class JsonOutput(object): def __init__(self, file, logLimit): self.jsonOutputFile = file self.jsonOutput = {} self.jsonOutputCoins = {} self.jsonOutput["raw_data"] = self.jsonOutputCoins self.jsonOutputLog = RingBuffer(logLimit); def status(self, status, time): self.jsonOutput["last_update"] = time self.jsonOutput["last_status"] = status def printline(self, line): self.jsonOutputLog.append(line) def writeJsonFile(self): with io.open(self.jsonOutputFile, 'w', encoding='utf-8') as f: self.jsonOutput["log"] = self.jsonOutputLog.get() f.write(unicode(json.dumps(self.jsonOutput, ensure_ascii=False, sort_keys=True))) f.close() def statusValue(self, coin, key, value): if(coin not in self.jsonOutputCoins): self.jsonOutputCoins[coin] = {} self.jsonOutputCoins[coin][key] = str(value) def clearStatusValues(self): self.jsonOutputCoins = {}
class JsonOutput(object): def __init__(self, file, logLimit, exchange=''): self.jsonOutputFile = file self.jsonOutput = {} self.clearStatusValues() self.jsonOutputLog = RingBuffer(logLimit) self.jsonOutput['exchange'] = exchange def status(self, status, time, days_remaining_msg): self.jsonOutput["last_update"] = time + days_remaining_msg self.jsonOutput["last_status"] = status def printline(self, line): line = line.replace("\n", ' | ') self.jsonOutputLog.append(line) def writeJsonFile(self): with io.open(self.jsonOutputFile, 'w', encoding='utf-8') as f: self.jsonOutput["log"] = self.jsonOutputLog.get() f.write(unicode(json.dumps(self.jsonOutput, ensure_ascii=False, sort_keys=True))) f.close() def statusValue(self, coin, key, value): if coin not in self.jsonOutputCoins: self.jsonOutputCoins[coin] = {} self.jsonOutputCoins[coin][key] = str(value) def clearStatusValues(self): self.jsonOutputCoins = {} self.jsonOutput["raw_data"] = self.jsonOutputCoins self.jsonOutputCurrency = {} self.jsonOutput["outputCurrency"] = self.jsonOutputCurrency def outputCurrency(self, key, value): self.jsonOutputCurrency[key] = str(value)
class JsonOutput(object): def __init__(self, file, logLimit): self.jsonOutputFile = file self.jsonOutput = {} self.jsonOutputCoins = {} self.jsonOutput["raw_data"] = self.jsonOutputCoins self.jsonOutputLog = RingBuffer(logLimit) def status(self, status, time): self.jsonOutput["last_update"] = time self.jsonOutput["last_status"] = status def printline(self, line): self.jsonOutputLog.append(line) def writeJsonFile(self): with io.open(self.jsonOutputFile, 'w', encoding='utf-8') as f: self.jsonOutput["log"] = self.jsonOutputLog.get() f.write( unicode( json.dumps(self.jsonOutput, ensure_ascii=False, sort_keys=True))) f.close() def statusValue(self, coin, key, value): if (coin not in self.jsonOutputCoins): self.jsonOutputCoins[coin] = {} self.jsonOutputCoins[coin][key] = str(value) def clearStatusValues(self): self.jsonOutputCoins = {}
class RingBufferTestCase(unittest.TestCase): def setUp(self): self.ringbuffer = RingBuffer(2) def test_create_ring_buffer(self): self.assertIsNotNone(self.ringbuffer) self.assertEquals(2, self.ringbuffer.size) def test_addOneItem_SameItemReturns(self): self.ringbuffer.add(1) self.assertEquals(1, self.ringbuffer.get()) def test_addFullBuffer_sameOrderReturns(self): self.ringbuffer.add(1) self.ringbuffer.add(2) self.assertEquals(1, self.ringbuffer.get()) self.assertEquals(2, self.ringbuffer.get()) def test_addOneMoreItemThanSize_FirstItemOverwritten(self): self.ringbuffer.add(1) self.ringbuffer.add(2) self.ringbuffer.add(3) self.assertEquals(2, self.ringbuffer.get()) self.assertEquals(3, self.ringbuffer.get()) self.assertEquals(None, self.ringbuffer.get()) def test_firstAddAndReadThenAddFullBuffer_sameOrderAsPutInAfterRead(self): self.ringbuffer.add(1) self.assertEquals(1, self.ringbuffer.get()) self.ringbuffer.add(2) self.ringbuffer.add(3) self.assertEquals(2, self.ringbuffer.get()) self.assertEquals(3, self.ringbuffer.get()) self.assertEquals(None, self.ringbuffer.get()) def test_firstAddAndReadThenAddOneMoreItemThanSize_SecondItemOverwritten(self): self.ringbuffer.add(1) self.assertEquals(1, self.ringbuffer.get()) self.ringbuffer.add(2) self.ringbuffer.add(3) self.ringbuffer.add(4) self.assertEquals(3, self.ringbuffer.get()) self.assertEquals(4, self.ringbuffer.get()) self.assertEquals(None, self.ringbuffer.get())
class DistanceData: def __init__(self, nodeid, buffersize=10): self.nodeid = nodeid self.buffersize = buffersize self.buffer = RingBuffer(buffersize) def add_observation(self, observation): self.buffer.append(observation) def get_filtered_value(self): #TODO implement filtering curr_vals = self.buffer.get() curr_vals.sort() return curr_vals[len(curr_vals)/2]
class DistanceData: def __init__(self, nodeid, buffersize=10): self.nodeid = nodeid self.buffersize = buffersize self.buffer = RingBuffer(buffersize) def add_observation(self, observation): self.buffer.append(observation) def get_filtered_value(self): #TODO implement filtering curr_vals = self.buffer.get() curr_vals.sort() return curr_vals[len(curr_vals) / 2]
class JsonOutput(object): def __init__(self, file, logLimit): self.jsonOutputFile = file self.jsonOutput = {} self.jsonOutputLog = RingBuffer(logLimit); def status(self, status, time): self.jsonOutput["last_update"] = time self.jsonOutput["last_status"] = status self.writeJsonFile() def printline(self, line): self.jsonOutputLog.append(line) def writeJsonFile(self): with io.open(self.jsonOutputFile, 'w', encoding='utf-8') as f: self.jsonOutput["log"] = self.jsonOutputLog.get() f.write(unicode(json.dumps(self.jsonOutput, ensure_ascii=False, sort_keys=True))) f.close()
class JsonOutput(object): def __init__(self, file, logLimit, exchange=''): self.jsonOutputFile = file self.jsonOutput = {} self.clearStatusValues() self.jsonOutputLog = RingBuffer(logLimit) self.jsonOutput['exchange'] = exchange self.jsonOutput['label'] = Config.get("BOT", "label", "Lending Bot") def status(self, status, time, days_remaining_msg): self.jsonOutput["last_update"] = time + days_remaining_msg self.jsonOutput["last_status"] = status def printline(self, line): line = line.replace("\n", ' | ') self.jsonOutputLog.append(line) def writeJsonFile(self): with io.open(self.jsonOutputFile, 'w', encoding='utf-8') as f: self.jsonOutput["log"] = self.jsonOutputLog.get() f.write(unicode(json.dumps(self.jsonOutput, ensure_ascii=True, sort_keys=True), errors='replace')) f.close() def addSectionLog(self, section, key, value): if section not in self.jsonOutput: self.jsonOutput[section] = {} if key not in self.jsonOutput[section]: self.jsonOutput[section][key] = {} self.jsonOutput[section][key] = value def statusValue(self, coin, key, value): if coin not in self.jsonOutputCoins: self.jsonOutputCoins[coin] = {} self.jsonOutputCoins[coin][key] = str(value) def clearStatusValues(self): self.jsonOutputCoins = {} self.jsonOutput["raw_data"] = self.jsonOutputCoins self.jsonOutputCurrency = {} self.jsonOutput["outputCurrency"] = self.jsonOutputCurrency def outputCurrency(self, key, value): self.jsonOutputCurrency[key] = str(value)
class JsonOutput(object): def __init__(self, file, logLimit): self.jsonOutputFile = file self.jsonOutput = {} self.jsonOutputLog = RingBuffer(logLimit) def status(self, status, time): self.jsonOutput["last_update"] = time self.jsonOutput["last_status"] = status self.writeJsonFile() def printline(self, line): self.jsonOutputLog.append(line) def writeJsonFile(self): with io.open(self.jsonOutputFile, 'w', encoding='utf-8') as f: self.jsonOutput["log"] = self.jsonOutputLog.get() f.write( unicode( json.dumps(self.jsonOutput, ensure_ascii=False, sort_keys=True))) f.close()
class Process: """ Represents a Process that can be launched, monitored and killed """ def __init__(self, process_config_obj=None): if process_config_obj != None and type(process_config_obj) == type( ProcessConfig()): self.name = process_config_obj.name self.startcmds = process_config_obj.startcmds self.wdir = process_config_obj.wdir self.outlines = process_config_obj.outlines self.outbuffer = RingBuffer(self.outlines) self.stdin_pipe = process_config_obj.stdin_pipe self.envdict = process_config_obj.envdict self.startuplvl = process_config_obj.startuplvl ### self.retcode = None self.subproc = None self.started = False self.tmpfpath = None self.stdout_queue = None self.stdout_thread = None def start(self): if self.started: return tmpfhandle, self.tmpfpath = tempfile.mkstemp() tmpf = os.fdopen(tmpfhandle, 'w') tmpf.write('#!{shellcmd}\n'.format(shellcmd=SHELL_CMD)) tmpf.write('trap "exit" INT TERM\n') tmpf.write('trap "kill 0" EXIT\n') tmpf.write('cd "{path}"\n'.format(path=self.wdir)) if type(self.startcmds) == type([]): for l in self.startcmds: if not l.endswith(' $'): l = l + ' $' tmpf.write(l + "\n") elif type(self.startcmds) == type(""): bgmode = ' $' if not self.startcmds.endswith(' $') else '' tmpf.write(self.startcmds + bgmode + "\n") tmpf.write('sleep infinity\n') tmpf.close() #tmpf = open(self.tmpfpath, "r") #for l in tmpf: # print("{l}".format(l=l)) #tmpf.close() stdin_arg = subprocess.PIPE if self.stdin_pipe else None env_arg = dict(os.environ) env_arg.update(self.envdict) try: self.subproc = subprocess.Popen([SHELL_CMD, self.tmpfpath], bufsize=0, preexec_fn=os.setpgrp, stdin=stdin_arg, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.wdir, env=env_arg) except OSError: print("Process.start(): subprocess.Popen(...) raised OSError") return self.stdout_queue = Queue() self.stdout_thread = Thread(target=self._enqueue_stdout, args=(self.subproc.stdout, self.stdout_queue)) self.stdout_thread.daemon = True self.stdout_thread.start() self.started = True def _update_out_buffer(self): if not self.started: return pollresult = self.subproc.poll() if pollresult != None: self.retcode = pollresult while True: try: line = self.stdout_queue.get_nowait() except Empty: break if (len(line) > 0 and line[-1] == '\n'): line = line[:-1] self.outbuffer.append(line) def terminate(self): if not self.started: return os.killpg(os.getpgid(self.subproc.pid), signal.SIGTERM) self.started = False def kill(self): if not self.started: return os.killpg(os.getpgid(self.subproc.pid), signal.SIGKILL) self.started = False def terminated(self): return self.retcode != None def running(self): return self.started and not self.terminated() def return_code(self): return self.retcode def get_output(self): if not self.started: return self._update_out_buffer() return self.outbuffer.get() def send(self, input_data): if not self.started: return stdoutdata, stderrdata = self.subproc.communicate(input_data) pollresult = self.subproc.poll() if pollresult != None: self.retcode = pollresult return lines = stdoutdata.split('\n') for l in lines: self.outbuffer.append(l) def _enqueue_stdout(self, out, queue): for line in iter(out.readline, b''): queue.put(line.decode("utf-8")) out.close()
class RingBufferTests(unittest.TestCase): def assert_arrays(self,a1,a2,msg="Array's not equal"): self.assertTrue(a1.size==a2.size,"{0}: {1} != {2}".format(msg,a1,a2)) self.assertTrue(np.array_equal(a1,a1),"{0}: {1} != {2}".format(msg,a1,a2)) # @property # def log(self): # return self._log def setUp(self): # self._log = logging.getLogger("RingBuffer.Tests") self.sample_data = np.arange(10.0) self.ring_buffer = RingBuffer(10) def tearDown(self): del self.ring_buffer def test_size_total(self): self.assertEqual(self.ring_buffer.size_total,10,'total size incorrect') def test_size_used(self): d = [0,1,3] self.ring_buffer.add(d) self.assertEqual(self.ring_buffer.size_used,len(d),'used size incorrect') self.ring_buffer.get(2) self.assertEqual(self.ring_buffer.size_used,1,'used size incorrect') def test_size_used(self): d = [0,1,3] self.ring_buffer.add(d) self.assertEqual(self.ring_buffer.size_remaining,self.ring_buffer.size_total-len(d),'remaining size incorrect') self.ring_buffer.get(2) self.assertEqual(self.ring_buffer.size_remaining,9,'remaining size incorrect') def test_basic_get(self): self.ring_buffer.add(self.sample_data[0:5]) data = self.ring_buffer.get(2) self.assert_arrays(data,self.sample_data[0:2]) def test_second_get(self): self.ring_buffer.add(self.sample_data[0:5]) data = self.ring_buffer.get(2) data = self.ring_buffer.get(2) self.assert_arrays(data,self.sample_data[3:5]) def test_read_all_data_when_number_is_larger_than_buffer(self): d = self.sample_data[0:2] self.ring_buffer.add(d) g = self.ring_buffer.get(4) self.assert_arrays(d,g) self.assertEqual(self.ring_buffer.size_used,0) def test_throw_on_read_from_empty_buffer(self): self.ring_buffer.add(self.sample_data[0:2]) self.ring_buffer.get(2) # Should throw exception if reading beyond buffer with self.assertRaises(RingBuffer.Empty): data = self.ring_buffer.get(2) def test_throw_on_write_to_full_buffer(self): # Should throw if buffer is full self.ring_buffer.add(self.sample_data) # Fill buffer with self.assertRaises(RingBuffer.Full): self.ring_buffer.add([0,1]) # once we get some items, we should be able to add self.ring_buffer.get(2) self.ring_buffer.add([0,1]) # adding anything else should throw again with self.assertRaises(RingBuffer.Full): self.ring_buffer.add([4]) def test_should_be_thread_safe(self): def write_thread(ring,w_buf): #print "Writing to ring buffer" for j in range(0,w_buf.size,2): ring.add(w_buf[j:j+2]) #print "Write:Sleep for .1 sec" time.sleep(.1) def read_thread(ring,buf): #print "Reading from RingBuffer" for i in range(buf.size): buf[i] = ring.get(1) #print "Read:Sleep for .06 sec" time.sleep(0.06) write_buf = np.arange(10.0,20.0) # Write thread wt = threading.Thread(target=write_thread,args=(self.ring_buffer,write_buf)) data = np.zeros(shape=10) rt = threading.Thread(target=read_thread,args=(self.ring_buffer,data)) wt.start() time.sleep(0.01) rt.start() wt.join() rt.join() self.assert_arrays(write_buf,data)