class JsonOutput(object):
def __init__(self, file, logLimit):
self.jsonOutputFile = file
self.jsonOutput = {}
self.jsonOutputCoins = {}
self.jsonOutput["raw_data"] = self.jsonOutputCoins
self.jsonOutputLog = RingBuffer(logLimit);
def status(self, status, time):
self.jsonOutput["last_update"] = time
self.jsonOutput["last_status"] = status
def printline(self, line):
self.jsonOutputLog.append(line)
def writeJsonFile(self):
with io.open(self.jsonOutputFile, 'w', encoding='utf-8') as f:
self.jsonOutput["log"] = self.jsonOutputLog.get()
f.write(unicode(json.dumps(self.jsonOutput, ensure_ascii=False, sort_keys=True)))
f.close()
def statusValue(self, coin, key, value):
if(coin not in self.jsonOutputCoins):
self.jsonOutputCoins[coin] = {}
self.jsonOutputCoins[coin][key] = str(value)
def clearStatusValues(self):
self.jsonOutputCoins = {}
class JsonOutput(object):
def __init__(self, file, logLimit, exchange=''):
self.jsonOutputFile = file
self.jsonOutput = {}
self.clearStatusValues()
self.jsonOutputLog = RingBuffer(logLimit)
self.jsonOutput['exchange'] = exchange
def status(self, status, time, days_remaining_msg):
self.jsonOutput["last_update"] = time + days_remaining_msg
self.jsonOutput["last_status"] = status
def printline(self, line):
line = line.replace("\n", ' | ')
self.jsonOutputLog.append(line)
def writeJsonFile(self):
with io.open(self.jsonOutputFile, 'w', encoding='utf-8') as f:
self.jsonOutput["log"] = self.jsonOutputLog.get()
f.write(unicode(json.dumps(self.jsonOutput, ensure_ascii=False, sort_keys=True)))
f.close()
def statusValue(self, coin, key, value):
if coin not in self.jsonOutputCoins:
self.jsonOutputCoins[coin] = {}
self.jsonOutputCoins[coin][key] = str(value)
def clearStatusValues(self):
self.jsonOutputCoins = {}
self.jsonOutput["raw_data"] = self.jsonOutputCoins
self.jsonOutputCurrency = {}
self.jsonOutput["outputCurrency"] = self.jsonOutputCurrency
def outputCurrency(self, key, value):
self.jsonOutputCurrency[key] = str(value)
class JsonOutput(object):
def __init__(self, file, logLimit):
self.jsonOutputFile = file
self.jsonOutput = {}
self.jsonOutputCoins = {}
self.jsonOutput["raw_data"] = self.jsonOutputCoins
self.jsonOutputLog = RingBuffer(logLimit)
def status(self, status, time):
self.jsonOutput["last_update"] = time
self.jsonOutput["last_status"] = status
def printline(self, line):
self.jsonOutputLog.append(line)
def writeJsonFile(self):
with io.open(self.jsonOutputFile, 'w', encoding='utf-8') as f:
self.jsonOutput["log"] = self.jsonOutputLog.get()
f.write(
unicode(
json.dumps(self.jsonOutput,
ensure_ascii=False,
sort_keys=True)))
f.close()
def statusValue(self, coin, key, value):
if (coin not in self.jsonOutputCoins):
self.jsonOutputCoins[coin] = {}
self.jsonOutputCoins[coin][key] = str(value)
def clearStatusValues(self):
self.jsonOutputCoins = {}
class RingBufferTestCase(unittest.TestCase):
def setUp(self):
self.ringbuffer = RingBuffer(2)
def test_create_ring_buffer(self):
self.assertIsNotNone(self.ringbuffer)
self.assertEquals(2, self.ringbuffer.size)
def test_addOneItem_SameItemReturns(self):
self.ringbuffer.add(1)
self.assertEquals(1, self.ringbuffer.get())
def test_addFullBuffer_sameOrderReturns(self):
self.ringbuffer.add(1)
self.ringbuffer.add(2)
self.assertEquals(1, self.ringbuffer.get())
self.assertEquals(2, self.ringbuffer.get())
def test_addOneMoreItemThanSize_FirstItemOverwritten(self):
self.ringbuffer.add(1)
self.ringbuffer.add(2)
self.ringbuffer.add(3)
self.assertEquals(2, self.ringbuffer.get())
self.assertEquals(3, self.ringbuffer.get())
self.assertEquals(None, self.ringbuffer.get())
def test_firstAddAndReadThenAddFullBuffer_sameOrderAsPutInAfterRead(self):
self.ringbuffer.add(1)
self.assertEquals(1, self.ringbuffer.get())
self.ringbuffer.add(2)
self.ringbuffer.add(3)
self.assertEquals(2, self.ringbuffer.get())
self.assertEquals(3, self.ringbuffer.get())
self.assertEquals(None, self.ringbuffer.get())
def test_firstAddAndReadThenAddOneMoreItemThanSize_SecondItemOverwritten(self):
self.ringbuffer.add(1)
self.assertEquals(1, self.ringbuffer.get())
self.ringbuffer.add(2)
self.ringbuffer.add(3)
self.ringbuffer.add(4)
self.assertEquals(3, self.ringbuffer.get())
self.assertEquals(4, self.ringbuffer.get())
self.assertEquals(None, self.ringbuffer.get())
class DistanceData:
def __init__(self, nodeid, buffersize=10):
self.nodeid = nodeid
self.buffersize = buffersize
self.buffer = RingBuffer(buffersize)
def add_observation(self, observation):
self.buffer.append(observation)
def get_filtered_value(self):
#TODO implement filtering
curr_vals = self.buffer.get()
curr_vals.sort()
return curr_vals[len(curr_vals)/2]
class DistanceData:
def __init__(self, nodeid, buffersize=10):
self.nodeid = nodeid
self.buffersize = buffersize
self.buffer = RingBuffer(buffersize)
def add_observation(self, observation):
self.buffer.append(observation)
def get_filtered_value(self):
#TODO implement filtering
curr_vals = self.buffer.get()
curr_vals.sort()
return curr_vals[len(curr_vals) / 2]
class JsonOutput(object):
def __init__(self, file, logLimit):
self.jsonOutputFile = file
self.jsonOutput = {}
self.jsonOutputLog = RingBuffer(logLimit);
def status(self, status, time):
self.jsonOutput["last_update"] = time
self.jsonOutput["last_status"] = status
self.writeJsonFile()
def printline(self, line):
self.jsonOutputLog.append(line)
def writeJsonFile(self):
with io.open(self.jsonOutputFile, 'w', encoding='utf-8') as f:
self.jsonOutput["log"] = self.jsonOutputLog.get()
f.write(unicode(json.dumps(self.jsonOutput, ensure_ascii=False, sort_keys=True)))
f.close()
class JsonOutput(object):
def __init__(self, file, logLimit, exchange=''):
self.jsonOutputFile = file
self.jsonOutput = {}
self.clearStatusValues()
self.jsonOutputLog = RingBuffer(logLimit)
self.jsonOutput['exchange'] = exchange
self.jsonOutput['label'] = Config.get("BOT", "label", "Lending Bot")
def status(self, status, time, days_remaining_msg):
self.jsonOutput["last_update"] = time + days_remaining_msg
self.jsonOutput["last_status"] = status
def printline(self, line):
line = line.replace("\n", ' | ')
self.jsonOutputLog.append(line)
def writeJsonFile(self):
with io.open(self.jsonOutputFile, 'w', encoding='utf-8') as f:
self.jsonOutput["log"] = self.jsonOutputLog.get()
f.write(unicode(json.dumps(self.jsonOutput, ensure_ascii=True, sort_keys=True), errors='replace'))
f.close()
def addSectionLog(self, section, key, value):
if section not in self.jsonOutput:
self.jsonOutput[section] = {}
if key not in self.jsonOutput[section]:
self.jsonOutput[section][key] = {}
self.jsonOutput[section][key] = value
def statusValue(self, coin, key, value):
if coin not in self.jsonOutputCoins:
self.jsonOutputCoins[coin] = {}
self.jsonOutputCoins[coin][key] = str(value)
def clearStatusValues(self):
self.jsonOutputCoins = {}
self.jsonOutput["raw_data"] = self.jsonOutputCoins
self.jsonOutputCurrency = {}
self.jsonOutput["outputCurrency"] = self.jsonOutputCurrency
def outputCurrency(self, key, value):
self.jsonOutputCurrency[key] = str(value)
class JsonOutput(object):
def __init__(self, file, logLimit):
self.jsonOutputFile = file
self.jsonOutput = {}
self.jsonOutputLog = RingBuffer(logLimit)
def status(self, status, time):
self.jsonOutput["last_update"] = time
self.jsonOutput["last_status"] = status
self.writeJsonFile()
def printline(self, line):
self.jsonOutputLog.append(line)
def writeJsonFile(self):
with io.open(self.jsonOutputFile, 'w', encoding='utf-8') as f:
self.jsonOutput["log"] = self.jsonOutputLog.get()
f.write(
unicode(
json.dumps(self.jsonOutput,
ensure_ascii=False,
sort_keys=True)))
f.close()
class Process:
""" Represents a Process that can be launched, monitored and killed """
def __init__(self, process_config_obj=None):
if process_config_obj != None and type(process_config_obj) == type(
ProcessConfig()):
self.name = process_config_obj.name
self.startcmds = process_config_obj.startcmds
self.wdir = process_config_obj.wdir
self.outlines = process_config_obj.outlines
self.outbuffer = RingBuffer(self.outlines)
self.stdin_pipe = process_config_obj.stdin_pipe
self.envdict = process_config_obj.envdict
self.startuplvl = process_config_obj.startuplvl
###
self.retcode = None
self.subproc = None
self.started = False
self.tmpfpath = None
self.stdout_queue = None
self.stdout_thread = None
def start(self):
if self.started:
return
tmpfhandle, self.tmpfpath = tempfile.mkstemp()
tmpf = os.fdopen(tmpfhandle, 'w')
tmpf.write('#!{shellcmd}\n'.format(shellcmd=SHELL_CMD))
tmpf.write('trap "exit" INT TERM\n')
tmpf.write('trap "kill 0" EXIT\n')
tmpf.write('cd "{path}"\n'.format(path=self.wdir))
if type(self.startcmds) == type([]):
for l in self.startcmds:
if not l.endswith(' $'):
l = l + ' $'
tmpf.write(l + "\n")
elif type(self.startcmds) == type(""):
bgmode = ' $' if not self.startcmds.endswith(' $') else ''
tmpf.write(self.startcmds + bgmode + "\n")
tmpf.write('sleep infinity\n')
tmpf.close()
#tmpf = open(self.tmpfpath, "r")
#for l in tmpf:
# print("{l}".format(l=l))
#tmpf.close()
stdin_arg = subprocess.PIPE if self.stdin_pipe else None
env_arg = dict(os.environ)
env_arg.update(self.envdict)
try:
self.subproc = subprocess.Popen([SHELL_CMD, self.tmpfpath],
bufsize=0,
preexec_fn=os.setpgrp,
stdin=stdin_arg,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=self.wdir,
env=env_arg)
except OSError:
print("Process.start(): subprocess.Popen(...) raised OSError")
return
self.stdout_queue = Queue()
self.stdout_thread = Thread(target=self._enqueue_stdout,
args=(self.subproc.stdout,
self.stdout_queue))
self.stdout_thread.daemon = True
self.stdout_thread.start()
self.started = True
def _update_out_buffer(self):
if not self.started:
return
pollresult = self.subproc.poll()
if pollresult != None:
self.retcode = pollresult
while True:
try:
line = self.stdout_queue.get_nowait()
except Empty:
break
if (len(line) > 0 and line[-1] == '\n'):
line = line[:-1]
self.outbuffer.append(line)
def terminate(self):
if not self.started:
return
os.killpg(os.getpgid(self.subproc.pid), signal.SIGTERM)
self.started = False
def kill(self):
if not self.started:
return
os.killpg(os.getpgid(self.subproc.pid), signal.SIGKILL)
self.started = False
def terminated(self):
return self.retcode != None
def running(self):
return self.started and not self.terminated()
def return_code(self):
return self.retcode
def get_output(self):
if not self.started:
return
self._update_out_buffer()
return self.outbuffer.get()
def send(self, input_data):
if not self.started:
return
stdoutdata, stderrdata = self.subproc.communicate(input_data)
pollresult = self.subproc.poll()
if pollresult != None:
self.retcode = pollresult
return
lines = stdoutdata.split('\n')
for l in lines:
self.outbuffer.append(l)
def _enqueue_stdout(self, out, queue):
for line in iter(out.readline, b''):
queue.put(line.decode("utf-8"))
out.close()
class RingBufferTests(unittest.TestCase):
def assert_arrays(self,a1,a2,msg="Array's not equal"):
self.assertTrue(a1.size==a2.size,"{0}: {1} != {2}".format(msg,a1,a2))
self.assertTrue(np.array_equal(a1,a1),"{0}: {1} != {2}".format(msg,a1,a2))
# @property
# def log(self):
# return self._log
def setUp(self):
# self._log = logging.getLogger("RingBuffer.Tests")
self.sample_data = np.arange(10.0)
self.ring_buffer = RingBuffer(10)
def tearDown(self):
del self.ring_buffer
def test_size_total(self):
self.assertEqual(self.ring_buffer.size_total,10,'total size incorrect')
def test_size_used(self):
d = [0,1,3]
self.ring_buffer.add(d)
self.assertEqual(self.ring_buffer.size_used,len(d),'used size incorrect')
self.ring_buffer.get(2)
self.assertEqual(self.ring_buffer.size_used,1,'used size incorrect')
def test_size_used(self):
d = [0,1,3]
self.ring_buffer.add(d)
self.assertEqual(self.ring_buffer.size_remaining,self.ring_buffer.size_total-len(d),'remaining size incorrect')
self.ring_buffer.get(2)
self.assertEqual(self.ring_buffer.size_remaining,9,'remaining size incorrect')
def test_basic_get(self):
self.ring_buffer.add(self.sample_data[0:5])
data = self.ring_buffer.get(2)
self.assert_arrays(data,self.sample_data[0:2])
def test_second_get(self):
self.ring_buffer.add(self.sample_data[0:5])
data = self.ring_buffer.get(2)
data = self.ring_buffer.get(2)
self.assert_arrays(data,self.sample_data[3:5])
def test_read_all_data_when_number_is_larger_than_buffer(self):
d = self.sample_data[0:2]
self.ring_buffer.add(d)
g = self.ring_buffer.get(4)
self.assert_arrays(d,g)
self.assertEqual(self.ring_buffer.size_used,0)
def test_throw_on_read_from_empty_buffer(self):
self.ring_buffer.add(self.sample_data[0:2])
self.ring_buffer.get(2)
# Should throw exception if reading beyond buffer
with self.assertRaises(RingBuffer.Empty):
data = self.ring_buffer.get(2)
def test_throw_on_write_to_full_buffer(self):
# Should throw if buffer is full
self.ring_buffer.add(self.sample_data) # Fill buffer
with self.assertRaises(RingBuffer.Full):
self.ring_buffer.add([0,1])
# once we get some items, we should be able to add
self.ring_buffer.get(2)
self.ring_buffer.add([0,1])
# adding anything else should throw again
with self.assertRaises(RingBuffer.Full):
self.ring_buffer.add([4])
def test_should_be_thread_safe(self):
def write_thread(ring,w_buf):
#print "Writing to ring buffer"
for j in range(0,w_buf.size,2):
ring.add(w_buf[j:j+2])
#print "Write:Sleep for .1 sec"
time.sleep(.1)
def read_thread(ring,buf):
#print "Reading from RingBuffer"
for i in range(buf.size):
buf[i] = ring.get(1)
#print "Read:Sleep for .06 sec"
time.sleep(0.06)
write_buf = np.arange(10.0,20.0)
# Write thread
wt = threading.Thread(target=write_thread,args=(self.ring_buffer,write_buf))
data = np.zeros(shape=10)
rt = threading.Thread(target=read_thread,args=(self.ring_buffer,data))
wt.start()
time.sleep(0.01)
rt.start()
wt.join()
rt.join()
self.assert_arrays(write_buf,data)
