def test_s3_buffer():
root = 'bucket/__test'
backend = Datasink.S3
teststr = 'hello world'
sink = Datasink(root=root, ext=ext, backend=backend)
sink.write(teststr)
assert sink._file.getvalue() == teststr + '\n'
def test_minute_create_file():
resolution = Datasink.MINUTE
sink = Datasink(root=root, ext=ext, resolution=resolution)
sink.write('test')
minute = datetime.now().strftime('%Y/%m/%d/%H/%M')
assert os.path.isdir(root)
assert os.path.isfile('{}/{}.{}'.format(root, minute, ext))
del sink
shutil.rmtree(root)
def test_hour_create_file():
resolution = Datasink.HOUR
sink = Datasink(root=root, ext=ext, resolution=resolution)
sink.write('test')
hour = datetime.now().strftime('%Y/%m/%d/%H')
assert os.path.isdir(root)
assert os.path.isfile('{}/{}.{}'.format(root, hour, ext))
del sink
shutil.rmtree(root)
def test_month_create_file():
resolution = Datasink.MONTH
sink = Datasink(root=root, ext=ext, resolution=resolution)
sink.write('test')
month = datetime.today().strftime('%Y/%m')
assert os.path.isdir(root)
assert os.path.isfile('{}/{}.{}'.format(root, month, ext))
del sink
shutil.rmtree(root)
def test_day_create_file():
resolution = Datasink.DAY
sink = Datasink(root=root, ext=ext, resolution=resolution)
sink.write('test')
today = datetime.today().strftime('%Y/%m/%d')
assert os.path.isdir(root)
assert os.path.isfile('{}/{}.{}'.format(root, today, ext))
del sink
shutil.rmtree(root)
def test_headers():
header = 'header'
teststr = 'test'
sink = Datasink(root=root, ext=ext, header=header, resolution=resolution)
sink.write(teststr)
sink._file
with open(sink._file.name) as f:
assert f.readline() == header + '\n'
assert f.readline() == teststr + '\n'
del sink
shutil.rmtree(root)
def test_no_overwrite_existing_file():
resolution = Datasink.MONTH
existing_filename = '{}/{}.{}'.format(root,
datetime.now().strftime('/%Y/%m'),
ext)
Path(existing_filename).parent.mkdir(mode=0o775,
parents=True,
exist_ok=True)
open(existing_filename, mode='w').write('hello world')
with pytest.raises(FileExistsError) as execinfo:
sink = Datasink(root=root, ext=ext, resolution=resolution)
sink.write()
shutil.rmtree(root)
def main(*,
root='cryptle-exchange/bitstamp-diff',
pairs=('btcusd', 'bchusd', 'ethusd', 'xrpusd'),
resolution=Datasink.MINUTE,
backend='os'):
# Use csv header
header = [
'time', 'id', 'price', 'volume', 'order_type', 'diff_type', 'src_time'
]
header = ','.join(header)
ext = 'csv'
# Prepare sinks
sinks = {}
for pair in pairs:
sinks[pair] = Datasink(
root='-'.join([root, pair]),
ext=ext,
header=header,
namemode=2,
resolution=resolution,
backend=backend,
)
conn = bitstamp.BitstampFeed()
conn.connect()
for pair in pairs:
conn.onCreate(
pair, partial(record_diff, diff_type='create', sink=sinks[pair]))
conn.onDelete(
pair, partial(record_diff, diff_type='delete', sink=sinks[pair]))
conn.onChange(pair,
partial(record_diff, diff_type='take', sink=sinks[pair]))
while True:
try:
while conn.is_connected():
time.sleep(0.2)
except ConnectionError:
# reconnect
conn.connect()
except KeyboardInterrupt:
print('\rTerminating...')
conn.close()
return 0
except Exception:
logging.error('Uncaught exception %s', e)
return 1
def test_mode_1():
entry = 'hello world'
sink = Datasink(root, namemode=1)
sink.write(entry)
f = str(sink._filepath)
sink.close()
# Filename is unix timestamp, should be long
assert len(f.split('/')[-1]) > 12
assert open(f).readline() == entry + '\n'
del sink
shutil.rmtree(root)
def test_mode_0():
entry = 'hello world'
sink = Datasink(root, namemode=0)
sink.write(entry)
f = str(sink._filepath)
sink.close()
# Filename is smallest time unit, shouldn't be too long
assert len(f.split('/')[-1]) < 8
assert open(f).readline() == entry + '\n'
del sink
shutil.rmtree(root)
def main(*,
root='cryptle-exchange/bitstamp-tick',
pairs=('btcusd', 'bchusd', 'ethusd', 'xrpusd'),
resolution=Datasink.MINUTE,
backend='os'):
header = ['id', 'price', 'amount', 'time']
header = ','.join(header)
ext = 'csv'
# Prepare sinks
sinks = {}
for pair in pairs:
sinks[pair] = Datasink(
root='-'.join([root, pair]),
ext=ext,
header=header,
namemode=2,
resolution=resolution,
backend=backend,
)
conn = bitstamp.BitstampFeed()
conn.connect()
for pair in pairs:
conn.onTrade(pair, partial(write_tick_to_sink, sink=sinks[pair]))
while True:
try:
while conn.is_connected():
time.sleep(0.2)
except ConnectionError:
# reconnect
conn.connect()
except KeyboardInterrupt:
print('\rTerminating...')
conn.close()
return 0
except Exception:
logging.error('Uncaught exception %s', e)
return 1
def main(*,
root='cryptle-exchange/bitstamp-book',
backend='os',
resolution='day',
pairs=('btcusd', 'bchusd', 'ethusd', 'xrpusd')):
ext = 'json'
sinks = {}
for pair in pairs:
sinks[pair] = Datasink(root='-'.join([root, pair]),
ext=ext,
namemode=1,
resolution=resolution,
backend=backend)
consec_fail_count = 0
while True:
try:
for pair in pairs:
data = req_orderbook(pair)
write_orderbook_to_sink(data, sinks[pair])
consec_fail_count = 0 # reset counter
time.sleep(600)
except ConnectionError:
consec_fail_count += 1
if consec_fail_count < 10:
# HTTP errors are not important so we can tolerate a number of them
# Sleep for a bit before trying again
time.sleep(5)
else:
# Connection may have degraded, wait a bit longer
time.sleep(300)
except KeyboardInterrupt:
print('\rTerminating...')
return 0
except Exception:
logging.error('Uncaught exception %s', e)
return 1