def test_sql():
db = kydb.connect('memory://db')
db.upload_objdb_config(OBJDB_CONFIG)
sql_template_path = '/templates/my_test_template.sql'
db[sql_template_path] = SQL_TEMPLATE
ts = db.new('DummySqlTimeSeries',
'/fxcm/GBPUSD_MINUTELY',
sql_template_path=sql_template_path,
sql_kwargs={'pair': 'GBPUSD'})
expected = '''
WITH t AS
(SELECT date_parse(datetime,
'%m/%d/%Y %H:%i:%S.000') AS dt,
(bidclose+askclose) / 2 AS closeprice
FROM fxcm
WHERE pair = 'GBPUSD'
AND year BETWEEN '2017' AND '2017')
SELECT *
FROM t
WHERE dt
BETWEEN timestamp '2017-05-01 00:00:00.000'
AND timestamp '2017-05-02 00:00:00.000'
'''
actual = ts.sql(datetime(2017, 5, 1), datetime(2017, 5, 2))
assert actual == expected
actual = ts.sql(date(2017, 5, 1), date(2017, 5, 2))
assert actual == expected
def load_model(self):
db = kydb.connect(self.dbname)
data = db[self.model_path]
filename = self.model_path.rsplit('/', 1)[1]
with open(filename, 'wb') as f:
f.write(data)
return load_model(filename)
def ts():
db = kydb.connect('memory://db')
db.upload_objdb_config(OBJDB_CONFIG)
sql_template_path = '/templates/my_test_template.sql'
db[sql_template_path] = SQL_TEMPLATE
ts = db.new('AthenaTimeSeries', '/fxcm/GBPUSD_MINUTELY',
database='epython-marketdata',
res_s3_path='s3://epython-athena/result-cache/',
sql_template_path=sql_template_path,
sql_kwargs={'pair': 'GBPUSD'})
return ts
def test_with_basepath(db_type: str):
base_path = 'unittests/my/base/path'
db = get_db(db_type, base_path)
key = '/apple'
db[key] = 123
assert db[key] == 123
assert db.read(key, reload=True) == 123
# Going to the root and including the base_path should be equivalent
db2 = kydb.connect(DB_URLS[db_type])
assert db2.read(base_path + key, reload=True) == 123
db2.rm_tree('/unittests/my')
def ts():
with qconnection.QConnection(host='localhost', port=5001) as q:
for cmd in COMMANDS:
try:
q(cmd)
except QException as msg:
print('q error: \'%s' % msg)
db = kydb.connect('memory://kdb')
db.upload_objdb_config(OBJDB_CONFIG)
ts = db.new('KDBTimeSeries', '/Bloomerg/VOD.L')
return ts
def test_list_dir():
db = kydb.connect('memory://union_db3;memory://union_db4')
db1, db2 = db.dbs
db1['/a/b/obj1'] = 1
db1['/a/b/obj2'] = 2
db1['/a/obj3'] = 3
db2['/a/b/obj4'] = 4
db2['/a/obj5'] = 5
db2['obj6'] = 6
assert set(db.ls('/')) == set(['a/', 'obj6'])
assert set(db.ls('/a/')) == set(['b/', 'obj3', 'obj5'])
assert set(db.ls('/a/b/')) == set(['obj1', 'obj2', 'obj4'])
assert set(db.ls('/', False)) == set(['obj6'])
assert set(db.ls('/a/', False)) == set(['obj3', 'obj5'])
assert set(db.ls('/a/b/', False)) == set(['obj1', 'obj2', 'obj4'])
def test_cache_context():
db = kydb.connect('memory://unittests')
db['global'] = 'always_here'
assert db._cache == {'/global': 'always_here'}
with db.cache_context():
db['foo'] = 123
assert db._cache == {'/global': 'always_here', '/foo': 123}
with db.cache_context():
db['bar'] = 456
assert db._cache == {
'/foo': 123,
'/bar': 456,
'/global': 'always_here'
}
assert db._cache == {'/global': 'always_here', '/foo': 123}
assert db._cache == {'/global': 'always_here'}
def test_basic():
db = kydb.connect('memory://union_db1;memory://union_db2')
db1, db2 = db.dbs
db1['/foo'] = 1
db2['/foo'] = 2
assert db['/foo'] == 1
assert db.ls('/') == ['foo']
db['/foo'] = 3
assert db['/foo'] == 3
assert db1['/foo'] == 3
assert db2['/foo'] == 2
db.delete('/foo')
assert db['/foo'] == 2
assert list(db.list_dir('/')) == ['foo']
assert db.ls('/') == ['foo']
db2.delete('/foo')
assert list(db.list_dir('/')) == []
assert db.ls('/') == []
def db():
"""
The fixture was prepared with this::
with open('test_http_basic', 'w') as f:
f.write(pickle.dumps(123).hex())
val = {
'my_int': 123,
'my_float': 123.456,
'my_str': 'hello',
'my_list': [1, 2, 3],
'my_datetime': datetime.now()
}
with open('test_http_dict', 'w') as f:
f.write(pickle.dumps(val).hex())
aws s3 cp test_http_basic s3://files.tayglobal.com/kinyu-demo/db/tests/
aws s3 cp test_http_dict s3://files.tayglobal.com/kinyu-demo/db/tests/
"""
return kydb.connect(BASE_URL)
def test_union():
db = kydb.connect('memory://db1;memory://db2')
# upload the config
db.upload_objdb_config(DBOBJ_CONFIG)
key = '/unittest/dbobj/greeter001'
obj = db.new('Greeter', key)
assert obj.name() == 'John'
# Object does not exist anywhere in the DB
assert not db.exists(key)
assert not db.dbs[0].exists(key)
assert not db.dbs[1].exists(key)
obj.write()
# object exists only in front DB
assert db.exists(key)
assert db.dbs[0].exists(key)
assert not db.dbs[1].exists(key)
obj.delete()
# Object does not exist anywhere in the DB
assert not db.exists(key)
assert not db.dbs[0].exists(key)
assert not db.dbs[1].exists(key)
def __init__(self, url: str):
self.url = url
self.db = kydb.connect(url)
from plotly.data import iris
import kydb
from datetime import date
import pandas as pd
import plotly.express as px
from .server import app
DEFAULT_DATES = {
('bitflyer', 'daily'): (date(2020, 5, 1), date(2020, 6, 4)),
('bitflyer', 'minutely'): (date(2020, 6, 1), date(2020, 6, 4)),
('fxcm', 'minutely'): (date(2018, 6, 4), date(2018, 6, 6)),
('ml', 'minutely'): (date(2018, 6, 4), date(2018, 6, 6)),
}
tsdb = kydb.connect('dynamodb://epython/timeseries')
exchanges = [x[:-1] for x in tsdb.ls('/symbols/')]
exchange_dropdown = dcc.Dropdown(id='exchange-dropdown',
options=[{
'label': x,
'value': x
} for x in exchanges],
value='bitflyer')
exchange = 'bitflyer'
resolution_dropdown = dcc.Dropdown(id='resolution-dropdown', value='minutely')
symbols_dropdown = dcc.Dropdown(id='ts-symbol')
date_range = dcc.DatePickerRange(
def __init__(self, srcdb: str, localpath: str, recursive=True):
self.db = kydb.connect(srcdb)
self.localpath = localpath
self.recursive = recursive
"""
Used to update Redis in the backend.
Only need 1 instance running
"""
import redis
import pickle
import time
import kydb
if __name__ == '__main__':
db = kydb.connect('dynamodb://epython')
config = db['/demos/epython-dash-demo/config']
host = config.get('REDIS_HOST', '127.0.0.1')
port = config.get('REDIS_PORT', '6379')
r = redis.Redis(host=host, port=port)
r.delete('trades')
r.set('tick', 0)
while True:
time.sleep(1)
r.incr('tick', 1)
print(int(r.get('tick')))
def test__get_dbobj_config_no_cfg():
# Missing config yaml in db
db = kydb.connect('memory://unittest')
with pytest.raises(DbObjException):
db._get_dbobj_config('MyClass')
def db():
db = kydb.connect('memory://unittest')
# upload the config
db.upload_objdb_config(DBOBJ_CONFIG)
return db
def get_db(db_type, base_path):
return kydb.connect(DB_URLS[db_type] + '/' + base_path)
def test_uniondb():
db = kydb.connect('memory://db1;memory://db2')
assert repr(db) == '<UnionDB memory://db1;memory://db2>'