def test_list_artifact_tags(db: sqldb.SQLDB):
db.store_artifact('k1', {}, '1', tag='t1', project='p1')
db.store_artifact('k1', {}, '2', tag='t2', project='p1')
db.store_artifact('k1', {}, '2', tag='t2', project='p2')
tags = db.list_artifact_tags('p1')
assert {'t1', 't2'} == set(tags), 'bad tags'
def test_update_run(db: sqldb.SQLDB):
uid = 'uid83'
run = new_run('s1', {'l1': 'v1', 'l2': 'v2'}, x=1)
db.store_run(run, uid)
val = 13
db.update_run({'x': val}, uid)
r = db.read_run(uid)
assert val == r['x'], 'bad update'
def test_update_run(db: sqldb.SQLDB):
uid = "uid83"
run = new_run("s1", {"l1": "v1", "l2": "v2"}, x=1)
db.store_run(run, uid)
val = 13
db.update_run({"x": val}, uid)
r = db.read_run(uid)
assert val == r["x"], "bad update"
def get_run_db_instance(db_session: Session):
db = get_db()
if isinstance(db, SQLDB):
run_db = SQLRunDB(db.dsn, db_session)
else:
run_db = db.db
run_db.connect()
return run_db
def test_schedules(db: sqldb.SQLDB):
count = 7
for i in range(count):
data = {'i': i}
db.store_schedule(data)
scheds = list(db.list_schedules())
assert count == len(scheds), 'wrong number of schedules'
assert set(range(count)) == set(s['i'] for s in scheds), 'bad scheds'
def get_run_db_instance(db_session: Session,
leader_session: typing.Optional[str] = None):
db = get_db()
if isinstance(db, SQLDB):
run_db = SQLRunDB(db.dsn, db_session, leader_session)
else:
run_db = db.db
run_db.connect()
return run_db
def test_list_artifact_date(db: sqldb.SQLDB):
t1 = datetime(2020, 2, 16)
t2 = t1 - timedelta(days=7)
t3 = t2 - timedelta(days=7)
prj = 'p7'
db.store_artifact('k1', {'updated': t1}, 'u1', project=prj)
db.store_artifact('k2', {'updated': t2}, 'u2', project=prj)
db.store_artifact('k3', {'updated': t3}, 'u3', project=prj)
arts = db.list_artifacts(project=prj, since=t3, tag='*')
assert 3 == len(arts), 'since t3'
arts = db.list_artifacts(project=prj, since=t2, tag='*')
assert 2 == len(arts), 'since t2'
arts = db.list_artifacts(project=prj,
since=t1 + timedelta(days=1),
tag='*')
assert not arts, 'since t1+'
arts = db.list_artifacts(project=prj, until=t2, tag='*')
assert 2 == len(arts), 'until t2'
arts = db.list_artifacts(project=prj, since=t2, until=t2, tag='*')
assert 1 == len(arts), 'since/until t2'
def test_list_runs(db: sqldb.SQLDB):
uid = 'u183'
run = new_run('s1', ['l1', 'l2'], x=1)
count = 5
for iter in range(count):
db.store_run(run, uid, iter=iter)
runs = list(db.list_runs(uid=uid))
assert 1 == len(runs), 'iter=False'
runs = list(db.list_runs(uid=uid, iter=True))
assert 5 == len(runs), 'iter=True'
def test_projects(db: sqldb.SQLDB):
prj1 = {
'name': 'p1',
'description': 'banana',
# 'users': ['u1', 'u2'],
'spec': {
'company': 'ACME'
},
'state': 'active',
'created': datetime.now(),
}
pid1 = db.add_project(prj1)
p1 = db.get_project(project_id=pid1)
assert p1, f'project {pid1} not found'
out = {
'name': p1.name,
'description': p1.description,
# 'users': sorted(u.name for u in p1.users),
'spec': p1.spec,
'state': p1.state,
'created': p1.created,
}
assert prj1 == out, 'bad project'
data = {'description': 'lemon'}
db.update_project(p1.name, data)
p1 = db.get_project(project_id=pid1)
assert data['description'] == p1.description, 'bad update'
prj2 = {'name': 'p2'}
db.add_project(prj2)
prjs = {p.name for p in db.list_projects()}
assert {prj1['name'], prj2['name']} == prjs, 'list'
def test_list_tags(db: sqldb.SQLDB):
p1, tags1 = 'prj1', ['a', 'b', 'c']
tag_objs(db, 17, p1, tags1)
p2, tags2 = 'prj2', ['b', 'c', 'd', 'e']
tag_objs(db, 11, p2, tags2)
tags = db.list_tags(p1)
assert set(tags) == set(tags1), 'tags'
def test_log(db: sqldb.SQLDB):
uid = 'm33'
data1, data2 = b'ab', b'cd'
db.store_log(uid, body=data1)
_, log = db.get_log(uid)
assert data1 == log, 'get log 1'
db.store_log(uid, body=data2, append=True)
_, log = db.get_log(uid)
assert data1 + data2 == log, 'get log 2'
db.store_log(uid, body=data1, append=False)
_, log = db.get_log(uid)
assert data1 == log, 'get log append=False'
def init_app():
global _db, _logs_dir, _k8s, _scheduler
logger.info('configuration dump\n%s', config.dump_yaml())
if config.httpdb.db_type == 'sqldb':
logger.info('using SQLDB')
_db = SQLDB(config.httpdb.dsn)
else:
logger.info('using FileRunDB')
_db = FileRunDB(config.httpdb.dirpath)
_db.connect()
_logs_dir = Path(config.httpdb.logs_path)
try:
_k8s = K8sHelper()
except Exception:
pass
# @yaronha - Initialize here
task = periodic.Task()
periodic.schedule(task, 60)
_scheduler = Scheduler()
for data in _db.list_schedules():
if 'schedule' not in data:
logger.warning('bad scheduler data - %s', data)
continue
_submit(data)
def test_list_functions(db: sqldb.SQLDB):
name = 'fn'
fn1 = new_func(['l1', 'l2'], x=1)
db.store_function(fn1, name)
fn2 = new_func(['l2', 'l3'], x=2)
db.store_function(fn2, name, tag='t1')
fn3 = new_func(['l3'], x=3)
db.store_function(fn3, name, tag='t2')
funcs = db.list_functions(name, labels=['l2'])
assert 2 == len(funcs), 'num of funcs'
assert {1, 2} == {fn['x'] for fn in funcs}, 'xs'
def test_artifacts_latest(db: sqldb.SQLDB):
k1, u1, art1 = 'k1', 'u1', {'a': 1}
prj = 'p38'
db.store_artifact(k1, art1, u1, project=prj)
arts = db.list_artifacts(project=prj, tag='latest')
assert art1['a'] == arts[0]['a'], 'bad artifact'
u2, art2 = 'u2', {'a': 17}
db.store_artifact(k1, art2, u2, project=prj)
arts = db.list_artifacts(project=prj, tag='latest')
assert 1 == len(arts), 'count'
assert art2['a'] == arts[0]['a'], 'bad artifact'
k2, u3, art3 = 'k2', 'u3', {'a': 99}
db.store_artifact(k2, art3, u3, project=prj)
arts = db.list_artifacts(project=prj, tag='latest')
assert 2 == len(arts), 'number'
assert {17, 99} == set(art['a'] for art in arts), 'latest'
def test_cache_projects(db: sqldb.SQLDB):
assert 0 == len(db._projects), 'empty cache'
name = 'prj348'
db.add_project({'name': name})
assert {name} == db._projects, 'project'
mock = Mock()
with patch(db, add_project=mock):
db._create_project_if_not_exists(name)
mock.assert_not_called()
mock = Mock()
with patch(db, add_project=mock):
db._create_project_if_not_exists(name + '-new')
mock.assert_called_once()
def test_tags(db: sqldb.SQLDB, cls):
p1, n1 = 'prj1', 'name1'
obj1, obj2, obj3 = cls(), cls(), cls()
db.session.add(obj1)
db.session.add(obj2)
db.session.add(obj3)
db.session.commit()
db.tag_objects([obj1, obj2], p1, n1)
objs = db.find_tagged(p1, n1)
assert {obj1, obj2} == set(objs), 'find tags'
db.del_tag(p1, n1)
objs = db.find_tagged(p1, n1)
assert [] == objs, 'find tags after del'
def test_artifacts(db: sqldb.SQLDB):
k1, u1, art1 = 'k1', 'u1', {'a': 1}
db.store_artifact(k1, art1, u1)
art = db.read_artifact(k1, u1)
assert art1['a'] == art['a'], 'get artifact'
art = db.read_artifact(k1)
assert art1['a'] == art['a'], 'get latest artifact'
prj = 'p1'
k2, u2, art2 = 'k2', 'u2', {'a': 2}
db.store_artifact(k2, art2, u2, project=prj)
k3, u3, art3 = 'k3', 'u3', {'a': 3}
db.store_artifact(k3, art3, u3, project=prj)
arts = db.list_artifacts(project=prj)
assert 2 == len(arts), 'list artifacts length'
assert {2, 3} == {a['a'] for a in arts}, 'list artifact a'
db.del_artifact(key=k1)
with pytest.raises(sqldb.RunDBError):
db.read_artifact(k1)
def test_save_get_function(db: sqldb.SQLDB):
func, name, proj = {'x': 1, 'y': 2}, 'f1', 'p2'
db.store_function(func, name, proj)
db_func = db.get_function(name, proj)
assert func == db_func, 'wrong func'
def test_runs(db: sqldb.SQLDB):
run1 = new_run('s1', ['l1', 'l2'], x=1)
db.store_run(run1, 'uid1')
run2 = new_run('s1', ['l2', 'l3'], x=2)
db.store_run(run2, 'uid2')
run3 = new_run('s2', ['l3'], x=2)
uid3 = 'uid3'
db.store_run(run3, uid3)
db.store_run(run3, uid3) # should not raise
updates = {
'status': {
'start_time': run_now(),
},
}
db.update_run(updates, uid3)
runs = db.list_runs(labels=['l2'])
assert 2 == len(runs), 'labels length'
assert {1, 2} == {r['x'] for r in runs}, 'xs labels'
runs = db.list_runs(state='s2')
assert 1 == len(runs), 'state length'
run3['status'] = updates['status']
assert run3 == runs[0], 'state run'
db.del_run(uid3)
with pytest.raises(sqldb.RunDBError):
db.read_run(uid3)
label = 'l1'
db.del_runs(labels=[label])
for run in db.list_runs():
assert label not in run['metadata']['labels'], 'del_runs'
def test_list_projects(db: sqldb.SQLDB):
for i in range(10):
run = new_run('s1', {'l1': 'v1', 'l2': 'v2'}, x=1)
db.store_run(run, 'u7', project=f'prj{i%3}', iter=i)
assert {'prj0', 'prj1', 'prj2'} == {p.name for p in db.list_projects()}
def test_run_iter0(db: sqldb.SQLDB):
uid, prj = 'uid39', 'lemon'
run = new_run('s1', {'l1': 'v1', 'l2': 'v2'}, x=1)
for i in range(7):
db.store_run(run, uid, prj, i)
db._get_run(uid, prj, 0) # See issue 140
def test_list_projects(db: sqldb.SQLDB):
for i in range(10):
run = new_run('s1', ['l1', 'l2'], x=1)
db.store_run(run, 'u7', project=f'prj{i%3}', iter=i)
assert {'prj0', 'prj1', 'prj2'} == set(db.list_projects())
