def NOtest_do_if_locked():
app = create_test_app()
new_map = update_aux_db_engine_discovery_map(
app, force_update=True, aux_db_engine_name_prefix='test_aux_')
sm = AuxSessionManager(app=app)
sn = 1
session = sm.get(sn)
def test_add_remove():
app = create_test_app()
sm = AuxSessionManager(app=app)
sn = 1
session = sm.get(sn)
assert session.query(BlockModel).count() == 0
assert session.query(HeaderModel).count() == 0
assert session.query(TxModel).count() == 0
assert session.query(ParamModel).count() == 0
assert session.query(ErrorModel).count() == 0
block = Block(b64decode_hashes=True, from_detailed_dict=d3[0])
block_m = BlockModel.update_from_block(block, session=sm.get(sn))
assert isinstance(block_m, BlockModel)
assert session.query(BlockModel).count() == 1
assert session.query(HeaderModel).count() == 1
assert session.query(TxModel).count() == 2
assert session.query(ParamModel).count() == 4
assert session.query(ErrorModel).count() == 0
block_m.add_error(error="Test Error", raw_error="Test Raw Error")
assert session.query(ErrorModel).count() == 1
block_m.delete(session=session)
assert session.query(BlockModel).count() == 0
assert session.query(HeaderModel).count() == 0
assert session.query(TxModel).count() == 0
assert session.query(ParamModel).count() == 0
assert session.query(ErrorModel).count() == 0
def test_add():
app = create_test_app()
new_map = update_aux_db_engine_discovery_map(
app, force_update=True, aux_db_engine_name_prefix='test_aux_')
sm = AuxSessionManager(app=app)
seq_num = 1
session = sm.get(seq_num)
def test_delete_expired_locks():
app = create_test_app()
new_map = update_aux_db_engine_discovery_map(
app, force_update=True, aux_db_engine_name_prefix='test_aux_')
sm = AuxSessionManager(app=app)
seq_num = 1
session = sm.get(seq_num)
create_tables([LockModel], sm.get_engine(seq_num))
assert LockModel.is_locked(session=session) == False
to = 1
eqs = LockModel.get_expired_locks(session=session, timeout_secs=to)
assert len(eqs) == 0
LockModel.lock(session=session)
assert LockModel.is_locked(session=session) == True
assert session.query(LockModel).count() == 1
lock = LockModel.get_latest_lock(session=session)
eqs = LockModel.get_expired_locks(session=session, timeout_secs=to)
assert len(eqs) == 0
print("Pausing for %d seconds ..." % (to + 1))
time.sleep(to + 1)
LockModel.lock(session=session)
assert session.query(LockModel).count() == 2
eqs = LockModel.get_expired_locks(session=session, timeout_secs=to)
assert len(eqs) == 1
LockModel.delete_expired_locks(session=session, timeout_secs=to)
eqs = LockModel.get_expired_locks(session=session, timeout_secs=to)
assert len(eqs) == 0
assert session.query(LockModel).count() == 1
def test_member_add_record_from_dict():
app = create_test_app()
update_aux_db_engine_discovery_map(app,
force_update=True,
aux_db_engine_name_prefix='test_aux_')
asm = AuxSessionManager(app=app)
seq_num = 1
engine = asm.get_engine(seq_num)
session = asm.get(seq_num)
session.query(MemberModel).delete()
assert session.query(MemberModel).count() == 0
d = {
'ecosystem_id': 1,
'key_id': -123,
'address': '456',
'pub': 'a9823432',
'amount': 13.1,
'maxpay': 10.2,
'multi': 1,
'deleted': 1,
'blocked': 1,
}
MemberModel.add_record_from_dict(d, session=session)
assert session.query(MemberModel).count() == 1
i = session.query(MemberModel).first()
for k, v in d.items():
assert hasattr(i, k)
val = getattr(i, k)
if k in ('amount', 'maxpay'):
assert float(v) == float(val)
else:
assert val == v
def test_add_all_records_for_all_ecosystems():
app = create_test_app()
update_aux_db_engine_discovery_map(app,
force_update=True,
aux_db_engine_name_prefix='test_aux_')
asm = AuxSessionManager(app=app)
seq_num = 1
engine = asm.get_engine(seq_num)
session = asm.get(seq_num)
m_session = sm.get(seq_num)
session.query(MemberModel).delete()
assert session.query(MemberModel).count() == 0
assert m_session.query(EsKeys).count() > 0
MemberModel.add_all_records_for_all_ecosystems(EsKeys,
ecosystem_model=Ecosystem,
session=session,
model_session=m_session)
total = 0
for es in m_session.query(Ecosystem).all():
EsKeys.set_ecosystem(es.id)
for m_item in m_session.query(EsKeys).all():
item = session.query(MemberModel).filter_by(key_id=m_item.id).one()
assert_eskeys_and_member_instances_eq(m_item, item)
total += m_session.query(EsKeys).count()
assert session.query(MemberModel).count() == total
def test_update_from_param():
app = create_test_app()
new_map = update_aux_db_engine_discovery_map(
app, force_update=True, aux_db_engine_name_prefix='test_aux_')
sm = AuxSessionManager(app=app)
sn = 1
session = sm.get(sn)
create_tables([TxModel, ParamModel], sm.get_engine(sn))
# TODO: fix when:
# td = get_block_data_from_dict(d3[0])['transactions'][0]['params'] # [0]
td = get_block_data_from_dict(d3[0])['transactions'][1]['params'] # [0]
print("test_update_from_param 2 td: %s" % td)
p0n = tuple(td.keys())[0]
p0v = td[p0n]
pd = {p0n: p0v}
app = create_test_app()
assert len(ParamModel.query.with_session(session).all()) == 0
p = Param(**pd)
ParamModel.update_from_param(p, session=session)
assert len(ParamModel.query.with_session(session).all()) == 1
pm = ParamModel.query.with_session(session).all()[0]
assert pm.name == p0n
assert pm.value == p0v
try:
ParamModel.update_from_param(p, session=session)
except exc.IntegrityError:
pass
def test_update_from_block_model_instance_via_manager():
app = create_test_app()
bhm = BlockHelperManager(app=app, engine_echo=False)
sm = AuxSessionManager(app=app)
seq_num = 1
bm_session = sm.get(seq_num)
bm_engine = sm.get_engine(seq_num)
create_tables([BlockModel], bm_engine, recreate_if_exists=True)
len_bm_first = len(BlockModel.query.with_session(bm_session).all())
assert len_bm_first == 0
block = Block(b64decode_hashes=True, from_detailed_dict=d3[0])
BlockModel.update_from_block(block, session=bm_session)
assert len(BlockModel.query.with_session(bm_session).all()) == len_bm_first + 1
bm_instance = BlockModel.query.with_session(bm_session).all()[0]
len_first = len(bhm.session.query(bhm.model).all())
assert len_first == 0
bhm.model.update_from_main_model_instance(bm_instance, session=bhm.session, main_model_session=bm_session, seq_num=seq_num)
num_of_bhm_records = len(BlockModel.__table__.columns) - len(BlockHelperModel.get_drop_fields()) + 2
assert len(bhm.session.query(bhm.model).all()) == len_first + num_of_bhm_records
bhm.session.query(bhm.model).filter_by(seq_num=seq_num, block_id=bm_instance.id).delete()
bhm.model.update_from_main_model_instance(bm_instance, session=bhm.session, main_model_session=bm_session, seq_num=seq_num)
assert len(bhm.session.query(bhm.model).all()) == len_first + num_of_bhm_records
def test_add_clear():
app = create_test_app()
sm = AuxSessionManager(app=app)
sn = 1
session = sm.get(sn)
assert session.query(BlockModel).count() == 0
assert session.query(HeaderModel).count() == 0
assert session.query(TxModel).count() == 0
assert session.query(ParamModel).count() == 0
assert session.query(ErrorModel).count() == 0
block1 = Block(b64decode_hashes=True, from_detailed_dict=d3[0])
block1_m = BlockModel.update_from_block(block1, session=sm.get(sn))
block2 = Block(b64decode_hashes=True, from_detailed_dict=d3[1])
block2_m = BlockModel.update_from_block(block2, session=sm.get(sn))
assert isinstance(block1_m, BlockModel)
assert isinstance(block2_m, BlockModel)
assert block1_m != block2_m
assert session.query(BlockModel).count() == 2
assert session.query(HeaderModel).count() == 2
assert session.query(TxModel).count() == 4
assert session.query(ParamModel).count() == 8
assert session.query(ErrorModel).count() == 0
block1_m.add_error(error="Test 1 Error", raw_error="Test 1 Raw Error")
assert session.query(ErrorModel).count() == 1
block2_m.add_error(error="Test 2 Error", raw_error="Test 2 Raw Error")
assert session.query(ErrorModel).count() == 2
BlockModel.clear(session=session)
assert session.query(BlockModel).count() == 0
assert session.query(HeaderModel).count() == 0
assert session.query(ParamModel).count() == 0
assert session.query(TxModel).count() == 0
assert session.query(ErrorModel).count() == 0
def NOtest_add_fill_error():
app = create_test_app()
sm = AuxSessionManager(app=app)
sn = 1
len_first = BlockModel.query.with_session(sm.get(sn)).count()
block_id = tuple(d3[0].keys())[0]
BlockModel.add_fill_error(block_id, session=sm.get(sn))
len_after = BlockModel.query.with_session(sm.get(sn)).count()
def test_get_engine():
app = create_test_app()
sm = AuxSessionManager(app=app)
if len(app.config['AUX_DB_ENGINE_DISCOVERY_MAP']) == 0:
try:
sm.get_engine(1)
except DbEngineMapIsEmptyError as e:
print("AUX_DB_ENGINE_DISCOVERY_MAP is empty")
return
assert isinstance(sm.get_engine(1), Engine)
def test_table_manager_object():
app = create_test_app()
models = [BlockModel, HeaderModel, TxModel, ParamModel]
sm = AuxSessionManager(app=app)
tm = TableManager(app=app, recreate_if_exists=True)
tm.create_tables()
for seq_num in range(1, get_num_of_backends(app) + 1):
engine = sm.get_engine(seq_num)
for model in models:
assert engine.dialect.has_table(engine, model.__tablename__)
def create_tables_by_seq_nums(**kwargs):
app = kwargs.get('app', create_test_app())
recreate_if_exists = kwargs.get('recreate_if_exists', True)
_seq_nums = kwargs.get('seq_nums', seq_nums)
_involved_models = kwargs.get('involved_models', involved_models)
for seq_num in _seq_nums:
asm = AuxSessionManager(app=app)
create_tables(_involved_models,
asm.get_engine(seq_num),
recreate_if_exists=recreate_if_exists)
def test_fill_block():
app = create_test_app()
sm = AuxSessionManager(app=app)
seq_num = 1
session = sm.get(seq_num)
BlockModel.clear(session=session)
assert session.query(BlockModel).count() == 0
f = BlockFiller(app=app, seq_num=seq_num, recreate_tables_if_exist=False)
block_id = 3
f.fill_block(block_id)
assert session.query(BlockModel).count() == 1
def test_get_bind_name():
app = create_test_app()
sm = AuxSessionManager(app=app)
if len(app.config['AUX_DB_ENGINE_DISCOVERY_MAP']) == 0:
try:
sm.get_bind_name(1)
except DbEngineMapIsEmptyError as e:
print("AUX_DB_ENGINE_DISCOVERY_MAP is empty")
return
assert sm.get_bind_name(1) == tuple(
app.config['AUX_DB_ENGINE_DISCOVERY_MAP'].keys())[0]
def test_member_object_creation():
app = create_test_app()
update_aux_db_engine_discovery_map(app,
force_update=True,
aux_db_engine_name_prefix='test_aux_')
asm = AuxSessionManager(app=app)
seq_num = 1
engine = asm.get_engine(seq_num)
session = asm.get(seq_num)
create_tables(involved_models, engine)
assert session.query(MemberModel).with_session(
asm.get(seq_num)).count() == 0
def test_update_from_header():
app = create_test_app()
new_map = update_aux_db_engine_discovery_map(app, force_update=True,
aux_db_engine_name_prefix='test_aux_')
sm = AuxSessionManager(app=app)
sn = 1
session = sm.get(sn)
hd = get_block_data_from_dict(d3[0])['header']
len_first = len(HeaderModel.query.with_session(session).all())
h = Header(b64decode_hashes=True, from_dict=hd)
HeaderModel.update_from_header(h, session=session)
assert len(HeaderModel.query.with_session(session).all()) == len_first + 1
def test_update_from_dict():
app = create_test_app()
new_map = update_aux_db_engine_discovery_map(app, force_update=True,
aux_db_engine_name_prefix='test_aux_')
sm = AuxSessionManager(app=app)
sn = 1
session = sm.get(sn)
create_tables([BlockModel, HeaderModel], sm.get_engine(sn))
hd = get_block_data_from_dict(d3[0])['header']
len_first = len(HeaderModel.query.with_session(session).all())
HeaderModel.update_from_dict(hd, session=session)
assert len(HeaderModel.query.with_session(session).all()) == len_first + 1
def test_fill_blocks__1():
app = create_test_app()
sm = AuxSessionManager(app=app)
seq_num = 1
session = sm.get(seq_num)
f = BlockFiller(app=app,
seq_num=seq_num,
recreate_tables_if_exist=False,
fetch_num_of_blocks=15)
assert session.query(BlockModel).count() == 0
f.fill_blocks(1, 10)
assert session.query(BlockModel).count() == 10
def NOtest_update_from_list_of_dicts():
app = create_test_app()
new_map = update_aux_db_engine_discovery_map(app, force_update=True,
aux_db_engine_name_prefix='test_aux_')
sm = AuxSessionManager(app=app)
sn = 1
session = sm.get(sn)
td = get_block_data_from_dict(d3[1])['transactions']
len_first = len(TxModel.query.with_session(session).all())
TxModel.update_from_list_of_dicts(td, session=session)
assert len(TxModel.query.with_session(session).all()) == len_first + len(td)
def NOtest_update_from_tx_set():
app = create_test_app()
new_map = update_aux_db_engine_discovery_map(app, force_update=True,
aux_db_engine_name_prefix='test_aux_')
sm = AuxSessionManager(app=app)
sn = 1
session = sm.get(sn)
td = get_block_data_from_dict(d3[0])['transactions']
len_first = len(TxModel.query.with_session(session).all())
tx = TxSet(b64decode_hashes=True, from_list=td)
TxModel.update_from_tx_set(tx, session=session)
assert len(TxModel.query.with_session(session).all()) == len_first + tx.count
def test_fill_all_blocks():
app = create_test_app()
sm = AuxSessionManager(app=app)
seq_num = 1
session = sm.get(seq_num)
BlockModel.clear(session=session)
assert session.query(BlockModel).count() == 0
f = BlockFiller(app=app,
seq_num=seq_num,
recreate_tables_if_exist=False,
fetch_num_of_blocks=10)
f.fill_all_blocks()
assert session.query(BlockModel).count() == get_max_block_id(seq_num)
def test_update_from_main_model_instance_via_manager():
app = create_test_app()
mhm = MemberHelperManager(app=app, engine_echo=True)
sm = AuxSessionManager(app=app)
seq_num = 1
mm_session = sm.get(seq_num)
mm_engine = sm.get_engine(seq_num)
create_tables([MemberModel], mm_engine, recreate_if_exists=True)
len_mm_first = len(MemberModel.query.with_session(mm_session).all())
assert len_mm_first == 0
d = {
'ecosystem_id': 1,
'key_id': -123,
'address': '456',
'pub': 'a9823432',
'amount': 13.1,
'maxpay': 10.2,
'multi': 1,
'deleted': 1,
'blocked': 1,
}
MemberModel.add_record_from_dict(d, session=mm_session)
assert len(
MemberModel.query.with_session(mm_session).all()) == len_mm_first + 1
mm_instance = MemberModel.query.with_session(mm_session).all()[0]
len_first = len(mhm.session.query(mhm.model).all())
assert len_first == 0
mhm.model.update_from_main_model_instance(mm_instance,
session=mhm.session,
main_model_session=mm_session,
seq_num=seq_num,
ecosystem_id=1)
num_of_mhm_records = len(MemberModel.__table__.columns) - len(
MemberHelperModel.get_drop_fields())
assert len(mhm.session.query(
mhm.model).all()) == len_first + num_of_mhm_records
print("---------------ALL: %s" % mhm.session.query(mhm.model).filter_by(
seq_num=seq_num, id=mm_instance.id).all())
mhm.session.query(mhm.model).filter_by(seq_num=seq_num,
id=mm_instance.id).delete()
mhm.model.update_from_main_model_instance(mm_instance,
session=mhm.session,
main_model_session=mm_session,
seq_num=seq_num,
ecosystem_id=1)
print("len of mhm: %s" % len(mhm.session.query(mhm.model).all()))
def NOtest_get_bind_name():
app = create_test_app()
new_map = update_aux_db_engine_discovery_map(
app, force_update=True, aux_db_engine_name_prefix='test_aux_')
sm = AuxSessionManager(app=app)
sn = 1
if len(app.config['AUX_DB_ENGINE_DISCOVERY_MAP']) == 0:
try:
sm.get_bind_name(sn)
except DbEngineMapIsEmptyError as e:
print("AUX_DB_ENGINE_DISCOVERY_MAP is empty")
return
assert sm.get_bind_name(sn) == tuple(
app.config['AUX_DB_ENGINE_DISCOVERY_MAP'].keys())[0]
def test_add():
app = create_test_app()
new_map = update_aux_db_engine_discovery_map(
app, force_update=True, aux_db_engine_name_prefix='test_aux_')
sm = AuxSessionManager(app=app)
seq_num = 1
session = sm.get(seq_num)
create_tables([LogModel], sm.get_engine(seq_num))
assert len(LogModel.query.with_session(session=session).all()) == 0
LogModel.add(session=session, context='filler', caller='fill_block')
assert len(LogModel.query.with_session(session=session).all()) == 1
LogModel.add(session=session, context='filler', caller='fill_block')
assert len(LogModel.query.with_session(session=session).all()) == 2
def nontest_update_from_main_model_instance():
app = create_test_app()
update_aux_db_engine_discovery_map(app,
force_update=True,
aux_db_engine_name_prefix='test_aux_')
engine = get_aux_helpers_engine(app)
metadata = MetaData()
metadata.create_all(engine)
session = get_aux_helpers_session(app)
create_tables([TxHelperModel], engine, recreate_if_exists=True)
session.query(TxHelperModel).all()
sm = AuxSessionManager(app=app)
seq_num = 1
bm_session = sm.get(seq_num)
bm_engine = sm.get_engine(seq_num)
create_tables([TxModel], bm_engine, recreate_if_exists=True)
len_bm_first = len(TxModel.query.with_session(bm_session).all())
assert len_bm_first == 0
block = Block(b64decode_hashes=True, from_detailed_dict=d3[0])
BlockModel.update_from_block(block, session=bm_session)
assert len(
BlockModel.query.with_session(bm_session).all()) == len_bm_first + 1
bm_instance = BlockModel.query.with_session(bm_session).all()[0]
len_first = len(BlockHelperModel.query.with_session(session).all())
assert len_first == 0
BlockHelperModel.update_from_block_model_instance(
bm_instance,
session=session,
block_model_session=bm_session,
seq_num=seq_num)
num_of_bm_fields = len(BlockModel.__table__.columns)
assert len(
BlockHelperModel.query.with_session(session).filter_by(
seq_num=seq_num).all()) == len_first + num_of_bm_fields - 1
BlockHelperModel.query.with_session(session).filter_by(
seq_num=seq_num, block_id=bm_instance.id).delete()
BlockHelperModel.update_from_block_model_instance(
bm_instance,
session=session,
block_model_session=bm_session,
seq_num=seq_num)
assert len(
BlockHelperModel.query.with_session(session).filter_by(
seq_num=seq_num).all()) == len_first + num_of_bm_fields - 1
def NOtest_add_delete():
app = create_test_app()
sm = AuxSessionManager(app=app)
sn = 1
session = sm.get(sn)
td = get_block_data_from_dict(d3[0])['transactions'][0]
assert TxModel.query.with_session(session).count() == 0
assert ParamModel.query.with_session(session).count() == 0
tx_m = TxModel.update_from_dict(td, session=session)
assert isinstance(tx_m, TxModel)
assert TxModel.query.with_session(session).count() == 1
assert ParamModel.query.with_session(session).count() == 2
tx_m.delete(session=session)
assert TxModel.query.with_session(session).count() == 0
assert ParamModel.query.with_session(session).count() == 0
def test_get_latest_lock():
app = create_test_app()
new_map = update_aux_db_engine_discovery_map(
app, force_update=True, aux_db_engine_name_prefix='test_aux_')
sm = AuxSessionManager(app=app)
seq_num = 1
session = sm.get(seq_num)
create_tables([LockModel], sm.get_engine(seq_num))
assert LockModel.is_locked(session=session) == False
LockModel.lock(session=session)
assert LockModel.is_locked(session=session) == True
q = LockModel.get_latest_lock(session=session)
assert isinstance(q, LockModel)
LockModel.unlock(session=session)
q = LockModel.get_latest_lock(session=session)
assert q is None
def test_clear_garbage():
app = create_test_app()
new_map = update_aux_db_engine_discovery_map(
app, force_update=True, aux_db_engine_name_prefix='test_aux_')
sm = AuxSessionManager(app=app)
seq_num = 1
session = sm.get(seq_num)
create_tables([LockModel], sm.get_engine(seq_num))
assert LockModel.is_locked(session=session) == False
to = 1
fake_pid = get_fake_pid()
LockModel.lock(session=session)
time.sleep(to + 1)
LockModel.lock(session=session)
LockModel.lock(session=session, process_id=fake_pid)
assert LockModel.is_locked(session=session) == True
assert session.query(LockModel).count() == 3
qs = session.query(LockModel).all()
qs_pids = [q.process_id for q in qs]
assert fake_pid in qs_pids
q = LockModel.get_latest_lock(session=session)
assert isinstance(q, LockModel)
assert session.query(LockModel).count() == 3
zqs = LockModel.get_zombie_locks(session=session)
zqs_pids = [q.process_id for q in zqs]
assert len(zqs_pids) == 1
assert fake_pid in zqs_pids
eqs = LockModel.get_expired_locks(session=session, timeout_secs=to)
assert len(eqs) == 1
LockModel.clear_garbage(session=session, timeout_secs=to)
zqs = LockModel.get_zombie_locks(session=session)
zqs_pids = [q.process_id for q in zqs]
assert len(zqs_pids) == 0
eqs = LockModel.get_expired_locks(session=session, timeout_secs=to)
assert len(eqs) == 0
LockModel.unlock(session=session)
assert session.query(LockModel).count() == 0
def test_add_clear():
app = create_test_app()
sm = AuxSessionManager(app=app)
sn = 1
session = sm.get(sn)
td1 = get_block_data_from_dict(d3[0])['transactions'][0]
td2 = get_block_data_from_dict(d3[1])['transactions'][0]
assert TxModel.query.with_session(session).count() == 0
assert ParamModel.query.with_session(session).count() == 0
tx1_m = TxModel.update_from_dict(td1, session=session)
tx2_m = TxModel.update_from_dict(td2, session=session)
assert isinstance(tx1_m, TxModel)
assert isinstance(tx2_m, TxModel)
assert TxModel.query.with_session(session).count() == 2
assert ParamModel.query.with_session(session).count() == 4
TxModel.clear(session=session)
assert TxModel.query.with_session(session).count() == 0
assert ParamModel.query.with_session(session).count() == 0