def NOtest_do_if_locked(): app = create_test_app() new_map = update_aux_db_engine_discovery_map( app, force_update=True, aux_db_engine_name_prefix='test_aux_') sm = AuxSessionManager(app=app) sn = 1 session = sm.get(sn)
def test_add_remove(): app = create_test_app() sm = AuxSessionManager(app=app) sn = 1 session = sm.get(sn) assert session.query(BlockModel).count() == 0 assert session.query(HeaderModel).count() == 0 assert session.query(TxModel).count() == 0 assert session.query(ParamModel).count() == 0 assert session.query(ErrorModel).count() == 0 block = Block(b64decode_hashes=True, from_detailed_dict=d3[0]) block_m = BlockModel.update_from_block(block, session=sm.get(sn)) assert isinstance(block_m, BlockModel) assert session.query(BlockModel).count() == 1 assert session.query(HeaderModel).count() == 1 assert session.query(TxModel).count() == 2 assert session.query(ParamModel).count() == 4 assert session.query(ErrorModel).count() == 0 block_m.add_error(error="Test Error", raw_error="Test Raw Error") assert session.query(ErrorModel).count() == 1 block_m.delete(session=session) assert session.query(BlockModel).count() == 0 assert session.query(HeaderModel).count() == 0 assert session.query(TxModel).count() == 0 assert session.query(ParamModel).count() == 0 assert session.query(ErrorModel).count() == 0
def test_add(): app = create_test_app() new_map = update_aux_db_engine_discovery_map( app, force_update=True, aux_db_engine_name_prefix='test_aux_') sm = AuxSessionManager(app=app) seq_num = 1 session = sm.get(seq_num)
def test_delete_expired_locks(): app = create_test_app() new_map = update_aux_db_engine_discovery_map( app, force_update=True, aux_db_engine_name_prefix='test_aux_') sm = AuxSessionManager(app=app) seq_num = 1 session = sm.get(seq_num) create_tables([LockModel], sm.get_engine(seq_num)) assert LockModel.is_locked(session=session) == False to = 1 eqs = LockModel.get_expired_locks(session=session, timeout_secs=to) assert len(eqs) == 0 LockModel.lock(session=session) assert LockModel.is_locked(session=session) == True assert session.query(LockModel).count() == 1 lock = LockModel.get_latest_lock(session=session) eqs = LockModel.get_expired_locks(session=session, timeout_secs=to) assert len(eqs) == 0 print("Pausing for %d seconds ..." % (to + 1)) time.sleep(to + 1) LockModel.lock(session=session) assert session.query(LockModel).count() == 2 eqs = LockModel.get_expired_locks(session=session, timeout_secs=to) assert len(eqs) == 1 LockModel.delete_expired_locks(session=session, timeout_secs=to) eqs = LockModel.get_expired_locks(session=session, timeout_secs=to) assert len(eqs) == 0 assert session.query(LockModel).count() == 1
def test_member_add_record_from_dict(): app = create_test_app() update_aux_db_engine_discovery_map(app, force_update=True, aux_db_engine_name_prefix='test_aux_') asm = AuxSessionManager(app=app) seq_num = 1 engine = asm.get_engine(seq_num) session = asm.get(seq_num) session.query(MemberModel).delete() assert session.query(MemberModel).count() == 0 d = { 'ecosystem_id': 1, 'key_id': -123, 'address': '456', 'pub': 'a9823432', 'amount': 13.1, 'maxpay': 10.2, 'multi': 1, 'deleted': 1, 'blocked': 1, } MemberModel.add_record_from_dict(d, session=session) assert session.query(MemberModel).count() == 1 i = session.query(MemberModel).first() for k, v in d.items(): assert hasattr(i, k) val = getattr(i, k) if k in ('amount', 'maxpay'): assert float(v) == float(val) else: assert val == v
def test_add_all_records_for_all_ecosystems(): app = create_test_app() update_aux_db_engine_discovery_map(app, force_update=True, aux_db_engine_name_prefix='test_aux_') asm = AuxSessionManager(app=app) seq_num = 1 engine = asm.get_engine(seq_num) session = asm.get(seq_num) m_session = sm.get(seq_num) session.query(MemberModel).delete() assert session.query(MemberModel).count() == 0 assert m_session.query(EsKeys).count() > 0 MemberModel.add_all_records_for_all_ecosystems(EsKeys, ecosystem_model=Ecosystem, session=session, model_session=m_session) total = 0 for es in m_session.query(Ecosystem).all(): EsKeys.set_ecosystem(es.id) for m_item in m_session.query(EsKeys).all(): item = session.query(MemberModel).filter_by(key_id=m_item.id).one() assert_eskeys_and_member_instances_eq(m_item, item) total += m_session.query(EsKeys).count() assert session.query(MemberModel).count() == total
def test_update_from_param(): app = create_test_app() new_map = update_aux_db_engine_discovery_map( app, force_update=True, aux_db_engine_name_prefix='test_aux_') sm = AuxSessionManager(app=app) sn = 1 session = sm.get(sn) create_tables([TxModel, ParamModel], sm.get_engine(sn)) # TODO: fix when: # td = get_block_data_from_dict(d3[0])['transactions'][0]['params'] # [0] td = get_block_data_from_dict(d3[0])['transactions'][1]['params'] # [0] print("test_update_from_param 2 td: %s" % td) p0n = tuple(td.keys())[0] p0v = td[p0n] pd = {p0n: p0v} app = create_test_app() assert len(ParamModel.query.with_session(session).all()) == 0 p = Param(**pd) ParamModel.update_from_param(p, session=session) assert len(ParamModel.query.with_session(session).all()) == 1 pm = ParamModel.query.with_session(session).all()[0] assert pm.name == p0n assert pm.value == p0v try: ParamModel.update_from_param(p, session=session) except exc.IntegrityError: pass
def test_update_from_block_model_instance_via_manager(): app = create_test_app() bhm = BlockHelperManager(app=app, engine_echo=False) sm = AuxSessionManager(app=app) seq_num = 1 bm_session = sm.get(seq_num) bm_engine = sm.get_engine(seq_num) create_tables([BlockModel], bm_engine, recreate_if_exists=True) len_bm_first = len(BlockModel.query.with_session(bm_session).all()) assert len_bm_first == 0 block = Block(b64decode_hashes=True, from_detailed_dict=d3[0]) BlockModel.update_from_block(block, session=bm_session) assert len(BlockModel.query.with_session(bm_session).all()) == len_bm_first + 1 bm_instance = BlockModel.query.with_session(bm_session).all()[0] len_first = len(bhm.session.query(bhm.model).all()) assert len_first == 0 bhm.model.update_from_main_model_instance(bm_instance, session=bhm.session, main_model_session=bm_session, seq_num=seq_num) num_of_bhm_records = len(BlockModel.__table__.columns) - len(BlockHelperModel.get_drop_fields()) + 2 assert len(bhm.session.query(bhm.model).all()) == len_first + num_of_bhm_records bhm.session.query(bhm.model).filter_by(seq_num=seq_num, block_id=bm_instance.id).delete() bhm.model.update_from_main_model_instance(bm_instance, session=bhm.session, main_model_session=bm_session, seq_num=seq_num) assert len(bhm.session.query(bhm.model).all()) == len_first + num_of_bhm_records
def test_add_clear(): app = create_test_app() sm = AuxSessionManager(app=app) sn = 1 session = sm.get(sn) assert session.query(BlockModel).count() == 0 assert session.query(HeaderModel).count() == 0 assert session.query(TxModel).count() == 0 assert session.query(ParamModel).count() == 0 assert session.query(ErrorModel).count() == 0 block1 = Block(b64decode_hashes=True, from_detailed_dict=d3[0]) block1_m = BlockModel.update_from_block(block1, session=sm.get(sn)) block2 = Block(b64decode_hashes=True, from_detailed_dict=d3[1]) block2_m = BlockModel.update_from_block(block2, session=sm.get(sn)) assert isinstance(block1_m, BlockModel) assert isinstance(block2_m, BlockModel) assert block1_m != block2_m assert session.query(BlockModel).count() == 2 assert session.query(HeaderModel).count() == 2 assert session.query(TxModel).count() == 4 assert session.query(ParamModel).count() == 8 assert session.query(ErrorModel).count() == 0 block1_m.add_error(error="Test 1 Error", raw_error="Test 1 Raw Error") assert session.query(ErrorModel).count() == 1 block2_m.add_error(error="Test 2 Error", raw_error="Test 2 Raw Error") assert session.query(ErrorModel).count() == 2 BlockModel.clear(session=session) assert session.query(BlockModel).count() == 0 assert session.query(HeaderModel).count() == 0 assert session.query(ParamModel).count() == 0 assert session.query(TxModel).count() == 0 assert session.query(ErrorModel).count() == 0
def NOtest_add_fill_error(): app = create_test_app() sm = AuxSessionManager(app=app) sn = 1 len_first = BlockModel.query.with_session(sm.get(sn)).count() block_id = tuple(d3[0].keys())[0] BlockModel.add_fill_error(block_id, session=sm.get(sn)) len_after = BlockModel.query.with_session(sm.get(sn)).count()
def test_get_engine(): app = create_test_app() sm = AuxSessionManager(app=app) if len(app.config['AUX_DB_ENGINE_DISCOVERY_MAP']) == 0: try: sm.get_engine(1) except DbEngineMapIsEmptyError as e: print("AUX_DB_ENGINE_DISCOVERY_MAP is empty") return assert isinstance(sm.get_engine(1), Engine)
def test_table_manager_object(): app = create_test_app() models = [BlockModel, HeaderModel, TxModel, ParamModel] sm = AuxSessionManager(app=app) tm = TableManager(app=app, recreate_if_exists=True) tm.create_tables() for seq_num in range(1, get_num_of_backends(app) + 1): engine = sm.get_engine(seq_num) for model in models: assert engine.dialect.has_table(engine, model.__tablename__)
def create_tables_by_seq_nums(**kwargs): app = kwargs.get('app', create_test_app()) recreate_if_exists = kwargs.get('recreate_if_exists', True) _seq_nums = kwargs.get('seq_nums', seq_nums) _involved_models = kwargs.get('involved_models', involved_models) for seq_num in _seq_nums: asm = AuxSessionManager(app=app) create_tables(_involved_models, asm.get_engine(seq_num), recreate_if_exists=recreate_if_exists)
def test_fill_block(): app = create_test_app() sm = AuxSessionManager(app=app) seq_num = 1 session = sm.get(seq_num) BlockModel.clear(session=session) assert session.query(BlockModel).count() == 0 f = BlockFiller(app=app, seq_num=seq_num, recreate_tables_if_exist=False) block_id = 3 f.fill_block(block_id) assert session.query(BlockModel).count() == 1
def test_get_bind_name(): app = create_test_app() sm = AuxSessionManager(app=app) if len(app.config['AUX_DB_ENGINE_DISCOVERY_MAP']) == 0: try: sm.get_bind_name(1) except DbEngineMapIsEmptyError as e: print("AUX_DB_ENGINE_DISCOVERY_MAP is empty") return assert sm.get_bind_name(1) == tuple( app.config['AUX_DB_ENGINE_DISCOVERY_MAP'].keys())[0]
def test_member_object_creation(): app = create_test_app() update_aux_db_engine_discovery_map(app, force_update=True, aux_db_engine_name_prefix='test_aux_') asm = AuxSessionManager(app=app) seq_num = 1 engine = asm.get_engine(seq_num) session = asm.get(seq_num) create_tables(involved_models, engine) assert session.query(MemberModel).with_session( asm.get(seq_num)).count() == 0
def test_update_from_header(): app = create_test_app() new_map = update_aux_db_engine_discovery_map(app, force_update=True, aux_db_engine_name_prefix='test_aux_') sm = AuxSessionManager(app=app) sn = 1 session = sm.get(sn) hd = get_block_data_from_dict(d3[0])['header'] len_first = len(HeaderModel.query.with_session(session).all()) h = Header(b64decode_hashes=True, from_dict=hd) HeaderModel.update_from_header(h, session=session) assert len(HeaderModel.query.with_session(session).all()) == len_first + 1
def test_update_from_dict(): app = create_test_app() new_map = update_aux_db_engine_discovery_map(app, force_update=True, aux_db_engine_name_prefix='test_aux_') sm = AuxSessionManager(app=app) sn = 1 session = sm.get(sn) create_tables([BlockModel, HeaderModel], sm.get_engine(sn)) hd = get_block_data_from_dict(d3[0])['header'] len_first = len(HeaderModel.query.with_session(session).all()) HeaderModel.update_from_dict(hd, session=session) assert len(HeaderModel.query.with_session(session).all()) == len_first + 1
def test_fill_blocks__1(): app = create_test_app() sm = AuxSessionManager(app=app) seq_num = 1 session = sm.get(seq_num) f = BlockFiller(app=app, seq_num=seq_num, recreate_tables_if_exist=False, fetch_num_of_blocks=15) assert session.query(BlockModel).count() == 0 f.fill_blocks(1, 10) assert session.query(BlockModel).count() == 10
def NOtest_update_from_list_of_dicts(): app = create_test_app() new_map = update_aux_db_engine_discovery_map(app, force_update=True, aux_db_engine_name_prefix='test_aux_') sm = AuxSessionManager(app=app) sn = 1 session = sm.get(sn) td = get_block_data_from_dict(d3[1])['transactions'] len_first = len(TxModel.query.with_session(session).all()) TxModel.update_from_list_of_dicts(td, session=session) assert len(TxModel.query.with_session(session).all()) == len_first + len(td)
def NOtest_update_from_tx_set(): app = create_test_app() new_map = update_aux_db_engine_discovery_map(app, force_update=True, aux_db_engine_name_prefix='test_aux_') sm = AuxSessionManager(app=app) sn = 1 session = sm.get(sn) td = get_block_data_from_dict(d3[0])['transactions'] len_first = len(TxModel.query.with_session(session).all()) tx = TxSet(b64decode_hashes=True, from_list=td) TxModel.update_from_tx_set(tx, session=session) assert len(TxModel.query.with_session(session).all()) == len_first + tx.count
def test_fill_all_blocks(): app = create_test_app() sm = AuxSessionManager(app=app) seq_num = 1 session = sm.get(seq_num) BlockModel.clear(session=session) assert session.query(BlockModel).count() == 0 f = BlockFiller(app=app, seq_num=seq_num, recreate_tables_if_exist=False, fetch_num_of_blocks=10) f.fill_all_blocks() assert session.query(BlockModel).count() == get_max_block_id(seq_num)
def test_update_from_main_model_instance_via_manager(): app = create_test_app() mhm = MemberHelperManager(app=app, engine_echo=True) sm = AuxSessionManager(app=app) seq_num = 1 mm_session = sm.get(seq_num) mm_engine = sm.get_engine(seq_num) create_tables([MemberModel], mm_engine, recreate_if_exists=True) len_mm_first = len(MemberModel.query.with_session(mm_session).all()) assert len_mm_first == 0 d = { 'ecosystem_id': 1, 'key_id': -123, 'address': '456', 'pub': 'a9823432', 'amount': 13.1, 'maxpay': 10.2, 'multi': 1, 'deleted': 1, 'blocked': 1, } MemberModel.add_record_from_dict(d, session=mm_session) assert len( MemberModel.query.with_session(mm_session).all()) == len_mm_first + 1 mm_instance = MemberModel.query.with_session(mm_session).all()[0] len_first = len(mhm.session.query(mhm.model).all()) assert len_first == 0 mhm.model.update_from_main_model_instance(mm_instance, session=mhm.session, main_model_session=mm_session, seq_num=seq_num, ecosystem_id=1) num_of_mhm_records = len(MemberModel.__table__.columns) - len( MemberHelperModel.get_drop_fields()) assert len(mhm.session.query( mhm.model).all()) == len_first + num_of_mhm_records print("---------------ALL: %s" % mhm.session.query(mhm.model).filter_by( seq_num=seq_num, id=mm_instance.id).all()) mhm.session.query(mhm.model).filter_by(seq_num=seq_num, id=mm_instance.id).delete() mhm.model.update_from_main_model_instance(mm_instance, session=mhm.session, main_model_session=mm_session, seq_num=seq_num, ecosystem_id=1) print("len of mhm: %s" % len(mhm.session.query(mhm.model).all()))
def NOtest_get_bind_name(): app = create_test_app() new_map = update_aux_db_engine_discovery_map( app, force_update=True, aux_db_engine_name_prefix='test_aux_') sm = AuxSessionManager(app=app) sn = 1 if len(app.config['AUX_DB_ENGINE_DISCOVERY_MAP']) == 0: try: sm.get_bind_name(sn) except DbEngineMapIsEmptyError as e: print("AUX_DB_ENGINE_DISCOVERY_MAP is empty") return assert sm.get_bind_name(sn) == tuple( app.config['AUX_DB_ENGINE_DISCOVERY_MAP'].keys())[0]
def test_add(): app = create_test_app() new_map = update_aux_db_engine_discovery_map( app, force_update=True, aux_db_engine_name_prefix='test_aux_') sm = AuxSessionManager(app=app) seq_num = 1 session = sm.get(seq_num) create_tables([LogModel], sm.get_engine(seq_num)) assert len(LogModel.query.with_session(session=session).all()) == 0 LogModel.add(session=session, context='filler', caller='fill_block') assert len(LogModel.query.with_session(session=session).all()) == 1 LogModel.add(session=session, context='filler', caller='fill_block') assert len(LogModel.query.with_session(session=session).all()) == 2
def nontest_update_from_main_model_instance(): app = create_test_app() update_aux_db_engine_discovery_map(app, force_update=True, aux_db_engine_name_prefix='test_aux_') engine = get_aux_helpers_engine(app) metadata = MetaData() metadata.create_all(engine) session = get_aux_helpers_session(app) create_tables([TxHelperModel], engine, recreate_if_exists=True) session.query(TxHelperModel).all() sm = AuxSessionManager(app=app) seq_num = 1 bm_session = sm.get(seq_num) bm_engine = sm.get_engine(seq_num) create_tables([TxModel], bm_engine, recreate_if_exists=True) len_bm_first = len(TxModel.query.with_session(bm_session).all()) assert len_bm_first == 0 block = Block(b64decode_hashes=True, from_detailed_dict=d3[0]) BlockModel.update_from_block(block, session=bm_session) assert len( BlockModel.query.with_session(bm_session).all()) == len_bm_first + 1 bm_instance = BlockModel.query.with_session(bm_session).all()[0] len_first = len(BlockHelperModel.query.with_session(session).all()) assert len_first == 0 BlockHelperModel.update_from_block_model_instance( bm_instance, session=session, block_model_session=bm_session, seq_num=seq_num) num_of_bm_fields = len(BlockModel.__table__.columns) assert len( BlockHelperModel.query.with_session(session).filter_by( seq_num=seq_num).all()) == len_first + num_of_bm_fields - 1 BlockHelperModel.query.with_session(session).filter_by( seq_num=seq_num, block_id=bm_instance.id).delete() BlockHelperModel.update_from_block_model_instance( bm_instance, session=session, block_model_session=bm_session, seq_num=seq_num) assert len( BlockHelperModel.query.with_session(session).filter_by( seq_num=seq_num).all()) == len_first + num_of_bm_fields - 1
def NOtest_add_delete(): app = create_test_app() sm = AuxSessionManager(app=app) sn = 1 session = sm.get(sn) td = get_block_data_from_dict(d3[0])['transactions'][0] assert TxModel.query.with_session(session).count() == 0 assert ParamModel.query.with_session(session).count() == 0 tx_m = TxModel.update_from_dict(td, session=session) assert isinstance(tx_m, TxModel) assert TxModel.query.with_session(session).count() == 1 assert ParamModel.query.with_session(session).count() == 2 tx_m.delete(session=session) assert TxModel.query.with_session(session).count() == 0 assert ParamModel.query.with_session(session).count() == 0
def test_get_latest_lock(): app = create_test_app() new_map = update_aux_db_engine_discovery_map( app, force_update=True, aux_db_engine_name_prefix='test_aux_') sm = AuxSessionManager(app=app) seq_num = 1 session = sm.get(seq_num) create_tables([LockModel], sm.get_engine(seq_num)) assert LockModel.is_locked(session=session) == False LockModel.lock(session=session) assert LockModel.is_locked(session=session) == True q = LockModel.get_latest_lock(session=session) assert isinstance(q, LockModel) LockModel.unlock(session=session) q = LockModel.get_latest_lock(session=session) assert q is None
def test_clear_garbage(): app = create_test_app() new_map = update_aux_db_engine_discovery_map( app, force_update=True, aux_db_engine_name_prefix='test_aux_') sm = AuxSessionManager(app=app) seq_num = 1 session = sm.get(seq_num) create_tables([LockModel], sm.get_engine(seq_num)) assert LockModel.is_locked(session=session) == False to = 1 fake_pid = get_fake_pid() LockModel.lock(session=session) time.sleep(to + 1) LockModel.lock(session=session) LockModel.lock(session=session, process_id=fake_pid) assert LockModel.is_locked(session=session) == True assert session.query(LockModel).count() == 3 qs = session.query(LockModel).all() qs_pids = [q.process_id for q in qs] assert fake_pid in qs_pids q = LockModel.get_latest_lock(session=session) assert isinstance(q, LockModel) assert session.query(LockModel).count() == 3 zqs = LockModel.get_zombie_locks(session=session) zqs_pids = [q.process_id for q in zqs] assert len(zqs_pids) == 1 assert fake_pid in zqs_pids eqs = LockModel.get_expired_locks(session=session, timeout_secs=to) assert len(eqs) == 1 LockModel.clear_garbage(session=session, timeout_secs=to) zqs = LockModel.get_zombie_locks(session=session) zqs_pids = [q.process_id for q in zqs] assert len(zqs_pids) == 0 eqs = LockModel.get_expired_locks(session=session, timeout_secs=to) assert len(eqs) == 0 LockModel.unlock(session=session) assert session.query(LockModel).count() == 0
def test_add_clear(): app = create_test_app() sm = AuxSessionManager(app=app) sn = 1 session = sm.get(sn) td1 = get_block_data_from_dict(d3[0])['transactions'][0] td2 = get_block_data_from_dict(d3[1])['transactions'][0] assert TxModel.query.with_session(session).count() == 0 assert ParamModel.query.with_session(session).count() == 0 tx1_m = TxModel.update_from_dict(td1, session=session) tx2_m = TxModel.update_from_dict(td2, session=session) assert isinstance(tx1_m, TxModel) assert isinstance(tx2_m, TxModel) assert TxModel.query.with_session(session).count() == 2 assert ParamModel.query.with_session(session).count() == 4 TxModel.clear(session=session) assert TxModel.query.with_session(session).count() == 0 assert ParamModel.query.with_session(session).count() == 0