def test_prepare_contract_data_with_exception( self, mocked_loop, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db, mocked_logger, mocked_gevent): static_number = 12 true_list = [True for i in xrange(0, static_number)] true_list.append(False) mocked_loop.__nonzero__.side_effect = true_list cb = ContractingDataBridge({'main': {}}) for i in range(static_number): cb.handicap_contracts_queue.put({'id': i, 'tender_id': i + 1111}) tender_data = MagicMock() tender_data.data = {'no_owner': '', 'no_tender_token': ''} cb.get_tender_credentials = MagicMock(return_value=tender_data) cb.prepare_contract_data() list_calls = mocked_gevent.sleep.call_args_list calls_logs = mocked_logger.info.call_args_list calls_with_error_delay = call(cb.on_error_delay) self.assertEqual( self._get_calls_count(list_calls, calls_with_error_delay), static_number) reconnecting_log = call('Reconnecting tenders client', extra={ 'JOURNAL_TENDER_ID': 1120, 'MESSAGE_ID': 'c_bridge_reconnect', 'JOURNAL_CONTRACT_ID': 9 }) self.assertEqual(self._get_calls_count(calls_logs, reconnecting_log), 1)
def test_run_with_Exception(self, mocked_loop, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db, mocked_logger, mocked_gevent): cb = ContractingDataBridge({'main': {}}) true_list = [True for i in xrange(0, 21)] true_list.append(False) mocked_loop.__nonzero__.side_effect = true_list e = Exception('Error!') cb._restart_synchronization_workers = MagicMock(side_effect=e) cb.run() mocked_logger.exception.assert_called_once_with(e)
def test_prepare_contract_data_retry_with_exception( self, mocked_loop, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db, mocked_logger, mocked_gevent): true_list = [True, False] mocked_loop.__nonzero__.side_effect = true_list cb = ContractingDataBridge({'main': {}}) contract = {'id': 0, 'tender_id': 1111} cb.handicap_contracts_queue_retry.get = MagicMock( return_value=contract) e = Exception("Error!!! prepare_contract_data_retry") cb.get_tender_data_with_retry = MagicMock(side_effect=e) cb.prepare_contract_data_retry() mocked_logger.exception.assert_called_with(e)
def test_run_with_KeyboardInterrupt(self, mocked_loop, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db, mocked_logger, mocked_gevent): true_list = [True for i in xrange(0, 21)] true_list.append(False) mocked_loop.__nonzero__.side_effect = true_list cb = ContractingDataBridge({'main': {}}) cb._restart_synchronization_workers = MagicMock( side_effect=KeyboardInterrupt) cb.run() gevent_calls = mocked_gevent.killall.call_args_list logger_calls = mocked_logger.info.call_args_list keyboard_interrut_log = call('Exiting...') kill_all_jobs = call(cb.jobs, timeout=5) kill_all_immortal_jobs = call(cb.immortal_jobs, timeout=5) self.assertEqual( self._get_calls_count(logger_calls, keyboard_interrut_log), 1) self.assertEqual(self._get_calls_count(gevent_calls, kill_all_jobs), 1) self.assertEqual( self._get_calls_count(gevent_calls, kill_all_immortal_jobs), 1) cb._start_contract_sculptors = MagicMock(side_effect=KeyboardInterrupt) with self.assertRaises(KeyboardInterrupt) as e: cb.run() isinstance(e.exception, exceptions.KeyboardInterrupt)
def test_prepare_contract_data(self, mocked_loop, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db, mocked_logger, mocked_gevent): true_list = [True, False] mocked_loop.__nonzero__.side_effect = true_list cb = ContractingDataBridge({'main': {}}) contract = {'id': 0, 'tender_id': 1111} tender_data = MagicMock() tender_data.data = {'owner': 'owner', 'tender_token': 'tender_token'} cb.handicap_contracts_queue.get = MagicMock(return_value=contract) cb.get_tender_credentials = MagicMock(return_value=tender_data) cb.prepare_contract_data() self.assertEquals(cb.contracts_put_queue.qsize(), 1) self.assertEquals(cb.contracts_put_queue.get(), contract)
def test_put_tender_in_cache_by_contract(self, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db): cb = ContractingDataBridge({'main': {}}) tender_id = '2001' cb.basket = {'1': 'one', '2': 'two', '42': 'why'} cb.cache_db = MagicMock() cb._put_tender_in_cache_by_contract({'id': '1984'}, tender_id) self.assertEqual(cb.basket.get('42', None), 'why') self.assertEqual(cb.cache_db.put.called, False) cb._put_tender_in_cache_by_contract({'id': '42'}, tender_id) self.assertEqual(cb.basket.get('42', None), None) cb.cache_db.put.assert_called_once_with('2001', 'why')
def test_get_tender_contracts_resource_gone(self, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db, mocked_logger, mocked_gevent): error_msg = { "status": "error", "errors": [{ "location": "url", "name": "contract_id", "description": "Archived" }] } resp = MagicMock() resp.body_string.return_value = json.dumps(error_msg) resp.text = json.dumps(error_msg) resp.status_code = 410 resp.status_int = 410 cb = ContractingDataBridge({'main': {}}) tender_to_sync = { 'id': '1' * 32, 'dateModified': datetime.now().isoformat() } tender = { "data": { "id": "1" * 32, "dateModified": tender_to_sync['dateModified'], "contracts": [{ "id": "2" * 32, "status": "active" }] } } cb.tenders_queue.put(tender_to_sync) cb.cache_db = MagicMock() cb.cache_db.has.return_value = False cb.tenders_sync_client = MagicMock() cb.tenders_sync_client.get_tender.return_value = tender exception = ResourceGone(response=resp) cb.contracting_client_ro = MagicMock() cb.contracting_client_ro.get_contract.side_effect = [exception] cb._get_tender_contracts() logger_msg = 'Sync contract {} of tender {} has been archived'.format( "2" * 32, "1" * 32) extra = { 'JOURNAL_TENDER_ID': '1' * 32, 'MESSAGE_ID': 'c_bridge_contract_to_sync', 'JOURNAL_CONTRACT_ID': '2' * 32 } mocked_logger.info.assert_has_calls([call(logger_msg, extra=extra)])
def test_get_tender_credentials(self, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db): cb = ContractingDataBridge({'main': {}}) cb.client = MagicMock() tender_id = '42' cb.client.extract_credentials.side_effect = (Exception(), Exception(), Exception(), tender_id) with self.assertRaises(Exception): cb.get_tender_credentials(tender_id) extract_credentials_calls = cb.client.extract_credentials.call_args_list self.assertEqual( self._get_calls_count(extract_credentials_calls, call(tender_id)), 3) self.assertEqual(len(extract_credentials_calls), 3) cb.client = MagicMock() cb.client.extract_credentials.return_value = tender_id data = cb.get_tender_credentials(tender_id) self.assertEqual(data, tender_id) cb.client.extract_credentials.assert_called_once_with(tender_id) cb.client = MagicMock() cb.client.extract_credentials.side_effect = (Exception('Boom!'), Exception('Boom!'), tender_id) data = cb.get_tender_credentials(tender_id) extract_credentials_calls = cb.client.extract_credentials.call_args_list self.assertEqual( self._get_calls_count(extract_credentials_calls, call(tender_id)), 3) self.assertEqual(len(extract_credentials_calls), 3) self.assertEqual(data, tender_id)
def test_sync_single_tender_Exception(self, mocked_loop, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db, mocked_logger, mocked_gevent): cb = ContractingDataBridge({'main': {}}) tender_id = 33 cb.tenders_sync_client.get_tender = MagicMock( return_value={ 'data': { 'contracts': [munchify({ 'status': 'active', 'id': 1 })], 'id': 2, 'status': 'active', 'procuringEntity': 'procuringEntity', 'owner': 'owner', 'tender_token': 'tender_token' } }) cb.get_tender_credentials = MagicMock( return_value={ 'data': { 'procuringEntity': 'procuringEntity', 'tender_token': 'tender_token' } }) cb.sync_single_tender(tender_id) calls_logs = mocked_logger.info.call_args_list self.assertEqual( self._get_calls_count(calls_logs, call('Contract exists 1')), 1) error = Exception('Error!') cb.contracting_client.get_contract = MagicMock(side_effect=error) with self.assertRaises(Exception) as e: cb.sync_single_tender(tender_id) mocked_logger.exception.assert_called_once_with(e.exception)
def test_restart_synchronization_workers(self, mocked_logger, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db): cb = ContractingDataBridge({'main': {}}) cb.clients_initialize = MagicMock() job_1 = MagicMock() job_2 = MagicMock() jobs_list = [job_1, job_2] cb.jobs = jobs_list cb._start_synchronization_workers = MagicMock() cb._restart_synchronization_workers() self.assertEqual(job_1.kill.call_count, 1) self.assertEqual(job_2.kill.call_count, 1) cb._start_synchronization_workers.assert_called_once_with() cb.clients_initialize.assert_called_once_with() mocked_logger.warn.assert_called_once_with( 'Restarting synchronization', extra={'MESSAGE_ID': 'c_bridge_restart'})
def test_put_contracts(self, mocked_loop, mocked_gevent, mocked_logger, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db): list_loop = [True, False] mocked_loop.__nonzero__.side_effect = list_loop contract = munch.munchify({'id': '42', 'tender_id': '1984'}) bridge = ContractingDataBridge({'main': {}}) bridge.contracts_put_queue = MagicMock() bridge.contracts_put_queue.get.return_value = contract bridge.contracting_client = MagicMock() bridge.contracts_retry_put_queue = MagicMock() bridge.contracting_client_init = MagicMock() bridge.cache_db = MagicMock() bridge._put_tender_in_cache_by_contract = MagicMock() bridge.put_contracts() bridge.contracts_put_queue.get.assert_called_once_with() bridge.contracting_client.create_contract.assert_called_once_with( {'data': contract.toDict()}) bridge.cache_db.put.assert_called_once_with(contract.id, True) bridge._put_tender_in_cache_by_contract.assert_called_once_with( contract.toDict(), contract.tender_id) mocked_gevent.sleep.assert_called_once_with(0) list_contracts = [] for i in range(0, 10): list_contracts.append(dict(id=i, tender_id=(i + 100))) bridge.contracts_put_queue = MagicMock() bridge.contracts_put_queue.get.side_effect = list_contracts list_loop = [True for i in range(0, 10)] list_loop.append(False) mocked_loop.__nonzero__.side_effect = list_loop bridge.put_contracts() extract_calls = [ data[0] for data, call in bridge.contracts_retry_put_queue.put.call_args_list ] for i in range(0, 10): assert extract_calls[i]['id'] == i self.assertEqual(len(extract_calls), 10) bridge.contracting_client_init.assert_called_once_with()
def test_retry_put_contracts(self, mocked_loop, mocked_gevent, mocked_logger, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db): true_list = [True, False] mocked_loop.__nonzero__.side_effect = true_list contract = {'id': '42', 'tender_id': '1984'} bridge = ContractingDataBridge({'main': {}}) remember_put_with_retry = bridge._put_with_retry bridge.contracts_retry_put_queue = MagicMock() bridge._put_with_retry = MagicMock() bridge.cache_db = MagicMock() bridge._put_tender_in_cache_by_contract = MagicMock() bridge.contracts_retry_put_queue.get.return_value = contract bridge.retry_put_contracts() bridge.contracts_retry_put_queue.get.assert_called_once_with() bridge._put_with_retry.assert_called_once_with(contract) bridge.cache_db.put.assert_called_once_with(contract['id'], True) bridge._put_tender_in_cache_by_contract.assert_called_once_with( contract, contract['tender_id']) mocked_gevent.sleep.assert_called_once_with(0) bridge._put_with_retry = remember_put_with_retry mocked_loop.__nonzero__.side_effect = true_list e = Exception('Boom!') bridge.contracting_client.create_contract = MagicMock( side_effect=[e, True]) contract = munch.munchify(contract) bridge.contracts_retry_put_queue.get.return_value = contract bridge.retry_put_contracts() mocked_logger.exception.assert_called_once_with(e)
def test_sync_single_tender(self, mocked_loop, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db, mocked_logger, mocked_gevent): cb = ContractingDataBridge({'main': {}}) tender_id = 33 cb.tenders_sync_client.get_tender = MagicMock( return_value={ 'data': { 'contracts': [munchify({ 'status': 'no_active', 'id': 1 })], 'id': 2, 'status': 'active', 'procuringEntity': 'procuringEntity', 'owner': 'owner', 'tender_token': 'tender_token' } }) cb.sync_single_tender(tender_id) calls_logs = mocked_logger.info.call_args_list self.assertEqual( self._get_calls_count(calls_logs, call("Skip contract 1 in status no_active")), 1) self.assertEqual( self._get_calls_count( calls_logs, call("Tender 33 does not contain contracts to transfer")), 1) cb.tenders_sync_client.get_tender = MagicMock( return_value={ 'data': { 'contracts': [munchify({ 'status': 'active', 'id': 1 })], 'id': 2, 'status': 'active', 'procuringEntity': 'procuringEntity', 'owner': 'owner', 'tender_token': 'tender_token' } }) cb.get_tender_credentials = MagicMock( return_value={ 'data': { 'procuringEntity': 'procuringEntity', 'tender_token': 'tender_token' } }) cb.contracting_client.get_contract = MagicMock( side_effect=ResourceNotFound) cb.contracting_client.create_contract = MagicMock( return_value={'data': ['test1', 'test2']}) cb.sync_single_tender(tender_id) calls_logs = mocked_logger.info.call_args_list self.assertEqual( self._get_calls_count(calls_logs, call("Getting tender {}".format(tender_id))), 2) self.assertEqual( self._get_calls_count(calls_logs, call("Got tender 2 in status active")), 2) self.assertEqual( self._get_calls_count(calls_logs, call('Getting tender 33 credentials')), 2) self.assertEqual( self._get_calls_count(calls_logs, call('Got tender 33 credentials')), 2) self.assertEqual( self._get_calls_count( calls_logs, call("Checking if contract 1 already exists")), 1) self.assertEqual( self._get_calls_count( calls_logs, call( 'Contract 1 does not exists. Prepare contract for creation.' )), 1) self.assertEqual( self._get_calls_count( calls_logs, call('Extending contract 1 with extra data')), 1) self.assertEqual( self._get_calls_count(calls_logs, call('Creating contract 1')), 1) self.assertEqual( self._get_calls_count(calls_logs, call('Contract 1 created')), 1) self.assertEqual( self._get_calls_count( calls_logs, call('Successfully transfered contracts: [1]')), 1)
def test_run_with_dead_all_jobs_and_workers(self, mocked_loop, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db, mocked_logger, mocked_gevent): # Prepare context true_list = [True for i in xrange(0, 21)] true_list.append(False) mocked_loop.__nonzero__.side_effect = true_list mocked_db()._backend = 'redis' mocked_db()._db_name = 'cache_db_name' mocked_db()._port = 6379 mocked_db()._host = 'localhost' cb = ContractingDataBridge({'main': {}}) # Check initialization msg = "Caching backend: '{}', db name: '{}', host: '{}', " \ "port: '{}'".format( cb.cache_db._backend, cb.cache_db._db_name, cb.cache_db._host, cb.cache_db._port ) calls = [ call(msg, extra={'MESSAGE_ID': DATABRIDGE_INFO}), call('Initialization contracting clients.', extra={'MESSAGE_ID': DATABRIDGE_INFO}) ] mocked_logger.info.assert_has_calls(calls) # Run: while loop has 22 iterations with dead workers and jobs cb.run() logger_calls = mocked_logger.info.call_args_list start_bridge = call('Start Contracting Data Bridge', extra={'MESSAGE_ID': 'c_bridge_start'}) current_stage = call( 'Current state: Tenders to process 0; Unhandled contracts 0; ' 'Contracts to create 0; Retrying to create 0', extra={ 'handicap_contracts_queue_size': 0, 'contracts_queue_size': 0, 'tenders_queue_size': 0, 'contracts_retry_queue': 0 }) starting_sync_workers = call( 'Starting forward and backward sync workers') init_clients = call('Initialization contracting clients.', extra={'MESSAGE_ID': 'c_bridge_info'}) self.assertEqual(self._get_calls_count(logger_calls, start_bridge), 1) self.assertEqual(self._get_calls_count(logger_calls, current_stage), 1) self.assertEqual( self._get_calls_count(logger_calls, starting_sync_workers), 22) self.assertEqual(self._get_calls_count(logger_calls, init_clients), 43) warn_calls = mocked_logger.warn.call_args_list restart_sync = call('Restarting synchronization', extra={'MESSAGE_ID': 'c_bridge_restart'}) restart_tender_worker = call('Restarting get_tender_contracts worker') restart_retry_worker = call('Restarting retry_put_contracts worker') restart_prepare_data = call('Restarting prepare_contract_data worker') restart_put = call('Restarting put_contracts worker') restart_prepare_data_retry = call( 'Restarting prepare_contract_data_retry worker') self.assertEqual(self._get_calls_count(warn_calls, restart_sync), 21) self.assertEqual( self._get_calls_count(warn_calls, restart_tender_worker), 21) self.assertEqual( self._get_calls_count(warn_calls, restart_retry_worker), 21) self.assertEqual( self._get_calls_count(warn_calls, restart_prepare_data), 21) self.assertEqual(self._get_calls_count(warn_calls, restart_put), 21) self.assertEqual( self._get_calls_count(warn_calls, restart_prepare_data_retry), 21) # TODO: calculate spawn calls with different args spawn_calls = mocked_gevent.spawn.call_args_list self.assertEqual(len(spawn_calls), 154) self.assertEqual( self._get_calls_count(spawn_calls, call(cb.get_tender_contracts)), 22) self.assertEqual( self._get_calls_count(spawn_calls, call(cb.prepare_contract_data)), 22) self.assertEqual( self._get_calls_count(spawn_calls, call(cb.prepare_contract_data_retry)), 22) self.assertEqual( self._get_calls_count(spawn_calls, call(cb.put_contracts)), 22) self.assertEqual( self._get_calls_count(spawn_calls, call(cb.retry_put_contracts)), 22) self.assertEqual( self._get_calls_count(spawn_calls, call(cb.get_tender_contracts_backward)), 22) self.assertEqual( self._get_calls_count(spawn_calls, call(cb.get_tender_contracts_forward)), 22)
def test_run_with_all_jobs_and_workers(self, mocked_loop, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db, mocked_logger, mocked_gevent): cb = ContractingDataBridge({'main': {}}) true_list = [True, False] mocked_loop.__nonzero__.side_effect = true_list def _start_conrtact_sculptors(cb): get_tender_contracts = MagicMock() prepare_contract_data = MagicMock() prepare_contract_data_retry = MagicMock() put_contracts = MagicMock() retry_put_contracts = MagicMock() get_tender_contracts.dead = False prepare_contract_data.dead = False prepare_contract_data_retry.dead = False put_contracts.dead = False retry_put_contracts.dead = False cb.immortal_jobs = { 'get_tender_contracts': get_tender_contracts, 'prepare_contract_data': prepare_contract_data, 'prepare_contract_data_retry': prepare_contract_data_retry, 'put_contracts': put_contracts, 'retry_put_contracts': retry_put_contracts, } def _start_synchronization_workers(cb): get_tender_contracts_backward = MagicMock() get_tender_contracts_forward = MagicMock() get_tender_contracts_backward.dead = False get_tender_contracts_forward.dead = False cb.jobs = [ get_tender_contracts_backward, get_tender_contracts_forward ] cb._start_contract_sculptors = MagicMock( side_effect=_start_conrtact_sculptors(cb)) cb._start_synchronization_workers = MagicMock( side_effect=_start_synchronization_workers(cb)) _restart_synchronization_workers = MagicMock() cb._restart_synchronization_workers = _restart_synchronization_workers cb.run() logger_calls = mocked_logger.info.call_args_list first_log = call( "Caching backend: '{}', db name: '{}', host: '{}', port: '{}'". format(cb.cache_db._backend, cb.cache_db._db_name, cb.cache_db._host, cb.cache_db._port), extra={"MESSAGE_ID": DATABRIDGE_INFO}) second_log = call('Initialization contracting clients.', extra={"MESSAGE_ID": DATABRIDGE_INFO}) thread_log = call('Start Contracting Data Bridge', extra=({ 'MESSAGE_ID': DATABRIDGE_START })) self.assertEqual(mocked_logger.info.call_count, 3) self.assertEqual(self._get_calls_count(logger_calls, first_log), 1) self.assertEqual(self._get_calls_count(logger_calls, second_log), 1) self.assertEqual(self._get_calls_count(logger_calls, thread_log), 1) self.assertEqual(mocked_logger.warn.call_count, 0) self.assertEqual(mocked_gevent.spawn.call_count, 0) self.assertEqual(cb._restart_synchronization_workers.call_count, 0)
def test_run_with_all_jobs_and_workers(self, mocked_loop, mocked_contract_client, mocked_tender_client, mocked_sync_client, mocked_db, mocked_logger, mocked_gevent): cb = ContractingDataBridge({'main': {}})