def setUp(self):
self.tender_id = uuid.uuid4().hex
self.award_id = uuid.uuid4().hex
self.qualification_id = uuid.uuid4().hex
self.document_id = generate_doc_id()
self.process_tracker = ProcessTracker(db=MagicMock())
self.process_tracker.set_item(self.tender_id, self.award_id, 1)
self.upload_to_tender_queue = Queue(10)
self.url = 'http://127.0.0.1:20604'
self.sleep_change_value = APIRateController()
self.data = Data(self.tender_id, self.award_id, '123', 'awards', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.qualification_data = Data(self.tender_id, self.qualification_id,
'123', 'qualifications', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.client = MagicMock()
self.worker = UploadFileToTender(self.client,
self.upload_to_tender_queue,
self.process_tracker, MagicMock(),
self.sleep_change_value)
self.worker.retry_upload_to_tender_queue = Queue(10)
def setUp(self):
self.tender_id = uuid.uuid4().hex
self.award_id = uuid.uuid4().hex
self.qualification_id = uuid.uuid4().hex
self.document_id = generate_doc_id()
self.process_tracker = ProcessTracker(db=MagicMock())
self.process_tracker.set_item(self.tender_id, self.award_id, 1)
self.upload_to_doc_service_queue = Queue(10)
self.upload_to_tender_queue = Queue(10)
self.sleep_change_value = APIRateController()
self.sna = event.Event()
self.sna.set()
self.data = Data(self.tender_id, self.award_id, '123', 'awards', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.qualification_data = Data(self.tender_id, self.qualification_id,
'123', 'qualifications', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.doc_service_client = DocServiceClient(host='127.0.0.1',
port='80',
user='',
password='')
self.worker = UploadFileToDocService(self.upload_to_doc_service_queue,
self.upload_to_tender_queue,
self.process_tracker,
self.doc_service_client, self.sna,
self.sleep_change_value)
self.url = '{url}'.format(url=self.doc_service_client.url)
def setUp(self):
self.filtered_tender_ids_queue = Queue(10)
self.edrpou_codes_queue = Queue(10)
self.process_tracker = ProcessTracker(Db(config))
self.tender_id = uuid.uuid4().hex
self.sleep_change_value = APIRateController()
self.worker = EdrDataBridge(config)
def setUp(self):
self.process_tracker = ProcessTracker(
MagicMock(has=MagicMock(return_value=False)))
self.tenders_id = [uuid.uuid4().hex for _ in range(4)]
self.sleep_change_value = APIRateController()
self.client = MagicMock()
self.tender_queue = Queue(10)
self.sna = event.Event()
self.sna.set()
self.worker = Scanner.spawn(self.client, self.tender_queue, self.sna,
self.process_tracker,
self.sleep_change_value)
def setUp(self):
self.filtered_tender_ids_queue = Queue(10)
self.edrpou_codes_queue = Queue(10)
self.db = Db(config)
self.process_tracker = ProcessTracker(self.db)
self.tender_id = uuid.uuid4().hex
self.filtered_tender_ids_queue.put(self.tender_id)
self.sleep_change_value = APIRateController()
self.client = MagicMock()
self.sna = event.Event()
self.sna.set()
self.worker = FilterTenders.spawn(self.client, self.filtered_tender_ids_queue, self.edrpou_codes_queue,
self.process_tracker, self.sna, self.sleep_change_value)
self.bid_ids = [uuid.uuid4().hex for _ in range(5)]
self.qualification_ids = [uuid.uuid4().hex for _ in range(5)]
self.award_ids = [uuid.uuid4().hex for _ in range(5)]
self.request_ids = [generate_request_id() for _ in range(2)]
self.response = ResponseMock({'X-Request-ID': self.request_ids[0]},
munchify({'prev_page': {'offset': '123'},
'next_page': {'offset': '1234'},
'data': {'status': "active.pre-qualification",
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'awards': [self.awards(0, 0, 'pending', CODES[0])]}}))
def __init__(self, config):
super(EdrDataBridge, self).__init__()
self.config = config
api_server = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
ro_api_server = self.config_get(
'public_tenders_api_server') or api_server
buffers_size = self.config_get('buffers_size') or 500
self.delay = self.config_get('delay') or 15
self.increment_step = self.config_get('increment_step') or 1
self.decrement_step = self.config_get('decrement_step') or 1
self.sleep_change_value = APIRateController(self.increment_step,
self.decrement_step)
self.doc_service_host = self.config_get('doc_service_server')
self.doc_service_port = self.config_get('doc_service_port') or 6555
self.sandbox_mode = os.environ.get('SANDBOX_MODE', 'False')
self.time_to_live = self.config_get('time_to_live') or 300
# init clients
self.tenders_sync_client = TendersClientSync(
'', host_url=ro_api_server, api_version=self.api_version)
self.client = TendersClient(self.config_get('api_token'),
host_url=api_server,
api_version=self.api_version)
self.proxy_client = ProxyClient(
host=self.config_get('proxy_server'),
user=self.config_get('proxy_user'),
password=self.config_get('proxy_password'),
port=self.config_get('proxy_port'),
version=self.config_get('proxy_version'))
self.doc_service_client = DocServiceClient(
host=self.doc_service_host,
port=self.doc_service_port,
user=self.config_get('doc_service_user'),
password=self.config_get('doc_service_password'))
# init queues for workers
self.filtered_tender_ids_queue = Queue(
maxsize=buffers_size
) # queue of tender IDs with appropriate status
self.edrpou_codes_queue = Queue(
maxsize=buffers_size
) # queue with edrpou codes (Data objects stored in it)
self.upload_to_doc_service_queue = Queue(
maxsize=buffers_size
) # queue with info from EDR (Data.file_content)
self.upload_to_tender_queue = Queue(maxsize=buffers_size)
# blockers
self.initialization_event = event.Event()
self.services_not_available = event.Event()
self.services_not_available.set()
self.db = Db(config)
self.process_tracker = ProcessTracker(self.db, self.time_to_live)
unprocessed_items = self.process_tracker.get_unprocessed_items()
for item in unprocessed_items:
self.upload_to_doc_service_queue.put(loads(item))
# Workers
self.scanner = partial(
Scanner.spawn,
tenders_sync_client=self.tenders_sync_client,
filtered_tender_ids_queue=self.filtered_tender_ids_queue,
services_not_available=self.services_not_available,
process_tracker=self.process_tracker,
sleep_change_value=self.sleep_change_value,
delay=self.delay)
self.filter_tender = partial(
FilterTenders.spawn,
tenders_sync_client=self.tenders_sync_client,
filtered_tender_ids_queue=self.filtered_tender_ids_queue,
edrpou_codes_queue=self.edrpou_codes_queue,
process_tracker=self.process_tracker,
services_not_available=self.services_not_available,
sleep_change_value=self.sleep_change_value,
delay=self.delay)
self.edr_handler = partial(
EdrHandler.spawn,
proxy_client=self.proxy_client,
edrpou_codes_queue=self.edrpou_codes_queue,
upload_to_doc_service_queue=self.upload_to_doc_service_queue,
process_tracker=self.process_tracker,
services_not_available=self.services_not_available,
delay=self.delay)
self.upload_file_to_doc_service = partial(
UploadFileToDocService.spawn,
upload_to_doc_service_queue=self.upload_to_doc_service_queue,
upload_to_tender_queue=self.upload_to_tender_queue,
process_tracker=self.process_tracker,
doc_service_client=self.doc_service_client,
services_not_available=self.services_not_available,
sleep_change_value=self.sleep_change_value,
delay=self.delay)
self.upload_file_to_tender = partial(
UploadFileToTender.spawn,
client=self.client,
upload_to_tender_queue=self.upload_to_tender_queue,
process_tracker=self.process_tracker,
services_not_available=self.services_not_available,
sleep_change_value=self.sleep_change_value,
delay=self.delay)
class EdrDataBridge(object):
""" Edr API Data Bridge """
def __init__(self, config):
super(EdrDataBridge, self).__init__()
self.config = config
api_server = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
ro_api_server = self.config_get(
'public_tenders_api_server') or api_server
buffers_size = self.config_get('buffers_size') or 500
self.delay = self.config_get('delay') or 15
self.increment_step = self.config_get('increment_step') or 1
self.decrement_step = self.config_get('decrement_step') or 1
self.sleep_change_value = APIRateController(self.increment_step,
self.decrement_step)
self.doc_service_host = self.config_get('doc_service_server')
self.doc_service_port = self.config_get('doc_service_port') or 6555
self.sandbox_mode = os.environ.get('SANDBOX_MODE', 'False')
self.time_to_live = self.config_get('time_to_live') or 300
# init clients
self.tenders_sync_client = TendersClientSync(
'', host_url=ro_api_server, api_version=self.api_version)
self.client = TendersClient(self.config_get('api_token'),
host_url=api_server,
api_version=self.api_version)
self.proxy_client = ProxyClient(
host=self.config_get('proxy_server'),
user=self.config_get('proxy_user'),
password=self.config_get('proxy_password'),
port=self.config_get('proxy_port'),
version=self.config_get('proxy_version'))
self.doc_service_client = DocServiceClient(
host=self.doc_service_host,
port=self.doc_service_port,
user=self.config_get('doc_service_user'),
password=self.config_get('doc_service_password'))
# init queues for workers
self.filtered_tender_ids_queue = Queue(
maxsize=buffers_size
) # queue of tender IDs with appropriate status
self.edrpou_codes_queue = Queue(
maxsize=buffers_size
) # queue with edrpou codes (Data objects stored in it)
self.upload_to_doc_service_queue = Queue(
maxsize=buffers_size
) # queue with info from EDR (Data.file_content)
self.upload_to_tender_queue = Queue(maxsize=buffers_size)
# blockers
self.initialization_event = event.Event()
self.services_not_available = event.Event()
self.services_not_available.set()
self.db = Db(config)
self.process_tracker = ProcessTracker(self.db, self.time_to_live)
unprocessed_items = self.process_tracker.get_unprocessed_items()
for item in unprocessed_items:
self.upload_to_doc_service_queue.put(loads(item))
# Workers
self.scanner = partial(
Scanner.spawn,
tenders_sync_client=self.tenders_sync_client,
filtered_tender_ids_queue=self.filtered_tender_ids_queue,
services_not_available=self.services_not_available,
process_tracker=self.process_tracker,
sleep_change_value=self.sleep_change_value,
delay=self.delay)
self.filter_tender = partial(
FilterTenders.spawn,
tenders_sync_client=self.tenders_sync_client,
filtered_tender_ids_queue=self.filtered_tender_ids_queue,
edrpou_codes_queue=self.edrpou_codes_queue,
process_tracker=self.process_tracker,
services_not_available=self.services_not_available,
sleep_change_value=self.sleep_change_value,
delay=self.delay)
self.edr_handler = partial(
EdrHandler.spawn,
proxy_client=self.proxy_client,
edrpou_codes_queue=self.edrpou_codes_queue,
upload_to_doc_service_queue=self.upload_to_doc_service_queue,
process_tracker=self.process_tracker,
services_not_available=self.services_not_available,
delay=self.delay)
self.upload_file_to_doc_service = partial(
UploadFileToDocService.spawn,
upload_to_doc_service_queue=self.upload_to_doc_service_queue,
upload_to_tender_queue=self.upload_to_tender_queue,
process_tracker=self.process_tracker,
doc_service_client=self.doc_service_client,
services_not_available=self.services_not_available,
sleep_change_value=self.sleep_change_value,
delay=self.delay)
self.upload_file_to_tender = partial(
UploadFileToTender.spawn,
client=self.client,
upload_to_tender_queue=self.upload_to_tender_queue,
process_tracker=self.process_tracker,
services_not_available=self.services_not_available,
sleep_change_value=self.sleep_change_value,
delay=self.delay)
def config_get(self, name):
return self.config.get('main').get(name)
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=retry_mult)
def check_doc_service(self):
try:
request("{host}:{port}/".format(host=self.doc_service_host,
port=self.doc_service_port))
except RequestError as e:
logger.info('DocService connection error, message {}'.format(e),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_DOC_SERVICE_CONN_ERROR},
{}))
raise e
else:
return True
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=retry_mult)
def check_proxy(self):
"""Check whether proxy is up and has the same sandbox mode (to prevent launching wrong pair of bot-proxy)"""
try:
self.proxy_client.health(self.sandbox_mode)
except RequestException as e:
logger.info('Proxy server connection error, message {} {}'.format(
e, self.sandbox_mode),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_PROXY_SERVER_CONN_ERROR},
{}))
raise e
else:
return True
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=retry_mult)
def check_openprocurement_api(self):
"""Makes request to the TendersClient, returns True if it's up, raises RequestError otherwise"""
try:
self.client.head('/api/{}/spore'.format(self.api_version))
except (RequestError, ResourceError) as e:
logger.info('TendersServer connection error, message {}'.format(e),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_DOC_SERVICE_CONN_ERROR},
{}))
raise e
else:
return True
def set_sleep(self):
self.services_not_available.clear()
def set_wake_up(self):
self.services_not_available.set()
def all_available(self):
try:
self.check_proxy() and self.check_openprocurement_api(
) and self.check_doc_service()
except Exception as e:
logger.info("Service is unavailable, message {}".format(e.message))
return False
else:
return True
def check_services(self):
if self.all_available():
logger.info("All services are available")
self.set_wake_up()
else:
logger.info("Pausing bot")
self.set_sleep()
def _start_jobs(self):
self.jobs = {
'scanner': self.scanner(),
'filter_tender': self.filter_tender(),
'edr_handler': self.edr_handler(),
'upload_file_to_doc_service': self.upload_file_to_doc_service(),
'upload_file_to_tender': self.upload_file_to_tender(),
}
def launch(self):
while True:
if self.all_available():
self.run()
break
gevent.sleep(self.delay)
def run(self):
logger.info('Start EDR API Data Bridge',
extra=journal_context({"MESSAGE_ID": DATABRIDGE_START},
{}))
self._start_jobs()
counter = 0
try:
while True:
gevent.sleep(self.delay)
self.check_services()
if counter == 20:
counter = 0
logger.info(
'Current state: Filtered tenders {}; Edrpou codes queue {}; Retry edrpou codes queue {};'
'Upload to doc service {}; Retry upload to doc service {}; '
'Upload to tender {}; Retry upload to tender {}'.
format(
self.filtered_tender_ids_queue.qsize(),
self.edrpou_codes_queue.qsize(),
self.jobs['edr_handler'].retry_edrpou_codes_queue.
qsize() if self.jobs['edr_handler'] else 0,
self.upload_to_doc_service_queue.qsize(),
self.jobs['upload_file_to_doc_service'].
retry_upload_to_doc_service_queue.qsize()
if self.jobs['upload_file_to_doc_service'] else 0,
self.upload_to_tender_queue.qsize(),
self.jobs['upload_file_to_tender'].
retry_upload_to_tender_queue.qsize()
if self.jobs['upload_file_to_tender'] else 0))
counter += 1
self.check_and_revive_jobs()
except KeyboardInterrupt:
logger.info('Exiting...')
gevent.killall(self.jobs, timeout=5)
except Exception as e:
logger.error(e)
def check_and_revive_jobs(self):
for name, job in self.jobs.items():
if job.dead:
self.revive_job(name)
def revive_job(self, name):
logger.warning('Restarting {} worker'.format(name),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_RESTART_WORKER}))
self.jobs[name] = gevent.spawn(getattr(self, name))
def setUp(self):
self.process_tracker = ProcessTracker(self.db)
self.tender_id = "111"
self.item_id = "222"
self.document_id = "333"
class TestUtils(TestCase):
__test__ = True
relative_to = os.path.dirname(__file__) # crafty line
redis = None
redis_process = None
PORT = 16379
@classmethod
def setUpClass(cls):
cls.redis_process = subprocess.Popen(
['redis-server', '--port',
str(cls.PORT), '--logfile /dev/null'])
time.sleep(0.1)
cls.db = Db(config)
cls.redis = StrictRedis(port=cls.PORT)
def setUp(self):
self.process_tracker = ProcessTracker(self.db)
self.tender_id = "111"
self.item_id = "222"
self.document_id = "333"
@classmethod
def tearDownClass(cls):
cls.redis_process.terminate()
cls.redis_process.wait()
def tearDown(self):
self.redis.flushall()
def test_db_init(self):
self.assertEqual(self.db._backend, "redis")
self.assertEqual(self.db._db_name, 0)
self.assertEqual(self.db._port, "16379")
self.assertEqual(self.db._host, "127.0.0.1")
def test_db_init_no_redis(self):
db = Db(config_no_redis)
self.assertIsNone(db._backend)
self.assertIsNone(db._db_name, 0)
self.assertIsNone(db._port)
self.assertIsNone(db._host)
def test_db_get(self):
self.assertIsNone(self.db.get("111"))
self.db.put("111", "test data")
self.assertEqual(self.db.get("111"), "test data")
def test_db_set(self):
self.db.put("111", "test data")
self.assertEqual(self.db.get("111"), "test data")
def test_db_has(self):
self.assertFalse(self.db.has("111"))
self.db.put("111", "test data")
self.assertTrue(self.db.has("111"))
def test_set_item(self):
self.assertEqual(self.process_tracker.processing_items, {})
self.assertEqual(self.process_tracker.tender_documents_to_process, {})
self.process_tracker.set_item(self.tender_id, self.item_id, 1)
self.assertEqual(self.process_tracker.processing_items,
{item_key(self.tender_id, self.item_id): 1})
self.assertEqual(self.process_tracker.tender_documents_to_process,
{db_key(self.tender_id): 1})
def test_add_docs_amount_to_tender(self):
self.assertEqual(self.process_tracker.tender_documents_to_process, {})
self.process_tracker._add_docs_amount_to_tender(self.tender_id, 2)
self.assertEqual(self.process_tracker.tender_documents_to_process,
{db_key(self.tender_id): 2})
self.process_tracker._add_docs_amount_to_tender(self.tender_id, 3)
self.assertEqual(self.process_tracker.tender_documents_to_process,
{db_key(self.tender_id): 5})
def test_remove_docs_amount_from_tender(self):
self.assertEqual(self.process_tracker.tender_documents_to_process, {})
self.process_tracker.tender_documents_to_process = {
db_key(self.tender_id): 2
}
self.assertEqual(self.process_tracker.tender_documents_to_process,
{db_key(self.tender_id): 2})
self.process_tracker._remove_docs_amount_from_tender(self.tender_id)
self.assertEqual(self.process_tracker.tender_documents_to_process,
{db_key(self.tender_id): 1})
self.process_tracker._remove_docs_amount_from_tender(self.tender_id)
self.assertEqual(self.process_tracker.tender_documents_to_process, {})
def test_check_processing_item(self):
self.assertEqual(self.process_tracker.processing_items, {})
self.assertFalse(
self.process_tracker.check_processing_item(self.tender_id,
self.item_id))
self.process_tracker.set_item(self.tender_id, self.item_id)
self.assertTrue(
self.process_tracker.check_processing_item(self.tender_id,
self.item_id))
def test_check_processed_item(self):
self.assertEqual(self.process_tracker.processed_items, {})
self.assertFalse(
self.process_tracker.check_processed_item(self.tender_id,
self.item_id))
self.process_tracker.set_item(self.tender_id, self.item_id)
self.process_tracker.update_items_and_tender(self.tender_id,
self.item_id,
self.document_id)
self.assertTrue(
self.process_tracker.check_processed_item(self.tender_id,
self.item_id))
def test_check_processed_tender(self):
self.assertFalse(
self.process_tracker.check_processed_tenders(self.tender_id))
self.redis.set(self.tender_id, "333")
self.assertTrue(
self.process_tracker.check_processed_tenders(self.tender_id))
def test_update_processing_items(self):
self.process_tracker.processing_items = {
item_key(self.tender_id, self.item_id): 2
}
self.assertEqual(self.process_tracker.processing_items,
{item_key(self.tender_id, self.item_id): 2})
self.process_tracker._update_processing_items(self.tender_id,
self.item_id,
self.document_id)
self.assertEqual(self.process_tracker.processing_items,
{item_key(self.tender_id, self.item_id): 1})
self.process_tracker._update_processing_items(self.tender_id,
self.item_id,
self.document_id)
self.assertEqual(self.process_tracker.processing_items, {})
def test_check_412_function(self):
func = check_412(
MagicMock(side_effect=ResourceError(
http_code=412, response=MagicMock(headers={'Set-Cookie': 1}))))
with self.assertRaises(ResourceError):
func(MagicMock(headers={'Cookie': 1}))
func = check_412(
MagicMock(side_effect=ResourceError(
http_code=403, response=MagicMock(headers={'Set-Cookie': 1}))))
with self.assertRaises(ResourceError):
func(MagicMock(headers={'Cookie': 1}))
f = check_412(MagicMock(side_effect=[1]))
self.assertEqual(f(1), 1)
class TestUploadFileToTenderWorker(unittest.TestCase):
__test__ = True
def setUp(self):
self.tender_id = uuid.uuid4().hex
self.award_id = uuid.uuid4().hex
self.qualification_id = uuid.uuid4().hex
self.document_id = generate_doc_id()
self.process_tracker = ProcessTracker(db=MagicMock())
self.process_tracker.set_item(self.tender_id, self.award_id, 1)
self.upload_to_tender_queue = Queue(10)
self.url = 'http://127.0.0.1:20604'
self.sleep_change_value = APIRateController()
self.data = Data(self.tender_id, self.award_id, '123', 'awards', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.qualification_data = Data(self.tender_id, self.qualification_id,
'123', 'qualifications', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.client = MagicMock()
self.worker = UploadFileToTender(self.client,
self.upload_to_tender_queue,
self.process_tracker, MagicMock(),
self.sleep_change_value)
self.worker.retry_upload_to_tender_queue = Queue(10)
def tearDown(self):
del self.worker
del self.upload_to_tender_queue
@staticmethod
def get_tender():
return {
'data': {
'id': uuid.uuid4().hex,
'documentOf': 'tender',
'documentType': DOC_TYPE,
'url': 'url'
}
}
def is_working(self, worker):
return self.upload_to_tender_queue.qsize(
) or worker.retry_upload_to_tender_queue.qsize()
def shutdown_when_done(self, worker):
worker.start()
while self.is_working(worker):
sleep(0.1)
worker.shutdown()
@patch('gevent.sleep')
def test_upload_to_tender_429(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.client._create_tender_resource_item = MagicMock(side_effect=[
ResourceError(http_code=429),
ResourceError(http_code=429),
ResourceError(http_code=403)
])
self.upload_to_tender_queue.put(self.data)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.worker.sleep_change_value.time_between_requests,
1)
@patch('gevent.sleep')
def test_upload_to_tender_exception(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.upload_to_tender_queue.put(self.data)
self.client._create_tender_resource_item = MagicMock(
side_effect=[Exception()])
self.worker.do_upload_to_tender_with_retry = MagicMock(
side_effect=ResourceError(http_code=403))
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.worker.sleep_change_value.time_between_requests,
0)
@patch('gevent.sleep')
def test_upload_to_tender_exception_status_int_none(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.upload_to_tender_queue.put(self.data)
client = MagicMock()
client._create_tender_resource_item = MagicMock(
side_effect=[Unauthorized()])
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.worker.sleep_change_value.time_between_requests,
0)
@patch('gevent.sleep')
def test_retry_upload_to_tender(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.client._create_tender_resource_item.side_effect = [
Unauthorized(http_code=401),
Unauthorized(http_code=403),
Unauthorized(http_code=429),
self.get_tender()
]
self.upload_to_tender_queue.put(self.data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
self.assertEqual(self.upload_to_tender_queue.qsize(), 1)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.process_tracker.processing_items,
{}) # test that item removed from processing_items
self.assertEqual(self.client._create_tender_resource_item.call_count,
4) # check upload to tender
@patch('gevent.sleep')
def test_retry_upload_to_tender_422(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.client_upload_to_tender = MagicMock(side_effect=ResourceError(
http_code=422))
self.worker.retry_upload_to_tender_queue = Queue(10)
self.worker.retry_upload_to_tender_queue.put(self.data)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
@patch('gevent.sleep')
def test_retry_upload_to_tender_429(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.client.client_upload_to_tender = MagicMock(side_effect=[
ResourceError(http_code=429),
ResourceError(http_code=403)
])
self.worker.retry_upload_to_tender_queue = Queue(10)
self.worker.retry_upload_to_tender_queue.put(self.data)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
@patch('gevent.sleep')
def test_retry_upload_to_tender_exception(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.worker.do_upload_to_tender_with_retry = MagicMock(
side_effect=[Exception(),
ResourceError(http_code=403)])
self.worker.retry_upload_to_tender_queue.put(self.data)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
@patch('gevent.sleep')
def test_upload_to_tender_queue_loop_exit(self, gevent_sleep):
""" Test LoopExit for upload_to_tender_queue """
gevent_sleep.side_effect = custom_sleep
self.client._create_tender_resource_item.side_effect = [
self.get_tender() for _ in range(2)
]
self.process_tracker.set_item(self.tender_id, self.award_id, 2)
self.worker.upload_to_tender_queue = MagicMock()
self.worker.upload_to_tender_queue.peek.side_effect = generate_answers(
answers=[LoopExit(), self.datum(),
self.datum()],
default=LoopExit())
self.worker.start()
sleep(1)
self.worker.shutdown()
self.assertEqual(self.process_tracker.processing_items, {})
self.assertIsNotNone(
self.client.request_history[0].headers['X-Client-Request-ID'])
self.assertIsNotNone(
self.client.request_history[1].headers['X-Client-Request-ID'])
self.assertEqual(self.client._create_tender_resource_item.call_count,
2) # check that processed just 1 request
def datum(self):
return Data(
self.tender_id, self.award_id, '123', 'awards', {
u'meta': {
u'id': self.document_id
},
u'url':
u'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f',
u'format': u'application/yaml',
u'hash': u'md5:9a0364b9e99bb480dd25e1f0284c8555',
u'title': file_name
})
@patch('gevent.sleep')
def test_retry_upload_to_tender_queue_loop_exit(self, gevent_sleep):
""" Test LoopExit for retry_upload_to_tender_queue """
gevent_sleep.side_effect = custom_sleep
self.client._create_tender_resource_item.side_effect = [
self.get_tender() for _ in range(2)
]
self.worker.retry_upload_to_tender_queue = MagicMock()
self.worker.retry_upload_to_tender_queue.peek.side_effect = generate_answers(
answers=[LoopExit(), self.datum(),
self.datum()],
default=LoopExit())
self.process_tracker.set_item(self.tender_id, self.award_id, 2)
self.worker.start()
sleep(1)
self.worker.shutdown()
self.assertEqual(self.process_tracker.processing_items, {})
self.assertIsNotNone(
self.client.request_history[0].headers['X-Client-Request-ID'])
self.assertIsNotNone(
self.client.request_history[1].headers['X-Client-Request-ID'])
self.assertEqual(self.client._create_tender_resource_item.call_count,
2) # check that processed just 1 request
@patch('gevent.sleep')
def test_request_failed_in_retry_item_status(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.client._create_tender_resource_item.side_effect = [
ResourceError(http_code=429)
] + [ResourceError(http_code=403) for _ in range(4)]
self.worker.retry_upload_to_tender_queue.put(self.data)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
@patch('gevent.sleep')
def test_request_failed_in_retry(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.worker.do_upload_to_tender_with_retry = MagicMock()
self.worker.do_upload_to_tender_with_retry.side_effect = [
ResourceError(http_code=429) for _ in range(5)
] + [ResourceError(http_code=403)]
self.sleep_change_value.increment_step = 3
self.sleep_change_value.decrement_step = 1.5
self.worker.retry_upload_to_tender_queue.put(self.data)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.worker.sleep_change_value.time_between_requests,
13.5)
@patch('gevent.sleep')
def test_process_412(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
api_server_bottle = Bottle()
api_server = WSGIServer(('127.0.0.1', 20604),
api_server_bottle,
log=None)
setup_routing(api_server_bottle, response_spore)
setup_routing(api_server_bottle,
response_412,
path='/api/2.3/tenders/{}/awards/{}/documents'.format(
self.tender_id, self.award_id),
method='POST')
api_server.start()
self.worker.client = TendersClient('',
host_url='http://127.0.0.1:20604',
api_version='2.3')
setup_routing(api_server_bottle,
generate_response,
path='/api/2.3/tenders/{}/awards/{}/documents'.format(
self.tender_id, self.award_id),
method='POST')
self.assertEqual(self.worker.client.headers['Cookie'],
'SERVER_ID={}'.format(SPORE_COOKIES))
self.upload_to_tender_queue.put(self.data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
self.assertEqual(self.upload_to_tender_queue.qsize(), 1)
self.shutdown_when_done(self.worker)
self.assertEqual(self.worker.client.headers['Cookie'],
'SERVER_ID={}'.format(COOKIES_412))
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.worker.retry_upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertItemsEqual(self.process_tracker.processing_items.keys(), [])
api_server.stop()
@patch('gevent.sleep')
def test_upload_worker(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.worker.services_not_available = MagicMock(wait=MagicMock())
self.worker.try_peek_data_and_upload_to_tender = MagicMock()
with patch.object(self.worker, 'exit', AlmostAlwaysFalse()):
self.worker.upload_worker()
self.worker.services_not_available.wait.assert_called_once()
self.worker.try_peek_data_and_upload_to_tender.assert_called_once_with(
False)
@patch('gevent.sleep')
def test_retry_upload_worker(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.worker.services_not_available = MagicMock(wait=MagicMock())
self.worker.try_peek_data_and_upload_to_tender = MagicMock()
with patch.object(self.worker, 'exit', AlmostAlwaysFalse()):
self.worker.retry_upload_worker()
self.worker.services_not_available.wait.assert_called_once()
self.worker.try_peek_data_and_upload_to_tender.assert_called_once_with(
True)
def test_peek_from_tender_queue(self):
self.worker.upload_to_tender_queue.put(self.data)
self.assertEqual(self.worker.peek_from_tender_queue(False), self.data)
def test_peek_from_tender_queue_retry(self):
self.worker.retry_upload_to_tender_queue.put(self.data)
self.assertEqual(self.worker.peek_from_tender_queue(True), self.data)
def test_peek_from_tender_queue_empty(self):
self.worker.upload_to_tender_queue = MagicMock(peek=MagicMock(
side_effect=LoopExit))
with self.assertRaises(LoopExit):
self.worker.peek_from_tender_queue(False)
def test_peek_from_tender_queue_retry_empty(self):
self.worker.retry_upload_to_tender_queue = MagicMock(peek=MagicMock(
side_effect=LoopExit))
with self.assertRaises(LoopExit):
self.worker.peek_from_tender_queue(True)
def test_try_peek_data_and_upload_to_tender(self):
self.worker.upload_to_tender_queue.put(self.data)
self.worker.try_upload_to_tender = MagicMock()
self.worker.try_peek_data_and_upload_to_tender(False)
self.worker.try_upload_to_tender.assert_called_once_with(
self.data, False)
def test_try_peek_data_and_upload_to_tender_retry(self):
self.worker.retry_upload_to_tender_queue.put(self.data)
self.worker.try_upload_to_tender = MagicMock()
self.worker.try_peek_data_and_upload_to_tender(True)
self.worker.try_upload_to_tender.assert_called_once_with(
self.data, True)
def test_try_upload_to_tender(self):
self.worker.update_headers_and_upload_to_tender = MagicMock()
self.worker.successfully_uploaded_to_tender = MagicMock()
self.worker.try_upload_to_tender(self.data, False)
self.worker.update_headers_and_upload_to_tender.assert_called_once_with(
self.data, False)
self.worker.successfully_uploaded_to_tender.assert_called_once_with(
self.data, False)
def test_try_upload_to_tender_retry(self):
self.worker.update_headers_and_upload_to_tender = MagicMock()
self.worker.successfully_uploaded_to_tender = MagicMock()
self.worker.try_upload_to_tender(self.data, True)
self.worker.update_headers_and_upload_to_tender.assert_called_once_with(
self.data, True)
self.worker.successfully_uploaded_to_tender.assert_called_once_with(
self.data, True)
def test_try_upload_to_tender_no_mock(self):
self.upload_to_tender_queue.put(self.data)
self.worker.try_upload_to_tender(self.data, False)
self.assertEqual(self.upload_to_tender_queue.qsize(), 0)
self.assertEqual(self.process_tracker.processing_items, {})
def test_try_upload_to_tender_no_mock_retry(self):
self.worker.retry_upload_to_tender_queue.put(self.data)
self.worker.try_upload_to_tender(self.data, True)
self.assertEqual(self.upload_to_tender_queue.qsize(), 0)
self.assertEqual(self.process_tracker.processing_items, {})
def test_try_upload_to_tender_resource_error(self):
re = ResourceError("test resource error")
self.worker.update_headers_and_upload_to_tender = MagicMock(
side_effect=re)
self.worker.remove_data_or_increase_wait = MagicMock()
self.worker.try_upload_to_tender(self.data, False)
self.worker.remove_data_or_increase_wait.assert_called_once_with(
re, self.data, False)
def test_try_upload_to_tender_exception(self):
e = Exception("exception")
self.worker.update_headers_and_upload_to_tender = MagicMock(
side_effect=e)
self.worker.handle_error = MagicMock()
self.worker.try_upload_to_tender(self.data, False)
self.worker.handle_error.assert_called_once_with(e, self.data, False)
def test_update_headers_and_upload_to_tender(self):
self.worker.do_upload_to_tender = MagicMock()
self.worker.update_headers_and_upload_to_tender(self.data, False)
self.worker.do_upload_to_tender.assert_called_once_with(self.data)
def test_update_headers_and_upload_to_tender_retry(self):
self.worker.do_upload_to_tender_with_retry = MagicMock()
self.worker.update_headers_and_upload_to_tender(self.data, True)
self.worker.do_upload_to_tender_with_retry.assert_called_once_with(
self.data)
def test_do_upload_to_tender(self):
api_server_bottle = Bottle()
api_server = WSGIServer(('127.0.0.1', 20604),
api_server_bottle,
log=None)
setup_routing(api_server_bottle, response_spore)
api_server.start()
self.worker.client = TendersClientSync(
'', host_url='http://127.0.0.1:20604', api_version='2.3')
setup_routing(api_server_bottle,
response_get_tender,
path='/api/2.3/tenders/{}/awards/{}/documents'.format(
self.tender_id, self.award_id),
method='POST')
self.worker.do_upload_to_tender(self.data)
api_server.stop()
def test_do_upload_to_tender_failure(self):
api_server_bottle = Bottle()
api_server = WSGIServer(('127.0.0.1', 20604),
api_server_bottle,
log=None)
setup_routing(api_server_bottle, response_spore)
api_server.start()
self.worker.client = TendersClientSync(
'', host_url='http://127.0.0.1:20604', api_version='2.3')
setup_routing(api_server_bottle,
response_412,
path='/api/2.3/tenders/{}/awards/{}/documents'.format(
self.tender_id, self.award_id),
method='POST')
with self.assertRaises(ResourceError):
self.worker.do_upload_to_tender(self.data)
api_server.stop()
def test_do_upload_to_tender_with_retry(self):
api_server_bottle = Bottle()
api_server = WSGIServer(('127.0.0.1', 20604),
api_server_bottle,
log=None)
setup_routing(api_server_bottle, response_spore)
api_server.start()
self.worker.client = TendersClientSync(
'', host_url='http://127.0.0.1:20604', api_version='2.3')
setup_routing(api_server_bottle,
response_get_tender,
path='/api/2.3/tenders/{}/awards/{}/documents'.format(
self.tender_id, self.award_id),
method='POST')
self.worker.do_upload_to_tender_with_retry(self.data)
api_server.stop()
def test_do_upload_to_tender_with_retry_fail_then_success(self):
api_server_bottle = Bottle()
api_server = WSGIServer(('127.0.0.1', 20604),
api_server_bottle,
log=None)
setup_routing(api_server_bottle, response_spore)
api_server.start()
self.worker.client = TendersClientSync(
'', host_url='http://127.0.0.1:20604', api_version='2.3')
setup_routing(api_server_bottle,
generate_response_retry,
path='/api/2.3/tenders/{}/awards/{}/documents'.format(
self.tender_id, self.award_id),
method='POST')
self.worker.do_upload_to_tender_with_retry(self.data)
api_server.stop()
def test_do_upload_to_tender_with_retry_fail(self):
api_server_bottle = Bottle()
api_server = WSGIServer(('127.0.0.1', 20604),
api_server_bottle,
log=None)
setup_routing(api_server_bottle, response_spore)
api_server.start()
self.worker.client = TendersClientSync(
'', host_url='http://127.0.0.1:20604', api_version='2.3')
setup_routing(api_server_bottle,
response_412,
path='/api/2.3/tenders/{}/awards/{}/documents'.format(
self.tender_id, self.award_id),
method='POST')
with self.assertRaises(ResourceError):
self.worker.do_upload_to_tender_with_retry(self.data)
api_server.stop()
def test_remove_data_or_increase_wait(self):
re = ResourceError("error")
self.worker.removing_data = MagicMock()
self.worker.remove_data_or_increase_wait(re, self.data, False)
self.worker.removing_data.assert_called_once_with(re, self.data, False)
def test_remove_data_or_increase_wait_429(self):
re = ResourceError("error", http_code=429)
self.worker.decrease_request_frequency = MagicMock()
self.worker.remove_data_or_increase_wait(re, self.data, False)
self.worker.decrease_request_frequency.assert_called_once_with(
re, self.data)
def test_remove_data_or_increase_wait_else(self):
re = ResourceError("error", http_code=404)
self.worker.handle_error = MagicMock()
self.worker.remove_data_or_increase_wait(re, self.data, False)
self.worker.handle_error.assert_called_once_with(re, self.data, False)
def test_removing_data(self):
re = ResourceError("error")
self.worker.sleep_change_value.time_between_requests = 1
self.worker.upload_to_tender_queue.put(self.data)
self.worker.removing_data(re, self.data, False)
self.assertEqual(self.worker.process_tracker.processing_items, {})
self.assertEqual(self.worker.upload_to_tender_queue.qsize(), 0)
self.assertEqual(self.worker.sleep_change_value.time_between_requests,
0)
def test_removing_data_retry(self):
re = ResourceError("error")
self.worker.sleep_change_value.time_between_requests = 1
self.worker.retry_upload_to_tender_queue.put(self.data)
self.worker.removing_data(re, self.data, True)
self.assertEqual(self.worker.process_tracker.processing_items, {})
self.assertEqual(self.worker.upload_to_tender_queue.qsize(), 0)
self.assertEqual(self.worker.retry_upload_to_tender_queue.qsize(), 0)
self.assertEqual(self.worker.sleep_change_value.time_between_requests,
0)
def test_decrease_request_frequency(self):
re = ResourceError("error", 429)
self.worker.decrease_request_frequency(re, self.data)
self.assertEqual(self.worker.sleep_change_value.time_between_requests,
1)
def test_handle_error(self):
re = ResourceError("error", 404)
self.worker.upload_to_tender_queue.put(self.data)
self.worker.handle_error(re, self.data, False)
self.assertEqual(self.worker.upload_to_tender_queue.qsize(), 0)
self.assertEqual(self.worker.retry_upload_to_tender_queue.get(),
self.data)
self.assertEqual(self.worker.retry_upload_to_tender_queue.qsize(), 0)
def test_handle_error_retry(self):
re = ResourceError("error", 404)
self.worker.upload_to_tender_queue.put(self.data)
self.worker.handle_error(re, self.data, True)
self.assertEqual(self.worker.upload_to_tender_queue.qsize(), 1)
self.assertEqual(self.worker.retry_upload_to_tender_queue.qsize(), 0)
def test_successfully_uploaded_to_tender(self):
self.worker.upload_to_tender_queue.put(self.data)
self.assertEqual(self.worker.process_tracker.processing_items,
{item_key(self.tender_id, self.award_id): 1})
self.worker.successfully_uploaded_to_tender(self.data, False)
self.assertEqual(self.worker.upload_to_tender_queue.qsize(), 0)
self.assertEqual(self.worker.process_tracker.processing_items, {})
def test_successfully_uploaded_to_tender_retry(self):
self.worker.retry_upload_to_tender_queue.put(self.data)
self.assertEqual(self.worker.process_tracker.processing_items,
{item_key(self.tender_id, self.award_id): 1})
self.worker.successfully_uploaded_to_tender(self.data, True)
self.assertEqual(self.worker.retry_upload_to_tender_queue.qsize(), 0)
self.assertEqual(self.worker.process_tracker.processing_items, {})
def test_run(self):
self.worker.delay = 1
upload_worker, retry_upload_worker = MagicMock(), MagicMock()
self.worker.upload_worker = upload_worker
self.worker.retry_upload_worker = retry_upload_worker
with patch.object(self.worker, 'exit', AlmostAlwaysFalse()):
self.worker._run()
self.assertEqual(self.worker.upload_worker.call_count, 1)
self.assertEqual(self.worker.retry_upload_worker.call_count, 1)
@patch('gevent.killall')
@patch('gevent.sleep')
def test_run_exception(self, gevent_sleep, killlall):
gevent_sleep.side_effect = custom_sleep
self.worker._start_jobs = MagicMock(return_value={"a": 1})
self.worker.check_and_revive_jobs = MagicMock(
side_effect=Exception("test error"))
self.worker._run()
killlall.assert_called_once_with([1], timeout=5)
@patch('gevent.killall')
def test_run_exception(self, killlall):
self.worker.delay = 1
self.worker._start_jobs = MagicMock(return_value={"a": 1})
self.worker.check_and_revive_jobs = MagicMock(
side_effect=Exception("test error"))
self.worker._run()
killlall.assert_called_once_with([1], timeout=5)
class TestUploadFileWorker(unittest.TestCase):
__test__ = True
def setUp(self):
self.tender_id = uuid.uuid4().hex
self.award_id = uuid.uuid4().hex
self.qualification_id = uuid.uuid4().hex
self.document_id = generate_doc_id()
self.process_tracker = ProcessTracker(db=MagicMock())
self.process_tracker.set_item(self.tender_id, self.award_id, 1)
self.upload_to_doc_service_queue = Queue(10)
self.upload_to_tender_queue = Queue(10)
self.sleep_change_value = APIRateController()
self.sna = event.Event()
self.sna.set()
self.data = Data(self.tender_id, self.award_id, '123', 'awards', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.qualification_data = Data(self.tender_id, self.qualification_id,
'123', 'qualifications', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.doc_service_client = DocServiceClient(host='127.0.0.1',
port='80',
user='',
password='')
self.worker = UploadFileToDocService(self.upload_to_doc_service_queue,
self.upload_to_tender_queue,
self.process_tracker,
self.doc_service_client, self.sna,
self.sleep_change_value)
self.url = '{url}'.format(url=self.doc_service_client.url)
@staticmethod
def stat_200():
return {
'data': {
'url':
'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f',
'format': 'application/yaml',
'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555',
'title': file_name
}
}
@staticmethod
def get_tender():
return {
'data': {
'id': uuid.uuid4().hex,
'documentOf': 'tender',
'documentType': DOC_TYPE,
'url': 'url'
}
}
def tearDown(self):
del self.worker
def is_working(self, worker):
return self.upload_to_doc_service_queue.qsize(
) or worker.retry_upload_to_doc_service_queue.qsize()
def shutdown_when_done(self, worker):
worker.start()
while self.is_working(worker):
sleep(0.1)
worker.shutdown()
def test_init(self):
worker = UploadFileToDocService.spawn(None, None, None, None, self.sna,
None)
self.assertGreater(datetime.datetime.now().isoformat(),
worker.start_time.isoformat())
self.assertEqual(worker.upload_to_doc_service_queue, None)
self.assertEqual(worker.upload_to_tender_queue, None)
self.assertEqual(worker.process_tracker, None)
self.assertEqual(worker.doc_service_client, None)
self.assertEqual(worker.services_not_available, self.sna)
self.assertEqual(worker.sleep_change_value, None)
self.assertEqual(worker.delay, 15)
self.assertEqual(worker.exit, False)
worker.shutdown()
self.assertEqual(worker.exit, True)
del worker
@requests_mock.Mocker()
@patch('gevent.sleep')
def test_successful_upload(self, mrequest, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
mrequest.post(self.url, json=self.stat_200(), status_code=200)
self.upload_to_doc_service_queue.put(self.data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.upload_to_tender_queue.qsize(), 1,
'Queue should be have 1 element')
self.assertEqual(mrequest.call_count, 1)
self.assertEqual(mrequest.request_history[0].url,
u'127.0.0.1:80/upload')
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
@requests_mock.Mocker()
@patch('gevent.sleep')
def test_retry_doc_service(self, mrequest, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
doc_service_client = DocServiceClient(host='127.0.0.1',
port='80',
user='',
password='')
mrequest.post(self.url, [{
'text': '',
'status_code': 401
} for _ in range(6)] + [{
'json': {
'data': {
'url': 'test url',
'format': 'application/yaml',
'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555',
'title': file_name
}
},
'status_code': 200
}])
self.upload_to_doc_service_queue.put(self.data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.upload_to_tender_queue.qsize(), 1,
'Queue should be have 1 element')
self.assertEqual(mrequest.call_count, 7)
self.assertEqual(mrequest.request_history[0].url,
u'127.0.0.1:80/upload')
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
@requests_mock.Mocker()
@patch('gevent.sleep')
def test_request_failed(self, mrequest, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
mrequest.post(self.url, json=self.stat_200(), status_code=200)
self.upload_to_doc_service_queue.put(self.data)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.upload_to_tender_queue.get(), self.data)
self.assertEqual(self.process_tracker.processing_items,
{item_key(self.tender_id, self.award_id): 1})
self.assertEqual(mrequest.call_count, 1)
self.assertEqual(mrequest.request_history[0].url,
u'127.0.0.1:80/upload')
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
@requests_mock.Mocker()
@patch('gevent.sleep')
def test_request_failed_item_status_change(self, mrequest, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
mrequest.post(self.url, json=self.stat_200(), status_code=200)
self.process_tracker.set_item(self.tender_id, self.qualification_id, 1)
self.upload_to_doc_service_queue.put(self.data)
self.upload_to_doc_service_queue.put(self.qualification_data)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.upload_to_tender_queue.get(), self.data)
self.assertEqual(self.upload_to_tender_queue.get(),
self.qualification_data)
self.assertEqual(mrequest.call_count, 2)
self.assertEqual(mrequest.request_history[0].url,
u'127.0.0.1:80/upload')
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
self.assertEqual(
self.process_tracker.processing_items, {
item_key(self.tender_id, self.award_id): 1,
item_key(self.tender_id, self.qualification_id): 1
})
@requests_mock.Mocker()
@patch('gevent.sleep')
def test_processing_items(self, mrequest, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
mrequest.post(self.url, [{
'json': self.stat_200(),
'status_code': 200
} for _ in range(2)])
self.process_tracker.set_item(self.tender_id, self.award_id, 2)
self.upload_to_doc_service_queue.put(self.data)
self.upload_to_doc_service_queue.put(self.data)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_tender_queue.get(), self.data)
self.assertEqual(self.upload_to_tender_queue.get(), self.data)
self.assertEqual(mrequest.request_history[0].url,
u'127.0.0.1:80/upload')
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
@requests_mock.Mocker()
@patch('gevent.sleep')
def test_upload_to_doc_service_queue_loop_exit(self, mrequest,
gevent_sleep):
""" Test LoopExit for upload_to_doc_service_queue """
gevent_sleep.side_effect = custom_sleep
self.process_tracker.set_item(self.tender_id, self.award_id, 2)
self.worker.upload_to_doc_service_queue = MagicMock()
self.worker.upload_to_doc_service_queue.peek.side_effect = generate_answers(
answers=[LoopExit(), self.data, self.data], default=LoopExit())
mrequest.post(self.url, [{
'json': self.stat_200(),
'status_code': 200
} for _ in range(2)])
self.worker.start()
sleep(1)
self.assertEqual(self.upload_to_tender_queue.get(), self.data)
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
self.assertIsNotNone(
mrequest.request_history[1].headers['X-Client-Request-ID'])
self.assertEqual(self.process_tracker.processing_items,
{item_key(self.tender_id, self.award_id): 2})
@requests_mock.Mocker()
@patch('gevent.sleep')
def test_retry_upload_to_doc_service_queue_loop_exit(
self, mrequest, gevent_sleep):
""" Test LoopExit for retry_upload_to_doc_service_queue """
gevent_sleep.side_effect = custom_sleep
mrequest.post(self.url, [{
'json': self.stat_200(),
'status_code': 200
} for _ in range(2)])
self.process_tracker.set_item(self.tender_id, self.award_id, 2)
self.worker.retry_upload_to_doc_service_queue = MagicMock()
self.worker.retry_upload_to_doc_service_queue.peek.side_effect = generate_answers(
answers=[LoopExit(), self.data, self.data], default=LoopExit())
self.worker.start()
sleep(1)
self.worker.shutdown()
self.assertEqual(self.upload_to_tender_queue.get(), self.data)
self.assertEqual(self.process_tracker.processing_items,
{item_key(self.tender_id, self.award_id): 2})
self.assertEqual(mrequest.request_history[0].url,
u'127.0.0.1:80/upload')
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
def test_remove_bad_data(self):
self.worker.upload_to_doc_service_queue = MagicMock(get=MagicMock())
self.worker.process_tracker = MagicMock(
update_items_and_tender=MagicMock())
self.worker.remove_bad_data(self.data, Exception("test message"),
False)
self.worker.upload_to_doc_service_queue.get.assert_called_once()
self.assertEqual(self.worker.retry_upload_to_doc_service_queue.get(),
self.data)
def test_remove_bad_data_retry(self):
self.worker.retry_upload_to_doc_service_queue = MagicMock(
get=MagicMock())
self.worker.process_tracker = MagicMock(
update_items_and_tender=MagicMock())
with self.assertRaises(Exception):
self.worker.remove_bad_data(self.data, Exception("test message"),
True)
self.worker.retry_upload_to_doc_service_queue.get.assert_called_once()
self.worker.process_tracker.update_items_and_tender.assert_called_with(
self.data.tender_id, self.data.item_id, self.document_id)
def test_try_upload_to_doc_service(self):
e = Exception("test error")
self.worker.update_headers_and_upload = MagicMock(side_effect=e)
self.worker.remove_bad_data = MagicMock()
self.worker.try_upload_to_doc_service(self.data, False)
self.worker.update_headers_and_upload.assert_called_once()
self.worker.remove_bad_data.assert_called_once_with(
self.data, e, False)
def test_try_upload_to_doc_service_retry(self):
e = Exception("test error")
self.worker.update_headers_and_upload = MagicMock(side_effect=e)
self.worker.remove_bad_data = MagicMock()
self.worker.try_upload_to_doc_service(self.data, True)
self.worker.update_headers_and_upload.assert_called_once()
self.worker.remove_bad_data.assert_called_with(self.data, e, True)
def test_run(self):
self.worker.delay = 1
upload_worker, retry_upload_worker = MagicMock(), MagicMock()
self.worker.upload_worker = upload_worker
self.worker.retry_upload_worker = retry_upload_worker
with patch.object(self.worker, 'exit', AlmostAlwaysFalse()):
self.worker._run()
self.assertEqual(self.worker.upload_worker.call_count, 1)
self.assertEqual(self.worker.retry_upload_worker.call_count, 1)
@patch('gevent.killall')
def test_run_exception(self, killlall):
self.worker.delay = 1
self.worker._start_jobs = MagicMock(return_value={"a": 1})
self.worker.check_and_revive_jobs = MagicMock(
side_effect=Exception("test error"))
self.worker._run()
killlall.assert_called_once_with([1], timeout=5)
@patch('gevent.killall')
@patch('gevent.sleep')
def test_run_exception(self, gevent_sleep, killlall):
gevent_sleep.side_effect = custom_sleep
self.worker._start_jobs = MagicMock(return_value={"a": 1})
self.worker.check_and_revive_jobs = MagicMock(
side_effect=Exception("test error"))
self.worker._run()
killlall.assert_called_once_with([1], timeout=5)