def test_fill_resource_items_queue(self):
bridge = EdgeDataBridge(self.config)
db_dict_list = [
{
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.utcnow().isoformat()
},
{
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.utcnow().isoformat()
}]
with patch('__builtin__.True', AlmostAlwaysTrue(1)):
bridge.fill_resource_items_queue()
self.assertEqual(bridge.resource_items_queue.qsize(), 0)
for item in db_dict_list:
bridge.input_queue.put({
'id': item['id'],
'dateModified': datetime.datetime.utcnow().isoformat()
})
view_return_list = [
munchify({
'id': db_dict_list[0]['id'],
'key': db_dict_list[0]['dateModified']
})
]
bridge.db.view = MagicMock(return_value=view_return_list)
with patch('__builtin__.True', AlmostAlwaysTrue(1)):
bridge.fill_resource_items_queue()
self.assertEqual(bridge.resource_items_queue.qsize(), 1)
def test_perfomance_watcher(self):
bridge = EdgeDataBridge(self.config)
for i in xrange(0, 3):
bridge.create_api_client()
req_duration = 1
for _, info in bridge.api_clients_info.items():
info['request_durations'][datetime.datetime.now()] = req_duration
req_duration += 1
self.assertEqual(info.get('grown', False), False)
self.assertEqual(len(info['request_durations']), 1)
self.assertEqual(len(bridge.api_clients_info), 3)
self.assertEqual(bridge.api_clients_queue.qsize(), 3)
sleep(1)
bridge.perfomance_watcher()
grown = 0
with_new_cookies = 0
for cid, info in bridge.api_clients_info.items():
if info.get('grown', False):
grown += 1
if info['drop_cookies']:
with_new_cookies += 1
self.assertEqual(len(info['request_durations']), 0)
self.assertEqual(len(bridge.api_clients_info), 3)
self.assertEqual(bridge.api_clients_queue.qsize(), 3)
self.assertEqual(grown, 3)
self.assertEqual(with_new_cookies, 1)
def test_config_get(self):
test_config = {
'main': {
'resources_api_server':
'https://lb.api-sandbox.openprocurement.org',
'resources_api_version': "0",
'public_resources_api_server':
'https://lb.api-sandbox.openprocurement.org',
'couch_url': 'http://localhost:5984',
'db_name': 'test_db',
'retrievers_params': {
'down_requests_sleep': 5,
'up_requests_sleep': 1,
'up_wait_sleep': 30,
'queue_size': 101
}
},
'version': 1
}
bridge = EdgeDataBridge(test_config)
couch_url_config = bridge.config_get('couch_url')
self.assertEqual(couch_url_config, test_config['main']['couch_url'])
del bridge.config['main']['couch_url']
couch_url_config = bridge.config_get('couch_url')
self.assertEqual(couch_url_config, None)
server = Server(test_config['main'].get('couch_url')
or 'http://127.0.0.1:5984')
del server[test_config['main']['db_name']]
del bridge.config['main']
with self.assertRaises(DataBridgeConfigError):
bridge.config_get('couch_url')
def test_gevent_watcher(self, mock_APIClient, mock_riw_spawn, mock_spawn):
bridge = EdgeDataBridge(self.config)
return_dict = {
'type':
'indexer',
'database':
bridge.db_name,
'design_document':
'_design/{}'.format(bridge.workers_config['resource']),
'progress':
99
}
bridge.server.tasks = MagicMock(return_value=[return_dict])
bridge.filler = MagicMock()
bridge.filler.exception = Exception('test_filler')
bridge.input_queue_filler = MagicMock()
bridge.input_queue_filler.exception = Exception('test_temp_filler')
self.assertEqual(bridge.workers_pool.free_count(), bridge.workers_max)
self.assertEqual(bridge.retry_workers_pool.free_count(),
bridge.retry_workers_max)
bridge.gevent_watcher()
self.assertEqual(bridge.workers_pool.free_count(),
bridge.workers_max - bridge.workers_min)
self.assertEqual(bridge.retry_workers_pool.free_count(),
bridge.retry_workers_max - bridge.retry_workers_min)
del bridge
def test_config_get(self):
test_config = {
'main': {
'tenders_api_server':
'https://lb.api-sandbox.openprocurement.org',
'tenders_api_version': "0",
'public_tenders_api_server':
'https://lb.api-sandbox.openprocurement.org',
'couch_url': 'http://localhost:5984',
'public_db': 'public_db'
},
'version': 1
}
bridge = EdgeDataBridge(test_config)
couch_url_config = bridge.config_get('couch_url')
self.assertEqual(couch_url_config, test_config['main']['couch_url'])
del bridge.config['main']['couch_url']
couch_url_config = bridge.config_get('couch_url')
self.assertEqual(couch_url_config, None)
del bridge.config['main']
with self.assertRaises(DataBridgeConfigError):
bridge.config_get('couch_url')
def test__calculate_st_dev(self):
bridge = EdgeDataBridge(self.config)
values = [1.1, 1.11, 1.12, 1.13, 1.14]
stdev = bridge._calculate_st_dev(values)
self.assertEqual(stdev, 0.014)
stdev = bridge._calculate_st_dev([])
self.assertEqual(stdev, 0)
def test__get_average_request_duration(self):
bridge = EdgeDataBridge(self.config)
bridge.create_api_client()
bridge.create_api_client()
bridge.create_api_client()
res, _ = bridge._get_average_requests_duration()
self.assertEqual(res, 0)
request_duration = 1
for k in bridge.api_clients_info:
for i in xrange(0, 3):
bridge.api_clients_info[k]['request_durations'][datetime.datetime.now()] =\
request_duration
request_duration += 1
res, res_list = bridge._get_average_requests_duration()
self.assertEqual(res, 2)
self.assertEqual(len(res_list), 3)
delta = datetime.timedelta(seconds=301)
grown_date = datetime.datetime.now() - delta
bridge.api_clients_info[uuid.uuid4().hex] = {
'request_durations': {grown_date: 1},
'destroy': False,
'request_interval': 0,
'avg_duration': 0
}
self.assertEqual(len(bridge.api_clients_info), 4)
res, res_list = bridge._get_average_requests_duration()
grown = 0
for k in bridge.api_clients_info:
if bridge.api_clients_info[k].get('grown', False):
grown += 1
self.assertEqual(res, 1.75)
self.assertEqual(len(res_list), 4)
self.assertEqual(grown, 1)
def test__mark_bad_clients(self):
bridge = EdgeDataBridge(self.config)
self.assertEqual(bridge.api_clients_queue.qsize(), 0)
self.assertEqual(len(bridge.api_clients_info), 0)
bridge.create_api_client()
bridge.create_api_client()
bridge.create_api_client()
self.assertEqual(len(bridge.api_clients_info), 3)
avg_duration = 1
req_intervals = [0, 2, 0, 0]
for cid in bridge.api_clients_info:
self.assertEqual(bridge.api_clients_info[cid]['drop_cookies'],
False)
bridge.api_clients_info[cid]['avg_duration'] = avg_duration
bridge.api_clients_info[cid]['grown'] = True
bridge.api_clients_info[cid]['request_interval'] = req_intervals[
avg_duration]
avg_duration += 1
avg = 1.5
bridge._mark_bad_clients(avg)
self.assertEqual(len(bridge.api_clients_info), 3)
self.assertEqual(bridge.api_clients_queue.qsize(), 3)
to_destroy = 0
for cid in bridge.api_clients_info:
if bridge.api_clients_info[cid]['drop_cookies']:
to_destroy += 1
self.assertEqual(to_destroy, 3)
def test_create_api_client(self, mock_APIClient):
mock_APIClient.side_effect = [RequestFailed(), munchify({
'session': {'headers': {'User-Agent': 'test.agent'}}
})]
bridge = EdgeDataBridge(self.config)
self.assertEqual(bridge.api_clients_queue.qsize(), 0)
bridge.create_api_client()
self.assertEqual(bridge.api_clients_queue.qsize(), 1)
del bridge
def test_run(self, mock_gevent, mock_perfomance, mock_controller,
mock_fill, mock_fill_input_queue):
bridge = EdgeDataBridge(self.config)
self.assertEqual(len(bridge.filter_workers_pool), 0)
with patch('__builtin__.True', AlmostAlwaysTrue(4)):
bridge.run()
self.assertEqual(mock_fill.call_count, 1)
self.assertEqual(mock_controller.call_count, 1)
self.assertEqual(mock_gevent.call_count, 1)
self.assertEqual(mock_fill_input_queue.call_count, 1)
def test_fill_input_queue(self):
bridge = EdgeDataBridge(self.config)
bridge.workers_config['historical'] = True
mock_api_client = MagicMock()
client_dict = {
'id': uuid.uuid4().hex,
'request_interval': 0.02,
'client': mock_api_client
}
bridge.api_clients_queue.put(client_dict)
return_value = [{
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.utcnow().isoformat()
}]
resource_item_return_value = {
'id': return_value[0]['id'],
'x_revision_n': randint(10, 99)
}
mock_api_client.get_resource_item_historical.return_value = resource_item_return_value
bridge.feeder.get_resource_items = MagicMock(return_value=return_value)
self.assertEqual(bridge.input_queue.qsize(), 0)
bridge.fill_input_queue()
self.assertEqual(bridge.resource_items_queue.qsize(),
resource_item_return_value['x_revision_n'])
bridge.resource_items_queue = Queue()
mock_api_client.get_resource_item_historical.side_effect = RequestFailed(
munchify({'status_code': 429}))
bridge.fill_input_queue()
self.assertEqual(bridge.resource_items_queue.qsize(), 0)
bridge = EdgeDataBridge(self.config)
return_value = [{
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.utcnow().isoformat()
}]
bridge.feeder.get_resource_items = MagicMock(return_value=return_value)
self.assertEqual(bridge.input_queue.qsize(), 0)
bridge.fill_input_queue()
self.assertEqual(bridge.input_queue.qsize(), 1)
self.assertEqual(bridge.input_queue.get(), return_value[0])
def test_get_tenders_list(self, mock_get_tenders):
tid = uuid.uuid4().hex
t_date_modified = datetime.datetime.utcnow().isoformat()
mock_get_tenders.return_value = [{
'id': tid,
'dateModified': t_date_modified
}]
bridge = EdgeDataBridge(self.config)
for tender_id, date_modified in bridge.get_teders_list():
self.assertEqual(tender_id, tid)
self.assertEqual(t_date_modified, date_modified)
def test_fill_input_queue(self):
bridge = EdgeDataBridge(self.config)
return_value = [
{'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.utcnow().isoformat()}
]
bridge.feeder.get_resource_items = MagicMock(return_value=return_value)
self.assertEqual(bridge.input_queue.qsize(), 0)
bridge.fill_input_queue()
self.assertEqual(bridge.input_queue.qsize(), 1)
self.assertEqual(bridge.input_queue.get(), return_value[0])
def test_send_bulk(self):
old_date_modified = datetime.datetime.utcnow().isoformat()
id_1 = uuid.uuid4().hex
date_modified_1 = datetime.datetime.utcnow().isoformat()
id_2 = uuid.uuid4().hex
date_modified_2 = datetime.datetime.utcnow().isoformat()
input_dict = {id_1: date_modified_1, id_2: date_modified_2}
return_value = [
munchify({
'id': id_1,
'key': date_modified_1
}),
munchify({
'id': id_2,
'key': old_date_modified
})
]
bridge = EdgeDataBridge(self.config)
bridge.db.view = MagicMock(return_value=return_value)
self.assertEqual(bridge.resource_items_queue.qsize(), 0)
bridge.send_bulk(input_dict)
self.assertEqual(bridge.resource_items_queue.qsize(), 1)
bridge.db.view.side_effect = [
Exception(), Exception(),
Exception('test')
]
input_dict = {}
with self.assertRaises(Exception) as e:
bridge.send_bulk(input_dict)
self.assertEqual(e.exception.message, 'test')
# historical
rev_1 = randint(10, 99)
input_dict = {id_1: rev_1}
bridge = EdgeDataBridge(self.config)
bridge.workers_config['historical'] = True
self.assertEqual(bridge.resource_items_queue.qsize(), 0)
bridge.send_bulk(input_dict)
self.assertEqual(bridge.resource_items_queue.qsize(), 1)
def test_bridge_stats(self):
bridge = EdgeDataBridge(self.config)
bridge.feeder = MagicMock()
bridge.feeder.queue.qsize.return_value = 44
bridge.feeder.backward_info = {
'last_response': datetime.datetime.now(),
'status': 1,
'resource_item_count': 13
}
bridge.feeder.forward_info = {
'last_response': datetime.datetime.now(),
'resource_item_count': 31
}
bridge.filler = MagicMock()
bridge.filler.exception = Exception('test_filler')
bridge.input_queue_filler = MagicMock()
bridge.input_queue_filler.exception = Exception('test_temp_filler')
sleep(1)
res = bridge.bridge_stats()
keys = ['resource_items_queue_size', 'retry_resource_items_queue_size', 'workers_count',
'api_clients_count', 'avg_request_duration', 'filter_workers_count',
'retry_workers_count', 'min_avg_request_duration', 'max_avg_request_duration']
for k, v in bridge.log_dict.items():
self.assertEqual(res[k], v)
for k in keys:
self.assertEqual(res[k], 0)
self.assertEqual(res['sync_backward_response_len'],
bridge.feeder.backward_info['resource_item_count'])
self.assertEqual(res['sync_forward_response_len'],
bridge.feeder.forward_info['resource_item_count'])
self.assertGreater(res['vms'], 0)
self.assertGreater(res['rss'], 0)
self.assertGreater(res['sync_backward_last_response'], 0)
self.assertGreater(res['sync_forward_last_response'], 0)
self.assertEqual(res['sync_queue'], 44)
self.assertEqual(res['resource'], bridge.workers_config['resource'])
self.assertEqual(res['_id'], bridge.workers_config['resource'])
self.assertNotEqual(res['time'], '')
bridge.feeder.backward_info['status'] = 0
bridge.api_clients_info['c1'] = {'request_durations': {datetime.datetime.now(): 1}}
bridge.api_clients_info['c2'] = {'request_durations': {datetime.datetime.now(): 2}}
res = bridge.bridge_stats()
self.assertEqual(res['sync_backward_last_response'], 0)
self.assertNotEqual(res['max_avg_request_duration'], 0)
self.assertNotEqual(res['min_avg_request_duration'], 0)
def test_queues_controller(self, mock_riw_spawn, mock_APIClient):
bridge = EdgeDataBridge(self.config)
bridge.resource_items_queue_size = 10
bridge.resource_items_queue = Queue(10)
for i in xrange(0, 10):
bridge.resource_items_queue.put('a')
self.assertEqual(len(bridge.workers_pool), 0)
self.assertEqual(bridge.resource_items_queue.qsize(), 10)
with patch('__builtin__.True', AlmostAlwaysTrue()):
bridge.queues_controller()
self.assertEqual(len(bridge.workers_pool), 1)
bridge.workers_pool.add(mock_riw_spawn)
self.assertEqual(len(bridge.workers_pool), 2)
for i in xrange(0, 10):
bridge.resource_items_queue.get()
with patch('__builtin__.True', AlmostAlwaysTrue()):
bridge.queues_controller()
self.assertEqual(len(bridge.workers_pool), 1)
self.assertEqual(bridge.resource_items_queue.qsize(), 0)
def test_resource_items_filter(self):
bridge = EdgeDataBridge(self.config)
date_modified_old = datetime.datetime.utcnow().isoformat()
date_modified_newest = datetime.datetime.utcnow().isoformat()
side_effect = [{'dateModified': date_modified_old},
{'dateModified': date_modified_newest}, None,
Exception('db exception')]
bridge.db.get = MagicMock(side_effect=side_effect)
result = bridge.resource_items_filter(uuid.uuid4().hex,
date_modified_newest)
self.assertEqual(result, True)
result = bridge.resource_items_filter(uuid.uuid4().hex,
date_modified_old)
self.assertEqual(result, False)
result = bridge.resource_items_filter(uuid.uuid4().hex,
date_modified_old)
self.assertEqual(result, True)
result = bridge.resource_items_filter(uuid.uuid4().hex,
date_modified_old)
self.assertEqual(result, True)
def test_run(self):
log_string = io.BytesIO()
stream_handler = logging.StreamHandler(log_string)
logger.addHandler(stream_handler)
bridge = EdgeDataBridge(self.config)
mock_tender = {'data': test_tender_data}
bridge.client.get_tender = MagicMock(return_value=mock_tender)
tid = uuid.uuid4().hex
t_date_modified = datetime.datetime.utcnow().isoformat()
mock_tender['data']['dateModified'] = t_date_modified
bridge.save_tender_in_db(tid, t_date_modified)
bridge.get_teders_list = MagicMock(
return_value=[[tid, datetime.datetime.utcnow().isoformat()]])
bridge.run()
x = log_string.getvalue().split('\n')
self.assertEqual(x[2], 'Start Edge Bridge')
self.assertEqual(x[3], 'Start data sync...')
del_tender = bridge.db.get(tid)
bridge.db.delete(del_tender)
logger.removeHandler(stream_handler)
log_string.close()
def test_init(self):
bridge = EdgeDataBridge(self.config)
self.assertIn('tenders_api_server', bridge.config['main'])
self.assertIn('tenders_api_version', bridge.config['main'])
self.assertIn('public_tenders_api_server', bridge.config['main'])
self.assertIn('couch_url', bridge.config['main'])
self.assertIn('public_db', bridge.config['main'])
self.assertEqual(
self.config['main']['couch_url'] + '/' +
self.config['main']['public_db'], bridge.couch_url)
test_config = {}
# Create EdgeDataBridge object with wrong config variable structure
test_config = {
'mani': {
'tenders_api_server':
'https://lb.api-sandbox.openprocurement.org',
'tenders_api_version': "0",
'public_tenders_api_server':
'https://lb.api-sandbox.openprocurement.org',
'couch_url': 'http://localhost:5984',
'public_db': 'public_db'
},
'version': 1
}
with self.assertRaises(DataBridgeConfigError):
EdgeDataBridge(test_config)
# Create EdgeDataBridge object without variable 'tenders_api_server' in config
del test_config['mani']
test_config['main'] = {}
with self.assertRaises(DataBridgeConfigError):
EdgeDataBridge(test_config)
with self.assertRaises(KeyError):
test_config['main']['tenders_api_server']
# Create EdgeDataBridge object with empty tenders_api_server
test_config['main']['tenders_api_server'] = ''
with self.assertRaises(DataBridgeConfigError):
EdgeDataBridge(test_config)
# Create EdgeDataBridge object with wrong tenders_api_server
test_config['main'][
'tenders_api_server'] = 'https://lb.api-sandbox.openprocurement.or'
with self.assertRaises(ConnectionError):
EdgeDataBridge(test_config)
test_config['main'][
'tenders_api_server'] = 'https://lb.api-sandbox.openprocurement.org'
# Create EdgeDataBridge object without 'couch_url'
with self.assertRaises(DataBridgeConfigError):
EdgeDataBridge(test_config)
with self.assertRaises(KeyError):
test_config['main']['couch_url']
# Create EdgeDataBridge object with empty 'couch_url'
test_config['main']['couch_url'] = ''
with self.assertRaises(DataBridgeConfigError):
EdgeDataBridge(test_config)
# Create EdgeDataBridge object with wrong 'couch_url'
test_config['main']['couch_url'] = 'http://localhost:598'
with self.assertRaises(DataBridgeConfigError):
EdgeDataBridge(test_config)
test_config['main']['couch_url'] = 'http://localhost:5984'
# Create EdgeDataBridge object without 'public_db'
with self.assertRaises(DataBridgeConfigError):
EdgeDataBridge(test_config)
with self.assertRaises(KeyError):
test_config['main']['public_db']
# Create EdgeDataBridge object with empty 'public_db'
test_config['main']['public_db'] = ''
with self.assertRaises(DataBridgeConfigError):
EdgeDataBridge(test_config)
# Create EdgeDataBridge object with wrong 'public_db'
test_config['main']['public_db'] = 'wrong_db'
with self.assertRaises(DataBridgeConfigError):
EdgeDataBridge(test_config)
test_config['main']['public_db'] = 'public_db'
test_config['main']['tenders_api_version'] = "0"
test_config['main'][
'public_tenders_api_server'] = 'https://lb.api-sandbox.openprocurement.org'
# Create EdgeDataBridge object with deleting config variables step by step
del test_config['main']['couch_url']
bridge = EdgeDataBridge(test_config)
self.assertEqual(type(bridge), EdgeDataBridge)
with self.assertRaises(KeyError):
test_config['main']['couch_url']
del bridge
del test_config['main']['tenders_api_version']
bridge = EdgeDataBridge(test_config)
self.assertEqual(type(bridge), EdgeDataBridge)
with self.assertRaises(KeyError):
test_config['main']['tenders_api_version']
del bridge
del test_config['main']['public_tenders_api_server']
bridge = EdgeDataBridge(test_config)
self.assertEqual(type(bridge), EdgeDataBridge)
with self.assertRaises(KeyError):
test_config['main']['public_tenders_api_server']
del bridge
del test_config['main']['public_db']
with self.assertRaises(DataBridgeConfigError):
EdgeDataBridge(test_config)
with self.assertRaises(KeyError):
test_config['main']['public_db']
def test_fill_api_clients_queue(self, mock_APIClient):
bridge = EdgeDataBridge(self.config)
self.assertEqual(bridge.api_clients_queue.qsize(), 0)
bridge.fill_api_clients_queue()
self.assertEqual(bridge.api_clients_queue.qsize(), bridge.workers_min)
def test_init(self):
self.config['main']['input_queue_size'] = -1
bridge = EdgeDataBridge(self.config)
self.assertEqual(bridge.input_queue_size, -1)
self.config['main']['input_queue_size'] = 1
bridge = EdgeDataBridge(self.config)
self.assertIn('resources_api_server', bridge.config['main'])
self.assertIn('resources_api_version', bridge.config['main'])
self.assertIn('public_resources_api_server', bridge.config['main'])
self.assertIn('couch_url', bridge.config['main'])
self.assertIn('db_name', bridge.config['main'])
self.assertEqual(self.config['main']['couch_url'], bridge.couch_url)
self.assertEqual(len(bridge.server.uuids()[0]), 32)
del bridge
self.config['main']['resource_items_queue_size'] = 101
self.config['main']['retry_resource_items_queue_size'] = 101
bridge = EdgeDataBridge(self.config)
self.config['main']['couch_url'] = 'http://127.0.0.1:5987'
with self.assertRaises(DataBridgeConfigError) as e:
bridge = EdgeDataBridge(self.config)
self.assertEqual(e.exception.message, 'Connection refused')
del bridge
self.config['main']['couch_url'] = 'http://127.0.0.1:5984'
try:
server = Server(self.config['main'].get('couch_url'))
del server[self.config['main']['db_name']]
except:
pass
test_config = {}
# Create EdgeDataBridge object with wrong config variable structure
test_config = {
'mani': {
'resources_api_server':
'https://lb.api-sandbox.openprocurement.org',
'resources_api_version': "0",
'public_resources_api_server':
'https://lb.api-sandbox.openprocurement.org',
'couch_url': 'http://localhost:5984',
'db_name': 'test_db',
'retrievers_params': {
'down_requests_sleep': 5,
'up_requests_sleep': 1,
'up_wait_sleep': 30,
'queue_size': 101
}
},
'version': 1
}
with self.assertRaises(DataBridgeConfigError) as e:
EdgeDataBridge(test_config)
self.assertEqual(e.exception.message, 'In config dictionary missed '
'section \'main\'')
# Create EdgeDataBridge object without variable 'resources_api_server' in config
del test_config['mani']
test_config['main'] = {
'retrievers_params': {
'down_requests_sleep': 5,
'up_requests_sleep': 1,
'up_wait_sleep': 30,
'queue_size': 101
}
}
with self.assertRaises(DataBridgeConfigError) as e:
EdgeDataBridge(test_config)
self.assertEqual(
e.exception.message, 'In config dictionary empty or '
'missing \'tenders_api_server\'')
with self.assertRaises(KeyError) as e:
test_config['main']['resources_api_server']
self.assertEqual(e.exception.message, 'resources_api_server')
# Create EdgeDataBridge object with empty resources_api_server
test_config['main']['resources_api_server'] = ''
with self.assertRaises(DataBridgeConfigError) as e:
EdgeDataBridge(test_config)
self.assertEqual(
e.exception.message, 'In config dictionary empty or '
'missing \'tenders_api_server\'')
# Create EdgeDataBridge object with invalid resources_api_server
test_config['main']['resources_api_server'] = 'my_server'
with self.assertRaises(DataBridgeConfigError) as e:
EdgeDataBridge(test_config)
self.assertEqual(e.exception.message, 'Invalid \'tenders_api_server\' '
'url.')
test_config['main'][
'resources_api_server'] = 'https://lb.api-sandbox.openprocurement.org'
test_config['main']['db_name'] = 'public'
test_config['main']['resources_api_version'] = "0"
test_config['main']['public_resources_api_server'] \
= 'https://lb.api-sandbox.openprocurement.org'
# Create EdgeDataBridge object with deleting config variables step by step
bridge = EdgeDataBridge(test_config)
self.assertEqual(type(bridge), EdgeDataBridge)
with self.assertRaises(KeyError) as e:
test_config['main']['couch_url']
self.assertEqual(e.exception.message, 'couch_url')
del bridge
del test_config['main']['resources_api_version']
bridge = EdgeDataBridge(test_config)
self.assertEqual(type(bridge), EdgeDataBridge)
with self.assertRaises(KeyError) as e:
test_config['main']['resources_api_version']
self.assertEqual(e.exception.message, 'resources_api_version')
del bridge
del test_config['main']['public_resources_api_server']
bridge = EdgeDataBridge(test_config)
self.assertEqual(type(bridge), EdgeDataBridge)
with self.assertRaises(KeyError) as e:
test_config['main']['public_resources_api_server']
self.assertEqual(e.exception.message, 'public_resources_api_server')
del bridge
server = Server(test_config['main'].get('couch_url')
or 'http://127.0.0.1:5984')
del server[test_config['main']['db_name']]
test_config['main']['retrievers_params']['up_wait_sleep'] = 0
with self.assertRaises(DataBridgeConfigError) as e:
EdgeDataBridge(test_config)
self.assertEqual(
e.exception.message, 'Invalid \'up_wait_sleep\' in '
'\'retrievers_params\'. Value must be grater than 30.')
def test_save_tender_in_db(self):
log_string = io.BytesIO()
stream_handler = logging.StreamHandler(log_string)
logger.addHandler(stream_handler)
bridge = EdgeDataBridge(self.config)
mock_tender = {'data': test_tender_data}
bridge.client.get_tender = MagicMock(return_value=mock_tender)
# Save tender
tid = uuid.uuid4().hex
t_date_modified = datetime.datetime.utcnow().isoformat()
mock_tender['data']['dateModified'] = t_date_modified
bridge.save_tender_in_db(tid, t_date_modified)
x = log_string.getvalue().split('\n')
self.assertEqual(x[1].strip(), 'Save tender ' + tid)
tender_in_db = bridge.db.get(tid)
self.assertEqual(tender_in_db.id, tid)
# Tender exist in db and not modified
result = bridge.save_tender_in_db(tid, t_date_modified)
self.assertEqual(result, None)
# Update tender
t_date_modified = datetime.datetime.utcnow().isoformat()
mock_tender['data']['dateModified'] = t_date_modified
bridge.save_tender_in_db(tid, t_date_modified)
x = log_string.getvalue().split('\n')
self.assertEqual(x[2].strip(), 'Update tender ' + tid)
updated_tender = bridge.db.get(tid)
self.assertEqual(updated_tender['dateModified'],
unicode(t_date_modified))
# Tender not found
bridge.client.get_tender = MagicMock(return_value=test_tender_data)
bridge.save_tender_in_db(tid, datetime.datetime.utcnow().isoformat())
x = log_string.getvalue().split('\n')
self.assertEqual(x[3].strip(), 'Tender ' + tid + ' not found')
bridge.db.delete(updated_tender)
# Saving tender with exception
bridge.client.get_tender = MagicMock(return_value=mock_tender)
bridge.config['main']['couch_url'] = ''
bridge.config['main']['public_db'] = ''
bridge.db = Database('bridge.couch_url',
session=Session(retry_delays=range(10)))
new_mock_tender = mock_tender
new_mock_tender['dateModified'] = datetime.datetime.utcnow().isoformat(
)
new_mock_tender['_rev'] = '2-' + uuid.uuid4().hex
bridge.db.get = MagicMock(return_value=new_mock_tender)
tid = uuid.uuid4().hex
bridge.save_tender_in_db(tid, datetime.datetime.utcnow().isoformat())
x = log_string.getvalue().split('\n')
self.assertEqual(
x[5].strip(), 'Saving tender ' + tid +
' fail with error (400, (u\'illegal_database_name\', u"Name: \'bridge.couch_url\'. Only lowercase characters (a-z), digits (0-9), and any of the characters _, $, (, ), +, -, and / are allowed. Must begin with a letter."))'
)
logger.removeHandler(stream_handler)
log_string.close()