def test_init(self):
stream = MagicMock()
stream.__iter__ = MagicMock()
stream.__next__ = MagicMock()
stream.next = MagicMock()
os_stream = OpenstackStream(stream, size=1234)
self.assertEqual(os_stream.size, 1234)
self.assertEqual(os_stream.__len__(), 1234)
self.assertEqual(os_stream.read(), stream.read(size=None))
self.assertEqual(os_stream.__iter__(), stream.__iter__())
self.assertEqual(os_stream.__next__(), stream.__next__())
self.assertEqual(os_stream.next(), stream.__next__())
def test_init(self):
stream = MagicMock()
stream.__iter__ = MagicMock()
stream.__next__ = MagicMock()
stream.next = MagicMock()
os_stream = OpenstackStream(stream, size=1234)
self.assertEquals(os_stream.size, 1234)
self.assertEquals(os_stream.__len__(), 1234)
self.assertEquals(os_stream.read(), stream.read(size=None))
self.assertEquals(os_stream.__iter__(), stream.__iter__())
self.assertEquals(os_stream.__next__(), stream.__next__())
self.assertEquals(os_stream.next(), stream.__next__())
def test_read_vehicle_data(self):
csv_data = []
csv_data.append(["", "", "", "", "", "", "", "", "", "", "", ""])
csv_data.append([
"", "Vehicle", "Time", "Speed", "", "Speed", "Vehicle", "Time", "",
"", "", ""
])
csv_data.append(
["", "1", "5:00", "25", "", "25", "2", "5:01", "", "", "", ""])
csv_data.append([
"", "Vehicle", "Time", "Speed", "", "Speed", "Vehicle", "Time", "",
"", "", ""
])
csv_data.append(
["", "3", "5:00", "25", "", "25", "4", "5:01", "", "", "", ""])
csv_reader = MagicMock()
csv_reader.__iter__ = MagicMock(return_value=iter(csv_data))
csv_reader.line_num = -1
file_header = {'date': '2/2/2016'}
data = read_vehicle_data(file_header, csv_reader)
self.assertEqual(len(data), 4)
vehicle_1 = data[0] if data[0].get('vehicle') == '1' else data[1]
self.assertEqual(vehicle_1['speed'], '25')
self.assertEqual(vehicle_1['vehicle'], '1')
self.assertEqual(vehicle_1['time'], '5:00')
self.assertEqual(vehicle_1['date'], '2/2/2016')
def patch_aws_encryption_sdk_stream(mocker):
mocker.patch.object(io_handling.aws_encryption_sdk, "stream")
mock_stream = MagicMock()
io_handling.aws_encryption_sdk.stream.return_value.__enter__.return_value = mock_stream
mock_stream.__iter__ = MagicMock(return_value=iter((sentinel.chunk_1,
sentinel.chunk_2)))
yield io_handling.aws_encryption_sdk.stream
def my_setup(self, children=None):
if children is None:
children = []
root = MagicMock(pk='root')
mock_children = MagicMock()
mock_children.__iter__ = MagicMock(return_value=iter(children))
mock_children.count.return_value = len(children)
root.get_children.return_value = mock_children
self.root = root
def test_export_seeds(self, mock_api_client_cls):
mock_warc_iter_cls = MagicMock()
mock_table_cls = MagicMock()
mock_table = MagicMock(spec=BaseTable)
mock_table_cls.side_effect = [mock_table]
mock_table.__iter__ = Mock(return_value=iter([[("key1", "key2"), ("k1v1", "k2v1"), ("k1v2", "k2v2")]]))
mock_api_client = MagicMock(spec=ApiClient)
mock_api_client_cls.side_effect = [mock_api_client]
mock_api_client.warcs.side_effect = [self.warcs]
export_message = {
"id": "test2",
"type": "test_user",
"seeds": [
{"id": "005b131f5f854402afa2b08a4b7ba960", "uid": "uid1"},
{"id": "105b131f5f854402afa2b08a4b7ba960", "uid": "uid2"},
],
"format": "csv",
"segment_size": None,
"path": self.export_path,
}
exporter = BaseExporter(
"http://test",
mock_warc_iter_cls,
mock_table_cls,
self.working_path,
warc_base_path=self.warc_base_path,
host="testhost",
)
exporter.routing_key = "export.start.test.test_user"
exporter.message = export_message
exporter.on_message()
mock_api_client_cls.assert_called_once_with("http://test")
mock_api_client.warcs.assert_called_once_with(
exclude_web=True,
collection_id=None,
seed_ids=["005b131f5f854402afa2b08a4b7ba960", "105b131f5f854402afa2b08a4b7ba960"],
harvest_date_start=None,
harvest_date_end=None,
)
mock_table_cls.assert_called_once_with(self.warc_filepaths, False, None, None, ["uid1", "uid2"], None)
self.assertTrue(exporter.result.success)
csv_filepath = os.path.join(self.export_path, "test2_001.csv")
self.assertTrue(os.path.exists(csv_filepath))
with open(csv_filepath, "r") as f:
lines = f.readlines()
self.assertEqual(3, len(lines))
def test_process_async_jobs_doesnt_send_email_for_bulk_created_course(self, filter_mock, client,
send_email_helper, **kwargs):
"""
test that the send_email_helper is not called for a bulk created course,
irrespective of the workflow_state
"""
mock_client_json(client, 'this can be anything')
iterable_ccmjob_mock = MagicMock()
filter_mock.return_value = iterable_ccmjob_mock
iterable_ccmjob_mock.__iter__ = Mock(return_value=iter([self.m_canvas_content_migration_job_with_bulk_id]))
start_job_with_noargs()
self.assertFalse(send_email_helper.called)
def test_process_async_jobs_on_failure_for_bulk_course_calls_tech_logger(self, mock_logger, filter_mock, client,
get_canvas_user_profile, send_email_helper,
tech_logger, **kwargs):
"""
test that the tech_logger is called even for bulk jobs when there is a failure.
"""
mock_client_json(client, 'failed')
iterable_ccmjob_mock = MagicMock()
filter_mock.return_value = iterable_ccmjob_mock
iterable_ccmjob_mock.__iter__ = Mock(return_value=iter([self.m_canvas_content_migration_job_with_bulk_id]))
start_job_with_noargs()
self.assertTrue(mock_logger.called)
def test_incoming_push_pop(self):
Resource = Mock(name="resource")
ctx = mock_context()
field = IterableField(attribute="foo", resource_class=Resource)
source_dict = {"foo": {"bar": 20}}
target_object = MagicMock()
target_object.__iter__ = iter([])
field.handle_incoming(ctx, source_dict, target_object)
ctx.assert_has_calls([mock.call.push(target_object), mock.call.pop()])
def test_process_async_jobs_on_failure_for_bulk_course_calls_tech_logger(
self, mock_logger, filter_mock, client, get_canvas_user_profile,
send_email_helper, tech_logger, **kwargs):
"""
test that the tech_logger is called even for bulk jobs when there is a failure.
"""
mock_client_json(client, 'failed')
iterable_ccmjob_mock = MagicMock()
filter_mock.return_value = iterable_ccmjob_mock
iterable_ccmjob_mock.__iter__ = Mock(return_value=iter(
[self.m_canvas_content_migration_job_with_bulk_id]))
start_job_with_noargs()
self.assertTrue(mock_logger.called)
def test_process_async_jobs_doesnt_send_email_for_bulk_created_course(
self, filter_mock, client, send_email_helper, **kwargs):
"""
test that the send_email_helper is not called for a bulk created course,
irrespective of the workflow_state
"""
mock_client_json(client, 'this can be anything')
iterable_ccmjob_mock = MagicMock()
filter_mock.return_value = iterable_ccmjob_mock
iterable_ccmjob_mock.__iter__ = Mock(return_value=iter(
[self.m_canvas_content_migration_job_with_bulk_id]))
start_job_with_noargs()
self.assertFalse(send_email_helper.called)
def test_process_notification(self, mock_init):
"""process_notification should get the target controller, call process_notification on it with the notification
name, data and request, then return a list of queued events"""
notification_data = {'data': ['some', 'data']}
notification_centre = MagicMock()
self.mock_vs.notification_centre = notification_centre
notifications = [{'name': 'message1'}, {'name': 'message2'}]
notification_centre.__iter__ = MagicMock(return_value=iter(notifications))
self.mock_controller.handle_notification = MagicMock()
queued_notifications = dispatch_notification('controller.path', 'vs_id', 'notification-name', notification_data,
self.session, 'request')
self.mock_vsm.get_view_state.assert_called_with('vs_id', no_create=True)
self.mock_vs.controller_from_path.assert_called_with('controller.path')
self.mock_controller.handle_notification.assert_called_with('notification-name', notification_data, 'request')
self.assertEqual(queued_notifications, notifications)
def test_log_invalid_vehicle(self):
csv_data = []
csv_data.append(["", "Vehicle", "Time", "Speed", ""])
csv_data.append(["", "1", "5:00", "bogus", ""])
csv_reader = MagicMock()
csv_reader.__iter__ = MagicMock(return_value=iter(csv_data))
csv_reader.line_num = 99
file_header = {'date': '2/2/2016', "filename": "fakename"}
logging_mock = MagicMock()
with patch("logging.info", logging_mock):
read_vehicle_data(file_header, csv_reader)
self.assertEqual(logging_mock.call_count, 1)
my_log_str = logging_mock.call_args[0][0]
self.assertTrue("fakename" in my_log_str)
self.assertTrue("99" in my_log_str)
self.assertTrue("bogus" in my_log_str)
def test_kwargs_pass(self, mock_init):
"""dispatch_notification should pass kwargs to the target's handle_notification."""
notifications = [{'name': 'load:scroll_top', 'target': 'test'}]
notification_centre = MagicMock()
self.mock_vs.notification_centre = notification_centre
notification_centre.__iter__ = MagicMock(return_value=iter(notifications))
mock_controller = MagicMock()
mock_controller.handle_notification = MagicMock()
mock_controller.render = MagicMock(return_value='html')
mock_controller.class_map_tree = MagicMock(return_value='class_map')
self.mock_vs.controller_from_path = MagicMock(return_value=mock_controller)
dispatch_notification('controller.path', 'vs_id', 'notification-name', {}, self.session, 'request',
environment='env')
mock_controller.handle_notification.assert_called_with('notification-name', {}, 'request', environment='env')
mock_controller.render.assert_called_with(request='request', environment='env')
def test_log_invalid_vehicle(self):
csv_data = []
csv_data.append(["", "Vehicle", "Time", "Speed", ""])
csv_data.append(["", "1", "5:00", "bogus", ""])
csv_reader = MagicMock()
csv_reader.__iter__ = MagicMock(return_value=iter(csv_data))
csv_reader.line_num = 99
file_header = {'date': '2/2/2016', "filename": "fakename"}
logging_mock = MagicMock()
with patch("logging.info", logging_mock):
read_vehicle_data(file_header, csv_reader)
self.assertEqual(logging_mock.call_count, 1)
my_log_str = logging_mock.call_args[0][0]
self.assertTrue("fakename" in my_log_str)
self.assertTrue("99" in my_log_str)
self.assertTrue("bogus" in my_log_str)
def test_get_success(self):
mock_domain_one = {
'owner': 0,
'status': 'active',
'domain': 'vegadns.org',
'domain_id': 1
}
mock_model_one = MagicMock()
mock_model_one.to_clean_dict = MagicMock(return_value=mock_domain_one)
mock_domain_two = {
'owner': 0,
'status': 'active',
'domain': 'vegadns.net',
'domain_id': 2
}
mock_model_two = MagicMock()
mock_model_two.to_clean_dict = MagicMock(return_value=mock_domain_two)
domain_list = MagicMock()
domain_list.__iter__ = MagicMock(
return_value=iter([mock_model_one, mock_model_two])
)
domain_list.count = lambda: 2
vegadns.api.endpoints.domains.Domains.get_domain_list = MagicMock(
return_value=domain_list
)
self.mock_auth('*****@*****.**', 'test')
response = self.open_with_basic_auth(
'/domains',
'GET',
'*****@*****.**',
'test'
)
self.assertEqual(response.status, "200 OK")
decoded = json.loads(response.data)
self.assertEqual(decoded['status'], "ok")
self.assertEqual(decoded['domains'][0]['domain_id'], 1)
self.assertEqual(decoded['domains'][0]['domain'], 'vegadns.org')
self.assertEqual(decoded['domains'][0]['owner'], 0)
self.assertEqual(decoded['domains'][1]['domain_id'], 2)
self.assertEqual(decoded['domains'][1]['domain'], 'vegadns.net')
self.assertEqual(decoded['domains'][1]['owner'], 0)
def test_update(self):
manager = Manager(connection=self.connection)
doc = MagicMock()
doc.__iter__.return_value = [('name', 'John'), ('age', 77)]
data_list = MagicMock()
data_list.__iter__ = Mock(return_value=iter([doc, doc]))
self.collection.find.return_value = data_list
updates = {'age': 77}
lookup_keys = {'name': 'John'}
updated = manager.update(lookup_keys, updates)
doc.update.assert_called_with(updates)
doc.save.assert_called()
self.assertEqual(updated, [doc, doc])
def test_incoming_push_pop(self):
Resource = Mock(name='resource')
ctx = mock_context()
field = IterableField(attribute='foo', resource_class=Resource)
source_dict = {
'foo': {'bar': 20},
}
target_object = MagicMock()
target_object.__iter__ = iter([])
field.handle_incoming(ctx, source_dict, target_object)
ctx.assert_has_calls([
mock.call.push(target_object),
mock.call.pop(),
])
def test_read_vehicle_data(self):
csv_data = []
csv_data.append(["", "", "", "", "", "", "", "", "", "", "", ""])
csv_data.append(["", "Vehicle", "Time", "Speed", "", "Speed", "Vehicle", "Time", "", "", "", ""])
csv_data.append(["", "1", "5:00", "25", "", "25", "2", "5:01", "", "", "", ""])
csv_data.append(["", "Vehicle", "Time", "Speed", "", "Speed", "Vehicle", "Time", "", "", "", ""])
csv_data.append(["", "3", "5:00", "25", "", "25", "4", "5:01", "", "", "", ""])
csv_reader = MagicMock()
csv_reader.__iter__ = MagicMock(return_value=iter(csv_data))
csv_reader.line_num = -1
file_header = {'date': '2/2/2016'}
data = read_vehicle_data(file_header, csv_reader)
self.assertEqual(len(data), 4)
vehicle_1 = data[0] if data[0].get('vehicle') == '1' else data[1]
self.assertEqual(vehicle_1['speed'], '25')
self.assertEqual(vehicle_1['vehicle'], '1')
self.assertEqual(vehicle_1['time'], '5:00')
self.assertEqual(vehicle_1['date'], '2/2/2016')
def test_controller_data_added_on_load(self, mock_init):
"""On a load notification the NotificationCentre gets the target controller that is to be loaded and bundles its
data into the request, unless it has data already."""
notifications = [{
'name': 'load:scroll_top',
'target': 'test'
}, {
'name': 'load',
'target': 'test'
}, {
'name': 'load:scroll_top',
'target': 'test',
'data': 'existing'
}, {
'name': 'load',
'target': 'test',
'data': ''
}]
notification_centre = MagicMock()
self.mock_vs.notification_centre = notification_centre
notification_centre.__iter__ = MagicMock(
return_value=iter(notifications))
mock_controller = MagicMock()
mock_controller.render = MagicMock(return_value='html')
mock_controller.class_map_tree = MagicMock(return_value='class_map')
self.mock_vs.controller_from_path = MagicMock(
return_value=mock_controller)
queued_notifications = dispatch_notification('controller.path',
'vs_id',
'notification-name', {},
self.session, 'request')
self.assertEqual(len(queued_notifications), 4)
self.assertEqual(queued_notifications[0]['data'], {
'html': 'html',
'class_map': 'class_map'
})
self.assertEqual(queued_notifications[1]['data'], {
'html': 'html',
'class_map': 'class_map'
})
self.assertEqual(queued_notifications[2]['data'], 'existing')
self.assertEqual(queued_notifications[3]['data'], '')
def test_process_notification(self, mock_init):
"""process_notification should get the target controller, call process_notification on it with the notification
name, data and request, then return a list of queued events"""
notification_data = {'data': ['some', 'data']}
notification_centre = MagicMock()
self.mock_vs.notification_centre = notification_centre
notifications = [{'name': 'message1'}, {'name': 'message2'}]
notification_centre.__iter__ = MagicMock(
return_value=iter(notifications))
self.mock_controller.handle_notification = MagicMock()
queued_notifications = dispatch_notification('controller.path',
'vs_id',
'notification-name',
notification_data,
self.session, 'request')
self.mock_vsm.get_view_state.assert_called_with('vs_id',
no_create=True)
self.mock_vs.controller_from_path.assert_called_with('controller.path')
self.mock_controller.handle_notification.assert_called_with(
'notification-name', notification_data, 'request')
self.assertEqual(queued_notifications, notifications)
def test_process_empty_formdata_exception(self, reset_honeypot):
field = self.get_field()
formdata = MagicMock()
formdata.__getitem__.side_effect = StopIteration
formdata.__iter__ = Mock(return_value=iter([HONEY_POT_PREFIX + '_1']))
unbound_field = Mock(name='UnboundField')
bound_field = Mock(name='BoundField')
unbound_field.bind.return_value = bound_field
field.unbound_field = unbound_field
field.process(formdata)
actual_process = bound_field.process.call_args_list
expected_process = [
call(formdata, u''),
]
self.assertEqual(expected_process, actual_process)
def test_controller_data_added_on_load(self, mock_init):
"""On a load notification the NotificationCentre gets the target controller that is to be loaded and bundles its
data into the request, unless it has data already."""
notifications = [{'name': 'load:scroll_top', 'target': 'test'}, {'name': 'load', 'target': 'test'},
{'name': 'load:scroll_top', 'target': 'test', 'data': 'existing'},
{'name': 'load', 'target': 'test', 'data': ''}]
notification_centre = MagicMock()
self.mock_vs.notification_centre = notification_centre
notification_centre.__iter__ = MagicMock(return_value=iter(notifications))
mock_controller = MagicMock()
mock_controller.render = MagicMock(return_value='html')
mock_controller.class_map_tree = MagicMock(return_value='class_map')
self.mock_vs.controller_from_path = MagicMock(return_value=mock_controller)
queued_notifications = dispatch_notification('controller.path', 'vs_id', 'notification-name', {},
self.session, 'request')
self.assertEqual(len(queued_notifications), 4)
self.assertEqual(queued_notifications[0]['data'], {'html': 'html', 'class_map': 'class_map'})
self.assertEqual(queued_notifications[1]['data'], {'html': 'html', 'class_map': 'class_map'})
self.assertEqual(queued_notifications[2]['data'], 'existing')
self.assertEqual(queued_notifications[3]['data'], '')
def test_kwargs_pass(self, mock_init):
"""dispatch_notification should pass kwargs to the target's handle_notification."""
notifications = [{'name': 'load:scroll_top', 'target': 'test'}]
notification_centre = MagicMock()
self.mock_vs.notification_centre = notification_centre
notification_centre.__iter__ = MagicMock(
return_value=iter(notifications))
mock_controller = MagicMock()
mock_controller.handle_notification = MagicMock()
mock_controller.render = MagicMock(return_value='html')
mock_controller.class_map_tree = MagicMock(return_value='class_map')
self.mock_vs.controller_from_path = MagicMock(
return_value=mock_controller)
dispatch_notification('controller.path',
'vs_id',
'notification-name', {},
self.session,
'request',
environment='env')
mock_controller.handle_notification.assert_called_with(
'notification-name', {}, 'request', environment='env')
mock_controller.render.assert_called_with(request='request',
environment='env')
def test_export_dehydrate(self, mock_producer, mock_api_client_cls):
mock_warc_iter_cls = MagicMock()
mock_table_cls = MagicMock()
mock_table = MagicMock(spec=BaseTable)
mock_table_cls.side_effect = [mock_table]
mock_table.__iter__ = Mock(return_value=iter([
[("key1", "key2"), ("k1v1", "k2v1"), ("k1v2", "k2v2")],
]))
mock_table.id_field.return_value = "key2"
mock_api_client = MagicMock(spec=ApiClient)
mock_api_client_cls.side_effect = [mock_api_client]
mock_api_client.warcs.side_effect = [self.warcs]
mock_connection = MagicMock(spec=Connection)
mock_exchange = MagicMock(spec=Exchange)
mock_exchange.name = "test exchange"
export_message = {
"id": "test1",
"type": "test_user",
"collection": {
"id": "005b131f5f854402afa2b08a4b7ba960"
},
"format": "dehydrate",
"segment_size": None,
"path": self.export_path,
}
exporter = BaseExporter("http://test",
mock_warc_iter_cls,
mock_table_cls,
self.working_path,
warc_base_path=self.warc_base_path,
host="testhost")
exporter.mq_config = True
exporter._producer_connection = mock_connection
exporter.exchange = mock_exchange
exporter.routing_key = "export.start.test.test_user"
exporter.message = export_message
exporter.on_message()
mock_api_client_cls.assert_called_once_with("http://test")
mock_api_client.warcs.assert_called_once_with(
collection_id="005b131f5f854402afa2b08a4b7ba960",
seed_ids=[],
harvest_date_end=None,
harvest_date_start=None)
mock_table_cls.assert_called_once_with(self.warc_filepaths, False,
None, None, [], None)
self.assertTrue(exporter.result.success)
txt_filepath = os.path.join(self.export_path, "test1_001.txt")
self.assertTrue(os.path.exists(txt_filepath))
with open(txt_filepath, "r") as f:
lines = f.readlines()
self.assertEqual(2, len(lines))
self.assertEqual("k2v1\n", lines[0])
name, _, kwargs = mock_producer.mock_calls[3]
self.assertEqual("export.status.test.test_user", kwargs["routing_key"])
export_status_message = kwargs["body"]
self.assertEqual("completed success", export_status_message["status"])
self.assertEqual("test1", export_status_message["id"])
def test_export_seeds(self, mock_api_client_cls):
mock_warc_iter_cls = MagicMock()
mock_table_cls = MagicMock()
mock_table = MagicMock(spec=BaseTable)
mock_table_cls.side_effect = [mock_table]
mock_table.__iter__ = Mock(return_value=iter([
[("key1", "key2"), ("k1v1", "k2v1"), ("k1v2", "k2v2")],
]))
mock_api_client = MagicMock(spec=ApiClient)
mock_api_client_cls.side_effect = [mock_api_client]
mock_api_client.warcs.side_effect = [self.warcs]
export_message = {
"id":
"test2",
"type":
"test_user",
"seeds": [
{
"id": "005b131f5f854402afa2b08a4b7ba960",
"uid": "uid1"
},
{
"id": "105b131f5f854402afa2b08a4b7ba960",
"uid": "uid2"
},
],
"format":
"csv",
"segment_size":
None,
"path":
self.export_path,
}
exporter = BaseExporter("http://test",
mock_warc_iter_cls,
mock_table_cls,
self.working_path,
warc_base_path=self.warc_base_path,
host="testhost")
exporter.routing_key = "export.start.test.test_user"
exporter.message = export_message
exporter.on_message()
mock_api_client_cls.assert_called_once_with("http://test")
mock_api_client.warcs.assert_called_once_with(
collection_id=None,
seed_ids=[
"005b131f5f854402afa2b08a4b7ba960",
"105b131f5f854402afa2b08a4b7ba960"
],
harvest_date_start=None,
harvest_date_end=None)
mock_table_cls.assert_called_once_with(self.warc_filepaths, False,
None, None, ["uid1", "uid2"],
None)
self.assertTrue(exporter.result.success)
csv_filepath = os.path.join(self.export_path, "test2_001.csv")
self.assertTrue(os.path.exists(csv_filepath))
with open(csv_filepath, "r") as f:
lines = f.readlines()
self.assertEqual(3, len(lines))
def test_export_collection(self, mock_producer, mock_api_client_cls):
mock_warc_iter_cls = MagicMock()
mock_table_cls = MagicMock()
mock_table = MagicMock(spec=BaseTable)
mock_table_cls.side_effect = [mock_table]
mock_table.__iter__ = Mock(return_value=iter([
[("key1", "key2"), ("k1v1", "k2v1"), ("k1v2", "k2v2")],
]))
mock_api_client = MagicMock(spec=ApiClient)
mock_api_client_cls.side_effect = [mock_api_client]
mock_api_client.warcs.side_effect = [self.warcs]
mock_connection = MagicMock(spec=Connection)
mock_exchange = MagicMock(spec=Exchange)
mock_exchange.name = "test exchange"
item_date_start = "2007-01-25T12:00:00Z"
item_datetime_start = iso8601.parse_date(item_date_start)
item_date_end = "2008-02-25T12:00:00Z"
item_datetime_end = iso8601.parse_date(item_date_end)
harvest_date_start = "2007-03-25T12:00:00Z"
harvest_date_end = "2008-04-25T12:00:00Z"
export_message = {
"id": "test1",
"type": "test_user",
"collection": {
"id": "005b131f5f854402afa2b08a4b7ba960"
},
"format": "csv",
"segment_size": None,
"path": self.export_path,
"dedupe": True,
"item_date_start": item_date_start,
"item_date_end": item_date_end,
"harvest_date_start": harvest_date_start,
"harvest_date_end": harvest_date_end,
}
exporter = BaseExporter("http://test",
mock_warc_iter_cls,
mock_table_cls,
self.working_path,
warc_base_path=self.warc_base_path,
host="testhost")
exporter.mq_config = True
exporter._producer_connection = mock_connection
exporter.exchange = mock_exchange
exporter.routing_key = "export.start.test.test_user"
exporter.message = export_message
exporter.on_message()
mock_api_client_cls.assert_called_once_with("http://test")
mock_api_client.warcs.assert_called_once_with(
collection_id="005b131f5f854402afa2b08a4b7ba960",
seed_ids=[],
harvest_date_start=harvest_date_start,
harvest_date_end=harvest_date_end)
mock_table_cls.assert_called_once_with(self.warc_filepaths, True,
item_datetime_start,
item_datetime_end, [], None)
self.assertTrue(exporter.result.success)
csv_filepath = os.path.join(self.export_path, "test1_001.csv")
self.assertTrue(os.path.exists(csv_filepath))
with open(csv_filepath, "r") as f:
lines = f.readlines()
self.assertEqual(3, len(lines))
name, _, kwargs = mock_producer.mock_calls[1]
self.assertEqual("export.status.test.test_user", kwargs["routing_key"])
export_status_message = kwargs["body"]
self.assertEqual("running", export_status_message["status"])
self.assertTrue(
iso8601.parse_date(export_status_message["date_started"]))
self.assertEqual("test1", export_status_message["id"])
self.assertEqual("Base Exporter", export_status_message["service"])
self.assertEqual("testhost", export_status_message["host"])
self.assertTrue(export_status_message["instance"])
name, _, kwargs = mock_producer.mock_calls[3]
self.assertEqual("export.status.test.test_user", kwargs["routing_key"])
export_status_message = kwargs["body"]
self.assertEqual("completed success", export_status_message["status"])
self.assertTrue(
iso8601.parse_date(export_status_message["date_started"]))
self.assertTrue(iso8601.parse_date(
export_status_message["date_ended"]))
self.assertEqual("test1", export_status_message["id"])
self.assertEqual("Base Exporter", export_status_message["service"])
self.assertEqual("testhost", export_status_message["host"])
self.assertTrue(export_status_message["instance"])
def test_export_dehydrate(self, mock_producer_cls, mock_api_client_cls):
mock_warc_iter_cls = MagicMock()
mock_table_cls = MagicMock()
mock_table = MagicMock(spec=BaseTable)
mock_table_cls.side_effect = [mock_table]
mock_table.__iter__ = Mock(return_value=iter([[("key1", "key2"), ("k1v1", "k2v1"), ("k1v2", "k2v2")]]))
mock_table.id_field.return_value = "key2"
mock_api_client = MagicMock(spec=ApiClient)
mock_api_client_cls.side_effect = [mock_api_client]
mock_api_client.warcs.side_effect = [self.warcs]
mock_connection = MagicMock(spec=Connection)
mock_exchange = MagicMock(spec=Exchange)
mock_exchange.name = "test exchange"
mock_producer = MagicMock(spec=Producer)
mock_producer_cls.return_value = mock_producer
export_message = {
"id": "test1",
"type": "test_user",
"collection": {"id": "005b131f5f854402afa2b08a4b7ba960"},
"format": "dehydrate",
"segment_size": None,
"path": self.export_path,
}
exporter = BaseExporter(
"http://test",
mock_warc_iter_cls,
mock_table_cls,
self.working_path,
warc_base_path=self.warc_base_path,
host="testhost",
)
exporter.mq_config = True
exporter._producer_connection = mock_connection
exporter.exchange = mock_exchange
exporter.routing_key = "export.start.test.test_user"
exporter.message = export_message
exporter.on_message()
mock_api_client_cls.assert_called_once_with("http://test")
mock_api_client.warcs.assert_called_once_with(
exclude_web=True,
collection_id="005b131f5f854402afa2b08a4b7ba960",
seed_ids=[],
harvest_date_end=None,
harvest_date_start=None,
)
mock_table_cls.assert_called_once_with(self.warc_filepaths, False, None, None, [], None)
self.assertTrue(exporter.result.success)
txt_filepath = os.path.join(self.export_path, "test1_001.txt")
self.assertTrue(os.path.exists(txt_filepath))
with open(txt_filepath, "r") as f:
lines = f.readlines()
self.assertEqual(2, len(lines))
self.assertEqual("k2v1\n", lines[0])
name, _, kwargs = mock_producer.mock_calls[1]
self.assertEqual("publish", name)
self.assertEqual("export.status.test.test_user", kwargs["routing_key"])
export_status_message = kwargs["body"]
self.assertEqual("completed success", export_status_message["status"])
self.assertEqual("test1", export_status_message["id"])
def test_traffic(monkeypatch):
r53conn = Mock(name='r53conn')
monkeypatch.setattr('boto.ec2.connect_to_region', MagicMock())
monkeypatch.setattr('boto.ec2.elb.connect_to_region', MagicMock())
monkeypatch.setattr('boto.cloudformation.connect_to_region', MagicMock())
monkeypatch.setattr('boto.route53.connect_to_region', r53conn)
stacks = [
StackVersion('myapp', 'v1', 'myapp.example.org', 'some-lb'),
StackVersion('myapp', 'v2', 'myapp.example.org', 'another-elb'),
StackVersion('myapp', 'v3', 'myapp.example.org', 'elb-3'),
StackVersion('myapp', 'v4', 'myapp.example.org', 'elb-4'),
]
monkeypatch.setattr('senza.traffic.get_stack_versions', MagicMock(return_value=stacks))
# start creating mocking of the route53 record sets and Application Versions
# this is a lot of dirty and nasty code. Please, somebody help this code.
def record(dns_identifier, weight):
rec = MagicMock(name=dns_identifier + '-record',
weight=weight,
identifier=dns_identifier,
type='CNAME')
rec.name = 'myapp.example.org.'
return rec
rr = MagicMock()
records = collections.OrderedDict()
for ver, percentage in [('v1', 60),
('v2', 30),
('v3', 10),
('v4', 0)]:
dns_identifier = 'myapp-{}'.format(ver)
records[dns_identifier] = record(dns_identifier, percentage * PERCENT_RESOLUTION)
rr.__iter__ = lambda x: iter(records.values())
def add_change(op, dns_name, rtype, ttl, identifier, weight):
if op == 'CREATE':
x = MagicMock(weight=weight, identifier=identifier)
x.name = "myapp.example.org."
x.type = "CNAME"
records[identifier] = x
return MagicMock(name='change')
def add_change_record(op, record):
if op == 'DELETE':
records[record.identifier].weight = 0
elif op == 'UPSERT':
records[record.identifier].weight = record.weight
rr.add_change = add_change
rr.add_change_record = add_change_record
r53conn().get_zone().get_records.return_value = rr
runner = CliRunner()
common_opts = ['traffic', '--region=my-region', 'myapp']
def run(opts):
result = runner.invoke(cli, common_opts + opts, catch_exceptions=False)
return result
def weights():
return [r.weight for r in records.values()]
with runner.isolated_filesystem():
run(['v4', '100'])
assert weights() == [0, 0, 0, 200]
run(['v3', '10'])
assert weights() == [0, 0, 20, 180]
run(['v2', '0.5'])
assert weights() == [0, 1, 20, 179]
run(['v1', '1'])
assert weights() == [2, 1, 19, 178]
run(['v4', '95'])
assert weights() == [1, 1, 13, 185]
run(['v4', '100'])
assert weights() == [0, 0, 0, 200]
run(['v4', '10'])
assert weights() == [0, 0, 0, 200]
run(['v4', '0'])
assert weights() == [0, 0, 0, 0]
def mock_open(mock=None, read_data='', match=None):
'''
A helper function to create a mock to replace the use of `open`. It works
for `open` called directly or used as a context manager.
The `mock` argument is the mock object to configure. If `None` (the
default) then a `MagicMock` will be created for you, with the API limited
to methods or attributes available on standard file handles.
`read_data` is a string for the `read` methoddline`, and `readlines` of the
file handle to return. This is an empty string by default.
If passed, `match` can be either a string or an iterable containing
patterns to attempt to match using fnmatch.fnmatch(). A side_effect will be
added to the mock object returned, which will cause an IOError(2, 'No such
file or directory') to be raised when the file path is not a match. This
allows you to make your mocked filehandle only work for certain file paths.
'''
# Normalize read_data, Python 2 filehandles should never produce unicode
# types on read.
if six.PY2:
read_data = salt.utils.stringutils.to_str(read_data)
def _readlines_side_effect(*args, **kwargs):
if handle.readlines.return_value is not None:
return handle.readlines.return_value
return list(_data)
def _read_side_effect(*args, **kwargs):
if handle.read.return_value is not None:
return handle.read.return_value
joiner = b'' if isinstance(read_data, six.binary_type) else ''
return joiner.join(_data)
def _readline_side_effect():
if handle.readline.return_value is not None:
while True:
yield handle.readline.return_value
for line in _data:
yield line
global file_spec
if file_spec is None:
if six.PY3:
import _io
file_spec = list(set(dir(_io.TextIOWrapper)).union(set(dir(_io.BytesIO))))
else:
file_spec = file # pylint: disable=undefined-variable
if mock is None:
mock = MagicMock(name='open', spec=open)
handle = MagicMock(spec=file_spec)
handle.__enter__.return_value = handle
_data = _iterate_read_data(read_data)
handle.write.return_value = None
handle.read.return_value = None
handle.readline.return_value = None
handle.readlines.return_value = None
# Support iteration via for loop
handle.__iter__ = lambda x: _readline_side_effect()
# This is salt specific and not in the upstream mock
handle.read.side_effect = _read_side_effect
handle.readline.side_effect = _readline_side_effect()
handle.readlines.side_effect = _readlines_side_effect
if match is not None:
if isinstance(match, six.string_types):
match = [match]
def fopen_side_effect(name, *args, **kwargs):
for pat in match:
if fnmatch.fnmatch(name, pat):
return DEFAULT
raise IOError(errno.ENOENT, 'No such file or directory', name)
mock.side_effect = fopen_side_effect
mock.return_value = handle
return mock
def test_export_collection(self, mock_producer_cls, mock_api_client_cls):
mock_warc_iter_cls = MagicMock()
mock_table_cls = MagicMock()
mock_table = MagicMock(spec=BaseTable)
mock_table_cls.side_effect = [mock_table]
mock_table.__iter__ = Mock(return_value=iter([[("key1", "key2"), ("k1v1", "k2v1"), ("k1v2", "k2v2")]]))
mock_api_client = MagicMock(spec=ApiClient)
mock_api_client_cls.side_effect = [mock_api_client]
mock_api_client.warcs.side_effect = [self.warcs]
mock_connection = MagicMock(spec=Connection)
mock_exchange = MagicMock(spec=Exchange)
mock_exchange.name = "test exchange"
mock_producer = MagicMock(spec=Producer)
mock_producer_cls.return_value = mock_producer
item_date_start = "2007-01-25T12:00:00Z"
item_datetime_start = iso8601.parse_date(item_date_start)
item_date_end = "2008-02-25T12:00:00Z"
item_datetime_end = iso8601.parse_date(item_date_end)
harvest_date_start = "2007-03-25T12:00:00Z"
harvest_date_end = "2008-04-25T12:00:00Z"
export_message = {
"id": "test1",
"type": "test_user",
"collection": {"id": "005b131f5f854402afa2b08a4b7ba960"},
"format": "csv",
"segment_size": None,
"path": self.export_path,
"dedupe": True,
"item_date_start": item_date_start,
"item_date_end": item_date_end,
"harvest_date_start": harvest_date_start,
"harvest_date_end": harvest_date_end,
}
exporter = BaseExporter(
"http://test",
mock_warc_iter_cls,
mock_table_cls,
self.working_path,
warc_base_path=self.warc_base_path,
host="testhost",
)
exporter.mq_config = True
exporter._producer_connection = mock_connection
exporter.exchange = mock_exchange
exporter.routing_key = "export.start.test.test_user"
exporter.message = export_message
exporter.on_message()
mock_api_client_cls.assert_called_once_with("http://test")
mock_api_client.warcs.assert_called_once_with(
exclude_web=True,
collection_id="005b131f5f854402afa2b08a4b7ba960",
seed_ids=[],
harvest_date_start=harvest_date_start,
harvest_date_end=harvest_date_end,
)
mock_table_cls.assert_called_once_with(
self.warc_filepaths, True, item_datetime_start, item_datetime_end, [], None
)
self.assertTrue(exporter.result.success)
csv_filepath = os.path.join(self.export_path, "test1_001.csv")
self.assertTrue(os.path.exists(csv_filepath))
with open(csv_filepath, "r") as f:
lines = f.readlines()
self.assertEqual(3, len(lines))
name, _, kwargs = mock_producer.mock_calls[0]
self.assertEqual("publish", name)
self.assertEqual("export.status.test.test_user", kwargs["routing_key"])
export_status_message = kwargs["body"]
self.assertEqual("running", export_status_message["status"])
self.assertTrue(iso8601.parse_date(export_status_message["date_started"]))
self.assertEqual("test1", export_status_message["id"])
self.assertEqual("Base Exporter", export_status_message["service"])
self.assertEqual("testhost", export_status_message["host"])
self.assertTrue(export_status_message["instance"])
name, _, kwargs = mock_producer.mock_calls[1]
self.assertEqual("publish", name)
self.assertEqual("export.status.test.test_user", kwargs["routing_key"])
export_status_message = kwargs["body"]
self.assertEqual("completed success", export_status_message["status"])
self.assertTrue(iso8601.parse_date(export_status_message["date_started"]))
self.assertTrue(iso8601.parse_date(export_status_message["date_ended"]))
self.assertEqual("test1", export_status_message["id"])
self.assertEqual("Base Exporter", export_status_message["service"])
self.assertEqual("testhost", export_status_message["host"])
self.assertTrue(export_status_message["instance"])
