def test_processing(self):
factor = 3
for _ in range(0, settings['csv_bulk_threshold'] * factor):
scrape_wrapper = MqScrapeWrapper()
scrape_wrapper.scrape_state_doc = create_scrape_state()
if scrape_wrapper.scrape_state_doc.scrape_state == STATE_SUCCESS:
scrape_wrapper.scrape_content_doc = create_scrape_content()
mq_message = TestMessage()
mq_message.body = scrape_wrapper.document
self.worker._mq_callback(mq_message)
self.worker._flush_aggregated_objects()
self.assertTrue(len(self.publisher_mock.publish.call_args_list) >= 2)
for i in range(0, len(self.publisher_mock.publish.call_args_list)):
json_obj = self.publisher_mock.publish.call_args_list[0][0][0]
merge_request = MqMergerRequest(json_obj)
# assert that files were created
s3_key = boto.s3.key.Key(self.worker.s3_bucket)
for file_name in [
merge_request.s3_file_scrape_state,
merge_request.s3_file_scrape_content
]:
s3_prefix, s3_file_key_name = break_s3_file_uri(file_name)
manifest_s3_key = boto.s3.key.Key(bucket=s3_key.bucket,
name=s3_file_key_name)
self.assertTrue(manifest_s3_key.exists(),
'Manifest file %s was not found' % file_name)
def test_android_success_packages(self):
for package_name in success_android_playstore:
message = TestMessage()
message.body = {
FIELD_PLATFORM: PLATFORM_ANDROID,
FIELD_PACKAGE_NAME: package_name
}
self.worker._mq_callback(message)
json_obj = self.publisher_mock.publish.call_args_list[0][0][0]
scrape_wrapper = MqScrapeWrapper.from_json(json_obj)
self.assertTrue(
scrape_wrapper.scrape_state_doc.scrape_state == STATE_SUCCESS,
'Valid package %s reported as FAILED' % package_name)
def test_android_failure_packages(self):
for package_name in failure_android_playstore:
message = TestMessage()
message.body = {
FIELD_PLATFORM: PLATFORM_ANDROID,
FIELD_PACKAGE_NAME: package_name
}
self.worker._mq_callback(message)
json_obj = self.publisher_mock.publish.call_args_list[0][0][0]
scrape_wrapper = MqScrapeWrapper.from_json(json_obj)
self.assertTrue(
scrape_wrapper.scrape_state_doc.scrape_state == STATE_FAILURE,
'Mistakenly valid package %s with URL %s' %
(package_name, scrape_wrapper.scrape_state_doc.scrape_url))
def perform_aggregation(self):
message = TestMessage()
message.body = str(self.uow_id)
self.aggregator._mq_callback(message)
if self.generate_output:
i = 0
for key in self.aggregator.aggregated_objects:
print self.output_prefix + '_%02d = %s' % (i, self.aggregator.aggregated_objects[key].get_document())
i += 1
if self.compare_results:
expected_values = base_fixtures.get_field_starting_with(self.output_prefix, self.output_module)
if len(expected_values) == 0:
assert 1 == 0
for obj in expected_values:
document = self.aggregator.aggregated_objects[self._get_key(obj)].get_document()
base_fixtures.compare_dictionaries(document, obj)
def test_logging(self):
self.worker = ChattyWorker(self.process_name)
message = TestMessage(process_name=self.process_name,
uow_id=self.uow_id)
self.worker._mq_callback(message)
uow_log = self.log_recording_dao.get_one(self.uow_id)
messages = INFO_LOG_MESSAGES + WARN_LOG_MESSAGES # + STD_MESSAGES
self.assertLessEqual(len(messages), len(uow_log.log))
for index, message in enumerate(messages):
self.assertIn(message, uow_log.log[index])
def test_exception_logging(self):
self.worker = ExceptionWorker(self.process_name)
message = TestMessage(process_name=self.process_name,
uow_id=self.uow_id)
self.worker._mq_callback(message)
uow_log = self.log_recording_dao.get_one(self.uow_id)
messages = [
'Exception: Artificially triggered exception to test Uow Exception Logging',
'Method ExceptionWorker._process_uow returned None. Assuming happy flow.',
'at INVALID_TIMEPERIOD: Success/Failure 0/0 entries'
]
for index, message in enumerate(messages):
self.assertIn(message, uow_log.log[index])
def perform_aggregation(self):
message = TestMessage(process_name=self.process_name,
uow_id=self.uow_id)
self.aggregator._mq_callback(message)
if self.generate_output:
i = 0
for key in self.aggregator.aggregated_objects:
print(
f'{self.output_prefix}_{i:02d} = {self.aggregator.aggregated_objects[key].document}'
)
i += 1
if self.compare_results:
expected_values = base_fixtures.get_field_starting_with(
self.output_prefix, self.output_module)
if len(expected_values) == 0:
assert 1 == 0
for obj in expected_values:
document = self.aggregator.aggregated_objects[self._get_key(
obj)].document
base_fixtures.compare_dictionaries(document, obj)
def test_aggregation(self):
message = TestMessage()
message.body = str(self.uow_id)
self.aggregator._mq_callback(message)
def test_execution(self):
message = TestMessage()
message.body = str(self.uow_id)
self.worker._mq_callback(message)
self.assertTrue(True)
