def test_processing(self): factor = 3 for _ in range(0, settings['csv_bulk_threshold'] * factor): scrape_wrapper = MqScrapeWrapper() scrape_wrapper.scrape_state_doc = create_scrape_state() if scrape_wrapper.scrape_state_doc.scrape_state == STATE_SUCCESS: scrape_wrapper.scrape_content_doc = create_scrape_content() mq_message = TestMessage() mq_message.body = scrape_wrapper.document self.worker._mq_callback(mq_message) self.worker._flush_aggregated_objects() self.assertTrue(len(self.publisher_mock.publish.call_args_list) >= 2) for i in range(0, len(self.publisher_mock.publish.call_args_list)): json_obj = self.publisher_mock.publish.call_args_list[0][0][0] merge_request = MqMergerRequest(json_obj) # assert that files were created s3_key = boto.s3.key.Key(self.worker.s3_bucket) for file_name in [ merge_request.s3_file_scrape_state, merge_request.s3_file_scrape_content ]: s3_prefix, s3_file_key_name = break_s3_file_uri(file_name) manifest_s3_key = boto.s3.key.Key(bucket=s3_key.bucket, name=s3_file_key_name) self.assertTrue(manifest_s3_key.exists(), 'Manifest file %s was not found' % file_name)
def test_android_success_packages(self): for package_name in success_android_playstore: message = TestMessage() message.body = { FIELD_PLATFORM: PLATFORM_ANDROID, FIELD_PACKAGE_NAME: package_name } self.worker._mq_callback(message) json_obj = self.publisher_mock.publish.call_args_list[0][0][0] scrape_wrapper = MqScrapeWrapper.from_json(json_obj) self.assertTrue( scrape_wrapper.scrape_state_doc.scrape_state == STATE_SUCCESS, 'Valid package %s reported as FAILED' % package_name)
def test_android_failure_packages(self): for package_name in failure_android_playstore: message = TestMessage() message.body = { FIELD_PLATFORM: PLATFORM_ANDROID, FIELD_PACKAGE_NAME: package_name } self.worker._mq_callback(message) json_obj = self.publisher_mock.publish.call_args_list[0][0][0] scrape_wrapper = MqScrapeWrapper.from_json(json_obj) self.assertTrue( scrape_wrapper.scrape_state_doc.scrape_state == STATE_FAILURE, 'Mistakenly valid package %s with URL %s' % (package_name, scrape_wrapper.scrape_state_doc.scrape_url))
def perform_aggregation(self): message = TestMessage() message.body = str(self.uow_id) self.aggregator._mq_callback(message) if self.generate_output: i = 0 for key in self.aggregator.aggregated_objects: print self.output_prefix + '_%02d = %s' % (i, self.aggregator.aggregated_objects[key].get_document()) i += 1 if self.compare_results: expected_values = base_fixtures.get_field_starting_with(self.output_prefix, self.output_module) if len(expected_values) == 0: assert 1 == 0 for obj in expected_values: document = self.aggregator.aggregated_objects[self._get_key(obj)].get_document() base_fixtures.compare_dictionaries(document, obj)
def test_logging(self): self.worker = ChattyWorker(self.process_name) message = TestMessage(process_name=self.process_name, uow_id=self.uow_id) self.worker._mq_callback(message) uow_log = self.log_recording_dao.get_one(self.uow_id) messages = INFO_LOG_MESSAGES + WARN_LOG_MESSAGES # + STD_MESSAGES self.assertLessEqual(len(messages), len(uow_log.log)) for index, message in enumerate(messages): self.assertIn(message, uow_log.log[index])
def test_exception_logging(self): self.worker = ExceptionWorker(self.process_name) message = TestMessage(process_name=self.process_name, uow_id=self.uow_id) self.worker._mq_callback(message) uow_log = self.log_recording_dao.get_one(self.uow_id) messages = [ 'Exception: Artificially triggered exception to test Uow Exception Logging', 'Method ExceptionWorker._process_uow returned None. Assuming happy flow.', 'at INVALID_TIMEPERIOD: Success/Failure 0/0 entries' ] for index, message in enumerate(messages): self.assertIn(message, uow_log.log[index])
def perform_aggregation(self): message = TestMessage(process_name=self.process_name, uow_id=self.uow_id) self.aggregator._mq_callback(message) if self.generate_output: i = 0 for key in self.aggregator.aggregated_objects: print( f'{self.output_prefix}_{i:02d} = {self.aggregator.aggregated_objects[key].document}' ) i += 1 if self.compare_results: expected_values = base_fixtures.get_field_starting_with( self.output_prefix, self.output_module) if len(expected_values) == 0: assert 1 == 0 for obj in expected_values: document = self.aggregator.aggregated_objects[self._get_key( obj)].document base_fixtures.compare_dictionaries(document, obj)
def test_aggregation(self): message = TestMessage() message.body = str(self.uow_id) self.aggregator._mq_callback(message)
def test_execution(self): message = TestMessage() message.body = str(self.uow_id) self.worker._mq_callback(message) self.assertTrue(True)