def bulk_process(self, data): """ """ for doc in data: doc_obj = DocumentObj(data=doc) self.process(doc_obj)
class WatchdogBaseTestCase(TestCase): """ Tests the Watchdog class. """ fixtures = get_fixtures(['watchdogs', 'distilleries']) doc_id = DOC_ID data = DATA doc_obj = DocumentObj(data=DATA, doc_id=DOC_ID, collection='mongodb.test_database.test_docs') @classmethod def setUpClass(cls): super(WatchdogBaseTestCase, cls).setUpClass() logging.disable(logging.ERROR) @classmethod def tearDownClass(cls): logging.disable(logging.NOTSET) super(WatchdogBaseTestCase, cls).tearDownClass() def setUp(self): self.distillery = Distillery.objects.get_by_natural_key( 'mongodb.test_database.test_docs') self.email_wdog = Watchdog.objects.get_by_natural_key('inspect_emails') self.log_wdog = Watchdog.objects.get_by_natural_key('inspect_logs')
def setUp(self): self.distillery = Distillery.objects.get_by_natural_key( 'mongodb', 'test_database', 'test_posts') doc = {'text': 'this is a text post'} self.doc_obj = DocumentObj(data=doc, doc_id=2, collection=str(self.distillery))
class WatchdogTransactionTestCase(TransactionTestCase): """ Tests the Watchdog class. """ fixtures = get_fixtures(['watchdogs', 'distilleries']) doc_id = DOC_ID data = DATA doc_obj = DocumentObj(data=DATA, doc_id=DOC_ID, collection='mongodb.test_database.test_docs') def setUp(self): self.distillery = Distillery.objects.get_by_natural_key( 'mongodb.test_database.test_docs') self.email_wdog = Watchdog.objects.get_by_natural_key('inspect_emails') @patch_find_by_id def test_multiprocess_muzzled(self): """ Tests muzzling when multiple duplicate Alerts are being processed concurrently. """ with patch('alerts.models.Alert._format_title', return_value=self.data['content']['subject']): incident_num = 20 alert_count = Alert.objects.count() args = [self.doc_obj] alert = self.email_wdog.process(*args) # check that a new Alert was created self.assertEqual(Alert.objects.count(), alert_count + 1) # NOTE: we can't use multiprocessing with Mocks, # so we have to settle for using threading to mimic concurrency threads = [] for dummy_index in range(incident_num): new_thread = threading.Thread(target=self.email_wdog.process, args=args) threads.append(new_thread) new_thread.start() for thread in threads: thread.join() # NOTE: we can't check Alert counts because saved Alerts # won't be committed in the TransactionTestCase # but we can check that the previous Alert was incremented alert = Alert.objects.get(pk=alert.pk) self.assertEqual(alert.incidents, incident_num + 1)
def setUp(self): super(MailChuteManagerTestCase, self).setUp() self.msg = MIMEText('example text', 'plain') self.msg['Message-ID'] = '*****@*****.**' self.msg['Date'] = 'Tue, 8 Sep 2015 16:08:59 -0400' self.doc_obj = DocumentObj(data=self.msg) # clear cached property try: del MailChute.objects._default_munger except AttributeError: pass
def test_process_nonmatch(self): """ Tests the process method for a nonmatching email. """ email = {'Message-ID': 'abc', 'Subject': 'This is an Urgent Alert'} doc_obj = DocumentObj(data=email) mailchute = MailChute.objects.get(pk=1) mailchute.munger.process = Mock(return_value=None) doc_id = mailchute.process(doc_obj) self.assertEqual(doc_id, None)
def test_update_monitors(self): """ Tests the update_monitors signal receiver. """ assert Monitor.objects.get(pk=3).status == 'RED' assert Monitor.objects.get(pk=4).status == 'RED' distillery = Distillery.objects.get(pk=1) doc_obj = DocumentObj(data={}, doc_id='1', collection=str(distillery)) document_saved.send(sender='document_saved', doc_obj=doc_obj) self.assertEqual(Monitor.objects.get(pk=3).status, 'GREEN') self.assertEqual(Monitor.objects.get(pk=4).status, 'RED')
def test_process_nonmatch(self): """ Tests the process method for a nonmatching data dictionary. """ data = {'id': 123, 'Subject': 'This is an Urgent Alert'} doc_obj = DocumentObj(data=data) datachute = DataChute.objects.get(pk=3) datachute.munger.process = Mock(return_value=None) doc_id = datachute.process(doc_obj) self.assertEqual(doc_id, None)
def test_process_match(self): """ Tests the process method for a matching data dictionary. """ mock_doc_id = 1 data = {'id': 123, 'subject': 'This is a Critical Alert'} doc_obj = DocumentObj(data=data) datachute = DataChute.objects.get(pk=3) datachute.munger.process = Mock(return_value=mock_doc_id) doc_id = datachute.process(doc_obj) datachute.munger.process.assert_called_once_with(doc_obj) self.assertEqual(doc_id, mock_doc_id)
def handle_message(sender, message, **args): """ Takes a signal sender and a Django Mailbox Message and processes the message. """ # get the Python email.message.Message object email = message.get_email_object() doc_obj = DocumentObj( data=email, doc_id=email['Message-ID'], collection=_MAILSIFTER_SETTINGS['MAIL_COLLECTION'], ) # process the email through enabled MailChutes MailChute.objects.process(doc_obj)
def test_process_match(self): """ Tests the process method for a matching email. """ mock_doc_id = 1 email = {'Message-ID': 'abc', 'Subject': 'This is a Critical Alert'} doc_obj = DocumentObj(data=email) mailchute = MailChute.objects.get(pk=1) mailchute.munger.process = Mock(return_value=mock_doc_id) doc_id = mailchute.process(doc_obj) mailchute.munger.process.assert_called_once_with(doc_obj) self.assertEqual(doc_id, mock_doc_id), None
def test_process_no_sieve(self): """ Tests the process method for a chute with no sieve. """ mock_doc_id = 1 data = {'id': 123, 'Subject': 'This is an Urgent Alert'} doc_obj = DocumentObj(data=data) datachute = DataChute.objects.get(pk=4) datachute.munger.process = Mock(return_value=mock_doc_id) doc_id = datachute.process(doc_obj) datachute.munger.process.assert_called_once_with(doc_obj) self.assertEqual(doc_id, mock_doc_id)
def test_process_no_sieve(self): """ Tests the process method for a chute with no sieve. """ mock_doc_id = 1 email = {'Message-ID': 'abc', 'Subject': 'This is an Urgent Alert'} doc_obj = DocumentObj(data=email) mailchute = MailChute.objects.get(pk=3) mailchute.enabled = True mailchute.munger.process = Mock(return_value=mock_doc_id) doc_id = mailchute.process(doc_obj) mailchute.munger.process.assert_called_once_with(doc_obj) self.assertEqual(doc_id, mock_doc_id)
def test_save_data(self): """ Tests the save_data method. """ mock_doc_id = 1 self.distillery.collection.insert = Mock(return_value=mock_doc_id) doc_obj = DocumentObj(data=self.bottled_data, doc_id='551d54e6f861c95f3123e5f6', collection='mongodb.test_database.twitter', platform='twitter') with patch('distilleries.models.timezone.now', return_value=self.time): doc_id = self.distillery.save_data(doc_obj) bottled_with_meta = copy.deepcopy(self.bottled_data) bottled_with_meta.update(self.meta) self.distillery.collection.insert.assert_called_once_with( bottled_with_meta) self.assertEqual(doc_id, mock_doc_id)
def test_process(self): """ Tests the process method. """ mock_doc = Mock() mock_doc_id = 1 data = {'id': 123, 'subject': 'This is a Critical Alert'} doc_obj = DocumentObj(data=data) datamunger = DataMunger.objects.get(pk=1) datamunger.condenser.process = Mock(return_value=mock_doc) datamunger.distillery.save_data = Mock(return_value=mock_doc_id) doc_id = datamunger.process(doc_obj) datamunger.condenser.process.assert_called_once_with(data) assert datamunger.distillery.save_data.call_count == 1 self.assertEqual(doc_id, mock_doc_id)
def test_process(self): """ Tests the process method. """ mock_doc = Mock() mock_doc_id = '1' email = {'Message-ID': 'abc', 'Subject': 'This is a Critical Alert'} doc_obj = DocumentObj(data=email) mailmunger = MailMunger.objects.get_by_natural_key('mail') mailmunger.condenser.process = Mock(return_value=mock_doc) mailmunger.distillery.save_data = Mock(return_value=mock_doc_id) doc_id = mailmunger.process(doc_obj) mailmunger.condenser.process.assert_called_once_with( data=email, company=mailmunger.distillery.company) assert mailmunger.distillery.save_data.call_count == 1 self.assertEqual(doc_id, mock_doc_id)
def create_doc_obj(body): """Turn a message str into a |DocumentObj|. Parses a message body, adds an ``_id`` field based on the document's ``@uuid`` field, and creates a |DocumentObj| from the result. Parameters ---------- body : str Returns ------- |DocumentObj| """ if isinstance(body, six.binary_type): body = body.decode('utf-8') data = json.loads(body) doc_id = data.get('@uuid') data['_id'] = doc_id collection = data.get('collection') return DocumentObj(data=data, doc_id=doc_id, collection=collection)
def test_process(self, mock_now): """ Tests the process method of the Monitor class. """ monitor = self.monitor_red assert monitor.last_healthy != LATE assert monitor.last_active_distillery.pk != 2 assert monitor.last_saved_doc != '11' assert monitor.status != 'GREEN' distillery = Distillery.objects.get(pk=2) collection = str(distillery) doc_id = '11' doc_obj = DocumentObj(doc_id=doc_id, collection=collection) monitor.process(doc_obj) # get a fresh instance from the database updated_monitor = Monitor.objects.get(pk=monitor.pk) self.assertEqual(updated_monitor.last_healthy, LATE) self.assertEqual(updated_monitor.last_active_distillery.pk, distillery.pk) self.assertEqual(updated_monitor.last_saved_doc, doc_id) self.assertEqual(updated_monitor.status, 'GREEN')
class DistilleryTestCase(TestCase, DistilleryTestCaseMixin): """ Tests the Distillery class. """ fixtures = get_fixtures(['distilleries', 'funnels', 'tastes']) doc = {'foo': 'bar'} doc_obj = DocumentObj(collection='elasticsearch.cyphon.syslog', doc_id='1', data=doc) @classmethod def setUpTestData(cls): cls.distillery = cls._create_example_distillery(cls) def test_add_raw_data_info_for_none(self): """ Tests the _add_raw_data_info method when no Collection name is given. """ with LogCapture() as log_capture: doc_obj = self.doc_obj doc_obj.collection = None actual = self.distillery._add_raw_data_info(self.doc, doc_obj) expected = self.doc log_capture.check( ('cyphon.documents', 'ERROR', 'Info for raw data document None:1 could not be added'), ) self.assertEqual(actual, expected) def test_add_raw_data_info_bad_str(self): """ Tests the _add_raw_data_info method for a malformed collection string. """ with LogCapture() as log_capture: doc_obj = self.doc_obj doc_obj.collection = 'bad_string' actual = self.distillery._add_raw_data_info(self.doc, doc_obj) expected = self.doc log_capture.check( ('cyphon.documents', 'ERROR', 'Info for raw data document bad_string:1 could not be added' ), ) self.assertEqual(actual, expected) def test_get_backend(self): """ Tests the get_backend method. """ self.assertEqual(self.distillery.get_backend(), 'mongodb') def test_get_warehouse_name(self): """ Tests the get_backend_name method. """ self.assertEqual(self.distillery.get_warehouse_name(), 'test_database') def test_get_collection(self): """ Tests the get_collection_name method. """ self.assertEqual(self.distillery.get_collection_name(), 'test_posts') def test_find(self): """ Tests the find method. """ mock_docs = Mock() mock_fieldsets = Mock() self.distillery.collection.find = Mock(return_value=mock_docs) docs = self.distillery.find(mock_fieldsets, 'AND') self.distillery.collection.find.assert_called_once_with( mock_fieldsets, 'AND', 1, _PAGE_SIZE) self.assertEqual(docs, mock_docs) def test_filter_ids(self): """ Tests the filter_ids method. """ mock_doc_ids = [1, 2] test_ids = [1, 2, 3, 4] self.distillery.collection.filter_ids = Mock(return_value=mock_doc_ids) doc_ids = self.distillery.filter_ids_by_content(doc_ids=test_ids, value='cat') bottled_with_meta = copy.deepcopy(self.bottled_data) bottled_with_meta.update(self.meta) call_args = self.distillery.collection.filter_ids.call_args[0] ids = call_args[0] field1 = call_args[1][0] field2 = call_args[1][1] field3 = call_args[1][2] field4 = call_args[1][3] value = call_args[2] self.assertEqual(ids, test_ids) self.assertEqual(field1.field_name, 'body') self.assertEqual(field1.field_type, 'TextField') self.assertEqual(field1.target_type, 'Keyword') self.assertEqual(field2.field_name, 'from') self.assertEqual(field2.field_type, 'EmailField') self.assertEqual(field2.target_type, 'Account') self.assertEqual(field3.field_name, 'priority') self.assertEqual(field3.field_type, 'CharField') self.assertEqual(field3.target_type, 'Keyword') self.assertEqual(field4.field_name, 'subject') self.assertEqual(field4.field_type, 'TextField') self.assertEqual(field4.target_type, 'Keyword') self.assertEqual(value, 'cat') self.assertEqual(doc_ids, mock_doc_ids) def test_get_sample(self): """ Tests the get_sample method. """ data = { '_raw_data': { 'backend': 'elasticsearch', 'database': 'test_index', 'collection': 'test_docs' }, 'date': '2015-01-01T12:00:00-05:00', 'from': '*****@*****.**', 'subject': 'This is a test', 'body': 'This is only a test.' } actual = self.distillery.get_sample(data) expected = { 'collection': 'elasticsearch.test_index.test_docs', 'date': dateutil.parser.parse('2015-01-01T12:00:00-05:00'), 'author': '*****@*****.**', 'title': 'This is a test', 'content': 'This is only a test.', 'location': None } self.assertEqual(actual, expected) def test_get_blind_sample_w_codebk(self): """ Tests the get_blind_sample method for a Distillery with a CodeBook. """ data = {'subject': 'test title'} sample = {'title': 'test title'} distillery = Distillery.objects.get_by_natural_key( 'mongodb.test_database.test_posts') with patch('distilleries.models.Distillery.container') \ as mock_container: mock_container.get_blind_sample = Mock(return_value=sample) actual = distillery.get_blind_sample(data) mock_container.get_blind_sample.assert_called_once_with( data, distillery.codebook) self.assertEqual(actual, sample) def test_get_blind_sample_wo_codebk(self): """ Tests the get_blind_sample method for a Distillery without a CodeBook. """ data = {'subject': 'test title'} sample = {'title': 'test title'} with patch('distilleries.models.Distillery.container') \ as mock_container: mock_container.get_sample = Mock(return_value=sample) actual = self.distillery.get_blind_sample(data) mock_container.get_sample.assert_called_once_with(data) self.assertEqual(actual, sample)
def _create_doc_obj(self, doc, doc_id): """ Takes a data dictionary and a document id for a document in the Distillery, and returns a DocumentObj for the document. """ return DocumentObj(data=doc, doc_id=doc_id, collection=str(self))
def _create_doc_obj(body): """Turn a message str into a DocumentObj.""" data = json.loads(body) doc_id = data.get('@uuid') collection = data.get('collection') return DocumentObj(data=data, doc_id=doc_id, collection=collection)