def bulk_process(self, data):
"""
"""
for doc in data:
doc_obj = DocumentObj(data=doc)
self.process(doc_obj)
class WatchdogBaseTestCase(TestCase):
"""
Tests the Watchdog class.
"""
fixtures = get_fixtures(['watchdogs', 'distilleries'])
doc_id = DOC_ID
data = DATA
doc_obj = DocumentObj(data=DATA,
doc_id=DOC_ID,
collection='mongodb.test_database.test_docs')
@classmethod
def setUpClass(cls):
super(WatchdogBaseTestCase, cls).setUpClass()
logging.disable(logging.ERROR)
@classmethod
def tearDownClass(cls):
logging.disable(logging.NOTSET)
super(WatchdogBaseTestCase, cls).tearDownClass()
def setUp(self):
self.distillery = Distillery.objects.get_by_natural_key(
'mongodb.test_database.test_docs')
self.email_wdog = Watchdog.objects.get_by_natural_key('inspect_emails')
self.log_wdog = Watchdog.objects.get_by_natural_key('inspect_logs')
def setUp(self):
self.distillery = Distillery.objects.get_by_natural_key(
'mongodb', 'test_database', 'test_posts')
doc = {'text': 'this is a text post'}
self.doc_obj = DocumentObj(data=doc,
doc_id=2,
collection=str(self.distillery))
class WatchdogTransactionTestCase(TransactionTestCase):
"""
Tests the Watchdog class.
"""
fixtures = get_fixtures(['watchdogs', 'distilleries'])
doc_id = DOC_ID
data = DATA
doc_obj = DocumentObj(data=DATA,
doc_id=DOC_ID,
collection='mongodb.test_database.test_docs')
def setUp(self):
self.distillery = Distillery.objects.get_by_natural_key(
'mongodb.test_database.test_docs')
self.email_wdog = Watchdog.objects.get_by_natural_key('inspect_emails')
@patch_find_by_id
def test_multiprocess_muzzled(self):
"""
Tests muzzling when multiple duplicate Alerts are being processed
concurrently.
"""
with patch('alerts.models.Alert._format_title',
return_value=self.data['content']['subject']):
incident_num = 20
alert_count = Alert.objects.count()
args = [self.doc_obj]
alert = self.email_wdog.process(*args)
# check that a new Alert was created
self.assertEqual(Alert.objects.count(), alert_count + 1)
# NOTE: we can't use multiprocessing with Mocks,
# so we have to settle for using threading to mimic concurrency
threads = []
for dummy_index in range(incident_num):
new_thread = threading.Thread(target=self.email_wdog.process,
args=args)
threads.append(new_thread)
new_thread.start()
for thread in threads:
thread.join()
# NOTE: we can't check Alert counts because saved Alerts
# won't be committed in the TransactionTestCase
# but we can check that the previous Alert was incremented
alert = Alert.objects.get(pk=alert.pk)
self.assertEqual(alert.incidents, incident_num + 1)
def setUp(self):
super(MailChuteManagerTestCase, self).setUp()
self.msg = MIMEText('example text', 'plain')
self.msg['Message-ID'] = '*****@*****.**'
self.msg['Date'] = 'Tue, 8 Sep 2015 16:08:59 -0400'
self.doc_obj = DocumentObj(data=self.msg)
# clear cached property
try:
del MailChute.objects._default_munger
except AttributeError:
pass
def test_process_nonmatch(self):
"""
Tests the process method for a nonmatching email.
"""
email = {'Message-ID': 'abc', 'Subject': 'This is an Urgent Alert'}
doc_obj = DocumentObj(data=email)
mailchute = MailChute.objects.get(pk=1)
mailchute.munger.process = Mock(return_value=None)
doc_id = mailchute.process(doc_obj)
self.assertEqual(doc_id, None)
def test_update_monitors(self):
"""
Tests the update_monitors signal receiver.
"""
assert Monitor.objects.get(pk=3).status == 'RED'
assert Monitor.objects.get(pk=4).status == 'RED'
distillery = Distillery.objects.get(pk=1)
doc_obj = DocumentObj(data={}, doc_id='1', collection=str(distillery))
document_saved.send(sender='document_saved', doc_obj=doc_obj)
self.assertEqual(Monitor.objects.get(pk=3).status, 'GREEN')
self.assertEqual(Monitor.objects.get(pk=4).status, 'RED')
def test_process_nonmatch(self):
"""
Tests the process method for a nonmatching data dictionary.
"""
data = {'id': 123, 'Subject': 'This is an Urgent Alert'}
doc_obj = DocumentObj(data=data)
datachute = DataChute.objects.get(pk=3)
datachute.munger.process = Mock(return_value=None)
doc_id = datachute.process(doc_obj)
self.assertEqual(doc_id, None)
def test_process_match(self):
"""
Tests the process method for a matching data dictionary.
"""
mock_doc_id = 1
data = {'id': 123, 'subject': 'This is a Critical Alert'}
doc_obj = DocumentObj(data=data)
datachute = DataChute.objects.get(pk=3)
datachute.munger.process = Mock(return_value=mock_doc_id)
doc_id = datachute.process(doc_obj)
datachute.munger.process.assert_called_once_with(doc_obj)
self.assertEqual(doc_id, mock_doc_id)
def handle_message(sender, message, **args):
"""
Takes a signal sender and a Django Mailbox Message and processes
the message.
"""
# get the Python email.message.Message object
email = message.get_email_object()
doc_obj = DocumentObj(
data=email,
doc_id=email['Message-ID'],
collection=_MAILSIFTER_SETTINGS['MAIL_COLLECTION'],
)
# process the email through enabled MailChutes
MailChute.objects.process(doc_obj)
def test_process_match(self):
"""
Tests the process method for a matching email.
"""
mock_doc_id = 1
email = {'Message-ID': 'abc', 'Subject': 'This is a Critical Alert'}
doc_obj = DocumentObj(data=email)
mailchute = MailChute.objects.get(pk=1)
mailchute.munger.process = Mock(return_value=mock_doc_id)
doc_id = mailchute.process(doc_obj)
mailchute.munger.process.assert_called_once_with(doc_obj)
self.assertEqual(doc_id, mock_doc_id), None
def test_process_no_sieve(self):
"""
Tests the process method for a chute with no sieve.
"""
mock_doc_id = 1
data = {'id': 123, 'Subject': 'This is an Urgent Alert'}
doc_obj = DocumentObj(data=data)
datachute = DataChute.objects.get(pk=4)
datachute.munger.process = Mock(return_value=mock_doc_id)
doc_id = datachute.process(doc_obj)
datachute.munger.process.assert_called_once_with(doc_obj)
self.assertEqual(doc_id, mock_doc_id)
def test_process_no_sieve(self):
"""
Tests the process method for a chute with no sieve.
"""
mock_doc_id = 1
email = {'Message-ID': 'abc', 'Subject': 'This is an Urgent Alert'}
doc_obj = DocumentObj(data=email)
mailchute = MailChute.objects.get(pk=3)
mailchute.enabled = True
mailchute.munger.process = Mock(return_value=mock_doc_id)
doc_id = mailchute.process(doc_obj)
mailchute.munger.process.assert_called_once_with(doc_obj)
self.assertEqual(doc_id, mock_doc_id)
def test_save_data(self):
"""
Tests the save_data method.
"""
mock_doc_id = 1
self.distillery.collection.insert = Mock(return_value=mock_doc_id)
doc_obj = DocumentObj(data=self.bottled_data,
doc_id='551d54e6f861c95f3123e5f6',
collection='mongodb.test_database.twitter',
platform='twitter')
with patch('distilleries.models.timezone.now', return_value=self.time):
doc_id = self.distillery.save_data(doc_obj)
bottled_with_meta = copy.deepcopy(self.bottled_data)
bottled_with_meta.update(self.meta)
self.distillery.collection.insert.assert_called_once_with(
bottled_with_meta)
self.assertEqual(doc_id, mock_doc_id)
def test_process(self):
"""
Tests the process method.
"""
mock_doc = Mock()
mock_doc_id = 1
data = {'id': 123, 'subject': 'This is a Critical Alert'}
doc_obj = DocumentObj(data=data)
datamunger = DataMunger.objects.get(pk=1)
datamunger.condenser.process = Mock(return_value=mock_doc)
datamunger.distillery.save_data = Mock(return_value=mock_doc_id)
doc_id = datamunger.process(doc_obj)
datamunger.condenser.process.assert_called_once_with(data)
assert datamunger.distillery.save_data.call_count == 1
self.assertEqual(doc_id, mock_doc_id)
def test_process(self):
"""
Tests the process method.
"""
mock_doc = Mock()
mock_doc_id = '1'
email = {'Message-ID': 'abc', 'Subject': 'This is a Critical Alert'}
doc_obj = DocumentObj(data=email)
mailmunger = MailMunger.objects.get_by_natural_key('mail')
mailmunger.condenser.process = Mock(return_value=mock_doc)
mailmunger.distillery.save_data = Mock(return_value=mock_doc_id)
doc_id = mailmunger.process(doc_obj)
mailmunger.condenser.process.assert_called_once_with(
data=email, company=mailmunger.distillery.company)
assert mailmunger.distillery.save_data.call_count == 1
self.assertEqual(doc_id, mock_doc_id)
def create_doc_obj(body):
"""Turn a message str into a |DocumentObj|.
Parses a message body, adds an ``_id`` field based on the document's
``@uuid`` field, and creates a |DocumentObj| from the result.
Parameters
----------
body : str
Returns
-------
|DocumentObj|
"""
if isinstance(body, six.binary_type):
body = body.decode('utf-8')
data = json.loads(body)
doc_id = data.get('@uuid')
data['_id'] = doc_id
collection = data.get('collection')
return DocumentObj(data=data, doc_id=doc_id, collection=collection)
def test_process(self, mock_now):
"""
Tests the process method of the Monitor class.
"""
monitor = self.monitor_red
assert monitor.last_healthy != LATE
assert monitor.last_active_distillery.pk != 2
assert monitor.last_saved_doc != '11'
assert monitor.status != 'GREEN'
distillery = Distillery.objects.get(pk=2)
collection = str(distillery)
doc_id = '11'
doc_obj = DocumentObj(doc_id=doc_id, collection=collection)
monitor.process(doc_obj)
# get a fresh instance from the database
updated_monitor = Monitor.objects.get(pk=monitor.pk)
self.assertEqual(updated_monitor.last_healthy, LATE)
self.assertEqual(updated_monitor.last_active_distillery.pk, distillery.pk)
self.assertEqual(updated_monitor.last_saved_doc, doc_id)
self.assertEqual(updated_monitor.status, 'GREEN')
class DistilleryTestCase(TestCase, DistilleryTestCaseMixin):
"""
Tests the Distillery class.
"""
fixtures = get_fixtures(['distilleries', 'funnels', 'tastes'])
doc = {'foo': 'bar'}
doc_obj = DocumentObj(collection='elasticsearch.cyphon.syslog',
doc_id='1',
data=doc)
@classmethod
def setUpTestData(cls):
cls.distillery = cls._create_example_distillery(cls)
def test_add_raw_data_info_for_none(self):
"""
Tests the _add_raw_data_info method when no Collection name is
given.
"""
with LogCapture() as log_capture:
doc_obj = self.doc_obj
doc_obj.collection = None
actual = self.distillery._add_raw_data_info(self.doc, doc_obj)
expected = self.doc
log_capture.check(
('cyphon.documents', 'ERROR',
'Info for raw data document None:1 could not be added'), )
self.assertEqual(actual, expected)
def test_add_raw_data_info_bad_str(self):
"""
Tests the _add_raw_data_info method for a malformed collection string.
"""
with LogCapture() as log_capture:
doc_obj = self.doc_obj
doc_obj.collection = 'bad_string'
actual = self.distillery._add_raw_data_info(self.doc, doc_obj)
expected = self.doc
log_capture.check(
('cyphon.documents', 'ERROR',
'Info for raw data document bad_string:1 could not be added'
), )
self.assertEqual(actual, expected)
def test_get_backend(self):
"""
Tests the get_backend method.
"""
self.assertEqual(self.distillery.get_backend(), 'mongodb')
def test_get_warehouse_name(self):
"""
Tests the get_backend_name method.
"""
self.assertEqual(self.distillery.get_warehouse_name(), 'test_database')
def test_get_collection(self):
"""
Tests the get_collection_name method.
"""
self.assertEqual(self.distillery.get_collection_name(), 'test_posts')
def test_find(self):
"""
Tests the find method.
"""
mock_docs = Mock()
mock_fieldsets = Mock()
self.distillery.collection.find = Mock(return_value=mock_docs)
docs = self.distillery.find(mock_fieldsets, 'AND')
self.distillery.collection.find.assert_called_once_with(
mock_fieldsets, 'AND', 1, _PAGE_SIZE)
self.assertEqual(docs, mock_docs)
def test_filter_ids(self):
"""
Tests the filter_ids method.
"""
mock_doc_ids = [1, 2]
test_ids = [1, 2, 3, 4]
self.distillery.collection.filter_ids = Mock(return_value=mock_doc_ids)
doc_ids = self.distillery.filter_ids_by_content(doc_ids=test_ids,
value='cat')
bottled_with_meta = copy.deepcopy(self.bottled_data)
bottled_with_meta.update(self.meta)
call_args = self.distillery.collection.filter_ids.call_args[0]
ids = call_args[0]
field1 = call_args[1][0]
field2 = call_args[1][1]
field3 = call_args[1][2]
field4 = call_args[1][3]
value = call_args[2]
self.assertEqual(ids, test_ids)
self.assertEqual(field1.field_name, 'body')
self.assertEqual(field1.field_type, 'TextField')
self.assertEqual(field1.target_type, 'Keyword')
self.assertEqual(field2.field_name, 'from')
self.assertEqual(field2.field_type, 'EmailField')
self.assertEqual(field2.target_type, 'Account')
self.assertEqual(field3.field_name, 'priority')
self.assertEqual(field3.field_type, 'CharField')
self.assertEqual(field3.target_type, 'Keyword')
self.assertEqual(field4.field_name, 'subject')
self.assertEqual(field4.field_type, 'TextField')
self.assertEqual(field4.target_type, 'Keyword')
self.assertEqual(value, 'cat')
self.assertEqual(doc_ids, mock_doc_ids)
def test_get_sample(self):
"""
Tests the get_sample method.
"""
data = {
'_raw_data': {
'backend': 'elasticsearch',
'database': 'test_index',
'collection': 'test_docs'
},
'date': '2015-01-01T12:00:00-05:00',
'from': '*****@*****.**',
'subject': 'This is a test',
'body': 'This is only a test.'
}
actual = self.distillery.get_sample(data)
expected = {
'collection': 'elasticsearch.test_index.test_docs',
'date': dateutil.parser.parse('2015-01-01T12:00:00-05:00'),
'author': '*****@*****.**',
'title': 'This is a test',
'content': 'This is only a test.',
'location': None
}
self.assertEqual(actual, expected)
def test_get_blind_sample_w_codebk(self):
"""
Tests the get_blind_sample method for a Distillery with a
CodeBook.
"""
data = {'subject': 'test title'}
sample = {'title': 'test title'}
distillery = Distillery.objects.get_by_natural_key(
'mongodb.test_database.test_posts')
with patch('distilleries.models.Distillery.container') \
as mock_container:
mock_container.get_blind_sample = Mock(return_value=sample)
actual = distillery.get_blind_sample(data)
mock_container.get_blind_sample.assert_called_once_with(
data, distillery.codebook)
self.assertEqual(actual, sample)
def test_get_blind_sample_wo_codebk(self):
"""
Tests the get_blind_sample method for a Distillery without a
CodeBook.
"""
data = {'subject': 'test title'}
sample = {'title': 'test title'}
with patch('distilleries.models.Distillery.container') \
as mock_container:
mock_container.get_sample = Mock(return_value=sample)
actual = self.distillery.get_blind_sample(data)
mock_container.get_sample.assert_called_once_with(data)
self.assertEqual(actual, sample)
def _create_doc_obj(self, doc, doc_id):
"""
Takes a data dictionary and a document id for a document in the
Distillery, and returns a DocumentObj for the document.
"""
return DocumentObj(data=doc, doc_id=doc_id, collection=str(self))
def _create_doc_obj(body):
"""Turn a message str into a DocumentObj."""
data = json.loads(body)
doc_id = data.get('@uuid')
collection = data.get('collection')
return DocumentObj(data=data, doc_id=doc_id, collection=collection)
