def setUp(self):
self.options = {
'exporter_options': {
'LOG_LEVEL':
'DEBUG',
'LOGGER_NAME':
'export-pipeline',
'notifications': [{
'name':
'exporters.notifications.s3_mail_notifier.SESMailNotifier',
'options': {
'team_mails': ['*****@*****.**'],
'client_mails': ['*****@*****.**'],
'access_key': 'somelogin',
'secret_key': 'somekey',
'mail_from': _MAIL_FROM,
}
}]
},
'writer': {
'name': 'somewriter',
'options': {
'some_option': 'some_value',
'bucket': 'SOMEBUCKET',
'filebase': 'FILEBASE',
}
}
}
self.meta = ExportMeta(self.options)
self.meta.per_module['writer']['items_count'] = 2
self.notifier = SESMailNotifier(
self.options['exporter_options']['notifications'][0], self.meta)
def setUp(self):
self.options = {
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline',
'notifications': [
{
'name': 'exporters.notifications.s3_mail_notifier.SESMailNotifier',
'options':
{
'team_mails': ['*****@*****.**'],
'client_mails': ['*****@*****.**'],
'access_key': 'somelogin',
'secret_key': 'somekey'
}
}
]
},
'writer': {
'name': 'somewriter',
'options': {
'some_option': 'some_value',
'bucket': 'SOMEBUCKET',
'filebase': 'FILEBASE',
}
}
}
self.meta = ExportMeta(self.options)
self.meta.per_module['writer']['items_count'] = 2
self.notifier = SESMailNotifier(
self.options['exporter_options']['notifications'][0], self.meta)
def test_invalid_mails(self):
options = {
'exporter_options': {
'LOG_LEVEL':
'DEBUG',
'LOGGER_NAME':
'export-pipeline',
'notifications': [{
'name':
'exporters.notifications.s3_mail_notifier.S3MailNotifier',
'options': {
'team_mails': ['badmail'],
'client_mails': [],
'access_key': 'somelogin',
'secret_key': 'somekey',
'mail_from': _MAIL_FROM,
}
}]
},
'writer': {
'name': 'somewriter',
'options': {
'some_option': 'some_value'
}
}
}
with self.assertRaises(InvalidMailProvided):
SESMailNotifier(options['exporter_options']['notifications'][0],
{})
class SESMailNotifierTest(unittest.TestCase):
def setUp(self):
self.options = {
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline',
'notifications': [
{
'name': 'exporters.notifications.s3_mail_notifier.SESMailNotifier',
'options':
{
'team_mails': ['*****@*****.**'],
'client_mails': ['*****@*****.**'],
'access_key': 'somelogin',
'secret_key': 'somekey'
}
}
]
},
'writer': {
'name': 'somewriter',
'options': {
'some_option': 'some_value',
'bucket': 'SOMEBUCKET',
'filebase': 'FILEBASE',
}
}
}
self.meta = ExportMeta(self.options)
self.meta.per_module['writer']['items_count'] = 2
self.notifier = SESMailNotifier(
self.options['exporter_options']['notifications'][0], self.meta)
@mock.patch('boto.connect_ses')
def test_start_dump(self, mock_ses):
self.notifier.notify_start_dump([CLIENTS, TEAM])
mock_ses.return_value.send_email.assert_called_once_with(
DEFAULT_MAIN_FROM,
'Started Customer export job',
u'\nExport job started with following parameters:\n\nWriter: somewriter'
u'\nBucket: SOMEBUCKET\nFilebase: FILEBASE',
['*****@*****.**', '*****@*****.**']
)
@mock.patch('boto.connect_ses')
def test_notify_with_custom_emails(self, mock_ses):
self.notifier.notify_start_dump(['*****@*****.**'])
mock_ses.return_value.send_email.assert_called_once_with(
mock.ANY,
mock.ANY,
mock.ANY,
['*****@*****.**']
)
@mock.patch('boto.connect_ses')
def test_complete_dump(self, mock_ses):
self.notifier.notify_complete_dump([CLIENTS, TEAM])
mock_ses.return_value.send_email.assert_called_once_with(
DEFAULT_MAIN_FROM,
'Customer export job finished',
u'\nExport job finished successfully.\n\nTotal records exported: 2.\n\n'
'If you have any questions or concerns about the data you have received, '
'email us at [email protected].\n',
['*****@*****.**', '*****@*****.**']
)
@mock.patch('boto.connect_ses')
def test_complete_dump_no_accurate_count(self, mock_ses):
self.meta.accurate_items_count = False
self.notifier.notify_complete_dump(['*****@*****.**'])
self.meta.accurate_items_count = False
mock_ses.return_value.send_email.assert_called_once_with(
DEFAULT_MAIN_FROM,
'Customer export job finished',
u'\nExport job finished successfully.\n\n\n\n'
'If you have any questions or concerns about the data you have received, '
'email us at [email protected].\n',
mock.ANY
)
@mock.patch('boto.connect_ses')
def test_failed_dump(self, mock_ses):
self.notifier.notify_failed_job('REASON', 'STACKTRACE', ['*****@*****.**'])
mock_ses.return_value.send_email.assert_called_once_with(
DEFAULT_MAIN_FROM,
'Failed export job for Customer',
u'\nExport job failed with following error:\n\n'
u'REASON\n\n'
u'Stacktrace:\nSTACKTRACE\n\n'
u'Configuration:\n' + json.dumps(self.options),
['*****@*****.**']
)
@mock.patch('boto.connect_ses')
def test_failed_dump_in_scrapy_cloud(self, mock_ses):
with environment(dict(SHUB_JOBKEY='10804/1/12')):
self.notifier.notify_failed_job('REASON', 'STACKTRACE', ['*****@*****.**'])
mock_ses.return_value.send_email.assert_called_once_with(
DEFAULT_MAIN_FROM,
'Failed export job for Customer',
u'\nExport job failed with following error:\n\n'
u'REASON\n\n'
u'Job key: 10804/1/12\n'
u'Job: https://dash.scrapinghub.com/p/10804/job/1/12\n\n'
u'Stacktrace:\nSTACKTRACE\n\n'
u'Configuration:\n' + json.dumps(self.options),
mock.ANY
)
def test_invalid_mails(self):
options = {
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline',
'notifications': [
{
'name': 'exporters.notifications.s3_mail_notifier.S3MailNotifier',
'options':
{
'team_mails': ['badmail'],
'client_mails': [],
'access_key': 'somelogin',
'secret_key': 'somekey'
}
}
]
},
'writer': {
'name': 'somewriter',
'options': {
'some_option': 'some_value'
}
}
}
with self.assertRaises(InvalidMailProvided):
SESMailNotifier(options['exporter_options']['notifications'][0], {})
class SESMailNotifierTest(unittest.TestCase):
def setUp(self):
self.options = {
'exporter_options': {
'LOG_LEVEL':
'DEBUG',
'LOGGER_NAME':
'export-pipeline',
'notifications': [{
'name':
'exporters.notifications.s3_mail_notifier.SESMailNotifier',
'options': {
'team_mails': ['*****@*****.**'],
'client_mails': ['*****@*****.**'],
'access_key': 'somelogin',
'secret_key': 'somekey',
'mail_from': _MAIL_FROM,
}
}]
},
'writer': {
'name': 'somewriter',
'options': {
'some_option': 'some_value',
'bucket': 'SOMEBUCKET',
'filebase': 'FILEBASE',
}
}
}
self.meta = ExportMeta(self.options)
self.meta.per_module['writer']['items_count'] = 2
self.notifier = SESMailNotifier(
self.options['exporter_options']['notifications'][0], self.meta)
@mock.patch('boto.connect_ses')
def test_start_dump(self, mock_ses):
self.notifier.notify_start_dump([CLIENTS, TEAM])
mock_ses.return_value.send_email.assert_called_once_with(
_MAIL_FROM, 'Started Customer export job',
u'\nExport job started with following parameters:\n\nWriter: somewriter'
u'\nBucket: SOMEBUCKET\nFilebase: FILEBASE',
['*****@*****.**', '*****@*****.**'])
@mock.patch('boto.connect_ses')
def test_notify_with_custom_emails(self, mock_ses):
self.notifier.notify_start_dump(['*****@*****.**'])
mock_ses.return_value.send_email.assert_called_once_with(
mock.ANY, mock.ANY, mock.ANY, ['*****@*****.**'])
@mock.patch('boto.connect_ses')
def test_complete_dump(self, mock_ses):
self.notifier.notify_complete_dump([CLIENTS, TEAM])
mock_ses.return_value.send_email.assert_called_once_with(
_MAIL_FROM, 'Customer export job finished',
u'\nExport job finished successfully.\n\nTotal records exported: 2.',
['*****@*****.**', '*****@*****.**'])
@mock.patch('boto.connect_ses')
def test_complete_dump_no_accurate_count(self, mock_ses):
self.meta.accurate_items_count = False
self.notifier.notify_complete_dump(['*****@*****.**'])
self.meta.accurate_items_count = False
mock_ses.return_value.send_email.assert_called_once_with(
_MAIL_FROM, 'Customer export job finished',
u'\nExport job finished successfully.\n\n', mock.ANY)
@mock.patch('boto.connect_ses')
def test_failed_dump(self, mock_ses):
self.notifier.notify_failed_job('REASON', 'STACKTRACE',
['*****@*****.**'])
mock_ses.return_value.send_email.assert_called_once_with(
_MAIL_FROM, 'Failed export job for Customer',
u'\nExport job failed with following error:\n\n'
u'REASON\n\n'
u'Stacktrace:\nSTACKTRACE\n\n'
u'Configuration:\n' + json.dumps(self.options), ['*****@*****.**'])
@mock.patch('boto.connect_ses')
def test_failed_dump_in_scrapy_cloud(self, mock_ses):
with environment(dict(SHUB_JOBKEY='10804/1/12')):
self.notifier.notify_failed_job('REASON', 'STACKTRACE',
['*****@*****.**'])
mock_ses.return_value.send_email.assert_called_once_with(
_MAIL_FROM, 'Failed export job for Customer',
u'\nExport job failed with following error:\n\n'
u'REASON\n\n'
u'Job key: 10804/1/12\n'
u'Job: https://dash.scrapinghub.com/p/10804/job/1/12\n\n'
u'Stacktrace:\nSTACKTRACE\n\n'
u'Configuration:\n' + json.dumps(self.options), mock.ANY)
def test_invalid_mails(self):
options = {
'exporter_options': {
'LOG_LEVEL':
'DEBUG',
'LOGGER_NAME':
'export-pipeline',
'notifications': [{
'name':
'exporters.notifications.s3_mail_notifier.S3MailNotifier',
'options': {
'team_mails': ['badmail'],
'client_mails': [],
'access_key': 'somelogin',
'secret_key': 'somekey',
'mail_from': _MAIL_FROM,
}
}]
},
'writer': {
'name': 'somewriter',
'options': {
'some_option': 'some_value'
}
}
}
with self.assertRaises(InvalidMailProvided):
SESMailNotifier(options['exporter_options']['notifications'][0],
{})