def test_run_job_with_mocked_data(self):
config_manager = self._setup_config_manager()
self._insert_waterwolf_mock_data()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['daily-url']
assert not information['daily-url']['last_error']
assert information['daily-url']['last_success']
# this should have created two .csv.gz files
now = datetime.datetime.utcnow() - datetime.timedelta(days=1)
private = now.strftime('%Y%m%d-crashdata.csv.gz')
public = now.strftime('%Y%m%d-pub-crashdata.csv.gz')
self.assertTrue(private in os.listdir(self.tempdir))
self.assertTrue(public in os.listdir(self.tempdir))
private_path = os.path.join(self.tempdir, private)
f = gzip.open(private_path)
try:
content = f.read()
self.assertTrue(content)
lines = content.splitlines()
header = lines[0]
payload = lines[1:]
self.assertEqual(header.split('\t')[0], 'signature')
self.assertEqual(header.split('\t')[1], 'url')
urls = [x.split('\t')[1] for x in payload]
self.assertTrue('http://p**n.xxx' in urls)
signatures = [x.split('\t')[0] for x in payload]
self.assertEqual(sorted(signatures), [
'FakeSignature1', 'FakeSignature2', 'FakeSignature3',
'FakeSignature4'
])
finally:
f.close()
public_path = os.path.join(self.tempdir, public)
f = gzip.open(public_path)
try:
content = f.read()
self.assertTrue(content)
lines = content.splitlines()
header = lines[0]
payload = lines[1:]
self.assertEqual(header.split('\t')[0], 'signature')
self.assertEqual(header.split('\t')[1], 'URL (removed)')
urls = [x.split('\t')[1] for x in payload]
self.assertTrue('http://p**n.xxx' not in urls)
signatures = [x.split('\t')[0] for x in payload]
self.assertEqual(sorted(signatures), [
'FakeSignature1', 'FakeSignature2', 'FakeSignature3',
'FakeSignature4'
])
finally:
f.close()
def test_all_matviews(self, mocked_utc_now):
# Pretend it's 03AM UTC
def mock_utc_now():
n = utc_now()
n = n.replace(hour=3)
return n
mocked_utc_now.side_effect = mock_utc_now
config_manager = self._setup_config_manager(
'socorro.unittest.cron.jobs.test_matviews.ReportsCleanJob|1d\n'
'socorro.unittest.cron.jobs.test_matviews.FTPScraperJob|1d\n'
''
'socorro.cron.jobs.matviews.ProductVersionsCronApp|1d\n'
'socorro.cron.jobs.matviews.SignaturesCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.TCBSCronApp|1d\n'
'socorro.cron.jobs.matviews.ADUCronApp|1d\n'
'socorro.cron.jobs.matviews.NightlyBuildsCronApp|1d\n'
'socorro.cron.jobs.matviews.BuildADUCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.CrashesByUserCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.CrashesByUserBuildCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.CorrelationsCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.HomePageGraphCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.HomePageGraphBuildCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.TCBSBuildCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.ExplosivenessCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.GraphicsDeviceCronApp|1d|02:00\n'
)
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
for app_name in ('product-versions-matview',
'signatures-matview',
'tcbs-matview',
'adu-matview',
'nightly-builds-matview',
'build-adu-matview',
'crashes-by-user-matview',
'crashes-by-user-build-matview',
'correlations-matview',
'home-page-graph-matview',
'home-page-graph-matview-build',
'tcbs-build-matview',
'explosiveness-matview',
'graphics-device-matview',):
self.assertTrue(app_name in information, app_name)
self.assertTrue(
not information[app_name]['last_error'],
app_name
)
self.assertTrue(
information[app_name]['last_success'],
app_name
)
def test_duplicates(self):
config_manager, json_file = self._setup_config_manager(
'socorro.cron.jobs.matviews.DuplicatesCronApp|1d')
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = json.load(open(json_file))
assert information['duplicates']
assert not information['duplicates']['last_error']
assert information['duplicates']['last_success']
# not a huge fan of this test because it's so specific
proc_name = 'update_reports_duplicates'
calls = self.psycopg2().cursor().callproc.mock_calls
call1, call2 = calls
__, called, __ = call1
assert called[0] == proc_name, called[0]
start, end = called[1]
self.assertEqual(end - start, datetime.timedelta(hours=1))
__, called, __ = call2
assert called[0] == proc_name, called[0]
start, end = called[1]
self.assertEqual(end - start, datetime.timedelta(hours=1))
def test_run(self):
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['suspicious-crashes']
assert not information['suspicious-crashes']['last_error']
assert information['suspicious-crashes']['last_success']
cursor = self.conn.cursor()
cursor.execute("""
SELECT signatures.signature, scs.report_date
FROM suspicious_crash_signatures scs
JOIN signatures ON scs.signature_id=signatures.signature_id
""")
count = 0
today = (utc_now() - datetime.timedelta(1)).date()
for row in cursor.fetchall():
self.assertEquals('sig', row[0])
self.assertEquals(today, row[1].date())
count += 1
self.assertEquals(1, count)
def test_basic_run_job_without_reports(self):
config_manager = self._setup_config_manager(3)
cursor = self.conn.cursor()
cursor.execute('select count(*) from reports')
count, = cursor.fetchone()
assert count == 0, "reports table not cleaned"
cursor.execute('select count(*) from bugs')
count, = cursor.fetchone()
assert count == 0, "'bugs' table not cleaned"
cursor.execute('select count(*) from bug_associations')
count, = cursor.fetchone()
assert count == 0, "'bug_associations' table not cleaned"
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
# now, because there we no matching signatures in the reports table
# it means that all bugs are rejected
cursor.execute('select count(*) from bugs')
count, = cursor.fetchone()
self.assertTrue(not count)
cursor.execute('select count(*) from bug_associations')
count, = cursor.fetchone()
self.assertTrue(not count)
def test_run_job_with_no_data_with_ssh_errors(self):
config_manager, json_file = self._setup_config_manager(
public_output_path='',
private_user='******',
private_server='secure.mozilla.org',
private_location='/var/data/',
private_ssh_command='chmod 0640 /var/data/*',
)
self._insert_waterwolf_mock_data()
# any mutable so we can keep track of the number of times
# the side_effect function is called
calls = []
def comm():
if calls:
# some errors
return '', "CRAP!"
else:
calls.append(1)
return '', ''
self.Popen().communicate.side_effect = comm
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = json.load(open(json_file))
assert information['daily-url']
assert not information['daily-url']['last_error']
assert information['daily-url']['last_success']
self.assertTrue(config.logger.warn.called)
def test_run_job_with_mocked_data_with_wrong_products(self):
config_manager, json_file = self._setup_config_manager(
product='Thunderbird,SeaMonkey',
version='1.0,2.0',
public_output_path=False
)
self._insert_waterwolf_mock_data()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = json.load(open(json_file))
assert information['daily-url']
assert not information['daily-url']['last_error']
assert information['daily-url']['last_success']
# this should have created two .csv.gz files
now = datetime.datetime.utcnow() - datetime.timedelta(days=1)
private = now.strftime('%Y%m%d-crashdata.csv.gz')
public = now.strftime('%Y%m%d-pub-crashdata.csv.gz')
self.assertTrue(private in os.listdir(self.tempdir))
self.assertTrue(public not in os.listdir(self.tempdir))
private_path = os.path.join(self.tempdir, private)
f = gzip.open(private_path)
try:
self.assertEqual(f.read(), '')
finally:
f.close()
def test_basic_run_job_no_data(self):
config_manager, json_file = self._setup_config_manager()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = json.load(open(json_file))
assert information['daily-url']
assert not information['daily-url']['last_error']
assert information['daily-url']['last_success']
# this should have created two .csv.gz files
now = datetime.datetime.utcnow() - datetime.timedelta(days=1)
private = now.strftime('%Y%m%d-crashdata.csv.gz')
public = now.strftime('%Y%m%d-pub-crashdata.csv.gz')
self.assertTrue(private in os.listdir(self.tempdir))
self.assertTrue(public in os.listdir(self.tempdir))
private_path = os.path.join(self.tempdir, private)
f = gzip.open(private_path)
try:
self.assertEqual(f.read(), '')
finally:
f.close()
public_path = os.path.join(self.tempdir, public)
f = gzip.open(public_path)
try:
self.assertEqual(f.read(), '')
finally:
f.close()
def test_reprocessing(self):
""" Simple test of reprocessing"""
config_manager = self._setup_config_manager()
cursor = self.conn.cursor()
# Create partitions to support the status query
# Load report_partition_info data
cursor.execute("""
INSERT into reprocessing_jobs
(crash_id)
VALUES
('13c4a348-5d04-11e3-8118-d231feb1dc81')
""")
# We have to do this here to accommodate separate crontabber processes
self.conn.commit()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
cursor.execute('select count(*) from reprocessing_jobs')
res_expected = 0
res, = cursor.fetchone()
self.assertEqual(res, res_expected)
def test_run_weekly_reports_partitions(self):
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['weekly-reports-partitions']
assert not information['weekly-reports-partitions']['last_error']
assert information['weekly-reports-partitions']['last_success']
def test_reports_clean_dependency_prerequisite(self):
config_manager, json_file = self._setup_config_manager(
'socorro.cron.jobs.matviews.ReportsCleanCronApp|1d')
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
# no file is created because it's unable to run anything
self.assertTrue(not os.path.isfile(json_file))
def test_cron_job(self, exacttarget_mock):
config_manager = self._setup_config_manager()
et_mock = exacttarget_mock.return_value
# Make get_subscriber raise an exception
list_service = et_mock.list.return_value = mock.Mock()
list_service.get_subscriber = mock.Mock(
side_effect=exacttarget.NewsletterException()
)
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
self.assertEqual(et_mock.trigger_send.call_count, 4)
last_email = u'z\[email protected]'
# Verify the last call to trigger_send
fields = {
'EMAIL_ADDRESS_': last_email,
'EMAIL_FORMAT_': 'H',
'TOKEN': last_email
}
et_mock.trigger_send.assert_called_with('socorro_dev_test', fields)
# Verify that user's data was updated
conf = config.crontabber['class-AutomaticEmailsCronApp']
es = SuperS().es(
urls=conf.elasticsearch.elasticsearch_urls,
timeout=conf.elasticsearch.elasticsearch_timeout,
)
search = es.indexes(conf.elasticsearch.elasticsearch_emails_index)
search = search.doctypes('emails')
es.get_es().refresh()
emails_list = (
'*****@*****.**',
'"Quidam" <*****@*****.**>',
'*****@*****.**'
)
search = search.filter(_id__in=emails_list)
res = search.values_list('last_sending')
self.assertEqual(len(res), 3)
now = utc_now()
for row in res:
date = string_to_datetime(row[0])
self.assertEqual(date.year, now.year)
self.assertEqual(date.month, now.month)
self.assertEqual(date.day, now.day)
def test_all_matviews(self):
config_manager, json_file = self._setup_config_manager(
'socorro.unittest.cron.jobs.test_matviews.ReportsCleanJob|1d\n'
'socorro.unittest.cron.jobs.test_matviews.FTPScraperJob|1d\n'
''
'socorro.cron.jobs.matviews.ProductVersionsCronApp|1d\n'
'socorro.cron.jobs.matviews.SignaturesCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.TCBSCronApp|1d\n'
'socorro.cron.jobs.matviews.ADUCronApp|1d\n'
'socorro.cron.jobs.matviews.HangReportCronApp|1d\n'
'socorro.cron.jobs.matviews.NightlyBuildsCronApp|1d\n'
'socorro.cron.jobs.matviews.BuildADUCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.CrashesByUserCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.CrashesByUserBuildCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.CorrelationsCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.HomePageGraphCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.HomePageGraphBuildCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.TCBSBuildCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.ExplosivenessCronApp|1d|02:00\n')
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = json.load(open(json_file))
for app_name in ('product-versions-matview', 'signatures-matview',
'tcbs-matview', 'adu-matview',
'hang-report-matview', 'nightly-builds-matview',
'build-adu-matview', 'crashes-by-user-matview',
'crashes-by-user-build-matview',
'correlations-matview', 'home-page-graph-matview',
'home-page-graph-matview-build',
'tcbs-build-matview', 'explosiveness-matview'):
self.assertTrue(app_name in information, app_name)
self.assertTrue(not information[app_name]['last_error'],
app_name)
self.assertTrue(information[app_name]['last_success'],
app_name)
self.assertEqual(self.psycopg2().cursor().callproc.call_count, 14)
for call in self.psycopg2().cursor().callproc.mock_calls:
__, call_args, __ = call
if len(call_args) > 1:
# e.g. ('update_signatures', [datetime.date(2012, 6, 25)])
# then check that it's a datetime.date instance
self.assertTrue(isinstance(call_args[1][0], datetime.date))
# the reason we expect 14 * 2 + 2 commit() calls is because,
# for each job it commits when it writes to the JSON database but
# postgresql jobs also commit the actual run. We have 16 jobs,
# 14 of them are postgresql jobs writing twice, 2 of them are
# regular jobs writing only once.
self.assertEqual(self.psycopg2().commit.call_count, 14 * 2 + 2)
def test_duplicates(self):
config_manager = self._setup_config_manager(
'socorro.cron.jobs.matviews.DuplicatesCronApp|1d'
)
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['duplicates']
assert not information['duplicates']['last_error']
assert information['duplicates']['last_success']
def test_basic_run_job_with_reports_with_existing_bugs_different(self):
config_manager = self._setup_config_manager(3)
cursor = self.conn.cursor()
cursor.execute('select count(*) from bugs')
count, = cursor.fetchone()
assert not count
cursor.execute('select count(*) from bug_associations')
count, = cursor.fetchone()
assert not count
# these are matching the SAMPLE_CSV above
cursor.execute("""insert into reports
(uuid,signature)
values
('123', 'legitimate(sig)');
""")
cursor.execute("""insert into reports
(uuid,signature)
values
('456', 'MWSBAR.DLL@0x2589f');
""")
cursor.execute("""insert into bugs
(id,status,resolution,short_desc)
values
(8, 'CLOSED', 'RESOLVED', 'Different');
""")
cursor.execute("""insert into bug_associations
(bug_id,signature)
values
(8, '@different');
""")
self.conn.commit()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
cursor.execute('select id, short_desc from bugs where id = 8')
bug = cursor.fetchone()
self.assertEqual(bug[1], 'newlines in sigs')
cursor.execute(
'select signature from bug_associations where bug_id = 8')
association = cursor.fetchone()
self.assertEqual(association[0], 'legitimate(sig)')
def test_reports_clean_with_dependency(self):
config_manager = self._setup_config_manager(
'socorro.cron.jobs.matviews.DuplicatesCronApp|1h\n'
'socorro.cron.jobs.matviews.ReportsCleanCronApp|1h'
)
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['reports-clean']
assert not information['reports-clean']['last_error']
assert information['reports-clean']['last_success']
def test_run_weekly_reports_partitions(self):
config_manager, json_file = self._setup_config_manager()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = json.load(open(json_file))
assert information['weekly-reports-partitions']
assert not information['weekly-reports-partitions']['last_error']
assert information['weekly-reports-partitions']['last_success']
# see https://bugzilla.mozilla.org/show_bug.cgi?id=828071
# it'll be twice,
# once for the stored proc and once for updating crontabber_state
self.assertEqual(self.psycopg2().commit.call_count, 2)
def test_cron_job(self, exacttarget_mock):
(config_manager, json_file) = self._setup_config_manager()
et_mock = exacttarget_mock.return_value
# Make get_subscriber raise an exception
list_service = et_mock.list.return_value = mock.Mock()
list_service.get_subscriber = mock.Mock(
side_effect=exacttarget.NewsletterException()
)
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = json.load(open(json_file))
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
self.assertEqual(et_mock.trigger_send.call_count, 3)
# Verify the last call to trigger_send
fields = {
'EMAIL_ADDRESS_': '*****@*****.**',
'EMAIL_FORMAT_': 'H',
'TOKEN': '*****@*****.**'
}
et_mock.trigger_send.assert_called_with('socorro_dev_test', fields)
# Verify that user's data was updated
cursor = self.conn.cursor()
emails_list = (
'*****@*****.**',
'*****@*****.**',
'*****@*****.**'
)
sql = """
SELECT last_sending
FROM emails
WHERE email IN %s
""" % (emails_list,)
cursor.execute(sql)
self.assertEqual(cursor.rowcount, 3)
now = utc_now()
for row in cursor.fetchall():
self.assertEqual(row[0].year, now.year)
self.assertEqual(row[0].month, now.month)
self.assertEqual(row[0].day, now.day)
def test_basic_run_no_errors(self):
# a mutable where commands sent are stored
commands_sent = []
self.Popen.side_effect = functools.partial(
mocked_Popen,
_commands_sent=commands_sent,
_exit_code=0,
_stdout='Bla bla',
_stderr='',
)
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['modulelist']
#print information['modulelist']['last_error']
#print information['modulelist']['last_error']['traceback']
if information['modulelist']['last_error']:
raise AssertionError(information['modulelist']['last_error'])
assert len(commands_sent) == 3
first = commands_sent[0]
second = commands_sent[1]
third = commands_sent[2]
yesterday = utc_now()
yesterday -= datetime.timedelta(days=1)
yesterday_fmt = yesterday.strftime('%Y%m%d')
self.assertTrue('PIG_CLASSPATH=/some/place pig' in first)
self.assertTrue('-param start_date=%s' % yesterday_fmt in first)
self.assertTrue('-param end_date=%s' % yesterday_fmt in first)
self.assertTrue('/some/place/modulelist.pig' in first)
self.assertTrue(
'PIG_CLASSPATH=/some/place hadoop fs -getmerge' in second)
self.assertTrue('modulelist-%s-%s' %
(yesterday_fmt, yesterday_fmt) in second)
self.assertTrue('/some/other/place/%s-modulelist.txt' %
(yesterday_fmt, ) in second)
self.assertTrue('PIG_CLASSPATH=/some/place hadoop fs ' in third)
self.assertTrue('modulelist-%s-%s' %
(yesterday_fmt, yesterday_fmt) in second)
# note that all jobs spew out 'Bla bla' on stdout
config.logger.info.assert_called_with('Bla bla')
def test_one_matview_alone(self):
config_manager, json_file = self._setup_config_manager(
'socorro.unittest.cron.jobs.test_matviews.ReportsCleanJob|1d\n'
'socorro.unittest.cron.jobs.test_matviews.FTPScraperJob|1d\n'
''
'socorro.cron.jobs.matviews.ProductVersionsCronApp|1d')
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
# not a huge fan of this test because it's so specific
from socorro.cron.jobs.matviews import ProductVersionsCronApp
proc_name = ProductVersionsCronApp.proc_name
(self.psycopg2().cursor().callproc.assert_called_once_with(
proc_name))
def test_run_job_no_data_but_scped(self):
config_manager, json_file = self._setup_config_manager(
public_output_path='',
private_user='******',
private_server='secure.mozilla.org',
private_location='/var/data/',
private_ssh_command='chmod 0640 /var/data/*',
)
def comm():
# no errors
return '', ''
self.Popen().communicate.side_effect = comm
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = json.load(open(json_file))
assert information['daily-url']
assert not information['daily-url']['last_error']
assert information['daily-url']['last_success']
# even though the files created are empty they should nevertheless
# be scp'ed
# can expect the command exactly
now = datetime.datetime.utcnow() - datetime.timedelta(days=1)
private = now.strftime('%Y%m%d-crashdata.csv.gz')
private_path = os.path.join(self.tempdir, private)
assert os.path.isfile(private_path)
scp_command = 'scp "%s" "[email protected]:/var/data/"' % private_path
ssh_command = 'ssh "*****@*****.**" "chmod 0640 /var/data/*"'
self.Popen.assert_any_call(
scp_command,
stdin=PIPE, stderr=PIPE, stdout=PIPE,
shell=True
)
self.Popen.assert_any_call(
ssh_command,
stdin=PIPE, stderr=PIPE, stdout=PIPE,
shell=True
)
def test_reports_clean_with_dependency(self):
config_manager, json_file = self._setup_config_manager(
'socorro.cron.jobs.matviews.DuplicatesCronApp|1h\n'
'socorro.cron.jobs.matviews.ReportsCleanCronApp|1h')
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = json.load(open(json_file))
assert information['reports-clean']
assert not information['reports-clean']['last_error']
assert information['reports-clean']['last_success']
# not a huge fan of this test because it's so specific
calls = self.psycopg2().cursor().callproc.mock_calls
call = calls[-1]
__, called, __ = list(call)
self.assertEqual(called[0], 'update_reports_clean')
def test_reprocessing_exception(self):
config_manager = self._setup_config_manager()
cursor = self.conn.cursor()
# Test exception handling
cursor.execute('drop table reprocessing_jobs')
self.conn.commit()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
cursor.execute("""
select json_extract_path_text(last_error, 'type')
from crontabber
""")
res_expected = "<class 'psycopg2.ProgrammingError'>"
res, = cursor.fetchone()
self.assertEqual(res, res_expected)
def test_cleanup_radix(self):
self.fsrts._current_slot = lambda: ['00', '00_00']
self.fsrts.save_raw_crash({"test": "TEST"}, {
'foo': 'bar',
self.fsrts.config.dump_field: 'baz'
}, self.CRASH_ID)
self.fsrts._current_slot = lambda: ['10', '00_01']
self.assertEqual(list(self.fsrts.new_crashes()), [self.CRASH_ID])
self.assertEqual(list(self.fsrts.new_crashes()), [])
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['cleanup_radix']
assert not information['cleanup_radix']['last_error']
assert information['cleanup_radix']['last_success']
self.assertEqual(os.listdir(self.fsrts.config.fs_root), [])
future = (utc_now() + datetime.timedelta(days=10)).strftime("%Y%m%d")
future_id = "0bba929f-8721-460c-dead-a43c%s" % future
self.fsrts._current_slot = lambda: ['00', '00_00']
self.fsrts.save_raw_crash({"test": "TEST"}, {
'foo': 'bar',
self.fsrts.config.dump_field: 'baz'
}, future_id)
self.fsrts._current_slot = lambda: ['10', '00_01']
self.assertEqual(list(self.fsrts.new_crashes()), [future_id])
self.assertEqual(list(self.fsrts.new_crashes()), [])
tab.run_all()
self.assertEqual(os.listdir(self.fsrts.config.fs_root), [future])
def test_basic_run_job_with_some_reports(self):
config_manager = self._setup_config_manager(3)
cursor = self.conn.cursor()
# these are matching the SAMPLE_CSV above
cursor.execute("""insert into reports
(uuid,signature)
values
('123', 'legitimate(sig)');
""")
cursor.execute("""insert into reports
(uuid,signature)
values
('456', 'MWSBAR.DLL@0x2589f');
""")
self.conn.commit()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
cursor.execute('select id from bugs order by id')
bugs = cursor.fetchall()
self.assertEqual(len(bugs), 2)
# the only bugs with matching those signatures are: 5 and 8
bug_ids = [x[0] for x in bugs]
self.assertEqual(bug_ids, [5, 8])
cursor.execute('select bug_id from bug_associations order by bug_id')
associations = cursor.fetchall()
self.assertEqual(len(associations), 2)
bug_ids = [x[0] for x in associations]
self.assertEqual(bug_ids, [5, 8])
def test_failing_pig_job(self):
# a mutable where commands sent are stored
commands_sent = []
self.Popen.side_effect = functools.partial(
mocked_Popen,
_commands_sent=commands_sent,
_exit_code=1,
_stdout='',
_stderr='First command failed :(',
)
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['modulelist']
assert information['modulelist']['last_error']
_traceback = information['modulelist']['last_error']['traceback']
self.assertTrue('pig run failed' in _traceback)
# the other two where cancelled
self.assertEqual(len(commands_sent), 1)
config.logger.error.assert_called_with('First command failed :(')
def test_failing_hadoop_cleanup_job(self):
# a mutable where commands sent are stored
commands_sent = []
self.Popen.side_effect = functools.partial(
mocked_Popen,
_commands_sent=commands_sent,
_exit_code=lambda cmd: 1 if cmd.count('-rmr') else 0,
_stdout='',
_stderr=lambda cmd: 'Poop' if cmd.count('-rmr') else '',
)
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['modulelist']
assert information['modulelist']['last_error']
_traceback = information['modulelist']['last_error']['traceback']
self.assertTrue('hadoop cleanup failed' in _traceback)
# the other two where cancelled
self.assertEqual(len(commands_sent), 3)
config.logger.error.assert_called_with('Poop')
def test_run_job_with_mocked_data_with_scp_errors(self):
config_manager, json_file = self._setup_config_manager(
public_output_path='',
private_user='******',
private_server='secure.mozilla.org',
private_location='/var/data/',
)
self._insert_waterwolf_mock_data()
def comm():
# some errors
return '', "CRAP!"
self.Popen().communicate.side_effect = comm
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = json.load(open(json_file))
assert information['daily-url']
assert not information['daily-url']['last_error']
assert information['daily-url']['last_success']
self.assertTrue(config.logger.warn.called)
def test_one_matview_alone(self):
config_manager = self._setup_config_manager(
'socorro.unittest.cron.jobs.test_matviews.ReportsCleanJob|1d\n'
'socorro.unittest.cron.jobs.test_matviews.FTPScraperJob|1d\n'
''
'socorro.cron.jobs.matviews.ProductVersionsCronApp|1d'
)
with config_manager.context() as config:
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['reports-clean']
assert not information['reports-clean']['last_error']
assert information['reports-clean']['last_success']
assert information['ftpscraper']
assert not information['ftpscraper']['last_error']
assert information['ftpscraper']['last_success']
assert information['product-versions-matview']
assert not information['product-versions-matview']['last_error']
assert information['product-versions-matview']['last_success']
def test_email_after_delay(self, exacttarget_mock):
"""Test that a user will receive an email if he or she sends us a new
crash report after the delay is passed (but not before). """
config_manager = self._setup_config_manager(
delay_between_emails=1, restrict_products=['EarthRaccoon'])
list_service_mock = exacttarget_mock.return_value.list.return_value
list_service_mock.get_subscriber.return_value = {'token': '*****@*****.**'}
trigger_send_mock = exacttarget_mock.return_value.trigger_send
tomorrow = utc_now() + datetime.timedelta(days=1, hours=2)
twohourslater = utc_now() + datetime.timedelta(hours=2)
with config_manager.context() as config:
# 1. Send an email to the user and update emailing data
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
exacttarget_mock.return_value.trigger_send.assert_called_with(
'socorro_dev_test', {
'EMAIL_ADDRESS_': '*****@*****.**',
'EMAIL_FORMAT_': 'H',
'TOKEN': '*****@*****.**'
})
self.assertEqual(trigger_send_mock.call_count, 1)
# 2. Test that before 'delay' is passed user doesn't receive
# another email
# Insert a new crash report with the same email address
cursor = self.conn.cursor()
cursor.execute(
"""
INSERT INTO reports
(uuid, email, product, version, release_channel, date_processed)
VALUES (
'50',
'*****@*****.**',
'EarthRaccoon',
'20.0',
'Release',
'%(onehourlater)s'
)
""" % {'onehourlater': utc_now() + datetime.timedelta(hours=1)})
self.conn.commit()
# Run crontabber with time pushed by two hours
with mock.patch('socorro.cron.crontabber.utc_now') as cronutc_mock:
with mock.patch('socorro.cron.base.utc_now') as baseutc_mock:
cronutc_mock.return_value = twohourslater
baseutc_mock.return_value = twohourslater
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
# No new email was sent
self.assertEqual(trigger_send_mock.call_count, 1)
# 3. Verify that, after 'delay' is passed, a new email is sent
# to our user
# Insert a new crash report with the same email address
cursor = self.conn.cursor()
cursor.execute("""
INSERT INTO reports
(uuid, email, product, version, release_channel, date_processed)
VALUES (
'51',
'*****@*****.**',
'EarthRaccoon',
'20.0',
'Release',
'%(tomorrow)s'
)
""" % {'tomorrow': utc_now() + datetime.timedelta(days=1)})
self.conn.commit()
# Run crontabber with time pushed by a day
with mock.patch('socorro.cron.crontabber.utc_now') as cronutc_mock:
with mock.patch('socorro.cron.base.utc_now') as baseutc_mock:
cronutc_mock.return_value = tomorrow
baseutc_mock.return_value = tomorrow
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
# A new email was sent
self.assertEqual(trigger_send_mock.call_count, 2)