def test_run_job_with_mocked_data(self): config_manager = self._setup_config_manager() self._insert_waterwolf_mock_data() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['daily-url'] assert not information['daily-url']['last_error'] assert information['daily-url']['last_success'] # this should have created two .csv.gz files now = datetime.datetime.utcnow() - datetime.timedelta(days=1) private = now.strftime('%Y%m%d-crashdata.csv.gz') public = now.strftime('%Y%m%d-pub-crashdata.csv.gz') self.assertTrue(private in os.listdir(self.tempdir)) self.assertTrue(public in os.listdir(self.tempdir)) private_path = os.path.join(self.tempdir, private) f = gzip.open(private_path) try: content = f.read() self.assertTrue(content) lines = content.splitlines() header = lines[0] payload = lines[1:] self.assertEqual(header.split('\t')[0], 'signature') self.assertEqual(header.split('\t')[1], 'url') urls = [x.split('\t')[1] for x in payload] self.assertTrue('http://p**n.xxx' in urls) signatures = [x.split('\t')[0] for x in payload] self.assertEqual(sorted(signatures), [ 'FakeSignature1', 'FakeSignature2', 'FakeSignature3', 'FakeSignature4' ]) finally: f.close() public_path = os.path.join(self.tempdir, public) f = gzip.open(public_path) try: content = f.read() self.assertTrue(content) lines = content.splitlines() header = lines[0] payload = lines[1:] self.assertEqual(header.split('\t')[0], 'signature') self.assertEqual(header.split('\t')[1], 'URL (removed)') urls = [x.split('\t')[1] for x in payload] self.assertTrue('http://p**n.xxx' not in urls) signatures = [x.split('\t')[0] for x in payload] self.assertEqual(sorted(signatures), [ 'FakeSignature1', 'FakeSignature2', 'FakeSignature3', 'FakeSignature4' ]) finally: f.close()
def test_all_matviews(self, mocked_utc_now): # Pretend it's 03AM UTC def mock_utc_now(): n = utc_now() n = n.replace(hour=3) return n mocked_utc_now.side_effect = mock_utc_now config_manager = self._setup_config_manager( 'socorro.unittest.cron.jobs.test_matviews.ReportsCleanJob|1d\n' 'socorro.unittest.cron.jobs.test_matviews.FTPScraperJob|1d\n' '' 'socorro.cron.jobs.matviews.ProductVersionsCronApp|1d\n' 'socorro.cron.jobs.matviews.SignaturesCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.TCBSCronApp|1d\n' 'socorro.cron.jobs.matviews.ADUCronApp|1d\n' 'socorro.cron.jobs.matviews.NightlyBuildsCronApp|1d\n' 'socorro.cron.jobs.matviews.BuildADUCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.CrashesByUserCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.CrashesByUserBuildCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.CorrelationsCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.HomePageGraphCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.HomePageGraphBuildCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.TCBSBuildCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.ExplosivenessCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.GraphicsDeviceCronApp|1d|02:00\n' ) with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() for app_name in ('product-versions-matview', 'signatures-matview', 'tcbs-matview', 'adu-matview', 'nightly-builds-matview', 'build-adu-matview', 'crashes-by-user-matview', 'crashes-by-user-build-matview', 'correlations-matview', 'home-page-graph-matview', 'home-page-graph-matview-build', 'tcbs-build-matview', 'explosiveness-matview', 'graphics-device-matview',): self.assertTrue(app_name in information, app_name) self.assertTrue( not information[app_name]['last_error'], app_name ) self.assertTrue( information[app_name]['last_success'], app_name )
def test_duplicates(self): config_manager, json_file = self._setup_config_manager( 'socorro.cron.jobs.matviews.DuplicatesCronApp|1d') with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = json.load(open(json_file)) assert information['duplicates'] assert not information['duplicates']['last_error'] assert information['duplicates']['last_success'] # not a huge fan of this test because it's so specific proc_name = 'update_reports_duplicates' calls = self.psycopg2().cursor().callproc.mock_calls call1, call2 = calls __, called, __ = call1 assert called[0] == proc_name, called[0] start, end = called[1] self.assertEqual(end - start, datetime.timedelta(hours=1)) __, called, __ = call2 assert called[0] == proc_name, called[0] start, end = called[1] self.assertEqual(end - start, datetime.timedelta(hours=1))
def test_run(self): config_manager = self._setup_config_manager() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['suspicious-crashes'] assert not information['suspicious-crashes']['last_error'] assert information['suspicious-crashes']['last_success'] cursor = self.conn.cursor() cursor.execute(""" SELECT signatures.signature, scs.report_date FROM suspicious_crash_signatures scs JOIN signatures ON scs.signature_id=signatures.signature_id """) count = 0 today = (utc_now() - datetime.timedelta(1)).date() for row in cursor.fetchall(): self.assertEquals('sig', row[0]) self.assertEquals(today, row[1].date()) count += 1 self.assertEquals(1, count)
def test_basic_run_job_without_reports(self): config_manager = self._setup_config_manager(3) cursor = self.conn.cursor() cursor.execute('select count(*) from reports') count, = cursor.fetchone() assert count == 0, "reports table not cleaned" cursor.execute('select count(*) from bugs') count, = cursor.fetchone() assert count == 0, "'bugs' table not cleaned" cursor.execute('select count(*) from bug_associations') count, = cursor.fetchone() assert count == 0, "'bug_associations' table not cleaned" with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] assert not information['bugzilla-associations']['last_error'] assert information['bugzilla-associations']['last_success'] # now, because there we no matching signatures in the reports table # it means that all bugs are rejected cursor.execute('select count(*) from bugs') count, = cursor.fetchone() self.assertTrue(not count) cursor.execute('select count(*) from bug_associations') count, = cursor.fetchone() self.assertTrue(not count)
def test_run_job_with_no_data_with_ssh_errors(self): config_manager, json_file = self._setup_config_manager( public_output_path='', private_user='******', private_server='secure.mozilla.org', private_location='/var/data/', private_ssh_command='chmod 0640 /var/data/*', ) self._insert_waterwolf_mock_data() # any mutable so we can keep track of the number of times # the side_effect function is called calls = [] def comm(): if calls: # some errors return '', "CRAP!" else: calls.append(1) return '', '' self.Popen().communicate.side_effect = comm with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = json.load(open(json_file)) assert information['daily-url'] assert not information['daily-url']['last_error'] assert information['daily-url']['last_success'] self.assertTrue(config.logger.warn.called)
def test_run_job_with_mocked_data_with_wrong_products(self): config_manager, json_file = self._setup_config_manager( product='Thunderbird,SeaMonkey', version='1.0,2.0', public_output_path=False ) self._insert_waterwolf_mock_data() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = json.load(open(json_file)) assert information['daily-url'] assert not information['daily-url']['last_error'] assert information['daily-url']['last_success'] # this should have created two .csv.gz files now = datetime.datetime.utcnow() - datetime.timedelta(days=1) private = now.strftime('%Y%m%d-crashdata.csv.gz') public = now.strftime('%Y%m%d-pub-crashdata.csv.gz') self.assertTrue(private in os.listdir(self.tempdir)) self.assertTrue(public not in os.listdir(self.tempdir)) private_path = os.path.join(self.tempdir, private) f = gzip.open(private_path) try: self.assertEqual(f.read(), '') finally: f.close()
def test_basic_run_job_no_data(self): config_manager, json_file = self._setup_config_manager() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = json.load(open(json_file)) assert information['daily-url'] assert not information['daily-url']['last_error'] assert information['daily-url']['last_success'] # this should have created two .csv.gz files now = datetime.datetime.utcnow() - datetime.timedelta(days=1) private = now.strftime('%Y%m%d-crashdata.csv.gz') public = now.strftime('%Y%m%d-pub-crashdata.csv.gz') self.assertTrue(private in os.listdir(self.tempdir)) self.assertTrue(public in os.listdir(self.tempdir)) private_path = os.path.join(self.tempdir, private) f = gzip.open(private_path) try: self.assertEqual(f.read(), '') finally: f.close() public_path = os.path.join(self.tempdir, public) f = gzip.open(public_path) try: self.assertEqual(f.read(), '') finally: f.close()
def test_reprocessing(self): """ Simple test of reprocessing""" config_manager = self._setup_config_manager() cursor = self.conn.cursor() # Create partitions to support the status query # Load report_partition_info data cursor.execute(""" INSERT into reprocessing_jobs (crash_id) VALUES ('13c4a348-5d04-11e3-8118-d231feb1dc81') """) # We have to do this here to accommodate separate crontabber processes self.conn.commit() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() cursor.execute('select count(*) from reprocessing_jobs') res_expected = 0 res, = cursor.fetchone() self.assertEqual(res, res_expected)
def test_run_weekly_reports_partitions(self): config_manager = self._setup_config_manager() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['weekly-reports-partitions'] assert not information['weekly-reports-partitions']['last_error'] assert information['weekly-reports-partitions']['last_success']
def test_reports_clean_dependency_prerequisite(self): config_manager, json_file = self._setup_config_manager( 'socorro.cron.jobs.matviews.ReportsCleanCronApp|1d') with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() # no file is created because it's unable to run anything self.assertTrue(not os.path.isfile(json_file))
def test_cron_job(self, exacttarget_mock): config_manager = self._setup_config_manager() et_mock = exacttarget_mock.return_value # Make get_subscriber raise an exception list_service = et_mock.list.return_value = mock.Mock() list_service.get_subscriber = mock.Mock( side_effect=exacttarget.NewsletterException() ) with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['automatic-emails'] assert not information['automatic-emails']['last_error'] assert information['automatic-emails']['last_success'] self.assertEqual(et_mock.trigger_send.call_count, 4) last_email = u'z\[email protected]' # Verify the last call to trigger_send fields = { 'EMAIL_ADDRESS_': last_email, 'EMAIL_FORMAT_': 'H', 'TOKEN': last_email } et_mock.trigger_send.assert_called_with('socorro_dev_test', fields) # Verify that user's data was updated conf = config.crontabber['class-AutomaticEmailsCronApp'] es = SuperS().es( urls=conf.elasticsearch.elasticsearch_urls, timeout=conf.elasticsearch.elasticsearch_timeout, ) search = es.indexes(conf.elasticsearch.elasticsearch_emails_index) search = search.doctypes('emails') es.get_es().refresh() emails_list = ( '*****@*****.**', '"Quidam" <*****@*****.**>', '*****@*****.**' ) search = search.filter(_id__in=emails_list) res = search.values_list('last_sending') self.assertEqual(len(res), 3) now = utc_now() for row in res: date = string_to_datetime(row[0]) self.assertEqual(date.year, now.year) self.assertEqual(date.month, now.month) self.assertEqual(date.day, now.day)
def test_all_matviews(self): config_manager, json_file = self._setup_config_manager( 'socorro.unittest.cron.jobs.test_matviews.ReportsCleanJob|1d\n' 'socorro.unittest.cron.jobs.test_matviews.FTPScraperJob|1d\n' '' 'socorro.cron.jobs.matviews.ProductVersionsCronApp|1d\n' 'socorro.cron.jobs.matviews.SignaturesCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.TCBSCronApp|1d\n' 'socorro.cron.jobs.matviews.ADUCronApp|1d\n' 'socorro.cron.jobs.matviews.HangReportCronApp|1d\n' 'socorro.cron.jobs.matviews.NightlyBuildsCronApp|1d\n' 'socorro.cron.jobs.matviews.BuildADUCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.CrashesByUserCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.CrashesByUserBuildCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.CorrelationsCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.HomePageGraphCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.HomePageGraphBuildCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.TCBSBuildCronApp|1d|02:00\n' 'socorro.cron.jobs.matviews.ExplosivenessCronApp|1d|02:00\n') with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = json.load(open(json_file)) for app_name in ('product-versions-matview', 'signatures-matview', 'tcbs-matview', 'adu-matview', 'hang-report-matview', 'nightly-builds-matview', 'build-adu-matview', 'crashes-by-user-matview', 'crashes-by-user-build-matview', 'correlations-matview', 'home-page-graph-matview', 'home-page-graph-matview-build', 'tcbs-build-matview', 'explosiveness-matview'): self.assertTrue(app_name in information, app_name) self.assertTrue(not information[app_name]['last_error'], app_name) self.assertTrue(information[app_name]['last_success'], app_name) self.assertEqual(self.psycopg2().cursor().callproc.call_count, 14) for call in self.psycopg2().cursor().callproc.mock_calls: __, call_args, __ = call if len(call_args) > 1: # e.g. ('update_signatures', [datetime.date(2012, 6, 25)]) # then check that it's a datetime.date instance self.assertTrue(isinstance(call_args[1][0], datetime.date)) # the reason we expect 14 * 2 + 2 commit() calls is because, # for each job it commits when it writes to the JSON database but # postgresql jobs also commit the actual run. We have 16 jobs, # 14 of them are postgresql jobs writing twice, 2 of them are # regular jobs writing only once. self.assertEqual(self.psycopg2().commit.call_count, 14 * 2 + 2)
def test_duplicates(self): config_manager = self._setup_config_manager( 'socorro.cron.jobs.matviews.DuplicatesCronApp|1d' ) with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['duplicates'] assert not information['duplicates']['last_error'] assert information['duplicates']['last_success']
def test_basic_run_job_with_reports_with_existing_bugs_different(self): config_manager = self._setup_config_manager(3) cursor = self.conn.cursor() cursor.execute('select count(*) from bugs') count, = cursor.fetchone() assert not count cursor.execute('select count(*) from bug_associations') count, = cursor.fetchone() assert not count # these are matching the SAMPLE_CSV above cursor.execute("""insert into reports (uuid,signature) values ('123', 'legitimate(sig)'); """) cursor.execute("""insert into reports (uuid,signature) values ('456', 'MWSBAR.DLL@0x2589f'); """) cursor.execute("""insert into bugs (id,status,resolution,short_desc) values (8, 'CLOSED', 'RESOLVED', 'Different'); """) cursor.execute("""insert into bug_associations (bug_id,signature) values (8, '@different'); """) self.conn.commit() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] assert not information['bugzilla-associations']['last_error'] assert information['bugzilla-associations']['last_success'] cursor.execute('select id, short_desc from bugs where id = 8') bug = cursor.fetchone() self.assertEqual(bug[1], 'newlines in sigs') cursor.execute( 'select signature from bug_associations where bug_id = 8') association = cursor.fetchone() self.assertEqual(association[0], 'legitimate(sig)')
def test_reports_clean_with_dependency(self): config_manager = self._setup_config_manager( 'socorro.cron.jobs.matviews.DuplicatesCronApp|1h\n' 'socorro.cron.jobs.matviews.ReportsCleanCronApp|1h' ) with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['reports-clean'] assert not information['reports-clean']['last_error'] assert information['reports-clean']['last_success']
def test_run_weekly_reports_partitions(self): config_manager, json_file = self._setup_config_manager() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = json.load(open(json_file)) assert information['weekly-reports-partitions'] assert not information['weekly-reports-partitions']['last_error'] assert information['weekly-reports-partitions']['last_success'] # see https://bugzilla.mozilla.org/show_bug.cgi?id=828071 # it'll be twice, # once for the stored proc and once for updating crontabber_state self.assertEqual(self.psycopg2().commit.call_count, 2)
def test_cron_job(self, exacttarget_mock): (config_manager, json_file) = self._setup_config_manager() et_mock = exacttarget_mock.return_value # Make get_subscriber raise an exception list_service = et_mock.list.return_value = mock.Mock() list_service.get_subscriber = mock.Mock( side_effect=exacttarget.NewsletterException() ) with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = json.load(open(json_file)) assert information['automatic-emails'] assert not information['automatic-emails']['last_error'] assert information['automatic-emails']['last_success'] self.assertEqual(et_mock.trigger_send.call_count, 3) # Verify the last call to trigger_send fields = { 'EMAIL_ADDRESS_': '*****@*****.**', 'EMAIL_FORMAT_': 'H', 'TOKEN': '*****@*****.**' } et_mock.trigger_send.assert_called_with('socorro_dev_test', fields) # Verify that user's data was updated cursor = self.conn.cursor() emails_list = ( '*****@*****.**', '*****@*****.**', '*****@*****.**' ) sql = """ SELECT last_sending FROM emails WHERE email IN %s """ % (emails_list,) cursor.execute(sql) self.assertEqual(cursor.rowcount, 3) now = utc_now() for row in cursor.fetchall(): self.assertEqual(row[0].year, now.year) self.assertEqual(row[0].month, now.month) self.assertEqual(row[0].day, now.day)
def test_basic_run_no_errors(self): # a mutable where commands sent are stored commands_sent = [] self.Popen.side_effect = functools.partial( mocked_Popen, _commands_sent=commands_sent, _exit_code=0, _stdout='Bla bla', _stderr='', ) config_manager = self._setup_config_manager() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['modulelist'] #print information['modulelist']['last_error'] #print information['modulelist']['last_error']['traceback'] if information['modulelist']['last_error']: raise AssertionError(information['modulelist']['last_error']) assert len(commands_sent) == 3 first = commands_sent[0] second = commands_sent[1] third = commands_sent[2] yesterday = utc_now() yesterday -= datetime.timedelta(days=1) yesterday_fmt = yesterday.strftime('%Y%m%d') self.assertTrue('PIG_CLASSPATH=/some/place pig' in first) self.assertTrue('-param start_date=%s' % yesterday_fmt in first) self.assertTrue('-param end_date=%s' % yesterday_fmt in first) self.assertTrue('/some/place/modulelist.pig' in first) self.assertTrue( 'PIG_CLASSPATH=/some/place hadoop fs -getmerge' in second) self.assertTrue('modulelist-%s-%s' % (yesterday_fmt, yesterday_fmt) in second) self.assertTrue('/some/other/place/%s-modulelist.txt' % (yesterday_fmt, ) in second) self.assertTrue('PIG_CLASSPATH=/some/place hadoop fs ' in third) self.assertTrue('modulelist-%s-%s' % (yesterday_fmt, yesterday_fmt) in second) # note that all jobs spew out 'Bla bla' on stdout config.logger.info.assert_called_with('Bla bla')
def test_one_matview_alone(self): config_manager, json_file = self._setup_config_manager( 'socorro.unittest.cron.jobs.test_matviews.ReportsCleanJob|1d\n' 'socorro.unittest.cron.jobs.test_matviews.FTPScraperJob|1d\n' '' 'socorro.cron.jobs.matviews.ProductVersionsCronApp|1d') with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() # not a huge fan of this test because it's so specific from socorro.cron.jobs.matviews import ProductVersionsCronApp proc_name = ProductVersionsCronApp.proc_name (self.psycopg2().cursor().callproc.assert_called_once_with( proc_name))
def test_run_job_no_data_but_scped(self): config_manager, json_file = self._setup_config_manager( public_output_path='', private_user='******', private_server='secure.mozilla.org', private_location='/var/data/', private_ssh_command='chmod 0640 /var/data/*', ) def comm(): # no errors return '', '' self.Popen().communicate.side_effect = comm with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = json.load(open(json_file)) assert information['daily-url'] assert not information['daily-url']['last_error'] assert information['daily-url']['last_success'] # even though the files created are empty they should nevertheless # be scp'ed # can expect the command exactly now = datetime.datetime.utcnow() - datetime.timedelta(days=1) private = now.strftime('%Y%m%d-crashdata.csv.gz') private_path = os.path.join(self.tempdir, private) assert os.path.isfile(private_path) scp_command = 'scp "%s" "[email protected]:/var/data/"' % private_path ssh_command = 'ssh "*****@*****.**" "chmod 0640 /var/data/*"' self.Popen.assert_any_call( scp_command, stdin=PIPE, stderr=PIPE, stdout=PIPE, shell=True ) self.Popen.assert_any_call( ssh_command, stdin=PIPE, stderr=PIPE, stdout=PIPE, shell=True )
def test_reports_clean_with_dependency(self): config_manager, json_file = self._setup_config_manager( 'socorro.cron.jobs.matviews.DuplicatesCronApp|1h\n' 'socorro.cron.jobs.matviews.ReportsCleanCronApp|1h') with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = json.load(open(json_file)) assert information['reports-clean'] assert not information['reports-clean']['last_error'] assert information['reports-clean']['last_success'] # not a huge fan of this test because it's so specific calls = self.psycopg2().cursor().callproc.mock_calls call = calls[-1] __, called, __ = list(call) self.assertEqual(called[0], 'update_reports_clean')
def test_reprocessing_exception(self): config_manager = self._setup_config_manager() cursor = self.conn.cursor() # Test exception handling cursor.execute('drop table reprocessing_jobs') self.conn.commit() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() cursor.execute(""" select json_extract_path_text(last_error, 'type') from crontabber """) res_expected = "<class 'psycopg2.ProgrammingError'>" res, = cursor.fetchone() self.assertEqual(res, res_expected)
def test_cleanup_radix(self): self.fsrts._current_slot = lambda: ['00', '00_00'] self.fsrts.save_raw_crash({"test": "TEST"}, { 'foo': 'bar', self.fsrts.config.dump_field: 'baz' }, self.CRASH_ID) self.fsrts._current_slot = lambda: ['10', '00_01'] self.assertEqual(list(self.fsrts.new_crashes()), [self.CRASH_ID]) self.assertEqual(list(self.fsrts.new_crashes()), []) config_manager = self._setup_config_manager() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['cleanup_radix'] assert not information['cleanup_radix']['last_error'] assert information['cleanup_radix']['last_success'] self.assertEqual(os.listdir(self.fsrts.config.fs_root), []) future = (utc_now() + datetime.timedelta(days=10)).strftime("%Y%m%d") future_id = "0bba929f-8721-460c-dead-a43c%s" % future self.fsrts._current_slot = lambda: ['00', '00_00'] self.fsrts.save_raw_crash({"test": "TEST"}, { 'foo': 'bar', self.fsrts.config.dump_field: 'baz' }, future_id) self.fsrts._current_slot = lambda: ['10', '00_01'] self.assertEqual(list(self.fsrts.new_crashes()), [future_id]) self.assertEqual(list(self.fsrts.new_crashes()), []) tab.run_all() self.assertEqual(os.listdir(self.fsrts.config.fs_root), [future])
def test_basic_run_job_with_some_reports(self): config_manager = self._setup_config_manager(3) cursor = self.conn.cursor() # these are matching the SAMPLE_CSV above cursor.execute("""insert into reports (uuid,signature) values ('123', 'legitimate(sig)'); """) cursor.execute("""insert into reports (uuid,signature) values ('456', 'MWSBAR.DLL@0x2589f'); """) self.conn.commit() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] assert not information['bugzilla-associations']['last_error'] assert information['bugzilla-associations']['last_success'] cursor.execute('select id from bugs order by id') bugs = cursor.fetchall() self.assertEqual(len(bugs), 2) # the only bugs with matching those signatures are: 5 and 8 bug_ids = [x[0] for x in bugs] self.assertEqual(bug_ids, [5, 8]) cursor.execute('select bug_id from bug_associations order by bug_id') associations = cursor.fetchall() self.assertEqual(len(associations), 2) bug_ids = [x[0] for x in associations] self.assertEqual(bug_ids, [5, 8])
def test_failing_pig_job(self): # a mutable where commands sent are stored commands_sent = [] self.Popen.side_effect = functools.partial( mocked_Popen, _commands_sent=commands_sent, _exit_code=1, _stdout='', _stderr='First command failed :(', ) config_manager = self._setup_config_manager() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['modulelist'] assert information['modulelist']['last_error'] _traceback = information['modulelist']['last_error']['traceback'] self.assertTrue('pig run failed' in _traceback) # the other two where cancelled self.assertEqual(len(commands_sent), 1) config.logger.error.assert_called_with('First command failed :(')
def test_failing_hadoop_cleanup_job(self): # a mutable where commands sent are stored commands_sent = [] self.Popen.side_effect = functools.partial( mocked_Popen, _commands_sent=commands_sent, _exit_code=lambda cmd: 1 if cmd.count('-rmr') else 0, _stdout='', _stderr=lambda cmd: 'Poop' if cmd.count('-rmr') else '', ) config_manager = self._setup_config_manager() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['modulelist'] assert information['modulelist']['last_error'] _traceback = information['modulelist']['last_error']['traceback'] self.assertTrue('hadoop cleanup failed' in _traceback) # the other two where cancelled self.assertEqual(len(commands_sent), 3) config.logger.error.assert_called_with('Poop')
def test_run_job_with_mocked_data_with_scp_errors(self): config_manager, json_file = self._setup_config_manager( public_output_path='', private_user='******', private_server='secure.mozilla.org', private_location='/var/data/', ) self._insert_waterwolf_mock_data() def comm(): # some errors return '', "CRAP!" self.Popen().communicate.side_effect = comm with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = json.load(open(json_file)) assert information['daily-url'] assert not information['daily-url']['last_error'] assert information['daily-url']['last_success'] self.assertTrue(config.logger.warn.called)
def test_one_matview_alone(self): config_manager = self._setup_config_manager( 'socorro.unittest.cron.jobs.test_matviews.ReportsCleanJob|1d\n' 'socorro.unittest.cron.jobs.test_matviews.FTPScraperJob|1d\n' '' 'socorro.cron.jobs.matviews.ProductVersionsCronApp|1d' ) with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['reports-clean'] assert not information['reports-clean']['last_error'] assert information['reports-clean']['last_success'] assert information['ftpscraper'] assert not information['ftpscraper']['last_error'] assert information['ftpscraper']['last_success'] assert information['product-versions-matview'] assert not information['product-versions-matview']['last_error'] assert information['product-versions-matview']['last_success']
def test_email_after_delay(self, exacttarget_mock): """Test that a user will receive an email if he or she sends us a new crash report after the delay is passed (but not before). """ config_manager = self._setup_config_manager( delay_between_emails=1, restrict_products=['EarthRaccoon']) list_service_mock = exacttarget_mock.return_value.list.return_value list_service_mock.get_subscriber.return_value = {'token': '*****@*****.**'} trigger_send_mock = exacttarget_mock.return_value.trigger_send tomorrow = utc_now() + datetime.timedelta(days=1, hours=2) twohourslater = utc_now() + datetime.timedelta(hours=2) with config_manager.context() as config: # 1. Send an email to the user and update emailing data tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['automatic-emails'] assert not information['automatic-emails']['last_error'] assert information['automatic-emails']['last_success'] exacttarget_mock.return_value.trigger_send.assert_called_with( 'socorro_dev_test', { 'EMAIL_ADDRESS_': '*****@*****.**', 'EMAIL_FORMAT_': 'H', 'TOKEN': '*****@*****.**' }) self.assertEqual(trigger_send_mock.call_count, 1) # 2. Test that before 'delay' is passed user doesn't receive # another email # Insert a new crash report with the same email address cursor = self.conn.cursor() cursor.execute( """ INSERT INTO reports (uuid, email, product, version, release_channel, date_processed) VALUES ( '50', '*****@*****.**', 'EarthRaccoon', '20.0', 'Release', '%(onehourlater)s' ) """ % {'onehourlater': utc_now() + datetime.timedelta(hours=1)}) self.conn.commit() # Run crontabber with time pushed by two hours with mock.patch('socorro.cron.crontabber.utc_now') as cronutc_mock: with mock.patch('socorro.cron.base.utc_now') as baseutc_mock: cronutc_mock.return_value = twohourslater baseutc_mock.return_value = twohourslater tab.run_all() information = self._load_structure() assert information['automatic-emails'] assert not information['automatic-emails']['last_error'] assert information['automatic-emails']['last_success'] # No new email was sent self.assertEqual(trigger_send_mock.call_count, 1) # 3. Verify that, after 'delay' is passed, a new email is sent # to our user # Insert a new crash report with the same email address cursor = self.conn.cursor() cursor.execute(""" INSERT INTO reports (uuid, email, product, version, release_channel, date_processed) VALUES ( '51', '*****@*****.**', 'EarthRaccoon', '20.0', 'Release', '%(tomorrow)s' ) """ % {'tomorrow': utc_now() + datetime.timedelta(days=1)}) self.conn.commit() # Run crontabber with time pushed by a day with mock.patch('socorro.cron.crontabber.utc_now') as cronutc_mock: with mock.patch('socorro.cron.base.utc_now') as baseutc_mock: cronutc_mock.return_value = tomorrow baseutc_mock.return_value = tomorrow tab.run_all() information = self._load_structure() assert information['automatic-emails'] assert not information['automatic-emails']['last_error'] assert information['automatic-emails']['last_success'] # A new email was sent self.assertEqual(trigger_send_mock.call_count, 2)