def test_cron_job(self, exacttarget_mock):
config_manager = self._setup_config_manager()
et_mock = exacttarget_mock.return_value
# Make get_subscriber raise an exception
list_service = et_mock.list.return_value = mock.Mock()
list_service.get_subscriber = mock.Mock(
side_effect=exacttarget.NewsletterException())
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
eq_(et_mock.trigger_send.call_count, 4)
last_email = u'z\[email protected]'
# Verify the last call to trigger_send
fields = {
'EMAIL_ADDRESS_': last_email,
'EMAIL_FORMAT_': 'H',
'TOKEN': last_email
}
et_mock.trigger_send.assert_called_with('socorro_dev_test', fields)
# Verify that user's data was updated
conf = config.crontabber['class-AutomaticEmailsCronApp']
es = SuperS().es(
urls=conf.elasticsearch.elasticsearch_urls,
timeout=conf.elasticsearch.elasticsearch_timeout,
)
search = es.indexes(conf.elasticsearch.elasticsearch_emails_index)
search = search.doctypes('emails')
es.get_es().refresh()
emails_list = ('*****@*****.**',
'"Quidam" <*****@*****.**>',
'*****@*****.**')
search = search.filter(_id__in=emails_list)
res = search.values_list('last_sending')
eq_(len(res), 3)
now = utc_now()
for row in res:
date = string_to_datetime(row[0])
eq_(date.year, now.year)
eq_(date.month, now.month)
eq_(date.day, now.day)
def test_mocked_fetch(self):
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['fetch-adi-from-hive']
assert not information['fetch-adi-from-hive']['last_error']
config.logger.info.assert_called_with(
'Faking the fetching of ADI from Hive :)')
def test_reports_clean_with_dependency(self):
config_manager = self._setup_config_manager(
'socorro.cron.jobs.matviews.DuplicatesCronApp|1h\n'
'socorro.cron.jobs.matviews.ReportsCleanCronApp|1h')
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['reports-clean']
assert not information['reports-clean']['last_error']
assert information['reports-clean']['last_success']
def test_run_job_based_on_last_success(self, requests_mocker):
"""specifically setting 0 days back and no prior run
will pick it up from now's date"""
requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
config_manager = self._setup_config_manager(0)
cursor = self.conn.cursor()
# these are matching the SAMPLE_CSV above
cursor.execute("""insert into bug_associations
(bug_id,signature)
values
(8, 'legitimate(sig)');
""")
self.conn.commit()
# second time
config_manager = self._setup_config_manager(0)
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
state = tab.job_state_database.copy()
self._wind_clock(state, days=1)
tab.job_state_database.update(state)
# Create a CSV file for one day back.
# This'll make sure there's a .csv file whose day
# is that of the last run.
self._setup_config_manager(1)
config_manager = self._setup_config_manager(0)
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
def test_duplicates(self):
config_manager = self._setup_config_manager(
'socorro.cron.jobs.matviews.DuplicatesCronApp|1d'
)
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['duplicates']
assert not information['duplicates']['last_error']
assert information['duplicates']['last_success']
def test_all_matviews(self, mocked_utc_now):
# Pretend it's 03AM UTC
def mock_utc_now():
n = utc_now()
n = n.replace(hour=3)
return n
mocked_utc_now.side_effect = mock_utc_now
config_manager = self._setup_config_manager(
'socorro.unittest.cron.jobs.test_matviews.ReportsCleanJob|1d\n'
'socorro.unittest.cron.jobs.test_matviews.FTPScraperJob|1d\n'
''
'socorro.cron.jobs.matviews.ProductVersionsCronApp|1d\n'
'socorro.cron.jobs.matviews.SignaturesCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.TCBSCronApp|1d\n'
'socorro.cron.jobs.matviews.ADUCronApp|1d\n'
'socorro.cron.jobs.matviews.BuildADUCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.CrashesByUserCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.CrashesByUserBuildCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.HomePageGraphCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.HomePageGraphBuildCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.TCBSBuildCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.ExplosivenessCronApp|1d|02:00\n'
'socorro.cron.jobs.matviews.GraphicsDeviceCronApp|1d|02:00\n')
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
for app_name in (
'product-versions-matview',
'signatures-matview',
'tcbs-matview',
'adu-matview',
'build-adu-matview',
'crashes-by-user-matview',
'crashes-by-user-build-matview',
'home-page-graph-matview',
'home-page-graph-matview-build',
'tcbs-build-matview',
'explosiveness-matview',
'graphics-device-matview',
):
ok_(app_name in information, app_name)
ok_(not information[app_name]['last_error'], app_name)
ok_(information[app_name]['last_success'], app_name)
def test_basic_run_job_with_reports_with_existing_bugs_same(self):
config_manager = self._setup_config_manager(3)
cursor = self.conn.cursor()
# these are matching the SAMPLE_CSV above
cursor.execute("""insert into reports
(uuid,signature)
values
('123', 'legitimate(sig)');
""")
cursor.execute("""insert into reports
(uuid,signature)
values
('456', 'MWSBAR.DLL@0x2589f');
""")
# exactly the same as the fixture
cursor.execute("""insert into bugs
(id,status,resolution,short_desc)
values
(8, 'CLOSED', 'RESOLVED', 'newlines in sigs');
""")
cursor.execute("""insert into bug_associations
(bug_id,signature)
values
(8, 'legitimate(sig)');
""")
self.conn.commit()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
cursor.execute('select id, short_desc from bugs where id = 8')
bug = cursor.fetchone()
eq_(bug[1], 'newlines in sigs')
cursor.execute(
'select signature from bug_associations where bug_id = 8')
association = cursor.fetchone()
eq_(association[0], 'legitimate(sig)')
cursor.execute('select * from bug_associations')
def test_basic_run_no_errors(self):
# a mutable where commands sent are stored
commands_sent = []
self.Popen.side_effect = functools.partial(
mocked_Popen,
_commands_sent=commands_sent,
_exit_code=0,
_stdout='Bla bla',
_stderr='',
)
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['modulelist']
#print information['modulelist']['last_error']
#print information['modulelist']['last_error']['traceback']
if information['modulelist']['last_error']:
raise AssertionError(information['modulelist']['last_error'])
assert len(commands_sent) == 3
first = commands_sent[0]
second = commands_sent[1]
third = commands_sent[2]
yesterday = utc_now()
yesterday -= datetime.timedelta(days=1)
yesterday_fmt = yesterday.strftime('%Y%m%d')
ok_('PIG_CLASSPATH=/some/place pig' in first)
ok_('-param start_date=%s' % yesterday_fmt in first)
ok_('-param end_date=%s' % yesterday_fmt in first)
ok_('/some/place/modulelist.pig' in first)
ok_('PIG_CLASSPATH=/some/place hadoop fs -getmerge' in second)
ok_('modulelist-%s-%s' % (yesterday_fmt, yesterday_fmt) in second)
ok_('/some/other/place/%s-modulelist.txt' %
(yesterday_fmt, ) in second)
ok_('PIG_CLASSPATH=/some/place hadoop fs ' in third)
ok_('modulelist-%s-%s' % (yesterday_fmt, yesterday_fmt) in second)
# note that all jobs spew out 'Bla bla' on stdout
config.logger.info.assert_called_with('Bla bla')
def test_download_error(self, rget):
config_manager = self._setup_config_manager()
def mocked_get(url):
return Response('not here', status_code=404)
rget.side_effect = mocked_get
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['featured-versions-automatic']
assert information['featured-versions-automatic']['last_error']
error = information['featured-versions-automatic']['last_error']
ok_('DownloadError' in error['type'])
ok_('404' in error['value'])
def test_symbols_unpack_non_archive_file(self):
source_file = os.path.join(self.temp_source_directory,
os.path.basename(__file__))
shutil.copy(__file__, source_file)
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
config.logger.warning.assert_called_with(
"Don't know how to unpack %s" % (source_file, ))
information = self._load_structure()
assert information['symbols-unpack']
assert not information['symbols-unpack']['last_error']
assert information['symbols-unpack']['last_success']
def test_run_job_no_data_but_scped(self):
config_manager = self._setup_config_manager(
public_output_path='',
private_user='******',
private_server='secure.mozilla.org',
private_location='/var/data/',
private_ssh_command='chmod 0640 /var/data/*',
)
def comm():
# no errors
return '', ''
self.Popen().communicate.side_effect = comm
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['daily-url']
assert not information['daily-url']['last_error']
assert information['daily-url']['last_success']
# even though the files created are empty they should nevertheless
# be scp'ed
# can expect the command exactly
now = datetime.datetime.utcnow() - datetime.timedelta(days=1)
private = now.strftime('%Y%m%d-crashdata.csv.gz')
private_path = os.path.join(self.tempdir, private)
assert os.path.isfile(private_path)
scp_command = 'scp "%s" "[email protected]:/var/data/"' % private_path
ssh_command = 'ssh "*****@*****.**" "chmod 0640 /var/data/*"'
self.Popen.assert_any_call(scp_command,
stdin=PIPE,
stderr=PIPE,
stdout=PIPE,
shell=True)
self.Popen.assert_any_call(ssh_command,
stdin=PIPE,
stderr=PIPE,
stdout=PIPE,
shell=True)
def test_with_bugzilla_failure(self, requests_mocker):
requests_mocker.get(
BUGZILLA_BASE_URL,
text='error loading content',
status_code=500
)
config_manager = self._setup_config_manager(3)
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
# There has been an error.
last_error = information['bugzilla-associations']['last_error']
assert last_error
assert 'HTTPError' in last_error['type']
assert not information['bugzilla-associations']['last_success']
def test_server_status(self):
""" Simple test of status monitor """
config_manager = self._setup_config_manager()
cursor = self.conn.cursor()
# Create partitions to support the status query
# Load report_partition_info data
cursor.execute("""
INSERT into report_partition_info
(table_name, build_order, keys, indexes,
fkeys, partition_column, timetype)
VALUES
('reports', '1', '{id,uuid}',
'{date_processed,hangid,"product,version",reason,signature,url}',
'{}', 'date_processed', 'TIMESTAMPTZ')
""")
cursor.execute('SELECT weekly_report_partitions()')
# We have to do this here to accommodate separate crontabber processes
self.conn.commit()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
cursor.execute('select count(*) from server_status')
res_expected = 1
res, = cursor.fetchone()
eq_(res, res_expected)
cursor.execute("""select
date_recently_completed
, date_oldest_job_queued -- is NULL until we upgrade Rabbit
, avg_process_sec
, waiting_job_count -- should be 1
-- , date_created -- leaving timestamp verification out
from server_status
""")
res_expected = (None, None, 0.0, 1)
res = cursor.fetchone()
eq_(res, res_expected)
def test_run(self, connect_s3):
key = mock.MagicMock()
connect_s3().get_bucket().get_key.return_value = None
connect_s3().get_bucket().new_key.return_value = key
with self._setup_config_manager().context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
app_name = 'upload-crash-report-json-schema'
ok_(information[app_name])
ok_(not information[app_name]['last_error'])
ok_(information[app_name]['last_success'])
key.set_contents_from_string.assert_called_with(
CRASH_REPORT_JSON_SCHEMA_AS_STRING
)
def test_run_job_one_day_back(self):
"""specifically setting 0 days back and no priror run
will pick it up from now's date"""
config_manager = self._setup_config_manager(1)
cursor = self.conn.cursor()
# these are matching the SAMPLE_CSV above
cursor.execute("""insert into reports
(uuid,signature)
values
('123', 'legitimate(sig)');
""")
cursor.execute("""insert into reports
(uuid,signature)
values
('456', 'MWSBAR.DLL@0x2589f');
""")
# exactly the same as the fixture
cursor.execute("""insert into bugs
(id,status,resolution,short_desc)
values
(8, 'CLOSED', 'RESOLVED', 'newlines in sigs');
""")
cursor.execute("""insert into bug_associations
(bug_id,signature)
values
(8, 'legitimate(sig)');
""")
self.conn.commit()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
cursor.execute('select id, short_desc from bugs where id = 8')
bug = cursor.fetchone()
eq_(bug[1], 'newlines in sigs')
def test_basic_run(self):
# We need to prepare to return a size for the new key
self.mock_key.size = 123456789
self.mock_key.generate_url.return_value = (
'https://s3.example.com/latest.csv')
# Run the crontabber job to remove the test table.
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
# Basic assertion test of stored procedure.
information = self._load_structure()
assert information['missing-symbols']
assert not information['missing-symbols']['last_error']
assert information['missing-symbols']['last_success']
self.mock_boto_class()._connect.assert_called_with()
self.mock_boto_class.close.assert_called_with()
self.mock_bucket.new_key.assert_called_with('latest.csv')
content = StringIO()
writer = csv.writer(content)
writer.writerow((
'debug_file',
'debug_id',
'code_file',
'code_id',
))
writer.writerow((
'nvwgf2um.pdb',
'9D492B844FF34800B34320464AA1E7E41',
'nvwgf2um.dll',
'561D1D4Ff58000',
))
self.mock_key.set_contents_from_string.assert_called_with(
content.getvalue())
# this is becausse 123456789 bytes is 117.74 Mb
tab.config.logger.info.assert_called_with(
'Generated https://s3.example.com/latest.csv '
'(123,456,789 bytes, 117.74 Mb)')
def test_half_way_exception(self):
"""If the save_raw_crash() fails on the second crash_id of 2,
the first one should be removed from the table."""
config_manager = self._setup_config_manager()
cursor = self.conn.cursor()
cursor.execute("""
INSERT INTO reprocessing_jobs
(crash_id)
VALUES
('13c4a348-5d04-11e3-8118-d231feb1dc81'),
('23d5b459-6e15-22f4-9229-e342ffc2ed92')
""")
self.conn.commit()
def mocked_save_raw_crash(raw_crash, dumps, crash_id):
if crash_id == '23d5b459-6e15-22f4-9229-e342ffc2ed92':
raise Exception('something unpredictable happened')
self.rabbit_queue_mocked().save_raw_crash.side_effect = (
mocked_save_raw_crash
)
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = tab.job_state_database['reprocessing-jobs']
# Note, we're expecting it to fail.
assert information['last_error']
eq_(
information['last_error']['value'],
'something unpredictable happened'
)
assert not information['last_success']
cursor = self.conn.cursor()
cursor.execute('select crash_id from reprocessing_jobs')
records = cursor.fetchall()
eq_(len(records), 1)
crash_id, = records[0]
eq_(crash_id, '23d5b459-6e15-22f4-9229-e342ffc2ed92')
def test_fetch_with_zero_hive_results(self, fake_hive):
config_manager = self._setup_config_manager()
def return_test_data(fake):
# a generator that yields literally nothing
# http://stackoverflow.com/a/13243870/205832
return
yield
fake_hive.connect.return_value \
.cursor.return_value.__iter__ = return_test_data
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['fetch-adi-from-hive']
assert information['fetch-adi-from-hive']['last_error']
ok_(
NoRowsWritten.__name__ in
information['fetch-adi-from-hive']['last_error']['type']
)
assert information['fetch-adi-from-hive']['last_error']
fake_hive.connect.assert_called_with(
database='default',
authMechanism='PLAIN',
host='localhost',
user='******',
password='******',
port=10000,
timeout=1800000,
)
pgcursor = self.conn.cursor()
pgcursor.execute(
"select count(*) from raw_adi_logs"
)
count, = pgcursor.fetchone()
eq_(count, 0)
def test_symbols_unpack_subdirectories_careful_dir_cleanup(self):
"""same test almost as test_symbols_unpack_subdirectories()
but this time we put a file in one of the directories and assert
that that does not get deleted"""
root_dir = self.temp_source_directory
foo_dir = os.path.join(root_dir, 'foo')
os.mkdir(foo_dir)
with open(os.path.join(foo_dir, 'some.file'), 'w') as f:
f.write('anything')
bar_dir = os.path.join(foo_dir, 'bar')
os.mkdir(bar_dir)
source_file = os.path.join(bar_dir, os.path.basename(ZIP_FILE))
shutil.copy(ZIP_FILE, source_file)
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['symbols-unpack']
assert not information['symbols-unpack']['last_error'], (
information['symbols-unpack']['last_error'])
assert information['symbols-unpack']['last_success']
ok_(os.listdir(self.temp_destination_directory), 'empty')
# there should now be a directory named `sample-<todays date>
destination_dir = self.temp_destination_directory
ok_(os.path.isdir(destination_dir))
# and it should contain files
ok_(os.listdir(destination_dir))
# and the original should now have been deleted
ok_(not os.path.isfile(source_file))
# because there was nothing else in that directory, or its parent
# those directories should be removed now
ok_(not os.path.isdir(bar_dir))
ok_(os.path.isdir(foo_dir))
ok_(os.path.isfile(os.path.join(foo_dir, 'some.file')))
assert os.path.isdir(root_dir)
def test_reprocessing(self):
""" Simple test of reprocessing"""
config_manager = self._setup_config_manager()
c = config_manager.get_config()
cursor = self.conn.cursor()
# Create partitions to support the status query
# Load report_partition_info data
cursor.execute("""
INSERT into reprocessing_jobs
(crash_id)
VALUES
('13c4a348-5d04-11e3-8118-d231feb1dc81')
""")
# We have to do this here to accommodate separate crontabber processes
self.conn.commit()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = tab.job_state_database['reprocessing-jobs']
assert not information['last_error']
assert information['last_success']
cursor = self.conn.cursor()
cursor.execute('select count(*) from reprocessing_jobs')
res_expected = 0
res, = cursor.fetchone()
eq_(res, res_expected)
self.rabbit_queue_mocked.return_value.save_raw_crash \
.assert_called_once_with(
DotDict({'legacy_processing': 0}),
[],
'13c4a348-5d04-11e3-8118-d231feb1dc81'
)
def test_basic_run_job_with_some_reports(self):
config_manager = self._setup_config_manager(3)
cursor = self.conn.cursor()
# these are matching the SAMPLE_CSV above
cursor.execute("""insert into reports
(uuid,signature)
values
('123', 'legitimate(sig)');
""")
cursor.execute("""insert into reports
(uuid,signature)
values
('456', 'MWSBAR.DLL@0x2589f');
""")
self.conn.commit()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
cursor.execute('select id from bugs order by id')
bugs = cursor.fetchall()
eq_(len(bugs), 2)
# the only bugs with matching those signatures are: 5 and 8
bug_ids = [x[0] for x in bugs]
eq_(bug_ids, [5, 8])
cursor.execute('select bug_id from bug_associations order by bug_id')
associations = cursor.fetchall()
eq_(len(associations), 2)
bug_ids = [x[0] for x in associations]
eq_(bug_ids, [5, 8])
def test_failing_hadoop_cleanup_job(self):
# a mutable where commands sent are stored
commands_sent = []
self.Popen.side_effect = functools.partial(
mocked_Popen,
_commands_sent=commands_sent,
_exit_code=lambda cmd: 1 if cmd.count('-rmr') else 0,
_stdout='',
_stderr=lambda cmd: 'Iniquity' if cmd.count('-rmr') else '',
)
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['modulelist']
assert information['modulelist']['last_error']
_traceback = information['modulelist']['last_error']['traceback']
ok_('hadoop cleanup failed' in _traceback)
# the other two where cancelled
eq_(len(commands_sent), 3)
config.logger.error.has_calls([mock.call('Iniquity')])
def test_one_matview_alone(self):
config_manager = self._setup_config_manager(
'socorro.unittest.cron.jobs.test_matviews.ReportsCleanJob|1d\n'
'socorro.unittest.cron.jobs.test_matviews.FTPScraperJob|1d\n'
''
'socorro.cron.jobs.matviews.ProductVersionsCronApp|1d')
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['reports-clean']
assert not information['reports-clean']['last_error']
assert information['reports-clean']['last_success']
assert information['ftpscraper']
assert not information['ftpscraper']['last_error']
assert information['ftpscraper']['last_success']
assert information['product-versions-matview']
assert not information['product-versions-matview']['last_error']
assert information['product-versions-matview']['last_success']
def test_symbols_unpack_zip_file(self):
source_file = os.path.join(self.temp_source_directory,
os.path.basename(ZIP_FILE))
shutil.copy(ZIP_FILE, source_file)
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['symbols-unpack']
assert not information['symbols-unpack']['last_error']
assert information['symbols-unpack']['last_success']
ok_(os.listdir(self.temp_destination_directory), 'empty')
# there should now be a directory named `sample-<todays date>
destination_dir = self.temp_destination_directory
ok_(os.path.isdir(destination_dir))
# and it should contain files
ok_(os.listdir(destination_dir))
# and the original should now have been deleted
ok_(not os.path.isfile(source_file))
def test_run_job_with_mocked_data_with_scp_errors(self):
config_manager = self._setup_config_manager(
public_output_path='',
private_user='******',
private_server='secure.mozilla.org',
private_location='/var/data/',
)
self._insert_waterwolf_mock_data()
def comm():
# some errors
return '', "CRAP!"
self.Popen().communicate.side_effect = comm
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['daily-url']
assert not information['daily-url']['last_error']
assert information['daily-url']['last_success']
ok_(config.logger.warn.called)
def test_mocked_fetch(self):
config_manager = self._setup_config_manager()
def return_test_data(fake):
test_data = [[
'2014-01-01', 'WinterWolf', 'Ginko', '2.3.1', '10.0.4',
'nightly-ww3v20', 'nightly', 'a-guid', '1'
],
[
'2019-01-01', 'NothingMuch', 'Ginko', '2.3.2',
'10.0.5a', None, 'release', 'a-guid', '2'
]]
for item in test_data:
yield item
with patch('socorro.cron.jobs.fetch_adi_from_hive.pyhs2') as fake_hive:
fake_hive.connect.return_value \
.cursor.return_value.__iter__ = return_test_data
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['fetch-adi-from-hive']
if information['fetch-adi-from-hive']['last_error']:
raise AssertionError(
information['fetch-adi-from-hive']['last_error'])
fake_hive.connect.assert_called_with(database='default',
authMechanism='PLAIN',
host='localhost',
user='******',
password='******',
port=10000)
pgcursor = self.conn.cursor()
columns = ('report_date', 'product_name', 'product_os_platform',
'product_os_version', 'product_version', 'build',
'build_channel', 'product_guid', 'count')
pgcursor.execute(""" select %s from raw_adi_logs """ %
','.join(columns))
adi = [dict(zip(columns, row)) for row in pgcursor.fetchall()]
eq_(adi, [{
'report_date': datetime.date(2014, 1, 1),
'product_name': 'WinterWolf',
'product_os_platform': 'Ginko',
'product_os_version': '2.3.1',
'product_version': '10.0.4',
'build': 'nightly-ww3v20',
'build_channel': 'nightly',
'product_guid': 'a-guid',
'count': 1
}, {
'report_date': datetime.date(2019, 1, 1),
'product_name': 'NothingMuch',
'product_os_platform': 'Ginko',
'product_os_version': '2.3.2',
'product_version': '10.0.5a',
'build': None,
'build_channel': 'release',
'product_guid': 'a-guid',
'count': 2
}])
def test_basic_run_job(self, rget):
config_manager = self._setup_config_manager()
def mocked_get(url):
if 'firefox_versions.json' in url:
return Response({
'FIREFOX_NIGHTLY': '52.0a1',
# Kept for legacy and smooth transition.
# We USED to consider the latest AURORA version a
# featured version but we no longer build aurora
# so Socorro shouldn't pick this up any more
# even if product-details.mozilla.org supplies it.
'FIREFOX_AURORA': '51.0a2',
'FIREFOX_ESR': '45.4.0esr',
'FIREFOX_ESR_NEXT': '',
'LATEST_FIREFOX_DEVEL_VERSION': '50.0b7',
'LATEST_FIREFOX_OLDER_VERSION': '3.6.28',
'LATEST_FIREFOX_RELEASED_DEVEL_VERSION': '50.0b7',
'LATEST_FIREFOX_VERSION': '49.0.1'
})
elif 'mobile_versions.json' in url:
return Response({
'nightly_version': '52.0a1',
'alpha_version': '51.0a2',
'beta_version': '50.0b6',
'version': '49.0',
'ios_beta_version': '6.0',
'ios_version': '5.0'
})
elif 'thunderbird_versions.json' in url:
return Response({
'LATEST_THUNDERBIRD_VERSION': '45.4.0',
'LATEST_THUNDERBIRD_DEVEL_VERSION': '50.0b1',
'LATEST_THUNDERBIRD_ALPHA_VERSION': '51.0a2',
'LATEST_THUNDERBIRD_NIGHTLY_VERSION': '52.0a1',
})
else:
raise NotImplementedError(url)
rget.side_effect = mocked_get
# Check what's set up in the fixture
rows = execute_query_fetchall(
self.conn, 'select product_name, version_string, featured_version '
'from product_versions order by version_string')
assert sorted(rows) == [
('Firefox', '15.0a1', True),
('Firefox', '24.5.0', True),
('Firefox', '49.0.1', False),
('Firefox', '50.0b', False),
('Firefox', '51.0a2', False),
('Firefox', '52.0a1', False),
]
# This is necessary so we get a new cursor when we do other
# selects after the crontabber app has run.
self.conn.commit()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['featured-versions-automatic']
assert not information['featured-versions-automatic']['last_error']
assert information['featured-versions-automatic']['last_success']
config.logger.info.assert_called_with(
'Set featured versions for Thunderbird to: '
'45.4.0, 50.0b1, 52.0a1')
rows = execute_query_fetchall(
self.conn, 'select product_name, version_string, featured_version '
'from product_versions')
eq_(
sorted(rows),
[
('Firefox', '15.0a1', False),
('Firefox', '24.5.0', False),
('Firefox', '49.0.1', True),
('Firefox', '50.0b', True),
# Note that the 'Aurora' branch is still mentioned but
# note that it's NOT featured (hence 'False').
('Firefox', '51.0a2', False),
('Firefox', '52.0a1', True),
])
def test_scrape_json_releases(self):
@responsify
def mocked_get(url, today=None, timeout=None):
if today is None:
today = utc_now()
html_wrap = "<html><body>\n%s\n</body></html>"
if url.endswith('/firefox/'):
return html_wrap % """
<a href="candidates/">candidates</a>
<a href="nightly/">nightly</a>
"""
if url.endswith('/firefox/candidates/'):
return html_wrap % """
<a href="28.0-candidates/">28.0-candidiates</a>
<a href="10.0b4-candidates/">10.0b4-candidiates</a>
<a href="None-candidates/">None-candidiates</a>
"""
if url.endswith('-candidates/'):
return html_wrap % """
<a href="build1/">build1</a>
"""
if url.endswith('/build1/'):
return html_wrap % """
<a href="linux-i686/">linux-i686</a>
"""
if url.endswith('/firefox/candidates/28.0-candidates/'
'build1/linux-i686/en-US/'):
return html_wrap % """
<a href="firefox-28.0.json">firefox-28.0.json</a>
"""
if url.endswith('/firefox/candidates/10.0b4-candidates/'
'build1/linux-i686/en-US/'):
return html_wrap % """
<a href="firefox-10.0b4.json">firefox-10.0b4.json</a>
<a href="firefox-10.0b4.en-US.linux-i686.mozinfo.json">
firefox-10.0b4.en-US.linux-i686.mozinfo.json</a>
<a href="JUNK.json">
JUNK.json</a>
"""
if url.endswith('/firefox/candidates/None-candidates/'
'build1/linux-i686/en-US/'):
return html_wrap % """
<a href="None.json">None.json</a>
"""
if 'None.json' in url:
return """ """
if 'firefox-28.0.json' in url:
return """
{
"buildid": "20140113161827",
"moz_app_maxversion": "28.0.*",
"moz_app_name": "firefox",
"moz_app_vendor": "Mozilla",
"moz_app_version": "28.0",
"moz_pkg_platform": "linux-i686",
"moz_source_repo":
"http://hg.mozilla.org/releases/mozilla-release",
"moz_update_channel": "release"
}
"""
if 'firefox-10.0b4.json' in url:
return """
{
"buildid": "20140113161826",
"moz_app_maxversion": "10.0.*",
"moz_app_name": "firefox",
"moz_app_vendor": "Mozilla",
"moz_app_version": "27.0",
"moz_pkg_platform": "linux-i686",
"moz_source_repo":
"http://hg.mozilla.org/releases/mozilla-beta",
"moz_update_channel": "beta"
}
"""
# Ignore unrecognized JSON files, see bug 1065071
if 'JUNK.json' in url:
return """
{
"something": "unexpected",
"nothing": "else"
}
"""
# Nightly tests for nightly and aurora builds
if url.endswith('/firefox/nightly/'):
return html_wrap % """
<a href="2014/">2014</a>
"""
if url.endswith(today.strftime('/firefox/nightly/%Y/%m/')):
return html_wrap % """
<a href="%s-03-02-03-mozilla-central/">txt</a>
<a href="%s-03-02-04-mozilla-central/">txt</a>
""" % (today.strftime('%Y-%m-%d'), today.strftime('%Y-%m-%d'))
if url.endswith(
today.strftime('/firefox/nightly/%Y/%m/'
'%Y-%m-%d-03-02-03-mozilla-central/')):
return html_wrap % """
<a href="firefox-30.0a1.en-US.linux-i686.json">txt</a>
"""
if url.endswith(
today.strftime(
'/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/'
)):
return html_wrap % """
<a href="firefox-30.0a2.en-US.linux-i686.json">txt</a>
"""
if url.endswith(
today.strftime(
'/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/'
'firefox-30.0a2.en-US.linux-i686.json')):
return """
{
"as": "$(CC)",
"buildid": "20140205030204",
"cc": "/usr/bin/ccache stuff",
"cxx": "/usr/bin/ccache stuff",
"host_alias": "x86_64-unknown-linux-gnu",
"host_cpu": "x86_64",
"host_os": "linux-gnu",
"host_vendor": "unknown",
"ld": "ld",
"moz_app_id": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}",
"moz_app_maxversion": "30.0a2",
"moz_app_name": "firefox",
"moz_app_vendor": "Mozilla",
"moz_app_version": "30.0a2",
"moz_pkg_platform": "linux-i686",
"moz_source_repo":
"https://hg.mozilla.org/mozilla-central",
"moz_source_stamp": "1f170f9fead0",
"moz_update_channel": "nightly",
"target_alias": "i686-pc-linux",
"target_cpu": "i686",
"target_os": "linux-gnu",
"target_vendor": "pc"
}
"""
if url.endswith(
today.strftime(
'/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-03-mozilla-central/'
'firefox-30.0a1.en-US.linux-i686.json')):
return """
{
"as": "$(CC)",
"buildid": "20140205030203",
"cc": "/usr/bin/ccache ",
"cxx": "/usr/bin/ccache stuff",
"host_alias": "x86_64-unknown-linux-gnu",
"host_cpu": "x86_64",
"host_os": "linux-gnu",
"host_vendor": "unknown",
"ld": "ld",
"moz_app_id": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}",
"moz_app_maxversion": "30.0a1",
"moz_app_name": "firefox",
"moz_app_vendor": "Mozilla",
"moz_app_version": "30.0a1",
"moz_pkg_platform": "linux-i686",
"moz_source_repo":
"https://hg.mozilla.org/mozilla-central",
"moz_source_stamp": "1f170f9fead0",
"moz_update_channel": "nightly",
"target_alias": "i686-pc-linux",
"target_cpu": "i686",
"target_os": "linux-gnu",
"target_vendor": "pc"
}
"""
raise NotImplementedError(url)
self.mocked_session().get.side_effect = mocked_get
config_manager = self._setup_config_manager_firefox()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['ftpscraper']
assert not information['ftpscraper']['last_error']
assert information['ftpscraper']['last_success']
config.logger.warning.assert_any_call(
'Unable to JSON parse content %r', ' ', exc_info=True)
base_url = config.crontabber['class-FTPScraperCronApp'].base_url
config.logger.warning.assert_any_call(
'warning, unsupported JSON file: %s',
base_url + 'firefox/candidates/'
'10.0b4-candidates/build1/linux-i686/en-US/JUNK.json')
cursor = self.conn.cursor()
columns = 'product_name', 'build_id', 'build_type'
cursor.execute("""
select %s
from releases_raw
""" % ','.join(columns))
builds = [dict(zip(columns, row)) for row in cursor.fetchall()]
build_ids = dict((str(x['build_id']), x) for x in builds)
assert '20140113161827' in build_ids
assert '20140113161826' in build_ids
assert '20140205030203' in build_ids
assert len(build_ids) == 4
expected = [{
'build_id': 20140113161827,
'product_name': 'firefox',
'build_type': 'release'
}, {
'build_id': 20140113161827,
'product_name': 'firefox',
'build_type': 'beta'
}, {
'build_id': 20140113161826,
'product_name': 'firefox',
'build_type': 'beta'
}, {
'build_id': 20140205030203,
'product_name': 'firefox',
'build_type': 'nightly'
}, {
'build_id': 20140205030204,
'product_name': 'firefox',
'build_type': 'aurora'
}]
assert builds == expected
def test_email_after_delay(self, exacttarget_mock):
"""Test that a user will receive an email if he or she sends us a new
crash report after the delay is passed (but not before). """
config_manager = self._setup_config_manager(
delay_between_emails=1, restrict_products=['EarthRaccoon'])
email = '*****@*****.**'
list_service_mock = exacttarget_mock.return_value.list.return_value
list_service_mock.get_subscriber.return_value = {'token': email}
trigger_send_mock = exacttarget_mock.return_value.trigger_send
tomorrow = utc_now() + datetime.timedelta(days=1, hours=2)
twohourslater = utc_now() + datetime.timedelta(hours=2)
storage_config_manager = self._setup_storage_config()
with storage_config_manager.context() as storage_config:
storage = ElasticSearchCrashStorage(storage_config)
with config_manager.context() as config:
# 1. Send an email to the user and update emailing data
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
exacttarget_mock.return_value.trigger_send.assert_called_with(
'socorro_dev_test', {
'EMAIL_ADDRESS_': email,
'EMAIL_FORMAT_': 'H',
'TOKEN': email
})
eq_(trigger_send_mock.call_count, 1)
# 2. Test that before 'delay' is passed user doesn't receive
# another email
# Insert a new crash report with the same email address
storage.save_processed({
'uuid':
'50',
'email':
email,
'product':
'EarthRaccoon',
'version':
'20.0',
'release_channel':
'Release',
'date_processed':
utc_now() + datetime.timedelta(hours=1)
})
storage.es.refresh()
# Run crontabber with time pushed by two hours
with mock.patch('crontabber.app.utc_now') as cronutc_mock:
with mock.patch('crontabber.base.utc_now') as baseutc_mock:
cronutc_mock.return_value = twohourslater
baseutc_mock.return_value = twohourslater
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
# No new email was sent
eq_(trigger_send_mock.call_count, 1)
# 3. Verify that, after 'delay' is passed, a new email is sent
# to our user
# Insert a new crash report with the same email address
storage.save_processed({
'uuid':
'51',
'email':
email,
'product':
'EarthRaccoon',
'version':
'20.0',
'release_channel':
'Release',
'date_processed':
utc_now() + datetime.timedelta(days=1)
})
storage.es.refresh()
# Run crontabber with time pushed by a day
with mock.patch('crontabber.app.utc_now') as cronutc_mock:
with mock.patch('crontabber.base.utc_now') as baseutc_mock:
cronutc_mock.return_value = tomorrow
baseutc_mock.return_value = tomorrow
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
# A new email was sent
eq_(trigger_send_mock.call_count, 2)
def test_email_cannot_be_sent_twice(self, exacttarget_mock):
config_manager = self._setup_config_manager(
restrict_products=['NightlyTrain'])
et_mock = exacttarget_mock.return_value
# Prepare failures
_failures = []
_email_sent = []
class SomeRandomError(Exception):
pass
def trigger_send(template, fields):
email = fields['EMAIL_ADDRESS_']
if email == '*****@*****.**' and email not in _failures:
_failures.append(email)
raise SomeRandomError('This is an error. ')
else:
_email_sent.append(email)
et_mock.trigger_send = trigger_send
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert information['automatic-emails']['last_error']
eq_(information['automatic-emails']['last_error']['type'],
str(SomeRandomError))
# Verify that user's data was updated, but not all of it
eq_(_email_sent, ['*****@*****.**', '*****@*****.**'])
emails_list = ('*****@*****.**', '*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**')
conf = config.crontabber['class-AutomaticEmailsCronApp']
es = SuperS().es(
urls=conf.elasticsearch.elasticsearch_urls,
timeout=conf.elasticsearch.elasticsearch_timeout,
)
search = es.indexes(conf.elasticsearch.elasticsearch_emails_index)
search = search.doctypes('emails')
es.get_es().refresh()
search = search.filter(_id__in=emails_list)
res = search.execute()
eq_(res.count, 2)
now = utc_now()
for row in res.results:
assert row['_id'] in ('*****@*****.**', '*****@*****.**')
date = string_to_datetime(row['_source']['last_sending'])
eq_(date.year, now.year)
eq_(date.month, now.month)
eq_(date.day, now.day)
# Run crontabber again and verify that all users are updated,
# and emails are not sent twice
state = tab.job_state_database['automatic-emails']
self._wind_clock(state, hours=1)
tab.job_state_database['automatic-emails'] = state
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
# Verify that users were not sent an email twice
eq_(_email_sent, [
'*****@*****.**', '*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**'
])
