def test_basic_run(self):
cur = self.conn.cursor()
# Ensure test table is present.
statement = """
INSERT INTO missing_symbols
(date_processed, debug_file, debug_id, code_file, code_id)
VALUES
(%(first)s, 'foo.pdb', '0420', 'foo.py', '123'),
(%(second)s, 'bar.pdb', '65EA9', 'bar.py', null)
"""
second = utc_now().date()
first = second - datetime.timedelta(days=1)
cur.execute(statement, {'first': first, 'second': second})
self.conn.commit()
# Run the crontabber job to remove the test table.
config_manager = self._setup_config_manager(days_to_keep=1)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
# Basic assertion test of stored procedure.
information = self._load_structure()
assert information['clean-missing-symbols']
assert not information['clean-missing-symbols']['last_error']
assert information['clean-missing-symbols']['last_success']
# Ensure expected test row was removed
cur.execute("""
SELECT date_processed FROM missing_symbols
""")
first, = cur.fetchall()
date_processed = first[0]
assert date_processed == second
def test_run_drop_old_partitions(self):
cur = self.conn.cursor()
# Ensure test table is present.
statement = """
SELECT COUNT(*) FROM raw_crashes_20120102;
"""
cur.execute(statement)
result = cur.fetchone()
assert result[0] == 2L
# need to get out of this transaction to allow crontabber to acquire a lock
self.conn.commit()
# Run the crontabber job to remove the test table.
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
# Basic assertion test of stored procedure.
information = self._load_structure()
print(information['truncate-partitions']['last_error'])
assert information['truncate-partitions']
assert not information['truncate-partitions']['last_error']
assert information['truncate-partitions']['last_success']
# Ensure test table was removed.
statement = """
SELECT COUNT(*) FROM raw_crashes_20120102;
"""
cur.execute(statement)
result = cur.fetchone()
assert result[0] == 0
def test_basic_run_job(self, requests_mocker):
requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
config_manager = self._setup_config_manager(3)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
cursor = self.conn.cursor()
cursor.execute('select bug_id from bug_associations order by bug_id')
associations = cursor.fetchall()
# Verify we have the expected number of associations.
assert len(associations) == 8
bug_ids = [x[0] for x in associations]
# Verify bugs with no crash signatures are missing.
assert 6 not in bug_ids
cursor.execute(
'select signature from bug_associations where bug_id = 8'
)
associations = cursor.fetchall()
# New signatures have correctly been inserted.
assert len(associations) == 2
assert ('another::legitimate(sig)',) in associations
assert ('legitimate(sig)',) in associations
def test_basic_run(self):
cur = self.conn.cursor()
# Ensure test table is present.
statement = """
INSERT INTO raw_adi
(date, product_name, adi_count) VALUES
(%(first)s, 'WinterFox', 11),
(%(second)s, 'WinterFox', 23)
"""
second = utc_now().date()
first = second - datetime.timedelta(days=1)
cur.execute(statement, {'first': first, 'second': second})
self.conn.commit()
# Run the crontabber job to remove the test table.
config_manager = self._setup_config_manager(days_to_keep=1)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
# Basic assertion test of stored procedure.
information = self._load_structure()
assert information['clean-raw-adi']
assert not information['clean-raw-adi']['last_error']
assert information['clean-raw-adi']['last_success']
# Ensure test row was removed
cur.execute("""
SELECT date FROM raw_adi
""")
result, = cur.fetchall()
report_date = result[0]
assert report_date == second
def test_all_matviews(self, mocked_utc_now):
# Pretend it's 03AM UTC
def mock_utc_now():
n = utc_now()
n = n.replace(hour=3)
return n
mocked_utc_now.side_effect = mock_utc_now
config_manager = self._setup_config_manager(
'socorro.unittest.cron.jobs.test_matviews.FTPScraperJob|1d\n'
'socorro.unittest.cron.jobs.test_matviews.FetchADIFromHiveCronApp|1d\n'
''
'socorro.cron.jobs.matviews.ProductVersionsCronApp|1d\n'
'socorro.cron.jobs.matviews.ADUCronApp|1d\n'
'socorro.cron.jobs.matviews.BuildADUCronApp|1d|02:00\n'
)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
for app_name in ('product-versions-matview',
'adu-matview',
'build-adu-matview'):
assert app_name in information
assert not information[app_name]['last_error']
assert information[app_name]['last_success']
def test_run_job_with_reports_with_existing_bugs_different(self, requests_mocker):
"""Verify that an association to a signature that no longer is part
of the crash signatures list gets removed.
"""
requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
config_manager = self._setup_config_manager(3)
cursor = self.conn.cursor()
cursor.execute("""
insert into bug_associations (bug_id, signature)
values (8, '@different');
""")
self.conn.commit()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
cursor.execute(
'select signature from bug_associations where bug_id = 8'
)
associations = cursor.fetchall()
# The previous association, to signature '@different' that is not in
# crash signatures, is now missing.
assert ('@different',) not in associations
def test_run_job_with_reports_with_existing_bugs_same(self, requests_mocker):
requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
config_manager = self._setup_config_manager(3)
cursor = self.conn.cursor()
cursor.execute("""
insert into bug_associations (bug_id, signature)
values (8, 'legitimate(sig)');
""")
self.conn.commit()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
cursor.execute(
'select signature from bug_associations where bug_id = 8'
)
associations = cursor.fetchall()
# New signatures have correctly been inserted.
assert len(associations) == 2
assert ('another::legitimate(sig)',) in associations
assert ('legitimate(sig)',) in associations
def test_basic_run_job(self, requests_mocker):
requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
config_manager = self._setup_config_manager(3)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
cursor = self.conn.cursor()
cursor.execute('select bug_id from bug_associations order by bug_id')
associations = cursor.fetchall()
# Verify we have the expected number of associations.
assert len(associations) == 8
bug_ids = [x[0] for x in associations]
# Verify bugs with no crash signatures are missing.
assert 6 not in bug_ids
cursor.execute(
'select signature from bug_associations where bug_id = 8'
)
associations = cursor.fetchall()
# New signatures have correctly been inserted.
assert len(associations) == 2
assert ('another::legitimate(sig)',) in associations
assert ('legitimate(sig)',) in associations
def test_run_drop_old_partitions(self):
cur = self.conn.cursor()
# Ensure test table is present.
statement = """
SELECT COUNT(*) FROM information_schema.tables
WHERE table_name = 'phrotest_20120102';
"""
cur.execute(statement)
result = cur.fetchone()
assert result[0] == 1
# Run the crontabber job to remove the test table.
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
# Basic assertion test of stored procedure.
information = self._load_structure()
assert information['drop-old-partitions']
assert not information['drop-old-partitions']['last_error']
assert information['drop-old-partitions']['last_success']
# Ensure test table was removed.
statement = """
SELECT COUNT(*) FROM information_schema.tables
WHERE table_name = 'phrotest_20120102';
"""
cur.execute(statement)
result = cur.fetchone()
assert result[0] == 0
def test_basic_run_job(self, requests_mocker):
requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
config_manager = self._setup_config_manager(3)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
associations = self.fetch_data()
# Verify we have the expected number of associations
assert len(associations) == 8
bug_ids = set([x['bug_id'] for x in associations])
# Verify bugs with no crash signatures are missing
assert 6 not in bug_ids
bug_8_signatures = [
item['signature'] for item in associations if item['bug_id'] == '8'
]
# New signatures have correctly been inserted
assert len(bug_8_signatures) == 2
assert 'another::legitimate(sig)' in bug_8_signatures
assert 'legitimate(sig)' in bug_8_signatures
def test_run_job_with_reports_with_existing_bugs_same(self, requests_mocker):
requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
config_manager = self._setup_config_manager(3)
cursor = self.conn.cursor()
cursor.execute("""
insert into bug_associations (bug_id, signature)
values (8, 'legitimate(sig)');
""")
self.conn.commit()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
cursor.execute(
'select signature from bug_associations where bug_id = 8'
)
associations = cursor.fetchall()
# New signatures have correctly been inserted.
assert len(associations) == 2
assert ('another::legitimate(sig)',) in associations
assert ('legitimate(sig)',) in associations
def test_run_job_with_reports_with_existing_bugs_different(self, requests_mocker):
"""Verify that an association to a signature that no longer is part
of the crash signatures list gets removed.
"""
requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
config_manager = self._setup_config_manager(3)
cursor = self.conn.cursor()
cursor.execute("""
insert into bug_associations (bug_id, signature)
values (8, '@different');
""")
self.conn.commit()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
cursor.execute(
'select signature from bug_associations where bug_id = 8'
)
associations = cursor.fetchall()
# The previous association, to signature '@different' that is not in
# crash signatures, is now missing.
assert ('@different',) not in associations
def test_run_weekly_reports_partitions(self):
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['weekly-reports-partitions']
assert not information['weekly-reports-partitions']['last_error']
assert information['weekly-reports-partitions']['last_success']
def test_run(self, connection_context):
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['elasticsearch-cleanup']
assert not information['elasticsearch-cleanup']['last_error']
assert information['elasticsearch-cleanup']['last_success']
def test_mocked_fetch_with_secondary_destination(self, fake_hive):
class MockedPGConnectionContext:
connection = mock.MagicMock()
def __init__(self, config):
self.config = config
@contextlib.contextmanager
def __call__(self):
yield self.connection
config_manager = self._setup_config_manager(
overrides={
'crontabber.class-FetchADIFromHiveCronApp.'
'secondary_destination.'
'database_class':
MockedPGConnectionContext,
})
yesterday = (datetime.datetime.utcnow() -
datetime.timedelta(days=1)).date()
def return_test_data(fake):
yield [
yesterday, 'WinterWolf', 'Ginko', '2.3.1', '10.0.4',
'nightly-ww3v20', 'nightly', 'a-guid', 1
]
fake_hive.connect.return_value \
.cursor.return_value.__iter__ = return_test_data
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['fetch-adi-from-hive']
assert not information['fetch-adi-from-hive']['last_error']
fake_hive.connect.assert_called_with(
database='default',
authMechanism='PLAIN',
host='localhost',
user='******',
password='******',
port=10000,
timeout=1800000,
)
# Critical test here.
# We make sure the secondary database class gets used
# for a `cursor.copy_from()` call.
assert MockedPGConnectionContext.connection.cursor().copy_from.called
def test_mocked_fetch(self):
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['fetch-adi-from-hive']
assert not information['fetch-adi-from-hive']['last_error']
config.logger.info.assert_called_with(
'Faking the fetching of ADI from Hive :)')
def test_duplicates(self):
config_manager = self._setup_config_manager(
'socorro.cron.jobs.matviews.DuplicatesCronApp|1d'
)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['duplicates']
assert not information['duplicates']['last_error']
assert information['duplicates']['last_success']
def test_mocked_fetch(self):
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['fetch-adi-from-hive']
assert not information['fetch-adi-from-hive']['last_error']
config.logger.info.assert_called_with(
'Faking the fetching of ADI from Hive :)'
)
def test_reports_clean_with_dependency(self):
config_manager = self._setup_config_manager(
'socorro.cron.jobs.matviews.DuplicatesCronApp|1h\n'
'socorro.cron.jobs.matviews.ReportsCleanCronApp|1h'
)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['reports-clean']
assert not information['reports-clean']['last_error']
assert information['reports-clean']['last_success']
def test_download_error(self, req_mock):
config_manager = self._setup_config_manager()
req_mock.get('http://example.com/firefox_versions.json',
status_code=404)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['featured-versions-automatic']
assert information['featured-versions-automatic']['last_error']
error = information['featured-versions-automatic']['last_error']
assert 'DownloadError' in error['type']
assert '404' in error['value']
def test_basic_run(self):
cur = self.conn.cursor()
# Ensure test table is present.
statement = """
INSERT INTO raw_adi
(date, product_name, adi_count) VALUES
(%(first)s, 'WinterFox', 11),
(%(second)s, 'WinterFox', 23)
"""
second = utc_now().date()
first = second - datetime.timedelta(days=1)
cur.execute(statement, {'first': first, 'second': second})
# Ensure test table is present.
statement = """
INSERT INTO raw_adi_logs
(report_date, product_name, count) VALUES
(%(first)s, 'WinterFox', 11),
(%(second)s, 'WinterFox', 23)
"""
second = utc_now().date()
first = second - datetime.timedelta(days=1)
cur.execute(statement, {'first': first, 'second': second})
self.conn.commit()
# Run the crontabber job to remove the test table.
config_manager = self._setup_config_manager(days_to_keep=1)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
# Basic assertion test of stored procedure.
information = self._load_structure()
assert information['clean-raw-adi']
assert not information['clean-raw-adi']['last_error']
assert information['clean-raw-adi']['last_success']
# Ensure test row was removed
cur.execute('SELECT date FROM raw_adi')
result, = cur.fetchall()
report_date = result[0]
assert report_date == second
cur.execute('SELECT report_date FROM raw_adi_logs')
result, = cur.fetchall()
report_date = result[0]
assert report_date == second
def test_with_bugzilla_failure(self, requests_mocker):
requests_mocker.get(BUGZILLA_BASE_URL,
text='error loading content',
status_code=500)
config_manager = self._setup_config_manager(3)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
# Verify there has been an error
last_error = information['bugzilla-associations']['last_error']
assert last_error
assert 'HTTPError' in last_error['type']
assert not information['bugzilla-associations']['last_success']
def test_basic_run(self):
# We need to prepare to return a size for the new key
self.mock_key.size = 123456789
self.mock_key.generate_url.return_value = (
'https://s3.example.com/latest.csv'
)
# Run the crontabber job to remove the test table.
config_manager = self._setup_config_manager()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
# Basic assertion test of stored procedure.
information = self._load_structure()
assert information['missing-symbols']
assert not information['missing-symbols']['last_error']
assert information['missing-symbols']['last_success']
self.mock_boto_class()._connect.assert_called_with()
self.mock_boto_class.close.assert_called_with()
self.mock_bucket.new_key.assert_called_with('latest.csv')
content = StringIO()
writer = csv.writer(content)
writer.writerow((
'debug_file',
'debug_id',
'code_file',
'code_id',
))
writer.writerow((
'nvwgf2um.pdb',
'9D492B844FF34800B34320464AA1E7E41',
'nvwgf2um.dll',
'561D1D4Ff58000',
))
self.mock_key.set_contents_from_string.assert_called_with(
content.getvalue()
)
# this is becausse 123456789 bytes is 117.74 Mb
tab.config.logger.info.assert_called_with(
'Generated https://s3.example.com/latest.csv '
'(123,456,789 bytes, 117.74 Mb)'
)
def test_run(self, connect_s3):
key = mock.MagicMock()
connect_s3().get_bucket().get_key.return_value = None
connect_s3().get_bucket().new_key.return_value = key
with self._setup_config_manager().context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
app_name = 'upload-crash-report-json-schema'
assert information[app_name]
assert not information[app_name]['last_error']
assert information[app_name]['last_success']
key.set_contents_from_string.assert_called_with(
CRASH_REPORT_JSON_SCHEMA_AS_STRING)
def test_download_error(self, rget):
config_manager = self._setup_config_manager()
def mocked_get(url):
return Response('not here', status_code=404)
rget.side_effect = mocked_get
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['featured-versions-automatic']
assert information['featured-versions-automatic']['last_error']
error = information['featured-versions-automatic']['last_error']
assert 'DownloadError' in error['type']
assert '404' in error['value']
def test_one_matview_alone(self):
config_manager = self._setup_config_manager(
'socorro.unittest.cron.jobs.test_matviews.FTPScraperJob|1d\n'
''
'socorro.cron.jobs.matviews.ProductVersionsCronApp|1d'
)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['ftpscraper']
assert not information['ftpscraper']['last_error']
assert information['ftpscraper']['last_success']
assert information['product-versions-matview']
assert not information['product-versions-matview']['last_error']
assert information['product-versions-matview']['last_success']
def test_with_bugzilla_failure(self, requests_mocker):
requests_mocker.get(
BUGZILLA_BASE_URL,
text='error loading content',
status_code=500
)
config_manager = self._setup_config_manager(3)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
# There has been an error.
last_error = information['bugzilla-associations']['last_error']
assert last_error
assert 'HTTPError' in last_error['type']
assert not information['bugzilla-associations']['last_success']
def test_fetch_with_zero_hive_results(self, fake_hive):
config_manager = self._setup_config_manager()
def return_test_data(fake):
# a generator that yields literally nothing
# http://stackoverflow.com/a/13243870/205832
return
yield
fake_hive.connect.return_value \
.cursor.return_value.__iter__ = return_test_data
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['fetch-adi-from-hive']
assert information['fetch-adi-from-hive']['last_error']
assert (
NoRowsWritten.__name__ in
information['fetch-adi-from-hive']['last_error']['type']
)
assert information['fetch-adi-from-hive']['last_error']
fake_hive.connect.assert_called_with(
database='default',
authMechanism='PLAIN',
host='localhost',
user='******',
password='******',
port=10000,
timeout=1800000,
)
pgcursor = self.conn.cursor()
pgcursor.execute(
"select count(*) from raw_adi_logs"
)
count, = pgcursor.fetchone()
assert count == 0
def test_run_job_based_on_last_success(self, requests_mocker):
"""specifically setting 0 days back and no prior run
will pick it up from now's date"""
requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
config_manager = self._setup_config_manager(0)
cursor = self.conn.cursor()
# these are matching the SAMPLE_CSV above
cursor.execute("""insert into bug_associations
(bug_id,signature)
values
(8, 'legitimate(sig)');
""")
self.conn.commit()
# second time
config_manager = self._setup_config_manager(0)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
state = tab.job_state_database.copy()
self._wind_clock(state, days=1)
tab.job_state_database.update(state)
# Create a CSV file for one day back.
# This'll make sure there's a .csv file whose day
# is that of the last run.
self._setup_config_manager(1)
config_manager = self._setup_config_manager(0)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
def test_run_job_with_reports_with_existing_bugs_different(
self, requests_mocker):
"""Verify that an association to a signature that no longer is part
of the crash signatures list gets removed.
"""
requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
self.insert_data(bug_id='8', signature='@different')
config_manager = self._setup_config_manager(3)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
# The previous association, to signature '@different' that is not in
# crash signatures, is now missing
associations = self.fetch_data()
assert '@different' not in [item['signature'] for item in associations]
def test_run_job_based_on_last_success(self, requests_mocker):
"""specifically setting 0 days back and no prior run
will pick it up from now's date"""
requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
config_manager = self._setup_config_manager(0)
cursor = self.conn.cursor()
# these are matching the SAMPLE_CSV above
cursor.execute("""insert into bug_associations
(bug_id,signature)
values
(8, 'legitimate(sig)');
""")
self.conn.commit()
# second time
config_manager = self._setup_config_manager(0)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
state = tab.job_state_database.copy()
self._wind_clock(state, days=1)
tab.job_state_database.update(state)
# Create a CSV file for one day back.
# This'll make sure there's a .csv file whose day
# is that of the last run.
self._setup_config_manager(1)
config_manager = self._setup_config_manager(0)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
def test_run_job_with_reports_with_existing_bugs_same(
self, requests_mocker):
requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
self.insert_data(bug_id='8', signature='legitimate(sig)')
config_manager = self._setup_config_manager(3)
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['bugzilla-associations']
assert not information['bugzilla-associations']['last_error']
assert information['bugzilla-associations']['last_success']
associations = self.fetch_data()
associations = [
item['signature'] for item in associations if item['bug_id'] == '8'
]
# New signatures have correctly been inserted
assert len(associations) == 2
assert associations == ['another::legitimate(sig)', 'legitimate(sig)']
def test_basic_run_job(self, req_mock):
config_manager = self._setup_config_manager()
req_mock.get(
'http://example.com/firefox_versions.json',
json={
'FIREFOX_NIGHTLY': '52.0a1',
# Kept for legacy and smooth transition. We USED to consider
# the latest AURORA version a featured version but we no
# longer build aurora so Socorro shouldn't pick this up any
# more even if product-details.mozilla.org supplies it.
'FIREFOX_AURORA': '51.0a2',
'FIREFOX_ESR': '45.4.0esr',
'FIREFOX_ESR_NEXT': '',
'LATEST_FIREFOX_DEVEL_VERSION': '50.0b7',
'LATEST_FIREFOX_OLDER_VERSION': '3.6.28',
'LATEST_FIREFOX_RELEASED_DEVEL_VERSION': '50.0b7',
'LATEST_FIREFOX_VERSION': '49.0.1',
})
req_mock.get('http://example.com/mobile_versions.json',
json={
'nightly_version': '52.0a1',
'alpha_version': '51.0a2',
'beta_version': '50.0b6',
'version': '49.0',
'ios_beta_version': '6.0',
'ios_version': '5.0',
})
req_mock.get('http://example.com/thunderbird_versions.json',
json={
'LATEST_THUNDERBIRD_VERSION': '45.4.0',
'LATEST_THUNDERBIRD_DEVEL_VERSION': '50.0b1',
'LATEST_THUNDERBIRD_ALPHA_VERSION': '51.0a2',
'LATEST_THUNDERBIRD_NIGHTLY_VERSION': '52.0a1',
})
# Check what's set up in the fixture
rows = execute_query_fetchall(
self.conn, 'select product_name, version_string, featured_version '
'from product_versions order by version_string')
assert sorted(rows) == [
('Firefox', '15.0a1', True),
('Firefox', '24.5.0', True),
('Firefox', '49.0.1', False),
('Firefox', '50.0b', False),
('Firefox', '51.0a2', False),
('Firefox', '52.0a1', False),
]
# This is necessary so we get a new cursor when we do other selects
# after the crontabber app has run.
self.conn.commit()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['featured-versions-automatic']
assert not information['featured-versions-automatic']['last_error']
assert information['featured-versions-automatic']['last_success']
config.logger.info.assert_called_with(
'Set featured versions for Thunderbird to: '
'45.4.0, 50.0b1, 52.0a1')
rows = execute_query_fetchall(
self.conn, 'select product_name, version_string, featured_version '
'from product_versions')
expected = [
('Firefox', '15.0a1', False),
('Firefox', '24.5.0', False),
('Firefox', '49.0.1', True),
('Firefox', '50.0b', True),
# Note that the 'Aurora' branch is still mentioned but note that it's NOT featured
# (hence 'False').
('Firefox', '51.0a2', False),
('Firefox', '52.0a1', True),
]
assert sorted(rows) == expected
def test_scrape_json_releases(self):
@responsify
def mocked_get(url, today=None, timeout=None):
if today is None:
today = utc_now()
html_wrap = "<html><body>\n%s\n</body></html>"
if url.endswith('/firefox/'):
return html_wrap % """
<a href="candidates/">candidates</a>
<a href="nightly/">nightly</a>
"""
if url.endswith('/firefox/candidates/'):
return html_wrap % """
<a href="28.0-candidates/">28.0-candidiates</a>
<a href="10.0b4-candidates/">10.0b4-candidiates</a>
<a href="None-candidates/">None-candidiates</a>
"""
if url.endswith('-candidates/'):
return html_wrap % """
<a href="build1/">build1</a>
"""
if url.endswith('/build1/'):
return html_wrap % """
<a href="linux-i686/">linux-i686</a>
"""
if url.endswith('/firefox/candidates/28.0-candidates/'
'build1/linux-i686/en-US/'):
return html_wrap % """
<a href="firefox-28.0.json">firefox-28.0.json</a>
"""
if url.endswith('/firefox/candidates/10.0b4-candidates/'
'build1/linux-i686/en-US/'):
return html_wrap % """
<a href="firefox-10.0b4.json">firefox-10.0b4.json</a>
<a href="firefox-10.0b4.en-US.linux-i686.mozinfo.json">
firefox-10.0b4.en-US.linux-i686.mozinfo.json</a>
<a href="JUNK.json">
JUNK.json</a>
"""
if url.endswith('/firefox/candidates/None-candidates/'
'build1/linux-i686/en-US/'):
return html_wrap % """
<a href="None.json">None.json</a>
"""
if 'None.json' in url:
return """ """
if 'firefox-28.0.json' in url:
return """
{
"buildid": "20140113161827",
"moz_app_maxversion": "28.0.*",
"moz_app_name": "firefox",
"moz_app_vendor": "Mozilla",
"moz_app_version": "28.0",
"moz_pkg_platform": "linux-i686",
"moz_source_repo":
"http://hg.mozilla.org/releases/mozilla-release",
"moz_update_channel": "release"
}
"""
if 'firefox-10.0b4.json' in url:
return """
{
"buildid": "20140113161826",
"moz_app_maxversion": "10.0.*",
"moz_app_name": "firefox",
"moz_app_vendor": "Mozilla",
"moz_app_version": "27.0",
"moz_pkg_platform": "linux-i686",
"moz_source_repo":
"http://hg.mozilla.org/releases/mozilla-beta",
"moz_update_channel": "beta"
}
"""
# Ignore unrecognized JSON files, see bug 1065071
if 'JUNK.json' in url:
return """
{
"something": "unexpected",
"nothing": "else"
}
"""
# Nightly tests for nightly and aurora builds
if url.endswith('/firefox/nightly/'):
return html_wrap % """
<a href="2014/">2014</a>
"""
if url.endswith(today.strftime('/firefox/nightly/%Y/%m/')):
return html_wrap % """
<a href="%s-03-02-03-mozilla-central/">txt</a>
<a href="%s-03-02-04-mozilla-central/">txt</a>
""" % (today.strftime('%Y-%m-%d'), today.strftime('%Y-%m-%d'))
if url.endswith(
today.strftime('/firefox/nightly/%Y/%m/'
'%Y-%m-%d-03-02-03-mozilla-central/')):
return html_wrap % """
<a href="firefox-30.0a1.en-US.linux-i686.json">txt</a>
"""
if url.endswith(
today.strftime(
'/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/'
)):
return html_wrap % """
<a href="firefox-30.0a2.en-US.linux-i686.json">txt</a>
"""
if url.endswith(
today.strftime(
'/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/'
'firefox-30.0a2.en-US.linux-i686.json')):
return """
{
"as": "$(CC)",
"buildid": "20140205030204",
"cc": "/usr/bin/ccache stuff",
"cxx": "/usr/bin/ccache stuff",
"host_alias": "x86_64-unknown-linux-gnu",
"host_cpu": "x86_64",
"host_os": "linux-gnu",
"host_vendor": "unknown",
"ld": "ld",
"moz_app_id": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}",
"moz_app_maxversion": "30.0a2",
"moz_app_name": "firefox",
"moz_app_vendor": "Mozilla",
"moz_app_version": "30.0a2",
"moz_pkg_platform": "linux-i686",
"moz_source_repo":
"https://hg.mozilla.org/mozilla-central",
"moz_source_stamp": "1f170f9fead0",
"moz_update_channel": "nightly",
"target_alias": "i686-pc-linux",
"target_cpu": "i686",
"target_os": "linux-gnu",
"target_vendor": "pc"
}
"""
if url.endswith(
today.strftime(
'/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-03-mozilla-central/'
'firefox-30.0a1.en-US.linux-i686.json')):
return """
{
"as": "$(CC)",
"buildid": "20140205030203",
"cc": "/usr/bin/ccache ",
"cxx": "/usr/bin/ccache stuff",
"host_alias": "x86_64-unknown-linux-gnu",
"host_cpu": "x86_64",
"host_os": "linux-gnu",
"host_vendor": "unknown",
"ld": "ld",
"moz_app_id": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}",
"moz_app_maxversion": "30.0a1",
"moz_app_name": "firefox",
"moz_app_vendor": "Mozilla",
"moz_app_version": "30.0a1",
"moz_pkg_platform": "linux-i686",
"moz_source_repo":
"https://hg.mozilla.org/mozilla-central",
"moz_source_stamp": "1f170f9fead0",
"moz_update_channel": "nightly",
"target_alias": "i686-pc-linux",
"target_cpu": "i686",
"target_os": "linux-gnu",
"target_vendor": "pc"
}
"""
raise NotImplementedError(url)
self.mocked_session().get.side_effect = mocked_get
config_manager = self._setup_config_manager_firefox()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['ftpscraper']
assert not information['ftpscraper']['last_error']
assert information['ftpscraper']['last_success']
config.logger.warning.assert_any_call(
'Unable to JSON parse content %r', ' ', exc_info=True)
base_url = config.crontabber['class-FTPScraperCronApp'].base_url
config.logger.warning.assert_any_call(
'warning, unsupported JSON file: %s',
base_url + 'firefox/candidates/'
'10.0b4-candidates/build1/linux-i686/en-US/JUNK.json')
cursor = self.conn.cursor()
columns = 'product_name', 'build_id', 'build_type'
cursor.execute("""
select %s
from releases_raw
""" % ','.join(columns))
builds = [dict(zip(columns, row)) for row in cursor.fetchall()]
build_ids = dict((str(x['build_id']), x) for x in builds)
assert '20140113161827' in build_ids
assert '20140113161826' in build_ids
assert '20140205030203' in build_ids
assert len(build_ids) == 4
expected = [{
'build_id': 20140113161827,
'product_name': 'firefox',
'build_type': 'release'
}, {
'build_id': 20140113161827,
'product_name': 'firefox',
'build_type': 'beta'
}, {
'build_id': 20140113161826,
'product_name': 'firefox',
'build_type': 'beta'
}, {
'build_id': 20140205030203,
'product_name': 'firefox',
'build_type': 'nightly'
}, {
'build_id': 20140205030204,
'product_name': 'firefox',
'build_type': 'aurora'
}]
assert builds == expected
def test_mocked_fetch(self, fake_hive):
config_manager = self._setup_config_manager()
yesterday = (
datetime.datetime.utcnow() - datetime.timedelta(days=1)
).date()
def return_test_data(fake):
yield [
yesterday,
'WinterWolf',
'Ginko',
'2.3.1',
'10.0.4',
'nightly-ww3v20',
'nightly',
'a-guid',
1
]
yield [
yesterday,
'NothingMuch',
'Ginko',
'3.2.1',
'10.0.4',
'release-ww3v20',
'release-cck-blah',
'*****@*****.**',
1
]
yield [
'2019-01-01',
'NothingMuch',
u'Ginko☢\0',
'2.3.2',
'10.0.5a',
'release',
'release-cck-\\',
'%7Ba-guid%7D',
2
]
yield [
'2019-01-01',
'Missing',
'Ginko',
'2.3.2',
'',
None,
'release',
'%7Ba-guid%7D',
2
]
yield [
yesterday,
'FennecAndroid', # product name
'Ginko', # platform?
'3.1415', # platform version
'38.0', # product version
'20150427090529', # build
'release', # update channel
'a-guid', # product guid
666 # count
]
fake_hive.connect.return_value \
.cursor.return_value.__iter__ = return_test_data
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['fetch-adi-from-hive']
assert not information['fetch-adi-from-hive']['last_error']
config.logger.info.assert_called_with(
'Wrote 4 rows from doing hive query'
)
fake_hive.connect.assert_called_with(
database='default',
authMechanism='PLAIN',
host='localhost',
user='******',
password='******',
port=10000,
timeout=1800000,
)
pgcursor = self.conn.cursor()
columns = (
'report_date',
'product_name',
'product_os_platform',
'product_os_version',
'product_version',
'build',
'build_channel',
'product_guid',
'count'
)
pgcursor.execute(
"select %s from raw_adi_logs" % ','.join(columns)
)
adi_logs = [dict(zip(columns, row)) for row in pgcursor.fetchall()]
expected = {
'report_date': yesterday,
'product_name': 'WinterWolf',
'product_os_platform': 'Ginko',
'product_os_version': '2.3.1',
'product_version': '10.0.4',
'build': 'nightly-ww3v20',
'build_channel': 'nightly',
'product_guid': 'a-guid',
'count': 1
}
assert adi_logs[0] == expected
expected = {
'report_date': yesterday,
'product_name': 'NothingMuch',
'product_os_platform': 'Ginko',
'product_os_version': '3.2.1',
'product_version': '10.0.4',
'build': 'release-ww3v20',
'build_channel': 'release-cck-blah',
'product_guid': '*****@*****.**',
'count': 1
}
assert adi_logs[1] == expected
expected = {
'report_date': datetime.date(2019, 1, 1),
'product_name': 'NothingMuch',
'product_os_platform': 'Ginko\xe2\x98\xa2',
'product_os_version': '2.3.2',
'product_version': '10.0.5a',
'build': 'release',
'build_channel': 'release-cck-\\',
'product_guid': '{a-guid}',
'count': 2
}
assert adi_logs[2] == expected
expected = {
'report_date': yesterday,
'product_name': 'FennecAndroid',
'product_os_platform': 'Ginko',
'product_os_version': '3.1415',
'product_version': '38.0',
'build': '20150427090529',
'build_channel': 'release',
'product_guid': 'a-guid',
'count': 666
}
assert adi_logs[3] == expected
columns = (
'adi_count',
'date',
'product_name',
'product_os_platform',
'product_os_version',
'product_version',
'build',
'product_guid',
'update_channel',
)
pgcursor.execute(
""" select %s from raw_adi
order by update_channel desc""" % ','.join(columns)
)
adi = [dict(zip(columns, row)) for row in pgcursor.fetchall()]
expected = {
'update_channel': 'release',
'product_guid': '{[email protected]}',
'product_version': '10.0.4',
'adi_count': 1,
'product_os_platform': 'Ginko',
'build': 'release-ww3v20',
'date': yesterday,
'product_os_version': '3.2.1',
'product_name': 'NothingMuch'
}
assert adi[0] == expected
expected = {
'update_channel': 'nightly',
'product_guid': 'a-guid',
'product_version': '10.0.4',
'adi_count': 1,
'product_os_platform': 'Ginko',
'build': 'nightly-ww3v20',
'date': yesterday,
'product_os_version': '2.3.1',
'product_name': 'WinterWolf'
}
assert adi[1] == expected
expected = {
'update_channel': 'beta',
'product_guid': 'a-guid',
'product_version': '38.0',
'adi_count': 666,
'product_os_platform': 'Ginko',
'build': '20150427090529',
'date': yesterday,
'product_os_version': '3.1415',
'product_name': 'FennecAndroid'
}
assert adi[2] == expected
def test_mocked_fetch_with_secondary_destination(self, fake_hive):
class MockedPGConnectionContext:
connection = mock.MagicMock()
def __init__(self, config):
self.config = config
@contextlib.contextmanager
def __call__(self):
yield self.connection
config_manager = self._setup_config_manager(
overrides={
'crontabber.class-FetchADIFromHiveCronApp.'
'secondary_destination.'
'database_class': MockedPGConnectionContext,
}
)
yesterday = (
datetime.datetime.utcnow() - datetime.timedelta(days=1)
).date()
def return_test_data(fake):
yield [
yesterday,
'WinterWolf',
'Ginko',
'2.3.1',
'10.0.4',
'nightly-ww3v20',
'nightly',
'a-guid',
1
]
fake_hive.connect.return_value \
.cursor.return_value.__iter__ = return_test_data
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['fetch-adi-from-hive']
assert not information['fetch-adi-from-hive']['last_error']
fake_hive.connect.assert_called_with(
database='default',
authMechanism='PLAIN',
host='localhost',
user='******',
password='******',
port=10000,
timeout=1800000,
)
# Critical test here.
# We make sure the secondary database class gets used
# for a `cursor.copy_from()` call.
assert MockedPGConnectionContext.connection.cursor().copy_from.called
def test_scrape_json_releases(self, requests_mocker):
today = utc_now()
requests_mocker.get(BASE_URL + 'firefox/',
text="""<html><body>
<a href="candidates/">candidates</a>
<a href="nightly/">nightly</a>
</body></html>""")
requests_mocker.get(BASE_URL + 'firefox/candidates/',
text="""<html><body>
<a href="28.0-candidates/">28.0-candidiates</a>
<a href="10.0b4-candidates/">10.0b4-candidiates</a>
<a href="None-candidates/">None-candidiates</a>
</body></html>""")
requests_mocker.get(BASE_URL + 'firefox/candidates/28.0-candidates/',
text="""<html><body>
<a href="build1/">build1</a>
</body></html>""")
requests_mocker.get(BASE_URL +
'firefox/candidates/28.0-candidates/build1/',
text="""<html><body>
<a href="linux-i686/">linux-i686</a>
</body></html>""")
requests_mocker.get(
BASE_URL + 'firefox/candidates/28.0-candidates/build1/linux-i686/',
text="""<html><body>
<a href="en-US/">en-US</a>
</body></html>""")
requests_mocker.get(
BASE_URL +
'firefox/candidates/28.0-candidates/build1/linux-i686/en-US/',
text="""<html><body>
<a href="firefox-28.0.json">firefox-28.0.json</a>
</body></html>""")
requests_mocker.get(
BASE_URL +
'firefox/candidates/28.0-candidates/build1/linux-i686/en-US/firefox-28.0.json',
json={
'buildid': '20140113161827',
'moz_app_maxversion': '28.0.*',
'moz_app_name': 'firefox',
'moz_app_vendor': 'Mozilla',
'moz_app_version': '28.0',
'moz_pkg_platform': 'linux-i686',
'moz_source_repo':
'http://hg.mozilla.org/releases/mozilla-release',
'moz_update_channel': 'release'
})
requests_mocker.get(BASE_URL + 'firefox/candidates/10.0b4-candidates/',
text="""<html><body>
<a href="build1/">build1</a>
</body></html>""")
requests_mocker.get(BASE_URL +
'firefox/candidates/10.0b4-candidates/build1/',
text="""<html><body>
<a href="linux-i686/">linux-i686</a>
</body></html>""")
requests_mocker.get(
BASE_URL +
'firefox/candidates/10.0b4-candidates/build1/linux-i686/',
text="""<html><body>
<a href="en-US/">en-US</a>
</body></html>""")
requests_mocker.get(
BASE_URL +
'firefox/candidates/10.0b4-candidates/build1/linux-i686/en-US/',
text="""<html><body>
<a href="firefox-10.0b4.json">firefox-10.0b4.json</a>
<a href="firefox-10.0b4.en-US.linux-i686.mozinfo.json">
firefox-10.0b4.en-US.linux-i686.mozinfo.json</a>
<a href="JUNK.json">JUNK.json</a>
</body></html>""")
requests_mocker.get(
BASE_URL +
'firefox/candidates/10.0b4-candidates/build1/linux-i686/en-US/firefox-10.0b4.json',
json={
'buildid': '20140113161826',
'moz_app_maxversion': '10.0.*',
'moz_app_name': 'firefox',
'moz_app_vendor': 'Mozilla',
'moz_app_version': '27.0',
'moz_pkg_platform': 'linux-i686',
'moz_source_repo':
'http://hg.mozilla.org/releases/mozilla-beta',
'moz_update_channel': 'beta'
})
# Ignore unrecognized JSON files, see bug 1065071
requests_mocker.get(
BASE_URL +
'firefox/candidates/10.0b4-candidates/build1/linux-i686/en-US/JUNK.json',
json={
'something': 'unexpected',
'nothing': 'else'
})
requests_mocker.get(BASE_URL + 'firefox/candidates/None-candidates/',
text="""<html><body>
<a href="build1/">build1</a>
</body></html>""")
requests_mocker.get(BASE_URL +
'firefox/candidates/None-candidates/build1/',
text="""<html><body>
<a href="linux-i686/">linux-i686</a>
</body></html>""")
requests_mocker.get(
BASE_URL + 'firefox/candidates/None-candidates/build1/linux-i686/',
text="""<html><body>
<a href="en-US/">en-US</a>
</body></html>""")
requests_mocker.get(
BASE_URL +
'firefox/candidates/None-candidates/build1/linux-i686/en-US/',
text="""<html><body>
<a href="None.json">None.json</a>
</body></html>""")
requests_mocker.get(
BASE_URL +
'firefox/candidates/None-candidates/build1/linux-i686/en-US/None.json',
text=""" """)
requests_mocker.get(BASE_URL + 'firefox/nightly/',
text="""<html><body>
<a href="%(year)s/">%(year)s</a>
</body></html>""" % {'year': today.strftime('%Y')})
requests_mocker.get(today.strftime(BASE_URL + 'firefox/nightly/%Y/'),
text="""<html><body>
<a href="%(month)s/">%(month)s</a>
</body></html>""" % {'month': today.strftime('%m')})
requests_mocker.get(today.strftime(BASE_URL +
'firefox/nightly/%Y/%m/'),
text="""<html><body>
<a href="%(date)s-03-02-03-mozilla-central/">txt</a>
<a href="%(date)s-03-02-04-mozilla-central/">txt</a>
""" % {'date': today.strftime('%Y-%m-%d')})
requests_mocker.get(today.strftime(
BASE_URL +
'firefox/nightly/%Y/%m/%Y-%m-%d-03-02-03-mozilla-central/'),
text="""<html><body>
<a href="firefox-30.0a1.en-US.linux-i686.json">txt</a>
</body></html>""")
requests_mocker.get(today.strftime(
BASE_URL +
'firefox/nightly/%Y/%m/%Y-%m-%d-03-02-03-mozilla-central/' +
'firefox-30.0a1.en-US.linux-i686.json'),
json={
'as': '$(CC)',
'buildid': '20140205030203',
'cc': '/usr/bin/ccache ',
'cxx': '/usr/bin/ccache stuff',
'host_alias': 'x86_64-unknown-linux-gnu',
'host_cpu': 'x86_64',
'host_os': 'linux-gnu',
'host_vendor': 'unknown',
'ld': 'ld',
'moz_app_id':
'{ec8030f7-c20a-464f-9b0e-13a3a9e97384}',
'moz_app_maxversion': '30.0a1',
'moz_app_name': 'firefox',
'moz_app_vendor': 'Mozilla',
'moz_app_version': '30.0a1',
'moz_pkg_platform': 'linux-i686',
'moz_source_repo':
'https://hg.mozilla.org/mozilla-central',
'moz_source_stamp': '1f170f9fead0',
'moz_update_channel': 'nightly',
'target_alias': 'i686-pc-linux',
'target_cpu': 'i686',
'target_os': 'linux-gnu',
'target_vendor': 'pc'
})
requests_mocker.get(today.strftime(
BASE_URL +
'firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/'),
text="""<html><body>
<a href="firefox-30.0a2.en-US.linux-i686.json">txt</a>
</body></html>""")
requests_mocker.get(today.strftime(
BASE_URL +
'firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/' +
'firefox-30.0a2.en-US.linux-i686.json'),
json={
'as': '$(CC)',
'buildid': '20140205030204',
'cc': '/usr/bin/ccache stuff',
'cxx': '/usr/bin/ccache stuff',
'host_alias': 'x86_64-unknown-linux-gnu',
'host_cpu': 'x86_64',
'host_os': 'linux-gnu',
'host_vendor': 'unknown',
'ld': 'ld',
'moz_app_id':
'{ec8030f7-c20a-464f-9b0e-13a3a9e97384}',
'moz_app_maxversion': '30.0a2',
'moz_app_name': 'firefox',
'moz_app_vendor': 'Mozilla',
'moz_app_version': '30.0a2',
'moz_pkg_platform': 'linux-i686',
'moz_source_repo':
'https://hg.mozilla.org/mozilla-central',
'moz_source_stamp': '1f170f9fead0',
'moz_update_channel': 'nightly',
'target_alias': 'i686-pc-linux',
'target_cpu': 'i686',
'target_os': 'linux-gnu',
'target_vendor': 'pc'
})
config_manager = self._setup_config_manager_firefox()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['ftpscraper']
assert not information['ftpscraper']['last_error']
assert information['ftpscraper']['last_success']
config.logger.warning.assert_any_call(
'Unable to JSON parse content %r', ' ', exc_info=True)
config.logger.warning.assert_any_call(
'warning, unsupported JSON file: %s',
BASE_URL + 'firefox/candidates/'
'10.0b4-candidates/build1/linux-i686/en-US/JUNK.json')
cursor = self.conn.cursor()
columns = 'product_name', 'build_id', 'build_type'
cursor.execute("""
select %s
from releases_raw
""" % ','.join(columns))
builds = [dict(zip(columns, row)) for row in cursor.fetchall()]
build_ids = dict((str(x['build_id']), x) for x in builds)
assert '20140113161827' in build_ids
assert '20140113161826' in build_ids
assert '20140205030203' in build_ids
assert len(build_ids) == 4
expected = [{
'build_id': 20140113161827,
'product_name': 'firefox',
'build_type': 'release'
}, {
'build_id': 20140113161827,
'product_name': 'firefox',
'build_type': 'beta'
}, {
'build_id': 20140113161826,
'product_name': 'firefox',
'build_type': 'beta'
}, {
'build_id': 20140205030203,
'product_name': 'firefox',
'build_type': 'nightly'
}, {
'build_id': 20140205030204,
'product_name': 'firefox',
'build_type': 'aurora'
}]
assert builds == expected
def test_scrape_json_releases(self):
@responsify
def mocked_get(url, today=None, timeout=None):
if today is None:
today = utc_now()
html_wrap = "<html><body>\n%s\n</body></html>"
if url.endswith('/firefox/'):
return html_wrap % """
<a href="candidates/">candidates</a>
<a href="nightly/">nightly</a>
"""
if url.endswith('/firefox/candidates/'):
return html_wrap % """
<a href="28.0-candidates/">28.0-candidiates</a>
<a href="10.0b4-candidates/">10.0b4-candidiates</a>
<a href="None-candidates/">None-candidiates</a>
"""
if url.endswith('-candidates/'):
return html_wrap % """
<a href="build1/">build1</a>
"""
if url.endswith('/build1/'):
return html_wrap % """
<a href="linux-i686/">linux-i686</a>
"""
if url.endswith('/firefox/candidates/28.0-candidates/'
'build1/linux-i686/en-US/'):
return html_wrap % """
<a href="firefox-28.0.json">firefox-28.0.json</a>
"""
if url.endswith('/firefox/candidates/10.0b4-candidates/'
'build1/linux-i686/en-US/'):
return html_wrap % """
<a href="firefox-10.0b4.json">firefox-10.0b4.json</a>
<a href="firefox-10.0b4.en-US.linux-i686.mozinfo.json">
firefox-10.0b4.en-US.linux-i686.mozinfo.json</a>
<a href="JUNK.json">
JUNK.json</a>
"""
if url.endswith('/firefox/candidates/None-candidates/'
'build1/linux-i686/en-US/'):
return html_wrap % """
<a href="None.json">None.json</a>
"""
if 'None.json' in url:
return """ """
if 'firefox-28.0.json' in url:
return """
{
"buildid": "20140113161827",
"moz_app_maxversion": "28.0.*",
"moz_app_name": "firefox",
"moz_app_vendor": "Mozilla",
"moz_app_version": "28.0",
"moz_pkg_platform": "linux-i686",
"moz_source_repo":
"http://hg.mozilla.org/releases/mozilla-release",
"moz_update_channel": "release"
}
"""
if 'firefox-10.0b4.json' in url:
return """
{
"buildid": "20140113161826",
"moz_app_maxversion": "10.0.*",
"moz_app_name": "firefox",
"moz_app_vendor": "Mozilla",
"moz_app_version": "27.0",
"moz_pkg_platform": "linux-i686",
"moz_source_repo":
"http://hg.mozilla.org/releases/mozilla-beta",
"moz_update_channel": "beta"
}
"""
# Ignore unrecognized JSON files, see bug 1065071
if 'JUNK.json' in url:
return """
{
"something": "unexpected",
"nothing": "else"
}
"""
# Nightly tests for nightly and aurora builds
if url.endswith('/firefox/nightly/'):
return html_wrap % """
<a href="2014/">2014</a>
"""
if url.endswith(today.strftime('/firefox/nightly/%Y/%m/')):
return html_wrap % """
<a href="%s-03-02-03-mozilla-central/">txt</a>
<a href="%s-03-02-04-mozilla-central/">txt</a>
""" % (today.strftime('%Y-%m-%d'), today.strftime('%Y-%m-%d'))
if url.endswith(today.strftime('/firefox/nightly/%Y/%m/'
'%Y-%m-%d-03-02-03-mozilla-central/')):
return html_wrap % """
<a href="firefox-30.0a1.en-US.linux-i686.json">txt</a>
"""
if url.endswith(today.strftime(
'/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/'
)):
return html_wrap % """
<a href="firefox-30.0a2.en-US.linux-i686.json">txt</a>
"""
if url.endswith(today.strftime(
'/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/'
'firefox-30.0a2.en-US.linux-i686.json'
)):
return """
{
"as": "$(CC)",
"buildid": "20140205030204",
"cc": "/usr/bin/ccache stuff",
"cxx": "/usr/bin/ccache stuff",
"host_alias": "x86_64-unknown-linux-gnu",
"host_cpu": "x86_64",
"host_os": "linux-gnu",
"host_vendor": "unknown",
"ld": "ld",
"moz_app_id": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}",
"moz_app_maxversion": "30.0a2",
"moz_app_name": "firefox",
"moz_app_vendor": "Mozilla",
"moz_app_version": "30.0a2",
"moz_pkg_platform": "linux-i686",
"moz_source_repo":
"https://hg.mozilla.org/mozilla-central",
"moz_source_stamp": "1f170f9fead0",
"moz_update_channel": "nightly",
"target_alias": "i686-pc-linux",
"target_cpu": "i686",
"target_os": "linux-gnu",
"target_vendor": "pc"
}
"""
if url.endswith(today.strftime(
'/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-03-mozilla-central/'
'firefox-30.0a1.en-US.linux-i686.json'
)):
return """
{
"as": "$(CC)",
"buildid": "20140205030203",
"cc": "/usr/bin/ccache ",
"cxx": "/usr/bin/ccache stuff",
"host_alias": "x86_64-unknown-linux-gnu",
"host_cpu": "x86_64",
"host_os": "linux-gnu",
"host_vendor": "unknown",
"ld": "ld",
"moz_app_id": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}",
"moz_app_maxversion": "30.0a1",
"moz_app_name": "firefox",
"moz_app_vendor": "Mozilla",
"moz_app_version": "30.0a1",
"moz_pkg_platform": "linux-i686",
"moz_source_repo":
"https://hg.mozilla.org/mozilla-central",
"moz_source_stamp": "1f170f9fead0",
"moz_update_channel": "nightly",
"target_alias": "i686-pc-linux",
"target_cpu": "i686",
"target_os": "linux-gnu",
"target_vendor": "pc"
}
"""
raise NotImplementedError(url)
self.mocked_session().get.side_effect = mocked_get
config_manager = self._setup_config_manager_firefox()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['ftpscraper']
assert not information['ftpscraper']['last_error']
assert information['ftpscraper']['last_success']
config.logger.warning.assert_any_call(
'Unable to JSON parse content %r',
' ',
exc_info=True
)
base_url = config.crontabber['class-FTPScraperCronApp'].base_url
config.logger.warning.assert_any_call(
'warning, unsupported JSON file: %s',
base_url + 'firefox/candidates/'
'10.0b4-candidates/build1/linux-i686/en-US/JUNK.json'
)
cursor = self.conn.cursor()
columns = 'product_name', 'build_id', 'build_type'
cursor.execute("""
select %s
from releases_raw
""" % ','.join(columns))
builds = [dict(zip(columns, row)) for row in cursor.fetchall()]
build_ids = dict((str(x['build_id']), x) for x in builds)
assert '20140113161827' in build_ids
assert '20140113161826' in build_ids
assert '20140205030203' in build_ids
assert len(build_ids) == 4
expected = [
{
'build_id': 20140113161827,
'product_name': 'firefox',
'build_type': 'release'
},
{
'build_id': 20140113161827,
'product_name': 'firefox',
'build_type': 'beta'
},
{
'build_id': 20140113161826,
'product_name': 'firefox',
'build_type': 'beta'
},
{
'build_id': 20140205030203,
'product_name': 'firefox',
'build_type': 'nightly'
},
{
'build_id': 20140205030204,
'product_name': 'firefox',
'build_type': 'aurora'
}
]
assert builds == expected