def test_basic_run(self): cur = self.conn.cursor() # Ensure test table is present. statement = """ INSERT INTO missing_symbols (date_processed, debug_file, debug_id, code_file, code_id) VALUES (%(first)s, 'foo.pdb', '0420', 'foo.py', '123'), (%(second)s, 'bar.pdb', '65EA9', 'bar.py', null) """ second = utc_now().date() first = second - datetime.timedelta(days=1) cur.execute(statement, {'first': first, 'second': second}) self.conn.commit() # Run the crontabber job to remove the test table. config_manager = self._setup_config_manager(days_to_keep=1) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() # Basic assertion test of stored procedure. information = self._load_structure() assert information['clean-missing-symbols'] assert not information['clean-missing-symbols']['last_error'] assert information['clean-missing-symbols']['last_success'] # Ensure expected test row was removed cur.execute(""" SELECT date_processed FROM missing_symbols """) first, = cur.fetchall() date_processed = first[0] assert date_processed == second
def test_run_drop_old_partitions(self): cur = self.conn.cursor() # Ensure test table is present. statement = """ SELECT COUNT(*) FROM raw_crashes_20120102; """ cur.execute(statement) result = cur.fetchone() assert result[0] == 2L # need to get out of this transaction to allow crontabber to acquire a lock self.conn.commit() # Run the crontabber job to remove the test table. config_manager = self._setup_config_manager() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() # Basic assertion test of stored procedure. information = self._load_structure() print(information['truncate-partitions']['last_error']) assert information['truncate-partitions'] assert not information['truncate-partitions']['last_error'] assert information['truncate-partitions']['last_success'] # Ensure test table was removed. statement = """ SELECT COUNT(*) FROM raw_crashes_20120102; """ cur.execute(statement) result = cur.fetchone() assert result[0] == 0
def test_basic_run_job(self, requests_mocker): requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS) config_manager = self._setup_config_manager(3) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] assert not information['bugzilla-associations']['last_error'] assert information['bugzilla-associations']['last_success'] cursor = self.conn.cursor() cursor.execute('select bug_id from bug_associations order by bug_id') associations = cursor.fetchall() # Verify we have the expected number of associations. assert len(associations) == 8 bug_ids = [x[0] for x in associations] # Verify bugs with no crash signatures are missing. assert 6 not in bug_ids cursor.execute( 'select signature from bug_associations where bug_id = 8' ) associations = cursor.fetchall() # New signatures have correctly been inserted. assert len(associations) == 2 assert ('another::legitimate(sig)',) in associations assert ('legitimate(sig)',) in associations
def test_basic_run(self): cur = self.conn.cursor() # Ensure test table is present. statement = """ INSERT INTO raw_adi (date, product_name, adi_count) VALUES (%(first)s, 'WinterFox', 11), (%(second)s, 'WinterFox', 23) """ second = utc_now().date() first = second - datetime.timedelta(days=1) cur.execute(statement, {'first': first, 'second': second}) self.conn.commit() # Run the crontabber job to remove the test table. config_manager = self._setup_config_manager(days_to_keep=1) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() # Basic assertion test of stored procedure. information = self._load_structure() assert information['clean-raw-adi'] assert not information['clean-raw-adi']['last_error'] assert information['clean-raw-adi']['last_success'] # Ensure test row was removed cur.execute(""" SELECT date FROM raw_adi """) result, = cur.fetchall() report_date = result[0] assert report_date == second
def test_all_matviews(self, mocked_utc_now): # Pretend it's 03AM UTC def mock_utc_now(): n = utc_now() n = n.replace(hour=3) return n mocked_utc_now.side_effect = mock_utc_now config_manager = self._setup_config_manager( 'socorro.unittest.cron.jobs.test_matviews.FTPScraperJob|1d\n' 'socorro.unittest.cron.jobs.test_matviews.FetchADIFromHiveCronApp|1d\n' '' 'socorro.cron.jobs.matviews.ProductVersionsCronApp|1d\n' 'socorro.cron.jobs.matviews.ADUCronApp|1d\n' 'socorro.cron.jobs.matviews.BuildADUCronApp|1d|02:00\n' ) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() for app_name in ('product-versions-matview', 'adu-matview', 'build-adu-matview'): assert app_name in information assert not information[app_name]['last_error'] assert information[app_name]['last_success']
def test_run_job_with_reports_with_existing_bugs_different(self, requests_mocker): """Verify that an association to a signature that no longer is part of the crash signatures list gets removed. """ requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS) config_manager = self._setup_config_manager(3) cursor = self.conn.cursor() cursor.execute(""" insert into bug_associations (bug_id, signature) values (8, '@different'); """) self.conn.commit() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] assert not information['bugzilla-associations']['last_error'] assert information['bugzilla-associations']['last_success'] cursor.execute( 'select signature from bug_associations where bug_id = 8' ) associations = cursor.fetchall() # The previous association, to signature '@different' that is not in # crash signatures, is now missing. assert ('@different',) not in associations
def test_run_job_with_reports_with_existing_bugs_same(self, requests_mocker): requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS) config_manager = self._setup_config_manager(3) cursor = self.conn.cursor() cursor.execute(""" insert into bug_associations (bug_id, signature) values (8, 'legitimate(sig)'); """) self.conn.commit() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] assert not information['bugzilla-associations']['last_error'] assert information['bugzilla-associations']['last_success'] cursor.execute( 'select signature from bug_associations where bug_id = 8' ) associations = cursor.fetchall() # New signatures have correctly been inserted. assert len(associations) == 2 assert ('another::legitimate(sig)',) in associations assert ('legitimate(sig)',) in associations
def test_basic_run_job(self, requests_mocker): requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS) config_manager = self._setup_config_manager(3) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] assert not information['bugzilla-associations']['last_error'] assert information['bugzilla-associations']['last_success'] cursor = self.conn.cursor() cursor.execute('select bug_id from bug_associations order by bug_id') associations = cursor.fetchall() # Verify we have the expected number of associations. assert len(associations) == 8 bug_ids = [x[0] for x in associations] # Verify bugs with no crash signatures are missing. assert 6 not in bug_ids cursor.execute( 'select signature from bug_associations where bug_id = 8' ) associations = cursor.fetchall() # New signatures have correctly been inserted. assert len(associations) == 2 assert ('another::legitimate(sig)',) in associations assert ('legitimate(sig)',) in associations
def test_run_drop_old_partitions(self): cur = self.conn.cursor() # Ensure test table is present. statement = """ SELECT COUNT(*) FROM information_schema.tables WHERE table_name = 'phrotest_20120102'; """ cur.execute(statement) result = cur.fetchone() assert result[0] == 1 # Run the crontabber job to remove the test table. config_manager = self._setup_config_manager() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() # Basic assertion test of stored procedure. information = self._load_structure() assert information['drop-old-partitions'] assert not information['drop-old-partitions']['last_error'] assert information['drop-old-partitions']['last_success'] # Ensure test table was removed. statement = """ SELECT COUNT(*) FROM information_schema.tables WHERE table_name = 'phrotest_20120102'; """ cur.execute(statement) result = cur.fetchone() assert result[0] == 0
def test_basic_run_job(self, requests_mocker): requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS) config_manager = self._setup_config_manager(3) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] assert not information['bugzilla-associations']['last_error'] assert information['bugzilla-associations']['last_success'] associations = self.fetch_data() # Verify we have the expected number of associations assert len(associations) == 8 bug_ids = set([x['bug_id'] for x in associations]) # Verify bugs with no crash signatures are missing assert 6 not in bug_ids bug_8_signatures = [ item['signature'] for item in associations if item['bug_id'] == '8' ] # New signatures have correctly been inserted assert len(bug_8_signatures) == 2 assert 'another::legitimate(sig)' in bug_8_signatures assert 'legitimate(sig)' in bug_8_signatures
def test_run_job_with_reports_with_existing_bugs_same(self, requests_mocker): requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS) config_manager = self._setup_config_manager(3) cursor = self.conn.cursor() cursor.execute(""" insert into bug_associations (bug_id, signature) values (8, 'legitimate(sig)'); """) self.conn.commit() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] assert not information['bugzilla-associations']['last_error'] assert information['bugzilla-associations']['last_success'] cursor.execute( 'select signature from bug_associations where bug_id = 8' ) associations = cursor.fetchall() # New signatures have correctly been inserted. assert len(associations) == 2 assert ('another::legitimate(sig)',) in associations assert ('legitimate(sig)',) in associations
def test_run_job_with_reports_with_existing_bugs_different(self, requests_mocker): """Verify that an association to a signature that no longer is part of the crash signatures list gets removed. """ requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS) config_manager = self._setup_config_manager(3) cursor = self.conn.cursor() cursor.execute(""" insert into bug_associations (bug_id, signature) values (8, '@different'); """) self.conn.commit() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] assert not information['bugzilla-associations']['last_error'] assert information['bugzilla-associations']['last_success'] cursor.execute( 'select signature from bug_associations where bug_id = 8' ) associations = cursor.fetchall() # The previous association, to signature '@different' that is not in # crash signatures, is now missing. assert ('@different',) not in associations
def test_run_weekly_reports_partitions(self): config_manager = self._setup_config_manager() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['weekly-reports-partitions'] assert not information['weekly-reports-partitions']['last_error'] assert information['weekly-reports-partitions']['last_success']
def test_run(self, connection_context): config_manager = self._setup_config_manager() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['elasticsearch-cleanup'] assert not information['elasticsearch-cleanup']['last_error'] assert information['elasticsearch-cleanup']['last_success']
def test_mocked_fetch_with_secondary_destination(self, fake_hive): class MockedPGConnectionContext: connection = mock.MagicMock() def __init__(self, config): self.config = config @contextlib.contextmanager def __call__(self): yield self.connection config_manager = self._setup_config_manager( overrides={ 'crontabber.class-FetchADIFromHiveCronApp.' 'secondary_destination.' 'database_class': MockedPGConnectionContext, }) yesterday = (datetime.datetime.utcnow() - datetime.timedelta(days=1)).date() def return_test_data(fake): yield [ yesterday, 'WinterWolf', 'Ginko', '2.3.1', '10.0.4', 'nightly-ww3v20', 'nightly', 'a-guid', 1 ] fake_hive.connect.return_value \ .cursor.return_value.__iter__ = return_test_data with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['fetch-adi-from-hive'] assert not information['fetch-adi-from-hive']['last_error'] fake_hive.connect.assert_called_with( database='default', authMechanism='PLAIN', host='localhost', user='******', password='******', port=10000, timeout=1800000, ) # Critical test here. # We make sure the secondary database class gets used # for a `cursor.copy_from()` call. assert MockedPGConnectionContext.connection.cursor().copy_from.called
def test_mocked_fetch(self): config_manager = self._setup_config_manager() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['fetch-adi-from-hive'] assert not information['fetch-adi-from-hive']['last_error'] config.logger.info.assert_called_with( 'Faking the fetching of ADI from Hive :)')
def test_duplicates(self): config_manager = self._setup_config_manager( 'socorro.cron.jobs.matviews.DuplicatesCronApp|1d' ) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['duplicates'] assert not information['duplicates']['last_error'] assert information['duplicates']['last_success']
def test_mocked_fetch(self): config_manager = self._setup_config_manager() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['fetch-adi-from-hive'] assert not information['fetch-adi-from-hive']['last_error'] config.logger.info.assert_called_with( 'Faking the fetching of ADI from Hive :)' )
def test_reports_clean_with_dependency(self): config_manager = self._setup_config_manager( 'socorro.cron.jobs.matviews.DuplicatesCronApp|1h\n' 'socorro.cron.jobs.matviews.ReportsCleanCronApp|1h' ) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['reports-clean'] assert not information['reports-clean']['last_error'] assert information['reports-clean']['last_success']
def test_download_error(self, req_mock): config_manager = self._setup_config_manager() req_mock.get('http://example.com/firefox_versions.json', status_code=404) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['featured-versions-automatic'] assert information['featured-versions-automatic']['last_error'] error = information['featured-versions-automatic']['last_error'] assert 'DownloadError' in error['type'] assert '404' in error['value']
def test_basic_run(self): cur = self.conn.cursor() # Ensure test table is present. statement = """ INSERT INTO raw_adi (date, product_name, adi_count) VALUES (%(first)s, 'WinterFox', 11), (%(second)s, 'WinterFox', 23) """ second = utc_now().date() first = second - datetime.timedelta(days=1) cur.execute(statement, {'first': first, 'second': second}) # Ensure test table is present. statement = """ INSERT INTO raw_adi_logs (report_date, product_name, count) VALUES (%(first)s, 'WinterFox', 11), (%(second)s, 'WinterFox', 23) """ second = utc_now().date() first = second - datetime.timedelta(days=1) cur.execute(statement, {'first': first, 'second': second}) self.conn.commit() # Run the crontabber job to remove the test table. config_manager = self._setup_config_manager(days_to_keep=1) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() # Basic assertion test of stored procedure. information = self._load_structure() assert information['clean-raw-adi'] assert not information['clean-raw-adi']['last_error'] assert information['clean-raw-adi']['last_success'] # Ensure test row was removed cur.execute('SELECT date FROM raw_adi') result, = cur.fetchall() report_date = result[0] assert report_date == second cur.execute('SELECT report_date FROM raw_adi_logs') result, = cur.fetchall() report_date = result[0] assert report_date == second
def test_with_bugzilla_failure(self, requests_mocker): requests_mocker.get(BUGZILLA_BASE_URL, text='error loading content', status_code=500) config_manager = self._setup_config_manager(3) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] # Verify there has been an error last_error = information['bugzilla-associations']['last_error'] assert last_error assert 'HTTPError' in last_error['type'] assert not information['bugzilla-associations']['last_success']
def test_basic_run(self): # We need to prepare to return a size for the new key self.mock_key.size = 123456789 self.mock_key.generate_url.return_value = ( 'https://s3.example.com/latest.csv' ) # Run the crontabber job to remove the test table. config_manager = self._setup_config_manager() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() # Basic assertion test of stored procedure. information = self._load_structure() assert information['missing-symbols'] assert not information['missing-symbols']['last_error'] assert information['missing-symbols']['last_success'] self.mock_boto_class()._connect.assert_called_with() self.mock_boto_class.close.assert_called_with() self.mock_bucket.new_key.assert_called_with('latest.csv') content = StringIO() writer = csv.writer(content) writer.writerow(( 'debug_file', 'debug_id', 'code_file', 'code_id', )) writer.writerow(( 'nvwgf2um.pdb', '9D492B844FF34800B34320464AA1E7E41', 'nvwgf2um.dll', '561D1D4Ff58000', )) self.mock_key.set_contents_from_string.assert_called_with( content.getvalue() ) # this is becausse 123456789 bytes is 117.74 Mb tab.config.logger.info.assert_called_with( 'Generated https://s3.example.com/latest.csv ' '(123,456,789 bytes, 117.74 Mb)' )
def test_run(self, connect_s3): key = mock.MagicMock() connect_s3().get_bucket().get_key.return_value = None connect_s3().get_bucket().new_key.return_value = key with self._setup_config_manager().context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() app_name = 'upload-crash-report-json-schema' assert information[app_name] assert not information[app_name]['last_error'] assert information[app_name]['last_success'] key.set_contents_from_string.assert_called_with( CRASH_REPORT_JSON_SCHEMA_AS_STRING)
def test_download_error(self, rget): config_manager = self._setup_config_manager() def mocked_get(url): return Response('not here', status_code=404) rget.side_effect = mocked_get with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['featured-versions-automatic'] assert information['featured-versions-automatic']['last_error'] error = information['featured-versions-automatic']['last_error'] assert 'DownloadError' in error['type'] assert '404' in error['value']
def test_one_matview_alone(self): config_manager = self._setup_config_manager( 'socorro.unittest.cron.jobs.test_matviews.FTPScraperJob|1d\n' '' 'socorro.cron.jobs.matviews.ProductVersionsCronApp|1d' ) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['ftpscraper'] assert not information['ftpscraper']['last_error'] assert information['ftpscraper']['last_success'] assert information['product-versions-matview'] assert not information['product-versions-matview']['last_error'] assert information['product-versions-matview']['last_success']
def test_with_bugzilla_failure(self, requests_mocker): requests_mocker.get( BUGZILLA_BASE_URL, text='error loading content', status_code=500 ) config_manager = self._setup_config_manager(3) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] # There has been an error. last_error = information['bugzilla-associations']['last_error'] assert last_error assert 'HTTPError' in last_error['type'] assert not information['bugzilla-associations']['last_success']
def test_fetch_with_zero_hive_results(self, fake_hive): config_manager = self._setup_config_manager() def return_test_data(fake): # a generator that yields literally nothing # http://stackoverflow.com/a/13243870/205832 return yield fake_hive.connect.return_value \ .cursor.return_value.__iter__ = return_test_data with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['fetch-adi-from-hive'] assert information['fetch-adi-from-hive']['last_error'] assert ( NoRowsWritten.__name__ in information['fetch-adi-from-hive']['last_error']['type'] ) assert information['fetch-adi-from-hive']['last_error'] fake_hive.connect.assert_called_with( database='default', authMechanism='PLAIN', host='localhost', user='******', password='******', port=10000, timeout=1800000, ) pgcursor = self.conn.cursor() pgcursor.execute( "select count(*) from raw_adi_logs" ) count, = pgcursor.fetchone() assert count == 0
def test_run_job_based_on_last_success(self, requests_mocker): """specifically setting 0 days back and no prior run will pick it up from now's date""" requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS) config_manager = self._setup_config_manager(0) cursor = self.conn.cursor() # these are matching the SAMPLE_CSV above cursor.execute("""insert into bug_associations (bug_id,signature) values (8, 'legitimate(sig)'); """) self.conn.commit() # second time config_manager = self._setup_config_manager(0) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() state = tab.job_state_database.copy() self._wind_clock(state, days=1) tab.job_state_database.update(state) # Create a CSV file for one day back. # This'll make sure there's a .csv file whose day # is that of the last run. self._setup_config_manager(1) config_manager = self._setup_config_manager(0) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] assert not information['bugzilla-associations']['last_error'] assert information['bugzilla-associations']['last_success']
def test_run_job_with_reports_with_existing_bugs_different( self, requests_mocker): """Verify that an association to a signature that no longer is part of the crash signatures list gets removed. """ requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS) self.insert_data(bug_id='8', signature='@different') config_manager = self._setup_config_manager(3) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] assert not information['bugzilla-associations']['last_error'] assert information['bugzilla-associations']['last_success'] # The previous association, to signature '@different' that is not in # crash signatures, is now missing associations = self.fetch_data() assert '@different' not in [item['signature'] for item in associations]
def test_run_job_based_on_last_success(self, requests_mocker): """specifically setting 0 days back and no prior run will pick it up from now's date""" requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS) config_manager = self._setup_config_manager(0) cursor = self.conn.cursor() # these are matching the SAMPLE_CSV above cursor.execute("""insert into bug_associations (bug_id,signature) values (8, 'legitimate(sig)'); """) self.conn.commit() # second time config_manager = self._setup_config_manager(0) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() state = tab.job_state_database.copy() self._wind_clock(state, days=1) tab.job_state_database.update(state) # Create a CSV file for one day back. # This'll make sure there's a .csv file whose day # is that of the last run. self._setup_config_manager(1) config_manager = self._setup_config_manager(0) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] assert not information['bugzilla-associations']['last_error'] assert information['bugzilla-associations']['last_success']
def test_run_job_with_reports_with_existing_bugs_same( self, requests_mocker): requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS) self.insert_data(bug_id='8', signature='legitimate(sig)') config_manager = self._setup_config_manager(3) with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['bugzilla-associations'] assert not information['bugzilla-associations']['last_error'] assert information['bugzilla-associations']['last_success'] associations = self.fetch_data() associations = [ item['signature'] for item in associations if item['bug_id'] == '8' ] # New signatures have correctly been inserted assert len(associations) == 2 assert associations == ['another::legitimate(sig)', 'legitimate(sig)']
def test_basic_run_job(self, req_mock): config_manager = self._setup_config_manager() req_mock.get( 'http://example.com/firefox_versions.json', json={ 'FIREFOX_NIGHTLY': '52.0a1', # Kept for legacy and smooth transition. We USED to consider # the latest AURORA version a featured version but we no # longer build aurora so Socorro shouldn't pick this up any # more even if product-details.mozilla.org supplies it. 'FIREFOX_AURORA': '51.0a2', 'FIREFOX_ESR': '45.4.0esr', 'FIREFOX_ESR_NEXT': '', 'LATEST_FIREFOX_DEVEL_VERSION': '50.0b7', 'LATEST_FIREFOX_OLDER_VERSION': '3.6.28', 'LATEST_FIREFOX_RELEASED_DEVEL_VERSION': '50.0b7', 'LATEST_FIREFOX_VERSION': '49.0.1', }) req_mock.get('http://example.com/mobile_versions.json', json={ 'nightly_version': '52.0a1', 'alpha_version': '51.0a2', 'beta_version': '50.0b6', 'version': '49.0', 'ios_beta_version': '6.0', 'ios_version': '5.0', }) req_mock.get('http://example.com/thunderbird_versions.json', json={ 'LATEST_THUNDERBIRD_VERSION': '45.4.0', 'LATEST_THUNDERBIRD_DEVEL_VERSION': '50.0b1', 'LATEST_THUNDERBIRD_ALPHA_VERSION': '51.0a2', 'LATEST_THUNDERBIRD_NIGHTLY_VERSION': '52.0a1', }) # Check what's set up in the fixture rows = execute_query_fetchall( self.conn, 'select product_name, version_string, featured_version ' 'from product_versions order by version_string') assert sorted(rows) == [ ('Firefox', '15.0a1', True), ('Firefox', '24.5.0', True), ('Firefox', '49.0.1', False), ('Firefox', '50.0b', False), ('Firefox', '51.0a2', False), ('Firefox', '52.0a1', False), ] # This is necessary so we get a new cursor when we do other selects # after the crontabber app has run. self.conn.commit() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['featured-versions-automatic'] assert not information['featured-versions-automatic']['last_error'] assert information['featured-versions-automatic']['last_success'] config.logger.info.assert_called_with( 'Set featured versions for Thunderbird to: ' '45.4.0, 50.0b1, 52.0a1') rows = execute_query_fetchall( self.conn, 'select product_name, version_string, featured_version ' 'from product_versions') expected = [ ('Firefox', '15.0a1', False), ('Firefox', '24.5.0', False), ('Firefox', '49.0.1', True), ('Firefox', '50.0b', True), # Note that the 'Aurora' branch is still mentioned but note that it's NOT featured # (hence 'False'). ('Firefox', '51.0a2', False), ('Firefox', '52.0a1', True), ] assert sorted(rows) == expected
def test_scrape_json_releases(self): @responsify def mocked_get(url, today=None, timeout=None): if today is None: today = utc_now() html_wrap = "<html><body>\n%s\n</body></html>" if url.endswith('/firefox/'): return html_wrap % """ <a href="candidates/">candidates</a> <a href="nightly/">nightly</a> """ if url.endswith('/firefox/candidates/'): return html_wrap % """ <a href="28.0-candidates/">28.0-candidiates</a> <a href="10.0b4-candidates/">10.0b4-candidiates</a> <a href="None-candidates/">None-candidiates</a> """ if url.endswith('-candidates/'): return html_wrap % """ <a href="build1/">build1</a> """ if url.endswith('/build1/'): return html_wrap % """ <a href="linux-i686/">linux-i686</a> """ if url.endswith('/firefox/candidates/28.0-candidates/' 'build1/linux-i686/en-US/'): return html_wrap % """ <a href="firefox-28.0.json">firefox-28.0.json</a> """ if url.endswith('/firefox/candidates/10.0b4-candidates/' 'build1/linux-i686/en-US/'): return html_wrap % """ <a href="firefox-10.0b4.json">firefox-10.0b4.json</a> <a href="firefox-10.0b4.en-US.linux-i686.mozinfo.json"> firefox-10.0b4.en-US.linux-i686.mozinfo.json</a> <a href="JUNK.json"> JUNK.json</a> """ if url.endswith('/firefox/candidates/None-candidates/' 'build1/linux-i686/en-US/'): return html_wrap % """ <a href="None.json">None.json</a> """ if 'None.json' in url: return """ """ if 'firefox-28.0.json' in url: return """ { "buildid": "20140113161827", "moz_app_maxversion": "28.0.*", "moz_app_name": "firefox", "moz_app_vendor": "Mozilla", "moz_app_version": "28.0", "moz_pkg_platform": "linux-i686", "moz_source_repo": "http://hg.mozilla.org/releases/mozilla-release", "moz_update_channel": "release" } """ if 'firefox-10.0b4.json' in url: return """ { "buildid": "20140113161826", "moz_app_maxversion": "10.0.*", "moz_app_name": "firefox", "moz_app_vendor": "Mozilla", "moz_app_version": "27.0", "moz_pkg_platform": "linux-i686", "moz_source_repo": "http://hg.mozilla.org/releases/mozilla-beta", "moz_update_channel": "beta" } """ # Ignore unrecognized JSON files, see bug 1065071 if 'JUNK.json' in url: return """ { "something": "unexpected", "nothing": "else" } """ # Nightly tests for nightly and aurora builds if url.endswith('/firefox/nightly/'): return html_wrap % """ <a href="2014/">2014</a> """ if url.endswith(today.strftime('/firefox/nightly/%Y/%m/')): return html_wrap % """ <a href="%s-03-02-03-mozilla-central/">txt</a> <a href="%s-03-02-04-mozilla-central/">txt</a> """ % (today.strftime('%Y-%m-%d'), today.strftime('%Y-%m-%d')) if url.endswith( today.strftime('/firefox/nightly/%Y/%m/' '%Y-%m-%d-03-02-03-mozilla-central/')): return html_wrap % """ <a href="firefox-30.0a1.en-US.linux-i686.json">txt</a> """ if url.endswith( today.strftime( '/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/' )): return html_wrap % """ <a href="firefox-30.0a2.en-US.linux-i686.json">txt</a> """ if url.endswith( today.strftime( '/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/' 'firefox-30.0a2.en-US.linux-i686.json')): return """ { "as": "$(CC)", "buildid": "20140205030204", "cc": "/usr/bin/ccache stuff", "cxx": "/usr/bin/ccache stuff", "host_alias": "x86_64-unknown-linux-gnu", "host_cpu": "x86_64", "host_os": "linux-gnu", "host_vendor": "unknown", "ld": "ld", "moz_app_id": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}", "moz_app_maxversion": "30.0a2", "moz_app_name": "firefox", "moz_app_vendor": "Mozilla", "moz_app_version": "30.0a2", "moz_pkg_platform": "linux-i686", "moz_source_repo": "https://hg.mozilla.org/mozilla-central", "moz_source_stamp": "1f170f9fead0", "moz_update_channel": "nightly", "target_alias": "i686-pc-linux", "target_cpu": "i686", "target_os": "linux-gnu", "target_vendor": "pc" } """ if url.endswith( today.strftime( '/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-03-mozilla-central/' 'firefox-30.0a1.en-US.linux-i686.json')): return """ { "as": "$(CC)", "buildid": "20140205030203", "cc": "/usr/bin/ccache ", "cxx": "/usr/bin/ccache stuff", "host_alias": "x86_64-unknown-linux-gnu", "host_cpu": "x86_64", "host_os": "linux-gnu", "host_vendor": "unknown", "ld": "ld", "moz_app_id": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}", "moz_app_maxversion": "30.0a1", "moz_app_name": "firefox", "moz_app_vendor": "Mozilla", "moz_app_version": "30.0a1", "moz_pkg_platform": "linux-i686", "moz_source_repo": "https://hg.mozilla.org/mozilla-central", "moz_source_stamp": "1f170f9fead0", "moz_update_channel": "nightly", "target_alias": "i686-pc-linux", "target_cpu": "i686", "target_os": "linux-gnu", "target_vendor": "pc" } """ raise NotImplementedError(url) self.mocked_session().get.side_effect = mocked_get config_manager = self._setup_config_manager_firefox() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['ftpscraper'] assert not information['ftpscraper']['last_error'] assert information['ftpscraper']['last_success'] config.logger.warning.assert_any_call( 'Unable to JSON parse content %r', ' ', exc_info=True) base_url = config.crontabber['class-FTPScraperCronApp'].base_url config.logger.warning.assert_any_call( 'warning, unsupported JSON file: %s', base_url + 'firefox/candidates/' '10.0b4-candidates/build1/linux-i686/en-US/JUNK.json') cursor = self.conn.cursor() columns = 'product_name', 'build_id', 'build_type' cursor.execute(""" select %s from releases_raw """ % ','.join(columns)) builds = [dict(zip(columns, row)) for row in cursor.fetchall()] build_ids = dict((str(x['build_id']), x) for x in builds) assert '20140113161827' in build_ids assert '20140113161826' in build_ids assert '20140205030203' in build_ids assert len(build_ids) == 4 expected = [{ 'build_id': 20140113161827, 'product_name': 'firefox', 'build_type': 'release' }, { 'build_id': 20140113161827, 'product_name': 'firefox', 'build_type': 'beta' }, { 'build_id': 20140113161826, 'product_name': 'firefox', 'build_type': 'beta' }, { 'build_id': 20140205030203, 'product_name': 'firefox', 'build_type': 'nightly' }, { 'build_id': 20140205030204, 'product_name': 'firefox', 'build_type': 'aurora' }] assert builds == expected
def test_mocked_fetch(self, fake_hive): config_manager = self._setup_config_manager() yesterday = ( datetime.datetime.utcnow() - datetime.timedelta(days=1) ).date() def return_test_data(fake): yield [ yesterday, 'WinterWolf', 'Ginko', '2.3.1', '10.0.4', 'nightly-ww3v20', 'nightly', 'a-guid', 1 ] yield [ yesterday, 'NothingMuch', 'Ginko', '3.2.1', '10.0.4', 'release-ww3v20', 'release-cck-blah', '*****@*****.**', 1 ] yield [ '2019-01-01', 'NothingMuch', u'Ginko☢\0', '2.3.2', '10.0.5a', 'release', 'release-cck-\\', '%7Ba-guid%7D', 2 ] yield [ '2019-01-01', 'Missing', 'Ginko', '2.3.2', '', None, 'release', '%7Ba-guid%7D', 2 ] yield [ yesterday, 'FennecAndroid', # product name 'Ginko', # platform? '3.1415', # platform version '38.0', # product version '20150427090529', # build 'release', # update channel 'a-guid', # product guid 666 # count ] fake_hive.connect.return_value \ .cursor.return_value.__iter__ = return_test_data with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['fetch-adi-from-hive'] assert not information['fetch-adi-from-hive']['last_error'] config.logger.info.assert_called_with( 'Wrote 4 rows from doing hive query' ) fake_hive.connect.assert_called_with( database='default', authMechanism='PLAIN', host='localhost', user='******', password='******', port=10000, timeout=1800000, ) pgcursor = self.conn.cursor() columns = ( 'report_date', 'product_name', 'product_os_platform', 'product_os_version', 'product_version', 'build', 'build_channel', 'product_guid', 'count' ) pgcursor.execute( "select %s from raw_adi_logs" % ','.join(columns) ) adi_logs = [dict(zip(columns, row)) for row in pgcursor.fetchall()] expected = { 'report_date': yesterday, 'product_name': 'WinterWolf', 'product_os_platform': 'Ginko', 'product_os_version': '2.3.1', 'product_version': '10.0.4', 'build': 'nightly-ww3v20', 'build_channel': 'nightly', 'product_guid': 'a-guid', 'count': 1 } assert adi_logs[0] == expected expected = { 'report_date': yesterday, 'product_name': 'NothingMuch', 'product_os_platform': 'Ginko', 'product_os_version': '3.2.1', 'product_version': '10.0.4', 'build': 'release-ww3v20', 'build_channel': 'release-cck-blah', 'product_guid': '*****@*****.**', 'count': 1 } assert adi_logs[1] == expected expected = { 'report_date': datetime.date(2019, 1, 1), 'product_name': 'NothingMuch', 'product_os_platform': 'Ginko\xe2\x98\xa2', 'product_os_version': '2.3.2', 'product_version': '10.0.5a', 'build': 'release', 'build_channel': 'release-cck-\\', 'product_guid': '{a-guid}', 'count': 2 } assert adi_logs[2] == expected expected = { 'report_date': yesterday, 'product_name': 'FennecAndroid', 'product_os_platform': 'Ginko', 'product_os_version': '3.1415', 'product_version': '38.0', 'build': '20150427090529', 'build_channel': 'release', 'product_guid': 'a-guid', 'count': 666 } assert adi_logs[3] == expected columns = ( 'adi_count', 'date', 'product_name', 'product_os_platform', 'product_os_version', 'product_version', 'build', 'product_guid', 'update_channel', ) pgcursor.execute( """ select %s from raw_adi order by update_channel desc""" % ','.join(columns) ) adi = [dict(zip(columns, row)) for row in pgcursor.fetchall()] expected = { 'update_channel': 'release', 'product_guid': '{[email protected]}', 'product_version': '10.0.4', 'adi_count': 1, 'product_os_platform': 'Ginko', 'build': 'release-ww3v20', 'date': yesterday, 'product_os_version': '3.2.1', 'product_name': 'NothingMuch' } assert adi[0] == expected expected = { 'update_channel': 'nightly', 'product_guid': 'a-guid', 'product_version': '10.0.4', 'adi_count': 1, 'product_os_platform': 'Ginko', 'build': 'nightly-ww3v20', 'date': yesterday, 'product_os_version': '2.3.1', 'product_name': 'WinterWolf' } assert adi[1] == expected expected = { 'update_channel': 'beta', 'product_guid': 'a-guid', 'product_version': '38.0', 'adi_count': 666, 'product_os_platform': 'Ginko', 'build': '20150427090529', 'date': yesterday, 'product_os_version': '3.1415', 'product_name': 'FennecAndroid' } assert adi[2] == expected
def test_mocked_fetch_with_secondary_destination(self, fake_hive): class MockedPGConnectionContext: connection = mock.MagicMock() def __init__(self, config): self.config = config @contextlib.contextmanager def __call__(self): yield self.connection config_manager = self._setup_config_manager( overrides={ 'crontabber.class-FetchADIFromHiveCronApp.' 'secondary_destination.' 'database_class': MockedPGConnectionContext, } ) yesterday = ( datetime.datetime.utcnow() - datetime.timedelta(days=1) ).date() def return_test_data(fake): yield [ yesterday, 'WinterWolf', 'Ginko', '2.3.1', '10.0.4', 'nightly-ww3v20', 'nightly', 'a-guid', 1 ] fake_hive.connect.return_value \ .cursor.return_value.__iter__ = return_test_data with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['fetch-adi-from-hive'] assert not information['fetch-adi-from-hive']['last_error'] fake_hive.connect.assert_called_with( database='default', authMechanism='PLAIN', host='localhost', user='******', password='******', port=10000, timeout=1800000, ) # Critical test here. # We make sure the secondary database class gets used # for a `cursor.copy_from()` call. assert MockedPGConnectionContext.connection.cursor().copy_from.called
def test_scrape_json_releases(self, requests_mocker): today = utc_now() requests_mocker.get(BASE_URL + 'firefox/', text="""<html><body> <a href="candidates/">candidates</a> <a href="nightly/">nightly</a> </body></html>""") requests_mocker.get(BASE_URL + 'firefox/candidates/', text="""<html><body> <a href="28.0-candidates/">28.0-candidiates</a> <a href="10.0b4-candidates/">10.0b4-candidiates</a> <a href="None-candidates/">None-candidiates</a> </body></html>""") requests_mocker.get(BASE_URL + 'firefox/candidates/28.0-candidates/', text="""<html><body> <a href="build1/">build1</a> </body></html>""") requests_mocker.get(BASE_URL + 'firefox/candidates/28.0-candidates/build1/', text="""<html><body> <a href="linux-i686/">linux-i686</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/28.0-candidates/build1/linux-i686/', text="""<html><body> <a href="en-US/">en-US</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/28.0-candidates/build1/linux-i686/en-US/', text="""<html><body> <a href="firefox-28.0.json">firefox-28.0.json</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/28.0-candidates/build1/linux-i686/en-US/firefox-28.0.json', json={ 'buildid': '20140113161827', 'moz_app_maxversion': '28.0.*', 'moz_app_name': 'firefox', 'moz_app_vendor': 'Mozilla', 'moz_app_version': '28.0', 'moz_pkg_platform': 'linux-i686', 'moz_source_repo': 'http://hg.mozilla.org/releases/mozilla-release', 'moz_update_channel': 'release' }) requests_mocker.get(BASE_URL + 'firefox/candidates/10.0b4-candidates/', text="""<html><body> <a href="build1/">build1</a> </body></html>""") requests_mocker.get(BASE_URL + 'firefox/candidates/10.0b4-candidates/build1/', text="""<html><body> <a href="linux-i686/">linux-i686</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/10.0b4-candidates/build1/linux-i686/', text="""<html><body> <a href="en-US/">en-US</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/10.0b4-candidates/build1/linux-i686/en-US/', text="""<html><body> <a href="firefox-10.0b4.json">firefox-10.0b4.json</a> <a href="firefox-10.0b4.en-US.linux-i686.mozinfo.json"> firefox-10.0b4.en-US.linux-i686.mozinfo.json</a> <a href="JUNK.json">JUNK.json</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/10.0b4-candidates/build1/linux-i686/en-US/firefox-10.0b4.json', json={ 'buildid': '20140113161826', 'moz_app_maxversion': '10.0.*', 'moz_app_name': 'firefox', 'moz_app_vendor': 'Mozilla', 'moz_app_version': '27.0', 'moz_pkg_platform': 'linux-i686', 'moz_source_repo': 'http://hg.mozilla.org/releases/mozilla-beta', 'moz_update_channel': 'beta' }) # Ignore unrecognized JSON files, see bug 1065071 requests_mocker.get( BASE_URL + 'firefox/candidates/10.0b4-candidates/build1/linux-i686/en-US/JUNK.json', json={ 'something': 'unexpected', 'nothing': 'else' }) requests_mocker.get(BASE_URL + 'firefox/candidates/None-candidates/', text="""<html><body> <a href="build1/">build1</a> </body></html>""") requests_mocker.get(BASE_URL + 'firefox/candidates/None-candidates/build1/', text="""<html><body> <a href="linux-i686/">linux-i686</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/None-candidates/build1/linux-i686/', text="""<html><body> <a href="en-US/">en-US</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/None-candidates/build1/linux-i686/en-US/', text="""<html><body> <a href="None.json">None.json</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/None-candidates/build1/linux-i686/en-US/None.json', text=""" """) requests_mocker.get(BASE_URL + 'firefox/nightly/', text="""<html><body> <a href="%(year)s/">%(year)s</a> </body></html>""" % {'year': today.strftime('%Y')}) requests_mocker.get(today.strftime(BASE_URL + 'firefox/nightly/%Y/'), text="""<html><body> <a href="%(month)s/">%(month)s</a> </body></html>""" % {'month': today.strftime('%m')}) requests_mocker.get(today.strftime(BASE_URL + 'firefox/nightly/%Y/%m/'), text="""<html><body> <a href="%(date)s-03-02-03-mozilla-central/">txt</a> <a href="%(date)s-03-02-04-mozilla-central/">txt</a> """ % {'date': today.strftime('%Y-%m-%d')}) requests_mocker.get(today.strftime( BASE_URL + 'firefox/nightly/%Y/%m/%Y-%m-%d-03-02-03-mozilla-central/'), text="""<html><body> <a href="firefox-30.0a1.en-US.linux-i686.json">txt</a> </body></html>""") requests_mocker.get(today.strftime( BASE_URL + 'firefox/nightly/%Y/%m/%Y-%m-%d-03-02-03-mozilla-central/' + 'firefox-30.0a1.en-US.linux-i686.json'), json={ 'as': '$(CC)', 'buildid': '20140205030203', 'cc': '/usr/bin/ccache ', 'cxx': '/usr/bin/ccache stuff', 'host_alias': 'x86_64-unknown-linux-gnu', 'host_cpu': 'x86_64', 'host_os': 'linux-gnu', 'host_vendor': 'unknown', 'ld': 'ld', 'moz_app_id': '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}', 'moz_app_maxversion': '30.0a1', 'moz_app_name': 'firefox', 'moz_app_vendor': 'Mozilla', 'moz_app_version': '30.0a1', 'moz_pkg_platform': 'linux-i686', 'moz_source_repo': 'https://hg.mozilla.org/mozilla-central', 'moz_source_stamp': '1f170f9fead0', 'moz_update_channel': 'nightly', 'target_alias': 'i686-pc-linux', 'target_cpu': 'i686', 'target_os': 'linux-gnu', 'target_vendor': 'pc' }) requests_mocker.get(today.strftime( BASE_URL + 'firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/'), text="""<html><body> <a href="firefox-30.0a2.en-US.linux-i686.json">txt</a> </body></html>""") requests_mocker.get(today.strftime( BASE_URL + 'firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/' + 'firefox-30.0a2.en-US.linux-i686.json'), json={ 'as': '$(CC)', 'buildid': '20140205030204', 'cc': '/usr/bin/ccache stuff', 'cxx': '/usr/bin/ccache stuff', 'host_alias': 'x86_64-unknown-linux-gnu', 'host_cpu': 'x86_64', 'host_os': 'linux-gnu', 'host_vendor': 'unknown', 'ld': 'ld', 'moz_app_id': '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}', 'moz_app_maxversion': '30.0a2', 'moz_app_name': 'firefox', 'moz_app_vendor': 'Mozilla', 'moz_app_version': '30.0a2', 'moz_pkg_platform': 'linux-i686', 'moz_source_repo': 'https://hg.mozilla.org/mozilla-central', 'moz_source_stamp': '1f170f9fead0', 'moz_update_channel': 'nightly', 'target_alias': 'i686-pc-linux', 'target_cpu': 'i686', 'target_os': 'linux-gnu', 'target_vendor': 'pc' }) config_manager = self._setup_config_manager_firefox() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['ftpscraper'] assert not information['ftpscraper']['last_error'] assert information['ftpscraper']['last_success'] config.logger.warning.assert_any_call( 'Unable to JSON parse content %r', ' ', exc_info=True) config.logger.warning.assert_any_call( 'warning, unsupported JSON file: %s', BASE_URL + 'firefox/candidates/' '10.0b4-candidates/build1/linux-i686/en-US/JUNK.json') cursor = self.conn.cursor() columns = 'product_name', 'build_id', 'build_type' cursor.execute(""" select %s from releases_raw """ % ','.join(columns)) builds = [dict(zip(columns, row)) for row in cursor.fetchall()] build_ids = dict((str(x['build_id']), x) for x in builds) assert '20140113161827' in build_ids assert '20140113161826' in build_ids assert '20140205030203' in build_ids assert len(build_ids) == 4 expected = [{ 'build_id': 20140113161827, 'product_name': 'firefox', 'build_type': 'release' }, { 'build_id': 20140113161827, 'product_name': 'firefox', 'build_type': 'beta' }, { 'build_id': 20140113161826, 'product_name': 'firefox', 'build_type': 'beta' }, { 'build_id': 20140205030203, 'product_name': 'firefox', 'build_type': 'nightly' }, { 'build_id': 20140205030204, 'product_name': 'firefox', 'build_type': 'aurora' }] assert builds == expected
def test_scrape_json_releases(self): @responsify def mocked_get(url, today=None, timeout=None): if today is None: today = utc_now() html_wrap = "<html><body>\n%s\n</body></html>" if url.endswith('/firefox/'): return html_wrap % """ <a href="candidates/">candidates</a> <a href="nightly/">nightly</a> """ if url.endswith('/firefox/candidates/'): return html_wrap % """ <a href="28.0-candidates/">28.0-candidiates</a> <a href="10.0b4-candidates/">10.0b4-candidiates</a> <a href="None-candidates/">None-candidiates</a> """ if url.endswith('-candidates/'): return html_wrap % """ <a href="build1/">build1</a> """ if url.endswith('/build1/'): return html_wrap % """ <a href="linux-i686/">linux-i686</a> """ if url.endswith('/firefox/candidates/28.0-candidates/' 'build1/linux-i686/en-US/'): return html_wrap % """ <a href="firefox-28.0.json">firefox-28.0.json</a> """ if url.endswith('/firefox/candidates/10.0b4-candidates/' 'build1/linux-i686/en-US/'): return html_wrap % """ <a href="firefox-10.0b4.json">firefox-10.0b4.json</a> <a href="firefox-10.0b4.en-US.linux-i686.mozinfo.json"> firefox-10.0b4.en-US.linux-i686.mozinfo.json</a> <a href="JUNK.json"> JUNK.json</a> """ if url.endswith('/firefox/candidates/None-candidates/' 'build1/linux-i686/en-US/'): return html_wrap % """ <a href="None.json">None.json</a> """ if 'None.json' in url: return """ """ if 'firefox-28.0.json' in url: return """ { "buildid": "20140113161827", "moz_app_maxversion": "28.0.*", "moz_app_name": "firefox", "moz_app_vendor": "Mozilla", "moz_app_version": "28.0", "moz_pkg_platform": "linux-i686", "moz_source_repo": "http://hg.mozilla.org/releases/mozilla-release", "moz_update_channel": "release" } """ if 'firefox-10.0b4.json' in url: return """ { "buildid": "20140113161826", "moz_app_maxversion": "10.0.*", "moz_app_name": "firefox", "moz_app_vendor": "Mozilla", "moz_app_version": "27.0", "moz_pkg_platform": "linux-i686", "moz_source_repo": "http://hg.mozilla.org/releases/mozilla-beta", "moz_update_channel": "beta" } """ # Ignore unrecognized JSON files, see bug 1065071 if 'JUNK.json' in url: return """ { "something": "unexpected", "nothing": "else" } """ # Nightly tests for nightly and aurora builds if url.endswith('/firefox/nightly/'): return html_wrap % """ <a href="2014/">2014</a> """ if url.endswith(today.strftime('/firefox/nightly/%Y/%m/')): return html_wrap % """ <a href="%s-03-02-03-mozilla-central/">txt</a> <a href="%s-03-02-04-mozilla-central/">txt</a> """ % (today.strftime('%Y-%m-%d'), today.strftime('%Y-%m-%d')) if url.endswith(today.strftime('/firefox/nightly/%Y/%m/' '%Y-%m-%d-03-02-03-mozilla-central/')): return html_wrap % """ <a href="firefox-30.0a1.en-US.linux-i686.json">txt</a> """ if url.endswith(today.strftime( '/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/' )): return html_wrap % """ <a href="firefox-30.0a2.en-US.linux-i686.json">txt</a> """ if url.endswith(today.strftime( '/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/' 'firefox-30.0a2.en-US.linux-i686.json' )): return """ { "as": "$(CC)", "buildid": "20140205030204", "cc": "/usr/bin/ccache stuff", "cxx": "/usr/bin/ccache stuff", "host_alias": "x86_64-unknown-linux-gnu", "host_cpu": "x86_64", "host_os": "linux-gnu", "host_vendor": "unknown", "ld": "ld", "moz_app_id": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}", "moz_app_maxversion": "30.0a2", "moz_app_name": "firefox", "moz_app_vendor": "Mozilla", "moz_app_version": "30.0a2", "moz_pkg_platform": "linux-i686", "moz_source_repo": "https://hg.mozilla.org/mozilla-central", "moz_source_stamp": "1f170f9fead0", "moz_update_channel": "nightly", "target_alias": "i686-pc-linux", "target_cpu": "i686", "target_os": "linux-gnu", "target_vendor": "pc" } """ if url.endswith(today.strftime( '/firefox/nightly/%Y/%m/%Y-%m-%d-03-02-03-mozilla-central/' 'firefox-30.0a1.en-US.linux-i686.json' )): return """ { "as": "$(CC)", "buildid": "20140205030203", "cc": "/usr/bin/ccache ", "cxx": "/usr/bin/ccache stuff", "host_alias": "x86_64-unknown-linux-gnu", "host_cpu": "x86_64", "host_os": "linux-gnu", "host_vendor": "unknown", "ld": "ld", "moz_app_id": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}", "moz_app_maxversion": "30.0a1", "moz_app_name": "firefox", "moz_app_vendor": "Mozilla", "moz_app_version": "30.0a1", "moz_pkg_platform": "linux-i686", "moz_source_repo": "https://hg.mozilla.org/mozilla-central", "moz_source_stamp": "1f170f9fead0", "moz_update_channel": "nightly", "target_alias": "i686-pc-linux", "target_cpu": "i686", "target_os": "linux-gnu", "target_vendor": "pc" } """ raise NotImplementedError(url) self.mocked_session().get.side_effect = mocked_get config_manager = self._setup_config_manager_firefox() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['ftpscraper'] assert not information['ftpscraper']['last_error'] assert information['ftpscraper']['last_success'] config.logger.warning.assert_any_call( 'Unable to JSON parse content %r', ' ', exc_info=True ) base_url = config.crontabber['class-FTPScraperCronApp'].base_url config.logger.warning.assert_any_call( 'warning, unsupported JSON file: %s', base_url + 'firefox/candidates/' '10.0b4-candidates/build1/linux-i686/en-US/JUNK.json' ) cursor = self.conn.cursor() columns = 'product_name', 'build_id', 'build_type' cursor.execute(""" select %s from releases_raw """ % ','.join(columns)) builds = [dict(zip(columns, row)) for row in cursor.fetchall()] build_ids = dict((str(x['build_id']), x) for x in builds) assert '20140113161827' in build_ids assert '20140113161826' in build_ids assert '20140205030203' in build_ids assert len(build_ids) == 4 expected = [ { 'build_id': 20140113161827, 'product_name': 'firefox', 'build_type': 'release' }, { 'build_id': 20140113161827, 'product_name': 'firefox', 'build_type': 'beta' }, { 'build_id': 20140113161826, 'product_name': 'firefox', 'build_type': 'beta' }, { 'build_id': 20140205030203, 'product_name': 'firefox', 'build_type': 'nightly' }, { 'build_id': 20140205030204, 'product_name': 'firefox', 'build_type': 'aurora' } ] assert builds == expected