def updateCronjobsTable(connection, cronjobName, success, lastTargetTime, failureMessage=None): cursor = connection.cursor() params = [lastTargetTime] if success: params.append(utc_now()) sql = """ /* socorro.cron.dailyMatviews updateCronjobsTable */ UPDATE cronjobs SET last_target_time = %s, last_success = %s WHERE cronjob = %s """ else: params.append(utc_now()) params.append(failureMessage) sql = """ /* socorro.cron.dailyMatviews updateCronjobsTable */ UPDATE cronjobs SET last_target_time = %s, last_failure = %s, failure_message = %s WHERE cronjob = %s """ params.append(cronjobName) cursor.execute(sql, params) connection.commit()
def _set_ongoing_job(self, class_): app_name = class_.app_name info = self.job_state_database.get(app_name) if info: # Was it already ongoing? if info.get('ongoing'): # Unless it's been ongoing for ages, raise OngoingJobError age_hours = (utc_now() - info['ongoing']).seconds / 3600.0 if age_hours < self.config.crontabber.max_ongoing_age_hours: raise OngoingJobError(info['ongoing']) else: self.logger.debug( '{} has been ongoing for {:2} hours. Ignore it and running the app anyway.' .format(app_name, age_hours) ) info['ongoing'] = utc_now() else: info = { 'next_run': None, 'first_run': None, 'last_run': None, 'last_success': None, 'last_error': {}, 'error_count': 0, 'depends_on': [], 'ongoing': utc_now(), } self.job_state_database[app_name] = info
def _normal_jobs_iter(self): """ Yields a list of job tuples pulled from the 'jobs' table for which the owner is this process and the started datetime is null. This iterator is perpetual - it never raises the StopIteration exception """ get_normal_job_sql = ( "select" " j.id," " j.uuid," " priority " "from" " jobs j " "where" " j.owner = %d" " and j.starteddatetime is null " "order by queueddatetime" " limit %d" % (self.processor_id, self.config.batchJobLimit)) normal_jobs_list = [] last_query_timestamp = utc_now() while True: polling_threshold = utc_now() - self.config.pollingInterval if not normal_jobs_list and \ last_query_timestamp < polling_threshold: # get more normal_jobs_list = self.transaction( execute_query_fetchall, get_normal_job_sql ) last_query_timestamp = utc_now() if normal_jobs_list: while normal_jobs_list: yield normal_jobs_list.pop(0) else: yield None
def test_sending_many_emails(self, exacttarget_mock): """Test that we can send emails to a lot of users in the same run. """ # First add a lot of emails. now = utc_now() - datetime.timedelta(minutes=30) config_manager = self._setup_storage_config() with config_manager.context() as config: storage = ElasticSearchCrashStorage(config) for i in range(21): storage.save_processed({ 'uuid': 'fake-%s' % i, 'email': '*****@*****.**' % i, 'product': 'WaterWolf', 'version': '20.0', 'release_channel': 'Release', 'date_processed': now, }) storage.es.refresh() config_manager = self._setup_simple_config() with config_manager.context() as config: job = automatic_emails.AutomaticEmailsCronApp(config, '') job.run(utc_now()) et_mock = exacttarget_mock.return_value # Verify that we have the default 4 results + the 21 we added. self.assertEqual(et_mock.trigger_send.call_count, 25)
def test_no_new_crashes(self): new_crash_source = ESNewCrashSource(self.config) self.health_check() generator = new_crash_source.new_crashes( utc_now() - datetime.timedelta(days=1), 'Firefox', ['43.0.1'] ) assert list(generator) == [] self.index_crash( a_processed_crash, raw_crash=a_raw_crash, crash_id=a_processed_crash['uuid'] ) self.refresh_index() # Same test now that there is a processed crash in there # but notably under a different name and version. generator = new_crash_source.new_crashes( utc_now() - datetime.timedelta(days=1), 'Firefox', ['43.0.1'] ) assert list(generator) == []
def time_to_run(self, class_, time_): """return true if it's time to run the job. This is true if there is no previous information about its last run or if the last time it ran and set its next_run to a date that is now past. """ app_name = class_.app_name try: info = self.database[app_name] except KeyError: if time_: h, m = [int(x) for x in time_.split(':')] # only run if this hour and minute is < now now = utc_now() if now.hour > h: return True elif now.hour == h and now.minute >= m: return True return False else: # no past information, run now return True next_run = info['next_run'] if next_run < utc_now(): return True return False
def test_slow_run_job(self): config_manager, json_file = self._setup_config_manager( 'socorro.unittest.cron.test_crontabber.SlowJob|1h' ) with config_manager.context() as config: tab = crontabber.CronTabber(config) time_before = utc_now() tab.run_all() time_after = utc_now() time_taken = (time_after - time_before).seconds #time_taken = (time_after - time_before).microseconds / 1000.0 / 1000.0 #print time_taken self.assertEqual(round(time_taken), 1.0) # check that this was written to the JSON file # and that the next_run is going to be 1 day from now assert os.path.isfile(json_file) structure = json.load(open(json_file)) information = structure['slow-job'] self.assertEqual(information['error_count'], 0) self.assertEqual(information['last_error'], {}) self.assertTrue(information['next_run'].startswith( (time_before + datetime.timedelta(hours=1)) .strftime('%Y-%m-%d %H:%M:%S')))
def run(self): # if this is non-zero, we use it. if self.config.days_into_past: last_run = ( utc_now() - datetime.timedelta(days=self.config.days_into_past) ) else: try: # KeyError if it's never run successfully # TypeError if self.job_information is None last_run = self.job_information['last_success'] except (KeyError, TypeError): # basically, the "virgin run" of this job last_run = utc_now() # bugzilla runs on PST, so we need to communicate in its time zone PST = tz.gettz('PST8PDT') last_run_formatted = last_run.astimezone(PST).strftime('%Y-%m-%d') query = self.config.query % last_run_formatted for ( bug_id, signature_set ) in self._iterator(query): try: # each run of this loop is a transaction self.database_transaction_executor( self.inner_transaction, bug_id, signature_set ) except NothingUsefulHappened: pass
def test_delete_old_indices(self): # Create old indices to be deleted. self.index_client.create('socorro200142', {}) self.indices.append('socorro200142') self.index_client.create('socorro200000', {}) self.indices.append('socorro200000') # Create an old aliased index. self.index_client.create('socorro200201_20030101', {}) self.indices.append('socorro200201_20030101') self.index_client.put_alias( index='socorro200201_20030101', name='socorro200201', ) # Create a recent aliased index. last_week_index = self.get_index_for_date( utc_now() - datetime.timedelta(weeks=1) ) self.index_client.create('socorro_some_aliased_index', {}) self.indices.append('socorro_some_aliased_index') self.index_client.put_alias( index='socorro_some_aliased_index', name=last_week_index, ) # Create a recent index that should not be deleted. now_index = self.get_index_for_date(utc_now()) self.index_client.create(now_index, {}) self.indices.append(now_index) # These will raise an error if an index was not correctly created. assert self.index_client.exists('socorro200142') assert self.index_client.exists('socorro200000') assert self.index_client.exists('socorro200201') assert self.index_client.exists(now_index) assert self.index_client.exists(last_week_index) api = IndexCleaner(self.config) api.delete_old_indices() # Verify the recent index is still there. ok_(self.index_client.exists(now_index)) ok_(self.index_client.exists(last_week_index)) # Verify the old indices are gone. ok_(not self.index_client.exists('socorro200142')) ok_(not self.index_client.exists('socorro200000')) ok_(not self.index_client.exists('socorro200201'))
def test_run(self): config_manager = self._setup_config_manager() with config_manager.context() as config: tab = CronTabber(config) tab.run_all() information = self._load_structure() assert information['suspicious-crashes'] assert not information['suspicious-crashes']['last_error'] assert information['suspicious-crashes']['last_success'] cursor = self.conn.cursor() cursor.execute(""" SELECT signatures.signature, scs.report_date FROM suspicious_crash_signatures scs JOIN signatures ON scs.signature_id=signatures.signature_id """) count = 0 today = (utc_now() - datetime.timedelta(1)).date() for row in cursor.fetchall(): eq_('sig', row[0]) eq_(today, row[1].date()) count += 1 eq_(1, count)
def transferOne( self, ooid, anotherJsonDumpStorage, createLinks=True, removeOld=False, webheadHostName=None, aDate=None ): """ Transfer data from another JsonDumpStorage instance into this instance of JsonDumpStorage ooid - the id of the data to transfer anotherJsonDumpStorage - An instance of JsonDumpStorage holding the data to be transferred createLinks - If true, create symlinks from and to date subdir removeOld - If true, attempt to delete the files and symlinks in source file tree webheadHostName: Used if known aDate: Used if unable to parse date from source directories and uuid NOTE: Assumes that the path names and suffixes for anotherJsonDumpStorage are the same as for self """ self.logger.debug("%s - transferOne %s %s", threading.currentThread().getName(), ooid, aDate) jsonFromFile = anotherJsonDumpStorage.getJson(ooid) self.logger.debug("%s - fetched json", threading.currentThread().getName()) dumpFromFile = os.path.splitext(jsonFromFile)[0] + anotherJsonDumpStorage.dumpSuffix if createLinks: self.logger.debug("%s - fetching stamp", threading.currentThread().getName()) stamp = anotherJsonDumpStorage.pathToDate(anotherJsonDumpStorage.lookupOoidInDatePath(None, ooid, None)[0]) else: self.logger.debug("%s - not bothering to fetch stamp", threading.currentThread().getName()) stamp = None self.logger.debug("%s - fetched pathToDate ", threading.currentThread().getName()) if not stamp: if not aDate: aDate = utc_now() stamp = aDate self.logger.debug("%s - about to copyFrom ", threading.currentThread().getName()) self.copyFrom(ooid, jsonFromFile, dumpFromFile, webheadHostName, stamp, createLinks, removeOld)
def test_test_mapping(self): """Much test. So meta. Wow test_test_. """ # First test a valid mapping. mapping = self.api.get_mapping() assert self.api.test_mapping(mapping) is None # Insert an invalid storage mapping. mapping = self.api.get_mapping({ 'name': 'fake_field', 'namespace': 'raw_crash', 'in_database_name': 'fake_field', 'storage_mapping': { 'type': 'unkwown' } }) with pytest.raises(BadArgumentError): self.api.test_mapping(mapping) # Test with a correct mapping but with data that cannot be indexed. self.index_crash({ 'date_processed': datetimeutil.utc_now(), 'product': 'WaterWolf', }) self.refresh_index() mapping = self.api.get_mapping({ 'name': 'product', 'storage_mapping': { 'type': 'long' } }) with pytest.raises(BadArgumentError): self.api.test_mapping(mapping)
def setUp(self): super(IntegrationTestSettings, self).setUp() config = self.get_config_context() self.storage = crashstorage.ElasticSearchCrashStorage(config) # clear the indices cache so the index is created on every test self.storage.indices_cache = set() self.now = utc_now() # Create the supersearch fields. self.storage.es.bulk_index( index=config.webapi.elasticsearch_default_index, doc_type='supersearch_fields', docs=SUPERSEARCH_FIELDS.values(), id_field='name', refresh=True, ) # Create the index that will be used. es_index = self.storage.get_index_for_crash(self.now) self.storage.create_socorro_index(es_index) self.api = SuperSearch(config=config)
def fillProcessorTable(cursor, processorCount, stamp=None, processorMap = {},logger = None): """ Puts some entries into the processor table. Also creates priority_jobs_NNN for each processor id, unless that table exists Given a map of id->timestamp, sets the lastseendatetime for each successive processor to that stamp (Ignores ids generated by the count or in the processorMap, and uses database's serial id generator) """ if not logger: logger = logging.getLogger() if not stamp: stamp = utc_now() if not processorCount and not processorMap: return sql = "INSERT INTO processors (name,startdatetime,lastseendatetime) VALUES (%s,%s,%s);" data = [] if processorMap: data.extend([('test_%d'%(id),stamp,processorMap.get(id,stamp)) for id in processorMap.keys() ]) else: data.extend([('test_%d'%(x),stamp, stamp) for x in range(1,processorCount+1) ]) try: cursor.executemany(sql,data) cursor.connection.commit() sql = "SELECT id from processors;" cursor.execute(sql) allIds = cursor.fetchall() cursor.connection.rollback() sql = "CREATE TABLE priority_jobs_%s (uuid varchar(50) not null primary key);" for tup in allIds: try: cursor.execute(sql%(tup[0])) cursor.connection.commit() except psycopg2.ProgrammingError: cursor.connection.rollback() finally: cursor.connection.rollback()
def test_update_user(self): config_manager = self._setup_simple_config() with config_manager.context() as config: job = automatic_emails.AutomaticEmailsCronApp(config, '') now = utc_now() report = { 'email': '*****@*****.**' } job.update_user(report, now, self.conn) cursor = self.conn.cursor() cursor.execute(""" SELECT last_sending FROM emails WHERE email=%(email)s """, report) self.assertEqual(cursor.rowcount, 1) row = cursor.fetchone() self.assertEqual(row[0], now) # Test with a non-existing user report = { 'email': '*****@*****.**' } job.update_user(report, now, self.conn) cursor = self.conn.cursor() cursor.execute(""" SELECT last_sending FROM emails WHERE email=%(email)s """, report) self.assertEqual(cursor.rowcount, 1) row = cursor.fetchone() self.assertEqual(row[0], now)
def test_create_release(self): self._insert_release_channels() self._insert_products() config_manager = self._setup_config_manager() with config_manager.context() as config: app = middleware_app.MiddlewareApp(config) app.main() server = middleware_app.application now = datetimeutil.utc_now() response = self.post( server, "/releases/release/", { "product": "Firefox", "version": "1.0", "update_channel": "beta", "build_id": now.strftime("%Y%m%d%H%M"), "platform": "Windows", "beta_number": "1", "release_channel": "Beta", "throttle": "1", }, ) eq_(response.data, True)
def get_signatures(self, **kwargs): """Return top crashers by signatures. See http://socorro.readthedocs.org/en/latest/middleware.html#tcbs """ filters = [ ("product", None, "str"), ("version", None, "str"), ("crash_type", "all", "str"), ("to_date", datetimeutil.utc_now(), "datetime"), ("duration", datetime.timedelta(7), "timedelta"), ("os", None, "str"), ("limit", 100, "int"), ("date_range_type", None, "str") ] params = external_common.parse_arguments(filters, kwargs) params.logger = logger try: connection = self.database.connection() cursor = connection.cursor() return tcbs.twoPeriodTopCrasherComparison(cursor, params) finally: connection.close()
def get_signatures(self, **kwargs): """Return top crashers by signatures. See http://socorro.readthedocs.org/en/latest/middleware.html#tcbs """ filters = [ ("product", None, "str"), ("version", None, "str"), ("crash_type", "all", "str"), ("to_date", datetimeutil.utc_now(), "datetime"), ("duration", datetime.timedelta(7), "timedelta"), ("os", None, "str"), ("limit", 100, "int"), ("date_range_type", None, "str") ] params = external_common.parse_arguments(filters, kwargs) params.logger = logger # what the twoPeriodTopCrasherComparison() function does is that it # makes a start date from taking the to_date - duration if params.duration > datetime.timedelta(30): raise BadArgumentError('Duration too long. Max 30 days.') with self.get_connection() as connection: return tcbs.twoPeriodTopCrasherComparison(connection, params)
def test_cleanup_radix(self): self.fsrts._current_slot = lambda: ["00", "00_00"] self.fsrts.save_raw_crash({"test": "TEST"}, {"foo": "bar", self.fsrts.config.dump_field: "baz"}, self.CRASH_ID) self.fsrts._current_slot = lambda: ["10", "00_01"] self.assertEqual(list(self.fsrts.new_crashes()), [self.CRASH_ID]) self.assertEqual(list(self.fsrts.new_crashes()), []) config_manager, json_file = self._setup_config_manager() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() self.assertEqual(os.listdir(self.fsrts.config.fs_root), []) future = (utc_now() + datetime.timedelta(days=10)).strftime("%Y%m%d") future_id = "0bba929f-8721-460c-dead-a43c%s" % future self.fsrts._current_slot = lambda: ["00", "00_00"] self.fsrts.save_raw_crash({"test": "TEST"}, {"foo": "bar", self.fsrts.config.dump_field: "baz"}, future_id) self.fsrts._current_slot = lambda: ["10", "00_01"] self.assertEqual(list(self.fsrts.new_crashes()), [future_id]) self.assertEqual(list(self.fsrts.new_crashes()), []) tab.run_all() self.assertEqual(os.listdir(self.fsrts.config.fs_root), [future])
def mocked_urlopener(url, today=None): if today is None: today = utc_now() html_wrap = "<html><body>\n%s\n</body></html>" if url.endswith('/firefox/'): return html_wrap % """ <a href="candidates/">candidates</a> <a href="nightly/">nightly</a> """ if url.endswith('/firefox/nightly/'): return html_wrap % """ <a href="10.0-candidates/">10.0-candidiates</a> """ if url.endswith('/firefox/candidates/'): return html_wrap % """ <a href="10.0b4-candidates/">10.0b4-candidiates</a> """ if (url.endswith('/firefox/nightly/10.0-candidates/') or url.endswith('/firefox/candidates/10.0b4-candidates/')): return html_wrap % """ <a href="build1/">build1</a> """ if (url.endswith('/firefox/nightly/10.0-candidates/build1/') or url.endswith('/firefox/candidates/10.0b4-candidates/build1/')): return html_wrap % """ <a href="linux_info.txt">linux_info.txt</a> """ if url.endswith(today.strftime('/firefox/nightly/%Y/%m/')): return html_wrap % today.strftime(""" <a href="%Y-%m-%d-trunk/">%Y-%m-%d-trunk</a> """) if url.endswith(today.strftime( '/firefox/nightly/%Y/%m/%Y-%m-%d-trunk/')): return html_wrap % """ <a href="mozilla-nightly-15.0a1.en-US.linux-x86_64.txt">txt</a> <a href="mozilla-nightly-15.0a2.en-US.linux-x86_64.txt">txt</a> """ if url.endswith(today.strftime( '/firefox/nightly/%Y/%m/%Y-%m-%d-trunk/mozilla-nightly-15.0a1.en' '-US.linux-x86_64.txt')): return ( "20120505030510\n" "http://hg.mozilla.org/mozilla-central/rev/0a48e6561534" ) if url.endswith(today.strftime( '/firefox/nightly/%Y/%m/%Y-%m-%d-trunk/mozilla-nightly-15.0a2.en' '-US.linux-x86_64.txt')): return ( "20120505443322\n" "http://hg.mozilla.org/mozilla-central/rev/xxx123" ) if url.endswith( '/firefox/nightly/10.0-candidates/build1/linux_info.txt'): return "buildID=20120516113045" if url.endswith( '/firefox/candidates/10.0b4-candidates/build1/linux_info.txt'): return "buildID=20120516114455" # bad testing boy! raise NotImplementedError(url)
def __init__(self,*args,**kwargs): """ Passes appropriate kwargs to Config, pays local attention to these keys: updateInterval: default: '0' format: 'dd:hh:mm:ss', leading parts optional. Must be >= 0 seconds. updateFunction: default: noop(). Takes self as argument. Behavior: Updates default values in argument reEvaluateFunction: default: noop(). Takes self as argument. Behavior: Mutates values in argument signalNumber: default: SIGALRM (14). If 0, then signals will not be handled. Instances that share the same signalNumber will all be update()-ed at every signal. self.internal.updateFunction may be set after construction if desired: Avoids double-work at construction self.internal.reEvalutateFunction may be set after construction if desired, but this is not recommended. """ skwargs = dict([(x,kwargs[x]) for x in socorro_config.getDefaultedConfigOptions().keys() if x in kwargs]) for i in range(len(args)): skwargs[socorro_config.namedConfigOptions[i]] = args[i] super(DynamicConfig,self).__init__(**skwargs) self.internal.updateFunction = kwargs.get('updateFunction',noop) self.internal.reEvaluateFunction = kwargs.get('reEvaluateFunction',noop) self.internal.signalNumber = kwargs.get('signalNumber',14) self.internal.nextUpdate = None updateInterval = kwargs.get('updateInterval','0:0:0:0') self.internal.updateDelta = socorro_config.timeDeltaConverter(updateInterval) if self.internal.updateDelta: if self.internal.updateDelta < datetime.timedelta(0): raise ValueError("updateInterval must be non-negative, but %s"%self.internal.updateDelta) self.internal.nextUpdate = utc_now() + self.internal.updateDelta # finally: make sure we are current if self.internal.signalNumber: priorSignal = signal.signal(self.internal.signalNumber,DynamicConfig.handleAlarm) self.doUpdate() DynamicConfig.instances[id(self)] = self
def test_create_release_with_beta_number_null(self): self._insert_release_channels() service = Releases(config=self.config) now = datetimeutil.utc_now() build_id = now.strftime('%Y%m%d%H%M') params = dict( product='Firefox', version='1.0', update_channel='beta', build_id=build_id, platform='Windows', beta_number=None, release_channel='Beta', throttle=1 ) res = service.create_release(**params) ok_(res) # but... params['beta_number'] = 0 assert_raises( MissingArgumentError, service.create_release, **params )
def test_basic_run(self): cur = self.conn.cursor() # Ensure test table is present. statement = """ INSERT INTO raw_adi (date, product_name, adi_count) VALUES (%(first)s, 'WinterFox', 11), (%(second)s, 'WinterFox', 23) """ second = utc_now().date() first = second - datetime.timedelta(days=1) cur.execute(statement, {'first': first, 'second': second}) self.conn.commit() # Run the crontabber job to remove the test table. config_manager = self._setup_config_manager(days_to_keep=1) with config_manager.context() as config: tab = CronTabber(config) tab.run_all() # Basic assertion test of stored procedure. information = self._load_structure() assert information['clean-raw-adi'] assert not information['clean-raw-adi']['last_error'] assert information['clean-raw-adi']['last_success'] # Ensure test row was removed cur.execute(""" SELECT date FROM raw_adi """) result, = cur.fetchall() report_date = result[0] eq_(report_date, second)
def test_update_user(self): config_manager = self._setup_simple_config() with config_manager.context() as config: job = automatic_emails.AutomaticEmailsCronApp(config, '') now = utc_now().isoformat() es = SuperS().es( urls=config.elasticsearch.elasticsearch_urls, timeout=config.elasticsearch.elasticsearch_timeout, ) search = es.indexes( config.elasticsearch.elasticsearch_emails_index ) search = search.doctypes('emails') connection = es.get_es() job.update_user('*****@*****.**', now, connection) connection.refresh() s = search.filter(_id='*****@*****.**') res = list(s.values_list('last_sending')) self.assertEqual(len(res), 1) self.assertEqual(res[0][0], now) # Test with a non-existing user job.update_user('*****@*****.**', now, connection) connection.refresh() s = search.filter(_id='*****@*****.**') res = list(s.values_list('last_sending')) self.assertEqual(len(res), 1) self.assertEqual(res[0][0], now)
def POST(self, *args): raw_crash, dumps = self._get_raw_crash_from_form() current_timestamp = utc_now() raw_crash.submitted_timestamp = current_timestamp.isoformat() # legacy - ought to be removed someday raw_crash.timestamp = time.time() if (not self.config.accept_submitted_crash_id or 'crash_id' not in raw_crash ): crash_id = createNewOoid(current_timestamp) raw_crash.crash_id = crash_id self.logger.info('%s received', crash_id) else: crash_id = raw_crash.crash_id self.logger.info('%s received with existing crash_id:', crash_id) raw_crash.type_tag = self.type_tag self.crash_storage.save_raw_crash( raw_crash, dumps, crash_id ) self.logger.info('%s accepted', crash_id) return "CrashID=%s%s\n" % (self.type_tag, crash_id)
def testLookupOoidInDatePath(self): d = dumpStorage.DumpStorage(self.testDir) expected = {} count = 0 for ooid,v in createJDS.jsonFileData.items(): dateS = v[0] if 0 == count%2: nd,dd = d.newEntry(ooid,datetime.datetime(*[int(x) for x in dateS.split('-')], tzinfo=UTC)) expected[ooid] = dd elif 0 == count%5: expected[ooid] = None pass else: nd,dd = d.newEntry(ooid) expected[ooid] = dd count += 1 dateS = v[0] count = 0 for ooid in createJDS.jsonFileData.keys(): dateS = v[0] if expected[ooid]: exEnd = datetime.datetime(*[int(x) for x in dateS.split('-')], tzinfo=UTC) passDate = utc_now() if 0 == count%3: passDate = None else: passDate = exEnd got,ignore = d.lookupOoidInDatePath(passDate,ooid) assert expected[ooid] == got, 'For %s: Expected %s, got %s'%(ooid,expected[ooid],got)
def POST(self, *args): raw_crash, dumps = \ self._make_raw_crash_and_dumps(web.webapi.rawinput()) current_timestamp = utc_now() raw_crash.submitted_timestamp = current_timestamp.isoformat() # legacy - ought to be removed someday raw_crash.timestamp = time.time() crash_id = createNewOoid(current_timestamp) raw_crash.legacy_processing = self.legacy_throttler.throttle(raw_crash) if raw_crash.legacy_processing == LegacyThrottler.DISCARD: self.logger.info('%s discarded', crash_id) return "Discarded=1\n" if raw_crash.legacy_processing == LegacyThrottler.IGNORE: self.logger.info('%s ignored', crash_id) return "Unsupported=1\n" crash_storage = self.context.crashStoragePool.crashStorage() try: crash_storage.save_raw( crash_id, raw_crash, dumps ) except PolyStorageError, x: self.logger.error('%s storage exception: %s', crash_id, str(x.exceptions), # log internal error set exc_info=True) raise
def test_create_release(self): self._insert_release_channels() self._insert_products() config_manager = self._setup_config_manager() with config_manager.context() as config: app = middleware_app.MiddlewareApp(config) app.main() server = middleware_app.application now = datetimeutil.utc_now() response = self.post( server, '/releases/release/', { 'product': 'Firefox', 'version': '1.0', 'update_channel': 'beta', 'build_id': now.strftime('%Y%m%d%H%M'), 'platform': 'Windows', 'beta_number': '1', 'release_channel': 'Beta', 'throttle': '1' } ) eq_(response.data, True)
def test_mapping(self, mapping): """Verify that a mapping is correct. This function does so by first creating a new, temporary index in elasticsearch using the mapping. It then takes some recent crash reports that are in elasticsearch and tries to insert them in the temporary index. Any failure in any of those steps will raise an exception. If any is raised, that means the mapping is incorrect in some way (either it doesn't validate against elasticsearch's rules, or is not compatible with the data we currently store). If no exception is raised, the mapping is likely correct. This function is to be used in any place that can change the `storage_mapping` field in any Super Search Field. Methods `create_field` and `update_field` use it, see above. """ temp_index = 'socorro_mapping_test' es_connection = self.get_connection() # Import at runtime to avoid dependency circle. from socorro.external.es.index_creator import IndexCreator index_creator = IndexCreator(self.config) try: index_creator.create_index( temp_index, mapping, ) now = datetimeutil.utc_now() last_week = now - datetime.timedelta(days=7) current_indices = self.generate_list_of_indexes(last_week, now) crashes_sample = es_connection.search( index=current_indices, doc_type=self.config.elasticsearch.elasticsearch_doctype, size=self.config.elasticsearch.mapping_test_crash_number, ) crashes = [x['_source'] for x in crashes_sample['hits']['hits']] for crash in crashes: es_connection.index( index=temp_index, doc_type=self.config.elasticsearch.elasticsearch_doctype, body=crash, ) except elasticsearch.exceptions.ElasticsearchException as e: raise BadArgumentError( 'storage_mapping', msg='Indexing existing data in Elasticsearch failed with the ' 'new mapping. Error is: %s' % str(e), ) finally: try: index_creator.get_index_client().delete(temp_index) except elasticsearch.exceptions.NotFoundError: # If the index does not exist (if the index creation failed # for example), we don't need to do anything. pass
def testSaveCampaign(): context = getDummyContext() product = 'Foobar' versions = '5' signature = 'JohnHancock' subject = 'email subject' body = 'email body' start_date = utc_now() end_date = start_date + timedelta(hours=1) author = '*****@*****.**' email_count = 0 parameters = (product, versions, signature, subject, body, start_date, end_date, email_count, author) sql = """INSERT INTO email_campaigns (product, versions, signature, subject, body, start_date, end_date, email_count, author) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""" dummyCursor = expect.DummyObjectWithExpectations() dummyCursor.expect('mogrify', (sql, list(parameters)), {}, None) dummyCursor.expect('execute', (sql, list(parameters)), {}, None) dummyCursor.expect('fetchone', (), {}, ['123']) campaign = ecc.EmailCampaignCreate(context) campaignId = campaign.save_campaign(dummyCursor, product, versions, signature, subject, body, start_date, end_date, author) assert campaignId == '123'
def process_crash(self, raw_crash, raw_dumps, processed_crash): """Take a raw_crash and its associated raw_dumps and return a processed_crash. """ # processor_meta_data will be used to ferry "inside information" to # transformation rules. Sometimes rules need a bit more extra # information about the transformation process itself. processor_meta_data = DotDict() processor_meta_data.processor_notes = [ self.config.processor_name, self.__class__.__name__ ] processor_meta_data.quit_check = self.quit_check processor_meta_data.processor = self processor_meta_data.config = self.config if "processor_notes" in processed_crash: original_processor_notes = [ x.strip() for x in processed_crash.processor_notes.split(";") ] processor_meta_data.processor_notes.append( "earlier processing: %s" % processed_crash.get("started_datetime", 'Unknown Date')) else: original_processor_notes = [] processed_crash.success = False processed_crash.started_datetime = utc_now() # for backwards compatibility: processed_crash.startedDateTime = processed_crash.started_datetime processed_crash.signature = 'EMPTY: crash failed to process' crash_id = raw_crash['uuid'] try: # quit_check calls ought to be scattered around the code to allow # the processor to be responsive to requests to shut down. self.quit_check() start_time = self.config.logger.info( "starting transform for crash: %s", crash_id) processor_meta_data.started_timestamp = start_time # apply_all_rules for rule in self.rules: rule.act(raw_crash, raw_dumps, processed_crash, processor_meta_data) self.quit_check() # the crash made it through the processor rules with no exceptions # raised, call it a success. processed_crash.success = True except Exception as exception: self.config.logger.warning('Error while processing %s: %s', crash_id, str(exception), exc_info=True) processor_meta_data.processor_notes.append( 'unrecoverable processor error: %s' % exception) # the processor notes are in the form of a list. Join them all # together to make a single string processor_meta_data.processor_notes.extend(original_processor_notes) processed_crash.processor_notes = '; '.join( processor_meta_data.processor_notes) completed_datetime = utc_now() processed_crash.completed_datetime = completed_datetime # for backwards compatibility: processed_crash.completeddatetime = completed_datetime self.config.logger.info( "finishing %s transform for crash: %s", 'successful' if processed_crash.success else 'failed', crash_id) return processed_crash
def get_parameters(kwargs): """ Return a dictionary of parameters with default values. Optional arguments: data_type -- Type of data to return. Default is None, to be determined by each service if needed. terms -- Terms to search for. Can be a string or a list of strings. Default is none. fields -- Fields to search into. Can be a string or a list of strings. Default to signature, not implemented for PostgreSQL. search_mode -- How to search for terms. Must be one of the following: "default", "contains", "is_exactly" or "starts_with". Default to "default" for ElasticSearch, "starts_with" for PostgreSQL. from_date -- Only elements after this date. Format must be "YYYY-mm-dd HH:ii:ss.S". Default is a week ago. to_date -- Only elements before this date. Format must be "YYYY-mm-dd HH:ii:ss.S". Default is now. products -- Products concerned by this search. Can be a string or a list of strings. Default is Firefox. os -- Restrict search to those operating systems. Can be a string or a list of strings. Default is all. versions -- Version of the software. Can be a string or a list of strings. Default is all. build_ids -- Restrict search to a particular build of the software. Can be a string or a list of strings. Default is all. reasons -- Restrict search to crashes caused by this reason. Default is all. release_channels -- Restrict search to crashes in this release channels. Default is all. report_type -- Retrict to a type of report. Can be any, crash or hang. Default is any. report_process -- How was the report processed. Can be any, crash or hang. Default is any. plugin_terms -- Search for terms concerning plugins. Can be a string or a list of strings. Default is none. plugin_in -- What field to look into. Can be "name" or "filename". Default is 'name'. plugin_search_mode -- How to search into plugins. Must be one of the following: "contains", "is_exactly" or "starts_with". Default to "contains". result_number -- Number of results to get. Default is 100. result_offset -- Get results from this offset. Default is 0. """ # Default dates now = datetimeutil.utc_now() lastweek = now - datetime.timedelta(7) filters = [("data_type", "signatures", "str"), ("terms", None, ["list", "str"]), ("signature", None, "str"), ("fields", "signature", ["list", "str"]), ("search_mode", "default", "str"), ("from_date", lastweek, "datetime"), ("to_date", now, "datetime"), ("products", None, ["list", "str"]), ("versions", None, ["list", "str"]), ("os", None, ["list", "str"]), ("reasons", None, ["list", "str"]), ("release_channels", None, ["list", "str"]), ("build_ids", None, ["list", "str"]), ("build_from", lastweek, "datetime"), ("build_to", now, "datetime"), ("report_process", "any", "str"), ("report_type", "any", "str"), ("plugin_terms", None, ["list", "str"]), ("plugin_in", "name", ["list", "str"]), ("plugin_search_mode", "default", "str"), ("result_number", 100, "int"), ("result_offset", 0, "int")] params = extern.parse_arguments(filters, kwargs) # To be moved into a config file? authorized_modes = ["default", "starts_with", "contains", "is_exactly"] if params["search_mode"] not in authorized_modes: params["search_mode"] = "default" if params["plugin_search_mode"] not in authorized_modes: params["plugin_search_mode"] = "default" # Do not search in the future and make sure we have dates where expected if params["to_date"] is None or params["to_date"] > now: params["to_date"] = now if params["from_date"] is None: params["from_date"] = lastweek if params["build_to"] is None or params["build_to"] > now: params["build_to"] = now if params["build_from"] is None: params["build_from"] = lastweek # Securing fields params['fields'] = restrict_fields(params['fields'], ['signature', 'dump']) params['plugin_in'] = restrict_fields(params['plugin_in'], ['filename', 'name']) return params
def test_email_after_delay(self, exacttarget_mock): """Test that a user will receive an email if he or she sends us a new crash report after the delay is passed (but not before). """ config_manager = self._setup_config_manager( delay_between_emails=1, restrict_products=['EarthRaccoon'] ) email = '*****@*****.**' list_service_mock = exacttarget_mock.return_value.list.return_value list_service_mock.get_subscriber.return_value = { 'token': email } trigger_send_mock = exacttarget_mock.return_value.trigger_send tomorrow = utc_now() + datetime.timedelta(days=1, hours=2) twohourslater = utc_now() + datetime.timedelta(hours=2) storage_config_manager = self._setup_storage_config() with storage_config_manager.context() as storage_config: storage = ElasticSearchCrashStorage(storage_config) with config_manager.context() as config: # 1. Send an email to the user and update emailing data tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['automatic-emails'] assert not information['automatic-emails']['last_error'] assert information['automatic-emails']['last_success'] exacttarget_mock.return_value.trigger_send.assert_called_with( 'socorro_dev_test', { 'EMAIL_ADDRESS_': email, 'EMAIL_FORMAT_': 'H', 'TOKEN': email } ) self.assertEqual(trigger_send_mock.call_count, 1) # 2. Test that before 'delay' is passed user doesn't receive # another email # Insert a new crash report with the same email address storage.save_processed({ 'uuid': '50', 'email': email, 'product': 'EarthRaccoon', 'version': '20.0', 'release_channel': 'Release', 'date_processed': utc_now() + datetime.timedelta(hours=1) }) storage.es.refresh() # Run crontabber with time pushed by two hours with mock.patch('socorro.cron.crontabber.utc_now') as cronutc_mock: with mock.patch('socorro.cron.base.utc_now') as baseutc_mock: cronutc_mock.return_value = twohourslater baseutc_mock.return_value = twohourslater tab.run_all() information = self._load_structure() assert information['automatic-emails'] assert not information['automatic-emails']['last_error'] assert information['automatic-emails']['last_success'] # No new email was sent self.assertEqual(trigger_send_mock.call_count, 1) # 3. Verify that, after 'delay' is passed, a new email is sent # to our user # Insert a new crash report with the same email address storage.save_processed({ 'uuid': '51', 'email': email, 'product': 'EarthRaccoon', 'version': '20.0', 'release_channel': 'Release', 'date_processed': utc_now() + datetime.timedelta(days=1) }) storage.es.refresh() # Run crontabber with time pushed by a day with mock.patch('socorro.cron.crontabber.utc_now') as cronutc_mock: with mock.patch('socorro.cron.base.utc_now') as baseutc_mock: cronutc_mock.return_value = tomorrow baseutc_mock.return_value = tomorrow tab.run_all() information = self._load_structure() assert information['automatic-emails'] assert not information['automatic-emails']['last_error'] assert information['automatic-emails']['last_success'] # A new email was sent self.assertEqual(trigger_send_mock.call_count, 2)
def setUp(self): super(IntegrationTestSuperSearch, self).setUp() self.api = SuperSearch(config=self.config) self.now = datetimeutil.utc_now()
def test_get_missing_fields(self): config = self.get_base_config(es_index='socorro_integration_test_%W') fake_mappings = [ { 'mappings': { config.elasticsearch.elasticsearch_doctype: { 'properties': { # Add a bunch of unknown fields. 'field_z': { 'type': 'string' }, 'namespace1': { 'type': 'object', 'properties': { 'field_a': { 'type': 'string' }, 'field_b': { 'type': 'long' } } }, 'namespace2': { 'type': 'object', 'properties': { 'subspace1': { 'type': 'object', 'properties': { 'field_b': { 'type': 'long' } } } } }, # Add a few known fields that should not appear. 'processed_crash': { 'type': 'object', 'properties': { 'signature': { 'type': 'string' }, 'product': { 'type': 'string' }, } } } } } }, { 'mappings': { config.elasticsearch.elasticsearch_doctype: { 'properties': { 'namespace1': { 'type': 'object', 'properties': { 'subspace1': { 'type': 'object', 'properties': { 'field_d': { 'type': 'long' } } } } } } } } }, ] now = datetimeutil.utc_now() indices = [] try: # Using "2" here means that an index will be missing, hence testing # that it swallows the subsequent error. for i in range(2): date = now - datetime.timedelta(weeks=i) index = date.strftime(config.elasticsearch.elasticsearch_index) mapping = fake_mappings[i % len(fake_mappings)] self.index_creator.create_index(index, mapping) indices.append(index) api = SuperSearchFields(config=config) missing_fields = api.get_missing_fields() expected = [ 'field_z', 'namespace1.field_a', 'namespace1.field_b', 'namespace1.subspace1.field_d', 'namespace2.subspace1.field_b', ] eq_(missing_fields['hits'], expected) eq_(missing_fields['total'], 5) finally: for index in indices: self.index_client.delete(index=index)
def setUpClass(cls): """ Populate product_info table with fake data """ super(IntegrationTestSignatureSummary, cls).setUpClass() cursor = cls.connection.cursor() # Insert data cls.now = datetimeutil.utc_now() now = cls.now.date() yesterday = now - datetime.timedelta(days=1) lastweek = now - datetime.timedelta(days=7) cursor.execute(""" INSERT INTO products (product_name, sort, rapid_release_version, release_name) VALUES ( 'Firefox', 1, '8.0', 'firefox' ), ( 'Fennec', 3, '11.0', 'mobile' ), ( 'Thunderbird', 2, '10.0', 'thunderbird' ); """) cursor.execute(""" INSERT INTO release_channels (release_channel, sort) VALUES ( 'Release', 1 ), ( 'Beta', 2 ); """) cursor.execute(""" INSERT INTO product_release_channels (product_name, release_channel, throttle) VALUES ( 'Firefox', 'Release', '0.1' ), ( 'Fennec', 'Release', '0.1' ), ( 'Fennec', 'Beta', '1.0' ), ( 'Thunderbird', 'Release', '0.1' ); """) # Insert versions, contains an expired version cursor.execute(""" INSERT INTO product_versions (product_version_id, product_name, major_version, release_version, version_string, build_date, sunset_date, featured_version, build_type, version_sort, has_builds, is_rapid_beta) VALUES ( 1, 'Firefox', '8.0', '8.0', '8.0', '%(now)s', '%(now)s', False, 'Release', '0008000', True, False ), ( 2, 'Firefox', '9.0', '9.0', '9.0', '%(lastweek)s', '%(lastweek)s', False, 'Nightly', '0009000', True, False ), ( 3, 'Fennec', '11.0', '11.0', '11.0.1', '%(now)s', '%(now)s', False, 'Release', '0011001', True, False ), ( 4, 'Fennec', '12.0', '12.0', '12.0b1', '%(now)s', '%(now)s', False, 'Beta', '00120b1', True, False ), ( 5, 'Thunderbird', '10.0', '10.0', '10.0.2b', '%(now)s', '%(now)s', False, 'Release', '001002b', True, False ); """ % { 'now': now, 'lastweek': lastweek }) cursor.execute(""" INSERT INTO signatures (first_build, first_report, signature) VALUES ('20130701120000', '%(now)s', 'Fake Signature #1') """ % {'now': now}) cursor.execute(""" SELECT signature_id FROM signatures WHERE signature = 'Fake Signature #1' """) signature_id = cursor.fetchone()[0] cursor.execute(""" SELECT product_version_id FROM product_versions WHERE product_name = 'Firefox' and version_string = '8.0' """) product_version_id = cursor.fetchone()[0] cursor.execute(""" SELECT product_version_id FROM product_versions WHERE product_name = 'Firefox' and version_string = '9.0' """) other_product_version_id = cursor.fetchone()[0] cursor.execute( """ INSERT INTO signature_summary_products (signature_id, product_version_id, product_name, version_string, report_date, report_count) VALUES (%(signature_id)s, %(product_version_id)s, 'Firefox', '8.0', '%(yesterday)s', 1) """ % { 'yesterday': yesterday, 'product_version_id': product_version_id, 'signature_id': signature_id }) cursor.execute( """ INSERT INTO signature_summary_products (signature_id, product_version_id, product_name, version_string, report_date, report_count) VALUES (%(signature_id)s, %(product_version_id)s, 'Firefox', '9.0', '%(yesterday)s', 1) """ % { 'yesterday': yesterday, 'product_version_id': other_product_version_id, 'signature_id': signature_id }) cursor.execute( """ INSERT INTO signature_summary_architecture (signature_id, architecture, product_version_id, product_name, report_date, report_count, version_string) VALUES (%(signature_id)s, 'amd64', %(product_version_id)s, 'Firefox', '%(yesterday)s', 1, '8.0') """ % { 'yesterday': yesterday, 'product_version_id': product_version_id, 'signature_id': signature_id }) cursor.execute( """ INSERT INTO signature_summary_architecture (signature_id, architecture, product_version_id, product_name, report_date, report_count, version_string) VALUES (%(signature_id)s, 'amd64', %(product_version_id)s, 'Firefox', '%(yesterday)s', 1, '9.0') """ % { 'yesterday': yesterday, 'product_version_id': other_product_version_id, 'signature_id': signature_id }) cursor.execute( """ INSERT INTO signature_summary_flash_version (signature_id, flash_version, product_version_id, product_name, report_date, report_count, version_string) VALUES (%(signature_id)s, '1.0', %(product_version_id)s, 'Firefox', '%(yesterday)s', 1, '8.0') """ % { 'yesterday': yesterday, 'product_version_id': product_version_id, 'signature_id': signature_id }) cursor.execute( """ INSERT INTO signature_summary_flash_version (signature_id, flash_version, product_version_id, product_name, report_date, report_count, version_string) VALUES (%(signature_id)s, '1.0', %(product_version_id)s, 'Firefox', '%(yesterday)s', 1, '9.0') """ % { 'yesterday': yesterday, 'product_version_id': other_product_version_id, 'signature_id': signature_id }) cursor.execute(""" INSERT INTO signature_summary_installations (signature_id, product_name, version_string, report_date, crash_count, install_count) VALUES (%(signature_id)s, 'Firefox', '8.0', '%(yesterday)s', 10, 8) """ % { 'yesterday': yesterday, 'signature_id': signature_id }) cursor.execute( """ INSERT INTO signature_summary_os (signature_id, os_version_string, product_version_id, product_name, report_date, report_count, version_string) VALUES (%(signature_id)s, 'Windows NT 6.4', %(product_version_id)s, 'Firefox', '%(yesterday)s', 1, '8.0') """ % { 'yesterday': yesterday, 'product_version_id': product_version_id, 'signature_id': signature_id }) cursor.execute( """ INSERT INTO signature_summary_process_type (signature_id, process_type, product_version_id, product_name, report_date, report_count, version_string) VALUES (%(signature_id)s, 'plugin', %(product_version_id)s, 'Firefox', '%(yesterday)s', 1, '8.0') """ % { 'yesterday': yesterday, 'product_version_id': product_version_id, 'signature_id': signature_id }) cursor.execute( """ INSERT INTO signature_summary_uptime (signature_id, uptime_string, product_version_id, product_name, report_date, report_count, version_string) VALUES (%(signature_id)s, '15-30 minutes', %(product_version_id)s, 'Firefox', '%(yesterday)s', 1, '8.0') """ % { 'yesterday': yesterday, 'product_version_id': product_version_id, 'signature_id': signature_id }) cursor.execute( """ INSERT INTO exploitability_reports (signature_id, product_version_id, product_name, version_string, signature, report_date, null_count, none_count, low_count, medium_count, high_count) VALUES (%(signature_id)s, %(product_version_id)s, 'Firefox', '8.0', 'Fake Signature #1', '%(yesterday)s', 1, 2, 3, 4, 5) """ % { 'yesterday': yesterday, 'signature_id': signature_id, 'product_version_id': product_version_id }) cursor.execute(""" INSERT INTO android_devices (android_cpu_abi, android_manufacturer, android_model, android_version) VALUES ('armeabi-v7a', 'samsung', 'GT-P5100', '16 (REL)') """) cursor.execute(""" SELECT android_device_id FROM android_devices WHERE android_cpu_abi = 'armeabi-v7a' AND android_manufacturer = 'samsung' AND android_model = 'GT-P5100' AND android_version = '16 (REL)' """) device_id = cursor.fetchone()[0] cursor.execute( """ INSERT INTO signature_summary_device (report_date, signature_id, product_version_id, product_name, version_string, android_device_id, report_count) VALUES ('%(yesterday)s', %(signature_id)s, %(product_version_id)s, 'Firefox', '8.0', %(device_id)s, 123) """ % { 'yesterday': yesterday, 'signature_id': signature_id, 'device_id': device_id, 'product_version_id': product_version_id }) cursor.execute( """ INSERT INTO signature_summary_device (report_date, signature_id, product_version_id, product_name, version_string, android_device_id, report_count) VALUES ('%(yesterday)s', %(signature_id)s, %(product_version_id)s, 'Firefox', '9.0', %(device_id)s, 123) """ % { 'yesterday': yesterday, 'signature_id': signature_id, 'device_id': device_id, 'product_version_id': other_product_version_id }) cursor.execute(""" INSERT INTO graphics_device (vendor_hex, adapter_hex, vendor_name, adapter_name) VALUES ('0x1234', '0x5678', 'Test Vendor', 'Test Adapter') """) cursor.execute(""" SELECT graphics_device_id FROM graphics_device WHERE vendor_hex = '0x1234' AND adapter_hex = '0x5678' """) graphics_device_id = cursor.fetchone()[0] cursor.execute( """ INSERT INTO signature_summary_graphics (report_date, signature_id, graphics_device_id, product_version_id, product_name, version_string, report_count) VALUES ('%(yesterday)s', %(signature_id)s, %(device_id)s, %(product_version_id)s, 'Firefox', '8.0', 123) """ % { 'yesterday': yesterday, 'signature_id': signature_id, 'device_id': graphics_device_id, 'product_version_id': product_version_id }) cls.connection.commit() def add_product_version_builds(self): cursor = self.connection.cursor() cursor.execute(""" SELECT product_version_id FROM product_versions WHERE product_name = 'Firefox' and version_string = '8.0' """) product_version_id = cursor.fetchone()[0] cursor.execute(""" INSERT INTO product_version_builds (build_id, platform, product_version_id) VALUES (1, 'Windows NT', %(product_version_id)s) """ % {'product_version_id': product_version_id}) self.connection.commit()
def setUp(self): """Set up this test class by populating the reports table with fake data. """ super(IntegrationTestCrashes, self).setUp() cursor = self.connection.cursor() self.now = datetimeutil.utc_now() yesterday = self.now - datetime.timedelta(days=1) day_before_yesterday = self.now - datetime.timedelta(days=2) build_date = self.now - datetime.timedelta(days=30) sunset_date = self.now + datetime.timedelta(days=30) cursor.execute(""" INSERT INTO products (product_name, sort, release_name) VALUES ( 'WaterWolf', 1, 'WaterWolf' ); """) cursor.execute(""" INSERT INTO product_versions (product_version_id, product_name, major_version, release_version, version_string, version_sort, build_date, sunset_date, featured_version, build_type, is_rapid_beta, rapid_beta_id) VALUES ( 1, 'WaterWolf', '1.0', '1.0', '1.0', '10000011000', '%(build_date)s', '%(sunset_date)s', 't', 'Nightly', False, NULL ), ( 2, 'WaterWolf', '2.0', '2.0', '2.0', '10000012000', '%(build_date)s', '%(sunset_date)s', 't', 'Nightly', False, NULL ); """ % { "build_date": build_date, "sunset_date": sunset_date }) cursor.execute(""" INSERT INTO gccrashes (report_date, product_version_id, build, gc_count_madu) VALUES ('%s', '%s', '%s', '%s'), ('%s', '%s', '%s', '%s'), ('%s', '%s', '%s', '%s'); """ % (yesterday, "1", "10000011000", "42", day_before_yesterday, "1", "10000011000", "42", yesterday, "2", "10000012000", "24")) self.connection.commit() cursor.close()
def _run_one(self, job_class, config, force=False): seconds = convert_frequency(config.frequency) time_ = config.time if not force: if not self.time_to_run(job_class, time_): self.logger.debug("skipping %r because it's not time to run", job_class) return self.logger.debug('about to run %r', job_class) app_name = job_class.app_name info = self.job_state_database.get(app_name) last_success = None now = utc_now() log_run = True exc_type = exc_value = exc_tb = None try: t0 = time.time() for last_success in self._run_job(job_class, config, info): t1 = time.time() self.logger.debug('successfully ran %r on %s', job_class, last_success) self._remember_success(job_class, last_success, t1 - t0) # _run_job() returns a generator, so we don't know how # many times this will loop. Anyway, we need to reset the # 't0' for the next loop if there is one. t0 = time.time() except (OngoingJobError, RowLevelLockError): # It's not an actual runtime error. It just basically means # you can't start crontabber right now. log_run = False raise except Exception: t1 = time.time() exc_type, exc_value, exc_tb = sys.exc_info() if self.config.sentry and self.config.sentry.dsn: client = sentry_client.get_client(self.config.sentry.dsn) identifier = client.get_ident(client.captureException()) self.logger.info('Error captured in Sentry. Reference: %s' % identifier) self.logger.debug( 'error when running %r on %s', job_class, last_success, exc_info=True ) self._remember_failure( job_class, t1 - t0, exc_type, exc_value, exc_tb ) finally: if log_run: self._log_run( job_class, seconds, time_, last_success, now, exc_type, exc_value, exc_tb )
def setUp(self): super(IntegrationTestCrashAduByBuildSignature, self).setUp() cursor = self.connection.cursor() # Insert data self.now = datetimeutil.utc_now() self.tomorrow = self.now + datetime.timedelta(days=1) tomorrow = self.tomorrow.date() now = self.now.date() cursor.execute(""" INSERT INTO products (product_name, sort, rapid_release_version, release_name) VALUES ( 'Firefox', 1, '8.0', 'firefox' ); """) cursor.execute(""" INSERT INTO release_channels (release_channel, sort) VALUES ( 'Release', 1 ), ( 'Beta', 2 ); """) cursor.execute(""" INSERT INTO product_versions (product_version_id, product_name, major_version, release_version, version_string, build_date, sunset_date, featured_version, build_type, version_sort, has_builds, is_rapid_beta, build_type_enum) VALUES ( 1, 'Firefox', '8.0', '8.0', '8.0', '%(now)s', '%(now)s', False, 'release', '0008000', True, False, 'release' ); """ % {'now': now}) cursor.execute(""" INSERT INTO signatures (first_build, first_report, signature) VALUES ('20130701120000', '%(now)s', 'Fake Signature #1') """ % {'now': now}) cursor.execute(""" SELECT signature_id FROM signatures WHERE signature = 'Fake Signature #1' """) signature_id = cursor.fetchone()[0] cursor.execute(""" SELECT product_version_id FROM product_versions WHERE product_name = 'Firefox' and version_string = '8.0' """) product_version_id = cursor.fetchone()[0] cursor.execute( """ INSERT INTO reports_clean (address_id, build, date_processed, domain_id, flash_version_id, os_name, os_version_id, process_type, reason_id, release_channel, signature_id, uuid, build_type, product_version_id) VALUES (1, '%(build)s', '%(now)s', 1, 1, 'windows', '9', 'browser', 1, 'release', '%(signature_id)s', 'a1', 'release', '%(product_version_id)s')""" % { 'now': now, 'build': now.strftime('%Y%m%d'), 'signature_id': signature_id, 'product_version_id': product_version_id }) cursor.execute( """ INSERT INTO build_adu (product_version_id, build_date, adu_date, os_name, adu_count) VALUES (%(product_version_id)s, '%(now)s', '%(now)s', 'windows', 123), (%(product_version_id)s, '%(tomorrow)s', '%(tomorrow)s', 'windows', 321) """ % { 'product_version_id': product_version_id, 'now': now, 'tomorrow': tomorrow })
def mock_utc_now(): n = utc_now() n = n.replace(hour=3) return n
def process_crash(self, raw_crash, raw_dumps, processed_crash): """Take a raw_crash and its associated raw_dumps and return a processed_crash If this throws an exception, the crash was not processed correctly. """ # processor_meta_data will be used to ferry "inside information" to # transformation rules. Sometimes rules need a bit more extra # information about the transformation process itself. processor_meta_data = DotDict() processor_meta_data.processor_notes = [ self.config.processor_name, self.__class__.__name__ ] processor_meta_data.quit_check = self.quit_check processor_meta_data.processor = self processor_meta_data.config = self.config if "processor_notes" in processed_crash: original_processor_notes = [ x.strip() for x in processed_crash.processor_notes.split(";") ] processor_meta_data.processor_notes.append( "earlier processing: %s" % processed_crash.get("started_datetime", 'Unknown Date')) else: original_processor_notes = [] processed_crash.success = False processed_crash.started_datetime = utc_now() # for backwards compatibility: processed_crash.startedDateTime = processed_crash.started_datetime processed_crash.signature = 'EMPTY: crash failed to process' crash_id = raw_crash['uuid'] # quit_check calls ought to be scattered around the code to allow # the processor to be responsive to requests to shut down. self.quit_check() start_time = self.logger.info('starting transform for crash: %s', crash_id) processor_meta_data.started_timestamp = start_time # Apply rules; if a rule fails, capture the error and continue onward for rule in self.rules: try: rule.act(raw_crash, raw_dumps, processed_crash, processor_meta_data) except Exception as exc: # If a rule throws an error, capture it and toss it in the # processor notes sentry_client.capture_error(sentry_dsn=self.sentry_dsn, logger=self.logger, extra={'crash_id': crash_id}) # NOTE(willkg): notes are public, so we can't put exception # messages in them processor_meta_data.processor_notes.append( 'rule %s failed: %s' % (rule.__class__.__name__, exc.__class__.__name__)) self.quit_check() # The crash made it through the processor rules with no exceptions # raised, call it a success processed_crash.success = True # The processor notes are in the form of a list. Join them all # together to make a single string processor_meta_data.processor_notes.extend(original_processor_notes) processed_crash.processor_notes = '; '.join( processor_meta_data.processor_notes) completed_datetime = utc_now() processed_crash.completed_datetime = completed_datetime # For backwards compatibility processed_crash.completeddatetime = completed_datetime self.logger.info("finishing %s transform for crash: %s", 'successful' if processed_crash.success else 'failed', crash_id) return processed_crash
def fix_date_parameter(self, parameters): """Correct the date parameter. If there is no date parameter, set default values. Otherwise, make sure there is exactly one lower bound value and one greater bound value. """ default_date_range = datetime.timedelta( days=self.config.search_default_date_range ) maximum_date_range = datetime.timedelta( days=self.config.search_maximum_date_range ) if not parameters.get('date'): now = datetimeutil.utc_now() lastweek = now - default_date_range parameters['date'] = [] parameters['date'].append(SearchParam( 'date', lastweek, '>=', 'datetime' )) parameters['date'].append(SearchParam( 'date', now, '<=', 'datetime' )) else: lower_than = None greater_than = None for param in parameters['date']: if ( '<' in param.operator and ( not lower_than or (lower_than and lower_than.value > param.value) ) ): lower_than = param if ( '>' in param.operator and ( not greater_than or (greater_than and greater_than.value < param.value) ) ): greater_than = param # Remove all the existing parameters so we have exactly # one lower value and one greater value parameters['date'] = [] if not lower_than: # add a lower than that is now lower_than = SearchParam( 'date', datetimeutil.utc_now(), '<=', 'datetime' ) if not greater_than: # add a greater than that is lower_than minus the date range greater_than = SearchParam( 'date', lower_than.value - default_date_range, '>=', 'datetime' ) # Verify the date range is not too big. delta = lower_than.value - greater_than.value if delta > maximum_date_range: raise BadArgumentError( 'date', msg='Date range is bigger than %s days' % self.config.search_maximum_date_range ) parameters['date'].append(lower_than) parameters['date'].append(greater_than)
def setUp(self): """Set up this test class by populating the reports table with fake data. """ super(IntegrationTestSearch, self).setUp() cursor = self.connection.cursor() # Insert data now = datetimeutil.utc_now() yesterday = now - datetime.timedelta(days=1) cursor.execute(""" INSERT INTO reports ( id, uuid, date_processed, product, version, build, signature, reason, os_name, hangid, process_type, release_channel ) VALUES ( 1, '1', '%(yesterday)s', 'WaterWolf', '1.0', '20001212010203', 'sig1', 'STACK_OVERFLOW', 'Linux', 1, 'browser', 'Release' ), ( 2, '2', '%(yesterday)s', 'WaterWolf', '2.0', '20001212010204', 'sig1', 'SIGFAULT', 'Windows NT', 2, 'browser', 'Release' ), ( 3, '3', '%(yesterday)s', 'WaterWolf', '1.0', '20001212010205', 'sig1', 'BIG_FAILURE', 'Windows NT', null, 'plugin', 'Release' ), ( 4, '4', '%(yesterday)s', 'WaterWolf', '1.0', '20001212010203', 'sig1', 'STACK_OVERFLOW', 'Windows NT', null, 'browser', 'Release' ), ( 5, '5', '%(yesterday)s', 'WaterWolf', '1.0', '20001212010203', 'sig2', 'STACK_OVERFLOW', 'Linux', null, 'browser', 'Release' ), ( 6, '6', '%(yesterday)s', 'WaterWolf', '3.0', '20001212010203', 'sig2', 'STACK_OVERFLOW', 'Windows NT', null, 'browser', 'Release' ), ( 7, '7', '%(yesterday)s', 'NightlyTrain', '1.0', '20001212010203', 'sig2', 'STACK_OVERFLOW', 'Linux', null, 'plugin', 'Release' ), ( 8, '8', '%(yesterday)s', 'WaterWolf', '1.0', '20001212010204', 'sig3', 'STACK_OVERFLOW', 'Linux', null, 'browser', 'Release' ), ( 9, '9', '%(yesterday)s', 'NightlyTrain', '1.0', '20001212010203', 'js::functions::call::hello_world', 'SIGFAULT', 'Linux', null, 'browser', 'Release' ), ( 10, '10', '%(yesterday)s', 'WindBear', '1.0', '20001212010203', 'this/is+a=C|signature', 'STACK_OVERFLOW', 'Linux', null, 'browser', 'Release' ); """ % { 'yesterday': yesterday }) cursor.execute(""" INSERT INTO plugins_reports ( report_id, plugin_id, date_processed, version ) VALUES ( 3, 1, '%(yesterday)s', '1.23.001' ), ( 7, 2, '%(yesterday)s', '2.0.1' ); """ % { 'yesterday': yesterday }) cursor.execute(""" INSERT INTO plugins ( id, filename, name ) VALUES ( 1, 'flash.dll', 'Flash' ), ( 2, 'NPSWF32_11_5_502_146.dll', 'someplugin' ); """) self.connection.commit()
def query(self, from_date, to_date, json_query): """ Send a query directly to ElasticSearch and return the result. """ # Default dates now = dtutil.utc_now().date() lastweek = now - timedelta(7) from_date = dtutil.string_to_datetime(from_date) or lastweek to_date = dtutil.string_to_datetime(to_date) or now # Create the indexes to use for querying. daterange = [] delta_day = to_date - from_date for delta in range(0, delta_day.days + 1): day = from_date + timedelta(delta) index = "socorro_%s" % day.strftime("%y%m%d") # Cache protection for limitating the number of HTTP calls if index not in self.cache or not self.cache[index]: daterange.append(index) can_return = False # - # This code is here to avoid failing queries caused by missing # indexes. It should not happen on prod, but doing this makes # sure users will never see a 500 Error because of this eventuality. # - # Iterate until we can return an actual result and not an error while not can_return: if not daterange: http_response = "{}" break datestring = ",".join(daterange) uri = "/%s/_search" % datestring with self.http: http_response = self.http.post(uri, json_query) # If there has been an error, # then we get a dict instead of some json. if isinstance(http_response, dict): data = http_response["error"]["data"] # If an index is missing, # try to remove it from the list of indexes and retry. if (http_response["error"]["code"] == 404 and data.find("IndexMissingException") >= 0): index = data[data.find("[[") + 2:data.find("]")] # Cache protection for limitating the number of HTTP calls self.cache[index] = True try: daterange.remove(index) except Exception: raise else: can_return = True return (http_response, "text/json")
def test_email_cannot_be_sent_twice(self, exacttarget_mock): (config_manager, json_file) = self._setup_config_manager( restrict_products=['NightlyTrain'] ) et_mock = exacttarget_mock.return_value # Prepare failures _failures = [] _email_sent = [] class SomeRandomError(Exception): pass def trigger_send(template, fields): email = fields['EMAIL_ADDRESS_'] if email == '*****@*****.**' and email not in _failures: _failures.append(email) raise SomeRandomError('This is an error. ') else: _email_sent.append(email) et_mock.trigger_send = trigger_send with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = json.load(open(json_file)) assert information['automatic-emails'] assert information['automatic-emails']['last_error'] self.assertEqual( information['automatic-emails']['last_error']['type'], str(SomeRandomError) ) # Verify that user's data was updated, but not all of it self.assertEqual(_email_sent, ['*****@*****.**', '*****@*****.**']) cursor = self.conn.cursor() emails_list = ( '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**' ) sql = """ SELECT email, last_sending FROM emails WHERE email IN %s """ % (emails_list,) cursor.execute(sql) now = utc_now() self.assertEqual(cursor.rowcount, 2) for row in cursor.fetchall(): assert row[0] in ('*****@*****.**', '*****@*****.**') self.assertEqual(row[1].year, now.year) self.assertEqual(row[1].month, now.month) self.assertEqual(row[1].day, now.day) # Run crontabber again and verify that all users are updated, # and emails are not sent twice self._wind_clock(json_file, hours=1) # This forces a crontabber instance to reload the JSON file tab._database = None tab.run_all() information = json.load(open(json_file)) assert information['automatic-emails'] assert not information['automatic-emails']['last_error'] assert information['automatic-emails']['last_success'] # Verify that users were not sent an email twice self.assertEqual(_email_sent, [ '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**' ])
def setUpClass(cls): """ Populate product_info table with fake data """ super(IntegrationTestProducts, cls).setUpClass() cursor = cls.connection.cursor() # Insert data cls.now = datetimeutil.utc_now() now = cls.now.date() lastweek = now - datetime.timedelta(days=7) cursor.execute(""" INSERT INTO products (product_name, sort, rapid_release_version, release_name) VALUES ( 'Firefox', 1, '8.0', 'firefox' ), ( 'Fennec', 3, '11.0', 'mobile' ), ( 'Thunderbird', 2, '10.0', 'thunderbird' ); """) cursor.execute(""" INSERT INTO release_channels (release_channel, sort) VALUES ( 'Release', 1 ), ( 'Beta', 2 ); """) cursor.execute(""" INSERT INTO product_release_channels (product_name, release_channel, throttle) VALUES ( 'Firefox', 'Release', '0.1' ), ( 'Fennec', 'Release', '0.1' ), ( 'Fennec', 'Beta', '1.0' ), ( 'Thunderbird', 'Release', '0.1' ); """) # Insert versions, contains an expired version cursor.execute(""" INSERT INTO product_versions (product_name, major_version, release_version, version_string, build_date, sunset_date, featured_version, build_type, version_sort) VALUES ( 'Firefox', '8.0', '8.0', '8.0', '%(now)s', '%(now)s', False, 'Release', '0008000' ), ( 'Firefox', '9.0', '9.0', '9.0', '%(lastweek)s', '%(lastweek)s', False, 'Nightly', '0009000' ), ( 'Fennec', '11.0', '11.0', '11.0.1', '%(now)s', '%(now)s', False, 'Release', '0011001' ), ( 'Fennec', '12.0', '12.0', '12.0b1', '%(now)s', '%(now)s', False, 'Beta', '00120b1' ), ( 'Thunderbird', '10.0', '10.0', '10.0.2b', '%(now)s', '%(now)s', False, 'Release', '001002b' ); """ % { 'now': now, 'lastweek': lastweek }) cls.connection.commit()
def setUp(self): super(IntegrationTestAutomaticEmails, self).setUp() # prep a fake table now = utc_now() - datetime.timedelta(minutes=30) last_month = now - datetime.timedelta(days=31) config_manager = self._setup_storage_config() with config_manager.context() as config: storage = ElasticSearchCrashStorage(config) # clear the indices cache so the index is created on every test storage.indices_cache = set() storage.save_processed({ 'uuid': '1', 'email': '*****@*****.**', 'product': 'WaterWolf', 'version': '20.0', 'release_channel': 'Release', 'date_processed': now, 'classifications': { 'support': { 'classification': 'unknown' } } }) storage.save_processed({ 'uuid': '2', 'email': '"Quidam" <*****@*****.**>', 'product': 'WaterWolf', 'version': '20.0', 'release_channel': 'Release', 'date_processed': now, 'classifications': { 'support': { 'classification': None } } }) storage.save_processed({ 'uuid': '3', 'email': '*****@*****.**', 'product': 'WaterWolf', 'version': '20.0', 'release_channel': 'Release', 'date_processed': now, 'classifications': { 'support': { 'classification': 'bitguard' } } }) storage.save_processed({ 'uuid': '4', 'email': '*****@*****.**', 'product': 'NightlyTrain', 'version': '1.0', 'release_channel': 'Nightly', 'date_processed': now }) storage.save_processed({ 'uuid': '5', 'email': '*****@*****.**', 'product': 'NightlyTrain', 'version': '1.0', 'release_channel': 'Nightly', 'date_processed': now }) storage.save_processed({ 'uuid': '6', 'email': '*****@*****.**', 'product': 'NightlyTrain', 'version': '1.0', 'release_channel': 'Nightly', 'date_processed': now }) storage.save_processed({ 'uuid': '7', 'email': '*****@*****.**', 'product': 'NightlyTrain', 'version': '1.0', 'release_channel': 'Nightly', 'date_processed': now }) storage.save_processed({ 'uuid': '8', 'email': '*****@*****.**', 'product': 'NightlyTrain', 'version': '1.0', 'release_channel': 'Nightly', 'date_processed': now }) storage.save_processed({ 'uuid': '9', 'email': '*****@*****.**', 'product': 'EarthRaccoon', 'version': '1.0', 'release_channel': 'Nightly', 'date_processed': now }) storage.save_processed({ 'uuid': '18', 'email': 'z\xc3\[email protected]', 'product': 'WaterWolf', 'version': '20.0', 'release_channel': 'Release', 'date_processed': now }) # Let's insert a duplicate storage.save_processed({ 'uuid': '10', 'email': '*****@*****.**', 'product': 'WaterWolf', 'version': '20.0', 'release_channel': 'Release', 'date_processed': now }) # And let's insert some invalid crashes storage.save_processed({ 'uuid': '11', 'email': None, 'product': 'WaterWolf', 'version': '20.0', 'release_channel': 'Release', 'date_processed': now }) storage.save_processed({ 'uuid': '12', 'email': '*****@*****.**', 'product': 'WaterWolf', 'version': '20.0', 'release_channel': 'Release', 'date_processed': last_month }) storage.save_processed({ 'uuid': '13', 'email': '*****@*****.**', 'product': 'WaterWolf', 'version': '20.0', 'release_channel': 'Release', 'date_processed': now }) storage.save_processed({ 'uuid': '14', 'email': '*****@*****.**', 'product': 'WindBear', 'version': '20.0', 'release_channel': 'Release', 'date_processed': now }) # Finally some invalid email addresses storage.save_processed({ 'uuid': '15', 'email': ' ', 'product': 'WaterWolf', 'version': '20.0', 'release_channel': 'Release', 'date_processed': now }) storage.save_processed({ 'uuid': '16', 'email': 'invalid@email', 'product': 'WaterWolf', 'version': '20.0', 'release_channel': 'Release', 'date_processed': now }) storage.save_processed({ 'uuid': '17', 'email': 'i.do.not.work', 'product': 'WaterWolf', 'version': '20.0', 'release_channel': 'Release', 'date_processed': now }) # Create some email addresses. storage.create_emails_index() storage.es.index( index=config.elasticsearch_emails_index, doc_type='emails', doc={ 'email': '*****@*****.**', 'last_sending': last_month }, id='*****@*****.**', ) storage.es.index( index=config.elasticsearch_emails_index, doc_type='emails', doc={ 'email': '"Quidam" <*****@*****.**>', 'last_sending': last_month }, id='"Quidam" <*****@*****.**>', ) storage.es.index( index=config.elasticsearch_emails_index, doc_type='emails', doc={ 'email': '*****@*****.**', 'last_sending': now }, id='*****@*****.**', ) # As indexing is asynchronous, we need to force elasticsearch to # make the newly created content searchable before we run the # tests. storage.es.refresh()
def setUp(self): """ Populate product_info table with fake data """ super(IntegrationTestProducts, self).setUp() cursor = self.connection.cursor() # Insert data self.now = datetimeutil.utc_now() now = self.now.date() lastweek = now - datetime.timedelta(days=7) cursor.execute(""" INSERT INTO products (product_name, sort, rapid_release_version, release_name) VALUES ( 'Firefox', 1, '8.0', 'firefox' ), ( 'Fennec', 3, '11.0', 'mobile' ), ( 'Thunderbird', 2, '10.0', 'thunderbird' ); """) cursor.execute(""" INSERT INTO release_channels (release_channel, sort) VALUES ( 'Release', 1 ), ( 'Beta', 2 ); """) cursor.execute(""" INSERT INTO product_release_channels (product_name, release_channel, throttle) VALUES ( 'Firefox', 'Release', '0.1' ), ( 'Fennec', 'Release', '0.1' ), ( 'Fennec', 'Beta', '1.0' ), ( 'Thunderbird', 'Release', '0.1' ); """) # Insert versions, contains an expired version cursor.execute(""" INSERT INTO product_versions (product_name, major_version, release_version, version_string, build_date, sunset_date, featured_version, build_type, version_sort) VALUES ( 'Firefox', '8.0', '8.0', '8.0', '%(now)s', '%(now)s', False, 'Release', '0008000' ), ( 'Firefox', '9.0', '9.0', '9.0', '%(lastweek)s', '%(lastweek)s', False, 'Nightly', '0009000' ), ( 'Fennec', '11.0', '11.0', '11.0.1', '%(now)s', '%(now)s', False, 'Release', '0011001' ), ( 'Fennec', '12.0', '12.0', '12.0b1', '%(now)s', '%(now)s', False, 'Beta', '00120b1' ), ( 'Thunderbird', '10.0', '10.0', '10.0.2b', '%(now)s', '%(now)s', False, 'Release', '001002b' ); """ % { 'now': now, 'lastweek': lastweek }) # insert bixie errors cursor.execute(""" INSERT INTO bixie.raw_product_releases (id, product_name, version, build, build_type, platform, repository, stability) VALUES ( 1, 'EmailApp', '0.1', 1234567890, 'Release', 'mobile', 'repo', 'stable' ), ( 2, 'EmailApp', '0.2', 1234567890, 'Beta', 'mobile', 'repo', 'stable' ), ( 3, 'ClockOClock', '1.0.18', 1234567890, 'Release', 'mobile', 'repo', 'stable' ) """) self.connection.commit()
def setUp(self): super(IntegrationElasticsearchSearch, self).setUp() config = self.get_config_context() self.api = Search(config=config) self.storage = crashstorage.ElasticSearchCrashStorage(config) # clear the indices cache so the index is created on every test self.storage.indices_cache = set() # Create the supersearch fields. self.storage.es.bulk_index( index=config.webapi.elasticsearch_default_index, doc_type='supersearch_fields', docs=SUPERSEARCH_FIELDS.values(), id_field='name', refresh=True, ) now = datetimeutil.utc_now() yesterday = now - datetime.timedelta(days=1) yesterday = datetimeutil.date_to_string(yesterday) last_month = now - datetime.timedelta(weeks=4) last_month = datetimeutil.date_to_string(last_month) # insert data into elasticsearch default_crash_report = { 'uuid': 100, 'signature': 'js::break_your_browser', 'date_processed': yesterday, 'product': 'WaterWolf', 'version': '1.0', 'release_channel': 'release', 'os_name': 'Linux', 'build': '1234567890', 'reason': 'MOZALLOC_WENT_WRONG', 'hangid': None, 'process_type': None, } self.storage.save_processed(default_crash_report) self.storage.save_processed( dict(default_crash_report, uuid=1, product='EarthRaccoon') ) self.storage.save_processed( dict(default_crash_report, uuid=2, version='2.0') ) self.storage.save_processed( dict(default_crash_report, uuid=3, release_channel='aurora') ) self.storage.save_processed( dict(default_crash_report, uuid=4, os_name='Windows NT') ) self.storage.save_processed( dict(default_crash_report, uuid=5, build='0987654321') ) self.storage.save_processed( dict(default_crash_report, uuid=6, reason='VERY_BAD_EXCEPTION') ) self.storage.save_processed( dict(default_crash_report, uuid=7, hangid='12') ) self.storage.save_processed( dict(default_crash_report, uuid=8, process_type='plugin') ) self.storage.save_processed( dict(default_crash_report, uuid=9, signature='my_bad') ) self.storage.save_processed( dict( default_crash_report, uuid=10, date_processed=last_month, signature='my_little_signature', ) ) # for plugin terms test self.storage.save_processed( dict( default_crash_report, uuid=11, product='PluginSoft', process_type='plugin', PluginFilename='carly.dll', PluginName='Hey I just met you', PluginVersion='1.2', ) ) self.storage.save_processed( dict( default_crash_report, uuid=12, product='PluginSoft', process_type='plugin', PluginFilename='hey.dll', PluginName='Hey Plugin', PluginVersion='10.7.0.2a', ) ) self.storage.save_processed( dict( default_crash_report, uuid=13, product='EarlyOwl', version='11.0b1', release_channel='beta', ) ) self.storage.save_processed( dict( default_crash_report, uuid=14, product='EarlyOwl', version='11.0b2', release_channel='beta', ) ) # As indexing is asynchronous, we need to force elasticsearch to # make the newly created content searchable before we run the tests self.storage.es.refresh()
def setUp(self): super(TestFunctionalAutomaticEmails, self).setUp() # prep a fake table now = utc_now() - datetime.timedelta(minutes=30) last_month = now - datetime.timedelta(days=31) cursor = self.conn.cursor() cursor.execute(""" INSERT INTO reports (uuid, email, product, version, release_channel, date_processed) VALUES ( '1', '*****@*****.**', 'WaterWolf', '20.0', 'Release', '%(now)s' ), ( '2', '*****@*****.**', 'WaterWolf', '20.0', 'Release', '%(now)s' ), ( '3', '*****@*****.**', 'WaterWolf', '20.0', 'Release', '%(now)s' ), ( '4', '*****@*****.**', 'NightlyTrain', '1.0', 'Nightly', '%(now)s' ), ( '5', '*****@*****.**', 'NightlyTrain', '1.0', 'Nightly', '%(now)s' ), ( '6', '*****@*****.**', 'NightlyTrain', '1.0', 'Nightly', '%(now)s' ), ( '7', '*****@*****.**', 'NightlyTrain', '1.0', 'Nightly', '%(now)s' ), ( '8', '*****@*****.**', 'NightlyTrain', '1.0', 'Nightly', '%(now)s' ) """ % {'now': now}) # Let's insert a duplicate cursor.execute(""" INSERT INTO reports (uuid, email, product, version, release_channel, date_processed) VALUES ( '10', '*****@*****.**', 'WaterWolf', '20.0', 'Release', '%(now)s' ) """ % {'now': now}) # And let's insert some invalid crashes cursor.execute(""" INSERT INTO reports (uuid, email, product, version, release_channel, date_processed) VALUES ( '11', null, 'WaterWolf', '20.0', 'Release', '%(now)s' ), ( '12', '*****@*****.**', 'WaterWolf', '20.0', 'Release', '%(last_month)s' ), ( '13', '*****@*****.**', 'WaterWolf', '20.0', 'Release', '%(now)s' ), ( '14', '*****@*****.**', 'WindBear', '20.0', 'Release', '%(now)s' ) """ % {'now': now, 'last_month': last_month}) cursor.execute(""" INSERT INTO emails (email, last_sending) VALUES ( '*****@*****.**', '%(last_month)s' ), ( '*****@*****.**', '%(last_month)s' ), ( '*****@*****.**', '%(now)s' ) """ % {'now': now, 'last_month': last_month}) self.conn.commit()
def test_email_cannot_be_sent_twice(self, exacttarget_mock): config_manager = self._setup_config_manager( restrict_products=['NightlyTrain'] ) et_mock = exacttarget_mock.return_value # Prepare failures _failures = [] _email_sent = [] class SomeRandomError(Exception): pass def trigger_send(template, fields): email = fields['EMAIL_ADDRESS_'] if email == '*****@*****.**' and email not in _failures: _failures.append(email) raise SomeRandomError('This is an error. ') else: _email_sent.append(email) et_mock.trigger_send = trigger_send with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['automatic-emails'] assert information['automatic-emails']['last_error'] self.assertEqual( information['automatic-emails']['last_error']['type'], str(SomeRandomError) ) # Verify that user's data was updated, but not all of it self.assertEqual(_email_sent, ['*****@*****.**', '*****@*****.**']) emails_list = ( '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**' ) conf = config.crontabber['class-AutomaticEmailsCronApp'] es = SuperS().es( urls=conf.elasticsearch.elasticsearch_urls, timeout=conf.elasticsearch.elasticsearch_timeout, ) search = es.indexes( conf.elasticsearch.elasticsearch_emails_index ) search = search.doctypes('emails') es.get_es().refresh() search = search.filter(_id__in=emails_list) res = search.execute() self.assertEqual(res.count, 2) now = utc_now() for row in res.results: assert row['_id'] in ('*****@*****.**', '*****@*****.**') date = string_to_datetime(row['_source']['last_sending']) self.assertEqual(date.year, now.year) self.assertEqual(date.month, now.month) self.assertEqual(date.day, now.day) # Run crontabber again and verify that all users are updated, # and emails are not sent twice state = tab.database['automatic-emails'] self._wind_clock(state, hours=1) tab.database['automatic-emails'] = state tab.run_all() information = self._load_structure() assert information['automatic-emails'] assert not information['automatic-emails']['last_error'] assert information['automatic-emails']['last_success'] # Verify that users were not sent an email twice self.assertEqual(_email_sent, [ '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**' ])
def setUp(self): super(TestIntegrationFTPScraper, self).setUp() cursor = self.conn.cursor() # Insert data now = utc_now() build_date = now - datetime.timedelta(days=30) sunset_date = now + datetime.timedelta(days=30) cursor.execute(""" TRUNCATE products CASCADE; INSERT INTO products (product_name, sort, release_name) VALUES ( 'Firefox', 1, 'firefox' ), ( 'Fennec', 1, 'mobile' ); """) cursor.execute(""" TRUNCATE product_versions CASCADE; INSERT INTO product_versions (product_version_id, product_name, major_version, release_version, version_string, version_sort, build_date, sunset_date, featured_version, build_type) VALUES ( 1, 'Firefox', '15.0', '15.0', '15.0a1', '000000150a1', '%(build_date)s', '%(sunset_date)s', 't', 'nightly' ) ,( 2, 'Firefox', '24.5', '24.5.0esr', '24.5.0esr', '024005000x000', '%(build_date)s', '%(sunset_date)s', 't', 'esr' ) ; """ % { "build_date": build_date, "sunset_date": sunset_date }) cursor.execute(""" TRUNCATE release_channels CASCADE; INSERT INTO release_channels (release_channel, sort) VALUES ('nightly', 1), ('aurora', 2), ('beta', 3), ('release', 4); """) cursor.execute(""" TRUNCATE product_release_channels CASCADE; INSERT INTO product_release_channels (product_name, release_channel, throttle) VALUES ('Firefox', 'nightly', 1), ('Firefox', 'aurora', 1), ('Firefox', 'beta', 1), ('Firefox', 'release', 1), ('Fennec', 'release', 1), ('Fennec', 'beta', 1); """) self.conn.commit() self.mocked_session = requests.Session() def download(url): return self.mocked_session.get(url).content def skip_json_file(url): return False self.scrapers = ftpscraper.ScrapersMixin() self.scrapers.download = download self.scrapers.skip_json_file = skip_json_file self.scrapers.config = DotDict({'logger': mock.Mock()})
def setUp(self): super(IntegrationTestFeaturedVersionsAutomatic, self).setUp() self.__truncate() now = utc_now() build_date = now - datetime.timedelta(days=30) sunset_date = now + datetime.timedelta(days=30) execute_no_results( self.conn, """ INSERT INTO products (product_name, sort, release_name) VALUES ('Firefox', 1, 'firefox'), ('Fennec', 1, 'mobile') """) execute_no_results( self.conn, """ INSERT INTO product_versions (product_version_id, product_name, major_version, release_version, version_string, version_sort, build_date, sunset_date, featured_version, build_type) VALUES ( 1, 'Firefox', '15.0', '15.0', '15.0a1', '000000150a1', %(build_date)s, %(sunset_date)s, true, 'release' ), ( 2, 'Firefox', '24.5', '24.5.0', '24.5.0', '024005000x000', %(build_date)s, %(sunset_date)s, true, 'nightly' ), ( 3, 'Firefox', '49.0.1', '49.0.1', '49.0.1', '000000150a1', %(build_date)s, %(sunset_date)s, false, 'release' ), ( 4, 'Firefox', '50.0b', '50.0b', '50.0b', '024005000x000', %(build_date)s, %(sunset_date)s, false, 'beta' ), ( 5, 'Firefox', '51.0a2', '51.0a2', '51.0a2', '000000150a1', %(build_date)s, %(sunset_date)s, false, 'aurora' ), ( 6, 'Firefox', '52.0a1', '52.0a1', '52.0a1', '024005000x000', %(build_date)s, %(sunset_date)s, false, 'nightly' ) """, { 'build_date': build_date, 'sunset_date': sunset_date }) execute_no_results( self.conn, """ INSERT INTO release_channels (release_channel, sort) VALUES ('nightly', 1), ('aurora', 2), ('beta', 3), ('release', 4) """) execute_no_results( self.conn, """ INSERT INTO product_release_channels (product_name, release_channel, throttle) VALUES ('Firefox', 'nightly', 1), ('Firefox', 'aurora', 1), ('Firefox', 'beta', 1), ('Firefox', 'release', 1), ('Fennec', 'release', 1), ('Fennec', 'beta', 1) """)
def test_scrape_json_releases(self, requests_mocker): today = utc_now() requests_mocker.get(BASE_URL + 'firefox/', text="""<html><body> <a href="candidates/">candidates</a> <a href="nightly/">nightly</a> </body></html>""") requests_mocker.get(BASE_URL + 'firefox/candidates/', text="""<html><body> <a href="28.0-candidates/">28.0-candidiates</a> <a href="10.0b4-candidates/">10.0b4-candidiates</a> <a href="None-candidates/">None-candidiates</a> </body></html>""") requests_mocker.get(BASE_URL + 'firefox/candidates/28.0-candidates/', text="""<html><body> <a href="build1/">build1</a> </body></html>""") requests_mocker.get(BASE_URL + 'firefox/candidates/28.0-candidates/build1/', text="""<html><body> <a href="linux-i686/">linux-i686</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/28.0-candidates/build1/linux-i686/', text="""<html><body> <a href="en-US/">en-US</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/28.0-candidates/build1/linux-i686/en-US/', text="""<html><body> <a href="firefox-28.0.json">firefox-28.0.json</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/28.0-candidates/build1/linux-i686/en-US/firefox-28.0.json', json={ 'buildid': '20140113161827', 'moz_app_maxversion': '28.0.*', 'moz_app_name': 'firefox', 'moz_app_vendor': 'Mozilla', 'moz_app_version': '28.0', 'moz_pkg_platform': 'linux-i686', 'moz_source_repo': 'http://hg.mozilla.org/releases/mozilla-release', 'moz_update_channel': 'release' }) requests_mocker.get(BASE_URL + 'firefox/candidates/10.0b4-candidates/', text="""<html><body> <a href="build1/">build1</a> </body></html>""") requests_mocker.get(BASE_URL + 'firefox/candidates/10.0b4-candidates/build1/', text="""<html><body> <a href="linux-i686/">linux-i686</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/10.0b4-candidates/build1/linux-i686/', text="""<html><body> <a href="en-US/">en-US</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/10.0b4-candidates/build1/linux-i686/en-US/', text="""<html><body> <a href="firefox-10.0b4.json">firefox-10.0b4.json</a> <a href="firefox-10.0b4.en-US.linux-i686.mozinfo.json"> firefox-10.0b4.en-US.linux-i686.mozinfo.json</a> <a href="JUNK.json">JUNK.json</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/10.0b4-candidates/build1/linux-i686/en-US/firefox-10.0b4.json', json={ 'buildid': '20140113161826', 'moz_app_maxversion': '10.0.*', 'moz_app_name': 'firefox', 'moz_app_vendor': 'Mozilla', 'moz_app_version': '27.0', 'moz_pkg_platform': 'linux-i686', 'moz_source_repo': 'http://hg.mozilla.org/releases/mozilla-beta', 'moz_update_channel': 'beta' }) # Ignore unrecognized JSON files, see bug 1065071 requests_mocker.get( BASE_URL + 'firefox/candidates/10.0b4-candidates/build1/linux-i686/en-US/JUNK.json', json={ 'something': 'unexpected', 'nothing': 'else' }) requests_mocker.get(BASE_URL + 'firefox/candidates/None-candidates/', text="""<html><body> <a href="build1/">build1</a> </body></html>""") requests_mocker.get(BASE_URL + 'firefox/candidates/None-candidates/build1/', text="""<html><body> <a href="linux-i686/">linux-i686</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/None-candidates/build1/linux-i686/', text="""<html><body> <a href="en-US/">en-US</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/None-candidates/build1/linux-i686/en-US/', text="""<html><body> <a href="None.json">None.json</a> </body></html>""") requests_mocker.get( BASE_URL + 'firefox/candidates/None-candidates/build1/linux-i686/en-US/None.json', text=""" """) requests_mocker.get(BASE_URL + 'firefox/nightly/', text="""<html><body> <a href="%(year)s/">%(year)s</a> </body></html>""" % {'year': today.strftime('%Y')}) requests_mocker.get(today.strftime(BASE_URL + 'firefox/nightly/%Y/'), text="""<html><body> <a href="%(month)s/">%(month)s</a> </body></html>""" % {'month': today.strftime('%m')}) requests_mocker.get(today.strftime(BASE_URL + 'firefox/nightly/%Y/%m/'), text="""<html><body> <a href="%(date)s-03-02-03-mozilla-central/">txt</a> <a href="%(date)s-03-02-04-mozilla-central/">txt</a> """ % {'date': today.strftime('%Y-%m-%d')}) requests_mocker.get(today.strftime( BASE_URL + 'firefox/nightly/%Y/%m/%Y-%m-%d-03-02-03-mozilla-central/'), text="""<html><body> <a href="firefox-30.0a1.en-US.linux-i686.json">txt</a> </body></html>""") requests_mocker.get(today.strftime( BASE_URL + 'firefox/nightly/%Y/%m/%Y-%m-%d-03-02-03-mozilla-central/' + 'firefox-30.0a1.en-US.linux-i686.json'), json={ 'as': '$(CC)', 'buildid': '20140205030203', 'cc': '/usr/bin/ccache ', 'cxx': '/usr/bin/ccache stuff', 'host_alias': 'x86_64-unknown-linux-gnu', 'host_cpu': 'x86_64', 'host_os': 'linux-gnu', 'host_vendor': 'unknown', 'ld': 'ld', 'moz_app_id': '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}', 'moz_app_maxversion': '30.0a1', 'moz_app_name': 'firefox', 'moz_app_vendor': 'Mozilla', 'moz_app_version': '30.0a1', 'moz_pkg_platform': 'linux-i686', 'moz_source_repo': 'https://hg.mozilla.org/mozilla-central', 'moz_source_stamp': '1f170f9fead0', 'moz_update_channel': 'nightly', 'target_alias': 'i686-pc-linux', 'target_cpu': 'i686', 'target_os': 'linux-gnu', 'target_vendor': 'pc' }) requests_mocker.get(today.strftime( BASE_URL + 'firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/'), text="""<html><body> <a href="firefox-30.0a2.en-US.linux-i686.json">txt</a> </body></html>""") requests_mocker.get(today.strftime( BASE_URL + 'firefox/nightly/%Y/%m/%Y-%m-%d-03-02-04-mozilla-central/' + 'firefox-30.0a2.en-US.linux-i686.json'), json={ 'as': '$(CC)', 'buildid': '20140205030204', 'cc': '/usr/bin/ccache stuff', 'cxx': '/usr/bin/ccache stuff', 'host_alias': 'x86_64-unknown-linux-gnu', 'host_cpu': 'x86_64', 'host_os': 'linux-gnu', 'host_vendor': 'unknown', 'ld': 'ld', 'moz_app_id': '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}', 'moz_app_maxversion': '30.0a2', 'moz_app_name': 'firefox', 'moz_app_vendor': 'Mozilla', 'moz_app_version': '30.0a2', 'moz_pkg_platform': 'linux-i686', 'moz_source_repo': 'https://hg.mozilla.org/mozilla-central', 'moz_source_stamp': '1f170f9fead0', 'moz_update_channel': 'nightly', 'target_alias': 'i686-pc-linux', 'target_cpu': 'i686', 'target_os': 'linux-gnu', 'target_vendor': 'pc' }) config_manager = self._setup_config_manager_firefox() with config_manager.context() as config: tab = CronTabberApp(config) tab.run_all() information = self._load_structure() assert information['ftpscraper'] assert not information['ftpscraper']['last_error'] assert information['ftpscraper']['last_success'] config.logger.warning.assert_any_call( 'Unable to JSON parse content %r', ' ', exc_info=True) config.logger.warning.assert_any_call( 'warning, unsupported JSON file: %s', BASE_URL + 'firefox/candidates/' '10.0b4-candidates/build1/linux-i686/en-US/JUNK.json') cursor = self.conn.cursor() columns = 'product_name', 'build_id', 'build_type' cursor.execute(""" select %s from releases_raw """ % ','.join(columns)) builds = [dict(zip(columns, row)) for row in cursor.fetchall()] build_ids = dict((str(x['build_id']), x) for x in builds) assert '20140113161827' in build_ids assert '20140113161826' in build_ids assert '20140205030203' in build_ids assert len(build_ids) == 4 expected = [{ 'build_id': 20140113161827, 'product_name': 'firefox', 'build_type': 'release' }, { 'build_id': 20140113161827, 'product_name': 'firefox', 'build_type': 'beta' }, { 'build_id': 20140113161826, 'product_name': 'firefox', 'build_type': 'beta' }, { 'build_id': 20140205030203, 'product_name': 'firefox', 'build_type': 'nightly' }, { 'build_id': 20140205030204, 'product_name': 'firefox', 'build_type': 'aurora' }] assert builds == expected
def _run_one(self, job_class, config, force=False): _debug = self.config.logger.debug seconds = convert_frequency(config.frequency) time_ = config.time if not force: if not self.time_to_run(job_class, time_): _debug("skipping %r because it's not time to run", job_class) return ok, dependency_error = self.check_dependencies(job_class) if not ok: _debug("skipping %r dependencies aren't met [%s]", job_class, dependency_error) return _debug('about to run %r', job_class) app_name = job_class.app_name info = self.database.get(app_name) last_success = None now = utc_now() try: t0 = time.time() for last_success in self._run_job(job_class, config, info): t1 = time.time() _debug('successfully ran %r on %s', job_class, last_success) self._remember_success(job_class, last_success, t1 - t0) # _run_job() returns a generator, so we don't know how # many times this will loop. Anyway, we need to reset the # 't0' for the next loop if there is one. t0 = time.time() exc_type = exc_value = exc_tb = None except: t1 = time.time() exc_type, exc_value, exc_tb = sys.exc_info() # when debugging tests that mock logging, uncomment this otherwise # the exc_info=True doesn't compute and record what the exception # was #raise if self.config.sentry and self.config.sentry.dsn: try: client = raven.Client(dsn=self.config.sentry.dsn) identifier = client.get_ident(client.captureException()) self.config.logger.info( 'Error captured in Sentry. Reference: %s' % identifier) except Exception: # Blank exceptions like this is evil but a failure to send # the exception to Sentry is much less important than for # crontabber to carry on. This is especially true # considering that raven depends on network I/O. _debug('Failed to capture and send error to Sentry', exc_info=True) _debug('error when running %r on %s', job_class, last_success, exc_info=True) self._remember_failure(job_class, t1 - t0, exc_type, exc_value, exc_tb) finally: self._log_run(job_class, seconds, time_, last_success, now, exc_type, exc_value, exc_tb)
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os from django.conf import settings from crashstats.crashstats.models import MissingProcessedCrash from crashstats.crashstats.management.commands.verifyprocessed import Command from socorro.lib.datetimeutil import utc_now from socorro.lib.ooid import create_new_ooid, date_from_ooid TODAY = utc_now().strftime("%Y%m%d") BUCKET_NAME = os.environ.get("resource.boto.bucket_name") def get_small_entropy(self): """Returns small entropy so we're not spending ages cycling through things.""" yield from ["000", "111", "222"] class TestVerifyProcessed: def fetch_crashids(self): return MissingProcessedCrash.objects.order_by("crash_id").values_list( "crash_id", flat=True) def create_raw_crash_in_s3(self, boto_helper, crash_id): boto_helper.upload_fileobj( bucket_name=BUCKET_NAME, key="v2/raw_crash/%s/%s/%s" % (crash_id[0:3], TODAY, crash_id),
def test_basic_run_no_errors(self): # a mutable where commands sent are stored commands_sent = [] self.Popen.side_effect = functools.partial( mocked_Popen, _commands_sent=commands_sent, _exit_code=0, _stdout='Bla bla', _stderr='', ) config_manager = self._setup_config_manager() with config_manager.context() as config: tab = crontabber.CronTabber(config) tab.run_all() information = self._load_structure() assert information['modulelist'] #print information['modulelist']['last_error'] #print information['modulelist']['last_error']['traceback'] if information['modulelist']['last_error']: raise AssertionError(information['modulelist']['last_error']) assert len(commands_sent) == 3 first = commands_sent[0] second = commands_sent[1] third = commands_sent[2] yesterday = utc_now() yesterday -= datetime.timedelta(days=1) yesterday_fmt = yesterday.strftime('%Y%m%d') self.assertTrue( 'PIG_CLASSPATH=/some/place pig' in first ) self.assertTrue( '-param start_date=%s' % yesterday_fmt in first ) self.assertTrue( '-param end_date=%s' % yesterday_fmt in first ) self.assertTrue( '/some/place/modulelist.pig' in first ) self.assertTrue( 'PIG_CLASSPATH=/some/place hadoop fs -getmerge' in second ) self.assertTrue( 'modulelist-%s-%s' % (yesterday_fmt, yesterday_fmt) in second ) self.assertTrue( '/some/other/place/%s-modulelist.txt' % (yesterday_fmt,) in second ) self.assertTrue( 'PIG_CLASSPATH=/some/place hadoop fs ' in third ) self.assertTrue( 'modulelist-%s-%s' % (yesterday_fmt, yesterday_fmt) in second ) # note that all jobs spew out 'Bla bla' on stdout config.logger.info.assert_called_with('Bla bla')
def setUp(self): """ Populate tables with fake data """ super(TestUtil, self).setUp() cursor = self.connection.cursor() self.now = datetimeutil.utc_now() now = self.now.date() cursor.execute(""" INSERT INTO products (product_name, sort, rapid_release_version, release_name) VALUES ( 'Firefox', 1, '8.0', 'firefox' ), ( 'Fennec', 2, '11.0', 'mobile' ), ( 'Thunderbird', 3, '10.0', 'thunderbird' ), ( 'WaterWolf', 4, '100.0', 'waterwolf' ); """) cursor.execute(""" INSERT INTO release_channels (release_channel, sort) VALUES ( 'Release', 1 ), ( 'Beta', 2 ); """) cursor.execute(""" INSERT INTO product_release_channels (product_name, release_channel, throttle) VALUES ( 'Firefox', 'Release', '0.1' ), ( 'Fennec', 'Release', '0.1' ), ( 'Fennec', 'Beta', '1.0' ), ( 'Thunderbird', 'Release', '0.1' ), ( 'WaterWolf', 'Beta', '0.1' ); """) cursor.execute(""" INSERT INTO product_versions (product_version_id, product_name, major_version, release_version, version_string, build_date, sunset_date, featured_version, build_type, version_sort, is_rapid_beta, rapid_beta_id) VALUES ( 1, 'Firefox', '8.0', '8.0', '8.0', '%(now)s', '%(now)s', False, 'Release', '0008000', False, NULL ), ( 2, 'Fennec', '11.0', '11.0', '11.0.1', '%(now)s', '%(now)s', False, 'Release', '0011001', False, NULL ), ( 3, 'Fennec', '12.0', '12.0', '12.0b1', '%(now)s', '%(now)s', False, 'Beta', '00120b1', False, NULL ), ( 4, 'Thunderbird', '10.0', '10.0', '10.0.2b', '%(now)s', '%(now)s', False, 'Release', '001002b', False, NULL ), ( 5, 'WaterWolf', '3.0b', '3.0b', '3.0b', '%(now)s', '%(now)s', False, 'Beta', '003002b000', True, 4 ), ( 6, 'WaterWolf', '3.0b', '3.0b', '3.0b1', '%(now)s', '%(now)s', False, 'Beta', '003002b001', False, 5 ), ( 7, 'WaterWolf', '3.0b', '3.0b', '3.0b2', '%(now)s', '%(now)s', False, 'Beta', '003002b002', False, 5 ); """ % {'now': now}) cursor.execute(""" INSERT INTO product_version_builds (build_id, platform, product_version_id) VALUES (1, 'Linux', 1), (2, 'Linux', 2), (3, 'Linux', 3), (4, 'Linux', 4), (5, 'Linux', 5), (6, 'Linux', 6), (7, 'Linux', 7); """) self.connection.commit()
def main(argv=None): parser = argparse.ArgumentParser(formatter_class=WrappedTextHelpFormatter, description=DESCRIPTION.strip()) parser.add_argument("--host", default=DEFAULT_HOST, help="host for system to fetch crashids from") parser.add_argument( "--date", default="", help= ('date to pull crash ids from as YYYY-MM-DD, "yesterday", "today", or "now"; ' 'defaults to "yesterday"'), ) parser.add_argument( "--signature-contains", default="", dest="signature", help="signature contains this string", ) parser.add_argument( "--product", default="Firefox", help='Product to fetch for or "all" for all; defaults to "Firefox"', ) parser.add_argument("--url", default="", help="Super Search url to base query on") parser.add_argument( "--num", default=100, help='number of crash ids you want or "all" for all of them', ) parser.add_argument("-v", "--verbose", action="store_true", help="increase verbosity of output") if argv is None: args = parser.parse_args() else: args = parser.parse_args(argv) host = args.host.rstrip("/") # Start with params from --url value or empty dict if args.url: params = extract_params(args.url) else: params = {} params["_columns"] = "uuid" params["product"] = params.get("product", args.product) if params["product"] == "all": # If the user specified "all", then we don't want to pass a product filter # in del params["product"] # Override with date if specified if "date" not in params or args.date: datestamp = args.date or "yesterday" if datestamp == "now": # Create a start -> end window that has wiggle room on either side # to deal with time differences between the client and server, but # also big enough to pick up results even in stage where it doesn't # process much enddate = utc_now() + datetime.timedelta(hours=1) startdate = enddate - datetime.timedelta(hours=12) # For "now", we want precision so we don't hit cache and we want to # sort by reverse date so that we get the most recent crashes startdate = startdate.strftime("%Y-%m-%dT%H:%M:%S.000Z") enddate = enddate.strftime("%Y-%m-%dT%H:%M:%S.000Z") params["_sort"] = "-date" else: if datestamp == "today": startdate = utc_now() elif datestamp == "yesterday": startdate = utc_now() - datetime.timedelta(days=1) else: startdate = datetime.datetime.strptime(datestamp, "%Y-%m-%d") enddate = startdate + datetime.timedelta(days=1) # For "today", "yesterday", and other dates, we want a day # precision so that Socorro can cache it startdate = startdate.strftime("%Y-%m-%d") enddate = enddate.strftime("%Y-%m-%d") params["date"] = [">=%s" % startdate, "<%s" % enddate] # Override with signature-contains if specified sig = args.signature if sig: params["signature"] = "~" + sig num_results = args.num if num_results == "all": num_results = INFINITY else: try: num_results = int(num_results) except ValueError: print('num needs to be an integer or "all"') return 1 if args.verbose: print("Params: %s" % params) for crashid in fetch_crashids(host, params, num_results): print(crashid) return 0
def main(argv=None): parser = argparse.ArgumentParser( formatter_class=WrappedTextHelpFormatter, description=DESCRIPTION.strip(), ) parser.add_argument('--host', default=DEFAULT_HOST, help='host for system to fetch crashids from') parser.add_argument( '--date', default='', help= ('date to pull crash ids from as YYYY-MM-DD, "yesterday", "today", or "now"; ' 'defaults to "yesterday"')) parser.add_argument('--signature-contains', default='', dest='signature', help='signature contains this string') parser.add_argument('--url', default='', help='Super Search url to base query on') parser.add_argument( '--num', default=100, help='number of crash ids you want or "all" for all of them') parser.add_argument('-v', '--verbose', action='store_true', help='increase verbosity of output') if argv is None: args = parser.parse_args() else: args = parser.parse_args(argv) host = args.host.rstrip('/') # Start with params from --url value or product=Firefox if args.url: params = extract_params(args.url) else: params = {'product': 'Firefox'} params['_columns'] = 'uuid' # Override with date if specified if 'date' not in params or args.date: datestamp = args.date or 'yesterday' if datestamp == 'now': # Create a start -> end window that has wiggle room on either side # to deal with time differences between the client and server, but # also big enough to pick up results even in stage where it doesn't # process much enddate = utc_now() + datetime.timedelta(hours=1) startdate = enddate - datetime.timedelta(hours=12) # For "now", we want precision so we don't hit cache and we want to # sort by reverse date so that we get the most recent crashes startdate = startdate.strftime('%Y-%m-%dT%H:%M:%S.000Z') enddate = enddate.strftime('%Y-%m-%dT%H:%M:%S.000Z') params['_sort'] = '-date' else: if datestamp == 'today': startdate = utc_now() elif datestamp == 'yesterday': startdate = utc_now() - datetime.timedelta(days=1) else: startdate = datetime.datetime.strptime(datestamp, '%Y-%m-%d') enddate = startdate + datetime.timedelta(days=1) # For "today", "yesterday", and other dates, we want a day # precision so that Socorro can cache it startdate = startdate.strftime('%Y-%m-%d') enddate = enddate.strftime('%Y-%m-%d') params['date'] = ['>=%s' % startdate, '<%s' % enddate] # Override with signature-contains if specified sig = args.signature if sig: params['signature'] = '~' + sig num_results = args.num if num_results == 'all': num_results = INFINITY else: try: num_results = int(num_results) except ValueError: print('num needs to be an integer or "all"') return 1 if args.verbose: print('Params: %s' % params) for crashid in fetch_crashids(host, params, num_results): print(crashid) return 0