def get(self, **kwargs): """Return JSON data of a crash report, given its uuid. """ filters = [ ('uuid', None, 'str'), ('datatype', None, 'str') ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingArgumentError('uuid') if not params.datatype: raise MissingArgumentError('datatype') store = self.config.filesystem.filesystem_class(self.config.filesystem) datatype_method_mapping = { 'raw': 'get_raw_dump', 'meta': 'get_raw_crash', 'processed': 'get_processed' } get = store.__getattribute__(datatype_method_mapping[params.datatype]) try: if params.datatype == 'raw': return (get(params.uuid), 'application/octet-stream') else: return get(params.uuid) except CrashIDNotFound: if params.datatype == 'processed': self.get(datatype='raw', uuid=params.uuid) j = priorityjobs.Priorityjobs(config=self.config) j.create(uuid=params.uuid) raise ResourceUnavailable(params.uuid) raise ResourceNotFound(params.uuid)
def post(self, **kwargs): params = external_common.parse_arguments(self.filters, kwargs) if not params.category: raise MissingArgumentError('category') if not params.rule: raise MissingArgumentError('rule') sql = """ /* socorro.external.postgresql.skiplist.SkipList.post */ INSERT INTO skiplist (category, rule) VALUES (%s, %s); """ sql_params = [params.category, params.rule] connection = self.database.connection() try: with connection.cursor() as cur: cur.execute(sql, sql_params) connection.commit() except psycopg2.Error: connection.rollback() error_message = "Failed updating skip list in PostgreSQL" logger.error(error_message) raise DatabaseError(error_message) finally: connection.close() return True
def get(self, **kwargs): """Return JSON data of a crash report, given its uuid. """ filters = [ ('uuid', None, 'str'), ('datatype', None, 'str'), ('name', None, 'str') # only applicable if datatype == 'raw' ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingArgumentError('uuid') if not params.datatype: raise MissingArgumentError('datatype') # get a generic crashstorage instance from whatever external resource # is implementing this service. store = self.get_storage() datatype_method_mapping = { 'raw': 'get_raw_dump', 'meta': 'get_raw_crash', 'processed': 'get_processed', 'unredacted': 'get_unredacted_processed', } get = store.__getattribute__(datatype_method_mapping[params.datatype]) try: if params.datatype == 'raw': return (get(params.uuid, name=params.name), 'application/octet-stream') else: return get(params.uuid) except CrashIDNotFound: if params.datatype in ('processed', 'unredacted'): # try to fetch a raw crash just to ensure that the raw crash # exists. If this line fails, there's no reason to actually # submit the priority job. try: store.get_raw_crash(params.uuid) except CrashIDNotFound: raise ResourceNotFound(params.uuid) # search through the existing other services to find the # Priorityjob service. try: priorityjob_service_impl = self.all_services[ 'Priorityjobs'] except KeyError: raise ServiceUnavailable('Priorityjobs') # get the underlying implementation of the Priorityjob # service and instantiate it. priority_job_service = priorityjob_service_impl.cls( config=self.config) # create the priority job for this crash_ids priority_job_service.create(uuid=params.uuid) raise ResourceUnavailable(params.uuid) raise ResourceNotFound(params.uuid)
def get(self, **kwargs): """Return JSON data of a crash report, given its uuid. """ filters = [ ("uuid", None, "str"), ("datatype", None, "str") ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingArgumentError('uuid') if not params.datatype: raise MissingArgumentError('datatype') if hasattr(self.config, 'hbase'): config = self.config.hbase store = crashstorage.HBaseCrashStorage(config) datatype_method_mapping = { "raw": "get_raw_dump", "meta": "get_raw_crash", "processed": "get_processed" } else: # old middleware config = self.config import socorro.storage.crashstorage as cs store = cs.CrashStoragePool( config, storageClass=config.hbaseStorageClass ).crashStorage() datatype_method_mapping = { "raw": "get_raw_dump", "meta": "get_meta", "processed": "get_processed" } get = store.__getattribute__(datatype_method_mapping[params.datatype]) try: if params.datatype == 'raw': return (get(params.uuid), 'application/octet-stream') else: return get(params.uuid) except (CrashIDNotFound, OoidNotFoundException): if params.datatype == 'processed': self.get(datatype='raw', uuid=params.uuid) j = priorityjobs.Priorityjobs(config=self.config) j.create(uuid=params.uuid) raise ResourceUnavailable(params.uuid) raise ResourceNotFound(params.uuid)
def get(self, **kwargs): '''Return data about a field from its name. ''' filters = [ ('name', None, 'str'), ] params = external_common.parse_arguments(filters, kwargs) if not params.name: raise MissingArgumentError("name") sql = '''/* socorro.external.postgresql.field.Field.get */ SELECT raw_field, transforms, product FROM data_dictionary WHERE raw_field=%(name)s ''' error_message = 'Failed to retrieve field data from PostgreSQL' results = self.query(sql, params, error_message=error_message) field_data = {'name': None, 'transforms': None, 'product': None} if not results: return field_data field_data = dict(zip(('name', 'transforms', 'product'), results[0])) return field_data
def get(self, **kwargs): """Return a job in the priority queue. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingArgumentError('uuid') sql = """ /* socorro.external.postgresql.priorityjobs.Priorityjobs.get */ SELECT uuid FROM priorityjobs WHERE uuid=%(uuid)s """ error_message = "Failed to retrieve priorityjobs data from PostgreSQL" results = self.query(sql, params, error_message=error_message) jobs = [] for row in results: job = dict(zip(("uuid",), row)) jobs.append(job) return { "hits": jobs, "total": len(jobs) }
def get(self, **kwargs): """Return a job in the job queue. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingArgumentError('uuid') fields = [ "id", "pathname", "uuid", "owner", "priority", "queueddatetime", "starteddatetime", "completeddatetime", "success", "message" ] sql = """ /* socorro.external.postgresql.job.Job.get */ SELECT %s FROM jobs WHERE uuid=%%(uuid)s """ % ", ".join(fields) error_message = "Failed to retrieve jobs data from PostgreSQL" results = self.query(sql, params, error_message=error_message) jobs = [] for row in results: job = dict(zip(fields, row)) # Make sure all dates are turned into strings for i in job: if isinstance(job[i], datetime.datetime): job[i] = datetimeutil.date_to_string(job[i]) jobs.append(job) return {"hits": jobs, "total": len(jobs)}
def post(self, **kwargs): params = external_common.parse_arguments(self.filters, kwargs) if not params['signatures']: raise MissingArgumentError('signatures') sql_params = [tuple(params['signatures'])] sql = """ SELECT signature, first_report AS first_date, first_build FROM signatures WHERE signature IN %s """ error_message = 'Failed to retrieve signatures from PostgreSQL' results = self.query(sql, sql_params, error_message=error_message) signatures = [] for sig in results.zipped(): sig.first_date = datetimeutil.date_to_string(sig.first_date) signatures.append(sig) return {'hits': signatures, 'total': len(signatures)}
def get(self, **kwargs): filters = [ ("vendor_hex", None, "str"), ("adapter_hex", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) for key in ('vendor_hex', 'adapter_hex'): if not params[key]: raise MissingArgumentError(key) sql_where = """ WHERE vendor_hex = %(vendor_hex)s AND adapter_hex = %(adapter_hex)s """ sql_query = """ SELECT vendor_hex, adapter_hex, vendor_name, adapter_name FROM graphics_device """ results = self.query(sql_query + sql_where, params) keys = 'vendor_hex', 'adapter_hex', 'vendor_name', 'adapter_name' hits = [dict(zip(keys, x)) for x in results] return {'hits': hits, 'total': len(hits)}
def post(self, **kwargs): """Return a list of signature - bug id associations. """ params = external_common.parse_arguments(self.filters, kwargs) if not params.signatures: raise MissingArgumentError('signatures') # Preparing variables for the SQL query signatures = [] sql_params = {} for i, elem in enumerate(params.signatures): signatures.append("%%(signature%s)s" % i) sql_params["signature%s" % i] = elem sql = """/* socorro.external.postgresql.bugs.Bugs.get */ SELECT ba.signature, bugs.id FROM bugs JOIN bug_associations AS ba ON bugs.id = ba.bug_id WHERE EXISTS( SELECT 1 FROM bug_associations WHERE bug_associations.bug_id = bugs.id AND signature IN (%s) ) """ % ", ".join(signatures) sql = str(" ".join(sql.split())) # better formatting of the sql string error_message = "Failed to retrieve bugs associations from PostgreSQL" results = self.query(sql, sql_params, error_message=error_message) bugs = [] for row in results: bug = dict(zip(("signature", "id"), row)) bugs.append(bug) return {"hits": bugs, "total": len(bugs)}
def get(self, **kwargs): """Return a dict that holds the throttling value per build type for a specific product.""" filters = [ ('product', None, 'str'), ] params = external_common.parse_arguments(filters, kwargs) required = ('product',) for key in required: if not params.get(key): raise MissingArgumentError(key) sql = """ SELECT build_type, throttle FROM product_build_types WHERE product_name = %(product)s """ results = self.query(sql, params) build_types = {} for row in results.zipped(): build_types[row.build_type] = row.throttle return { 'hits': build_types, }
def create(self, **kwargs): """Add a new job to the priority queue """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingArgumentError('uuid') with self.context() as connection: try: self.config.logger.debug( 'Inserting priority job into RabbitMQ %s', params.uuid) connection.channel.basic_publish( exchange='', routing_key=self.config.priority_queue_name, body=params.uuid, properties=pika.BasicProperties(delivery_mode=2)) except ChannelClosed: self.config.logger.error( "Failed inserting priorityjobs data into RabbitMQ", exc_info=True) return False return True
def get(self, **kwargs): filters = [ ("vendor_hex", None, ["list", "str"]), ("adapter_hex", None, ["list", "str"]), ] params = external_common.parse_arguments(filters, kwargs) for key in ('vendor_hex', 'adapter_hex'): param = params[key] if not param: raise MissingArgumentError(key) params[key] = tuple(params[key]) sql_query = """ SELECT vendor_hex, adapter_hex, vendor_name, adapter_name FROM graphics_device WHERE vendor_hex IN %(vendor_hex)s AND adapter_hex IN %(adapter_hex)s """ results = self.query(sql_query, params) hits = results.zipped() return { 'hits': hits, 'total': len(hits) }
def get_paireduuid(self, **kwargs): """Return paired uuid given a uuid and an optional hangid. If a hangid is passed, then return only one result. Otherwise, return all found paired uuids. """ filters = [ ("uuid", None, "str"), ("hangid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingArgumentError('uuid') crash_date = datetimeutil.uuid_to_date(params.uuid) sql = """ /* socorro.external.postgresql.crashes.Crashes.get_paireduuid */ SELECT uuid FROM reports r WHERE r.uuid != %(uuid)s AND r.date_processed BETWEEN TIMESTAMP %(crash_date)s - CAST('1 day' AS INTERVAL) AND TIMESTAMP %(crash_date)s + CAST('1 day' AS INTERVAL) """ sql_params = {"uuid": params.uuid, "crash_date": crash_date} if params.hangid is not None: sql = """%s AND r.hangid = %%(hangid)s LIMIT 1 """ % sql sql_params["hangid"] = params.hangid else: sql = """%s AND r.hangid IN ( SELECT hangid FROM reports r2 WHERE r2.date_processed BETWEEN TIMESTAMP %%(crash_date)s - CAST('1 day' AS INTERVAL) AND TIMESTAMP %%(crash_date)s + CAST('1 day' AS INTERVAL) AND r2.uuid = %%(uuid)s ) """ % sql # Query the database error_message = "Failed to retrieve paired uuids from PostgreSQL" results = self.query(sql, sql_params, error_message=error_message) # Transforming the results into what we want uuids = [dict(zip(("uuid", ), row)) for row in results] return {"hits": uuids, "total": len(uuids)}
def get_adu_by_signature(self, **kwargs): """Return a list of ADUs and crash counts by signature and ADU date """ now = datetimeutil.utc_now().date() lastweek = now - datetime.timedelta(weeks=1) filters = [ ("start_date", lastweek, "date"), ("end_date", now, "date"), ("signature", None, "str"), ("channel", None, "str"), ("product_name", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) for param in ("start_date", "end_date", "signature", "channel"): if not params[param]: raise MissingArgumentError(param) if (params.end_date - params.start_date) > datetime.timedelta(days=365): raise BadArgumentError('Duration too long. Max 365 days.') sql_query = """ SELECT product_name, signature, adu_date::TEXT, build_date::TEXT, buildid::TEXT, crash_count, adu_count, os_name, channel FROM crash_adu_by_build_signature WHERE adu_date BETWEEN %(start_date)s AND %(end_date)s AND product_name = %(product_name)s AND channel = %(channel)s AND signature = %(signature)s ORDER BY buildid """ error_message = ( "Failed to retrieve crash ADU by build signature from PostgreSQL") results = self.query(sql_query, params, error_message=error_message) fields = [ 'product_name', 'signature', 'adu_date', 'build_date', 'buildid', 'crash_count', 'adu_count', 'os_name', 'channel' ] crashes = [dict(zip(fields, row)) for row in results] return {"hits": crashes, "total": len(crashes)}
def create_field(self, **kwargs): """Create a new field in the database, to be used by supersearch and all elasticsearch related services. """ filters = [ ('name', None, 'str'), ('data_validation_type', 'enum', 'str'), ('default_value', None, 'str'), ('description', None, 'str'), ('form_field_choices', None, ['list', 'str']), ('has_full_version', False, 'bool'), ('in_database_name', None, 'str'), ('is_exposed', False, 'bool'), ('is_returned', False, 'bool'), ('is_mandatory', False, 'bool'), ('query_type', 'enum', 'str'), ('namespace', None, 'str'), ('permissions_needed', None, ['list', 'str']), ('storage_mapping', None, 'json'), ] params = external_common.parse_arguments(filters, kwargs) mandatory_params = ('name', 'in_database_name') for param in mandatory_params: if not params[param]: raise MissingArgumentError(param) # Before making the change, make sure it does not break indexing. new_mapping = self.get_mapping(overwrite_mapping=params) # Try the mapping. If there is an error, an exception will be raised. # If an exception is raised, the new mapping will be rejected. self.test_mapping(new_mapping) es_connection = self.get_connection().get_es() try: es_connection.index( index=self.config.elasticsearch_default_index, doc_type='supersearch_fields', doc=params, id=params['name'], overwrite_existing=False, refresh=True, ) except ElasticHttpError, e: if e.status_code == 409: # This field exists in the database, it thus cannot be created! raise InsertionError( 'The field "%s" already exists in the database, ' 'impossible to create it. ' % params['name']) # Else this is an unexpected error and we want to know about it. raise
def get(self, **kwargs): """Return a single crash report from its UUID. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if params.uuid is None: raise MissingArgumentError("uuid") crash_date = datetimeutil.uuid_to_date(params.uuid) logger.debug("Looking for crash %s during day %s" % (params.uuid, crash_date)) sql = """/* socorro.external.postgresql.crash.Crash.get */ SELECT reports.signature, reports.email, reports.url, reports.addons_checked, reports.exploitability, ( SELECT reports_duplicates.duplicate_of FROM reports_duplicates WHERE reports_duplicates.uuid = reports.uuid ) as duplicate_of FROM reports WHERE reports.uuid=%(uuid)s AND reports.success IS NOT NULL AND utc_day_is( reports.date_processed, %(crash_date)s) """ sql_params = { "uuid": params.uuid, "crash_date": crash_date } error_message = "Failed to retrieve crash data from PostgreSQL" results = self.query(sql, sql_params, error_message=error_message) crashes = [] for row in results: crash = dict(zip(( "signature", "email", "url", "addons_checked", "exploitability", "duplicate_of"), row)) crashes.append(crash) return { "hits": crashes, "total": len(crashes) }
def get(self, **kwargs): '''Return the result of a custom query. ''' params = external_common.parse_arguments(self.filters, kwargs) if not params.query: raise MissingArgumentError('query') try: query = json.loads(params.query) except ValueError: raise BadArgumentError( 'query', msg="Invalid JSON value for parameter 'query'" ) es = pyelasticsearch.ElasticSearch( urls=self.config.elasticsearch_urls, timeout=self.config.elasticsearch_timeout_extended, ) # Set indices. indices = [] if not params.indices: # By default, use the last two indices. today = utc_now() last_week = today - datetime.timedelta(days=7) indices = self.generate_list_of_indexes(last_week, today) elif len(params.indices) == 1 and params.indices[0] == 'ALL': # If we want all indices, just do nothing. pass else: indices = params.indices search_args = {} if indices: search_args['index'] = indices search_args['doc_type'] = self.config.elasticsearch_doctype try: results = es.search( query, **search_args ) except ElasticHttpNotFoundError, e: missing_index = re.findall(BAD_INDEX_REGEX, e.error)[0] raise ResourceNotFound( "elasticsearch index '%s' does not exist" % missing_index )
def prepare_search_params(self, **kwargs): """Return a dictionary of parameters for a search-like SQL query. Uses socorro.lib.search_common.get_parameters() for arguments filtering. """ params = search_common.get_parameters(kwargs) if not params["signature"]: raise MissingArgumentError('signature') params["terms"] = params["signature"] params["search_mode"] = "is_exactly" # Default mode falls back to starts_with for postgres if params["plugin_search_mode"] == "default": params["plugin_search_mode"] = "starts_with" # Searching for terms in plugins if params["report_process"] == "plugin" and params["plugin_terms"]: params["plugin_terms"] = " ".join(params["plugin_terms"]) params["plugin_terms"] = Crashes.prepare_terms( params["plugin_terms"], params["plugin_search_mode"] ) # Get information about the versions util_service = Util(config=self.context) params["versions_info"] = util_service.versions_info(**params) # Parsing the versions params["versions_string"] = params["versions"] (params["versions"], params["products"]) = Crashes.parse_versions( params["versions"], params["products"] ) # Changing the OS ids to OS names if hasattr(self.context, 'webapi'): context = self.context.webapi else: # old middleware context = self.context for i, elem in enumerate(params["os"]): for platform in context.platforms: if platform["id"] == elem: params["os"][i] = platform["name"] return params
def delete(self, **kwargs): params = external_common.parse_arguments(self.filters, kwargs) if not params.category: raise MissingArgumentError('category') if not params.rule: raise MissingArgumentError('rule') sql_params = [params.category, params.rule] count_sql = """ /* socorro.external.postgresql.skiplist.SkipList.delete */ SELECT COUNT(*) FROM skiplist WHERE category=%s AND rule=%s """ sql = """ /* socorro.external.postgresql.skiplist.SkipList.delete */ DELETE FROM skiplist WHERE category=%s AND rule=%s """ connection = self.database.connection() try: cur = connection.cursor() count = self.count(count_sql, sql_params, connection=connection) if not count: return False cur.execute(sql, sql_params) connection.commit() except psycopg2.Error: connection.rollback() error_message = "Failed delete skip list in PostgreSQL" logger.error(error_message) raise DatabaseError(error_message) finally: connection.close() return True
def create(self, **kwargs): """Add a new job to the priority queue if not already in that queue. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingArgumentError('uuid') sql = """ /* socorro.external.postgresql.priorityjobs.Priorityjobs.create */ INSERT INTO priorityjobs (uuid) VALUES (%(uuid)s) """ sql_exists = """ /* socorro.external.postgresql.priorityjobs.Priorityjobs.create */ SELECT 1 FROM priorityjobs WHERE uuid=%(uuid)s """ with self.get_connection() as connection: try: cur = connection.cursor() # Verifying that the uuid is not already in the queue cur.execute(sql_exists, params) if cur.rowcount: logger.debug('The uuid %s is already in the priorityjobs ' 'table' % params.uuid) return False logger.debug('Adding the uuid %s to the priorityjobs table' % params.uuid) cur.execute(sql, params) except psycopg2.Error: error = "Failed inserting priorityjobs data into PostgreSQL" logger.error(error, exc_info=True) connection.rollback() return False else: connection.commit() return bool(cur.rowcount) finally: cur.close() return True
def get_count_by_day(self, **kwargs): """Returns the number of crashes on a daily basis""" filters = [ ("signature", None, "str"), ("start_date", None, "date"), ("end_date", None, "date") ] DATE_FORMAT = "%Y-%m-%d" params = external_common.parse_arguments(filters, kwargs) for param in ("signature", "start_date"): if not params[param]: raise MissingArgumentError(param) if not params.end_date: params.end_date = params.start_date + datetime.timedelta(1) sql = """ SELECT COUNT(*), date_processed::date FROM reports_clean rc JOIN signatures ON rc.signature_id=signatures.signature_id WHERE rc.date_processed >= %(start_date)s AND rc.date_processed::date < %(end_date)s AND signatures.signature=%(signature)s GROUP BY rc.date_processed::date """ hits = {} for count, date in self.query(sql, params): hits[date.strftime(DATE_FORMAT)] = count current = params.start_date while current < params.end_date: hits.setdefault(current.strftime(DATE_FORMAT), 0) current += datetime.timedelta(1) return {"hits": hits, "total": len(hits)}
def post(self, **kwargs): """Return a list of signatures-to-bug_ids or bug_ids-to-signatures associations. """ params = external_common.parse_arguments(self.filters, kwargs) if not params['signatures'] and not params['bug_ids']: raise MissingArgumentError('specify one of signatures or bug_ids') elif params['signatures'] and params['bug_ids']: raise BadArgumentError('specify only one of signatures or bug_ids') sql_params = [] if params['signatures']: sql_params.append(tuple(params.signatures)) sql = """/* socorro.external.postgresql.bugs.Bugs.get */ SELECT ba.signature, bugs.id FROM bugs JOIN bug_associations AS ba ON bugs.id = ba.bug_id WHERE EXISTS( SELECT 1 FROM bug_associations WHERE bug_associations.bug_id = bugs.id AND signature IN %s ) """ elif params['bug_ids']: sql_params.append(tuple(params.bug_ids)) sql = """/* socorro.external.postgresql.bugs.Bugs.get */ SELECT ba.signature, bugs.id FROM bugs JOIN bug_associations AS ba ON bugs.id = ba.bug_id WHERE bugs.id IN %s """ error_message = "Failed to retrieve bug associations from PostgreSQL" results = self.query(sql, sql_params, error_message=error_message) bugs = [] for row in results: bug = dict(zip(("signature", "id"), row)) bugs.append(bug) return {"hits": bugs, "total": len(bugs)}
def get(self, **kwargs): """ return GC crashes per build ID """ for arg in ['product', 'version']: if not kwargs.get(arg): raise MissingArgumentError(arg) now = datetimeutil.utc_now().date() lastweek = now - datetime.timedelta(weeks=1) filters = [ ("product", None, "str"), ("version", None, "str"), ("from_date", lastweek, "date"), ("to_date", now, "date"), ] params = external_common.parse_arguments(filters, kwargs) result = self.query( """ /* socorro.external.postgresql.gccrashes.GCCrashes.get */ SELECT build::text, sum(gc_count_madu) FROM gccrashes JOIN product_versions USING (product_version_id) WHERE product_name = %(product)s AND version_string = %(version)s AND report_date BETWEEN %(from_date)s AND %(to_date)s AND build IS NOT NULL GROUP BY build ORDER BY build """, params) # Because we don't return a list of dicts, we turn it into a # pure list first so it becomes a list of tuples. rows = list(result) return {'hits': rows, 'total': len(rows)}
def get(self, **kwargs): filters = [ ("backfill_type", None, "str"), ("reports_clean", True, "bool"), ("check_period", '01:00:00', "str"), ("table_name", None, "str"), ("update_day", None, "datetime"), ("start_date", None, "datetime"), ("end_date", None, "datetime"), ] params = external_common.parse_arguments(filters, kwargs) if not params.backfill_type: raise MissingArgumentError('backfill_type') date_param = ['update_day', 'start_date', 'end_date'] for i in date_param: if i in kwargs: params[i] = str(params[i].date()) try: query = 'SELECT backfill_%(backfill_type)s (%(params)s); ' required_params = BACKFILL_PARAMETERS[params.backfill_type] query_params = [(i, params[i]) for i in required_params] query_params_str = ', '.join('%(' + str(i[0]) + ')s' for i in query_params) query = query % { 'backfill_type': params.backfill_type, 'params': query_params_str } except: raise BadArgumentError(kwargs['backfill_type']) error_message = "Failed to retrieve backfill %s from PostgreSQL" error_message = error_message % kwargs['backfill_type'] results = self.query(query, params, error_message=error_message) return results
def get(self, **kwargs): """Return a single error report from its UUID. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if params.uuid is None: raise MissingArgumentError('uuid') crash_date = datetimeutil.uuid_to_date(params.uuid) sql = """/* socorro.external.postgresql.error.Error.get */ SELECT bixie.crashes.signature, bixie.crashes.product, bixie.crashes.error FROM bixie.crashes WHERE bixie.crashes.crash_id=%(uuid)s AND bixie.crashes.success IS NOT NULL AND utc_day_is( bixie.crashes.processor_completed_datetime, %(crash_date)s ) """ sql_params = {"uuid": params.uuid, "crash_date": crash_date} error_message = "Failed to retrieve error data from PostgreSQL" results = self.query(sql, sql_params, error_message=error_message) errors = [] for row in results: error = dict(zip(("product", "error", "signature"), row)) errors.append(error) return {"hits": errors, "total": len(errors)}
def delete_field(self, **kwargs): """Remove a field from the database. Removing a field means that it won't be indexed in elasticsearch anymore, nor will it be exposed or accessible via supersearch. It doesn't delete the data from crash reports though, so it would be possible to re-create the field and reindex some indices to get that data back. """ filters = [ ('name', None, 'str'), ] params = external_common.parse_arguments(filters, kwargs) if not params['name']: raise MissingArgumentError('name') es_connection = self.get_connection() es_connection.delete( index=self.config.elasticsearch.elasticsearch_default_index, doc_type='supersearch_fields', id=params['name'], refresh=True, )
def get_parameters(self, **kwargs): parameters = {} for param in self.filters: values = kwargs.get(param.name, param.default) if values in ('', []): # Those values are equivalent to None here. # Note that we cannot use bool(), because 0 is not equivalent # to None in our case. values = None if values is None and param.mandatory: raise MissingArgumentError(param.name) if values is None and param.default is not None: values = param.default # all values can be a list, so we make them all lists to simplify if values is not None and not isinstance(values, (list, tuple)): values = [values] if values is not None: no_operator_param = None for value in values: operator = None operator_not = False operators = OPERATORS_MAP.get(param.data_type, OPERATORS_MAP['default']) if isinstance(value, basestring): if value.startswith(OPERATOR_NOT): operator_not = True value = value[1:] for ope in operators: if value.startswith(ope): operator = ope value = value[len(ope):] break # ensure the right data type try: value = convert_to_type(value, param.data_type) except ValueError: raise BadArgumentError( param.name, msg='Bad value for parameter %s:' ' "%s" is not a valid %s' % (param.name, value, param.data_type)) if not param.name in parameters: parameters[param.name] = [] if not operator: if not no_operator_param: no_operator_param = SearchParam( param.name, [value], operator, param.data_type, operator_not) else: no_operator_param.value.append(value) else: parameters[param.name].append( SearchParam(param.name, value, operator, param.data_type, operator_not)) if no_operator_param: parameters[no_operator_param.name].append( no_operator_param) self.fix_date_parameter(parameters) self.fix_process_type_parameter(parameters) self.fix_hang_type_parameter(parameters) return parameters
def get_parameters(self, **kwargs): parameters = {} fields = kwargs['_fields'] assert fields if fields: self.build_filters(fields) for param in self.filters: values = kwargs.get(param.name, param.default) if values in ('', []): # Those values are equivalent to None here. # Note that we cannot use bool(), because 0 is not equivalent # to None in our case. values = None if values is None and param.mandatory: raise MissingArgumentError(param.name) if values is None and param.default is not None: values = param.default # all values can be a list, so we make them all lists to simplify if values is not None and not isinstance(values, (list, tuple)): values = [values] if values is not None: # There should only be one parameter with no operator, and # we want to stack all values into it. That's why we want # to keep track of it. # Actually, we want _two_ parameters with no operator: one # for each possible value of "operator_not". no_operator_param = {True: None, False: None} for value in values: operator = None operator_not = False operators = OPERATORS_MAP.get(param.data_type, OPERATORS_MAP['default']) if isinstance(value, basestring): if value.startswith(OPERATOR_NOT): operator_not = True value = value[1:] for ope in operators: if value.startswith(ope): operator = ope value = value[len(ope):] break # ensure the right data type try: value = convert_to_type(value, param.data_type) except ValueError: raise BadArgumentError( param.name, msg='Bad value for parameter %s:' ' "%s" is not a valid %s' % (param.name, value, param.data_type)) if param.name not in parameters: parameters[param.name] = [] if not operator: if not no_operator_param[operator_not]: no_operator_param[operator_not] = SearchParam( param.name, [value], operator, param.data_type, operator_not) else: no_operator_param[operator_not].value.append(value) else: parameters[param.name].append( SearchParam(param.name, value, operator, param.data_type, operator_not)) for value in no_operator_param.values(): if value: parameters[value.name].append(value) self.fix_date_parameter(parameters) self.fix_process_type_parameter(parameters) self.fix_hang_type_parameter(parameters) self.fix_version_parameter(parameters) return parameters
def get_exploitability(self, **kwargs): """Return a list of exploitable crash reports. See socorro.lib.external_common.parse_arguments() for all filters. """ now = datetimeutil.utc_now().date() lastweek = now - datetime.timedelta(weeks=1) filters = [ ("start_date", lastweek, "date"), ("end_date", now, "date"), ("product", None, "str"), ("version", None, "str"), ("page", None, "int"), ("batch", None, "int"), ] params = external_common.parse_arguments(filters, kwargs) sql_where = """ report_date BETWEEN %(start_date)s AND %(end_date)s AND null_count + none_count + low_count + medium_count + high_count > 4 """ if params.product: sql_where += " AND pv.product_name = %(product)s" if params.version: sql_where += " AND pv.version_string = %(version)s" inner_with_sql = """ SELECT signature, SUM(high_count) AS high_count, SUM(medium_count) AS medium_count, SUM(low_count) AS low_count, SUM(null_count) AS null_count, SUM(none_count) AS none_count, SUM(high_count) + SUM(medium_count) AS med_or_high FROM exploitability_reports JOIN product_versions AS pv USING (product_version_id) WHERE high_count + medium_count + null_count + none_count > 4 AND %s GROUP BY signature """ % (sql_where, ) count_sql_query = """ /* external.postgresql.crashes.Crashes.get_exploitability */ WITH sums AS ( %s ) SELECT count(signature) FROM sums """ % (inner_with_sql, ) results = self.query( count_sql_query, params, error_message="Failed to retrieve exploitable crashes count") total_crashes_count, = results[0] sql_query = """ /* external.postgresql.crashes.Crashes.get_exploitability */ WITH sums AS ( %s ) SELECT signature, high_count, medium_count, low_count, null_count, none_count FROM sums ORDER BY med_or_high DESC, signature ASC """ % (inner_with_sql, ) if params['page'] is not None: if params['page'] <= 0: raise BadArgumentError('page', params['page'], 'starts on 1') if params['batch'] is None: raise MissingArgumentError('batch') sql_query += """ LIMIT %(limit)s OFFSET %(offset)s """ params['limit'] = params['batch'] params['offset'] = params['batch'] * (params['page'] - 1) error_message = ( "Failed to retrieve exploitable crashes from PostgreSQL") results = self.query(sql_query, params, error_message=error_message) # Transforming the results into what we want crashes = [] for row in results: crash = dict( zip(("signature", "high_count", "medium_count", "low_count", "null_count", "none_count"), row)) crashes.append(crash) return {"hits": crashes, "total": total_crashes_count}