def get(self, **kwargs): """Return JSON data of a crash report, given its uuid. """ filters = [('uuid', None, 'str'), ('datatype', None, 'str')] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") if not params.datatype: raise MissingOrBadArgumentError( "Mandatory parameter 'datatype' is missing or empty") store = crashstorage.FileSystemCrashStorage(self.config.filesystem) datatype_method_mapping = { 'raw': 'get_raw_dump', 'meta': 'get_raw_crash', 'processed': 'get_processed' } get = store.__getattribute__(datatype_method_mapping[params.datatype]) try: if params.datatype == 'raw': return (get(params.uuid), 'application/octet-stream') else: return get(params.uuid) except CrashIDNotFound: if params.datatype == 'processed': self.get(datatype='raw', uuid=params.uuid) j = priorityjobs.Priorityjobs(config=self.config) j.create(uuid=params.uuid) raise ResourceUnavailable(params.uuid) raise ResourceNotFound(params.uuid)
def get(self, **kwargs): """Return JSON data of a crash report, given its uuid. """ filters = [ ("uuid", None, "str"), ("datatype", None, "str") ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") if not params.datatype: raise MissingOrBadArgumentError( "Mandatory parameter 'datatype' is missing or empty") if hasattr(self.config, 'hbase'): config = self.config.hbase store = crashstorage.HBaseCrashStorage(config) datatype_method_mapping = { "raw": "get_raw_dump", "meta": "get_raw_crash", "processed": "get_processed" } else: # old middleware config = self.config import socorro.storage.crashstorage as cs store = cs.CrashStoragePool( config, storageClass=config.hbaseStorageClass ).crashStorage() datatype_method_mapping = { "raw": "get_raw_dump", "meta": "get_meta", "processed": "get_processed" } get = store.__getattribute__(datatype_method_mapping[params.datatype]) try: if params.datatype == 'raw': return (get(params.uuid), 'application/octet-stream') else: return get(params.uuid) except (CrashIDNotFound, OoidNotFoundException): if params.datatype == 'processed': self.get(datatype='raw', uuid=params.uuid) j = priorityjobs.Priorityjobs(config=self.config) j.create(uuid=params.uuid) raise ResourceUnavailable(params.uuid) raise ResourceNotFound(params.uuid)
def get(self, **kwargs): """Return a job in the priority queue. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") sql = """ /* socorro.external.postgresql.priorityjobs.Priorityjobs.get */ SELECT uuid FROM priorityjobs WHERE uuid=%(uuid)s """ error_message = "Failed to retrieve priorityjobs data from PostgreSQL" results = self.query(sql, params, error_message=error_message) jobs = [] for row in results: job = dict(zip(("uuid", ), row)) jobs.append(job) return {"hits": jobs, "total": len(jobs)}
def post(self, **kwargs): """Return a list of signature - bug id associations. """ params = external_common.parse_arguments(self.filters, kwargs) if not params.signatures: raise MissingOrBadArgumentError( "Mandatory parameter 'signatures' is missing or empty") # Preparing variables for the SQL query signatures = [] sql_params = {} for i, elem in enumerate(params.signatures): signatures.append("%%(signature%s)s" % i) sql_params["signature%s" % i] = elem sql = """/* socorro.external.postgresql.bugs.Bugs.get */ SELECT ba.signature, bugs.id FROM bugs JOIN bug_associations AS ba ON bugs.id = ba.bug_id WHERE EXISTS( SELECT 1 FROM bug_associations WHERE bug_associations.bug_id = bugs.id AND signature IN (%s) ) """ % ", ".join(signatures) sql = str(" ".join(sql.split())) # better formatting of the sql string error_message = "Failed to retrieve bugs associations from PostgreSQL" results = self.query(sql, sql_params, error_message=error_message) bugs = [] for row in results: bug = dict(zip(("signature", "id"), row)) bugs.append(bug) return {"hits": bugs, "total": len(bugs)}
def get(self, **kwargs): """Return a job in the job queue. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") fields = [ "id", "pathname", "uuid", "owner", "priority", "queueddatetime", "starteddatetime", "completeddatetime", "success", "message" ] sql = """ /* socorro.external.postgresql.job.Job.get */ SELECT %s FROM jobs WHERE uuid=%%(uuid)s """ % ", ".join(fields) error_message = "Failed to retrieve jobs data from PostgreSQL" results = self.query(sql, params, error_message=error_message) jobs = [] for row in results: job = dict(zip(fields, row)) # Make sure all dates are turned into strings for i in job: if isinstance(job[i], datetime.datetime): job[i] = datetimeutil.date_to_string(job[i]) jobs.append(job) return {"hits": jobs, "total": len(jobs)}
def get_paireduuid(self, **kwargs): """Return paired uuid given a uuid and an optional hangid. If a hangid is passed, then return only one result. Otherwise, return all found paired uuids. """ filters = [ ("uuid", None, "str"), ("hangid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") crash_date = datetimeutil.uuid_to_date(params.uuid) sql = """ /* socorro.external.postgresql.crashes.Crashes.get_paireduuid */ SELECT uuid FROM reports r WHERE r.uuid != %(uuid)s AND r.date_processed BETWEEN TIMESTAMP %(crash_date)s - CAST('1 day' AS INTERVAL) AND TIMESTAMP %(crash_date)s + CAST('1 day' AS INTERVAL) """ sql_params = {"uuid": params.uuid, "crash_date": crash_date} if params.hangid is not None: sql = """%s AND r.hangid = %%(hangid)s LIMIT 1 """ % sql sql_params["hangid"] = params.hangid else: sql = """%s AND r.hangid IN ( SELECT hangid FROM reports r2 WHERE r2.date_processed BETWEEN TIMESTAMP %%(crash_date)s - CAST('1 day' AS INTERVAL) AND TIMESTAMP %%(crash_date)s + CAST('1 day' AS INTERVAL) AND r2.uuid = %%(uuid)s ) """ % sql # Query the database error_message = "Failed to retrieve paired uuids from PostgreSQL" results = self.query(sql, sql_params, error_message=error_message) # Transforming the results into what we want uuids = [dict(zip(("uuid", ), row)) for row in results] return {"hits": uuids, "total": len(uuids)}
def _require_parameters(self, params, *required): """ Checks that all required parameters are present in and non-empty in params, where required is a list of strings. Raises MissingOrBadArgumentError if a required parameter is missing or empty. """ for p in required: if not params.get(p, None): raise MissingOrBadArgumentError( "Mandatory parameter '%s' is missing or empty" % p)
def get(self, **kwargs): """Return a single crash report from it's UUID. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if params.uuid is None: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") crash_date = datetimeutil.uuid_to_date(params.uuid) logger.debug("Looking for crash %s during day %s" % (params.uuid, crash_date)) sql = """/* socorro.external.postgresql.crash.Crash.get */ SELECT reports.email, reports.url, reports.addons_checked, ( SELECT reports_duplicates.duplicate_of FROM reports_duplicates WHERE reports_duplicates.uuid = reports.uuid ) as duplicate_of FROM reports WHERE reports.uuid=%(uuid)s AND reports.success IS NOT NULL AND utc_day_is( reports.date_processed, %(crash_date)s) """ sql_params = {"uuid": params.uuid, "crash_date": crash_date} results = [] # Creating the connection to the DB self.connection = self.database.connection() cur = self.connection.cursor() try: results = db.execute(cur, sql, sql_params) except psycopg2.Error: util.reportExceptionAndContinue(logger) json_result = {"total": 0, "hits": []} for crash in results: row = dict( zip(("email", "url", "addons_checked", "duplicate_of"), crash)) json_result["hits"].append(row) json_result["total"] = len(json_result["hits"]) self.connection.close() return json_result
def create(self, **kwargs): """Add a new job to the priority queue if not already in that queue. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") sql = """ /* socorro.external.postgresql.priorityjobs.Priorityjobs.create */ INSERT INTO priorityjobs (uuid) VALUES (%(uuid)s) """ sql_exists = """ /* socorro.external.postgresql.priorityjobs.Priorityjobs.create */ SELECT 1 FROM priorityjobs WHERE uuid=%(uuid)s """ connection = None try: connection = self.database.connection() cur = connection.cursor() # Verifying that the uuid is not already in the queue cur.execute(sql_exists, params) if cur.rowcount: logger.debug('The uuid %s is already in the priorityjobs ' 'table' % params.uuid) return False logger.debug('Adding the uuid %s to the priorityjobs table' % params.uuid) cur.execute(sql, params) except psycopg2.Error: logger.error("Failed inserting priorityjobs data into PostgreSQL", exc_info=True) connection.rollback() return False else: connection.commit() return bool(cur.rowcount) finally: if connection: connection.close() return True
def prepare_search_params(self, **kwargs): """Return a dictionary of parameters for a search-like SQL query. Uses socorro.lib.search_common.get_parameters() for arguments filtering. """ params = search_common.get_parameters(kwargs) if not params["signature"]: raise MissingOrBadArgumentError( "Mandatory parameter 'signature' is missing or empty") params["terms"] = params["signature"] params["search_mode"] = "is_exactly" # Default mode falls back to starts_with for postgres if params["plugin_search_mode"] == "default": params["plugin_search_mode"] = "starts_with" # Searching for terms in plugins if params["report_process"] == "plugin" and params["plugin_terms"]: params["plugin_terms"] = " ".join(params["plugin_terms"]) params["plugin_terms"] = Crashes.prepare_terms( params["plugin_terms"], params["plugin_search_mode"]) # Get information about the versions util_service = Util(config=self.context) params["versions_info"] = util_service.versions_info(**params) # Parsing the versions params["versions_string"] = params["versions"] (params["versions"], params["products"]) = Crashes.parse_versions( params["versions"], params["products"]) # Changing the OS ids to OS names if hasattr(self.context, 'webapi'): context = self.context.webapi else: # old middleware context = self.context for i, elem in enumerate(params["os"]): for platform in context.platforms: if platform["id"] == elem: params["os"][i] = platform["name"] return params
def post(self, **kwargs): """Return a list of signature - bug id associations. """ params = external_common.parse_arguments(self.filters, kwargs) if not params.signatures: raise MissingOrBadArgumentError( "Mandatory parameter 'signatures' is missing or empty") # Preparing variables for the SQL query signatures = [] sql_params = {} for i, elem in enumerate(params.signatures): signatures.append("%%(signature%s)s" % i) sql_params["signature%s" % i] = elem sql = """/* socorro.external.postgresql.bugs.Bugs.get */ SELECT ba.signature, bugs.id FROM bugs JOIN bug_associations AS ba ON bugs.id = ba.bug_id WHERE EXISTS( SELECT 1 FROM bug_associations WHERE bug_associations.bug_id = bugs.id AND signature IN (%s) ) """ % ", ".join(signatures) sql = str(" ".join(sql.split())) # better formatting of the sql string connection = None try: connection = self.database.connection() cur = connection.cursor() #~ logger.debug(cur.mogrify(sql, sql_params)) results = db.execute(cur, sql, sql_params) except psycopg2.Error: logger.error("Failed retrieving extensions data from PostgreSQL", exc_info=True) else: result = {"total": 0, "hits": []} for crash in results: row = dict(zip(("signature", "id"), crash)) result["hits"].append(row) result["total"] = len(result["hits"]) return result finally: if connection: connection.close()
def get(self, **kwargs): """Return a job in the job queue. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") fields = [ "id", "pathname", "uuid", "owner", "priority", "queueddatetime", "starteddatetime", "completeddatetime", "success", "message" ] sql = """ /* socorro.external.postgresql.job.Job.get */ SELECT %s FROM jobs WHERE uuid=%%(uuid)s """ % ", ".join(fields) json_result = {"total": 0, "hits": []} connection = None try: # Creating the connection to the DB connection = self.database.connection() cur = connection.cursor() results = db.execute(cur, sql, params) except psycopg2.Error: logger.error("Failed retrieving jobs data from PostgreSQL", exc_info=True) else: for job in results: row = dict(zip(fields, job)) # Make sure all dates are turned into strings for i in row: if isinstance(row[i], datetime.datetime): row[i] = datetimeutil.date_to_string(row[i]) json_result["hits"].append(row) json_result["total"] = len(json_result["hits"]) finally: if connection: connection.close() return json_result
def get(self, **kwargs): """Return a single crash report from its UUID. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if params.uuid is None: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") crash_date = datetimeutil.uuid_to_date(params.uuid) logger.debug("Looking for crash %s during day %s" % (params.uuid, crash_date)) sql = """/* socorro.external.postgresql.crash.Crash.get */ SELECT reports.email, reports.url, reports.addons_checked, reports.exploitability, ( SELECT reports_duplicates.duplicate_of FROM reports_duplicates WHERE reports_duplicates.uuid = reports.uuid ) as duplicate_of FROM reports WHERE reports.uuid=%(uuid)s AND reports.success IS NOT NULL AND utc_day_is( reports.date_processed, %(crash_date)s) """ sql_params = {"uuid": params.uuid, "crash_date": crash_date} error_message = "Failed to retrieve crash data from PostgreSQL" results = self.query(sql, sql_params, error_message=error_message) crashes = [] for row in results: crash = dict( zip(("email", "url", "addons_checked", "exploitability", "duplicate_of"), row)) crashes.append(crash) return {"hits": crashes, "total": len(crashes)}
def get(self, **kwargs): """Return a job in the priority queue. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") sql = """ /* socorro.external.postgresql.priorityjobs.Priorityjobs.get */ SELECT uuid FROM priorityjobs WHERE uuid=%(uuid)s """ json_result = { "total": 0, "hits": [] } try: # Creating the connection to the DB connection = self.database.connection() cur = connection.cursor() results = db.execute(cur, sql, params) except psycopg2.Error: logger.error("Failed retrieving priorityjobs data from PostgreSQL", exc_info=True) else: for job in results: row = dict(zip(("uuid",), job)) json_result["hits"].append(row) json_result["total"] = len(json_result["hits"]) finally: connection.close() return json_result
def get_paireduuid(self, **kwargs): """Return paired uuid given a uuid and an optional hangid. If a hangid is passed, then return only one result. Otherwise, return all found paired uuids. """ filters = [ ("uuid", None, "str"), ("hangid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") crash_date = datetimeutil.uuid_to_date(params.uuid) sql = """ /* socorro.external.postgresql.crashes.Crashes.get_paireduuid */ SELECT uuid FROM reports r WHERE r.uuid != %(uuid)s AND r.date_processed BETWEEN TIMESTAMP %(crash_date)s - CAST('1 day' AS INTERVAL) AND TIMESTAMP %(crash_date)s + CAST('1 day' AS INTERVAL) """ sql_params = {"uuid": params.uuid, "crash_date": crash_date} if params.hangid is not None: sql = """%s AND r.hangid = %%(hangid)s LIMIT 1 """ % sql sql_params["hangid"] = params.hangid else: sql = """%s AND r.hangid IN ( SELECT hangid FROM reports r2 WHERE r2.date_processed BETWEEN TIMESTAMP %%(crash_date)s - CAST('1 day' AS INTERVAL) AND TIMESTAMP %%(crash_date)s + CAST('1 day' AS INTERVAL) AND r2.uuid = %%(uuid)s ) """ % sql result = {"total": 0, "hits": []} try: connection = self.database.connection() cur = connection.cursor() results = db.execute(cur, sql, sql_params) # Transforming the results into what we want for report in results: row = dict(zip(("uuid", ), report)) result["hits"].append(row) result["total"] = len(result["hits"]) except psycopg2.Error: logger.error("Failed to retrieve paired uuids from database", exc_info=True) finally: connection.close() return result
def get_daily(self, **kwargs): """Return crashes by active daily users. """ now = datetimeutil.utc_now().date() lastweek = now - datetime.timedelta(weeks=1) filters = [ ("product", None, "str"), ("versions", None, ["list", "str"]), ("from_date", lastweek, "date"), ("to_date", now, "date"), ("os", None, ["list", "str"]), ("report_type", None, ["list", "str"]), ("separated_by", None, "str"), ("date_range_type", "date", "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.product: raise MissingOrBadArgumentError( "Mandatory parameter 'product' is missing or empty") if not params.versions or not params.versions[0]: raise MissingOrBadArgumentError( "Mandatory parameter 'versions' is missing or empty") params.versions = tuple(params.versions) # simple version, for home page graphs mainly if ((not params.os or not params.os[0]) and (not params.report_type or not params.report_type[0]) and (not params.separated_by or not params.separated_by[0])): if params.date_range_type == "build": table_to_use = "home_page_graph_build_view" date_range_field = "build_date" else: table_to_use = "home_page_graph_view" date_range_field = "report_date" db_fields = ("product_name", "version_string", date_range_field, "report_count", "adu", "crash_hadu") out_fields = ("product", "version", "date", "report_count", "adu", "crash_hadu") sql = """ /* socorro.external.postgresql.crashes.Crashes.get_daily */ SELECT %(db_fields)s FROM %(table_to_use)s WHERE product_name=%%(product)s AND version_string IN %%(versions)s AND %(date_range_field)s BETWEEN %%(from_date)s AND %%(to_date)s """ % { "db_fields": ", ".join(db_fields), "date_range_field": date_range_field, "table_to_use": table_to_use } # complex version, for daily crashes page mainly else: if params.date_range_type == "build": table_to_use = "crashes_by_user_build_view" date_range_field = "build_date" else: table_to_use = "crashes_by_user_view" date_range_field = "report_date" db_fields = [ "product_name", "version_string", date_range_field, "sum(adjusted_report_count)::bigint as report_count", "sum(adu)::bigint as adu", """crash_hadu(sum(report_count)::bigint, sum(adu)::bigint, avg(throttle)) as crash_hadu""", "avg(throttle) as throttle" ] out_fields = [ "product", "version", "date", "report_count", "adu", "crash_hadu", "throttle" ] db_group = ["product_name", "version_string", date_range_field] if params.separated_by == "os": db_fields += ["os_name", "os_short_name"] db_group += ["os_name", "os_short_name"] out_fields += ["os", "os_short"] sql_where = [] if params.os and params.os[0]: sql_where.append("os_short_name IN %(os)s") params.os = tuple(x[0:3].lower() for x in params.os) if params.report_type and params.report_type[0]: sql_where.append("crash_type_short IN %(report_type)s") params.report_type = tuple(params.report_type) if sql_where: sql_where = "AND %s" % " AND ".join(sql_where) else: sql_where = '' sql = """ /* socorro.external.postgresql.crashes.Crashes.get_daily */ SELECT %(db_fields)s FROM ( SELECT product_name, version_string, %(date_range_field)s, os_name, os_short_name, SUM(report_count)::int as report_count, SUM(adjusted_report_count)::int as adjusted_report_count, MAX(adu) as adu, AVG(throttle) as throttle FROM %(table_to_use)s WHERE product_name=%%(product)s AND version_string IN %%(versions)s AND %(date_range_field)s BETWEEN %%(from_date)s AND %%(to_date)s %(sql_where)s GROUP BY product_name, version_string, %(date_range_field)s, os_name, os_short_name ) as aggregated_crashes_by_user """ % { "db_fields": ", ".join(db_fields), "date_range_field": date_range_field, "table_to_use": table_to_use, "sql_where": sql_where } if db_group: sql = "%s GROUP BY %s" % (sql, ", ".join(db_group)) sql = str(" ".join(sql.split())) # better formatting of the sql string try: connection = self.database.connection() cursor = connection.cursor() # logger.debug(cursor.mogrify(sql, params)) cursor.execute(sql, params) results = cursor.fetchall() except psycopg2.Error: logger.error("Failed retrieving daily crash data from PostgreSQL", exc_info=True) results = [] finally: connection.close() hits = {} for row in results: daily_data = dict(zip(out_fields, row)) if "throttle" in daily_data: daily_data["throttle"] = float(daily_data["throttle"]) daily_data["crash_hadu"] = float(daily_data["crash_hadu"]) daily_data["date"] = daily_data["date"].isoformat() key = "%s:%s" % (daily_data["product"], daily_data["version"]) if params.separated_by == "os": key = "%s:%s" % (key, daily_data["os_short"]) if "os_short" in daily_data: del daily_data["os_short"] if key not in hits: hits[key] = {} hits[key][daily_data["date"]] = daily_data return {"hits": hits}
def get(self, **kwargs): """ Return urls for signature """ filters = [ ("signature", None, "str"), ("start_date", None, "datetime"), ("end_date", None, "datetime"), ("products", None, ["list", "str"]), ("versions", None, ["list", "str"]), ] params = external_common.parse_arguments(filters, kwargs) #Because no parameters are optional, we need to loop through #all parameters to ensure each has been set and is not None missingParams = [] for param in params: if not params[param]: missingParams.append(param) if len(missingParams) > 0: raise MissingOrBadArgumentError( "Mandatory parameter(s) '%s' is missing or empty" % ", ".join(missingParams)) all_products_versions_sql = """ /* socorro.external.postgresql.signature_urls.SignatureURLs.get */ SELECT url, count(*) as crash_count FROM reports_clean JOIN reports_user_info USING ( UUID ) JOIN signatures USING ( signature_id ) WHERE reports_clean.date_processed BETWEEN %(start_date)s AND %(end_date)s AND reports_user_info.date_processed BETWEEN %(start_date)s AND %(end_date)s AND signature = %(signature)s AND url <> '' """ sql = """ /* socorro.external.postgresql.signature_urls.SignatureURLs.get */ SELECT url, count(*) as crash_count FROM reports_clean JOIN reports_user_info USING ( UUID ) JOIN signatures USING ( signature_id ) JOIN product_versions USING ( product_version_id ) WHERE reports_clean.date_processed BETWEEN %(start_date)s AND %(end_date)s AND reports_user_info.date_processed BETWEEN %(start_date)s AND %(end_date)s AND signature = %(signature)s AND url <> '' AND ( """ sql_group_order = """ GROUP BY url ORDER BY crash_count DESC LIMIT 100""" sql_params = { "start_date": params.start_date, "end_date": params.end_date, "signature": params.signature } # if this query is for all products the 'ALL' keyword will be # the only item in the products list and this will then also # be for all versions. if 'ALL' in params['products']: sql_query = " ".join((all_products_versions_sql, sql_group_order)) # if this query is for all versions the 'ALL' keyword will be # the only item in the versions list. elif 'ALL' in params['versions']: sql_products = " product_name IN ('%s') )" % ("', '".join( [product for product in params.products])) sql_date_range_limit = """AND '%s' BETWEEN product_versions.build_date AND product_versions.sunset_date""" % params.end_date sql_query = " ".join( (sql, sql_products, sql_date_range_limit, sql_group_order)) else: products = [] (params["products_versions"], products) = self.parse_versions(params["versions"], []) versions_list = [] products_list = [] for x in range(0, len(params["products_versions"]), 2): products_list.append(params["products_versions"][x]) versions_list.append(params["products_versions"][x + 1]) product_version_list = [] for prod in params["products"]: versions = [] [ versions.append(versions_list[i]) for i, x in enumerate(products_list) if x == prod ] product_version_list.append(tuple(versions)) sql_product_version_ids = [ """( product_name = %%(product%s)s AND version_string IN %%(version%s)s ) """ % (x, x) for x in range(len(product_version_list)) ] sql_params = add_param_to_dict(sql_params, "version", product_version_list) sql_params = add_param_to_dict(sql_params, "product", params.products) sql_query = " ".join((sql, " OR ".join(sql_product_version_ids), " ) " + sql_group_order)) json_result = {"total": 0, "hits": []} connection = None try: connection = self.database.connection() cur = connection.cursor() results = db.execute(cur, sql_query, sql_params) except psycopg2.Error: logger.error( "Failed retrieving urls for signature data from PostgreSQL", exc_info=True) else: for url in results: row = dict(zip(("url", "crash_count"), url)) json_result["hits"].append(row) json_result["total"] = len(json_result["hits"]) return json_result finally: if connection: connection.close()
def get_list(self, **kwargs): """ List all crashes with a given signature and return them. Optional arguments: see SearchCommon.get_parameters() """ # aliases if "from" in kwargs and "from_date" not in kwargs: kwargs["from_date"] = kwargs.get("from") if "to" in kwargs and "to_date" not in kwargs: kwargs["to_date"] = kwargs.get("to") params = search_common.get_parameters(kwargs) if not params["signature"]: raise MissingOrBadArgumentError( "Mandatory parameter 'signature' is missing or empty" ) params["terms"] = params["signature"] params["search_mode"] = "is_exactly" # Default mode falls back to starts_with for postgres if params["plugin_search_mode"] == "default": params["plugin_search_mode"] = "starts_with" # Limiting to a signature if params["terms"]: params["terms"] = self.prepare_terms(params["terms"], params["search_mode"]) # Searching for terms in plugins if params["report_process"] == "plugin" and params["plugin_terms"]: params["plugin_terms"] = " ".join(params["plugin_terms"]) params["plugin_terms"] = self.prepare_terms( params["plugin_terms"], params["plugin_search_mode"]) # Get information about the versions util_service = Util(config=self.context) params["versions_info"] = util_service.versions_info(**params) # Parsing the versions params["versions_string"] = params["versions"] (params["versions"], params["products"]) = self.parse_versions( params["versions"], params["products"]) if hasattr(self.context, 'webapi'): context = self.context.webapi else: # old middleware context = self.context # Changing the OS ids to OS names for i, elem in enumerate(params["os"]): for platform in context.platforms: if platform["id"][:3] == elem[:3]: params["os"][i] = platform["name"] # Creating the parameters for the sql query sql_params = { } # Preparing the different parts of the sql query sql_select = """ SELECT r.date_processed, r.uptime, r.user_comments, r.uuid, r.product, r.version, r.build, r.signature, r.url, r.os_name, r.os_version, r.cpu_name, r.cpu_info, r.address, r.reason, r.last_crash, r.install_age, r.hangid, r.process_type, (r.client_crash_date - (r.install_age * INTERVAL '1 second')) AS install_time, rd.duplicate_of """ sql_from = self.build_reports_sql_from(params) sql_from = """%s LEFT OUTER JOIN reports_duplicates rd ON r.uuid = rd.uuid """ % sql_from (sql_where, sql_params) = self.build_reports_sql_where(params, sql_params, self.context) sql_order = """ ORDER BY r.date_processed DESC """ (sql_limit, sql_params) = self.build_reports_sql_limit(params, sql_params) # Assembling the query sql_query = " ".join(( "/* socorro.external.postgresql.report.Report.list */", sql_select, sql_from, sql_where, sql_order, sql_limit)) # Query for counting the results sql_count_query = " ".join(( "/* socorro.external.postgresql.report.Report.list */", "SELECT count(*)", sql_from, sql_where)) # Querying the DB try: connection = self.database.connection() total = self.count( sql_count_query, sql_params, error_message="Failed to count crashes from PostgreSQL.", connection=connection ) results = [] # No need to call Postgres if we know there will be no results if total != 0: results = self.query( sql_query, sql_params, error_message="Failed to retrieve crashes from PostgreSQL", connection=connection ) except psycopg2.Error: raise DatabaseError("Failed to retrieve crashes from PostgreSQL") finally: if connection: connection.close() # Transforming the results into what we want crashes = [] for row in results: crash = dict(zip(( "date_processed", "uptime", "user_comments", "uuid", "product", "version", "build", "signature", "url", "os_name", "os_version", "cpu_name", "cpu_info", "address", "reason", "last_crash", "install_age", "hangid", "process_type", "install_time", "duplicate_of" ), row)) for i in crash: try: crash[i] = datetimeutil.date_to_string(crash[i]) except TypeError: pass crashes.append(crash) return { "hits": crashes, "total": total }