def get(self, **kwargs):
"""Return JSON data of a crash report, given its uuid. """
filters = [('uuid', None, 'str'), ('datatype', None, 'str')]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
if not params.datatype:
raise MissingOrBadArgumentError(
"Mandatory parameter 'datatype' is missing or empty")
store = crashstorage.FileSystemCrashStorage(self.config.filesystem)
datatype_method_mapping = {
'raw': 'get_raw_dump',
'meta': 'get_raw_crash',
'processed': 'get_processed'
}
get = store.__getattribute__(datatype_method_mapping[params.datatype])
try:
if params.datatype == 'raw':
return (get(params.uuid), 'application/octet-stream')
else:
return get(params.uuid)
except CrashIDNotFound:
if params.datatype == 'processed':
self.get(datatype='raw', uuid=params.uuid)
j = priorityjobs.Priorityjobs(config=self.config)
j.create(uuid=params.uuid)
raise ResourceUnavailable(params.uuid)
raise ResourceNotFound(params.uuid)
def get(self, **kwargs):
"""Return JSON data of a crash report, given its uuid. """
filters = [
("uuid", None, "str"),
("datatype", None, "str")
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
if not params.datatype:
raise MissingOrBadArgumentError(
"Mandatory parameter 'datatype' is missing or empty")
if hasattr(self.config, 'hbase'):
config = self.config.hbase
store = crashstorage.HBaseCrashStorage(config)
datatype_method_mapping = {
"raw": "get_raw_dump",
"meta": "get_raw_crash",
"processed": "get_processed"
}
else:
# old middleware
config = self.config
import socorro.storage.crashstorage as cs
store = cs.CrashStoragePool(
config,
storageClass=config.hbaseStorageClass
).crashStorage()
datatype_method_mapping = {
"raw": "get_raw_dump",
"meta": "get_meta",
"processed": "get_processed"
}
get = store.__getattribute__(datatype_method_mapping[params.datatype])
try:
if params.datatype == 'raw':
return (get(params.uuid), 'application/octet-stream')
else:
return get(params.uuid)
except (CrashIDNotFound, OoidNotFoundException):
if params.datatype == 'processed':
self.get(datatype='raw', uuid=params.uuid)
j = priorityjobs.Priorityjobs(config=self.config)
j.create(uuid=params.uuid)
raise ResourceUnavailable(params.uuid)
raise ResourceNotFound(params.uuid)
def get(self, **kwargs):
"""Return a job in the priority queue. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
sql = """
/* socorro.external.postgresql.priorityjobs.Priorityjobs.get */
SELECT uuid FROM priorityjobs WHERE uuid=%(uuid)s
"""
error_message = "Failed to retrieve priorityjobs data from PostgreSQL"
results = self.query(sql, params, error_message=error_message)
jobs = []
for row in results:
job = dict(zip(("uuid", ), row))
jobs.append(job)
return {"hits": jobs, "total": len(jobs)}
def post(self, **kwargs):
"""Return a list of signature - bug id associations. """
params = external_common.parse_arguments(self.filters, kwargs)
if not params.signatures:
raise MissingOrBadArgumentError(
"Mandatory parameter 'signatures' is missing or empty")
# Preparing variables for the SQL query
signatures = []
sql_params = {}
for i, elem in enumerate(params.signatures):
signatures.append("%%(signature%s)s" % i)
sql_params["signature%s" % i] = elem
sql = """/* socorro.external.postgresql.bugs.Bugs.get */
SELECT ba.signature, bugs.id
FROM bugs
JOIN bug_associations AS ba ON bugs.id = ba.bug_id
WHERE EXISTS(
SELECT 1 FROM bug_associations
WHERE bug_associations.bug_id = bugs.id
AND signature IN (%s)
)
""" % ", ".join(signatures)
sql = str(" ".join(sql.split())) # better formatting of the sql string
error_message = "Failed to retrieve bugs associations from PostgreSQL"
results = self.query(sql, sql_params, error_message=error_message)
bugs = []
for row in results:
bug = dict(zip(("signature", "id"), row))
bugs.append(bug)
return {"hits": bugs, "total": len(bugs)}
def get(self, **kwargs):
"""Return a job in the job queue. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
fields = [
"id", "pathname", "uuid", "owner", "priority", "queueddatetime",
"starteddatetime", "completeddatetime", "success", "message"
]
sql = """
/* socorro.external.postgresql.job.Job.get */
SELECT %s FROM jobs WHERE uuid=%%(uuid)s
""" % ", ".join(fields)
error_message = "Failed to retrieve jobs data from PostgreSQL"
results = self.query(sql, params, error_message=error_message)
jobs = []
for row in results:
job = dict(zip(fields, row))
# Make sure all dates are turned into strings
for i in job:
if isinstance(job[i], datetime.datetime):
job[i] = datetimeutil.date_to_string(job[i])
jobs.append(job)
return {"hits": jobs, "total": len(jobs)}
def get_paireduuid(self, **kwargs):
"""Return paired uuid given a uuid and an optional hangid.
If a hangid is passed, then return only one result. Otherwise, return
all found paired uuids.
"""
filters = [
("uuid", None, "str"),
("hangid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
crash_date = datetimeutil.uuid_to_date(params.uuid)
sql = """
/* socorro.external.postgresql.crashes.Crashes.get_paireduuid */
SELECT uuid
FROM reports r
WHERE r.uuid != %(uuid)s
AND r.date_processed BETWEEN
TIMESTAMP %(crash_date)s - CAST('1 day' AS INTERVAL) AND
TIMESTAMP %(crash_date)s + CAST('1 day' AS INTERVAL)
"""
sql_params = {"uuid": params.uuid, "crash_date": crash_date}
if params.hangid is not None:
sql = """%s
AND r.hangid = %%(hangid)s
LIMIT 1
""" % sql
sql_params["hangid"] = params.hangid
else:
sql = """%s
AND r.hangid IN (
SELECT hangid
FROM reports r2
WHERE r2.date_processed BETWEEN
TIMESTAMP %%(crash_date)s - CAST('1 day' AS INTERVAL)
AND
TIMESTAMP %%(crash_date)s + CAST('1 day' AS INTERVAL)
AND r2.uuid = %%(uuid)s
)
""" % sql
# Query the database
error_message = "Failed to retrieve paired uuids from PostgreSQL"
results = self.query(sql, sql_params, error_message=error_message)
# Transforming the results into what we want
uuids = [dict(zip(("uuid", ), row)) for row in results]
return {"hits": uuids, "total": len(uuids)}
def _require_parameters(self, params, *required):
"""
Checks that all required parameters are present in and non-empty in
params, where required is a list of strings.
Raises MissingOrBadArgumentError if a required parameter is
missing or empty.
"""
for p in required:
if not params.get(p, None):
raise MissingOrBadArgumentError(
"Mandatory parameter '%s' is missing or empty" % p)
def get(self, **kwargs):
"""Return a single crash report from it's UUID. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if params.uuid is None:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
crash_date = datetimeutil.uuid_to_date(params.uuid)
logger.debug("Looking for crash %s during day %s" %
(params.uuid, crash_date))
sql = """/* socorro.external.postgresql.crash.Crash.get */
SELECT reports.email, reports.url, reports.addons_checked,
( SELECT reports_duplicates.duplicate_of
FROM reports_duplicates
WHERE reports_duplicates.uuid = reports.uuid
) as duplicate_of
FROM reports
WHERE reports.uuid=%(uuid)s
AND reports.success IS NOT NULL
AND utc_day_is( reports.date_processed, %(crash_date)s)
"""
sql_params = {"uuid": params.uuid, "crash_date": crash_date}
results = []
# Creating the connection to the DB
self.connection = self.database.connection()
cur = self.connection.cursor()
try:
results = db.execute(cur, sql, sql_params)
except psycopg2.Error:
util.reportExceptionAndContinue(logger)
json_result = {"total": 0, "hits": []}
for crash in results:
row = dict(
zip(("email", "url", "addons_checked", "duplicate_of"), crash))
json_result["hits"].append(row)
json_result["total"] = len(json_result["hits"])
self.connection.close()
return json_result
def create(self, **kwargs):
"""Add a new job to the priority queue if not already in that queue.
"""
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
sql = """
/* socorro.external.postgresql.priorityjobs.Priorityjobs.create */
INSERT INTO priorityjobs (uuid) VALUES (%(uuid)s)
"""
sql_exists = """
/* socorro.external.postgresql.priorityjobs.Priorityjobs.create */
SELECT 1 FROM priorityjobs WHERE uuid=%(uuid)s
"""
connection = None
try:
connection = self.database.connection()
cur = connection.cursor()
# Verifying that the uuid is not already in the queue
cur.execute(sql_exists, params)
if cur.rowcount:
logger.debug('The uuid %s is already in the priorityjobs '
'table' % params.uuid)
return False
logger.debug('Adding the uuid %s to the priorityjobs table' %
params.uuid)
cur.execute(sql, params)
except psycopg2.Error:
logger.error("Failed inserting priorityjobs data into PostgreSQL",
exc_info=True)
connection.rollback()
return False
else:
connection.commit()
return bool(cur.rowcount)
finally:
if connection:
connection.close()
return True
def prepare_search_params(self, **kwargs):
"""Return a dictionary of parameters for a search-like SQL query.
Uses socorro.lib.search_common.get_parameters() for arguments
filtering.
"""
params = search_common.get_parameters(kwargs)
if not params["signature"]:
raise MissingOrBadArgumentError(
"Mandatory parameter 'signature' is missing or empty")
params["terms"] = params["signature"]
params["search_mode"] = "is_exactly"
# Default mode falls back to starts_with for postgres
if params["plugin_search_mode"] == "default":
params["plugin_search_mode"] = "starts_with"
# Searching for terms in plugins
if params["report_process"] == "plugin" and params["plugin_terms"]:
params["plugin_terms"] = " ".join(params["plugin_terms"])
params["plugin_terms"] = Crashes.prepare_terms(
params["plugin_terms"],
params["plugin_search_mode"])
# Get information about the versions
util_service = Util(config=self.context)
params["versions_info"] = util_service.versions_info(**params)
# Parsing the versions
params["versions_string"] = params["versions"]
(params["versions"], params["products"]) = Crashes.parse_versions(
params["versions"],
params["products"])
# Changing the OS ids to OS names
if hasattr(self.context, 'webapi'):
context = self.context.webapi
else:
# old middleware
context = self.context
for i, elem in enumerate(params["os"]):
for platform in context.platforms:
if platform["id"] == elem:
params["os"][i] = platform["name"]
return params
def post(self, **kwargs):
"""Return a list of signature - bug id associations. """
params = external_common.parse_arguments(self.filters, kwargs)
if not params.signatures:
raise MissingOrBadArgumentError(
"Mandatory parameter 'signatures' is missing or empty")
# Preparing variables for the SQL query
signatures = []
sql_params = {}
for i, elem in enumerate(params.signatures):
signatures.append("%%(signature%s)s" % i)
sql_params["signature%s" % i] = elem
sql = """/* socorro.external.postgresql.bugs.Bugs.get */
SELECT ba.signature, bugs.id
FROM bugs
JOIN bug_associations AS ba ON bugs.id = ba.bug_id
WHERE EXISTS(
SELECT 1 FROM bug_associations
WHERE bug_associations.bug_id = bugs.id
AND signature IN (%s)
)
""" % ", ".join(signatures)
sql = str(" ".join(sql.split())) # better formatting of the sql string
connection = None
try:
connection = self.database.connection()
cur = connection.cursor()
#~ logger.debug(cur.mogrify(sql, sql_params))
results = db.execute(cur, sql, sql_params)
except psycopg2.Error:
logger.error("Failed retrieving extensions data from PostgreSQL",
exc_info=True)
else:
result = {"total": 0, "hits": []}
for crash in results:
row = dict(zip(("signature", "id"), crash))
result["hits"].append(row)
result["total"] = len(result["hits"])
return result
finally:
if connection:
connection.close()
def get(self, **kwargs):
"""Return a job in the job queue. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
fields = [
"id", "pathname", "uuid", "owner", "priority", "queueddatetime",
"starteddatetime", "completeddatetime", "success", "message"
]
sql = """
/* socorro.external.postgresql.job.Job.get */
SELECT %s FROM jobs WHERE uuid=%%(uuid)s
""" % ", ".join(fields)
json_result = {"total": 0, "hits": []}
connection = None
try:
# Creating the connection to the DB
connection = self.database.connection()
cur = connection.cursor()
results = db.execute(cur, sql, params)
except psycopg2.Error:
logger.error("Failed retrieving jobs data from PostgreSQL",
exc_info=True)
else:
for job in results:
row = dict(zip(fields, job))
# Make sure all dates are turned into strings
for i in row:
if isinstance(row[i], datetime.datetime):
row[i] = datetimeutil.date_to_string(row[i])
json_result["hits"].append(row)
json_result["total"] = len(json_result["hits"])
finally:
if connection:
connection.close()
return json_result
def get(self, **kwargs):
"""Return a single crash report from its UUID. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if params.uuid is None:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
crash_date = datetimeutil.uuid_to_date(params.uuid)
logger.debug("Looking for crash %s during day %s" %
(params.uuid, crash_date))
sql = """/* socorro.external.postgresql.crash.Crash.get */
SELECT
reports.email,
reports.url,
reports.addons_checked,
reports.exploitability,
( SELECT reports_duplicates.duplicate_of
FROM reports_duplicates
WHERE reports_duplicates.uuid = reports.uuid
) as duplicate_of
FROM reports
WHERE reports.uuid=%(uuid)s
AND reports.success IS NOT NULL
AND utc_day_is( reports.date_processed, %(crash_date)s)
"""
sql_params = {"uuid": params.uuid, "crash_date": crash_date}
error_message = "Failed to retrieve crash data from PostgreSQL"
results = self.query(sql, sql_params, error_message=error_message)
crashes = []
for row in results:
crash = dict(
zip(("email", "url", "addons_checked", "exploitability",
"duplicate_of"), row))
crashes.append(crash)
return {"hits": crashes, "total": len(crashes)}
def get(self, **kwargs):
"""Return a job in the priority queue. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
sql = """
/* socorro.external.postgresql.priorityjobs.Priorityjobs.get */
SELECT uuid FROM priorityjobs WHERE uuid=%(uuid)s
"""
json_result = {
"total": 0,
"hits": []
}
try:
# Creating the connection to the DB
connection = self.database.connection()
cur = connection.cursor()
results = db.execute(cur, sql, params)
except psycopg2.Error:
logger.error("Failed retrieving priorityjobs data from PostgreSQL",
exc_info=True)
else:
for job in results:
row = dict(zip(("uuid",), job))
json_result["hits"].append(row)
json_result["total"] = len(json_result["hits"])
finally:
connection.close()
return json_result
def get_paireduuid(self, **kwargs):
"""Return paired uuid given a uuid and an optional hangid.
If a hangid is passed, then return only one result. Otherwise, return
all found paired uuids.
"""
filters = [
("uuid", None, "str"),
("hangid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
crash_date = datetimeutil.uuid_to_date(params.uuid)
sql = """
/* socorro.external.postgresql.crashes.Crashes.get_paireduuid */
SELECT uuid
FROM reports r
WHERE r.uuid != %(uuid)s
AND r.date_processed BETWEEN
TIMESTAMP %(crash_date)s - CAST('1 day' AS INTERVAL) AND
TIMESTAMP %(crash_date)s + CAST('1 day' AS INTERVAL)
"""
sql_params = {"uuid": params.uuid, "crash_date": crash_date}
if params.hangid is not None:
sql = """%s
AND r.hangid = %%(hangid)s
LIMIT 1
""" % sql
sql_params["hangid"] = params.hangid
else:
sql = """%s
AND r.hangid IN (
SELECT hangid
FROM reports r2
WHERE r2.date_processed BETWEEN
TIMESTAMP %%(crash_date)s - CAST('1 day' AS INTERVAL)
AND
TIMESTAMP %%(crash_date)s + CAST('1 day' AS INTERVAL)
AND r2.uuid = %%(uuid)s
)
""" % sql
result = {"total": 0, "hits": []}
try:
connection = self.database.connection()
cur = connection.cursor()
results = db.execute(cur, sql, sql_params)
# Transforming the results into what we want
for report in results:
row = dict(zip(("uuid", ), report))
result["hits"].append(row)
result["total"] = len(result["hits"])
except psycopg2.Error:
logger.error("Failed to retrieve paired uuids from database",
exc_info=True)
finally:
connection.close()
return result
def get_daily(self, **kwargs):
"""Return crashes by active daily users. """
now = datetimeutil.utc_now().date()
lastweek = now - datetime.timedelta(weeks=1)
filters = [
("product", None, "str"),
("versions", None, ["list", "str"]),
("from_date", lastweek, "date"),
("to_date", now, "date"),
("os", None, ["list", "str"]),
("report_type", None, ["list", "str"]),
("separated_by", None, "str"),
("date_range_type", "date", "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.product:
raise MissingOrBadArgumentError(
"Mandatory parameter 'product' is missing or empty")
if not params.versions or not params.versions[0]:
raise MissingOrBadArgumentError(
"Mandatory parameter 'versions' is missing or empty")
params.versions = tuple(params.versions)
# simple version, for home page graphs mainly
if ((not params.os or not params.os[0])
and (not params.report_type or not params.report_type[0])
and (not params.separated_by or not params.separated_by[0])):
if params.date_range_type == "build":
table_to_use = "home_page_graph_build_view"
date_range_field = "build_date"
else:
table_to_use = "home_page_graph_view"
date_range_field = "report_date"
db_fields = ("product_name", "version_string", date_range_field,
"report_count", "adu", "crash_hadu")
out_fields = ("product", "version", "date", "report_count", "adu",
"crash_hadu")
sql = """
/* socorro.external.postgresql.crashes.Crashes.get_daily */
SELECT %(db_fields)s
FROM %(table_to_use)s
WHERE product_name=%%(product)s
AND version_string IN %%(versions)s
AND %(date_range_field)s BETWEEN %%(from_date)s
AND %%(to_date)s
""" % {
"db_fields": ", ".join(db_fields),
"date_range_field": date_range_field,
"table_to_use": table_to_use
}
# complex version, for daily crashes page mainly
else:
if params.date_range_type == "build":
table_to_use = "crashes_by_user_build_view"
date_range_field = "build_date"
else:
table_to_use = "crashes_by_user_view"
date_range_field = "report_date"
db_fields = [
"product_name", "version_string", date_range_field,
"sum(adjusted_report_count)::bigint as report_count",
"sum(adu)::bigint as adu",
"""crash_hadu(sum(report_count)::bigint, sum(adu)::bigint,
avg(throttle)) as crash_hadu""",
"avg(throttle) as throttle"
]
out_fields = [
"product", "version", "date", "report_count", "adu",
"crash_hadu", "throttle"
]
db_group = ["product_name", "version_string", date_range_field]
if params.separated_by == "os":
db_fields += ["os_name", "os_short_name"]
db_group += ["os_name", "os_short_name"]
out_fields += ["os", "os_short"]
sql_where = []
if params.os and params.os[0]:
sql_where.append("os_short_name IN %(os)s")
params.os = tuple(x[0:3].lower() for x in params.os)
if params.report_type and params.report_type[0]:
sql_where.append("crash_type_short IN %(report_type)s")
params.report_type = tuple(params.report_type)
if sql_where:
sql_where = "AND %s" % " AND ".join(sql_where)
else:
sql_where = ''
sql = """
/* socorro.external.postgresql.crashes.Crashes.get_daily */
SELECT %(db_fields)s
FROM (
SELECT
product_name,
version_string,
%(date_range_field)s,
os_name,
os_short_name,
SUM(report_count)::int as report_count,
SUM(adjusted_report_count)::int
as adjusted_report_count,
MAX(adu) as adu,
AVG(throttle) as throttle
FROM %(table_to_use)s
WHERE product_name=%%(product)s
AND version_string IN %%(versions)s
AND %(date_range_field)s BETWEEN %%(from_date)s
AND %%(to_date)s
%(sql_where)s
GROUP BY product_name, version_string,
%(date_range_field)s, os_name, os_short_name
) as aggregated_crashes_by_user
""" % {
"db_fields": ", ".join(db_fields),
"date_range_field": date_range_field,
"table_to_use": table_to_use,
"sql_where": sql_where
}
if db_group:
sql = "%s GROUP BY %s" % (sql, ", ".join(db_group))
sql = str(" ".join(sql.split())) # better formatting of the sql string
try:
connection = self.database.connection()
cursor = connection.cursor()
# logger.debug(cursor.mogrify(sql, params))
cursor.execute(sql, params)
results = cursor.fetchall()
except psycopg2.Error:
logger.error("Failed retrieving daily crash data from PostgreSQL",
exc_info=True)
results = []
finally:
connection.close()
hits = {}
for row in results:
daily_data = dict(zip(out_fields, row))
if "throttle" in daily_data:
daily_data["throttle"] = float(daily_data["throttle"])
daily_data["crash_hadu"] = float(daily_data["crash_hadu"])
daily_data["date"] = daily_data["date"].isoformat()
key = "%s:%s" % (daily_data["product"], daily_data["version"])
if params.separated_by == "os":
key = "%s:%s" % (key, daily_data["os_short"])
if "os_short" in daily_data:
del daily_data["os_short"]
if key not in hits:
hits[key] = {}
hits[key][daily_data["date"]] = daily_data
return {"hits": hits}
def get(self, **kwargs):
""" Return urls for signature """
filters = [
("signature", None, "str"),
("start_date", None, "datetime"),
("end_date", None, "datetime"),
("products", None, ["list", "str"]),
("versions", None, ["list", "str"]),
]
params = external_common.parse_arguments(filters, kwargs)
#Because no parameters are optional, we need to loop through
#all parameters to ensure each has been set and is not None
missingParams = []
for param in params:
if not params[param]:
missingParams.append(param)
if len(missingParams) > 0:
raise MissingOrBadArgumentError(
"Mandatory parameter(s) '%s' is missing or empty" %
", ".join(missingParams))
all_products_versions_sql = """
/* socorro.external.postgresql.signature_urls.SignatureURLs.get */
SELECT url, count(*) as crash_count FROM reports_clean
JOIN reports_user_info USING ( UUID )
JOIN signatures USING ( signature_id )
WHERE reports_clean.date_processed
BETWEEN %(start_date)s AND %(end_date)s
AND reports_user_info.date_processed
BETWEEN %(start_date)s AND %(end_date)s
AND signature = %(signature)s
AND url <> ''
"""
sql = """
/* socorro.external.postgresql.signature_urls.SignatureURLs.get */
SELECT url, count(*) as crash_count FROM reports_clean
JOIN reports_user_info USING ( UUID )
JOIN signatures USING ( signature_id )
JOIN product_versions USING ( product_version_id )
WHERE reports_clean.date_processed
BETWEEN %(start_date)s AND %(end_date)s
AND reports_user_info.date_processed
BETWEEN %(start_date)s AND %(end_date)s
AND signature = %(signature)s
AND url <> ''
AND (
"""
sql_group_order = """ GROUP BY url
ORDER BY crash_count DESC LIMIT 100"""
sql_params = {
"start_date": params.start_date,
"end_date": params.end_date,
"signature": params.signature
}
# if this query is for all products the 'ALL' keyword will be
# the only item in the products list and this will then also
# be for all versions.
if 'ALL' in params['products']:
sql_query = " ".join((all_products_versions_sql, sql_group_order))
# if this query is for all versions the 'ALL' keyword will be
# the only item in the versions list.
elif 'ALL' in params['versions']:
sql_products = " product_name IN ('%s') )" % ("', '".join(
[product for product in params.products]))
sql_date_range_limit = """AND '%s' BETWEEN
product_versions.build_date
AND product_versions.sunset_date""" % params.end_date
sql_query = " ".join(
(sql, sql_products, sql_date_range_limit, sql_group_order))
else:
products = []
(params["products_versions"],
products) = self.parse_versions(params["versions"], [])
versions_list = []
products_list = []
for x in range(0, len(params["products_versions"]), 2):
products_list.append(params["products_versions"][x])
versions_list.append(params["products_versions"][x + 1])
product_version_list = []
for prod in params["products"]:
versions = []
[
versions.append(versions_list[i])
for i, x in enumerate(products_list) if x == prod
]
product_version_list.append(tuple(versions))
sql_product_version_ids = [
"""( product_name = %%(product%s)s
AND version_string IN %%(version%s)s ) """ % (x, x)
for x in range(len(product_version_list))
]
sql_params = add_param_to_dict(sql_params, "version",
product_version_list)
sql_params = add_param_to_dict(sql_params, "product",
params.products)
sql_query = " ".join((sql, " OR ".join(sql_product_version_ids),
" ) " + sql_group_order))
json_result = {"total": 0, "hits": []}
connection = None
try:
connection = self.database.connection()
cur = connection.cursor()
results = db.execute(cur, sql_query, sql_params)
except psycopg2.Error:
logger.error(
"Failed retrieving urls for signature data from PostgreSQL",
exc_info=True)
else:
for url in results:
row = dict(zip(("url", "crash_count"), url))
json_result["hits"].append(row)
json_result["total"] = len(json_result["hits"])
return json_result
finally:
if connection:
connection.close()
def get_list(self, **kwargs):
"""
List all crashes with a given signature and return them.
Optional arguments: see SearchCommon.get_parameters()
"""
# aliases
if "from" in kwargs and "from_date" not in kwargs:
kwargs["from_date"] = kwargs.get("from")
if "to" in kwargs and "to_date" not in kwargs:
kwargs["to_date"] = kwargs.get("to")
params = search_common.get_parameters(kwargs)
if not params["signature"]:
raise MissingOrBadArgumentError(
"Mandatory parameter 'signature' is missing or empty"
)
params["terms"] = params["signature"]
params["search_mode"] = "is_exactly"
# Default mode falls back to starts_with for postgres
if params["plugin_search_mode"] == "default":
params["plugin_search_mode"] = "starts_with"
# Limiting to a signature
if params["terms"]:
params["terms"] = self.prepare_terms(params["terms"],
params["search_mode"])
# Searching for terms in plugins
if params["report_process"] == "plugin" and params["plugin_terms"]:
params["plugin_terms"] = " ".join(params["plugin_terms"])
params["plugin_terms"] = self.prepare_terms(
params["plugin_terms"],
params["plugin_search_mode"])
# Get information about the versions
util_service = Util(config=self.context)
params["versions_info"] = util_service.versions_info(**params)
# Parsing the versions
params["versions_string"] = params["versions"]
(params["versions"], params["products"]) = self.parse_versions(
params["versions"],
params["products"])
if hasattr(self.context, 'webapi'):
context = self.context.webapi
else:
# old middleware
context = self.context
# Changing the OS ids to OS names
for i, elem in enumerate(params["os"]):
for platform in context.platforms:
if platform["id"][:3] == elem[:3]:
params["os"][i] = platform["name"]
# Creating the parameters for the sql query
sql_params = {
}
# Preparing the different parts of the sql query
sql_select = """
SELECT
r.date_processed,
r.uptime,
r.user_comments,
r.uuid,
r.product,
r.version,
r.build,
r.signature,
r.url,
r.os_name,
r.os_version,
r.cpu_name,
r.cpu_info,
r.address,
r.reason,
r.last_crash,
r.install_age,
r.hangid,
r.process_type,
(r.client_crash_date - (r.install_age * INTERVAL '1 second'))
AS install_time,
rd.duplicate_of
"""
sql_from = self.build_reports_sql_from(params)
sql_from = """%s
LEFT OUTER JOIN reports_duplicates rd ON r.uuid = rd.uuid
""" % sql_from
(sql_where, sql_params) = self.build_reports_sql_where(params,
sql_params,
self.context)
sql_order = """
ORDER BY r.date_processed DESC
"""
(sql_limit, sql_params) = self.build_reports_sql_limit(params,
sql_params)
# Assembling the query
sql_query = " ".join((
"/* socorro.external.postgresql.report.Report.list */",
sql_select, sql_from, sql_where, sql_order, sql_limit))
# Query for counting the results
sql_count_query = " ".join((
"/* socorro.external.postgresql.report.Report.list */",
"SELECT count(*)", sql_from, sql_where))
# Querying the DB
try:
connection = self.database.connection()
total = self.count(
sql_count_query,
sql_params,
error_message="Failed to count crashes from PostgreSQL.",
connection=connection
)
results = []
# No need to call Postgres if we know there will be no results
if total != 0:
results = self.query(
sql_query,
sql_params,
error_message="Failed to retrieve crashes from PostgreSQL",
connection=connection
)
except psycopg2.Error:
raise DatabaseError("Failed to retrieve crashes from PostgreSQL")
finally:
if connection:
connection.close()
# Transforming the results into what we want
crashes = []
for row in results:
crash = dict(zip((
"date_processed",
"uptime",
"user_comments",
"uuid",
"product",
"version",
"build",
"signature",
"url",
"os_name",
"os_version",
"cpu_name",
"cpu_info",
"address",
"reason",
"last_crash",
"install_age",
"hangid",
"process_type",
"install_time",
"duplicate_of"
), row))
for i in crash:
try:
crash[i] = datetimeutil.date_to_string(crash[i])
except TypeError:
pass
crashes.append(crash)
return {
"hits": crashes,
"total": total
}