def get(self, **kwargs):
"""Return JSON data of a crash report, given its uuid. """
filters = [
('uuid', None, 'str'),
('datatype', None, 'str')
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingArgumentError('uuid')
if not params.datatype:
raise MissingArgumentError('datatype')
store = self.config.filesystem.filesystem_class(self.config.filesystem)
datatype_method_mapping = {
'raw': 'get_raw_dump',
'meta': 'get_raw_crash',
'processed': 'get_processed'
}
get = store.__getattribute__(datatype_method_mapping[params.datatype])
try:
if params.datatype == 'raw':
return (get(params.uuid), 'application/octet-stream')
else:
return get(params.uuid)
except CrashIDNotFound:
if params.datatype == 'processed':
self.get(datatype='raw', uuid=params.uuid)
j = priorityjobs.Priorityjobs(config=self.config)
j.create(uuid=params.uuid)
raise ResourceUnavailable(params.uuid)
raise ResourceNotFound(params.uuid)
def post(self, **kwargs):
params = external_common.parse_arguments(self.filters, kwargs)
if not params.category:
raise MissingArgumentError('category')
if not params.rule:
raise MissingArgumentError('rule')
sql = """
/* socorro.external.postgresql.skiplist.SkipList.post */
INSERT INTO skiplist (category, rule)
VALUES (%s, %s);
"""
sql_params = [params.category, params.rule]
connection = self.database.connection()
try:
with connection.cursor() as cur:
cur.execute(sql, sql_params)
connection.commit()
except psycopg2.Error:
connection.rollback()
error_message = "Failed updating skip list in PostgreSQL"
logger.error(error_message)
raise DatabaseError(error_message)
finally:
connection.close()
return True
def get(self, **kwargs):
"""Return JSON data of a crash report, given its uuid. """
filters = [
('uuid', None, 'str'),
('datatype', None, 'str'),
('name', None, 'str') # only applicable if datatype == 'raw'
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingArgumentError('uuid')
if not params.datatype:
raise MissingArgumentError('datatype')
# get a generic crashstorage instance from whatever external resource
# is implementing this service.
store = self.get_storage()
datatype_method_mapping = {
'raw': 'get_raw_dump',
'meta': 'get_raw_crash',
'processed': 'get_processed',
'unredacted': 'get_unredacted_processed',
}
get = store.__getattribute__(datatype_method_mapping[params.datatype])
try:
if params.datatype == 'raw':
return (get(params.uuid,
name=params.name), 'application/octet-stream')
else:
return get(params.uuid)
except CrashIDNotFound:
if params.datatype in ('processed', 'unredacted'):
# try to fetch a raw crash just to ensure that the raw crash
# exists. If this line fails, there's no reason to actually
# submit the priority job.
try:
store.get_raw_crash(params.uuid)
except CrashIDNotFound:
raise ResourceNotFound(params.uuid)
# search through the existing other services to find the
# Priorityjob service.
try:
priorityjob_service_impl = self.all_services[
'Priorityjobs']
except KeyError:
raise ServiceUnavailable('Priorityjobs')
# get the underlying implementation of the Priorityjob
# service and instantiate it.
priority_job_service = priorityjob_service_impl.cls(
config=self.config)
# create the priority job for this crash_ids
priority_job_service.create(uuid=params.uuid)
raise ResourceUnavailable(params.uuid)
raise ResourceNotFound(params.uuid)
def get(self, **kwargs):
"""Return JSON data of a crash report, given its uuid. """
filters = [
("uuid", None, "str"),
("datatype", None, "str")
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingArgumentError('uuid')
if not params.datatype:
raise MissingArgumentError('datatype')
if hasattr(self.config, 'hbase'):
config = self.config.hbase
store = crashstorage.HBaseCrashStorage(config)
datatype_method_mapping = {
"raw": "get_raw_dump",
"meta": "get_raw_crash",
"processed": "get_processed"
}
else:
# old middleware
config = self.config
import socorro.storage.crashstorage as cs
store = cs.CrashStoragePool(
config,
storageClass=config.hbaseStorageClass
).crashStorage()
datatype_method_mapping = {
"raw": "get_raw_dump",
"meta": "get_meta",
"processed": "get_processed"
}
get = store.__getattribute__(datatype_method_mapping[params.datatype])
try:
if params.datatype == 'raw':
return (get(params.uuid), 'application/octet-stream')
else:
return get(params.uuid)
except (CrashIDNotFound, OoidNotFoundException):
if params.datatype == 'processed':
self.get(datatype='raw', uuid=params.uuid)
j = priorityjobs.Priorityjobs(config=self.config)
j.create(uuid=params.uuid)
raise ResourceUnavailable(params.uuid)
raise ResourceNotFound(params.uuid)
def get(self, **kwargs):
'''Return data about a field from its name. '''
filters = [
('name', None, 'str'),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.name:
raise MissingArgumentError("name")
sql = '''/* socorro.external.postgresql.field.Field.get */
SELECT
raw_field,
transforms,
product
FROM data_dictionary
WHERE raw_field=%(name)s
'''
error_message = 'Failed to retrieve field data from PostgreSQL'
results = self.query(sql, params, error_message=error_message)
field_data = {'name': None, 'transforms': None, 'product': None}
if not results:
return field_data
field_data = dict(zip(('name', 'transforms', 'product'), results[0]))
return field_data
def get(self, **kwargs):
"""Return a job in the priority queue. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingArgumentError('uuid')
sql = """
/* socorro.external.postgresql.priorityjobs.Priorityjobs.get */
SELECT uuid FROM priorityjobs WHERE uuid=%(uuid)s
"""
error_message = "Failed to retrieve priorityjobs data from PostgreSQL"
results = self.query(sql, params, error_message=error_message)
jobs = []
for row in results:
job = dict(zip(("uuid",), row))
jobs.append(job)
return {
"hits": jobs,
"total": len(jobs)
}
def get(self, **kwargs):
"""Return a job in the job queue. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingArgumentError('uuid')
fields = [
"id", "pathname", "uuid", "owner", "priority", "queueddatetime",
"starteddatetime", "completeddatetime", "success", "message"
]
sql = """
/* socorro.external.postgresql.job.Job.get */
SELECT %s FROM jobs WHERE uuid=%%(uuid)s
""" % ", ".join(fields)
error_message = "Failed to retrieve jobs data from PostgreSQL"
results = self.query(sql, params, error_message=error_message)
jobs = []
for row in results:
job = dict(zip(fields, row))
# Make sure all dates are turned into strings
for i in job:
if isinstance(job[i], datetime.datetime):
job[i] = datetimeutil.date_to_string(job[i])
jobs.append(job)
return {"hits": jobs, "total": len(jobs)}
def post(self, **kwargs):
params = external_common.parse_arguments(self.filters, kwargs)
if not params['signatures']:
raise MissingArgumentError('signatures')
sql_params = [tuple(params['signatures'])]
sql = """
SELECT
signature,
first_report AS first_date,
first_build
FROM signatures
WHERE signature IN %s
"""
error_message = 'Failed to retrieve signatures from PostgreSQL'
results = self.query(sql, sql_params, error_message=error_message)
signatures = []
for sig in results.zipped():
sig.first_date = datetimeutil.date_to_string(sig.first_date)
signatures.append(sig)
return {'hits': signatures, 'total': len(signatures)}
def get(self, **kwargs):
filters = [
("vendor_hex", None, "str"),
("adapter_hex", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
for key in ('vendor_hex', 'adapter_hex'):
if not params[key]:
raise MissingArgumentError(key)
sql_where = """
WHERE
vendor_hex = %(vendor_hex)s
AND
adapter_hex = %(adapter_hex)s
"""
sql_query = """
SELECT
vendor_hex, adapter_hex, vendor_name, adapter_name
FROM graphics_device
"""
results = self.query(sql_query + sql_where, params)
keys = 'vendor_hex', 'adapter_hex', 'vendor_name', 'adapter_name'
hits = [dict(zip(keys, x)) for x in results]
return {'hits': hits, 'total': len(hits)}
def post(self, **kwargs):
"""Return a list of signature - bug id associations. """
params = external_common.parse_arguments(self.filters, kwargs)
if not params.signatures:
raise MissingArgumentError('signatures')
# Preparing variables for the SQL query
signatures = []
sql_params = {}
for i, elem in enumerate(params.signatures):
signatures.append("%%(signature%s)s" % i)
sql_params["signature%s" % i] = elem
sql = """/* socorro.external.postgresql.bugs.Bugs.get */
SELECT ba.signature, bugs.id
FROM bugs
JOIN bug_associations AS ba ON bugs.id = ba.bug_id
WHERE EXISTS(
SELECT 1 FROM bug_associations
WHERE bug_associations.bug_id = bugs.id
AND signature IN (%s)
)
""" % ", ".join(signatures)
sql = str(" ".join(sql.split())) # better formatting of the sql string
error_message = "Failed to retrieve bugs associations from PostgreSQL"
results = self.query(sql, sql_params, error_message=error_message)
bugs = []
for row in results:
bug = dict(zip(("signature", "id"), row))
bugs.append(bug)
return {"hits": bugs, "total": len(bugs)}
def get(self, **kwargs):
"""Return a dict that holds the throttling value per build type
for a specific product."""
filters = [
('product', None, 'str'),
]
params = external_common.parse_arguments(filters, kwargs)
required = ('product',)
for key in required:
if not params.get(key):
raise MissingArgumentError(key)
sql = """
SELECT
build_type,
throttle
FROM product_build_types
WHERE product_name = %(product)s
"""
results = self.query(sql, params)
build_types = {}
for row in results.zipped():
build_types[row.build_type] = row.throttle
return {
'hits': build_types,
}
def create(self, **kwargs):
"""Add a new job to the priority queue
"""
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingArgumentError('uuid')
with self.context() as connection:
try:
self.config.logger.debug(
'Inserting priority job into RabbitMQ %s', params.uuid)
connection.channel.basic_publish(
exchange='',
routing_key=self.config.priority_queue_name,
body=params.uuid,
properties=pika.BasicProperties(delivery_mode=2))
except ChannelClosed:
self.config.logger.error(
"Failed inserting priorityjobs data into RabbitMQ",
exc_info=True)
return False
return True
def get(self, **kwargs):
filters = [
("vendor_hex", None, ["list", "str"]),
("adapter_hex", None, ["list", "str"]),
]
params = external_common.parse_arguments(filters, kwargs)
for key in ('vendor_hex', 'adapter_hex'):
param = params[key]
if not param:
raise MissingArgumentError(key)
params[key] = tuple(params[key])
sql_query = """
SELECT
vendor_hex, adapter_hex, vendor_name, adapter_name
FROM graphics_device
WHERE vendor_hex IN %(vendor_hex)s
AND adapter_hex IN %(adapter_hex)s
"""
results = self.query(sql_query, params)
hits = results.zipped()
return {
'hits': hits,
'total': len(hits)
}
def get_paireduuid(self, **kwargs):
"""Return paired uuid given a uuid and an optional hangid.
If a hangid is passed, then return only one result. Otherwise, return
all found paired uuids.
"""
filters = [
("uuid", None, "str"),
("hangid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingArgumentError('uuid')
crash_date = datetimeutil.uuid_to_date(params.uuid)
sql = """
/* socorro.external.postgresql.crashes.Crashes.get_paireduuid */
SELECT uuid
FROM reports r
WHERE r.uuid != %(uuid)s
AND r.date_processed BETWEEN
TIMESTAMP %(crash_date)s - CAST('1 day' AS INTERVAL) AND
TIMESTAMP %(crash_date)s + CAST('1 day' AS INTERVAL)
"""
sql_params = {"uuid": params.uuid, "crash_date": crash_date}
if params.hangid is not None:
sql = """%s
AND r.hangid = %%(hangid)s
LIMIT 1
""" % sql
sql_params["hangid"] = params.hangid
else:
sql = """%s
AND r.hangid IN (
SELECT hangid
FROM reports r2
WHERE r2.date_processed BETWEEN
TIMESTAMP %%(crash_date)s - CAST('1 day' AS INTERVAL)
AND
TIMESTAMP %%(crash_date)s + CAST('1 day' AS INTERVAL)
AND r2.uuid = %%(uuid)s
)
""" % sql
# Query the database
error_message = "Failed to retrieve paired uuids from PostgreSQL"
results = self.query(sql, sql_params, error_message=error_message)
# Transforming the results into what we want
uuids = [dict(zip(("uuid", ), row)) for row in results]
return {"hits": uuids, "total": len(uuids)}
def get_adu_by_signature(self, **kwargs):
"""Return a list of ADUs and crash counts by signature and ADU date
"""
now = datetimeutil.utc_now().date()
lastweek = now - datetime.timedelta(weeks=1)
filters = [
("start_date", lastweek, "date"),
("end_date", now, "date"),
("signature", None, "str"),
("channel", None, "str"),
("product_name", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
for param in ("start_date", "end_date", "signature", "channel"):
if not params[param]:
raise MissingArgumentError(param)
if (params.end_date -
params.start_date) > datetime.timedelta(days=365):
raise BadArgumentError('Duration too long. Max 365 days.')
sql_query = """
SELECT
product_name,
signature,
adu_date::TEXT,
build_date::TEXT,
buildid::TEXT,
crash_count,
adu_count,
os_name,
channel
FROM crash_adu_by_build_signature
WHERE adu_date BETWEEN %(start_date)s AND %(end_date)s
AND product_name = %(product_name)s
AND channel = %(channel)s
AND signature = %(signature)s
ORDER BY buildid
"""
error_message = (
"Failed to retrieve crash ADU by build signature from PostgreSQL")
results = self.query(sql_query, params, error_message=error_message)
fields = [
'product_name', 'signature', 'adu_date', 'build_date', 'buildid',
'crash_count', 'adu_count', 'os_name', 'channel'
]
crashes = [dict(zip(fields, row)) for row in results]
return {"hits": crashes, "total": len(crashes)}
def create_field(self, **kwargs):
"""Create a new field in the database, to be used by supersearch and
all elasticsearch related services.
"""
filters = [
('name', None, 'str'),
('data_validation_type', 'enum', 'str'),
('default_value', None, 'str'),
('description', None, 'str'),
('form_field_choices', None, ['list', 'str']),
('has_full_version', False, 'bool'),
('in_database_name', None, 'str'),
('is_exposed', False, 'bool'),
('is_returned', False, 'bool'),
('is_mandatory', False, 'bool'),
('query_type', 'enum', 'str'),
('namespace', None, 'str'),
('permissions_needed', None, ['list', 'str']),
('storage_mapping', None, 'json'),
]
params = external_common.parse_arguments(filters, kwargs)
mandatory_params = ('name', 'in_database_name')
for param in mandatory_params:
if not params[param]:
raise MissingArgumentError(param)
# Before making the change, make sure it does not break indexing.
new_mapping = self.get_mapping(overwrite_mapping=params)
# Try the mapping. If there is an error, an exception will be raised.
# If an exception is raised, the new mapping will be rejected.
self.test_mapping(new_mapping)
es_connection = self.get_connection().get_es()
try:
es_connection.index(
index=self.config.elasticsearch_default_index,
doc_type='supersearch_fields',
doc=params,
id=params['name'],
overwrite_existing=False,
refresh=True,
)
except ElasticHttpError, e:
if e.status_code == 409:
# This field exists in the database, it thus cannot be created!
raise InsertionError(
'The field "%s" already exists in the database, '
'impossible to create it. ' % params['name'])
# Else this is an unexpected error and we want to know about it.
raise
def get(self, **kwargs):
"""Return a single crash report from its UUID. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if params.uuid is None:
raise MissingArgumentError("uuid")
crash_date = datetimeutil.uuid_to_date(params.uuid)
logger.debug("Looking for crash %s during day %s" % (params.uuid,
crash_date))
sql = """/* socorro.external.postgresql.crash.Crash.get */
SELECT
reports.signature,
reports.email,
reports.url,
reports.addons_checked,
reports.exploitability,
( SELECT reports_duplicates.duplicate_of
FROM reports_duplicates
WHERE reports_duplicates.uuid = reports.uuid
) as duplicate_of
FROM reports
WHERE reports.uuid=%(uuid)s
AND reports.success IS NOT NULL
AND utc_day_is( reports.date_processed, %(crash_date)s)
"""
sql_params = {
"uuid": params.uuid,
"crash_date": crash_date
}
error_message = "Failed to retrieve crash data from PostgreSQL"
results = self.query(sql, sql_params, error_message=error_message)
crashes = []
for row in results:
crash = dict(zip((
"signature",
"email",
"url",
"addons_checked",
"exploitability",
"duplicate_of"), row))
crashes.append(crash)
return {
"hits": crashes,
"total": len(crashes)
}
def get(self, **kwargs):
'''Return the result of a custom query. '''
params = external_common.parse_arguments(self.filters, kwargs)
if not params.query:
raise MissingArgumentError('query')
try:
query = json.loads(params.query)
except ValueError:
raise BadArgumentError(
'query',
msg="Invalid JSON value for parameter 'query'"
)
es = pyelasticsearch.ElasticSearch(
urls=self.config.elasticsearch_urls,
timeout=self.config.elasticsearch_timeout_extended,
)
# Set indices.
indices = []
if not params.indices:
# By default, use the last two indices.
today = utc_now()
last_week = today - datetime.timedelta(days=7)
indices = self.generate_list_of_indexes(last_week, today)
elif len(params.indices) == 1 and params.indices[0] == 'ALL':
# If we want all indices, just do nothing.
pass
else:
indices = params.indices
search_args = {}
if indices:
search_args['index'] = indices
search_args['doc_type'] = self.config.elasticsearch_doctype
try:
results = es.search(
query,
**search_args
)
except ElasticHttpNotFoundError, e:
missing_index = re.findall(BAD_INDEX_REGEX, e.error)[0]
raise ResourceNotFound(
"elasticsearch index '%s' does not exist" % missing_index
)
def prepare_search_params(self, **kwargs):
"""Return a dictionary of parameters for a search-like SQL query.
Uses socorro.lib.search_common.get_parameters() for arguments
filtering.
"""
params = search_common.get_parameters(kwargs)
if not params["signature"]:
raise MissingArgumentError('signature')
params["terms"] = params["signature"]
params["search_mode"] = "is_exactly"
# Default mode falls back to starts_with for postgres
if params["plugin_search_mode"] == "default":
params["plugin_search_mode"] = "starts_with"
# Searching for terms in plugins
if params["report_process"] == "plugin" and params["plugin_terms"]:
params["plugin_terms"] = " ".join(params["plugin_terms"])
params["plugin_terms"] = Crashes.prepare_terms(
params["plugin_terms"],
params["plugin_search_mode"]
)
# Get information about the versions
util_service = Util(config=self.context)
params["versions_info"] = util_service.versions_info(**params)
# Parsing the versions
params["versions_string"] = params["versions"]
(params["versions"], params["products"]) = Crashes.parse_versions(
params["versions"],
params["products"]
)
# Changing the OS ids to OS names
if hasattr(self.context, 'webapi'):
context = self.context.webapi
else:
# old middleware
context = self.context
for i, elem in enumerate(params["os"]):
for platform in context.platforms:
if platform["id"] == elem:
params["os"][i] = platform["name"]
return params
def delete(self, **kwargs):
params = external_common.parse_arguments(self.filters, kwargs)
if not params.category:
raise MissingArgumentError('category')
if not params.rule:
raise MissingArgumentError('rule')
sql_params = [params.category, params.rule]
count_sql = """
/* socorro.external.postgresql.skiplist.SkipList.delete */
SELECT COUNT(*) FROM skiplist
WHERE category=%s AND rule=%s
"""
sql = """
/* socorro.external.postgresql.skiplist.SkipList.delete */
DELETE FROM skiplist
WHERE category=%s AND rule=%s
"""
connection = self.database.connection()
try:
cur = connection.cursor()
count = self.count(count_sql, sql_params, connection=connection)
if not count:
return False
cur.execute(sql, sql_params)
connection.commit()
except psycopg2.Error:
connection.rollback()
error_message = "Failed delete skip list in PostgreSQL"
logger.error(error_message)
raise DatabaseError(error_message)
finally:
connection.close()
return True
def create(self, **kwargs):
"""Add a new job to the priority queue if not already in that queue.
"""
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingArgumentError('uuid')
sql = """
/* socorro.external.postgresql.priorityjobs.Priorityjobs.create */
INSERT INTO priorityjobs (uuid) VALUES (%(uuid)s)
"""
sql_exists = """
/* socorro.external.postgresql.priorityjobs.Priorityjobs.create */
SELECT 1 FROM priorityjobs WHERE uuid=%(uuid)s
"""
with self.get_connection() as connection:
try:
cur = connection.cursor()
# Verifying that the uuid is not already in the queue
cur.execute(sql_exists, params)
if cur.rowcount:
logger.debug('The uuid %s is already in the priorityjobs '
'table' % params.uuid)
return False
logger.debug('Adding the uuid %s to the priorityjobs table' %
params.uuid)
cur.execute(sql, params)
except psycopg2.Error:
error = "Failed inserting priorityjobs data into PostgreSQL"
logger.error(error, exc_info=True)
connection.rollback()
return False
else:
connection.commit()
return bool(cur.rowcount)
finally:
cur.close()
return True
def get_count_by_day(self, **kwargs):
"""Returns the number of crashes on a daily basis"""
filters = [
("signature", None, "str"),
("start_date", None, "date"),
("end_date", None, "date")
]
DATE_FORMAT = "%Y-%m-%d"
params = external_common.parse_arguments(filters, kwargs)
for param in ("signature", "start_date"):
if not params[param]:
raise MissingArgumentError(param)
if not params.end_date:
params.end_date = params.start_date + datetime.timedelta(1)
sql = """
SELECT
COUNT(*),
date_processed::date
FROM
reports_clean rc
JOIN signatures ON
rc.signature_id=signatures.signature_id
WHERE
rc.date_processed >= %(start_date)s AND
rc.date_processed::date < %(end_date)s AND
signatures.signature=%(signature)s
GROUP BY
rc.date_processed::date
"""
hits = {}
for count, date in self.query(sql, params):
hits[date.strftime(DATE_FORMAT)] = count
current = params.start_date
while current < params.end_date:
hits.setdefault(current.strftime(DATE_FORMAT), 0)
current += datetime.timedelta(1)
return {"hits": hits, "total": len(hits)}
def post(self, **kwargs):
"""Return a list of signatures-to-bug_ids or bug_ids-to-signatures
associations. """
params = external_common.parse_arguments(self.filters, kwargs)
if not params['signatures'] and not params['bug_ids']:
raise MissingArgumentError('specify one of signatures or bug_ids')
elif params['signatures'] and params['bug_ids']:
raise BadArgumentError('specify only one of signatures or bug_ids')
sql_params = []
if params['signatures']:
sql_params.append(tuple(params.signatures))
sql = """/* socorro.external.postgresql.bugs.Bugs.get */
SELECT ba.signature, bugs.id
FROM bugs
JOIN bug_associations AS ba ON bugs.id = ba.bug_id
WHERE EXISTS(
SELECT 1 FROM bug_associations
WHERE bug_associations.bug_id = bugs.id
AND signature IN %s
)
"""
elif params['bug_ids']:
sql_params.append(tuple(params.bug_ids))
sql = """/* socorro.external.postgresql.bugs.Bugs.get */
SELECT ba.signature, bugs.id
FROM bugs
JOIN bug_associations AS ba ON bugs.id = ba.bug_id
WHERE bugs.id IN %s
"""
error_message = "Failed to retrieve bug associations from PostgreSQL"
results = self.query(sql, sql_params, error_message=error_message)
bugs = []
for row in results:
bug = dict(zip(("signature", "id"), row))
bugs.append(bug)
return {"hits": bugs, "total": len(bugs)}
def get(self, **kwargs):
""" return GC crashes per build ID """
for arg in ['product', 'version']:
if not kwargs.get(arg):
raise MissingArgumentError(arg)
now = datetimeutil.utc_now().date()
lastweek = now - datetime.timedelta(weeks=1)
filters = [
("product", None, "str"),
("version", None, "str"),
("from_date", lastweek, "date"),
("to_date", now, "date"),
]
params = external_common.parse_arguments(filters, kwargs)
result = self.query(
"""
/* socorro.external.postgresql.gccrashes.GCCrashes.get */
SELECT
build::text,
sum(gc_count_madu)
FROM gccrashes
JOIN product_versions
USING (product_version_id)
WHERE product_name = %(product)s
AND version_string = %(version)s
AND report_date BETWEEN %(from_date)s AND %(to_date)s
AND build IS NOT NULL
GROUP BY build
ORDER BY build
""", params)
# Because we don't return a list of dicts, we turn it into a
# pure list first so it becomes a list of tuples.
rows = list(result)
return {'hits': rows, 'total': len(rows)}
def get(self, **kwargs):
filters = [
("backfill_type", None, "str"),
("reports_clean", True, "bool"),
("check_period", '01:00:00', "str"),
("table_name", None, "str"),
("update_day", None, "datetime"),
("start_date", None, "datetime"),
("end_date", None, "datetime"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.backfill_type:
raise MissingArgumentError('backfill_type')
date_param = ['update_day', 'start_date', 'end_date']
for i in date_param:
if i in kwargs:
params[i] = str(params[i].date())
try:
query = 'SELECT backfill_%(backfill_type)s (%(params)s); '
required_params = BACKFILL_PARAMETERS[params.backfill_type]
query_params = [(i, params[i]) for i in required_params]
query_params_str = ', '.join('%(' + str(i[0]) + ')s'
for i in query_params)
query = query % {
'backfill_type': params.backfill_type,
'params': query_params_str
}
except:
raise BadArgumentError(kwargs['backfill_type'])
error_message = "Failed to retrieve backfill %s from PostgreSQL"
error_message = error_message % kwargs['backfill_type']
results = self.query(query, params, error_message=error_message)
return results
def get(self, **kwargs):
"""Return a single error report from its UUID. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if params.uuid is None:
raise MissingArgumentError('uuid')
crash_date = datetimeutil.uuid_to_date(params.uuid)
sql = """/* socorro.external.postgresql.error.Error.get */
SELECT
bixie.crashes.signature,
bixie.crashes.product,
bixie.crashes.error
FROM bixie.crashes
WHERE bixie.crashes.crash_id=%(uuid)s
AND bixie.crashes.success IS NOT NULL
AND utc_day_is(
bixie.crashes.processor_completed_datetime,
%(crash_date)s
)
"""
sql_params = {"uuid": params.uuid, "crash_date": crash_date}
error_message = "Failed to retrieve error data from PostgreSQL"
results = self.query(sql, sql_params, error_message=error_message)
errors = []
for row in results:
error = dict(zip(("product", "error", "signature"), row))
errors.append(error)
return {"hits": errors, "total": len(errors)}
def delete_field(self, **kwargs):
"""Remove a field from the database.
Removing a field means that it won't be indexed in elasticsearch
anymore, nor will it be exposed or accessible via supersearch. It
doesn't delete the data from crash reports though, so it would be
possible to re-create the field and reindex some indices to get that
data back.
"""
filters = [
('name', None, 'str'),
]
params = external_common.parse_arguments(filters, kwargs)
if not params['name']:
raise MissingArgumentError('name')
es_connection = self.get_connection()
es_connection.delete(
index=self.config.elasticsearch.elasticsearch_default_index,
doc_type='supersearch_fields',
id=params['name'],
refresh=True,
)
def get_parameters(self, **kwargs):
parameters = {}
for param in self.filters:
values = kwargs.get(param.name, param.default)
if values in ('', []):
# Those values are equivalent to None here.
# Note that we cannot use bool(), because 0 is not equivalent
# to None in our case.
values = None
if values is None and param.mandatory:
raise MissingArgumentError(param.name)
if values is None and param.default is not None:
values = param.default
# all values can be a list, so we make them all lists to simplify
if values is not None and not isinstance(values, (list, tuple)):
values = [values]
if values is not None:
no_operator_param = None
for value in values:
operator = None
operator_not = False
operators = OPERATORS_MAP.get(param.data_type,
OPERATORS_MAP['default'])
if isinstance(value, basestring):
if value.startswith(OPERATOR_NOT):
operator_not = True
value = value[1:]
for ope in operators:
if value.startswith(ope):
operator = ope
value = value[len(ope):]
break
# ensure the right data type
try:
value = convert_to_type(value, param.data_type)
except ValueError:
raise BadArgumentError(
param.name,
msg='Bad value for parameter %s:'
' "%s" is not a valid %s' %
(param.name, value, param.data_type))
if not param.name in parameters:
parameters[param.name] = []
if not operator:
if not no_operator_param:
no_operator_param = SearchParam(
param.name, [value], operator, param.data_type,
operator_not)
else:
no_operator_param.value.append(value)
else:
parameters[param.name].append(
SearchParam(param.name, value, operator,
param.data_type, operator_not))
if no_operator_param:
parameters[no_operator_param.name].append(
no_operator_param)
self.fix_date_parameter(parameters)
self.fix_process_type_parameter(parameters)
self.fix_hang_type_parameter(parameters)
return parameters
def get_parameters(self, **kwargs):
parameters = {}
fields = kwargs['_fields']
assert fields
if fields:
self.build_filters(fields)
for param in self.filters:
values = kwargs.get(param.name, param.default)
if values in ('', []):
# Those values are equivalent to None here.
# Note that we cannot use bool(), because 0 is not equivalent
# to None in our case.
values = None
if values is None and param.mandatory:
raise MissingArgumentError(param.name)
if values is None and param.default is not None:
values = param.default
# all values can be a list, so we make them all lists to simplify
if values is not None and not isinstance(values, (list, tuple)):
values = [values]
if values is not None:
# There should only be one parameter with no operator, and
# we want to stack all values into it. That's why we want
# to keep track of it.
# Actually, we want _two_ parameters with no operator: one
# for each possible value of "operator_not".
no_operator_param = {True: None, False: None}
for value in values:
operator = None
operator_not = False
operators = OPERATORS_MAP.get(param.data_type,
OPERATORS_MAP['default'])
if isinstance(value, basestring):
if value.startswith(OPERATOR_NOT):
operator_not = True
value = value[1:]
for ope in operators:
if value.startswith(ope):
operator = ope
value = value[len(ope):]
break
# ensure the right data type
try:
value = convert_to_type(value, param.data_type)
except ValueError:
raise BadArgumentError(
param.name,
msg='Bad value for parameter %s:'
' "%s" is not a valid %s' %
(param.name, value, param.data_type))
if param.name not in parameters:
parameters[param.name] = []
if not operator:
if not no_operator_param[operator_not]:
no_operator_param[operator_not] = SearchParam(
param.name, [value], operator, param.data_type,
operator_not)
else:
no_operator_param[operator_not].value.append(value)
else:
parameters[param.name].append(
SearchParam(param.name, value, operator,
param.data_type, operator_not))
for value in no_operator_param.values():
if value:
parameters[value.name].append(value)
self.fix_date_parameter(parameters)
self.fix_process_type_parameter(parameters)
self.fix_hang_type_parameter(parameters)
self.fix_version_parameter(parameters)
return parameters
def get_exploitability(self, **kwargs):
"""Return a list of exploitable crash reports.
See socorro.lib.external_common.parse_arguments() for all filters.
"""
now = datetimeutil.utc_now().date()
lastweek = now - datetime.timedelta(weeks=1)
filters = [
("start_date", lastweek, "date"),
("end_date", now, "date"),
("product", None, "str"),
("version", None, "str"),
("page", None, "int"),
("batch", None, "int"),
]
params = external_common.parse_arguments(filters, kwargs)
sql_where = """
report_date BETWEEN %(start_date)s AND %(end_date)s
AND
null_count + none_count + low_count + medium_count + high_count > 4
"""
if params.product:
sql_where += " AND pv.product_name = %(product)s"
if params.version:
sql_where += " AND pv.version_string = %(version)s"
inner_with_sql = """
SELECT
signature,
SUM(high_count) AS high_count,
SUM(medium_count) AS medium_count,
SUM(low_count) AS low_count,
SUM(null_count) AS null_count,
SUM(none_count) AS none_count,
SUM(high_count) + SUM(medium_count) AS med_or_high
FROM exploitability_reports
JOIN product_versions AS pv USING (product_version_id)
WHERE
high_count + medium_count + null_count + none_count > 4
AND
%s
GROUP BY signature
""" % (sql_where, )
count_sql_query = """
/* external.postgresql.crashes.Crashes.get_exploitability */
WITH sums AS (
%s
)
SELECT
count(signature)
FROM sums
""" % (inner_with_sql, )
results = self.query(
count_sql_query,
params,
error_message="Failed to retrieve exploitable crashes count")
total_crashes_count, = results[0]
sql_query = """
/* external.postgresql.crashes.Crashes.get_exploitability */
WITH sums AS (
%s
)
SELECT
signature,
high_count,
medium_count,
low_count,
null_count,
none_count
FROM sums
ORDER BY
med_or_high DESC, signature ASC
""" % (inner_with_sql, )
if params['page'] is not None:
if params['page'] <= 0:
raise BadArgumentError('page', params['page'], 'starts on 1')
if params['batch'] is None:
raise MissingArgumentError('batch')
sql_query += """
LIMIT %(limit)s
OFFSET %(offset)s
"""
params['limit'] = params['batch']
params['offset'] = params['batch'] * (params['page'] - 1)
error_message = (
"Failed to retrieve exploitable crashes from PostgreSQL")
results = self.query(sql_query, params, error_message=error_message)
# Transforming the results into what we want
crashes = []
for row in results:
crash = dict(
zip(("signature", "high_count", "medium_count", "low_count",
"null_count", "none_count"), row))
crashes.append(crash)
return {"hits": crashes, "total": total_crashes_count}
