def test_parse_arguments_with_class_validators(self):
class NumberConverter(object):
def clean(self, value):
conv = {'one': 1, 'two': 2, 'three': 3}
try:
return conv[value]
except KeyError:
raise ValueError('No idea?!')
# Define a set of filters with some types being non-trivial types
# but instead a custom validator.
filters = [
("param1", 0, NumberConverter()),
]
arguments = {
"param1": "one",
}
params_exp = DotDict()
params_exp.param1 = 1
params = external_common.parse_arguments(
filters,
arguments,
modern=True
)
assert params == params_exp
# note that a ValueError becomes a BadArgumentError
arguments = {
"param1": "will cause a ValueError in NumberConverter.clean",
}
with pytest.raises(BadArgumentError):
external_common.parse_arguments(filters, arguments, modern=True)
def test_parse_arguments_with_class_validators(self):
class NumberConverter(object):
def clean(self, value):
conv = {'one': 1, 'two': 2, 'three': 3}
try:
return conv[value]
except KeyError:
raise ValueError('No idea?!')
# Define a set of filters with some types being non-trivial types
# but instead a custom validator.
filters = [
("param1", 0, NumberConverter()),
]
arguments = {
"param1": "one",
}
params_exp = util.DotDict()
params_exp.param1 = 1
params = external_common.parse_arguments(filters,
arguments,
modern=True)
assert params == params_exp
# note that a ValueError becomes a BadArgumentError
arguments = {
"param1": "will cause a ValueError in NumberConverter.clean",
}
with pytest.raises(BadArgumentError):
external_common.parse_arguments(filters, arguments, modern=True)
def get(self, **kwargs):
filters = [
("vendor_hex", None, "str"),
("adapter_hex", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
for key in ('vendor_hex', 'adapter_hex'):
if not params[key]:
raise MissingArgumentError(key)
sql_where = """
WHERE
vendor_hex = %(vendor_hex)s
AND
adapter_hex = %(adapter_hex)s
"""
sql_query = """
SELECT
vendor_hex, adapter_hex, vendor_name, adapter_name
FROM graphics_device
"""
results = self.query(sql_query + sql_where, params)
keys = 'vendor_hex', 'adapter_hex', 'vendor_name', 'adapter_name'
hits = [dict(zip(keys, x)) for x in results]
return {'hits': hits, 'total': len(hits)}
def get(self, **kwargs):
"""Return a dict that holds the throttling value per build type
for a specific product."""
filters = [
('product', None, 'str'),
]
params = external_common.parse_arguments(filters, kwargs)
required = ('product',)
for key in required:
if not params.get(key):
raise MissingArgumentError(key)
sql = """
SELECT
build_type,
throttle::REAL
FROM product_build_types
WHERE product_name = %(product)s
"""
results = self.query(sql, params)
build_types = {}
for row in results.zipped():
build_types[row['build_type']] = row['throttle']
return {
'hits': build_types,
}
def get(self, **kwargs):
filters = [
("vendor_hex", None, "str"),
("adapter_hex", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
for key in ('vendor_hex', 'adapter_hex'):
if not params[key]:
raise MissingArgumentError(key)
sql_where = """
WHERE
vendor_hex = %(vendor_hex)s
AND
adapter_hex = %(adapter_hex)s
"""
sql_query = """
SELECT
vendor_hex, adapter_hex, vendor_name, adapter_name
FROM graphics_device
"""
results = self.query(sql_query + sql_where, params)
keys = 'vendor_hex', 'adapter_hex', 'vendor_name', 'adapter_name'
hits = [dict(zip(keys, x)) for x in results]
return {'hits': hits, 'total': len(hits)}
def get_adu_by_signature(self, **kwargs):
"""Return a list of ADUs and crash counts by signature and ADU date
"""
now = datetimeutil.utc_now().date()
lastweek = now - datetime.timedelta(weeks=1)
filters = [
("start_date", lastweek, "date"),
("end_date", now, "date"),
("signature", None, "str"),
("channel", None, "str")
]
params = external_common.parse_arguments(filters, kwargs)
for param in ("start_date", "end_date", "signature", "channel"):
if not params[param]:
raise MissingArgumentError(param)
if (params.end_date - params.start_date) > datetime.timedelta(days=60):
raise BadArgumentError('Duration too long. Max 60 days.')
sql_query = """
SELECT
signature,
adu_date::TEXT,
build_date::TEXT,
buildid::TEXT,
crash_count,
adu_count,
os_name,
channel
FROM crash_adu_by_build_signature
WHERE adu_date BETWEEN %(start_date)s AND %(end_date)s
AND channel = %(channel)s
AND signature = %(signature)s
ORDER BY buildid
"""
error_message = (
"Failed to retrieve crash ADU by build signature from PostgreSQL"
)
results = self.query(sql_query, params, error_message=error_message)
fields = [
'signature',
'adu_date',
'build_date',
'buildid',
'crash_count',
'adu_count',
'os_name',
'channel'
]
crashes = [dict(zip(fields, row)) for row in results]
return {
"hits": crashes,
"total": len(crashes)
}
def get_signatures(self, **kwargs):
"""Return top crashers by signatures.
See http://socorro.readthedocs.org/en/latest/middleware.html#tcbs
"""
filters = [
("product", None, "str"),
("version", None, "str"),
("crash_type", "all", "str"),
("to_date", datetimeutil.utc_now(), "datetime"),
("duration", datetime.timedelta(7), "timedelta"),
("os", None, "str"),
("limit", 100, "int"),
("date_range_type", None, "str")
]
params = external_common.parse_arguments(filters, kwargs)
params.logger = logger
try:
connection = self.database.connection()
cursor = connection.cursor()
return tcbs.twoPeriodTopCrasherComparison(cursor, params)
finally:
connection.close()
def get(self, **kwargs):
filters = [
('product', None, 'str'),
('version', None, 'str'),
('build_id', None, 'int'),
]
params = external_common.parse_arguments(filters, kwargs)
required = ('product', 'build_id', 'version')
for key in required:
if not params.get(key):
raise MissingArgumentError(key)
sql = """
SELECT
pv.version_string
FROM product_versions pv
LEFT JOIN product_version_builds pvb ON
(pv.product_version_id = pvb.product_version_id)
WHERE pv.product_name = %(product)s
AND pv.release_version = %(version)s
AND pvb.build_id = %(build_id)s
"""
results = self.query(sql, params)
# The query can return multiple results, but they're the same value. So
# we just return the first one.
version_string = [
row['version_string'] for row in results.zipped()
]
if version_string:
version_string = [version_string[0]]
return {
'hits': version_string
}
def get_featured(self, **kwargs):
"""Return a list of featured versions for one, several or all products.
"""
filters = [
("products", None, ["list", "str"]),
]
params = external_common.parse_arguments(filters, kwargs)
sql = """
SELECT product_name, version_string
FROM product_info
WHERE is_featured = true
"""
sql_params = {}
if params.products and params.products[0]:
sql += " AND product_name IN %(product)s"
sql_params['product'] = tuple(params.products)
error_message = "Failed to retrieve featured versions from PostgreSQL"
sql_results = self.query(sql, sql_params, error_message=error_message)
hits = {}
total = 0
for row in sql_results:
total += 1
version = dict(zip(("product", "version"), row))
if version["product"] not in hits:
hits[version["product"]] = [version["version"]]
else:
hits[version["product"]].append(version["version"])
return {"total": total, "hits": hits}
def get(self, **kwargs):
filters = [("date", datetime.datetime.utcnow().date(), "date"), ("product", "Firefox", "str")]
params = external_common.parse_arguments(filters, kwargs)
params["tomorrow"] = params["date"] + datetime.timedelta(days=1)
results = self.query(SQL, params)
hits = results.zipped()
return {"hits": hits, "total": len(hits)}
def get(self, **kwargs):
"""Return a single crash report from it's UUID. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
day = int(params.uuid[-2:])
month = int(params.uuid[-4:-2])
# assuming we won't use this after year 2099
year = int("20%s" % params.uuid[-6:-4])
crash_date = datetime.date(year=year, month=month, day=day)
logger.debug("Looking for crash %s during day %s" % (params.uuid,
crash_date))
sql = """/* socorro.external.postgresql.crash.Crash.get */
SELECT reports.email, reports.url, reports.addons_checked,
( SELECT reports_duplicates.duplicate_of
FROM reports_duplicates
WHERE reports_duplicates.uuid = reports.uuid
) as duplicate_of
FROM reports
WHERE reports.uuid=%(uuid)s
AND reports.success IS NOT NULL
AND utc_day_is( reports.date_processed, %(crash_date)s)
"""
sql_params = {
"uuid": params.uuid,
"crash_date": crash_date
}
results = []
# Creating the connection to the DB
self.connection = self.database.connection()
cur = self.connection.cursor()
try:
results = db.execute(cur, sql, sql_params)
except psycopg2.Error:
util.reportExceptionAndContinue(logger)
json_result = {
"total": 0,
"hits": []
}
for crash in results:
row = dict(zip((
"email",
"url",
"addons_checked",
"duplicate_of"), crash))
json_result["hits"].append(row)
json_result["total"] = len(json_result["hits"])
self.connection.close()
return json_result
def get_default_version(self, **kwargs):
"""Return the default version of one or several products. """
filters = [
("products", None, ["list", "str"])
]
params = external_common.parse_arguments(filters, kwargs)
sql = """
/* socorro.external.postgresql.products.get_default_version */
SELECT
product_name AS product,
version_string AS version
FROM default_versions
"""
if params.products and params.products[0] != "":
params.products = tuple(params.products)
sql = "%s WHERE product_name IN %%(products)s" % sql
error_message = "Failed to retrieve default versions from PostgreSQL"
results = self.query(sql, params, error_message=error_message)
products = {}
for product in results.zipped():
products[product.product] = product.version
return {
"hits": products
}
def get(self, **kwargs):
"""Return a dict that holds the throttling value per build type
for a specific product."""
filters = [
('product', None, 'str'),
]
params = external_common.parse_arguments(filters, kwargs)
required = ('product', )
for key in required:
if not params.get(key):
raise MissingArgumentError(key)
sql = """
SELECT
build_type,
throttle::REAL
FROM product_build_types
WHERE product_name = %(product)s
"""
results = self.query(sql, params)
build_types = {}
for row in results.zipped():
build_types[row['build_type']] = row['throttle']
return {
'hits': build_types,
}
def post(self, **kwargs):
"""Return a list of signature - bug id associations. """
params = external_common.parse_arguments(self.filters, kwargs)
if not params.signatures:
raise MissingArgumentError('signatures')
# Preparing variables for the SQL query
signatures = []
sql_params = {}
for i, elem in enumerate(params.signatures):
signatures.append("%%(signature%s)s" % i)
sql_params["signature%s" % i] = elem
sql = """/* socorro.external.postgresql.bugs.Bugs.get */
SELECT ba.signature, bugs.id
FROM bugs
JOIN bug_associations AS ba ON bugs.id = ba.bug_id
WHERE EXISTS(
SELECT 1 FROM bug_associations
WHERE bug_associations.bug_id = bugs.id
AND signature IN (%s)
)
""" % ", ".join(signatures)
sql = str(" ".join(sql.split())) # better formatting of the sql string
error_message = "Failed to retrieve bugs associations from PostgreSQL"
results = self.query(sql, sql_params, error_message=error_message)
bugs = []
for row in results:
bug = dict(zip(("signature", "id"), row))
bugs.append(bug)
return {"hits": bugs, "total": len(bugs)}
def get(self, **kwargs):
'''Return data about a field from its name. '''
filters = [
('name', None, 'str'),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.name:
raise MissingArgumentError("name")
sql = '''/* socorro.external.postgresql.field.Field.get */
SELECT
raw_field AS name,
transforms,
product
FROM data_dictionary
WHERE raw_field=%(name)s
'''
error_message = 'Failed to retrieve field data from PostgreSQL'
results = self.query(sql, params, error_message=error_message)
field_data = {
'name': None,
'transforms': None,
'product': None
}
if not results:
return field_data
field_data = results.zipped()[0]
return field_data
def post(self, **kwargs):
params = external_common.parse_arguments(self.filters, kwargs)
if not params['signatures']:
raise MissingArgumentError('signatures')
sql_params = [tuple(params['signatures'])]
sql = """
SELECT
signature,
first_report AS first_date,
first_build
FROM signatures
WHERE signature IN %s
"""
error_message = 'Failed to retrieve signatures from PostgreSQL'
results = self.query(sql, sql_params, error_message=error_message)
signatures = []
for sig in results.zipped():
sig.first_date = datetimeutil.date_to_string(sig.first_date)
signatures.append(sig)
return {'hits': signatures, 'total': len(signatures)}
def get(self, **kwargs):
"""Return a job in the priority queue. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
sql = """
/* socorro.external.postgresql.priorityjobs.Priorityjobs.get */
SELECT uuid FROM priorityjobs WHERE uuid=%(uuid)s
"""
json_result = {"total": 0, "hits": []}
try:
# Creating the connection to the DB
connection = self.database.connection()
cur = connection.cursor()
results = db.execute(cur, sql, params)
except psycopg2.Error:
logger.error("Failed retrieving priorityjobs data from PostgreSQL",
exc_info=True)
else:
for job in results:
row = dict(zip(("uuid", ), job))
json_result["hits"].append(row)
json_result["total"] = len(json_result["hits"])
finally:
connection.close()
return json_result
def get(self, **kwargs):
"""Return JSON data of a crash report, given its uuid. """
filters = [
('uuid', None, 'str'),
('datatype', None, 'str')
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingArgumentError('uuid')
if not params.datatype:
raise MissingArgumentError('datatype')
store = self.config.filesystem.filesystem_class(self.config.filesystem)
datatype_method_mapping = {
'raw': 'get_raw_dump',
'meta': 'get_raw_crash',
'processed': 'get_processed'
}
get = store.__getattribute__(datatype_method_mapping[params.datatype])
try:
if params.datatype == 'raw':
return (get(params.uuid), 'application/octet-stream')
else:
return get(params.uuid)
except CrashIDNotFound:
if params.datatype == 'processed':
self.get(datatype='raw', uuid=params.uuid)
j = priorityjobs.Priorityjobs(config=self.config)
j.create(uuid=params.uuid)
raise ResourceUnavailable(params.uuid)
raise ResourceNotFound(params.uuid)
def create(self, **kwargs):
"""
Create a new build for a product.
See http://socorro.readthedocs.org/en/latest/middleware.html#builds
Required keyword arguments:
product - Concerned product, e.g. firefox
version - Concerned version, e.g. 9.0a1
platform - Platform for this build, e.g. win32
build_id - Build ID for this build (yyyymmdd######)
build_type - Type of build, e.g. Nightly, Beta, Aurora, Release
Required if build_type is Beta:
beta_number - The beta number, e.g. 9.0b#
Optional keyword arguments:
repository - Repository this build came from
Return: (product_name, version)
"""
# Parse arguments
filters = [
("product", None, "str"),
("version", None, "str"),
("platform", None, "str"),
("build_id", None, "str"),
("build_type", None, "str"),
("beta_number", None, "str"),
("repository", "", "str")
]
params = external_common.parse_arguments(filters, kwargs)
self._require_parameters(params, "product", "version", "platform",
"build_id", "build_type")
if params["build_type"].lower() == "beta":
self._require_parameters(params, "beta_number")
try:
connection = self.database.connection()
cursor = connection.cursor()
buildutil.insert_build(cursor, params["product"],
params["version"], params["platform"],
params["build_id"], params["build_type"],
params["beta_number"],
params["repository"])
except psycopg2.Error:
logger.error("Failed inserting build data into PostgresSQL",
exc_info=True)
connection.rollback()
raise
else:
connection.commit()
finally:
connection.close()
return (params["product"], params["version"])
def get_signatures(self, **kwargs):
"""Return top crashers by signatures.
See http://socorro.readthedocs.org/en/latest/middleware.html#tcbs
"""
filters = [
("product", None, "str"),
("version", None, "str"),
("crash_type", "all", "str"),
("to_date", datetimeutil.utc_now(), "datetime"),
("duration", datetime.timedelta(7), "timedelta"),
("os", None, "str"),
("limit", 100, "int"),
("date_range_type", None, "str")
]
params = external_common.parse_arguments(filters, kwargs)
params.logger = logger
try:
connection = self.database.connection()
cursor = connection.cursor()
return tcbs.twoPeriodTopCrasherComparison(cursor, params)
finally:
connection.close()
def get_default_version(self, **kwargs):
"""Return the default version of one or several products. """
filters = [("products", None, ["list", "str"])]
params = external_common.parse_arguments(filters, kwargs)
sql = """
/* socorro.external.postgresql.products.Products.get_default_version */
SELECT product_name, version_string
FROM default_versions
"""
if params.products and params.products[0] != "":
params.products = tuple(params.products)
sql = "%s WHERE product_name IN %%(products)s" % sql
try:
connection = self.database.connection()
cursor = connection.cursor()
cursor.execute(sql, params)
results = cursor.fetchall()
except psycopg2.Error:
results = []
logger.error("Failed to retrieve default versions from PostgreSQL",
exc_info=True)
finally:
connection.close()
products = {}
for row in results:
product = dict(zip(("product", "version"), row))
products[product["product"]] = product["version"]
return {"hits": products}
def create(self, **kwargs):
"""
Create a new build for a product.
See http://socorro.readthedocs.org/en/latest/middleware.html#builds
Required keyword arguments:
product - Concerned product, e.g. firefox
version - Concerned version, e.g. 9.0a1
platform - Platform for this build, e.g. win32
build_id - Build ID for this build (yyyymmdd######)
build_type - Type of build, e.g. Nightly, Beta, Aurora, Release
Required if build_type is Beta:
beta_number - The beta number, e.g. 9.0b#
Optional keyword arguments:
repository - Repository this build came from
Return: (product_name, version)
"""
# Parse arguments
filters = [
("product", None, "str"),
("version", None, "str"),
("platform", None, "str"),
("build_id", None, "int"),
("build_type", None, "str"),
("beta_number", None, "int"),
("repository", "", "str")
]
params = external_common.parse_arguments(filters, kwargs)
self._require_parameters(params, "product", "version", "platform",
"build_id", "build_type")
if params["build_type"].lower() == "beta":
self._require_parameters(params, "beta_number")
connection = None
try:
connection = self.database.connection()
cursor = connection.cursor()
buildutil.insert_build(cursor, params["product"],
params["version"], params["platform"],
params["build_id"], params["build_type"],
params["beta_number"],
params["repository"])
except psycopg2.Error, e:
error = str(e)
logger.error("Failed inserting build data into PostgresSQL, "
"reason: %s" % error,
exc_info=True)
connection.rollback()
if "CONTEXT" in error:
error = error[0:error.index("CONTEXT")]
raise InsertionError(error)
def get_signatures(self, **kwargs):
"""Return top crashers by signatures.
See http://socorro.readthedocs.org/en/latest/middleware.html#tcbs
"""
filters = [
("product", None, "str"),
("version", None, "str"),
("crash_type", "all", "str"),
("to_date", datetimeutil.utc_now(), "datetime"),
("duration", datetime.timedelta(7), "timedelta"),
("os", None, "str"),
("limit", 100, "int"),
("date_range_type", None, "str")
]
params = external_common.parse_arguments(filters, kwargs)
params.logger = logger
# what the twoPeriodTopCrasherComparison() function does is that it
# makes a start date from taking the to_date - duration
if params.duration > datetime.timedelta(30):
raise BadArgumentError('Duration too long. Max 30 days.')
with self.get_connection() as connection:
return tcbs.twoPeriodTopCrasherComparison(connection, params)
def post(self, **kwargs):
params = external_common.parse_arguments(self.filters, kwargs)
if not params.category:
raise MissingArgumentError('category')
if not params.rule:
raise MissingArgumentError('rule')
sql = """
/* socorro.external.postgresql.skiplist.SkipList.post */
INSERT INTO skiplist (category, rule)
VALUES (%s, %s);
"""
sql_params = [params.category, params.rule]
connection = self.database.connection()
try:
with connection.cursor() as cur:
cur.execute(sql, sql_params)
connection.commit()
except psycopg2.Error:
connection.rollback()
error_message = "Failed updating skip list in PostgreSQL"
logger.error(error_message)
raise DatabaseError(error_message)
finally:
connection.close()
return True
def get_default_version(self, **kwargs):
"""Return the default version of one or several products. """
filters = [("products", None, ["list", "str"])]
params = external_common.parse_arguments(filters, kwargs)
sql = """
/* socorro.external.postgresql.products.Products.get_default_version */
SELECT product_name, version_string
FROM default_versions
"""
if params.products and params.products[0] != "":
params.products = tuple(params.products)
sql = "%s WHERE product_name IN %%(products)s" % sql
try:
connection = self.database.connection()
cursor = connection.cursor()
cursor.execute(sql, params)
results = cursor.fetchall()
except psycopg2.Error:
results = []
logger.error("Failed to retrieve default versions from PostgreSQL", exc_info=True)
finally:
connection.close()
products = {}
for row in results:
product = dict(zip(("product", "version"), row))
products[product["product"]] = product["version"]
return {"hits": products}
def get(self, **kwargs):
params = external_common.parse_arguments(self.filters, kwargs)
sql_params = []
sql = """
/* socorro.external.postgresql.skiplist.SkipList.get */
SELECT category,
rule
FROM skiplist
WHERE 1=1
"""
if params.category:
sql += 'AND category=%s'
sql_params.append(params.category)
if params.rule:
sql += 'AND rule=%s'
sql_params.append(params.rule)
# Use `UPPER()` to make the sort case insensitive
# which makes it more user-friendly on the UI later
sql += """
ORDER BY UPPER(category), UPPER(rule)
"""
error_message = "Failed to retrieve skip list data from PostgreSQL"
sql_results = self.query(sql, sql_params, error_message=error_message)
results = [dict(zip(("category", "rule"), x)) for x in sql_results]
return {'hits': results, 'total': len(results)}
def get(self, **kwargs):
"""Return JSON data of a crash report, given its uuid. """
filters = [
('uuid', None, str),
('datatype', None, str),
('name', None, str) # only applicable if datatype == 'raw'
]
params = external_common.parse_arguments(filters, kwargs, modern=True)
if not params.uuid:
raise MissingArgumentError('uuid')
if not params.datatype:
raise MissingArgumentError('datatype')
datatype_method_mapping = {
'raw': 'get_raw_dump',
'meta': 'get_raw_crash',
'processed': 'get_processed',
'unredacted': 'get_unredacted_processed',
}
if params.datatype not in datatype_method_mapping:
raise BadArgumentError(params.datatype)
get = self.__getattribute__(datatype_method_mapping[params.datatype])
try:
if params.datatype == 'raw':
return get(params.uuid, name=params.name)
else:
return get(params.uuid)
except CrashIDNotFound:
# The CrashIDNotFound exception that happens inside the
# crashstorage is too revealing as exception message
# contains information about buckets and prefix keys.
# Re-wrap it here so the message is just the crash ID.
raise CrashIDNotFound(params.uuid)
def get(self, **kwargs):
'''Return data about a field from its name. '''
filters = [
('name', None, 'str'),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.name:
raise MissingArgumentError("name")
sql = '''/* socorro.external.postgresql.field.Field.get */
SELECT
raw_field,
transforms,
product
FROM data_dictionary
WHERE raw_field=%(name)s
'''
error_message = 'Failed to retrieve field data from PostgreSQL'
results = self.query(sql, params, error_message=error_message)
field_data = {'name': None, 'transforms': None, 'product': None}
if not results:
return field_data
field_data = dict(zip(('name', 'transforms', 'product'), results[0]))
return field_data
def get_default_version(self, **kwargs):
"""Return the default version of one or several products. """
filters = [
("products", None, ["list", "str"])
]
params = external_common.parse_arguments(filters, kwargs)
sql = """
/* socorro.external.postgresql.products.get_default_version */
SELECT product_name, version_string
FROM default_versions
"""
if params.products and params.products[0] != "":
params.products = tuple(params.products)
sql = "%s WHERE product_name IN %%(products)s" % sql
error_message = "Failed to retrieve default versions from PostgreSQL"
results = self.query(sql, params, error_message=error_message)
products = {}
for row in results:
product = dict(zip(("product", "version"), row))
products[product["product"]] = product["version"]
return {
"hits": products
}
def get(self, **kwargs):
filters = [
("backfill_type", None, "str"),
("reports_clean", True, "bool"),
("check_period", "01:00:00", "str"),
("table_name", None, "str"),
("update_day", None, "datetime"),
("start_date", None, "datetime"),
("end_date", None, "datetime"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.backfill_type:
raise MissingArgumentError("backfill_type")
date_param = ["update_day", "start_date", "end_date"]
for i in date_param:
if i in kwargs:
params[i] = str(params[i].date())
try:
query = "SELECT backfill_%(backfill_type)s (%(params)s); "
required_params = BACKFILL_PARAMETERS[params.backfill_type]
query_params = [(i, params[i]) for i in required_params]
query_params_str = ", ".join("%(" + str(i[0]) + ")s" for i in query_params)
query = query % {"backfill_type": params.backfill_type, "params": query_params_str}
except:
raise BadArgumentError(kwargs["backfill_type"])
error_message = "Failed to retrieve backfill %s from PostgreSQL"
error_message = error_message % kwargs["backfill_type"]
results = self.query(query, params, error_message=error_message)
return results
def get_channels(self, **kwargs):
"""Return a list of release channels for one, several or all products.
"""
filters = [
("products", None, ["list", "str"]),
]
params = external_common.parse_arguments(filters, kwargs)
sql = """
SELECT build_type, product_name
FROM product_info
"""
sql_params = {}
if params.products and params.products[0]:
sql += " WHERE product_name IN %(products)s"
sql_params['products'] = tuple(params.products)
error_message = "Failed to retrieve release channels from PostgreSQL"
sql_results = self.query(sql, sql_params, error_message=error_message)
channels = {}
for row in sql_results:
res = dict(zip(("channel", "product"), row))
if res["product"] not in channels:
channels[res["product"]] = [res["channel"]]
else:
channels[res["product"]].append(res["channel"])
return channels
def _get_sql_params(self, **kwargs):
filters = [
(
'date',
(
datetime.datetime.utcnow() - datetime.timedelta(days=1)
).date(),
'date'
),
(
'limit',
None,
int
),
]
params = external_common.parse_arguments(filters, kwargs)
sql = """
SELECT debug_file, debug_id, code_file, code_id
FROM missing_symbols
WHERE
date_processed = %(date)s AND
debug_file != '' AND
debug_id != ''
GROUP BY debug_file, debug_id, code_file, code_id
"""
if params['limit'] is not None:
sql += '\nLIMIT %(limit)s'
return sql, params
def get_channels(self, **kwargs):
"""Return a list of release channels for one, several or all products.
"""
filters = [
("products", None, ["list", "str"]),
]
params = external_common.parse_arguments(filters, kwargs)
sql = """
SELECT build_type, product_name
FROM product_info
"""
sql_params = {}
if params.products and params.products[0]:
sql += " WHERE product_name IN %(products)s"
sql_params['products'] = tuple(params.products)
error_message = "Failed to retrieve release channels from PostgreSQL"
sql_results = self.query(sql, sql_params, error_message=error_message)
channels = {}
for row in sql_results:
res = dict(zip(("channel", "product"), row))
if res["product"] not in channels:
channels[res["product"]] = [res["channel"]]
else:
channels[res["product"]].append(res["channel"])
return channels
def get(self, **kwargs):
filters = [
("report_date", None, "datetime"),
("report_type", None, "str"),
("product", None, "str"),
("version", None, "str"),
("signature", None, "str"),
("platform", None, "str"),
("min_crashes", 10, "int"),
("min_baseline_diff", 0.05, "float"),
]
params = external_common.parse_arguments(filters, kwargs)
hits = []
if params['report_type'] == 'interesting-addons':
hits = self.interesting_addons(params)
elif params['report_type'] == 'interesting-modules':
hits = self.interesting_modules(params)
elif params['report_type'] == 'interesting-addons-with-version':
hits = self.interesting_addons_with_version(params)
elif params['report_type'] == 'interesting-modules-with-version':
hits = self.interesting_modules_with_version(params)
elif params['report_type'] == 'core-counts':
hits = self.core_counts(params)
else:
raise BadArgumentError(
'report_type',
received=report_type
)
return {
'hits': hits,
'total': len(hits)
}
def create(self, **kwargs):
"""Add a new job to the priority queue
"""
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingArgumentError('uuid')
with self.context() as connection:
try:
self.config.logger.debug(
'Inserting priority job into RabbitMQ %s', params.uuid)
connection.channel.basic_publish(
exchange='',
routing_key=self.config.priority_queue_name,
body=params.uuid,
properties=pika.BasicProperties(delivery_mode=2))
except ChannelClosed:
self.config.logger.error(
"Failed inserting priorityjobs data into RabbitMQ",
exc_info=True)
return False
return True
def get_signatures(self, **kwargs):
"""Return top crashers by signatures.
See http://socorro.readthedocs.org/en/latest/middleware.html#tcbs
"""
filters = [
("product", None, "str"),
("version", None, "str"),
("crash_type", "all", "str"),
("to_date", datetimeutil.utc_now(), "datetime"),
("duration", datetime.timedelta(7), "timedelta"),
("os", None, "str"),
("limit", 100, "int"),
("date_range_type", None, "str")
]
params = external_common.parse_arguments(filters, kwargs)
params.logger = logger
# what the twoPeriodTopCrasherComparison() function does is that it
# makes a start date from taking the to_date - duration
if params.duration > datetime.timedelta(30):
raise BadArgumentError('Duration too long. Max 30 days.')
with self.get_connection() as connection:
return tcbs.twoPeriodTopCrasherComparison(connection, params)
def get_featured(self, **kwargs):
"""Return a list of featured versions for one, several or all products.
"""
filters = [
("products", None, ["list", "str"]),
]
params = external_common.parse_arguments(filters, kwargs)
sql = """
SELECT product_name, version_string
FROM product_info
WHERE is_featured = true
"""
sql_params = {}
if params.products and params.products[0]:
sql += " AND product_name IN %(product)s"
sql_params['product'] = tuple(params.products)
error_message = "Failed to retrieve featured versions from PostgreSQL"
sql_results = self.query(sql, sql_params, error_message=error_message)
hits = {}
total = 0
for row in sql_results:
total += 1
version = dict(zip(("product", "version"), row))
if version["product"] not in hits:
hits[version["product"]] = [version["version"]]
else:
hits[version["product"]].append(version["version"])
return {"total": total, "hits": hits}
def get(self, **kwargs):
"""Return a job in the priority queue. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingArgumentError('uuid')
sql = """
/* socorro.external.postgresql.priorityjobs.Priorityjobs.get */
SELECT uuid FROM priorityjobs WHERE uuid=%(uuid)s
"""
error_message = "Failed to retrieve priorityjobs data from PostgreSQL"
results = self.query(sql, params, error_message=error_message)
jobs = []
for row in results:
job = dict(zip(("uuid",), row))
jobs.append(job)
return {
"hits": jobs,
"total": len(jobs)
}
def get(self, **kwargs):
filters = [
("vendor_hex", None, ["list", "str"]),
("adapter_hex", None, ["list", "str"]),
]
params = external_common.parse_arguments(filters, kwargs)
for key in ('vendor_hex', 'adapter_hex'):
param = params[key]
if not param:
raise MissingArgumentError(key)
params[key] = tuple(params[key])
sql_query = """
SELECT
vendor_hex, adapter_hex, vendor_name, adapter_name
FROM graphics_device
WHERE vendor_hex IN %(vendor_hex)s
AND adapter_hex IN %(adapter_hex)s
"""
results = self.query(sql_query, params)
hits = results.zipped()
return {
'hits': hits,
'total': len(hits)
}
def post(self, **kwargs):
params = external_common.parse_arguments(self.filters, kwargs)
if not params['signatures']:
raise MissingArgumentError('signatures')
sql_params = [tuple(params['signatures'])]
sql = """
SELECT
signature,
first_report AS first_date,
first_build
FROM signatures
WHERE signature IN %s
"""
error_message = 'Failed to retrieve signatures from PostgreSQL'
results = self.query(sql, sql_params, error_message=error_message)
signatures = []
for sig in results.zipped():
sig.first_date = datetimeutil.date_to_string(sig.first_date)
signatures.append(sig)
return {
'hits': signatures,
'total': len(signatures)
}
def create(self, **kwargs):
"""Add a new job to the priority queue
"""
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingArgumentError('uuid')
with self.context() as connection:
try:
self.config.logger.debug(
'Inserting priority job into RabbitMQ %s', params.uuid
)
connection.channel.basic_publish(
exchange='',
routing_key=self.config.priority_queue_name,
body=params.uuid,
properties=pika.BasicProperties(delivery_mode=2)
)
except ChannelClosed:
self.config.logger.error(
"Failed inserting priorityjobs data into RabbitMQ",
exc_info=True
)
return False
return True
def get(self, **kwargs):
"""Return JSON data of a crash report, given its uuid. """
filters = [
('uuid', None, str),
('datatype', None, str),
('name', None, str) # only applicable if datatype == 'raw'
]
params = external_common.parse_arguments(filters, kwargs, modern=True)
if not params.uuid:
raise MissingArgumentError('uuid')
if not params.datatype:
raise MissingArgumentError('datatype')
datatype_method_mapping = {
'raw': 'get_raw_dump',
'meta': 'get_raw_crash',
'processed': 'get_processed',
'unredacted': 'get_unredacted_processed',
}
if params.datatype not in datatype_method_mapping:
raise BadArgumentError(params.datatype)
get = self.__getattribute__(datatype_method_mapping[params.datatype])
try:
if params.datatype == 'raw':
return get(params.uuid, name=params.name)
else:
return get(params.uuid)
except CrashIDNotFound:
# The CrashIDNotFound exception that happens inside the
# crashstorage is too revealing as exception message
# contains information about buckets and prefix keys.
# Re-wrap it here so the message is just the crash ID.
raise CrashIDNotFound(params.uuid)
def get(self, **kwargs):
"""Return a job in the job queue. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
fields = [
"id", "pathname", "uuid", "owner", "priority", "queueddatetime",
"starteddatetime", "completeddatetime", "success", "message"
]
sql = """
/* socorro.external.postgresql.job.Job.get */
SELECT %s FROM jobs WHERE uuid=%%(uuid)s
""" % ", ".join(fields)
error_message = "Failed to retrieve jobs data from PostgreSQL"
results = self.query(sql, params, error_message=error_message)
jobs = []
for row in results:
job = dict(zip(fields, row))
# Make sure all dates are turned into strings
for i in job:
if isinstance(job[i], datetime.datetime):
job[i] = datetimeutil.date_to_string(job[i])
jobs.append(job)
return {"hits": jobs, "total": len(jobs)}
def get(self, **kwargs):
"""Return a single crash report from it's UUID. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if params.uuid is None:
raise MissingOrBadArgumentException(
"Mandatory parameter 'uuid' is missing or empty")
crash_date = datetimeutil.uuid_to_date(params.uuid)
logger.debug("Looking for crash %s during day %s" % (params.uuid,
crash_date))
sql = """/* socorro.external.postgresql.crash.Crash.get */
SELECT reports.email, reports.url, reports.addons_checked,
( SELECT reports_duplicates.duplicate_of
FROM reports_duplicates
WHERE reports_duplicates.uuid = reports.uuid
) as duplicate_of
FROM reports
WHERE reports.uuid=%(uuid)s
AND reports.success IS NOT NULL
AND utc_day_is( reports.date_processed, %(crash_date)s)
"""
sql_params = {
"uuid": params.uuid,
"crash_date": crash_date
}
results = []
# Creating the connection to the DB
self.connection = self.database.connection()
cur = self.connection.cursor()
try:
results = db.execute(cur, sql, sql_params)
except psycopg2.Error:
util.reportExceptionAndContinue(logger)
json_result = {
"total": 0,
"hits": []
}
for crash in results:
row = dict(zip((
"email",
"url",
"addons_checked",
"duplicate_of"), crash))
json_result["hits"].append(row)
json_result["total"] = len(json_result["hits"])
self.connection.close()
return json_result
def get(self, **kwargs):
"""Return a single crash report from it's UUID. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if params.uuid is None:
raise MissingOrBadArgumentException(
"Mandatory parameter 'uuid' is missing or empty")
crash_date = datetimeutil.uuid_to_date(params.uuid)
logger.debug("Looking for crash %s during day %s" % (params.uuid,
crash_date))
sql = """/* socorro.external.postgresql.crash.Crash.get */
SELECT reports.email, reports.url, reports.addons_checked,
( SELECT reports_duplicates.duplicate_of
FROM reports_duplicates
WHERE reports_duplicates.uuid = reports.uuid
) as duplicate_of
FROM reports
WHERE reports.uuid=%(uuid)s
AND reports.success IS NOT NULL
AND utc_day_is( reports.date_processed, %(crash_date)s)
"""
sql_params = {
"uuid": params.uuid,
"crash_date": crash_date
}
results = []
# Creating the connection to the DB
self.connection = self.database.connection()
cur = self.connection.cursor()
try:
results = db.execute(cur, sql, sql_params)
except psycopg2.Error:
util.reportExceptionAndContinue(logger)
json_result = {
"total": 0,
"hits": []
}
for crash in results:
row = dict(zip((
"email",
"url",
"addons_checked",
"duplicate_of"), crash))
json_result["hits"].append(row)
json_result["total"] = len(json_result["hits"])
self.connection.close()
return json_result
def get(self, **kwargs):
"""Return JSON data of a crash report, given its uuid. """
filters = [
('uuid', None, 'str'),
('datatype', None, 'str')
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingArgumentError('uuid')
if not params.datatype:
raise MissingArgumentError('datatype')
# get a generic crashstorage instance from whatever external resource
# is implementing this service.
store = self.get_storage()
datatype_method_mapping = {
'raw': 'get_raw_dump',
'meta': 'get_raw_crash',
'processed': 'get_processed',
'unredacted': 'get_unredacted_processed',
}
get = store.__getattribute__(datatype_method_mapping[params.datatype])
try:
if params.datatype == 'raw':
return (get(params.uuid), 'application/octet-stream')
else:
return get(params.uuid)
except CrashIDNotFound:
if params.datatype in ('processed', 'unredacted'):
# try to fetch a raw crash just to ensure that the raw crash
# exists. If this line fails, there's no reason to actually
# submit the priority job.
try:
store.get_raw_crash(params.uuid)
except CrashIDNotFound:
raise ResourceNotFound(params.uuid)
# search through the existing other services to find the
# Priorityjob service.
try:
priorityjob_service_impl = self.all_services[
'Priorityjobs'
]
except KeyError:
raise ServiceUnavailable('Priorityjobs')
# get the underlying implementation of the Priorityjob
# service and instantiate it.
priority_job_service = priorityjob_service_impl.cls(
config=self.config
)
# create the priority job for this crash_ids
priority_job_service.create(uuid=params.uuid)
raise ResourceUnavailable(params.uuid)
raise ResourceNotFound(params.uuid)
def get(self, **kwargs):
"""Return JSON data of a crash report, given its uuid. """
filters = [
('uuid', None, 'str'),
('datatype', None, 'str'),
('name', None, 'str') # only applicable if datatype == 'raw'
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingArgumentError('uuid')
if not params.datatype:
raise MissingArgumentError('datatype')
# get a generic crashstorage instance from whatever external resource
# is implementing this service.
store = self.get_storage()
datatype_method_mapping = {
'raw': 'get_raw_dump',
'meta': 'get_raw_crash',
'processed': 'get_processed',
'unredacted': 'get_unredacted_processed',
}
get = store.__getattribute__(datatype_method_mapping[params.datatype])
try:
if params.datatype == 'raw':
return (get(params.uuid,
name=params.name), 'application/octet-stream')
else:
return get(params.uuid)
except CrashIDNotFound:
if params.datatype in ('processed', 'unredacted'):
# try to fetch a raw crash just to ensure that the raw crash
# exists. If this line fails, there's no reason to actually
# submit the priority job.
try:
store.get_raw_crash(params.uuid)
except CrashIDNotFound:
raise ResourceNotFound(params.uuid)
# search through the existing other services to find the
# Priorityjob service.
try:
priorityjob_service_impl = self.all_services[
'Priorityjobs']
except KeyError:
raise ServiceUnavailable('Priorityjobs')
# get the underlying implementation of the Priorityjob
# service and instantiate it.
priority_job_service = priorityjob_service_impl.cls(
config=self.config)
# create the priority job for this crash_ids
priority_job_service.create(uuid=params.uuid)
raise ResourceUnavailable(params.uuid)
raise ResourceNotFound(params.uuid)
def get(self, **kwargs):
"""Return a job in the job queue. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
fields = [
"id",
"pathname",
"uuid",
"owner",
"priority",
"queueddatetime",
"starteddatetime",
"completeddatetime",
"success",
"message"
]
sql = """
/* socorro.external.postgresql.job.Job.get */
SELECT %s FROM jobs WHERE uuid=%%(uuid)s
""" % ", ".join(fields)
json_result = {
"total": 0,
"hits": []
}
connection = None
try:
# Creating the connection to the DB
connection = self.database.connection()
cur = connection.cursor()
results = db.execute(cur, sql, params)
except psycopg2.Error:
logger.error("Failed retrieving jobs data from PostgreSQL",
exc_info=True)
else:
for job in results:
row = dict(zip(fields, job))
# Make sure all dates are turned into strings
for i in row:
if isinstance(row[i], datetime.datetime):
row[i] = datetimeutil.date_to_string(row[i])
json_result["hits"].append(row)
json_result["total"] = len(json_result["hits"])
finally:
if connection:
connection.close()
return json_result
def get(self, **kwargs):
yesterday = datetime.datetime.utcnow() - datetime.timedelta(1)
tomorrow = yesterday + datetime.timedelta(2)
yesterday = yesterday.date()
tomorrow = tomorrow.date()
filters = [
('start_date', yesterday, 'date'),
('end_date', tomorrow, 'date'),
('product', '', 'str'),
('versions', [], 'list'),
('platforms', [], 'list'),
]
params = external_common.parse_arguments(filters, kwargs)
required = (
'start_date',
'end_date',
'product',
'versions',
'platforms',
)
missing = []
for each in required:
if not params.get(each):
missing.append(each)
if missing:
raise MissingArgumentError(', '.join(missing))
SQL = """
SELECT
SUM(adu_count) AS adi_count,
adu_date AS date,
pv.build_type,
pv.version_string AS version
FROM
product_adu
LEFT OUTER JOIN product_versions pv USING (product_version_id)
WHERE
pv.product_name = %(product)s
AND pv.version_string IN %(versions)s
AND os_name IN %(platforms)s
AND adu_date BETWEEN %(start_date)s AND %(end_date)s
GROUP BY
adu_date,
build_type,
version_string
"""
params['versions'] = tuple(params['versions'])
params['platforms'] = tuple(params['platforms'])
results = self.query(SQL, params)
rows = []
for row in results.zipped():
row.date = row.date.strftime('%Y-%m-%d')
# BIGINTs become Decimal which becomes floating point in JSON
row.adi_count = long(row.adi_count)
rows.append(row)
return {'hits': rows, 'total': len(rows)}
def get(self, **kwargs):
filters = [
("report_types", None, ["list", "str"]),
("report_type", None, "str"),
("signature", None, "str"),
("start_date", None, "datetime"),
("end_date", None, "datetime"),
("versions", None, ["list", "str"]),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.get('report_types') and params.get('report_type'):
# to support the legacy case
individual_report = True
report_types = [params['report_type']]
else:
individual_report = False
report_types = params['report_types']
# check that all the report types are recognized
for report_type in report_types:
query_params = report_type_sql.get(report_type, {})
known_report_types = ('products', 'distinct_install',
'exploitability', 'devices', 'graphics')
if (report_type not in known_report_types
and 'first_col' not in query_params):
raise BadArgumentError(report_type)
products = []
versions = []
# Get information about the versions
util_service = Util(config=self.context)
versions_info = util_service.versions_info(**params)
if versions_info:
for i, elem in enumerate(versions_info):
products.append(versions_info[elem]["product_name"])
versions.append(str(versions_info[elem]["version_string"]))
# This MUST be a tuple otherwise it gets cast to an array
params['product'] = tuple(products)
params['version'] = tuple(versions)
all_results = {}
with self.get_connection() as connection:
for report_type in report_types:
result_cols, query_string, query_parameters = self._get_query(
report_type, params)
sql_results = self.query(query_string,
params=query_parameters,
connection=connection)
results = [dict(zip(result_cols, row)) for row in sql_results]
all_results[report_type] = results
if individual_report:
return all_results.values()[0]
else:
return {'reports': all_results}
def get(self, **kwargs):
"""Return a job in the job queue. """
filters = [
("uuid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
fields = [
"id",
"pathname",
"uuid",
"owner",
"priority",
"queueddatetime",
"starteddatetime",
"completeddatetime",
"success",
"message"
]
sql = """
/* socorro.external.postgresql.job.Job.get */
SELECT %s FROM jobs WHERE uuid=%%(uuid)s
""" % ", ".join(fields)
json_result = {
"total": 0,
"hits": []
}
connection = None
try:
# Creating the connection to the DB
connection = self.database.connection()
cur = connection.cursor()
results = db.execute(cur, sql, params)
except psycopg2.Error:
logger.error("Failed retrieving jobs data from PostgreSQL",
exc_info=True)
else:
for job in results:
row = dict(zip(fields, job))
# Make sure all dates are turned into strings
for i in row:
if isinstance(row[i], datetime.datetime):
row[i] = datetimeutil.date_to_string(row[i])
json_result["hits"].append(row)
json_result["total"] = len(json_result["hits"])
finally:
if connection:
connection.close()
return json_result
def get_paireduuid(self, **kwargs):
"""Return paired uuid given a uuid and an optional hangid.
If a hangid is passed, then return only one result. Otherwise, return
all found paired uuids.
"""
filters = [
("uuid", None, "str"),
("hangid", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
crash_date = datetimeutil.uuid_to_date(params.uuid)
sql = """
/* socorro.external.postgresql.crashes.Crashes.get_paireduuid */
SELECT uuid
FROM reports r
WHERE r.uuid != %(uuid)s
AND r.date_processed BETWEEN
TIMESTAMP %(crash_date)s - CAST('1 day' AS INTERVAL) AND
TIMESTAMP %(crash_date)s + CAST('1 day' AS INTERVAL)
"""
sql_params = {"uuid": params.uuid, "crash_date": crash_date}
if params.hangid is not None:
sql = """%s
AND r.hangid = %%(hangid)s
LIMIT 1
""" % sql
sql_params["hangid"] = params.hangid
else:
sql = """%s
AND r.hangid IN (
SELECT hangid
FROM reports r2
WHERE r2.date_processed BETWEEN
TIMESTAMP %%(crash_date)s - CAST('1 day' AS INTERVAL)
AND
TIMESTAMP %%(crash_date)s + CAST('1 day' AS INTERVAL)
AND r2.uuid = %%(uuid)s
)
""" % sql
# Query the database
error_message = "Failed to retrieve paired uuids from PostgreSQL"
results = self.query(sql, sql_params, error_message=error_message)
# Transforming the results into what we want
uuids = [dict(zip(("uuid", ), row)) for row in results]
return {"hits": uuids, "total": len(uuids)}
def create_field(self, **kwargs):
"""Create a new field in the database, to be used by supersearch and
all elasticsearch related services.
"""
filters = [
('name', None, 'str'),
('data_validation_type', 'enum', 'str'),
('default_value', None, 'str'),
('description', None, 'str'),
('form_field_choices', None, ['list', 'str']),
('has_full_version', False, 'bool'),
('in_database_name', None, 'str'),
('is_exposed', False, 'bool'),
('is_returned', False, 'bool'),
('is_mandatory', False, 'bool'),
('query_type', 'enum', 'str'),
('namespace', None, 'str'),
('permissions_needed', None, ['list', 'str']),
('storage_mapping', None, 'json'),
]
params = external_common.parse_arguments(filters, kwargs)
mandatory_params = ('name', 'in_database_name')
for param in mandatory_params:
if not params[param]:
raise MissingArgumentError(param)
# Before making the change, make sure it does not break indexing.
new_mapping = self.get_mapping(overwrite_mapping=params)
# Try the mapping. If there is an error, an exception will be raised.
# If an exception is raised, the new mapping will be rejected.
self.test_mapping(new_mapping)
es_connection = self.get_connection().get_es()
try:
es_connection.index(
index=self.config.elasticsearch_default_index,
doc_type='supersearch_fields',
doc=params,
id=params['name'],
overwrite_existing=False,
refresh=True,
)
except ElasticHttpError, e:
if e.status_code == 409:
# This field exists in the database, it thus cannot be created!
raise InsertionError(
'The field "%s" already exists in the database, '
'impossible to create it. ' % params['name']
)
# Else this is an unexpected error and we want to know about it.
raise
def get(self, **kwargs):
filters = [
("start_date", None, "datetime"),
("end_date", None, "datetime"),
("product", None, "str"),
("version", None, "str"),
]
params = external_common.parse_arguments(filters, kwargs)
results = [] # So we have something to return.
query_string = """SELECT product_name,
version_string,
product_version_id,
report_date,
nightly_builds.build_date,
days_out,
sum(report_count) as report_count
FROM nightly_builds
JOIN product_versions USING ( product_version_id )
WHERE report_date <= %(end_date)s
AND report_date >= %(start_date)s
AND product_name = %(product)s
AND version_string = %(version)s
GROUP BY product_name,
version_string,
product_version_id,
report_date,
nightly_builds.build_date,
days_out"""
try:
connection = self.database.connection()
cursor = connection.cursor()
sql_results = db.execute(cursor, query_string, params)
except psycopg2.Error:
logger.error("Failed retrieving crashtrends data from PostgreSQL",
exc_info=True)
else:
for trend in sql_results:
row = dict(zip((
"product_name",
"version_string",
"product_version_id",
"report_date",
"build_date",
"days_out",
"report_count"), trend))
row['report_date'] = datetimeutil.date_to_string(row['report_date'])
row['build_date'] = datetimeutil.date_to_string(row['build_date'])
results.append(row)
finally:
connection.close()
results = {'crashtrends' : results}
return results
