def test_parse_arguments_with_class_validators(self): class NumberConverter(object): def clean(self, value): conv = {'one': 1, 'two': 2, 'three': 3} try: return conv[value] except KeyError: raise ValueError('No idea?!') # Define a set of filters with some types being non-trivial types # but instead a custom validator. filters = [ ("param1", 0, NumberConverter()), ] arguments = { "param1": "one", } params_exp = DotDict() params_exp.param1 = 1 params = external_common.parse_arguments( filters, arguments, modern=True ) assert params == params_exp # note that a ValueError becomes a BadArgumentError arguments = { "param1": "will cause a ValueError in NumberConverter.clean", } with pytest.raises(BadArgumentError): external_common.parse_arguments(filters, arguments, modern=True)
def test_parse_arguments_with_class_validators(self): class NumberConverter(object): def clean(self, value): conv = {'one': 1, 'two': 2, 'three': 3} try: return conv[value] except KeyError: raise ValueError('No idea?!') # Define a set of filters with some types being non-trivial types # but instead a custom validator. filters = [ ("param1", 0, NumberConverter()), ] arguments = { "param1": "one", } params_exp = util.DotDict() params_exp.param1 = 1 params = external_common.parse_arguments(filters, arguments, modern=True) assert params == params_exp # note that a ValueError becomes a BadArgumentError arguments = { "param1": "will cause a ValueError in NumberConverter.clean", } with pytest.raises(BadArgumentError): external_common.parse_arguments(filters, arguments, modern=True)
def get(self, **kwargs): filters = [ ("vendor_hex", None, "str"), ("adapter_hex", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) for key in ('vendor_hex', 'adapter_hex'): if not params[key]: raise MissingArgumentError(key) sql_where = """ WHERE vendor_hex = %(vendor_hex)s AND adapter_hex = %(adapter_hex)s """ sql_query = """ SELECT vendor_hex, adapter_hex, vendor_name, adapter_name FROM graphics_device """ results = self.query(sql_query + sql_where, params) keys = 'vendor_hex', 'adapter_hex', 'vendor_name', 'adapter_name' hits = [dict(zip(keys, x)) for x in results] return {'hits': hits, 'total': len(hits)}
def get(self, **kwargs): """Return a dict that holds the throttling value per build type for a specific product.""" filters = [ ('product', None, 'str'), ] params = external_common.parse_arguments(filters, kwargs) required = ('product',) for key in required: if not params.get(key): raise MissingArgumentError(key) sql = """ SELECT build_type, throttle::REAL FROM product_build_types WHERE product_name = %(product)s """ results = self.query(sql, params) build_types = {} for row in results.zipped(): build_types[row['build_type']] = row['throttle'] return { 'hits': build_types, }
def get_adu_by_signature(self, **kwargs): """Return a list of ADUs and crash counts by signature and ADU date """ now = datetimeutil.utc_now().date() lastweek = now - datetime.timedelta(weeks=1) filters = [ ("start_date", lastweek, "date"), ("end_date", now, "date"), ("signature", None, "str"), ("channel", None, "str") ] params = external_common.parse_arguments(filters, kwargs) for param in ("start_date", "end_date", "signature", "channel"): if not params[param]: raise MissingArgumentError(param) if (params.end_date - params.start_date) > datetime.timedelta(days=60): raise BadArgumentError('Duration too long. Max 60 days.') sql_query = """ SELECT signature, adu_date::TEXT, build_date::TEXT, buildid::TEXT, crash_count, adu_count, os_name, channel FROM crash_adu_by_build_signature WHERE adu_date BETWEEN %(start_date)s AND %(end_date)s AND channel = %(channel)s AND signature = %(signature)s ORDER BY buildid """ error_message = ( "Failed to retrieve crash ADU by build signature from PostgreSQL" ) results = self.query(sql_query, params, error_message=error_message) fields = [ 'signature', 'adu_date', 'build_date', 'buildid', 'crash_count', 'adu_count', 'os_name', 'channel' ] crashes = [dict(zip(fields, row)) for row in results] return { "hits": crashes, "total": len(crashes) }
def get_signatures(self, **kwargs): """Return top crashers by signatures. See http://socorro.readthedocs.org/en/latest/middleware.html#tcbs """ filters = [ ("product", None, "str"), ("version", None, "str"), ("crash_type", "all", "str"), ("to_date", datetimeutil.utc_now(), "datetime"), ("duration", datetime.timedelta(7), "timedelta"), ("os", None, "str"), ("limit", 100, "int"), ("date_range_type", None, "str") ] params = external_common.parse_arguments(filters, kwargs) params.logger = logger try: connection = self.database.connection() cursor = connection.cursor() return tcbs.twoPeriodTopCrasherComparison(cursor, params) finally: connection.close()
def get(self, **kwargs): filters = [ ('product', None, 'str'), ('version', None, 'str'), ('build_id', None, 'int'), ] params = external_common.parse_arguments(filters, kwargs) required = ('product', 'build_id', 'version') for key in required: if not params.get(key): raise MissingArgumentError(key) sql = """ SELECT pv.version_string FROM product_versions pv LEFT JOIN product_version_builds pvb ON (pv.product_version_id = pvb.product_version_id) WHERE pv.product_name = %(product)s AND pv.release_version = %(version)s AND pvb.build_id = %(build_id)s """ results = self.query(sql, params) # The query can return multiple results, but they're the same value. So # we just return the first one. version_string = [ row['version_string'] for row in results.zipped() ] if version_string: version_string = [version_string[0]] return { 'hits': version_string }
def get_featured(self, **kwargs): """Return a list of featured versions for one, several or all products. """ filters = [ ("products", None, ["list", "str"]), ] params = external_common.parse_arguments(filters, kwargs) sql = """ SELECT product_name, version_string FROM product_info WHERE is_featured = true """ sql_params = {} if params.products and params.products[0]: sql += " AND product_name IN %(product)s" sql_params['product'] = tuple(params.products) error_message = "Failed to retrieve featured versions from PostgreSQL" sql_results = self.query(sql, sql_params, error_message=error_message) hits = {} total = 0 for row in sql_results: total += 1 version = dict(zip(("product", "version"), row)) if version["product"] not in hits: hits[version["product"]] = [version["version"]] else: hits[version["product"]].append(version["version"]) return {"total": total, "hits": hits}
def get(self, **kwargs): filters = [("date", datetime.datetime.utcnow().date(), "date"), ("product", "Firefox", "str")] params = external_common.parse_arguments(filters, kwargs) params["tomorrow"] = params["date"] + datetime.timedelta(days=1) results = self.query(SQL, params) hits = results.zipped() return {"hits": hits, "total": len(hits)}
def get(self, **kwargs): """Return a single crash report from it's UUID. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) day = int(params.uuid[-2:]) month = int(params.uuid[-4:-2]) # assuming we won't use this after year 2099 year = int("20%s" % params.uuid[-6:-4]) crash_date = datetime.date(year=year, month=month, day=day) logger.debug("Looking for crash %s during day %s" % (params.uuid, crash_date)) sql = """/* socorro.external.postgresql.crash.Crash.get */ SELECT reports.email, reports.url, reports.addons_checked, ( SELECT reports_duplicates.duplicate_of FROM reports_duplicates WHERE reports_duplicates.uuid = reports.uuid ) as duplicate_of FROM reports WHERE reports.uuid=%(uuid)s AND reports.success IS NOT NULL AND utc_day_is( reports.date_processed, %(crash_date)s) """ sql_params = { "uuid": params.uuid, "crash_date": crash_date } results = [] # Creating the connection to the DB self.connection = self.database.connection() cur = self.connection.cursor() try: results = db.execute(cur, sql, sql_params) except psycopg2.Error: util.reportExceptionAndContinue(logger) json_result = { "total": 0, "hits": [] } for crash in results: row = dict(zip(( "email", "url", "addons_checked", "duplicate_of"), crash)) json_result["hits"].append(row) json_result["total"] = len(json_result["hits"]) self.connection.close() return json_result
def get_default_version(self, **kwargs): """Return the default version of one or several products. """ filters = [ ("products", None, ["list", "str"]) ] params = external_common.parse_arguments(filters, kwargs) sql = """ /* socorro.external.postgresql.products.get_default_version */ SELECT product_name AS product, version_string AS version FROM default_versions """ if params.products and params.products[0] != "": params.products = tuple(params.products) sql = "%s WHERE product_name IN %%(products)s" % sql error_message = "Failed to retrieve default versions from PostgreSQL" results = self.query(sql, params, error_message=error_message) products = {} for product in results.zipped(): products[product.product] = product.version return { "hits": products }
def get(self, **kwargs): """Return a dict that holds the throttling value per build type for a specific product.""" filters = [ ('product', None, 'str'), ] params = external_common.parse_arguments(filters, kwargs) required = ('product', ) for key in required: if not params.get(key): raise MissingArgumentError(key) sql = """ SELECT build_type, throttle::REAL FROM product_build_types WHERE product_name = %(product)s """ results = self.query(sql, params) build_types = {} for row in results.zipped(): build_types[row['build_type']] = row['throttle'] return { 'hits': build_types, }
def post(self, **kwargs): """Return a list of signature - bug id associations. """ params = external_common.parse_arguments(self.filters, kwargs) if not params.signatures: raise MissingArgumentError('signatures') # Preparing variables for the SQL query signatures = [] sql_params = {} for i, elem in enumerate(params.signatures): signatures.append("%%(signature%s)s" % i) sql_params["signature%s" % i] = elem sql = """/* socorro.external.postgresql.bugs.Bugs.get */ SELECT ba.signature, bugs.id FROM bugs JOIN bug_associations AS ba ON bugs.id = ba.bug_id WHERE EXISTS( SELECT 1 FROM bug_associations WHERE bug_associations.bug_id = bugs.id AND signature IN (%s) ) """ % ", ".join(signatures) sql = str(" ".join(sql.split())) # better formatting of the sql string error_message = "Failed to retrieve bugs associations from PostgreSQL" results = self.query(sql, sql_params, error_message=error_message) bugs = [] for row in results: bug = dict(zip(("signature", "id"), row)) bugs.append(bug) return {"hits": bugs, "total": len(bugs)}
def get(self, **kwargs): '''Return data about a field from its name. ''' filters = [ ('name', None, 'str'), ] params = external_common.parse_arguments(filters, kwargs) if not params.name: raise MissingArgumentError("name") sql = '''/* socorro.external.postgresql.field.Field.get */ SELECT raw_field AS name, transforms, product FROM data_dictionary WHERE raw_field=%(name)s ''' error_message = 'Failed to retrieve field data from PostgreSQL' results = self.query(sql, params, error_message=error_message) field_data = { 'name': None, 'transforms': None, 'product': None } if not results: return field_data field_data = results.zipped()[0] return field_data
def post(self, **kwargs): params = external_common.parse_arguments(self.filters, kwargs) if not params['signatures']: raise MissingArgumentError('signatures') sql_params = [tuple(params['signatures'])] sql = """ SELECT signature, first_report AS first_date, first_build FROM signatures WHERE signature IN %s """ error_message = 'Failed to retrieve signatures from PostgreSQL' results = self.query(sql, sql_params, error_message=error_message) signatures = [] for sig in results.zipped(): sig.first_date = datetimeutil.date_to_string(sig.first_date) signatures.append(sig) return {'hits': signatures, 'total': len(signatures)}
def get(self, **kwargs): """Return a job in the priority queue. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") sql = """ /* socorro.external.postgresql.priorityjobs.Priorityjobs.get */ SELECT uuid FROM priorityjobs WHERE uuid=%(uuid)s """ json_result = {"total": 0, "hits": []} try: # Creating the connection to the DB connection = self.database.connection() cur = connection.cursor() results = db.execute(cur, sql, params) except psycopg2.Error: logger.error("Failed retrieving priorityjobs data from PostgreSQL", exc_info=True) else: for job in results: row = dict(zip(("uuid", ), job)) json_result["hits"].append(row) json_result["total"] = len(json_result["hits"]) finally: connection.close() return json_result
def get(self, **kwargs): """Return JSON data of a crash report, given its uuid. """ filters = [ ('uuid', None, 'str'), ('datatype', None, 'str') ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingArgumentError('uuid') if not params.datatype: raise MissingArgumentError('datatype') store = self.config.filesystem.filesystem_class(self.config.filesystem) datatype_method_mapping = { 'raw': 'get_raw_dump', 'meta': 'get_raw_crash', 'processed': 'get_processed' } get = store.__getattribute__(datatype_method_mapping[params.datatype]) try: if params.datatype == 'raw': return (get(params.uuid), 'application/octet-stream') else: return get(params.uuid) except CrashIDNotFound: if params.datatype == 'processed': self.get(datatype='raw', uuid=params.uuid) j = priorityjobs.Priorityjobs(config=self.config) j.create(uuid=params.uuid) raise ResourceUnavailable(params.uuid) raise ResourceNotFound(params.uuid)
def create(self, **kwargs): """ Create a new build for a product. See http://socorro.readthedocs.org/en/latest/middleware.html#builds Required keyword arguments: product - Concerned product, e.g. firefox version - Concerned version, e.g. 9.0a1 platform - Platform for this build, e.g. win32 build_id - Build ID for this build (yyyymmdd######) build_type - Type of build, e.g. Nightly, Beta, Aurora, Release Required if build_type is Beta: beta_number - The beta number, e.g. 9.0b# Optional keyword arguments: repository - Repository this build came from Return: (product_name, version) """ # Parse arguments filters = [ ("product", None, "str"), ("version", None, "str"), ("platform", None, "str"), ("build_id", None, "str"), ("build_type", None, "str"), ("beta_number", None, "str"), ("repository", "", "str") ] params = external_common.parse_arguments(filters, kwargs) self._require_parameters(params, "product", "version", "platform", "build_id", "build_type") if params["build_type"].lower() == "beta": self._require_parameters(params, "beta_number") try: connection = self.database.connection() cursor = connection.cursor() buildutil.insert_build(cursor, params["product"], params["version"], params["platform"], params["build_id"], params["build_type"], params["beta_number"], params["repository"]) except psycopg2.Error: logger.error("Failed inserting build data into PostgresSQL", exc_info=True) connection.rollback() raise else: connection.commit() finally: connection.close() return (params["product"], params["version"])
def get_default_version(self, **kwargs): """Return the default version of one or several products. """ filters = [("products", None, ["list", "str"])] params = external_common.parse_arguments(filters, kwargs) sql = """ /* socorro.external.postgresql.products.Products.get_default_version */ SELECT product_name, version_string FROM default_versions """ if params.products and params.products[0] != "": params.products = tuple(params.products) sql = "%s WHERE product_name IN %%(products)s" % sql try: connection = self.database.connection() cursor = connection.cursor() cursor.execute(sql, params) results = cursor.fetchall() except psycopg2.Error: results = [] logger.error("Failed to retrieve default versions from PostgreSQL", exc_info=True) finally: connection.close() products = {} for row in results: product = dict(zip(("product", "version"), row)) products[product["product"]] = product["version"] return {"hits": products}
def create(self, **kwargs): """ Create a new build for a product. See http://socorro.readthedocs.org/en/latest/middleware.html#builds Required keyword arguments: product - Concerned product, e.g. firefox version - Concerned version, e.g. 9.0a1 platform - Platform for this build, e.g. win32 build_id - Build ID for this build (yyyymmdd######) build_type - Type of build, e.g. Nightly, Beta, Aurora, Release Required if build_type is Beta: beta_number - The beta number, e.g. 9.0b# Optional keyword arguments: repository - Repository this build came from Return: (product_name, version) """ # Parse arguments filters = [ ("product", None, "str"), ("version", None, "str"), ("platform", None, "str"), ("build_id", None, "int"), ("build_type", None, "str"), ("beta_number", None, "int"), ("repository", "", "str") ] params = external_common.parse_arguments(filters, kwargs) self._require_parameters(params, "product", "version", "platform", "build_id", "build_type") if params["build_type"].lower() == "beta": self._require_parameters(params, "beta_number") connection = None try: connection = self.database.connection() cursor = connection.cursor() buildutil.insert_build(cursor, params["product"], params["version"], params["platform"], params["build_id"], params["build_type"], params["beta_number"], params["repository"]) except psycopg2.Error, e: error = str(e) logger.error("Failed inserting build data into PostgresSQL, " "reason: %s" % error, exc_info=True) connection.rollback() if "CONTEXT" in error: error = error[0:error.index("CONTEXT")] raise InsertionError(error)
def get_signatures(self, **kwargs): """Return top crashers by signatures. See http://socorro.readthedocs.org/en/latest/middleware.html#tcbs """ filters = [ ("product", None, "str"), ("version", None, "str"), ("crash_type", "all", "str"), ("to_date", datetimeutil.utc_now(), "datetime"), ("duration", datetime.timedelta(7), "timedelta"), ("os", None, "str"), ("limit", 100, "int"), ("date_range_type", None, "str") ] params = external_common.parse_arguments(filters, kwargs) params.logger = logger # what the twoPeriodTopCrasherComparison() function does is that it # makes a start date from taking the to_date - duration if params.duration > datetime.timedelta(30): raise BadArgumentError('Duration too long. Max 30 days.') with self.get_connection() as connection: return tcbs.twoPeriodTopCrasherComparison(connection, params)
def post(self, **kwargs): params = external_common.parse_arguments(self.filters, kwargs) if not params.category: raise MissingArgumentError('category') if not params.rule: raise MissingArgumentError('rule') sql = """ /* socorro.external.postgresql.skiplist.SkipList.post */ INSERT INTO skiplist (category, rule) VALUES (%s, %s); """ sql_params = [params.category, params.rule] connection = self.database.connection() try: with connection.cursor() as cur: cur.execute(sql, sql_params) connection.commit() except psycopg2.Error: connection.rollback() error_message = "Failed updating skip list in PostgreSQL" logger.error(error_message) raise DatabaseError(error_message) finally: connection.close() return True
def get(self, **kwargs): params = external_common.parse_arguments(self.filters, kwargs) sql_params = [] sql = """ /* socorro.external.postgresql.skiplist.SkipList.get */ SELECT category, rule FROM skiplist WHERE 1=1 """ if params.category: sql += 'AND category=%s' sql_params.append(params.category) if params.rule: sql += 'AND rule=%s' sql_params.append(params.rule) # Use `UPPER()` to make the sort case insensitive # which makes it more user-friendly on the UI later sql += """ ORDER BY UPPER(category), UPPER(rule) """ error_message = "Failed to retrieve skip list data from PostgreSQL" sql_results = self.query(sql, sql_params, error_message=error_message) results = [dict(zip(("category", "rule"), x)) for x in sql_results] return {'hits': results, 'total': len(results)}
def get(self, **kwargs): """Return JSON data of a crash report, given its uuid. """ filters = [ ('uuid', None, str), ('datatype', None, str), ('name', None, str) # only applicable if datatype == 'raw' ] params = external_common.parse_arguments(filters, kwargs, modern=True) if not params.uuid: raise MissingArgumentError('uuid') if not params.datatype: raise MissingArgumentError('datatype') datatype_method_mapping = { 'raw': 'get_raw_dump', 'meta': 'get_raw_crash', 'processed': 'get_processed', 'unredacted': 'get_unredacted_processed', } if params.datatype not in datatype_method_mapping: raise BadArgumentError(params.datatype) get = self.__getattribute__(datatype_method_mapping[params.datatype]) try: if params.datatype == 'raw': return get(params.uuid, name=params.name) else: return get(params.uuid) except CrashIDNotFound: # The CrashIDNotFound exception that happens inside the # crashstorage is too revealing as exception message # contains information about buckets and prefix keys. # Re-wrap it here so the message is just the crash ID. raise CrashIDNotFound(params.uuid)
def get(self, **kwargs): '''Return data about a field from its name. ''' filters = [ ('name', None, 'str'), ] params = external_common.parse_arguments(filters, kwargs) if not params.name: raise MissingArgumentError("name") sql = '''/* socorro.external.postgresql.field.Field.get */ SELECT raw_field, transforms, product FROM data_dictionary WHERE raw_field=%(name)s ''' error_message = 'Failed to retrieve field data from PostgreSQL' results = self.query(sql, params, error_message=error_message) field_data = {'name': None, 'transforms': None, 'product': None} if not results: return field_data field_data = dict(zip(('name', 'transforms', 'product'), results[0])) return field_data
def get_default_version(self, **kwargs): """Return the default version of one or several products. """ filters = [ ("products", None, ["list", "str"]) ] params = external_common.parse_arguments(filters, kwargs) sql = """ /* socorro.external.postgresql.products.get_default_version */ SELECT product_name, version_string FROM default_versions """ if params.products and params.products[0] != "": params.products = tuple(params.products) sql = "%s WHERE product_name IN %%(products)s" % sql error_message = "Failed to retrieve default versions from PostgreSQL" results = self.query(sql, params, error_message=error_message) products = {} for row in results: product = dict(zip(("product", "version"), row)) products[product["product"]] = product["version"] return { "hits": products }
def get(self, **kwargs): filters = [ ("backfill_type", None, "str"), ("reports_clean", True, "bool"), ("check_period", "01:00:00", "str"), ("table_name", None, "str"), ("update_day", None, "datetime"), ("start_date", None, "datetime"), ("end_date", None, "datetime"), ] params = external_common.parse_arguments(filters, kwargs) if not params.backfill_type: raise MissingArgumentError("backfill_type") date_param = ["update_day", "start_date", "end_date"] for i in date_param: if i in kwargs: params[i] = str(params[i].date()) try: query = "SELECT backfill_%(backfill_type)s (%(params)s); " required_params = BACKFILL_PARAMETERS[params.backfill_type] query_params = [(i, params[i]) for i in required_params] query_params_str = ", ".join("%(" + str(i[0]) + ")s" for i in query_params) query = query % {"backfill_type": params.backfill_type, "params": query_params_str} except: raise BadArgumentError(kwargs["backfill_type"]) error_message = "Failed to retrieve backfill %s from PostgreSQL" error_message = error_message % kwargs["backfill_type"] results = self.query(query, params, error_message=error_message) return results
def get_channels(self, **kwargs): """Return a list of release channels for one, several or all products. """ filters = [ ("products", None, ["list", "str"]), ] params = external_common.parse_arguments(filters, kwargs) sql = """ SELECT build_type, product_name FROM product_info """ sql_params = {} if params.products and params.products[0]: sql += " WHERE product_name IN %(products)s" sql_params['products'] = tuple(params.products) error_message = "Failed to retrieve release channels from PostgreSQL" sql_results = self.query(sql, sql_params, error_message=error_message) channels = {} for row in sql_results: res = dict(zip(("channel", "product"), row)) if res["product"] not in channels: channels[res["product"]] = [res["channel"]] else: channels[res["product"]].append(res["channel"]) return channels
def _get_sql_params(self, **kwargs): filters = [ ( 'date', ( datetime.datetime.utcnow() - datetime.timedelta(days=1) ).date(), 'date' ), ( 'limit', None, int ), ] params = external_common.parse_arguments(filters, kwargs) sql = """ SELECT debug_file, debug_id, code_file, code_id FROM missing_symbols WHERE date_processed = %(date)s AND debug_file != '' AND debug_id != '' GROUP BY debug_file, debug_id, code_file, code_id """ if params['limit'] is not None: sql += '\nLIMIT %(limit)s' return sql, params
def get(self, **kwargs): filters = [ ("report_date", None, "datetime"), ("report_type", None, "str"), ("product", None, "str"), ("version", None, "str"), ("signature", None, "str"), ("platform", None, "str"), ("min_crashes", 10, "int"), ("min_baseline_diff", 0.05, "float"), ] params = external_common.parse_arguments(filters, kwargs) hits = [] if params['report_type'] == 'interesting-addons': hits = self.interesting_addons(params) elif params['report_type'] == 'interesting-modules': hits = self.interesting_modules(params) elif params['report_type'] == 'interesting-addons-with-version': hits = self.interesting_addons_with_version(params) elif params['report_type'] == 'interesting-modules-with-version': hits = self.interesting_modules_with_version(params) elif params['report_type'] == 'core-counts': hits = self.core_counts(params) else: raise BadArgumentError( 'report_type', received=report_type ) return { 'hits': hits, 'total': len(hits) }
def create(self, **kwargs): """Add a new job to the priority queue """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingArgumentError('uuid') with self.context() as connection: try: self.config.logger.debug( 'Inserting priority job into RabbitMQ %s', params.uuid) connection.channel.basic_publish( exchange='', routing_key=self.config.priority_queue_name, body=params.uuid, properties=pika.BasicProperties(delivery_mode=2)) except ChannelClosed: self.config.logger.error( "Failed inserting priorityjobs data into RabbitMQ", exc_info=True) return False return True
def get(self, **kwargs): """Return a job in the priority queue. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingArgumentError('uuid') sql = """ /* socorro.external.postgresql.priorityjobs.Priorityjobs.get */ SELECT uuid FROM priorityjobs WHERE uuid=%(uuid)s """ error_message = "Failed to retrieve priorityjobs data from PostgreSQL" results = self.query(sql, params, error_message=error_message) jobs = [] for row in results: job = dict(zip(("uuid",), row)) jobs.append(job) return { "hits": jobs, "total": len(jobs) }
def get(self, **kwargs): filters = [ ("vendor_hex", None, ["list", "str"]), ("adapter_hex", None, ["list", "str"]), ] params = external_common.parse_arguments(filters, kwargs) for key in ('vendor_hex', 'adapter_hex'): param = params[key] if not param: raise MissingArgumentError(key) params[key] = tuple(params[key]) sql_query = """ SELECT vendor_hex, adapter_hex, vendor_name, adapter_name FROM graphics_device WHERE vendor_hex IN %(vendor_hex)s AND adapter_hex IN %(adapter_hex)s """ results = self.query(sql_query, params) hits = results.zipped() return { 'hits': hits, 'total': len(hits) }
def post(self, **kwargs): params = external_common.parse_arguments(self.filters, kwargs) if not params['signatures']: raise MissingArgumentError('signatures') sql_params = [tuple(params['signatures'])] sql = """ SELECT signature, first_report AS first_date, first_build FROM signatures WHERE signature IN %s """ error_message = 'Failed to retrieve signatures from PostgreSQL' results = self.query(sql, sql_params, error_message=error_message) signatures = [] for sig in results.zipped(): sig.first_date = datetimeutil.date_to_string(sig.first_date) signatures.append(sig) return { 'hits': signatures, 'total': len(signatures) }
def create(self, **kwargs): """Add a new job to the priority queue """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingArgumentError('uuid') with self.context() as connection: try: self.config.logger.debug( 'Inserting priority job into RabbitMQ %s', params.uuid ) connection.channel.basic_publish( exchange='', routing_key=self.config.priority_queue_name, body=params.uuid, properties=pika.BasicProperties(delivery_mode=2) ) except ChannelClosed: self.config.logger.error( "Failed inserting priorityjobs data into RabbitMQ", exc_info=True ) return False return True
def get(self, **kwargs): """Return a job in the job queue. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") fields = [ "id", "pathname", "uuid", "owner", "priority", "queueddatetime", "starteddatetime", "completeddatetime", "success", "message" ] sql = """ /* socorro.external.postgresql.job.Job.get */ SELECT %s FROM jobs WHERE uuid=%%(uuid)s """ % ", ".join(fields) error_message = "Failed to retrieve jobs data from PostgreSQL" results = self.query(sql, params, error_message=error_message) jobs = [] for row in results: job = dict(zip(fields, row)) # Make sure all dates are turned into strings for i in job: if isinstance(job[i], datetime.datetime): job[i] = datetimeutil.date_to_string(job[i]) jobs.append(job) return {"hits": jobs, "total": len(jobs)}
def get(self, **kwargs): """Return a single crash report from it's UUID. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if params.uuid is None: raise MissingOrBadArgumentException( "Mandatory parameter 'uuid' is missing or empty") crash_date = datetimeutil.uuid_to_date(params.uuid) logger.debug("Looking for crash %s during day %s" % (params.uuid, crash_date)) sql = """/* socorro.external.postgresql.crash.Crash.get */ SELECT reports.email, reports.url, reports.addons_checked, ( SELECT reports_duplicates.duplicate_of FROM reports_duplicates WHERE reports_duplicates.uuid = reports.uuid ) as duplicate_of FROM reports WHERE reports.uuid=%(uuid)s AND reports.success IS NOT NULL AND utc_day_is( reports.date_processed, %(crash_date)s) """ sql_params = { "uuid": params.uuid, "crash_date": crash_date } results = [] # Creating the connection to the DB self.connection = self.database.connection() cur = self.connection.cursor() try: results = db.execute(cur, sql, sql_params) except psycopg2.Error: util.reportExceptionAndContinue(logger) json_result = { "total": 0, "hits": [] } for crash in results: row = dict(zip(( "email", "url", "addons_checked", "duplicate_of"), crash)) json_result["hits"].append(row) json_result["total"] = len(json_result["hits"]) self.connection.close() return json_result
def get(self, **kwargs): """Return JSON data of a crash report, given its uuid. """ filters = [ ('uuid', None, 'str'), ('datatype', None, 'str') ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingArgumentError('uuid') if not params.datatype: raise MissingArgumentError('datatype') # get a generic crashstorage instance from whatever external resource # is implementing this service. store = self.get_storage() datatype_method_mapping = { 'raw': 'get_raw_dump', 'meta': 'get_raw_crash', 'processed': 'get_processed', 'unredacted': 'get_unredacted_processed', } get = store.__getattribute__(datatype_method_mapping[params.datatype]) try: if params.datatype == 'raw': return (get(params.uuid), 'application/octet-stream') else: return get(params.uuid) except CrashIDNotFound: if params.datatype in ('processed', 'unredacted'): # try to fetch a raw crash just to ensure that the raw crash # exists. If this line fails, there's no reason to actually # submit the priority job. try: store.get_raw_crash(params.uuid) except CrashIDNotFound: raise ResourceNotFound(params.uuid) # search through the existing other services to find the # Priorityjob service. try: priorityjob_service_impl = self.all_services[ 'Priorityjobs' ] except KeyError: raise ServiceUnavailable('Priorityjobs') # get the underlying implementation of the Priorityjob # service and instantiate it. priority_job_service = priorityjob_service_impl.cls( config=self.config ) # create the priority job for this crash_ids priority_job_service.create(uuid=params.uuid) raise ResourceUnavailable(params.uuid) raise ResourceNotFound(params.uuid)
def get(self, **kwargs): """Return JSON data of a crash report, given its uuid. """ filters = [ ('uuid', None, 'str'), ('datatype', None, 'str'), ('name', None, 'str') # only applicable if datatype == 'raw' ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingArgumentError('uuid') if not params.datatype: raise MissingArgumentError('datatype') # get a generic crashstorage instance from whatever external resource # is implementing this service. store = self.get_storage() datatype_method_mapping = { 'raw': 'get_raw_dump', 'meta': 'get_raw_crash', 'processed': 'get_processed', 'unredacted': 'get_unredacted_processed', } get = store.__getattribute__(datatype_method_mapping[params.datatype]) try: if params.datatype == 'raw': return (get(params.uuid, name=params.name), 'application/octet-stream') else: return get(params.uuid) except CrashIDNotFound: if params.datatype in ('processed', 'unredacted'): # try to fetch a raw crash just to ensure that the raw crash # exists. If this line fails, there's no reason to actually # submit the priority job. try: store.get_raw_crash(params.uuid) except CrashIDNotFound: raise ResourceNotFound(params.uuid) # search through the existing other services to find the # Priorityjob service. try: priorityjob_service_impl = self.all_services[ 'Priorityjobs'] except KeyError: raise ServiceUnavailable('Priorityjobs') # get the underlying implementation of the Priorityjob # service and instantiate it. priority_job_service = priorityjob_service_impl.cls( config=self.config) # create the priority job for this crash_ids priority_job_service.create(uuid=params.uuid) raise ResourceUnavailable(params.uuid) raise ResourceNotFound(params.uuid)
def get(self, **kwargs): """Return a job in the job queue. """ filters = [ ("uuid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") fields = [ "id", "pathname", "uuid", "owner", "priority", "queueddatetime", "starteddatetime", "completeddatetime", "success", "message" ] sql = """ /* socorro.external.postgresql.job.Job.get */ SELECT %s FROM jobs WHERE uuid=%%(uuid)s """ % ", ".join(fields) json_result = { "total": 0, "hits": [] } connection = None try: # Creating the connection to the DB connection = self.database.connection() cur = connection.cursor() results = db.execute(cur, sql, params) except psycopg2.Error: logger.error("Failed retrieving jobs data from PostgreSQL", exc_info=True) else: for job in results: row = dict(zip(fields, job)) # Make sure all dates are turned into strings for i in row: if isinstance(row[i], datetime.datetime): row[i] = datetimeutil.date_to_string(row[i]) json_result["hits"].append(row) json_result["total"] = len(json_result["hits"]) finally: if connection: connection.close() return json_result
def get(self, **kwargs): yesterday = datetime.datetime.utcnow() - datetime.timedelta(1) tomorrow = yesterday + datetime.timedelta(2) yesterday = yesterday.date() tomorrow = tomorrow.date() filters = [ ('start_date', yesterday, 'date'), ('end_date', tomorrow, 'date'), ('product', '', 'str'), ('versions', [], 'list'), ('platforms', [], 'list'), ] params = external_common.parse_arguments(filters, kwargs) required = ( 'start_date', 'end_date', 'product', 'versions', 'platforms', ) missing = [] for each in required: if not params.get(each): missing.append(each) if missing: raise MissingArgumentError(', '.join(missing)) SQL = """ SELECT SUM(adu_count) AS adi_count, adu_date AS date, pv.build_type, pv.version_string AS version FROM product_adu LEFT OUTER JOIN product_versions pv USING (product_version_id) WHERE pv.product_name = %(product)s AND pv.version_string IN %(versions)s AND os_name IN %(platforms)s AND adu_date BETWEEN %(start_date)s AND %(end_date)s GROUP BY adu_date, build_type, version_string """ params['versions'] = tuple(params['versions']) params['platforms'] = tuple(params['platforms']) results = self.query(SQL, params) rows = [] for row in results.zipped(): row.date = row.date.strftime('%Y-%m-%d') # BIGINTs become Decimal which becomes floating point in JSON row.adi_count = long(row.adi_count) rows.append(row) return {'hits': rows, 'total': len(rows)}
def get(self, **kwargs): filters = [ ("report_types", None, ["list", "str"]), ("report_type", None, "str"), ("signature", None, "str"), ("start_date", None, "datetime"), ("end_date", None, "datetime"), ("versions", None, ["list", "str"]), ] params = external_common.parse_arguments(filters, kwargs) if not params.get('report_types') and params.get('report_type'): # to support the legacy case individual_report = True report_types = [params['report_type']] else: individual_report = False report_types = params['report_types'] # check that all the report types are recognized for report_type in report_types: query_params = report_type_sql.get(report_type, {}) known_report_types = ('products', 'distinct_install', 'exploitability', 'devices', 'graphics') if (report_type not in known_report_types and 'first_col' not in query_params): raise BadArgumentError(report_type) products = [] versions = [] # Get information about the versions util_service = Util(config=self.context) versions_info = util_service.versions_info(**params) if versions_info: for i, elem in enumerate(versions_info): products.append(versions_info[elem]["product_name"]) versions.append(str(versions_info[elem]["version_string"])) # This MUST be a tuple otherwise it gets cast to an array params['product'] = tuple(products) params['version'] = tuple(versions) all_results = {} with self.get_connection() as connection: for report_type in report_types: result_cols, query_string, query_parameters = self._get_query( report_type, params) sql_results = self.query(query_string, params=query_parameters, connection=connection) results = [dict(zip(result_cols, row)) for row in sql_results] all_results[report_type] = results if individual_report: return all_results.values()[0] else: return {'reports': all_results}
def get_paireduuid(self, **kwargs): """Return paired uuid given a uuid and an optional hangid. If a hangid is passed, then return only one result. Otherwise, return all found paired uuids. """ filters = [ ("uuid", None, "str"), ("hangid", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") crash_date = datetimeutil.uuid_to_date(params.uuid) sql = """ /* socorro.external.postgresql.crashes.Crashes.get_paireduuid */ SELECT uuid FROM reports r WHERE r.uuid != %(uuid)s AND r.date_processed BETWEEN TIMESTAMP %(crash_date)s - CAST('1 day' AS INTERVAL) AND TIMESTAMP %(crash_date)s + CAST('1 day' AS INTERVAL) """ sql_params = {"uuid": params.uuid, "crash_date": crash_date} if params.hangid is not None: sql = """%s AND r.hangid = %%(hangid)s LIMIT 1 """ % sql sql_params["hangid"] = params.hangid else: sql = """%s AND r.hangid IN ( SELECT hangid FROM reports r2 WHERE r2.date_processed BETWEEN TIMESTAMP %%(crash_date)s - CAST('1 day' AS INTERVAL) AND TIMESTAMP %%(crash_date)s + CAST('1 day' AS INTERVAL) AND r2.uuid = %%(uuid)s ) """ % sql # Query the database error_message = "Failed to retrieve paired uuids from PostgreSQL" results = self.query(sql, sql_params, error_message=error_message) # Transforming the results into what we want uuids = [dict(zip(("uuid", ), row)) for row in results] return {"hits": uuids, "total": len(uuids)}
def create_field(self, **kwargs): """Create a new field in the database, to be used by supersearch and all elasticsearch related services. """ filters = [ ('name', None, 'str'), ('data_validation_type', 'enum', 'str'), ('default_value', None, 'str'), ('description', None, 'str'), ('form_field_choices', None, ['list', 'str']), ('has_full_version', False, 'bool'), ('in_database_name', None, 'str'), ('is_exposed', False, 'bool'), ('is_returned', False, 'bool'), ('is_mandatory', False, 'bool'), ('query_type', 'enum', 'str'), ('namespace', None, 'str'), ('permissions_needed', None, ['list', 'str']), ('storage_mapping', None, 'json'), ] params = external_common.parse_arguments(filters, kwargs) mandatory_params = ('name', 'in_database_name') for param in mandatory_params: if not params[param]: raise MissingArgumentError(param) # Before making the change, make sure it does not break indexing. new_mapping = self.get_mapping(overwrite_mapping=params) # Try the mapping. If there is an error, an exception will be raised. # If an exception is raised, the new mapping will be rejected. self.test_mapping(new_mapping) es_connection = self.get_connection().get_es() try: es_connection.index( index=self.config.elasticsearch_default_index, doc_type='supersearch_fields', doc=params, id=params['name'], overwrite_existing=False, refresh=True, ) except ElasticHttpError, e: if e.status_code == 409: # This field exists in the database, it thus cannot be created! raise InsertionError( 'The field "%s" already exists in the database, ' 'impossible to create it. ' % params['name'] ) # Else this is an unexpected error and we want to know about it. raise
def get(self, **kwargs): filters = [ ("start_date", None, "datetime"), ("end_date", None, "datetime"), ("product", None, "str"), ("version", None, "str"), ] params = external_common.parse_arguments(filters, kwargs) results = [] # So we have something to return. query_string = """SELECT product_name, version_string, product_version_id, report_date, nightly_builds.build_date, days_out, sum(report_count) as report_count FROM nightly_builds JOIN product_versions USING ( product_version_id ) WHERE report_date <= %(end_date)s AND report_date >= %(start_date)s AND product_name = %(product)s AND version_string = %(version)s GROUP BY product_name, version_string, product_version_id, report_date, nightly_builds.build_date, days_out""" try: connection = self.database.connection() cursor = connection.cursor() sql_results = db.execute(cursor, query_string, params) except psycopg2.Error: logger.error("Failed retrieving crashtrends data from PostgreSQL", exc_info=True) else: for trend in sql_results: row = dict(zip(( "product_name", "version_string", "product_version_id", "report_date", "build_date", "days_out", "report_count"), trend)) row['report_date'] = datetimeutil.date_to_string(row['report_date']) row['build_date'] = datetimeutil.date_to_string(row['build_date']) results.append(row) finally: connection.close() results = {'crashtrends' : results} return results