示例#1
0
    def get(self, **kwargs):
        filters = [
            ("vendor_hex", None, ["list", "str"]),
            ("adapter_hex", None, ["list", "str"]),
        ]
        params = external_common.parse_arguments(filters, kwargs)
        for key in ('vendor_hex', 'adapter_hex'):
            param = params[key]
            if not param:
                raise MissingArgumentError(key)

            params[key] = tuple(params[key])

        sql_query = """
            SELECT
                vendor_hex, adapter_hex, vendor_name, adapter_name
            FROM graphics_device
            WHERE vendor_hex IN %(vendor_hex)s
            AND adapter_hex IN %(adapter_hex)s
        """

        results = self.query(sql_query, params)
        hits = results.zipped()

        return {
            'hits': hits,
            'total': len(hits)
        }
示例#2
0
    def post(self, **kwargs):
        params = external_common.parse_arguments(self.filters, kwargs)

        if not params['signatures']:
            raise MissingArgumentError('signatures')

        sql_params = [tuple(params['signatures'])]
        sql = """
            SELECT
                signature,
                first_report AS first_date,
                first_build
            FROM signatures
            WHERE signature IN %s
        """

        error_message = 'Failed to retrieve signatures from PostgreSQL'
        results = self.query(sql, sql_params, error_message=error_message)

        signatures = []
        for sig in results.zipped():
            sig['first_date'] = datetimeutil.date_to_string(sig['first_date'])
            signatures.append(sig)

        return {
            'hits': signatures,
            'total': len(signatures)
        }
示例#3
0
    def create(self, **kwargs):
        """Add a new job to the priority queue
        """
        filters = [
            ("uuid", None, "str"),
        ]
        params = external_common.parse_arguments(filters, kwargs)

        if not params.uuid:
            raise MissingArgumentError('uuid')

        with self.context() as connection:
            try:
                self.config.logger.debug(
                    'Inserting priority job into RabbitMQ %s', params.uuid)
                connection.channel.basic_publish(
                    exchange='',
                    routing_key=self.config.priority_queue_name,
                    body=params.uuid,
                    properties=pika.BasicProperties(delivery_mode=2))
            except ChannelClosed:
                self.config.logger.error(
                    "Failed inserting priorityjobs data into RabbitMQ",
                    exc_info=True)
                return False

        return True
示例#4
0
    def get_signatures(self, **kwargs):
        """Return top crashers by signatures.

        See http://socorro.readthedocs.org/en/latest/middleware.html#tcbs
        """
        filters = [
            ("product", None, "str"),
            ("version", None, "str"),
            ("crash_type", "all", "str"),
            ("to_date", datetimeutil.utc_now(), "datetime"),
            ("duration", datetime.timedelta(7), "timedelta"),
            ("os", None, "str"),
            ("limit", 100, "int"),
            ("date_range_type", None, "str")
        ]

        params = external_common.parse_arguments(filters, kwargs)
        params.logger = logger

        # what the twoPeriodTopCrasherComparison() function does is that it
        # makes a start date from taking the to_date - duration
        if params.duration > datetime.timedelta(30):
            raise BadArgumentError('Duration too long. Max 30 days.')

        with self.get_connection() as connection:
            return tcbs.twoPeriodTopCrasherComparison(connection, params)
示例#5
0
    def get_default_version(self, **kwargs):
        """Return the default version of one or several products. """
        filters = [
            ("products", None, ["list", "str"])
        ]
        params = external_common.parse_arguments(filters, kwargs)

        sql = """
            /* socorro.external.postgresql.products.get_default_version */
            SELECT
                product_name AS product,
                version_string AS version
            FROM default_versions
        """

        if params.products and params.products[0] != "":
            params.products = tuple(params.products)
            sql = "%s WHERE product_name IN %%(products)s" % sql

        error_message = "Failed to retrieve default versions from PostgreSQL"
        results = self.query(sql, params, error_message=error_message)

        products = {}
        for product in results.zipped():
            products[product['product']] = product['version']

        return {
            "hits": products
        }
示例#6
0
    def get(self, **kwargs):
        """Return a dict that holds the throttling value per build type
        for a specific product."""
        filters = [
            ('product', None, 'str'),
        ]
        params = external_common.parse_arguments(filters, kwargs)
        required = ('product',)
        for key in required:
            if not params.get(key):
                raise MissingArgumentError(key)

        sql = """
            SELECT
                build_type,
                throttle
            FROM product_build_types
            WHERE product_name = %(product)s
        """
        results = self.query(sql, params)

        build_types = {}
        for row in results.zipped():
            build_types[row['build_type']] = row['throttle']

        return {
            'hits': build_types,
        }
示例#7
0
    def get_default_version(self, **kwargs):
        """Return the default version of one or several products. """
        filters = [("products", None, ["list", "str"])]
        params = external_common.parse_arguments(filters, kwargs)

        sql = """
            /* socorro.external.postgresql.products.get_default_version */
            SELECT
                product_name AS product,
                version_string AS version
            FROM default_versions
        """

        if params.products and params.products[0] != "":
            params.products = tuple(params.products)
            sql = "%s WHERE product_name IN %%(products)s" % sql

        error_message = "Failed to retrieve default versions from PostgreSQL"
        results = self.query(sql, params, error_message=error_message)

        products = {}
        for product in results.zipped():
            products[product['product']] = product['version']

        return {"hits": products}
示例#8
0
 def _get_sql_params(self, **kwargs):
     filters = [
         (
             'date',
             (
                 datetime.datetime.utcnow() - datetime.timedelta(days=1)
             ).date(),
             'date'
         ),
         (
             'limit',
             None,
             int
         ),
     ]
     params = external_common.parse_arguments(filters, kwargs)
     sql = """
     SELECT debug_file, debug_id, code_file, code_id
     FROM missing_symbols
     WHERE
         date_processed = %(date)s AND
         debug_file != '' AND
         debug_id != ''
     GROUP BY debug_file, debug_id, code_file, code_id
     """
     if params['limit'] is not None:
         sql += '\nLIMIT %(limit)s'
     return sql, params
示例#9
0
    def get_channels(self, **kwargs):
        """Return a list of release channels for one, several or all products.
        """
        filters = [
            ("products", None, ["list", "str"]),
        ]
        params = external_common.parse_arguments(filters, kwargs)

        sql = """
            SELECT
                build_type AS channel,
                product_name AS product
            FROM product_info
        """
        sql_params = {}

        if params.products and params.products[0]:
            sql += " WHERE product_name IN %(products)s"
            sql_params['products'] = tuple(params.products)

        error_message = "Failed to retrieve release channels from PostgreSQL"
        sql_results = self.query(sql, sql_params, error_message=error_message)

        channels = {}
        for res in sql_results.zipped():
            if res['product'] not in channels:
                channels[res['product']] = [res['channel']]
            else:
                channels[res['product']].append(res['channel'])

        return channels
示例#10
0
    def get(self, **kwargs):
        filters = [
            ("report_date", None, "datetime"),
            ("report_type", None, "str"),
            ("product", None, "str"),
            ("version", None, "str"),
            ("signature", None, "str"),
            ("platform", None, "str"),
            ("min_crashes", 10, "int"),
            ("min_baseline_diff", 0.05, "float"),
        ]

        params = external_common.parse_arguments(filters, kwargs)

        hits = []
        if params['report_type'] == 'interesting-addons':
            hits = self.interesting_addons(params)
        elif params['report_type'] == 'interesting-modules':
            hits = self.interesting_modules(params)
        elif params['report_type'] == 'interesting-addons-with-version':
            hits = self.interesting_addons_with_version(params)
        elif params['report_type'] == 'interesting-modules-with-version':
            hits = self.interesting_modules_with_version(params)
        elif params['report_type'] == 'core-counts':
            hits = self.core_counts(params)
        else:
            raise BadArgumentError(
                'report_type',
                received=report_type
            )

        return {
            'hits': hits,
            'total': len(hits)
        }
示例#11
0
    def get(self, **kwargs):
        """Return a dict that holds the throttling value per build type
        for a specific product."""
        filters = [
            ('product', None, 'str'),
        ]
        params = external_common.parse_arguments(filters, kwargs)
        required = ('product', )
        for key in required:
            if not params.get(key):
                raise MissingArgumentError(key)

        sql = """
            SELECT
                build_type,
                throttle::REAL
            FROM product_build_types
            WHERE product_name = %(product)s
        """
        results = self.query(sql, params)

        build_types = {}
        for row in results.zipped():
            build_types[row['build_type']] = row['throttle']

        return {
            'hits': build_types,
        }
示例#12
0
    def get(self, **kwargs):
        """Return JSON data of a crash report, given its uuid. """
        filters = [
            ('uuid', None, str),
            ('datatype', None, str),
            ('name', None, str)  # only applicable if datatype == 'raw'
        ]
        params = external_common.parse_arguments(filters, kwargs, modern=True)

        if not params.uuid:
            raise MissingArgumentError('uuid')

        if not params.datatype:
            raise MissingArgumentError('datatype')

        datatype_method_mapping = {
            'raw': 'get_raw_dump',
            'meta': 'get_raw_crash',
            'processed': 'get_processed',
            'unredacted': 'get_unredacted_processed',
        }
        get = self.__getattribute__(datatype_method_mapping[params.datatype])
        try:
            if params.datatype == 'raw':
                return get(params.uuid, name=params.name)
            else:
                return get(params.uuid)
        except CrashIDNotFound:
            # The CrashIDNotFound exception that happens inside the
            # crashstorage is too revealing as exception message
            # contains information about buckets and prefix keys.
            # Re-wrap it here so the message is just the crash ID.
            raise CrashIDNotFound(params.uuid)
    def test_parse_arguments_with_class_validators(self):
        class NumberConverter(object):
            def clean(self, value):
                conv = {'one': 1, 'two': 2, 'three': 3}
                try:
                    return conv[value]
                except KeyError:
                    raise ValueError('No idea?!')

        # Define a set of filters with some types being non-trivial types
        # but instead a custom validator.

        filters = [
            ("param1", 0, NumberConverter()),
        ]
        arguments = {
            "param1": "one",
        }
        params_exp = util.DotDict()
        params_exp.param1 = 1

        params = external_common.parse_arguments(filters,
                                                 arguments,
                                                 modern=True)
        eq_(params, params_exp)

        # note that a ValueError becomes a BadArgumentError
        arguments = {
            "param1": "will cause a ValueError in NumberConverter.clean",
        }
        assert_raises(BadArgumentError,
                      external_common.parse_arguments,
                      filters,
                      arguments,
                      modern=True)
示例#14
0
    def post(self, **kwargs):
        params = external_common.parse_arguments(self.filters, kwargs)
        if not params.category:
            raise MissingArgumentError('category')
        if not params.rule:
            raise MissingArgumentError('rule')

        sql = """
            /* socorro.external.postgresql.skiplist.SkipList.post */
            INSERT INTO skiplist (category, rule)
            VALUES (%s, %s);
        """

        sql_params = [params.category, params.rule]
        connection = self.database.connection()
        try:
            with connection.cursor() as cur:
                cur.execute(sql, sql_params)
            connection.commit()
        except psycopg2.Error:
            connection.rollback()
            error_message = "Failed updating skip list in PostgreSQL"
            logger.error(error_message)
            raise DatabaseError(error_message)
        finally:
            connection.close()

        return True
示例#15
0
    def get(self, **kwargs):
        filters = [
            ("report_date", None, "datetime"),
            ("report_type", None, "str"),
            ("product", None, "str"),
            ("version", None, "str"),
            ("signature", None, "str"),
            ("platform", None, "str"),
            ("min_crashes", 10, "int"),
            ("min_baseline_diff", 0.05, "float"),
        ]

        params = external_common.parse_arguments(filters, kwargs)

        hits = []
        if params['report_type'] == 'interesting-addons':
            hits = self.interesting_addons(params)
        elif params['report_type'] == 'interesting-modules':
            hits = self.interesting_modules(params)
        elif params['report_type'] == 'interesting-addons-with-version':
            hits = self.interesting_addons_with_version(params)
        elif params['report_type'] == 'interesting-modules-with-version':
            hits = self.interesting_modules_with_version(params)
        elif params['report_type'] == 'core-counts':
            hits = self.core_counts(params)
        else:
            raise BadArgumentError(
                'report_type',
                received=report_type
            )

        return {
            'hits': hits,
            'total': len(hits)
        }
示例#16
0
    def get(self, **kwargs):
        params = external_common.parse_arguments(self.filters, kwargs)
        sql_params = []
        sql = """
            /* socorro.external.postgresql.skiplist.SkipList.get */
            SELECT category,
                   rule
            FROM skiplist
            WHERE 1=1
        """
        if params.category:
            sql += 'AND category=%s'
            sql_params.append(params.category)
        if params.rule:
            sql += 'AND rule=%s'
            sql_params.append(params.rule)
        # Use `UPPER()` to make the sort case insensitive
        # which makes it more user-friendly on the UI later
        sql += """
            ORDER BY UPPER(category), UPPER(rule)
        """

        error_message = "Failed to retrieve skip list data from PostgreSQL"
        sql_results = self.query(sql, sql_params, error_message=error_message)

        results = sql_results.zipped()

        return {'hits': results, 'total': len(results)}
示例#17
0
    def get(self, **kwargs):
        '''Return data about a field from its name. '''
        filters = [
            ('name', None, 'str'),
        ]
        params = external_common.parse_arguments(filters, kwargs)

        if not params.name:
            raise MissingArgumentError("name")

        sql = '''/* socorro.external.postgresql.field.Field.get */
            SELECT
                raw_field AS name,
                transforms,
                product
            FROM data_dictionary
            WHERE raw_field=%(name)s
        '''

        error_message = 'Failed to retrieve field data from PostgreSQL'
        results = self.query(sql, params, error_message=error_message)

        field_data = {
            'name': None,
            'transforms': None,
            'product': None
        }

        if not results:
            return field_data

        field_data = results.zipped()[0]

        return field_data
示例#18
0
    def get(self, **kwargs):
        filters = [
            ("vendor_hex", None, ["list", "str"]),
            ("adapter_hex", None, ["list", "str"]),
        ]
        params = external_common.parse_arguments(filters, kwargs)
        for key in ('vendor_hex', 'adapter_hex'):
            param = params[key]
            if not param:
                raise MissingArgumentError(key)

            params[key] = tuple(params[key])

        sql_query = """
            SELECT
                vendor_hex, adapter_hex, vendor_name, adapter_name
            FROM graphics_device
            WHERE vendor_hex IN %(vendor_hex)s
            AND adapter_hex IN %(adapter_hex)s
        """

        results = self.query(sql_query, params)
        hits = results.zipped()

        return {'hits': hits, 'total': len(hits)}
示例#19
0
    def create(self, **kwargs):
        """Add a new job to the priority queue
        """
        filters = [
            ("uuid", None, "str"),
        ]
        params = external_common.parse_arguments(filters, kwargs)

        if not params.uuid:
            raise MissingArgumentError('uuid')

        with self.context() as connection:
            try:
                self.config.logger.debug(
                    'Inserting priority job into RabbitMQ %s', params.uuid
                )
                connection.channel.basic_publish(
                    exchange='',
                    routing_key=self.config.priority_queue_name,
                    body=params.uuid,
                    properties=pika.BasicProperties(delivery_mode=2)
                )
            except ChannelClosed:
                self.config.logger.error(
                    "Failed inserting priorityjobs data into RabbitMQ",
                    exc_info=True
                )
                return False

        return True
示例#20
0
    def get_adu_by_signature(self, **kwargs):
        """Return a list of ADUs and crash counts by signature and ADU date
        """
        now = datetimeutil.utc_now().date()
        lastweek = now - datetime.timedelta(weeks=1)

        filters = [
            ("start_date", lastweek, "date"),
            ("end_date", now, "date"),
            ("signature", None, "str"),
            ("channel", None, "str"),
            ("product_name", None, "str"),
        ]

        params = external_common.parse_arguments(filters, kwargs)

        for param in ("start_date", "end_date", "signature", "channel"):
            if not params[param]:
                raise MissingArgumentError(param)

        if (params.end_date - params.start_date) > datetime.timedelta(days=365):
            raise BadArgumentError('Duration too long. Max 365 days.')

        sql_query = """
            SELECT
                product_name,
                signature,
                adu_date::TEXT,
                build_date::TEXT,
                buildid::TEXT,
                crash_count,
                adu_count,
                os_name,
                channel
            FROM crash_adu_by_build_signature
            WHERE adu_date BETWEEN %(start_date)s AND %(end_date)s
            AND product_name = %(product_name)s
            AND channel = %(channel)s
            AND signature = %(signature)s
            ORDER BY buildid
        """

        error_message = (
            "Failed to retrieve crash ADU by build signature from PostgreSQL"
        )
        results = self.query(sql_query, params, error_message=error_message)

        crashes = results.zipped()

        return {
            "hits": crashes,
            "total": len(crashes)
        }
示例#21
0
 def get(self, **kwargs):
     filters = [
         ('date', datetime.datetime.utcnow().date(), 'date'),
         ('product', 'Firefox', 'str'),
     ]
     params = external_common.parse_arguments(filters, kwargs)
     params['tomorrow'] = params['date'] + datetime.timedelta(days=1)
     results = self.query(SQL, params)
     hits = results.zipped()
     return {
         'hits': hits,
         'total': len(hits),
     }
示例#22
0
    def get(self, **kwargs):
        filters = [
            ('report_type', None, 'str'),
            ('product', None, 'str'),
            ('version', None, 'str'),
            ('platforms', None, 'list'),
        ]

        params = external_common.parse_arguments(filters, kwargs)
        try:
            content = self._get_content(params)
        except NotFoundError, msg:
            self.config.logger.info('Failed to download %s' % msg)
            return
示例#23
0
    def get_adu_by_signature(self, **kwargs):
        """Return a list of ADUs and crash counts by signature and ADU date
        """
        now = datetimeutil.utc_now().date()
        lastweek = now - datetime.timedelta(weeks=1)

        filters = [
            ("start_date", lastweek, "date"),
            ("end_date", now, "date"),
            ("signature", None, "str"),
            ("channel", None, "str"),
            ("product_name", None, "str"),
        ]

        params = external_common.parse_arguments(filters, kwargs)

        for param in ("start_date", "end_date", "signature", "channel"):
            if not params[param]:
                raise MissingArgumentError(param)

        if (params.end_date -
                params.start_date) > datetime.timedelta(days=365):
            raise BadArgumentError('Duration too long. Max 365 days.')

        sql_query = """
            SELECT
                product_name,
                signature,
                adu_date::TEXT,
                build_date::TEXT,
                buildid::TEXT,
                crash_count,
                adu_count,
                os_name,
                channel
            FROM crash_adu_by_build_signature
            WHERE adu_date BETWEEN %(start_date)s AND %(end_date)s
            AND product_name = %(product_name)s
            AND channel = %(channel)s
            AND signature = %(signature)s
            ORDER BY buildid
        """

        error_message = (
            "Failed to retrieve crash ADU by build signature from PostgreSQL")
        results = self.query(sql_query, params, error_message=error_message)

        crashes = results.zipped()

        return {"hits": crashes, "total": len(crashes)}
示例#24
0
文件: query.py 项目: ahlfors/socorro
    def post(self, **kwargs):
        '''Return the result of a custom query. '''
        params = external_common.parse_arguments(self.filters, kwargs)

        if not params.query:
            raise MissingArgumentError('query')

        try:
            query = json.loads(params.query)
        except ValueError:
            raise BadArgumentError(
                'query',
                msg="Invalid JSON value for parameter 'query'"
            )

        # Set indices.
        indices = []
        if not params.indices:
            # By default, use the last two indices.
            today = datetimeutil.utc_now()
            last_week = today - datetime.timedelta(days=7)

            indices = self.generate_list_of_indexes(last_week, today)
        elif len(params.indices) == 1 and params.indices[0] == 'ALL':
            # If we want all indices, just do nothing.
            pass
        else:
            indices = params.indices

        search_args = {}
        if indices:
            search_args['index'] = indices
            search_args['doc_type'] = (
                self.config.elasticsearch.elasticsearch_doctype
            )

        connection = self.get_connection()

        try:
            results = connection.search(
                body=query,
                **search_args
            )
        except elasticsearch.exceptions.NotFoundError, e:
            missing_index = re.findall(BAD_INDEX_REGEX, e.error)[0]
            raise ResourceNotFound(
                "elasticsearch index '%s' does not exist" % missing_index
            )
示例#25
0
    def test_parse_arguments(self):
        """Test external_common.parse_arguments(). """
        filters = [
            ("param1", "default", [str]),
            ("param2", None, int),
            ("param3", ["some", "default", "list"], [str]),
            ("param4", ["list", "of", 4, "values"], [str]),
            ("param5", None, bool),
            ("param6", None, datetime.date),
            ("param7", None, datetime.date),
            ("param8", None, datetime.datetime),
            ("param9", None, [str]),
        ]
        arguments = {
            "param1": "value1",
            "unknown": 12345,
            "param5": "true",
            "param7": datetime.date(2016, 2, 9).isoformat(),
            "param8": datetime.datetime(2016, 2, 9).isoformat(),
            # note the 'param9' is deliberately not specified.
        }
        params_exp = util.DotDict()
        params_exp.param1 = ["value1"]
        params_exp.param2 = None
        params_exp.param3 = ['some', 'default', 'list']
        params_exp.param4 = ["list", "of", "4", "values"]
        params_exp.param5 = True
        params_exp.param6 = None
        params_exp.param7 = datetime.date(2016, 2, 9)
        params_exp.param8 = datetime.datetime(2016, 2, 9).replace(
            tzinfo=isodate.UTC
        )
        params_exp.param9 = None

        params = external_common.parse_arguments(
            filters,
            arguments,
            modern=True
        )
        for key in params:
            eq_(params[key], params_exp[key], '{}: {!r} != {!r}'.format(
                key,
                params[key],
                params_exp[key]
            ))
        eq_(params, params_exp)
示例#26
0
    def get_count_by_day(self, **kwargs):
        """Returns the number of crashes on a daily basis"""
        filters = [
            ("signature", None, "str"),
            ("start_date", None, "date"),
            ("end_date", None, "date")
        ]

        DATE_FORMAT = "%Y-%m-%d"

        params = external_common.parse_arguments(filters, kwargs)

        for param in ("signature", "start_date"):
            if not params[param]:
                raise MissingArgumentError(param)

        if not params.end_date:
            params.end_date = params.start_date + datetime.timedelta(1)

        sql = """
            SELECT
                COUNT(*),
                date_processed::date
            FROM
                reports_clean rc
            JOIN signatures ON
                rc.signature_id=signatures.signature_id
            WHERE
                rc.date_processed >= %(start_date)s AND
                rc.date_processed::date < %(end_date)s AND
                signatures.signature=%(signature)s
            GROUP BY
                rc.date_processed::date
        """

        hits = {}

        for count, date in self.query(sql, params):
            hits[date.strftime(DATE_FORMAT)] = count

        current = params.start_date
        while current < params.end_date:
            hits.setdefault(current.strftime(DATE_FORMAT), 0)
            current += datetime.timedelta(1)

        return {"hits": hits, "total": len(hits)}
    def test_parse_arguments_old_way(self):
        """Test external_common.parse_arguments(). """
        filters = [("param1", "default", ["list", "str"]),
                   ("param2", None, "int"),
                   ("param3", ["list", "of", 4, "values"], ["list", "str"])]
        arguments = {"param1": "value1", "unknown": 12345}
        params_exp = util.DotDict()
        params_exp.param1 = ["value1"]
        params_exp.param2 = None
        params_exp.param3 = ["list", "of", "4", "values"]

        params = external_common.parse_arguments(
            filters,
            arguments,
            modern=False,
        )

        eq_(params, params_exp)
示例#28
0
    def test_parse_arguments(self):
        """Test external_common.parse_arguments(). """
        filters = [
            ("param1", "default", ["list", "str"]),
            ("param2", None, "int"),
            ("param3", ["list", "of", 4, "values"], ["list", "str"])
        ]
        arguments = {
            "param1": "value1",
            "unknown": 12345
        }
        params_exp = util.DotDict()
        params_exp.param1 = ["value1"]
        params_exp.param2 = None
        params_exp.param3 = ["list", "of", "4", "values"]

        params = external_common.parse_arguments(filters, arguments)

        eq_(params, params_exp)
示例#29
0
    def get_count_by_day(self, **kwargs):
        """Returns the number of crashes on a daily basis"""
        filters = [("signature", None, "str"), ("start_date", None, "date"),
                   ("end_date", None, "date")]

        DATE_FORMAT = "%Y-%m-%d"

        params = external_common.parse_arguments(filters, kwargs)

        for param in ("signature", "start_date"):
            if not params[param]:
                raise MissingArgumentError(param)

        if not params.end_date:
            params.end_date = params.start_date + datetime.timedelta(1)

        sql = """
            SELECT
                COUNT(*),
                date_processed::date
            FROM
                reports_clean rc
            JOIN signatures ON
                rc.signature_id=signatures.signature_id
            WHERE
                rc.date_processed >= %(start_date)s AND
                rc.date_processed::date < %(end_date)s AND
                signatures.signature=%(signature)s
            GROUP BY
                rc.date_processed::date
        """

        hits = {}

        for count, date in self.query(sql, params):
            hits[date.strftime(DATE_FORMAT)] = count

        current = params.start_date
        while current < params.end_date:
            hits.setdefault(current.strftime(DATE_FORMAT), 0)
            current += datetime.timedelta(1)

        return {"hits": hits, "total": len(hits)}
示例#30
0
文件: bugs.py 项目: ahlfors/socorro
    def get(self, **kwargs):
        """Return a list of signatures-to-bug_ids or bug_ids-to-signatures
           associations. """
        params = external_common.parse_arguments(self.filters, kwargs)

        if not params['signatures'] and not params['bug_ids']:
            raise MissingArgumentError('specify one of signatures or bug_ids')
        elif params['signatures'] and params['bug_ids']:
            raise BadArgumentError('specify only one of signatures or bug_ids')

        sql_params = []
        if params['signatures']:
            sql_params.append(tuple(params.signatures))

            sql = """/* socorro.external.postgresql.bugs.Bugs.get */
                SELECT ba.signature, bugs.id
                FROM bugs
                    JOIN bug_associations AS ba ON bugs.id = ba.bug_id
                WHERE EXISTS(
                    SELECT 1 FROM bug_associations
                    WHERE bug_associations.bug_id = bugs.id
                    AND signature IN %s
                )
            """
        elif params['bug_ids']:
            sql_params.append(tuple(params.bug_ids))

            sql = """/* socorro.external.postgresql.bugs.Bugs.get */
                SELECT ba.signature, bugs.id
                FROM bugs
                    JOIN bug_associations AS ba ON bugs.id = ba.bug_id
                WHERE bugs.id IN %s
            """

        error_message = "Failed to retrieve bug associations from PostgreSQL"
        results = self.query(sql, sql_params, error_message=error_message)

        bugs = results.zipped()

        return {
            "hits": bugs,
            "total": len(bugs)
        }
示例#31
0
    def get(self, **kwargs):
        """Return a list of signatures-to-bug_ids or bug_ids-to-signatures
           associations. """
        params = external_common.parse_arguments(self.filters,
                                                 kwargs,
                                                 modern=True)

        if not params['signatures'] and not params['bug_ids']:
            raise MissingArgumentError('specify one of signatures or bug_ids')
        elif params['signatures'] and params['bug_ids']:
            raise BadArgumentError('specify only one of signatures or bug_ids')

        sql_params = []
        if params['signatures']:
            sql_params.append(tuple(params.signatures))

            sql = """/* socorro.external.postgresql.bugs.Bugs.get */
                SELECT ba.signature, bugs.id
                FROM bugs
                    JOIN bug_associations AS ba ON bugs.id = ba.bug_id
                WHERE EXISTS(
                    SELECT 1 FROM bug_associations
                    WHERE bug_associations.bug_id = bugs.id
                    AND signature IN %s
                )
            """
        elif params['bug_ids']:
            sql_params.append(tuple(params.bug_ids))

            sql = """/* socorro.external.postgresql.bugs.Bugs.get */
                SELECT ba.signature, bugs.id
                FROM bugs
                    JOIN bug_associations AS ba ON bugs.id = ba.bug_id
                WHERE bugs.id IN %s
            """

        error_message = "Failed to retrieve bug associations from PostgreSQL"
        results = self.query(sql, sql_params, error_message=error_message)

        bugs = results.zipped()

        return {"hits": bugs, "total": len(bugs)}
示例#32
0
 def post(self, **kwargs):
     """adding a new product"""
     filters = [
         ("product", None, "str"),
         ("version", None, "str"),
     ]
     params = external_common.parse_arguments(filters, kwargs)
     with self.get_connection() as connection:
         try:
             result, = single_row_sql(
                 connection,
                 "SELECT add_new_product(%s, %s)",
                 (params['product'], params['version']),
             )
         except psycopg2.Error:
             connection.rollback()
             return False
         else:
             connection.commit()
         return result
示例#33
0
 def _get_sql_params(self, **kwargs):
     filters = [
         ('date',
          (datetime.datetime.utcnow() - datetime.timedelta(days=1)).date(),
          'date'),
         ('limit', None, int),
     ]
     params = external_common.parse_arguments(filters, kwargs)
     sql = """
     SELECT debug_file, debug_id, code_file, code_id
     FROM missing_symbols
     WHERE
         date_processed = %(date)s AND
         debug_file != '' AND
         debug_id != ''
     GROUP BY debug_file, debug_id, code_file, code_id
     """
     if params['limit'] is not None:
         sql += '\nLIMIT %(limit)s'
     return sql, params
示例#34
0
 def post(self, **kwargs):
     """adding a new product"""
     filters = [
         ("product", None, "str"),
         ("version", None, "str"),
     ]
     params = external_common.parse_arguments(filters, kwargs)
     with self.get_connection() as connection:
         try:
             result, = single_row_sql(
                 connection,
                 "SELECT add_new_product(%s, %s)",
                 (params['product'], params['version']),
             )
         except psycopg2.Error:
             connection.rollback()
             return False
         else:
             connection.commit()
         return result
示例#35
0
    def test_parse_arguments_with_class_validators(self):

        class NumberConverter(object):

            def clean(self, value):
                conv = {'one': 1, 'two': 2, 'three': 3}
                try:
                    return conv[value]
                except KeyError:
                    raise ValueError('No idea?!')

        # Define a set of filters with some types being non-trivial types
        # but instead a custom validator.

        filters = [
            ("param1", 0, NumberConverter()),
        ]
        arguments = {
            "param1": "one",
        }
        params_exp = util.DotDict()
        params_exp.param1 = 1

        params = external_common.parse_arguments(
            filters,
            arguments,
            modern=True
        )
        eq_(params, params_exp)

        # note that a ValueError becomes a BadArgumentError
        arguments = {
            "param1": "will cause a ValueError in NumberConverter.clean",
        }
        assert_raises(
            BadArgumentError,
            external_common.parse_arguments,
            filters,
            arguments,
            modern=True
        )
示例#36
0
    def post(self, **kwargs):
        '''Return the result of a custom query. '''
        params = external_common.parse_arguments(self.filters, kwargs)

        if not params.query:
            raise MissingArgumentError('query')

        try:
            query = json.loads(params.query)
        except ValueError:
            raise BadArgumentError(
                'query', msg="Invalid JSON value for parameter 'query'")

        # Set indices.
        indices = []
        if not params.indices:
            # By default, use the last two indices.
            today = datetimeutil.utc_now()
            last_week = today - datetime.timedelta(days=7)

            indices = self.generate_list_of_indexes(last_week, today)
        elif len(params.indices) == 1 and params.indices[0] == 'ALL':
            # If we want all indices, just do nothing.
            pass
        else:
            indices = params.indices

        search_args = {}
        if indices:
            search_args['index'] = indices
            search_args['doc_type'] = (
                self.config.elasticsearch.elasticsearch_doctype)

        connection = self.get_connection()

        try:
            results = connection.search(body=query, **search_args)
        except elasticsearch.exceptions.NotFoundError, e:
            missing_index = re.findall(BAD_INDEX_REGEX, e.error)[0]
            raise ResourceNotFound("elasticsearch index '%s' does not exist" %
                                   missing_index)
    def get(self, **kwargs):
        params = external_common.parse_arguments(self.filters, kwargs)

        if not params['signatures']:
            raise MissingArgumentError('signatures')

        sql_params = [tuple(params['signatures'])]
        sql = """
            SELECT
                signature,
                first_report AS first_date,
                first_build::VARCHAR
            FROM signatures
            WHERE signature IN %s
        """

        error_message = 'Failed to retrieve signatures from PostgreSQL'
        results = self.query(sql, sql_params, error_message=error_message)

        signatures = results.zipped()
        return {'hits': signatures, 'total': len(signatures)}
    def test_parse_arguments(self):
        """Test external_common.parse_arguments(). """
        filters = [
            ("param1", "default", [str]),
            ("param2", None, int),
            ("param3", ["some", "default", "list"], [str]),
            ("param4", ["list", "of", 4, "values"], [str]),
            ("param5", None, bool),
            ("param6", None, datetime.date),
            ("param7", None, datetime.date),
            ("param8", None, datetime.datetime),
            ("param9", None, [str]),
        ]
        arguments = {
            "param1": "value1",
            "unknown": 12345,
            "param5": "true",
            "param7": datetime.date(2016, 2, 9).isoformat(),
            "param8": datetime.datetime(2016, 2, 9).isoformat(),
            # note the 'param9' is deliberately not specified.
        }
        params_exp = util.DotDict()
        params_exp.param1 = ["value1"]
        params_exp.param2 = None
        params_exp.param3 = ['some', 'default', 'list']
        params_exp.param4 = ["list", "of", "4", "values"]
        params_exp.param5 = True
        params_exp.param6 = None
        params_exp.param7 = datetime.date(2016, 2, 9)
        params_exp.param8 = datetime.datetime(2016, 2,
                                              9).replace(tzinfo=isodate.UTC)
        params_exp.param9 = None

        params = external_common.parse_arguments(filters,
                                                 arguments,
                                                 modern=True)
        for key in params:
            eq_(params[key], params_exp[key],
                '{}: {!r} != {!r}'.format(key, params[key], params_exp[key]))
        eq_(params, params_exp)
示例#39
0
    def get(self, **kwargs):
        """ return GC crashes per build ID """

        for arg in ['product', 'version']:
            if not kwargs.get(arg):
                raise MissingArgumentError(arg)

        now = datetimeutil.utc_now().date()
        lastweek = now - datetime.timedelta(weeks=1)

        filters = [
            ("product", None, "str"),
            ("version", None, "str"),
            ("from_date", lastweek, "date"),
            ("to_date", now, "date"),
        ]

        params = external_common.parse_arguments(filters, kwargs)

        result = self.query(
            """
            /* socorro.external.postgresql.gccrashes.GCCrashes.get */
            SELECT
                build::text,
                sum(gc_count_madu)
            FROM gccrashes
            JOIN product_versions
            USING (product_version_id)
            WHERE product_name = %(product)s
            AND version_string = %(version)s
            AND report_date BETWEEN %(from_date)s AND %(to_date)s
            AND build IS NOT NULL
            GROUP BY build
            ORDER BY build
        """, params)

        # Because we don't return a list of dicts, we turn it into a
        # pure list first so it becomes a list of tuples.
        rows = list(result)
        return {'hits': rows, 'total': len(rows)}
示例#40
0
    def get(self, **kwargs):
        """ return GC crashes per build ID """

        for arg in ['product', 'version']:
            if not kwargs.get(arg):
                raise MissingArgumentError(arg)

        now = datetimeutil.utc_now().date()
        lastweek = now - datetime.timedelta(weeks=1)

        filters = [
            ("product", None, "str"),
            ("version", None, "str"),
            ("from_date", lastweek, "date"),
            ("to_date", now, "date"),
        ]

        params = external_common.parse_arguments(filters, kwargs)

        result = self.query("""
            /* socorro.external.postgresql.gccrashes.GCCrashes.get */
            SELECT
                build::text,
                sum(gc_count_madu)
            FROM gccrashes
            JOIN product_versions
            USING (product_version_id)
            WHERE product_name = %(product)s
            AND version_string = %(version)s
            AND report_date BETWEEN %(from_date)s AND %(to_date)s
            AND build IS NOT NULL
            GROUP BY build
            ORDER BY build
        """, params)

        # Because we don't return a list of dicts, we turn it into a
        # pure list first so it becomes a list of tuples.
        rows = list(result)
        return {'hits': rows, 'total': len(rows)}
示例#41
0
    def get_signatures(self, **kwargs):
        """Return top crashers by signatures.

        See https://socorro.readthedocs.io/en/latest/middleware.html#tcbs
        """
        filters = [("product", None, "str"), ("version", None, "str"),
                   ("crash_type", "all", "str"),
                   ("to_date", datetimeutil.utc_now(), "datetime"),
                   ("duration", datetime.timedelta(7), "timedelta"),
                   ("os", None, "str"), ("limit", 100, "int"),
                   ("date_range_type", None, "str")]

        params = external_common.parse_arguments(filters, kwargs)
        params.logger = logger

        # what the twoPeriodTopCrasherComparison() function does is that it
        # makes a start date from taking the to_date - duration
        if params.duration > datetime.timedelta(30):
            raise BadArgumentError('Duration too long. Max 30 days.')

        with self.get_connection() as connection:
            return tcbs.twoPeriodTopCrasherComparison(connection, params)
示例#42
0
    def get(self, **kwargs):

        filters = [
            ("backfill_type", None, "str"),
            ("reports_clean", True, "bool"),
            ("check_period", '01:00:00', "str"),
            ("table_name", None, "str"),
            ("update_day", None, "datetime"),
            ("start_date", None, "datetime"),
            ("end_date", None, "datetime"),
        ]

        params = external_common.parse_arguments(filters, kwargs)

        if not params.backfill_type:
            raise MissingArgumentError('backfill_type')

        date_param = ['update_day', 'start_date', 'end_date']
        for i in date_param:
            if i in kwargs:
                params[i] = str(params[i].date())

        try:
            query = 'SELECT backfill_%(backfill_type)s (%(params)s); '
            required_params = BACKFILL_PARAMETERS[params.backfill_type]
            query_params = [(i, params[i]) for i in required_params]
            query_params_str = ', '.join(
                '%(' + str(i[0]) + ')s' for i in query_params
            )
            query = query % {'backfill_type': params.backfill_type,
                             'params': query_params_str}
        except:
            raise BadArgumentError(kwargs['backfill_type'])

        error_message = "Failed to retrieve backfill %s from PostgreSQL"
        error_message = error_message % kwargs['backfill_type']
        results = self.query(query, params, error_message=error_message)
        return results
示例#43
0
    def get_featured(self, **kwargs):
        """Return a list of featured versions for one, several or all products.
        """
        filters = [
            ("products", None, ["list", "str"]),
        ]
        params = external_common.parse_arguments(filters, kwargs)

        sql = """
            SELECT
                product_name AS product,
                version_string AS version
            FROM product_info
            WHERE is_featured = true
        """
        sql_params = {}

        if params.products and params.products[0]:
            sql += " AND product_name IN %(product)s"
            sql_params['product'] = tuple(params.products)

        error_message = "Failed to retrieve featured versions from PostgreSQL"
        sql_results = self.query(sql, sql_params, error_message=error_message)

        hits = {}
        total = 0

        for version in sql_results.zipped():
            total += 1
            if version['product'] not in hits:
                hits[version['product']] = [version['version']]
            else:
                hits[version['product']].append(version['version'])

        return {
            "total": total,
            "hits": hits
        }
示例#44
0
    def delete(self, **kwargs):
        params = external_common.parse_arguments(self.filters, kwargs)
        if not params.category:
            raise MissingArgumentError('category')
        if not params.rule:
            raise MissingArgumentError('rule')

        sql_params = [params.category, params.rule]
        count_sql = """
            /* socorro.external.postgresql.skiplist.SkipList.delete */
            SELECT COUNT(*) FROM skiplist
            WHERE category=%s AND rule=%s
        """
        sql = """
            /* socorro.external.postgresql.skiplist.SkipList.delete */
            DELETE FROM skiplist
            WHERE category=%s AND rule=%s
        """

        connection = self.database.connection()
        try:
            cur = connection.cursor()
            count = self.count(count_sql, sql_params, connection=connection)
            if not count:
                return False
            cur.execute(sql, sql_params)
            connection.commit()
        except psycopg2.Error:
            connection.rollback()
            error_message = "Failed delete skip list in PostgreSQL"
            logger.error(error_message)
            raise DatabaseError(error_message)
        finally:
            connection.close()

        return True
示例#45
0
    def get(self, **kwargs):
        params = external_common.parse_arguments(self.filters, kwargs)

        if not params['signatures']:
            raise MissingArgumentError('signatures')

        sql_params = [tuple(params['signatures'])]
        sql = """
            SELECT
                signature,
                first_report AS first_date,
                first_build::VARCHAR
            FROM signatures
            WHERE signature IN %s
        """

        error_message = 'Failed to retrieve signatures from PostgreSQL'
        results = self.query(sql, sql_params, error_message=error_message)

        signatures = results.zipped()
        return {
            'hits': signatures,
            'total': len(signatures)
        }
示例#46
0
    def delete_field(self, **kwargs):
        """Remove a field from the database.

        Removing a field means that it won't be indexed in elasticsearch
        anymore, nor will it be exposed or accessible via supersearch. It
        doesn't delete the data from crash reports though, so it would be
        possible to re-create the field and reindex some indices to get that
        data back.
        """
        filters = [
            ('name', None, 'str'),
        ]
        params = external_common.parse_arguments(filters, kwargs)

        if not params['name']:
            raise MissingArgumentError('name')

        es_connection = self.get_connection()
        es_connection.delete(
            index=self.config.elasticsearch.elasticsearch_default_index,
            doc_type='supersearch_fields',
            id=params['name'],
            refresh=True,
        )
示例#47
0
    def get_signature_history(self, **kwargs):
        """Return the history of a signature.

        See https://socorro.readthedocs.io/en/latest/middleware.html
        """
        now = datetimeutil.utc_now()
        lastweek = now - datetime.timedelta(days=7)

        filters = [
            ('product', None, 'str'),
            ('version', None, 'str'),
            ('signature', None, 'str'),
            ('end_date', now, 'datetime'),
            ('start_date', lastweek, 'datetime'),
        ]
        params = external_common.parse_arguments(filters, kwargs)

        for param in ('product', 'version', 'signature'):
            if not params[param]:
                raise MissingArgumentError(param)

        if params.signature == '##null##':
            signature_where = 'AND signature IS NULL'
        else:
            signature_where = 'AND signature = %(signature)s'

        if params.signature == '##empty##':
            params.signature = ''

        sql = """
            /* external.postgresql.crashes.Crashes.get_signature_history */
            WITH hist AS (
                SELECT
                    report_date,
                    report_count
                FROM
                    tcbs JOIN signatures using (signature_id)
                         JOIN product_versions using (product_version_id)
                WHERE
                    report_date BETWEEN %%(start_date)s AND %%(end_date)s
                    AND product_name = %%(product)s
                    AND version_string = %%(version)s
                    %s
                GROUP BY
                    report_date, report_count
                ORDER BY 1
            ),
            scaling_window AS (
                SELECT
                    hist.*,
                    SUM(report_count) over () AS total_crashes
                FROM hist
            )
            SELECT
                report_date AS date,
                report_count AS count,
                report_count / total_crashes::float * 100 AS percent_of_total
            FROM scaling_window
            ORDER BY report_date DESC
        """ % signature_where

        error_message = 'Failed to retrieve signature history from PostgreSQL'
        results = self.query(sql, params, error_message=error_message)

        # Transforming the results into what we want
        history = []
        for dot in results.zipped():
            dot['date'] = datetimeutil.date_to_string(dot['date'])
            history.append(dot)

        return {'hits': history, 'total': len(history)}
示例#48
0
    def get(self, **kwargs):
        filters = [
            ("version", None, [str]),
            ("product", None, [str]),
            ("is_featured", None, bool),
            ("start_date", None, SmartDate()),
            ("end_date", None, SmartDate()),
            ("active", None, bool),
            ("is_rapid_beta", None, bool),
            ("build_type", None, [str]),
        ]
        params = external_common.parse_arguments(filters, kwargs, modern=True)
        where = []
        sql_params = {}
        for param, value in params.items():
            if value is None:
                continue
            param = {
                'product': 'pv.product_name',
                'version': 'version_string',
                'is_featured': 'featured_version',
                'end_date': 'sunset_date',
                'start_date': 'build_date',
            }.get(param, param)

            if param == 'active':
                # This is a convenient exception. It makes it possible
                # to query for only productversions that are NOT sunset
                # without having to do any particular date arithmetic.
                param = 'sunset_date'
                operator_ = value and '>=' or '<'
                value = datetime.datetime.utcnow().date().isoformat()
            elif isinstance(value, list):
                operator_ = 'IN'
                value = tuple(value)
            elif isinstance(value, tuple):
                assert len(value) == 2
                operator_, value = value
            else:
                operator_ = '='
            where.append('{} {} %({})s'.format(param, operator_, param))
            sql_params[param] = value

        # rewrite it to a string
        if where:
            sql_where = 'WHERE ' + ' AND '.join(where)
        else:
            sql_where = ''

        sql = """
            /* socorro.external.postgresql.products.ProductVersions.get */
            SELECT
                pv.product_name AS product,
                pv.version_string AS version,
                pv.build_date AS start_date,
                pv.sunset_date AS end_date,
                ((prc.throttle * (100)::numeric))::REAL AS throttle,
                pv.featured_version AS is_featured,
                pv.build_type,
                pv.has_builds,
                pv.is_rapid_beta
            FROM (
                ( product_versions pv
                  JOIN product_release_channels prc
                    ON (
                        pv.product_name = prc.product_name AND
                        pv.build_type = prc.release_channel
                  )
                  JOIN products
                    ON pv.product_name = products.product_name
                )
                JOIN release_channels
                    ON pv.build_type = release_channels.release_channel
            )
            {}
            ORDER BY products.sort, version_sort DESC, release_channels.sort
        """.format(sql_where)
        results = self.query(sql, sql_params).zipped()

        return {
            'hits': results,
            'total': len(results),
        }
示例#49
0
    def get(self, **kwargs):
        """ Return product information, or version information for one
        or more product:version combinations """
        warnings.warn('This class is deprecated. Use ProductVersions instead.',
                      DeprecationWarning)
        filters = [
            ("versions", None, ["list", "str"]),  # for legacy, to be removed
        ]
        params = external_common.parse_arguments(filters, kwargs)
        if params.versions and params.versions[0]:
            return self._get_versions(params)

        sql = """
            /* socorro.external.postgresql.products.Products.get */
            SELECT
                product_name AS product,
                version_string AS version,
                start_date,
                end_date,
                throttle,
                is_featured AS featured,
                build_type AS release,
                has_builds
            FROM product_info
            ORDER BY product_sort, version_sort DESC, channel_sort
        """

        error_message = "Failed to retrieve products/versions from PostgreSQL"
        results = self.query(sql, error_message=error_message)

        products = []
        versions_per_product = {}

        for version in results.zipped():
            try:
                version['end_date'] = datetimeutil.date_to_string(
                    version['end_date'])
            except TypeError:
                pass
            try:
                version['start_date'] = datetimeutil.date_to_string(
                    version['start_date'])
            except TypeError:
                pass

            version['throttle'] = float(version['throttle'])

            product = version['product']
            if product not in products:
                products.append(product)

            if product not in versions_per_product:
                versions_per_product[product] = [version]
            else:
                versions_per_product[product].append(version)

        return {
            'products': products,
            'hits': versions_per_product,
            'total': len(results)
        }
示例#50
0
    def get(self, **kwargs):
        filters = [
            ("report_types", None, ["list", "str"]),
            ("report_type", None, "str"),
            ("signature", None, "str"),
            ("start_date", None, "date"),
            ("end_date", None, "date"),
            ("versions", None, ["list", "str"]),
        ]

        params = external_common.parse_arguments(filters, kwargs)
        if not params.get('report_types') and params.get('report_type'):
            # to support the legacy case
            individual_report = True
            report_types = [params['report_type']]
        else:
            individual_report = False
            report_types = params['report_types']

        # check that all the report types are recognized
        for report_type in report_types:
            query_params = report_type_sql.get(report_type, {})
            known_report_types = ('products', 'distinct_install',
                                  'exploitability', 'devices', 'graphics')
            if (report_type not in known_report_types
                    and 'first_col' not in query_params):
                raise BadArgumentError(report_type)

        products = []
        versions = []

        # Get information about the versions
        util_service = Util(config=self.context)
        versions_info = util_service.versions_info(**params)
        if versions_info:
            for i, elem in enumerate(versions_info):
                products.append(versions_info[elem]["product_name"])
                versions.append(str(versions_info[elem]["version_string"]))

        # This MUST be a tuple otherwise it gets cast to an array
        params['product'] = tuple(products)
        params['version'] = tuple(versions)

        all_results = {}
        assert isinstance(params['start_date'], datetime.date)
        assert isinstance(params['end_date'], datetime.date)

        with self.get_connection() as connection:
            for report_type in report_types:
                result_cols, query_string, query_parameters = self._get_query(
                    report_type, params)
                sql_results = self.query(query_string,
                                         params=query_parameters,
                                         connection=connection)
                results = [dict(zip(result_cols, row)) for row in sql_results]
                all_results[report_type] = results

            if individual_report:
                return all_results.values()[0]
            else:
                return {'reports': all_results}
示例#51
0
    def get(self, **kwargs):
        """Return JSON data of a crash report, given its uuid. """

        filters = [
            ('uuid', None, 'str'),
            ('datatype', None, 'str'),
            ('name', None, 'str')  # only applicable if datatype == 'raw'
        ]
        params = external_common.parse_arguments(filters, kwargs)

        if not params.uuid:
            raise MissingArgumentError('uuid')

        if not params.datatype:
            raise MissingArgumentError('datatype')

        # get a generic crashstorage instance from whatever external resource
        # is implementing this service.
        store = self.get_storage()

        datatype_method_mapping = {
            'raw': 'get_raw_dump',
            'meta': 'get_raw_crash',
            'processed': 'get_processed',
            'unredacted': 'get_unredacted_processed',
        }

        get = store.__getattribute__(datatype_method_mapping[params.datatype])
        try:
            if params.datatype == 'raw':
                return (
                    get(params.uuid, name=params.name),
                    'application/octet-stream'
                )
            else:
                return get(params.uuid)
        except CrashIDNotFound:
            if params.datatype in ('processed', 'unredacted'):
                # try to fetch a raw crash just to ensure that the raw crash
                # exists.  If this line fails, there's no reason to actually
                # submit the priority job.
                try:
                    store.get_raw_crash(params.uuid)
                except CrashIDNotFound:
                    raise ResourceNotFound(params.uuid)
                # search through the existing other services to find the
                # Priorityjob service.
                try:
                    priorityjob_service_impl = self.all_services[
                        'Priorityjobs'
                    ]
                except KeyError:
                    raise ServiceUnavailable('Priorityjobs')
                # get the underlying implementation of the Priorityjob
                # service and instantiate it.
                priority_job_service = priorityjob_service_impl.cls(
                    config=self.config
                )
                # create the priority job for this crash_ids
                priority_job_service.create(uuid=params.uuid)
                raise ResourceUnavailable(params.uuid)
            raise ResourceNotFound(params.uuid)
示例#52
0
    def versions_info(self, **kwargs):
        """
        Return information about versions of a product.

        See https://socorro.readthedocs.io/en/latest/middleware.html

        Keyword arguments:
        versions - List of products and versions.

        Return:
        None if versions is null or empty ;
        Otherwise a dictionary of data about a version, i.e.:
        {
            "product_name:version_string": {
                "product_version_id": integer,
                "version_string": "string",
                "product_name": "string",
                "major_version": "string" or None,
                "release_channel": "string" or None,
                "build_id": [list, of, decimals] or None
            }
        }

        """
        # Parse arguments
        filters = [("versions", None, ["list", "str"])]
        params = external_common.parse_arguments(filters, kwargs)

        if "versions" not in params or not params["versions"]:
            return None

        products_list = []
        (versions_list,
         products_list) = self.parse_versions(params["versions"],
                                              products_list)

        if not versions_list:
            return None

        versions = []
        products = []
        for x in xrange(0, len(versions_list), 2):
            products.append(versions_list[x])
            versions.append(versions_list[x + 1])

        params = {}
        params = self.dispatch_params(params, "product", products)
        params = self.dispatch_params(params, "version", versions)
        prefixed_versions = ["^%s" % x for x in versions]
        params = self.dispatch_params(params, "start_with_version",
                                      prefixed_versions)

        where_product = []
        where_rapid_beta = []
        for i in range(len(products)):
            where_product.append("""
                (pv.product_name = %%(product%(i)s)s
                AND pv.version_string ~ %%(start_with_version%(i)s)s)
            """ % {'i': i})
            where_rapid_beta.append("""
                (
                    i1.version_string = %%(version%(i)s)s
                    AND i1.version_string = i2.version_string
                ) OR (
                    i1.rapid_beta_id = i2.product_version_id
                    AND i2.version_string = %%(version%(i)s)s
                    AND i2.is_rapid_beta IS TRUE
                )
            """ % {'i': i})

        sql = """
            /* socorro.external.postgresql.util.Util.versions_info */
            WITH infos AS (
                SELECT
                    pv.product_version_id,
                    pv.version_string,
                    pv.product_name,
                    pv.release_version,
                    pv.build_type,
                    pvb.build_id,
                    pv.is_rapid_beta,
                    pv.rapid_beta_id,
                    pv.version_sort
                FROM product_versions pv
                    LEFT JOIN product_version_builds pvb ON
                        (pv.product_version_id = pvb.product_version_id)
                WHERE %(product_filters)s
            )
            SELECT DISTINCT
                i1.product_version_id,
                i1.product_name,
                i1.version_string,
                i1.release_version,
                i1.build_type,
                i1.build_id,
                i1.is_rapid_beta,
                i2.is_rapid_beta AS is_from_rapid_beta,
                (i2.product_name || ':' || i2.version_string)
                    AS from_beta_version,
                i1.version_sort
            FROM infos i1
                LEFT JOIN infos i2 ON (
                    i1.product_name = i2.product_name
                    AND i1.release_version = i2.release_version
                    AND i1.build_type = i2.build_type
                )
            WHERE %(rapid_beta_filters)s
            ORDER BY i1.version_sort
        """ % {
            "product_filters": " OR ".join(where_product),
            "rapid_beta_filters": " OR ".join(where_rapid_beta),
        }

        error_message = "Failed to retrieve versions data from PostgreSQL"
        results = self.query(sql, params, error_message=error_message)

        res = {}
        for row in results:
            version = dict(
                zip((
                    "product_version_id",
                    "product_name",
                    "version_string",
                    "major_version",
                    "release_channel",
                    "build_id",
                    "is_rapid_beta",
                    "is_from_rapid_beta",
                    "from_beta_version",
                    "version_sort",
                ), row))

            key = ":".join(
                (version["product_name"], version["version_string"]))

            del version["version_sort"]  # no need to send this back

            if key in res:
                # That key already exists, just add it the new buildid
                res[key]["build_id"].append(int(version["build_id"]))
            else:
                if version["build_id"]:
                    version["build_id"] = [int(version["build_id"])]

                res[key] = version

        return res
示例#53
0
def get_parameters(kwargs):
    """
    Return a dictionary of parameters with default values.

    Optional arguments:
    data_type -- Type of data to return.
        Default is None, to be determined by each service if needed.
    terms -- Terms to search for.
        Can be a string or a list of strings.
        Default is none.
    fields -- Fields to search into.
        Can be a string or a list of strings.
        Default to signature, not implemented for PostgreSQL.
    search_mode -- How to search for terms.
        Must be one of the following:
            "default", "contains", "is_exactly" or "starts_with".
        Default to "default" for ElasticSearch,
            "starts_with" for PostgreSQL.
    from_date -- Only elements after this date.
        Format must be "YYYY-mm-dd HH:ii:ss.S".
        Default is a week ago.
    to_date -- Only elements before this date.
        Format must be "YYYY-mm-dd HH:ii:ss.S".
        Default is now.
    products -- Products concerned by this search.
        Can be a string or a list of strings.
        Default is Firefox.
    os -- Restrict search to those operating systems.
        Can be a string or a list of strings.
        Default is all.
    versions -- Version of the software.
        Can be a string or a list of strings.
        Default is all.
    build_ids -- Restrict search to a particular build of the software.
        Can be a string or a list of strings.
        Default is all.
    reasons -- Restrict search to crashes caused by this reason.
        Default is all.
    release_channels -- Restrict search to crashes in this release channels.
        Default is all.
    report_type -- Retrict to a type of report.
        Can be any, crash or hang.
        Default is any.
    report_process -- How was the report processed.
        Can be any, crash or hang.
        Default is any.
    plugin_terms -- Search for terms concerning plugins.
        Can be a string or a list of strings.
        Default is none.
    plugin_in -- What field to look into.
        Can be "name" or "filename".
        Default is 'name'.
    plugin_search_mode -- How to search into plugins.
        Must be one of the following:
            "contains", "is_exactly" or "starts_with".
        Default to "contains".
    result_number -- Number of results to get.
        Default is 100.
    result_offset -- Get results from this offset.
        Default is 0.
    """
    # Default dates
    now = datetimeutil.utc_now()
    lastweek = now - datetime.timedelta(7)

    filters = [
        ("data_type", "signatures", "str"),
        ("terms", None, ["list", "str"]),
        ("signature", None, "str"),
        ("fields", "signature", ["list", "str"]),
        ("search_mode", "default", "str"),
        ("from_date", lastweek, "datetime"),
        ("to_date", now, "datetime"),
        ("products", None, ["list", "str"]),
        ("versions", None, ["list", "str"]),
        ("os", None, ["list", "str"]),
        ("reasons", None, ["list", "str"]),
        ("release_channels", None, ["list", "str"]),
        ("build_ids", None, ["list", "str"]),
        ("build_from", lastweek, "datetime"),
        ("build_to", now, "datetime"),
        ("report_process", "any", "str"),
        ("report_type", "any", "str"),
        ("plugin_terms", None, ["list", "str"]),
        ("plugin_in", "name", ["list", "str"]),
        ("plugin_search_mode", "default", "str"),
        ("result_number", 100, "int"),
        ("result_offset", 0, "int")
    ]

    params = extern.parse_arguments(filters, kwargs)

    # To be moved into a config file?
    authorized_modes = [
        "default",
        "starts_with",
        "contains",
        "is_exactly"
    ]
    if params["search_mode"] not in authorized_modes:
        params["search_mode"] = "default"
    if params["plugin_search_mode"] not in authorized_modes:
        params["plugin_search_mode"] = "default"

    # Do not search in the future and make sure we have dates where expected
    if params["to_date"] is None or params["to_date"] > now:
        params["to_date"] = now
    if params["from_date"] is None:
        params["from_date"] = lastweek

    if params["build_to"] is None or params["build_to"] > now:
        params["build_to"] = now
    if params["build_from"] is None:
        params["build_from"] = lastweek

    # Securing fields
    params['fields'] = restrict_fields(
        params['fields'],
        ['signature', 'dump']
    )
    params['plugin_in'] = restrict_fields(
        params['plugin_in'],
        ['filename', 'name']
    )

    return params
示例#54
0
    def get(self, **kwargs):
        """Return JSON data of a crash report, given its uuid. """

        filters = [
            ('uuid', None, 'str'),
            ('datatype', None, 'str'),
            ('name', None, 'str')  # only applicable if datatype == 'raw'
        ]
        params = external_common.parse_arguments(filters, kwargs)

        if not params.uuid:
            raise MissingArgumentError('uuid')

        if not params.datatype:
            raise MissingArgumentError('datatype')

        # get a generic crashstorage instance from whatever external resource
        # is implementing this service.
        store = self.get_storage()

        datatype_method_mapping = {
            'raw': 'get_raw_dump',
            'meta': 'get_raw_crash',
            'processed': 'get_processed',
            'unredacted': 'get_unredacted_processed',
        }

        get = store.__getattribute__(datatype_method_mapping[params.datatype])
        try:
            if params.datatype == 'raw':
                return (
                    get(params.uuid, name=params.name),
                    'application/octet-stream'
                )
            else:
                return get(params.uuid)
        except CrashIDNotFound:
            if params.datatype in ('processed', 'unredacted'):
                # try to fetch a raw crash just to ensure that the raw crash
                # exists.  If this line fails, there's no reason to actually
                # submit the priority job.
                try:
                    store.get_raw_crash(params.uuid)
                except CrashIDNotFound:
                    raise ResourceNotFound(params.uuid)
                # search through the existing other services to find the
                # Priorityjob service.
                try:
                    priorityjob_service_impl = self.all_services[
                        'Priorityjobs'
                    ]
                except KeyError:
                    raise ServiceUnavailable('Priorityjobs')
                # get the underlying implementation of the Priorityjob
                # service and instantiate it.
                priority_job_service = priorityjob_service_impl.cls(
                    config=self.config
                )
                # create the priority job for this crash_ids
                priority_job_service.create(uuid=params.uuid)
                raise ResourceUnavailable(params.uuid)
            raise ResourceNotFound(params.uuid)
示例#55
0
    def get(self, **kwargs):
        """ Return urls for signature """
        filters = [
            ("signature", None, "str"),
            ("start_date", None, "datetime"),
            ("end_date", None, "datetime"),
            ("products", None, ["list", "str"]),
            ("versions", None, ["list", "str"]),
        ]

        params = external_common.parse_arguments(filters, kwargs)

        #Because no parameters are optional, we need to loop through
        #all parameters to ensure each has been set and is not None
        missingParams = []
        for param in params:
            if not params[param]:
                if param == 'versions':
                    # force versions parameter to being 'ALL' if empty
                    params[param] = 'ALL'
                    continue
                missingParams.append(param)

        if len(missingParams) > 0:
            raise MissingArgumentError(", ".join(missingParams))

        all_products_versions_sql = """
        /* socorro.external.postgresql.signature_urls.SignatureURLs.get */
            SELECT url, count(*) as crash_count FROM reports_clean
            JOIN reports_user_info USING ( UUID )
            JOIN signatures USING ( signature_id )
            WHERE reports_clean.date_processed
                BETWEEN %(start_date)s AND %(end_date)s
            AND reports_user_info.date_processed
                BETWEEN %(start_date)s AND %(end_date)s
            AND signature = %(signature)s
            AND url <> ''
        """

        sql = """
        /* socorro.external.postgresql.signature_urls.SignatureURLs.get */
            SELECT url, count(*) as crash_count FROM reports_clean
            JOIN reports_user_info USING ( UUID )
            JOIN signatures USING ( signature_id )
            JOIN product_versions USING ( product_version_id )
            WHERE reports_clean.date_processed
                BETWEEN %(start_date)s AND %(end_date)s
            AND reports_user_info.date_processed
                BETWEEN %(start_date)s AND %(end_date)s
            AND signature = %(signature)s
            AND url <> ''
            AND (
        """

        sql_group_order = """ GROUP BY url
            ORDER BY crash_count DESC LIMIT 100"""
        sql_params = {
            "start_date": params.start_date,
            "end_date": params.end_date,
            "signature": params.signature
        }

        # if this query is for all products the 'ALL' keyword will be
        # the only item in the products list and this will then also
        # be for all versions.
        if 'ALL' in params['products']:
            sql_query = " ".join((all_products_versions_sql, sql_group_order))
        # if this query is for all versions the 'ALL' keyword will be
        # the only item in the versions list.
        elif 'ALL' in params['versions']:
            sql_products = " product_name IN %(products)s )"
            sql_params['products'] = tuple(params.products)

            sql_date_range_limit = """AND %(end_date)s BETWEEN
                product_versions.build_date
                    AND product_versions.sunset_date"""
            sql_query = " ".join((sql, sql_products,
                                  sql_date_range_limit, sql_group_order))
        else:
            products = []
            (params["products_versions"],
             products) = self.parse_versions(params["versions"], [])

            if len(params["products_versions"]) == 0:
                raise BadArgumentError(", ".join(params["versions"]))

            versions_list = []
            products_list = []
            for x in range(0, len(params["products_versions"]), 2):
                products_list.append(params["products_versions"][x])
                versions_list.append(params["products_versions"][x + 1])

            product_version_list = []
            for prod in params["products"]:
                versions = []
                [versions.append(versions_list[i])
                 for i, x in enumerate(products_list)
                 if x == prod]
                product_version_list.append(tuple(versions))

            sql_product_version_ids = [
                """( product_name = %%(product%s)s
                    AND version_string IN %%(version%s)s ) """
                        % (x, x) for x in range(len(product_version_list))]

            sql_params = add_param_to_dict(sql_params, "version",
                                       product_version_list)

            sql_params = add_param_to_dict(sql_params, "product",
                                       params.products)

            sql_query = " ".join((sql, " OR ".join(sql_product_version_ids),
                              " ) " + sql_group_order))

        error_message = "Failed to retrieve urls for signature from PostgreSQL"
        results = self.query(sql_query, sql_params,
                             error_message=error_message)
        urls = results.zipped()
        return {
            "hits": urls,
            "total": len(urls)
        }
示例#56
0
    def get_exploitability(self, **kwargs):
        """Return a list of exploitable crash reports.

        See socorrolib.lib.external_common.parse_arguments() for all filters.
        """
        now = datetimeutil.utc_now().date()
        lastweek = now - datetime.timedelta(weeks=1)

        filters = [
            ("start_date", lastweek, "date"),
            ("end_date", now, "date"),
            ("product", None, "str"),
            ("version", None, "str"),
            ("page", None, "int"),
            ("batch", None, "int"),
        ]

        params = external_common.parse_arguments(filters, kwargs)

        sql_where = """
            report_date BETWEEN %(start_date)s AND %(end_date)s
            AND
            null_count + none_count + low_count + medium_count + high_count > 4
        """

        if params.product:
            sql_where += " AND pv.product_name = %(product)s"
        if params.version:
            sql_where += " AND pv.version_string = %(version)s"

        inner_with_sql = """
            SELECT
                signature,
                SUM(high_count) AS high_count,
                SUM(medium_count) AS medium_count,
                SUM(low_count) AS low_count,
                SUM(null_count) AS null_count,
                SUM(none_count) AS none_count,
                SUM(high_count) + SUM(medium_count) AS med_or_high
            FROM exploitability_reports
            JOIN product_versions AS pv USING (product_version_id)
            WHERE
                high_count + medium_count + null_count + none_count > 4
                AND
                %s
            GROUP BY signature
        """ % (sql_where, )

        count_sql_query = """
            /* external.postgresql.crashes.Crashes.get_exploitability */
            WITH sums AS (
                %s
            )
            SELECT
                count(signature)
            FROM sums
        """ % (inner_with_sql, )

        results = self.query(
            count_sql_query,
            params,
            error_message="Failed to retrieve exploitable crashes count")
        total_crashes_count, = results[0]

        sql_query = """
            /* external.postgresql.crashes.Crashes.get_exploitability */
            WITH sums AS (
                %s
            )
            SELECT
                signature,
                high_count,
                medium_count,
                low_count,
                null_count,
                none_count
            FROM sums
            ORDER BY
                med_or_high DESC, signature ASC
        """ % (inner_with_sql, )

        if params['page'] is not None:
            if params['page'] <= 0:
                raise BadArgumentError('page', params['page'], 'starts on 1')
            if params['batch'] is None:
                raise MissingArgumentError('batch')
            sql_query += """
            LIMIT %(limit)s
            OFFSET %(offset)s
            """
            params['limit'] = params['batch']
            params['offset'] = params['batch'] * (params['page'] - 1)

        error_message = (
            "Failed to retrieve exploitable crashes from PostgreSQL")
        results = self.query(sql_query, params, error_message=error_message)

        # Transforming the results into what we want
        crashes = results.zipped()

        return {"hits": crashes, "total": total_crashes_count}