def main(argv=None): parser = argparse.ArgumentParser( formatter_class=WrappedTextHelpFormatter, description='Send crash id to rabbitmq queue for processing', epilog=EPILOG.strip(), ) parser.add_argument('queue', help='the queue to add the crash id to') parser.add_argument('crashid', nargs='*', help='one or more crash ids to add') if argv is None: args = parser.parse_args() else: args = parser.parse_args(argv) # This will pull crash ids from the command line if specified, or stdin crashids_iterable = args.crashid or sys.stdin crashids = [crashid.strip() for crashid in crashids_iterable if crashid.strip()] # Verify crash ids first for crashid in crashids: if not is_crash_id_valid(crashid): print('Crash id "%s" is not valid. Exiting.' % crashid) return 1 # NOTE(willkg): This matches what's in socorro.external.rabbitmq classes without us having to # use configman and ConnectionContext and deal with switching between configured queues host = get_envvar('resource.rabbitmq.host') port = int(get_envvar('resource.rabbitmq.port', '5672')) user = get_envvar('secrets.rabbitmq.rabbitmq_user') password = get_envvar('secrets.rabbitmq.rabbitmq_password') virtual_host = get_envvar('resource.rabbitmq.virtual_host', '/') print('Configuration:') print('host: %s' % host) print('port: %s' % port) print('user: %s' % user) print('password: ********') print('virtual_host: %s' % virtual_host) print('queue: %s' % args.queue) print('# crashids: %s' % len(crashids)) print('') conn = build_pika_connection(host, port, virtual_host, user, password) props = pika.BasicProperties(delivery_mode=2) channel = conn.channel() for crashid in crashids: print('Sending %s to %s....' % (crashid, args.queue)) channel.basic_publish( exchange='', routing_key=args.queue, body=crashid, properties=props ) print('Done!') return 0
def main(argv): parser = argparse.ArgumentParser( formatter_class=WrappedTextHelpFormatter, prog=os.path.basename(__file__), description='Send crash id to rabbitmq queue for processing', epilog=EPILOG.strip(), ) parser.add_argument('queue', help='the queue to add the crash id to') parser.add_argument('crashid', nargs='*', help='one or more crash ids to add') args = parser.parse_args(argv) # This will pull crash ids from the command line if specified, or stdin crashids_iterable = args.crashid or sys.stdin crashids = [ crashid.strip() for crashid in crashids_iterable if crashid.strip() ] # Verify crash ids first for crashid in crashids: if not is_crash_id_valid(crashid): print('Crash id "%s" is not valid. Exiting.' % crashid) return 1 # NOTE(willkg): This matches what's in socorro.external.rabbitmq classes without us having to # use configman and ConnectionContext and deal with switching between configured queues host = get_envvar('resource.rabbitmq.host') port = int(get_envvar('resource.rabbitmq.port', '5672')) user = get_envvar('secrets.rabbitmq.rabbitmq_user') password = get_envvar('secrets.rabbitmq.rabbitmq_password') virtual_host = get_envvar('resource.rabbitmq.virtual_host', '/') print('Configuration:') print('host: %s' % host) print('port: %s' % port) print('user: %s' % user) print('password: ********') print('virtual_host: %s' % virtual_host) print('queue: %s' % args.queue) print('# crashids: %s' % len(crashids)) print('') conn = build_pika_connection(host, port, virtual_host, user, password) props = pika.BasicProperties(delivery_mode=2) channel = conn.channel() for crashid in crashids: print('Sending %s to %s....' % (crashid, args.queue)) channel.basic_publish(exchange='', routing_key=args.queue, body=crashid, properties=props) print('Done!') return 0
def crash_verify(request): """Verifies crash data in crash data destinations""" crash_id = request.GET.get('crash_id', None) if not crash_id or not is_crash_id_valid(crash_id): return http.JsonResponse({'error': 'unknown crash id'}, status=400) data = {'uuid': crash_id} # Check S3 crash bucket for raw and processed crash data raw_api = models.RawCrash() try: raw_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True) has_raw_crash = True except CrashIDNotFound: has_raw_crash = False data['s3_raw_crash'] = has_raw_crash processed_api = models.ProcessedCrash() try: processed_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True) has_processed_crash = True except CrashIDNotFound: has_processed_crash = False data['s3_processed_crash'] = has_processed_crash # Check Elasticsearch for crash data supersearch_api = supersearch_models.SuperSearch() params = { '_columns': ['uuid'], '_results_number': 1, 'uuid': crash_id, 'dont_cache': True, 'refresh_cache': True } results = supersearch_api.get(**params) data['elasticsearch_crash'] = (results['total'] == 1 and results['hits'][0]['uuid'] == crash_id) # Check S3 telemetry bucket for crash data telemetry_api = models.TelemetryCrash() try: telemetry_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True) has_telemetry_crash = True except CrashIDNotFound: has_telemetry_crash = False data['s3_telemetry_crash'] = has_telemetry_crash # NOTE(willkg): This doesn't check postgres because that's being phased # out. return http.HttpResponse(json.dumps(data), content_type='application/json')
def post(self, **data): crash_ids = data["crash_ids"] if not isinstance(crash_ids, (list, tuple)): crash_ids = [crash_ids] # If one of them isn't a crash id, raise a 400. for crash_id in crash_ids: if not is_crash_id_valid(crash_id): raise BadArgumentError("Crash id '%s' is not valid." % crash_id) return self.get_implementation().publish( queue="reprocessing", crash_ids=crash_ids )
def crash_verify(request): """Verifies crash data in crash data destinations""" crash_id = request.GET.get("crash_id", None) if not crash_id or not is_crash_id_valid(crash_id): return http.JsonResponse({"error": "unknown crash id"}, status=400) data = {"uuid": crash_id} # Check S3 crash bucket for raw and processed crash data raw_api = models.RawCrash() try: raw_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True) has_raw_crash = True except CrashIDNotFound: has_raw_crash = False data["s3_raw_crash"] = has_raw_crash processed_api = models.ProcessedCrash() try: processed_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True) has_processed_crash = True except CrashIDNotFound: has_processed_crash = False data["s3_processed_crash"] = has_processed_crash # Check Elasticsearch for crash data supersearch_api = supersearch_models.SuperSearch() params = { "_columns": ["uuid"], "_results_number": 1, "uuid": crash_id, "dont_cache": True, "refresh_cache": True, } results = supersearch_api.get(**params) data["elasticsearch_crash"] = (results["total"] == 1 and results["hits"][0]["uuid"] == crash_id) # Check S3 telemetry bucket for crash data telemetry_api = models.TelemetryCrash() try: telemetry_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True) has_telemetry_crash = True except CrashIDNotFound: has_telemetry_crash = False data["s3_telemetry_crash"] = has_telemetry_crash return http.HttpResponse(json.dumps(data), content_type="application/json")
def crash_verify(request): """Verifies crash data in crash data destinations""" crash_id = request.GET.get('crash_id', None) if not crash_id or not is_crash_id_valid(crash_id): return http.JsonResponse({'error': 'unknown crash id'}, status=400) data = { 'uuid': crash_id } # Check S3 crash bucket for raw and processed crash data raw_api = models.RawCrash() try: raw_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True) has_raw_crash = True except CrashIDNotFound: has_raw_crash = False data['s3_raw_crash'] = has_raw_crash processed_api = models.ProcessedCrash() try: processed_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True) has_processed_crash = True except CrashIDNotFound: has_processed_crash = False data['s3_processed_crash'] = has_processed_crash # Check Elasticsearch for crash data supersearch_api = supersearch_models.SuperSearch() params = { '_columns': ['uuid'], '_results_number': 1, 'uuid': crash_id, 'dont_cache': True, 'refresh_cache': True } results = supersearch_api.get(**params) data['elasticsearch_crash'] = ( results['total'] == 1 and results['hits'][0]['uuid'] == crash_id ) # Check S3 telemetry bucket for crash data telemetry_api = models.TelemetryCrash() try: telemetry_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True) has_telemetry_crash = True except CrashIDNotFound: has_telemetry_crash = False data['s3_telemetry_crash'] = has_telemetry_crash return http.HttpResponse(json.dumps(data), content_type='application/json')
def get(self, **kwargs): """Return JSON data of a crash report, given its uuid. """ filters = [ ("uuid", None, str), ("datatype", None, str), ("name", None, str), # only applicable if datatype == 'raw' ] params = external_common.parse_arguments(filters, kwargs, modern=True) if not params.uuid: raise MissingArgumentError("uuid") if not ooid.is_crash_id_valid(params.uuid): raise BadArgumentError("uuid") if not params.datatype: raise MissingArgumentError("datatype") datatype_method_mapping = { "raw": "get_raw_dump", "meta": "get_raw_crash", "processed": "get_processed", "unredacted": "get_unredacted_processed", } if params.datatype not in datatype_method_mapping: raise BadArgumentError(params.datatype) get = self.__getattribute__(datatype_method_mapping[params.datatype]) try: if params.datatype == "raw": return get(params.uuid, name=params.name) else: return get(params.uuid) except CrashIDNotFound as cidnf: self.logger.warning( "%(datatype)s not found: %(exception)s", { "datatype": params.datatype, "exception": cidnf }, ) # The CrashIDNotFound exception that happens inside the # crashstorage is too revealing as exception message # contains information about buckets and prefix keys. # Re-wrap it here so the message is just the crash ID. raise CrashIDNotFound(params.uuid)
def get(self, **kwargs): """Return JSON data of a crash report, given its uuid. """ filters = [ ('uuid', None, str), ('datatype', None, str), ('name', None, str) # only applicable if datatype == 'raw' ] params = external_common.parse_arguments(filters, kwargs, modern=True) if not params.uuid: raise MissingArgumentError('uuid') if not ooid.is_crash_id_valid(params.uuid): raise BadArgumentError('uuid') if not params.datatype: raise MissingArgumentError('datatype') datatype_method_mapping = { 'raw': 'get_raw_dump', 'meta': 'get_raw_crash', 'processed': 'get_processed', 'unredacted': 'get_unredacted_processed', } if params.datatype not in datatype_method_mapping: raise BadArgumentError(params.datatype) get = self.__getattribute__(datatype_method_mapping[params.datatype]) try: if params.datatype == 'raw': return get(params.uuid, name=params.name) else: return get(params.uuid) except CrashIDNotFound as cidnf: self.config.logger.error('%s not found: %s' % (params.datatype, cidnf)) # The CrashIDNotFound exception that happens inside the # crashstorage is too revealing as exception message # contains information about buckets and prefix keys. # Re-wrap it here so the message is just the crash ID. raise CrashIDNotFound(params.uuid)
def test_validate_crash_id(crashid, expected): assert ooid.is_crash_id_valid(crashid) == expected