def main(argv=None):
parser = argparse.ArgumentParser(
formatter_class=WrappedTextHelpFormatter,
description='Send crash id to rabbitmq queue for processing',
epilog=EPILOG.strip(),
)
parser.add_argument('queue', help='the queue to add the crash id to')
parser.add_argument('crashid', nargs='*', help='one or more crash ids to add')
if argv is None:
args = parser.parse_args()
else:
args = parser.parse_args(argv)
# This will pull crash ids from the command line if specified, or stdin
crashids_iterable = args.crashid or sys.stdin
crashids = [crashid.strip() for crashid in crashids_iterable if crashid.strip()]
# Verify crash ids first
for crashid in crashids:
if not is_crash_id_valid(crashid):
print('Crash id "%s" is not valid. Exiting.' % crashid)
return 1
# NOTE(willkg): This matches what's in socorro.external.rabbitmq classes without us having to
# use configman and ConnectionContext and deal with switching between configured queues
host = get_envvar('resource.rabbitmq.host')
port = int(get_envvar('resource.rabbitmq.port', '5672'))
user = get_envvar('secrets.rabbitmq.rabbitmq_user')
password = get_envvar('secrets.rabbitmq.rabbitmq_password')
virtual_host = get_envvar('resource.rabbitmq.virtual_host', '/')
print('Configuration:')
print('host: %s' % host)
print('port: %s' % port)
print('user: %s' % user)
print('password: ********')
print('virtual_host: %s' % virtual_host)
print('queue: %s' % args.queue)
print('# crashids: %s' % len(crashids))
print('')
conn = build_pika_connection(host, port, virtual_host, user, password)
props = pika.BasicProperties(delivery_mode=2)
channel = conn.channel()
for crashid in crashids:
print('Sending %s to %s....' % (crashid, args.queue))
channel.basic_publish(
exchange='',
routing_key=args.queue,
body=crashid,
properties=props
)
print('Done!')
return 0
def main(argv):
parser = argparse.ArgumentParser(
formatter_class=WrappedTextHelpFormatter,
prog=os.path.basename(__file__),
description='Send crash id to rabbitmq queue for processing',
epilog=EPILOG.strip(),
)
parser.add_argument('queue', help='the queue to add the crash id to')
parser.add_argument('crashid',
nargs='*',
help='one or more crash ids to add')
args = parser.parse_args(argv)
# This will pull crash ids from the command line if specified, or stdin
crashids_iterable = args.crashid or sys.stdin
crashids = [
crashid.strip() for crashid in crashids_iterable if crashid.strip()
]
# Verify crash ids first
for crashid in crashids:
if not is_crash_id_valid(crashid):
print('Crash id "%s" is not valid. Exiting.' % crashid)
return 1
# NOTE(willkg): This matches what's in socorro.external.rabbitmq classes without us having to
# use configman and ConnectionContext and deal with switching between configured queues
host = get_envvar('resource.rabbitmq.host')
port = int(get_envvar('resource.rabbitmq.port', '5672'))
user = get_envvar('secrets.rabbitmq.rabbitmq_user')
password = get_envvar('secrets.rabbitmq.rabbitmq_password')
virtual_host = get_envvar('resource.rabbitmq.virtual_host', '/')
print('Configuration:')
print('host: %s' % host)
print('port: %s' % port)
print('user: %s' % user)
print('password: ********')
print('virtual_host: %s' % virtual_host)
print('queue: %s' % args.queue)
print('# crashids: %s' % len(crashids))
print('')
conn = build_pika_connection(host, port, virtual_host, user, password)
props = pika.BasicProperties(delivery_mode=2)
channel = conn.channel()
for crashid in crashids:
print('Sending %s to %s....' % (crashid, args.queue))
channel.basic_publish(exchange='',
routing_key=args.queue,
body=crashid,
properties=props)
print('Done!')
return 0
def crash_verify(request):
"""Verifies crash data in crash data destinations"""
crash_id = request.GET.get('crash_id', None)
if not crash_id or not is_crash_id_valid(crash_id):
return http.JsonResponse({'error': 'unknown crash id'}, status=400)
data = {'uuid': crash_id}
# Check S3 crash bucket for raw and processed crash data
raw_api = models.RawCrash()
try:
raw_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True)
has_raw_crash = True
except CrashIDNotFound:
has_raw_crash = False
data['s3_raw_crash'] = has_raw_crash
processed_api = models.ProcessedCrash()
try:
processed_api.get(crash_id=crash_id,
dont_cache=True,
refresh_cache=True)
has_processed_crash = True
except CrashIDNotFound:
has_processed_crash = False
data['s3_processed_crash'] = has_processed_crash
# Check Elasticsearch for crash data
supersearch_api = supersearch_models.SuperSearch()
params = {
'_columns': ['uuid'],
'_results_number': 1,
'uuid': crash_id,
'dont_cache': True,
'refresh_cache': True
}
results = supersearch_api.get(**params)
data['elasticsearch_crash'] = (results['total'] == 1
and results['hits'][0]['uuid'] == crash_id)
# Check S3 telemetry bucket for crash data
telemetry_api = models.TelemetryCrash()
try:
telemetry_api.get(crash_id=crash_id,
dont_cache=True,
refresh_cache=True)
has_telemetry_crash = True
except CrashIDNotFound:
has_telemetry_crash = False
data['s3_telemetry_crash'] = has_telemetry_crash
# NOTE(willkg): This doesn't check postgres because that's being phased
# out.
return http.HttpResponse(json.dumps(data), content_type='application/json')
def post(self, **data):
crash_ids = data["crash_ids"]
if not isinstance(crash_ids, (list, tuple)):
crash_ids = [crash_ids]
# If one of them isn't a crash id, raise a 400.
for crash_id in crash_ids:
if not is_crash_id_valid(crash_id):
raise BadArgumentError("Crash id '%s' is not valid." % crash_id)
return self.get_implementation().publish(
queue="reprocessing", crash_ids=crash_ids
)
def crash_verify(request):
"""Verifies crash data in crash data destinations"""
crash_id = request.GET.get("crash_id", None)
if not crash_id or not is_crash_id_valid(crash_id):
return http.JsonResponse({"error": "unknown crash id"}, status=400)
data = {"uuid": crash_id}
# Check S3 crash bucket for raw and processed crash data
raw_api = models.RawCrash()
try:
raw_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True)
has_raw_crash = True
except CrashIDNotFound:
has_raw_crash = False
data["s3_raw_crash"] = has_raw_crash
processed_api = models.ProcessedCrash()
try:
processed_api.get(crash_id=crash_id,
dont_cache=True,
refresh_cache=True)
has_processed_crash = True
except CrashIDNotFound:
has_processed_crash = False
data["s3_processed_crash"] = has_processed_crash
# Check Elasticsearch for crash data
supersearch_api = supersearch_models.SuperSearch()
params = {
"_columns": ["uuid"],
"_results_number": 1,
"uuid": crash_id,
"dont_cache": True,
"refresh_cache": True,
}
results = supersearch_api.get(**params)
data["elasticsearch_crash"] = (results["total"] == 1
and results["hits"][0]["uuid"] == crash_id)
# Check S3 telemetry bucket for crash data
telemetry_api = models.TelemetryCrash()
try:
telemetry_api.get(crash_id=crash_id,
dont_cache=True,
refresh_cache=True)
has_telemetry_crash = True
except CrashIDNotFound:
has_telemetry_crash = False
data["s3_telemetry_crash"] = has_telemetry_crash
return http.HttpResponse(json.dumps(data), content_type="application/json")
def crash_verify(request):
"""Verifies crash data in crash data destinations"""
crash_id = request.GET.get('crash_id', None)
if not crash_id or not is_crash_id_valid(crash_id):
return http.JsonResponse({'error': 'unknown crash id'}, status=400)
data = {
'uuid': crash_id
}
# Check S3 crash bucket for raw and processed crash data
raw_api = models.RawCrash()
try:
raw_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True)
has_raw_crash = True
except CrashIDNotFound:
has_raw_crash = False
data['s3_raw_crash'] = has_raw_crash
processed_api = models.ProcessedCrash()
try:
processed_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True)
has_processed_crash = True
except CrashIDNotFound:
has_processed_crash = False
data['s3_processed_crash'] = has_processed_crash
# Check Elasticsearch for crash data
supersearch_api = supersearch_models.SuperSearch()
params = {
'_columns': ['uuid'],
'_results_number': 1,
'uuid': crash_id,
'dont_cache': True,
'refresh_cache': True
}
results = supersearch_api.get(**params)
data['elasticsearch_crash'] = (
results['total'] == 1 and
results['hits'][0]['uuid'] == crash_id
)
# Check S3 telemetry bucket for crash data
telemetry_api = models.TelemetryCrash()
try:
telemetry_api.get(crash_id=crash_id, dont_cache=True, refresh_cache=True)
has_telemetry_crash = True
except CrashIDNotFound:
has_telemetry_crash = False
data['s3_telemetry_crash'] = has_telemetry_crash
return http.HttpResponse(json.dumps(data), content_type='application/json')
def get(self, **kwargs):
"""Return JSON data of a crash report, given its uuid. """
filters = [
("uuid", None, str),
("datatype", None, str),
("name", None, str), # only applicable if datatype == 'raw'
]
params = external_common.parse_arguments(filters, kwargs, modern=True)
if not params.uuid:
raise MissingArgumentError("uuid")
if not ooid.is_crash_id_valid(params.uuid):
raise BadArgumentError("uuid")
if not params.datatype:
raise MissingArgumentError("datatype")
datatype_method_mapping = {
"raw": "get_raw_dump",
"meta": "get_raw_crash",
"processed": "get_processed",
"unredacted": "get_unredacted_processed",
}
if params.datatype not in datatype_method_mapping:
raise BadArgumentError(params.datatype)
get = self.__getattribute__(datatype_method_mapping[params.datatype])
try:
if params.datatype == "raw":
return get(params.uuid, name=params.name)
else:
return get(params.uuid)
except CrashIDNotFound as cidnf:
self.logger.warning(
"%(datatype)s not found: %(exception)s",
{
"datatype": params.datatype,
"exception": cidnf
},
)
# The CrashIDNotFound exception that happens inside the
# crashstorage is too revealing as exception message
# contains information about buckets and prefix keys.
# Re-wrap it here so the message is just the crash ID.
raise CrashIDNotFound(params.uuid)
def get(self, **kwargs):
"""Return JSON data of a crash report, given its uuid. """
filters = [
('uuid', None, str),
('datatype', None, str),
('name', None, str) # only applicable if datatype == 'raw'
]
params = external_common.parse_arguments(filters, kwargs, modern=True)
if not params.uuid:
raise MissingArgumentError('uuid')
if not ooid.is_crash_id_valid(params.uuid):
raise BadArgumentError('uuid')
if not params.datatype:
raise MissingArgumentError('datatype')
datatype_method_mapping = {
'raw': 'get_raw_dump',
'meta': 'get_raw_crash',
'processed': 'get_processed',
'unredacted': 'get_unredacted_processed',
}
if params.datatype not in datatype_method_mapping:
raise BadArgumentError(params.datatype)
get = self.__getattribute__(datatype_method_mapping[params.datatype])
try:
if params.datatype == 'raw':
return get(params.uuid, name=params.name)
else:
return get(params.uuid)
except CrashIDNotFound as cidnf:
self.config.logger.error('%s not found: %s' %
(params.datatype, cidnf))
# The CrashIDNotFound exception that happens inside the
# crashstorage is too revealing as exception message
# contains information about buckets and prefix keys.
# Re-wrap it here so the message is just the crash ID.
raise CrashIDNotFound(params.uuid)
def get(self, **kwargs):
"""Return JSON data of a crash report, given its uuid. """
filters = [
('uuid', None, str),
('datatype', None, str),
('name', None, str) # only applicable if datatype == 'raw'
]
params = external_common.parse_arguments(filters, kwargs, modern=True)
if not params.uuid:
raise MissingArgumentError('uuid')
if not ooid.is_crash_id_valid(params.uuid):
raise BadArgumentError('uuid')
if not params.datatype:
raise MissingArgumentError('datatype')
datatype_method_mapping = {
'raw': 'get_raw_dump',
'meta': 'get_raw_crash',
'processed': 'get_processed',
'unredacted': 'get_unredacted_processed',
}
if params.datatype not in datatype_method_mapping:
raise BadArgumentError(params.datatype)
get = self.__getattribute__(datatype_method_mapping[params.datatype])
try:
if params.datatype == 'raw':
return get(params.uuid, name=params.name)
else:
return get(params.uuid)
except CrashIDNotFound as cidnf:
self.config.logger.error('%s not found: %s' % (params.datatype, cidnf))
# The CrashIDNotFound exception that happens inside the
# crashstorage is too revealing as exception message
# contains information about buckets and prefix keys.
# Re-wrap it here so the message is just the crash ID.
raise CrashIDNotFound(params.uuid)
def test_validate_crash_id(crashid, expected):
assert ooid.is_crash_id_valid(crashid) == expected
def test_validate_crash_id(crashid, expected):
assert ooid.is_crash_id_valid(crashid) == expected
