def test_primitives(self):
# Test all the primitives
assert dotdict_to_dict(None) is None
assert dotdict_to_dict([]) == []
assert dotdict_to_dict('') == ''
assert dotdict_to_dict(1) == 1
assert dotdict_to_dict({}) == {}
def test_primitives(self):
# Test all the primitives
assert dotdict_to_dict(None) is None
assert dotdict_to_dict([]) == []
assert dotdict_to_dict("") == ""
assert dotdict_to_dict(1) == 1
assert dotdict_to_dict({}) == {}
def comp(data, expected):
# First dotdict_to_dict the data and compare it.
new_dict = dotdict_to_dict(data)
assert new_dict == expected
# Now deepcopy the new dict to make sure it's ok.
copy.deepcopy(new_dict)
def action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
info = raw_crash["ModuleSignatureInfo"]
if isinstance(info, DotDict):
# Sometimes the value is a DotDict which json.dumps doesn't work with so
# convert it to a dict first
info = dotdict_to_dict(info)
raw_crash["ModuleSignatureInfo"] = json.dumps(info)
def comp(data, expected):
# First dotdict_to_dict the data and compare it.
new_dict = dotdict_to_dict(data)
assert new_dict == expected
# Now deepcopy the new dict to make sure it's ok.
copy.deepcopy(new_dict)
def _temp_raw_crash_json_file(self, raw_crash, crash_id):
file_pathname = os.path.join(
self.config.temporary_file_system_storage_path,
'%s.%s.TEMPORARY.json' % (crash_id, threading.currentThread().getName())
)
with open(file_pathname, "w") as f:
json.dump(dotdict_to_dict(raw_crash), f)
try:
yield file_pathname
finally:
os.unlink(file_pathname)
def _temp_raw_crash_json_file(self, raw_crash, crash_id):
file_pathname = os.path.join(
self.config.temporary_file_system_storage_path,
'%s.%s.TEMPORARY.json' % (crash_id, threading.currentThread().getName())
)
with open(file_pathname, "w") as f:
json.dump(dotdict_to_dict(raw_crash), f)
try:
yield file_pathname
finally:
os.unlink(file_pathname)
def _transform(self, crash_id):
"""Transform a raw crash into a process crash.
The ``crash_id`` passed in is used as a key to fetch the raw crash data
from the ``source``, the ``processor_class`` processes the crash and
the processed crash is saved to the ``destination``.
"""
# Fetch the raw crash data
try:
raw_crash = self.source.get_raw_crash(crash_id)
dumps = self.source.get_raw_dumps_as_files(crash_id)
except CrashIDNotFound:
# If the crash isn't found, we just reject it--no need to capture
# errors here
self.processor.reject_raw_crash(crash_id, 'crash cannot be found in raw crash storage')
return
except Exception as x:
# We don't know what this error is, so we should capture it
self._capture_error(crash_id, sys.exc_info())
self.logger.warning('error loading crash %s', crash_id, exc_info=True)
self.processor.reject_raw_crash(crash_id, 'error in loading: %s' % x)
return
# Fetch processed crash data--there won't be any if this crash hasn't
# been processed, yet
try:
processed_crash = self.source.get_unredacted_processed(crash_id)
except CrashIDNotFound:
processed_crash = DotDict()
# Process the crash and remove any temporary artifacts from disk
try:
# Process the crash to generate a processed crash
processed_crash = self.processor.process_crash(raw_crash, dumps, processed_crash)
# Convert the raw and processed crashes from DotDict into Python standard data
# structures
raw_crash = dotdict_to_dict(raw_crash)
processed_crash = dotdict_to_dict(processed_crash)
# bug 866973 - save_raw_and_processed() instead of just save_processed().
# The raw crash may have been modified by the processor rules. The
# individual crash storage implementations may choose to honor re-saving
# the raw_crash or not.
self.destination.save_raw_and_processed(raw_crash, None, processed_crash, crash_id)
self.logger.info('saved - %s', crash_id)
except Exception:
# Capture the exception so we don't lose it as we do other things
exc_type, exc_value, exc_tb = sys.exc_info()
# PolyStorage can throw a PolyStorageException which is a sequence
# of exc_info items, so we need to capture each one
if isinstance(exc_value, collections.Sequence):
exc_info = exc_value
else:
exc_info = [(exc_type, exc_value, exc_tb)]
for exc_info_item in exc_info:
self._capture_error(crash_id, exc_info_item)
self.logger.warning('error in processing or saving crash %s', crash_id)
# Re-raise the original exception with the correct traceback
six.reraise(exc_type, exc_value, exc_tb)
finally:
# Clean up any dump files saved to the file system
for a_dump_pathname in dumps.values():
if 'TEMPORARY' in a_dump_pathname:
try:
os.unlink(a_dump_pathname)
except OSError as x:
self.logger.info('deletion of dump failed: %s', x)
def _transform(self, crash_id):
"""Transform a raw crash into a process crash.
The ``crash_id`` passed in is used as a key to fetch the raw crash data
from the ``source``, the ``processor_class`` processes the crash and
the processed crash is saved to the ``destination``.
"""
# Fetch the raw crash data
try:
raw_crash = self.source.get_raw_crash(crash_id)
dumps = self.source.get_raw_dumps_as_files(crash_id)
except CrashIDNotFound:
# If the crash isn't found, we just reject it--no need to capture
# errors here
self.processor.reject_raw_crash(
crash_id, 'crash cannot be found in raw crash storage')
return
except Exception as x:
# We don't know what this error is, so we should capture it
self._capture_error(sys.exc_info(), crash_id)
self.logger.warning('error loading crash %s',
crash_id,
exc_info=True)
self.processor.reject_raw_crash(crash_id,
'error in loading: %s' % x)
return
# Fetch processed crash data--there won't be any if this crash hasn't
# been processed, yet
try:
processed_crash = self.source.get_unredacted_processed(crash_id)
except CrashIDNotFound:
processed_crash = DotDict()
# Process the crash and remove any temporary artifacts from disk
try:
# Process the crash to generate a processed crash
processed_crash = self.processor.process_crash(
raw_crash, dumps, processed_crash)
# Convert the raw and processed crashes from DotDict into Python standard data
# structures
raw_crash = dotdict_to_dict(raw_crash)
processed_crash = dotdict_to_dict(processed_crash)
# bug 866973 - save_raw_and_processed() instead of just save_processed().
# The raw crash may have been modified by the processor rules. The
# individual crash storage implementations may choose to honor re-saving
# the raw_crash or not.
self.destination.save_raw_and_processed(raw_crash, None,
processed_crash, crash_id)
self.logger.info('saved - %s', crash_id)
except Exception:
# Capture the exception so we don't lose it as we do other things
exc_type, exc_value, exc_tb = sys.exc_info()
# PolyStorage can throw a PolyStorageException which is a sequence
# of exc_info items, so we need to capture each one
if isinstance(exc_value, collections.Sequence):
exc_info = exc_value
else:
exc_info = [(exc_type, exc_value, exc_tb)]
for exc_info_item in exc_info:
self._capture_error(exc_info_item, crash_id)
self.logger.warning('error in processing or saving crash %s',
crash_id)
# Re-raise the original exception with the correct traceback
six.reraise(exc_type, exc_value, exc_tb)
finally:
# Clean up any dump files saved to the file system
for a_dump_pathname in dumps.values():
if 'TEMPORARY' in a_dump_pathname:
try:
os.unlink(a_dump_pathname)
except OSError as x:
self.logger.info('deletion of dump failed: %s', x)
def dict_to_str(a_mapping):
if isinstance(a_mapping, DotDict):
a_mapping = dotdict_to_dict(a_mapping)
return json.dumps(a_mapping, cls=JSONISOEncoder)
def _fake_unredacted_processed_crash_as_string(self):
d = self._fake_unredacted_processed_crash()
s = json.dumps(dotdict_to_dict(d))
return s
def _transform(self, crash_id):
"""Transform a raw crash into a process crash.
The ``crash_id`` passed in is used as a key to fetch the raw crash data
from the ``source``, the ``processor_class`` processes the crash and
the processed crash is saved to the ``destination``.
"""
# Fetch the raw crash data
try:
raw_crash = self.source.get_raw_crash(crash_id)
dumps = self.source.get_raw_dumps_as_files(crash_id)
except CrashIDNotFound:
# If the crash isn't found, we just reject it--no need to capture
# errors here
self.processor.reject_raw_crash(
crash_id, "crash cannot be found in raw crash storage"
)
return
except Exception as x:
# We don't know what this error is, so we should capture it
self._capture_error(sys.exc_info(), crash_id)
self.logger.warning("error loading crash %s", crash_id, exc_info=True)
self.processor.reject_raw_crash(crash_id, "error in loading: %s" % x)
return
# Fetch processed crash data--there won't be any if this crash hasn't
# been processed, yet
try:
processed_crash = self.source.get_unredacted_processed(crash_id)
except CrashIDNotFound:
processed_crash = DotDict()
# Process the crash and remove any temporary artifacts from disk
try:
# Process the crash to generate a processed crash
processed_crash = self.processor.process_crash(
raw_crash, dumps, processed_crash
)
# Convert the raw and processed crashes from DotDict into Python standard
# data structures
raw_crash = dotdict_to_dict(raw_crash)
processed_crash = dotdict_to_dict(processed_crash)
self.destination.save_processed_crash(raw_crash, processed_crash)
self.logger.info("saved - %s", crash_id)
except PolyStorageError as poly_storage_error:
# Capture and log the exceptions raised by storage backends
for storage_error in poly_storage_error:
self._capture_error(storage_error, crash_id)
self.logger.warning("error in processing or saving crash %s", crash_id)
# Re-raise the original exception with the correct traceback
raise
finally:
# Clean up any dump files saved to the file system
for a_dump_pathname in dumps.values():
if "TEMPORARY" in a_dump_pathname:
try:
os.unlink(a_dump_pathname)
except OSError as x:
self.logger.info("deletion of dump failed: %s", x)
def _fake_unredacted_processed_crash_as_string(self):
d = self._fake_unredacted_processed_crash()
s = json.dumps(dotdict_to_dict(d))
return s
