def test_primitives(self): # Test all the primitives assert dotdict_to_dict(None) is None assert dotdict_to_dict([]) == [] assert dotdict_to_dict('') == '' assert dotdict_to_dict(1) == 1 assert dotdict_to_dict({}) == {}
def test_primitives(self): # Test all the primitives assert dotdict_to_dict(None) is None assert dotdict_to_dict([]) == [] assert dotdict_to_dict("") == "" assert dotdict_to_dict(1) == 1 assert dotdict_to_dict({}) == {}
def comp(data, expected): # First dotdict_to_dict the data and compare it. new_dict = dotdict_to_dict(data) assert new_dict == expected # Now deepcopy the new dict to make sure it's ok. copy.deepcopy(new_dict)
def action(self, raw_crash, raw_dumps, processed_crash, processor_meta): info = raw_crash["ModuleSignatureInfo"] if isinstance(info, DotDict): # Sometimes the value is a DotDict which json.dumps doesn't work with so # convert it to a dict first info = dotdict_to_dict(info) raw_crash["ModuleSignatureInfo"] = json.dumps(info)
def _temp_raw_crash_json_file(self, raw_crash, crash_id): file_pathname = os.path.join( self.config.temporary_file_system_storage_path, '%s.%s.TEMPORARY.json' % (crash_id, threading.currentThread().getName()) ) with open(file_pathname, "w") as f: json.dump(dotdict_to_dict(raw_crash), f) try: yield file_pathname finally: os.unlink(file_pathname)
def _transform(self, crash_id): """Transform a raw crash into a process crash. The ``crash_id`` passed in is used as a key to fetch the raw crash data from the ``source``, the ``processor_class`` processes the crash and the processed crash is saved to the ``destination``. """ # Fetch the raw crash data try: raw_crash = self.source.get_raw_crash(crash_id) dumps = self.source.get_raw_dumps_as_files(crash_id) except CrashIDNotFound: # If the crash isn't found, we just reject it--no need to capture # errors here self.processor.reject_raw_crash(crash_id, 'crash cannot be found in raw crash storage') return except Exception as x: # We don't know what this error is, so we should capture it self._capture_error(crash_id, sys.exc_info()) self.logger.warning('error loading crash %s', crash_id, exc_info=True) self.processor.reject_raw_crash(crash_id, 'error in loading: %s' % x) return # Fetch processed crash data--there won't be any if this crash hasn't # been processed, yet try: processed_crash = self.source.get_unredacted_processed(crash_id) except CrashIDNotFound: processed_crash = DotDict() # Process the crash and remove any temporary artifacts from disk try: # Process the crash to generate a processed crash processed_crash = self.processor.process_crash(raw_crash, dumps, processed_crash) # Convert the raw and processed crashes from DotDict into Python standard data # structures raw_crash = dotdict_to_dict(raw_crash) processed_crash = dotdict_to_dict(processed_crash) # bug 866973 - save_raw_and_processed() instead of just save_processed(). # The raw crash may have been modified by the processor rules. The # individual crash storage implementations may choose to honor re-saving # the raw_crash or not. self.destination.save_raw_and_processed(raw_crash, None, processed_crash, crash_id) self.logger.info('saved - %s', crash_id) except Exception: # Capture the exception so we don't lose it as we do other things exc_type, exc_value, exc_tb = sys.exc_info() # PolyStorage can throw a PolyStorageException which is a sequence # of exc_info items, so we need to capture each one if isinstance(exc_value, collections.Sequence): exc_info = exc_value else: exc_info = [(exc_type, exc_value, exc_tb)] for exc_info_item in exc_info: self._capture_error(crash_id, exc_info_item) self.logger.warning('error in processing or saving crash %s', crash_id) # Re-raise the original exception with the correct traceback six.reraise(exc_type, exc_value, exc_tb) finally: # Clean up any dump files saved to the file system for a_dump_pathname in dumps.values(): if 'TEMPORARY' in a_dump_pathname: try: os.unlink(a_dump_pathname) except OSError as x: self.logger.info('deletion of dump failed: %s', x)
def _transform(self, crash_id): """Transform a raw crash into a process crash. The ``crash_id`` passed in is used as a key to fetch the raw crash data from the ``source``, the ``processor_class`` processes the crash and the processed crash is saved to the ``destination``. """ # Fetch the raw crash data try: raw_crash = self.source.get_raw_crash(crash_id) dumps = self.source.get_raw_dumps_as_files(crash_id) except CrashIDNotFound: # If the crash isn't found, we just reject it--no need to capture # errors here self.processor.reject_raw_crash( crash_id, 'crash cannot be found in raw crash storage') return except Exception as x: # We don't know what this error is, so we should capture it self._capture_error(sys.exc_info(), crash_id) self.logger.warning('error loading crash %s', crash_id, exc_info=True) self.processor.reject_raw_crash(crash_id, 'error in loading: %s' % x) return # Fetch processed crash data--there won't be any if this crash hasn't # been processed, yet try: processed_crash = self.source.get_unredacted_processed(crash_id) except CrashIDNotFound: processed_crash = DotDict() # Process the crash and remove any temporary artifacts from disk try: # Process the crash to generate a processed crash processed_crash = self.processor.process_crash( raw_crash, dumps, processed_crash) # Convert the raw and processed crashes from DotDict into Python standard data # structures raw_crash = dotdict_to_dict(raw_crash) processed_crash = dotdict_to_dict(processed_crash) # bug 866973 - save_raw_and_processed() instead of just save_processed(). # The raw crash may have been modified by the processor rules. The # individual crash storage implementations may choose to honor re-saving # the raw_crash or not. self.destination.save_raw_and_processed(raw_crash, None, processed_crash, crash_id) self.logger.info('saved - %s', crash_id) except Exception: # Capture the exception so we don't lose it as we do other things exc_type, exc_value, exc_tb = sys.exc_info() # PolyStorage can throw a PolyStorageException which is a sequence # of exc_info items, so we need to capture each one if isinstance(exc_value, collections.Sequence): exc_info = exc_value else: exc_info = [(exc_type, exc_value, exc_tb)] for exc_info_item in exc_info: self._capture_error(exc_info_item, crash_id) self.logger.warning('error in processing or saving crash %s', crash_id) # Re-raise the original exception with the correct traceback six.reraise(exc_type, exc_value, exc_tb) finally: # Clean up any dump files saved to the file system for a_dump_pathname in dumps.values(): if 'TEMPORARY' in a_dump_pathname: try: os.unlink(a_dump_pathname) except OSError as x: self.logger.info('deletion of dump failed: %s', x)
def dict_to_str(a_mapping): if isinstance(a_mapping, DotDict): a_mapping = dotdict_to_dict(a_mapping) return json.dumps(a_mapping, cls=JSONISOEncoder)
def _fake_unredacted_processed_crash_as_string(self): d = self._fake_unredacted_processed_crash() s = json.dumps(dotdict_to_dict(d)) return s
def _transform(self, crash_id): """Transform a raw crash into a process crash. The ``crash_id`` passed in is used as a key to fetch the raw crash data from the ``source``, the ``processor_class`` processes the crash and the processed crash is saved to the ``destination``. """ # Fetch the raw crash data try: raw_crash = self.source.get_raw_crash(crash_id) dumps = self.source.get_raw_dumps_as_files(crash_id) except CrashIDNotFound: # If the crash isn't found, we just reject it--no need to capture # errors here self.processor.reject_raw_crash( crash_id, "crash cannot be found in raw crash storage" ) return except Exception as x: # We don't know what this error is, so we should capture it self._capture_error(sys.exc_info(), crash_id) self.logger.warning("error loading crash %s", crash_id, exc_info=True) self.processor.reject_raw_crash(crash_id, "error in loading: %s" % x) return # Fetch processed crash data--there won't be any if this crash hasn't # been processed, yet try: processed_crash = self.source.get_unredacted_processed(crash_id) except CrashIDNotFound: processed_crash = DotDict() # Process the crash and remove any temporary artifacts from disk try: # Process the crash to generate a processed crash processed_crash = self.processor.process_crash( raw_crash, dumps, processed_crash ) # Convert the raw and processed crashes from DotDict into Python standard # data structures raw_crash = dotdict_to_dict(raw_crash) processed_crash = dotdict_to_dict(processed_crash) self.destination.save_processed_crash(raw_crash, processed_crash) self.logger.info("saved - %s", crash_id) except PolyStorageError as poly_storage_error: # Capture and log the exceptions raised by storage backends for storage_error in poly_storage_error: self._capture_error(storage_error, crash_id) self.logger.warning("error in processing or saving crash %s", crash_id) # Re-raise the original exception with the correct traceback raise finally: # Clean up any dump files saved to the file system for a_dump_pathname in dumps.values(): if "TEMPORARY" in a_dump_pathname: try: os.unlink(a_dump_pathname) except OSError as x: self.logger.info("deletion of dump failed: %s", x)