def __init__(self, config, quit_check_callback=None): super(FileSystemRawCrashStorage, self).__init__(config) self.std_crash_store = JsonDumpStorage( root=config.std_fs_root, maxDirectoryEntries=config.dump_dir_count, jsonSuffix=config.json_file_suffix, dumpSuffix=config.dump_file_suffix, dumpGID=config.dump_gid, dumpPermissions=config.dump_permissions, dirPermissions=config.dir_permissions, logger=config.logger) self.hostname = os.uname()[1]
def __init__(self, config, quit_check_callback=None): super(FileSystemRawCrashStorage, self).__init__(config) self.std_crash_store = JsonDumpStorage( root=config.std_fs_root, maxDirectoryEntries=config.dump_dir_count, jsonSuffix=config.json_file_suffix, dumpSuffix=config.dump_file_suffix, dumpGID=config.dump_gid, dumpPermissions=config.dump_permissions, dirPermissions=config.dir_permissions, logger=config.logger ) self.hostname = os.uname()[1]
class FileSystemRawCrashStorage(CrashStorageBase): """This crash storage class impements only the raw crash part of the api. Raw crashes (the json file and the binary dump) are stored in a file system. This class is appropriate for fast storage of crashes into a local file system. In 2011, a varient of this code base was adopted by the Socorro Collector for fast temporary storage as crashes came in.""" required_config = Namespace() required_config.add_option( 'std_fs_root', doc='a path to a local file system', default='./primaryCrashStore', reference_value_from='resource.filesystem', ) required_config.add_option( 'dump_dir_count', doc='the number of dumps to be stored in a single directory in the ' 'local file system', default=1024, reference_value_from='resource.filesystem', ) required_config.add_option( 'dump_gid', doc='the group ID for saved crashes in local file system (optional)', default='', reference_value_from='resource.filesystem', ) required_config.add_option( 'dump_permissions', doc='a number used for permissions crash dump files in the local ' 'file system', default=stat.S_IRGRP | stat.S_IWGRP | stat.S_IRUSR | stat.S_IWUSR, reference_value_from='resource.filesystem', ) required_config.add_option( 'dir_permissions', doc='a number used for permissions for directories in the local ' 'file system', default=(stat.S_IRGRP | stat.S_IXGRP | stat.S_IWGRP | stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR), reference_value_from='resource.filesystem', ) required_config.add_option( 'json_file_suffix', doc='the suffix used to identify a json file', default='.json', reference_value_from='resource.filesystem', ) required_config.add_option( 'dump_file_suffix', doc='the suffix used to identify a dump file', default='.dump', reference_value_from='resource.filesystem', ) #-------------------------------------------------------------------------- def __init__(self, config, quit_check_callback=None): super(FileSystemRawCrashStorage, self).__init__(config) self.std_crash_store = JsonDumpStorage( root=config.std_fs_root, maxDirectoryEntries=config.dump_dir_count, jsonSuffix=config.json_file_suffix, dumpSuffix=config.dump_file_suffix, dumpGID=config.dump_gid, dumpPermissions=config.dump_permissions, dirPermissions=config.dir_permissions, logger=config.logger ) self.hostname = os.uname()[1] #-------------------------------------------------------------------------- def _load_raw_crash_from_file(self, pathname): with open(pathname) as json_file: raw_crash = json.load(json_file, object_hook=DotDict) return raw_crash #-------------------------------------------------------------------------- def _do_save_raw(self, json_storage_system, raw_crash, dumps, crash_id): json_storage_system.new_entry( crash_id, raw_crash, dumps, self.hostname ) #-------------------------------------------------------------------------- def save_raw_crash(self, raw_crash, dumps, crash_id): """forward the raw_crash and the dump to the underlying file system""" self._do_save_raw(self.std_crash_store, raw_crash, dumps, crash_id) def save_raw_and_processed(self, raw_crash, dumps, processed_crash, crash_id): """ bug 866973 - do not try to save dumps=None into the Filesystem We are doing this in lieu of a queuing solution that could allow us to operate an independent crashmover. When the queuing system is implemented, we could remove this, and have the raw crash saved by a crashmover that's consuming crash_ids the same way that the processor consumes them. Even though it is ok to resave the raw_crash in this case to the filesystem, the fs does not know what to do with a dumps=None when passed to save_raw, so we are going to avoid that. """ self.save_processed(processed_crash) #-------------------------------------------------------------------------- def get_raw_crash(self, crash_id): """fetch the raw crash from the underlying file system""" try: pathname = self.std_crash_store.getJson(crash_id) return self._load_raw_crash_from_file(pathname) except OSError: raise CrashIDNotFound(crash_id) except ValueError: # empty json file? return DotDict() #-------------------------------------------------------------------------- def get_raw_dump(self, crash_id, dump_name=None): """read the binary crash dump from the underlying file system by getting the pathname and then opening and reading the file.""" try: job_pathname = self.std_crash_store.getDump(crash_id, dump_name) with open(job_pathname) as dump_file: binary = dump_file.read() return binary except OSError: raise CrashIDNotFound(crash_id) #-------------------------------------------------------------------------- def _do_get_raw_dumps(self, crash_id, crash_store): try: dumpname_paths_map = crash_store.get_dumps(crash_id) dumpname_dump_map = {} for dump_name, dump_pathname in dumpname_paths_map.iteritems(): with open(dump_pathname, 'rb') as f: dumpname_dump_map[dump_name] = f.read() return dumpname_dump_map except OSError: raise CrashIDNotFound(crash_id) #-------------------------------------------------------------------------- def get_raw_dumps(self, crash_id): """read the all the binary crash dumps from the underlying file system by getting the pathnames and then opening and reading the files. returns a dict of dump names to binary dumps""" return self._do_get_raw_dumps(crash_id, self.std_crash_store) #-------------------------------------------------------------------------- def get_raw_dumps_as_files(self, crash_id): """read the all the binary crash dumps from the underlying file system by getting the pathnames and then opening and reading the files. returns a dict of dump names to binary dumps""" return self.std_crash_store.get_dumps(crash_id) #-------------------------------------------------------------------------- def new_crashes(self): """return an iterator that yields a list of crash_ids of raw crashes that were added to the file system since the last time this iterator was requested.""" # why is this called 'destructiveDateWalk'? The underlying code # that manages the filesystem uses a tree of radix date directories # and symbolic links to track "new" raw crashes. As the the crash_ids # are fetched from the file system, the symbolic links are removed and # directories are deleted. Essentially, the state of what is # considered to be new is saved within the file system by those links. return self.std_crash_store.destructiveDateWalk() #-------------------------------------------------------------------------- def remove(self, crash_id): """delegate removal of a raw crash to the underlying filesystem""" try: self.std_crash_store.quickDelete(crash_id) except NoSuchUuidFound: raise CrashIDNotFound(crash_id)
class FileSystemRawCrashStorage(CrashStorageBase): """This crash storage class impements only the raw crash part of the api. Raw crashes (the json file and the binary dump) are stored in a file system. This class is appropriate for fast storage of crashes into a local file system. In 2011, a varient of this code base was adopted by the Socorro Collector for fast temporary storage as crashes came in.""" required_config = Namespace() required_config.add_option( "std_fs_root", doc="a path to a local file system", default="/home/socorro/primaryCrashStore" ) required_config.add_option( "dump_dir_count", doc="the number of dumps to be stored in a single directory in the " "local file system", default=1024, ) required_config.add_option( "dump_gid", doc="the group ID for saved crashes in local file system (optional)", default="" ) required_config.add_option( "dump_permissions", doc="a number used for permissions crash dump files in the local " "file system", default=stat.S_IRGRP | stat.S_IWGRP | stat.S_IRUSR | stat.S_IWUSR, ) required_config.add_option( "dir_permissions", doc="a number used for permissions for directories in the local " "file system", default=(stat.S_IRGRP | stat.S_IXGRP | stat.S_IWGRP | stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR), ) required_config.add_option("json_file_suffix", doc="the suffix used to identify a json file", default=".json") required_config.add_option("dump_file_suffix", doc="the suffix used to identify a dump file", default=".dump") # -------------------------------------------------------------------------- def __init__(self, config, quit_check_callback=None): super(FileSystemRawCrashStorage, self).__init__(config) self.std_crash_store = JsonDumpStorage( root=config.std_fs_root, maxDirectoryEntries=config.dump_dir_count, jsonSuffix=config.json_file_suffix, dumpSuffix=config.dump_file_suffix, dumpGID=config.dump_gid, dumpPermissions=config.dump_permissions, dirPermissions=config.dir_permissions, logger=config.logger, ) self.hostname = os.uname()[1] # -------------------------------------------------------------------------- def _load_raw_crash_from_file(self, pathname): with open(pathname) as json_file: raw_crash = json.load(json_file, object_hook=DotDict) return raw_crash # -------------------------------------------------------------------------- def _do_save_raw(self, json_storage_system, raw_crash, dump, crash_id): try: json_file_handle, dump_file_handle = json_storage_system.newEntry( crash_id, self.hostname # from base class ) try: dump_file_handle.write(dump) json.dump(raw_crash, json_file_handle) finally: dump_file_handle.close() json_file_handle.close() self.logger.debug("saved - %s", crash_id) except Exception: self.logger.critical("storage has failed for: %s", crash_id, exc_info=True) raise # -------------------------------------------------------------------------- def save_raw_crash(self, raw_crash, dump, crash_id): """forward the raw_crash and the dump to the underlying file system""" self._do_save_raw(self.std_crash_store, raw_crash, dump, crash_id) # -------------------------------------------------------------------------- def get_raw_crash(self, crash_id): """fetch the raw crash from the underlying file system""" try: pathname = self.std_crash_store.getJson(crash_id) return self._load_raw_crash_from_file(pathname) except OSError: raise CrashIDNotFound(crash_id) except ValueError: # empty json file? return DotDict() # -------------------------------------------------------------------------- def get_raw_dump(self, crash_id): """read the binary crash dump from the underlying file system by getting the pathname and then opening and reading the file.""" try: job_pathname = self.std_crash_store.getDump(crash_id) with open(job_pathname) as dump_file: binary = dump_file.read() return binary except OSError: raise CrashIDNotFound(crash_id) # -------------------------------------------------------------------------- def new_crashes(self): """return an iterator that yields a list of crash_ids of raw crashes that were added to the file system since the last time this iterator was requested.""" # why is this called 'destructiveDateWalk'? The underlying code # that manages the filesystem uses a tree of radix date directories # and symbolic links to track "new" raw crashes. As the the crash_ids # are fetched from the file system, the symbolic links are removed and # directories are deleted. Essentially, the state of what is # considered to be new is saved within the file system by those links. return self.std_crash_store.destructiveDateWalk() # -------------------------------------------------------------------------- def remove(self, crash_id): """delegate removal of a raw crash to the underlying filesystem""" try: self.std_crash_store.remove(crash_id) except NoSuchUuidFound: raise CrashIDNotFound(crash_id)
class FileSystemRawCrashStorage(CrashStorageBase): """This crash storage class impements only the raw crash part of the api. Raw crashes (the json file and the binary dump) are stored in a file system. This class is appropriate for fast storage of crashes into a local file system. In 2011, a varient of this code base was adopted by the Socorro Collector for fast temporary storage as crashes came in.""" required_config = Namespace() required_config.add_option('std_fs_root', doc='a path to a local file system', default='./primaryCrashStore') required_config.add_option( 'dump_dir_count', doc='the number of dumps to be stored in a single directory in the ' 'local file system', default=1024) required_config.add_option( 'dump_gid', doc='the group ID for saved crashes in local file system (optional)', default='') required_config.add_option( 'dump_permissions', doc='a number used for permissions crash dump files in the local ' 'file system', default=stat.S_IRGRP | stat.S_IWGRP | stat.S_IRUSR | stat.S_IWUSR) required_config.add_option( 'dir_permissions', doc='a number used for permissions for directories in the local ' 'file system', default=(stat.S_IRGRP | stat.S_IXGRP | stat.S_IWGRP | stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR)) required_config.add_option('json_file_suffix', doc='the suffix used to identify a json file', default='.json') required_config.add_option('dump_file_suffix', doc='the suffix used to identify a dump file', default='.dump') #-------------------------------------------------------------------------- def __init__(self, config, quit_check_callback=None): super(FileSystemRawCrashStorage, self).__init__(config) self.std_crash_store = JsonDumpStorage( root=config.std_fs_root, maxDirectoryEntries=config.dump_dir_count, jsonSuffix=config.json_file_suffix, dumpSuffix=config.dump_file_suffix, dumpGID=config.dump_gid, dumpPermissions=config.dump_permissions, dirPermissions=config.dir_permissions, logger=config.logger) self.hostname = os.uname()[1] #-------------------------------------------------------------------------- def _load_raw_crash_from_file(self, pathname): with open(pathname) as json_file: raw_crash = json.load(json_file, object_hook=DotDict) return raw_crash #-------------------------------------------------------------------------- def _do_save_raw(self, json_storage_system, raw_crash, dumps, crash_id): json_storage_system.new_entry(crash_id, raw_crash, dumps, self.hostname) #-------------------------------------------------------------------------- def save_raw_crash(self, raw_crash, dumps, crash_id): """forward the raw_crash and the dump to the underlying file system""" self._do_save_raw(self.std_crash_store, raw_crash, dumps, crash_id) #-------------------------------------------------------------------------- def get_raw_crash(self, crash_id): """fetch the raw crash from the underlying file system""" try: pathname = self.std_crash_store.getJson(crash_id) return self._load_raw_crash_from_file(pathname) except OSError: raise CrashIDNotFound(crash_id) except ValueError: # empty json file? return DotDict() #-------------------------------------------------------------------------- def get_raw_dump(self, crash_id, dump_name=None): """read the binary crash dump from the underlying file system by getting the pathname and then opening and reading the file.""" try: job_pathname = self.std_crash_store.getDump(crash_id, dump_name) with open(job_pathname) as dump_file: binary = dump_file.read() return binary except OSError: raise CrashIDNotFound(crash_id) #-------------------------------------------------------------------------- def _do_get_raw_dumps(self, crash_id, crash_store): try: dumpname_paths_map = crash_store.get_dumps(crash_id) dumpname_dump_map = {} for dump_name, dump_pathname in dumpname_paths_map.iteritems(): with open(dump_pathname, 'rb') as f: dumpname_dump_map[dump_name] = f.read() return dumpname_dump_map except OSError: raise CrashIDNotFound(crash_id) #-------------------------------------------------------------------------- def get_raw_dumps(self, crash_id): """read the all the binary crash dumps from the underlying file system by getting the pathnames and then opening and reading the files. returns a dict of dump names to binary dumps""" return self._do_get_raw_dumps(crash_id, self.std_crash_store) #-------------------------------------------------------------------------- def get_raw_dumps_as_files(self, crash_id): """read the all the binary crash dumps from the underlying file system by getting the pathnames and then opening and reading the files. returns a dict of dump names to binary dumps""" return self.std_crash_store.get_dumps(crash_id) #-------------------------------------------------------------------------- def new_crashes(self): """return an iterator that yields a list of crash_ids of raw crashes that were added to the file system since the last time this iterator was requested.""" # why is this called 'destructiveDateWalk'? The underlying code # that manages the filesystem uses a tree of radix date directories # and symbolic links to track "new" raw crashes. As the the crash_ids # are fetched from the file system, the symbolic links are removed and # directories are deleted. Essentially, the state of what is # considered to be new is saved within the file system by those links. return self.std_crash_store.destructiveDateWalk() #-------------------------------------------------------------------------- def remove(self, crash_id): """delegate removal of a raw crash to the underlying filesystem""" try: self.std_crash_store.remove(crash_id) except NoSuchUuidFound: raise CrashIDNotFound(crash_id)