def __init__(self, config, quit_check_callback=None):
super(FileSystemRawCrashStorage, self).__init__(config)
self.std_crash_store = JsonDumpStorage(
root=config.std_fs_root,
maxDirectoryEntries=config.dump_dir_count,
jsonSuffix=config.json_file_suffix,
dumpSuffix=config.dump_file_suffix,
dumpGID=config.dump_gid,
dumpPermissions=config.dump_permissions,
dirPermissions=config.dir_permissions,
logger=config.logger)
self.hostname = os.uname()[1]
def __init__(self, config, quit_check_callback=None):
super(FileSystemRawCrashStorage, self).__init__(config)
self.std_crash_store = JsonDumpStorage(
root=config.std_fs_root,
maxDirectoryEntries=config.dump_dir_count,
jsonSuffix=config.json_file_suffix,
dumpSuffix=config.dump_file_suffix,
dumpGID=config.dump_gid,
dumpPermissions=config.dump_permissions,
dirPermissions=config.dir_permissions,
logger=config.logger
)
self.hostname = os.uname()[1]
class FileSystemRawCrashStorage(CrashStorageBase):
"""This crash storage class impements only the raw crash part of the
api. Raw crashes (the json file and the binary dump) are stored in a
file system. This class is appropriate for fast storage of crashes into
a local file system. In 2011, a varient of this code base was adopted
by the Socorro Collector for fast temporary storage as crashes came in."""
required_config = Namespace()
required_config.add_option(
'std_fs_root',
doc='a path to a local file system',
default='./primaryCrashStore',
reference_value_from='resource.filesystem',
)
required_config.add_option(
'dump_dir_count',
doc='the number of dumps to be stored in a single directory in the '
'local file system',
default=1024,
reference_value_from='resource.filesystem',
)
required_config.add_option(
'dump_gid',
doc='the group ID for saved crashes in local file system (optional)',
default='',
reference_value_from='resource.filesystem',
)
required_config.add_option(
'dump_permissions',
doc='a number used for permissions crash dump files in the local '
'file system',
default=stat.S_IRGRP | stat.S_IWGRP | stat.S_IRUSR | stat.S_IWUSR,
reference_value_from='resource.filesystem',
)
required_config.add_option(
'dir_permissions',
doc='a number used for permissions for directories in the local '
'file system',
default=(stat.S_IRGRP | stat.S_IXGRP | stat.S_IWGRP | stat.S_IRUSR
| stat.S_IXUSR | stat.S_IWUSR),
reference_value_from='resource.filesystem',
)
required_config.add_option(
'json_file_suffix',
doc='the suffix used to identify a json file',
default='.json',
reference_value_from='resource.filesystem',
)
required_config.add_option(
'dump_file_suffix',
doc='the suffix used to identify a dump file',
default='.dump',
reference_value_from='resource.filesystem',
)
#--------------------------------------------------------------------------
def __init__(self, config, quit_check_callback=None):
super(FileSystemRawCrashStorage, self).__init__(config)
self.std_crash_store = JsonDumpStorage(
root=config.std_fs_root,
maxDirectoryEntries=config.dump_dir_count,
jsonSuffix=config.json_file_suffix,
dumpSuffix=config.dump_file_suffix,
dumpGID=config.dump_gid,
dumpPermissions=config.dump_permissions,
dirPermissions=config.dir_permissions,
logger=config.logger
)
self.hostname = os.uname()[1]
#--------------------------------------------------------------------------
def _load_raw_crash_from_file(self, pathname):
with open(pathname) as json_file:
raw_crash = json.load(json_file, object_hook=DotDict)
return raw_crash
#--------------------------------------------------------------------------
def _do_save_raw(self,
json_storage_system,
raw_crash,
dumps,
crash_id):
json_storage_system.new_entry(
crash_id,
raw_crash,
dumps,
self.hostname
)
#--------------------------------------------------------------------------
def save_raw_crash(self, raw_crash, dumps, crash_id):
"""forward the raw_crash and the dump to the underlying file system"""
self._do_save_raw(self.std_crash_store, raw_crash, dumps, crash_id)
def save_raw_and_processed(self, raw_crash, dumps, processed_crash, crash_id):
""" bug 866973 - do not try to save dumps=None into the Filesystem
We are doing this in lieu of a queuing solution that could allow
us to operate an independent crashmover. When the queuing system
is implemented, we could remove this, and have the raw crash
saved by a crashmover that's consuming crash_ids the same way
that the processor consumes them.
Even though it is ok to resave the raw_crash in this case to the
filesystem, the fs does not know what to do with a dumps=None
when passed to save_raw, so we are going to avoid that.
"""
self.save_processed(processed_crash)
#--------------------------------------------------------------------------
def get_raw_crash(self, crash_id):
"""fetch the raw crash from the underlying file system"""
try:
pathname = self.std_crash_store.getJson(crash_id)
return self._load_raw_crash_from_file(pathname)
except OSError:
raise CrashIDNotFound(crash_id)
except ValueError: # empty json file?
return DotDict()
#--------------------------------------------------------------------------
def get_raw_dump(self, crash_id, dump_name=None):
"""read the binary crash dump from the underlying file system by
getting the pathname and then opening and reading the file."""
try:
job_pathname = self.std_crash_store.getDump(crash_id, dump_name)
with open(job_pathname) as dump_file:
binary = dump_file.read()
return binary
except OSError:
raise CrashIDNotFound(crash_id)
#--------------------------------------------------------------------------
def _do_get_raw_dumps(self, crash_id, crash_store):
try:
dumpname_paths_map = crash_store.get_dumps(crash_id)
dumpname_dump_map = {}
for dump_name, dump_pathname in dumpname_paths_map.iteritems():
with open(dump_pathname, 'rb') as f:
dumpname_dump_map[dump_name] = f.read()
return dumpname_dump_map
except OSError:
raise CrashIDNotFound(crash_id)
#--------------------------------------------------------------------------
def get_raw_dumps(self, crash_id):
"""read the all the binary crash dumps from the underlying file system
by getting the pathnames and then opening and reading the files.
returns a dict of dump names to binary dumps"""
return self._do_get_raw_dumps(crash_id, self.std_crash_store)
#--------------------------------------------------------------------------
def get_raw_dumps_as_files(self, crash_id):
"""read the all the binary crash dumps from the underlying file system
by getting the pathnames and then opening and reading the files.
returns a dict of dump names to binary dumps"""
return self.std_crash_store.get_dumps(crash_id)
#--------------------------------------------------------------------------
def new_crashes(self):
"""return an iterator that yields a list of crash_ids of raw crashes
that were added to the file system since the last time this iterator
was requested."""
# why is this called 'destructiveDateWalk'? The underlying code
# that manages the filesystem uses a tree of radix date directories
# and symbolic links to track "new" raw crashes. As the the crash_ids
# are fetched from the file system, the symbolic links are removed and
# directories are deleted. Essentially, the state of what is
# considered to be new is saved within the file system by those links.
return self.std_crash_store.destructiveDateWalk()
#--------------------------------------------------------------------------
def remove(self, crash_id):
"""delegate removal of a raw crash to the underlying filesystem"""
try:
self.std_crash_store.quickDelete(crash_id)
except NoSuchUuidFound:
raise CrashIDNotFound(crash_id)
class FileSystemRawCrashStorage(CrashStorageBase):
"""This crash storage class impements only the raw crash part of the
api. Raw crashes (the json file and the binary dump) are stored in a
file system. This class is appropriate for fast storage of crashes into
a local file system. In 2011, a varient of this code base was adopted
by the Socorro Collector for fast temporary storage as crashes came in."""
required_config = Namespace()
required_config.add_option(
"std_fs_root", doc="a path to a local file system", default="/home/socorro/primaryCrashStore"
)
required_config.add_option(
"dump_dir_count",
doc="the number of dumps to be stored in a single directory in the " "local file system",
default=1024,
)
required_config.add_option(
"dump_gid", doc="the group ID for saved crashes in local file system (optional)", default=""
)
required_config.add_option(
"dump_permissions",
doc="a number used for permissions crash dump files in the local " "file system",
default=stat.S_IRGRP | stat.S_IWGRP | stat.S_IRUSR | stat.S_IWUSR,
)
required_config.add_option(
"dir_permissions",
doc="a number used for permissions for directories in the local " "file system",
default=(stat.S_IRGRP | stat.S_IXGRP | stat.S_IWGRP | stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR),
)
required_config.add_option("json_file_suffix", doc="the suffix used to identify a json file", default=".json")
required_config.add_option("dump_file_suffix", doc="the suffix used to identify a dump file", default=".dump")
# --------------------------------------------------------------------------
def __init__(self, config, quit_check_callback=None):
super(FileSystemRawCrashStorage, self).__init__(config)
self.std_crash_store = JsonDumpStorage(
root=config.std_fs_root,
maxDirectoryEntries=config.dump_dir_count,
jsonSuffix=config.json_file_suffix,
dumpSuffix=config.dump_file_suffix,
dumpGID=config.dump_gid,
dumpPermissions=config.dump_permissions,
dirPermissions=config.dir_permissions,
logger=config.logger,
)
self.hostname = os.uname()[1]
# --------------------------------------------------------------------------
def _load_raw_crash_from_file(self, pathname):
with open(pathname) as json_file:
raw_crash = json.load(json_file, object_hook=DotDict)
return raw_crash
# --------------------------------------------------------------------------
def _do_save_raw(self, json_storage_system, raw_crash, dump, crash_id):
try:
json_file_handle, dump_file_handle = json_storage_system.newEntry(
crash_id, self.hostname # from base class
)
try:
dump_file_handle.write(dump)
json.dump(raw_crash, json_file_handle)
finally:
dump_file_handle.close()
json_file_handle.close()
self.logger.debug("saved - %s", crash_id)
except Exception:
self.logger.critical("storage has failed for: %s", crash_id, exc_info=True)
raise
# --------------------------------------------------------------------------
def save_raw_crash(self, raw_crash, dump, crash_id):
"""forward the raw_crash and the dump to the underlying file system"""
self._do_save_raw(self.std_crash_store, raw_crash, dump, crash_id)
# --------------------------------------------------------------------------
def get_raw_crash(self, crash_id):
"""fetch the raw crash from the underlying file system"""
try:
pathname = self.std_crash_store.getJson(crash_id)
return self._load_raw_crash_from_file(pathname)
except OSError:
raise CrashIDNotFound(crash_id)
except ValueError: # empty json file?
return DotDict()
# --------------------------------------------------------------------------
def get_raw_dump(self, crash_id):
"""read the binary crash dump from the underlying file system by
getting the pathname and then opening and reading the file."""
try:
job_pathname = self.std_crash_store.getDump(crash_id)
with open(job_pathname) as dump_file:
binary = dump_file.read()
return binary
except OSError:
raise CrashIDNotFound(crash_id)
# --------------------------------------------------------------------------
def new_crashes(self):
"""return an iterator that yields a list of crash_ids of raw crashes
that were added to the file system since the last time this iterator
was requested."""
# why is this called 'destructiveDateWalk'? The underlying code
# that manages the filesystem uses a tree of radix date directories
# and symbolic links to track "new" raw crashes. As the the crash_ids
# are fetched from the file system, the symbolic links are removed and
# directories are deleted. Essentially, the state of what is
# considered to be new is saved within the file system by those links.
return self.std_crash_store.destructiveDateWalk()
# --------------------------------------------------------------------------
def remove(self, crash_id):
"""delegate removal of a raw crash to the underlying filesystem"""
try:
self.std_crash_store.remove(crash_id)
except NoSuchUuidFound:
raise CrashIDNotFound(crash_id)
class FileSystemRawCrashStorage(CrashStorageBase):
"""This crash storage class impements only the raw crash part of the
api. Raw crashes (the json file and the binary dump) are stored in a
file system. This class is appropriate for fast storage of crashes into
a local file system. In 2011, a varient of this code base was adopted
by the Socorro Collector for fast temporary storage as crashes came in."""
required_config = Namespace()
required_config.add_option('std_fs_root',
doc='a path to a local file system',
default='./primaryCrashStore')
required_config.add_option(
'dump_dir_count',
doc='the number of dumps to be stored in a single directory in the '
'local file system',
default=1024)
required_config.add_option(
'dump_gid',
doc='the group ID for saved crashes in local file system (optional)',
default='')
required_config.add_option(
'dump_permissions',
doc='a number used for permissions crash dump files in the local '
'file system',
default=stat.S_IRGRP | stat.S_IWGRP | stat.S_IRUSR | stat.S_IWUSR)
required_config.add_option(
'dir_permissions',
doc='a number used for permissions for directories in the local '
'file system',
default=(stat.S_IRGRP | stat.S_IXGRP | stat.S_IWGRP | stat.S_IRUSR
| stat.S_IXUSR | stat.S_IWUSR))
required_config.add_option('json_file_suffix',
doc='the suffix used to identify a json file',
default='.json')
required_config.add_option('dump_file_suffix',
doc='the suffix used to identify a dump file',
default='.dump')
#--------------------------------------------------------------------------
def __init__(self, config, quit_check_callback=None):
super(FileSystemRawCrashStorage, self).__init__(config)
self.std_crash_store = JsonDumpStorage(
root=config.std_fs_root,
maxDirectoryEntries=config.dump_dir_count,
jsonSuffix=config.json_file_suffix,
dumpSuffix=config.dump_file_suffix,
dumpGID=config.dump_gid,
dumpPermissions=config.dump_permissions,
dirPermissions=config.dir_permissions,
logger=config.logger)
self.hostname = os.uname()[1]
#--------------------------------------------------------------------------
def _load_raw_crash_from_file(self, pathname):
with open(pathname) as json_file:
raw_crash = json.load(json_file, object_hook=DotDict)
return raw_crash
#--------------------------------------------------------------------------
def _do_save_raw(self, json_storage_system, raw_crash, dumps, crash_id):
json_storage_system.new_entry(crash_id, raw_crash, dumps,
self.hostname)
#--------------------------------------------------------------------------
def save_raw_crash(self, raw_crash, dumps, crash_id):
"""forward the raw_crash and the dump to the underlying file system"""
self._do_save_raw(self.std_crash_store, raw_crash, dumps, crash_id)
#--------------------------------------------------------------------------
def get_raw_crash(self, crash_id):
"""fetch the raw crash from the underlying file system"""
try:
pathname = self.std_crash_store.getJson(crash_id)
return self._load_raw_crash_from_file(pathname)
except OSError:
raise CrashIDNotFound(crash_id)
except ValueError: # empty json file?
return DotDict()
#--------------------------------------------------------------------------
def get_raw_dump(self, crash_id, dump_name=None):
"""read the binary crash dump from the underlying file system by
getting the pathname and then opening and reading the file."""
try:
job_pathname = self.std_crash_store.getDump(crash_id, dump_name)
with open(job_pathname) as dump_file:
binary = dump_file.read()
return binary
except OSError:
raise CrashIDNotFound(crash_id)
#--------------------------------------------------------------------------
def _do_get_raw_dumps(self, crash_id, crash_store):
try:
dumpname_paths_map = crash_store.get_dumps(crash_id)
dumpname_dump_map = {}
for dump_name, dump_pathname in dumpname_paths_map.iteritems():
with open(dump_pathname, 'rb') as f:
dumpname_dump_map[dump_name] = f.read()
return dumpname_dump_map
except OSError:
raise CrashIDNotFound(crash_id)
#--------------------------------------------------------------------------
def get_raw_dumps(self, crash_id):
"""read the all the binary crash dumps from the underlying file system
by getting the pathnames and then opening and reading the files.
returns a dict of dump names to binary dumps"""
return self._do_get_raw_dumps(crash_id, self.std_crash_store)
#--------------------------------------------------------------------------
def get_raw_dumps_as_files(self, crash_id):
"""read the all the binary crash dumps from the underlying file system
by getting the pathnames and then opening and reading the files.
returns a dict of dump names to binary dumps"""
return self.std_crash_store.get_dumps(crash_id)
#--------------------------------------------------------------------------
def new_crashes(self):
"""return an iterator that yields a list of crash_ids of raw crashes
that were added to the file system since the last time this iterator
was requested."""
# why is this called 'destructiveDateWalk'? The underlying code
# that manages the filesystem uses a tree of radix date directories
# and symbolic links to track "new" raw crashes. As the the crash_ids
# are fetched from the file system, the symbolic links are removed and
# directories are deleted. Essentially, the state of what is
# considered to be new is saved within the file system by those links.
return self.std_crash_store.destructiveDateWalk()
#--------------------------------------------------------------------------
def remove(self, crash_id):
"""delegate removal of a raw crash to the underlying filesystem"""
try:
self.std_crash_store.remove(crash_id)
except NoSuchUuidFound:
raise CrashIDNotFound(crash_id)
