def __init__(self, config, quit_check_callback=None):
super(FileSystemCrashStorage, self).__init__(config)
self.pro_crash_store = ProcessedDumpStorage(
root=config.pro_fs_root,
minutesPerSlot=config.minutes_per_slot,
subSlotCount=config.sub_slot_count,
indexName=config.index_name,
dateName=config.date_name,
fileSuffix=config.processed_crash_file_suffix,
gzipCompression=config.gzip_compression_level,
storageDepth=config.storage_depth,
dumpGID=config.dump_gid,
dumpPermissions=config.dump_permissions,
dirPermissions=config.dir_permissions,
)
def __init__(self, config, quit_check_callback=None):
super(FileSystemCrashStorage, self).__init__(config)
self.pro_crash_store = ProcessedDumpStorage(
root=config.pro_fs_root,
minutesPerSlot=config.minutes_per_slot,
subSlotCount=config.sub_slot_count,
indexName=config.index_name,
dateName=config.date_name,
fileSuffix=config.processed_crash_file_suffix,
gzipCompression=config.gzip_compression_level,
storageDepth=config.storage_depth,
dumpGID=config.dump_gid,
dumpPermissions=config.dump_permissions,
dirPermissions=config.dir_permissions,
)
class FileSystemCrashStorage(FileSystemThrottledCrashStorage):
"""This storage class is the only file system based crash storage system
appropriate for storing both raw and processed crashes. This class uses
the same segregating raw crash storage as the previous class and adds
processed storage. Processed crashes are stored in their own file system
root, 'pro_fs_root' (processed file system root) using the same radix
directory system as the raw crashes."""
required_config = Namespace()
required_config.add_option(
'pro_fs_root',
doc='a path to a local file system for processed storage',
default='./processedCrashStore',
reference_value_from='resource.filesystem',
)
required_config.add_option(
'minutes_per_slot',
doc='the number of minutes in the lowest date directory',
default=1,
reference_value_from='resource.filesystem',
)
required_config.add_option(
'sub_slot_count',
doc='distribute data evenly among this many sub timeslots',
default=1,
reference_value_from='resource.filesystem',
)
required_config.add_option(
'index_name',
doc='the relative path to the top of the name storage tree from '
'root parameter',
default='name',
reference_value_from='resource.filesystem',
)
required_config.add_option(
'date_name',
doc='the relative path to the top of the date storage tree from '
'root parameter',
default='date',
reference_value_from='resource.filesystem',
)
required_config.add_option(
'processed_crash_file_suffix',
doc='the processed crash filename suffix',
default='.jsonz',
reference_value_from='resource.filesystem',
)
required_config.add_option(
'gzip_compression_level',
doc='the level of compression to use',
default=9,
reference_value_from='resource.filesystem',
)
required_config.add_option(
'storage_depth',
doc='the length of branches in the radix storage tree',
default=2,
reference_value_from='resource.filesystem',
)
#--------------------------------------------------------------------------
def __init__(self, config, quit_check_callback=None):
super(FileSystemCrashStorage, self).__init__(config)
self.pro_crash_store = ProcessedDumpStorage(
root=config.pro_fs_root,
minutesPerSlot=config.minutes_per_slot,
subSlotCount=config.sub_slot_count,
indexName=config.index_name,
dateName=config.date_name,
fileSuffix=config.processed_crash_file_suffix,
gzipCompression=config.gzip_compression_level,
storageDepth=config.storage_depth,
dumpGID=config.dump_gid,
dumpPermissions=config.dump_permissions,
dirPermissions=config.dir_permissions,
)
#--------------------------------------------------------------------------
def save_processed(self, processed_crash):
"""save a processed crash (in the form of a Mapping) into a json
file. It first gets the underlying file system to give it a file
handle open for writing, then it uses the 'json' module to write
the mapping to the open file handle."""
try:
crash_id = processed_crash['uuid']
except KeyError:
raise CrashIDNotFound("uuid missing from processed_crash")
try:
self._stringify_dates_in_dict(processed_crash)
processed_crash_file_handle = \
self.pro_crash_store.newEntry(crash_id)
try:
json.dump(processed_crash, processed_crash_file_handle)
finally:
processed_crash_file_handle.close()
self.logger.debug('saved processed- %s', crash_id)
except Exception:
self.logger.critical(
'processed file system storage has failed for: %s',
crash_id,
exc_info=True
)
raise
#--------------------------------------------------------------------------
def get_unredacted_processed(self, crash_id):
"""fetch a processed json file from the underlying file system"""
try:
return self.pro_crash_store.getDumpFromFile(crash_id)
except OSError:
raise CrashIDNotFound(crash_id)
#--------------------------------------------------------------------------
def remove(self, crash_id):
"""remove the all traces of a crash, both raw and processed from the
file system."""
try:
super(FileSystemCrashStorage, self).remove(crash_id)
except CrashIDNotFound:
self.logger.warning(
'raw crash not found for deletion: %s',
crash_id
)
try:
self.pro_crash_store.removeDumpFile(crash_id)
except OSError:
self.logger.warning('processed crash not found for deletion: %s',
crash_id)
#--------------------------------------------------------------------------
@staticmethod
def _stringify_dates_in_dict(a_dict):
for name, value in a_dict.iteritems():
if isinstance(value, datetime.datetime):
a_dict[name] = ("%4d-%02d-%02d %02d:%02d:%02d.%d" %
(value.year,
value.month,
value.day,
value.hour,
value.minute,
value.second,
value.microsecond)
)
class FileSystemCrashStorage(FileSystemThrottledCrashStorage):
"""This storage class is the only file system based crash storage system
appropriate for storing both raw and processed crashes. This class uses
the same segregating raw crash storage as the previous class and adds
processed storage. Processed crashes are stored in their own file system
root, 'pro_fs_root' (processed file system root) using the same radix
directory system as the raw crashes."""
required_config = Namespace()
required_config.add_option(
"pro_fs_root",
doc="a path to a local file system for processed storage",
default="/home/socorro/processedCrashStore",
)
required_config.add_option("minutes_per_slot", doc="the number of minutes in the lowest date directory", default=1)
required_config.add_option("sub_slot_count", doc="distribute data evenly among this many sub timeslots", default=1)
required_config.add_option(
"index_name", doc="the relative path to the top of the name storage tree from " "root parameter", default="name"
)
required_config.add_option(
"date_name", doc="the relative path to the top of the date storage tree from " "root parameter", default="date"
)
required_config.add_option(
"processed_crash_file_suffix", doc="the processed crash filename suffix", default=".jsonz"
)
required_config.add_option("gzip_compression_level", doc="the level of compression to use", default=9)
required_config.add_option("storage_depth", doc="the length of branches in the radix storage tree", default=2)
required_config.add_option(
"forbidden_keys",
doc="a comma delimited list of keys to not allowed in the processed " "crash",
default="url, email, user_id",
from_string_converter=lambda x: [i.strip() for i in x.split(",")],
)
# --------------------------------------------------------------------------
def __init__(self, config, quit_check_callback=None):
super(FileSystemCrashStorage, self).__init__(config)
self.pro_crash_store = ProcessedDumpStorage(
root=config.pro_fs_root,
minutesPerSlot=config.minutes_per_slot,
subSlotCount=config.sub_slot_count,
indexName=config.index_name,
dateName=config.date_name,
fileSuffix=config.processed_crash_file_suffix,
gzipCompression=config.gzip_compression_level,
storageDepth=config.storage_depth,
dumpGID=config.dump_gid,
dumpPermissions=config.dump_permissions,
dirPermissions=config.dir_permissions,
)
# --------------------------------------------------------------------------
def save_processed(self, processed_crash):
"""save a processed crash (in the form of a Mapping) into a json
file. It first gets the underlying file system to give it a file
handle open for writing, then it uses the 'json' module to write
the mapping to the open file handle."""
try:
crash_id = processed_crash["uuid"]
except KeyError:
raise CrashIDNotFound("uuid missing from processed_crash")
try:
processed_crash = self.sanitize_processed_crash(processed_crash, self.config.forbidden_keys)
self._stringify_dates_in_dict(processed_crash)
processed_crash_file_handle = self.pro_crash_store.newEntry(crash_id)
try:
json.dump(processed_crash, processed_crash_file_handle)
finally:
processed_crash_file_handle.close()
self.logger.debug("saved processed- %s", crash_id)
except Exception:
self.logger.critical("processed file system storage has failed for: %s", crash_id, exc_info=True)
raise
# --------------------------------------------------------------------------
def get_processed(self, crash_id):
"""fetch a processed json file from the underlying file system"""
try:
return self.pro_crash_store.getDumpFromFile(crash_id)
except OSError:
raise CrashIDNotFound(crash_id)
# --------------------------------------------------------------------------
def remove(self, crash_id):
"""remove the all traces of a crash, both raw and processed from the
file system."""
try:
super(FileSystemCrashStorage, self).remove(crash_id)
except CrashIDNotFound:
self.logger.warning("raw crash not found for deletion: %s", crash_id)
try:
self.pro_crash_store.removeDumpFile(crash_id)
except OSError:
self.logger.warning("processed crash not found for deletion: %s", crash_id)
# --------------------------------------------------------------------------
@staticmethod
def sanitize_processed_crash(processed_crash, forbidden_keys):
"""returns a copy of a processed_crash with the forbidden keys removed.
parameters:
processed_crash - the processed crash in the form of a mapping
forbidden_keys - a list of strings to be removed from the
processed crash
returns:
a mapping that is a shallow copy of the original processed_crash
minus the forbidden keys and values"""
a_copy = processed_crash.copy()
for a_forbidden_key in forbidden_keys:
if a_forbidden_key in a_copy:
del a_copy[a_forbidden_key]
return a_copy
# --------------------------------------------------------------------------
@staticmethod
def _stringify_dates_in_dict(a_dict):
for name, value in a_dict.iteritems():
if isinstance(value, datetime.datetime):
a_dict[name] = "%4d-%02d-%02d %02d:%02d:%02d.%d" % (
value.year,
value.month,
value.day,
value.hour,
value.minute,
value.second,
value.microsecond,
)
class FileSystemCrashStorage(FileSystemThrottledCrashStorage):
"""This storage class is the only file system based crash storage system
appropriate for storing both raw and processed crashes. This class uses
the same segregating raw crash storage as the previous class and adds
processed storage. Processed crashes are stored in their own file system
root, 'pro_fs_root' (processed file system root) using the same radix
directory system as the raw crashes."""
required_config = Namespace()
required_config.add_option(
'pro_fs_root',
doc='a path to a local file system for processed storage',
default='./processedCrashStore')
required_config.add_option(
'minutes_per_slot',
doc='the number of minutes in the lowest date directory',
default=1)
required_config.add_option(
'sub_slot_count',
doc='distribute data evenly among this many sub timeslots',
default=1)
required_config.add_option(
'index_name',
doc='the relative path to the top of the name storage tree from '
'root parameter',
default='name')
required_config.add_option(
'date_name',
doc='the relative path to the top of the date storage tree from '
'root parameter',
default='date')
required_config.add_option('processed_crash_file_suffix',
doc='the processed crash filename suffix',
default='.jsonz')
required_config.add_option('gzip_compression_level',
doc='the level of compression to use',
default=9)
required_config.add_option(
'storage_depth',
doc='the length of branches in the radix storage tree',
default=2)
required_config.add_option(
'forbidden_keys',
doc='a comma delimited list of keys to not allowed in the processed '
'crash',
default='url, email, user_id, exploitability',
from_string_converter=lambda x: [i.strip() for i in x.split(',')])
#--------------------------------------------------------------------------
def __init__(self, config, quit_check_callback=None):
super(FileSystemCrashStorage, self).__init__(config)
self.pro_crash_store = ProcessedDumpStorage(
root=config.pro_fs_root,
minutesPerSlot=config.minutes_per_slot,
subSlotCount=config.sub_slot_count,
indexName=config.index_name,
dateName=config.date_name,
fileSuffix=config.processed_crash_file_suffix,
gzipCompression=config.gzip_compression_level,
storageDepth=config.storage_depth,
dumpGID=config.dump_gid,
dumpPermissions=config.dump_permissions,
dirPermissions=config.dir_permissions,
)
#--------------------------------------------------------------------------
def save_processed(self, processed_crash):
"""save a processed crash (in the form of a Mapping) into a json
file. It first gets the underlying file system to give it a file
handle open for writing, then it uses the 'json' module to write
the mapping to the open file handle."""
try:
crash_id = processed_crash['uuid']
except KeyError:
raise CrashIDNotFound("uuid missing from processed_crash")
try:
processed_crash = self.sanitize_processed_crash(
processed_crash, self.config.forbidden_keys)
self._stringify_dates_in_dict(processed_crash)
processed_crash_file_handle = \
self.pro_crash_store.newEntry(crash_id)
try:
json.dump(processed_crash, processed_crash_file_handle)
finally:
processed_crash_file_handle.close()
self.logger.debug('saved processed- %s', crash_id)
except Exception:
self.logger.critical(
'processed file system storage has failed for: %s',
crash_id,
exc_info=True)
raise
#--------------------------------------------------------------------------
def get_processed(self, crash_id):
"""fetch a processed json file from the underlying file system"""
try:
return self.pro_crash_store.getDumpFromFile(crash_id)
except OSError:
raise CrashIDNotFound(crash_id)
#--------------------------------------------------------------------------
def remove(self, crash_id):
"""remove the all traces of a crash, both raw and processed from the
file system."""
try:
super(FileSystemCrashStorage, self).remove(crash_id)
except CrashIDNotFound:
self.logger.warning('raw crash not found for deletion: %s',
crash_id)
try:
self.pro_crash_store.removeDumpFile(crash_id)
except OSError:
self.logger.warning('processed crash not found for deletion: %s',
crash_id)
#--------------------------------------------------------------------------
@staticmethod
def sanitize_processed_crash(processed_crash, forbidden_keys):
"""returns a copy of a processed_crash with the forbidden keys removed.
parameters:
processed_crash - the processed crash in the form of a mapping
forbidden_keys - a list of strings to be removed from the
processed crash
returns:
a mapping that is a shallow copy of the original processed_crash
minus the forbidden keys and values"""
a_copy = processed_crash.copy()
for a_forbidden_key in forbidden_keys:
if a_forbidden_key in a_copy:
del a_copy[a_forbidden_key]
return a_copy
#--------------------------------------------------------------------------
@staticmethod
def _stringify_dates_in_dict(a_dict):
for name, value in a_dict.iteritems():
if isinstance(value, datetime.datetime):
a_dict[name] = (
"%4d-%02d-%02d %02d:%02d:%02d.%d" %
(value.year, value.month, value.day, value.hour,
value.minute, value.second, value.microsecond))