def test_classes_in_namespaces_converter_4(self):
n = Namespace()
n.add_option('kls_list',
default='configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha',
from_string_converter=
converters.classes_in_namespaces_converter(
'kls%d',
'kls',
instantiate_classes=True))
cm = ConfigurationManager(
n,
[{'kls_list':'configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Beta, '
'configman.tests.test_converters.Beta, '
'configman.tests.test_converters.Alpha'}])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4)
for x in config.kls_list.subordinate_namespace_names:
self.assertTrue(x in config)
self.assertTrue('kls_instance' in config[x])
self.assertTrue(isinstance(config[x].kls_instance,
config[x].kls))
def test_classes_in_namespaces_converter_4(self):
n = Namespace()
n.add_option('kls_list',
default='configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha',
from_string_converter=
converters.classes_in_namespaces_converter(
'kls%d',
'kls',
instantiate_classes=True))
cm = ConfigurationManager(
n,
[{'kls_list':'configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Beta, '
'configman.tests.test_converters.Beta, '
'configman.tests.test_converters.Alpha'}])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4)
for x in config.kls_list.subordinate_namespace_names:
self.assertTrue(x in config)
self.assertTrue('kls_instance' in config[x])
self.assertTrue(isinstance(config[x].kls_instance,
config[x].kls))
def test_classes_in_namespaces_converter_4(self):
n = Namespace()
n.add_option(
"kls_list",
default="configman.tests.test_converters.Alpha, "
"configman.tests.test_converters.Alpha, "
"configman.tests.test_converters.Alpha",
from_string_converter=converters.classes_in_namespaces_converter("kls%d", "kls", instantiate_classes=True),
)
cm = ConfigurationManager(
n,
[
{
"kls_list": "configman.tests.test_converters.Alpha, "
"configman.tests.test_converters.Beta, "
"configman.tests.test_converters.Beta, "
"configman.tests.test_converters.Alpha"
}
],
)
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4)
for x in config.kls_list.subordinate_namespace_names:
self.assertTrue(x in config)
self.assertTrue("kls_instance" in config[x])
self.assertTrue(isinstance(config[x].kls_instance, config[x].kls))
def test_classes_in_namespaces_converter_3(self):
n = Namespace()
n.add_option('kls_list',
default='configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha',
from_string_converter=
converters.classes_in_namespaces_converter('kls%d'))
cm = ConfigurationManager(n, argv_source=[])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 3)
for x in config.kls_list.subordinate_namespace_names:
self.assertTrue(x in config)
self.assertEqual(config[x].cls, Alpha)
self.assertTrue('cls_instance' not in config[x])
class RadixCleanupCronApp(BaseCronApp):
app_name = 'cleanup_radix'
app_description = 'Cleans up dead radix directories'
required_config = Namespace()
required_config.add_option(
'dated_storage_classes',
doc='a comma delimited list of storage classes',
default='',
from_string_converter=classes_in_namespaces_converter(
template_for_namespace='storage%d',
name_of_class_option='crashstorage_class',
instantiate_classes=False, # we instantiate manually for thread
# safety
))
def __init__(self, config, *args, **kwargs):
super(RadixCleanupCronApp, self).__init__(config, *args, **kwargs)
self.storage_namespaces = \
config.dated_storage_classes.subordinate_namespace_names
self.stores = DotDict()
for a_namespace in self.storage_namespaces:
self.stores[a_namespace] = \
config[a_namespace].crashstorage_class(config[a_namespace])
def run(self):
today = utc_now().strftime("%Y%m%d")
for storage in self.stores.values():
for date in os.listdir(storage.config.fs_root):
if date >= today:
continue # don't process today's crashes or any crashes
# from the future
if os.listdir(
os.sep.join([
storage.config.fs_root, date,
storage.config.date_branch_base
])):
self.config.logger.error(
"Could not delete crashes for "
"date %s: branch isn't empty", date)
continue # if the date branch isn't empty, then it's not
# safe to nuke
shutil.rmtree(os.sep.join([storage.config.fs_root, date]))
def test_classes_in_namespaces_converter_3(self):
n = Namespace()
n.add_option('kls_list',
default='configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha, '
'configman.tests.test_converters.Alpha',
from_string_converter=
converters.classes_in_namespaces_converter('kls%d'))
cm = ConfigurationManager(n, argv_source=[])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 3)
for x in config.kls_list.subordinate_namespace_names:
self.assertTrue(x in config)
self.assertEqual(config[x].cls, Alpha)
self.assertTrue('cls_instance' not in config[x])
def test_classes_in_namespaces_converter_1(self):
converter_fn = converters.classes_in_namespaces_converter("HH%d")
class_list_str = "configman.tests.test_converters.Foo," "configman.tests.test_converters.Bar"
result = converter_fn(class_list_str)
self.assertTrue(hasattr(result, "required_config"))
req = result.required_config
self.assertEqual(len(req), 2)
self.assertTrue("HH0" in req)
self.assertEqual(len(req.HH0), 1)
self.assertTrue("cls" in req.HH0)
self.assertTrue("HH1" in req)
self.assertEqual(len(req.HH1), 1)
self.assertTrue("cls" in req.HH1)
self.assertEqual(
sorted([x.strip() for x in class_list_str.split(",")]),
sorted([x.strip() for x in converters.py_obj_to_str(result).split(",")]),
)
def test_classes_in_namespaces_converter_1(self):
converter_fn = converters.classes_in_namespaces_converter('HH%d')
class_list_str = ('configman.tests.test_converters.Foo,'
'configman.tests.test_converters.Bar')
result = converter_fn(class_list_str)
self.assertTrue(hasattr(result, 'required_config'))
req = result.required_config
self.assertEqual(len(req), 2)
self.assertTrue('HH0' in req)
self.assertEqual(len(req.HH0), 1)
self.assertTrue('cls' in req.HH0)
self.assertTrue('HH1' in req)
self.assertEqual(len(req.HH1), 1)
self.assertTrue('cls' in req.HH1)
self.assertEqual(
sorted([x.strip() for x in class_list_str.split(',')]),
sorted([x.strip() for x in
converters.py_obj_to_str(result).split(',')]))
def test_classes_in_namespaces_converter_1(self):
converter_fn = converters.classes_in_namespaces_converter('HH%d')
class_list_str = ('configman.tests.test_converters.Foo,'
'configman.tests.test_converters.Bar')
result = converter_fn(class_list_str)
self.assertTrue(hasattr(result, 'required_config'))
req = result.required_config
self.assertEqual(len(req), 2)
self.assertTrue('HH0' in req)
self.assertEqual(len(req.HH0), 1)
self.assertTrue('cls' in req.HH0)
self.assertTrue('HH1' in req)
self.assertEqual(len(req.HH1), 1)
self.assertTrue('cls' in req.HH1)
self.assertEqual(
sorted([x.strip() for x in class_list_str.split(',')]),
sorted([x.strip() for x in
converters.py_obj_to_str(result).split(',')]))
class PolyCrashStorage(CrashStorageBase):
"""a crashstorage implementation that encapsulates a collection of other
crashstorage instances. Any save operation applied to an instance of this
class will be applied to all the crashstorge in the collection.
This class is useful for 'save' operations only. It does not implement
the 'get' operations.
The contained crashstorage instances are specified in the configuration.
Each class specified in the 'storage_classes' config option will be given
its own numbered namespace in the form 'storage%d'. With in the namespace,
the class itself will be referred to as just 'store'. Any configuration
requirements within the class 'store' will be isolated within the local
namespace. That allows multiple instances of the same storageclass to
avoid name collisions.
"""
required_config = Namespace()
required_config.add_option(
'storage_classes',
doc='a comma delimited list of storage classes',
default='',
from_string_converter=classes_in_namespaces_converter(
template_for_namespace='storage%d',
name_of_class_option='crashstorage_class',
instantiate_classes=False, # we instantiate manually for thread
# safety
),
likely_to_be_changed=True,
)
#--------------------------------------------------------------------------
def __init__(self, config, quit_check_callback=None):
"""instantiate all the subordinate crashstorage instances
parameters:
config - a configman dot dict holding configuration information
quit_check_callback - a function to be called periodically during
long running operations.
instance variables:
self.storage_namespaces - the list of the namespaces inwhich the
subordinate instances are stored.
self.stores - instances of the subordinate crash stores
"""
super(PolyCrashStorage, self).__init__(config, quit_check_callback)
self.storage_namespaces = \
config.storage_classes.subordinate_namespace_names
self.stores = DotDict()
for a_namespace in self.storage_namespaces:
self.stores[a_namespace] = \
config[a_namespace].crashstorage_class(
config[a_namespace],
quit_check_callback
)
#--------------------------------------------------------------------------
def close(self):
"""iterate through the subordinate crash stores and close them.
Even though the classes are closed in sequential order, all are
assured to close even if an earlier one raises an exception. When all
are closed, any exceptions that were raised are reraised in a
PolyStorageError
raises:
PolyStorageError - an exception container holding a list of the
exceptions raised by the subordinate storage
systems"""
storage_exception = PolyStorageError()
for a_store in self.stores.itervalues():
try:
a_store.close()
except Exception, x:
self.logger.error('%s failure: %s', a_store.__class__, str(x))
storage_exception.gather_current_exception()
if storage_exception.has_exceptions():
raise storage_exception
class PolyCrashStorage(CrashStorageBase):
"""a crashstorage implementation that encapsulates a collection of other
crashstorage instances. Any save operation applied to an instance of this
class will be applied to all the crashstorge in the collection.
This class is useful for 'save' operations only. It does not implement
the 'get' operations.
The contained crashstorage instances are specified in the configuration.
Each class specified in the 'storage_classes' config option will be given
its own numbered namespace in the form 'storage%d'. With in the namespace,
the class itself will be referred to as just 'store'. Any configuration
requirements within the class 'store' will be isolated within the local
namespace. That allows multiple instances of the same storageclass to
avoid name collisions.
"""
required_config = Namespace()
required_config.add_option(
'storage_classes',
doc='a comma delimited list of storage classes',
default='',
from_string_converter=classes_in_namespaces_converter(
template_for_namespace='storage%d',
name_of_class_option='crashstorage_class',
# we instantiate manually for thread safety
instantiate_classes=False,
),
likely_to_be_changed=True,
)
def __init__(self, config, quit_check_callback=None):
"""instantiate all the subordinate crashstorage instances
parameters:
config - a configman dot dict holding configuration information
quit_check_callback - a function to be called periodically during
long running operations.
instance variables:
self.storage_namespaces - the list of the namespaces inwhich the
subordinate instances are stored.
self.stores - instances of the subordinate crash stores
"""
super(PolyCrashStorage, self).__init__(config, quit_check_callback)
self.storage_namespaces = config.storage_classes.subordinate_namespace_names
self.stores = ConfigmanDotDict()
for a_namespace in self.storage_namespaces:
self.stores[a_namespace] = config[a_namespace].crashstorage_class(
config[a_namespace],
quit_check_callback
)
def close(self):
"""iterate through the subordinate crash stores and close them.
Even though the classes are closed in sequential order, all are
assured to close even if an earlier one raises an exception. When all
are closed, any exceptions that were raised are reraised in a
PolyStorageError
raises:
PolyStorageError - an exception container holding a list of the
exceptions raised by the subordinate storage
systems"""
storage_exception = PolyStorageError()
for a_store in self.stores.itervalues():
try:
a_store.close()
except Exception as x:
self.logger.error('%s failure: %s', a_store.__class__,
str(x))
storage_exception.gather_current_exception()
if storage_exception.has_exceptions():
raise storage_exception
def save_raw_crash(self, raw_crash, dumps, crash_id):
"""iterate through the subordinate crash stores saving the raw_crash
and the dump to each of them.
parameters:
raw_crash - the meta data mapping
dumps - a mapping of dump name keys to dump binary values
crash_id - the id of the crash to use"""
storage_exception = PolyStorageError()
for a_store in self.stores.itervalues():
self.quit_check()
try:
a_store.save_raw_crash(raw_crash, dumps, crash_id)
except Exception as x:
self.logger.error('%s failure: %s', a_store.__class__,
str(x))
storage_exception.gather_current_exception()
if storage_exception.has_exceptions():
raise storage_exception
def save_processed(self, processed_crash):
"""iterate through the subordinate crash stores saving the
processed_crash to each of the.
parameters:
processed_crash - a mapping containing the processed crash"""
storage_exception = PolyStorageError()
for a_store in self.stores.itervalues():
self.quit_check()
try:
a_store.save_processed(processed_crash)
except Exception as x:
self.logger.error('%s failure: %s', a_store.__class__,
str(x), exc_info=True)
storage_exception.gather_current_exception()
if storage_exception.has_exceptions():
raise storage_exception
def save_raw_and_processed(self, raw_crash, dump, processed_crash,
crash_id):
storage_exception = PolyStorageError()
# Later we're going to need to clone this per every crash storage
# in the loop. But, to save time, before we do that, convert the
# processed crash which is a SocorroDotDict into a pure python
# dict which we can more easily copy.deepcopy() operate on.
processed_crash_as_dict = socorrodotdict_to_dict(processed_crash)
raw_crash_as_dict = socorrodotdict_to_dict(raw_crash)
for a_store in self.stores.itervalues():
self.quit_check()
try:
actual_store = getattr(a_store, 'wrapped_object', a_store)
if hasattr(actual_store, 'is_mutator') and actual_store.is_mutator():
# We do this because `a_store.save_raw_and_processed`
# expects the processed crash to be a DotDict but
# you can't deepcopy those, so we deepcopy the
# pure dict version and then dress it back up as a
# DotDict.
my_processed_crash = SocorroDotDict(
copy.deepcopy(processed_crash_as_dict)
)
my_raw_crash = SocorroDotDict(
copy.deepcopy(raw_crash_as_dict)
)
else:
my_processed_crash = processed_crash
my_raw_crash = raw_crash
a_store.save_raw_and_processed(
my_raw_crash,
dump,
my_processed_crash,
crash_id
)
except Exception:
store_class = getattr(
a_store, 'wrapped_object', a_store.__class__
)
self.logger.error(
'%r failed (crash id: %s)',
store_class,
crash_id,
exc_info=True
)
storage_exception.gather_current_exception()
if storage_exception.has_exceptions():
raise storage_exception
def test_classes_in_namespaces_converter_2(self):
converter_fn = converters.classes_in_namespaces_converter('HH%d')
class_sequence = (Foo, Bar)
self.assertRaises(TypeError, converter_fn, class_sequence)
def test_classes_in_namespaces_converter_2(self):
converter_fn = converters.classes_in_namespaces_converter('HH%d')
class_sequence = (Foo, Bar)
self.assertRaises(TypeError, converter_fn, class_sequence)
