def test_classes_in_namespaces_converter_5(self):
n = Namespace()
n.add_option(
'kls_list',
default=('socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha'),
from_string_converter=str_to_classes_in_namespaces_converter(
'%(name)s_%(index)02d'))
cm = ConfigurationManager(n, [{
'kls_list': ('socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Alpha'),
'Alpha_00.a':
21,
'Beta_01.b':
38,
}])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4)
for i, (a_class_name, a_class,
ns_name) in (enumerate(config.kls_list.class_list)):
self.assertTrue(isinstance(a_class_name, str))
self.assertEqual(a_class_name, a_class.__name__)
self.assertEqual(ns_name, "%s_%02d" % (a_class_name, i))
def test_classes_in_namespaces_converter_4(self):
n = Namespace()
n.add_option(
'kls_list',
default=(
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha'
),
from_string_converter=str_to_classes_in_namespaces_converter(
'%(name)s_%(index)02d'
)
)
cm = ConfigurationManager(
n,
[{
'kls_list': (
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Alpha'
),
'Alpha_00.a': 21,
'Beta_01.b': 38,
}]
)
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4)
for x in config.kls_list.subordinate_namespace_names:
self.assertTrue(x in config)
self.assertEqual(config.Alpha_00.a, 21)
self.assertEqual(config.Beta_01.b, 38)
def test_classes_in_namespaces_converter_2(self):
converter_fn = str_to_classes_in_namespaces_converter(
'class_%(name)s'
)
class_sequence = (Foo, Bar)
# ought to raise TypeError because 'class_sequence' is not a string
self.assertRaises(TypeError, converter_fn, class_sequence)
def test_classes_in_namespaces_converter_5(self):
n = Namespace()
n.add_option(
'kls_list',
default=(
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha'
),
from_string_converter=str_to_classes_in_namespaces_converter(
'%(name)s_%(index)02d'
)
)
cm = ConfigurationManager(
n,
[{
'kls_list': (
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Alpha'
),
'Alpha_00.a': 21,
'Beta_01.b': 38,
}]
)
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4)
for i, (a_class_name, a_class, ns_name) in \
enumerate(config.kls_list.class_list):
self.assertTrue(isinstance(a_class_name, str))
self.assertEqual(a_class_name, a_class.__name__)
self.assertEqual(ns_name, "%s_%02d" % (a_class_name, i))
def test_classes_in_namespaces_converter_4(self):
n = Namespace()
n.add_option(
'kls_list',
default=('socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Alpha'),
from_string_converter=str_to_classes_in_namespaces_converter(
'%(name)s_%(index)02d'))
cm = ConfigurationManager(n, [{
'kls_list': ('socorro.unittest.lib.test_converters.Alpha, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Beta, '
'socorro.unittest.lib.test_converters.Alpha'),
'Alpha_00.a':
21,
'Beta_01.b':
38,
}])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 4)
for x in config.kls_list.subordinate_namespace_names:
self.assertTrue(x in config)
self.assertEqual(config.Alpha_00.a, 21)
self.assertEqual(config.Beta_01.b, 38)
class TransformRuleSystem(RequiredConfig):
"""A collection of TransformRules that can be applied together"""
required_config = Namespace()
required_config.add_option(
name='rules_list',
default=[],
from_string_converter=str_to_classes_in_namespaces_converter())
required_config.add_option(
'chatty_rules',
doc='should the rules announce what they are doing?',
default=False,
)
def __init__(self, config=None, quit_check=None):
if quit_check:
self._quit_check = quit_check
else:
self._quit_check = self._null_quit_check
self.rules = []
if not config:
config = DotDict()
if 'chatty_rules' not in config:
config.chatty_rules = False
self.config = config
if "rules_list" in config:
self.tag = config.tag
self.act = getattr(self, config.action)
list_of_rules = config.rules_list.class_list
for a_rule_class_name, a_rule_class, ns_name in list_of_rules:
try:
self.rules.append(a_rule_class(config[ns_name]))
except KeyError, x:
self.rules.append(a_rule_class(config))
class ProcessorRuleSets(RequiredConfig):
# why do rules come in sets? Why not just have a big list of rules?
# rule sets are containers for rules with a similar purpose and
# execution mode. For example, there are rule sets for adjusting the
# raw_crash, transforming raw to processed, post processing the
# processed_crash and then all the different forms of classifiers.
# Some rule sets have different execution modes: run all the rules,
# run the rules until one fails, run the rules until one succeeds,
# etc.
required_config = Namespace()
names = []
for (name, tag, rule_set_class_str, action_str, default_rules_str) \
in rule_sets:
names.append(name)
required_config.namespace(name)
required_config[name].add_option(
name='tag',
doc='the lookup tag associated with this rule set',
default=tag
)
required_config[name].add_option(
name='rule_system_class',
default=rule_set_class_str,
doc='the fully qualified name of the rule system class',
from_string_converter=str_to_python_object,
likely_to_be_changed=True,
)
required_config[name].add_option(
name='action',
default=action_str,
doc=(
'the name of the rule set method to run to processes '
'these rules'
),
likely_to_be_changed=True,
)
required_config[name].add_option(
name='rules_list',
doc='a list of fully qualified class names for the rules',
default=default_rules_str,
from_string_converter=str_to_classes_in_namespaces_converter(
name_of_class_option='rule_class'
),
likely_to_be_changed=True,
)
@classmethod
def to_str(klass):
return "'%s'" % rule_sets_as_string
def test_classes_in_namespaces_converter_1(self):
converter_fn = str_to_classes_in_namespaces_converter('class_%(name)s')
class_list_str = ('socorro.unittest.lib.test_converters.Foo,'
'socorro.unittest.lib.test_converters.Bar')
result = converter_fn(class_list_str)
self.assertTrue(hasattr(result, 'required_config'))
req = result.required_config
self.assertEqual(len(req), 2)
self.assertTrue('class_Foo' in req)
self.assertEqual(len(req.class_Foo), 1)
self.assertTrue('class_Bar' in req)
self.assertEqual(len(req.class_Bar), 1)
self.assertEqual(
sorted([x.strip() for x in class_list_str.split(',')]),
sorted([x.strip() for x in to_str(result).strip("'").split(',')]))
def test_classes_in_namespaces_converter_1(self):
converter_fn = str_to_classes_in_namespaces_converter()
class_list_str = ('socorro.unittest.lib.test_converters.Foo,'
'socorro.unittest.lib.test_converters.Bar')
result = converter_fn(class_list_str)
assert hasattr(result, 'required_config')
req = result.required_config
assert len(req) == 2
assert 'Foo' in req
assert len(req.Foo) == 1
assert 'Bar' in req
assert len(req.Bar) == 1
expected = sorted(
[x.strip() for x in to_str(result).strip("'").split(',')])
assert sorted([x.strip()
for x in class_list_str.split(',')]) == expected
def test_classes_in_namespaces_converter_1(self):
converter_fn = str_to_classes_in_namespaces_converter()
class_list_str = (
'socorro.unittest.lib.test_converters.Foo,'
'socorro.unittest.lib.test_converters.Bar'
)
result = converter_fn(class_list_str)
assert hasattr(result, 'required_config')
req = result.required_config
assert len(req) == 2
assert 'Foo' in req
assert len(req.Foo) == 1
assert 'Bar' in req
assert len(req.Bar) == 1
expected = sorted([x.strip() for x in to_str(result).strip("'").split(',')])
assert sorted([x.strip() for x in class_list_str.split(',')]) == expected
def test_classes_in_namespaces_converter_3(self):
n = Namespace()
n.add_option(
'kls_list',
default=('socorro.unittest.lib.test_converters.Foo, '
'socorro.unittest.lib.test_converters.Foo, '
'socorro.unittest.lib.test_converters.Foo'),
from_string_converter=str_to_classes_in_namespaces_converter(
'%(name)s_%(index)02d'))
cm = ConfigurationManager(n, argv_source=[])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 3)
self.assertTrue('Foo_00' in config)
self.assertTrue('Foo_01' in config)
self.assertTrue('Foo_02' in config)
def test_classes_in_namespaces_converter_3(self):
n = Namespace()
n.add_option(
'kls_list',
default=(
'socorro.unittest.lib.test_converters.Foo, '
'socorro.unittest.lib.test_converters.Foo, '
'socorro.unittest.lib.test_converters.Foo'
),
from_string_converter= str_to_classes_in_namespaces_converter(
'%(name)s_%(index)02d'
)
)
cm = ConfigurationManager(n, argv_source=[])
config = cm.get_config()
self.assertEqual(len(config.kls_list.subordinate_namespace_names), 3)
self.assertTrue('Foo_00' in config)
self.assertTrue('Foo_01' in config)
self.assertTrue('Foo_02' in config)
def test_classes_in_namespaces_converter_1(self):
converter_fn = str_to_classes_in_namespaces_converter(
'class_%(name)s'
)
class_list_str = (
'socorro.unittest.lib.test_converters.Foo,'
'socorro.unittest.lib.test_converters.Bar'
)
result = converter_fn(class_list_str)
self.assertTrue(hasattr(result, 'required_config'))
req = result.required_config
self.assertEqual(len(req), 2)
self.assertTrue('class_Foo' in req)
self.assertEqual(len(req.class_Foo), 1)
self.assertTrue('class_Bar' in req)
self.assertEqual(len(req.class_Bar), 1)
self.assertEqual(
sorted([x.strip() for x in class_list_str.split(',')]),
sorted([
x.strip() for x in
to_str(result).strip("'").split(',')
])
)
def test_classes_in_namespaces_converter_2(self):
converter_fn = str_to_classes_in_namespaces_converter()
class_sequence = (Foo, Bar)
# ought to raise TypeError because 'class_sequence' is not a string
with pytest.raises(TypeError):
converter_fn(class_sequence)
class TransformRuleSystem(RequiredConfig):
"""A collection of TransformRules that can be applied together"""
required_config = Namespace()
required_config.add_option(
name='rules_list',
default=[],
from_string_converter=str_to_classes_in_namespaces_converter()
)
required_config.add_option(
'chatty_rules',
doc='should the rules announce what they are doing?',
default=False,
)
#--------------------------------------------------------------------------
def __init__(self, config=None, quit_check=None):
if quit_check:
self._quit_check = quit_check
else:
self._quit_check = self._null_quit_check
self.rules = []
if not config:
config = DotDict()
if 'chatty_rules' not in config:
config.chatty_rules = False
self.config = config
if "rules_list" in config:
self.tag = config.tag
self.act = getattr(self, config.action)
list_of_rules = config.rules_list.class_list
for a_rule_class_name, a_rule_class, ns_name in list_of_rules:
try:
self.rules.append(
a_rule_class(config[ns_name])
)
except KeyError:
self.rules.append(
a_rule_class(config)
)
#--------------------------------------------------------------------------
def _null_quit_check(self):
"a no-op method to do nothing if no quit check method has been defined"
pass
#--------------------------------------------------------------------------
def load_rules(self, an_iterable):
"""cycle through a collection of Transform rule tuples loading them
into the TransformRuleSystem"""
self.rules = [
TransformRule(*x, config=self.config) for x in an_iterable
]
#--------------------------------------------------------------------------
def append_rules(self, an_iterable):
"""add rules to the TransformRuleSystem"""
self.rules.extend(
TransformRule(*x, config=self.config) for x in an_iterable
)
#--------------------------------------------------------------------------
def apply_all_rules(self, *args, **kwargs):
"""cycle through all rules and apply them all without regard to
success or failure
returns:
True - since success or failure is ignored"""
for x in self.rules:
self._quit_check()
if self.config.chatty_rules:
self.config.logger.debug(
'apply_all_rules: %s',
to_str(x.__class__)
)
predicate_result, action_result = x.act(*args, **kwargs)
if self.config.chatty_rules:
self.config.logger.debug(
' : pred - %s; act - %s',
predicate_result,
action_result
)
return True
#--------------------------------------------------------------------------
def apply_until_action_succeeds(self, *args, **kwargs):
"""cycle through all rules until an action is run and succeeds
returns:
True - if an action is run and succeeds
False - if no action succeeds"""
for x in self.rules:
self._quit_check()
if self.config.chatty_rules:
self.config.logger.debug(
'apply_until_action_succeeds: %s',
to_str(x.__class__)
)
predicate_result, action_result = x.act(*args, **kwargs)
if self.config.chatty_rules:
self.config.logger.debug(
' : pred - %s; act - %s',
predicate_result,
action_result
)
if action_result:
return True
return False
#--------------------------------------------------------------------------
def apply_until_action_fails(self, *args, **kwargs):
"""cycle through all rules until an action is run and fails
returns:
True - an action ran and it failed
False - no action ever failed"""
for x in self.rules:
self._quit_check()
if self.config.chatty_rules:
self.config.logger.debug(
'apply_until_action_fails: %s',
to_str(x.__class__)
)
predicate_result, action_result = x.act(*args, **kwargs)
if self.config.chatty_rules:
self.config.logger.debug(
' : pred - %s; act - %s',
predicate_result,
action_result
)
if not action_result:
return True
return False
#--------------------------------------------------------------------------
def apply_until_predicate_succeeds(self, *args, **kwargs):
"""cycle through all rules until a predicate returns True
returns:
True - an action ran and it succeeded
False - an action ran and it failed
None - no predicate ever succeeded"""
for x in self.rules:
self._quit_check()
if self.config.chatty_rules:
self.config.logger.debug(
'apply_until_predicate_succeeds: %s',
to_str(x.__class__)
)
predicate_result, action_result = x.act(*args, **kwargs)
if self.config.chatty_rules:
self.config.logger.debug(
' : pred - %s; act - %s',
predicate_result,
action_result
)
if predicate_result:
return action_result
return None
#--------------------------------------------------------------------------
def apply_until_predicate_fails(self, *args, **kwargs):
"""cycle through all rules until a predicate returns False
returns:
False - a predicate ran and it failed
None - no predicate ever failed"""
for x in self.rules:
self._quit_check()
if self.config.chatty_rules:
self.config.logger.debug(
'apply_until_predicate_fails: %s',
to_str(x.__class__)
)
predicate_result, action_result = x.act(*args, **kwargs)
if self.config.chatty_rules:
self.config.logger.debug(
' : pred - %s; act - %s',
predicate_result,
action_result
)
if not predicate_result:
return False
return None
class TransformRuleSystem(RequiredConfig):
"""A collection of TransformRules that can be applied together"""
required_config = Namespace()
required_config.add_option(
name='rules_list',
default=[],
from_string_converter=str_to_classes_in_namespaces_converter())
#--------------------------------------------------------------------------
def __init__(self, config=None):
self.rules = []
self.config = config
if config:
if "rules_list" in config:
for a_rule_class in config.rules_list:
try:
self.rules.append(
a_rule_class(config[a_rule_class.__name__]))
except KeyError:
self.config.logger.debug(
'Rule %s configuration is missing',
a_rule_class.__name__)
#--------------------------------------------------------------------------
def load_rules(self, an_iterable):
"""cycle through a collection of Transform rule tuples loading them
into the TransformRuleSystem"""
self.rules = [
TransformRule(*x, config=self.config) for x in an_iterable
]
#--------------------------------------------------------------------------
def append_rules(self, an_iterable):
"""add rules to the TransformRuleSystem"""
self.rules.extend(
TransformRule(*x, config=self.config) for x in an_iterable)
#--------------------------------------------------------------------------
def apply_all_rules(self, *args, **kwargs):
"""cycle through all rules and apply them all without regard to
success or failure
returns:
True - since success or failure is ignored"""
# print 'args:', args, 'kwargs:', kwargs
for x in self.rules:
x.act(*args, **kwargs)
return True
#--------------------------------------------------------------------------
def apply_until_action_succeeds(self, *args, **kwargs):
"""cycle through all rules until an action is run and succeeds
returns:
True - if an action is run and succeeds
False - if no action succeeds"""
for x in self.rules:
predicate_result, action_result = x.act(*args, **kwargs)
if action_result:
return True
return False
#--------------------------------------------------------------------------
def apply_until_action_fails(self, *args, **kwargs):
"""cycle through all rules until an action is run and fails
returns:
True - an action ran and it failed
False - no action ever failed"""
for x in self.rules:
predicate_result, action_result = x.act(*args, **kwargs)
if not action_result:
return True
return False
#--------------------------------------------------------------------------
def apply_until_predicate_succeeds(self, *args, **kwargs):
"""cycle through all rules until a predicate returns True
returns:
True - an action ran and it succeeded
False - an action ran and it failed
None - no predicate ever succeeded"""
for x in self.rules:
predicate_result, action_result = x.act(*args, **kwargs)
if predicate_result:
return action_result
return None
#--------------------------------------------------------------------------
def apply_until_predicate_fails(self, *args, **kwargs):
"""cycle through all rules until a predicate returns False
returns:
False - a predicate ran and it failed
None - no predicate ever failed"""
for x in self.rules:
predicate_result, action_result = x.act(*args, **kwargs)
if not predicate_result:
return False
return None
def test_classes_in_namespaces_converter_2(self):
converter_fn = str_to_classes_in_namespaces_converter()
class_sequence = (Foo, Bar)
# ought to raise TypeError because 'class_sequence' is not a string
with pytest.raises(TypeError):
converter_fn(class_sequence)
def test_classes_in_namespaces_converter_2(self):
converter_fn = str_to_classes_in_namespaces_converter('class_%(name)s')
class_sequence = (Foo, Bar)
# ought to raise TypeError because 'class_sequence' is not a string
self.assertRaises(TypeError, converter_fn, class_sequence)
