def action(self, *args, **kwargs):
"""the default action for Support Classifiers invokes any derivied
_action function, trapping any exceptions raised in the process. We
are obligated to catch these exceptions to give subsequent rules the
opportunity act and perhaps (mitigate the error). An error during the
action application is a failure of the rule, not a failure of the
classification system itself."""
try:
return self._action(*args, **kwargs)
except KeyError as x:
if not self.config:
if 'processor' in kwargs:
self.config = kwargs['processor'].config
else:
raise
self.config.logger.debug(
'Rule %s action failed because of missing key "%s"',
to_str(self.__class__),
x,
)
except Exception as x:
if not self.config and 'processor' in kwargs:
self.config = kwargs['processor'].config
if not self.config:
raise
self.config.logger.debug(
'Rule %s action failed because of "%s"',
to_str(self.__class__),
x,
exc_info=True
)
return False
def close(self):
for a_rule in self.rules:
try:
self.config.logger.debug('trying to close %s', to_str(a_rule.__class__))
a_rule.close()
except AttributeError:
self.config.logger.debug('%s has no close', to_str(a_rule.__class__))
# no close method mean no need to close
pass
def close(self):
for a_rule in self.rules:
try:
self.config.logger.debug('trying to close %s', to_str(a_rule.__class__))
close_method = a_rule.close
except AttributeError:
self.config.logger.debug('%s has no close', to_str(a_rule.__class__))
# no close method mean no need to close
continue
close_method()
def __init__(self, value):
if not isinstance(value, six.string_types):
value_str = to_str(value)
else:
value_str = value
self.value_str = value_str
self.value = value
def __init__(self, source, config_manager=None, top_level_section_name=''):
self.delayed_parser_instantiation = False
self.top_level_section_name = top_level_section_name
if source is configobj.ConfigObj:
try:
app = config_manager._get_option('admin.application')
source = "%s.%s" % (app.value.app_name, file_name_extension)
except AttributeError:
# we likely don't have the admin.application object set up yet.
# we need to delay the instantiation of the config parser
# until later.
if source is None:
raise NotEnoughInformationException(
"Can't setup an ini file without knowing the file name"
)
self.delayed_parser_instantiation = True
return
if not os.path.exists(source) and config_manager.config_optional:
return
if isinstance(source, (six.binary_type, six.text_type)):
source = to_str(source)
if (isinstance(source, six.string_types)
and source.endswith(file_name_extension)):
try:
self.config_obj = ConfigObjWithIncludes(source)
except Exception as x:
raise LoadingIniFileFailsException(
"ConfigObj cannot load ini: %s" % str(x))
else:
raise CantHandleTypeException()
def __init__(self, source, the_config_manager=None):
self.values = None
if source is json:
try:
app = the_config_manager._get_option('admin.application')
source = "%s.%s" % (app.value.app_name, file_name_extension)
except (AttributeError, KeyError):
raise NotEnoughInformationException(
"Can't setup an json file without knowing the file name")
if isinstance(source, (six.binary_type, six.text_type)):
source = to_str(source)
if (isinstance(source, six.string_types)
and source.endswith(file_name_extension)):
try:
with open(source) as fp:
self.values = json.load(fp)
except IOError as x:
# The file doesn't exist. That's ok, we'll give warning
# but this isn't a fatal error
import warnings
warnings.warn("%s doesn't exist" % source)
self.values = {}
except ValueError:
raise LoadingJsonFileFailsException("Cannot load json: %s" %
str(x))
else:
raise CantHandleTypeException()
def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
crash_id = raw_crash.uuid
old_processed_crash = self.crashstore.get_unredacted_processed(crash_id)
for key, value in old_processed_crash.iteritems():
if 'date_processed' in key:
processed_crash[key] = date_to_string(
string_to_datetime(value) - self.config.time_delta
)
print processed_crash.uuid, value, processed_crash[key]
else:
if key != 'uptime' and key != 'crash_time' and (
'time' in key or "date" in key or 'Date' in key
):
value = date_to_string(string_to_datetime(value))
processed_crash[key] = value
processor_meta.processor_notes.append(
'DateProcessedTimeMachine has pushed date_processed into the past'
' by "%s" (D HH:MM:SS)' % to_str(self.config.time_delta)
)
processor_meta.processor_notes.append(
'Original processor_notes: %s'
% old_processed_crash['processor_notes']
)
return True
def __init__(self, source, the_config_manager=None):
self.values = None
if source is json:
try:
app = the_config_manager._get_option('admin.application')
source = "%s.%s" % (app.value.app_name, file_name_extension)
except (AttributeError, KeyError):
raise NotEnoughInformationException(
"Can't setup an json file without knowing the file name"
)
if isinstance(source, (six.binary_type, six.text_type)):
source = to_str(source)
if (
isinstance(source, six.string_types)
and source.endswith(file_name_extension)
):
try:
with open(source) as fp:
self.values = json.load(fp)
except IOError as x:
# The file doesn't exist. That's ok, we'll give warning
# but this isn't a fatal error
import warnings
warnings.warn("%s doesn't exist" % source)
self.values = {}
except ValueError:
raise LoadingJsonFileFailsException(
"Cannot load json: %s" % str(x)
)
else:
raise CantHandleTypeException()
def local_to_str(a_thing):
try:
return local_to_string_converters[type(a_thing)](a_thing)
except KeyError:
try:
return find_to_string_converter(a_thing)(a_thing)
except TypeError:
return to_str(a_thing)
def string_to_string(a_string):
a_string = to_str(a_string)
quote = '"'
if '"' in a_string:
quote = "'"
if "'" in a_string:
quote = '"""'
if "/n" in a_string:
quote = "'''"
return "%s%s%s" % (quote, a_string, quote)
def __str__(self):
"""return an instance of Option's value as a string.
The option instance doesn't actually have to be from the Option class.
All it requires is that the passed option instance has a ``value``
attribute.
"""
try:
return self.to_string_converter(self.value)
except TypeError:
return to_str(self.value)
def __str__(self):
"""return an instance of Option's value as a string.
The option instance doesn't actually have to be from the Option class.
All it requires is that the passed option instance has a ``value``
attribute.
"""
try:
return self.to_string_converter(self.value)
except TypeError:
return to_str(self.value)
def __init__(self, candidate, the_config_manager=None):
if isinstance(candidate, (six.binary_type, six.text_type)):
candidate = to_str(candidate)
if (
isinstance(candidate, six.string_types) and
candidate.endswith(file_name_extension)
):
# we're trusting the string represents a filename
opener = functools.partial(open, candidate)
elif isinstance(candidate, function_type):
# we're trusting that the function when called with no parameters
# will return a Context Manager Type.
opener = candidate
else:
raise CantHandleTypeException()
self.values = {}
try:
with opener() as f:
previous_key = None
for line in f:
line = to_str(line)
if line.strip().startswith('#') or not line.strip():
continue
if line[0] in ' \t' and previous_key:
line = line[1:]
self.values[previous_key] = (
'%s%s' % (self.values[previous_key], line.rstrip())
)
continue
try:
key, value = line.split("=", 1)
self.values[key.strip()] = value.strip()
previous_key = key
except ValueError:
self.values[line] = ''
except Exception as x:
raise NotAConfigFileError(
"Conf couldn't interpret %s as a config file: %s"
% (candidate, str(x))
)
def _option_to_args_list(self, an_option, key):
if an_option.is_argument:
if an_option.foreign_data is not None:
nargs = an_option.foreign_data.argparse.kwargs.get(
'nargs', None)
else:
if isinstance(an_option.value,
(six.binary_type, six.text_type)):
an_option.value = to_str(an_option.value)
return an_option.value
if an_option.to_string_converter:
return an_option.to_string_converter(an_option.value)
return to_str(an_option.value)
if (nargs is not None
and isinstance(an_option.value, collections.Sequence)):
if isinstance(an_option.value,
(six.binary_type, six.text_type)):
an_option.value = to_str(an_option.value)
return an_option.value
return [to_str(x) for x in an_option.value]
if an_option.value is None:
return []
return to_str(an_option.value)
#if an_option.foreign_data.argparse.kwargs.nargs == 0:
#return None
if an_option.from_string_converter in (bool, boolean_converter):
if an_option.value:
return "--%s" % key
return None
if an_option.value is None:
return None
return '--%s="%s"' % (key, to_str(an_option))
def predicate(self, *args, **kwargs):
"""the default predicate for Support Classifiers invokes any derivied
_predicate function, trapping any exceptions raised in the process. We
are obligated to catch these exceptions to give subsequent rules the
opportunity to act. An error during the predicate application is a
failure of the rule, not a failure of the classification system itself
"""
try:
return self._predicate(*args, **kwargs)
except Exception, x:
self.config.logger.debug(
'Rule %s predicicate failed because of "%s"', to_str(self.__class__), x, exc_info=True
)
return False
def apply_all_rules(self, *args, **kwargs):
"""cycle through all rules and apply them all without regard to
success or failure
returns:
True - since success or failure is ignored"""
for x in self.rules:
self._quit_check()
if self.config.chatty_rules:
self.config.logger.debug("apply_all_rules: %s", to_str(x.__class__))
predicate_result, action_result = x.act(*args, **kwargs)
if self.config.chatty_rules:
self.config.logger.debug(" : pred - %s; act - %s", predicate_result, action_result)
return True
def _load(self, infile, configspec):
"""this overrides the original ConfigObj method of the same name. It
runs through the input file collecting lines into a list. When
completed, this method submits the list of lines to the super class'
function of the same name. ConfigObj proceeds, completely unaware
that it's input file has been preprocessed."""
if isinstance(infile, (six.binary_type, six.text_type)):
infile = to_str(infile)
original_path = os.path.dirname(infile)
expanded_file_contents = self._expand_files(infile, original_path)
super(ConfigObjWithIncludes, self)._load(expanded_file_contents,
configspec)
else:
super(ConfigObjWithIncludes, self)._load(infile, configspec)
def __init__(self, candidate, the_config_manager=None):
if isinstance(candidate, (six.binary_type, six.text_type)):
candidate = to_str(candidate)
if (isinstance(candidate, six.string_types)
and candidate.endswith(file_name_extension)):
# we're trusting the string represents a filename
opener = functools.partial(open, candidate)
elif isinstance(candidate, function_type):
# we're trusting that the function when called with no parameters
# will return a Context Manager Type.
opener = candidate
else:
raise CantHandleTypeException()
self.values = {}
try:
with opener() as f:
previous_key = None
for line in f:
line = to_str(line)
if line.strip().startswith('#') or not line.strip():
continue
if line[0] in ' \t' and previous_key:
line = line[1:]
self.values[previous_key] = (
'%s%s' %
(self.values[previous_key], line.rstrip()))
continue
try:
key, value = line.split("=", 1)
self.values[key.strip()] = value.strip()
previous_key = key
except ValueError:
self.values[line] = ''
except Exception as x:
raise NotAConfigFileError(
"Conf couldn't interpret %s as a config file: %s" %
(candidate, str(x)))
def action(self, *args, **kwargs):
"""the default action for Support Classifiers invokes any derivied
_action function, trapping any exceptions raised in the process. We
are obligated to catch these exceptions to give subsequent rules the
opportunity to act and perhaps mitigate the error. An error during the
action application is a failure of the rule, not a failure of the
classification system itself."""
try:
return self._action(*args, **kwargs)
except KeyError, x:
self.config.logger.debug(
'Rule %s action failed because of missing key "%s"',
to_str(self.__class__),
x,
)
def test_classes_in_namespaces_converter_1(self):
converter_fn = str_to_classes_in_namespaces_converter('class_%(name)s')
class_list_str = ('socorro.unittest.lib.test_converters.Foo,'
'socorro.unittest.lib.test_converters.Bar')
result = converter_fn(class_list_str)
self.assertTrue(hasattr(result, 'required_config'))
req = result.required_config
self.assertEqual(len(req), 2)
self.assertTrue('class_Foo' in req)
self.assertEqual(len(req.class_Foo), 1)
self.assertTrue('class_Bar' in req)
self.assertEqual(len(req.class_Bar), 1)
self.assertEqual(
sorted([x.strip() for x in class_list_str.split(',')]),
sorted([x.strip() for x in to_str(result).strip("'").split(',')]))
def action(self, *args, **kwargs):
"""the default action for Support Classifiers invokes any derivied
_action function, trapping any exceptions raised in the process. We
are obligated to catch these exceptions to give subsequent rules the
opportunity to act and perhaps mitigate the error. An error during the
action application is a failure of the rule, not a failure of the
classification system itself."""
try:
return self._action(*args, **kwargs)
except KeyError, x:
self.config.logger.debug(
'Rule %s action failed because of missing key "%s"',
to_str(self.__class__),
x,
)
def predicate(self, *args, **kwargs):
"""the default predicate for Support Classifiers invokes any derivied
_predicate function, trapping any exceptions raised in the process. We
are obligated to catch these exceptions to give subsequent rules the
opportunity to act. An error during the predicate application is a
failure of the rule, not a failure of the classification system itself
"""
try:
return self._predicate(*args, **kwargs)
except Exception, x:
self.config.logger.debug(
'Rule %s predicicate failed because of "%s"',
to_str(self.__class__),
x,
exc_info=True)
return False
def dispatch_request_to_write(config_file_type, options_mapping, opener):
if isinstance(config_file_type, (six.binary_type, six.text_type)):
config_file_type = to_str(config_file_type)
try:
writer_fn = file_extension_dispatch[config_file_type]
except KeyError:
raise UnknownFileExtensionException(
"%s isn't a registered file name extension" % config_file_type)
with opener() as output_stream:
writer_fn(options_mapping, output_stream=output_stream)
else:
# this is the case where we've not gotten a file extension, but a
# for_handler module. Use the module's ValueSource's write method
with opener() as output_stream:
config_file_type.ValueSource.write(options_mapping,
output_stream=output_stream)
def test_classes_in_namespaces_converter_1(self):
converter_fn = str_to_classes_in_namespaces_converter()
class_list_str = (
'socorro.unittest.lib.test_converters.Foo,'
'socorro.unittest.lib.test_converters.Bar'
)
result = converter_fn(class_list_str)
assert hasattr(result, 'required_config')
req = result.required_config
assert len(req) == 2
assert 'Foo' in req
assert len(req.Foo) == 1
assert 'Bar' in req
assert len(req.Bar) == 1
expected = sorted([x.strip() for x in to_str(result).strip("'").split(',')])
assert sorted([x.strip() for x in class_list_str.split(',')]) == expected
def test_classes_in_namespaces_converter_1(self):
converter_fn = str_to_classes_in_namespaces_converter()
class_list_str = ('socorro.unittest.lib.test_converters.Foo,'
'socorro.unittest.lib.test_converters.Bar')
result = converter_fn(class_list_str)
assert hasattr(result, 'required_config')
req = result.required_config
assert len(req) == 2
assert 'Foo' in req
assert len(req.Foo) == 1
assert 'Bar' in req
assert len(req.Bar) == 1
expected = sorted(
[x.strip() for x in to_str(result).strip("'").split(',')])
assert sorted([x.strip()
for x in class_list_str.split(',')]) == expected
def __init__(
self,
name,
default=None,
doc=None,
from_string_converter=None,
to_string_converter=None,
value=None,
short_form=None,
exclude_from_print_conf=False,
exclude_from_dump_conf=False,
is_argument=False,
likely_to_be_changed=False,
not_for_definition=False,
reference_value_from=None,
secret=False,
has_changed=False,
foreign_data=None,
):
self.name = name
self.short_form = short_form
self.default = default
if isinstance(doc, (six.binary_type, six.text_type)):
doc = to_str(doc).strip()
self.doc = doc
if from_string_converter is None:
if default is not None:
# take a qualified guess from the default value
from_string_converter = self._deduce_converter(default)
if isinstance(from_string_converter, (six.binary_type, six.text_type)):
from_string_converter = str_to_python_object(from_string_converter)
self.from_string_converter = from_string_converter
# if this is not set, the type is used in converters.py to attempt
# the conversion
self.to_string_converter = to_string_converter
if value is None:
value = default
self.value = value
self.is_argument = is_argument
self.exclude_from_print_conf = exclude_from_print_conf
self.exclude_from_dump_conf = exclude_from_dump_conf
self.likely_to_be_changed = likely_to_be_changed
self.not_for_definition = not_for_definition
self.reference_value_from = reference_value_from
self.secret = secret
self.has_changed = has_changed
self.foreign_data = foreign_data
def get_import_for_type(t):
"""given a type, return a tuple of the (module-path, type_name)
or (None, None) if it is a built in."""
t_as_string = to_str(t)
if not is_identifier(t_as_string):
# this class expanded into something other than a single identifier
# we can ignore it. This is the case when we encounter something
# like the configman.converter.str_to_classes_in_namespaces
# InnerClassList. We can safely ignore these things here.
return (None, None)
if '.' in t_as_string:
parts = t_as_string.split('.')
return ('.'.join(parts[:-1]), parts[-1])
else:
if t_as_string in known_mapping_str_to_type:
return (None, None)
return (None, t_as_string)
def apply_all_rules(self, *args, **kwargs):
"""cycle through all rules and apply them all without regard to
success or failure
returns:
True - since success or failure is ignored"""
for x in self.rules:
self._quit_check()
if self.config.chatty_rules:
self.config.logger.debug('apply_all_rules: %s',
to_str(x.__class__))
predicate_result, action_result = x.act(*args, **kwargs)
if self.config.chatty_rules:
self.config.logger.debug(
' : pred - %s; act - %s', predicate_result,
action_result)
return True
def wrap_with_value_source_api(value_source_list, a_config_manager):
wrapped_sources = []
for a_source in value_source_list:
if a_source is ConfigFileFutureProxy:
a_source = a_config_manager._get_option('admin.conf').default
# raise hell if the config file doesn't exist
if isinstance(a_source, (six.binary_type, six.text_type)):
a_source = to_str(a_source)
config_file_doesnt_exist = not os.path.isfile(a_source)
if config_file_doesnt_exist:
if a_config_manager.config_optional:
continue # no file, it's optional, ignore it
raise IOError(a_source) # no file, it's required, raise
if a_source == a_config_manager.config_pathname:
# the config file has not been set to anything other than
# the default value. Force this into be the degenerate case
# and skip the wrapping process. We'll read the file later.
continue
if a_source is None:
# this means the source is degenerate - like the case where
# the config file name has not been specified
continue
handlers = type_handler_dispatch.get_handlers(a_source)
wrapped_source = None
error_history = []
for a_handler in handlers:
try:
wrapped_source = a_handler.ValueSource(a_source,
a_config_manager)
break
except (ValueException, CannotConvertError) as x:
# a failure is not necessarily fatal, we need to try all of
# the handlers. It's only fatal when they've all failed
exception_as_str = str(x)
if exception_as_str:
error_history.append(str(x))
if wrapped_source is None:
if error_history:
errors = '; '.join(error_history)
raise AllHandlersFailedException(errors)
else:
raise NoHandlerForType(type(a_source))
wrapped_sources.append(wrapped_source)
return wrapped_sources
def apply_until_predicate_fails(self, *args, **kwargs):
"""cycle through all rules until a predicate returns False
returns:
False - a predicate ran and it failed
None - no predicate ever failed"""
for x in self.rules:
self._quit_check()
if self.config.chatty_rules:
self.config.logger.debug("apply_until_predicate_fails: %s", to_str(x.__class__))
predicate_result, action_result = x.act(*args, **kwargs)
if self.config.chatty_rules:
self.config.logger.debug(
" : pred - %s; act - %s", predicate_result, action_result
)
if not predicate_result:
return False
return None
def apply_until_action_succeeds(self, *args, **kwargs):
"""cycle through all rules until an action is run and succeeds
returns:
True - if an action is run and succeeds
False - if no action succeeds"""
for x in self.rules:
self._quit_check()
if self.config.chatty_rules:
self.config.logger.debug("apply_until_action_succeeds: %s", to_str(x.__class__))
predicate_result, action_result = x.act(*args, **kwargs)
if self.config.chatty_rules:
self.config.logger.debug(
" : pred - %s; act - %s", predicate_result, action_result
)
if action_result:
return True
return False
def apply_until_predicate_fails(self, *args, **kwargs):
"""cycle through all rules until a predicate returns False
returns:
False - a predicate ran and it failed
None - no predicate ever failed"""
for x in self.rules:
self._quit_check()
if self.config.chatty_rules:
self.config.logger.debug('apply_until_predicate_fails: %s',
to_str(x.__class__))
predicate_result, action_result = x.act(*args, **kwargs)
if self.config.chatty_rules:
self.config.logger.debug(
' : pred - %s; act - %s',
predicate_result, action_result)
if not predicate_result:
return False
return None
def apply_until_action_succeeds(self, *args, **kwargs):
"""cycle through all rules until an action is run and succeeds
returns:
True - if an action is run and succeeds
False - if no action succeeds"""
for x in self.rules:
self._quit_check()
if self.config.chatty_rules:
self.config.logger.debug('apply_until_action_succeeds: %s',
to_str(x.__class__))
predicate_result, action_result = x.act(*args, **kwargs)
if self.config.chatty_rules:
self.config.logger.debug(
' : pred - %s; act - %s',
predicate_result, action_result)
if action_result:
return True
return False
def action(self, *args, **kwargs):
"""the default action for Support Classifiers invokes any derivied
_action function, trapping any exceptions raised in the process. We
are obligated to catch these exceptions to give subsequent rules the
opportunity to act and perhaps mitigate the error. An error during the
action application is a failure of the rule, not a failure of the
classification system itself."""
try:
return self._action(*args, **kwargs)
except Exception as exception:
if not self._send_to_sentry('action', self.action, *args, **
kwargs):
# Only log if it couldn't be sent to Sentry
self.config.logger.debug(
'Rule %s action failed because of "%s"',
to_str(self.__class__),
exception,
exc_info=True)
return False
def apply_until_action_fails(self, *args, **kwargs):
"""cycle through all rules until an action is run and fails
returns:
True - an action ran and it failed
False - no action ever failed"""
for x in self.rules:
if self.config.chatty_rules:
self.config.logger.debug(
'apply_until_action_fails: %s',
to_str(x.__class__)
)
predicate_result, action_result = x.act(*args, **kwargs)
if self.config.chatty_rules:
self.config.logger.debug(
' : pred - %s; act - %s',
predicate_result,
action_result
)
if not action_result:
return True
return False
def str_to_timedelta(input_str):
""" a string conversion function for timedelta for strings in the format
DD:HH:MM:SS or D HH:MM:SS
"""
try:
input_str = input_str.replace(' ', ':')
except (TypeError, AttributeError):
from configman.converters import to_str
raise TypeError('%s should have been a string' % to_str(input_str))
days, hours, minutes, seconds = 0, 0, 0, 0
details = input_str.split(':')
if len(details) >= 4:
days = int(details[-4])
if len(details) >= 3:
hours = int(details[-3])
if len(details) >= 2:
minutes = int(details[-2])
if len(details) >= 1:
seconds = int(details[-1])
return datetime.timedelta(days=days,
hours=hours,
minutes=minutes,
seconds=seconds)
def apply_until_predicate_succeeds(self, *args, **kwargs):
"""cycle through all rules until a predicate returns True
returns:
True - an action ran and it succeeded
False - an action ran and it failed
None - no predicate ever succeeded"""
for x in self.rules:
if self.config.chatty_rules:
self.config.logger.debug(
'apply_until_predicate_succeeds: %s',
to_str(x.__class__)
)
predicate_result, action_result = x.act(*args, **kwargs)
if self.config.chatty_rules:
self.config.logger.debug(
' : pred - %s; act - %s',
predicate_result,
action_result
)
if predicate_result:
return action_result
return None
def test_classes_in_namespaces_converter_1(self):
converter_fn = str_to_classes_in_namespaces_converter(
'class_%(name)s'
)
class_list_str = (
'socorro.unittest.lib.test_converters.Foo,'
'socorro.unittest.lib.test_converters.Bar'
)
result = converter_fn(class_list_str)
self.assertTrue(hasattr(result, 'required_config'))
req = result.required_config
self.assertEqual(len(req), 2)
self.assertTrue('class_Foo' in req)
self.assertEqual(len(req.class_Foo), 1)
self.assertTrue('class_Bar' in req)
self.assertEqual(len(req.class_Bar), 1)
self.assertEqual(
sorted([x.strip() for x in class_list_str.split(',')]),
sorted([
x.strip() for x in
to_str(result).strip("'").split(',')
])
)
def action(self, *args, **kwargs):
"""the default action for Support Classifiers invokes any derivied
_action function, trapping any exceptions raised in the process. We
are obligated to catch these exceptions to give subsequent rules the
opportunity to act and perhaps mitigate the error. An error during the
action application is a failure of the rule, not a failure of the
classification system itself."""
try:
return self._action(*args, **kwargs)
except Exception as exception:
if not self._send_to_sentry(
'action',
self.action,
*args,
**kwargs
):
# Only log if it couldn't be sent to Sentry
self.config.logger.debug(
'Rule %s action failed because of "%s"',
to_str(self.__class__),
exception,
exc_info=True
)
return False
def set_value(self, val=None):
if val is None:
val = self.default
if isinstance(val, (six.binary_type, six.text_type)):
val = to_str(val)
try:
new_value = self.from_string_converter(val)
self.has_changed = new_value != self.value
self.value = new_value
except TypeError:
self.has_changed = val != self.value
self.value = val
except ValueError:
error_message = "In '%s', '%s' fails to convert '%s'" % (
self.name, self.from_string_converter, val)
raise CannotConvertError(error_message)
elif isinstance(val, Option):
self.has_changed = val.default != self.value
self.value = val.default
elif isinstance(val, collections.Mapping) and 'default' in val:
self.set_value(val["default"])
else:
self.has_changed = val != self.value
self.value = val
def _option_to_args_list(self, an_option, key):
if an_option.is_argument:
if an_option.foreign_data is not None:
nargs = an_option.foreign_data.argparse.kwargs.get(
'nargs',
None
)
else:
if isinstance(an_option.value, (six.binary_type, six.text_type)):
an_option.value = to_str(an_option.value)
return an_option.value
if an_option.to_string_converter:
return an_option.to_string_converter(an_option.value)
return to_str(an_option.value)
if (
nargs is not None
and isinstance(an_option.value, collections.Sequence)
):
if isinstance(an_option.value, (six.binary_type, six.text_type)):
an_option.value = to_str(an_option.value)
return an_option.value
return [to_str(x) for x in an_option.value]
if an_option.value is None:
return []
return to_str(an_option.value)
#if an_option.foreign_data.argparse.kwargs.nargs == 0:
#return None
if an_option.from_string_converter in (bool, boolean_converter):
if an_option.value:
return "--%s" % key
return None
if an_option.value is None:
return None
return '--%s="%s"' % (
key,
to_str(an_option)
)
def summary_name(self):
return to_str(self.__class__)
def test_to_str(self):
self.assertEqual(converters.to_str(1), "1")
self.assertEqual(converters.to_str(3.1415), "3.1415")
self.assertEqual(converters.to_str('hello'), "hello")
self.assertEqual(
converters.to_str(u"'你好'"),
u"'\u4f60\u597d'"
)
self.assertEqual(
converters.list_to_str([1, 2, 3]),
"1, 2, 3"
)
self.assertEqual(
converters.to_str(True),
"True"
)
self.assertEqual(
converters.to_str(False),
"False"
)
self.assertEqual(
converters.to_str(datetime.datetime(1960, 5, 4, 15, 10)),
"1960-05-04T15:10:00"
)
self.assertEqual(
converters.to_str(datetime.date(1960, 5, 4)),
"1960-05-04"
)
self.assertEqual(
converters.to_str(datetime.timedelta(days=1, seconds=1)),
"1 00:00:01"
)
self.assertEqual(converters.to_str(int), 'int')
self.assertEqual(converters.to_str(float), 'float')
self.assertEqual(converters.to_str(unittest), 'unittest')
self.assertEqual(
converters.to_str(converters.to_str),
'configman.converters.to_str'
)
import re
r = re.compile('.*')
self.assertEqual(converters.to_str(r), '.*')
are obligated to catch these exceptions to give subsequent rules the
opportunity to act and perhaps mitigate the error. An error during the
action application is a failure of the rule, not a failure of the
classification system itself."""
try:
return self._action(*args, **kwargs)
except KeyError, x:
self.config.logger.debug(
'Rule %s action failed because of missing key "%s"',
to_str(self.__class__),
x,
)
except Exception, x:
self.config.logger.debug(
'Rule %s action failed because of "%s"',
to_str(self.__class__),
x,
exc_info=True
)
return False
#--------------------------------------------------------------------------
def _action(self, *args, **kwargs):
"""Rules derived from this base class ought to override this method
with an actual classification rule. Successful application of this
method should include a call to '_add_classification'.
returns:
True - this rule was applied successfully and no further rules
should be applied
False - this rule did not succeed and further rules should be
def __init__(
self,
definition_source=None,
values_source_list=None,
argv_source=None,
#use_config_files=True,
use_auto_help=True,
use_admin_controls=True,
quit_after_admin=True,
options_banned_from_help=None,
app_name='',
app_version='',
app_description='',
config_pathname='.',
config_optional=True,
value_source_object_hook=DotDict,
):
"""create and initialize a configman object.
parameters:
definition_source - a namespace or list of namespaces from which
configman is to fetch the definitions of the
configuration parameters.
values_source_list - (optional) a hierarchical list of sources for
values for the configuration parameters.
As values are copied from these sources,
conficting values are resolved with sources
on the right getting preference over sources on
the left.
argv_source - if the values_source_list contains a commandline
source, this value is an alternative source for
actual command line arguments. Useful for testing or
preprocessing command line arguments.
use_auto_help - set to True if configman is to automatically set up
help output for command line invocations.
use_admin_controls - configman can add command line flags that it
interprets independently of the app defined
arguments. True enables this capability, while,
False supresses it.
quit_after_admin - if True and admin controls are enabled and used,
call sys.exit to end the app. This is useful to
stop the app from running if all that was done
was to write a config file or stop after help.
options_banned_from_help - a list of strings that will censor the
output of help to prevent specified
options from being listed in the help
output. This is useful for hiding debug
or secret command line arguments.
app_name - assigns a name to the app. This is used in help output
and as a default basename for config files.
app_version - assigns a version for the app used help output.
app_description - assigns a description for the app to be used in
the help output.
config_pathname - a hard coded path to the directory of or the full
path and name of the configuration file.
config_optional - a boolean indicating if a missing default config
file is optional. Note: this is only for the
default config file. If a config file is specified
on the commandline, it _must_ exist.
value_source_object_hook - a class used for the internal
representation of a value source.
This is used to enable any special
processing, like key translations.
"""
# instead of allowing mutables as default keyword argument values...
if definition_source is None:
definition_source_list = []
elif (
isinstance(definition_source, collections.Sequence) and
not isinstance(definition_source, (six.binary_type, six.text_type))
):
definition_source_list = list(definition_source)
else:
if isinstance(definition_source, (six.binary_type, six.text_type)):
definition_source = to_str(definition_source)
definition_source_list = [definition_source]
if argv_source is None:
self.argv_source = sys.argv[1:]
self.app_invocation_name = sys.argv[0]
else:
self.argv_source = argv_source
self.app_invocation_name = app_name
if options_banned_from_help is None:
options_banned_from_help = ['application']
self.config_pathname = config_pathname
self.config_optional = config_optional
self.use_auto_help = use_auto_help
self.value_source_object_hook = value_source_object_hook
self.app_name = app_name
self.app_version = app_version
self.app_description = app_description
self.args = [] # extra commandline arguments that are not switches
# will be stored here.
self._config = None # eventual container for DOM-like config object
self.option_definitions = Namespace()
self.definition_source_list = definition_source_list
command_line_value_source = command_line
if values_source_list is None:
# nothing set, assume defaults
if use_admin_controls:
values_source_list = (
ConfigFileFutureProxy,
environment,
command_line_value_source
)
else:
values_source_list = (
environment,
command_line_value_source
)
# determine which command_line facility to use for help
if self.use_auto_help:
# we need to iterate through all of our value sources looking for
# one that can interact with the user on the commandline.
for a_value_source in values_source_list:
if inspect.ismodule(a_value_source):
handler = \
type_handler_dispatch[a_value_source][0].ValueSource
try:
# if a value source is able to handle the command line
# it will have defined 'command_line_value_source' as
# true. Not all values sources may have this attribute
if handler.command_line_value_source:
handler._setup_auto_help(self)
break
except AttributeError:
# not a commandline source because it doesn't have
# the 'command_line_value_source' OR it doesn't have
# a method that allows it to setup a help system.
# this is OK, we can ignore it and move on until we
# find an appropriate source.
pass
else:
# While not actually necessary to have implemented, this
# is the case where the value source is not a module.
# So we know nothing about its interface. We cannot even
# try to use it as a commandline value source.
pass
admin_tasks_done = False
self.keys_blocked_from_output = [
'help',
'admin.conf',
'admin.dump_conf',
'admin.print_conf',
'admin.strict',
'admin.expose_secrets',
]
self.options_banned_from_help = options_banned_from_help
if use_admin_controls:
admin_options = self._setup_admin_options(values_source_list)
self.definition_source_list.append(admin_options)
# iterate through the option definitions to create the nested dict
# hierarchy of all the options called 'option_definitions'
for a_definition_source in self.definition_source_list:
try:
safe_copy_of_def_source = a_definition_source.safe_copy()
except AttributeError:
# apparently, the definition source was not in the form of a
# Namespace object. This isn't a show stopper, but we don't
# know how to make a copy of this object safely: we know from
# experience that the stock copy.copy method leads to grief
# as many sub-objects within an option definition source can
# not be copied that way (classes, for example).
# The only action we can take is to trust and continue with the
# original copy of the definition source.
safe_copy_of_def_source = a_definition_source
setup_definitions(
safe_copy_of_def_source,
self.option_definitions
)
if use_admin_controls:
# the name of the config file needs to be loaded from the command
# line prior to processing the rest of the command line options.
config_filename = config_filename_from_commandline(self)
if (
config_filename
and ConfigFileFutureProxy in values_source_list
):
self.option_definitions.admin.conf.default = config_filename
self.values_source_list = wrap_with_value_source_api(
values_source_list,
self
)
known_keys = self._overlay_expand()
self._check_for_mismatches(known_keys)
# the app_name, app_version and app_description are to come from
# if 'application' option if it is present. If it is not present,
# get the app_name,et al, from parameters passed into the constructor.
# if those are empty, set app_name, et al, to empty strings
try:
app_option = self._get_option('application')
self.app_name = getattr(app_option.value, 'app_name', '')
self.app_version = getattr(app_option.value, 'app_version', '')
self.app_description = getattr(
app_option.value,
'app_description',
''
)
except NotAnOptionError:
# there is no 'application' option, continue to use the
# 'app_name' from the parameters passed in, if they exist.
pass
try:
if use_auto_help and self._get_option('help').value:
self.output_summary()
admin_tasks_done = True
except NotAnOptionError:
# the current command-line implementation already has a help
# mechanism of its own that doesn't require the use of a
# option in configman. This error is ignorable
pass
# keys that end with a "$" are called "blocked_by_suffix".
# This means that these options are not to be written out to
# configuration files.
keys_blocked_by_suffix = [
key
for key in self.option_definitions.keys_breadth_first()
if key.endswith('$')
]
self.keys_blocked_from_output.extend(keys_blocked_by_suffix)
if use_admin_controls and self._get_option('admin.print_conf').value:
self.print_conf()
admin_tasks_done = True
if use_admin_controls and self._get_option('admin.dump_conf').value:
self.dump_conf()
admin_tasks_done = True
if quit_after_admin and admin_tasks_done:
sys.exit()
def __init__(
self,
definition_source=None,
values_source_list=None,
argv_source=None,
#use_config_files=True,
use_auto_help=True,
use_admin_controls=True,
quit_after_admin=True,
options_banned_from_help=None,
app_name='',
app_version='',
app_description='',
config_pathname='.',
config_optional=True,
value_source_object_hook=DotDict,
):
"""create and initialize a configman object.
parameters:
definition_source - a namespace or list of namespaces from which
configman is to fetch the definitions of the
configuration parameters.
values_source_list - (optional) a hierarchical list of sources for
values for the configuration parameters.
As values are copied from these sources,
conficting values are resolved with sources
on the right getting preference over sources on
the left.
argv_source - if the values_source_list contains a commandline
source, this value is an alternative source for
actual command line arguments. Useful for testing or
preprocessing command line arguments.
use_auto_help - set to True if configman is to automatically set up
help output for command line invocations.
use_admin_controls - configman can add command line flags that it
interprets independently of the app defined
arguments. True enables this capability, while,
False supresses it.
quit_after_admin - if True and admin controls are enabled and used,
call sys.exit to end the app. This is useful to
stop the app from running if all that was done
was to write a config file or stop after help.
options_banned_from_help - a list of strings that will censor the
output of help to prevent specified
options from being listed in the help
output. This is useful for hiding debug
or secret command line arguments.
app_name - assigns a name to the app. This is used in help output
and as a default basename for config files.
app_version - assigns a version for the app used help output.
app_description - assigns a description for the app to be used in
the help output.
config_pathname - a hard coded path to the directory of or the full
path and name of the configuration file.
config_optional - a boolean indicating if a missing default config
file is optional. Note: this is only for the
default config file. If a config file is specified
on the commandline, it _must_ exist.
value_source_object_hook - a class used for the internal
representation of a value source.
This is used to enable any special
processing, like key translations.
"""
# instead of allowing mutables as default keyword argument values...
if definition_source is None:
definition_source_list = []
elif (isinstance(definition_source, collections.Sequence)
and not isinstance(definition_source,
(six.binary_type, six.text_type))):
definition_source_list = list(definition_source)
else:
if isinstance(definition_source, (six.binary_type, six.text_type)):
definition_source = to_str(definition_source)
definition_source_list = [definition_source]
if argv_source is None:
self.argv_source = sys.argv[1:]
self.app_invocation_name = sys.argv[0]
else:
self.argv_source = argv_source
self.app_invocation_name = app_name
if options_banned_from_help is None:
options_banned_from_help = ['application']
self.config_pathname = config_pathname
self.config_optional = config_optional
self.use_auto_help = use_auto_help
self.value_source_object_hook = value_source_object_hook
self.app_name = app_name
self.app_version = app_version
self.app_description = app_description
self.args = [] # extra commandline arguments that are not switches
# will be stored here.
self._config = None # eventual container for DOM-like config object
self.option_definitions = Namespace()
self.definition_source_list = definition_source_list
command_line_value_source = command_line
if values_source_list is None:
# nothing set, assume defaults
if use_admin_controls:
values_source_list = (ConfigFileFutureProxy, environment,
command_line_value_source)
else:
values_source_list = (environment, command_line_value_source)
# determine which command_line facility to use for help
if self.use_auto_help:
# we need to iterate through all of our value sources looking for
# one that can interact with the user on the commandline.
for a_value_source in values_source_list:
if inspect.ismodule(a_value_source):
handler = \
type_handler_dispatch[a_value_source][0].ValueSource
try:
# if a value source is able to handle the command line
# it will have defined 'command_line_value_source' as
# true. Not all values sources may have this attribute
if handler.command_line_value_source:
handler._setup_auto_help(self)
break
except AttributeError:
# not a commandline source because it doesn't have
# the 'command_line_value_source' OR it doesn't have
# a method that allows it to setup a help system.
# this is OK, we can ignore it and move on until we
# find an appropriate source.
pass
else:
# While not actually necessary to have implemented, this
# is the case where the value source is not a module.
# So we know nothing about its interface. We cannot even
# try to use it as a commandline value source.
pass
admin_tasks_done = False
self.keys_blocked_from_output = [
'help',
'admin.conf',
'admin.dump_conf',
'admin.print_conf',
'admin.strict',
'admin.expose_secrets',
]
self.options_banned_from_help = options_banned_from_help
if use_admin_controls:
admin_options = self._setup_admin_options(values_source_list)
self.definition_source_list.append(admin_options)
# iterate through the option definitions to create the nested dict
# hierarchy of all the options called 'option_definitions'
for a_definition_source in self.definition_source_list:
try:
safe_copy_of_def_source = a_definition_source.safe_copy()
except AttributeError:
# apparently, the definition source was not in the form of a
# Namespace object. This isn't a show stopper, but we don't
# know how to make a copy of this object safely: we know from
# experience that the stock copy.copy method leads to grief
# as many sub-objects within an option definition source can
# not be copied that way (classes, for example).
# The only action we can take is to trust and continue with the
# original copy of the definition source.
safe_copy_of_def_source = a_definition_source
setup_definitions(safe_copy_of_def_source, self.option_definitions)
if use_admin_controls:
# the name of the config file needs to be loaded from the command
# line prior to processing the rest of the command line options.
config_filename = config_filename_from_commandline(self)
if (config_filename
and ConfigFileFutureProxy in values_source_list):
self.option_definitions.admin.conf.default = config_filename
self.values_source_list = wrap_with_value_source_api(
values_source_list, self)
known_keys = self._overlay_expand()
self._check_for_mismatches(known_keys)
# the app_name, app_version and app_description are to come from
# if 'application' option if it is present. If it is not present,
# get the app_name,et al, from parameters passed into the constructor.
# if those are empty, set app_name, et al, to empty strings
try:
app_option = self._get_option('application')
self.app_name = getattr(app_option.value, 'app_name', '')
self.app_version = getattr(app_option.value, 'app_version', '')
self.app_description = getattr(app_option.value, 'app_description',
'')
except NotAnOptionError:
# there is no 'application' option, continue to use the
# 'app_name' from the parameters passed in, if they exist.
pass
try:
if use_auto_help and self._get_option('help').value:
self.output_summary()
admin_tasks_done = True
except NotAnOptionError:
# the current command-line implementation already has a help
# mechanism of its own that doesn't require the use of a
# option in configman. This error is ignorable
pass
# keys that end with a "$" are called "blocked_by_suffix".
# This means that these options are not to be written out to
# configuration files.
keys_blocked_by_suffix = [
key for key in self.option_definitions.keys_breadth_first()
if key.endswith('$')
]
self.keys_blocked_from_output.extend(keys_blocked_by_suffix)
if use_admin_controls and self._get_option('admin.print_conf').value:
self.print_conf()
admin_tasks_done = True
if use_admin_controls and self._get_option('admin.dump_conf').value:
self.dump_conf()
admin_tasks_done = True
if quit_after_admin and admin_tasks_done:
sys.exit()
return True
# --------------------------------------------------------------------------
def action(self, *args, **kwargs):
"""the default action for Support Classifiers invokes any derivied
_action function, trapping any exceptions raised in the process. We
are obligated to catch these exceptions to give subsequent rules the
opportunity to act and perhaps mitigate the error. An error during the
action application is a failure of the rule, not a failure of the
classification system itself."""
try:
return self._action(*args, **kwargs)
except KeyError, x:
self.config.logger.debug('Rule %s action failed because of missing key "%s"', to_str(self.__class__), x)
except Exception, x:
self.config.logger.debug('Rule %s action failed because of "%s"', to_str(self.__class__), x, exc_info=True)
return False
# --------------------------------------------------------------------------
def _action(self, *args, **kwargs):
"""Rules derived from this base class ought to override this method
with an actual classification rule. Successful application of this
method should include a call to '_add_classification'.
returns:
True - this rule was applied successfully and no further rules
should be applied
False - this rule did not succeed and further rules should be
tried
"""
return True
are obligated to catch these exceptions to give subsequent rules the
opportunity to act and perhaps mitigate the error. An error during the
action application is a failure of the rule, not a failure of the
classification system itself."""
try:
return self._action(*args, **kwargs)
except KeyError, x:
self.config.logger.debug(
'Rule %s action failed because of missing key "%s"',
to_str(self.__class__),
x,
)
except Exception, x:
self.config.logger.debug(
'Rule %s action failed because of "%s"',
to_str(self.__class__),
x,
exc_info=True
)
return False
#--------------------------------------------------------------------------
def _action(self, *args, **kwargs):
"""Rules derived from this base class ought to override this method
with an actual classification rule. Successful application of this
method should include a call to '_add_classification'.
returns:
True - this rule was applied successfully and no further rules
should be applied
False - this rule did not succeed and further rules should be
def _write_ini(source_dict,
namespace_name=None,
level=0,
indent_size=4,
output_stream=sys.stdout):
"""this function prints the components of a configobj ini file. It is
recursive for outputing the nested sections of the ini file."""
options = [
value for value in source_dict.values()
if isinstance(value, Option)
]
options.sort(key=lambda x: x.name)
indent_spacer = " " * (level * indent_size)
for an_option in options:
print("%s# %s" % (indent_spacer, an_option.doc),
file=output_stream)
option_value = to_str(an_option)
if an_option.reference_value_from:
print('%s# see "%s.%s" for the default or override it here' %
(indent_spacer, an_option.reference_value_from,
an_option.name),
file=output_stream)
if an_option.likely_to_be_changed or an_option.has_changed:
option_format = '%s%s=%s\n'
else:
option_format = '%s#%s=%s\n'
if isinstance(option_value, six.string_types) and \
',' in option_value:
# quote lists unless they're already quoted
if option_value[0] not in '\'"':
option_value = '"%s"' % option_value
print(option_format %
(indent_spacer, an_option.name, option_value),
file=output_stream)
next_level = level + 1
namespaces = [(key, value) for key, value in source_dict.items()
if isinstance(value, Namespace)]
namespaces.sort(key=ValueSource._namespace_reference_value_from_sort)
for key, namespace in namespaces:
next_level_spacer = " " * next_level * indent_size
print("%s%s%s%s\n" %
(indent_spacer, "[" * next_level, key, "]" * next_level),
file=output_stream)
if namespace._doc:
print("%s%s" % (next_level_spacer, namespace._doc),
file=output_stream)
if namespace._reference_value_from:
print("%s#+include ./common_%s.ini\n" %
(next_level_spacer, key),
file=output_stream)
if namespace_name:
ValueSource._write_ini(source_dict=namespace,
namespace_name="%s.%s" %
(namespace_name, key),
level=level + 1,
indent_size=indent_size,
output_stream=output_stream)
else:
ValueSource._write_ini(source_dict=namespace,
namespace_name=key,
level=level + 1,
indent_size=indent_size,
output_stream=output_stream)
def test_to_str(self):
self.assertEqual(converters.to_str(1), "1")
self.assertEqual(converters.to_str(3.1415), "3.1415")
self.assertEqual(converters.to_str('hello'), "hello")
self.assertEqual(converters.to_str(u"'你好'"), u"'\u4f60\u597d'")
self.assertEqual(converters.list_to_str([1, 2, 3]), "1, 2, 3")
self.assertEqual(converters.to_str(True), "True")
self.assertEqual(converters.to_str(False), "False")
self.assertEqual(
converters.to_str(datetime.datetime(1960, 5, 4, 15, 10)),
"1960-05-04T15:10:00")
self.assertEqual(converters.to_str(datetime.date(1960, 5, 4)),
"1960-05-04")
self.assertEqual(
converters.to_str(datetime.timedelta(days=1, seconds=1)),
"1 00:00:01")
self.assertEqual(converters.to_str(int), 'int')
self.assertEqual(converters.to_str(float), 'float')
self.assertEqual(converters.to_str(unittest), 'unittest')
self.assertEqual(converters.to_str(converters.to_str),
'configman.converters.to_str')
import re
r = re.compile('.*')
self.assertEqual(converters.to_str(r), '.*')