def load_yaml(filename, **options):
loaded_yaml = AttrDict()
# Find the requested yaml in the yaml dir
if os.path.exists(filename):
with open(filename) as config_fh:
try:
loaded_yaml.update(yaml.load(config_fh, Loader=OrderedDictYAMLLoader))
except:
warn('Unable to parse configuration file at {}'.format(filename), ConfigInvalid)
return loaded_yaml
def load_yaml(filename, **options):
loaded_yaml = AttrDict()
# Find the requested yaml in the yaml dir
if os.path.exists(filename):
with open(filename) as config_fh:
try:
loaded_yaml.update(
yaml.load(config_fh, Loader=OrderedDictYAMLLoader))
except:
warn(
'Unable to parse configuration file at {}'.format(
filename), ConfigInvalid)
return loaded_yaml
def _apply_runtime_overrides(self, yaml_dict, keys, value):
base = yaml_dict
for key in keys[:-1]:
if key not in base:
base[key] = AttrDict()
base = base[key]
base[keys[-1]] = value
def __init__(self, **config):
for k, v in config.viewitems():
try: x = getattr(self, k)
except AttributeError:
raise AttributeError('Unrecognized configuration key: {}'.format(k))
if isinstance(x, Mapping) and isinstance(v, Mapping):
v = AttrDict(v)
v.rebase(AttrDict(x))
setattr(self, k, v)
pool = QuietHTTPConnectionPool(reactor, persistent=True)
for k, v in self.request_pool_options.viewitems():
getattr(pool, k) # to somewhat protect against typos
setattr(pool, k, v)
self.request_agent = ContentDecoderAgent(RedirectAgent(Agent(
reactor, TLSContextFactory(self.ca_certs_files), pool=pool )), [('gzip', GzipDecoder)])
def pytest_collection_modifyitems(session, config, items):
for item in items:
try:
item._metadata = AttrDict(item.function.meta.kwargs)
except AttributeError:
logger.warning('AttributeError getting metadata from item: {}'.format(
str(item.nodeid))
)
item._metadata = AttrDict()
meta = item.get_marker("meta")
if meta is None:
continue
metas = reversed([x.kwargs for x in meta]) # Extract the kwargs, reverse the order
for meta in metas:
item._metadata.update(meta)
yield
def pytest_collection_modifyitems(session, config, items):
from cfme.utils.log import logger
for item in items:
try:
item._metadata = AttrDict(item.function.meta.kwargs)
except AttributeError:
logger.debug('AttributeError getting metadata from item: {}'.format(
str(item.nodeid))
)
item._metadata = AttrDict()
# Extract the kwargs, reverse the order
metas = reversed([mark.kwargs
for _, mark in item.iter_markers_with_node('meta') or []])
for meta in metas:
item._metadata.update(meta)
yield
def __getitem__(self, key):
# Attempt a normal dict lookup to pull a cached conf
if key not in self:
super(Config, self).__setitem__(key, AttrDict())
# Cache miss, populate this conf key
# Call out to dict's setitem here since this is the only place where we're allowed to
# create a new key and we want the default behavior instead of the override below
self._populate(key)
return super(Config, self).__getitem__(key)
def __init__(self, **config):
for k, v in config.viewitems():
try:
x = getattr(self, k)
except AttributeError:
raise AttributeError(
'Unrecognized configuration key: {}'.format(k))
if isinstance(x, Mapping) and isinstance(v, Mapping):
v = AttrDict(v)
v.rebase(AttrDict(x))
setattr(self, k, v)
pool = QuietHTTPConnectionPool(reactor, persistent=True)
for k, v in self.request_pool_options.viewitems():
getattr(pool, k) # to somewhat protect against typos
setattr(pool, k, v)
self.request_agent = ContentDecoderAgent(
RedirectAgent(
Agent(reactor,
TLSContextFactory(self.ca_certs_files),
pool=pool)), [('gzip', GzipDecoder)])
def _populate(self, key):
yaml_dict = self._load_yaml(key)
# Graft in local yaml updates if they're available
with catch_warnings():
local_yaml = '%s.local' % key
local_yaml_dict = self._load_yaml(local_yaml, warn_on_fail=False)
if local_yaml_dict:
yaml_dict.update_dict(local_yaml_dict)
# Graft on the runtime overrides
if key in self._runtime:
AttrDict(self._runtime)[key]._.apply_flat(
partial(self._apply_runtime_overrides, yaml_dict))
self[key].update(yaml_dict)
self._inherit(key)
def _scavenge_class(cls, ignore_navigate=False):
"""Scavenges locations and nav functions from the class. Recursively goes through so no loops.
Args:
ignore_navigate: Useful for the initial - root class that has no navigate function.
"""
if "navigate" not in cls.__dict__ and not ignore_navigate:
raise ValueError(
"The nav class {} must contain navigation staticmethod".format(
cls.__name__))
elif not ignore_navigate:
navigate = (cls.navigate.im_func
if hasattr(cls.navigate, "im_func") else cls.navigate)
contents = AttrDict({"subclasses": {}, "direct_navs": {}})
for key, value in cls.__dict__.iteritems():
if key.startswith("_") or key == "navigate":
continue
if inspect.isclass(value):
contents.subclasses[value.__name__] = _scavenge_class(value)
elif callable(value) and value.__name__ != "<lambda>":
contents.direct_navs[value.__name__] = value
elif hasattr(
value, "im_func"
): # An unbound method, we just take the raw function from it
contents.direct_navs[value.__name__] = value.im_func
# Skipping others
if not contents.subclasses and not contents.direct_navs and not ignore_navigate:
# Leaf tree location generator
return navigate
elif ignore_navigate:
# Root tree location generator
result_dict = {}
result_dict.update(contents.subclasses)
result_dict.update(contents.direct_navs)
return result_dict
else:
# Non-leaf tree location generator
result_dict = {}
result_dict.update(contents.subclasses)
result_dict.update(contents.direct_navs)
return [navigate, result_dict]
def main():
import argparse
parser = argparse.ArgumentParser(
description='Collect and dispatch various metrics to destinations.')
parser.add_argument('-t', '--destination', metavar='host[:port]',
help='host[:port] (default port: 2003, can be overidden'
' via config file) of sink destination endpoint (e.g. carbon'
' linereceiver tcp port, by default).')
parser.add_argument('-i', '--interval', type=int, metavar='seconds',
help='Interval between collecting and sending the datapoints.')
parser.add_argument('-e', '--collector-enable',
action='append', metavar='collector', default=list(),
help='Enable only the specified metric collectors,'
' can be specified multiple times.')
parser.add_argument('-d', '--collector-disable',
action='append', metavar='collector', default=list(),
help='Explicitly disable specified metric collectors,'
' can be specified multiple times. Overrides --collector-enable.')
parser.add_argument('-s', '--sink-enable',
action='append', metavar='sink', default=list(),
help='Enable only the specified datapoint sinks,'
' can be specified multiple times.')
parser.add_argument('-x', '--sink-disable',
action='append', metavar='sink', default=list(),
help='Explicitly disable specified datapoint sinks,'
' can be specified multiple times. Overrides --sink-enable.')
parser.add_argument('-p', '--processor-enable',
action='append', metavar='processor', default=list(),
help='Enable only the specified datapoint processors,'
' can be specified multiple times.')
parser.add_argument('-z', '--processor-disable',
action='append', metavar='processor', default=list(),
help='Explicitly disable specified datapoint processors,'
' can be specified multiple times. Overrides --processor-enable.')
parser.add_argument('-c', '--config',
action='append', metavar='path', default=list(),
help='Configuration files to process.'
' Can be specified more than once.'
' Values from the latter ones override values in the former.'
' Available CLI options override the values in any config.')
parser.add_argument('-a', '--xattr-emulation', metavar='db-path',
help='Emulate filesystem extended attributes (used in'
' some collectors like sysstat or cron_log), storing per-path'
' data in a simple shelve db.')
parser.add_argument('-n', '--dry-run',
action='store_true', help='Do not actually send data.')
parser.add_argument('--debug',
action='store_true', help='Verbose operation mode.')
optz = parser.parse_args()
# Read configuration files
cfg = AttrDict.from_yaml('{}.yaml'.format(
os.path.splitext(os.path.realpath(__file__))[0] ))
for k in optz.config: cfg.update_yaml(k)
# Logging
import logging
configure_logging( cfg.logging,
logging.DEBUG if optz.debug else logging.WARNING )
if not cfg.logging.tracebacks:
class NoTBLogger(logging.Logger):
def exception(self, *argz, **kwz): self.error(*argz, **kwz)
logging.setLoggerClass(NoTBLogger)
log = logging.getLogger(__name__)
# Fill "auto-detected" blanks in the configuration, CLI overrides
try:
if optz.destination: cfg.sinks._default.host = optz.destination
cfg.sinks._default.host = cfg.sinks._default.host.rsplit(':', 1)
if len(cfg.sinks._default.host) == 1:
cfg.sinks._default.host =\
cfg.sinks._default.host[0], cfg.sinks._default.default_port
else: cfg.sinks._default.host[1] = int(cfg.sinks._default.host[1])
except KeyError: pass
if optz.interval: cfg.loop.interval = optz.interval
if optz.dry_run: cfg.debug.dry_run = optz.dry_run
if optz.xattr_emulation: cfg.core.xattr_emulation = optz.xattr_emulation
# Fake "xattr" module, if requested
if cfg.core.xattr_emulation:
import shelve
xattr_db = shelve.open(cfg.core.xattr_emulation, 'c')
class xattr_path(object):
def __init__(self, base):
assert isinstance(base, str)
self.base = base
def key(self, k): return '{}\0{}'.format(self.base, k)
def __setitem__(self, k, v): xattr_db[self.key(k)] = v
def __getitem__(self, k): return xattr_db[self.key(k)]
def __del__(self): xattr_db.sync()
class xattr_module(object): xattr = xattr_path
sys.modules['xattr'] = xattr_module
# Override "enabled" collector/sink parameters, based on CLI
ep_conf = dict()
for ep, enabled, disabled in\
[ ('collectors', optz.collector_enable, optz.collector_disable),
('processors', optz.processor_enable, optz.processor_disable),
('sinks', optz.sink_enable, optz.sink_disable) ]:
conf = cfg[ep]
conf_base = conf.pop('_default')
if 'debug' not in conf_base: conf_base['debug'] = cfg.debug
ep_conf[ep] = conf_base, conf, OrderedDict(), enabled, disabled
# Init global cfg for collectors/sinks' usage
from graphite_metrics import collectors, sinks, loops
collectors.cfg = sinks.cfg = loops.cfg = cfg
# Init pluggable components
import pkg_resources
for ep_type in 'collector', 'processor', 'sink':
ep_key = '{}s'.format(ep_type) # a bit of a hack
conf_base, conf, objects, enabled, disabled = ep_conf[ep_key]
ep_dict = dict( (ep.name, ep) for ep in
pkg_resources.iter_entry_points('graphite_metrics.{}'.format(ep_key)) )
eps = OrderedDict(
(name, (ep_dict.pop(name), subconf or AttrDict()))
for name, subconf in conf.viewitems() if name in ep_dict )
eps.update( (name, (module, conf_base))
for name, module in ep_dict.viewitems() )
for ep_name, (ep_module, subconf) in eps.viewitems():
if ep_name[0] == '_':
log.debug( 'Skipping {} enty point,'
' prefixed by underscore: {}'.format(ep_type, ep_name) )
subconf.rebase(conf_base) # fill in "_default" collector parameters
if enabled:
if ep_name in enabled: subconf['enabled'] = True
else: subconf['enabled'] = False
if disabled and ep_name in disabled: subconf['enabled'] = False
if subconf.get('enabled', True):
log.debug('Loading {}: {}'.format(ep_type, ep_name))
try: obj = getattr(ep_module.load(), ep_type)(subconf)
except Exception as err:
log.exception('Failed to load/init {} ({}): {}'.format(ep_type, ep_name, err))
subconf.enabled = False
obj = None
if subconf.get('enabled', True): objects[ep_name] = obj
else:
log.debug(( '{} {} (entry point: {})'
' was disabled after init' ).format(ep_type.title(), obj, ep_name))
if ep_type != 'processor' and not objects:
log.fatal('No {}s were properly enabled/loaded, bailing out'.format(ep_type))
sys.exit(1)
log.debug('{}: {}'.format(ep_key.title(), objects))
loop = dict( (ep.name, ep) for ep in
pkg_resources.iter_entry_points('graphite_metrics.loops') )
conf = AttrDict(**cfg.loop)
if 'debug' not in conf: conf.debug = cfg.debug
loop = loop[cfg.loop.name].load().loop(conf)
collectors, processors, sinks = it.imap( op.itemgetter(2),
op.itemgetter('collectors', 'processors', 'sinks')(ep_conf) )
log.debug(
'Starting main loop: {} ({} collectors, {} processors, {} sinks)'\
.format(loop, len(collectors), len(processors), len(sinks)) )
loop.start(collectors, processors, sinks)
import lya
from lya import AttrDict
from pdb import set_trace as pause
import os.path as osp
__C = AttrDict()
cfg = __C
cfg.ROOT_DIR = osp.abspath(osp.join(osp.dirname(__file__), '..', '..'))
cfg.DATA_DIR = osp.abspath(osp.join(cfg.ROOT_DIR, 'data'))
def load_config(cfg_file):
update_config(lya.AttrDict.from_yaml(cfg_file))
def update_config(cfg_other):
__C.update_dict(cfg_other)
def __init__(self):
self._settings = AttrDict.from_yaml(settings_default.as_posix())
if settings_local.exists():
self._settings.update_yaml(settings_local.as_posix())
from utils.path import project_path
from utils.timeutil import parsetime
# Default blocking time before giving up on an ssh command execution,
# in seconds (float)
RUNCMD_TIMEOUT = 1200.0
SSHResult = namedtuple("SSHResult", ["rc", "output"])
_ssh_key_file = project_path.join('.generated_ssh_key')
_ssh_pubkey_file = project_path.join('.generated_ssh_key.pub')
# enum
_ssh_keystate = AttrDict({
'not_installed': 0,
'installing': 1,
'installed': 2
})
# enum reverse lookup
_ssh_keystate.update({v: k for k, v in _ssh_keystate.items()})
_client_session = []
class SSHClient(paramiko.SSHClient):
"""paramiko.SSHClient wrapper
Allows copying/overriding and use as a context manager
Constructor kwargs are handed directly to paramiko.SSHClient.connect()
"""
def __init__(self, stream_output=False, keystate=_ssh_keystate.not_installed,
class Config(dict):
"""Configuration YAML loader and cache"""
def __init__(self, config_dir, **kwargs):
# private yaycl conf, plugins can stash info here for configuration
self._yaycl = AttrDict({'config_dir': config_dir})
self._runtime = ConfigTree(self)
self._extension = kwargs.pop('extension', '.yaml')
self._yaycl.update(kwargs)
def _runtime_overrides(self):
return self._runtime
def _set_runtime_overrides(self, overrides_dict):
self._runtime.update(overrides_dict)
def _del_runtime_overrides(self):
self._runtime.clear()
runtime = property(_runtime_overrides, _set_runtime_overrides,
_del_runtime_overrides)
def save(self, key):
"""Write out an in-memory config to the conf dir
Warning: This will destroy any formatting or ordering that existed in the original yaml
"""
with open(os.path.join(self._yaycl.config_dir, '%s.yaml' % key),
'w') as conf_file:
self[key].dump(conf_file)
# Support for descriptor access, e.g. instance.attrname
# Note that this is only on the get side, for support of nefarious things
# like setting and deleting, use the normal dict interface.
def __getattribute__(self, attr):
# Attempt normal object attr lookup; delegate to the dict interface if that fails
try:
return super(Config, self).__getattribute__(attr)
except AttributeError:
# try to load from cache first if this conf is already known
if attr in self:
return self[attr]
if attr.startswith('_'):
# don't try to load private names
raise
# If we're here, trigger the loader via getitem
return self[attr]
def __getitem__(self, key):
# Attempt a normal dict lookup to pull a cached conf
if key not in self:
super(Config, self).__setitem__(key, AttrDict())
# Cache miss, populate this conf key
# Call out to dict's setitem here since this is the only place where we're allowed to
# create a new key and we want the default behavior instead of the override below
self._populate(key)
return super(Config, self).__getitem__(key)
def __setitem__(self, key, value):
self[key].clear()
self[key].update(value)
def __delitem__(self, key):
self[key].clear()
self._populate(key)
def _inherit(self, conf_key):
"""Recurses through an object looking for 'inherit' clauses and replaces them with their
real counterparts. In the case of a dict, the inherit clause remains, in the case of
anything else, a replacement occurs such that:
sim5:
key: value
newkey: newvalue
sim6:
tags:
- tag1
- tag2
sim7:
inherit: management_systems/sim5
test:
tags:
inherit: management_systems/sim6/tags
Will produce the following output if requesting management_systems/sim7
inherit: management_systems/sim5
key: value
newkey: newvalue
test:
tags:
- tag1
- tag2
"""
for keys in self._needs_inherit(conf_key):
# get the dict containing inherit key
keys, root_key = keys[:-1], keys[-1]
root = self[conf_key]
for k in keys:
root = root[k]
# find the dict we're inheriting based on value
base = self[conf_key]
try:
for path_element in root[root_key]['inherit'].split('/'):
base = base[path_element]
except KeyError:
warn(
'{} path cannot be traversed, {} does not exist'.format(
root[root_key]['inherit'], path_element),
InvalidInheritPath)
# rebase if the base was an attrdict,
# otherwise overwrite the key in-place
if isinstance(base, AttrDict):
del (root[root_key]['inherit'])
root[root_key].rebase(base)
else:
root[root_key] = base
def _needs_inherit(self, conf_key):
conf = self[conf_key]
# loop over keys until all the inherits are gone
while True:
seen_inherit = False
for k, v in conf.flatten_dict(conf):
if k[-1] == 'inherit':
# give back the keys needed to get to a dict containing an inherit key
seen_inherit = True
yield k[:-1]
if not seen_inherit:
break
def _populate(self, key):
yaml_dict = self._load_yaml(key)
# Graft in local yaml updates if they're available
with catch_warnings():
local_yaml = '%s.local' % key
local_yaml_dict = self._load_yaml(local_yaml, warn_on_fail=False)
if local_yaml_dict:
yaml_dict.update_dict(local_yaml_dict)
# Graft on the runtime overrides
if key in self._runtime:
AttrDict(self._runtime)[key]._.apply_flat(
partial(self._apply_runtime_overrides, yaml_dict))
self[key].update(yaml_dict)
self._inherit(key)
def _apply_runtime_overrides(self, yaml_dict, keys, value):
base = yaml_dict
for key in keys[:-1]:
if key not in base:
base[key] = AttrDict()
base = base[key]
base[keys[-1]] = value
def clear(self):
# because of the 'from conf import foo' mechanism, we need to clear each key in-place,
# and reload the runtime overrides. Once a key is created in this dict, its value MUST NOT
# change to a different dict object.
for key in self:
# clear the conf dict in-place
self[key].clear()
self._populate(key)
def _load_yaml(self, conf_key, warn_on_fail=True):
return config_file(self.file_path(conf_key),
warn_on_fail=warn_on_fail,
**self._yaycl)
def file_path(self, conf_key):
file_name = '{}{}'.format(conf_key, self._extension)
return os.path.join(self._yaycl.config_dir, file_name)
class Config(dict):
"""Configuration YAML loader and cache"""
def __init__(self, config_dir, **kwargs):
# private yaycl conf, plugins can stash info here for configuration
self._yaycl = AttrDict({'config_dir': config_dir})
self._runtime = ConfigTree(self)
self._extension = kwargs.pop('extension', '.yaml')
self._yaycl.update(kwargs)
def _runtime_overrides(self):
return self._runtime
def _set_runtime_overrides(self, overrides_dict):
self._runtime.update(overrides_dict)
def _del_runtime_overrides(self):
self._runtime.clear()
runtime = property(_runtime_overrides, _set_runtime_overrides, _del_runtime_overrides)
def save(self, key):
"""Write out an in-memory config to the conf dir
Warning: This will destroy any formatting or ordering that existed in the original yaml
"""
with open(os.path.join(self._yaycl.config_dir, '%s.yaml' % key), 'w') as conf_file:
self[key].dump(conf_file)
# Support for descriptor access, e.g. instance.attrname
# Note that this is only on the get side, for support of nefarious things
# like setting and deleting, use the normal dict interface.
def __getattribute__(self, attr):
# Attempt normal object attr lookup; delegate to the dict interface if that fails
try:
return super(Config, self).__getattribute__(attr)
except AttributeError:
# try to load from cache first if this conf is already known
if attr in self:
return self[attr]
if attr.startswith('_'):
# don't try to load private names
raise
# If we're here, trigger the loader via getitem
return self[attr]
def __getitem__(self, key):
# Attempt a normal dict lookup to pull a cached conf
if key not in self:
super(Config, self).__setitem__(key, AttrDict())
# Cache miss, populate this conf key
# Call out to dict's setitem here since this is the only place where we're allowed to
# create a new key and we want the default behavior instead of the override below
self._populate(key)
return super(Config, self).__getitem__(key)
def __setitem__(self, key, value):
self[key].clear()
self[key].update(value)
def __delitem__(self, key):
self[key].clear()
self._populate(key)
def _inherit(self, conf_key):
"""Recurses through an object looking for 'inherit' clauses and replaces them with their
real counterparts. In the case of a dict, the inherit clause remains, in the case of
anything else, a replacement occurs such that:
sim5:
key: value
newkey: newvalue
sim6:
tags:
- tag1
- tag2
sim7:
inherit: management_systems/sim5
test:
tags:
inherit: management_systems/sim6/tags
Will produce the following output if requesting management_systems/sim7
inherit: management_systems/sim5
key: value
newkey: newvalue
test:
tags:
- tag1
- tag2
"""
for keys in self._needs_inherit(conf_key):
# get the dict containing inherit key
keys, root_key = keys[:-1], keys[-1]
root = self[conf_key]
for k in keys:
root = root[k]
# find the dict we're inheriting based on value
base = self[conf_key]
try:
for path_element in root[root_key]['inherit'].split('/'):
base = base[path_element]
except KeyError:
warn('{} path cannot be traversed, {} does not exist'.format(
root[root_key]['inherit'], path_element), InvalidInheritPath)
# rebase if the base was an attrdict,
# otherwise overwrite the key in-place
if isinstance(base, AttrDict):
del(root[root_key]['inherit'])
root[root_key].rebase(base)
else:
root[root_key] = base
def _needs_inherit(self, conf_key):
conf = self[conf_key]
# loop over keys until all the inherits are gone
while True:
seen_inherit = False
for k, v in conf.flatten_dict(conf):
if k[-1] == 'inherit':
# give back the keys needed to get to a dict containing an inherit key
seen_inherit = True
yield k[:-1]
if not seen_inherit:
break
def _populate(self, key):
yaml_dict = self._load_yaml(key)
# Graft in local yaml updates if they're available
with catch_warnings():
local_yaml = '%s.local' % key
local_yaml_dict = self._load_yaml(local_yaml, warn_on_fail=False)
if local_yaml_dict:
yaml_dict.update_dict(local_yaml_dict)
# Graft on the runtime overrides
if key in self._runtime:
AttrDict(self._runtime)[key]._.apply_flat(
partial(self._apply_runtime_overrides, yaml_dict))
self[key].update(yaml_dict)
self._inherit(key)
def _apply_runtime_overrides(self, yaml_dict, keys, value):
base = yaml_dict
for key in keys[:-1]:
if key not in base:
base[key] = AttrDict()
base = base[key]
base[keys[-1]] = value
def clear(self):
# because of the 'from conf import foo' mechanism, we need to clear each key in-place,
# and reload the runtime overrides. Once a key is created in this dict, its value MUST NOT
# change to a different dict object.
for key in self:
# clear the conf dict in-place
self[key].clear()
self._populate(key)
def _load_yaml(self, conf_key, warn_on_fail=True):
return config_file(self.file_path(conf_key), warn_on_fail=warn_on_fail, **self._yaycl)
def file_path(self, conf_key):
file_name = '{}{}'.format(conf_key, self._extension)
return os.path.join(self._yaycl.config_dir, file_name)
def __init__(self, config_dir, **kwargs):
# private yaycl conf, plugins can stash info here for configuration
self._yaycl = AttrDict({'config_dir': config_dir})
self._runtime = ConfigTree(self)
self._extension = kwargs.pop('extension', '.yaml')
self._yaycl.update(kwargs)
def __init__(self, config_dir, **kwargs):
# private yaycl conf, plugins can stash info here for configuration
self._yaycl = AttrDict({'config_dir': config_dir})
self._runtime = ConfigTree(self)
self._extension = kwargs.pop('extension', '.yaml')
self._yaycl.update(kwargs)
def main():
import argparse
parser = argparse.ArgumentParser(
description='Check integrity of mirrored files.')
parser.add_argument('-c', '--config',
action='append', metavar='path', default=list(),
help='Configuration files to process.'
' Can be specified more than once.'
' Values from the latter ones override values in the former.'
' Available CLI options override the values in any config.')
parser.add_argument('-m', '--mtime-after',
type=int, metavar='unix_time',
help='Only act on files with'
' mtime larger than the given value (unix timestamp).')
parser.add_argument('-l', '--list', nargs='?', metavar='types', default=False,
help='Instead of usual action, just dump current state of all the files and exit.'
'Can take an optional "type" (of mirrors wrt file) argument (comma-separated type(s)):'
' consistent, inconsistent, unavailable, undergoal (default: inconsistent, unavailable).')
parser.add_argument('-n', '--skip-nx',
action='store_true', help='Do not retry known-404 mirrors.')
parser.add_argument('--debug',
action='store_true', help='Verbose operation mode.')
optz = parser.parse_args()
## Read configuration files
from lya import AttrDict
cfg = AttrDict.from_yaml('{}.yaml'.format(
os.path.splitext(os.path.realpath(__file__))[0] ))
for k in optz.config: cfg.update_yaml(k)
## Logging
import logging
logging.basicConfig(
level=logging.WARNING if not optz.debug else logging.DEBUG )
global log
log = logging.getLogger()
## Modules
modules_manifest_db = dict(dbm=ManifestDBM)
modules_remote = dict(
(k, ft.partial(v, conf=cfg.checks.get(k.split('__', 1)[0], dict())))
for k,v in dict( gentoo_portage=check_portage,
rsync=check_rsync, rsync__batched=check_rsync_batched,
mirrors=check_mirror ).viewitems() )
## Catch-up with local fs
check_fs_ts = time()
manifest_db = modules_manifest_db[cfg.manifest.type](cfg.manifest)
log.debug('Updating manifest-db with hashes of local files')
for path in cfg.local:
check_fs( path, manifest_db, hashes=cfg.manifest.hashes,
local_changes_warn=cfg.goal.warn.local_changes,
ts=check_fs_ts, ts_min=optz.mtime_after )
log.debug('Manifest-db cleanup')
ts_gc_min = check_fs_ts - cfg.goal.gc_timeout * 24 * 3600
if optz.mtime_after and optz.mtime_after < ts_gc_min:
check_gc(manifest_db, ts_min=ts_gc_min)
## List of remotes in order of preference
remotes = list()
for rtype, urls in cfg.remote.viewitems():
for url in urls or list(): remotes.append((rtype, url))
## Build a set of excluded filename-patterns
exclude = set(cfg.exclude.patterns or set())
for src in cfg.exclude.from_files or list():
with open(src, 'rb') as src:
for pat in it.ifilter( None,
it.imap(op.methodcaller('strip'), src) ):
exclude.add(pat)
## Build a large-enough list of stuff to be checked
qratio = int(len(remotes) * cfg.goal.query.ratio)
qmin = min(len(remotes), cfg.goal.query.hard_min or qratio)
qmax = min(len(remotes), max(qmin, cfg.goal.query.hard_max or qratio))
limit = (cfg.goal.limit.files or None) if not optz.list else None
undergoal = manifest_db.undergoal_order(
qmin, qmax, limit=limit,
remotes=remotes, ts_min=optz.mtime_after )
# Check that each listed path actually exists now and isn't conf-excluded
drop = set()
for path in undergoal:
for pat in exclude:
if fnmatch(pat, basename(path)):
drop.add(path)
continue
try:
meta = manifest_db[path]
if meta['ts'] != check_fs_ts and not os.path.exists(path):
raise KeyError(path)
except KeyError:
log.debug('Skipping check for unlisted/nx path: %s', path)
drop.add(path)
undergoal = list(path for path in undergoal if path not in drop)
## Just dump the list, if requested
if optz.list is not False:
log.debug('Just listing all the remotes')
if not optz.list: optz.list = 'inconsistent, unavailable'
mtypes = sorted(it.imap(op.methodcaller('strip'), optz.list.split(',')))
mtypes_err = set(mtypes).difference([
'consistent', 'inconsistent', 'unavailable', 'undergoal' ])
if mtypes_err:
parser.error('Unrecognized check states: {}'.format(', '.join(mtypes_err)))
if 'undergoal' in mtypes:
undergoal_check = True
mtypes = sorted(
set(mtypes).difference(['undergoal'])\
.union(['consistent', 'inconsistent', 'unavailable']) )
else: undergoal_check = False
for path in undergoal:
meta_remotes = manifest_db[path].get('remotes', dict())
if undergoal_check and not (
len(set(meta_remotes.get(
'consistent', set() )).intersection(remotes)) < qmin\
or set(meta_remotes.get(
'inconsistent', set() )).intersection(remotes) ): continue
if not meta_remotes:
print('Path: {}\n No consistency data available'.format(path))
continue
path_line = False
for mtype in mtypes:
mtype_remotes = set(meta_remotes.get(mtype, set())).intersection(remotes)
if not mtype_remotes: continue
if not path_line:
print('Path: {}'.format(path))
path_line = True
print(' {}{}'.format(mtype, ' ({}{}):\n {}'.format(
len(mtype_remotes), ''
if mtype != 'consistent' else ', min/max: {}/{}'.format(qmin, qmax),
'\n '.join(it.starmap('({}) {}'.format, sorted(mtype_remotes))) )))
sys.exit()
## Check against remotes
log.debug('Checking manifest-db against remotes')
check_remotes(
undergoal, remotes, manifest_db, modules_remote,
thresh_err=min( cfg.goal.warn.inconsistency.max,
int(len(remotes) * cfg.goal.warn.inconsistency.ratio) ),
thresh_nx=cfg.goal.warn.unavailable, thresh_bug=cfg.goal.limit.errors,
skip_nx=optz.skip_nx, warn_skip=cfg.goal.warn.skipped_remote )
log.debug('Done')
#!/usr/bin/env python
import sys
import argparse
from lya import AttrDict
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-f', '--file', action='store', default='/etc/goodrain/docker-compose.yaml', help='yaml file, default is %(default)s')
parser.add_argument('-u', '--update', action='store_true', help="update yaml config")
parser.add_argument('-d', '--delete', action='store', help="delete a service")
parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), default=sys.stdin)
args = parser.parse_args()
yaml_file = args.file
cfg = AttrDict.from_yaml(yaml_file)
if args.update:
cfg.update_yaml(args.infile)
with open(yaml_file, 'w') as f:
cfg.dump(f)
elif args.delete:
service = args.delete
if 'services' in cfg:
if service in cfg.services:
cfg.services.pop(service)
if not cfg.services:
cfg.pop('services')
with open(yaml_file, 'w') as f:
cfg.dump(f)
def __init__(self, master=None):
"""Creates application main window with sizes self.WIDTH and self.HEIGHT.
:param master: instance - Tkinter.Tk instance
"""
super(Application, self).__init__(master)
self.master.title('Engine Game')
self.master.geometry('{}x{}'.format(self.WIDTH, self.HEIGHT))
self.master.protocol('WM_DELETE_WINDOW', self.exit)
self.source = None
self._map = None
self.points = None
self.lines = None
self.captured_point = None
self.x0 = None
self.y0 = None
self.scale_x = None
self.scale_y = None
self.font_size = None
self.coordinates = {}
self.captured_lines = {}
self.canvas_obj = AttrDict()
self.icons = {
0: PhotoImage(file=join('icons', 'player_city.png')),
1: PhotoImage(file=join('icons', 'city.png')),
2: PhotoImage(file=join('icons', 'market.png')),
3: PhotoImage(file=join('icons', 'store.png')),
4: PhotoImage(file=join('icons', 'point.png')),
5: PhotoImage(file=join('icons', 'player_train.png')),
6: PhotoImage(file=join('icons', 'train.png')),
7: PhotoImage(file=join('icons', 'crashed_train.png')),
8: PhotoImage(file=join('icons', 'collision.png')),
9: PhotoImage(file=join('icons', 'play.png')),
10: PhotoImage(file=join('icons', 'play_pressed.png')),
11: PhotoImage(file=join('icons', 'stop.png')),
12: PhotoImage(file=join('icons', 'stop_pressed.png'))
}
self.queue_requests = {
0: self.set_status_bar,
1: self.set_player_idx,
2: self.build_map,
3: self.refresh_map,
4: self.set_available_games,
99: self.bot_control
}
self.settings_window = None
if exists(expanduser(self.DEFAULTS)):
with open(expanduser(self.DEFAULTS), 'r') as cfg:
defaults = DefaultsDict.from_yaml(cfg)
self.host = None if not defaults.host else str(defaults.host)
self.port = None if not defaults.port else int(defaults.port)
self.timeout = None if not defaults.timeout else int(defaults.timeout)
self.username = None if not defaults.username else str(defaults.username)
self.password = None if not defaults.password else str(defaults.password)
else:
self.host, self.port, self.timeout, self.username, self.password = None, None, None, None, None
self.player_idx = None
self.posts = {}
self.trains = {}
self.select_game_window = False
self.available_games = None
self.game = None
self.num_players = None
self.num_turns = None
self.bot = Bot()
self.bot_thread = None
self.menu = Menu(self)
filemenu = Menu(self.menu)
filemenu.add_command(label='Open file', command=self.file_open)
filemenu.add_command(label='Server settings', command=self.open_server_settings)
filemenu.add_command(label='Select game', command=self.select_game)
filemenu.add_command(label='Exit', command=self.exit)
self.menu.add_cascade(label='Menu', menu=filemenu)
master.config(menu=self.menu)
self._status_bar = StringVar()
self.label = Label(master, textvariable=self._status_bar)
self.label.pack()
self.frame = Frame(self)
self.frame.bind('<Configure>', self._resize_frame)
self.canvas = Canvas(self.frame, bg=self.BG, scrollregion=(0, 0, self.winfo_width(), self.winfo_height()))
self.canvas.bind('<Button-1>', self._capture_point)
self.canvas.bind('<Motion>', self._move_point)
self.canvas.bind('<B1-ButtonRelease>', self._release_point)
self.canvas.bind('<Configure>', self._resize_canvas)
hbar = Scrollbar(self.frame, orient=HORIZONTAL)
hbar.pack(side=BOTTOM, fill=X)
hbar.config(command=self.canvas.xview)
vbar = Scrollbar(self.frame, orient=VERTICAL)
vbar.pack(side=RIGHT, fill=Y)
vbar.config(command=self.canvas.yview)
self.canvas.config(xscrollcommand=hbar.set, yscrollcommand=vbar.set)
self.canvas.pack(fill=BOTH, expand=True)
self.play = Label(self.canvas, bg='white')
self.play.configure(image=self.icons[9])
self.play.bind('<Button-1>', self._play_press)
self.play.bind('<B1-ButtonRelease>', self._play_release)
self.stop = Label(self.canvas, bg='white')
self.stop.configure(image=self.icons[11])
self.stop.bind('<Button-1>', self._stop_press)
self.stop.bind('<B1-ButtonRelease>', self._stop_release)
self.frame.pack(fill=BOTH, expand=True)
self.weighted = IntVar(value=1)
self.weighted_check = Checkbutton(self, text='Proportionally to length', variable=self.weighted,
command=self._proportionally)
self.weighted_check.pack(side=LEFT)
self.show_weight = IntVar()
self.show_weight_check = Checkbutton(self, text='Show length', variable=self.show_weight,
command=self.show_weights)
self.show_weight_check.pack(side=LEFT)
self.pack(fill=BOTH, expand=True)
self.requests_executor()
self.get_available_games()
self.set_status_bar('Click Play to start the game')
self.play.place(rely=0.5, relx=0.5, anchor=CENTER)