class Problem(Base):
_attrs = dict(constraints=Attr(List(Constraint)), domain=Attr(Domain))
_opts = dict(init_validate=True, args=('domain', 'constraints'))
@init_hook
def _init(self):
self.domain = self.domain.copy()
self.var_constraint = defaultdict(set)
for con in self.constraints:
con.preprocess(self.domain)
for var in con.args:
self.var_constraint[var].add(con)
def check(self, theory):
for con in self.constraints:
if set(con.args) <= set(theory):
if not con.check(**theory):
return False
return True
def display(self, **kwargs):
strs = [self.domain.display(**kwargs)]
for con in self.constraints:
s = con.display(**kwargs)
if s:
strs.append(s)
return '\n'.join(strs)
def validate(self):
super(Problem, self).validate()
if not set(self.var_constraint).issubset(set(self.domain)):
raise ValueError('Some constraints defined in over '
'undefined variables')
class ClassWrapper(SetLeaf):
'''The idea is that a type implicitly represents the set of all of its
subclasses, including itself.
'''
_attrs = dict(type=Attr(type, doc=''),
subclasses=Attr(List(type),
doc='',
init=lambda self:
([self.type] + subclasses(self.type))))
_opts = dict(args=('type', ))
def __init__(self, *args, **kwargs):
super(ClassWrapper, self).__init__(*args, **kwargs)
self.subclasses = [self.type] + subclasses(self.type)
def display(self, **kwargs):
return 'ClassWrapper({})'.format(get_typename(self.type))
def size(self):
return len(self.subclasses)
def hasmember(self, item):
return item in self.subclasses
def sample(self, **kwargs):
ret = choice(self.subclasses)
return ret
def enumerate(self, **kwargs):
args = Args(**kwargs)
maxenum = min(args.max_enumerate, len(self.subclasses))
for k, item in enumerate(self.subclasses):
if k >= maxenum:
break
yield item
def to_set(self, **kwargs):
return super(SetLeaf, self).to_set(**kwargs)
def from_yaml(cls, name, dct):
if isinstance(dct, STR):
return cls(commands=[Command(dct)])
elif List(STR).query(dct):
cmds = [Command(s) for s in dct]
return cls(commands=cmds)
elif isinstance(dct, dict):
if set(dct.keys()).issuperset({'command'}):
ret = Task.from_yaml(name, dct['command'])
if 'context' in dct:
for cmd in ret.commands:
cmd.context = dct['context']
if 'args' in dct:
ret.args = dct['args']
if 'kwargs' in dct:
ret.kwargs = dct['kwargs']
if 'for' in dct:
ret.loop = For.from_yaml(dct['for'])
if 'if' in dct:
ret.condition = Task.from_yaml(name + '-if', dct['if'])
ret.condition_type = True
elif 'ifnot' in dct:
ret.condition = Task.from_yaml(name + '-ifnot', dct['ifnot'])
ret.condition_type = False
ret.validate()
return ret
raise ValidationError('Invalid data for task: {}'.format(name))
class For(Base):
_attrs = dict(var = Attr((STR, List(STR)), doc='The loop variable(s)'),
in_ = Attr((STR, List((STR, int, list))),
doc='Name(s) of list macro(s) to loop over'))
_opts = dict(init_validate = True,
args = ('var', 'in_'))
@classmethod
def from_yaml(cls, dct):
if isinstance(dct, STR):
m = re.match(FOREX, dct)
if m:
return cls(var=m.groups()[0], in_=m.groups()[1])
else:
raise ValidationError('Invalid for loop specifier: {}'
.format(dct))
kwargs = {}
get_delete(dct, kwargs, 'var', None)
get_delete(dct, kwargs, 'in', None, 'in_')
if dct:
raise ValidationError('Invalid for keys: {}'
.format(','.join(dct.keys())))
return cls(**kwargs)
def resolve_macros(self, env, **kwargs):
var = self.var
in_ = self.in_
if not isinstance(var, list):
var = [self.var]
in_ = [self.in_]
outs = []
for k, v in enumerate(var):
name = in_[k]
if isinstance(name, list):
val = name
else:
val = env.env[name]
if not isinstance(val, list):
raise ValidationError('For loop "in" specifier must be name of '
'list macro: {}'.format(in_[k]))
outs.append(val)
return var, outs
def loop(self, env, **kwargs):
var, in_ = self.resolve_macros(env, **kwargs)
for tup in product(*in_):
yld = env.copy(**kwargs)
for name, val in zip(var, tup):
yld.env[name] = val
yield yld
def validate(self):
super(For, self).validate()
if isinstance(self.var, list):
if len(self.var) != len(self.in_):
raise ValidationError('"var" and "in" lists must be same '
'length')
class Task(Base):
_attrs = dict(commands = Attr(List(Command)),
condition = Attr(This, optional=True),
loop = Attr(For, optional=True),
args = Attr(List((STR, int)), init=lambda self: list()),
kwargs = Attr(Dict((STR, int)), init=lambda self: dict()),
condition_type = Attr(bool, True))
_opts = dict(init_validate = True,
optional_none = True)
@classmethod
def from_yaml(cls, name, dct):
if isinstance(dct, STR):
return cls(commands=[Command(dct)])
elif List(STR).query(dct):
cmds = [Command(s) for s in dct]
return cls(commands=cmds)
elif isinstance(dct, dict):
if set(dct.keys()).issuperset({'command'}):
ret = Task.from_yaml(name, dct['command'])
if 'context' in dct:
for cmd in ret.commands:
cmd.context = dct['context']
if 'args' in dct:
ret.args = dct['args']
if 'kwargs' in dct:
ret.kwargs = dct['kwargs']
if 'for' in dct:
ret.loop = For.from_yaml(dct['for'])
if 'if' in dct:
ret.condition = Task.from_yaml(name + '-if', dct['if'])
ret.condition_type = True
elif 'ifnot' in dct:
ret.condition = Task.from_yaml(name + '-ifnot', dct['ifnot'])
ret.condition_type = False
ret.validate()
return ret
raise ValidationError('Invalid data for task: {}'.format(name))
def run(self, env, **kwargs):
codes = []
looping = kwargs.get('looping', False)
exit_on_error = kwargs.get('exit_on_error',
env.settings.get('exit_on_error', True))
preview_conditionals = kwargs.get('preview_conditionals',
env.settings.get('preview_conditionals', True))
if self.condition and not looping:
if preview_conditionals:
kwargs['preview_pre'] = 'if: ' if self.condition_type else 'ifnot: '
codes_ = self.condition.run(env, **kwargs)
code = max(codes_)
if (((self.condition_type is True and code != 0) or
(self.condition_type is False and code == 0)) and
code is not None):
return []
if preview_conditionals:
kwargs['preview_pre'] = '\t'
if (self.args or self.kwargs) and not looping:
env = env.copy(**kwargs)
if self.args:
for k, arg in enumerate(self.args):
env.env['_{}'.format(k + 1)] = arg
if self.kwargs:
for name, value in self.kwargs.items():
env.env[name] = value
if self.loop and not looping:
n = 0
kwargs['looping'] = True
for env_ in self.loop.loop(env, **kwargs):
env_.env[env_.settings['loop_count_macro']] = n
codes_ = self.run(env_, **kwargs)
codes.extend(codes_)
n += 1
return codes
for cmd in self.commands:
if cmd.command in env.tasks:
codes_ = env.tasks[cmd.command].run(env, **kwargs)
codes.extend(codes_)
else:
code = cmd.run(env, **kwargs)
codes.append(code)
if exit_on_error and any(c != 0 for c in codes if c is not None):
break
return codes
def run_preview(self, env, **kwargs):
kwargs['preview'] = True
kwargs['verbose'] = True
kwargs['run_preview'] = True
with assign(sys, 'stdout', cStringIO()):
self.run(env, **kwargs)
ret = sys.stdout.getvalue()
return ret
class Document(Base):
_attrs = dict(imports = Attr(List(STR), init=lambda self:list()),
includes = Attr(List(STR), init=lambda self: list()),
secrets = Attr(List(STR), init=lambda self: list()),
macros = Attr(Dict((STR, int, float,
List((STR, int, float, list, dict)),
Dict((STR, int, float, list, dict)))),
init=lambda self: dict()),
contexts = Attr(Dict(Context), init=lambda self: dict()),
tasks = Attr(Dict(Task), init=lambda self: dict()),
secret_values = Attr(Dict(STR), init=lambda self: dict()),
captures = Attr(Dict(STR), init=lambda self: dict()),
files = Attr(Dict(STR), init=lambda self: dict()),
settings = Attr(Dict(None), init=lambda self: dict()),
default_task = Attr(STR, '', 'Task to run if no task is '
'specified at the command line'),
env = Attr(Env, init=lambda self: Env(), internal=True),
dirname = Attr(STR, doc='Relative path for includes'),
cachedir = Attr(STR, '', 'Directory to store downloaded files'),
pull = Attr(bool, False, 'Force-pull URLs'),
)
_opts = dict(init_validate = True)
@classmethod
def from_path(cls, path, **kwargs):
with open(path, 'r') as f:
dct = yaml.load(f)
return cls.from_yaml(dct,
os.path.abspath(os.path.dirname(path)),
**kwargs)
@classmethod
def from_yaml(cls, dct, dirname, **kwargs):
get_delete(dct, kwargs, 'import', [], 'imports')
get_delete(dct, kwargs, 'include', [], 'includes')
get_delete(dct, kwargs, 'capture', {}, 'captures')
get_delete(dct, kwargs, 'files', {})
get_delete(dct, kwargs, 'secrets', [])
get_delete(dct, kwargs, 'macros', {})
get_delete(dct, kwargs, 'contexts', {})
get_delete(dct, kwargs, 'tasks', {})
get_delete(dct, kwargs, 'default', '', 'default_task')
init_macros = kwargs.get('initial_macros', {})
for name, value in init_macros.items():
if name not in kwargs['macros']:
kwargs['macros'][name] = value
settings = dict(dct.get('settings', {}))
settings.update(kwargs.get('settings', {}))
kwargs['settings'] = settings
if 'settings' in dct:
del dct['settings']
if dct:
raise ValidationError('Invalid top-level keys: {}'
.format(','.join(dct.keys())))
for key in list(kwargs['contexts'].keys()):
kwargs['contexts'][key] = \
Context.from_yaml(key, kwargs['contexts'][key])
for key in list(kwargs['tasks'].keys()):
kwargs['tasks'][key] = \
Task.from_yaml(key, kwargs['tasks'][key])
kwargs['dirname'] = dirname
return cls(**kwargs)
@init_hook
def process(self, **kwargs):
cachedir = DEFAULT_CACHE_DIR
if self.cachedir:
cachedir = self.cachedir
self.process_settings(**kwargs)
jenv = Env().jenv
pre_macros = dict(self.macros)
for name, macro in ordered_macros(pre_macros, lenient=True):
try:
pre_macros[name] = resolve(macro, pre_macros, lenient=True,
jenv=jenv)
except:
pass # There might be macros defined in terms of
# jinja_functions to be imported
def process(path):
return resolve_url(resolve(path, pre_macros, jenv=jenv),
cachedir=cachedir, force=self.pull)
with chdir(self.dirname):
for path in map(process, self.imports):
if not os.path.exists(path):
raise ValidationError("Module path does not exist: {}"
.format(path))
self.process_import(path, **kwargs)
for path in map(process, self.includes):
if not os.path.isfile(path):
raise ValidationError("Include path does not exist: {}"
.format(path))
self.process_include(path, **kwargs)
files = {}
for fname, fpath in self.files.items():
path = process(fpath)
if not os.path.isfile(path):
raise ValidationError("File path does not exist: {}"
.format(path))
files[fname] = path
for name in self.secrets:
self.process_secret(name, **kwargs)
env = Env(macros=self.macros, contexts=self.contexts, tasks=self.tasks,
secret_values=self.secret_values, captures=self.captures,
settings=self.settings, default_task=self.default_task,
files=files)
self.env.update(env, **kwargs)
def post_process(self, **kwargs):
with chdir(self.dirname):
self.env.resolve_macros()
self.validate()
def process_import(self, path, **kwargs):
mod = imp.load_source('yatr_module_import', path)
if not hasattr(mod, 'env'):
raise ImportError("yatr extension module '{}' has no env"
.format(path))
self.env.update(mod.env, **kwargs)
def process_include(self, path, **kwargs):
doc = Document.from_path(path, pull=self.pull, cachedir=self.cachedir,
settings=self.settings)
self.env.update(doc.env, **kwargs)
def process_secret(self, name, **kwargs):
raise NotImplementedError('Secrets currently unsupported')
def process_settings(self, **kwargs):
self.settings['silent'] = \
str_to_bool(self.settings.get('silent', False))
self.settings['loop_count_macro'] = \
self.settings.get('loop_count_macro', '_n')
self.settings['preview_conditionals'] = \
str_to_bool(self.settings.get('preview_conditionals', True))
self.settings['exit_on_error'] = \
str_to_bool(self.settings.get('exit_on_error', True))
def run(self, name, **kwargs):
try:
with chdir(self.dirname):
return self.env.tasks[name].run(self.env, **kwargs)
except KeyError:
raise RuntimeError('No such task: {}'.format(name))
def validate(self):
super(Document, self).validate()
self.env.validate()
import os
import imp
import yaml
from syn.base_utils import chdir
from syn.base import Base, Attr, init_hook
from syn.type import List, Dict
from syn.five import STR
from .base import ValidationError, resolve_url, resolve, ordered_macros,\
DEFAULT_CACHE_DIR, str_to_bool, get_delete
from .context import Context
from .task import Task
from .env import Env
STRList = List(STR)
#-------------------------------------------------------------------------------
# Document
class Document(Base):
_attrs = dict(imports = Attr(List(STR), init=lambda self:list()),
includes = Attr(List(STR), init=lambda self: list()),
secrets = Attr(List(STR), init=lambda self: list()),
macros = Attr(Dict((STR, int, float,
List((STR, int, float, list, dict)),
Dict((STR, int, float, list, dict)))),
init=lambda self: dict()),
contexts = Attr(Dict(Context), init=lambda self: dict()),
tasks = Attr(Dict(Task), init=lambda self: dict()),
secret_values = Attr(Dict(STR), init=lambda self: dict()),
captures = Attr(Dict(STR), init=lambda self: dict()),
class Env(Base, Copyable, Updateable):
_groups = (UP, AUP)
_attrs = dict(macros = Attr(Dict((STR, int, float,
List((STR, int, float, list, dict)),
Dict((STR, int, float, list, dict)))),
init=lambda self: dict(),
doc='Macro definitions', groups=(UP, CP)),
contexts = Attr(Dict(Context), init=lambda self: dict(),
doc='Execution context definitions',
groups=(UP, CP)),
tasks = Attr(Dict(Task), init=lambda self: dict(),
doc='Task definitions', groups=(UP, CP)),
secret_values = Attr(Dict(STR), init=lambda self: dict(),
doc='Secret value store',
groups=(UP, CP)),
captures = Attr(Dict(STR), init=lambda self: dict(),
doc='Commands to captures output of',
groups=(UP, CP)),
files = Attr(Dict(STR), init=lambda self: dict(),
doc='File name macros', groups=(UP, CP)),
settings = Attr(Dict(None), init=lambda self: dict(),
doc='Global settings of various sorts',
groups=(UP, CP)),
jinja_filters = Attr(Dict(Callable),
init=lambda self: \
dict(DEFAULT_JINJA_FILTERS),
doc='Custom Jinja2 filters',
groups=(UP, CP)),
jinja_functions = Attr(Dict(Callable),
init=lambda self: \
dict(DEFAULT_JINJA_FUNCTIONS),
doc='Custom Jinja2 functions',
groups=(UP, CP)),
function_aliases = Attr(Dict(STR), init=lambda self: dict(),
internal=True, groups=(UP, CP),
doc='Jinja function aliases'),
env = Attr(Dict((STR, int, float,
List((STR, int, float, list, dict)),
Dict((STR, int, float, list, dict)))),
init=lambda self: dict(),
doc='Current name resolution environment',
groups=(UP, CP)),
jenv = Attr(Environment, doc='Jinja2 environment', group='eq_exclude',
init=lambda self: Environment(undefined=StrictUndefined)),
default_task = Attr(STR, '', 'Task to run if no task is '
'specified at the command line',
group=AUP),
default_context = Attr(Context, doc='Execution context to use '
'if none is specified in task definition',
group=AUP))
_opts = dict(init_validate=True)
@init_hook
def _init_populate(self):
self.macros.update(INITIAL_MACROS)
self.contexts.update(BUILTIN_CONTEXTS)
if not hasattr(self, 'default_context'):
self.default_context = self.contexts['null']
@init_hook
def _set_jenv(self, **kwargs):
filts = dict(self.jinja_filters)
for name, filt in filts.items():
filts[name] = partial(filt, env=self)
self.jenv.filters.update(filts)
funcs = dict(self.jinja_functions)
for name, func in funcs.items():
funcs[name] = partial(func, env=self)
self.jenv.globals.update(funcs)
def _update_post(self, other, **kwargs):
self._set_jenv(**kwargs)
def capture_value(self, cmd, **kwargs):
out, code = get_output(cmd)
# These are intended to be macro values, so newlines and extra
# white space probably aren't desirable
return out.strip()
def macro_env(self, **kwargs):
dct = dict(self.macros)
dct.update(self.secret_values)
dct.update(self.files)
return dct
def resolve_macros(self, **kwargs):
env = self.macro_env(**kwargs)
macros = dict(self.macros)
macros.update(self.captures)
# TODO: better error message if there is a cycle
potential_problems = set(env) & set(self.jinja_functions)
for name, template in ordered_macros(macros,
jenv=self.jenv,
funcs=self.jinja_functions):
if name in self.macros:
fixed = fix_functions(template, potential_problems, self)
env[name] = resolve(fixed, env, jenv=self.jenv)
if name in self.captures:
cmd = resolve(template, env, jenv=self.jenv)
env[name] = self.capture_value(cmd, **kwargs)
self.env = env
def resolve(self, template):
return resolve(template, self.env, jenv=self.jenv)
def validate(self):
super(Env, self).validate()
for name, ctx in self.contexts.items():
ctx.validate()
for name, task in self.tasks.items():
task.validate()
class B(Base, YAMLMixin):
_opts = dict(init_validate=True)
_attrs = dict(a=Attr(int), b=Attr(List(A)))
class Dependencies(Base):
'''Representation of the various dependency sets'''
special_contexts = ('includes', )
_attrs = dict(contexts=Attr(Mapping(List(Dependency), AttrDict),
init=lambda x: AttrDict(),
doc='Diction of dependencies in their '
'various contexts'),
includes=Attr(Mapping(List(STR), AttrDict),
init=lambda x: AttrDict(),
doc='Specification of which contexts to '
'include in others'))
_opts = dict(init_validate=True, coerce_args=True)
@classmethod
def _contexts(cls, dct):
return [key for key in dct if key not in cls.special_contexts]
@classmethod
def _from_context(cls, dct):
deps = []
for key in dct:
typ = DEPENDENCY_KEYS[key]
deps.extend([typ.from_conf(obj) for obj in dct[key]])
deps.sort(key=attrgetter('order'))
return deps
@classmethod
def from_yaml(cls, fil):
dct = yaml.load(fil)
contexts = cls._contexts(dct)
includes = dct.get('includes', {})
contexts = {
context: cls._from_context(dct.get(context, {}))
for context in contexts
}
kwargs = dict(contexts=AttrDict(contexts), includes=AttrDict(includes))
return cls(**kwargs)
def contexts_from_includes(self, context, contexts):
new = self.includes.get(context, [])
if new:
contexts.extend(list(filter(lambda c: c not in contexts, new)))
for con in new:
self.contexts_from_includes(con, contexts)
def deps_from_context(self, context):
contexts = [context]
if context == 'all':
contexts = self.contexts
elif context not in self.contexts:
raise ValueError('Invalid context: {}'.format(context))
self.contexts_from_includes(context, contexts)
ret = []
for con in contexts:
ret += getattr(self.contexts, con)
ret.sort(key=attrgetter('order'))
return ret
def dependencies_to_satisfy(self, context, deptype=None):
if deptype is None:
deptype = AnyType()
deps = []
for dep in self.deps_from_context(context):
if deptype.query(dep):
deps.append(dep)
return deps
def dependency_operations(self, deps):
groups = defaultdict(list)
for dep in deps:
groups[dep.order].append(dep)
dep_to_group = {}
for group in groups.values():
for dep in group:
dep_to_group[dep] = group[0].order
name_to_dep = {dep.name: dep for dep in deps}
special_deps = [] # before or after relations specified
for dep in deps:
if dep.before or dep.after:
special_deps.append(dep)
groups[dep.order].remove(dep)
dep_to_group[dep] = dep
if not groups[dep.order]:
del groups[dep.order]
sorted_groups = []
for order in sorted(groups.keys()):
groups[order].sort(key=attrgetter('name'))
sorted_groups.append(order)
rels = []
nodes = special_deps + sorted_groups
for k in xrange(len(sorted_groups) - 1):
rels.append(Precedes(sorted_groups[k], sorted_groups[k + 1]))
def query(name):
try:
return dep_to_group[name_to_dep[name]]
except KeyError:
pass
for dep in special_deps:
if dep.before:
rels.append(Precedes(dep, query(dep.before)))
if dep.after:
# For when the target is in a different context
target = query(dep.after)
if target:
rels.append(Precedes(target, dep))
def parents(d):
ret = []
for rel in rels:
if rel.B is d:
ret.append(rel.A)
return ret
def children(d):
ret = []
for rel in rels:
if rel.A is d:
ret.append(rel.B)
return ret
def order(d):
if isinstance(d, int):
return d
return d.order
# Preserve intuitive order for dangling elements
# (e.g. dangling apt should precede any pip if possible)
for dep in special_deps:
if dep.after:
if not children(dep):
for par in parents(dep):
for ch in children(par):
if ch is not dep:
if order(ch) > dep.order:
rels.append(Precedes(dep, ch))
ret = []
sorting = topological_sorting(nodes, rels)
for item in sorting:
if isinstance(item, Dependency):
ops = item.satisfy()
else:
ops = []
for dep in groups[item]:
ops.extend(dep.satisfy())
Operation.optimize(ops)
ret.extend(ops)
return ret
def satisfy(self, context, deptype=None, execute=True):
deps = self.dependencies_to_satisfy(context, deptype)
ops = self.dependency_operations(deps)
if execute:
for op in ops:
op.execute()
return ops
def export_header(self):
from . import __version__ as ver
return '# Auto-generated by depman {}\n'.format(ver)
def export(self,
context,
deptype,
outfile,
write=True,
include_header=True):
deps = [
dep for dep in self.deps_from_context(context)
if deptype.query(dep)
]
out = ''
if include_header:
out = self.export_header()
exports = sorted([dep.export() for dep in deps])
out += '\n'.join(exports)
if write:
outfile.write(out)
return out
def validate(self):
super(Dependencies, self).validate()
for key, value in self.includes.items():
assert key in self.contexts, \
"Each key ({}) in includes must be a valid context".format(key)
for con in value:
assert con in self.contexts, \
"Each list item ({}) must be a valid context".format(con)