class Problem(Base): _attrs = dict(constraints=Attr(List(Constraint)), domain=Attr(Domain)) _opts = dict(init_validate=True, args=('domain', 'constraints')) @init_hook def _init(self): self.domain = self.domain.copy() self.var_constraint = defaultdict(set) for con in self.constraints: con.preprocess(self.domain) for var in con.args: self.var_constraint[var].add(con) def check(self, theory): for con in self.constraints: if set(con.args) <= set(theory): if not con.check(**theory): return False return True def display(self, **kwargs): strs = [self.domain.display(**kwargs)] for con in self.constraints: s = con.display(**kwargs) if s: strs.append(s) return '\n'.join(strs) def validate(self): super(Problem, self).validate() if not set(self.var_constraint).issubset(set(self.domain)): raise ValueError('Some constraints defined in over ' 'undefined variables')
class ClassWrapper(SetLeaf): '''The idea is that a type implicitly represents the set of all of its subclasses, including itself. ''' _attrs = dict(type=Attr(type, doc=''), subclasses=Attr(List(type), doc='', init=lambda self: ([self.type] + subclasses(self.type)))) _opts = dict(args=('type', )) def __init__(self, *args, **kwargs): super(ClassWrapper, self).__init__(*args, **kwargs) self.subclasses = [self.type] + subclasses(self.type) def display(self, **kwargs): return 'ClassWrapper({})'.format(get_typename(self.type)) def size(self): return len(self.subclasses) def hasmember(self, item): return item in self.subclasses def sample(self, **kwargs): ret = choice(self.subclasses) return ret def enumerate(self, **kwargs): args = Args(**kwargs) maxenum = min(args.max_enumerate, len(self.subclasses)) for k, item in enumerate(self.subclasses): if k >= maxenum: break yield item def to_set(self, **kwargs): return super(SetLeaf, self).to_set(**kwargs)
def from_yaml(cls, name, dct): if isinstance(dct, STR): return cls(commands=[Command(dct)]) elif List(STR).query(dct): cmds = [Command(s) for s in dct] return cls(commands=cmds) elif isinstance(dct, dict): if set(dct.keys()).issuperset({'command'}): ret = Task.from_yaml(name, dct['command']) if 'context' in dct: for cmd in ret.commands: cmd.context = dct['context'] if 'args' in dct: ret.args = dct['args'] if 'kwargs' in dct: ret.kwargs = dct['kwargs'] if 'for' in dct: ret.loop = For.from_yaml(dct['for']) if 'if' in dct: ret.condition = Task.from_yaml(name + '-if', dct['if']) ret.condition_type = True elif 'ifnot' in dct: ret.condition = Task.from_yaml(name + '-ifnot', dct['ifnot']) ret.condition_type = False ret.validate() return ret raise ValidationError('Invalid data for task: {}'.format(name))
class For(Base): _attrs = dict(var = Attr((STR, List(STR)), doc='The loop variable(s)'), in_ = Attr((STR, List((STR, int, list))), doc='Name(s) of list macro(s) to loop over')) _opts = dict(init_validate = True, args = ('var', 'in_')) @classmethod def from_yaml(cls, dct): if isinstance(dct, STR): m = re.match(FOREX, dct) if m: return cls(var=m.groups()[0], in_=m.groups()[1]) else: raise ValidationError('Invalid for loop specifier: {}' .format(dct)) kwargs = {} get_delete(dct, kwargs, 'var', None) get_delete(dct, kwargs, 'in', None, 'in_') if dct: raise ValidationError('Invalid for keys: {}' .format(','.join(dct.keys()))) return cls(**kwargs) def resolve_macros(self, env, **kwargs): var = self.var in_ = self.in_ if not isinstance(var, list): var = [self.var] in_ = [self.in_] outs = [] for k, v in enumerate(var): name = in_[k] if isinstance(name, list): val = name else: val = env.env[name] if not isinstance(val, list): raise ValidationError('For loop "in" specifier must be name of ' 'list macro: {}'.format(in_[k])) outs.append(val) return var, outs def loop(self, env, **kwargs): var, in_ = self.resolve_macros(env, **kwargs) for tup in product(*in_): yld = env.copy(**kwargs) for name, val in zip(var, tup): yld.env[name] = val yield yld def validate(self): super(For, self).validate() if isinstance(self.var, list): if len(self.var) != len(self.in_): raise ValidationError('"var" and "in" lists must be same ' 'length')
class Task(Base): _attrs = dict(commands = Attr(List(Command)), condition = Attr(This, optional=True), loop = Attr(For, optional=True), args = Attr(List((STR, int)), init=lambda self: list()), kwargs = Attr(Dict((STR, int)), init=lambda self: dict()), condition_type = Attr(bool, True)) _opts = dict(init_validate = True, optional_none = True) @classmethod def from_yaml(cls, name, dct): if isinstance(dct, STR): return cls(commands=[Command(dct)]) elif List(STR).query(dct): cmds = [Command(s) for s in dct] return cls(commands=cmds) elif isinstance(dct, dict): if set(dct.keys()).issuperset({'command'}): ret = Task.from_yaml(name, dct['command']) if 'context' in dct: for cmd in ret.commands: cmd.context = dct['context'] if 'args' in dct: ret.args = dct['args'] if 'kwargs' in dct: ret.kwargs = dct['kwargs'] if 'for' in dct: ret.loop = For.from_yaml(dct['for']) if 'if' in dct: ret.condition = Task.from_yaml(name + '-if', dct['if']) ret.condition_type = True elif 'ifnot' in dct: ret.condition = Task.from_yaml(name + '-ifnot', dct['ifnot']) ret.condition_type = False ret.validate() return ret raise ValidationError('Invalid data for task: {}'.format(name)) def run(self, env, **kwargs): codes = [] looping = kwargs.get('looping', False) exit_on_error = kwargs.get('exit_on_error', env.settings.get('exit_on_error', True)) preview_conditionals = kwargs.get('preview_conditionals', env.settings.get('preview_conditionals', True)) if self.condition and not looping: if preview_conditionals: kwargs['preview_pre'] = 'if: ' if self.condition_type else 'ifnot: ' codes_ = self.condition.run(env, **kwargs) code = max(codes_) if (((self.condition_type is True and code != 0) or (self.condition_type is False and code == 0)) and code is not None): return [] if preview_conditionals: kwargs['preview_pre'] = '\t' if (self.args or self.kwargs) and not looping: env = env.copy(**kwargs) if self.args: for k, arg in enumerate(self.args): env.env['_{}'.format(k + 1)] = arg if self.kwargs: for name, value in self.kwargs.items(): env.env[name] = value if self.loop and not looping: n = 0 kwargs['looping'] = True for env_ in self.loop.loop(env, **kwargs): env_.env[env_.settings['loop_count_macro']] = n codes_ = self.run(env_, **kwargs) codes.extend(codes_) n += 1 return codes for cmd in self.commands: if cmd.command in env.tasks: codes_ = env.tasks[cmd.command].run(env, **kwargs) codes.extend(codes_) else: code = cmd.run(env, **kwargs) codes.append(code) if exit_on_error and any(c != 0 for c in codes if c is not None): break return codes def run_preview(self, env, **kwargs): kwargs['preview'] = True kwargs['verbose'] = True kwargs['run_preview'] = True with assign(sys, 'stdout', cStringIO()): self.run(env, **kwargs) ret = sys.stdout.getvalue() return ret
class Document(Base): _attrs = dict(imports = Attr(List(STR), init=lambda self:list()), includes = Attr(List(STR), init=lambda self: list()), secrets = Attr(List(STR), init=lambda self: list()), macros = Attr(Dict((STR, int, float, List((STR, int, float, list, dict)), Dict((STR, int, float, list, dict)))), init=lambda self: dict()), contexts = Attr(Dict(Context), init=lambda self: dict()), tasks = Attr(Dict(Task), init=lambda self: dict()), secret_values = Attr(Dict(STR), init=lambda self: dict()), captures = Attr(Dict(STR), init=lambda self: dict()), files = Attr(Dict(STR), init=lambda self: dict()), settings = Attr(Dict(None), init=lambda self: dict()), default_task = Attr(STR, '', 'Task to run if no task is ' 'specified at the command line'), env = Attr(Env, init=lambda self: Env(), internal=True), dirname = Attr(STR, doc='Relative path for includes'), cachedir = Attr(STR, '', 'Directory to store downloaded files'), pull = Attr(bool, False, 'Force-pull URLs'), ) _opts = dict(init_validate = True) @classmethod def from_path(cls, path, **kwargs): with open(path, 'r') as f: dct = yaml.load(f) return cls.from_yaml(dct, os.path.abspath(os.path.dirname(path)), **kwargs) @classmethod def from_yaml(cls, dct, dirname, **kwargs): get_delete(dct, kwargs, 'import', [], 'imports') get_delete(dct, kwargs, 'include', [], 'includes') get_delete(dct, kwargs, 'capture', {}, 'captures') get_delete(dct, kwargs, 'files', {}) get_delete(dct, kwargs, 'secrets', []) get_delete(dct, kwargs, 'macros', {}) get_delete(dct, kwargs, 'contexts', {}) get_delete(dct, kwargs, 'tasks', {}) get_delete(dct, kwargs, 'default', '', 'default_task') init_macros = kwargs.get('initial_macros', {}) for name, value in init_macros.items(): if name not in kwargs['macros']: kwargs['macros'][name] = value settings = dict(dct.get('settings', {})) settings.update(kwargs.get('settings', {})) kwargs['settings'] = settings if 'settings' in dct: del dct['settings'] if dct: raise ValidationError('Invalid top-level keys: {}' .format(','.join(dct.keys()))) for key in list(kwargs['contexts'].keys()): kwargs['contexts'][key] = \ Context.from_yaml(key, kwargs['contexts'][key]) for key in list(kwargs['tasks'].keys()): kwargs['tasks'][key] = \ Task.from_yaml(key, kwargs['tasks'][key]) kwargs['dirname'] = dirname return cls(**kwargs) @init_hook def process(self, **kwargs): cachedir = DEFAULT_CACHE_DIR if self.cachedir: cachedir = self.cachedir self.process_settings(**kwargs) jenv = Env().jenv pre_macros = dict(self.macros) for name, macro in ordered_macros(pre_macros, lenient=True): try: pre_macros[name] = resolve(macro, pre_macros, lenient=True, jenv=jenv) except: pass # There might be macros defined in terms of # jinja_functions to be imported def process(path): return resolve_url(resolve(path, pre_macros, jenv=jenv), cachedir=cachedir, force=self.pull) with chdir(self.dirname): for path in map(process, self.imports): if not os.path.exists(path): raise ValidationError("Module path does not exist: {}" .format(path)) self.process_import(path, **kwargs) for path in map(process, self.includes): if not os.path.isfile(path): raise ValidationError("Include path does not exist: {}" .format(path)) self.process_include(path, **kwargs) files = {} for fname, fpath in self.files.items(): path = process(fpath) if not os.path.isfile(path): raise ValidationError("File path does not exist: {}" .format(path)) files[fname] = path for name in self.secrets: self.process_secret(name, **kwargs) env = Env(macros=self.macros, contexts=self.contexts, tasks=self.tasks, secret_values=self.secret_values, captures=self.captures, settings=self.settings, default_task=self.default_task, files=files) self.env.update(env, **kwargs) def post_process(self, **kwargs): with chdir(self.dirname): self.env.resolve_macros() self.validate() def process_import(self, path, **kwargs): mod = imp.load_source('yatr_module_import', path) if not hasattr(mod, 'env'): raise ImportError("yatr extension module '{}' has no env" .format(path)) self.env.update(mod.env, **kwargs) def process_include(self, path, **kwargs): doc = Document.from_path(path, pull=self.pull, cachedir=self.cachedir, settings=self.settings) self.env.update(doc.env, **kwargs) def process_secret(self, name, **kwargs): raise NotImplementedError('Secrets currently unsupported') def process_settings(self, **kwargs): self.settings['silent'] = \ str_to_bool(self.settings.get('silent', False)) self.settings['loop_count_macro'] = \ self.settings.get('loop_count_macro', '_n') self.settings['preview_conditionals'] = \ str_to_bool(self.settings.get('preview_conditionals', True)) self.settings['exit_on_error'] = \ str_to_bool(self.settings.get('exit_on_error', True)) def run(self, name, **kwargs): try: with chdir(self.dirname): return self.env.tasks[name].run(self.env, **kwargs) except KeyError: raise RuntimeError('No such task: {}'.format(name)) def validate(self): super(Document, self).validate() self.env.validate()
import os import imp import yaml from syn.base_utils import chdir from syn.base import Base, Attr, init_hook from syn.type import List, Dict from syn.five import STR from .base import ValidationError, resolve_url, resolve, ordered_macros,\ DEFAULT_CACHE_DIR, str_to_bool, get_delete from .context import Context from .task import Task from .env import Env STRList = List(STR) #------------------------------------------------------------------------------- # Document class Document(Base): _attrs = dict(imports = Attr(List(STR), init=lambda self:list()), includes = Attr(List(STR), init=lambda self: list()), secrets = Attr(List(STR), init=lambda self: list()), macros = Attr(Dict((STR, int, float, List((STR, int, float, list, dict)), Dict((STR, int, float, list, dict)))), init=lambda self: dict()), contexts = Attr(Dict(Context), init=lambda self: dict()), tasks = Attr(Dict(Task), init=lambda self: dict()), secret_values = Attr(Dict(STR), init=lambda self: dict()), captures = Attr(Dict(STR), init=lambda self: dict()),
class Env(Base, Copyable, Updateable): _groups = (UP, AUP) _attrs = dict(macros = Attr(Dict((STR, int, float, List((STR, int, float, list, dict)), Dict((STR, int, float, list, dict)))), init=lambda self: dict(), doc='Macro definitions', groups=(UP, CP)), contexts = Attr(Dict(Context), init=lambda self: dict(), doc='Execution context definitions', groups=(UP, CP)), tasks = Attr(Dict(Task), init=lambda self: dict(), doc='Task definitions', groups=(UP, CP)), secret_values = Attr(Dict(STR), init=lambda self: dict(), doc='Secret value store', groups=(UP, CP)), captures = Attr(Dict(STR), init=lambda self: dict(), doc='Commands to captures output of', groups=(UP, CP)), files = Attr(Dict(STR), init=lambda self: dict(), doc='File name macros', groups=(UP, CP)), settings = Attr(Dict(None), init=lambda self: dict(), doc='Global settings of various sorts', groups=(UP, CP)), jinja_filters = Attr(Dict(Callable), init=lambda self: \ dict(DEFAULT_JINJA_FILTERS), doc='Custom Jinja2 filters', groups=(UP, CP)), jinja_functions = Attr(Dict(Callable), init=lambda self: \ dict(DEFAULT_JINJA_FUNCTIONS), doc='Custom Jinja2 functions', groups=(UP, CP)), function_aliases = Attr(Dict(STR), init=lambda self: dict(), internal=True, groups=(UP, CP), doc='Jinja function aliases'), env = Attr(Dict((STR, int, float, List((STR, int, float, list, dict)), Dict((STR, int, float, list, dict)))), init=lambda self: dict(), doc='Current name resolution environment', groups=(UP, CP)), jenv = Attr(Environment, doc='Jinja2 environment', group='eq_exclude', init=lambda self: Environment(undefined=StrictUndefined)), default_task = Attr(STR, '', 'Task to run if no task is ' 'specified at the command line', group=AUP), default_context = Attr(Context, doc='Execution context to use ' 'if none is specified in task definition', group=AUP)) _opts = dict(init_validate=True) @init_hook def _init_populate(self): self.macros.update(INITIAL_MACROS) self.contexts.update(BUILTIN_CONTEXTS) if not hasattr(self, 'default_context'): self.default_context = self.contexts['null'] @init_hook def _set_jenv(self, **kwargs): filts = dict(self.jinja_filters) for name, filt in filts.items(): filts[name] = partial(filt, env=self) self.jenv.filters.update(filts) funcs = dict(self.jinja_functions) for name, func in funcs.items(): funcs[name] = partial(func, env=self) self.jenv.globals.update(funcs) def _update_post(self, other, **kwargs): self._set_jenv(**kwargs) def capture_value(self, cmd, **kwargs): out, code = get_output(cmd) # These are intended to be macro values, so newlines and extra # white space probably aren't desirable return out.strip() def macro_env(self, **kwargs): dct = dict(self.macros) dct.update(self.secret_values) dct.update(self.files) return dct def resolve_macros(self, **kwargs): env = self.macro_env(**kwargs) macros = dict(self.macros) macros.update(self.captures) # TODO: better error message if there is a cycle potential_problems = set(env) & set(self.jinja_functions) for name, template in ordered_macros(macros, jenv=self.jenv, funcs=self.jinja_functions): if name in self.macros: fixed = fix_functions(template, potential_problems, self) env[name] = resolve(fixed, env, jenv=self.jenv) if name in self.captures: cmd = resolve(template, env, jenv=self.jenv) env[name] = self.capture_value(cmd, **kwargs) self.env = env def resolve(self, template): return resolve(template, self.env, jenv=self.jenv) def validate(self): super(Env, self).validate() for name, ctx in self.contexts.items(): ctx.validate() for name, task in self.tasks.items(): task.validate()
class B(Base, YAMLMixin): _opts = dict(init_validate=True) _attrs = dict(a=Attr(int), b=Attr(List(A)))
class Dependencies(Base): '''Representation of the various dependency sets''' special_contexts = ('includes', ) _attrs = dict(contexts=Attr(Mapping(List(Dependency), AttrDict), init=lambda x: AttrDict(), doc='Diction of dependencies in their ' 'various contexts'), includes=Attr(Mapping(List(STR), AttrDict), init=lambda x: AttrDict(), doc='Specification of which contexts to ' 'include in others')) _opts = dict(init_validate=True, coerce_args=True) @classmethod def _contexts(cls, dct): return [key for key in dct if key not in cls.special_contexts] @classmethod def _from_context(cls, dct): deps = [] for key in dct: typ = DEPENDENCY_KEYS[key] deps.extend([typ.from_conf(obj) for obj in dct[key]]) deps.sort(key=attrgetter('order')) return deps @classmethod def from_yaml(cls, fil): dct = yaml.load(fil) contexts = cls._contexts(dct) includes = dct.get('includes', {}) contexts = { context: cls._from_context(dct.get(context, {})) for context in contexts } kwargs = dict(contexts=AttrDict(contexts), includes=AttrDict(includes)) return cls(**kwargs) def contexts_from_includes(self, context, contexts): new = self.includes.get(context, []) if new: contexts.extend(list(filter(lambda c: c not in contexts, new))) for con in new: self.contexts_from_includes(con, contexts) def deps_from_context(self, context): contexts = [context] if context == 'all': contexts = self.contexts elif context not in self.contexts: raise ValueError('Invalid context: {}'.format(context)) self.contexts_from_includes(context, contexts) ret = [] for con in contexts: ret += getattr(self.contexts, con) ret.sort(key=attrgetter('order')) return ret def dependencies_to_satisfy(self, context, deptype=None): if deptype is None: deptype = AnyType() deps = [] for dep in self.deps_from_context(context): if deptype.query(dep): deps.append(dep) return deps def dependency_operations(self, deps): groups = defaultdict(list) for dep in deps: groups[dep.order].append(dep) dep_to_group = {} for group in groups.values(): for dep in group: dep_to_group[dep] = group[0].order name_to_dep = {dep.name: dep for dep in deps} special_deps = [] # before or after relations specified for dep in deps: if dep.before or dep.after: special_deps.append(dep) groups[dep.order].remove(dep) dep_to_group[dep] = dep if not groups[dep.order]: del groups[dep.order] sorted_groups = [] for order in sorted(groups.keys()): groups[order].sort(key=attrgetter('name')) sorted_groups.append(order) rels = [] nodes = special_deps + sorted_groups for k in xrange(len(sorted_groups) - 1): rels.append(Precedes(sorted_groups[k], sorted_groups[k + 1])) def query(name): try: return dep_to_group[name_to_dep[name]] except KeyError: pass for dep in special_deps: if dep.before: rels.append(Precedes(dep, query(dep.before))) if dep.after: # For when the target is in a different context target = query(dep.after) if target: rels.append(Precedes(target, dep)) def parents(d): ret = [] for rel in rels: if rel.B is d: ret.append(rel.A) return ret def children(d): ret = [] for rel in rels: if rel.A is d: ret.append(rel.B) return ret def order(d): if isinstance(d, int): return d return d.order # Preserve intuitive order for dangling elements # (e.g. dangling apt should precede any pip if possible) for dep in special_deps: if dep.after: if not children(dep): for par in parents(dep): for ch in children(par): if ch is not dep: if order(ch) > dep.order: rels.append(Precedes(dep, ch)) ret = [] sorting = topological_sorting(nodes, rels) for item in sorting: if isinstance(item, Dependency): ops = item.satisfy() else: ops = [] for dep in groups[item]: ops.extend(dep.satisfy()) Operation.optimize(ops) ret.extend(ops) return ret def satisfy(self, context, deptype=None, execute=True): deps = self.dependencies_to_satisfy(context, deptype) ops = self.dependency_operations(deps) if execute: for op in ops: op.execute() return ops def export_header(self): from . import __version__ as ver return '# Auto-generated by depman {}\n'.format(ver) def export(self, context, deptype, outfile, write=True, include_header=True): deps = [ dep for dep in self.deps_from_context(context) if deptype.query(dep) ] out = '' if include_header: out = self.export_header() exports = sorted([dep.export() for dep in deps]) out += '\n'.join(exports) if write: outfile.write(out) return out def validate(self): super(Dependencies, self).validate() for key, value in self.includes.items(): assert key in self.contexts, \ "Each key ({}) in includes must be a valid context".format(key) for con in value: assert con in self.contexts, \ "Each list item ({}) must be a valid context".format(con)