def test_patch_complex(): c = ConfigDict({ "a": 1, "b": { "x": 3, "y": 4 }, "c": { "x": 5, "y": 6 }, "d": { "x": 7, "y": 8 } }) d = {"a": 2, "b": {"z": 5}, "c": [1, 2], "d": {"y": 9}} c.patch(d) assert c.a == 2 assert c.b.x == 3 assert c.b.y == 4 assert c.b.z == 5 assert c.c == [1, 2] assert c.d.x == 7 assert c.d.y == 9
def read(text): if isinstance(text, File): text = text.read_all() t = env.from_string(text) data = yaml.load(t.render(context or dict())) conf = ConfigDict(data) if 'patches' in conf and patchable: leaf = conf parent = leaf['patches'] del leaf['patches'] parent = File(f.parent.child(parent)) conf = read(parent) conf.patch(leaf) return conf
def __init__(self, data=None): if isinstance(data, Config): self.data = data.data else: # Assume dict self.data = ConfigDict(data) self.default_search_paths = [os.getcwd()]
def test_patch_complex(): c = ConfigDict({ "a": 1, "b": {"x": 3, "y": 4}, "c": {"x": 5, "y": 6}, "d": {"x": 7, "y": 8} }) d = {"a": 2, "b": {"z": 5}, "c": [1, 2], "d": {"y": 9}} c.patch(d) assert c.a == 2 assert c.b.x == 3 assert c.b.y == 4 assert c.b.z == 5 assert c.c == [1, 2] assert c.d.x == 7 assert c.d.y == 9
class Config(AutoProp): def __init__(self, data=None): if isinstance(data, Config): self.data = data.data else: # Assume dict self.data = ConfigDict(data) self.default_search_paths = [os.getcwd()] def flatten(self): return dict(self.data) @AutoProp.default def publish(self): return self.data.get('publish', None) @AutoProp.default def source_dir(self): try: default_source = File(self.data.file_path).parent.path except AttributeError: default_source = os.getcwd() source = self.data.get('source_dir', default_source) return Folder(default_source).child_folder(source) @AutoProp.default def source_patterns(self): patterns = [] config_patterns = self.data.get('source_patterns', ['*.json']) extappend(patterns, config_patterns) return patterns @AutoProp.default def output_dir(self): out = self.data.get('output_dir', 'out') return Folder(self.source_dir).parent.child_folder(out) @AutoProp.default def search_paths(self): paths = [] paths.extend(self.default_search_paths) paths.append(self.source_dir.path) config_paths = [] extappend(config_paths, self.data.get('search_paths', [])) config_paths = [Folder(self.source_dir).parent.child(path) for path in config_paths] extappend(paths, config_paths) return list(set(paths)) @AutoProp.default def context(self): return self.data.get('context') @AutoProp.default def region(self): return self.data.get('region', 'us-east-1')
def publish(data, push_www=True, push_app=False): data = ConfigDict(data) www_archive = upload_www(data, push_www) app_archive = upload_app(data, push_app) config_file = File(File(__file__).parent.child('stack/gitbot.yaml')) config = yaml.load(config_file.read_all()) config['file_path'] = config_file.path params = data.get('stack_params', dict()) params.update(dict(AppSource=app_archive, WebSource=www_archive)) worker_params = get_worker_outputs(data) if not worker_params or 'QueueURL' not in worker_params: raise Exception('Failed to create the worker stack') params.update(dict( WorkerQueueURL=worker_params['QueueURL'], ManagerAccessKey=worker_params['ManagerKey'], ManagerSecretKey=worker_params['ManagerSecret'] )) config['data'] = data stack.publish_stack(config, params=params, debug=True, wait=True)
def __init__(self, name, config=None, env=None, settings=None, context=None, conf_path=None): if not name: raise ValueError("project name is required") self.name = name self.config = ConfigDict(config or {}) self.config.work_root = self.config.get('work_root', Folder(os.getcwd()).child('out')) self.env = env self.settings = settings self.conf_path = conf_path self.context = context self.annotation_header_template = config.get( 'annotation_header_template', ANNOTATION_TEMPLATE) self.depends_template = config.get( 'depends_template', DEPENDS_TEMPLATE) self._depends = None
def _load(cls, project_name, env='build', context=None, conf_path=None): if project_name in __PROJECTS__: return __PROJECTS__[project_name] settings = __ENV__.get(env, None) if not settings: file_name = env + '.yaml' if conf_path: file_name = Folder(conf_path).child(file_name) settings = yinja.load(file_name, context=context) __ENV__[env] = settings config = ConfigDict() config.update(settings.config) config.patch(settings.projects.get(project_name, dict())) project_type = config.get('type_name', 'gitbot.lib.build.Project') project_class = load_python_object(project_type) project = project_class(project_name, config, env, settings, context, conf_path) __PROJECTS__[project_name] = project return project
class Project(AutoProp): def __init__(self, name, config=None, env=None, settings=None, context=None, conf_path=None): if not name: raise ValueError("project name is required") self.name = name self.config = ConfigDict(config or {}) self.config.work_root = self.config.get('work_root', Folder(os.getcwd()).child('out')) self.env = env self.settings = settings self.conf_path = conf_path self.context = context self.annotation_header_template = config.get( 'annotation_header_template', ANNOTATION_TEMPLATE) self.depends_template = config.get( 'depends_template', DEPENDS_TEMPLATE) self._depends = None @property def depends(self): if not self._depends: depends = self.config.get('depends', {}) self._depends = { project_name: self.load_dependent(project_name, project_env) for project_name, project_env in depends.items() } return self._depends @AutoProp.default def work_root(self): return self.config.get('work_root') @AutoProp.default def source_root(self): return self.config.get('source_root', Folder(self.work_root).child('sources')) @AutoProp.default def build_root(self): return self.config.get('build_root', Folder(self.work_root).child('dist')) @AutoProp.default def tools_root(self): return self.config.get('tools_root', Folder(self.work_root).child('tools')) @AutoProp.default def dateformat(self): return self.config.get('dateformat', '%A, %d. %B %Y %I:%M%p') @property def needs_source(self): return self.config.get('needs_source', True) @AutoProp.default def version(self): return self.config.get('version') @AutoProp.default def dir_name(self): return self.config.get('dir_name', self.name) @AutoProp.default def source_dir(self): return self.config.get('source_dir', '%s/%s' % (self.source_root, self.dir_name)) @AutoProp.default def build_dir(self): return self.config.get('build_dir', '%s/%s' % (self.build_root, self.dir_name)) @AutoProp.default def repo_base(self): return self.config.get('repo_base', 'https://github.com') @AutoProp.default def repo_owner(self): return self.config.get('repo_owner') @AutoProp.default def repo_name(self): return self.config.get('repo_name', self.name) @AutoProp.default def repo(self): return self.config.get('repo', '%s/%s/%s' % (self.repo_base, self.repo_owner, self.repo_name)) @AutoProp.default def remote(self): return self.config.get('remote', 'origin') @AutoProp.default def branch(self): return self.config.get('branch', 'master') @property def version_published(self): return self.tree.tagger.check(str(self.version)) @property def tree(self): return Tree(self.source_dir, self.repo, self.remote, self.branch) def has_source_changed(self): return self.tree.has_changes() @property def revision(self): try: return self.tree.get_revision(short=False).strip() except GitException: return self.tree.get_revision_remote().strip() @property def modified(self): return self.tree.get_last_committed().strftime( self.dateformat) def pull(self, tip_only=False, pull_depends=False): self.tree.pull(tip_only) if pull_depends: for dep in self.depends.itervalues(): dep.pull(tip_only=tip_only) def merge(self, ref): if not Folder(self.source_dir).exists: self.pull() self.tree.fetch_ref(ref, local='gitbot/local', notags=True) try: self.tree.merge('gitbot/local', ff_only=False) except GitException, gitex: raise BuildConflictedException(unicode(gitex))
def publish_stack(config, params=None, debug=False, wait=False, **kwargs): if isinstance(config, File): file_path = config.path config = yaml.load(config.read_all()) config["file_path"] = file_path uploaded = upload_stack(config, **kwargs) config = uploaded.config main_stack = _get_main_stack(uploaded.config, uploaded.files) try: main = uploaded.result[main_stack] except KeyError: raise Exception("Cannot find the main stack[{main}]".format(main=main_stack)) try: stack_name = config.publish.stack_name except AttributeError: raise Exception("Stack name is required in configuration[publish.stack_name].") defaults = get_params(config) args = ConfigDict({name: info["value"] for name, info in defaults.iteritems()}) args.patch(params) params = args region = config.publish.get("region", "us-east-1") # Connect to cloud formation and create the stack cf = connect_cf(region, **kwargs) try: cf.describe_stacks(stack_name) update = True except: update = False params = _transform_params(config.flatten(), params, uploaded.result) fn = cf.update_stack if update else cf.create_stack try: fn( stack_name, disable_rollback=debug, capabilities=["CAPABILITY_IAM"], template_url=main["url"], parameters=params, ) except BotoServerError as bse: try: error = json.loads(bse.error_message) except: raise bse try: message = error["Error"]["Message"] except KeyError: raise bse if message == "No updates are to be performed.": print "Stack is already up to date." wait = False else: raise bse if wait: __wait_while_status(cf, "CREATE_IN_PROGRESS" if not update else "UPDATE_IN_PROGRESS") return stack_name
def test_copy(): c = ConfigDict({"a": 1, "b": {"c": 3}}) d = c.copy() assert c == d c.b.c = 4 assert c != d
def test_patch_simple(): c = ConfigDict({"a": 1, "b": {"c": 3, "e": 4}}) d = {"b": {"e": 5}} c.patch(d) assert c.b.c == 3 assert c.b.e == 5
def test_init(): c = ConfigDict({"a": 1}) assert c.a == 1 assert c["a"] == 1
def test_two_levels_assignment(): c = ConfigDict({"a": 1, "b": {"c": 3}}) d = {"d": 5} c.b = d assert c.b.d == 5 assert c.b == d
def test_list(): c = ConfigDict({"a": 1, "b": {"c": 3}}) c.d = [dict(e=1), dict(f=2)] assert c.d[0].e == 1 assert c.d[1].f == 2
def test_two_levels_patch(): c = ConfigDict({"a": 1, "b": {"c": 3}}) d = {"d": 5} c.b.d = d assert c.b.c == 3 assert c.b.d == d
def test_two_levels(): c = ConfigDict({"a": 1, "b": {"c": 3}}) assert c.b.c == 3
def test_change(): c = ConfigDict({"a": 1}) assert c.a == 1 c.a = 2 assert c["a"] == 2
def test_operator(): c = ConfigDict({"a": 1, "b": {"c": 3}}) from operator import attrgetter assert attrgetter('b.c')(c) == 3