def test_patch_complex():
c = ConfigDict({
"a": 1,
"b": {
"x": 3,
"y": 4
},
"c": {
"x": 5,
"y": 6
},
"d": {
"x": 7,
"y": 8
}
})
d = {"a": 2, "b": {"z": 5}, "c": [1, 2], "d": {"y": 9}}
c.patch(d)
assert c.a == 2
assert c.b.x == 3
assert c.b.y == 4
assert c.b.z == 5
assert c.c == [1, 2]
assert c.d.x == 7
assert c.d.y == 9
def read(text):
if isinstance(text, File):
text = text.read_all()
t = env.from_string(text)
data = yaml.load(t.render(context or dict()))
conf = ConfigDict(data)
if 'patches' in conf and patchable:
leaf = conf
parent = leaf['patches']
del leaf['patches']
parent = File(f.parent.child(parent))
conf = read(parent)
conf.patch(leaf)
return conf
def __init__(self, data=None):
if isinstance(data, Config):
self.data = data.data
else:
# Assume dict
self.data = ConfigDict(data)
self.default_search_paths = [os.getcwd()]
def test_patch_complex():
c = ConfigDict({
"a": 1,
"b": {"x": 3, "y": 4},
"c": {"x": 5, "y": 6},
"d": {"x": 7, "y": 8}
})
d = {"a": 2, "b": {"z": 5}, "c": [1, 2], "d": {"y": 9}}
c.patch(d)
assert c.a == 2
assert c.b.x == 3
assert c.b.y == 4
assert c.b.z == 5
assert c.c == [1, 2]
assert c.d.x == 7
assert c.d.y == 9
class Config(AutoProp):
def __init__(self, data=None):
if isinstance(data, Config):
self.data = data.data
else:
# Assume dict
self.data = ConfigDict(data)
self.default_search_paths = [os.getcwd()]
def flatten(self):
return dict(self.data)
@AutoProp.default
def publish(self):
return self.data.get('publish', None)
@AutoProp.default
def source_dir(self):
try:
default_source = File(self.data.file_path).parent.path
except AttributeError:
default_source = os.getcwd()
source = self.data.get('source_dir', default_source)
return Folder(default_source).child_folder(source)
@AutoProp.default
def source_patterns(self):
patterns = []
config_patterns = self.data.get('source_patterns', ['*.json'])
extappend(patterns, config_patterns)
return patterns
@AutoProp.default
def output_dir(self):
out = self.data.get('output_dir', 'out')
return Folder(self.source_dir).parent.child_folder(out)
@AutoProp.default
def search_paths(self):
paths = []
paths.extend(self.default_search_paths)
paths.append(self.source_dir.path)
config_paths = []
extappend(config_paths, self.data.get('search_paths', []))
config_paths = [Folder(self.source_dir).parent.child(path)
for path in config_paths]
extappend(paths, config_paths)
return list(set(paths))
@AutoProp.default
def context(self):
return self.data.get('context')
@AutoProp.default
def region(self):
return self.data.get('region', 'us-east-1')
def publish(data, push_www=True, push_app=False):
data = ConfigDict(data)
www_archive = upload_www(data, push_www)
app_archive = upload_app(data, push_app)
config_file = File(File(__file__).parent.child('stack/gitbot.yaml'))
config = yaml.load(config_file.read_all())
config['file_path'] = config_file.path
params = data.get('stack_params', dict())
params.update(dict(AppSource=app_archive, WebSource=www_archive))
worker_params = get_worker_outputs(data)
if not worker_params or 'QueueURL' not in worker_params:
raise Exception('Failed to create the worker stack')
params.update(dict(
WorkerQueueURL=worker_params['QueueURL'],
ManagerAccessKey=worker_params['ManagerKey'],
ManagerSecretKey=worker_params['ManagerSecret']
))
config['data'] = data
stack.publish_stack(config,
params=params,
debug=True,
wait=True)
def __init__(self, name, config=None,
env=None,
settings=None,
context=None,
conf_path=None):
if not name:
raise ValueError("project name is required")
self.name = name
self.config = ConfigDict(config or {})
self.config.work_root = self.config.get('work_root',
Folder(os.getcwd()).child('out'))
self.env = env
self.settings = settings
self.conf_path = conf_path
self.context = context
self.annotation_header_template = config.get(
'annotation_header_template',
ANNOTATION_TEMPLATE)
self.depends_template = config.get(
'depends_template',
DEPENDS_TEMPLATE)
self._depends = None
def _load(cls, project_name, env='build', context=None, conf_path=None):
if project_name in __PROJECTS__:
return __PROJECTS__[project_name]
settings = __ENV__.get(env, None)
if not settings:
file_name = env + '.yaml'
if conf_path:
file_name = Folder(conf_path).child(file_name)
settings = yinja.load(file_name, context=context)
__ENV__[env] = settings
config = ConfigDict()
config.update(settings.config)
config.patch(settings.projects.get(project_name, dict()))
project_type = config.get('type_name', 'gitbot.lib.build.Project')
project_class = load_python_object(project_type)
project = project_class(project_name,
config,
env,
settings,
context,
conf_path)
__PROJECTS__[project_name] = project
return project
class Project(AutoProp):
def __init__(self, name, config=None,
env=None,
settings=None,
context=None,
conf_path=None):
if not name:
raise ValueError("project name is required")
self.name = name
self.config = ConfigDict(config or {})
self.config.work_root = self.config.get('work_root',
Folder(os.getcwd()).child('out'))
self.env = env
self.settings = settings
self.conf_path = conf_path
self.context = context
self.annotation_header_template = config.get(
'annotation_header_template',
ANNOTATION_TEMPLATE)
self.depends_template = config.get(
'depends_template',
DEPENDS_TEMPLATE)
self._depends = None
@property
def depends(self):
if not self._depends:
depends = self.config.get('depends', {})
self._depends = {
project_name: self.load_dependent(project_name,
project_env)
for project_name, project_env in depends.items()
}
return self._depends
@AutoProp.default
def work_root(self):
return self.config.get('work_root')
@AutoProp.default
def source_root(self):
return self.config.get('source_root',
Folder(self.work_root).child('sources'))
@AutoProp.default
def build_root(self):
return self.config.get('build_root',
Folder(self.work_root).child('dist'))
@AutoProp.default
def tools_root(self):
return self.config.get('tools_root',
Folder(self.work_root).child('tools'))
@AutoProp.default
def dateformat(self):
return self.config.get('dateformat', '%A, %d. %B %Y %I:%M%p')
@property
def needs_source(self):
return self.config.get('needs_source', True)
@AutoProp.default
def version(self):
return self.config.get('version')
@AutoProp.default
def dir_name(self):
return self.config.get('dir_name', self.name)
@AutoProp.default
def source_dir(self):
return self.config.get('source_dir',
'%s/%s' % (self.source_root, self.dir_name))
@AutoProp.default
def build_dir(self):
return self.config.get('build_dir',
'%s/%s' % (self.build_root, self.dir_name))
@AutoProp.default
def repo_base(self):
return self.config.get('repo_base', 'https://github.com')
@AutoProp.default
def repo_owner(self):
return self.config.get('repo_owner')
@AutoProp.default
def repo_name(self):
return self.config.get('repo_name', self.name)
@AutoProp.default
def repo(self):
return self.config.get('repo',
'%s/%s/%s' % (self.repo_base, self.repo_owner, self.repo_name))
@AutoProp.default
def remote(self):
return self.config.get('remote', 'origin')
@AutoProp.default
def branch(self):
return self.config.get('branch', 'master')
@property
def version_published(self):
return self.tree.tagger.check(str(self.version))
@property
def tree(self):
return Tree(self.source_dir,
self.repo,
self.remote,
self.branch)
def has_source_changed(self):
return self.tree.has_changes()
@property
def revision(self):
try:
return self.tree.get_revision(short=False).strip()
except GitException:
return self.tree.get_revision_remote().strip()
@property
def modified(self):
return self.tree.get_last_committed().strftime(
self.dateformat)
def pull(self, tip_only=False, pull_depends=False):
self.tree.pull(tip_only)
if pull_depends:
for dep in self.depends.itervalues():
dep.pull(tip_only=tip_only)
def merge(self, ref):
if not Folder(self.source_dir).exists:
self.pull()
self.tree.fetch_ref(ref, local='gitbot/local', notags=True)
try:
self.tree.merge('gitbot/local', ff_only=False)
except GitException, gitex:
raise BuildConflictedException(unicode(gitex))
def publish_stack(config, params=None, debug=False, wait=False, **kwargs):
if isinstance(config, File):
file_path = config.path
config = yaml.load(config.read_all())
config["file_path"] = file_path
uploaded = upload_stack(config, **kwargs)
config = uploaded.config
main_stack = _get_main_stack(uploaded.config, uploaded.files)
try:
main = uploaded.result[main_stack]
except KeyError:
raise Exception("Cannot find the main stack[{main}]".format(main=main_stack))
try:
stack_name = config.publish.stack_name
except AttributeError:
raise Exception("Stack name is required in configuration[publish.stack_name].")
defaults = get_params(config)
args = ConfigDict({name: info["value"] for name, info in defaults.iteritems()})
args.patch(params)
params = args
region = config.publish.get("region", "us-east-1")
# Connect to cloud formation and create the stack
cf = connect_cf(region, **kwargs)
try:
cf.describe_stacks(stack_name)
update = True
except:
update = False
params = _transform_params(config.flatten(), params, uploaded.result)
fn = cf.update_stack if update else cf.create_stack
try:
fn(
stack_name,
disable_rollback=debug,
capabilities=["CAPABILITY_IAM"],
template_url=main["url"],
parameters=params,
)
except BotoServerError as bse:
try:
error = json.loads(bse.error_message)
except:
raise bse
try:
message = error["Error"]["Message"]
except KeyError:
raise bse
if message == "No updates are to be performed.":
print "Stack is already up to date."
wait = False
else:
raise bse
if wait:
__wait_while_status(cf, "CREATE_IN_PROGRESS" if not update else "UPDATE_IN_PROGRESS")
return stack_name
def test_copy():
c = ConfigDict({"a": 1, "b": {"c": 3}})
d = c.copy()
assert c == d
c.b.c = 4
assert c != d
def test_patch_simple():
c = ConfigDict({"a": 1, "b": {"c": 3, "e": 4}})
d = {"b": {"e": 5}}
c.patch(d)
assert c.b.c == 3
assert c.b.e == 5
def test_init():
c = ConfigDict({"a": 1})
assert c.a == 1
assert c["a"] == 1
def test_two_levels_assignment():
c = ConfigDict({"a": 1, "b": {"c": 3}})
d = {"d": 5}
c.b = d
assert c.b.d == 5
assert c.b == d
def test_list():
c = ConfigDict({"a": 1, "b": {"c": 3}})
c.d = [dict(e=1), dict(f=2)]
assert c.d[0].e == 1
assert c.d[1].f == 2
def test_two_levels_patch():
c = ConfigDict({"a": 1, "b": {"c": 3}})
d = {"d": 5}
c.b.d = d
assert c.b.c == 3
assert c.b.d == d
def test_two_levels_assignment():
c = ConfigDict({"a": 1, "b": {"c": 3}})
d = {"d": 5}
c.b = d
assert c.b.d == 5
assert c.b == d
def test_two_levels():
c = ConfigDict({"a": 1, "b": {"c": 3}})
assert c.b.c == 3
def test_change():
c = ConfigDict({"a": 1})
assert c.a == 1
c.a = 2
assert c["a"] == 2
def test_operator():
c = ConfigDict({"a": 1, "b": {"c": 3}})
from operator import attrgetter
assert attrgetter('b.c')(c) == 3
def test_copy():
c = ConfigDict({"a": 1, "b": {"c": 3}})
d = c.copy()
assert c == d
c.b.c = 4
assert c != d
def test_patch_simple():
c = ConfigDict({"a": 1, "b": {"c": 3, "e": 4}})
d = {"b": {"e": 5}}
c.patch(d)
assert c.b.c == 3
assert c.b.e == 5
def test_list():
c = ConfigDict({"a": 1, "b": {"c": 3}})
c.d = [dict(e=1), dict(f=2)]
assert c.d[0].e == 1
assert c.d[1].f == 2
def test_change():
c = ConfigDict({"a": 1})
assert c.a == 1
c.a = 2
assert c["a"] == 2
