def init(self, ref, path, config=None, env=None, deployment=None, sync=False, **kwargs):
app, service = self.parse_app_ref(ref, kwargs, app_only=True)
if config:
config_file = os.path.expanduser(config)
else:
config_file = os.path.join(path, 'mcloud.yml')
config = YamlConfig(file=config_file, app_name=app)
config.load(process=False)
deployment_info = yield self._remote_exec('deployment_info', name=deployment)
if deployment_info:
if not deployment:
deployment = deployment_info['name']
if not deployment_info['local']:
yield self._remote_exec('init', app, config=config.export(), env=env, deployment=deployment)
if sync:
yield self.sync(os.path.realpath(path), '%s@%s' % (app, self.host), no_remove=False, force=True,
full=True)
else:
yield self._remote_exec('init', app, path=os.path.realpath(path), config=config.export(),
deployment=deployment)
else:
print('There is no deployments configured yet.\n\n'
'You can create new local deployment using following command:\n'
'\n $ mcloud deployment-create local\n\n')
reactor.stop()
def test_filter_env_non_dict():
"""
pass through non-dict elements
"""
yc = YamlConfig(env='xx')
result = yc.filter_env('foo')
assert result == 'foo'
def test_load_config_from_config():
config = YamlConfig(source='{"foo": "bar"}', app_name='myapp')
flexmock(config).should_receive('prepare').with_args({'foo': 'bar'}).once().and_return({'foo': 'bar1'})
flexmock(config).should_receive('validate').with_args({'foo': 'bar1'}).once()
flexmock(config).should_receive('process').with_args(OrderedDict([('foo', 'bar1')]), path=None, app_name='myapp', client='booo').once()
config.load(client='booo')
def test_filter_env_remove():
"""
~xxx syntax: remove elements that not match
"""
yc = YamlConfig(env='xx')
result = yc.filter_env({'~foo': 'bar'})
assert result == {}
def test_load_config_prepare():
config = {
'foo': {
'image': 'foo',
'env': {
'bar': 'baz'
},
'cmd': 'some'
},
'bar': {
'extend': 'foo',
'cmd': 'other'
}
}
yc = YamlConfig()
processed = yc.prepare(config)
assert processed['bar'] == {
'image': 'foo',
'env': {
'bar': 'baz'
},
'cmd': 'other'
}
def test_load_config_prepare():
config = {
'foo': {
'image': 'foo',
'env': {
'bar': 'baz'
},
'cmd': 'some'
},
'bar': {
'extend': 'foo',
'cmd': 'other'
}
}
yc = YamlConfig()
processed = yc.prepare(config)
assert processed['bar'] == {
'image': 'foo',
'env': {
'bar': 'baz'
},
'cmd': 'other'
}
def test_filter_env_non_dict():
"""
pass through non-dict elements
"""
yc = YamlConfig(env='xx')
result = yc.filter_env('foo')
assert result == 'foo'
def test_process_with_local_config():
c = YamlConfig(source='{"nginx": {"image": "bar"}, "---": {"commands": {"bar": ["foo"]}}}')
flexmock(c)
c.should_receive('process_local_config').once().with_args({"commands": {"bar": ["foo"]}})
c.load(process=False)
def test_build_build_volumes_several(tmpdir):
s = Service()
c = YamlConfig()
foo1 = tmpdir.mkdir('foo1')
foo2 = tmpdir.mkdir('foo2')
foo3 = tmpdir.mkdir('foo3')
c.process_volumes_build(
s, {'volumes': {
'foo1': 'bar1',
'foo2': 'bar2',
'foo3': 'bar3',
}}, str(tmpdir))
assert s.volumes == [{
'local': str(foo1),
'remote': 'bar1'
}, {
'local': str(foo2),
'remote': 'bar2'
}, {
'local': str(foo3),
'remote': 'bar3'
}]
def test_load_config_prepare_env():
yc = YamlConfig(env='myenv')
flexmock(yc).should_receive('filter_env').with_args({'foo': {'bar': 'baz'}}).once().and_return({'fas': {'bar': 'baz'}})
processed = yc.prepare({'foo': {'bar': 'baz'}})
assert processed == {'fas': {'bar': 'baz'}}
def test_validate_ordered_dict():
c = YamlConfig()
config = OrderedDict([
('web',
OrderedDict([('image', 'orchardup/nginx'),
('volumes', OrderedDict([('public', '/var/www')]))]))
])
assert c.validate(config)
def test_build_image_dockerfile():
s = Service()
c = YamlConfig()
c.process_image_build(s, {'build': 'foo/bar'}, '/base/path')
assert isinstance(s.image_builder, DockerfileImageBuilder)
assert s.image_builder.path == '/base/path/foo/bar'
def test_build_image_image():
s = Service()
c = YamlConfig()
c.process_image_build(s, {'image': 'foo/bar'}, '/base/path')
assert isinstance(s.image_builder, PrebuiltImageBuilder)
assert s.image_builder.image == 'foo/bar'
def test_build_image_dockerfile():
s = Service()
c = YamlConfig()
c.process_image_build(s, {'build': 'foo/bar'}, '/base/path')
assert isinstance(s.image_builder, DockerfileImageBuilder)
assert s.image_builder.path == '/base/path/foo/bar'
def test_hosts_config():
c = YamlConfig()
c.hosts = OrderedDict(
(('boo', '*****@*****.**'), ('foo', '*****@*****.**')))
assert c.get_command_host() == '*****@*****.**'
assert c.get_command_host('boo') == '*****@*****.**'
assert c.get_command_host('foo') == '*****@*****.**'
def test_filter_env_non_dict_in_match():
"""
~xxx syntax: should contain dict
"""
yc = YamlConfig(env='foo')
with pytest.raises(TypeError):
yc.filter_env({'~foo': 'bar'})
def test_build_image_image():
s = Service()
c = YamlConfig()
c.process_image_build(s, {'image': 'foo/bar'}, '/base/path')
assert isinstance(s.image_builder, PrebuiltImageBuilder)
assert s.image_builder.image == 'foo/bar'
def test_build_inline_dockerfile():
s = Service()
c = YamlConfig()
c.process_image_build(s, {'dockerfile': 'FROM foo\nWORKDIR boo'}, '/base/path')
assert isinstance(s.image_builder, InlineDockerfileImageBuilder)
assert s.image_builder.files['Dockerfile'] == 'FROM foo\nWORKDIR boo'
def test_filter_env_remove():
"""
~xxx syntax: remove elements that not match
"""
yc = YamlConfig(env='xx')
result = yc.filter_env({
'~foo': 'bar'
})
assert result == {}
def test_build_inline_dockerfile():
s = Service()
c = YamlConfig()
c.process_image_build(s, {'dockerfile': 'FROM foo\nWORKDIR boo'},
'/base/path')
assert isinstance(s.image_builder, InlineDockerfileImageBuilder)
assert s.image_builder.files['Dockerfile'] == 'FROM foo\nWORKDIR boo'
def test_filter_env_dict_no_env():
"""
pass through dict elements without ~
"""
yc = YamlConfig(env='xx')
flexmock(yc).should_call('filter_env').with_args({'foo': 'bar'}).once()
flexmock(yc).should_call('filter_env').with_args('bar').once()
result = yc.filter_env({'foo': 'bar'})
assert result == {'foo': 'bar'}
def test_filter_env_non_dict_in_match():
"""
~xxx syntax: should contain dict
"""
yc = YamlConfig(env='foo')
with pytest.raises(TypeError):
yc.filter_env({
'~foo': 'bar'
})
def test_load_config(tmpdir):
p = tmpdir.join('mcloud.yml')
p.write('foo: bar')
config = YamlConfig(file=p.realpath(), app_name='myapp')
flexmock(config).should_receive('prepare').with_args({'foo': 'bar'}).once().and_return({'foo': 'bar1'})
flexmock(config).should_receive('validate').with_args({'foo': 'bar1'}).once()
flexmock(config).should_receive('process').with_args(OrderedDict([('foo', 'bar1')]), path=None, app_name='myapp', client='booo').once()
config.load(client='booo')
def test_filter_env_dict_no_env():
"""
pass through dict elements without ~
"""
yc = YamlConfig(env='xx')
flexmock(yc).should_call('filter_env').with_args({'foo': 'bar'}).once()
flexmock(yc).should_call('filter_env').with_args('bar').once()
result = yc.filter_env({'foo': 'bar'})
assert result == {'foo': 'bar'}
def test_hosts_config():
c = YamlConfig()
c.hosts = OrderedDict((
('boo', '*****@*****.**'),
('foo', '*****@*****.**')
))
assert c.get_command_host() == '*****@*****.**'
assert c.get_command_host('boo') == '*****@*****.**'
assert c.get_command_host('foo') == '*****@*****.**'
def test_build_build_volumes_basepath(tmpdir):
s = Service()
c = YamlConfig()
c.process_volumes_build(s, {'volumes': {
'.': 'bar1',
}}, str(tmpdir))
assert s.volumes == [
{'local': str(tmpdir), 'remote': 'bar1'},
]
def test_build_build_volumes_hackish_paths(path, result):
s = Service()
c = YamlConfig()
c.process_volumes_build(s, {'volumes': {
path: 'bar',
}}, '/foo')
assert s.volumes == [
{'local': result, 'remote': 'bar'},
]
def test_load_config_not_valid(tmpdir):
p = tmpdir.join('mcloud.yml')
p.write('foo: bar')
config = YamlConfig(file=p.realpath(), app_name='myapp')
flexmock(config).should_receive('prepare').with_args({'foo': 'bar'}).once().and_return({'foo': 'bar1'})
flexmock(config).should_receive('validate').with_args({'foo': 'bar1'}).once().and_raise(ValueError('boo'))
flexmock(config).should_receive('process').times(0)
with pytest.raises(ConfigParseError):
config.load()
def test_filter_env_keep():
"""
~xxx syntax: keep elements that match
"""
yc = YamlConfig(env='foo')
flexmock(yc).should_call('filter_env').with_args({'~foo': {'bar': 'baz'}}).once().and_return({'bar': 'baz'})
flexmock(yc).should_call('filter_env').with_args({'bar': 'baz'}).once().and_return({'bar': 'baz'})
flexmock(yc).should_call('filter_env').with_args('baz').once().and_return('baz')
result = yc.filter_env({
'~foo': {'bar': 'baz'}
})
assert result == {'bar': 'baz'}
def test_load_config_prepare_env():
yc = YamlConfig(env='myenv')
flexmock(yc).should_receive('filter_env').with_args({
'foo': {
'bar': 'baz'
}
}).once().and_return({'fas': {
'bar': 'baz'
}})
processed = yc.prepare({'foo': {'bar': 'baz'}})
assert processed == {'fas': {'bar': 'baz'}}
def test_build_build_volumes_single_file(tmpdir):
s = Service()
c = YamlConfig()
tmpdir.join('nginx.conf').write('foo')
c.process_volumes_build(s, {'volumes': {
'nginx.conf': 'bar1',
}}, str(tmpdir))
assert s.volumes == [
{'local': str(tmpdir.join('nginx.conf')), 'remote': 'bar1'},
]
def test_build_build_volumes_hackish_paths(path, result):
s = Service()
c = YamlConfig()
c.process_volumes_build(s, {'volumes': {
path: 'bar',
}}, '/foo')
assert s.volumes == [
{
'local': result,
'remote': 'bar'
},
]
def test_build_build_volumes_basepath(tmpdir):
s = Service()
c = YamlConfig()
c.process_volumes_build(s, {'volumes': {
'.': 'bar1',
}}, str(tmpdir))
assert s.volumes == [
{
'local': str(tmpdir),
'remote': 'bar1'
},
]
def test_build_build_volumes_single_file(tmpdir):
s = Service()
c = YamlConfig()
tmpdir.join('nginx.conf').write('foo')
c.process_volumes_build(s, {'volumes': {
'nginx.conf': 'bar1',
}}, str(tmpdir))
assert s.volumes == [
{
'local': str(tmpdir.join('nginx.conf')),
'remote': 'bar1'
},
]
def test_build_build_env_several():
s = Service()
c = YamlConfig(env='prod')
c.process_env_build(s, {'env': {
'foo1': 'bar1',
'foo2': 'bar2',
'foo3': 'bar3',
}}, '/base/path')
assert s.env == {
'env': 'prod',
'foo1': 'bar1',
'foo2': 'bar2',
'foo3': 'bar3',
}
def test_load_config_from_config():
config = YamlConfig(source='{"foo": "bar"}', app_name='myapp')
flexmock(config).should_receive('prepare').with_args({
'foo': 'bar'
}).once().and_return({'foo': 'bar1'})
flexmock(config).should_receive('validate').with_args({
'foo': 'bar1'
}).once()
flexmock(config).should_receive('process').with_args(OrderedDict([
('foo', 'bar1')
]),
path=None,
app_name='myapp',
client='booo').once()
config.load(client='booo')
def test_load_config_not_valid(tmpdir):
p = tmpdir.join('mcloud.yml')
p.write('foo: bar')
config = YamlConfig(file=p.realpath(), app_name='myapp')
flexmock(config).should_receive('prepare').with_args({
'foo': 'bar'
}).once().and_return({'foo': 'bar1'})
flexmock(config).should_receive('validate').with_args({
'foo': 'bar1'
}).once().and_raise(ValueError('boo'))
flexmock(config).should_receive('process').times(0)
with pytest.raises(ConfigParseError):
config.load()
def test_build_build_env_several():
s = Service()
c = YamlConfig(env='prod')
c.process_env_build(
s, {'env': {
'foo1': 'bar1',
'foo2': 'bar2',
'foo3': 'bar3',
}}, '/base/path')
assert s.env == {
'env': 'prod',
'foo1': 'bar1',
'foo2': 'bar2',
'foo3': 'bar3',
}
def test_process_local_config_commands():
c = YamlConfig()
c.process_local_config({'commands': {
'push (Upload code to remove server)': ['sync . {host} --path ticcet/'],
'pull': ['foo', 'bar']
}})
assert c.get_commands() == {
'push': {
'help': 'Upload code to remove server',
'commands': ['sync . {host} --path ticcet/']
},
'pull': {
'help': 'pull command',
'commands': ['foo', 'bar']
}
}
def test_filter_env_keep():
"""
~xxx syntax: keep elements that match
"""
yc = YamlConfig(env='foo')
flexmock(yc).should_call('filter_env').with_args({
'~foo': {
'bar': 'baz'
}
}).once().and_return({'bar': 'baz'})
flexmock(yc).should_call('filter_env').with_args({
'bar': 'baz'
}).once().and_return({'bar': 'baz'})
flexmock(yc).should_call('filter_env').with_args('baz').once().and_return(
'baz')
result = yc.filter_env({'~foo': {'bar': 'baz'}})
assert result == {'bar': 'baz'}
def test_build_build_volumes_several(tmpdir):
s = Service()
c = YamlConfig()
foo1 = tmpdir.mkdir('foo1')
foo2 = tmpdir.mkdir('foo2')
foo3 = tmpdir.mkdir('foo3')
c.process_volumes_build(s, {'volumes': {
'foo1': 'bar1',
'foo2': 'bar2',
'foo3': 'bar3',
}}, str(tmpdir))
assert s.volumes == [
{'local': str(foo1), 'remote': 'bar1'},
{'local': str(foo2), 'remote': 'bar2'},
{'local': str(foo3), 'remote': 'bar3'}
]
def test_load_config(tmpdir):
p = tmpdir.join('mcloud.yml')
p.write('foo: bar')
config = YamlConfig(file=p.realpath(), app_name='myapp')
flexmock(config).should_receive('prepare').with_args({
'foo': 'bar'
}).once().and_return({'foo': 'bar1'})
flexmock(config).should_receive('validate').with_args({
'foo': 'bar1'
}).once()
flexmock(config).should_receive('process').with_args(OrderedDict([
('foo', 'bar1')
]),
path=None,
app_name='myapp',
client='booo').once()
config.load(client='booo')
def test_process_with_app_name():
c = YamlConfig()
flexmock(c)
c.should_receive('process_image_build').once()
c.should_receive('process_volumes_build').once()
c.should_receive('process_command_build').once()
c.process({'nginx': {'foo': 'bar'}}, path='foo', app_name='myapp')
assert isinstance(c.services['nginx.myapp'], Service)
assert c.services['nginx.myapp'].name == 'nginx.myapp'
def test_process_local_config_commands():
c = YamlConfig()
c.process_local_config({
'commands': {
'push (Upload code to remove server)':
['sync . {host} --path ticcet/'],
'pull': ['foo', 'bar']
}
})
assert c.get_commands() == {
'push': {
'help': 'Upload code to remove server',
'commands': ['sync . {host} --path ticcet/']
},
'pull': {
'help': 'pull command',
'commands': ['foo', 'bar']
}
}
def config(self, ref, diff=False, config=None, update=False, set_env=None, **kwargs):
app, service = self.parse_app_ref(ref, kwargs, app_only=True)
app_config = yield self._remote_exec('config', app)
parser_env = set_env or app_config['env']
if diff or (not update and not set_env):
old_config = YamlConfig(source=unicode(app_config['source']), app_name=app, env=parser_env)
old_config.load(process=False)
from collections import OrderedDict
yaml.add_representer(unicode, yaml.representer.SafeRepresenter.represent_unicode)
yaml.add_representer(OrderedDict, self.represent_ordereddict)
olds = yaml.dump(old_config.config, default_flow_style=False)
if not update and not diff and not set_env:
x = PrettyTable(["Name", "Value"], hrules=ALL, align='l', header=False)
x.align = "l"
x.add_row(['Config', olds])
x.add_row(['Environment', app_config['env']])
x.add_row(['Path', app_config['path']])
print(x)
else:
if config:
config_file = os.path.expanduser(config)
else:
config_file = 'mcloud.yml'
new_config = YamlConfig(file=config_file, app_name=app, env=parser_env)
new_config.load(process=False)
if diff:
yaml.add_representer(unicode, yaml.representer.SafeRepresenter.represent_unicode)
yaml.add_representer(OrderedDict, self.represent_ordereddict)
news = yaml.dump(new_config.config, default_flow_style=False)
if olds == news:
print('Configs are identical.')
else:
for line in unified_diff(olds.splitlines(1), news.splitlines(1)):
if line.endswith('\n'):
line = line[0:-1]
if line.startswith('+'):
print color_text(line, color='green')
elif line.startswith('-'):
print color_text(line, color='red')
else:
print line
else:
if set_env and not update:
yield self._remote_exec('update', app, env=set_env)
else:
yield self._remote_exec('update', app, config=new_config.export(), env=set_env)
def load(self, need_details=False):
try:
if 'source' in self.config:
yaml_config = YamlConfig(source=self.config['source'], app_name=self.name, path=self.config['path'], env=self.get_env())
elif 'path' in self.config:
yaml_config = YamlConfig(file=os.path.join(self.config['path'], 'mcloud.yml'), app_name=self.name, path=self.config['path'])
else:
self.error = {
'msg': 'Can not parse config'
}
defer.returnValue(None)
deployment = yield self.get_deployment()
if not deployment:
self.error = {
'msg': 'No deployment found'
}
else:
client = deployment.get_client()
yield yaml_config.load(client=client)
yield defer.gatherResults([service.inspect() for service in yaml_config.get_services().values()])
if need_details:
defer.returnValue(self._details(yaml_config, deployment))
else:
defer.returnValue(yaml_config)
except (ValueError, DeploymentDoesNotExist) as e:
config_ = {'name': self.name, 'config': self.config, 'services': [], 'running': False, 'status': 'error',
'message': '%s When loading config: %s' % (e.message, self.config)}
defer.returnValue(config_)
def test_process_with_app_name():
c = YamlConfig()
flexmock(c)
c.should_receive('process_image_build').once()
c.should_receive('process_volumes_build').once()
c.should_receive('process_command_build').once()
c.process({
'nginx': {'foo': 'bar'}
}, path='foo', app_name='myapp')
assert isinstance(c.services['nginx.myapp'], Service)
assert c.services['nginx.myapp'].name == 'nginx.myapp'
def test_process_with_local_config():
c = YamlConfig(
source=
'{"nginx": {"image": "bar"}, "---": {"commands": {"bar": ["foo"]}}}')
flexmock(c)
c.should_receive('process_local_config').once().with_args(
{"commands": {
"bar": ["foo"]
}})
c.load(process=False)
def init(self,
ref,
path,
config=None,
env=None,
deployment=None,
sync=False,
**kwargs):
app, service = self.parse_app_ref(ref, kwargs, app_only=True)
if config:
config_file = os.path.expanduser(config)
else:
config_file = os.path.join(path, 'mcloud.yml')
config = YamlConfig(file=config_file, app_name=app)
config.load(process=False)
deployment_info = yield self._remote_exec('deployment_info',
name=deployment)
if deployment_info:
if not deployment:
deployment = deployment_info['name']
if not deployment_info['local']:
yield self._remote_exec('init',
app,
config=config.export(),
env=env,
deployment=deployment)
if sync:
yield self.sync(os.path.realpath(path),
'%s@%s' % (app, self.host),
no_remove=False,
force=True,
full=True)
else:
yield self._remote_exec('init',
app,
path=os.path.realpath(path),
config=config.export(),
deployment=deployment)
else:
print(
'There is no deployments configured yet.\n\n'
'You can create new local deployment using following command:\n'
'\n $ mcloud deployment-create local\n\n')
reactor.stop()
def load(self, need_details=False):
try:
if 'source' in self.config:
yaml_config = YamlConfig(source=self.config['source'],
app_name=self.name,
path=self.config['path'],
env=self.get_env())
elif 'path' in self.config:
yaml_config = YamlConfig(file=os.path.join(
self.config['path'], 'mcloud.yml'),
app_name=self.name,
path=self.config['path'])
else:
self.error = {'msg': 'Can not parse config'}
defer.returnValue(None)
deployment = yield self.get_deployment()
if not deployment:
self.error = {'msg': 'No deployment found'}
else:
client = deployment.get_client()
yield yaml_config.load(client=client)
yield defer.gatherResults([
service.inspect()
for service in yaml_config.get_services().values()
])
if need_details:
defer.returnValue(self._details(yaml_config, deployment))
else:
defer.returnValue(yaml_config)
except (ValueError, DeploymentDoesNotExist) as e:
config_ = {
'name': self.name,
'config': self.config,
'services': [],
'running': False,
'status': 'error',
'message':
'%s When loading config: %s' % (e.message, self.config)
}
defer.returnValue(config_)
def test_build_image_dockerfile_no_path():
s = Service()
c = YamlConfig()
with pytest.raises(ConfigParseError):
c.process_image_build(s, {'build': 'foo/bar'}, None)
"""
:param config:
:type config: mcloud.config.YamlConfig
:return:
"""
for name, command in config.get_commands().items():
cmd_instance = LocalCommand(config, command)
cmd = subparsers.add_parser('*%s' % name, help=command['help'])
cmd.add_argument('to', help='Host to use with command', default=None, choices=config.hosts, nargs='?')
cmd.set_defaults(func=cmd_instance.call)
if os.path.exists('mcloud.yml'):
config = YamlConfig(file='mcloud.yml')
config.load(process=False)
load_commands(config)
def cli(help_, arguments=None, by_ref=False, name=None):
def cmd_decorator(func):
cmd = subparsers.add_parser(name or func.__name__.replace('_', '-'), help=help_)
command_settings[func.__name__] = {
'need_app': False
}
if arguments:
def test_build_image_empty():
s = Service()
c = YamlConfig()
with pytest.raises(ValueError) as e:
c.process_image_build(s, {}, '/base/path')
def test_get_service():
c = YamlConfig()
c.services = {'foo': 'bar'}
assert c.get_service('foo') == 'bar'
def test_get_service_no():
c = YamlConfig()
c.services = {'foo': 'bar'}
with pytest.raises(UnknownServiceError):
c.get_service('baz')
def test_not_existent_file():
with pytest.raises(ValueError):
YamlConfig(file='Not existent path')
def test_get_service_no():
c = YamlConfig()
c.services = {'foo': 'bar'}
with pytest.raises(UnknownServiceError):
c.get_service('baz')
def test_get_service():
c = YamlConfig()
c.services = {'foo': 'bar'}
assert c.get_service('foo') == 'bar'
def test_build_image_empty():
s = Service()
c = YamlConfig()
with pytest.raises(ValueError) as e:
c.process_image_build(s, {}, '/base/path')
def test_build_image_dockerfile_no_path():
s = Service()
c = YamlConfig()
with pytest.raises(ConfigParseError):
c.process_image_build(s, {'build': 'foo/bar'}, None)