def testReplaceEnv(self):
x = {'a': '%{env.ES_PORT}'}
m = {'a': '9300'}
os.environ['ES_PORT'] = '9300'
self.assertEquals(
util.replace(x,
util.EnvironmentVariables(util.FixedVariables({}))),
m)
def testReplace(self):
x = {
'a': 'abc%{var}def %{var}',
'b': ['%{var} %{ var2 } %{var3}'],
'c': {
'd': '%{var2} %{var3}'
}
}
m = {'a': 'abc1def 1', 'b': ['1 2 3'], 'c': {'d': '2 3'}}
self.assertEquals(
m,
util.replace(
x, util.FixedVariables({
'var': '1',
'var2': 2,
'var3': '3'
})))
def parse_service(filename):
with open(filename, 'r') as fd:
document = yaml.load(fd)
# Merge globals.yml files into document
path = os.path.dirname(os.path.abspath(filename))
while '/' in path:
candidate = os.path.join(path, 'globals.yml')
if os.path.exists(candidate):
with open(candidate, 'r') as fd2:
document = util.merge(yaml.load(fd2), document)
path = path[0:path.rindex('/')]
# Start from a service section if it exists
config = document.get('service', {})
# Fetch and merge json template from maven
if util.rget(document,'maven','version') or util.rget(document,'maven','resolve'):
coord = document['maven']
resolver = maven.ArtifactResolver(coord['repository'], coord['groupid'], coord['artifactid'], coord.get('classifier'))
version = coord.get('version') or resolver.resolve(coord['resolve'])
artifact = resolver.fetch(version)
document['variables'] = util.merge(
document.get('variables', {}),
{'lighter.version': artifact.version, 'lighter.uniqueVersion': artifact.uniqueVersion})
config = util.merge(config, artifact.body)
# Merge overrides into json template
config = util.merge(config, document.get('override', {}))
# Substitute variables into the config
config = util.replace(config, document.get('variables', {}))
return Service(filename, document, config)
def parse_service(filename, targetdir=None, verifySecrets=False, profiles=[]):
logging.info("Processing %s", filename)
# Start from a service section if it exists
with open(filename, 'r') as fd:
try:
document = yaml.load(fd)
except yaml.YAMLError as e:
raise RuntimeError("Error parsing file %s: %s" % (filename, e))
# Merge globals.yml files into document
path = os.path.dirname(os.path.abspath(filename))
while '/' in path:
candidate = os.path.join(path, 'globals.yml')
if os.path.exists(candidate):
document = merge_with_service(candidate, document)
path = path[0:path.rindex('/')]
# Merge profile .yml files into document
document = merge_with_profiles(document, profiles)
variables = util.FixedVariables(document.get('variables', {}))
# Environment variables has higher precedence
variables = util.EnvironmentVariables(variables)
# Replace variables in entire document
document = util.replace(document,
variables,
raiseError=False,
escapeVar=False)
config = document.get('service', {})
# Allow resolving version/uniqueVersion variables from docker registry
variables = docker.ImageVariables.create(
variables, document, util.rget(config, 'container', 'docker', 'image'))
# Fetch and merge json template from maven
if util.rget(document, 'maven', 'version') or util.rget(
document, 'maven', 'resolve'):
coord = document['maven']
versionspec = coord.get('version')
if not versionspec:
versionspec = coord['resolve']
logging.warn(
"The 'resolve:' tag is deprecated, please switch to 'version:' which is a drop-in replacement in %s"
% filename)
resolver = maven.ArtifactResolver(coord['repository'],
coord['groupid'],
coord['artifactid'],
coord.get('classifier'))
version = resolver.resolve(versionspec)
artifact = resolver.fetch(version)
config = util.merge(config, artifact.body)
variables = maven.ArtifactVariables(variables, artifact)
# Merge overrides into json template
config = util.merge(config, document.get('override', {}))
# Substitute variables into the config
try:
config = util.replace(config, variables)
except KeyError as e:
raise RuntimeError(
'Failed to parse %s with the following message: %s' %
(filename, str(e.message)))
if 'env' in config:
config['env'] = process_env(filename, verifySecrets, config['env'])
# Generate deploy keys and encrypt secrets
config = secretary.apply(document, config)
checksum = util.checksum(config)
# Include hash of config to detect if an element has been removed
config['labels'] = config.get('labels', {})
config['labels']['com.meltwater.lighter.checksum'] = checksum
# Include a docker label to sort on
if util.rget(config, 'container', 'docker'):
config['container']['docker']['parameters'] = config['container'][
'docker'].get('parameters', [])
config['container']['docker']['parameters'].append({
'key':
'label',
'value':
'com.meltwater.lighter.appid=' + config['id']
})
# Write json file to disk for logging purposes
if targetdir:
outputfile = os.path.join(targetdir, filename + '.json')
# Exception if directory exists, e.g. because another thread created it concurrently
try:
os.makedirs(os.path.dirname(outputfile))
except OSError as e:
pass
with open(outputfile, 'w') as fd:
fd.write(util.toJson(config, indent=4))
return Service(filename, document, config)
def testReplace(self):
x = {'a':'abc%{var}def', 'b':['%{var} %{var2} %{var3}'], 'c': {'d': '%{var2} %{var3}'}}
m = {'a':'abc1def', 'b':['1 2 3'], 'c': {'d': '2 3'}}
self.assertEquals(util.replace(x, {'var':'1', 'var2':2, 'var3': '3'}), m)
def testReplaceEscape(self):
x = {'a': 'abc%%{var}def %%{var}def', 'b': 'abc%{var}def %{var}'}
m = {'a': 'abc%{var}def %{var}def', 'b': 'abc1def 1'}
self.assertEquals(m, util.replace(x, util.FixedVariables({'var': '1'})))
def testReplacePreserveType(self):
x = {'a': 'abc%{var}def', 'b': '%{var}'}
m = {'a': 'abc1def', 'b': 1}
self.assertEquals(m, util.replace(x, util.FixedVariables({'var': 1})))
def parse_service(filename, targetdir=None, verifySecrets=False, profiles=[]):
logging.info("Processing %s", filename)
# Start from a service section if it exists
with open(filename, 'r') as fd:
try:
document = yaml.load(fd)
except yaml.YAMLError as e:
raise RuntimeError("Error parsing file %s: %s" % (filename, e))
# Merge globals.yml files into document
path = os.path.dirname(os.path.abspath(filename))
while '/' in path:
candidate = os.path.join(path, 'globals.yml')
if os.path.exists(candidate):
document = merge_with_service(candidate, document)
path = path[0:path.rindex('/')]
# Merge profile .yml files into document
document = merge_with_profiles(document, profiles)
variables = util.FixedVariables(document.get('variables', {}))
# Environment variables has higher precedence
variables = util.EnvironmentVariables(variables)
# Replace variables in entire document
document = util.replace(document, variables, raiseError=False, escapeVar=False)
config = document.get('service', {})
# Allow resolving version/uniqueVersion variables from docker registry
variables = docker.ImageVariables.create(
variables, document, util.rget(config, 'container', 'docker', 'image'))
# Fetch and merge json template from maven
if util.rget(document, 'maven', 'version') or util.rget(document, 'maven', 'resolve'):
coord = document['maven']
versionspec = coord.get('version')
if not versionspec:
versionspec = coord['resolve']
logging.warn("The 'resolve:' tag is deprecated, please switch to 'version:' which is a drop-in replacement in %s" % filename)
resolver = maven.ArtifactResolver(coord['repository'], coord['groupid'], coord['artifactid'], coord.get('classifier'))
version = resolver.resolve(versionspec)
artifact = resolver.fetch(version)
config = util.merge(config, artifact.body)
variables = maven.ArtifactVariables(variables, artifact)
# Merge overrides into json template
config = util.merge(config, document.get('override', {}))
# Substitute variables into the config
try:
config = util.replace(config, variables)
except KeyError as e:
raise RuntimeError('Failed to parse %s with the following message: %s' % (filename, str(e.message)))
if 'env' in config:
config['env'] = process_env(filename, verifySecrets, config['env'])
# Generate deploy keys and encrypt secrets
config = secretary.apply(document, config)
checksum = util.checksum(config)
# Include hash of config to detect if an element has been removed
config['labels'] = config.get('labels', {})
config['labels']['com.meltwater.lighter.checksum'] = checksum
# Include a docker label to sort on
if util.rget(config, 'container', 'docker'):
config['container']['docker']['parameters'] = config['container']['docker'].get('parameters', [])
config['container']['docker']['parameters'].append({'key': 'label', 'value': 'com.meltwater.lighter.appid='+config['id']})
# Write json file to disk for logging purposes
if targetdir:
outputfile = os.path.join(targetdir, filename + '.json')
# Exception if directory exists, e.g. because another thread created it concurrently
try:
os.makedirs(os.path.dirname(outputfile))
except OSError as e:
pass
with open(outputfile, 'w') as fd:
fd.write(util.toJson(config, indent=4))
return Service(filename, document, config)
def testReplaceEnv(self):
x = {'a': '%{env.ES_PORT}'}
m = {'a': '9300'}
os.environ['ES_PORT'] = '9300'
self.assertEquals(util.replace(x, util.EnvironmentVariables(util.FixedVariables({}))), m)
def testReplaceEscape(self):
x = {'a': 'abc%%{var}def %%{var}def', 'b': 'abc%{var}def %{var}'}
m = {'a': 'abc%{var}def %{var}def', 'b': 'abc1def 1'}
self.assertEquals(m, util.replace(x, util.FixedVariables({'var': '1'})))
def testReplacePreserveType(self):
x = {'a': 'abc%{var}def', 'b': '%{var}'}
m = {'a': 'abc1def', 'b': 1}
self.assertEquals(m, util.replace(x, util.FixedVariables({'var': 1})))
def testReplace(self):
x = {'a': 'abc%{var}def %{var}', 'b': ['%{var} %{ var2 } %{var3}'], 'c': {'d': '%{var2} %{var3}'}}
m = {'a': 'abc1def 1', 'b': ['1 2 3'], 'c': {'d': '2 3'}}
self.assertEquals(m, util.replace(x, util.FixedVariables({'var': '1', 'var2': 2, 'var3': '3'})))