def testMergeListNonexisting(self):
x = {'a': {'b': [1, 2]}}
y = {'a': {'b': {1: 3, 4: 4}}}
try:
util.merge(x, y)
self.fail("Expected exception ValueError")
except IndexError, e:
self.assertEquals("The given list override index a.b[4] doesn't exist", e.message)
def testMergeListNonexisting(self):
x = {'a': {'b': [1, 2]}}
y = {'a': {'b': {1: 3, 4: 4}}}
try:
util.merge(x, y)
self.fail("Expected exception ValueError")
except IndexError, e:
self.assertEquals("The given list override index a.b[4] doesn't exist", e.message)
def testMerge(self):
x = {'a': 1, 'b': 2}
y = {'b': 3, 'c': 4}
z = {'c': 5, 'd': 6}
m = {'a': 1, 'b': 3, 'c': 5, 'd': 6}
self.assertEqual(util.merge(x, y, z),m)
m = {'a': 1, 'b': 2, 'c': 4, 'd': 6}
self.assertEqual(util.merge(z, y, x),m)
def testMerge(self):
x = {'a': 1, 'b': 2}
y = {'b': 3, 'c': 4}
z = {'c': 5, 'd': 6}
m = {'a': 1, 'b': 3, 'c': 5, 'd': 6}
self.assertEqual(m, util.merge(util.merge(x, y), z))
m = {'a': 1, 'b': 2, 'c': 4, 'd': 6}
self.assertEqual(m, util.merge(util.merge(z, y), x))
def merge_with_service(override_file, document):
if not os.path.exists(override_file):
raise RuntimeError('Could not read file %s' % override_file)
with open(override_file, 'r') as fd2:
document = util.merge(yaml.load(fd2), document)
return document
def merge_with_service(override_file, document):
if not os.path.exists(override_file):
raise RuntimeError('Could not read file %s' % override_file)
with open(override_file, 'r') as fd2:
document = util.merge(yaml.load(fd2), document)
return document
def notify(self, payload):
logging.debug("Sending Slack message: %s", json.dumps(payload))
headers = {
'Content-type': 'application/json; charset=UTF-8',
'Authorization': 'Bearer ' + str(self._token or '')
}
for channel in self._channels:
payload_object = {
"channel": channel,
"attachments": [util.merge(payload, self._message_attribs)]
}
self._call('/chat.postMessage', payload_object, headers)
def parse_service(filename):
with open(filename, 'r') as fd:
document = yaml.load(fd)
# Merge globals.yml files into document
path = os.path.dirname(os.path.abspath(filename))
while '/' in path:
candidate = os.path.join(path, 'globals.yml')
if os.path.exists(candidate):
with open(candidate, 'r') as fd2:
document = util.merge(yaml.load(fd2), document)
path = path[0:path.rindex('/')]
# Start from a service section if it exists
config = document.get('service', {})
# Fetch and merge json template from maven
if util.rget(document,'maven','version') or util.rget(document,'maven','resolve'):
coord = document['maven']
resolver = maven.ArtifactResolver(coord['repository'], coord['groupid'], coord['artifactid'], coord.get('classifier'))
version = coord.get('version') or resolver.resolve(coord['resolve'])
artifact = resolver.fetch(version)
document['variables'] = util.merge(
document.get('variables', {}),
{'lighter.version': artifact.version, 'lighter.uniqueVersion': artifact.uniqueVersion})
config = util.merge(config, artifact.body)
# Merge overrides into json template
config = util.merge(config, document.get('override', {}))
# Substitute variables into the config
config = util.replace(config, document.get('variables', {}))
return Service(filename, document, config)
def notify(self, message):
logging.debug("Sending HipChat message: %s", message)
for room in self._rooms:
self._call('/v2/room/%s/notification' % room,
util.merge({'message': message}, self._message_attribs))
def notify(self, message):
logging.debug("Sending HipChat message: %s", message)
for room in self._rooms:
self._call("/v2/room/%s/notification" % room, util.merge({"message": message}, self._message_attribs))
def testMergeListOverrideDeep(self):
x = {'a': [1, {'a': 2, 'b': 3}]}
y = {'a': {1: {'a': 4}}}
m = {'a': [1, {'a': 4, 'b': 3}]}
self.assertEquals(m, util.merge(x, y))
def testMergeLists(self):
x = {'a': [1, 2]}
y = {'a': [2, 3]}
m = {'a': [1, 2, 2, 3]}
self.assertEquals(util.merge(x, y), m)
def notify(targetMarathonUrl, service):
parsedMarathonUrl = urlparse(targetMarathonUrl)
tags = ["environment:%s" % service.environment, "service:%s" % service.id]
title = "Deployed %s to the %s environment" % (service.id,
service.environment)
# Send HipChat notification
notify_message = "Deployed <b>%s</b> with image <b>%s</b> to <b>%s</b> (%s)" % (
service.id, service.image, service.environment,
parsedMarathonUrl.netloc)
if service.releaseNotes:
notify_message += service.releaseNotes
hipchat = HipChat(util.rget(service.document, 'hipchat', 'token'),
util.rget(service.document, 'hipchat', 'url'),
util.rget(service.document, 'hipchat', 'rooms'))
hipchat.notify(notify_message)
# Send Slack notification
notify_payload = {
"title_link":
targetMarathonUrl,
"fields": [{
"title": "Service",
"value": service.id,
"short": 'true'
}, {
"title": "Environment",
"value": service.environment,
"short": 'true'
}, {
"title": "Image",
"value": service.image,
"short": 'true'
}],
"ts":
int(time.time())
}
if service.releaseNotes:
notify_payload = util.merge({'text': service.releaseNotes},
notify_payload)
slack = Slack(util.rget(service.document, 'slack', 'token'),
util.rget(service.document, 'slack', 'url'),
util.rget(service.document, 'slack', 'channels'))
slack.notify(notify_payload)
# Send NewRelic deployment notification
newrelic = NewRelic(util.rget(service.document, 'newrelic', 'token'))
newrelic.notify(util.rget(service.config, 'env', 'NEW_RELIC_APP_NAME'),
service.uniqueVersion)
# Send Datadog deployment notification
datadog = Datadog(
util.rget(service.document, 'datadog', 'token'),
util.toList(util.rget(service.document, 'datadog', 'tags')))
datadog.notify(
aggregation_key="%s_%s" % (service.environment, service.id),
title=title,
message=
"%%%%%% \n Lighter deployed **%s** with image **%s** to **%s** (%s) \n %%%%%%"
% (service.id, service.image, service.environment,
parsedMarathonUrl.netloc),
tags=tags)
# Send Graphite deployment notification
prefix = (util.rget(service.document, 'graphite', 'prefix')
or 'lighter').strip('.')
metricname = '%s.%s.%s.deployments' % (
prefix, service.environment, '.'.join(
filter(bool, service.id.split('/'))))
graphite = Graphite(
util.rget(service.document, 'graphite', 'address'),
util.rget(service.document, 'graphite', 'url'),
util.toList(util.rget(service.document, 'graphite', 'tags')))
graphite.notify(metricname=metricname,
title=title,
message="Lighter deployed %s with image %s to %s (%s)" %
(service.id, service.image, service.environment,
parsedMarathonUrl.netloc),
tags=tags)
def parse_service(filename, targetdir=None, verifySecrets=False, profiles=[]):
logging.info("Processing %s", filename)
# Start from a service section if it exists
with open(filename, 'r') as fd:
try:
document = yaml.load(fd)
except yaml.YAMLError as e:
raise RuntimeError("Error parsing file %s: %s" % (filename, e))
# Merge globals.yml files into document
path = os.path.dirname(os.path.abspath(filename))
while '/' in path:
candidate = os.path.join(path, 'globals.yml')
if os.path.exists(candidate):
document = merge_with_service(candidate, document)
path = path[0:path.rindex('/')]
# Merge profile .yml files into document
document = merge_with_profiles(document, profiles)
variables = util.FixedVariables(document.get('variables', {}))
# Environment variables has higher precedence
variables = util.EnvironmentVariables(variables)
# Replace variables in entire document
document = util.replace(document, variables, raiseError=False, escapeVar=False)
config = document.get('service', {})
# Allow resolving version/uniqueVersion variables from docker registry
variables = docker.ImageVariables.create(
variables, document, util.rget(config, 'container', 'docker', 'image'))
# Fetch and merge json template from maven
if util.rget(document, 'maven', 'version') or util.rget(document, 'maven', 'resolve'):
coord = document['maven']
versionspec = coord.get('version')
if not versionspec:
versionspec = coord['resolve']
logging.warn("The 'resolve:' tag is deprecated, please switch to 'version:' which is a drop-in replacement in %s" % filename)
resolver = maven.ArtifactResolver(coord['repository'], coord['groupid'], coord['artifactid'], coord.get('classifier'))
version = resolver.resolve(versionspec)
artifact = resolver.fetch(version)
config = util.merge(config, artifact.body)
variables = maven.ArtifactVariables(variables, artifact)
# Merge overrides into json template
config = util.merge(config, document.get('override', {}))
# Substitute variables into the config
try:
config = util.replace(config, variables)
except KeyError as e:
raise RuntimeError('Failed to parse %s with the following message: %s' % (filename, str(e.message)))
if 'env' in config:
config['env'] = process_env(filename, verifySecrets, config['env'])
# Generate deploy keys and encrypt secrets
config = secretary.apply(document, config)
checksum = util.checksum(config)
# Include hash of config to detect if an element has been removed
config['labels'] = config.get('labels', {})
config['labels']['com.meltwater.lighter.checksum'] = checksum
# Include a docker label to sort on
if util.rget(config, 'container', 'docker'):
config['container']['docker']['parameters'] = config['container']['docker'].get('parameters', [])
config['container']['docker']['parameters'].append({'key': 'label', 'value': 'com.meltwater.lighter.appid='+config['id']})
# Write json file to disk for logging purposes
if targetdir:
outputfile = os.path.join(targetdir, filename + '.json')
# Exception if directory exists, e.g. because another thread created it concurrently
try:
os.makedirs(os.path.dirname(outputfile))
except OSError as e:
pass
with open(outputfile, 'w') as fd:
fd.write(util.toJson(config, indent=4))
return Service(filename, document, config)
def parse_service(filename, targetdir=None, verifySecrets=False, profiles=[]):
logging.info("Processing %s", filename)
# Start from a service section if it exists
with open(filename, 'r') as fd:
try:
document = yaml.load(fd)
except yaml.YAMLError as e:
raise RuntimeError("Error parsing file %s: %s" % (filename, e))
# Merge globals.yml files into document
path = os.path.dirname(os.path.abspath(filename))
while '/' in path:
candidate = os.path.join(path, 'globals.yml')
if os.path.exists(candidate):
document = merge_with_service(candidate, document)
path = path[0:path.rindex('/')]
# Merge profile .yml files into document
document = merge_with_profiles(document, profiles)
variables = util.FixedVariables(document.get('variables', {}))
# Environment variables has higher precedence
variables = util.EnvironmentVariables(variables)
# Replace variables in entire document
document = util.replace(document,
variables,
raiseError=False,
escapeVar=False)
config = document.get('service', {})
# Allow resolving version/uniqueVersion variables from docker registry
variables = docker.ImageVariables.create(
variables, document, util.rget(config, 'container', 'docker', 'image'))
# Fetch and merge json template from maven
if util.rget(document, 'maven', 'version') or util.rget(
document, 'maven', 'resolve'):
coord = document['maven']
versionspec = coord.get('version')
if not versionspec:
versionspec = coord['resolve']
logging.warn(
"The 'resolve:' tag is deprecated, please switch to 'version:' which is a drop-in replacement in %s"
% filename)
resolver = maven.ArtifactResolver(coord['repository'],
coord['groupid'],
coord['artifactid'],
coord.get('classifier'))
version = resolver.resolve(versionspec)
artifact = resolver.fetch(version)
config = util.merge(config, artifact.body)
variables = maven.ArtifactVariables(variables, artifact)
# Merge overrides into json template
config = util.merge(config, document.get('override', {}))
# Substitute variables into the config
try:
config = util.replace(config, variables)
except KeyError as e:
raise RuntimeError(
'Failed to parse %s with the following message: %s' %
(filename, str(e.message)))
if 'env' in config:
config['env'] = process_env(filename, verifySecrets, config['env'])
# Generate deploy keys and encrypt secrets
config = secretary.apply(document, config)
checksum = util.checksum(config)
# Include hash of config to detect if an element has been removed
config['labels'] = config.get('labels', {})
config['labels']['com.meltwater.lighter.checksum'] = checksum
# Include a docker label to sort on
if util.rget(config, 'container', 'docker'):
config['container']['docker']['parameters'] = config['container'][
'docker'].get('parameters', [])
config['container']['docker']['parameters'].append({
'key':
'label',
'value':
'com.meltwater.lighter.appid=' + config['id']
})
# Write json file to disk for logging purposes
if targetdir:
outputfile = os.path.join(targetdir, filename + '.json')
# Exception if directory exists, e.g. because another thread created it concurrently
try:
os.makedirs(os.path.dirname(outputfile))
except OSError as e:
pass
with open(outputfile, 'w') as fd:
fd.write(util.toJson(config, indent=4))
return Service(filename, document, config)
def testMergeListOverride(self):
x = {'a': [1, 2]}
y = {'a': {1: 3}}
m = {'a': [1, 3]}
self.assertEquals(m, util.merge(x, y))
def testMergeLists(self):
x = {'a': [1, 2]}
y = {'a': [2, 3]}
m = {'a': [1, 2, 2, 3]}
self.assertEquals(m, util.merge(x, y))
def testMergeListOverrideDeep(self):
x = {'a': [1, {'a': 2, 'b': 3}]}
y = {'a': {1: {'a': 4}}}
m = {'a': [1, {'a': 4, 'b': 3}]}
self.assertEquals(m, util.merge(x, y))
def testMergeListOverride(self):
x = {'a': [1, 2]}
y = {'a': {1: 3}}
m = {'a': [1, 3]}
self.assertEquals(m, util.merge(x, y))
