def deploy(marathonurl, filenames, noop=False, force=False, targetdir=None, profiles=[]):
services = parse_services(filenames, targetdir, profiles=profiles)
for service in services:
try:
targetMarathonUrl = marathonurl or util.rget(service.document, 'marathon', 'url')
if not targetMarathonUrl:
raise RuntimeError("No Marathon URL defined for service %s" % service.filename)
# See if service config has changed by comparing the checksum
appurl = get_marathon_url(targetMarathonUrl, service.config['id'], force)
prevVersion = get_marathon_app(appurl)
if util.rget(prevVersion, 'labels', 'com.meltwater.lighter.checksum') == service.checksum:
logging.info("Service already deployed with same config: %s", service.filename)
continue
# Skip deployment if noop flag is given
if noop:
continue
# Deploy new service config
logging.info("Deploying %s", service.filename)
util.jsonRequest(appurl, data=service.config, method='PUT')
# Send deployment notifications
notify(targetMarathonUrl, service)
except urllib2.HTTPError as e:
raise RuntimeError("Failed to deploy %s HTTP %d (%s) - Response: %s" % (service.filename, e.code, e, e.read())), None, sys.exc_info()[2]
except urllib2.URLError as e:
raise RuntimeError("Failed to deploy %s (%s)" % (service.filename, e)), None, sys.exc_info()[2]
def testRedeployWithoutChange(self):
service1 = lighter.parse_service('src/resources/yaml/staging/myservice-servicekey.yml')
service2 = lighter.parse_service('src/resources/yaml/staging/myservice-servicekey.yml')
self.assertNotEqual(service1.config, service2.config)
checksum1 = util.rget(service1.config, 'labels', 'com.meltwater.lighter.checksum')
self.assertIsNotNone(checksum1)
self.assertEqual(checksum1, util.rget(service2.config, 'labels', 'com.meltwater.lighter.checksum'))
self.assertNotEqual(service1.config['env']['DEPLOY_PUBLIC_KEY'], service2.config['env']['DEPLOY_PUBLIC_KEY'])
self.assertNotEqual(service1.config['env']['DEPLOY_PRIVATE_KEY'], service2.config['env']['DEPLOY_PRIVATE_KEY'])
def testRedeployWithoutChange(self):
service1 = lighter.parse_service('src/resources/yaml/staging/myservice-servicekey.yml')
service2 = lighter.parse_service('src/resources/yaml/staging/myservice-servicekey.yml')
self.assertNotEqual(service1.config, service2.config)
checksum1 = util.rget(service1.config, 'labels', 'com.meltwater.lighter.checksum')
self.assertIsNotNone(checksum1)
self.assertEqual(checksum1, util.rget(service2.config, 'labels', 'com.meltwater.lighter.checksum'))
self.assertNotEqual(service1.config['env']['DEPLOY_PUBLIC_KEY'], service2.config['env']['DEPLOY_PUBLIC_KEY'])
self.assertNotEqual(service1.config['env']['DEPLOY_PRIVATE_KEY'], service2.config['env']['DEPLOY_PRIVATE_KEY'])
def releaseNotes(self):
notes = util.rget(self.document, 'hipchat', 'releaseNotes')
if notes:
return notes
docker_label = util.rget(self.document, 'hipchat',
'message.image.label')
if not docker_label:
return None
labels = docker.ImageVariables.create(self.config, self.document,
self.image).pop('labels')
if labels:
return labels.get(docker_label)
return None
def _fetch(self, version, uniqueVersion=None, metadata={}):
url = '{0}/{1}/{2}/{3}/{2}-{4}'.format(self._url, self._groupid.replace('.', '/'), self._artifactid, version, uniqueVersion or version)
if self._classifier is not None:
url += '-' + self._classifier
url += '.json'
# Extract unique version number from metadata
if not uniqueVersion:
timestamp = util.rget(metadata,'versioning','snapshot','timestamp') or util.rget(metadata,'versioning','lastUpdated')
buildNumber = util.rget(metadata,'versioning','snapshot','buildNumber')
if timestamp or buildNumber:
uniqueVersion = '%s-%s-%s' % (version.replace('-SNAPSHOT',''), timestamp, buildNumber)
try:
return Artifact(version, uniqueVersion, self._classifier, util.jsonRequest(url))
except urllib2.HTTPError, e:
raise RuntimeError("Failed to retrieve %s HTTP %d (%s)" % (url, e.code, e)), None, sys.exc_info()[2]
def deploy(marathonurl,
filenames,
noop=False,
force=False,
canaryGroup=None,
profiles=[]):
services = parse_services(filenames, canaryGroup, profiles)
for service in services:
try:
targetMarathonUrl = marathonurl or util.rget(
service.document, 'marathon', 'url')
if not targetMarathonUrl:
raise RuntimeError("No Marathon URL defined for service %s" %
service.filename)
# See if service config has changed by comparing the checksum
appurl = get_marathon_appurl(targetMarathonUrl,
service.config['id'], force)
prevVersion = get_marathon_app(appurl)
if util.rget(prevVersion, 'labels',
'com.meltwater.lighter.checksum') == service.checksum:
logging.info("Service already deployed with same config: %s",
service.filename)
continue
# Skip deployment if noop flag is given
if noop:
continue
# Deploy new service config
logging.info("Deploying %s", service.filename)
util.jsonRequest(appurl, data=service.config, method='PUT')
# Send deployment notifications
notify(targetMarathonUrl, service)
except urllib2.HTTPError as e:
raise RuntimeError(
"Failed to deploy %s HTTP %d (%s) - Response: %s" %
(service.filename, e.code, e,
e.read())), None, sys.exc_info()[2]
except urllib2.URLError as e:
raise RuntimeError("Failed to deploy %s (%s)" %
(service.filename, e)), None, sys.exc_info()[2]
return services
def _fetch(self, version, uniqueVersion=None, metadata={}):
url = '{0}/{1}/{2}/{3}/{2}-{4}'.format(self._url, self._groupid.replace('.', '/'), self._artifactid, version, uniqueVersion or version)
if self._classifier is not None:
url += '-' + self._classifier
url += '.json'
# Extract unique version number from metadata
if not uniqueVersion:
timestamp = util.rget(metadata, 'versioning', 'snapshot', 'timestamp') or util.rget(metadata, 'versioning', 'lastUpdated')
buildNumber = util.rget(metadata, 'versioning', 'snapshot', 'buildNumber')
if timestamp or buildNumber:
uniqueVersion = '-'.join(filter(bool, [version.replace('-SNAPSHOT', ''), timestamp, buildNumber]))
try:
return Artifact(version, uniqueVersion, self._classifier, util.jsonRequest(url))
except urllib2.HTTPError as e:
raise RuntimeError("Failed to retrieve %s HTTP %d (%s)" % (url, e.code, e)), None, sys.exc_info()[2]
except urllib2.URLError as e:
raise RuntimeError("Failed to retrieve %s (%s)" % (url, e)), None, sys.exc_info()[2]
def apply(document, config):
"""
Generates a deploy key, injects config/master keys and performs the extra deployment time encryption of secrets.
"""
url = util.rget(document, 'secretary', 'url')
if not url:
return config
# Check for encrypted secrets
found = False
for key, value in config.get('env', {}).iteritems():
envelopes = extractEnvelopes(value)
found = found or envelopes
# e.g. dots aren't valid in a shell identifier, so `secretary decrypt -e` wouldn't work with them
if envelopes and not _SHELL_IDENTIFIER_RE.match(key):
raise RuntimeError((
"The env var '%s' has an encrypted value but its name is not a valid shell script identifier and not supported by Secretary. "
+
"Only alphanumeric characters and underscores are supported, starting with an alphabetic or underscore character."
+
"Please check https://github.com/meltwater/lighter#secrets-management ."
) % key)
# Avoid adding public keys if no secrets present
if not found:
return config
result = deepcopy(config)
masterKey = decodePublicKey(
util.rget(document, 'secretary', 'master', 'publickey'))
result['env'] = result.get('env', {})
result['env']['SECRETARY_URL'] = url
result['env']['MASTER_PUBLIC_KEY'] = encodeKey(masterKey)
# Autogenerate a deploy key
deployKey = PrivateKey.generate()
result['env']['DEPLOY_PRIVATE_KEY'] = KeyValue(encodeKey(deployKey))
result['env']['DEPLOY_PUBLIC_KEY'] = KeyValue(
encodeKey(deployKey.public_key))
return result
def fetch(self, version):
trailer = '-SNAPSHOT'
if not version.endswith(trailer):
return self._fetch(version)
# Try to resolve unique/timestamped snapshot versions from maven-metadata.xml
logging.debug('Trying to resolve %s to a unique timestamp-buildnumber version', version)
url = '{0}/{1}/{2}/{3}/maven-metadata.xml'.format(self._url, self._groupid.replace('.', '/'), self._artifactid, version)
metadata = {}
try:
metadata = util.xmlRequest(url)
except urllib2.URLError:
logging.debug('Failed to fetch %s', url)
# Find a matching snapshot version (Gradle doesn't create <snapshotVersions> but Maven does)
timestamp = util.rget(metadata, 'versioning', 'snapshot', 'timestamp')
buildNumber = util.rget(metadata, 'versioning', 'snapshot', 'buildNumber')
snapshot = '-'.join(filter(bool, [version[0:len(version) - len(trailer)], timestamp, buildNumber])
) if (timestamp is not None and buildNumber is not None) else None
return self._fetch(version, snapshot, metadata)
def resolve(self, expression):
# If it's not a valid version range expression, assume it's a specific version
if not VersionRange.isExpression(expression):
return expression
# Fetch the available versions for this artifact
metadata = util.xmlRequest('{0}/{1}/{2}/maven-metadata.xml'.format(self._url, self._groupid.replace('.', '/'), self._artifactid))
versions = util.toList(util.rget(metadata, 'versioning', 'versions', 'version'))
logging.debug('%s:%s candidate versions %s', self._groupid, self._artifactid, versions)
# Select the version that best matches the version range expression
return self.selectVersion(expression, versions)
def apply_canary(canaryGroup, service):
if not canaryGroup:
return
mangledGroup = util.mangle(canaryGroup)
config = service.config
config['id'] = '%s-canary-%s-%s' % (
config.get('id', ''), mangledGroup,
hashlib.md5('%s-%s' % (canaryGroup, service.filename)).hexdigest())
config['instances'] = 1
# Rewrite service ports and add label for meltwater/proxymatic to read
config['labels'] = config.get('labels', {})
ports = config.get('ports', [])
for port, i in zip(ports, range(len(ports))):
config['labels']['com.meltwater.proxymatic.port.%d.servicePort' %
i] = str(port)
config['ports'][i] = 0
mappings = util.toList(
util.rget(config, 'container', 'docker', 'portMappings'))
for mapping, i in zip(mappings, range(len(mappings))):
config['labels']['com.meltwater.proxymatic.port.%d.servicePort' %
i] = str(mapping['servicePort'])
mapping['servicePort'] = 0
# Apply canary label to task so old canaries can be destroyed
config['labels']['com.meltwater.lighter.filename'] = service.filename
config['labels']['com.meltwater.lighter.canary.group'] = mangledGroup
# Apply canary label to container so container metrics can be aggregated
if util.rget(config, 'container', 'docker'):
config['container']['docker']['parameters'] = config['container'][
'docker'].get('parameters', [])
config['container']['docker']['parameters'].append({
'key':
'label',
'value':
'com.meltwater.lighter.canary.group=' + mangledGroup
})
def apply(document, config):
"""
Generates a deploy key, injects config/master keys and performs the extra deployment time encryption of secrets.
"""
url = util.rget(document, 'secretary', 'url')
if not url:
return config
# Check for encrypted secrets
found = False
for key, value in config.get('env', {}).iteritems():
envelopes = extractEnvelopes(value)
found = found or envelopes
# e.g. dots aren't valid in a shell identifier, so `secretary decrypt -e` wouldn't work with them
if envelopes and not _SHELL_IDENTIFIER_RE.match(key):
raise RuntimeError(
("The env var '%s' is not a valid shell script identifier and not supported by Secretary. " +
"Only alphanumeric characters and underscores are supported, starting with an alphabetic or underscore character.") % key)
# Avoid adding public keys if no secrets present
if not found:
return config
result = deepcopy(config)
masterKey = decodePublicKey(util.rget(document, 'secretary', 'master', 'publickey'))
result['env'] = result.get('env', {})
result['env']['SECRETARY_URL'] = url
result['env']['MASTER_PUBLIC_KEY'] = encodeKey(masterKey)
# Autogenerate a deploy key
deployKey = PrivateKey.generate()
result['env']['DEPLOY_PRIVATE_KEY'] = KeyValue(encodeKey(deployKey))
result['env']['DEPLOY_PUBLIC_KEY'] = KeyValue(encodeKey(deployKey.public_key))
return result
def parse_service(filename):
with open(filename, 'r') as fd:
document = yaml.load(fd)
# Merge globals.yml files into document
path = os.path.dirname(os.path.abspath(filename))
while '/' in path:
candidate = os.path.join(path, 'globals.yml')
if os.path.exists(candidate):
with open(candidate, 'r') as fd2:
document = util.merge(yaml.load(fd2), document)
path = path[0:path.rindex('/')]
# Start from a service section if it exists
config = document.get('service', {})
# Fetch and merge json template from maven
if util.rget(document,'maven','version') or util.rget(document,'maven','resolve'):
coord = document['maven']
resolver = maven.ArtifactResolver(coord['repository'], coord['groupid'], coord['artifactid'], coord.get('classifier'))
version = coord.get('version') or resolver.resolve(coord['resolve'])
artifact = resolver.fetch(version)
document['variables'] = util.merge(
document.get('variables', {}),
{'lighter.version': artifact.version, 'lighter.uniqueVersion': artifact.uniqueVersion})
config = util.merge(config, artifact.body)
# Merge overrides into json template
config = util.merge(config, document.get('override', {}))
# Substitute variables into the config
config = util.replace(config, document.get('variables', {}))
return Service(filename, document, config)
def deploy(marathonurl, filenames, noop=False, force=False, targetdir=None):
parsedMarathonUrl = urlparse(marathonurl)
services = parse_services(filenames, targetdir)
for service in services:
try:
appurl = get_marathon_url(marathonurl, service.config['id'], force)
# See if service config has changed
prevConfig = get_marathon_app(appurl)
if compare_service_versions(service.config, prevConfig):
logging.debug("Service already deployed with same config: %s", service.filename)
continue
# Skip deployment if noop flag is given
if noop:
continue
# Deploy new service config
logging.info("Deploying %s", service.filename)
util.jsonRequest(appurl, data=service.config, method='PUT')
# Send HipChat notification
hipchat = HipChat(
util.rget(service.document,'hipchat','token'),
util.rget(service.document,'hipchat','url'),
util.rget(service.document,'hipchat','rooms'))
hipchat.notify("Deployed <b>%s</b> with image <b>%s</b> to <b>%s</b> (%s)" %
(service.id, service.image, service.environment, parsedMarathonUrl.netloc))
# Send NewRelic deployment notification
newrelic = NewRelic(util.rget(service.document, 'newrelic', 'token'))
newrelic.notify(
util.rget(service.config, 'env', 'NEW_RELIC_APP_NAME'),
service.uniqueVersion
)
# Send Datadog deployment notification
datadog = Datadog(util.rget(service.document, 'datadog', 'token'))
datadog.notify(
title="Deployed %s to the %s environment" % (service.id, service.environment),
message="%%%%%% \n Lighter deployed **%s** with image **%s** to **%s** (%s) \n %%%%%%" % (service.id, service.image, service.environment, parsedMarathonUrl.netloc),
id=service.id,
tags=[
"environment:%s" % service.environment,
"service:%s" % service.id
]
)
except urllib2.HTTPError, e:
raise RuntimeError("Failed to deploy %s HTTP %d (%s)" % (service.filename, e.code, e)), None, sys.exc_info()[2]
except urllib2.URLError, e:
raise RuntimeError("Failed to deploy %s (%s)" % (service.filename, e)), None, sys.exc_info()[2]
def _tryRegistryV2Response(self, url):
"""
Return an image first compatibility history object using the Docker Registry V2 API
"""
try:
expandedurl = self._expandurl(url)
response = util.jsonRequest(expandedurl, timeout=15)
except urllib2.HTTPError as e:
if e.code != 404:
obfuscatedurl = self._expandurl(url, obfuscateauth=True)
raise RuntimeError("Failed to call %s (%s)" %
(obfuscatedurl, e)), None, sys.exc_info()[2]
return None
except urllib2.URLError as e:
return None
# Extract the first compatibility image id if present
return util.rget(response, 'history', 0, 'v1Compatibility')
def __init__(self, wrappedResolver, document, image):
m = re.search(
'^(?:([\w\-]+[\.:].[\w\.\-:]+)/)?(?:([\w\.\-]+)/)?([\w\.\-/]+)(?::([\w\.\-/]+))?$',
image)
if not m:
raise ValueError("Failed to parse Docker image coordinates '%s'" %
image)
self._wrappedResolver = wrappedResolver
self._document = document
self._image = image
self._registry = m.group(1) if m.group(1) else 'registry-1.docker.io'
self._organization = m.group(2)
self._repository = m.group(3)
self._tag = m.group(4) if m.group(4) else 'latest'
self._auth = util.rget(document, 'docker', 'registries',
self._registry, 'auth')
logging.debug("Parsed image '%s' as %s" %
(self._image, (self._registry, self._organization,
self._repository, self._tag)))
def parse_service(filename, targetdir=None, verifySecrets=False, profiles=[]):
logging.info("Processing %s", filename)
# Start from a service section if it exists
with open(filename, 'r') as fd:
try:
document = yaml.load(fd)
except yaml.YAMLError as e:
raise RuntimeError("Error parsing file %s: %s" % (filename, e))
# Merge globals.yml files into document
path = os.path.dirname(os.path.abspath(filename))
while '/' in path:
candidate = os.path.join(path, 'globals.yml')
if os.path.exists(candidate):
document = merge_with_service(candidate, document)
path = path[0:path.rindex('/')]
# Merge profile .yml files into document
document = merge_with_profiles(document, profiles)
variables = util.FixedVariables(document.get('variables', {}))
# Environment variables has higher precedence
variables = util.EnvironmentVariables(variables)
# Replace variables in entire document
document = util.replace(document,
variables,
raiseError=False,
escapeVar=False)
config = document.get('service', {})
# Allow resolving version/uniqueVersion variables from docker registry
variables = docker.ImageVariables.create(
variables, document, util.rget(config, 'container', 'docker', 'image'))
# Fetch and merge json template from maven
if util.rget(document, 'maven', 'version') or util.rget(
document, 'maven', 'resolve'):
coord = document['maven']
versionspec = coord.get('version')
if not versionspec:
versionspec = coord['resolve']
logging.warn(
"The 'resolve:' tag is deprecated, please switch to 'version:' which is a drop-in replacement in %s"
% filename)
resolver = maven.ArtifactResolver(coord['repository'],
coord['groupid'],
coord['artifactid'],
coord.get('classifier'))
version = resolver.resolve(versionspec)
artifact = resolver.fetch(version)
config = util.merge(config, artifact.body)
variables = maven.ArtifactVariables(variables, artifact)
# Merge overrides into json template
config = util.merge(config, document.get('override', {}))
# Substitute variables into the config
try:
config = util.replace(config, variables)
except KeyError as e:
raise RuntimeError(
'Failed to parse %s with the following message: %s' %
(filename, str(e.message)))
if 'env' in config:
config['env'] = process_env(filename, verifySecrets, config['env'])
# Generate deploy keys and encrypt secrets
config = secretary.apply(document, config)
checksum = util.checksum(config)
# Include hash of config to detect if an element has been removed
config['labels'] = config.get('labels', {})
config['labels']['com.meltwater.lighter.checksum'] = checksum
# Include a docker label to sort on
if util.rget(config, 'container', 'docker'):
config['container']['docker']['parameters'] = config['container'][
'docker'].get('parameters', [])
config['container']['docker']['parameters'].append({
'key':
'label',
'value':
'com.meltwater.lighter.appid=' + config['id']
})
# Write json file to disk for logging purposes
if targetdir:
outputfile = os.path.join(targetdir, filename + '.json')
# Exception if directory exists, e.g. because another thread created it concurrently
try:
os.makedirs(os.path.dirname(outputfile))
except OSError as e:
pass
with open(outputfile, 'w') as fd:
fd.write(util.toJson(config, indent=4))
return Service(filename, document, config)
def notify(targetMarathonUrl, service):
parsedMarathonUrl = urlparse(targetMarathonUrl)
tags = ["environment:%s" % service.environment, "service:%s" % service.id]
title = "Deployed %s to the %s environment" % (service.id, service.environment)
# Send HipChat notification
hipchat = HipChat(
util.rget(service.document, 'hipchat', 'token'),
util.rget(service.document, 'hipchat', 'url'),
util.rget(service.document, 'hipchat', 'rooms'))
hipchat.notify("Deployed <b>%s</b> with image <b>%s</b> to <b>%s</b> (%s)" %
(service.id, service.image, service.environment, parsedMarathonUrl.netloc))
# Send NewRelic deployment notification
newrelic = NewRelic(util.rget(service.document, 'newrelic', 'token'))
newrelic.notify(
util.rget(service.config, 'env', 'NEW_RELIC_APP_NAME'),
service.uniqueVersion)
# Send Datadog deployment notification
datadog = Datadog(
util.rget(service.document, 'datadog', 'token'),
util.toList(util.rget(service.document, 'datadog', 'tags')))
datadog.notify(
aggregation_key="%s_%s" % (service.environment, service.id),
title=title,
message="%%%%%% \n Lighter deployed **%s** with image **%s** to **%s** (%s) \n %%%%%%" % (
service.id, service.image, service.environment, parsedMarathonUrl.netloc),
tags=tags)
# Send Graphite deployment notification
prefix = (util.rget(service.document, 'graphite', 'prefix') or 'lighter').strip('.')
metricname = '%s.%s.%s.deployments' % (
prefix, service.environment,
'.'.join(filter(bool, service.id.split('/'))))
graphite = Graphite(
util.rget(service.document, 'graphite', 'address'),
util.rget(service.document, 'graphite', 'url'),
util.toList(util.rget(service.document, 'graphite', 'tags')))
graphite.notify(
metricname=metricname,
title=title,
message="Lighter deployed %s with image %s to %s (%s)" % (
service.id, service.image, service.environment, parsedMarathonUrl.netloc),
tags=tags)
def parse_service(filename, targetdir=None, verifySecrets=False, profiles=[]):
logging.info("Processing %s", filename)
# Start from a service section if it exists
with open(filename, 'r') as fd:
try:
document = yaml.load(fd)
except yaml.YAMLError as e:
raise RuntimeError("Error parsing file %s: %s" % (filename, e))
# Merge globals.yml files into document
path = os.path.dirname(os.path.abspath(filename))
while '/' in path:
candidate = os.path.join(path, 'globals.yml')
if os.path.exists(candidate):
document = merge_with_service(candidate, document)
path = path[0:path.rindex('/')]
# Merge profile .yml files into document
document = merge_with_profiles(document, profiles)
variables = util.FixedVariables(document.get('variables', {}))
# Environment variables has higher precedence
variables = util.EnvironmentVariables(variables)
# Replace variables in entire document
document = util.replace(document, variables, raiseError=False, escapeVar=False)
config = document.get('service', {})
# Allow resolving version/uniqueVersion variables from docker registry
variables = docker.ImageVariables.create(
variables, document, util.rget(config, 'container', 'docker', 'image'))
# Fetch and merge json template from maven
if util.rget(document, 'maven', 'version') or util.rget(document, 'maven', 'resolve'):
coord = document['maven']
versionspec = coord.get('version')
if not versionspec:
versionspec = coord['resolve']
logging.warn("The 'resolve:' tag is deprecated, please switch to 'version:' which is a drop-in replacement in %s" % filename)
resolver = maven.ArtifactResolver(coord['repository'], coord['groupid'], coord['artifactid'], coord.get('classifier'))
version = resolver.resolve(versionspec)
artifact = resolver.fetch(version)
config = util.merge(config, artifact.body)
variables = maven.ArtifactVariables(variables, artifact)
# Merge overrides into json template
config = util.merge(config, document.get('override', {}))
# Substitute variables into the config
try:
config = util.replace(config, variables)
except KeyError as e:
raise RuntimeError('Failed to parse %s with the following message: %s' % (filename, str(e.message)))
if 'env' in config:
config['env'] = process_env(filename, verifySecrets, config['env'])
# Generate deploy keys and encrypt secrets
config = secretary.apply(document, config)
checksum = util.checksum(config)
# Include hash of config to detect if an element has been removed
config['labels'] = config.get('labels', {})
config['labels']['com.meltwater.lighter.checksum'] = checksum
# Include a docker label to sort on
if util.rget(config, 'container', 'docker'):
config['container']['docker']['parameters'] = config['container']['docker'].get('parameters', [])
config['container']['docker']['parameters'].append({'key': 'label', 'value': 'com.meltwater.lighter.appid='+config['id']})
# Write json file to disk for logging purposes
if targetdir:
outputfile = os.path.join(targetdir, filename + '.json')
# Exception if directory exists, e.g. because another thread created it concurrently
try:
os.makedirs(os.path.dirname(outputfile))
except OSError as e:
pass
with open(outputfile, 'w') as fd:
fd.write(util.toJson(config, indent=4))
return Service(filename, document, config)
def testRGet(self):
self.assertEqual('c', util.rget({'a': [{'b': 'c'}]}, 'a', 0, 'b'))
self.assertEqual(None, util.rget({'a': [{'b': 'c'}]}, 'a', 0, 'd'))
self.assertEqual(None, util.rget({'a': [{'b': 'c'}]}, 'a', 1, 'b'))
self.assertEqual(None, util.rget({'a': [{'b': 'c'}]}, 'a', -1, 'b'))
def notify(targetMarathonUrl, service):
parsedMarathonUrl = urlparse(targetMarathonUrl)
tags = ["environment:%s" % service.environment, "service:%s" % service.id]
title = "Deployed %s to the %s environment" % (service.id,
service.environment)
# Send HipChat notification
notify_message = "Deployed <b>%s</b> with image <b>%s</b> to <b>%s</b> (%s)" % (
service.id, service.image, service.environment,
parsedMarathonUrl.netloc)
if service.releaseNotes:
notify_message += service.releaseNotes
hipchat = HipChat(util.rget(service.document, 'hipchat', 'token'),
util.rget(service.document, 'hipchat', 'url'),
util.rget(service.document, 'hipchat', 'rooms'))
hipchat.notify(notify_message)
# Send Slack notification
notify_payload = {
"title_link":
targetMarathonUrl,
"fields": [{
"title": "Service",
"value": service.id,
"short": 'true'
}, {
"title": "Environment",
"value": service.environment,
"short": 'true'
}, {
"title": "Image",
"value": service.image,
"short": 'true'
}],
"ts":
int(time.time())
}
if service.releaseNotes:
notify_payload = util.merge({'text': service.releaseNotes},
notify_payload)
slack = Slack(util.rget(service.document, 'slack', 'token'),
util.rget(service.document, 'slack', 'url'),
util.rget(service.document, 'slack', 'channels'))
slack.notify(notify_payload)
# Send NewRelic deployment notification
newrelic = NewRelic(util.rget(service.document, 'newrelic', 'token'))
newrelic.notify(util.rget(service.config, 'env', 'NEW_RELIC_APP_NAME'),
service.uniqueVersion)
# Send Datadog deployment notification
datadog = Datadog(
util.rget(service.document, 'datadog', 'token'),
util.toList(util.rget(service.document, 'datadog', 'tags')))
datadog.notify(
aggregation_key="%s_%s" % (service.environment, service.id),
title=title,
message=
"%%%%%% \n Lighter deployed **%s** with image **%s** to **%s** (%s) \n %%%%%%"
% (service.id, service.image, service.environment,
parsedMarathonUrl.netloc),
tags=tags)
# Send Graphite deployment notification
prefix = (util.rget(service.document, 'graphite', 'prefix')
or 'lighter').strip('.')
metricname = '%s.%s.%s.deployments' % (
prefix, service.environment, '.'.join(
filter(bool, service.id.split('/'))))
graphite = Graphite(
util.rget(service.document, 'graphite', 'address'),
util.rget(service.document, 'graphite', 'url'),
util.toList(util.rget(service.document, 'graphite', 'tags')))
graphite.notify(metricname=metricname,
title=title,
message="Lighter deployed %s with image %s to %s (%s)" %
(service.id, service.image, service.environment,
parsedMarathonUrl.netloc),
tags=tags)
def resolve(self, expression):
metadata = util.xmlRequest('{0}/{1}/{2}/maven-metadata.xml'.format(self._url, self._groupid.replace('.', '/'), self._artifactid))
versions = util.toList(util.rget(metadata,'versioning','versions','version'))
logging.debug('%s:%s candidate versions %s', self._groupid, self._artifactid, versions)
return self.selectVersion(expression, versions)
def notify(targetMarathonUrl, service):
parsedMarathonUrl = urlparse(targetMarathonUrl)
tags = ["environment:%s" % service.environment, "service:%s" % service.id]
title = "Deployed %s to the %s environment" % (service.id,
service.environment)
# Send HipChat notification
hipchat = HipChat(util.rget(service.document, 'hipchat', 'token'),
util.rget(service.document, 'hipchat', 'url'),
util.rget(service.document, 'hipchat', 'rooms'))
hipchat.notify(
"Deployed <b>%s</b> with image <b>%s</b> to <b>%s</b> (%s)" %
(service.id, service.image, service.environment,
parsedMarathonUrl.netloc))
# Send NewRelic deployment notification
newrelic = NewRelic(util.rget(service.document, 'newrelic', 'token'))
newrelic.notify(util.rget(service.config, 'env', 'NEW_RELIC_APP_NAME'),
service.uniqueVersion)
# Send Datadog deployment notification
datadog = Datadog(
util.rget(service.document, 'datadog', 'token'),
util.toList(util.rget(service.document, 'datadog', 'tags')))
datadog.notify(
aggregation_key="%s_%s" % (service.environment, service.id),
title=title,
message=
"%%%%%% \n Lighter deployed **%s** with image **%s** to **%s** (%s) \n %%%%%%"
% (service.id, service.image, service.environment,
parsedMarathonUrl.netloc),
tags=tags)
# Send Graphite deployment notification
prefix = (util.rget(service.document, 'graphite', 'prefix')
or 'lighter').strip('.')
metricname = '%s.%s.%s.deployments' % (
prefix, service.environment, '.'.join(
filter(bool, service.id.split('/'))))
graphite = Graphite(
util.rget(service.document, 'graphite', 'address'),
util.rget(service.document, 'graphite', 'url'),
util.toList(util.rget(service.document, 'graphite', 'tags')))
graphite.notify(metricname=metricname,
title=title,
message="Lighter deployed %s with image %s to %s (%s)" %
(service.id, service.image, service.environment,
parsedMarathonUrl.netloc),
tags=tags)
def uniqueVersion(self):
return util.rget(self.document, 'variables', 'lighter.uniqueVersion') or \
(self.image.split(':')[1] if ':' in self.image else 'latest')
def testRGet(self):
self.assertEqual('c', util.rget({'a': [{'b': 'c'}]}, 'a', 0, 'b'))
self.assertEqual(None, util.rget({'a': [{'b': 'c'}]}, 'a', 0, 'd'))
self.assertEqual(None, util.rget({'a': [{'b': 'c'}]}, 'a', 1, 'b'))
self.assertEqual(None, util.rget({'a': [{'b': 'c'}]}, 'a', -1, 'b'))
def environment(self):
return util.rget(self.document, 'facts', 'environment') or 'default'
def image(self):
return util.rget(self.config, 'container', 'docker', 'image') or ''
def image(self):
return util.rget(self.config,'container','docker','image') or ''
def uniqueVersion(self):
return util.rget(self.document, 'variables', 'lighter.uniqueVersion') or \
(self.image.split(':')[1] if ':' in self.image else 'latest')
def environment(self):
return util.rget(self.document,'facts','environment') or 'default'
