def deploy(marathonurl, filenames, noop=False, force=False, targetdir=None, profiles=[]):
services = parse_services(filenames, targetdir, profiles=profiles)
for service in services:
try:
targetMarathonUrl = marathonurl or util.rget(service.document, 'marathon', 'url')
if not targetMarathonUrl:
raise RuntimeError("No Marathon URL defined for service %s" % service.filename)
# See if service config has changed by comparing the checksum
appurl = get_marathon_url(targetMarathonUrl, service.config['id'], force)
prevVersion = get_marathon_app(appurl)
if util.rget(prevVersion, 'labels', 'com.meltwater.lighter.checksum') == service.checksum:
logging.info("Service already deployed with same config: %s", service.filename)
continue
# Skip deployment if noop flag is given
if noop:
continue
# Deploy new service config
logging.info("Deploying %s", service.filename)
util.jsonRequest(appurl, data=service.config, method='PUT')
# Send deployment notifications
notify(targetMarathonUrl, service)
except urllib2.HTTPError as e:
raise RuntimeError("Failed to deploy %s HTTP %d (%s) - Response: %s" % (service.filename, e.code, e, e.read())), None, sys.exc_info()[2]
except urllib2.URLError as e:
raise RuntimeError("Failed to deploy %s (%s)" % (service.filename, e)), None, sys.exc_info()[2]
def deploy(marathonurl, filenames, noop=False, force=False, targetdir=None):
parsedMarathonUrl = urlparse(marathonurl)
services = parse_services(filenames, targetdir)
for service in services:
try:
appurl = get_marathon_url(marathonurl, service.config['id'], force)
# See if service config has changed
prevConfig = get_marathon_app(appurl)
if compare_service_versions(service.config, prevConfig):
logging.debug("Service already deployed with same config: %s", service.filename)
continue
# Skip deployment if noop flag is given
if noop:
continue
# Deploy new service config
logging.info("Deploying %s", service.filename)
util.jsonRequest(appurl, data=service.config, method='PUT')
# Send HipChat notification
hipchat = HipChat(
util.rget(service.document,'hipchat','token'),
util.rget(service.document,'hipchat','url'),
util.rget(service.document,'hipchat','rooms'))
hipchat.notify("Deployed <b>%s</b> with image <b>%s</b> to <b>%s</b> (%s)" %
(service.id, service.image, service.environment, parsedMarathonUrl.netloc))
# Send NewRelic deployment notification
newrelic = NewRelic(util.rget(service.document, 'newrelic', 'token'))
newrelic.notify(
util.rget(service.config, 'env', 'NEW_RELIC_APP_NAME'),
service.uniqueVersion
)
# Send Datadog deployment notification
datadog = Datadog(util.rget(service.document, 'datadog', 'token'))
datadog.notify(
title="Deployed %s to the %s environment" % (service.id, service.environment),
message="%%%%%% \n Lighter deployed **%s** with image **%s** to **%s** (%s) \n %%%%%%" % (service.id, service.image, service.environment, parsedMarathonUrl.netloc),
id=service.id,
tags=[
"environment:%s" % service.environment,
"service:%s" % service.id
]
)
except urllib2.HTTPError, e:
raise RuntimeError("Failed to deploy %s HTTP %d (%s)" % (service.filename, e.code, e)), None, sys.exc_info()[2]
except urllib2.URLError, e:
raise RuntimeError("Failed to deploy %s (%s)" % (service.filename, e)), None, sys.exc_info()[2]
def _call(self, endpoint, data, headers):
if not self._url or not self._token:
logging.debug('Slack is not enabled')
return
try:
url = self._url.rstrip('/') + endpoint
logging.debug('Calling Slack endpoint %s', endpoint)
util.jsonRequest(url, data=data, method='POST', headers=headers)
except urllib2.URLError as e:
logging.warn(str(e))
return {}
def _call(self, endpoint, data):
if not self._address or not self._url:
logging.debug('Graphite is not enabled')
return
try:
url = self._url.rstrip('/') + endpoint
logging.debug('Calling Graphite endpoint %s', endpoint)
util.jsonRequest(url, data=data, method='POST')
except urllib2.URLError as e:
logging.warn('Graphite._call: ' + str(e))
return {}
def _call(self, endpoint, data):
if not self._url or not self._token:
logging.debug('Datadog is not enabled')
return
try:
url = self._url.rstrip('/') + endpoint + '?api_key=' + self._token
logging.debug('Calling Datadog endpoint %s', endpoint)
util.jsonRequest(url, data=data, method='POST')
except urllib2.URLError as e:
logging.warn(str(e))
return {}
def _call(self, endpoint, data):
if not self._url or not self._token:
logging.debug("HipChat is not enabled")
return
try:
url = self._url.rstrip("/") + endpoint + "?auth_token=" + self._token
logging.debug("Calling HipChat endpoint %s", endpoint)
util.jsonRequest(url, data=data, method="POST")
except urllib2.URLError, e:
logging.warn(str(e))
return {}
def _call(self, endpoint, data):
if not self._address or not self._url:
logging.debug('Graphite is not enabled')
return
try:
url = self._url.rstrip('/') + endpoint
logging.debug('Calling Graphite endpoint %s', endpoint)
util.jsonRequest(url, data=data, method='POST')
except urllib2.URLError as e:
logging.warn(str(e))
return {}
def _resolvePost(self, url, data=None, *args, **kwargs):
if url.startswith("file:"):
return jsonRequest(url, data, *args, **kwargs)
if "/v2/apps" in url and data:
self.assertEquals(data["container"]["docker"]["image"], "meltwater/myservice:1.0.0")
self._resolvePostCalled = True
return {"app": {}}
def wrapper(url, method="GET", data=None, *args, **kwargs):
if url.startswith("file:"):
return jsonRequest(url, data, *args, **kwargs)
if url == appurl and method == "PUT" and data:
return {}
if url == appurl and method == "GET":
return {"app": {}}
return None
def wrapper(url, method='GET', data=None, *args, **kwargs):
if url.startswith('file:'):
return jsonRequest(url, data, *args, **kwargs)
if url == appurl and method == 'PUT' and data:
return {}
if url == appurl and method == 'GET':
return {'app': {}}
return None
def deploy(marathonurl,
filenames,
noop=False,
force=False,
canaryGroup=None,
profiles=[]):
services = parse_services(filenames, canaryGroup, profiles)
for service in services:
try:
targetMarathonUrl = marathonurl or util.rget(
service.document, 'marathon', 'url')
if not targetMarathonUrl:
raise RuntimeError("No Marathon URL defined for service %s" %
service.filename)
# See if service config has changed by comparing the checksum
appurl = get_marathon_appurl(targetMarathonUrl,
service.config['id'], force)
prevVersion = get_marathon_app(appurl)
if util.rget(prevVersion, 'labels',
'com.meltwater.lighter.checksum') == service.checksum:
logging.info("Service already deployed with same config: %s",
service.filename)
continue
# Skip deployment if noop flag is given
if noop:
continue
# Deploy new service config
logging.info("Deploying %s", service.filename)
util.jsonRequest(appurl, data=service.config, method='PUT')
# Send deployment notifications
notify(targetMarathonUrl, service)
except urllib2.HTTPError as e:
raise RuntimeError(
"Failed to deploy %s HTTP %d (%s) - Response: %s" %
(service.filename, e.code, e,
e.read())), None, sys.exc_info()[2]
except urllib2.URLError as e:
raise RuntimeError("Failed to deploy %s (%s)" %
(service.filename, e)), None, sys.exc_info()[2]
return services
def wrapper(url, method='GET', data=None, *args, **kwargs):
if url.startswith('file:'):
return jsonRequest(url, data, *args, **kwargs)
if url == appurl and method == 'PUT' and data:
self.assertEquals(data['container']['docker']['image'], 'meltwater/myservice:1.0.0')
self._called = True
return {}
if url == appurl and method == 'GET':
return {'app': {}}
return None
def request(url, data=None, *args, **kwargs):
if url.startswith('file:'):
return jsonRequest(url, data, *args, **kwargs)
if url == knownurl:
self._dockerRegistryCalled = True
return response
logging.debug(url)
raise urllib2.URLError("Unknown URL %s" % url)
def wrapper(url, method='GET', data=None, *args, **kwargs):
if url.startswith('file:') and method == 'GET':
return jsonRequest(url, data, *args, **kwargs)
if url == '%s/v2/apps?label=com.meltwater.lighter.canary.group%%3D%%3Dgeneric' % marathonurl and method == 'GET':
return {'apps': [canary2.config, canary3.config]}
if url == '%s/v2/apps%s' % (
marathonurl, canary3.config['id']) and method == 'DELETE':
self._deleted = True
return {}
raise RuntimeError("Unexpected HTTP %s call to %s" % (method, url))
def request(url, data=None, *args, **kwargs):
if url.startswith('file:'):
return jsonRequest(url, data, *args, **kwargs)
if url == knownurl:
self._dockerRegistryCalled = True
return response
logging.debug(url)
raise urllib2.URLError("Unknown URL %s" % url)
def wrapper(url, method='GET', data=None, *args, **kwargs):
if url.startswith('file:'):
return jsonRequest(url, data, *args, **kwargs)
if url == appurl and method == 'PUT' and data:
self.assertEquals(data['container']['docker']['image'],
'meltwater/myservice:1.0.0')
self._called = True
return {}
if url == appurl and method == 'GET':
return {'app': {}}
return None
def get_marathon_app(url):
try:
return util.jsonRequest(url)['app']
except urllib2.HTTPError as e:
logging.debug(str(e))
if e.code == 404:
return {}
else:
raise RuntimeError("Failed to get app info %s HTTP %d (%s) - Response: %s" % (url, e.code, e, e.read())), None, sys.exc_info()[2]
except urllib2.URLError as e:
logging.debug(str(e))
raise RuntimeError("Failed to get app info %s (%s)" % (url, e)), None, sys.exc_info()[2]
def get_marathon_app(url):
try:
return util.jsonRequest(url)['app']
except urllib2.HTTPError as e:
logging.debug(str(e))
if e.code == 404:
return {}
else:
raise RuntimeError(
"Failed to get app info %s HTTP %d (%s) - Response: %s" %
(url, e.code, e, e.read())), None, sys.exc_info()[2]
except urllib2.URLError as e:
logging.debug(str(e))
raise RuntimeError("Failed to get app info %s (%s)" %
(url, e)), None, sys.exc_info()[2]
def _fetch(self, version, uniqueVersion=None, metadata={}):
url = '{0}/{1}/{2}/{3}/{2}-{4}'.format(self._url, self._groupid.replace('.', '/'), self._artifactid, version, uniqueVersion or version)
if self._classifier is not None:
url += '-' + self._classifier
url += '.json'
# Extract unique version number from metadata
if not uniqueVersion:
timestamp = util.rget(metadata,'versioning','snapshot','timestamp') or util.rget(metadata,'versioning','lastUpdated')
buildNumber = util.rget(metadata,'versioning','snapshot','buildNumber')
if timestamp or buildNumber:
uniqueVersion = '%s-%s-%s' % (version.replace('-SNAPSHOT',''), timestamp, buildNumber)
try:
return Artifact(version, uniqueVersion, self._classifier, util.jsonRequest(url))
except urllib2.HTTPError, e:
raise RuntimeError("Failed to retrieve %s HTTP %d (%s)" % (url, e.code, e)), None, sys.exc_info()[2]
def get_marathon_apps(url, labelFilter):
appsurl = url.rstrip('/') + '/v2/apps?' + urllib.urlencode(
{'label': labelFilter})
try:
return util.jsonRequest(appsurl)['apps']
except urllib2.HTTPError as e:
logging.debug(str(e))
if e.code == 404:
return {}
else:
raise RuntimeError(
"Failed to fetch apps %s HTTP %d (%s) - Response: %s" %
(url, e.code, e, e.read())), None, sys.exc_info()[2]
except urllib2.URLError as e:
logging.debug(str(e))
raise RuntimeError("Failed to fetch apps %s (%s)" %
(url, e)), None, sys.exc_info()[2]
def get_marathon_app(url, method='GET'):
try:
response = util.jsonRequest(url, method=method)
if method == 'GET':
return response['app']
return response
except urllib2.HTTPError as e:
logging.debug(str(e))
if e.code == 404:
return {}
else:
raise RuntimeError(
"Failed to %s app %s HTTP %d (%s) - Response: %s" %
(method, url, e.code, e, e.read())), None, sys.exc_info()[2]
except urllib2.URLError as e:
logging.debug(str(e))
raise RuntimeError("Failed to %s app %s (%s)" %
(method, url, e)), None, sys.exc_info()[2]
def _tryRegistryV2Response(self, url):
"""
Return an image first compatibility history object using the Docker Registry V2 API
"""
try:
expandedurl = self._expandurl(url)
response = util.jsonRequest(expandedurl, timeout=15)
except urllib2.HTTPError as e:
if e.code != 404:
obfuscatedurl = self._expandurl(url, obfuscateauth=True)
raise RuntimeError("Failed to call %s (%s)" %
(obfuscatedurl, e)), None, sys.exc_info()[2]
return None
except urllib2.URLError as e:
return None
# Extract the first compatibility image id if present
return util.rget(response, 'history', 0, 'v1Compatibility')
def _fetch(self, version, uniqueVersion=None, metadata={}):
url = '{0}/{1}/{2}/{3}/{2}-{4}'.format(self._url, self._groupid.replace('.', '/'), self._artifactid, version, uniqueVersion or version)
if self._classifier is not None:
url += '-' + self._classifier
url += '.json'
# Extract unique version number from metadata
if not uniqueVersion:
timestamp = util.rget(metadata, 'versioning', 'snapshot', 'timestamp') or util.rget(metadata, 'versioning', 'lastUpdated')
buildNumber = util.rget(metadata, 'versioning', 'snapshot', 'buildNumber')
if timestamp or buildNumber:
uniqueVersion = '-'.join(filter(bool, [version.replace('-SNAPSHOT', ''), timestamp, buildNumber]))
try:
return Artifact(version, uniqueVersion, self._classifier, util.jsonRequest(url))
except urllib2.HTTPError as e:
raise RuntimeError("Failed to retrieve %s HTTP %d (%s)" % (url, e.code, e)), None, sys.exc_info()[2]
except urllib2.URLError as e:
raise RuntimeError("Failed to retrieve %s (%s)" % (url, e)), None, sys.exc_info()[2]
def _tryRegistryV1(self, url):
"""
Resolves an image id using the Docker Registry V1 API
"""
try:
expandedurl = self._expandurl(url, defaultrepo=True)
response = util.jsonRequest(expandedurl, timeout=15)
except urllib2.HTTPError as e:
if e.code != 404:
obfuscatedurl = self._expandurl(url,
defaultrepo=True,
obfuscateauth=True)
raise RuntimeError("Failed to call %s (%s)" %
(obfuscatedurl, e)), None, sys.exc_info()[2]
return None
except urllib2.URLError as e:
return None
# Docker Registry v1 returns image id as a string
if isinstance(response, (str, unicode)):
return unicode(response)
return None
def _parseErrorPost(self, url, *args, **kwargs):
if url.startswith("file:"):
return jsonRequest(url, *args, **kwargs)
raise self.fail("Should not POST into Marathon")
def _parseErrorPost(self, url, *args, **kwargs):
if url.startswith('file:'):
return jsonRequest(url, *args, **kwargs)
raise self.fail('Should not POST into Marathon')
def get_marathon_app(url):
try:
return util.jsonRequest(url)['app']
except urllib2.URLError, e:
logging.debug(str(e))
return {}