def testMergeListNonexisting(self): x = {'a': {'b': [1, 2]}} y = {'a': {'b': {1: 3, 4: 4}}} try: util.merge(x, y) self.fail("Expected exception ValueError") except IndexError, e: self.assertEquals("The given list override index a.b[4] doesn't exist", e.message)
def testMerge(self): x = {'a': 1, 'b': 2} y = {'b': 3, 'c': 4} z = {'c': 5, 'd': 6} m = {'a': 1, 'b': 3, 'c': 5, 'd': 6} self.assertEqual(util.merge(x, y, z),m) m = {'a': 1, 'b': 2, 'c': 4, 'd': 6} self.assertEqual(util.merge(z, y, x),m)
def testMerge(self): x = {'a': 1, 'b': 2} y = {'b': 3, 'c': 4} z = {'c': 5, 'd': 6} m = {'a': 1, 'b': 3, 'c': 5, 'd': 6} self.assertEqual(m, util.merge(util.merge(x, y), z)) m = {'a': 1, 'b': 2, 'c': 4, 'd': 6} self.assertEqual(m, util.merge(util.merge(z, y), x))
def merge_with_service(override_file, document): if not os.path.exists(override_file): raise RuntimeError('Could not read file %s' % override_file) with open(override_file, 'r') as fd2: document = util.merge(yaml.load(fd2), document) return document
def notify(self, payload): logging.debug("Sending Slack message: %s", json.dumps(payload)) headers = { 'Content-type': 'application/json; charset=UTF-8', 'Authorization': 'Bearer ' + str(self._token or '') } for channel in self._channels: payload_object = { "channel": channel, "attachments": [util.merge(payload, self._message_attribs)] } self._call('/chat.postMessage', payload_object, headers)
def parse_service(filename): with open(filename, 'r') as fd: document = yaml.load(fd) # Merge globals.yml files into document path = os.path.dirname(os.path.abspath(filename)) while '/' in path: candidate = os.path.join(path, 'globals.yml') if os.path.exists(candidate): with open(candidate, 'r') as fd2: document = util.merge(yaml.load(fd2), document) path = path[0:path.rindex('/')] # Start from a service section if it exists config = document.get('service', {}) # Fetch and merge json template from maven if util.rget(document,'maven','version') or util.rget(document,'maven','resolve'): coord = document['maven'] resolver = maven.ArtifactResolver(coord['repository'], coord['groupid'], coord['artifactid'], coord.get('classifier')) version = coord.get('version') or resolver.resolve(coord['resolve']) artifact = resolver.fetch(version) document['variables'] = util.merge( document.get('variables', {}), {'lighter.version': artifact.version, 'lighter.uniqueVersion': artifact.uniqueVersion}) config = util.merge(config, artifact.body) # Merge overrides into json template config = util.merge(config, document.get('override', {})) # Substitute variables into the config config = util.replace(config, document.get('variables', {})) return Service(filename, document, config)
def notify(self, message): logging.debug("Sending HipChat message: %s", message) for room in self._rooms: self._call('/v2/room/%s/notification' % room, util.merge({'message': message}, self._message_attribs))
def notify(self, message): logging.debug("Sending HipChat message: %s", message) for room in self._rooms: self._call("/v2/room/%s/notification" % room, util.merge({"message": message}, self._message_attribs))
def testMergeListOverrideDeep(self): x = {'a': [1, {'a': 2, 'b': 3}]} y = {'a': {1: {'a': 4}}} m = {'a': [1, {'a': 4, 'b': 3}]} self.assertEquals(m, util.merge(x, y))
def testMergeLists(self): x = {'a': [1, 2]} y = {'a': [2, 3]} m = {'a': [1, 2, 2, 3]} self.assertEquals(util.merge(x, y), m)
def notify(targetMarathonUrl, service): parsedMarathonUrl = urlparse(targetMarathonUrl) tags = ["environment:%s" % service.environment, "service:%s" % service.id] title = "Deployed %s to the %s environment" % (service.id, service.environment) # Send HipChat notification notify_message = "Deployed <b>%s</b> with image <b>%s</b> to <b>%s</b> (%s)" % ( service.id, service.image, service.environment, parsedMarathonUrl.netloc) if service.releaseNotes: notify_message += service.releaseNotes hipchat = HipChat(util.rget(service.document, 'hipchat', 'token'), util.rget(service.document, 'hipchat', 'url'), util.rget(service.document, 'hipchat', 'rooms')) hipchat.notify(notify_message) # Send Slack notification notify_payload = { "title_link": targetMarathonUrl, "fields": [{ "title": "Service", "value": service.id, "short": 'true' }, { "title": "Environment", "value": service.environment, "short": 'true' }, { "title": "Image", "value": service.image, "short": 'true' }], "ts": int(time.time()) } if service.releaseNotes: notify_payload = util.merge({'text': service.releaseNotes}, notify_payload) slack = Slack(util.rget(service.document, 'slack', 'token'), util.rget(service.document, 'slack', 'url'), util.rget(service.document, 'slack', 'channels')) slack.notify(notify_payload) # Send NewRelic deployment notification newrelic = NewRelic(util.rget(service.document, 'newrelic', 'token')) newrelic.notify(util.rget(service.config, 'env', 'NEW_RELIC_APP_NAME'), service.uniqueVersion) # Send Datadog deployment notification datadog = Datadog( util.rget(service.document, 'datadog', 'token'), util.toList(util.rget(service.document, 'datadog', 'tags'))) datadog.notify( aggregation_key="%s_%s" % (service.environment, service.id), title=title, message= "%%%%%% \n Lighter deployed **%s** with image **%s** to **%s** (%s) \n %%%%%%" % (service.id, service.image, service.environment, parsedMarathonUrl.netloc), tags=tags) # Send Graphite deployment notification prefix = (util.rget(service.document, 'graphite', 'prefix') or 'lighter').strip('.') metricname = '%s.%s.%s.deployments' % ( prefix, service.environment, '.'.join( filter(bool, service.id.split('/')))) graphite = Graphite( util.rget(service.document, 'graphite', 'address'), util.rget(service.document, 'graphite', 'url'), util.toList(util.rget(service.document, 'graphite', 'tags'))) graphite.notify(metricname=metricname, title=title, message="Lighter deployed %s with image %s to %s (%s)" % (service.id, service.image, service.environment, parsedMarathonUrl.netloc), tags=tags)
def parse_service(filename, targetdir=None, verifySecrets=False, profiles=[]): logging.info("Processing %s", filename) # Start from a service section if it exists with open(filename, 'r') as fd: try: document = yaml.load(fd) except yaml.YAMLError as e: raise RuntimeError("Error parsing file %s: %s" % (filename, e)) # Merge globals.yml files into document path = os.path.dirname(os.path.abspath(filename)) while '/' in path: candidate = os.path.join(path, 'globals.yml') if os.path.exists(candidate): document = merge_with_service(candidate, document) path = path[0:path.rindex('/')] # Merge profile .yml files into document document = merge_with_profiles(document, profiles) variables = util.FixedVariables(document.get('variables', {})) # Environment variables has higher precedence variables = util.EnvironmentVariables(variables) # Replace variables in entire document document = util.replace(document, variables, raiseError=False, escapeVar=False) config = document.get('service', {}) # Allow resolving version/uniqueVersion variables from docker registry variables = docker.ImageVariables.create( variables, document, util.rget(config, 'container', 'docker', 'image')) # Fetch and merge json template from maven if util.rget(document, 'maven', 'version') or util.rget(document, 'maven', 'resolve'): coord = document['maven'] versionspec = coord.get('version') if not versionspec: versionspec = coord['resolve'] logging.warn("The 'resolve:' tag is deprecated, please switch to 'version:' which is a drop-in replacement in %s" % filename) resolver = maven.ArtifactResolver(coord['repository'], coord['groupid'], coord['artifactid'], coord.get('classifier')) version = resolver.resolve(versionspec) artifact = resolver.fetch(version) config = util.merge(config, artifact.body) variables = maven.ArtifactVariables(variables, artifact) # Merge overrides into json template config = util.merge(config, document.get('override', {})) # Substitute variables into the config try: config = util.replace(config, variables) except KeyError as e: raise RuntimeError('Failed to parse %s with the following message: %s' % (filename, str(e.message))) if 'env' in config: config['env'] = process_env(filename, verifySecrets, config['env']) # Generate deploy keys and encrypt secrets config = secretary.apply(document, config) checksum = util.checksum(config) # Include hash of config to detect if an element has been removed config['labels'] = config.get('labels', {}) config['labels']['com.meltwater.lighter.checksum'] = checksum # Include a docker label to sort on if util.rget(config, 'container', 'docker'): config['container']['docker']['parameters'] = config['container']['docker'].get('parameters', []) config['container']['docker']['parameters'].append({'key': 'label', 'value': 'com.meltwater.lighter.appid='+config['id']}) # Write json file to disk for logging purposes if targetdir: outputfile = os.path.join(targetdir, filename + '.json') # Exception if directory exists, e.g. because another thread created it concurrently try: os.makedirs(os.path.dirname(outputfile)) except OSError as e: pass with open(outputfile, 'w') as fd: fd.write(util.toJson(config, indent=4)) return Service(filename, document, config)
def parse_service(filename, targetdir=None, verifySecrets=False, profiles=[]): logging.info("Processing %s", filename) # Start from a service section if it exists with open(filename, 'r') as fd: try: document = yaml.load(fd) except yaml.YAMLError as e: raise RuntimeError("Error parsing file %s: %s" % (filename, e)) # Merge globals.yml files into document path = os.path.dirname(os.path.abspath(filename)) while '/' in path: candidate = os.path.join(path, 'globals.yml') if os.path.exists(candidate): document = merge_with_service(candidate, document) path = path[0:path.rindex('/')] # Merge profile .yml files into document document = merge_with_profiles(document, profiles) variables = util.FixedVariables(document.get('variables', {})) # Environment variables has higher precedence variables = util.EnvironmentVariables(variables) # Replace variables in entire document document = util.replace(document, variables, raiseError=False, escapeVar=False) config = document.get('service', {}) # Allow resolving version/uniqueVersion variables from docker registry variables = docker.ImageVariables.create( variables, document, util.rget(config, 'container', 'docker', 'image')) # Fetch and merge json template from maven if util.rget(document, 'maven', 'version') or util.rget( document, 'maven', 'resolve'): coord = document['maven'] versionspec = coord.get('version') if not versionspec: versionspec = coord['resolve'] logging.warn( "The 'resolve:' tag is deprecated, please switch to 'version:' which is a drop-in replacement in %s" % filename) resolver = maven.ArtifactResolver(coord['repository'], coord['groupid'], coord['artifactid'], coord.get('classifier')) version = resolver.resolve(versionspec) artifact = resolver.fetch(version) config = util.merge(config, artifact.body) variables = maven.ArtifactVariables(variables, artifact) # Merge overrides into json template config = util.merge(config, document.get('override', {})) # Substitute variables into the config try: config = util.replace(config, variables) except KeyError as e: raise RuntimeError( 'Failed to parse %s with the following message: %s' % (filename, str(e.message))) if 'env' in config: config['env'] = process_env(filename, verifySecrets, config['env']) # Generate deploy keys and encrypt secrets config = secretary.apply(document, config) checksum = util.checksum(config) # Include hash of config to detect if an element has been removed config['labels'] = config.get('labels', {}) config['labels']['com.meltwater.lighter.checksum'] = checksum # Include a docker label to sort on if util.rget(config, 'container', 'docker'): config['container']['docker']['parameters'] = config['container'][ 'docker'].get('parameters', []) config['container']['docker']['parameters'].append({ 'key': 'label', 'value': 'com.meltwater.lighter.appid=' + config['id'] }) # Write json file to disk for logging purposes if targetdir: outputfile = os.path.join(targetdir, filename + '.json') # Exception if directory exists, e.g. because another thread created it concurrently try: os.makedirs(os.path.dirname(outputfile)) except OSError as e: pass with open(outputfile, 'w') as fd: fd.write(util.toJson(config, indent=4)) return Service(filename, document, config)
def testMergeListOverride(self): x = {'a': [1, 2]} y = {'a': {1: 3}} m = {'a': [1, 3]} self.assertEquals(m, util.merge(x, y))
def testMergeLists(self): x = {'a': [1, 2]} y = {'a': [2, 3]} m = {'a': [1, 2, 2, 3]} self.assertEquals(m, util.merge(x, y))