class Hooks:
def __init__(self):
self.utils = Utils()
self.whobj = WebHook()
self.statsd_message_list = []
self.config_file = ""
def run_hook(self, hookname, appdata, path, hook_input_metric):
try:
exit_code = 0
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS'
self.whobj.invoke_webhook(appdata, hook_input_metric,
self.config_file)
abs_path = os.path.abspath(path)
if "hooks" in appdata and hookname in appdata["hooks"]:
command = appdata["hooks"][hookname]
with chdir(abs_path):
print("About to run {} hook [{}] at path {}".format(
hookname, command, abs_path))
exit_code = os.system(command)
except (Exception) as e:
print("The following error occurred: %s" % e, file=sys.stderr)
execution_result = 'FAILURE'
raise
finally:
try:
if 'execution_result' not in globals(
) and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'function_execution_start_time' not in globals(
) and 'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
sc = self.utils.getStatsClient()
time_take_milliseonds = (
(datetime.now() -
function_execution_start_time).total_seconds() * 1000)
hook_input_metric = hook_input_metric + ",outcome=" + str(
execution_result)
tup = (hook_input_metric, time_take_milliseonds)
self.statsd_message_list.append(tup)
except (Exception) as e:
print("The following error occurred: %s" % e, file=sys.stderr)
raise
return exit_code
class Hooks:
def __init__(self):
self.utils = Utils()
self.whobj = WebHook()
self.statsd_message_list = []
self.config_file = ""
def run_hook(self, hookname, appdata, path, hook_input_metric):
try:
exit_code = 0
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS'
self.whobj.invoke_webhook(appdata, hook_input_metric, self.config_file)
abs_path = os.path.abspath(path)
if "hooks" in appdata and hookname in appdata["hooks"]:
command = appdata["hooks"][hookname]
with chdir(abs_path):
print("About to run {} hook [{}] at path {}".format(
hookname, command, abs_path))
exit_code = os.system(command)
except (Exception) as e:
printException(e)
execution_result = 'FAILURE'
raise
finally:
try:
if 'execution_result' not in globals() and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
sc = self.utils.getStatsClient()
time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
hook_input_metric = hook_input_metric + ",outcome=" + str(execution_result)
tup = (hook_input_metric, time_take_milliseonds)
self.statsd_message_list.append(tup)
except (Exception) as e:
printException(e)
raise
return exit_code
class RogerDeploy(object):
def __init__(self):
self.rogerGitPullObject = RogerGitPull()
self.rogerPushObject = RogerPush()
self.rogerBuildObject = RogerBuild()
self.dockerUtilsObject = DockerUtils()
self.dockerObject = Docker()
self.utils = Utils()
self.slack = None
self.statsd_message_list = []
self.registry = ""
self.image_name = ""
# To remove a temporary directory created by roger-deploy if this
# script exits
def removeDirTree(self, work_dir, args, temp_dir_created):
exists = os.path.exists(os.path.abspath(work_dir))
if exists and (temp_dir_created is True):
shutil.rmtree(work_dir)
print("Deleted temporary dir:{0}".format(work_dir))
def getNextVersion(self, config, roger_env, application, branch, work_dir, repo, args, gitObj):
sha = getGitSha(work_dir, repo, branch, gitObj)
docker_search = self.dockerUtilsObject.docker_search(roger_env['registry'], config['name'], application)
image_version_list = []
version = ''
envs = []
for each_key in roger_env["environments"].keys():
envs.append(each_key)
for line in docker_search.split('\n'):
image = line.split(' ')[0]
# (vmahedia) todo: This belongs in a separate DockerImage class. This class should know
# how to generate the image name, how to search the image or return this name pattern we
# then use to search this image on the registry.
matchObj = re.match("^{0}-{1}-.*/v.*".format(config['name'], application), image)
if matchObj and matchObj.group().startswith(config['name'] + '-' + application):
skip_image = False
for env in envs:
if matchObj.group().startswith("{0}-{1}-{2}".format(config['name'], application, env)):
skip_image = True
break
if skip_image is False:
if verify(str(matchObj.group().split('v')[-1])):
image_version_list.append(matchObj.group().split('v')[-1])
if len(image_version_list) == 0: # Create initial version
version = "{0}/v0.1.0".format(sha)
print("No version currently exist in the Docker Registry. \nDeploying version:{0}".format(
version))
else:
version = self.incrementVersion(sha, image_version_list, args)
return version
def incrementVersion(self, sha, image_version_list, args):
latest = max(image_version_list, key=self.splitVersion)
ver_tuple = self.splitVersion(latest)
latest_version = ''
if args.incr_major:
latest_version = "{0}/v{1}.0.0".format(sha,
(int(ver_tuple[0]) + 1))
return latest_version
if args.incr_patch:
latest_version = "{0}/v{1}.{2}.{3}".format(
sha, int(ver_tuple[0]), int(ver_tuple[1]), (int(ver_tuple[2]) + 1))
return latest_version
latest_version = "{0}/v{1}.{2}.0".format(
sha, int(ver_tuple[0]), (int(ver_tuple[1]) + 1))
return latest_version
def splitVersion(self, version):
major, _, rest = version.partition('.')
minor, _, rest = rest.partition('.')
patch, _, rest = rest.partition('.')
return int(major), int(minor) if minor else 0, int(patch) if patch else 0
def parseArgs(self):
self.parser = argparse.ArgumentParser(
prog='roger deploy', description=describe())
self.parser.add_argument('-e', '--environment', metavar='env',
help="environment to deploy to. Example: 'dev' or 'stage'")
self.parser.add_argument('-b', '--branch', metavar='branch', default='master',
help="branch to pull code from. Defaults to master. Example: 'production' or 'master'")
self.parser.add_argument('-sg', '--skip-gitpull', action="store_true",
help="skip the gitpull step. Defaults to false.")
self.parser.add_argument('-s', '--skip-build', action="store_true",
help="whether to skip the build step. Defaults to false.")
self.parser.add_argument('-M', '--incr-major', action="store_true",
help="increment major in version. Defaults to false.")
self.parser.add_argument('-sp', '--skip-push', action="store_true",
help="skip the push step. Defaults to false.'")
self.parser.add_argument('-v', '--verbose', help="verbose mode for debugging. Defaults to false.", action="store_true")
self.parser.add_argument('-f', '--force-push', action="store_true",
help="force push. Not recommended. Forces push even if validation checks failed. Applies only if skip_push is false. Defaults to false.")
self.parser.add_argument('-p', '--incr-patch', action="store_true",
help="increment patch in version. Defaults to false.'")
self.parser.add_argument('-S', '--secrets-file',
help="specifies an optional secrets file for deployment runtime variables.")
self.parser.add_argument('-d', '--directory',
help="working directory. Uses a temporary directory if not specified.")
self.parser.add_argument('application', metavar='application', help="application to deploy. \
Can also push specific containers(comma seperated). Example: 'all' \
or 'app1:app2' or 'kairos' or 'app_name[container1,container2]' \
or'app1[container1,container2]:app2[container3,container4]' or 'app1:app2[container]'")
self.parser.add_argument('app_repo', metavar="app_repo",
help="Application's git repository name, repo must be under 'seomoz' organization")
self.parser.add_argument('config_file', metavar='config_file',
help="configuration file to be use. Example: 'content.json' or 'kwe.json'")
return self.parser
def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args):
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS'
settingObj = settingObject
appObj = appObject
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
roger_env = appObj.getRogerEnv(config_dir)
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
if 'registry' not in roger_env:
raise ValueError('Registry not found in roger-mesos-tools.config file.')
else:
self.registry = roger_env['registry']
# Setup for Slack-Client, token, and git user
# (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event
# Event should be registered and SlackNotification should be one of the members. it can have
# N notifications on a particular "event", Notifications.Notify will broadcast notification to
# all the interested parties.
if 'notifications' in config:
self.slack = Slack(config['notifications'],
'/home/vagrant/.roger_cli.conf.d/slack_token')
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application)
apps = []
apps_container_dict = {}
if args.application == 'all':
apps = config['apps'].keys()
else:
if ":" not in args.application and "[" not in args.application:
apps.append(args.application)
else:
for item in args.application.split(":"):
if '[' in item:
matchObj = re.match(r'(.*)\[(.*)\]', item)
apps.append(matchObj.group(1))
apps_container_dict[matchObj.group(1)] = matchObj.group(2)
else:
apps.append(item)
common_repo = config.get('repo', '')
environment = roger_env.get('default_environment', '')
work_dir = ''
if args.directory:
work_dir = args.directory
temp_dir_created = False
if args.verbose:
print("Using {0} as the working directory".format(work_dir))
else:
work_dir = mkdtemp()
temp_dir_created = True
if args.verbose:
print("Created a temporary dir: {0}".format(work_dir))
if args.environment is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file")
else:
print(
"Using value {} from environment variable $ROGER_ENV".format(env_var))
environment = env_var
else:
environment = args.environment
if environment not in roger_env['environments']:
self.removeDirTree(work_dir, args, temp_dir_created)
raise ValueError('Environment not found in roger-mesos-tools.config file.')
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
try:
for app in apps:
if app not in config['apps']:
raise ValueError('Application {} specified not found.'.format(app))
else:
try:
if args.verbose:
print("Deploying {} ...".format(app))
self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj,
root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict)
except (IOError, ValueError) as e:
error_msg = "Error when deploying {}: {}".format(app, repr(e))
printErrorMsg(error_msg)
pass # try deploying the next app
except (Exception) as e:
printException(e)
raise
except (Exception) as e:
execution_result = 'FAILURE'
printException(e)
raise
finally:
# Check if the initializition of variables carried out
if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
if 'execution_result' not in globals() and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals() and 'config_name' not in locals():
config_name = ""
if 'environment' not in globals() and 'environment' not in locals():
environment = "dev"
if not hasattr(args, "application"):
args.application = ""
if 'settingObj' not in globals() and 'settingObj' not in locals():
settingObj = Settings()
if 'work_dir' not in globals() and 'work_dir' not in locals():
work_dir = ''
temp_dir_created = False
if not (self.rogerGitPullObject.outcome is 1 and self.rogerBuildObject.outcome is 1 and self.rogerPushObject.outcome is 1):
execution_result = 'FAILURE'
try:
# If the deploy fails before going through any steps
sc = self.utils.getStatsClient()
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application)
args.application = self.utils.extract_app_name(args.application)
time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.application) + ",event=deploy" + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******",identifier=" + str(self.identifier)
tup = (input_metric, time_take_milliseonds)
self.statsd_message_list.append(tup)
self.removeDirTree(work_dir, args, temp_dir_created)
except (Exception) as e:
error_msg = "Error when deploying {}: {}".format(app, repr(e))
printErrorMsg(error_msg)
raise
def deployApp(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config,
roger_env, work_dir, config_dir, environment, app, branch, slack, config_file, common_repo, temp_dir_created, apps_container_dict):
startTime = datetime.now()
settingObj = settingObject
appObj = appObject
frameworkUtils = frameworkUtilsObject
environmentObj = roger_env['environments'][environment]
data = appObj.getAppData(config_dir, config_file, app)
frameworkObj = frameworkUtils.getFramework(data)
framework = frameworkObj.getName()
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', app)
image_name = ''
image = ''
skip_gitpull = True if args.skip_gitpull else False
# get/update target source(s)
if not skip_gitpull:
args.app_name = app
args.directory = work_dir
self.rogerGitPullObject.statsd_message_list = self.statsd_message_list
self.rogerGitPullObject.identifier = self.identifier
self.rogerGitPullObject.main(settingObj, appObj, gitObj, hooksObj, args)
skip_build = True if args.skip_build else False
skip_push = True if args.skip_push else False
secrets_file = args.secrets_file if args.secrets_file else None
# Set initial version
# todo (vmahedia) #image_name naming should not be magic, make it explicit
# todo (vmahedia) ExtractClass GitInfo, no need to pass args, we already have the information.
# We deal with only one repo at a time. It may change in future but we can change the code then.
image_git_sha = getGitSha(work_dir, repo, branch, gitObj)
image_name = "{0}-{1}-{2}/v0.1.0".format(config['name'], app, image_git_sha)
print(colored("******Fetching current version deployed or latest version from registry.\
This is used to bump to next version.******", "grey"))
if skip_build:
curr_image_ver = frameworkObj.getCurrentImageVersion(
roger_env, environment, app)
self.image_name = curr_image_ver
if args.verbose:
print("Current image version deployed on {0} is :{1}".format(framework, curr_image_ver))
if curr_image_ver is not None:
image_name = "{0}-{1}-{2}".format(
config['name'], app, curr_image_ver)
if args.verbose:
print("Image current version from {0} endpoint is:{1}".format(framework, image_name))
else:
if args.verbose:
print("Using base version for image:{0}".format(image_name))
else:
# Docker build,tag and push
image_name = self.getNextVersion(
config, roger_env, app, branch, work_dir, repo, args, gitObj)
print(colored("******Done finding latest version******", "green"))
image_name = "{0}-{1}-{2}".format(config['name'], app, image_name)
print(colored("Bumped up image to version:{0}".format(image_name), "green"))
self.image_name = image_name
build_args = args
build_args.app_name = app
build_args.directory = os.path.abspath(work_dir)
build_args.tag_name = image_name
build_args.config_file = config_file
build_args.env = environment
build_args.push = True
build_args.verbose = args.verbose
try:
self.rogerBuildObject.identifier = self.identifier
self.rogerBuildObject.statsd_message_list = self.statsd_message_list
self.rogerBuildObject.main(settingObj, appObject, hooksObj,
self.dockerUtilsObject, self.dockerObject, build_args)
except ValueError:
raise
print("Image Version is: {}".format(colored(image_name, "cyan")))
# Deploying the app to framework
args.image_name = image_name
args.config_file = config_file
args.env = environment
if app in apps_container_dict:
args.app_name = str(app) + ":" + apps_container_dict[app]
else:
args.app_name = app
self.rogerPushObject.identifier = self.identifier
self.rogerPushObject.statsd_message_list = self.statsd_message_list
self.rogerPushObject.main(settingObj, appObj, frameworkUtils,
hooksObj, args)
deployTime = datetime.now() - startTime
username = settingObj.getUser()
deployMessage = "{0}'s deploy for {1} / {2} / {3} completed in {4} seconds.".format(
username, app, environment, branch, deployTime.total_seconds())
if slack is not None:
slack.api_call(deployMessage)
print(colored(deployMessage, "green"))
def locateConfigFile(self, args, gitObj):
# Clone the git repo first because config lives there, there's nothing that we can do without this file
# this is not the clean way but the code is very convulted for now to implement this in a clean manner
# For now, we will clone the repo silently and use that config. We have to clone it everytime because we
# have to assume that there's always a change, although it's not true - we can check if there's a change
# and only pull then, but that is another mess. Let's make it simple and pull everytime
branch = args.branch if args.branch else "master"
repo_dir = os.path.join(args.directory,args.app_repo)
if os.path.exists(args.config_file):
# skip the git clone
# this file path could be inside the cloned repo or outside
if args.config_file.startswith(args.directory):
# chdir is a decorator above but we should move it to utils or somewhere else for every file to use
with chdir(repo_dir):
# file is inside cloned repo so Pull to fetch changes
rc = gitObj.gitPull(branch, args.verbose)
if rc:
print(colored("WARNING: Unable to Pull branch - {}, from repo - {}. Config file will"
"not contain latest changes".format(args.branch, args.app_repo)), "yellow")
# File is not inside cloned repo but somewhere else
# just use it, no need to do anything
else:
if args.app_repo:
if os.path.exists(repo_dir):
raise ValueError("Repo directory - {} already exists but config file - {} is not present"\
.format(repo_dir, args.config_file))
# chdir is a decorator defined above
with chdir(args.directory):
if args.verbose:
print(colored("Cloning repo - {} at - {} for config file - {}".\
format(args.app_repo, repo_dir, args.config_file)))
# clone the repo
# file does not exist and we need to clone the repo it is in repo since repo is
# defined and we mandate it to be in repo
exit_code = gitObj.gitShallowClone(args.app_repo, branch, args.verbose)
if exit_code:
raise ValueError("Error cloning repo {} while looking for config file".format(args.app_repo))
# check now if we have config file in the cloned repo, otherwise bailout.
if not os.path.exists(args.config_file):
# we checked out the repo and either the path is not in repo and file does not exist
# the path is in repo but the repo doesn't have the config file
raise ValueError("Config file - {} does not exist".format(args.config_file))
else:
raise ValueError("Config file - {} does not exist, no app repo defined either".format(args.config_file))
class RogerGitPull(object):
def __init__(self):
self.utils = Utils()
self.statsd_message_list = []
self.outcome = 1
def parse_args(self):
self.parser = argparse.ArgumentParser(
prog='roger gitpull', description=describe())
self.parser.add_argument('app_name', metavar='app_name',
help="application for which code is to be pulled. Example: 'agora' or 'grafana'")
self.parser.add_argument('directory', metavar='directory',
help="working directory. The repo has its own directory it this. Example: '/home/vagrant/work_dir'")
self.parser.add_argument('-b', '--branch', metavar='branch',
help="git branch to pull code from. Example: 'production' or 'master'. Defaults to master.")
self.parser.add_argument('config_file', metavar='config_file',
help="configuration file to use. Example: 'content.json' or 'kwe.json'")
return self.parser
def main(self, settings, appConfig, gitObject, hooksObj, args):
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs
environment = "dev"
if hasattr(args, "environment"):
environment = args.environment
settingObj = settings
appObj = appConfig
gitObj = gitObject
config_dir = settingObj.getConfigDir()
hooksObj.config_file = args.config_file
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
common_repo = config.get('repo', '')
data = appObj.getAppData(config_dir, args.config_file, args.app_name)
if not data:
raise ValueError('Application with name [{}] or data for it not found at {}/{}.'.format(
args.app_name, config_dir, args.config_file))
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', args.app_name)
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
if not os.path.exists(args.directory):
try:
os.makedirs(args.directory)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
if not hasattr(args, "app_name"):
args.app_name = ""
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
args.app_name = self.utils.extract_app_name(args.app_name)
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "pre_gitpull"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{0}/{1}".format(args.directory, repo_name)
if os.path.isdir(path):
with chdir(path):
exit_code = gitObj.gitPull(branch)
else:
with chdir('{0}'.format(args.directory)):
exit_code = gitObj.gitShallowClone(repo, branch)
if exit_code != 0:
raise ValueError('gitpull failed.')
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "post_gitpull"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"The following error occurred: %s" %
e, file=sys.stderr)
execution_result = 'FAILURE'
raise
finally:
try:
# If the gitpull fails before going through any steps
if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
if 'execution_result' not in globals() and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals() and 'config_name' not in locals():
config_name = ""
if 'environment' not in globals() and 'environment' not in locals():
environment = "dev"
if not hasattr(args, "app_name"):
args.app_name = ""
if 'settingObj' not in globals() and 'settingObj' not in locals():
settingObj = Settings()
if 'execution_result' is 'FAILURE':
self.outcome = 0
sc = self.utils.getStatsClient()
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=gitpull" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"The following error occurred: %s" %
e, file=sys.stderr)
raise
class RogerGitPull(object):
def __init__(self):
self.utils = Utils()
self.statsd_message_list = []
self.outcome = 1
def parse_args(self):
self.parser = argparse.ArgumentParser(
prog='roger gitpull', description=describe())
self.parser.add_argument('app_name', metavar='app_name',
help="application for which code is to be pulled. Example: 'agora' or 'grafana'")
self.parser.add_argument('directory', metavar='directory',
help="working directory. The repo has its own directory it this. Example: '/home/vagrant/work_dir'")
self.parser.add_argument('-v', '--verbose', help="verbose mode for debugging", action="store_true")
self.parser.add_argument('-b', '--branch', metavar='branch',
help="git branch to pull code from. Example: 'production' or 'master'. Defaults to master.")
self.parser.add_argument('config_file', metavar='config_file',
help="configuration file to use. Example: 'content.json' or 'kwe.json'")
return self.parser
def main(self, settings, appConfig, gitObject, hooksObj, args):
print(colored("******Executing GIT PULL of application repo******", "grey"))
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs
environment = "dev"
if hasattr(args, "environment"):
environment = args.environment
settingObj = settings
appObj = appConfig
gitObj = gitObject
config_dir = settingObj.getConfigDir()
hooksObj.config_file = args.config_file
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
common_repo = config.get('repo', '')
data = appObj.getAppData(config_dir, args.config_file, args.app_name)
if not data:
raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format(
args.app_name, config_dir, args.config_file))
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', args.app_name)
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
if not os.path.exists(args.directory):
try:
os.makedirs(args.directory)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
if not hasattr(args, "app_name"):
args.app_name = ""
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
args.app_name = self.utils.extract_app_name(args.app_name)
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "pre_gitpull"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname))
# get/update target source(s)
repo_name = appObj.getRepoName(repo)
path = "{0}/{1}".format(args.directory, repo_name)
if os.path.isdir(path):
with chdir(path):
exit_code = gitObj.gitPull(branch, args.verbose)
else:
with chdir('{0}'.format(args.directory)):
exit_code = gitObj.gitShallowClone(repo, branch, args.verbose)
if exit_code != 0:
raise ValueError("Gitpull failed.")
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "post_gitpull"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname))
except (Exception) as e:
printException(e)
execution_result = 'FAILURE'
raise
finally:
try:
# If the gitpull fails before going through any steps
if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
if 'execution_result' not in globals() and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals() and 'config_name' not in locals():
config_name = ""
if 'environment' not in globals() and 'environment' not in locals():
environment = "dev"
if not hasattr(args, "app_name"):
args.app_name = ""
if 'settingObj' not in globals() and 'settingObj' not in locals():
settingObj = Settings()
if 'execution_result' is 'FAILURE':
self.outcome = 0
sc = self.utils.getStatsClient()
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=gitpull" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"ERROR - %s" %
e, file=sys.stderr)
raise
print(colored("******Completed the GIT PULL step successfully******", "green"))
class RogerPush(object):
def __init__(self):
self.utils = Utils()
self.task_id = []
self.statsd_message_list = []
self.statsd_push_list = []
self.outcome = 1
self.registry = ""
self.image_name = ""
def parse_args(self):
self.parser = argparse.ArgumentParser(
prog='roger push', description=describe())
self.parser.add_argument('app_name', metavar='app_name', help="application to push. Can also push specific"
" containers(comma seperated). Example: 'agora' or 'app_name:container1,container2'")
self.parser.add_argument('-e', '--env', metavar='env',
help="environment to push to. Example: 'dev' or 'prod'")
self.parser.add_argument('-v', '--verbose', help="verbose mode for debugging", action="store_true")
self.parser.add_argument('directory', metavar='directory',
help="working directory. Example: '/home/vagrant/work_dir'")
self.parser.add_argument('image_name', metavar='image_name',
help="image name that includes version to use. Example: 'roger-collectd-v0.20' or 'elasticsearch-v0.07'")
self.parser.add_argument('config_file', metavar='config_file',
help="configuration file to use. Example: 'content.json' or 'kwe.json'")
self.parser.add_argument(
'--skip-push', '-s', help="skips push. Only generates components for review. Defaults to false.", action="store_true")
self.parser.add_argument(
'--force-push', '-f', help="force push. Not Recommended. Forces push even if validation checks failed. Defaults to false.", action="store_true")
self.parser.add_argument('--secrets-file', '-S',
help="specifies an optional secrets file for deploy runtime variables.")
return self.parser
# (vmahedia) todo: https://seomoz.atlassian.net/browse/ROGER-2396
# this has a lot of redundant messages and logic of assuming the
# secret file location is annoying, make it obvious and simple
def loadSecrets(self, secrets_dir, file_name, args, environment):
if args.secrets_file is not None:
print("Using specified secrets file: {}".format(args.secrets_file))
file_name = args.secrets_file
exists = os.path.exists(secrets_dir)
if exists is False:
os.makedirs(secrets_dir)
# (vmahedia) WE SHOULD NOT DO ANY GUESSING GAME BE EXPLICIT
# about where we expect what and argument should make that very clear to customers
# Two possible paths -- first without environment, second with
path1 = "{}/{}".format(secrets_dir, file_name)
path2 = "{}/{}/{}".format(secrets_dir, environment, file_name)
if args.verbose:
print(colored("Trying to load secrets from file {} or {}".format(path1, path2), "cyan"))
try:
with open(path1) as f:
return_file = yaml.load(f) if path1.lower().endswith('.yml') else json.load(f)
return return_file
except IOError:
pass
except ValueError as e:
raise ValueError("Error while loading json from {} - {}".format(path1, e))
try:
with open(path2) as f:
return_file = yaml.load(f) if path2.lower().endswith('.yml') else json.load(f)
return return_file
except IOError:
if args.verbose:
print("WARNING - Couldn't load any secrets file. Searched {} and {}. \nIGNORE this above warning if you do not have secrets or your secrets file is passed in using the optional argument and does not reside in the above 2 looked up paths.".format(path1, path2))
return {}
except ValueError as e:
raise ValueError("Error while loading json from {} - {}".format(path2, e))
def replaceSecrets(self, output_dict, secrets_dict):
if type(output_dict) is not dict:
return output_dict
for key in output_dict:
if output_dict[key] == "SECRET":
if key in secrets_dict.keys():
output_dict[key] = secrets_dict[key]
if type(output_dict[key]) is list:
temp_list = []
for list_elem in output_dict[key]:
temp_list.append(self.replaceSecrets(
list_elem, secrets_dict))
output_dict[key] = temp_list
if type(output_dict[key]) is dict:
temp_dict = self.replaceSecrets(output_dict[key], secrets_dict)
output_dict[key] = temp_dict
return output_dict
def mergeSecrets(self, json_str, secrets):
'''Given a JSON string and an object of secret environment variables, replaces
parses the JSON keys with the secret variables. Returns back
a JSON string. Raises an error if there are any SECRET variables still exists.'''
output_dict = json.loads(json_str)
json_str = json.dumps(self.replaceSecrets(
output_dict, secrets), indent=4)
if '\"SECRET\"' in json_str:
print(colored("ERROR - Found the \"SECRET\" keyword in the template file -- does your secrets file have all secret environment variables?", "red"))
print(colored("ERROR - The use of \"SECRET\" is deprecated. Please switch to using Jinja variables. To do so,"
" use '{{ <actual variable name> }}' instead of \"SECRET\" in the template file.", "red"))
return "StandardError"
return json_str
def renderTemplate(self, template, environment, image, app_data, config, container, container_name, additional_vars):
variables = {'environment': environment, 'image': image}
# Copy variables from config-wide, app-wide, then container-wide variable
# configs, each one from "global" and then environment-specific.
for obj in [config, app_data, container]:
if type(obj) == dict and 'vars' in obj:
variables.update(obj['vars'].get('global', {}))
variables.update(obj['vars'].get('environment', {}).get(environment, {}))
variables.update(additional_vars)
return template.render(variables)
def statsd_counter_logging(self, metric):
sc = self.utils.getStatsClient()
sc.incr(metric, 1)
def repo_relative_path(self, appConfig, args, repo, path):
'''Returns a path relative to the repo, assumed to be under [args.directory]/[repo name]'''
repo_name = appConfig.getRepoName(repo)
abs_path = os.path.abspath(args.directory)
return os.path.join(args.directory, repo_name, path)
def getContainerName(self, container):
return str(container.keys()[0]) if type(container) == dict else container
def main(self, settings, appConfig, frameworkObject, hooksObj, args):
print(colored("******Deploying application to framework******", "grey"))
try:
validation_failed = False
settingObj = settings
appObj = appConfig
frameworkUtils = frameworkObject
config_dir = settingObj.getConfigDir()
hooksObj.config_file = args.config_file
cur_file_path = os.path.dirname(os.path.realpath(__file__))
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
act_as_user = ""
if 'name' in config:
config_name = config['name']
if 'act-as' in config:
act_as_user = config['act-as']
roger_env = appObj.getRogerEnv(config_dir)
if not hasattr(args, "app_name"):
args.app_name = ""
if 'registry' not in roger_env.keys():
raise ValueError("Registry not found in roger-mesos-tools.config file.")
else:
self.registry = roger_env['registry']
if hasattr(args, "image_name"):
self.image_name = args.image_name
environment = roger_env.get('default_environment', '')
if args.env is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(colored("WARNING - Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file", "yellow"))
else:
if args.verbose:
print(colored("Using value {} from environment variable $ROGER_ENV".format(env_var), "grey"))
environment = env_var
else:
environment = args.env
# ----------------------------------------------
if environment not in roger_env['environments']:
raise ValueError("Environment not found in roger-mesos-tools.config file.")
# ----------------------------------------------
# GetEnvironmentConfig(environment)
# ----------------------------------------------
environmentObj = roger_env['environments'][environment]
common_repo = config.get('repo', '')
# ----------------------------------------------
# GetContainersForApp(app)
# ----------------------------------------------
app_name = args.app_name
container_list = []
# todo (vmahedia): What does ':' signify? Put explanation.
if ':' in app_name:
tokens = app_name.split(':')
app_name = tokens[0]
# todo (vmahedia): it's container list - need to explain syntax
if ',' in tokens[1]:
container_list = tokens[1].split(',')
else:
container_list.append(tokens[1])
# ----------------------------------------------
data = appObj.getAppData(config_dir, args.config_file, app_name)
if not data:
raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format(
app_name, config_dir, args.config_file))
configured_container_list = []
for task in data['containers']:
if type(task) == dict:
configured_container_list.append(task.keys()[0])
else:
configured_container_list.append(task)
if not set(container_list) <= set(configured_container_list):
raise ValueError("List of containers [{}] passed do not match list of acceptable containers: [{}]".format(
container_list, configured_container_list))
frameworkObj = frameworkUtils.getFramework(data)
framework = frameworkObj.getName()
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', app_name)
comp_dir = settingObj.getComponentsDir()
templ_dir = settingObj.getTemplatesDir()
secrets_dir = settingObj.getSecretsDir()
# Create comp_dir if it doesn't exist
if not os.path.isdir(comp_dir):
os.makedirs(comp_dir)
if not container_list:
data_containers = data['containers']
else:
data_containers = container_list
failed_container_dict = {}
# (vmahedia) upto this point it's all getting and checking the
# configuration parameters
template = ''
# Required for when work_dir,component_dir,template_dir or
# secret_env_dir is something like '.' or './temp"
os.chdir(cur_file_path)
app_path = ''
if 'template_path' in data:
app_path = self.repo_relative_path(appObj, args, repo, data['template_path'])
else:
app_path = templ_dir
extra_vars = {}
if 'extra_variables_path' in data:
ev_path = self.repo_relative_path(appObj, args, repo, data['extra_variables_path'])
with open(ev_path) as f:
extra_vars = yaml.load(f) if ev_path.lower(
).endswith('.yml') else json.load(f)
if not app_path.endswith('/'):
app_path = app_path + '/'
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
args.app_name = self.utils.extract_app_name(args.app_name)
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "pre_push"
hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname))
# ----------------------------------------------
# (vmahedia) Figure out what the hell this loop does
# and name it appropriately
# it seems first part is just finding a template and Rendering
# it against the given config, checking to see if there are errors
# ----------------------------------------------
# (vmahedia) Meat starts from here, probably.
for container in data_containers:
container_name = self.getContainerName(container)
containerConfig = "{0}-{1}.json".format(config['name'], container_name)
env = Environment(loader = FileSystemLoader("{}".format(app_path)), undefined = StrictUndefined)
template_with_path = "[{}{}]".format(app_path, containerConfig)
try:
template = env.get_template(containerConfig)
except exceptions.TemplateNotFound as e:
raise ValueError("The template file {} does not exist".format(template_with_path))
except Exception as e:
raise ValueError("Error while reading template from {} - {}".format(template_with_path, e))
additional_vars = {}
# (vmahedia)variables likes this should be at least visible within one
# scroll up or down, move this code to near to context
# Why are we getting the secrets everytime, this requires the file to be
# present
additional_vars.update(extra_vars)
secret_vars = self.loadSecrets(secrets_dir, containerConfig, args, environment)
additional_vars.update(secret_vars)
image_path = "{0}/{1}".format(
roger_env['registry'], args.image_name)
print("Rendering content from template {} for environment [{}]".format(
template_with_path, environment))
try:
output = self.renderTemplate(template, environment, image_path, data, config, container, container_name, additional_vars)
except exceptions.UndefinedError as e:
error_str = "The following Undefined Jinja variable error occurred. %s.\n" % e
print(colored(error_str, "red"), file=sys.stderr)
failed_container_dict[container_name] = error_str
# we are going to fail even if one of the container config is not valid but we will
# still go through the loop and collect all the errors before we bail out
validation_failed = True
pass
# ----------------------------------------------
# it seems the checks above can finish independent of the
# following code, decouple this two parts, later when the code
# is well understood
# ----------------------------------------------
# Adding check to see if all jinja variables git resolved fot
# the container
if container_name not in failed_container_dict:
# Adding check so that not all apps try to mergeSecrets
try:
outputObj = json.loads(output)
except Exception as e:
raise ValueError("Error while loading json from {} - {}".format(template_with_path, e))
if '\"SECRET\"' in output:
output = self.mergeSecrets(output, self.loadSecrets(
secrets_dir, containerConfig, args, environment))
if output != "StandardError":
try:
comp_exists = os.path.exists("{0}".format(comp_dir))
if comp_exists is False:
os.makedirs("{0}".format(comp_dir))
comp_env_exists = os.path.exists(
"{0}/{1}".format(comp_dir, environment))
if comp_env_exists is False:
os.makedirs(
"{0}/{1}".format(comp_dir, environment))
except Exception as e:
logging.error(traceback.format_exc())
# (vmahedia) Should we write out the files even though there is an error with one of the
# containers. Although maybe users would want to see some output
with open("{0}/{1}/{2}".format(comp_dir, environment, containerConfig), 'wb') as fh:
fh.write(output)
else:
raise ValueError("Error while loading secrets to render template file variables")
# Notify container error messages
# let failed_container_dict just be for now, but report all the errors
if validation_failed:
raise Exception("Unable to render Jinja template")
deployment_check_failed = False
# fail if the deployment check fails
for container in data_containers:
container_name = self.getContainerName(container)
containerConfig = "{0}-{1}.json".format(config['name'], container_name)
config_file_path = "{0}/{1}/{2}".format(comp_dir, environment, containerConfig)
result = frameworkObj.runDeploymentChecks(config_file_path, environment)
if not result:
# need to give more indication about what can they do to fix this and what exactly failed
# in the deployment check function, we should print an error in that function as well
print(colored("Deployment checks failed for container - {}".format(framework, container)), "red")
deployment_check_failed = True
if deployment_check_failed:
raise Exception("Deployment Check failed for one or more containers, check logs for more info!")
if args.skip_push:
print(colored("Skipping push to {} framework. The rendered config file(s) are under {}/{}/".format(
framework, colored(comp_dir, "cyan"), colored(environment, "cyan")), "yellow"))
else:
# push to roger framework
if 'owner' in config:
frameworkObj.act_as_user = config['owner']
tools_version_value = self.utils.get_version()
image_name = self.registry + "/" + args.image_name
image_tag_value = urllib.quote("'" + image_name + "'")
for container in data_containers:
try:
function_execution_start_time = datetime.now()
# Assume SUCCESS unless exception
execution_result = 'SUCCESS'
sc = self.utils.getStatsClient()
except (Exception) as e:
raise ValueError("{} Error : {}".format(getDebugInfo(), e))
try:
# this is where actual push is happening
# we only push if forced, in case of failures
# in deployment checks
#
# (vmahedia) todo:
# list down scenarios in which this features
# will be useful
resp, task_id = frameworkObj.put(config_file_path, environmentObj,
container_name, environment, act_as_user)
container_task_id = self.utils.modify_task_id(task_id)
self.task_id.extend(container_task_id)
if hasattr(resp, "status_code"):
status_code = resp.status_code
except (Exception) as e:
print("ERROR - : %s" %
e, file=sys.stderr)
execution_result = 'FAILURE'
raise
finally:
try:
if 'function_execution_start_time' not in globals() and \
'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
if 'execution_result' not in globals() and \
'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals() and \
'config_name' not in locals():
config_name = ""
if 'environment' not in globals() and \
'environment' not in locals():
environment = "dev"
if 'container_name' not in globals() and \
'container_name' not in locals():
container_name = ""
if 'status_code' not in globals() and \
'status_code' not in locals():
status_code = "500"
if not hasattr(args, "app_name"):
args.app_name = ""
if 'settingObj' not in globals() and \
'settingObj' not in locals():
settingObj = Settings()
if 'container_task_id' not in globals() and 'container_task_id' not in locals():
container_task_id = []
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(),
args.app_name)
if not str(status_code).startswith("20"):
execution_result = 'FAILURE'
self.outcome = 0
time_taken = (datetime.now() - function_execution_start_time).total_seconds()
for task_id in container_task_id:
input_metric = "roger-tools.rogeros_tools_exec_time" + \
",app_name=" + str(args.app_name) + \
",event=push" + \
",container_name=" + str(container_name) + \
",identifier=" + str(self.identifier) + \
",outcome=" + str(execution_result) + \
",response_code=" + str(status_code) + \
",config_name=" + str(config_name) + \
",env=" + str(environment) + \
",user="******"20"):
metric = input_metric.replace("rogeros_tools_exec_time", "rogeros_events")
metric = metric + ",source=tools" + ",task_id=" + task_id
self.statsd_counter_logging(metric)
except (Exception) as e:
printException(e)
raise
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "post_push"
hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + \
",app_name=" + str(args.app_name) + \
",identifier=" + str(self.identifier) + \
",config_name=" + str(config_name) + \
",env=" + str(environment) + \
",user="******"{} hook failed.".format(hookname))
print(colored("******Done with the PUSH step******", "green"))
except (Exception) as e:
raise ValueError("ERROR - {}".format(e))
class RogerDeploy(object):
def __init__(self):
self.rogerGitPullObject = RogerGitPull()
self.rogerPushObject = RogerPush()
self.rogerBuildObject = RogerBuild()
self.dockerUtilsObject = DockerUtils()
self.dockerObject = Docker()
self.utils = Utils()
self.slack = None
self.statsd_message_list = []
self.registry = ""
self.image_name = ""
# To remove a temporary directory created by roger-deploy if this
# script exits
def removeDirTree(self, work_dir, args, temp_dir_created):
exists = os.path.exists(os.path.abspath(work_dir))
if exists and (temp_dir_created is True):
shutil.rmtree(work_dir)
print("Deleted temporary dir:{0}".format(work_dir))
def getNextVersion(self, config, roger_env, application, branch, work_dir,
repo, args, gitObj):
sha = getGitSha(work_dir, repo, branch, gitObj)
docker_search = self.dockerUtilsObject.docker_search(
roger_env['registry'], config['name'], application)
image_version_list = []
version = ''
envs = []
for each_key in roger_env["environments"].keys():
envs.append(each_key)
for line in docker_search.split('\n'):
image = line.split(' ')[0]
matchObj = re.match(
"^{0}-{1}-.*/v.*".format(config['name'], application), image)
if matchObj and matchObj.group().startswith(config['name'] + '-' +
application):
skip_image = False
for env in envs:
if matchObj.group().startswith("{0}-{1}-{2}".format(
config['name'], application, env)):
skip_image = True
break
if skip_image is False:
if verify(str(matchObj.group().split('v')[-1])):
image_version_list.append(
matchObj.group().split('v')[-1])
if len(image_version_list) == 0: # Create initial version
version = "{0}/v0.1.0".format(sha)
print(
"No version currently exist in the Docker Registry. \nDeploying version:{0}"
.format(version))
else:
version = self.incrementVersion(sha, image_version_list, args)
return version
def incrementVersion(self, sha, image_version_list, args):
latest = max(image_version_list, key=self.splitVersion)
ver_tuple = self.splitVersion(latest)
latest_version = ''
if args.incr_major:
latest_version = "{0}/v{1}.0.0".format(sha,
(int(ver_tuple[0]) + 1))
return latest_version
if args.incr_patch:
latest_version = "{0}/v{1}.{2}.{3}".format(sha, int(ver_tuple[0]),
int(ver_tuple[1]),
(int(ver_tuple[2]) + 1))
return latest_version
latest_version = "{0}/v{1}.{2}.0".format(sha, int(ver_tuple[0]),
(int(ver_tuple[1]) + 1))
return latest_version
def splitVersion(self, version):
major, _, rest = version.partition('.')
minor, _, rest = rest.partition('.')
patch, _, rest = rest.partition('.')
return int(major), int(minor) if minor else 0, int(
patch) if patch else 0
def parseArgs(self):
self.parser = argparse.ArgumentParser(prog='roger deploy',
description=describe())
self.parser.add_argument(
'-e',
'--environment',
metavar='env',
help="environment to deploy to. Example: 'dev' or 'stage'")
self.parser.add_argument(
'application',
metavar='application',
help="application to deploy. Can also push specific"
" containers(comma seperated). Example: 'all' or 'app1:app2' or 'kairos' or 'app_name[container1,container2]' or 'app1[container1,container2]:app2[container3,container4]' or 'app1:app2[container]'"
)
self.parser.add_argument(
'-b',
'--branch',
metavar='branch',
help=
"branch to pull code from. Defaults to master. Example: 'production' or 'master'"
)
self.parser.add_argument(
'-sg',
'--skip-gitpull',
action="store_true",
help="skip the gitpull step. Defaults to false.")
self.parser.add_argument(
'-s',
'--skip-build',
action="store_true",
help="whether to skip the build step. Defaults to false.")
self.parser.add_argument(
'config_file',
metavar='config_file',
help=
"configuration file to be use. Example: 'content.json' or 'kwe.json'"
)
self.parser.add_argument(
'-M',
'--incr-major',
action="store_true",
help="increment major in version. Defaults to false.")
self.parser.add_argument(
'-sp',
'--skip-push',
action="store_true",
help="skip the push step. Defaults to false.'")
self.parser.add_argument(
'-f',
'--force-push',
action="store_true",
help=
"force push. Not recommended. Forces push even if validation checks failed. Applies only if skip_push is false. Defaults to false."
)
self.parser.add_argument(
'-p',
'--incr-patch',
action="store_true",
help="increment patch in version. Defaults to false.'")
self.parser.add_argument(
'-S',
'--secrets-file',
help=
"specifies an optional secrets file for deployment runtime variables."
)
self.parser.add_argument(
'-d',
'--directory',
help=
"working directory. Uses a temporary directory if not specified.")
return self.parser
def main(self, settingObject, appObject, frameworkUtilsObject, gitObj,
hooksObj, args):
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS'
settingObj = settingObject
appObj = appObject
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
roger_env = appObj.getRogerEnv(config_dir)
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
if 'registry' not in roger_env:
raise ValueError(
'Registry not found in roger-mesos-tools.config file.')
else:
self.registry = roger_env['registry']
# Setup for Slack-Client, token, and git user
if 'notifications' in config:
self.slack = Slack(
config['notifications'],
'/home/vagrant/.roger_cli.conf.d/slack_token')
self.identifier = self.utils.get_identifier(
config_name, settingObj.getUser(), args.application)
apps = []
apps_container_dict = {}
if args.application == 'all':
apps = config['apps'].keys()
else:
if ":" not in args.application and "[" not in args.application:
apps.append(args.application)
else:
for item in args.application.split(":"):
if '[' in item:
matchObj = re.match(r'(.*)\[(.*)\]', item)
apps.append(matchObj.group(1))
apps_container_dict[matchObj.group(
1)] = matchObj.group(2)
else:
apps.append(item)
common_repo = config.get('repo', '')
environment = roger_env.get('default_environment', '')
work_dir = ''
if args.directory:
work_dir = args.directory
temp_dir_created = False
print("Using {0} as the working directory".format(work_dir))
else:
work_dir = mkdtemp()
temp_dir_created = True
print("Created a temporary dir: {0}".format(work_dir))
if args.environment is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file"
)
else:
print(
"Using value {} from environment variable $ROGER_ENV"
.format(env_var))
environment = env_var
else:
environment = args.environment
if environment not in roger_env['environments']:
self.removeDirTree(work_dir, args, temp_dir_created)
raise ValueError(
'Environment not found in roger-mesos-tools.config file.')
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
try:
for app in apps:
if app not in config['apps']:
print(
'Application {} specified not found.'.format(app))
else:
try:
print("Deploying {} ...".format(app))
self.deployApp(settingObject, appObject,
frameworkUtilsObject, gitObj,
hooksObj, root, args, config,
roger_env, work_dir, config_dir,
environment, app, branch,
self.slack, args.config_file,
common_repo, temp_dir_created,
apps_container_dict)
except (IOError, ValueError) as e:
print(
"The following error occurred when deploying {}: {}"
.format(app, e),
file=sys.stderr)
pass # try deploying the next app
except (Exception) as e:
print("The following error occurred: %s" % e, file=sys.stderr)
raise
except (Exception) as e:
execution_result = 'FAILURE'
print("The following error occurred: %s" % e, file=sys.stderr)
raise
finally:
# Check if the initializition of variables carried out
if 'function_execution_start_time' not in globals(
) and 'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
if 'execution_result' not in globals(
) and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals() and 'config_name' not in locals(
):
config_name = ""
if 'environment' not in globals() and 'environment' not in locals(
):
environment = "dev"
if not hasattr(args, "application"):
args.application = ""
if 'settingObj' not in globals() and 'settingObj' not in locals():
settingObj = Settings()
if 'work_dir' not in globals() and 'work_dir' not in locals():
work_dir = ''
temp_dir_created = False
if not (self.rogerGitPullObject.outcome is 1
and self.rogerBuildObject.outcome is 1
and self.rogerPushObject.outcome is 1):
execution_result = 'FAILURE'
try:
# If the deploy fails before going through any steps
sc = self.utils.getStatsClient()
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(
config_name, settingObj.getUser(), args.application)
args.application = self.utils.extract_app_name(
args.application)
time_take_milliseonds = (
(datetime.now() -
function_execution_start_time).total_seconds() * 1000)
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(
args.application) + ",event=deploy" + ",outcome=" + str(
execution_result
) + ",config_name=" + str(config_name) + ",env=" + str(
environment) + ",user="******",identifier=" + str(
self.identifier)
tup = (input_metric, time_take_milliseonds)
self.statsd_message_list.append(tup)
self.removeDirTree(work_dir, args, temp_dir_created)
except (Exception) as e:
print("The following error occurred: %s" % e, file=sys.stderr)
raise
def deployApp(self, settingObject, appObject, frameworkUtilsObject, gitObj,
hooksObj, root, args, config, roger_env, work_dir,
config_dir, environment, app, branch, slack, config_file,
common_repo, temp_dir_created, apps_container_dict):
startTime = datetime.now()
settingObj = settingObject
appObj = appObject
frameworkUtils = frameworkUtilsObject
environmentObj = roger_env['environments'][environment]
data = appObj.getAppData(config_dir, config_file, app)
frameworkObj = frameworkUtils.getFramework(data)
framework = frameworkObj.getName()
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', app)
image_name = ''
image = ''
skip_gitpull = False
if args.skip_gitpull is not None:
skip_gitpull = args.skip_gitpull
# get/update target source(s)
if not skip_gitpull:
args.app_name = app
args.directory = work_dir
self.rogerGitPullObject.statsd_message_list = self.statsd_message_list
self.rogerGitPullObject.identifier = self.identifier
self.rogerGitPullObject.main(settingObj, appObj, gitObj, hooksObj,
args)
skip_build = False
if args.skip_build is not None:
skip_build = args.skip_build
skip_push = False
if args.skip_push is not None:
skip_push = args.skip_push
secrets_file = None
if args.secrets_file is not None:
secrets_file = args.secrets_file
# Set initial version
image_git_sha = getGitSha(work_dir, repo, branch, gitObj)
image_name = "{0}-{1}-{2}/v0.1.0".format(config['name'], app,
image_git_sha)
if skip_build:
curr_image_ver = frameworkObj.getCurrentImageVersion(
roger_env, environment, app)
self.image_name = curr_image_ver
print("Current image version deployed on {0} is :{1}".format(
framework, curr_image_ver))
if curr_image_ver is not None:
image_name = "{0}-{1}-{2}".format(config['name'], app,
curr_image_ver)
print("Image current version from {0} endpoint is:{1}".format(
framework, image_name))
else:
print("Using base version for image:{0}".format(image_name))
else:
# Docker build,tag and push
image_name = self.getNextVersion(config, roger_env, app, branch,
work_dir, repo, args, gitObj)
image_name = "{0}-{1}-{2}".format(config['name'], app, image_name)
print("Bumped up image to version:{0}".format(image_name))
self.image_name = image_name
build_args = args
build_args.app_name = app
build_args.directory = os.path.abspath(work_dir)
build_args.tag_name = image_name
build_args.config_file = config_file
build_args.env = environment
build_args.push = True
try:
self.rogerBuildObject.identifier = self.identifier
self.rogerBuildObject.statsd_message_list = self.statsd_message_list
self.rogerBuildObject.main(settingObj, appObject, hooksObj,
self.dockerUtilsObject,
self.dockerObject, build_args)
except ValueError:
raise
print("Version is:" + image_name)
# Deploying the app to framework
args.image_name = image_name
args.config_file = config_file
args.env = environment
if app in apps_container_dict:
args.app_name = str(app) + ":" + apps_container_dict[app]
else:
args.app_name = app
self.rogerPushObject.identifier = self.identifier
self.rogerPushObject.statsd_message_list = self.statsd_message_list
self.rogerPushObject.main(settingObj, appObj, frameworkUtils, hooksObj,
args)
deployTime = datetime.now() - startTime
username = settingObj.getUser()
deployMessage = "{0}'s deploy for {1} / {2} / {3} completed in {4} seconds.".format(
username, app, environment, branch, deployTime.total_seconds())
if slack is not None:
slack.api_call(deployMessage)
print(deployMessage)
class RogerBuild(object):
def __init__(self):
self.utils = Utils()
self.statsd_message_list = []
self.outcome = 1
self.registry = ""
self.tag_name = ""
def parse_args(self):
self.parser = argparse.ArgumentParser(
prog='roger build', description=describe())
self.parser.add_argument('app_name', metavar='app_name',
help="application to build. Example: 'agora'.")
self.parser.add_argument('directory', metavar='directory',
help="working directory. Example: '/home/vagrant/work_dir'.")
self.parser.add_argument('tag_name', metavar='tag_name',
help="tag for the built image. Example: 'roger-collectd:0.20'.")
self.parser.add_argument('config_file', metavar='config_file',
help="configuration file to use. Example: 'content.json'.")
self.parser.add_argument(
'--push', '-p', help="Also push to registry. Defaults to false.", action="store_true")
return self.parser
def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj, args):
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
config = appObj.getConfig(config_dir, args.config_file)
hooksObj.config_file = args.config_file
roger_env = appObj.getRogerEnv(config_dir)
config_name = ""
if 'name' in config:
config_name = config['name']
common_repo = config.get('repo', '')
if not hasattr(args, "env"):
args.env = "dev"
data = appObj.getAppData(config_dir, args.config_file, args.app_name)
if not data:
raise ValueError('Application with name [{}] or data for it not found at {}/{}.'.format(
args.app_name, config_dir, args.config_file))
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', args.app_name)
build_args = {}
if 'build-args' in data:
if 'environment' in data['build-args']:
if args.env in data['build-args']['environment']:
build_args = data['build-args']['environment'][args.env]
projects = data.get('privateProjects', [])
docker_path = data.get('path', 'none')
# get/update target source(s)
file_exists = True
file_path = ''
cur_dir = ''
if "PWD" in os.environ:
cur_dir = os.environ.get('PWD')
abs_path = os.path.abspath(args.directory)
repo_name = appObj.getRepoName(repo)
if docker_path != 'none':
if abs_path == args.directory:
file_path = "{0}/{1}/{2}".format(args.directory,
repo_name, docker_path)
else:
file_path = "{0}/{1}/{2}/{3}".format(
cur_dir, args.directory, repo_name, docker_path)
else:
if abs_path == args.directory:
file_path = "{0}/{1}".format(args.directory, repo_name)
else:
file_path = "{0}/{1}/{2}".format(cur_dir,
args.directory, repo_name)
if not hasattr(args, "app_name"):
args.app_name = ""
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
args.app_name = self.utils.extract_app_name(args.app_name)
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "pre_build"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"{0}/{1}".format(file_path, data['build_filename']))
file_exists = os.path.exists(build_filename)
if not file_exists:
raise ValueError("Specified build file: {} does not exist. Exiting build.".format(build_filename))
else:
file_exists = os.path.exists("{0}/Dockerfile".format(file_path))
if file_exists:
if 'registry' not in roger_env:
raise ValueError('Registry not found in roger-mesos-tools.config file.')
else:
self.registry = roger_env['registry']
self.tag_name = args.tag_name
image = "{0}/{1}".format(roger_env['registry'], args.tag_name)
try:
if abs_path == args.directory:
try:
dockerObj.docker_build(
dockerUtilsObj, appObj, args.directory, repo, projects, docker_path, image, build_args, build_filename)
except ValueError:
print('Docker build failed.')
raise
else:
directory = '{0}/{1}'.format(cur_dir, args.directory)
try:
dockerObj.docker_build(
dockerUtilsObj, appObj, directory, repo, projects, docker_path, image, build_args, build_filename)
except ValueError:
print('Docker build failed.')
raise
build_message = "Image {0} built".format(image)
if(args.push):
exit_code = dockerUtilsObj.docker_push(image)
if exit_code != 0:
raise ValueError(
'Docker push failed.')
build_message += " and pushed to registry {}".format(roger_env[
'registry'])
print(build_message)
except (IOError) as e:
print("The folowing error occurred.(Error: %s).\n" %
e, file=sys.stderr)
raise
else:
print("Dockerfile does not exist in dir: {}".format(file_path))
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "post_build"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"The following error occurred: %s" %
e, file=sys.stderr)
raise
finally:
try:
# If the build fails before going through any steps
if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
if 'execution_result' not in globals() and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals() and 'config_name' not in locals():
config_name = ""
if not hasattr(args, "env"):
args.env = "dev"
if not hasattr(args, "app_name"):
args.app_name = ""
if 'settingObj' not in globals() and 'settingObj' not in locals():
settingObj = Settings()
if 'execution_result' is 'FAILURE':
self.outcome = 0
sc = self.utils.getStatsClient()
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=build" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"The following error occurred: %s" %
e, file=sys.stderr)
raise
class RogerBuild(object):
def __init__(self):
self.utils = Utils()
self.statsd_message_list = []
self.outcome = 1
self.registry = ""
self.tag_name = ""
def parse_args(self):
self.parser = argparse.ArgumentParser(
prog='roger build', description=describe())
self.parser.add_argument('app_name', metavar='app_name',
help="application to build. Example: 'agora'.")
self.parser.add_argument('directory', metavar='directory',
help="working directory. Example: '/home/vagrant/work_dir'.")
self.parser.add_argument('tag_name', metavar='tag_name',
help="tag for the built image. Example: 'roger-collectd:0.20'.")
self.parser.add_argument('config_file', metavar='config_file',
help="configuration file to use. Example: 'content.json'.")
self.parser.add_argument('-v', '--verbose', help="verbose mode for debugging. Defaults to false.", action="store_true")
self.parser.add_argument(
'--push', '-p', help="Also push to registry. Defaults to false.", action="store_true")
return self.parser
def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj, args):
print(colored("******Building the Docker image now******", "grey"))
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
config = appObj.getConfig(config_dir, args.config_file)
hooksObj.config_file = args.config_file
roger_env = appObj.getRogerEnv(config_dir)
config_name = ""
if 'name' in config:
config_name = config['name']
common_repo = config.get('repo', '')
if not hasattr(args, "env"):
args.env = "dev"
data = appObj.getAppData(config_dir, args.config_file, args.app_name)
if not data:
raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format(
args.app_name, config_dir, args.config_file))
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', args.app_name)
build_args = {}
if 'build-args' in data:
if 'environment' in data['build-args']:
if args.env in data['build-args']['environment']:
build_args = data['build-args']['environment'][args.env]
projects = data.get('privateProjects', [])
# get/update target source(s)
file_exists = True
file_path = ''
cur_dir = ''
if "PWD" in os.environ:
cur_dir = os.environ.get('PWD')
# This is bad code, assuming current directory and then trying to again guess, this is not rocket science
# it's a f*****g file path, as simple as that. https://seomoz.atlassian.net/browse/ROGER-2405
# dockerfile location possibilities
# 1. Path relative to the repo, we know repo path for cli is <checkout_dir>/<repo>
# 2. Absolute path
# This path comes from config file and not passed on commandline so we should not try to prefix current
# working directory if the relative path is passed, don't try to guess too much.
# changelog : relative path from current directory won't work for working_directory or checkout_dir
# changelog : working_directory or checkout_dir should be absolute path, not backward-compatible
checkout_dir = os.path.abspath(args.directory)
repo_name = appObj.getRepoName(repo)
# (vmahedia) todo : this should be called docker_file_dir
dockerfile_rel_repo_path = data.get('path', '')
file_path = os.path.join(checkout_dir, repo_name, dockerfile_rel_repo_path)
if not hasattr(args, "app_name"):
args.app_name = ""
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
args.app_name = self.utils.extract_app_name(args.app_name)
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "pre_build"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"{} hook failed.".format(hookname))
build_filename = 'Dockerfile'
if 'build_filename' in data:
build_filename = ("{0}/{1}".format(file_path, data['build_filename']))
file_exists = os.path.exists(build_filename)
if not file_exists:
raise ValueError("Specified build file: {} does not exist. Exiting build.".format(build_filename))
else:
file_exists = os.path.exists("{0}/Dockerfile".format(file_path))
if file_exists:
# (vmahedia) todo: We know what parameters are required for build command so we should not wait until
# now to bailout. Config parser should have a validator for every command to see if all the Required
# parameters are passed or not. Why do all this up to this point if we know we will fail on this.
# RequiredParameter, below, "registry"
if 'registry' not in roger_env:
raise ValueError("Registry not found in roger-mesos-tools.config file.")
else:
self.registry = roger_env['registry']
self.tag_name = args.tag_name
image = "{0}/{1}".format(roger_env['registry'], args.tag_name)
try:
if checkout_dir == args.directory:
try:
dockerObj.docker_build(
dockerUtilsObj, appObj, args.directory, repo, projects, dockerfile_rel_repo_path, image, build_args, args.verbose, build_filename)
except ValueError:
raise ValueError("Docker build failed")
else:
directory = '{0}/{1}'.format(cur_dir, args.directory)
try:
dockerObj.docker_build(
dockerUtilsObj, appObj, directory, repo, projects, dockerfile_rel_repo_path, image, build_args, args.verbose, build_filename)
except ValueError:
print('Docker build failed.')
raise
print(colored("******Successfully built Docker image******", "green"))
build_message = "Image [{}]".format(image)
if(args.push):
print(colored("******Pushing Docker image to registry******", "grey"))
exit_code = dockerUtilsObj.docker_push(image, args.verbose)
if exit_code != 0:
raise ValueError(
'Docker push failed.')
build_message += " successfully pushed to registry [{}]*******".format(roger_env[
'registry'])
print(colored(build_message, "green"))
except (IOError) as e:
printException(e)
raise
else:
print(colored("Dockerfile does not exist in dir: {}".format(file_path), "red"))
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "post_build"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******""
if not hasattr(args, "env"):
args.env = "dev"
if not hasattr(args, "app_name"):
args.app_name = ""
if 'settingObj' not in globals() and 'settingObj' not in locals():
settingObj = Settings()
if 'execution_result' is 'FAILURE':
self.outcome = 0
sc = self.utils.getStatsClient()
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=build" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user=" + str(settingObj.getUser())
tup = (input_metric, time_take_milliseonds)
self.statsd_message_list.append(tup)
except (Exception) as e:
printException(e)
raise
class RogerPush(object):
def __init__(self):
self.utils = Utils()
self.task_id = []
self.statsd_message_list = []
self.statsd_push_list = []
self.outcome = 1
self.registry = ""
self.image_name = ""
def parse_args(self):
self.parser = argparse.ArgumentParser(prog='roger push',
description=describe())
self.parser.add_argument(
'app_name',
metavar='app_name',
help="application to push. Can also push specific"
" containers(comma seperated). Example: 'agora' or 'app_name:container1,container2'"
)
self.parser.add_argument(
'-e',
'--env',
metavar='env',
help="environment to push to. Example: 'dev' or 'prod'")
self.parser.add_argument(
'directory',
metavar='directory',
help="working directory. Example: '/home/vagrant/work_dir'")
self.parser.add_argument(
'image_name',
metavar='image_name',
help=
"image name that includes version to use. Example: 'roger-collectd-v0.20' or 'elasticsearch-v0.07'"
)
self.parser.add_argument(
'config_file',
metavar='config_file',
help=
"configuration file to use. Example: 'content.json' or 'kwe.json'")
self.parser.add_argument(
'--skip-push',
'-s',
help=
"skips push. Only generates components for review. Defaults to false.",
action="store_true")
self.parser.add_argument(
'--force-push',
'-f',
help=
"force push. Not Recommended. Forces push even if validation checks failed. Defaults to false.",
action="store_true")
self.parser.add_argument(
'--secrets-file',
'-S',
help=
"specifies an optional secrets file for deploy runtime variables.")
return self.parser
def loadSecrets(self, secrets_dir, file_name, args, environment):
if args.secrets_file is not None:
print("Using specified secrets file: {}".format(args.secrets_file))
file_name = args.secrets_file
exists = os.path.exists(secrets_dir)
if exists is False:
os.makedirs(secrets_dir)
# Two possible paths -- first without environment, second with
path1 = "{}/{}".format(secrets_dir, file_name)
path2 = "{}/{}/{}".format(secrets_dir, environment, file_name)
print(" Loading secrets from {} or {}".format(path1, path2))
try:
with open(path1) as f:
return_file = yaml.load(f) if path1.lower().endswith(
'.yml') else json.load(f)
return return_file
except IOError:
pass
except ValueError as e:
raise ValueError(" Error while loading json from {} - {}".format(
path1, e))
try:
with open(path2) as f:
return_file = yaml.load(f) if path2.lower().endswith(
'.yml') else json.load(f)
return return_file
except IOError:
print(" Couldn't load secrets file environment in %s or %s\n" %
(path1, path2),
file=sys.stderr)
return {}
except ValueError as e:
raise ValueError(" Error while loading json from {} - {}".format(
path2, e))
def replaceSecrets(self, output_dict, secrets_dict):
if type(output_dict) is not dict:
return output_dict
for key in output_dict:
if output_dict[key] == "SECRET":
if key in secrets_dict.keys():
output_dict[key] = secrets_dict[key]
if type(output_dict[key]) is list:
temp_list = []
for list_elem in output_dict[key]:
temp_list.append(
self.replaceSecrets(list_elem, secrets_dict))
output_dict[key] = temp_list
if type(output_dict[key]) is dict:
temp_dict = self.replaceSecrets(output_dict[key], secrets_dict)
output_dict[key] = temp_dict
return output_dict
def mergeSecrets(self, json_str, secrets):
'''Given a JSON string and an object of secret environment variables, replaces
parses the JSON keys with the secret variables. Returns back
a JSON string. Raises an error if there are any SECRET variables still exists.'''
print(
"WARNING - The use of \"SECRET\" is deprecated. Please switch to using Jinja variables. To do so,"
" use '{{ <actual variable name> }}' instead of \"SECRET\" in the target file."
)
output_dict = json.loads(json_str)
json_str = json.dumps(self.replaceSecrets(output_dict, secrets),
indent=4)
if '\"SECRET\"' in json_str:
print(
'There are still "SECRET" values -- does your secrets file have all secret environment variables?'
)
return "StandardError"
return json_str
def renderTemplate(self, template, environment, image, app_data, config,
container, container_name, additional_vars):
variables = {'environment': environment, 'image': image}
# Copy variables from config-wide, app-wide, then container-wide variable
# configs, each one from "global" and then environment-specific.
for obj in [config, app_data, container]:
if type(obj) == dict and 'vars' in obj:
variables.update(obj['vars'].get('global', {}))
variables.update(obj['vars'].get('environment',
{}).get(environment, {}))
variables.update(additional_vars)
return template.render(variables)
def statsd_counter_logging(self, metric):
sc = self.utils.getStatsClient()
sc.incr(metric, 1)
def repo_relative_path(self, appConfig, args, repo, path):
'''Returns a path relative to the repo, assumed to be under [args.directory]/[repo name]'''
repo_name = appConfig.getRepoName(repo)
abs_path = os.path.abspath(args.directory)
if abs_path == args.directory:
return "{0}/{1}/{2}".format(args.directory, repo_name, path)
else:
return "{0}/{1}/{2}/{3}".format(os.environ.get('PWD', ''),
args.directory, repo_name, path)
def main(self, settings, appConfig, frameworkObject, hooksObj, args):
try:
settingObj = settings
appObj = appConfig
frameworkUtils = frameworkObject
config_dir = settingObj.getConfigDir()
hooksObj.config_file = args.config_file
cur_file_path = os.path.dirname(os.path.realpath(__file__))
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
act_as_user = ""
if 'name' in config:
config_name = config['name']
if 'act-as' in config:
act_as_user = config['act-as']
roger_env = appObj.getRogerEnv(config_dir)
if not hasattr(args, "app_name"):
args.app_name = ""
if 'registry' not in roger_env.keys():
raise ValueError(
'Registry not found in roger-mesos-tools.config file.')
else:
self.registry = roger_env['registry']
if hasattr(args, "image_name"):
self.image_name = args.image_name
environment = roger_env.get('default_environment', '')
if args.env is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file"
)
else:
print(
"Using value {} from environment variable $ROGER_ENV"
.format(env_var))
environment = env_var
else:
environment = args.env
if environment not in roger_env['environments']:
raise ValueError(
'Environment not found in roger-mesos-tools.config file.')
environmentObj = roger_env['environments'][environment]
common_repo = config.get('repo', '')
app_name = args.app_name
container_list = []
if ':' in app_name:
tokens = app_name.split(':')
app_name = tokens[0]
if ',' in tokens[1]:
container_list = tokens[1].split(',')
else:
container_list.append(tokens[1])
data = appObj.getAppData(config_dir, args.config_file, app_name)
if not data:
raise ValueError(
'Application with name [{}] or data for it not found at {}/{}.'
.format(app_name, config_dir, args.config_file))
configured_container_list = []
for task in data['containers']:
if type(task) == dict:
configured_container_list.append(task.keys()[0])
else:
configured_container_list.append(task)
if not set(container_list) <= set(configured_container_list):
raise ValueError(
'List of containers [{}] passed do not match list of acceptable containers: [{}]'
.format(container_list, configured_container_list))
frameworkObj = frameworkUtils.getFramework(data)
framework = frameworkObj.getName()
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', app_name)
comp_dir = settingObj.getComponentsDir()
templ_dir = settingObj.getTemplatesDir()
secrets_dir = settingObj.getSecretsDir()
# Create comp_dir if it doesn't exist
if not os.path.isdir(comp_dir):
os.makedirs(comp_dir)
# template marathon files
if not container_list:
data_containers = data['containers']
else:
data_containers = container_list
failed_container_dict = {}
template = ''
# Required for when work_dir,component_dir,template_dir or
# secret_env_dir is something like '.' or './temp"
os.chdir(cur_file_path)
app_path = ''
if 'template_path' in data:
app_path = self.repo_relative_path(appObj, args, repo,
data['template_path'])
else:
app_path = templ_dir
extra_vars = {}
if 'extra_variables_path' in data:
ev_path = self.repo_relative_path(appObj, args, repo,
data['extra_variables_path'])
with open(ev_path) as f:
extra_vars = yaml.load(f) if ev_path.lower().endswith(
'.yml') else json.load(f)
if not app_path.endswith('/'):
app_path = app_path + '/'
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(
config_name, settingObj.getUser(), args.app_name)
args.app_name = self.utils.extract_app_name(args.app_name)
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "pre_push"
hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(
args.app_name) + ",identifier=" + str(
self.identifier) + ",config_name=" + str(
config_name) + ",env=" + str(
environment) + ",user="******"{0}-{1}.json".format(
config['name'], container_name)
else:
container_name = container
containerConfig = "{0}-{1}.json".format(
config['name'], container)
env = Environment(loader=FileSystemLoader(
"{}".format(app_path)),
undefined=StrictUndefined)
template_with_path = "[{}{}]".format(app_path, containerConfig)
try:
template = env.get_template(containerConfig)
except exceptions.TemplateNotFound as e:
raise ValueError(
"The template file {} does not exist".format(
template_with_path))
except Exception as e:
raise ValueError(
"Error while reading template from {} - {}".format(
template_with_path, e))
additional_vars = {}
additional_vars.update(extra_vars)
secret_vars = self.loadSecrets(secrets_dir, containerConfig,
args, environment)
additional_vars.update(secret_vars)
image_path = "{0}/{1}".format(roger_env['registry'],
args.image_name)
print(
"Rendering content from template {} for environment [{}]".
format(template_with_path, environment))
try:
output = self.renderTemplate(template, environment,
image_path, data, config,
container, container_name,
additional_vars)
except exceptions.UndefinedError as e:
error_str = "The following error occurred. %s.\n" % e
print(error_str, file=sys.stderr)
failed_container_dict[container_name] = error_str
pass
# Adding check to see if all jinja variables git resolved fot the
# container
if container_name not in failed_container_dict:
# Adding check so that not all apps try to mergeSecrets
try:
outputObj = json.loads(output)
except Exception as e:
raise ValueError(
"Error while loading json from {} - {}".format(
template_with_path, e))
if 'SECRET' in output:
output = self.mergeSecrets(
output,
self.loadSecrets(secrets_dir, containerConfig,
args, environment))
if output != "StandardError":
try:
comp_exists = os.path.exists(
"{0}".format(comp_dir))
if comp_exists is False:
os.makedirs("{0}".format(comp_dir))
comp_env_exists = os.path.exists("{0}/{1}".format(
comp_dir, environment))
if comp_env_exists is False:
os.makedirs("{0}/{1}".format(
comp_dir, environment))
except Exception as e:
logging.error(traceback.format_exc())
with open(
"{0}/{1}/{2}".format(comp_dir, environment,
containerConfig),
'wb') as fh:
fh.write(output)
if args.skip_push:
print(
"Skipping push to {} framework. The rendered config file(s) are under {}/{}"
.format(framework, comp_dir, environment))
else:
# push to roger framework
if 'owner' in config:
frameworkObj.act_as_user = config['owner']
tools_version_value = self.utils.get_version()
image_name = self.registry + "/" + args.image_name
image_tag_value = urllib.quote("'" + image_name + "'")
for container in data_containers:
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs
sc = self.utils.getStatsClient()
except (Exception) as e:
print("The following error occurred: %s" % e,
file=sys.stderr)
try:
if type(container) == dict:
container_name = str(container.keys()[0])
containerConfig = "{0}-{1}.json".format(
config['name'], container_name)
else:
container_name = container
containerConfig = "{0}-{1}.json".format(
config['name'], container)
if container_name in failed_container_dict:
print(
"Failed push to {} framework for container {} as unresolved Jinja variables present in template."
.format(framework, container_name))
else:
config_file_path = "{0}/{1}/{2}".format(
comp_dir, environment, containerConfig)
result = frameworkObj.runDeploymentChecks(
config_file_path, environment)
if args.force_push or result is True:
resp, task_id = frameworkObj.put(
config_file_path, environmentObj,
container_name, environment, act_as_user)
container_task_id = self.utils.modify_task_id(
task_id)
self.task_id.extend(container_task_id)
if hasattr(resp, "status_code"):
status_code = resp.status_code
else:
print(
"Skipping push to {} framework for container {} as Validation Checks failed."
.format(framework, container))
except (Exception) as e:
print("The following error occurred: %s" % e,
file=sys.stderr)
execution_result = 'FAILURE'
raise
finally:
try:
if 'function_execution_start_time' not in globals(
) and 'function_execution_start_time' not in locals(
):
function_execution_start_time = datetime.now()
if 'execution_result' not in globals(
) and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals(
) and 'config_name' not in locals():
config_name = ""
if 'environment' not in globals(
) and 'environment' not in locals():
environment = "dev"
if 'container_name' not in globals(
) and 'container_name' not in locals():
container_name = ""
if 'status_code' not in globals(
) and 'status_code' not in locals():
status_code = "500"
if not hasattr(args, "app_name"):
args.app_name = ""
if 'settingObj' not in globals(
) and 'settingObj' not in locals():
settingObj = Settings()
if 'container_task_id' not in globals(
) and 'container_task_id' not in locals():
container_task_id = []
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(
config_name, settingObj.getUser(),
args.app_name)
if not str(status_code).startswith("20"):
execution_result = 'FAILURE'
self.outcome = 0
time_take_milliseonds = (
(datetime.now() - function_execution_start_time
).total_seconds() * 1000)
for task_id in container_task_id:
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(
args.app_name
) + ",event=push" + ",container_name=" + str(
container_name) + ",identifier=" + str(
self.identifier) + ",outcome=" + str(
execution_result
) + ",response_code=" + str(
status_code
) + ",config_name=" + str(
config_name) + ",env=" + str(
environment) + ",user="******",task_id=" + str(
task_id
) + ",tools_version=" + str(
tools_version_value
) + ",image_tag=" + str(
image_tag_value)
tup = (input_metric, time_take_milliseonds)
self.statsd_push_list.append(tup)
if str(status_code).startswith("20"):
metric = input_metric.replace(
"rogeros_tools_exec_time",
"rogeros_events")
metric = metric + ",source=tools" + ",task_id=" + task_id
self.statsd_counter_logging(metric)
except (Exception) as e:
print("The following error occurred: %s" % e,
file=sys.stderr)
raise
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "post_push"
hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(
args.app_name) + ",identifier=" + str(
self.identifier) + ",config_name=" + str(
config_name) + ",env=" + str(
environment) + ",user="******"The following error occurred: %s" % e, file=sys.stderr)
raise