def custom_api_call(self, text, channel):
""" Makes the webhook call
Keyword arguments:
text -- message to be posted
channel -- to which channel
posts a message if rogeros - bot is present
"""
try:
self.webhookSetting()
if len(channel) == 0:
channel = self.defChannel
var = self.sc.api_call("channels.list")
length = len(var['channels'])
for iterator in range(0, length):
# getting rid of # for comparison
if channel[1:] == var['channels'][iterator]['name']:
if self.botid in var['channels'][iterator]['members']:
if not self.disabled:
self.client.notify(channel=channel, username=self.username,
icon_emoji=self.emoji, text=text)
except (Exception) as e:
# notify to channel and log it as well
printException(e)
raise
def get_containerid_mesostaskid(self, appTaskId, hostname):
containerId = ''
mesosTaskId = ''
try:
containers = subprocess.check_output(
"docker -H tcp://{}:4243 ps -q".format(hostname), shell=True)
except:
print("No route to host. Please check hostname '{}'".format(
hostname),
file=sys.stderr)
return
for container in containers.split('\n'):
if container.strip() != '':
try:
mesosTaskId = subprocess.check_output(
"docker -H tcp://{0}:4243 exec {1} \
printenv MESOS_TASK_ID".format(hostname, container),
stderr=subprocess.STDOUT,
shell=True)
except Exception as e:
if ("Cannot connect to the Docker daemon"
in str(e.output)):
printException(e)
break
else:
# This is the case when a container does not have a
# MESOS_TASK_ID in its ENV variables
pass
if mesosTaskId.startswith(appTaskId):
containerId = container.strip()
break
return containerId, mesosTaskId
def get_containerid_mesostaskid(self, appTaskId, hostname):
containerId = ''
mesosTaskId = ''
try:
containers = subprocess.check_output(
"docker -H tcp://{}:4243 ps -q".format(hostname), shell=True)
except:
print("No route to host. Please check hostname '{}'".format(
hostname), file=sys.stderr)
return
for container in containers.split('\n'):
if container.strip() != '':
try:
mesosTaskId = subprocess.check_output("docker -H tcp://{0}:4243 exec {1} \
printenv MESOS_TASK_ID".format(hostname, container), stderr=subprocess.STDOUT, shell=True)
except Exception as e:
if ("Cannot connect to the Docker daemon" in str(e.output)):
printException(e)
break
else:
# This is the case when a container does not have a
# MESOS_TASK_ID in its ENV variables
pass
if mesosTaskId.startswith(appTaskId):
containerId = container.strip()
break
return containerId, mesosTaskId
def run_hook(self, hookname, appdata, path, hook_input_metric):
try:
exit_code = 0
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS'
self.whobj.invoke_webhook(appdata, hook_input_metric, self.config_file)
abs_path = os.path.abspath(path)
if "hooks" in appdata and hookname in appdata["hooks"]:
command = appdata["hooks"][hookname]
with chdir(abs_path):
print("About to run {} hook [{}] at path {}".format(
hookname, command, abs_path))
exit_code = os.system(command)
except (Exception) as e:
printException(e)
execution_result = 'FAILURE'
raise
finally:
try:
if 'execution_result' not in globals() and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
sc = self.utils.getStatsClient()
time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
hook_input_metric = hook_input_metric + ",outcome=" + str(execution_result)
tup = (hook_input_metric, time_take_milliseonds)
self.statsd_message_list.append(tup)
except (Exception) as e:
printException(e)
raise
return exit_code
def configLevelSettings(self, config_file):
""" Prepares all the config_level settings as variables
Keyword arguments:
config_file -- This is the file name passed as argument
return three sets of channels, envs and commands.
self variable as config level variable not expected to change for the run
"""
if (not self.configLoadFlag):
self.config = self.appconfigObj.getConfig(self.config_dir, config_file)
self.configLoadFlag = True
if(not self.areBasicKeysAvailableInConfig(self.config)):
return
try:
self.config_channels = Set(self.config['notifications']['channels'])
if 'envs' in self.config['notifications'].keys():
self.config_envs = Set(self.config['notifications']['envs'])
if 'commands' in self.config['notifications'].keys():
self.config_commands = Set(self.config['notifications']['commands'])
if (len(self.config_channels) == 0 or len(self.config_envs) == 0 or len(self.config_commands) == 0):
return
except (Exception, KeyError, ValueError) as e:
# notify to channel and log it as well
printException(e)
raise
def invoke_webhook(self, appdata, config_file, action, env, user):
""" Pepares set and posts to slack channel
Keyword arguments:
appdata -- this is value related to an app
hook_input_metric -- value it gets from hook class per app
config_file -- the file name under for the app deployment
"""
envSet = []
commandsSet = []
self.function_execution_start_time = datetime.now()
self.webhookSetting()
self.configLevelSettings(config_file)
self.action = action
self.envr = env
self.user = user
self.app_name = appdata['name']
try:
if(not self.areBasicKeysAvailableInAppdata(appdata)):
return
if 'notifications' in appdata:
channelsSet = Set(appdata['notifications']['channels'])
# to handle if tag is there but no data is present
if (len(self.config_channels) != 0):
channelsSet = channelsSet.union(self.config_channels)
# to handle if tag is there
if 'envs' in appdata['notifications'].keys():
envSet = Set(appdata['notifications']['envs'])
# to handle if tag is there but data is not there
if (len(self.config_envs) != 0):
envSet = envSet.union(self.config_envs)
if 'commands' in appdata['notifications'].keys():
commandsSet = Set(appdata['notifications']['commands'])
if (len(self.config_commands) != 0):
commandsSet = commandsSet.union(self.config_commands)
if (len(channelsSet) == 0 or len(envSet) == 0 or len(commandsSet) == 0):
return
else:
if list(self.config_envs)[0] == 'all':
self.config_envs = ['dev', 'production', 'staging', 'local']
if list(self.config_commands)[0] == 'all':
self.config_commands = ['pull', 'build', 'push']
self.postToSlack(self.action, self.config_envs, self.config_commands, self.config_channels)
return
if list(envSet)[0] == 'all':
envSet = ['dev', 'production', 'staging', 'local']
if list(commandsSet)[0] == 'all':
commandsSet = ['pull', 'build', 'push']
except (Exception, KeyError, ValueError) as e:
printException(e)
raise
try:
self.postToSlack(self.action, envSet, commandsSet, channelsSet)
except (Exception) as e:
self.custom_api_call("Error : %s" % e, self.defChannel)
printException(e)
raise
def docker_search(self, registry, name, application):
result = ""
try:
result = self.docker_search_v2(registry)
except (Exception) as e:
error_message = "Error when attempting search using docker v2 catalog: {} ".format(e)
printException(error_message)
print(colored("Attempting docker v1 search", "yellow"))
result = self.docker_search_v1(registry, name, application)
return result
def docker_search(self, registry, name, application):
result = ""
try:
result = self.docker_search_v2(registry)
except (Exception) as e:
error_message = "Error when attempting search using docker v2 catalog: {} ".format(e)
printException(error_message)
print(colored("Attempting docker v1 search", "yellow"))
result = self.docker_search_v1(registry, name, application)
return result
def main(self):
self.parser = self.parse_args()
args = self.parser.parse_args()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
environment = roger_env.get('default_environment', '')
if args.env is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file")
else:
if args.verbose:
print(colored("Using value {} from environment variable $ROGER_ENV".format(env_var), "yellow"))
environment = env_var
else:
environment = args.env
if environment not in roger_env['environments']:
raise ValueError(colored("Environment not found in roger-mesos-tools.config file.", "red"))
hostname = ''
containerId = ''
if args.hostname is None:
hostname = containerconfig.get_hostname_from_marathon(
environment, roger_env, args.appTaskId)
else:
hostname = args.hostname
if hostname != '': # Hostname maybe empty when the given appTaskId does not match any taskId from Marathon
(containerId, mesosTaskId) = containerconfig.get_containerid_mesostaskid(
args.appTaskId, hostname)
else:
if args.verbose:
print(colored("Most likely hostname could not be retrieved with appTaskId {0}. Hostname is also \
an optional argument. See -h for usage.".format(args.appTaskId), "cyan"))
if containerId is not '' and containerId is not None:
if args.verbose:
print(colored("INFO - If there are multiple containers that pattern match the given mesos task Id, \
then will log into the first one", "cyan"))
print(colored("Executing bash in docker container - {0} on host - {1} for mesosTaskId - {2}".format(
containerId, hostname, mesosTaskId), "yellow"))
try:
subprocess.check_call(
"docker -H tcp://{0}:4243 exec -it {1} bash".format(hostname, containerId), shell=True)
except Exception as e:
printException(e)
else:
print(colored("No Container found on host {0} with application Task Id {1}".format(
hostname, args.appTaskId), "red"))
def run_hook(self, hookname, appdata, path, env, user):
try:
exit_code = 0
execution_result = 'SUCCESS'
self.whobj.invoke_webhook(appdata, self.config_file, hookname, env, user)
abs_path = os.path.abspath(path)
if "hooks" in appdata and hookname in appdata["hooks"]:
command = appdata["hooks"][hookname]
with chdir(abs_path):
print("About to run {} hook [{}] at path {}".format(
hookname, command, abs_path))
exit_code = os.system(command)
except (Exception) as e:
printException(e)
execution_result = 'FAILURE'
raise
finally:
# todo: maybe send a datadog event ?
pass
return exit_code
def postToSlack(self, action, envSet, commandsSet, channelsSet):
""" Prepares post to slack channel
Keyword arguments:
action -- action extracted from hookname_input_metric
envSet -- set of accepted environment
commandsSet -- set of accepted commandssets
channelsSet -- set of accepted channelsets
"""
try:
if ('post' in action and self.envr in envSet and action.split('_')[1] in commandsSet):
for channel in channelsSet:
timeElapsed = '{0:.2f}'.format((datetime.now() - self.function_execution_start_time).total_seconds() * 1000)
timeasInt = float(timeElapsed)
h, m, s, ms = self.makeTimeReadable(timeasInt)
readableTime = self.createMesage(h, m, s, ms)
slackMessage = ("Completed *" + action.split('_')[1] + "* of *" + self.app_name + "* on *" + self.envr + "* in *" + readableTime + "* (triggered by *" + self.user + "*)")
self.custom_api_call(slackMessage, '#' + channel)
except (Exception) as e:
self.custom_api_call("Error : %s" % e, self.defChannel)
printException(e)
raise
def main(self, settings, appConfig, frameworkObject, hooksObj, args):
print(colored("******Deploying application to framework******", "grey"))
try:
validation_failed = False
settingObj = settings
appObj = appConfig
frameworkUtils = frameworkObject
config_dir = settingObj.getConfigDir()
hooksObj.config_file = args.config_file
cur_file_path = os.path.dirname(os.path.realpath(__file__))
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
act_as_user = ""
if 'name' in config:
config_name = config['name']
if 'act-as' in config:
act_as_user = config['act-as']
roger_env = appObj.getRogerEnv(config_dir)
if not hasattr(args, "app_name"):
args.app_name = ""
if 'registry' not in roger_env.keys():
raise ValueError("Registry not found in roger-mesos-tools.config file.")
else:
self.registry = roger_env['registry']
if hasattr(args, "image_name"):
self.image_name = args.image_name
environment = roger_env.get('default_environment', '')
if args.env is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(colored("WARNING - Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file", "yellow"))
else:
if args.verbose:
print(colored("Using value {} from environment variable $ROGER_ENV".format(env_var), "grey"))
environment = env_var
else:
environment = args.env
# ----------------------------------------------
if environment not in roger_env['environments']:
raise ValueError("Environment not found in roger-mesos-tools.config file.")
# ----------------------------------------------
# GetEnvironmentConfig(environment)
# ----------------------------------------------
environmentObj = roger_env['environments'][environment]
common_repo = config.get('repo', '')
# ----------------------------------------------
# GetContainersForApp(app)
# ----------------------------------------------
app_name = args.app_name
container_list = []
# todo (vmahedia): What does ':' signify? Put explanation.
if ':' in app_name:
tokens = app_name.split(':')
app_name = tokens[0]
# todo (vmahedia): it's container list - need to explain syntax
if ',' in tokens[1]:
container_list = tokens[1].split(',')
else:
container_list.append(tokens[1])
# ----------------------------------------------
data = appObj.getAppData(config_dir, args.config_file, app_name)
if not data:
raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format(
app_name, config_dir, args.config_file))
configured_container_list = []
for task in data['containers']:
if type(task) == dict:
configured_container_list.append(task.keys()[0])
else:
configured_container_list.append(task)
if not set(container_list) <= set(configured_container_list):
raise ValueError("List of containers [{}] passed do not match list of acceptable containers: [{}]".format(
container_list, configured_container_list))
frameworkObj = frameworkUtils.getFramework(data)
framework = frameworkObj.getName()
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', app_name)
comp_dir = settingObj.getComponentsDir()
templ_dir = settingObj.getTemplatesDir()
secrets_dir = settingObj.getSecretsDir()
# Create comp_dir if it doesn't exist
if not os.path.isdir(comp_dir):
os.makedirs(comp_dir)
if not container_list:
data_containers = data['containers']
else:
data_containers = container_list
failed_container_dict = {}
# (vmahedia) upto this point it's all getting and checking the
# configuration parameters
template = ''
# Required for when work_dir,component_dir,template_dir or
# secret_env_dir is something like '.' or './temp"
os.chdir(cur_file_path)
app_path = ''
if 'template_path' in data:
app_path = self.repo_relative_path(appObj, args, repo, data['template_path'])
else:
app_path = templ_dir
extra_vars = {}
if 'extra_variables_path' in data:
ev_path = self.repo_relative_path(appObj, args, repo, data['extra_variables_path'])
with open(ev_path) as f:
extra_vars = yaml.load(f) if ev_path.lower(
).endswith('.yml') else json.load(f)
if not app_path.endswith('/'):
app_path = app_path + '/'
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
args.app_name = self.utils.extract_app_name(args.app_name)
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "pre_push"
hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname))
# ----------------------------------------------
# (vmahedia) Figure out what the hell this loop does
# and name it appropriately
# it seems first part is just finding a template and Rendering
# it against the given config, checking to see if there are errors
# ----------------------------------------------
# (vmahedia) Meat starts from here, probably.
for container in data_containers:
container_name = self.getContainerName(container)
containerConfig = "{0}-{1}.json".format(config['name'], container_name)
env = Environment(loader = FileSystemLoader("{}".format(app_path)), undefined = StrictUndefined)
template_with_path = "[{}{}]".format(app_path, containerConfig)
try:
template = env.get_template(containerConfig)
except exceptions.TemplateNotFound as e:
raise ValueError("The template file {} does not exist".format(template_with_path))
except Exception as e:
raise ValueError("Error while reading template from {} - {}".format(template_with_path, e))
additional_vars = {}
# (vmahedia)variables likes this should be at least visible within one
# scroll up or down, move this code to near to context
# Why are we getting the secrets everytime, this requires the file to be
# present
additional_vars.update(extra_vars)
secret_vars = self.loadSecrets(secrets_dir, containerConfig, args, environment)
additional_vars.update(secret_vars)
image_path = "{0}/{1}".format(
roger_env['registry'], args.image_name)
print("Rendering content from template {} for environment [{}]".format(
template_with_path, environment))
try:
output = self.renderTemplate(template, environment, image_path, data, config, container, container_name, additional_vars)
except exceptions.UndefinedError as e:
error_str = "The following Undefined Jinja variable error occurred. %s.\n" % e
print(colored(error_str, "red"), file=sys.stderr)
failed_container_dict[container_name] = error_str
# we are going to fail even if one of the container config is not valid but we will
# still go through the loop and collect all the errors before we bail out
validation_failed = True
pass
# ----------------------------------------------
# it seems the checks above can finish independent of the
# following code, decouple this two parts, later when the code
# is well understood
# ----------------------------------------------
# Adding check to see if all jinja variables git resolved fot
# the container
if container_name not in failed_container_dict:
# Adding check so that not all apps try to mergeSecrets
try:
outputObj = json.loads(output)
except Exception as e:
raise ValueError("Error while loading json from {} - {}".format(template_with_path, e))
if '\"SECRET\"' in output:
output = self.mergeSecrets(output, self.loadSecrets(
secrets_dir, containerConfig, args, environment))
if output != "StandardError":
try:
comp_exists = os.path.exists("{0}".format(comp_dir))
if comp_exists is False:
os.makedirs("{0}".format(comp_dir))
comp_env_exists = os.path.exists(
"{0}/{1}".format(comp_dir, environment))
if comp_env_exists is False:
os.makedirs(
"{0}/{1}".format(comp_dir, environment))
except Exception as e:
logging.error(traceback.format_exc())
# (vmahedia) Should we write out the files even though there is an error with one of the
# containers. Although maybe users would want to see some output
with open("{0}/{1}/{2}".format(comp_dir, environment, containerConfig), 'wb') as fh:
fh.write(output)
else:
raise ValueError("Error while loading secrets to render template file variables")
# Notify container error messages
# let failed_container_dict just be for now, but report all the errors
if validation_failed:
raise Exception("Unable to render Jinja template")
deployment_check_failed = False
# fail if the deployment check fails
for container in data_containers:
container_name = self.getContainerName(container)
containerConfig = "{0}-{1}.json".format(config['name'], container_name)
config_file_path = "{0}/{1}/{2}".format(comp_dir, environment, containerConfig)
result = frameworkObj.runDeploymentChecks(config_file_path, environment)
if not result:
# need to give more indication about what can they do to fix this and what exactly failed
# in the deployment check function, we should print an error in that function as well
print(colored("Deployment checks failed for container - {}".format(framework, container)), "red")
deployment_check_failed = True
if deployment_check_failed:
raise Exception("Deployment Check failed for one or more containers, check logs for more info!")
if args.skip_push:
print(colored("Skipping push to {} framework. The rendered config file(s) are under {}/{}/".format(
framework, colored(comp_dir, "cyan"), colored(environment, "cyan")), "yellow"))
else:
# push to roger framework
if 'owner' in config:
frameworkObj.act_as_user = config['owner']
tools_version_value = self.utils.get_version()
image_name = self.registry + "/" + args.image_name
image_tag_value = urllib.quote("'" + image_name + "'")
for container in data_containers:
try:
function_execution_start_time = datetime.now()
# Assume SUCCESS unless exception
execution_result = 'SUCCESS'
sc = self.utils.getStatsClient()
except (Exception) as e:
raise ValueError("{} Error : {}".format(getDebugInfo(), e))
try:
# this is where actual push is happening
# we only push if forced, in case of failures
# in deployment checks
#
# (vmahedia) todo:
# list down scenarios in which this features
# will be useful
resp, task_id = frameworkObj.put(config_file_path, environmentObj,
container_name, environment, act_as_user)
container_task_id = self.utils.modify_task_id(task_id)
self.task_id.extend(container_task_id)
if hasattr(resp, "status_code"):
status_code = resp.status_code
except (Exception) as e:
print("ERROR - : %s" %
e, file=sys.stderr)
execution_result = 'FAILURE'
raise
finally:
try:
if 'function_execution_start_time' not in globals() and \
'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
if 'execution_result' not in globals() and \
'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals() and \
'config_name' not in locals():
config_name = ""
if 'environment' not in globals() and \
'environment' not in locals():
environment = "dev"
if 'container_name' not in globals() and \
'container_name' not in locals():
container_name = ""
if 'status_code' not in globals() and \
'status_code' not in locals():
status_code = "500"
if not hasattr(args, "app_name"):
args.app_name = ""
if 'settingObj' not in globals() and \
'settingObj' not in locals():
settingObj = Settings()
if 'container_task_id' not in globals() and 'container_task_id' not in locals():
container_task_id = []
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(),
args.app_name)
if not str(status_code).startswith("20"):
execution_result = 'FAILURE'
self.outcome = 0
time_taken = (datetime.now() - function_execution_start_time).total_seconds()
for task_id in container_task_id:
input_metric = "roger-tools.rogeros_tools_exec_time" + \
",app_name=" + str(args.app_name) + \
",event=push" + \
",container_name=" + str(container_name) + \
",identifier=" + str(self.identifier) + \
",outcome=" + str(execution_result) + \
",response_code=" + str(status_code) + \
",config_name=" + str(config_name) + \
",env=" + str(environment) + \
",user="******"20"):
metric = input_metric.replace("rogeros_tools_exec_time", "rogeros_events")
metric = metric + ",source=tools" + ",task_id=" + task_id
self.statsd_counter_logging(metric)
except (Exception) as e:
printException(e)
raise
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "post_push"
hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + \
",app_name=" + str(args.app_name) + \
",identifier=" + str(self.identifier) + \
",config_name=" + str(config_name) + \
",env=" + str(environment) + \
",user="******"{} hook failed.".format(hookname))
print(colored("******Done with the PUSH step******", "green"))
except (Exception) as e:
raise ValueError("ERROR - {}".format(e))
def main(self):
self.parser = self.parse_args()
args = self.parser.parse_args()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
environment = roger_env.get('default_environment', '')
if args.env is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file"
)
else:
if args.verbose:
print(
colored(
"Using value {} from environment variable $ROGER_ENV"
.format(env_var), "yellow"))
environment = env_var
else:
environment = args.env
if environment not in roger_env['environments']:
raise ValueError(
colored(
"Environment not found in roger-mesos-tools.config file.",
"red"))
hostname = ''
containerId = ''
if args.hostname is None:
hostname = containerconfig.get_hostname_from_marathon(
environment, roger_env, args.appTaskId)
else:
hostname = args.hostname
if hostname != '': # Hostname maybe empty when the given appTaskId does not match any taskId from Marathon
(containerId,
mesosTaskId) = containerconfig.get_containerid_mesostaskid(
args.appTaskId, hostname)
else:
if args.verbose:
print(
colored(
"Most likely hostname could not be retrieved with appTaskId {0}. Hostname is also \
an optional argument. See -h for usage.".format(args.appTaskId), "cyan"))
if containerId is not '' and containerId is not None:
if args.verbose:
print(
colored(
"INFO - If there are multiple containers that pattern match the given mesos task Id, \
then will log into the first one", "cyan"))
print(
colored(
"Executing bash in docker container - {0} on host - {1} for mesosTaskId - {2}"
.format(containerId, hostname, mesosTaskId), "yellow"))
try:
subprocess.check_call(
"docker -H tcp://{0}:4243 exec -it {1} bash".format(
hostname, containerId),
shell=True)
except Exception as e:
printException(e)
else:
print(
colored(
"No Container found on host {0} with application Task Id {1}"
.format(hostname, args.appTaskId), "red"))
def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args):
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS'
settingObj = settingObject
appObj = appObject
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
roger_env = appObj.getRogerEnv(config_dir)
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
if 'registry' not in roger_env:
raise ValueError('Registry not found in roger-mesos-tools.config file.')
else:
self.registry = roger_env['registry']
# Setup for Slack-Client, token, and git user
# (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event
# Event should be registered and SlackNotification should be one of the members. it can have
# N notifications on a particular "event", Notifications.Notify will broadcast notification to
# all the interested parties.
if 'notifications' in config:
self.slack = Slack(config['notifications'],
'/home/vagrant/.roger_cli.conf.d/slack_token')
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application)
apps = []
apps_container_dict = {}
if args.application == 'all':
apps = config['apps'].keys()
else:
if ":" not in args.application and "[" not in args.application:
apps.append(args.application)
else:
for item in args.application.split(":"):
if '[' in item:
matchObj = re.match(r'(.*)\[(.*)\]', item)
apps.append(matchObj.group(1))
apps_container_dict[matchObj.group(1)] = matchObj.group(2)
else:
apps.append(item)
common_repo = config.get('repo', '')
environment = roger_env.get('default_environment', '')
work_dir = ''
if args.directory:
work_dir = args.directory
temp_dir_created = False
if args.verbose:
print("Using {0} as the working directory".format(work_dir))
else:
work_dir = mkdtemp()
temp_dir_created = True
if args.verbose:
print("Created a temporary dir: {0}".format(work_dir))
if args.environment is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file")
else:
print(
"Using value {} from environment variable $ROGER_ENV".format(env_var))
environment = env_var
else:
environment = args.environment
if environment not in roger_env['environments']:
self.removeDirTree(work_dir, args, temp_dir_created)
raise ValueError('Environment not found in roger-mesos-tools.config file.')
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
try:
for app in apps:
if app not in config['apps']:
raise ValueError('Application {} specified not found.'.format(app))
else:
try:
if args.verbose:
print("Deploying {} ...".format(app))
self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj,
root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict)
except (IOError, ValueError) as e:
error_msg = "Error when deploying {}: {}".format(app, repr(e))
printErrorMsg(error_msg)
pass # try deploying the next app
except (Exception) as e:
printException(e)
raise
except (Exception) as e:
execution_result = 'FAILURE'
printException(e)
raise
finally:
# Check if the initializition of variables carried out
if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
if 'execution_result' not in globals() and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals() and 'config_name' not in locals():
config_name = ""
if 'environment' not in globals() and 'environment' not in locals():
environment = "dev"
if not hasattr(args, "application"):
args.application = ""
if 'settingObj' not in globals() and 'settingObj' not in locals():
settingObj = Settings()
if 'work_dir' not in globals() and 'work_dir' not in locals():
work_dir = ''
temp_dir_created = False
if not (self.rogerGitPullObject.outcome is 1 and self.rogerBuildObject.outcome is 1 and self.rogerPushObject.outcome is 1):
execution_result = 'FAILURE'
try:
# If the deploy fails before going through any steps
sc = self.utils.getStatsClient()
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application)
args.application = self.utils.extract_app_name(args.application)
time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.application) + ",event=deploy" + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******",identifier=" + str(self.identifier)
tup = (input_metric, time_take_milliseonds)
self.statsd_message_list.append(tup)
self.removeDirTree(work_dir, args, temp_dir_created)
except (Exception) as e:
error_msg = "Error when deploying {}: {}".format(app, repr(e))
printErrorMsg(error_msg)
raise
def main(self):
self.parser = self.parse_args()
args = self.parser.parse_args()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
environment = roger_env.get('default_environment', '')
if args.env is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set.Using the default set from roger-mesos-tools.config file"
)
else:
print(
"Using value {} from environment variable $ROGER_ENV".
format(env_var))
environment = env_var
else:
environment = args.env
if environment not in roger_env['environments']:
raise ValueError(
'Environment not found in roger-mesos-tools.config file.')
hostname = ''
containerId = ''
if args.hostname is None:
hostname = containerconfig.get_hostname_from_marathon(
environment, roger_env, args.appTaskId)
else:
hostname = args.hostname
if hostname != '': # Hostname maybe empty when the given appTaskId does not match any taskId from Marathon
(containerId,
mesosTaskId) = containerconfig.get_containerid_mesostaskid(
args.appTaskId, hostname)
else:
print(
"Most likely hostname could not be retrieved with appTaskId {0}. Hostname is also \
an optional argument. See -h for usage.".format(args.appTaskId))
if containerId is not '' and containerId is not None:
print(
"If there are multiple containers that pattern match the given mesos task Id, \
then will log into the first one")
print(
"Displaying logs in docker container - {0} on host - {1} for mesosTask Id {2}"
.format(containerId, hostname, mesosTaskId))
command = "docker -H tcp://{0}:4243 logs ".format(hostname)
if args.follow:
command = "{} -f=true".format(command)
else:
command = "{} -f=false".format(command)
if args.since:
command = "{} --since=\"{}\"".format(command, args.since)
if args.timestamps:
command = "{} -t".format(command, args.since)
if args.tail:
command = "{} --tail=\"{}\"".format(command, args.tail)
command = "{} {}".format(command, containerId)
try:
subprocess.check_call("{}".format(command), shell=True)
except (KeyboardInterrupt, SystemExit):
print("Exited.")
except (subprocess.CalledProcessError) as e:
printException(e)
else:
print(
"No Container found on host {0} with application Task Id {1}".
format(hostname, args.appTaskId))
def main(self, settings, appConfig, gitObject, hooksObj, args):
print(colored("******Executing GIT PULL of application repo******", "grey"))
try:
function_execution_start_time = datetime.now()
environment = "dev"
if hasattr(args, "environment"):
environment = args.environment
settingObj = settings
appObj = appConfig
gitObj = gitObject
config_dir = settingObj.getConfigDir()
hooksObj.config_file = args.config_file
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
common_repo = config.get('repo', '')
data = appObj.getAppData(config_dir, args.config_file, args.app_name)
if not data:
raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format(
args.app_name, config_dir, args.config_file))
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', args.app_name)
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
if not os.path.exists(args.directory):
try:
os.makedirs(args.directory)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
hookname = "pre_gitpull"
exit_code = hooksObj.run_hook(hookname, data, args.directory, environment, settingObj.getUser())
if exit_code != 0:
raise ValueError("{} hook failed.".format(hookname))
# get/update target source(s)
repo_name = appObj.getRepoName(repo)
path = "{0}/{1}".format(args.directory, repo_name)
if os.path.isdir(path):
with chdir(path):
exit_code = gitObj.gitPull(branch, args.verbose)
else:
with chdir('{0}'.format(args.directory)):
exit_code = gitObj.gitShallowClone(repo, branch, args.verbose)
if exit_code != 0:
raise ValueError("Gitpull failed.")
hookname = "post_gitpull"
exit_code = hooksObj.run_hook(hookname, data, args.directory, environment, settingObj.getUser())
if exit_code != 0:
raise ValueError("{} hook failed.".format(hookname))
except (Exception) as e:
printException(e)
raise
finally:
# todo: maybe send a datadog event?
pass
print(colored("******Completed the GIT PULL step successfully******", "green"))
def main(self, settings, appConfig, gitObject, hooksObj, args):
print(colored("******Executing GIT PULL of application repo******", "grey"))
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs
environment = "dev"
if hasattr(args, "environment"):
environment = args.environment
settingObj = settings
appObj = appConfig
gitObj = gitObject
config_dir = settingObj.getConfigDir()
hooksObj.config_file = args.config_file
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
common_repo = config.get('repo', '')
data = appObj.getAppData(config_dir, args.config_file, args.app_name)
if not data:
raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format(
args.app_name, config_dir, args.config_file))
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', args.app_name)
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
if not os.path.exists(args.directory):
try:
os.makedirs(args.directory)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
if not hasattr(args, "app_name"):
args.app_name = ""
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
args.app_name = self.utils.extract_app_name(args.app_name)
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "pre_gitpull"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname))
# get/update target source(s)
repo_name = appObj.getRepoName(repo)
path = "{0}/{1}".format(args.directory, repo_name)
if os.path.isdir(path):
with chdir(path):
exit_code = gitObj.gitPull(branch, args.verbose)
else:
with chdir('{0}'.format(args.directory)):
exit_code = gitObj.gitShallowClone(repo, branch, args.verbose)
if exit_code != 0:
raise ValueError("Gitpull failed.")
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "post_gitpull"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname))
except (Exception) as e:
printException(e)
execution_result = 'FAILURE'
raise
finally:
try:
# If the gitpull fails before going through any steps
if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
if 'execution_result' not in globals() and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals() and 'config_name' not in locals():
config_name = ""
if 'environment' not in globals() and 'environment' not in locals():
environment = "dev"
if not hasattr(args, "app_name"):
args.app_name = ""
if 'settingObj' not in globals() and 'settingObj' not in locals():
settingObj = Settings()
if 'execution_result' is 'FAILURE':
self.outcome = 0
sc = self.utils.getStatsClient()
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=gitpull" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"ERROR - %s" %
e, file=sys.stderr)
raise
print(colored("******Completed the GIT PULL step successfully******", "green"))
def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj, args):
print(colored("******Building the Docker image now******", "grey"))
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
config = appObj.getConfig(config_dir, args.config_file)
hooksObj.config_file = args.config_file
roger_env = appObj.getRogerEnv(config_dir)
config_name = ""
if 'name' in config:
config_name = config['name']
common_repo = config.get('repo', '')
if not hasattr(args, "env"):
args.env = "dev"
data = appObj.getAppData(config_dir, args.config_file, args.app_name)
if not data:
raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format(
args.app_name, config_dir, args.config_file))
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', args.app_name)
build_args = {}
if 'build-args' in data:
if 'environment' in data['build-args']:
if args.env in data['build-args']['environment']:
build_args = data['build-args']['environment'][args.env]
projects = data.get('privateProjects', [])
# get/update target source(s)
file_exists = True
file_path = ''
cur_dir = ''
if "PWD" in os.environ:
cur_dir = os.environ.get('PWD')
# This is bad code, assuming current directory and then trying to again guess, this is not rocket science
# it's a f*****g file path, as simple as that. https://seomoz.atlassian.net/browse/ROGER-2405
# dockerfile location possibilities
# 1. Path relative to the repo, we know repo path for cli is <checkout_dir>/<repo>
# 2. Absolute path
# This path comes from config file and not passed on commandline so we should not try to prefix current
# working directory if the relative path is passed, don't try to guess too much.
# changelog : relative path from current directory won't work for working_directory or checkout_dir
# changelog : working_directory or checkout_dir should be absolute path, not backward-compatible
checkout_dir = os.path.abspath(args.directory)
repo_name = appObj.getRepoName(repo)
# (vmahedia) todo : this should be called docker_file_dir
dockerfile_rel_repo_path = data.get('path', '')
file_path = os.path.join(checkout_dir, repo_name, dockerfile_rel_repo_path)
if not hasattr(args, "app_name"):
args.app_name = ""
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
args.app_name = self.utils.extract_app_name(args.app_name)
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "pre_build"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"{} hook failed.".format(hookname))
build_filename = 'Dockerfile'
if 'build_filename' in data:
build_filename = ("{0}/{1}".format(file_path, data['build_filename']))
file_exists = os.path.exists(build_filename)
if not file_exists:
raise ValueError("Specified build file: {} does not exist. Exiting build.".format(build_filename))
else:
file_exists = os.path.exists("{0}/Dockerfile".format(file_path))
if file_exists:
# (vmahedia) todo: We know what parameters are required for build command so we should not wait until
# now to bailout. Config parser should have a validator for every command to see if all the Required
# parameters are passed or not. Why do all this up to this point if we know we will fail on this.
# RequiredParameter, below, "registry"
if 'registry' not in roger_env:
raise ValueError("Registry not found in roger-mesos-tools.config file.")
else:
self.registry = roger_env['registry']
self.tag_name = args.tag_name
image = "{0}/{1}".format(roger_env['registry'], args.tag_name)
try:
if checkout_dir == args.directory:
try:
dockerObj.docker_build(
dockerUtilsObj, appObj, args.directory, repo, projects, dockerfile_rel_repo_path, image, build_args, args.verbose, build_filename)
except ValueError:
raise ValueError("Docker build failed")
else:
directory = '{0}/{1}'.format(cur_dir, args.directory)
try:
dockerObj.docker_build(
dockerUtilsObj, appObj, directory, repo, projects, dockerfile_rel_repo_path, image, build_args, args.verbose, build_filename)
except ValueError:
print('Docker build failed.')
raise
print(colored("******Successfully built Docker image******", "green"))
build_message = "Image [{}]".format(image)
if(args.push):
print(colored("******Pushing Docker image to registry******", "grey"))
exit_code = dockerUtilsObj.docker_push(image, args.verbose)
if exit_code != 0:
raise ValueError(
'Docker push failed.')
build_message += " successfully pushed to registry [{}]*******".format(roger_env[
'registry'])
print(colored(build_message, "green"))
except (IOError) as e:
printException(e)
raise
else:
print(colored("Dockerfile does not exist in dir: {}".format(file_path), "red"))
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "post_build"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******""
if not hasattr(args, "env"):
args.env = "dev"
if not hasattr(args, "app_name"):
args.app_name = ""
if 'settingObj' not in globals() and 'settingObj' not in locals():
settingObj = Settings()
if 'execution_result' is 'FAILURE':
self.outcome = 0
sc = self.utils.getStatsClient()
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=build" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user=" + str(settingObj.getUser())
tup = (input_metric, time_take_milliseonds)
self.statsd_message_list.append(tup)
except (Exception) as e:
printException(e)
raise
def main(self, settingObject, appObject, frameworkUtilsObject, gitObj,
hooksObj, args):
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS'
settingObj = settingObject
appObj = appObject
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
roger_env = appObj.getRogerEnv(config_dir)
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
if 'registry' not in roger_env:
raise ValueError(
'Registry not found in roger-mesos-tools.config file.')
else:
self.registry = roger_env['registry']
# Setup for Slack-Client, token, and git user
# (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event
# Event should be registered and SlackNotification should be one of the members. it can have
# N notifications on a particular "event", Notifications.Notify will broadcast notification to
# all the interested parties.
if 'notifications' in config:
self.slack = Slack(
config['notifications'],
'/home/vagrant/.roger_cli.conf.d/slack_token')
self.identifier = self.utils.get_identifier(
config_name, settingObj.getUser(), args.application)
apps = []
apps_container_dict = {}
if args.application == 'all':
apps = config['apps'].keys()
else:
if ":" not in args.application and "[" not in args.application:
apps.append(args.application)
else:
for item in args.application.split(":"):
if '[' in item:
matchObj = re.match(r'(.*)\[(.*)\]', item)
apps.append(matchObj.group(1))
apps_container_dict[matchObj.group(
1)] = matchObj.group(2)
else:
apps.append(item)
common_repo = config.get('repo', '')
environment = roger_env.get('default_environment', '')
work_dir = ''
if args.directory:
work_dir = args.directory
temp_dir_created = False
if args.verbose:
print(
"Using {0} as the working directory".format(work_dir))
else:
work_dir = mkdtemp()
temp_dir_created = True
if args.verbose:
print("Created a temporary dir: {0}".format(work_dir))
if args.environment is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file"
)
else:
print(
"Using value {} from environment variable $ROGER_ENV"
.format(env_var))
environment = env_var
else:
environment = args.environment
if environment not in roger_env['environments']:
self.removeDirTree(work_dir, args, temp_dir_created)
raise ValueError(
'Environment not found in roger-mesos-tools.config file.')
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
try:
for app in apps:
if app not in config['apps']:
raise ValueError(
'Application {} specified not found.'.format(app))
else:
try:
if args.verbose:
print("Deploying {} ...".format(app))
self.deployApp(settingObject, appObject,
frameworkUtilsObject, gitObj,
hooksObj, root, args, config,
roger_env, work_dir, config_dir,
environment, app, branch,
self.slack, args.config_file,
common_repo, temp_dir_created,
apps_container_dict)
except (IOError, ValueError) as e:
error_msg = "Error when deploying {}: {}".format(
app, repr(e))
printErrorMsg(error_msg)
pass # try deploying the next app
except (Exception) as e:
printException(e)
raise
except (Exception) as e:
execution_result = 'FAILURE'
printException(e)
raise
finally:
# todo: maybe send the datadog event, need to look
pass
def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj,
args):
print(colored("******Building the Docker image now******", "grey"))
try:
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
config = appObj.getConfig(config_dir, args.config_file)
hooksObj.config_file = args.config_file
roger_env = appObj.getRogerEnv(config_dir)
config_name = ""
if 'name' in config:
config_name = config['name']
common_repo = config.get('repo', '')
if not hasattr(args, "env"):
args.env = "dev"
data = appObj.getAppData(config_dir, args.config_file,
args.app_name)
if not data:
raise ValueError(
"Application with name [{}] or data for it not found at {}/{}."
.format(args.app_name, config_dir, args.config_file))
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', args.app_name)
docker_build_args = {}
if 'build-args' in data:
if 'environment' in data['build-args']:
if args.env in data['build-args']['environment']:
docker_build_args = data['build-args']['environment'][
args.env]
# read the build-args from commandline like docker does as well
# build-args defined on command line will override the ones from the config file, for the same keys
# so this update of dictionary has to be done after we have read build arg values from the config file
if args.build_arg:
docker_build_args.update(
dict(
arg_key_val_str.split('=')
for arg_key_val_str in args.build_arg))
projects = data.get('privateProjects', [])
# get/update target source(s)
file_exists = True
file_path = ''
cur_dir = ''
if "PWD" in os.environ:
cur_dir = os.environ.get('PWD')
# This is bad code, assuming current directory and then trying to again guess, this is not rocket science
# it's a f*****g file path, as simple as that. https://seomoz.atlassian.net/browse/ROGER-2405
# dockerfile location possibilities
# 1. Path relative to the repo, we know repo path for cli is <checkout_dir>/<repo>
# 2. Absolute path
# This path comes from config file and not passed on commandline so we should not try to prefix current
# working directory if the relative path is passed, don't try to guess too much.
# changelog : relative path from current directory won't work for working_directory or checkout_dir
# changelog : working_directory or checkout_dir should be absolute path, not backward-compatible
checkout_dir = os.path.abspath(args.directory)
repo_name = appObj.getRepoName(repo)
# (vmahedia) todo : this should be called docker_file_dir
dockerfile_rel_repo_path = data.get('path', '')
file_path = os.path.join(checkout_dir, repo_name,
dockerfile_rel_repo_path)
if not hasattr(args, "app_name"):
args.app_name = ""
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(
config_name, settingObj.getUser(), args.app_name)
args.app_name = self.utils.extract_app_name(args.app_name)
hookname = "pre_build"
exit_code = hooksObj.run_hook(hookname, data, file_path, args.env,
settingObj.getUser())
if exit_code != 0:
raise ValueError("{} hook failed.".format(hookname))
build_filename = 'Dockerfile'
if 'build_filename' in data:
build_filename = ("{0}/{1}".format(file_path,
data['build_filename']))
file_exists = os.path.exists(build_filename)
if not file_exists:
raise ValueError(
"Specified build file: {} does not exist. Exiting build."
.format(build_filename))
else:
file_exists = os.path.exists(
"{0}/Dockerfile".format(file_path))
if file_exists:
# (vmahedia) todo: We know what parameters are required for build command so we should not wait until
# now to bailout. Config parser should have a validator for every command to see if all the Required
# parameters are passed or not. Why do all this up to this point if we know we will fail on this.
# RequiredParameter, below, "registry"
if 'registry' not in roger_env:
raise ValueError(
"Registry not found in roger-mesos-tools.config file.")
else:
self.registry = roger_env['registry']
self.tag_name = args.tag_name
image = "{0}/{1}".format(roger_env['registry'], args.tag_name)
try:
if checkout_dir == args.directory:
try:
dockerObj.docker_build(
dockerUtilsObj, appObj, args.directory, repo,
projects, dockerfile_rel_repo_path, image,
docker_build_args, args.verbose,
build_filename, args.disable_swaparoo)
except ValueError:
raise ValueError("Docker build failed")
else:
directory = os.path.join(cur_dir, args.directory)
try:
dockerObj.docker_build(
dockerUtilsObj, appObj, directory, repo,
projects, dockerfile_rel_repo_path, image,
docker_build_args, args.verbose,
build_filename, args.disable_swaparoo)
except ValueError:
print('Docker build failed.')
raise
print(
colored("******Successfully built Docker image******",
"green"))
build_message = "Image [{}]".format(image)
if (args.push):
print(
colored(
"******Pushing Docker image to registry******",
"grey"))
exit_code = dockerUtilsObj.docker_push(
image, args.verbose)
if exit_code != 0:
raise ValueError('Docker push failed.')
build_message += " successfully pushed to registry [{}]*******".format(
roger_env['registry'])
print(colored(build_message, "green"))
except (IOError) as e:
printException(e)
raise
else:
print(
colored(
"Dockerfile does not exist in dir: {}".format(
file_path), "red"))
hookname = "post_build"
exit_code = hooksObj.run_hook(hookname, data, file_path, args.env,
settingObj.getUser())
if exit_code != 0:
raise ValueError('{} hook failed.'.format(hookname))
except (Exception) as e:
printException(e)
raise
finally:
# todo: maybe send a datadog event?
pass
def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj, args):
print(colored("******Building the Docker image now******", "grey"))
try:
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
config = appObj.getConfig(config_dir, args.config_file)
hooksObj.config_file = args.config_file
roger_env = appObj.getRogerEnv(config_dir)
config_name = ""
if 'name' in config:
config_name = config['name']
common_repo = config.get('repo', '')
if not hasattr(args, "env"):
args.env = "dev"
data = appObj.getAppData(config_dir, args.config_file, args.app_name)
if not data:
raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format(
args.app_name, config_dir, args.config_file))
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', args.app_name)
docker_build_args = {}
if 'build-args' in data:
if 'environment' in data['build-args']:
if args.env in data['build-args']['environment']:
docker_build_args = data['build-args']['environment'][args.env]
# read the build-args from commandline like docker does as well
# build-args defined on command line will override the ones from the config file, for the same keys
# so this update of dictionary has to be done after we have read build arg values from the config file
if args.build_arg:
docker_build_args.update(dict(arg_key_val_str.split('=') for arg_key_val_str in args.build_arg))
projects = data.get('privateProjects', [])
# get/update target source(s)
file_exists = True
file_path = ''
cur_dir = ''
if "PWD" in os.environ:
cur_dir = os.environ.get('PWD')
# This is bad code, assuming current directory and then trying to again guess, this is not rocket science
# it's a f*****g file path, as simple as that. https://seomoz.atlassian.net/browse/ROGER-2405
# dockerfile location possibilities
# 1. Path relative to the repo, we know repo path for cli is <checkout_dir>/<repo>
# 2. Absolute path
# This path comes from config file and not passed on commandline so we should not try to prefix current
# working directory if the relative path is passed, don't try to guess too much.
# changelog : relative path from current directory won't work for working_directory or checkout_dir
# changelog : working_directory or checkout_dir should be absolute path, not backward-compatible
checkout_dir = os.path.abspath(args.directory)
repo_name = appObj.getRepoName(repo)
# (vmahedia) todo : this should be called docker_file_dir
dockerfile_rel_repo_path = data.get('path', '')
file_path = os.path.join(checkout_dir, repo_name, dockerfile_rel_repo_path)
if not hasattr(args, "app_name"):
args.app_name = ""
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
args.app_name = self.utils.extract_app_name(args.app_name)
hookname = "pre_build"
exit_code = hooksObj.run_hook(hookname, data, file_path, args.env, settingObj.getUser())
if exit_code != 0:
raise ValueError("{} hook failed.".format(hookname))
build_filename = 'Dockerfile'
if 'build_filename' in data:
build_filename = ("{0}/{1}".format(file_path, data['build_filename']))
file_exists = os.path.exists(build_filename)
if not file_exists:
raise ValueError("Specified build file: {} does not exist. Exiting build.".format(build_filename))
else:
file_exists = os.path.exists("{0}/Dockerfile".format(file_path))
if file_exists:
# (vmahedia) todo: We know what parameters are required for build command so we should not wait until
# now to bailout. Config parser should have a validator for every command to see if all the Required
# parameters are passed or not. Why do all this up to this point if we know we will fail on this.
# RequiredParameter, below, "registry"
if 'registry' not in roger_env:
raise ValueError("Registry not found in roger-mesos-tools.config file.")
else:
self.registry = roger_env['registry']
self.tag_name = args.tag_name
image = "{0}/{1}".format(roger_env['registry'], args.tag_name)
try:
if checkout_dir == args.directory:
try:
dockerObj.docker_build(
dockerUtilsObj, appObj, args.directory, repo, projects, dockerfile_rel_repo_path, image, docker_build_args, args.verbose, build_filename, args.disable_swaparoo)
except ValueError:
raise ValueError("Docker build failed")
else:
directory = os.path.join(cur_dir, args.directory)
try:
dockerObj.docker_build(
dockerUtilsObj, appObj, directory, repo, projects, dockerfile_rel_repo_path, image, docker_build_args, args.verbose, build_filename, args.disable_swaparoo)
except ValueError:
print('Docker build failed.')
raise
print(colored("******Successfully built Docker image******", "green"))
build_message = "Image [{}]".format(image)
if(args.push):
print(colored("******Pushing Docker image to registry******", "grey"))
exit_code = dockerUtilsObj.docker_push(image, args.verbose)
if exit_code != 0:
raise ValueError(
'Docker push failed.')
build_message += " successfully pushed to registry [{}]*******".format(roger_env[
'registry'])
print(colored(build_message, "green"))
except (IOError) as e:
printException(e)
raise
else:
print(colored("Dockerfile does not exist in dir: {}".format(file_path), "red"))
hookname = "post_build"
exit_code = hooksObj.run_hook(hookname, data, file_path, args.env, settingObj.getUser())
if exit_code != 0:
raise ValueError('{} hook failed.'.format(hookname))
except (Exception) as e:
printException(e)
raise
finally:
# todo: maybe send a datadog event?
pass
def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args):
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS'
settingObj = settingObject
appObj = appObject
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
roger_env = appObj.getRogerEnv(config_dir)
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
if 'registry' not in roger_env:
raise ValueError('Registry not found in roger-mesos-tools.config file.')
else:
self.registry = roger_env['registry']
# Setup for Slack-Client, token, and git user
# (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event
# Event should be registered and SlackNotification should be one of the members. it can have
# N notifications on a particular "event", Notifications.Notify will broadcast notification to
# all the interested parties.
if 'notifications' in config:
self.slack = Slack(config['notifications'],
'/home/vagrant/.roger_cli.conf.d/slack_token')
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application)
apps = []
apps_container_dict = {}
if args.application == 'all':
apps = config['apps'].keys()
else:
if ":" not in args.application and "[" not in args.application:
apps.append(args.application)
else:
for item in args.application.split(":"):
if '[' in item:
matchObj = re.match(r'(.*)\[(.*)\]', item)
apps.append(matchObj.group(1))
apps_container_dict[matchObj.group(1)] = matchObj.group(2)
else:
apps.append(item)
common_repo = config.get('repo', '')
environment = roger_env.get('default_environment', '')
work_dir = ''
if args.directory:
work_dir = args.directory
temp_dir_created = False
if args.verbose:
print("Using {0} as the working directory".format(work_dir))
else:
work_dir = mkdtemp()
temp_dir_created = True
if args.verbose:
print("Created a temporary dir: {0}".format(work_dir))
if args.environment is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file")
else:
print(
"Using value {} from environment variable $ROGER_ENV".format(env_var))
environment = env_var
else:
environment = args.environment
if environment not in roger_env['environments']:
self.removeDirTree(work_dir, args, temp_dir_created)
raise ValueError('Environment not found in roger-mesos-tools.config file.')
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
try:
for app in apps:
if app not in config['apps']:
raise ValueError('Application {} specified not found.'.format(app))
else:
try:
if args.verbose:
print("Deploying {} ...".format(app))
self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj,
root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict)
except (IOError, ValueError) as e:
error_msg = "Error when deploying {}: {}".format(app, repr(e))
printErrorMsg(error_msg)
pass # try deploying the next app
except (Exception) as e:
printException(e)
raise
except (Exception) as e:
execution_result = 'FAILURE'
printException(e)
raise
finally:
# todo: maybe send the datadog event, need to look
pass
def main(self):
self.parser = self.parse_args()
args = self.parser.parse_args()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
environment = roger_env.get('default_environment', '')
if args.env is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set.Using the default set from roger-mesos-tools.config file")
else:
print(
"Using value {} from environment variable $ROGER_ENV".format(env_var))
environment = env_var
else:
environment = args.env
if environment not in roger_env['environments']:
raise ValueError('Environment not found in roger-mesos-tools.config file.')
hostname = ''
containerId = ''
if args.hostname is None:
hostname = containerconfig.get_hostname_from_marathon(
environment, roger_env, args.appTaskId)
else:
hostname = args.hostname
if hostname != '': # Hostname maybe empty when the given appTaskId does not match any taskId from Marathon
(containerId, mesosTaskId) = containerconfig.get_containerid_mesostaskid(
args.appTaskId, hostname)
else:
print("Most likely hostname could not be retrieved with appTaskId {0}. Hostname is also \
an optional argument. See -h for usage.".format(args.appTaskId))
if containerId is not '' and containerId is not None:
print("If there are multiple containers that pattern match the given mesos task Id, \
then will log into the first one")
print("Displaying logs in docker container - {0} on host - {1} for mesosTask Id {2}".format(
containerId, hostname, mesosTaskId))
command = "docker -H tcp://{0}:4243 logs ".format(hostname)
if args.follow:
command = "{} -f=true".format(command)
else:
command = "{} -f=false".format(command)
if args.since:
command = "{} --since=\"{}\"".format(command, args.since)
if args.timestamps:
command = "{} -t".format(command, args.since)
if args.tail:
command = "{} --tail=\"{}\"".format(command, args.tail)
command = "{} {}".format(command, containerId)
try:
subprocess.check_call("{}".format(command), shell=True)
except (KeyboardInterrupt, SystemExit):
print("Exited.")
except (subprocess.CalledProcessError) as e:
printException(e)
else:
print("No Container found on host {0} with application Task Id {1}".format(hostname, args.appTaskId))
self.task_id.extend(container_task_id)
except (Exception) as e:
print("ERROR - : %s" %e, file=sys.stderr)
execution_result = 'FAILURE'
raise
finally:
# todo: maybe send datadog event from here?
pass
hookname = "post_push"
exit_code = hooksObj.run_hook(hookname, data, app_path, args.env, settingObj.getUser())
if exit_code != 0:
raise ValueError("{} hook failed.".format(hookname))
print(colored("******Done with the PUSH step******", "green"))
except (Exception) as e:
raise ValueError("ERROR - {}".format(e))
if __name__ == "__main__":
settingObj = Settings()
appObj = AppConfig()
frameworkUtils = FrameworkUtils()
hooksObj = Hooks()
roger_push = RogerPush()
try:
roger_push.parser = roger_push.parse_args()
roger_push.args = roger_push.parser.parse_args()
roger_push.main(settingObj, appObj, frameworkUtils, hooksObj, roger_push.args)
except (Exception) as e:
printException(e)
def main(self, settings, appConfig, gitObject, hooksObj, args):
print(
colored("******Executing GIT PULL of application repo******",
"grey"))
try:
function_execution_start_time = datetime.now()
environment = "dev"
if hasattr(args, "environment"):
environment = args.environment
settingObj = settings
appObj = appConfig
gitObj = gitObject
config_dir = settingObj.getConfigDir()
hooksObj.config_file = args.config_file
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
common_repo = config.get('repo', '')
data = appObj.getAppData(config_dir, args.config_file,
args.app_name)
if not data:
raise ValueError(
"Application with name [{}] or data for it not found at {}/{}."
.format(args.app_name, config_dir, args.config_file))
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', args.app_name)
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
if not os.path.exists(args.directory):
try:
os.makedirs(args.directory)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
hookname = "pre_gitpull"
exit_code = hooksObj.run_hook(hookname, data, args.directory,
environment, settingObj.getUser())
if exit_code != 0:
raise ValueError("{} hook failed.".format(hookname))
# get/update target source(s)
repo_name = appObj.getRepoName(repo)
path = "{0}/{1}".format(args.directory, repo_name)
if os.path.isdir(path):
with chdir(path):
exit_code = gitObj.gitPull(branch, args.verbose)
else:
with chdir('{0}'.format(args.directory)):
exit_code = gitObj.gitShallowClone(repo, branch,
args.verbose)
if exit_code != 0:
raise ValueError("Gitpull failed.")
hookname = "post_gitpull"
exit_code = hooksObj.run_hook(hookname, data, args.directory,
environment, settingObj.getUser())
if exit_code != 0:
raise ValueError("{} hook failed.".format(hookname))
except (Exception) as e:
printException(e)
raise
finally:
# todo: maybe send a datadog event?
pass
print(
colored("******Completed the GIT PULL step successfully******",
"green"))
