def custom_api_call(self, text, channel): """ Makes the webhook call Keyword arguments: text -- message to be posted channel -- to which channel posts a message if rogeros - bot is present """ try: self.webhookSetting() if len(channel) == 0: channel = self.defChannel var = self.sc.api_call("channels.list") length = len(var['channels']) for iterator in range(0, length): # getting rid of # for comparison if channel[1:] == var['channels'][iterator]['name']: if self.botid in var['channels'][iterator]['members']: if not self.disabled: self.client.notify(channel=channel, username=self.username, icon_emoji=self.emoji, text=text) except (Exception) as e: # notify to channel and log it as well printException(e) raise
def get_containerid_mesostaskid(self, appTaskId, hostname): containerId = '' mesosTaskId = '' try: containers = subprocess.check_output( "docker -H tcp://{}:4243 ps -q".format(hostname), shell=True) except: print("No route to host. Please check hostname '{}'".format( hostname), file=sys.stderr) return for container in containers.split('\n'): if container.strip() != '': try: mesosTaskId = subprocess.check_output( "docker -H tcp://{0}:4243 exec {1} \ printenv MESOS_TASK_ID".format(hostname, container), stderr=subprocess.STDOUT, shell=True) except Exception as e: if ("Cannot connect to the Docker daemon" in str(e.output)): printException(e) break else: # This is the case when a container does not have a # MESOS_TASK_ID in its ENV variables pass if mesosTaskId.startswith(appTaskId): containerId = container.strip() break return containerId, mesosTaskId
def get_containerid_mesostaskid(self, appTaskId, hostname): containerId = '' mesosTaskId = '' try: containers = subprocess.check_output( "docker -H tcp://{}:4243 ps -q".format(hostname), shell=True) except: print("No route to host. Please check hostname '{}'".format( hostname), file=sys.stderr) return for container in containers.split('\n'): if container.strip() != '': try: mesosTaskId = subprocess.check_output("docker -H tcp://{0}:4243 exec {1} \ printenv MESOS_TASK_ID".format(hostname, container), stderr=subprocess.STDOUT, shell=True) except Exception as e: if ("Cannot connect to the Docker daemon" in str(e.output)): printException(e) break else: # This is the case when a container does not have a # MESOS_TASK_ID in its ENV variables pass if mesosTaskId.startswith(appTaskId): containerId = container.strip() break return containerId, mesosTaskId
def run_hook(self, hookname, appdata, path, hook_input_metric): try: exit_code = 0 function_execution_start_time = datetime.now() execution_result = 'SUCCESS' self.whobj.invoke_webhook(appdata, hook_input_metric, self.config_file) abs_path = os.path.abspath(path) if "hooks" in appdata and hookname in appdata["hooks"]: command = appdata["hooks"][hookname] with chdir(abs_path): print("About to run {} hook [{}] at path {}".format( hookname, command, abs_path)) exit_code = os.system(command) except (Exception) as e: printException(e) execution_result = 'FAILURE' raise finally: try: if 'execution_result' not in globals() and 'execution_result' not in locals(): execution_result = 'FAILURE' if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals(): function_execution_start_time = datetime.now() sc = self.utils.getStatsClient() time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000) hook_input_metric = hook_input_metric + ",outcome=" + str(execution_result) tup = (hook_input_metric, time_take_milliseonds) self.statsd_message_list.append(tup) except (Exception) as e: printException(e) raise return exit_code
def configLevelSettings(self, config_file): """ Prepares all the config_level settings as variables Keyword arguments: config_file -- This is the file name passed as argument return three sets of channels, envs and commands. self variable as config level variable not expected to change for the run """ if (not self.configLoadFlag): self.config = self.appconfigObj.getConfig(self.config_dir, config_file) self.configLoadFlag = True if(not self.areBasicKeysAvailableInConfig(self.config)): return try: self.config_channels = Set(self.config['notifications']['channels']) if 'envs' in self.config['notifications'].keys(): self.config_envs = Set(self.config['notifications']['envs']) if 'commands' in self.config['notifications'].keys(): self.config_commands = Set(self.config['notifications']['commands']) if (len(self.config_channels) == 0 or len(self.config_envs) == 0 or len(self.config_commands) == 0): return except (Exception, KeyError, ValueError) as e: # notify to channel and log it as well printException(e) raise
def invoke_webhook(self, appdata, config_file, action, env, user): """ Pepares set and posts to slack channel Keyword arguments: appdata -- this is value related to an app hook_input_metric -- value it gets from hook class per app config_file -- the file name under for the app deployment """ envSet = [] commandsSet = [] self.function_execution_start_time = datetime.now() self.webhookSetting() self.configLevelSettings(config_file) self.action = action self.envr = env self.user = user self.app_name = appdata['name'] try: if(not self.areBasicKeysAvailableInAppdata(appdata)): return if 'notifications' in appdata: channelsSet = Set(appdata['notifications']['channels']) # to handle if tag is there but no data is present if (len(self.config_channels) != 0): channelsSet = channelsSet.union(self.config_channels) # to handle if tag is there if 'envs' in appdata['notifications'].keys(): envSet = Set(appdata['notifications']['envs']) # to handle if tag is there but data is not there if (len(self.config_envs) != 0): envSet = envSet.union(self.config_envs) if 'commands' in appdata['notifications'].keys(): commandsSet = Set(appdata['notifications']['commands']) if (len(self.config_commands) != 0): commandsSet = commandsSet.union(self.config_commands) if (len(channelsSet) == 0 or len(envSet) == 0 or len(commandsSet) == 0): return else: if list(self.config_envs)[0] == 'all': self.config_envs = ['dev', 'production', 'staging', 'local'] if list(self.config_commands)[0] == 'all': self.config_commands = ['pull', 'build', 'push'] self.postToSlack(self.action, self.config_envs, self.config_commands, self.config_channels) return if list(envSet)[0] == 'all': envSet = ['dev', 'production', 'staging', 'local'] if list(commandsSet)[0] == 'all': commandsSet = ['pull', 'build', 'push'] except (Exception, KeyError, ValueError) as e: printException(e) raise try: self.postToSlack(self.action, envSet, commandsSet, channelsSet) except (Exception) as e: self.custom_api_call("Error : %s" % e, self.defChannel) printException(e) raise
def docker_search(self, registry, name, application): result = "" try: result = self.docker_search_v2(registry) except (Exception) as e: error_message = "Error when attempting search using docker v2 catalog: {} ".format(e) printException(error_message) print(colored("Attempting docker v1 search", "yellow")) result = self.docker_search_v1(registry, name, application) return result
def main(self): self.parser = self.parse_args() args = self.parser.parse_args() config_dir = settingObj.getConfigDir() roger_env = appObj.getRogerEnv(config_dir) environment = roger_env.get('default_environment', '') if args.env is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print( "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file") else: if args.verbose: print(colored("Using value {} from environment variable $ROGER_ENV".format(env_var), "yellow")) environment = env_var else: environment = args.env if environment not in roger_env['environments']: raise ValueError(colored("Environment not found in roger-mesos-tools.config file.", "red")) hostname = '' containerId = '' if args.hostname is None: hostname = containerconfig.get_hostname_from_marathon( environment, roger_env, args.appTaskId) else: hostname = args.hostname if hostname != '': # Hostname maybe empty when the given appTaskId does not match any taskId from Marathon (containerId, mesosTaskId) = containerconfig.get_containerid_mesostaskid( args.appTaskId, hostname) else: if args.verbose: print(colored("Most likely hostname could not be retrieved with appTaskId {0}. Hostname is also \ an optional argument. See -h for usage.".format(args.appTaskId), "cyan")) if containerId is not '' and containerId is not None: if args.verbose: print(colored("INFO - If there are multiple containers that pattern match the given mesos task Id, \ then will log into the first one", "cyan")) print(colored("Executing bash in docker container - {0} on host - {1} for mesosTaskId - {2}".format( containerId, hostname, mesosTaskId), "yellow")) try: subprocess.check_call( "docker -H tcp://{0}:4243 exec -it {1} bash".format(hostname, containerId), shell=True) except Exception as e: printException(e) else: print(colored("No Container found on host {0} with application Task Id {1}".format( hostname, args.appTaskId), "red"))
def run_hook(self, hookname, appdata, path, env, user): try: exit_code = 0 execution_result = 'SUCCESS' self.whobj.invoke_webhook(appdata, self.config_file, hookname, env, user) abs_path = os.path.abspath(path) if "hooks" in appdata and hookname in appdata["hooks"]: command = appdata["hooks"][hookname] with chdir(abs_path): print("About to run {} hook [{}] at path {}".format( hookname, command, abs_path)) exit_code = os.system(command) except (Exception) as e: printException(e) execution_result = 'FAILURE' raise finally: # todo: maybe send a datadog event ? pass return exit_code
def postToSlack(self, action, envSet, commandsSet, channelsSet): """ Prepares post to slack channel Keyword arguments: action -- action extracted from hookname_input_metric envSet -- set of accepted environment commandsSet -- set of accepted commandssets channelsSet -- set of accepted channelsets """ try: if ('post' in action and self.envr in envSet and action.split('_')[1] in commandsSet): for channel in channelsSet: timeElapsed = '{0:.2f}'.format((datetime.now() - self.function_execution_start_time).total_seconds() * 1000) timeasInt = float(timeElapsed) h, m, s, ms = self.makeTimeReadable(timeasInt) readableTime = self.createMesage(h, m, s, ms) slackMessage = ("Completed *" + action.split('_')[1] + "* of *" + self.app_name + "* on *" + self.envr + "* in *" + readableTime + "* (triggered by *" + self.user + "*)") self.custom_api_call(slackMessage, '#' + channel) except (Exception) as e: self.custom_api_call("Error : %s" % e, self.defChannel) printException(e) raise
def main(self, settings, appConfig, frameworkObject, hooksObj, args): print(colored("******Deploying application to framework******", "grey")) try: validation_failed = False settingObj = settings appObj = appConfig frameworkUtils = frameworkObject config_dir = settingObj.getConfigDir() hooksObj.config_file = args.config_file cur_file_path = os.path.dirname(os.path.realpath(__file__)) config = appObj.getConfig(config_dir, args.config_file) config_name = "" act_as_user = "" if 'name' in config: config_name = config['name'] if 'act-as' in config: act_as_user = config['act-as'] roger_env = appObj.getRogerEnv(config_dir) if not hasattr(args, "app_name"): args.app_name = "" if 'registry' not in roger_env.keys(): raise ValueError("Registry not found in roger-mesos-tools.config file.") else: self.registry = roger_env['registry'] if hasattr(args, "image_name"): self.image_name = args.image_name environment = roger_env.get('default_environment', '') if args.env is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print(colored("WARNING - Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file", "yellow")) else: if args.verbose: print(colored("Using value {} from environment variable $ROGER_ENV".format(env_var), "grey")) environment = env_var else: environment = args.env # ---------------------------------------------- if environment not in roger_env['environments']: raise ValueError("Environment not found in roger-mesos-tools.config file.") # ---------------------------------------------- # GetEnvironmentConfig(environment) # ---------------------------------------------- environmentObj = roger_env['environments'][environment] common_repo = config.get('repo', '') # ---------------------------------------------- # GetContainersForApp(app) # ---------------------------------------------- app_name = args.app_name container_list = [] # todo (vmahedia): What does ':' signify? Put explanation. if ':' in app_name: tokens = app_name.split(':') app_name = tokens[0] # todo (vmahedia): it's container list - need to explain syntax if ',' in tokens[1]: container_list = tokens[1].split(',') else: container_list.append(tokens[1]) # ---------------------------------------------- data = appObj.getAppData(config_dir, args.config_file, app_name) if not data: raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format( app_name, config_dir, args.config_file)) configured_container_list = [] for task in data['containers']: if type(task) == dict: configured_container_list.append(task.keys()[0]) else: configured_container_list.append(task) if not set(container_list) <= set(configured_container_list): raise ValueError("List of containers [{}] passed do not match list of acceptable containers: [{}]".format( container_list, configured_container_list)) frameworkObj = frameworkUtils.getFramework(data) framework = frameworkObj.getName() repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', app_name) comp_dir = settingObj.getComponentsDir() templ_dir = settingObj.getTemplatesDir() secrets_dir = settingObj.getSecretsDir() # Create comp_dir if it doesn't exist if not os.path.isdir(comp_dir): os.makedirs(comp_dir) if not container_list: data_containers = data['containers'] else: data_containers = container_list failed_container_dict = {} # (vmahedia) upto this point it's all getting and checking the # configuration parameters template = '' # Required for when work_dir,component_dir,template_dir or # secret_env_dir is something like '.' or './temp" os.chdir(cur_file_path) app_path = '' if 'template_path' in data: app_path = self.repo_relative_path(appObj, args, repo, data['template_path']) else: app_path = templ_dir extra_vars = {} if 'extra_variables_path' in data: ev_path = self.repo_relative_path(appObj, args, repo, data['extra_variables_path']) with open(ev_path) as f: extra_vars = yaml.load(f) if ev_path.lower( ).endswith('.yml') else json.load(f) if not app_path.endswith('/'): app_path = app_path + '/' if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) args.app_name = self.utils.extract_app_name(args.app_name) hooksObj.statsd_message_list = self.statsd_message_list hookname = "pre_push" hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname)) # ---------------------------------------------- # (vmahedia) Figure out what the hell this loop does # and name it appropriately # it seems first part is just finding a template and Rendering # it against the given config, checking to see if there are errors # ---------------------------------------------- # (vmahedia) Meat starts from here, probably. for container in data_containers: container_name = self.getContainerName(container) containerConfig = "{0}-{1}.json".format(config['name'], container_name) env = Environment(loader = FileSystemLoader("{}".format(app_path)), undefined = StrictUndefined) template_with_path = "[{}{}]".format(app_path, containerConfig) try: template = env.get_template(containerConfig) except exceptions.TemplateNotFound as e: raise ValueError("The template file {} does not exist".format(template_with_path)) except Exception as e: raise ValueError("Error while reading template from {} - {}".format(template_with_path, e)) additional_vars = {} # (vmahedia)variables likes this should be at least visible within one # scroll up or down, move this code to near to context # Why are we getting the secrets everytime, this requires the file to be # present additional_vars.update(extra_vars) secret_vars = self.loadSecrets(secrets_dir, containerConfig, args, environment) additional_vars.update(secret_vars) image_path = "{0}/{1}".format( roger_env['registry'], args.image_name) print("Rendering content from template {} for environment [{}]".format( template_with_path, environment)) try: output = self.renderTemplate(template, environment, image_path, data, config, container, container_name, additional_vars) except exceptions.UndefinedError as e: error_str = "The following Undefined Jinja variable error occurred. %s.\n" % e print(colored(error_str, "red"), file=sys.stderr) failed_container_dict[container_name] = error_str # we are going to fail even if one of the container config is not valid but we will # still go through the loop and collect all the errors before we bail out validation_failed = True pass # ---------------------------------------------- # it seems the checks above can finish independent of the # following code, decouple this two parts, later when the code # is well understood # ---------------------------------------------- # Adding check to see if all jinja variables git resolved fot # the container if container_name not in failed_container_dict: # Adding check so that not all apps try to mergeSecrets try: outputObj = json.loads(output) except Exception as e: raise ValueError("Error while loading json from {} - {}".format(template_with_path, e)) if '\"SECRET\"' in output: output = self.mergeSecrets(output, self.loadSecrets( secrets_dir, containerConfig, args, environment)) if output != "StandardError": try: comp_exists = os.path.exists("{0}".format(comp_dir)) if comp_exists is False: os.makedirs("{0}".format(comp_dir)) comp_env_exists = os.path.exists( "{0}/{1}".format(comp_dir, environment)) if comp_env_exists is False: os.makedirs( "{0}/{1}".format(comp_dir, environment)) except Exception as e: logging.error(traceback.format_exc()) # (vmahedia) Should we write out the files even though there is an error with one of the # containers. Although maybe users would want to see some output with open("{0}/{1}/{2}".format(comp_dir, environment, containerConfig), 'wb') as fh: fh.write(output) else: raise ValueError("Error while loading secrets to render template file variables") # Notify container error messages # let failed_container_dict just be for now, but report all the errors if validation_failed: raise Exception("Unable to render Jinja template") deployment_check_failed = False # fail if the deployment check fails for container in data_containers: container_name = self.getContainerName(container) containerConfig = "{0}-{1}.json".format(config['name'], container_name) config_file_path = "{0}/{1}/{2}".format(comp_dir, environment, containerConfig) result = frameworkObj.runDeploymentChecks(config_file_path, environment) if not result: # need to give more indication about what can they do to fix this and what exactly failed # in the deployment check function, we should print an error in that function as well print(colored("Deployment checks failed for container - {}".format(framework, container)), "red") deployment_check_failed = True if deployment_check_failed: raise Exception("Deployment Check failed for one or more containers, check logs for more info!") if args.skip_push: print(colored("Skipping push to {} framework. The rendered config file(s) are under {}/{}/".format( framework, colored(comp_dir, "cyan"), colored(environment, "cyan")), "yellow")) else: # push to roger framework if 'owner' in config: frameworkObj.act_as_user = config['owner'] tools_version_value = self.utils.get_version() image_name = self.registry + "/" + args.image_name image_tag_value = urllib.quote("'" + image_name + "'") for container in data_containers: try: function_execution_start_time = datetime.now() # Assume SUCCESS unless exception execution_result = 'SUCCESS' sc = self.utils.getStatsClient() except (Exception) as e: raise ValueError("{} Error : {}".format(getDebugInfo(), e)) try: # this is where actual push is happening # we only push if forced, in case of failures # in deployment checks # # (vmahedia) todo: # list down scenarios in which this features # will be useful resp, task_id = frameworkObj.put(config_file_path, environmentObj, container_name, environment, act_as_user) container_task_id = self.utils.modify_task_id(task_id) self.task_id.extend(container_task_id) if hasattr(resp, "status_code"): status_code = resp.status_code except (Exception) as e: print("ERROR - : %s" % e, file=sys.stderr) execution_result = 'FAILURE' raise finally: try: if 'function_execution_start_time' not in globals() and \ 'function_execution_start_time' not in locals(): function_execution_start_time = datetime.now() if 'execution_result' not in globals() and \ 'execution_result' not in locals(): execution_result = 'FAILURE' if 'config_name' not in globals() and \ 'config_name' not in locals(): config_name = "" if 'environment' not in globals() and \ 'environment' not in locals(): environment = "dev" if 'container_name' not in globals() and \ 'container_name' not in locals(): container_name = "" if 'status_code' not in globals() and \ 'status_code' not in locals(): status_code = "500" if not hasattr(args, "app_name"): args.app_name = "" if 'settingObj' not in globals() and \ 'settingObj' not in locals(): settingObj = Settings() if 'container_task_id' not in globals() and 'container_task_id' not in locals(): container_task_id = [] if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) if not str(status_code).startswith("20"): execution_result = 'FAILURE' self.outcome = 0 time_taken = (datetime.now() - function_execution_start_time).total_seconds() for task_id in container_task_id: input_metric = "roger-tools.rogeros_tools_exec_time" + \ ",app_name=" + str(args.app_name) + \ ",event=push" + \ ",container_name=" + str(container_name) + \ ",identifier=" + str(self.identifier) + \ ",outcome=" + str(execution_result) + \ ",response_code=" + str(status_code) + \ ",config_name=" + str(config_name) + \ ",env=" + str(environment) + \ ",user="******"20"): metric = input_metric.replace("rogeros_tools_exec_time", "rogeros_events") metric = metric + ",source=tools" + ",task_id=" + task_id self.statsd_counter_logging(metric) except (Exception) as e: printException(e) raise hooksObj.statsd_message_list = self.statsd_message_list hookname = "post_push" hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + \ ",app_name=" + str(args.app_name) + \ ",identifier=" + str(self.identifier) + \ ",config_name=" + str(config_name) + \ ",env=" + str(environment) + \ ",user="******"{} hook failed.".format(hookname)) print(colored("******Done with the PUSH step******", "green")) except (Exception) as e: raise ValueError("ERROR - {}".format(e))
def main(self): self.parser = self.parse_args() args = self.parser.parse_args() config_dir = settingObj.getConfigDir() roger_env = appObj.getRogerEnv(config_dir) environment = roger_env.get('default_environment', '') if args.env is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print( "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file" ) else: if args.verbose: print( colored( "Using value {} from environment variable $ROGER_ENV" .format(env_var), "yellow")) environment = env_var else: environment = args.env if environment not in roger_env['environments']: raise ValueError( colored( "Environment not found in roger-mesos-tools.config file.", "red")) hostname = '' containerId = '' if args.hostname is None: hostname = containerconfig.get_hostname_from_marathon( environment, roger_env, args.appTaskId) else: hostname = args.hostname if hostname != '': # Hostname maybe empty when the given appTaskId does not match any taskId from Marathon (containerId, mesosTaskId) = containerconfig.get_containerid_mesostaskid( args.appTaskId, hostname) else: if args.verbose: print( colored( "Most likely hostname could not be retrieved with appTaskId {0}. Hostname is also \ an optional argument. See -h for usage.".format(args.appTaskId), "cyan")) if containerId is not '' and containerId is not None: if args.verbose: print( colored( "INFO - If there are multiple containers that pattern match the given mesos task Id, \ then will log into the first one", "cyan")) print( colored( "Executing bash in docker container - {0} on host - {1} for mesosTaskId - {2}" .format(containerId, hostname, mesosTaskId), "yellow")) try: subprocess.check_call( "docker -H tcp://{0}:4243 exec -it {1} bash".format( hostname, containerId), shell=True) except Exception as e: printException(e) else: print( colored( "No Container found on host {0} with application Task Id {1}" .format(hostname, args.appTaskId), "red"))
def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args): try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' settingObj = settingObject appObj = appObject config_dir = settingObj.getConfigDir() root = settingObj.getCliDir() roger_env = appObj.getRogerEnv(config_dir) config = appObj.getConfig(config_dir, args.config_file) config_name = "" if 'name' in config: config_name = config['name'] if 'registry' not in roger_env: raise ValueError('Registry not found in roger-mesos-tools.config file.') else: self.registry = roger_env['registry'] # Setup for Slack-Client, token, and git user # (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event # Event should be registered and SlackNotification should be one of the members. it can have # N notifications on a particular "event", Notifications.Notify will broadcast notification to # all the interested parties. if 'notifications' in config: self.slack = Slack(config['notifications'], '/home/vagrant/.roger_cli.conf.d/slack_token') self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application) apps = [] apps_container_dict = {} if args.application == 'all': apps = config['apps'].keys() else: if ":" not in args.application and "[" not in args.application: apps.append(args.application) else: for item in args.application.split(":"): if '[' in item: matchObj = re.match(r'(.*)\[(.*)\]', item) apps.append(matchObj.group(1)) apps_container_dict[matchObj.group(1)] = matchObj.group(2) else: apps.append(item) common_repo = config.get('repo', '') environment = roger_env.get('default_environment', '') work_dir = '' if args.directory: work_dir = args.directory temp_dir_created = False if args.verbose: print("Using {0} as the working directory".format(work_dir)) else: work_dir = mkdtemp() temp_dir_created = True if args.verbose: print("Created a temporary dir: {0}".format(work_dir)) if args.environment is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print( "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file") else: print( "Using value {} from environment variable $ROGER_ENV".format(env_var)) environment = env_var else: environment = args.environment if environment not in roger_env['environments']: self.removeDirTree(work_dir, args, temp_dir_created) raise ValueError('Environment not found in roger-mesos-tools.config file.') branch = "master" # master by default if args.branch is not None: branch = args.branch try: for app in apps: if app not in config['apps']: raise ValueError('Application {} specified not found.'.format(app)) else: try: if args.verbose: print("Deploying {} ...".format(app)) self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict) except (IOError, ValueError) as e: error_msg = "Error when deploying {}: {}".format(app, repr(e)) printErrorMsg(error_msg) pass # try deploying the next app except (Exception) as e: printException(e) raise except (Exception) as e: execution_result = 'FAILURE' printException(e) raise finally: # Check if the initializition of variables carried out if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals(): function_execution_start_time = datetime.now() if 'execution_result' not in globals() and 'execution_result' not in locals(): execution_result = 'FAILURE' if 'config_name' not in globals() and 'config_name' not in locals(): config_name = "" if 'environment' not in globals() and 'environment' not in locals(): environment = "dev" if not hasattr(args, "application"): args.application = "" if 'settingObj' not in globals() and 'settingObj' not in locals(): settingObj = Settings() if 'work_dir' not in globals() and 'work_dir' not in locals(): work_dir = '' temp_dir_created = False if not (self.rogerGitPullObject.outcome is 1 and self.rogerBuildObject.outcome is 1 and self.rogerPushObject.outcome is 1): execution_result = 'FAILURE' try: # If the deploy fails before going through any steps sc = self.utils.getStatsClient() if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application) args.application = self.utils.extract_app_name(args.application) time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000) input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.application) + ",event=deploy" + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******",identifier=" + str(self.identifier) tup = (input_metric, time_take_milliseonds) self.statsd_message_list.append(tup) self.removeDirTree(work_dir, args, temp_dir_created) except (Exception) as e: error_msg = "Error when deploying {}: {}".format(app, repr(e)) printErrorMsg(error_msg) raise
def main(self): self.parser = self.parse_args() args = self.parser.parse_args() config_dir = settingObj.getConfigDir() roger_env = appObj.getRogerEnv(config_dir) environment = roger_env.get('default_environment', '') if args.env is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print( "Environment variable $ROGER_ENV is not set.Using the default set from roger-mesos-tools.config file" ) else: print( "Using value {} from environment variable $ROGER_ENV". format(env_var)) environment = env_var else: environment = args.env if environment not in roger_env['environments']: raise ValueError( 'Environment not found in roger-mesos-tools.config file.') hostname = '' containerId = '' if args.hostname is None: hostname = containerconfig.get_hostname_from_marathon( environment, roger_env, args.appTaskId) else: hostname = args.hostname if hostname != '': # Hostname maybe empty when the given appTaskId does not match any taskId from Marathon (containerId, mesosTaskId) = containerconfig.get_containerid_mesostaskid( args.appTaskId, hostname) else: print( "Most likely hostname could not be retrieved with appTaskId {0}. Hostname is also \ an optional argument. See -h for usage.".format(args.appTaskId)) if containerId is not '' and containerId is not None: print( "If there are multiple containers that pattern match the given mesos task Id, \ then will log into the first one") print( "Displaying logs in docker container - {0} on host - {1} for mesosTask Id {2}" .format(containerId, hostname, mesosTaskId)) command = "docker -H tcp://{0}:4243 logs ".format(hostname) if args.follow: command = "{} -f=true".format(command) else: command = "{} -f=false".format(command) if args.since: command = "{} --since=\"{}\"".format(command, args.since) if args.timestamps: command = "{} -t".format(command, args.since) if args.tail: command = "{} --tail=\"{}\"".format(command, args.tail) command = "{} {}".format(command, containerId) try: subprocess.check_call("{}".format(command), shell=True) except (KeyboardInterrupt, SystemExit): print("Exited.") except (subprocess.CalledProcessError) as e: printException(e) else: print( "No Container found on host {0} with application Task Id {1}". format(hostname, args.appTaskId))
def main(self, settings, appConfig, gitObject, hooksObj, args): print(colored("******Executing GIT PULL of application repo******", "grey")) try: function_execution_start_time = datetime.now() environment = "dev" if hasattr(args, "environment"): environment = args.environment settingObj = settings appObj = appConfig gitObj = gitObject config_dir = settingObj.getConfigDir() hooksObj.config_file = args.config_file config = appObj.getConfig(config_dir, args.config_file) config_name = "" if 'name' in config: config_name = config['name'] common_repo = config.get('repo', '') data = appObj.getAppData(config_dir, args.config_file, args.app_name) if not data: raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format( args.app_name, config_dir, args.config_file)) repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', args.app_name) branch = "master" # master by default if args.branch is not None: branch = args.branch if not os.path.exists(args.directory): try: os.makedirs(args.directory) except OSError as exception: if exception.errno != errno.EEXIST: raise hookname = "pre_gitpull" exit_code = hooksObj.run_hook(hookname, data, args.directory, environment, settingObj.getUser()) if exit_code != 0: raise ValueError("{} hook failed.".format(hookname)) # get/update target source(s) repo_name = appObj.getRepoName(repo) path = "{0}/{1}".format(args.directory, repo_name) if os.path.isdir(path): with chdir(path): exit_code = gitObj.gitPull(branch, args.verbose) else: with chdir('{0}'.format(args.directory)): exit_code = gitObj.gitShallowClone(repo, branch, args.verbose) if exit_code != 0: raise ValueError("Gitpull failed.") hookname = "post_gitpull" exit_code = hooksObj.run_hook(hookname, data, args.directory, environment, settingObj.getUser()) if exit_code != 0: raise ValueError("{} hook failed.".format(hookname)) except (Exception) as e: printException(e) raise finally: # todo: maybe send a datadog event? pass print(colored("******Completed the GIT PULL step successfully******", "green"))
def main(self, settings, appConfig, gitObject, hooksObj, args): print(colored("******Executing GIT PULL of application repo******", "grey")) try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs environment = "dev" if hasattr(args, "environment"): environment = args.environment settingObj = settings appObj = appConfig gitObj = gitObject config_dir = settingObj.getConfigDir() hooksObj.config_file = args.config_file config = appObj.getConfig(config_dir, args.config_file) config_name = "" if 'name' in config: config_name = config['name'] common_repo = config.get('repo', '') data = appObj.getAppData(config_dir, args.config_file, args.app_name) if not data: raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format( args.app_name, config_dir, args.config_file)) repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', args.app_name) branch = "master" # master by default if args.branch is not None: branch = args.branch if not os.path.exists(args.directory): try: os.makedirs(args.directory) except OSError as exception: if exception.errno != errno.EEXIST: raise if not hasattr(args, "app_name"): args.app_name = "" if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) args.app_name = self.utils.extract_app_name(args.app_name) hooksObj.statsd_message_list = self.statsd_message_list hookname = "pre_gitpull" hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname)) # get/update target source(s) repo_name = appObj.getRepoName(repo) path = "{0}/{1}".format(args.directory, repo_name) if os.path.isdir(path): with chdir(path): exit_code = gitObj.gitPull(branch, args.verbose) else: with chdir('{0}'.format(args.directory)): exit_code = gitObj.gitShallowClone(repo, branch, args.verbose) if exit_code != 0: raise ValueError("Gitpull failed.") hooksObj.statsd_message_list = self.statsd_message_list hookname = "post_gitpull" hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname)) except (Exception) as e: printException(e) execution_result = 'FAILURE' raise finally: try: # If the gitpull fails before going through any steps if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals(): function_execution_start_time = datetime.now() if 'execution_result' not in globals() and 'execution_result' not in locals(): execution_result = 'FAILURE' if 'config_name' not in globals() and 'config_name' not in locals(): config_name = "" if 'environment' not in globals() and 'environment' not in locals(): environment = "dev" if not hasattr(args, "app_name"): args.app_name = "" if 'settingObj' not in globals() and 'settingObj' not in locals(): settingObj = Settings() if 'execution_result' is 'FAILURE': self.outcome = 0 sc = self.utils.getStatsClient() if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000) input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=gitpull" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"ERROR - %s" % e, file=sys.stderr) raise print(colored("******Completed the GIT PULL step successfully******", "green"))
def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj, args): print(colored("******Building the Docker image now******", "grey")) try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs config_dir = settingObj.getConfigDir() root = settingObj.getCliDir() config = appObj.getConfig(config_dir, args.config_file) hooksObj.config_file = args.config_file roger_env = appObj.getRogerEnv(config_dir) config_name = "" if 'name' in config: config_name = config['name'] common_repo = config.get('repo', '') if not hasattr(args, "env"): args.env = "dev" data = appObj.getAppData(config_dir, args.config_file, args.app_name) if not data: raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format( args.app_name, config_dir, args.config_file)) repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', args.app_name) build_args = {} if 'build-args' in data: if 'environment' in data['build-args']: if args.env in data['build-args']['environment']: build_args = data['build-args']['environment'][args.env] projects = data.get('privateProjects', []) # get/update target source(s) file_exists = True file_path = '' cur_dir = '' if "PWD" in os.environ: cur_dir = os.environ.get('PWD') # This is bad code, assuming current directory and then trying to again guess, this is not rocket science # it's a f*****g file path, as simple as that. https://seomoz.atlassian.net/browse/ROGER-2405 # dockerfile location possibilities # 1. Path relative to the repo, we know repo path for cli is <checkout_dir>/<repo> # 2. Absolute path # This path comes from config file and not passed on commandline so we should not try to prefix current # working directory if the relative path is passed, don't try to guess too much. # changelog : relative path from current directory won't work for working_directory or checkout_dir # changelog : working_directory or checkout_dir should be absolute path, not backward-compatible checkout_dir = os.path.abspath(args.directory) repo_name = appObj.getRepoName(repo) # (vmahedia) todo : this should be called docker_file_dir dockerfile_rel_repo_path = data.get('path', '') file_path = os.path.join(checkout_dir, repo_name, dockerfile_rel_repo_path) if not hasattr(args, "app_name"): args.app_name = "" if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) args.app_name = self.utils.extract_app_name(args.app_name) hooksObj.statsd_message_list = self.statsd_message_list hookname = "pre_build" hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"{} hook failed.".format(hookname)) build_filename = 'Dockerfile' if 'build_filename' in data: build_filename = ("{0}/{1}".format(file_path, data['build_filename'])) file_exists = os.path.exists(build_filename) if not file_exists: raise ValueError("Specified build file: {} does not exist. Exiting build.".format(build_filename)) else: file_exists = os.path.exists("{0}/Dockerfile".format(file_path)) if file_exists: # (vmahedia) todo: We know what parameters are required for build command so we should not wait until # now to bailout. Config parser should have a validator for every command to see if all the Required # parameters are passed or not. Why do all this up to this point if we know we will fail on this. # RequiredParameter, below, "registry" if 'registry' not in roger_env: raise ValueError("Registry not found in roger-mesos-tools.config file.") else: self.registry = roger_env['registry'] self.tag_name = args.tag_name image = "{0}/{1}".format(roger_env['registry'], args.tag_name) try: if checkout_dir == args.directory: try: dockerObj.docker_build( dockerUtilsObj, appObj, args.directory, repo, projects, dockerfile_rel_repo_path, image, build_args, args.verbose, build_filename) except ValueError: raise ValueError("Docker build failed") else: directory = '{0}/{1}'.format(cur_dir, args.directory) try: dockerObj.docker_build( dockerUtilsObj, appObj, directory, repo, projects, dockerfile_rel_repo_path, image, build_args, args.verbose, build_filename) except ValueError: print('Docker build failed.') raise print(colored("******Successfully built Docker image******", "green")) build_message = "Image [{}]".format(image) if(args.push): print(colored("******Pushing Docker image to registry******", "grey")) exit_code = dockerUtilsObj.docker_push(image, args.verbose) if exit_code != 0: raise ValueError( 'Docker push failed.') build_message += " successfully pushed to registry [{}]*******".format(roger_env[ 'registry']) print(colored(build_message, "green")) except (IOError) as e: printException(e) raise else: print(colored("Dockerfile does not exist in dir: {}".format(file_path), "red")) hooksObj.statsd_message_list = self.statsd_message_list hookname = "post_build" hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"" if not hasattr(args, "env"): args.env = "dev" if not hasattr(args, "app_name"): args.app_name = "" if 'settingObj' not in globals() and 'settingObj' not in locals(): settingObj = Settings() if 'execution_result' is 'FAILURE': self.outcome = 0 sc = self.utils.getStatsClient() if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000) input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=build" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user=" + str(settingObj.getUser()) tup = (input_metric, time_take_milliseonds) self.statsd_message_list.append(tup) except (Exception) as e: printException(e) raise
def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args): try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' settingObj = settingObject appObj = appObject config_dir = settingObj.getConfigDir() root = settingObj.getCliDir() roger_env = appObj.getRogerEnv(config_dir) config = appObj.getConfig(config_dir, args.config_file) config_name = "" if 'name' in config: config_name = config['name'] if 'registry' not in roger_env: raise ValueError( 'Registry not found in roger-mesos-tools.config file.') else: self.registry = roger_env['registry'] # Setup for Slack-Client, token, and git user # (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event # Event should be registered and SlackNotification should be one of the members. it can have # N notifications on a particular "event", Notifications.Notify will broadcast notification to # all the interested parties. if 'notifications' in config: self.slack = Slack( config['notifications'], '/home/vagrant/.roger_cli.conf.d/slack_token') self.identifier = self.utils.get_identifier( config_name, settingObj.getUser(), args.application) apps = [] apps_container_dict = {} if args.application == 'all': apps = config['apps'].keys() else: if ":" not in args.application and "[" not in args.application: apps.append(args.application) else: for item in args.application.split(":"): if '[' in item: matchObj = re.match(r'(.*)\[(.*)\]', item) apps.append(matchObj.group(1)) apps_container_dict[matchObj.group( 1)] = matchObj.group(2) else: apps.append(item) common_repo = config.get('repo', '') environment = roger_env.get('default_environment', '') work_dir = '' if args.directory: work_dir = args.directory temp_dir_created = False if args.verbose: print( "Using {0} as the working directory".format(work_dir)) else: work_dir = mkdtemp() temp_dir_created = True if args.verbose: print("Created a temporary dir: {0}".format(work_dir)) if args.environment is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print( "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file" ) else: print( "Using value {} from environment variable $ROGER_ENV" .format(env_var)) environment = env_var else: environment = args.environment if environment not in roger_env['environments']: self.removeDirTree(work_dir, args, temp_dir_created) raise ValueError( 'Environment not found in roger-mesos-tools.config file.') branch = "master" # master by default if args.branch is not None: branch = args.branch try: for app in apps: if app not in config['apps']: raise ValueError( 'Application {} specified not found.'.format(app)) else: try: if args.verbose: print("Deploying {} ...".format(app)) self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict) except (IOError, ValueError) as e: error_msg = "Error when deploying {}: {}".format( app, repr(e)) printErrorMsg(error_msg) pass # try deploying the next app except (Exception) as e: printException(e) raise except (Exception) as e: execution_result = 'FAILURE' printException(e) raise finally: # todo: maybe send the datadog event, need to look pass
def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj, args): print(colored("******Building the Docker image now******", "grey")) try: config_dir = settingObj.getConfigDir() root = settingObj.getCliDir() config = appObj.getConfig(config_dir, args.config_file) hooksObj.config_file = args.config_file roger_env = appObj.getRogerEnv(config_dir) config_name = "" if 'name' in config: config_name = config['name'] common_repo = config.get('repo', '') if not hasattr(args, "env"): args.env = "dev" data = appObj.getAppData(config_dir, args.config_file, args.app_name) if not data: raise ValueError( "Application with name [{}] or data for it not found at {}/{}." .format(args.app_name, config_dir, args.config_file)) repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', args.app_name) docker_build_args = {} if 'build-args' in data: if 'environment' in data['build-args']: if args.env in data['build-args']['environment']: docker_build_args = data['build-args']['environment'][ args.env] # read the build-args from commandline like docker does as well # build-args defined on command line will override the ones from the config file, for the same keys # so this update of dictionary has to be done after we have read build arg values from the config file if args.build_arg: docker_build_args.update( dict( arg_key_val_str.split('=') for arg_key_val_str in args.build_arg)) projects = data.get('privateProjects', []) # get/update target source(s) file_exists = True file_path = '' cur_dir = '' if "PWD" in os.environ: cur_dir = os.environ.get('PWD') # This is bad code, assuming current directory and then trying to again guess, this is not rocket science # it's a f*****g file path, as simple as that. https://seomoz.atlassian.net/browse/ROGER-2405 # dockerfile location possibilities # 1. Path relative to the repo, we know repo path for cli is <checkout_dir>/<repo> # 2. Absolute path # This path comes from config file and not passed on commandline so we should not try to prefix current # working directory if the relative path is passed, don't try to guess too much. # changelog : relative path from current directory won't work for working_directory or checkout_dir # changelog : working_directory or checkout_dir should be absolute path, not backward-compatible checkout_dir = os.path.abspath(args.directory) repo_name = appObj.getRepoName(repo) # (vmahedia) todo : this should be called docker_file_dir dockerfile_rel_repo_path = data.get('path', '') file_path = os.path.join(checkout_dir, repo_name, dockerfile_rel_repo_path) if not hasattr(args, "app_name"): args.app_name = "" if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier( config_name, settingObj.getUser(), args.app_name) args.app_name = self.utils.extract_app_name(args.app_name) hookname = "pre_build" exit_code = hooksObj.run_hook(hookname, data, file_path, args.env, settingObj.getUser()) if exit_code != 0: raise ValueError("{} hook failed.".format(hookname)) build_filename = 'Dockerfile' if 'build_filename' in data: build_filename = ("{0}/{1}".format(file_path, data['build_filename'])) file_exists = os.path.exists(build_filename) if not file_exists: raise ValueError( "Specified build file: {} does not exist. Exiting build." .format(build_filename)) else: file_exists = os.path.exists( "{0}/Dockerfile".format(file_path)) if file_exists: # (vmahedia) todo: We know what parameters are required for build command so we should not wait until # now to bailout. Config parser should have a validator for every command to see if all the Required # parameters are passed or not. Why do all this up to this point if we know we will fail on this. # RequiredParameter, below, "registry" if 'registry' not in roger_env: raise ValueError( "Registry not found in roger-mesos-tools.config file.") else: self.registry = roger_env['registry'] self.tag_name = args.tag_name image = "{0}/{1}".format(roger_env['registry'], args.tag_name) try: if checkout_dir == args.directory: try: dockerObj.docker_build( dockerUtilsObj, appObj, args.directory, repo, projects, dockerfile_rel_repo_path, image, docker_build_args, args.verbose, build_filename, args.disable_swaparoo) except ValueError: raise ValueError("Docker build failed") else: directory = os.path.join(cur_dir, args.directory) try: dockerObj.docker_build( dockerUtilsObj, appObj, directory, repo, projects, dockerfile_rel_repo_path, image, docker_build_args, args.verbose, build_filename, args.disable_swaparoo) except ValueError: print('Docker build failed.') raise print( colored("******Successfully built Docker image******", "green")) build_message = "Image [{}]".format(image) if (args.push): print( colored( "******Pushing Docker image to registry******", "grey")) exit_code = dockerUtilsObj.docker_push( image, args.verbose) if exit_code != 0: raise ValueError('Docker push failed.') build_message += " successfully pushed to registry [{}]*******".format( roger_env['registry']) print(colored(build_message, "green")) except (IOError) as e: printException(e) raise else: print( colored( "Dockerfile does not exist in dir: {}".format( file_path), "red")) hookname = "post_build" exit_code = hooksObj.run_hook(hookname, data, file_path, args.env, settingObj.getUser()) if exit_code != 0: raise ValueError('{} hook failed.'.format(hookname)) except (Exception) as e: printException(e) raise finally: # todo: maybe send a datadog event? pass
def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj, args): print(colored("******Building the Docker image now******", "grey")) try: config_dir = settingObj.getConfigDir() root = settingObj.getCliDir() config = appObj.getConfig(config_dir, args.config_file) hooksObj.config_file = args.config_file roger_env = appObj.getRogerEnv(config_dir) config_name = "" if 'name' in config: config_name = config['name'] common_repo = config.get('repo', '') if not hasattr(args, "env"): args.env = "dev" data = appObj.getAppData(config_dir, args.config_file, args.app_name) if not data: raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format( args.app_name, config_dir, args.config_file)) repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', args.app_name) docker_build_args = {} if 'build-args' in data: if 'environment' in data['build-args']: if args.env in data['build-args']['environment']: docker_build_args = data['build-args']['environment'][args.env] # read the build-args from commandline like docker does as well # build-args defined on command line will override the ones from the config file, for the same keys # so this update of dictionary has to be done after we have read build arg values from the config file if args.build_arg: docker_build_args.update(dict(arg_key_val_str.split('=') for arg_key_val_str in args.build_arg)) projects = data.get('privateProjects', []) # get/update target source(s) file_exists = True file_path = '' cur_dir = '' if "PWD" in os.environ: cur_dir = os.environ.get('PWD') # This is bad code, assuming current directory and then trying to again guess, this is not rocket science # it's a f*****g file path, as simple as that. https://seomoz.atlassian.net/browse/ROGER-2405 # dockerfile location possibilities # 1. Path relative to the repo, we know repo path for cli is <checkout_dir>/<repo> # 2. Absolute path # This path comes from config file and not passed on commandline so we should not try to prefix current # working directory if the relative path is passed, don't try to guess too much. # changelog : relative path from current directory won't work for working_directory or checkout_dir # changelog : working_directory or checkout_dir should be absolute path, not backward-compatible checkout_dir = os.path.abspath(args.directory) repo_name = appObj.getRepoName(repo) # (vmahedia) todo : this should be called docker_file_dir dockerfile_rel_repo_path = data.get('path', '') file_path = os.path.join(checkout_dir, repo_name, dockerfile_rel_repo_path) if not hasattr(args, "app_name"): args.app_name = "" if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) args.app_name = self.utils.extract_app_name(args.app_name) hookname = "pre_build" exit_code = hooksObj.run_hook(hookname, data, file_path, args.env, settingObj.getUser()) if exit_code != 0: raise ValueError("{} hook failed.".format(hookname)) build_filename = 'Dockerfile' if 'build_filename' in data: build_filename = ("{0}/{1}".format(file_path, data['build_filename'])) file_exists = os.path.exists(build_filename) if not file_exists: raise ValueError("Specified build file: {} does not exist. Exiting build.".format(build_filename)) else: file_exists = os.path.exists("{0}/Dockerfile".format(file_path)) if file_exists: # (vmahedia) todo: We know what parameters are required for build command so we should not wait until # now to bailout. Config parser should have a validator for every command to see if all the Required # parameters are passed or not. Why do all this up to this point if we know we will fail on this. # RequiredParameter, below, "registry" if 'registry' not in roger_env: raise ValueError("Registry not found in roger-mesos-tools.config file.") else: self.registry = roger_env['registry'] self.tag_name = args.tag_name image = "{0}/{1}".format(roger_env['registry'], args.tag_name) try: if checkout_dir == args.directory: try: dockerObj.docker_build( dockerUtilsObj, appObj, args.directory, repo, projects, dockerfile_rel_repo_path, image, docker_build_args, args.verbose, build_filename, args.disable_swaparoo) except ValueError: raise ValueError("Docker build failed") else: directory = os.path.join(cur_dir, args.directory) try: dockerObj.docker_build( dockerUtilsObj, appObj, directory, repo, projects, dockerfile_rel_repo_path, image, docker_build_args, args.verbose, build_filename, args.disable_swaparoo) except ValueError: print('Docker build failed.') raise print(colored("******Successfully built Docker image******", "green")) build_message = "Image [{}]".format(image) if(args.push): print(colored("******Pushing Docker image to registry******", "grey")) exit_code = dockerUtilsObj.docker_push(image, args.verbose) if exit_code != 0: raise ValueError( 'Docker push failed.') build_message += " successfully pushed to registry [{}]*******".format(roger_env[ 'registry']) print(colored(build_message, "green")) except (IOError) as e: printException(e) raise else: print(colored("Dockerfile does not exist in dir: {}".format(file_path), "red")) hookname = "post_build" exit_code = hooksObj.run_hook(hookname, data, file_path, args.env, settingObj.getUser()) if exit_code != 0: raise ValueError('{} hook failed.'.format(hookname)) except (Exception) as e: printException(e) raise finally: # todo: maybe send a datadog event? pass
def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args): try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' settingObj = settingObject appObj = appObject config_dir = settingObj.getConfigDir() root = settingObj.getCliDir() roger_env = appObj.getRogerEnv(config_dir) config = appObj.getConfig(config_dir, args.config_file) config_name = "" if 'name' in config: config_name = config['name'] if 'registry' not in roger_env: raise ValueError('Registry not found in roger-mesos-tools.config file.') else: self.registry = roger_env['registry'] # Setup for Slack-Client, token, and git user # (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event # Event should be registered and SlackNotification should be one of the members. it can have # N notifications on a particular "event", Notifications.Notify will broadcast notification to # all the interested parties. if 'notifications' in config: self.slack = Slack(config['notifications'], '/home/vagrant/.roger_cli.conf.d/slack_token') self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application) apps = [] apps_container_dict = {} if args.application == 'all': apps = config['apps'].keys() else: if ":" not in args.application and "[" not in args.application: apps.append(args.application) else: for item in args.application.split(":"): if '[' in item: matchObj = re.match(r'(.*)\[(.*)\]', item) apps.append(matchObj.group(1)) apps_container_dict[matchObj.group(1)] = matchObj.group(2) else: apps.append(item) common_repo = config.get('repo', '') environment = roger_env.get('default_environment', '') work_dir = '' if args.directory: work_dir = args.directory temp_dir_created = False if args.verbose: print("Using {0} as the working directory".format(work_dir)) else: work_dir = mkdtemp() temp_dir_created = True if args.verbose: print("Created a temporary dir: {0}".format(work_dir)) if args.environment is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print( "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file") else: print( "Using value {} from environment variable $ROGER_ENV".format(env_var)) environment = env_var else: environment = args.environment if environment not in roger_env['environments']: self.removeDirTree(work_dir, args, temp_dir_created) raise ValueError('Environment not found in roger-mesos-tools.config file.') branch = "master" # master by default if args.branch is not None: branch = args.branch try: for app in apps: if app not in config['apps']: raise ValueError('Application {} specified not found.'.format(app)) else: try: if args.verbose: print("Deploying {} ...".format(app)) self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict) except (IOError, ValueError) as e: error_msg = "Error when deploying {}: {}".format(app, repr(e)) printErrorMsg(error_msg) pass # try deploying the next app except (Exception) as e: printException(e) raise except (Exception) as e: execution_result = 'FAILURE' printException(e) raise finally: # todo: maybe send the datadog event, need to look pass
def main(self): self.parser = self.parse_args() args = self.parser.parse_args() config_dir = settingObj.getConfigDir() roger_env = appObj.getRogerEnv(config_dir) environment = roger_env.get('default_environment', '') if args.env is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print( "Environment variable $ROGER_ENV is not set.Using the default set from roger-mesos-tools.config file") else: print( "Using value {} from environment variable $ROGER_ENV".format(env_var)) environment = env_var else: environment = args.env if environment not in roger_env['environments']: raise ValueError('Environment not found in roger-mesos-tools.config file.') hostname = '' containerId = '' if args.hostname is None: hostname = containerconfig.get_hostname_from_marathon( environment, roger_env, args.appTaskId) else: hostname = args.hostname if hostname != '': # Hostname maybe empty when the given appTaskId does not match any taskId from Marathon (containerId, mesosTaskId) = containerconfig.get_containerid_mesostaskid( args.appTaskId, hostname) else: print("Most likely hostname could not be retrieved with appTaskId {0}. Hostname is also \ an optional argument. See -h for usage.".format(args.appTaskId)) if containerId is not '' and containerId is not None: print("If there are multiple containers that pattern match the given mesos task Id, \ then will log into the first one") print("Displaying logs in docker container - {0} on host - {1} for mesosTask Id {2}".format( containerId, hostname, mesosTaskId)) command = "docker -H tcp://{0}:4243 logs ".format(hostname) if args.follow: command = "{} -f=true".format(command) else: command = "{} -f=false".format(command) if args.since: command = "{} --since=\"{}\"".format(command, args.since) if args.timestamps: command = "{} -t".format(command, args.since) if args.tail: command = "{} --tail=\"{}\"".format(command, args.tail) command = "{} {}".format(command, containerId) try: subprocess.check_call("{}".format(command), shell=True) except (KeyboardInterrupt, SystemExit): print("Exited.") except (subprocess.CalledProcessError) as e: printException(e) else: print("No Container found on host {0} with application Task Id {1}".format(hostname, args.appTaskId))
self.task_id.extend(container_task_id) except (Exception) as e: print("ERROR - : %s" %e, file=sys.stderr) execution_result = 'FAILURE' raise finally: # todo: maybe send datadog event from here? pass hookname = "post_push" exit_code = hooksObj.run_hook(hookname, data, app_path, args.env, settingObj.getUser()) if exit_code != 0: raise ValueError("{} hook failed.".format(hookname)) print(colored("******Done with the PUSH step******", "green")) except (Exception) as e: raise ValueError("ERROR - {}".format(e)) if __name__ == "__main__": settingObj = Settings() appObj = AppConfig() frameworkUtils = FrameworkUtils() hooksObj = Hooks() roger_push = RogerPush() try: roger_push.parser = roger_push.parse_args() roger_push.args = roger_push.parser.parse_args() roger_push.main(settingObj, appObj, frameworkUtils, hooksObj, roger_push.args) except (Exception) as e: printException(e)
def main(self, settings, appConfig, gitObject, hooksObj, args): print( colored("******Executing GIT PULL of application repo******", "grey")) try: function_execution_start_time = datetime.now() environment = "dev" if hasattr(args, "environment"): environment = args.environment settingObj = settings appObj = appConfig gitObj = gitObject config_dir = settingObj.getConfigDir() hooksObj.config_file = args.config_file config = appObj.getConfig(config_dir, args.config_file) config_name = "" if 'name' in config: config_name = config['name'] common_repo = config.get('repo', '') data = appObj.getAppData(config_dir, args.config_file, args.app_name) if not data: raise ValueError( "Application with name [{}] or data for it not found at {}/{}." .format(args.app_name, config_dir, args.config_file)) repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', args.app_name) branch = "master" # master by default if args.branch is not None: branch = args.branch if not os.path.exists(args.directory): try: os.makedirs(args.directory) except OSError as exception: if exception.errno != errno.EEXIST: raise hookname = "pre_gitpull" exit_code = hooksObj.run_hook(hookname, data, args.directory, environment, settingObj.getUser()) if exit_code != 0: raise ValueError("{} hook failed.".format(hookname)) # get/update target source(s) repo_name = appObj.getRepoName(repo) path = "{0}/{1}".format(args.directory, repo_name) if os.path.isdir(path): with chdir(path): exit_code = gitObj.gitPull(branch, args.verbose) else: with chdir('{0}'.format(args.directory)): exit_code = gitObj.gitShallowClone(repo, branch, args.verbose) if exit_code != 0: raise ValueError("Gitpull failed.") hookname = "post_gitpull" exit_code = hooksObj.run_hook(hookname, data, args.directory, environment, settingObj.getUser()) if exit_code != 0: raise ValueError("{} hook failed.".format(hookname)) except (Exception) as e: printException(e) raise finally: # todo: maybe send a datadog event? pass print( colored("******Completed the GIT PULL step successfully******", "green"))