def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args): try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' settingObj = settingObject appObj = appObject config_dir = settingObj.getConfigDir() root = settingObj.getCliDir() roger_env = appObj.getRogerEnv(config_dir) config = appObj.getConfig(config_dir, args.config_file) config_name = "" if 'name' in config: config_name = config['name'] if 'registry' not in roger_env: raise ValueError('Registry not found in roger-mesos-tools.config file.') else: self.registry = roger_env['registry'] # Setup for Slack-Client, token, and git user # (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event # Event should be registered and SlackNotification should be one of the members. it can have # N notifications on a particular "event", Notifications.Notify will broadcast notification to # all the interested parties. if 'notifications' in config: self.slack = Slack(config['notifications'], '/home/vagrant/.roger_cli.conf.d/slack_token') self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application) apps = [] apps_container_dict = {} if args.application == 'all': apps = config['apps'].keys() else: if ":" not in args.application and "[" not in args.application: apps.append(args.application) else: for item in args.application.split(":"): if '[' in item: matchObj = re.match(r'(.*)\[(.*)\]', item) apps.append(matchObj.group(1)) apps_container_dict[matchObj.group(1)] = matchObj.group(2) else: apps.append(item) common_repo = config.get('repo', '') environment = roger_env.get('default_environment', '') work_dir = '' if args.directory: work_dir = args.directory temp_dir_created = False if args.verbose: print("Using {0} as the working directory".format(work_dir)) else: work_dir = mkdtemp() temp_dir_created = True if args.verbose: print("Created a temporary dir: {0}".format(work_dir)) if args.environment is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print( "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file") else: print( "Using value {} from environment variable $ROGER_ENV".format(env_var)) environment = env_var else: environment = args.environment if environment not in roger_env['environments']: self.removeDirTree(work_dir, args, temp_dir_created) raise ValueError('Environment not found in roger-mesos-tools.config file.') branch = "master" # master by default if args.branch is not None: branch = args.branch try: for app in apps: if app not in config['apps']: raise ValueError('Application {} specified not found.'.format(app)) else: try: if args.verbose: print("Deploying {} ...".format(app)) self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict) except (IOError, ValueError) as e: error_msg = "Error when deploying {}: {}".format(app, repr(e)) printErrorMsg(error_msg) pass # try deploying the next app except (Exception) as e: printException(e) raise except (Exception) as e: execution_result = 'FAILURE' printException(e) raise finally: # Check if the initializition of variables carried out if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals(): function_execution_start_time = datetime.now() if 'execution_result' not in globals() and 'execution_result' not in locals(): execution_result = 'FAILURE' if 'config_name' not in globals() and 'config_name' not in locals(): config_name = "" if 'environment' not in globals() and 'environment' not in locals(): environment = "dev" if not hasattr(args, "application"): args.application = "" if 'settingObj' not in globals() and 'settingObj' not in locals(): settingObj = Settings() if 'work_dir' not in globals() and 'work_dir' not in locals(): work_dir = '' temp_dir_created = False if not (self.rogerGitPullObject.outcome is 1 and self.rogerBuildObject.outcome is 1 and self.rogerPushObject.outcome is 1): execution_result = 'FAILURE' try: # If the deploy fails before going through any steps sc = self.utils.getStatsClient() if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application) args.application = self.utils.extract_app_name(args.application) time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000) input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.application) + ",event=deploy" + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******",identifier=" + str(self.identifier) tup = (input_metric, time_take_milliseonds) self.statsd_message_list.append(tup) self.removeDirTree(work_dir, args, temp_dir_created) except (Exception) as e: error_msg = "Error when deploying {}: {}".format(app, repr(e)) printErrorMsg(error_msg) raise
def main(self, settings, appConfig, gitObject, hooksObj, args): print(colored("******Executing GIT PULL of application repo******", "grey")) try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs environment = "dev" if hasattr(args, "environment"): environment = args.environment settingObj = settings appObj = appConfig gitObj = gitObject config_dir = settingObj.getConfigDir() hooksObj.config_file = args.config_file config = appObj.getConfig(config_dir, args.config_file) config_name = "" if 'name' in config: config_name = config['name'] common_repo = config.get('repo', '') data = appObj.getAppData(config_dir, args.config_file, args.app_name) if not data: raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format( args.app_name, config_dir, args.config_file)) repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', args.app_name) branch = "master" # master by default if args.branch is not None: branch = args.branch if not os.path.exists(args.directory): try: os.makedirs(args.directory) except OSError as exception: if exception.errno != errno.EEXIST: raise if not hasattr(args, "app_name"): args.app_name = "" if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) args.app_name = self.utils.extract_app_name(args.app_name) hooksObj.statsd_message_list = self.statsd_message_list hookname = "pre_gitpull" hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname)) # get/update target source(s) repo_name = appObj.getRepoName(repo) path = "{0}/{1}".format(args.directory, repo_name) if os.path.isdir(path): with chdir(path): exit_code = gitObj.gitPull(branch, args.verbose) else: with chdir('{0}'.format(args.directory)): exit_code = gitObj.gitShallowClone(repo, branch, args.verbose) if exit_code != 0: raise ValueError("Gitpull failed.") hooksObj.statsd_message_list = self.statsd_message_list hookname = "post_gitpull" hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname)) except (Exception) as e: printException(e) execution_result = 'FAILURE' raise finally: try: # If the gitpull fails before going through any steps if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals(): function_execution_start_time = datetime.now() if 'execution_result' not in globals() and 'execution_result' not in locals(): execution_result = 'FAILURE' if 'config_name' not in globals() and 'config_name' not in locals(): config_name = "" if 'environment' not in globals() and 'environment' not in locals(): environment = "dev" if not hasattr(args, "app_name"): args.app_name = "" if 'settingObj' not in globals() and 'settingObj' not in locals(): settingObj = Settings() if 'execution_result' is 'FAILURE': self.outcome = 0 sc = self.utils.getStatsClient() if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000) input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=gitpull" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"ERROR - %s" % e, file=sys.stderr) raise print(colored("******Completed the GIT PULL step successfully******", "green"))
def main(self, settings, appConfig, gitObject, hooksObj, args): try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs environment = "dev" if hasattr(args, "environment"): environment = args.environment settingObj = settings appObj = appConfig gitObj = gitObject config_dir = settingObj.getConfigDir() hooksObj.config_file = args.config_file config = appObj.getConfig(config_dir, args.config_file) config_name = "" if 'name' in config: config_name = config['name'] common_repo = config.get('repo', '') data = appObj.getAppData(config_dir, args.config_file, args.app_name) if not data: raise ValueError('Application with name [{}] or data for it not found at {}/{}.'.format( args.app_name, config_dir, args.config_file)) repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', args.app_name) branch = "master" # master by default if args.branch is not None: branch = args.branch if not os.path.exists(args.directory): try: os.makedirs(args.directory) except OSError as exception: if exception.errno != errno.EEXIST: raise if not hasattr(args, "app_name"): args.app_name = "" if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) args.app_name = self.utils.extract_app_name(args.app_name) hooksObj.statsd_message_list = self.statsd_message_list hookname = "pre_gitpull" hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{0}/{1}".format(args.directory, repo_name) if os.path.isdir(path): with chdir(path): exit_code = gitObj.gitPull(branch) else: with chdir('{0}'.format(args.directory)): exit_code = gitObj.gitShallowClone(repo, branch) if exit_code != 0: raise ValueError('gitpull failed.') hooksObj.statsd_message_list = self.statsd_message_list hookname = "post_gitpull" hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"The following error occurred: %s" % e, file=sys.stderr) execution_result = 'FAILURE' raise finally: try: # If the gitpull fails before going through any steps if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals(): function_execution_start_time = datetime.now() if 'execution_result' not in globals() and 'execution_result' not in locals(): execution_result = 'FAILURE' if 'config_name' not in globals() and 'config_name' not in locals(): config_name = "" if 'environment' not in globals() and 'environment' not in locals(): environment = "dev" if not hasattr(args, "app_name"): args.app_name = "" if 'settingObj' not in globals() and 'settingObj' not in locals(): settingObj = Settings() if 'execution_result' is 'FAILURE': self.outcome = 0 sc = self.utils.getStatsClient() if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000) input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=gitpull" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"The following error occurred: %s" % e, file=sys.stderr) raise
def main(self, settings, appConfig, frameworkObject, hooksObj, args): print(colored("******Deploying application to framework******", "grey")) try: validation_failed = False settingObj = settings appObj = appConfig frameworkUtils = frameworkObject config_dir = settingObj.getConfigDir() hooksObj.config_file = args.config_file cur_file_path = os.path.dirname(os.path.realpath(__file__)) config = appObj.getConfig(config_dir, args.config_file) config_name = "" act_as_user = "" if 'name' in config: config_name = config['name'] if 'act-as' in config: act_as_user = config['act-as'] roger_env = appObj.getRogerEnv(config_dir) if not hasattr(args, "app_name"): args.app_name = "" if 'registry' not in roger_env.keys(): raise ValueError("Registry not found in roger-mesos-tools.config file.") else: self.registry = roger_env['registry'] if hasattr(args, "image_name"): self.image_name = args.image_name environment = roger_env.get('default_environment', '') if args.env is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print(colored("WARNING - Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file", "yellow")) else: if args.verbose: print(colored("Using value {} from environment variable $ROGER_ENV".format(env_var), "grey")) environment = env_var else: environment = args.env # ---------------------------------------------- if environment not in roger_env['environments']: raise ValueError("Environment not found in roger-mesos-tools.config file.") # ---------------------------------------------- # GetEnvironmentConfig(environment) # ---------------------------------------------- environmentObj = roger_env['environments'][environment] common_repo = config.get('repo', '') # ---------------------------------------------- # GetContainersForApp(app) # ---------------------------------------------- app_name = args.app_name container_list = [] # todo (vmahedia): What does ':' signify? Put explanation. if ':' in app_name: tokens = app_name.split(':') app_name = tokens[0] # todo (vmahedia): it's container list - need to explain syntax if ',' in tokens[1]: container_list = tokens[1].split(',') else: container_list.append(tokens[1]) # ---------------------------------------------- data = appObj.getAppData(config_dir, args.config_file, app_name) if not data: raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format( app_name, config_dir, args.config_file)) configured_container_list = [] for task in data['containers']: if type(task) == dict: configured_container_list.append(task.keys()[0]) else: configured_container_list.append(task) if not set(container_list) <= set(configured_container_list): raise ValueError("List of containers [{}] passed do not match list of acceptable containers: [{}]".format( container_list, configured_container_list)) frameworkObj = frameworkUtils.getFramework(data) framework = frameworkObj.getName() repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', app_name) comp_dir = settingObj.getComponentsDir() templ_dir = settingObj.getTemplatesDir() secrets_dir = settingObj.getSecretsDir() # Create comp_dir if it doesn't exist if not os.path.isdir(comp_dir): os.makedirs(comp_dir) if not container_list: data_containers = data['containers'] else: data_containers = container_list failed_container_dict = {} # (vmahedia) upto this point it's all getting and checking the # configuration parameters template = '' # Required for when work_dir,component_dir,template_dir or # secret_env_dir is something like '.' or './temp" os.chdir(cur_file_path) app_path = '' if 'template_path' in data: app_path = self.repo_relative_path(appObj, args, repo, data['template_path']) else: app_path = templ_dir extra_vars = {} if 'extra_variables_path' in data: ev_path = self.repo_relative_path(appObj, args, repo, data['extra_variables_path']) with open(ev_path) as f: extra_vars = yaml.load(f) if ev_path.lower( ).endswith('.yml') else json.load(f) if not app_path.endswith('/'): app_path = app_path + '/' if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) args.app_name = self.utils.extract_app_name(args.app_name) hooksObj.statsd_message_list = self.statsd_message_list hookname = "pre_push" hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname)) # ---------------------------------------------- # (vmahedia) Figure out what the hell this loop does # and name it appropriately # it seems first part is just finding a template and Rendering # it against the given config, checking to see if there are errors # ---------------------------------------------- # (vmahedia) Meat starts from here, probably. for container in data_containers: container_name = self.getContainerName(container) containerConfig = "{0}-{1}.json".format(config['name'], container_name) env = Environment(loader = FileSystemLoader("{}".format(app_path)), undefined = StrictUndefined) template_with_path = "[{}{}]".format(app_path, containerConfig) try: template = env.get_template(containerConfig) except exceptions.TemplateNotFound as e: raise ValueError("The template file {} does not exist".format(template_with_path)) except Exception as e: raise ValueError("Error while reading template from {} - {}".format(template_with_path, e)) additional_vars = {} # (vmahedia)variables likes this should be at least visible within one # scroll up or down, move this code to near to context # Why are we getting the secrets everytime, this requires the file to be # present additional_vars.update(extra_vars) secret_vars = self.loadSecrets(secrets_dir, containerConfig, args, environment) additional_vars.update(secret_vars) image_path = "{0}/{1}".format( roger_env['registry'], args.image_name) print("Rendering content from template {} for environment [{}]".format( template_with_path, environment)) try: output = self.renderTemplate(template, environment, image_path, data, config, container, container_name, additional_vars) except exceptions.UndefinedError as e: error_str = "The following Undefined Jinja variable error occurred. %s.\n" % e print(colored(error_str, "red"), file=sys.stderr) failed_container_dict[container_name] = error_str # we are going to fail even if one of the container config is not valid but we will # still go through the loop and collect all the errors before we bail out validation_failed = True pass # ---------------------------------------------- # it seems the checks above can finish independent of the # following code, decouple this two parts, later when the code # is well understood # ---------------------------------------------- # Adding check to see if all jinja variables git resolved fot # the container if container_name not in failed_container_dict: # Adding check so that not all apps try to mergeSecrets try: outputObj = json.loads(output) except Exception as e: raise ValueError("Error while loading json from {} - {}".format(template_with_path, e)) if '\"SECRET\"' in output: output = self.mergeSecrets(output, self.loadSecrets( secrets_dir, containerConfig, args, environment)) if output != "StandardError": try: comp_exists = os.path.exists("{0}".format(comp_dir)) if comp_exists is False: os.makedirs("{0}".format(comp_dir)) comp_env_exists = os.path.exists( "{0}/{1}".format(comp_dir, environment)) if comp_env_exists is False: os.makedirs( "{0}/{1}".format(comp_dir, environment)) except Exception as e: logging.error(traceback.format_exc()) # (vmahedia) Should we write out the files even though there is an error with one of the # containers. Although maybe users would want to see some output with open("{0}/{1}/{2}".format(comp_dir, environment, containerConfig), 'wb') as fh: fh.write(output) else: raise ValueError("Error while loading secrets to render template file variables") # Notify container error messages # let failed_container_dict just be for now, but report all the errors if validation_failed: raise Exception("Unable to render Jinja template") deployment_check_failed = False # fail if the deployment check fails for container in data_containers: container_name = self.getContainerName(container) containerConfig = "{0}-{1}.json".format(config['name'], container_name) config_file_path = "{0}/{1}/{2}".format(comp_dir, environment, containerConfig) result = frameworkObj.runDeploymentChecks(config_file_path, environment) if not result: # need to give more indication about what can they do to fix this and what exactly failed # in the deployment check function, we should print an error in that function as well print(colored("Deployment checks failed for container - {}".format(framework, container)), "red") deployment_check_failed = True if deployment_check_failed: raise Exception("Deployment Check failed for one or more containers, check logs for more info!") if args.skip_push: print(colored("Skipping push to {} framework. The rendered config file(s) are under {}/{}/".format( framework, colored(comp_dir, "cyan"), colored(environment, "cyan")), "yellow")) else: # push to roger framework if 'owner' in config: frameworkObj.act_as_user = config['owner'] tools_version_value = self.utils.get_version() image_name = self.registry + "/" + args.image_name image_tag_value = urllib.quote("'" + image_name + "'") for container in data_containers: try: function_execution_start_time = datetime.now() # Assume SUCCESS unless exception execution_result = 'SUCCESS' sc = self.utils.getStatsClient() except (Exception) as e: raise ValueError("{} Error : {}".format(getDebugInfo(), e)) try: # this is where actual push is happening # we only push if forced, in case of failures # in deployment checks # # (vmahedia) todo: # list down scenarios in which this features # will be useful resp, task_id = frameworkObj.put(config_file_path, environmentObj, container_name, environment, act_as_user) container_task_id = self.utils.modify_task_id(task_id) self.task_id.extend(container_task_id) if hasattr(resp, "status_code"): status_code = resp.status_code except (Exception) as e: print("ERROR - : %s" % e, file=sys.stderr) execution_result = 'FAILURE' raise finally: try: if 'function_execution_start_time' not in globals() and \ 'function_execution_start_time' not in locals(): function_execution_start_time = datetime.now() if 'execution_result' not in globals() and \ 'execution_result' not in locals(): execution_result = 'FAILURE' if 'config_name' not in globals() and \ 'config_name' not in locals(): config_name = "" if 'environment' not in globals() and \ 'environment' not in locals(): environment = "dev" if 'container_name' not in globals() and \ 'container_name' not in locals(): container_name = "" if 'status_code' not in globals() and \ 'status_code' not in locals(): status_code = "500" if not hasattr(args, "app_name"): args.app_name = "" if 'settingObj' not in globals() and \ 'settingObj' not in locals(): settingObj = Settings() if 'container_task_id' not in globals() and 'container_task_id' not in locals(): container_task_id = [] if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) if not str(status_code).startswith("20"): execution_result = 'FAILURE' self.outcome = 0 time_taken = (datetime.now() - function_execution_start_time).total_seconds() for task_id in container_task_id: input_metric = "roger-tools.rogeros_tools_exec_time" + \ ",app_name=" + str(args.app_name) + \ ",event=push" + \ ",container_name=" + str(container_name) + \ ",identifier=" + str(self.identifier) + \ ",outcome=" + str(execution_result) + \ ",response_code=" + str(status_code) + \ ",config_name=" + str(config_name) + \ ",env=" + str(environment) + \ ",user="******"20"): metric = input_metric.replace("rogeros_tools_exec_time", "rogeros_events") metric = metric + ",source=tools" + ",task_id=" + task_id self.statsd_counter_logging(metric) except (Exception) as e: printException(e) raise hooksObj.statsd_message_list = self.statsd_message_list hookname = "post_push" hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + \ ",app_name=" + str(args.app_name) + \ ",identifier=" + str(self.identifier) + \ ",config_name=" + str(config_name) + \ ",env=" + str(environment) + \ ",user="******"{} hook failed.".format(hookname)) print(colored("******Done with the PUSH step******", "green")) except (Exception) as e: raise ValueError("ERROR - {}".format(e))
def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args): try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' settingObj = settingObject appObj = appObject config_dir = settingObj.getConfigDir() root = settingObj.getCliDir() roger_env = appObj.getRogerEnv(config_dir) config = appObj.getConfig(config_dir, args.config_file) config_name = "" if 'name' in config: config_name = config['name'] if 'registry' not in roger_env: raise ValueError( 'Registry not found in roger-mesos-tools.config file.') else: self.registry = roger_env['registry'] # Setup for Slack-Client, token, and git user if 'notifications' in config: self.slack = Slack( config['notifications'], '/home/vagrant/.roger_cli.conf.d/slack_token') self.identifier = self.utils.get_identifier( config_name, settingObj.getUser(), args.application) apps = [] apps_container_dict = {} if args.application == 'all': apps = config['apps'].keys() else: if ":" not in args.application and "[" not in args.application: apps.append(args.application) else: for item in args.application.split(":"): if '[' in item: matchObj = re.match(r'(.*)\[(.*)\]', item) apps.append(matchObj.group(1)) apps_container_dict[matchObj.group( 1)] = matchObj.group(2) else: apps.append(item) common_repo = config.get('repo', '') environment = roger_env.get('default_environment', '') work_dir = '' if args.directory: work_dir = args.directory temp_dir_created = False print("Using {0} as the working directory".format(work_dir)) else: work_dir = mkdtemp() temp_dir_created = True print("Created a temporary dir: {0}".format(work_dir)) if args.environment is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print( "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file" ) else: print( "Using value {} from environment variable $ROGER_ENV" .format(env_var)) environment = env_var else: environment = args.environment if environment not in roger_env['environments']: self.removeDirTree(work_dir, args, temp_dir_created) raise ValueError( 'Environment not found in roger-mesos-tools.config file.') branch = "master" # master by default if args.branch is not None: branch = args.branch try: for app in apps: if app not in config['apps']: print( 'Application {} specified not found.'.format(app)) else: try: print("Deploying {} ...".format(app)) self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict) except (IOError, ValueError) as e: print( "The following error occurred when deploying {}: {}" .format(app, e), file=sys.stderr) pass # try deploying the next app except (Exception) as e: print("The following error occurred: %s" % e, file=sys.stderr) raise except (Exception) as e: execution_result = 'FAILURE' print("The following error occurred: %s" % e, file=sys.stderr) raise finally: # Check if the initializition of variables carried out if 'function_execution_start_time' not in globals( ) and 'function_execution_start_time' not in locals(): function_execution_start_time = datetime.now() if 'execution_result' not in globals( ) and 'execution_result' not in locals(): execution_result = 'FAILURE' if 'config_name' not in globals() and 'config_name' not in locals( ): config_name = "" if 'environment' not in globals() and 'environment' not in locals( ): environment = "dev" if not hasattr(args, "application"): args.application = "" if 'settingObj' not in globals() and 'settingObj' not in locals(): settingObj = Settings() if 'work_dir' not in globals() and 'work_dir' not in locals(): work_dir = '' temp_dir_created = False if not (self.rogerGitPullObject.outcome is 1 and self.rogerBuildObject.outcome is 1 and self.rogerPushObject.outcome is 1): execution_result = 'FAILURE' try: # If the deploy fails before going through any steps sc = self.utils.getStatsClient() if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier( config_name, settingObj.getUser(), args.application) args.application = self.utils.extract_app_name( args.application) time_take_milliseonds = ( (datetime.now() - function_execution_start_time).total_seconds() * 1000) input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str( args.application) + ",event=deploy" + ",outcome=" + str( execution_result ) + ",config_name=" + str(config_name) + ",env=" + str( environment) + ",user="******",identifier=" + str( self.identifier) tup = (input_metric, time_take_milliseonds) self.statsd_message_list.append(tup) self.removeDirTree(work_dir, args, temp_dir_created) except (Exception) as e: print("The following error occurred: %s" % e, file=sys.stderr) raise
def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj, args): try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs config_dir = settingObj.getConfigDir() root = settingObj.getCliDir() config = appObj.getConfig(config_dir, args.config_file) hooksObj.config_file = args.config_file roger_env = appObj.getRogerEnv(config_dir) config_name = "" if 'name' in config: config_name = config['name'] common_repo = config.get('repo', '') if not hasattr(args, "env"): args.env = "dev" data = appObj.getAppData(config_dir, args.config_file, args.app_name) if not data: raise ValueError('Application with name [{}] or data for it not found at {}/{}.'.format( args.app_name, config_dir, args.config_file)) repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', args.app_name) build_args = {} if 'build-args' in data: if 'environment' in data['build-args']: if args.env in data['build-args']['environment']: build_args = data['build-args']['environment'][args.env] projects = data.get('privateProjects', []) docker_path = data.get('path', 'none') # get/update target source(s) file_exists = True file_path = '' cur_dir = '' if "PWD" in os.environ: cur_dir = os.environ.get('PWD') abs_path = os.path.abspath(args.directory) repo_name = appObj.getRepoName(repo) if docker_path != 'none': if abs_path == args.directory: file_path = "{0}/{1}/{2}".format(args.directory, repo_name, docker_path) else: file_path = "{0}/{1}/{2}/{3}".format( cur_dir, args.directory, repo_name, docker_path) else: if abs_path == args.directory: file_path = "{0}/{1}".format(args.directory, repo_name) else: file_path = "{0}/{1}/{2}".format(cur_dir, args.directory, repo_name) if not hasattr(args, "app_name"): args.app_name = "" if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) args.app_name = self.utils.extract_app_name(args.app_name) hooksObj.statsd_message_list = self.statsd_message_list hookname = "pre_build" hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"{0}/{1}".format(file_path, data['build_filename'])) file_exists = os.path.exists(build_filename) if not file_exists: raise ValueError("Specified build file: {} does not exist. Exiting build.".format(build_filename)) else: file_exists = os.path.exists("{0}/Dockerfile".format(file_path)) if file_exists: if 'registry' not in roger_env: raise ValueError('Registry not found in roger-mesos-tools.config file.') else: self.registry = roger_env['registry'] self.tag_name = args.tag_name image = "{0}/{1}".format(roger_env['registry'], args.tag_name) try: if abs_path == args.directory: try: dockerObj.docker_build( dockerUtilsObj, appObj, args.directory, repo, projects, docker_path, image, build_args, build_filename) except ValueError: print('Docker build failed.') raise else: directory = '{0}/{1}'.format(cur_dir, args.directory) try: dockerObj.docker_build( dockerUtilsObj, appObj, directory, repo, projects, docker_path, image, build_args, build_filename) except ValueError: print('Docker build failed.') raise build_message = "Image {0} built".format(image) if(args.push): exit_code = dockerUtilsObj.docker_push(image) if exit_code != 0: raise ValueError( 'Docker push failed.') build_message += " and pushed to registry {}".format(roger_env[ 'registry']) print(build_message) except (IOError) as e: print("The folowing error occurred.(Error: %s).\n" % e, file=sys.stderr) raise else: print("Dockerfile does not exist in dir: {}".format(file_path)) hooksObj.statsd_message_list = self.statsd_message_list hookname = "post_build" hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"The following error occurred: %s" % e, file=sys.stderr) raise finally: try: # If the build fails before going through any steps if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals(): function_execution_start_time = datetime.now() if 'execution_result' not in globals() and 'execution_result' not in locals(): execution_result = 'FAILURE' if 'config_name' not in globals() and 'config_name' not in locals(): config_name = "" if not hasattr(args, "env"): args.env = "dev" if not hasattr(args, "app_name"): args.app_name = "" if 'settingObj' not in globals() and 'settingObj' not in locals(): settingObj = Settings() if 'execution_result' is 'FAILURE': self.outcome = 0 sc = self.utils.getStatsClient() if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000) input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=build" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"The following error occurred: %s" % e, file=sys.stderr) raise
def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj, args): print(colored("******Building the Docker image now******", "grey")) try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs config_dir = settingObj.getConfigDir() root = settingObj.getCliDir() config = appObj.getConfig(config_dir, args.config_file) hooksObj.config_file = args.config_file roger_env = appObj.getRogerEnv(config_dir) config_name = "" if 'name' in config: config_name = config['name'] common_repo = config.get('repo', '') if not hasattr(args, "env"): args.env = "dev" data = appObj.getAppData(config_dir, args.config_file, args.app_name) if not data: raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format( args.app_name, config_dir, args.config_file)) repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', args.app_name) build_args = {} if 'build-args' in data: if 'environment' in data['build-args']: if args.env in data['build-args']['environment']: build_args = data['build-args']['environment'][args.env] projects = data.get('privateProjects', []) # get/update target source(s) file_exists = True file_path = '' cur_dir = '' if "PWD" in os.environ: cur_dir = os.environ.get('PWD') # This is bad code, assuming current directory and then trying to again guess, this is not rocket science # it's a f*****g file path, as simple as that. https://seomoz.atlassian.net/browse/ROGER-2405 # dockerfile location possibilities # 1. Path relative to the repo, we know repo path for cli is <checkout_dir>/<repo> # 2. Absolute path # This path comes from config file and not passed on commandline so we should not try to prefix current # working directory if the relative path is passed, don't try to guess too much. # changelog : relative path from current directory won't work for working_directory or checkout_dir # changelog : working_directory or checkout_dir should be absolute path, not backward-compatible checkout_dir = os.path.abspath(args.directory) repo_name = appObj.getRepoName(repo) # (vmahedia) todo : this should be called docker_file_dir dockerfile_rel_repo_path = data.get('path', '') file_path = os.path.join(checkout_dir, repo_name, dockerfile_rel_repo_path) if not hasattr(args, "app_name"): args.app_name = "" if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) args.app_name = self.utils.extract_app_name(args.app_name) hooksObj.statsd_message_list = self.statsd_message_list hookname = "pre_build" hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"{} hook failed.".format(hookname)) build_filename = 'Dockerfile' if 'build_filename' in data: build_filename = ("{0}/{1}".format(file_path, data['build_filename'])) file_exists = os.path.exists(build_filename) if not file_exists: raise ValueError("Specified build file: {} does not exist. Exiting build.".format(build_filename)) else: file_exists = os.path.exists("{0}/Dockerfile".format(file_path)) if file_exists: # (vmahedia) todo: We know what parameters are required for build command so we should not wait until # now to bailout. Config parser should have a validator for every command to see if all the Required # parameters are passed or not. Why do all this up to this point if we know we will fail on this. # RequiredParameter, below, "registry" if 'registry' not in roger_env: raise ValueError("Registry not found in roger-mesos-tools.config file.") else: self.registry = roger_env['registry'] self.tag_name = args.tag_name image = "{0}/{1}".format(roger_env['registry'], args.tag_name) try: if checkout_dir == args.directory: try: dockerObj.docker_build( dockerUtilsObj, appObj, args.directory, repo, projects, dockerfile_rel_repo_path, image, build_args, args.verbose, build_filename) except ValueError: raise ValueError("Docker build failed") else: directory = '{0}/{1}'.format(cur_dir, args.directory) try: dockerObj.docker_build( dockerUtilsObj, appObj, directory, repo, projects, dockerfile_rel_repo_path, image, build_args, args.verbose, build_filename) except ValueError: print('Docker build failed.') raise print(colored("******Successfully built Docker image******", "green")) build_message = "Image [{}]".format(image) if(args.push): print(colored("******Pushing Docker image to registry******", "grey")) exit_code = dockerUtilsObj.docker_push(image, args.verbose) if exit_code != 0: raise ValueError( 'Docker push failed.') build_message += " successfully pushed to registry [{}]*******".format(roger_env[ 'registry']) print(colored(build_message, "green")) except (IOError) as e: printException(e) raise else: print(colored("Dockerfile does not exist in dir: {}".format(file_path), "red")) hooksObj.statsd_message_list = self.statsd_message_list hookname = "post_build" hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"" if not hasattr(args, "env"): args.env = "dev" if not hasattr(args, "app_name"): args.app_name = "" if 'settingObj' not in globals() and 'settingObj' not in locals(): settingObj = Settings() if 'execution_result' is 'FAILURE': self.outcome = 0 sc = self.utils.getStatsClient() if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name) time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000) input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=build" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user=" + str(settingObj.getUser()) tup = (input_metric, time_take_milliseonds) self.statsd_message_list.append(tup) except (Exception) as e: printException(e) raise
def main(self, settings, appConfig, frameworkObject, hooksObj, args): try: settingObj = settings appObj = appConfig frameworkUtils = frameworkObject config_dir = settingObj.getConfigDir() hooksObj.config_file = args.config_file cur_file_path = os.path.dirname(os.path.realpath(__file__)) config = appObj.getConfig(config_dir, args.config_file) config_name = "" act_as_user = "" if 'name' in config: config_name = config['name'] if 'act-as' in config: act_as_user = config['act-as'] roger_env = appObj.getRogerEnv(config_dir) if not hasattr(args, "app_name"): args.app_name = "" if 'registry' not in roger_env.keys(): raise ValueError( 'Registry not found in roger-mesos-tools.config file.') else: self.registry = roger_env['registry'] if hasattr(args, "image_name"): self.image_name = args.image_name environment = roger_env.get('default_environment', '') if args.env is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print( "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file" ) else: print( "Using value {} from environment variable $ROGER_ENV" .format(env_var)) environment = env_var else: environment = args.env if environment not in roger_env['environments']: raise ValueError( 'Environment not found in roger-mesos-tools.config file.') environmentObj = roger_env['environments'][environment] common_repo = config.get('repo', '') app_name = args.app_name container_list = [] if ':' in app_name: tokens = app_name.split(':') app_name = tokens[0] if ',' in tokens[1]: container_list = tokens[1].split(',') else: container_list.append(tokens[1]) data = appObj.getAppData(config_dir, args.config_file, app_name) if not data: raise ValueError( 'Application with name [{}] or data for it not found at {}/{}.' .format(app_name, config_dir, args.config_file)) configured_container_list = [] for task in data['containers']: if type(task) == dict: configured_container_list.append(task.keys()[0]) else: configured_container_list.append(task) if not set(container_list) <= set(configured_container_list): raise ValueError( 'List of containers [{}] passed do not match list of acceptable containers: [{}]' .format(container_list, configured_container_list)) frameworkObj = frameworkUtils.getFramework(data) framework = frameworkObj.getName() repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', app_name) comp_dir = settingObj.getComponentsDir() templ_dir = settingObj.getTemplatesDir() secrets_dir = settingObj.getSecretsDir() # Create comp_dir if it doesn't exist if not os.path.isdir(comp_dir): os.makedirs(comp_dir) # template marathon files if not container_list: data_containers = data['containers'] else: data_containers = container_list failed_container_dict = {} template = '' # Required for when work_dir,component_dir,template_dir or # secret_env_dir is something like '.' or './temp" os.chdir(cur_file_path) app_path = '' if 'template_path' in data: app_path = self.repo_relative_path(appObj, args, repo, data['template_path']) else: app_path = templ_dir extra_vars = {} if 'extra_variables_path' in data: ev_path = self.repo_relative_path(appObj, args, repo, data['extra_variables_path']) with open(ev_path) as f: extra_vars = yaml.load(f) if ev_path.lower().endswith( '.yml') else json.load(f) if not app_path.endswith('/'): app_path = app_path + '/' if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier( config_name, settingObj.getUser(), args.app_name) args.app_name = self.utils.extract_app_name(args.app_name) hooksObj.statsd_message_list = self.statsd_message_list hookname = "pre_push" hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str( args.app_name) + ",identifier=" + str( self.identifier) + ",config_name=" + str( config_name) + ",env=" + str( environment) + ",user="******"{0}-{1}.json".format( config['name'], container_name) else: container_name = container containerConfig = "{0}-{1}.json".format( config['name'], container) env = Environment(loader=FileSystemLoader( "{}".format(app_path)), undefined=StrictUndefined) template_with_path = "[{}{}]".format(app_path, containerConfig) try: template = env.get_template(containerConfig) except exceptions.TemplateNotFound as e: raise ValueError( "The template file {} does not exist".format( template_with_path)) except Exception as e: raise ValueError( "Error while reading template from {} - {}".format( template_with_path, e)) additional_vars = {} additional_vars.update(extra_vars) secret_vars = self.loadSecrets(secrets_dir, containerConfig, args, environment) additional_vars.update(secret_vars) image_path = "{0}/{1}".format(roger_env['registry'], args.image_name) print( "Rendering content from template {} for environment [{}]". format(template_with_path, environment)) try: output = self.renderTemplate(template, environment, image_path, data, config, container, container_name, additional_vars) except exceptions.UndefinedError as e: error_str = "The following error occurred. %s.\n" % e print(error_str, file=sys.stderr) failed_container_dict[container_name] = error_str pass # Adding check to see if all jinja variables git resolved fot the # container if container_name not in failed_container_dict: # Adding check so that not all apps try to mergeSecrets try: outputObj = json.loads(output) except Exception as e: raise ValueError( "Error while loading json from {} - {}".format( template_with_path, e)) if 'SECRET' in output: output = self.mergeSecrets( output, self.loadSecrets(secrets_dir, containerConfig, args, environment)) if output != "StandardError": try: comp_exists = os.path.exists( "{0}".format(comp_dir)) if comp_exists is False: os.makedirs("{0}".format(comp_dir)) comp_env_exists = os.path.exists("{0}/{1}".format( comp_dir, environment)) if comp_env_exists is False: os.makedirs("{0}/{1}".format( comp_dir, environment)) except Exception as e: logging.error(traceback.format_exc()) with open( "{0}/{1}/{2}".format(comp_dir, environment, containerConfig), 'wb') as fh: fh.write(output) if args.skip_push: print( "Skipping push to {} framework. The rendered config file(s) are under {}/{}" .format(framework, comp_dir, environment)) else: # push to roger framework if 'owner' in config: frameworkObj.act_as_user = config['owner'] tools_version_value = self.utils.get_version() image_name = self.registry + "/" + args.image_name image_tag_value = urllib.quote("'" + image_name + "'") for container in data_containers: try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs sc = self.utils.getStatsClient() except (Exception) as e: print("The following error occurred: %s" % e, file=sys.stderr) try: if type(container) == dict: container_name = str(container.keys()[0]) containerConfig = "{0}-{1}.json".format( config['name'], container_name) else: container_name = container containerConfig = "{0}-{1}.json".format( config['name'], container) if container_name in failed_container_dict: print( "Failed push to {} framework for container {} as unresolved Jinja variables present in template." .format(framework, container_name)) else: config_file_path = "{0}/{1}/{2}".format( comp_dir, environment, containerConfig) result = frameworkObj.runDeploymentChecks( config_file_path, environment) if args.force_push or result is True: resp, task_id = frameworkObj.put( config_file_path, environmentObj, container_name, environment, act_as_user) container_task_id = self.utils.modify_task_id( task_id) self.task_id.extend(container_task_id) if hasattr(resp, "status_code"): status_code = resp.status_code else: print( "Skipping push to {} framework for container {} as Validation Checks failed." .format(framework, container)) except (Exception) as e: print("The following error occurred: %s" % e, file=sys.stderr) execution_result = 'FAILURE' raise finally: try: if 'function_execution_start_time' not in globals( ) and 'function_execution_start_time' not in locals( ): function_execution_start_time = datetime.now() if 'execution_result' not in globals( ) and 'execution_result' not in locals(): execution_result = 'FAILURE' if 'config_name' not in globals( ) and 'config_name' not in locals(): config_name = "" if 'environment' not in globals( ) and 'environment' not in locals(): environment = "dev" if 'container_name' not in globals( ) and 'container_name' not in locals(): container_name = "" if 'status_code' not in globals( ) and 'status_code' not in locals(): status_code = "500" if not hasattr(args, "app_name"): args.app_name = "" if 'settingObj' not in globals( ) and 'settingObj' not in locals(): settingObj = Settings() if 'container_task_id' not in globals( ) and 'container_task_id' not in locals(): container_task_id = [] if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier( config_name, settingObj.getUser(), args.app_name) if not str(status_code).startswith("20"): execution_result = 'FAILURE' self.outcome = 0 time_take_milliseonds = ( (datetime.now() - function_execution_start_time ).total_seconds() * 1000) for task_id in container_task_id: input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str( args.app_name ) + ",event=push" + ",container_name=" + str( container_name) + ",identifier=" + str( self.identifier) + ",outcome=" + str( execution_result ) + ",response_code=" + str( status_code ) + ",config_name=" + str( config_name) + ",env=" + str( environment) + ",user="******",task_id=" + str( task_id ) + ",tools_version=" + str( tools_version_value ) + ",image_tag=" + str( image_tag_value) tup = (input_metric, time_take_milliseonds) self.statsd_push_list.append(tup) if str(status_code).startswith("20"): metric = input_metric.replace( "rogeros_tools_exec_time", "rogeros_events") metric = metric + ",source=tools" + ",task_id=" + task_id self.statsd_counter_logging(metric) except (Exception) as e: print("The following error occurred: %s" % e, file=sys.stderr) raise hooksObj.statsd_message_list = self.statsd_message_list hookname = "post_push" hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str( args.app_name) + ",identifier=" + str( self.identifier) + ",config_name=" + str( config_name) + ",env=" + str( environment) + ",user="******"The following error occurred: %s" % e, file=sys.stderr) raise