Exemplo n.º 1
0
class RogerDeploy(object):

    def __init__(self):
        self.rogerGitPullObject = RogerGitPull()
        self.rogerPushObject = RogerPush()
        self.rogerBuildObject = RogerBuild()
        self.dockerUtilsObject = DockerUtils()
        self.dockerObject = Docker()
        self.utils = Utils()
        self.slack = None
        self.statsd_message_list = []
        self.registry = ""
        self.image_name = ""

        # To remove a temporary directory created by roger-deploy if this
        # script exits
    def removeDirTree(self, work_dir, args, temp_dir_created):
        exists = os.path.exists(os.path.abspath(work_dir))
        if exists and (temp_dir_created is True):
            shutil.rmtree(work_dir)
            print("Deleted temporary dir:{0}".format(work_dir))

    def getNextVersion(self, config, roger_env, application, branch, work_dir, repo, args, gitObj):
        sha = getGitSha(work_dir, repo, branch, gitObj)
        docker_search = self.dockerUtilsObject.docker_search(roger_env['registry'], config['name'], application)
        image_version_list = []
        version = ''
        envs = []
        for each_key in roger_env["environments"].keys():
            envs.append(each_key)
        for line in docker_search.split('\n'):
            image = line.split(' ')[0]
            # (vmahedia) todo: This belongs in a separate DockerImage class. This class should know
            # how to generate the image name, how to search the image or return this name pattern we
            # then use to search this image on the registry.
            matchObj = re.match("^{0}-{1}-.*/v.*".format(config['name'], application), image)
            if matchObj and matchObj.group().startswith(config['name'] + '-' + application):
                skip_image = False
                for env in envs:
                    if matchObj.group().startswith("{0}-{1}-{2}".format(config['name'], application, env)):
                        skip_image = True
                        break
                if skip_image is False:
                    if verify(str(matchObj.group().split('v')[-1])):
                        image_version_list.append(matchObj.group().split('v')[-1])

        if len(image_version_list) == 0:  # Create initial version
            version = "{0}/v0.1.0".format(sha)
            print("No version currently exist in the Docker Registry. \nDeploying version:{0}".format(
                version))
        else:
            version = self.incrementVersion(sha, image_version_list, args)
        return version

    def incrementVersion(self, sha, image_version_list, args):
        latest = max(image_version_list, key=self.splitVersion)
        ver_tuple = self.splitVersion(latest)
        latest_version = ''
        if args.incr_major:
            latest_version = "{0}/v{1}.0.0".format(sha,
                                                   (int(ver_tuple[0]) + 1))
            return latest_version
        if args.incr_patch:
            latest_version = "{0}/v{1}.{2}.{3}".format(
                sha, int(ver_tuple[0]), int(ver_tuple[1]), (int(ver_tuple[2]) + 1))
            return latest_version

        latest_version = "{0}/v{1}.{2}.0".format(
            sha, int(ver_tuple[0]), (int(ver_tuple[1]) + 1))
        return latest_version

    def splitVersion(self, version):
        major, _, rest = version.partition('.')
        minor, _, rest = rest.partition('.')
        patch, _, rest = rest.partition('.')
        return int(major), int(minor) if minor else 0, int(patch) if patch else 0

    def parseArgs(self):
        self.parser = argparse.ArgumentParser(
            prog='roger deploy', description=describe())
        self.parser.add_argument('-e', '--environment', metavar='env',
                                 help="environment to deploy to. Example: 'dev' or 'stage'")
        self.parser.add_argument('-b', '--branch', metavar='branch', default='master',
                                 help="branch to pull code from. Defaults to master. Example: 'production' or 'master'")
        self.parser.add_argument('-sg', '--skip-gitpull', action="store_true",
                                 help="skip the gitpull step. Defaults to false.")
        self.parser.add_argument('-s', '--skip-build', action="store_true",
                                 help="whether to skip the build step. Defaults to false.")
        self.parser.add_argument('-M', '--incr-major', action="store_true",
                                 help="increment major in version. Defaults to false.")
        self.parser.add_argument('-sp', '--skip-push', action="store_true",
                                 help="skip the push step. Defaults to false.'")
        self.parser.add_argument('-v', '--verbose', help="verbose mode for debugging. Defaults to false.", action="store_true")
        self.parser.add_argument('-f', '--force-push', action="store_true",
                                 help="force push. Not recommended. Forces push even if validation checks failed. Applies only if skip_push is false. Defaults to false.")
        self.parser.add_argument('-p', '--incr-patch', action="store_true",
                                 help="increment patch in version. Defaults to false.'")
        self.parser.add_argument('-S', '--secrets-file',
                                 help="specifies an optional secrets file for deployment runtime variables.")
        self.parser.add_argument('-d', '--directory',
                                 help="working directory. Uses a temporary directory if not specified.")
        self.parser.add_argument('application', metavar='application', help="application to deploy. \
                                 Can also push specific containers(comma seperated). Example: 'all' \
                                 or 'app1:app2' or 'kairos' or 'app_name[container1,container2]' \
                                 or'app1[container1,container2]:app2[container3,container4]' or 'app1:app2[container]'")
        self.parser.add_argument('app_repo', metavar="app_repo",
                                 help="Application's git repository name, repo must be under 'seomoz' organization")
        self.parser.add_argument('config_file', metavar='config_file',
                                 help="configuration file to be use. Example: 'content.json' or 'kwe.json'")

        return self.parser

    def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args):
        try:
            function_execution_start_time = datetime.now()
            execution_result = 'SUCCESS'
            settingObj = settingObject
            appObj = appObject
            config_dir = settingObj.getConfigDir()
            root = settingObj.getCliDir()
            roger_env = appObj.getRogerEnv(config_dir)
            config = appObj.getConfig(config_dir, args.config_file)
            config_name = ""
            if 'name' in config:
                config_name = config['name']
            if 'registry' not in roger_env:
                raise ValueError('Registry not found in roger-mesos-tools.config file.')
            else:
                self.registry = roger_env['registry']

            # Setup for Slack-Client, token, and git user
            # (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event
            # Event should be registered and SlackNotification should be one of the members. it can have
            # N notifications on a particular "event", Notifications.Notify will broadcast notification to
            # all the interested parties.
            if 'notifications' in config:
                self.slack = Slack(config['notifications'],
                                   '/home/vagrant/.roger_cli.conf.d/slack_token')

            self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application)

            apps = []
            apps_container_dict = {}
            if args.application == 'all':
                apps = config['apps'].keys()
            else:
                if ":" not in args.application and "[" not in args.application:
                    apps.append(args.application)
                else:
                    for item in args.application.split(":"):
                        if '[' in item:
                            matchObj = re.match(r'(.*)\[(.*)\]', item)
                            apps.append(matchObj.group(1))
                            apps_container_dict[matchObj.group(1)] = matchObj.group(2)
                        else:
                            apps.append(item)

            common_repo = config.get('repo', '')
            environment = roger_env.get('default_environment', '')

            work_dir = ''
            if args.directory:
                work_dir = args.directory
                temp_dir_created = False
                if args.verbose:
                    print("Using {0} as the working directory".format(work_dir))
            else:
                work_dir = mkdtemp()
                temp_dir_created = True
                if args.verbose:
                    print("Created a temporary dir: {0}".format(work_dir))

            if args.environment is None:
                if "ROGER_ENV" in os.environ:
                    env_var = os.environ.get('ROGER_ENV')
                    if env_var.strip() == '':
                        print(
                            "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file")
                    else:
                        print(
                            "Using value {} from environment variable $ROGER_ENV".format(env_var))
                        environment = env_var
            else:
                environment = args.environment

            if environment not in roger_env['environments']:
                self.removeDirTree(work_dir, args, temp_dir_created)
                raise ValueError('Environment not found in roger-mesos-tools.config file.')

            branch = "master"  # master by default
            if args.branch is not None:
                branch = args.branch

            try:
                for app in apps:
                    if app not in config['apps']:
                        raise ValueError('Application {} specified not found.'.format(app))
                    else:
                        try:
                            if args.verbose:
                                print("Deploying {} ...".format(app))
                            self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj,
                                           root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict)
                        except (IOError, ValueError) as e:
                            error_msg = "Error when deploying {}: {}".format(app, repr(e))
                            printErrorMsg(error_msg)
                            pass    # try deploying the next app
            except (Exception) as e:
                printException(e)
                raise
        except (Exception) as e:
            execution_result = 'FAILURE'
            printException(e)
            raise
        finally:
            # Check if the initializition of variables carried out
            if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
                function_execution_start_time = datetime.now()

            if 'execution_result' not in globals() and 'execution_result' not in locals():
                execution_result = 'FAILURE'

            if 'config_name' not in globals() and 'config_name' not in locals():
                config_name = ""

            if 'environment' not in globals() and 'environment' not in locals():
                environment = "dev"

            if not hasattr(args, "application"):
                args.application = ""

            if 'settingObj' not in globals() and 'settingObj' not in locals():
                settingObj = Settings()

            if 'work_dir' not in globals() and 'work_dir' not in locals():
                work_dir = ''
                temp_dir_created = False

            if not (self.rogerGitPullObject.outcome is 1 and self.rogerBuildObject.outcome is 1 and self.rogerPushObject.outcome is 1):
                execution_result = 'FAILURE'

            try:
                # If the deploy fails before going through any steps
                sc = self.utils.getStatsClient()
                if not hasattr(self, "identifier"):
                    self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application)
                args.application = self.utils.extract_app_name(args.application)
                time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
                input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.application) + ",event=deploy" + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******",identifier=" + str(self.identifier)
                tup = (input_metric, time_take_milliseonds)
                self.statsd_message_list.append(tup)
                self.removeDirTree(work_dir, args, temp_dir_created)
            except (Exception) as e:
                error_msg = "Error when deploying {}: {}".format(app, repr(e))
                printErrorMsg(error_msg)
                raise

    def deployApp(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config,
                  roger_env, work_dir, config_dir, environment, app, branch, slack, config_file, common_repo, temp_dir_created, apps_container_dict):

        startTime = datetime.now()
        settingObj = settingObject
        appObj = appObject
        frameworkUtils = frameworkUtilsObject
        environmentObj = roger_env['environments'][environment]
        data = appObj.getAppData(config_dir, config_file, app)
        frameworkObj = frameworkUtils.getFramework(data)
        framework = frameworkObj.getName()

        repo = ''
        if common_repo != '':
            repo = data.get('repo', common_repo)
        else:
            repo = data.get('repo', app)

        image_name = ''
        image = ''

        skip_gitpull = True if args.skip_gitpull else False

        # get/update target source(s)
        if not skip_gitpull:
            args.app_name = app
            args.directory = work_dir
            self.rogerGitPullObject.statsd_message_list = self.statsd_message_list
            self.rogerGitPullObject.identifier = self.identifier
            self.rogerGitPullObject.main(settingObj, appObj, gitObj, hooksObj, args)

        skip_build = True if args.skip_build else False
        skip_push = True if args.skip_push else False
        secrets_file = args.secrets_file if args.secrets_file else None

        # Set initial version
        # todo (vmahedia) #image_name naming should not be magic, make it explicit

        # todo (vmahedia) ExtractClass GitInfo, no need to pass args, we already have the information.
        # We deal with only one repo at a time. It may change in future but we can change the code then.
        image_git_sha = getGitSha(work_dir, repo, branch, gitObj)
        image_name = "{0}-{1}-{2}/v0.1.0".format(config['name'], app, image_git_sha)
        print(colored("******Fetching current version deployed or latest version from registry.\
                       This is used to bump to next version.******", "grey"))
        if skip_build:
            curr_image_ver = frameworkObj.getCurrentImageVersion(
                roger_env, environment, app)
            self.image_name = curr_image_ver

            if args.verbose:
                print("Current image version deployed on {0} is :{1}".format(framework, curr_image_ver))
            if curr_image_ver is not None:
                image_name = "{0}-{1}-{2}".format(
                    config['name'], app, curr_image_ver)
                if args.verbose:
                    print("Image current version from {0} endpoint is:{1}".format(framework, image_name))
            else:
                if args.verbose:
                    print("Using base version for image:{0}".format(image_name))
        else:
            # Docker build,tag and push
            image_name = self.getNextVersion(
                config, roger_env, app, branch, work_dir, repo, args, gitObj)
            print(colored("******Done finding latest version******", "green"))
            image_name = "{0}-{1}-{2}".format(config['name'], app, image_name)
            print(colored("Bumped up image to version:{0}".format(image_name), "green"))
            self.image_name = image_name
            build_args = args
            build_args.app_name = app
            build_args.directory = os.path.abspath(work_dir)
            build_args.tag_name = image_name
            build_args.config_file = config_file
            build_args.env = environment
            build_args.push = True
            build_args.verbose = args.verbose
            try:
                self.rogerBuildObject.identifier = self.identifier
                self.rogerBuildObject.statsd_message_list = self.statsd_message_list
                self.rogerBuildObject.main(settingObj, appObject, hooksObj,
                                           self.dockerUtilsObject, self.dockerObject, build_args)
            except ValueError:
                raise

        print("Image Version is: {}".format(colored(image_name, "cyan")))

        # Deploying the app to framework
        args.image_name = image_name
        args.config_file = config_file
        args.env = environment
        if app in apps_container_dict:
            args.app_name = str(app) + ":" + apps_container_dict[app]
        else:
            args.app_name = app
        self.rogerPushObject.identifier = self.identifier
        self.rogerPushObject.statsd_message_list = self.statsd_message_list
        self.rogerPushObject.main(settingObj, appObj, frameworkUtils,
                                  hooksObj, args)

        deployTime = datetime.now() - startTime

        username = settingObj.getUser()

        deployMessage = "{0}'s deploy for {1} / {2} / {3} completed in {4} seconds.".format(
            username, app, environment, branch, deployTime.total_seconds())
        if slack is not None:
            slack.api_call(deployMessage)
        print(colored(deployMessage, "green"))

    def locateConfigFile(self, args, gitObj):
        # Clone the git repo first because config lives there, there's nothing that we can do without this file
        # this is not the clean way but the code is very convulted for now to implement this in a clean manner
        # For now, we will clone the repo silently and use that config. We have to clone it everytime because we
        # have to assume that there's always a change, although it's not true - we can check if there's a change
        # and only pull then, but that is another mess. Let's make it simple and pull everytime
        branch = args.branch if args.branch else "master"
        repo_dir = os.path.join(args.directory,args.app_repo)
        if os.path.exists(args.config_file):
            # skip the git clone
            # this file path could be inside the cloned repo or outside
            if args.config_file.startswith(args.directory):
                # chdir is a decorator above but we should move it to utils or somewhere else for every file to use
                with chdir(repo_dir):
                    # file is inside cloned repo so Pull to fetch changes
                    rc = gitObj.gitPull(branch, args.verbose)
                    if rc:
                        print(colored("WARNING: Unable to Pull branch - {}, from repo - {}. Config file will"
                                       "not contain latest changes".format(args.branch, args.app_repo)), "yellow")
                    # File is not inside cloned repo but somewhere else
                    # just use it, no need to do anything
        else:
             if args.app_repo:
                 if os.path.exists(repo_dir):
                     raise ValueError("Repo directory - {} already exists but config file - {} is not present"\
                                      .format(repo_dir, args.config_file))
                 # chdir is a decorator defined above
                 with chdir(args.directory):
                    if args.verbose:
                        print(colored("Cloning repo - {} at - {} for config file - {}".\
                              format(args.app_repo, repo_dir, args.config_file)))
                    # clone the repo
                    # file does not exist and we need to clone the repo it is in repo since repo is
                    # defined and we mandate it to be in repo
                    exit_code = gitObj.gitShallowClone(args.app_repo, branch, args.verbose)
                    if exit_code:
                        raise ValueError("Error cloning repo {} while looking for config file".format(args.app_repo))
                    # check now if we have config file in the cloned repo, otherwise bailout.
                    if not os.path.exists(args.config_file):
                        # we checked out the repo and either the path is not in repo and file does not exist
                        # the path is in repo but the repo doesn't have the config file
                        raise ValueError("Config file - {} does not exist".format(args.config_file))
             else:
                raise ValueError("Config file - {} does not exist, no app repo defined either".format(args.config_file))
Exemplo n.º 2
0
class RogerPush(object):

    def __init__(self):
        self.utils = Utils()
        self.task_id = []
        self.outcome = 1
        self.registry = ""
        self.image_name = ""

    def parse_args(self):
        self.parser = argparse.ArgumentParser(
            prog='roger push', description=describe())
        self.parser.add_argument('app_name', metavar='app_name',
                                 help="Name of the App to be pushed, as defined in config file."
                                      "To deploy specific containers from an App, look at example B"
                                      "Example: A. 'agora' B. 'app_name:container1,container2'")
        self.parser.add_argument('-e', '--env', metavar='env',
                                 help="Environment to push to. Example: 'dev' or 'prod'")
        self.parser.add_argument('-v', '--verbose', help="Verbose mode", action="store_false")
        # Changelog: todo - Change this to checkout_dir but it's complicated to make thsi change in other commands
        # so for now, leaving it as it is and will have to take care of this in one change.
        self.parser.add_argument('directory', metavar='directory',
                                 help="App Repo will be checked out here, this is the working dir CLI will use."
                                      "Example: '/home/vagrant/work_dir'")
        self.parser.add_argument('image_name', metavar='image_name',
                                 help="image name that includes version to use."
                                      "Example: 'roger-collectd-v0.20' or 'elasticsearch-v0.07'")
        self.parser.add_argument('config_file', metavar='config_file',
                                 help="Configuration file to use."
                                      "Example: A. 'local.yml' or B.'content.json'")
        self.parser.add_argument('--skip-push', '-s', help="App is not pushed. Only renders template with config."
                                                           "Use it to check generated file before deploying or to debug"
                                                           "Defaults to false.", action="store_true")
        self.parser.add_argument(
            '--force-push', '-f', help="force push. Not Recommended. Forces push even if validation checks failed. Defaults to false.", action="store_true")
        self.parser.add_argument('--secrets-file', '-S',
                                 help="specifies an optional secrets file for deploy runtime variables.")
        return self.parser

    def loadSecrets(self, secrets_dir, file_name, args, environment):
        if args.secrets_file:
            print("Using specified secrets file: {}".format(args.secrets_file))
            file_name = args.secrets_file

        exists = os.path.exists(secrets_dir)
        if exists is False:
            os.makedirs(secrets_dir)

        # (vmahedia) WE SHOULD NOT DO ANY GUESSING GAME BE EXPLICIT
        # about where we expect what and argument should make that very clear to customers
        # Two possible paths -- first without environment, second with
        path1 = "{}/{}".format(secrets_dir, file_name)
        path2 = "{}/{}/{}".format(secrets_dir, environment, file_name)
        secrets_file_paths = [file_name] if os.path.isabs(file_name) else [path1, path2]
        for secrets_file_path in secrets_file_paths:
            if args.verbose:
                print(colored("Trying to load secrets from file {}".format(secrets_file_path), "cyan"))
            try:
                with open(secrets_file_path) as f:
                    return_file = yaml.load(f) if secrets_file_path.lower().endswith('.yml') else json.load(f)
                if args.verbose:
                    print(colored("Loaded secrets from file {}".format(secrets_file_path), "cyan"))
                return return_file
            except IOError:
                if args.verbose:
                    print(colored("Unable to find secrets file {}".format(secrets_file_path), "cyan"))
                pass
            except ValueError as e:
                raise ValueError("Error while loading secrets from {} - {}".format(secrets_file_path, e))

    def replaceSecrets(self, output_dict, secrets_dict):
        if type(output_dict) is not dict:
            return output_dict

        for key in output_dict:
            if output_dict[key] == "SECRET":
                if key in secrets_dict.keys():
                    output_dict[key] = secrets_dict[key]

            if type(output_dict[key]) is list:
                temp_list = []
                for list_elem in output_dict[key]:
                    temp_list.append(self.replaceSecrets(
                        list_elem, secrets_dict))
                    output_dict[key] = temp_list

            if type(output_dict[key]) is dict:
                temp_dict = self.replaceSecrets(output_dict[key], secrets_dict)
                output_dict[key] = temp_dict

        return output_dict

    def mergeSecrets(self, json_str, secrets):
        '''Given a JSON string and an object of secret environment variables, replaces
        parses the JSON keys with the secret variables. Returns back
        a JSON string. Raises an error if there are any SECRET variables still exists.'''
        output_dict = json.loads(json_str)
        json_str = json.dumps(self.replaceSecrets(output_dict, secrets), indent=4)

        if '\"SECRET\"' in json_str:
            print(colored("ERROR - Found the \"SECRET\" keyword in the template file -- does your secrets file have all secret environment variables?", "red"))
            print(colored("ERROR - The use of \"SECRET\" is deprecated. Please switch to using Jinja variables. To do so,"
              " use '{{ <actual variable name> }}' instead of \"SECRET\" in the template file.", "red"))
            return "StandardError"
        return json_str

    def renderTemplate(self, template, environment, image, app_data, config, container, container_name, additional_vars):

        variables = {'environment': environment, 'image': image}

        # Copy variables from config-wide, app-wide, then container-wide variable
        # configs, each one from "global" and then environment-specific.
        for obj in [config, app_data, container]:
            if type(obj) == dict and 'vars' in obj:
                variables.update(obj['vars'].get('global', {}))
                variables.update(obj['vars'].get('environment', {}).get(environment, {}))

        variables.update(additional_vars)
        return template.render(variables)

    def repo_relative_path(self, appConfig, args, repo, path):
        '''Returns a path relative to the repo, assumed to be under [args.directory]/[repo name]'''
        repo_name = appConfig.getRepoName(repo)
        abs_path = os.path.abspath(args.directory)
        return os.path.join(args.directory, repo_name, path)


    def getContainerName(self, container):
         return str(container.keys()[0]) if type(container) == dict else container

    def getContainersList(self, app_name):
        container_list = []
        # todo (vmahedia): What does ':' signify? Put explanation.
        if ':' in app_name:
            tokens = app_name.split(':')
            app_name = tokens[0]
            # todo (vmahedia): it's container list - need to explain syntax
            if ',' in tokens[1]:
                container_list = tokens[1].split(',')
            else:
                container_list.append(tokens[1])
        return container_list

    def getConfiguredContainersList(self, app_data):
        configured_container_list = []
        for task in app_data['containers']:
            if type(task) == dict:
                configured_container_list.append(task.keys()[0])
            else:
                configured_container_list.append(task)
        return configured_container_list

    # vmahedia: Why does this have to be so complex? Maybe just define on the commandline explicitly
    def getTargetEnvironment(self, roger_env, args):
        environment = roger_env.get('default_environment', '')
        if args.env is None:
            if "ROGER_ENV" in os.environ:
                env_var = os.environ.get('ROGER_ENV')
                if env_var.strip() == '':
                    print(colored("WARNING - Environment variable $ROGER_ENV is not set. Using the default set "
                                  "from roger-mesos-tools.config file", "yellow"))
                else:
                    if args.verbose:
                        print(colored("Using value {} from environment variable $ROGER_ENV".format(env_var), "grey"))
                    environment = env_var
        else:
            environment = args.env
        return environment

    def getRepository(self, app_data, common_repo, app_name):
        repo = ''
        if common_repo != '':
            repo = app_data.get('repo', common_repo)
        else:
            repo = app_data.get('repo', app_name)
        return repo

    def getAppPath(self, appObj, args, data, repo, templ_dir):
        app_path = ''
        if 'template_path' in data:
            app_path = self.repo_relative_path(appObj, args, repo, data['template_path'])
        else:
            app_path = templ_dir
        return app_path

    def main(self, settings, appConfig, frameworkObject, hooksObj, args):
        print(colored("******Deploying application to framework******", "grey"))
        try:
            validation_failed = False
            settingObj = settings
            appObj = appConfig
            frameworkUtils = frameworkObject
            config_dir = settingObj.getConfigDir()
            hooksObj.config_file = args.config_file
            cur_file_path = os.path.dirname(os.path.realpath(__file__))
            config = appObj.getConfig(config_dir, args.config_file)
            config_name = ""
            act_as_user = ""
            if 'name' in config:
                config_name = config['name']
            if 'act-as' in config:
                act_as_user = config['act-as']
            roger_env = appObj.getRogerEnv(config_dir)

            if not hasattr(args, "app_name"):
                args.app_name = ""

            try:
                self.registry = roger_env['registry']
            except KeyError:
                raise ValueError("Registry not found in roger-mesos-tools.config file.")

            if hasattr(args, "image_name"):
                self.image_name = args.image_name

            environment = self.getTargetEnvironment(roger_env, args)
            # ----------------------------------------------
            # GetEnvironmentConfig(environment)
            # ----------------------------------------------
            try:
                environmentObj = roger_env['environments'][environment]
            except KeyError as e:
                raise ValueError("'environment' not defined in roger-mesos-tools.config file. - {}".format(e))

            data = appObj.getAppData(config_dir, args.config_file, self.utils.extract_app_name(args.app_name))
            if not data:
                raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format(
                                 args.app_name, config_dir, args.config_file))

            container_list = self.getContainersList(args.app_name)
            configured_container_list = self.getConfiguredContainersList(data)

            if not set(container_list) <= set(configured_container_list):
                raise ValueError("List of containers [{}] passed are more than list of containers configured in config"
                                 "file: [{}]".format(container_list, configured_container_list))

            frameworkObj = frameworkUtils.getFramework(data)
            framework = frameworkObj.getName()
            common_repo = config.get('repo', '')

            repo = self.getRepository(data, common_repo, args.app_name)
            comp_dir = settingObj.getComponentsDir()
            templ_dir = settingObj.getTemplatesDir()
            secrets_dir = settingObj.getSecretsDir()

            # Create comp_dir if it doesn't exist
            if not os.path.isdir(comp_dir):
                os.makedirs(comp_dir)

            data_containers = data['containers'] if not container_list else container_list
            failed_container_dict = {}

            # (vmahedia) upto this point it's all getting and checking the
            # configuration parameters

            # Required for when work_dir,component_dir,template_dir or
            # secret_env_dir is something like '.' or './temp"
            os.chdir(cur_file_path)
            app_path = self.getAppPath(appObj, args, data, repo, templ_dir)

            env = Environment(loader = FileSystemLoader("{}".format(app_path)), undefined = StrictUndefined)
            extra_vars = {}
            if 'extra_variables_path' in data:
                ev_path = self.repo_relative_path(appObj, args, repo, data['extra_variables_path'])
                with open(ev_path) as f:
                    extra_vars = yaml.load(f) if ev_path.lower().endswith('.yml') else json.load(f)

            if not hasattr(self, "identifier"):
                self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)

            args.app_name = self.utils.extract_app_name(args.app_name)
            hookname = "pre_push"
            exit_code = hooksObj.run_hook(hookname, data, app_path, args.env, settingObj.getUser())
            if exit_code != 0:
                raise ValueError("{} hook failed.".format(hookname))

            # ----------------------------------------------
            # (vmahedia) Figure out what the hell this loop does
            # and name it appropriately
            # it seems first part is just finding a template and Rendering
            # it against the given config, checking to see if there are errors
            # ----------------------------------------------
            # (vmahedia) Meat starts from here, probably.
            template = ''
            for container in data_containers:
                container_name = self.getContainerName(container)
                if type(container) == dict:
                    container = container[container_name]
                containerConfig = "{0}-{1}.json".format(config['name'], container_name)

                template_with_path = os.path.join(app_path, containerConfig)
                try:
                    template = env.get_template(containerConfig)
                except exceptions.TemplateNotFound as e:
                    raise ValueError("The template file {} does not exist".format(template_with_path))
                except Exception as e:
                    raise ValueError("Error while reading template from {} - {}".format(template_with_path, e))

                additional_vars = {}
                # (vmahedia)variables likes this should be at least visible within one
                # scroll up or down, move this code to near to context
                # Why are we getting the secrets everytime, this requires the file to be
                # present
                additional_vars.update(extra_vars)
                secret_vars = self.loadSecrets(secrets_dir, containerConfig, args, environment)
                if secret_vars is not None:
                    additional_vars.update(secret_vars)

                image_path = "{0}/{1}".format(roger_env['registry'], args.image_name)
                print("Rendering content from template {} for environment [{}]".format(template_with_path, environment))
                try:
                    output = self.renderTemplate(template, environment, image_path, data,
                                                 config, container, container_name, additional_vars)
                except exceptions.UndefinedError as e:
                    error_str = "The following Undefined Jinja variable error occurred. %s.\n" % e
                    print(colored(error_str, "red"), file=sys.stderr)
                    failed_container_dict[container_name] = error_str
                    # we are going to fail even if one of the container config is not valid but we will
                    # still go through the loop and collect all the errors before we bail out
                    validation_failed = True
                    pass
                # ----------------------------------------------
                # it seems the checks above can finish independent of the
                # following code, decouple this two parts, later when  the code
                # is well understood
                # ----------------------------------------------
                # Adding check to see if all jinja variables git resolved fot
                # the container
                if container_name not in failed_container_dict:
                    # Adding check so that not all apps try to mergeSecrets
                    try:
                        outputObj = json.loads(output)
                    except Exception as e:
                        raise ValueError("Error while loading json from {} - {}".format(template_with_path, e))

                    if '\"SECRET\"' in output and not args.secrets_file:
                        raise ValueError('"SECRET" string present in template, replace'
                                         'with template variables named in sercrets file')
                    output = self.mergeSecrets(output, secret_vars)
                    if output != "StandardError":
                        try:
                            comp_dir_exists = os.path.exists("{0}".format(comp_dir))
                            if not comp_dir_exists:
                                os.makedirs("{0}".format(comp_dir))
                            comp_env_dir_exists = os.path.exists("{0}/{1}".format(comp_dir, environment))
                            if not comp_env_dir_exists:
                                os.makedirs("{0}/{1}".format(comp_dir, environment))
                        except Exception as e:
                            logging.error(traceback.format_exc())
                        # (vmahedia) Should we write out the files even though there is an error with one of the
                        # containers. Although maybe users would want to see some output
                        with open("{0}/{1}/{2}".format(comp_dir, environment, containerConfig), 'wb') as fh:
                            fh.write(output)
                    else:
                        raise ValueError("Error while loading secrets to render template file variables")

            # Notify container error messages
            # let failed_container_dict just be for now, but report all the errors
            if validation_failed:
                raise Exception("Unable to render Jinja template")

            deployment_check_failed = False
            # fail if the deployment check fails

            for container in data_containers:
                container_name = self.getContainerName(container)
                containerConfig = "{0}-{1}.json".format(config['name'], container_name)
                config_file_path = "{0}/{1}/{2}".format(comp_dir, environment, containerConfig)
                result = frameworkObj.runDeploymentChecks(config_file_path, environment)
                if not result:
                    # need to give more indication about what can they do to fix this and what exactly failed
                    # in the deployment check function, we should print an error in that function as well
                    print(colored("Deployment checks failed for container - {}".format(framework, container)), "red")
                    deployment_check_failed = True

            if deployment_check_failed:
                raise Exception("Deployment Check failed for one or more containers, check logs for more info!")

            if args.skip_push:
                print(colored("Skipping push to {} framework. The rendered config file(s) are under {}/{}/".format(
                    framework, colored(comp_dir, "cyan"), colored(environment, "cyan")), "yellow"))
            else:
                # push to roger framework
                if 'owner' in config:
                    frameworkObj.act_as_user = config['owner']

                tools_version_value = self.utils.get_version()
                if self.registry not in args.image_name:
                    image_name = self.registry + "/" + args.image_name

                for container in data_containers:
                    try:
                        function_execution_start_time = datetime.now()
                        # Assume SUCCESS unless exception
                        execution_result = 'SUCCESS'
                    except (Exception) as e:
                        raise ValueError("{} Error : {}".format(getDebugInfo(), e))
                    try:
                        container_name = self.getContainerName(container)
                        containerConfig = "{0}-{1}.json".format(config['name'], container_name)
                        config_file_path = "{0}/{1}/{2}".format(comp_dir, environment, containerConfig)
                        # this is where actual push is happening
                        # we only push if forced, in case of failures
                        # in deployment checks
                        #
                        # (vmahedia) todo:
                        # list down scenarios in which this features
                        # will be useful
                        resp, task_id = frameworkObj.put(config_file_path, environmentObj,
                                                         container_name, environment, act_as_user)
                        # // operator does floor division, rounds up to integer
                        color = "green" if resp.status_code // 100 == 2 else "red"
                        if not resp.status_code == 204: # empty response
                            print(colored(json.dumps(resp.json(), indent=4), color))
                        container_task_id = self.utils.modify_task_id(task_id)
                        self.task_id.extend(container_task_id)
                    except (Exception) as e:
                        print("ERROR - : %s" %e, file=sys.stderr)
                        execution_result = 'FAILURE'
                        raise
                    finally:
                        # todo: maybe send datadog event from here?
                        pass

            hookname = "post_push"
            exit_code = hooksObj.run_hook(hookname, data, app_path, args.env, settingObj.getUser())
            if exit_code != 0:
                raise ValueError("{} hook failed.".format(hookname))
            print(colored("******Done with the PUSH step******", "green"))

        except (Exception) as e:
            raise ValueError("ERROR - {}".format(e))
Exemplo n.º 3
0
class RogerGitPull(object):

    def __init__(self):
        self.utils = Utils()
        self.statsd_message_list = []
        self.outcome = 1

    def parse_args(self):
        self.parser = argparse.ArgumentParser(
            prog='roger gitpull', description=describe())
        self.parser.add_argument('app_name', metavar='app_name',
                                 help="application for which code is to be pulled. Example: 'agora' or 'grafana'")
        self.parser.add_argument('directory', metavar='directory',
                                 help="working directory. The repo has its own directory it this. Example: '/home/vagrant/work_dir'")
        self.parser.add_argument('-v', '--verbose', help="verbose mode for debugging", action="store_true")
        self.parser.add_argument('-b', '--branch', metavar='branch',
                                 help="git branch to pull code from. Example: 'production' or 'master'. Defaults to master.")
        self.parser.add_argument('config_file', metavar='config_file',
                                 help="configuration file to use. Example: 'content.json' or 'kwe.json'")
        return self.parser

    def main(self, settings, appConfig, gitObject, hooksObj, args):
        print(colored("******Executing GIT PULL of application repo******", "grey"))
        try:
            function_execution_start_time = datetime.now()
            execution_result = 'SUCCESS'  # Assume the execution_result to be SUCCESS unless exception occurs
            environment = "dev"
            if hasattr(args, "environment"):
                environment = args.environment
            settingObj = settings
            appObj = appConfig
            gitObj = gitObject
            config_dir = settingObj.getConfigDir()
            hooksObj.config_file = args.config_file
            config = appObj.getConfig(config_dir, args.config_file)
            config_name = ""
            if 'name' in config:
                config_name = config['name']
            common_repo = config.get('repo', '')
            data = appObj.getAppData(config_dir, args.config_file, args.app_name)
            if not data:
                raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format(
                    args.app_name, config_dir, args.config_file))
            repo = ''
            if common_repo != '':
                repo = data.get('repo', common_repo)
            else:
                repo = data.get('repo', args.app_name)

            branch = "master"  # master by default
            if args.branch is not None:
                branch = args.branch

            if not os.path.exists(args.directory):
                try:
                    os.makedirs(args.directory)
                except OSError as exception:
                    if exception.errno != errno.EEXIST:
                        raise

            if not hasattr(args, "app_name"):
                args.app_name = ""

            if not hasattr(self, "identifier"):
                self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)

            args.app_name = self.utils.extract_app_name(args.app_name)

            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "pre_gitpull"
            hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname))

            # get/update target source(s)
            repo_name = appObj.getRepoName(repo)
            path = "{0}/{1}".format(args.directory, repo_name)
            if os.path.isdir(path):
                with chdir(path):
                    exit_code = gitObj.gitPull(branch, args.verbose)
            else:
                with chdir('{0}'.format(args.directory)):
                    exit_code = gitObj.gitShallowClone(repo, branch, args.verbose)

            if exit_code != 0:
                raise ValueError("Gitpull failed.")

            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "post_gitpull"
            hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname))
        except (Exception) as e:
            printException(e)
            execution_result = 'FAILURE'
            raise
        finally:
            try:
                # If the gitpull fails before going through any steps
                if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
                    function_execution_start_time = datetime.now()

                if 'execution_result' not in globals() and 'execution_result' not in locals():
                    execution_result = 'FAILURE'

                if 'config_name' not in globals() and 'config_name' not in locals():
                    config_name = ""

                if 'environment' not in globals() and 'environment' not in locals():
                    environment = "dev"

                if not hasattr(args, "app_name"):
                    args.app_name = ""

                if 'settingObj' not in globals() and 'settingObj' not in locals():
                    settingObj = Settings()

                if 'execution_result' is 'FAILURE':
                    self.outcome = 0

                sc = self.utils.getStatsClient()
                if not hasattr(self, "identifier"):
                    self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
                time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
                input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=gitpull" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"ERROR - %s" %
                      e, file=sys.stderr)
                raise
            print(colored("******Completed the GIT PULL step successfully******", "green"))
Exemplo n.º 4
0
class RogerGitPull(object):

    def __init__(self):
        self.utils = Utils()
        self.statsd_message_list = []
        self.outcome = 1

    def parse_args(self):
        self.parser = argparse.ArgumentParser(
            prog='roger gitpull', description=describe())
        self.parser.add_argument('app_name', metavar='app_name',
                                 help="application for which code is to be pulled. Example: 'agora' or 'grafana'")
        self.parser.add_argument('directory', metavar='directory',
                                 help="working directory. The repo has its own directory it this. Example: '/home/vagrant/work_dir'")
        self.parser.add_argument('-b', '--branch', metavar='branch',
                                 help="git branch to pull code from. Example: 'production' or 'master'. Defaults to master.")
        self.parser.add_argument('config_file', metavar='config_file',
                                 help="configuration file to use. Example: 'content.json' or 'kwe.json'")
        return self.parser

    def main(self, settings, appConfig, gitObject, hooksObj, args):
        try:
            function_execution_start_time = datetime.now()
            execution_result = 'SUCCESS'  # Assume the execution_result to be SUCCESS unless exception occurs
            environment = "dev"
            if hasattr(args, "environment"):
                environment = args.environment
            settingObj = settings
            appObj = appConfig
            gitObj = gitObject
            config_dir = settingObj.getConfigDir()
            hooksObj.config_file = args.config_file
            config = appObj.getConfig(config_dir, args.config_file)
            config_name = ""
            if 'name' in config:
                config_name = config['name']
            common_repo = config.get('repo', '')
            data = appObj.getAppData(config_dir, args.config_file, args.app_name)
            if not data:
                raise ValueError('Application with name [{}] or data for it not found at {}/{}.'.format(
                    args.app_name, config_dir, args.config_file))
            repo = ''
            if common_repo != '':
                repo = data.get('repo', common_repo)
            else:
                repo = data.get('repo', args.app_name)

            branch = "master"  # master by default
            if args.branch is not None:
                branch = args.branch

            if not os.path.exists(args.directory):
                try:
                    os.makedirs(args.directory)
                except OSError as exception:
                    if exception.errno != errno.EEXIST:
                        raise

            if not hasattr(args, "app_name"):
                args.app_name = ""

            if not hasattr(self, "identifier"):
                self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)

            args.app_name = self.utils.extract_app_name(args.app_name)

            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "pre_gitpull"
            hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{0}/{1}".format(args.directory, repo_name)
            if os.path.isdir(path):
                with chdir(path):
                    exit_code = gitObj.gitPull(branch)
            else:
                with chdir('{0}'.format(args.directory)):
                    exit_code = gitObj.gitShallowClone(repo, branch)

            if exit_code != 0:
                raise ValueError('gitpull failed.')

            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "post_gitpull"
            hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"The following error occurred: %s" %
                  e, file=sys.stderr)
            execution_result = 'FAILURE'
            raise
        finally:
            try:
                # If the gitpull fails before going through any steps
                if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
                    function_execution_start_time = datetime.now()

                if 'execution_result' not in globals() and 'execution_result' not in locals():
                    execution_result = 'FAILURE'

                if 'config_name' not in globals() and 'config_name' not in locals():
                    config_name = ""

                if 'environment' not in globals() and 'environment' not in locals():
                    environment = "dev"

                if not hasattr(args, "app_name"):
                    args.app_name = ""

                if 'settingObj' not in globals() and 'settingObj' not in locals():
                    settingObj = Settings()

                if 'execution_result' is 'FAILURE':
                    self.outcome = 0

                sc = self.utils.getStatsClient()
                if not hasattr(self, "identifier"):
                    self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
                time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
                input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=gitpull" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"The following error occurred: %s" %
                      e, file=sys.stderr)
                raise
Exemplo n.º 5
0
class RogerDeploy(object):
    def __init__(self):
        self.rogerGitPullObject = RogerGitPull()
        self.rogerPushObject = RogerPush()
        self.rogerBuildObject = RogerBuild()
        self.dockerUtilsObject = DockerUtils()
        self.dockerObject = Docker()
        self.utils = Utils()
        self.slack = None
        self.statsd_message_list = []
        self.registry = ""
        self.image_name = ""

        # To remove a temporary directory created by roger-deploy if this
        # script exits
    def removeDirTree(self, work_dir, args, temp_dir_created):
        exists = os.path.exists(os.path.abspath(work_dir))
        if exists and (temp_dir_created is True):
            shutil.rmtree(work_dir)
            print("Deleted temporary dir:{0}".format(work_dir))

    def getNextVersion(self, config, roger_env, application, branch, work_dir,
                       repo, args, gitObj):
        sha = getGitSha(work_dir, repo, branch, gitObj)
        docker_search = self.dockerUtilsObject.docker_search(
            roger_env['registry'], config['name'], application)
        image_version_list = []
        version = ''
        envs = []
        for each_key in roger_env["environments"].keys():
            envs.append(each_key)
        for line in docker_search.split('\n'):
            image = line.split(' ')[0]
            matchObj = re.match(
                "^{0}-{1}-.*/v.*".format(config['name'], application), image)
            if matchObj and matchObj.group().startswith(config['name'] + '-' +
                                                        application):
                skip_image = False
                for env in envs:
                    if matchObj.group().startswith("{0}-{1}-{2}".format(
                            config['name'], application, env)):
                        skip_image = True
                        break
                if skip_image is False:
                    if verify(str(matchObj.group().split('v')[-1])):
                        image_version_list.append(
                            matchObj.group().split('v')[-1])

        if len(image_version_list) == 0:  # Create initial version
            version = "{0}/v0.1.0".format(sha)
            print(
                "No version currently exist in the Docker Registry. \nDeploying version:{0}"
                .format(version))
        else:
            version = self.incrementVersion(sha, image_version_list, args)
        return version

    def incrementVersion(self, sha, image_version_list, args):
        latest = max(image_version_list, key=self.splitVersion)
        ver_tuple = self.splitVersion(latest)
        latest_version = ''
        if args.incr_major:
            latest_version = "{0}/v{1}.0.0".format(sha,
                                                   (int(ver_tuple[0]) + 1))
            return latest_version
        if args.incr_patch:
            latest_version = "{0}/v{1}.{2}.{3}".format(sha, int(ver_tuple[0]),
                                                       int(ver_tuple[1]),
                                                       (int(ver_tuple[2]) + 1))
            return latest_version

        latest_version = "{0}/v{1}.{2}.0".format(sha, int(ver_tuple[0]),
                                                 (int(ver_tuple[1]) + 1))
        return latest_version

    def splitVersion(self, version):
        major, _, rest = version.partition('.')
        minor, _, rest = rest.partition('.')
        patch, _, rest = rest.partition('.')
        return int(major), int(minor) if minor else 0, int(
            patch) if patch else 0

    def parseArgs(self):
        self.parser = argparse.ArgumentParser(prog='roger deploy',
                                              description=describe())
        self.parser.add_argument(
            '-e',
            '--environment',
            metavar='env',
            help="environment to deploy to. Example: 'dev' or 'stage'")
        self.parser.add_argument(
            'application',
            metavar='application',
            help="application to deploy. Can also push specific"
            " containers(comma seperated). Example: 'all' or 'app1:app2' or 'kairos' or 'app_name[container1,container2]' or 'app1[container1,container2]:app2[container3,container4]' or 'app1:app2[container]'"
        )
        self.parser.add_argument(
            '-b',
            '--branch',
            metavar='branch',
            help=
            "branch to pull code from. Defaults to master. Example: 'production' or 'master'"
        )
        self.parser.add_argument(
            '-sg',
            '--skip-gitpull',
            action="store_true",
            help="skip the gitpull step. Defaults to false.")
        self.parser.add_argument(
            '-s',
            '--skip-build',
            action="store_true",
            help="whether to skip the build step. Defaults to false.")
        self.parser.add_argument(
            'config_file',
            metavar='config_file',
            help=
            "configuration file to be use. Example: 'content.json' or 'kwe.json'"
        )
        self.parser.add_argument(
            '-M',
            '--incr-major',
            action="store_true",
            help="increment major in version. Defaults to false.")
        self.parser.add_argument(
            '-sp',
            '--skip-push',
            action="store_true",
            help="skip the push step. Defaults to false.'")
        self.parser.add_argument(
            '-f',
            '--force-push',
            action="store_true",
            help=
            "force push. Not recommended. Forces push even if validation checks failed. Applies only if skip_push is false. Defaults to false."
        )
        self.parser.add_argument(
            '-p',
            '--incr-patch',
            action="store_true",
            help="increment patch in version. Defaults to false.'")
        self.parser.add_argument(
            '-S',
            '--secrets-file',
            help=
            "specifies an optional secrets file for deployment runtime variables."
        )
        self.parser.add_argument(
            '-d',
            '--directory',
            help=
            "working directory. Uses a temporary directory if not specified.")
        return self.parser

    def main(self, settingObject, appObject, frameworkUtilsObject, gitObj,
             hooksObj, args):
        try:
            function_execution_start_time = datetime.now()
            execution_result = 'SUCCESS'
            settingObj = settingObject
            appObj = appObject
            config_dir = settingObj.getConfigDir()
            root = settingObj.getCliDir()
            roger_env = appObj.getRogerEnv(config_dir)
            config = appObj.getConfig(config_dir, args.config_file)
            config_name = ""
            if 'name' in config:
                config_name = config['name']
            if 'registry' not in roger_env:
                raise ValueError(
                    'Registry not found in roger-mesos-tools.config file.')
            else:
                self.registry = roger_env['registry']

            # Setup for Slack-Client, token, and git user
            if 'notifications' in config:
                self.slack = Slack(
                    config['notifications'],
                    '/home/vagrant/.roger_cli.conf.d/slack_token')

            self.identifier = self.utils.get_identifier(
                config_name, settingObj.getUser(), args.application)

            apps = []
            apps_container_dict = {}
            if args.application == 'all':
                apps = config['apps'].keys()
            else:
                if ":" not in args.application and "[" not in args.application:
                    apps.append(args.application)
                else:
                    for item in args.application.split(":"):
                        if '[' in item:
                            matchObj = re.match(r'(.*)\[(.*)\]', item)
                            apps.append(matchObj.group(1))
                            apps_container_dict[matchObj.group(
                                1)] = matchObj.group(2)
                        else:
                            apps.append(item)

            common_repo = config.get('repo', '')
            environment = roger_env.get('default_environment', '')

            work_dir = ''
            if args.directory:
                work_dir = args.directory
                temp_dir_created = False
                print("Using {0} as the working directory".format(work_dir))
            else:
                work_dir = mkdtemp()
                temp_dir_created = True
                print("Created a temporary dir: {0}".format(work_dir))

            if args.environment is None:
                if "ROGER_ENV" in os.environ:
                    env_var = os.environ.get('ROGER_ENV')
                    if env_var.strip() == '':
                        print(
                            "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file"
                        )
                    else:
                        print(
                            "Using value {} from environment variable $ROGER_ENV"
                            .format(env_var))
                        environment = env_var
            else:
                environment = args.environment

            if environment not in roger_env['environments']:
                self.removeDirTree(work_dir, args, temp_dir_created)
                raise ValueError(
                    'Environment not found in roger-mesos-tools.config file.')

            branch = "master"  # master by default
            if args.branch is not None:
                branch = args.branch

            try:
                for app in apps:
                    if app not in config['apps']:
                        print(
                            'Application {} specified not found.'.format(app))
                    else:
                        try:
                            print("Deploying {} ...".format(app))
                            self.deployApp(settingObject, appObject,
                                           frameworkUtilsObject, gitObj,
                                           hooksObj, root, args, config,
                                           roger_env, work_dir, config_dir,
                                           environment, app, branch,
                                           self.slack, args.config_file,
                                           common_repo, temp_dir_created,
                                           apps_container_dict)
                        except (IOError, ValueError) as e:
                            print(
                                "The following error occurred when deploying {}: {}"
                                .format(app, e),
                                file=sys.stderr)
                            pass  # try deploying the next app
            except (Exception) as e:
                print("The following error occurred: %s" % e, file=sys.stderr)
                raise
        except (Exception) as e:
            execution_result = 'FAILURE'
            print("The following error occurred: %s" % e, file=sys.stderr)
            raise
        finally:
            # Check if the initializition of variables carried out
            if 'function_execution_start_time' not in globals(
            ) and 'function_execution_start_time' not in locals():
                function_execution_start_time = datetime.now()

            if 'execution_result' not in globals(
            ) and 'execution_result' not in locals():
                execution_result = 'FAILURE'

            if 'config_name' not in globals() and 'config_name' not in locals(
            ):
                config_name = ""

            if 'environment' not in globals() and 'environment' not in locals(
            ):
                environment = "dev"

            if not hasattr(args, "application"):
                args.application = ""

            if 'settingObj' not in globals() and 'settingObj' not in locals():
                settingObj = Settings()

            if 'work_dir' not in globals() and 'work_dir' not in locals():
                work_dir = ''
                temp_dir_created = False

            if not (self.rogerGitPullObject.outcome is 1
                    and self.rogerBuildObject.outcome is 1
                    and self.rogerPushObject.outcome is 1):
                execution_result = 'FAILURE'

            try:
                # If the deploy fails before going through any steps
                sc = self.utils.getStatsClient()
                if not hasattr(self, "identifier"):
                    self.identifier = self.utils.get_identifier(
                        config_name, settingObj.getUser(), args.application)
                args.application = self.utils.extract_app_name(
                    args.application)
                time_take_milliseonds = (
                    (datetime.now() -
                     function_execution_start_time).total_seconds() * 1000)
                input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(
                    args.application) + ",event=deploy" + ",outcome=" + str(
                        execution_result
                    ) + ",config_name=" + str(config_name) + ",env=" + str(
                        environment) + ",user="******",identifier=" + str(
                                self.identifier)
                tup = (input_metric, time_take_milliseonds)
                self.statsd_message_list.append(tup)
                self.removeDirTree(work_dir, args, temp_dir_created)
            except (Exception) as e:
                print("The following error occurred: %s" % e, file=sys.stderr)
                raise

    def deployApp(self, settingObject, appObject, frameworkUtilsObject, gitObj,
                  hooksObj, root, args, config, roger_env, work_dir,
                  config_dir, environment, app, branch, slack, config_file,
                  common_repo, temp_dir_created, apps_container_dict):

        startTime = datetime.now()
        settingObj = settingObject
        appObj = appObject
        frameworkUtils = frameworkUtilsObject
        environmentObj = roger_env['environments'][environment]
        data = appObj.getAppData(config_dir, config_file, app)
        frameworkObj = frameworkUtils.getFramework(data)
        framework = frameworkObj.getName()

        repo = ''
        if common_repo != '':
            repo = data.get('repo', common_repo)
        else:
            repo = data.get('repo', app)

        image_name = ''
        image = ''

        skip_gitpull = False
        if args.skip_gitpull is not None:
            skip_gitpull = args.skip_gitpull

        # get/update target source(s)
        if not skip_gitpull:
            args.app_name = app
            args.directory = work_dir
            self.rogerGitPullObject.statsd_message_list = self.statsd_message_list
            self.rogerGitPullObject.identifier = self.identifier
            self.rogerGitPullObject.main(settingObj, appObj, gitObj, hooksObj,
                                         args)

        skip_build = False
        if args.skip_build is not None:
            skip_build = args.skip_build

        skip_push = False
        if args.skip_push is not None:
            skip_push = args.skip_push

        secrets_file = None
        if args.secrets_file is not None:
            secrets_file = args.secrets_file

        # Set initial version
        image_git_sha = getGitSha(work_dir, repo, branch, gitObj)
        image_name = "{0}-{1}-{2}/v0.1.0".format(config['name'], app,
                                                 image_git_sha)

        if skip_build:
            curr_image_ver = frameworkObj.getCurrentImageVersion(
                roger_env, environment, app)
            self.image_name = curr_image_ver

            print("Current image version deployed on {0} is :{1}".format(
                framework, curr_image_ver))
            if curr_image_ver is not None:
                image_name = "{0}-{1}-{2}".format(config['name'], app,
                                                  curr_image_ver)
                print("Image current version from {0} endpoint is:{1}".format(
                    framework, image_name))
            else:
                print("Using base version for image:{0}".format(image_name))
        else:
            # Docker build,tag and push
            image_name = self.getNextVersion(config, roger_env, app, branch,
                                             work_dir, repo, args, gitObj)
            image_name = "{0}-{1}-{2}".format(config['name'], app, image_name)
            print("Bumped up image to version:{0}".format(image_name))
            self.image_name = image_name

            build_args = args
            build_args.app_name = app
            build_args.directory = os.path.abspath(work_dir)
            build_args.tag_name = image_name
            build_args.config_file = config_file
            build_args.env = environment
            build_args.push = True
            try:
                self.rogerBuildObject.identifier = self.identifier
                self.rogerBuildObject.statsd_message_list = self.statsd_message_list
                self.rogerBuildObject.main(settingObj, appObject, hooksObj,
                                           self.dockerUtilsObject,
                                           self.dockerObject, build_args)
            except ValueError:
                raise

        print("Version is:" + image_name)

        # Deploying the app to framework
        args.image_name = image_name
        args.config_file = config_file
        args.env = environment
        if app in apps_container_dict:
            args.app_name = str(app) + ":" + apps_container_dict[app]
        else:
            args.app_name = app
        self.rogerPushObject.identifier = self.identifier
        self.rogerPushObject.statsd_message_list = self.statsd_message_list
        self.rogerPushObject.main(settingObj, appObj, frameworkUtils, hooksObj,
                                  args)

        deployTime = datetime.now() - startTime

        username = settingObj.getUser()

        deployMessage = "{0}'s deploy for {1} / {2} / {3} completed in {4} seconds.".format(
            username, app, environment, branch, deployTime.total_seconds())
        if slack is not None:
            slack.api_call(deployMessage)
        print(deployMessage)
Exemplo n.º 6
0
class RogerPush(object):

    def __init__(self):
        self.utils = Utils()
        self.task_id = []
        self.statsd_message_list = []
        self.statsd_push_list = []
        self.outcome = 1
        self.registry = ""
        self.image_name = ""

    def parse_args(self):
        self.parser = argparse.ArgumentParser(
            prog='roger push', description=describe())
        self.parser.add_argument('app_name', metavar='app_name', help="application to push. Can also push specific"
                                 " containers(comma seperated). Example: 'agora' or 'app_name:container1,container2'")
        self.parser.add_argument('-e', '--env', metavar='env',
                                 help="environment to push to. Example: 'dev' or 'prod'")
        self.parser.add_argument('-v', '--verbose', help="verbose mode for debugging", action="store_true")
        self.parser.add_argument('directory', metavar='directory',
                                 help="working directory. Example: '/home/vagrant/work_dir'")
        self.parser.add_argument('image_name', metavar='image_name',
                                 help="image name that includes version to use. Example: 'roger-collectd-v0.20' or 'elasticsearch-v0.07'")
        self.parser.add_argument('config_file', metavar='config_file',
                                 help="configuration file to use. Example: 'content.json' or 'kwe.json'")
        self.parser.add_argument(
            '--skip-push', '-s', help="skips push. Only generates components for review. Defaults to false.", action="store_true")
        self.parser.add_argument(
            '--force-push', '-f', help="force push. Not Recommended. Forces push even if validation checks failed. Defaults to false.", action="store_true")
        self.parser.add_argument('--secrets-file', '-S',
                                 help="specifies an optional secrets file for deploy runtime variables.")
        return self.parser

    # (vmahedia) todo: https://seomoz.atlassian.net/browse/ROGER-2396
    # this has a lot of redundant messages and logic of assuming the
    # secret file location is annoying, make it obvious and simple
    def loadSecrets(self, secrets_dir, file_name, args, environment):
        if args.secrets_file is not None:
            print("Using specified secrets file: {}".format(args.secrets_file))
            file_name = args.secrets_file
        exists = os.path.exists(secrets_dir)
        if exists is False:
            os.makedirs(secrets_dir)

        # (vmahedia) WE SHOULD NOT DO ANY GUESSING GAME BE EXPLICIT
        # about where we expect what and argument should make that very clear to customers
        # Two possible paths -- first without environment, second with
        path1 = "{}/{}".format(secrets_dir, file_name)
        path2 = "{}/{}/{}".format(secrets_dir, environment, file_name)
        if args.verbose:
            print(colored("Trying to load secrets from file {} or {}".format(path1, path2), "cyan"))

        try:
            with open(path1) as f:
                return_file = yaml.load(f) if path1.lower().endswith('.yml') else json.load(f)
            return return_file
        except IOError:
            pass
        except ValueError as e:
            raise ValueError("Error while loading json from {} - {}".format(path1, e))

        try:
            with open(path2) as f:
                return_file = yaml.load(f) if path2.lower().endswith('.yml') else json.load(f)
            return return_file
        except IOError:
            if args.verbose:
                print("WARNING - Couldn't load any secrets file. Searched {} and {}. \nIGNORE this above warning if you do not have secrets or your secrets file is passed in using the optional argument and does not reside in the above 2 looked up paths.".format(path1, path2))
            return {}
        except ValueError as e:
            raise ValueError("Error while loading json from {} - {}".format(path2, e))

    def replaceSecrets(self, output_dict, secrets_dict):
        if type(output_dict) is not dict:
            return output_dict

        for key in output_dict:
            if output_dict[key] == "SECRET":
                if key in secrets_dict.keys():
                    output_dict[key] = secrets_dict[key]

            if type(output_dict[key]) is list:
                temp_list = []
                for list_elem in output_dict[key]:
                    temp_list.append(self.replaceSecrets(
                        list_elem, secrets_dict))
                    output_dict[key] = temp_list

            if type(output_dict[key]) is dict:
                temp_dict = self.replaceSecrets(output_dict[key], secrets_dict)
                output_dict[key] = temp_dict

        return output_dict

    def mergeSecrets(self, json_str, secrets):
        '''Given a JSON string and an object of secret environment variables, replaces
        parses the JSON keys with the secret variables. Returns back
        a JSON string. Raises an error if there are any SECRET variables still exists.'''
        output_dict = json.loads(json_str)
        json_str = json.dumps(self.replaceSecrets(
            output_dict, secrets), indent=4)

        if '\"SECRET\"' in json_str:
            print(colored("ERROR - Found the \"SECRET\" keyword in the template file -- does your secrets file have all secret environment variables?", "red"))
            print(colored("ERROR - The use of \"SECRET\" is deprecated. Please switch to using Jinja variables. To do so,"
              " use '{{ <actual variable name> }}' instead of \"SECRET\" in the template file.", "red"))
            return "StandardError"
        return json_str

    def renderTemplate(self, template, environment, image, app_data, config, container, container_name, additional_vars):

        variables = {'environment': environment, 'image': image}

        # Copy variables from config-wide, app-wide, then container-wide variable
        # configs, each one from "global" and then environment-specific.
        for obj in [config, app_data, container]:
            if type(obj) == dict and 'vars' in obj:
                variables.update(obj['vars'].get('global', {}))
                variables.update(obj['vars'].get('environment', {}).get(environment, {}))

        variables.update(additional_vars)

        return template.render(variables)

    def statsd_counter_logging(self, metric):
        sc = self.utils.getStatsClient()
        sc.incr(metric, 1)

    def repo_relative_path(self, appConfig, args, repo, path):
        '''Returns a path relative to the repo, assumed to be under [args.directory]/[repo name]'''
        repo_name = appConfig.getRepoName(repo)
        abs_path = os.path.abspath(args.directory)
        return os.path.join(args.directory, repo_name, path)


    def getContainerName(self, container):
         return str(container.keys()[0]) if type(container) == dict else container

    def main(self, settings, appConfig, frameworkObject, hooksObj, args):
        print(colored("******Deploying application to framework******", "grey"))
        try:
            validation_failed = False
            settingObj = settings
            appObj = appConfig
            frameworkUtils = frameworkObject
            config_dir = settingObj.getConfigDir()
            hooksObj.config_file = args.config_file
            cur_file_path = os.path.dirname(os.path.realpath(__file__))
            config = appObj.getConfig(config_dir, args.config_file)
            config_name = ""
            act_as_user = ""
            if 'name' in config:
                config_name = config['name']
            if 'act-as' in config:
                act_as_user = config['act-as']
            roger_env = appObj.getRogerEnv(config_dir)

            if not hasattr(args, "app_name"):
                args.app_name = ""

            if 'registry' not in roger_env.keys():
                raise ValueError("Registry not found in roger-mesos-tools.config file.")
            else:
                self.registry = roger_env['registry']

            if hasattr(args, "image_name"):
                self.image_name = args.image_name

            environment = roger_env.get('default_environment', '')
            if args.env is None:
                if "ROGER_ENV" in os.environ:
                    env_var = os.environ.get('ROGER_ENV')
                    if env_var.strip() == '':
                        print(colored("WARNING - Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file", "yellow"))
                    else:
                        if args.verbose:
                            print(colored("Using value {} from environment variable $ROGER_ENV".format(env_var), "grey"))
                        environment = env_var
            else:
                environment = args.env
            # ----------------------------------------------

            if environment not in roger_env['environments']:
                raise ValueError("Environment not found in roger-mesos-tools.config file.")

            # ----------------------------------------------
            # GetEnvironmentConfig(environment)
            # ----------------------------------------------
            environmentObj = roger_env['environments'][environment]
            common_repo = config.get('repo', '')

            # ----------------------------------------------
            # GetContainersForApp(app)
            # ----------------------------------------------
            app_name = args.app_name
            container_list = []
            # todo (vmahedia): What does ':' signify? Put explanation.
            if ':' in app_name:
                tokens = app_name.split(':')
                app_name = tokens[0]
                # todo (vmahedia): it's container list - need to explain syntax
                if ',' in tokens[1]:
                    container_list = tokens[1].split(',')
                else:
                    container_list.append(tokens[1])
            # ----------------------------------------------
            data = appObj.getAppData(config_dir, args.config_file, app_name)
            if not data:
                raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format(
                    app_name, config_dir, args.config_file))

            configured_container_list = []
            for task in data['containers']:
                if type(task) == dict:
                    configured_container_list.append(task.keys()[0])
                else:
                    configured_container_list.append(task)
            if not set(container_list) <= set(configured_container_list):
                raise ValueError("List of containers [{}] passed do not match list of acceptable containers: [{}]".format(
                    container_list, configured_container_list))

            frameworkObj = frameworkUtils.getFramework(data)
            framework = frameworkObj.getName()

            repo = ''
            if common_repo != '':
                repo = data.get('repo', common_repo)
            else:
                repo = data.get('repo', app_name)

            comp_dir = settingObj.getComponentsDir()
            templ_dir = settingObj.getTemplatesDir()
            secrets_dir = settingObj.getSecretsDir()

            # Create comp_dir if it doesn't exist
            if not os.path.isdir(comp_dir):
                os.makedirs(comp_dir)

            if not container_list:
                data_containers = data['containers']
            else:
                data_containers = container_list

            failed_container_dict = {}

            # (vmahedia) upto this point it's all getting and checking the
            # configuration parameters
            template = ''
            # Required for when work_dir,component_dir,template_dir or
            # secret_env_dir is something like '.' or './temp"
            os.chdir(cur_file_path)
            app_path = ''
            if 'template_path' in data:
                app_path = self.repo_relative_path(appObj, args, repo, data['template_path'])
            else:
                app_path = templ_dir

            extra_vars = {}
            if 'extra_variables_path' in data:
                ev_path = self.repo_relative_path(appObj, args, repo, data['extra_variables_path'])
                with open(ev_path) as f:
                    extra_vars = yaml.load(f) if ev_path.lower(
                    ).endswith('.yml') else json.load(f)

            if not app_path.endswith('/'):
                app_path = app_path + '/'

            if not hasattr(self, "identifier"):
                self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)

            args.app_name = self.utils.extract_app_name(args.app_name)
            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "pre_push"
            hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{} hook failed.".format(hookname))

            # ----------------------------------------------
            # (vmahedia) Figure out what the hell this loop does
            # and name it appropriately
            # it seems first part is just finding a template and Rendering
            # it against the given config, checking to see if there are errors
            # ----------------------------------------------
            # (vmahedia) Meat starts from here, probably.
            for container in data_containers:
                container_name = self.getContainerName(container)
                containerConfig = "{0}-{1}.json".format(config['name'], container_name)

                env = Environment(loader = FileSystemLoader("{}".format(app_path)), undefined = StrictUndefined)
                template_with_path = "[{}{}]".format(app_path, containerConfig)
                try:
                    template = env.get_template(containerConfig)
                except exceptions.TemplateNotFound as e:
                    raise ValueError("The template file {} does not exist".format(template_with_path))
                except Exception as e:
                    raise ValueError("Error while reading template from {} - {}".format(template_with_path, e))

                additional_vars = {}
                # (vmahedia)variables likes this should be at least visible within one
                # scroll up or down, move this code to near to context
                # Why are we getting the secrets everytime, this requires the file to be
                # present
                additional_vars.update(extra_vars)
                secret_vars = self.loadSecrets(secrets_dir, containerConfig, args, environment)
                additional_vars.update(secret_vars)

                image_path = "{0}/{1}".format(
                    roger_env['registry'], args.image_name)
                print("Rendering content from template {} for environment [{}]".format(
                    template_with_path, environment))
                try:
                    output = self.renderTemplate(template, environment, image_path, data, config, container, container_name, additional_vars)
                except exceptions.UndefinedError as e:
                    error_str = "The following Undefined Jinja variable error occurred. %s.\n" % e
                    print(colored(error_str, "red"), file=sys.stderr)
                    failed_container_dict[container_name] = error_str

                    # we are going to fail even if one of the container config is not valid but we will
                    # still go through the loop and collect all the errors before we bail out
                    validation_failed = True
                    pass
                # ----------------------------------------------
                # it seems the checks above can finish independent of the
                # following code, decouple this two parts, later when  the code
                # is well understood
                # ----------------------------------------------
                # Adding check to see if all jinja variables git resolved fot
                # the container
                if container_name not in failed_container_dict:
                    # Adding check so that not all apps try to mergeSecrets
                    try:
                        outputObj = json.loads(output)
                    except Exception as e:
                        raise ValueError("Error while loading json from {} - {}".format(template_with_path, e))

                    if '\"SECRET\"' in output:
                        output = self.mergeSecrets(output, self.loadSecrets(
                            secrets_dir, containerConfig, args, environment))
                    if output != "StandardError":
                        try:
                            comp_exists = os.path.exists("{0}".format(comp_dir))
                            if comp_exists is False:
                                os.makedirs("{0}".format(comp_dir))
                            comp_env_exists = os.path.exists(
                                "{0}/{1}".format(comp_dir, environment))
                            if comp_env_exists is False:
                                os.makedirs(
                                    "{0}/{1}".format(comp_dir, environment))
                        except Exception as e:
                            logging.error(traceback.format_exc())
                        # (vmahedia) Should we write out the files even though there is an error with one of the
                        # containers. Although maybe users would want to see some output
                        with open("{0}/{1}/{2}".format(comp_dir, environment, containerConfig), 'wb') as fh:
                            fh.write(output)
                    else:
                        raise ValueError("Error while loading secrets to render template file variables")

            # Notify container error messages
            # let failed_container_dict just be for now, but report all the errors
            if validation_failed:
                raise Exception("Unable to render Jinja template")

            deployment_check_failed = False
            # fail if the deployment check fails

            for container in data_containers:
                container_name = self.getContainerName(container)
                containerConfig = "{0}-{1}.json".format(config['name'], container_name)
                config_file_path = "{0}/{1}/{2}".format(comp_dir, environment, containerConfig)
                result = frameworkObj.runDeploymentChecks(config_file_path, environment)
                if not result:
                    # need to give more indication about what can they do to fix this and what exactly failed
                    # in the deployment check function, we should print an error in that function as well
                    print(colored("Deployment checks failed for container - {}".format(framework, container)), "red")
                    deployment_check_failed = True

            if deployment_check_failed:
                raise Exception("Deployment Check failed for one or more containers, check logs for more info!")

            if args.skip_push:
                print(colored("Skipping push to {} framework. The rendered config file(s) are under {}/{}/".format(
                    framework, colored(comp_dir, "cyan"), colored(environment, "cyan")), "yellow"))
            else:
                # push to roger framework
                if 'owner' in config:
                    frameworkObj.act_as_user = config['owner']

                tools_version_value = self.utils.get_version()
                image_name = self.registry + "/" + args.image_name
                image_tag_value = urllib.quote("'" + image_name + "'")

                for container in data_containers:
                    try:
                        function_execution_start_time = datetime.now()
                        # Assume SUCCESS unless exception
                        execution_result = 'SUCCESS'
                        sc = self.utils.getStatsClient()
                    except (Exception) as e:
                        raise ValueError("{} Error : {}".format(getDebugInfo(), e))
                    try:
                        # this is where actual push is happening
                        # we only push if forced, in case of failures
                        # in deployment checks
                        #
                        # (vmahedia) todo:
                        # list down scenarios in which this features
                        # will be useful
                        resp, task_id = frameworkObj.put(config_file_path, environmentObj,
                                                         container_name, environment, act_as_user)
                        container_task_id = self.utils.modify_task_id(task_id)
                        self.task_id.extend(container_task_id)
                        if hasattr(resp, "status_code"):
                            status_code = resp.status_code
                    except (Exception) as e:
                        print("ERROR - : %s" %
                              e, file=sys.stderr)
                        execution_result = 'FAILURE'
                        raise
                    finally:
                        try:
                            if 'function_execution_start_time' not in globals() and \
                               'function_execution_start_time' not in locals():
                                function_execution_start_time = datetime.now()

                            if 'execution_result' not in globals() and \
                               'execution_result' not in locals():
                                execution_result = 'FAILURE'

                            if 'config_name' not in globals() and \
                               'config_name' not in locals():
                                config_name = ""

                            if 'environment' not in globals() and \
                               'environment' not in locals():
                                environment = "dev"

                            if 'container_name' not in globals() and \
                               'container_name' not in locals():
                                container_name = ""

                            if 'status_code' not in globals() and \
                               'status_code' not in locals():
                                status_code = "500"

                            if not hasattr(args, "app_name"):
                                args.app_name = ""

                            if 'settingObj' not in globals() and \
                               'settingObj' not in locals():
                                settingObj = Settings()

                            if 'container_task_id' not in globals() and 'container_task_id' not in locals():
                                container_task_id = []

                            if not hasattr(self, "identifier"):
                                self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(),
                                                                            args.app_name)

                            if not str(status_code).startswith("20"):
                                execution_result = 'FAILURE'
                                self.outcome = 0

                            time_taken = (datetime.now() - function_execution_start_time).total_seconds()
                            for task_id in container_task_id:
                                input_metric = "roger-tools.rogeros_tools_exec_time" + \
                                               ",app_name=" + str(args.app_name) + \
                                               ",event=push" + \
                                               ",container_name=" + str(container_name) + \
                                               ",identifier=" + str(self.identifier) + \
                                               ",outcome=" + str(execution_result) + \
                                               ",response_code=" + str(status_code) + \
                                               ",config_name=" + str(config_name) + \
                                               ",env=" + str(environment) + \
                                               ",user="******"20"):
                                    metric = input_metric.replace("rogeros_tools_exec_time", "rogeros_events")
                                    metric = metric + ",source=tools" + ",task_id=" + task_id
                                    self.statsd_counter_logging(metric)

                        except (Exception) as e:
                            printException(e)
                            raise

            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "post_push"
            hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + \
                                                                         ",app_name=" + str(args.app_name) + \
                                                                         ",identifier=" + str(self.identifier) + \
                                                                         ",config_name=" + str(config_name) + \
                                                                         ",env=" + str(environment) + \
                                                                         ",user="******"{} hook failed.".format(hookname))
            print(colored("******Done with the PUSH step******", "green"))

        except (Exception) as e:
            raise ValueError("ERROR - {}".format(e))
Exemplo n.º 7
0
class RogerBuild(object):

    def __init__(self):
        self.utils = Utils()
        self.statsd_message_list = []
        self.outcome = 1
        self.registry = ""
        self.tag_name = ""

    def parse_args(self):
        self.parser = argparse.ArgumentParser(
            prog='roger build', description=describe())
        self.parser.add_argument('app_name', metavar='app_name',
                                 help="application to build. Example: 'agora'.")
        self.parser.add_argument('directory', metavar='directory',
                                 help="working directory. Example: '/home/vagrant/work_dir'.")
        self.parser.add_argument('tag_name', metavar='tag_name',
                                 help="tag for the built image. Example: 'roger-collectd:0.20'.")
        self.parser.add_argument('config_file', metavar='config_file',
                                 help="configuration file to use. Example: 'content.json'.")
        self.parser.add_argument(
            '--push', '-p', help="Also push to registry. Defaults to false.", action="store_true")
        return self.parser

    def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj, args):
        try:
            function_execution_start_time = datetime.now()
            execution_result = 'SUCCESS'  # Assume the execution_result to be SUCCESS unless exception occurs
            config_dir = settingObj.getConfigDir()
            root = settingObj.getCliDir()
            config = appObj.getConfig(config_dir, args.config_file)
            hooksObj.config_file = args.config_file
            roger_env = appObj.getRogerEnv(config_dir)
            config_name = ""
            if 'name' in config:
                config_name = config['name']
            common_repo = config.get('repo', '')
            if not hasattr(args, "env"):
                args.env = "dev"
            data = appObj.getAppData(config_dir, args.config_file, args.app_name)
            if not data:
                raise ValueError('Application with name [{}] or data for it not found at {}/{}.'.format(
                    args.app_name, config_dir, args.config_file))
            repo = ''
            if common_repo != '':
                repo = data.get('repo', common_repo)
            else:
                repo = data.get('repo', args.app_name)

            build_args = {}
            if 'build-args' in data:
                if 'environment' in data['build-args']:
                    if args.env in data['build-args']['environment']:
                        build_args = data['build-args']['environment'][args.env]

            projects = data.get('privateProjects', [])
            docker_path = data.get('path', 'none')

            # get/update target source(s)
            file_exists = True
            file_path = ''
            cur_dir = ''
            if "PWD" in os.environ:
                cur_dir = os.environ.get('PWD')
            abs_path = os.path.abspath(args.directory)
            repo_name = appObj.getRepoName(repo)
            if docker_path != 'none':
                if abs_path == args.directory:
                    file_path = "{0}/{1}/{2}".format(args.directory,
                                                     repo_name, docker_path)
                else:
                    file_path = "{0}/{1}/{2}/{3}".format(
                        cur_dir, args.directory, repo_name, docker_path)
            else:
                if abs_path == args.directory:
                    file_path = "{0}/{1}".format(args.directory, repo_name)
                else:
                    file_path = "{0}/{1}/{2}".format(cur_dir,
                                                     args.directory, repo_name)

            if not hasattr(args, "app_name"):
                args.app_name = ""

            if not hasattr(self, "identifier"):
                self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)

            args.app_name = self.utils.extract_app_name(args.app_name)
            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "pre_build"
            hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"{0}/{1}".format(file_path, data['build_filename']))
                file_exists = os.path.exists(build_filename)
                if not file_exists:
                    raise ValueError("Specified build file: {} does not exist. Exiting build.".format(build_filename))
            else:
                file_exists = os.path.exists("{0}/Dockerfile".format(file_path))

            if file_exists:
                if 'registry' not in roger_env:
                    raise ValueError('Registry not found in roger-mesos-tools.config file.')
                else:
                    self.registry = roger_env['registry']
                self.tag_name = args.tag_name
                image = "{0}/{1}".format(roger_env['registry'], args.tag_name)
                try:
                    if abs_path == args.directory:
                        try:
                            dockerObj.docker_build(
                                dockerUtilsObj, appObj, args.directory, repo, projects, docker_path, image, build_args, build_filename)
                        except ValueError:
                            print('Docker build failed.')
                            raise
                    else:
                        directory = '{0}/{1}'.format(cur_dir, args.directory)
                        try:
                            dockerObj.docker_build(
                                dockerUtilsObj, appObj, directory, repo, projects, docker_path, image, build_args, build_filename)
                        except ValueError:
                            print('Docker build failed.')
                            raise
                    build_message = "Image {0} built".format(image)
                    if(args.push):
                        exit_code = dockerUtilsObj.docker_push(image)
                        if exit_code != 0:
                            raise ValueError(
                                'Docker push failed.')
                        build_message += " and pushed to registry {}".format(roger_env[
                                                                             'registry'])
                    print(build_message)
                except (IOError) as e:
                    print("The folowing error occurred.(Error: %s).\n" %
                          e, file=sys.stderr)
                    raise
            else:
                print("Dockerfile does not exist in dir: {}".format(file_path))

            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "post_build"
            hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"The following error occurred: %s" %
                  e, file=sys.stderr)
            raise
        finally:
            try:
                # If the build fails before going through any steps
                if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
                    function_execution_start_time = datetime.now()

                if 'execution_result' not in globals() and 'execution_result' not in locals():
                    execution_result = 'FAILURE'

                if 'config_name' not in globals() and 'config_name' not in locals():
                    config_name = ""

                if not hasattr(args, "env"):
                    args.env = "dev"

                if not hasattr(args, "app_name"):
                    args.app_name = ""

                if 'settingObj' not in globals() and 'settingObj' not in locals():
                    settingObj = Settings()

                if 'execution_result' is 'FAILURE':
                    self.outcome = 0

                sc = self.utils.getStatsClient()
                if not hasattr(self, "identifier"):
                    self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
                time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
                input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=build" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"The following error occurred: %s" %
                      e, file=sys.stderr)
                raise
Exemplo n.º 8
0
class RogerBuild(object):
    def __init__(self):
        self.utils = Utils()
        self.outcome = 1
        self.registry = ""
        self.tag_name = ""

    def parse_args(self):
        self.parser = argparse.ArgumentParser(prog='roger build',
                                              description=describe())
        self.parser.add_argument(
            'app_name',
            metavar='app_name',
            help="application to build. Example: 'agora'.")
        self.parser.add_argument(
            'directory',
            metavar='directory',
            help=
            "App Repo will be checked out here, this is the working dir CLI will use."
            "A temporary directory is created if no directory specified."
            "Example: '/home/vagrant/work_dir'.")
        self.parser.add_argument(
            'tag_name',
            metavar='tag_name',
            help="tag for the built image. Example: 'roger-collectd:0.20'.")
        self.parser.add_argument(
            'config_file',
            metavar='config_file',
            help="configuration file to use. Example: 'content.json'.")
        self.parser.add_argument(
            '-v',
            '--verbose',
            help="verbose mode for debugging. Defaults to false.",
            action="store_true")
        self.parser.add_argument(
            '--push',
            '-p',
            help="Also push to registry. Defaults to false.",
            action="store_true")
        self.parser.add_argument(
            '--build-arg',
            action='append',
            help=
            'docker build-arg; Use flags multiple times to pass more than one arg'
        )
        self.parser.add_argument('-ns',
                                 '--disable-swaparoo',
                                 help="Disables swaparoo functionality",
                                 action="store_true")
        return self.parser

    def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj,
             args):
        print(colored("******Building the Docker image now******", "grey"))
        try:
            config_dir = settingObj.getConfigDir()
            root = settingObj.getCliDir()
            config = appObj.getConfig(config_dir, args.config_file)
            hooksObj.config_file = args.config_file
            roger_env = appObj.getRogerEnv(config_dir)
            config_name = ""
            if 'name' in config:
                config_name = config['name']
            common_repo = config.get('repo', '')
            if not hasattr(args, "env"):
                args.env = "dev"
            data = appObj.getAppData(config_dir, args.config_file,
                                     args.app_name)
            if not data:
                raise ValueError(
                    "Application with name [{}] or data for it not found at {}/{}."
                    .format(args.app_name, config_dir, args.config_file))
            repo = ''
            if common_repo != '':
                repo = data.get('repo', common_repo)
            else:
                repo = data.get('repo', args.app_name)

            docker_build_args = {}

            if 'build-args' in data:
                if 'environment' in data['build-args']:
                    if args.env in data['build-args']['environment']:
                        docker_build_args = data['build-args']['environment'][
                            args.env]

            # read the build-args from commandline like docker does as well
            # build-args defined on command line will override the ones from the config file, for the same keys
            # so this update of dictionary has to be done after we have read build arg values from the config file
            if args.build_arg:
                docker_build_args.update(
                    dict(
                        arg_key_val_str.split('=')
                        for arg_key_val_str in args.build_arg))

            projects = data.get('privateProjects', [])

            # get/update target source(s)
            file_exists = True
            file_path = ''
            cur_dir = ''
            if "PWD" in os.environ:
                cur_dir = os.environ.get('PWD')

            # This is bad code, assuming current directory and then trying to again guess, this is not rocket science
            # it's a f*****g file path, as simple as that. https://seomoz.atlassian.net/browse/ROGER-2405
            # dockerfile location possibilities
            #    1. Path relative to the repo, we know repo path for cli is <checkout_dir>/<repo>
            #    2. Absolute path
            # This path comes from config file and not passed on commandline so we should not try to prefix current
            # working directory if the relative path is passed, don't try to guess too much.
            # changelog : relative path from current directory won't work for working_directory or checkout_dir
            # changelog : working_directory or checkout_dir should be absolute path, not backward-compatible
            checkout_dir = os.path.abspath(args.directory)
            repo_name = appObj.getRepoName(repo)
            # (vmahedia) todo : this should be called docker_file_dir
            dockerfile_rel_repo_path = data.get('path', '')
            file_path = os.path.join(checkout_dir, repo_name,
                                     dockerfile_rel_repo_path)

            if not hasattr(args, "app_name"):
                args.app_name = ""

            if not hasattr(self, "identifier"):
                self.identifier = self.utils.get_identifier(
                    config_name, settingObj.getUser(), args.app_name)

            args.app_name = self.utils.extract_app_name(args.app_name)
            hookname = "pre_build"
            exit_code = hooksObj.run_hook(hookname, data, file_path, args.env,
                                          settingObj.getUser())
            if exit_code != 0:
                raise ValueError("{} hook failed.".format(hookname))

            build_filename = 'Dockerfile'

            if 'build_filename' in data:
                build_filename = ("{0}/{1}".format(file_path,
                                                   data['build_filename']))
                file_exists = os.path.exists(build_filename)
                if not file_exists:
                    raise ValueError(
                        "Specified build file: {} does not exist. Exiting build."
                        .format(build_filename))
            else:
                file_exists = os.path.exists(
                    "{0}/Dockerfile".format(file_path))

            if file_exists:
                # (vmahedia) todo: We know what parameters are required for build command so we should not wait until
                # now to bailout. Config parser should have a validator for every command to see if all the Required
                # parameters are passed or not. Why do all this up to this point if we know we will fail on this.
                # RequiredParameter, below, "registry"
                if 'registry' not in roger_env:
                    raise ValueError(
                        "Registry not found in roger-mesos-tools.config file.")
                else:
                    self.registry = roger_env['registry']
                self.tag_name = args.tag_name
                image = "{0}/{1}".format(roger_env['registry'], args.tag_name)
                try:
                    if checkout_dir == args.directory:
                        try:
                            dockerObj.docker_build(
                                dockerUtilsObj, appObj, args.directory, repo,
                                projects, dockerfile_rel_repo_path, image,
                                docker_build_args, args.verbose,
                                build_filename, args.disable_swaparoo)
                        except ValueError:
                            raise ValueError("Docker build failed")
                    else:
                        directory = os.path.join(cur_dir, args.directory)
                        try:
                            dockerObj.docker_build(
                                dockerUtilsObj, appObj, directory, repo,
                                projects, dockerfile_rel_repo_path, image,
                                docker_build_args, args.verbose,
                                build_filename, args.disable_swaparoo)
                        except ValueError:
                            print('Docker build failed.')
                            raise
                    print(
                        colored("******Successfully built Docker image******",
                                "green"))
                    build_message = "Image [{}]".format(image)
                    if (args.push):
                        print(
                            colored(
                                "******Pushing Docker image to registry******",
                                "grey"))
                        exit_code = dockerUtilsObj.docker_push(
                            image, args.verbose)
                        if exit_code != 0:
                            raise ValueError('Docker push failed.')
                        build_message += " successfully pushed to registry [{}]*******".format(
                            roger_env['registry'])
                    print(colored(build_message, "green"))
                except (IOError) as e:
                    printException(e)
                    raise
            else:
                print(
                    colored(
                        "Dockerfile does not exist in dir: {}".format(
                            file_path), "red"))

            hookname = "post_build"
            exit_code = hooksObj.run_hook(hookname, data, file_path, args.env,
                                          settingObj.getUser())
            if exit_code != 0:
                raise ValueError('{} hook failed.'.format(hookname))
        except (Exception) as e:
            printException(e)
            raise
        finally:
            # todo: maybe send a datadog event?
            pass
Exemplo n.º 9
0
class RogerBuild(object):

    def __init__(self):
        self.utils = Utils()
        self.statsd_message_list = []
        self.outcome = 1
        self.registry = ""
        self.tag_name = ""

    def parse_args(self):
        self.parser = argparse.ArgumentParser(
            prog='roger build', description=describe())
        self.parser.add_argument('app_name', metavar='app_name',
                                 help="application to build. Example: 'agora'.")
        self.parser.add_argument('directory', metavar='directory',
                                 help="working directory. Example: '/home/vagrant/work_dir'.")
        self.parser.add_argument('tag_name', metavar='tag_name',
                                 help="tag for the built image. Example: 'roger-collectd:0.20'.")
        self.parser.add_argument('config_file', metavar='config_file',
                                 help="configuration file to use. Example: 'content.json'.")
        self.parser.add_argument('-v', '--verbose', help="verbose mode for debugging. Defaults to false.", action="store_true")
        self.parser.add_argument(
            '--push', '-p', help="Also push to registry. Defaults to false.", action="store_true")
        return self.parser

    def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj, args):
        print(colored("******Building the Docker image now******", "grey"))
        try:
            function_execution_start_time = datetime.now()
            execution_result = 'SUCCESS'  # Assume the execution_result to be SUCCESS unless exception occurs
            config_dir = settingObj.getConfigDir()
            root = settingObj.getCliDir()
            config = appObj.getConfig(config_dir, args.config_file)
            hooksObj.config_file = args.config_file
            roger_env = appObj.getRogerEnv(config_dir)
            config_name = ""
            if 'name' in config:
                config_name = config['name']
            common_repo = config.get('repo', '')
            if not hasattr(args, "env"):
                args.env = "dev"
            data = appObj.getAppData(config_dir, args.config_file, args.app_name)
            if not data:
                raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format(
                    args.app_name, config_dir, args.config_file))
            repo = ''
            if common_repo != '':
                repo = data.get('repo', common_repo)
            else:
                repo = data.get('repo', args.app_name)

            build_args = {}
            if 'build-args' in data:
                if 'environment' in data['build-args']:
                    if args.env in data['build-args']['environment']:
                        build_args = data['build-args']['environment'][args.env]

            projects = data.get('privateProjects', [])


            # get/update target source(s)
            file_exists = True
            file_path = ''
            cur_dir = ''
            if "PWD" in os.environ:
                cur_dir = os.environ.get('PWD')


            # This is bad code, assuming current directory and then trying to again guess, this is not rocket science
            # it's a f*****g file path, as simple as that. https://seomoz.atlassian.net/browse/ROGER-2405
            # dockerfile location possibilities
            #    1. Path relative to the repo, we know repo path for cli is <checkout_dir>/<repo>
            #    2. Absolute path
            # This path comes from config file and not passed on commandline so we should not try to prefix current
            # working directory if the relative path is passed, don't try to guess too much.
            # changelog : relative path from current directory won't work for working_directory or checkout_dir
            # changelog : working_directory or checkout_dir should be absolute path, not backward-compatible
            checkout_dir = os.path.abspath(args.directory)
            repo_name = appObj.getRepoName(repo)
            # (vmahedia) todo : this should be called docker_file_dir 
            dockerfile_rel_repo_path = data.get('path', '')
            file_path = os.path.join(checkout_dir, repo_name, dockerfile_rel_repo_path)


            if not hasattr(args, "app_name"):
                args.app_name = ""

            if not hasattr(self, "identifier"):
                self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)

            args.app_name = self.utils.extract_app_name(args.app_name)
            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "pre_build"
            hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"{} hook failed.".format(hookname))

            build_filename = 'Dockerfile'

            if 'build_filename' in data:
                build_filename = ("{0}/{1}".format(file_path, data['build_filename']))
                file_exists = os.path.exists(build_filename)
                if not file_exists:
                    raise ValueError("Specified build file: {} does not exist. Exiting build.".format(build_filename))
            else:
                file_exists = os.path.exists("{0}/Dockerfile".format(file_path))

            if file_exists:
                # (vmahedia) todo: We know what parameters are required for build command so we should not wait until
                # now to bailout. Config parser should have a validator for every command to see if all the Required
                # parameters are passed or not. Why do all this up to this point if we know we will fail on this.
                # RequiredParameter, below, "registry"
                if 'registry' not in roger_env:
                    raise ValueError("Registry not found in roger-mesos-tools.config file.")
                else:
                    self.registry = roger_env['registry']
                self.tag_name = args.tag_name
                image = "{0}/{1}".format(roger_env['registry'], args.tag_name)
                try:
                    if checkout_dir == args.directory:
                        try:
                            dockerObj.docker_build(
                                dockerUtilsObj, appObj, args.directory, repo, projects, dockerfile_rel_repo_path, image, build_args, args.verbose, build_filename)
                        except ValueError:
                            raise ValueError("Docker build failed")
                    else:
                        directory = '{0}/{1}'.format(cur_dir, args.directory)
                        try:
                            dockerObj.docker_build(
                                dockerUtilsObj, appObj, directory, repo, projects, dockerfile_rel_repo_path, image, build_args, args.verbose, build_filename)
                        except ValueError:
                            print('Docker build failed.')
                            raise
                    print(colored("******Successfully built Docker image******", "green"))
                    build_message = "Image [{}]".format(image)
                    if(args.push):
                        print(colored("******Pushing Docker image to registry******", "grey"))
                        exit_code = dockerUtilsObj.docker_push(image, args.verbose)
                        if exit_code != 0:
                            raise ValueError(
                                'Docker push failed.')
                        build_message += " successfully pushed to registry [{}]*******".format(roger_env[
                                                                             'registry'])
                    print(colored(build_message, "green"))
                except (IOError) as e:
                    printException(e)
                    raise
            else:
                print(colored("Dockerfile does not exist in dir: {}".format(file_path), "red"))

            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "post_build"
            hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******""

                if not hasattr(args, "env"):
                    args.env = "dev"

                if not hasattr(args, "app_name"):
                    args.app_name = ""

                if 'settingObj' not in globals() and 'settingObj' not in locals():
                    settingObj = Settings()

                if 'execution_result' is 'FAILURE':
                    self.outcome = 0

                sc = self.utils.getStatsClient()
                if not hasattr(self, "identifier"):
                    self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
                time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
                input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=build" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user=" + str(settingObj.getUser())
                tup = (input_metric, time_take_milliseonds)
                self.statsd_message_list.append(tup)
            except (Exception) as e:
                printException(e)
                raise
Exemplo n.º 10
0
class RogerBuild(object):

    def __init__(self):
        self.utils = Utils()
        self.outcome = 1
        self.registry = ""
        self.tag_name = ""

    def parse_args(self):
        self.parser = argparse.ArgumentParser(prog='roger build', description=describe())
        self.parser.add_argument('app_name', metavar='app_name',
                                 help="application to build. Example: 'agora'.")
        self.parser.add_argument('directory', metavar='directory',
                                 help="App Repo will be checked out here, this is the working dir CLI will use."
                                      "A temporary directory is created if no directory specified."
                                      "Example: '/home/vagrant/work_dir'.")
        self.parser.add_argument('tag_name', metavar='tag_name',
                                 help="tag for the built image. Example: 'roger-collectd:0.20'.")
        self.parser.add_argument('config_file', metavar='config_file',
                                 help="configuration file to use. Example: 'content.json'.")
        self.parser.add_argument('-v', '--verbose', help="verbose mode for debugging. Defaults to false.", action="store_true")
        self.parser.add_argument('--push', '-p', help="Also push to registry. Defaults to false.", action="store_true")
        self.parser.add_argument('--build-arg', action='append',
                                 help='docker build-arg; Use flags multiple times to pass more than one arg')
        self.parser.add_argument('-ns', '--disable-swaparoo', help="Disables swaparoo functionality", action="store_true")
        return self.parser

    def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj, args):
        print(colored("******Building the Docker image now******", "grey"))
        try:
            config_dir = settingObj.getConfigDir()
            root = settingObj.getCliDir()
            config = appObj.getConfig(config_dir, args.config_file)
            hooksObj.config_file = args.config_file
            roger_env = appObj.getRogerEnv(config_dir)
            config_name = ""
            if 'name' in config:
                config_name = config['name']
            common_repo = config.get('repo', '')
            if not hasattr(args, "env"):
                args.env = "dev"
            data = appObj.getAppData(config_dir, args.config_file, args.app_name)
            if not data:
                raise ValueError("Application with name [{}] or data for it not found at {}/{}.".format(
                    args.app_name, config_dir, args.config_file))
            repo = ''
            if common_repo != '':
                repo = data.get('repo', common_repo)
            else:
                repo = data.get('repo', args.app_name)

            docker_build_args = {}

            if 'build-args' in data:
                if 'environment' in data['build-args']:
                    if args.env in data['build-args']['environment']:
                        docker_build_args = data['build-args']['environment'][args.env]

            # read the build-args from commandline like docker does as well
            # build-args defined on command line will override the ones from the config file, for the same keys
            # so this update of dictionary has to be done after we have read build arg values from the config file
            if args.build_arg:
                docker_build_args.update(dict(arg_key_val_str.split('=') for arg_key_val_str in args.build_arg))

            projects = data.get('privateProjects', [])


            # get/update target source(s)
            file_exists = True
            file_path = ''
            cur_dir = ''
            if "PWD" in os.environ:
                cur_dir = os.environ.get('PWD')


            # This is bad code, assuming current directory and then trying to again guess, this is not rocket science
            # it's a f*****g file path, as simple as that. https://seomoz.atlassian.net/browse/ROGER-2405
            # dockerfile location possibilities
            #    1. Path relative to the repo, we know repo path for cli is <checkout_dir>/<repo>
            #    2. Absolute path
            # This path comes from config file and not passed on commandline so we should not try to prefix current
            # working directory if the relative path is passed, don't try to guess too much.
            # changelog : relative path from current directory won't work for working_directory or checkout_dir
            # changelog : working_directory or checkout_dir should be absolute path, not backward-compatible
            checkout_dir = os.path.abspath(args.directory)
            repo_name = appObj.getRepoName(repo)
            # (vmahedia) todo : this should be called docker_file_dir
            dockerfile_rel_repo_path = data.get('path', '')
            file_path = os.path.join(checkout_dir, repo_name, dockerfile_rel_repo_path)


            if not hasattr(args, "app_name"):
                args.app_name = ""

            if not hasattr(self, "identifier"):
                self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)

            args.app_name = self.utils.extract_app_name(args.app_name)
            hookname = "pre_build"
            exit_code = hooksObj.run_hook(hookname, data, file_path, args.env, settingObj.getUser())
            if exit_code != 0:
                raise ValueError("{} hook failed.".format(hookname))

            build_filename = 'Dockerfile'

            if 'build_filename' in data:
                build_filename = ("{0}/{1}".format(file_path, data['build_filename']))
                file_exists = os.path.exists(build_filename)
                if not file_exists:
                    raise ValueError("Specified build file: {} does not exist. Exiting build.".format(build_filename))
            else:
                file_exists = os.path.exists("{0}/Dockerfile".format(file_path))

            if file_exists:
                # (vmahedia) todo: We know what parameters are required for build command so we should not wait until
                # now to bailout. Config parser should have a validator for every command to see if all the Required
                # parameters are passed or not. Why do all this up to this point if we know we will fail on this.
                # RequiredParameter, below, "registry"
                if 'registry' not in roger_env:
                    raise ValueError("Registry not found in roger-mesos-tools.config file.")
                else:
                    self.registry = roger_env['registry']
                self.tag_name = args.tag_name
                image = "{0}/{1}".format(roger_env['registry'], args.tag_name)
                try:
                    if checkout_dir == args.directory:
                        try:
                            dockerObj.docker_build(
                                dockerUtilsObj, appObj, args.directory, repo, projects, dockerfile_rel_repo_path, image, docker_build_args, args.verbose, build_filename, args.disable_swaparoo)
                        except ValueError:
                            raise ValueError("Docker build failed")
                    else:
                        directory = os.path.join(cur_dir, args.directory)
                        try:
                            dockerObj.docker_build(
                                dockerUtilsObj, appObj, directory, repo, projects, dockerfile_rel_repo_path, image, docker_build_args, args.verbose, build_filename, args.disable_swaparoo)
                        except ValueError:
                            print('Docker build failed.')
                            raise
                    print(colored("******Successfully built Docker image******", "green"))
                    build_message = "Image [{}]".format(image)
                    if(args.push):
                        print(colored("******Pushing Docker image to registry******", "grey"))
                        exit_code = dockerUtilsObj.docker_push(image, args.verbose)
                        if exit_code != 0:
                            raise ValueError(
                                'Docker push failed.')
                        build_message += " successfully pushed to registry [{}]*******".format(roger_env[
                                                                             'registry'])
                    print(colored(build_message, "green"))
                except (IOError) as e:
                    printException(e)
                    raise
            else:
                print(colored("Dockerfile does not exist in dir: {}".format(file_path), "red"))

            hookname = "post_build"
            exit_code = hooksObj.run_hook(hookname, data, file_path, args.env, settingObj.getUser())
            if exit_code != 0:
                raise ValueError('{} hook failed.'.format(hookname))
        except (Exception) as e:
            printException(e)
            raise
        finally:
            # todo: maybe send a datadog event? 
            pass
Exemplo n.º 11
0
class RogerPush(object):
    def __init__(self):
        self.utils = Utils()
        self.task_id = []
        self.statsd_message_list = []
        self.statsd_push_list = []
        self.outcome = 1
        self.registry = ""
        self.image_name = ""

    def parse_args(self):
        self.parser = argparse.ArgumentParser(prog='roger push',
                                              description=describe())
        self.parser.add_argument(
            'app_name',
            metavar='app_name',
            help="application to push. Can also push specific"
            " containers(comma seperated). Example: 'agora' or 'app_name:container1,container2'"
        )
        self.parser.add_argument(
            '-e',
            '--env',
            metavar='env',
            help="environment to push to. Example: 'dev' or 'prod'")
        self.parser.add_argument(
            'directory',
            metavar='directory',
            help="working directory. Example: '/home/vagrant/work_dir'")
        self.parser.add_argument(
            'image_name',
            metavar='image_name',
            help=
            "image name that includes version to use. Example: 'roger-collectd-v0.20' or 'elasticsearch-v0.07'"
        )
        self.parser.add_argument(
            'config_file',
            metavar='config_file',
            help=
            "configuration file to use. Example: 'content.json' or 'kwe.json'")
        self.parser.add_argument(
            '--skip-push',
            '-s',
            help=
            "skips push. Only generates components for review. Defaults to false.",
            action="store_true")
        self.parser.add_argument(
            '--force-push',
            '-f',
            help=
            "force push. Not Recommended. Forces push even if validation checks failed. Defaults to false.",
            action="store_true")
        self.parser.add_argument(
            '--secrets-file',
            '-S',
            help=
            "specifies an optional secrets file for deploy runtime variables.")
        return self.parser

    def loadSecrets(self, secrets_dir, file_name, args, environment):
        if args.secrets_file is not None:
            print("Using specified secrets file: {}".format(args.secrets_file))
            file_name = args.secrets_file
        exists = os.path.exists(secrets_dir)
        if exists is False:
            os.makedirs(secrets_dir)

        # Two possible paths -- first without environment, second with
        path1 = "{}/{}".format(secrets_dir, file_name)
        path2 = "{}/{}/{}".format(secrets_dir, environment, file_name)
        print(" Loading secrets from {} or {}".format(path1, path2))

        try:
            with open(path1) as f:
                return_file = yaml.load(f) if path1.lower().endswith(
                    '.yml') else json.load(f)
            return return_file
        except IOError:
            pass
        except ValueError as e:
            raise ValueError(" Error while loading json from {} - {}".format(
                path1, e))

        try:
            with open(path2) as f:
                return_file = yaml.load(f) if path2.lower().endswith(
                    '.yml') else json.load(f)
            return return_file
        except IOError:
            print(" Couldn't load secrets file environment in %s or %s\n" %
                  (path1, path2),
                  file=sys.stderr)
            return {}
        except ValueError as e:
            raise ValueError(" Error while loading json from {} - {}".format(
                path2, e))

    def replaceSecrets(self, output_dict, secrets_dict):
        if type(output_dict) is not dict:
            return output_dict

        for key in output_dict:
            if output_dict[key] == "SECRET":
                if key in secrets_dict.keys():
                    output_dict[key] = secrets_dict[key]

            if type(output_dict[key]) is list:
                temp_list = []
                for list_elem in output_dict[key]:
                    temp_list.append(
                        self.replaceSecrets(list_elem, secrets_dict))
                    output_dict[key] = temp_list

            if type(output_dict[key]) is dict:
                temp_dict = self.replaceSecrets(output_dict[key], secrets_dict)
                output_dict[key] = temp_dict

        return output_dict

    def mergeSecrets(self, json_str, secrets):
        '''Given a JSON string and an object of secret environment variables, replaces
        parses the JSON keys with the secret variables. Returns back
        a JSON string. Raises an error if there are any SECRET variables still exists.'''
        print(
            "WARNING - The use of \"SECRET\" is deprecated. Please switch to using Jinja variables. To do so,"
            " use '{{ <actual variable name> }}' instead of \"SECRET\" in the target file."
        )
        output_dict = json.loads(json_str)
        json_str = json.dumps(self.replaceSecrets(output_dict, secrets),
                              indent=4)

        if '\"SECRET\"' in json_str:
            print(
                'There are still "SECRET" values -- does your secrets file have all secret environment variables?'
            )
            return "StandardError"
        return json_str

    def renderTemplate(self, template, environment, image, app_data, config,
                       container, container_name, additional_vars):

        variables = {'environment': environment, 'image': image}

        # Copy variables from config-wide, app-wide, then container-wide variable
        # configs, each one from "global" and then environment-specific.
        for obj in [config, app_data, container]:
            if type(obj) == dict and 'vars' in obj:
                variables.update(obj['vars'].get('global', {}))
                variables.update(obj['vars'].get('environment',
                                                 {}).get(environment, {}))

        variables.update(additional_vars)

        return template.render(variables)

    def statsd_counter_logging(self, metric):
        sc = self.utils.getStatsClient()
        sc.incr(metric, 1)

    def repo_relative_path(self, appConfig, args, repo, path):
        '''Returns a path relative to the repo, assumed to be under [args.directory]/[repo name]'''
        repo_name = appConfig.getRepoName(repo)
        abs_path = os.path.abspath(args.directory)
        if abs_path == args.directory:
            return "{0}/{1}/{2}".format(args.directory, repo_name, path)
        else:
            return "{0}/{1}/{2}/{3}".format(os.environ.get('PWD', ''),
                                            args.directory, repo_name, path)

    def main(self, settings, appConfig, frameworkObject, hooksObj, args):
        try:
            settingObj = settings
            appObj = appConfig
            frameworkUtils = frameworkObject
            config_dir = settingObj.getConfigDir()
            hooksObj.config_file = args.config_file
            cur_file_path = os.path.dirname(os.path.realpath(__file__))
            config = appObj.getConfig(config_dir, args.config_file)
            config_name = ""
            act_as_user = ""
            if 'name' in config:
                config_name = config['name']
            if 'act-as' in config:
                act_as_user = config['act-as']
            roger_env = appObj.getRogerEnv(config_dir)

            if not hasattr(args, "app_name"):
                args.app_name = ""

            if 'registry' not in roger_env.keys():
                raise ValueError(
                    'Registry not found in roger-mesos-tools.config file.')
            else:
                self.registry = roger_env['registry']

            if hasattr(args, "image_name"):
                self.image_name = args.image_name

            environment = roger_env.get('default_environment', '')
            if args.env is None:
                if "ROGER_ENV" in os.environ:
                    env_var = os.environ.get('ROGER_ENV')
                    if env_var.strip() == '':
                        print(
                            "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file"
                        )
                    else:
                        print(
                            "Using value {} from environment variable $ROGER_ENV"
                            .format(env_var))
                        environment = env_var
            else:
                environment = args.env

            if environment not in roger_env['environments']:
                raise ValueError(
                    'Environment not found in roger-mesos-tools.config file.')

            environmentObj = roger_env['environments'][environment]
            common_repo = config.get('repo', '')
            app_name = args.app_name
            container_list = []
            if ':' in app_name:
                tokens = app_name.split(':')
                app_name = tokens[0]
                if ',' in tokens[1]:
                    container_list = tokens[1].split(',')
                else:
                    container_list.append(tokens[1])

            data = appObj.getAppData(config_dir, args.config_file, app_name)
            if not data:
                raise ValueError(
                    'Application with name [{}] or data for it not found at {}/{}.'
                    .format(app_name, config_dir, args.config_file))

            configured_container_list = []
            for task in data['containers']:
                if type(task) == dict:
                    configured_container_list.append(task.keys()[0])
                else:
                    configured_container_list.append(task)
            if not set(container_list) <= set(configured_container_list):
                raise ValueError(
                    'List of containers [{}] passed do not match list of acceptable containers: [{}]'
                    .format(container_list, configured_container_list))

            frameworkObj = frameworkUtils.getFramework(data)
            framework = frameworkObj.getName()

            repo = ''
            if common_repo != '':
                repo = data.get('repo', common_repo)
            else:
                repo = data.get('repo', app_name)

            comp_dir = settingObj.getComponentsDir()
            templ_dir = settingObj.getTemplatesDir()
            secrets_dir = settingObj.getSecretsDir()

            # Create comp_dir if it doesn't exist
            if not os.path.isdir(comp_dir):
                os.makedirs(comp_dir)

            # template marathon files
            if not container_list:
                data_containers = data['containers']
            else:
                data_containers = container_list

            failed_container_dict = {}

            template = ''
            # Required for when work_dir,component_dir,template_dir or
            # secret_env_dir is something like '.' or './temp"
            os.chdir(cur_file_path)
            app_path = ''
            if 'template_path' in data:
                app_path = self.repo_relative_path(appObj, args, repo,
                                                   data['template_path'])
            else:
                app_path = templ_dir

            extra_vars = {}
            if 'extra_variables_path' in data:
                ev_path = self.repo_relative_path(appObj, args, repo,
                                                  data['extra_variables_path'])
                with open(ev_path) as f:
                    extra_vars = yaml.load(f) if ev_path.lower().endswith(
                        '.yml') else json.load(f)

            if not app_path.endswith('/'):
                app_path = app_path + '/'

            if not hasattr(self, "identifier"):
                self.identifier = self.utils.get_identifier(
                    config_name, settingObj.getUser(), args.app_name)

            args.app_name = self.utils.extract_app_name(args.app_name)
            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "pre_push"
            hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(
                args.app_name) + ",identifier=" + str(
                    self.identifier) + ",config_name=" + str(
                        config_name) + ",env=" + str(
                            environment) + ",user="******"{0}-{1}.json".format(
                        config['name'], container_name)
                else:
                    container_name = container
                    containerConfig = "{0}-{1}.json".format(
                        config['name'], container)

                env = Environment(loader=FileSystemLoader(
                    "{}".format(app_path)),
                                  undefined=StrictUndefined)
                template_with_path = "[{}{}]".format(app_path, containerConfig)
                try:
                    template = env.get_template(containerConfig)
                except exceptions.TemplateNotFound as e:
                    raise ValueError(
                        "The template file {} does not exist".format(
                            template_with_path))
                except Exception as e:
                    raise ValueError(
                        "Error while reading template from {} - {}".format(
                            template_with_path, e))

                additional_vars = {}
                additional_vars.update(extra_vars)
                secret_vars = self.loadSecrets(secrets_dir, containerConfig,
                                               args, environment)
                additional_vars.update(secret_vars)

                image_path = "{0}/{1}".format(roger_env['registry'],
                                              args.image_name)
                print(
                    "Rendering content from template {} for environment [{}]".
                    format(template_with_path, environment))
                try:
                    output = self.renderTemplate(template, environment,
                                                 image_path, data, config,
                                                 container, container_name,
                                                 additional_vars)
                except exceptions.UndefinedError as e:
                    error_str = "The following error occurred. %s.\n" % e
                    print(error_str, file=sys.stderr)
                    failed_container_dict[container_name] = error_str
                    pass

                # Adding check to see if all jinja variables git resolved fot the
                # container
                if container_name not in failed_container_dict:
                    # Adding check so that not all apps try to mergeSecrets
                    try:
                        outputObj = json.loads(output)
                    except Exception as e:
                        raise ValueError(
                            "Error while loading json from {} - {}".format(
                                template_with_path, e))

                    if 'SECRET' in output:
                        output = self.mergeSecrets(
                            output,
                            self.loadSecrets(secrets_dir, containerConfig,
                                             args, environment))
                    if output != "StandardError":
                        try:
                            comp_exists = os.path.exists(
                                "{0}".format(comp_dir))
                            if comp_exists is False:
                                os.makedirs("{0}".format(comp_dir))
                            comp_env_exists = os.path.exists("{0}/{1}".format(
                                comp_dir, environment))
                            if comp_env_exists is False:
                                os.makedirs("{0}/{1}".format(
                                    comp_dir, environment))
                        except Exception as e:
                            logging.error(traceback.format_exc())
                        with open(
                                "{0}/{1}/{2}".format(comp_dir, environment,
                                                     containerConfig),
                                'wb') as fh:
                            fh.write(output)

            if args.skip_push:
                print(
                    "Skipping push to {} framework. The rendered config file(s) are under {}/{}"
                    .format(framework, comp_dir, environment))
            else:
                # push to roger framework
                if 'owner' in config:
                    frameworkObj.act_as_user = config['owner']

                tools_version_value = self.utils.get_version()
                image_name = self.registry + "/" + args.image_name
                image_tag_value = urllib.quote("'" + image_name + "'")

                for container in data_containers:
                    try:
                        function_execution_start_time = datetime.now()
                        execution_result = 'SUCCESS'  # Assume the execution_result to be SUCCESS unless exception occurs
                        sc = self.utils.getStatsClient()
                    except (Exception) as e:
                        print("The following error occurred: %s" % e,
                              file=sys.stderr)
                    try:
                        if type(container) == dict:
                            container_name = str(container.keys()[0])
                            containerConfig = "{0}-{1}.json".format(
                                config['name'], container_name)
                        else:
                            container_name = container
                            containerConfig = "{0}-{1}.json".format(
                                config['name'], container)

                        if container_name in failed_container_dict:
                            print(
                                "Failed push to {} framework for container {} as unresolved Jinja variables present in template."
                                .format(framework, container_name))
                        else:
                            config_file_path = "{0}/{1}/{2}".format(
                                comp_dir, environment, containerConfig)

                            result = frameworkObj.runDeploymentChecks(
                                config_file_path, environment)

                            if args.force_push or result is True:
                                resp, task_id = frameworkObj.put(
                                    config_file_path, environmentObj,
                                    container_name, environment, act_as_user)

                                container_task_id = self.utils.modify_task_id(
                                    task_id)
                                self.task_id.extend(container_task_id)

                                if hasattr(resp, "status_code"):
                                    status_code = resp.status_code
                            else:
                                print(
                                    "Skipping push to {} framework for container {} as Validation Checks failed."
                                    .format(framework, container))
                    except (Exception) as e:
                        print("The following error occurred: %s" % e,
                              file=sys.stderr)
                        execution_result = 'FAILURE'
                        raise
                    finally:
                        try:

                            if 'function_execution_start_time' not in globals(
                            ) and 'function_execution_start_time' not in locals(
                            ):
                                function_execution_start_time = datetime.now()

                            if 'execution_result' not in globals(
                            ) and 'execution_result' not in locals():
                                execution_result = 'FAILURE'

                            if 'config_name' not in globals(
                            ) and 'config_name' not in locals():
                                config_name = ""

                            if 'environment' not in globals(
                            ) and 'environment' not in locals():
                                environment = "dev"

                            if 'container_name' not in globals(
                            ) and 'container_name' not in locals():
                                container_name = ""

                            if 'status_code' not in globals(
                            ) and 'status_code' not in locals():
                                status_code = "500"

                            if not hasattr(args, "app_name"):
                                args.app_name = ""

                            if 'settingObj' not in globals(
                            ) and 'settingObj' not in locals():
                                settingObj = Settings()

                            if 'container_task_id' not in globals(
                            ) and 'container_task_id' not in locals():
                                container_task_id = []

                            if not hasattr(self, "identifier"):
                                self.identifier = self.utils.get_identifier(
                                    config_name, settingObj.getUser(),
                                    args.app_name)

                            if not str(status_code).startswith("20"):
                                execution_result = 'FAILURE'
                                self.outcome = 0

                            time_take_milliseonds = (
                                (datetime.now() - function_execution_start_time
                                 ).total_seconds() * 1000)
                            for task_id in container_task_id:
                                input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(
                                    args.app_name
                                ) + ",event=push" + ",container_name=" + str(
                                    container_name) + ",identifier=" + str(
                                        self.identifier) + ",outcome=" + str(
                                            execution_result
                                        ) + ",response_code=" + str(
                                            status_code
                                        ) + ",config_name=" + str(
                                            config_name) + ",env=" + str(
                                                environment) + ",user="******",task_id=" + str(
                                                    task_id
                                                ) + ",tools_version=" + str(
                                                    tools_version_value
                                                ) + ",image_tag=" + str(
                                                    image_tag_value)
                                tup = (input_metric, time_take_milliseonds)
                                self.statsd_push_list.append(tup)

                                if str(status_code).startswith("20"):
                                    metric = input_metric.replace(
                                        "rogeros_tools_exec_time",
                                        "rogeros_events")
                                    metric = metric + ",source=tools" + ",task_id=" + task_id
                                    self.statsd_counter_logging(metric)

                        except (Exception) as e:
                            print("The following error occurred: %s" % e,
                                  file=sys.stderr)
                            raise

            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "post_push"
            hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(
                args.app_name) + ",identifier=" + str(
                    self.identifier) + ",config_name=" + str(
                        config_name) + ",env=" + str(
                            environment) + ",user="******"The following error occurred: %s" % e, file=sys.stderr)
            raise