Exemple #1
0
 def setUp(self):
     self.gitObj = GitUtils()
     self.settingObj = Settings()
     self.base_dir = self.settingObj.getCliDir()
     self.configs_dir = self.base_dir + "/tests/configs"
     self.work_dir = self.base_dir + "/tests/work_dir"
     self.branch = "master"
Exemple #2
0
    def setUp(self):
        parser = argparse.ArgumentParser(description='Args for test')
        parser.add_argument(
            '-e',
            '--env',
            metavar='env',
            help="Environment to deploy to. example: 'dev' or 'stage'")
        parser.add_argument(
            '--skip-push',
            '-s',
            help="Don't push. Only generate components. Defaults to false.",
            action="store_true")
        parser.add_argument(
            '--secrets-file',
            '-S',
            help=
            "Specify an optional secrets file for deploy runtime variables.")
        self.parser = parser
        self.args = parser

        self.settingObj = Settings()
        self.base_dir = self.settingObj.getCliDir()
        self.configs_dir = self.base_dir + "/tests/configs"

        with open(self.configs_dir +
                  '/roger_single_container_var_tests.json') as config:
            config = json.load(config)
        with open(self.configs_dir + '/roger-mesos-tools.config') as roger:
            roger_env = yaml.load(roger)
        data = config['apps']['container-vars']
        self.config = config
        self.roger_env = roger_env
        self.data = data
Exemple #3
0
    def _get_template_path(self,
                           container_name,
                           config_dir,
                           args,
                           app_name,
                           app_object=AppConfig(),
                           settings_object=Settings()):
        """
        Returns the template path for a given container_name

        Each framework requires an template_path for the app_id method

        :Params:
        :config_dir [str]: path to the config directory
        :args [argparse.NameSpace]: how to get acceses to the values passed
        :app_name [str]: name of app
        :app_object [cli.appconfig.AppConfig]: instance of AppConfig
        :settings_object [cli.settings.Settings]: instance of Settings

        """

        data = app_object.getConfig(config_dir, args.config)
        repo = self._config_resolver('repo', app_name, args.config)
        template_path = self._config_resolver('template_path', app_name,
                                              args.config)

        # this path is always relative to the root repo dir, so join
        if template_path and not os.path.isabs(template_path):
            app_path = os.path.join(self._temp_dir, repo, template_path)
        else:
            app_path = settings_object.getTemplatesDir()

        file_name = "{0}-{1}.json".format(data['name'], container_name)

        return os.path.join(app_path, file_name)
Exemple #4
0
    def setUp(self):
        parser = argparse.ArgumentParser(description='Args for test')
        parser.add_argument('app_name', metavar='app_name',
                            help="application to build. Example: 'agora'.")
        parser.add_argument('env', metavar='env',
                            help="environment. Example: 'test'.")
        parser.add_argument('directory', metavar='directory',
                            help="working directory. Example: '/home/vagrant/work_dir'.")
        parser.add_argument('tag_name', metavar='tag_name',
                            help="tag for the built image. Example: 'roger-collectd:0.20'.")
        parser.add_argument('config_file', metavar='config_file',
                            help="configuration file to use. Example: 'content.json'.")
        parser.add_argument(
            '--push', '-p', help="Also push to registry. Defaults to false.", action="store_true")

        self.parser = parser
        self.args = self.parser
        self.settingObj = Settings()
        self.base_dir = self.settingObj.getCliDir()
        self.configs_dir = self.base_dir + "/tests/configs"
        self.components_dir = self.base_dir + '/tests/components/dev'
        with open(self.configs_dir + '/app.json') as config:
            config = json.load(config)
        with open(self.configs_dir + '/roger-mesos-tools.config') as roger:
            roger_env = yaml.load(roger)
        data = config['apps']['grafana_test_app']
        self.config = config
        self.roger_env = roger_env
        self.data = data
 def setUp(self):
     self.webhook = WebHook()
     self.settingObj = Settings()
     self.base_dir = self.settingObj.getCliDir()
     self.configs_dir = self.base_dir + "/tests/configs"
     self.components_dir = self.base_dir + '/tests/components/dev'
     with open(self.configs_dir + '/roger-mesos-tools.config') as roger:
         roger_env = yaml.load(roger)
     self.roger_env = roger_env
 def setUp(self):
     self.settingObj = Settings()
     self.base_dir = self.settingObj.getCliDir()
     self.configs_dir = self.base_dir + "/tests/configs"
     self.components_dir = self.base_dir + '/tests/components/dev'
     self.configs_dir = self.base_dir + "/tests/configs"
     with open(self.configs_dir + '/app.json') as config:
         config = json.load(config)
     self.config = config
Exemple #7
0
def main():
    args = getCliArguments()
    settings = Settings(args.settings)
    machine = machineFactory(settings)

    output = []

    for letter in list(args.message):
        output.append(machine.encode(letter))

    print(''.join(output))
Exemple #8
0
 def get_haproxy_config(self, environment):
     haproxy_config = ""
     settingObj = Settings()
     appObj = AppConfig()
     config_dir = settingObj.getConfigDir()
     roger_env = appObj.getRogerEnv(config_dir)
     host = roger_env['environments'][environment]['host']
     haproxy_config_path = roger_env['environments'][
         environment]['haproxy_config_path']
     url = "{}{}".format(host, haproxy_config_path)
     haproxy_config = requests.get(url, stream=True)
     return haproxy_config.text
 def get_proxy_config(self, environment):
     proxy_config = ""
     settingObj = Settings()
     appObj = AppConfig()
     config_dir = settingObj.getConfigDir()
     roger_env = appObj.getRogerEnv(config_dir)
     host = roger_env['environments'][environment]['host']
     proxy_config_path = roger_env['environments'][environment][
         'proxy_config_path']
     url = "{}{}".format(host, proxy_config_path)
     proxy_config = requests.get(url).json()
     return proxy_config
Exemple #10
0
 def getStatsClient(self):
     settingObj = Settings()
     appObj = AppConfig()
     config_dir = settingObj.getConfigDir()
     roger_env = appObj.getRogerEnv(config_dir)
     statsd_url = ""
     statsd_port = ""
     if 'statsd_endpoint' in roger_env.keys():
         statsd_url = roger_env['statsd_endpoint']
     if 'statsd_port' in roger_env.keys():
         statsd_port = int(roger_env['statsd_port'])
     return statsd.StatsClient(statsd_url, statsd_port)
Exemple #11
0
 def __init__(self,
              app_config=AppConfig(),
              settings=Settings(),
              framework_utils=FrameworkUtils(),
              framework=Marathon()):
     self._app_config = app_config
     self._settings = settings
     self._framework_utils = framework_utils
     self._framework = framework
     self._config_dir = None
     self._roger_env = None
     self._temp_dir = None
Exemple #12
0
 def __init__(self):
     self.disabled = True
     self.emoji = ':rocket:'
     self.defChannel = ''
     self.config_dir = ''
     self.username = '******'
     self.settingObj = Settings()
     self.appconfigObj = AppConfig()
     self.configLoadFlag = False
     self.config = ''
     self.config_channels = []
     self.config_envs = []
     self.config_commands = []
Exemple #13
0
 def getRepoUrl(self, repo):
     if repo.startswith('*****@*****.**'):
         return repo
     else:
         settingObj = Settings()
         config_dir = settingObj.getConfigDir()
         roger_env = self.getRogerEnv(config_dir)
         if 'default_github_repo_prefix' in roger_env.keys():
             prefix = roger_env['default_github_repo_prefix']
         else:
             raise ValueError(
                 "Could not determine github repo.Please provide default \"github repo prefix\" or ensure repo startswith [email protected]"
             )
         return str(prefix + '{}.git'.format(repo))
    def setUp(self):
        parser = argparse.ArgumentParser(description='Args for test')
        parser.add_argument(
            'app_name',
            metavar='app_name',
            help=
            "application for which code is to be pulled. Example: 'agora' or 'grafana'"
        )
        parser.add_argument(
            'directory',
            metavar='directory',
            help=
            "working directory. The repo has its own directory it this. Example: '/home/vagrant/work_dir'"
        )
        parser.add_argument(
            '-b',
            '--branch',
            metavar='branch',
            help=
            "git branch to pull code from. Example: 'production' or 'master'. Defaults to master."
        )
        parser.add_argument(
            'config_file',
            metavar='config_file',
            help=
            "configuration file to use. Example: 'content.json' or 'kwe.json'")

        self.settingObj = Settings()
        self.base_dir = self.settingObj.getCliDir()
        self.configs_dir = self.base_dir + "/tests/configs"
        self.work_dir = self.base_dir + "/tests/work_dir"
        self.components_dir = self.base_dir + '/tests/components/dev'
        self.args = parser

        with open(self.configs_dir + '/app.json') as config:
            config = json.load(config)
        with open(self.configs_dir + '/roger-mesos-tools.config') as roger:
            roger_env = yaml.load(roger)

        data = config['apps']['grafana_test_app']
        self.roger_env = roger_env
        self.data = data
        self.config = config

        pass
Exemple #15
0
    def setUp(self):
        parser = argparse.ArgumentParser(description='Args for test')
        parser.add_argument('-e', '--environment', metavar='env',
                            help="Environment to deploy to. example: 'dev' or 'stage'")
        parser.add_argument('-s', '--skip-build', action="store_true",
                            help="Flag that skips roger-build when set to true. Defaults to false.'")
        parser.add_argument('-M', '--incr-major', action="store_true",
                            help="Increment major in version. Defaults to false.'")
        parser.add_argument('-p', '--incr-patch', action="store_true",
                            help="Increment patch in version. Defaults to false.'")
        parser.add_argument('-sp', '--skip-push', action="store_true",
                            help="Flag that skips roger push when set to true. Defaults to false.'")
        parser.add_argument(
            '-S', '--secrets-file', help="Specify an optional secrets file for deployment runtime variables.")
        parser.add_argument(
            '-d', '--directory', help="Specify an optional directory to pull out the repo. This is the working directory.")
        self.parser = parser
        self.args = parser

        self.settingObj = Settings()
        self.base_dir = self.settingObj.getCliDir()
        self.configs_dir = self.base_dir + "/tests/configs"

        config = {u'repo': u'roger', u'notifications': {u'username': u'Roger Deploy', u'method': u'chat.postMessage', u'channel': u'Channel ID', u'emoji': u':rocket:'},
                  u'apps': {u'test_app': {u'imageBase': u'test_app_base', u'name': u'test_app', u'containers': [u'container_name1', u'container_name2']},
                            u'test_app1': {u'framework': u'chronos', u'name': u'test_app1', u'containers': [u'container_name1', u'container_name2'], u'imageBase': u'test_app_base'},
                            u'grafana_test_app': {u'imageBase': u'test_app_base', u'name': u'test_app_grafana', u'containers': [u'grafana', {u'grafana1': {u'vars': {u'environment': {u'prod': {u'mem': u'2048', u'cpus': u'2'}, u'dev': {u'mem': u'512', u'cpus': u'0.5'}},
                                                                                                                                                                     u'global': {u'mem': u'128', u'cpus': u'0.1'}}}}, {u'grafana2': {u'vars': {u'environment': {u'prod': {u'mem': u'2048', u'cpus': u'2'}, u'dev': {u'mem': u'1024', u'cpus': u'1'}},
                                                                                                                                                                                                                                               u'global': {u'mem': u'128', u'cpus': u'0.1'}}}}]}}, u'name': u'test-app', u'vars': {u'environment': {u'prod': {u'mem': u'2048', u'cpus': u'2'}, u'dev': {u'mem': u'512', u'cpus': u'1'},
                                                                                                                                                                                                                                                                                                                                                    u'stage': {u'mem': u'1024', u'cpus': u'1'}}, u'global': {u'instances': u'1', u'network': u'BRIDGE'}}}

        roger_env = {u'default_environment': u'dev', u'registry': u'example.com:5000', u'environments': {u'prod': {u'chronos_endpoint': u'http://prod.example.com:4400',
                                                                                                         u'marathon_endpoint': u'http://prod.example.com:8080'}, u'dev': {u'chronos_endpoint': u'http://dev.example.com:4400', u'marathon_endpoint': u'http://dev.example.com:8080'},
                                                                                                         u'stage': {u'chronos_endpoint': u'http://stage.example.com:4400', u'marathon_endpoint': u'http://stage.example.com:8080'}}}

        data = config['apps']['grafana_test_app']
        self.config = config
        self.roger_env = roger_env
        self.data = data
Exemple #16
0
    def setUp(self):
        parser = argparse.ArgumentParser(description='Args for test')
        parser.add_argument('-e', '--env', metavar='env',
                            help="Environment to deploy to. example: 'dev' or 'stage'")
        parser.add_argument(
            '--skip-push', '-s', help="Don't push. Only generate components. Defaults to false.", action="store_true")
        parser.add_argument(
            '--secrets-file', '-S', help="Specify an optional secrets file for deploy runtime variables.")
        self.parser = parser
        self.args = parser

        self.settingObj = Settings()
        self.base_dir = self.settingObj.getCliDir()
        self.configs_dir = self.base_dir + "/tests/configs"
        self.components_dir = self.base_dir + '/tests/components/dev'

        config = {u'repo': u'roger', u'act-as': u'test', u'notifications': {u'username': u'Roger Deploy', u'method': u'chat.postMessage', u'channel': u'Channel ID', u'emoji': u':rocket:'},
                  u'apps': {u'test_app': {u'imageBase': u'test_app_base', u'name': u'test_app', u'containers': [u'container_name1', u'container_name2']},
                            u'test_app1': {u'vars': {u'global': {u'env_value1': u'12', u'env_value2': u'16'}, u'environment': {u'test': {u'env_value1': u'20', u'env_value2': u'24'}}}, u'framework': u'chronos', u'name': u'test_app1', u'containers': [u'container_name1', u'container_name2'], u'imageBase': u'test_app_base'},
                            u'grafana_test_app': {u'imageBase': u'test_app_base', u'name': u'test_app_grafana', u'containers': [u'grafana', {u'grafana1': {u'vars': {u'environment': {u'prod': {u'mem': u'2048', u'cpus': u'2'}, u'dev': {u'mem': u'512', u'cpus': u'0.5'}, u'test': {u'env_value1': u'64', u'env_value2': u'128'}}, u'global': {u'mem': u'128', u'cpus': u'0.1', u'env_value1': u'30', u'env_value2': u'54'}}}}, {u'grafana2': {u'vars': {u'environment': {u'prod': {u'mem': u'2048', u'cpus': u'2'}, u'dev': {u'mem': u'1024', u'cpus': u'1'}}, u'global': {u'mem': u'128', u'cpus': u'0.1'}}}}]}}, u'name': u'test-app',
                  u'vars': {u'environment': {u'prod': {u'mem': u'2048', u'cpus': u'2'}, u'test': {u'env_value1': u'4', u'env_value2': u'8'}, u'dev': {u'mem': u'512', u'cpus': u'1'}, u'stage': {u'mem': u'1024', u'cpus': u'1'}}, u'global': {u'instances': u'1', u'network': u'BRIDGE', u'env_value1': u'3', u'env_value2': u'3'}}}

        with open(self.configs_dir + '/roger-mesos-tools.config') as roger:
            roger_env = yaml.load(roger)
        data = config['apps']['grafana_test_app']
        self.config = config
        self.roger_env = roger_env
        self.data = data
        test_config = config
        test_data = test_config['apps']['grafana_test_app']
        self.test_config = test_config
        self.test_data = test_data
        template = Template(
            '{ "env": { "ENV_VAR1": "{{ env_value1 }}", "ENV_VAR2": "{{ env_value2 }}" }}')
        self.template = template
        self.additional_vars = {}
Exemple #17
0
    def main(self, settings, appConfig, frameworkObject, hooksObj, args):
        try:
            settingObj = settings
            appObj = appConfig
            frameworkUtils = frameworkObject
            config_dir = settingObj.getConfigDir()
            hooksObj.config_file = args.config_file
            cur_file_path = os.path.dirname(os.path.realpath(__file__))
            config = appObj.getConfig(config_dir, args.config_file)
            config_name = ""
            act_as_user = ""
            if 'name' in config:
                config_name = config['name']
            if 'act-as' in config:
                act_as_user = config['act-as']
            roger_env = appObj.getRogerEnv(config_dir)

            if not hasattr(args, "app_name"):
                args.app_name = ""

            if 'registry' not in roger_env.keys():
                raise ValueError(
                    'Registry not found in roger-mesos-tools.config file.')
            else:
                self.registry = roger_env['registry']

            if hasattr(args, "image_name"):
                self.image_name = args.image_name

            environment = roger_env.get('default_environment', '')
            if args.env is None:
                if "ROGER_ENV" in os.environ:
                    env_var = os.environ.get('ROGER_ENV')
                    if env_var.strip() == '':
                        print(
                            "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file"
                        )
                    else:
                        print(
                            "Using value {} from environment variable $ROGER_ENV"
                            .format(env_var))
                        environment = env_var
            else:
                environment = args.env

            if environment not in roger_env['environments']:
                raise ValueError(
                    'Environment not found in roger-mesos-tools.config file.')

            environmentObj = roger_env['environments'][environment]
            common_repo = config.get('repo', '')
            app_name = args.app_name
            container_list = []
            if ':' in app_name:
                tokens = app_name.split(':')
                app_name = tokens[0]
                if ',' in tokens[1]:
                    container_list = tokens[1].split(',')
                else:
                    container_list.append(tokens[1])

            data = appObj.getAppData(config_dir, args.config_file, app_name)
            if not data:
                raise ValueError(
                    'Application with name [{}] or data for it not found at {}/{}.'
                    .format(app_name, config_dir, args.config_file))

            configured_container_list = []
            for task in data['containers']:
                if type(task) == dict:
                    configured_container_list.append(task.keys()[0])
                else:
                    configured_container_list.append(task)
            if not set(container_list) <= set(configured_container_list):
                raise ValueError(
                    'List of containers [{}] passed do not match list of acceptable containers: [{}]'
                    .format(container_list, configured_container_list))

            frameworkObj = frameworkUtils.getFramework(data)
            framework = frameworkObj.getName()

            repo = ''
            if common_repo != '':
                repo = data.get('repo', common_repo)
            else:
                repo = data.get('repo', app_name)

            comp_dir = settingObj.getComponentsDir()
            templ_dir = settingObj.getTemplatesDir()
            secrets_dir = settingObj.getSecretsDir()

            # Create comp_dir if it doesn't exist
            if not os.path.isdir(comp_dir):
                os.makedirs(comp_dir)

            # template marathon files
            if not container_list:
                data_containers = data['containers']
            else:
                data_containers = container_list

            failed_container_dict = {}

            template = ''
            # Required for when work_dir,component_dir,template_dir or
            # secret_env_dir is something like '.' or './temp"
            os.chdir(cur_file_path)
            app_path = ''
            if 'template_path' in data:
                app_path = self.repo_relative_path(appObj, args, repo,
                                                   data['template_path'])
            else:
                app_path = templ_dir

            extra_vars = {}
            if 'extra_variables_path' in data:
                ev_path = self.repo_relative_path(appObj, args, repo,
                                                  data['extra_variables_path'])
                with open(ev_path) as f:
                    extra_vars = yaml.load(f) if ev_path.lower().endswith(
                        '.yml') else json.load(f)

            if not app_path.endswith('/'):
                app_path = app_path + '/'

            if not hasattr(self, "identifier"):
                self.identifier = self.utils.get_identifier(
                    config_name, settingObj.getUser(), args.app_name)

            args.app_name = self.utils.extract_app_name(args.app_name)
            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "pre_push"
            hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(
                args.app_name) + ",identifier=" + str(
                    self.identifier) + ",config_name=" + str(
                        config_name) + ",env=" + str(
                            environment) + ",user="******"{0}-{1}.json".format(
                        config['name'], container_name)
                else:
                    container_name = container
                    containerConfig = "{0}-{1}.json".format(
                        config['name'], container)

                env = Environment(loader=FileSystemLoader(
                    "{}".format(app_path)),
                                  undefined=StrictUndefined)
                template_with_path = "[{}{}]".format(app_path, containerConfig)
                try:
                    template = env.get_template(containerConfig)
                except exceptions.TemplateNotFound as e:
                    raise ValueError(
                        "The template file {} does not exist".format(
                            template_with_path))
                except Exception as e:
                    raise ValueError(
                        "Error while reading template from {} - {}".format(
                            template_with_path, e))

                additional_vars = {}
                additional_vars.update(extra_vars)
                secret_vars = self.loadSecrets(secrets_dir, containerConfig,
                                               args, environment)
                additional_vars.update(secret_vars)

                image_path = "{0}/{1}".format(roger_env['registry'],
                                              args.image_name)
                print(
                    "Rendering content from template {} for environment [{}]".
                    format(template_with_path, environment))
                try:
                    output = self.renderTemplate(template, environment,
                                                 image_path, data, config,
                                                 container, container_name,
                                                 additional_vars)
                except exceptions.UndefinedError as e:
                    error_str = "The following error occurred. %s.\n" % e
                    print(error_str, file=sys.stderr)
                    failed_container_dict[container_name] = error_str
                    pass

                # Adding check to see if all jinja variables git resolved fot the
                # container
                if container_name not in failed_container_dict:
                    # Adding check so that not all apps try to mergeSecrets
                    try:
                        outputObj = json.loads(output)
                    except Exception as e:
                        raise ValueError(
                            "Error while loading json from {} - {}".format(
                                template_with_path, e))

                    if 'SECRET' in output:
                        output = self.mergeSecrets(
                            output,
                            self.loadSecrets(secrets_dir, containerConfig,
                                             args, environment))
                    if output != "StandardError":
                        try:
                            comp_exists = os.path.exists(
                                "{0}".format(comp_dir))
                            if comp_exists is False:
                                os.makedirs("{0}".format(comp_dir))
                            comp_env_exists = os.path.exists("{0}/{1}".format(
                                comp_dir, environment))
                            if comp_env_exists is False:
                                os.makedirs("{0}/{1}".format(
                                    comp_dir, environment))
                        except Exception as e:
                            logging.error(traceback.format_exc())
                        with open(
                                "{0}/{1}/{2}".format(comp_dir, environment,
                                                     containerConfig),
                                'wb') as fh:
                            fh.write(output)

            if args.skip_push:
                print(
                    "Skipping push to {} framework. The rendered config file(s) are under {}/{}"
                    .format(framework, comp_dir, environment))
            else:
                # push to roger framework
                if 'owner' in config:
                    frameworkObj.act_as_user = config['owner']

                tools_version_value = self.utils.get_version()
                image_name = self.registry + "/" + args.image_name
                image_tag_value = urllib.quote("'" + image_name + "'")

                for container in data_containers:
                    try:
                        function_execution_start_time = datetime.now()
                        execution_result = 'SUCCESS'  # Assume the execution_result to be SUCCESS unless exception occurs
                        sc = self.utils.getStatsClient()
                    except (Exception) as e:
                        print("The following error occurred: %s" % e,
                              file=sys.stderr)
                    try:
                        if type(container) == dict:
                            container_name = str(container.keys()[0])
                            containerConfig = "{0}-{1}.json".format(
                                config['name'], container_name)
                        else:
                            container_name = container
                            containerConfig = "{0}-{1}.json".format(
                                config['name'], container)

                        if container_name in failed_container_dict:
                            print(
                                "Failed push to {} framework for container {} as unresolved Jinja variables present in template."
                                .format(framework, container_name))
                        else:
                            config_file_path = "{0}/{1}/{2}".format(
                                comp_dir, environment, containerConfig)

                            result = frameworkObj.runDeploymentChecks(
                                config_file_path, environment)

                            if args.force_push or result is True:
                                resp, task_id = frameworkObj.put(
                                    config_file_path, environmentObj,
                                    container_name, environment, act_as_user)

                                container_task_id = self.utils.modify_task_id(
                                    task_id)
                                self.task_id.extend(container_task_id)

                                if hasattr(resp, "status_code"):
                                    status_code = resp.status_code
                            else:
                                print(
                                    "Skipping push to {} framework for container {} as Validation Checks failed."
                                    .format(framework, container))
                    except (Exception) as e:
                        print("The following error occurred: %s" % e,
                              file=sys.stderr)
                        execution_result = 'FAILURE'
                        raise
                    finally:
                        try:

                            if 'function_execution_start_time' not in globals(
                            ) and 'function_execution_start_time' not in locals(
                            ):
                                function_execution_start_time = datetime.now()

                            if 'execution_result' not in globals(
                            ) and 'execution_result' not in locals():
                                execution_result = 'FAILURE'

                            if 'config_name' not in globals(
                            ) and 'config_name' not in locals():
                                config_name = ""

                            if 'environment' not in globals(
                            ) and 'environment' not in locals():
                                environment = "dev"

                            if 'container_name' not in globals(
                            ) and 'container_name' not in locals():
                                container_name = ""

                            if 'status_code' not in globals(
                            ) and 'status_code' not in locals():
                                status_code = "500"

                            if not hasattr(args, "app_name"):
                                args.app_name = ""

                            if 'settingObj' not in globals(
                            ) and 'settingObj' not in locals():
                                settingObj = Settings()

                            if 'container_task_id' not in globals(
                            ) and 'container_task_id' not in locals():
                                container_task_id = []

                            if not hasattr(self, "identifier"):
                                self.identifier = self.utils.get_identifier(
                                    config_name, settingObj.getUser(),
                                    args.app_name)

                            if not str(status_code).startswith("20"):
                                execution_result = 'FAILURE'
                                self.outcome = 0

                            time_take_milliseonds = (
                                (datetime.now() - function_execution_start_time
                                 ).total_seconds() * 1000)
                            for task_id in container_task_id:
                                input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(
                                    args.app_name
                                ) + ",event=push" + ",container_name=" + str(
                                    container_name) + ",identifier=" + str(
                                        self.identifier) + ",outcome=" + str(
                                            execution_result
                                        ) + ",response_code=" + str(
                                            status_code
                                        ) + ",config_name=" + str(
                                            config_name) + ",env=" + str(
                                                environment) + ",user="******",task_id=" + str(
                                                    task_id
                                                ) + ",tools_version=" + str(
                                                    tools_version_value
                                                ) + ",image_tag=" + str(
                                                    image_tag_value)
                                tup = (input_metric, time_take_milliseonds)
                                self.statsd_push_list.append(tup)

                                if str(status_code).startswith("20"):
                                    metric = input_metric.replace(
                                        "rogeros_tools_exec_time",
                                        "rogeros_events")
                                    metric = metric + ",source=tools" + ",task_id=" + task_id
                                    self.statsd_counter_logging(metric)

                        except (Exception) as e:
                            print("The following error occurred: %s" % e,
                                  file=sys.stderr)
                            raise

            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "post_push"
            hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(
                args.app_name) + ",identifier=" + str(
                    self.identifier) + ",config_name=" + str(
                        config_name) + ",env=" + str(
                            environment) + ",user="******"The following error occurred: %s" % e, file=sys.stderr)
            raise
Exemple #18
0
 def setUp(self):
     self.settingObj = Settings()
     self.base_dir = self.settingObj.getCliDir()
     self.config_dir = self.base_dir + "/tests/configs"
     self.template_dir = self.base_dir + "/tests/templates"
     pass
 def setUp(self):
     self.settingObj = Settings()
     self.base_dir = self.settingObj.getCliDir()
                        self.task_id.extend(container_task_id)
                    except (Exception) as e:
                        print("ERROR - : %s" %e, file=sys.stderr)
                        execution_result = 'FAILURE'
                        raise
                    finally:
                        # todo: maybe send datadog event from here?
                        pass

            hookname = "post_push"
            exit_code = hooksObj.run_hook(hookname, data, app_path, args.env, settingObj.getUser())
            if exit_code != 0:
                raise ValueError("{} hook failed.".format(hookname))
            print(colored("******Done with the PUSH step******", "green"))

        except (Exception) as e:
            raise ValueError("ERROR - {}".format(e))

if __name__ == "__main__":
    settingObj = Settings()
    appObj = AppConfig()
    frameworkUtils = FrameworkUtils()
    hooksObj = Hooks()
    roger_push = RogerPush()
    try:
        roger_push.parser = roger_push.parse_args()
        roger_push.args = roger_push.parser.parse_args()
        roger_push.main(settingObj, appObj, frameworkUtils, hooksObj, roger_push.args)
    except (Exception) as e:
        printException(e)
Exemple #21
0
    def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj, args):
        try:
            function_execution_start_time = datetime.now()
            execution_result = 'SUCCESS'  # Assume the execution_result to be SUCCESS unless exception occurs
            config_dir = settingObj.getConfigDir()
            root = settingObj.getCliDir()
            config = appObj.getConfig(config_dir, args.config_file)
            hooksObj.config_file = args.config_file
            roger_env = appObj.getRogerEnv(config_dir)
            config_name = ""
            if 'name' in config:
                config_name = config['name']
            common_repo = config.get('repo', '')
            if not hasattr(args, "env"):
                args.env = "dev"
            data = appObj.getAppData(config_dir, args.config_file, args.app_name)
            if not data:
                raise ValueError('Application with name [{}] or data for it not found at {}/{}.'.format(
                    args.app_name, config_dir, args.config_file))
            repo = ''
            if common_repo != '':
                repo = data.get('repo', common_repo)
            else:
                repo = data.get('repo', args.app_name)

            build_args = {}
            if 'build-args' in data:
                if 'environment' in data['build-args']:
                    if args.env in data['build-args']['environment']:
                        build_args = data['build-args']['environment'][args.env]

            projects = data.get('privateProjects', [])
            docker_path = data.get('path', 'none')

            # get/update target source(s)
            file_exists = True
            file_path = ''
            cur_dir = ''
            if "PWD" in os.environ:
                cur_dir = os.environ.get('PWD')
            abs_path = os.path.abspath(args.directory)
            repo_name = appObj.getRepoName(repo)
            if docker_path != 'none':
                if abs_path == args.directory:
                    file_path = "{0}/{1}/{2}".format(args.directory,
                                                     repo_name, docker_path)
                else:
                    file_path = "{0}/{1}/{2}/{3}".format(
                        cur_dir, args.directory, repo_name, docker_path)
            else:
                if abs_path == args.directory:
                    file_path = "{0}/{1}".format(args.directory, repo_name)
                else:
                    file_path = "{0}/{1}/{2}".format(cur_dir,
                                                     args.directory, repo_name)

            if not hasattr(args, "app_name"):
                args.app_name = ""

            if not hasattr(self, "identifier"):
                self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)

            args.app_name = self.utils.extract_app_name(args.app_name)
            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "pre_build"
            hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"{0}/{1}".format(file_path, data['build_filename']))
                file_exists = os.path.exists(build_filename)
                if not file_exists:
                    raise ValueError("Specified build file: {} does not exist. Exiting build.".format(build_filename))
            else:
                file_exists = os.path.exists("{0}/Dockerfile".format(file_path))

            if file_exists:
                if 'registry' not in roger_env:
                    raise ValueError('Registry not found in roger-mesos-tools.config file.')
                else:
                    self.registry = roger_env['registry']
                self.tag_name = args.tag_name
                image = "{0}/{1}".format(roger_env['registry'], args.tag_name)
                try:
                    if abs_path == args.directory:
                        try:
                            dockerObj.docker_build(
                                dockerUtilsObj, appObj, args.directory, repo, projects, docker_path, image, build_args, build_filename)
                        except ValueError:
                            print('Docker build failed.')
                            raise
                    else:
                        directory = '{0}/{1}'.format(cur_dir, args.directory)
                        try:
                            dockerObj.docker_build(
                                dockerUtilsObj, appObj, directory, repo, projects, docker_path, image, build_args, build_filename)
                        except ValueError:
                            print('Docker build failed.')
                            raise
                    build_message = "Image {0} built".format(image)
                    if(args.push):
                        exit_code = dockerUtilsObj.docker_push(image)
                        if exit_code != 0:
                            raise ValueError(
                                'Docker push failed.')
                        build_message += " and pushed to registry {}".format(roger_env[
                                                                             'registry'])
                    print(build_message)
                except (IOError) as e:
                    print("The folowing error occurred.(Error: %s).\n" %
                          e, file=sys.stderr)
                    raise
            else:
                print("Dockerfile does not exist in dir: {}".format(file_path))

            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "post_build"
            hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"The following error occurred: %s" %
                  e, file=sys.stderr)
            raise
        finally:
            try:
                # If the build fails before going through any steps
                if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
                    function_execution_start_time = datetime.now()

                if 'execution_result' not in globals() and 'execution_result' not in locals():
                    execution_result = 'FAILURE'

                if 'config_name' not in globals() and 'config_name' not in locals():
                    config_name = ""

                if not hasattr(args, "env"):
                    args.env = "dev"

                if not hasattr(args, "app_name"):
                    args.app_name = ""

                if 'settingObj' not in globals() and 'settingObj' not in locals():
                    settingObj = Settings()

                if 'execution_result' is 'FAILURE':
                    self.outcome = 0

                sc = self.utils.getStatsClient()
                if not hasattr(self, "identifier"):
                    self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
                time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
                input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=build" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"The following error occurred: %s" %
                      e, file=sys.stderr)
                raise
Exemple #22
0
import json
import yaml
import re
from jinja2 import Environment, FileSystemLoader
from termcolor import colored
from cli.framework import Framework
from cli.utils import Utils
from cli.settings import Settings
from cli.marathonvalidator import MarathonValidator
from cli.proxyparser import ProxyParser
from cli.appconfig import AppConfig
from distutils.version import LooseVersion
requests.packages.urllib3.disable_warnings()

utils = Utils()
settings = Settings()


class Marathon(Framework):
    def __init__(self):
        self.user = None
        self.passw = None
        self.marathonvalidator = MarathonValidator()
        self.proxyparser = ProxyParser()

    def getName(self):
        return "Marathon"

    def get(self, roger_env, environment):
        url = roger_env['environments'][environment][
            'marathon_endpoint'] + "/v2/apps"
Exemple #23
0
    def main(self, settings, appConfig, gitObject, hooksObj, args):
        try:
            function_execution_start_time = datetime.now()
            execution_result = 'SUCCESS'  # Assume the execution_result to be SUCCESS unless exception occurs
            environment = "dev"
            if hasattr(args, "environment"):
                environment = args.environment
            settingObj = settings
            appObj = appConfig
            gitObj = gitObject
            config_dir = settingObj.getConfigDir()
            hooksObj.config_file = args.config_file
            config = appObj.getConfig(config_dir, args.config_file)
            config_name = ""
            if 'name' in config:
                config_name = config['name']
            common_repo = config.get('repo', '')
            data = appObj.getAppData(config_dir, args.config_file, args.app_name)
            if not data:
                raise ValueError('Application with name [{}] or data for it not found at {}/{}.'.format(
                    args.app_name, config_dir, args.config_file))
            repo = ''
            if common_repo != '':
                repo = data.get('repo', common_repo)
            else:
                repo = data.get('repo', args.app_name)

            branch = "master"  # master by default
            if args.branch is not None:
                branch = args.branch

            if not os.path.exists(args.directory):
                try:
                    os.makedirs(args.directory)
                except OSError as exception:
                    if exception.errno != errno.EEXIST:
                        raise

            if not hasattr(args, "app_name"):
                args.app_name = ""

            if not hasattr(self, "identifier"):
                self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)

            args.app_name = self.utils.extract_app_name(args.app_name)

            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "pre_gitpull"
            hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{0}/{1}".format(args.directory, repo_name)
            if os.path.isdir(path):
                with chdir(path):
                    exit_code = gitObj.gitPull(branch)
            else:
                with chdir('{0}'.format(args.directory)):
                    exit_code = gitObj.gitShallowClone(repo, branch)

            if exit_code != 0:
                raise ValueError('gitpull failed.')

            hooksObj.statsd_message_list = self.statsd_message_list
            hookname = "post_gitpull"
            hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"The following error occurred: %s" %
                  e, file=sys.stderr)
            execution_result = 'FAILURE'
            raise
        finally:
            try:
                # If the gitpull fails before going through any steps
                if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
                    function_execution_start_time = datetime.now()

                if 'execution_result' not in globals() and 'execution_result' not in locals():
                    execution_result = 'FAILURE'

                if 'config_name' not in globals() and 'config_name' not in locals():
                    config_name = ""

                if 'environment' not in globals() and 'environment' not in locals():
                    environment = "dev"

                if not hasattr(args, "app_name"):
                    args.app_name = ""

                if 'settingObj' not in globals() and 'settingObj' not in locals():
                    settingObj = Settings()

                if 'execution_result' is 'FAILURE':
                    self.outcome = 0

                sc = self.utils.getStatsClient()
                if not hasattr(self, "identifier"):
                    self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
                time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
                input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=gitpull" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"The following error occurred: %s" %
                      e, file=sys.stderr)
                raise
Exemple #24
0
    def main(self, settingObject, appObject, frameworkUtilsObject, gitObj,
             hooksObj, args):
        try:
            function_execution_start_time = datetime.now()
            execution_result = 'SUCCESS'
            settingObj = settingObject
            appObj = appObject
            config_dir = settingObj.getConfigDir()
            root = settingObj.getCliDir()
            roger_env = appObj.getRogerEnv(config_dir)
            config = appObj.getConfig(config_dir, args.config_file)
            config_name = ""
            if 'name' in config:
                config_name = config['name']
            if 'registry' not in roger_env:
                raise ValueError(
                    'Registry not found in roger-mesos-tools.config file.')
            else:
                self.registry = roger_env['registry']

            # Setup for Slack-Client, token, and git user
            if 'notifications' in config:
                self.slack = Slack(
                    config['notifications'],
                    '/home/vagrant/.roger_cli.conf.d/slack_token')

            self.identifier = self.utils.get_identifier(
                config_name, settingObj.getUser(), args.application)

            apps = []
            apps_container_dict = {}
            if args.application == 'all':
                apps = config['apps'].keys()
            else:
                if ":" not in args.application and "[" not in args.application:
                    apps.append(args.application)
                else:
                    for item in args.application.split(":"):
                        if '[' in item:
                            matchObj = re.match(r'(.*)\[(.*)\]', item)
                            apps.append(matchObj.group(1))
                            apps_container_dict[matchObj.group(
                                1)] = matchObj.group(2)
                        else:
                            apps.append(item)

            common_repo = config.get('repo', '')
            environment = roger_env.get('default_environment', '')

            work_dir = ''
            if args.directory:
                work_dir = args.directory
                temp_dir_created = False
                print("Using {0} as the working directory".format(work_dir))
            else:
                work_dir = mkdtemp()
                temp_dir_created = True
                print("Created a temporary dir: {0}".format(work_dir))

            if args.environment is None:
                if "ROGER_ENV" in os.environ:
                    env_var = os.environ.get('ROGER_ENV')
                    if env_var.strip() == '':
                        print(
                            "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file"
                        )
                    else:
                        print(
                            "Using value {} from environment variable $ROGER_ENV"
                            .format(env_var))
                        environment = env_var
            else:
                environment = args.environment

            if environment not in roger_env['environments']:
                self.removeDirTree(work_dir, args, temp_dir_created)
                raise ValueError(
                    'Environment not found in roger-mesos-tools.config file.')

            branch = "master"  # master by default
            if args.branch is not None:
                branch = args.branch

            try:
                for app in apps:
                    if app not in config['apps']:
                        print(
                            'Application {} specified not found.'.format(app))
                    else:
                        try:
                            print("Deploying {} ...".format(app))
                            self.deployApp(settingObject, appObject,
                                           frameworkUtilsObject, gitObj,
                                           hooksObj, root, args, config,
                                           roger_env, work_dir, config_dir,
                                           environment, app, branch,
                                           self.slack, args.config_file,
                                           common_repo, temp_dir_created,
                                           apps_container_dict)
                        except (IOError, ValueError) as e:
                            print(
                                "The following error occurred when deploying {}: {}"
                                .format(app, e),
                                file=sys.stderr)
                            pass  # try deploying the next app
            except (Exception) as e:
                print("The following error occurred: %s" % e, file=sys.stderr)
                raise
        except (Exception) as e:
            execution_result = 'FAILURE'
            print("The following error occurred: %s" % e, file=sys.stderr)
            raise
        finally:
            # Check if the initializition of variables carried out
            if 'function_execution_start_time' not in globals(
            ) and 'function_execution_start_time' not in locals():
                function_execution_start_time = datetime.now()

            if 'execution_result' not in globals(
            ) and 'execution_result' not in locals():
                execution_result = 'FAILURE'

            if 'config_name' not in globals() and 'config_name' not in locals(
            ):
                config_name = ""

            if 'environment' not in globals() and 'environment' not in locals(
            ):
                environment = "dev"

            if not hasattr(args, "application"):
                args.application = ""

            if 'settingObj' not in globals() and 'settingObj' not in locals():
                settingObj = Settings()

            if 'work_dir' not in globals() and 'work_dir' not in locals():
                work_dir = ''
                temp_dir_created = False

            if not (self.rogerGitPullObject.outcome is 1
                    and self.rogerBuildObject.outcome is 1
                    and self.rogerPushObject.outcome is 1):
                execution_result = 'FAILURE'

            try:
                # If the deploy fails before going through any steps
                sc = self.utils.getStatsClient()
                if not hasattr(self, "identifier"):
                    self.identifier = self.utils.get_identifier(
                        config_name, settingObj.getUser(), args.application)
                args.application = self.utils.extract_app_name(
                    args.application)
                time_take_milliseonds = (
                    (datetime.now() -
                     function_execution_start_time).total_seconds() * 1000)
                input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(
                    args.application) + ",event=deploy" + ",outcome=" + str(
                        execution_result
                    ) + ",config_name=" + str(config_name) + ",env=" + str(
                        environment) + ",user="******",identifier=" + str(
                                self.identifier)
                tup = (input_metric, time_take_milliseonds)
                self.statsd_message_list.append(tup)
                self.removeDirTree(work_dir, args, temp_dir_created)
            except (Exception) as e:
                print("The following error occurred: %s" % e, file=sys.stderr)
                raise
 def setUp(self):
     self.appObj = AppConfig()
     self.settingObj = Settings()
     self.base_dir = self.settingObj.getCliDir()
     self.configs_dir = self.base_dir + "/tests/configs"