def setUp(self):
self.gitObj = GitUtils()
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
self.configs_dir = self.base_dir + "/tests/configs"
self.work_dir = self.base_dir + "/tests/work_dir"
self.branch = "master"
def setUp(self):
parser = argparse.ArgumentParser(description='Args for test')
parser.add_argument(
'-e',
'--env',
metavar='env',
help="Environment to deploy to. example: 'dev' or 'stage'")
parser.add_argument(
'--skip-push',
'-s',
help="Don't push. Only generate components. Defaults to false.",
action="store_true")
parser.add_argument(
'--secrets-file',
'-S',
help=
"Specify an optional secrets file for deploy runtime variables.")
self.parser = parser
self.args = parser
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
self.configs_dir = self.base_dir + "/tests/configs"
with open(self.configs_dir +
'/roger_single_container_var_tests.json') as config:
config = json.load(config)
with open(self.configs_dir + '/roger-mesos-tools.config') as roger:
roger_env = yaml.load(roger)
data = config['apps']['container-vars']
self.config = config
self.roger_env = roger_env
self.data = data
def _get_template_path(self,
container_name,
config_dir,
args,
app_name,
app_object=AppConfig(),
settings_object=Settings()):
"""
Returns the template path for a given container_name
Each framework requires an template_path for the app_id method
:Params:
:config_dir [str]: path to the config directory
:args [argparse.NameSpace]: how to get acceses to the values passed
:app_name [str]: name of app
:app_object [cli.appconfig.AppConfig]: instance of AppConfig
:settings_object [cli.settings.Settings]: instance of Settings
"""
data = app_object.getConfig(config_dir, args.config)
repo = self._config_resolver('repo', app_name, args.config)
template_path = self._config_resolver('template_path', app_name,
args.config)
# this path is always relative to the root repo dir, so join
if template_path and not os.path.isabs(template_path):
app_path = os.path.join(self._temp_dir, repo, template_path)
else:
app_path = settings_object.getTemplatesDir()
file_name = "{0}-{1}.json".format(data['name'], container_name)
return os.path.join(app_path, file_name)
def setUp(self):
parser = argparse.ArgumentParser(description='Args for test')
parser.add_argument('app_name', metavar='app_name',
help="application to build. Example: 'agora'.")
parser.add_argument('env', metavar='env',
help="environment. Example: 'test'.")
parser.add_argument('directory', metavar='directory',
help="working directory. Example: '/home/vagrant/work_dir'.")
parser.add_argument('tag_name', metavar='tag_name',
help="tag for the built image. Example: 'roger-collectd:0.20'.")
parser.add_argument('config_file', metavar='config_file',
help="configuration file to use. Example: 'content.json'.")
parser.add_argument(
'--push', '-p', help="Also push to registry. Defaults to false.", action="store_true")
self.parser = parser
self.args = self.parser
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
self.configs_dir = self.base_dir + "/tests/configs"
self.components_dir = self.base_dir + '/tests/components/dev'
with open(self.configs_dir + '/app.json') as config:
config = json.load(config)
with open(self.configs_dir + '/roger-mesos-tools.config') as roger:
roger_env = yaml.load(roger)
data = config['apps']['grafana_test_app']
self.config = config
self.roger_env = roger_env
self.data = data
def setUp(self):
self.webhook = WebHook()
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
self.configs_dir = self.base_dir + "/tests/configs"
self.components_dir = self.base_dir + '/tests/components/dev'
with open(self.configs_dir + '/roger-mesos-tools.config') as roger:
roger_env = yaml.load(roger)
self.roger_env = roger_env
def setUp(self):
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
self.configs_dir = self.base_dir + "/tests/configs"
self.components_dir = self.base_dir + '/tests/components/dev'
self.configs_dir = self.base_dir + "/tests/configs"
with open(self.configs_dir + '/app.json') as config:
config = json.load(config)
self.config = config
def main():
args = getCliArguments()
settings = Settings(args.settings)
machine = machineFactory(settings)
output = []
for letter in list(args.message):
output.append(machine.encode(letter))
print(''.join(output))
def get_haproxy_config(self, environment):
haproxy_config = ""
settingObj = Settings()
appObj = AppConfig()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
host = roger_env['environments'][environment]['host']
haproxy_config_path = roger_env['environments'][
environment]['haproxy_config_path']
url = "{}{}".format(host, haproxy_config_path)
haproxy_config = requests.get(url, stream=True)
return haproxy_config.text
def get_proxy_config(self, environment):
proxy_config = ""
settingObj = Settings()
appObj = AppConfig()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
host = roger_env['environments'][environment]['host']
proxy_config_path = roger_env['environments'][environment][
'proxy_config_path']
url = "{}{}".format(host, proxy_config_path)
proxy_config = requests.get(url).json()
return proxy_config
def getStatsClient(self):
settingObj = Settings()
appObj = AppConfig()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
statsd_url = ""
statsd_port = ""
if 'statsd_endpoint' in roger_env.keys():
statsd_url = roger_env['statsd_endpoint']
if 'statsd_port' in roger_env.keys():
statsd_port = int(roger_env['statsd_port'])
return statsd.StatsClient(statsd_url, statsd_port)
def __init__(self,
app_config=AppConfig(),
settings=Settings(),
framework_utils=FrameworkUtils(),
framework=Marathon()):
self._app_config = app_config
self._settings = settings
self._framework_utils = framework_utils
self._framework = framework
self._config_dir = None
self._roger_env = None
self._temp_dir = None
def __init__(self):
self.disabled = True
self.emoji = ':rocket:'
self.defChannel = ''
self.config_dir = ''
self.username = '******'
self.settingObj = Settings()
self.appconfigObj = AppConfig()
self.configLoadFlag = False
self.config = ''
self.config_channels = []
self.config_envs = []
self.config_commands = []
def getRepoUrl(self, repo):
if repo.startswith('*****@*****.**'):
return repo
else:
settingObj = Settings()
config_dir = settingObj.getConfigDir()
roger_env = self.getRogerEnv(config_dir)
if 'default_github_repo_prefix' in roger_env.keys():
prefix = roger_env['default_github_repo_prefix']
else:
raise ValueError(
"Could not determine github repo.Please provide default \"github repo prefix\" or ensure repo startswith [email protected]"
)
return str(prefix + '{}.git'.format(repo))
def setUp(self):
parser = argparse.ArgumentParser(description='Args for test')
parser.add_argument(
'app_name',
metavar='app_name',
help=
"application for which code is to be pulled. Example: 'agora' or 'grafana'"
)
parser.add_argument(
'directory',
metavar='directory',
help=
"working directory. The repo has its own directory it this. Example: '/home/vagrant/work_dir'"
)
parser.add_argument(
'-b',
'--branch',
metavar='branch',
help=
"git branch to pull code from. Example: 'production' or 'master'. Defaults to master."
)
parser.add_argument(
'config_file',
metavar='config_file',
help=
"configuration file to use. Example: 'content.json' or 'kwe.json'")
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
self.configs_dir = self.base_dir + "/tests/configs"
self.work_dir = self.base_dir + "/tests/work_dir"
self.components_dir = self.base_dir + '/tests/components/dev'
self.args = parser
with open(self.configs_dir + '/app.json') as config:
config = json.load(config)
with open(self.configs_dir + '/roger-mesos-tools.config') as roger:
roger_env = yaml.load(roger)
data = config['apps']['grafana_test_app']
self.roger_env = roger_env
self.data = data
self.config = config
pass
def setUp(self):
parser = argparse.ArgumentParser(description='Args for test')
parser.add_argument('-e', '--environment', metavar='env',
help="Environment to deploy to. example: 'dev' or 'stage'")
parser.add_argument('-s', '--skip-build', action="store_true",
help="Flag that skips roger-build when set to true. Defaults to false.'")
parser.add_argument('-M', '--incr-major', action="store_true",
help="Increment major in version. Defaults to false.'")
parser.add_argument('-p', '--incr-patch', action="store_true",
help="Increment patch in version. Defaults to false.'")
parser.add_argument('-sp', '--skip-push', action="store_true",
help="Flag that skips roger push when set to true. Defaults to false.'")
parser.add_argument(
'-S', '--secrets-file', help="Specify an optional secrets file for deployment runtime variables.")
parser.add_argument(
'-d', '--directory', help="Specify an optional directory to pull out the repo. This is the working directory.")
self.parser = parser
self.args = parser
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
self.configs_dir = self.base_dir + "/tests/configs"
config = {u'repo': u'roger', u'notifications': {u'username': u'Roger Deploy', u'method': u'chat.postMessage', u'channel': u'Channel ID', u'emoji': u':rocket:'},
u'apps': {u'test_app': {u'imageBase': u'test_app_base', u'name': u'test_app', u'containers': [u'container_name1', u'container_name2']},
u'test_app1': {u'framework': u'chronos', u'name': u'test_app1', u'containers': [u'container_name1', u'container_name2'], u'imageBase': u'test_app_base'},
u'grafana_test_app': {u'imageBase': u'test_app_base', u'name': u'test_app_grafana', u'containers': [u'grafana', {u'grafana1': {u'vars': {u'environment': {u'prod': {u'mem': u'2048', u'cpus': u'2'}, u'dev': {u'mem': u'512', u'cpus': u'0.5'}},
u'global': {u'mem': u'128', u'cpus': u'0.1'}}}}, {u'grafana2': {u'vars': {u'environment': {u'prod': {u'mem': u'2048', u'cpus': u'2'}, u'dev': {u'mem': u'1024', u'cpus': u'1'}},
u'global': {u'mem': u'128', u'cpus': u'0.1'}}}}]}}, u'name': u'test-app', u'vars': {u'environment': {u'prod': {u'mem': u'2048', u'cpus': u'2'}, u'dev': {u'mem': u'512', u'cpus': u'1'},
u'stage': {u'mem': u'1024', u'cpus': u'1'}}, u'global': {u'instances': u'1', u'network': u'BRIDGE'}}}
roger_env = {u'default_environment': u'dev', u'registry': u'example.com:5000', u'environments': {u'prod': {u'chronos_endpoint': u'http://prod.example.com:4400',
u'marathon_endpoint': u'http://prod.example.com:8080'}, u'dev': {u'chronos_endpoint': u'http://dev.example.com:4400', u'marathon_endpoint': u'http://dev.example.com:8080'},
u'stage': {u'chronos_endpoint': u'http://stage.example.com:4400', u'marathon_endpoint': u'http://stage.example.com:8080'}}}
data = config['apps']['grafana_test_app']
self.config = config
self.roger_env = roger_env
self.data = data
def setUp(self):
parser = argparse.ArgumentParser(description='Args for test')
parser.add_argument('-e', '--env', metavar='env',
help="Environment to deploy to. example: 'dev' or 'stage'")
parser.add_argument(
'--skip-push', '-s', help="Don't push. Only generate components. Defaults to false.", action="store_true")
parser.add_argument(
'--secrets-file', '-S', help="Specify an optional secrets file for deploy runtime variables.")
self.parser = parser
self.args = parser
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
self.configs_dir = self.base_dir + "/tests/configs"
self.components_dir = self.base_dir + '/tests/components/dev'
config = {u'repo': u'roger', u'act-as': u'test', u'notifications': {u'username': u'Roger Deploy', u'method': u'chat.postMessage', u'channel': u'Channel ID', u'emoji': u':rocket:'},
u'apps': {u'test_app': {u'imageBase': u'test_app_base', u'name': u'test_app', u'containers': [u'container_name1', u'container_name2']},
u'test_app1': {u'vars': {u'global': {u'env_value1': u'12', u'env_value2': u'16'}, u'environment': {u'test': {u'env_value1': u'20', u'env_value2': u'24'}}}, u'framework': u'chronos', u'name': u'test_app1', u'containers': [u'container_name1', u'container_name2'], u'imageBase': u'test_app_base'},
u'grafana_test_app': {u'imageBase': u'test_app_base', u'name': u'test_app_grafana', u'containers': [u'grafana', {u'grafana1': {u'vars': {u'environment': {u'prod': {u'mem': u'2048', u'cpus': u'2'}, u'dev': {u'mem': u'512', u'cpus': u'0.5'}, u'test': {u'env_value1': u'64', u'env_value2': u'128'}}, u'global': {u'mem': u'128', u'cpus': u'0.1', u'env_value1': u'30', u'env_value2': u'54'}}}}, {u'grafana2': {u'vars': {u'environment': {u'prod': {u'mem': u'2048', u'cpus': u'2'}, u'dev': {u'mem': u'1024', u'cpus': u'1'}}, u'global': {u'mem': u'128', u'cpus': u'0.1'}}}}]}}, u'name': u'test-app',
u'vars': {u'environment': {u'prod': {u'mem': u'2048', u'cpus': u'2'}, u'test': {u'env_value1': u'4', u'env_value2': u'8'}, u'dev': {u'mem': u'512', u'cpus': u'1'}, u'stage': {u'mem': u'1024', u'cpus': u'1'}}, u'global': {u'instances': u'1', u'network': u'BRIDGE', u'env_value1': u'3', u'env_value2': u'3'}}}
with open(self.configs_dir + '/roger-mesos-tools.config') as roger:
roger_env = yaml.load(roger)
data = config['apps']['grafana_test_app']
self.config = config
self.roger_env = roger_env
self.data = data
test_config = config
test_data = test_config['apps']['grafana_test_app']
self.test_config = test_config
self.test_data = test_data
template = Template(
'{ "env": { "ENV_VAR1": "{{ env_value1 }}", "ENV_VAR2": "{{ env_value2 }}" }}')
self.template = template
self.additional_vars = {}
def main(self, settings, appConfig, frameworkObject, hooksObj, args):
try:
settingObj = settings
appObj = appConfig
frameworkUtils = frameworkObject
config_dir = settingObj.getConfigDir()
hooksObj.config_file = args.config_file
cur_file_path = os.path.dirname(os.path.realpath(__file__))
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
act_as_user = ""
if 'name' in config:
config_name = config['name']
if 'act-as' in config:
act_as_user = config['act-as']
roger_env = appObj.getRogerEnv(config_dir)
if not hasattr(args, "app_name"):
args.app_name = ""
if 'registry' not in roger_env.keys():
raise ValueError(
'Registry not found in roger-mesos-tools.config file.')
else:
self.registry = roger_env['registry']
if hasattr(args, "image_name"):
self.image_name = args.image_name
environment = roger_env.get('default_environment', '')
if args.env is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file"
)
else:
print(
"Using value {} from environment variable $ROGER_ENV"
.format(env_var))
environment = env_var
else:
environment = args.env
if environment not in roger_env['environments']:
raise ValueError(
'Environment not found in roger-mesos-tools.config file.')
environmentObj = roger_env['environments'][environment]
common_repo = config.get('repo', '')
app_name = args.app_name
container_list = []
if ':' in app_name:
tokens = app_name.split(':')
app_name = tokens[0]
if ',' in tokens[1]:
container_list = tokens[1].split(',')
else:
container_list.append(tokens[1])
data = appObj.getAppData(config_dir, args.config_file, app_name)
if not data:
raise ValueError(
'Application with name [{}] or data for it not found at {}/{}.'
.format(app_name, config_dir, args.config_file))
configured_container_list = []
for task in data['containers']:
if type(task) == dict:
configured_container_list.append(task.keys()[0])
else:
configured_container_list.append(task)
if not set(container_list) <= set(configured_container_list):
raise ValueError(
'List of containers [{}] passed do not match list of acceptable containers: [{}]'
.format(container_list, configured_container_list))
frameworkObj = frameworkUtils.getFramework(data)
framework = frameworkObj.getName()
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', app_name)
comp_dir = settingObj.getComponentsDir()
templ_dir = settingObj.getTemplatesDir()
secrets_dir = settingObj.getSecretsDir()
# Create comp_dir if it doesn't exist
if not os.path.isdir(comp_dir):
os.makedirs(comp_dir)
# template marathon files
if not container_list:
data_containers = data['containers']
else:
data_containers = container_list
failed_container_dict = {}
template = ''
# Required for when work_dir,component_dir,template_dir or
# secret_env_dir is something like '.' or './temp"
os.chdir(cur_file_path)
app_path = ''
if 'template_path' in data:
app_path = self.repo_relative_path(appObj, args, repo,
data['template_path'])
else:
app_path = templ_dir
extra_vars = {}
if 'extra_variables_path' in data:
ev_path = self.repo_relative_path(appObj, args, repo,
data['extra_variables_path'])
with open(ev_path) as f:
extra_vars = yaml.load(f) if ev_path.lower().endswith(
'.yml') else json.load(f)
if not app_path.endswith('/'):
app_path = app_path + '/'
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(
config_name, settingObj.getUser(), args.app_name)
args.app_name = self.utils.extract_app_name(args.app_name)
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "pre_push"
hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(
args.app_name) + ",identifier=" + str(
self.identifier) + ",config_name=" + str(
config_name) + ",env=" + str(
environment) + ",user="******"{0}-{1}.json".format(
config['name'], container_name)
else:
container_name = container
containerConfig = "{0}-{1}.json".format(
config['name'], container)
env = Environment(loader=FileSystemLoader(
"{}".format(app_path)),
undefined=StrictUndefined)
template_with_path = "[{}{}]".format(app_path, containerConfig)
try:
template = env.get_template(containerConfig)
except exceptions.TemplateNotFound as e:
raise ValueError(
"The template file {} does not exist".format(
template_with_path))
except Exception as e:
raise ValueError(
"Error while reading template from {} - {}".format(
template_with_path, e))
additional_vars = {}
additional_vars.update(extra_vars)
secret_vars = self.loadSecrets(secrets_dir, containerConfig,
args, environment)
additional_vars.update(secret_vars)
image_path = "{0}/{1}".format(roger_env['registry'],
args.image_name)
print(
"Rendering content from template {} for environment [{}]".
format(template_with_path, environment))
try:
output = self.renderTemplate(template, environment,
image_path, data, config,
container, container_name,
additional_vars)
except exceptions.UndefinedError as e:
error_str = "The following error occurred. %s.\n" % e
print(error_str, file=sys.stderr)
failed_container_dict[container_name] = error_str
pass
# Adding check to see if all jinja variables git resolved fot the
# container
if container_name not in failed_container_dict:
# Adding check so that not all apps try to mergeSecrets
try:
outputObj = json.loads(output)
except Exception as e:
raise ValueError(
"Error while loading json from {} - {}".format(
template_with_path, e))
if 'SECRET' in output:
output = self.mergeSecrets(
output,
self.loadSecrets(secrets_dir, containerConfig,
args, environment))
if output != "StandardError":
try:
comp_exists = os.path.exists(
"{0}".format(comp_dir))
if comp_exists is False:
os.makedirs("{0}".format(comp_dir))
comp_env_exists = os.path.exists("{0}/{1}".format(
comp_dir, environment))
if comp_env_exists is False:
os.makedirs("{0}/{1}".format(
comp_dir, environment))
except Exception as e:
logging.error(traceback.format_exc())
with open(
"{0}/{1}/{2}".format(comp_dir, environment,
containerConfig),
'wb') as fh:
fh.write(output)
if args.skip_push:
print(
"Skipping push to {} framework. The rendered config file(s) are under {}/{}"
.format(framework, comp_dir, environment))
else:
# push to roger framework
if 'owner' in config:
frameworkObj.act_as_user = config['owner']
tools_version_value = self.utils.get_version()
image_name = self.registry + "/" + args.image_name
image_tag_value = urllib.quote("'" + image_name + "'")
for container in data_containers:
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs
sc = self.utils.getStatsClient()
except (Exception) as e:
print("The following error occurred: %s" % e,
file=sys.stderr)
try:
if type(container) == dict:
container_name = str(container.keys()[0])
containerConfig = "{0}-{1}.json".format(
config['name'], container_name)
else:
container_name = container
containerConfig = "{0}-{1}.json".format(
config['name'], container)
if container_name in failed_container_dict:
print(
"Failed push to {} framework for container {} as unresolved Jinja variables present in template."
.format(framework, container_name))
else:
config_file_path = "{0}/{1}/{2}".format(
comp_dir, environment, containerConfig)
result = frameworkObj.runDeploymentChecks(
config_file_path, environment)
if args.force_push or result is True:
resp, task_id = frameworkObj.put(
config_file_path, environmentObj,
container_name, environment, act_as_user)
container_task_id = self.utils.modify_task_id(
task_id)
self.task_id.extend(container_task_id)
if hasattr(resp, "status_code"):
status_code = resp.status_code
else:
print(
"Skipping push to {} framework for container {} as Validation Checks failed."
.format(framework, container))
except (Exception) as e:
print("The following error occurred: %s" % e,
file=sys.stderr)
execution_result = 'FAILURE'
raise
finally:
try:
if 'function_execution_start_time' not in globals(
) and 'function_execution_start_time' not in locals(
):
function_execution_start_time = datetime.now()
if 'execution_result' not in globals(
) and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals(
) and 'config_name' not in locals():
config_name = ""
if 'environment' not in globals(
) and 'environment' not in locals():
environment = "dev"
if 'container_name' not in globals(
) and 'container_name' not in locals():
container_name = ""
if 'status_code' not in globals(
) and 'status_code' not in locals():
status_code = "500"
if not hasattr(args, "app_name"):
args.app_name = ""
if 'settingObj' not in globals(
) and 'settingObj' not in locals():
settingObj = Settings()
if 'container_task_id' not in globals(
) and 'container_task_id' not in locals():
container_task_id = []
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(
config_name, settingObj.getUser(),
args.app_name)
if not str(status_code).startswith("20"):
execution_result = 'FAILURE'
self.outcome = 0
time_take_milliseonds = (
(datetime.now() - function_execution_start_time
).total_seconds() * 1000)
for task_id in container_task_id:
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(
args.app_name
) + ",event=push" + ",container_name=" + str(
container_name) + ",identifier=" + str(
self.identifier) + ",outcome=" + str(
execution_result
) + ",response_code=" + str(
status_code
) + ",config_name=" + str(
config_name) + ",env=" + str(
environment) + ",user="******",task_id=" + str(
task_id
) + ",tools_version=" + str(
tools_version_value
) + ",image_tag=" + str(
image_tag_value)
tup = (input_metric, time_take_milliseonds)
self.statsd_push_list.append(tup)
if str(status_code).startswith("20"):
metric = input_metric.replace(
"rogeros_tools_exec_time",
"rogeros_events")
metric = metric + ",source=tools" + ",task_id=" + task_id
self.statsd_counter_logging(metric)
except (Exception) as e:
print("The following error occurred: %s" % e,
file=sys.stderr)
raise
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "post_push"
hook_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(
args.app_name) + ",identifier=" + str(
self.identifier) + ",config_name=" + str(
config_name) + ",env=" + str(
environment) + ",user="******"The following error occurred: %s" % e, file=sys.stderr)
raise
def setUp(self):
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
self.config_dir = self.base_dir + "/tests/configs"
self.template_dir = self.base_dir + "/tests/templates"
pass
def setUp(self):
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
self.task_id.extend(container_task_id)
except (Exception) as e:
print("ERROR - : %s" %e, file=sys.stderr)
execution_result = 'FAILURE'
raise
finally:
# todo: maybe send datadog event from here?
pass
hookname = "post_push"
exit_code = hooksObj.run_hook(hookname, data, app_path, args.env, settingObj.getUser())
if exit_code != 0:
raise ValueError("{} hook failed.".format(hookname))
print(colored("******Done with the PUSH step******", "green"))
except (Exception) as e:
raise ValueError("ERROR - {}".format(e))
if __name__ == "__main__":
settingObj = Settings()
appObj = AppConfig()
frameworkUtils = FrameworkUtils()
hooksObj = Hooks()
roger_push = RogerPush()
try:
roger_push.parser = roger_push.parse_args()
roger_push.args = roger_push.parser.parse_args()
roger_push.main(settingObj, appObj, frameworkUtils, hooksObj, roger_push.args)
except (Exception) as e:
printException(e)
def main(self, settingObj, appObj, hooksObj, dockerUtilsObj, dockerObj, args):
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
config = appObj.getConfig(config_dir, args.config_file)
hooksObj.config_file = args.config_file
roger_env = appObj.getRogerEnv(config_dir)
config_name = ""
if 'name' in config:
config_name = config['name']
common_repo = config.get('repo', '')
if not hasattr(args, "env"):
args.env = "dev"
data = appObj.getAppData(config_dir, args.config_file, args.app_name)
if not data:
raise ValueError('Application with name [{}] or data for it not found at {}/{}.'.format(
args.app_name, config_dir, args.config_file))
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', args.app_name)
build_args = {}
if 'build-args' in data:
if 'environment' in data['build-args']:
if args.env in data['build-args']['environment']:
build_args = data['build-args']['environment'][args.env]
projects = data.get('privateProjects', [])
docker_path = data.get('path', 'none')
# get/update target source(s)
file_exists = True
file_path = ''
cur_dir = ''
if "PWD" in os.environ:
cur_dir = os.environ.get('PWD')
abs_path = os.path.abspath(args.directory)
repo_name = appObj.getRepoName(repo)
if docker_path != 'none':
if abs_path == args.directory:
file_path = "{0}/{1}/{2}".format(args.directory,
repo_name, docker_path)
else:
file_path = "{0}/{1}/{2}/{3}".format(
cur_dir, args.directory, repo_name, docker_path)
else:
if abs_path == args.directory:
file_path = "{0}/{1}".format(args.directory, repo_name)
else:
file_path = "{0}/{1}/{2}".format(cur_dir,
args.directory, repo_name)
if not hasattr(args, "app_name"):
args.app_name = ""
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
args.app_name = self.utils.extract_app_name(args.app_name)
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "pre_build"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"{0}/{1}".format(file_path, data['build_filename']))
file_exists = os.path.exists(build_filename)
if not file_exists:
raise ValueError("Specified build file: {} does not exist. Exiting build.".format(build_filename))
else:
file_exists = os.path.exists("{0}/Dockerfile".format(file_path))
if file_exists:
if 'registry' not in roger_env:
raise ValueError('Registry not found in roger-mesos-tools.config file.')
else:
self.registry = roger_env['registry']
self.tag_name = args.tag_name
image = "{0}/{1}".format(roger_env['registry'], args.tag_name)
try:
if abs_path == args.directory:
try:
dockerObj.docker_build(
dockerUtilsObj, appObj, args.directory, repo, projects, docker_path, image, build_args, build_filename)
except ValueError:
print('Docker build failed.')
raise
else:
directory = '{0}/{1}'.format(cur_dir, args.directory)
try:
dockerObj.docker_build(
dockerUtilsObj, appObj, directory, repo, projects, docker_path, image, build_args, build_filename)
except ValueError:
print('Docker build failed.')
raise
build_message = "Image {0} built".format(image)
if(args.push):
exit_code = dockerUtilsObj.docker_push(image)
if exit_code != 0:
raise ValueError(
'Docker push failed.')
build_message += " and pushed to registry {}".format(roger_env[
'registry'])
print(build_message)
except (IOError) as e:
print("The folowing error occurred.(Error: %s).\n" %
e, file=sys.stderr)
raise
else:
print("Dockerfile does not exist in dir: {}".format(file_path))
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "post_build"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"The following error occurred: %s" %
e, file=sys.stderr)
raise
finally:
try:
# If the build fails before going through any steps
if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
if 'execution_result' not in globals() and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals() and 'config_name' not in locals():
config_name = ""
if not hasattr(args, "env"):
args.env = "dev"
if not hasattr(args, "app_name"):
args.app_name = ""
if 'settingObj' not in globals() and 'settingObj' not in locals():
settingObj = Settings()
if 'execution_result' is 'FAILURE':
self.outcome = 0
sc = self.utils.getStatsClient()
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=build" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(args.env) + ",user="******"The following error occurred: %s" %
e, file=sys.stderr)
raise
import json
import yaml
import re
from jinja2 import Environment, FileSystemLoader
from termcolor import colored
from cli.framework import Framework
from cli.utils import Utils
from cli.settings import Settings
from cli.marathonvalidator import MarathonValidator
from cli.proxyparser import ProxyParser
from cli.appconfig import AppConfig
from distutils.version import LooseVersion
requests.packages.urllib3.disable_warnings()
utils = Utils()
settings = Settings()
class Marathon(Framework):
def __init__(self):
self.user = None
self.passw = None
self.marathonvalidator = MarathonValidator()
self.proxyparser = ProxyParser()
def getName(self):
return "Marathon"
def get(self, roger_env, environment):
url = roger_env['environments'][environment][
'marathon_endpoint'] + "/v2/apps"
def main(self, settings, appConfig, gitObject, hooksObj, args):
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS' # Assume the execution_result to be SUCCESS unless exception occurs
environment = "dev"
if hasattr(args, "environment"):
environment = args.environment
settingObj = settings
appObj = appConfig
gitObj = gitObject
config_dir = settingObj.getConfigDir()
hooksObj.config_file = args.config_file
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
common_repo = config.get('repo', '')
data = appObj.getAppData(config_dir, args.config_file, args.app_name)
if not data:
raise ValueError('Application with name [{}] or data for it not found at {}/{}.'.format(
args.app_name, config_dir, args.config_file))
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', args.app_name)
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
if not os.path.exists(args.directory):
try:
os.makedirs(args.directory)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
if not hasattr(args, "app_name"):
args.app_name = ""
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
args.app_name = self.utils.extract_app_name(args.app_name)
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "pre_gitpull"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"{0}/{1}".format(args.directory, repo_name)
if os.path.isdir(path):
with chdir(path):
exit_code = gitObj.gitPull(branch)
else:
with chdir('{0}'.format(args.directory)):
exit_code = gitObj.gitShallowClone(repo, branch)
if exit_code != 0:
raise ValueError('gitpull failed.')
hooksObj.statsd_message_list = self.statsd_message_list
hookname = "post_gitpull"
hookname_input_metric = "roger-tools.rogeros_tools_exec_time," + "event=" + hookname + ",app_name=" + str(args.app_name) + ",identifier=" + str(self.identifier) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"The following error occurred: %s" %
e, file=sys.stderr)
execution_result = 'FAILURE'
raise
finally:
try:
# If the gitpull fails before going through any steps
if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
if 'execution_result' not in globals() and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals() and 'config_name' not in locals():
config_name = ""
if 'environment' not in globals() and 'environment' not in locals():
environment = "dev"
if not hasattr(args, "app_name"):
args.app_name = ""
if 'settingObj' not in globals() and 'settingObj' not in locals():
settingObj = Settings()
if 'execution_result' is 'FAILURE':
self.outcome = 0
sc = self.utils.getStatsClient()
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.app_name)
time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.app_name) + ",event=gitpull" + ",identifier=" + str(self.identifier) + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******"The following error occurred: %s" %
e, file=sys.stderr)
raise
def main(self, settingObject, appObject, frameworkUtilsObject, gitObj,
hooksObj, args):
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS'
settingObj = settingObject
appObj = appObject
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
roger_env = appObj.getRogerEnv(config_dir)
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
if 'registry' not in roger_env:
raise ValueError(
'Registry not found in roger-mesos-tools.config file.')
else:
self.registry = roger_env['registry']
# Setup for Slack-Client, token, and git user
if 'notifications' in config:
self.slack = Slack(
config['notifications'],
'/home/vagrant/.roger_cli.conf.d/slack_token')
self.identifier = self.utils.get_identifier(
config_name, settingObj.getUser(), args.application)
apps = []
apps_container_dict = {}
if args.application == 'all':
apps = config['apps'].keys()
else:
if ":" not in args.application and "[" not in args.application:
apps.append(args.application)
else:
for item in args.application.split(":"):
if '[' in item:
matchObj = re.match(r'(.*)\[(.*)\]', item)
apps.append(matchObj.group(1))
apps_container_dict[matchObj.group(
1)] = matchObj.group(2)
else:
apps.append(item)
common_repo = config.get('repo', '')
environment = roger_env.get('default_environment', '')
work_dir = ''
if args.directory:
work_dir = args.directory
temp_dir_created = False
print("Using {0} as the working directory".format(work_dir))
else:
work_dir = mkdtemp()
temp_dir_created = True
print("Created a temporary dir: {0}".format(work_dir))
if args.environment is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file"
)
else:
print(
"Using value {} from environment variable $ROGER_ENV"
.format(env_var))
environment = env_var
else:
environment = args.environment
if environment not in roger_env['environments']:
self.removeDirTree(work_dir, args, temp_dir_created)
raise ValueError(
'Environment not found in roger-mesos-tools.config file.')
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
try:
for app in apps:
if app not in config['apps']:
print(
'Application {} specified not found.'.format(app))
else:
try:
print("Deploying {} ...".format(app))
self.deployApp(settingObject, appObject,
frameworkUtilsObject, gitObj,
hooksObj, root, args, config,
roger_env, work_dir, config_dir,
environment, app, branch,
self.slack, args.config_file,
common_repo, temp_dir_created,
apps_container_dict)
except (IOError, ValueError) as e:
print(
"The following error occurred when deploying {}: {}"
.format(app, e),
file=sys.stderr)
pass # try deploying the next app
except (Exception) as e:
print("The following error occurred: %s" % e, file=sys.stderr)
raise
except (Exception) as e:
execution_result = 'FAILURE'
print("The following error occurred: %s" % e, file=sys.stderr)
raise
finally:
# Check if the initializition of variables carried out
if 'function_execution_start_time' not in globals(
) and 'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
if 'execution_result' not in globals(
) and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals() and 'config_name' not in locals(
):
config_name = ""
if 'environment' not in globals() and 'environment' not in locals(
):
environment = "dev"
if not hasattr(args, "application"):
args.application = ""
if 'settingObj' not in globals() and 'settingObj' not in locals():
settingObj = Settings()
if 'work_dir' not in globals() and 'work_dir' not in locals():
work_dir = ''
temp_dir_created = False
if not (self.rogerGitPullObject.outcome is 1
and self.rogerBuildObject.outcome is 1
and self.rogerPushObject.outcome is 1):
execution_result = 'FAILURE'
try:
# If the deploy fails before going through any steps
sc = self.utils.getStatsClient()
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(
config_name, settingObj.getUser(), args.application)
args.application = self.utils.extract_app_name(
args.application)
time_take_milliseonds = (
(datetime.now() -
function_execution_start_time).total_seconds() * 1000)
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(
args.application) + ",event=deploy" + ",outcome=" + str(
execution_result
) + ",config_name=" + str(config_name) + ",env=" + str(
environment) + ",user="******",identifier=" + str(
self.identifier)
tup = (input_metric, time_take_milliseonds)
self.statsd_message_list.append(tup)
self.removeDirTree(work_dir, args, temp_dir_created)
except (Exception) as e:
print("The following error occurred: %s" % e, file=sys.stderr)
raise
def setUp(self):
self.appObj = AppConfig()
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
self.configs_dir = self.base_dir + "/tests/configs"