def getGitSha(self, repo, branch, work_dir):
appObj = AppConfig()
repo_name = appObj.getRepoName(repo)
with chdir("{0}/{1}".format(work_dir, repo_name)):
proc = subprocess.Popen(
["git rev-parse origin/{} --verify HEAD".format(branch)],
stdout=subprocess.PIPE, shell=True)
out = proc.communicate()
return out[0].split('\n')[0]
def gitClone(self, repo, branch):
appObj = AppConfig()
try:
repo_url = appObj.getRepoUrl(repo)
except (ValueError) as e:
print("The folowing error occurred.(Error: %s).\n" %
e, file=sys.stderr)
exit_code = os.system(
"git clone --branch {} {}".format(branch, repo_url))
return exit_code
def gitClone(self, repo, branch):
appObj = AppConfig()
try:
repo_url = appObj.getRepoUrl(repo)
except (ValueError) as e:
print("The folowing error occurred.(Error: %s).\n" % e,
file=sys.stderr)
exit_code = os.system("git clone --branch {} {}".format(
branch, repo_url))
return exit_code
def getGitSha(self, repo, branch, work_dir):
appObj = AppConfig()
repo_name = appObj.getRepoName(repo)
with chdir("{0}/{1}".format(work_dir, repo_name)):
proc = subprocess.Popen(
["git rev-parse origin/{} --verify HEAD".format(branch)],
stdout=subprocess.PIPE,
shell=True)
out = proc.communicate()
return out[0].split('\n')[0]
def get_proxy_config(self, environment):
proxy_config = ""
settingObj = Settings()
appObj = AppConfig()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
host = roger_env['environments'][environment]['host']
proxy_config_path = roger_env['environments'][environment][
'proxy_config_path']
url = "{}{}".format(host, proxy_config_path)
proxy_config = requests.get(url).json()
return proxy_config
def getStatsClient(self):
settingObj = Settings()
appObj = AppConfig()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
statsd_url = ""
statsd_port = ""
if 'statsd_endpoint' in roger_env.keys():
statsd_url = roger_env['statsd_endpoint']
if 'statsd_port' in roger_env.keys():
statsd_port = int(roger_env['statsd_port'])
return statsd.StatsClient(statsd_url, statsd_port)
def get_haproxy_config(self, environment):
haproxy_config = ""
settingObj = Settings()
appObj = AppConfig()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
host = roger_env['environments'][environment]['host']
haproxy_config_path = roger_env['environments'][
environment]['haproxy_config_path']
url = "{}{}".format(host, haproxy_config_path)
haproxy_config = requests.get(url, stream=True)
return haproxy_config.text
def getStatsClient(self):
settingObj = Settings()
appObj = AppConfig()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
statsd_url = ""
statsd_port = ""
if 'statsd_endpoint' in roger_env.keys():
statsd_url = roger_env['statsd_endpoint']
if 'statsd_port' in roger_env.keys():
statsd_port = int(roger_env['statsd_port'])
return statsd.StatsClient(statsd_url, statsd_port)
def get_proxy_config(self, environment):
proxy_config = ""
settingObj = Settings()
appObj = AppConfig()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
host = roger_env['environments'][environment]['host']
proxy_config_path = roger_env['environments'][
environment]['proxy_config_path']
url = "{}{}".format(host, proxy_config_path)
proxy_config = requests.get(url).json()
return proxy_config
def gitShallowClone(self, repo, branch, verbose):
appObj = AppConfig()
try:
repo_url = appObj.getRepoUrl(repo)
except (ValueError) as e:
print("The folowing error occurred.(Error: %s).\n" % e,
file=sys.stderr)
redirect = " >/dev/null 2>&1"
if verbose:
redirect = ""
exit_code = os.system("git clone --depth 1 --branch {} {} {}".format(
branch, repo_url, redirect))
return exit_code
def __init__(self):
self.disabled = True
self.emoji = ':rocket:'
self.defChannel = ''
self.config_dir = ''
self.username = '******'
self.settingObj = Settings()
self.appconfigObj = AppConfig()
self.configLoadFlag = False
self.config = ''
self.config_channels = []
self.config_envs = []
self.config_commands = []
def gitShallowClone(self, repo, branch, verbose):
appObj = AppConfig()
try:
repo_url = appObj.getRepoUrl(repo)
except (ValueError) as e:
print("The folowing error occurred.(Error: %s).\n" %
e, file=sys.stderr)
redirect = " >/dev/null 2>&1"
if verbose:
redirect = ""
exit_code = os.system(
"git clone --depth 1 --branch {} {} {}".format(branch, repo_url, redirect))
return exit_code
def get_image_name(self,
username,
password,
env,
app_id,
config_dir,
config_file,
app_config_object=AppConfig()):
"""
returns the application image name
:params:
:username [str]: auth username
:password [str]: auth password
:env: [str]: enviroment
:app_id [str]: app_id
:config_file [str]: file name
:config_dir [str]: config directory path
:app_config_object [object]: AppConfig object
"""
config = app_config_object.getRogerEnv(config_dir)
location = config['environments'][env]['marathon_endpoint']
url = '{location}/v2/apps/{app_id}'.format(location=location,
app_id=app_id)
res = requests.get(url, auth=(username, password))
image = res.json()['app']['container']['docker']['image']
return image
def _get_template_path(self,
container_name,
config_dir,
args,
app_name,
app_object=AppConfig(),
settings_object=Settings()):
"""
Returns the template path for a given container_name
Each framework requires an template_path for the app_id method
:Params:
:config_dir [str]: path to the config directory
:args [argparse.NameSpace]: how to get acceses to the values passed
:app_name [str]: name of app
:app_object [cli.appconfig.AppConfig]: instance of AppConfig
:settings_object [cli.settings.Settings]: instance of Settings
"""
data = app_object.getConfig(config_dir, args.config)
repo = self._config_resolver('repo', app_name, args.config)
template_path = self._config_resolver('template_path', app_name,
args.config)
# this path is always relative to the root repo dir, so join
if template_path and not os.path.isabs(template_path):
app_path = os.path.join(self._temp_dir, repo, template_path)
else:
app_path = settings_object.getTemplatesDir()
file_name = "{0}-{1}.json".format(data['name'], container_name)
return os.path.join(app_path, file_name)
def __init__(self,
app_config=AppConfig(),
settings=Settings(),
framework_utils=FrameworkUtils(),
framework=Marathon()):
self._app_config = app_config
self._settings = settings
self._framework_utils = framework_utils
self._framework = framework
self._config_dir = None
self._roger_env = None
self._temp_dir = None
class TestAppConfig(unittest.TestCase):
def setUp(self):
self.appObj = AppConfig()
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
self.configs_dir = self.base_dir + "/tests/configs"
def test_getRogerEnv(self):
roger_env = self.appObj.getRogerEnv(self.configs_dir)
assert roger_env['registry'] == "example.com:5000"
assert roger_env['default_environment'] == "dev"
assert roger_env['environments']['dev'][
'marathon_endpoint'] == "http://dev.example.com:8080"
assert roger_env['environments']['prod'][
'chronos_endpoint'] == "http://prod.example.com:4400"
def test_getConfig(self):
config = self.appObj.getConfig(self.configs_dir, "app.json")
assert config['name'] == "test-app"
assert config['repo'] == "roger"
assert config['vars']['environment']['prod']['mem'] == "2048"
assert len(config['apps'].keys()) == 3
for app in config['apps']:
assert "test_app" in app
assert config['apps'][app]['imageBase'] == "test_app_base"
def test_getAppData(self):
app_data = self.appObj.getAppData(self.configs_dir, "app.json",
"app_name")
assert app_data == ''
app_data = self.appObj.getAppData(self.configs_dir, "app.json",
"test_app")
assert app_data['imageBase'] == "test_app_base"
assert len(app_data['containers']) == 2
def tearDown(self):
pass
def get_image_name(
self,
username,
password,
env,
name,
config_dir,
config_file,
app_config_object=AppConfig()
):
config = app_config_object.getRogerEnv(config_dir)
location = config['environments'][env]['chronos_endpoint']
url = '{location}/scheduler/jobs/search?name={name}'.format(
location=location, name=name)
res = requests.get(url, auth=(username, password))
imagename = res.json()[0]['container']['image']
return imagename
def setUp(self):
parser = argparse.ArgumentParser(description='Args for test')
parser.add_argument(
'-e',
'--env',
metavar='env',
help="Environment to deploy to. example: 'dev' or 'stage'")
parser.add_argument(
'--skip-push',
'-s',
help="Don't push. Only generate components. Defaults to false.",
action="store_true")
parser.add_argument(
'--secrets-file',
'-S',
help=
"Specify an optional secrets file for deploy runtime variables.")
self.parser = parser
self.args = parser
self.utils = Utils()
self.appConfig = AppConfig()
"Environment variable $ROGER_ENV is not set.Using the default set from roger-mesos-tools.config file"
)
else:
print(
"Using value {} from environment variable $ROGER_ENV".
format(env_var))
environment = env_var
else:
environment = args.env
if environment not in roger_env['environments']:
raise ValueError(
colored(
"Environment not found in roger-mesos-tools.config file.",
"red"))
app_details = self.get_app_details(framework, proxyparser, environment,
args, roger_env)
self.print_app_details(app_details, args)
if __name__ == '__main__':
settings = Settings()
appconfig = AppConfig()
framework = Marathon()
proxyparser = ProxyParser()
roger_ps = RogerPS()
roger_ps.parser = roger_ps.parse_args()
roger_ps.args = roger_ps.parser.parse_args()
roger_ps.main(settings, appconfig, framework, proxyparser, roger_ps.args)
def setUp(self):
self.marathon = Marathon()
self.appconfig = AppConfig()
self.task_id.extend(container_task_id)
except (Exception) as e:
print("ERROR - : %s" %e, file=sys.stderr)
execution_result = 'FAILURE'
raise
finally:
# todo: maybe send datadog event from here?
pass
hookname = "post_push"
exit_code = hooksObj.run_hook(hookname, data, app_path, args.env, settingObj.getUser())
if exit_code != 0:
raise ValueError("{} hook failed.".format(hookname))
print(colored("******Done with the PUSH step******", "green"))
except (Exception) as e:
raise ValueError("ERROR - {}".format(e))
if __name__ == "__main__":
settingObj = Settings()
appObj = AppConfig()
frameworkUtils = FrameworkUtils()
hooksObj = Hooks()
roger_push = RogerPush()
try:
roger_push.parser = roger_push.parse_args()
roger_push.args = roger_push.parser.parse_args()
roger_push.main(settingObj, appObj, frameworkUtils, hooksObj, roger_push.args)
except (Exception) as e:
printException(e)
class WebHook:
def __init__(self):
self.disabled = True
self.emoji = ':rocket:'
self.defChannel = ''
self.config_dir = ''
self.username = '******'
self.settingObj = Settings()
self.appconfigObj = AppConfig()
self.configLoadFlag = False
self.config = ''
self.config_channels = []
self.config_envs = []
self.config_commands = []
def webhookSetting(self):
"""
Prepares webhook setting from roger-mesos-tools.config file
File should have all the required variables otherwise it exits
with a warning message
"""
try:
self.config_dir = self.settingObj.getConfigDir()
roger_env = self.appconfigObj.getRogerEnv(self.config_dir)
if 'slack_webhook_url' in roger_env.keys():
self.webhookURL = roger_env['slack_webhook_url']
if 'slack_api_token' in roger_env.keys():
self.token = roger_env['slack_api_token']
if 'slack_default_channel' in roger_env.keys():
self.defChannel = roger_env['slack_default_channel']
if 'slack_deploy_botid' in roger_env.keys():
self.botid = roger_env['slack_deploy_botid']
except (Exception) as e:
print("Warning: slackweb basic initialization failed (error: %s).\
Not using slack." % e)
return
try:
self.sc = SlackClient(self.token)
self.client = slackweb.Slack(url=self.webhookURL)
self.disabled = False
except (Exception) as e:
print("Warning: slackweb basic initialization failed (error: %s).\
Not using slack." % e)
return # disabled flag remains False
def custom_api_call(self, text, channel):
""" Makes the webhook call
Keyword arguments:
text -- message to be posted
channel -- to which channel
posts a message if rogeros - bot is present
"""
try:
self.webhookSetting()
if len(channel) == 0:
channel = self.defChannel
var = self.sc.api_call("channels.list")
length = len(var['channels'])
for iterator in range(0, length):
# getting rid of # for comparison
if channel[1:] == var['channels'][iterator]['name']:
if self.botid in var['channels'][iterator]['members']:
if not self.disabled:
self.client.notify(channel=channel, username=self.username,
icon_emoji=self.emoji, text=text)
except (Exception) as e:
# notify to channel and log it as well
printException(e)
raise
def areBasicKeysAvailableInConfig(self, config):
if 'notifications' not in config:
return False
tempList = config['notifications'].keys()
if 'channels' not in tempList:
return False # if no channel is available nothing can be done
if 'envs' not in tempList:
self.config_envs = ['dev', 'production', 'staging', 'local']
if 'commands' not in tempList:
self.config_commands = ['pull', 'build', 'push']
return True
def areBasicKeysAvailableInAppdata(self, appdata):
tempList = appdata.keys()
if 'notifications' not in tempList:
return False
appKeys = appdata['notifications'].keys()
if 'channels' not in appKeys:
return False
return True
def configLevelSettings(self, config_file):
""" Prepares all the config_level settings as variables
Keyword arguments:
config_file -- This is the file name passed as argument
return three sets of channels, envs and commands.
self variable as config level variable not expected to change for the run
"""
if (not self.configLoadFlag):
self.config = self.appconfigObj.getConfig(self.config_dir, config_file)
self.configLoadFlag = True
if(not self.areBasicKeysAvailableInConfig(self.config)):
return
try:
self.config_channels = Set(self.config['notifications']['channels'])
if 'envs' in self.config['notifications'].keys():
self.config_envs = Set(self.config['notifications']['envs'])
if 'commands' in self.config['notifications'].keys():
self.config_commands = Set(self.config['notifications']['commands'])
if (len(self.config_channels) == 0 or len(self.config_envs) == 0 or len(self.config_commands) == 0):
return
except (Exception, KeyError, ValueError) as e:
# notify to channel and log it as well
printException(e)
raise
def invoke_webhook(self, appdata, config_file, action, env, user):
""" Pepares set and posts to slack channel
Keyword arguments:
appdata -- this is value related to an app
hook_input_metric -- value it gets from hook class per app
config_file -- the file name under for the app deployment
"""
envSet = []
commandsSet = []
self.function_execution_start_time = datetime.now()
self.webhookSetting()
self.configLevelSettings(config_file)
self.action = action
self.envr = env
self.user = user
self.app_name = appdata['name']
try:
if(not self.areBasicKeysAvailableInAppdata(appdata)):
return
if 'notifications' in appdata:
channelsSet = Set(appdata['notifications']['channels'])
# to handle if tag is there but no data is present
if (len(self.config_channels) != 0):
channelsSet = channelsSet.union(self.config_channels)
# to handle if tag is there
if 'envs' in appdata['notifications'].keys():
envSet = Set(appdata['notifications']['envs'])
# to handle if tag is there but data is not there
if (len(self.config_envs) != 0):
envSet = envSet.union(self.config_envs)
if 'commands' in appdata['notifications'].keys():
commandsSet = Set(appdata['notifications']['commands'])
if (len(self.config_commands) != 0):
commandsSet = commandsSet.union(self.config_commands)
if (len(channelsSet) == 0 or len(envSet) == 0 or len(commandsSet) == 0):
return
else:
if list(self.config_envs)[0] == 'all':
self.config_envs = ['dev', 'production', 'staging', 'local']
if list(self.config_commands)[0] == 'all':
self.config_commands = ['pull', 'build', 'push']
self.postToSlack(self.action, self.config_envs, self.config_commands, self.config_channels)
return
if list(envSet)[0] == 'all':
envSet = ['dev', 'production', 'staging', 'local']
if list(commandsSet)[0] == 'all':
commandsSet = ['pull', 'build', 'push']
except (Exception, KeyError, ValueError) as e:
printException(e)
raise
try:
self.postToSlack(self.action, envSet, commandsSet, channelsSet)
except (Exception) as e:
self.custom_api_call("Error : %s" % e, self.defChannel)
printException(e)
raise
def postToSlack(self, action, envSet, commandsSet, channelsSet):
""" Prepares post to slack channel
Keyword arguments:
action -- action extracted from hookname_input_metric
envSet -- set of accepted environment
commandsSet -- set of accepted commandssets
channelsSet -- set of accepted channelsets
"""
try:
if ('post' in action and self.envr in envSet and action.split('_')[1] in commandsSet):
for channel in channelsSet:
timeElapsed = '{0:.2f}'.format((datetime.now() - self.function_execution_start_time).total_seconds() * 1000)
timeasInt = float(timeElapsed)
h, m, s, ms = self.makeTimeReadable(timeasInt)
readableTime = self.createMesage(h, m, s, ms)
slackMessage = ("Completed *" + action.split('_')[1] + "* of *" + self.app_name + "* on *" + self.envr + "* in *" + readableTime + "* (triggered by *" + self.user + "*)")
self.custom_api_call(slackMessage, '#' + channel)
except (Exception) as e:
self.custom_api_call("Error : %s" % e, self.defChannel)
printException(e)
raise
def makeTimeReadable(self, ms):
""" Make time readable
Keyword arguments:
ms -- time in miliseconds
"""
s = ms / 1000
m, s = divmod(s, 60)
h, m = divmod(m, 60)
return h, m, s, ms
def createMesage(self, h, m, s, ms):
message = ''
if int(h) > 0:
message += str(int(h)) + "h "
if int(m) > 0:
message += str(int(m)) + "m "
if int(s) > 0:
message += str(int(s)) + "s "
message += str(ms) + "ms "
return message
def setUp(self):
self.appObj = AppConfig()
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
self.configs_dir = self.base_dir + "/tests/configs"
else:
raise ValueError("Config file - {} does not exist, no app repo defined either".format(args.config_file))
if __name__ == "__main__":
roger_deploy = RogerDeploy()
roger_deploy.parser = roger_deploy.parseArgs()
args = roger_deploy.parser.parse_args()
gitObj = GitUtils()
# appropriate exception will be thrown if config file is not found, no point of catching it
# since we can't do anything about it except print an error message which exception already has
roger_deploy.locateConfigFile(args, gitObj)
settingObj = Settings()
# if we reach here, we have already located a config file otherwise locateConfigFile would have thrown exception
appObj = AppConfig()
appObj.config_file_path = args.config_file
frameworkUtils = FrameworkUtils()
hooksObj = Hooks()
roger_deploy.main(settingObj, appObj, frameworkUtils,
gitObj, hooksObj, args)
result_list = []
tools_version_value = roger_deploy.utils.get_version()
image_name = roger_deploy.registry + "/" + roger_deploy.image_name
image_tag_value = urllib.quote("'" + image_name + "'")
try:
for task_id_value in roger_deploy.rogerPushObject.task_id:
statsd_message_list = roger_deploy.utils.append_arguments(roger_deploy.statsd_message_list, task_id=task_id_value, tools_version=tools_version_value, image_tag=image_tag_value)
result_list.append(statsd_message_list)
sc = roger_deploy.utils.getStatsClient()
