def getStatsClient(self):
settingObj = Settings()
appObj = AppConfig()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
statsd_url = ""
statsd_port = ""
if 'statsd_endpoint' in roger_env.keys():
statsd_url = roger_env['statsd_endpoint']
if 'statsd_port' in roger_env.keys():
statsd_port = int(roger_env['statsd_port'])
return statsd.StatsClient(statsd_url, statsd_port)
def get_haproxy_config(self, environment):
haproxy_config = ""
settingObj = Settings()
appObj = AppConfig()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
host = roger_env['environments'][environment]['host']
haproxy_config_path = roger_env['environments'][
environment]['haproxy_config_path']
url = "{}{}".format(host, haproxy_config_path)
haproxy_config = requests.get(url, stream=True)
return haproxy_config.text
def get_proxy_config(self, environment):
proxy_config = ""
settingObj = Settings()
appObj = AppConfig()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
host = roger_env['environments'][environment]['host']
proxy_config_path = roger_env['environments'][environment][
'proxy_config_path']
url = "{}{}".format(host, proxy_config_path)
proxy_config = requests.get(url).json()
return proxy_config
def getStatsClient(self):
settingObj = Settings()
appObj = AppConfig()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
statsd_url = ""
statsd_port = ""
if 'statsd_endpoint' in roger_env.keys():
statsd_url = roger_env['statsd_endpoint']
if 'statsd_port' in roger_env.keys():
statsd_port = int(roger_env['statsd_port'])
return statsd.StatsClient(statsd_url, statsd_port)
def get_proxy_config(self, environment):
proxy_config = ""
settingObj = Settings()
appObj = AppConfig()
config_dir = settingObj.getConfigDir()
roger_env = appObj.getRogerEnv(config_dir)
host = roger_env['environments'][environment]['host']
proxy_config_path = roger_env['environments'][
environment]['proxy_config_path']
url = "{}{}".format(host, proxy_config_path)
proxy_config = requests.get(url).json()
return proxy_config
def getRepoUrl(self, repo):
if repo.startswith('*****@*****.**'):
return repo
else:
settingObj = Settings()
config_dir = settingObj.getConfigDir()
roger_env = self.getRogerEnv(config_dir)
if 'default_github_repo_prefix' in roger_env.keys():
prefix = roger_env['default_github_repo_prefix']
else:
raise ValueError(
"Could not determine github repo.Please provide default \"github repo prefix\" or ensure repo startswith [email protected]")
return str(prefix + '{}.git'.format(repo))
def getRepoUrl(self, repo):
if repo.startswith('*****@*****.**'):
return repo
else:
settingObj = Settings()
config_dir = settingObj.getConfigDir()
roger_env = self.getRogerEnv(config_dir)
if 'default_github_repo_prefix' in roger_env.keys():
prefix = roger_env['default_github_repo_prefix']
else:
raise ValueError(
"Could not determine github repo.Please provide default \"github repo prefix\" or ensure repo startswith [email protected]"
)
return str(prefix + '{}.git'.format(repo))
class TestSettings(unittest.TestCase):
def setUp(self):
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
def test_getConfigDir(self):
set_config_dir = ''
if "ROGER_CONFIG_DIR" in os.environ:
set_config_dir = os.environ.get('ROGER_CONFIG_DIR')
os.environ["ROGER_CONFIG_DIR"] = self.base_dir + "/testconfigdir"
config_dir = self.settingObj.getConfigDir()
assert config_dir == self.base_dir + "/testconfigdir"
del os.environ['ROGER_CONFIG_DIR']
try:
config_dir = self.settingObj.getConfigDir()
except (ValueError) as e:
assert (
"Environment variable $ROGER_CONFIG_DIR is not set." in e)
if set_config_dir.strip() != '':
os.environ["ROGER_CONFIG_DIR"] = "{}".format(set_config_dir)
def test_getComponentsDir(self):
set_comp_dir = ''
if "ROGER_COMPONENTS_DIR" in os.environ:
set_comp_dir = os.environ.get('ROGER_COMPONENTS_DIR')
os.environ["ROGER_COMPONENTS_DIR"] = self.base_dir + "/testcompdir"
comp_dir = self.settingObj.getComponentsDir()
assert comp_dir == self.base_dir + "/testcompdir"
del os.environ['ROGER_COMPONENTS_DIR']
try:
comp_dir = self.settingObj.getComponentsDir()
except (ValueError) as e:
assert (
"Environment variable $ROGER_COMPONENTS_DIR is not set." in e)
if set_comp_dir.strip() != '':
os.environ["ROGER_COMPONENTS_DIR"] = "{}".format(set_comp_dir)
def test_getTemplatesDir(self):
set_temp_dir = ''
if "ROGER_TEMPLATES_DIR" in os.environ:
set_temp_dir = os.environ.get('ROGER_TEMPLATES_DIR')
os.environ["ROGER_TEMPLATES_DIR"] = self.base_dir + "/testtempldir"
temp_dir = self.settingObj.getTemplatesDir()
assert temp_dir == self.base_dir + "/testtempldir"
del os.environ['ROGER_TEMPLATES_DIR']
try:
temp_dir = self.settingObj.getTemplatesDir()
except (ValueError) as e:
assert (
"Environment variable $ROGER_TEMPLATES_DIR is not set." in e)
if set_temp_dir.strip() != '':
os.environ["ROGER_TEMPLATES_DIR"] = "{}".format(set_temp_dir)
def test_getSecretsDir(self):
set_sect_dir = ''
if "ROGER_SECRETS_DIR" in os.environ:
set_sect_dir = os.environ.get('ROGER_SECRETS_DIR')
os.environ["ROGER_SECRETS_DIR"] = self.base_dir + "/testsectdir"
sect_dir = self.settingObj.getSecretsDir()
assert sect_dir == self.base_dir + "/testsectdir"
del os.environ['ROGER_SECRETS_DIR']
try:
sect_dir = self.settingObj.getSecretsDir()
except (ValueError) as e:
assert (
"Environment variable $ROGER_SECRETS_DIR is not set." in e)
if set_sect_dir.strip() != '':
os.environ["ROGER_SECRETS_DIR"] = "{}".format(set_sect_dir)
def test_getCliDir(self):
set_cli_dir = ''
cli_dir = self.settingObj.getCliDir()
assert cli_dir is not None
def tearDown(self):
pass
class WebHook:
def __init__(self):
self.disabled = True
self.emoji = ':rocket:'
self.defChannel = ''
self.config_dir = ''
self.username = '******'
self.settingObj = Settings()
self.appconfigObj = AppConfig()
self.configLoadFlag = False
self.config = ''
self.config_channels = []
self.config_envs = []
self.config_commands = []
def webhookSetting(self):
"""
Prepares webhook setting from roger-mesos-tools.config file
File should have all the required variables otherwise it exits
with a warning message
"""
try:
self.config_dir = self.settingObj.getConfigDir()
roger_env = self.appconfigObj.getRogerEnv(self.config_dir)
if 'slack_webhook_url' in roger_env.keys():
self.webhookURL = roger_env['slack_webhook_url']
if 'slack_api_token' in roger_env.keys():
self.token = roger_env['slack_api_token']
if 'slack_default_channel' in roger_env.keys():
self.defChannel = roger_env['slack_default_channel']
if 'slack_deploy_botid' in roger_env.keys():
self.botid = roger_env['slack_deploy_botid']
except (Exception) as e:
print("Warning: slackweb basic initialization failed (error: %s).\
Not using slack." % e)
return
try:
self.sc = SlackClient(self.token)
self.client = slackweb.Slack(url=self.webhookURL)
self.disabled = False
except (Exception) as e:
print("Warning: slackweb basic initialization failed (error: %s).\
Not using slack." % e)
return # disabled flag remains False
def custom_api_call(self, text, channel):
""" Makes the webhook call
Keyword arguments:
text -- message to be posted
channel -- to which channel
posts a message if rogeros - bot is present
"""
try:
self.webhookSetting()
if len(channel) == 0:
channel = self.defChannel
var = self.sc.api_call("channels.list")
length = len(var['channels'])
for iterator in range(0, length):
# getting rid of # for comparison
if channel[1:] == var['channels'][iterator]['name']:
if self.botid in var['channels'][iterator]['members']:
if not self.disabled:
self.client.notify(channel=channel, username=self.username,
icon_emoji=self.emoji, text=text)
except (Exception) as e:
# notify to channel and log it as well
printException(e)
raise
def areBasicKeysAvailableInConfig(self, config):
if 'notifications' not in config:
return False
tempList = config['notifications'].keys()
if 'channels' not in tempList:
return False # if no channel is available nothing can be done
if 'envs' not in tempList:
self.config_envs = ['dev', 'production', 'staging', 'local']
if 'commands' not in tempList:
self.config_commands = ['pull', 'build', 'push']
return True
def areBasicKeysAvailableInAppdata(self, appdata):
tempList = appdata.keys()
if 'notifications' not in tempList:
return False
appKeys = appdata['notifications'].keys()
if 'channels' not in appKeys:
return False
return True
def configLevelSettings(self, config_file):
""" Prepares all the config_level settings as variables
Keyword arguments:
config_file -- This is the file name passed as argument
return three sets of channels, envs and commands.
self variable as config level variable not expected to change for the run
"""
if (not self.configLoadFlag):
self.config = self.appconfigObj.getConfig(self.config_dir, config_file)
self.configLoadFlag = True
if(not self.areBasicKeysAvailableInConfig(self.config)):
return
try:
self.config_channels = Set(self.config['notifications']['channels'])
if 'envs' in self.config['notifications'].keys():
self.config_envs = Set(self.config['notifications']['envs'])
if 'commands' in self.config['notifications'].keys():
self.config_commands = Set(self.config['notifications']['commands'])
if (len(self.config_channels) == 0 or len(self.config_envs) == 0 or len(self.config_commands) == 0):
return
except (Exception, KeyError, ValueError) as e:
# notify to channel and log it as well
printException(e)
raise
def invoke_webhook(self, appdata, config_file, action, env, user):
""" Pepares set and posts to slack channel
Keyword arguments:
appdata -- this is value related to an app
hook_input_metric -- value it gets from hook class per app
config_file -- the file name under for the app deployment
"""
envSet = []
commandsSet = []
self.function_execution_start_time = datetime.now()
self.webhookSetting()
self.configLevelSettings(config_file)
self.action = action
self.envr = env
self.user = user
self.app_name = appdata['name']
try:
if(not self.areBasicKeysAvailableInAppdata(appdata)):
return
if 'notifications' in appdata:
channelsSet = Set(appdata['notifications']['channels'])
# to handle if tag is there but no data is present
if (len(self.config_channels) != 0):
channelsSet = channelsSet.union(self.config_channels)
# to handle if tag is there
if 'envs' in appdata['notifications'].keys():
envSet = Set(appdata['notifications']['envs'])
# to handle if tag is there but data is not there
if (len(self.config_envs) != 0):
envSet = envSet.union(self.config_envs)
if 'commands' in appdata['notifications'].keys():
commandsSet = Set(appdata['notifications']['commands'])
if (len(self.config_commands) != 0):
commandsSet = commandsSet.union(self.config_commands)
if (len(channelsSet) == 0 or len(envSet) == 0 or len(commandsSet) == 0):
return
else:
if list(self.config_envs)[0] == 'all':
self.config_envs = ['dev', 'production', 'staging', 'local']
if list(self.config_commands)[0] == 'all':
self.config_commands = ['pull', 'build', 'push']
self.postToSlack(self.action, self.config_envs, self.config_commands, self.config_channels)
return
if list(envSet)[0] == 'all':
envSet = ['dev', 'production', 'staging', 'local']
if list(commandsSet)[0] == 'all':
commandsSet = ['pull', 'build', 'push']
except (Exception, KeyError, ValueError) as e:
printException(e)
raise
try:
self.postToSlack(self.action, envSet, commandsSet, channelsSet)
except (Exception) as e:
self.custom_api_call("Error : %s" % e, self.defChannel)
printException(e)
raise
def postToSlack(self, action, envSet, commandsSet, channelsSet):
""" Prepares post to slack channel
Keyword arguments:
action -- action extracted from hookname_input_metric
envSet -- set of accepted environment
commandsSet -- set of accepted commandssets
channelsSet -- set of accepted channelsets
"""
try:
if ('post' in action and self.envr in envSet and action.split('_')[1] in commandsSet):
for channel in channelsSet:
timeElapsed = '{0:.2f}'.format((datetime.now() - self.function_execution_start_time).total_seconds() * 1000)
timeasInt = float(timeElapsed)
h, m, s, ms = self.makeTimeReadable(timeasInt)
readableTime = self.createMesage(h, m, s, ms)
slackMessage = ("Completed *" + action.split('_')[1] + "* of *" + self.app_name + "* on *" + self.envr + "* in *" + readableTime + "* (triggered by *" + self.user + "*)")
self.custom_api_call(slackMessage, '#' + channel)
except (Exception) as e:
self.custom_api_call("Error : %s" % e, self.defChannel)
printException(e)
raise
def makeTimeReadable(self, ms):
""" Make time readable
Keyword arguments:
ms -- time in miliseconds
"""
s = ms / 1000
m, s = divmod(s, 60)
h, m = divmod(m, 60)
return h, m, s, ms
def createMesage(self, h, m, s, ms):
message = ''
if int(h) > 0:
message += str(int(h)) + "h "
if int(m) > 0:
message += str(int(m)) + "m "
if int(s) > 0:
message += str(int(s)) + "s "
message += str(ms) + "ms "
return message
class TestSettings(unittest.TestCase):
def setUp(self):
self.settingObj = Settings()
self.base_dir = self.settingObj.getCliDir()
def test_getConfigDir(self):
set_config_dir = ''
if "ROGER_CONFIG_DIR" in os.environ:
set_config_dir = os.environ.get('ROGER_CONFIG_DIR')
os.environ["ROGER_CONFIG_DIR"] = self.base_dir + "/testconfigdir"
config_dir = self.settingObj.getConfigDir()
assert config_dir == self.base_dir + "/testconfigdir"
del os.environ['ROGER_CONFIG_DIR']
try:
config_dir = self.settingObj.getConfigDir()
except (ValueError) as e:
assert ("Environment variable $ROGER_CONFIG_DIR is not set." in e)
if set_config_dir.strip() != '':
os.environ["ROGER_CONFIG_DIR"] = "{}".format(set_config_dir)
def test_getComponentsDir(self):
set_comp_dir = ''
if "ROGER_COMPONENTS_DIR" in os.environ:
set_comp_dir = os.environ.get('ROGER_COMPONENTS_DIR')
os.environ["ROGER_COMPONENTS_DIR"] = self.base_dir + "/testcompdir"
comp_dir = self.settingObj.getComponentsDir()
assert comp_dir == self.base_dir + "/testcompdir"
del os.environ['ROGER_COMPONENTS_DIR']
try:
comp_dir = self.settingObj.getComponentsDir()
except (ValueError) as e:
assert ("Environment variable $ROGER_COMPONENTS_DIR is not set."
in e)
if set_comp_dir.strip() != '':
os.environ["ROGER_COMPONENTS_DIR"] = "{}".format(set_comp_dir)
def test_getTemplatesDir(self):
set_temp_dir = ''
if "ROGER_TEMPLATES_DIR" in os.environ:
set_temp_dir = os.environ.get('ROGER_TEMPLATES_DIR')
os.environ["ROGER_TEMPLATES_DIR"] = self.base_dir + "/testtempldir"
temp_dir = self.settingObj.getTemplatesDir()
assert temp_dir == self.base_dir + "/testtempldir"
del os.environ['ROGER_TEMPLATES_DIR']
try:
temp_dir = self.settingObj.getTemplatesDir()
except (ValueError) as e:
assert ("Environment variable $ROGER_TEMPLATES_DIR is not set."
in e)
if set_temp_dir.strip() != '':
os.environ["ROGER_TEMPLATES_DIR"] = "{}".format(set_temp_dir)
def test_getSecretsDir(self):
set_sect_dir = ''
if "ROGER_SECRETS_DIR" in os.environ:
set_sect_dir = os.environ.get('ROGER_SECRETS_DIR')
os.environ["ROGER_SECRETS_DIR"] = self.base_dir + "/testsectdir"
sect_dir = self.settingObj.getSecretsDir()
assert sect_dir == self.base_dir + "/testsectdir"
del os.environ['ROGER_SECRETS_DIR']
try:
sect_dir = self.settingObj.getSecretsDir()
except (ValueError) as e:
assert ("Environment variable $ROGER_SECRETS_DIR is not set." in e)
if set_sect_dir.strip() != '':
os.environ["ROGER_SECRETS_DIR"] = "{}".format(set_sect_dir)
def test_getCliDir(self):
set_cli_dir = ''
cli_dir = self.settingObj.getCliDir()
assert cli_dir is not None
def tearDown(self):
pass