def test_unresolved_jinja_variable_fails(self):
with open(self.configs_dir + '/roger_push_unresolved_jinja.json') as config:
config = json.load(config)
with open(self.configs_dir + '/roger-mesos-tools.config') as roger:
roger_env = yaml.load(roger)
data = config['apps']['container-vars']
self.config = config
self.roger_env = roger_env
self.data = data
settings = mock(Settings)
appConfig = mock(AppConfig)
roger_push = RogerPush()
roger_push.utils = mock(Utils)
marathon = mock(Marathon)
mockedHooks = mock(Hooks)
when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0)
roger_env = self.roger_env
config = self.config
data = self.data
sc = mock(StatsClient)
when(sc).timing(any(), any()).thenReturn(any())
when(roger_push.utils).getStatsClient().thenReturn(sc)
when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any())
when(roger_push.utils).extract_app_name(any()).thenReturn("test")
when(roger_push.utils).get_version().thenReturn(any())
frameworkUtils = mock(FrameworkUtils)
when(frameworkUtils).getFramework(data).thenReturn(marathon)
when(marathon).getName().thenReturn('Marathon')
when(settings).getComponentsDir().thenReturn(
self.base_dir + "/tests/components")
when(settings).getSecretsDir().thenReturn(
self.base_dir + "/tests/secrets")
when(settings).getTemplatesDir().thenReturn(
self.base_dir + "/tests/templates")
when(settings).getConfigDir().thenReturn(self.configs_dir)
when(settings).getCliDir().thenReturn(self.base_dir)
when(settings).getUser().thenReturn(any())
when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env)
when(appConfig).getConfig(self.configs_dir,
"roger_push_unresolved_jinja.json").thenReturn(config)
when(appConfig).getAppData(self.configs_dir,
"roger_push_unresolved_jinja.json", "container-vars").thenReturn(data)
args = self.args
args.env = "dev"
args.skip_push = False
args.force_push = False
args.secrets_dir = ""
args.secrets_file = ""
args.app_name = 'container-vars'
args.config_file = 'roger_push_unresolved_jinja.json'
args.directory = self.base_dir + '/tests/testrepo'
args.image_name = 'tests/v0.1.0'
args.verbose = False
roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args)
verify(frameworkUtils, times=0).put(any(), any(), any(), any())
def test_roger_push_env_from_ROGER_ENV_VAR(self):
settings = mock(Settings)
appConfig = mock(AppConfig)
roger_push = RogerPush()
marathon = mock(Marathon)
mockedHooks = mock(Hooks)
roger_env = self.roger_env
config = self.config
frameworkUtils = mock(FrameworkUtils)
when(settings).getConfigDir().thenReturn(self.configs_dir)
when(mockedHooks).run_hook(any(), any(), any()).thenReturn(0)
when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env)
when(appConfig).getConfig(any(), any()).thenReturn(config)
args = self.args
args.env = None
args.secrets_file = ""
args.skip_push = True
args.app_name = 'grafana_test_app'
args.config_file = 'test.json'
roger_env = appConfig.getRogerEnv(self.configs_dir)
roger_env["default_environment"] = None
# Setting ROGER_ENV to specific value
os.environ["ROGER_ENV"] = "test_env"
with self.assertRaises(ValueError):
roger_push.main(settings, appConfig,
frameworkUtils, mockedHooks, args)
def test_roger_push_with_incorrect_container_name(self):
settings = mock(Settings)
appConfig = mock(AppConfig)
roger_push = RogerPush()
marathon = mock(Marathon)
mockedHooks = mock(Hooks)
roger_env = self.roger_env
config = self.config
appdata = self.data
sc = mock(StatsClient)
when(sc).timing(any(), any()).thenReturn(any())
when(roger_push.utils).getStatsClient().thenReturn(sc)
when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any())
when(roger_push.utils).extract_app_name(any()).thenReturn("test")
frameworkUtils = mock(FrameworkUtils)
when(marathon).getName().thenReturn('Marathon')
when(settings).getConfigDir().thenReturn(self.configs_dir)
when(settings).getUser().thenReturn(any())
when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0)
when(appConfig).getAppData(any(), any(), any()).thenReturn(appdata)
when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env)
when(appConfig).getConfig(any(), any()).thenReturn(config)
args = self.args
args.env = "dev"
args.secrets_file = ""
args.skip_push = True
args.app_name = 'grafana_test_app:test'
args.config_file = 'test.json'
with self.assertRaises(ValueError):
roger_push.main(settings, appConfig,
frameworkUtils, mockedHooks, args)
def test_push_happens_with_validation_error_when_force_push_set(self):
settings = mock(Settings)
appConfig = mock(AppConfig)
roger_push = RogerPush()
roger_push.utils = mock(Utils)
marathon = mock(Marathon)
mockedHooks = mock(Hooks)
roger_env = self.roger_env
config = self.config
data = self.data
when(marathon).getName().thenReturn('Marathon')
when(marathon).put(any(), any(), any(),
any(), any()).thenReturn(["Response [200]", any()])
when(marathon).runDeploymentChecks(any(), any()).thenReturn(True)
frameworkUtils = mock(FrameworkUtils)
frameworkUtils = mock(FrameworkUtils)
sc = mock(StatsClient)
when(sc).timing(any(), any()).thenReturn(any())
when(roger_push.utils).getStatsClient().thenReturn(sc)
when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any())
when(roger_push.utils).extract_app_name(any()).thenReturn("test")
when(roger_push.utils).modify_task_id(any()).thenReturn([any()])
when(roger_push.utils).get_version().thenReturn(any())
when(frameworkUtils).getFramework(data).thenReturn(marathon)
when(settings).getComponentsDir().thenReturn(
self.base_dir + "/tests/components")
when(settings).getSecretsDir().thenReturn(
self.base_dir + "/tests/secrets")
when(settings).getTemplatesDir().thenReturn(
self.base_dir + "/tests/templates")
when(settings).getConfigDir().thenReturn(self.configs_dir)
when(settings).getCliDir().thenReturn(self.base_dir)
when(settings).getUser().thenReturn(any())
when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0)
when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env)
when(appConfig).getConfig(any(), any()).thenReturn(config)
when(appConfig).getAppData(any(), any(), any()).thenReturn(data)
args = self.args
args.env = "dev"
args.secrets_file = ""
args.skip_push = False
args.force_push = True
args.app_name = 'grafana_test_app'
args.config_file = 'test.json'
args.directory = self.base_dir + '/tests/testrepo'
args.image_name = 'grafana/grafana:2.1.3'
args.verbose = False
raised_exception = False
try:
roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args)
except (Exception) as e:
raised_exception = True
self.assertFalse(raised_exception)
def test_roger_push_with_correct_container_name(self):
settings = mock(Settings)
appConfig = mock(AppConfig)
roger_push = RogerPush()
roger_push.utils = mock(Utils)
marathon = mock(Marathon)
mockedHooks = mock(Hooks)
frameworkUtils = mock(FrameworkUtils)
roger_env = self.roger_env
appdata = self.data
config = self.config
sc = mock(StatsClient)
when(sc).timing(any(), any()).thenReturn(any())
when(roger_push.utils).getStatsClient().thenReturn(sc)
when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any())
when(roger_push.utils).extract_app_name(any()).thenReturn("test")
when(frameworkUtils).getFramework(any()).thenReturn(marathon)
when(marathon).getName().thenReturn('Marathon')
when(settings).getComponentsDir().thenReturn(
self.base_dir + "/tests/components")
when(settings).getSecretsDir().thenReturn(
self.base_dir + "/tests/secrets")
when(settings).getTemplatesDir().thenReturn(
self.base_dir + "/tests/templates")
when(settings).getConfigDir().thenReturn(self.configs_dir)
when(settings).getCliDir().thenReturn(self.base_dir)
when(settings).getUser().thenReturn(any())
when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env)
when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0)
when(appConfig).getRogerEnv(any()).thenReturn(roger_env)
appdata["hooks"] = dict([("post_push", "some command")])
when(appConfig).getAppData(any(), any(), any()).thenReturn(appdata)
when(appConfig).getConfig(any(), any()).thenReturn(config)
when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env)
when(appConfig).getConfig(any(), any()).thenReturn(config)
args = self.args
args.env = "dev"
args.secrets_file = ""
args.skip_push = True
args.app_name = 'grafana_test_app:grafana'
args.config_file = 'test.json'
args.image_name = 'grafana/grafana:2.1.3'
args.verbose = False
raised_exception = False
try:
roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args)
except (Exception) as e:
raised_exception = True
self.assertFalse(raised_exception)
verify(frameworkUtils).getFramework(any())
def test_roger_push_grafana_test_app(self):
settings = mock(Settings)
appConfig = mock(AppConfig)
roger_push = RogerPush()
roger_push.utils = mock(Utils)
marathon = mock(Marathon)
mockedHooks = mock(Hooks)
when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0)
roger_env = self.roger_env
config = self.config
data = self.data
sc = mock(StatsClient)
when(sc).timing(any(), any()).thenReturn(any())
when(roger_push.utils).getStatsClient().thenReturn(sc)
when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any())
when(roger_push.utils).extract_app_name(any()).thenReturn("test")
when(marathon).put(any(), any(), any()).thenReturn("Response [200]", any())
frameworkUtils = mock(FrameworkUtils)
when(frameworkUtils).getFramework(data).thenReturn(marathon)
when(marathon).getName().thenReturn('Marathon')
when(settings).getComponentsDir().thenReturn(
self.base_dir + "/tests/components")
when(settings).getSecretsDir().thenReturn(
self.base_dir + "/tests/secrets")
when(settings).getTemplatesDir().thenReturn(
self.base_dir + "/tests/templates")
when(settings).getConfigDir().thenReturn(self.configs_dir)
when(settings).getCliDir().thenReturn(self.base_dir)
when(settings).getUser().thenReturn(any())
when(appConfig).getRogerEnv(any()).thenReturn(roger_env)
when(appConfig).getConfig(any(), any()).thenReturn(config)
when(appConfig).getAppData(any(), any(), any()).thenReturn(data)
args = self.args
args.env = "dev"
args.secrets_file = ""
args.skip_push = True
args.app_name = 'grafana_test_app'
args.config_file = 'test.json'
args.directory = self.base_dir + '/tests/testrepo'
args.image_name = 'grafana/grafana:2.1.3'
args.verbose = False
roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args)
with open(self.base_dir + '/tests/templates/test-app-grafana.json') as output:
output = json.load(output)
assert output['container']['docker'][
'image'] == "grafana/grafana:2.1.3"
verify(settings).getConfigDir()
verify(settings).getComponentsDir()
verify(settings).getTemplatesDir()
verify(settings).getSecretsDir()
verify(frameworkUtils).getFramework(data)
def test_roger_push_with_no_registry_fails(self):
settings = mock(Settings)
appConfig = mock(AppConfig)
roger_push = RogerPush()
marathon = mock(Marathon)
mockedHooks = mock(Hooks)
when(mockedHooks).run_hook(any(), any(), any()).thenReturn(0)
roger_env = self.roger_env
config = self.config
data = self.data
when(marathon).put(any(), any(), any()).thenReturn("Response [200]")
frameworkUtils = mock(FrameworkUtils)
when(frameworkUtils).getFramework(data).thenReturn(marathon)
when(settings).getComponentsDir().thenReturn(
self.base_dir + "/tests/components")
when(settings).getSecretsDir().thenReturn(
self.base_dir + "/tests/secrets")
when(settings).getTemplatesDir().thenReturn(
self.base_dir + "/tests/templates")
when(settings).getConfigDir().thenReturn(self.configs_dir)
when(settings).getCliDir().thenReturn(self.base_dir)
when(appConfig).getRogerEnv(any()).thenReturn(roger_env)
when(appConfig).getConfig(any(), any()).thenReturn(config)
when(appConfig).getAppData(any(), any(), any()).thenReturn(data)
args = self.args
args.env = "dev"
args.secrets_file = ""
args.skip_push = True
args.app_name = 'grafana_test_app'
args.config_file = 'test.json'
args.directory = self.base_dir + '/tests/testrepo'
args.image_name = 'grafana/grafana:2.1.3'
config_dir = settings.getConfigDir()
roger_env = appConfig.getRogerEnv(config_dir)
# Remove registry key from dictionary
del roger_env['registry']
with self.assertRaises(ValueError):
roger_push.main(settings, appConfig,
frameworkUtils, mockedHooks, args)
def test_roger_push_with_no_app_fails(self):
settings = mock(Settings)
appConfig = mock(AppConfig)
roger_push = RogerPush()
roger_push.utils = mock(Utils)
marathon = mock(Marathon)
mockedHooks = mock(Hooks)
when(mockedHooks).run_hook(any(), any(), any()).thenReturn(0)
roger_env = self.roger_env
config = self.config
data = self.data
sc = mock(StatsClient)
when(sc).timing(any(), any()).thenReturn(any())
when(roger_push.utils).getStatsClient().thenReturn(sc)
when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any())
when(roger_push.utils).extract_app_name(any()).thenReturn("test")
when(marathon).put(any(), any(), any()).thenReturn("Response [200]")
frameworkUtils = mock(FrameworkUtils)
when(frameworkUtils).getFramework(data).thenReturn(marathon)
when(settings).getComponentsDir().thenReturn(
self.base_dir + "/tests/components")
when(settings).getSecretsDir().thenReturn(
self.base_dir + "/tests/secrets")
when(settings).getTemplatesDir().thenReturn(
self.base_dir + "/tests/templates")
when(settings).getConfigDir().thenReturn(self.configs_dir)
when(settings).getCliDir().thenReturn(self.base_dir)
when(settings).getUser().thenReturn(any())
when(appConfig).getRogerEnv(any()).thenReturn(roger_env)
when(appConfig).getConfig(any(), any()).thenReturn(config)
when(appConfig).getAppData(any(), any(), any()).thenReturn(data)
args = self.args
args.app_name = ''
args.config_file = 'test.json'
args.env = 'some_test_env'
with self.assertRaises(ValueError):
roger_push.main(settings, appConfig,
frameworkUtils, mockedHooks, args)
def test_roger_push_secrets_replaced(self):
settings = mock(Settings)
appConfig = mock(AppConfig)
roger_push = RogerPush()
roger_push.utils = mock(Utils)
marathon = mock(Marathon)
mockedHooks = mock(Hooks)
roger_env = self.roger_env
config = self.config
data = self.data
sc = mock(StatsClient)
when(sc).timing(any(), any()).thenReturn(any())
when(roger_push.utils).getStatsClient().thenReturn(sc)
when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any())
when(roger_push.utils).extract_app_name(any()).thenReturn("test")
when(marathon).put(any(), any(), any()).thenReturn("Response [200]")
frameworkUtils = mock(FrameworkUtils)
when(frameworkUtils).getFramework(data).thenReturn(marathon)
when(marathon).getName().thenReturn('Marathon')
when(settings).getComponentsDir().thenReturn(
self.base_dir + "/tests/components")
when(settings).getSecretsDir().thenReturn(
self.base_dir + "/tests/secrets")
when(settings).getTemplatesDir().thenReturn(
self.base_dir + "/tests/templates")
when(settings).getConfigDir().thenReturn(self.configs_dir)
when(settings).getCliDir().thenReturn(self.base_dir)
when(settings).getUser().thenReturn(any())
when(appConfig).getRogerEnv(any()).thenReturn(roger_env)
when(appConfig).getConfig(any(), any()).thenReturn(config)
when(appConfig).getAppData(any(), any(), any()).thenReturn(data)
when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0)
args = self.args
args.env = "dev"
args.secrets_file = ""
args.skip_push = True
args.app_name = 'grafana_test_app'
args.config_file = 'test.json'
args.directory = self.base_dir + '/tests/testrepo'
args.image_name = 'grafana/grafana:2.1.3'
args.verbose = False
exit_code = roger_push.main(
settings, appConfig, frameworkUtils, mockedHooks, args)
file_path = ("{0}/{1}/{2}".format(self.components_dir,
args.env, args.config_file))
assert (os.path.isfile(file_path) is not True)
class RogerDeploy(object):
def __init__(self):
self.rogerGitPullObject = RogerGitPull()
self.rogerPushObject = RogerPush()
self.rogerBuildObject = RogerBuild()
self.dockerUtilsObject = DockerUtils()
self.dockerObject = Docker()
self.utils = Utils()
self.slack = None
self.registry = ""
self.image_name = ""
# To remove a temporary directory created by roger-deploy if this
# script exits
def removeDirTree(self, work_dir, args, temp_dir_created):
exists = os.path.exists(os.path.abspath(work_dir))
if exists and (temp_dir_created is True):
shutil.rmtree(work_dir)
print("Deleted temporary dir:{0}".format(work_dir))
def getNextVersion(self, config, roger_env, application, branch, work_dir, repo, args, gitObj):
sha = getGitSha(work_dir, repo, branch, gitObj)
docker_search = self.dockerUtilsObject.docker_search(roger_env['registry'], config['name'], application)
image_version_list = []
version = ''
envs = []
for each_key in roger_env["environments"].keys():
envs.append(each_key)
for line in docker_search.split('\n'):
image = line.split(' ')[0]
# (vmahedia) todo: This belongs in a separate DockerImage class. This class should know
# how to generate the image name, how to search the image or return this name pattern we
# then use to search this image on the registry.
matchObj = re.match("^{0}-{1}-.*/v.*".format(config['name'], application), image)
if matchObj and matchObj.group().startswith(config['name'] + '-' + application):
skip_image = False
for env in envs:
if matchObj.group().startswith("{0}-{1}-{2}".format(config['name'], application, env)):
skip_image = True
break
if skip_image is False:
if verify(str(matchObj.group().split('v')[-1])):
image_version_list.append(matchObj.group().split('v')[-1])
if len(image_version_list) == 0: # Create initial version
version = "{0}/v0.1.0".format(sha)
print("No version currently exist in the Docker Registry. \nDeploying version:{0}".format(
version))
else:
version = self.incrementVersion(sha, image_version_list, args)
return version
def incrementVersion(self, sha, image_version_list, args):
latest = max(image_version_list, key=self.splitVersion)
ver_tuple = self.splitVersion(latest)
latest_version = ''
if args.incr_major:
latest_version = "{0}/v{1}.0.0".format(sha,
(int(ver_tuple[0]) + 1))
return latest_version
if args.incr_patch:
latest_version = "{0}/v{1}.{2}.{3}".format(
sha, int(ver_tuple[0]), int(ver_tuple[1]), (int(ver_tuple[2]) + 1))
return latest_version
latest_version = "{0}/v{1}.{2}.0".format(
sha, int(ver_tuple[0]), (int(ver_tuple[1]) + 1))
return latest_version
def splitVersion(self, version):
major, _, rest = version.partition('.')
minor, _, rest = rest.partition('.')
patch, _, rest = rest.partition('.')
return int(major), int(minor) if minor else 0, int(patch) if patch else 0
def parseArgs(self):
self.parser = argparse.ArgumentParser(
prog='roger deploy', description=describe())
self.parser.add_argument('-e', '--environment', metavar='env',
help="environment to deploy to. Example: 'dev' or 'stage'")
self.parser.add_argument('-b', '--branch', metavar='branch', default='master',
help="branch to pull code from. Defaults to master. Example: 'production' or 'master'")
self.parser.add_argument('-sg', '--skip-gitpull', action="store_true",
help="skip the gitpull step. Defaults to false.")
self.parser.add_argument('-s', '--skip-build', action="store_true",
help="whether to skip the build step. Defaults to false.")
self.parser.add_argument('-M', '--incr-major', action="store_true",
help="increment major in version. Defaults to false.")
self.parser.add_argument('-sp', '--skip-push', action="store_true",
help="skip the push step. Defaults to false.'")
self.parser.add_argument('-v', '--verbose', help="verbose mode for debugging. Defaults to false.", action="store_true")
self.parser.add_argument('-ns', '--disable-swaparoo', help="Disables swaparoo functionality", action="store_true")
self.parser.add_argument('-f', '--force-push', action="store_true",
help="force push. Not recommended. Forces push even if validation checks failed. Applies only if skip_push is false. Defaults to false.")
self.parser.add_argument('-p', '--incr-patch', action="store_true",
help="increment patch in version. Defaults to false.'")
self.parser.add_argument('-S', '--secrets-file',
help="specifies an optional secrets file for deployment runtime variables.")
self.parser.add_argument('--build-arg', action='append',
help='docker build-arg; Use flags multiple times to pass more than one arg')
# (vmahedia) todo Changing this name is slightly more complicated, so making it more verbose for now
self.parser.add_argument('-d', '--directory',
help="App Repo will be checked out here, this is the working dir CLI will use."
"A temporary directory is created if no directory specified."
"Example: '/home/vagrant/work_dir'.")
self.parser.add_argument('application', metavar='application', help="application to deploy. \
Can also push specific containers(comma seperated). Example: 'all' \
or 'app1:app2' or 'kairos' or 'app_name[container1,container2]' \
or'app1[container1,container2]:app2[container3,container4]' or 'app1:app2[container]'")
self.parser.add_argument('config_file', metavar='config_file',
help="Path to a Roger configuration file. This can either be an absolute "
"path or a relative path that is taken relative to the ROGER_CONFIG_DIR "
"environment variable.")
return self.parser
def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args):
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS'
settingObj = settingObject
appObj = appObject
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
roger_env = appObj.getRogerEnv(config_dir)
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
if 'registry' not in roger_env:
raise ValueError('Registry not found in roger-mesos-tools.config file.')
else:
self.registry = roger_env['registry']
# Setup for Slack-Client, token, and git user
# (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event
# Event should be registered and SlackNotification should be one of the members. it can have
# N notifications on a particular "event", Notifications.Notify will broadcast notification to
# all the interested parties.
if 'notifications' in config:
self.slack = Slack(config['notifications'],
'/home/vagrant/.roger_cli.conf.d/slack_token')
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application)
apps = []
apps_container_dict = {}
if args.application == 'all':
apps = config['apps'].keys()
else:
if ":" not in args.application and "[" not in args.application:
apps.append(args.application)
else:
for item in args.application.split(":"):
if '[' in item:
matchObj = re.match(r'(.*)\[(.*)\]', item)
apps.append(matchObj.group(1))
apps_container_dict[matchObj.group(1)] = matchObj.group(2)
else:
apps.append(item)
common_repo = config.get('repo', '')
environment = roger_env.get('default_environment', '')
work_dir = ''
if args.directory:
work_dir = args.directory
temp_dir_created = False
if args.verbose:
print("Using {0} as the working directory".format(work_dir))
else:
work_dir = mkdtemp()
temp_dir_created = True
if args.verbose:
print("Created a temporary dir: {0}".format(work_dir))
if args.environment is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file")
else:
print(
"Using value {} from environment variable $ROGER_ENV".format(env_var))
environment = env_var
else:
environment = args.environment
if environment not in roger_env['environments']:
self.removeDirTree(work_dir, args, temp_dir_created)
raise ValueError('Environment not found in roger-mesos-tools.config file.')
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
try:
for app in apps:
if app not in config['apps']:
raise ValueError('Application {} specified not found.'.format(app))
else:
try:
if args.verbose:
print("Deploying {} ...".format(app))
self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj,
root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict)
except (IOError, ValueError) as e:
error_msg = "Error when deploying {}: {}".format(app, repr(e))
printErrorMsg(error_msg)
pass # try deploying the next app
except (Exception) as e:
printException(e)
raise
except (Exception) as e:
execution_result = 'FAILURE'
printException(e)
raise
finally:
# todo: maybe send the datadog event, need to look
pass
def deployApp(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config,
roger_env, work_dir, config_dir, environment, app, branch, slack, config_file, common_repo, temp_dir_created, apps_container_dict):
startTime = datetime.now()
settingObj = settingObject
appObj = appObject
frameworkUtils = frameworkUtilsObject
environmentObj = roger_env['environments'][environment]
data = appObj.getAppData(config_dir, config_file, app)
frameworkObj = frameworkUtils.getFramework(data)
framework = frameworkObj.getName()
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', app)
image_name = ''
image = ''
skip_gitpull = True if args.skip_gitpull else False
# get/update target source(s)
if not skip_gitpull:
args.app_name = app
args.directory = work_dir
self.rogerGitPullObject.identifier = self.identifier
self.rogerGitPullObject.main(settingObj, appObj, gitObj, hooksObj, args)
skip_build = True if args.skip_build else False
skip_push = True if args.skip_push else False
secrets_file = args.secrets_file if args.secrets_file else None
# Set initial version
# todo (vmahedia) #image_name naming should not be magic, make it explicit
# todo (vmahedia) ExtractClass GitInfo, no need to pass args, we already have the information.
# We deal with only one repo at a time. It may change in future but we can change the code then.
image_git_sha = getGitSha(work_dir, repo, branch, gitObj)
image_name = "{0}-{1}-{2}/v0.1.0".format(config['name'], app, image_git_sha)
print(colored("******Fetching current version deployed or latest version from registry."
"This is used to bump to next version.******", "grey"))
if skip_build:
curr_image_ver = frameworkObj.getCurrentImageVersion(roger_env, environment, app)
if not curr_image_ver:
raise ValueError("No version for this app {} is deployed on marathon; You have to build the image and "
"deploy it. When skip_build is defined, cli tries to fetch deployed image ".format(app))
self.image_name = curr_image_ver
if args.verbose:
print("Current image version deployed on {0} is :{1}".format(framework, curr_image_ver))
if curr_image_ver is not None:
image_name = "{0}-{1}-{2}".format(config['name'], app, curr_image_ver)
if args.verbose:
print("Image current version from {0} endpoint is:{1}".format(framework, image_name))
else:
if args.verbose:
print("Using base version for image:{0}".format(image_name))
else:
# Docker build,tag and push
image_name = self.getNextVersion(config, roger_env, app, branch, work_dir, repo, args, gitObj)
print(colored("******Done finding latest version******", "green"))
image_name = "{0}-{1}-{2}".format(config['name'], app, image_name)
print(colored("Bumped up image to version:{0}".format(image_name), "green"))
self.image_name = image_name
build_args = args
build_args.app_name = app
build_args.directory = os.path.abspath(work_dir)
build_args.tag_name = image_name
build_args.config_file = config_file
build_args.env = environment
build_args.push = True
build_args.verbose = args.verbose
try:
self.rogerBuildObject.identifier = self.identifier
self.rogerBuildObject.main(settingObj, appObject, hooksObj,
self.dockerUtilsObject, self.dockerObject, build_args)
except ValueError:
raise
print("Image Version is: {}".format(colored(image_name, "cyan")))
# Deploying the app to framework
args.image_name = image_name
args.config_file = config_file
args.env = environment
if app in apps_container_dict:
args.app_name = str(app) + ":" + apps_container_dict[app]
else:
args.app_name = app
self.rogerPushObject.identifier = self.identifier
self.rogerPushObject.main(settingObj, appObj, frameworkUtils,
hooksObj, args)
deployTime = datetime.now() - startTime
username = settingObj.getUser()
deployMessage = "{0}'s deploy for {1} / {2} / {3} completed in {4} seconds.".format(
username, app, environment, branch, deployTime.total_seconds())
if slack is not None:
slack.api_call(deployMessage)
print(colored(deployMessage, "green"))
class RogerDeploy(object):
def __init__(self):
self.rogerGitPullObject = RogerGitPull()
self.rogerPushObject = RogerPush()
self.rogerBuildObject = RogerBuild()
self.dockerUtilsObject = DockerUtils()
self.dockerObject = Docker()
self.utils = Utils()
self.slack = None
self.statsd_message_list = []
self.registry = ""
self.image_name = ""
# To remove a temporary directory created by roger-deploy if this
# script exits
def removeDirTree(self, work_dir, args, temp_dir_created):
exists = os.path.exists(os.path.abspath(work_dir))
if exists and (temp_dir_created is True):
shutil.rmtree(work_dir)
print("Deleted temporary dir:{0}".format(work_dir))
def getNextVersion(self, config, roger_env, application, branch, work_dir, repo, args, gitObj):
sha = getGitSha(work_dir, repo, branch, gitObj)
docker_search = self.dockerUtilsObject.docker_search(roger_env['registry'], config['name'], application)
image_version_list = []
version = ''
envs = []
for each_key in roger_env["environments"].keys():
envs.append(each_key)
for line in docker_search.split('\n'):
image = line.split(' ')[0]
# (vmahedia) todo: This belongs in a separate DockerImage class. This class should know
# how to generate the image name, how to search the image or return this name pattern we
# then use to search this image on the registry.
matchObj = re.match("^{0}-{1}-.*/v.*".format(config['name'], application), image)
if matchObj and matchObj.group().startswith(config['name'] + '-' + application):
skip_image = False
for env in envs:
if matchObj.group().startswith("{0}-{1}-{2}".format(config['name'], application, env)):
skip_image = True
break
if skip_image is False:
if verify(str(matchObj.group().split('v')[-1])):
image_version_list.append(matchObj.group().split('v')[-1])
if len(image_version_list) == 0: # Create initial version
version = "{0}/v0.1.0".format(sha)
print("No version currently exist in the Docker Registry. \nDeploying version:{0}".format(
version))
else:
version = self.incrementVersion(sha, image_version_list, args)
return version
def incrementVersion(self, sha, image_version_list, args):
latest = max(image_version_list, key=self.splitVersion)
ver_tuple = self.splitVersion(latest)
latest_version = ''
if args.incr_major:
latest_version = "{0}/v{1}.0.0".format(sha,
(int(ver_tuple[0]) + 1))
return latest_version
if args.incr_patch:
latest_version = "{0}/v{1}.{2}.{3}".format(
sha, int(ver_tuple[0]), int(ver_tuple[1]), (int(ver_tuple[2]) + 1))
return latest_version
latest_version = "{0}/v{1}.{2}.0".format(
sha, int(ver_tuple[0]), (int(ver_tuple[1]) + 1))
return latest_version
def splitVersion(self, version):
major, _, rest = version.partition('.')
minor, _, rest = rest.partition('.')
patch, _, rest = rest.partition('.')
return int(major), int(minor) if minor else 0, int(patch) if patch else 0
def parseArgs(self):
self.parser = argparse.ArgumentParser(
prog='roger deploy', description=describe())
self.parser.add_argument('-e', '--environment', metavar='env',
help="environment to deploy to. Example: 'dev' or 'stage'")
self.parser.add_argument('-b', '--branch', metavar='branch', default='master',
help="branch to pull code from. Defaults to master. Example: 'production' or 'master'")
self.parser.add_argument('-sg', '--skip-gitpull', action="store_true",
help="skip the gitpull step. Defaults to false.")
self.parser.add_argument('-s', '--skip-build', action="store_true",
help="whether to skip the build step. Defaults to false.")
self.parser.add_argument('-M', '--incr-major', action="store_true",
help="increment major in version. Defaults to false.")
self.parser.add_argument('-sp', '--skip-push', action="store_true",
help="skip the push step. Defaults to false.'")
self.parser.add_argument('-v', '--verbose', help="verbose mode for debugging. Defaults to false.", action="store_true")
self.parser.add_argument('-f', '--force-push', action="store_true",
help="force push. Not recommended. Forces push even if validation checks failed. Applies only if skip_push is false. Defaults to false.")
self.parser.add_argument('-p', '--incr-patch', action="store_true",
help="increment patch in version. Defaults to false.'")
self.parser.add_argument('-S', '--secrets-file',
help="specifies an optional secrets file for deployment runtime variables.")
self.parser.add_argument('-d', '--directory',
help="working directory. Uses a temporary directory if not specified.")
self.parser.add_argument('application', metavar='application', help="application to deploy. \
Can also push specific containers(comma seperated). Example: 'all' \
or 'app1:app2' or 'kairos' or 'app_name[container1,container2]' \
or'app1[container1,container2]:app2[container3,container4]' or 'app1:app2[container]'")
self.parser.add_argument('app_repo', metavar="app_repo",
help="Application's git repository name, repo must be under 'seomoz' organization")
self.parser.add_argument('config_file', metavar='config_file',
help="configuration file to be use. Example: 'content.json' or 'kwe.json'")
return self.parser
def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args):
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS'
settingObj = settingObject
appObj = appObject
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
roger_env = appObj.getRogerEnv(config_dir)
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
if 'registry' not in roger_env:
raise ValueError('Registry not found in roger-mesos-tools.config file.')
else:
self.registry = roger_env['registry']
# Setup for Slack-Client, token, and git user
# (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event
# Event should be registered and SlackNotification should be one of the members. it can have
# N notifications on a particular "event", Notifications.Notify will broadcast notification to
# all the interested parties.
if 'notifications' in config:
self.slack = Slack(config['notifications'],
'/home/vagrant/.roger_cli.conf.d/slack_token')
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application)
apps = []
apps_container_dict = {}
if args.application == 'all':
apps = config['apps'].keys()
else:
if ":" not in args.application and "[" not in args.application:
apps.append(args.application)
else:
for item in args.application.split(":"):
if '[' in item:
matchObj = re.match(r'(.*)\[(.*)\]', item)
apps.append(matchObj.group(1))
apps_container_dict[matchObj.group(1)] = matchObj.group(2)
else:
apps.append(item)
common_repo = config.get('repo', '')
environment = roger_env.get('default_environment', '')
work_dir = ''
if args.directory:
work_dir = args.directory
temp_dir_created = False
if args.verbose:
print("Using {0} as the working directory".format(work_dir))
else:
work_dir = mkdtemp()
temp_dir_created = True
if args.verbose:
print("Created a temporary dir: {0}".format(work_dir))
if args.environment is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file")
else:
print(
"Using value {} from environment variable $ROGER_ENV".format(env_var))
environment = env_var
else:
environment = args.environment
if environment not in roger_env['environments']:
self.removeDirTree(work_dir, args, temp_dir_created)
raise ValueError('Environment not found in roger-mesos-tools.config file.')
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
try:
for app in apps:
if app not in config['apps']:
raise ValueError('Application {} specified not found.'.format(app))
else:
try:
if args.verbose:
print("Deploying {} ...".format(app))
self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj,
root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict)
except (IOError, ValueError) as e:
error_msg = "Error when deploying {}: {}".format(app, repr(e))
printErrorMsg(error_msg)
pass # try deploying the next app
except (Exception) as e:
printException(e)
raise
except (Exception) as e:
execution_result = 'FAILURE'
printException(e)
raise
finally:
# Check if the initializition of variables carried out
if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals():
function_execution_start_time = datetime.now()
if 'execution_result' not in globals() and 'execution_result' not in locals():
execution_result = 'FAILURE'
if 'config_name' not in globals() and 'config_name' not in locals():
config_name = ""
if 'environment' not in globals() and 'environment' not in locals():
environment = "dev"
if not hasattr(args, "application"):
args.application = ""
if 'settingObj' not in globals() and 'settingObj' not in locals():
settingObj = Settings()
if 'work_dir' not in globals() and 'work_dir' not in locals():
work_dir = ''
temp_dir_created = False
if not (self.rogerGitPullObject.outcome is 1 and self.rogerBuildObject.outcome is 1 and self.rogerPushObject.outcome is 1):
execution_result = 'FAILURE'
try:
# If the deploy fails before going through any steps
sc = self.utils.getStatsClient()
if not hasattr(self, "identifier"):
self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application)
args.application = self.utils.extract_app_name(args.application)
time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000)
input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.application) + ",event=deploy" + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******",identifier=" + str(self.identifier)
tup = (input_metric, time_take_milliseonds)
self.statsd_message_list.append(tup)
self.removeDirTree(work_dir, args, temp_dir_created)
except (Exception) as e:
error_msg = "Error when deploying {}: {}".format(app, repr(e))
printErrorMsg(error_msg)
raise
def deployApp(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config,
roger_env, work_dir, config_dir, environment, app, branch, slack, config_file, common_repo, temp_dir_created, apps_container_dict):
startTime = datetime.now()
settingObj = settingObject
appObj = appObject
frameworkUtils = frameworkUtilsObject
environmentObj = roger_env['environments'][environment]
data = appObj.getAppData(config_dir, config_file, app)
frameworkObj = frameworkUtils.getFramework(data)
framework = frameworkObj.getName()
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', app)
image_name = ''
image = ''
skip_gitpull = True if args.skip_gitpull else False
# get/update target source(s)
if not skip_gitpull:
args.app_name = app
args.directory = work_dir
self.rogerGitPullObject.statsd_message_list = self.statsd_message_list
self.rogerGitPullObject.identifier = self.identifier
self.rogerGitPullObject.main(settingObj, appObj, gitObj, hooksObj, args)
skip_build = True if args.skip_build else False
skip_push = True if args.skip_push else False
secrets_file = args.secrets_file if args.secrets_file else None
# Set initial version
# todo (vmahedia) #image_name naming should not be magic, make it explicit
# todo (vmahedia) ExtractClass GitInfo, no need to pass args, we already have the information.
# We deal with only one repo at a time. It may change in future but we can change the code then.
image_git_sha = getGitSha(work_dir, repo, branch, gitObj)
image_name = "{0}-{1}-{2}/v0.1.0".format(config['name'], app, image_git_sha)
print(colored("******Fetching current version deployed or latest version from registry.\
This is used to bump to next version.******", "grey"))
if skip_build:
curr_image_ver = frameworkObj.getCurrentImageVersion(
roger_env, environment, app)
self.image_name = curr_image_ver
if args.verbose:
print("Current image version deployed on {0} is :{1}".format(framework, curr_image_ver))
if curr_image_ver is not None:
image_name = "{0}-{1}-{2}".format(
config['name'], app, curr_image_ver)
if args.verbose:
print("Image current version from {0} endpoint is:{1}".format(framework, image_name))
else:
if args.verbose:
print("Using base version for image:{0}".format(image_name))
else:
# Docker build,tag and push
image_name = self.getNextVersion(
config, roger_env, app, branch, work_dir, repo, args, gitObj)
print(colored("******Done finding latest version******", "green"))
image_name = "{0}-{1}-{2}".format(config['name'], app, image_name)
print(colored("Bumped up image to version:{0}".format(image_name), "green"))
self.image_name = image_name
build_args = args
build_args.app_name = app
build_args.directory = os.path.abspath(work_dir)
build_args.tag_name = image_name
build_args.config_file = config_file
build_args.env = environment
build_args.push = True
build_args.verbose = args.verbose
try:
self.rogerBuildObject.identifier = self.identifier
self.rogerBuildObject.statsd_message_list = self.statsd_message_list
self.rogerBuildObject.main(settingObj, appObject, hooksObj,
self.dockerUtilsObject, self.dockerObject, build_args)
except ValueError:
raise
print("Image Version is: {}".format(colored(image_name, "cyan")))
# Deploying the app to framework
args.image_name = image_name
args.config_file = config_file
args.env = environment
if app in apps_container_dict:
args.app_name = str(app) + ":" + apps_container_dict[app]
else:
args.app_name = app
self.rogerPushObject.identifier = self.identifier
self.rogerPushObject.statsd_message_list = self.statsd_message_list
self.rogerPushObject.main(settingObj, appObj, frameworkUtils,
hooksObj, args)
deployTime = datetime.now() - startTime
username = settingObj.getUser()
deployMessage = "{0}'s deploy for {1} / {2} / {3} completed in {4} seconds.".format(
username, app, environment, branch, deployTime.total_seconds())
if slack is not None:
slack.api_call(deployMessage)
print(colored(deployMessage, "green"))
def locateConfigFile(self, args, gitObj):
# Clone the git repo first because config lives there, there's nothing that we can do without this file
# this is not the clean way but the code is very convulted for now to implement this in a clean manner
# For now, we will clone the repo silently and use that config. We have to clone it everytime because we
# have to assume that there's always a change, although it's not true - we can check if there's a change
# and only pull then, but that is another mess. Let's make it simple and pull everytime
branch = args.branch if args.branch else "master"
repo_dir = os.path.join(args.directory,args.app_repo)
if os.path.exists(args.config_file):
# skip the git clone
# this file path could be inside the cloned repo or outside
if args.config_file.startswith(args.directory):
# chdir is a decorator above but we should move it to utils or somewhere else for every file to use
with chdir(repo_dir):
# file is inside cloned repo so Pull to fetch changes
rc = gitObj.gitPull(branch, args.verbose)
if rc:
print(colored("WARNING: Unable to Pull branch - {}, from repo - {}. Config file will"
"not contain latest changes".format(args.branch, args.app_repo)), "yellow")
# File is not inside cloned repo but somewhere else
# just use it, no need to do anything
else:
if args.app_repo:
if os.path.exists(repo_dir):
raise ValueError("Repo directory - {} already exists but config file - {} is not present"\
.format(repo_dir, args.config_file))
# chdir is a decorator defined above
with chdir(args.directory):
if args.verbose:
print(colored("Cloning repo - {} at - {} for config file - {}".\
format(args.app_repo, repo_dir, args.config_file)))
# clone the repo
# file does not exist and we need to clone the repo it is in repo since repo is
# defined and we mandate it to be in repo
exit_code = gitObj.gitShallowClone(args.app_repo, branch, args.verbose)
if exit_code:
raise ValueError("Error cloning repo {} while looking for config file".format(args.app_repo))
# check now if we have config file in the cloned repo, otherwise bailout.
if not os.path.exists(args.config_file):
# we checked out the repo and either the path is not in repo and file does not exist
# the path is in repo but the repo doesn't have the config file
raise ValueError("Config file - {} does not exist".format(args.config_file))
else:
raise ValueError("Config file - {} does not exist, no app repo defined either".format(args.config_file))
def test_vars_single_container(self):
settings = mock(Settings)
appConfig = mock(AppConfig)
marathon = mock(Marathon)
mockedHooks = mock(Hooks)
roger_push = RogerPush()
roger_push.utils = mock(Utils)
when(mockedHooks).run_hook(any(), any(), any(), any(), any(), any()).thenReturn(0)
roger_env = self.roger_env
config = self.config
data = self.data
frameworkUtils = mock(FrameworkUtils)
when(frameworkUtils).getFramework(data).thenReturn(marathon)
when(marathon).getName().thenReturn('Marathon')
when(settings).getComponentsDir().thenReturn(
self.base_dir + "/tests/components")
when(settings).getSecretsDir().thenReturn(
self.base_dir + "/tests/secrets")
when(settings).getTemplatesDir().thenReturn(
self.base_dir + "/tests/templates")
when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any())
when(roger_push.utils).extract_app_name(any()).thenReturn("test")
when(settings).getConfigDir().thenReturn(self.configs_dir)
when(settings).getCliDir().thenReturn(self.base_dir)
when(settings).getUser().thenReturn(any())
when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env)
when(appConfig).getConfig(self.configs_dir,
"roger_single_container_var_tests.json").thenReturn(config)
when(appConfig).getAppData(self.configs_dir,
"roger_single_container_var_tests.json", "container-vars").thenReturn(data)
args = self.args
args.env = "dev"
args.skip_push = True
args.secrets_dir = ""
args.app_name = 'container-vars'
args.config_file = 'roger_single_container_var_tests.json'
args.directory = self.base_dir + '/tests/testrepo'
args.image_name = 'tests/v0.1.0'
args.secrets_file = 'test'
args.verbose = False
roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args)
with open(self.base_dir + '/tests/components/dev/roger-single-container-var-tests.json') as output:
output = json.load(output)
var1 = output["env"]["VAR_1"]
var3 = output["env"]["VAR_3"]
var4 = output["env"]["VAR_4"]
print ("Expected Value -> Var1 : environment_value_1")
print ("Actual Value : {}".format(var1))
print ("Expected Value -> Var3 : value_3")
print ("Actual Value : {}".format(var3))
assert var3 == "value_3"
assert var1 == "environment_value_1"
class RogerDeploy(object):
def __init__(self):
self.rogerGitPullObject = RogerGitPull()
self.rogerPushObject = RogerPush()
self.rogerBuildObject = RogerBuild()
self.dockerUtilsObject = DockerUtils()
self.dockerObject = Docker()
self.utils = Utils()
self.slack = None
self.registry = ""
self.image_name = ""
# To remove a temporary directory created by roger-deploy if this
# script exits
def removeDirTree(self, work_dir, args, temp_dir_created):
exists = os.path.exists(os.path.abspath(work_dir))
if exists and (temp_dir_created is True):
shutil.rmtree(work_dir)
print("Deleted temporary dir:{0}".format(work_dir))
def getNextVersion(self, config, roger_env, application, branch, work_dir,
repo, args, gitObj):
sha = getGitSha(work_dir, repo, branch, gitObj)
docker_search = self.dockerUtilsObject.docker_search(
roger_env['registry'], config['name'], application)
image_version_list = []
version = ''
envs = []
for each_key in roger_env["environments"].keys():
envs.append(each_key)
for line in docker_search.split('\n'):
image = line.split(' ')[0]
# (vmahedia) todo: This belongs in a separate DockerImage class. This class should know
# how to generate the image name, how to search the image or return this name pattern we
# then use to search this image on the registry.
matchObj = re.match(
"^{0}-{1}-.*/v.*".format(config['name'], application), image)
if matchObj and matchObj.group().startswith(config['name'] + '-' +
application):
skip_image = False
for env in envs:
if matchObj.group().startswith("{0}-{1}-{2}".format(
config['name'], application, env)):
skip_image = True
break
if skip_image is False:
if verify(str(matchObj.group().split('v')[-1])):
image_version_list.append(
matchObj.group().split('v')[-1])
if len(image_version_list) == 0: # Create initial version
version = "{0}/v0.1.0".format(sha)
print(
"No version currently exist in the Docker Registry. \nDeploying version:{0}"
.format(version))
else:
version = self.incrementVersion(sha, image_version_list, args)
return version
def incrementVersion(self, sha, image_version_list, args):
latest = max(image_version_list, key=self.splitVersion)
ver_tuple = self.splitVersion(latest)
latest_version = ''
if args.incr_major:
latest_version = "{0}/v{1}.0.0".format(sha,
(int(ver_tuple[0]) + 1))
return latest_version
if args.incr_patch:
latest_version = "{0}/v{1}.{2}.{3}".format(sha, int(ver_tuple[0]),
int(ver_tuple[1]),
(int(ver_tuple[2]) + 1))
return latest_version
latest_version = "{0}/v{1}.{2}.0".format(sha, int(ver_tuple[0]),
(int(ver_tuple[1]) + 1))
return latest_version
def splitVersion(self, version):
major, _, rest = version.partition('.')
minor, _, rest = rest.partition('.')
patch, _, rest = rest.partition('.')
return int(major), int(minor) if minor else 0, int(
patch) if patch else 0
def parseArgs(self):
self.parser = argparse.ArgumentParser(prog='roger deploy',
description=describe())
self.parser.add_argument(
'-e',
'--environment',
metavar='env',
help="environment to deploy to. Example: 'dev' or 'stage'")
self.parser.add_argument(
'-b',
'--branch',
metavar='branch',
default='master',
help=
"branch to pull code from. Defaults to master. Example: 'production' or 'master'"
)
self.parser.add_argument(
'-sg',
'--skip-gitpull',
action="store_true",
help="skip the gitpull step. Defaults to false.")
self.parser.add_argument(
'-s',
'--skip-build',
action="store_true",
help="whether to skip the build step. Defaults to false.")
self.parser.add_argument(
'-M',
'--incr-major',
action="store_true",
help="increment major in version. Defaults to false.")
self.parser.add_argument(
'-sp',
'--skip-push',
action="store_true",
help="skip the push step. Defaults to false.'")
self.parser.add_argument(
'-v',
'--verbose',
help="verbose mode for debugging. Defaults to false.",
action="store_true")
self.parser.add_argument('-ns',
'--disable-swaparoo',
help="Disables swaparoo functionality",
action="store_true")
self.parser.add_argument(
'-f',
'--force-push',
action="store_true",
help=
"force push. Not recommended. Forces push even if validation checks failed. Applies only if skip_push is false. Defaults to false."
)
self.parser.add_argument(
'-p',
'--incr-patch',
action="store_true",
help="increment patch in version. Defaults to false.'")
self.parser.add_argument(
'-S',
'--secrets-file',
help=
"specifies an optional secrets file for deployment runtime variables."
)
self.parser.add_argument(
'--build-arg',
action='append',
help=
'docker build-arg; Use flags multiple times to pass more than one arg'
)
# (vmahedia) todo Changing this name is slightly more complicated, so making it more verbose for now
self.parser.add_argument(
'-d',
'--directory',
help=
"App Repo will be checked out here, this is the working dir CLI will use."
"A temporary directory is created if no directory specified."
"Example: '/home/vagrant/work_dir'.")
self.parser.add_argument('application',
metavar='application',
help="application to deploy. \
Can also push specific containers(comma seperated). Example: 'all' \
or 'app1:app2' or 'kairos' or 'app_name[container1,container2]' \
or'app1[container1,container2]:app2[container3,container4]' or 'app1:app2[container]'"
)
self.parser.add_argument(
'config_file',
metavar='config_file',
help=
"Path to a Roger configuration file. This can either be an absolute "
"path or a relative path that is taken relative to the ROGER_CONFIG_DIR "
"environment variable.")
return self.parser
def main(self, settingObject, appObject, frameworkUtilsObject, gitObj,
hooksObj, args):
try:
function_execution_start_time = datetime.now()
execution_result = 'SUCCESS'
settingObj = settingObject
appObj = appObject
config_dir = settingObj.getConfigDir()
root = settingObj.getCliDir()
roger_env = appObj.getRogerEnv(config_dir)
config = appObj.getConfig(config_dir, args.config_file)
config_name = ""
if 'name' in config:
config_name = config['name']
if 'registry' not in roger_env:
raise ValueError(
'Registry not found in roger-mesos-tools.config file.')
else:
self.registry = roger_env['registry']
# Setup for Slack-Client, token, and git user
# (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event
# Event should be registered and SlackNotification should be one of the members. it can have
# N notifications on a particular "event", Notifications.Notify will broadcast notification to
# all the interested parties.
if 'notifications' in config:
self.slack = Slack(
config['notifications'],
'/home/vagrant/.roger_cli.conf.d/slack_token')
self.identifier = self.utils.get_identifier(
config_name, settingObj.getUser(), args.application)
apps = []
apps_container_dict = {}
if args.application == 'all':
apps = config['apps'].keys()
else:
if ":" not in args.application and "[" not in args.application:
apps.append(args.application)
else:
for item in args.application.split(":"):
if '[' in item:
matchObj = re.match(r'(.*)\[(.*)\]', item)
apps.append(matchObj.group(1))
apps_container_dict[matchObj.group(
1)] = matchObj.group(2)
else:
apps.append(item)
common_repo = config.get('repo', '')
environment = roger_env.get('default_environment', '')
work_dir = ''
if args.directory:
work_dir = args.directory
temp_dir_created = False
if args.verbose:
print(
"Using {0} as the working directory".format(work_dir))
else:
work_dir = mkdtemp()
temp_dir_created = True
if args.verbose:
print("Created a temporary dir: {0}".format(work_dir))
if args.environment is None:
if "ROGER_ENV" in os.environ:
env_var = os.environ.get('ROGER_ENV')
if env_var.strip() == '':
print(
"Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file"
)
else:
print(
"Using value {} from environment variable $ROGER_ENV"
.format(env_var))
environment = env_var
else:
environment = args.environment
if environment not in roger_env['environments']:
self.removeDirTree(work_dir, args, temp_dir_created)
raise ValueError(
'Environment not found in roger-mesos-tools.config file.')
branch = "master" # master by default
if args.branch is not None:
branch = args.branch
try:
for app in apps:
if app not in config['apps']:
raise ValueError(
'Application {} specified not found.'.format(app))
else:
try:
if args.verbose:
print("Deploying {} ...".format(app))
self.deployApp(settingObject, appObject,
frameworkUtilsObject, gitObj,
hooksObj, root, args, config,
roger_env, work_dir, config_dir,
environment, app, branch,
self.slack, args.config_file,
common_repo, temp_dir_created,
apps_container_dict)
except (IOError, ValueError) as e:
error_msg = "Error when deploying {}: {}".format(
app, repr(e))
printErrorMsg(error_msg)
pass # try deploying the next app
except (Exception) as e:
printException(e)
raise
except (Exception) as e:
execution_result = 'FAILURE'
printException(e)
raise
finally:
# todo: maybe send the datadog event, need to look
pass
def deployApp(self, settingObject, appObject, frameworkUtilsObject, gitObj,
hooksObj, root, args, config, roger_env, work_dir,
config_dir, environment, app, branch, slack, config_file,
common_repo, temp_dir_created, apps_container_dict):
startTime = datetime.now()
settingObj = settingObject
appObj = appObject
frameworkUtils = frameworkUtilsObject
environmentObj = roger_env['environments'][environment]
data = appObj.getAppData(config_dir, config_file, app)
frameworkObj = frameworkUtils.getFramework(data)
framework = frameworkObj.getName()
repo = ''
if common_repo != '':
repo = data.get('repo', common_repo)
else:
repo = data.get('repo', app)
image_name = ''
image = ''
skip_gitpull = True if args.skip_gitpull else False
# get/update target source(s)
if not skip_gitpull:
args.app_name = app
args.directory = work_dir
self.rogerGitPullObject.identifier = self.identifier
self.rogerGitPullObject.main(settingObj, appObj, gitObj, hooksObj,
args)
skip_build = True if args.skip_build else False
skip_push = True if args.skip_push else False
secrets_file = args.secrets_file if args.secrets_file else None
# Set initial version
# todo (vmahedia) #image_name naming should not be magic, make it explicit
# todo (vmahedia) ExtractClass GitInfo, no need to pass args, we already have the information.
# We deal with only one repo at a time. It may change in future but we can change the code then.
image_git_sha = getGitSha(work_dir, repo, branch, gitObj)
image_name = "{0}-{1}-{2}/v0.1.0".format(config['name'], app,
image_git_sha)
print(
colored(
"******Fetching current version deployed or latest version from registry."
"This is used to bump to next version.******", "grey"))
if skip_build:
curr_image_ver = frameworkObj.getCurrentImageVersion(
roger_env, environment, app)
if not curr_image_ver:
raise ValueError(
"No version for this app {} is deployed on marathon; You have to build the image and "
"deploy it. When skip_build is defined, cli tries to fetch deployed image "
.format(app))
self.image_name = curr_image_ver
if args.verbose:
print("Current image version deployed on {0} is :{1}".format(
framework, curr_image_ver))
if curr_image_ver is not None:
image_name = "{0}-{1}-{2}".format(config['name'], app,
curr_image_ver)
if args.verbose:
print("Image current version from {0} endpoint is:{1}".
format(framework, image_name))
else:
if args.verbose:
print(
"Using base version for image:{0}".format(image_name))
else:
# Docker build,tag and push
image_name = self.getNextVersion(config, roger_env, app, branch,
work_dir, repo, args, gitObj)
print(colored("******Done finding latest version******", "green"))
image_name = "{0}-{1}-{2}".format(config['name'], app, image_name)
print(
colored("Bumped up image to version:{0}".format(image_name),
"green"))
self.image_name = image_name
build_args = args
build_args.app_name = app
build_args.directory = os.path.abspath(work_dir)
build_args.tag_name = image_name
build_args.config_file = config_file
build_args.env = environment
build_args.push = True
build_args.verbose = args.verbose
try:
self.rogerBuildObject.identifier = self.identifier
self.rogerBuildObject.main(settingObj, appObject, hooksObj,
self.dockerUtilsObject,
self.dockerObject, build_args)
except ValueError:
raise
print("Image Version is: {}".format(colored(image_name, "cyan")))
# Deploying the app to framework
args.image_name = image_name
args.config_file = config_file
args.env = environment
if app in apps_container_dict:
args.app_name = str(app) + ":" + apps_container_dict[app]
else:
args.app_name = app
self.rogerPushObject.identifier = self.identifier
self.rogerPushObject.main(settingObj, appObj, frameworkUtils, hooksObj,
args)
deployTime = datetime.now() - startTime
username = settingObj.getUser()
deployMessage = "{0}'s deploy for {1} / {2} / {3} completed in {4} seconds.".format(
username, app, environment, branch, deployTime.total_seconds())
if slack is not None:
slack.api_call(deployMessage)
print(colored(deployMessage, "green"))
def test_vars_single_container(self):
settings = mock(Settings)
appConfig = mock(AppConfig)
marathon = mock(Marathon)
mockedHooks = mock(Hooks)
roger_push = RogerPush()
roger_push.utils = mock(Utils)
when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0)
roger_env = self.roger_env
config = self.config
data = self.data
sc = mock(StatsClient)
frameworkUtils = mock(FrameworkUtils)
when(frameworkUtils).getFramework(data).thenReturn(marathon)
when(marathon).getName().thenReturn('Marathon')
when(settings).getComponentsDir().thenReturn(self.base_dir +
"/tests/components")
when(settings).getSecretsDir().thenReturn(self.base_dir +
"/tests/secrets")
when(settings).getTemplatesDir().thenReturn(self.base_dir +
"/tests/templates")
when(sc).timing(any(), any()).thenReturn(any())
when(roger_push.utils).getStatsClient().thenReturn(sc)
when(roger_push.utils).get_identifier(any(), any(),
any()).thenReturn(any())
when(roger_push.utils).extract_app_name(any()).thenReturn("test")
when(settings).getConfigDir().thenReturn(self.configs_dir)
when(settings).getCliDir().thenReturn(self.base_dir)
when(settings).getUser().thenReturn(any())
when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env)
when(appConfig).getConfig(
self.configs_dir,
"roger_single_container_var_tests.json").thenReturn(config)
when(appConfig).getAppData(self.configs_dir,
"roger_single_container_var_tests.json",
"container-vars").thenReturn(data)
args = self.args
args.env = "dev"
args.skip_push = True
args.secrets_dir = ""
args.app_name = 'container-vars'
args.config_file = 'roger_single_container_var_tests.json'
args.directory = self.base_dir + '/tests/testrepo'
args.image_name = 'tests/v0.1.0'
args.secrets_file = 'test'
roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args)
with open(self.base_dir +
'/tests/components/dev/roger-single-container-var-tests.json'
) as output:
output = json.load(output)
var1 = output["env"]["VAR_1"]
var3 = output["env"]["VAR_3"]
var4 = output["env"]["VAR_4"]
print("Expected Value -> Var1 : environment_value_1")
print("Actual Value : {}".format(var1))
print("Expected Value -> Var3 : value_3")
print("Actual Value : {}".format(var3))
assert var3 == "value_3"
assert var1 == "environment_value_1"