def test_template_render_for_container_level_variables(self): roger_push = RogerPush() app_data = self.config['apps']['grafana_test_app'] container = "grafana" # Passing environment that doesn't exist output = roger_push.renderTemplate(self.template, "non_existing_env", "test_image", app_data, self.config, container, "grafana", self.additional_vars) result = json.loads(output) assert result['env']['ENV_VAR1'] == '3' assert result['env']['ENV_VAR2'] == '3' # Existing environment output = roger_push.renderTemplate( self.template, "test", "test_image", app_data, self.config, container, "grafana", self.additional_vars) result = json.loads(output) assert result['env']['ENV_VAR1'] == '4' assert result['env']['ENV_VAR2'] == '8' container = app_data['containers'][1]['grafana1'] output = roger_push.renderTemplate(self.template, "non_existing_env", "test_image", app_data, self.config, container, "grafana1", self.additional_vars) result = json.loads(output) assert result['env']['ENV_VAR1'] == '30' assert result['env']['ENV_VAR2'] == '54' output = roger_push.renderTemplate( self.template, "test", "test_image", app_data, self.config, container, "grafana1", self.additional_vars) result = json.loads(output) assert result['env']['ENV_VAR1'] == '64' assert result['env']['ENV_VAR2'] == '128'
def test_roger_push_with_incorrect_container_name(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() marathon = mock(Marathon) mockedHooks = mock(Hooks) roger_env = self.roger_env config = self.config appdata = self.data when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") frameworkUtils = mock(FrameworkUtils) when(marathon).getName().thenReturn('Marathon') when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getUser().thenReturn(any()) when(mockedHooks).run_hook(any(), any(), any(), any(), any(), any()).thenReturn(0) when(appConfig).getAppData(any(), any(), any()).thenReturn(appdata) when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) args = self.args args.env = "dev" args.secrets_file = "" args.skip_push = True args.app_name = 'grafana_test_app:test' args.config_file = 'test.json' with self.assertRaises(ValueError): roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args)
def test_roger_push_env_from_ROGER_ENV_VAR(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() marathon = mock(Marathon) mockedHooks = mock(Hooks) roger_env = self.roger_env config = self.config frameworkUtils = mock(FrameworkUtils) when(settings).getConfigDir().thenReturn(self.configs_dir) when(mockedHooks).run_hook(any(), any(), any()).thenReturn(0) when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) args = self.args args.env = None args.secrets_file = "" args.skip_push = True args.app_name = 'grafana_test_app' args.config_file = 'test.json' roger_env = appConfig.getRogerEnv(self.configs_dir) roger_env["default_environment"] = None # Setting ROGER_ENV to specific value os.environ["ROGER_ENV"] = "test_env" with self.assertRaises(ValueError): roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args)
def test_roger_push_with_incorrect_container_name(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() marathon = mock(Marathon) mockedHooks = mock(Hooks) roger_env = self.roger_env config = self.config appdata = self.data sc = mock(StatsClient) when(sc).timing(any(), any()).thenReturn(any()) when(roger_push.utils).getStatsClient().thenReturn(sc) when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") frameworkUtils = mock(FrameworkUtils) when(marathon).getName().thenReturn('Marathon') when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getUser().thenReturn(any()) when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0) when(appConfig).getAppData(any(), any(), any()).thenReturn(appdata) when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) args = self.args args.env = "dev" args.secrets_file = "" args.skip_push = True args.app_name = 'grafana_test_app:test' args.config_file = 'test.json' with self.assertRaises(ValueError): roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args)
def test_secret_vars_replacement_with_secrets_file(self): args = self.args args.env = "test" args.secrets_file = "test_app-container1.json" args.app_name = 'test_app' args.image_name = 'test_image' args.verbose = False secrets_dir = self.base_dir + "/tests/secrets" roger_push = RogerPush() app_data = self.config['apps'][args.app_name] container = "container1" additional_vars = self.additional_vars extra_vars = {} extra_vars['env_value1'] = "100" extra_vars['env_value2'] = "200" additional_vars.update(extra_vars) secret_vars = roger_push.loadSecrets( secrets_dir, args.secrets_file, args, args.env) print(secret_vars) additional_vars.update(secret_vars) output = roger_push.renderTemplate( self.template, args.env, args.image_name, app_data, self.config, container, "container1", additional_vars) result = json.loads(output) assert result['env']['ENV_VAR1'] == 'test_value1' assert result['env']['ENV_VAR2'] == 'test_value2'
def test_unresolved_jinja_variable_fails(self): with open(self.configs_dir + '/roger_push_unresolved_jinja.json') as config: config = json.load(config) with open(self.configs_dir + '/roger-mesos-tools.config') as roger: roger_env = yaml.load(roger) data = config['apps']['container-vars'] self.config = config self.roger_env = roger_env self.data = data settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() roger_push.utils = mock(Utils) marathon = mock(Marathon) mockedHooks = mock(Hooks) when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0) roger_env = self.roger_env config = self.config data = self.data sc = mock(StatsClient) when(sc).timing(any(), any()).thenReturn(any()) when(roger_push.utils).getStatsClient().thenReturn(sc) when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(roger_push.utils).get_version().thenReturn(any()) frameworkUtils = mock(FrameworkUtils) when(frameworkUtils).getFramework(data).thenReturn(marathon) when(marathon).getName().thenReturn('Marathon') when(settings).getComponentsDir().thenReturn( self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn( self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn( self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env) when(appConfig).getConfig(self.configs_dir, "roger_push_unresolved_jinja.json").thenReturn(config) when(appConfig).getAppData(self.configs_dir, "roger_push_unresolved_jinja.json", "container-vars").thenReturn(data) args = self.args args.env = "dev" args.skip_push = False args.force_push = False args.secrets_dir = "" args.secrets_file = "" args.app_name = 'container-vars' args.config_file = 'roger_push_unresolved_jinja.json' args.directory = self.base_dir + '/tests/testrepo' args.image_name = 'tests/v0.1.0' args.verbose = False roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args) verify(frameworkUtils, times=0).put(any(), any(), any(), any())
def test_secret_vars_replacement_with_secrets_file(self): args = self.args args.env = "test" args.secrets_file = "test_app-container1.json" args.app_name = 'test_app' args.image_name = 'test_image' secrets_dir = self.base_dir + "/tests/secrets" roger_push = RogerPush() app_data = self.config['apps'][args.app_name] container = "container1" additional_vars = self.additional_vars extra_vars = {} extra_vars['env_value1'] = "100" extra_vars['env_value2'] = "200" additional_vars.update(extra_vars) secret_vars = roger_push.loadSecrets(secrets_dir, args.secrets_file, args, args.env) print(secret_vars) additional_vars.update(secret_vars) output = roger_push.renderTemplate(self.template, args.env, args.image_name, app_data, self.config, container, "container1", additional_vars) result = json.loads(output) assert result['env']['ENV_VAR1'] == 'test_value1' assert result['env']['ENV_VAR2'] == 'test_value2'
def test_template_render_for_config_level_variables(self): roger_push = RogerPush() app_data = self.config['apps']['test_app'] container = "container_name1" output = roger_push.renderTemplate( self.template, "test", "test_image", app_data, self.config, container, "container_name1", self.additional_vars) result = json.loads(output) assert result['env']['ENV_VAR1'] == '4' assert result['env']['ENV_VAR2'] == '8'
def test_unresolved_jinja_variable_fails(self): with open(self.configs_dir + '/roger_push_unresolved_jinja.json') as config: config = json.load(config) with open(self.configs_dir + '/roger-mesos-tools.config') as roger: roger_env = yaml.load(roger) data = config['apps']['container-vars'] self.config = config self.roger_env = roger_env self.data = data settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() roger_push.utils = mock(Utils) marathon = mock(Marathon) mockedHooks = mock(Hooks) when(mockedHooks).run_hook(any(), any(), any(), any(), any(), any()).thenReturn(0) roger_env = self.roger_env config = self.config data = self.data when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(roger_push.utils).get_version().thenReturn(any()) frameworkUtils = mock(FrameworkUtils) when(frameworkUtils).getFramework(data).thenReturn(marathon) when(marathon).getName().thenReturn('Marathon') when(settings).getComponentsDir().thenReturn( self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn( self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn( self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env) when(appConfig).getConfig(self.configs_dir, "roger_push_unresolved_jinja.json").thenReturn(config) when(appConfig).getAppData(self.configs_dir, "roger_push_unresolved_jinja.json", "container-vars").thenReturn(data) args = self.args args.env = "dev" args.skip_push = False args.force_push = False args.secrets_dir = "" args.secrets_file = "" args.app_name = 'container-vars' args.config_file = 'roger_push_unresolved_jinja.json' args.directory = self.base_dir + '/tests/testrepo' args.image_name = 'tests/v0.1.0' args.verbose = False roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args) verify(frameworkUtils, times=0).put(any(), any(), any(), any())
def test_push_fails_with_validation_error(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() roger_push.utils = mock(Utils) marathon = mock(Marathon) mockedHooks = mock(Hooks) roger_env = self.roger_env config = self.config data = self.data sc = mock(StatsClient) when(sc).timing(any(), any()).thenReturn(any()) when(roger_push.utils).getStatsClient().thenReturn(sc) when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(roger_push.utils).modify_task_id(any()).thenReturn([any()]) when(roger_push.utils).get_version().thenReturn(any()) frameworkUtils = mock(FrameworkUtils) when(frameworkUtils).getFramework(data).thenReturn(marathon) when(marathon).getName().thenReturn('Marathon') when(marathon).put(any(), any(), any(), any(), any()).thenReturn(["Response [200]", any()]) when(marathon).runDeploymentChecks(any(), any()).thenReturn(True) when(settings).getComponentsDir().thenReturn( self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn( self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn( self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0) when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) when(appConfig).getAppData(any(), any(), any()).thenReturn(data) args = self.args args.env = "dev" args.secrets_file = "" args.skip_push = False args.force_push = False args.app_name = 'grafana_test_app' args.config_file = 'test.json' args.directory = self.base_dir + '/tests/testrepo' args.image_name = 'grafana/grafana:2.1.3' args.verbose = False raised_exception = False try: return_code = roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args) except (Exception) as e: raised_exception = True self.assertFalse(raised_exception) verify(frameworkUtils, times=0).put(any(), any(), any(), any())
def __init__(self): self.rogerGitPullObject = RogerGitPull() self.rogerPushObject = RogerPush() self.rogerBuildObject = RogerBuild() self.dockerUtilsObject = DockerUtils() self.dockerObject = Docker() self.utils = Utils() self.slack = None self.registry = "" self.image_name = ""
def test_roger_push_grafana_test_app(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() roger_push.utils = mock(Utils) marathon = mock(Marathon) mockedHooks = mock(Hooks) when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0) roger_env = self.roger_env config = self.config data = self.data sc = mock(StatsClient) when(sc).timing(any(), any()).thenReturn(any()) when(roger_push.utils).getStatsClient().thenReturn(sc) when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(marathon).put(any(), any(), any()).thenReturn("Response [200]", any()) frameworkUtils = mock(FrameworkUtils) when(frameworkUtils).getFramework(data).thenReturn(marathon) when(marathon).getName().thenReturn('Marathon') when(settings).getComponentsDir().thenReturn(self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn(self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn(self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(appConfig).getRogerEnv(any()).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) when(appConfig).getAppData(any(), any(), any()).thenReturn(data) args = self.args args.env = "dev" args.secrets_file = "" args.skip_push = True args.app_name = 'grafana_test_app' args.config_file = 'test.json' args.directory = self.base_dir + '/tests/testrepo' args.image_name = 'grafana/grafana:2.1.3' roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args) with open(self.base_dir + '/tests/templates/test-app-grafana.json') as output: output = json.load(output) assert output['container']['docker'][ 'image'] == "grafana/grafana:2.1.3" verify(settings).getConfigDir() verify(settings).getComponentsDir() verify(settings).getTemplatesDir() verify(settings).getSecretsDir() verify(frameworkUtils).getFramework(data)
def test_template_render_for_extra_variables(self): roger_push = RogerPush() app_data = self.config['apps']['grafana_test_app'] container = app_data['containers'][1]['grafana1'] additional_vars = self.additional_vars additional_vars['env_value1'] = "100" additional_vars['env_value2'] = "200" output = roger_push.renderTemplate( self.template, "test", "test_image", app_data, self.config, container, "grafana1", additional_vars) result = json.loads(output) assert result['env']['ENV_VAR1'] == '100' assert result['env']['ENV_VAR2'] == '200'
def test_roger_push_with_correct_container_name(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() roger_push.utils = mock(Utils) marathon = mock(Marathon) mockedHooks = mock(Hooks) frameworkUtils = mock(FrameworkUtils) roger_env = self.roger_env appdata = self.data config = self.config sc = mock(StatsClient) when(sc).timing(any(), any()).thenReturn(any()) when(roger_push.utils).getStatsClient().thenReturn(sc) when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(frameworkUtils).getFramework(any()).thenReturn(marathon) when(marathon).getName().thenReturn('Marathon') when(settings).getComponentsDir().thenReturn( self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn( self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn( self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env) when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0) when(appConfig).getRogerEnv(any()).thenReturn(roger_env) appdata["hooks"] = dict([("post_push", "some command")]) when(appConfig).getAppData(any(), any(), any()).thenReturn(appdata) when(appConfig).getConfig(any(), any()).thenReturn(config) when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) args = self.args args.env = "dev" args.secrets_file = "" args.skip_push = True args.app_name = 'grafana_test_app:grafana' args.config_file = 'test.json' args.image_name = 'grafana/grafana:2.1.3' args.verbose = False raised_exception = False try: roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args) except (Exception) as e: raised_exception = True self.assertFalse(raised_exception) verify(frameworkUtils).getFramework(any())
def test_push_fails_with_validation_error(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() roger_push.utils = mock(Utils) marathon = mock(Marathon) mockedHooks = mock(Hooks) roger_env = self.roger_env config = self.config data = self.data sc = mock(StatsClient) when(sc).timing(any(), any()).thenReturn(any()) when(roger_push.utils).getStatsClient().thenReturn(sc) when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(roger_push.utils).modify_task_id(any()).thenReturn([any()]) when(roger_push.utils).get_version().thenReturn(any()) frameworkUtils = mock(FrameworkUtils) when(frameworkUtils).getFramework(data).thenReturn(marathon) when(marathon).getName().thenReturn('Marathon') when(marathon).put(any(), any(), any(), any(), any()).thenReturn(["Response [200]", any()]) when(marathon).runDeploymentChecks(any(), any()).thenReturn(True) when(settings).getComponentsDir().thenReturn(self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn(self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn(self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0) when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) when(appConfig).getAppData(any(), any(), any()).thenReturn(data) args = self.args args.env = "dev" args.secrets_file = "" args.skip_push = False args.force_push = False args.app_name = 'grafana_test_app' args.config_file = 'test.json' args.directory = self.base_dir + '/tests/testrepo' args.image_name = 'grafana/grafana:2.1.3' return_code = roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args) verify(frameworkUtils, times=0).put(any(), any(), any(), any())
def test_roger_push_grafana_test_app(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() roger_push.utils = mock(Utils) marathon = mock(Marathon) mockedHooks = mock(Hooks) when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0) roger_env = self.roger_env config = self.config data = self.data sc = mock(StatsClient) when(sc).timing(any(), any()).thenReturn(any()) when(roger_push.utils).getStatsClient().thenReturn(sc) when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(marathon).put(any(), any(), any()).thenReturn("Response [200]", any()) frameworkUtils = mock(FrameworkUtils) when(frameworkUtils).getFramework(data).thenReturn(marathon) when(marathon).getName().thenReturn('Marathon') when(settings).getComponentsDir().thenReturn( self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn( self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn( self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(appConfig).getRogerEnv(any()).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) when(appConfig).getAppData(any(), any(), any()).thenReturn(data) args = self.args args.env = "dev" args.secrets_file = "" args.skip_push = True args.app_name = 'grafana_test_app' args.config_file = 'test.json' args.directory = self.base_dir + '/tests/testrepo' args.image_name = 'grafana/grafana:2.1.3' args.verbose = False roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args) with open(self.base_dir + '/tests/templates/test-app-grafana.json') as output: output = json.load(output) assert output['container']['docker'][ 'image'] == "grafana/grafana:2.1.3" verify(settings).getConfigDir() verify(settings).getComponentsDir() verify(settings).getTemplatesDir() verify(settings).getSecretsDir() verify(frameworkUtils).getFramework(data)
def test_roger_push_with_correct_container_name(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() roger_push.utils = mock(Utils) marathon = mock(Marathon) mockedHooks = mock(Hooks) frameworkUtils = mock(FrameworkUtils) roger_env = self.roger_env appdata = self.data config = self.config when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(frameworkUtils).getFramework(any()).thenReturn(marathon) when(marathon).getName().thenReturn('Marathon') when(settings).getComponentsDir().thenReturn( self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn( self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn( self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env) when(mockedHooks).run_hook(any(), any(), any(), any(), any(), any()).thenReturn(0) when(appConfig).getRogerEnv(any()).thenReturn(roger_env) appdata["hooks"] = dict([("post_push", "some command")]) when(appConfig).getAppData(any(), any(), any()).thenReturn(appdata) when(appConfig).getConfig(any(), any()).thenReturn(config) when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) args = self.args args.env = "dev" args.secrets_file = "" args.skip_push = True args.app_name = 'grafana_test_app:grafana' args.config_file = 'test.json' args.image_name = 'grafana/grafana:2.1.3' args.verbose = False raised_exception = False try: roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args) except (Exception) as e: raised_exception = True self.assertFalse(raised_exception) verify(frameworkUtils).getFramework(any())
def test_roger_push_secrets_replaced(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() roger_push.utils = mock(Utils) marathon = mock(Marathon) mockedHooks = mock(Hooks) roger_env = self.roger_env config = self.config data = self.data sc = mock(StatsClient) when(sc).timing(any(), any()).thenReturn(any()) when(roger_push.utils).getStatsClient().thenReturn(sc) when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(marathon).put(any(), any(), any()).thenReturn("Response [200]") frameworkUtils = mock(FrameworkUtils) when(frameworkUtils).getFramework(data).thenReturn(marathon) when(marathon).getName().thenReturn('Marathon') when(settings).getComponentsDir().thenReturn( self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn( self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn( self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(appConfig).getRogerEnv(any()).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) when(appConfig).getAppData(any(), any(), any()).thenReturn(data) when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0) args = self.args args.env = "dev" args.secrets_file = "" args.skip_push = True args.app_name = 'grafana_test_app' args.config_file = 'test.json' args.directory = self.base_dir + '/tests/testrepo' args.image_name = 'grafana/grafana:2.1.3' args.verbose = False exit_code = roger_push.main( settings, appConfig, frameworkUtils, mockedHooks, args) file_path = ("{0}/{1}/{2}".format(self.components_dir, args.env, args.config_file)) assert (os.path.isfile(file_path) is not True)
def test_roger_push_secrets_replaced(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() roger_push.utils = mock(Utils) marathon = mock(Marathon) mockedHooks = mock(Hooks) roger_env = self.roger_env config = self.config data = self.data sc = mock(StatsClient) when(sc).timing(any(), any()).thenReturn(any()) when(roger_push.utils).getStatsClient().thenReturn(sc) when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(marathon).put(any(), any(), any()).thenReturn("Response [200]") frameworkUtils = mock(FrameworkUtils) when(frameworkUtils).getFramework(data).thenReturn(marathon) when(marathon).getName().thenReturn('Marathon') when(settings).getComponentsDir().thenReturn(self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn(self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn(self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(appConfig).getRogerEnv(any()).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) when(appConfig).getAppData(any(), any(), any()).thenReturn(data) when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0) args = self.args args.env = "dev" args.secrets_file = "" args.skip_push = True args.app_name = 'grafana_test_app' args.config_file = 'test.json' args.directory = self.base_dir + '/tests/testrepo' args.image_name = 'grafana/grafana:2.1.3' exit_code = roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args) file_path = ("{0}/{1}/{2}".format(self.components_dir, args.env, args.config_file)) assert (os.path.isfile(file_path) is not True)
def test_roger_push_calls_postpush_hook_when_present(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() roger_push.utils = mock(Utils) marathon = mock(Marathon) mockedHooks = mock(Hooks) frameworkUtils = mock(FrameworkUtils) roger_env = self.roger_env appdata = self.data config = self.config sc = mock(StatsClient) when(sc).timing(any(), any()).thenReturn(any()) when(roger_push.utils).getStatsClient().thenReturn(sc) when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(frameworkUtils).getFramework(any()).thenReturn(marathon) when(marathon).getName().thenReturn('Marathon') when(settings).getComponentsDir().thenReturn(self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn(self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn(self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env) when(mockedHooks).run_hook(any(), any(), any(), any()).thenReturn(0) when(appConfig).getRogerEnv(any()).thenReturn(roger_env) appdata["hooks"] = dict([("post_push", "some command")]) when(appConfig).getAppData(any(), any(), any()).thenReturn(appdata) when(appConfig).getConfig(any(), any()).thenReturn(config) args = self.args args.config_file = 'any.json' args.app_name = 'any app' args.env = 'dev' args.image_name = 'tests/v0.1.0' args.directory = '/tmp' args.secrets_file = "" args.skip_push = True return_code = roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args) verify(mockedHooks).run_hook("post_push", any(), any(), any())
def test_template_render_for_app_level_variables(self): roger_push = RogerPush() app_data = self.config['apps']['test_app1'] container = "container_name1" # Passing environment that doesn't exist output = roger_push.renderTemplate(self.template, "non_existing_env", "test_image", app_data, self.config, container, "container_name1", self.additional_vars) result = json.loads(output) assert result['env']['ENV_VAR1'] == '12' assert result['env']['ENV_VAR2'] == '16' # Existing environment output = roger_push.renderTemplate( self.template, "test", "test_image", app_data, self.config, container, "container_name1", self.additional_vars) result = json.loads(output) assert result['env']['ENV_VAR1'] == '20' assert result['env']['ENV_VAR2'] == '24'
def test_roger_push_with_no_registry_fails(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() marathon = mock(Marathon) mockedHooks = mock(Hooks) when(mockedHooks).run_hook(any(), any(), any()).thenReturn(0) roger_env = self.roger_env config = self.config data = self.data when(marathon).put(any(), any(), any()).thenReturn("Response [200]") frameworkUtils = mock(FrameworkUtils) when(frameworkUtils).getFramework(data).thenReturn(marathon) when(settings).getComponentsDir().thenReturn(self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn(self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn(self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(appConfig).getRogerEnv(any()).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) when(appConfig).getAppData(any(), any(), any()).thenReturn(data) args = self.args args.env = "dev" args.secrets_file = "" args.skip_push = True args.app_name = 'grafana_test_app' args.config_file = 'test.json' args.directory = self.base_dir + '/tests/testrepo' args.image_name = 'grafana/grafana:2.1.3' config_dir = settings.getConfigDir() roger_env = appConfig.getRogerEnv(config_dir) # Remove registry key from dictionary del roger_env['registry'] with self.assertRaises(ValueError): roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args)
def test_roger_push_with_no_app_fails(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() roger_push.utils = mock(Utils) marathon = mock(Marathon) mockedHooks = mock(Hooks) when(mockedHooks).run_hook(any(), any(), any()).thenReturn(0) roger_env = self.roger_env config = self.config data = self.data sc = mock(StatsClient) when(sc).timing(any(), any()).thenReturn(any()) when(roger_push.utils).getStatsClient().thenReturn(sc) when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(marathon).put(any(), any(), any()).thenReturn("Response [200]") frameworkUtils = mock(FrameworkUtils) when(frameworkUtils).getFramework(data).thenReturn(marathon) when(settings).getComponentsDir().thenReturn(self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn(self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn(self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(appConfig).getRogerEnv(any()).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) when(appConfig).getAppData(any(), any(), any()).thenReturn(data) args = self.args args.app_name = '' args.config_file = 'test.json' args.env = 'some_test_env' with self.assertRaises(ValueError): roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args)
def test_roger_push_with_no_registry_fails(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() marathon = mock(Marathon) mockedHooks = mock(Hooks) when(mockedHooks).run_hook(any(), any(), any()).thenReturn(0) roger_env = self.roger_env config = self.config data = self.data when(marathon).put(any(), any(), any()).thenReturn("Response [200]") frameworkUtils = mock(FrameworkUtils) when(frameworkUtils).getFramework(data).thenReturn(marathon) when(settings).getComponentsDir().thenReturn( self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn( self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn( self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(appConfig).getRogerEnv(any()).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) when(appConfig).getAppData(any(), any(), any()).thenReturn(data) args = self.args args.env = "dev" args.secrets_file = "" args.skip_push = True args.app_name = 'grafana_test_app' args.config_file = 'test.json' args.directory = self.base_dir + '/tests/testrepo' args.image_name = 'grafana/grafana:2.1.3' config_dir = settings.getConfigDir() roger_env = appConfig.getRogerEnv(config_dir) # Remove registry key from dictionary del roger_env['registry'] with self.assertRaises(ValueError): roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args)
def test_roger_push_with_no_app_fails(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() roger_push.utils = mock(Utils) marathon = mock(Marathon) mockedHooks = mock(Hooks) when(mockedHooks).run_hook(any(), any(), any()).thenReturn(0) roger_env = self.roger_env config = self.config data = self.data sc = mock(StatsClient) when(sc).timing(any(), any()).thenReturn(any()) when(roger_push.utils).getStatsClient().thenReturn(sc) when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(marathon).put(any(), any(), any()).thenReturn("Response [200]") frameworkUtils = mock(FrameworkUtils) when(frameworkUtils).getFramework(data).thenReturn(marathon) when(settings).getComponentsDir().thenReturn( self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn( self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn( self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(appConfig).getRogerEnv(any()).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) when(appConfig).getAppData(any(), any(), any()).thenReturn(data) args = self.args args.app_name = '' args.config_file = 'test.json' args.env = 'some_test_env' with self.assertRaises(ValueError): roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args)
def test_template_render_for_container_level_variables(self): roger_push = RogerPush() app_data = self.config['apps']['grafana_test_app'] container = "grafana" # Passing environment that doesn't exist output = roger_push.renderTemplate(self.template, "non_existing_env", "test_image", app_data, self.config, container, "grafana", self.additional_vars) result = json.loads(output) assert result['env']['ENV_VAR1'] == '3' assert result['env']['ENV_VAR2'] == '3' # Existing environment output = roger_push.renderTemplate(self.template, "test", "test_image", app_data, self.config, container, "grafana", self.additional_vars) result = json.loads(output) assert result['env']['ENV_VAR1'] == '4' assert result['env']['ENV_VAR2'] == '8' container = app_data['containers'][1]['grafana1'] output = roger_push.renderTemplate(self.template, "non_existing_env", "test_image", app_data, self.config, container, "grafana1", self.additional_vars) result = json.loads(output) assert result['env']['ENV_VAR1'] == '30' assert result['env']['ENV_VAR2'] == '54' output = roger_push.renderTemplate(self.template, "test", "test_image", app_data, self.config, container, "grafana1", self.additional_vars) result = json.loads(output) assert result['env']['ENV_VAR1'] == '64' assert result['env']['ENV_VAR2'] == '128'
def test_vars_single_container(self): settings = mock(Settings) appConfig = mock(AppConfig) marathon = mock(Marathon) mockedHooks = mock(Hooks) roger_push = RogerPush() roger_push.utils = mock(Utils) when(mockedHooks).run_hook(any(), any(), any(), any(), any(), any()).thenReturn(0) roger_env = self.roger_env config = self.config data = self.data frameworkUtils = mock(FrameworkUtils) when(frameworkUtils).getFramework(data).thenReturn(marathon) when(marathon).getName().thenReturn('Marathon') when(settings).getComponentsDir().thenReturn( self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn( self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn( self.base_dir + "/tests/templates") when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env) when(appConfig).getConfig(self.configs_dir, "roger_single_container_var_tests.json").thenReturn(config) when(appConfig).getAppData(self.configs_dir, "roger_single_container_var_tests.json", "container-vars").thenReturn(data) args = self.args args.env = "dev" args.skip_push = True args.secrets_dir = "" args.app_name = 'container-vars' args.config_file = 'roger_single_container_var_tests.json' args.directory = self.base_dir + '/tests/testrepo' args.image_name = 'tests/v0.1.0' args.secrets_file = 'test' args.verbose = False roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args) with open(self.base_dir + '/tests/components/dev/roger-single-container-var-tests.json') as output: output = json.load(output) var1 = output["env"]["VAR_1"] var3 = output["env"]["VAR_3"] var4 = output["env"]["VAR_4"] print ("Expected Value -> Var1 : environment_value_1") print ("Actual Value : {}".format(var1)) print ("Expected Value -> Var3 : value_3") print ("Actual Value : {}".format(var3)) assert var3 == "value_3" assert var1 == "environment_value_1"
class RogerDeploy(object): def __init__(self): self.rogerGitPullObject = RogerGitPull() self.rogerPushObject = RogerPush() self.rogerBuildObject = RogerBuild() self.dockerUtilsObject = DockerUtils() self.dockerObject = Docker() self.utils = Utils() self.slack = None self.registry = "" self.image_name = "" # To remove a temporary directory created by roger-deploy if this # script exits def removeDirTree(self, work_dir, args, temp_dir_created): exists = os.path.exists(os.path.abspath(work_dir)) if exists and (temp_dir_created is True): shutil.rmtree(work_dir) print("Deleted temporary dir:{0}".format(work_dir)) def getNextVersion(self, config, roger_env, application, branch, work_dir, repo, args, gitObj): sha = getGitSha(work_dir, repo, branch, gitObj) docker_search = self.dockerUtilsObject.docker_search(roger_env['registry'], config['name'], application) image_version_list = [] version = '' envs = [] for each_key in roger_env["environments"].keys(): envs.append(each_key) for line in docker_search.split('\n'): image = line.split(' ')[0] # (vmahedia) todo: This belongs in a separate DockerImage class. This class should know # how to generate the image name, how to search the image or return this name pattern we # then use to search this image on the registry. matchObj = re.match("^{0}-{1}-.*/v.*".format(config['name'], application), image) if matchObj and matchObj.group().startswith(config['name'] + '-' + application): skip_image = False for env in envs: if matchObj.group().startswith("{0}-{1}-{2}".format(config['name'], application, env)): skip_image = True break if skip_image is False: if verify(str(matchObj.group().split('v')[-1])): image_version_list.append(matchObj.group().split('v')[-1]) if len(image_version_list) == 0: # Create initial version version = "{0}/v0.1.0".format(sha) print("No version currently exist in the Docker Registry. \nDeploying version:{0}".format( version)) else: version = self.incrementVersion(sha, image_version_list, args) return version def incrementVersion(self, sha, image_version_list, args): latest = max(image_version_list, key=self.splitVersion) ver_tuple = self.splitVersion(latest) latest_version = '' if args.incr_major: latest_version = "{0}/v{1}.0.0".format(sha, (int(ver_tuple[0]) + 1)) return latest_version if args.incr_patch: latest_version = "{0}/v{1}.{2}.{3}".format( sha, int(ver_tuple[0]), int(ver_tuple[1]), (int(ver_tuple[2]) + 1)) return latest_version latest_version = "{0}/v{1}.{2}.0".format( sha, int(ver_tuple[0]), (int(ver_tuple[1]) + 1)) return latest_version def splitVersion(self, version): major, _, rest = version.partition('.') minor, _, rest = rest.partition('.') patch, _, rest = rest.partition('.') return int(major), int(minor) if minor else 0, int(patch) if patch else 0 def parseArgs(self): self.parser = argparse.ArgumentParser( prog='roger deploy', description=describe()) self.parser.add_argument('-e', '--environment', metavar='env', help="environment to deploy to. Example: 'dev' or 'stage'") self.parser.add_argument('-b', '--branch', metavar='branch', default='master', help="branch to pull code from. Defaults to master. Example: 'production' or 'master'") self.parser.add_argument('-sg', '--skip-gitpull', action="store_true", help="skip the gitpull step. Defaults to false.") self.parser.add_argument('-s', '--skip-build', action="store_true", help="whether to skip the build step. Defaults to false.") self.parser.add_argument('-M', '--incr-major', action="store_true", help="increment major in version. Defaults to false.") self.parser.add_argument('-sp', '--skip-push', action="store_true", help="skip the push step. Defaults to false.'") self.parser.add_argument('-v', '--verbose', help="verbose mode for debugging. Defaults to false.", action="store_true") self.parser.add_argument('-ns', '--disable-swaparoo', help="Disables swaparoo functionality", action="store_true") self.parser.add_argument('-f', '--force-push', action="store_true", help="force push. Not recommended. Forces push even if validation checks failed. Applies only if skip_push is false. Defaults to false.") self.parser.add_argument('-p', '--incr-patch', action="store_true", help="increment patch in version. Defaults to false.'") self.parser.add_argument('-S', '--secrets-file', help="specifies an optional secrets file for deployment runtime variables.") self.parser.add_argument('--build-arg', action='append', help='docker build-arg; Use flags multiple times to pass more than one arg') # (vmahedia) todo Changing this name is slightly more complicated, so making it more verbose for now self.parser.add_argument('-d', '--directory', help="App Repo will be checked out here, this is the working dir CLI will use." "A temporary directory is created if no directory specified." "Example: '/home/vagrant/work_dir'.") self.parser.add_argument('application', metavar='application', help="application to deploy. \ Can also push specific containers(comma seperated). Example: 'all' \ or 'app1:app2' or 'kairos' or 'app_name[container1,container2]' \ or'app1[container1,container2]:app2[container3,container4]' or 'app1:app2[container]'") self.parser.add_argument('config_file', metavar='config_file', help="Path to a Roger configuration file. This can either be an absolute " "path or a relative path that is taken relative to the ROGER_CONFIG_DIR " "environment variable.") return self.parser def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args): try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' settingObj = settingObject appObj = appObject config_dir = settingObj.getConfigDir() root = settingObj.getCliDir() roger_env = appObj.getRogerEnv(config_dir) config = appObj.getConfig(config_dir, args.config_file) config_name = "" if 'name' in config: config_name = config['name'] if 'registry' not in roger_env: raise ValueError('Registry not found in roger-mesos-tools.config file.') else: self.registry = roger_env['registry'] # Setup for Slack-Client, token, and git user # (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event # Event should be registered and SlackNotification should be one of the members. it can have # N notifications on a particular "event", Notifications.Notify will broadcast notification to # all the interested parties. if 'notifications' in config: self.slack = Slack(config['notifications'], '/home/vagrant/.roger_cli.conf.d/slack_token') self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application) apps = [] apps_container_dict = {} if args.application == 'all': apps = config['apps'].keys() else: if ":" not in args.application and "[" not in args.application: apps.append(args.application) else: for item in args.application.split(":"): if '[' in item: matchObj = re.match(r'(.*)\[(.*)\]', item) apps.append(matchObj.group(1)) apps_container_dict[matchObj.group(1)] = matchObj.group(2) else: apps.append(item) common_repo = config.get('repo', '') environment = roger_env.get('default_environment', '') work_dir = '' if args.directory: work_dir = args.directory temp_dir_created = False if args.verbose: print("Using {0} as the working directory".format(work_dir)) else: work_dir = mkdtemp() temp_dir_created = True if args.verbose: print("Created a temporary dir: {0}".format(work_dir)) if args.environment is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print( "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file") else: print( "Using value {} from environment variable $ROGER_ENV".format(env_var)) environment = env_var else: environment = args.environment if environment not in roger_env['environments']: self.removeDirTree(work_dir, args, temp_dir_created) raise ValueError('Environment not found in roger-mesos-tools.config file.') branch = "master" # master by default if args.branch is not None: branch = args.branch try: for app in apps: if app not in config['apps']: raise ValueError('Application {} specified not found.'.format(app)) else: try: if args.verbose: print("Deploying {} ...".format(app)) self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict) except (IOError, ValueError) as e: error_msg = "Error when deploying {}: {}".format(app, repr(e)) printErrorMsg(error_msg) pass # try deploying the next app except (Exception) as e: printException(e) raise except (Exception) as e: execution_result = 'FAILURE' printException(e) raise finally: # todo: maybe send the datadog event, need to look pass def deployApp(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config, roger_env, work_dir, config_dir, environment, app, branch, slack, config_file, common_repo, temp_dir_created, apps_container_dict): startTime = datetime.now() settingObj = settingObject appObj = appObject frameworkUtils = frameworkUtilsObject environmentObj = roger_env['environments'][environment] data = appObj.getAppData(config_dir, config_file, app) frameworkObj = frameworkUtils.getFramework(data) framework = frameworkObj.getName() repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', app) image_name = '' image = '' skip_gitpull = True if args.skip_gitpull else False # get/update target source(s) if not skip_gitpull: args.app_name = app args.directory = work_dir self.rogerGitPullObject.identifier = self.identifier self.rogerGitPullObject.main(settingObj, appObj, gitObj, hooksObj, args) skip_build = True if args.skip_build else False skip_push = True if args.skip_push else False secrets_file = args.secrets_file if args.secrets_file else None # Set initial version # todo (vmahedia) #image_name naming should not be magic, make it explicit # todo (vmahedia) ExtractClass GitInfo, no need to pass args, we already have the information. # We deal with only one repo at a time. It may change in future but we can change the code then. image_git_sha = getGitSha(work_dir, repo, branch, gitObj) image_name = "{0}-{1}-{2}/v0.1.0".format(config['name'], app, image_git_sha) print(colored("******Fetching current version deployed or latest version from registry." "This is used to bump to next version.******", "grey")) if skip_build: curr_image_ver = frameworkObj.getCurrentImageVersion(roger_env, environment, app) if not curr_image_ver: raise ValueError("No version for this app {} is deployed on marathon; You have to build the image and " "deploy it. When skip_build is defined, cli tries to fetch deployed image ".format(app)) self.image_name = curr_image_ver if args.verbose: print("Current image version deployed on {0} is :{1}".format(framework, curr_image_ver)) if curr_image_ver is not None: image_name = "{0}-{1}-{2}".format(config['name'], app, curr_image_ver) if args.verbose: print("Image current version from {0} endpoint is:{1}".format(framework, image_name)) else: if args.verbose: print("Using base version for image:{0}".format(image_name)) else: # Docker build,tag and push image_name = self.getNextVersion(config, roger_env, app, branch, work_dir, repo, args, gitObj) print(colored("******Done finding latest version******", "green")) image_name = "{0}-{1}-{2}".format(config['name'], app, image_name) print(colored("Bumped up image to version:{0}".format(image_name), "green")) self.image_name = image_name build_args = args build_args.app_name = app build_args.directory = os.path.abspath(work_dir) build_args.tag_name = image_name build_args.config_file = config_file build_args.env = environment build_args.push = True build_args.verbose = args.verbose try: self.rogerBuildObject.identifier = self.identifier self.rogerBuildObject.main(settingObj, appObject, hooksObj, self.dockerUtilsObject, self.dockerObject, build_args) except ValueError: raise print("Image Version is: {}".format(colored(image_name, "cyan"))) # Deploying the app to framework args.image_name = image_name args.config_file = config_file args.env = environment if app in apps_container_dict: args.app_name = str(app) + ":" + apps_container_dict[app] else: args.app_name = app self.rogerPushObject.identifier = self.identifier self.rogerPushObject.main(settingObj, appObj, frameworkUtils, hooksObj, args) deployTime = datetime.now() - startTime username = settingObj.getUser() deployMessage = "{0}'s deploy for {1} / {2} / {3} completed in {4} seconds.".format( username, app, environment, branch, deployTime.total_seconds()) if slack is not None: slack.api_call(deployMessage) print(colored(deployMessage, "green"))
class RogerDeploy(object): def __init__(self): self.rogerGitPullObject = RogerGitPull() self.rogerPushObject = RogerPush() self.rogerBuildObject = RogerBuild() self.dockerUtilsObject = DockerUtils() self.dockerObject = Docker() self.utils = Utils() self.slack = None self.statsd_message_list = [] self.registry = "" self.image_name = "" # To remove a temporary directory created by roger-deploy if this # script exits def removeDirTree(self, work_dir, args, temp_dir_created): exists = os.path.exists(os.path.abspath(work_dir)) if exists and (temp_dir_created is True): shutil.rmtree(work_dir) print("Deleted temporary dir:{0}".format(work_dir)) def getNextVersion(self, config, roger_env, application, branch, work_dir, repo, args, gitObj): sha = getGitSha(work_dir, repo, branch, gitObj) docker_search = self.dockerUtilsObject.docker_search(roger_env['registry'], config['name'], application) image_version_list = [] version = '' envs = [] for each_key in roger_env["environments"].keys(): envs.append(each_key) for line in docker_search.split('\n'): image = line.split(' ')[0] # (vmahedia) todo: This belongs in a separate DockerImage class. This class should know # how to generate the image name, how to search the image or return this name pattern we # then use to search this image on the registry. matchObj = re.match("^{0}-{1}-.*/v.*".format(config['name'], application), image) if matchObj and matchObj.group().startswith(config['name'] + '-' + application): skip_image = False for env in envs: if matchObj.group().startswith("{0}-{1}-{2}".format(config['name'], application, env)): skip_image = True break if skip_image is False: if verify(str(matchObj.group().split('v')[-1])): image_version_list.append(matchObj.group().split('v')[-1]) if len(image_version_list) == 0: # Create initial version version = "{0}/v0.1.0".format(sha) print("No version currently exist in the Docker Registry. \nDeploying version:{0}".format( version)) else: version = self.incrementVersion(sha, image_version_list, args) return version def incrementVersion(self, sha, image_version_list, args): latest = max(image_version_list, key=self.splitVersion) ver_tuple = self.splitVersion(latest) latest_version = '' if args.incr_major: latest_version = "{0}/v{1}.0.0".format(sha, (int(ver_tuple[0]) + 1)) return latest_version if args.incr_patch: latest_version = "{0}/v{1}.{2}.{3}".format( sha, int(ver_tuple[0]), int(ver_tuple[1]), (int(ver_tuple[2]) + 1)) return latest_version latest_version = "{0}/v{1}.{2}.0".format( sha, int(ver_tuple[0]), (int(ver_tuple[1]) + 1)) return latest_version def splitVersion(self, version): major, _, rest = version.partition('.') minor, _, rest = rest.partition('.') patch, _, rest = rest.partition('.') return int(major), int(minor) if minor else 0, int(patch) if patch else 0 def parseArgs(self): self.parser = argparse.ArgumentParser( prog='roger deploy', description=describe()) self.parser.add_argument('-e', '--environment', metavar='env', help="environment to deploy to. Example: 'dev' or 'stage'") self.parser.add_argument('-b', '--branch', metavar='branch', default='master', help="branch to pull code from. Defaults to master. Example: 'production' or 'master'") self.parser.add_argument('-sg', '--skip-gitpull', action="store_true", help="skip the gitpull step. Defaults to false.") self.parser.add_argument('-s', '--skip-build', action="store_true", help="whether to skip the build step. Defaults to false.") self.parser.add_argument('-M', '--incr-major', action="store_true", help="increment major in version. Defaults to false.") self.parser.add_argument('-sp', '--skip-push', action="store_true", help="skip the push step. Defaults to false.'") self.parser.add_argument('-v', '--verbose', help="verbose mode for debugging. Defaults to false.", action="store_true") self.parser.add_argument('-f', '--force-push', action="store_true", help="force push. Not recommended. Forces push even if validation checks failed. Applies only if skip_push is false. Defaults to false.") self.parser.add_argument('-p', '--incr-patch', action="store_true", help="increment patch in version. Defaults to false.'") self.parser.add_argument('-S', '--secrets-file', help="specifies an optional secrets file for deployment runtime variables.") self.parser.add_argument('-d', '--directory', help="working directory. Uses a temporary directory if not specified.") self.parser.add_argument('application', metavar='application', help="application to deploy. \ Can also push specific containers(comma seperated). Example: 'all' \ or 'app1:app2' or 'kairos' or 'app_name[container1,container2]' \ or'app1[container1,container2]:app2[container3,container4]' or 'app1:app2[container]'") self.parser.add_argument('app_repo', metavar="app_repo", help="Application's git repository name, repo must be under 'seomoz' organization") self.parser.add_argument('config_file', metavar='config_file', help="configuration file to be use. Example: 'content.json' or 'kwe.json'") return self.parser def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args): try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' settingObj = settingObject appObj = appObject config_dir = settingObj.getConfigDir() root = settingObj.getCliDir() roger_env = appObj.getRogerEnv(config_dir) config = appObj.getConfig(config_dir, args.config_file) config_name = "" if 'name' in config: config_name = config['name'] if 'registry' not in roger_env: raise ValueError('Registry not found in roger-mesos-tools.config file.') else: self.registry = roger_env['registry'] # Setup for Slack-Client, token, and git user # (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event # Event should be registered and SlackNotification should be one of the members. it can have # N notifications on a particular "event", Notifications.Notify will broadcast notification to # all the interested parties. if 'notifications' in config: self.slack = Slack(config['notifications'], '/home/vagrant/.roger_cli.conf.d/slack_token') self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application) apps = [] apps_container_dict = {} if args.application == 'all': apps = config['apps'].keys() else: if ":" not in args.application and "[" not in args.application: apps.append(args.application) else: for item in args.application.split(":"): if '[' in item: matchObj = re.match(r'(.*)\[(.*)\]', item) apps.append(matchObj.group(1)) apps_container_dict[matchObj.group(1)] = matchObj.group(2) else: apps.append(item) common_repo = config.get('repo', '') environment = roger_env.get('default_environment', '') work_dir = '' if args.directory: work_dir = args.directory temp_dir_created = False if args.verbose: print("Using {0} as the working directory".format(work_dir)) else: work_dir = mkdtemp() temp_dir_created = True if args.verbose: print("Created a temporary dir: {0}".format(work_dir)) if args.environment is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print( "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file") else: print( "Using value {} from environment variable $ROGER_ENV".format(env_var)) environment = env_var else: environment = args.environment if environment not in roger_env['environments']: self.removeDirTree(work_dir, args, temp_dir_created) raise ValueError('Environment not found in roger-mesos-tools.config file.') branch = "master" # master by default if args.branch is not None: branch = args.branch try: for app in apps: if app not in config['apps']: raise ValueError('Application {} specified not found.'.format(app)) else: try: if args.verbose: print("Deploying {} ...".format(app)) self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict) except (IOError, ValueError) as e: error_msg = "Error when deploying {}: {}".format(app, repr(e)) printErrorMsg(error_msg) pass # try deploying the next app except (Exception) as e: printException(e) raise except (Exception) as e: execution_result = 'FAILURE' printException(e) raise finally: # Check if the initializition of variables carried out if 'function_execution_start_time' not in globals() and 'function_execution_start_time' not in locals(): function_execution_start_time = datetime.now() if 'execution_result' not in globals() and 'execution_result' not in locals(): execution_result = 'FAILURE' if 'config_name' not in globals() and 'config_name' not in locals(): config_name = "" if 'environment' not in globals() and 'environment' not in locals(): environment = "dev" if not hasattr(args, "application"): args.application = "" if 'settingObj' not in globals() and 'settingObj' not in locals(): settingObj = Settings() if 'work_dir' not in globals() and 'work_dir' not in locals(): work_dir = '' temp_dir_created = False if not (self.rogerGitPullObject.outcome is 1 and self.rogerBuildObject.outcome is 1 and self.rogerPushObject.outcome is 1): execution_result = 'FAILURE' try: # If the deploy fails before going through any steps sc = self.utils.getStatsClient() if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier(config_name, settingObj.getUser(), args.application) args.application = self.utils.extract_app_name(args.application) time_take_milliseonds = ((datetime.now() - function_execution_start_time).total_seconds() * 1000) input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str(args.application) + ",event=deploy" + ",outcome=" + str(execution_result) + ",config_name=" + str(config_name) + ",env=" + str(environment) + ",user="******",identifier=" + str(self.identifier) tup = (input_metric, time_take_milliseonds) self.statsd_message_list.append(tup) self.removeDirTree(work_dir, args, temp_dir_created) except (Exception) as e: error_msg = "Error when deploying {}: {}".format(app, repr(e)) printErrorMsg(error_msg) raise def deployApp(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config, roger_env, work_dir, config_dir, environment, app, branch, slack, config_file, common_repo, temp_dir_created, apps_container_dict): startTime = datetime.now() settingObj = settingObject appObj = appObject frameworkUtils = frameworkUtilsObject environmentObj = roger_env['environments'][environment] data = appObj.getAppData(config_dir, config_file, app) frameworkObj = frameworkUtils.getFramework(data) framework = frameworkObj.getName() repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', app) image_name = '' image = '' skip_gitpull = True if args.skip_gitpull else False # get/update target source(s) if not skip_gitpull: args.app_name = app args.directory = work_dir self.rogerGitPullObject.statsd_message_list = self.statsd_message_list self.rogerGitPullObject.identifier = self.identifier self.rogerGitPullObject.main(settingObj, appObj, gitObj, hooksObj, args) skip_build = True if args.skip_build else False skip_push = True if args.skip_push else False secrets_file = args.secrets_file if args.secrets_file else None # Set initial version # todo (vmahedia) #image_name naming should not be magic, make it explicit # todo (vmahedia) ExtractClass GitInfo, no need to pass args, we already have the information. # We deal with only one repo at a time. It may change in future but we can change the code then. image_git_sha = getGitSha(work_dir, repo, branch, gitObj) image_name = "{0}-{1}-{2}/v0.1.0".format(config['name'], app, image_git_sha) print(colored("******Fetching current version deployed or latest version from registry.\ This is used to bump to next version.******", "grey")) if skip_build: curr_image_ver = frameworkObj.getCurrentImageVersion( roger_env, environment, app) self.image_name = curr_image_ver if args.verbose: print("Current image version deployed on {0} is :{1}".format(framework, curr_image_ver)) if curr_image_ver is not None: image_name = "{0}-{1}-{2}".format( config['name'], app, curr_image_ver) if args.verbose: print("Image current version from {0} endpoint is:{1}".format(framework, image_name)) else: if args.verbose: print("Using base version for image:{0}".format(image_name)) else: # Docker build,tag and push image_name = self.getNextVersion( config, roger_env, app, branch, work_dir, repo, args, gitObj) print(colored("******Done finding latest version******", "green")) image_name = "{0}-{1}-{2}".format(config['name'], app, image_name) print(colored("Bumped up image to version:{0}".format(image_name), "green")) self.image_name = image_name build_args = args build_args.app_name = app build_args.directory = os.path.abspath(work_dir) build_args.tag_name = image_name build_args.config_file = config_file build_args.env = environment build_args.push = True build_args.verbose = args.verbose try: self.rogerBuildObject.identifier = self.identifier self.rogerBuildObject.statsd_message_list = self.statsd_message_list self.rogerBuildObject.main(settingObj, appObject, hooksObj, self.dockerUtilsObject, self.dockerObject, build_args) except ValueError: raise print("Image Version is: {}".format(colored(image_name, "cyan"))) # Deploying the app to framework args.image_name = image_name args.config_file = config_file args.env = environment if app in apps_container_dict: args.app_name = str(app) + ":" + apps_container_dict[app] else: args.app_name = app self.rogerPushObject.identifier = self.identifier self.rogerPushObject.statsd_message_list = self.statsd_message_list self.rogerPushObject.main(settingObj, appObj, frameworkUtils, hooksObj, args) deployTime = datetime.now() - startTime username = settingObj.getUser() deployMessage = "{0}'s deploy for {1} / {2} / {3} completed in {4} seconds.".format( username, app, environment, branch, deployTime.total_seconds()) if slack is not None: slack.api_call(deployMessage) print(colored(deployMessage, "green")) def locateConfigFile(self, args, gitObj): # Clone the git repo first because config lives there, there's nothing that we can do without this file # this is not the clean way but the code is very convulted for now to implement this in a clean manner # For now, we will clone the repo silently and use that config. We have to clone it everytime because we # have to assume that there's always a change, although it's not true - we can check if there's a change # and only pull then, but that is another mess. Let's make it simple and pull everytime branch = args.branch if args.branch else "master" repo_dir = os.path.join(args.directory,args.app_repo) if os.path.exists(args.config_file): # skip the git clone # this file path could be inside the cloned repo or outside if args.config_file.startswith(args.directory): # chdir is a decorator above but we should move it to utils or somewhere else for every file to use with chdir(repo_dir): # file is inside cloned repo so Pull to fetch changes rc = gitObj.gitPull(branch, args.verbose) if rc: print(colored("WARNING: Unable to Pull branch - {}, from repo - {}. Config file will" "not contain latest changes".format(args.branch, args.app_repo)), "yellow") # File is not inside cloned repo but somewhere else # just use it, no need to do anything else: if args.app_repo: if os.path.exists(repo_dir): raise ValueError("Repo directory - {} already exists but config file - {} is not present"\ .format(repo_dir, args.config_file)) # chdir is a decorator defined above with chdir(args.directory): if args.verbose: print(colored("Cloning repo - {} at - {} for config file - {}".\ format(args.app_repo, repo_dir, args.config_file))) # clone the repo # file does not exist and we need to clone the repo it is in repo since repo is # defined and we mandate it to be in repo exit_code = gitObj.gitShallowClone(args.app_repo, branch, args.verbose) if exit_code: raise ValueError("Error cloning repo {} while looking for config file".format(args.app_repo)) # check now if we have config file in the cloned repo, otherwise bailout. if not os.path.exists(args.config_file): # we checked out the repo and either the path is not in repo and file does not exist # the path is in repo but the repo doesn't have the config file raise ValueError("Config file - {} does not exist".format(args.config_file)) else: raise ValueError("Config file - {} does not exist, no app repo defined either".format(args.config_file))
class RogerDeploy(object): def __init__(self): self.rogerGitPullObject = RogerGitPull() self.rogerPushObject = RogerPush() self.rogerBuildObject = RogerBuild() self.dockerUtilsObject = DockerUtils() self.dockerObject = Docker() self.utils = Utils() self.slack = None self.statsd_message_list = [] self.registry = "" self.image_name = "" # To remove a temporary directory created by roger-deploy if this # script exits def removeDirTree(self, work_dir, args, temp_dir_created): exists = os.path.exists(os.path.abspath(work_dir)) if exists and (temp_dir_created is True): shutil.rmtree(work_dir) print("Deleted temporary dir:{0}".format(work_dir)) def getNextVersion(self, config, roger_env, application, branch, work_dir, repo, args, gitObj): sha = getGitSha(work_dir, repo, branch, gitObj) docker_search = self.dockerUtilsObject.docker_search( roger_env['registry'], config['name'], application) image_version_list = [] version = '' envs = [] for each_key in roger_env["environments"].keys(): envs.append(each_key) for line in docker_search.split('\n'): image = line.split(' ')[0] matchObj = re.match( "^{0}-{1}-.*/v.*".format(config['name'], application), image) if matchObj and matchObj.group().startswith(config['name'] + '-' + application): skip_image = False for env in envs: if matchObj.group().startswith("{0}-{1}-{2}".format( config['name'], application, env)): skip_image = True break if skip_image is False: if verify(str(matchObj.group().split('v')[-1])): image_version_list.append( matchObj.group().split('v')[-1]) if len(image_version_list) == 0: # Create initial version version = "{0}/v0.1.0".format(sha) print( "No version currently exist in the Docker Registry. \nDeploying version:{0}" .format(version)) else: version = self.incrementVersion(sha, image_version_list, args) return version def incrementVersion(self, sha, image_version_list, args): latest = max(image_version_list, key=self.splitVersion) ver_tuple = self.splitVersion(latest) latest_version = '' if args.incr_major: latest_version = "{0}/v{1}.0.0".format(sha, (int(ver_tuple[0]) + 1)) return latest_version if args.incr_patch: latest_version = "{0}/v{1}.{2}.{3}".format(sha, int(ver_tuple[0]), int(ver_tuple[1]), (int(ver_tuple[2]) + 1)) return latest_version latest_version = "{0}/v{1}.{2}.0".format(sha, int(ver_tuple[0]), (int(ver_tuple[1]) + 1)) return latest_version def splitVersion(self, version): major, _, rest = version.partition('.') minor, _, rest = rest.partition('.') patch, _, rest = rest.partition('.') return int(major), int(minor) if minor else 0, int( patch) if patch else 0 def parseArgs(self): self.parser = argparse.ArgumentParser(prog='roger deploy', description=describe()) self.parser.add_argument( '-e', '--environment', metavar='env', help="environment to deploy to. Example: 'dev' or 'stage'") self.parser.add_argument( 'application', metavar='application', help="application to deploy. Can also push specific" " containers(comma seperated). Example: 'all' or 'app1:app2' or 'kairos' or 'app_name[container1,container2]' or 'app1[container1,container2]:app2[container3,container4]' or 'app1:app2[container]'" ) self.parser.add_argument( '-b', '--branch', metavar='branch', help= "branch to pull code from. Defaults to master. Example: 'production' or 'master'" ) self.parser.add_argument( '-sg', '--skip-gitpull', action="store_true", help="skip the gitpull step. Defaults to false.") self.parser.add_argument( '-s', '--skip-build', action="store_true", help="whether to skip the build step. Defaults to false.") self.parser.add_argument( 'config_file', metavar='config_file', help= "configuration file to be use. Example: 'content.json' or 'kwe.json'" ) self.parser.add_argument( '-M', '--incr-major', action="store_true", help="increment major in version. Defaults to false.") self.parser.add_argument( '-sp', '--skip-push', action="store_true", help="skip the push step. Defaults to false.'") self.parser.add_argument( '-f', '--force-push', action="store_true", help= "force push. Not recommended. Forces push even if validation checks failed. Applies only if skip_push is false. Defaults to false." ) self.parser.add_argument( '-p', '--incr-patch', action="store_true", help="increment patch in version. Defaults to false.'") self.parser.add_argument( '-S', '--secrets-file', help= "specifies an optional secrets file for deployment runtime variables." ) self.parser.add_argument( '-d', '--directory', help= "working directory. Uses a temporary directory if not specified.") return self.parser def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args): try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' settingObj = settingObject appObj = appObject config_dir = settingObj.getConfigDir() root = settingObj.getCliDir() roger_env = appObj.getRogerEnv(config_dir) config = appObj.getConfig(config_dir, args.config_file) config_name = "" if 'name' in config: config_name = config['name'] if 'registry' not in roger_env: raise ValueError( 'Registry not found in roger-mesos-tools.config file.') else: self.registry = roger_env['registry'] # Setup for Slack-Client, token, and git user if 'notifications' in config: self.slack = Slack( config['notifications'], '/home/vagrant/.roger_cli.conf.d/slack_token') self.identifier = self.utils.get_identifier( config_name, settingObj.getUser(), args.application) apps = [] apps_container_dict = {} if args.application == 'all': apps = config['apps'].keys() else: if ":" not in args.application and "[" not in args.application: apps.append(args.application) else: for item in args.application.split(":"): if '[' in item: matchObj = re.match(r'(.*)\[(.*)\]', item) apps.append(matchObj.group(1)) apps_container_dict[matchObj.group( 1)] = matchObj.group(2) else: apps.append(item) common_repo = config.get('repo', '') environment = roger_env.get('default_environment', '') work_dir = '' if args.directory: work_dir = args.directory temp_dir_created = False print("Using {0} as the working directory".format(work_dir)) else: work_dir = mkdtemp() temp_dir_created = True print("Created a temporary dir: {0}".format(work_dir)) if args.environment is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print( "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file" ) else: print( "Using value {} from environment variable $ROGER_ENV" .format(env_var)) environment = env_var else: environment = args.environment if environment not in roger_env['environments']: self.removeDirTree(work_dir, args, temp_dir_created) raise ValueError( 'Environment not found in roger-mesos-tools.config file.') branch = "master" # master by default if args.branch is not None: branch = args.branch try: for app in apps: if app not in config['apps']: print( 'Application {} specified not found.'.format(app)) else: try: print("Deploying {} ...".format(app)) self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict) except (IOError, ValueError) as e: print( "The following error occurred when deploying {}: {}" .format(app, e), file=sys.stderr) pass # try deploying the next app except (Exception) as e: print("The following error occurred: %s" % e, file=sys.stderr) raise except (Exception) as e: execution_result = 'FAILURE' print("The following error occurred: %s" % e, file=sys.stderr) raise finally: # Check if the initializition of variables carried out if 'function_execution_start_time' not in globals( ) and 'function_execution_start_time' not in locals(): function_execution_start_time = datetime.now() if 'execution_result' not in globals( ) and 'execution_result' not in locals(): execution_result = 'FAILURE' if 'config_name' not in globals() and 'config_name' not in locals( ): config_name = "" if 'environment' not in globals() and 'environment' not in locals( ): environment = "dev" if not hasattr(args, "application"): args.application = "" if 'settingObj' not in globals() and 'settingObj' not in locals(): settingObj = Settings() if 'work_dir' not in globals() and 'work_dir' not in locals(): work_dir = '' temp_dir_created = False if not (self.rogerGitPullObject.outcome is 1 and self.rogerBuildObject.outcome is 1 and self.rogerPushObject.outcome is 1): execution_result = 'FAILURE' try: # If the deploy fails before going through any steps sc = self.utils.getStatsClient() if not hasattr(self, "identifier"): self.identifier = self.utils.get_identifier( config_name, settingObj.getUser(), args.application) args.application = self.utils.extract_app_name( args.application) time_take_milliseonds = ( (datetime.now() - function_execution_start_time).total_seconds() * 1000) input_metric = "roger-tools.rogeros_tools_exec_time," + "app_name=" + str( args.application) + ",event=deploy" + ",outcome=" + str( execution_result ) + ",config_name=" + str(config_name) + ",env=" + str( environment) + ",user="******",identifier=" + str( self.identifier) tup = (input_metric, time_take_milliseonds) self.statsd_message_list.append(tup) self.removeDirTree(work_dir, args, temp_dir_created) except (Exception) as e: print("The following error occurred: %s" % e, file=sys.stderr) raise def deployApp(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config, roger_env, work_dir, config_dir, environment, app, branch, slack, config_file, common_repo, temp_dir_created, apps_container_dict): startTime = datetime.now() settingObj = settingObject appObj = appObject frameworkUtils = frameworkUtilsObject environmentObj = roger_env['environments'][environment] data = appObj.getAppData(config_dir, config_file, app) frameworkObj = frameworkUtils.getFramework(data) framework = frameworkObj.getName() repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', app) image_name = '' image = '' skip_gitpull = False if args.skip_gitpull is not None: skip_gitpull = args.skip_gitpull # get/update target source(s) if not skip_gitpull: args.app_name = app args.directory = work_dir self.rogerGitPullObject.statsd_message_list = self.statsd_message_list self.rogerGitPullObject.identifier = self.identifier self.rogerGitPullObject.main(settingObj, appObj, gitObj, hooksObj, args) skip_build = False if args.skip_build is not None: skip_build = args.skip_build skip_push = False if args.skip_push is not None: skip_push = args.skip_push secrets_file = None if args.secrets_file is not None: secrets_file = args.secrets_file # Set initial version image_git_sha = getGitSha(work_dir, repo, branch, gitObj) image_name = "{0}-{1}-{2}/v0.1.0".format(config['name'], app, image_git_sha) if skip_build: curr_image_ver = frameworkObj.getCurrentImageVersion( roger_env, environment, app) self.image_name = curr_image_ver print("Current image version deployed on {0} is :{1}".format( framework, curr_image_ver)) if curr_image_ver is not None: image_name = "{0}-{1}-{2}".format(config['name'], app, curr_image_ver) print("Image current version from {0} endpoint is:{1}".format( framework, image_name)) else: print("Using base version for image:{0}".format(image_name)) else: # Docker build,tag and push image_name = self.getNextVersion(config, roger_env, app, branch, work_dir, repo, args, gitObj) image_name = "{0}-{1}-{2}".format(config['name'], app, image_name) print("Bumped up image to version:{0}".format(image_name)) self.image_name = image_name build_args = args build_args.app_name = app build_args.directory = os.path.abspath(work_dir) build_args.tag_name = image_name build_args.config_file = config_file build_args.env = environment build_args.push = True try: self.rogerBuildObject.identifier = self.identifier self.rogerBuildObject.statsd_message_list = self.statsd_message_list self.rogerBuildObject.main(settingObj, appObject, hooksObj, self.dockerUtilsObject, self.dockerObject, build_args) except ValueError: raise print("Version is:" + image_name) # Deploying the app to framework args.image_name = image_name args.config_file = config_file args.env = environment if app in apps_container_dict: args.app_name = str(app) + ":" + apps_container_dict[app] else: args.app_name = app self.rogerPushObject.identifier = self.identifier self.rogerPushObject.statsd_message_list = self.statsd_message_list self.rogerPushObject.main(settingObj, appObj, frameworkUtils, hooksObj, args) deployTime = datetime.now() - startTime username = settingObj.getUser() deployMessage = "{0}'s deploy for {1} / {2} / {3} completed in {4} seconds.".format( username, app, environment, branch, deployTime.total_seconds()) if slack is not None: slack.api_call(deployMessage) print(deployMessage)
def test_container_resolution(self): settings = mock(Settings) appConfig = mock(AppConfig) roger_push = RogerPush() roger_push.utils = mock(Utils) marathon = mock(Marathon) mockedHooks = mock(Hooks) when(mockedHooks).run_hook(any(), any(), any(), any(), any(), any()).thenReturn(0) roger_env = self.roger_env config = self.test_config data = self.test_data when(roger_push.utils).get_identifier(any(), any(), any()).thenReturn(any()) when(roger_push.utils).extract_app_name(any()).thenReturn("test") when(marathon).getName().thenReturn('Marathon') when(marathon).put(any(), any(), any()).thenReturn("Response [200]") when(marathon).put(any(), any(), any()).thenReturn("Response [200]") when(marathon).put(any(), any(), any()).thenReturn("Response [200]") frameworkUtils = mock(FrameworkUtils) when(frameworkUtils).getFramework(data).thenReturn(marathon) when(settings).getComponentsDir().thenReturn( self.base_dir + "/tests/components") when(settings).getSecretsDir().thenReturn( self.base_dir + "/tests/secrets") when(settings).getTemplatesDir().thenReturn( self.base_dir + "/tests/templates") when(settings).getConfigDir().thenReturn(self.configs_dir) when(settings).getCliDir().thenReturn(self.base_dir) when(settings).getUser().thenReturn(any()) when(appConfig).getRogerEnv(self.configs_dir).thenReturn(roger_env) when(appConfig).getConfig(any(), any()).thenReturn(config) when(appConfig).getAppData(any(), any(), any()).thenReturn(data) args = self.args args.env = "dev" args.secrets_file = "" args.skip_push = True args.app_name = 'grafana_test_app' args.directory = self.base_dir + '/tests/testrepo' args.config_file = 'test.json' args.image_name = 'grafana/grafana:2.1.3' args.verbose = False roger_push.main(settings, appConfig, frameworkUtils, mockedHooks, args) with open(self.base_dir + '/tests/templates/test-app-grafana.json') as output: output = json.load(output) assert output['container']['docker'][ 'image'] == "grafana/grafana:2.1.3" assert output['cpus'] == 2 assert output['mem'] == 1024 assert output['uris'] == ["abc", "xyz", "$ENV_VAR"] with open(self.base_dir + '/tests/templates/test-app-grafana1.json') as output: output = json.load(output) assert output['container']['docker'][ 'image'] == "grafana/grafana:2.1.3" assert output['cpus'] == 0.5 assert output['mem'] == 512 with open(self.base_dir + '/tests/templates/test-app-grafana2.json') as output: output = json.load(output) assert output['container']['docker'][ 'image'] == "grafana/grafana:2.1.3" assert output['cpus'] == 1 assert output['mem'] == 1024
class RogerDeploy(object): def __init__(self): self.rogerGitPullObject = RogerGitPull() self.rogerPushObject = RogerPush() self.rogerBuildObject = RogerBuild() self.dockerUtilsObject = DockerUtils() self.dockerObject = Docker() self.utils = Utils() self.slack = None self.registry = "" self.image_name = "" # To remove a temporary directory created by roger-deploy if this # script exits def removeDirTree(self, work_dir, args, temp_dir_created): exists = os.path.exists(os.path.abspath(work_dir)) if exists and (temp_dir_created is True): shutil.rmtree(work_dir) print("Deleted temporary dir:{0}".format(work_dir)) def getNextVersion(self, config, roger_env, application, branch, work_dir, repo, args, gitObj): sha = getGitSha(work_dir, repo, branch, gitObj) docker_search = self.dockerUtilsObject.docker_search( roger_env['registry'], config['name'], application) image_version_list = [] version = '' envs = [] for each_key in roger_env["environments"].keys(): envs.append(each_key) for line in docker_search.split('\n'): image = line.split(' ')[0] # (vmahedia) todo: This belongs in a separate DockerImage class. This class should know # how to generate the image name, how to search the image or return this name pattern we # then use to search this image on the registry. matchObj = re.match( "^{0}-{1}-.*/v.*".format(config['name'], application), image) if matchObj and matchObj.group().startswith(config['name'] + '-' + application): skip_image = False for env in envs: if matchObj.group().startswith("{0}-{1}-{2}".format( config['name'], application, env)): skip_image = True break if skip_image is False: if verify(str(matchObj.group().split('v')[-1])): image_version_list.append( matchObj.group().split('v')[-1]) if len(image_version_list) == 0: # Create initial version version = "{0}/v0.1.0".format(sha) print( "No version currently exist in the Docker Registry. \nDeploying version:{0}" .format(version)) else: version = self.incrementVersion(sha, image_version_list, args) return version def incrementVersion(self, sha, image_version_list, args): latest = max(image_version_list, key=self.splitVersion) ver_tuple = self.splitVersion(latest) latest_version = '' if args.incr_major: latest_version = "{0}/v{1}.0.0".format(sha, (int(ver_tuple[0]) + 1)) return latest_version if args.incr_patch: latest_version = "{0}/v{1}.{2}.{3}".format(sha, int(ver_tuple[0]), int(ver_tuple[1]), (int(ver_tuple[2]) + 1)) return latest_version latest_version = "{0}/v{1}.{2}.0".format(sha, int(ver_tuple[0]), (int(ver_tuple[1]) + 1)) return latest_version def splitVersion(self, version): major, _, rest = version.partition('.') minor, _, rest = rest.partition('.') patch, _, rest = rest.partition('.') return int(major), int(minor) if minor else 0, int( patch) if patch else 0 def parseArgs(self): self.parser = argparse.ArgumentParser(prog='roger deploy', description=describe()) self.parser.add_argument( '-e', '--environment', metavar='env', help="environment to deploy to. Example: 'dev' or 'stage'") self.parser.add_argument( '-b', '--branch', metavar='branch', default='master', help= "branch to pull code from. Defaults to master. Example: 'production' or 'master'" ) self.parser.add_argument( '-sg', '--skip-gitpull', action="store_true", help="skip the gitpull step. Defaults to false.") self.parser.add_argument( '-s', '--skip-build', action="store_true", help="whether to skip the build step. Defaults to false.") self.parser.add_argument( '-M', '--incr-major', action="store_true", help="increment major in version. Defaults to false.") self.parser.add_argument( '-sp', '--skip-push', action="store_true", help="skip the push step. Defaults to false.'") self.parser.add_argument( '-v', '--verbose', help="verbose mode for debugging. Defaults to false.", action="store_true") self.parser.add_argument('-ns', '--disable-swaparoo', help="Disables swaparoo functionality", action="store_true") self.parser.add_argument( '-f', '--force-push', action="store_true", help= "force push. Not recommended. Forces push even if validation checks failed. Applies only if skip_push is false. Defaults to false." ) self.parser.add_argument( '-p', '--incr-patch', action="store_true", help="increment patch in version. Defaults to false.'") self.parser.add_argument( '-S', '--secrets-file', help= "specifies an optional secrets file for deployment runtime variables." ) self.parser.add_argument( '--build-arg', action='append', help= 'docker build-arg; Use flags multiple times to pass more than one arg' ) # (vmahedia) todo Changing this name is slightly more complicated, so making it more verbose for now self.parser.add_argument( '-d', '--directory', help= "App Repo will be checked out here, this is the working dir CLI will use." "A temporary directory is created if no directory specified." "Example: '/home/vagrant/work_dir'.") self.parser.add_argument('application', metavar='application', help="application to deploy. \ Can also push specific containers(comma seperated). Example: 'all' \ or 'app1:app2' or 'kairos' or 'app_name[container1,container2]' \ or'app1[container1,container2]:app2[container3,container4]' or 'app1:app2[container]'" ) self.parser.add_argument( 'config_file', metavar='config_file', help= "Path to a Roger configuration file. This can either be an absolute " "path or a relative path that is taken relative to the ROGER_CONFIG_DIR " "environment variable.") return self.parser def main(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, args): try: function_execution_start_time = datetime.now() execution_result = 'SUCCESS' settingObj = settingObject appObj = appObject config_dir = settingObj.getConfigDir() root = settingObj.getCliDir() roger_env = appObj.getRogerEnv(config_dir) config = appObj.getConfig(config_dir, args.config_file) config_name = "" if 'name' in config: config_name = config['name'] if 'registry' not in roger_env: raise ValueError( 'Registry not found in roger-mesos-tools.config file.') else: self.registry = roger_env['registry'] # Setup for Slack-Client, token, and git user # (vmahedia) todo: ExtractClass Notifications, it should know who all to notify on what event # Event should be registered and SlackNotification should be one of the members. it can have # N notifications on a particular "event", Notifications.Notify will broadcast notification to # all the interested parties. if 'notifications' in config: self.slack = Slack( config['notifications'], '/home/vagrant/.roger_cli.conf.d/slack_token') self.identifier = self.utils.get_identifier( config_name, settingObj.getUser(), args.application) apps = [] apps_container_dict = {} if args.application == 'all': apps = config['apps'].keys() else: if ":" not in args.application and "[" not in args.application: apps.append(args.application) else: for item in args.application.split(":"): if '[' in item: matchObj = re.match(r'(.*)\[(.*)\]', item) apps.append(matchObj.group(1)) apps_container_dict[matchObj.group( 1)] = matchObj.group(2) else: apps.append(item) common_repo = config.get('repo', '') environment = roger_env.get('default_environment', '') work_dir = '' if args.directory: work_dir = args.directory temp_dir_created = False if args.verbose: print( "Using {0} as the working directory".format(work_dir)) else: work_dir = mkdtemp() temp_dir_created = True if args.verbose: print("Created a temporary dir: {0}".format(work_dir)) if args.environment is None: if "ROGER_ENV" in os.environ: env_var = os.environ.get('ROGER_ENV') if env_var.strip() == '': print( "Environment variable $ROGER_ENV is not set. Using the default set from roger-mesos-tools.config file" ) else: print( "Using value {} from environment variable $ROGER_ENV" .format(env_var)) environment = env_var else: environment = args.environment if environment not in roger_env['environments']: self.removeDirTree(work_dir, args, temp_dir_created) raise ValueError( 'Environment not found in roger-mesos-tools.config file.') branch = "master" # master by default if args.branch is not None: branch = args.branch try: for app in apps: if app not in config['apps']: raise ValueError( 'Application {} specified not found.'.format(app)) else: try: if args.verbose: print("Deploying {} ...".format(app)) self.deployApp(settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config, roger_env, work_dir, config_dir, environment, app, branch, self.slack, args.config_file, common_repo, temp_dir_created, apps_container_dict) except (IOError, ValueError) as e: error_msg = "Error when deploying {}: {}".format( app, repr(e)) printErrorMsg(error_msg) pass # try deploying the next app except (Exception) as e: printException(e) raise except (Exception) as e: execution_result = 'FAILURE' printException(e) raise finally: # todo: maybe send the datadog event, need to look pass def deployApp(self, settingObject, appObject, frameworkUtilsObject, gitObj, hooksObj, root, args, config, roger_env, work_dir, config_dir, environment, app, branch, slack, config_file, common_repo, temp_dir_created, apps_container_dict): startTime = datetime.now() settingObj = settingObject appObj = appObject frameworkUtils = frameworkUtilsObject environmentObj = roger_env['environments'][environment] data = appObj.getAppData(config_dir, config_file, app) frameworkObj = frameworkUtils.getFramework(data) framework = frameworkObj.getName() repo = '' if common_repo != '': repo = data.get('repo', common_repo) else: repo = data.get('repo', app) image_name = '' image = '' skip_gitpull = True if args.skip_gitpull else False # get/update target source(s) if not skip_gitpull: args.app_name = app args.directory = work_dir self.rogerGitPullObject.identifier = self.identifier self.rogerGitPullObject.main(settingObj, appObj, gitObj, hooksObj, args) skip_build = True if args.skip_build else False skip_push = True if args.skip_push else False secrets_file = args.secrets_file if args.secrets_file else None # Set initial version # todo (vmahedia) #image_name naming should not be magic, make it explicit # todo (vmahedia) ExtractClass GitInfo, no need to pass args, we already have the information. # We deal with only one repo at a time. It may change in future but we can change the code then. image_git_sha = getGitSha(work_dir, repo, branch, gitObj) image_name = "{0}-{1}-{2}/v0.1.0".format(config['name'], app, image_git_sha) print( colored( "******Fetching current version deployed or latest version from registry." "This is used to bump to next version.******", "grey")) if skip_build: curr_image_ver = frameworkObj.getCurrentImageVersion( roger_env, environment, app) if not curr_image_ver: raise ValueError( "No version for this app {} is deployed on marathon; You have to build the image and " "deploy it. When skip_build is defined, cli tries to fetch deployed image " .format(app)) self.image_name = curr_image_ver if args.verbose: print("Current image version deployed on {0} is :{1}".format( framework, curr_image_ver)) if curr_image_ver is not None: image_name = "{0}-{1}-{2}".format(config['name'], app, curr_image_ver) if args.verbose: print("Image current version from {0} endpoint is:{1}". format(framework, image_name)) else: if args.verbose: print( "Using base version for image:{0}".format(image_name)) else: # Docker build,tag and push image_name = self.getNextVersion(config, roger_env, app, branch, work_dir, repo, args, gitObj) print(colored("******Done finding latest version******", "green")) image_name = "{0}-{1}-{2}".format(config['name'], app, image_name) print( colored("Bumped up image to version:{0}".format(image_name), "green")) self.image_name = image_name build_args = args build_args.app_name = app build_args.directory = os.path.abspath(work_dir) build_args.tag_name = image_name build_args.config_file = config_file build_args.env = environment build_args.push = True build_args.verbose = args.verbose try: self.rogerBuildObject.identifier = self.identifier self.rogerBuildObject.main(settingObj, appObject, hooksObj, self.dockerUtilsObject, self.dockerObject, build_args) except ValueError: raise print("Image Version is: {}".format(colored(image_name, "cyan"))) # Deploying the app to framework args.image_name = image_name args.config_file = config_file args.env = environment if app in apps_container_dict: args.app_name = str(app) + ":" + apps_container_dict[app] else: args.app_name = app self.rogerPushObject.identifier = self.identifier self.rogerPushObject.main(settingObj, appObj, frameworkUtils, hooksObj, args) deployTime = datetime.now() - startTime username = settingObj.getUser() deployMessage = "{0}'s deploy for {1} / {2} / {3} completed in {4} seconds.".format( username, app, environment, branch, deployTime.total_seconds()) if slack is not None: slack.api_call(deployMessage) print(colored(deployMessage, "green"))