def test_split_groups_read_on_specific_group(self, group_b_fixture): with application.test_request_context('/v2/groups/group-b', method='GET') as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) with RequestsMock() as rsps: rsps.add(method='GET', url=conf.MARATHON_ADDRESSES[0] + '/v2/groups//dev/group-b', body=json.dumps(group_b_fixture), status=200) apps = list(request_parser.split()) self.assertEqual(2, len(apps)) original_app_one = MarathonApp.from_json( {"id": "/dev/group-b/appb0"}) original_app_two = MarathonApp.from_json( {"id": "/dev/group-b/group-b0/app0"}) expected_apps = [ (request_parser.merge_marathon_apps( MarathonApp(), original_app_one), original_app_one), (request_parser.merge_marathon_apps( MarathonApp(), original_app_two), original_app_two), ] self.assertEqual(expected_apps, apps)
def test_change_request_path_if_is_read_single_app( self, single_full_app_fixture): with application.test_request_context('/v2/apps/foo', method='GET') as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) single_full_app_fixture['id'] = "/dev/foo" apps = [(MarathonApp.from_json(single_full_app_fixture), MarathonApp.from_json(single_full_app_fixture))] request = request_parser.join(apps) self.assertIsInstance(request, HollowmanRequest) self.assertEqual("/v2/apps/dev/foo", request.path)
def test_split_groups_write_PUT_on_group(self): """ Atualmente, o único body que chega em um PUT em /v2/groups é: {"scaleBy": <N>} onde `<N>` é o fator que será multiplicado pelo atual número de TASK_RUNNING de cada app. O problema é que o Request.split() retorna uma lista de apps, e o Request.join() potencialmente vai reconstruir um body com essa lista de apps. O problema é que isso gera um request body *diferente* do orignal, já que agora temos um body contendo um APP_GROUP com todas suas apps (e sub apps). E se fazemos apenas isso, a informação do "scaleBy" se perdeu, pois se mandamos um request com o TASK_GROUP inteiro para o upstream, nada vai mudar já que as apps não foram modificadas. Uma ideia é o Core do hollowman decobrir essa ação de scaleBy e chamar o métoro "scale_by" dos filtros, já com a request_app tendo seu atributo "instances" multiplicado pelo fator. Opcionalmente o fator poderia ser passado como parametro para o filtro. Isso nos daria a possibilidade de "corrigir" um problema atual do scaleby que é: Quando damos scale_by = 2 em um app que está suspended, ela continua suspended já que 2 * 0 = 0. A ideia é que suspended apps também sejam ligadas considerando esse fator. O que faríamos no filtro seria, para toda app que instances = 0, consideramos instances = 1 e multiplicamos pelo fator. Enfim, apenas uma ideia. Temos que ver o que fazemos com esse teste aqui. """ with application.test_request_context("/v2/groups/group-b", method="PUT") as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) with RequestsMock() as rsps: rsps.add( method="GET", url=conf.MARATHON_ADDRESSES[0] + "/v2/groups//dev/group-b", body=json.dumps(group_b_fixture), status=200, ) apps = list(request_parser.split()) self.assertEqual(2, len(apps)) expected_apps = [ ( MarathonApp(), MarathonApp.from_json({"id": "/dev/group-b/appb0"}), ), ( MarathonApp(), MarathonApp.from_json( {"id": "/dev/group-b/group-b0/app0"}), ), ] self.assertEqual(expected_apps, apps)
def test_can_read_app_if_already_migrated(self, single_full_app_fixture): """ Conferimos que é possível fazer um GET em /v2/apps/<app-id> para uma app que já está migrada. O <app-id> usado é sempre *sem* namespace """ request_data = deepcopy(single_full_app_fixture) single_full_app_fixture["id"] = "/dev/foo" with application.test_request_context("/v2/apps/foo", method="GET") as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) with RequestsMock() as rsps: rsps.add( method="GET", url=conf.MARATHON_ADDRESSES[0] + "/v2/apps//dev/foo", body=json.dumps({"app": single_full_app_fixture}), status=200, ) apps = list(request_parser.split()) original_app = MarathonApp.from_json(single_full_app_fixture) expected_app = ( request_parser.merge_marathon_apps(MarathonApp(), original_app), original_app, ) self.assertEqual(apps, [expected_app])
def test_multiapp_response_returns_multiple_marathonapp_instances( self, fixture): modified_app = fixture.copy() modified_app["id"] = "/xablau" apps = [fixture, modified_app] with application.test_request_context("/v2/apps/", method="GET", data=b"") as ctx: response = FlaskResponse( response=json.dumps({"apps": apps}), status=HTTPStatus.OK, headers={}, ) response = Response(ctx.request, response) with patch.object(response, "marathon_client") as client: original_apps = [MarathonApp.from_json(app) for app in apps] client.get_app.side_effect = original_apps apps = list(response.split()) self.assertEqual( apps, [ (AsgardApp.from_json(fixture), original_apps[0]), (AsgardApp.from_json(modified_app), original_apps[1]), ], )
def test_multiapp_response_returns_multiple_marathonapp_instances(self, fixture): modified_app = fixture.copy() modified_app['id'] = '/xablau' apps = [fixture, modified_app] with application.test_request_context('/v2/apps/', method='GET', data=b'') as ctx: response = FlaskResponse(response=json.dumps({"apps": apps}), status=HTTPStatus.OK, headers={}) response = Response(ctx.request, response) with patch.object(response, 'marathon_client') as client: original_apps = [MarathonApp.from_json(app) for app in apps] client.get_app.side_effect = original_apps apps = list(response.split()) self.assertEqual([call("/foo"), call("/xablau")], client.get_app.call_args_list) self.assertEqual( apps, [ (AsgardApp.from_json(fixture), original_apps[0]), (AsgardApp.from_json(modified_app), original_apps[1]) ] )
def deploy_marathon_app(client, marathon_json, sleep_secs=10, retries=3): app_id = marathon_json['id'] CONFIG_URI = os.getenv('CONFIG_URI') if CONFIG_URI: marathon_json['uris'].append(CONFIG_URI) print("Attempting deploy Marathon app with id: %s" % app_id) print(marathon_json, file=sys.stderr) marathon_app = MarathonApp.from_json(marathon_json) # We are going to retry, in the case of blocked deployments attempt = 0 while attempt < retries: try: try: client.get_app(app_id) response = client.update_app(app_id, marathon_app) except NotFoundError: response = client.create_app(app_id, marathon_app) print(response, file=sys.stderr) print('Deployment succeeded.') break except Exception, ex: attempt += 1 print(ex.message) print('Failure attempting to deploy app. Retrying...') time.sleep(sleep_secs)
def test_join_one_app_should_produce_one_app_not_a_list(self, fixture): """ Um POST em /v2/apps, apesar de receber no body apens uma app ({...}), após o request.join(), restá produzindo um request com uma lista de apps: [{...}], e a API do marathon não aceita lista no POST apenas no PUT. O problema parece ser no request.join():89, onde fazemos if self.is_list_app_request(). Precisamos olhar se é PUT ou POST e gerar list() ou dict() apropriadamente. """ with application.test_request_context("/v2/apps/", method="POST", data=json.dumps(fixture)) as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) mock_app = get_fixture("single_full_app.json") mock_apps = [(MarathonApp.from_json(mock_app), Mock())] joined_request = request_parser.join(mock_apps) self.assertIsInstance(joined_request, HollowmanRequest) joined_request_data = json.loads(joined_request.data) self.assertFalse( isinstance(joined_request_data, list), "Body não deveria ser uma lista", ) self.assertEqual("/foo", joined_request_data["id"])
def split(self) -> Apps: if self.is_read_request(): if self.is_list_apps_request(): apps = self.marathon_client.list_apps() for app in apps: yield self.merge_marathon_apps(MarathonApp(), app), app elif self.is_app_request(): app = self._get_original_app(self.request.user, self.object_id) yield self.merge_marathon_apps(MarathonApp(), app), app elif self.is_group_request(): self.group = self._get_original_group(self.request.user, self.object_id) for app in self.group.iterate_apps(): yield self.merge_marathon_apps(MarathonApp(), app), app return # Request is a WRITE if self.is_app_request(): for app in self.get_request_data(): request_app = MarathonApp.from_json(app) app = self._get_original_app(self.request.user, self.object_id or request_app.id) yield self.merge_marathon_apps(request_app, app), app elif self.is_tasks_request(): request_data = self.request.get_json() for task_id in request_data['ids']: request_task = MarathonTask.from_json({"id": task_id}) yield request_task, request_task return
def _get_original_app(self, user, app_id): app_id_with_namespace = "/{}/{}".format(user.current_account.namespace, app_id.strip("/")) try: return self.marathon_client.get_app(app_id_with_namespace) except NotFoundError as e: return MarathonApp.from_json({"id": app_id_with_namespace})
def test_empty_labels(self): app_dict = {"labels": {}} request_app = MarathonApp.from_json(app_dict) filtered_app = self.filter.write(Mock(), request_app, Mock()) filtered_app = filtered_app.json_repr() self.assertDictEqual(filtered_app["labels"], {})
def test_a_request_for_write_operation_with_appid_in_url_path_returns_a_tuple_of_marathonapp( self, fixture): scale_up = {'instances': 10} with application.test_request_context( '/v2/apps/foo', method='PUT', data=json.dumps(scale_up)) as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) with RequestsMock() as rsps: rsps.add(method='GET', url=conf.MARATHON_ADDRESSES[0] + '/v2/apps//dev/foo', body=json.dumps({'app': fixture}), status=200) apps = list(request_parser.split()) original_app = MarathonApp.from_json(fixture) expected_apps = (request_parser.merge_marathon_apps( MarathonApp.from_json(scale_up), original_app), original_app) self.assertEqual(apps, [expected_apps])
def test_split_does_not_break_when_removing_force_parameter_if_request_is_a_list( self, fixture): request_data = {"id": "/foo", "instances": 2} with application.test_request_context( '/v2/apps/', method='PUT', data=json.dumps(request_data)) as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) with RequestsMock() as rsps: rsps.add(method='GET', url=conf.MARATHON_ADDRESSES[0] + '/v2/apps//dev/foo', body=json.dumps({'app': fixture}), status=200) apps = list(request_parser.split()) original_app = MarathonApp.from_json(fixture) expected_app = (request_parser.merge_marathon_apps( MarathonApp.from_json(request_data), original_app), original_app) self.assertEqual(apps, [expected_app])
def test_split_groups_read_on_root_group(self, group_dev_namespace_fixture): with application.test_request_context("/v2/groups", method="GET") as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) with RequestsMock() as rsps: rsps.add( method="GET", url=conf.MARATHON_ADDRESSES[0] + "/v2/groups//dev/", body=json.dumps(group_dev_namespace_fixture), status=200, ) apps = list(request_parser.split()) self.assertEqual(3, len(apps)) original_app_one = MarathonApp.from_json({"id": "/dev/a/app0"}) original_app_two = MarathonApp.from_json( {"id": "/dev/group-b/appb0"}) original_app_three = MarathonApp.from_json( {"id": "/dev/group-b/group-b0/app0"}) expected_apps = [ ( request_parser.merge_marathon_apps( MarathonApp(), original_app_one), original_app_one, ), ( request_parser.merge_marathon_apps( MarathonApp(), original_app_two), original_app_two, ), ( request_parser.merge_marathon_apps( MarathonApp(), original_app_three), original_app_three, ), ] self.assertEqual(apps, expected_apps)
def do_full_rollback(client: MarathonClient, rollback: list): print('------------------\nPerforming rollback in order:') print('\n'.join(rollback)) print('------------------') for each in rollback: if os.path.isfile(each): with open(each) as json_file: app = MarathonApp.from_json(json.load(json_file)) _update_application(client, app, each, False) else: deployment = client.delete_app(each, True) wait_for_deployment(client, deployment)
def test_it_recreates_a_post_request_for_a_single_app(self, fixture): with application.test_request_context('/v2/apps//foo', method='POST', data=json.dumps(fixture)) as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) with patch.object(request_parser, 'marathon_client') as client: client.get_app.return_value = MarathonApp.from_json(fixture) apps = list(request_parser.split()) request = request_parser.join(apps) self.assertIsInstance(request, HollowmanRequest) self.assertEqual(request.get_json()['id'], '/foo')
def test_it_recreates_a_get_request_for_a_single_app(self, fixture): with application.test_request_context("/v2/apps//foo", method="GET", data=b"") as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) with patch.object(request_parser, "marathon_client") as client: client.get_app.return_value = MarathonApp.from_json(fixture) apps = list(request_parser.split()) request = request_parser.join(apps) self.assertIsInstance(request, HollowmanRequest) self.assertEqual(request, ctx.request) self.assertEqual(request.data, b"")
def test_it_recreates_a_put_request_for_multiple_apps(self, fixture): with application.test_request_context('/v2/apps/', method='PUT', data=json.dumps(fixture)) as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) mock_app = get_fixture('single_full_app.json') mock_apps = [(MarathonApp.from_json(mock_app), Mock()) for _ in range(2)] request = request_parser.join(mock_apps) self.assertIsInstance(request, HollowmanRequest) self.assertCountEqual( [app['id'] for app in json.loads(request.data)], [app.id for app, _ in mock_apps])
def test_a_read_single_app_request_returns_a_single_marathonapp_if_app_exists( self, fixture): with application.test_request_context('/v2/apps//foo', method='GET', data=b'') as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) with patch.object(request_parser, 'marathon_client') as client: original_app = MarathonApp.from_json(fixture) client.get_app.return_value = original_app apps = list(request_parser.split()) self.assertEqual(apps, [(request_parser.merge_marathon_apps( MarathonApp(), original_app), client.get_app.return_value)])
def test_absent_env_and_labels_keys(self): app_dict = { "id": "/foo/bar", "cmd": "sleep 5000", "cpus": 1, "mem": 128, "disk": 0, "instances": 1, } request_app = MarathonApp.from_json(app_dict) filtered_app = self.filter.write(Mock(), request_app, Mock()) # filtered_app = filtered_app.json_repr() self.assertEqual(filtered_app.env, {}) self.assertEqual(filtered_app.labels, {})
def test_a_request_for_a_new_app_will_return_a_tuple_with_an_empty_marathonapp( self, fixture): with application.test_request_context('/v2/apps//foo', method='PUT', data=json.dumps(fixture)) as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) with patch.object(request_parser, 'marathon_client') as client: response_mock = Mock() response_mock.headers.get.return_value = 'application/json' client.get_app.side_effect = NotFoundError( response=response_mock) apps = list(request_parser.split()) self.assertEqual(apps, [(AsgardApp.from_json(fixture), MarathonApp.from_json({"id": "/dev/foo"}))])
def test_it_trims_labels(self): app_dict = { "labels": { "Label": "xablau", " MY_LABEL ": " abc ", " OTHER_LABEL ": " 10.0.0.1 ", } } request_app = MarathonApp.from_json(app_dict) filtered_app = self.filter.write(Mock(), request_app, Mock()) filtered_app = filtered_app.json_repr() self.assertDictEqual(filtered_app['labels'], { "Label": "xablau", "MY_LABEL": "abc", "OTHER_LABEL": "10.0.0.1" })
def test_it_trims_envvars(self): app_dict = { "env": { "ENV": "xablau", " MY_ENV ": " abc ", " OTHER_ENV ": " 10.0.0.1 ", } } request_app = MarathonApp.from_json(app_dict) filtered_app = self.filter.write(Mock(), request_app, Mock()) filtered_app = filtered_app.json_repr() self.assertDictEqual(filtered_app['env'], { "ENV": "xablau", "MY_ENV": "abc", "OTHER_ENV": "10.0.0.1" })
def deploy_marathon_app(client, marathon_json, sleep_secs=10, retries=3): app_id = marathon_json['id'] print("Attempting deploy Marathon app with id: %s" % app_id) print(marathon_json, file=sys.stderr) marathon_app = MarathonApp.from_json(marathon_json) # We are going to retry, in the case of blocked deployments attempt = 0 while attempt < retries: try: response = client.create_app(app_id, marathon_app) print(response, file=sys.stderr) print('Deployment succeeded.') break except Exception, ex: attempt += 1 print(ex.message) print('Failure attempting to deploy app. Retrying...') time.sleep(sleep_secs)
def test_a_request_for_restart_operation_with_appid_in_url_path_returns_a_tuple_of_marathonapp( self, fixture): with application.test_request_context('/v2/apps/xablau/restart', method='PUT', data=b'{"force": true}') as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) with RequestsMock() as rsps: rsps.add(method='GET', url=conf.MARATHON_ADDRESSES[0] + '/v2/apps//dev/xablau', body=json.dumps({'app': fixture}), status=200) apps = list(request_parser.split()) original_app = MarathonApp.from_json(fixture) expected_app = (request_parser.merge_marathon_apps( MarathonApp(), original_app), original_app) self.assertEqual(apps, [expected_app])
def put_app(client: MarathonClient, definition_path: str, fullrollback: bool) -> str: rollback_order = None if os.path.isdir(definition_path): prompt = input( 'The path {} is a directory. Deploy applications defined in it?\nType \'YES\'' ' to confirm: '.format(definition_path)) if prompt != 'YES': print("Aborting") sys.exit(2) if fullrollback: print( 'If you cancel any deployment, all previous applications (although successfully deployed) ' 'will be rolled back to their previous states.\nAre you totally sure?' ) if input('Type \'YES\' to confirm: ') != 'YES': print('Aborting') sys.exit(2) rollback_order = [] for definition_filename in sorted(os.listdir(definition_path)): definition_filepath = os.path.join(definition_path, definition_filename) if not definition_filename.startswith('#') and os.path.isfile( definition_filepath): # Commented files support deployed = put_app(client, definition_filepath, False) if deployed is False and rollback_order is not None: # Initiate full rollback!! rollback_order.sort(reverse=True) do_full_rollback(client, rollback_order) if rollback_order is not None: rollback_order.append(deployed) return definition_path with open(definition_path) as json_file: app = MarathonApp.from_json(json.load(json_file)) appid = app.id if app.id.startswith('/') else '/' + app.id if any(filter(lambda x: x.id == appid, client.list_apps())): return _update_application(client, app, definition_path) return _create_application(client, app, definition_path)
def create_app_from_json(self, json_data ): a = MarathonApp.from_json(json_data) return MarathonClient.create_app(self, a.id, a)
mesos_agent_map_string = os.getenv("MESOS_AGENT_MAP", None) mesos_master_urls = os.getenv("MESOS_MASTER_URLS", "http://localhost:5050").split(',') exit_code = 0 auth = None if marathon_user and marathon_password: auth = (marathon_user, marathon_password) ### Setup Logging logging.basicConfig(format="%(levelname)-8s [[[%(message)s]]]", level=getattr(logging, log_level.upper())) logging.getLogger('marathon').setLevel(logging.WARN) # INFO is too chatty logging.info("Parsing JSON app definition...") app_definition = MarathonApp.from_json(json.loads(marathon_app)) try: logging.info("Connecting to Marathon...") client = MarathonClient(marathon_urls, username=marathon_user, password=marathon_password, verify=False) except MarathonError as e: logging.error("Failed to connect to Marathon! {}".format(e)) exit_code = 1 sys.exit(exit_code) logging.info("Deploying application...") try: app = client.get_app(marathon_app_id)
def create_app_from_json(self, json_data): a = MarathonApp.from_json(json_data) return MarathonClient.create_app(self, a.id, a)
def update_app_from_json(self, json_data, force): a = MarathonApp.from_json(json_data) return MarathonClient.update_app(self, a.id, a, force)
def update_app_from_json( self, json_data, force ): a = MarathonApp.from_json(json_data) return MarathonClient.update_app(self, a.id, a, force)
def deploy(app_definition, marathon_url, instances, auth_token, zero, force): old_appids = [] # Connect to Marathon print("\nConnecting to Marathon...") c = MarathonClient(marathon_url, auth_token=auth_token) print("Connected to", marathon_url) # Pick up the Marathon App Definition file app_json = open(app_definition).read() app = MarathonApp.from_json(json.loads(app_json)) new_app_id = app.id service_name = new_app_id.split("/")[-1].split(".")[0] # Instantiate the new application on DC/OS but don't launch it yet # The application definition instances field should be 0 by default # If forced, the application will be relaunched even if the ID already exists print("\nInstantiating new application on Marathon with", app.instances, "instances...") try: c.create_app(new_app_id, app) except: if force == 'Yes': print("\nForcing redeploy of the same app id...", new_app_id) c.update_app(new_app_id, app, force=True, minimal=True) check_deployment(c, new_app_id) pass else: sys.exit() print("Created app", new_app_id) # List and find currently running apps of the same service # This assumes the naming convention (id): /some/group/service_name.uniquevalue print("\nFinding any existing apps for service:", service_name) for app in c.list_apps(): existing_service_name = app.id.split("/")[-1].split(".")[0] if (service_name == existing_service_name) and app.instances > 0: print("Found up and running application id:", app.id) old_appids.append(app.id) # If it's the first deployment ever, just launch the desired number of instances # Otherwise perform a hybrid release # Finally clean up any older app instances running if not old_appids: if instances is None: instances = 2 print("No current apps found. Launching brand new service with", instances, "instances...") c.scale_app(new_app_id, instances=instances) check_deployment(c, new_app_id) check_health(c, new_app_id) else: old_appids.reverse() if zero == 'Yes': print("\nStarting zero downtime deployment for...", new_app_id) for old_appid in old_appids: if instances is None: instances = c.get_app(old_appid).instances if (old_appid == '' or old_appid == new_app_id or old_appid == '/' + new_app_id): print("Scaling existing app_id", new_app_id, "to", instances, "instances...") c.scale_app(new_app_id, instances=instances) check_deployment(c, new_app_id) check_health(c, new_app_id) else: print("Target number of total instances:", instances) delta = int(round(instances * .50)) delta = (delta if delta > 0 else 1) scale(c, new_app_id, old_appid, delta) if (c.get_app(new_app_id).instances != instances): print("\nLaunch", instances - delta, "remaining instance(s) of the new version...") c.scale_app(new_app_id, instances=instances) check_deployment(c, new_app_id) check_health(c, new_app_id) if (c.get_app(old_appid).instances > 0): print( "Finish shutting down remaining instances of the old version..." ) c.scale_app(old_appid, instances=0) check_deployment(c, old_appid) else: print("Started deployment with downtime...") for old_appid in old_appids: c.scale_app(old_appid, instances=0) check_deployment(c, old_appid) c.scale_app(new_app_id, instances=instances) check_deployment(c, new_app_id) check_health(c, new_app_id) print("\nSUCCESS:\nNew application ID:", new_app_id, "\nRunning instances:", instances)