def test_iterate_sub_groups_three_levels(self): """ Grupos: + / + /foo + /bla + /baz """ data = { "id": "/", "groups": [ { "id": "/foo", "apps": [], "groups": [ { "id": "/foo/bar", "apps": [], "groups": [{"id": "/foo/bar/baz"}] }, ] }, ], "apps": [] } group = AsgardAppGroup(MarathonGroup.from_json(data)) self.assertEqual(group.id, "/") expected_all_group_ids = ["/", "/foo", "/foo/bar", "/foo/bar/baz"] returned_groups = list(group.iterate_groups()) self.assertEqual(expected_all_group_ids, [g.id for g in returned_groups])
def test_response_group_remove_namespace_from_group_name(self, group_dev_namespace_fixture): with application.test_request_context("/v2/groups/group-b", method="GET") as ctx: ctx.request.user = self.user ok_response = FlaskResponse( response=json.dumps(group_dev_namespace_fixture['groups'][1]), status=HTTPStatus.OK, headers={} ) response_group = original_group = MarathonGroup.from_json(group_dev_namespace_fixture['groups'][1]) original_group = original_group = MarathonGroup.from_json(group_dev_namespace_fixture['groups'][1]) response_wrapper = Response(ctx.request, ok_response) filtered_group = self.filter.response_group(self.user, response_group, original_group) self.assertEqual("/group-b", filtered_group.id) self.assertEqual(1, len(filtered_group.groups)) # Não removemos o namespace de subgrupos. self.assertEqual("/dev/group-b/group-b0", filtered_group.groups[0].id)
def test_filters_can_modify_all_apps_from_group_and_subgroups(self, group_dev_namespace_fixture): """ Um fltro deve poder alterar todas as apps de todos os grupos de um response. """ with application.test_request_context("/v2/groups/group-b", method="GET") as ctx: ctx.request.user = self.user ok_response = FlaskResponse( response=json.dumps(group_dev_namespace_fixture['groups'][1]), status=HTTPStatus.OK, headers={} ) with RequestsMock() as rsps: rsps.add(method='GET', url=conf.MARATHON_ADDRESSES[0] + '/v2/groups//dev/group-b', body=json.dumps(deepcopy(group_dev_namespace_fixture['groups'][1])), status=200) rsps.add(method='GET', url=conf.MARATHON_ADDRESSES[0] + '/v2/groups//dev/group-b/group-b0', body=json.dumps(deepcopy(group_dev_namespace_fixture['groups'][1]['groups'][0])), status=200) response_wrapper = Response(ctx.request, ok_response) final_response = dispatch(user=self.user, request=response_wrapper, filters_pipeline={OperationType.READ: [DummyFilter()]}, filter_method_name_callback=lambda *args: "response_group") final_response_data = json.loads(final_response.data) returned_group = MarathonGroup.from_json(final_response_data) self.assertEqual("/dummy/dev/group-b/appb0", returned_group.apps[0].id) self.assertEqual("/dummy/dev/group-b/group-b0/app0", returned_group.groups[0].apps[0].id)
def test_response_group_remove_namespace_from_group_name( self, group_dev_namespace_fixture): with application.test_request_context("/v2/groups/group-b", method="GET") as ctx: ctx.request.user = self.user response_group = original_group = MarathonGroup.from_json( group_dev_namespace_fixture["groups"][1]) original_group = original_group = MarathonGroup.from_json( group_dev_namespace_fixture["groups"][1]) filtered_group = self.filter.response_group( self.user, response_group, original_group) self.assertEqual("/group-b", filtered_group.id) self.assertEqual(1, len(filtered_group.groups)) # Não removemos o namespace de subgrupos. self.assertEqual("/dev/group-b/group-b0", filtered_group.groups[0].id)
def test_changes_from_all_filters_are_persisted_after_response_join( self, group_dev_namespace_fixture): """ Certificamos que uma modificação de um filtro não é perdida durante a execução do pipeline. No response final, todas as modificações devem estar disponíveis. """ with application.test_request_context("/v2/groups/group-b", method="GET") as ctx: ctx.request.user = self.user ok_response = FlaskResponse( response=json.dumps(group_dev_namespace_fixture["groups"][1]), status=HTTPStatus.OK, headers={}, ) ctx.request.user = self.user response_wrapper = Response(ctx.request, ok_response) final_response = dispatch( user=self.user, request=response_wrapper, filters_pipeline={ OperationType.READ: [DummyFilter(), FooFilter()] }, filter_method_name_callback=lambda *args: "response_group", ) final_response_data = json.loads(final_response.data) returned_group = MarathonGroup.from_json(final_response_data) self.assertEqual("/foo/dummy/dev/group-b/appb0", returned_group.apps[0].id) self.assertEqual( "/foo/dummy/dev/group-b/group-b0/app0", returned_group.groups[0].apps[0].id, )
def test_call_filter_method_passing_each_group( self, group_dev_namespace_fixture): """ Todos os grupos de um response devem ser passados para os filtros, um de cada vez """ with application.test_request_context("/v2/groups/group-b", method="GET") as ctx: ctx.request.user = self.user ok_response = FlaskResponse( response=json.dumps(group_dev_namespace_fixture["groups"][1]), status=HTTPStatus.OK, headers={}, ) response_wrapper = Response(ctx.request, ok_response) final_response = dispatch( user=self.user, request=response_wrapper, filters_pipeline={OperationType.READ: [DummyFilter()]}, filter_method_name_callback=lambda *args: "response_group", ) final_response_data = json.loads(final_response.data) returned_group = MarathonGroup.from_json(final_response_data) self.assertEqual("/dummy/dev/group-b", returned_group.id) self.assertEqual("/dummy/dev/group-b/group-b0", returned_group.groups[0].id)
def test_split_groups_read_on_root_group(self, group_dev_namespace_fixture): with application.test_request_context('/v2/groups/', method='GET') as ctx: response = FlaskResponse( response=json.dumps(group_dev_namespace_fixture), status=HTTPStatus.OK, headers={} ) with RequestsMock() as rsps: rsps.add(method='GET', url=conf.MARATHON_ADDRESSES[0] + '/v2/groups//dev/', body=json.dumps(deepcopy(group_dev_namespace_fixture)), status=200) rsps.add(method='GET', url=conf.MARATHON_ADDRESSES[0] + '/v2/groups//dev/a', body=json.dumps(deepcopy(group_dev_namespace_fixture['groups'][0])), status=200) rsps.add(method='GET', url=conf.MARATHON_ADDRESSES[0] + '/v2/groups//dev/group-b', body=json.dumps(deepcopy(group_dev_namespace_fixture['groups'][1])), status=200) rsps.add(method='GET', url=conf.MARATHON_ADDRESSES[0] + '/v2/groups//dev/group-b/group-b0', body=json.dumps(deepcopy(group_dev_namespace_fixture['groups'][1]['groups'][0])), status=200) rsps.add(method='GET', url=conf.MARATHON_ADDRESSES[0] + '/v2/groups//dev/group-c', body=json.dumps(deepcopy(group_dev_namespace_fixture['groups'][2])), status=200) ctx.request.user = self.user response = Response(ctx.request, response) groups_tuple = list(response.split()) self.assertEqual(5, len(groups_tuple)) expected_groups = [AsgardAppGroup(g) for g in AsgardAppGroup(MarathonGroup.from_json(group_dev_namespace_fixture)).iterate_groups()] # Compara com os groups originais self.assertEqual(expected_groups, [g[1] for g in groups_tuple])
def test_filters_can_modify_all_apps_from_group_and_subgroups( self, group_dev_namespace_fixture): """ Um fltro deve poder alterar todas as apps de todos os grupos de um response. """ with application.test_request_context("/v2/groups/group-b", method="GET") as ctx: ctx.request.user = self.user ok_response = FlaskResponse( response=json.dumps(group_dev_namespace_fixture["groups"][1]), status=HTTPStatus.OK, headers={}, ) response_wrapper = Response(ctx.request, ok_response) final_response = dispatch( user=self.user, request=response_wrapper, filters_pipeline={OperationType.READ: [DummyFilter()]}, filter_method_name_callback=lambda *args: "response_group", ) final_response_data = json.loads(final_response.data) returned_group = MarathonGroup.from_json(final_response_data) self.assertEqual("/dummy/dev/group-b/appb0", returned_group.apps[0].id) self.assertEqual( "/dummy/dev/group-b/group-b0/app0", returned_group.groups[0].apps[0].id, )
def split(self) -> Apps: if self.is_read_request(): response_content = json.loads(self.response.data) if self.is_list_apps_request(): all_apps = list( AsgardAppGroup.from_json(response_content).iterate_apps() ) for response_app in all_apps: yield response_app, response_app return elif self.is_group_request(): response_group = AsgardAppGroup( MarathonGroup.from_json(response_content) ) for current_group in response_group.iterate_groups(): yield current_group, current_group return elif self.is_tasks_request(): for task in response_content["tasks"]: response_task = MarathonTask.from_json(task) yield response_task, response_task return elif self.is_deployment(): content = response_content deployments = ( MarathonDeployment.from_json(deploy) for deploy in content ) for deployment in deployments: yield deployment, deployment return elif self.is_queue_request(): queue_data = response_content queued_apps = ( MarathonQueueItem.from_json(queue_item) for queue_item in queue_data["queue"] ) for queued_app in queued_apps: yield queued_app, queued_app return else: response_app = AsgardApp.from_json( response_content.get("app") or response_content ) app = self.marathon_client.get_app(self.object_id) yield response_app, app return if self.is_write_request(): response_content = json.loads(self.response.data) if "tasks" in response_content: for task in response_content["tasks"]: response_task = MarathonTask.from_json(task) yield response_task, response_task return return yield AsgardApp(), self.marathon_client.get_app(self.app_id)
def split(self) -> Apps: if self.is_read_request(): response_content = json.loads(self.response.data) if self.is_list_apps_request(): for app in response_content['apps']: response_app = SieveMarathonApp.from_json(app) app = self.marathon_client.get_app(self.object_id or response_app.id) yield response_app, app return elif self.is_group_request(): response_group = SieveAppGroup( MarathonGroup.from_json(response_content)) for current_group in response_group.iterate_groups(): group_id = current_group.id group_id_without_namespace = self._remove_namespace_if_exists( self.request.user.current_account.namespace, group_id) original_group = self._get_original_group( self.request.user, group_id_without_namespace) yield current_group, original_group return elif self.is_tasks_request(): for task in response_content['tasks']: response_task = MarathonTask.from_json(task) yield response_task, response_task return elif self.is_deployment(): content = response_content deployments = (MarathonDeployment.from_json(deploy) for deploy in content) for deployment in deployments: yield deployment, deployment return elif self.is_queue_request(): queue_data = response_content queued_apps = (MarathonQueueItem.from_json(queue_item) for queue_item in queue_data['queue']) for queued_app in queued_apps: yield queued_app, queued_app return else: response_app = SieveMarathonApp.from_json( response_content.get('app') or response_content) app = self.marathon_client.get_app(self.object_id) yield response_app, app return if self.is_write_request(): response_content = json.loads(self.response.data) if 'tasks' in response_content: for task in response_content['tasks']: response_task = MarathonTask.from_json(task) yield response_task, response_task return return yield SieveMarathonApp(), self.marathon_client.get_app(self.app_id)
def _get_original_group(self, user, group_id): group_id_with_namespace = "/{}/{}".format( user.current_account.namespace, (group_id or "/").strip("/")) try: return SieveAppGroup( self.marathon_client.get_group(group_id_with_namespace)) except NotFoundError as e: return SieveAppGroup( MarathonGroup.from_json({"id": group_id_with_namespace}))
def test_modify_some_apps(self): data = {"id": "/", "apps": [{"id": "/foo"}, {"id": "/bla"}]} group = AsgardAppGroup(MarathonGroup.from_json(data)) apps = list(group.iterate_apps()) apps[0].id = "/foo0" apps[1].id = "/bla0" apps_modified = list(group.iterate_apps()) self.assertEqual(["/foo0", "/bla0"], [app.id for app in apps_modified])
def test_response_group_remove_namespace_from_app_name( self, group_dev_namespace_fixture): """ Devemos remover o namespace de todas as apps to grupo. Mas não das apps dos subgrupos """ with application.test_request_context("/v2/groups/group-b", method="GET") as ctx: ctx.request.user = self.user response_group = original_group = MarathonGroup.from_json( group_dev_namespace_fixture["groups"][1]) original_group = original_group = MarathonGroup.from_json( group_dev_namespace_fixture["groups"][1]) filtered_group = self.filter.response_group( self.user, response_group, original_group) self.assertEqual("/group-b", filtered_group.id) self.assertEqual(1, len(filtered_group.apps)) self.assertEqual("/group-b/appb0", filtered_group.apps[0].id)
def test_response_group_remove_namespace_from_all_tasks_of_all_apps_empty_task_list( self, group_dev_namespace_fixture): with application.test_request_context("/v2/groups/a", method="GET") as ctx: ctx.request.user = self.user response_group = original_group = MarathonGroup.from_json( group_dev_namespace_fixture["groups"][0]) response_group.apps[0].tasks = [] original_group.apps[0].tasks = [] filtered_group = self.filter.response_group( self.user, response_group, original_group) self.assertEqual(1, len(filtered_group.apps))
def test_iterate_sub_groups_one_level(self): data = { "id": "/", "groups": [ {"id": "/foo", "apps": []}, {"id": "/bla", "apps": []}, ], "apps": [] } group = AsgardAppGroup(MarathonGroup.from_json(data)) self.assertEqual(group.id, "/") all_group_ids = ["/", "/foo", "/bla"] self.assertEqual(all_group_ids, [g.id for g in group.iterate_groups()])
def test_iterate_group_apps(self): data = { "id": "/", "groups": [ { "id": "/foo", "apps": [ { "id": "/foo/app0" }, { "id": "/foo/app1" }, ], "groups": [ { "id": "/foo/bar", "apps": [], "groups": [{ "id": "/foo/bar/baz", "apps": [ { "id": "/foo/bar/baz/app0" }, { "id": "/foo/bar/baz/app1" }, ] }] }, ] }, ], "apps": [{ "id": "/app0" }, { "id": "/app1" }] } group = SieveAppGroup(MarathonGroup.from_json(data)) self.assertEqual(group.id, "/") expected_all_apps_ids = [ "/app0", "/app1", "/foo/app0", "/foo/app1", "/foo/bar/baz/app0", "/foo/bar/baz/app1" ] returned_apps = list(group.iterate_apps()) self.assertEqual(expected_all_apps_ids, [g.id for g in returned_apps])
def test_response_group_remove_namespace_from_all_tasks_of_all_apps_empty_task_list(self, group_dev_namespace_fixture): with application.test_request_context("/v2/groups/a", method="GET") as ctx: ctx.request.user = self.user ok_response = FlaskResponse( response=json.dumps(group_dev_namespace_fixture['groups'][0]), status=HTTPStatus.OK, headers={} ) response_group = original_group = MarathonGroup.from_json(group_dev_namespace_fixture['groups'][0]) response_group.apps[0].tasks = [] original_group.apps[0].tasks = [] response_wrapper = Response(ctx.request, ok_response) filtered_group = self.filter.response_group(self.user, response_group, original_group) self.assertEqual(1, len(filtered_group.apps))
def test_response_group_remove_namespace_from_all_tasks_of_all_apps(self, group_dev_namespace_fixture): with application.test_request_context("/v2/groups/a", method="GET") as ctx: ctx.request.user = self.user ok_response = FlaskResponse( response=json.dumps(group_dev_namespace_fixture['groups'][0]), status=HTTPStatus.OK, headers={} ) response_group = original_group = MarathonGroup.from_json(group_dev_namespace_fixture['groups'][0]) response_wrapper = Response(ctx.request, ok_response) filtered_group = self.filter.response_group(self.user, response_group, original_group) self.assertEqual(1, len(filtered_group.apps)) self.assertEqual("a_app0.a31dfafb-be63-11e7-8ef1-0242a8c1e33e", filtered_group.apps[0].tasks[0].id) self.assertEqual("/a/app0", filtered_group.apps[0].tasks[0].app_id) self.assertEqual("a_app0.a31dfafb-be63-11e7-8ef1-0242a8c1e44o", filtered_group.apps[0].tasks[1].id) self.assertEqual("/a/app0", filtered_group.apps[0].tasks[1].app_id)
def test_get_original_group_not_found(self): """ Tenta buscar um grupo que não existe. """ with application.test_request_context('/v2/groups//not-found', method='GET') as ctx: with RequestsMock() as rsps: rsps.add(method='GET', url=conf.MARATHON_ADDRESSES[0] + '/v2/groups//dev/not-found', status=404) ctx.request.user = self.user request = Request(ctx.request) group = request._get_original_group(self.user, "/not-found") self.assertEqual( AsgardAppGroup( MarathonGroup.from_json({"id": "/dev/not-found"})), group)
def test_split_groups_read_on_specific_group(self, group_dev_namespace_fixture): with application.test_request_context("/v2/groups/group-b", method="GET") as ctx: response = FlaskResponse( response=json.dumps(group_dev_namespace_fixture["groups"][1]), status=HTTPStatus.OK, headers={}, ) ctx.request.user = self.user response = Response(ctx.request, response) groups_tuple = list(response.split()) self.assertEqual(2, len(groups_tuple)) expected_groups = [ AsgardAppGroup(g) for g in AsgardAppGroup( MarathonGroup.from_json( group_dev_namespace_fixture["groups"] [1])).iterate_groups() ] # Compara com os groups originais self.assertEqual(expected_groups, [g[1] for g in groups_tuple])
def test_response_group_remove_namespace_from_all_tasks_of_all_apps( self, group_dev_namespace_fixture): with application.test_request_context("/v2/groups/a", method="GET") as ctx: ctx.request.user = self.user response_group = original_group = MarathonGroup.from_json( group_dev_namespace_fixture["groups"][0]) filtered_group = self.filter.response_group( self.user, response_group, original_group) self.assertEqual(1, len(filtered_group.apps)) self.assertEqual( "a_app0.a31dfafb-be63-11e7-8ef1-0242a8c1e33e", filtered_group.apps[0].tasks[0].id, ) self.assertEqual("/a/app0", filtered_group.apps[0].tasks[0].app_id) self.assertEqual( "a_app0.a31dfafb-be63-11e7-8ef1-0242a8c1e44o", filtered_group.apps[0].tasks[1].id, ) self.assertEqual("/a/app0", filtered_group.apps[0].tasks[1].app_id)
import sys import json def _patch_docker_params(params): for p in params: for k, v in p.items(): if "=" in v: idx_ = v.index("=") parts = v.split("=") v = parts[0] + "=" + "/sieve" + parts[1] p[k] = v _group = SieveAppGroup(MarathonGroup.from_json(json.loads(sys.stdin.read()))) for group in _group.iterate_groups(): group.id = "/sieve{}".format(group.id) del group.version print(group.id, file=sys.stderr) for app in group.apps: app.id = "/sieve{}".format(app.id) app.fetch = [] del app.version _patch_docker_params([ p for p in app.container.docker.parameters if p['value'].startswith("hollowman.appname") ]) print(" >", app.id, app.container.docker.parameters, file=sys.stderr)
def from_json(cls, *args, **kwargs): return SieveAppGroup(MarathonGroup.from_json(*args, **kwargs))
def from_json(self, data): self._marathon_group = MarathonGroup.from_json(data) return self
def from_json(cls, *args, **kwargs): return AsgardAppGroup(MarathonGroup.from_json(*args, **kwargs))
import sys import json def _patch_docker_params(params): for p in params: for k, v in p.items(): if "=" in v: idx_ = v.index("=") parts = v.split("=") v = parts[0] + "=" + "/sieve" + parts[1] p[k] = v _group = AsgardAppGroup(MarathonGroup.from_json(json.loads(sys.stdin.read()))) for group in _group.iterate_groups(): group.id = "/sieve{}".format(group.id) del group.version print(group.id, file=sys.stderr) for app in group.apps: app.id = "/sieve{}".format(app.id) app.fetch = [] del app.version _patch_docker_params([ p for p in app.container.docker.parameters if p['value'].startswith("hollowman.appname") ]) print(" >", app.id, app.container.docker.parameters, file=sys.stderr)
#!/usr/bin/env python # Script que percorre todas as apps e groups de um json (de Grupo) removendo: # app.version # app.fetch # group.version import sys import json from hollowman.marathon.group import AsgardAppGroup from marathon.models.group import MarathonGroup from marathon.util import MarathonJsonEncoder data = open(sys.argv[1]).read() _g = AsgardAppGroup(MarathonGroup.from_json(json.loads(data))) for group in _g.iterate_groups(): del group.version for app in group.apps: del app.version del app.fetch data_output = json.dumps(_g._marathon_group, cls=MarathonJsonEncoder) print(data_output)
#!/usr/bin/env python # Script que percorre todas as apps e groups de um json (de Grupo) removendo: # app.version # app.fetch # group.version import sys import json from hollowman.marathon.group import SieveAppGroup from marathon.models.group import MarathonGroup from marathon.util import MarathonJsonEncoder data = open(sys.argv[1]).read() _g = SieveAppGroup(MarathonGroup.from_json(json.loads(data))) for group in _g.iterate_groups(): del group.version for app in group.apps: del app.version del app.fetch data_output = json.dumps(_g._marathon_group, cls=MarathonJsonEncoder) print(data_output)