def test_dispatch_should_pass_an_instance_if_SieveMarathonApp_to_filters(self):
"""
Certifica que quando um request remove todas as constraints e algum filtro adiciona novas
constraints, essas constraints adicionadas pelo filtro são preservadas
"""
class AddNewConstraintFilter:
def write(self, user, request_app, original_app):
assert isinstance(request_app, AsgardApp)
return request_app
pipeline = {
OperationType.WRITE: [AddNewConstraintFilter(), ]
}
request_data = {"constraints": []}
request_app = AsgardApp.from_json(request_data)
original_app = AsgardApp.from_json(deepcopy(self.single_full_app_fixture))
with application.test_request_context("/v2/apps/foo",
method="PUT",
headers={"Content-type": "application/json"},
data=json.dumps(request_data)) as ctx:
ctx.request.user = self.user
request = Request(ctx.request)
filtered_request = dispatch(self.user, request, filters_pipeline=pipeline)
def write(self, user, request_app: AsgardApp, app):
if not request_app.has_constraint(self.mesos_constraint.field):
request_app.constraints.append(self.mesos_constraint)
if not request_app.has_constraint(self.workload_constraint.field):
request_app.constraints.append(self.workload_constraint)
return request_app
def test_preserve_upgrade_strategy_added_by_filter(self):
class AddNewUpgradeStrategyFilter:
def write(self, user, request_app, original_app):
us_data = {
"maximumOverCapacity": 1,
"minimumHealthCapacity": 0.75
}
request_app.upgrade_strategy = MarathonUpgradeStrategy.from_json(us_data)
return request_app
pipeline = {
OperationType.WRITE: [AddNewUpgradeStrategyFilter(), ]
}
# Simulando uma app que veio sem o campo "upgradeStrategy"
request_data = {}
request_app = AsgardApp.from_json(request_data)
original_app = AsgardApp.from_json(deepcopy(self.single_full_app_fixture))
with application.test_request_context("/v2/apps/foo",
method="PUT",
headers={"Content-type": "application/json"},
data=json.dumps(request_data)) as ctx:
ctx.request.user = self.user
request = Request(ctx.request)
filtered_request = dispatch(self.user, request, filters_pipeline=pipeline)
filtered_app = AsgardApp.from_json(filtered_request.get_json())
self.assertEqual(1, filtered_app.upgrade_strategy.maximum_over_capacity)
self.assertEqual(0.75, filtered_app.upgrade_strategy.minimum_health_capacity)
self._check_other_fields("upgradeStrategy", filtered_app)
def test_preserve_constraints_added_by_filter(self):
"""
Certifica que quando um request remove todas as constraints e algum filtro adiciona novas
constraints, essas constraints adicionadas pelo filtro são preservadas
"""
class AddNewConstraintFilter:
def write(self, user, request_app, original_app):
request_app.constraints.append(MarathonConstraint.from_json("key:LIKE:value".split(":")))
return request_app
pipeline = {
OperationType.WRITE: [AddNewConstraintFilter(), ]
}
request_data = {"constraints": []}
request_app = AsgardApp.from_json(request_data)
original_app = AsgardApp.from_json(deepcopy(self.single_full_app_fixture))
with application.test_request_context("/v2/apps/foo",
method="PUT",
headers={"Content-type": "application/json"},
data=json.dumps(request_data)) as ctx:
ctx.request.user = self.user
request = Request(ctx.request)
filtered_request = dispatch(self.user, request, filters_pipeline=pipeline)
filtered_request_app = AsgardApp.from_json(filtered_request.get_json())
self.assertEqual(1, len(filtered_request_app.constraints))
self._check_other_fields("constraints", filtered_request_app)
def test_preserve_envs_added_by_filter(self):
class AddNewEnvFilter:
def write(self, user, request_app, original_app):
request_app.env["env1"] = "env-value1"
return request_app
pipeline = {
OperationType.WRITE: [AddNewEnvFilter(), ]
}
request_data = {"env": []}
request_app = AsgardApp.from_json(request_data)
original_app = AsgardApp.from_json(deepcopy(self.single_full_app_fixture))
with application.test_request_context("/v2/apps/foo",
method="PUT",
headers={"Content-type": "application/json"},
data=json.dumps(request_data)) as ctx:
ctx.request.user = self.user
request = Request(ctx.request)
filtered_request = dispatch(self.user, request, filters_pipeline=pipeline)
filtered_app = AsgardApp.from_json(filtered_request.get_json())
self.assertEqual(1, len(filtered_app.env.keys()))
self.assertEqual({"env1": "env-value1"}, filtered_app.env)
self._check_other_fields("env", filtered_app)
def test_multiapp_response_returns_multiple_marathonapp_instances(
self, fixture):
modified_app = fixture.copy()
modified_app["id"] = "/xablau"
apps = [fixture, modified_app]
with application.test_request_context("/v2/apps/",
method="GET",
data=b"") as ctx:
response = FlaskResponse(
response=json.dumps({"apps": apps}),
status=HTTPStatus.OK,
headers={},
)
response = Response(ctx.request, response)
with patch.object(response, "marathon_client") as client:
original_apps = [MarathonApp.from_json(app) for app in apps]
client.get_app.side_effect = original_apps
apps = list(response.split())
self.assertEqual(
apps,
[
(AsgardApp.from_json(fixture), original_apps[0]),
(AsgardApp.from_json(modified_app), original_apps[1]),
],
)
def test_multiapp_response_returns_multiple_marathonapp_instances(self, fixture):
modified_app = fixture.copy()
modified_app['id'] = '/xablau'
apps = [fixture, modified_app]
with application.test_request_context('/v2/apps/',
method='GET', data=b'') as ctx:
response = FlaskResponse(response=json.dumps({"apps": apps}),
status=HTTPStatus.OK,
headers={})
response = Response(ctx.request, response)
with patch.object(response, 'marathon_client') as client:
original_apps = [MarathonApp.from_json(app) for app in apps]
client.get_app.side_effect = original_apps
apps = list(response.split())
self.assertEqual([call("/foo"), call("/xablau")], client.get_app.call_args_list)
self.assertEqual(
apps,
[
(AsgardApp.from_json(fixture), original_apps[0]),
(AsgardApp.from_json(modified_app), original_apps[1])
]
)
def setUp(self, single_full_app_fixture):
self.filter = NameSpaceFilter()
self.request_app = AsgardApp.from_json(single_full_app_fixture)
self.original_app = AsgardApp.from_json(single_full_app_fixture)
self.account = Account(name="Dev Account", namespace="dev", owner="company")
self.user = User(tx_email="*****@*****.**")
self.user.current_account = self.account
def setUp(self, single_full_app_fixture):
self.docker_auth_uri = "file:///etc/docker.tar.bz2"
self.base_uris = ["http://google.com", "file://etc/file.txt"]
self.single_full_app_fixture = single_full_app_fixture
self.request_app = AsgardApp.from_json(self.single_full_app_fixture)
self.original_app = AsgardApp.from_json(self.single_full_app_fixture)
self.filter = AddURIFilter()
def test_update_app_remove_all_constraints(self):
"""
Certifica que um request que remove todas as constraints,
remove essas constraints na app original
"""
class DummyFilter:
def write(self, user, request_app, original_app):
return request_app
pipeline = {
OperationType.WRITE: [DummyFilter(), ]
}
request_data = {"constraints": []}
request_app = AsgardApp.from_json(request_data)
original_app = AsgardApp.from_json(deepcopy(self.single_full_app_fixture))
with application.test_request_context("/v2/apps/foo",
method="PUT",
data=json.dumps(request_data),
headers={"Content-type": "application/json"}) as ctx:
ctx.request.user = self.user
request = Request(ctx.request)
filtered_request = dispatch(self.user, request, filters_pipeline=pipeline)
filtered_app = AsgardApp.from_json(filtered_request.get_json())
self.assertEqual(0, len(filtered_app.constraints))
self._check_other_fields("constraints", filtered_app)
def test_update_app_change_all_constraints(self):
"""
Devemos respeitar as constraints quem estão da request,
elas devem substituir as constrains da app original
"""
class DummyFilter:
def write(self, user, request_app, original_app):
return request_app
pipeline = {
OperationType.WRITE: [DummyFilter(), ]
}
request_data = {"constraints": [["hostname", "LIKE", "myhost"]]}
request_app = AsgardApp.from_json(request_data)
original_app = AsgardApp.from_json(deepcopy(self.single_full_app_fixture))
with application.test_request_context("/v2/apps/foo",
method="PUT",
data=json.dumps(request_data),
headers={"Content-type": "application/json"}) as ctx:
ctx.request.user = self.user
request = Request(ctx.request)
filtered_request = dispatch(self.user, request, filters_pipeline=pipeline)
filtered_app = AsgardApp.from_json(filtered_request.get_json())
self.assertEqual(1, len(filtered_app.constraints))
self.assertEqual(["hostname", "LIKE", "myhost"], filtered_app.constraints[0].json_repr())
self._check_other_fields("constraints", filtered_app)
def split(self) -> Apps:
if self.is_read_request():
response_content = json.loads(self.response.data)
if self.is_list_apps_request():
all_apps = list(
AsgardAppGroup.from_json(response_content).iterate_apps()
)
for response_app in all_apps:
yield response_app, response_app
return
elif self.is_group_request():
response_group = AsgardAppGroup(
MarathonGroup.from_json(response_content)
)
for current_group in response_group.iterate_groups():
yield current_group, current_group
return
elif self.is_tasks_request():
for task in response_content["tasks"]:
response_task = MarathonTask.from_json(task)
yield response_task, response_task
return
elif self.is_deployment():
content = response_content
deployments = (
MarathonDeployment.from_json(deploy) for deploy in content
)
for deployment in deployments:
yield deployment, deployment
return
elif self.is_queue_request():
queue_data = response_content
queued_apps = (
MarathonQueueItem.from_json(queue_item)
for queue_item in queue_data["queue"]
)
for queued_app in queued_apps:
yield queued_app, queued_app
return
else:
response_app = AsgardApp.from_json(
response_content.get("app") or response_content
)
app = self.marathon_client.get_app(self.object_id)
yield response_app, app
return
if self.is_write_request():
response_content = json.loads(self.response.data)
if "tasks" in response_content:
for task in response_content["tasks"]:
response_task = MarathonTask.from_json(task)
yield response_task, response_task
return
return
yield AsgardApp(), self.marathon_client.get_app(self.app_id)
def split(self) -> Apps:
if self.is_read_request():
response_content = json.loads(self.response.data)
if self.is_list_apps_request():
for app in response_content['apps']:
response_app = AsgardApp.from_json(app)
app = self.marathon_client.get_app(self.object_id
or response_app.id)
yield response_app, app
return
elif self.is_group_request():
response_group = AsgardAppGroup(
MarathonGroup.from_json(response_content))
for current_group in response_group.iterate_groups():
group_id = current_group.id
group_id_without_namespace = self._remove_namespace_if_exists(
self.request.user.current_account.namespace, group_id)
original_group = self._get_original_group(
self.request.user, group_id_without_namespace)
yield current_group, original_group
return
elif self.is_tasks_request():
for task in response_content['tasks']:
response_task = MarathonTask.from_json(task)
yield response_task, response_task
return
elif self.is_deployment():
content = response_content
deployments = (MarathonDeployment.from_json(deploy)
for deploy in content)
for deployment in deployments:
yield deployment, deployment
return
elif self.is_queue_request():
queue_data = response_content
queued_apps = (MarathonQueueItem.from_json(queue_item)
for queue_item in queue_data['queue'])
for queued_app in queued_apps:
yield queued_app, queued_app
return
else:
response_app = AsgardApp.from_json(
response_content.get('app') or response_content)
app = self.marathon_client.get_app(self.object_id)
yield response_app, app
return
if self.is_write_request():
response_content = json.loads(self.response.data)
if 'tasks' in response_content:
for task in response_content['tasks']:
response_task = MarathonTask.from_json(task)
yield response_task, response_task
return
return
yield AsgardApp(), self.marathon_client.get_app(self.app_id)
def test_create_app_do_not_add_uri_if_exist(self):
self.single_full_app_fixture["uris"] = copy(
self.base_uris) + [self.docker_auth_uri]
self.request_app = AsgardApp.from_json(self.single_full_app_fixture)
filtered_app = self.filter.write(None, self.request_app, AsgardApp())
self.assertEqual(3, len(filtered_app.uris))
self.assertEqual(self.base_uris + [self.docker_auth_uri],
filtered_app.uris)
def test_update_suspended_app_set_instances_to_zero(self):
self.request_app = AsgardApp.from_json({"env": {"ENV_A": "VALUE"}})
self.original_app.instances = 10
filtered_app = self.filter.write(self.user, self.request_app,
AsgardApp())
self.assertIsNone(filtered_app.labels.get("traefik.enable"))
def test_create_app_add_uri_with_other_existing_uris(self):
"""
Mesmo se a app já tiver utras uris, temos que adicionar a nossa
"""
self.single_full_app_fixture['uris'] = copy(self.base_uris)
self.request_app = AsgardApp.from_json(self.single_full_app_fixture)
filtered_app = self.filter.write(None, self.request_app, AsgardApp())
self.assertEqual(3, len(filtered_app.uris))
self.assertEqual(self.base_uris + [self.docker_auth_uri], filtered_app.uris)
def _transform_to_new_format(self, app: AsgardApp):
if app.container.docker.network.lower() == NET_BRIDGE.lower():
app.networks = [{"mode": "container/bridge"}]
else:
app.networks = [{"mode": "host"}]
del app.container.docker.network
app.container.port_mappings = app.container.docker.port_mappings
return app
def setUp(self):
single_full_app_fixture = get_fixture("single_full_app.json")
self.filter = AddOwnerConstraintFilter()
self.request_app = AsgardApp.from_json(single_full_app_fixture)
self.original_app = AsgardApp.from_json(single_full_app_fixture)
self.user = UserDB()
self.account_dev = Account(**ACCOUNT_DEV_DICT)
self.user.current_account = self.account_dev
def test_transform_json_before_upstream_absent_port_mappings(
self, app_json_new_format):
del app_json_new_format["container"]["portMappings"]
request_app = AsgardApp.from_json(app_json_new_format)
original_app = AsgardApp.from_json(app_json_new_format)
filtered_app = self.filter.write(None, request_app, original_app)
self.assertIsNone(filtered_app.container.docker.port_mappings)
self.assertFalse(hasattr(filtered_app.container, "port_mappings"))
self.assertFalse(hasattr(filtered_app, "networks"))
def test_transform_json_to_new_format_change_network_before_response_to_client(
self, full_app_old_format):
"""
Movemos o dict de `container.docker.network` para `networks`
"""
del full_app_old_format["container"]["docker"]["portMappings"]
request_app = AsgardApp.from_json(full_app_old_format)
original_app = AsgardApp.from_json(full_app_old_format)
filtered_app = self.filter.response(None, request_app, original_app)
self.assertTrue(filtered_app.networks)
self.assertEqual(1, len(filtered_app.networks))
self.assertEqual("container/bridge", filtered_app.networks[0]["mode"])
def _transform_to_new_format(self, app: AsgardApp):
network_attr = getattr(app.container.docker, "network",
NET_BRIDGE.lower())
if network_attr.lower() == NET_BRIDGE.lower():
app.networks = [{"mode": "container/bridge"}]
else:
app.networks = [{"mode": "host"}]
if hasattr(app.container.docker, "network"):
del app.container.docker.network
if (hasattr(app.container.docker, "port_mappings")
and app.container.docker.port_mappings):
app.container.port_mappings = app.container.docker.port_mappings
return app
def test_noop_if_env_is_disabled(self, app_json_new_format,
app_json_old_format):
"""
O filtro nao deve rodar se a não houver indicações de que a UI já é a versão nova.
Essa indicação é feita através de um header `X-UI-Version` que é passado em todos os requests (apenas pela UI nova)
"""
with mock.patch.dict(os.environ,
ASGARD_FILTER_TRANSFORMJSON_ENABLED="0"
), application.test_request_context(
"/v2/apps/dev/foo", headers={}):
asgard_app_new_format = AsgardApp.from_json(app_json_new_format)
asgard_app_old_format = AsgardApp.from_json(app_json_old_format)
filterd_before_upstream_request = self.filter.write(
None, asgard_app_new_format, asgard_app_new_format)
filterd_before_response_to_client = self.filter.response(
None, asgard_app_old_format, asgard_app_old_format)
# Não convertemos a App para formato velho
self.assertEqual(
filterd_before_upstream_request.networks,
app_json_new_format["networks"],
)
self.assertEqual(
filterd_before_upstream_request.container.port_mappings[0].
json_repr(),
app_json_new_format["container"]["portMappings"][0],
)
self.assertFalse(
hasattr(filterd_before_upstream_request.container.docker,
"network"))
self.assertFalse(
filterd_before_upstream_request.container.docker.port_mappings)
# Não convertemos a app para o formato novo
self.assertFalse(filterd_before_response_to_client.networks)
self.assertFalse(
hasattr(filterd_before_response_to_client.container,
"port_mappings"))
self.assertEqual(
filterd_before_response_to_client.container.docker.network,
app_json_old_format["container"]["docker"]["network"],
)
self.assertEqual(
filterd_before_response_to_client.container.docker.
port_mappings[0].json_repr(),
app_json_old_format["container"]["docker"]["portMappings"][0],
)
def setUp(self):
self.single_full_app_fixture = get_fixture("single_full_app.json")
self.filter = BasicConstraintFilter()
self.request_app = AsgardApp.from_json(self.single_full_app_fixture)
self.original_app = Mock()
self.user = Mock()
self.constraints = (BasicConstraintFilter.workload_constraint,)
def test_it_recreates_a_get_response_for_multiple_apps(self, fixture):
modified_app = deepcopy(fixture)
modified_app["id"] = "/xablau"
fixtures = [fixture, modified_app]
expected_response = deepcopy(fixtures)
with application.test_request_context("/v2/apps/",
method="GET",
data=b"") as ctx:
response = FlaskResponse(
response=json.dumps({"apps": fixtures}),
status=HTTPStatus.OK,
headers={},
)
response = Response(ctx.request, response)
with patch.object(response, "marathon_client") as client:
original_apps = [AsgardApp.from_json(app) for app in fixtures]
client.get_app.side_effect = original_apps
apps = list(response.split())
joined_response = response.join(apps)
self.assertIsInstance(joined_response, FlaskResponse)
self.assertDictEqual(json.loads(joined_response.data),
{"apps": expected_response})
def test_response_apps_remove_namespace_from_all_tasks(
self, single_full_app_with_tasks_fixture):
request_app = original_app = AsgardApp.from_json(
single_full_app_with_tasks_fixture)
self.assertEqual(3, len(request_app.tasks))
modified_app = self.filter.response(self.user, request_app,
original_app)
self.assertEqual("foo.a29b3666-be63-11e7-8ef1-0242a8c1e33e",
modified_app.tasks[0].id)
self.assertEqual("/foo", modified_app.tasks[0].app_id)
self.assertEqual("foo.a31e220e-be63-11e7-8ef1-0242a8c1e33e",
modified_app.tasks[1].id)
self.assertEqual("/foo", modified_app.tasks[1].app_id)
self.assertEqual("foo.a31dfafb-be63-11e7-8ef1-0242a8c1e33e",
modified_app.tasks[2].id)
self.assertEqual("/foo", modified_app.tasks[2].app_id)
self.assertEqual(
"foo.bb1f57d0-e755-11e8-9eac-0242eb39892d",
modified_app.last_task_failure.task_id,
)
self.assertEqual("/foo", modified_app.last_task_failure.app_id)
def test_add_namespace_create_new_app(self):
"""
Para novas apps, sempre vamos adicionar o prefixo.
"""
modified_app = self.filter.write(self.user, self.request_app,
AsgardApp())
self.assertEqual("/dev/foo", modified_app.id)
def test_transform_json_already_new_format_before_response_to_client(
self, app_json_new_format):
"""
Não fazemos nada se o JSOM que vem do backend ja estiver no formato novo.
Isso vai acontecr quando atualizarmos pro Marathon 1.5.t
"""
response_app = AsgardApp.from_json(app_json_new_format)
original_app = AsgardApp.from_json(app_json_new_format)
filtered_app = self.filter.response(None, response_app, original_app)
self.assertTrue(filtered_app.networks)
self.assertTrue(hasattr(filtered_app.container, "port_mappings"))
# O model da App tem o campo, mas deve estar vazio
self.assertFalse(filtered_app.container.docker.port_mappings)
self.assertFalse(hasattr(filtered_app.container.docker, "network"))
def merge_marathon_apps(self, modified_app, base_app):
"""
A junção das duas apps (request_app (aqui modified_app) e original_app (aqui base_app)) é
sempre feita pegando todos os dados da original_app e jogando os dados da requst_app "em cima".
Não podemos usar o `minimal=Fase` na request_app pois para requests que estão *incompletos*, ou seja,
sem alguns vampos (já veremos exemplo) se esássemos minimal=False, iríramos apagar esses "campos faltantes"
da original_app. Exemplos:
request_app = {"instances": 10}
original_app está completa, com envs, constraints e tudo mais.
se usamos `minimal=False` na request_app, teremos um JSON com *todos* os campos em branco, menos o "instances".
Então quando fizermos `merged.update(modified_app.json_repr(minimal=False))`, vamos no final ter um JSON apenas com
o campo "instances" perrnchido e todo o restante vazio.
"""
merged = base_app.json_repr(minimal=False)
merged.update(modified_app.json_repr(minimal=True))
try:
raw_request_data = json.loads(self.request.data)
for key in REMOVABLE_KEYS:
if key in raw_request_data:
merged[key] = raw_request_data[key]
except Exception as e:
pass
if isinstance(base_app, MarathonTask):
return MarathonTask.from_json(merged)
return AsgardApp.from_json(merged)
def test_create_app_with_zero_instances(self):
self.request_app.instances = 0
filtered_app = self.filter.write(self.user, self.request_app,
AsgardApp())
self.assertTrue(
"hollowman.default_scale" not in filtered_app.labels.keys(),
"Não deveria ter adicionado label default_scale",
)
def test_response_apps_returns_none_if_outside_current_namespace(
self, single_full_app_with_tasks_fixture):
request_app = original_app = AsgardApp.from_json(
single_full_app_with_tasks_fixture)
request_app.id = original_app.id = "/othernamespace/foo"
self.assertIsNone(
self.filter.response(self.user, request_app, original_app))
