async def test_get_apps_stats_with_data(self): """ Prepara um ElasticSearch com alguns dados e faz o cálculo agregado do uso de CPU e RAM """ app_stats_datapoints = get_fixture( f"agents/ead07ffb-5a61-42c9-9386-21b680597e6c-S0/app_stats.json") app = MesosApp(id="infra/asgard/api") await self._load_app_stats_into_storage(self.INDEX_NAME, self.utc_now, app_stats_datapoints) backend = MarathonAppsBackend() user = User(**USER_WITH_MULTIPLE_ACCOUNTS_DICT) account = Account(**ACCOUNT_DEV_DICT) async with Elasticsearch([settings.STATS_API_URL]) as es: raw = await es.search(index=self.INDEX_NAME) cpu_pcts = [ hit["_source"]["cpu_pct"] for hit in raw["hits"]["hits"] ] mem_pcts = [ hit["_source"]["mem_pct"] for hit in raw["hits"]["hits"] ] self.assertEqual(5, len(cpu_pcts)) self.assertEqual(5, len(mem_pcts)) app_stats = await backend.get_app_stats(app, user, account) self.assertEqual( AppStats(cpu_pct="0.25", ram_pct="15.05", cpu_thr_pct="1.00"), app_stats, )
def test_join_one_app_should_produce_one_app_not_a_list(self, fixture): """ Um POST em /v2/apps, apesar de receber no body apens uma app ({...}), após o request.join(), restá produzindo um request com uma lista de apps: [{...}], e a API do marathon não aceita lista no POST apenas no PUT. O problema parece ser no request.join():89, onde fazemos if self.is_list_app_request(). Precisamos olhar se é PUT ou POST e gerar list() ou dict() apropriadamente. """ with application.test_request_context("/v2/apps/", method="POST", data=json.dumps(fixture)) as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) mock_app = get_fixture("single_full_app.json") mock_apps = [(MarathonApp.from_json(mock_app), Mock())] joined_request = request_parser.join(mock_apps) self.assertIsInstance(joined_request, HollowmanRequest) joined_request_data = json.loads(joined_request.data) self.assertFalse( isinstance(joined_request_data, list), "Body não deveria ser uma lista", ) self.assertEqual("/foo", joined_request_data["id"])
async def test_call_on_event_when_sse_event_is_found(self): """ A cada par de linhas: event: ... data: ... Chamamos o método `on_event()` """ content = get_fixture("sse/single-event.txt") with asynctest.patch.object( self.consumer, "keep_runnig", side_effect=[True, False]), asynctest.patch.object( self.consumer, "on_event") as on_event_mock: with aioresponses() as m: m.get("http://localhost:8080/v2/events", status=200, body=content) await self.consumer.start() args_list = on_event_mock.await_args_list self.assertEqual( [ asynctest.mock.call( b"event_stream_attached", b'{"remoteAddress":"172.18.0.1","eventType":"event_stream_attached","timestamp":"2018-09-03T18:03:45.685Z"}', ) ], args_list, )
async def test_apps_stats_app_not_found(self): async with HttpClientContext(app) as client: local_address = ( f"http://{client._server.host}:{client._server.port}") with aioresponses( passthrough=[local_address, settings.STATS_API_URL ]) as rsps: agent_id = "ead07ffb-5a61-42c9-9386-21b680597e6c-S0" build_mesos_cluster(rsps, agent_id) # namespace=asgard-infra app_stats_datapoints = get_fixture( f"agents/{agent_id}/app_stats.json") resp = await client.get( f"/apps/asgard/api/not-exist/stats?account_id={ACCOUNT_DEV_ID}", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, ) self.assertEqual(HTTPStatus.OK, resp.status) data = await resp.json() self.assertEqual( AppStats(cpu_pct="0", ram_pct="0", cpu_thr_pct="0").dict(), data["stats"], )
def action_expander(): """ :return: :rtype: ActionExpander """ all_actions_source_data = get_fixture('policies-gen.json.js') return ActionExpander(PolicyGenActionsMasterList(all_actions_source_data))
def setUp(self): self.single_full_app_fixture = get_fixture("single_full_app.json") self.filter = BasicConstraintFilter() self.request_app = SieveMarathonApp.from_json(self.single_full_app_fixture) self.original_app = Mock() self.user = Mock() self.constraints = (BasicConstraintFilter.mesos_constraint, BasicConstraintFilter.workload_constraint)
async def test_apps_stats_with_avg_1_min(self): with aioresponses(passthrough=[ TEST_LOCAL_AIOHTTP_ADDRESS, settings.STATS_API_URL ]) as rsps: agent_id = "ead07ffb-5a61-42c9-9386-21b680597e6c-S0" build_mesos_cluster(rsps, agent_id) # namespace=asgard-infra app_stats_datapoints = get_fixture( f"agents/{agent_id}/app_stats.json") await self._load_app_stats_into_storage(self.INDEX_NAME, self.utc_now, app_stats_datapoints) resp = await self.client.get( f"/apps/infra/asgard/api/stats/avg-1min?account_id={ACCOUNT_DEV_ID}", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, ) self.assertEqual(HTTPStatus.OK, resp.status) data = await resp.json() self.assertEqual( AppStats(cpu_pct="0.25", ram_pct="15.05", cpu_thr_pct="1.00").dict(), data["stats"], )
async def test_get_agent_by_id_logs_error_if_failed(self): """ O campo total_apps retorna com valor 0 caso não tenha sido possível obter a lista de apps rodando em um Agent. Nesse caso o campo `errors` deverá estar preenchido """ agent_data = get_fixture( "agents/ead07ffb-5a61-42c9-9386-21b680597e6c-S10/info.json") agent_id = agent_data["id"] agent_hostname = agent_data["hostname"] with aioresponses() as rsps, mock.patch.object( log, "logger") as logger_mock: build_mesos_cluster(rsps, { "id": agent_id, "apps": ClusterOptions.CONNECTION_ERROR }) agent = await self.mesos_backend.get_agent_by_id( agent_id, self.user, self.account) self.assertEqual(agent_id, agent.id) self.assertEqual([], agent.applications) self.assertEqual(0, agent.total_apps) self.assertEqual({"total_apps": "INDISPONIVEL"}, agent.errors) logger_mock.exception.assert_called_with({ "event": "Erro buscando agent applications", "agent": agent_id, "hostname": agent_hostname, })
def test_transform(): epub = WLDocument.from_file( get_fixture('text', 'asnyk_zbior.xml'), provider=DirDocProvider(get_fixture('text', '')) ).as_epub(flags=['without_fonts']).get_file() zipf = ZipFile(epub) # Check contributor list. last = zipf.open('OPS/last.html') tree = html.parse(last) editors_attribution = False for par in tree.findall("//p"): if par.text.startswith(u'Opracowanie redakcyjne i przypisy:'): editors_attribution = True assert_equal(par.text.rstrip(), u'Opracowanie redakcyjne i przypisy: ' u'Adam Fikcyjny, Aleksandra Sekuła, Olga Sutkowska.') assert_true(editors_attribution)
def test_transform(): epub = WLDocument.from_file( get_fixture('text', 'asnyk_zbior.xml'), provider=DirDocProvider(get_fixture( 'text', ''))).as_epub(flags=['without_fonts']).get_file() zipf = ZipFile(epub) # Check contributor list. last = zipf.open('OPS/last.html') tree = html.parse(last) editors_attribution = False for par in tree.findall("//p"): if par.text.startswith(u'Opracowanie redakcyjne i przypisy:'): editors_attribution = True assert_equal( par.text.rstrip(), u'Opracowanie redakcyjne i przypisy: ' u'Adam Fikcyjny, Aleksandra Sekuła, Olga Sutkowska.') assert_true(editors_attribution)
def setUp(self): single_full_app_fixture = get_fixture("single_full_app.json") self.filter = AddOwnerConstraintFilter() self.request_app = AsgardApp.from_json(single_full_app_fixture) self.original_app = AsgardApp.from_json(single_full_app_fixture) self.user = UserDB() self.account_dev = Account(**ACCOUNT_DEV_DICT) self.user.current_account = self.account_dev
async def test_mesos_agent_calculate_stats(self): """ Each agent should be capable of calculating its ocupancy """ agent_info = get_fixture( "agents/ead07ffb-5a61-42c9-9386-21b680597e6c-S10/info.json") agent = MesosAgent(**agent_info) self.assertEqual({}, agent.stats) await agent.calculate_stats() self.assertEqual({"cpu_pct": "16.00", "ram_pct": "22.50"}, agent.stats)
async def setUp(self): self.backend = ChronosScheduledJobsBackend() self.chronos_dev_job_fixture = get_fixture( "scheduled-jobs/chronos/dev-with-infra-in-name.json") self.asgard_job = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**self.chronos_dev_job_fixture)) self.user = User(**USER_WITH_MULTIPLE_ACCOUNTS_DICT) self.account = Account(**ACCOUNT_DEV_DICT)
def test_wlpicture(): wlp = picture.WLPicture.from_file(open(get_fixture("picture", "angelus-novus.xml"))) pi = wlp.picture_info # from nose.tools import set_trace; set_trace() assert pi.type[0] == u"Image" assert pi.mime_type == u"image/jpeg" == wlp.mime_type assert wlp.slug == "angelus-novus" assert path.exists(wlp.image_path) f = wlp.image_file("r") f.close()
def test_wlpicture(): wlp = picture.WLPicture.from_file(open(get_fixture('picture', 'angelus-novus.xml'))) pi = wlp.picture_info # from nose.tools import set_trace; set_trace() assert pi.type[0] == u"Image" assert pi.mime_type == u'image/png' == wlp.mime_type assert wlp.slug == 'angelus-novus' assert path.exists(wlp.image_path) f = wlp.image_file('r') f.close()
async def test_get_apps_returns_empty_list_if_no_apps_running_on_agent( self): agent_id = "ead07ffb-5a61-42c9-9386-21b680597e6c-S4" slave = get_fixture(f"agents/{agent_id}/info.json") slave_id = slave["id"] self.account.owner = slave["attributes"]["owner"] with aioresponses() as rsps: build_mesos_cluster(rsps, agent_id) agent = await self.mesos_backend.get_agent_by_id( slave_id, self.user, self.account) apps = await self.mesos_backend.get_apps_running_for_agent( self.user, agent) self.assertEqual(0, len(apps))
def test_wlpicture(): wlp = picture.WLPicture.from_file( open(get_fixture('picture', 'angelus-novus.xml'))) pi = wlp.picture_info # from nose.tools import set_trace; set_trace() assert pi.type[0] == u"Image" assert pi.mime_type == u'image/jpeg' == wlp.mime_type assert wlp.slug == 'angelus-novus' assert path.exists(wlp.image_path) f = wlp.image_file('r') f.close()
def test_it_recreates_a_put_request_for_multiple_apps(self, fixture): with application.test_request_context('/v2/apps/', method='PUT', data=json.dumps(fixture)) as ctx: ctx.request.user = self.user request_parser = Request(ctx.request) mock_app = get_fixture('single_full_app.json') mock_apps = [(MarathonApp.from_json(mock_app), Mock()) for _ in range(2)] request = request_parser.join(mock_apps) self.assertIsInstance(request, HollowmanRequest) self.assertCountEqual( [app['id'] for app in json.loads(request.data)], [app.id for app, _ in mock_apps])
def test_picture_parts(): wlp = picture.WLPicture.from_file(open(get_fixture('picture', 'angelus-novus.xml'))) parts = list(wlp.partiter()) assert len(parts) == 5, "there should be %d parts of the picture" % 5 motifs = set() names = set() print parts for p in parts: for m in p['themes']: motifs.add(m) for p in parts: if p['object']: names.add(p['object']) assert motifs == set([u'anioł historii', u'spojrzenie']), "missing motifs, got: %s" % motifs assert names == set([u'obraz cały', u'skrzydło']), 'missing objects, got: %s' % names
def test_picture_parts(): wlp = picture.WLPicture.from_file(open(get_fixture("picture", "angelus-novus.xml"))) parts = list(wlp.partiter()) assert len(parts) == 5, "there should be %d parts of the picture" % 5 motifs = set() names = set() print parts for p in parts: for m in p["themes"]: motifs.add(m) for p in parts: if p["object"]: names.add(p["object"]) assert motifs == set([u"anioł historii", u"spojrzenie"]), "missing motifs, got: %s" % motifs assert names == set([u"obraz cały", u"skrzydło"]), "missing objects, got: %s" % names
async def test_get_apps_stats_no_data(self): """ Prepara um ElasticSearch com alguns dados e faz o cálculo agregado do uso de CPU e RAM """ app_stats_datapoints = get_fixture( f"agents/ead07ffb-5a61-42c9-9386-21b680597e6c-S0/app_stats.json") app = MesosApp(id="infra/app-does-not-exist") await self._load_app_stats_into_storage(self.INDEX_NAME, self.utc_now, app_stats_datapoints) backend = MarathonAppsBackend() user = User(**USER_WITH_MULTIPLE_ACCOUNTS_DICT) account = Account(**ACCOUNT_DEV_DICT) app_stats = await backend.get_app_stats(app, user, account) self.assertEqual(AppStats(cpu_pct="0", ram_pct="0", cpu_thr_pct="0"), app_stats)
async def test_call_on_event_ignore_blank_lines(self): content = get_fixture("sse/multi-event-blanklines-in-between.txt") with asynctest.patch.object( self.consumer, "keep_runnig", side_effect=[True, False]), asynctest.patch.object( self.consumer, "on_event") as on_event_mock: with aioresponses() as m: m.get("http://localhost:8080/v2/events", status=200, body=content) await self.consumer.start() self.assertEqual(4, on_event_mock.await_count) args_list = on_event_mock.await_args_list event_1 = ( b"group_change_success", b'{"groupId":"/asgard-dev","version":"2018-09-04T12:58:57.000Z","eventType":"group_change_success","timestamp":"2018-09-04T12:58:57.072Z"}', ) event_2 = ( b"api_post_event", b'{"clientIp":"172.18.0.1","uri":"/v2/apps//asgard-dev/bla","appDefinition":{"id":"/asgard-dev/bla","cmd":"sleep 1000","args":null,"user":null,"env":{},"instances":0,"cpus":1,"mem":128,"disk":0,"gpus":0,"executor":"","constraints":[],"uris":[],"fetch":[],"storeUrls":[],"backoffSeconds":1,"backoffFactor":1.15,"maxLaunchDelaySeconds":3600,"container":{"type":"DOCKER","volumes":[],"docker":{"image":"alpine","network":"BRIDGE","portMappings":[],"privileged":false,"parameters":[],"forcePullImage":false}},"healthChecks":[],"readinessChecks":[],"dependencies":[],"upgradeStrategy":{"minimumHealthCapacity":1,"maximumOverCapacity":1},"labels":{},"ipAddress":null,"version":"2018-09-04T12:58:57.000Z","residency":null,"secrets":{},"taskKillGracePeriodSeconds":null,"unreachableStrategy":{"inactiveAfterSeconds":0,"expungeAfterSeconds":0},"killSelection":"YOUNGEST_FIRST","ports":[10006],"portDefinitions":[{"port":10006,"protocol":"tcp","name":"default","labels":{}}],"requirePorts":false,"versionInfo":{"lastScalingAt":"2018-09-04T12:58:57.000Z","lastConfigChangeAt":"2018-09-04T12:56:56.861Z"}},"eventType":"api_post_event","timestamp":"2018-09-04T12:58:57.073Z"}', ) event_3 = ( b"status_update_event", b'{"slaveId":"40cf614e-b392-4d31-9230-090ca3c7aa83-S0","taskId":"asgard-dev_bla.0139427f-b042-11e8-9638-0242ac12001f","taskStatus":"TASK_KILLED","message":"Container exited with status 137","appId":"/asgard-dev/bla","host":"172.18.0.51","ipAddresses":[{"ipAddress":"172.17.0.4","protocol":"IPv4"}],"ports":[30989],"version":"2018-09-04T12:56:56.861Z","eventType":"status_update_event","timestamp":"2018-09-04T12:59:57.728Z"}', ) event_4 = ( b"instance_changed_event", b'{"instanceId":"asgard-dev_bla.marathon-0139427f-b042-11e8-9638-0242ac12001f","condition":"Killed","runSpecId":"/asgard-dev/bla","agentId":"40cf614e-b392-4d31-9230-090ca3c7aa83-S0","host":"172.18.0.51","runSpecVersion":"2018-09-04T12:56:56.861Z","timestamp":"2018-09-04T12:59:57.739Z","eventType":"instance_changed_event"}', ) self.assertEqual( [ asynctest.mock.call(*event_1), asynctest.mock.call(*event_2), asynctest.mock.call(*event_3), asynctest.mock.call(*event_4), ], args_list, )
async def setUp(self): await super(HollowmanAppTest, self).setUp() fixture = get_fixture("single_full_app.json") self.user = UserDB( tx_email="*****@*****.**", tx_name="John Doe", tx_authkey="69ed620926be4067a36402c3f7e9ddf0", ) self.account_dev = AccountDB( id=4, name="Dev Team", namespace="dev", owner="company" ) responses.add( method="GET", url=conf.MARATHON_ADDRESSES[0] + "/v2/apps", body=json.dumps({"apps": [fixture]}), status=200, headers={"Content-Encoding": "chunked"}, ) responses.start()
async def test_call_on_exception(self): """ Call on_exceptin when an unhandled exception is raised """ content = get_fixture("sse/single-event.txt") with asynctest.patch.object( self.consumer, "keep_runnig", side_effect=self.total_loops(1) ), asynctest.patch.object( self.consumer, "on_event", side_effect=Exception() ), asynctest.patch.object( self.consumer, "on_exception", side_effect=CoroutineMock() ) as on_exception_mock: with aioresponses() as m: m.get( "http://localhost:8080/v2/events", status=200, body=content ) await self.consumer.start() self.assertEqual(1, on_exception_mock.await_count)
async def setUp(self): self.container_spec = {"image": "alpine:3", "network": "BRIDGE"} self.schedule_spec = { "value": "20190811T2100+00:00/R", "tz": "America/Sao_Paulo", } self.required_fields_scheduled_job = { "id": "my-sheduled-app", "cpus": 5.0, "mem": 512, "container": { **self.container_spec }, "schedule": self.schedule_spec, "description": "A Scheduled Task", } dev_chronos_job_fixture = get_fixture( "scheduled-jobs/chronos/dev-another-job.json") self.asgard_job = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**dev_chronos_job_fixture))
def test_picture_parts(): wlp = picture.WLPicture.from_file( open(get_fixture('picture', 'angelus-novus.xml'))) parts = list(wlp.partiter()) expect_parts = 4 assert len( parts ) == expect_parts, "there should be %d parts of the picture" % expect_parts motifs = set() names = set() for p in parts: for m in p['themes']: motifs.add(m) for p in parts: if p['object']: names.add(p['object']) assert motifs == {u'anioł historii', u'spojrzenie'}, "missing motifs, got: %s" % motifs assert names == {u'obraz cały', u'skrzydło'}, 'missing objects, got: %s' % names
async def test_apps_stats_app_not_found(self): with aioresponses(passthrough=[ TEST_LOCAL_AIOHTTP_ADDRESS, settings.STATS_API_URL ]) as rsps: agent_id = "ead07ffb-5a61-42c9-9386-21b680597e6c-S0" build_mesos_cluster(rsps, agent_id) # namespace=asgard-infra app_stats_datapoints = get_fixture( f"agents/{agent_id}/app_stats.json") resp = await self.client.get( f"/apps/asgard/api/not-exist/stats?account_id={ACCOUNT_DEV_ID}", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, ) self.assertEqual(200, resp.status) data = await resp.json() self.assertEqual( AppStats(cpu_pct="0", ram_pct="0", cpu_thr_pct="0").dict(), data["stats"], )
async def test_mesos_client_try_all_mesos_master_urls_on_exception(self): """ Se uma (ou mais) das URLs estiver com problemas, devemos tentar todas as outras antes de retornar uma exception """ agent_id = "ead07ffb-5a61-42c9-9386-21b680597e6c-S0" agent_info = get_fixture( "agents/ead07ffb-5a61-42c9-9386-21b680597e6c-S0/info.json") async with MesosClient(*settings.MESOS_API_URLS) as mesos: with aioresponses() as rsps: rsps.get( f"{settings.MESOS_API_URLS[0]}/slaves?slave_id={agent_id}", exception=Exception("Connection Error"), ) rsps.get( f"{settings.MESOS_API_URLS[1]}/slaves?slave_id={agent_id}", status=200, payload={ "slaves": [agent_info], "recovered_slaves": [] }, ) agent = await mesos.get_agent_by_id(agent_id=agent_id) self.assertEqual(agent_id, agent.id)
async def test_get_apps_returns_apps_running_on_agent(self): agent_id = "ead07ffb-5a61-42c9-9386-21b680597e6c-S0" slave = get_fixture(f"agents/{agent_id}/info.json") slave_id = slave["id"] slave_owner = slave["attributes"]["owner"] self.account.owner = slave_owner with aioresponses() as rsps: build_mesos_cluster(rsps, agent_id) agent = await self.mesos_backend.get_agent_by_id( slave_id, self.user, self.account) apps = await self.mesos_backend.get_apps_running_for_agent( self.user, agent) self.assertEqual(5, len(apps)) expected_app_ids = sorted([ "captura/wetl/visitcentral", "portal/api", "captura/kirby/feeder", "infra/asgard/api", "infra/rabbitmq", ]) self.assertEqual(expected_app_ids, sorted([app.id for app in apps]))
def test_transform_hyphenate(): epub = WLDocument.from_file(get_fixture('text', 'asnyk_zbior.xml'), provider=DirDocProvider(get_fixture( 'text', ''))).as_epub(flags=['without_fonts'], hyphenate=True).get_file()
async def setUp(self): self.agent_id = "ead07ffb-5a61-42c9-9386-21b680597e6c-S0" agent_info = get_fixture(f"agents/{self.agent_id}/info.json") self.agent = MesosAgent(**agent_info)
def test_transform(): mobi = WLDocument.from_file( get_fixture('text', 'asnyk_zbior.xml'), provider=DirDocProvider(get_fixture('text', '')) ).as_mobi(converter_path='true').get_file()
def test_v2_deployments_get(self): fixture = get_fixture("deployments/get.json") with application.test_client() as client, RequestsMock() as rsps: rsps.add( url=f"{conf.MARATHON_ADDRESSES[0]}/v2/deployments", body=json.dumps(fixture), method="GET", status=200, ) response = client.get("/v2/deployments", headers=self.auth_header) self.assertEqual( json.loads(response.data), [{ "affectedApps": ["/foo"], "currentActions": [{ "action": "ScaleApplication", "app": "/foo", "apps": None, "pod": None, "type": None, "readinessCheckResults": [{ "taskId": "foo.c9de6033", "lastResponse": { "body": "{}", "contentType": "application/json", "status": 500, }, "name": "myReadyCheck", "ready": False, }], }], "currentStep": 2, "id": "97c136bf-5a28-4821-9d94-480d9fbb01c8", "steps": [ { "actions": [{ "action": "StartApplication", "app": "/foo", "apps": None, "pod": None, "type": None, "readinessCheckResults": None, }] }, { "actions": [{ "action": "ScaleApplication", "app": "/foo", "apps": None, "pod": None, "type": None, "readinessCheckResults": None, }] }, ], "totalSteps": 2, "version": "2015-09-30T09:09:17.614Z", }], )