async def test_list_jobs_return_ordered_by_name(self, dev_with_infra_fixture, dev_another_job_fixture): await _load_jobs_into_chronos(dev_another_job_fixture, dev_with_infra_fixture) account = Account(**ACCOUNT_DEV_DICT) resp = await self.client.get( "/jobs", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, ) self.assertEqual(HTTPStatus.OK, resp.status) expected_asgard_jobs = [ ChronosScheduledJobConverter.to_asgard_model( ChronosJob( **dev_another_job_fixture)).remove_namespace(account), ChronosScheduledJobConverter.to_asgard_model( ChronosJob( **dev_with_infra_fixture)).remove_namespace(account), ] resp_data = await resp.json() self.assertEqual(expected_asgard_jobs[0], resp_data["jobs"][0]) self.assertEqual(expected_asgard_jobs[1], resp_data["jobs"][1])
async def test_convert_to_client_full_model(self, chronos_job_fixture): """ Confirma que os campos que são, na verdade, sub-modelos também são incluídos na conversão. """ chronos_job_original = ChronosJob(**chronos_job_fixture) asgard_job = ChronosScheduledJobConverter.to_asgard_model( chronos_job_original) chronos_job_converted = ChronosScheduledJobConverter.to_client_model( asgard_job) self.assertEqual(chronos_job_original.dict(), chronos_job_converted.dict())
async def test_does_not_include_auth_header_if_no_auth_provided( self, dev_job_fixture ): chronos_job = ChronosJob(**dev_job_fixture) with aioresponses() as rsps: rsps.get( "http://chronos/v1/scheduler/job/job-id", status=200, payload=chronos_job.dict(), ) resp = await self.client.get_job_by_id("job-id") self.assertEqual(chronos_job.dict(), resp.dict()) self.assert_auth_header_present( rsps, "get", "http://chronos/v1/scheduler/job/job-id", {} )
async def test_list_jobs_do_not_include_jobs_from_alternate_account( self, dev_job_fixture, infra_job_fixture): """ Valida o parametro ?account_id= """ await _load_jobs_into_chronos(dev_job_fixture, infra_job_fixture) account = Account(**ACCOUNT_INFRA_DICT) resp = await self.client.get( "/jobs", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, params={"account_id": ACCOUNT_INFRA_ID}, ) self.assertEqual(HTTPStatus.OK, resp.status) expected_asgard_jobs = [ ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**infra_job_fixture)).remove_namespace(account) ] resp_data = await resp.json() self.assertEqual( ScheduledJobsListResource(jobs=expected_asgard_jobs).dict(), resp_data, )
async def test_delete_job_job_exist(self, dev_job_fixture): await _load_jobs_into_chronos(dev_job_fixture) asgard_job = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**dev_job_fixture)).remove_namespace(self.account) resp = await self.client.delete( f"/jobs/{asgard_job.id}", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, ) self.assertEqual(HTTPStatus.OK, resp.status) resp_data = await resp.json() self.assertEqual( ScheduledJobResource(job=asgard_job).dict(), resp_data) resp = await self.client.get( f"/jobs/{asgard_job.id}", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, ) self.assertEqual(HTTPStatus.NOT_FOUND, resp.status)
async def test_update_job_job_exist(self, dev_job_fixture): """ Conferimos que um job é atualizado corretamente """ await _load_jobs_into_chronos(dev_job_fixture) asgard_job = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**dev_job_fixture)) asgard_job.remove_namespace(self.account) self.assertEqual(asgard_job.cpus, dev_job_fixture["cpus"]) self.assertEqual(asgard_job.mem, dev_job_fixture["mem"]) asgard_job.cpus = 2 asgard_job.mem = 2048 resp = await self.client.put( f"/jobs/{asgard_job.id}", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, json=asgard_job.dict(), ) self.assertEqual(HTTPStatus.ACCEPTED, resp.status) updated_job_response = await self.client.get( f"/jobs/{asgard_job.id}", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, ) updated_job_data = await updated_job_response.json() updated_job_resource = CreateScheduledJobResource(**updated_job_data) self.assertEqual(asgard_job.cpus, updated_job_resource.job.cpus) self.assertEqual(asgard_job.mem, updated_job_resource.job.mem)
async def test_create_job_validation_error(self, infra_job_fixture): """ Validamos que retornamos HTTPStatus.UNPROCESSABLE_ENTITY caso a entrada esteja incompleta """ account = Account(**ACCOUNT_DEV_DICT) asgard_job_no_namespace = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**infra_job_fixture)).remove_namespace(account) incomplete_asgard_job = asgard_job_no_namespace.dict() del incomplete_asgard_job["container"] resp = await self.client.post( "/jobs", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, json=incomplete_asgard_job, ) self.assertEqual(HTTPStatus.UNPROCESSABLE_ENTITY, resp.status) resp_data = await resp.json() expected_error_msg = """1 validation error for ScheduledJob\ncontainer\n field required (type=value_error.missing)""" self.assertEqual( ErrorResource(errors=[ErrorDetail(msg=expected_error_msg)]).dict(), resp_data, )
async def test_create_job_name_has_namespace_from_another_account( self, infra_job_fixture): await _cleanup_chronos() account = Account(**ACCOUNT_DEV_DICT) asgard_job_no_namespace = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**infra_job_fixture)).remove_namespace(account) resp = await self.client.post( "/jobs", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, json=asgard_job_no_namespace.dict(), ) self.assertEqual(HTTPStatus.CREATED, resp.status) resp_data = await resp.json() self.assertEqual( f"{asgard_job_no_namespace.id}", CreateScheduledJobResource(**resp_data).job.id, ) await asyncio.sleep(0.3) resp_created_job = await self.client.get( f"/jobs/{asgard_job_no_namespace.id}", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, ) self.assertEqual(HTTPStatus.OK, resp_created_job.status)
async def test_convert_to_asgard_model_enabled_field( self, chronos_job_fixture): """ O Campo orignal no chronos é "disabled". Como nosso campo é "enabled", os valores devem ser invertidos no momento da conversão dos modelos """ chronos_job = ChronosJob(**chronos_job_fixture) asgard_scheduled_job = ChronosScheduledJobConverter.to_asgard_model( chronos_job) self.assertTrue(asgard_scheduled_job.enabled) chronos_job.disabled = True asgard_scheduled_job = ChronosScheduledJobConverter.to_asgard_model( chronos_job) self.assertFalse(asgard_scheduled_job.enabled)
async def test_use_auth_data_on_create(self, dev_job_fixture): url = "http://chronos/v1/scheduler/iso8601" job = ChronosJob(**dev_job_fixture) with aioresponses() as rsps: rsps.post(url, status=200, payload={}) await self.client_with_auth.create_job(job) self.assert_auth_header_present(rsps, "post", url, self.auth_header)
async def test_jobs_get_by_id_job_exist(self, chronos_job_fixture): chronos_job_fixture["name"] = f"{self.account.namespace}-my-job" async with http_client as client: await client.post( f"{settings.SCHEDULED_JOBS_SERVICE_ADDRESS}/v1/scheduler/iso8601", json=chronos_job_fixture, ) # Para dar tempo do chronos registra e responder no request log abaixo await asyncio.sleep(1) asgard_job = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**chronos_job_fixture)) # A busca deve ser feita sempre *sem* o namespace asgard_job.remove_namespace(self.account) resp = await self.client.get( f"/jobs/{asgard_job.id}", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, ) self.assertEqual(HTTPStatus.OK, resp.status) resp_data = await resp.json() self.assertEqual( ScheduledJobResource(job=asgard_job).dict(), resp_data)
async def test_update_job_add_default_fetch_uri(self): del self.chronos_dev_job_fixture["fetch"] new_fetch_uri = FetchURLSpec( uri="https://static.server.com/assets/main.css") expected_fetch_list = [ new_fetch_uri, FetchURLSpec( uri=settings.SCHEDULED_JOBS_DEFAULT_FETCH_URIS[0].uri), FetchURLSpec( uri=settings.SCHEDULED_JOBS_DEFAULT_FETCH_URIS[1].uri), ] await _load_jobs_into_chronos(self.chronos_dev_job_fixture) asgard_job = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**self.chronos_dev_job_fixture)) asgard_job.add_fetch_uri(new_fetch_uri) asgard_job.remove_namespace(self.account) await self.backend.update_job(asgard_job, self.user, self.account) stored_job = await self.backend.get_job_by_id(asgard_job.id, self.user, self.account) self.assertEqual(expected_fetch_list, stored_job.fetch)
async def test_create_job_on_alternate_account(self, dev_job_fixture): """ Confirmar que podemos fazer POST /jobs?account_id=<id> o o job será criado com o namespace da account de id = <id> """ await _cleanup_chronos() resp = await self.client.post( "/jobs", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, params={"account_id": ACCOUNT_INFRA_ID}, json=ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**dev_job_fixture)).dict(), ) self.assertEqual(HTTPStatus.CREATED, resp.status) resp_data = await resp.json() self.assertEqual( f"{dev_job_fixture['name']}", CreateScheduledJobResource(**resp_data).job.id, ) resp_created_job = await self.client.get( f"/jobs/{dev_job_fixture['name']}", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, params={"account_id": ACCOUNT_INFRA_ID}, ) self.assertEqual(HTTPStatus.OK, resp_created_job.status)
def to_client_model(cls, other: ScheduledJob) -> ChronosJob: env_list = None fetch_list = None constraints_list = None if other.env: env_list = ChronosEnvSpecConverter.to_client_model(other.env) if other.fetch: fetch_list = ChronosFetchURLSpecConverter.to_client_model( other.fetch) if other.constraints: constraints_list = ChronosConstraintSpecConverter.to_client_model( other.constraints) return ChronosJob( name=other.id, command=other.command, arguments=other.arguments, description=other.description, cpus=other.cpus, shell=other.shell, retires=other.retries, disabled=not other.enabled, concurrent=other.concurrent, mem=other.mem, disk=other.disk, schedule=other.schedule.value, scheduleTimeZone=other.schedule.tz, container=ChronosContainerSpecConverter.to_client_model( other.container), environmentVariables=env_list, fetch=fetch_list, constraints=constraints_list, )
async def test_convert_to_asgard_model_constraints_field( self, chronos_job_fixture): chronos_job = ChronosJob(**chronos_job_fixture) asgard_job = ChronosScheduledJobConverter.to_asgard_model(chronos_job) self.assertEqual( ["hostname:LIKE:10.0.0.1", "workload:LIKE:general"], asgard_job.constraints, )
async def test_delete_job_job_does_not_exist(self): await _cleanup_chronos() job_not_found = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**self.chronos_dev_job_fixture)) job_not_found.id = "this-job-does-not-exist" with self.assertRaises(NotFoundEntity): await self.backend.delete_job(job_not_found, self.user, self.account)
async def test_to_client_model_retries_field(self, chronos_job_fixture): chronos_job = ChronosJob(**chronos_job_fixture) asgard_job = ChronosScheduledJobConverter.to_asgard_model(chronos_job) self.assertEqual(chronos_job_fixture["retries"], asgard_job.retries) asgard_job.retries = 4 chronos_job_converted = ChronosScheduledJobConverter.to_client_model( asgard_job) self.assertEqual(asgard_job.retries, chronos_job_converted.retries)
async def test_to_client_model_disabled_field(self, chronos_job_fixture): asgard_job = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**chronos_job_fixture)) self.assertFalse( ChronosScheduledJobConverter.to_client_model(asgard_job).disabled) asgard_job.enabled = False self.assertTrue( ChronosScheduledJobConverter.to_client_model(asgard_job).disabled)
async def test_use_auth_data_on_delete(self, dev_job_fixture): job = ChronosJob(**dev_job_fixture) url = f"http://chronos/v1/scheduler/job/{dev_job_fixture['name']}" with aioresponses() as rsps: rsps.delete(url, status=200, payload={}) await self.client_with_auth.delete_job(job) self.assert_auth_header_present( rsps, "delete", url, self.auth_header )
async def create_job(self, job: ChronosJob) -> ChronosJob: """ O Chronos, pelo menos até a versão v3.0.2, tem um problema com jobs que usam timezone diferente de UTC. Quando colocamos, por exemplo, tz=America/Sao_Paulo o jobs fica programado para a hora certa, mas quando o momento chega o job fica com status OVERDUE mas *não roda*, nem aparece nos logs a tentativa de rodar o jobs. """ await self._request("post", f"{self.base_url}/iso8601", json=job.dict()) return job
async def test_list_jobs_no_not_include_jobs_from_other_namespaces( self, infra_job_fixture, dev_job_fixture): await _load_jobs_into_chronos(infra_job_fixture, dev_job_fixture) user = User(**USER_WITH_MULTIPLE_ACCOUNTS_DICT) account = Account(**ACCOUNT_DEV_DICT) jobs = await self.backend.list_jobs(user, account) expected_asgard_job = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**dev_job_fixture)).remove_namespace(account) self.assertCountEqual([expected_asgard_job], jobs)
async def get_job_by_id(self, job_id: str) -> ChronosJob: async with http_client as client: resp = await client.get(f"{self.address}/v1/scheduler/job/{job_id}" ) if resp.status == HTTPStatus.BAD_REQUEST: # `/job/{name}` retorna 400 se o job não existe. # Isso acontece por causa dessa linha: # https://github.com/mesosphere/chronos/blob/7eff5e0e2d666a94bf240608a05afcbad5f2235f/src/main/scala/org/apache/mesos/chronos/scheduler/api/JobManagementResource.scala#L51 raise Http404() data = await resp.json() return ChronosJob(**data)
async def test_convert_to_asgard_model_schedule_field( self, chronos_job_fixture): chronos_job = ChronosJob(**chronos_job_fixture) asgard_job = ChronosScheduledJobConverter.to_asgard_model(chronos_job) self.assertEqual( { "value": chronos_job_fixture["schedule"], "tz": chronos_job_fixture["scheduleTimeZone"], }, asgard_job.schedule.dict(), )
async def setUp(self): self.backend = ChronosScheduledJobsBackend() self.chronos_dev_job_fixture = get_fixture( "scheduled-jobs/chronos/dev-with-infra-in-name.json") self.asgard_job = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**self.chronos_dev_job_fixture)) self.user = User(**USER_WITH_MULTIPLE_ACCOUNTS_DICT) self.account = Account(**ACCOUNT_DEV_DICT)
async def test_to_asgard_model_required_fields(self, chronos_job_fixture): del chronos_job_fixture["environmentVariables"] del chronos_job_fixture["constraints"] del chronos_job_fixture["fetch"] asgard_job = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**chronos_job_fixture)) asgard_job_converted = ChronosScheduledJobConverter.to_asgard_model( ChronosScheduledJobConverter.to_client_model(asgard_job)) self.assertEqual(asgard_job_converted.dict(), asgard_job.dict())
async def search(self, name: str) -> List[ChronosJob]: """ Procura por todos os jobs que contenham o termo `name` em seu nome. """ resp = await self._request( "get", f"{self.address}/v1/scheduler/jobs/search", params={"name": name}, ) data = await resp.json() jobs = [ChronosJob(**job) for job in data] return jobs
async def test_convert_to_asgard_model_env_field(self, chronos_job_fixture): chronos_job = ChronosJob(**chronos_job_fixture) asgard_job = ChronosScheduledJobConverter.to_asgard_model(chronos_job) self.assertEqual( { "ENV_1": "VALUE_1", "ENV_2": "VALUE_2", "ENV_3": "VALUE_3" }, asgard_job.dict()["env"], )
async def test_create_job_add_internal_field_values(self, dev_job_fixture): """ Conferimos que quando um job é criado adicionamos os valores obrigatórios de alguns campos """ await _cleanup_chronos() account = Account(**ACCOUNT_DEV_DICT) asgard_job_no_namespace = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**dev_job_fixture)).remove_namespace(account) resp = await self.client.post( "/jobs", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, json=asgard_job_no_namespace.dict(), ) self.assertEqual(HTTPStatus.CREATED, resp.status) resp_data = await resp.json() self.assertEqual( f"{asgard_job_no_namespace.id}", CreateScheduledJobResource(**resp_data).job.id, ) await asyncio.sleep(0.5) resp_created_job = await self.client.get( f"/jobs/{asgard_job_no_namespace.id}", headers={ "Authorization": f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}" }, ) created_job_resource = ScheduledJobResource( **await resp_created_job.json()) self.assertEqual(HTTPStatus.OK, resp_created_job.status) expected_constraints = asgard_job_no_namespace.constraints + [ f"owner:LIKE:{account.owner}" ] self.assertEqual(created_job_resource.job.constraints, expected_constraints) expected_fetch_uris = asgard_job_no_namespace.fetch + [ FetchURLSpec( uri=settings.SCHEDULED_JOBS_DEFAULT_FETCH_URIS[0].uri), FetchURLSpec( uri=settings.SCHEDULED_JOBS_DEFAULT_FETCH_URIS[1].uri), ] self.assertEqual(created_job_resource.job.fetch, expected_fetch_uris)
async def test_to_client_model_required_fields(self, chronos_job_fixture): asgard_job_dict = ChronosScheduledJobConverter.to_asgard_model( ChronosJob(**chronos_job_fixture)).dict() del asgard_job_dict["env"] del asgard_job_dict["fetch"] del asgard_job_dict["constraints"] chronos_job = ChronosScheduledJobConverter.to_client_model( ScheduledJob(**asgard_job_dict)) chronos_converted = ChronosScheduledJobConverter.to_client_model( ChronosScheduledJobConverter.to_asgard_model(chronos_job)) self.assertEqual(chronos_converted.dict(), chronos_job.dict())
async def get_job_by_id(self, job_id: str) -> ChronosJob: """ Retorna um Job do Chronos, dado seu id (nome). Raise asgard.http.exceptions.HTTPNotFound() se o job não existir """ try: resp = await self._request( "get", f"{self.address}/v1/scheduler/job/{job_id}") except HTTPBadRequest as e: # `/job/{name}` retorna 400 se o job não existe. # Isso acontece por causa dessa linha: # https://github.com/mesosphere/chronos/blob/7eff5e0e2d666a94bf240608a05afcbad5f2235f/src/main/scala/org/apache/mesos/chronos/scheduler/api/JobManagementResource.scala#L51 raise HTTPNotFound(request_info=e.request_info) data = await resp.json() return ChronosJob(**data)