async def test_list_jobs_return_ordered_by_name(self,
dev_with_infra_fixture,
dev_another_job_fixture):
await _load_jobs_into_chronos(dev_another_job_fixture,
dev_with_infra_fixture)
account = Account(**ACCOUNT_DEV_DICT)
resp = await self.client.get(
"/jobs",
headers={
"Authorization":
f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}"
},
)
self.assertEqual(HTTPStatus.OK, resp.status)
expected_asgard_jobs = [
ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(
**dev_another_job_fixture)).remove_namespace(account),
ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(
**dev_with_infra_fixture)).remove_namespace(account),
]
resp_data = await resp.json()
self.assertEqual(expected_asgard_jobs[0], resp_data["jobs"][0])
self.assertEqual(expected_asgard_jobs[1], resp_data["jobs"][1])
async def test_convert_to_client_full_model(self, chronos_job_fixture):
"""
Confirma que os campos que são, na verdade, sub-modelos também são
incluídos na conversão.
"""
chronos_job_original = ChronosJob(**chronos_job_fixture)
asgard_job = ChronosScheduledJobConverter.to_asgard_model(
chronos_job_original)
chronos_job_converted = ChronosScheduledJobConverter.to_client_model(
asgard_job)
self.assertEqual(chronos_job_original.dict(),
chronos_job_converted.dict())
async def test_does_not_include_auth_header_if_no_auth_provided(
self, dev_job_fixture
):
chronos_job = ChronosJob(**dev_job_fixture)
with aioresponses() as rsps:
rsps.get(
"http://chronos/v1/scheduler/job/job-id",
status=200,
payload=chronos_job.dict(),
)
resp = await self.client.get_job_by_id("job-id")
self.assertEqual(chronos_job.dict(), resp.dict())
self.assert_auth_header_present(
rsps, "get", "http://chronos/v1/scheduler/job/job-id", {}
)
async def test_list_jobs_do_not_include_jobs_from_alternate_account(
self, dev_job_fixture, infra_job_fixture):
"""
Valida o parametro ?account_id=
"""
await _load_jobs_into_chronos(dev_job_fixture, infra_job_fixture)
account = Account(**ACCOUNT_INFRA_DICT)
resp = await self.client.get(
"/jobs",
headers={
"Authorization":
f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}"
},
params={"account_id": ACCOUNT_INFRA_ID},
)
self.assertEqual(HTTPStatus.OK, resp.status)
expected_asgard_jobs = [
ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**infra_job_fixture)).remove_namespace(account)
]
resp_data = await resp.json()
self.assertEqual(
ScheduledJobsListResource(jobs=expected_asgard_jobs).dict(),
resp_data,
)
async def test_delete_job_job_exist(self, dev_job_fixture):
await _load_jobs_into_chronos(dev_job_fixture)
asgard_job = ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**dev_job_fixture)).remove_namespace(self.account)
resp = await self.client.delete(
f"/jobs/{asgard_job.id}",
headers={
"Authorization":
f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}"
},
)
self.assertEqual(HTTPStatus.OK, resp.status)
resp_data = await resp.json()
self.assertEqual(
ScheduledJobResource(job=asgard_job).dict(), resp_data)
resp = await self.client.get(
f"/jobs/{asgard_job.id}",
headers={
"Authorization":
f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}"
},
)
self.assertEqual(HTTPStatus.NOT_FOUND, resp.status)
async def test_update_job_job_exist(self, dev_job_fixture):
"""
Conferimos que um job é atualizado corretamente
"""
await _load_jobs_into_chronos(dev_job_fixture)
asgard_job = ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**dev_job_fixture))
asgard_job.remove_namespace(self.account)
self.assertEqual(asgard_job.cpus, dev_job_fixture["cpus"])
self.assertEqual(asgard_job.mem, dev_job_fixture["mem"])
asgard_job.cpus = 2
asgard_job.mem = 2048
resp = await self.client.put(
f"/jobs/{asgard_job.id}",
headers={
"Authorization":
f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}"
},
json=asgard_job.dict(),
)
self.assertEqual(HTTPStatus.ACCEPTED, resp.status)
updated_job_response = await self.client.get(
f"/jobs/{asgard_job.id}",
headers={
"Authorization":
f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}"
},
)
updated_job_data = await updated_job_response.json()
updated_job_resource = CreateScheduledJobResource(**updated_job_data)
self.assertEqual(asgard_job.cpus, updated_job_resource.job.cpus)
self.assertEqual(asgard_job.mem, updated_job_resource.job.mem)
async def test_create_job_validation_error(self, infra_job_fixture):
"""
Validamos que retornamos HTTPStatus.UNPROCESSABLE_ENTITY caso a entrada esteja incompleta
"""
account = Account(**ACCOUNT_DEV_DICT)
asgard_job_no_namespace = ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**infra_job_fixture)).remove_namespace(account)
incomplete_asgard_job = asgard_job_no_namespace.dict()
del incomplete_asgard_job["container"]
resp = await self.client.post(
"/jobs",
headers={
"Authorization":
f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}"
},
json=incomplete_asgard_job,
)
self.assertEqual(HTTPStatus.UNPROCESSABLE_ENTITY, resp.status)
resp_data = await resp.json()
expected_error_msg = """1 validation error for ScheduledJob\ncontainer\n field required (type=value_error.missing)"""
self.assertEqual(
ErrorResource(errors=[ErrorDetail(msg=expected_error_msg)]).dict(),
resp_data,
)
async def test_create_job_name_has_namespace_from_another_account(
self, infra_job_fixture):
await _cleanup_chronos()
account = Account(**ACCOUNT_DEV_DICT)
asgard_job_no_namespace = ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**infra_job_fixture)).remove_namespace(account)
resp = await self.client.post(
"/jobs",
headers={
"Authorization":
f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}"
},
json=asgard_job_no_namespace.dict(),
)
self.assertEqual(HTTPStatus.CREATED, resp.status)
resp_data = await resp.json()
self.assertEqual(
f"{asgard_job_no_namespace.id}",
CreateScheduledJobResource(**resp_data).job.id,
)
await asyncio.sleep(0.3)
resp_created_job = await self.client.get(
f"/jobs/{asgard_job_no_namespace.id}",
headers={
"Authorization":
f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}"
},
)
self.assertEqual(HTTPStatus.OK, resp_created_job.status)
async def test_convert_to_asgard_model_enabled_field(
self, chronos_job_fixture):
"""
O Campo orignal no chronos é "disabled". Como nosso campo é
"enabled", os valores devem ser invertidos no momento da conversão
dos modelos
"""
chronos_job = ChronosJob(**chronos_job_fixture)
asgard_scheduled_job = ChronosScheduledJobConverter.to_asgard_model(
chronos_job)
self.assertTrue(asgard_scheduled_job.enabled)
chronos_job.disabled = True
asgard_scheduled_job = ChronosScheduledJobConverter.to_asgard_model(
chronos_job)
self.assertFalse(asgard_scheduled_job.enabled)
async def test_use_auth_data_on_create(self, dev_job_fixture):
url = "http://chronos/v1/scheduler/iso8601"
job = ChronosJob(**dev_job_fixture)
with aioresponses() as rsps:
rsps.post(url, status=200, payload={})
await self.client_with_auth.create_job(job)
self.assert_auth_header_present(rsps, "post", url, self.auth_header)
async def test_jobs_get_by_id_job_exist(self, chronos_job_fixture):
chronos_job_fixture["name"] = f"{self.account.namespace}-my-job"
async with http_client as client:
await client.post(
f"{settings.SCHEDULED_JOBS_SERVICE_ADDRESS}/v1/scheduler/iso8601",
json=chronos_job_fixture,
)
# Para dar tempo do chronos registra e responder no request log abaixo
await asyncio.sleep(1)
asgard_job = ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**chronos_job_fixture))
# A busca deve ser feita sempre *sem* o namespace
asgard_job.remove_namespace(self.account)
resp = await self.client.get(
f"/jobs/{asgard_job.id}",
headers={
"Authorization":
f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}"
},
)
self.assertEqual(HTTPStatus.OK, resp.status)
resp_data = await resp.json()
self.assertEqual(
ScheduledJobResource(job=asgard_job).dict(), resp_data)
async def test_update_job_add_default_fetch_uri(self):
del self.chronos_dev_job_fixture["fetch"]
new_fetch_uri = FetchURLSpec(
uri="https://static.server.com/assets/main.css")
expected_fetch_list = [
new_fetch_uri,
FetchURLSpec(
uri=settings.SCHEDULED_JOBS_DEFAULT_FETCH_URIS[0].uri),
FetchURLSpec(
uri=settings.SCHEDULED_JOBS_DEFAULT_FETCH_URIS[1].uri),
]
await _load_jobs_into_chronos(self.chronos_dev_job_fixture)
asgard_job = ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**self.chronos_dev_job_fixture))
asgard_job.add_fetch_uri(new_fetch_uri)
asgard_job.remove_namespace(self.account)
await self.backend.update_job(asgard_job, self.user, self.account)
stored_job = await self.backend.get_job_by_id(asgard_job.id, self.user,
self.account)
self.assertEqual(expected_fetch_list, stored_job.fetch)
async def test_create_job_on_alternate_account(self, dev_job_fixture):
"""
Confirmar que podemos fazer POST /jobs?account_id=<id>
o o job será criado com o namespace da account de id = <id>
"""
await _cleanup_chronos()
resp = await self.client.post(
"/jobs",
headers={
"Authorization":
f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}"
},
params={"account_id": ACCOUNT_INFRA_ID},
json=ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**dev_job_fixture)).dict(),
)
self.assertEqual(HTTPStatus.CREATED, resp.status)
resp_data = await resp.json()
self.assertEqual(
f"{dev_job_fixture['name']}",
CreateScheduledJobResource(**resp_data).job.id,
)
resp_created_job = await self.client.get(
f"/jobs/{dev_job_fixture['name']}",
headers={
"Authorization":
f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}"
},
params={"account_id": ACCOUNT_INFRA_ID},
)
self.assertEqual(HTTPStatus.OK, resp_created_job.status)
def to_client_model(cls, other: ScheduledJob) -> ChronosJob:
env_list = None
fetch_list = None
constraints_list = None
if other.env:
env_list = ChronosEnvSpecConverter.to_client_model(other.env)
if other.fetch:
fetch_list = ChronosFetchURLSpecConverter.to_client_model(
other.fetch)
if other.constraints:
constraints_list = ChronosConstraintSpecConverter.to_client_model(
other.constraints)
return ChronosJob(
name=other.id,
command=other.command,
arguments=other.arguments,
description=other.description,
cpus=other.cpus,
shell=other.shell,
retires=other.retries,
disabled=not other.enabled,
concurrent=other.concurrent,
mem=other.mem,
disk=other.disk,
schedule=other.schedule.value,
scheduleTimeZone=other.schedule.tz,
container=ChronosContainerSpecConverter.to_client_model(
other.container),
environmentVariables=env_list,
fetch=fetch_list,
constraints=constraints_list,
)
async def test_convert_to_asgard_model_constraints_field(
self, chronos_job_fixture):
chronos_job = ChronosJob(**chronos_job_fixture)
asgard_job = ChronosScheduledJobConverter.to_asgard_model(chronos_job)
self.assertEqual(
["hostname:LIKE:10.0.0.1", "workload:LIKE:general"],
asgard_job.constraints,
)
async def test_delete_job_job_does_not_exist(self):
await _cleanup_chronos()
job_not_found = ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**self.chronos_dev_job_fixture))
job_not_found.id = "this-job-does-not-exist"
with self.assertRaises(NotFoundEntity):
await self.backend.delete_job(job_not_found, self.user,
self.account)
async def test_to_client_model_retries_field(self, chronos_job_fixture):
chronos_job = ChronosJob(**chronos_job_fixture)
asgard_job = ChronosScheduledJobConverter.to_asgard_model(chronos_job)
self.assertEqual(chronos_job_fixture["retries"], asgard_job.retries)
asgard_job.retries = 4
chronos_job_converted = ChronosScheduledJobConverter.to_client_model(
asgard_job)
self.assertEqual(asgard_job.retries, chronos_job_converted.retries)
async def test_to_client_model_disabled_field(self, chronos_job_fixture):
asgard_job = ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**chronos_job_fixture))
self.assertFalse(
ChronosScheduledJobConverter.to_client_model(asgard_job).disabled)
asgard_job.enabled = False
self.assertTrue(
ChronosScheduledJobConverter.to_client_model(asgard_job).disabled)
async def test_use_auth_data_on_delete(self, dev_job_fixture):
job = ChronosJob(**dev_job_fixture)
url = f"http://chronos/v1/scheduler/job/{dev_job_fixture['name']}"
with aioresponses() as rsps:
rsps.delete(url, status=200, payload={})
await self.client_with_auth.delete_job(job)
self.assert_auth_header_present(
rsps, "delete", url, self.auth_header
)
async def create_job(self, job: ChronosJob) -> ChronosJob:
"""
O Chronos, pelo menos até a versão v3.0.2, tem um problema com jobs que usam timezone diferente de UTC.
Quando colocamos, por exemplo, tz=America/Sao_Paulo o jobs fica programado para a hora certa, mas quando o momento
chega o job fica com status OVERDUE mas *não roda*, nem aparece nos logs a tentativa de rodar o jobs.
"""
await self._request("post",
f"{self.base_url}/iso8601",
json=job.dict())
return job
async def test_list_jobs_no_not_include_jobs_from_other_namespaces(
self, infra_job_fixture, dev_job_fixture):
await _load_jobs_into_chronos(infra_job_fixture, dev_job_fixture)
user = User(**USER_WITH_MULTIPLE_ACCOUNTS_DICT)
account = Account(**ACCOUNT_DEV_DICT)
jobs = await self.backend.list_jobs(user, account)
expected_asgard_job = ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**dev_job_fixture)).remove_namespace(account)
self.assertCountEqual([expected_asgard_job], jobs)
async def get_job_by_id(self, job_id: str) -> ChronosJob:
async with http_client as client:
resp = await client.get(f"{self.address}/v1/scheduler/job/{job_id}"
)
if resp.status == HTTPStatus.BAD_REQUEST:
# `/job/{name}` retorna 400 se o job não existe.
# Isso acontece por causa dessa linha:
# https://github.com/mesosphere/chronos/blob/7eff5e0e2d666a94bf240608a05afcbad5f2235f/src/main/scala/org/apache/mesos/chronos/scheduler/api/JobManagementResource.scala#L51
raise Http404()
data = await resp.json()
return ChronosJob(**data)
async def test_convert_to_asgard_model_schedule_field(
self, chronos_job_fixture):
chronos_job = ChronosJob(**chronos_job_fixture)
asgard_job = ChronosScheduledJobConverter.to_asgard_model(chronos_job)
self.assertEqual(
{
"value": chronos_job_fixture["schedule"],
"tz": chronos_job_fixture["scheduleTimeZone"],
},
asgard_job.schedule.dict(),
)
async def setUp(self):
self.backend = ChronosScheduledJobsBackend()
self.chronos_dev_job_fixture = get_fixture(
"scheduled-jobs/chronos/dev-with-infra-in-name.json")
self.asgard_job = ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**self.chronos_dev_job_fixture))
self.user = User(**USER_WITH_MULTIPLE_ACCOUNTS_DICT)
self.account = Account(**ACCOUNT_DEV_DICT)
async def test_to_asgard_model_required_fields(self, chronos_job_fixture):
del chronos_job_fixture["environmentVariables"]
del chronos_job_fixture["constraints"]
del chronos_job_fixture["fetch"]
asgard_job = ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**chronos_job_fixture))
asgard_job_converted = ChronosScheduledJobConverter.to_asgard_model(
ChronosScheduledJobConverter.to_client_model(asgard_job))
self.assertEqual(asgard_job_converted.dict(), asgard_job.dict())
async def search(self, name: str) -> List[ChronosJob]:
"""
Procura por todos os jobs que contenham o termo `name` em seu nome.
"""
resp = await self._request(
"get",
f"{self.address}/v1/scheduler/jobs/search",
params={"name": name},
)
data = await resp.json()
jobs = [ChronosJob(**job) for job in data]
return jobs
async def test_convert_to_asgard_model_env_field(self,
chronos_job_fixture):
chronos_job = ChronosJob(**chronos_job_fixture)
asgard_job = ChronosScheduledJobConverter.to_asgard_model(chronos_job)
self.assertEqual(
{
"ENV_1": "VALUE_1",
"ENV_2": "VALUE_2",
"ENV_3": "VALUE_3"
},
asgard_job.dict()["env"],
)
async def test_create_job_add_internal_field_values(self, dev_job_fixture):
"""
Conferimos que quando um job é criado adicionamos os valores obrigatórios
de alguns campos
"""
await _cleanup_chronos()
account = Account(**ACCOUNT_DEV_DICT)
asgard_job_no_namespace = ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**dev_job_fixture)).remove_namespace(account)
resp = await self.client.post(
"/jobs",
headers={
"Authorization":
f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}"
},
json=asgard_job_no_namespace.dict(),
)
self.assertEqual(HTTPStatus.CREATED, resp.status)
resp_data = await resp.json()
self.assertEqual(
f"{asgard_job_no_namespace.id}",
CreateScheduledJobResource(**resp_data).job.id,
)
await asyncio.sleep(0.5)
resp_created_job = await self.client.get(
f"/jobs/{asgard_job_no_namespace.id}",
headers={
"Authorization":
f"Token {USER_WITH_MULTIPLE_ACCOUNTS_AUTH_KEY}"
},
)
created_job_resource = ScheduledJobResource(
**await resp_created_job.json())
self.assertEqual(HTTPStatus.OK, resp_created_job.status)
expected_constraints = asgard_job_no_namespace.constraints + [
f"owner:LIKE:{account.owner}"
]
self.assertEqual(created_job_resource.job.constraints,
expected_constraints)
expected_fetch_uris = asgard_job_no_namespace.fetch + [
FetchURLSpec(
uri=settings.SCHEDULED_JOBS_DEFAULT_FETCH_URIS[0].uri),
FetchURLSpec(
uri=settings.SCHEDULED_JOBS_DEFAULT_FETCH_URIS[1].uri),
]
self.assertEqual(created_job_resource.job.fetch, expected_fetch_uris)
async def test_to_client_model_required_fields(self, chronos_job_fixture):
asgard_job_dict = ChronosScheduledJobConverter.to_asgard_model(
ChronosJob(**chronos_job_fixture)).dict()
del asgard_job_dict["env"]
del asgard_job_dict["fetch"]
del asgard_job_dict["constraints"]
chronos_job = ChronosScheduledJobConverter.to_client_model(
ScheduledJob(**asgard_job_dict))
chronos_converted = ChronosScheduledJobConverter.to_client_model(
ChronosScheduledJobConverter.to_asgard_model(chronos_job))
self.assertEqual(chronos_converted.dict(), chronos_job.dict())
async def get_job_by_id(self, job_id: str) -> ChronosJob:
"""
Retorna um Job do Chronos, dado seu id (nome).
Raise asgard.http.exceptions.HTTPNotFound() se o job não existir
"""
try:
resp = await self._request(
"get", f"{self.address}/v1/scheduler/job/{job_id}")
except HTTPBadRequest as e:
# `/job/{name}` retorna 400 se o job não existe.
# Isso acontece por causa dessa linha:
# https://github.com/mesosphere/chronos/blob/7eff5e0e2d666a94bf240608a05afcbad5f2235f/src/main/scala/org/apache/mesos/chronos/scheduler/api/JobManagementResource.scala#L51
raise HTTPNotFound(request_info=e.request_info)
data = await resp.json()
return ChronosJob(**data)
