async def check_rename_directory_on_storage(self, path_from: str,
path_to: str) -> None:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
await client.storage.mv(URL(f"{self.tmpstorage}{path_from}"),
URL(f"{self.tmpstorage}{path_to}"))
async def check_blob_size(self, bucket_name: str, key: str,
size: int) -> None:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
blob = await client.blob_storage.head_blob(bucket_name, key)
assert blob.size == size
async def kill_job(self, id_or_name: str) -> None:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
id = await resolve_job(id_or_name, client=client)
with suppress(ResourceNotFound, IllegalArgumentError):
await client.jobs.kill(id)
async def test_port_forward(nmrc_path: Path, nginx_job_async: Tuple[str, str]) -> None:
loop_sleep = 1
service_wait_time = 10 * 60
async def get_(url: str) -> int:
status = 999
start_time = time()
async with aiohttp.ClientSession() as session:
while status != 200 and (int(time() - start_time) < service_wait_time):
try:
async with session.get(url) as resp:
status = resp.status
text = await resp.text()
assert text == nginx_job_async[1], (
f"Secret not found "
f"via {url}. Like as it's not our test server."
)
except aiohttp.ClientConnectionError:
status = 599
if status != 200:
await asyncio.sleep(loop_sleep)
return status
async with api_get(path=nmrc_path, timeout=CLIENT_TIMEOUT) as client:
port = unused_port()
# We test client instead of run_cli as asyncio subprocesses do
# not work if run from thread other than main.
async with client.jobs.port_forward(
nginx_job_async[0], port, 80, no_key_check=True
):
await asyncio.sleep(loop_sleep)
url = f"http://127.0.0.1:{port}/secret.txt"
probe = await get_(url)
assert probe == 200
async def nginx_job_async(
nmrc_path: Path, loop: asyncio.AbstractEventLoop
) -> AsyncIterator[Tuple[str, str]]:
async with api_get(path=nmrc_path) as client:
secret = uuid4()
command = (
f"bash -c \"echo -n '{secret}' > /usr/share/nginx/html/secret.txt; "
f"timeout 15m /usr/sbin/nginx -g 'daemon off;'\""
)
container = Container(
image=RemoteImage.new_external_image(name="nginx", tag="latest"),
command=command,
resources=Resources(20, 0.1, None, None, True, None, None),
)
job = await client.jobs.run(
container, is_preemptible=False, description="test NGINX job"
)
try:
for i in range(60):
status = await client.jobs.status(job.id)
if status.status == JobStatus.RUNNING:
break
await asyncio.sleep(1)
else:
raise AssertionError("Cannot start NGINX job")
yield job.id, str(secret)
finally:
with suppress(Exception):
await client.jobs.kill(job.id)
async def check_file_absent_on_storage(self, name: str, path: str) -> None:
__tracebackhide__ = True
url = URL(self.tmpstorage + path)
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
async for file in client.storage.ls(url):
if file.type == FileStatusType.FILE and file.path == name:
raise AssertionError(f"File {name} found in {url}")
async def upload_blob(self, bucket_name: str, key: str,
file: Path) -> None:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
await client.blob_storage.upload_file(
URL("file:" + str(file)),
client.blob_storage.make_url(bucket_name, key))
async def check_dir_absent_on_storage(self, name: str, path: str) -> None:
__tracebackhide__ = True
url = URL(self.tmpstorage + path)
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
files = await client.storage.ls(url)
for file in files:
if file.type == FileStatusType.DIRECTORY and file.path == name:
raise AssertionError(f"Dir {name} found in {url}")
async def check_rm_file_on_storage(self,
name: str,
path: str,
*,
fromhome: bool = False) -> None:
__tracebackhide__ = True
url = self.make_uri(path, fromhome=fromhome)
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
await client.storage.rm(url / name)
async def check_blob_checksum(self, bucket_name: str, key: str,
checksum: str, tmp_path: Path) -> None:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
await client.blob_storage.download_file(
client.blob_storage.make_url(bucket_name, key),
URL("file:" + str(tmp_path)),
)
assert self.hash_hex(
tmp_path) == checksum, "checksum test failed for {url}"
async def _get_storage_cookie(nmrc_path: Optional[Path]) -> None:
async with api_get(timeout=CLIENT_TIMEOUT, path=nmrc_path) as client:
await client.storage.ls(URL("storage:/"))
cookie = client._get_session_cookie()
if cookie is not None:
new_config = dataclasses.replace(
client._config,
cookie_session=_CookieSession(cookie=cookie.value,
timestamp=int(time())),
)
Factory(nmrc_path)._save(new_config)
async def drop_stale_buckets(self, bucket_prefix: str) -> None:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
buckets = await client.blob_storage.list_buckets()
for bucket in buckets:
if (bucket.name.startswith(bucket_prefix)
and bucket.creation_time < time() - 3600 * 4 and
bucket.permission in (Action.WRITE, Action.MANAGE)):
await self.acleanup_bucket(bucket.name)
await self.adelete_bucket(bucket.name)
async def check_upload_file_to_storage(self, name: str, path: str,
local_file: str) -> None:
__tracebackhide__ = True
url = URL(self.tmpstorage + path)
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
if name is None:
await client.storage.upload_file(URL("file:" + local_file),
url)
else:
await client.storage.upload_file(URL("file:" + local_file),
URL(f"{url}/{name}"))
async def resolve_job_name_to_id(self, job_name: str) -> str:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
return await resolve_job(
job_name,
client=client,
status={
JobStatus.PENDING,
JobStatus.RUNNING,
JobStatus.SUCCEEDED,
JobStatus.FAILED,
},
)
async def drop_stale_buckets(self, bucket_prefix: str) -> None:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
buckets = await client.blob_storage.list_buckets()
for bucket in buckets:
if (bucket.name.startswith(bucket_prefix)
and bucket.creation_time < time() - 3600 * 4 and
bucket.permission in (Action.WRITE, Action.MANAGE)):
with suppress(ResourceNotFound):
# bucket can be deleted by another parallel test run,
# ignore ResourceNotFound errors
await self.acleanup_bucket(bucket.name)
await self.adelete_bucket(bucket.name)
async def job_info(self,
job_id: str,
wait_start: bool = False) -> JobDescription:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
job = await client.jobs.status(job_id)
start_time = time()
while (wait_start and job.status == JobStatus.PENDING
and time() - start_time < JOB_TIMEOUT):
job = await client.jobs.status(job_id)
if int(time() - start_time) > JOB_TIMEOUT:
raise AssertionError(
f"timeout exceeded, last output: '{job.status}'")
return job
async def check_file_exists_on_storage(self,
name: str,
path: str,
size: int,
*,
fromhome: bool = False) -> None:
__tracebackhide__ = True
url = self.make_uri(path, fromhome=fromhome)
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
async for file in client.storage.ls(url):
if (file.type == FileStatusType.FILE and file.name == name
and file.size == size):
return
raise AssertionError(
f"File {name} with size {size} not found in {url}")
async def akill_job(self, id_or_name: str, *, wait: bool = True) -> None:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
id = await resolve_job(
id_or_name,
client=client,
status={JobStatus.PENDING, JobStatus.RUNNING})
with suppress(ResourceNotFound, IllegalArgumentError):
await client.jobs.kill(id)
if wait:
while True:
stat = await client.jobs.status(id)
if stat.status not in (JobStatus.PENDING,
JobStatus.RUNNING):
break
async def acreate_bucket(self, name: str, *, wait: bool = False) -> None:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
await client.blob_storage.create_bucket(name)
if wait:
t0 = time()
delay = 1
while time() - t0 < 30:
try:
await client.blob_storage.list_blobs(name, max_keys=10)
return
except ResourceNotFound:
delay = min(delay * 2, 15)
await asyncio.sleep(delay)
raise RuntimeError(
f"Bucket {name} doesn't exist after the creation")
async def wait_job_change_state_from(
self,
job_id: str,
wait_state: JobStatus,
stop_state: Optional[JobStatus] = None) -> None:
__tracebackhide__ = True
start_time = time()
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
job = await client.jobs.status(job_id)
while job.status == wait_state and (int(time() - start_time) <
JOB_TIMEOUT):
if stop_state == job.status:
raise JobWaitStateStopReached(
f"failed running job {job_id}: {stop_state}")
await asyncio.sleep(JOB_WAIT_SLEEP_SECONDS)
job = await client.jobs.status(job_id)
async def check_file_on_storage_checksum(self, name: str, path: str,
checksum: str, tmpdir: str,
tmpname: str) -> None:
__tracebackhide__ = True
url = URL(self.tmpstorage + path)
if tmpname:
target = join(tmpdir, tmpname)
target_file = target
else:
target = tmpdir
target_file = join(tmpdir, name)
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
await client.storage.download_file(url / name,
URL("file:" + target))
assert (self.hash_hex(target_file) == checksum
), "checksum test failed for {url}"
async def check_rename_file_on_storage(self, name_from: str,
path_from: str, name_to: str,
path_to: str) -> None:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
await client.storage.mv(
URL(f"{self.tmpstorage}{path_from}/{name_from}"),
URL(f"{self.tmpstorage}{path_to}/{name_to}"),
)
files1 = await client.storage.ls(
URL(f"{self.tmpstorage}{path_from}"))
names1 = {f.name for f in files1}
assert name_from not in names1
files2 = await client.storage.ls(URL(f"{self.tmpstorage}{path_to}")
)
names2 = {f.name for f in files2}
assert name_to in names2
async def acleanup_bucket(self, bucket_name: str) -> None:
__tracebackhide__ = True
# Each test needs a clean bucket state and we can't delete bucket until it's
# cleaned
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
blobs, _ = await client.blob_storage.list_blobs(bucket_name,
recursive=True)
if not blobs:
return
# XXX: We do assume we will not have tests that run 10000 of objects. If we
# do, please add a semaphore here.
tasks = []
for blob in blobs:
log.info("Removing %s %s", bucket_name, blob.key)
tasks.append(
client.blob_storage.delete_blob(bucket_name, key=blob.key))
await asyncio.gather(*tasks)
async def wait_job_change_state_to(
self,
job_id: str,
target_state: JobStatus,
stop_state: Optional[JobStatus] = None,
) -> None:
__tracebackhide__ = True
start_time = time()
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
job = await client.jobs.status(job_id)
while target_state != job.status:
if stop_state and stop_state == job.status:
raise JobWaitStateStopReached(
f"failed running job {job_id}: '{stop_state}'")
if int(time() - start_time) > JOB_TIMEOUT:
raise AssertionError(
f"timeout exceeded, last output: '{job.status}'")
await asyncio.sleep(JOB_WAIT_SLEEP_SECONDS)
job = await client.jobs.status(job_id)
async def arun_job_and_wait_state(
self,
image: str,
command: str = "",
*,
description: Optional[str] = None,
name: Optional[str] = None,
tty: bool = False,
wait_state: JobStatus = JobStatus.RUNNING,
stop_state: JobStatus = JobStatus.FAILED,
) -> str:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
preset = client.presets["cpu-micro"]
resources = Resources(memory_mb=preset.memory_mb, cpu=preset.cpu)
container = Container(
image=client.parse.remote_image(image),
command=command,
resources=resources,
tty=tty,
)
job = await client.jobs.run(
container,
is_preemptible=preset.is_preemptible,
description=description,
name=name,
)
start_time = time()
while job.status != wait_state:
if stop_state == job.status:
raise JobWaitStateStopReached(
f"failed running job {job.id}: {stop_state}")
if int(time() - start_time) > JOB_TIMEOUT:
raise AssertionError(
f"timeout exceeded, last output: '{job.status}'")
await asyncio.sleep(JOB_WAIT_SLEEP_SECONDS)
job = await client.jobs.status(job.id)
return job.id
async def _check_job_output() -> AsyncIterator[bytes]:
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
async for chunk in client.jobs.monitor(job_id):
yield chunk
async def assert_job_state(self, job_id: str, state: JobStatus) -> None:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
job = await client.jobs.status(job_id)
assert job.status == state
async def mkdir(self, path: str, **kwargs: bool) -> None:
__tracebackhide__ = True
url = URL(self.tmpstorage + path)
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
await client.storage.mkdir(url, **kwargs)
async def rm(self, path: str, *, recursive: bool = False) -> None:
__tracebackhide__ = True
url = URL(self.tmpstorage + path)
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
await client.storage.rm(url, recursive=recursive)
async def resolve_job_name_to_id(self, job_name: str) -> str:
__tracebackhide__ = True
async with api_get(timeout=CLIENT_TIMEOUT,
path=self._nmrc_path) as client:
return await resolve_job(job_name, client=client)
