def _watch_batch(self,
batch: Batch,
timeout: int = 10800,
interval: Union[int, float] = 60) -> Batch:
tqdm = get_tqdm_progress_bar()
iterations_until_timeout = math.ceil(timeout / interval)
bar_format = '{l_bar}{bar}| {n_fmt}/{total_fmt} [{postfix[0]}]'
with tqdm(total=len(batch),
bar_format=bar_format,
postfix=[f'timeout in {timeout} s']) as progress_bar:
for ii in range(iterations_until_timeout):
batch = self.refresh(batch)
counts = batch._count_statuses()
complete = counts['SUCCEEDED'] + counts['FAILED']
progress_bar.postfix = [
f'timeout in {timeout - ii * interval}s'
]
# to control n/total manually; update is n += value
progress_bar.n = complete
progress_bar.update(0)
if batch.complete():
return batch
time.sleep(interval)
raise HyP3Error(f'Timeout occurred while waiting for {batch}')
def test_batch_len():
batch = Batch()
assert len(batch) == 0
batch = Batch([])
assert len(batch) == 0
batch = Batch([Job.from_dict(SUCCEEDED_JOB), Job.from_dict(FAILED_JOB)])
assert len(batch) == 2
def test_getitem(get_mock_job):
unexpired_time = (datetime.now(tz=tz.UTC) + timedelta(days=7)).isoformat(timespec='seconds')
j0 = Job.from_dict(SUCCEEDED_JOB)
j1 = Job.from_dict(FAILED_JOB)
j2 = get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time,
files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}])
batch = Batch([j0, j1, j2])
assert j0 == batch[0]
assert j1 == batch[1]
assert j2 == batch[2]
assert Batch([j1, j2]) == batch[1:]
def test_batch_iadd():
a = Batch([Job.from_dict(SUCCEEDED_JOB)])
b = Batch([Job.from_dict(FAILED_JOB)])
j = Job.from_dict(SUCCEEDED_JOB)
j.status_code = 'RUNNING'
a += b
assert len(a) == 2
assert a.jobs[0].succeeded()
assert a.jobs[1].failed()
a += j
assert len(a) == 3
assert a.jobs[0].succeeded()
assert a.jobs[1].failed()
assert a.jobs[2].running()
def test_batch_add():
a = Batch([Job.from_dict(SUCCEEDED_JOB)])
b = Batch([Job.from_dict(FAILED_JOB)])
j = Job.from_dict(SUCCEEDED_JOB)
j.status_code = 'RUNNING'
c = a + b
assert len(c) == 2
assert c.jobs[0].succeeded()
assert c.jobs[1].failed()
d = c + j
assert len(d) == 3
assert d.jobs[0].succeeded()
assert d.jobs[1].failed()
assert d.jobs[2].running()
def test_batch_any_expired():
job1 = Job.from_dict(SUCCEEDED_JOB)
job1.expiration_time = datetime.now(tz.UTC) + timedelta(days=7)
job2 = copy(job1)
job2.expiration_time = datetime.now(tz.UTC) + timedelta(days=2)
batch = Batch([job1, job2])
assert not batch.any_expired()
# ignore jobs without expiration times
job3 = Job.from_dict(FAILED_JOB)
batch += job3
assert not batch.any_expired()
job4 = copy(job1)
job4.expiration_time = datetime.now(tz.UTC) - timedelta(days=2)
batch += job4
assert batch.any_expired()
def test_batch_download_expired(tmp_path, get_mock_job):
expired_time = (datetime.now(tz=tz.UTC) - timedelta(days=7)).isoformat(timespec='seconds')
unexpired_time = (datetime.now(tz=tz.UTC) + timedelta(days=7)).isoformat(timespec='seconds')
batch = Batch([
get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time,
files=[{'url': 'https://foo.com/file1', 'size': 0, 'filename': 'file1'}]),
get_mock_job(status_code='SUCCEEDED', expiration_time=expired_time,
files=[{'url': 'https://foo.com/file2', 'size': 0, 'filename': 'file2'}]),
get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time,
files=[{'url': 'https://foo.com/file3', 'size': 0, 'filename': 'file3'}])
])
responses.add(responses.GET, 'https://foo.com/file1', body='foobar1')
responses.add(responses.GET, 'https://foo.com/file2', body='foobar2')
responses.add(responses.GET, 'https://foo.com/file3', body='foobar3')
paths = batch.download_files(tmp_path)
contents = [path.read_text() for path in paths]
assert len(paths) == 2
assert set(paths) == {tmp_path / 'file1', tmp_path / 'file3'}
assert set(contents) == {'foobar1', 'foobar3'}
def test_contains(get_mock_job):
unexpired_time = (datetime.now(tz=tz.UTC) + timedelta(days=7)).isoformat(timespec='seconds')
j1 = Job.from_dict(SUCCEEDED_JOB)
j2 = Job.from_dict(FAILED_JOB)
j3 = get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time,
files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}])
a = Batch([j1, j2])
assert j1 in a
assert j2 in a
assert j3 not in a
def test_batch_complete_succeeded():
batch = Batch([Job.from_dict(SUCCEEDED_JOB), Job.from_dict(SUCCEEDED_JOB)])
assert batch.complete()
assert batch.succeeded()
batch += Job.from_dict(FAILED_JOB)
assert batch.complete()
assert not batch.succeeded()
running = Job.from_dict(FAILED_JOB)
running.status_code = 'RUNNING'
batch += running
assert not batch.complete()
assert not batch.succeeded()
def test_reverse(get_mock_job):
unexpired_time = (datetime.now(tz=tz.UTC) + timedelta(days=7)).isoformat(timespec='seconds')
j0 = Job.from_dict(SUCCEEDED_JOB)
j1 = Job.from_dict(FAILED_JOB)
j2 = get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time,
files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}])
batch = Batch([j0, j1, j2])
batch_reversed = reversed(batch)
assert next(batch_reversed) == j2
assert next(batch_reversed) == j1
assert next(batch_reversed) == j0
def test_batch_filter_jobs():
succeeded_job = Job.from_dict(SUCCEEDED_JOB)
succeeded_job.expiration_time = datetime.now(tz.UTC) + timedelta(days=7)
expired_job = Job.from_dict(SUCCEEDED_JOB)
expired_job.expiration_time = datetime.now(tz.UTC) - timedelta(days=7)
running_job = Job.from_dict(FAILED_JOB)
running_job.status_code = 'RUNNING'
pending_job = Job.from_dict(FAILED_JOB)
pending_job.status_code = 'PENDING'
batch = Batch([succeeded_job, running_job, expired_job, pending_job, Job.from_dict(FAILED_JOB)])
not_failed = batch.filter_jobs()
assert len(not_failed) == 4
assert not_failed.jobs[0].succeeded() and not not_failed.jobs[0].expired()
assert not_failed.jobs[1].running()
assert not_failed.jobs[2].succeeded() and not_failed.jobs[2].expired()
assert not_failed.jobs[3].running()
not_failed_or_expired = batch.filter_jobs(include_expired=False)
assert len(not_failed_or_expired) == 3
assert not_failed_or_expired.jobs[0].succeeded() and not not_failed_or_expired.jobs[0].expired()
assert not_failed_or_expired.jobs[1].running()
assert not_failed_or_expired.jobs[2].running()
succeeded = batch.filter_jobs(running=False)
assert len(succeeded) == 2
assert succeeded.jobs[0].succeeded() and not succeeded.jobs[0].expired()
assert succeeded.jobs[1].succeeded() and succeeded.jobs[1].expired()
running = batch.filter_jobs(succeeded=False)
assert len(running) == 2
assert running.jobs[0].running()
assert running.jobs[1].running()
failed = batch.filter_jobs(succeeded=False, running=False, failed=True)
assert len(failed) == 1
assert failed.jobs[0].failed()
everything = batch.filter_jobs(failed=True)
assert len(everything) == len(batch)
for ii, job in enumerate(everything.jobs):
assert job.status_code == batch.jobs[ii].status_code
if job.succeeded():
assert job.expired() == batch.jobs[ii].expired()
def test_delitem():
j0 = Job.from_dict(SUCCEEDED_JOB)
j1 = Job.from_dict(FAILED_JOB)
batch = Batch([j0, j1])
assert j0 in batch
assert j1 in batch
del batch[1]
assert j0 in batch
assert j1 not in batch
batch += j1
del batch[0]
assert j0 not in batch
assert j1 in batch
def find_jobs(self,
start: Optional[datetime] = None,
end: Optional[datetime] = None,
status_code: Optional[str] = None,
name: Optional[str] = None,
job_type: Optional[str] = None) -> Batch:
"""Gets a Batch of jobs from HyP3 matching the provided search criteria
Args:
start: only jobs submitted after given time
end: only jobs submitted before given time
status_code: only jobs matching this status (SUCCEEDED, FAILED, RUNNING, PENDING)
name: only jobs with this name
job_type: only jobs with this job_type
Returns:
A Batch object containing the found jobs
"""
params = {}
for param_name in ('start', 'end', 'status_code', 'name', 'job_type'):
param_value = locals().get(param_name)
if param_value is not None:
if isinstance(param_value, datetime):
if param_value.tzinfo is None:
param_value = param_value.replace(tzinfo=timezone.utc)
param_value = param_value.isoformat(timespec='seconds')
params[param_name] = param_value
response = self.session.get(urljoin(self.url, '/jobs'), params=params)
_raise_for_hyp3_status(response)
jobs = [Job.from_dict(job) for job in response.json()['jobs']]
while 'next' in response.json():
next_url = response.json()['next']
response = self.session.get(next_url)
_raise_for_hyp3_status(response)
jobs.extend(
[Job.from_dict(job) for job in response.json()['jobs']])
return Batch(jobs)
def submit_prepared_jobs(self, prepared_jobs: Union[dict,
List[dict]]) -> Batch:
"""Submit a prepared job dictionary, or list of prepared job dictionaries
Args:
prepared_jobs: A prepared job dictionary, or list of prepared job dictionaries
Returns:
A Batch object containing the submitted job(s)
"""
if isinstance(prepared_jobs, dict):
payload = {'jobs': [prepared_jobs]}
else:
payload = {'jobs': prepared_jobs}
response = self.session.post(urljoin(self.url, '/jobs'), json=payload)
_raise_for_hyp3_status(response)
batch = Batch()
for job in response.json()['jobs']:
batch += Job.from_dict(job)
return batch
def _refresh_batch(self, batch: Batch):
jobs = []
for job in batch.jobs:
jobs.append(self.refresh(job))
return Batch(jobs)
def test_batch_iter():
defined_jobs = [Job.from_dict(SUCCEEDED_JOB), Job.from_dict(FAILED_JOB)]
batch = Batch(defined_jobs)
for batch_job, defined_job in zip(batch, defined_jobs):
assert batch_job == defined_job