def test_batch_len():
batch = Batch()
assert len(batch) == 0
batch = Batch([])
assert len(batch) == 0
batch = Batch([Job.from_dict(SUCCEEDED_JOB), Job.from_dict(FAILED_JOB)])
assert len(batch) == 2
def test_setitem(get_mock_job):
unexpired_time = (datetime.now(tz=tz.UTC) + timedelta(days=7)).isoformat(timespec='seconds')
j0 = Job.from_dict(SUCCEEDED_JOB)
j1 = Job.from_dict(FAILED_JOB)
j2 = get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time,
files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}])
batch = Batch([j0, j1])
assert batch[1] == j1
batch[1] = j2
assert batch[1] == j2
def test_job_attributes():
job = Job.from_dict(SUCCEEDED_JOB)
for key in SUCCEEDED_JOB.keys():
assert job.__getattribute__(key)
job = Job.from_dict(FAILED_JOB)
for key in FAILED_JOB.keys():
assert job.__getattribute__(key)
unprovided_attributes = set(vars(job).keys()) - set(FAILED_JOB.keys())
for key in unprovided_attributes:
assert job.__getattribute__(key) is None
def test_contains(get_mock_job):
unexpired_time = (datetime.now(tz=tz.UTC) + timedelta(days=7)).isoformat(timespec='seconds')
j1 = Job.from_dict(SUCCEEDED_JOB)
j2 = Job.from_dict(FAILED_JOB)
j3 = get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time,
files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}])
a = Batch([j1, j2])
assert j1 in a
assert j2 in a
assert j3 not in a
def test_job_dict_transforms():
job = Job.from_dict(SUCCEEDED_JOB)
assert job.to_dict() == SUCCEEDED_JOB
retry = job.to_dict(for_resubmit=True)
assert retry.keys() == Job._attributes_for_resubmit
job = Job.from_dict(FAILED_JOB)
assert job.to_dict() == FAILED_JOB
retry = job.to_dict(for_resubmit=True)
assert retry.keys() == Job._attributes_for_resubmit
def test_batch_complete_succeeded():
batch = Batch([Job.from_dict(SUCCEEDED_JOB), Job.from_dict(SUCCEEDED_JOB)])
assert batch.complete()
assert batch.succeeded()
batch += Job.from_dict(FAILED_JOB)
assert batch.complete()
assert not batch.succeeded()
running = Job.from_dict(FAILED_JOB)
running.status_code = 'RUNNING'
batch += running
assert not batch.complete()
assert not batch.succeeded()
def test_batch_filter_jobs():
succeeded_job = Job.from_dict(SUCCEEDED_JOB)
succeeded_job.expiration_time = datetime.now(tz.UTC) + timedelta(days=7)
expired_job = Job.from_dict(SUCCEEDED_JOB)
expired_job.expiration_time = datetime.now(tz.UTC) - timedelta(days=7)
running_job = Job.from_dict(FAILED_JOB)
running_job.status_code = 'RUNNING'
pending_job = Job.from_dict(FAILED_JOB)
pending_job.status_code = 'PENDING'
batch = Batch([succeeded_job, running_job, expired_job, pending_job, Job.from_dict(FAILED_JOB)])
not_failed = batch.filter_jobs()
assert len(not_failed) == 4
assert not_failed.jobs[0].succeeded() and not not_failed.jobs[0].expired()
assert not_failed.jobs[1].running()
assert not_failed.jobs[2].succeeded() and not_failed.jobs[2].expired()
assert not_failed.jobs[3].running()
not_failed_or_expired = batch.filter_jobs(include_expired=False)
assert len(not_failed_or_expired) == 3
assert not_failed_or_expired.jobs[0].succeeded() and not not_failed_or_expired.jobs[0].expired()
assert not_failed_or_expired.jobs[1].running()
assert not_failed_or_expired.jobs[2].running()
succeeded = batch.filter_jobs(running=False)
assert len(succeeded) == 2
assert succeeded.jobs[0].succeeded() and not succeeded.jobs[0].expired()
assert succeeded.jobs[1].succeeded() and succeeded.jobs[1].expired()
running = batch.filter_jobs(succeeded=False)
assert len(running) == 2
assert running.jobs[0].running()
assert running.jobs[1].running()
failed = batch.filter_jobs(succeeded=False, running=False, failed=True)
assert len(failed) == 1
assert failed.jobs[0].failed()
everything = batch.filter_jobs(failed=True)
assert len(everything) == len(batch)
for ii, job in enumerate(everything.jobs):
assert job.status_code == batch.jobs[ii].status_code
if job.succeeded():
assert job.expired() == batch.jobs[ii].expired()
def test_batch_add():
a = Batch([Job.from_dict(SUCCEEDED_JOB)])
b = Batch([Job.from_dict(FAILED_JOB)])
j = Job.from_dict(SUCCEEDED_JOB)
j.status_code = 'RUNNING'
c = a + b
assert len(c) == 2
assert c.jobs[0].succeeded()
assert c.jobs[1].failed()
d = c + j
assert len(d) == 3
assert d.jobs[0].succeeded()
assert d.jobs[1].failed()
assert d.jobs[2].running()
def test_batch_iadd():
a = Batch([Job.from_dict(SUCCEEDED_JOB)])
b = Batch([Job.from_dict(FAILED_JOB)])
j = Job.from_dict(SUCCEEDED_JOB)
j.status_code = 'RUNNING'
a += b
assert len(a) == 2
assert a.jobs[0].succeeded()
assert a.jobs[1].failed()
a += j
assert len(a) == 3
assert a.jobs[0].succeeded()
assert a.jobs[1].failed()
assert a.jobs[2].running()
def test_delitem():
j0 = Job.from_dict(SUCCEEDED_JOB)
j1 = Job.from_dict(FAILED_JOB)
batch = Batch([j0, j1])
assert j0 in batch
assert j1 in batch
del batch[1]
assert j0 in batch
assert j1 not in batch
batch += j1
del batch[0]
assert j0 not in batch
assert j1 in batch
def test_job_expired():
job = Job.from_dict(SUCCEEDED_JOB)
job.expiration_time = datetime.now(tz.UTC) + timedelta(days=7)
assert not job.expired()
job.expiration_time = datetime.now(tz.UTC) - timedelta(days=7)
assert job.expired()
assert 'expiration_time' not in FAILED_JOB
job = Job.from_dict(FAILED_JOB)
assert job.expiration_time is None
assert not job.expired()
failed_job_with_expiration_time = FAILED_JOB.copy()
failed_job_with_expiration_time['expiration_time'] = None
job = Job.from_dict(failed_job_with_expiration_time)
assert job.expiration_time is None
assert not job.expired()
def test_batch_any_expired():
job1 = Job.from_dict(SUCCEEDED_JOB)
job1.expiration_time = datetime.now(tz.UTC) + timedelta(days=7)
job2 = copy(job1)
job2.expiration_time = datetime.now(tz.UTC) + timedelta(days=2)
batch = Batch([job1, job2])
assert not batch.any_expired()
# ignore jobs without expiration times
job3 = Job.from_dict(FAILED_JOB)
batch += job3
assert not batch.any_expired()
job4 = copy(job1)
job4.expiration_time = datetime.now(tz.UTC) - timedelta(days=2)
batch += job4
assert batch.any_expired()
def find_jobs(self,
start: Optional[datetime] = None,
end: Optional[datetime] = None,
status_code: Optional[str] = None,
name: Optional[str] = None,
job_type: Optional[str] = None) -> Batch:
"""Gets a Batch of jobs from HyP3 matching the provided search criteria
Args:
start: only jobs submitted after given time
end: only jobs submitted before given time
status_code: only jobs matching this status (SUCCEEDED, FAILED, RUNNING, PENDING)
name: only jobs with this name
job_type: only jobs with this job_type
Returns:
A Batch object containing the found jobs
"""
params = {}
for param_name in ('start', 'end', 'status_code', 'name', 'job_type'):
param_value = locals().get(param_name)
if param_value is not None:
if isinstance(param_value, datetime):
if param_value.tzinfo is None:
param_value = param_value.replace(tzinfo=timezone.utc)
param_value = param_value.isoformat(timespec='seconds')
params[param_name] = param_value
response = self.session.get(urljoin(self.url, '/jobs'), params=params)
_raise_for_hyp3_status(response)
jobs = [Job.from_dict(job) for job in response.json()['jobs']]
while 'next' in response.json():
next_url = response.json()['next']
response = self.session.get(next_url)
_raise_for_hyp3_status(response)
jobs.extend(
[Job.from_dict(job) for job in response.json()['jobs']])
return Batch(jobs)
def _watch_job(self,
job: Job,
timeout: int = 10800,
interval: Union[int, float] = 60) -> Job:
tqdm = get_tqdm_progress_bar()
iterations_until_timeout = math.ceil(timeout / interval)
bar_format = '{n_fmt}/{total_fmt} [{postfix[0]}]'
with tqdm(total=1,
bar_format=bar_format,
postfix=[f'timeout in {timeout} s']) as progress_bar:
for ii in range(iterations_until_timeout):
job = self.refresh(job)
progress_bar.postfix = [
f'timeout in {timeout - ii * interval}s'
]
progress_bar.update(int(job.complete()))
if job.complete():
return job
time.sleep(interval)
raise HyP3Error(f'Timeout occurred while waiting for {job}')
def get_job_by_id(self, job_id: str) -> Job:
"""Get job by job ID
Args:
job_id: A job ID
Returns:
A Job object
"""
response = self.session.get(urljoin(self.url, f'/jobs/{job_id}'))
_raise_for_hyp3_status(response)
return Job.from_dict(response.json())
def test_job_complete_succeeded_failed_running():
job = Job.from_dict(SUCCEEDED_JOB)
assert job.complete()
assert job.succeeded()
assert not job.failed()
assert not job.running()
job = Job.from_dict(FAILED_JOB)
assert job.complete()
assert not job.succeeded()
assert job.failed()
assert not job.running()
job.status_code = 'PENDING'
assert not job.complete()
assert not job.succeeded()
assert not job.failed()
assert job.running()
job.status_code = 'RUNNING'
assert not job.complete()
assert not job.succeeded()
assert not job.failed()
assert job.running()
def submit_prepared_jobs(self, prepared_jobs: Union[dict,
List[dict]]) -> Batch:
"""Submit a prepared job dictionary, or list of prepared job dictionaries
Args:
prepared_jobs: A prepared job dictionary, or list of prepared job dictionaries
Returns:
A Batch object containing the submitted job(s)
"""
if isinstance(prepared_jobs, dict):
payload = {'jobs': [prepared_jobs]}
else:
payload = {'jobs': prepared_jobs}
response = self.session.post(urljoin(self.url, '/jobs'), json=payload)
_raise_for_hyp3_status(response)
batch = Batch()
for job in response.json()['jobs']:
batch += Job.from_dict(job)
return batch
def test_batch_iter():
defined_jobs = [Job.from_dict(SUCCEEDED_JOB), Job.from_dict(FAILED_JOB)]
batch = Batch(defined_jobs)
for batch_job, defined_job in zip(batch, defined_jobs):
assert batch_job == defined_job
