def _watch_batch(self, batch: Batch, timeout: int = 10800, interval: Union[int, float] = 60) -> Batch: tqdm = get_tqdm_progress_bar() iterations_until_timeout = math.ceil(timeout / interval) bar_format = '{l_bar}{bar}| {n_fmt}/{total_fmt} [{postfix[0]}]' with tqdm(total=len(batch), bar_format=bar_format, postfix=[f'timeout in {timeout} s']) as progress_bar: for ii in range(iterations_until_timeout): batch = self.refresh(batch) counts = batch._count_statuses() complete = counts['SUCCEEDED'] + counts['FAILED'] progress_bar.postfix = [ f'timeout in {timeout - ii * interval}s' ] # to control n/total manually; update is n += value progress_bar.n = complete progress_bar.update(0) if batch.complete(): return batch time.sleep(interval) raise HyP3Error(f'Timeout occurred while waiting for {batch}')
def test_batch_len(): batch = Batch() assert len(batch) == 0 batch = Batch([]) assert len(batch) == 0 batch = Batch([Job.from_dict(SUCCEEDED_JOB), Job.from_dict(FAILED_JOB)]) assert len(batch) == 2
def test_getitem(get_mock_job): unexpired_time = (datetime.now(tz=tz.UTC) + timedelta(days=7)).isoformat(timespec='seconds') j0 = Job.from_dict(SUCCEEDED_JOB) j1 = Job.from_dict(FAILED_JOB) j2 = get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time, files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}]) batch = Batch([j0, j1, j2]) assert j0 == batch[0] assert j1 == batch[1] assert j2 == batch[2] assert Batch([j1, j2]) == batch[1:]
def test_batch_iadd(): a = Batch([Job.from_dict(SUCCEEDED_JOB)]) b = Batch([Job.from_dict(FAILED_JOB)]) j = Job.from_dict(SUCCEEDED_JOB) j.status_code = 'RUNNING' a += b assert len(a) == 2 assert a.jobs[0].succeeded() assert a.jobs[1].failed() a += j assert len(a) == 3 assert a.jobs[0].succeeded() assert a.jobs[1].failed() assert a.jobs[2].running()
def test_batch_add(): a = Batch([Job.from_dict(SUCCEEDED_JOB)]) b = Batch([Job.from_dict(FAILED_JOB)]) j = Job.from_dict(SUCCEEDED_JOB) j.status_code = 'RUNNING' c = a + b assert len(c) == 2 assert c.jobs[0].succeeded() assert c.jobs[1].failed() d = c + j assert len(d) == 3 assert d.jobs[0].succeeded() assert d.jobs[1].failed() assert d.jobs[2].running()
def test_batch_any_expired(): job1 = Job.from_dict(SUCCEEDED_JOB) job1.expiration_time = datetime.now(tz.UTC) + timedelta(days=7) job2 = copy(job1) job2.expiration_time = datetime.now(tz.UTC) + timedelta(days=2) batch = Batch([job1, job2]) assert not batch.any_expired() # ignore jobs without expiration times job3 = Job.from_dict(FAILED_JOB) batch += job3 assert not batch.any_expired() job4 = copy(job1) job4.expiration_time = datetime.now(tz.UTC) - timedelta(days=2) batch += job4 assert batch.any_expired()
def test_batch_download_expired(tmp_path, get_mock_job): expired_time = (datetime.now(tz=tz.UTC) - timedelta(days=7)).isoformat(timespec='seconds') unexpired_time = (datetime.now(tz=tz.UTC) + timedelta(days=7)).isoformat(timespec='seconds') batch = Batch([ get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time, files=[{'url': 'https://foo.com/file1', 'size': 0, 'filename': 'file1'}]), get_mock_job(status_code='SUCCEEDED', expiration_time=expired_time, files=[{'url': 'https://foo.com/file2', 'size': 0, 'filename': 'file2'}]), get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time, files=[{'url': 'https://foo.com/file3', 'size': 0, 'filename': 'file3'}]) ]) responses.add(responses.GET, 'https://foo.com/file1', body='foobar1') responses.add(responses.GET, 'https://foo.com/file2', body='foobar2') responses.add(responses.GET, 'https://foo.com/file3', body='foobar3') paths = batch.download_files(tmp_path) contents = [path.read_text() for path in paths] assert len(paths) == 2 assert set(paths) == {tmp_path / 'file1', tmp_path / 'file3'} assert set(contents) == {'foobar1', 'foobar3'}
def test_contains(get_mock_job): unexpired_time = (datetime.now(tz=tz.UTC) + timedelta(days=7)).isoformat(timespec='seconds') j1 = Job.from_dict(SUCCEEDED_JOB) j2 = Job.from_dict(FAILED_JOB) j3 = get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time, files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}]) a = Batch([j1, j2]) assert j1 in a assert j2 in a assert j3 not in a
def test_batch_complete_succeeded(): batch = Batch([Job.from_dict(SUCCEEDED_JOB), Job.from_dict(SUCCEEDED_JOB)]) assert batch.complete() assert batch.succeeded() batch += Job.from_dict(FAILED_JOB) assert batch.complete() assert not batch.succeeded() running = Job.from_dict(FAILED_JOB) running.status_code = 'RUNNING' batch += running assert not batch.complete() assert not batch.succeeded()
def test_reverse(get_mock_job): unexpired_time = (datetime.now(tz=tz.UTC) + timedelta(days=7)).isoformat(timespec='seconds') j0 = Job.from_dict(SUCCEEDED_JOB) j1 = Job.from_dict(FAILED_JOB) j2 = get_mock_job(status_code='SUCCEEDED', expiration_time=unexpired_time, files=[{'url': 'https://foo.com/file', 'size': 0, 'filename': 'file'}]) batch = Batch([j0, j1, j2]) batch_reversed = reversed(batch) assert next(batch_reversed) == j2 assert next(batch_reversed) == j1 assert next(batch_reversed) == j0
def test_batch_filter_jobs(): succeeded_job = Job.from_dict(SUCCEEDED_JOB) succeeded_job.expiration_time = datetime.now(tz.UTC) + timedelta(days=7) expired_job = Job.from_dict(SUCCEEDED_JOB) expired_job.expiration_time = datetime.now(tz.UTC) - timedelta(days=7) running_job = Job.from_dict(FAILED_JOB) running_job.status_code = 'RUNNING' pending_job = Job.from_dict(FAILED_JOB) pending_job.status_code = 'PENDING' batch = Batch([succeeded_job, running_job, expired_job, pending_job, Job.from_dict(FAILED_JOB)]) not_failed = batch.filter_jobs() assert len(not_failed) == 4 assert not_failed.jobs[0].succeeded() and not not_failed.jobs[0].expired() assert not_failed.jobs[1].running() assert not_failed.jobs[2].succeeded() and not_failed.jobs[2].expired() assert not_failed.jobs[3].running() not_failed_or_expired = batch.filter_jobs(include_expired=False) assert len(not_failed_or_expired) == 3 assert not_failed_or_expired.jobs[0].succeeded() and not not_failed_or_expired.jobs[0].expired() assert not_failed_or_expired.jobs[1].running() assert not_failed_or_expired.jobs[2].running() succeeded = batch.filter_jobs(running=False) assert len(succeeded) == 2 assert succeeded.jobs[0].succeeded() and not succeeded.jobs[0].expired() assert succeeded.jobs[1].succeeded() and succeeded.jobs[1].expired() running = batch.filter_jobs(succeeded=False) assert len(running) == 2 assert running.jobs[0].running() assert running.jobs[1].running() failed = batch.filter_jobs(succeeded=False, running=False, failed=True) assert len(failed) == 1 assert failed.jobs[0].failed() everything = batch.filter_jobs(failed=True) assert len(everything) == len(batch) for ii, job in enumerate(everything.jobs): assert job.status_code == batch.jobs[ii].status_code if job.succeeded(): assert job.expired() == batch.jobs[ii].expired()
def test_delitem(): j0 = Job.from_dict(SUCCEEDED_JOB) j1 = Job.from_dict(FAILED_JOB) batch = Batch([j0, j1]) assert j0 in batch assert j1 in batch del batch[1] assert j0 in batch assert j1 not in batch batch += j1 del batch[0] assert j0 not in batch assert j1 in batch
def find_jobs(self, start: Optional[datetime] = None, end: Optional[datetime] = None, status_code: Optional[str] = None, name: Optional[str] = None, job_type: Optional[str] = None) -> Batch: """Gets a Batch of jobs from HyP3 matching the provided search criteria Args: start: only jobs submitted after given time end: only jobs submitted before given time status_code: only jobs matching this status (SUCCEEDED, FAILED, RUNNING, PENDING) name: only jobs with this name job_type: only jobs with this job_type Returns: A Batch object containing the found jobs """ params = {} for param_name in ('start', 'end', 'status_code', 'name', 'job_type'): param_value = locals().get(param_name) if param_value is not None: if isinstance(param_value, datetime): if param_value.tzinfo is None: param_value = param_value.replace(tzinfo=timezone.utc) param_value = param_value.isoformat(timespec='seconds') params[param_name] = param_value response = self.session.get(urljoin(self.url, '/jobs'), params=params) _raise_for_hyp3_status(response) jobs = [Job.from_dict(job) for job in response.json()['jobs']] while 'next' in response.json(): next_url = response.json()['next'] response = self.session.get(next_url) _raise_for_hyp3_status(response) jobs.extend( [Job.from_dict(job) for job in response.json()['jobs']]) return Batch(jobs)
def submit_prepared_jobs(self, prepared_jobs: Union[dict, List[dict]]) -> Batch: """Submit a prepared job dictionary, or list of prepared job dictionaries Args: prepared_jobs: A prepared job dictionary, or list of prepared job dictionaries Returns: A Batch object containing the submitted job(s) """ if isinstance(prepared_jobs, dict): payload = {'jobs': [prepared_jobs]} else: payload = {'jobs': prepared_jobs} response = self.session.post(urljoin(self.url, '/jobs'), json=payload) _raise_for_hyp3_status(response) batch = Batch() for job in response.json()['jobs']: batch += Job.from_dict(job) return batch
def _refresh_batch(self, batch: Batch): jobs = [] for job in batch.jobs: jobs.append(self.refresh(job)) return Batch(jobs)
def test_batch_iter(): defined_jobs = [Job.from_dict(SUCCEEDED_JOB), Job.from_dict(FAILED_JOB)] batch = Batch(defined_jobs) for batch_job, defined_job in zip(batch, defined_jobs): assert batch_job == defined_job