def test_throws_when_deployment_layer_id_is_missing(self, m: Mocker): mangled_response = json.loads(RESPONSE_DEPLOY_SUCCESS) mangled_response['data']['result']['deployment'].pop('layer') m.get('/job/test-job-id', json=mangled_response) with self.assertRaisesRegex(piazza.InvalidResponse, 'missing `data.result.deployment.layer`'): piazza.get_status('test-job-id')
def test_throws_when_result_is_missing(self, m: Mocker): mangled_response = json.loads(RESPONSE_JOB_SUCCESS) mangled_response['data'].pop('result') m.get('/job/test-job-id', json=mangled_response) with self.assertRaisesRegex(piazza.InvalidResponse, 'missing `data.result`'): piazza.get_status('test-job-id')
def test_throws_when_status_is_ambiguous(self, m: Mocker): mangled_response = json.loads(RESPONSE_JOB_SUCCESS) mangled_response['data']['status'] = 'lolwut' m.get('/job/test-job-id', json=mangled_response) with self.assertRaisesRegex(piazza.InvalidResponse, 'ambiguous value for `data.status`'): piazza.get_status('test-job-id')
def test_returns_correct_status_for_successful_deployment_job( self, m: Mocker): m.get('/job/test-job-id', text=RESPONSE_JOB_SUCCESS) status = piazza.get_status('test-job-id') self.assertEqual('Success', status.status) self.assertEqual('test-data-id', status.data_id) self.assertIsNone(status.error_message)
def _updater(self, job_id: str, age: timedelta, index: int): log = self._log job_ttl = self._job_ttl # Get latest status try: status = piazza.get_status(job_id) except piazza.Unauthorized: log.error('<%03d/%s> credentials rejected during polling!', index, job_id) return except (piazza.ServerError, piazza.Error) as err: if isinstance(err, piazza.ServerError) and err.status_code == 404: log.warning('<%03d/%s> Job not found', index, job_id) _save_execution_error(job_id, STEP_POLLING, 'Job not found') return else: log.error('<%03d/%s> call to Piazza failed: %s', index, job_id, err.message) return # Emit console feedback log.info('<%03d/%s> polled (%s; age=%s)', index, job_id, status.status, age) # Determine appropriate action by status if status.status in (piazza.STATUS_SUBMITTED, piazza.STATUS_PENDING): if age > job_ttl: log.warning( '<%03d/%s> appears to have stalled and will no longer be tracked', index, job_id) _save_execution_error(job_id, STEP_QUEUED, 'Submission wait time exceeded', status=STATUS_TIMED_OUT) return conn = db.get_connection() try: db.jobs.update_status(conn, job_id=job_id, status=status.status) except db.DatabaseError as err: log.error('<%03d/%s> Could not save status to database', index, job_id) db.print_diagnostics(err) return finally: conn.close() elif status.status == piazza.STATUS_RUNNING: if age > job_ttl: log.warning( '<%03d/%s> appears to have stalled and will no longer be tracked', index, job_id) _save_execution_error(job_id, STEP_PROCESSING, 'Processing time exceeded', status=STATUS_TIMED_OUT) return conn = db.get_connection() try: db.jobs.update_status(conn, job_id=job_id, status=status.status) except db.DatabaseError as err: log.error('<%03d/%s> Could not save status to database', index, job_id) db.print_diagnostics(err) return finally: conn.close() elif status.status == piazza.STATUS_SUCCESS: log.info('<%03d/%s> Resolving detections data ID (via <%s>)', index, job_id, status.data_id) try: detections_data_id = _resolve_detections_data_id( status.data_id) except PostprocessingError as err: log.error('<%03d/%s> Could not resolve detections data ID: %s', index, job_id, err) _save_execution_error(job_id, STEP_RESOLVE, str(err)) return log.info('<%03d/%s> Fetching detections from Piazza', index, job_id) try: geojson = piazza.get_file(detections_data_id).text except piazza.ServerError as err: log.error('<%03d/%s> Could not fetch data ID <%s>: %s', index, job_id, detections_data_id, err) _save_execution_error( job_id, STEP_COLLECT_GEOJSON, 'Could not retrieve GeoJSON from Piazza') return log.info('<%03d/%s> Saving detections to database (%0.1fMB)', index, job_id, len(geojson) / 1024000) conn = db.get_connection() transaction = conn.begin() try: db.jobs.insert_detection(conn, job_id=job_id, feature_collection=geojson) db.jobs.update_status( conn, job_id=job_id, status=piazza.STATUS_SUCCESS, ) transaction.commit() except db.DatabaseError as err: transaction.rollback() transaction.close() log.error( '<%03d/%s> Could not save status and detections to database', index, job_id) db.print_diagnostics(err) _save_execution_error(job_id, STEP_COLLECT_GEOJSON, 'Could not insert GeoJSON to database') return finally: conn.close() elif status.status in (piazza.STATUS_ERROR, piazza.STATUS_FAIL): # FIXME -- use heuristics to generate a more descriptive error message _save_execution_error(job_id, STEP_ALGORITHM, 'Job failed during algorithm execution') elif status.status == piazza.STATUS_CANCELLED: _save_execution_error(job_id, STEP_ALGORITHM, 'Job was cancelled', status=piazza.STATUS_CANCELLED)
def test_throws_when_job_doesnt_exist(self, m: Mocker): m.get('/job/test-job-id', text=RESPONSE_JOB_NOT_FOUND, status_code=404) with self.assertRaises(piazza.ServerError): piazza.get_status('test-job-id')
def test_throws_when_credentials_are_rejected(self, m: Mocker): m.get('/job/test-job-id', text=RESPONSE_ERROR_UNAUTHORIZED, status_code=401) with self.assertRaises(piazza.Unauthorized): piazza.get_status('test-job-id')
def test_throws_when_piazza_is_unreachable(self, _): with unittest.mock.patch('requests.get') as stub: stub.side_effect = ConnectionError(request=unittest.mock.Mock()) with self.assertRaises(piazza.Unreachable): piazza.get_status('test-job-id')
def test_handles_http_errors_gracefully(self, m: Mocker): m.get('/job/test-job-id', text=RESPONSE_JOB_RUNNING, status_code=500) with self.assertRaises(piazza.ServerError): piazza.get_status('test-job-id')
def test_returns_correct_status_for_canceled_job(self, m: Mocker): m.get('/job/test-job-id', text=RESPONSE_JOB_CANCELLED) status = piazza.get_status('test-job-id') self.assertEqual('Cancelled', status.status) self.assertIsNone(status.data_id) self.assertIsNone(status.error_message)
def test_returns_correct_status_for_failed_job(self, m: Mocker): m.get('/job/test-job-id', text=RESPONSE_JOB_ERROR) status = piazza.get_status('test-job-id') self.assertEqual('Error', status.status) self.assertIsNone(status.data_id) self.assertEqual('test-failure-message', status.error_message)
def test_returns_correct_status_for_pending_job(self, m: Mocker): m.get('/job/test-job-id', text=RESPONSE_JOB_PENDING) status = piazza.get_status('test-job-id') self.assertEqual('Pending', status.status) self.assertIsNone(status.data_id) self.assertIsNone(status.error_message)
def test_returns_correct_status_for_submitted_job(self, m: Mocker): m.get('/job/test-job-id', text=RESPONSE_JOB_SUBMITTED) status = piazza.get_status('test-job-id') self.assertEqual('Submitted', status.status) self.assertIsNone(status.data_id) self.assertIsNone(status.error_message)
def test_calls_correct_url(self, m: Mocker): m.get('/job/test-job-id', text=RESPONSE_JOB_RUNNING) piazza.get_status('test-job-id') self.assertEqual( 'https://test-piazza-host.localdomain/job/test-job-id', m.request_history[0].url)