def test_throws_when_deployment_layer_id_is_missing(self, m: Mocker):
mangled_response = json.loads(RESPONSE_DEPLOY_SUCCESS)
mangled_response['data']['result']['deployment'].pop('layer')
m.get('/job/test-job-id', json=mangled_response)
with self.assertRaisesRegex(piazza.InvalidResponse,
'missing `data.result.deployment.layer`'):
piazza.get_status('test-job-id')
def test_throws_when_result_is_missing(self, m: Mocker):
mangled_response = json.loads(RESPONSE_JOB_SUCCESS)
mangled_response['data'].pop('result')
m.get('/job/test-job-id', json=mangled_response)
with self.assertRaisesRegex(piazza.InvalidResponse,
'missing `data.result`'):
piazza.get_status('test-job-id')
def test_throws_when_status_is_ambiguous(self, m: Mocker):
mangled_response = json.loads(RESPONSE_JOB_SUCCESS)
mangled_response['data']['status'] = 'lolwut'
m.get('/job/test-job-id', json=mangled_response)
with self.assertRaisesRegex(piazza.InvalidResponse,
'ambiguous value for `data.status`'):
piazza.get_status('test-job-id')
def test_returns_correct_status_for_successful_deployment_job(
self, m: Mocker):
m.get('/job/test-job-id', text=RESPONSE_JOB_SUCCESS)
status = piazza.get_status('test-job-id')
self.assertEqual('Success', status.status)
self.assertEqual('test-data-id', status.data_id)
self.assertIsNone(status.error_message)
def _updater(self, job_id: str, age: timedelta, index: int):
log = self._log
job_ttl = self._job_ttl
# Get latest status
try:
status = piazza.get_status(job_id)
except piazza.Unauthorized:
log.error('<%03d/%s> credentials rejected during polling!', index,
job_id)
return
except (piazza.ServerError, piazza.Error) as err:
if isinstance(err, piazza.ServerError) and err.status_code == 404:
log.warning('<%03d/%s> Job not found', index, job_id)
_save_execution_error(job_id, STEP_POLLING, 'Job not found')
return
else:
log.error('<%03d/%s> call to Piazza failed: %s', index, job_id,
err.message)
return
# Emit console feedback
log.info('<%03d/%s> polled (%s; age=%s)', index, job_id, status.status,
age)
# Determine appropriate action by status
if status.status in (piazza.STATUS_SUBMITTED, piazza.STATUS_PENDING):
if age > job_ttl:
log.warning(
'<%03d/%s> appears to have stalled and will no longer be tracked',
index, job_id)
_save_execution_error(job_id,
STEP_QUEUED,
'Submission wait time exceeded',
status=STATUS_TIMED_OUT)
return
conn = db.get_connection()
try:
db.jobs.update_status(conn,
job_id=job_id,
status=status.status)
except db.DatabaseError as err:
log.error('<%03d/%s> Could not save status to database', index,
job_id)
db.print_diagnostics(err)
return
finally:
conn.close()
elif status.status == piazza.STATUS_RUNNING:
if age > job_ttl:
log.warning(
'<%03d/%s> appears to have stalled and will no longer be tracked',
index, job_id)
_save_execution_error(job_id,
STEP_PROCESSING,
'Processing time exceeded',
status=STATUS_TIMED_OUT)
return
conn = db.get_connection()
try:
db.jobs.update_status(conn,
job_id=job_id,
status=status.status)
except db.DatabaseError as err:
log.error('<%03d/%s> Could not save status to database', index,
job_id)
db.print_diagnostics(err)
return
finally:
conn.close()
elif status.status == piazza.STATUS_SUCCESS:
log.info('<%03d/%s> Resolving detections data ID (via <%s>)',
index, job_id, status.data_id)
try:
detections_data_id = _resolve_detections_data_id(
status.data_id)
except PostprocessingError as err:
log.error('<%03d/%s> Could not resolve detections data ID: %s',
index, job_id, err)
_save_execution_error(job_id, STEP_RESOLVE, str(err))
return
log.info('<%03d/%s> Fetching detections from Piazza', index,
job_id)
try:
geojson = piazza.get_file(detections_data_id).text
except piazza.ServerError as err:
log.error('<%03d/%s> Could not fetch data ID <%s>: %s', index,
job_id, detections_data_id, err)
_save_execution_error(
job_id, STEP_COLLECT_GEOJSON,
'Could not retrieve GeoJSON from Piazza')
return
log.info('<%03d/%s> Saving detections to database (%0.1fMB)',
index, job_id,
len(geojson) / 1024000)
conn = db.get_connection()
transaction = conn.begin()
try:
db.jobs.insert_detection(conn,
job_id=job_id,
feature_collection=geojson)
db.jobs.update_status(
conn,
job_id=job_id,
status=piazza.STATUS_SUCCESS,
)
transaction.commit()
except db.DatabaseError as err:
transaction.rollback()
transaction.close()
log.error(
'<%03d/%s> Could not save status and detections to database',
index, job_id)
db.print_diagnostics(err)
_save_execution_error(job_id, STEP_COLLECT_GEOJSON,
'Could not insert GeoJSON to database')
return
finally:
conn.close()
elif status.status in (piazza.STATUS_ERROR, piazza.STATUS_FAIL):
# FIXME -- use heuristics to generate a more descriptive error message
_save_execution_error(job_id, STEP_ALGORITHM,
'Job failed during algorithm execution')
elif status.status == piazza.STATUS_CANCELLED:
_save_execution_error(job_id,
STEP_ALGORITHM,
'Job was cancelled',
status=piazza.STATUS_CANCELLED)
def test_throws_when_job_doesnt_exist(self, m: Mocker):
m.get('/job/test-job-id', text=RESPONSE_JOB_NOT_FOUND, status_code=404)
with self.assertRaises(piazza.ServerError):
piazza.get_status('test-job-id')
def test_throws_when_credentials_are_rejected(self, m: Mocker):
m.get('/job/test-job-id',
text=RESPONSE_ERROR_UNAUTHORIZED,
status_code=401)
with self.assertRaises(piazza.Unauthorized):
piazza.get_status('test-job-id')
def test_throws_when_piazza_is_unreachable(self, _):
with unittest.mock.patch('requests.get') as stub:
stub.side_effect = ConnectionError(request=unittest.mock.Mock())
with self.assertRaises(piazza.Unreachable):
piazza.get_status('test-job-id')
def test_handles_http_errors_gracefully(self, m: Mocker):
m.get('/job/test-job-id', text=RESPONSE_JOB_RUNNING, status_code=500)
with self.assertRaises(piazza.ServerError):
piazza.get_status('test-job-id')
def test_returns_correct_status_for_canceled_job(self, m: Mocker):
m.get('/job/test-job-id', text=RESPONSE_JOB_CANCELLED)
status = piazza.get_status('test-job-id')
self.assertEqual('Cancelled', status.status)
self.assertIsNone(status.data_id)
self.assertIsNone(status.error_message)
def test_returns_correct_status_for_failed_job(self, m: Mocker):
m.get('/job/test-job-id', text=RESPONSE_JOB_ERROR)
status = piazza.get_status('test-job-id')
self.assertEqual('Error', status.status)
self.assertIsNone(status.data_id)
self.assertEqual('test-failure-message', status.error_message)
def test_returns_correct_status_for_pending_job(self, m: Mocker):
m.get('/job/test-job-id', text=RESPONSE_JOB_PENDING)
status = piazza.get_status('test-job-id')
self.assertEqual('Pending', status.status)
self.assertIsNone(status.data_id)
self.assertIsNone(status.error_message)
def test_returns_correct_status_for_submitted_job(self, m: Mocker):
m.get('/job/test-job-id', text=RESPONSE_JOB_SUBMITTED)
status = piazza.get_status('test-job-id')
self.assertEqual('Submitted', status.status)
self.assertIsNone(status.data_id)
self.assertIsNone(status.error_message)
def test_calls_correct_url(self, m: Mocker):
m.get('/job/test-job-id', text=RESPONSE_JOB_RUNNING)
piazza.get_status('test-job-id')
self.assertEqual(
'https://test-piazza-host.localdomain/job/test-job-id',
m.request_history[0].url)
