def test_state():
c = create_client_mock_for_container_tests(3, 7)
mf = _model.ModelFuture(3, 7, client=c)
ret = mf.state
assert ret == 'foo'
c.scripts.get_containers_runs.return_value = Response({'id': 7,
'container_id': 3,
'state': 'failed'})
mf = _model.ModelFuture(3, 7, client=c)
assert mf.state == 'failed'
def test_modelfuture_constructor(mock_adc, mock_spe):
c = create_client_mock_for_container_tests(7, 17)
mf = _model.ModelFuture(job_id=7, run_id=17, client=c)
assert mf.is_training is True
assert mf.train_run_id == 17
assert mf.train_job_id == 7
mf = _model.ModelFuture(job_id=7, run_id=17,
train_job_id=23, train_run_id=29, client=c)
assert mf.is_training is False
assert mf.train_run_id == 29
assert mf.train_job_id == 23
def test_set_model_exception_memory_error():
mock_client = setup_client_mock(1, 2, state='failed')
err_msg = ('Process ran out of its allowed 3000 MiB of '
'memory and was killed.')
logs = [{
'created_at': '2017-05-10T12:00:00.000Z',
'id': 10005,
'level': 'error',
'message': 'Failed'
}, {
'created_at':
'2017-05-10T12:00:00.000Z',
'id':
10003,
'level':
'error',
'message':
'Error on job: Process ended with an '
'error, exiting: 137.'
}, {
'created_at': '2017-05-10T12:00:00.000Z',
'id': 10000,
'level': 'error',
'message': err_msg
}]
mock_client.scripts.list_containers_runs_logs.return_value = logs
fut = _model.ModelFuture(1, 2, client=mock_client)
with pytest.raises(MemoryError) as err:
fut.result()
assert str(err.value) == err_msg
def test_table_None(mock_res, mock_lt, mock_meta):
mock_lt.side_effect = FileNotFoundError()
c = create_client_mock_for_container_tests(3, 7)
mf = _model.ModelFuture(3, 7, client=c)
assert mf.table is None
mock_lt.assert_called_once_with(3, 7, 'predictions.csv',
index_col=0, client=c)
def test_set_model_exception_unknown_error():
# If we really don't recognize the error, at least give the
# user a few lines of logs so they can maybe figure it out themselves.
mock_client = setup_client_mock(1, 2, state='failed')
logs = [{
'created_at': '2017-05-10T12:00:00.000Z',
'id': 10005,
'level': 'error',
'message': 'Failed'
}, {
'created_at':
'2017-05-10T12:00:00.000Z',
'id':
10003,
'level':
'error',
'message':
'Error on job: Process ended with an '
'error, exiting: 137.'
}, {
'created_at': '2017-05-10T12:00:00.000Z',
'id': 10000,
'level': 'error',
'message': 'Oops'
}]
err_msg = '\n'.join([l['message'] for l in logs])
mock_client.scripts.list_containers_runs_logs.return_value = logs
fut = _model.ModelFuture(1, 2, client=mock_client)
with pytest.raises(CivisJobFailure) as err:
fut.result()
assert str(err.value).startswith(err_msg)
def test_set_job_exception_unknown_error(m_sleep):
# If we really don't recognize the error, at least give the
# user a few lines of logs so they can maybe figure it out themselves.
logs = [{'created_at': '2017-05-10T12:00:00.000Z',
'id': 10005,
'level': 'error',
'message': 'Failed'},
{'created_at': '2017-05-10T12:00:00.000Z',
'id': 10003,
'level': 'error',
'message': 'Error on job: Process ended with an '
'error, exiting: 137.'},
{'created_at': '2017-05-10T12:00:00.000Z',
'id': 10000,
'level': 'error',
'message': 'Oops'}]
mock_client = create_client_mock_for_container_tests(
1, 2, state='failed', log_outputs=logs)
err_msg = (
"(From job 1 / run 2) "
+ '\n'.join([x['message'] for x in logs][::-1]))
fut = _model.ModelFuture(1, 2, client=mock_client)
with pytest.raises(CivisJobFailure) as err:
fut.result()
assert str(err.value).startswith(err_msg)
def test_train_data_(mock_cio):
def poller(*args, **kwargs):
return Response({'state': 'succeeded'})
mock_client = mock.MagicMock()
mock_client.scripts.get_containers_runs = poller
path = '/green/eggs/and/ham'
training_meta = {
'run': {
'configuration': {
'data': {
'location': path
}
},
'status': 'succeeded'
}
}
mock_cio.file_to_json.return_value = training_meta
mf = _model.ModelFuture(job_id=1,
run_id=2,
train_job_id=11,
train_run_id=13,
client=mock_client)
assert mf.training_metadata == training_meta
mock_cio.file_id_from_run_output.assert_called_with('model_info.json',
11,
13,
client=mock_client)
def test_metrics_training(mock_file_id_from_run_output):
mock_file_id_from_run_output.return_value = 11
c = setup_client_mock(3, 7)
mf = _model.ModelFuture(3, 7, client=c)
assert mf.metrics == 'foo'
mock_file_id_from_run_output.assert_called_with('metrics.json', 3, 7,
client=mock.ANY)
def test_metrics_prediction(mock_file_id_from_run_output):
mock_file_id_from_run_output.return_value = 11
c = create_client_mock_for_container_tests(3, 7)
mf = _model.ModelFuture(1, 2, 3, 7, client=c)
assert mf.metrics == 'foo'
mock_file_id_from_run_output.assert_called_with('metrics.json', 3, 7,
client=mock.ANY)
def test_set_model_exception_no_exception(mock_f2j):
# If nothing went wrong, we shouldn't set an exception
ro = [{'name': 'model_info.json', 'object_id': 137, 'object_type': 'File'},
{'name': 'metrics.json', 'object_id': 139, 'object_type': 'File'}]
ro = [Response(o) for o in ro]
mock_client = setup_client_mock(1, 2, state='succeeded', run_outputs=ro)
fut = _model.ModelFuture(1, 2, client=mock_client)
assert fut.exception() is None
def test_table(mock_res, mock_lt):
c = setup_client_mock(3, 7)
mf = _model.ModelFuture(3, 7, client=c)
assert mf.table == 'bar'
mock_lt.assert_called_once_with(3,
7,
'predictions.csv',
index_col=0,
client=c)
def test_validation_metadata_prediction(mock_spe, mock_f2f,
mock_file_id_from_run_output):
mock_file_id_from_run_output.return_value = 11
c = create_client_mock_for_container_tests(3, 7)
mf = _model.ModelFuture(1, 2, 3, 7, client=c)
assert mf.validation_metadata == 'foo'
mock_f2f.assert_called_once_with(11, client=c)
mock_file_id_from_run_output.assert_called_with('metrics.json', 3, 7,
client=mock.ANY)
def test_validation_metadata_training(mock_spe, mock_f2f,
mock_file_id_from_run_output):
mock_file_id_from_run_output.return_value = 11
c = setup_client_mock(3, 7)
mf = _model.ModelFuture(3, 7, client=c)
assert mf.validation_metadata == 'foo'
mock_f2f.assert_called_once_with(11, client=c)
mock_file_id_from_run_output.assert_called_with('metrics.json', 3, 7,
client=mock.ANY)
def test_table_None(mock_res, mock_lt):
mock_lt.side_effect = FileNotFoundError()
c = setup_client_mock(3, 7)
mf = _model.ModelFuture(3, 7, client=c)
assert mf.table is None
mock_lt.assert_called_once_with(3,
7,
'predictions.csv',
index_col=0,
client=c)
def test_getstate():
c = setup_client_mock(3, 7)
mf = _model.ModelFuture(3, 7, client=c)
ret = mf.__getstate__()
assert ret['_done_callbacks'] == []
assert not ret['_self_polling_executor']
assert 'client' not in ret
assert 'poller' not in ret
assert '_condition' not in ret
def test_estimator(mock_le):
c = setup_client_mock(3, 7)
mf = _model.ModelFuture(3, 7, client=c)
assert mock_le.call_count == 0, "Estimator retrieval is lazy."
assert mf.estimator == 'spam'
assert mock_le.call_count == 1
assert mf.estimator == 'spam'
assert mock_le.call_count == 1,\
"The Estimator is only downloaded once and cached."
def test_validation_metadata_missing(mock_spe, mock_f2f,
mock_file_id_from_run_output):
# Make sure that missing validation metadata doesn't cause an error
mock_file_id_from_run_output.side_effect = FileNotFoundError
c = setup_client_mock(3, 7)
mf = _model.ModelFuture(3, 7, client=c)
assert mf.validation_metadata is None
assert mf.metrics is None
assert mock_f2f.call_count == 0
assert mock_file_id_from_run_output.call_count == 1
def test_train_data_fname(m_sleep):
# swap out the poller with a simple function that accepts *args, **kwargs
# and returns a simple successful Response object
def poller(*args, **kwargs):
return Response({'state': 'succeeded'})
mock_client = mock.MagicMock()
mock_client.scripts.get_containers_runs = poller
mf = _model.ModelFuture(job_id=1, run_id=2, client=mock_client)
path = '/green/eggs/and/ham'
training_meta = {'run': {'configuration': {'data': {'location': path}}}}
mf._train_metadata = training_meta
assert mf.train_data_fname == 'ham'
def test_train_data_exc_handling(mock_load_table, m_sleep):
def poller(*args, **kwargs):
return Response({'state': 'succeeded'})
mock_client = mock.MagicMock()
mock_client.scripts.get_containers_runs = poller
mf = _model.ModelFuture(job_id=1, run_id=2, client=mock_client)
mf._train_data_fname = 'placeholder.csv'
# check we catch 404 error and raise some intelligible
r = Response({'content': None, 'status_code': 404, 'reason': None})
mock_load_table.side_effect = [CivisAPIError(r)]
with pytest.raises(ValueError):
mf.train_data
def test_metrics_training_None(mock_file_to_json,
mock_file_id_from_run_output):
mock_file_to_json.return_value = mock.MagicMock(
return_value={'metrics': 'foo',
'run': {'status': 'succeeded'}})
mock_file_id_from_run_output.return_value = 11
c = create_client_mock_for_container_tests(3, 7)
mf = _model.ModelFuture(3, 7, client=c)
# override validation metadata to be None, as though we ran
# a train job without validation
mf._val_metadata = None
mock_file_to_json.return_value = None
assert mf.metrics is None
mock_file_id_from_run_output.assert_called_with('metrics.json', 3, 7,
client=mock.ANY)
def test_set_job_exception_memory_error(m_sleep):
err_msg = ('Process ran out of its allowed 3000 MiB of '
'memory and was killed.')
logs = [{'created_at': '2017-05-10T12:00:00.000Z',
'id': 10005,
'level': 'error',
'message': 'Failed'},
{'created_at': '2017-05-10T12:00:00.000Z',
'id': 10003,
'level': 'error',
'message': 'Error on job: Process ended with an '
'error, exiting: 137.'},
{'created_at': '2017-05-10T12:00:00.000Z',
'id': 10000,
'level': 'error',
'message': err_msg}]
mock_client = create_client_mock_for_container_tests(
1, 2, state='failed', log_outputs=logs)
fut = _model.ModelFuture(1, 2, client=mock_client)
with pytest.raises(MemoryError) as err:
fut.result()
assert str(err.value) == f"(From job 1 / run 2) {err_msg}"
def test_modelfuture_pickle_smoke(mock_client):
mf = _model.ModelFuture(job_id=7, run_id=13,
client=create_client_mock_for_container_tests())
mf.result()
mf_pickle = pickle.dumps(mf)
pickle.loads(mf_pickle)
def test_metadata(mock_spec, mock_f2j):
c = setup_client_mock(3, 7)
mf = _model.ModelFuture(3, 7, client=c)
assert mf.metadata == 'bar'
mock_f2j.assert_called_once_with(11, client=c)
def test_table_no_pkey(mock_res, mock_lt, mock_meta):
c = create_client_mock_for_container_tests(3, 7)
mf = _model.ModelFuture(3, 7, client=c)
assert mf.table == 'bar'
mock_lt.assert_called_once_with(3, 7, 'predictions.csv',
index_col=False, client=c)
def test_metadata(mock_spec, mock_f2j):
c = create_client_mock_for_container_tests(3, 7)
mf = _model.ModelFuture(3, 7, client=c)
assert mf.metadata == {'foo': 'bar'}
mock_f2j.assert_called_once_with(11, client=c)
def test_modelfuture_pickle_smoke(mock_client):
mf = _model.ModelFuture(job_id=7, run_id=13, client=setup_client_mock())
mf.result()
mf_pickle = pickle.dumps(mf)
pickle.loads(mf_pickle)
