def test_state(): c = create_client_mock_for_container_tests(3, 7) mf = _model.ModelFuture(3, 7, client=c) ret = mf.state assert ret == 'foo' c.scripts.get_containers_runs.return_value = Response({'id': 7, 'container_id': 3, 'state': 'failed'}) mf = _model.ModelFuture(3, 7, client=c) assert mf.state == 'failed'
def test_modelfuture_constructor(mock_adc, mock_spe): c = create_client_mock_for_container_tests(7, 17) mf = _model.ModelFuture(job_id=7, run_id=17, client=c) assert mf.is_training is True assert mf.train_run_id == 17 assert mf.train_job_id == 7 mf = _model.ModelFuture(job_id=7, run_id=17, train_job_id=23, train_run_id=29, client=c) assert mf.is_training is False assert mf.train_run_id == 29 assert mf.train_job_id == 23
def test_set_model_exception_memory_error(): mock_client = setup_client_mock(1, 2, state='failed') err_msg = ('Process ran out of its allowed 3000 MiB of ' 'memory and was killed.') logs = [{ 'created_at': '2017-05-10T12:00:00.000Z', 'id': 10005, 'level': 'error', 'message': 'Failed' }, { 'created_at': '2017-05-10T12:00:00.000Z', 'id': 10003, 'level': 'error', 'message': 'Error on job: Process ended with an ' 'error, exiting: 137.' }, { 'created_at': '2017-05-10T12:00:00.000Z', 'id': 10000, 'level': 'error', 'message': err_msg }] mock_client.scripts.list_containers_runs_logs.return_value = logs fut = _model.ModelFuture(1, 2, client=mock_client) with pytest.raises(MemoryError) as err: fut.result() assert str(err.value) == err_msg
def test_table_None(mock_res, mock_lt, mock_meta): mock_lt.side_effect = FileNotFoundError() c = create_client_mock_for_container_tests(3, 7) mf = _model.ModelFuture(3, 7, client=c) assert mf.table is None mock_lt.assert_called_once_with(3, 7, 'predictions.csv', index_col=0, client=c)
def test_set_model_exception_unknown_error(): # If we really don't recognize the error, at least give the # user a few lines of logs so they can maybe figure it out themselves. mock_client = setup_client_mock(1, 2, state='failed') logs = [{ 'created_at': '2017-05-10T12:00:00.000Z', 'id': 10005, 'level': 'error', 'message': 'Failed' }, { 'created_at': '2017-05-10T12:00:00.000Z', 'id': 10003, 'level': 'error', 'message': 'Error on job: Process ended with an ' 'error, exiting: 137.' }, { 'created_at': '2017-05-10T12:00:00.000Z', 'id': 10000, 'level': 'error', 'message': 'Oops' }] err_msg = '\n'.join([l['message'] for l in logs]) mock_client.scripts.list_containers_runs_logs.return_value = logs fut = _model.ModelFuture(1, 2, client=mock_client) with pytest.raises(CivisJobFailure) as err: fut.result() assert str(err.value).startswith(err_msg)
def test_set_job_exception_unknown_error(m_sleep): # If we really don't recognize the error, at least give the # user a few lines of logs so they can maybe figure it out themselves. logs = [{'created_at': '2017-05-10T12:00:00.000Z', 'id': 10005, 'level': 'error', 'message': 'Failed'}, {'created_at': '2017-05-10T12:00:00.000Z', 'id': 10003, 'level': 'error', 'message': 'Error on job: Process ended with an ' 'error, exiting: 137.'}, {'created_at': '2017-05-10T12:00:00.000Z', 'id': 10000, 'level': 'error', 'message': 'Oops'}] mock_client = create_client_mock_for_container_tests( 1, 2, state='failed', log_outputs=logs) err_msg = ( "(From job 1 / run 2) " + '\n'.join([x['message'] for x in logs][::-1])) fut = _model.ModelFuture(1, 2, client=mock_client) with pytest.raises(CivisJobFailure) as err: fut.result() assert str(err.value).startswith(err_msg)
def test_train_data_(mock_cio): def poller(*args, **kwargs): return Response({'state': 'succeeded'}) mock_client = mock.MagicMock() mock_client.scripts.get_containers_runs = poller path = '/green/eggs/and/ham' training_meta = { 'run': { 'configuration': { 'data': { 'location': path } }, 'status': 'succeeded' } } mock_cio.file_to_json.return_value = training_meta mf = _model.ModelFuture(job_id=1, run_id=2, train_job_id=11, train_run_id=13, client=mock_client) assert mf.training_metadata == training_meta mock_cio.file_id_from_run_output.assert_called_with('model_info.json', 11, 13, client=mock_client)
def test_metrics_training(mock_file_id_from_run_output): mock_file_id_from_run_output.return_value = 11 c = setup_client_mock(3, 7) mf = _model.ModelFuture(3, 7, client=c) assert mf.metrics == 'foo' mock_file_id_from_run_output.assert_called_with('metrics.json', 3, 7, client=mock.ANY)
def test_metrics_prediction(mock_file_id_from_run_output): mock_file_id_from_run_output.return_value = 11 c = create_client_mock_for_container_tests(3, 7) mf = _model.ModelFuture(1, 2, 3, 7, client=c) assert mf.metrics == 'foo' mock_file_id_from_run_output.assert_called_with('metrics.json', 3, 7, client=mock.ANY)
def test_set_model_exception_no_exception(mock_f2j): # If nothing went wrong, we shouldn't set an exception ro = [{'name': 'model_info.json', 'object_id': 137, 'object_type': 'File'}, {'name': 'metrics.json', 'object_id': 139, 'object_type': 'File'}] ro = [Response(o) for o in ro] mock_client = setup_client_mock(1, 2, state='succeeded', run_outputs=ro) fut = _model.ModelFuture(1, 2, client=mock_client) assert fut.exception() is None
def test_table(mock_res, mock_lt): c = setup_client_mock(3, 7) mf = _model.ModelFuture(3, 7, client=c) assert mf.table == 'bar' mock_lt.assert_called_once_with(3, 7, 'predictions.csv', index_col=0, client=c)
def test_validation_metadata_prediction(mock_spe, mock_f2f, mock_file_id_from_run_output): mock_file_id_from_run_output.return_value = 11 c = create_client_mock_for_container_tests(3, 7) mf = _model.ModelFuture(1, 2, 3, 7, client=c) assert mf.validation_metadata == 'foo' mock_f2f.assert_called_once_with(11, client=c) mock_file_id_from_run_output.assert_called_with('metrics.json', 3, 7, client=mock.ANY)
def test_validation_metadata_training(mock_spe, mock_f2f, mock_file_id_from_run_output): mock_file_id_from_run_output.return_value = 11 c = setup_client_mock(3, 7) mf = _model.ModelFuture(3, 7, client=c) assert mf.validation_metadata == 'foo' mock_f2f.assert_called_once_with(11, client=c) mock_file_id_from_run_output.assert_called_with('metrics.json', 3, 7, client=mock.ANY)
def test_table_None(mock_res, mock_lt): mock_lt.side_effect = FileNotFoundError() c = setup_client_mock(3, 7) mf = _model.ModelFuture(3, 7, client=c) assert mf.table is None mock_lt.assert_called_once_with(3, 7, 'predictions.csv', index_col=0, client=c)
def test_getstate(): c = setup_client_mock(3, 7) mf = _model.ModelFuture(3, 7, client=c) ret = mf.__getstate__() assert ret['_done_callbacks'] == [] assert not ret['_self_polling_executor'] assert 'client' not in ret assert 'poller' not in ret assert '_condition' not in ret
def test_estimator(mock_le): c = setup_client_mock(3, 7) mf = _model.ModelFuture(3, 7, client=c) assert mock_le.call_count == 0, "Estimator retrieval is lazy." assert mf.estimator == 'spam' assert mock_le.call_count == 1 assert mf.estimator == 'spam' assert mock_le.call_count == 1,\ "The Estimator is only downloaded once and cached."
def test_validation_metadata_missing(mock_spe, mock_f2f, mock_file_id_from_run_output): # Make sure that missing validation metadata doesn't cause an error mock_file_id_from_run_output.side_effect = FileNotFoundError c = setup_client_mock(3, 7) mf = _model.ModelFuture(3, 7, client=c) assert mf.validation_metadata is None assert mf.metrics is None assert mock_f2f.call_count == 0 assert mock_file_id_from_run_output.call_count == 1
def test_train_data_fname(m_sleep): # swap out the poller with a simple function that accepts *args, **kwargs # and returns a simple successful Response object def poller(*args, **kwargs): return Response({'state': 'succeeded'}) mock_client = mock.MagicMock() mock_client.scripts.get_containers_runs = poller mf = _model.ModelFuture(job_id=1, run_id=2, client=mock_client) path = '/green/eggs/and/ham' training_meta = {'run': {'configuration': {'data': {'location': path}}}} mf._train_metadata = training_meta assert mf.train_data_fname == 'ham'
def test_train_data_exc_handling(mock_load_table, m_sleep): def poller(*args, **kwargs): return Response({'state': 'succeeded'}) mock_client = mock.MagicMock() mock_client.scripts.get_containers_runs = poller mf = _model.ModelFuture(job_id=1, run_id=2, client=mock_client) mf._train_data_fname = 'placeholder.csv' # check we catch 404 error and raise some intelligible r = Response({'content': None, 'status_code': 404, 'reason': None}) mock_load_table.side_effect = [CivisAPIError(r)] with pytest.raises(ValueError): mf.train_data
def test_metrics_training_None(mock_file_to_json, mock_file_id_from_run_output): mock_file_to_json.return_value = mock.MagicMock( return_value={'metrics': 'foo', 'run': {'status': 'succeeded'}}) mock_file_id_from_run_output.return_value = 11 c = create_client_mock_for_container_tests(3, 7) mf = _model.ModelFuture(3, 7, client=c) # override validation metadata to be None, as though we ran # a train job without validation mf._val_metadata = None mock_file_to_json.return_value = None assert mf.metrics is None mock_file_id_from_run_output.assert_called_with('metrics.json', 3, 7, client=mock.ANY)
def test_set_job_exception_memory_error(m_sleep): err_msg = ('Process ran out of its allowed 3000 MiB of ' 'memory and was killed.') logs = [{'created_at': '2017-05-10T12:00:00.000Z', 'id': 10005, 'level': 'error', 'message': 'Failed'}, {'created_at': '2017-05-10T12:00:00.000Z', 'id': 10003, 'level': 'error', 'message': 'Error on job: Process ended with an ' 'error, exiting: 137.'}, {'created_at': '2017-05-10T12:00:00.000Z', 'id': 10000, 'level': 'error', 'message': err_msg}] mock_client = create_client_mock_for_container_tests( 1, 2, state='failed', log_outputs=logs) fut = _model.ModelFuture(1, 2, client=mock_client) with pytest.raises(MemoryError) as err: fut.result() assert str(err.value) == f"(From job 1 / run 2) {err_msg}"
def test_modelfuture_pickle_smoke(mock_client): mf = _model.ModelFuture(job_id=7, run_id=13, client=create_client_mock_for_container_tests()) mf.result() mf_pickle = pickle.dumps(mf) pickle.loads(mf_pickle)
def test_metadata(mock_spec, mock_f2j): c = setup_client_mock(3, 7) mf = _model.ModelFuture(3, 7, client=c) assert mf.metadata == 'bar' mock_f2j.assert_called_once_with(11, client=c)
def test_table_no_pkey(mock_res, mock_lt, mock_meta): c = create_client_mock_for_container_tests(3, 7) mf = _model.ModelFuture(3, 7, client=c) assert mf.table == 'bar' mock_lt.assert_called_once_with(3, 7, 'predictions.csv', index_col=False, client=c)
def test_metadata(mock_spec, mock_f2j): c = create_client_mock_for_container_tests(3, 7) mf = _model.ModelFuture(3, 7, client=c) assert mf.metadata == {'foo': 'bar'} mock_f2j.assert_called_once_with(11, client=c)
def test_modelfuture_pickle_smoke(mock_client): mf = _model.ModelFuture(job_id=7, run_id=13, client=setup_client_mock()) mf.result() mf_pickle = pickle.dumps(mf) pickle.loads(mf_pickle)