def create_mock_client_with_job():
mock_client = create_client_mock()
job_response = Response({"id": 1, "name": "test"})
run_post_response = Response({"id": 1})
run_get_response = Response({
"id": run_post_response.id,
"state": "succeeded",
"is_cancel_requested": False,
"error": None,
"custom_id": run_post_response.id,
})
mock_client.scripts.post_custom.return_value = job_response
mock_client.scripts.post_custom_runs.return_value = run_post_response
mock_client.scripts.get_custom_runs.return_value = run_get_response
return mock_client
def test_file_to_dataframe_infer_gzip():
m_client = mock.Mock()
m_client.files.get.return_value = Response({'name': 'spam.csv.gz',
'file_url': 'url'})
with mock.patch.object(civis.io._files.pd, 'read_csv') as mock_read_csv:
civis.io.file_to_dataframe(121, compression='infer', client=m_client)
assert mock_read_csv.called_once_with(121, compression='gzip')
def test_sql_script():
sql = "SELECT SPECIAL SQL QUERY"
export_job_id = 32
database_id = 29
credential_id = 3920
response = Response({'id': export_job_id})
mock_client = create_client_mock()
mock_client.scripts.post_sql.return_value = response
mock_client.get_database_id.return_value = database_id
mock_client.default_credential = credential_id
civis.io._tables._sql_script(client=mock_client,
sql=sql,
database='fake-db',
job_name='My job',
credential_id=None,
hidden=False,
csv_settings=None)
mock_client.scripts.post_sql.assert_called_once_with(
'My job',
remote_host_id=database_id,
credential_id=credential_id,
sql=sql,
hidden=False,
csv_settings={})
mock_client.scripts.post_sql_runs.assert_called_once_with(export_job_id)
def _container_response_stub(from_template_id=8387):
arguments = {
'MODEL': 'sparse_logistic',
'TARGET_COLUMN': 'brushes_teeth_much',
'PRIMARY_KEY': 'voterbase_id',
'CALIBRATION': 'sigmoid',
'EXCLUDE_COLS': 'dog cat lizard frog',
'CVPARAMS': '{}',
'PARAMS': '{}',
'REQUIRED_CPU': 1000,
'REQUIRED_MEMORY': 9999,
'REQUIRED_DISK_SPACE': -20,
'DEPENDENCIES': 'A B C D',
'GIT_CRED': 9876
}
notifications = {
'urls': [],
'failureEmailAddresses': [],
'failureOn': True,
'stallWarningMinutes': None,
'successEmailAddresses': [],
'successEmailBody': None,
'successEmailSubject': None,
'successOn': True
}
return Response(dict(arguments=arguments,
notifications=notifications,
required_resources={},
docker_image_tag=None,
docker_command=None,
repo_http_uri=None,
repo_ref=None,
name='Civis Model Train',
from_template_id=from_template_id,
))
def test_modelpipeline_classmethod_constructor(mock_future,
container_response_stub):
mock_client = mock.Mock()
mock_client.scripts.get_containers.return_value = \
container = container_response_stub
mock_client.credentials.get.return_value = Response({'name': 'Token'})
resources = {
'REQUIRED_CPU': 1000,
'REQUIRED_MEMORY': 9999,
'REQUIRED_DISK_SPACE': -20
}
# test everything is working fine
mp = _model.ModelPipeline.from_existing(1, 1, client=mock_client)
assert isinstance(mp, _model.ModelPipeline)
assert mp.dependent_variable == [container.arguments['TARGET_COLUMN']]
assert mp.primary_key == container.arguments['PRIMARY_KEY']
excluded = container.arguments.get('EXCLUDE_COLS', None)
assert mp.excluded_columns == excluded.split() if excluded else None
assert mp.model == container.arguments['MODEL']
assert mp.calibration == container.arguments['CALIBRATION']
assert mp.cv_params == json.loads(container.arguments['CVPARAMS'])
assert mp.parameters == json.loads(container.arguments['PARAMS'])
assert mp.job_resources == resources
assert mp.model_name == container.name[:-6]
assert mp.notifications == {
camel_to_snake(key): val
for key, val in container.notifications.items()
}
deps = container.arguments.get('DEPENDENCIES', None)
assert mp.dependencies == deps.split() if deps else None
assert mp.git_token_name == 'Token'
def mock_job():
return Response(
dict(params=[{
'name': 'spam'
}],
arguments={'spam': 'eggs'},
**_MOCK_JOB_KWARGS))
def test_set_model_exception_no_exception(mock_f2j):
# If nothing went wrong, we shouldn't set an exception
ro = [{'name': 'model_info.json', 'object_id': 137, 'object_type': 'File'},
{'name': 'metrics.json', 'object_id': 139, 'object_type': 'File'}]
ro = [Response(o) for o in ro]
mock_client = setup_client_mock(1, 2, state='succeeded', run_outputs=ro)
fut = _model.ModelFuture(1, 2, client=mock_client)
assert fut.exception() is None
def test_poller_returns_none(self):
poller = mock.Mock(
side_effect=[None, None,
Response({'state': 'success'})])
polling_thread = _ResultPollingThread(poller, (),
polling_interval=0.01)
polling_thread.run()
assert poller.call_count == 3
def _set_api_exception(self, exc, result=None):
with self._condition:
if result is None:
result = Response({"state": FAILED[0]})
self._result = result
self._last_result = self._result
self.set_exception(exc)
self.cleanup()
def test_file_id_from_run_output_exact():
m_client = mock.Mock()
m_client.scripts.list_containers_runs_outputs.return_value = \
[Response({'name': 'spam', 'object_id': 2013,
'object_type': 'File'})]
fid = civis.io.file_id_from_run_output('spam', 17, 13, client=m_client)
assert fid == 2013
def test_result_callback_no_get(mock_civis):
# Test that the completed callback happens even if we don't call `get`
callback = mock.MagicMock()
mock_civis.io.civis_to_file.side_effect = make_to_file_mock('spam')
fut = ContainerFuture(1, 2, client=mock.MagicMock())
fut.set_result(Response({'state': 'success'}))
civis.parallel._CivisBackendResult(fut, callback)
assert callback.call_count == 1
def test_repeated_polling():
# Verify that we poll the expected number of times.
poll_interval = 0.2
poller = mock.Mock(return_value=Response({"state": "running"}))
pollable = PollableResult(poller, (), polling_interval=poll_interval)
pollable.done() # Check status once to start the polling thread
assert poller.call_count == 1, "Poll once on the first status check"
time.sleep(2.2 * poll_interval)
assert poller.call_count == 3, "After waiting 2.2x the polling interval"
def test_file_to_dataframe_kwargs():
m_client = mock.Mock()
m_client.files.get.return_value = Response({'name': 'spam.csv',
'file_url': 'url'})
with mock.patch.object(civis.io._files.pd, 'read_csv') as mock_read_csv:
civis.io.file_to_dataframe(121, compression='special', client=m_client,
delimiter='|', nrows=10)
assert mock_read_csv.called_once_with(121, compression='special',
delimiter='|', nrows=10)
def test_poll_on_creation(self):
poller = mock.Mock(return_value=Response({"state": "running"}))
pollable = PollableResult(poller, (),
polling_interval=0.01,
poll_on_creation=False)
pollable.done() # Check status once to start the polling thread
assert poller.call_count == 0
time.sleep(0.015)
assert poller.call_count == 1
def test_file_to_dataframe_infer():
m_client = mock.Mock()
url = 'url'
m_client.files.get.return_value = Response({'name': 'spam.csv',
'file_url': url})
with mock.patch.object(civis.io._files.pd, 'read_csv',
autospec=True) as mock_read_csv:
civis.io.file_to_dataframe(121, compression='infer', client=m_client)
mock_read_csv.assert_called_once_with(url, compression='infer')
def test_poll_on_creation(self):
poller = mock.Mock(side_effect=Response({"state": "running"}))
pollable = PollableResult(poller, (),
polling_interval=0.01,
poll_on_creation=False)
repr(pollable)
assert poller.call_count == 0
time.sleep(0.02)
assert poller.call_count == 1
def test_file_id_from_run_output_no_file():
# Get an IOError if we request a file which doesn't exist
m_client = mock.Mock()
m_client.scripts.list_containers_runs_outputs.return_value = [
Response({'name': 'spam', 'object_id': 2013,
'object_type': 'File'})]
with pytest.raises(FileNotFoundError) as err:
civis.io.file_id_from_run_output('eggs', 17, 13, client=m_client)
assert 'not an output' in str(err.value)
def test_file_id_from_run_output_approximate():
# Test fuzzy name matching
m_client = mock.Mock()
m_client.scripts.list_containers_runs_outputs.return_value = \
[Response({'name': 'spam.csv.gz', 'object_id': 2013,
'object_type': 'File'})]
fid = civis.io.file_id_from_run_output('^spam', 17, 13, regex=True,
client=m_client)
assert fid == 2013
def test_result_success(mock_civis):
# Test that we can get a result back from a succeeded job.
callback = mock.MagicMock()
mock_civis.io.civis_to_file.side_effect = make_to_file_mock('spam')
fut = ContainerFuture(1, 2, client=mock.MagicMock())
fut.set_result(Response({'state': 'success'}))
res = civis.parallel._CivisBackendResult(fut, callback)
assert res.get() == 'spam'
assert callback.call_count == 1
def test_file_to_dataframe_expired():
m_client = mock.Mock()
url = None
m_client.files.get.return_value = Response({
'name': 'spam.csv',
'file_url': url
})
expected_err = 'Unable to locate file 121. If it previously ' + \
'existed, it may have expired.'
with pytest.raises(EmptyResultError, match=expected_err):
civis.io.file_to_dataframe(121, client=m_client)
def test_timeout(self):
# Note: Something about the test framework seems to prevent the
# Pollable result from being destroyed while the polling
# thread is running. The test will hang if the PollableResult
# never completes. I haven't seen the same problem in
# the interpreter.
pollable = PollableResult(
mock.Mock(side_effect=[Response({"state": "running"}),
ValueError()]), (),
polling_interval=0.1)
pytest.raises(futures.TimeoutError, pollable.result, timeout=0.05)
def test_modelpipeline_init_err():
mock_client = mock.MagicMock()
r = Response({'content': None, 'status_code': 9999, 'reason': None})
mock_client.templates.get_scripts.side_effect = CivisAPIError(r)
with pytest.raises(NotImplementedError):
_model.ModelPipeline(LogisticRegression(),
'test',
etl=LogisticRegression(),
client=mock_client)
# clean up
del _model._NEWEST_CIVISML_VERSION
def mock_client_no_json_output():
mock_client = create_mock_client_with_job()
mock_output = [
Response({
"object_type": int,
"object_id": 10,
"name": "output",
"value": 100,
})
]
mock_client.scripts.list_custom_runs_outputs.return_value = mock_output
return mock_client
def test_modelpipeline_classmethod_constructor_defaults(
mock_future, container_response_stub):
del container_response_stub.arguments['PARAMS']
del container_response_stub.arguments['CVPARAMS']
mock_client = mock.Mock()
mock_client.scripts.get_containers.return_value = container_response_stub
mock_client.credentials.get.return_value = Response({'name': 'Token'})
# test everything is working fine
mp = _model.ModelPipeline.from_existing(1, 1, client=mock_client)
assert mp.cv_params == {}
assert mp.parameters == {}
def test_result_exception(mock_civis):
# An error in the job should be raised by the result
callback = mock.MagicMock()
exc = ZeroDivisionError()
mock_civis.io.civis_to_file.side_effect = make_to_file_mock(exc)
fut = ContainerFuture(1, 2, client=mock.MagicMock())
fut._set_api_exception(Response({'state': 'failed'}))
res = civis.parallel._CivisBackendResult(fut, callback)
with pytest.raises(ZeroDivisionError):
res.get()
assert callback.call_count == 0
def test_list_models():
resp = [Response({'id': 2834, 'name': 'RFC model'})]
m_client = create_client_mock()
m_client.scripts.list_custom.return_value = resp
out = list_models(job_type='train', client=m_client)
assert out == resp
out = list_models(job_type='predict', client=m_client)
assert out == resp
out = list_models(job_type=None, client=m_client)
assert out == resp
def test_state():
c = create_client_mock_for_container_tests(3, 7)
mf = _model.ModelFuture(3, 7, client=c)
ret = mf.state
assert ret == 'foo'
c.scripts.get_containers_runs.return_value = Response({'id': 7,
'container_id': 3,
'state': 'failed'})
mf = _model.ModelFuture(3, 7, client=c)
assert mf.state == 'failed'
def mock_child_job():
params = [{
'name': 'spam'
}, {
'name': 'CIVIS_PARENT_JOB_ID',
'value': '123'
}, {
'name': 'CIVIS_PARENT_RUN_ID',
'value': '456'
}]
args = {'spam': 'eggs'}
return Response(dict(params=params, arguments=args, **_MOCK_JOB_KWARGS))
def test_file_id_from_run_output_approximate_multiple():
# Fuzzy name matching with muliple matches should return the first
m_cl = mock.Mock()
m_cl.scripts.list_containers_runs_outputs.return_value = [
Response({
'name': 'spam.csv.gz',
'object_id': 2013,
'object_type': 'File'
}),
Response({
'name': 'eggs.csv.gz',
'object_id': 2014,
'object_type': 'File'
})
]
fid = civis.io.file_id_from_run_output('.csv',
17,
13,
regex=True,
client=m_cl)
assert fid == 2013
def test_result_eventual_success(mock_civis):
# Test that we can get a result back from a succeeded job,
# even if we need to retry a few times to succeed with the download.
callback = mock.MagicMock()
exc = requests.ConnectionError()
se = make_to_file_mock('spam', max_n_err=2, exc=exc)
mock_civis.io.civis_to_file.side_effect = se
fut = ContainerFuture(1, 2, client=mock.MagicMock())
fut.set_result(Response({'state': 'success'}))
res = civis.parallel._CivisBackendResult(fut, callback)
assert res.get() == 'spam'
assert callback.call_count == 1