def test_empty_dataset_warning_message():
a1 = autom8.Accumulator()
a2 = autom8.Accumulator()
a3 = autom8.Accumulator()
autom8.create_matrix([[]], receiver=a1)
autom8.create_matrix([[], []], receiver=a2)
autom8.create_matrix([[], [], []], receiver=a3)
assert a1.warnings == ['Dropped 1 empty row from dataset.']
assert a2.warnings == ['Dropped 2 empty rows from dataset.']
assert a3.warnings == ['Dropped 3 empty rows from dataset.']
def test_columns_with_numbers_as_strings():
dataset = [
['A', 'B', 'C'],
['1.1', '$4', 7],
['2.2', '$5', 8],
['3.3', '6%', 9],
]
acc = autom8.Accumulator()
matrix = autom8.create_matrix(_add_labels(dataset), receiver=acc)
ctx = autom8.create_context(matrix, receiver=acc)
autom8.clean_dataset(ctx)
assert len(acc.warnings) == 0
assert len(ctx.steps) == 2
assert ctx.matrix.tolist() == [[1.1, 4, 7], [2.2, 5, 8], [3.3, 6, 9]]
vectors = [['A', 'B', 'C'], [1, '2%', 'foo'], ['3', 4.0, 'bar']]
matrix = autom8.create_matrix(vectors, receiver=acc)
out = PlaybackContext(matrix, receiver=acc)
playback(ctx.steps, out)
assert out.matrix.tolist() == [[1, 2, 'foo'], [3, 4, 'bar']]
assert out.matrix.columns[0].dtype == int
assert out.matrix.columns[1].dtype == float
def test_column_with_some_blank_strings():
# Repeat the previous test, only replace most of the empty strings with
# blank strings.
dataset = [
['A', 'B', 'C'],
[True, 1.1, 20],
[' ', 2.2, 30],
[False, '\t', 40],
[False, 3.3, ' \t \r \n\t'],
['', 4.4, ' '],
]
acc = autom8.Accumulator()
matrix = autom8.create_matrix(_add_labels(dataset), receiver=acc)
ctx = autom8.create_context(matrix, receiver=acc)
autom8.clean_dataset(ctx)
assert ctx.matrix.tolist() == [
[True, True, 1.1, True, 20, True],
[False, False, 2.2, True, 30, True],
[False, True, 0.0, False, 40, True],
[False, True, 3.3, True, 0, False],
[False, False, 4.4, True, 0, False],
]
assert ctx.matrix.formulas == [
'A',
['is-defined', 'A'],
'B',
['is-defined', 'B'],
'C',
['is-defined', 'C'],
]
def test_creating_simple_matrix_with_names_and_roles():
acc = autom8.Accumulator()
matrix = autom8.create_matrix(
dataset=[['hi', True], ['bye', False]],
column_names=['msg', 'flag'],
column_roles=['textual', 'encoded'],
receiver=acc,
)
c1, c2 = matrix.columns
e1 = np.array(['hi', 'bye'], dtype=object)
e2 = np.array([True, False], dtype=None)
assert np.array_equal(c1.values, e1)
assert np.array_equal(c2.values, e2)
assert c1.name == 'msg'
assert c2.name == 'flag'
assert c1.role == 'textual'
assert c2.role == 'encoded'
assert c1.is_original
assert c2.is_original
assert len(acc.warnings) == 0
def test_column_of_ints_and_floats():
dataset = [
['A', 'B'],
[1, 3.3],
[2.2, 4],
[None, None],
]
acc = autom8.Accumulator()
matrix = autom8.create_matrix(_add_labels(dataset), receiver=acc)
ctx = autom8.create_context(matrix, receiver=acc)
autom8.clean_dataset(ctx)
assert len(ctx.steps) == 4
assert len(acc.warnings) == 2
assert ctx.matrix.tolist() == [
[1.0, True, 3.3, True],
[2.2, True, 4.0, True],
[0.0, False, 0.0, False],
]
vectors = [['A', 'B'], [None, 10], [20.0, None], [30, 40]]
matrix = autom8.create_matrix(vectors, receiver=acc)
out = PlaybackContext(matrix, receiver=acc)
playback(ctx.steps, out)
assert out.matrix.tolist() == [
[0.0, False, 10.0, True],
[20.0, True, 0.0, False],
[30.0, True, 40.0, True],
]
assert out.matrix.columns[0].dtype == float
assert out.matrix.columns[2].dtype == float
def test_creating_simple_matrix_from_list():
acc = autom8.Accumulator()
matrix = autom8.create_matrix(
[['hi', 1, True], ['bye', 2, False]],
receiver=acc,
)
c1, c2, c3 = matrix.columns
e1 = np.array(['hi', 'bye'], dtype=object)
e2 = np.array([1, 2], dtype=None)
e3 = np.array([True, False], dtype=None)
assert np.array_equal(c1.values, e1)
assert np.array_equal(c2.values, e2)
assert np.array_equal(c3.values, e3)
assert c1.name == 'A'
assert c2.name == 'B'
assert c3.name == 'C'
assert c1.role is None
assert c2.role is None
assert c3.role is None
assert c1.is_original
assert c2.is_original
assert c3.is_original
assert len(acc.warnings) == 0
def test_one_hot_encode_categories_when_something_goes_wrong():
import autom8.categories
features = [
[1, 10, 'foo', 'bar', -1.0],
[2, 20, 'bar', 'foo', -1.0],
[3, -1, 'foo', 'foo', -1.0],
]
roles = ['numerical'] + ['categorical'] * 4
matrix = _create_matrix(features, roles)
ctx = _create_context(features, roles)
autom8.encode_categories(ctx, method='one-hot', only_strings=False)
encoder = ctx.steps[0].args[0]
acc = autom8.Accumulator()
plc = PlaybackContext(matrix, receiver=acc)
# As in the previous test, just monkey-patch in a "steps" list.
# (Again, this is pretty terrible.)
ctx.steps = []
# Break the encoder so that our function will raise an exception.
encoder.transform = None
autom8.categories.encode(plc, encoder, [1, 2, 3, 4])
assert ctx.matrix.formulas == plc.matrix.formulas
assert plc.matrix.tolist() == [
[1, 0, 0, 0, 0, 0, 0, 0, 0],
[2, 0, 0, 0, 0, 0, 0, 0, 0],
[3, 0, 0, 0, 0, 0, 0, 0, 0],
]
assert len(acc.warnings) == 1
def test_is_recording_property():
matrix = autom8.create_matrix([[1, 2]])
c1 = autom8.create_context(matrix)
c2 = PlaybackContext(matrix, autom8.Accumulator())
assert c1.is_recording
assert not c2.is_recording
assert hasattr(c1, 'receiver')
assert hasattr(c2, 'receiver')
def test_duplicate_column_names():
acc = autom8.Accumulator()
matrix = autom8.create_matrix(
dataset=[[1, 2, 3]],
column_names=['A', 'B', 'A'],
receiver=acc,
)
assert len(acc.warnings) == 1
assert 'Column names are not unique' in acc.warnings[0]
def _playback(fitted, roles, features, receiver=None):
if receiver is None:
receiver = autom8.Accumulator()
matrix = _create_matrix(features, roles)
ctx = PlaybackContext(matrix, receiver=receiver)
playback(fitted.steps, ctx)
assert fitted.matrix.formulas == ctx.matrix.formulas
return ctx.matrix.tolist()
def test_columns_with_numbers_with_commas():
dataset = [['A'], ['1,100.0'], ['2,200'], ['3,300'], ['50']]
acc = autom8.Accumulator()
matrix = autom8.create_matrix(_add_labels(dataset), receiver=acc)
ctx = autom8.create_context(matrix, receiver=acc)
autom8.clean_dataset(ctx)
assert len(acc.warnings) == 0
assert len(ctx.steps) == 1
assert ctx.matrix.tolist() == [[1100], [2200], [3300], [50]]
def run(name):
dataset = load(name)
acc = autom8.Accumulator()
with warnings.catch_warnings():
warnings.simplefilter('ignore')
autom8.run(dataset, receiver=acc)
try_json_encoding_everything(acc)
return acc
def test_planner_decorator():
matrix = autom8.create_matrix([[1, 1], [2, 2]])
c1 = autom8.create_context(matrix)
c2 = PlaybackContext(matrix, autom8.Accumulator())
# This should not raise an exception.
autom8.drop_duplicate_columns(c1)
# But this should raise one.
with pytest.raises(autom8.Autom8Exception) as excinfo:
autom8.drop_duplicate_columns(c2)
excinfo.match('Expected.*RecordingContext')
def test_extra_columns_warning_message():
a1 = autom8.Accumulator()
a2 = autom8.Accumulator()
m1 = autom8.create_matrix([[1, 2], [1, 2, 3]], receiver=a1)
m2 = autom8.create_matrix([[1], [1, 2, 3], [1, 2, 3, 4], [1, 2, 3, 4]],
receiver=a2)
assert len(m1.columns), 2
assert a1.warnings == [
'Dropped 1 extra column from dataset.'
' Keeping first 2 columns.'
' To avoid this behavior, ensure that each row in the dataset has'
' the same number of columns.'
]
assert len(m2.columns), 1
assert a2.warnings == [
'Dropped 3 extra columns from dataset.'
' Keeping first 1 column.'
' To avoid this behavior, ensure that each row in the dataset has'
' the same number of columns.'
]
def test_mixed_up_columns_with_strings_and_numbers():
dataset = [
['A', 'B'],
[True, 'foo'],
[1.1, 30],
[20, 4.4],
['bar', False],
['', 'baz'],
[50, 'fiz'],
[None, True],
]
acc = autom8.Accumulator()
matrix = autom8.create_matrix(_add_labels(dataset), receiver=acc)
ctx = autom8.create_context(matrix, receiver=acc)
autom8.clean_dataset(ctx)
assert len(ctx.steps) == 6
assert len(acc.warnings) == 0
assert ctx.matrix.tolist() == [
[1.0, '', 0.0, 'foo'],
[1.1, '', 30.0, ''],
[20.0, '', 4.4, ''],
[0.0, 'bar', 0.0, ''],
[0.0, '', 0.0, 'baz'],
[50.0, '', 0.0, 'fiz'],
[0.0, '', 1.0, ''],
]
assert ctx.matrix.formulas == [
['number', 'A'],
['string', 'A'],
['number', 'B'],
['string', 'B'],
]
vectors = [['A', 'B'], [False, 'buz'], ['zim', 10], [2, None]]
matrix = autom8.create_matrix(vectors, receiver=acc)
out = PlaybackContext(matrix, receiver=acc)
playback(ctx.steps, out)
assert out.matrix.tolist() == [
[0.0, '', 0.0, 'buz'],
[0.0, 'zim', 10.0, ''],
[2.0, '', 0.0, ''],
]
assert out.matrix.formulas == [
['number', 'A'],
['string', 'A'],
['number', 'B'],
['string', 'B'],
]
def test_clean_numeric_labels():
dataset = [
['A', 'B', 'C'],
[1, 2, '3'],
[3, 4, '4'],
[5, 6, 5],
[7, 8, None],
[9, 9, ''],
]
acc = autom8.Accumulator()
ctx = autom8.create_context(dataset, receiver=acc)
assert len(acc.warnings) == 1
assert ctx.labels.original.tolist() == [3, 4, 5, 0, 0]
def test_columns_with_some_empty_strings():
dataset = [
['A', 'B', 'C'],
[True, 1.1, 20],
['', 2.2, 30],
[False, '', 40],
[False, 3.3, ''],
['', 4.4, ''],
]
acc = autom8.Accumulator()
matrix = autom8.create_matrix(_add_labels(dataset), receiver=acc)
ctx = autom8.create_context(matrix, receiver=acc)
autom8.clean_dataset(ctx)
assert len(ctx.steps) == 6
assert len(acc.warnings) == 3
assert ctx.matrix.tolist() == [
[True, True, 1.1, True, 20, True],
[False, False, 2.2, True, 30, True],
[False, True, 0.0, False, 40, True],
[False, True, 3.3, True, 0, False],
[False, False, 4.4, True, 0, False],
]
assert ctx.matrix.formulas == [
'A',
['is-defined', 'A'],
'B',
['is-defined', 'B'],
'C',
['is-defined', 'C'],
]
vectors = [['A', 'B', 'C'], ['', 5.5, ''], [True, '', 50]]
matrix = autom8.create_matrix(vectors, receiver=acc)
out = PlaybackContext(matrix, receiver=acc)
playback(ctx.steps, out)
assert out.matrix.tolist() == [
[False, False, 5.5, True, 0, False],
[True, True, 0.0, False, 50, True],
]
assert out.matrix.formulas == [
'A',
['is-defined', 'A'],
'B',
['is-defined', 'B'],
'C',
['is-defined', 'C'],
]
def test_creating_simple_matrix_from_numpy_array():
acc = autom8.Accumulator()
matrix = autom8.create_matrix(
np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]]),
receiver=acc,
)
c1, c2, c3 = matrix.columns
e1 = np.array([1, 4, 7, 10], dtype=object)
e2 = np.array([2, 5, 8, 11], dtype=None)
e3 = np.array([3, 6, 9, 12], dtype=None)
assert np.array_equal(c1.values, e1)
assert np.array_equal(c2.values, e2)
assert np.array_equal(c3.values, e3)
def test_column_of_all_strings():
dataset = [
['A', 'B'],
['1', 2],
['3', 4],
['n', 0],
]
acc = autom8.Accumulator()
matrix = autom8.create_matrix(_add_labels(dataset), receiver=acc)
ctx = autom8.create_context(matrix, receiver=acc)
autom8.clean_dataset(ctx)
assert len(acc.warnings) == 0
assert len(ctx.steps) == 0
assert ctx.matrix.tolist() == [['1', 2], ['3', 4], ['n', 0]]
def test_boston_dataset():
acc = datasets.run('boston.csv')
# Assert that we at least got 10 candidates.
assert len(acc.candidates) >= 10
# Make sure each candidate has an r2_score.
for candidate in acc.candidates:
s1 = candidate.train.metrics['r2_score']
s2 = candidate.test.metrics['r2_score']
assert s1 <= 1.0
assert s2 <= 1.0
assert isinstance(s1, float)
assert isinstance(s2, float)
# Assert that the best test score is better than 0.6.
best = max(i.test.metrics['r2_score'] for i in acc.candidates)
assert best > 0.6
# Make sure each pipeline can make predictions.
vectors = [
[
'CRIM',
'ZN',
'INDUS',
'CHAS',
'NOX',
'RM',
'AGE',
'DIS',
'RAD',
'TAX',
'PTRATIO',
'B',
'LSTAT',
],
[0.007, 18, 2.3, 0, 0.5, 6.5, 65, 4, 1, 296, 15.4, 396.9, 4.98],
[0.05, 0, 2.4, 0, 0.5, 7.8, 53, 3, 3, 193, 18, 392.63, 4.45],
]
for candidate in acc.candidates:
tmp = autom8.Accumulator()
pred = candidate.pipeline.run(vectors, receiver=tmp)
assert len(pred.predictions) == 2
assert isinstance(pred.predictions[0], float)
assert isinstance(pred.predictions[1], float)
assert not tmp.warnings
def test_evaluate_pipeline():
acc = autom8.Accumulator()
inputs = [
[1, 2],
[3, 4],
[5, 6],
[7, 8],
[9, 10],
[11, 12],
[13, 14],
[15, 16],
]
dataset = [i + [i[0] + i[1]] for i in inputs]
ctx = autom8.create_context(dataset, receiver=acc)
# For now, just hack in the test_indices that we want.
ctx.test_indices = [2, 5]
autom8.add_column_of_ones(ctx)
ctx << sklearn.linear_model.LinearRegression()
assert len(acc.candidates) == 1
candidate = acc.candidates[0]
assert candidate.train.metrics['r2_score'] == 1.0
assert candidate.test.metrics['r2_score'] == 1.0
assert np.allclose(
candidate.train.predictions,
np.array([1 + 2, 3 + 4, 7 + 8, 9 + 10, 13 + 14, 15 + 16]),
)
assert np.allclose(
candidate.test.predictions,
np.array([5 + 6, 11 + 12]),
)
# Try using the pipeline to make some predictions.
result = candidate.pipeline.run([[17, 18], [19, 20], [21, 22]],
receiver=acc)
assert np.allclose(result.predictions,
np.array([17 + 18, 19 + 20, 21 + 22]))
assert result.probabilities is None
assert not acc.warnings
def test_ignoring_text_columns():
vocab = ['foo', 'bar', 'baz', 'zim', 'zam', 'fiz', 'buz', 'bim', 'bam',
'call', 'step', 'term', 'type', 'protocol', 'hand', 'head', 'foot']
random_range = lambda: range(random.randint(2, 12))
random_word = lambda: random.choice(vocab)
new_text = lambda: ' '.join(random_word() for _ in random_range())
dataset = [[row[0], new_text(), row[1]] for row in _make_dataset()]
acc = autom8.Accumulator()
with warnings.catch_warnings():
warnings.simplefilter('ignore')
autom8.run(dataset, receiver=acc)
did_ignore_text = any('drop_text_columns' in step.func.__name__
for candidate in acc.candidates
for step in candidate.pipeline.steps)
assert did_ignore_text
def test_primitives_with_object_dtype():
dataset = [
['A', 'B', 'C'],
[True, 1.1, 2],
[False, 3.1, 4],
[True, 5.1, 6],
]
acc = autom8.Accumulator()
matrix = autom8.create_matrix(_add_labels(dataset), receiver=acc)
for col in matrix.columns:
col.values = col.values.astype(object)
ctx = autom8.create_context(matrix, receiver=acc)
autom8.clean_dataset(ctx)
dtypes = [c.dtype for c in ctx.matrix.columns]
assert dtypes[0] == bool
assert dtypes[1] == float
assert dtypes[2] == int
vectors = [['A', 'B', 'C'], [1, 2, 3.0], [0, 4, 5.0], [1, False, 6.9]]
matrix = autom8.create_matrix(vectors, receiver=acc)
out = PlaybackContext(matrix, receiver=acc)
playback(ctx.steps, out)
assert out.matrix.tolist() == [[True, 2.0, 3], [False, 4.0, 5],
[True, 0.0, 6]]
dtypes = [c.dtype for c in out.matrix.columns]
assert dtypes[0] == bool
assert dtypes[1] == float
assert dtypes[2] == int
vectors = [['A', 'B', 'C'], ['1', '2', None], ['', None, ()]]
matrix = autom8.create_matrix(vectors, receiver=acc)
out = PlaybackContext(matrix, receiver=acc)
playback(ctx.steps, out)
# Just use repr to avoid having to fart around with nan.
assert repr(out.matrix.tolist()) == ("[[True, 2.0, 0], [False, nan, 0]]")
def test_matrix_with_unexpected_value():
dataset = [
['A', 'B', 'C'],
[1, 2, ()],
[3, 4, {}],
[5, 6, object()],
]
acc = autom8.Accumulator()
matrix = autom8.create_matrix(_add_labels(dataset), receiver=acc)
ctx = autom8.create_context(matrix, receiver=acc)
autom8.clean_dataset(ctx)
assert len(acc.warnings) == 1
assert 'Dropping column' in acc.warnings[0]
assert 'contain booleans, numbers' in acc.warnings[0]
assert ctx.matrix.tolist() == [[1, 2], [3, 4], [5, 6]]
vectors = [['A', 'B', 'C'], [1, 2, 'foo'], [3, 4, 'bar']]
matrix = autom8.create_matrix(vectors, receiver=acc)
out = PlaybackContext(matrix, receiver=acc)
playback(ctx.steps, out)
assert out.matrix.tolist() == [[1, 2], [3, 4]]
def test_column_of_all_strings_and_none_values():
dataset = [
['A', 'B'],
['1', 2],
['foo', 4],
[None, 0],
]
acc = autom8.Accumulator()
matrix = autom8.create_matrix(_add_labels(dataset), receiver=acc)
ctx = autom8.create_context(matrix, receiver=acc)
autom8.clean_dataset(ctx)
assert len(acc.warnings) == 0
assert len(ctx.steps) == 1
assert ctx.matrix.tolist() == [['1', 2], ['foo', 4], ['', 0]]
vectors = [['A', 'B'], [None, 'bar'], ['baz', None]]
matrix = autom8.create_matrix(vectors, receiver=acc)
out = PlaybackContext(matrix, receiver=acc)
playback(ctx.steps, out)
assert out.matrix.tolist() == [['', 'bar'], ['baz', None]]
def test_column_with_all_none():
dataset = [
['A', 'B', 'C'],
[True, None, 2],
[False, None, 4],
[True, None, 6],
]
acc = autom8.Accumulator()
matrix = autom8.create_matrix(_add_labels(dataset), receiver=acc)
ctx = autom8.create_context(matrix, receiver=acc)
autom8.clean_dataset(ctx)
assert len(acc.warnings) == 1
assert 'Dropping column' in acc.warnings[0]
assert ctx.matrix.tolist() == [[True, 2], [False, 4], [True, 6]]
vectors = [['A', 'B', 'C'], [1, 2, 'foo'], [3, 4, 'bar']]
matrix = autom8.create_matrix(vectors, receiver=acc)
out = PlaybackContext(matrix, receiver=acc)
playback(ctx.steps, out)
assert out.matrix.tolist() == [[1, 'foo'], [3, 'bar']]
def test_run():
dataset = _make_dataset()
acc = autom8.Accumulator()
with warnings.catch_warnings():
warnings.simplefilter('ignore')
autom8.run(dataset, receiver=acc)
# Assert that we at least got 10 candidates.
assert len(acc.candidates) >= 10
# Make sure they all have an r2_score.
for candidate in acc.candidates:
s1 = candidate.train.metrics['r2_score']
s2 = candidate.test.metrics['r2_score']
assert isinstance(s1, float)
assert isinstance(s2, float)
# Make sure the best scores are at least 0.5.
best_train = max(r.train.metrics['r2_score'] for r in acc.candidates)
best_test = max(r.test.metrics['r2_score'] for r in acc.candidates)
assert best_train > 0.5
assert best_test > 0.5
def check_classifier_predictions(acc, valid_labels, vectors):
for candidate in acc.candidates:
tmp = autom8.Accumulator()
pred = candidate.pipeline.run(vectors, receiver=tmp)
assert not tmp.warnings
# Just make sure that we can json-encode the predictions.
json.dumps(pred)
assert len(pred.predictions) == len(vectors) - 1
for label in pred.predictions:
assert label in valid_labels
if pred.probabilities is None:
continue
assert len(pred.probabilities) == len(pred.predictions)
for probs in pred.probabilities:
# Make sure we have between one and three pairs.
assert 1 <= len(probs) <= 3
# Make sure each pair is a valid label and a valid probability.
for label, score in probs:
assert label in valid_labels
assert 0 < score <= 1
# Make sure that the probabilities are sorted from highest to lowest.
scores = [score for _, score in probs]
assert sorted(scores, reverse=True) == scores
# Make sure that they don't all add up to something greater than 1.
0 < sum(score for _, score in probs) <= 1
# Make sure that the prediction is the first label.
for label, probs in zip(pred.predictions, pred.probabilities):
assert label == probs[0][0]
def test_empty_datasets():
for data in [[], (), np.array([]), autom8.Matrix([])]:
acc = autom8.Accumulator()
matrix = autom8.create_matrix(data, receiver=acc)
assert len(matrix.columns) == 0
assert len(acc.warnings) == 0
def test_empty_dataset_with_empty_rows():
# Assert that we see one warning when we have three empty rows.
acc = autom8.Accumulator()
matrix = autom8.create_matrix([[], [], []], receiver=acc)
assert len(matrix.columns) == 0
assert len(acc.warnings) == 1
