def test_predictor_deduplicate_data(self):
n_points = 100
input_dataframe = pd.DataFrame({
'numeric_int': [x % 44 for x in list(range(n_points))],
'numeric_int_2': [x % 20 for x in list(range(n_points))],
}, index=list(range(n_points)))
input_dataframe['y'] = input_dataframe['numeric_int'] % 10
# Add duplicate row
input_dataframe = input_dataframe.append(input_dataframe.iloc[99], ignore_index=True)
mdb = Predictor(name='test_drop_duplicates')
mdb.learn(
from_data=input_dataframe,
to_predict='y',
stop_training_in_x_seconds=1,
use_gpu=False
)
model_data = F.get_model_data('test_drop_duplicates')
# Ensure duplicate row was not used for training, or analysis
assert model_data['data_preparation']['total_row_count'] == n_points
assert model_data['data_preparation']['used_row_count'] <= n_points
assert sum([model_data['data_preparation']['train_row_count'],
model_data['data_preparation']['validation_row_count'],
model_data['data_preparation']['test_row_count']]) == n_points
assert sum([mdb.transaction.input_data.train_df.shape[0],
mdb.transaction.input_data.test_df.shape[0],
mdb.transaction.input_data.validation_df.shape[0]]) == n_points
# Disable deduplication and ensure the duplicate row is used
mdb = Predictor(name='test_drop_duplicates')
mdb.learn(
from_data=input_dataframe,
to_predict='y',
stop_training_in_x_seconds=1,
use_gpu=False,
advanced_args={
'deduplicate_data': False
}
)
model_data = F.get_model_data('test_drop_duplicates')
# Duplicate row was used for analysis and training
assert model_data['data_preparation']['total_row_count'] == n_points+1
assert model_data['data_preparation']['used_row_count'] <= n_points+1
assert sum([model_data['data_preparation']['train_row_count'],
model_data['data_preparation']['validation_row_count'],
model_data['data_preparation']['test_row_count']]) == n_points+1
assert sum([mdb.transaction.input_data.train_df.shape[0],
mdb.transaction.input_data.test_df.shape[0],
mdb.transaction.input_data.validation_df.shape[0]]) == n_points+1
def test_category_tags_output(self):
vocab = random.sample(SMALL_VOCAB, 10)
vocab = {i: word for i, word in enumerate(vocab)}
# x1 contains the index of first tag present
# x2 contains the index of second tag present
# if a tag is missing then x1/x2 contain -1 instead
# Thus the dataset should be perfectly predicted
n_points = 5000
x1 = [
random.randint(0,
len(vocab) - 1) if random.random() > 0.1 else -1
for i in range(n_points)
]
x2 = [
random.randint(0,
len(vocab) - 1) if random.random() > 0.1 else -1
for i in range(n_points)
]
tags = []
for x1_index, x2_index in zip(x1, x2):
row_tags = set([vocab.get(x1_index), vocab.get(x2_index)])
row_tags = [x for x in row_tags if x is not None]
tags.append(','.join(row_tags))
df = pd.DataFrame({'x1': x1, 'x2': x2, 'tags': tags})
df_train = df.iloc[:round(n_points * 0.9)]
df_test = df.iloc[round(n_points * 0.9):]
predictor = Predictor('test')
predictor.learn(from_data=df_train,
to_predict='tags',
advanced_args=dict(deduplicate_data=False),
stop_training_in_x_seconds=60,
use_gpu=False)
model_data = F.get_model_data('test')
assert model_data['data_analysis_v2']['tags']['typing'][
'data_type'] == DATA_TYPES.CATEGORICAL
assert model_data['data_analysis_v2']['tags']['typing'][
'data_subtype'] == DATA_SUBTYPES.TAGS
predictions = predictor.predict(when_data=df_test)
test_tags = df_test.tags.apply(lambda x: x.split(','))
predicted_tags = []
for i in range(len(predictions)):
predicted_tags.append(predictions[i]['tags'])
test_tags_encoded = predictor.transaction.model_backend.predictor._mixer.encoders[
'tags'].encode(test_tags)
pred_labels_encoded = predictor.transaction.model_backend.predictor._mixer.encoders[
'tags'].encode(predicted_tags)
score = f1_score(test_tags_encoded,
pred_labels_encoded,
average='weighted')
assert score >= 0.3
def test_timeseries(self, tmp_path):
ts_hours = 12
data_len = 120
train_file_name = os.path.join(str(tmp_path), 'train_data.csv')
test_file_name = os.path.join(str(tmp_path), 'test_data.csv')
features = generate_value_cols(['date', 'int'], data_len,
ts_hours * 3600)
labels = [generate_timeseries_labels(features)]
feature_headers = list(map(lambda col: col[0], features))
label_headers = list(map(lambda col: col[0], labels))
# Create the training dataset and save it to a file
columns_train = list(
map(lambda col: col[1:int(len(col) * 3 / 4)], features))
columns_train.extend(
list(map(lambda col: col[1:int(len(col) * 3 / 4)], labels)))
columns_to_file(columns_train,
train_file_name,
headers=[*feature_headers, *label_headers])
# Create the testing dataset and save it to a file
columns_test = list(
map(lambda col: col[int(len(col) * 3 / 4):], features))
columns_to_file(columns_test, test_file_name, headers=feature_headers)
mdb = Predictor(name='test_timeseries')
mdb.learn(from_data=train_file_name,
to_predict=label_headers,
timeseries_settings={
'order_by': [feature_headers[0]],
'window': 3
},
stop_training_in_x_seconds=10,
use_gpu=False,
advanced_args={'force_predict': True})
results = mdb.predict(when_data=test_file_name, use_gpu=False)
for row in results:
expect_columns = [
label_headers[0], label_headers[0] + '_confidence'
]
for col in expect_columns:
assert col in row
models = F.get_models()
model_data = F.get_model_data(models[0]['name'])
assert model_data
def get_model_data(self, name, native_view=False):
model = F.get_model_data(name)
if native_view:
return model
data_analysis = model['data_analysis_v2']
for column in data_analysis['columns']:
if len(data_analysis[column]) == 0 or data_analysis[column].get(
'empty', {}).get('is_empty', False):
data_analysis[column]['typing'] = {
'data_subtype': DATA_SUBTYPES.INT
}
return model
def test_category_tags_input(self):
vocab = random.sample(SMALL_VOCAB, 10)
# tags contains up to 2 randomly selected tags
# y contains the sum of indices of tags
# the dataset should be nearly perfectly predicted
n_points = 5000
tags = []
y = []
for i in range(n_points):
row_tags = []
row_y = 0
for k in range(2):
if random.random() > 0.2:
selected_index = random.randint(0, len(vocab) - 1)
if vocab[selected_index] not in row_tags:
row_tags.append(vocab[selected_index])
row_y += selected_index
tags.append(','.join(row_tags))
y.append(row_y)
df = pd.DataFrame({'tags': tags, 'y': y})
df_train = df.iloc[:round(n_points * 0.9)]
df_test = df.iloc[round(n_points * 0.9):]
predictor = Predictor(name='test')
predictor.learn(from_data=df_train,
to_predict='y',
advanced_args=dict(deduplicate_data=False),
stop_training_in_x_seconds=40,
use_gpu=False)
model_data = F.get_model_data('test')
assert model_data['data_analysis_v2']['tags']['typing'][
'data_type'] == DATA_TYPES.CATEGORICAL
assert model_data['data_analysis_v2']['tags']['typing'][
'data_subtype'] == DATA_SUBTYPES.TAGS
predictions = predictor.predict(when_data=df_test)
test_y = df_test.y.apply(str)
predicted_y = []
for i in range(len(predictions)):
predicted_y.append(predictions[i]['y'])
score = accuracy_score(test_y, predicted_y)
assert score >= 0.2
def get_model_data(self, name, db_fix=True):
model = F.get_model_data(name)
# Make some corrections for databases not to break when dealing with empty columns
if db_fix:
data_analysis = model['data_analysis_v2']
for column in data_analysis['columns']:
analysis = data_analysis.get(column)
if isinstance(analysis,
dict) and (len(analysis) == 0 or analysis.get(
'empty', {}).get('is_empty', False)):
data_analysis[column]['typing'] = {
'data_subtype': DATA_SUBTYPES.INT
}
return model
def test_multilabel_prediction(self, tmp_path):
train_file_name = os.path.join(str(tmp_path), 'train_data.csv')
test_file_name = os.path.join(str(tmp_path), 'test_data.csv')
data_len = 60
features = generate_value_cols(['int', 'float', 'int', 'float'], data_len)
labels = []
labels.append(generate_log_labels(features))
labels.append(generate_timeseries_labels(features))
feature_headers = list(map(lambda col: col[0], features))
label_headers = list(map(lambda col: col[0], labels))
# Create the training dataset and save it to a file
columns_train = list(
map(lambda col: col[1:int(len(col) * 3 / 4)], features))
columns_train.extend(
list(map(lambda col: col[1:int(len(col) * 3 / 4)], labels)))
columns_to_file(columns_train, train_file_name,
headers=[*feature_headers, *label_headers])
# Create the testing dataset and save it to a file
columns_test = list(
map(lambda col: col[int(len(col) * 3 / 4):], features))
columns_to_file(columns_test, test_file_name,
headers=feature_headers)
mdb = Predictor(name='test_multilabel_prediction')
mdb.learn(
from_data=train_file_name,
to_predict=label_headers,
stop_training_in_x_seconds=1,
use_gpu=False,
advanced_args={'force_predict': True}
)
results = mdb.predict(when_data=test_file_name)
models = F.get_models()
model_data = F.get_model_data(models[0]['name'])
assert model_data
for i in range(len(results)):
row = results[i]
for label in label_headers:
expect_columns = [label, label + '_confidence']
for col in expect_columns:
assert col in row
def get_model_data(self, name, db_fix=True):
from mindsdb_native import F
from mindsdb_native.libs.constants.mindsdb import DATA_SUBTYPES
from mindsdb.interfaces.storage.db import session, Predictor
predictor_record = Predictor.query.filter_by(
company_id=self.company_id, name=name, is_custom=False).first()
predictor_record = self._try_outdate_db_status(predictor_record)
model = predictor_record.data
if model is None or model['status'] == 'training':
try:
self.fs_store.get(
name, f'predictor_{self.company_id}_{predictor_record.id}',
self.config['paths']['predictors'])
new_model_data = F.get_model_data(name)
except Exception:
new_model_data = None
if predictor_record.data is None or (
new_model_data is not None
and len(new_model_data) > len(predictor_record.data)):
predictor_record.data = new_model_data
model = new_model_data
session.commit()
# Make some corrections for databases not to break when dealing with empty columns
if db_fix:
data_analysis = model['data_analysis_v2']
for column in model['columns']:
analysis = data_analysis.get(column)
if isinstance(analysis,
dict) and (len(analysis) == 0 or analysis.get(
'empty', {}).get('is_empty', False)):
data_analysis[column]['typing'] = {
'data_subtype': DATA_SUBTYPES.INT
}
model['created_at'] = str(
parse_datetime(str(predictor_record.created_at).split('.')[0]))
model['updated_at'] = str(
parse_datetime(str(predictor_record.updated_at).split('.')[0]))
model['predict'] = predictor_record.to_predict
model['update'] = predictor_record.update_status
return self._pack(model)
def predict(self, name, when_data=None, kwargs={}):
if name not in self.predictor_cache:
# Clear the cache entirely if we have less than .12 GB left
if psutil.virtual_memory().available < 1.2 * pow(10, 9):
self.predictor_cache = {}
if F.get_model_data(name)['status'] == 'complete':
self.predictor_cache[name] = {
'predictor':
mindsdb_native.Predictor(name=name,
run_env={'trigger': 'mindsdb'}),
'created':
datetime.datetime.now()
}
predictions = self.predictor_cache[name]['predictor'].predict(
when_data=when_data, **kwargs)
return predictions
def test_house_pricing(self, use_gpu):
"""
Tests whole pipeline from downloading the dataset to making predictions and explanations.
"""
# Create & Learn
name = 'home_rentals_price'
mdb = Predictor(name=name)
mdb.learn(
to_predict='rental_price',
from_data=
"https://s3.eu-west-2.amazonaws.com/mindsdb-example-data/home_rentals.csv",
backend='lightwood',
stop_training_in_x_seconds=80,
use_gpu=use_gpu)
def assert_prediction_interface(predictions):
for prediction in predictions:
assert hasattr(prediction, 'explanation')
test_results = mdb.test(
when_data=
"https://s3.eu-west-2.amazonaws.com/mindsdb-example-data/home_rentals.csv",
accuracy_score_functions=r2_score,
predict_args={'use_gpu': use_gpu})
assert test_results['rental_price_accuracy'] >= 0.8
predictions = mdb.predict(
when_data=
"https://s3.eu-west-2.amazonaws.com/mindsdb-example-data/home_rentals.csv",
use_gpu=use_gpu)
assert_prediction_interface(predictions)
predictions = mdb.predict(when_data={'sqft': 300}, use_gpu=use_gpu)
assert_prediction_interface(predictions)
amd = F.get_model_data(name)
assert isinstance(json.dumps(amd), str)
for k in [
'status', 'name', 'version', 'data_source', 'current_phase',
'updated_at', 'created_at', 'train_end_at'
]:
assert isinstance(amd[k], str)
assert isinstance(amd['predict'], (list, str))
assert isinstance(amd['is_active'], bool)
for k in ['validation_set_accuracy', 'accuracy']:
assert isinstance(amd[k], float)
for k in amd['data_preparation']:
assert isinstance(amd['data_preparation'][k], (int, float))
for k in amd['data_analysis']:
assert (len(amd['data_analysis'][k]) > 0)
assert isinstance(amd['data_analysis'][k][0], dict)
model_analysis = amd['model_analysis']
assert (len(model_analysis) > 0)
assert isinstance(model_analysis[0], dict)
input_importance = model_analysis[0]["overall_input_importance"]
assert (len(input_importance) > 0)
assert isinstance(input_importance, dict)
for k in ['train', 'test', 'valid']:
assert isinstance(model_analysis[0][k + '_data_accuracy'], dict)
assert len(model_analysis[0][k + '_data_accuracy']) == 1
assert model_analysis[0][k +
'_data_accuracy']['rental_price'] > 0.4
for column, importance in zip(input_importance["x"],
input_importance["y"]):
assert isinstance(column, str)
assert (len(column) > 0)
assert isinstance(importance, (float, int))
assert (importance >= 0 and importance <= 10)
# Test confidence estimation after save -> load
p = None
F.export_predictor(name)
F.import_model(f"{name}.zip", f"{name}-new")
p = Predictor(name=f'{name}-new')
predictions = p.predict(when_data={'sqft': 1000},
use_gpu=use_gpu,
run_confidence_variation_analysis=True)
assert_prediction_interface(predictions)
def get_model_data(self, name):
return F.get_model_data(name)