def test_predictor_deduplicate_data(self): n_points = 100 input_dataframe = pd.DataFrame({ 'numeric_int': [x % 44 for x in list(range(n_points))], 'numeric_int_2': [x % 20 for x in list(range(n_points))], }, index=list(range(n_points))) input_dataframe['y'] = input_dataframe['numeric_int'] % 10 # Add duplicate row input_dataframe = input_dataframe.append(input_dataframe.iloc[99], ignore_index=True) mdb = Predictor(name='test_drop_duplicates') mdb.learn( from_data=input_dataframe, to_predict='y', stop_training_in_x_seconds=1, use_gpu=False ) model_data = F.get_model_data('test_drop_duplicates') # Ensure duplicate row was not used for training, or analysis assert model_data['data_preparation']['total_row_count'] == n_points assert model_data['data_preparation']['used_row_count'] <= n_points assert sum([model_data['data_preparation']['train_row_count'], model_data['data_preparation']['validation_row_count'], model_data['data_preparation']['test_row_count']]) == n_points assert sum([mdb.transaction.input_data.train_df.shape[0], mdb.transaction.input_data.test_df.shape[0], mdb.transaction.input_data.validation_df.shape[0]]) == n_points # Disable deduplication and ensure the duplicate row is used mdb = Predictor(name='test_drop_duplicates') mdb.learn( from_data=input_dataframe, to_predict='y', stop_training_in_x_seconds=1, use_gpu=False, advanced_args={ 'deduplicate_data': False } ) model_data = F.get_model_data('test_drop_duplicates') # Duplicate row was used for analysis and training assert model_data['data_preparation']['total_row_count'] == n_points+1 assert model_data['data_preparation']['used_row_count'] <= n_points+1 assert sum([model_data['data_preparation']['train_row_count'], model_data['data_preparation']['validation_row_count'], model_data['data_preparation']['test_row_count']]) == n_points+1 assert sum([mdb.transaction.input_data.train_df.shape[0], mdb.transaction.input_data.test_df.shape[0], mdb.transaction.input_data.validation_df.shape[0]]) == n_points+1
def test_category_tags_output(self): vocab = random.sample(SMALL_VOCAB, 10) vocab = {i: word for i, word in enumerate(vocab)} # x1 contains the index of first tag present # x2 contains the index of second tag present # if a tag is missing then x1/x2 contain -1 instead # Thus the dataset should be perfectly predicted n_points = 5000 x1 = [ random.randint(0, len(vocab) - 1) if random.random() > 0.1 else -1 for i in range(n_points) ] x2 = [ random.randint(0, len(vocab) - 1) if random.random() > 0.1 else -1 for i in range(n_points) ] tags = [] for x1_index, x2_index in zip(x1, x2): row_tags = set([vocab.get(x1_index), vocab.get(x2_index)]) row_tags = [x for x in row_tags if x is not None] tags.append(','.join(row_tags)) df = pd.DataFrame({'x1': x1, 'x2': x2, 'tags': tags}) df_train = df.iloc[:round(n_points * 0.9)] df_test = df.iloc[round(n_points * 0.9):] predictor = Predictor('test') predictor.learn(from_data=df_train, to_predict='tags', advanced_args=dict(deduplicate_data=False), stop_training_in_x_seconds=60, use_gpu=False) model_data = F.get_model_data('test') assert model_data['data_analysis_v2']['tags']['typing'][ 'data_type'] == DATA_TYPES.CATEGORICAL assert model_data['data_analysis_v2']['tags']['typing'][ 'data_subtype'] == DATA_SUBTYPES.TAGS predictions = predictor.predict(when_data=df_test) test_tags = df_test.tags.apply(lambda x: x.split(',')) predicted_tags = [] for i in range(len(predictions)): predicted_tags.append(predictions[i]['tags']) test_tags_encoded = predictor.transaction.model_backend.predictor._mixer.encoders[ 'tags'].encode(test_tags) pred_labels_encoded = predictor.transaction.model_backend.predictor._mixer.encoders[ 'tags'].encode(predicted_tags) score = f1_score(test_tags_encoded, pred_labels_encoded, average='weighted') assert score >= 0.3
def test_timeseries(self, tmp_path): ts_hours = 12 data_len = 120 train_file_name = os.path.join(str(tmp_path), 'train_data.csv') test_file_name = os.path.join(str(tmp_path), 'test_data.csv') features = generate_value_cols(['date', 'int'], data_len, ts_hours * 3600) labels = [generate_timeseries_labels(features)] feature_headers = list(map(lambda col: col[0], features)) label_headers = list(map(lambda col: col[0], labels)) # Create the training dataset and save it to a file columns_train = list( map(lambda col: col[1:int(len(col) * 3 / 4)], features)) columns_train.extend( list(map(lambda col: col[1:int(len(col) * 3 / 4)], labels))) columns_to_file(columns_train, train_file_name, headers=[*feature_headers, *label_headers]) # Create the testing dataset and save it to a file columns_test = list( map(lambda col: col[int(len(col) * 3 / 4):], features)) columns_to_file(columns_test, test_file_name, headers=feature_headers) mdb = Predictor(name='test_timeseries') mdb.learn(from_data=train_file_name, to_predict=label_headers, timeseries_settings={ 'order_by': [feature_headers[0]], 'window': 3 }, stop_training_in_x_seconds=10, use_gpu=False, advanced_args={'force_predict': True}) results = mdb.predict(when_data=test_file_name, use_gpu=False) for row in results: expect_columns = [ label_headers[0], label_headers[0] + '_confidence' ] for col in expect_columns: assert col in row models = F.get_models() model_data = F.get_model_data(models[0]['name']) assert model_data
def get_model_data(self, name, native_view=False): model = F.get_model_data(name) if native_view: return model data_analysis = model['data_analysis_v2'] for column in data_analysis['columns']: if len(data_analysis[column]) == 0 or data_analysis[column].get( 'empty', {}).get('is_empty', False): data_analysis[column]['typing'] = { 'data_subtype': DATA_SUBTYPES.INT } return model
def test_category_tags_input(self): vocab = random.sample(SMALL_VOCAB, 10) # tags contains up to 2 randomly selected tags # y contains the sum of indices of tags # the dataset should be nearly perfectly predicted n_points = 5000 tags = [] y = [] for i in range(n_points): row_tags = [] row_y = 0 for k in range(2): if random.random() > 0.2: selected_index = random.randint(0, len(vocab) - 1) if vocab[selected_index] not in row_tags: row_tags.append(vocab[selected_index]) row_y += selected_index tags.append(','.join(row_tags)) y.append(row_y) df = pd.DataFrame({'tags': tags, 'y': y}) df_train = df.iloc[:round(n_points * 0.9)] df_test = df.iloc[round(n_points * 0.9):] predictor = Predictor(name='test') predictor.learn(from_data=df_train, to_predict='y', advanced_args=dict(deduplicate_data=False), stop_training_in_x_seconds=40, use_gpu=False) model_data = F.get_model_data('test') assert model_data['data_analysis_v2']['tags']['typing'][ 'data_type'] == DATA_TYPES.CATEGORICAL assert model_data['data_analysis_v2']['tags']['typing'][ 'data_subtype'] == DATA_SUBTYPES.TAGS predictions = predictor.predict(when_data=df_test) test_y = df_test.y.apply(str) predicted_y = [] for i in range(len(predictions)): predicted_y.append(predictions[i]['y']) score = accuracy_score(test_y, predicted_y) assert score >= 0.2
def get_model_data(self, name, db_fix=True): model = F.get_model_data(name) # Make some corrections for databases not to break when dealing with empty columns if db_fix: data_analysis = model['data_analysis_v2'] for column in data_analysis['columns']: analysis = data_analysis.get(column) if isinstance(analysis, dict) and (len(analysis) == 0 or analysis.get( 'empty', {}).get('is_empty', False)): data_analysis[column]['typing'] = { 'data_subtype': DATA_SUBTYPES.INT } return model
def test_multilabel_prediction(self, tmp_path): train_file_name = os.path.join(str(tmp_path), 'train_data.csv') test_file_name = os.path.join(str(tmp_path), 'test_data.csv') data_len = 60 features = generate_value_cols(['int', 'float', 'int', 'float'], data_len) labels = [] labels.append(generate_log_labels(features)) labels.append(generate_timeseries_labels(features)) feature_headers = list(map(lambda col: col[0], features)) label_headers = list(map(lambda col: col[0], labels)) # Create the training dataset and save it to a file columns_train = list( map(lambda col: col[1:int(len(col) * 3 / 4)], features)) columns_train.extend( list(map(lambda col: col[1:int(len(col) * 3 / 4)], labels))) columns_to_file(columns_train, train_file_name, headers=[*feature_headers, *label_headers]) # Create the testing dataset and save it to a file columns_test = list( map(lambda col: col[int(len(col) * 3 / 4):], features)) columns_to_file(columns_test, test_file_name, headers=feature_headers) mdb = Predictor(name='test_multilabel_prediction') mdb.learn( from_data=train_file_name, to_predict=label_headers, stop_training_in_x_seconds=1, use_gpu=False, advanced_args={'force_predict': True} ) results = mdb.predict(when_data=test_file_name) models = F.get_models() model_data = F.get_model_data(models[0]['name']) assert model_data for i in range(len(results)): row = results[i] for label in label_headers: expect_columns = [label, label + '_confidence'] for col in expect_columns: assert col in row
def get_model_data(self, name, db_fix=True): from mindsdb_native import F from mindsdb_native.libs.constants.mindsdb import DATA_SUBTYPES from mindsdb.interfaces.storage.db import session, Predictor predictor_record = Predictor.query.filter_by( company_id=self.company_id, name=name, is_custom=False).first() predictor_record = self._try_outdate_db_status(predictor_record) model = predictor_record.data if model is None or model['status'] == 'training': try: self.fs_store.get( name, f'predictor_{self.company_id}_{predictor_record.id}', self.config['paths']['predictors']) new_model_data = F.get_model_data(name) except Exception: new_model_data = None if predictor_record.data is None or ( new_model_data is not None and len(new_model_data) > len(predictor_record.data)): predictor_record.data = new_model_data model = new_model_data session.commit() # Make some corrections for databases not to break when dealing with empty columns if db_fix: data_analysis = model['data_analysis_v2'] for column in model['columns']: analysis = data_analysis.get(column) if isinstance(analysis, dict) and (len(analysis) == 0 or analysis.get( 'empty', {}).get('is_empty', False)): data_analysis[column]['typing'] = { 'data_subtype': DATA_SUBTYPES.INT } model['created_at'] = str( parse_datetime(str(predictor_record.created_at).split('.')[0])) model['updated_at'] = str( parse_datetime(str(predictor_record.updated_at).split('.')[0])) model['predict'] = predictor_record.to_predict model['update'] = predictor_record.update_status return self._pack(model)
def predict(self, name, when_data=None, kwargs={}): if name not in self.predictor_cache: # Clear the cache entirely if we have less than .12 GB left if psutil.virtual_memory().available < 1.2 * pow(10, 9): self.predictor_cache = {} if F.get_model_data(name)['status'] == 'complete': self.predictor_cache[name] = { 'predictor': mindsdb_native.Predictor(name=name, run_env={'trigger': 'mindsdb'}), 'created': datetime.datetime.now() } predictions = self.predictor_cache[name]['predictor'].predict( when_data=when_data, **kwargs) return predictions
def test_house_pricing(self, use_gpu): """ Tests whole pipeline from downloading the dataset to making predictions and explanations. """ # Create & Learn name = 'home_rentals_price' mdb = Predictor(name=name) mdb.learn( to_predict='rental_price', from_data= "https://s3.eu-west-2.amazonaws.com/mindsdb-example-data/home_rentals.csv", backend='lightwood', stop_training_in_x_seconds=80, use_gpu=use_gpu) def assert_prediction_interface(predictions): for prediction in predictions: assert hasattr(prediction, 'explanation') test_results = mdb.test( when_data= "https://s3.eu-west-2.amazonaws.com/mindsdb-example-data/home_rentals.csv", accuracy_score_functions=r2_score, predict_args={'use_gpu': use_gpu}) assert test_results['rental_price_accuracy'] >= 0.8 predictions = mdb.predict( when_data= "https://s3.eu-west-2.amazonaws.com/mindsdb-example-data/home_rentals.csv", use_gpu=use_gpu) assert_prediction_interface(predictions) predictions = mdb.predict(when_data={'sqft': 300}, use_gpu=use_gpu) assert_prediction_interface(predictions) amd = F.get_model_data(name) assert isinstance(json.dumps(amd), str) for k in [ 'status', 'name', 'version', 'data_source', 'current_phase', 'updated_at', 'created_at', 'train_end_at' ]: assert isinstance(amd[k], str) assert isinstance(amd['predict'], (list, str)) assert isinstance(amd['is_active'], bool) for k in ['validation_set_accuracy', 'accuracy']: assert isinstance(amd[k], float) for k in amd['data_preparation']: assert isinstance(amd['data_preparation'][k], (int, float)) for k in amd['data_analysis']: assert (len(amd['data_analysis'][k]) > 0) assert isinstance(amd['data_analysis'][k][0], dict) model_analysis = amd['model_analysis'] assert (len(model_analysis) > 0) assert isinstance(model_analysis[0], dict) input_importance = model_analysis[0]["overall_input_importance"] assert (len(input_importance) > 0) assert isinstance(input_importance, dict) for k in ['train', 'test', 'valid']: assert isinstance(model_analysis[0][k + '_data_accuracy'], dict) assert len(model_analysis[0][k + '_data_accuracy']) == 1 assert model_analysis[0][k + '_data_accuracy']['rental_price'] > 0.4 for column, importance in zip(input_importance["x"], input_importance["y"]): assert isinstance(column, str) assert (len(column) > 0) assert isinstance(importance, (float, int)) assert (importance >= 0 and importance <= 10) # Test confidence estimation after save -> load p = None F.export_predictor(name) F.import_model(f"{name}.zip", f"{name}-new") p = Predictor(name=f'{name}-new') predictions = p.predict(when_data={'sqft': 1000}, use_gpu=use_gpu, run_confidence_variation_analysis=True) assert_prediction_interface(predictions)
def get_model_data(self, name): return F.get_model_data(name)