def post(self, request): scenario = create_test_scenario(request.data) inference_time = 0 try: check_test_consistency(scenario) except ValueError as e: return Response({ 'detail': str(e), }, status=status.HTTP_400_BAD_REQUEST) # Load pipeline model from pipeline file pipeline = load_model(scenario.pipeline.file) if not pipeline: return Response( {'detail': 'The selected file isn\'t a pipeline objects'}, status=status.HTTP_400_BAD_REQUEST) # Extract pipeline information from loaded model pipeline = MLManager.extract_pipeline_components(pipeline) if not pipeline: return Response( {'detail': 'The selected file isn\'t a pipeline objects'}, status=status.HTTP_400_BAD_REQUEST) # Model params scenario.model = pipeline.get('model') scenario.transforms = json.dumps(pipeline.get('transforms', [])) # Dataset if scenario.run_db: # Get features from table features = get_columns(scenario.db_url, scenario.table) else: data_extractor_start = datetime.now() # Get Dataset ds = get_dataset(scenario) features = ds.columns.to_list() data_extractor_end = datetime.now() data_extractor_time = (data_extractor_end - data_extractor_start).total_seconds() inference_time += data_extractor_time if scenario.labels_type == 'column': # Remove Label column if exists features.remove(scenario.labels) # ML Manager manager = MLManager() # Testing Phase query = None if scenario.run_db: inference_start = datetime.now() # Generate query using MLManager dbms = DBMSUtils.get_dbms_from_str_connection(scenario.db_url) queries, query = manager.generate_query(scenario.pipeline.file, scenario.table, features, dbms, scenario.optimizer) # Execute query y_pred = execute_multi_queries(scenario.db_url, queries) y_pred = pd.Series(y_pred.iloc[:, 0], name='Label') inference_end = datetime.now() inference_time += (inference_end - inference_start).total_seconds() else: inference_start = datetime.now() # Execute predict using MLManager and ML Library y_pred = manager.predict(ds[features], scenario.pipeline.file) y_pred = pd.Series(y_pred, name='Label') inference_end = datetime.now() inference_time += (inference_end - inference_start).total_seconds() # Label labels = [] # Compute evaluation if scenario.labels_type: if scenario.labels_type == 'file': # Get labels from file labels_document = get_document_object(scenario.labels) labels = get_dataframe(labels_document.file) if labels is None: return Response( { 'detail': 'The selected labels file {} isn\'t valid'.format( labels_document.filename) }, status=status.HTTP_400_BAD_REQUEST) # Get first column from file labels = labels.iloc[:, 0].to_list() elif scenario.labels_type == 'table': # Get labels from table labels = get_table(scenario.db_url, scenario.labels) if not labels: return Response( { 'detail': 'The selected table {} isn\'t valid for label'. format(scenario.labels) }, status=status.HTTP_400_BAD_REQUEST) # Get first column from table labels = labels.iloc[:, 0].to_list() elif scenario.labels_type == 'column' and not scenario.run_db: # Get labels from column labels = ds[scenario.labels].to_list() elif scenario.labels_type == 'column' and scenario.run_db: # Get labels from table labels = get_column(scenario.db_url, scenario.table, scenario.labels) else: return Response({'detail': 'Select the wrong labels type'}, status=status.HTTP_400_BAD_REQUEST) # Compute evaluation res_evaluation = None if labels and scenario.metric: res_evaluation = manager.evaluate(scenario.metric, labels, y_pred) # Create predictions file test_result_name = "test_{}_{}.csv".format(scenario.model, datetime.now()) test_result_name = test_result_name.replace(' ', '_') test_result_name = test_result_name.replace(':', '_') y_pred.to_csv(test_result_name, index=False, header=True) # Save predictions in Document model f = open(test_result_name, 'rb') document = Document(file=File(f), filename=test_result_name) document.save() f.close() # Remove temporally predictions file os.remove(test_result_name) # Save Scenario model scenario.save() # Save ResultScenario result_scenario = ResultScenario() result_scenario.scenario = scenario result_scenario.execution_time = inference_time result_scenario.throughput = result_scenario.execution_time / len( y_pred) result_scenario.score = res_evaluation result_scenario.file_result = document.filename result_scenario.query = query result_scenario.save() return Response( { 'detail': 'Successfully predicted result', 'filename': test_result_name, 'scenario_id': scenario.id }, status=status.HTTP_201_CREATED)
def post(self, request): # Params scenario = create_simulation_scenario(request.data) try: check_simulation_consistency(scenario) except ValueError as e: return Response({ 'detail': str(e), }, status=status.HTTP_400_BAD_REQUEST) # Get Dataset t_load_start = datetime.now() ds = get_table(scenario['db_url'], scenario['table']) features = ds.columns.to_list() if scenario['labels_type'] == 'column': # Remove Label column if exists features.remove(scenario['labels']) t_load_end = datetime.now() # ML Manager manager = MLManager() t_ml = [] t_db = [] # Testing Phase for i in range(scenario['batch_number']): ds_batch = get_batch(ds, i, scenario['batch_size']) if ds_batch.empty: break # Execute predict using MLManager and ML Library t_start = datetime.now() _ = manager.predict(ds[features], scenario['pipeline'].file) t_end = datetime.now() t_ml.append(t_end - t_start) # Create Batch for DBMS ds_batch.to_sql('batch', con=scenario["db_url"], if_exists="replace", index=False) # Generate query using MLManager dbms = DBMSUtils.get_dbms_from_str_connection(scenario['db_url']) queries, query = manager.generate_query(scenario['pipeline'].file, scenario['table'], features, dbms, scenario['optimizer']) # Execute query t_start = datetime.now() # Execute query _ = execute_multi_queries(scenario["db_url"], queries) t_end = datetime.now() t_db.append(t_end - t_start) # Finish Simulation return Response( { 'detail': 'Successfully predicted result', 'ml_results': { 'execution_time': (np.mean(t_ml) + (t_load_end - t_load_start)) }, 'dbms_results': { 'execution_time': np.mean(t_db) }, }, status=status.HTTP_200_OK)
def main(data_conf, pipeline_conf, str_db_conn, task, optimizer, debug=False): data_conf['str_db_conn'] = str_db_conn data_conf = check_data_config(data_conf) train = data_conf['train'] y_train = data_conf['y_train'] test = data_conf['test'] y_test = data_conf['y_test'] test_table_name = data_conf['test_table_name'] features = list(data_conf['train'].columns) conn = data_conf['db_conn'] tasks = ['regression', 'binary_classification', 'multi_classification'] if task not in tasks: raise ValueError(f"Wrong task {task}. Available tasks: {tasks}") if task == 'regression': eval_fn = evaluate_regression_results elif task == 'binary_classification': eval_fn = evaluate_binary_classification_results else: eval_fn = evaluate_multi_classification_results check_pipeline_config(pipeline_conf, features) model_name = pipeline_conf['model']['name'] mlmanager = MLManager() pipeline = create_pipeline(pipeline_conf) # fit print("\nStarting training...") pipeline.fit(train, y_train) _check_fitted_pipeline(pipeline, model_name, train) print("Training completed.\n") fitted_model = pipeline.steps[1][1] # ML predict print("\nStarting the ML inference...") ml_preds = pipeline.predict(test) ml_preds = pd.Series(ml_preds) print(ml_preds[:10]) eval_fn(model_name, y_test, ml_preds) print("ML inference completed.\n") # SQL conversion print("\nStarting the SQL conversion...") pipeline = extract_pipeline(pipeline) dbms = DBMSUtils.get_dbms_from_str_connection(data_conf['str_db_conn']) queries, all_query = create_query(pipeline, mlmanager, features, test_table_name, optimizer, dbms, debug) print("SQL Conversion completed.\n") # SQL predict print("\nStarting the SQL inference...") for q in queries[:-1]: try: for qq in q.split(';'): conn.execute(qq) except Exception as e: pass try: sql_preds = pd.read_sql(queries[-1], conn) except Exception as e: logging.error(e.args[0]) return sql_preds = pd.Series(sql_preds.iloc[:, 0]) print(sql_preds[:10]) null_val = False if sql_preds.isnull().sum() == 0: eval_fn(f"{model_name} SQL", y_test, sql_preds) else: null_val = True print("SQL inference completed.\n") # Null value test if null_val: print("\nNull value test") null_val_cnt = 0 for sample_id in sql_preds[sql_preds.isnull()].index: print(sample_id) for (attr, val) in zip(test.columns, test.iloc[sample_id, :].values): print("\t", attr, '=', val) null_val_cnt += 1 print(f"Found {null_val_cnt} null predictions.") # Accuracy test print("\nAccuracy test") equals = False for prec in range(10, 0, -1): ml_preds = ml_preds.map(lambda x: round(x, prec)) sql_preds = sql_preds.map(lambda x: round(x, prec)) if ml_preds.equals(sql_preds): print( f"The prediction scores are equal with {prec} decimal precision." ) print(":)") equals = True break if not equals: print("The prediction scores are not equal.") print(":(\n") ne_preds = 0 for i in range(len(ml_preds)): ml_pred = ml_preds.iloc[i] sql_pred = sql_preds.iloc[i] if ml_pred != sql_pred: if debug: print(i, ml_pred, sql_pred) for (attr, val) in zip(test.columns, test.iloc[i, :].values): print("\t", attr, '=', val) ne_preds += 1 print(f"Found {ne_preds} incorrect predictions.")