def run_kfold(topic, no_folds, results_type, conf_matrix): """ Runs kfold test using credentials in ../Credentials.py """ # get credentials, import + export folders import Credentials active_adoption = Credentials.active_adoption instance_creds = Credentials.ctx[active_adoption] workspace_id = Credentials.workspace_id[active_adoption][topic] workspace_thresh = Credentials.calculate_workspace_thresh(topic) conversation_version = Credentials.conversation_version # import + export folders import config import time data_folder = config.data_dir export_folder = config.output_folder timestr = time.strftime("%Y%m%d-%H%M") output_loc_results = os.path.join( export_folder, "{}_kfold_results_raw_{}.csv".format(topic, timestr)) output_loc_metrics = os.path.join( export_folder, "{}_kfold_results_metrics_{}.csv".format(topic, timestr)) output_loc_confmat = os.path.join( export_folder, "{}_kfold_confmat_{}.png".format(topic, timestr)) # authenticate if 'apikey' in instance_creds: logger.debug("Authenticating (apikey)") kf = kfoldtest(n_folds=no_folds, apikey=instance_creds['apikey'], url=instance_creds['url'], threshold=workspace_thresh, version=conversation_version) elif 'password' in instance_creds: logger.debug("Authenticating (username/password)") kf = kfoldtest(n_folds=no_folds, username=instance_creds['username'], password=instance_creds['password'], url=instance_creds['url'], threshold=workspace_thresh, version=conversation_version) # get train df from watson + check there are sufficient workspaces to run the test train_df = kf.intent_df_from_watson(workspace_id) kf.check_sufficient_workspaces() # create folds in WA if above is true folds = kf.create_folds(method='kfold') kf.create_kfold_WA(folds) available_flag = False while available_flag == False: logger.info("Checking workspaces..") available_flag = kf.check_workspaces_status() time.sleep(20) # run kfold test try: results = kf.run_kfold_test(folds) if (results_type == 'raw') or (results_type == 'all'): results.to_csv(output_loc_results) classification_report = kf.create_classification_report(results) if (results_type == 'metrics') or (results_type == 'all'): metrics = Metrics(workspace_thresh) metric_df = metrics.get_all_metrics_CV( results, fold_col='fold', detailed_results=False) metric_df.to_csv(output_loc_metrics) # TODO: confusion matrix if conf_matrix: from confusionmatrix import ConfusionMatrix cfn = ConfusionMatrix(workspace_thresh=workspace_thresh) cfn.create(results, fig_path=output_loc_confmat) logger.info("Confusion matrix saved to {}".format( output_loc_confmat)) finally: # regardless of what happens above, delete the temporary workspaces before exiting kf.delete_kfold_workspaces()
def run_blindset(topic, results_type, conf_matrix, blindset_name): """ Runs blindset test using credentials in ../Credentials.py """ # get credentials, import + export folders import Credentials active_adoption = Credentials.active_adoption instance_creds = Credentials.ctx[active_adoption] print(instance_creds) print('print works') workspace_id = Credentials.workspace_id[active_adoption][topic] workspace_thresh = Credentials.calculate_workspace_thresh(topic) conversation_version = Credentials.conversation_version # import + export folders import config import time data_folder = config.data_dir export_folder = config.output_folder timestr = time.strftime("%Y%m%d-%H%M") blindset_name = blindset_name or topic + "_blindset.csv" output_loc_results = os.path.join( export_folder, "{}_results_raw_{}.csv".format(topic, timestr)) output_loc_metrics = os.path.join( export_folder, "{}_results_metrics_{}.csv".format(topic, timestr)) output_loc_confmat = os.path.join( export_folder, "{}_confmat_{}.png".format(topic, timestr)) # authenticate if 'apikey' in instance_creds: logger.debug("Authenticating (apikey)") bs = blindset(apikey=instance_creds['apikey'], url=instance_creds['url'], threshold=workspace_thresh, version=conversation_version) elif 'password' in instance_creds: logger.debug("Authenticating (username/password)") bs = blindset(username=instance_creds['username'], password=instance_creds['password'], url=instance_creds['url'], threshold=workspace_thresh, version=conversation_version) # run test blindset_df = bs.import_blindset(os.path.join(data_folder, blindset_name)) # TODO: check blindset df results = bs.run_blind_test(blindset_df, workspace_id) # exports + metrics if (results_type == 'raw') or (results_type == 'all'): cols_export = [ col for col in results.columns.values if col != 'intent_correct' ] results[cols_export].to_csv(output_loc_results, encoding='utf-8') logger.info("Raw results exported to {}".format(output_loc_results)) if (results_type == 'metrics') or (results_type == 'all'): met = Metrics(workspace_thresh) metric_df, _ = met.get_all_metrics(results, detailed_results=True) metric_df.to_csv(output_loc_metrics, encoding='utf-8') logger.info( "Metrics per intent exported to {}".format(output_loc_metrics)) # confusion matrix if conf_matrix: from confusionmatrix import ConfusionMatrix cfn = ConfusionMatrix(workspace_thresh=workspace_thresh) cfn.create(results, fig_path=output_loc_confmat) #bs.plot_confusion_matrix(results, output_loc_confmat) logger.info("Confusion matrix saved to {}".format(output_loc_confmat)) # print high-level metrics overall_metrics = bs.calculate_overall_metrics(results, av_method="weighted") logger.info("Overall metrics for the workspace (weighted):") logger.info(overall_metrics)