def main(corpus_label, exp_name, precomputed_queries_filename="predictions.json"): experiment["precomputed_queries_filename"] = precomputed_queries_filename if corpus_label.lower() == "aac": corpus = ez_connect("AAC", SERVER) experiment["name"] = exp_name experiment["load_preselected_test_files_list"] = "test_guids_c3_aac.txt" experiment["test_files_condition"] = "metadata.num_in_collection_references:>0 AND metadata.year:>2010" experiment["doc_methods"] = doc_methods_aac elif corpus_label.lower() == "pmc": corpus = ez_connect("PMC_CSC", SERVER) experiment["files_dict_filename"] = "files_dict_test.json" # experiment["files_dict_filename"] = "precomputed_queries_test.json" experiment["name"] = exp_name experiment["load_preselected_test_files_list"] = "test_guids_pmc_c3_1k.txt" experiment["test_files_condition"] = "metadata.num_in_collection_references:>0 AND metadata.year:>2013" experiment["doc_methods"] = doc_methods_pmc # corpus.closeIndex("idx_*") exp = Experiment(experiment, options, use_celery=False, process_command_line=False) print("exp_dir: ", exp.exp["exp_dir"]) print("Query file: ", precomputed_queries_filename) exp.run() pass
def initialize_experiment(): experiment = Experiment(data) experiment.add_classifier(TemporalEvidencesClassifier(data.features, data.target_names), name="Our method") experiment.add_classifier(NaiveBayesClassifier(data.features, data.target_names), name="Naive Bayes") return experiment
def main(): from multi.config import MINERVA_ELASTICSEARCH_ENDPOINT cp.useElasticCorpus() cp.Corpus.connectCorpus("g:\\nlp\\phd\\aac", endpoint=MINERVA_ELASTICSEARCH_ENDPOINT) cp.Corpus.setCorpusFilter("AAC") exp=Experiment(experiment, options, True) exp.run()
def main(): from multi.celery_app import MINERVA_ELASTICSEARCH_ENDPOINT cp.useElasticCorpus() cp.Corpus.connectCorpus("g:\\nlp\\phd\\aac", endpoint=MINERVA_ELASTICSEARCH_ENDPOINT) cp.Corpus.setCorpusFilter("AAC") ## experiment["test_files"]=["456f8c80-9807-46a9-8455-cd4a7e346f9d"] exp = Experiment(experiment, options, False) exp.run()
def main(): from multi.config import MINERVA_ELASTICSEARCH_ENDPOINT cp.useElasticCorpus() root_dir = getRootDir("aac") cp.Corpus.connectCorpus(root_dir, endpoint=MINERVA_ELASTICSEARCH_ENDPOINT) cp.Corpus.setCorpusFilter("AAC") ## experiment["test_files"]=["456f8c80-9807-46a9-8455-cd4a7e346f9d"] exp = Experiment(experiment, options, False) exp.run()
def main(): drive = "g" ## cp.useLocalCorpus() cp.useElasticCorpus() cp.Corpus.setCorpusFilter(collection_id="AAC") cp.Corpus.connectCorpus(drive + ":\\nlp\\phd\\aac") ## generator=AtharQueryGenerator(drive+r":\NLP\PhD\citation_context\doc_dict.json", reassign_guids=True) ## experiment["test_files"]=cp.Corpus.listPapers("year >= 2011") experiment["test_files"] = cp.Corpus.listPapers("year:>=2011") exp = Experiment(experiment, options) exp.run() pass
def _run_synth_exp(self): # initialise result array results = np.zeros((self.param_values.shape[0], len(SCORE_NAMES), len(self.methods), self.num_runs)) # iterate through parameter settings for param_idx in range(self.param_values.shape[0]): print('parameter setting: {0}'.format(param_idx)) # read parameter directory path_pattern = self.output_dir + 'data/param{0}/set{1}/' # iterate through data sets for run_idx in range(self.num_runs): print('data set number: {0}'.format(run_idx)) data_path = path_pattern.format(*(param_idx, run_idx)) if os.path.exists(os.path.join(data_path, 'pred_.csv')): os.remove(os.path.join(data_path, 'pred_.csv')) if os.path.exists(os.path.join(data_path, 'probs_.csv')): os.remove(os.path.join(data_path, 'probs_.csv')) # read data doc_start, gt, annos = self.generator.read_data_file(data_path + 'full_data.csv') exp = Experiment(data_path, self.generator.num_labels, annos, gt, doc_start, None, alpha0_factor=self.alpha0_factor, alpha0_diags=self.alpha0_diags, beta0_factor=self.beta0_factor, begin_factor=self.begin_factor, max_iter=self.max_iter, crf_probs=self.crf_probs, bootstrapping=False) exp.methods = self.methods # run methods results[param_idx, :, :, run_idx], preds, probabilities, _, _, _ = exp.run_methods(new_data=True, save_with_timestamp=False) if self.save_results: if not os.path.exists(self.output_dir): os.makedirs(self.output_dir) print('Saving results...') # np.savetxt(self.output_dir + 'results.csv', np.mean(results, 3)[:, :, 0]) results.dump(self.output_dir + 'results') if self.show_plots or self.save_plots: plot_results(self.param_values, self.methods, self.param_idx, results, self.show_plots, self.save_plots, self.output_dir) return results
def main(): drive = "g" ## cp.useLocalCorpus() cp.useElasticCorpus() cp.Corpus.setCorpusFilter(collection_id="AAC") cp.Corpus.connectCorpus(drive + ":\\nlp\\phd\\aac") generator = AtharQueryGenerator( drive + r":\NLP\PhD\citation_context\doc_dict.json", reassign_guids=True) experiment["test_files"] = list(generator.docs.keys()) exp = Experiment(experiment, options) exp.query_generator = generator exp.run() pass
def initialize_experiment(): experiment = Experiment(data) experiment.add_classifier(TemporalEvidencesClassifier( data.features, data.target_names), name="Our method") experiment.add_classifier(NaiveBayesClassifier(data.features, data.target_names), name="Naive Bayes") return experiment
print('loading ground truth labels...') gt = pd.read_csv(savepath + '/gt.csv', header=None).values #gt = IOB2_to_IOB(gt) # debug with subset ------- # s = 1000 # gt = gt[:s] # annos = annos[:s] # doc_start = doc_start[:s] # text = text[:s] # gt_dev = gt_dev[:s] # doc_start_dev = doc_start_dev[:s] # text_dev = text_dev[:s] # ------------------------- exp = Experiment(None, 3, annos.shape[1], None) exp.alpha0_factor = 1 exp.alpha0_diags = 100 exp.save_results = True exp.opt_hyper = False # True # run all the methods that don't require tuning here exp.methods = [ 'ibcc', 'majority', 'best', 'worst', ]
def main(): corpus = ez_connect("AAC", "aws-server") exp = Experiment(experiment, options, False) exp.run()
def main(): cp.useElasticCorpus() cp.Corpus.connectCorpus("g:\\nlp\\phd\\pmc_coresc", endpoint={ "host": "129.215.90.202", "port": 9200 }) ## experiment["test_files"]=["456f8c80-9807-46a9-8455-cd4a7e346f9d"] experiment["test_files"] = [ ## "bdc9a118-cb76-4d26-9c4d-e886794428f5", ## "65a4319e-4324-4529-96fa-66c52e392da0", ## "0cc28fb0-b116-4990-b816-3dc066273c34", ## "ef3e4284-c527-4e83-9b59-f8996b09df76", ## "b3129460-d284-4f69-83a8-f87f588e7800", ## "d8548dab-ff28-4f93-b2ae-16887e59e8ad", ## "42efd8ec-4c06-4754-a527-3045eed87766", ## "f4374057-7ab2-4567-b73b-aa5b72328d3e", ## "cbf989c5-79f5-4317-8515-2192e2a3fe2a", ## "37d1cc24-68a5-4a36-b55d-94acdfad08c1", ## "2b5202ec-e71b-4d1a-8ef4-439c4f505342", ## "11ba9f31-13f8-4a40-8bfc-6c9c7725e7ba", ## "e047f55f-ff56-44a6-a07c-794887330752", ## "d39d353a-f9ca-4ce3-ab42-e9a16f5bd372", ## "a407716f-4516-4cba-9c52-d4e3b09bcda6", ## "680724b2-50e7-4809-a86f-e63326059f7e", ## "1ce857ab-7692-4a95-9ba0-f517179a940e", ## "e12b2e84-a91d-4170-88a6-6ba983ceab1b", ## "5a6c0a35-dbe0-486a-8edf-3c3d3638f06e", ## "c40d5876-208c-4eb4-b239-652ed14f8560", ## "9a764770-fd73-474e-8f38-cf0128371e2c", ## "54432fc8-c1c4-42f9-95b0-c5fad39f8317", ## "a7dab0f1-5891-4d83-92c2-d25069c49d27", ## "283ed90d-3ff9-4161-8c4d-4e55a555973e", ## "6478c6ca-e16c-473f-9f4c-060143b3cc8f", ## "666f2c58-3180-465b-877c-28d14cbcdf98", ## "f5dedb99-f2a1-4ae9-b4a0-3c23e33cbfc9", ## "e5ed924b-8b78-4c76-bb6c-54d9790c8a15", ## "b8ace4e7-8523-471f-847b-b45aee8ccfc1", ## "ff30447d-828e-4699-bbf7-ce586aae9764", ## "aec8d55c-43e0-42cb-b832-77f888c2325a", ## "067862a3-d8fd-4252-b831-f6f120af82a1", ## "64956609-5a4d-4e05-bad1-0445c3d1834d", ## "cd1cd1ec-ecc9-4e70-96b3-7f1447ec0df3", ## "d61b922b-622b-440c-b040-3db563fd6f0e", ## "51d71d97-5abb-4a4d-ba77-7d18a11343f0", ## "b4c2215a-0a38-4e44-a5ab-f0d0114d89fc", ## "d3265a02-86ba-47e9-879c-a15043ca5808", ## "3e53830f-33a5-4192-9159-bcd01a3e66d3", ## "be50acb5-e165-4afb-b259-eeb9f28d0f2e", ## "fb8e6675-46d9-41c8-8ba0-598842a63fe8", ## "34043f1e-3424-4c4a-b782-9489fc274db5", ## "e07f7715-d400-4958-a0ac-2e6dab3b1843", ## "44f52aea-cae3-4e1c-85ce-da0038cbcea1", ## "b99f5b2d-6edd-4787-a50d-fef7d030ff05", ## "55ef06b7-ffc6-4e43-9362-daf4b9f6735f", ## "89c63f73-988a-4ece-99fd-e6c91fc9f6fd", ## "83293e90-8f3e-45db-8dae-49a179568d3e", ## "7fbcd237-d40d-4d44-9c9f-7e2f462e547e", ## "5303a3a3-c1cd-458c-9bef-56df3080169d", ## ## "a3af153f-cf9d-40ea-b64d-c6e52e0a187b", ## "38a67959-7ad9-426d-8357-51ab376b7a4b", ## "5f71b848-2f22-49bf-b32a-c1cc441a6dbe", ## "e875b2dc-3757-4728-8e8b-47c6a1d8241c", ## "a1bd31a8-66bc-4be9-aa85-e2e821aa18f5", ## "ef030f01-cdcb-4aaf-8ec1-c1a8778095da", ## "5b4f7822-1127-4d45-84ca-6755e1debaab", ## "a11c9c92-e294-4dfc-8e73-f432ad460776", ## "6c647939-ef22-4d8b-b887-121272168829", ## "d4c97daa-790c-40fe-a17e-fffa8e7fbd36", ## "65ee7543-549d-4821-b51d-8fc27dbe85cb", ## "4fe8ae7f-47bf-41fd-95ef-14ee7831f37e", ## "45d1bde0-2bd5-413b-89e3-9151d5a73ffb", ## "7bedaa57-30ff-4569-8456-59236171a80f", ## "67a054e0-744e-477b-80a4-06a268064bc7", ## "51a51cbb-952b-450b-970e-f6a23ecf9ce6", ## "b14209b3-d868-41cd-b1d0-f1a1489220f3", ## "53230a94-3baf-4825-a039-e8125890e737", ## "bb576feb-658e-45ef-810a-617b586159e5", ## "7d5ad1b5-2f3d-4728-b583-ec1ebbc3dac6", ## "f27cf9db-0d2d-490b-9917-c076a5ebca2c", ## "3b6679e5-deae-43ee-a98b-cac7029e92f4", ## "0d44cbba-1989-4654-b250-1b41285359ea", ## "b0cceb78-5f66-4084-accb-171040521cda", ## "18bf7b21-2456-49da-882a-06032ec46bec", ## "588b99bd-c358-440b-b30c-e1f3dc10b96b", ## "c4c1d5c0-7f40-465a-bbdb-351b4c9948a8", ## "0efcb373-ecd3-4e10-9f2a-1bbd3a6cbf58", ## "9bb0db11-2821-4d55-8f34-9bfd5d58f444", ## "49ff3f83-b4d7-4979-800f-785460c95552", ## "58e0a5d1-6343-4e2e-b544-6f690bff023e", ## "5a84843d-d7b0-43b0-846c-d30d3196ee8a", ## "6f244f35-8f61-4eb9-9de0-dfbbca63532b", ## "9d3cf2ea-162b-4e78-a311-b7333ad65c3a", ## "75e80547-50f5-4a12-8db6-e799a2e5029b", ## "f6c70cbd-e6c3-4ea5-b99f-ac0c455d832a", ## "f72d2af0-1e8a-40f5-9acf-ddbe9ddc4a7b", ## "9a81337e-1280-4b11-9b00-7e516d298ea1", ## "c7a83006-6ed3-46ec-b476-c49770dc4979", ## "aa45f968-61a4-421d-a532-a036fb8336ef", ## "f38092bc-b1e2-4ba8-ad60-b964825e52ac", ## "a755f020-c04d-4640-8b43-fb63b560bd6e", ## "1f06edd3-09d6-4033-b65e-a96d6a78f748", ## "33273c27-bcb7-4a4d-b339-c8af16c97b91", ## "802d2b57-0425-410e-82b6-f0024bc6f0dd", ## "699e837a-a662-49a0-b4c5-b3ef113eff34", ## "5b019a09-e21f-4109-a757-2c8396c8f169", ## "d4a70f39-7c5c-4566-8b5c-72208f3929ea", ## "6cf4f22d-c77b-4f0a-9e3c-378d7803f62b", ## "5cc55656-d309-4906-9cbf-7e34e734c352", ## "4222902b-e5fc-4eef-a7a7-79ec85d8e7c0", ## "4e408d49-6e51-441a-9c1c-8720d0d7032a", ## "aba078e1-c385-45b2-9adf-1ab7901b373b", ## "772eaabf-8996-486f-9bba-355cbf0c15e1", ## "38048193-6565-45ea-9950-64c7e4c266a3", ## "4d36eeb9-9121-4510-847e-99b80c77473e", ## "7b9e39c9-18a3-4112-ba9f-36b70d60f60f", ## "e31fd474-e2a0-4b3c-9d36-b31003b3bbc6", ## "69b92870-e050-4277-bd00-08f79aa6d9e6", ## "a67be750-73dc-427b-ac6a-e46adcaf7430", ## "1f8a95c4-856a-4f39-9c45-52d309d8c075", ## "b3606482-e22f-4809-948a-385a4f1e47cb", ## "7a4a67fb-3f4c-4c26-a060-775e8a4b7480", ## "bc39625f-1bc1-49ce-8ea3-4f8debe90b01", ## "202ef49f-c3f9-4d3a-b971-2d8094c06242", ## "28967c8d-2584-4898-9b62-0bfc669e2490", ## "355d6857-06ee-4430-a511-aaf0e8eaf23d", ## "0ce7eebd-0815-4f5d-b1c9-fafb65584994", ## "870c4608-525d-44d1-960f-4eb73589618c", ## "2420a665-d848-459f-9f51-456275d42e8b", ## "deb5362b-af92-4973-970f-ebe3fec12ee9", ## "d244e4d9-808c-4abd-a627-02716e9609c9", ## "de3f08f8-12fa-41d1-82cc-30c2c43cf52e", ## "575e9d63-94d7-483d-a980-8c974afc0ad9", ## "31410622-7133-4472-8225-8cf6b1eb1683", ## "80f3bd59-c5d6-43a0-97e3-155c1af50275", ## "cf2e8b40-5fab-4b17-acb1-6b676d909aa6", ## "51375367-8a16-4070-bab7-5ebcca3427c4", ## "dc293831-b099-45e2-a9ef-87ec8ccb8722", ## "aad3943f-37f9-4774-aa77-f312650b699e", ## "cdf828fc-2fb5-4c6f-8b42-8ff7c8ff0ff0", ## "6569f681-77e3-4ceb-a956-04e7a751f2b3", ## "2edffd94-b1db-46b7-bc4b-e4da9dcf4f51", ## "45a922bc-814a-40bd-a76a-fcbeca77bc81", ## "be4f19f7-de07-4674-8f03-1fbec9c7dd04", ## "48d9f4cf-c081-4520-b350-6ca3142987a7", ## "f35243d1-a3e3-4402-99b3-e576a27cde0d", ## "e8f567f8-3179-4214-bcbc-79332c1cfd1d", ## "209e32f7-a3cd-4e86-afee-2935a1f25514", ## "1cd47a2c-58c1-4c89-a689-cbdc0dd1f6b7", ## "2c64d4d5-3883-4fee-8c2c-1c0afb3835cf", ## "1323e0b5-c986-4ca6-855a-0b147d938e50", ## "d293f62a-983f-4ddc-a227-84d82bb36af1", ## "5b6439c9-466d-4bc0-aff2-e85de8eb9337", ## "da2b0b43-26b1-458b-b57a-83279ceb314e", ## "c21e2afc-0f92-490d-aa1b-7f826a83221d", ## "c5e67372-cf98-45db-bb7a-e3f4e7662774", ## "c7f91884-cfc8-406b-919e-658008c21279", ## "753b9d9a-ce8d-4fba-ac74-106526416738", ## "799680bf-5150-4fb2-b9b6-91fd0edc2593", ] exp = Experiment(experiment, options, True) exp.run()
# doc_start = doc_start[idxs] # text = text[idxs] # gt_task1_val = gt_task1_val[idxs] # ------------------------- num_reps = 10 batch_frac = 0.03 AL_iters = 10 for rep in range(1, num_reps): output_dir = '../data/bayesian_sequence_combination/output/ner_al_super_new/' if not os.path.isdir(output_dir): os.mkdir(output_dir) exp = Experiment(None, 9, annos.shape[1], None, max_iter=20, crf_probs=True, rep=rep) exp.save_results = True exp.opt_hyper = False#True exp.nu0_factor = 0.1 exp.alpha0_diags = 1 # best_diags exp.alpha0_factor = 1#9 # best_factor exp.methods = [ 'bac_seq_integrateIF', 'HMM_crowd', ] results, preds, probs, results_nocrowd, preds_nocrowd, probs_nocrowd = exp.run_methods( annos, gt, doc_start, output_dir, text, ground_truth_val=gt_val, doc_start_val=doc_start_val, text_val=text_val,
@author: Edwin Simpson ''' from evaluation.experiment import Experiment import data.load_data as load_data import numpy as np output_dir = '../../data/bayesian_sequence_combination/output/ner/' regen_data = False gt, annos, doc_start, text, gt_nocrowd, doc_start_nocrowd, text_nocrowd, gt_task1_val, gt_val, doc_start_val, text_val, _ = \ load_data.load_ner_data(regen_data) # ------------------------------------------------------------------------------------------------ exp = Experiment(None, 9, annos.shape[1], None, max_iter=20) exp.save_results = True exp.opt_hyper = False #True best_bac_wm = 'bac_seq' #'unknown' # choose model with best score for the different BAC worker models best_bac_wm_score = -np.inf best_nu0factor = 0.1 best_diags = 1 best_factor = 1 best_acc_bias = 0 exp.alpha0_diags = best_diags exp.alpha0_factor = best_factor exp.nu0_factor = best_nu0factor exp.alpha0_acc_bias = best_acc_bias
num_reps = 10 batch_frac = 0.03 AL_iters = 10 output_dir = os.path.join(evaluation.experiment.output_root_dir, 'ner_al') if not os.path.isdir(output_dir): os.mkdir(output_dir) # ACTIVE LEARNING WITH UNCERTAINTY SAMPLING for rep in range(1, num_reps): beta0_factor = 0.1 alpha0_diags = 1 alpha0_factor = 1 exp = Experiment(output_dir, 9, annos, gt, doc_start, text, annos, gt_val, doc_start, text, alpha0_factor=alpha0_factor, alpha0_diags=alpha0_diags, beta0_factor=beta0_factor, max_iter=20, crf_probs=True, rep=rep) exp.methods = [ 'bac_seq_integrateIF', 'HMM_crowd', ] results, preds, probs, results_nocrowd, preds_nocrowd, probs_nocrowd = exp.run_methods( active_learning=True, AL_batch_fraction=batch_frac, max_AL_iters=AL_iters ) beta0_factor = 10 alpha0_diags = 1 alpha0_factor = 1 exp = Experiment(output_dir, 9, annos, gt, doc_start, text, annos, gt_val, doc_start, text, alpha0_factor=alpha0_factor, alpha0_diags=alpha0_diags, beta0_factor=beta0_factor, max_iter=20, crf_probs=True, rep=rep)
beta0_factor = 1 alpha0_diags = 10 alpha0_factor = 10 best_begin_factor = 10 output_dir = os.path.join( evaluation.experiment.output_root_dir, 'pico3_%f_%f_%f' % (beta0_factor, alpha0_diags, alpha0_factor)) exp = Experiment(output_dir, 3, annos, gt, doc_start, features, annos, gt_val, doc_start, features, alpha0_factor=alpha0_factor, alpha0_diags=alpha0_diags, beta0_factor=beta0_factor, max_iter=20, begin_factor=best_begin_factor) # # run all the methods that don't require tuning here exp.methods = [ # 'bac_seq_integrateIF', 'bac_seq_integrateIF_thenLSTM', 'bac_seq_integrateIF_integrateLSTM_atEnd', ] # this will run task 1 -- train on all crowdsourced data, test on the labelled portion thereof exp.run_methods(new_data=regen_data)
def main(): corpus = ez_connect("PMC_CSC", "aws-server") corpus.closeIndex("idx_inlink_*") exp = Experiment(experiment, options, use_celery=False) exp.run() pass
data = load_dataset("../datasets/houseA.csv", "../datasets/houseA.config") intervals_to_test = [ #test various settings for delta t_max ("Delta t_max=1200s", initialize_bins(start=0, end=60, width=10) + initialize_bins(start=60, end=1200, width=30)), ("Delta t_max=120s", initialize_bins(start=0, end=60, width=10) + initialize_bins(start=60, end=120, width=30)), ("Delta t_max=60s", initialize_bins(start=0, end=60, width=10)), ("Delta t_max=30s", initialize_bins(start=0, end=30, width=10)), ("Delta t_max=10s", initialize_bins(start=0, end=10, width=10)), #test various interval widths ("all intervals 2s wide", initialize_bins(start=0, end=300, width=2)), ("all intervals 4s wide", initialize_bins(start=0, end=300, width=4)), ("all intervals 6s wide", initialize_bins(start=0, end=300, width=6)), ("all intervals 8s wide", initialize_bins(start=0, end=300, width=8)), ("all intervals 30s wide", initialize_bins(start=0, end=300, width=30)), ("all intervals 50s wide", initialize_bins(start=0, end=300, width=50)), ("all intervals 100s wide", initialize_bins(start=0, end=300, width=100)) ] #run 10-fold cross-validation for each of the configured intervals experiment = Experiment(data) for (name, bins) in intervals_to_test: experiment.add_classifier(TemporalEvidencesClassifier(data.features, data.target_names, bins=bins), name=name) results = experiment.run(folds=10) results.print_quality_comparison_at_cutoff( cutoff=1, metrics=["Recall", "Precision", "F1"])
def main(): corpus = ez_connect("AAC", "koko") corpus.closeIndex("idx_*") exp = Experiment(experiment, options, use_celery=False) exp.run() pass
# ------------------------------------------------------------------------------------------------ # Rerunning with found parameters... beta0_factor = 0.1 alpha0_diags = 0.1 alpha0_factor = 0.1 output_dir = os.path.join( evaluation.experiment.output_root_dir, 'ner3_%f_%f_%f' % (beta0_factor, alpha0_diags, alpha0_factor)) exp = Experiment(output_dir, 9, annos, gt, doc_start, features, annos, gt_val, doc_start, features, alpha0_factor=alpha0_factor, alpha0_diags=alpha0_diags, beta0_factor=beta0_factor, max_iter=20) exp.methods = [ 'best', # does not use the hyperparameters 'worst', # does not use the hyperparameters 'majority', # does not use the hyperparameters 'mace', # worked best with its own default hyperparameters, smoothing=0.001, alpha=0.5, beta=0.5 'ds', # does not use the hyperparameters 'HMM_Crowd', # does not use alpha0_diags; default values happened to work best ]
# , debug_subset_size=1000) # include this argument to debug with small dataset # ------------------------------------------------------------------------------------------------ # only hmm_Crowd actually uses these hyperparameters beta0_factor = 0.1 alpha0_diags = 0.1 alpha0_factor = 0.1 output_dir = os.path.join(evaluation.experiment.output_root_dir, 'pico3') exp = Experiment(output_dir, 3, annos, gt, doc_start, features, annos, gt_val, doc_start, features, alpha0_factor=alpha0_factor, alpha0_diags=alpha0_diags, beta0_factor=beta0_factor, max_iter=20) # # run all the methods that don't require tuning here exp.methods = [ 'best', 'worst', 'majority', 'ds', 'mace', 'HMM_crowd', ]
# exp.nu0_factor = nu_factors[best_idxs[0]] # exp.alpha0_diags = diags[best_idxs[1]] # exp.alpha0_factor = factors[best_idxs[2]] # # print('Best values for %s: %f, %f, %f' % (method, exp.nu0_factor, exp.alpha0_diags, exp.alpha0_factor)) # # # this will run task 1 -- train on all crowdsourced data, test on the labelled portion thereof # exp.methods = [method] # exp.run_methods(annos, gt, doc_start, output_dir, text, rerun_all=True, return_model=True, # ground_truth_val=gt_dev, doc_start_val=doc_start_dev, text_val=text_dev, # new_data=regen_data # ) # ------------------------------------------------------------------------------------------------------------------ exp = Experiment(None, nclasses, annos.shape[1], None, max_iter=5) exp.save_results = True exp.opt_hyper = False #True # values obtained from tuning on dev: best_nu0factor = 1000 #1 best_diags = 1000 best_factor = 0.1 exp.nu0_factor = best_nu0factor exp.alpha0_diags = best_diags exp.alpha0_factor = best_factor exp.methods = [ # 'bac_seq_integrateIF',
def main(): global options, experiment corpus = ez_connect("AAC", "koko") # to_generate = "training_min2" to_generate = "training_min1" # to_generate = "testing" # to_generate = "PRUEBA" # to_generate = "export" # experiment["keyword_selector"] = "MultiMaximalSetSelector" # suffix = "mms" experiment["keyword_selector"] = "AllSelector" suffix = "w" if to_generate.startswith("training"): experiment["max_queries_generated"] = 100000000 experiment["max_test_files"] = 500000 # experiment["test_files_condition"] = "metadata.num_in_collection_references:>0 " # experiment["features_field_name"] = "_full_text" experiment[ "test_files_condition"] = "metadata.num_in_collection_references:>0 AND metadata.year:>2010" experiment["features_field_name"] = "_all_text" # experiment["test_guids_to_ignore_file"] = "test_guids_c3_aac_desktop.txt" experiment[ "load_preselected_test_files_list"] = "train_guids_aac_c6.txt" match = re.search(r"_min(\d)", to_generate) if match: min_multi = int(match.group(1)) experiment["resolvable_cit_min_multi"] = min_multi experiment[ "precomputed_queries_filename"] = "precomputed_queries_training_min%d.json" % min_multi experiment[ "files_dict_filename"] = "files_dict_training_min%d.json" % min_multi experiment[ "feature_data_filename"] = "feature_data_at_%s_min%d.json.gz" % ( suffix, min_multi) else: experiment["resolvable_cit_min_multi"] = 1 experiment[ "precomputed_queries_filename"] = "precomputed_queries_training.json" experiment["files_dict_filename"] = "files_dict_training.json" experiment[ "feature_data_filename"] = "feature_data_at_%s.json.gz" % suffix elif to_generate == "testing": experiment[ "load_preselected_test_files_list"] = "test_guids_c3_aac_desktop.txt" experiment["name"] = "aac_generate_kw_trace_TEST" experiment["max_queries_generated"] = 1000 experiment[ "precomputed_queries_filename"] = "precomputed_queries_new1k.json" experiment["files_dict_filename"] = "files_dict_new1k.json" experiment[ "feature_data_filename"] = "feature_data_test_at_%s.json.gz" % suffix # experiment["resolvable_cit_min_multi"]= 1 experiment["max_test_files"] = 1000 experiment[ "test_files_condition"] = "metadata.num_in_collection_references:>0 AND metadata.year:>2010" elif to_generate == "export": experiment[ "feature_data_filename"] = "feature_data_at_%s.json.gz" % suffix options["clear_existing_prr_results"] = 0 options = { "run_prebuild_bows": 0, # should the whole BOW building process run? "overwrite_existing_bows": 0, # if a BOW exists already, should we overwrite it? "build_indexes": 0, # rebuild indices? "generate_queries": 0, # precompute the queries? "force_regenerate_resolvable_citations": 0, # find again the resolvable citations in a file? "overwrite_existing_queries": 0, # force rebuilding of queries too? # delete previous precomputed results? i.e. start from scratch "run_precompute_retrieval": 0, "run_feature_annotation": 0, "refresh_results_cache": 1, # should we clean the offline reader cache and redownload it all from elastic? "run_package_features": 1, "list_missing_files": 0, } exp = Experiment(experiment, options, False) exp.run()
sys.path.append("..") import pandas from evaluation.experiment import Experiment from evaluation.metrics import quality_metrics from recsys.classifiers.temporal import TemporalEvidencesClassifier, configure_dynamic_cutoff from recsys.dataset import load_dataset #configuration data = load_dataset("../datasets/houseA.csv", "../datasets/houseA.config") methods_to_test = [("Fixed cutoff", None), ("dynamic cutoff=4", configure_dynamic_cutoff(1.0, 0.4, 4)), ("dynamic cutoff=2", configure_dynamic_cutoff(1.0, 0.4, 2))] #run all configured cutoffs with 10-fold cross-validation experiment = Experiment(data) for name, method in methods_to_test: experiment.add_classifier(TemporalEvidencesClassifier(data.features, data.target_names, postprocess=method), name=name) results = experiment.run(folds=10) #print results pandas.set_option('expand_frame_repr', False) pandas.set_option('max_columns', 4) print "Maximum 5 recommendations" results.print_quality_comparison_at_cutoff(cutoff=5, metrics=quality_metrics) print "Maximum 10 recommendations" results.print_quality_comparison_at_cutoff(cutoff=10, metrics=quality_metrics)
cutoff_results_at = 14 return data, cutoff_results_at def houseB(): """ This dataset is partially dominated by one of the sensors, which makes the evaluation results less statistically sound, e.g. it leads to large confidence intervals when running 10-fold cross-validation. """ data = load_dataset("../datasets/houseB.csv", "../datasets/houseB.config") cutoff_results_at = 15 return data, cutoff_results_at #configuration data, cutoff_results_at = houseA() #run several classifiers on the same dataset, use 10-fold cross-validation experiment = Experiment(data) experiment.add_classifier(TemporalEvidencesClassifier(data.features, data.target_names), name="Our method") experiment.add_classifier(NaiveBayesClassifier(data.features, data.target_names), name="Naive Bayes") experiment.add_classifier(RandomClassifier(data.features, data.target_names), name="Random") results = experiment.run(folds=10) #print and plot results results.print_quality_comparison_at_cutoff(cutoff=1, metrics=["Recall", "Precision", "F1"]) results.print_runtime_comparison() plot_conf = plot.plot_config(config.plot_directory, sub_dirs=[data.name], img_type=config.img_type) results.plot_quality_comparison(metrics=["Recall", "Precision", "F1"], plot_config=plot_conf, cutoff_results_at=cutoff_results_at)
def houseB(): """ This dataset is partially dominated by one of the sensors, which makes the evaluation results less statistically sound, e.g. it leads to large confidence intervals when running 10-fold cross-validation. """ data = load_dataset("../datasets/houseB.csv", "../datasets/houseB.config") cutoff_results_at = 15 return data, cutoff_results_at #configuration data, cutoff_results_at = houseA() #run several classifiers on the same dataset, use 10-fold cross-validation experiment = Experiment(data) experiment.add_classifier(TemporalEvidencesClassifier(data.features, data.target_names), name="Our method") experiment.add_classifier(NaiveBayesClassifier(data.features, data.target_names), name="Naive Bayes") experiment.add_classifier(RandomClassifier(data.features, data.target_names), name="Random") results = experiment.run(folds=10) #print and plot results results.print_quality_comparison_at_cutoff( cutoff=1, metrics=["Recall", "Precision", "F1"]) results.print_runtime_comparison() plot_conf = plot.plot_config(config.plot_directory,
from recsys.classifiers.binning import initialize_bins from recsys.dataset import load_dataset #configuration data = load_dataset("../datasets/houseA.csv", "../datasets/houseA.config") intervals_to_test = [#test various settings for delta t_max ("Delta t_max=1200s", initialize_bins(start=0, end=60, width=10) + initialize_bins(start=60, end=1200, width=30)), ("Delta t_max=120s", initialize_bins(start=0, end=60, width=10) + initialize_bins(start=60, end=120, width=30)), ("Delta t_max=60s", initialize_bins(start=0, end=60, width=10)), ("Delta t_max=30s", initialize_bins(start=0, end=30, width=10)), ("Delta t_max=10s", initialize_bins(start=0, end=10, width=10)), #test various interval widths ("all intervals 2s wide", initialize_bins(start=0, end=300, width=2)), ("all intervals 4s wide", initialize_bins(start=0, end=300, width=4)), ("all intervals 6s wide", initialize_bins(start=0, end=300, width=6)), ("all intervals 8s wide", initialize_bins(start=0, end=300, width=8)), ("all intervals 30s wide", initialize_bins(start=0, end=300, width=30)), ("all intervals 50s wide", initialize_bins(start=0, end=300, width=50)), ("all intervals 100s wide", initialize_bins(start=0, end=300, width=100))] #run 10-fold cross-validation for each of the configured intervals experiment = Experiment(data) for (name, bins) in intervals_to_test: experiment.add_classifier(TemporalEvidencesClassifier(data.features, data.target_names, bins=bins), name=name) results = experiment.run(folds=10) results.print_quality_comparison_at_cutoff(cutoff=1, metrics=["Recall", "Precision", "F1"])