reparsed = minidom.parseString(rough_string) return reparsed.toprettyxml(indent=" ").replace( "<?xml version=\"1.0\" ?>\n", "") logging.basicConfig(filename="logs/parse_xml_parameters4indri.log", level=logging.DEBUG) if __name__ == '__main__': config_file = sys.argv[1] config = json.load(open(config_file)) logging.info('Config: ' + json.dumps(config, indent=2)) out = config["output"] # process queries: queries = extract_trec_million_queries(config["queries"]) queries_text = {} stoplist = set(stopwords.words("english")) if config["stopwords"] else {} qrels_MQ = [] # get only judged queries for file in os.listdir(config["qrels_MQ"]): qrels_MQ += list(read_values(os.path.join(config["qrels_MQ"], file), 0)) qrels_MQ = set(qrels_MQ) q_times = defaultdict(int) print("Pre-process queries %d queries..." % len(qrels_MQ)) logging.info("Pre-process queries %d queries..." % len(qrels_MQ)) for q in tqdm(qrels_MQ): q_text = clean(queries[q], config["stemmer"], stoplist) q_times[q_text] += 1 # queries with duplicate content queries_text[q] = q_text if q_times[q_text] == 1 else ' '.join( [q_text, str(q_times[q_text])])
from os.path import join if __name__ == '__main__': print("Split data into folds for training,tast and validation") config_file = sys.argv[1] config_all = json.load(open(config_file)) config = config_all["parameters"] print('Config: '+json.dumps(config, indent=2)) if config["only_queries"] or config["only_docs"]: to_split = [] if config["only_queries"]: queries = extractTopics(config["queries_folder"]) if config["queries_format"] == "trec" \ else extract_trec_million_queries(config["queries_folder"]) to_split = list(queries.keys()) elif config["only_docs"]: index = pyndri.Index(config["index"]) docs_list = [doc for doc in range(index.document_base(), index.maximum_document())] external_doc_id = [index.document(doc)[0] for doc in docs_list] to_split = external_doc_id folds = {} random.shuffle(to_split) split = chunkIt(to_split, config["folds_num"]) if config["validation"]: for i in tqdm(range(config["folds_num"])): # print("fold ",i, end="\t") test = split[i]
print("Data extraction\nConfiguration: ") print(json.dumps(config, indent=2), end='\n') print("Reading index ...") index = pyndri.Index(config["indexed_data"]) _, id2token, _ = index.get_dictionary() externalDocId = {} for doc in range(index.document_base(), index.maximum_document()): extD, _ = index.document(doc) externalDocId[extD] = doc print("Extract queries ...") queries = {} if config["train_queries"] == config["test_queries"]: queries = extractTopics(config["train_queries"]) if config["train_queries_format"] == "trec"\ else extract_trec_million_queries(config["train_queries"]) else: train_queries = extractTopics(config["train_queries"]) if config["train_queries_format"] == "trec" \ else extract_trec_million_queries(config["train_queries"]) test_queries = extractTopics(config["test_queries"]) if config["test_queries_format"] == "trec" \ else extract_trec_million_queries(config["test_queries"]) queries = {**train_queries, **test_queries} print("{n} queries to process.".format(n=len(queries))) queries_text = {} q_times = defaultdict(int) print("Preprocess queries ...") for q in tqdm(queries): q_text = clean(queries[q], "krovetz", {}) q_times[q_text] += 1 queries_text[q] = q_text if q_times[q_text] == 1 else ' '.join(
Usage: construcrQueryFile.py <topics_folder> <collection_name> <outputfolder> [--tq | --mq] Options: --t if topics_folder contains trec query files. --tm if topics_folder contains trec_million query files. """) print("\nparameters : \n") print(args) topics = {} if args["--tq"]: topics = extract_trec_topics(args["<topics_folder>"]) elif args["--mq"]: topics = extract_trec_million_queries(args["<topics_folder>"]) else: print("No queries to extract") outputFile = open(join(args["<outputfolder>"], "RetrievalParameterFile_{name}.xml".format(name=args["<collection_name>"])), 'w') outputFile.write("<parameters>\n") tokenizer = MosesTokenizer() prog = re.compile("[_\-\(]*([A-Z]\.(\ )*)*[_\-\(]*") tops = {} for top in topics: terms = topics[top].split() toptext = "" for t in terms: