def get(self, doc_id, location, expression, force_update=False):
if not doc_id in self.cached_locations:
self.cached_locations[doc_id] = {}
if location in self.cached_locations[doc_id] and not force_update:
return self.cached_locations[doc_id][location]
else:
#first time the expression is seen, check....
if expression in self.cached_expressions and not force_update:
#expression has been retrieved before but at different location...
prev_doc_id, prev_location = self.cached_expressions[
expression]
return self.cached_locations[prev_doc_id][prev_location]
else:
control = Control(self.control_filename
) # control file name (after indexing)
document_finder = MathDocument(control)
mathml = document_finder.find_mathml(doc_id, location)
mathml = MathExtractor.isolate_pmml(mathml)
if isinstance(mathml, bytes):
mathml = mathml.decode('UTF-8')
# save on cache...
self.cached_locations[doc_id][location] = mathml
self.cached_expressions[expression] = (doc_id, location)
return mathml
def find_formula_ids(tsv_results, control_filename, mathids_cache):
control = Control(control_filename)
document_finder = MathDocument(control)
for query_offset in tsv_results:
print("Processing Query: " + str(query_offset))
total_locs = len(tsv_results[query_offset]["results"])
for index, result in enumerate(tsv_results[query_offset]["results"]):
doc, loc = result
math_id = mathids_cache.get_mathid(document_finder, doc, loc)
#print(str((query_offset, doc, loc, math_id)))
tsv_results[query_offset]["math_ids"].append(math_id)
if index > 0 and (index + 1) % 25 == 0:
print("... done " + str(index + 1) + " of " + str(total_locs),
end="\r")
import codecs
import sys
from sys import argv
from tangent.utility.control import Control
from tangent.math.mathdocument import MathDocument
__author__ = 'FWTompa'
if __name__ == '__main__':
if sys.stdout.encoding != 'utf8':
sys.stdout = codecs.getwriter('utf8')(sys.stdout.buffer, 'strict')
if sys.stderr.encoding != 'utf8':
sys.stderr = codecs.getwriter('utf8')(sys.stderr.buffer, 'strict')
if len(argv) != 4 or argv[1] == "help":
print("Use: python get_math.py <cntl> <doc#> <expr#>")
print(" where (doc# < 0) => use queryfile")
sys.exit()
cntl = Control(argv[1]) # control file name (after indexing)
d = MathDocument(cntl)
docno = int(argv[2])
exprno = int(argv[3])
print("doc " + argv[2] + ": " +
d.find_doc_file(docno)) #print document file name
print(d.find_mathml(docno, exprno)) # doc_num and pos_num
if __name__ == '__main__':
if sys.stdout.encoding != 'utf8':
sys.stdout = codecs.getwriter('utf8')(sys.stdout.buffer, 'strict')
if sys.stderr.encoding != 'utf8':
sys.stderr = codecs.getwriter('utf8')(sys.stderr.buffer, 'strict')
if (len(argv) > 2
or (len(argv) == 2 and argv[1]
== 'help')): # uses control file to control all parameters
print_help_and_exit()
else:
start = time.time()
try:
cntl = Control(argv[1]) if len(argv) == 2 else Control()
except Exception as err:
print("Error in reading <cntl-file>: " + str(err))
print_help_and_exit()
query_file = cntl.read("queries")
if not query_file:
print("<cntl-file> missing queries")
print_help_and_exit()
window = cntl.read("window", default=0, num=True)
if window and window < 0: # negative window values make no sense
print('Negative window values not permitted -- using 1')
window = 1
topk = cntl.read("topk", default=100, num=True)
if topk < 1: # non-positive values make no sense
print('Non-positive topk values not permitted -- using 100')
def main():
if len(sys.argv) < 5:
print("Usage")
print(
"\tpython3 rerank_results.py control input_results metric output_results"
)
print("")
print("Where:")
print("\tcontrol:\tPath to tangent control file")
print("\tinput_results:\tPath to file with results to re-rank")
print("\tmetric:\t\tSimilarity metric to use [0-4]")
print(
"\toutput_results:\tPath to file where re-ranked results will be stored"
)
print("")
print("Optional:")
print("\t-w\twindow\t\t: Window for pair generation")
print("\t-h\thtml_prefix\t: Prefix for HTML output (requires dot)")
print("\t-c\tcondition\t: Current test condition")
print("\t-s\tstats\t\t: File to store stats")
print("\t-t\ttimes\t\t: File to accumulate time stats")
print("\t-k\tmax_results\t: K number of results to rerank as maximum")
return
control_filename = sys.argv[1]
input_filename = sys.argv[2]
try:
metric = int(sys.argv[3])
if metric < -1 or metric > 11:
print("Invalid similarity metric function")
return
except:
print("Invalid similarity metric function")
return
output_filename = sys.argv[4]
optional_params = optional_parameters(sys.argv[5:])
#load control file
control = Control(control_filename) # control file name (after indexing)
math_doc = MathDocument(control)
if "w" in optional_params:
try:
window = int(optional_params["w"])
if window <= 0:
print("Invalid window")
return
except:
print("Invalid value for window")
return
else:
window = int(control.read("window"))
if "h" in optional_params:
html_prefix = optional_params["h"]
if not os.path.isdir(html_prefix):
os.makedirs(html_prefix)
else:
html_prefix = None
if "c" in optional_params:
condition = optional_params["c"]
print("testing condition: " + condition)
else:
condition = "undefined"
if "s" in optional_params:
stats_file = optional_params["s"]
else:
stats_file = None
if "k" in optional_params:
try:
max_k = int(optional_params["k"])
except:
print("Invalid max_results parameter")
return
else:
max_k = 0
if "t" in optional_params:
times_file = optional_params["t"]
else:
times_file = None
in_file = open(input_filename, 'r', newline='', encoding='utf-8')
reader = csv.reader(in_file,
delimiter='\t',
lineterminator='\n',
quoting=csv.QUOTE_NONE,
escapechar="\\")
lines = [row for row in reader]
in_file.close()
mathml_cache_file = control_filename + ".retrieval_2.cache"
if not os.path.exists(mathml_cache_file):
mathml_cache = MathMLCache(control_filename)
else:
cache_file = open(mathml_cache_file, "rb")
mathml_cache = pickle.load(cache_file)
cache_file.close()
current_query = None
current_name = None
current_tuple_retrieval_time = 'undefined'
all_queries = []
#read all results to re-rank
for idx, line in enumerate(lines):
#parts = line.strip().split("\t")
parts = line
if len(parts) == 2:
if parts[0][0] == "Q":
current_name = parts[1]
current_query = None
elif parts[0][0] == "E":
if current_name is None:
print("invalid expression at " + str(idx) +
": query name expected first")
else:
query_expression = parts[1]
#query_offset = len(all_queries)
query_offset = int(current_name.split("-")[-1]) - 1
if html_prefix != None:
mathml = mathml_cache.get(-1, query_offset,
query_expression, True)
# create empty directories for this query ...
if not os.path.isdir(html_prefix + "/" + current_name):
os.makedirs(html_prefix + "/" + current_name)
if not os.path.isdir(html_prefix + "/" + current_name +
"/images"):
os.makedirs(html_prefix + "/" + current_name +
"/images")
else:
mathml = None
current_query = Query(current_name, query_expression,
mathml, current_tuple_retrieval_time,
max_k)
current_name = None
all_queries.append(current_query)
print("Query: " + current_query.name + ": " +
current_query.expression)
#print(mathml)
#current_query.tree.save_as_dot("expre_" + str(idx) + ".gv")
elif parts[0][0] == "C":
if current_query is None:
print("invalid constraint at " + str(idx) +
": query expression expected first")
else:
# create a constraint tree
current_query.set_constraints(parts[1])
# RZ: Record tuple-based retrieval time and other metrics.
if len(parts) == 3 and parts[0][0] == "I" and current_query != None:
if parts[1] == "qt":
current_query.initRetrievalTime = float(parts[2])
elif parts[1] == "post":
current_query.postings = int(parts[2])
elif parts[1] == "expr":
current_query.matchedFormulae = int(parts[2])
elif parts[1] == "doc":
current_query.matchedDocs = int(parts[2])
if len(parts) == 5:
if parts[0][0] == "R":
doc_id = int(parts[1])
location = int(parts[2])
doc_name = math_doc.find_doc_file(doc_id)
expression = parts[3]
score = float(parts[4])
if html_prefix != None:
mathml = mathml_cache.get(doc_id, location, expression)
else:
mathml = None
if current_query is None:
print("Error: result listed before a query, line " +
str(idx))
else:
current_query.add_result(doc_id, doc_name, location,
expression, score, mathml)
cache_file = open(mathml_cache_file, "wb")
pickle.dump(mathml_cache, cache_file, pickle.HIGHEST_PROTOCOL)
cache_file.close()
# now, re-rank...
print("Results loaded, reranking ...")
# compute similarity first...
start_time = time.time()
for q_idx, query in enumerate(all_queries):
#print("Evaluating: " + query.name + " - " + query.expression)
query_start_time = time.time() * 1000 # RZ: ms
for res_idx, exp_result in enumerate(query.results):
result = query.results[exp_result]
#print("Candidate: " + result.expression)
scores = [0.0]
if metric == -1:
# bypass mode, generate HTML for original core ranking
scores = [result.original_score]
matched_c = {}
elif metric == 0:
# same as original based on f-measure of matched pairs..
pairs_query = query.tree.root.get_pairs("", window)
pairs_candidate = result.tree.root.get_pairs("", window)
scores, matched_q, matched_c = similarity_v00(
pairs_query, pairs_candidate)
elif metric == 1:
# based on testing of alignments....
scores, matched_q, matched_c = similarity_v01(
query.tree, result.tree)
elif metric == 2:
# Same as 0 but limiting to matching total symbols first...
pairs_query = query.tree.root.get_pairs("", window)
pairs_candidate = result.tree.root.get_pairs("", window)
scores, matched_q, matched_c = similarity_v02(
pairs_query, pairs_candidate)
elif metric == 3:
# modified version of 2 which performs unification....
pairs_candidate = result.tree.root.get_pairs("", window)
scores, matched_q, matched_c, unified_c = similarity_v03(
pairs_query, pairs_candidate)
result.set_unified_elements(unified_c)
elif metric == 4:
# modified version of 1 which performs unification ...
sim_res = similarity_v04(query.tree, result.tree,
query.constraints)
scores, matched_q, matched_c, unified_c, wildcard_c, unified = sim_res
result.set_unified_elements(unified_c)
result.set_wildcard_matches(wildcard_c)
result.set_all_unified(unified)
elif metric == 5:
# modified version of 4 which allows multiple sub matches
sim_res = similarity_v05(query.tree, result.tree,
query.constraints)
scores, matched_q, matched_c, unified_c, wildcard_c = sim_res
result.set_unified_elements(unified_c)
result.set_wildcard_matches(wildcard_c)
elif metric == 6:
# modified version of 4 which allows subtree matches for wildcards (partial support)...
sim_res = similarity_v06(query.tree, result.tree,
query.constraints)
scores, matched_q, matched_c, unified_c, wildcard_c, unified = sim_res
result.set_unified_elements(unified_c)
result.set_wildcard_matches(wildcard_c)
result.set_all_unified(unified)
elif metric == 7:
# modified version of 4 which allows subtree matches for wildcards (partial support)...
sim_res = similarity_v07(query.tree, result.tree,
query.constraints)
scores, matched_q, matched_c, unified_c, wildcard_c, unified = sim_res
result.set_unified_elements(unified_c)
result.set_wildcard_matches(wildcard_c)
result.set_all_unified(unified)
elif metric == 8:
# modified version of 4 which allows subtree matches for wildcards (partial support)...
sim_res = similarity_v08(query.tree, result.tree,
query.constraints)
scores, matched_q, matched_c, unified_c, wildcard_c, unified = sim_res
result.set_unified_elements(unified_c)
result.set_wildcard_matches(wildcard_c)
result.set_all_unified(unified)
elif metric == 9:
# modified version of 4 which allows subtree matches for wildcards (partial support)...
sim_res = similarity_v09(query.tree, result.tree,
query.constraints)
scores, matched_q, matched_c, unified_c, wildcard_c, unified = sim_res
result.set_unified_elements(unified_c)
result.set_wildcard_matches(wildcard_c)
result.set_all_unified(unified)
elif metric == 10:
# modified version of 4 which allows subtree matches for wildcards (partial support)...
sim_res = similarity_v10(query.tree, result.tree,
query.constraints)
scores, matched_q, matched_c, unified_c, wildcard_c, unified = sim_res
result.set_unified_elements(unified_c)
result.set_wildcard_matches(wildcard_c)
result.set_all_unified(unified)
elif metric == 11:
# matching of metric 06 with scores from metric 04 (MSS)
sim_res = similarity_v11(query.tree, result.tree,
query.constraints)
scores, matched_q, matched_c, unified_c, wildcard_c, unified = sim_res
result.set_unified_elements(unified_c)
result.set_wildcard_matches(wildcard_c)
result.set_all_unified(unified)
result.set_matched_elements(matched_c)
result.new_scores = scores
query_end_time = time.time() * 1000 # RZ: ms
# re-rank based on new score(s)
query.sort_results()
query.sort_documents()
query.elapsed_time = query_end_time - query_start_time
end_time = time.time()
elapsed = end_time - start_time
print("Elapsed Time Ranking: " + str(elapsed) + "s")
#now, store the re-ranked results...
out_file = open(output_filename, 'w', newline='', encoding='utf-8')
csv_writer = csv.writer(out_file,
delimiter='\t',
lineterminator='\n',
quoting=csv.QUOTE_NONE,
escapechar="\\")
for query in all_queries:
csv_writer.writerow([])
query.output_query(csv_writer)
query.output_sorted_results(csv_writer)
if html_prefix is not None:
print("Saving " + query.name + " to HTML file.....")
query.save_html(html_prefix + "/" + query.name)
out_file.close()
#if stats file is requested ...
if stats_file is not None:
out_file = open(stats_file, "w")
out_file.write(Query.stats_header("\t"))
for query in all_queries:
query.output_stats(out_file, "\t", condition)
out_file.close()
# if times file is requested ...
if times_file is not None:
sorted_queries = sorted([(query.name.strip(), query)
for query in all_queries])
if os.path.exists(times_file):
out_file = open(times_file, "a")
else:
out_file = open(times_file, "w")
header = "condition," + ",".join(
[name for (name, query) in sorted_queries])
out_file.write(header + "\n")
line = condition
for name, query in sorted_queries:
line += "," + str(query.elapsed_time)
out_file.write(line + "\n")
out_file.close()
print("Finished successfully")
ntcir_main_count = 1000 # main task require 1000 results returned
ntcir_wiki_count = 100
if sys.stdout.encoding != 'utf8':
sys.stdout = codecs.getwriter('utf8')(sys.stdout.buffer, 'strict')
if sys.stderr.encoding != 'utf8':
sys.stderr = codecs.getwriter('utf8')(sys.stderr.buffer, 'strict')
if (len(argv) > 2 or (len(argv) == 2 and argv[1] == 'help')): # uses control file to control all parameters
print_help_and_exit()
else:
start = time.time()
try:
cntl = Control(argv[1]) if len(argv) == 2 else Control()
except Exception as err:
print("Error in reading <cntl-file>: " +str(err))
print_help_and_exit()
db = cntl.read("database")
if not db:
print("<cntl-file> missing database")
print_help_and_exit()
query_file = cntl.read("queries")
if not query_file:
print("<cntl-file> missing queries")
print_help_and_exit()
window = cntl.read("window",num=True)
if window and window < 2: # window values smaller than 2 make no sense
print('Window values smaller than 2 not permitted -- using 2')
ntcir_wiki_count = 100
if sys.stdout.encoding != 'utf8':
sys.stdout = codecs.getwriter('utf8')(sys.stdout.buffer, 'strict')
if sys.stderr.encoding != 'utf8':
sys.stderr = codecs.getwriter('utf8')(sys.stderr.buffer, 'strict')
if (len(argv) > 2
or (len(argv) == 2 and argv[1]
== 'help')): # uses control file to control all parameters
print_help_and_exit()
else:
start = time.time()
try:
cntl = Control(argv[1]) if len(argv) == 2 else Control()
except Exception as err:
print("Error in reading <cntl-file>: " + str(err))
print_help_and_exit()
db = cntl.read("database")
if not db:
print("<cntl-file> missing database")
print_help_and_exit()
query_file = cntl.read("queries")
if not query_file:
print("<cntl-file> missing queries")
print_help_and_exit()
window = cntl.read("window", num=True)
if window and window < 1: # window values smaller than 1 make no sense
print('Window values smaller than 1 not permitted -- using 1')
def main():
if len(sys.argv) < 5:
print("Usage")
print("\tpython3 rerank_results.py control input_results metric output_results")
print("")
print("Where:")
print("\tcontrol:\tPath to tangent control file")
print("\tinput_results:\tPath to file with results to re-rank")
print("\tmetric:\t\tSimilarity metric to use [0-4]")
print("\toutput_results:\tPath to file where re-ranked results will be stored")
print("")
print("Optional:")
print("\t-w\twindow\t\t: Window for pair generation")
print("\t-h\thtml_prefix\t: Prefix for HTML output (requires dot)")
print("\t-c\tcondition\t: Current test condition")
print("\t-s\tstats\t\t: File to store stats")
print("\t-t\ttimes\t\t: File to accumulate time stats")
return
control_filename = sys.argv[1]
input_filename = sys.argv[2]
try:
metric = int(sys.argv[3])
if metric < 0 or metric > 5:
print("Invalid similarity metric function")
return
except:
print("Invalid similarity metric function")
return
output_filename = sys.argv[4]
optional_params = optional_parameters(sys.argv[5:])
#load control file
control = Control(control_filename) # control file name (after indexing)
math_doc = MathDocument(control)
if "w" in optional_params:
try:
window = int(optional_params["w"])
if window <= 0:
print("Invalid window")
return
except:
print("Invalid value for window")
return
else:
window = int(control.read("window"))
if "h" in optional_params:
html_prefix = optional_params["h"]
if not os.path.isdir(html_prefix):
os.makedirs(html_prefix)
if not os.path.isdir(html_prefix + "/images"):
os.makedirs(html_prefix + "/images")
else:
html_prefix = None
if "c" in optional_params:
condition = optional_params["c"]
print("testing condition: " + condition)
else:
condition = "undefined"
if "s" in optional_params:
stats_file = optional_params["s"]
else:
stats_file = None
if "t" in optional_params:
times_file = optional_params["t"]
else:
times_file = None
in_file = open(input_filename, 'r', encoding="utf-8")
lines = in_file.readlines()
in_file.close()
mathml_cache_file = control_filename + ".retrieval_2.cache"
if not os.path.exists(mathml_cache_file):
mathml_cache = MathMLCache(control_filename)
else:
cache_file = open(mathml_cache_file, "rb")
mathml_cache = pickle.load(cache_file)
cache_file.close()
current_query = None
current_name = None
current_tuple_retrieval_time = 'undefined'
all_queries = []
#read all results to re-rank
for idx, line in enumerate(lines):
parts = line.strip().split("\t")
if len(parts) == 2:
if parts[0][0] == "Q":
current_name = parts[1]
current_query = None
elif parts[0][0] == "E":
if current_name is None:
print("invalid expression at " + str(idx) + ": query name expected first")
else:
query_expression = parts[1]
if html_prefix != None:
mathml = mathml_cache.get(-1, len(all_queries), query_expression)
else:
mathml = None
current_query = Query(current_name, query_expression, mathml, current_tuple_retrieval_time)
current_name = None
all_queries.append(current_query)
print("Query: " + current_query.name + ": " + current_query.expression, flush=True)
#print(mathml)
#current_query.tree.save_as_dot("expre_" + str(idx) + ".gv")
elif parts[0][0] == "C":
if current_query is None:
print("invalid constraint at " + str(idx) + ": query expression expected first")
else:
# create a constraint tree
current_query.set_constraints(parts[1])
# RZ: Record tuple-based retrieval time and other metrics.
if len(parts) == 3 and parts[0][0] == "I" and current_query != None:
if parts[1] == "qt":
current_query.initRetrievalTime = float( parts[2] )
elif parts[1] == "post":
current_query.postings = int( parts[2] )
elif parts[1] == "expr":
current_query.matchedFormulae = int( parts[2] )
elif parts[1] == "doc":
current_query.matchedDocs = int( parts[2] )
if len(parts) == 5:
if parts[0][0] == "R":
doc_id = int(parts[1])
location = int(parts[2])
doc_name = math_doc.find_doc_file(doc_id)
expression = parts[3]
score = float(parts[4])
if html_prefix != None:
mathml = mathml_cache.get(doc_id, location, expression)
else:
mathml = None
if current_query is None:
print("Error: result listed before a query, line " + str(idx))
else:
current_query.add_result(doc_id, doc_name, location, expression, score, mathml)
cache_file = open(mathml_cache_file, "wb")
pickle.dump(mathml_cache, cache_file, pickle.HIGHEST_PROTOCOL)
cache_file.close()
# now, re-rank...
# compute similarity first...
start_time = time.time()
for q_idx, query in enumerate(all_queries):
pairs_query = query.tree.root.get_pairs("", window)
#print("Evaluating: " + query.expression)
query_start_time = time.time() * 1000 # RZ: ms
for res_idx, exp_result in enumerate(query.results):
result = query.results[exp_result]
#print("Candidate: " + result.expression)
scores = [0.0]
if metric == 0:
# same as original based on f-measure of matched pairs...
pairs_candidate = result.tree.root.get_pairs("", window)
scores, matched_q, matched_c = similarity_v00(pairs_query, pairs_candidate)
elif metric == 1:
# based on testing of alignments....
scores, matched_q, matched_c = similarity_v01(query.tree, result.tree)
elif metric == 2:
# Same as 0 but limiting to matching total symbols first...
pairs_candidate = result.tree.root.get_pairs("", window)
scores, matched_q, matched_c = similarity_v02(pairs_query, pairs_candidate)
elif metric == 3:
# modified version of 2 which performs unification....
pairs_candidate = result.tree.root.get_pairs("", window)
scores, matched_q, matched_c, unified_c = similarity_v03(pairs_query, pairs_candidate)
result.set_unified_elements(unified_c)
elif metric == 4:
# modified version of 1 which performs unification ...
scores, matched_q, matched_c, unified_c = similarity_v04(query.tree, result.tree, query.constraints)
result.set_unified_elements(unified_c)
elif metric == 5:
# modified version of 4 which allows multiple sub matches
scores, matched_q, matched_c, unified_c = similarity_v05(query.tree, result.tree, query.constraints)
result.set_unified_elements(unified_c)
result.set_matched_elements(matched_c)
result.new_scores = scores
query_end_time = time.time() * 1000 # RZ: ms
# re-rank based on new score(s)
query.sort_results()
query.sort_documents()
query.elapsed_time = query_end_time - query_start_time
end_time = time.time()
elapsed = end_time - start_time
print("Elapsed Time Ranking: " + str(elapsed) + "s")
#now, store the re-ranked results...
out_file = open(output_filename, "w")
for query in all_queries:
out_file.write("\n")
query.output_query(out_file)
query.output_sorted_results(out_file)
if html_prefix is not None:
print("Saving " + query.name + " to HTML file.....")
query.save_html(html_prefix)
out_file.close()
#if stats file is requested ...
if stats_file is not None:
out_file = open(stats_file, "w")
out_file.write(Query.stats_header("\t"))
for query in all_queries:
query.output_stats(out_file,"\t", condition)
out_file.close()
# if times file is requested ...
if times_file is not None:
sorted_queries = sorted([(query.name.strip(), query) for query in all_queries])
if os.path.exists(times_file):
out_file = open(times_file, "a")
else:
out_file = open(times_file, "w")
header = "condition," + ",".join([name for (name, query) in sorted_queries])
out_file.write(header + "\n")
line = condition
for name, query in sorted_queries:
line += "," + str(query.elapsed_time)
out_file.write(line + "\n")
out_file.close()
print("Finished successfully")
print("%s is done saving to database %s" % (chunkid,fileid), flush=True)
return fileid, combined_stats
if __name__ == '__main__':
if sys.stdout.encoding != 'utf8':
sys.stdout = codecs.getwriter('utf8')(sys.stdout.buffer, 'strict')
if sys.stderr.encoding != 'utf8':
sys.stderr = codecs.getwriter('utf8')(sys.stderr.buffer, 'strict')
if (len(argv) > 2 or (len(argv) == 2 and argv[1] == 'help')): # uses control file to control all parameters
print_help_and_exit()
else:
start = time.time()
try:
cntl = Control(argv[1]) if len(argv) == 2 else Control()
except Exception as err:
print("Error in reading <cntl-file>: " +str(err))
print_help_and_exit()
doc_id_mapping_path = cntl.read("doc_list")
if not doc_id_mapping_path:
print("<cntl-file> missing doc_list")
print_help_and_exit()
database_name = cntl.read("database")
if not database_name:
print("<cntl-file> missing database")
print_help_and_exit()
window = cntl.read("window",num=True)
if window and window < 2: # window values smaller than 2 make no sense
print('Window values smaller than 2 not permitted -- using 2')
print(" math_results\\t<file with results from core formula retrieval engine")
print(" text_results\\t<file with results from text search engine>")
print(" combined_results\\t<file to store combined results>")
print(" combine_math\\t{'rerank' | 'average'} (mechanism for combining math results)")
print(" mweight\\t0..100 (percentage of weight on formula matches)")
print("and may optionally include:")
print(" run\\t<arbitrary name for query run>")
print("as well as other pairs.")
print("")
print("Optional additional command line parameters:")
print("\t-w\twindow\t\t: Window for pair generation")
exit()
#load control file
control = Control(sys.argv[1]) # control file name (after indexing)
math_doc = MathDocument(control)
minput_filename = control.read("math_results")
tinput_filename = control.read("text_results")
combiner = control.read("combine_math")
mweight = control.read("mweight",num=True,default=70)
output_filename = control.read("combined_results")
optional_params = optional_parameters(sys.argv[2:])
window = control.read("window",num=True,default=1)
if "w" in optional_params:
try:
w = int(optional_params["w"])
if __name__ == '__main__':
if sys.stdout.encoding != 'utf8':
sys.stdout = codecs.getwriter('utf8')(sys.stdout.buffer, 'strict')
if sys.stderr.encoding != 'utf8':
sys.stderr = codecs.getwriter('utf8')(sys.stderr.buffer, 'strict')
if (len(argv) > 2
or (len(argv) == 2 and argv[1]
== 'help')): # uses control file to control all parameters
print_help_and_exit()
else:
start = time.time()
try:
cntl = Control(argv[1]) if len(argv) == 2 else Control()
except Exception as err:
print("Error in reading <cntl-file>: " + str(err))
print_help_and_exit()
doc_id_mapping_path = cntl.read("doc_list")
if not doc_id_mapping_path:
print("<cntl-file> missing doc_list")
print_help_and_exit()
window = cntl.read("window", num=True)
if window and window < 1: # window values smaller than 2 make no sense
print('Window values smaller than 1 not permitted -- using 1')
window = 1
chunk_size = cntl.read("chunk_size", num=True, default=200)
print("reading %s" % doc_id_mapping_path, flush=True)
