def lines_in_files(path):
all_files = csv_access.list_files_in_directory(path)
total = 0
for f in all_files:
with open(f, "r") as g:
lines = g.readlines()
total += len(lines)
print("Total lines: " + str(total))
def get_spo_from_uns(path="/data/smalldatasets/clean_triple_relations/",
loc_dic=None):
all_files = csv_access.list_files_in_directory(path)
if loc_dic is not None:
# retrieve max index
loc_idx = max(list(loc_dic.values())) + 1 # the next one
spos = []
for fname in all_files:
if loc_dic is not None:
loc_dic[fname] = loc_idx
loc_idx += 1
df = pd.read_csv(fname, encoding='latin1')
for index, row in df.iterrows():
s, p, o = row['s'], row['p'], row['o']
s = nlp_utils.filter(s)
p = nlp_utils.filter(p)
o = nlp_utils.filter(o)
if s == "" or p == "" or o == "":
continue
spo = (s, p, o)
spos.append(spo)
print("Original data: " + str(len(spos)))
# Identify contained elements and remove those facts
to_remove = set()
for idx in range(len(spos)):
if idx in to_remove:
continue
s, p, o = spos[idx]
fact = s + " " + p + " " + o
ftok = fact.split(" ")
for jdx in range(len(spos)):
if jdx in to_remove:
continue
if jdx == idx:
continue
sj, pj, oj = spos[jdx]
alt_fact = sj + " " + pj + " " + oj
aftok = alt_fact.split(" ")
if len(set(ftok) - set(aftok)) == 0:
to_remove.add(idx)
# filter out non-answer bits as well as indexes deemed to remove
spos = [el for idx, el in enumerate(spos) if idx not in to_remove]
# for el in data:
# print(el)
print("Proc data: " + str(len(spos)))
return spos, loc_dic
def statistics_words(path):
all_files = csv_access.list_files_in_directory(path)
total_tokens = 0
total_lines = 0
for f in all_files:
with open(f, "r") as g:
lines = g.readlines()
total_lines += len(lines)
for l in lines:
tokens = l.split(" ")
total_tokens += len(tokens)
print("Total tokens: " + str(total_tokens))
print("Avg tokens: " + str(float(total_tokens/total_lines)))
def extract_data_from_directory(path="/Users/ra-mit/data/mitdwhdata/",
rows_per_relation=5):
all_files = csv_access.list_files_in_directory(path)
pairs = []
for filename in all_files:
name = filename.split("/")[-1].split(".")[0]
df = pd.read_csv(filename, encoding='latin1')
columns = df.columns
for c in columns:
pair = (name, c, 0)
pairs.append(pair) # filename - cols
current_rows = 0
for idx, row in df.iterrows():
current_rows += 1
# # filter cols based on type
# valid_type_columns = []
# for c in columns:
# if df[c].dtype == string:
# valid_type_columns.append(c)
# columns = valid_type_columns
for c in columns:
if re.search('[0-9]', str(row[c])) is not None:
continue
if pd.isnull(row[c]) or pd.isnull(row[c]):
continue
if str(row[c]) == 'nan':
continue
pair = (c, row[c], 0)
pairs.append(pair) # cols - colvalues
colref = columns[0]
for c1 in columns:
if re.search('[0-9]', str(
row[colref])) is not None or re.search(
'[0-9]', str(row[c1])) is not None:
continue
if pd.isnull(row[colref]) or pd.isnull(row[c1]):
continue
# if type(row[colref]) == float and np.isnan(row[colref]):
# continue
# if type(row[c1]) == float and np.isnan(row[c1]):
# continue
if str(row[colref]) == 'nan' or str(row[c1]) == 'nan':
continue
pair = (row[colref], row[c1], 0)
pairs.append(pair)
if current_rows > rows_per_relation:
break # go to next file
return pairs
def get_location_dictionary(path):
files = csv_access.list_files_in_directory(path)
return get_location_dictionary_from_files(files)
def get_sqa(path="/data/smalldatasets/clean_triple_relations/",
filter_stopwords=False,
loc_dic=None):
all_files = csv_access.list_files_in_directory(path)
data = []
if loc_dic is not None:
# retrieve max index
loc_idx = max(list(loc_dic.values())) + 1 # the next one
for fname in all_files:
if loc_dic is not None:
loc_dic[fname] = loc_idx
loc_idx += 1
df = pd.read_csv(fname, encoding='latin1')
for index, row in df.iterrows():
s, p, o = row['s'], row['p'], row['o']
s = nlp_utils.filter(s)
p = nlp_utils.filter(p)
o = nlp_utils.filter(o)
if s == "" or p == "" or o == "":
continue
this_support = s.split(" ") + p.split(" ") + o.split(" ")
this_question = s.split(" ") + p.split(" ")
this_answer = o.split(" ")
this_question2 = p.split(" ") + o.split(" ")
this_answer2 = s.split(" ")
# if filter_stopwords:
# this_support = [w for w in this_support if w not in english]
# this_question = [w for w in this_question if w not in english]
# this_question2 = [w for w in this_question2 if w not in english]
# this_answer = [w for w in this_answer if w not in english]
# this_answer2 = [w for w in this_answer2 if w not in english]
#if len(this_support) != 0 and len(this_question) != 0 and len(this_answer) != 0:
el1 = this_support, this_question, this_answer
# print(el1)
data.append(el1)
#if len(this_support) != 0 and len(this_question2) != 0 and len(this_answer2) != 0:
el2 = this_support, this_question2, this_answer2
# print(el2)
data.append(el2)
print("Original data: " + str(len(data)))
# Identify contained elements and remove those facts
to_remove = set()
for idx in range(0, len(data), 2):
fact, _, _ = data[idx]
for jdx in range(0, len(data), 2):
if jdx == idx:
continue
alt_fact, _, _ = data[jdx]
if len(set(fact) - set(alt_fact)) == 0:
to_remove.add(idx)
to_remove.add(idx + 1) # there are pairs of them
#break # move on to next fact
# filter out non-answer bits as well as indexes deemed to remove
data = [
el for idx, el in enumerate(data)
if idx not in to_remove and len(el[0]) > 2
]
# for el in data:
# print(el)
print("Proc data: " + str(len(data)))
return data, loc_dic
total_tokens += len(tokens)
print("Total tokens: " + str(total_tokens))
print("Avg tokens: " + str(float(total_tokens/total_lines)))
if __name__ == "__main__":
print("Process relations")
statistics_words("/Users/ra-mit/data/fabric/academic/clean_relations/")
# exit()
path = "/Users/ra-mit/data/fabric/academic/relations/"
out_path = "/Users/ra-mit/data/fabric/academic/clean_relations/"
pronouns = ["She", "He", "she", "he"]
all_files = csv_access.list_files_in_directory(path)
#all_files = [all_files[0]]
for fpath in all_files:
name = (fpath.split("/")[-1]).split(".")[0]
pre_processed_tokens = []
with open(fpath, "r") as f:
relations = f.readlines()
for r in relations:
tokens = r.split(" ")[1::] # remove number
pre_tokens = tp.tokenize(" ".join(tokens), " ") # clean stuff
pre_tokens = [el.strip() for el in pre_tokens]
# change pronouns by names
for idx in range(len(pre_tokens)):
def main(argv):
ifiles = ""
ofile = ""
training_data_path = ""
term_dictionary_path = ""
sparsity_code_size = 16
try:
opts, args = getopt.getopt(argv, "hvi:o:t:d:s:")
except getopt.GetoptError:
print("use it correctly ")
sys.exit(2)
for opt, arg in opts:
if opt == "-h":
print("wrong!")
sys.exit()
elif opt in "-i":
ifiles = arg
elif opt in "-o":
ofile = arg
elif opt in "-t":
training_data_path = arg
elif opt in "-d":
term_dictionary_path = arg
elif opt in "-s":
sparsity_code_size = int(arg)
# load existing dic
term_dic = None
with open(term_dictionary_path, "rb") as f:
term_dic = pickle.load(f)
# create vectorizer
idx_vectorizer = IndexVectorizer(vocab_index=term_dic,
sparsity_code_size=sparsity_code_size)
vectorizer = tp.CustomVectorizer(idx_vectorizer)
# reload existing training data into new file
with gzip.open(ofile + "/pairsdata/training_data.pklz", "wb") as g:
with gzip.open(training_data_path, "rb") as f:
try:
while True:
x1, x2, y = pickle.load(f)
pickle.dump((x1, x2, y), g)
except EOFError:
print("rewritten")
# read all unstructured files
all_files = csv_access.list_files_in_directory(ifiles)
#offset = 1
for i in range(len(all_files)):
ifile = all_files[i]
# get positive pairs from ifile
for x1, x2 in gen_pos_pairs(ifile, vectorizer):
pickle.dump((x1, x2, 0), g)
# gen negative pairs from all the jfiles
#for jfile in all_files[offset::]:
for jfile in all_files:
#jfile = all_files[j]
if ifile == jfile:
continue
for x1, x2 in gen_neg_pairs(ifile, jfile,
vectorizer): # neg from i to j
pickle.dump((x1, x2, 1), g)
#offset += 1 # advance offset to not repeat negative pairs
with gzip.open(ofile + "/training_data.pklz", "rb") as f:
with gzip.open(ofile + "/baedata/training_data.pklz", "wb") as g:
try:
while True:
x1, x2, y = pickle.load(f)
pickle.dump((x1, y), g)
pickle.dump((x2, y), g)
except EOFError:
print("rewritten")
vocab, inv_vocab = vectorizer.get_vocab_dictionaries()
with open(term_dictionary_path, "wb") as f:
pickle.dump(vocab, f)
print("Done!")