def test(model_props=None,
model_name=None,
weights_file='best_weights',
dataset_name='test',
save_output=True,
save_scores=False):
if model_props is None:
model_props = model_properties.MentionRankingProps(
name=model_name,
load_weights_from=model_name,
weights_file=weights_file)
print "Loading data"
vectors = np.load(directories.RELEVANT_VECTORS + 'word_vectors.npy')
dataset = datasets.DocumentBatchedDataset(dataset_name,
model_props,
with_ids=True)
docs = utils.load_pickle(directories.DOCUMENTS + dataset_name +
'_docs.pkl')
stats = {}
print "Building model"
model, _ = pairwise_models.get_model(dataset, vectors, model_props)
print "Evaluating model on", dataset_name
evaluate_model(dataset,
docs,
model,
model_props,
stats,
save_output=save_output,
save_scores=save_scores)
timer.clear()
utils.write_pickle(stats, model_props.path + dataset_name + "_results.pkl")
def write_action_spaces(dataset_name,
action_space_path,
model_path,
ltr=False):
output_file = action_space_path + dataset_name + "_action_space.pkl"
print "Writing candidate actions to " + output_file
scores = utils.load_pickle(model_path + dataset_name + "_scores.pkl")
write_probable_pairs(dataset_name, action_space_path, scores)
probable_pairs = utils.load_pickle(action_space_path + dataset_name +
'_probable_pairs.pkl')
possible_pairs_total = 0
action_spaces = []
for did in scores:
if did in probable_pairs:
actions = defaultdict(list)
for (m1, m2) in probable_pairs[did]:
actions[m2].append(m1)
if ltr:
actions = sorted(actions.items(),
cmp=lambda (ana1, ants1), (ana2, ants2): -1
if (ana1, ana2) in scores[did] else 1)
for i in range(len(actions) - 1):
assert (actions[i][0], actions[i + 1][0]) in scores[did]
else:
actions = sorted(actions.items(),
key=lambda (ana, ants): max(scores[did][
(ant, ana)] - scores[did][(-1, ana)]
for ant in ants))
possible_pairs = get_possible_pairs(probable_pairs[did])
possible_pairs_total += len(possible_pairs)
action_spaces.append(ActionSpace(did, actions, possible_pairs))
utils.write_pickle(action_spaces, output_file)
def get_asp_valid_path(model_str,
player_img,
answer_set_filename,
save=True):
# Initialize start and goal fact variables
start_nodes = []
goal_nodes = []
is_start_idx = State.prolog_state_contents_is_start_index()
goal_reached_idx = State.prolog_state_contents_goal_reached_index()
# Create new graph for model
graph = nx.Graph()
# Add nodes from reachable facts
reachable_facts = Solver.get_facts_as_list(model_str,
fact_name='reachable')
for reachable_fact in reachable_facts:
reachable_contents = Solver.get_fact_contents_as_list(
reachable_fact)
reachable_node = str(reachable_contents)
graph.add_node(reachable_node)
if reachable_contents[is_start_idx] == '1':
start_nodes.append(reachable_node)
if reachable_contents[goal_reached_idx] == '1':
goal_nodes.append(reachable_node)
# Check that reachable start and goal states exist
if len(start_nodes) == 0:
error_exit('No reachable start states found in model str')
if len(goal_nodes) == 0:
error_exit('No reachable goal states found in model str')
# Add edges from link facts
link_facts = Solver.get_facts_as_list(model_str, fact_name='link')
for link_fact in link_facts:
link_contents = Solver.get_fact_contents_as_list(link_fact)
src_node = str(link_contents[:len(link_contents) // 2])
dest_node = str(link_contents[len(link_contents) // 2:])
graph.add_edge(src_node, dest_node)
# Check if valid path exists from start to goal
for start_node in start_nodes:
for goal_node in goal_nodes:
valid_path_exists = nx.has_path(graph,
source=start_node,
target=goal_node)
if valid_path_exists:
valid_path = nx.dijkstra_path(graph,
source=start_node,
target=goal_node)
if save:
valid_path_str = " => \n".join(valid_path)
valid_path_file = get_filepath(
"level_saved_files_%s/generated_level_paths" %
player_img, "%s.pickle" % answer_set_filename)
write_pickle(valid_path_file, valid_path_str)
return valid_path
return None
def main(save_filename, unique_metatiles_file, player_img):
print("Creating id maps from unique metatiles file %s..." %
unique_metatiles_file)
start_time = datetime.now()
metatile_count = 0
id_metatile_map = {}
metatile_id_map = {}
unique_metatiles = read_pickle(unique_metatiles_file)
for metatile in unique_metatiles:
metatile_count += 1
metatile_id = "t%d" % metatile_count
metatile_str = metatile.to_str()
id_metatile_map[metatile_id] = metatile_str
metatile_id_map[metatile_str] = metatile_id
level_saved_files_dir = "level_saved_files_%s/" % player_img
outfile = "%s.pickle" % save_filename
id_metatile_map_file = get_filepath(
level_saved_files_dir + "id_metatile_maps", outfile)
metatile_id_map_file = get_filepath(
level_saved_files_dir + "metatile_id_maps", outfile)
write_pickle(id_metatile_map_file, id_metatile_map)
write_pickle(metatile_id_map_file, metatile_id_map)
end_time = datetime.now()
runtime = str(end_time - start_time)
print("Runtime: %s" % runtime)
return id_metatile_map_file, metatile_id_map_file, runtime
def main(save_filename, unique_metatiles_file, player_img, print_stats):
print("Calculating states per metatile stats for the given unique_metatiles_file: %s" % unique_metatiles_file)
start_time = datetime.now()
save_directory = "level_saved_files_%s/metatile_num_states_dicts/" % player_img
save_file = "%s.pickle" % save_filename
metatile_num_states_dict_file = get_filepath(save_directory, save_file)
unique_metatiles = read_pickle(unique_metatiles_file)
metatile_num_states_dict = {}
for metatile in unique_metatiles:
metatile_str = metatile.to_str()
metatile_graph = nx.DiGraph(metatile.graph_as_dict)
num_states = len(metatile_graph.nodes())
metatile_num_states_dict[metatile_str] = num_states
write_pickle(metatile_num_states_dict_file, metatile_num_states_dict)
end_time = datetime.now()
runtime = str(end_time-start_time)
if print_stats:
print(get_metatile_num_states_stats(metatile_num_states_dict))
print("Runtime: %s\n" % runtime)
return runtime
def preprocess_names(config):
word_counts = utils.load_pickle(config.word_counts_raw)
capitalized_counts = utils.load_pickle(config.capitalized_counts)
remove = ["english", "french", "german", "august", "president", "colonel", "lord", "june",
"major", "states", "august", "sunday", "christmas", "america", "paris", "france",
"florence", "roman", "israel", "ireland", "bible", "france", "england"]
name_stats = []
names, wordlike_names = set(), set()
for name_file in [config.first_names_txt, config.last_names_txt]:
with open(name_file) as f:
for line in f:
split = line.split()
name = split[0].lower()
freq = float(line.split()[1])
count_ratio = word_counts[name] / max(1, float(capitalized_counts[name]))
if name in remove and freq < 0.002:
print name
if name not in remove and freq > 0.002:
name_stats.append((name, word_counts[name], capitalized_counts[name],
count_ratio))
if count_ratio < 0.75:
names.add(name)
if count_ratio > 0.2:
wordlike_names.add(name)
for w, wc, cc, cr in sorted(name_stats, key=itemgetter(-1), reverse=True):
if cc > 500:
print w, wc, cc, cr
utils.write_pickle(names, config.names)
utils.write_pickle(wordlike_names, config.wordlike_names)
def run_evaluation(self):
train_scores, train_loss, dev_pairs = evaluate(self, self.dev_docs, self.dev_data,
"Evaluating on train")
test_scores, test_loss, test_pairs = evaluate(self, self.test_docs, self.test_data,
"Evaluating on test")
epoch_stats = {
"epoch": self.epoch,
"n": self.n,
"train_loss": train_loss,
"test_loss": test_loss
}
epoch_stats.update({"train " + k: v for k, v in train_scores.iteritems()})
epoch_stats.update({"test " + k: v for k, v in test_scores.iteritems()})
self.history.append(epoch_stats)
utils.write_pickle(self.history, self.model_props.path + 'history.pkl')
timer.print_totals()
test_conll = epoch_stats["test conll"]
if self.epoch % self.write_every == 0:
self.best_conll_window = 0
if test_conll > self.best_conll:
self.best_conll = test_conll
print "New best CoNLL, saving model"
self.save_progress(dev_pairs, test_pairs, "best")
if test_conll > self.best_conll_window:
self.best_conll_window = test_conll
print "New best CoNLL in window, saving model"
self.save_progress(dev_pairs, test_pairs,
str(self.write_every * int(self.epoch / self.write_every)))
self.model.save_weights(self.model_props.path + "weights.hdf5", overwrite=True)
def eval(self):
# self.logger.info('--------------------Evaluation: mAP@50-------------------')
self.ImgNet.eval().cuda()
self.TxtNet.eval().cuda()
re_BI, re_BT, re_LT, qu_BI, qu_BT, qu_LT, indexes = generate_hashes_from_dataloader(
self.database_loader, self.test_loader, self.ImgNet, self.TxtNet,
self.cfg.LABEL_DIM)
qu_BI = self.get_each_5th_element(qu_BI)
re_BI = self.get_each_5th_element(re_BI)
qu_LI = self.get_each_5th_element(qu_LT)
re_LI = self.get_each_5th_element(re_LT)
indexes = list(indexes)
indexes[0] = self.get_each_5th_element(indexes[0])
indexes[2] = self.get_each_5th_element(indexes[2])
self.visualize_retrieval(qu_BI, qu_BT, re_BI, re_BT, qu_LI, qu_LT,
re_LI, re_LT, indexes, 'DJSRH')
MAP_I2T, MAP_T2I, MAP_I2I, MAP_T2T, MAP_AVG = self.calc_maps_k(
qu_BI, qu_BT, re_BI, re_BT, qu_LI, qu_LT, re_LI, re_LT,
self.cfg.MAP_K)
MAPS = (MAP_I2T, MAP_T2I, MAP_I2I, MAP_T2T)
maps5 = (MAP_I2T, MAP_T2I, MAP_I2I, MAP_T2T, MAP_AVG)
maps10 = self.calc_maps_k(qu_BI, qu_BT, re_BI, re_BT, qu_LI, qu_LT,
re_LI, re_LT, 10)
maps20 = self.calc_maps_k(qu_BI, qu_BT, re_BI, re_BT, qu_LI, qu_LT,
re_LI, re_LT, 20)
mapshr = self.calc_maps_rad(qu_BI, qu_BT, re_BI, re_BT, qu_LI, qu_LT,
re_LI, re_LT, [0, 1, 2, 3, 4, 5])
top_k_hists(qu_BI, qu_BT, re_BI, re_BT, model='DJSRH')
hr_hists(qu_BI, qu_BT, re_BI, re_BT, model='DJSRH')
build_binary_hists(qu_BI, qu_BT, re_BI, re_BT, 'DJSRH',
[i[0] for i in mapshr])
maps_eval = (maps5, maps10, maps20, mapshr)
if (self.best_it + self.best_ti + self.best_ii +
self.best_tt) < (MAP_I2T + MAP_T2I + MAP_I2I + MAP_T2T):
self.best_it = MAP_I2T
self.best_ti = MAP_T2I
self.best_ii = MAP_I2I
self.best_tt = MAP_T2T
if not self.cfg.TEST:
self.save_checkpoints('best.pth')
if not self.cfg.TEST:
self.save_checkpoints('last.pth')
write_pickle(
osp.join(self.cfg.MODEL_DIR, self.path, 'maps_eval.pkl'),
maps_eval)
def export(self):
path = '../yelp_dataset/fm_res/'
V_filename = 'FM_V.pickle'
W_filename = 'FM_W.pickle'
V = self.V.detach()
W = [self.W.detach(), self.w0.detach()]
write_pickle(path + V_filename, V)
write_pickle(path + W_filename, W)
def write_feature_names():
utils.write_pickle(
{
f: i
for i, f in enumerate(
next(utils.load_json_lines(directories.RAW +
'train'))["pair_feature_names"])
}, directories.MISC + 'pair_feature_names.pkl')
def write_docs(dataset_name):
gold, mention_to_gold = load_gold(dataset_name)
mentions = load_mentions(dataset_name)
docs = []
for did in gold:
docs.append(Document(did, mentions[did],
gold[did], mention_to_gold[did]))
utils.write_pickle(docs, directories.DOCUMENTS + dataset_name + '_docs.pkl')
def write_docs(dataset_name):
gold, mention_to_gold = load_gold(dataset_name)
mentions = load_mentions(dataset_name)
docs = []
for did in gold:
docs.append(
Document(did, mentions[did], gold[did], mention_to_gold[did]))
utils.write_pickle(docs,
directories.DOCUMENTS + dataset_name + '_docs.pkl')
def write_genres():
sources = set()
for dataset_name in ["train"]:
print "Adding sources from", dataset_name
for d in docs(dataset_name):
sources.add(d["document_features"]["source"])
print sources
utils.write_pickle({source: i for i, source in enumerate(sorted(sources))},
directories.MISC + 'genres.pkl')
def export(self, filepath, metapath):
r"""
export the embeddings to files
"""
user_factors = self.user_factors.detach().cpu()
item_factors = self.item_factors.detach().cpu()
user_file = filepath + metapath + '_user' + '.pickle'
item_file = filepath + metapath + '_item' + '.pickle'
write_pickle(user_file, user_factors)
write_pickle(item_file, item_factors)
def prep_data(config):
vocab = vocabulary.Vocabulary(config)
for era in ["historic", "modern"]:
sequences = []
for sentence in sentences(config, True,
include_historic=(era == "historic"),
include_modern=(era == "modern")):
print " ".join([vocab[vocab[w]] for w in sentence])
sequences.append([vocab[w] for w in sentence])
utils.write_pickle(sequences, config.all_sequences[era])
def write_genres():
sources = set()
for dataset_name in ["train"]:
print "Adding sources from", dataset_name
for d in docs(dataset_name):
sources.add(d["document_features"]["source"])
print sources
utils.write_pickle({source: i
for i, source in enumerate(sorted(sources))},
directories.MISC + 'genres.pkl')
def process_answer_set(self, model_str):
player_img, prolog_filename = Solver.parse_prolog_filepath(
self.prolog_file)
answer_set_filename = self.get_cur_answer_set_filename(prolog_filename)
# Create assignments dictionary {(tile_x, tile_y): tile_id}
assignments_dict = Solver.create_assignments_dict(model_str)
# Create and save structural txt file for the generated level
level_structural_txt = ""
for row in range(self.level_h):
for col in range(self.level_w):
tile_xy = (col, row)
tile_id = assignments_dict.get(tile_xy)
tile_char = self.get_tile_char(tile_id)
level_structural_txt += tile_char
level_structural_txt += "\n"
if self.save:
generated_level_txt_dir = "level_structural_layers/generated/"
level_structural_txt_file = get_filepath(
generated_level_txt_dir, "%s.txt" % answer_set_filename)
write_file(level_structural_txt_file, level_structural_txt)
generated_level_assignments_dir = "level_saved_files_%s/generated_level_assignments_dicts/" % player_img
level_assignments_file = get_filepath(
generated_level_assignments_dir,
"%s.pickle" % answer_set_filename)
write_pickle(level_assignments_file, assignments_dict)
generated_level_model_str_dir = "level_saved_files_%s/generated_level_model_strs/" % player_img
level_model_str_file = get_filepath(generated_level_model_str_dir,
"%s.txt" % answer_set_filename)
write_pickle(level_model_str_file, model_str)
if self.print_level:
print(level_structural_txt)
if self.validate:
asp_valid = Solver.asp_is_valid(
check_path=True,
check_onground=self.require_all_platforms_reachable,
check_bonus=self.require_all_bonus_tiles_reachable,
model_str=model_str,
player_img=player_img,
answer_set_filename=answer_set_filename,
tile_ids=self.tile_ids.copy(),
save=self.save)
self.asp_valid_levels_count += 1 if asp_valid else 0
# state_graph_valid_path = Solver.get_state_graph_valid_path(assignments_dict, player_img, prolog_filename,
# answer_set_filename, save=self.save)
# self.state_graph_valid_levels_count += 1 if state_graph_valid_path is not None else 0
self.increment_answer_set_count()
def main(data_path):
# if os.path.isfile(data_path):
# df_loans = read_pickle("{}preprocessed_loans.pickle".format(data_path))
# else:
contract_addresses = get_contract_addresses()
loans = get_all_loans(contract_addresses)
write_pickle(loans, "{}loans.pickle".format(data_path))
#
# loans = read_pickle("{}loans.pickle".format(data_path))
df_loans = preprocess(loans)
write_pickle(df_loans, "{}preprocessed_loans.pickle".format(data_path))
def write_words():
words = Counter()
for dataset_name in ["train", "dev", "test"]:
inc = 1 if dataset_name == "train" else 0
print "Adding words from", dataset_name
for d in docs(dataset_name):
for mention in d["mentions"].values():
for w in mention["sentence"]:
words[word_vectors.normalize(w)] += inc
words[word_vectors.normalize(mention["dep_relation"])] += 1
utils.write_pickle(words, directories.MISC + 'word_counts.pkl')
def write_words():
words = Counter()
for dataset_name in ["train", "dev", "test"]:
inc = 1 if dataset_name == "train" else 0
print "Adding words from", dataset_name
for d in docs(dataset_name):
for mention in d["mentions"].values():
for w in mention["sentence"]:
words[word_vectors.normalize(w)] += inc
words[word_vectors.normalize(mention["dep_relation"])] += 1
utils.write_pickle(words, directories.MISC + 'word_counts.pkl')
def build_data():
training_data = {}
testing_data = {}
train_size = 120
for i in range(10):
imgs = get_imgs_by_number(i)
random.shuffle(imgs)
training_data[i] = [img[1] for img in imgs[:train_size]]
testing_data[i] = [img[1] for img in imgs[train_size:]]
utils.write_pickle(training_data, 'train_data.pkl')
utils.write_pickle(testing_data, 'test_data.pkl')
print('training and testing data is shuffled and written')
def count_words(config, normalize):
word_counts = Counter()
capitalized_counts = Counter()
for sentence in sentences(config, normalize):
for i in range(len(sentence)):
w = sentence[i]
word_counts[w] += 1
if w[0].isupper() and not first_word(sentence, i):
capitalized_counts[w.lower()] += 1
utils.write_pickle(word_counts, config.word_counts if normalize else config.word_counts_raw)
if not normalize:
utils.write_pickle(capitalized_counts, config.capitalized_counts)
def main(save_filename, metatile_id_map_file, id_metatile_map_file,
metatile_coords_dict_files, player_img):
print("Constructing tile_id constraints dictionary...")
start_time = datetime.now()
# Create save file path
metatile_constraints_dir = "level_saved_files_%s/metatile_constraints" % player_img
metatile_constraints_file = get_filepath(metatile_constraints_dir,
"%s.pickle" % save_filename)
# Load in files
metatile_id_map = read_pickle(metatile_id_map_file)
id_metatile_map = read_pickle(id_metatile_map_file)
metatile_coords_dicts = [
read_pickle(file) for file in metatile_coords_dict_files
]
coord_metatiles_dict = get_coord_metatiles_dict(metatile_coords_dicts)
coord_tile_ids_map = get_coord_tile_ids_map(metatile_id_map,
coord_metatiles_dict)
tile_id_constraints_dict = {}
for tile_id, metatile_str in id_metatile_map.items():
metatile = Metatile.from_str(metatile_str)
tile_id_constraints_dict[tile_id] = {
"type": metatile.type,
"graph": metatile.graph_as_dict,
"games": metatile.games,
"levels": metatile.levels,
"adjacent": {
TOP: [],
BOTTOM: [],
LEFT: [],
RIGHT: [],
TOP_LEFT: [],
BOTTOM_LEFT: [],
TOP_RIGHT: [],
BOTTOM_RIGHT: []
}
}
tile_id_constraints_dict = populate_tile_id_constraints_adjacencies(
tile_id_constraints_dict, coord_tile_ids_map)
end_time = datetime.now()
runtime = str(end_time - start_time)
write_pickle(metatile_constraints_file, tile_id_constraints_dict)
print("Runtime: %s\n" % runtime)
return metatile_constraints_file, runtime
def main(config, train_size=0.8, min_length=6, max_length=30):
for era in ["historic", "modern"]:
seqs = utils.load_pickle(config.all_sequences[era])
n = len(seqs)
seqs = [s for s in seqs if min_length <= len(s) <= max_length]
print len(seqs), n
lengths = [len(s) for s in seqs]
#sns.distplot(lengths)
#plt.show()
random.shuffle(seqs)
train_size = int(train_size * len(seqs))
utils.write_pickle(seqs[:train_size], config.train_sequences[era])
utils.write_pickle(seqs[train_size:], config.dev_sequences[era])
def stemWords():
from operator import add
import utils
from nltk.stem.porter import *
stemmer = PorterStemmer()
D2 = utils.read_pickle('../all-word-series')
D3 = {}
for k, v in D2.iteritems():
sk = stemmer.stem(k)
if sk not in D3:
D3[sk] = v
else:
D3[sk] = map(add, D3[sk], v)
utils.write_pickle(D3, '../stemmed-all-unigrams')
def save_process_runtimes(process_key, process_runtimes):
all_levels_process_info_file = utils.get_filepath(
"", "all_levels_process_info.pickle")
if os.path.exists(all_levels_process_info_file):
all_levels_process_info = utils.read_pickle(
all_levels_process_info_file)
else:
all_levels_process_info = {}
if all_levels_process_info.get(process_key) is None:
all_levels_process_info[process_key] = {}
for process_step, runtime_str in process_runtimes:
all_levels_process_info[process_key][process_step] = runtime_str
utils.write_pickle(all_levels_process_info_file, all_levels_process_info)
def write_probable_pairs(dataset_name, action_space_path, scores):
probable_pairs = {}
margin_removals = 0
total_pairs = 0
total_size = 0
for did in utils.logged_loop(scores):
doc_scores = scores[did]
pairs = sorted([pair for pair in doc_scores.keys() if pair[0] != -1],
key=lambda pr: doc_scores[pr] - (-1 - 0.3 * doc_scores[
(-1, pr[1])]),
reverse=True)
total_pairs += len(pairs)
probable_pairs[did] = []
for pair in pairs:
score = doc_scores[pair] - (-1 - 0.3 * doc_scores[(-1, pair[1])])
if score < SCORE_THRESHOLD:
break
probable_pairs[did].append(pair)
max_scores = {}
for pair in probable_pairs[did]:
if pair[1] not in max_scores:
max_scores[pair[1]] = max(doc_scores[pair],
-1 - 0.3 * doc_scores[(-1, pair[1])])
else:
max_scores[pair[1]] = max(max_scores[pair[1]],
doc_scores[pair])
margin_removals += len(probable_pairs[did])
probable_pairs[did] = [
p for p in probable_pairs[did]
if doc_scores[p] - max_scores[p[1]] > MARGIN_THRESHOLD
]
margin_removals -= len(probable_pairs[did])
total_size += len(probable_pairs[did])
print "num docs:", len(scores)
print "avg size without filter: {:.1f}".format(total_pairs /
float(len(scores)))
print "avg size: {:.1f}".format(total_size / float(len(scores)))
print "margin removals size: {:.1f}".format(margin_removals /
float(len(scores)))
utils.write_pickle(
probable_pairs,
action_space_path + dataset_name + '_probable_pairs.pkl')
shutil.copyfile('clustering_preprocessing.py',
action_space_path + 'clustering_preprocessing.py')
def write_feature_names():
raw_train = directories.RAW + 'train'
try:
utils.write_pickle(
{
f: i
for i, f in enumerate(
next(utils.load_json_lines(raw_train))
["pair_feature_names"])
}, directories.MISC + 'pair_feature_names.pkl')
except FileNotFoundError as e:
if e.filename == raw_train:
raise FileNotFoundError(
'Raw training data not found. Perhaps you need to copy the original dataset first: %s'
% e.filename) from e
else:
raise
def eval_acc(data):
query_path = './queries.pkl'
rewards_path = './rewards.txt'
write_pickle(query_path, data)
while(not os.path.isfile(rewards_path)):
sleep(0.1)
rewards_file = open(rewards_path, 'r')
acc = rewards_file.read()
os.remove(rewards_path)
try:
acc = float(acc)
except Exception:
acc = 0.0
return acc
def test(model_props=None, model_name=None, weights_file='best_weights', dataset_name='test',
save_output=True, save_scores=False):
if model_props is None:
model_props = model_properties.MentionRankingProps(name=model_name,
load_weights_from=model_name,
weights_file=weights_file)
print "Loading data"
vectors = np.load(directories.RELEVANT_VECTORS + 'word_vectors.npy')
dataset = datasets.DocumentBatchedDataset(dataset_name, model_props, with_ids=True)
docs = utils.load_pickle(directories.DOCUMENTS + dataset_name + '_docs.pkl')
stats = {}
print "Building model"
model, _ = pairwise_models.get_model(dataset, vectors, model_props)
print "Evaluating model on", dataset_name
evaluate_model(dataset, docs, model, model_props, stats,
save_output=save_output, save_scores=save_scores)
timer.clear()
utils.write_pickle(stats, model_props.path + dataset_name + "_results.pkl")
def write_document_vectors():
vectors = word_vectors.WordVectors(load=True)
for dataset_name in ["train", "dev", "test"]:
print "Building document vectors for", dataset_name
doc_vectors = {}
for d in docs(dataset_name):
sentences = {}
did = None
for mention_num in sorted(d["mentions"].keys(), key=int):
m = d["mentions"][mention_num]
did = m["doc_id"]
if m['sent_num'] not in sentences:
sentences[m['sent_num']] = m['sentence']
v = np.zeros(vectors.vectors[0].size)
n = 0
for s in sentences.values():
for w in s:
v += vectors.vectors[vectors[w]]
n += 1
doc_vectors[did] = v / n
utils.write_pickle(doc_vectors, directories.MISC + dataset_name + "_document_vectors.pkl")
def evaluate_model(dataset, docs, model, model_props, stats, save_output=False, save_scores=False,
print_table=False):
prog = utils.Progbar(dataset.n_batches)
mt = RankingMetricsTracker(dataset.name, model_props=model_props) \
if model_props.ranking else ClassificationMetricsTracker(dataset.name)
mta = ClassificationMetricsTracker(dataset.name + " anaphoricity", anaphoricity=True)
docs_by_id = {doc.did: doc for doc in docs} if model_props.ranking else {}
saved_links, saved_scores = (defaultdict(list) if save_output else None,
defaultdict(dict) if save_scores else None)
for i, X in enumerate(dataset):
if X['y'].size == 0:
continue
progress = []
scores = model.predict_on_batch(X)
if model_props.ranking:
update_doc(docs_by_id[X['did']], X, scores,
saved_links=saved_links, saved_scores=saved_scores)
if model_props.anaphoricity and not model_props.ranking:
progress.append(("anaphoricity loss", mta.update(X, scores[0][:, 0])))
if not model_props.anaphoricity_only:
progress.append(("loss", mt.update(
X, scores if model_props.ranking else
scores[1 if model_props.anaphoricity else 0][:, 0])))
prog.update(i + 1, exact=progress)
if save_scores:
print "Writing scores"
utils.write_pickle(saved_scores, model_props.path + dataset.name + '_scores.pkl')
if save_output:
print "Writing output"
utils.write_pickle(saved_links, model_props.path + dataset.name + '_links.pkl')
utils.write_pickle(docs, model_props.path + dataset.name + '_processed_docs.pkl')
timer.start("metrics")
if model_props.ranking:
stats.update(compute_metrics(docs, dataset.name))
stats["validate time"] = time.time() - prog.start
if model_props.anaphoricity and not model_props.ranking:
mta.finish(stats)
if not model_props.anaphoricity_only:
mt.finish(stats)
timer.stop("metrics")
if print_table:
print " & ".join(map(lambda x: "{:.2f}".format(x * 100), [
stats[dataset.name + " muc precision"],
stats[dataset.name + " muc recall"],
stats[dataset.name + " muc"],
stats[dataset.name + " b3 precision"],
stats[dataset.name + " b3 recall"],
stats[dataset.name + " b3"],
stats[dataset.name + " ceafe precision"],
stats[dataset.name + " ceafe recall"],
stats[dataset.name + " ceafe"],
stats[dataset.name + " conll"],
]))
def run_evaluation(self):
train_scores, train_loss, dev_pairs = evaluate(self, self.dev_docs,
self.dev_data,
"Evaluating on train")
test_scores, test_loss, test_pairs = evaluate(self, self.test_docs,
self.test_data,
"Evaluating on test")
epoch_stats = {
"epoch": self.epoch,
"n": self.n,
"train_loss": train_loss,
"test_loss": test_loss
}
epoch_stats.update(
{"train " + k: v
for k, v in train_scores.iteritems()})
epoch_stats.update(
{"test " + k: v
for k, v in test_scores.iteritems()})
self.history.append(epoch_stats)
utils.write_pickle(self.history, self.model_props.path + 'history.pkl')
timer.print_totals()
test_conll = epoch_stats["test conll"]
if self.epoch % self.write_every == 0:
self.best_conll_window = 0
if test_conll > self.best_conll:
self.best_conll = test_conll
print "New best CoNLL, saving model"
self.save_progress(dev_pairs, test_pairs, "best")
if test_conll > self.best_conll_window:
self.best_conll_window = test_conll
print "New best CoNLL in window, saving model"
self.save_progress(
dev_pairs, test_pairs,
str(self.write_every * int(self.epoch / self.write_every)))
self.model.save_weights(self.model_props.path + "weights.hdf5",
overwrite=True)
def test_model_actively(self):
"""
Starts the active learning process
"""
logging.info(f'Starting validation for {self.amine}')
# Run forward pass on the validation data
logging.debug(f'Weights for loss function: {self.weights}')
iters, all_data, all_labels, x_t, y_t, x_v, y_v = self.setup_active_learning(
)
for i in range(iters):
logging.debug(
f'Doing active learning with {len(x_t)} example. Iteration: {i}'
)
# Update available datapoints in the pool and evaluate current
# model performance
x_t, y_t, x_v, y_v = self.active_learning(all_data, all_labels,
x_t, y_t, x_v, y_v)
# Save this dictionary in case we need it later
write_pickle(self.dst_folder / Path('cv_statistics.pkl'),
self.cv_statistics)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--attn-data-file",
required=True,
help="Pickle file containing extracted attention maps.")
parser.add_argument("--outfile",
required=True,
help="Where to write out the distances between heads.")
args = parser.parse_args()
print("Loading attention data")
data = utils.load_pickle(args.attn_data_file)
print("Computing head distances")
js_distances = np.zeros([144, 144])
for doc in utils.logged_loop(data, n_steps=None):
if "attns" not in doc:
continue
tokens, attns = doc["tokens"], np.array(doc["attns"])
attns_flat = attns.reshape([144, attns.shape[2], attns.shape[3]])
for head in range(144):
head_attns = np.expand_dims(attns_flat[head], 0)
head_attns_smoothed = (0.001 / head_attns.shape[1]) + (head_attns *
0.999)
attns_flat_smoothed = (0.001 / attns_flat.shape[1]) + (attns_flat *
0.999)
m = (head_attns_smoothed + attns_flat_smoothed) / 2
js = -head_attns_smoothed * np.log(m / head_attns_smoothed)
js += -attns_flat_smoothed * np.log(m / attns_flat_smoothed)
js /= 2
js = js.sum(-1).sum(-1)
js_distances[head] += js
utils.write_pickle(js_distances, args.outfile)
def main(game, level, metatile_coords_dict_file, metatile_id_map_file,
player_img):
start_time = datetime.now()
print("\nCreating {(tile id, extra_info): coords} map ...")
metatile_coords_dict = read_pickle(metatile_coords_dict_file)
metatile_id_map = read_pickle(metatile_id_map_file)
tile_id_extra_info_coords_map = {}
for metatile_str, coords in metatile_coords_dict.items():
metatile = Metatile.from_str(metatile_str)
metatile_id = metatile_id_map.get(metatile_str)
if metatile_id is None:
error_exit("metatile_str not found in metatile_id_map")
has_graph = bool(metatile.graph_as_dict)
extra_info = ""
if not has_graph:
extra_info += "E" # metatile graph is empty
if len(coords) == 1:
extra_info += "S" # metatile was only used for one tile
tile_id_extra_info_coords_map[(metatile_id, extra_info)] = coords
save_directory = "level_saved_files_%s/tile_id_coords_maps/%s/" % (
player_img, game)
tile_id_extra_info_coords_map_file = get_filepath(save_directory,
"%s.pickle" % level)
write_pickle(tile_id_extra_info_coords_map_file,
tile_id_extra_info_coords_map)
end_time = datetime.now()
runtime = str(end_time - start_time)
print("Runtime: %s\n" % runtime)
return tile_id_extra_info_coords_map_file, runtime
def write_document_vectors():
vectors = word_vectors.WordVectors(load=True)
for dataset_name in ["train", "dev", "test"]:
print "Building document vectors for", dataset_name
doc_vectors = {}
for d in docs(dataset_name):
sentences = {}
did = None
for mention_num in sorted(d["mentions"].keys(), key=int):
m = d["mentions"][mention_num]
did = m["doc_id"]
if m['sent_num'] not in sentences:
sentences[m['sent_num']] = m['sentence']
v = np.zeros(vectors.vectors[0].size)
n = 0
for s in sentences.values():
for w in s:
v += vectors.vectors[vectors[w]]
n += 1
doc_vectors[did] = v / n
utils.write_pickle(
doc_vectors,
directories.MISC + dataset_name + "_document_vectors.pkl")
def write(self, path):
utils.write_pickle(self.__dict__, path)
def train(model_props, n_epochs=10000, reduced=False, dev_set_name='dev'):
print "Training", model_props.path
pprint(model_props.__dict__)
model_props.write(model_props.path + 'model_props.pkl')
utils.rmkdir(model_props.path + 'src')
for fname in os.listdir('.'):
if fname.endswith('.py'):
shutil.copyfile(fname, model_props.path + 'src/' + fname)
if model_props.ranking or \
model_props.top_pairs:
write_start = 0
write_every = 10
else:
write_start = 80
write_every = 20
print "Loading data"
vectors = np.load(directories.RELEVANT_VECTORS + 'word_vectors.npy')
train = datasets.DocumentBatchedDataset("train_reduced" if reduced else "train",
model_props, with_ids=True)
dev = datasets.DocumentBatchedDataset(dev_set_name + "_reduced" if reduced else dev_set_name,
model_props, with_ids=True)
print "Building model"
model, _ = pairwise_models.get_model(dev, vectors, model_props)
json_string = model.to_json()
open(model_props.path + 'architecture.json', 'w').write(json_string)
best_val_score = 1000
best_val_score_in_window = 1000
history = []
for epoch in range(n_epochs):
timer.start("train")
print "EPOCH {:}, model = {:}".format((epoch + 1), model_props.path)
epoch_stats = {}
model_weights = model.get_weights()
train_docs = utils.load_pickle(directories.DOCUMENTS + 'train_docs.pkl')
dev_docs = utils.load_pickle(directories.DOCUMENTS + dev_set_name + '_docs.pkl')
if reduced:
dev_docs = dev_docs[:3]
if model_props.ranking:
print "Running over training set"
run_model_over_docs(train, train_docs, model)
epoch_stats.update(compute_metrics(train_docs, "train"))
if model_props.use_rewards:
print "Setting costs"
set_costs(train, train_docs)
print "Training"
prog = utils.Progbar(train.n_batches)
train.shuffle()
loss_sum, n_examples = 0, 0
for i, X in enumerate(train):
if X['y'].size == 0:
continue
batch_loss = model.train_on_batch(X)
loss_sum += batch_loss * train.scale_factor
n_examples += X['y'].size
prog.update(i + 1, exact=[("train loss", loss_sum / n_examples)])
epoch_stats["train time"] = time.time() - prog.start
for k in prog.unique_values:
epoch_stats[k] = prog.sum_values[k][0] / max(1, prog.sum_values[k][1])
epoch_stats["weight diffs"] = [
(np.sum(np.abs(new_weight - old_weight)), new_weight.size)
for new_weight, old_weight in zip(model.get_weights(), model_weights)]
summed = np.sum(map(np.array, epoch_stats["weight diffs"][1:]), axis=0)
epoch_stats["total weight diff"] = tuple(summed)
print "Testing on dev set"
evaluate_model(dev, dev_docs, model, model_props, epoch_stats)
history.append(epoch_stats)
utils.write_pickle(history, model_props.path + 'history.pkl')
score = -epoch_stats["dev conll"] if model_props.ranking else \
(epoch_stats["dev loss"] if not model_props.anaphoricity_only else
epoch_stats["dev anaphoricity loss"])
if score < best_val_score:
best_val_score = score
print "New best {:}, saving model".format(
"CoNLL F1" if model_props.ranking else "validation loss")
model.save_weights(model_props.path + "best_weights.hdf5", overwrite=True)
if score < best_val_score_in_window and epoch > write_start:
print "Best in last {:}, saved to weights_{:}".format(
write_every, write_every * (epoch / write_every))
best_val_score_in_window = score
model.save_weights(model_props.path + "weights_{:}.hdf5".format(
write_every * (epoch / write_every)), overwrite=True)
if epoch + write_every >= n_epochs:
model.save_weights(model_props.path + "final_weights.hdf5", overwrite=True)
if epoch % write_every == 0:
best_val_score_in_window = 1000
timer.stop("train")
timer.print_totals()
print
timer.clear()
def write_feature_names():
utils.write_pickle({f: i for i, f in enumerate(next(
utils.load_json_lines(directories.RAW + 'train'))["pair_feature_names"])},
directories.MISC + 'pair_feature_names.pkl')
