def fit_and_plot(histname, fittype, fitrange=[], fitparam=[], fitparlim=[]):
h_invmass = []
c = []
hist = open_hist(histname)
print(utils.nbins)
for p in range(utils.nbins):
name = "InvMass" + histname + str(utils.pt_low[p]) + "_to_" + str(
utils.pt_high[p])
thefit = utils.set_fit_function(fittype, "thefit", fitrange, fitparam,
fitparlim)
ptlow = utils.get_bin(hist, utils.pt_low[p], False)
pthigh = utils.get_bin(hist, utils.pt_high[p], False)
h_invmass.insert(p, hist.ProjectionX(name, ptlow, pthigh - 1))
h_invmass[p].Fit("thefit", "R")
c.insert(p, ROOT.TCanvas("c" + str(p), "c" + str(p), 700, 700))
ROOT.gStyle.SetOptStat(0)
title_name = "Inv Mass " + histname + " {} < pt < {}".format(
utils.pt_low[p], utils.pt_high[p])
utils.setup_hist_to_draw(h_invmass[p], title_name, "inv mass", "",
[1.7, 2.05])
h_invmass[p].Draw("E0")
title_name = "Inv Mass {} {} < pt < {}".format("Reflections",
utils.pt_low[p],
utils.pt_high[p])
c[p].SaveAs(plot_dir + histname + "_InvMass_pt_" +
str(utils.pt_low[p]) + "_to_" + str(utils.pt_high[p]) +
png)
def color_stdout(cmd, text):
exe = get_bin(cmd)
exe_sub = [exe] + flags.get(cmd, []) + text
p = subprocess.run(exe_sub,
stdout=subprocess.PIPE,
encoding=encoding,
env=environment)
color_stdout = func.get(cmd, defaultc)(p.stdout)
sys.stdout.write(color_stdout)
def generate_vec(doc, sentences):
vectorized_sentences = []
for i in range(len(sentences)):
nodes = sentences[i]
vectorize_words = list_vectorize(nodes, words)
vectorized_sentences.append(vectorize_words)
vec, mask = mask_array(vectorized_sentences)
doc.vec = vec
doc.mask = mask
inpt_len = numpy.sum(doc.mask, 1)
doc.sentence_len = inpt_len
pre_index, post_index, zp2sen_dict, zp2real = generate_ZP_index(inpt_len)
doc.zp_pre_index = pre_index
doc.zp_post_index = post_index
np_index_start, np_index_end, np_indecs, np_mask, np_sen2index_dict, np2real = generate_NP_index(
inpt_len, MAX=nnargs["max_width"])
doc.np_index_start = np_index_start
doc.np_index_end = np_index_end
doc.np_indecs = np_indecs
doc.np_mask = np_mask
doc.zp2real = zp2real # zp_index: real_sentence_num,real_index
doc.np2real = np2real # np_index: real_sentence_num,real_start_index,real_end_index
zp2candi_dict = {} #zp_index: [np_index]
for i in range(len(pre_index)):
zp_index = pre_index[i]
sen_index = zp2sen_dict[i]
zp2candi_dict[i] = []
for sen_id in range(max(0, sen_index - 2), sen_index + 1):
np_indexs = np_sen2index_dict[sen_id]
for np_index in np_indexs:
if not ((sen_id == sen_index) and
(np_index_end[np_index] > zp_index)):
zp2candi_dict[i].append(np_index)
doc.zp2candi_dict = zp2candi_dict
zp_candi_distance_dict = {}
gold_azp = []
for zp_index in doc.zp2candi_dict:
gold_azp_add = 0
if len(doc.zp2candi_dict[zp_index]) > 0:
this_zp_real_sentence_num, this_zp_real_index = doc.zp2real[
zp_index]
np_indexes_of_zp = doc.zp2candi_dict[zp_index]
max_index = max(np_indexes_of_zp)
zp_candi_distance_dict[zp_index] = [] #utils.get_bin(distance)
for ii, np_index in enumerate(np_indexes_of_zp):
distance = max_index - np_index
zp_candi_distance_dict[zp_index].append(
utils.get_bin(distance))
doc.zp_candi_distance_dict = zp_candi_distance_dict
def insert(self, filename):
"""
This function is used to upload log data into the SQL DB.
:param: Takes class params and set of params to be inserted.
:type: list
:rtype: string,list
:return: returns a query string and a list of contents.
"""
query = 'INSERT INTO ' + self.TABLE_NAME + '(data_id, bin, filename, time_stamp) ' \
'VALUES (%s,%s,%s, CURRENT_TIMESTAMP);'
content = (self.data_id, get_bin(self.read_bin_dir + filename),
filename)
return (query, content)
def integrate_hist(hist, xvalues=[], yvalues=[]):
if xvalues:
if yvalues:
integral_val = hist.Integral(
utils.get_bin(hist, xvalues[0]),
utils.get_bin(hist, xvalues[1]),
utils.get_bin(hist, yvalues[0], False),
utils.get_bin(hist, yvalues[1], False))
else:
integral_val = hist.Integral(utils.get_bin(hist, xvalues[0]),
utils.get_bin(hist, xvalues[1]))
else:
integral_val = hist.Integral()
return integral_val
IMAGE_WIDTH = 256
IMAGE_HEIGHT = 256
########################################################################################################################
model = load_model('Model_Task2',
custom_objects={'loss_fun_task2': loss_fun_task2})
model.load_weights('./10-epochs.h5')
########################################################################################################################
data_x_vel = np.load('data_x_vel.npy')
data_y = np.load('data_y.npy')
num_classes = 100
bin_vx = get_bin(data_y[:, 0], num_classes)
bin_vy = get_bin(data_y[:, 1], num_classes)
dataset_path = './Generated_dataset/Task-2/'
test_subtask_no = str(98)
subtask = 'Task-2-' + test_subtask_no
subtask_folder = os.path.join(dataset_path, subtask)
task_no = subtask.split('-')[1]
subtask_no = subtask.split('-')[2]
file = 'task-0000' + str(task_no) + '_' + str(subtask_no) + '.npy'
data = np.load(os.path.join(dataset_path, file))
print(data.shape)
def generate_vec(data_path, docs):
read_f = file("./data/emb", "rb")
embedding, words, _ = cPickle.load(read_f)
read_f.close()
for doc in docs:
# generate vector for this doc
vectorized_sentences = []
for i in range(doc.sentence_num):
nodes = doc.filter_nodes[i]
vectorize_words = list_vectorize(nodes, words)
vectorized_sentences.append(vectorize_words)
vec, mask = mask_array(vectorized_sentences)
doc.vec = vec
doc.mask = mask
inpt_len = numpy.sum(doc.mask, 1)
doc.sentence_len = inpt_len
pre_index, post_index, zp2sen_dict, zp2real = opt.generate_ZP_index(
inpt_len)
doc.zp_pre_index = pre_index
doc.zp_post_index = post_index
np_index_start, np_index_end, np_indecs, np_mask, np_sen2index_dict, np2real = opt.generate_NP_index(
inpt_len, MAX=nnargs["max_width"])
doc.np_index_start = np_index_start
doc.np_index_end = np_index_end
doc.np_indecs = np_indecs
doc.np_mask = np_mask
doc.zp2real = zp2real # zp_index: real_sentence_num,real_index
doc.np2real = np2real # np_index: real_sentence_num,real_start_index,real_end_index
zp2candi_dict = {} #zp_index: [np_index]
for i in range(len(pre_index)):
zp_index = pre_index[i]
sen_index = zp2sen_dict[i]
zp2candi_dict[i] = []
for sen_id in range(max(0, sen_index - 2), sen_index + 1):
np_indexs = np_sen2index_dict[sen_id]
for np_index in np_indexs:
if not ((sen_id == sen_index) and
(np_index_end[np_index] > zp_index)):
zp2candi_dict[i].append(np_index)
doc.zp2candi_dict = zp2candi_dict
zp_candi_distance_dict = {}
zp_candi_coref = {
} # zp_index: [0,1,0,0,0] coreference result for each of its candidate
gold_np = [0] * len(np2real)
for i, (this_np_real_sentence_num, this_np_real_start,
this_np_real_end) in enumerate(np2real):
if (this_np_real_sentence_num, this_np_real_start,
this_np_real_end) in doc.np_dict:
gold_np[i] = 1
gold_azp = []
train_ante = []
gold_ante = [0] * len(gold_np)
for zp_index in doc.zp2candi_dict:
gold_azp_add = 0
if len(doc.zp2candi_dict[zp_index]) > 0:
this_zp_real_sentence_num, this_zp_real_index = doc.zp2real[
zp_index]
this_zp = None
if (this_zp_real_sentence_num,
this_zp_real_index) in doc.zp_dict:
this_zp = doc.zp_dict[(this_zp_real_sentence_num,
this_zp_real_index)]
np_indexes_of_zp = doc.zp2candi_dict[zp_index]
max_index = max(np_indexes_of_zp)
zp_candi_coref[zp_index] = numpy.array(
[0] * (len(np_indexes_of_zp) +
1)) #last index = 1 means zp is not azp
zp_candi_distance_dict[zp_index] = [] #utils.get_bin(distance)
for ii, np_index in enumerate(np_indexes_of_zp):
distance = max_index - np_index
zp_candi_distance_dict[zp_index].append(
utils.get_bin(distance))
this_np_real_sentence_num, this_np_real_start, this_np_real_end = doc.np2real[
np_index]
if (this_np_real_sentence_num, this_np_real_start,
this_np_real_end) in doc.np_dict:
this_candi = doc.np_dict[(this_np_real_sentence_num,
this_np_real_start,
this_np_real_end)]
if this_zp:
if this_candi in this_zp.antecedent:
zp_candi_coref[zp_index][ii] = 1
gold_ante[np_index] = 1
train_ante.append(np_index)
gold_azp_add = 1
if sum(zp_candi_coref[zp_index]) == 0:
zp_candi_coref[zp_index][-1] = 1
gold_azp_add = 0
gold_azp.append(gold_azp_add)
#print sum(gold_azp),len(doc.all_azps)
train_azp = numpy.array(gold_azp).nonzero()[0]
doc.zp_candi_coref = zp_candi_coref
doc.zp_candi_distance_dict = zp_candi_distance_dict
doc.gold_azp = gold_azp
doc.gold_np = gold_np
doc.gold_ante = gold_ante
doc.train_ante = train_ante
doc.train_azp = train_azp
save_f = file(data_path + "docs", 'wb')
cPickle.dump(docs, save_f, protocol=cPickle.HIGHEST_PROTOCOL)
save_f.close()
data_y = np.asarray(data_y).astype(np.float32)
np.save('./Transformed_dataset/data_x_images.npy', data_x_images)
np.save('./Transformed_dataset/data_x_features.npy', data_x_features)
np.save('./Transformed_dataset/data_y.npy', data_y)
data_y = np.load('./Transformed_dataset/data_y.npy')
num_classes = 100
vx_bins = []
vy_bins = []
theta_bins = []
for obj in range(4):
bin_vx = get_bin(data_y[:, obj, 0], num_classes)
bin_vy = get_bin(data_y[:, obj, 1], num_classes)
theta = get_bin(data_y[:, obj, 2], num_classes)
vx_bins.append(bin_vx)
vy_bins.append(bin_vy)
theta_bins.append(theta)
# ONE-HOT ENCODING DATA_Y
one_hot_encoded_data_y = []
for frame in range(data_y.shape[0]):
target = []
for obj in range(data_y.shape[1]):
one_hot_vx = convert_to_one_hot_encoded_bin(data_y[frame, obj, 0], vx_bins[obj], num_classes)
def generate_vec(data_path, docs):
read_f = file("./data/emb", "rb")
embedding, words, _ = cPickle.load(read_f)
read_f.close()
#np_len = [5,6,7,8,9,10]
np_len = [3, 4, 5]
for doc in docs:
coref_nps = []
for ZP in doc.all_azps:
for np in ZP.antecedent:
coref_nps.append(np)
doc.coref_nps = coref_nps
for width in np_len:
nps = 0
hit_np = 0
hit_ante = 0
hit_azp = 0
for doc in docs:
# generate vector for this doc
vectorized_sentences = []
for i in range(doc.sentence_num):
nodes = doc.filter_nodes[i]
vectorize_words = list_vectorize(nodes, words)
vectorized_sentences.append(vectorize_words)
vec, mask = mask_array(vectorized_sentences)
doc.vec = vec
doc.mask = mask
inpt_len = numpy.sum(doc.mask, 1)
doc.sentence_len = inpt_len
pre_index, post_index, zp2sen_dict, zp2real = opt.generate_ZP_index(
inpt_len)
doc.zp_pre_index = pre_index
doc.zp_post_index = post_index
np_index_start, np_index_end, np_indecs, np_mask, np_sen2index_dict, np2real = opt.generate_NP_index(
inpt_len, MAX=width)
nps += len(np_indecs)
#doc.np_dict[(this_np_real_sentence_num,this_np_real_start,this_np_real_end)]
np_d = {}
gold_np = [0] * len(np2real)
for i, (this_np_real_sentence_num, this_np_real_start,
this_np_real_end) in enumerate(np2real):
if (this_np_real_sentence_num, this_np_real_start,
this_np_real_end) in doc.np_dict:
gold_np[i] = 1
hit_np += 1
this_np = doc.np_dict[(this_np_real_sentence_num,
this_np_real_start,
this_np_real_end)]
if this_np in doc.coref_nps:
hit_ante += 1
np_d[this_np] = (this_np_real_sentence_num,
this_np_real_start, this_np_real_end)
for ZP in doc.all_azps:
for np in ZP.antecedent:
if np in doc.coref_nps:
if np in np_d:
this_np_real_sentence_num, this_np_real_start, this_np_real_end = np_d[
np]
if (this_np_real_sentence_num >= (ZP.sen_id - 2)
and this_np_real_sentence_num < ZP.sen_id
) or (this_np_real_sentence_num == ZP.sen_id
and this_np_real_end <= ZP.index):
hit_azp += 1
break
continue
print width, "All", nps, "NPs", hit_np, "Antes", hit_ante, float(
hit_np) / float(nps), float(hit_ante) / float(nps), hit_azp
return nps
if 1:
doc.np_index_start = np_index_start
doc.np_index_end = np_index_end
doc.np_indecs = np_indecs
doc.np_mask = np_mask
doc.zp2real = zp2real # zp_index: real_sentence_num,real_index
doc.np2real = np2real # np_index: real_sentence_num,real_start_index,real_end_index
zp2candi_dict = {} #zp_index: [np_index]
for i in range(len(pre_index)):
zp_index = pre_index[i]
sen_index = zp2sen_dict[i]
zp2candi_dict[i] = []
for sen_id in range(max(0, sen_index - 2), sen_index + 1):
np_indexs = np_sen2index_dict[sen_id]
for np_index in np_indexs:
if not ((sen_id == sen_index) and
(np_index_end[np_index] > zp_index)):
zp2candi_dict[i].append(np_index)
doc.zp2candi_dict = zp2candi_dict
zp_candi_distance_dict = {}
zp_candi_coref = {
} # zp_index: [0,1,0,0,0] coreference result for each of its candidate
gold_azp = []
gold_np = [0] * len(np2real)
for zp_index in doc.zp2candi_dict:
gold_azp_add = 0
if len(doc.zp2candi_dict[zp_index]) > 0:
this_zp_real_sentence_num, this_zp_real_index = doc.zp2real[
zp_index]
this_zp = None
if (this_zp_real_sentence_num,
this_zp_real_index) in doc.zp_dict:
this_zp = doc.zp_dict[(this_zp_real_sentence_num,
this_zp_real_index)]
np_indexes = doc.zp2candi_dict[zp_index]
max_index = max(np_indexes)
#zp_candi_coref[zp_index] = numpy.array([0]*(max(np_indexes)+2)) #last index = 1 means zp is not azp
zp_candi_coref[zp_index] = numpy.array(
[0] *
(len(np_indexes) + 1)) #last index = 1 means zp is not azp
zp_candi_distance_dict[zp_index] = [] #utils.get_bin(distance)
for ii, np_index in enumerate(np_indexes):
distance = max_index - np_index
zp_candi_distance_dict[zp_index].append(
utils.get_bin(distance))
this_np_real_sentence_num, this_np_real_start, this_np_real_end = doc.np2real[
np_index]
if (this_np_real_sentence_num, this_np_real_start,
this_np_real_end) in doc.np_dict:
this_candi = doc.np_dict[(this_np_real_sentence_num,
this_np_real_start,
this_np_real_end)]
if this_zp:
if this_candi in this_zp.antecedent:
zp_candi_coref[zp_index][ii] = 1
gold_np[np_index] = 1
gold_azp_add = 1
if sum(zp_candi_coref[zp_index]) == 0:
zp_candi_coref[zp_index][-1] = 1
gold_azp_add = 0
gold_azp.append(gold_azp_add)
doc.zp_candi_coref = zp_candi_coref
doc.zp_candi_distance_dict = zp_candi_distance_dict
doc.gold_azp = gold_azp
doc.gold_np = gold_np
save_f = file(data_path + "docs", 'wb')
cPickle.dump(docs, save_f, protocol=cPickle.HIGHEST_PROTOCOL)
save_f.close()