def pre_training_loss(truth, pred):
feat,pred = split_feat_pred(pred)
d = create_index_dict(truth, pred)
feat = create_feature_dict(feat)
truthxy = tf.concat([d['truthHitAssignedX'],d['truthHitAssignedY']],axis=-1)
#print('>>>>> >>> > > > > >', truthxy.shape, d['predCCoords'].shape, d['predXY'].shape, )
diff = 10.*d['predCCoords'] - truthxy
diff = d['truthNoNoise']*(0.1*d['predBeta'])*diff**2
beta_med = (d['predBeta']-0.5)**2
posl = (d['predXY'] - truthxy)**2 / 1000.
loss = tf.reduce_mean(diff) + tf.reduce_mean(beta_med) + tf.reduce_mean(posl)
tf.print('pretrain loss',loss, 'posl**2', tf.reduce_mean(posl))
return loss
def full_obj_cond_loss(truth, pred_in, rowsplits):
global full_obj_cond_loss_counter
full_obj_cond_loss_counter+=1
start_time = time.time()
if truth.shape[0] is None:
return tf.constant(0., tf.float32)
rowsplits = tf.cast(rowsplits, tf.int64)#just for first loss evaluation from stupid keras
feat,pred = split_feat_pred(pred_in)
d = create_index_dict(truth, pred, n_ccoords=config.n_ccoords)
feat = create_feature_dict(feat)
#print('feat',feat.shape)
#d['predBeta'] = tf.clip_by_value(d['predBeta'],1e-6,1.-1e-6)
row_splits = rowsplits[ : rowsplits[-1,0],0]
classes = d['truthHitAssignementIdx'][...,0]
energyweights = d['truthHitAssignedEnergies']
energyweights = tf.math.log(0.1 * energyweights + 1.)
if not config.use_energy_weights:
energyweights *= 0.
energyweights += 1.
#just to mitigate the biased sample
energyweights = tf.where(d['truthHitAssignedEnergies']>10.,energyweights+0.1, energyweights*(d['truthHitAssignedEnergies']/10.+0.1))
if not config.downweight_low_energy:
energyweights = tf.zeros_like(energyweights) + 1.
if config.use_energy_weights:
energyweights = tf.math.log(0.1 * d['truthHitAssignedEnergies'] + 1.)
#also using log now, scale back in evaluation #
den_offset = config.energy_den_offset
if config.log_energy:
raise ValueError("loss config log_energy is not supported anymore. Please use the 'ExpMinusOne' layer within the model instead to scale the output.")
energy_diff = (d['predEnergy'] - d['truthHitAssignedEnergies'])
scaled_true_energy = d['truthHitAssignedEnergies']
if config.rel_energy_mse:
scaled_true_energy *= scaled_true_energy
sqrt_true_en = tf.sqrt(scaled_true_energy + 1e-6)
energy_loss = energy_diff/(sqrt_true_en+den_offset)
if config.huber_energy_scale>0:
huber_scale = config.huber_energy_scale * sqrt_true_en
energy_loss = huber(energy_loss, huber_scale)
else:
energy_loss = energy_loss**2
pos_offs = None
payload_loss = None
xdiff = d['predX']+feat['recHitX'] - d['truthHitAssignedX']
ydiff = d['predY']+feat['recHitY'] - d['truthHitAssignedY']
pos_offs = tf.reduce_sum(tf.concat( [xdiff, ydiff],axis=-1)**2, axis=-1, keepdims=True)
tdiff = d['predT'] - d['truthHitAssignedT']
#print("d['truthHitAssignedT']", tf.reduce_mean(d['truthHitAssignedT']), tdiff)
tdiff = (1e6 * tdiff)**2
# self.timing_loss_weight
payload_loss = tf.concat([config.energy_loss_weight * energy_loss ,
config.position_loss_weight * pos_offs,
config.timing_loss_weight * tdiff], axis=-1)
if not config.use_spectators:
d['truthIsSpectator'] = tf.zeros_like(d['truthIsSpectator'])
attractive_loss, rep_loss, noise_loss, min_beta_loss, payload_loss_full, too_much_beta_loss = oc_loss(
x = d['predCCoords'],
beta = d['predBeta'],
truth_indices = d['truthHitAssignementIdx'],
row_splits=row_splits,
is_spectator = d['truthIsSpectator'],
payload_loss=payload_loss,
Q_MIN=config.q_min,
S_B=config.s_b,
energyweights=energyweights,
use_average_cc_pos=config.use_average_cc_pos,
payload_rel_threshold=config.payload_rel_threshold,
cont_beta_loss=config.cont_beta_loss
)
attractive_loss *= config.potential_scaling
rep_loss *= config.potential_scaling * config.repulsion_scaling
min_beta_loss *= config.beta_loss_scale
noise_loss *= config.noise_scaler
too_much_beta_loss *= config.too_much_beta_scale
spectator_beta_penalty = tf.constant(0.)
if config.use_spectators:
spectator_beta_penalty = 0.1 * spectator_penalty(d,row_splits)
spectator_beta_penalty = tf.where(tf.math.is_nan(spectator_beta_penalty),0,spectator_beta_penalty)
#attractive_loss = tf.where(tf.math.is_nan(attractive_loss),0,attractive_loss)
#rep_loss = tf.where(tf.math.is_nan(rep_loss),0,rep_loss)
#min_beta_loss = tf.where(tf.math.is_nan(min_beta_loss),0,min_beta_loss)
#noise_loss = tf.where(tf.math.is_nan(noise_loss),0,noise_loss)
#payload_loss_full = tf.where(tf.math.is_nan(payload_loss_full),0,payload_loss_full)
energy_loss = payload_loss_full[0]
pos_loss = payload_loss_full[1]
time_loss = payload_loss_full[2]
#energy_loss *= 0.0000001
# neglect energy loss almost fully
loss = attractive_loss + rep_loss + min_beta_loss + noise_loss + energy_loss + time_loss + pos_loss + spectator_beta_penalty + too_much_beta_loss
loss = tf.debugging.check_numerics(loss,"loss has nan")
if config.pre_train:
preloss = pre_training_loss(truth,pred_in)
loss /= 10.
loss += preloss
print('call ',full_obj_cond_loss_counter)
print('loss',loss.numpy(),
'attractive_loss',attractive_loss.numpy(),
'rep_loss', rep_loss.numpy(),
'min_beta_loss', min_beta_loss.numpy(),
'noise_loss' , noise_loss.numpy(),
'energy_loss', energy_loss.numpy(),
'pos_loss', pos_loss.numpy(),
'time_loss', time_loss.numpy(),
'spectator_beta_penalty', spectator_beta_penalty.numpy(),
'too_much_beta_loss', too_much_beta_loss.numpy())
print('time for this loss eval',int((time.time()-start_time)*1000),'ms')
global g_time
print('time for total batch',int((time.time()-g_time)*1000),'ms')
g_time=time.time()
return loss
pred = predicted[0]
feat = td.transferFeatureListToNumpy()
rs = feat[1]
feat = feat[0]
#weights = td.transferWeightListToNumpy()
truth = td.transferTruthListToNumpy()[0]
td.clear()
print(feat.shape)
print(truth.shape)
fig = plt.figure(figsize=(10, 4))
ax = [fig.add_subplot(1, 2, 1, projection='3d'), fig.add_subplot(1, 2, 2)]
data = create_index_dict(truth, pred, usetf=False)
feats = create_feature_dict(feat)
make_cluster_coordinates_plot(
plt,
ax[1],
data['truthHitAssignementIdx'], #[ V ]
data['predBeta'], #[ V ]
data['predCCoords'])
make_original_truth_shower_plot(plt, ax[0], data['truthHitAssignementIdx'],
feats['recHitEnergy'], feats['recHitX'],
feats['recHitY'], feats['recHitZ'])
plt.tight_layout()
fig.savefig("event_" + args.e + ".pdf")
plt.close()
def _make_plot(self, counter, feat, predicted, truth):
# make sure it gets reloaded in the fork
# doesn't really seem to help though
# from importlib import reload
# global matplotlib
# matplotlib=reload(matplotlib)
# matplotlib.use('Agg')
# import matplotlib.pyplot as plt
# import matplotlib.gridspec as gridspec
# exception handling is weird for keras fit right now... explicitely print exceptions at the end
try:
pred = predicted[0]
feat = feat[0] # remove row splits
truth = truth[0] # remove row splits, anyway one event
_, pred = split_feat_pred(pred)
data = create_index_dict(truth,
pred,
usetf=False,
n_ccoords=self.n_ccoords)
feats = create_feature_dict(feat)
fig = plt.figure(figsize=(10, 8))
ax = None
if self.n_ccoords == 2:
ax = [
fig.add_subplot(self.gs[0, 0], projection='3d'),
fig.add_subplot(self.gs[0, 1]),
fig.add_subplot(self.gs[1, :])
]
elif self.n_ccoords == 3:
ax = [
fig.add_subplot(self.gs[0, 0], projection='3d'),
fig.add_subplot(self.gs[0, 1], projection='3d'),
fig.add_subplot(self.gs[1, :])
]
data['predBeta'] = np.clip(data['predBeta'], 1e-6, 1. - 1e-6)
seed = truth.shape[0]
if self.cycle_colors:
seed += counter
cmap = createRandomizedColors('jet', seed=seed)
identified = make_cluster_coordinates_plot(
plt,
ax[1],
data['truthHitAssignementIdx'], # [ V x 1] or [ V ]
data['predBeta'], # [ V x 1] or [ V ]
data['predCCoords'],
beta_threshold=0.5,
distance_threshold=0.75,
beta_plot_threshold=0.01,
cmap=cmap)
make_original_truth_shower_plot(plt,
ax[0],
data['truthHitAssignementIdx'],
feats['recHitEnergy'],
feats['recHitX'],
feats['recHitY'],
feats['recHitZ'],
cmap=cmap,
predBeta=data['predBeta'])
angle_in = 10. * counter + 60.
while angle_in >= 360:
angle_in -= 360
while angle_in <= -360:
angle_in -= 360
ax[0].view_init(30, angle_in)
if self.n_ccoords == 3:
ax[1].view_init(30, angle_in)
predEnergy = data['predEnergy']
if self.log_energy:
predEnergy = np.exp(predEnergy) - 1
def calc_eta(x, y, z):
rsq = np.sqrt(x**2 + y**2)
return -1 * np.sign(z) * np.log(rsq / np.abs(z + 1e-3) / 2.)
def calc_phi(x, y):
return np.arctan2(x, y)
predEta = calc_eta(data['predX'] + feats['recHitX'],
data['predY'] + feats['recHitY'],
np.sign(feats['recHitZ']) * 322.1)
predPhi = calc_phi(data['predX'] + feats['recHitX'],
data['predY'] + feats['recHitY'])
trueEta = calc_eta(data['truthHitAssignedX'],
data['truthHitAssignedY'] + 1e-3,
data['truthHitAssignedZ'] + 1e-3)
truePhi = calc_phi(data['truthHitAssignedX'],
data['truthHitAssignedY'] + 1e-3)
make_eta_phi_projection_truth_plot(
plt,
ax[2],
data['truthHitAssignementIdx'],
feats['recHitEnergy'],
calc_eta(feats['recHitX'], feats['recHitY'], feats['recHitZ']),
calc_phi(feats['recHitX'], feats['recHitY']),
predEta, # data['predEta']+feats['recHitEta'],
predPhi, # data['predPhi']+feats['recHitRelPhi'],
trueEta,
truePhi,
data['truthHitAssignedEnergies'],
data['predBeta'],
data['predCCoords'],
cmap=cmap,
identified=identified,
predEnergy=predEnergy)
plt.tight_layout()
fig.savefig(self.outputfile + str(self.keep_counter) + ".pdf")
if self.publish is not None:
temp_name = next(tempfile._get_candidate_names())
temp_name = self.outputfile + temp_name + '.png'
fig.savefig(temp_name)
cpstring = 'cp -f '
if "@" in self.publish:
cpstring = 'scp '
os.system(cpstring + temp_name + ' ' + self.publish +
'.png > /dev/null')
os.system('rm -f ' + temp_name)
fig.clear()
plt.close(fig)
plt.clf()
plt.cla()
plt.close()
except Exception as e:
print(e)
raise e
def _make_plot(self, counter, feat, predicted, truth):
#exception handling is weird for keras fit right now... explicitely print exceptions at the end
try:
pred = predicted[0]
feat = feat[0] #remove row splits
truth = truth[0] #remove row splits, anyway one event
fig = plt.figure(figsize=(10, 8))
ax = [
fig.add_subplot(self.gs[0, 0], projection='3d'),
fig.add_subplot(self.gs[0, 1], projection='3d'),
fig.add_subplot(self.gs[1, 0], projection='3d'),
fig.add_subplot(self.gs[1, 1], projection='3d')
]
_, pred = split_feat_pred(pred)
data = create_index_dict(truth, pred, usetf=False)
feats = create_feature_dict(feat)
seed = truth.shape[0]
if self.cycle_colors:
seed += counter
cmap = createRandomizedColors('jet', seed=seed)
make_original_truth_shower_plot(plt,
ax[0],
data['truthHitAssignementIdx'],
feats['recHitEnergy'],
feats['recHitX'],
feats['recHitY'],
feats['recHitZ'],
cmap=cmap)
angle_in = counter + 60.
while angle_in >= 360:
angle_in -= 360
while angle_in <= -360:
angle_in -= 360
ax[0].view_init(30, angle_in)
make_original_truth_shower_plot(plt,
ax[1],
data['truthHitAssignementIdx'],
feats['recHitEnergy'],
pred[:, self.coordinates_a[0]],
pred[:, self.coordinates_a[1]],
pred[:, self.coordinates_a[2]],
cmap=cmap)
make_original_truth_shower_plot(plt,
ax[2],
data['truthHitAssignementIdx'],
feats['recHitEnergy'],
pred[:, self.coordinates_b[0]],
pred[:, self.coordinates_b[1]],
pred[:, self.coordinates_b[2]],
cmap=cmap)
make_original_truth_shower_plot(plt,
ax[3],
data['truthHitAssignementIdx'],
feats['recHitEnergy'],
pred[:, self.coordinates_c[0]],
pred[:, self.coordinates_c[1]],
pred[:, self.coordinates_c[2]],
cmap=cmap)
plt.tight_layout()
fig.savefig(self.outputfile + str(self.keep_counter) + ".pdf")
fig.clear()
plt.close(fig)
plt.clf()
plt.cla()
plt.close()
except Exception as e:
print(e)
raise e
gen.feedNumpyData()) # this is [ [features],[truth],[None] ]
if gen.lastBatch():
break
row_splits = feat[1][:, 0]
with tf.device('/CPU:0'):
feat, _ = ragged_constructor((feat[0], row_splits))
truth, row_splits = ragged_constructor((truth[0], row_splits))
for i in range(len(row_splits) - 1):
with tf.device('/CPU:0'):
features_s = feat[row_splits[i]:row_splits[i + 1]].numpy()
truth_s = truth[row_splits[i]:row_splits[i + 1]].numpy()
feature_dict = index_dicts.create_feature_dict(
features_s) # follow this
truth_dict = index_dicts.create_truth_dict(
truth_s) # and this function to know all the keys
print(feature_dict['recHitEnergy'].shape) # rechit energies
print(feature_dict['recHitEta'].shape) # rechit eta
print(feature_dict['recHitRelPhi'].shape) # rechit phi
print(feature_dict['recHitX'].shape) # rechit X
print(feature_dict['recHitY'].shape) # rechit Y
print(feature_dict['recHitZ'].shape) # rechit Z
print(feature_dict['recHitTime'].shape) # rechit Z
print(truth_dict['truthHitAssignementIdx'].shape) # shower id
print(truth_dict['truthRealEnergy'].shape) # energy of the shower
print("Going to next window\n\n")