def train(self):
print('\n:: training started\n')
epochs = self.config['epochs']
best_dev_accuracy = 0.0
epochs_without_improvement = 0
random_input_prob = self.config.get('random_input_prob', 0.0)
unk_action_id = self.action_templates.index(UNK)
for j in range(epochs):
batch_gen = batch_generator(self.data_train, self.batch_size)
for batch in batch_gen:
batch_copy = [np.copy(elem) for elem in batch]
enc_inp, dec_inp, dec_out, bow_out, context_features, action_masks, y = batch_copy
num_turns = np.sum(np.vectorize(lambda x: x != 0)(y))
for idx in range(num_turns):
if np.random.random() < random_input_prob:
random_input_idx = np.random.choice(
range(self.random_input[0].shape[0]))
random_input = [
random_input_i[random_input_idx]
for random_input_i in self.random_input
]
enc_inp[0][idx], dec_inp[0][idx], dec_out[0][
idx], bow_out[0][idx] = random_input
y[0][idx] = unk_action_id
batch_loss_dict, lr = self.net.train_step(
enc_inp, dec_inp, dec_out, bow_out, context_features,
action_masks, y)
# evaluate every epoch
train_accuracy, train_mean_losses = evaluate(self.net,
self.data_train,
runs_number=1)
train_loss_report = ' '.join([
'{}: {:.3f}'.format(key, value)
for key, value in train_mean_losses.items()
])
dev_accuracy, dev_mean_losses = evaluate(self.net,
self.data_dev,
runs_number=1)
dev_loss_report = ' '.join([
'{}: {:.3f}'.format(key, value)
for key, value in dev_mean_losses.items()
])
print(
':: {}@lr={:.5f} || trn accuracy {:.3f} {} || dev accuracy {:.3f} {}'
.format(j + 1, lr, train_accuracy, train_loss_report,
dev_accuracy, dev_loss_report))
if best_dev_accuracy < dev_accuracy:
print('New best dev accuracy. Saving checkpoint')
self.net.save(self.model_folder)
best_dev_accuracy = dev_accuracy
epochs_without_improvement = 0
else:
epochs_without_improvement += 1
if self.config[
'early_stopping_threshold'] < epochs_without_improvement:
print(
'Finished after {} epochs due to early stopping'.format(j))
break
def train(self):
print('\n:: training started\n')
epochs = self.config['epochs']
best_dev_accuracy = 0.0
epochs_without_improvement = 0
for j in range(epochs):
losses = []
batch_gen = batch_generator(self.data_train, self.batch_size)
for batch in batch_gen: # batch_x, batch_context_features, batch_action_masks, batch_y in batch_gen:
batch_copy = [np.copy(elem) for elem in batch]
dropped_out_batch = self.drop_out_batch(batch_copy)
batch_loss_dict, lr = self.net.train_step(*dropped_out_batch)
# evaluate every epoch
train_accuracy, train_loss_dict = evaluate(self.net, self.data_train)
train_loss_report = ' '.join(['{}: {:.3f}'.format(key, value) for key, value in train_loss_dict.items()])
dev_accuracy, dev_loss_dict = evaluate(self.net, self.data_dev, runs_number=3)
dev_loss_report = ' '.join(['{}: {:.3f}'.format(key, value) for key, value in dev_loss_dict.items()])
print(':: {}@lr={:.5f} || trn accuracy {:.3f} {} || dev accuracy {:.3f} {}'.format(j + 1, lr, train_accuracy, train_loss_report, dev_accuracy, dev_loss_report))
eval_stats_noisy = evaluate_advanced(self.net,
self.data_test,
self.action_templates,
BABI_CONFIG['backoff_utterance'].lower(),
post_ood_turns=self.post_ood_turns_noisy,
runs_number=1)
print('\n\n')
print('Noisy dataset: {} turns overall, {} turns after the first OOD'.format(eval_stats_noisy['total_turns'],
eval_stats_noisy['total_turns_after_ood']))
print('Accuracy:')
accuracy = eval_stats_noisy['correct_turns'] / eval_stats_noisy['total_turns']
accuracy_after_ood = eval_stats_noisy['correct_turns_after_ood'] / eval_stats_noisy['total_turns_after_ood'] \
if eval_stats_noisy['total_turns_after_ood'] != 0 \
else 0
accuracy_post_ood = eval_stats_noisy['correct_post_ood_turns'] / eval_stats_noisy['total_post_ood_turns'] \
if eval_stats_noisy['total_post_ood_turns'] != 0 \
else 0
accuracy_ood = eval_stats_noisy['correct_ood_turns'] / eval_stats_noisy['total_ood_turns'] \
if eval_stats_noisy['total_ood_turns'] != 0 \
else 0
print('overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}; OOD: {:.3f}'.format(accuracy,
accuracy_after_ood,
accuracy_post_ood,
accuracy_ood))
if best_dev_accuracy < dev_accuracy:
print('New best dev accuracy. Saving checkpoint')
self.net.save(self.model_folder)
best_dev_accuracy = dev_accuracy
epochs_without_improvement = 0
else:
epochs_without_improvement += 1
if self.config['early_stopping_threshold'] < epochs_without_improvement:
print('Finished after {} epochs due to early stopping'.format(j))
break
default=False)
args = argpar.parse_args()
t = posTagger()
if os.stat(args.in_file).st_size == 0:
print "Input file is empty"
else:
if args.train:
print "Running in training mode\n"
if not args.top_x_form:
print args.top_x_form
top_x = [args.top_x_form, args.top_x_word_len, args.top_x_position, args.top_x_prefix, args.top_x_suffix,
args.top_x_lettercombs]
t.train(args.in_file, args.model, int(args.epochs), top_x, args.decrease_alpha, args.shuffle_sentences,
args.batch_training)
elif args.test:
print "Running in test mode\n"
t.test(args.in_file, args.model, args.output_file)
elif args.evaluate:
print "Running in evaluation mode\n"
out_stream = open(args.output_file, 'w')
evaluate(args.in_file, out_stream)
out_stream.close()
elif args.tag:
print "Running in tag mode\n"
t.tag(args.in_file, args.model, args.output_file)
t1 = time.time()
print "\n\tDone. Total time: " + str(t1 - t0) + "sec.\n"
def cosipy_core(DATA, indY, indX, GRID_RESTART=None, stake_names=None, stake_data=None):
# Local variables
_RRR = np.full(len(DATA.time), np.nan)
_RAIN = np.full(len(DATA.time), np.nan)
_SNOWFALL = np.full(len(DATA.time), np.nan)
_LWin = np.full(len(DATA.time), np.nan)
_LWout = np.full(len(DATA.time), np.nan)
_H = np.full(len(DATA.time), np.nan)
_LE = np.full(len(DATA.time), np.nan)
_B = np.full(len(DATA.time), np.nan)
_MB = np.full(len(DATA.time), np.nan)
_surfMB = np.full(len(DATA.time), np.nan)
_MB = np.full(len(DATA.time), np.nan)
_Q = np.full(len(DATA.time), np.nan)
_SNOWHEIGHT = np.full(len(DATA.time), np.nan)
_TOTALHEIGHT = np.full(len(DATA.time), np.nan)
_TS = np.full(len(DATA.time), np.nan)
_ALBEDO = np.full(len(DATA.time), np.nan)
_ME = np.full(len(DATA.time), np.nan)
_intMB = np.full(len(DATA.time), np.nan)
_EVAPORATION = np.full(len(DATA.time), np.nan)
_SUBLIMATION = np.full(len(DATA.time), np.nan)
_CONDENSATION = np.full(len(DATA.time), np.nan)
_DEPOSITION = np.full(len(DATA.time), np.nan)
_REFREEZE = np.full(len(DATA.time), np.nan)
_NLAYERS = np.full(len(DATA.time), np.nan)
_subM = np.full(len(DATA.time), np.nan)
_Z0 = np.full(len(DATA.time), np.nan)
_surfM = np.full(len(DATA.time), np.nan)
_LAYER_HEIGHT = np.full((len(DATA.time),max_layers), np.nan)
_LAYER_RHO = np.full((len(DATA.time),max_layers), np.nan)
_LAYER_T = np.full((len(DATA.time),max_layers), np.nan)
_LAYER_LWC = np.full((len(DATA.time),max_layers), np.nan)
_LAYER_CC = np.full((len(DATA.time),max_layers), np.nan)
_LAYER_POROSITY = np.full((len(DATA.time),max_layers), np.nan)
_LAYER_ICE_FRACTION = np.full((len(DATA.time),max_layers), np.nan)
_LAYER_IRREDUCIBLE_WATER = np.full((len(DATA.time),max_layers), np.nan)
_LAYER_REFREEZE = np.full((len(DATA.time),max_layers), np.nan)
# Start logging
logger = logging.getLogger(__name__)
#--------------------------------------------
# Initialize snowpack or load restart grid
#--------------------------------------------
if GRID_RESTART is None:
GRID = init_snowpack(DATA)
else:
GRID = load_snowpack(GRID_RESTART)
# Create the local output datasets
logger.debug('Create local datasets')
IO = IOClass(DATA)
RESTART = IO.create_local_restart_dataset()
# Merge grid layers, if necessary
logger.debug('Create local datasets')
# hours since the last snowfall (albedo module)
hours_since_snowfall = 0
#--------------------------------------------
# Get data from file
#--------------------------------------------
T2 = DATA.T2.values
RH2 = DATA.RH2.values
PRES = DATA.PRES.values
G = DATA.G.values
U2 = DATA.U2.values
#--------------------------------------------
# Checks for optional input variables
#--------------------------------------------
if ('SNOWFALL' in DATA) and ('RRR' in DATA):
SNOWF = DATA.SNOWFALL.values
RRR = DATA.RRR.values
elif ('SNOWFALL' in DATA):
SNOWF = DATA.SNOWFALL.values
RRR = None
RAIN = None
else:
SNOWF = None
RRR = DATA.RRR.values
if force_use_TP is True:
SNOWF = None
# Check whether longwave data is availible -> used for surface temperature calculations
if ('LWin' in DATA) and ('N' in DATA):
LWin = DATA.LWin.values
N = DATA.N.values
elif ('LWin' in DATA):
LWin = DATA.LWin.values
else:
LWin = None
N = DATA.N.values
if ('SLOPE' in DATA):
SLOPE = DATA.SLOPE.values
else:
SLOPE = 0.0
# Cumulative mass balance variable
MB_cum = 0
# Create pandas dataframe for stake evaluation
_df = pd.DataFrame(index=stake_data.index, columns=['mb'], dtype='float')
# Profiling with bokeh
cp = cProfile.Profile()
#--------------------------------------------
# TIME LOOP
#--------------------------------------------
logger.debug('Start time loop')
for t in np.arange(len(DATA.time)):
# Check grid
GRID.grid_check()
# get seconds since start
timestamp = dt*t
# Calc fresh snow density
density_fresh_snow = np.maximum(109.0+6.0*(T2[t]-273.16)+26.0*np.sqrt(U2[t]), 50.0)
if (SNOWF is not None):
SNOWFALL = SNOWF[t]
else:
# , else convert rainfall [mm] to snowheight [m]
# liquid/solid fraction
SNOWFALL = (RRR[t]/1000.0)*(ice_density/density_fresh_snow)*(0.5*(-np.tanh(((T2[t]-zero_temperature) / center_snow_transfer_function) * spread_snow_transfer_function) + 1.0))
RAIN = RRR[t]-SNOWFALL*(density_fresh_snow/ice_density) * 1000
# if snowfall is smaller than the threshold
if SNOWFALL<minimum_snow_layer_height:
SNOWFALL = 0.0
# if rainfall is smaller than the threshold
if RAIN<(minimum_snow_layer_height*(density_fresh_snow/ice_density)*1000):
RAIN = 0.0
if SNOWFALL > 0.0:
# Add a new snow node on top
GRID.add_fresh_snow(SNOWFALL, density_fresh_snow, np.minimum(float(T2[t]),zero_temperature), 0.0, timestamp)
# Guarantee that solar radiation is greater equal zero
if (G[t]<0.0):
G[t] = 0.0
#--------------------------------------------
# Merge grid layers, if necessary
#--------------------------------------------
GRID.update_grid()
#--------------------------------------------
# Calculate albedo and roughness length changes if first layer is snow
#--------------------------------------------
alpha = updateAlbedo(GRID, timestamp)
#--------------------------------------------
# Update roughness length
#--------------------------------------------
z0 = updateRoughness(GRID, timestamp)
#--------------------------------------------
# Surface Energy Balance
#--------------------------------------------
# Calculate net shortwave radiation
SWnet = G[t] * (1 - alpha)
# Penetrating SW radiation and subsurface melt
if SWnet > 0.0:
subsurface_melt, G_penetrating = penetrating_radiation(GRID, SWnet, dt)
else:
subsurface_melt = 0.0
G_penetrating = 0.0
# Calculate residual incoming shortwave radiation (penetrating part removed)
G_resid = G[t] - G_penetrating
if LWin is not None:
# Find new surface temperature (LW is used from the input file)
fun, surface_temperature, lw_radiation_in, lw_radiation_out, sensible_heat_flux, latent_heat_flux, \
ground_heat_flux, sw_radiation_net, rho, Lv, Cs_t, Cs_q, q0, q2, qdiff, phi \
= update_surface_temperature(GRID, alpha, z0, T2[t], RH2[t], PRES[t], G_resid, U2[t], SLOPE, LWin=LWin[t])
else:
# Find new surface temperature (LW is parametrized using cloud fraction)
fun, surface_temperature, lw_radiation_in, lw_radiation_out, sensible_heat_flux, latent_heat_flux, \
ground_heat_flux, sw_radiation_net, rho, Lv, Cs_t, Cs_q, q0, q2, qdiff, phi \
= update_surface_temperature(GRID, alpha, z0, T2[t], RH2[t], PRES[t], G_resid, U2[t], SLOPE, N=N[t])
#--------------------------------------------
# Surface mass fluxes [m w.e.q.]
#--------------------------------------------
if surface_temperature < zero_temperature:
sublimation = min(latent_heat_flux / (1000.0 * lat_heat_sublimation), 0) * dt
deposition = max(latent_heat_flux / (1000.0 * lat_heat_sublimation), 0) * dt
evaporation = 0
condensation = 0
else:
sublimation = 0
deposition = 0
evaporation = min(latent_heat_flux / (1000.0 * lat_heat_vaporize), 0) * dt
condensation = max(latent_heat_flux / (1000.0 * lat_heat_vaporize), 0) * dt
#--------------------------------------------
# Melt process - mass changes of snowpack (melting, sublimation, deposition, evaporation, condensation)
#--------------------------------------------
# Melt energy in [W m^-2 or J s^-1 m^-2]
melt_energy = max(0, sw_radiation_net + lw_radiation_in + lw_radiation_out + ground_heat_flux +
sensible_heat_flux + latent_heat_flux)
# Convert melt energy to m w.e.q.
melt = melt_energy * dt / (1000 * lat_heat_melting)
# Remove melt [m w.e.q.]
GRID.remove_melt_weq(melt - sublimation - deposition - evaporation)
#--------------------------------------------
# Percolation
#--------------------------------------------
Q = percolation(GRID, melt - condensation, dt)
#--------------------------------------------
# Refreezing
#--------------------------------------------
water_refreezed = refreezing(GRID)
#--------------------------------------------
# Solve the heat equation
#--------------------------------------------
solveHeatEquation(GRID, dt)
#--------------------------------------------
# Calculate new density to densification
#--------------------------------------------
densification(GRID,SLOPE)
#--------------------------------------------
# Calculate mass balance
#--------------------------------------------
surface_mass_balance = SNOWFALL * (density_fresh_snow / ice_density) - melt - sublimation - deposition - evaporation
internal_mass_balance = water_refreezed - subsurface_melt
mass_balance = surface_mass_balance + internal_mass_balance
internal_mass_balance2 = melt-Q #+ subsurface_melt
mass_balance_check = surface_mass_balance + internal_mass_balance2
#GRID.grid_check()
# Write results
logger.debug('Write data into local result structure')
# TOBI
# Cumulative mass balance for stake evaluation
MB_cum = MB_cum + mass_balance
# Store cumulative MB in pandas frame for validation
if stake_names:
if (DATA.isel(time=t).time.values in stake_data.index):
_df['mb'].loc[DATA.isel(time=t).time.values] = MB_cum
# Save results
_RAIN[t] = RAIN
_SNOWFALL[t] = SNOWFALL
_LWin[t] = lw_radiation_in
_LWout[t] = lw_radiation_out
_H[t] = sensible_heat_flux
_LE[t] = latent_heat_flux
_B[t] = ground_heat_flux
_MB[t] = mass_balance
_surfMB[t] = surface_mass_balance
_MB[t] = mass_balance
_Q[t] = Q
_SNOWHEIGHT[t] = GRID.get_total_snowheight()
_TOTALHEIGHT[t] = GRID.get_total_height()
_TS[t] = surface_temperature
_ALBEDO[t] = alpha
_NLAYERS[t] = GRID.get_number_layers()
_ME[t] = melt_energy
_intMB[t] = internal_mass_balance
_EVAPORATION[t] = evaporation
_SUBLIMATION[t] = sublimation
_CONDENSATION[t] = condensation
_DEPOSITION[t] = deposition
_REFREEZE[t] = water_refreezed
_subM[t] = subsurface_melt
_Z0[t] = z0
_surfM[t] = melt
if full_field:
if GRID.get_number_layers()>max_layers:
logger.error('Maximum number of layers reached')
else:
_LAYER_HEIGHT[t, 0:GRID.get_number_layers()] = GRID.get_height()
_LAYER_RHO[t, 0:GRID.get_number_layers()] = GRID.get_density()
_LAYER_T[t, 0:GRID.get_number_layers()] = GRID.get_temperature()
_LAYER_LWC[t, 0:GRID.get_number_layers()] = GRID.get_liquid_water_content()
_LAYER_CC[t, 0:GRID.get_number_layers()] = GRID.get_cold_content()
_LAYER_POROSITY[t, 0:GRID.get_number_layers()] = GRID.get_porosity()
_LAYER_ICE_FRACTION[t, 0:GRID.get_number_layers()] = GRID.get_ice_fraction()
_LAYER_IRREDUCIBLE_WATER[t, 0:GRID.get_number_layers()] = GRID.get_irreducible_water_content()
_LAYER_REFREEZE[t, 0:GRID.get_number_layers()] = GRID.get_refreeze()
else:
_LAYER_HEIGHT = None
_LAYER_RHO = None
_LAYER_T = None
_LAYER_LWC = None
_LAYER_CC = None
_LAYER_POROSITY = None
_LAYER_ICE_FRACTION = None
_LAYER_IRREDUCIBLE_WATER = None
_LAYER_REFREEZE = None
# Evaluate stakes
_stat = evaluate(stake_names, stake_data, _df)
# Restart
logger.debug('Write restart data into local restart structure')
RESTART['NLAYERS'] = GRID.get_number_layers()
RESTART.LAYER_HEIGHT[0:GRID.get_number_layers()] = GRID.get_height()
RESTART.LAYER_RHO[0:GRID.get_number_layers()] = GRID.get_density()
RESTART.LAYER_T[0:GRID.get_number_layers()] = GRID.get_temperature()
return (indY,indX,RESTART,_RAIN,_SNOWFALL,_LWin,_LWout,_H,_LE,_B, \
_MB,_surfMB,_Q,_SNOWHEIGHT,_TOTALHEIGHT,_TS,_ALBEDO,_NLAYERS, \
_ME,_intMB,_EVAPORATION,_SUBLIMATION,_CONDENSATION,_DEPOSITION,_REFREEZE, \
_subM,_Z0,_surfM, \
_LAYER_HEIGHT,_LAYER_RHO,_LAYER_T,_LAYER_LWC,_LAYER_CC,_LAYER_POROSITY,_LAYER_ICE_FRACTION, \
_LAYER_IRREDUCIBLE_WATER,_LAYER_REFREEZE,stake_names,_stat,_df)
def validate(args, net, val_data_loader, val_dataset, iteration_num):
"""Test a FPN network on an image database."""
iou_thresh = args.IOU_THRESH
num_samples = len(val_dataset)
logger.info('Validating at ' + str(iteration_num) +
' number of samples:: ' + str(num_samples))
print_time = True
val_step = 20
count = 0
torch.cuda.synchronize()
ts = time.perf_counter()
activation = torch.nn.Sigmoid().cuda()
ego_pds = []
ego_gts = []
det_boxes = []
gt_boxes_all = []
for nlt in range(args.num_label_types):
numc = args.num_classes_list[nlt]
det_boxes.append([[] for _ in range(numc)])
gt_boxes_all.append([])
net.eval()
with torch.no_grad():
for val_itr, (images, gt_boxes, gt_targets, ego_labels, batch_counts,
img_indexs, wh) in enumerate(val_data_loader):
torch.cuda.synchronize()
t1 = time.perf_counter()
batch_size = images.size(0)
images = images.cuda(0, non_blocking=True)
decoded_boxes, confidence, ego_preds = net(images)
ego_preds = activation(ego_preds).cpu().numpy()
ego_labels = ego_labels.numpy()
confidence = activation(confidence)
if print_time and val_itr % val_step == 0:
torch.cuda.synchronize()
tf = time.perf_counter()
logger.info('Forward Time {:0.3f}'.format(tf - t1))
seq_len = gt_targets.size(1)
for b in range(batch_size):
for s in range(seq_len):
if ego_labels[b, s] > -1:
ego_pds.append(ego_preds[b, s, :])
ego_gts.append(ego_labels[b, s])
width, height = wh[b][0], wh[b][1]
gt_boxes_batch = gt_boxes[b,
s, :batch_counts[b,
s], :].numpy()
gt_labels_batch = gt_targets[b,
s, :batch_counts[b,
s]].numpy()
decoded_boxes_frame = decoded_boxes[b, s].clone()
cc = 0
for nlt in range(args.num_label_types):
num_c = args.num_classes_list[nlt]
tgt_labels = gt_labels_batch[:, cc:cc + num_c]
# print(gt_boxes_batch.shape, tgt_labels.shape)
frame_gt = get_individual_labels(
gt_boxes_batch, tgt_labels)
gt_boxes_all[nlt].append(frame_gt)
for cl_ind in range(num_c):
scores = confidence[b, s, :, cc].clone().squeeze()
cc += 1
cls_dets = utils.filter_detections(
args, scores, decoded_boxes_frame)
det_boxes[nlt][cl_ind].append(cls_dets)
count += 1
if print_time and val_itr % val_step == 0:
torch.cuda.synchronize()
te = time.perf_counter()
logger.info(
'detections done: {:d}/{:d} time taken {:0.3f}'.format(
count, num_samples, te - ts))
torch.cuda.synchronize()
ts = time.perf_counter()
if print_time and val_itr % val_step == 0:
torch.cuda.synchronize()
te = time.perf_counter()
logger.info('NMS stuff Time {:0.3f}'.format(te - tf))
logger.info('Evaluating detections for epoch number ' + str(iteration_num))
mAP, ap_all, ap_strs = evaluate.evaluate(gt_boxes_all,
det_boxes,
args.all_classes,
iou_thresh=iou_thresh)
mAP_ego, ap_all_ego, ap_strs_ego = evaluate.evaluate_ego(
np.asarray(ego_gts), np.asarray(ego_pds), args.ego_classes)
return mAP + [mAP_ego], ap_all + [ap_all_ego], ap_strs + [ap_strs_ego]
def train(self):
print('\n:: training started\n')
epochs = self.config['epochs']
best_dev_accuracy = 0.0
epochs_without_improvement = 0
random_input_prob = self.config.get('random_input_prob', 0.0)
unk_action_id = self.action_templates.index(UNK)
for j in range(epochs):
losses = []
batch_gen = batch_generator([self.X_train, self.context_features_train, self.action_masks_train, self.prev_action_train, self.y_train], self.batch_size)
for batch in batch_gen: # batch_x, batch_context_features, batch_action_masks, batch_y in batch_gen:
batch_copy = [np.copy(elem) for elem in batch]
X, context_features, action_masks, prev_action, y = batch_copy
num_turns = np.sum(np.vectorize(lambda x: x!= 0)(y))
for idx in range(num_turns):
if np.random.random() < random_input_prob:
random_input_idx = np.random.choice(range(self.random_input[0].shape[0]))
random_input = [random_input_i[random_input_idx] for random_input_i in self.random_input]
X[0][idx] = random_input[0]
y[0][idx] = unk_action_id
if idx + 1 < num_turns:
prev_action[0][idx + 1] = unk_action_id
batch_loss_dict, lr = self.net.train_step(X, context_features, action_masks, prev_action, y)
# evaluate every epoch
train_accuracy, train_loss_dict = evaluate(self.net, (self.X_train, self.context_features_train, self.action_masks_train, self.prev_action_train, self.y_train))
train_loss_report = ' '.join(['{}: {:.3f}'.format(key, value) for key, value in train_loss_dict.items()])
dev_accuracy, dev_loss_dict = evaluate(self.net, (self.X_dev, self.context_features_dev, self.action_masks_dev, self.prev_action_dev, self.y_dev))
dev_loss_report = ' '.join(['{}: {:.3f}'.format(key, value) for key, value in dev_loss_dict.items()])
print(':: {}@lr={:.5f} || trn accuracy {:.3f} {} || dev accuracy {:.3f} {}'.format(j + 1, lr, train_accuracy, train_loss_report, dev_accuracy, dev_loss_report))
eval_stats_noisy = evaluate_advanced(self.net,
(self.X_test, self.context_features_test, self.action_masks_test, self.prev_action_test, self.y_test),
self.action_templates,
BABI_CONFIG['backoff_utterance'].lower(),
post_ood_turns=self.post_ood_turns_noisy,
runs_number=1)
print('\n\n')
print('Noisy dataset: {} turns overall, {} turns after the first OOD'.format(eval_stats_noisy['total_turns'],
eval_stats_noisy['total_turns_after_ood']))
print('Accuracy:')
accuracy = eval_stats_noisy['correct_turns'] / eval_stats_noisy['total_turns']
accuracy_after_ood = eval_stats_noisy['correct_turns_after_ood'] / eval_stats_noisy['total_turns_after_ood'] \
if eval_stats_noisy['total_turns_after_ood'] != 0 \
else 0
accuracy_post_ood = eval_stats_noisy['correct_post_ood_turns'] / eval_stats_noisy['total_post_ood_turns'] \
if eval_stats_noisy['total_post_ood_turns'] != 0 \
else 0
accuracy_ood = eval_stats_noisy['correct_ood_turns'] / eval_stats_noisy['total_ood_turns'] \
if eval_stats_noisy['total_ood_turns'] != 0 \
else 0
print('overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}; OOD: {:.3f}'.format(accuracy,
accuracy_after_ood,
accuracy_post_ood,
accuracy_ood))
if best_dev_accuracy < dev_accuracy:
print('New best dev loss. Saving checkpoint')
self.net.save(self.model_folder)
best_dev_accuracy = dev_accuracy
epochs_without_improvement = 0
else:
epochs_without_improvement += 1
if self.config['early_stopping_threshold'] < epochs_without_improvement:
print('Finished after {} epochs due to early stopping'.format(j))
break
def perform_detection(args, net, val_data_loader, val_dataset, iteration):
"""Test a network on a video database."""
num_images = len(val_dataset)
print_time = True
val_step = 50
count = 0
torch.cuda.synchronize()
ts = time.perf_counter()
activation = torch.nn.Sigmoid().cuda()
ego_pds = []
ego_gts = []
det_boxes = []
gt_boxes_all = []
for nlt in range(1):
numc = args.num_classes_list[nlt]
det_boxes.append([[] for _ in range(numc)])
gt_boxes_all.append([])
nlt = 0
processed_videos = []
with torch.no_grad():
for val_itr, (images, gt_boxes, gt_targets, ego_labels, batch_counts,
img_indexs, wh) in enumerate(val_data_loader):
torch.cuda.synchronize()
t1 = time.perf_counter()
batch_size = images.size(0)
images = images.cuda(0, non_blocking=True)
decoded_boxes, confidence, ego_preds = net(images)
ego_preds = activation(ego_preds).cpu().numpy()
ego_labels = ego_labels.numpy()
confidence = activation(confidence)
seq_len = ego_preds.shape[1]
if print_time and val_itr % val_step == 0:
torch.cuda.synchronize()
tf = time.perf_counter()
logger.info('Forward Time {:0.3f}'.format(tf - t1))
for b in range(batch_size):
index = img_indexs[b]
annot_info = val_dataset.ids[index]
video_id, frame_num, step_size = annot_info
videoname = val_dataset.video_list[video_id]
save_dir = '{:s}/{}'.format(args.det_save_dir, videoname)
store_last = False
if videoname not in processed_videos:
processed_videos.append(videoname)
store_last = True
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
count += 1
for s in range(seq_len):
if ego_labels[b, s] > -1:
ego_pds.append(ego_preds[b, s, :])
ego_gts.append(ego_labels[b, s])
gt_boxes_batch = gt_boxes[b,
s, :batch_counts[b,
s], :].numpy()
gt_labels_batch = gt_targets[b,
s, :batch_counts[b,
s]].numpy()
decoded_boxes_batch = decoded_boxes[b, s]
frame_gt = utils.get_individual_labels(
gt_boxes_batch, gt_labels_batch[:, :1])
gt_boxes_all[0].append(frame_gt)
confidence_batch = confidence[b, s]
scores = confidence_batch[:, 0].squeeze().clone()
cls_dets, save_data = utils.filter_detections_for_dumping(
args, scores, decoded_boxes_batch, confidence_batch)
det_boxes[0][0].append(cls_dets)
save_name = '{:s}/{:05d}.pkl'.format(
save_dir, frame_num + 1)
frame_num += step_size
save_data = {'ego': ego_preds[b, s, :], 'main': save_data}
if s < seq_len - args.skip_ending or store_last:
with open(save_name, 'wb') as ff:
pickle.dump(save_data, ff)
if print_time and val_itr % val_step == 0:
torch.cuda.synchronize()
te = time.perf_counter()
logger.info('im_detect: {:d}/{:d} time taken {:0.3f}'.format(
count, num_images, te - ts))
torch.cuda.synchronize()
ts = time.perf_counter()
if print_time and val_itr % val_step == 0:
torch.cuda.synchronize()
te = time.perf_counter()
logger.info('NMS stuff Time {:0.3f}'.format(te - tf))
mAP, ap_all, ap_strs = evaluate.evaluate(gt_boxes_all,
det_boxes,
args.all_classes,
iou_thresh=args.IOU_THRESH)
mAP_ego, ap_all_ego, ap_strs_ego = evaluate.evaluate_ego(
np.asarray(ego_gts), np.asarray(ego_pds), args.ego_classes)
return mAP + [mAP_ego], ap_all + [ap_all_ego], ap_strs + [ap_strs_ego]
arg_par.add_argument('-g', '--gold', dest='gold', help='gold', default='gold.conll06')
arg_par.add_argument('-o', '--output', dest='out_file', help='output file', default='predicted.conll06')
arg_par.add_argument('-e', '--epochs', dest='epochs', help='epochs', default='10')
arg_par.add_argument('-decrease-alpha', dest='decrease_alpha', action='store_true', help='decrease alpha',
default=False)
arg_par.add_argument('-shuffle-sentences', dest='shuffle_sentences', action='store_true', help='shuffle sentences',
default=False)
arguments = arg_par.parse_args()
if os.stat(arguments.in_file).st_size == 0:
print "Input file is empty"
else:
if arguments.train:
print "Running in training mode\n"
train(arguments)
# cProfile.run("train(arguments)")
elif arguments.test:
print "Running in test mode\n"
test(arguments)
elif arguments.evaluate:
print "Running in evaluation mode\n"
evaluate(arguments)
t1 = time.time()
print "\n\tDone. Total time: " + str(t1 - t0) + " sec.\n"