def train(obj, optimizer, dataset, xp, args, epoch): xp.Timer_Train.reset() stats = {} for i, x, y in tqdm(optimizer.get_sampler(dataset), desc='Train Epoch', leave=False, total=optimizer.get_sampler_len(dataset)): oracle_info = obj.oracle(optimizer.variables.w, x, y) oracle_info['i'] = i optimizer.step(oracle_info) # track statistics for monitoring stats['obj'] = float(oracle_info['obj']) stats['error'] = float(obj.task_error(optimizer.variables.w, x, y)) stats['size'] = float(x.size(0)) update_metrics(xp, stats) xp.Timer_Train.update() print('\nEpoch: [{0}] (Train) \t' '({timer:.2f}s) \t' 'Obj {obj:.3f}\t' 'Error {error:.2f}\t'.format( int(xp.Epoch.value), timer=xp.Timer_Train.value, error=xp.Error_Train.value, obj=xp.Obj_Train.value, )) log_metrics(xp, epoch)
def train(model, loss, optimizer, loader, xp, args): model.train() xp.Timer_Train.reset() stats_dict = {} for x, y in tqdm(loader, disable=not args.tqdm, desc='Train Epoch', leave=False, total=len(loader)): (x, y) = (x.cuda(), y.cuda()) if args.cuda else (x, y) # forward pass scores = model(x) # compute the loss function, possibly using smoothing with set_smoothing_enabled(args.smooth_svm): loss_value = loss(scores, y) # backward pass optimizer.zero_grad() loss_value.backward() # optimization step optimizer.step(lambda: float(loss_value)) # monitoring stats_dict['loss'] = float(loss(scores, y)) stats_dict['acc'] = float(accuracy(scores, y)) stats_dict['gamma'] = float(optimizer.gamma) stats_dict['size'] = float(scores.size(0)) update_metrics(xp, stats_dict) xp.Eta.update(optimizer.eta) xp.Reg.update(regularization(model, args.l2)) xp.Obj_Train.update(xp.Reg.value + xp.Loss_Train.value) xp.Timer_Train.update() print('\nEpoch: [{0}] (Train) \t' '({timer:.2f}s) \t' 'Obj {obj:.3f}\t' 'Loss {loss:.3f}\t' 'Acc {acc:.2f}%\t' .format(int(xp.Epoch.value), timer=xp.Timer_Train.value, acc=xp.Acc_Train.value, obj=xp.Obj_Train.value, loss=xp.Loss_Train.value)) log_metrics(xp)
def evaluate(model, loss_fn, test_loader, params, dirs, istest=False): '''Evaluate the model on the test set. Args: model: (torch.nn.Module) the Deep AR model loss_fn: a function that takes outputs and labels per timestep, and then computes the loss for the batch test_loader: load test data and labels params: (Params) hyperparameters ''' logger = logging.getLogger('DeepAR.Eval') model.eval() with torch.no_grad(): summary = {} metrics = utils.init_metrics(params, dirs) for i, (test_batch, labels) in enumerate(tqdm(test_loader)): test_batch = test_batch.permute(1, 0, 2).to(torch.float32).to( dirs.device) labels = labels.to(torch.float32).to(dirs.device) batch_size = test_batch.shape[1] hidden = model.init_hidden(batch_size) cell = model.init_cell(batch_size) for t in range(params.pred_start): _, hidden, cell = model(test_batch[t].unsqueeze(0), hidden, cell) #save some params of SQF for plot if istest and (i == 0): plot_param, _, _ = model( test_batch[params.pred_start].unsqueeze(0), hidden, cell) save_name = os.path.join(dirs.model_dir, 'sqf_param') with open(save_name, 'wb') as f: import pickle pickle.dump(plot_param, f) pickle.dump(test_batch[params.pred_start], f) samples, _, _ = model.predict(test_batch, hidden, cell, sampling=True) metrics = utils.update_metrics(metrics, samples, labels, params.pred_start) summary = utils.final_metrics(metrics) if istest == False: strings ='\nCRPS: '+str(summary['CRPS'])+\ '\nmre:'+str(summary['mre'].abs().max(dim=1)[0].mean().item())+\ '\nPINAW:'+str(summary['pinaw'].item()) logger.info('- Full test metrics: ' + strings) else: logger.info(' - Test Set CRPS: ' + str(summary['CRPS'].mean().item())) ss_metric = {} ss_metric['CRPS_Mean'] = summary['CRPS'].mean() ss_metric['mre'] = summary['mre'].abs().mean() ss_metric['pinaw'] = summary['pinaw'] for i, crps in enumerate(summary['CRPS']): ss_metric[f'CRPS_{i}'] = crps for i, mre in enumerate(summary['mre'].mean(dim=0)): ss_metric[f'mre_{i}'] = mre return ss_metric
def evaluate(model, test_loader, params, plot_num): '''Evaluate the model on the test set. Args: model: (torch.nn.Module) the Deep AR model loss_fn: a function that takes outputs and labels per timestep, and then computes the loss for the batch test_loader: load test data and labels params: (Params) hyperparameters plot_num: (-1): evaluation from evaluate.py; else (epoch): evaluation on epoch sample: (boolean) do ancestral sampling or directly use output mu from last time step ''' model.eval() with torch.no_grad(): plot_batch = np.random.randint(len(test_loader) - 1) summary_metric = {} raw_metrics = utils.init_metrics() sum_mu = torch.zeros([740, params.predict_steps]).to(params.device) sum_sigma = torch.zeros([740, params.predict_steps]).to(params.device) true = torch.zeros([740, params.predict_steps]).to(params.device) for i, (test_batch, id_batch, v, labels) in enumerate(tqdm(test_loader)): test_batch = test_batch.to(torch.float32).to(params.device) id_batch = id_batch.unsqueeze(-1).to(params.device) v_batch = v.to(torch.float32).to(params.device) labels = labels.to(torch.float32).to(params.device) batch_size = test_batch.shape[0] sample_mu, sample_q90 = transformer.test(model, params, test_batch, v_batch, id_batch) raw_metrics = utils.update_metrics( raw_metrics, sample_mu, labels, params.test_predict_start, relative=params.relative_metrics) if (i == 0): sum_mu = sample_mu sum_q90 = sample_q90 true = labels[:, -params.predict_steps:] else: sum_mu = torch.cat([sum_mu, sample_mu], 0) sum_q90 = torch.cat([sum_q90, sample_q90], 0) true = torch.cat([true, labels[:, -params.predict_steps:]], 0) summary_metric = utils.final_metrics(raw_metrics) summary_metric['q50'] = transformer.quantile_loss(0.5, sum_mu, true) summary_metric['q90'] = transformer.quantile_loss(0.5, sum_q90, true) summary_metric['MAPE'] = transformer.MAPE(sum_mu, true) metrics_string = '; '.join('{}: {:05.3f}'.format(k, v) for k, v in summary_metric.items()) logger.info('- Full test metrics: ' + metrics_string) return summary_metric
def evaluate_dataset(model, dataloader): model.eval() running_metrics = None for images_lowres, images_fullres, targets in dataloader: images_lowres, images_fullres, targets = utils.device( [images_lowres, images_fullres, targets]) with torch.no_grad(): predictions = model(images_lowres, images_fullres) psnr = compute_psnr(predictions, targets) running_metrics = utils.update_metrics(running_metrics, {'psnr': psnr}, len(dataloader)) model.train() return running_metrics
def evaluate(model, loss_fn, test_loader, params, plot_num, sample=True): '''Evaluate the model on the test set. Args: model: (torch.nn.Module) the Deep AR model loss_fn: a function that takes outputs and labels per timestep, and then computes the loss for the batch test_loader: load test data and labels params: (Params) hyperparameters plot_num: (-1): evaluation from evaluate.py; else (epoch): evaluation on epoch sample: (boolean) do ancestral sampling or directly use output mu from last time step ''' model.eval() with torch.no_grad(): plot_batch = np.random.randint(len(test_loader)-1) summary_metric = {} raw_metrics = utils.init_metrics(sample=sample) # Test_loader: # test_batch ([batch_size, train_window, 1+cov_dim]): z_{0:T-1} + x_{1:T}, note that z_0 = 0; # id_batch ([batch_size]): one integer denoting the time series id; # v ([batch_size, 2]): scaling factor for each window; # labels ([batch_size, train_window]): z_{1:T}. for i, (test_batch, id_batch, v, labels) in enumerate(tqdm(test_loader)): test_batch = test_batch.permute(1, 0, 2).to(torch.float32).to(params.device) id_batch = id_batch.unsqueeze(0).to(params.device) v_batch = v.to(torch.float32).to(params.device) labels = labels.to(torch.float32).to(params.device) batch_size = test_batch.shape[1] input_mu = torch.zeros(batch_size, params.test_predict_start, device=params.device) # scaled input_sigma = torch.zeros(batch_size, params.test_predict_start, device=params.device) # scaled hidden = model.init_hidden(batch_size) cell = model.init_cell(batch_size) for t in range(params.test_predict_start): # if z_t is missing, replace it by output mu from the last time step zero_index = (test_batch[t,:,0] == 0) if t > 0 and torch.sum(zero_index) > 0: test_batch[t,zero_index,0] = mu[zero_index] mu, sigma, hidden, cell = model(test_batch[t].unsqueeze(0), id_batch, hidden, cell) input_mu[:,t] = v_batch[:, 0] * mu + v_batch[:, 1] input_sigma[:,t] = v_batch[:, 0] * sigma if sample: samples, sample_mu, sample_sigma = model.test(test_batch, v_batch, id_batch, hidden, cell, sampling=True) raw_metrics = utils.update_metrics(raw_metrics, input_mu, input_sigma, sample_mu, labels, params.test_predict_start, samples, relative = params.relative_metrics) else: sample_mu, sample_sigma = model.test(test_batch, v_batch, id_batch, hidden, cell) raw_metrics = utils.update_metrics(raw_metrics, input_mu, input_sigma, sample_mu, labels, params.test_predict_start, relative = params.relative_metrics) if i == plot_batch: if sample: sample_metrics = utils.get_metrics(sample_mu, labels, params.test_predict_start, samples, relative = params.relative_metrics) else: sample_metrics = utils.get_metrics(sample_mu, labels, params.test_predict_start, relative = params.relative_metrics) # select 10 from samples with highest error and 10 from the rest top_10_nd_sample = (-sample_metrics['ND']).argsort()[:batch_size // 10] # hard coded to be 10 chosen = set(top_10_nd_sample.tolist()) all_samples = set(range(batch_size)) not_chosen = np.asarray(list(all_samples - chosen)) if batch_size < 100: # make sure there are enough unique samples to choose top 10 from random_sample_10 = np.random.choice(top_10_nd_sample, size=10, replace=True) else: random_sample_10 = np.random.choice(top_10_nd_sample, size=10, replace=False) if batch_size < 12: # make sure there are enough unique samples to choose bottom 90 from random_sample_90 = np.random.choice(not_chosen, size=10, replace=True) else: random_sample_90 = np.random.choice(not_chosen, size=10, replace=False) combined_sample = np.concatenate((random_sample_10, random_sample_90)) label_plot = labels[combined_sample].data.cpu().numpy() predict_mu = sample_mu[combined_sample].data.cpu().numpy() predict_sigma = sample_sigma[combined_sample].data.cpu().numpy() plot_mu = np.concatenate((input_mu[combined_sample].data.cpu().numpy(), predict_mu), axis=1) plot_sigma = np.concatenate((input_sigma[combined_sample].data.cpu().numpy(), predict_sigma), axis=1) plot_metrics = {_k: _v[combined_sample] for _k, _v in sample_metrics.items()} plot_eight_windows(params.plot_dir, plot_mu, plot_sigma, label_plot, params.test_window, params.test_predict_start, plot_num, plot_metrics, sample) summary_metric = utils.final_metrics(raw_metrics, sampling=sample) metrics_string = '; '.join('{}: {:05.3f}'.format(k, v) for k, v in summary_metric.items()) logger.info('- Full test metrics: ' + metrics_string) return summary_metric
def set_df(df, settings, metrics, db): machine_type = __name__.split('.')[-1] S, MIN_D, MAX_X, MAX_Y, USE_BIGCHAINDB = (settings[k] for k in ('S', 'MIN_D', 'MAX_X', 'MAX_Y', 'USE_BIGCHAINDB')) s = (df['machine_type']==machine_type) speed, reward, penalty = (settings['machines'][machine_type][k] for k in ('speed', 'reward', 'penalty')) # print('********************************', speed, reward, penalty) col_x_trg, col_y_trg = 'x_trg', 'y_trg' # df.loc[s,'x_near'] = df.loc[s,'x_trg'] # df.loc[s,'y_near'] = df.loc[s,'y_trg'] for ix, row in df.loc[s].iterrows(): #print(i,r) neighbour_ix = dist(row.x,row.y,df[df.index!=ix].x,df[df.index!=ix].y).idxmin() # if I am not the nearest set nearest as target # if ix!=neighbour_ix: df.loc[ix,'ix_near'] = int(neighbour_ix) df.loc[ix,'x_near'] = df.loc[neighbour_ix,'x'] df.loc[ix,'y_near'] = df.loc[neighbour_ix,'y'] # reward #reward = 1.0 df.loc[s,'ds'] = dist(df.loc[s,'x'], df.loc[s,'y'], df.loc[s,col_x_trg], df.loc[s,col_y_trg]) df.loc[s,'dx'] = df.loc[s,col_x_trg] - df.loc[s,'x'] df.loc[s,'dy'] = df.loc[s,col_y_trg] - df.loc[s,'y'] # penalty #penalty = 1.0#1.0 df.loc[s,'ds1'] = dist(df.loc[s,'x'], df.loc[s,'y'], df.loc[s,'x_near'], df.loc[s,'y_near']) df.loc[s,'dx1'] = -(df.loc[s,'x_near'] - df.loc[s,'x']) df.loc[s,'dy1'] = -(df.loc[s,'y_near'] - df.loc[s,'y']) # speed is max(the remaining dist, S) ds is always positive selection = s & (df['ds']>=MIN_D) #if not selection.empty: fr = df.loc[selection] df.loc[selection,'u'] = fr.ds.clip_upper(reward*speed) * fr['dx']/fr['ds'] df.loc[selection,'v'] = fr.ds.clip_upper(reward*speed) * fr['dy']/fr['ds'] #df.loc[selection,'u1'] = fr.ds1.clip_upper(penalty*speed) * fr['dx1']/(fr['ds1']**3) #df.loc[selection,'v1'] = fr.ds1.clip_upper(penalty*speed) * fr['dy1']/(fr['ds1']**3) df.loc[selection,'u1'] = (1.0/ (fr['dx1']**2)).clip_upper(penalty*speed) df.loc[selection,'v1'] = (1.0/ (fr['dy1']**2)).clip_upper(penalty*speed) # TODO: doesn't really work #selection = s & (df['ds']<MIN_Dspeed) df.loc[selection,'u'] += df.loc[selection,'u1'] df.loc[selection,'v'] += df.loc[selection,'v1'] # on collision turn left selection = s & (df['collision']) # fr = df.loc[selection] # choose random vector, noise evolution # df.loc[selection, 'u']=rnd_vec(len(df.loc[selection]),2.0)-1.0 # df.loc[selection, 'v']=rnd_vec(len(df.loc[selection]),2.0)-1.0 # df.loc[selection, 'ds']=(df.loc[selection, 'u']**2 + df.loc[selection, 'v']**2)**0.5 # df.loc[selection,'u'] = fr.ds.clip_upper(speed) * fr['u']/fr['ds'] # df.loc[selection,'v'] = fr.ds.clip_upper(speed) * fr['v']/fr['ds'] # df.loc[selection,'u'] = fr.ds.clip_upper(speed) * fr['u']/fr['ds'] # df.loc[selection,'v'] = fr.ds.clip_upper(speed) * fr['v']/fr['ds'] u = df.loc[selection, 'v'] v = -df.loc[selection, 'u'] df.loc[selection, 'u'] = u df.loc[selection, 'v'] = v update_metrics(db, metrics, 'collisions', machine_type, len(df.loc[selection])) # determine if target is reached and set new target # new target should come from blockchain selection = s & (df['ds'].abs()<MIN_D) #if not selection.empty: l = len(df.loc[selection]) if l>0: print('targets reached', l) df.loc[selection,'x_trg'] = rnd_vec(l,MAX_X) df.loc[selection, 'y_trg'] = rnd_vec(l,MAX_Y) # alex # this code uses simple binary state change to describe whether the robot is carrying # an asset (for example energy or a parcel or whatever) update_metrics(db, metrics, 'pickups', machine_type, len(df[selection & (df['state']=='empty')])) update_metrics(db, metrics, 'dropoffs', machine_type, len(df[selection & (df['state']=='carry')])) for ix, row in df.loc[selection].iterrows(): # for each bot that reached a new waypoint change its behaviour if df.loc[ix,'state'] == 'empty': print('empty bot changed to carry') # alex adjusted currentBot = df.loc[ix] if USE_BIGCHAINDB: createAsset(master_sender_private_key,master_sender_public_key) # mockup sender initialization, creation of paackage to take for bot transferAsset(master_sender_public_key,master_sender_private_key,currentBot.public_key,currentBot.private_key) # transfer parcel from master_sender to current robot df.loc[ix,'state'] = 'carry' elif df.loc[ix,'state'] == 'carry': print('carry bot changed to empty') # alex adjusted currentBot = df.loc[ix] if USE_BIGCHAINDB: transferAsset(currentBot.public_key,currentBot.private_key, master_receiver_public_key,master_receiver_private_key) # transfer parcel from current robot to master receiver df.loc[ix,'state'] = 'empty' # alex end
targets) in enumerate(train_loader): step_num = utils.step_num(epoch, batch, train_loader) images_lowres, images_fullres, targets = utils.device( [images_lowres, images_fullres, targets]) predictions = model(images_lowres, images_fullres) loss = F.mse_loss(predictions, targets) optimizer.zero_grad() loss.backward() optimizer.step() running_metrics = utils.update_metrics( running_metrics, { 'mse': loss.item(), 'psnr': compute_psnr_from_mse(loss) }, print_freq) if (step_num + 1) % print_freq == 0: utils.log_to_tensorboard(writer, running_metrics, step_num) utils.print_metrics(running_metrics, step_num, n_epochs * len(train_loader)) running_metrics = {l: 0 for l in running_metrics} print('Finished epoch %d\n' % epoch) print('Evaluation') eval_metrics = evaluate_dataset(model, test_loader) utils.log_to_tensorboard(writer, eval_metrics, step_num, log_prefix='test') utils.print_metrics(eval_metrics, step_num, n_epochs * len(train_loader))
def trainepoch(epoch): xp.Epoch.update(1).log() print('\nTRAINING : Epoch ' + str(epoch)) nli_net.train() # shuffle the data permutation = np.random.permutation(len(train['s1'])) s1 = train['s1'][permutation] s2 = train['s2'][permutation] target = train['label'][permutation] if epoch > 1 and params.opt == 'sgd': optimizer.param_groups[0]['lr'] *= params.decay optimizer.eta = optimizer.param_groups[0]['lr'] xp.Timer_Train.reset() stats = {} for stidx in tqdm(range(0, len(s1), params.batch_size), disable=not params.tqdm, desc='Train Epoch', leave=False): # prepare batch s1_batch, s1_len = get_batch(s1[stidx:stidx + params.batch_size], word_vec) s2_batch, s2_len = get_batch(s2[stidx:stidx + params.batch_size], word_vec) s1_batch, s2_batch = s1_batch.cuda(), s2_batch.cuda() tgt_batch = torch.LongTensor(target[stidx:stidx + params.batch_size]).cuda() # model forward scores = nli_net((s1_batch, s1_len), (s2_batch, s2_len)) with set_smoothing_enabled(params.smooth_svm): loss = loss_fn(scores, tgt_batch) # backward optimizer.zero_grad() loss.backward() if params.opt != 'dfw': adapt_grad_norm(nli_net, params.max_norm) # necessary information for the step-size of some optimizers -> provide closure optimizer.step(lambda: float(loss)) # track statistics for monitoring stats['loss'] = float(loss_fn(scores, tgt_batch)) stats['acc'] = float(accuracy(scores, tgt_batch)) stats['gamma'] = float(optimizer.gamma) stats['size'] = float(tgt_batch.size(0)) update_metrics(xp, stats) xp.Eta.update(optimizer.eta) xp.Reg.update(regularization(nli_net, params.l2)) xp.Obj_Train.update(xp.Reg.value + xp.Loss_Train.value) xp.Timer_Train.update() print('results : epoch {0} ; mean accuracy train : {1}'.format( epoch, xp.acc_train)) print('\nEpoch: [{0}] (Train) \t' '({timer:.2f}s) \t' 'Obj {obj:.3f}\t' 'Loss {loss:.3f}\t' 'Acc {acc:.2f}%\t'.format(int(xp.Epoch.value), timer=xp.Timer_Train.value, acc=xp.Acc_Train.value, obj=xp.Obj_Train.value, loss=xp.Loss_Train.value)) log_metrics(xp)
def evaluate(model, loss_fn, test_loader, params, sample=True): ''' Evaluate the model on the test set. Args: model: (torch.nn.Module) the Deep AR model loss_fn: a function that takes outputs and labels per timestep, and then computes the loss for the batch test_loader: load test data and labels params: (Params) hyperparameters sample: (boolean) do ancestral sampling or directly use output mu from last time step ''' model.eval() with torch.no_grad(): summary_metric = {} raw_metrics = utils.init_metrics(sample=sample) # Test_loader: # test_batch ([batch_size, train_window, 1+cov_dim]): z_{0:T-1} + x_{1:T}, note that z_0 = 0; # id_batch ([batch_size]): one integer denoting the time series id; # v ([batch_size, 2]): scaling factor for each window; # labels ([batch_size, train_window]): z_{1:T}. result_mu = [] result_sigma = [] for i, (test_batch, id_batch, v, labels) in enumerate(test_loader): test_batch = test_batch.permute(1, 0, 2).to(torch.float32).to(params.device) id_batch = id_batch.unsqueeze(0).to(params.device) v_batch = v.to(torch.float32).to(params.device) labels = labels.to(torch.float32).to(params.device) batch_size = test_batch.shape[1] input_mu = torch.zeros(batch_size, params.test_predict_start, device=params.device) # scaled input_sigma = torch.zeros(batch_size, params.test_predict_start, device=params.device) # scaled hidden = model.init_hidden(batch_size) cell = model.init_cell(batch_size) for t in range(params.test_predict_start): # 先计算encoder部分 # if z_t is missing, replace it by output mu from the last time step # 如果z_t缺失,用前一步预测值代替真实值作为输入 zero_index = (test_batch[t, :, 0] == 0) if t > 0 and torch.sum(zero_index) > 0: test_batch[t, zero_index, 0] = mu[zero_index] mu, sigma, hidden, cell = model(test_batch[t].unsqueeze(0), id_batch, hidden, cell) input_mu[:, t] = v_batch[:, 0] * mu + v_batch[:, 1] # v_batch[:, 1] == 0, useless input_sigma[:, t] = v_batch[:, 0] * sigma if not params.one_step: test_batch[params.test_predict_start, :, 0] = mu # 计算decoder部分 if sample: samples, sample_mu, sample_sigma = model.test(test_batch, v_batch, id_batch, hidden, cell, sampling=True, one_step=params.one_step) raw_metrics = utils.update_metrics(raw_metrics, input_mu, input_sigma, sample_mu, labels, params.test_predict_start, samples, relative = params.relative_metrics) else: sample_mu, sample_sigma = model.test(test_batch, v_batch, id_batch, hidden, cell, one_step=params.one_step) raw_metrics = utils.update_metrics(raw_metrics, input_mu, input_sigma, sample_mu, labels, params.test_predict_start, relative = params.relative_metrics) result_mu.append(sample_mu) result_sigma.append(sample_sigma) summary_metric = utils.final_metrics(raw_metrics, sampling=sample) metrics_string = '; '.join('{}: {:05.3f}'.format(k, v) for k, v in summary_metric.items()) # print('test metrics: ' + metrics_string) return summary_metric, result_mu, result_sigma