def test_remove_experiment(self): cc = CrayonClient(port=self.test_server_port) self.assertRaises(ValueError, cc.open_experiment, "foo") foo = cc.create_experiment("foo") foo.add_scalar_value("bar", 1, step=2, wall_time=0) self.assertRaises(ValueError, cc.create_experiment, "foo") cc.open_experiment("foo") cc.remove_experiment(foo.xp_name) self.assertRaises(ValueError, cc.remove_experiment, foo.xp_name) foo = cc.create_experiment("foo")
def test_open_experiment(self): cc = CrayonClient(port=self.test_server_port) foo = cc.create_experiment("foo") foo.add_scalar_value("bar", 1, step=2, wall_time=0) foo = cc.open_experiment("foo") foo.add_scalar_value("bar", 3, wall_time=1) self.assertEqual(foo.get_scalar_values("bar"), [[0.0, 2, 1.0], [1.0, 3, 3.0]])
class Monitor(object): def __init__(self, address, port): self.cc = CrayonClient(hostname=address, port=port) def start_experiment(self, name, clean=True): exps = self.cc.get_experiment_names() if name in exps: if clean: self.cc.remove_experiment(name) self.exp = self.cc.create_experiment(name) print 'clean and creat a new one' else: self.exp = self.cc.open_experiment(name) else: self.exp = self.cc.create_experiment(name) def push(self, data, wall_time=-1, step=-1): self.exp.add_scalar_dict(data, wall_time, step)
def make_crayon_experiments(experiment_name, new=True): client = CrayonClient(hostname=config.CRAYON_SERVER_HOSTNAME) train_experiment_name = f'{experiment_name}_train' valid_experiment_name = f'{experiment_name}_valid' if new: try: client.remove_experiment(train_experiment_name) except ValueError: pass try: client.remove_experiment(valid_experiment_name) except ValueError: pass train_experiment = client.create_experiment( train_experiment_name) train_experiment.scalar_steps['lr'] = 1 valid_experiment = client.create_experiment(valid_experiment_name) else: train_experiment = client.open_experiment(train_experiment_name) valid_experiment = client.open_experiment(valid_experiment_name) return train_experiment, valid_experiment
def main(): args = parse_args() ctx = mx.gpu(args.gpu) print(args) cc = CrayonClient(hostname='10.132.90.242') if args.exp_name is None: args.exp_name = datetime.now().strftime('frcnnEval_%m-%d') try: exp = cc.create_experiment(args.exp_name) except: exp = cc.open_experiment(args.exp_name) for x in args.epoch.split(","): mAp = test_rcnn(args.network, args.dataset, args.image_set, args.root_path, args.dataset_path, ctx, args.prefix, int(x), args.vis, args.shuffle, args.has_rpn, args.proposal, args.thresh, args.use_global_context, args.use_roi_align) exp.add_scalar_value('mAp', mAp) return
def get_crayon_experiment(exp_name, hostname='127.0.0.1', overwrite=True): cc = CrayonClient(hostname=hostname) cc_exp = None experiments = cc.get_experiment_names() if exp_name in experiments: if overwrite: cc.remove_experiment(exp_name) cc_exp = cc.create_experiment(exp_name) else: cc_exp = cc.open_experiment(exp_name) else: try: cc_exp = cc.create_experiment(exp_name) except ValueError: cc.remove_experiment(exp_name) cc_exp = cc.create_experiment(exp_name) return cc_exp
def init_crayon(hostname, experiment_name): try: from pycrayon import CrayonClient cc = CrayonClient(hostname) try: Logger.exp = cc.create_experiment(experiment_name) except ValueError as e: print(e) if input('Open the experiment (y/n)? ').lower() != 'y': raise Logger.exp = cc.open_experiment(experiment_name) except ImportError: print('Importing pycrayon has been failed. ' 'Some features of Logger will disabled.') except ValueError as e: print(e) if input('continue (y/n)? ').lower() != 'y': raise
optimizer = torch.optim.SGD(params[8:], lr=lr, momentum=momentum, weight_decay=weight_decay) if not os.path.exists(output_dir): os.mkdir(output_dir) # tensorboad use_tensorboard = use_tensorboard and CrayonClient is not None if use_tensorboard: cc = CrayonClient(hostname='127.0.0.1') if remove_all_log: cc.remove_all_experiments() if exp_name is None: exp_name = datetime.now().strftime('vgg16_%m-%d_%H-%M') exp = cc.create_experiment(exp_name) else: exp = cc.open_experiment(exp_name) # training train_loss = 0 tp, tf, fg, bg = 0., 0., 0, 0 step_cnt = 0 re_cnt = False t = Timer() t.tic() for step in range(start_step, end_step+1): # get one batch blobs = data_layer.forward() ###### im_data = blobs['data'] im_info = blobs['im_info']
use_tensorboard = False remove_all_log = True if use_tensorboard: cc = CrayonClient(hostname='127.0.0.1') if remove_all_log: print('remove all experiments') cc.remove_all_experiments() if start_epoch == 0: try: cc.remove_experiment(cfg.exp_name) except ValueError: pass exp = cc.create_experiment(cfg.exp_name) else: exp = cc.open_experiment(cfg.exp_name) train_loss = 0 bbox_loss, iou_loss, cls_loss = 0., 0., 0. cnt = 0 timer = Timer() # default input size network_size = cfg.inp_size for step in range(start_epoch * imdb.batch_per_epoch, cfg.max_epoch * imdb.batch_per_epoch + 1): timer.tic() # random change network size if step % cfg.network_size_rand_period == 0:
======= cc = CrayonClient(hostname="", port=8889) >>>>>>> d6f052dba3a3f893fd80497288d9412d8cc1c097 # delete this experiment from the server try: cc.remove_experiment("MNIST_DCGAN_GEN") cc.remove_experiment("MNIST_DCGAN_DIS") except: pass # create a new experiment try: tb_gen = cc.create_experiment("MNIST_DCGAN_GEN") tb_dis = cc.create_experiment("MNIST_DCGAN_DIS") except: tb_gen = cc.open_experiment("MNIST_DCGAN_GEN") tb_dis = cc.open_experiment("MNIST_DCGAN_DIS") # Training Data train, test = chainer.datasets.get_mnist() x_train, t_train = train._datasets x_train = (x_train - 0.5) / 0.5 x_test, t_test = test._datasets # 学習データ数 train_size = len(x_train) # テストデータ数 test_size = len(x_test) # エポック数 epoch_n = 50 # バッチサイズ
def main(): parser = argparse.ArgumentParser(description='mcnn worldexp.') parser.add_argument('--preload', type=int, default=1) parser.add_argument('--data', type=str, default="/mnt/m2/mzcc/crowd_data/worldexpo", help='train, test, etc') args = parser.parse_args() method = 'mcnn' dataset_name = 'worldexpo' output_dir = './saved_models/' data_path = args.data train_path = data_path+'/train_frame' train_gt_path = data_path+'/train_dmap' train_mask_path = os.path.join(data_path,'train_roi') val_path = data_path+'/test_frame' val_gt_path = data_path+'/test_dmap' val_mask_path = os.path.join(data_path, 'test_roi') #training configuration start_step = 0 end_step = 3000 lr = 0.000001 momentum = 0.9 disp_interval = 500 log_interval = 250 #Tensorboard config use_tensorboard = False save_exp_name = method + '_' + dataset_name + '_' + 'v1' remove_all_log = False # remove all historical experiments in TensorBoard exp_name = None # the previous experiment name in TensorBoard # ------------ rand_seed = 64678 if rand_seed is not None: np.random.seed(rand_seed) torch.manual_seed(rand_seed) torch.cuda.manual_seed(rand_seed) # load net net = CrowdCounter() network.weights_normal_init(net, dev=0.01) # network.weights_xavier_init(net, gain=0.01) net.cuda() net.train() params = list(net.parameters()) optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=lr) if not os.path.exists(output_dir): os.mkdir(output_dir) # tensorboad use_tensorboard = use_tensorboard and CrayonClient is not None if use_tensorboard: cc = CrayonClient(hostname='127.0.0.1') if remove_all_log: cc.remove_all_experiments() if exp_name is None: exp_name = save_exp_name exp = cc.create_experiment(exp_name) else: exp = cc.open_experiment(exp_name) # training train_loss = 0 step_cnt = 0 re_cnt = False t = Timer() t.tic() data_loader = ExrImageDataLoader(train_path, train_gt_path, mask_path=train_mask_path, shuffle=True, gt_downsample=True, pre_load=args.preload) data_loader_val = ExrImageDataLoader(val_path, val_gt_path, mask_path=val_mask_path, shuffle=False, gt_downsample=True, pre_load=False) best_mae = 10000000 for epoch in range(start_step, end_step+1): step = -1 train_loss = 0 for blob in data_loader: step = step + 1 im_data = blob['data'] gt_data = blob['gt_density'] mask = blob['mask'] density_map = net(im_data, gt_data, mask=mask) loss = net.loss train_loss += loss.item()#.data[0] step_cnt += 1 optimizer.zero_grad() loss.backward() optimizer.step() if step % disp_interval == 0: print("current loss: {}".format(loss.item())) duration = t.toc(average=False) fps = step_cnt / duration gt_count = np.sum(gt_data) density_map = density_map.data.cpu().numpy() et_count = np.sum(density_map) utils.save_results(im_data,gt_data,density_map, output_dir) log_text = 'epoch: %4d, step %4d, Time: %.4fs, gt_cnt: %4.1f, et_cnt: %4.1f' % (epoch, step, 1./fps, gt_count,et_count) log_print(log_text, color='green', attrs=['bold']) re_cnt = True if re_cnt: t.tic() re_cnt = False if (epoch % 2 == 0): save_name = os.path.join(output_dir, '{}_{}_{}.h5'.format(method,dataset_name,epoch)) network.save_net(save_name, net) #calculate error on the validation dataset mae,mse = evaluate_model(save_name, data_loader_val) if mae < best_mae: best_mae = mae best_mse = mse best_model = '{}_{}_{}.h5'.format(method,dataset_name,epoch) log_text = 'EPOCH: %d, MAE: %.1f, MSE: %0.1f' % (epoch,mae,mse) log_print(log_text, color='green', attrs=['bold']) log_text = 'BEST MAE: %0.1f, BEST MSE: %0.1f, BEST MODEL: %s' % (best_mae,best_mse, best_model) log_print(log_text, color='green', attrs=['bold']) if use_tensorboard: exp.add_scalar_value('MAE', mae, step=epoch) exp.add_scalar_value('MSE', mse, step=epoch) exp.add_scalar_value('train_loss', train_loss/data_loader.get_num_samples(), step=epoch)
# pycrayon 初期化 cc = CrayonClient(hostname="192.168.1.198", port=8889) # delete this experiment from the server try: cc.remove_experiment("AlexNet train (Adam)") cc.remove_experiment("AlexNet test (Adam)") except: pass # create a new experiment try: tb_alex_train = cc.create_experiment("AlexNet train (Adam)") tb_alex_test = cc.create_experiment("AlexNet test (Adam)") except: tb_alex_train = cc.open_experiment("AlexNet train (Adam)") tb_alex_test = cc.open_experiment("AlexNet test (Adam)") # x_train: 32*32*3 train, test = get_cifar10() x_train, t_train = train._datasets x_test, t_test = test._datasets # 学習データサイズ input_size = 32 # 学習データ数 train_size = len(x_train) # テストデータ数 test_size = len(x_test) # エポック数 epoch_n = 150
# pycrayon 初期化 cc = CrayonClient(hostname="192.168.1.90", port=8889) # delete this experiment from the server try: cc.remove_experiment("ResNet train") cc.remove_experiment("ResNet test") except: pass # create a new experiment try: tb_res_train = cc.create_experiment("ResNet train") tb_res_test = cc.create_experiment("ResNet test") except: tb_res_train = cc.open_experiment("ResNet train") tb_res_test = cc.open_experiment("ResNet test") # x_train: 32*32*3 train, test = get_cifar10() x_train, t_train = train._datasets x_test, t_test = test._datasets # 学習データサイズ input_size = 32 # 学習データ数 train_size = len(x_train) # テストデータ数 test_size = len(x_test) # エポック数 epoch_n = 400
# Set up the Crayon logging server. if opt.exp_host != "": from pycrayon import CrayonClient cc = CrayonClient(hostname=opt.exp_host) experiments = cc.get_experiment_names() print(experiments) ''' if opt.exp in experiments: cc.remove_experiment(opt.exp) experiment = cc.create_experiment(opt.exp) ''' if opt.exp in experiments: experiment = cc.open_experiment(opt.exp) else: experiment = cc.create_experiment(opt.exp) def report_func(epoch, batch, num_batches, start_time, lr, report_stats): """ This is the user-defined batch-level traing progress report function. Args: epoch(int): current epoch count. batch(int): current batch count. num_batches(int): total number of batches. start_time(float): last report time. lr(float): current learning rate.