def test_remove_experiment(self):
cc = CrayonClient(port=self.test_server_port)
self.assertRaises(ValueError, cc.open_experiment, "foo")
foo = cc.create_experiment("foo")
foo.add_scalar_value("bar", 1, step=2, wall_time=0)
self.assertRaises(ValueError, cc.create_experiment, "foo")
cc.open_experiment("foo")
cc.remove_experiment(foo.xp_name)
self.assertRaises(ValueError, cc.remove_experiment, foo.xp_name)
foo = cc.create_experiment("foo")
def test_open_experiment(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
foo.add_scalar_value("bar", 1, step=2, wall_time=0)
foo = cc.open_experiment("foo")
foo.add_scalar_value("bar", 3, wall_time=1)
self.assertEqual(foo.get_scalar_values("bar"),
[[0.0, 2, 1.0], [1.0, 3, 3.0]])
class Monitor(object):
def __init__(self, address, port):
self.cc = CrayonClient(hostname=address, port=port)
def start_experiment(self, name, clean=True):
exps = self.cc.get_experiment_names()
if name in exps:
if clean:
self.cc.remove_experiment(name)
self.exp = self.cc.create_experiment(name)
print 'clean and creat a new one'
else:
self.exp = self.cc.open_experiment(name)
else:
self.exp = self.cc.create_experiment(name)
def push(self, data, wall_time=-1, step=-1):
self.exp.add_scalar_dict(data, wall_time, step)
def make_crayon_experiments(experiment_name, new=True):
client = CrayonClient(hostname=config.CRAYON_SERVER_HOSTNAME)
train_experiment_name = f'{experiment_name}_train'
valid_experiment_name = f'{experiment_name}_valid'
if new:
try:
client.remove_experiment(train_experiment_name)
except ValueError:
pass
try:
client.remove_experiment(valid_experiment_name)
except ValueError:
pass
train_experiment = client.create_experiment( train_experiment_name)
train_experiment.scalar_steps['lr'] = 1
valid_experiment = client.create_experiment(valid_experiment_name)
else:
train_experiment = client.open_experiment(train_experiment_name)
valid_experiment = client.open_experiment(valid_experiment_name)
return train_experiment, valid_experiment
def main():
args = parse_args()
ctx = mx.gpu(args.gpu)
print(args)
cc = CrayonClient(hostname='10.132.90.242')
if args.exp_name is None:
args.exp_name = datetime.now().strftime('frcnnEval_%m-%d')
try:
exp = cc.create_experiment(args.exp_name)
except:
exp = cc.open_experiment(args.exp_name)
for x in args.epoch.split(","):
mAp = test_rcnn(args.network, args.dataset, args.image_set,
args.root_path, args.dataset_path, ctx, args.prefix,
int(x), args.vis, args.shuffle, args.has_rpn,
args.proposal, args.thresh, args.use_global_context,
args.use_roi_align)
exp.add_scalar_value('mAp', mAp)
return
def get_crayon_experiment(exp_name, hostname='127.0.0.1', overwrite=True):
cc = CrayonClient(hostname=hostname)
cc_exp = None
experiments = cc.get_experiment_names()
if exp_name in experiments:
if overwrite:
cc.remove_experiment(exp_name)
cc_exp = cc.create_experiment(exp_name)
else:
cc_exp = cc.open_experiment(exp_name)
else:
try:
cc_exp = cc.create_experiment(exp_name)
except ValueError:
cc.remove_experiment(exp_name)
cc_exp = cc.create_experiment(exp_name)
return cc_exp
def init_crayon(hostname, experiment_name):
try:
from pycrayon import CrayonClient
cc = CrayonClient(hostname)
try:
Logger.exp = cc.create_experiment(experiment_name)
except ValueError as e:
print(e)
if input('Open the experiment (y/n)? ').lower() != 'y':
raise
Logger.exp = cc.open_experiment(experiment_name)
except ImportError:
print('Importing pycrayon has been failed. '
'Some features of Logger will disabled.')
except ValueError as e:
print(e)
if input('continue (y/n)? ').lower() != 'y':
raise
optimizer = torch.optim.SGD(params[8:], lr=lr, momentum=momentum, weight_decay=weight_decay)
if not os.path.exists(output_dir):
os.mkdir(output_dir)
# tensorboad
use_tensorboard = use_tensorboard and CrayonClient is not None
if use_tensorboard:
cc = CrayonClient(hostname='127.0.0.1')
if remove_all_log:
cc.remove_all_experiments()
if exp_name is None:
exp_name = datetime.now().strftime('vgg16_%m-%d_%H-%M')
exp = cc.create_experiment(exp_name)
else:
exp = cc.open_experiment(exp_name)
# training
train_loss = 0
tp, tf, fg, bg = 0., 0., 0, 0
step_cnt = 0
re_cnt = False
t = Timer()
t.tic()
for step in range(start_step, end_step+1):
# get one batch
blobs = data_layer.forward()
######
im_data = blobs['data']
im_info = blobs['im_info']
use_tensorboard = False
remove_all_log = True
if use_tensorboard:
cc = CrayonClient(hostname='127.0.0.1')
if remove_all_log:
print('remove all experiments')
cc.remove_all_experiments()
if start_epoch == 0:
try:
cc.remove_experiment(cfg.exp_name)
except ValueError:
pass
exp = cc.create_experiment(cfg.exp_name)
else:
exp = cc.open_experiment(cfg.exp_name)
train_loss = 0
bbox_loss, iou_loss, cls_loss = 0., 0., 0.
cnt = 0
timer = Timer()
# default input size
network_size = cfg.inp_size
for step in range(start_epoch * imdb.batch_per_epoch, cfg.max_epoch * imdb.batch_per_epoch + 1):
timer.tic()
# random change network size
if step % cfg.network_size_rand_period == 0:
=======
cc = CrayonClient(hostname="", port=8889)
>>>>>>> d6f052dba3a3f893fd80497288d9412d8cc1c097
# delete this experiment from the server
try:
cc.remove_experiment("MNIST_DCGAN_GEN")
cc.remove_experiment("MNIST_DCGAN_DIS")
except:
pass
# create a new experiment
try:
tb_gen = cc.create_experiment("MNIST_DCGAN_GEN")
tb_dis = cc.create_experiment("MNIST_DCGAN_DIS")
except:
tb_gen = cc.open_experiment("MNIST_DCGAN_GEN")
tb_dis = cc.open_experiment("MNIST_DCGAN_DIS")
# Training Data
train, test = chainer.datasets.get_mnist()
x_train, t_train = train._datasets
x_train = (x_train - 0.5) / 0.5
x_test, t_test = test._datasets
# 学習データ数
train_size = len(x_train)
# テストデータ数
test_size = len(x_test)
# エポック数
epoch_n = 50
# バッチサイズ
def main():
parser = argparse.ArgumentParser(description='mcnn worldexp.')
parser.add_argument('--preload', type=int, default=1)
parser.add_argument('--data', type=str, default="/mnt/m2/mzcc/crowd_data/worldexpo", help='train, test, etc')
args = parser.parse_args()
method = 'mcnn'
dataset_name = 'worldexpo'
output_dir = './saved_models/'
data_path = args.data
train_path = data_path+'/train_frame'
train_gt_path = data_path+'/train_dmap'
train_mask_path = os.path.join(data_path,'train_roi')
val_path = data_path+'/test_frame'
val_gt_path = data_path+'/test_dmap'
val_mask_path = os.path.join(data_path, 'test_roi')
#training configuration
start_step = 0
end_step = 3000
lr = 0.000001
momentum = 0.9
disp_interval = 500
log_interval = 250
#Tensorboard config
use_tensorboard = False
save_exp_name = method + '_' + dataset_name + '_' + 'v1'
remove_all_log = False # remove all historical experiments in TensorBoard
exp_name = None # the previous experiment name in TensorBoard
# ------------
rand_seed = 64678
if rand_seed is not None:
np.random.seed(rand_seed)
torch.manual_seed(rand_seed)
torch.cuda.manual_seed(rand_seed)
# load net
net = CrowdCounter()
network.weights_normal_init(net, dev=0.01)
# network.weights_xavier_init(net, gain=0.01)
net.cuda()
net.train()
params = list(net.parameters())
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=lr)
if not os.path.exists(output_dir):
os.mkdir(output_dir)
# tensorboad
use_tensorboard = use_tensorboard and CrayonClient is not None
if use_tensorboard:
cc = CrayonClient(hostname='127.0.0.1')
if remove_all_log:
cc.remove_all_experiments()
if exp_name is None:
exp_name = save_exp_name
exp = cc.create_experiment(exp_name)
else:
exp = cc.open_experiment(exp_name)
# training
train_loss = 0
step_cnt = 0
re_cnt = False
t = Timer()
t.tic()
data_loader = ExrImageDataLoader(train_path, train_gt_path, mask_path=train_mask_path,
shuffle=True, gt_downsample=True, pre_load=args.preload)
data_loader_val = ExrImageDataLoader(val_path, val_gt_path, mask_path=val_mask_path,
shuffle=False, gt_downsample=True, pre_load=False)
best_mae = 10000000
for epoch in range(start_step, end_step+1):
step = -1
train_loss = 0
for blob in data_loader:
step = step + 1
im_data = blob['data']
gt_data = blob['gt_density']
mask = blob['mask']
density_map = net(im_data, gt_data, mask=mask)
loss = net.loss
train_loss += loss.item()#.data[0]
step_cnt += 1
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step % disp_interval == 0:
print("current loss: {}".format(loss.item()))
duration = t.toc(average=False)
fps = step_cnt / duration
gt_count = np.sum(gt_data)
density_map = density_map.data.cpu().numpy()
et_count = np.sum(density_map)
utils.save_results(im_data,gt_data,density_map, output_dir)
log_text = 'epoch: %4d, step %4d, Time: %.4fs, gt_cnt: %4.1f, et_cnt: %4.1f' % (epoch,
step, 1./fps, gt_count,et_count)
log_print(log_text, color='green', attrs=['bold'])
re_cnt = True
if re_cnt:
t.tic()
re_cnt = False
if (epoch % 2 == 0):
save_name = os.path.join(output_dir, '{}_{}_{}.h5'.format(method,dataset_name,epoch))
network.save_net(save_name, net)
#calculate error on the validation dataset
mae,mse = evaluate_model(save_name, data_loader_val)
if mae < best_mae:
best_mae = mae
best_mse = mse
best_model = '{}_{}_{}.h5'.format(method,dataset_name,epoch)
log_text = 'EPOCH: %d, MAE: %.1f, MSE: %0.1f' % (epoch,mae,mse)
log_print(log_text, color='green', attrs=['bold'])
log_text = 'BEST MAE: %0.1f, BEST MSE: %0.1f, BEST MODEL: %s' % (best_mae,best_mse, best_model)
log_print(log_text, color='green', attrs=['bold'])
if use_tensorboard:
exp.add_scalar_value('MAE', mae, step=epoch)
exp.add_scalar_value('MSE', mse, step=epoch)
exp.add_scalar_value('train_loss', train_loss/data_loader.get_num_samples(), step=epoch)
# pycrayon 初期化
cc = CrayonClient(hostname="192.168.1.198", port=8889)
# delete this experiment from the server
try:
cc.remove_experiment("AlexNet train (Adam)")
cc.remove_experiment("AlexNet test (Adam)")
except:
pass
# create a new experiment
try:
tb_alex_train = cc.create_experiment("AlexNet train (Adam)")
tb_alex_test = cc.create_experiment("AlexNet test (Adam)")
except:
tb_alex_train = cc.open_experiment("AlexNet train (Adam)")
tb_alex_test = cc.open_experiment("AlexNet test (Adam)")
# x_train: 32*32*3
train, test = get_cifar10()
x_train, t_train = train._datasets
x_test, t_test = test._datasets
# 学習データサイズ
input_size = 32
# 学習データ数
train_size = len(x_train)
# テストデータ数
test_size = len(x_test)
# エポック数
epoch_n = 150
# pycrayon 初期化
cc = CrayonClient(hostname="192.168.1.90", port=8889)
# delete this experiment from the server
try:
cc.remove_experiment("ResNet train")
cc.remove_experiment("ResNet test")
except:
pass
# create a new experiment
try:
tb_res_train = cc.create_experiment("ResNet train")
tb_res_test = cc.create_experiment("ResNet test")
except:
tb_res_train = cc.open_experiment("ResNet train")
tb_res_test = cc.open_experiment("ResNet test")
# x_train: 32*32*3
train, test = get_cifar10()
x_train, t_train = train._datasets
x_test, t_test = test._datasets
# 学習データサイズ
input_size = 32
# 学習データ数
train_size = len(x_train)
# テストデータ数
test_size = len(x_test)
# エポック数
epoch_n = 400
# Set up the Crayon logging server.
if opt.exp_host != "":
from pycrayon import CrayonClient
cc = CrayonClient(hostname=opt.exp_host)
experiments = cc.get_experiment_names()
print(experiments)
'''
if opt.exp in experiments:
cc.remove_experiment(opt.exp)
experiment = cc.create_experiment(opt.exp)
'''
if opt.exp in experiments:
experiment = cc.open_experiment(opt.exp)
else:
experiment = cc.create_experiment(opt.exp)
def report_func(epoch, batch, num_batches, start_time, lr, report_stats):
"""
This is the user-defined batch-level traing progress
report function.
Args:
epoch(int): current epoch count.
batch(int): current batch count.
num_batches(int): total number of batches.
start_time(float): last report time.
lr(float): current learning rate.
