def test_init_from_file(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
foo.add_scalar_value("bar", 2, wall_time=time.time(), step=1)
filename = foo.to_zip()
new = cc.create_experiment("new", filename)
os.remove(filename)
def test_remove_experiment(self):
cc = CrayonClient(port=self.test_server_port)
self.assertRaises(ValueError, cc.open_experiment, "foo")
foo = cc.create_experiment("foo")
foo.add_scalar_value("bar", 1, step=2, wall_time=0)
self.assertRaises(ValueError, cc.create_experiment, "foo")
cc.open_experiment("foo")
cc.remove_experiment(foo.xp_name)
self.assertRaises(ValueError, cc.remove_experiment, foo.xp_name)
foo = cc.create_experiment("foo")
def setup_tensorboard(exp_id, cur_t, hostname, port):
exp_filename = '{}_{}'.format(cur_t, exp_id)
tb = CrayonClient(hostname=hostname, port=port)
try:
tb_experiment = tb.create_experiment(exp_filename)
except:
# flush the data anew
tb.remove_experiment(exp_filename)
tb_experiment = tb.create_experiment(exp_filename)
return tb_experiment, tb
def test_get_scalar_names(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
foo.add_scalar_value("fizz", 0, wall_time=0)
foo.add_scalar_value("buzz", 0, wall_time=0)
self.assertEqual(sorted(foo.get_scalar_names()),
sorted(["fizz", "buzz"]))
def test_add_scalar_dict_wrong_data(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
data = {"fizz": "foo", "buzz": 5}
self.assertRaises(ValueError, foo.add_scalar_dict, data)
data = {3: 6, "buzz": 10}
self.assertRaises(ValueError, foo.add_scalar_dict, data)
def test_get_scalar_values_two_data(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
foo.add_scalar_value("bar", 0, wall_time=0, step=0)
foo.add_scalar_value("bar", 1, wall_time=1, step=1)
self.assertEqual(foo.get_scalar_values("bar"),
[[0.0, 0, 0.0], [1.0, 1, 1.0]])
def test_add_scalar_dict(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
data = {"fizz": 3, "buzz": 5}
foo.add_scalar_dict(data, wall_time=0, step=5)
data = {"fizz": 6, "buzz": 10}
foo.add_scalar_dict(data)
class Tensorboard(Callback):
def __init__(self,
experiment_name: str,
hostname=QB_TB_HOSTNAME,
port=QB_TB_PORT):
if host_is_up(hostname, port):
from pycrayon import CrayonClient
self.client = CrayonClient(hostname=hostname, port=port)
self.experiment_name = experiment_name
try:
self.client.remove_experiment(experiment_name)
except ValueError:
pass
self.experiment = self.client.create_experiment(experiment_name)
else:
log.info(
f'Tensorboard not found on http://{hostname}:{port}, experiment logging disabled'
)
self.client = None
self.experiment_name = None
self.experiment = None
def on_epoch_end(self, logs):
if self.client is not None:
self.experiment.add_scalar_value('train_loss',
logs['train_loss'][-1])
self.experiment.add_scalar_value('train_acc',
logs['train_acc'][-1])
self.experiment.add_scalar_value('test_loss',
logs['test_loss'][-1])
self.experiment.add_scalar_value('test_acc', logs['test_acc'][-1])
self.experiment.add_scalar_value('train_time',
logs['train_time'][-1])
def init_logger(tensorboard=True, prepend_text=""):
global logger, experimentLogger
logger = logging.getLogger('heel-contour-prediction')
#log file handler
print(settings.opt)
fileHandler = logging.FileHandler(os.path.join(settings.opt['save'], prepend_text + settings.opt['description'] + '.log'))
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
fileHandler.setFormatter(formatter)
fileHandler.setLevel(logging.INFO)
logger.addHandler(fileHandler)
#output stream handler
streamHandler = logging.StreamHandler()
streamHandler.setFormatter(formatter)
streamHandler.setLevel(logging.INFO)
logger.addHandler(streamHandler)
logger.setLevel(logging.INFO)
logger.info('file handler and stream handler are ready for logging')
if(tensorboard == True):
cc = CrayonClient(hostname="localhost")
experimentLogger = cc.create_experiment(Path(settings.opt['save']).name)
# log the configuration
logger.info(settings.opt)
def test_get_histogram_values_two_data(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
data = {
"min": 0,
"max": 100,
"num": 3,
"bucket_limit": [10, 50, 30],
"bucket": [5, 45, 25]
}
foo.add_histogram_value("bar", data, wall_time=0, step=0)
data = {
"min": 0,
"max": 100,
"num": 3,
"bucket_limit": [10, 50, 30],
"bucket": [5, 45, 25]
}
foo.add_histogram_value("bar", data, wall_time=1, step=1)
self.assertEqual(foo.get_histogram_values("bar"),
[[
0.0, 0,
[
0.0, 100.0, 3.0, 0.0, 0.0, [10.0, 50.0, 30.0],
[5.0, 45.0, 25.0]
]
],
[
1.0, 1,
[
0.0, 100.0, 3.0, 0.0, 0.0,
[10.0, 50.0, 30.0], [5.0, 45.0, 25.0]
]
]])
def test_open_experiment(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
foo.add_scalar_value("bar", 1, step=2, wall_time=0)
foo = cc.open_experiment("foo")
foo.add_scalar_value("bar", 3, wall_time=1)
self.assertEqual(foo.get_scalar_values("bar"),
[[0.0, 2, 1.0], [1.0, 3, 3.0]])
class Monitor(object):
def __init__(self, address, port):
self.cc = CrayonClient(hostname=address, port=port)
def start_experiment(self, name, clean=True):
exps = self.cc.get_experiment_names()
if name in exps:
if clean:
self.cc.remove_experiment(name)
self.exp = self.cc.create_experiment(name)
print 'clean and creat a new one'
else:
self.exp = self.cc.open_experiment(name)
else:
self.exp = self.cc.create_experiment(name)
def push(self, data, wall_time=-1, step=-1):
self.exp.add_scalar_dict(data, wall_time, step)
def test_add_histogram_value_wrong_data(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
data = ["lolz", "lulz", "lelz"]
self.assertRaises(ValueError,
foo.add_histogram_value,
"bar",
data,
tobuild=True)
def tensorboard():
'''
'''
from pycrayon import CrayonClient
cc = CrayonClient(hostname=TENSORBOARD_SERVER)
try:
cc.remove_experiment(EXP_NAME)
except:
pass
foo = cc.create_experiment(EXP_NAME)
def test_get_scalar_values_auto_step(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
foo.add_scalar_value("bar", 0, wall_time=0)
foo.add_scalar_value("bar", 1, wall_time=1)
foo.add_scalar_value("bar", 2, wall_time=2, step=10)
foo.add_scalar_value("bar", 3, wall_time=3)
self.assertEqual(
foo.get_scalar_values("bar"),
[[0.0, 0, 0.0], [1.0, 1, 1.0], [2.0, 10, 2.0], [3.0, 11, 3.0]])
def test_backup(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
foo.add_scalar_value("bar", 2, wall_time=time.time(), step=1)
foo.add_scalar_value("bar", 2, wall_time=time.time(), step=2)
foo_data = foo.get_scalar_values("bar")
filename = foo.to_zip()
cc.remove_experiment("foo")
foo = cc.create_experiment("foo", zip_file=filename)
new_data = foo.get_scalar_values("bar")
self.assertEqual(foo_data, new_data)
new = cc.create_experiment("new", zip_file=filename)
new_data = new.get_scalar_values("bar")
self.assertEqual(foo_data, new_data)
os.remove(filename)
def make_crayon_experiments(experiment_name, new=True):
client = CrayonClient(hostname=config.CRAYON_SERVER_HOSTNAME)
train_experiment_name = f'{experiment_name}_train'
valid_experiment_name = f'{experiment_name}_valid'
if new:
try:
client.remove_experiment(train_experiment_name)
except ValueError:
pass
try:
client.remove_experiment(valid_experiment_name)
except ValueError:
pass
train_experiment = client.create_experiment( train_experiment_name)
train_experiment.scalar_steps['lr'] = 1
valid_experiment = client.create_experiment(valid_experiment_name)
else:
train_experiment = client.open_experiment(train_experiment_name)
valid_experiment = client.open_experiment(valid_experiment_name)
return train_experiment, valid_experiment
def test_add_histogram_value_wrong_variable(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
data = {
"min": 0,
"max": 100,
"num": 3,
"bucket_limit": [10, 50, 30],
"bucket": [5, 45, 25]
}
self.assertRaises(ValueError, foo.add_histogram_value, "", data)
def get_crayon_experiment(exp_name, hostname='127.0.0.1', overwrite=True):
cc = CrayonClient(hostname=hostname)
cc_exp = None
experiments = cc.get_experiment_names()
if exp_name in experiments:
if overwrite:
cc.remove_experiment(exp_name)
cc_exp = cc.create_experiment(exp_name)
else:
cc_exp = cc.open_experiment(exp_name)
else:
try:
cc_exp = cc.create_experiment(exp_name)
except ValueError:
cc.remove_experiment(exp_name)
cc_exp = cc.create_experiment(exp_name)
return cc_exp
def test_get_scalar_dict(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
data = {"fizz": 3, "buzz": 5}
foo.add_scalar_dict(data, wall_time=0, step=5)
data = {"fizz": 6, "buzz": 10}
foo.add_scalar_dict(data, wall_time=1)
self.assertEqual(foo.get_scalar_values("fizz"),
[[0.0, 5, 3.0], [1.0, 6, 6.0]])
self.assertEqual(foo.get_scalar_values("buzz"),
[[0.0, 5, 5.0], [1.0, 6, 10.0]])
def test_add_histogram_value(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
data = {
"min": 0,
"max": 100,
"num": 3,
"bucket_limit": [10, 50, 30],
"bucket": [5, 45, 25]
}
foo.add_histogram_value("bar", data, wall_time=0, step=0)
foo.add_histogram_value("bar", data)
def test_get_histogram_values_wrong_variable(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
data = {
"min": 0,
"max": 100,
"num": 3,
"bucket_limit": [10, 50, 30],
"bucket": [5, 45, 25]
}
foo.add_histogram_value("bar", data, wall_time=0, step=0)
self.assertRaises(ValueError, foo.get_histogram_values, "")
def test_add_histogram_value_with_sumsq(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
data = {
"min": 0,
"max": 100,
"num": 3,
"bucket_limit": [10, 50, 30],
"bucket": [5, 45, 25],
"sum_squares": 5625
}
foo.add_histogram_value("bar", data)
def crayon_create_experiment(exp_name: str, cclient: CrayonClient,
overwrite: bool = True) -> CrayonExperiment:
"""
Create experiment name in the alband/crayon tensorboard
:param exp_name: name of experiment
:param cclient: handler of requests to crayon
:param overwrite: if the experiment already exists delete and recreate
:return:
"""
try:
ccexp = cclient.create_experiment(exp_name)
return ccexp
except ValueError as verr:
if overwrite:
cclient.remove_experiment(exp_name)
ccexp = cclient.create_experiment(exp_name)
return ccexp
else:
raise verr
def test_get_histogram_names(self):
cc = CrayonClient(port=self.test_server_port)
foo = cc.create_experiment("foo")
data = {
"min": 0,
"max": 100,
"num": 3,
"bucket_limit": [10, 50, 30],
"bucket": [5, 45, 25]
}
foo.add_histogram_value("fizz", data, wall_time=0, step=0)
foo.add_histogram_value("buzz", data, wall_time=1, step=1)
self.assertEqual(sorted(foo.get_histogram_names()),
sorted(["fizz", "buzz"]))
class Tensorboard(Callback):
def __init__(self, experiment_name: str, log_dir=None):
from pycrayon import CrayonClient
self.client = CrayonClient(port=6007)
self.experiment_name = experiment_name
try:
self.client.remove_experiment(experiment_name)
except ValueError:
pass
self.experiment = self.client.create_experiment(experiment_name)
def on_epoch_end(self, logs):
self.experiment.add_scalar_value('train_loss', logs['train_loss'][-1])
self.experiment.add_scalar_value('train_acc', logs['train_acc'][-1])
self.experiment.add_scalar_value('test_loss', logs['test_loss'][-1])
self.experiment.add_scalar_value('test_acc', logs['test_acc'][-1])
self.experiment.add_scalar_value('train_time', logs['train_time'][-1])
def train_tracking_model(config):
r"""
Training function for :class:`eurus.track.pytorch.train.TrackingModel`.
Parameters
----------
config : :class:`eurus.track.pytorch.train.TrackingModelConfig`
The configuration to run the training.
"""
if config.crayon_config is not None:
crayon_config = config.crayon_config
cc = CrayonClient(hostname=crayon_config.server_address)
crayon_logger = cc.create_experiment(crayon_config.experiment_name)
else:
crayon_logger = None
dataset = create_dataset(config.dataset_config)
dataloader = DataLoader(dataset, **config.dataloader_config.configuration)
model = create_tracking_model(config.tracking_model_config)
criterion = nn.MSELoss()
if torch.cuda.is_available():
model = model.cuda()
criterion = criterion.cuda()
# TODO: Optimizer config?
optimizer = optim.SGD(model.parameters(), lr=config.lr)
for epoch in range(1, config.n_epochs + 1):
loss_meter, time_meter = training_loop(
epoch, dataloader, model, criterion, optimizer, crayon_logger)
logger.info(
'Epoch: {0:05d} completed \t'
'Average Loss: {1:4.4f} \t'
'Total time: {2:4.4f}'.format(
epoch,
loss_meter.avg,
time_meter.sum))
if crayon_logger is not None:
crayon_logger.add_scalar_value('loss_epochs', loss_meter.avg)
crayon_logger.add_scalar_value('time_epochs', time_meter.sum)
torch.save(model.state_dict(), config.weights_file)
def create_crayon_logger(exp_name, port=8889):
"""
"""
# Connect Crayon Logger (TensorBoard "wrapper") to the server
cc = CrayonClient(hostname="localhost", port=port)
tb_log_exp_name = exp_name
# Remove previous experiment
try:
cc.remove_experiment(tb_log_exp_name)
except ValueError:
# experiment doesn't already exist - nothing to be done here
print("Experiment '{}' didn't exist already (nothing to be done).".format(\
tb_log_exp_name))
# Create a new experiment
tb_log = cc.create_experiment(tb_log_exp_name)
return tb_log
def main():
args = parse_args()
ctx = mx.gpu(args.gpu)
print(args)
cc = CrayonClient(hostname='10.132.90.242')
if args.exp_name is None:
args.exp_name = datetime.now().strftime('frcnnEval_%m-%d')
try:
exp = cc.create_experiment(args.exp_name)
except:
exp = cc.open_experiment(args.exp_name)
for x in args.epoch.split(","):
mAp = test_rcnn(args.network, args.dataset, args.image_set,
args.root_path, args.dataset_path, ctx, args.prefix,
int(x), args.vis, args.shuffle, args.has_rpn,
args.proposal, args.thresh, args.use_global_context,
args.use_roi_align)
exp.add_scalar_value('mAp', mAp)
return
def init_crayon(hostname, experiment_name):
try:
from pycrayon import CrayonClient
cc = CrayonClient(hostname)
try:
Logger.exp = cc.create_experiment(experiment_name)
except ValueError as e:
print(e)
if input('Open the experiment (y/n)? ').lower() != 'y':
raise
Logger.exp = cc.open_experiment(experiment_name)
except ImportError:
print('Importing pycrayon has been failed. '
'Some features of Logger will disabled.')
except ValueError as e:
print(e)
if input('continue (y/n)? ').lower() != 'y':
raise
if len(opt.gpuid) > 1:
sys.stderr.write("Sorry, multigpu isn't supported yet, coming soon!\n")
sys.exit(1)
# Set up the Crayon logging server.
if opt.exp_host != "":
from pycrayon import CrayonClient
cc = CrayonClient(hostname=opt.exp_host)
experiments = cc.get_experiment_names()
print(experiments)
if opt.exp in experiments:
cc.remove_experiment(opt.exp)
experiment = cc.create_experiment(opt.exp)
if opt.tensorboard:
from tensorboardX import SummaryWriter
writer = SummaryWriter(
opt.tensorboard_log_dir + datetime.now().strftime("/%b-%d_%H-%M-%S"),
comment="Onmt")
progress_step = 0
def report_func(epoch, batch, num_batches,
progress_step,
start_time, lr, report_stats):
"""
This is the user-defined batch-level traing progress
if cw == stop: break
if nchars and len(res) > nchars: break
return res
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('corpus', help='Path to the corpus file.')
parser.add_argument('crayserver', help='Server location for crayon.')
parser.add_argument('expname', help='Experiment name')
args = parser.parse_args()
# Connect to the server
cc = CrayonClient(hostname=args.crayserver)
#Create a new experiment
myexp = cc.create_experiment(args.expname)
train = util.CharsCorpusReader(args.corpus, begin="<s>")
vocab = util.Vocab.from_corpus(train)
VOCAB_SIZE = vocab.size()
model = dy.ParameterCollection()
trainer = dy.SimpleSGDTrainer(model)
#lm = RNNLanguageModel(model, LAYERS, INPUT_DIM, HIDDEN_DIM, VOCAB_SIZE, builder=dy.SimpleRNNBuilder)
lm = RNNLanguageModel(model, LAYERS, INPUT_DIM, HIDDEN_DIM, VOCAB_SIZE, builder=dy.LSTMBuilder)
train = list(train)
# Sort training sentences in descending order and count minibatches
train.sort(key=lambda x: -len(x))
import copy
import os
from PIL import ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True
from fine_tuning_config_file import *
## If you want to keep a track of your network on tensorboard, set USE_TENSORBOARD TO 1 in config file.
if USE_TENSORBOARD:
from pycrayon import CrayonClient
cc = CrayonClient(hostname=TENSORBOARD_SERVER)
try:
cc.remove_experiment(EXP_NAME)
except:
pass
foo = cc.create_experiment(EXP_NAME)
## If you want to use the GPU, set GPU_MODE TO 1 in config file
use_gpu = GPU_MODE
if use_gpu:
torch.cuda.set_device(CUDA_DEVICE)
count=0
### SECTION 2 - data loading and shuffling/augmentation/normalization : all handled by torch automatically.
# This is a little hard to understand initially, so I'll explain in detail here!
# For training, the data gets transformed by undergoing augmentation and normalization.
