def train():
config = Config()
datasets = Datasets(config)
train_txts, train_labels = get_dataset(config.filename_train_txt,
config.filename_train_label,
config.filename_vocab)
valid_txts, valid_labels = get_dataset(config.filename_valid_txt,
config.filename_valid_label,
config.filename_vocab)
train_datasets = data_util.TensorDataset(train_txts, train_labels)
valid_datasets = data_util.TensorDataset(valid_txts, valid_labels)
train_datasets = data_util.DataLoader(train_datasets,
config.batch_size,
shuffle=True,
num_workers=2)
train_datasets = data_util.DataLoader(valid_datasets,
config.batch_size,
shuffle=False,
num_workers=2)
model = Model(config)
if torch.cuda.is_available():
model = model.cuda()
train(model, config, (train_datasets, valid_datasets))
def main():
config = get_args()
config.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
# preprocess
if config.preprocess_frames:
preprocess.get_frames(config.train_vid, config.train_frames)
preprocess.get_frames(config.test_vid, config.test_frames)
if config.create_csv:
train_speeds = preprocess.read_speed(config.train_speeds)
preprocess.create_csv(config.train_frames, train_speeds,
config.csv_path)
# dataset creation
dataset = FrameDataset(config.csv_path, config.train_frames)
train_set, val_set = random_split(dataset, [16320, 4080])
# test set creation
transform = transforms.Compose([
transforms.Resize((66, 220)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
test_set = datasets.ImageFolder(config.test_frames, transform=transform)
# model selection
if config.model == 'simpleCNN':
model = models.simpleCNN().to(config.device)
elif config.model == 'ResNet':
model = models.ResNet().to(config.device)
# train/val/test
if config.train:
runner.train(config, model, train_set)
elif config.val:
runner.validate(config, model, val_set)
elif config.test:
runner.test(config, model, test_set)
def sample_loss(params):
hyperparameters = params_to_hyperparams(params)
print("Parameters:", hyperparameters)
task = Task(target_pos=np.array([0., 0., 10.]))
agent = ddpg.create(task, hyperparameters=hyperparameters)
results = runner.train(task,
agent,
nb_epochs=N_EPOCHS,
nb_train_steps_per_epoch=1)
best_reward = results.max()
print("Reward:", best_reward)
return best_reward
default=default_output_root)
parser.add_argument('--name', type=str, default='baseline')
args, _ = parser.parse_known_args()
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(ColoredFormatter(to_file=False))
stream_handler.addFilter(TFlogFilter())
stream_handler.setLevel(logging.INFO)
logger.addHandler(stream_handler)
output_root = args.output_root
name = args.name
config = configuration.Configuration(name, args.work_dir, args.data_dir,
args.output_root)
config_parser = config.to_parser()
update_args, _ = config_parser.parse_known_args()
config.from_args(update_args)
output_dir = maybe_mkdir(config.output_dir)
log_path = os.path.join(output_dir, config.start_time + '.log')
file_handler = logging.FileHandler(log_path)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(ColoredFormatter(to_file=True))
file_handler.addFilter(TFlogFilter())
logger.addHandler(file_handler)
try:
config.log_params()
runner.train(config, restore=False)
except:
logger.exception('Uncaught exception:')
sys.exit(1)
args = parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = args.cuda_id
config = {
LSTM_HID: 256,
NUM_LAYERS: args.num_layers_tf,
NUM_HEADS: args.num_heads_tf,
DROPOUT: args.dropout,
LR: args.lr,
BATCH_SIZE: args.batch_size,
DEP_BEAM_SIZE: args.dep_beam_size,
PTR_BEAM_SIZE: args.ptr_beam_size,
MODEL_TYPE: args.model_type, ###DEP_MODEL,POINTER_MODEL, DEP_BASELINE, LM_BASELINE###
DATASET_TYPE: args.dataset_type, # 1 is 100k, 2 is 250k
NUM_EDUS_BOUND: args.num_edus_bound,
WARMUP_STEPS: args.warmup_steps,
}
config[DEVICE] = th.cuda.current_device()
#config[DEVICE] = 'cpu'
config[EVAL] = args.eval
print("WARNING: THEY WILL CHANGE .TRANSPOSE() IN DGL TO HAVE OPPOSITE SEMANTICS SOON")
if args.prepare:
data_helper = DataHelper()
data_helper.create_data_helper(data_dir="../../data")
elif args.train:
train(config)
elif args.eval:
config[EVAL_LONG_DOCS] = args.eval_long_docs
evaluate(config, 0)
def test_train_v2(test_conf_v2):
conf = test_conf_v2
runner.train(conf)
assert True
def test_train_v2(test_conf_v2):
conf = test_conf_v2
runner.train(conf)
assert True
import numpy as np
from IPython import embed
sys.path.append('lib')
import corpus
import config
import runner
import mark
conf = config.parse_args()
corpus = conf.open_corpus()
if conf.mode() == 'console':
embed()
elif conf.mode() == 'train':
train_scores, test_scores = runner.train(conf)
runner.report_bleu_graph(train_scores, test_scores)
elif conf.mode() == 'restore_console':
encdec, opt, conf = runner.load(conf.load_prefix())
embed()
# usage: ---------------------------------------------------------------
# source = "this is a pen."
# batch, hyp = runner.predict(conf, encdec, source)
# x = batch.data_at(0)
# t, y = hyp[0]
# mark.decoded_vec_to_str(y)
#
# In [24]: corpus.tokenize(source, cleanup_tag=False)
# Out[24]: [u'<bos>', u'this', u'is', u'a', u'pen', u'.', u'<eos>']
#
# In [20]: corpus.ids_to_tokens(x)
use_distance_as_feature=args.use_distance_as_feature,
num_classes=2,
c=args.c
).double()
criterion = nn.CrossEntropyLoss()
if not args.sgd:
optimizer = geoopt.optim.RiemannianAdam(
model.parameters(),
lr=args.lr,
betas=(float(adam_betas[0]), float(adam_betas[1])),
stabilize=10,
weight_decay=args.wd
)
else:
optimizer = geoopt.optim.RiemannianSGD(
model.parameters(), args.lr, stabilize=10,
weight_decay=args.wd)
runner = runner.CustomRunner()
runner.train(
model=model,
criterion=criterion,
optimizer=optimizer,
loaders={"train": loader_train, "valid": loader_test},
callbacks=[PrecisionCallback(precision_args=[1])],
logdir=logdir,
n_epochs=n_epochs,
verbose=args.verbose,
)
import numpy as np
from IPython import embed
sys.path.append('lib')
import corpus
import config
import runner
import mark
conf = config.parse_args()
corpus = conf.open_corpus()
if conf.mode() == 'console':
embed()
elif conf.mode() == 'train':
train_scores, test_scores = runner.train(conf)
runner.report_bleu_graph(train_scores, test_scores)
elif conf.mode() == 'restore_console':
encdec, opt, conf = runner.load(conf.load_prefix())
embed()
# usage: ---------------------------------------------------------------
# source = "this is a pen."
# batch, hyp = runner.predict(conf, encdec, source)
# x = batch.data_at(0)
# t, y = hyp[0]
# mark.decoded_vec_to_str(y)
#
# In [24]: corpus.tokenize(source, cleanup_tag=False)
# Out[24]: [u'<bos>', u'this', u'is', u'a', u'pen', u'.', u'<eos>']
#
# In [20]: corpus.ids_to_tokens(x)
config.batch_size,
shuffle=False,
num_workers=2)
model = Model(config)
if torch.cuda.is_available():
model = model.cuda()
train(model, config, (train_datasets, valid_datasets))
def test():
config = Config()
datasets = Datasets(config)
test_txts, test_labels = get_dataset(config.filename_test_txt,
config.filename_test_label,
config.filename_vocab)
test_datasets = data_util.TensorDataset(test_txts, test_labels)
test_datasets = data_util.DataLoader(test_datasets,
config.batch_size,
shuffle=True,
num_workers=2)
model = Model(config)
model.load_state_dict(torch.load(filename, map_location='gpu'))
valid(model, config, test_datasets)
if __name__ == "__main__":
train()
