def build_validation_data_loader(self) -> DataLoader:
if self._should_evaluate_classifier():
# The BYOL paper performs validation in two steps:
# - A subset of the training data (CLS_VALIDATION) is used to select an optimal LR for the classifier.
# - Final results are reported on the test / validation data (TEST).
# We combine these two datasets, and then use a custom reducer to calculate the final result.
cls_val_dataset = build_dataset(
self.data_config,
DatasetSplit.CLS_VALIDATION,
)
test_dataset = build_dataset(self.data_config, DatasetSplit.TEST)
dataset: Dataset = JointDataset(
[cls_val_dataset, test_dataset], ["lr_val", "test"]
)
else:
# When only reporting self-supervised loss, we just use CLS_VALIDATION.
dataset = build_dataset(
self.data_config,
DatasetSplit.CLS_VALIDATION,
)
return DataLoader(
dataset,
batch_size=self.context.get_per_slot_batch_size(),
pin_memory=True,
num_workers=self.data_config.num_workers,
)
def train(model_name, model):
logdir = "logs/" + model_name + "/" + datetime.now().strftime("%Y%m%d-%H%M%S")
checkpoint_path = "model/" + model_name + "/checkpoint.ckpt"
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path, save_weights_only=True, verbose=1)
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=logdir)
train_dataset = build_dataset(TRAIN_CSV_PATH).repeat().prefetch(256).shuffle(256).batch(32)
valid_dataset = build_dataset(VALID_CSV_PATH).repeat().prefetch(100).shuffle(100).batch(16)
model.fit(train_dataset, epochs=300, steps_per_epoch=10, validation_steps=3, validation_data=valid_dataset, callbacks= [tensorboard_callback, cp_callback])
def get_data(args):
if args.setup == "ranker":
train_it, dev_it = {}, {}
for dataset in ["coco", "multi30k"]:
train_it_, dev_it_ = build_dataset(args, dataset)
train_it[dataset] = train_it_
dev_it[dataset] = dev_it_
else:
train_it, dev_it = build_dataset(args, args.dataset)
return train_it, dev_it
def random_search_param():
estimator = RandomForestClassifier()
dataset = build_dataset()
n_estimators = [
100
] #Pas besoin de l'optimiser, un n_estimator élevé produire toujours de meilleurs résultats
max_features = ["sqrt", "auto"]
max_depth = [int(x) for x in np.linspace(start=10, stop=200, num=20)]
max_depth.append(None)
min_samples_split = [
int(x) for x in np.linspace(start=2, stop=200, num=100)
]
bootstrap = ["True", "False"]
param_grid = {
'n_estimators': n_estimators,
'max_features': max_features,
'max_depth': max_depth,
'min_samples_split': min_samples_split,
'bootstrap': bootstrap
}
search = cv.RandomizedSearchCV(estimator, param_grid, n_iter=50, cv=3)
search.fit(dataset, "RandomForestRandomSearch.txt")
def main():
"""Build dataset according to args and train model"""
args = prepare_arg_parser().parse_args()
device = torch.device(args.device[0])
restricted = args.restricted
unsupervised = args.unsupervised
model_path = args.model[0]
output_path = args.output[0]
start = time.time()
print("Building dataset and adjectives")
dataset = data.build_dataset(restricted=restricted,
unsupervised=unsupervised)
data_loader = DataLoader(dataset=dataset,
batch_size=BATCH_SIZE,
shuffle=True)
print("Built dataset and adjectives in %ds" % (time.time() - start))
model, optimizer = initialize_model(model_path, device)
try:
print("Training on", device.type.upper())
training_loop(model, optimizer, data_loader)
finally:
# Always save model. This catches SIGINT kill signal.
# If stopping a model running in the background use:
# kill -s SIGINT <pid>
print("Saving model to", model_path)
torch.save(
{
"model_state_dict": model.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
},
output_path,
)
torch.cuda.empty_cache()
def evaluate(override_cfg, model_dir, continuous=True):
"""Run training and evaluation."""
tf.logging.info("model_dir = " + model_dir)
try:
cfg = _load_config(model_dir)
except tf.errors.NotFoundError:
tf.logging.info("Model directory does not exist yet. Creating new config.")
cfg = model.build_config(model_dir=model_dir, data_path=FLAGS.data_path)
tf.logging.info(cfg)
tf.logging.info(override_cfg)
cfg = utils.merge(cfg, override_cfg)
cfg.tpu.enable = False
cfg.dataset.max_length = None
# Construct inputs and estimator
_, eval_input = data.build_dataset(cfg.dataset, is_tpu=cfg.tpu.enable)
estimator = model.get_estimator(**cfg)
if continuous:
checkpoints_iterator = tf.contrib.training.checkpoints_iterator(
cfg.model_dir)
eval_metrics = None
for ckpt_path in checkpoints_iterator:
eval_metrics = estimator.evaluate(
input_fn=eval_input, checkpoint_path=ckpt_path)
tf.logging.info(pprint.pformat(eval_metrics))
return eval_metrics
else:
eval_metrics = estimator.evaluate(input_fn=eval_input)
return eval_metrics
def train_language_model(n):
"""
generating n-gram language models for n=2,3,4
n: value of n for generating n-grams, values are 2,3,4 (int)
"""
#Generating N-gram language model
for i, l in enumerate(lang):
path = os.path.join(train_path,
train_lang_path[i]) #path to train text
_, word_list = build_dataset(
path, l) #generating word list individual language
ngram = generating_ngrams(word_list, n) #generating N-gram
ngram.apply_freq_filter(
5) #filtering N-grams with frequency less that 5
ngram_model = ngram.ngram_fd.items()
ngram_model = sorted(ngram.ngram_fd.items(),
key=lambda item: item[1],
reverse=True)
print("Length of", l, "model:", len(ngram_model))
np.save(os.path.join(checkpoint_path, l + str(n) + "gram.npy"),
ngram_model)
def feature_importance():
estimator = RandomForestClassifier(max_depth=10,
min_samples_split=170,
bootstrap=True)
dataset = build_dataset()
testDate = date(2018, 1, 1)
trainSet = dataset[dataset["date"] < testDate.toordinal()]
testSet = dataset[dataset["date"] >= testDate.toordinal()]
trainX = trainSet.drop("outcome", axis=1)
trainY = trainSet["outcome"]
testX = testSet.drop("outcome", axis=1)
testY = testSet["outcome"]
estimator.fit(trainX, trainY)
scores = [x for x in estimator.feature_importances_]
featuresNames = trainX.columns
tmp = []
for i in range(len(scores)):
tmp.append((scores[i], featuresNames[i]))
tmp = sorted(tmp)
file = open("features_importance", "w")
for i in range(len(scores)):
file.write("{} = {}\n".format(tmp[i][1], tmp[i][0]))
file.close()
def grid_search_param():
estimator = RandomForestClassifier()
dataset = build_dataset()
# /!\ modifier en fonction de ce que retournera RandomSearch sinon la on est parti pour 3 ans
n_estimators = [100]
max_features = ["sqrt", "auto"]
max_depth = [int(x) for x in np.linspace(start=10, stop=50, num=5)]
min_samples_split = [
int(x) for x in np.linspace(start=120, stop=170, num=6)
]
bootstrap = ["True", "False"]
param_grid = {
'n_estimators': n_estimators,
'max_features': max_features,
'max_depth': max_depth,
'min_samples_split': min_samples_split,
'bootstrap': bootstrap
}
search = cv.GridSearchCV(estimator, param_grid, cv=5)
search.fit(dataset, "RandomForestGridSearch.txt")
def compute_score():
estimator = RandomForestClassifier(max_depth=10,
min_samples_split=170,
bootstrap=True)
dataset = build_dataset()
print(cv.cross_val_proba_score(dataset, estimator, 3))
def build_training_data_loader(self) -> DataLoader:
dataset_train = build_dataset(image_set="train", args=self.hparams)
return DataLoader(
dataset_train,
batch_size=self.context.get_per_slot_batch_size(),
collate_fn=unwrap_collate_fn,
num_workers=self.hparams.num_workers,
shuffle=True,
)
def main(args):
# Setup experiment
if not torch.cuda.is_available():
raise NotImplementedError("Training on CPU is not supported.")
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
args.experiment = args.experiment or f"{args.model.replace('_', '-')}"
args.experiment = "-".join([args.experiment])
args.experiment_dir = os.path.join(
args.output_dir, args.dataset,
args.experiment + ("-cumulative" if args.all else ""))
os.makedirs(args.experiment_dir, exist_ok=True)
tb_writer = TensorBoardLogger(save_dir=args.experiment_dir)
seed_everything(42, workers=True)
train_loaders, valid_loaders = data.build_dataset(
args.dataset,
args.data_path,
blurry=args.blurry,
batch_size=args.batch_size)
model = GCL(args, train_loaders)
trainer = Trainer(gpus=-1,
distributed_backend='ddp',
max_epochs=len(valid_loaders),
reload_dataloaders_every_epoch=True,
plugins=DDPPlugin(find_unused_parameters=False),
logger=tb_writer)
# for task_id in range(num_tasks):
# print(f"task: {task_id}")
# train_loader = DataLoader(train_loaders[task_id], batch_size=10)
metrics = MetricCollection(
[Accuracy(), F1(args.num_classes, average='macro')])
model.train_metrics = metrics.clone(prefix=f'train{model.curr_index}_')
trainer.fit(model)
trainer.train_loop.reset_train_val_dataloaders(model)
if model.curr_index:
temp = trainer.train_dataloader
labels = []
for batch in temp:
labels.append(batch[1])
print("Check if train loader if reloaded", labels)
#! TEST
test_loaders = [
DataLoader(ds, batch_size=len(ds))
for ds in valid_loaders[:model.curr_index + 1]
]
model.test_metrics = nn.ModuleList(
[metrics.clone(prefix=f'valid{i}_') for i in range(len(test_loaders))])
trainer.test(model, test_dataloaders=test_loaders)
def main(args):
np.random.seed(432)
torch.random.manual_seed(432)
try:
os.makedirs(args.outpath)
except OSError:
pass
experiment_path = utils.get_new_model_path(args.outpath)
print(experiment_path)
train_writer = SummaryWriter(os.path.join(experiment_path, 'train_logs'))
val_writer = SummaryWriter(os.path.join(experiment_path, 'val_logs'))
scheduler = cyclical_lr(5, 1e-5, 2e-3)
trainer = train.Trainer(train_writer, val_writer, scheduler=scheduler)
train_transform = data.build_preprocessing()
eval_transform = data.build_preprocessing()
trainds, evalds = data.build_dataset(args.datadir, None)
trainds.transform = train_transform
evalds.transform = eval_transform
model = models.resnet34()
base_opt = torch.optim.Adam(model.parameters())
opt = SWA(base_opt, swa_start=30, swa_freq=10)
trainloader = DataLoader(trainds,
batch_size=args.batch_size,
shuffle=True,
num_workers=8,
pin_memory=True)
evalloader = DataLoader(evalds,
batch_size=args.batch_size,
shuffle=False,
num_workers=16,
pin_memory=True)
export_path = os.path.join(experiment_path, 'last.pth')
best_lwlrap = 0
for epoch in range(args.epochs):
print('Epoch {} - lr {:.6f}'.format(epoch, scheduler(epoch)))
trainer.train_epoch(model, opt, trainloader, scheduler(epoch))
metrics = trainer.eval_epoch(model, evalloader)
print('Epoch: {} - lwlrap: {:.4f}'.format(epoch, metrics['lwlrap']))
# save best model
if metrics['lwlrap'] > best_lwlrap:
best_lwlrap = metrics['lwlrap']
torch.save(model.state_dict(), export_path)
print('Best metrics {:.4f}'.format(best_lwlrap))
opt.swap_swa_sgd()
def build_validation_data_loader(self) -> DataLoader:
dataset_val = build_dataset(image_set="val", args=self.hparams)
# Set up evaluator
self.base_ds = get_coco_api_from_dataset(dataset_val)
return DataLoader(
dataset_val,
batch_size=self.context.get_per_slot_batch_size(),
collate_fn=unwrap_collate_fn,
num_workers=self.hparams.num_workers,
shuffle=False,
)
def main(args):
np.random.seed(432)
torch.random.manual_seed(432)
try:
os.makedirs(args.outpath)
except OSError:
pass
experiment_path = utils.get_new_model_path(args.outpath)
train_writer = SummaryWriter(os.path.join(experiment_path, 'train_logs'))
val_writer = SummaryWriter(os.path.join(experiment_path, 'val_logs'))
trainer = train.Trainer(train_writer, val_writer)
# todo: add config
train_transform = data.build_preprocessing()
eval_transform = data.build_preprocessing()
trainds, evalds = data.build_dataset(args.datadir, None)
trainds.transform = train_transform
evalds.transform = eval_transform
model = models.resnet34()
opt = torch.optim.Adam(model.parameters())
trainloader = DataLoader(trainds,
batch_size=args.batch_size,
shuffle=True,
num_workers=8,
pin_memory=True)
evalloader = DataLoader(evalds,
batch_size=args.batch_size,
shuffle=False,
num_workers=16,
pin_memory=True)
for epoch in range(args.epochs):
trainer.train_epoch(model, opt, trainloader, 3e-4)
metrics = trainer.eval_epoch(model, evalloader)
state = dict(
epoch=epoch,
model_state_dict=model.state_dict(),
optimizer_state_dict=opt.state_dict(),
loss=metrics['loss'],
lwlrap=metrics['lwlrap'],
global_step=trainer.global_step,
)
export_path = os.path.join(experiment_path, 'last.pth')
torch.save(state, export_path)
def load_data(dataset,
drop,
emb_dim,
batch_size,
max_doc_len,
max_sen_len,
repeat,
split_by_period,
shuffle_train=True):
# Load data
print("Loading data...")
datasets = [
str(root / 'data' / dataset / s)
for s in ['train.ss', 'dev.ss', 'test.ss']
]
tfrecords = [
str(root / 'data' / dataset / 'tfrecords' / s)
for s in ['train.tfrecord', 'dev.tfrecord', 'test.tfrecord']
]
stats_filename = str(root / 'data' / dataset / 'stats' /
('stats.txt' + str(drop)))
embedding_filename = 'data/embedding_imdb_yelp13_elc_cd_clt.txt'
# if dataset in ['yelp13', 'imdb']:
# embedding_filename = 'data/embedding_imdb_yelp13.txt'
# elif dataset in ['cd', 'elc']:
# embedding_filename = 'data/embedding_cd_elc.txt'
print(embedding_filename)
text_filename = str(root / 'data' / dataset / 'word2vec_train.ss')
datasets, lengths, embedding, stats, wrd_dict = data.build_dataset(
datasets, tfrecords, stats_filename, embedding_filename, max_doc_len,
max_sen_len, split_by_period, emb_dim, text_filename, drop)
trainset, devset, testset = datasets
trainlen, devlen, testlen = lengths
# trainlen *= 1 - flags.drop
if repeat:
trainset = trainset.repeat()
devset = devset.repeat()
testset = testset.repeat()
if shuffle_train:
trainset = trainset.shuffle(30000)
devset = devset
testset = testset
if batch_size != 1:
trainset = trainset.batch(batch_size)
devset = devset.batch(batch_size)
testset = testset.batch(batch_size)
print("Data loaded.")
return embedding, trainset, devset, testset, trainlen, devlen, testlen, stats
def _build_cls_training_data_loader(self) -> DataLoader:
"""
Builds data loader for the on_validation_epoch_start classifier training when enabled.
"""
cls_train_dataset = build_dataset(self.data_config, DatasetSplit.CLS_TRAIN)
rounded_length = (
len(cls_train_dataset) // self.context.get_global_batch_size()
) * self.context.get_global_batch_size()
cls_train_dataset = torch.utils.data.Subset(
cls_train_dataset, range(rounded_length)
)
return DataLoader(
cls_train_dataset,
batch_size=self.context.get_per_slot_batch_size(),
pin_memory=True,
num_workers=self.data_config.num_workers,
shuffle=True,
drop_last=True,
)
def build_validation_data_loader(self) -> DataLoader:
dataset_val = build_dataset(image_set="val", args=self.hparams)
if "cat_ids" in self.hparams:
self.cat_ids = self.hparams.cat_ids
self.catIdtoCls = dataset_val.catIdtoCls
# Set up evaluator
self.base_ds = get_coco_api_from_dataset(dataset_val)
iou_types = tuple(k for k in ("segm", "bbox")
if k in self.postprocessors.keys())
self.reducer = self.context.wrap_reducer(
COCOReducer(self.base_ds, iou_types, self.cat_ids),
for_training=False,
for_validation=True,
)
return DataLoader(
dataset_val,
batch_size=self.context.get_per_slot_batch_size(),
collate_fn=unwrap_collate_fn,
num_workers=self.hparams.num_workers,
shuffle=False,
)
def build_training_data_loader(self) -> DataLoader:
if self.training_mode == TrainingMode.SELF_SUPERVISED:
split = DatasetSplit.TRAIN
elif self.training_mode == TrainingMode.CLASSIFIER_ONLY:
split = DatasetSplit.CLS_TRAIN
train_dataset = build_dataset(self.data_config, split=split)
# In order to ensure distributed training shards correctly, we round the dataset size
# down to be a multiple of the global batch size.
# See comment here for more details:
# https://github.com/determined-ai/determined/blob/b3a34baa7dcca788a090120a17f9a4f8dc1a4184/harness/determined/pytorch/samplers.py#L77
rounded_length = (
len(train_dataset) // self.context.get_global_batch_size()
) * self.context.get_global_batch_size()
train_dataset = torch.utils.data.Subset(train_dataset, range(rounded_length))
return DataLoader(
train_dataset,
batch_size=self.context.get_per_slot_batch_size(),
pin_memory=True,
num_workers=self.data_config.num_workers,
shuffle=True,
drop_last=True,
)
def train_and_eval(cfg, do_eval=True, report_fn=None):
"""Run training (and evaluation if on a GPU)."""
tf.logging.info("cfg.model_dir = " + cfg.model_dir)
# Save out config to model directory
assert FLAGS.mode == "train"
tf.gfile.MakeDirs(cfg.model_dir)
with tf.gfile.GFile(os.path.join(cfg.model_dir, "config.json"), "w") as f:
json.dump(cfg, f)
if not cfg.dataset.num_repeats and not cfg.steps_per_epoch:
raise ValueError("Must have a fixed num repeats or epoch step size.")
# Construct inputs and estimator
train_input, eval_input = data.build_dataset(
cfg.dataset, is_tpu=cfg.tpu.enable)
estimator = model.get_estimator(**cfg)
if do_eval:
eval_metrics = None
for i in range(cfg.num_epochs):
tf.logging.info("Starting epoch %s/%s" % (i + 1, cfg.num_epochs))
train_metrics = estimator.train(
input_fn=train_input, steps=cfg.steps_per_epoch or None)
tf.logging.info(pprint.pformat(train_metrics))
eval_metrics = estimator.evaluate(input_fn=eval_input)
tf.logging.info(pprint.pformat(eval_metrics))
if report_fn:
report_fn(eval_metrics)
return eval_metrics
else:
for i in range(cfg.num_epochs):
tf.logging.info("Starting epoch %s/%s" % (i + 1, cfg.num_epochs))
train_metrics = estimator.train(
input_fn=train_input, steps=cfg.steps_per_epoch)
tf.logging.info(pprint.pformat(train_metrics))
return dict()
def main():
parser = argparse.ArgumentParser(
description="Deep Neural Networks for 3D Anaglyph Image Generation")
parser.add_argument(
"--config-file",
default="",
metavar="file",
help="path to config file",
type=str,
)
parser.add_argument(
"--mode",
default="test",
metavar="mode",
help="'train' or 'test'",
type=str,
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
# build the config
cfg = get_cfg_defaults()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
dataset = build_dataset(cfg)
model, optimizer = build_model(cfg)
train(cfg, optimizer, dataset)
# print params
print("\nParameters:")
for attr, value in sorted(flags.__flags.items()):
print("{}={}".format(attr.upper(), value.value))
print("")
graph = tf.Graph()
with graph.as_default():
# Load data
print("Loading data...")
datasets = ['data/' + flags.dataset + s for s in ['/train.ss', '/dev.ss', '/test.ss']]
embeddingpath = 'data/' + flags.dataset + '/embedding.txt'
hierarchy = flags.model in ['dnsc']
datasets, lengths, embedding, usr_cnt, prd_cnt, wrd_dict = \
data.build_dataset(datasets, embeddingpath, flags.max_doc_len,
flags.max_sen_len, hierarchy)
trainset, devset, testset = datasets
trainlen, devlen, testlen = lengths
trainset = trainset.batch(flags.batch_size)
devset = devset.batch(flags.batch_size)
testset = testset.batch(flags.batch_size)
print("Loading data finished...")
# create the session
session_config = tf.ConfigProto(
allow_soft_placement=flags.allow_soft_placement,
log_device_placement=flags.log_device_placement
)
session_config.gpu_options.allow_growth = True
sess = tf.Session(config=session_config)
if flags.tfdbg == 'cli':
def main(args):
np.random.seed(432)
torch.random.manual_seed(432)
try:
os.makedirs(args.outpath)
except OSError:
pass
experiment_path = utils.get_new_model_path(args.outpath)
print(experiment_path)
train_writer = SummaryWriter(os.path.join(experiment_path, 'train_logs'))
val_writer = SummaryWriter(os.path.join(experiment_path, 'val_logs'))
trainer = train.Trainer(train_writer, val_writer)
# todo: add config
train_transform = data.build_preprocessing()
eval_transform = data.build_preprocessing()
trainds, evalds = data.build_dataset(args.datadir, None)
trainds.transform = train_transform
evalds.transform = eval_transform
model = models.resnet34()
opt = torch.optim.Adam(model.parameters(), lr=1e-8)
trainloader = DataLoader(trainds,
batch_size=args.batch_size,
shuffle=True,
num_workers=8,
pin_memory=True)
evalloader = DataLoader(evalds,
batch_size=args.batch_size,
shuffle=False,
num_workers=16,
pin_memory=True)
#find lr fast ai
criterion = torch.nn.BCEWithLogitsLoss()
lr_finder = LRFinder(model, opt, criterion, device="cuda")
# lr_finder.range_test(trainloader, val_loader=evalloader, end_lr=1, num_iter=10, step_mode="exp")
lr_finder.range_test(trainloader,
end_lr=100,
num_iter=100,
step_mode="exp")
#plot graph fast ai
skip_start = 6
skip_end = 3
lrs = lr_finder.history["lr"]
losses = lr_finder.history["loss"]
grad_norm = lr_finder.history["grad_norm"]
# ind = grad_norm.index(min(grad_norm))
# opt_lr = lrs[ind]
# print('LR with min grad_norm =', opt_lr)
lrs = lrs[skip_start:-skip_end]
losses = losses[skip_start:-skip_end]
fig = plt.figure(figsize=(12, 9))
plt.plot(lrs, losses)
plt.xscale("log")
plt.xlabel("Learning rate")
plt.ylabel("Loss")
train_writer.add_figure('loss_vs_lr', fig)
lr_finder.reset()
# fixed_lr = 1e-3
fixed_lr = 3e-4
opt = torch.optim.Adam(model.parameters(), lr=fixed_lr)
# #new
# lr = 1e-3
# eta_min = 1e-5
# t_max = 10
# opt = torch.optim.Adam(model.parameters(), lr=lr)
# scheduler = CosineAnnealingLR(opt, T_max=t_max, eta_min=eta_min)
# #new
# one cycle for 5 ehoches
# scheduler = CosineAnnealingLR(opt, 519*4, eta_min=1e-4)
scheduler = CosineAnnealingLR(opt, args.epochs)
# scheduler = CosineAnnealingLR(opt, 519, eta_min=1e-5)
# scheduler = StepLR(opt, step_size=3, gamma=0.1)
state_list = []
for epoch in range(args.epochs):
# t = epoch / args.epochs
# lr = np.exp((1 - t) * np.log(lr_begin) + t * np.log(lr_end))
# выставляем lr для всех параметров
trainer.train_epoch(model, opt, trainloader, fixed_lr, scheduler)
# trainer.train_epoch(model, opt, trainloader, 3e-4, scheduler)
# trainer.train_epoch(model, opt, trainloader, 9.0451e-4, scheduler)
metrics = trainer.eval_epoch(model, evalloader)
state = dict(
epoch=epoch,
model_state_dict=model.state_dict(),
optimizer_state_dict=opt.state_dict(),
loss=metrics['loss'],
lwlrap=metrics['lwlrap'],
global_step=trainer.global_step,
)
state_copy = copy.deepcopy(state)
state_list.append(state_copy)
export_path = os.path.join(experiment_path, 'last.pth')
torch.save(state, export_path)
# save the best path
best_export_path = os.path.join(experiment_path, 'best.pth')
max_lwlrap = 0
max_lwlrap_ind = 0
for i in range(args.epochs):
if state_list[i]['lwlrap'] > max_lwlrap:
max_lwlrap = state_list[i]['lwlrap']
max_lwlrap_ind = i
best_state = state_list[max_lwlrap_ind]
torch.save(best_state, best_export_path)
print("{}={}".format(attr.upper(), value.value), file=output_file)
print("")
output_file.close()
with tf.Graph().as_default():
# Load data
print("Loading data...")
datasets = ['data/' + flags.dataset + s for s in ['/train.ss', '/dev.ss', '/test.ss']]
tfrecords = ['data/' + flags.dataset + s for s in ['/train.tfrecord', '/dev.tfrecord', '/test.tfrecord']]
stats_filename = 'data/' + flags.dataset + '/stats.txt'
embeddingpath = 'data/' + flags.dataset + '/embedding' + str(flags.emb_dim) + '.txt'
text_filename = 'data/' + flags.dataset + '/text'
hierarchy = flags.split_by_sentence
datasets, lengths, embedding, usr_cnt, prd_cnt, wrd_dict = \
data.build_dataset(datasets, tfrecords, stats_filename, embeddingpath, flags.max_doc_len,
flags.max_sen_len, hierarchy, flags.emb_dim, text_filename)
trainset, devset, testset = datasets
trainlen, devlen, testlen = lengths
trainset = trainset.shuffle(300000).batch(flags.batch_size)
devset = devset.batch(flags.batch_size)
testset = testset.batch(flags.batch_size)
print("Loading data finished...")
# create the session
session_config = tf.ConfigProto(
allow_soft_placement=flags.allow_soft_placement,
log_device_placement=flags.log_device_placement
)
session_config.gpu_options.allow_growth = True
sess = tf.Session(config=session_config)
from model import build_model
if __name__ == '__main__':
tic = time.time()
opt, message = get_args()
print(message)
if not opt.isTest:
#random.seed(opt.seed)
#np.random.seed(opt.seed)
#torch.manual_seed(opt.seed)
#torch.cuda.manual_seed_all(opt.seed)
#torch.backends.cudnn.benchmark = True
# build the dataset according to the options
tr_loader, val_loader = build_dataset(opt)
model = build_model(opt)
model.setup()
metric = []
best_val_loss = 1000.0
for epoch in range(opt.start_epoch,
opt.epochs + opt.epochs_decay + opt.start_epoch):
epoch_start_time = time.time()
iter_data_time = time.time()
old_lr, lr = model.update_learning_rate()
tr_loss, tr_time, tr_data_time = model.train_one_epoch(tr_loader)
val_loss, val_time, val_data_time = model.test_one_epoch(
val_loader)
from model import build_model
from data import build_dataset
import os
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
# Parameters
version = 4
words = 10000
review_len = 128
vec_len = 300
patience = 5
batch_size = 15
epochs = 20
# Load data
X, y = build_dataset(review_len)
# Build model
model = build_model(words, vec_len, review_len)
# Early stopping
early_stopping_monitor = EarlyStopping(patience=patience,
monitor="loss",
mode="auto")
# Fit model
model.fit(X,
y,
epochs=epochs,
callbacks=[early_stopping_monitor],
batch_size=batch_size,
optimizer='adam',
metrics=['accuracy'])
model.summary()
return model
###
# Model estimation and evaluation
###
nb_values_total, nb_values_part = 4 * 60 * 90, 4 * 60 * 20
nb_samples = 10000
# Build dataset
dataset = build_dataset(nb_values_total, nb_values_part)
d_train_0, d_train_1, d_test_0, d_test_1 = sample(dataset, nb_samples,
nb_values_part)
X_train = np.concatenate((d_train_0[0], d_train_1[0]))
X_test = np.concatenate((d_test_0[0], d_test_1[0]))
Y_train = np.concatenate((d_train_0[1], d_train_1[1]))
Y_test = np.concatenate((d_test_0[1], d_test_1[1]))
# Model estimation
model = build_model_2(nb_values_part)
model.fit(X_train,
Y_train,
batch_size=16,
epochs=50,
validation_data=(X_test, Y_test),
import tensorflow as tf
from tensorflow.contrib import rnn
import numpy as np
import data
# Text file containing words for training
training_file = 'belling_the_cat.txt'
training_data = data.read_data(training_file)
dictionary, reverse_dictionary = data.build_dataset(training_data)
vocab_size = len(dictionary)
# Parameters
learning_rate = 0.001
training_iters = 50000
#training_iters = 5000
display_iters = 1000
n_input = 3
# number of units in RNN cell
n_hidden = 512
def model(x):
# reshape to [1, n_input]
#x = tf.reshape(x, [-1, n_input])
# Generate a n_input-element sequence of inputs
# (eg. [had] [a] [general] -> [20] [6] [33])
x2 = tf.split(x,n_input,1)
import tensorflow as tf
import pickle
from main import Summodel
from data import build_dict, build_dataset, batch_iter
with open("args.pickle", "rb") as f:
args = pickle.load(f)
print("Loading dictionary...")
word_dict, reversed_dict, article_max_len, summary_max_len = build_dict(
"valid")
print("Loading validation dataset...")
valid_x, valid_y = build_dataset("valid", word_dict, article_max_len,
summary_max_len)
valid_x_len = list(map(lambda x: len([y for y in x if y != 0]), valid_x))
with tf.Session() as sess:
print("Loading saved model...")
model = Summodel(reversed_dict,
article_max_len,
summary_max_len,
args,
Forward_only=True)
saver = tf.train.Saver(tf.global_variables())
ckpt = tf.train.get_checkpoint_state("./saved_model/")
saver.restore(sess, ckpt.model_checkpoint_path)
batches = batch_iter(valid_x, valid_y, args.batch_size, 1)
print("Writing summaries to 'train/result.txt'...")
for batch_x, batch_y in batches:
def __init__(self, cfg):
use_mixed_precision = cfg.dtype == "float16"
if use_mixed_precision:
policy = tf.keras.mixed_precision.experimental.Policy("mixed_float16")
tf.keras.mixed_precision.experimental.set_policy(policy)
self.train_dataset = build_dataset(**cfg.train.dataset.as_dict())
self.val_dataset = build_dataset(**cfg.val.dataset.as_dict())
self.model = build_model(**cfg.model.as_dict())
optimizer = build_optimizer(**cfg.optimizer.as_dict())
if cfg.lookahead:
optimizer = LookaheadOptimizer(optimizer, cfg.lookahead.steps, cfg.lookahead.alpha)
if use_mixed_precision:
optimizer = tf.keras.mixed_precision.experimental.LossScaleOptimizer(
optimizer=optimizer, loss_scale= "dynamic")
self.loss_fn = build_loss(**cfg.loss.as_dict())
self.optimizer = optimizer
self.use_mixed_precision = use_mixed_precision
self.cfg = cfg
self.total_train_steps = cfg.learning_rate_scheduler.train_steps
self.learning_rate_scheduler = build_learning_rate_scheduler(
**cfg.learning_rate_scheduler.as_dict())
self.global_step = tf.Variable(initial_value=0,
trainable=False,
name="global_step",
dtype=tf.int64)
self.learning_rate = tf.Variable(initial_value=0,
trainable=False,
name="learning_rate",
dtype=tf.float32)
self.checkpoint = tf.train.Checkpoint(optimizer=self.optimizer, model=self.model)
self.manager = tf.train.CheckpointManager(checkpoint=self.checkpoint,
directory=cfg.checkpoint_dir,
max_to_keep=10)
if os.path.exists(cfg.pretrained_weights_path):
self.model.load_weights(cfg.pretrained_weights_path, by_name=True, skip_mismatch=True)
latest_checkpoint = self.manager.latest_checkpoint
if latest_checkpoint is not None:
try:
steps = int(latest_checkpoint.split("-")[-1])
self.global_step.assign(steps)
except:
self.global_step.assign(0)
self.checkpoint.restore(latest_checkpoint)
tf.print(_time_to_string(), "Restored weights from %s." % latest_checkpoint)
else:
self.global_step.assign(0)
self.summary_writer = tf.summary.create_file_writer(logdir=cfg.summary_dir)
self.log_every_n_steps = cfg.log_every_n_steps
self.save_ckpt_steps = cfg.save_ckpt_steps
self.use_jit = tf.config.optimizer.get_jit() is not None
self.training_loss_metrics = {}
self.val_loss_metrics = {}
self.train_acc_metric = tf.keras.metrics.Accuracy()
self.train_auc_metric = tf.keras.metrics.AUC()
self.val_acc_metric = tf.keras.metrics.Accuracy()
self.val_auc_metric = tf.keras.metrics.AUC()
self._add_graph = True
