def main(cfg):
torch.multiprocessing.set_sharing_strategy('file_system')
seed_torch(seed=cfg.seed)
output_dir = os.path.join(cfg.output_dir, cfg.desc)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
train_dataset = build_dataset(cfg, phase='train')
test_dataset = build_dataset(cfg, phase='test')
if cfg.DATA.weighted_sample:
train_dl = DataLoader(train_dataset,
batch_size=32,
sampler=WeightedRandomSampler(
train_dataset.get_label_weight(),
num_samples=5000),
num_workers=0,
drop_last=True)
else:
train_dl = DataLoader(train_dataset,
batch_size=32,
shuffle=True,
num_workers=16,
drop_last=True)
test_dl = DataLoader(test_dataset,
batch_size=32,
num_workers=8,
drop_last=True)
solver = Solver(cfg)
solver.train(train_dl, test_dl)
def main():
args = parse_args()
log(args, '\n' + str(args))
if args.mode == 'train':
comparision.train(args)
elif args.mode == 'classify':
classify(args)
elif args.mode == 'dataset':
dataset.build_dataset(args)
elif args.mode == 'run':
run_thread = run.MyThread(0, args)
network_thread = run.MyThread(1, args)
run_thread.start()
network_thread.start()
while network_thread.is_alive():
continue
elif args.mode == 'normalize':
start_time = time.clock()
if args.split_dir != 'none':
splits = dataset.split_images(args, start_time)
log(
args, '{:.5f}'.format(time.clock() - start_time) + 's ' +
'Images have been split ')
if args.normalize:
dataset.normalize(args, start_time)
if args.split_dir != 'none':
dataset.normalize(args, start_time, dirs=splits)
else:
log(args,
'Please select a mode using the tag --mode, use --help for help.',
True)
def main(argv):
del argv # Unused.
if FLAGS.eager_run:
tf.config.experimental_run_functions_eagerly(True)
model_type = GraphTransformer
data_spec = json.load(open(FLAGS.data_spec))
if FLAGS.predict:
src_vocab = Vocabulary.load(data_spec["source_vocab"])
tgt_vocab = Vocabulary.load(data_spec["target_vocab"])
eval_set = build_dataset(
os.path.join(FLAGS.predict),
max_num_src_tokens=data_spec["max_num_src_tokens"],
max_num_tgt_tokens=data_spec["max_num_tgt_tokens"],
src_vocab=src_vocab,
tgt_vocab=tgt_vocab,
is_train=False,
predict_edge=True,
multiple=True)
eval_set = eval_set.batch(FLAGS.batch_size, drop_remainder=False)
predict(model_type, eval_set, src_vocab,
tgt_vocab, FLAGS.save_model_path,
json.load(open(FLAGS.ref_derivs)), FLAGS.predict_output)
else:
hparams = {
"batch_sz": FLAGS.batch_size,
"d_model": FLAGS.model_dim,
"biaffine": FLAGS.biaffine,
"num_src_tokens": data_spec["max_num_src_tokens"],
"num_tgt_tokens": data_spec["max_num_tgt_tokens"],
"dropout": FLAGS.dropout,
"predict_edge": True
}
src_vocab = Vocabulary.load(data_spec["source_vocab"])
tgt_vocab = Vocabulary.load(data_spec["target_vocab"])
train_set = build_dataset(data_spec["train_set"],
max_num_src_tokens=hparams["num_src_tokens"],
max_num_tgt_tokens=hparams["num_tgt_tokens"],
src_vocab=src_vocab,
tgt_vocab=tgt_vocab,
is_train=True,
predict_edge=True,
multiple=True)
train_set = train_set.batch(FLAGS.batch_size, drop_remainder=True)
dev_set = build_dataset(data_spec["dev_set"],
max_num_src_tokens=hparams["num_src_tokens"],
max_num_tgt_tokens=hparams["num_tgt_tokens"],
src_vocab=src_vocab,
tgt_vocab=tgt_vocab,
is_train=False,
predict_edge=True,
multiple=True)
dev_set = dev_set.batch(FLAGS.batch_size, drop_remainder=False)
train(model_type, FLAGS.epochs, train_set, dev_set, src_vocab,
tgt_vocab, hparams, FLAGS.save_model_path)
def main(args):
if flags.FLAGS.prepare:
directories = list_directories(flags.FLAGS.prepare)
dataset = os.path.join(flags.FLAGS.prepare, 'combined')
if not os.path.exists(dataset):
os.makedirs(dataset)
build_dataset(directories, dataset)
elif flags.FLAGS.test:
test(flags.FLAGS.dataset)
else:
model_fn = get_model_fn(flags.FLAGS.model)
train(flags.FLAGS.dataset, model_fn)
def build_astro_sets(batch_size):
files = 'data/train1, data/train2, data/train3, data/train4, data/train5, data/train6, data/train7, data/train8'
train = build_dataset(files,
batch_size,
reverse_time_series_prob=0.5,
shuffle_filenames=True)
val = build_dataset('data/val',
batch_size,
reverse_time_series_prob=0.5,
shuffle_filenames=True)
test = build_dataset('data/test', 1)
return train, val, test
def train_model(option):
train_loader = DataLoader(dataset=build_dataset('./data/train_x.pkl',
'./data/train_y.pkl'),
batch_size=option.batch_size,
shuffle=True)
test_loader = DataLoader(dataset=build_dataset('./data/test_x.pkl',
'./data/test_y.pkl'),
batch_size=option.batch_size,
shuffle=True)
model = biLSTM_CRF(option.embedding_size, option.hidden_size,
option.dict_number, option.num_labels)
optimizer = torch.optim.Adam(model.parameters(), lr=option.lr)
if option.use_gpu:
model.cuda()
if option.pre_trained:
model.load_state_dict(torch.load(option.pre_trained))
for epoch in range(option.epochs):
print(epoch)
# 创建评价
train_eva = evaluation(option.label_dict)
test_eva = evaluation(option.label_dict)
# 训练
model.train()
for step, (batch_x, batch_y, batch_masks) in enumerate(train_loader):
optimizer.zero_grad()
if option.use_gpu:
batch_x = batch_x.cuda()
batch_y = batch_y.cuda()
masks = masks.cuda()
y_pred, loss = model(batch_x, batch_y, batch_masks)
train_eva.add(y_pred, batch_y)
loss.backward()
optimizer.step()
# 通过测试集验证
model.eval()
for step, (batch_x, batch_y) in enumerate(test_loader):
if option.use_gpu:
batch_x = batch_x.cuda()
batch_y = batch_y.cuda()
masks = masks.cuda()
y_pred, loss = model(batch_x, batch_y, batch_masks)
test_eva.add(y_pred, batch_y)
print("train:")
print(train_eva.evaluate())
print("test:")
print(test_eva.evaluate())
def build(hidden_size, batch_size, max_len, cuda):
bidirectional = False
model_name = 'bert'
x = import_module('models.' + model_name)
config = x.Config(batch_size)
train_data = build_dataset(config)
train_dataloader = build_iterator(train_data, config)
val_data, test_data = build_dataset_eval(config)
val_dataloader = build_iterator_eval(val_data, config)
test_dataloader = build_iterator_eval(test_data, config)
encoder = x.Model(config).to(config.device)
decoder = DecoderRNN(len(config.tokenizer.vocab),
max_len,
hidden_size * 2 if bidirectional else hidden_size,
dropout_p=0.2,
use_attention=True,
bidirectional=bidirectional,
eos_id=config.tokenizer.convert_tokens_to_ids([SEP
])[0],
sos_id=config.tokenizer.convert_tokens_to_ids([CLS
])[0])
seq2seq = Seq2seq(encoder, decoder)
if cuda:
seq2seq.cuda()
optimizer = torch.optim.Adam(lr=1e-3, params=seq2seq.parameters())
Tensor = torch.cuda.LongTensor if cuda else torch.LongTensor
if cuda:
seq2seq.cuda()
Tensor = torch.cuda.LongTensor if cuda else torch.LongTensor
loss_fun = torch.nn.NLLLoss(reduce=False)
return seq2seq, optimizer, Tensor, train_dataloader, val_dataloader, test_dataloader, loss_fun, config
def __init__(self):
self._flags = tf.flags.FLAGS
tf.logging.warn('tensorflow flags:')
for key, value in sorted(self._flags.flag_values_dict().items()):
tf.logging.warn('{:25}: {}'.format(key, value))
self.dataset_args = {
'duration_thresholds': self._flags.duration_thresholds,
'weight_key': self._flags.weight_key,
'debug_labels': self._flags.debug_labels,
'use_sequence_weight': self._flags.use_sequence_weight,
'label_dtype': self._flags.label_dtype,
'mode': self._flags.mode,
'label_num': self._flags.label_num,
'multi_task_config': self._flags.multi_task_config,
'negative_rate': self._flags.negative_rate
}
self.Dataset = build_dataset(self._flags.dataset)
self._dataset = self.Dataset(self._flags.compression_type,
self._flags.label_key,
self._flags.schema,
self._flags.valid_path,
self._flags.train_path,
batch_size=self._flags.batch_size,
prebatch=self._flags.prebatch,
epochs=self._flags.epochs,
seq_train=self._flags.seq_train,
**self.dataset_args)
self._Network = network_factory(self._flags.network)
self._features = self._serving_features()
self._build_estimator()
self._assets = self._serving_warm_up()
self._output_cols = list()
def input_fn(params):
ds = build_dataset(params['data_loader'],
batch_size=params['batch_size'],
shuffle=256).prefetch(1)
features, labels = ds.make_one_shot_iterator().get_next()
return features, labels
def _build_dataset(self):
dataset = build_dataset(self._flags.dataset)
if dataset != WeightedDataset:
raise ValueError(
'Weighted LR model should be run with WeightedDataset, find {}'
.format(dataset.__name__))
return dataset
def get(name, args=None):
if name.startswith("physics"):
physics_map = {
"physics_human": "data/human_ann.json",
"physics": "data/ann.json"
}
return dict(
factory="COCODataset",
args=dict(root="", ann_file=physics_map[name]),
)
if name.startswith("annotated_physics"):
physics_map = {
"annotated_physics_human": "data/annotated_human_ann.json",
"annotated_physics_train": "data/annotated_ann.json",
"annotated_physics_val": "data/annotated_ann.json"
}
return build_dataset(
EasyDict(
NAME="OBJECT_PROPOSAL",
OBJECT_DERENDER_FILE=physics_map[name],
SPLIT="TRAIN"
if "train" in name else "VAL" if "val" in name else "TEST",
ROOT=DatasetCatalog.TRAIN_ROOT
if "human" not in name else DatasetCatalog.HUMAN_ROOT,
), args)
def main(_):
flags_dict = FLAGS.flag_values_dict()
train_mode = flags_dict['train_mode']
print(train_mode)
if train_mode is not TrainMode.FL:
train_dataset = build_dataset(config.TRAIN_DATA_PATH,
positive_only=True).shuffle(3200).batch(
config.BATCH_SIZE)
test_dataset = build_dataset(config.TEST_DATA_PATH,
positive_only=True).shuffle(3200).batch(
config.BATCH_SIZE)
else:
train_dataset = build_dataset(config.FL_TRAIN_DATA_PATH,
positive_only=False).shuffle(3200).batch(
config.BATCH_SIZE)
test_dataset = build_dataset(config.FL_TEST_DATA_PATH,
positive_only=False).shuffle(3200).batch(
config.BATCH_SIZE)
if train_mode in [TrainMode.GAN, TrainMode.ALL]:
print(
"************************* Training GAN ***************************"
)
gan_train(train_dataset, config.EPOCHS)
elif train_mode in [TrainMode.AE, TrainMode.AAE, TrainMode.ALL]:
print(
"************************* Training AE ***************************"
)
ae_train(train_dataset, test_dataset, config.EPOCHS, train_mode)
elif train_mode == TrainMode.VAEGAN:
print(
"************************* Training VAE-GAN ***************************"
)
# vaegan_train(train_dataset, test_dataset, pretrained_path='/data2/huangps/data_freakie/gx/code/elevator_GAN_FH/vaegan_training_checkpoints/ckpt_epoch_60-60')
vaegan_train(train_dataset, test_dataset)
elif train_mode == TrainMode.FL:
print(
"************************* Training VAE-GAN ***************************"
)
classifier_train(train_dataset, test_dataset)
def main(config):
run_logger = None
if config['azure_ml']:
run_logger = init_azure_logging(config)
if config['tall_kernel']:
model = tall_kernel(config)
else:
model = cnn(config)
with tf.device('/cpu:0'): # put data pipeline on CPU
ds_train = build_dataset(config, 'train')
ds_val = build_dataset(config, 'val')
loss = tf.losses.SparseCategoricalCrossentropy(from_logits=True)
optimizer = get_optimizer(config)
model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
add_logpaths(config)
save_state(config)
callbacks = build_callbacks(config, run_logger)
model.fit(x=ds_train,
validation_data=ds_val,
epochs=config['epochs'],
callbacks=callbacks,
validation_steps=90,
verbose=config['verbose'])
def main(config):
genc = build_genc()
writer_config = build_writer_config(config)
reader_config = build_dataset_reader_config(config)
for mode in ['train', 'val', 'test']:
writer = MyTFRWriter(writer_config, mode)
ds = dataset.build_dataset(reader_config, mode)
for batch in ds:
x, y = batch
x = genc(x) # shape = (batch_sz, 63, dim_z)
for idx in range(x.shape[0]):
example = serialized_example(x[idx, ...], y[idx])
writer.write(example)
writer.close()
def train_input_fn(params):
levels = build_levels()
ds = dataset.build_dataset(params['data_loader'],
levels=levels,
scale=params['scale'],
shuffle=4096,
augment=True)
ds = ds.map(lambda input: {
**input, 'image': preprocess_image(input['image'])
})
return ds.prefetch(1)
def get_loader(args):
train_transform = transforms.Compose([
transforms.RandomResizedCrop(args.cropsize, scale=(args.crop, 1.)),
transforms.RandomApply([transforms.ColorJitter(0.4, 0.4, 0.4, 0.1)], p=0.8),
transforms.RandomGrayscale(p=0.2),
transforms.RandomApply([GaussianBlur([.1, 2.])], p=0.5),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
train_dataset = build_dataset(args.dataset, args.data_dir, transform=train_transform, split = args.datasplit)
train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=args.batch_size, shuffle=False,
num_workers=args.num_workers, pin_memory=True,
sampler=train_sampler, drop_last=True)
return train_loader
def train():
parser = argparse.ArgumentParser()
train_dataset = '../data/corpus.small'
vocab_dataset = '../data/vocab.small'
output = '../output/bert.model'
model = 'bert'
parser.add_argument('--model', type=str, required=False, default=model)
parser.add_argument("-c", "--train_dataset", required=False, type=str, default=train_dataset,
help="train dataset for train bert")
parser.add_argument("-t", "--test_dataset", type=str, default=None, help="test set for evaluate train set")
parser.add_argument("-v", "--vocab_path", required=False, type=str, default=vocab_dataset,
help="built vocab model path with bert-vocab")
parser.add_argument("-o", "--output_path", required=False, type=str, default=output, help="exoutput/bert.model")
parser.add_argument("-b", "--batch_size", type=int, default=64, help="number of batch_size")
parser.add_argument("-e", "--epochs", type=int, default=10, help="number of epochs")
parser.add_argument("-w", "--num_workers", type=int, default=5, help="dataloader worker size")
parser.add_argument("--require_improvement", type=int, default=1000, help="patience of early stopping")
parser.add_argument("--with_cuda", type=bool, default=True, help="training with CUDA: true, or false")
parser.add_argument("--log_freq", type=int, default=10, help="printing loss every n iter: setting n")
parser.add_argument("--corpus_lines", type=int, default=None, help="total number of lines in corpus")
parser.add_argument("--cuda_devices", type=int, nargs='+', default=0, help="CUDA device ids")
parser.add_argument("--on_memory", type=bool, default=True, help="Loading on memory: true or false")
parser.add_argument("--lr", type=float, default=1e-3, help="learning rate of adam")
args = parser.parse_args()
model_name = args.model
module = import_module('models.' + model_name)
config = module.ModelConfig(args)
train_data, dev_data, test_data = build_dataset(config)
train_iter = build_iterator(train_data, config)
dev_iter = build_iterator(dev_data, config)
test_iter = build_iterator(test_data, config)
model = module.Model(config).to(config.device)
model_train(config, model, train_iter, dev_iter, test_iter)
test(config, model, test_iter)
def evaluate(model, word2id, id2word, step, path_submission):
# Create Dataset
ds_test = build_dataset(PATH_TEST, word2id)
ds_test = ds_test.batch(BATCH_SIZE)
# ds_test = ds_test.take(2) # uncomment for demo purposes
# Run Validation
accuracy, perp = validate(model=model,
dataset=ds_test,
id2word=id2word,
step=step)
# Write Submission File
perp = tf.concat(perp, axis=0)
print(f"Writing Submission: {path_submission}")
pd.DataFrame(perp.numpy()).to_csv(path_submission,
sep=' ',
header=False,
index=False,
float_format='%.3f')
def do_training(args):
trainloader, testloader = build_dataset(
args.dataset,
dataroot=args.dataroot,
batch_size=args.batch_size,
eval_batch_size=args.eval_batch_size,
num_workers=2)
model = build_model(args.arch, num_classes=num_classes(args.dataset))
model = torch.nn.DataParallel(model).cuda()
# Calculate total number of model parameters
num_params = sum(p.numel() for p in model.parameters())
track.metric(iteration=0, num_params=num_params)
optimizer = build_optimizer('SGD', params=model.parameters(), lr=args.lr)
criterion = torch.nn.CrossEntropyLoss()
best_acc = 0.0
for epoch in range(args.epochs):
track.debug("Starting epoch %d" % epoch)
args.lr = adjust_learning_rate(epoch, optimizer, args.lr,
args.schedule, args.gamma)
train_loss, train_acc = train(trainloader, model, criterion, optimizer,
epoch)
test_loss, test_acc = test(testloader, model, criterion, epoch)
track.debug(
'Finished epoch %d... | train loss %.3f | train acc %.3f | test loss %.3f | test acc %.3f'
% (epoch, train_loss, train_acc, test_loss, test_acc))
# Save model
model_fname = os.path.join(track.trial_dir(),
"model{}.ckpt".format(epoch))
torch.save(model, model_fname)
if test_acc > best_acc:
best_acc = test_acc
best_fname = os.path.join(track.trial_dir(), "best.ckpt")
track.debug("New best score! Saving model")
torch.save(model, best_fname)
def get_loader(args):
val_transform_list = []
if args.resize > 0:
val_transform_list.append(transforms.Resize(args.resize))
val_transform_list.append(transforms.CenterCrop(args.cropsize))
val_transform_list.append(transforms.ToTensor())
val_transform_list.append(transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]))
val_transform = transforms.Compose(val_transform_list)
clipdataset = build_dataset(
dataset=args.dataset,
root=args.data_dir,
split=args.datasplit,
transform=val_transform,
sample_num=args.datasamplenum,
)
dataloader = DataLoader(clipdataset,
batch_size=1,
num_workers=8,
sampler=DistributedSampler(clipdataset, shuffle=False),
shuffle=False,
pin_memory=True,
drop_last=False)
return dataloader, len(clipdataset)
def main(argv):
if FLAGS.eager_run:
tf.config.experimental_run_functions_eagerly(True)
model_type = Transformer
data_spec = json.load(open(FLAGS.data_spec))
src_vocab = Vocabulary.load(data_spec["source_vocab"])
tgt_vocab = Vocabulary.load(data_spec["target_vocab"])
eval_set = build_dataset(os.path.join(FLAGS.predict),
max_num_src_tokens=data_spec["max_num_src_tokens"],
max_num_tgt_tokens=data_spec["max_num_tgt_tokens"],
src_vocab=src_vocab,
tgt_vocab=tgt_vocab,
is_train=False)
eval_set = eval_set.batch(FLAGS.batch_size, drop_remainder=False)
get_top_beam_graphs(model_type,
eval_set,
src_vocab,
tgt_vocab,
FLAGS.save_model_path,
FLAGS.output_path,
graph_size=FLAGS.graph_size)
)
# If the restored model is ready for inference, save it and quit training.
if args.export_only:
if latest_checkpoint is None:
print("Warning: Model not restored from any checkpoint.")
print("Saving model to {} ...".format(export_dir))
model.save(export_dir)
print("Model saved at: {}".format(export_dir))
quit()
# Construct a dataset for evaluation.
dataset_test = build_dataset(test_files_dir,
"test",
number_marks=number_marks,
image_shape=input_shape,
training=False,
batch_size=args.batch_size,
shuffle=False,
prefetch=tf.data.experimental.AUTOTUNE)
# If only evaluation is required.
if args.eval_only:
model.evaluate(dataset_test)
quit()
# Finally, it's time to train the model.
# Compile the model and print the model summary.
model.compile(optimizer=keras.optimizers.Adam(0.0001),
loss=keras.losses.MeanSquaredError(),
metrics=[keras.metrics.MeanSquaredError()])
import numpy as np
from six.moves import urllib
import six.moves.urllib
from six.moves import xrange # pylint: disable=redefined-builtin
import tensorflow as tf
# Read the data into a list of strings.
words = ds.get_words()
print('Data size', len(words))
vocabulary_size = 50000
data, count, dictionary, reverse_dictionary = ds.build_dataset(words, vocabulary_size)
del words # Hint to reduce memory.
print('Most common words (+UNK)', count[:5])
print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])
data_index = 0
# Step 3: Function to generate a training batch for the skip-gram model.
def generate_batch(batch_size, num_skips, skip_window):
global data_index
assert batch_size % num_skips == 0
assert num_skips <= 2 * skip_window
batch = np.ndarray(shape=(batch_size), dtype=np.int32)
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
span = 2 * skip_window + 1 # [ skip_window target skip_window ]
def train_single_gpu(config, args):
#setup logger
str_t = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
train_dir = f'./res/{str_t}'
config['train']['train_dir'] = train_dir
os.makedirs(train_dir)
os.makedirs(train_dir + '/result')
os.makedirs(train_dir + '/model')
fh = logging.FileHandler(f'{train_dir}/train.log')
fh.setFormatter(logging_formatter)
logger.addHandler(fh)
logger.info(json.dumps(config, indent=4))
#train and validate files
batch_size = config['train']['batch_size']
input_config = config['input']
train_files = glob.glob(input_config['train'])
validate_files = glob.glob(input_config['validate'])
test_files = glob.glob(input_config['test'])
train_dataset = build_dataset(train_files, batch_size)
validate_dataset = build_dataset(validate_files, batch_size)
test_dataset = build_dataset(test_files, batch_size)
#metric
metric_train_loss = tf.keras.metrics.Mean(name='train_loss')
#model
model = ModelAttention(config['model'])
if args.init_model is not None:
x = tf.ones([batch_size, window_size, input_feature_dim],
dtype=tf.float32)
ref_aa = tf.ones((batch_size, ), dtype=tf.int32)
alt_aa = tf.ones((batch_size, ), dtype=tf.int32)
mask = tf.zeros((batch_size, window_size), dtype=tf.float32)
model((ref_aa, alt_aa, x), training=False, mask=mask)
model.load_weights(args.init_model)
#set other layers untrainable
for layer in model.layers:
if layer.name == 'gru_cell':
layer.trainable = True
else:
layer.trainable = True
print(layer.name, layer.trainable)
#double check
print('trainable variables')
for var in model.trainable_variables:
print(var.name)
#learning rate
init_learning_rate = config['train']['learning_rate']
end_learning_rate = config['train']['end_learning_rate']
warmup_steps, decay_steps = config['train']['warmup_steps'], config[
'train']['decay_steps']
learning_rate = LearningRate(init_learning_rate,
end_learning_rate=end_learning_rate,
warmup_steps=warmup_steps,
decay_steps=decay_steps)
#training algorithm
opt = config['train'].get('opt', 'adam')
if opt == 'adam':
optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
else:
raise NotImplementedError(f"opt {opt} not NotImplementedError")
def _save_res(var_id, target, pred, name, epoch):
with open(f'{train_dir}/result/epoch_{epoch}_{name}.score', 'w') as f:
f.write('var\ttarget\tScore\n')
for a, c, d in zip(var_id, target, pred):
f.write('{}\t{:d}\t{:f}\n'.format(a.numpy().decode('utf-8'),
int(c), d))
return True
@tf.function(input_signature=[train_dataset.element_spec])
def test_step(sample):
var, ref_aa, alt_aa, feature, label, padding_mask = sample
logit = model((ref_aa, alt_aa, feature), False, padding_mask)
loss = compute_loss(label, logit)
pred = model.predict_from_logit(logit)
return var, label, pred, loss
def test(test_dataset,
data_name,
epoch,
auc=False,
pvalue=False,
corr=False):
#metrics
metric_test = TestMetric()
metric_test_loss = tf.keras.metrics.Mean(name='test_loss')
metric_test.reset_states()
metric_test_loss.reset_states()
all_pred, all_label, all_var = [], [], []
for step, sample in enumerate(test_dataset):
var, label, pred, loss = test_step(sample)
metric_test.update_state(label, pred)
metric_test_loss.update_state(loss)
all_pred.extend(list(pred))
all_label.extend(list(label))
all_var.extend(list(var))
all_var = np.array(all_var)
all_label = np.array(all_label)
all_pred = np.array(all_pred)
_save_res(all_var, all_label, all_pred, data_name, epoch)
if auc:
logger.info(
f'{data_name} pos= {metric_test.result_pos()} neg= {metric_test.result_neg()} loss= {metric_test_loss.result()} auPR= {metric_test.result_auPR()} auROC= {metric_test.result_auROC()} max= {metric_test.result_max()}'
)
if pvalue:
logger.info(
f'{data_name} pos= {metric_test.result_pos()} neg= {metric_test.result_neg()} loss= {metric_test_loss.result()} pvalue= {metric_test.result_pvalue()}'
)
if corr:
corr, pvalue = metric_test.result_corr()
logger.info(
f'{data_name} pos= {metric_test.result_total()} corr= {corr} pvalue= {pvalue} max= {metric_test.result_max()}'
)
return metric_test.result_auROC()
@tf.function(input_signature=[train_dataset.element_spec])
def train_step(sample):
var, ref_aa, alt_aa, feature, label, padding_mask = sample
with tf.GradientTape() as tape:
logit = model((ref_aa, alt_aa, feature), True, padding_mask)
loss = compute_loss(label, logit)
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
metric_train_loss.update_state(loss)
#if optimizer.iterations % 512 == 0:
# _update_gradient_norm_summary(model.trainable_variables, gradients)
return loss
print('init model')
#test(test_dataset, 'test', 0, pvalue=False, auc=True, corr=False)
EPOCHS = 512
watch_auROC = -1.0
watch_epoch = -1
patience_epochs = 5
for epoch in range(EPOCHS):
start = time.time()
for step, samples in enumerate(train_dataset):
loss = train_step(samples)
#model summary
if optimizer.iterations == 1:
model.summary(print_fn=logger.info)
logger.info(f'Epoch {epoch} Loss {metric_train_loss.result():.4f}')
metric_train_loss.reset_states()
model.save_weights(f'{train_dir}/model/epoch-{epoch}.h5')
#validate and test
validate_auROC = test(validate_dataset,
'validate',
epoch,
pvalue=False,
auc=True,
corr=False)
if validate_auROC > watch_auROC:
watch_auROC = validate_auROC
watch_epoch = epoch
if epoch - watch_epoch == patience_epochs:
logger.info(
f'best_epoch {watch_epoch} best_validate_auROC= {watch_auROC}')
break
import dataset as ds
ds.build_dataset('./data/images/numbers/train', 'jpg',
'./npz/service_numbers_train.npz')
ds.plot_dataset('./data/service_numbers_train.npz')
ds.build_dataset('./data/images/numbers/test', 'jpg',
'./npz/service_numbers_test.npz')
ds.plot_dataset('./data/service_numbers_test.npz')
ds.build_dataset('./data/images/letters/train', 'jpg',
'./npz/service_letters_train.npz')
ds.plot_dataset('./data/service_letters_train.npz')
ds.build_dataset('./data/images/letters/test', 'jpg',
'./npz/service_letters_test.npz')
ds.plot_dataset('./data/service_letters_test.npz')
torch.manual_seed(seed)
if use_cuda:
os.environ['CUDA_VISIBLE_DEVICES'] = gpus
torch.cuda.manual_seed(seed)
model = Darknet(darknetcfg, learnetcfg)
region_loss = model.loss
model.load_weights(weightfile)
model.print_network()
###################################################
### Meta-model parameters
region_loss.seen = model.seen
processed_batches = 0 if cfg.tuning else model.seen / batch_size
trainlist = dataset.build_dataset(data_options)
nsamples = len(trainlist)
init_width = model.width
init_height = model.height
init_epoch = 0 if cfg.tuning else model.seen / nsamples
max_epochs = max_batches * batch_size / nsamples + 1
max_epochs = int(math.ceil(cfg.max_epoch * 1. /
cfg.repeat)) if cfg.tuning else max_epochs
print(cfg.repeat, nsamples, max_batches, batch_size)
print(num_workers)
kwargs = {'num_workers': num_workers, 'pin_memory': True} if use_cuda else {}
test_loader = torch.utils.data.DataLoader(dataset.listDataset(
testlist,
shape=(init_width, init_height),
shuffle=False,
def _build_dataset(self):
dataset = build_dataset(self._flags.dataset)
if dataset not in (DurationDataset, MutiTaskDataset):
raise ValueError('mmoe model should be run with DurationDataset or MutiTaskDataset, find {}'
.format(dataset.__name__))
return dataset
def colorization_train(args, wandb_run):
assert os.path.exists(
args.draft_weights_path), f"{args.draft_weights_path} not exists"
global logger, X
logger = wandb_run
os.makedirs(os.path.join(logger.dir, "image"), exist_ok=True)
##################### init values ##########################
X = stats.truncnorm(
(0 - args.mu) / args.sigma,
(1 - args.mu) / args.sigma,
loc=args.mu,
scale=args.sigma,
)
########################## dataset ##########################
train_transforms = ColorizationTransforms(args.image_size,
args.draft_image_r,
train=Mode.TRAIN)
test_transforms = ColorizationTransforms(args.image_size,
args.draft_image_r,
train=Mode.TEST)
train_dataset = build_dataset(os.path.join(args.root_dir, "train"),
train_transforms, args.batch_size)
test_dataset = build_dataset(os.path.join(args.root_dir, "test"),
test_transforms, 8)
test_batch = next(iter(test_dataset))
line, line_draft, hint, color = test_batch
mask = opt.mask_gen(list(hint.shape), X, 0)
hint = hint * mask
test_batch = (line, line_draft, hint, color)
########################## model ##########################
draft_image_size = args.image_size // args.draft_image_r
draft_gen = Generator(args.g_dim)
input_dict = {"line": line, "hint": hint}
draft_gen(input_dict)
draft_gen.load_weights(args.draft_weights_path)
draft_gen.trainable = False
gen = Generator(args.g_dim)
########################## optim ##########################
total_step = len(train_dataset) * args.epochs
gen_optim_lr_schedule = optim.schedules.ExponentialDecay(
args.lr,
decay_steps=int(total_step * args.decay_steps_rate),
decay_rate=args.decay_rate,
staircase=True,
)
gen_optim = optim.Adam(gen_optim_lr_schedule,
beta_1=args.beta_1,
beta_2=args.beta_2)
######################## training ##########################
for epoch in range(args.epochs):
print(f"Epoch: {epoch} start")
training_loop(train_dataset, test_batch, draft_gen, gen, gen_optim)
gen.save_weights(os.path.join(logger.dir, "gen.ckpt.h5"))
#################### artifacts loging ######################
artifacts_path = os.path.join(logger.dir, f"{args.mode}.h5")
gen.save(artifacts_path)
if args.upload_artifacts:
gen_artifacts = wandb.Artifact(args.mode, type="model")
gen_artifacts.add_file(artifacts_path, "weight")
logger.log_artifact(gen_artifacts)
def main():
levels = build_levels()
data_loader = Inferred('shapes', ['./tmp', 10, 500])
dataset = build_dataset(data_loader,
levels=levels,
scale=500,
shuffle=4096,
augment=True)
input = dataset.make_one_shot_iterator().get_next()
image = input['image']
classifications = input['detection']['classifications']
trainable_masks = input['trainable_masks']
regressions_postprocessed = utils.dict_starmap(
lambda r, l: utils.regression_postprocess(
r, tf.to_float(l.anchor_sizes / input['image_size'])),
[input['detection']['regressions'], levels])
batch = []
for i in range(image.shape[0]):
level_images = {}
for k in levels:
class_row = []
mask_row = []
boxes_row = []
for a in range(levels[k].anchor_sizes.shape[0]):
image_with_classes = draw_classmap(
image[i], classifications[k][i, :, :, a, :])
image_with_classes /= tf.reduce_max(image_with_classes)
class_row.append(image_with_classes)
image_with_mask = draw_mask(image[i],
trainable_masks[k][i, :, :, a])
image_with_mask /= tf.reduce_max(image_with_mask)
mask_row.append(image_with_mask)
decoded = utils.boxes_decode(
classifications[k][i, :, :, a, :],
regressions_postprocessed[k][i, :, :, a, :])
image_with_boxes = draw_bounding_boxes(image[i], decoded,
data_loader.class_names)
image_with_boxes /= tf.reduce_max(image_with_boxes)
boxes_row.append(image_with_boxes)
class_row = tf.concat(class_row, 1)
mask_row = tf.concat(mask_row, 1)
boxes_row = tf.concat(boxes_row, 1)
level_image = tf.concat([class_row, mask_row, boxes_row], 0)
level_images[k] = level_image
batch.append(level_images)
with tf.Session() as sess:
batch = sess.run(batch)
for level_images in batch:
for k in level_images:
plt.figure(figsize=(16, 8))
plt.imshow(level_images[k])
plt.title(k)
plt.show()
parser.add_argument("-t", "--table_path", type=str, required=True,
help="The path of table file.")
parser.add_argument("-w", "--image_width", type=int, default=100,
help="Image width, this parameter will affect the output "
"shape of the model, default is 100, so this model "
"can only predict up to 24 characters.")
parser.add_argument("-b", "--batch_size", type=int, default=256,
help="Batch size.")
parser.add_argument("-m", "--model", type=str, required=True,
help="The saved model.")
parser.add_argument("--channels", type=int, default=1, help="Image channels, "
"0: Use the number of channels in the image, "
"1: Grayscale image, "
"3: RGB image")
parser.add_argument("--ignore_case", action="store_true",
help="Whether ignore case.(default false)")
args = parser.parse_args()
eval_ds, size, num_classes = build_dataset(
args.annotation_paths,
args.table_path,
args.image_width,
args.channels,
args.ignore_case,
batch_size=args.batch_size)
print("Num of eval samples: {}".format(size))
model = keras.models.load_model(args.model, compile=False)
model.summary()
model.compile(loss=CTCLoss(), metrics=[WordAccuracy()])
model.evaluate(eval_ds)
def main():
args = arg_parser()
seed_everything(args.seed)
if cuda.is_available():
device = torch.device("cuda:0")
else:
device = torch.device("cpu")
train_df = pd.read_csv(args.train_df_path)
valid_df = pd.read_csv(args.valid_df_path)
valid_df_sub = valid_df.sample(
frac=1.0, random_state=42).reset_index(drop=True)[:40000]
valid_df_sub1 = valid_df.sample(
frac=1.0, random_state=52).reset_index(drop=True)[:40000]
valid_df_sub2 = valid_df.sample(
frac=1.0, random_state=62).reset_index(drop=True)[:40000]
del valid_df
gc.collect()
if args.DEBUG:
train_df = train_df[:1000]
valid_df_sub = valid_df_sub[:1000]
valid_df_sub1 = valid_df_sub1[:1000]
valid_df_sub2 = valid_df_sub2[:1000]
train_loader = build_dataset(args, train_df, is_train=True)
batch_num = len(train_loader)
valid_loader = build_dataset(args, valid_df_sub, is_train=False)
valid_loader1 = build_dataset(args, valid_df_sub1, is_train=False)
valid_loader2 = build_dataset(args, valid_df_sub2, is_train=False)
model = build_model(args, device)
if args.model == 'resnet50':
save_path = os.path.join(args.PATH, 'weights', f'resnet50_best.pt')
if args.model == 'resnext':
save_path = os.path.join(args.PATH, 'weights', f'resnext_best.pt')
elif args.model == 'xception':
save_path = os.path.join(args.PATH, 'weights', f'xception_best.pt')
else:
NotImplementedError
optimizer = build_optimizer(args, model)
scheduler = build_scheduler(args, optimizer, batch_num)
train_cfg = {
'train_loader': train_loader,
'valid_loader': valid_loader,
'valid_loader1': valid_loader1,
'valid_loader2': valid_loader2,
'model': model,
'criterion': nn.BCEWithLogitsLoss(),
'optimizer': optimizer,
'scheduler': scheduler,
'save_path': save_path,
'device': device
}
train_model(args, train_cfg)
