def __init__(self, args: argparse.Namespace):
self.args = args
self.train_writer = SummaryWriter('Logs/train')
self.test_writer = SummaryWriter('Logs/test')
self.wavenet = Wavenet(args, self.train_writer)
self.train_data_loader = DataLoader(
batch_size=args.batch_size * torch.cuda.device_count(),
shuffle=args.shuffle,
num_workers=args.num_workers,
train=True,
input_length=self.args.output_length +
self.wavenet.receptive_field + 1,
output_length=self.args.output_length,
dataset_length=self.args.sample_step * self.args.accumulate *
self.args.batch_size * torch.cuda.device_count())
self.test_data_loader = DataLoader(
batch_size=args.batch_size * torch.cuda.device_count(),
shuffle=args.shuffle,
num_workers=args.num_workers,
train=False,
input_length=self.args.output_length +
self.wavenet.receptive_field + 1,
output_length=self.args.output_length)
self.start_1 = 0
self.start_2 = 0
self.load_last_checkpoint(self.args.resume)
def main():
dl = DataLoader()
model = Model()
sess = tf.Session()
sess.run(tf.global_variables_initializer())
n_batch = len(dl.data['train']['Y']) # dl.batch_size
for epoch in range(NUM_EPOCHS):
for i, (user_id, gender_id, age_id, job_id, movie_id, category_ids,
movie_title, score) in enumerate(dl.next_train_batch()):
loss = model.train_batch(sess, user_id, gender_id, age_id, job_id,
movie_id, category_ids, movie_title,
score)
if i % 500 == 0:
print('Epoch [%d/%d] | Batch [%d/%d] | Loss: %.2f | LR: %.4f' %
(epoch, NUM_EPOCHS, i, n_batch, loss, model.lr))
losses = []
for i(user_id, gender_id, age_id, job_id, movie_id, category_ids,
movie_title, score) in enumerate(dl.next_test_batch()):
pred, loss = model.predict_batch(sess, user_id, gender_id, age_id,
job_id, movie_id, category_ids,
movie_title, score)
losses.append(loss)
print('-' * 30)
print('Testing losses:', sum(losses) / len(losses))
print('Prediction:%.2f,Actual:%.2f' % (pred[-5], score[-5]))
print('Prediction:%.2f,Actual:%.2f' % (pred[-4], score[-4]))
print('Prediction:%.2f,Actual:%.2f' % (pred[-3], score[-3]))
print('Prediction:%.2f,Actual:%.2f' % (pred[-2], score[-2]))
print('Prediction:%.2f,Actual:%.2f' % (pred[-1], score[-1]))
print('-' * 30)
def __init__(self, args):
logging.info('Loading dataset')
#self.loader = CNNDailyMail(config.dataset, data_type, ['src', 'trg'], config.bpe_model_filename)
self.loader = DataLoader(config.dataset, train_type, ['src', 'trg'], config.bpe_model_filename)
self.eval_loader = DataLoader(config.dataset, eval_type, ['src', 'trg'], config.bpe_model_filename)
logging.info('Dataset has been loaded.Total size: %s, maxlen: %s', self.loader.data_length, self.loader.max_len)
self.device = torch.device("cuda" if args.cuda and torch.cuda.is_available() else "cpu")
if config.multi_gpu:
self.n_gpu = torch.cuda.device_count()
else:
self.n_gpu = 0
os.environ["CUDA_VISIBLE_DEVICES"] = "3"
self.writer = SummaryWriter(config.log + config.prefix)
if args.pretrain_emb:
self.embeddings = torch.from_numpy(np.load(config.emb_filename)).float()
logging.info('Use vocabulary and embedding sizes from embedding dump.')
self.vocab_size, self.emb_size = self.embeddings.shape
else:
self.embeddings = None
self.vocab_size, self.emb_size = config.vocab_size, config.emb_size
self.args = args
self.m_args = {'max_seq_len': 638, 'vocab_size': self.vocab_size,
'enc_n_layers': config.enc_n_layers, 'dec_n_layers': config.dec_n_layers, 'global_layers': config.global_layers,
'batch_size': config.train_bs, 'emb_size': self.emb_size, 'dim_m': config.model_dim,
'n_heads': config.n_heads, 'dim_i': config.inner_dim, 'dropout': config.dropout,
'embedding_weights': self.embeddings, 'lexical_switch': args.lexical,
'emb_share_prj': args.emb_share_prj, 'global_encoding':config.global_encoding,
'encoding_gate': config.encoding_gate, 'stack': config.stack}
#if config.multi_gpu:
# self.m_args['batch_size'] *= n_gpu
self.error_file = open(config.error_filename, "w")
def main():
args = get_args(train=True)
set_random_seed(args.seed)
dl = DataLoader(video_dir=args.video_dir,
caption_filename=args.caption_filename,
load_word_dict=args.no_load_word_dict,
word_dict_filename=args.word_dict_filename)
train_video, train_caption = dl.load_data()
word_dict_len = dl.get_word_dict_len()
train(total_data=len(train_video),
train_video=train_video,
train_caption=train_caption,
prefix=args.prefix,
validation=args.validation,
batch_size=args.batch_size,
collate_fn=collate_fn,
margin=args.margin,
model_name=args.model,
vocabulary_size=word_dict_len,
embed_dim=args.embed_dim,
hidden_size=args.hidden_size,
rnn_layers=args.rnn_layers,
dropout_rate=args.dropout_rate,
bidirectional=args.no_bidirectional,
learning_rate=args.learning_rate,
epoches=args.epoches,
save_intervals=args.save_intervals,
use_cuda=args.no_cuda)
def __init__(self, path, size_x=448, size_y=448):
"""
description:
FCN class
input:
path: string, path to VOC PASCAL dataset
size_x: int, width to scale images to
size_y: int, height to scale images to
return:
NA
"""
self.path = path
self.size_x = size_x
self.size_y = size_y
self.n_classes = 21
self.data = DataLoader(self.path,
size_x=self.size_x,
size_y=self.size_y)
self.train_list, self.val_list = self.data.load_seg_lists()
self.fcn = self.model_init()
self.fcn.summary()
def train(self):
# TODO: split local and remote samples
data_loader = DataLoader(self.data_order)
dataset = data_loader.load(self.data_path, self.num_edge)
head_index = dataset['head_index']
tail_index = dataset['tail_index']
rel_index = dataset['rel_index']
model = Model(self.model_config)
model.share_memory()
for epoch in range(self.num_epoch):
# TODO: last batch may not be complete
perm = th.randperm(self.num_edge)
head_index = head_index[perm].view(self.num_proc, -1,
model.num_chunk * model.pos_num)
tail_index = tail_index[perm].view(self.num_proc, -1,
model.num_chunk * model.pos_num)
rel_index = rel_index[perm].view(self.num_proc, -1,
model.num_chunk * model.pos_num)
procs = []
for proc in range(self.num_proc):
p = mp.Process(target=train_proc,
args=(model, proc, head_index[proc],
tail_index[proc], rel_index[proc],
self.loss_func))
p.start()
procs.append(p)
for p in procs:
p.join()
def get_data_and_config(global_args):
data_loader = DataLoader(global_args)
train_set, test_set = data_loader.split()
model_args = get_model_args(data_loader.item_nums, global_args)
logging.info(dict_to_str(model_args, "Configurations"))
return model_args, train_set, test_set
def __init__(self, args):
logging.info('initializing...')
self.args = args
self.dataset = DataLoader(self.args)
self.embeddings_path = args.embeddings_path
self.small_embeddings_path = os.path.splitext(
self.embeddings_path)[0] + '_small.vec'
logging.info('initializing...[ok]')
def load_data(args):
import random
train_val_filelist = glob.glob(args.data_train)
random.shuffle(train_val_filelist)
n_train = int(args.train_val_split * len(train_val_filelist))
d = DataFormat(train_groups,
train_vars,
label_var,
wgtvar,
obs_vars,
filename=train_val_filelist[0],
point_mode='NCP')
logging.info('Using the following variables:\n' + '\n'.join([
v_group + '\n\t' + str(train_vars[v_group]) for v_group in train_groups
]))
logging.info('Using weight\n' + ','.join(wgtvar))
orig_metadata = os.path.join(os.path.dirname(train_val_filelist[0]),
'metadata.json')
output_metadata = os.path.join(os.path.dirname(args.model_prefix),
'preprocessing.json')
if args.predict:
test_filelist = glob.glob(args.data_test)
test = DataLoader(test_filelist,
d,
batch_size=args.batch_size,
predict_mode=True,
shuffle=False,
args=args)
return test
else:
train = DataLoader(train_val_filelist[:n_train],
d,
batch_size=args.batch_size,
args=args)
val = DataLoader(train_val_filelist[n_train:],
d,
batch_size=args.batch_size,
args=args)
if not os.path.exists(output_metadata):
train_shapes = {}
train_shapes['pfcand'] = (1, ) + (
100,
len(train_vars['pfcand']),
)
dump_input_metadata(orig_metadata,
groups=train_groups,
shapes=train_shapes,
var_names=train_vars,
output=output_metadata)
return (train, val)
def train_variational_autoencoder(
learning_rate: float,
epochs: int,
batch_size: int,
latent_variables: int = 10,
print_every: int = 50,
) -> None:
print(
f"Training a variational autoencoder for {epochs} epochs with batch size {batch_size}"
)
data_loader = DataLoader(batch_size)
image_loss = CrossEntropy()
divergence_loss = KLDivergenceStandardNormal()
encoder_mean = Model([Linear(784, 50), ReLU(), Linear(50, latent_variables)])
encoder_variance = Model(
[Linear(784, 50), ReLU(), Linear(50, latent_variables), Exponential()]
)
reparameterization = Reparameterization()
decoder = Model([Linear(latent_variables, 50), ReLU(), Linear(50, 784)])
for i in range(epochs):
# One training loop
training_data = data_loader.get_training_data()
for j, batch in enumerate(training_data):
input, target = batch
# Forward pass
mean = encoder_mean(input)
variance = encoder_variance(input)
z = reparameterization(mean=mean, variance=variance)
generated_samples = decoder(z)
# Loss calculation
divergence_loss_value = divergence_loss(mean, variance)
generation_loss = image_loss(generated_samples, input)
if j % print_every == 0:
print(
f"Epoch {i+1}/{epochs}, "
f"training iteration {j+1}/{len(training_data)}"
)
print(
f"KL loss {np.round(divergence_loss_value, 2)}\t"
f"Generation loss {np.round(generation_loss, 2)}"
)
# Backward pass
decoder_gradient = image_loss.gradient()
decoder_gradient = decoder.backward(decoder_gradient)
decoder_mean_gradient, decoder_variance_gradient = reparameterization.backward(
decoder_gradient
)
encoder_mean_gradient, encoder_variance_gradient = (
divergence_loss.gradient()
)
encoder_mean.backward(decoder_mean_gradient + encoder_mean_gradient)
encoder_variance.backward(
decoder_variance_gradient + encoder_variance_gradient
)
def get_data_and_config(global_args):
data_loader = DataLoader(global_args)
train_set, test_set = data_loader.split()
model_args = get_model_args(data_loader.item_nums, global_args)
logging.info(dict_to_str(model_args, "Model Configurations"))
if model_args["gen_sub_sess"]:
train_set = data_loader.generate_sub_sessions(train_set,
model_args["pad_token"])
return model_args, train_set, test_set
def evaluate_trainset(model=None):
with open("./data/labels.pkl", "rb") as f:
labels = pickle.load(f)
with open("./data/vectorizer.pkl", "rb") as f:
vectorizer = pickle.load(f)
with open(Config.train_path, "r", encoding="utf-8") as f:
n_val = 0
for line in f:
n_val += 1
n_labels = len(labels)
test_loader = DataLoader(Config.train_path,
batch_size=Config.batch_size,
n_labels=n_labels,
vectorizer=vectorizer)
if model is None:
model = load_model(model_path)
y_predict = model.predict_generator(
generator=test_loader.get_test_generator(),
steps=n_val // Config.batch_size + 1,
verbose=0)
with open("./prediction_trainset.csv", "w", encoding="utf-8-sig") as fout:
with open(Config.train_path, "r", encoding="utf-8") as fin:
cnt = 0
hit = 0
gt_total = 0
pt_total = 0
for line in fin:
cur_labels, description = line.strip().split(",")
ground_truth = [
labels[idx] for idx in map(int, cur_labels.split())
]
predict = [
labels[idx] for idx in np.where(y_predict[cnt] > 0.5)[0]
]
cur_hit = len(set(ground_truth).intersection(set(predict)))
hit += cur_hit
gt_total += len(ground_truth)
pt_total += len(predict)
fout.write("{},{},{}\n".format(" ".join(ground_truth),
" ".join(predict), cur_hit))
cnt += 1
precision = hit / pt_total if pt_total != 0 else 0
recall = hit / gt_total if gt_total != 0 else 0
print("precision:{:.4f}, recall:{:.4f}".format(precision, recall))
return precision, recall
def main():
# Define the parser
parser = argparse.ArgumentParser()
parser.add_argument('--batch_size',
type=int,
default=32,
help='minibatch size')
# Read the arguments
eval_args = parser.parse_args()
with open(os.path.join('saved_models', 'config.pkl'), 'rb') as f:
saved_args = pickle.load(f)
# Initialize with the saved args
model = Model(saved_args, 'eval')
# Initialize TensorFlow session
sess = tf.InteractiveSession()
# Initialize TensorFlow saver
saver = tf.train.Saver()
# restore the session
saver.restore(sess, 'saved_models/model.ckpt')
data_loader = DataLoader(eval_args.batch_size, mode='eval')
data_loader.load_data('/home/kevin/ncfm_data', '/home/kevin/test.txt')
data_loader.convert2images(maxn=saved_args.input_dim[0])
# Maintain the total_error until now
accuracy = 0
total_loss = 0.
counter = 0.
for e in range(1):
# Reset the data pointers of the data loader object
data_loader.reset_batch_pointer()
for b in range(data_loader.num_batches):
start = time.time()
# Get the source, target data for the next batch
x, y = data_loader.next_batch()
feed = {model.input_data: x, model.target_data: y}
loss, pred = sess.run([model.loss, model.pred], feed)
accuracy += np.sum(np.array(pred) == np.array(y))
total_loss += loss
end = time.time()
print "Processed batch number : ", b, "out of ", data_loader.num_batches, " batches, time consuming: ", end - start
counter += data_loader.batch_size
# Print the mean error across all the batches
print "Total mean accuracy of the evaluation is ", accuracy / counter
print "Total mean testing loss of the model is ", total_loss / counter
def __init__(self, args):
self.args = args
self.train_writer = SummaryWriter('Logs/train')
self.test_writer = SummaryWriter('Logs/test')
self.wavenet = Wavenet(args, self.train_writer)
self.train_data_loader = DataLoader(
args.batch_size * torch.cuda.device_count(), args.shuffle,
args.num_workers, True)
self.test_data_loader = DataLoader(
args.batch_size * torch.cuda.device_count(), args.shuffle,
args.num_workers, False)
self.wavenet.total = self.train_data_loader.__len__(
) * self.args.num_epochs
self.load_last_checkpoint(self.args.resume)
class EmbeddingsBuilder:
def __init__(self, args):
logging.info('initializing...')
self.args = args
self.dataset = DataLoader(self.args)
self.embeddings_path = args.embeddings_path
self.small_embeddings_path = os.path.splitext(
self.embeddings_path)[0] + '_small.vec'
logging.info('initializing...[ok]')
def build_embedding(self, vocab_dict):
"""
Load embedding vectors from a .txt file.
Optionally limit the vocabulary to save memory. `vocab` should be a set.
"""
num_words = len(vocab_dict)
num_found = 0
with open(self.small_embeddings_path, 'w') as out_file:
with tf.gfile.GFile(self.embeddings_path) as f:
header = next(f)
num_embeddings, embeddings_dim = header.split(' ')
num_embeddings = int(num_embeddings)
out_file.write(header)
for _, line in tqdm(enumerate(f),
'loading embeddings',
total=num_embeddings):
tokens = line.rstrip().split(" ")
word = tokens[0]
if word in vocab_dict:
num_found += 1
out_file.write(line)
tf.logging.info(
"Found embeddings for {} out of {} words in vocabulary".format(
num_found, num_words))
def run(self):
self.dataset.load()
X = self.dataset.X_train_labeled['moment'].values
X = np.append(X,
self.dataset.X_train_unlabeled['moment'].values,
axis=0)
X = np.append(X, self.dataset.X_test['moment'].values, axis=0)
tokenizer = Tokenizer()
tokenizer.fit_on_texts(X)
self.build_embedding(tokenizer.word_index)
def main():
model = tf.estimator.Estimator(
model_fn,
params={
'lr': 1e-4,
'movie_id_size': paddle.dataset.movielens.max_movie_id() + 1,
'job_id_size': paddle.dataset.movielens.max_job_id() + 1,
'user_id_size': paddle.dataset.movielens.max_user_id() + 1,
'age_id_size': len(paddle.dataset.movielens.age_table),
'movie_title_vocab_size': 5175,
})
dl = DataLoader(BATCH_SIZE)
for i in xrange(NUM_EPOCHS):
model.train(dl.train_pipeline())
print('Testing loss:', model.evaluate(dl.eval_pipeline())['mse'])
def __init__(self, args):
self.args = args
self.args.run_id = str(uuid.uuid4())
self.args.initial_timestamp = datetime.now().timestamp()
self.args.TEXT_COL = 1
self.dataset = DataLoader(self.args)
self.output_path = args.output_file
if os.path.exists(self.output_path):
os.remove(self.output_path)
print("PARAMS: {}".format(self.args))
# set random state
np.random.seed(args.random_state)
def main():
dl = DataLoader(
source_path='../temp/letters_source.txt',
target_path='../temp/letters_target.txt')
sources, targets = dl.load()
tf_estimator = tf.estimator.Estimator(
tf_estimator_model_fn, params=prepare_params(dl), model_dir=args.model_dir)
for epoch in range(args.num_epochs):
tf_estimator.train(tf.estimator.inputs.numpy_input_fn(
x = {'source':sources, 'target':targets},
batch_size = args.batch_size,
num_epochs = None,
shuffle = True), steps=1000)
greedy_decode(['apple', 'common', 'zhedong'], tf_estimator, dl)
def main():
if len(sys.argv) < 4:
raise SyntaxError("Missing inputs to the script")
DATASET_PATH = sys.argv[1]
DATE_COL = sys.argv[2]
SERIES_NAME = sys.argv[3]
FORECAST_STEPS = int(sys.argv[4])
OPTIONAL_PARAMETERS = sys.argv[5]
FREQUENCY = None
SEASONAL_PERIOD = None
# Extract optional parameters from the script inputs
if OPTIONAL_PARAMETERS is not None and OPTIONAL_PARAMETERS != "":
for parameter in OPTIONAL_PARAMETERS.split(","):
vals = parameter.split("=")
key = vals[0]
value = vals[1]
if key == "FREQUENCY":
FREQUENCY = value
elif key == "SEASONAL_PERIOD":
SEASONAL_PERIOD = int(value)
dataframe = DataLoader.load_data(DATASET_PATH,
date_col=DATE_COL,
frequency=FREQUENCY)
model = AutoARIMA()
model.fit(dataframe[SERIES_NAME], seasonal_period=SEASONAL_PERIOD)
print(model.predict(FORECAST_STEPS))
def predict(args, trainer=None, pretrained=None, use_cuda=False):
# load pretrained embeddings and model
if not trainer:
# load pretrained embeddings
pretrained = Pretrain(from_pt=args.embeddings)
# load model
logger.info("Loading model from {}".format(args.model))
trainer = Trainer(model_file=args.model, pretrain=pretrained, use_cuda=use_cuda)
# load data
logger.info("Loading prediction data...")
doc = Document(from_file=args.test_data, read_positions=get_read_format_args(args), write_positions=get_write_format_args(args), copy_untouched=args.copy_untouched)
data = DataLoader(doc, args.batch_size, vocab=trainer.vocab, pretrain=pretrained, evaluation=True)
if len(data) == 0:
raise RuntimeError("Cannot start prediction because no data is available")
logger.info("Start prediction...")
preds = []
for batch in data:
preds += trainer.predict(batch)[0] # don't keep loss
preds = unsort(preds, data.data_orig_idx)
# write to file and score
doc.add_predictions(preds)
doc.write_to_file(args.test_data_out)
display_results(doc, args.no_eval_feats, per_feature=True, report_oov=args.w_token_index >= 0)
def test():
du = DataLoader(**data_config)
params['src_vcb_size'] = du.vocab_size
params['tgt_vcb_size'] = du.vocab_size
params['batch_size'] = 1
tf.reset_default_graph()
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False,
gpu_options=tf.GPUOptions(allow_growth=True))
with tf.Session(config=config) as sess:
model = Seq2Seq(params, mode='decode')
sess.run(tf.global_variables_initializer())
model.load(sess, tf.train.latest_checkpoint('./logs/model/'))
# model.load(sess, './logs/model/model_16.ckpt')
# sent = input('you: ')
# while (sent):
# result = model.get_response(sess, sent, du)
# print('bot: ', result)
#
# sent = input('you: ')
sents = [('天王盖地虎', '宝塔镇妖河')]
for sent in sents:
result = model.get_response(sess, sent[0], du)
print('source : ', sent[0])
print('target : ', sent[1])
print('predict: ', result)
print('')
def run_test(epoch=-1, is_val=True):
print('Running Test')
opt = TestOptions().parse()
# No shuffling for test set
opt.serial_batches = True
opt.which_epoch = epoch
# Set batch_size to 1
opt.batch_size = 1
# If we are running on the test set change the folder path to where the test meshes are stored
if not is_val:
opt.phase = "test"
dataset = DataLoader(opt)
if opt.verbose:
print("DEBUG testpath: ", opt.dataroot)
print("DEBUG dataset length ", len(dataset))
model = create_model(opt)
writer = Writer(opt)
writer.reset_counter()
for i, data in enumerate(dataset):
model.set_input(data)
ncorrect, nexamples = model.test(epoch, is_val)
if opt.verbose:
print("DEBUG test ncorrect, nexamples ", ncorrect, nexamples)
writer.update_counter(ncorrect, nexamples)
writer.print_acc(epoch, writer.acc)
return writer.acc
def main():
"""
Main Function for searching process.
"""
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
cudnn.enabled = True
cudnn.benchmark = True
cudnn.deterministic = True
checkpoint = utils.checkpoint(args)
if checkpoint.ok:
data_loader = DataLoader(args)
loss = Loss(args, checkpoint) if not args.test_only else None
search_model = Controller(args, loss).cuda()
srdarts = Searcher(args, data_loader, search_model, loss, checkpoint)
while not srdarts.terminate():
srdarts.search()
srdarts.valid()
checkpoint.done()
def __init__(self, args):
self.args = args
self.train_writer = SummaryWriter('Logs/train')
self.test_writer = SummaryWriter('Logs/test')
self.wavenet = Wavenet(args.layer_size, args.stack_size, args.channels,
args.residual_channels, args.dilation_channels,
args.skip_channels, args.end_channels,
args.out_channels, args.learning_rate,
self.train_writer)
self.train_data_loader = DataLoader(
args.batch_size * torch.cuda.device_count(),
self.wavenet.receptive_field, args.shuffle, args.num_workers, True)
self.test_data_loader = DataLoader(
args.batch_size * torch.cuda.device_count(),
self.wavenet.receptive_field, args.shuffle, args.num_workers,
False)
def test_feature_detached(self):
loader = DataLoader.build_with(self.path, self.source, self.config)
batch = next(loader.iterator())
tensor = batch["avgpool_512"]
self.assertFalse(tensor.requires_grad)
def main():
opt = get_opt()
transform = torchvision.transforms.ToTensor()
coco = torchvision.datasets.CocoDetection('data/train2014/' , 'data/annotations/instances_train2014.json' , transform=transform)
if not os.path.exists('tensorboard'):
os.makedirs('tensorboard')
board = SummaryWriter(log_dir = os.path.join('tensorboard', 'SSD'))
train_dataset = Dataset(coco)
train_loader = DataLoader(opt, train_dataset)
model = ObjectDetection_SSD(nbr_classes = opt.nbr_classes)
start = time.time()
train(model , opt , train_loader , board)
end = time.time()
duree = end-start
torch.save(model.cpu().state_dict(), 'model.pth')
print('Finished training in' , duree)
def main(args, local_rank=0):
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO)
vocabs = dict()
vocabs['src'] = Vocab(args.src_vocab, 0, [BOS, EOS])
vocabs['tgt'] = Vocab(args.tgt_vocab, 0, [BOS, EOS])
logger.info(args)
for name in vocabs:
logger.info("vocab %s, size %d, coverage %.3f", name,
vocabs[name].size, vocabs[name].coverage)
set_seed(19940117)
#device = torch.device('cpu')
torch.cuda.set_device(local_rank)
device = torch.device('cuda', local_rank)
logger.info("start building model")
logger.info("building retriever")
if args.add_retrieval_loss:
retriever, another_model = Retriever.from_pretrained(
args.num_retriever_heads,
vocabs,
args.retriever,
args.nprobe,
args.topk,
local_rank,
load_response_encoder=True)
matchingmodel = MatchingModel(retriever.model, another_model)
matchingmodel = matchingmodel.to(device)
else:
retriever = Retriever.from_pretrained(args.num_retriever_heads, vocabs,
args.retriever, args.nprobe,
args.topk, local_rank)
logger.info("building retriever + generator")
model = RetrieverGenerator(vocabs, retriever, args.share_encoder,
args.embed_dim, args.ff_embed_dim,
args.num_heads, args.dropout, args.mem_dropout,
args.enc_layers, args.dec_layers,
args.mem_enc_layers, args.label_smoothing)
model = model.to(device)
model.eval()
dev_data = DataLoader(vocabs,
args.dev_data,
args.dev_batch_size,
for_train=False)
bleu = validate(device,
model,
dev_data,
beam_size=5,
alpha=0.6,
max_time_step=10)
def run_test(dataset=None, epoch=-1, phase="test"):
print('Running Test')
opt = TestOptions().parse()
opt.serial_batches = True # no shuffle
if dataset is None:
dataset = CreateDataset(opt)
# be consistent with training
dataset = torch.utils.data.Subset(dataset, range(len(dataset)))
dataset = DataLoader(dataset, opt)
else:
opt.nclasses = len(dataset.dataset.dataset.classes)
opt.input_nc = dataset.dataset.dataset.opt.input_nc
dataset.dataset.dataset.opt.num_aug = 1
# dataset.dataset.dataset.opt.is_train = False
model = ClassifierModel(opt)
writer = Writer(opt)
# test
writer.reset_counter()
for i, data in enumerate(dataset):
model.set_input(data, epoch)
loss, (prec1, prec5), y_pred, y_true = model.test()
writer.update_counter(loss, prec1, prec5, y_pred, y_true)
if epoch == -1:
writer.plot_summary("val", dataset.dataset.dataset.classes)
else:
writer.plot(epoch, phase, dataset.dataset.dataset.classes)
return writer.statistics.top1.avg
def train_encoder(mdl, crit, optim, sch, stat):
"""Train REL or EXT model"""
logger.info(f'*** Epoch {stat.epoch} ***')
mdl.train()
it = DataLoader(load_dataset(args.dir_data,
'train'), args.model_type, args.batch_size,
args.max_ntokens_src, spt_ids_B, spt_ids_C, eos_mapping)
for batch in it:
_, logits = mdl(batch)
mask_inp = utils.sequence_mask(batch.src_lens, batch.inp.size(1))
loss = crit(logits, batch.tgt, mask_inp)
loss.backward()
stat.update(loss,
'train',
args.model_type,
logits=logits,
labels=batch.tgt)
torch.nn.utils.clip_grad_norm_(model.parameters(), 5)
optim.step()
if stat.steps == 0:
continue
if stat.steps % args.log_interval == 0:
stat.lr = optim.param_groups[0]['lr']
stat.report()
sch.step(stat.avg_train_loss)
if stat.steps % args.valid_interval == 0:
valid_ret(mdl, crit, optim, stat)
def run(args):
if args.use_gpu:
try:
output = subprocess.check_output("free-gpu", shell=True)
output = int(output.decode('utf-8'))
gpu = output
print("claiming gpu {}".format(gpu))
torch.cuda.set_device(int(gpu))
a = torch.tensor([1]).cuda()
device = int(gpu)
except (IndexError, subprocess.CalledProcessError) as e:
device = None
else:
device = None
test_generator = DataLoader("../../data/raw/xkcd_colordata",
"../../data/raw/",
"test",
device=device)
model = load_model(args.model_path)
batch_count = 0
inst_count = 0
cos_accum = 0.0
for batch in test_generator:
X, Y = prepare_batch(batch, device)
Y_pred = model.forward(batch)
cos_accum += angle(Y_pred, Y)
batch_count += 1
inst_count += batch_size
if batch_count % 10 == 0:
print('batch_count:%d, inst_count:%d, avg_cosine:%.3f' %
(batch_count, inst_count, cos_accum / batch_count))
print(cos_accum / batch_count)
def run_cnn(model_params, optimization_params, dataset_path, dataset_params, filename_params, visual_params, epochs, verbose=False):
print(filename_params)
if not os.path.exists(filename_params.results):
os.makedirs(filename_params.results)
is_config, config_values = interface.command.get_command("-config")
is_curriculum, curriculum_set = interface.command.get_command("-curriculum")
is_batch_run, batch_index = interface.command.get_command("-batch", default="0")
is_init_params, param_path = interface.command.get_command("-params")
if is_config:
#Assume config is speficially for running bootstrapping batches.
config_arr = eval(config_values)
if len(config_arr) == 2:
loss_function = config_arr[0]
label_noise = float(config_arr[1])
dataset_params.label_noise = label_noise
model_params.loss = loss_function
batch_index = loss_function + "-" + str(label_noise) + "-" + batch_index
print(batch_index)
if is_curriculum:
dataset_path = curriculum_set
weights = None
if is_init_params:
store = ParamStorage()
if not param_path:
param_path = "./results/params.pkl"
weights = store.load_params(path=param_path)['params']
dataset = DataLoader.create()
dataset.load(dataset_path, dataset_params, optimization_params.batch_size) #Input stage
model = ConvModel(model_params, verbose=True) #Create network stage
evaluator = Evaluator(model, dataset, optimization_params, dataset_path)
evaluator.run(epochs=epochs, verbose=verbose, init=weights)
report = evaluator.get_result()
network_store_path = filename_params.network_save_name
result_path = filename_params.results + "/results.json"
if is_batch_run:
network_store_path = filename_params.results + "/batch" + batch_index + ".pkl"
result_path =filename_params.results + "/batch" + batch_index + ".json"
storage = ParamStorage(path=network_store_path)
storage.store_params(model.params)
dataset.destroy()
if visual_params.gui_enabled:
interface.server.stop_job(report)
printing.print_section('Evaluation precision and recall')
prc = PrecisionRecallCurve(pr_path, model.params, model_params, dataset_params)
test_datapoints = prc.get_curves_datapoints(optimization_params.batch_size, set_name="test")
valid_datapoints = prc.get_curves_datapoints(optimization_params.batch_size, set_name="valid")
#Stores the model params. Model can later be restored.
printing.print_section('Storing model parameters')
if visual_params.gui_enabled:
interface.server.send_precision_recall_data(test_datapoints, valid_datapoints)
storage.store_result(result_path, evaluator.events, test_datapoints, valid_datapoints)
