def test_01_model_train(self):
model_file = DIRECTORY_MODELS + 'arima.pickle'
date = '2018-11-20'
duration = 30
country = None
model(date, duration, country)
self.assertTrue(os.path.exists(model_file))
def train(model, train_dataloader, val_dataloader, optimizer,
criteriaOfEachLevel, n_ary, decode_level, eval_point, save_path):
total_batch = len(train_dataloader)
eval_point = int(eval_point * total_batch)
total_acc = 0
for i, batch in enumerate(train_dataloader):
batch_size = len(batch)
optimizer.zero_grad()
loss = 0
acc = 0
for j, sample in enumerate(batch):
# source to reference
output = model(sample["source"]["syntax"],
sample["reference"]["syntax"])
output = output.view(-1, output.size(2))
target = sample["reference"]["label"].to(cuda)
loss += computeLossByLevel(output, target,
sample["reference"]["level"],
criteriaOfEachLevel)
acc += ((output.argmax(1) == target).view(-1, n_ary).all(1)).sum()
# reference to source
output = model(sample["reference"]["syntax"],
sample["source"]["syntax"])
output = output.view(-1, output.size(2))
target = sample["source"]["label"].to(cuda)
loss += computeLossByLevel(output, target,
sample["source"]["level"],
criteriaOfEachLevel)
acc += ((output.argmax(1) == target).view(-1, n_ary).all(1)).sum()
total_acc += acc
acc = acc / (2 * batch_size * decode_level)
loss = loss / (2 * batch_size * decode_level)
print("Processing {:05d}/{} batch. loss: {:.5f} accuracy: {:.4f}\r".
format(i, total_batch, loss, acc),
end='')
loss.backward()
optimizer.step()
if i % eval_point == 0:
print("\nStart evaluation...\n")
eval_acc = evaluate(model, val_dataloader, n_ary, decode_level,
criteriaOfEachLevel)
path = os.path.join(
save_path,
"model_{}_{}_{:.3f}.pt".format(iteration, i / eval_point,
eval_acc))
torch.save(model.state_dict(), path)
print("Training Accuracy: {}".format(total_acc /
len(train_dataloader.dataset)))
def test_02_model_predict(self):
key = 'arima'
date = '2018-11-20'
duration = 30
country = None
result = model(date, duration, country)
self.assertTrue(key in result)
def step(hparams, tokens, past=None):
lm_output = model.model(hparams=hparams, X=tokens, past=past, reuse=tf.AUTO_REUSE)
logits = lm_output['logits'][:, :, :hparams.n_vocab]
presents = lm_output['present']
presents.set_shape(model.past_shape(hparams=hparams, batch_size=batch_size))
return {
'logits': logits,
'presents': presents,
}
def __init__(self,
batch_size=500,
lr=0.001,
training_times=500
):
"""
auto-encoder的实现
:param batch_size:每批图片的数量
:param lr: 学习率
:param training_times:epoch，训练次数
"""
self.batch_size = batch_size
self.lr = lr
self.training_times = training_times
self.device = t.device('cuda' if t.cuda.is_available() else 'cpu')
self.train_data, self.test_data = data_create(self.batch_size)
self.model = model().to(self.device)
self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr)
self.loss = MSELoss()
def predict(sentence, bpe_model, model):
model.eval()
if isinstance(sentence, str):
sentence = preprocess(sentence)
tokens = bpe_tokenizer(sentence)
else:
tokens = [int(token) for token in sentence]
src_indexes = tokens
# convert to tensor format
# since the inference is done on single sentence, batch size is 1
src_tensor = torch.LongTensor(src_indexes).unsqueeze(1).to(device)
# src_tensor => [seq_len, 1]
src_length = torch.LongTensor([len(src_indexes)])
# src_length => [1]
with torch.no_grad():
predictions = model(src_tensor, src_length)
return predictions
def predict():
# Check date parameter in request
if 'date' in request.args:
date = request.args['date']
else:
return "Error: No date parameter was provided."
# Check country parameter in request
if 'country' in request.args:
country = request.args['country']
else:
country = None
# Check duration parameter in request
if 'duration' in request.args:
duration = request.args['duration']
if duration == '':
duration = 30
else:
duration = int(duration)
else:
duration = 30
# Call model with parameters
result = model(date, duration, country)
# Return result
return jsonify({'data': result})
import matplotlib.pyplot as plt
import sys
import json
import time
import os
from OCC.Core.gp import gp_Pnt, gp_Vec, gp_Dir
from OCC.Core.gp import gp_Ax1, gp_Ax2, gp_Ax3
from OCC.Core.gp import gp_XYZ
from OCC.Core.gp import gp_Lin
from src.base import create_tempnum
from src.model import model, model_base
from src.jsonfile import write_json
obj = model(cfgfile="./cfg/model.json")
print(obj.cfg["name"])
md0 = obj.set_model(name="surf0")
md1 = obj.set_model(name="surf1")
md2 = obj.set_model(name="surf")
md2.rim = obj.make_PolyWire(skin=None)
md3 = model_base(meta={"name": "surf3"})
md3.axs.Translate(gp_Pnt(), gp_Pnt(0, 0, 100))
meta = {}
meta["surf1"] = md1.export_dict()
meta["surf2"] = md2.export_dict()
meta["surf3"] = md3.export_dict()
write_json(create_tempnum("model", obj.tmpdir, ext=".json"), meta)
emoji_to_pic = {
'happy': None,
'disgust': None,
'sad': None,
'surprise': None,
'fear': None,
'angry': None
}
# ATTENTION: CHANGE THE '\\' A/C TO YOUR OS
files = glob.glob(EMOJI_FILE_PATH + '\\*.png')
logger.info('loading the emoji png files in memory ...')
for file in tqdm.tqdm(files):
logger.debug('file path: {}'.format(file))
# ATTENTION: CHANGE THE '\\' A/C TO YOUR OS
emoji_to_pic[file.split('\\')[-1].split('.')[0]] = cv.imread(file, -1)
X = tf.placeholder(tf.float32, shape=[None, 48, 48, 1])
keep_prob = tf.placeholder(tf.float32)
y_conv = model(X, keep_prob)
saver = tf.train.Saver()
with tf.Session(config=config) as sess:
saver.restore(sess, os.path.join(CHECKPOINT_SAVE_PATH, 'model.ckpt'))
logger.info('Opening the camera for getting the video feed ...')
from_cam(sess)
def train(args):
if args.amp:
amp_handle = amp.init(enabled=args.fp16)
args.distributed = False
if 'WORLD_SIZE' in os.environ:
args.distributed = int(os.environ['WORLD_SIZE']) > 1
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend='nccl', init_method='env://')
args.N_gpu = torch.distributed.get_world_size()
else:
args.N_gpu = 1
dboxes = dboxes300_coco()
encoder = Encoder(dboxes)
cocoGt = get_coco_ground_truth(args)
ssd300 = model(args)
args.learning_rate = args.learning_rate * args.N_gpu * (args.batch_size / 32)
iteration = 0
loss_func = Loss(dboxes)
loss_func.cuda()
optimizer = torch.optim.SGD(
tencent_trick(ssd300),
lr=args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = MultiStepLR(
optimizer=optimizer,
milestones=args.multistep,
gamma=0.1)
if args.fp16:
if args.amp:
optimizer = amp_handle.wrap_optimizer(optimizer)
else:
optimizer = FP16_Optimizer(optimizer, static_loss_scale=128.)
val_dataloader, inv_map = get_val_dataloader(args)
train_loader = get_train_loader(args, dboxes)
acc = 0
logger = Logger(args.batch_size, args.local_rank)
for epoch in range(0, args.epochs):
logger.start_epoch()
scheduler.step()
iteration = train_loop(
ssd300, loss_func, epoch, optimizer,
train_loader, iteration, logger, args)
logger.end_epoch()
if epoch in args.evaluation:
acc = evaluate(ssd300, val_dataloader, cocoGt, encoder, inv_map, args)
if args.local_rank == 0:
print('Epoch {:2d}, Accuracy: {:4f} mAP'.format(epoch, acc))
if args.data_pipeline == 'dali':
train_loader.reset()
return acc, logger.average_speed()
def main():
image_root = os.path.join(cfg.DATA_DIR, cfg.DATASET, 'image')
datasetname = cfg.DATASET
dataset = Dataset(datasetname)
query_lists, _ = dataset.get_retrieval_list()
feature_file = open(os.path.join(cfg.FEATURE_DIR, cfg.DATASET, \
cfg.MODEL, cfg.DISTANCE_METRIC, \
cfg.SAVE_NAME), 'rb')
feature_dict = cPickle.load(feature_file)
save_root = os.path.join(cfg.RESULT_DIR, cfg.DATASET, cfg.MODEL,
cfg.DISTANCE_METRIC)
if not os.path.exists(save_root):
os.makedirs(save_root)
net = model(cfg.MODEL, cfg.CAFFEMODEL, cfg.PROTOTXT, cfg.FEATURE_NAME[0],
cfg.GPU_ID)
for idx, query_list in enumerate(query_lists):
query_name, query_id = query_list.strip().split(' ')
if query_name in feature_dict.keys():
q_feature = copy.deepcopy(feature_dict[query_name]['feature'])
q_id = copy.deepcopy(feature_dict[query_name]['id'])
else:
image_path = os.path.join(image_root, query_name + '.jpg')
image = cv2.imread(image_path)
q_feature = copy.deepcopy(net.forward(image, cfg.FEATURE_NAME[0]))
q_id = dataset.get_image_id(query_name)
retrieval_results = []
for key in feature_dict.keys():
p_feature = feature_dict[key]['feature']
p_id = feature_dict[key]['id']
if cfg.DISTANCE_METRIC == 'L2':
tmp = norm_L2_distance(q_feature, p_feature)
elif cfg.DISTANCE_METRIC == 'cosin':
tmp = norm_cosin_distance(q_feature, p_feature)
retrieval_results.append({
'filename': key,
'distance': tmp,
'id': p_id
})
retrieval_results.sort(lambda x, y: cmp(y['distance'], x['distance']))
#pdb.set_trace()
f = open(os.path.join(save_root, query_name + '.txt'), 'w')
# query image is also store in dataset
for i in range(0, cfg.TOP_K + 1):
filename = retrieval_results[i]['filename']
distance = retrieval_results[i]['distance']
f.writelines(
str(query_name) + '_' + str(filename) + ' ' + str(distance) +
'\n')
f.close()
view_bar(idx, len(query_lists))
def main():
# args = parser.parse_args()
args = Opts()
enc = encoder.get_encoder(args.model_name)
hparams = model.default_hparams()
with open(os.path.join('models', args.model_name, 'hparams.json')) as f:
# hparams.override_from_dict(json.load(f))
hparams.override_from_dict(json.loads(f.read()))
if args.sample_length > hparams.n_ctx:
raise ValueError("Can't get samples longer than window size: %s" %
hparams.n_ctx)
if args.model_name == '345M':
args.memory_saving_gradients = True
if args.optimizer == 'adam':
args.only_train_transformer_layers = True
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.graph_options.rewrite_options.layout_optimizer = rewriter_config_pb2.RewriterConfig.OFF
with tf.Session(config=config) as sess:
context = tf.placeholder(tf.int32, [args.batch_size, None])
context_in = randomize(context, hparams, args.noise)
output = model.model(hparams=hparams, X=context_in)
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=context[:, 1:], logits=output['logits'][:, :-1]))
if args.val_every > 0:
val_context = tf.placeholder(tf.int32, [args.val_batch_size, None])
val_output = model.model(hparams=hparams, X=val_context)
val_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=val_context[:, 1:],
logits=val_output['logits'][:, :-1]))
val_loss_summary = tf.summary.scalar('val_loss', val_loss)
tf_sample = sample.sample_sequence(hparams=hparams,
length=args.sample_length,
context=context,
batch_size=args.batch_size,
temperature=1.0,
top_k=args.top_k,
top_p=args.top_p)
all_vars = [v for v in tf.trainable_variables() if 'model' in v.name]
train_vars = [v for v in all_vars if '/h' in v.name
] if args.only_train_transformer_layers else all_vars
if args.optimizer == 'adam':
opt = tf.train.AdamOptimizer(learning_rate=args.learning_rate)
elif args.optimizer == 'sgd':
opt = tf.train.GradientDescentOptimizer(
learning_rate=args.learning_rate)
else:
exit('Bad optimizer:', args.optimizer)
if args.accumulate_gradients > 1:
if args.memory_saving_gradients:
exit(
"Memory saving gradients are not implemented for gradient accumulation yet."
)
opt = AccumulatingOptimizer(opt=opt, var_list=train_vars)
opt_reset = opt.reset()
opt_compute = opt.compute_gradients(loss)
opt_apply = opt.apply_gradients()
summary_loss = tf.summary.scalar('loss', opt_apply)
else:
if args.memory_saving_gradients:
opt_grads = memory_saving_gradients.gradients(loss, train_vars)
else:
opt_grads = tf.gradients(loss, train_vars)
opt_grads = list(zip(opt_grads, train_vars))
opt_apply = opt.apply_gradients(opt_grads)
summary_loss = tf.summary.scalar('loss', loss)
summary_lr = tf.summary.scalar('learning_rate', args.learning_rate)
summaries = tf.summary.merge([summary_lr, summary_loss])
summary_log = tf.summary.FileWriter(
os.path.join(CHECKPOINT_DIR, args.run_name))
saver = tf.train.Saver(var_list=all_vars,
max_to_keep=5,
keep_checkpoint_every_n_hours=2)
sess.run(tf.global_variables_initializer())
if args.restore_from == 'latest':
ckpt = tf.train.latest_checkpoint(
os.path.join(CHECKPOINT_DIR, args.run_name))
if ckpt is None:
# Get fresh GPT weights if new run.
ckpt = tf.train.latest_checkpoint(
os.path.join('models', args.model_name))
elif args.restore_from == 'fresh':
ckpt = tf.train.latest_checkpoint(
os.path.join('models', args.model_name))
else:
ckpt = tf.train.latest_checkpoint(args.restore_from)
print('Loading checkpoint', ckpt)
saver.restore(sess, ckpt)
print('Loading dataset...')
chunks = load_dataset(enc,
args.dataset,
args.combine,
encoding=args.encoding)
data_sampler = Sampler(chunks)
if args.val_every > 0:
if args.val_dataset:
val_chunks = load_dataset(enc,
args.val_dataset,
args.combine,
encoding=args.encoding)
else:
val_chunks = chunks
print('dataset has', data_sampler.total_size, 'tokens')
print('Training...')
if args.val_every > 0:
# Sample from validation set once with fixed seed to make
# it deterministic during training as well as across runs.
val_data_sampler = Sampler(val_chunks, seed=1)
val_batches = [[
val_data_sampler.sample(1024)
for _ in range(args.val_batch_size)
] for _ in range(args.val_batch_count)]
counter = 1
counter_path = os.path.join(CHECKPOINT_DIR, args.run_name, 'counter')
if os.path.exists(counter_path):
# Load the step number if we're resuming a run
# Add 1 so we don't immediately try to save again
with open(counter_path, 'r') as fp:
counter = int(fp.read()) + 1
def save():
maketree(os.path.join(CHECKPOINT_DIR, args.run_name))
print(
'Saving',
os.path.join(CHECKPOINT_DIR, args.run_name,
'model-{}').format(counter))
saver.save(sess,
os.path.join(CHECKPOINT_DIR, args.run_name, 'model'),
global_step=counter)
with open(counter_path, 'w') as fp:
fp.write(str(counter) + '\n')
def generate_samples():
print('Generating samples...')
context_tokens = data_sampler.sample(1)
all_text = []
index = 0
while index < args.sample_num:
out = sess.run(
tf_sample,
feed_dict={context: args.batch_size * [context_tokens]})
for i in range(min(args.sample_num - index, args.batch_size)):
text = enc.decode(out[i])
text = '======== SAMPLE {} ========\n{}\n'.format(
index + 1, text)
all_text.append(text)
index += 1
print(text)
maketree(os.path.join(SAMPLE_DIR, args.run_name))
with open(os.path.join(SAMPLE_DIR, args.run_name,
'samples-{}').format(counter),
'w',
encoding=args.encoding) as fp:
fp.write('\n'.join(all_text))
def validation():
print('Calculating validation loss...')
losses = []
for batch in tqdm.tqdm(val_batches):
losses.append(
sess.run(val_loss, feed_dict={val_context: batch}))
v_val_loss = np.mean(losses)
v_summary = sess.run(val_loss_summary,
feed_dict={val_loss: v_val_loss})
summary_log.add_summary(v_summary, counter)
summary_log.flush()
print('[{counter} | {time:2.2f}] validation loss = {loss:2.2f}'.
format(counter=counter,
time=time.time() - start_time,
loss=v_val_loss))
def sample_batch():
return [data_sampler.sample(1024) for _ in range(args.batch_size)]
avg_loss = (0.0, 0.0)
start_time = time.time()
try:
while True:
if counter % args.save_every == 0:
save()
if counter % args.sample_every == 0:
generate_samples()
if args.val_every > 0 and (counter % args.val_every == 0
or counter == 1):
validation()
if args.accumulate_gradients > 1:
sess.run(opt_reset)
for _ in range(args.accumulate_gradients):
sess.run(opt_compute,
feed_dict={context: sample_batch()})
(v_loss, v_summary) = sess.run((opt_apply, summaries))
else:
(_, v_loss, v_summary) = sess.run(
(opt_apply, loss, summaries),
feed_dict={context: sample_batch()})
summary_log.add_summary(v_summary, counter)
avg_loss = (avg_loss[0] * 0.99 + v_loss,
avg_loss[1] * 0.99 + 1.0)
print(
'[{counter} | {time:2.2f}] loss={loss:2.2f} avg={avg:2.2f}'
.format(counter=counter,
time=time.time() - start_time,
loss=v_loss,
avg=avg_loss[0] / avg_loss[1]))
counter += 1
except KeyboardInterrupt:
print('interrupted')
save()
def train_main(dataset,
model_name='117M',
seed=None,
batch_size=1,
sample_length=1023,
sample_num=50,
sample_every=100,
run_name='dnd_biographies08',
restore_from='latest',
mode="test",
max_iterations=50000,
loss_threshold=0.8,
save_every=1000):
enc = encoder.get_encoder(model_name)
hparams = model.default_hparams()
with open(os.path.join('models', model_name, 'hparams.json')) as f:
hparams.override_from_dict(json.load(f))
if sample_length is None:
sample_length = hparams.n_ctx // 2
elif sample_length > hparams.n_ctx:
raise ValueError("Can't get samples longer than window size: %s" %
hparams.n_ctx)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
context = tf.placeholder(tf.int32, [batch_size, None])
np.random.seed(seed)
tf.set_random_seed(seed)
output = model.model(hparams=hparams, X=context)
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=context[:, 1:], logits=output['logits'][:, :-1]))
tf_sample = sample.sample_sequence(hparams=hparams,
length=sample_length,
context=context,
batch_size=batch_size,
temperature=1.0,
top_k=40)
train_vars = [v for v in tf.trainable_variables() if 'model' in v.name]
opt = tf.train.AdamOptimizer(1e-4).minimize(loss, var_list=train_vars)
saver = tf.train.Saver(var_list=train_vars,
max_to_keep=5,
keep_checkpoint_every_n_hours=2)
sess.run(tf.global_variables_initializer())
if restore_from == 'latest':
ckpt = tf.train.latest_checkpoint(
os.path.join(CHECKPOINT_DIR, run_name))
if ckpt is None:
# Get fresh GPT weights if new run.
ckpt = tf.train.latest_checkpoint(
os.path.join('models', model_name))
elif restore_from == 'fresh':
ckpt = tf.train.latest_checkpoint(
os.path.join('models', model_name))
else:
ckpt = tf.train.latest_checkpoint(restore_from)
print('Loading checkpoint', ckpt)
saver.restore(sess, ckpt)
print('Loading dataset...')
chunks = load_dataset(enc, dataset)
data_sampler = Sampler(chunks)
print('dataset has', data_sampler.total_size, 'tokens')
print('Training...')
counter = 1
if os.path.exists(os.path.join(CHECKPOINT_DIR, run_name, 'counter')):
# Load the step number if we're resuming a run
# Add 1 so we don't immediately try to save again
with open(os.path.join(CHECKPOINT_DIR, run_name, 'counter'),
'r') as fp:
counter = int(fp.read()) + 1
def save():
maketree(os.path.join(CHECKPOINT_DIR, run_name))
print(
'Saving',
os.path.join(CHECKPOINT_DIR, run_name,
'model-{}').format(counter))
saver.save(sess,
os.path.join(CHECKPOINT_DIR, run_name, 'model'),
global_step=counter)
with open(os.path.join(CHECKPOINT_DIR, run_name, 'counter'),
'w') as fp:
fp.write(str(counter) + '\n')
def generate_samples():
context_tokens = data_sampler.sample(1)
all_text = []
index = 0
while index < sample_num or sample_num == 0:
out = sess.run(
tf_sample,
feed_dict={context: batch_size * [context_tokens]})
for i in range(min(sample_num - index, batch_size)):
text = enc.decode(out[i])
text = '======== SAMPLE {} ========\n{}\n'.format(
index + 1, text)
all_text.append(text)
index += 1
print(text)
# print(''.join(all_text))
maketree(os.path.join(SAMPLE_DIR, run_name))
with open(
os.path.join(SAMPLE_DIR, run_name,
'samples-{}').format(counter), 'w') as fp:
fp.write('\n'.join(all_text))
avg_loss = (0.0, 0.0)
start_time = time.time()
try:
if mode == "train":
while True and counter <= max_iterations:
if counter % save_every == 0:
save()
if counter % sample_every == 0:
generate_samples()
batch = [
data_sampler.sample(1024) for _ in range(batch_size)
]
_, lv = sess.run((opt, loss), feed_dict={context: batch})
avg_loss = (avg_loss[0] * 0.99 + lv,
avg_loss[1] * 0.99 + 1.0)
print(
'[{counter} | {time:2.2f}] loss={loss:2.2f} avg={avg:2.2f}'
.format(counter=counter,
time=time.time() - start_time,
loss=lv,
avg=avg_loss[0] / avg_loss[1]))
counter += 1
if counter > 100:
if (avg_loss[0] / avg_loss[1]) < loss_threshold:
counter = max_iterations + 1
else:
generate_samples()
except KeyboardInterrupt:
print('interrupted')
save()
parser.add_argument('--fp16-mode',
type=str,
default='off',
choices=['off', 'static', 'amp'],
help='Half precission mode to use')
opt = parser.parse_args()
if opt.fp16_mode != 'off':
opt.fp16 = True
opt.amp = (opt.fp16_mode == 'amp')
else:
opt.fp16 = False
opt.amp = False
if opt.amp:
amp_handle = amp.init(enabled=opt.fp16)
model = model(opt)
optimizer = torch.optim.SGD(model.parameters(), lr=opt.lr, momentum=0.9)
model = torch.nn.DataParallel(model, device_ids=[0, 1, 2, 3])
if opt.fp16:
print("INFO: Use Fp16")
if opt.amp:
model, optimizer = amp.initialize(model, optimizer, opt_level="O2")
# optimizer = amp_handle.wrap_optimizer(optimizer)
else:
optimizer = FP16_Optimizer(optimizer, static_loss_scale=128.)
# Prepare dataset
print("INFO: Prepare Datasets")
if opt.data_pipeline == 'dali':
eii = ExternalInputIterator(opt.batch_size, opt.img_path,
def main():
args = parser.parse_args()
if args.dataset == 'lpd_5':
tracks = ['Drums', 'Piano', 'Guitar', 'Bass', 'Strings']
elif args.dataset == 'lpd_17':
tracks = ['Drums', 'Piano', 'Chromatic Percussion', 'Organ', 'Guitar', 'Bass', 'Strings', 'Ensemble', 'Brass', 'Reed', 'Pipe', 'Synth Lead', 'Synth Pad', 'Synth Effects', 'Ethnic', 'Percussive', 'Sound Effects']
else:
print('invalid dataset name.')
exit()
trc_len = len(tracks)
note_size = 84
time_note = note_size*trc_len + 1
end_note = note_size*trc_len + 2
hparams = HParams(**{
"n_vocab": end_note+1,
"n_ctx": 1024,
"n_embd": 768,
"n_head": 12,
"n_layer": 12
})
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.visible_device_list = args.gpu
config.graph_options.rewrite_options.layout_optimizer = rewriter_config_pb2.RewriterConfig.OFF
with tf.Session(config=config) as sess:
context = tf.placeholder(tf.int32, [args.batch_size, None])
context_in = randomize(context, hparams, args.noise)
output = model.model(hparams=hparams, X=context_in)
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=context[:, 1:], logits=output['logits'][:, :-1]))
train_vars = [v for v in tf.trainable_variables() if 'model' in v.name]
opt = tf.train.AdamOptimizer(learning_rate=args.learning_rate)
opt_grads = tf.gradients(loss, train_vars)
opt_grads = list(zip(opt_grads, train_vars))
opt_apply = opt.apply_gradients(opt_grads)
summary_loss = tf.summary.scalar('loss', loss)
summary_lr = tf.summary.scalar('learning_rate', args.learning_rate)
summaries = tf.summary.merge([summary_lr, summary_loss])
summary_log = tf.summary.FileWriter(
os.path.join(CHECKPOINT_DIR, args.run_name))
saver = tf.train.Saver(
var_list=train_vars,
max_to_keep=5,
keep_checkpoint_every_n_hours=2)
sess.run(tf.global_variables_initializer())
if args.restore_from == 'latest':
ckpt = tf.train.latest_checkpoint(
os.path.join(CHECKPOINT_DIR, args.run_name))
print('Loading checkpoint', ckpt)
saver.restore(sess, ckpt)
elif args.restore_from != 'fresh':
ckpt = tf.train.latest_checkpoint(args.restore_from)
print('Loading checkpoint', ckpt)
saver.restore(sess, ckpt)
print('Loading dataset...')
chunks = load_dataset(args.input)
data_sampler = Sampler(chunks)
print('dataset has', data_sampler.total_size, 'tokens')
print('Training...')
counter = 1
counter_path = os.path.join(CHECKPOINT_DIR, args.run_name, 'counter')
if os.path.exists(counter_path):
# Load the step number if we're resuming a run
# Add 1 so we don't immediately try to save again
with open(counter_path, 'r') as fp:
counter = int(fp.read()) + 1
def save():
maketree(os.path.join(CHECKPOINT_DIR, args.run_name))
print(
'Saving',
os.path.join(CHECKPOINT_DIR, args.run_name,
'model-{}').format(counter))
saver.save(
sess,
os.path.join(CHECKPOINT_DIR, args.run_name, 'model'),
global_step=counter)
with open(counter_path, 'w') as fp:
fp.write(str(counter) + '\n')
def sample_batch():
return [data_sampler.sample(1024) for _ in range(args.batch_size)]
avg_loss = (0.0, 0.0)
start_time = time.time()
try:
while True:
if counter % args.save_every == 0:
save()
(_, v_loss, v_summary) = sess.run(
(opt_apply, loss, summaries),
feed_dict={context: sample_batch()})
summary_log.add_summary(v_summary, counter)
avg_loss = (avg_loss[0] * 0.99 + v_loss,
avg_loss[1] * 0.99 + 1.0)
print(
'[{counter} | {time:2.2f}] loss={loss:2.2f} avg={avg:2.2f}'
.format(
counter=counter,
time=time.time() - start_time,
loss=v_loss,
avg=avg_loss[0] / avg_loss[1]))
counter += 1
except KeyboardInterrupt:
print('interrupted')
save()
from src.utils import Timer
from src.dataset import Dataset
import copy
if __name__ == '__main__':
dataset = cfg.DATASET
layername = cfg.FEATURE_NAME[0]
modelname = cfg.MODEL
caffemodel = cfg.CAFFEMODEL
prototxt = cfg.PROTOTXT
gpu_id = cfg.GPU_ID
print('model: {}, caffemodel: {}, prototxt: {}'.format(
modelname, caffemodel, prototxt))
net = model(modelname, caffemodel, prototxt, layername, gpu_id)
dataset = Dataset(dataset)
image_iters = dataset.get_image_list()
#image_iters = image_iters[0:100]
for idx, image_iter in enumerate(image_iters):
image_name = image_iter.split('.')[0]
image_id = dataset.get_image_id(image_name)
if image_id:
image_path = os.path.join(dataset.data_path, image_name + '.jpg')
image = cv2.imread(image_path)
if args.input_profile is not None:
try:
input_profile = pd.read_csv(args.input_profile)
logger.info('Input profile data loaded from %s', args.input)
except FileNotFoundError as e:
logger.error('Input profile file %s not found', args.input)
raise SystemExit("Provide valid path!")
if args.step == 'upload':
upload_to_s3(args.lfp, **config['s3_upload'])
if args.step == 'download':
download_from_s3(args.lfp, **config['s3_download'])
elif args.step == 'clean':
output = clean_base(input, **config['clean'])
elif args.step == 'featurize':
output = featurize(input, input_profile, args.lfp,
**config['featurize'])
elif args.step == 'model':
model(input, args.lfp, **config['model'])
elif args.step == 'score':
output = scoring(args.lfp, **config['score'])
elif args.step == 'create_db':
if args.truncate:
create_score_db(input, 1, **config['database'])
else:
create_score_db(input, 0, **config['database'])
if args.output is not None and output is not None:
output.to_csv(args.output, index=False)
logger.info("Output saved to %s" % args.output)
def train(sess,
data,
labels,
steps,
run_name,
batch_size=1,
n_heads=None,
n_layers=None,
learning_rate=0.0001,
print_each=1,
save_every=1000,
accumulate=5,
use_class_entropy=False,
model_path="checkpoint/"):
model_path = os.path.join(model_path, run_name)
if not os.path.exists(model_path):
os.mkdir(model_path)
new_run = 'counter' not in os.listdir(model_path)
hparams = model.default_hparams()
#Set HyperParams
if n_layers: hparams.n_layer = n_layers
if n_heads: hparams.n_head = n_heads
if os.path.exists(model_path + "/hparams.json"):
with open(os.path.join(model_path, 'hparams.json')) as f:
hparams.override_from_dict(json.load(f))
#Spectrogram dimensions
d_shape = np.shape(data)
print(d_shape)
hparams.n_timestep = d_shape[1]
hparams.n_freq = d_shape[2]
hparams.n_cat = len(labels[0])
#Create TF graph
inp_specs = tf.placeholder(
tf.float32, [batch_size, hparams.n_timestep, hparams.n_freq])
logits = model.model(hparams, inp_specs, reuse=tf.AUTO_REUSE)
#Loss tensor = Softmax cross entropy
label_exp = tf.placeholder(tf.int8, [batch_size, hparams.n_cat])
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(labels=label_exp,
logits=logits['logits']))
all_vars = [v for v in tf.trainable_variables() if 'model' in v.name]
print("Using {} Parameter Network".format(str(len(all_vars))))
lr = tf.placeholder(tf.float32)
if accumulate > 1:
#Train step using AdamOtimizer with Accumulating gradients
opt = AccumulatingOptimizer(
opt=tf.train.AdamOptimizer(learning_rate=lr), var_list=all_vars)
opt_reset = opt.reset()
opt_compute = opt.compute_gradients(loss)
opt_apply = opt.apply_gradients()
else:
opt = tf.train.AdamOptimizer(learning_rate=lr)
opt_grads = tf.gradients(loss, all_vars)
opt_grads = list(zip(opt_grads, all_vars))
opt_apply = opt.apply_gradients(opt_grads)
#Create saveable graph and checkpoint + counter
saver = tf.train.Saver(var_list=all_vars)
sess.run(tf.global_variables_initializer())
if new_run:
saver.save(sess, model_path + "/{}.ckpt".format(run_name))
ckpt = tf.train.latest_checkpoint(model_path)
print('Restoring checkpoint', ckpt)
saver.restore(sess, ckpt)
#Training SetUp
#Get counter
counter = 1
counter_path = os.path.join(model_path, 'counter')
if os.path.exists(counter_path):
with open(counter_path, 'r') as fp:
counter = int(fp.read()) + 1
counter_base = counter
def save():
print('Saving',
os.path.join(model_path, 'model-{}').format(counter - 1))
saver.save(sess,
os.path.join(model_path, 'model'),
global_step=counter - 1)
with open(counter_path, 'w') as fp:
fp.write(str(counter - 1) + '\n')
def next_batch(num, data, lab):
'''
Return a total of `num` random samples and labels.
'''
idx = np.arange(0, len(data))
np.random.shuffle(idx)
idx = idx[:num]
data_shuffle = [data[i] for i in idx]
labels_shuffle = [lab[i] for i in idx]
return np.asarray(data_shuffle), np.asarray(labels_shuffle)
avg_loss = (0.0, 0.0)
start_time = time.time()
def class_entropy(y):
y = np.sum(y, 0)
e = sum([(i / sum(y)) * np.log(i / sum(y)) if i > 0 else 0 for i in y])
return np.abs(1 - (-np.log(1 / len(y)) + e))
try:
while counter < (counter_base + steps):
if (counter - 1) % save_every == 0 and counter > 1:
save()
# Get batch of specified size
x, lab = next_batch(batch_size, data, labels)
lrate = learning_rate * class_entropy(
lab) if use_class_entropy else learning_rate
if accumulate > 1:
sess.run(opt_reset)
#Run Gradient accumulation steps
for _ in range(accumulate):
sess.run(opt_compute,
feed_dict={
inp_specs: x,
label_exp: lab
})
else:
_, v_loss = sess.run((opt_apply, loss),
feed_dict={
inp_specs: x,
label_exp: lab,
lr: lrate,
"model/drop:0": 1.0
})
avg_loss = (avg_loss[0] * 0.99 + v_loss, avg_loss[1] * 0.99 + 1.0)
print(
'[{counter} | {time:2.2f}] loss={loss:2.2f} avg={avg:2.2f} lrate={lrate}'
.format(counter=counter,
time=time.time() - start_time,
loss=v_loss,
avg=avg_loss[0] / avg_loss[1],
lrate=str(lrate)))
if counter % print_each == 0:
sample = next_batch(batch_size, data, labels)
out = sess.run(logits,
feed_dict={
inp_specs: sample[0],
"model/drop:0": 1.0
})
acc = sum(
np.argmax(np.asarray(out['logits']), axis=1) == np.argmax(
sample[1], axis=1)) / batch_size
print("[Summary Step] Accuracy {}% for {} distribution".format(
str(acc * 100), str(np.sum(sample[1], 0))))
print("Class Entropy: {}".format(str(class_entropy(
sample[1]))))
counter += 1
save()
except KeyboardInterrupt:
print('interrupted')
save()
def predict(sess,
data,
run_name,
batch_size,
num_categories,
category_names,
model_path="checkpoint/"):
model_path = os.path.join(model_path, run_name)
# Load Hyperparams from model
hparams = model.default_hparams()
if os.path.exists(model_path + "/hparams.json"):
with open(os.path.join(model_path, 'hparams.json')) as f:
hparams.override_from_dict(json.load(f))
d_shape = np.shape(data)
print("Precicting for data: " + str(d_shape))
hparams.n_timestep = d_shape[1]
hparams.n_freq = d_shape[2]
hparams.n_cat = num_categories
# Create TF graph
inp_specs = tf.placeholder(
tf.float32, [batch_size, hparams.n_timestep, hparams.n_freq])
prediction = model.model(hparams, inp_specs)
# Get Model vars
all_vars = [v for v in tf.trainable_variables() if 'model' in v.name]
saver = tf.train.Saver(var_list=all_vars)
sess.run(tf.global_variables_initializer())
ckpt = tf.train.latest_checkpoint(model_path)
saver.restore(sess, ckpt)
predictions = np.zeros((len(data), num_categories))
num_batches = int(np.ceil(len(data) / batch_size))
for i in tqdm(range(num_batches)):
c = batch_size
if i * batch_size + c > len(data):
add = (i * batch_size + c) - len(data)
pred = sess.run(prediction,
feed_dict={
inp_specs:
np.concatenate((data[i * batch_size:],
np.zeros(
(add, hparams.n_timestep,
hparams.n_freq)))),
"model/drop:0":
1.0
})['logits']
predictions[i * batch_size:] = pred[:-add]
else:
predictions[i*batch_size: i*batch_size+c] =\
sess.run(prediction, feed_dict={inp_specs: data[i*batch_size: i*batch_size+batch_size],
"model/drop:0": 1.0})['logits']
cats = np.argmax(predictions, axis=1)
return {
"raw": predictions,
"category": cats,
"predictName": ["N", "S", "V", "F", "Q"],
"names": category_names
}
def test(model_name, test_audio_name):
csv_test_audio = csv_dir + test_audio_name + '/'
init, net1_optim, net2_optim, all_optim, x, x_face_id, y_landmark, y_phoneme, y_lipS, y_maya_param, dropout, cost, \
tensorboard_op, pred, clear_op, inc_op, avg, batch_size_placeholder, phase = model()
# start tf graph
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
max_to_keep = 20
saver = tf.train.Saver(max_to_keep=max_to_keep)
try_mkdir(pred_dir)
# Test sess, load ckpt
OLD_CHECKPOINT_FILE = model_dir + model_name + '/' + model_name +'.ckpt'
saver.restore(sess, OLD_CHECKPOINT_FILE)
print("Model loaded: " + model_dir + model_name)
total_epoch_num = 1
print(csv_test_audio)
data_dir = {'train': {}, 'test': {}}
data_dir['test']['wav'] = open(csv_test_audio + "test/wav.csv", 'r')
data_dir['test']['clip_len'] = open(csv_test_audio + "test/clip_len.csv", 'r')
cv_file_len = simple_read_clip_len(data_dir['test']['clip_len'])
print('Loading wav_raw.txt file in {:}'.format(csv_test_audio))
train_wav_raw = np.loadtxt(csv_test_audio + 'wav_raw.csv')
test_wav_raw = train_wav_raw
for epoch in range(0, total_epoch_num):
# clear data file header
# ============================== TRAIN SET CHUNK ITERATION ============================== #
sess.run(clear_op)
for key in ['train', 'test']:
for lpw_key in data_dir[key].keys():
data_dir[key][lpw_key].seek(0)
print("===================== TEST/CV CHUNK - {:} ======================".format(csv_test_audio))
eof = False
chunk_num = 0
chunk_size_sum = 0
batch_size = test_wav_raw.shape[0]
chunk_size = batch_size * batch_per_chunk_size
while (not eof):
cv_data, eof = read_chunk_data(data_dir, 'test', chunk_size)
chunk_num += 1
chunk_size_sum += len(cv_data['wav'])
print('Load Chunk {:d}, size {:d}, total_size {:d} ({:2.2f})'
.format(chunk_num, len(cv_data['wav']), chunk_size_sum, chunk_size_sum / cv_file_len))
full_idx_array = np.arange(len(cv_data['wav']))
# np.random.shuffle(full_idx_array)
for next_idx in range(0, int(np.floor(len(cv_data['wav']) / batch_size))):
batch_idx_array = full_idx_array[next_idx * batch_size: (next_idx + 1) * batch_size]
batch_x, batch_x_face_id, batch_x_pose, batch_y_landmark, batch_y_phoneme, batch_y_lipS, batch_y_maya_param = \
read_next_batch_easy_from_raw(test_wav_raw, cv_data, 'face_close', batch_idx_array, batch_size, n_steps, n_input, n_landmark,
n_phoneme, n_face_id)
npClose = np.loadtxt(lpw_dir + 'saved_param/maya_close_face.txt')
batch_x_face_id = np.tile(npClose, (batch_x_face_id.shape[0], 1))
test_pred, loss, _ = sess.run([pred, cost, inc_op],
feed_dict={x: batch_x,
x_face_id: batch_x_face_id,
y_landmark: batch_y_landmark,
y_phoneme: batch_y_phoneme,
y_lipS: batch_y_lipS,
dropout: 0,
batch_size_placeholder: batch_x.shape[0],
phase: 0,
y_maya_param: batch_y_maya_param})
def save_output(filename, npTxt, fmt):
f = open(filename, 'wb')
np.savetxt(f, npTxt, fmt=fmt)
f.close()
try_mkdir(pred_dir + test_audio_name)
def sigmoid(x):
return 1/(1+np.exp(-x))
save_output(pred_dir + test_audio_name + "/mayaparam_pred_cls.txt",
np.concatenate([test_pred['jali'], sigmoid(test_pred['v_cls'])], axis=1), '%.4f')
save_output(pred_dir + test_audio_name + "/mayaparam_pred_reg.txt",
np.concatenate([test_pred['jali'], test_pred['v_reg']], axis=1), '%.4f')
def evaluate(model,
dataloader,
n_ary,
decode_level,
criteriaOfEachLevel,
printResult=False):
def outputAccuracyEachLevel(accuracyEachLevel):
out = []
for level, acc in enumerate(accuracyEachLevel):
out.append("Level {}: {}, ".format(level, acc))
return ' '.join(out)
model.eval()
loss = 0
correctEachLevel = [0] * decode_level
countEachLevel = [0] * decode_level
with torch.no_grad():
for batch in tqdm(dataloader):
for sample in batch:
# source to reference
output = model(sample["source"]["syntax"],
sample["reference"]["syntax"])
output = output.view(-1, output.size(2))
target = sample["reference"]["label"].to(cuda)
loss += computeLossByLevel(output, target,
sample["reference"]["level"],
criteriaOfEachLevel)
accumulateAccuracyEachLayer(
correctEachLevel=correctEachLevel,
countEachLevel=countEachLevel,
predictIndexes=output.argmax(1),
target=target,
indexesEachLevel=sample["reference"]["level"],
n_ary=n_ary)
if printResult:
print(
"Target:",
indexesToSymbols(target, total_symbols,
sample["reference"]["level"]))
print(
"Predict:",
indexesToSymbols(output.argmax(1), total_symbols,
sample["reference"]["level"]))
print()
# reference to source
output = model(sample["reference"]["syntax"],
sample["source"]["syntax"])
output = output.view(-1, output.size(2))
target = sample["source"]["label"].to(cuda)
loss += computeLossByLevel(output, target,
sample["source"]["level"],
criteriaOfEachLevel)
accumulateAccuracyEachLayer(
correctEachLevel=correctEachLevel,
countEachLevel=countEachLevel,
predictIndexes=output.argmax(1),
target=target,
indexesEachLevel=sample["source"]["level"],
n_ary=n_ary)
if printResult:
print(
"Target:",
indexesToSymbols(target, total_symbols,
sample["source"]["level"]))
print(
"Predict:",
indexesToSymbols(output.argmax(1), total_symbols,
sample["source"]["level"]))
print()
loss = loss / (2 * len(dataloader) * dataloader.batch_size * decode_level)
accuracyEachLevel = []
for correct, count in zip(correctEachLevel, countEachLevel):
accuracyEachLevel.append(correct / count)
print("Evaluation Results. loss: {:.5f} accuracy at each level: {}".format(
loss, outputAccuracyEachLevel(accuracyEachLevel)))
model.train()
acc = torch.tensor(correctEachLevel).sum() / torch.tensor(
countEachLevel).sum()
return acc
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
device_str = "cuda:0" if torch.cuda.is_available() else "cpu"
print("Device: %s\n" % device_str)
device = torch.device(device_str)
# Hyperparameter for Cutmix
cutmix_beta = 0.3
# Hyperparameter
epochs = 100
lr = 0.01
train_loader, valid_loader = data.load_data(batch_size=64)
print("Train samples: %d" % len(train_loader.dataset))
print("Valid samples: %d" % len(valid_loader.dataset))
model = model.model()
model = model.to(device)
criterion_lss1 = nn.BCELoss()
criterion_lss2 = nn.KLDivLoss(reduction='batchmean')
criterion_ce = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=lr)
time_str = time.strftime("%m_%d-%Hh%Mm%Ss", time.localtime())
file = open("../log/%s.csv" % time_str, 'w')
writer = csv.writer(file)
headers = [
"train_loss", "train_acc", "train_lsl", "train_lss_1", "train_lss_2",
"train_lsd", "valid_loss", "valid_acc", "valid_lsl", "valid_lss_1",
"valid_lss_2", "valid_lsd"
epoch_range = trange(epoch, desc='Loss: {1.00000}', leave=True)
model.train().to(device)
# model.load_state_dict(torch.load('modelfiles/yeet.mdl'))
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
norm = t.normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
for e in epoch_range:
epoch_loss = []
for image, data_dict in data:
# c = torch.zeros((1, image.shape[1], image.shape[2]), device=device)
# image = torch.cat((image, image, image), dim=0)
image = norm({'image': image})['image']
optimizer.zero_grad()
loss = model(image.unsqueeze(0), [data_dict])
losses = 0
for key in loss:
losses += loss[key]
losses.backward()
epoch_loss.append(losses.item())
optimizer.step()
epoch_range.desc = 'Loss: ' + '{:.5f}'.format(
torch.tensor(epoch_loss).mean().item())
torch.save(model.state_dict(), 'modelfiles/yeet.mdl')
with torch.no_grad():
model.eval()
feature_extractor.to(device).eval()
reduced_images = torch.zeros((1, 2048), device=device)
