def do_convert(args, logdir1, logdir2):
# Load graph
model = Net2()
df = Net2DataFlow(hp.convert.data_path, hp.convert.batch_size)
ckpt1 = tf.train.latest_checkpoint(logdir1)
ckpt2 = '{}/{}'.format(logdir2, args.ckpt) if args.ckpt else tf.train.latest_checkpoint(logdir2)
session_inits = []
if ckpt2:
session_inits.append(SaverRestore(ckpt2))
if ckpt1:
session_inits.append(SaverRestore(ckpt1, ignore=['global_step']))
pred_conf = PredictConfig(
model=model,
input_names=get_eval_input_names(),
output_names=get_eval_output_names(),
session_init=ChainInit(session_inits))
predictor = OfflinePredictor(pred_conf)
audio, y_audio, ppgs = convert(predictor, df)
# Write the result
tf.summary.audio('A', y_audio, hp.default.sr, max_outputs=hp.convert.batch_size)
tf.summary.audio('B', audio, hp.default.sr, max_outputs=hp.convert.batch_size)
# Visualize PPGs
heatmap = np.expand_dims(ppgs, 3) # channel=1
tf.summary.image('PPG', heatmap, max_outputs=ppgs.shape[0])
writer = tf.summary.FileWriter(logdir2)
with tf.Session() as sess:
summ = sess.run(tf.summary.merge_all())
writer.add_summary(summ)
writer.close()
def eval(logdir1, logdir2):
# Load graph
model = Net2()
# dataflow
df = Net2DataFlow(hp.test2.data_path, hp.test2.batch_size)
ckpt1 = tf.train.latest_checkpoint(logdir1)
ckpt2 = tf.train.latest_checkpoint(logdir2)
session_inits = []
if ckpt2:
session_inits.append(SaverRestore(ckpt2))
if ckpt1:
session_inits.append(SaverRestore(ckpt1, ignore=['global_step']))
pred_conf = PredictConfig(model=model,
input_names=get_eval_input_names(),
output_names=get_eval_output_names(),
session_init=ChainInit(session_inits))
predictor = OfflinePredictor(pred_conf)
# x_mfccs, y_spec, _ = next(df().get_data())
summ_loss, = predictor(next(df().get_data()))
writer = tf.summary.FileWriter(logdir2)
writer.add_summary(summ_loss)
writer.close()
def do_convert(logdir1, logdir2, input_path, output_path):
# Load graph
model = Net2()
model.actual_duration = librosa.core.get_duration(filename=input_path,
sr=hp.default.sr)
# TODO isolate out logdirs, uhh and how to pre-dl from s3?
assert len(input_path) > 0, "must be non-empty input path"
df = Net2DataFlow(data_path=input_path, batch_size=1)
ckpt1 = tf.train.latest_checkpoint(logdir1)
ckpt2 = tf.train.latest_checkpoint(logdir2)
session_inits = []
session_inits.append(SaverRestore(ckpt2))
session_inits.append(SaverRestore(ckpt1, ignore=['global_step']))
pred_conf = PredictConfig(model=model,
input_names=get_eval_input_names(),
output_names=get_eval_output_names(),
session_init=ChainInit(session_inits))
predictor = OfflinePredictor(pred_conf)
audio, y_audio, ppgs = convert(predictor, df)
write_wav(audio[0], hp.default.sr, output_path)
def train(args, logdir1, logdir2):
# model
model = Net2()
preprocessing(data_path, logdir2)
# dataflow
df = Net2DataFlow(data_path, hp.train2.batch_size)
# set logger for event and model saver
logger.set_logger_dir(logdir2)
# session_conf = tf.ConfigProto(
# gpu_options=tf.GPUOptions(
# allow_growth=True,
# per_process_gpu_memory_fraction=0.6,
# ),
# )
dataset_size = len(glob.glob(data_path + '/wav/*.wav'))
print("\t\data_path : ", data_path)
print("\t\tDataset Size : ", dataset_size)
print("\t\tBatch Size : ", hp.train2.batch_size)
print("\t\tSteps per epoch : ", (dataset_size // hp.train2.batch_size))
from time import sleep
sleep(10)
session_inits = []
ckpt2 = '{}/{}'.format(
logdir2,
args.ckpt) if args.ckpt else tf.train.latest_checkpoint(logdir2)
if ckpt2:
session_inits.append(SaverRestore(ckpt2))
ckpt1 = tf.train.latest_checkpoint(logdir1)
if ckpt1:
session_inits.append(SaverRestore(ckpt1, ignore=['global_step']))
train_conf = AutoResumeTrainConfig(
model=model,
data=QueueInput(df(n_prefetch=1000, n_thread=8)),
callbacks=[
# TODO save on prefix net2
ModelSaver(checkpoint_dir=logdir2),
# ConvertCallback(logdir2, hp.train2.test_per_epoch),
],
max_epoch=hp.train2.num_epochs,
steps_per_epoch=dataset_size // hp.train2.batch_size,
session_init=ChainInit(session_inits))
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
train_conf.nr_tower = len(args.gpu.split(','))
gpu_list = args.gpu.split(',')
gpu_list = list(map(int, gpu_list))
#trainer = SimpleTrainer()
trainer = SyncMultiGPUTrainerReplicated(gpu_list)
#trainer = AsyncMultiGPUTrainer(gpu_list, False)
launch_train_with_config(train_conf, trainer=trainer)
def train(args, logdir1, logdir2):
# model
model = Net2()
# dataflow
df = Net2DataFlow(hp.train2.data_path, hp.train2.batch_size)
# set logger for event and model saver
logger.set_logger_dir(logdir2)
session_conf = tf.ConfigProto(
# log_device_placement=True,
allow_soft_placement=True,
gpu_options=tf.GPUOptions(
# allow_growth=True,
per_process_gpu_memory_fraction=0.6,
),
)
session_inits = []
ckpt2 = '{}/{}'.format(logdir2, args.ckpt) if args.ckpt else tf.train.latest_checkpoint(logdir2)
if ckpt2:
session_inits.append(SaverRestore(ckpt2))
ckpt1 = tf.train.latest_checkpoint(logdir1)
if ckpt1:
session_inits.append(SaverRestore(ckpt1, ignore=['global_step']))
train_conf = TrainConfig(
model=model,
data=QueueInput(df(n_prefetch=1000, n_thread=4)),
callbacks=[
# TODO save on prefix net2
ModelSaver(checkpoint_dir=logdir2),
# ConvertCallback(logdir2, hp.train2.test_per_epoch),
],
max_epoch=hp.train2.num_epochs,
steps_per_epoch=hp.train2.steps_per_epoch,
session_init=ChainInit(session_inits),
session_config=session_conf
)
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
train_conf.nr_tower = len(args.gpu.split(','))
#trainer = SyncMultiGPUTrainerParameterServer(hp.train2.num_gpu)
trainer = SimpleTrainer()
launch_train_with_config(train_conf, trainer=trainer)
def do_convert(args, logdir1, logdir2, input_dir):
# Load graph
model = Net2()
# input_dir = hp.convert.data_base_dir_original + hp.convert.data_path
df = Net2DataFlow(input_dir, hp.convert.batch_size)
ckpt1 = tf.train.latest_checkpoint(logdir1)
ckpt2 = '{}/{}'.format(
logdir2,
args.ckpt) if args.ckpt else tf.train.latest_checkpoint(logdir2)
session_inits = []
if ckpt2:
session_inits.append(SaverRestore(ckpt2))
if ckpt1:
session_inits.append(SaverRestore(ckpt1, ignore=['global_step']))
pred_conf = PredictConfig(model=model,
input_names=get_eval_input_names(),
output_names=get_eval_output_names(),
session_init=ChainInit(session_inits))
predictor = OfflinePredictor(pred_conf)
# loop over all the audio files
for wav_file in df.wav_files:
# check if file is present audio
out_path = wav_file.replace(hp.convert.data_base_dir_original,
hp.convert.data_base_dir_convert)
# change file extension from wv1/wv2 to wav
out_path = out_path[:-2] + 'av'
if os.path.isfile(out_path):
# file is already present, move on to the next one.
print("skipping " + wav_file)
continue
print("converting " + wav_file)
# convert audio
audio_len, feats = df.get_features(wav_file)
audio_full = []
for feat in feats:
input_arr = ([feat[0]], [feat[1]], [feat[2]])
audio, ppgs = convert(predictor, input_arr)
audio_full.append(
(audio[0] * hp.convert.amplitude_multiplier).astype(np.int16))
scipy.io.wavfile.write(out_path, hp.default.sr,
np.concatenate(audio_full)[:audio_len])
def train(args, logdir2):
# model
model = Net2()
# dataflow
df = Net2DataFlow(hp.train2.mel_path, hp.train2.ppgs_path,
hp.train2.batch_size)
session_inits = []
ckpt2 = '{}/{}'.format(
logdir2,
args.ckpt) if args.ckpt else tf.train.latest_checkpoint(logdir2)
if ckpt2:
session_inits.append(SaverRestore(ckpt2))
'''
ckpt1 = tf.train.latest_checkpoint(logdir1)
if ckpt1:
session_inits.append(SaverRestore(ckpt1, ignore=['global_step']))
'''
train_conf = TrainConfig(
model=model,
data=QueueInput(df(n_prefetch=1000, n_thread=4)),
callbacks=[
# TODO save on prefix net2
ModelSaver(checkpoint_dir=logdir2),
# ConvertCallback(logdir2, hp.train2.test_per_epoch),
],
max_epoch=hp.train2.num_epochs,
steps_per_epoch=hp.train2.steps_per_epoch,
session_init=ChainInit(session_inits))
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
train_conf.nr_tower = len(args.gpu.split(','))
trainer = SyncMultiGPUTrainerReplicated(hp.train2.num_gpu)
print("strated trainer")
launch_train_with_config(train_conf, trainer=trainer)