def split(self, validation_split):
"""Split dataset into traning and validaiton.
Arguments:
validation_split (float): Float between 0 and 1. Fraction of the dataset to be used
as validation data.
"""
val_size = int(validation_split * len(self))
train_size = len(self) - val_size
ds = []
if self.enable_pkbar:
message = (
'=> processing, splitting and caching dataset into cpu memory: \n'
+
'total batches: {}, train batches: {}, val batches: {}, batch_size: {}'
)
pbar = pkbar.Pbar(
message.format(len(self), train_size, val_size,
self.batch_size), len(self))
for i, _ in enumerate(self):
ds.append(self[i])
if self.enable_pkbar:
pbar.update(i)
train_dataset = ds[:train_size]
val_dataset = ds[train_size:]
return train_dataset, val_dataset
def initialize_H(five_tuple, k):
s, a, P_mat, R_vec, gamma = five_tuple
h = torch.zeros([s * a] * k, dtype=torch.float32)
pbar = pkbar.Pbar(name='initialize H, k=' + str(k), target=(s * a))
for i in range(s * a):
pbar.update(i)
in_edge = torch.sum(P_mat[..., i // a])
if k == 1:
h[i] = 1
if k == 2:
for j in range(s * a):
h[i, j] = P_mat[i, j]
if k == 3:
for j in range(s * a):
for l in range(s * a):
h[i, j, l] = P_mat[i, j] * P_mat[j, l]
if k == 4:
for j in range(s * a):
for l in range(s * a):
for m in range(s * a):
h[i, j, l, m] = P_mat[i, j] * P_mat[j, l] * P_mat[l, m]
h[i, ...] *= in_edge
for n in range(s * a):
h[..., n] *= R_vec[n]
h.requires_grad = False
return tn.Node(h * gamma**(k - 1), backend=backend)
def get_embedding_matrix (key, tokenizer, experiment, dataset, embedding_dim = 300, force = False):
# Cache file
cache_file = os.path.join (config.directories['cache'], experiment, dataset, key + '.npy')
# Restore cache
if (not force and os.path.isfile (cache_file)):
return np.load (cache_file)
# Adding again 1 because of reserved 0 index
vocab_size = len (tokenizer.word_index) + 1
# embedding_matrix
embedding_matrix = np.zeros ((vocab_size, embedding_dim))
# Get num lines for the progress bar
num_lines = sum (1 for line in open (config.pretrained_models[key]['vectors']))
# Create pkbar
pbar = pkbar.Pbar (name = 'generating ' + key, target = num_lines)
# Open vector file
index = 0
with open (config.pretrained_models[key]['vectors'], 'r', encoding = 'utf8', errors = 'ignore') as f:
for line in f:
# Get word and vector
word, *vector = line.split ()
# Check if the word is in the word index
if word in tokenizer.word_index:
# Get position
idx = tokenizer.word_index[word]
# Set weights
embedding_matrix[idx] = np.array (vector, dtype = np.float32)[:embedding_dim]
# Update list
index += 1
pbar.update (index)
# Store in cache
os.makedirs (os.path.dirname (cache_file), exist_ok = True)
np.save (cache_file, embedding_matrix)
# Return embedding matrix
return embedding_matrix
def prepareSimData(dataset, audiofiles, destloc, preprocess):
pbar = pkbar.Pbar(name='Preparing SpecImages for ' + dataset + ' (Sim)',
target=len(audiofiles))
for i in range(len(audiofiles)):
audio = audiofiles[i]
audio = audio.strip()
audioName, audioLoc = audio.split()
if dataset == 'Train':
cleanLoc = audioLoc.replace('REVERB_WSJCAM0_tr', 'WSJCAM0')
cleanLoc = cleanLoc.replace('/mc_train', '')
cleanLoc = cleanLoc.replace('_ch1', '')
else:
cleanLoc = audioLoc.replace('far_test', 'cln_test')
cleanLoc = cleanLoc.replace('near_test', 'cln_test')
cleanLoc = cleanLoc.replace('_ch1', '')
reverbPSD, cleanPSD = spectralImages_2D(audioName, audioLoc, cleanLoc,
preprocess)
reverb_arkfile = destloc + '/1ch/' + dataset + '/Sim/' + audioName + '.ark'
clean_arkfile = destloc + '/1ch/' + dataset + '/Clean/' + audioName + '.ark'
Path(os.path.dirname(reverb_arkfile)).mkdir(parents=True,
exist_ok=True)
Path(os.path.dirname(clean_arkfile)).mkdir(parents=True, exist_ok=True)
kaldiio.save_ark(reverb_arkfile,
reverbPSD,
scp=reverb_arkfile.replace('ark', 'scp'))
kaldiio.save_ark(clean_arkfile,
cleanPSD,
scp=clean_arkfile.replace('ark', 'scp'))
pbar.update(i)
os.system('cat ' + destloc + '/1ch/Dev/Sim/*.scp | sort > ' + destloc +
'/1ch/Dev/Sim/wav.scp')
os.system('cat ' + destloc + '/1ch/Dev/Clean/*.scp | sort > ' + destloc +
'/1ch/Dev/Clean/wav.scp')
return
def testLoadDataset(self):
print('=> test loading all dataset')
pbar = pkbar.Pbar('loading and processing dataset into cpu memory, total '
+ 'batches: {}, batch_size: {}'.format(len(self.ds), batch_size),
len(self.ds))
for i, _ in enumerate(self.ds):
pbar.update(i)
def load(self):
"""Cache dataset into CPU memory. If not called, dataset will be cached during the first epoch.
"""
if self.enable_pkbar:
pbar = pkbar.Pbar(
'=> processing and caching dataset into cpu memory: \ntotal ' +
'batches: {}, batch_size: {}'.format(
len(self), self.batch_size), len(self))
for i, _ in enumerate(self):
if self.enable_pkbar:
pbar.update(i)
def molecules():
for f in h5_files(path):
anidata = anidataloader(f)
anidata_size = anidata.group_size()
use_pbar = PKBAR_INSTALLED and verbose
if use_pbar:
pbar = pkbar.Pbar('=> loading {}, total molecules: {}'.format(f, anidata_size), anidata_size)
for i, m in enumerate(anidata):
yield m
if use_pbar:
pbar.update(i)
def VRL_train(five_tuple,
k,
data,
H,
H_core,
omega,
lr=0.00001,
epochs=int(1e4),
lam=1):
s, a, P_mat, R_vec, gamma = five_tuple
energy_history = []
Pbar = pkbar.Pbar(name='progress', target=epochs)
op = optim.SGD([data], lr=lr, momentum=0.9, weight_decay=5e-4)
for e in range(epochs):
spins = []
core = []
edge = []
energy = 0
regularity = 0
op.zero_grad()
for i in range(k):
H_core[i] = tensor_completion(H_core[i], H[i], omega[i])
core.append(tn.replicate_nodes(H_core[i]))
edge.append([])
for c in core[i]:
edge[i] += c.get_all_dangling()
for i in range(k):
spins.append(K_Spin(s, a, i + 1, data=data, softmax=True))
for j in range(i + 1):
edge[i][j] ^ spins[i].qubits[j][0]
for i in range(k):
energy -= contractors.branch(tn.reachable(core[i]),
nbranch=1).get_tensor()
energy_history.append(energy)
for j in range(s):
regularity += (1 - torch.sum(data[j * a:(j + 1) * a], 0))**2
target = energy + lam * regularity
target.backward()
op.step()
Pbar.update(e)
return spins, energy_history
def train_model(num_epochs, batch_size, learning_rate, heads, depth, loss_func):
"""
Trains the model with the specified parameters and returns the model
as well as the point estimate of losses on the validation set over each
epoch.
"""
global weights
weights = []
pbar = pkbar.Pbar(name='Training Model', target = num_epochs)
# Load data as torch.tensors
_, x_train, y_train = attn_dl.load_vectorized_data('train')
_, x_validate, y_validate = attn_dl.load_vectorized_data('dev')
model = TransformerModel(x_train.shape[2], heads, depth).double()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
train_ds = TensorDataset(x_train, y_train)
# Notice we shuffle our training data so the seasons are mixed!
train_dl = DataLoader(train_ds, batch_size=batch_size,
shuffle=True)
validate_ds = TensorDataset(x_validate, y_validate)
validate_dl = DataLoader(validate_ds,
batch_size=batch_size * 2)
# L1 loss is more robust to outliers
losses = []
for epoch in range(num_epochs):
model.train()
for xb, yb in train_dl:
pred = model(xb)
loss = loss_func(pred.double(), yb.double())
loss.backward()
optimizer.step()
optimizer.zero_grad()
model.eval()
with torch.no_grad():
epoch_loss = sum(loss_func(model(xb), yb) for xb, yb in validate_dl)
if epoch % 10 == 0:
print(epoch_loss/len(xb))
losses.append( epoch_loss / len(xb) )
pbar.update(epoch)
return model, losses
def build_from_path(hparams, in_dir, out_dir, num_workers=16):
executor = ProcessPoolExecutor(max_workers=num_workers)
spk_paths = [
p for p in glob.glob(os.path.join(in_dir, "*")) if os.path.isdir(p)
]
#files_path = glob.glob(os.path.join(data_root, '**/*.wav')) # 경로 부분은 알아서 자기 상황에 맞춰서 바꾸어야 함
mel_gen = Audio2Mel(hparams)
os.makedirs(out_dir, exist_ok=True)
futures = []
for i, spk_path in enumerate(spk_paths):
spk_name = os.path.basename(spk_path)
all_wav_path = glob.glob(os.path.join(spk_path, '*.wav'))
pbar = pkbar.Pbar(name='loading and processing dataset',
target=len(all_wav_path))
# make speaker directory
os.makedirs(os.path.join(out_dir, spk_name), exist_ok=True)
for j, wav_path in enumerate(all_wav_path):
wav_name = os.path.basename(wav_path)
npz_name = os.path.join(out_dir, spk_name, wav_name[:-4] + ".npz")
futures.append(
executor.submit(
partial(_processing_data, hparams, wav_path, i, npz_name,
mel_gen, pbar, j)))
results = [
future.result() for future in futures if future.result() is not None
]
all_mel = np.concatenate(results, axis=1)
mel_mean = np.mean(all_mel, axis=1)
mel_std = np.std(all_mel, axis=1)
np.save(os.path.join(out_dir, "mel_mean_std.npy"), [mel_mean, mel_std])
print('Finish Preprocessing')
def prepareRealData(dataset, audiofiles, destloc, preprocess):
pbar = pkbar.Pbar(name='Preparing SpecImages for ' + dataset + ' (Real)',
target=len(audiofiles))
for i in range(len(audiofiles)):
audio = audiofiles[i]
audio = audio.strip()
audioName, audioLoc = audio.split()
reverbPSD = spectralImages_1D(audioName, audioLoc, preprocess)
reverb_arkfile = destloc + '/1ch/Dev/Real/' + audioName + '.ark'
Path(os.path.dirname(reverb_arkfile)).mkdir(parents=True,
exist_ok=True)
kaldiio.save_ark(reverb_arkfile,
reverbPSD,
scp=reverb_arkfile.replace('ark', 'scp'))
pbar.update(i)
os.system('cat ' + destloc + '/1ch/Dev/Real/*.scp | sort > ' + destloc +
'/1ch/Dev/Real/wav.scp')
return
def build_from_path(hparams, in_dir, out_dir, num_workers=16):
executor = ProcessPoolExecutor(max_workers=num_workers)
#spk_paths = [p for p in glob.glob(os.path.join(in_dir, "*")) if os.path.isdir(p) and os.path.basename(p) in hparams.used_spks]
#files_path = glob.glob(os.path.join(data_root, '**/*.wav')) # 경로 부분은 알아서 자기 상황에 맞춰서 바꾸어야 함
with open(os.path.join(in_dir, "metadata.csv"), 'rt') as f:
all_meta = [row.rstrip() for row in f.readlines()]
mel_gen = Audio2Mel(hparams)
os.makedirs(out_dir, exist_ok=True)
pbar = pkbar.Pbar(name='loading and processing dataset',
target=len(all_meta))
futures = []
for i, row in enumerate(all_meta):
wav_path, text, spk_label, spk_name = row.split('|')
full_path = os.path.join(in_dir, wav_path)
# make speaker directory
os.makedirs(os.path.join(out_dir, spk_name), exist_ok=True)
wav_name = os.path.basename(wav_path)
npz_name = os.path.join(out_dir, spk_name, wav_name[:-4] + ".npz")
futures.append(
executor.submit(
partial(_processing_data, hparams, full_path, text, spk_label,
npz_name, mel_gen, pbar, i)))
results = [
future.result() for future in futures if future.result() is not None
]
all_mel = np.concatenate(results, axis=1)
mel_mean = np.mean(all_mel, axis=1)
mel_std = np.std(all_mel, axis=1)
np.save(os.path.join(out_dir, "mel_mean_std.npy"), [mel_mean, mel_std])
print('Finish Preprocessing')
def get_features(self):
loader = self.target_loader
model_path = osp.join('logs',self.folder_name,'best_model.pth')
self.activations = []
sd = torch.load(model_path)
self.model.load_state_dict(sd['model_state_dict'])
self.model.avgpool.register_forward_hook(self.hook)
self.model.eval()
pbar = pkbar.Pbar(name='Epoch Progress',target=len(loader))
labels = []
for it, ((data, jig_l, class_l), _) in enumerate(loader):
pbar.update(it)
data, jig_l, class_l = data.to(self.device), jig_l.to(self.device), class_l.to(self.device)
jigsaw_logit, class_logit = self.model(data)
labels.append(class_l.detach().cpu().numpy())
target_data = {}
target_data['activations'] = self.activations
target_data['labels'] = labels
with open(osp.join('logs',self.folder_name, 'act_labels.pkl'), 'wb') as handle:
pickle.dump(target_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
return self.activations
def get_features(self):
loader = self.target_loader
self.activations = []
sd = torch.load(self.model_path)
self.model.load_state_dict(sd['model_state_dict'])
if self.freeze_layer == 1:
self.model.avgpool.register_forward_hook(self.hook)
elif self.freeze_layer == 2:
self.model.layer4[1].bn1.register_forward_hook(self.hook)
elif self.freeze_layer == 3:
self.model.layer4[0].register_forward_hook(self.hook)
self.model.eval()
pbar = pkbar.Pbar(name='Epoch Progress', target=len(loader))
labels = []
for it, ((data, jig_l, class_l), _) in enumerate(loader):
pbar.update(it)
data, jig_l, class_l = data.to(self.device), jig_l.to(
self.device), class_l.to(self.device)
jigsaw_logit, class_logit = self.model(data)
labels.append(class_l.detach().cpu().numpy())
self.activations = functools.reduce(
(lambda x, y: np.append(x, y, axis=0)), self.activations)
labels = functools.reduce((lambda x, y: np.append(x, y, axis=0)),
labels)
print(self.activations.shape)
target_data = {}
target_data['features'] = self.activations
target_data['labels'] = labels
with open(
osp.join(
'logs', self.exp_type, 'act_labels_{}_layer_{}.pkl'.format(
self.args.target, self.freeze_layer)), 'wb') as handle:
pickle.dump(target_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
print("Activations saved to logs/{}/act_labels_{}_layer_{}.pkl".format(
self.exp_type, self.args.target, self.freeze_layer))
return self.activations
def deocdeData(dataloc, specImageloc, destfolder, model, device):
for dataset in ['Dev', 'Eval']:
audiofiles = __readscpFiles__(dataloc + '/' + dataset + '_SimData.scp')
audiofiles.update(
__readscpFiles__(dataloc + '/' + dataset + '_RealData.scp'))
pbar = pkbar.Pbar(name='Decoding ' + dataset + ' AudioFiles: ',
target=len(audiofiles))
data = SpecImages(specImageloc + '/' + dataset, mode='decode')
with torch.no_grad():
for i, (k, v) in enumerate(audiofiles.items()):
uttID = data.uttname2idx('MagdB_' + k)
audio = data.__getaudio__(uttID)
input_mag = audio['noisy_mag'].unsqueeze(1).to(device)
enhanced_mag = model(input_mag).cpu().numpy()
if enhanced_mag.shape[0] > 1:
enhanced_mag = np.hstack(np.squeeze(enhanced_mag))
else:
enhanced_mag = np.squeeze(enhanced_mag)
enhanced_mag = np.interp(enhanced_mag, [-1, 1],
audio['noisy_norm'])
temp = np.zeros((257, enhanced_mag.shape[1])) + 1j * np.zeros(
(257, enhanced_mag.shape[1]))
temp[:-1, :] = 10**(enhanced_mag / 20) * (np.cos(
audio['noisy_phase']) + np.sin(audio['noisy_phase']) * 1j)
enhanced_audio = istft(temp)
enhanced_audio = 0.98 * enhanced_audio / np.max(
np.abs(enhanced_audio))
enhanced_audio = enhanced_audio[:audio['utt_samples']]
destloc = destfolder + v.split('Reverb_Challenge')[1]
Path(os.path.dirname(destloc)).mkdir(parents=True,
exist_ok=True)
sf.write(destloc, enhanced_audio, 16000)
del audio, input_mag, enhanced_mag, temp, enhanced_audio
pbar.update(i)
def build_from_path(root, spk_emb_path):
executor = ProcessPoolExecutor(max_workers=cpu_count())
with open(spk_emb_path, 'rb') as f:
spk_meta = pickle.load(f)
all_spk_path = [p for p in glob.glob(os.path.join(root, '**/*')) if os.path.isdir(p)]
for spk_path in all_spk_path:
spk_name = os.path.basename(spk_path)
spk_emb = None
for meta in spk_meta:
if meta[0] == spk_name:
spk_emb = meta[1]
break
# speaker embedding이 없으면 continue
if spk_emb is None:
continue
all_npz_path = glob.glob(os.path.join(spk_path, '*.npz'))
pbar = pkbar.Pbar(name='loading and processing dataset', target=len(all_npz_path))
futures = []
for i, npz_path in enumerate(all_npz_path):
futures.append(executor.submit(partial(_add_spk_emb, npz_path, spk_emb, pbar, i)))
parser.add_argument('--audiofilelist',
type=str,
help='Location for scp file with audio locations',
required=True)
parser.add_argument(
'--specImageDir',
type=str,
help='Location for SpecImages (default: ./SpecImages)',
default=os.getcwd() + '/SpecImages')
parser.add_argument(
'--model',
type=str,
help=
'Absolute path for saved model (default: ./Saved_Model/best_model.ckpt)',
default=os.getcwd() + '/Saved_Model/best_model.ckpt')
args = parser.parse_args()
use_cuda = torch.cuda.is_available()
device = torch.device('cuda' if use_cuda else 'cpu')
args.specImageDir = '/data/scratch/vkk160330/Features/Reverb_Spec' # Comment this for your run
model = SkipConvNet(args.specImageDir).to(device)
saved_model = torch.load(args.model)
model.load_state_dict(saved_model['state_dict'])
model.eval()
audiofilelist = __readscpFiles__(args.audiofilelist)
pbar = pkbar.Pbar(name='Decoding AudioFiles: ', target=len(audiofilelist))
for i, (audioName, locations) in enumerate(audiofilelist.items()):
decode(audioName, locations, model, device)
pbar.update(i)
op = optim.SGD(model.parameters(),
lr=0.01,
momentum=0.9,
weight_decay=5e-4)
ops.append(op)
epochs = 10
acc_list = []
loss_list = []
global outputs_grad
for e in range(epochs):
correct = 0
total = 0
losses = 0
pbar = pkbar.Pbar(name='Epoch ' + str(e), target=60000 / batch_size)
for batch_idx, (x, y) in enumerate(trainloader):
dct_x = torch_shift(x)
dct_x = torch_apply(dct.dct, dct_x)
dct_x = dct_x.to(device)
y = y.to(device)
outputs_grad = []
outputs = []
for i in range(len(models)):
out = models[i](dct_x[i, ...])
outputs_grad.append(out)
outputs.append(out.detach())
# This line makes multiple calls to train_slice function
def __init__(self,
file_path,
batch_size=1000,
device='cpu',
chunk_threshold=20,
other_properties={},
include_energies=True,
species_order=['H', 'C', 'N', 'O'],
subtract_self_energies=False,
self_energies=[-0.600953, -38.08316, -54.707756, -75.194466]):
super(CachedDataset, self).__init__()
# example of species_dict will looks like
# species_dict: {'H': 0, 'C': 1, 'N': 2, 'O': 3}
# self_energies_dict: {'H': -0.600953, 'C': -38.08316, 'N': -54.707756, 'O': -75.194466}
species_dict = {}
self_energies_dict = {}
for i, s in enumerate(species_order):
species_dict[s] = i
self_energies_dict[s] = self_energies[i]
self.batch_size = batch_size
self.data_species = []
self.data_coordinates = []
data_self_energies = []
self.data_properties = {}
self.properties_info = other_properties
# whether include energies to properties
if include_energies:
self.add_energies_to_properties()
# let user check the properties will be loaded
self.check_properties()
# anidataloader
anidata = anidataloader(file_path)
anidata_size = anidata.group_size()
self.enable_pkbar = anidata_size > 5 and PKBAR_INSTALLED
if self.enable_pkbar:
pbar = pkbar.Pbar(
'=> loading h5 dataset into cpu memory, total molecules: {}'.
format(anidata_size), anidata_size)
# load h5 data into cpu memory as lists
for i, molecule in enumerate(anidata):
# conformations
num_conformations = len(molecule['coordinates'])
# species and coordinates
self.data_coordinates += list(molecule['coordinates'].reshape(
num_conformations, -1).astype(np.float32))
species = np.array([species_dict[x] for x in molecule['species']])
self.data_species += list(np.tile(species, (num_conformations, 1)))
# if subtract_self_energies
if subtract_self_energies:
self_energies = np.array(
sum([self_energies_dict[x] for x in molecule['species']]))
data_self_energies += list(
np.tile(self_energies, (num_conformations, 1)))
# properties
for key in self.data_properties:
self.data_properties[key] += list(molecule[key].reshape(
num_conformations, -1))
# pkbar update
if self.enable_pkbar:
pbar.update(i)
# if subtract self energies
if subtract_self_energies and 'energies' in self.properties_info[
'properties']:
self.data_properties['energies'] = np.array(
self.data_properties['energies']) - np.array(
data_self_energies)
del data_self_energies
gc.collect()
self.length = (len(self.data_species) + self.batch_size -
1) // self.batch_size
self.device = device
self.shuffled_index = np.arange(len(self.data_species))
np.random.shuffle(self.shuffled_index)
self.chunk_threshold = chunk_threshold
if not self.chunk_threshold:
self.chunk_threshold = np.inf
# clean trash
anidata.cleanup()
del num_conformations
del species
del anidata
gc.collect()
def __init__(self, file_path, batch_size, other_properties,
include_energies, species_order, subtract_self_energies,
self_energies):
super(TorchData, self).__init__()
species_dict = {}
self_energies_dict = {}
for i, s in enumerate(species_order):
species_dict[s] = i
self_energies_dict[s] = self_energies[i]
self.batch_size = batch_size
self.data_species = []
self.data_coordinates = []
data_self_energies = []
self.data_properties = {}
self.properties_info = other_properties
# whether include energies to properties
if include_energies:
self.add_energies_to_properties()
# let user check the properties will be loaded
self.check_properties()
# anidataloader
anidata = anidataloader(file_path)
anidata_size = anidata.group_size()
self.enable_pkbar = anidata_size > 5 and PKBAR_INSTALLED
if self.enable_pkbar:
pbar = pkbar.Pbar(
'=> loading h5 dataset into cpu memory, total molecules: {}'.
format(anidata_size), anidata_size)
# load h5 data into cpu memory as lists
for i, molecule in enumerate(anidata):
# conformations
num_conformations = len(molecule['coordinates'])
# species and coordinates
self.data_coordinates += list(molecule['coordinates'].reshape(
num_conformations, -1).astype(np.float32))
species = np.array([species_dict[x] for x in molecule['species']])
self.data_species += list(np.tile(species, (num_conformations, 1)))
# if subtract_self_energies
if subtract_self_energies:
self_energies = np.array(
sum([self_energies_dict[x] for x in molecule['species']]))
data_self_energies += list(
np.tile(self_energies, (num_conformations, 1)))
# properties
for key in self.data_properties:
self.data_properties[key] += list(molecule[key].reshape(
num_conformations, -1))
# pkbar update
if self.enable_pkbar:
pbar.update(i)
# if subtract self energies
if subtract_self_energies and 'energies' in self.properties_info[
'properties']:
self.data_properties['energies'] = np.array(
self.data_properties['energies']) - np.array(
data_self_energies)
del data_self_energies
gc.collect()
self.length = len(self.data_species)
# clean trash
anidata.cleanup()
del num_conformations
del species
del anidata
gc.collect()
def run(init_lr=INIT_LR,
warmup_steps=0,
max_epochs=100,
root=CHARADES_ROOT,
train_split=CHARADES_ANNO,
batch_size=BS * BS_UPSCALE,
frames=80,
save_dir=FINE_SAVE_DIR):
crop_size = {'S': 160, 'M': 224, 'XL': 312}[X3D_VERSION]
resize_size = {
'S': [180., 225.],
'M': [256., 256.],
'XL': [360., 450.]
}[X3D_VERSION] #[256.,320.]
gamma_tau = {'S': 6, 'M': 5 * 1, 'XL': 5}[X3D_VERSION] # 5
load_steps = st_steps = steps = 0
epochs = 0
num_steps_per_update = 1
cur_iterations = steps * num_steps_per_update
iterations_per_epoch = CHARADES_TR_SIZE // (batch_size * 1)
val_iterations_per_epoch = CHARADES_VAL_SIZE // (batch_size)
max_steps = iterations_per_epoch * max_epochs
val_spatial_transforms = Compose([
CenterCropScaled(crop_size),
ToTensor(255),
Normalize(CHARADES_MEAN, CHARADES_STD)
])
# SET 'TESTING' FOR BOTH, TO EXTRACT
dataset = Charades(train_split,
'testing',
root,
train_spatial_transforms,
task='loc',
frames=frames,
gamma_tau=gamma_tau,
crops=1)
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
shuffle=True,
num_workers=8,
pin_memory=True,
collate_fn=collate_fn)
val_dataset = Charades(train_split,
'testing',
root,
val_spatial_transforms,
task='loc',
frames=frames,
gamma_tau=gamma_tau,
crops=1)
val_dataloader = torch.utils.data.DataLoader(val_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=8,
pin_memory=True,
collate_fn=collate_fn)
dataloaders = {'train': dataloader, 'val': val_dataloader}
datasets = {'train': dataset, 'val': val_dataset}
print('train', len(datasets['train']), 'val', len(datasets['val']))
print('Total iterations:', max_steps, 'Total epochs:', max_epochs)
print('datasets created')
fine_net = x3d_fine.generate_model(x3d_version=X3D_VERSION,
n_classes=400,
n_input_channels=3,
task='loc',
dropout=0.5,
base_bn_splits=1,
global_tower=True)
fine_net.replace_logits(157)
load_ckpt = torch.load('models/fine_charades_039000_SAVE.pt')
state = fine_net.state_dict()
state.update(load_ckpt['model_state_dict'])
fine_net.load_state_dict(state)
fine_net.cuda()
fine_net = nn.DataParallel(fine_net)
print('model loaded')
lr = init_lr
print('LR:%f' % lr)
optimizer = optim.SGD(fine_net.parameters(),
lr=lr,
momentum=0.9,
weight_decay=1e-5)
lr_sched = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='min',
patience=3,
factor=0.1,
verbose=True)
if steps > 0:
optimizer.load_state_dict(load_ckpt['optimizer_state_dict'])
lr_sched.load_state_dict(load_ckpt['scheduler_state_dict'])
criterion = nn.BCEWithLogitsLoss()
val_apm = APMeter()
tr_apm = APMeter()
while epochs < max_epochs:
print('Step {} Epoch {}'.format(steps, epochs))
print('-' * 10)
# Each epoch has a training and validation phase
for phase in ['train'] + ['val']:
bar_st = iterations_per_epoch if phase == 'train' else val_iterations_per_epoch
bar = pkbar.Pbar(name='update: ', target=bar_st)
fine_net.train(False) # Set model to evaluate mode
# FOR EVAL AGGREGATE BN STATS
_ = fine_net.module.aggregate_sub_bn_stats()
torch.autograd.set_grad_enabled(False)
tot_loss = 0.0
tot_loc_loss = 0.0
tot_cls_loss = 0.0
tot_dis_loss = 0.0
tot_acc = 0.0
tot_corr = 0.0
tot_dat = 0.0
num_iter = 0
optimizer.zero_grad()
# Iterate over data.
print(phase)
for i, data in enumerate(dataloaders[phase]):
#for data in dataloaders[phase]:
num_iter += 1
bar.update(i)
inputs, labels, masks, meta, name = data
b, n, c, t, h, w = inputs.shape
inputs = inputs.view(b * n, c, t, h, w)
inputs = inputs.cuda() # B 3 T W H
tl = labels.size(2)
labels = labels.cuda() # B C TL
masks = masks.cuda() # B TL
valid_t = torch.sum(masks, dim=1).int()
feat, _ = fine_net([inputs, masks]) # N C T 1 1
keys = list(feat.keys())
print(i, name[0], feat[keys[0]].cpu().numpy().shape,
feat[keys[1]].cpu().numpy().shape,
feat[keys[2]].cpu().numpy().shape,
feat[keys[3]].cpu().numpy().shape,
feat[keys[4]].cpu().numpy().shape)
for k in feat:
torch.save(feat[k].data.cpu(),
os.path.join(save_dir, k, name[0]))
break
print(
'=> the first batch is ([chunk1, chunk2, ...], {"energies", "force", ...}) in which chunk1=(species, coordinates)'
)
chunks, properties = dataset[0]
elif parser.dataset == 'cache':
print('using cache dataset API')
print('=> loading dataset...')
dataset = torchani.data.CachedDataset(
file_path=parser.dataset_path,
species_order=['H', 'C', 'N', 'O'],
subtract_self_energies=True,
batch_size=parser.batch_size)
print('=> caching all dataset into cpu')
pbar = pkbar.Pbar(
'loading and processing dataset into cpu memory, total ' +
'batches: {}, batch_size: {}'.format(len(dataset),
parser.batch_size),
len(dataset))
for i, t in enumerate(dataset):
pbar.update(i)
print(
'=> the first batch is ([chunk1, chunk2, ...], {"energies", "force", ...}) in which chunk1=(species, coordinates)'
)
chunks, properties = dataset[0]
for i, chunk in enumerate(chunks):
print('chunk{}'.format(i + 1), list(chunk[0].size()),
list(chunk[1].size()))
print('energies', list(properties['energies'].size()))
print('=> start warming up')
def _do_epoch(self):
criterion = nn.CrossEntropyLoss()
self.model.train()
epoch_loss = 0
pbar = pkbar.Pbar(name='Epoch Progress',
target=len(self.source_loader))
for it, ((data, jig_l, class_l),
d_idx) in enumerate(self.source_loader):
pbar.update(it)
data, jig_l, class_l, d_idx = data.to(self.device), jig_l.to(
self.device), class_l.to(self.device), d_idx.to(self.device)
# absolute_iter_count = it + self.current_epoch * self.len_dataloader
# p = float(absolute_iter_count) / self.args.epochs / self.len_dataloader
# lambda_val = 2. / (1. + np.exp(-10 * p)) - 1
# if domain_error > 2.0:
# lambda_val = 0
# print("Shutting down LAMBDA to prevent implosion")
self.optimizer.zero_grad()
jigsaw_logit, class_logit = self.model(
data) # , lambda_val=lambda_val)
jigsaw_loss = criterion(jigsaw_logit, jig_l)
# domain_loss = criterion(domain_logit, d_idx)
# domain_error = domain_loss.item()
if self.only_non_scrambled:
if self.target_id is not None:
idx = (jig_l == 0) & (d_idx != self.target_id)
class_loss = criterion(class_logit[idx], class_l[idx])
else:
class_loss = criterion(class_logit[jig_l == 0],
class_l[jig_l == 0])
elif self.target_id:
class_loss = criterion(class_logit[d_idx != self.target_id],
class_l[d_idx != self.target_id])
else:
class_loss = criterion(class_logit, class_l)
_, cls_pred = class_logit.max(dim=1)
_, jig_pred = jigsaw_logit.max(dim=1)
if self.args.deep_all:
jigsaw_loss = torch.Tensor([0.0])
loss = class_loss
else:
loss = class_loss + jigsaw_loss * self.jig_weight # + 0.1 * domain_loss
# _, domain_pred = domain_logit.max(dim=1)
epoch_loss = epoch_loss + loss
loss.backward()
self.optimizer.step()
self.logger.log(
it,
len(self.source_loader),
{
"jigsaw": jigsaw_loss.item(),
"class":
class_loss.item() # , "domain": domain_loss.item()
},
# ,"lambda": lambda_val},
{
"jigsaw": torch.sum(jig_pred == jig_l.data).item(),
"class": torch.sum(cls_pred == class_l.data).item(),
# "domain": torch.sum(domain_pred == d_idx.data).item()
},
data.shape[0])
del loss, class_loss, jigsaw_loss, jigsaw_logit, class_logit
self.model.eval()
with torch.no_grad():
for phase, loader in self.test_loaders.items():
total = len(loader.dataset)
if loader.dataset.isMulti():
jigsaw_correct, class_correct, single_acc = self.do_test_multi(
loader)
print("Single vs multi: %g %g" %
(float(single_acc) / total,
float(class_correct) / total))
else:
jigsaw_correct, class_correct = self.do_test(loader)
jigsaw_acc = float(jigsaw_correct) / total
class_acc = float(class_correct) / total
self.logger.log_test(phase, {
"jigsaw": jigsaw_acc,
"class": class_acc
})
self.results[phase][self.current_epoch] = class_acc
if (self.results['val'][self.current_epoch] > self.best_class_acc):
self.best_class_acc = self.results['val'][self.current_epoch]
print("Saving new best at epoch: {}".format(self.current_epoch))
self.save_model(
os.path.join("logs", self.folder_name, 'best_model.pth'))
print("Saving latest at epoch: {}".format(self.current_epoch))
self.save_model(
os.path.join("logs", self.folder_name, 'latest_model.pth'))
import sys
sys.path.append("../")
import pkbar
import time
import numpy as np
import torch
print('=> using pbar (progress bar)')
pbar = pkbar.Pbar(name='loading and processing dataset', target=10)
for i in range(10):
time.sleep(0.1)
pbar.update(i)
# Hyper-parameters
input_size = 1
output_size = 1
num_epochs = 3
learning_rate = 0.001
train_per_epoch = 100
# Toy dataset
x_train = np.array(
[[3.3], [4.4], [5.5], [6.71], [6.93], [4.168], [9.779], [6.182], [7.59],
[2.167], [7.042], [10.791], [5.313], [7.997], [3.1]],
dtype=np.float32)
y_train = np.array(
[[1.7], [2.76], [2.09], [3.19], [1.694], [1.573], [3.366], [2.596], [2.53],
[1.221], [2.827], [3.465], [1.65], [2.904], [1.3]],
dtype=np.float32)
letter_list = [
'<pad>', "'", '+', '-', '.', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
'W', 'X', 'Y', 'Z', '_', ' ', '<sos>', '<eos>'
]
letter2index, index2letter = create_dict(letter_list)
inf_data_path = os.path.join(args.data_root_dir, args.inf_data_path)
print('Load Testing Data')
inf_data = load_data(inf_data_path)
inf_dataset = SpeechDataset(inf_data)
inf_loader = DataLoader(inf_dataset,
batch_size=1,
shuffle=False,
collate_fn=collate_wrapper_test)
pbar = pkbar.Pbar(name='Inference', target=len(inf_data))
model = Seq2Seq(input_dim=args.input_dim,
vocab_size=len(letter_list),
hidden_dim=args.hidden_dim,
value_size=args.value_size,
key_size=args.key_size,
pyramidlayers=args.encoder_layers)
model.to(DEVICE)
model.load_state_dict(torch.load('./models/76_model.pth.tar'))
prediction_list = test(model, inf_loader, index2letter, pbar)
index = np.linspace(0,
len(prediction_list) - 1,
len(prediction_list)).astype(np.int32)
test_result = pd.DataFrame({'Id': index, 'Predicted': prediction_list})
def train(net, trainloader, testloader, num_epochs, lrate, expt, arch,
milestones, temporal_length, model_path, model_path2, feat_map,
class_num, mode):
epoch_time = AverageMeter()
model = net
# Summary Writer Initiation
default_dir = '/fast-stripe/workspaces/deval/synthetic-data/hcn/'
ckpt_dir = os.path.join(default_dir, expt)
create_dir(ckpt_dir)
# Remove previous events for the same expt
files = []
for root, directory, filename in os.walk(ckpt_dir):
files.extend(filename)
break
for filename in files:
if "event" in filename:
filepath = os.path.join(ckpt_dir, filename)
os.remove(filepath)
writer = SummaryWriter(ckpt_dir)
# optimizer and scheduler
LR_steps = milestones
if mode == 'train':
if arch == 'hcn' or arch == 'hcn_ensemble':
optimizer = optim.Adam(model.parameters(),
lr=lrate,
betas=(0.9, 0.999),
eps=1e-8,
weight_decay=1e-4)
#optimizer = optim.SGD(model.parameters(), lr=lrate, momentum=0.9, weight_decay=1e-4)
scheduler = ExponentialLR(optimizer, gamma=0.99, last_epoch=-1)
# Parameters to save in the dict
train_loss = 0
train_acc = 0
best_train_epoch = 0
best_train_perf = 0
best_train_model = 0
best_test_acc = 0
best_test_model = 0
best_test_perf = 0
best_test_epoch = 0
for epoch in range(num_epochs):
print("Epoch", epoch + 1, "/", num_epochs)
batch_time = AverageMeter()
start_epoch = time.time()
losses = AverageMeter()
accuracies = AverageMeter()
test_acc = AverageMeter()
if mode == 'train':
pbar = pkbar.Pbar(name='Training...', target=len(trainloader))
model.train()
for i, data in enumerate(trainloader, 0):
start_batch = time.time()
body_3d_keypoints = data[0].to(device)
labels = data[1].long().to(device)
if arch == 'hcn':
inputs = body_3d_keypoints
optimizer.zero_grad()
outputs = model(inputs)
#print (outputs)
#print (outputs.size())
# Loss and gradient update
criterion = nn.CrossEntropyLoss()
train_loss = criterion(outputs, labels)
losses.update(train_loss, inputs.size(0))
train_loss.backward()
optimizer.step()
batch_time.update(time.time() - start_batch)
# Training accuracy
train_acc = calc_accuracy(outputs, labels)
accuracies.update(train_acc, outputs.size(0))
pbar.update(i)
lrate = optimizer.param_groups[0]['lr']
#if arch == 'vacnn':
# scheduler.step(train_loss)
#else:
scheduler.step()
# Train log - Summary Writer
writer.add_scalar('Train/Loss', losses.avg, epoch)
writer.add_scalar('Train/Acc', accuracies.avg, epoch)
writer.add_scalar('Lrate', lrate, epoch)
print("Training accuracy -->", accuracies.avg)
#print ("Training accuracy verify -->",np.mean(np.array(accuracies_verify)))
print("Epoch", epoch + 1, "Learning rate ->", lrate)
# Testing Loop
state = {
'last_epoch': epoch + 1,
'best_train_state_dict': best_train_model,
'last_state_dict': model.state_dict(),
'best_train_perf': best_train_perf,
'best_train_epoch': best_train_epoch,
'best_test_perf': best_test_perf,
'best_test_model': best_test_model,
'best_test_epoch': best_test_epoch,
'optimizer': optimizer.state_dict(),
'scheduler': scheduler
}
test_acc = test(testloader, state, None, model, arch, epoch,
writer, test_acc, feat_map, ckpt_dir, class_num,
mode)
if accuracies.avg > best_train_perf:
best_train_perf = accuracies.avg
state['best_train_perf'] = best_train_perf
best_train_epoch = epoch + 1
state['best_train_epoch'] = best_train_epoch
best_train_model = model.state_dict()
state['best_train_model'] = best_train_model
if test_acc.avg > best_test_acc:
best_test_acc = test_acc.avg
best_test_perf = test_acc.avg
state['best_test_perf'] = best_test_perf
best_test_model = model.state_dict()
state['best_test_model'] = best_test_model
best_test_epoch = epoch + 1
state['best_test_epoch'] = best_test_epoch
model_write_path = os.path.join(ckpt_dir, 'model_state.pth')
torch.save(state, model_write_path)
epoch_time.update((time.time() - start_epoch))
print("Average epoch time ->", epoch_time.avg)
print("Best Train epoch ->", best_train_epoch,
"Best Train Accuracy ->", best_train_perf)
print("Best Test epoch ->", best_test_epoch,
"Best Test Accuracy ->", best_test_perf)
elif mode == 'test':
state = model_path
state2 = model_path2
test_acc = test(testloader, state, state2, model, arch, epoch,
writer, test_acc, feat_map, ckpt_dir, class_num,
mode)
writer.close()
def build_from_path(hparams,
in_dir,
out_dir,
spk_emb_path,
spk2gen_path,
num_workers=16):
executor = ProcessPoolExecutor(max_workers=num_workers)
# load spk paths
if hparams.used_spks is not None:
spk_paths = [
p for p in glob.glob(os.path.join(in_dir, "*"))
if os.path.isdir(p) and os.path.basename(p) in hparams.used_spks
]
else:
spk_paths = [
p for p in glob.glob(os.path.join(in_dir, "*")) if os.path.isdir(p)
]
# load speaker embedding
if spk_emb_path:
spk_embs = pickle.load(open(spk_emb_path, 'rb'))
# load speaker to gender
if spk2gen_path is not None:
spk2gen = pickle.load(open(spk2gen_path, "rb"))
else:
raise ValueError
os.makedirs(out_dir, exist_ok=True)
# preprocessing per speaker
for i, spk_path in enumerate(spk_paths):
spk_name = os.path.basename(spk_path)
if spk_emb_path:
emb_idx = -1
for i in range(len(spk_embs)):
if spk_embs[i][0] == spk_name:
emb_idx = i
break
gender = spk2gen[spk_name]
assert gender == 'M' or gender == 'F'
# make speaker directory
os.makedirs(os.path.join(out_dir, spk_name), exist_ok=True)
os.makedirs(os.path.join(out_dir, spk_name, 'train'), exist_ok=True)
os.makedirs(os.path.join(out_dir, spk_name, 'val'), exist_ok=True)
os.makedirs(os.path.join(out_dir, spk_name, 'test'), exist_ok=True)
# glob all samples for a speaker
all_wav_path = glob.glob(os.path.join(spk_path, "*.wav"))
random.shuffle(all_wav_path)
total_num = len(all_wav_path)
train_num = int(total_num * 0.95)
val_num = total_num - train_num - 1
test_num = 1
pbar = pkbar.Pbar(name='loading and processing dataset',
target=len(all_wav_path))
futures = []
for j, wav_path in tqdm(enumerate(all_wav_path)):
wav_name = os.path.basename(wav_path)
spk_emb = spk_embs[emb_idx][1] if spk_emb_path else None
if j < train_num:
npz_name = os.path.join(out_dir, spk_name, 'train',
wav_name[:-4] + ".npz")
elif j >= train_num and j < train_num + val_num:
npz_name = os.path.join(out_dir, spk_name, 'val',
wav_name[:-4] + ".npz")
else:
npz_name = os.path.join(out_dir, spk_name, 'test',
wav_name[:-4] + ".npz")
futures.append(
executor.submit(
partial(_processing_data, hparams, wav_path, i, spk_emb,
gender, npz_name, pbar, i)))
results = [
future.result() for future in futures
if future.result() is not None
]
print('Finish Preprocessing')
