def main():
out_dir = path.join(".", "MNIST")
util_library.check_dir(out_dir)
data_dir = util_library.saving_path(path.join(data_path, "MNIST"), "MNIST")
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets(data_dir, one_hot=True)
# Training data
im_x, im_labels = mnist.train.next_batch(mnist.train.num_examples)
im_x, im_keys, im_labels = generate_image_lists(
im_x, im_labels, path.join(out_dir, "train_set_keys.txt"))
test_im_dict = get_dict(im_x, im_keys, im_labels)
save_fn = path.join(out_dir, "train")
speech_library.write_feats(test_im_dict, save_fn)
#Validation data
im_x, im_labels = mnist.validation.next_batch(
mnist.validation.num_examples)
im_x, im_keys, im_labels = generate_image_lists(
im_x, im_labels, path.join(out_dir, "validation_set_keys.txt"))
test_im_dict = get_dict(im_x, im_keys, im_labels)
save_fn = path.join(out_dir, "validation")
speech_library.write_feats(test_im_dict, save_fn)
# Testing data
im_x, im_labels = mnist.test.next_batch(mnist.test.num_examples)
im_x, im_keys, im_labels = generate_image_lists(
im_x, im_labels, path.join(out_dir, "test_set_keys.txt"))
test_im_dict = get_dict(im_x, im_keys, im_labels)
save_fn = path.join(out_dir, "test")
speech_library.write_feats(test_im_dict, save_fn)
def library_setup():
parameters = arguments_for_library_setup()
model_lib = default_model_lib.copy()
model_lib["data_type"] = parameters.data_type
model_lib["subset"] = parameters.subset
model_lib["M"] = parameters.M
model_lib["K"] = parameters.K
model_lib["Q"] = parameters.Q
if model_lib["subset"] == "val":
subset = "validation.npz"
else:
subset = model_lib["subset"] + ".npz"
model_lib["data_dir"] = path.join(
feats_path, model_lib["data_type"], subset
)
model_lib["image_input_dim"] = 28*28
model_lib["name"] = "M_{}_K_{}_Q_{}_{}_{}".format(
model_lib["M"], model_lib["K"], model_lib["Q"], model_lib["data_type"],
model_lib["subset"]
)
base_dir = path.join(".", "Episode_files")
util_library.check_dir(base_dir)
model_lib["data_fn"] = path.join(base_dir, model_lib["name"])
return model_lib
def library_setup():
parameters = arguments_for_library_setup()
model_lib = default_model_lib.copy()
model_lib["data_type"] = parameters.data_type
model_lib["features_type"] = parameters.features_type
model_lib["data_tag"] = parameters.data_tag
model_lib["subset"] = parameters.subset
model_lib["M"] = parameters.M
model_lib["K"] = parameters.K
model_lib["Q"] = parameters.Q
model_lib["data_dir"] = path.join(
feats_path, model_lib["data_type"], "Subsets", "Words",
model_lib["features_type"], model_lib["data_tag"] + "_" +
model_lib["subset"] + "_" + model_lib["features_type"] + ".npz")
model_lib["speech_input_dim"] = 13
model_lib["max_frames"] = 100
model_lib["name"] = "M_{}_K_{}_Q_{}_{}_{}".format(model_lib["M"],
model_lib["K"],
model_lib["Q"],
model_lib["data_type"],
model_lib["subset"])
base_dir = path.join(".", "Episode_files")
util_library.check_dir(base_dir)
model_lib["data_fn"] = path.join(base_dir, model_lib["name"])
return model_lib
def result(log_fn, keyword, key):
accuracies = []
rnd_seeds = []
for line in open(log_fn, 'r'):
if re.search("Model name: ", line):
model_name = line.strip().split(" ")[-1]
if re.search(keyword, line):
line_parts = line.strip().split(" ")
keyword_parts = keyword.strip().split(" ")
ind = np.where(np.asarray(line_parts) == keyword_parts[0])[0][0]
acc = float(line_parts[ind + 3])
rnd_seed = line_parts[-1]
if rnd_seed not in rnd_seeds:
accuracies.append(acc)
rnd_seeds.append(rnd_seed)
if len(accuracies) == 0:
print(f'Log file {log_fn} is empty.')
return
accuracies = np.asarray(accuracies)
mean = np.mean(accuracies)
std = np.std(accuracies)
results_fn = os.path.dirname(log_fn)
util_library.check_dir(results_fn)
results_fn = path.join(results_fn, key + "_mean_and_std.txt")
if os.path.isfile(results_fn) is False:
print("\n" + "-" * PRINT_LENGTH)
print(f'Calculating results for {log_fn}')
print("-" * PRINT_LENGTH + "\n")
print(f'\tMean: {mean*100:3.2f}%')
print(f'\tStandard deviation: {std*100:3.2f}%\n')
print(f'\tWriting: {results_fn}.\n')
results_file = open(results_fn, 'w')
results_file.write("Mean: {}\n".format(mean))
results_file.write("Standard deviation: {}\n".format(std))
results_file.write("\nMean: {}%\n".format(mean * 100))
results_file.write("Standard deviation: {}%\n".format(std * 100))
results_file.close()
else:
print(f'{results_fn} already exists.\n')
def main():
directories = os.walk("../Model_data/")
keyword = " at rnd_seed of "
log_restore_list = []
log_save_dict = {}
model_log = []
for root, dirs, files in directories:
for filename in files:
if filename.split("_")[-1] == "log.txt" and root.split("/")[2] not in end_to_end_models:
log = path.join(root, filename)
log_restore_list.append(log)
for directory in log_restore_list:
dir_parts = directory.split("/")
save_dir = path.join("./Unimodal_results", "/".join(dir_parts[2:5] + dir_parts[-2:]))
util_library.check_dir(path.dirname(save_dir))
log_save_dict[directory] = save_dir
for log_fn in log_save_dict:
all_model_instances = []
if os.path.isfile(log_fn):
for line in open(log_fn, 'r'):
if re.search(keyword, line):
line_parts = line.strip().split(" ")
all_model_instances.append(":".join(line_parts[0].split(":")[0:-1]))
if os.path.isfile(log_save_dict[log_fn]) is False and len(all_model_instances) in [3, 5]:
print(f'\tCopying {log_fn} to {log_save_dict[log_fn]}.\n')
shutil.copyfile(log_fn, log_save_dict[log_fn])
elif os.path.isfile(log_save_dict[log_fn]) is False and len(all_model_instances) == 0:
print(f'\t{log_fn} had no successfully trained and tested models.\n')
elif os.path.isfile(log_save_dict[log_fn]) is False and len(all_model_instances) not in [3, 5]:
print(f'\tNot all instances in {log_fn} trained.\n')
elif os.path.isfile(log_save_dict[log_fn]):
print(f'\t{log_fn} already copied.\n')
def extract_words(feats, feats_list, lib, train_val_or_test, **kwargs):
dataset = lib["dataset"]
out_dir = lib["out_dir"]
feats_type = lib["feats_type"]
include_labels = lib["include_labels"]
output_dict = {}
if include_labels:
for key in tqdm(sorted(feats_list)):
basekey = key.strip().split("_")[0]
base_start = int(key.strip().split("_")[1].split("-")[0])
for (start, end, label) in feats_list[key]:
start = int(start)
end = int(end)
new_key = "{}".format(
label) + "_" + basekey + "_{:06d}-{:06d}".format(
start, end)
if end - start - 1 != 0:
output_dict[new_key] = feats[key][start - base_start:end -
base_start, :]
else:
for key in tqdm(sorted(feats_list)):
basekey = key.strip().split("_")[0]
base_start = int(key.strip().split("_")[1].split("-")[0])
for (start, end) in feats_list[key]:
start = int(start)
end = int(end)
new_key = basekey + "_{:06d}-{:06d}".format(start, end)
if end - start - 1 != 0:
output_dict[new_key] = feats[key][start - base_start:end -
base_start, :]
segmented_subsets_path = path.join(out_dir, "Subsets", "Words", feats_type)
util_library.check_dir(segmented_subsets_path)
save_dir = path.join(segmented_subsets_path,
"gt_" + train_val_or_test + "_" + feats_type)
print("Number of words in subset: {}".format(len(output_dict)))
write_feats(output_dict, save_dir)
def extract_segments(feats, lib, **kwargs):
dataset = lib["dataset"]
out_dir = lib["out_dir"]
feats_type = lib["feats_type"]
vads = lib["vads"]
labels_to_exclude = lib["labels_to_exclude"]
include_labels = lib["include_labels"]
segments_or_words = "Segments"
extract_words_or_not = lib["extract_words_or_not"]
labels_given = lib["labels_given"]
util_library.check_dir(out_dir)
segment_path = path.join(out_dir, "Features", "Segments", feats_type)
util_library.check_dir(segment_path)
segments = segment_regions(feats, vads, include_labels, labels_to_exclude,
labels_given, **kwargs)
seg_info_dir = path.join(segment_path,
dataset + "_" + feats_type + "_segments_list")
write_feats(segments, seg_info_dir)
segmented_feats = get_segments(feats, segments)
seg_feats_dir = path.join(
segment_path, dataset + "_" + feats_type + "_segmented_features")
write_feats(segmented_feats, seg_feats_dir)
print("\n" + "-" * PRINT_LENGTH)
train_feats, train_list, val_feats, val_list, test_feats, test_list = extract_subsets(
lib, segments_or_words, **kwargs)
print("\n" + "-" * PRINT_LENGTH)
if extract_words_or_not:
print("\nExtracting ground truth words from subset segments:\n")
extract_words(train_feats, train_list, lib, "train", **kwargs)
extract_words(val_feats, val_list, lib, "val", **kwargs)
extract_words(test_feats, test_list, lib, "test", **kwargs)
print("\n" + "-" * PRINT_LENGTH)
def extract_subsets(lib, segments_or_words="Segments", **kwargs):
dataset = lib["dataset"]
out_dir = lib["out_dir"]
feats_type = lib["feats_type"]
print("\nExtracting subsets:\n")
if segments_or_words == "Segments":
feat_name = dataset + "_" + feats_type + "_segmented_features"
list_name = dataset + "_" + feats_type + "_segments_list"
util_library.check_dir(out_dir)
segmented_subsets_path = path.join(out_dir, "Subsets", segments_or_words,
feats_type)
util_library.check_dir(segmented_subsets_path)
in_dir = [
path.join(out_dir, "Features", segments_or_words, feats_type,
feat_name),
path.join(out_dir, "Features", segments_or_words, feats_type,
list_name)
]
seg_dir = [
lib["training_speakers_path"], lib["validation_speakers_path"],
lib["testing_speakers_path"]
]
out_seg_dir = [
path.join(segmented_subsets_path, "seg_train_subset_" + feats_type),
path.join(segmented_subsets_path,
"seg_train_subset_list_" + feats_type),
path.join(segmented_subsets_path, "seg_val_subset_" + feats_type),
path.join(segmented_subsets_path, "seg_val_subset_list_" + feats_type),
path.join(segmented_subsets_path, "seg_test_subset_" + feats_type),
path.join(segmented_subsets_path, "seg_test_subset_list_" + feats_type)
]
train_feats, train_list, val_feats, val_list, test_feats, test_list = subset_division(
feats_type, in_dir, seg_dir, out_seg_dir)
return train_feats, train_list, val_feats, val_list, test_feats, test_list
def main():
args = check_argv()
if args.metric == "cosine":
dist_func = "cosine"
elif args.metric == "euclidean":
dist_func = "euclidean"
elif args.metric == "euclidean_squared":
dist_func == "sqeuclidean"
print("Start time: {}".format(datetime.datetime.now()))
model = "classifier"
speech_not_image_pairs = True if args.feats_fn.split(
"/")[2] in SPEECH_DATASETS else False if args.feats_fn.split(
"/")[2] in IMAGE_DATASETS else "INVALID"
if speech_not_image_pairs == "INVALID":
print("Specified dataset to get pairs for, not valid.")
sys.exit(0)
key_pair_file = path.join(
pair_path, "/".join(args.feats_fn.split(".")[-2].split("/")[2:]))
util_library.check_dir(key_pair_file)
latent_npz = path.join(key_pair_file, model + "_latents",
model + "_feats.npz")
key_pair_file = path.join(key_pair_file, "key_" + model + "_pairs.list")
if os.path.isfile(latent_npz) is False:
print("Generate latents before calculating distances.")
sys.exit(0)
latents, keys = data_library.load_latent_data_from_npz(latent_npz)
latents = np.asarray(latents)
latents = np.squeeze(latents)
nan = False
for i in range(len(latents)):
for j in range(latents.shape[-1]):
if np.isnan(latents[i, j]):
nan = True
if nan:
print("Latents contain NAN values.")
sys.exit(0)
key_pair_file = open(key_pair_file, 'w')
for i in tqdm(range(len(latents)),
desc="Calculating distances",
ncols=COL_LENGTH):
distances = cdist(latents[i, :].reshape(1, latents.shape[-1]), latents,
dist_func)
distances = np.squeeze(distances)
indices = np.argsort(distances)
lower_limit = 0.05
upper_limit = 0.25
current_key = keys[i]
pairs = []
count = 0
while len(pairs) < args.num_pairs and count < len(indices):
pair_key = keys[indices[count]]
if current_key != pair_key:
if speech_not_image_pairs and current_key.split("_")[1].split(
"-")[0] != pair_key.split("_")[1].split("-")[0]:
pairs.append(pair_key)
elif speech_not_image_pairs is False and distances[
indices[count]] > lower_limit and distances[
indices[count]] < upper_limit:
pairs.append(pair_key)
count += 1
if len(pairs) == args.num_pairs:
key_pair_file.write(f'{current_key:<30}')
for pair in pairs:
key_pair_file.write(f'\t{pair:<30}')
key_pair_file.write("\n")
key_pair_file.close()
print("End time: {}".format(datetime.datetime.now()))
def main():
out_dir = path.join(".", "omniglot")
util_library.check_dir(out_dir)
data_dir = path.join(data_path, "Omniglot")
# Testing data
test_fn = path.join(data_dir, "test", "*", "*", "*.png")
test_dict = {}
save_fn = path.join(out_dir, "test")
for im_fn in tqdm(sorted(glob.glob(test_fn))):
im = ImageOps.invert(Image.open(im_fn).resize((28, 28)).convert('L'))
im = np.array(im)
im = im.reshape((1, im.shape[0] * im.shape[1]))
im_max = np.max(im)
im = im / im_max
for row in range(im.shape[0]):
for column in range(im.shape[1]):
if im[row, column] > 1.0 or im[row, column] < 0.0:
print("Image values not between 0 and 1")
if np.isnan(im[row, column]):
print("Image contains NaN values")
name = im_fn.split("/")[-1].split(".")[0]
test_dict[name] = im
speech_library.write_feats(test_dict, save_fn)
# Training and validation data
train_fn = path.join(data_dir, "train", "*", "*", "*.png")
temp_dict = {}
all_classes = []
train_dict = {}
val_dict = {}
save_fn = path.join(out_dir, "train")
val_save_fn = path.join(out_dir, "validation")
for im_fn in tqdm(sorted(glob.glob(train_fn))):
im = ImageOps.invert(Image.open(im_fn).resize((28, 28)).convert('L'))
im = np.array(im)
im = im.reshape((1, im.shape[0] * im.shape[1]))
im_max = np.max(im)
im = im / im_max
for row in range(im.shape[0]):
for column in range(im.shape[1]):
if im[row, column] > 1.0 or im[row, column] < 0.0:
print("Image values not between 0 and 1")
if np.isnan(im[row, column]):
print("Image contains NaN values")
name = im_fn.split("/")[-1].split(".")[0]
all_classes.append(name.split("_")[0])
temp_dict[name] = im
unique_classes = sorted(list(set(all_classes)))
train_classes = unique_classes[0:664]
val_classes = unique_classes[664:]
for im_name in temp_dict:
name = im_name.split("_")[0]
if name in train_classes: train_dict[im_name] = temp_dict[im_name]
elif name in val_classes: val_dict[im_name] = temp_dict[im_name]
speech_library.write_feats(train_dict, save_fn)
speech_library.write_feats(val_dict, val_save_fn)
def main():
args = check_argv()
if args.metric == "cosine":
dist_func = "cosine"
elif args.metric == "euclidean":
dist_func = "euclidean"
elif args.metric == "euclidean_squared":
dist_func == "sqeuclidean"
print("Start time: {}".format(datetime.datetime.now()))
if args.speech_feats_fn.split("_")[-2] == args.image_feats_fn.split(
"/")[-1].split('.')[0]:
subset = args.image_feats_fn.split("/")[-1].split('.')[0]
elif args.speech_feats_fn.split(
"_")[-2] == "val" and args.image_feats_fn.split("/")[-1].split(
'.')[0] == "validation":
subset = args.image_feats_fn.split("/")[-1].split('.')[0]
else:
sys.exit(0)
speech_not_image_pairs = True if args.speech_feats_fn.split(
"/")[2] in SPEECH_DATASETS and args.image_feats_fn.split(
"/")[2] in IMAGE_DATASETS else "INVALID"
if speech_not_image_pairs == "INVALID":
print("Specified dataset to get pairs for, not valid.")
sys.exit(0)
key_pair_file = path.join(
pair_path,
args.speech_feats_fn.split("/")[2] + "_speech_" +
args.image_feats_fn.split("/")[2] + "_image_pairs")
model = "classifier"
util_library.check_dir(key_pair_file)
if os.path.isfile(key_pair_file) is False:
speech_latent_npz = path.join(
pair_path,
"/".join(args.speech_feats_fn.split(".")[-2].split("/")[2:]),
model + "_latents", model + "_feats.npz")
image_latent_npz = path.join(
pair_path,
"/".join(args.image_feats_fn.split(".")[-2].split("/")[2:]),
model + "_latents", model + "_feats.npz")
image_latents, image_keys = data_library.load_latent_data_from_npz(
image_latent_npz)
image_latents = np.asarray(image_latents)
image_latents = np.squeeze(image_latents)
speech_latents, speech_keys = data_library.load_latent_data_from_npz(
speech_latent_npz)
speech_latents = np.asarray(speech_latents)
speech_latents = np.squeeze(speech_latents)
speech_latents = (speech_latents - speech_latents.mean(axis=0)
) / speech_latents.std(axis=0)
image_latents = (image_latents - image_latents.mean(axis=0)
) / image_latents.std(axis=0)
im_x, im_labels, im_keys = (data_library.load_image_data_from_npz(
args.image_feats_fn))
sp_x, sp_labels, sp_lengths, sp_keys = (
data_library.load_speech_data_from_npz(args.speech_feats_fn))
max_frames = 100
d_frame = 13
print("\nLimiting dimensionality: {}".format(d_frame))
print("Limiting number of frames: {}\n".format(max_frames))
data_library.truncate_data_dim(sp_x, sp_lengths, d_frame, max_frames)
support_set = few_shot_learning_library.construct_few_shot_support_set_with_keys(
sp_x, sp_labels, sp_keys, sp_lengths, im_x, im_labels, im_keys, 11,
5)
support_set_speech_keys = []
support_set_image_keys = []
support_set_speech_latents = []
support_set_image_latents = []
for key in support_set:
support_set_speech_keys.extend(support_set[key]["speech_keys"])
for sp_key in support_set[key]["speech_keys"]:
ind = np.where(np.asarray(speech_keys) == sp_key)[0][0]
support_set_speech_latents.append(speech_latents[ind, :])
support_set_image_keys.extend(support_set[key]["image_keys"])
for im_key in support_set[key]["image_keys"]:
ind = np.where(np.asarray(image_keys) == im_key)[0][0]
support_set_image_latents.append(image_latents[ind, :])
support_set_speech_latents = np.asarray(support_set_speech_latents)
support_set_image_latents = np.asarray(support_set_image_latents)
support_dict = {}
already_used = []
for key in support_set_speech_keys:
label = key.split("_")[0]
for im_key in support_set_image_keys:
if few_shot_learning_library.label_test(
im_key.split("_")[0],
label) and im_key not in already_used:
support_dict[key] = im_key
already_used.append(im_key)
break
speech_dict = {}
speech_distances = cdist(speech_latents, support_set_speech_latents,
dist_func)
speech_indexes = np.argsort(speech_distances, axis=1)
for i, sp_key in enumerate(speech_keys):
if sp_key not in support_set_speech_keys:
for count in range(speech_indexes.shape[-1]):
ind = speech_indexes[i, count]
speech_dict[sp_key] = support_dict[
support_set_speech_keys[ind]]
break
image_dict = {}
image_distances = cdist(image_latents, support_set_image_latents,
dist_func)
image_indexes = np.argsort(image_distances, axis=1)
for i, im_key in enumerate(image_keys):
if im_key not in support_set_image_keys:
for count in range(image_indexes.shape[-1]):
ind = image_indexes[i, count]
if support_set_image_keys[ind] not in image_dict:
image_dict[support_set_image_keys[ind]] = []
image_dict[support_set_image_keys[ind]].append(im_key)
break
already_used_im_keys = []
key_pair_file = open(path.join(key_pair_file, subset + "_pairs.txt"),
'w')
for sp_key in tqdm(speech_dict,
desc="Generating speech-image pairs",
ncols=COL_LENGTH):
possible_im_keys = image_dict[speech_dict[sp_key]]
for i in range(len(possible_im_keys)):
possible_key = possible_im_keys[i]
if possible_key not in already_used_im_keys:
key_pair_file.write(f'{sp_key}\t{possible_key}\n')
already_used_im_keys.append(possible_key)
image_dict[speech_dict[sp_key]].remove(possible_key)
break
key_pair_file.close()
print("End time: {}".format(datetime.datetime.now()))
def main():
args = check_argv()
print("Start time: {}".format(datetime.datetime.now()))
model = "classifier"
speech_not_image_pairs = True if args.feats_fn.split(
"/")[2] in SPEECH_DATASETS else False if args.feats_fn.split(
"/")[2] in IMAGE_DATASETS else "INVALID"
if speech_not_image_pairs == "INVALID":
print("Specified dataset to get pairs for, not valid.")
sys.exit(0)
VALID_DATASETS = SPEECH_DATASETS if speech_not_image_pairs else IMAGE_DATASETS
directories = os.walk("../Model_data/")
valid_dirs = []
for root, dirs, files in directories:
for filename in files:
if filename.split("_")[-1] == "log.txt" and root.split(
"/")[2] == model and root.split("/")[4] in VALID_DATASETS:
log = path.join(root, filename)
name = root.split("/")[-1]
valid_dirs.append((log, root, root.split("/")[-1]))
if len(valid_dirs) != 1:
print(
f'Number of models found to generate pairs is {len(valid_dirs)}, can only generate pairs from 1 model'
)
sys.exit(0)
acc_dict = {}
for line in open(valid_dirs[0][0], 'r'):
if re.search("rnd_seed", line):
line_parts = line.strip().split(" ")
keyword_parts = "at rnd_seed of ".strip().split(" ")
ind = np.where(np.asarray(line_parts) == keyword_parts[0])[0]
if ":".join(line_parts[0].split(":")[:-1]) not in acc_dict:
acc_dict[":".join(line_parts[0].split(
":")[:-1])] = (float(line_parts[ind[0] - 1]) +
float(line_parts[ind[1] - 1])) / 2.0
max_acc = -np.inf
max_name = ""
for name in acc_dict:
if acc_dict[name] > max_acc:
max_acc = acc_dict[name]
max_name = name
model_path = path.join(valid_dirs[0][1], max_name,
valid_dirs[0][2] + "_lib.pkl")
key_pair_file = path.join(
pair_path, "/".join(args.feats_fn.split(".")[-2].split("/")[2:]),
model + "_latents")
util_library.check_dir(key_pair_file)
key_pair_file = path.join(key_pair_file, model + "_feats.npz")
latent_dict = {}
if os.path.isfile(key_pair_file) is False:
if speech_not_image_pairs:
print(f'Restoring model from: {model_path}')
speech_lib = model_setup_library.restore_lib(model_path)
latents = speech_rnn_latent_values(speech_lib, args.feats_fn,
latent_dict)
else:
print(f'Restoring model from: {model_path}')
image_lib = model_setup_library.restore_lib(model_path)
latents = image_cnn_latent_values(image_lib, args.feats_fn,
latent_dict)
np.savez_compressed(key_pair_file, **latent_dict)
print("End time: {}".format(datetime.datetime.now()))
def main():
out_dir = path.join(".", "buckeye")
util_library.check_dir(out_dir)
data_dir = path.join(".", "buckeye", "Subsets", "Words", "mfcc")
util_library.check_dir(path.join(data_dir, "back_up"))
train_file = path.join(data_path, "buckeye", "train_classes.list")
val_file = path.join(data_path, "buckeye", "validation_classes.list")
test_file = path.join(data_path, "buckeye", "test_classes.list")
train_fn = path.join(data_dir, "gt_train_mfcc.npz")
val_fn = path.join(data_dir, "gt_val_mfcc.npz")
test_fn = path.join(data_dir, "gt_test_mfcc.npz")
train_dict = {}
val_dict = {}
test_dict = {}
train_x, train_labels, train_lengths, train_keys = (
data_library.load_speech_data_from_npz(train_fn))
train_x, train_labels, train_lengths, train_keys = (
data_library.remove_test_classes(train_x, train_labels, train_lengths,
train_keys,
model_setup_library.DIGIT_LIST))
data_library.test_classes(train_labels, model_setup_library.DIGIT_LIST,
"training")
val_x, val_labels, val_lengths, val_keys = (
data_library.load_speech_data_from_npz(val_fn))
val_x, val_labels, val_lengths, val_keys = (
data_library.remove_test_classes(val_x, val_labels, val_lengths,
val_keys,
model_setup_library.DIGIT_LIST))
data_library.test_classes(val_labels, model_setup_library.DIGIT_LIST,
"valdiation")
test_x, test_labels, test_lengths, test_keys = (
data_library.load_speech_data_from_npz(test_fn))
test_x, test_labels, test_lengths, test_keys = (
data_library.remove_test_classes(test_x, test_labels, test_lengths,
test_keys,
model_setup_library.DIGIT_LIST))
data_library.test_classes(test_labels, model_setup_library.DIGIT_LIST,
"testing")
train_classes = []
val_classes = []
test_classes = []
for line in open(train_file):
train_classes.append(line.strip())
for line in open(val_file):
val_classes.append(line.strip())
for line in open(test_file):
test_classes.append(line.strip())
for i, lab in enumerate(train_labels):
if lab in train_classes: train_dict[train_keys[i]] = train_x[i]
elif lab in val_classes: val_dict[train_keys[i]] = train_x[i]
elif lab in test_classes: test_dict[train_keys[i]] = train_x[i]
else: print(f'{lab} does not fit in anywhere :(')
for i, lab in enumerate(val_labels):
if lab in train_classes: train_dict[val_keys[i]] = val_x[i]
elif lab in val_classes: val_dict[val_keys[i]] = val_x[i]
elif lab in test_classes: test_dict[val_keys[i]] = val_x[i]
else: print(f'{lab} does not fit in anywhere :(')
for i, lab in enumerate(test_labels):
if lab in train_classes: train_dict[test_keys[i]] = test_x[i]
elif lab in val_classes: val_dict[test_keys[i]] = test_x[i]
elif lab in test_classes: test_dict[test_keys[i]] = test_x[i]
else: print(f'{lab} does not fit in anywhere :(')
speech_library.write_feats(train_dict, train_fn)
speech_library.write_feats(val_dict, val_fn)
speech_library.write_feats(test_dict, test_fn)
def extract_features(lib, **kwargs):
dataset = lib["dataset"]
out_dir = lib["out_dir"]
wavs = lib["wavs"]
feats_type = lib["feats_type"]
util_library.check_dir(out_dir)
feats = {}
print("\nExtracting features:")
for wav_fn in tqdm(sorted(glob.glob(wavs))):
samplerate, signal = wavfile.read(wav_fn)
if feats_type == "fbank":
wanted_feats = filterbank(signal,
sampling_frequency=samplerate,
preemphasis_fact=0.97,
winlen=0.025,
winstep=0.01,
winfunc=np.hamming,
lowf=0,
highf=None,
nfft=None,
nfilt=45,
return_energy=False,
**kwargs)
elif feats_type == "mfcc":
wanted_feats = mfcc(signal,
sampling_frequency=samplerate,
preemphasis_fact=0.97,
winlen=0.025,
winstep=0.01,
winfunc=np.hamming,
lowf=0,
highf=None,
nfft=None,
nfilt=24,
numcep=13,
ceplifter=22,
append_energy=True,
**kwargs)
if lib["dataset"] == "TIDigits":
parts = wav_fn.strip().split("/")
save_key = parts[-2] + "-" + parts[-1].strip().split(".")[0]
else:
save_key = path.splitext(path.split(wav_fn)[-1])[0]
feats[save_key] = wanted_feats
feats_dir = path.join(out_dir, "Features", "Raw", feats_type)
util_library.check_dir(feats_dir)
raw_feats_dir = path.join(feats_dir,
dataset + "_" + feats_type + "_features")
speakers = features_library.get_speakers(feats)
mean, variance = features_library.speaker_mean_and_variance(
feats, speakers)
feats = features_library.speaker_mean_variance_normalization(
feats, mean, variance)
write_feats(feats, raw_feats_dir)
print("\n" + "-" * PRINT_LENGTH)
return feats
def main():
directories = os.walk("../Model_data/")
keyword = " at rnd_seed of "
log_restore_list = []
log_save_dict = {}
zero_shot_format = {}
model_log = []
for root, dirs, files in directories:
for filename in files:
if filename.split("_")[-1] == "log.txt" and root.split(
"/")[2] in end_to_end_models:
log = path.join(root, filename)
log_restore_list.append(log)
for directory in log_restore_list:
dir_parts = directory.split("/")
filename = dir_parts[-1]
if filename.split("_")[1] == "unimodal" and filename.split(
"_")[2] == "speech":
save_dir = path.join("./Unimodal_results", dir_parts[2],
"unimodal_speech", dir_parts[3])
util_library.check_dir(save_dir)
log_save_dict[directory] = [
path.join(save_dir, "1_shot_results.txt"),
path.join(save_dir, "5_shot_results.txt")
]
zero_shot_format[directory] = False
elif filename.split("_")[1] == "unimodal" and filename.split(
"_")[2] == "image":
save_dir = path.join("./Unimodal_results", dir_parts[2],
"unimodal_image", dir_parts[3])
util_library.check_dir(save_dir)
log_save_dict[directory] = [
path.join(save_dir, "1_shot_results.txt"),
path.join(save_dir, "5_shot_results.txt")
]
zero_shot_format[directory] = False
elif filename.split("_")[1] == "multimodal":
save_dir = path.join("./Multimodal_results",
dir_parts[2] + "_" + dir_parts[3])
util_library.check_dir(save_dir)
if filename.split(".")[0].split("_")[2] == "zero":
log_save_dict[directory] = [
path.join(save_dir, "0_shot_results.txt")
]
zero_shot_format[directory] = True
else:
log_save_dict[directory] = [
path.join(save_dir, "1_shot_results.txt"),
path.join(save_dir, "5_shot_results.txt")
]
zero_shot_format[directory] = False
for log_fn in log_save_dict:
all_model_instances = []
if os.path.isfile(log_fn):
for line in open(log_fn, 'r'):
if re.search(keyword, line):
line_parts = line.strip().split(" ")
all_model_instances.append(":".join(
line_parts[0].split(":")[0:-1]))
if zero_shot_format[log_fn] is False:
if os.path.isfile(
log_save_dict[log_fn][0]) is False and os.path.isfile(
log_save_dict[log_fn][1]) is False and len(
all_model_instances) in [3, 5]:
if os.path.isfile(log_fn):
one_shot_text = ""
few_shot_text = ""
for line in open(log_fn, 'r'):
if re.search(keyword, line):
line_parts = line.strip().split(" ")
one_shot_text += "1-shot accuracy of " + line_parts[
4] + " at rnd_seed of " + line_parts[8] + "\n"
few_shot_text += "5-shot accuracy of " + line_parts[
-5] + " at rnd_seed of " + line_parts[-1] + "\n"
else:
one_shot_text += line
few_shot_text += line
with open(log_save_dict[log_fn][0], "w") as f:
f.write(one_shot_text)
f.close()
with open(log_save_dict[log_fn][1], "w") as f:
f.write(few_shot_text)
f.close()
elif zero_shot_format[log_fn]:
if os.path.isfile(log_save_dict[log_fn][0]) is False and len(
all_model_instances) in [3, 5]:
if os.path.isfile(log_fn):
zero_shot_text = ""
for line in open(log_fn, 'r'):
if re.search(keyword, line):
line_parts = line.strip().split(" ")
zero_shot_text += "Zero-shot accuracy of " + line_parts[
4] + " at rnd_seed of " + line_parts[8] + "\n"
else:
zero_shot_text += line
with open(log_save_dict[log_fn][0], "w") as f:
f.write(zero_shot_text)
f.close()
elif os.path.isfile(
log_save_dict[log_fn][0]) is False and os.path.isfile(
log_save_dict[log_fn][1]) is False and len(
all_model_instances) == 0:
print(
f'\t{log_fn} had no successfully trained and tested models.\n')
elif os.path.isfile(
log_save_dict[log_fn][0]) is False and os.path.isfile(
log_save_dict[log_fn][1]) is False and len(
all_model_instances) not in [3, 5]:
print(f'\tNot all instances in {log_fn} trained.\n')
elif os.path.isfile(log_save_dict[log_fn][0]) and os.path.isfile(
log_save_dict[log_fn][1]):
print(f'\t{log_fn} already copied.\n')
def model_library_setup(base_lib, param_dict):
model_lib = base_lib.copy()
temp_list = []
for pool_layer in model_lib["image_pool_layers"]:
if pool_layer == "None": pool_value = None
else: pool_value = pool_layer
temp_list.append(pool_value)
model_lib["image_pool_layers"] = temp_list
for key in param_dict:
if key != "rnd_seed":
model_lib[key] = param_dict[key]
model_lib["model_name"] = get_model_name(model_lib)
date = str(datetime.now()).split(" ")
model_lib["date"] = date
model_lib["model_instance"] = model_lib["model_name"] + "_" + date[
0] + "_" + date[1]
model_lib["output_fn"] = path.join(
model_path if model_lib["final_model"] else non_final_model_path,
model_lib["model_type"], model_lib["model_name"],
model_lib["model_instance"])
util_library.check_dir(model_lib["output_fn"])
model_lib["best_model_fn"] = util_library.saving_path(
model_lib["output_fn"],
model_lib["model_name"] + "_" + model_lib["model_type"] + "_best")
model_lib["intermediate_model_fn"] = util_library.saving_path(
model_lib["output_fn"],
model_lib["model_name"] + "_" + model_lib["model_type"] + "_last")
for key in param_dict:
if key == "rnd_seed":
model_lib[key] = param_dict[key]
if model_lib["batch_size"] <= 0:
print("Batch size must be greater than zero")
sys.exit(0)
if model_lib["n_buckets"] <= 0 and model_lib["divide_into_buckets"]:
print("Number of buckets must be greater than zero")
sys.exit(0)
if model_lib["epochs"] <= 0:
print("Epochs must be greater than zero")
sys.exit(0)
if model_lib["learning_rate"] <= 0.0:
print("Learning rate must be greater than zero")
sys.exit(0)
if model_lib["keep_prob"] < 0.0 and model_lib["keep_prob"] > 1.0:
print("Keep probability must be between 0.0 and 1.0")
sys.exit(0)
log_dict, model_instances = model_files(model_lib)
test_list = [
"multimodal", "multimodal_zero_shot", "unimodal_image",
"unimodal_speech"
]
exit_flag = True
if str(model_lib["rnd_seed"]) in log_dict:
print(f'This model with seed {model_lib["rnd_seed"]} already trained.')
lib_fn = path.join("/".join(model_lib["output_fn"].split("/")[0:-1]),
model_instances[str(model_lib["rnd_seed"])],
model_lib["model_name"] + "_lib.pkl")
model_lib = model_setup_library.restore_lib(lib_fn)
for test in test_list:
if test in log_dict[str(model_lib["rnd_seed"])]:
model_lib["do_" + test] = False
else:
model_lib["do_" + test] = True
exit_flag = False
return False, exit_flag, model_lib
else:
for test in test_list:
model_lib["do_" + test] = True
return True, False, model_lib
