def main():
if args.cifar10_mode:
train_image_size = 32
num_label = 10
dataset = create_dataset(args.tfrecord_path, preprocess_for_train, num_label, args.batch_size,
args.num_epoch, train_image_size, cifar10_mode=True)
else:
train_image_size = args.train_image_size
num_label = args.num_label
dataset = create_dataset(args.tfrecord_path, preprocess_for_train, num_label, args.batch_size, args.num_epoch, train_image_size)
# Clearing the session removes all the nodes left over
tf.keras.backend.clear_session()
# Learning rate config
learning_rate_decay = tf.optimizers.schedules.ExponentialDecay(0.001, 100, 0.9)
# Create Optimizer
optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate_decay)
# Cross-entorpy loss for classification
loss_fn = tf.keras.losses.CategoricalCrossentropy()
# Metrics for classification accuracy
compute_accuracy = tf.keras.metrics.CategoricalAccuracy()
# Create model
num_label = 10 if args.cifar10_mode else args.num_label
model = Inception(num_label)
# Set up checkpoint
checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=model)
# Training process
train(model, optimizer, loss_fn, dataset, compute_accuracy, args.num_epoch, num_label)
# Export the model to a checkpoint
#checkpoint.save(file_prefix=args.model_save_path + 'ckpt')
# Evaluate results
if args.test_tfrecord_path:
test_dataset = create_dataset(args.test_tfrecord_path, 1,
train_image_size, num_label,
preprocess_for_train, is_training=False)
evaluate_for_iwild(model, test_dataset, args.save_eval_result_path)
def test_create_dataset():
audio_path = os.path.join("tests", "files", "audio.wav")
converted_audio_path = os.path.join("tests", "files",
"audio-converted.wav")
text_path = os.path.join("tests", "files", "text.txt")
forced_alignment_path = "align.json"
output_directory = "wavs"
label_path = "metadata.csv"
info_path = "info.json"
min_confidence = 0.85
with mock.patch("dataset.clip_generator.align.transcribe",
wraps=fake_transcribe) as mock_transcribe:
create_dataset(
text_path=text_path,
audio_path=audio_path,
forced_alignment_path=forced_alignment_path,
output_path=output_directory,
label_path=label_path,
info_path=info_path,
)
assert os.listdir(output_directory) == list(
expected_clips.keys()), "Unexpected audio clips"
with open(label_path) as f:
lines = f.readlines()
expected_text = [
f"{name}|{text}\n" for name, text in expected_clips.items()
]
assert lines == expected_text, "Unexpected metadata contents"
with open(forced_alignment_path, "r") as forced_alignment_file:
data = json.load(forced_alignment_file)
for segment in data:
assert {"start", "end", "name", "score", "aligned"}.issubset(
segment.keys()), "Alignment JSON missing required keys"
assert segment[
"score"] >= min_confidence, "SWS score less than min confidence"
with open(info_path) as f:
data = json.load(f)
assert int(data["total_duration"]) == 7
assert data["total_clips"] == 3
os.remove(forced_alignment_path)
shutil.rmtree(output_directory)
os.remove(label_path)
os.remove(info_path)
os.remove(converted_audio_path)
def test_create_dataset():
audio_path = os.path.join("test_samples", "audio.wav")
converted_audio_path = os.path.join("test_samples", "audio-converted.wav")
text_path = os.path.join("test_samples", "text.txt")
dataset_directory = "test-create-dataset"
forced_alignment_path = os.path.join(dataset_directory, "align.json")
output_directory = os.path.join(dataset_directory, "wavs")
unlabelled_path = os.path.join(dataset_directory, "unlabelled")
label_path = os.path.join(dataset_directory, "metadata.csv")
info_path = os.path.join(dataset_directory, "info.json")
min_confidence = 1.0
create_dataset(
text_path=text_path,
audio_path=audio_path,
transcription_model=FakeTranscriptionModel(),
output_folder=dataset_directory,
min_confidence=min_confidence,
combine_clips=False,
)
assert os.listdir(output_directory) == EXPECTED_CLIPS, "Unexpected audio clips"
assert os.listdir(unlabelled_path) == UNMATCHED_CLIPS, "Unexpected unmatched audio clips"
with open(label_path) as f:
lines = f.readlines()
expected_text = [f"{name}|{text}\n" for name, text in TRANSCRIPTION.items() if name in EXPECTED_CLIPS]
assert lines == expected_text, "Unexpected metadata contents"
with open(forced_alignment_path, "r") as forced_alignment_file:
data = json.load(forced_alignment_file)
for segment in data:
assert {"name", "start", "end", "duration", "score", "transcript", "text"}.issubset(
segment.keys()
), "Alignment JSON missing required keys"
assert segment["score"] >= min_confidence, "SWS score less than min confidence"
with open(info_path) as f:
data = json.load(f)
assert int(data["total_duration"]) == 5
assert data["total_clips"] == 2
os.remove(converted_audio_path)
shutil.rmtree(dataset_directory)
import dataset.create_dataset as dataset
train_dir = 'd:/data/train'
train_label = 'd:/data/ICPR2012_Cells_Classification_Contest/cell_label_information_Train.txt'
test_dir = 'd:/data/test'
test_label = 'd:/data/ICPR2012_Cells_Classification_Contest/cell_label_information_Test.txt'
dataset_path = 'd:/data/cell_dataset.gz'
dataset.create_dataset(train_dir, train_label, test_dir, test_label,
dataset_path)
import dataset.create_dataset as dataset train_dir='d:/data/train' train_label='d:/data/ICPR2012_Cells_Classification_Contest/cell_label_information_Train.txt' test_dir='d:/data/test' test_label='d:/data/ICPR2012_Cells_Classification_Contest/cell_label_information_Test.txt' dataset_path='d:/data/cell_dataset.gz' dataset.create_dataset(train_dir, train_label, test_dir, test_label, dataset_path)