def project_details(project):
"""
Show the details for the selected project.
"""
project = urllib.parse.unquote(project)
path = project_utils.lookup_project_path(project)
config = project_utils.load_project_config(path)
stats = {}
for class_name, tags in config['classes'].items():
stats[class_name] = {}
for split in SPLITS:
videos_dir = directories.get_videos_dir(path, split, class_name)
tags_dir = directories.get_tags_dir(path, split, class_name)
stats[class_name][split] = {
'total':
len(os.listdir(videos_dir)),
'tagged':
len(os.listdir(tags_dir)) if os.path.exists(tags_dir) else 0,
}
return render_template('project_details.html',
config=config,
path=path,
stats=stats,
project=config['name'])
def training_page(project):
project = urllib.parse.unquote(project)
path = project_utils.lookup_project_path(project)
project_config = project_utils.load_project_config(path)
output_path_prefix = os.path.join(os.path.basename(path), 'checkpoints', '')
return render_template('training.html', project=project, path=path, models=utils.get_available_backbone_models(),
output_path_prefix=output_path_prefix, project_config=project_config)
def edit_class(project, class_name):
"""
Edit the class name and tags for an existing class in the given project.
"""
project = urllib.parse.unquote(project)
class_name = urllib.parse.unquote(class_name)
path = project_utils.lookup_project_path(project)
# Get new class name and tags
new_class_name, new_tag1, new_tag2 = utils.get_class_name_and_tags(
request.form)
# Update project config
config = project_utils.load_project_config(path)
del config['classes'][class_name]
config['classes'][new_class_name] = [new_tag1, new_tag2]
project_utils.write_project_config(path, config)
# Update directory names
data_dirs = []
for split in SPLITS:
data_dirs.extend([
directories.get_videos_dir(path, split),
directories.get_frames_dir(path, split),
directories.get_tags_dir(path, split),
])
# Feature directories follow the format <dataset_dir>/<split>/<model>/<num_layers_to_finetune>/<label>
features_dir = directories.get_features_dir(path, split)
if os.path.exists(features_dir):
model_dirs = [
os.path.join(features_dir, model_dir)
for model_dir in os.listdir(features_dir)
]
data_dirs.extend([
os.path.join(model_dir, tuned_layers)
for model_dir in model_dirs
for tuned_layers in os.listdir(model_dir)
])
logreg_dir = directories.get_logreg_dir(path)
if os.path.exists(logreg_dir):
data_dirs.extend([
os.path.join(logreg_dir, model_dir)
for model_dir in os.listdir(logreg_dir)
])
for base_dir in data_dirs:
class_dir = os.path.join(base_dir, class_name)
if os.path.exists(class_dir):
new_class_dir = os.path.join(base_dir, new_class_name)
os.rename(class_dir, new_class_dir)
return redirect(url_for('project_details', project=project))
def project_config():
"""
Provide the config for a given project.
"""
data = request.json
name = data['name']
path = project_utils.lookup_project_path(name)
# Get config
config = project_utils.load_project_config(path)
return jsonify(config)
def edit_tag(project, tag_idx):
data = request.form
project = urllib.parse.unquote(project)
path = project_utils.lookup_project_path(project)
config = project_utils.load_project_config(path)
tags = config['tags']
new_tag_name = data['newTagName']
# Update tag name
tags[tag_idx] = new_tag_name
project_utils.write_project_config(path, config)
return redirect(url_for('project_details', project=project))
def remove_tag_from_class():
"""
Remove selected tag from class label in project config.
"""
data = request.json
path = data['path']
tag_index = data['tagIndex']
class_name = data['className']
config = project_utils.load_project_config(path)
config['classes'][class_name].remove(int(tag_index))
project_utils.write_project_config(path, config)
return jsonify(success=True)
def remove_class(project, class_name):
"""
Remove the given class from the config file of the given project. No data will be deleted.
"""
project = urllib.parse.unquote(project)
class_name = urllib.parse.unquote(class_name)
path = project_utils.lookup_project_path(project)
# Update project config
config = project_utils.load_project_config(path)
del config['classes'][class_name]
project_utils.write_project_config(path, config)
return redirect(url_for("project_details", project=project))
def create_tag(project):
data = request.form
project = urllib.parse.unquote(project)
path = project_utils.lookup_project_path(project)
config = project_utils.load_project_config(path)
tag_name = data['newTagName']
tags = config['tags']
new_tag_index = config['max_tag_index'] + 1
tags[new_tag_index] = tag_name
config['max_tag_index'] = new_tag_index
project_utils.write_project_config(path, config)
return redirect(url_for('project_details', project=project))
def remove_tag(project, tag_idx):
project = urllib.parse.unquote(project)
path = project_utils.lookup_project_path(project)
config = project_utils.load_project_config(path)
tags = config['tags']
# Remove tag from the overall tags list
del tags[tag_idx]
# Remove tag from the classes
for class_label, class_tags in config['classes'].items():
if tag_idx in class_tags:
class_tags.remove(tag_idx)
project_utils.write_project_config(path, config)
return redirect(url_for('project_details', project=project))
def assign_tag_to_class():
"""
Assign selected tag to class label in project config.
"""
data = request.json
path = data['path']
tag_index = data['tagIndex']
class_name = data['className']
config = project_utils.load_project_config(path)
class_tags = config['classes'][class_name]
class_tags.append(int(tag_index))
class_tags.sort()
project_utils.write_project_config(path, config)
return jsonify(success=True)
def update_project():
"""
Update an existing project entry with a new path. If a config file exists in there, it will be
used, otherwise a new one will be created.
The project will keep the given project name.
"""
data = request.form
project_name = data['projectName']
path = data['path']
# Check for existing config file (might be None)
config = project_utils.load_project_config(path)
# Make sure the directory is correctly set up
project_utils.setup_new_project(project_name, path, config)
return redirect(url_for('project_details', project=project_name))
def import_project():
"""
Import an existing project from the given path. If a config file exists in there, it will be
used while also making sure that the project name is still unique. Otherwise, a new config
will be created and a unique project name will be constructed from the directory name.
"""
data = request.form
path = data['path']
# Check for existing config file and make sure project name is unique
config = project_utils.load_project_config(path)
if config:
project_name = project_utils.get_unique_project_name(config['name'])
else:
# Use folder name as project name and make sure it is unique
project_name = project_utils.get_unique_project_name(os.path.basename(path))
# Make sure the directory is correctly set up
project_utils.setup_new_project(project_name, path, config)
return redirect(url_for('project_details', project=project_name))
def start_testing():
data = request.json
path_in = data['inputVideoPath']
output_video_name = data['outputVideoName']
title = data['title']
path = data['path']
custom_classifier = os.path.join(path, data['classifier'])
config = project_utils.load_project_config(path)
if output_video_name:
output_dir = os.path.join(path, 'output_videos')
os.makedirs(output_dir, exist_ok=True)
path_out = os.path.join(output_dir, output_video_name + '.mp4')
else:
path_out = None
ctx = multiprocessing.get_context('spawn')
global queue_testing_output
global stop_event
queue_testing_output = ctx.Queue()
stop_event = ctx.Event()
testing_kwargs = {
'custom_classifier': custom_classifier,
'path_in': path_in,
'path_out': path_out,
'title': title,
'use_gpu': config['use_gpu'],
'display_fn': queue_testing_output.put,
'stop_event': stop_event,
}
global test_process
test_process = ctx.Process(target=run_custom_classifier,
kwargs=testing_kwargs)
test_process.start()
return jsonify(success=True)
def project_details(project):
"""
Show the details for the selected project.
"""
project = urllib.parse.unquote(project)
path = project_utils.lookup_project_path(project)
config = project_utils.load_project_config(path)
stats = {}
for class_name in config['classes']:
stats[class_name] = {}
for split in SPLITS:
videos_dir = directories.get_videos_dir(path, split, class_name)
tags_dir = directories.get_tags_dir(path, split, class_name)
stats[class_name][split] = {
'total': len(os.listdir(videos_dir)),
'tagged': len(os.listdir(tags_dir)) if os.path.exists(tags_dir) else 0,
'videos': natsorted([video for video in os.listdir(videos_dir) if video.endswith(VIDEO_EXT)], alg=ns.IC)
}
tags = config['tags']
return render_template('project_details.html', config=config, path=path, stats=stats, project=config['name'],
tags=tags)
def add_class(project):
"""
Add a new class to the given project.
"""
data = request.form
project = urllib.parse.unquote(project)
path = project_utils.lookup_project_path(project)
class_name = data['className']
# Update project config
config = project_utils.load_project_config(path)
config['classes'][class_name] = []
project_utils.write_project_config(path, config)
# Setup directory structure
for split in SPLITS:
videos_dir = directories.get_videos_dir(path, split, class_name)
if not os.path.exists(videos_dir):
os.mkdir(videos_dir)
return redirect(url_for("project_details", project=project))
def start_training():
data = request.json
path = data['path']
num_layers_to_finetune = data['layersToFinetune']
output_folder = data['outputFolder']
model_name = data['modelName']
epochs = data['epochs']
config = project_utils.load_project_config(path)
model_name, model_version = model_name.split('-')
path_out = os.path.join(path, 'checkpoints', output_folder)
ctx = multiprocessing.get_context('spawn')
global queue_train_logs
global confmat_event
queue_train_logs = ctx.Queue()
confmat_event = ctx.Event()
training_kwargs = {
'path_in': path,
'num_layers_to_finetune': int(num_layers_to_finetune),
'path_out': path_out,
'model_version': model_version,
'model_name': model_name,
'epochs': int(epochs),
'use_gpu': config['use_gpu'],
'temporal_training': config['temporal'],
'log_fn': queue_train_logs.put,
'confmat_event': confmat_event,
}
global train_process
train_process = ctx.Process(target=train_model, kwargs=training_kwargs)
train_process.start()
return jsonify(success=True)
def annotate(project, split, label, idx):
"""
For the given class label, show all frames for annotating the selected video.
"""
project = urllib.parse.unquote(project)
path = project_utils.lookup_project_path(project)
label = urllib.parse.unquote(label)
split = urllib.parse.unquote(split)
_, model_config = utils.load_feature_extractor(path)
frames_dir = directories.get_frames_dir(path, split, label)
features_dir = directories.get_features_dir(path,
split,
model_config,
label=label)
tags_dir = directories.get_tags_dir(path, split, label)
logreg_dir = directories.get_logreg_dir(path, model_config, label)
videos = os.listdir(frames_dir)
videos = natsorted(videos, alg=ns.IC)
# The list of images in the folder
images = [
image
for image in glob.glob(os.path.join(frames_dir, videos[idx], '*'))
if utils.is_image_file(image)
]
classes = [-1] * len(images)
# Load logistic regression model if available and assisted tagging is enabled
if utils.get_project_setting(path, 'assisted_tagging'):
logreg_path = os.path.join(logreg_dir, 'logreg.joblib')
features_path = os.path.join(features_dir, f'{videos[idx]}.npy')
if os.path.isfile(logreg_path) and os.path.isfile(features_path):
logreg = load(logreg_path)
features = np.load(features_path).mean(axis=(2, 3))
classes = list(logreg.predict(features))
# Natural sort images, so that they are sorted by number
images = natsorted(images, alg=ns.IC)
# Extract image file name (without full path) and include class label
images = [(os.path.basename(image), _class)
for image, _class in zip(images, classes)]
# Load existing annotations
annotations_file = os.path.join(tags_dir, f'{videos[idx]}.json')
if os.path.exists(annotations_file):
with open(annotations_file, 'r') as f:
data = json.load(f)
annotations = data['time_annotation']
else:
# Use "background" label for all frames per default
annotations = [0] * len(images)
# Read tags from config
config = project_utils.load_project_config(path)
tags = config['classes'][label]
return render_template('frame_annotation.html',
images=images,
annotations=annotations,
idx=idx,
fps=16,
n_images=len(images),
video_name=videos[idx],
project_config=config,
split=split,
label=label,
path=path,
tags=tags,
project=project,
n_videos=len(videos))
def train_logreg(path):
"""
(Re-)Train a logistic regression model on all annotations that have been submitted so far.
"""
inference_engine, model_config = utils.load_feature_extractor(path)
logreg_dir = directories.get_logreg_dir(path, model_config)
logreg_path = os.path.join(logreg_dir, 'logreg.joblib')
project_config = project_utils.load_project_config(path)
classes = project_config['classes']
all_features = []
all_annotations = []
for split in SPLITS:
for label, class_tags in classes.items():
videos_dir = directories.get_videos_dir(path, split, label)
frames_dir = directories.get_frames_dir(path, split, label)
features_dir = directories.get_features_dir(path,
split,
model_config,
label=label)
tags_dir = directories.get_tags_dir(path, split, label)
if not os.path.exists(tags_dir):
continue
# Compute the respective frames and features
compute_frames_and_features(inference_engine=inference_engine,
project_path=path,
videos_dir=videos_dir,
frames_dir=frames_dir,
features_dir=features_dir)
video_tag_files = os.listdir(tags_dir)
for video_tag_file in video_tag_files:
feature_file = os.path.join(
features_dir, video_tag_file.replace('.json', '.npy'))
annotation_file = os.path.join(tags_dir, video_tag_file)
features = np.load(feature_file)
for f in features:
all_features.append(f.mean(axis=(1, 2)))
with open(annotation_file, 'r') as f:
annotations = json.load(f)['time_annotation']
# Reset tags that have been removed from the class to 'background'
annotations = [
tag_idx if tag_idx in class_tags else 0
for tag_idx in annotations
]
all_annotations.extend(annotations)
# Use low class weight for background and higher weight for all present tags
annotated_tags = set(all_annotations)
class_weight = {tag: 2 for tag in annotated_tags}
class_weight[0] = 0.5
all_features = np.array(all_features)
all_annotations = np.array(all_annotations)
if len(annotated_tags) > 1:
os.makedirs(logreg_dir, exist_ok=True)
logreg = LogisticRegression(C=0.1, class_weight=class_weight)
logreg.fit(all_features, all_annotations)
dump(logreg, logreg_path)
def flip_videos():
"""
Flip the videos horizontally for given class and
copy tags of selected original videos for flipped version of it.
"""
data = request.json
project = data['projectName']
path = project_utils.lookup_project_path(project)
config = project_utils.load_project_config(path)
counterpart_class_name = str(data['counterpartClassName'])
original_class_name = str(data['originalClassName'])
copy_video_tags = data['videosToCopyTags']
if counterpart_class_name not in config['classes']:
config['classes'][counterpart_class_name] = config['classes'][original_class_name] \
if copy_video_tags['train'] or copy_video_tags['valid'] else []
project_utils.write_project_config(path, config)
for split in SPLITS:
videos_path_in = os.path.join(path, f'videos_{split}', original_class_name)
videos_path_out = os.path.join(path, f'videos_{split}', counterpart_class_name)
original_tags_path = os.path.join(path, f'tags_{split}', original_class_name)
counterpart_tags_path = os.path.join(path, f'tags_{split}', counterpart_class_name)
# Create directory to save flipped videos
os.makedirs(videos_path_out, exist_ok=True)
os.makedirs(counterpart_tags_path, exist_ok=True)
video_list = [video for video in os.listdir(videos_path_in) if video.endswith(VIDEO_EXT)]
for video in video_list:
output_video_list = [op_video for op_video in os.listdir(videos_path_out) if op_video.endswith(VIDEO_EXT)]
print(f'Processing video: {video}')
if '_flipped' in video:
flipped_video_name = ''.join(video.split('_flipped'))
else:
flipped_video_name = video.split('.')[0] + '_flipped' + VIDEO_EXT
if flipped_video_name not in output_video_list:
# Original video as input
original_video = ffmpeg.input(os.path.join(videos_path_in, video))
# Do horizontal flip
flipped_video = ffmpeg.hflip(original_video)
# Get flipped video output
flipped_video_output = ffmpeg.output(flipped_video,
filename=os.path.join(videos_path_out, flipped_video_name))
# Run to render and save video
ffmpeg.run(flipped_video_output)
# Copy tags of original video to flipped video (in train/valid set)
if video in copy_video_tags[split]:
original_tags_file = os.path.join(original_tags_path, video.replace(VIDEO_EXT, '.json'))
flipped_tags_file = os.path.join(counterpart_tags_path,
flipped_video_name.replace(VIDEO_EXT, '.json'))
if os.path.exists(original_tags_file):
with open(original_tags_file) as f:
original_video_tags = json.load(f)
original_video_tags['file'] = flipped_video_name
with open(flipped_tags_file, 'w') as f:
f.write(json.dumps(original_video_tags, indent=2))
print("Processing complete!")
return jsonify(status=True, url=url_for("project_details", project=project))
def train_model(path_in,
path_out,
model_name,
model_version,
num_layers_to_finetune,
epochs,
use_gpu=True,
overwrite=True,
temporal_training=None,
resume=False,
log_fn=print,
confmat_event=None):
os.makedirs(path_out, exist_ok=True)
# Check for existing files
saved_files = [
"last_classifier.checkpoint", "best_classifier.checkpoint",
"config.json", "label2int.json", "confusion_matrix.png",
"confusion_matrix.npy"
]
if not overwrite and any(
os.path.exists(os.path.join(path_out, file))
for file in saved_files):
print(f"Warning: This operation will overwrite files in {path_out}")
while True:
confirmation = input(
"Are you sure? Add --overwrite to hide this warning. (Y/N) ")
if confirmation.lower() == "y":
break
elif confirmation.lower() == "n":
sys.exit()
else:
print('Invalid input')
# Load weights
selected_config, weights = get_relevant_weights(
SUPPORTED_MODEL_CONFIGURATIONS,
model_name,
model_version,
log_fn,
)
backbone_weights = weights['backbone']
if resume:
# Load the last classifier
checkpoint_classifier = torch.load(
os.path.join(path_out, 'last_classifier.checkpoint'))
# Update original weights in case some intermediate layers have been finetuned
update_backbone_weights(backbone_weights, checkpoint_classifier)
# Load backbone network
backbone_network = build_backbone_network(selected_config,
backbone_weights)
# Get the required temporal dimension of feature tensors in order to
# finetune the provided number of layers
if num_layers_to_finetune > 0:
num_timesteps = backbone_network.num_required_frames_per_layer.get(
-num_layers_to_finetune)
if not num_timesteps:
# Remove 1 because we added 0 to temporal_dependencies
num_layers = len(
backbone_network.num_required_frames_per_layer) - 1
msg = (f'ERROR - Num of layers to finetune not compatible. '
f'Must be an integer between 0 and {num_layers}')
log_fn(msg)
raise IndexError(msg)
else:
num_timesteps = 1
# Extract layers to finetune
if num_layers_to_finetune > 0:
fine_tuned_layers = backbone_network.cnn[-num_layers_to_finetune:]
backbone_network.cnn = backbone_network.cnn[0:-num_layers_to_finetune]
# finetune the model
extract_features(path_in,
selected_config,
backbone_network,
num_layers_to_finetune,
use_gpu,
num_timesteps=num_timesteps,
log_fn=log_fn)
# Find label names
label_names = os.listdir(directories.get_videos_dir(path_in, 'train'))
label_names = [x for x in label_names if not x.startswith('.')]
label_names_temporal = ['background']
project_config = load_project_config(path_in)
if project_config:
for temporal_tags in project_config['classes'].values():
label_names_temporal.extend(temporal_tags)
else:
for label in label_names:
label_names_temporal.extend([f'{label}_tag1', f'{label}_tag2'])
label_names_temporal = sorted(set(label_names_temporal))
label2int_temporal_annotation = {
name: index
for index, name in enumerate(label_names_temporal)
}
label2int = {name: index for index, name in enumerate(label_names)}
extractor_stride = backbone_network.num_required_frames_per_layer_padding[
0]
# Create the data loaders
features_dir = directories.get_features_dir(path_in, 'train',
selected_config,
num_layers_to_finetune)
tags_dir = directories.get_tags_dir(path_in, 'train')
train_loader = generate_data_loader(
project_config,
features_dir,
tags_dir,
label_names,
label2int,
label2int_temporal_annotation,
num_timesteps=num_timesteps,
stride=extractor_stride,
temporal_annotation_only=temporal_training,
)
features_dir = directories.get_features_dir(path_in, 'valid',
selected_config,
num_layers_to_finetune)
tags_dir = directories.get_tags_dir(path_in, 'valid')
valid_loader = generate_data_loader(
project_config,
features_dir,
tags_dir,
label_names,
label2int,
label2int_temporal_annotation,
num_timesteps=None,
batch_size=1,
shuffle=False,
stride=extractor_stride,
temporal_annotation_only=temporal_training,
)
# Check if the data is loaded fully
if not train_loader or not valid_loader:
log_fn(
"ERROR - \n "
"\tMissing annotations for train or valid set.\n"
"\tHint: Check if tags_train and tags_valid directories exist.\n")
return
# Modify the network to generate the training network on top of the features
if temporal_training:
num_output = len(label_names_temporal)
else:
num_output = len(label_names)
# modify the network to generate the training network on top of the features
gesture_classifier = LogisticRegression(
num_in=backbone_network.feature_dim,
num_out=num_output,
use_softmax=False)
if resume:
gesture_classifier.load_state_dict(checkpoint_classifier)
if num_layers_to_finetune > 0:
# remove internal padding for training
fine_tuned_layers.apply(set_internal_padding_false)
net = Pipe(fine_tuned_layers, gesture_classifier)
else:
net = gesture_classifier
net.train()
if use_gpu:
net = net.cuda()
lr_schedule = {
0: 0.0001,
int(epochs / 2): 0.00001
} if epochs > 1 else {
0: 0.0001
}
num_epochs = epochs
# Save training config and label2int dictionary
config = {
'backbone_name': selected_config.model_name,
'backbone_version': selected_config.version,
'num_layers_to_finetune': num_layers_to_finetune,
'classifier': str(gesture_classifier),
'temporal_training': temporal_training,
'lr_schedule': lr_schedule,
'num_epochs': num_epochs,
'start_time': str(datetime.datetime.now()),
'end_time': '',
}
with open(os.path.join(path_out, 'config.json'), 'w') as f:
json.dump(config, f, indent=2)
with open(os.path.join(path_out, 'label2int.json'), 'w') as f:
json.dump(
label2int_temporal_annotation if temporal_training else label2int,
f,
indent=2)
# Train model
best_model_state_dict = training_loops(
net,
train_loader,
valid_loader,
use_gpu,
num_epochs,
lr_schedule,
label_names,
path_out,
temporal_annotation_training=temporal_training,
log_fn=log_fn,
confmat_event=confmat_event)
# Save best model
if isinstance(net, Pipe):
best_model_state_dict = {
clean_pipe_state_dict_key(key): value
for key, value in best_model_state_dict.items()
}
torch.save(best_model_state_dict,
os.path.join(path_out, "best_classifier.checkpoint"))
config['end_time'] = str(datetime.datetime.now())
with open(os.path.join(path_out, 'config.json'), 'w') as f:
json.dump(config, f, indent=2)
def inject_project_config(project):
path = project_utils.lookup_project_path(project)
return project_utils.load_project_config(path)
