def project_details(project): """ Show the details for the selected project. """ project = urllib.parse.unquote(project) path = project_utils.lookup_project_path(project) config = project_utils.load_project_config(path) stats = {} for class_name, tags in config['classes'].items(): stats[class_name] = {} for split in SPLITS: videos_dir = directories.get_videos_dir(path, split, class_name) tags_dir = directories.get_tags_dir(path, split, class_name) stats[class_name][split] = { 'total': len(os.listdir(videos_dir)), 'tagged': len(os.listdir(tags_dir)) if os.path.exists(tags_dir) else 0, } return render_template('project_details.html', config=config, path=path, stats=stats, project=config['name'])
def training_page(project): project = urllib.parse.unquote(project) path = project_utils.lookup_project_path(project) project_config = project_utils.load_project_config(path) output_path_prefix = os.path.join(os.path.basename(path), 'checkpoints', '') return render_template('training.html', project=project, path=path, models=utils.get_available_backbone_models(), output_path_prefix=output_path_prefix, project_config=project_config)
def edit_class(project, class_name): """ Edit the class name and tags for an existing class in the given project. """ project = urllib.parse.unquote(project) class_name = urllib.parse.unquote(class_name) path = project_utils.lookup_project_path(project) # Get new class name and tags new_class_name, new_tag1, new_tag2 = utils.get_class_name_and_tags( request.form) # Update project config config = project_utils.load_project_config(path) del config['classes'][class_name] config['classes'][new_class_name] = [new_tag1, new_tag2] project_utils.write_project_config(path, config) # Update directory names data_dirs = [] for split in SPLITS: data_dirs.extend([ directories.get_videos_dir(path, split), directories.get_frames_dir(path, split), directories.get_tags_dir(path, split), ]) # Feature directories follow the format <dataset_dir>/<split>/<model>/<num_layers_to_finetune>/<label> features_dir = directories.get_features_dir(path, split) if os.path.exists(features_dir): model_dirs = [ os.path.join(features_dir, model_dir) for model_dir in os.listdir(features_dir) ] data_dirs.extend([ os.path.join(model_dir, tuned_layers) for model_dir in model_dirs for tuned_layers in os.listdir(model_dir) ]) logreg_dir = directories.get_logreg_dir(path) if os.path.exists(logreg_dir): data_dirs.extend([ os.path.join(logreg_dir, model_dir) for model_dir in os.listdir(logreg_dir) ]) for base_dir in data_dirs: class_dir = os.path.join(base_dir, class_name) if os.path.exists(class_dir): new_class_dir = os.path.join(base_dir, new_class_name) os.rename(class_dir, new_class_dir) return redirect(url_for('project_details', project=project))
def project_config(): """ Provide the config for a given project. """ data = request.json name = data['name'] path = project_utils.lookup_project_path(name) # Get config config = project_utils.load_project_config(path) return jsonify(config)
def edit_tag(project, tag_idx): data = request.form project = urllib.parse.unquote(project) path = project_utils.lookup_project_path(project) config = project_utils.load_project_config(path) tags = config['tags'] new_tag_name = data['newTagName'] # Update tag name tags[tag_idx] = new_tag_name project_utils.write_project_config(path, config) return redirect(url_for('project_details', project=project))
def remove_tag_from_class(): """ Remove selected tag from class label in project config. """ data = request.json path = data['path'] tag_index = data['tagIndex'] class_name = data['className'] config = project_utils.load_project_config(path) config['classes'][class_name].remove(int(tag_index)) project_utils.write_project_config(path, config) return jsonify(success=True)
def remove_class(project, class_name): """ Remove the given class from the config file of the given project. No data will be deleted. """ project = urllib.parse.unquote(project) class_name = urllib.parse.unquote(class_name) path = project_utils.lookup_project_path(project) # Update project config config = project_utils.load_project_config(path) del config['classes'][class_name] project_utils.write_project_config(path, config) return redirect(url_for("project_details", project=project))
def create_tag(project): data = request.form project = urllib.parse.unquote(project) path = project_utils.lookup_project_path(project) config = project_utils.load_project_config(path) tag_name = data['newTagName'] tags = config['tags'] new_tag_index = config['max_tag_index'] + 1 tags[new_tag_index] = tag_name config['max_tag_index'] = new_tag_index project_utils.write_project_config(path, config) return redirect(url_for('project_details', project=project))
def remove_tag(project, tag_idx): project = urllib.parse.unquote(project) path = project_utils.lookup_project_path(project) config = project_utils.load_project_config(path) tags = config['tags'] # Remove tag from the overall tags list del tags[tag_idx] # Remove tag from the classes for class_label, class_tags in config['classes'].items(): if tag_idx in class_tags: class_tags.remove(tag_idx) project_utils.write_project_config(path, config) return redirect(url_for('project_details', project=project))
def assign_tag_to_class(): """ Assign selected tag to class label in project config. """ data = request.json path = data['path'] tag_index = data['tagIndex'] class_name = data['className'] config = project_utils.load_project_config(path) class_tags = config['classes'][class_name] class_tags.append(int(tag_index)) class_tags.sort() project_utils.write_project_config(path, config) return jsonify(success=True)
def update_project(): """ Update an existing project entry with a new path. If a config file exists in there, it will be used, otherwise a new one will be created. The project will keep the given project name. """ data = request.form project_name = data['projectName'] path = data['path'] # Check for existing config file (might be None) config = project_utils.load_project_config(path) # Make sure the directory is correctly set up project_utils.setup_new_project(project_name, path, config) return redirect(url_for('project_details', project=project_name))
def import_project(): """ Import an existing project from the given path. If a config file exists in there, it will be used while also making sure that the project name is still unique. Otherwise, a new config will be created and a unique project name will be constructed from the directory name. """ data = request.form path = data['path'] # Check for existing config file and make sure project name is unique config = project_utils.load_project_config(path) if config: project_name = project_utils.get_unique_project_name(config['name']) else: # Use folder name as project name and make sure it is unique project_name = project_utils.get_unique_project_name(os.path.basename(path)) # Make sure the directory is correctly set up project_utils.setup_new_project(project_name, path, config) return redirect(url_for('project_details', project=project_name))
def start_testing(): data = request.json path_in = data['inputVideoPath'] output_video_name = data['outputVideoName'] title = data['title'] path = data['path'] custom_classifier = os.path.join(path, data['classifier']) config = project_utils.load_project_config(path) if output_video_name: output_dir = os.path.join(path, 'output_videos') os.makedirs(output_dir, exist_ok=True) path_out = os.path.join(output_dir, output_video_name + '.mp4') else: path_out = None ctx = multiprocessing.get_context('spawn') global queue_testing_output global stop_event queue_testing_output = ctx.Queue() stop_event = ctx.Event() testing_kwargs = { 'custom_classifier': custom_classifier, 'path_in': path_in, 'path_out': path_out, 'title': title, 'use_gpu': config['use_gpu'], 'display_fn': queue_testing_output.put, 'stop_event': stop_event, } global test_process test_process = ctx.Process(target=run_custom_classifier, kwargs=testing_kwargs) test_process.start() return jsonify(success=True)
def project_details(project): """ Show the details for the selected project. """ project = urllib.parse.unquote(project) path = project_utils.lookup_project_path(project) config = project_utils.load_project_config(path) stats = {} for class_name in config['classes']: stats[class_name] = {} for split in SPLITS: videos_dir = directories.get_videos_dir(path, split, class_name) tags_dir = directories.get_tags_dir(path, split, class_name) stats[class_name][split] = { 'total': len(os.listdir(videos_dir)), 'tagged': len(os.listdir(tags_dir)) if os.path.exists(tags_dir) else 0, 'videos': natsorted([video for video in os.listdir(videos_dir) if video.endswith(VIDEO_EXT)], alg=ns.IC) } tags = config['tags'] return render_template('project_details.html', config=config, path=path, stats=stats, project=config['name'], tags=tags)
def add_class(project): """ Add a new class to the given project. """ data = request.form project = urllib.parse.unquote(project) path = project_utils.lookup_project_path(project) class_name = data['className'] # Update project config config = project_utils.load_project_config(path) config['classes'][class_name] = [] project_utils.write_project_config(path, config) # Setup directory structure for split in SPLITS: videos_dir = directories.get_videos_dir(path, split, class_name) if not os.path.exists(videos_dir): os.mkdir(videos_dir) return redirect(url_for("project_details", project=project))
def start_training(): data = request.json path = data['path'] num_layers_to_finetune = data['layersToFinetune'] output_folder = data['outputFolder'] model_name = data['modelName'] epochs = data['epochs'] config = project_utils.load_project_config(path) model_name, model_version = model_name.split('-') path_out = os.path.join(path, 'checkpoints', output_folder) ctx = multiprocessing.get_context('spawn') global queue_train_logs global confmat_event queue_train_logs = ctx.Queue() confmat_event = ctx.Event() training_kwargs = { 'path_in': path, 'num_layers_to_finetune': int(num_layers_to_finetune), 'path_out': path_out, 'model_version': model_version, 'model_name': model_name, 'epochs': int(epochs), 'use_gpu': config['use_gpu'], 'temporal_training': config['temporal'], 'log_fn': queue_train_logs.put, 'confmat_event': confmat_event, } global train_process train_process = ctx.Process(target=train_model, kwargs=training_kwargs) train_process.start() return jsonify(success=True)
def annotate(project, split, label, idx): """ For the given class label, show all frames for annotating the selected video. """ project = urllib.parse.unquote(project) path = project_utils.lookup_project_path(project) label = urllib.parse.unquote(label) split = urllib.parse.unquote(split) _, model_config = utils.load_feature_extractor(path) frames_dir = directories.get_frames_dir(path, split, label) features_dir = directories.get_features_dir(path, split, model_config, label=label) tags_dir = directories.get_tags_dir(path, split, label) logreg_dir = directories.get_logreg_dir(path, model_config, label) videos = os.listdir(frames_dir) videos = natsorted(videos, alg=ns.IC) # The list of images in the folder images = [ image for image in glob.glob(os.path.join(frames_dir, videos[idx], '*')) if utils.is_image_file(image) ] classes = [-1] * len(images) # Load logistic regression model if available and assisted tagging is enabled if utils.get_project_setting(path, 'assisted_tagging'): logreg_path = os.path.join(logreg_dir, 'logreg.joblib') features_path = os.path.join(features_dir, f'{videos[idx]}.npy') if os.path.isfile(logreg_path) and os.path.isfile(features_path): logreg = load(logreg_path) features = np.load(features_path).mean(axis=(2, 3)) classes = list(logreg.predict(features)) # Natural sort images, so that they are sorted by number images = natsorted(images, alg=ns.IC) # Extract image file name (without full path) and include class label images = [(os.path.basename(image), _class) for image, _class in zip(images, classes)] # Load existing annotations annotations_file = os.path.join(tags_dir, f'{videos[idx]}.json') if os.path.exists(annotations_file): with open(annotations_file, 'r') as f: data = json.load(f) annotations = data['time_annotation'] else: # Use "background" label for all frames per default annotations = [0] * len(images) # Read tags from config config = project_utils.load_project_config(path) tags = config['classes'][label] return render_template('frame_annotation.html', images=images, annotations=annotations, idx=idx, fps=16, n_images=len(images), video_name=videos[idx], project_config=config, split=split, label=label, path=path, tags=tags, project=project, n_videos=len(videos))
def train_logreg(path): """ (Re-)Train a logistic regression model on all annotations that have been submitted so far. """ inference_engine, model_config = utils.load_feature_extractor(path) logreg_dir = directories.get_logreg_dir(path, model_config) logreg_path = os.path.join(logreg_dir, 'logreg.joblib') project_config = project_utils.load_project_config(path) classes = project_config['classes'] all_features = [] all_annotations = [] for split in SPLITS: for label, class_tags in classes.items(): videos_dir = directories.get_videos_dir(path, split, label) frames_dir = directories.get_frames_dir(path, split, label) features_dir = directories.get_features_dir(path, split, model_config, label=label) tags_dir = directories.get_tags_dir(path, split, label) if not os.path.exists(tags_dir): continue # Compute the respective frames and features compute_frames_and_features(inference_engine=inference_engine, project_path=path, videos_dir=videos_dir, frames_dir=frames_dir, features_dir=features_dir) video_tag_files = os.listdir(tags_dir) for video_tag_file in video_tag_files: feature_file = os.path.join( features_dir, video_tag_file.replace('.json', '.npy')) annotation_file = os.path.join(tags_dir, video_tag_file) features = np.load(feature_file) for f in features: all_features.append(f.mean(axis=(1, 2))) with open(annotation_file, 'r') as f: annotations = json.load(f)['time_annotation'] # Reset tags that have been removed from the class to 'background' annotations = [ tag_idx if tag_idx in class_tags else 0 for tag_idx in annotations ] all_annotations.extend(annotations) # Use low class weight for background and higher weight for all present tags annotated_tags = set(all_annotations) class_weight = {tag: 2 for tag in annotated_tags} class_weight[0] = 0.5 all_features = np.array(all_features) all_annotations = np.array(all_annotations) if len(annotated_tags) > 1: os.makedirs(logreg_dir, exist_ok=True) logreg = LogisticRegression(C=0.1, class_weight=class_weight) logreg.fit(all_features, all_annotations) dump(logreg, logreg_path)
def flip_videos(): """ Flip the videos horizontally for given class and copy tags of selected original videos for flipped version of it. """ data = request.json project = data['projectName'] path = project_utils.lookup_project_path(project) config = project_utils.load_project_config(path) counterpart_class_name = str(data['counterpartClassName']) original_class_name = str(data['originalClassName']) copy_video_tags = data['videosToCopyTags'] if counterpart_class_name not in config['classes']: config['classes'][counterpart_class_name] = config['classes'][original_class_name] \ if copy_video_tags['train'] or copy_video_tags['valid'] else [] project_utils.write_project_config(path, config) for split in SPLITS: videos_path_in = os.path.join(path, f'videos_{split}', original_class_name) videos_path_out = os.path.join(path, f'videos_{split}', counterpart_class_name) original_tags_path = os.path.join(path, f'tags_{split}', original_class_name) counterpart_tags_path = os.path.join(path, f'tags_{split}', counterpart_class_name) # Create directory to save flipped videos os.makedirs(videos_path_out, exist_ok=True) os.makedirs(counterpart_tags_path, exist_ok=True) video_list = [video for video in os.listdir(videos_path_in) if video.endswith(VIDEO_EXT)] for video in video_list: output_video_list = [op_video for op_video in os.listdir(videos_path_out) if op_video.endswith(VIDEO_EXT)] print(f'Processing video: {video}') if '_flipped' in video: flipped_video_name = ''.join(video.split('_flipped')) else: flipped_video_name = video.split('.')[0] + '_flipped' + VIDEO_EXT if flipped_video_name not in output_video_list: # Original video as input original_video = ffmpeg.input(os.path.join(videos_path_in, video)) # Do horizontal flip flipped_video = ffmpeg.hflip(original_video) # Get flipped video output flipped_video_output = ffmpeg.output(flipped_video, filename=os.path.join(videos_path_out, flipped_video_name)) # Run to render and save video ffmpeg.run(flipped_video_output) # Copy tags of original video to flipped video (in train/valid set) if video in copy_video_tags[split]: original_tags_file = os.path.join(original_tags_path, video.replace(VIDEO_EXT, '.json')) flipped_tags_file = os.path.join(counterpart_tags_path, flipped_video_name.replace(VIDEO_EXT, '.json')) if os.path.exists(original_tags_file): with open(original_tags_file) as f: original_video_tags = json.load(f) original_video_tags['file'] = flipped_video_name with open(flipped_tags_file, 'w') as f: f.write(json.dumps(original_video_tags, indent=2)) print("Processing complete!") return jsonify(status=True, url=url_for("project_details", project=project))
def train_model(path_in, path_out, model_name, model_version, num_layers_to_finetune, epochs, use_gpu=True, overwrite=True, temporal_training=None, resume=False, log_fn=print, confmat_event=None): os.makedirs(path_out, exist_ok=True) # Check for existing files saved_files = [ "last_classifier.checkpoint", "best_classifier.checkpoint", "config.json", "label2int.json", "confusion_matrix.png", "confusion_matrix.npy" ] if not overwrite and any( os.path.exists(os.path.join(path_out, file)) for file in saved_files): print(f"Warning: This operation will overwrite files in {path_out}") while True: confirmation = input( "Are you sure? Add --overwrite to hide this warning. (Y/N) ") if confirmation.lower() == "y": break elif confirmation.lower() == "n": sys.exit() else: print('Invalid input') # Load weights selected_config, weights = get_relevant_weights( SUPPORTED_MODEL_CONFIGURATIONS, model_name, model_version, log_fn, ) backbone_weights = weights['backbone'] if resume: # Load the last classifier checkpoint_classifier = torch.load( os.path.join(path_out, 'last_classifier.checkpoint')) # Update original weights in case some intermediate layers have been finetuned update_backbone_weights(backbone_weights, checkpoint_classifier) # Load backbone network backbone_network = build_backbone_network(selected_config, backbone_weights) # Get the required temporal dimension of feature tensors in order to # finetune the provided number of layers if num_layers_to_finetune > 0: num_timesteps = backbone_network.num_required_frames_per_layer.get( -num_layers_to_finetune) if not num_timesteps: # Remove 1 because we added 0 to temporal_dependencies num_layers = len( backbone_network.num_required_frames_per_layer) - 1 msg = (f'ERROR - Num of layers to finetune not compatible. ' f'Must be an integer between 0 and {num_layers}') log_fn(msg) raise IndexError(msg) else: num_timesteps = 1 # Extract layers to finetune if num_layers_to_finetune > 0: fine_tuned_layers = backbone_network.cnn[-num_layers_to_finetune:] backbone_network.cnn = backbone_network.cnn[0:-num_layers_to_finetune] # finetune the model extract_features(path_in, selected_config, backbone_network, num_layers_to_finetune, use_gpu, num_timesteps=num_timesteps, log_fn=log_fn) # Find label names label_names = os.listdir(directories.get_videos_dir(path_in, 'train')) label_names = [x for x in label_names if not x.startswith('.')] label_names_temporal = ['background'] project_config = load_project_config(path_in) if project_config: for temporal_tags in project_config['classes'].values(): label_names_temporal.extend(temporal_tags) else: for label in label_names: label_names_temporal.extend([f'{label}_tag1', f'{label}_tag2']) label_names_temporal = sorted(set(label_names_temporal)) label2int_temporal_annotation = { name: index for index, name in enumerate(label_names_temporal) } label2int = {name: index for index, name in enumerate(label_names)} extractor_stride = backbone_network.num_required_frames_per_layer_padding[ 0] # Create the data loaders features_dir = directories.get_features_dir(path_in, 'train', selected_config, num_layers_to_finetune) tags_dir = directories.get_tags_dir(path_in, 'train') train_loader = generate_data_loader( project_config, features_dir, tags_dir, label_names, label2int, label2int_temporal_annotation, num_timesteps=num_timesteps, stride=extractor_stride, temporal_annotation_only=temporal_training, ) features_dir = directories.get_features_dir(path_in, 'valid', selected_config, num_layers_to_finetune) tags_dir = directories.get_tags_dir(path_in, 'valid') valid_loader = generate_data_loader( project_config, features_dir, tags_dir, label_names, label2int, label2int_temporal_annotation, num_timesteps=None, batch_size=1, shuffle=False, stride=extractor_stride, temporal_annotation_only=temporal_training, ) # Check if the data is loaded fully if not train_loader or not valid_loader: log_fn( "ERROR - \n " "\tMissing annotations for train or valid set.\n" "\tHint: Check if tags_train and tags_valid directories exist.\n") return # Modify the network to generate the training network on top of the features if temporal_training: num_output = len(label_names_temporal) else: num_output = len(label_names) # modify the network to generate the training network on top of the features gesture_classifier = LogisticRegression( num_in=backbone_network.feature_dim, num_out=num_output, use_softmax=False) if resume: gesture_classifier.load_state_dict(checkpoint_classifier) if num_layers_to_finetune > 0: # remove internal padding for training fine_tuned_layers.apply(set_internal_padding_false) net = Pipe(fine_tuned_layers, gesture_classifier) else: net = gesture_classifier net.train() if use_gpu: net = net.cuda() lr_schedule = { 0: 0.0001, int(epochs / 2): 0.00001 } if epochs > 1 else { 0: 0.0001 } num_epochs = epochs # Save training config and label2int dictionary config = { 'backbone_name': selected_config.model_name, 'backbone_version': selected_config.version, 'num_layers_to_finetune': num_layers_to_finetune, 'classifier': str(gesture_classifier), 'temporal_training': temporal_training, 'lr_schedule': lr_schedule, 'num_epochs': num_epochs, 'start_time': str(datetime.datetime.now()), 'end_time': '', } with open(os.path.join(path_out, 'config.json'), 'w') as f: json.dump(config, f, indent=2) with open(os.path.join(path_out, 'label2int.json'), 'w') as f: json.dump( label2int_temporal_annotation if temporal_training else label2int, f, indent=2) # Train model best_model_state_dict = training_loops( net, train_loader, valid_loader, use_gpu, num_epochs, lr_schedule, label_names, path_out, temporal_annotation_training=temporal_training, log_fn=log_fn, confmat_event=confmat_event) # Save best model if isinstance(net, Pipe): best_model_state_dict = { clean_pipe_state_dict_key(key): value for key, value in best_model_state_dict.items() } torch.save(best_model_state_dict, os.path.join(path_out, "best_classifier.checkpoint")) config['end_time'] = str(datetime.datetime.now()) with open(os.path.join(path_out, 'config.json'), 'w') as f: json.dump(config, f, indent=2)
def inject_project_config(project): path = project_utils.lookup_project_path(project) return project_utils.load_project_config(path)