def update_window(window, movie, nfo_lines, index, movie_count, quality_names, custom_format_names): def get_mediainfo(key, default=""): return get_by_path(movie, ["movieFile", "mediaInfo", key], default) update_key(window, "AUDIO_FMT", get_mediainfo("audioFormat")) update_key(window, "AUDIO_CHANNELS", get_mediainfo("audioChannels")) width = get_mediainfo("width", 0) height = get_mediainfo("height", 0) update_key(window, "VIDEO_DIMENSIONS", f"{width}x{height}") update_key(window, "VIDEO_BITRATE", f"{int(get_mediainfo('videoBitrate')/1024)} KB/s") update_key(window, "VIDEO_CODEC_ID", get_mediainfo("videoCodecID")) update_key(window, "VIDEO_CODEC_LIBRARY", get_mediainfo("videoCodecLibrary")) update_key(window, "VIDEO_FMT", get_mediainfo("videoFormat")) update_key(window, "VIDEO_FPS", get_mediainfo("videoFps")) update_key(window, "VIDEO_PROFILE", get_mediainfo("videoProfile")) update_key( window, "FILE_CONTAINER", get_mediainfo("containerFormat"), ) size = (humanbytes_storage(movie.get("sizeOnDisk")) if movie.get("sizeOnDisk") else "N/A") update_key(window, "FILE_SIZE", size) movie_path = "" if movie.get("path") and get_by_path(movie, ["movieFile", "relativePath"]): movie_path = Path(movie["path"]) / get_by_path( movie, ["movieFile", "relativePath"]) update_key(window, "FILE_PATH", movie_path) update_key(window, "MOVIE_TITLE", movie["title"]) update_key(window, "PROGRESS", f"{index+1}/{movie_count}") # Reset selection to movie's quality movie_quality = (get_by_path( movie, ["movieFile", "quality", "quality", "name"]) or UNKNOWN_QUALITY) quality_select_index = quality_names.index("Bluray-1080p") format_select_index = custom_format_names.index("Complex Surround") window["__QUAL__"].update(set_to_index=quality_select_index) # Clear selection window["__FMT__"].update(set_to_index=format_select_index) window["__NFO_LINES__"].update( "\n".join(nfo_lines) if nfo_lines else "No .nfo to show.")
def get_custom_formats(): api_fmts = _get(CUSTOM_FORMAT_PATH).json() # Radarr is messed up. See https://github.com/Radarr/Radarr/issues/4049 custom_fmt_names = sorted([f["name"] for f in api_fmts]) found_api_fmts = {} found_all_fmts = False for movie in get_movies(): for cf in get_by_path( movie, ["movieFile", "quality", "customFormats"], default=[] ): if cf["name"] not in found_api_fmts: found_api_fmts[cf["name"]] = cf # if sorted(found_api_fmts.keys()) == custom_fmt_names: # found_all_fmts = True # break # # if found_all_fmts: # break if sorted(found_api_fmts.keys()) != custom_fmt_names: print("Couldn't find all custom formats from movie files.") return found_api_fmts
def update_audio(): count = 0 custom_formats = get_custom_formats() needs_updating = [] for movie in get_movies(): if not movie.get("movieFile"): continue count += 1 audio_format = get_by_path(movie, ["movieFile", "mediaInfo", "audioFormat"]) audio_channels = get_by_path(movie, ["movieFile", "mediaInfo", "audioChannels"]) cf = None if audio_channels >= 6: cf_names = [ cf["name"] for cf in get_by_path(movie, ["movieFile", "quality", "customFormats"]) ] if "Complex Surround" not in cf_names: needs_updating.append(movie) count = 0 for movie in needs_updating: count += 1 audio_format = get_by_path(movie, ["movieFile", "mediaInfo", "audioFormat"]) audio_channels = get_by_path(movie, ["movieFile", "mediaInfo", "audioChannels"]) print(f"Adding Complex Surround to {count}/{len(needs_updating)}: ") print(movie["title"], audio_format, audio_channels) moviefile = get_moviefile(get_by_path(movie, ["movieFile", "id"])) assert moviefile add_custom_format(custom_formats["Complex Surround"], moviefile)
def update_unk_blu_complex(): updates = [] for movie in get_movies_for_downloaded_quality("Unknown"): width = get_by_path(movie, ["movieFile", "mediaInfo", "width"], 0) channels = get_by_path(movie, ["movieFile", "mediaInfo", "audioChannels"], 0) if 1900 <= width <= 1920 and channels > 3: updates.append(movie) if updates: blu_qual = get_quality_by_name("Bluray-1080p") custom_formats = [get_custom_formats()["Complex Surround"]] assert blu_qual for movie in updates: print(movie["title"]) moviefile = movie.get("movieFile") assert moviefile moviefile = get_moviefile(moviefile["id"]) assert moviefile set_quality(blu_qual, moviefile) set_custom_formats(custom_formats, moviefile) print(len(updates))
def get_mediainfo(key, default=""): return get_by_path(movie, ["movieFile", "mediaInfo", key], default)
window.finalize() # populate window with initial values update_window( window, movie, nfo_lines, idx, len(movies), quality_names, custom_format_names, ) while True: # get the quality data for currently-displayed movie current_quality_name = get_by_path( movie, ["movieFile", "quality", "quality", "name"]) current_formats_names = sorted([ cf["name"] for cf in get_by_path( movie, ["movieFile", "quality", "customFormats"], default=[]) ]) event, values = window.read() if event in (None, "Exit"): break if event == "Next": idx += 1 # get the quality data selected in the GUI selected_quality_name = values.get("__QUAL__", [None])[0]
def train(config): # setup GPU device if available, move model into configured device device, device_ids = prepare_device(config["n_gpu"]) # datasets train_datasets = dict() valid_datasets = dict() ## train keys = ["datasets", "train"] for name in get_by_path(config, keys): train_datasets[name] = config.init_obj([*keys, name]) ## valid valid_exist = False keys = ["datasets", "valid"] for name in get_by_path(config, keys): valid_exist = True valid_datasets[name] = config.init_obj([*keys, name]) ## compute inverse class frequency as class weight if config["datasets"].get("imbalanced", False): target = train_datasets["data"].y_train # TODO class_weight = compute_class_weight(class_weight="balanced", classes=target.unique(), y=target) class_weight = torch.FloatTensor(class_weight).to(device) else: class_weight = None # losses losses = dict() for name in config["losses"]: kwargs = {} if "balanced" in get_by_path(config, ["losses", name, "type"]): kwargs.update(class_weight=class_weight) losses[name] = config.init_obj(["losses", name], **kwargs) # metrics metrics_iter = [ getattr(module_metric, met) for met in config["metrics"]["per_iteration"] ] metrics_epoch = [ getattr(module_metric, met) for met in config["metrics"]["per_epoch"] ] if "pick_threshold" in config["metrics"]: metrics_threshold = config.init_obj(["metrics", "pick_threshold"]) else: metrics_threshold = None torch_objs = { "datasets": { "train": train_datasets, "valid": valid_datasets }, "losses": losses, "metrics": { "iter": metrics_iter, "epoch": metrics_epoch, "threshold": metrics_threshold, }, } k_fold = config["k_fold"] if k_fold > 1: # cross validation enabled train_datasets["data"].split_cv_indexes(k_fold) results = pd.DataFrame() Cross_Valid.create_CV(k_fold) start = time.time() for k in range(k_fold): # data_loaders train_data_loaders = dict() valid_data_loaders = dict() ## train keys = ["data_loaders", "train"] for name in get_by_path(config, keys): kwargs = {} if "imbalanced" in get_by_path(config, [*keys, name, "module"]): kwargs.update(class_weight=class_weight.cpu().detach().numpy(), target=target) dataset = train_datasets[name] loaders = config.init_obj([*keys, name], dataset, **kwargs) train_data_loaders[name] = loaders.train_loader if not valid_exist: valid_data_loaders[name] = loaders.valid_loader ## valid keys = ["data_loaders", "valid"] for name in get_by_path(config, keys): dataset = valid_datasets[name] loaders = config.init_obj([*keys, name], dataset) valid_data_loaders[name] = loaders.valid_loader # models models = dict() logger_model = get_logger("model", verbosity=1) for name in config["models"]: model = config.init_obj(["models", name]) logger_model.info(model) model = model.to(device) if len(device_ids) > 1: model = torch.nn.DataParallel(model, device_ids=device_ids) models[name] = model # optimizers optimizers = dict() for name in config["optimizers"]: trainable_params = filter(lambda p: p.requires_grad, models[name].parameters()) optimizers[name] = config.init_obj(["optimizers", name], trainable_params) # learning rate schedulers lr_schedulers = dict() for name in config["lr_schedulers"]: lr_schedulers[name] = config.init_obj(["lr_schedulers", name], optimizers[name]) torch_objs.update({ "data_loaders": { "train": train_data_loaders, "valid": valid_data_loaders, }, "models": models, "optimizers": optimizers, "lr_schedulers": lr_schedulers, "amp": None, }) # amp if config["trainer"]["kwargs"]["apex"]: # TODO: revise here if multiple models and optimizers models["model"], optimizers["model"] = amp.initialize( models["model"], optimizers["model"], opt_level="O1") torch_objs["amp"] = amp trainer = config.init_obj(["trainer"], torch_objs, config.save_dir, config.resume, device) train_log = trainer.train() results = pd.concat((results, train_log), axis=1) # cross validation if k_fold > 1: Cross_Valid.next_fold() msg = msg_box("result") end = time.time() total_time = consuming_time(start, end) msg += f"\nConsuming time: {total_time}." result = pd.DataFrame() result["mean"] = results.mean(axis=1) result["std"] = results.std(axis=1) msg += f"\n{result}" logger.info(msg) return result
def test(config): # setup GPU device if available, move model into configured device device, device_ids = prepare_device(config["n_gpu"]) # datasets test_datasets = dict() keys = ["datasets", "test"] for name in get_by_path(config, keys): test_datasets[name] = config.init_obj([*keys, name]) results = pd.DataFrame() k_fold = config["k_fold"] Cross_Valid.create_CV(k_fold) start = time.time() for k in range(k_fold): # data_loaders test_data_loaders = dict() keys = ["data_loaders", "test"] for name in get_by_path(config, keys): dataset = test_datasets[name] loaders = config.init_obj([*keys, name], dataset) test_data_loaders[name] = loaders.test_loader # models if k_fold > 1: fold_prefix = f"fold_{k}_" dirname = os.path.dirname(config.resume) basename = os.path.basename(config.resume) resume = os.path.join(dirname, fold_prefix + basename) else: resume = config.resume logger.info(f"Loading model: {resume} ...") checkpoint = torch.load(resume) models = dict() logger_model = get_logger("model", verbosity=0) for name in config["models"]: model = config.init_obj(["models", name]) logger_model.info(model) state_dict = checkpoint["models"][name] if config["n_gpu"] > 1: model = torch.nn.DataParallel(model) model.load_state_dict(state_dict) model = model.to(device) model.eval() models[name] = model model = models["model"] # losses loss_fn = config.init_obj(["losses", "loss"]) # metrics metrics_epoch = [ getattr(module_metric, met) for met in config["metrics"]["per_epoch"] ] keys_epoch = [m.__name__ for m in metrics_epoch] test_metrics = MetricTracker([], keys_epoch) if "pick_threshold" in config["metrics"]: threshold = checkpoint["threshold"] setattr(module_metric, "THRESHOLD", threshold) logger.info(f"threshold: {threshold}") with torch.no_grad(): print("testing...") test_loader = test_data_loaders["data"] if len(metrics_epoch) > 0: outputs = torch.FloatTensor().to(device) targets = torch.FloatTensor().to(device) for batch_idx, (data, target) in enumerate(test_loader): data, target = data.to(device), target.to(device) output = model(data) if len(metrics_epoch) > 0: outputs = torch.cat((outputs, output)) targets = torch.cat((targets, target)) # # save sample images, or do something with output here # for met in metrics_epoch: test_metrics.epoch_update(met.__name__, met(targets, outputs)) test_log = test_metrics.result() test_log = test_log["mean"].rename(k) results = pd.concat((results, test_log), axis=1) logger.info(test_log) # cross validation if k_fold > 1: Cross_Valid.next_fold() msg = msg_box("result") end = time.time() total_time = consuming_time(start, end) msg += f"\nConsuming time: {total_time}." result = pd.DataFrame() result["mean"] = results.mean(axis=1) result["std"] = results.std(axis=1) msg += f"\n{result}" logger.info(msg) # bootstrap if config.test_args.bootstrapping: assert k_fold == 1, "k-fold ensemble and bootstrap are mutually exclusive." N = config.test_args.bootstrap_times bootstrapping(targets, outputs, metrics_epoch, test_metrics, repeat=N)