def update_window(window, movie, nfo_lines, index, movie_count, quality_names,
custom_format_names):
def get_mediainfo(key, default=""):
return get_by_path(movie, ["movieFile", "mediaInfo", key], default)
update_key(window, "AUDIO_FMT", get_mediainfo("audioFormat"))
update_key(window, "AUDIO_CHANNELS", get_mediainfo("audioChannels"))
width = get_mediainfo("width", 0)
height = get_mediainfo("height", 0)
update_key(window, "VIDEO_DIMENSIONS", f"{width}x{height}")
update_key(window, "VIDEO_BITRATE",
f"{int(get_mediainfo('videoBitrate')/1024)} KB/s")
update_key(window, "VIDEO_CODEC_ID", get_mediainfo("videoCodecID"))
update_key(window, "VIDEO_CODEC_LIBRARY",
get_mediainfo("videoCodecLibrary"))
update_key(window, "VIDEO_FMT", get_mediainfo("videoFormat"))
update_key(window, "VIDEO_FPS", get_mediainfo("videoFps"))
update_key(window, "VIDEO_PROFILE", get_mediainfo("videoProfile"))
update_key(
window,
"FILE_CONTAINER",
get_mediainfo("containerFormat"),
)
size = (humanbytes_storage(movie.get("sizeOnDisk"))
if movie.get("sizeOnDisk") else "N/A")
update_key(window, "FILE_SIZE", size)
movie_path = ""
if movie.get("path") and get_by_path(movie, ["movieFile", "relativePath"]):
movie_path = Path(movie["path"]) / get_by_path(
movie, ["movieFile", "relativePath"])
update_key(window, "FILE_PATH", movie_path)
update_key(window, "MOVIE_TITLE", movie["title"])
update_key(window, "PROGRESS", f"{index+1}/{movie_count}")
# Reset selection to movie's quality
movie_quality = (get_by_path(
movie, ["movieFile", "quality", "quality", "name"]) or UNKNOWN_QUALITY)
quality_select_index = quality_names.index("Bluray-1080p")
format_select_index = custom_format_names.index("Complex Surround")
window["__QUAL__"].update(set_to_index=quality_select_index)
# Clear selection
window["__FMT__"].update(set_to_index=format_select_index)
window["__NFO_LINES__"].update(
"\n".join(nfo_lines) if nfo_lines else "No .nfo to show.")
def get_custom_formats():
api_fmts = _get(CUSTOM_FORMAT_PATH).json()
# Radarr is messed up. See https://github.com/Radarr/Radarr/issues/4049
custom_fmt_names = sorted([f["name"] for f in api_fmts])
found_api_fmts = {}
found_all_fmts = False
for movie in get_movies():
for cf in get_by_path(
movie, ["movieFile", "quality", "customFormats"], default=[]
):
if cf["name"] not in found_api_fmts:
found_api_fmts[cf["name"]] = cf
# if sorted(found_api_fmts.keys()) == custom_fmt_names:
# found_all_fmts = True
# break
#
# if found_all_fmts:
# break
if sorted(found_api_fmts.keys()) != custom_fmt_names:
print("Couldn't find all custom formats from movie files.")
return found_api_fmts
def update_audio():
count = 0
custom_formats = get_custom_formats()
needs_updating = []
for movie in get_movies():
if not movie.get("movieFile"):
continue
count += 1
audio_format = get_by_path(movie, ["movieFile", "mediaInfo", "audioFormat"])
audio_channels = get_by_path(movie, ["movieFile", "mediaInfo", "audioChannels"])
cf = None
if audio_channels >= 6:
cf_names = [
cf["name"]
for cf in get_by_path(movie, ["movieFile", "quality", "customFormats"])
]
if "Complex Surround" not in cf_names:
needs_updating.append(movie)
count = 0
for movie in needs_updating:
count += 1
audio_format = get_by_path(movie, ["movieFile", "mediaInfo", "audioFormat"])
audio_channels = get_by_path(movie, ["movieFile", "mediaInfo", "audioChannels"])
print(f"Adding Complex Surround to {count}/{len(needs_updating)}: ")
print(movie["title"], audio_format, audio_channels)
moviefile = get_moviefile(get_by_path(movie, ["movieFile", "id"]))
assert moviefile
add_custom_format(custom_formats["Complex Surround"], moviefile)
def update_unk_blu_complex():
updates = []
for movie in get_movies_for_downloaded_quality("Unknown"):
width = get_by_path(movie, ["movieFile", "mediaInfo", "width"], 0)
channels = get_by_path(movie, ["movieFile", "mediaInfo", "audioChannels"], 0)
if 1900 <= width <= 1920 and channels > 3:
updates.append(movie)
if updates:
blu_qual = get_quality_by_name("Bluray-1080p")
custom_formats = [get_custom_formats()["Complex Surround"]]
assert blu_qual
for movie in updates:
print(movie["title"])
moviefile = movie.get("movieFile")
assert moviefile
moviefile = get_moviefile(moviefile["id"])
assert moviefile
set_quality(blu_qual, moviefile)
set_custom_formats(custom_formats, moviefile)
print(len(updates))
def get_mediainfo(key, default=""):
return get_by_path(movie, ["movieFile", "mediaInfo", key], default)
window.finalize()
# populate window with initial values
update_window(
window,
movie,
nfo_lines,
idx,
len(movies),
quality_names,
custom_format_names,
)
while True:
# get the quality data for currently-displayed movie
current_quality_name = get_by_path(
movie, ["movieFile", "quality", "quality", "name"])
current_formats_names = sorted([
cf["name"] for cf in get_by_path(
movie, ["movieFile", "quality", "customFormats"],
default=[])
])
event, values = window.read()
if event in (None, "Exit"):
break
if event == "Next":
idx += 1
# get the quality data selected in the GUI
selected_quality_name = values.get("__QUAL__", [None])[0]
def train(config):
# setup GPU device if available, move model into configured device
device, device_ids = prepare_device(config["n_gpu"])
# datasets
train_datasets = dict()
valid_datasets = dict()
## train
keys = ["datasets", "train"]
for name in get_by_path(config, keys):
train_datasets[name] = config.init_obj([*keys, name])
## valid
valid_exist = False
keys = ["datasets", "valid"]
for name in get_by_path(config, keys):
valid_exist = True
valid_datasets[name] = config.init_obj([*keys, name])
## compute inverse class frequency as class weight
if config["datasets"].get("imbalanced", False):
target = train_datasets["data"].y_train # TODO
class_weight = compute_class_weight(class_weight="balanced",
classes=target.unique(),
y=target)
class_weight = torch.FloatTensor(class_weight).to(device)
else:
class_weight = None
# losses
losses = dict()
for name in config["losses"]:
kwargs = {}
if "balanced" in get_by_path(config, ["losses", name, "type"]):
kwargs.update(class_weight=class_weight)
losses[name] = config.init_obj(["losses", name], **kwargs)
# metrics
metrics_iter = [
getattr(module_metric, met)
for met in config["metrics"]["per_iteration"]
]
metrics_epoch = [
getattr(module_metric, met) for met in config["metrics"]["per_epoch"]
]
if "pick_threshold" in config["metrics"]:
metrics_threshold = config.init_obj(["metrics", "pick_threshold"])
else:
metrics_threshold = None
torch_objs = {
"datasets": {
"train": train_datasets,
"valid": valid_datasets
},
"losses": losses,
"metrics": {
"iter": metrics_iter,
"epoch": metrics_epoch,
"threshold": metrics_threshold,
},
}
k_fold = config["k_fold"]
if k_fold > 1: # cross validation enabled
train_datasets["data"].split_cv_indexes(k_fold)
results = pd.DataFrame()
Cross_Valid.create_CV(k_fold)
start = time.time()
for k in range(k_fold):
# data_loaders
train_data_loaders = dict()
valid_data_loaders = dict()
## train
keys = ["data_loaders", "train"]
for name in get_by_path(config, keys):
kwargs = {}
if "imbalanced" in get_by_path(config, [*keys, name, "module"]):
kwargs.update(class_weight=class_weight.cpu().detach().numpy(),
target=target)
dataset = train_datasets[name]
loaders = config.init_obj([*keys, name], dataset, **kwargs)
train_data_loaders[name] = loaders.train_loader
if not valid_exist:
valid_data_loaders[name] = loaders.valid_loader
## valid
keys = ["data_loaders", "valid"]
for name in get_by_path(config, keys):
dataset = valid_datasets[name]
loaders = config.init_obj([*keys, name], dataset)
valid_data_loaders[name] = loaders.valid_loader
# models
models = dict()
logger_model = get_logger("model", verbosity=1)
for name in config["models"]:
model = config.init_obj(["models", name])
logger_model.info(model)
model = model.to(device)
if len(device_ids) > 1:
model = torch.nn.DataParallel(model, device_ids=device_ids)
models[name] = model
# optimizers
optimizers = dict()
for name in config["optimizers"]:
trainable_params = filter(lambda p: p.requires_grad,
models[name].parameters())
optimizers[name] = config.init_obj(["optimizers", name],
trainable_params)
# learning rate schedulers
lr_schedulers = dict()
for name in config["lr_schedulers"]:
lr_schedulers[name] = config.init_obj(["lr_schedulers", name],
optimizers[name])
torch_objs.update({
"data_loaders": {
"train": train_data_loaders,
"valid": valid_data_loaders,
},
"models": models,
"optimizers": optimizers,
"lr_schedulers": lr_schedulers,
"amp": None,
})
# amp
if config["trainer"]["kwargs"]["apex"]:
# TODO: revise here if multiple models and optimizers
models["model"], optimizers["model"] = amp.initialize(
models["model"], optimizers["model"], opt_level="O1")
torch_objs["amp"] = amp
trainer = config.init_obj(["trainer"], torch_objs, config.save_dir,
config.resume, device)
train_log = trainer.train()
results = pd.concat((results, train_log), axis=1)
# cross validation
if k_fold > 1:
Cross_Valid.next_fold()
msg = msg_box("result")
end = time.time()
total_time = consuming_time(start, end)
msg += f"\nConsuming time: {total_time}."
result = pd.DataFrame()
result["mean"] = results.mean(axis=1)
result["std"] = results.std(axis=1)
msg += f"\n{result}"
logger.info(msg)
return result
def test(config):
# setup GPU device if available, move model into configured device
device, device_ids = prepare_device(config["n_gpu"])
# datasets
test_datasets = dict()
keys = ["datasets", "test"]
for name in get_by_path(config, keys):
test_datasets[name] = config.init_obj([*keys, name])
results = pd.DataFrame()
k_fold = config["k_fold"]
Cross_Valid.create_CV(k_fold)
start = time.time()
for k in range(k_fold):
# data_loaders
test_data_loaders = dict()
keys = ["data_loaders", "test"]
for name in get_by_path(config, keys):
dataset = test_datasets[name]
loaders = config.init_obj([*keys, name], dataset)
test_data_loaders[name] = loaders.test_loader
# models
if k_fold > 1:
fold_prefix = f"fold_{k}_"
dirname = os.path.dirname(config.resume)
basename = os.path.basename(config.resume)
resume = os.path.join(dirname, fold_prefix + basename)
else:
resume = config.resume
logger.info(f"Loading model: {resume} ...")
checkpoint = torch.load(resume)
models = dict()
logger_model = get_logger("model", verbosity=0)
for name in config["models"]:
model = config.init_obj(["models", name])
logger_model.info(model)
state_dict = checkpoint["models"][name]
if config["n_gpu"] > 1:
model = torch.nn.DataParallel(model)
model.load_state_dict(state_dict)
model = model.to(device)
model.eval()
models[name] = model
model = models["model"]
# losses
loss_fn = config.init_obj(["losses", "loss"])
# metrics
metrics_epoch = [
getattr(module_metric, met)
for met in config["metrics"]["per_epoch"]
]
keys_epoch = [m.__name__ for m in metrics_epoch]
test_metrics = MetricTracker([], keys_epoch)
if "pick_threshold" in config["metrics"]:
threshold = checkpoint["threshold"]
setattr(module_metric, "THRESHOLD", threshold)
logger.info(f"threshold: {threshold}")
with torch.no_grad():
print("testing...")
test_loader = test_data_loaders["data"]
if len(metrics_epoch) > 0:
outputs = torch.FloatTensor().to(device)
targets = torch.FloatTensor().to(device)
for batch_idx, (data, target) in enumerate(test_loader):
data, target = data.to(device), target.to(device)
output = model(data)
if len(metrics_epoch) > 0:
outputs = torch.cat((outputs, output))
targets = torch.cat((targets, target))
#
# save sample images, or do something with output here
#
for met in metrics_epoch:
test_metrics.epoch_update(met.__name__, met(targets, outputs))
test_log = test_metrics.result()
test_log = test_log["mean"].rename(k)
results = pd.concat((results, test_log), axis=1)
logger.info(test_log)
# cross validation
if k_fold > 1:
Cross_Valid.next_fold()
msg = msg_box("result")
end = time.time()
total_time = consuming_time(start, end)
msg += f"\nConsuming time: {total_time}."
result = pd.DataFrame()
result["mean"] = results.mean(axis=1)
result["std"] = results.std(axis=1)
msg += f"\n{result}"
logger.info(msg)
# bootstrap
if config.test_args.bootstrapping:
assert k_fold == 1, "k-fold ensemble and bootstrap are mutually exclusive."
N = config.test_args.bootstrap_times
bootstrapping(targets, outputs, metrics_epoch, test_metrics, repeat=N)