def __init__(
self,
src: AbsolutePath,
dst: AbsolutePath,
*,
buffer_size: int = 0,
notifier: Notifier = DEFAULT_NOTIFIER,
notify_end: bool = True,
):
src = AbsolutePath.ensure(src)
dst = AbsolutePath.ensure(dst)
buffer_size = buffer_size or 32 * 1024 * 1024
assert buffer_size > 0
if not src.isfile():
raise core_exceptions.NotAFileError(src)
if dst.exists():
raise FileExistsError(dst)
dst_dir = dst.get_directory()
if not dst_dir.isdir():
raise NotADirectoryError(dst_dir)
disk_usage = shutil.disk_usage(dst_dir.path)
total = src.get_size()
if total >= disk_usage.free:
raise core_exceptions.DiskSpaceError(total, disk_usage.free)
self.src = src
self.dst = dst
self.total = total
self.buffer_size = buffer_size
self.notifier = notifier
self.cancel = False
self.notify_end = notify_end
def open_database(self, path: AbsolutePath) -> Database:
path = AbsolutePath.ensure(path)
assert path in self.databases
if not self.databases[path]:
self.databases[path] = Database(
path, notifier=self.notifier, lang=self.lang
)
return self.databases[path]
def open_language(self, lang_path: AbsolutePath) -> DefaultLanguage:
lang_path = AbsolutePath.ensure(lang_path)
assert lang_path in self.languages
self.config.language = lang_path.title
if not self.languages[lang_path]:
self.languages[lang_path] = self._load_lang(lang_path)
self.save_config()
return self.lang
def get_video_from_filename(self, filename, required=True):
# type: (PathType, bool) -> Optional[Video]
filename = AbsolutePath.ensure(filename)
if filename in self.__videos:
return self.__videos[filename]
if required:
raise other.toolsaurus.application.exceptions.UnknownVideoFilename(
filename)
return None
def set_folders(self, folders) -> None:
folders = sorted(AbsolutePath.ensure(folder) for folder in folders)
if folders == sorted(self.folders):
return
folders_tree = PathTree(folders)
for video in self.videos.values():
video.discarded = not folders_tree.in_folders(video.filename)
self.folders = set(folders)
self.save()
def __init__(self, path: PathType):
path = AbsolutePath.ensure(path)
folder = path.get_directory()
self.__json_path = path
self.__prev_path = AbsolutePath.file_path(folder, path.title,
"prev.json")
self.__next_path = AbsolutePath.file_path(folder, path.title,
"next.json")
self.__thumb_folder = AbsolutePath.join(folder,
f"{path.title}.thumbnails")
def load_list_file(list_file_path):
# type: (Union[AbsolutePath, str]) -> Iterable[str]
strings = []
list_file_path = AbsolutePath.ensure(list_file_path)
if list_file_path.isfile():
with open(list_file_path.path, "r") as list_file:
for line in list_file:
line = line.strip()
if line and line[0] != "#":
strings.append(line)
return strings
def job_video_to_json(job):
input_file_name, output_file_name, job_count, job_id, jobn = job
nb_read = 0
nb_loaded = 0
end = False
input_file_path = AbsolutePath.ensure(input_file_name)
output_file_path = AbsolutePath.ensure(output_file_name)
assert input_file_path.isfile()
if output_file_path.exists():
output_file_path.delete()
env = os.environ.copy()
process = subprocess.Popen(
[RUN_VIDEO_RAPTOR_BATCH.path, input_file_name, output_file_name],
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
while True:
line = process.stdout.readline().decode().strip()
if not line and process.poll() is not None:
break
if line:
if line.startswith("#"):
if line.startswith("#count "):
nb_read = int(line[7:])
elif line.startswith("#loaded "):
nb_loaded = int(line[8:])
elif line == "#end":
end = True
else:
step = int(line)
jobn.progress(job_id, step, job_count)
program_errors = process.stderr.read().decode().strip()
if not end and program_errors:
raise exceptions.VideoToJsonError(program_errors)
assert nb_read == job_count
jobn.progress(job_id, job_count, job_count)
return nb_loaded
def path_in_directory(self: AbsolutePath, directory, is_case_insensitive=None):
directory = AbsolutePath.ensure(directory)
if not directory.isdir():
return False
directory = directory.standard_path
path = self.standard_path
if is_case_insensitive:
directory = directory.lower()
path = path.lower()
if len(directory) >= len(path):
return False
return path.startswith(
"%s%s" % (directory, "" if directory.endswith(os.sep) else os.sep))
def not_found_from_folder(self, folder: str):
folder = AbsolutePath.ensure(folder)
if not folder.isdir():
return ""
videos = []
for video in self.database.get_videos("readable", "not_found"):
if path_in_directory(
video.filename,
folder,
is_case_insensitive=self.database.sys_is_case_insensitive,
):
videos.append(video)
videos.sort(key=lambda v: v.filename)
return videos
def change_video_path(self, video_id: int,
path: AbsolutePath) -> AbsolutePath:
path = AbsolutePath.ensure(path)
assert path.isfile()
video = self.select_one("video", (), video_id=video_id)
assert video.filename != path
old_filename = video.filename
self.modify("video", [video_id], filename=str(path))
self.save()
self.__notify_filename_modified()
return old_filename
def __load(self, folders: Optional[Iterable[PathType]] = None):
to_save = False
json_dict = self.__backup.load()
if not isinstance(json_dict, dict):
raise exceptions.InvalidDatabaseJSON(self.__backup.path)
# Parsing settings.
self.settings.update(json_dict.get("settings", {}))
# Parsing date.
if "date" in json_dict:
self.date = DateModified(json_dict["date"])
# Parsing folders.
self.folders.update(AbsolutePath(path) for path in json_dict.get("folders", ()))
if folders:
lb = len(self.folders)
self.folders.update(AbsolutePath.ensure(path) for path in folders)
to_save = to_save or lb != len(self.folders)
# Parsing video property types.
for prop_dict in json_dict.get("prop_types", ()):
self.add_prop_type(PropType.from_dict(prop_dict), save=False)
# Parsing predictors
self.predictors = json_dict.get("predictors", {})
# Parsing videos.
folders_tree = PathTree(self.folders)
for video_dict in json_dict.get("videos", ()):
if video_dict["U"]:
video_state = VideoState.from_dict(video_dict)
else:
video_state = Video.from_dict(video_dict, database=self)
video_state.discarded = not folders_tree.in_folders(video_state.filename)
self.videos[video_state.filename] = video_state
self.save(on_new_identifiers=to_save)
self.__notifier.notify(notifications.DatabaseLoaded(self))
def get_video_id(self, filename):
return self.__videos[AbsolutePath.ensure(filename)].video_id
def ensure_thumbnails(self) -> None:
valid_thumb_names = set()
videos_without_thumbs = []
thumb_to_videos = {} # type: Dict[str, List[Video]]
thumb_errors = {}
# Collect videos with and without thumbnails.
existing_thumb_names = self.__check_thumbnails_on_disk()
with Profiler(self.lang.profile_check_video_thumbnails,
notifier=self.notifier):
for video in self.get_videos("readable"):
thumb_name = video.thumb_name
if not video.found:
video.runtime.has_thumbnail = thumb_name in existing_thumb_names
elif not video.unreadable_thumbnail:
if (thumb_name in existing_thumb_names
and existing_thumb_names[thumb_name] > video.date):
thumb_to_videos.setdefault(thumb_name,
[]).append(video)
else:
videos_without_thumbs.append(video)
# If a thumbnail name is associated to many videos,
# consider these videos don't have thumbnails.
with Profiler(self.lang.profile_check_unique_thumbnails,
notifier=self.notifier):
for valid_thumb_name, vds in thumb_to_videos.items():
if len(vds) == 1:
valid_thumb_names.add(valid_thumb_name)
vds[0].runtime.has_thumbnail = True
else:
videos_without_thumbs.extend(vds)
nb_videos_no_thumbs = len(videos_without_thumbs)
del thumb_to_videos
if not videos_without_thumbs:
self.save()
self.__notify_missing_thumbnails()
return
for video in videos_without_thumbs:
base_thumb_name = video.thumb_name
thumb_name_index = 0
thumb_name = base_thumb_name
while thumb_name in valid_thumb_names:
thumb_name = f"{base_thumb_name}_{thumb_name_index}"
thumb_name_index += 1
video.thumb_name = thumb_name
video.runtime.has_thumbnail = True
valid_thumb_names.add(thumb_name)
del valid_thumb_names
self.save()
job_notifier = job_notifications.CollectVideoThumbnails(
nb_videos_no_thumbs, self.notifier)
thumb_jobs = functions.dispatch_tasks(
[(video.filename.path, video.thumb_name)
for video in videos_without_thumbs],
CPU_COUNT,
[self.__db_folder, self.__thumb_folder.best_path, job_notifier],
)
del videos_without_thumbs
with Profiler(
title=self.lang.profile_collect_video_thumbnails.format(
cpu_count=CPU_COUNT),
notifier=self.notifier,
):
results = functions.parallelize(
backend_raptor.backend_video_thumbnails,
thumb_jobs,
cpu_count=CPU_COUNT,
)
for arr in results:
for d in arr:
assert len(d) == 2 and "f" in d and "e" in d
file_name = d["f"]
file_path = AbsolutePath.ensure(file_name)
thumb_errors[file_name] = d["e"]
video = self.videos[file_path]
video.unreadable_thumbnail = True
video.runtime.has_thumbnail = False
if thumb_errors:
self.notifier.notify(
notifications.VideoThumbnailErrors(thumb_errors))
self.save()
self.__notify_missing_thumbnails()
def update(self) -> None:
SpecialProperties.install(self)
current_date = DateModified.now()
all_file_names = self.__get_new_video_paths()
if not all_file_names:
return
job_notifier = job_notifications.CollectVideoInfos(
len(all_file_names), self.notifier)
jobs = functions.dispatch_tasks(all_file_names, CPU_COUNT,
[self.folder, job_notifier])
with Profiler(
self.lang.profile_collect_video_infos.format(
cpu_count=len(jobs)),
notifier=self.notifier,
):
results = functions.parallelize(backend_raptor.backend_video_infos,
jobs,
cpu_count=CPU_COUNT)
videos = {}
unreadable = []
for arr in results:
for d in arr:
file_path = AbsolutePath.ensure(d["f"])
if len(d) == 2:
video_state = VideoState(
filename=file_path,
size=file_path.get_size(),
errors=d["e"],
database=self,
)
unreadable.append(video_state)
else:
video_state = Video.from_dict(d, database=self)
# Get previous properties, if available
if file_path in self.videos and self.videos[
file_path].readable:
video_state.properties.update(
self.videos[file_path].properties)
video_state.similarity_id = self.videos[
file_path].similarity_id
# Set special properties
SpecialProperties.set(video_state)
videos[file_path] = video_state
self.videos.pop(file_path, None)
stat = FileSystem.stat(file_path.path)
video_state.runtime.is_file = True
video_state.runtime.mtime = stat.st_mtime
video_state.runtime.size = stat.st_size
video_state.runtime.driver_id = stat.st_dev
assert len(videos) == len(all_file_names)
if videos:
self.videos.update(videos)
self.date = current_date
self.save()
if unreadable:
self.notifier.notify(
notifications.VideoInfoErrors({
video_state.filename: video_state.errors
for video_state in unreadable
}))