def backend_video_thumbnails(job):
videos_data, job_id, db_folder, thumb_folder, job_notifier = job
list_file_path = AbsolutePath.file_path(db_folder, job_id, "thumbnails.list")
json_file_path = AbsolutePath.file_path(db_folder, job_id, "thumbnails.json")
with open(list_file_path.path, "wb") as file:
for file_path, thumb_name in videos_data:
file.write(f"{file_path}\t{thumb_folder}\t{thumb_name}\t\n".encode())
nb_loaded = job_video_thumbnails_to_json(
(
list_file_path.path,
json_file_path.path,
len(videos_data),
job_id,
job_notifier,
)
)
assert json_file_path.isfile()
arr = parse_json(json_file_path)
assert arr[-1] is None
arr.pop()
assert nb_loaded + len(arr) == len(videos_data)
list_file_path.delete()
json_file_path.delete()
return arr
def list_files(self, output: str):
self.database.list_files(output)
output_path = AbsolutePath(output)
if not output_path.isfile():
raise OSError("Unable to output videos file names in %s" %
output_path)
return str(output_path)
def _move_video_file(self, video_id: int, directory: str):
self.provider.register_notifications()
try:
filename = self.database.get_video_filename(video_id)
directory = AbsolutePath.ensure_directory(directory)
if not PathTree(self.database.video_folders).in_folders(directory):
raise exceptions.ForbiddenVideoFolder(
directory, self.database.video_folders)
dst = AbsolutePath.file_path(directory, filename.file_title,
filename.extension)
self.copy_work = FileCopier(filename,
dst,
notifier=self.notifier,
notify_end=False)
with Profiler(self.database.lang.profile_move,
notifier=self.notifier):
done = self.copy_work.move()
self.copy_work = None
if done:
old_path = self.database.change_video_path(video_id, dst)
if old_path:
old_path.delete()
self.provider.refresh()
self.notifier.notify(Done())
else:
self.notifier.notify(Cancelled())
except Exception as exc:
self.database.notifier.notify(
End(
self.database.lang.error_moving_file.format(
name=type(exc).__name__, message=exc)))
finally:
self.db_loading_thread = None
def __init__(self, path: PathType):
path = AbsolutePath.ensure(path)
folder = path.get_directory()
self.__json_path = path
self.__prev_path = AbsolutePath.file_path(folder, path.title,
"prev.json")
self.__next_path = AbsolutePath.file_path(folder, path.title,
"next.json")
self.__thumb_folder = AbsolutePath.join(folder,
f"{path.title}.thumbnails")
def main():
if len(sys.argv) != 2:
return
path = AbsolutePath(sys.argv[1]).assert_file()
assert path.extension.lower() == "webp"
image = Image.open(path.path)
image.info.pop("background", None)
assert image.is_animated
output_path = AbsolutePath.join(path.get_directory(), f"{path.title}.gif")
image.save(output_path.path, "gif", save_all=True, optimize=False)
print(output_path)
def main():
if len(sys.argv) != 2:
return
folder = AbsolutePath.ensure_directory(sys.argv[1])
for basename in folder.listdir():
try:
path = AbsolutePath.join(folder, basename)
if not path.isfile():
continue
image = Image.open(path.path)
if image.is_animated:
print(path)
except UnidentifiedImageError:
pass
def change_video_path(self, video_id: int,
path: AbsolutePath) -> AbsolutePath:
path = AbsolutePath.ensure(path)
assert path.isfile()
video = self.select_one("video", (), video_id=video_id)
assert video.filename != path
old_filename = video.filename
self.modify("video", [video_id], filename=str(path))
self.save()
self.__notify_filename_modified()
return old_filename
def __transfer_db_path(path: AbsolutePath, old_folder: AbsolutePath,
new_folder: AbsolutePath):
old_name = old_folder.title
old_basename = path.get_basename()
assert old_basename.startswith(old_name)
new_basename = f"{new_folder.title}{old_basename[(len(old_name)):]}"
new_path = AbsolutePath.join(new_folder, new_basename)
re_path = AbsolutePath.join(new_folder, old_basename)
if re_path.exists():
print("Database: renaming", re_path, file=sys.stderr)
if new_path.exists():
raise exceptions.PathAlreadyExists(new_path)
FileSystem.rename(re_path.path, new_path.path)
assert not re_path.exists()
assert new_path.exists()
return new_path
def rename(self, new_name) -> None:
old_db_folder = self.__db_folder
old_thumb_folder = self.__thumb_folder
old_json_path = self.__json_path
old_miniature_path = self.__miniatures_path
old_log_path = self.__log_path
new_name = new_name.strip()
if new_name == self.__db_folder.title:
return
if functions.has_discarded_characters(new_name):
raise exceptions.InvalidDatabaseName(new_name)
new_db_folder = AbsolutePath.join(old_db_folder.get_directory(),
new_name)
if new_db_folder.exists():
raise exceptions.DatabaseAlreadyExists(new_db_folder)
FileSystem.rename(old_db_folder.path, new_db_folder.path)
assert not old_db_folder.exists()
assert new_db_folder.isdir()
self.__db_folder = new_db_folder
self.__thumb_folder = self.__transfer_db_path(old_thumb_folder,
old_db_folder,
new_db_folder)
self.__json_path = self.__transfer_db_path(old_json_path,
old_db_folder,
new_db_folder)
self.__miniatures_path = self.__transfer_db_path(
old_miniature_path, old_db_folder, new_db_folder)
self.__log_path = self.__transfer_db_path(old_log_path, old_db_folder,
new_db_folder)
self.notifier.set_log_path(self.__log_path.path)
self.set_path(self.__json_path)
def main():
app = Application()
db = app.open_database(AbsolutePath.join(app.dbs_dir, "adult videos"))
all_miniatures = db.ensure_miniatures(
returns=True) # type: List[Miniature]
video_id_to_miniature = {m.video_id: m for m in all_miniatures}
videos = db.get_videos("readable", "with_thumbnails") # type: List[Video]
prop_name = "<?newsensations>"
classifier = {}
for video in videos:
prop_val = video.properties.get(prop_name, -1)
classifier.setdefault(prop_val, []).append(video)
assert 0 in classifier
assert 1 in classifier
negative_videos = classifier[0]
positive_videos = classifier[1]
video_id_to_class = {video.video_id: False for video in negative_videos}
video_id_to_class.update(
{video.video_id: True
for video in positive_videos})
miniatures = []
classes = []
for video_id, y in video_id_to_class.items():
miniatures.append(video_id_to_miniature[video_id])
classes.append(video_id_to_class[video_id])
theta = train(miniatures, classes)
print("Predictions:")
for i, (x, y) in enumerate(zip(miniatures, classes)):
p = predict(x, theta)
b = p >= 0.5 if y else p < 0.5
print("Image", i + 1, "expects", y, "predicts", p, "good?", b)
def open_database(self, path: AbsolutePath) -> Database:
path = AbsolutePath.ensure(path)
assert path in self.databases
if not self.databases[path]:
self.databases[path] = Database(
path, notifier=self.notifier, lang=self.lang
)
return self.databases[path]
def open_language(self, lang_path: AbsolutePath) -> DefaultLanguage:
lang_path = AbsolutePath.ensure(lang_path)
assert lang_path in self.languages
self.config.language = lang_path.title
if not self.languages[lang_path]:
self.languages[lang_path] = self._load_lang(lang_path)
self.save_config()
return self.lang
def set_folders(self, folders) -> None:
folders = sorted(AbsolutePath.ensure(folder) for folder in folders)
if folders == sorted(self.folders):
return
folders_tree = PathTree(folders)
for video in self.videos.values():
video.discarded = not folders_tree.in_folders(video.filename)
self.folders = set(folders)
self.save()
def main():
if len(sys.argv) != 2:
print("No path specified.")
return
folder = AbsolutePath(sys.argv[1]).assert_dir()
lines = Counter()
for file_name in folder.listdir():
path = AbsolutePath.join(folder, file_name)
if path.isfile():
with open(path.path) as file:
for line in file:
lines.update([line.strip()])
inv = {}
for line, count in lines.items():
inv.setdefault(count, []).append(line)
for count in sorted(inv, reverse=True):
for line in sorted(inv[count]):
print(f"{count}\t{line}")
def generate_temp_file_path(extension):
temp_file_id = 0
while True:
temp_file_path = AbsolutePath.file_path(
TEMP_DIR, "%s%s" % (TEMP_PREFIX, temp_file_id), extension)
if temp_file_path.exists():
temp_file_id += 1
else:
break
return temp_file_path
def generate_non_existing_path(directory, name, extension):
file_id = 0
while True:
file_path = AbsolutePath.file_path(directory, f"{name}{file_id}",
extension)
if file_path.exists():
file_id += 1
else:
break
return file_path
def load_list_file(list_file_path):
# type: (Union[AbsolutePath, str]) -> Iterable[str]
strings = []
list_file_path = AbsolutePath.ensure(list_file_path)
if list_file_path.isfile():
with open(list_file_path.path, "r") as list_file:
for line in list_file:
line = line.strip()
if line and line[0] != "#":
strings.append(line)
return strings
def job_video_to_json(job):
input_file_name, output_file_name, job_count, job_id, jobn = job
nb_read = 0
nb_loaded = 0
end = False
input_file_path = AbsolutePath.ensure(input_file_name)
output_file_path = AbsolutePath.ensure(output_file_name)
assert input_file_path.isfile()
if output_file_path.exists():
output_file_path.delete()
env = os.environ.copy()
process = subprocess.Popen(
[RUN_VIDEO_RAPTOR_BATCH.path, input_file_name, output_file_name],
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
while True:
line = process.stdout.readline().decode().strip()
if not line and process.poll() is not None:
break
if line:
if line.startswith("#"):
if line.startswith("#count "):
nb_read = int(line[7:])
elif line.startswith("#loaded "):
nb_loaded = int(line[8:])
elif line == "#end":
end = True
else:
step = int(line)
jobn.progress(job_id, step, job_count)
program_errors = process.stderr.read().decode().strip()
if not end and program_errors:
raise exceptions.VideoToJsonError(program_errors)
assert nb_read == job_count
jobn.progress(job_id, job_count, job_count)
return nb_loaded
def __load(self, folders: Optional[Iterable[PathType]] = None):
to_save = False
json_dict = self.__backup.load()
if not isinstance(json_dict, dict):
raise exceptions.InvalidDatabaseJSON(self.__backup.path)
# Parsing settings.
self.settings.update(json_dict.get("settings", {}))
# Parsing date.
if "date" in json_dict:
self.date = DateModified(json_dict["date"])
# Parsing folders.
self.folders.update(AbsolutePath(path) for path in json_dict.get("folders", ()))
if folders:
lb = len(self.folders)
self.folders.update(AbsolutePath.ensure(path) for path in folders)
to_save = to_save or lb != len(self.folders)
# Parsing video property types.
for prop_dict in json_dict.get("prop_types", ()):
self.add_prop_type(PropType.from_dict(prop_dict), save=False)
# Parsing predictors
self.predictors = json_dict.get("predictors", {})
# Parsing videos.
folders_tree = PathTree(self.folders)
for video_dict in json_dict.get("videos", ()):
if video_dict["U"]:
video_state = VideoState.from_dict(video_dict)
else:
video_state = Video.from_dict(video_dict, database=self)
video_state.discarded = not folders_tree.in_folders(video_state.filename)
self.videos[video_state.filename] = video_state
self.save(on_new_identifiers=to_save)
self.__notifier.notify(notifications.DatabaseLoaded(self))
def __init__(
self,
src: AbsolutePath,
dst: AbsolutePath,
*,
buffer_size: int = 0,
notifier: Notifier = DEFAULT_NOTIFIER,
notify_end: bool = True,
):
src = AbsolutePath.ensure(src)
dst = AbsolutePath.ensure(dst)
buffer_size = buffer_size or 32 * 1024 * 1024
assert buffer_size > 0
if not src.isfile():
raise core_exceptions.NotAFileError(src)
if dst.exists():
raise FileExistsError(dst)
dst_dir = dst.get_directory()
if not dst_dir.isdir():
raise NotADirectoryError(dst_dir)
disk_usage = shutil.disk_usage(dst_dir.path)
total = src.get_size()
if total >= disk_usage.free:
raise core_exceptions.DiskSpaceError(total, disk_usage.free)
self.src = src
self.dst = dst
self.total = total
self.buffer_size = buffer_size
self.notifier = notifier
self.cancel = False
self.notify_end = notify_end
def path_in_directory(self: AbsolutePath, directory, is_case_insensitive=None):
directory = AbsolutePath.ensure(directory)
if not directory.isdir():
return False
directory = directory.standard_path
path = self.standard_path
if is_case_insensitive:
directory = directory.lower()
path = path.lower()
if len(directory) >= len(path):
return False
return path.startswith(
"%s%s" % (directory, "" if directory.endswith(os.sep) else os.sep))
def new_database(self, name, folders: Iterable[AbsolutePath]):
if functions.has_discarded_characters(name):
raise exceptions.InvalidDatabaseName(name)
path = AbsolutePath.join(self.dbs_dir, name)
if path.title != name:
raise exceptions.InvalidDatabaseName(name)
if path in self.databases:
raise exceptions.DatabaseAlreadyExists(path)
if path.exists():
raise exceptions.DatabasePathUnavailable(path)
self.databases[path] = Database(
path.mkdir(), folders=folders, notifier=self.notifier, lang=self.lang
)
return self.databases[path]
def not_found_from_folder(self, folder: str):
folder = AbsolutePath.ensure(folder)
if not folder.isdir():
return ""
videos = []
for video in self.database.get_videos("readable", "not_found"):
if path_in_directory(
video.filename,
folder,
is_case_insensitive=self.database.sys_is_case_insensitive,
):
videos.append(video)
videos.sort(key=lambda v: v.filename)
return videos
def _collect_videos_info(folder: str, files: Dict[AbsolutePath,
VideoRuntimeInfo]):
entry: os.DirEntry
for entry in FileSystem.scandir(folder):
if entry.is_dir():
_collect_videos_info(entry.path, files)
elif (functions.get_file_extension(entry.name)
in constants.VIDEO_SUPPORTED_EXTENSIONS):
stat = entry.stat()
files[AbsolutePath(entry.path)] = VideoRuntimeInfo(
size=stat.st_size,
mtime=stat.st_mtime,
driver_id=stat.st_dev,
is_file=True,
)
def backend_video_infos(job):
file_names, job_id, database_folder, job_notifier = job
list_file_path = AbsolutePath.file_path(database_folder, str(job_id), "list")
json_file_path = AbsolutePath.file_path(database_folder, str(job_id), "json")
with open(list_file_path.path, "wb") as file:
for file_name in file_names:
file.write(f"{file_name}\n".encode())
count = job_video_to_json(
(
list_file_path.path,
json_file_path.path,
len(file_names),
job_id,
job_notifier,
)
)
assert json_file_path.isfile()
arr = parse_json(json_file_path)
# assert len(arr) == count, (len(arr), count)
list_file_path.delete()
json_file_path.delete()
return arr
def __init__(self, *, geometry=None):
super().__init__()
# setup interface
self.interface = Interface()
# setup page
html_path = AbsolutePath.join(package_dir(), "interface", "web",
"index.html").assert_file()
url = QUrl.fromLocalFile(html_path.path)
print("Loading", url)
with open(html_path.path) as file:
html = file.read()
html = html.replace(
"<!--headerScript-->",
'<script src="qrc:///qtwebchannel/qwebchannel.js"></script>',
)
html = html.replace('<script src="onload.js"></script>',
'<script src="qt.js"></script>')
self.web_page = CustomPage()
self.web_page.setHtml(html, url)
self.setPage(self.web_page)
# setup channel
self.channel = QWebChannel()
self.channel.registerObject("backend", self.interface)
self.page().setWebChannel(self.channel)
# setup geometry
if geometry:
assert isinstance(geometry, (tuple, list))
assert len(geometry) == 4
self.setGeometry(*geometry)
# setup player
self.player = None
if has_vlc:
self.player = Player(on_next=self._on_next_random_video)
self.interface.player_triggered.connect(self.player.show)
if geometry:
_, _, width, height = geometry
self.player.resize(int(width * 3 / 4), int(height * 3 / 4))
def __init__(self, path, folders=None, notifier=None, lang=None):
# type: (PathType, Iterable[PathType], Notifier, DefaultLanguage) -> None
# Paths
self.__db_folder = AbsolutePath.ensure_directory(path)
self.__thumb_folder = new_sub_folder(self.__db_folder,
"thumbnails").mkdir()
self.__json_path = new_sub_file(self.__db_folder, "json")
self.__miniatures_path = new_sub_file(self.__db_folder,
"miniatures.json")
self.__log_path = new_sub_file(self.__db_folder, "log")
# RAM data
self.__message = None
self.lang = lang or DefaultLanguage
# Set log file
notifier = notifier or DEFAULT_NOTIFIER
notifier.set_log_path(self.__log_path.path)
# Load database
super().__init__(self.__json_path, folders, notifier)
# Set special properties
with Profiler("install special properties", notifier=self.notifier):
SpecialProperties.install(self)
def flush_json_data(
data: object,
previous_file_path: AbsolutePath,
target_file_path: AbsolutePath,
next_file_path: AbsolutePath,
):
# Store JSON data to next file
with open(next_file_path.path, "w") as output_file:
json.dump(data, output_file)
# Remove previous file
previous_file_path.delete()
# Move target file to previous file
if target_file_path.isfile():
FileSystem.rename(target_file_path.path, previous_file_path.path)
assert not target_file_path.isfile()
assert previous_file_path.isfile()
# Move next file to target file
FileSystem.rename(next_file_path.path, target_file_path.path)
# Next file deleted
# Previous file may exists
# Target file contains data
assert not next_file_path.exists()
assert target_file_path.isfile()
