def __init__(self):
self.__log = pathlib.Path("./log.txt").open("w")
self.__config = DowConfig(pathlib.Path(".").joinpath("config.json"))
self.__db = DowDatabase(self.__config.ROOT_DIR, self.__config.DB_NAME)
self.__model = tf.keras.applications.VGG19(weights='imagenet', include_top=False)
self.__files, self.__features = self.__load_files()
self.__knn = NearestNeighbors(metric='cosine', algorithm='brute', n_jobs=-1)
self.__knn.fit(self.__features)
def __init__(self, app):
super().__init__()
self.__app = app
self.__config = DowConfig(pathlib.Path(".").joinpath("config.json"))
if pathlib.Path(self.__config.ROOT_DIR).joinpath(self.__config.DB_NAME).exists():
self.__db = DowDatabase(self.__config.ROOT_DIR, self.__config.DB_NAME)
else:
print("No db")
self.__slide_view = DowGlImage(self)
self.__slide_label = QtWidgets.QLabel("Test")
self.__slide_label.setFixedHeight(30)
self.__slide_label.setStyleSheet("QLabel { color : blue; font-size : 32px; }")
self.__slide_layout = QtWidgets.QVBoxLayout()
self.__slide_space = QtWidgets.QWidget()
self.__slide_layout.setContentsMargins(QtCore.QMargins(0,0,0,0))
self.__slide_layout.addWidget(self.__slide_label)
self.__slide_layout.addWidget(self.__slide_space)
self.__slide_layout.setAlignment(self.__slide_label, QtCore.Qt.AlignHCenter)
self.__slide_view.setLayout(self.__slide_layout)
main_layout = QtWidgets.QVBoxLayout()
main_layout.setContentsMargins(QtCore.QMargins(0,0,0,0))
main_layout.addWidget(self.__slide_view)
self.setLayout(main_layout)
self.__engine = DowSlideShow(self.__config, None)
import pathlib
import sys
from classes.Config import DowConfig
from classes.Database import DowDatabase
config = DowConfig(pathlib.Path(".").joinpath("config.json"))
db = DowDatabase(config.ROOT_DIR, config.DB_NAME)
o = db.SelectAllFilesLike(sys.argv[1])
if o is not None:
for f in o:
print(f[:-1])
print("\n")
else:
print("None")
from classes.Config import DowConfig
from classes.Database import DowDatabase
from classes.Sankaku2 import DowSankaku
from classes.Pixiv import DowPixiv
from classes.Worker import DowWorker
from classes.MimeType import DowMimeType
import pathlib
from multiprocessing import Process
config = DowConfig(pathlib.Path(".").joinpath("config.json"))
db = DowDatabase(config.ROOT_DIR, config.DB_NAME)
download_dir = pathlib.Path(config.ROOT_DIR).joinpath(config.DOWNLOAD_FOLDER)
sankaku = DowSankaku(config.SAN_USER, config.SAN_PASSWORD, download_dir,
config.SAN_USER_TAG + "+~webm+~mp4")
pixiv = DowPixiv(download_dir, config.PIXIV_TOKEN)
procs = []
if not sankaku.IsConnected():
exit(1)
def check_file_type(file):
file = pathlib.Path(file)
img_type = DowMimeType(file).GetType()
img_type = "jpg" if img_type == "jpeg" else img_type
if img_type != None and img_type != file.suffix:
img_type = "." + img_type
new_file = pathlib.Path(str(file).split(".")[0] + img_type)
return new_file
else:
import pathlib
from classes.Config import DowConfig
from classes.Database import DowDatabase
config = DowConfig(pathlib.Path(".").joinpath("config.json"))
db = DowDatabase(config.ROOT_DIR, config.DB_NAME)
files = db.SelectAll()
if files is None or len(files) == 0:
exit(0)
for line in files:
name, path, tags, features = line
if not pathlib.Path(path).joinpath(name).exists():
db.Delete(name)
from classes.Config import DowConfig
from classes.Database import DowDatabase
import pathlib
conf = DowConfig(pathlib.Path(".").joinpath("config.json"))
db = DowDatabase(conf.ROOT_DIR, conf.DB_NAME)
tagdict = dict()
for line in pathlib.Path(conf.ROOT_DIR).joinpath(conf.MAP_NAME).open("r"):
if ':' in line:
folder, tags_list = line.split(":", 1)
tagdict[folder] = set(tags_list.replace("\n", "").split(","))
for row in db.SelectAll():
name, path, tags, features = row
source = pathlib.Path(path).joinpath(name)
if not source.exists():
db.Delete(name)
continue
else:
tags = set(tags.replace("\n", "").split(" "))
found = False
for line in tagdict:
if tagdict[line].intersection(tags):
dest = pathlib.Path(conf.ROOT_DIR).joinpath(line)
dest.mkdir(parents=True, exist_ok=True)
if dest.joinpath(name) != source:
source.replace(dest.joinpath(name))
print(f"Move {source} to {dest}")
db.Update(name, "path", dest)
class Features():
def __init__(self):
self.__log = pathlib.Path("./log.txt").open("w")
self.__config = DowConfig(pathlib.Path(".").joinpath("config.json"))
self.__db = DowDatabase(self.__config.ROOT_DIR, self.__config.DB_NAME)
self.__model = tf.keras.applications.VGG19(weights='imagenet', include_top=False)
self.__files, self.__features = self.__load_files()
self.__knn = NearestNeighbors(metric='cosine', algorithm='brute', n_jobs=-1)
self.__knn.fit(self.__features)
def __del__(self):
self.__log.close()
def __load_image(self, file_name):
img = tf.keras.preprocessing.image.load_img(str(file_name), target_size=(224, 224))
x = tf.keras.preprocessing.image.img_to_array(img)
x = np.expand_dims(x, axis=0)
return tf.keras.applications.vgg19.preprocess_input(x)
def __move_to_error(self, file):
self.__log.write(str(file) + "\n")
self.__log.flush()
folder = pathlib.Path(self.__config.ROOT_DIR).joinpath(self.__config.ERROR_FOLDER)
folder.mkdir(parents=True, exist_ok=True)
file.replace(folder.joinpath(file.name))
print("Error : " + str(file))
def get_features(self, file_name):
return self.__model.predict(self.__load_image(file_name)).ravel()
def __load_files(self):
files = []
features = []
for f in pathlib.Path(self.__config.ROOT_DIR).glob("**/*"):
if f.suffix in DowMimeType("").image_formats_suffix_list:
db_res = self.__db.SelectFile(f.name)
if db_res is None:
self.__move_to_error(f)
elif db_res[3] == "no":
try:
fts = self.get_features(f)
st = ' '.join(str(x) for x in fts)
self.__db.Update(f.name, "features", st)
files.append(f)
features.append(fts)
except:
self.__move_to_error(f)
else:
ts = np.array(list(map(np.float32, db_res[3].split(' '))))
files.append(f)
features.append(ts)
return files, features
def find_dups(self):
dups = pathlib.Path(self.__config.ROOT_DIR).joinpath(self.__config.DUPS).open("w")
for i in range(len(self.__files)):
dist, indices = self.__knn.kneighbors(self.__features[i].reshape(1, -1), n_neighbors=5)
for j in indices[0]:
if self.__files[j] == self.__files[i]:
continue
elif dist[0][list(indices[0]).index(j)] < 0.3:
dups.write("%s | %s\n" % (self.__files[j], self.__files[i]))
dups.close()
def __init__(self, app):
super().__init__()
self.__app = app
self.__config = DowConfig(pathlib.Path(".").joinpath("config.json"))
#Controls
## Menu Bar
self.__file_menu = QtWidgets.QMenu("File")
self.__file_menu.addAction("Open Files", self.__show_open_files_dialog)
self.__file_menu.addAction("Open Directory",
self.__show_open_dir_dialog)
self.__file_menu.addAction("Clear Files", self.__clear_files)
self.__file_menu.addAction("Close", self.__close)
self.__menu_bar = QtWidgets.QMenuBar()
self.__menu_bar.addMenu(self.__file_menu)
## Left side
self.__image = DowGlImage(self) #QtWidgets.QLabel()
# self.__image.setAlignment(QtCore.Qt.AlignVCenter)
self.__controls_left_layout = QtWidgets.QVBoxLayout()
self.__left_box = QtWidgets.QWidget()
self.__left_box_layout = QtWidgets.QVBoxLayout(self.__left_box)
self.__left_box_layout.setContentsMargins(QtCore.QMargins(0, 0, 0, 0))
self.__left_box_layout.addWidget(self.__image)
self.__controls_left_layout.addWidget(self.__left_box)
## Right side
self.__search_box = QtWidgets.QLineEdit()
self.__all_tags = QtWidgets.QListView()
self.__all_tags.setMaximumHeight(150)
self.__selected_tags = QtWidgets.QListView()
self.__files = QtWidgets.QListView()
self.__files.setMinimumHeight(200)
self.__files.setMaximumHeight(200)
self.__back_button = QtWidgets.QPushButton("Back")
self.__next_button = QtWidgets.QPushButton("Next")
self.__save_button = QtWidgets.QPushButton("Save")
self.__buttons_right_layout = QtWidgets.QHBoxLayout()
self.__buttons_right_layout.addWidget(self.__back_button)
self.__buttons_right_layout.addWidget(self.__next_button)
self.__buttons_right_layout.addWidget(self.__save_button)
self.__right_box = QtWidgets.QWidget()
self.__right_box.setMinimumWidth(200)
self.__right_box.setMaximumWidth(300)
self.__controls_right_layout = QtWidgets.QVBoxLayout(self.__right_box)
self.__controls_right_layout.setContentsMargins(
QtCore.QMargins(0, 0, 0, 0))
self.__controls_right_layout.addWidget(self.__search_box)
self.__controls_right_layout.addWidget(self.__all_tags)
self.__controls_right_layout.addWidget(self.__selected_tags)
self.__controls_right_layout.addWidget(self.__files)
self.__controls_right_layout.addLayout(self.__buttons_right_layout)
#Layouts
self.__main_layout = QtWidgets.QHBoxLayout(self)
self.__main_layout.addLayout(self.__controls_left_layout)
self.__main_layout.addWidget(self.__right_box)
self.__main_layout.setMenuBar(self.__menu_bar)
#Logic
self.__logic = DowTagEditor(self.__all_tags, self.__selected_tags,
self.__files, self.__next_button,
self.__back_button, self.__save_button,
self.__search_box)
if pathlib.Path(self.__config.ROOT_DIR).joinpath(
self.__config.DB_NAME).exists():
self.__db = DowDatabase(self.__config.ROOT_DIR,
self.__config.DB_NAME)
self.__logic.SetDatabase(self.__db)
self.__logic.ImageChangeEvent(self.__load_image)
from classes.Config import DowConfig
from classes.Database import DowDatabase
from classes.MimeType import DowMimeType
import pathlib
config = DowConfig(pathlib.Path(".").joinpath("config.json"))
db = DowDatabase(config.ROOT_DIR, config.DB_NAME)
add_folder = pathlib.Path(config.ROOT_DIR, config.ADD_FOLDER)
for file in add_folder.glob("**/*"):
if file.suffix in DowMimeType("").all_formats_suffix_list:
if not db.IsFileIn(file.name):
db.Insert(file.name, file.parents[0], "tagme")