class Collector():
def __init__(self, cookie, capacity):
# group of illust_id
self.group = set()
self.cookie = cookie
self.downloader = Downloader(capacity)
# add illust_id from page/json_selector
def add(self, group):
self.group |= group
# optional function
# collect images' tags
# and dump into tags.json
def collect_tags(self):
self.tags = dict()
# a copy of self.group
group = self.group.copy()
pool = ParallelPool(len(group))
print('---tags collector start---')
while len(group) or not pool.empty():
time.sleep(THREAD_DELAY)
# send tags_collector to parallel pool
while not pool.full() and len(group):
illust_id = group.pop()
ref = 'https://www.pixiv.net/bookmark.php?type=user'
url = 'https://www.pixiv.net/artworks/' + illust_id
headers = {'Referer': ref}
pool.add(
CollectorUnit(url, self.cookie, tags_selector, headers))
# remove complete thread
finished = pool.finished_item()
while True:
try:
tag_group = next(finished)
illust_id = re.search('artworks/(\d+)',
tag_group.url).group(1)
self.tags[illust_id] = tag_group.group
except StopIteration:
break
with open(IMAGES_STORE_PATH + 'tags.json', 'w', encoding='utf-8') as f:
f.write(json.dumps(self.tags, indent=4, ensure_ascii=False))
print('\n---tags collector complete---')
# collect image from self.group
# and send to self.downloader
def collect(self):
if WITH_TAG:
self.collect_tags()
pool = ParallelPool(len(self.group))
print("---collector start---")
while len(self.group) or not pool.empty():
time.sleep(THREAD_DELAY)
# send json_collector to parallel pool
while not pool.full() and len(self.group):
illust_id = self.group.pop()
ref = 'https://www.pixiv.net/artworks/' + illust_id
url = 'https://www.pixiv.net/ajax/illust/' + illust_id + '/pages?lang=zh'
headers = {'Referer': ref, "x-user-id": USER_ID}
pool.add(
CollectorUnit(url, self.cookie, image_group_selector,
headers))
# remove complete thread
finished = pool.finished_item()
while True:
try:
image_group = next(finished)
self.downloader.add(image_group.group)
except StopIteration:
break
print("\n---collector complete---")
print("artworks: " + str(len(self.downloader.group)))
# self.downloader.download()
def download(self):
return self.downloader.download()
class main(Tk):
def __init__(self):
super().__init__()
self.settings = {}
self.pv = Preview(self.populate_previewer)
self.dc = Downloader(self.handle_download)
self.current_info = {"status": None}
try:
self.current_url = self.clipboard_get()
except TclError:
self.current_url = None
try:
with open("settings.json") as fp:
self.settings = json.load(fp)
except FileNotFoundError:
with open("settings.json", "w") as fp:
json.dump(self.settings, fp)
self.title("Youtube Downloader")
self.attributes("-topmost", True)
self.geometry(self.settings.get("geometry", "600x250+400+300"))
self.columnconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
self.minsize(600, 250)
self.previewer_frame = Previewer_frame(self)
self.previewer_frame.grid(padx=5, pady=5, sticky="nwe")
self.tv = Tree(self)
self.tv.grid(column=0, row=1, padx=5, pady=5, sticky="nswe")
self.bottom = Bottom(self)
self.bottom.grid(column=0, row=2, sticky="w")
try:
self.pv.add(self.clipboard_get())
except TclError:
pass
self.after(100, self.check_clipboard)
self.protocol("WM_DELETE_WINDOW", self.end)
self.mainloop()
def check_clipboard(self):
try:
url = self.clipboard_get()
except TclError:
pass
else:
if self.current_url != url:
self.current_url = url
if self.focus_get() is None:
self.pv.add(self.current_url)
self.after(100, self.check_clipboard)
def add(self, download_type):
if self.current_info["status"] == "Ok":
self.current_info["type"] = download_type
self.dc.add(self.current_info)
def handle_download(self, info):
print(info)
def populate_previewer(self, info):
self.previewer_frame.url = info["url"]
self.previewer_frame.thumbnail = info["thumbnail"]
self.previewer_frame.title = info["title"]
self.previewer_frame.uploader = info["uploader"]
self.current_info = info
def end(self):
self.settings["geometry"] = self.geometry()
with open("settings.json", "w") as fp:
json.dump(self.settings, fp)
self.destroy()
