class Collector(): def __init__(self, cookie, capacity): # group of illust_id self.group = set() self.cookie = cookie self.downloader = Downloader(capacity) # add illust_id from page/json_selector def add(self, group): self.group |= group # optional function # collect images' tags # and dump into tags.json def collect_tags(self): self.tags = dict() # a copy of self.group group = self.group.copy() pool = ParallelPool(len(group)) print('---tags collector start---') while len(group) or not pool.empty(): time.sleep(THREAD_DELAY) # send tags_collector to parallel pool while not pool.full() and len(group): illust_id = group.pop() ref = 'https://www.pixiv.net/bookmark.php?type=user' url = 'https://www.pixiv.net/artworks/' + illust_id headers = {'Referer': ref} pool.add( CollectorUnit(url, self.cookie, tags_selector, headers)) # remove complete thread finished = pool.finished_item() while True: try: tag_group = next(finished) illust_id = re.search('artworks/(\d+)', tag_group.url).group(1) self.tags[illust_id] = tag_group.group except StopIteration: break with open(IMAGES_STORE_PATH + 'tags.json', 'w', encoding='utf-8') as f: f.write(json.dumps(self.tags, indent=4, ensure_ascii=False)) print('\n---tags collector complete---') # collect image from self.group # and send to self.downloader def collect(self): if WITH_TAG: self.collect_tags() pool = ParallelPool(len(self.group)) print("---collector start---") while len(self.group) or not pool.empty(): time.sleep(THREAD_DELAY) # send json_collector to parallel pool while not pool.full() and len(self.group): illust_id = self.group.pop() ref = 'https://www.pixiv.net/artworks/' + illust_id url = 'https://www.pixiv.net/ajax/illust/' + illust_id + '/pages?lang=zh' headers = {'Referer': ref, "x-user-id": USER_ID} pool.add( CollectorUnit(url, self.cookie, image_group_selector, headers)) # remove complete thread finished = pool.finished_item() while True: try: image_group = next(finished) self.downloader.add(image_group.group) except StopIteration: break print("\n---collector complete---") print("artworks: " + str(len(self.downloader.group))) # self.downloader.download() def download(self): return self.downloader.download()
class main(Tk): def __init__(self): super().__init__() self.settings = {} self.pv = Preview(self.populate_previewer) self.dc = Downloader(self.handle_download) self.current_info = {"status": None} try: self.current_url = self.clipboard_get() except TclError: self.current_url = None try: with open("settings.json") as fp: self.settings = json.load(fp) except FileNotFoundError: with open("settings.json", "w") as fp: json.dump(self.settings, fp) self.title("Youtube Downloader") self.attributes("-topmost", True) self.geometry(self.settings.get("geometry", "600x250+400+300")) self.columnconfigure(0, weight=1) self.rowconfigure(1, weight=1) self.minsize(600, 250) self.previewer_frame = Previewer_frame(self) self.previewer_frame.grid(padx=5, pady=5, sticky="nwe") self.tv = Tree(self) self.tv.grid(column=0, row=1, padx=5, pady=5, sticky="nswe") self.bottom = Bottom(self) self.bottom.grid(column=0, row=2, sticky="w") try: self.pv.add(self.clipboard_get()) except TclError: pass self.after(100, self.check_clipboard) self.protocol("WM_DELETE_WINDOW", self.end) self.mainloop() def check_clipboard(self): try: url = self.clipboard_get() except TclError: pass else: if self.current_url != url: self.current_url = url if self.focus_get() is None: self.pv.add(self.current_url) self.after(100, self.check_clipboard) def add(self, download_type): if self.current_info["status"] == "Ok": self.current_info["type"] = download_type self.dc.add(self.current_info) def handle_download(self, info): print(info) def populate_previewer(self, info): self.previewer_frame.url = info["url"] self.previewer_frame.thumbnail = info["thumbnail"] self.previewer_frame.title = info["title"] self.previewer_frame.uploader = info["uploader"] self.current_info = info def end(self): self.settings["geometry"] = self.geometry() with open("settings.json", "w") as fp: json.dump(self.settings, fp) self.destroy()