def worker_run(self, queue, callback): while True: try: task = queue.get() if not task: continue illu_file = None # download if task.download_mode == DOWNLOAD_MODE_ID: illu_file = self.downloader.download_all_by_id( task.id, task.path, p_limit=task.p_limit if task.has_key('p_limit') else 0) elif task.download_mode == DOWNLOAD_MODE_URL: illu_file = self.downloader.download_all_by_url( task.url, task.path) elif task.download_mode == DOWNLOAD_MODE_DETAIL: illu_file = self.downloader.download_by_detail( task.illu, task.path, p_limit=task.p_limit) # callback if task.task_type == TASK_TYPE_ID and callback: msg = CommonUtils.build_callback_msg(illu_file, id=str(task.id)) callback(msg) elif task.task_type == TASK_TYPE_URL and callback: msg = CommonUtils.build_callback_msg(illu_file, url=str(task.url)) callback(msg) elif callback: if illu_file: if illu_file != PAGE_LIMIT_CONTINUE: callback("%s:%s\nFile:%s\n\n" % (task.get_from + " get", task.illu.get('id'), illu_file)) else: callback("%s:%s\nFile:%s\n\n" % (task.get_from + " get", task.illu.get('id'), 'Download Fail')) except Exception as e: print("error", e) finally: queue.task_done() print(threading.currentThread().getName() + ":Current Task Number:" + str(queue.qsize())) if callback and task and task.has_key( 'all_count') and task.all_count > 0: if illu_file: current_count = task.current_count.getAndInc() if current_count + 1 == task.all_count: afterEndCallback(task, callback)
def run(self): if not os.path.exists(self.path): try: os.makedirs(self.path) except Exception as e: error_log("make dir Fail:" + self.path) error_log(e) return try: path = ImageDownload.download_topics(self.url, self.path, create_path=self.create_path, downloader=self.downloader) if self.success: self.success(CommonUtils.build_callback_msg(path, url=self.url)) if self.callback_params and self.callback_params.has_key( 'current_count') and self.callback_params.has_key( 'all_count'): current_count = self.callback_params[ 'current_count'].getAndInc() if self.callback_params['all_count'] == (current_count + 1): self.success("Download from Pixivision:\n" + self.callback_params['url'] + "\nAll tasks are complete!\n\n") except Exception as e: print("Download topics fail") print(e) if self.fail: self.fail()