def do_download_worker(book_list, options, cpus, merge=False, notification=lambda x,y:x): ''' Master job, to launch child jobs to extract ISBN for a set of books This is run as a worker job in the background to keep the UI more responsive and get around the memory leak issues as it will launch a child job for each book as a worker process ''' server = Server(pool_size=cpus) logger.info(options['version']) total = 0 alreadybad = [] # Queue all the jobs logger.info("Adding jobs for URLs:") for book in book_list: logger.info("%s"%book['url']) if book['good']: total += 1 args = ['calibre_plugins.fanficfare_plugin.jobs', 'do_download_for_worker', (book,options,merge)] job = ParallelJob('arbitrary_n', "url:(%s) id:(%s)"%(book['url'],book['calibre_id']), done=None, args=args) job._book = book server.add_job(job) else: # was already bad before the subprocess ever started. alreadybad.append(book) # This server is an arbitrary_n job, so there is a notifier available. # Set the % complete to a small number to avoid the 'unavailable' indicator notification(0.01, _('Downloading FanFiction Stories')) # dequeue the job results as they arrive, saving the results count = 0 while True: job = server.changed_jobs_queue.get() # A job can 'change' when it is not finished, for example if it # produces a notification. Ignore these. job.update() if not job.is_finished: continue # A job really finished. Get the information. book_list.remove(job._book) book_list.append(job.result) book_id = job._book['calibre_id'] count = count + 1 notification(float(count)/total, '%d of %d stories finished downloading'%(count,total)) # Add this job's output to the current log logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title'])) logger.info(job.details) if count >= total: ## ordering first by good vs bad, then by listorder. good_list = filter(lambda x : x['good'], book_list) bad_list = filter(lambda x : not x['good'], book_list) good_list = sorted(good_list,key=lambda x : x['listorder']) bad_list = sorted(bad_list,key=lambda x : x['listorder']) logger.info("\n"+_("Download Results:")+"\n%s\n"%("\n".join([ "%(url)s %(comment)s" % book for book in good_list+bad_list]))) logger.info("\n"+_("Successful:")+"\n%s\n"%("\n".join([book['url'] for book in good_list]))) logger.info("\n"+_("Unsuccessful:")+"\n%s\n"%("\n".join([book['url'] for book in bad_list]))) break server.close() # return the book list as the job result return book_list
def do_download_worker(book_list, options, cpus, merge=False, notification=lambda x, y: x): ''' Coordinator job, to launch child jobs to extract ISBN for a set of books This is run as a worker job in the background to keep the UI more responsive and get around the memory leak issues as it will launch a child job for each book as a worker process ''' server = Server(pool_size=cpus) logger.info(options['version']) total = 0 alreadybad = [] # Queue all the jobs logger.info("Adding jobs for URLs:") for book in book_list: logger.info("%s" % book['url']) if book['good']: total += 1 args = [ 'calibre_plugins.fanficfare_plugin.jobs', 'do_download_for_worker', (book, options, merge) ] job = ParallelJob('arbitrary_n', "url:(%s) id:(%s)" % (book['url'], book['calibre_id']), done=None, args=args) job._book = book server.add_job(job) else: # was already bad before the subprocess ever started. alreadybad.append(book) # This server is an arbitrary_n job, so there is a notifier available. # Set the % complete to a small number to avoid the 'unavailable' indicator notification(0.01, _('Downloading FanFiction Stories')) # dequeue the job results as they arrive, saving the results count = 0 while True: job = server.changed_jobs_queue.get() # A job can 'change' when it is not finished, for example if it # produces a notification. Ignore these. job.update() if not job.is_finished: continue # A job really finished. Get the information. book_list.remove(job._book) book_list.append(job.result) book_id = job._book['calibre_id'] count = count + 1 notification( float(count) / total, _('%d of %d stories finished downloading') % (count, total)) # Add this job's output to the current log logger.info('Logfile for book ID %s (%s)' % (book_id, job._book['title'])) logger.info(job.details) if count >= total: book_list = sorted(book_list, key=lambda x: x['listorder']) logger.info("\n" + _("Download Results:") + "\n%s\n" % ("\n".join([ "%(status)s %(url)s %(comment)s" % book for book in book_list ]))) good_lists = defaultdict(list) bad_lists = defaultdict(list) for book in book_list: if book['good']: good_lists[book['status']].append(book) else: bad_lists[book['status']].append(book) order = [ _('Add'), _('Update'), _('Meta'), _('Different URL'), _('Rejected'), _('Skipped'), _('Bad'), _('Error'), ] j = 0 for d in [good_lists, bad_lists]: for status in order: if d[status]: l = d[status] logger.info("\n" + status + "\n%s\n" % ("\n".join([book['url'] for book in l]))) for book in l: book['reportorder'] = j j += 1 del d[status] # just in case a status is added but doesn't appear in order. for status in d.keys(): logger.info("\n" + status + "\n%s\n" % ("\n".join([book['url'] for book in d[status]]))) break server.close() # return the book list as the job result return book_list
def do_download_worker(book_list, options, cpus, notification=lambda x, y: x): ''' Master job, to launch child jobs to extract ISBN for a set of books This is run as a worker job in the background to keep the UI more responsive and get around the memory leak issues as it will launch a child job for each book as a worker process ''' server = Server(pool_size=cpus) logger.info(options['version']) total = 0 alreadybad = [] # Queue all the jobs logger.info("Adding jobs for URLs:") for book in book_list: logger.info("%s" % book['url']) if book['good']: total += 1 args = [ 'calibre_plugins.fanficfare_plugin.jobs', 'do_download_for_worker', (book, options) ] job = ParallelJob('arbitrary_n', "url:(%s) id:(%s)" % (book['url'], book['calibre_id']), done=None, args=args) job._book = book server.add_job(job) else: # was already bad before the subprocess ever started. alreadybad.append(book) # This server is an arbitrary_n job, so there is a notifier available. # Set the % complete to a small number to avoid the 'unavailable' indicator notification(0.01, _('Downloading FanFiction Stories')) # dequeue the job results as they arrive, saving the results count = 0 while True: job = server.changed_jobs_queue.get() # A job can 'change' when it is not finished, for example if it # produces a notification. Ignore these. job.update() if not job.is_finished: continue # A job really finished. Get the information. book_list.remove(job._book) book_list.append(job.result) book_id = job._book['calibre_id'] count = count + 1 notification( float(count) / total, '%d of %d stories finished downloading' % (count, total)) # Add this job's output to the current log logger.info('Logfile for book ID %s (%s)' % (book_id, job._book['title'])) logger.info(job.details) if count >= total: logger.info("\n" + _("Successful:") + "\n%s\n" % ("\n".join([ book['url'] for book in filter(lambda x: x['good'], book_list) ]))) logger.info("\n" + _("Unsuccessful:") + "\n%s\n" % ("\n".join([ book['url'] for book in filter(lambda x: not x['good'], book_list) ]))) break server.close() # return the book list as the job result return book_list