def single_covers(title, authors, identifiers, caches, tdir):
patch_plugins()
load_caches(caches)
log = GUILog()
results = Queue()
worker = Thread(target=run_download, args=(log, results, Event()),
kwargs=dict(title=title, authors=authors, identifiers=identifiers))
worker.daemon = True
worker.start()
c = Counter()
while worker.is_alive():
try:
plugin, width, height, fmt, data = results.get(True, 1)
except Empty:
continue
else:
name = plugin.name
if plugin.can_get_multiple_covers:
name += '{%d}'%c[plugin.name]
c[plugin.name] += 1
name = '%s,,%s,,%s,,%s.cover'%(name, width, height, fmt)
with open(os.path.join(tdir, name), 'wb') as f:
f.write(data)
os.mkdir(os.path.join(tdir, name+'.done'))
return log.dump()
def __init__(self, thread_type, thread_count=1):
self.thread_type = thread_type
self.thread_count = thread_count
self.tasks = Queue()
self.results = Queue()
self.threads = []
class Watcher(WatcherBase):
def __init__(self, root_dirs, worker, log):
WatcherBase.__init__(self, worker, log)
self.stream = Stream(self.notify,
*(x.encode('utf-8') for x in root_dirs),
file_events=True)
self.wait_queue = Queue()
def wakeup(self):
self.wait_queue.put(True)
def loop(self):
observer = Observer()
observer.schedule(self.stream)
observer.daemon = True
observer.start()
try:
while True:
try:
# Cannot use blocking get() as it is not interrupted by
# Ctrl-C
if self.wait_queue.get(10000) is True:
self.force_restart()
except Empty:
pass
finally:
observer.unschedule(self.stream)
observer.stop()
def notify(self, ev):
name = ev.name
if isinstance(name, bytes):
name = name.decode('utf-8')
if self.file_is_watched(name):
self.handle_modified({name})
def __init__(self, parent=None):
QWidget.__init__(self, parent)
self.discovery_counter = 0
self.last_hidden_text_warning = None
self.current_search = None
self.anchor_cfi = None
self.l = l = QVBoxLayout(self)
l.setContentsMargins(0, 0, 0, 0)
self.search_input = si = SearchInput(self)
self.searcher = None
self.search_tasks = Queue()
self.results_found.connect(self.on_result_found,
type=Qt.ConnectionType.QueuedConnection)
si.do_search.connect(self.search_requested)
si.cleared.connect(self.search_cleared)
si.go_back.connect(self.go_back)
l.addWidget(si)
self.results = r = Results(self)
r.count_changed.connect(self.count_changed)
r.show_search_result.connect(self.do_show_search_result,
type=Qt.ConnectionType.QueuedConnection)
r.current_result_changed.connect(self.update_hidden_message)
l.addWidget(r, 100)
self.spinner = s = BusySpinner(self)
s.setVisible(False)
l.addWidget(s)
self.hidden_message = la = QLabel(
_('This text is hidden in the book and cannot be displayed'))
la.setStyleSheet('QLabel { margin-left: 1ex }')
la.setWordWrap(True)
la.setVisible(False)
l.addWidget(la)
def __init__(self,
items,
level1=DEFAULT_LEVEL1,
level2=DEFAULT_LEVEL2,
level3=DEFAULT_LEVEL3,
scorer=None):
with wlock:
if not workers:
requests, results = Queue(), Queue()
w = [
Worker(requests, results)
for i in range(max(1, cpu_count()))
]
[x.start() for x in w]
workers.extend(w)
items = map(lambda x: normalize('NFC', unicode_type(x)),
filter(None, items))
self.items = items = tuple(items)
tasks = split(items, len(workers))
self.task_maps = [{j: i
for j, (i, _) in enumerate(task)} for task in tasks]
scorer = scorer or default_scorer
self.scorers = [
scorer(tuple(map(itemgetter(1), task_items)))
for task_items in tasks
]
self.sort_keys = None
def __init__(self, thread_type, thread_count=1):
self.thread_type = thread_type
self.thread_count = thread_count
self.tasks = Queue()
self.results = Queue()
self.threads = []
def single_covers(title, authors, identifiers, caches, tdir):
patch_plugins()
load_caches(caches)
log = GUILog()
results = Queue()
worker = Thread(target=run_download, args=(log, results, Event()),
kwargs=dict(title=title, authors=authors, identifiers=identifiers))
worker.daemon = True
worker.start()
c = Counter()
while worker.is_alive():
try:
plugin, width, height, fmt, data = results.get(True, 1)
except Empty:
continue
else:
name = plugin.name
if plugin.can_get_multiple_covers:
name += '{%d}'%c[plugin.name]
c[plugin.name] += 1
name = '%s,,%s,,%s,,%s.cover'%(name, width, height, fmt)
with open(os.path.join(tdir, name), 'wb') as f:
f.write(data)
os.mkdir(os.path.join(tdir, name+'.done'))
return log.dump()
class Watcher(WatcherBase):
def __init__(self, root_dirs, worker, log):
WatcherBase.__init__(self, worker, log)
self.stream = Stream(self.notify, *(x.encode('utf-8') for x in root_dirs), file_events=True)
self.wait_queue = Queue()
def wakeup(self):
self.wait_queue.put(True)
def loop(self):
observer = Observer()
observer.schedule(self.stream)
observer.daemon = True
observer.start()
try:
while True:
try:
# Cannot use blocking get() as it is not interrupted by
# Ctrl-C
if self.wait_queue.get(10000) is True:
self.force_restart()
except Empty:
pass
finally:
observer.unschedule(self.stream)
observer.stop()
def notify(self, ev):
name = ev.name
if isinstance(name, bytes):
name = name.decode('utf-8')
if self.file_is_watched(name):
self.handle_modified({name})
class DBThread(Thread):
CLOSE = '-------close---------'
def __init__(self, path, row_factory):
Thread.__init__(self)
self.setDaemon(True)
self.path = path
self.unhandled_error = (None, '')
self.row_factory = row_factory
self.requests = Queue(1)
self.results = Queue(1)
self.conn = None
def connect(self):
self.conn = do_connect(self.path, self.row_factory)
def run(self):
try:
self.connect()
while True:
func, args, kwargs = self.requests.get()
if func == self.CLOSE:
self.conn.close()
break
if func == 'dump':
try:
ok, res = True, tuple(self.conn.iterdump())
except Exception as err:
ok, res = False, (err, traceback.format_exc())
elif func == 'create_dynamic_filter':
try:
f = DynamicFilter(args[0])
self.conn.create_function(args[0], 1, f)
ok, res = True, f
except Exception as err:
ok, res = False, (err, traceback.format_exc())
else:
bfunc = getattr(self.conn, func)
try:
for i in range(3):
try:
ok, res = True, bfunc(*args, **kwargs)
break
except OperationalError as err:
# Retry if unable to open db file
e = str(err)
if 'unable to open' not in e or i == 2:
if 'unable to open' in e:
prints('Unable to open database for func',
func, reprlib.repr(args),
reprlib.repr(kwargs))
raise
time.sleep(0.5)
except Exception as err:
ok, res = False, (err, traceback.format_exc())
self.results.put((ok, res))
except Exception as err:
self.unhandled_error = (err, traceback.format_exc())
class DBThread(Thread):
CLOSE = '-------close---------'
def __init__(self, path, row_factory):
Thread.__init__(self)
self.setDaemon(True)
self.path = path
self.unhandled_error = (None, '')
self.row_factory = row_factory
self.requests = Queue(1)
self.results = Queue(1)
self.conn = None
def connect(self):
self.conn = do_connect(self.path, self.row_factory)
def run(self):
try:
self.connect()
while True:
func, args, kwargs = self.requests.get()
if func == self.CLOSE:
self.conn.close()
break
if func == 'dump':
try:
ok, res = True, tuple(self.conn.iterdump())
except Exception as err:
ok, res = False, (err, traceback.format_exc())
elif func == 'create_dynamic_filter':
try:
f = DynamicFilter(args[0])
self.conn.create_function(args[0], 1, f)
ok, res = True, f
except Exception as err:
ok, res = False, (err, traceback.format_exc())
else:
bfunc = getattr(self.conn, func)
try:
for i in range(3):
try:
ok, res = True, bfunc(*args, **kwargs)
break
except OperationalError as err:
# Retry if unable to open db file
e = str(err)
if 'unable to open' not in e or i == 2:
if 'unable to open' in e:
prints('Unable to open database for func',
func, reprlib.repr(args),
reprlib.repr(kwargs))
raise
time.sleep(0.5)
except Exception as err:
ok, res = False, (err, traceback.format_exc())
self.results.put((ok, res))
except Exception as err:
self.unhandled_error = (err, traceback.format_exc())
def __init__(self, path, row_factory):
Thread.__init__(self, daemon=True)
self.path = path
self.unhandled_error = (None, '')
self.row_factory = row_factory
self.requests = Queue(1)
self.results = Queue(1)
self.conn = None
class ConnectedWorker(Thread):
def __init__(self, worker, conn, rfile):
Thread.__init__(self)
self.daemon = True
self.conn = conn
self.worker = worker
self.notifications = Queue()
self._returncode = 'dummy'
self.killed = False
self.log_path = worker.log_path
self.rfile = rfile
self.close_log_file = getattr(worker, 'close_log_file', None)
def start_job(self, job):
notification = PARALLEL_FUNCS[job.name][-1] is not None
eintr_retry_call(self.conn.send, (job.name, job.args, job.kwargs, job.description))
if notification:
self.start()
else:
self.conn.close()
self.job = job
def run(self):
while True:
try:
x = eintr_retry_call(self.conn.recv)
self.notifications.put(x)
except BaseException:
break
try:
self.conn.close()
except BaseException:
pass
def kill(self):
self.killed = True
try:
self.worker.kill()
except BaseException:
pass
@property
def is_alive(self):
return not self.killed and self.worker.is_alive
@property
def returncode(self):
if self._returncode != 'dummy':
return self._returncode
r = self.worker.returncode
if self.killed and r is None:
self._returncode = 1
return 1
if r is not None:
self._returncode = r
return r
class ConnectedWorker(Thread):
def __init__(self, worker, conn, rfile):
Thread.__init__(self)
self.daemon = True
self.conn = conn
self.worker = worker
self.notifications = Queue()
self._returncode = 'dummy'
self.killed = False
self.log_path = worker.log_path
self.rfile = rfile
self.close_log_file = getattr(worker, 'close_log_file', None)
def start_job(self, job):
notification = PARALLEL_FUNCS[job.name][-1] is not None
eintr_retry_call(self.conn.send,
(job.name, job.args, job.kwargs, job.description))
if notification:
self.start()
else:
self.conn.close()
self.job = job
def run(self):
while True:
try:
x = eintr_retry_call(self.conn.recv)
self.notifications.put(x)
except BaseException:
break
try:
self.conn.close()
except BaseException:
pass
def kill(self):
self.killed = True
try:
self.worker.kill()
except BaseException:
pass
@property
def is_alive(self):
return not self.killed and self.worker.is_alive
@property
def returncode(self):
if self._returncode != 'dummy':
return self._returncode
r = self.worker.returncode
if self.killed and r is None:
self._returncode = 1
return 1
if r is not None:
self._returncode = r
return r
def __init__(self):
Thread.__init__(self)
self.daemon = True
self.lock = RLock()
self.queued_jobs = []
self.running_jobs = set()
self.changed_jobs = Queue()
self.keep_going = True
def __init__(self, path, row_factory):
Thread.__init__(self)
self.setDaemon(True)
self.path = path
self.unhandled_error = (None, '')
self.row_factory = row_factory
self.requests = Queue(1)
self.results = Queue(1)
self.conn = None
def __init__(self, log, abort, title, authors, identifiers, caches):
Thread.__init__(self, name='CoverWorker')
self.daemon = True
self.log, self.abort = log, abort
self.title, self.authors, self.identifiers = (title, authors,
identifiers)
self.caches = caches
self.rq = Queue()
self.error = None
def __init__(self, worker, conn, rfile):
Thread.__init__(self)
self.daemon = True
self.conn = conn
self.worker = worker
self.notifications = Queue()
self._returncode = 'dummy'
self.killed = False
self.log_path = worker.log_path
self.rfile = rfile
self.close_log_file = getattr(worker, 'close_log_file', None)
def __init__(self, result_callback=lambda x: x, worker_entry_point='main'):
Thread.__init__(self)
self.worker_entry_point = worker_entry_point
self.start()
self.main_queue = Queue()
self.result_callback = result_callback
self.reap_thread = None
self.shutting_down = False
self.connected = Event()
self.latest_completion_request_id = None
self.request_count = 0
self.lock = RLock()
def __init__(self, *args, **kwargs):
global conn_id
HTTPConnection.__init__(self, *args, **kwargs)
self.sendq = Queue()
self.control_frames = deque()
self.cf_lock = Lock()
self.sending = None
self.send_buf = None
self.frag_decoder = UTF8Decoder()
self.ws_close_received = self.ws_close_sent = False
conn_id += 1
self.websocket_connection_id = conn_id
self.stop_reading = False
class AnnotationsSaveWorker(Thread):
def __init__(self):
Thread.__init__(self, name='AnnotSaveWorker')
self.daemon = True
self.queue = Queue()
def shutdown(self):
if self.is_alive():
self.queue.put(None)
self.join()
def run(self):
while True:
x = self.queue.get()
if x is None:
return
annotations_list = x['annotations_list']
annotations_path_key = x['annotations_path_key']
bld = x['book_library_details']
pathtoebook = x['pathtoebook']
in_book_file = x['in_book_file']
sync_annots_user = x['sync_annots_user']
try:
save_annotations(annotations_list, annotations_path_key, bld,
pathtoebook, in_book_file, sync_annots_user)
except Exception:
import traceback
traceback.print_exc()
def save_annotations(self,
current_book_data,
in_book_file=True,
sync_annots_user=''):
alist = tuple(
annotations_as_copied_list(current_book_data['annotations_map']))
ebp = current_book_data['pathtoebook']
can_save_in_book_file = ebp.lower().endswith('.epub')
self.queue.put({
'annotations_list':
alist,
'annotations_path_key':
current_book_data['annotations_path_key'],
'book_library_details':
current_book_data['book_library_details'],
'pathtoebook':
current_book_data['pathtoebook'],
'in_book_file':
in_book_file and can_save_in_book_file,
'sync_annots_user':
sync_annots_user,
})
def __init__(self, max_workers=None, name=None):
Thread.__init__(self, name=name)
self.max_workers = max_workers or detect_ncpus()
self.available_workers = []
self.busy_workers = {}
self.pending_jobs = []
self.events = Queue()
self.results = Queue()
self.tracker = Queue()
self.terminal_failure = None
self.common_data = pickle_dumps(None)
self.worker_data = None
self.shutting_down = False
self.start()
def __init__(self, url, fname, parent):
QDialog.__init__(self, parent)
self.setWindowTitle(_('Download %s')%fname)
self.l = QVBoxLayout(self)
self.purl = urlparse(url)
self.msg = QLabel(_('Downloading <b>%(fname)s</b> from %(url)s')%dict(
fname=fname, url=self.purl.netloc))
self.msg.setWordWrap(True)
self.l.addWidget(self.msg)
self.pb = QProgressBar(self)
self.pb.setMinimum(0)
self.pb.setMaximum(0)
self.l.addWidget(self.pb)
self.bb = QDialogButtonBox(QDialogButtonBox.Cancel, Qt.Horizontal, self)
self.l.addWidget(self.bb)
self.bb.rejected.connect(self.reject)
sz = self.sizeHint()
self.resize(max(sz.width(), 400), sz.height())
fpath = PersistentTemporaryFile(os.path.splitext(fname)[1])
fpath.close()
self.fpath = fpath.name
self.worker = Worker(url, self.fpath, Queue())
self.rejected = False
def get_covers(themes, dialog, num_of_workers=8):
items = Queue()
tuple(map(items.put, themes))
def callback(metadata, x):
if not sip.isdeleted(dialog) and not dialog.dialog_closed:
dialog.cover_downloaded.emit(metadata, x)
def run():
while True:
try:
metadata = items.get_nowait()
except Empty:
return
try:
cdata = get_cover(metadata)
except Exception as e:
import traceback
traceback.print_exc()
callback(metadata, e)
else:
callback(metadata, cdata)
for w in range(num_of_workers):
t = Thread(name='IconThemeCover', target=run)
t.daemon = True
t.start()
def __init__(self, plugin, kwargs, abort):
Thread.__init__(self)
self.daemon = True
self.plugin, self.kwargs, self.rq = plugin, kwargs, Queue()
self.abort = abort
self.buf = StringIO()
self.log = create_log(self.buf)
def __init__(self, opts, log):
mj = opts.max_jobs
if mj < 1:
mj = detect_ncpus()
self.log = log
self.max_jobs = max(1, mj)
self.max_job_time = max(0, opts.max_job_time * 60)
self.lock = RLock()
self.jobs = {}
self.finished_jobs = {}
self.events = Queue()
self.job_id = count()
self.waiting_job_ids = set()
self.waiting_jobs = deque()
self.max_block = None
self.shutting_down = False
self.event_loop = None
def __init__(self, parent=None):
QWidget.__init__(self, parent)
self.current_search = None
self.l = l = QVBoxLayout(self)
l.setContentsMargins(0, 0, 0, 0)
self.search_input = si = SearchInput(self)
self.searcher = None
self.search_tasks = Queue()
self.results_found.connect(self.on_result_found, type=Qt.QueuedConnection)
si.do_search.connect(self.search_requested)
l.addWidget(si)
self.results = r = Results(self)
r.show_search_result.connect(self.do_show_search_result, type=Qt.QueuedConnection)
l.addWidget(r, 100)
self.spinner = s = BusySpinner(self)
s.setVisible(False)
l.addWidget(s)
def __init__(self):
Thread.__init__(self)
self.daemon = True
self.lock = RLock()
self.queued_jobs = []
self.running_jobs = set()
self.changed_jobs = Queue()
self.keep_going = True
def __init__(self, ps1='>>> ', ps2='... ', show_js=False, libdir=None):
Thread.__init__(self, name='RapydScriptREPL')
self.to_python = to_python
self.JSError = JSError
self.enc = getattr(sys.stdin, 'encoding', None) or 'utf-8'
try:
import readline
self.readline = readline
except ImportError:
pass
self.output = ANSIStream(sys.stdout)
self.to_repl = Queue()
self.from_repl = Queue()
self.ps1, self.ps2 = ps1, ps2
self.show_js, self.libdir = show_js, libdir
self.prompt = ''
self.completions = None
self.start()
def __init__(self, ps1='>>> ', ps2='... ', show_js=False, libdir=None):
Thread.__init__(self, name='RapydScriptREPL')
self.to_python = to_python
self.JSError = JSError
self.enc = getattr(sys.stdin, 'encoding', None) or 'utf-8'
try:
import readline
self.readline = readline
except ImportError:
pass
self.output = ANSIStream(sys.stdout)
self.to_repl = Queue()
self.from_repl = Queue()
self.ps1, self.ps2 = ps1, ps2
self.show_js, self.libdir = show_js, libdir
self.prompt = ''
self.completions = None
self.start()
def compress_images(container, report=None, names=None, jpeg_quality=None, progress_callback=lambda n, t, name:True):
images = get_compressible_images(container)
if names is not None:
images &= set(names)
results = {}
queue = Queue()
abort = Event()
for name in images:
queue.put(name)
def pc(name):
keep_going = progress_callback(len(results), len(images), name)
if not keep_going:
abort.set()
progress_callback(0, len(images), '')
[Worker(abort, 'CompressImage%d' % i, queue, results, container, jpeg_quality, pc) for i in range(min(detect_ncpus(), len(images)))]
queue.join()
before_total = after_total = 0
changed = False
for name, (ok, res) in iteritems(results):
name = force_unicode(name, filesystem_encoding)
if ok:
before, after = res
if before != after:
changed = True
before_total += before
after_total += after
if report:
if before != after:
report(_('{0} compressed from {1} to {2} bytes [{3:.1%} reduction]').format(
name, human_readable(before), human_readable(after), (before - after)/before))
else:
report(_('{0} could not be further compressed').format(name))
else:
report(_('Failed to process {0} with error:').format(name))
report(res)
if report:
if changed:
report('')
report(_('Total image filesize reduced from {0} to {1} [{2:.1%} reduction]').format(
human_readable(before_total), human_readable(after_total), (before_total - after_total)/before_total))
else:
report(_('Images are already fully optimized'))
return changed, results
class Progress(Thread):
def __init__(self, conn):
Thread.__init__(self)
self.daemon = True
self.conn = conn
self.queue = Queue()
def __call__(self, percent, msg=''):
self.queue.put((percent, msg))
def run(self):
while True:
x = self.queue.get()
if x is None:
break
try:
eintr_retry_call(self.conn.send, x)
except:
break
def __init__(self):
QAbstractTableModel.__init__(self)
SearchQueryParser.__init__(self, ['all'])
self.wait_icon = (QIcon(I('jobs.png')))
self.running_icon = (QIcon(I('exec.png')))
self.error_icon = (QIcon(I('dialog_error.png')))
self.done_icon = (QIcon(I('ok.png')))
self.jobs = []
self.add_job = Dispatcher(self._add_job)
self.server = Server(limit=config['worker_limit']//2,
enforce_cpu_limit=config['enforce_cpu_limit'])
self.threaded_server = ThreadedJobServer()
self.changed_queue = Queue()
self.timer = QTimer(self)
self.timer.timeout.connect(self.update, type=Qt.ConnectionType.QueuedConnection)
self.timer.start(1000)
def check_external_links(container, progress_callback=(lambda num, total:None), check_anchors=True):
progress_callback(0, 0)
external_links = defaultdict(list)
for name, mt in iteritems(container.mime_map):
if mt in OEB_DOCS or mt in OEB_STYLES:
for href, lnum, col in container.iterlinks(name):
purl = urlparse(href)
if purl.scheme in ('http', 'https'):
external_links[href].append((name, href, lnum, col))
if not external_links:
return []
items = Queue()
ans = []
tuple(map(items.put, iteritems(external_links)))
progress_callback(0, len(external_links))
done = []
downloaded_html_ids = {}
def check_links():
br = browser(honor_time=False, verify_ssl_certificates=False)
while True:
try:
full_href, locations = items.get_nowait()
except Empty:
return
href, frag = full_href.partition('#')[::2]
try:
res = br.open(href, timeout=10)
except Exception as e:
ans.append((locations, e, full_href))
else:
if frag and check_anchors:
ct = res.info().get('Content-Type')
if ct and ct.split(';')[0].lower() in {'text/html', XHTML_MIME}:
ids = downloaded_html_ids.get(href)
if ids is None:
try:
ids = downloaded_html_ids[href] = get_html_ids(res.read())
except Exception:
ids = downloaded_html_ids[href] = frozenset()
if frag not in ids:
ans.append((locations, ValueError('HTML anchor {} not found on the page'.format(frag)), full_href))
res.close()
finally:
done.append(None)
progress_callback(len(done), len(external_links))
workers = [Thread(name="CheckLinks", target=check_links) for i in range(min(10, len(external_links)))]
for w in workers:
w.daemon = True
w.start()
for w in workers:
w.join()
return ans
class Progress(Thread):
def __init__(self, conn):
Thread.__init__(self)
self.daemon = True
self.conn = conn
self.queue = Queue()
def __call__(self, percent, msg=''):
self.queue.put((percent, msg))
def run(self):
while True:
x = self.queue.get()
if x is None:
break
try:
eintr_retry_call(self.conn.send, x)
except:
break
def __init__(self, worker, conn, rfile):
Thread.__init__(self)
self.daemon = True
self.conn = conn
self.worker = worker
self.notifications = Queue()
self._returncode = 'dummy'
self.killed = False
self.log_path = worker.log_path
self.rfile = rfile
self.close_log_file = getattr(worker, 'close_log_file', None)
def __init__(self,
notify_on_job_done=lambda x: x,
pool_size=None,
limit=sys.maxsize,
enforce_cpu_limit=True):
Thread.__init__(self)
self.daemon = True
self.id = next(server_counter) + 1
if enforce_cpu_limit:
limit = min(limit, cpu_count())
self.pool_size = limit if pool_size is None else pool_size
self.notify_on_job_done = notify_on_job_done
self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue()
self.kill_queue = Queue()
self.waiting_jobs = []
self.workers = deque()
self.launched_worker_counter = count()
next(self.launched_worker_counter)
self.start()
def __init__(self, log, abort, title, authors, identifiers, caches):
Thread.__init__(self)
self.daemon = True
self.log, self.abort = log, abort
self.title, self.authors, self.identifiers = (title, authors,
identifiers)
self.caches = caches
self.rq = Queue()
self.error = None
def __init__(self, description, done=lambda x: x):
self.id = next(job_counter)
self.description = description
self.done = done
self.done2 = None
self.killed = False
self.failed = False
self.kill_on_start = False
self.start_time = None
self.result = None
self.duration = None
self.log_path = None
self.notifications = Queue()
self._run_state = self.WAITING
self.percent = 0
self._message = None
self._status_text = _('Waiting...')
self._done_called = False
self.core_usage = 1
self.timed_out = False
def compress_images(container,
report=None,
names=None,
jpeg_quality=None,
progress_callback=lambda n, t, name: True):
images = get_compressible_images(container)
if names is not None:
images &= set(names)
results = {}
queue = Queue()
abort = Event()
for name in images:
queue.put(name)
def pc(name):
keep_going = progress_callback(len(results), len(images), name)
if not keep_going:
abort.set()
progress_callback(0, len(images), '')
[
Worker(abort, 'CompressImage%d' % i, queue, results, container,
jpeg_quality, pc)
for i in range(min(detect_ncpus(), len(images)))
]
queue.join()
before_total = after_total = 0
changed = False
for name, (ok, res) in results.iteritems():
name = force_unicode(name, filesystem_encoding)
if ok:
before, after = res
if before != after:
changed = True
before_total += before
after_total += after
if report:
if before != after:
report(
_('{0} compressed from {1} to {2} bytes [{3:.1%} reduction]'
).format(name, human_readable(before),
human_readable(after),
(before - after) / before))
else:
report(
_('{0} could not be further compressed').format(name))
else:
report(_('Failed to process {0} with error:').format(name))
report(res)
if report:
if changed:
report('')
report(
_('Total image filesize reduced from {0} to {1} [{2:.1%} reduction]'
).format(human_readable(before_total),
human_readable(after_total),
(before_total - after_total) / before_total))
else:
report(_('Images are already fully optimized'))
return changed, results
def __init__(self, notify_on_job_done=lambda x: x, pool_size=None,
limit=sys.maxsize, enforce_cpu_limit=True):
Thread.__init__(self)
self.daemon = True
global _counter
self.id = _counter+1
_counter += 1
if enforce_cpu_limit:
limit = min(limit, cpu_count())
self.pool_size = limit if pool_size is None else pool_size
self.notify_on_job_done = notify_on_job_done
self.auth_key = os.urandom(32)
self.address, self.listener = create_listener(self.auth_key, backlog=4)
self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue()
self.kill_queue = Queue()
self.waiting_jobs = []
self.workers = deque()
self.launched_worker_count = 0
self._worker_launch_lock = RLock()
self.start()
def __init__(self, notify_on_job_done=lambda x: x, pool_size=None,
limit=sys.maxsize, enforce_cpu_limit=True):
Thread.__init__(self)
self.daemon = True
global _counter
self.id = _counter+1
_counter += 1
if enforce_cpu_limit:
limit = min(limit, cpu_count())
self.pool_size = limit if pool_size is None else pool_size
self.notify_on_job_done = notify_on_job_done
self.auth_key = os.urandom(32)
self.address, self.listener = create_listener(self.auth_key, backlog=4)
self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue()
self.kill_queue = Queue()
self.waiting_jobs = []
self.workers = deque()
self.launched_worker_count = 0
self._worker_launch_lock = RLock()
self.start()
def __init__(self, *args, **kwargs):
global conn_id
HTTPConnection.__init__(self, *args, **kwargs)
self.sendq = Queue()
self.control_frames = deque()
self.cf_lock = Lock()
self.sending = None
self.send_buf = None
self.frag_decoder = UTF8Decoder()
self.ws_close_received = self.ws_close_sent = False
conn_id += 1
self.websocket_connection_id = conn_id
self.stop_reading = False
class ThreadPool(object):
def __init__(self, log, notify_server, count=10, queue_size=1000):
self.request_queue, self.result_queue = Queue(queue_size), Queue(
queue_size)
self.workers = [
Worker(log, notify_server, i, self.request_queue,
self.result_queue) for i in range(count)
]
def start(self):
for w in self.workers:
w.start()
def put_nowait(self, job_id, func):
self.request_queue.put_nowait((job_id, func))
def get_nowait(self):
return self.result_queue.get_nowait()
def stop(self, wait_till):
for w in self.workers:
try:
self.request_queue.put_nowait(None)
except Full:
break
for w in self.workers:
now = monotonic()
if now >= wait_till:
break
w.join(wait_till - now)
self.workers = [w for w in self.workers if w.is_alive()]
@property
def busy(self):
return sum(int(w.working) for w in self.workers)
@property
def idle(self):
return sum(int(not w.working) for w in self.workers)
def __init__(self, result_callback=lambda x:x, worker_entry_point='main'):
Thread.__init__(self)
self.worker_entry_point = worker_entry_point
self.start()
self.main_queue = Queue()
self.result_callback = result_callback
self.reap_thread = None
self.shutting_down = False
self.connected = Event()
self.current_completion_request = None
self.latest_completion_request_id = None
self.request_count = 0
self.lock = RLock()
def download_cover(log, title=None, authors=None, identifiers={}, timeout=30):
'''
Synchronous cover download. Returns the "best" cover as per user
prefs/cover resolution.
Returned cover is a tuple: (plugin, width, height, fmt, data)
Returns None if no cover is found.
'''
rq = Queue()
abort = Event()
run_download(log,
rq,
abort,
title=title,
authors=authors,
identifiers=identifiers,
timeout=timeout,
get_best_cover=True)
results = []
while True:
try:
results.append(rq.get_nowait())
except Empty:
break
cp = msprefs['cover_priorities']
def keygen(result):
plugin, width, height, fmt, data = result
return (cp.get(plugin.name, 1), 1 / (width * height))
results.sort(key=keygen)
return results[0] if results else None
class Watcher(WatcherBase):
def __init__(self, root_dirs, worker, log):
WatcherBase.__init__(self, worker, log)
self.watchers = []
self.modified_queue = Queue()
for d in frozenset(root_dirs):
self.watchers.append(TreeWatcher(d, self.modified_queue))
def wakeup(self):
self.modified_queue.put(True)
def loop(self):
for w in self.watchers:
w.start()
with HandleInterrupt(lambda: self.modified_queue.put(None)):
while True:
path = self.modified_queue.get()
if path is None:
break
if path is True:
self.force_restart()
else:
self.handle_modified({path})
class Watcher(WatcherBase):
def __init__(self, root_dirs, worker, log):
WatcherBase.__init__(self, worker, log)
self.watchers = []
self.modified_queue = Queue()
for d in frozenset(root_dirs):
self.watchers.append(TreeWatcher(d, self.modified_queue))
def wakeup(self):
self.modified_queue.put(True)
def loop(self):
for w in self.watchers:
w.start()
with HandleInterrupt(lambda : self.modified_queue.put(None)):
while True:
path = self.modified_queue.get()
if path is None:
break
if path is True:
self.force_restart()
else:
self.handle_modified({path})
class ThreadPool(object):
def __init__(self, log, notify_server, count=10, queue_size=1000):
self.request_queue, self.result_queue = Queue(queue_size), Queue(queue_size)
self.workers = [Worker(log, notify_server, i, self.request_queue, self.result_queue) for i in range(count)]
def start(self):
for w in self.workers:
w.start()
def put_nowait(self, job_id, func):
self.request_queue.put_nowait((job_id, func))
def get_nowait(self):
return self.result_queue.get_nowait()
def stop(self, wait_till):
for w in self.workers:
try:
self.request_queue.put_nowait(None)
except Full:
break
for w in self.workers:
now = monotonic()
if now >= wait_till:
break
w.join(wait_till - now)
self.workers = [w for w in self.workers if w.is_alive()]
@property
def busy(self):
return sum(int(w.working) for w in self.workers)
@property
def idle(self):
return sum(int(not w.working) for w in self.workers)
def __init__(self, opts, log):
mj = opts.max_jobs
if mj < 1:
mj = detect_ncpus()
self.log = log
self.max_jobs = max(1, mj)
self.max_job_time = max(0, opts.max_job_time * 60)
self.lock = RLock()
self.jobs = {}
self.finished_jobs = {}
self.events = Queue()
self.job_id = count()
self.waiting_job_ids = set()
self.waiting_jobs = deque()
self.max_block = None
self.shutting_down = False
self.event_loop = None
def __init__(self):
QAbstractTableModel.__init__(self)
SearchQueryParser.__init__(self, ['all'])
self.wait_icon = (QIcon(I('jobs.png')))
self.running_icon = (QIcon(I('exec.png')))
self.error_icon = (QIcon(I('dialog_error.png')))
self.done_icon = (QIcon(I('ok.png')))
self.jobs = []
self.add_job = Dispatcher(self._add_job)
self.server = Server(limit=int(config['worker_limit']/2.0),
enforce_cpu_limit=config['enforce_cpu_limit'])
self.threaded_server = ThreadedJobServer()
self.changed_queue = Queue()
self.timer = QTimer(self)
self.timer.timeout.connect(self.update, type=Qt.QueuedConnection)
self.timer.start(1000)
def __init__(self, description, done=lambda x: x):
self.id = next(job_counter)
self.description = description
self.done = done
self.done2 = None
self.killed = False
self.failed = False
self.kill_on_start = False
self.start_time = None
self.result = None
self.duration = None
self.log_path = None
self.notifications = Queue()
self._run_state = self.WAITING
self.percent = 0
self._message = None
self._status_text = _('Waiting...')
self._done_called = False
self.core_usage = 1
self.timed_out = False
def __init__(self):
Thread.__init__(self)
self.requests = Queue()
class GenericDownloadThreadPool(object):
'''
add_task must be implemented in a subclass and must
GenericDownloadThreadPool.add_task must be called
at the end of the function.
'''
def __init__(self, thread_type, thread_count=1):
self.thread_type = thread_type
self.thread_count = thread_count
self.tasks = Queue()
self.results = Queue()
self.threads = []
def set_thread_count(self, thread_count):
self.thread_count = thread_count
def add_task(self):
'''
This must be implemented in a sub class and this function
must be called at the end of the add_task function in
the sub class.
The implementation of this function (in this base class)
starts any threads necessary to fill the pool if it is
not already full.
'''
for i in range(self.thread_count - self.running_threads_count()):
t = self.thread_type(self.tasks, self.results)
self.threads.append(t)
t.start()
def abort(self):
self.tasks = Queue()
self.results = Queue()
for t in self.threads:
t.abort()
self.threads = []
def has_tasks(self):
return not self.tasks.empty()
def get_result(self):
return self.results.get()
def get_result_no_wait(self):
return self.results.get_nowait()
def result_count(self):
return len(self.results)
def has_results(self):
return not self.results.empty()
def threads_running(self):
return self.running_threads_count() > 0
def running_threads_count(self):
count = 0
for t in self.threads:
if t.is_alive():
count += 1
return count
def __init__(self):
Thread.__init__(self)
self.requests = Queue()
self.request_count = 0
self.parse_items = {}
self.launch_error = None
class DeleteService(Thread):
''' Provide a blocking file delete implementation with support for the
recycle bin. On windows, deleting files to the recycle bin spins the event
loop, which can cause locking errors in the main thread. We get around this
by only moving the files/folders to be deleted out of the library in the
main thread, they are deleted to recycle bin in a separate worker thread.
This has the added advantage that doing a restore from the recycle bin wont
cause metadata.db and the file system to get out of sync. Also, deleting
becomes much faster, since in the common case, the move is done by a simple
os.rename(). The downside is that if the user quits calibre while a long
move to recycle bin is happening, the files may not all be deleted.'''
daemon = True
def __init__(self):
Thread.__init__(self)
self.requests = Queue()
def shutdown(self, timeout=20):
self.requests.put(None)
self.join(timeout)
def create_staging(self, library_path):
base_path = os.path.dirname(library_path)
base = os.path.basename(library_path)
try:
ans = tempfile.mkdtemp(prefix=base+' deleted ', dir=base_path)
except OSError:
ans = tempfile.mkdtemp(prefix=base+' deleted ')
atexit.register(remove_dir, ans)
return ans
def remove_dir_if_empty(self, path):
try:
os.rmdir(path)
except OSError as e:
if e.errno == errno.ENOTEMPTY or len(os.listdir(path)) > 0:
# Some linux systems appear to raise an EPERM instead of an
# ENOTEMPTY, see https://bugs.launchpad.net/bugs/1240797
return
raise
def delete_books(self, paths, library_path):
tdir = self.create_staging(library_path)
self.queue_paths(tdir, paths, delete_empty_parent=True)
def queue_paths(self, tdir, paths, delete_empty_parent=True):
try:
self._queue_paths(tdir, paths, delete_empty_parent=delete_empty_parent)
except:
if os.path.exists(tdir):
shutil.rmtree(tdir, ignore_errors=True)
raise
def _queue_paths(self, tdir, paths, delete_empty_parent=True):
requests = []
for path in paths:
if os.path.exists(path):
basename = os.path.basename(path)
c = 0
while True:
dest = os.path.join(tdir, basename)
if not os.path.exists(dest):
break
c += 1
basename = '%d - %s' % (c, os.path.basename(path))
try:
shutil.move(path, dest)
except EnvironmentError:
if os.path.isdir(path):
# shutil.move may have partially copied the directory,
# so the subsequent call to move() will fail as the
# destination directory already exists
raise
# Wait a little in case something has locked a file
time.sleep(1)
shutil.move(path, dest)
if delete_empty_parent:
remove_dir_if_empty(os.path.dirname(path), ignore_metadata_caches=True)
requests.append(dest)
if not requests:
remove_dir_if_empty(tdir)
else:
self.requests.put(tdir)
def delete_files(self, paths, library_path):
tdir = self.create_staging(library_path)
self.queue_paths(tdir, paths, delete_empty_parent=False)
def run(self):
while True:
x = self.requests.get()
try:
if x is None:
break
try:
self.do_delete(x)
except:
import traceback
traceback.print_exc()
finally:
self.requests.task_done()
def wait(self):
'Blocks until all pending deletes have completed'
self.requests.join()
def do_delete(self, tdir):
if os.path.exists(tdir):
try:
for x in os.listdir(tdir):
x = os.path.join(tdir, x)
if os.path.isdir(x):
delete_tree(x)
else:
delete_file(x)
finally:
shutil.rmtree(tdir)
class Repl(Thread):
LINE_CONTINUATION_CHARS = r'\:'
daemon = True
def __init__(self, ps1='>>> ', ps2='... ', show_js=False, libdir=None):
Thread.__init__(self, name='RapydScriptREPL')
self.to_python = to_python
self.JSError = JSError
self.enc = getattr(sys.stdin, 'encoding', None) or 'utf-8'
try:
import readline
self.readline = readline
except ImportError:
pass
self.output = ANSIStream(sys.stdout)
self.to_repl = Queue()
self.from_repl = Queue()
self.ps1, self.ps2 = ps1, ps2
self.show_js, self.libdir = show_js, libdir
self.prompt = ''
self.completions = None
self.start()
def init_ctx(self):
self.prompt = self.ps1
self.ctx = compiler()
self.ctx.g.Duktape.write = self.output.write
self.ctx.eval(r'''console = { log: function() { Duktape.write(Array.prototype.slice.call(arguments).join(' ') + '\n');}};
console['error'] = console['log'];''')
self.ctx.g.repl_options = {
'show_js': self.show_js,
'histfile':False,
'input':True, 'output':True, 'ps1':self.ps1, 'ps2':self.ps2,
'terminal':self.output.isatty,
'enum_global': 'Object.keys(this)',
'lib_path': self.libdir or os.path.dirname(P(COMPILER_PATH)) # TODO: Change this to load pyj files from the src code
}
def get_from_repl(self):
while True:
try:
return self.from_repl.get(True, 1)
except Empty:
if not self.is_alive():
raise SystemExit(1)
def run(self):
self.init_ctx()
rl = None
def set_prompt(p):
self.prompt = p
def prompt(lw):
self.from_repl.put(to_python(lw))
self.ctx.g.set_prompt = set_prompt
self.ctx.g.prompt = prompt
self.ctx.eval('''
listeners = {};
rl = {
setPrompt:set_prompt,
write:Duktape.write,
clearLine: function() {},
on: function(ev, cb) { listeners[ev] = cb; return rl; },
prompt: prompt,
sync_prompt: true,
send_line: function(line) { listeners['line'](line); },
send_interrupt: function() { listeners['SIGINT'](); },
close: function() {listeners['close'](); },
};
repl_options.readline = { createInterface: function(options) { rl.completer = options.completer; return rl; }};
exports.init_repl(repl_options)
''', fname='<init repl>')
rl = self.ctx.g.rl
completer = to_python(rl.completer)
send_interrupt = to_python(rl.send_interrupt)
send_line = to_python(rl.send_line)
while True:
ev, line = self.to_repl.get()
try:
if ev == 'SIGINT':
self.output.write('\n')
send_interrupt()
elif ev == 'line':
send_line(line)
else:
val = completer(line)
val = to_python(val)
self.from_repl.put(val[0])
except Exception as e:
if isinstance(e, JSError):
print(e.stack or error_message(e), file=sys.stderr)
else:
import traceback
traceback.print_exc()
for i in range(100):
# Do this many times to ensure we dont deadlock
self.from_repl.put(None)
def __call__(self):
if hasattr(self, 'readline'):
history = os.path.join(cache_dir(), 'pyj-repl-history.txt')
self.readline.parse_and_bind("tab: complete")
try:
self.readline.read_history_file(history)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
atexit.register(partial(self.readline.write_history_file, history))
def completer(text, num):
if self.completions is None:
self.to_repl.put(('complete', text))
self.completions = list(filter(None, self.get_from_repl()))
if not self.completions:
return None
try:
return self.completions[num]
except (IndexError, TypeError, AttributeError, KeyError):
self.completions = None
if hasattr(self, 'readline'):
self.readline.set_completer(completer)
while True:
lw = self.get_from_repl()
if lw is None:
raise SystemExit(1)
q = self.prompt
if hasattr(self, 'readline'):
self.readline.set_pre_input_hook(lambda:(self.readline.insert_text(lw), self.readline.redisplay()))
else:
q += lw
try:
line = raw_input(q)
self.to_repl.put(('line', line))
except EOFError:
return
except KeyboardInterrupt:
self.to_repl.put(('SIGINT', None))
def abort(self):
self.tasks = Queue()
self.results = Queue()
for t in self.threads:
t.abort()
self.threads = []
class Pool(Thread):
daemon = True
def __init__(self, max_workers=None, name=None):
Thread.__init__(self, name=name)
self.max_workers = max_workers or detect_ncpus()
self.available_workers = []
self.busy_workers = {}
self.pending_jobs = []
self.events = Queue()
self.results = Queue()
self.tracker = Queue()
self.terminal_failure = None
self.common_data = pickle_dumps(None)
self.worker_data = None
self.shutting_down = False
self.start()
def set_common_data(self, data=None):
''' Set some data that will be passed to all subsequent jobs without
needing to be transmitted every time. You must call this method before
queueing any jobs, otherwise the behavior is undefined. You can call it
after all jobs are done, then it will be used for the new round of
jobs. Can raise the :class:`Failure` exception is data could not be
sent to workers.'''
if self.failed:
raise Failure(self.terminal_failure)
self.events.put(data)
def __call__(self, job_id, module, func, *args, **kwargs):
'''
Schedule a job. The job will be run in a worker process, with the
result placed in self.results. If a terminal failure has occurred
previously, this method will raise the :class:`Failure` exception.
:param job_id: A unique id for the job. The result will have this id.
:param module: Either a fully qualified python module name or python
source code which will be executed as a module.
Source code is detected by the presence of newlines in module.
:param func: Name of the function from ``module`` that will be
executed. ``args`` and ``kwargs`` will be passed to the function.
'''
if self.failed:
raise Failure(self.terminal_failure)
job = Job(job_id, module, func, args, kwargs)
self.tracker.put(None)
self.events.put(job)
def wait_for_tasks(self, timeout=None):
''' Wait for all queued jobs to be completed, if timeout is not None,
will raise a RuntimeError if jobs are not completed in the specified
time. Will raise a :class:`Failure` exception if a terminal failure has
occurred previously. '''
if self.failed:
raise Failure(self.terminal_failure)
if timeout is None:
self.tracker.join()
else:
join_with_timeout(self.tracker, timeout)
def shutdown(self, wait_time=0.1):
''' Shutdown this pool, terminating all worker process. The pool cannot
be used after a shutdown. '''
self.shutting_down = True
self.events.put(None)
self.shutdown_workers(wait_time=wait_time)
def create_worker(self):
p = start_worker('from {0} import run_main, {1}; run_main({1})'.format(self.__class__.__module__, 'worker_main'))
sys.stdout.flush()
eintr_retry_call(p.stdin.write, self.worker_data)
p.stdin.flush(), p.stdin.close()
conn = eintr_retry_call(self.listener.accept)
w = Worker(p, conn, self.events, self.name)
if self.common_data != pickle_dumps(None):
w.set_common_data(self.common_data)
return w
def start_worker(self):
try:
w = self.create_worker()
if not self.shutting_down:
self.available_workers.append(w)
except Exception:
import traceback
self.terminal_failure = TerminalFailure('Failed to start worker process', traceback.format_exc(), None)
self.terminal_error()
return False
def run(self):
from calibre.utils.ipc.server import create_listener
self.auth_key = os.urandom(32)
self.address, self.listener = create_listener(self.auth_key)
self.worker_data = msgpack_dumps((self.address, self.auth_key))
if self.start_worker() is False:
return
while True:
event = self.events.get()
if event is None or self.shutting_down:
break
if self.handle_event(event) is False:
break
def handle_event(self, event):
if isinstance(event, Job):
job = event
if not self.available_workers:
if len(self.busy_workers) >= self.max_workers:
self.pending_jobs.append(job)
return
if self.start_worker() is False:
return False
return self.run_job(job)
elif isinstance(event, WorkerResult):
worker_result = event
self.busy_workers.pop(worker_result.worker, None)
self.available_workers.append(worker_result.worker)
self.tracker.task_done()
if worker_result.is_terminal_failure:
self.terminal_failure = TerminalFailure('Worker process crashed while executing job', worker_result.result.traceback, worker_result.id)
self.terminal_error()
return False
self.results.put(worker_result)
else:
self.common_data = pickle_dumps(event)
if len(self.common_data) > MAX_SIZE:
self.cd_file = PersistentTemporaryFile('pool_common_data')
with self.cd_file as f:
f.write(self.common_data)
self.common_data = pickle_dumps(File(f.name))
for worker in self.available_workers:
try:
worker.set_common_data(self.common_data)
except Exception:
import traceback
self.terminal_failure = TerminalFailure('Worker process crashed while sending common data', traceback.format_exc(), None)
self.terminal_error()
return False
while self.pending_jobs and self.available_workers:
if self.run_job(self.pending_jobs.pop()) is False:
return False
def run_job(self, job):
worker = self.available_workers.pop()
try:
worker(job)
except Exception:
import traceback
self.terminal_failure = TerminalFailure('Worker process crashed while sending job', traceback.format_exc(), job.id)
self.terminal_error()
return False
self.busy_workers[worker] = job
@property
def failed(self):
return self.terminal_failure is not None
def terminal_error(self):
if self.shutting_down:
return
for worker, job in iteritems(self.busy_workers):
self.results.put(WorkerResult(job.id, Result(None, None, None), True, worker))
self.tracker.task_done()
while self.pending_jobs:
job = self.pending_jobs.pop()
self.results.put(WorkerResult(job.id, Result(None, None, None), True, None))
self.tracker.task_done()
self.shutdown()
def shutdown_workers(self, wait_time=0.1):
self.worker_data = self.common_data = None
for worker in self.busy_workers:
if worker.process.poll() is None:
try:
worker.process.terminate()
except EnvironmentError:
pass # If the process has already been killed
workers = [w.process for w in self.available_workers + list(self.busy_workers)]
aw = list(self.available_workers)
def join():
for w in aw:
try:
w(None)
except Exception:
pass
for w in workers:
try:
w.wait()
except Exception:
pass
reaper = Thread(target=join, name='ReapPoolWorkers')
reaper.daemon = True
reaper.start()
reaper.join(wait_time)
for w in self.available_workers + list(self.busy_workers):
try:
w.conn.close()
except Exception:
pass
for w in workers:
if w.poll() is None:
try:
w.kill()
except EnvironmentError:
pass
del self.available_workers[:]
self.busy_workers.clear()
if hasattr(self, 'cd_file'):
try:
os.remove(self.cd_file.name)
except EnvironmentError:
pass
class CoverWorker(Thread): # {{{
def __init__(self, log, abort, title, authors, identifiers, caches):
Thread.__init__(self)
self.daemon = True
self.log, self.abort = log, abort
self.title, self.authors, self.identifiers = (title, authors,
identifiers)
self.caches = caches
self.rq = Queue()
self.error = None
def fake_run(self):
images = ['donate.png', 'config.png', 'column.png', 'eject.png', ]
time.sleep(2)
for pl, im in zip(metadata_plugins(['cover']), images):
self.rq.put((pl.name, 1, 1, 'png', I(im, data=True)))
def run(self):
try:
if DEBUG_DIALOG:
self.fake_run()
else:
self.run_fork()
except WorkerError as e:
self.error = force_unicode(e.orig_tb)
except:
import traceback
self.error = force_unicode(traceback.format_exc())
def run_fork(self):
with TemporaryDirectory('_single_metadata_download') as tdir:
self.keep_going = True
t = Thread(target=self.monitor_tdir, args=(tdir,))
t.daemon = True
t.start()
try:
res = fork_job('calibre.ebooks.metadata.sources.worker',
'single_covers',
(self.title, self.authors, self.identifiers, self.caches,
tdir),
no_output=True, abort=self.abort)
self.log.append_dump(res['result'])
finally:
self.keep_going = False
t.join()
def scan_once(self, tdir, seen):
for x in list(os.listdir(tdir)):
if x in seen:
continue
if x.endswith('.cover') and os.path.exists(os.path.join(tdir,
x+'.done')):
name = x.rpartition('.')[0]
try:
plugin_name, width, height, fmt = name.split(',,')
width, height = int(width), int(height)
with open(os.path.join(tdir, x), 'rb') as f:
data = f.read()
except:
import traceback
traceback.print_exc()
else:
seen.add(x)
self.rq.put((plugin_name, width, height, fmt, data))
def monitor_tdir(self, tdir):
seen = set()
while self.keep_going:
time.sleep(1)
self.scan_once(tdir, seen)
# One last scan after the download process has ended
self.scan_once(tdir, seen)
class JobsManager(object):
def __init__(self, opts, log):
mj = opts.max_jobs
if mj < 1:
mj = detect_ncpus()
self.log = log
self.max_jobs = max(1, mj)
self.max_job_time = max(0, opts.max_job_time * 60)
self.lock = RLock()
self.jobs = {}
self.finished_jobs = {}
self.events = Queue()
self.job_id = count()
self.waiting_job_ids = set()
self.waiting_jobs = deque()
self.max_block = None
self.shutting_down = False
self.event_loop = None
def start_job(self, name, module, func, args=(), kwargs=None, job_done_callback=None, job_data=None):
with self.lock:
if self.shutting_down:
return None
if self.event_loop is None:
self.event_loop = t = Thread(name='JobsEventLoop', target=self.run)
t.daemon = True
t.start()
job_id = next(self.job_id)
self.events.put(StartEvent(job_id, name, module, func, args, kwargs or {}, job_done_callback, job_data))
self.waiting_job_ids.add(job_id)
return job_id
def job_status(self, job_id):
with self.lock:
if not self.shutting_down:
if job_id in self.finished_jobs:
job = self.finished_jobs[job_id]
return 'finished', job.result, job.traceback, job.was_aborted
if job_id in self.jobs:
return 'running', None, None, None
if job_id in self.waiting_job_ids:
return 'waiting', None, None, None
return None, None, None, None
def abort_job(self, job_id):
job = self.jobs.get(job_id)
if job is not None:
job.abort_event.set()
def wait_for_running_job(self, job_id, timeout=None):
job = self.jobs.get(job_id)
if job is not None:
job.wait_for_end.wait(timeout)
if not job.done:
return False
while job_id not in self.finished_jobs:
time.sleep(0.001)
return True
def shutdown(self, timeout=5.0):
with self.lock:
self.shutting_down = True
for job in itervalues(self.jobs):
job.abort_event.set()
self.events.put(False)
def wait_for_shutdown(self, wait_till):
for job in itervalues(self.jobs):
delta = wait_till - monotonic()
if delta > 0:
job.join(delta)
if self.event_loop is not None:
delta = wait_till - monotonic()
if delta > 0:
self.event_loop.join(delta)
# Internal API {{{
def run(self):
while not self.shutting_down:
if self.max_block is None:
ev = self.events.get()
else:
try:
ev = self.events.get(block=True, timeout=self.max_block)
except Empty:
ev = None
if self.shutting_down:
break
if ev is None:
self.abort_hanging_jobs()
elif isinstance(ev, StartEvent):
self.waiting_jobs.append(ev)
self.start_waiting_jobs()
elif isinstance(ev, DoneEvent):
self.job_finished(ev.job_id)
elif ev is False:
break
def start_waiting_jobs(self):
with self.lock:
while self.waiting_jobs and len(self.jobs) < self.max_jobs:
ev = self.waiting_jobs.popleft()
self.jobs[ev.job_id] = Job(ev, self.events)
self.waiting_job_ids.discard(ev.job_id)
self.update_max_block()
def update_max_block(self):
with self.lock:
mb = None
now = monotonic()
for job in itervalues(self.jobs):
if not job.done and not job.abort_event.is_set():
delta = self.max_job_time - (now - job.start_time)
if delta <= 0:
self.max_block = 0
return
if mb is None:
mb = delta
else:
mb = min(mb, delta)
self.max_block = mb
def abort_hanging_jobs(self):
now = monotonic()
found = False
for job in itervalues(self.jobs):
if not job.done and not job.abort_event.is_set():
delta = self.max_job_time - (now - job.start_time)
if delta <= 0:
job.abort_event.set()
found = True
if found:
self.update_max_block()
def job_finished(self, job_id):
with self.lock:
self.finished_jobs[job_id] = job = self.jobs.pop(job_id)
if job.callback is not None:
try:
job.callback(job)
except Exception:
import traceback
self.log.error('Error running callback for job: %s:\n%s' % (job.name, traceback.format_exc()))
self.prune_finished_jobs()
if job.traceback and not job.was_aborted:
logdata = job.read_log()
self.log.error('The job: %s failed:\n%s\n%s' % (job.job_name, logdata, job.traceback))
job.remove_log()
self.start_waiting_jobs()
def prune_finished_jobs(self):
with self.lock:
remove = []
now = monotonic()
for job_id, job in iteritems(self.finished_jobs):
if now - job.end_time > 3600:
remove.append(job_id)
for job_id in remove:
del self.finished_jobs[job_id]
