def offload():
# The entry point for the offload worker process
address = cPickle.loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
key = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
func_cache = {}
with closing(Client(address, authkey=key)) as conn:
while True:
args = eintr_retry_call(conn.recv)
if args is None:
break
res = {'result':None, 'tb':None}
try:
mod, func, args, kwargs = args
if mod is None:
eintr_retry_call(conn.send, res)
continue
f = func_cache.get((mod, func), None)
if f is None:
try:
m = importlib.import_module(mod)
except ImportError:
importlib.import_module('calibre.customize.ui') # Load plugins
m = importlib.import_module(mod)
func_cache[(mod, func)] = f = getattr(m, func)
res['result'] = f(*args, **kwargs)
except:
import traceback
res['tb'] = traceback.format_exc()
eintr_retry_call(conn.send, res)
def send(self, data, conn=None):
conn = conn or self.control_conn
try:
eintr_retry_call(conn.send, data)
except:
if not self.shutting_down:
raise
def __call__(self, job):
eintr_retry_call(self.conn.send_bytes, cPickle.dumps(job, -1))
if job is not None:
self.job_id = job.id
t = Thread(target=self.recv, name='PoolWorker-'+self.name)
t.daemon = True
t.start()
def cli(port=4444):
prints('Connecting to remote debugger on port %d...' % port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
for i in range(20):
try:
sock.connect(('127.0.0.1', port))
break
except socket.error:
pass
time.sleep(0.1)
else:
try:
sock.connect(('127.0.0.1', port))
except socket.error as err:
prints('Failed to connect to remote debugger:', err, file=sys.stderr)
raise SystemExit(1)
prints('Connected to remote process')
import readline
histfile = os.path.join(cache_dir(), 'rpdb.history')
try:
readline.read_history_file(histfile)
except IOError:
pass
atexit.register(readline.write_history_file, histfile)
p = pdb.Pdb()
readline.set_completer(p.complete)
readline.parse_and_bind("tab: complete")
stdin = getattr(sys.stdin, 'buffer', sys.stdin)
stdout = getattr(sys.stdout, 'buffer', sys.stdout)
try:
while True:
recvd = b''
while not recvd.endswith(PROMPT) or select.select([sock], [], [], 0) == ([sock], [], []):
buf = eintr_retry_call(sock.recv, 16 * 1024)
if not buf:
return
recvd += buf
recvd = recvd[:-len(PROMPT)]
if recvd.startswith(QUESTION):
recvd = recvd[len(QUESTION):]
stdout.write(recvd)
raw = stdin.readline() or b'n'
else:
stdout.write(recvd)
raw = b''
try:
raw = raw_input(PROMPT.decode('utf-8'))
except (EOFError, KeyboardInterrupt):
pass
else:
if not isinstance(raw, bytes):
raw = raw.encode('utf-8')
raw += b'\n'
if not raw:
raw = b'quit\n'
eintr_retry_call(sock.send, raw)
except KeyboardInterrupt:
pass
def start_job(self, job):
notification = PARALLEL_FUNCS[job.name][-1] is not None
eintr_retry_call(self.conn.send, (job.name, job.args, job.kwargs, job.description))
if notification:
self.start()
else:
self.conn.close()
self.job = job
def run(self):
while True:
x = self.queue.get()
if x is None:
break
try:
eintr_retry_call(self.conn.send, x)
except:
break
def create_single_instance_mutex(name, per_user=True):
from calibre.utils.ipc import eintr_retry_call
path = singleinstance_path(name, per_user)
f = lopen(path, 'w')
try:
eintr_retry_call(fcntl.lockf, f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
return partial(_clean_lock_file, f)
except EnvironmentError as err:
if err.errno not in (errno.EAGAIN, errno.EACCES):
raise
def create_worker(self):
p = start_worker('from {0} import run_main, {1}; run_main({1})'.format(self.__class__.__module__, 'worker_main'))
sys.stdout.flush()
eintr_retry_call(p.stdin.write, self.worker_data)
p.stdin.flush(), p.stdin.close()
conn = eintr_retry_call(self.listener.accept)
w = Worker(p, conn, self.events, self.name)
if self.common_data != pickle_dumps(None):
w.set_common_data(self.common_data)
return w
def is_tdir_locked(path):
lf = os.path.join(path, TDIR_LOCK)
f = lopen(lf, 'w')
try:
eintr_retry_call(fcntl.lockf, f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
eintr_retry_call(fcntl.lockf, f.fileno(), fcntl.LOCK_UN)
return False
except EnvironmentError:
return True
finally:
f.close()
def run(self):
while True:
x = self.dataq.get()
if x is None:
break
try:
self.data_written = True
eintr_retry_call(self.conn.send_bytes, x)
except Exception as e:
self.resultq.put(as_unicode(e))
else:
self.resultq.put(None)
def launch_worker_process(self):
from calibre.utils.ipc.server import create_listener
from calibre.utils.ipc.simple_worker import start_pipe_worker
self.worker_process = p = start_pipe_worker(
'from {0} import run_main, {1}; run_main({1})'.format(self.__class__.__module__, self.worker_entry_point), stdout=None)
auth_key = os.urandom(32)
address, self.listener = create_listener(auth_key)
eintr_retry_call(p.stdin.write, cPickle.dumps((address, auth_key), -1))
p.stdin.flush(), p.stdin.close()
self.control_conn = eintr_retry_call(self.listener.accept)
self.data_conn = eintr_retry_call(self.listener.accept)
self.connected.set()
def create_worker(self):
from calibre.utils.ipc.simple_worker import start_pipe_worker
p = start_pipe_worker(
'from {0} import run_main, {1}; run_main({1})'.format(self.__class__.__module__, 'worker_main'), stdout=None)
sys.stdout.flush()
eintr_retry_call(p.stdin.write, self.worker_data)
p.stdin.flush(), p.stdin.close()
conn = eintr_retry_call(self.listener.accept)
w = Worker(p, conn, self.events, self.name)
if self.common_data != cPickle.dumps(None, -1):
w.set_common_data(self.common_data)
return w
def shutdown(self):
try:
eintr_retry_call(self.conn.send, None)
except:
import traceback
traceback.print_exc()
finally:
self.conn = None
try:
os.remove(self.worker.log_path)
except:
pass
self.kill_thread.start()
def delegate_recycle(path):
if '\n' in path:
raise ValueError('Cannot recycle paths that have newlines in them (%r)' % path)
with rlock:
start_recycler()
eintr_retry_call(print, path.encode('utf-8'), file=recycler.stdin)
recycler.stdin.flush()
# Theoretically this could be made non-blocking using a
# thread+queue, however the original implementation was blocking,
# so I am leaving it as blocking.
result = eintr_retry_call(recycler.stdout.readline)
if result.rstrip() != b'OK':
raise RuntimeError('recycler failed to recycle: %r' % path)
def worker_main(conn):
from importlib import import_module
common_data = None
while True:
try:
job = pickle_loads(eintr_retry_call(conn.recv_bytes))
except EOFError:
break
except KeyboardInterrupt:
break
except Exception:
prints('recv() failed in worker, terminating worker', file=sys.stderr)
import traceback
traceback.print_exc()
return 1
if job is None:
break
if not isinstance(job, Job):
if isinstance(job, File):
with lopen(job.name, 'rb') as f:
common_data = f.read()
common_data = pickle_loads(common_data)
else:
common_data = job
continue
try:
if '\n' in job.module:
import_module('calibre.customize.ui') # Load plugins
from calibre.utils.ipc.simple_worker import compile_code
mod = compile_code(job.module)
func = mod[job.func]
else:
func = getattr(import_module(job.module), job.func)
if common_data is not None:
job.kwargs['common_data'] = common_data
result = func(*job.args, **job.kwargs)
result = Result(result, None, None)
except Exception as err:
import traceback
result = Result(None, as_unicode(err), traceback.format_exc())
try:
eintr_retry_call(conn.send_bytes, pickle_dumps(result))
except EOFError:
break
except Exception:
prints('send() failed in worker, terminating worker', file=sys.stderr)
import traceback
traceback.print_exc()
return 1
return 0
def run(self):
conn = None
try:
conn = eintr_retry_call(self.listener.accept)
except BaseException:
self.tb = traceback.format_exc()
return
self.accepted = True
with closing(conn):
try:
eintr_retry_call(conn.send, self.args)
self.res = eintr_retry_call(conn.recv)
except BaseException:
self.tb = traceback.format_exc()
def call_func(self, func_name, args, kwargs, conn):
with Writer(conn) as f:
try:
self.dispatcher(f, func_name, args, kwargs)
except Exception as e:
if not f.data_written:
import traceback
# Try to tell the client process what error happened
try:
eintr_retry_call(conn.send_bytes, (_encode(('failed', (unicode(e),
as_unicode(traceback.format_exc()))))))
except:
pass
raise
def launch_worker_process(self):
from calibre.utils.ipc.server import create_listener
from calibre.utils.ipc.pool import start_worker
self.worker_process = p = start_worker(
'from {0} import run_main, {1}; run_main({1})'.format(self.__class__.__module__, self.worker_entry_point))
auth_key = os.urandom(32)
address, self.listener = create_listener(auth_key)
eintr_retry_call(p.stdin.write, cPickle.dumps((address, auth_key), -1))
p.stdin.flush(), p.stdin.close()
self.control_conn = eintr_retry_call(self.listener.accept)
self.data_conn = eintr_retry_call(self.listener.accept)
self.data_thread = t = Thread(name='CWData', target=self.handle_data_requests)
t.daemon = True
t.start()
self.connected.set()
def shutdown(self):
try:
eintr_retry_call(self.conn.send, None)
except:
import traceback
traceback.print_exc()
finally:
self.conn = None
t = Thread(target=self.worker.kill)
t.daemon = True
try:
os.remove(self.worker.log_path)
except:
pass
t.start()
def run_main(func):
from multiprocessing.connection import Client
from contextlib import closing
stdin = getattr(sys.stdin, 'buffer', sys.stdin)
address, key = msgpack_loads(eintr_retry_call(stdin.read))
with closing(Client(address, authkey=key)) as conn:
raise SystemExit(func(conn))
def main():
# The entry point for the simple worker process
address = cPickle.loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
key = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
with closing(Client(address, authkey=key)) as conn:
args = eintr_retry_call(conn.recv)
try:
mod, func, args, kwargs, module_is_source_code = args
if module_is_source_code:
importlib.import_module('calibre.customize.ui') # Load plugins
mod = compile_code(mod)
func = mod[func]
else:
try:
mod = importlib.import_module(mod)
except ImportError:
importlib.import_module('calibre.customize.ui') # Load plugins
mod = importlib.import_module(mod)
func = getattr(mod, func)
res = {'result':func(*args, **kwargs)}
except:
res = {'tb': traceback.format_exc()}
try:
conn.send(res)
except:
# Maybe EINTR
conn.send(res)
def run(self):
while self.keep_going:
try:
conn = eintr_retry_call(self.listener.accept)
self.handle_client(conn)
except:
pass
def main():
if iswindows:
if '--multiprocessing-fork' in sys.argv:
# We are using the multiprocessing module on windows to launch a
# worker process
from multiprocessing import freeze_support
freeze_support()
return 0
# Close open file descriptors inherited from parent
# On Unix this is done by the subprocess module
os.closerange(3, 256)
if isosx and 'CALIBRE_WORKER_ADDRESS' not in os.environ and 'CALIBRE_SIMPLE_WORKER' not in os.environ and '--pipe-worker' not in sys.argv:
# On some OS X computers launchd apparently tries to
# launch the last run process from the bundle
# so launch the gui as usual
from calibre.gui2.main import main as gui_main
return gui_main(['calibre'])
csw = os.environ.get('CALIBRE_SIMPLE_WORKER', None)
if csw:
mod, _, func = csw.partition(':')
mod = importlib.import_module(mod)
func = getattr(mod, func)
func()
return
if '--pipe-worker' in sys.argv:
try:
exec (sys.argv[-1])
except Exception:
print('Failed to run pipe worker with command:', sys.argv[-1])
raise
return
address = cPickle.loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
key = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
resultf = unhexlify(os.environ['CALIBRE_WORKER_RESULT']).decode('utf-8')
with closing(Client(address, authkey=key)) as conn:
name, args, kwargs, desc = eintr_retry_call(conn.recv)
if desc:
prints(desc)
sys.stdout.flush()
func, notification = get_func(name)
notifier = Progress(conn)
if notification:
kwargs[notification] = notifier
notifier.start()
result = func(*args, **kwargs)
if result is not None and os.path.exists(os.path.dirname(resultf)):
cPickle.dump(result, open(resultf, 'wb'), -1)
notifier.queue.put(None)
try:
sys.stdout.flush()
except EnvironmentError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
try:
sys.stderr.flush()
except EnvironmentError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
return 0
def recv(self, conn=None):
conn = conn or self.control_conn
try:
return eintr_retry_call(conn.recv)
except:
if not self.shutting_down:
raise
def recv(self):
try:
result = cPickle.loads(eintr_retry_call(self.conn.recv_bytes))
wr = WorkerResult(self.job_id, result, False, self)
except Exception as err:
import traceback
result = Result(None, as_unicode(err), traceback.format_exc())
wr = WorkerResult(self.job_id, result, True, self)
self.events.put(wr)
def singleinstance(name):
import socket
from calibre.utils.ipc import eintr_retry_call
name = '%s-singleinstance-%s-%d' % (__appname__, name, os.geteuid())
if not isinstance(name, bytes):
name = name.encode('utf-8')
address = b'\0' + name.replace(b' ', b'_')
sock = socket.socket(family=socket.AF_UNIX)
try:
eintr_retry_call(sock.bind, address)
except socket.error as err:
if getattr(err, 'errno', None) == errno.EADDRINUSE:
return False
raise
fd = sock.fileno()
old_flags = fcntl.fcntl(fd, fcntl.F_GETFD)
fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC)
atexit.register(sock.close)
return True
def singleinstance(name):
'''
Return True if no other instance of the application identified by name is running,
False otherwise.
@param name: The name to lock.
@type name: string
'''
from calibre.utils.ipc import eintr_retry_call
path = singleinstance_path(name)
f = open(path, 'w')
try:
eintr_retry_call(fcntl.lockf, f.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
atexit.register(_clean_lock_file, f)
return True
except IOError as err:
if err.errno == errno.EAGAIN:
return False
raise
return False
def read(amount):
ans = b''
left = amount
while left > 0:
raw = eintr_retry_call(sys.stdin.read, left)
if len(raw) == 0:
raise EOFError('')
left -= len(raw)
ans += raw
return ans
def create_single_instance_mutex(name, per_user=True):
import socket
from calibre.utils.ipc import eintr_retry_call
name = '%s-singleinstance-%s-%s' % (
__appname__, (os.geteuid() if per_user else ''), name
)
name = name.encode('utf-8')
address = b'\0' + name.replace(b' ', b'_')
sock = socket.socket(family=socket.AF_UNIX)
try:
eintr_retry_call(sock.bind, address)
except socket.error as err:
if getattr(err, 'errno', None) == errno.EADDRINUSE:
return
raise
fd = sock.fileno()
old_flags = fcntl.fcntl(fd, fcntl.F_GETFD)
fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC)
return sock.close
def run(self):
while True:
try:
x = eintr_retry_call(self.conn.recv)
self.notifications.put(x)
except BaseException:
break
try:
self.conn.close()
except BaseException:
pass
def do_launch(self, env, gui, redirect_output, rfile, job_name=None):
w = Worker(env, gui=gui, job_name=job_name)
try:
w(redirect_output=redirect_output)
conn = eintr_retry_call(self.listener.accept)
if conn is None:
raise Exception('Failed to launch worker process')
except BaseException:
try:
w.kill()
except:
pass
import traceback
return traceback.format_exc()
return ConnectedWorker(w, conn, rfile)
def main():
if iswindows:
if '--multiprocessing-fork' in sys.argv:
# We are using the multiprocessing module on windows to launch a
# worker process
from multiprocessing import freeze_support
freeze_support()
return 0
# Close open file descriptors inherited from parent
# On Unix this is done by the subprocess module
os.closerange(3, 256)
if isosx and 'CALIBRE_WORKER_ADDRESS' not in os.environ:
# On some OS X computers launchd apparently tries to
# launch the last run process from the bundle
# so launch the gui as usual
from calibre.gui2.main import main as gui_main
return gui_main(['calibre'])
csw = os.environ.get('CALIBRE_SIMPLE_WORKER', None)
if csw:
mod, _, func = csw.partition(':')
mod = importlib.import_module(mod)
func = getattr(mod, func)
func()
return
address = cPickle.loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
key = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
resultf = unhexlify(os.environ['CALIBRE_WORKER_RESULT']).decode('utf-8')
with closing(Client(address, authkey=key)) as conn:
name, args, kwargs, desc = eintr_retry_call(conn.recv)
if desc:
prints(desc)
sys.stdout.flush()
func, notification = get_func(name)
notifier = Progress(conn)
if notification:
kwargs[notification] = notifier
notifier.start()
result = func(*args, **kwargs)
if result is not None and os.path.exists(os.path.dirname(resultf)):
cPickle.dump(result, open(resultf, 'wb'), -1)
notifier.queue.put(None)
sys.stdout.flush()
sys.stderr.flush()
return 0
def _handle_client(self, conn):
while True:
try:
func_name, args, kwargs = eintr_retry_call(conn.recv)
except EOFError:
try:
conn.close()
except:
pass
return
else:
try:
self.call_func(func_name, args, kwargs, conn)
except:
try:
conn.close()
except:
pass
prints('Proxy function: %s with args: %r and'
' kwargs: %r failed')
if DEBUG:
import traceback
traceback.print_exc()
break
def run_main(func):
from multiprocessing.connection import Client
stdin = getattr(sys.stdin, 'buffer', sys.stdin)
address, key = msgpack_loads(eintr_retry_call(stdin.read))
with closing(Client(address, authkey=key)) as control_conn, closing(Client(address, authkey=key)) as data_conn:
func(control_conn, data_conn)
def set_common_data(self, data):
eintr_retry_call(self.conn.send_bytes, data)
def __call__(self, module, func, *args, **kwargs):
if self.conn is None:
self.conn = eintr_retry_call(self.listener.accept)
eintr_retry_call(self.conn.send, (module, func, args, kwargs))
return eintr_retry_call(self.conn.recv)
def run_main(func):
from multiprocessing.connection import Client
from contextlib import closing
address, key = cPickle.loads(eintr_retry_call(sys.stdin.read))
with closing(Client(address, authkey=key)) as conn:
raise SystemExit(func(conn))
def main():
if iswindows:
if '--multiprocessing-fork' in sys.argv:
# We are using the multiprocessing module on windows to launch a
# worker process
from multiprocessing import freeze_support
freeze_support()
return 0
# Close open file descriptors inherited from parent
# On Unix this is done by the subprocess module
os.closerange(3, 256)
if isosx and 'CALIBRE_WORKER_ADDRESS' not in os.environ and 'CALIBRE_SIMPLE_WORKER' not in os.environ and '--pipe-worker' not in sys.argv:
# On some OS X computers launchd apparently tries to
# launch the last run process from the bundle
# so launch the gui as usual
from calibre.gui2.main import main as gui_main
return gui_main(['calibre'])
csw = os.environ.get('CALIBRE_SIMPLE_WORKER', None)
if csw:
mod, _, func = csw.partition(':')
mod = importlib.import_module(mod)
func = getattr(mod, func)
func()
return
if '--pipe-worker' in sys.argv:
try:
exec (sys.argv[-1])
except Exception:
print('Failed to run pipe worker with command:', sys.argv[-1])
raise
return
address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
resultf = from_hex_unicode(os.environ['CALIBRE_WORKER_RESULT'])
with closing(Client(address, authkey=key)) as conn:
name, args, kwargs, desc = eintr_retry_call(conn.recv)
if desc:
prints(desc)
sys.stdout.flush()
func, notification = get_func(name)
notifier = Progress(conn)
if notification:
kwargs[notification] = notifier
notifier.start()
result = func(*args, **kwargs)
if result is not None and os.path.exists(os.path.dirname(resultf)):
with lopen(resultf, 'wb') as f:
f.write(pickle_dumps(result))
notifier.queue.put(None)
try:
sys.stdout.flush()
except EnvironmentError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
try:
sys.stderr.flush()
except EnvironmentError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
return 0
def __call__(self, module, func, *args, **kwargs):
eintr_retry_call(self.conn.send, (module, func, args, kwargs))
return eintr_retry_call(self.conn.recv)
def get_data(data_conn, data_type, data=None):
eintr_retry_call(data_conn.send, Request(None, data_type, data, None))
result, tb = eintr_retry_call(data_conn.recv)
if tb:
raise DataError(tb)
return result
def unlock_file(fobj):
from calibre.utils.ipc import eintr_retry_call
eintr_retry_call(fcntl.lockf, fobj.fileno(), fcntl.LOCK_UN)
fobj.close()
def send(self, msg):
if self.process is not None and not self.process.stdin.closed:
if not isinstance(msg, bytes):
msg = msg.encode('utf-8')
eintr_retry_call(self.process.stdin.write,
struct.pack(b'>L', len(msg)) + msg)
def lock_tdir(path):
lf = os.path.join(path, TDIR_LOCK)
f = lopen(lf, 'w')
eintr_retry_call(fcntl.lockf, f.fileno(),
fcntl.LOCK_EX | fcntl.LOCK_NB)
return f
def run_main(func):
from multiprocessing.connection import Client
address, key = cPickle.loads(eintr_retry_call(sys.stdin.read))
with closing(Client(address, authkey=key)) as control_conn, closing(
Client(address, authkey=key)) as data_conn:
func(control_conn, data_conn)
def main():
if iswindows:
if '--multiprocessing-fork' in sys.argv:
# We are using the multiprocessing module on windows to launch a
# worker process
from multiprocessing import freeze_support
freeze_support()
return 0
if ismacos and 'CALIBRE_WORKER_FD' not in os.environ and 'CALIBRE_SIMPLE_WORKER' not in os.environ and '--pipe-worker' not in sys.argv:
# On some OS X computers launchd apparently tries to
# launch the last run process from the bundle
# so launch the gui as usual
from calibre.gui2.main import main as gui_main
return gui_main(['calibre'])
niceness = os.environ.pop('CALIBRE_WORKER_NICENESS', None)
if niceness:
try:
os.nice(int(niceness))
except Exception:
pass
csw = os.environ.pop('CALIBRE_SIMPLE_WORKER', None)
if csw:
mod, _, func = csw.partition(':')
mod = importlib.import_module(mod)
func = getattr(mod, func)
func()
return
if '--pipe-worker' in sys.argv:
try:
exec(sys.argv[-1])
except Exception:
print('Failed to run pipe worker with command:', sys.argv[-1])
sys.stdout.flush()
raise
return
fd = int(os.environ['CALIBRE_WORKER_FD'])
resultf = from_hex_unicode(os.environ['CALIBRE_WORKER_RESULT'])
with Connection(fd) as conn:
name, args, kwargs, desc = eintr_retry_call(conn.recv)
if desc:
prints(desc)
sys.stdout.flush()
func, notification = get_func(name)
notifier = Progress(conn)
if notification:
kwargs[notification] = notifier
notifier.start()
result = func(*args, **kwargs)
if result is not None:
os.makedirs(os.path.dirname(resultf), exist_ok=True)
with lopen(resultf, 'wb') as f:
f.write(pickle_dumps(result))
notifier.queue.put(None)
try:
sys.stdout.flush()
except OSError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
try:
sys.stderr.flush()
except OSError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
return 0
def pdftohtml(output_dir, pdf_path, no_images, as_xml=False):
'''
Convert the pdf into html using the pdftohtml app.
This will write the html as index.html into output_dir.
It will also write all extracted images to the output_dir
'''
pdfsrc = os.path.join(output_dir, 'src.pdf')
index = os.path.join(output_dir, 'index.' + ('xml' if as_xml else 'html'))
with lopen(pdf_path, 'rb') as src, lopen(pdfsrc, 'wb') as dest:
shutil.copyfileobj(src, dest)
with CurrentDir(output_dir):
def a(x):
return os.path.basename(x)
exe = PDFTOHTML
cmd = [
exe, '-enc', 'UTF-8', '-noframes', '-p', '-nomerge', '-nodrm',
a(pdfsrc),
a(index)
]
if isbsd:
cmd.remove('-nodrm')
if no_images:
cmd.append('-i')
if as_xml:
cmd.append('-xml')
logf = PersistentTemporaryFile('pdftohtml_log')
try:
p = popen(cmd,
stderr=logf._fd,
stdout=logf._fd,
stdin=subprocess.PIPE)
except OSError as err:
if err.errno == errno.ENOENT:
raise ConversionError(
_('Could not find pdftohtml, check it is in your PATH'))
else:
raise
ret = eintr_retry_call(p.wait)
logf.flush()
logf.close()
out = lopen(logf.name, 'rb').read().decode('utf-8', 'replace').strip()
if ret != 0:
raise ConversionError('pdftohtml failed with return code: %d\n%s' %
(ret, out))
if out:
prints("pdftohtml log:")
prints(out)
if not os.path.exists(index) or os.stat(index).st_size < 100:
raise DRMError()
if not as_xml:
with lopen(index, 'r+b') as i:
raw = i.read().decode('utf-8')
raw = flip_images(raw)
raw = raw.replace(
'<head',
'<!-- created by calibre\'s pdftohtml -->\n <head', 1)
i.seek(0)
i.truncate()
# versions of pdftohtml >= 0.20 output self closing <br> tags, this
# breaks the pdf heuristics regexps, so replace them
raw = raw.replace('<br/>', '<br>')
raw = re.sub(r'<a\s+name=(\d+)',
r'<a id="\1"',
raw,
flags=re.I)
raw = re.sub(r'<a id="(\d+)"', r'<a id="p\1"', raw, flags=re.I)
raw = re.sub(r'<a href="index.html#(\d+)"',
r'<a href="#p\1"',
raw,
flags=re.I)
raw = replace_entities(raw)
raw = raw.replace('\u00a0', ' ')
i.write(raw.encode('utf-8'))
cmd = [
exe, '-f', '1', '-l', '1', '-xml', '-i', '-enc', 'UTF-8',
'-noframes', '-p', '-nomerge', '-nodrm', '-q', '-stdout',
a(pdfsrc)
]
if isbsd:
cmd.remove('-nodrm')
p = popen(cmd, stdout=subprocess.PIPE)
raw = p.stdout.read().strip()
if p.wait() == 0 and raw:
parse_outline(raw, output_dir)
try:
os.remove(pdfsrc)
except:
pass
def cli(port=4444):
prints('Connecting to remote debugger on port %d...' % port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
for i in range(20):
try:
sock.connect(('127.0.0.1', port))
break
except socket.error:
pass
time.sleep(0.1)
else:
try:
sock.connect(('127.0.0.1', port))
except socket.error as err:
prints('Failed to connect to remote debugger:',
err,
file=sys.stderr)
raise SystemExit(1)
prints('Connected to remote process')
import readline
histfile = os.path.join(cache_dir(), 'rpdb.history')
try:
readline.read_history_file(histfile)
except IOError:
pass
atexit.register(readline.write_history_file, histfile)
p = pdb.Pdb()
readline.set_completer(p.complete)
readline.parse_and_bind("tab: complete")
stdin = getattr(sys.stdin, 'buffer', sys.stdin)
stdout = getattr(sys.stdout, 'buffer', sys.stdout)
try:
while True:
recvd = b''
while not recvd.endswith(PROMPT) or select.select(
[sock], [], [], 0) == ([sock], [], []):
buf = eintr_retry_call(sock.recv, 16 * 1024)
if not buf:
return
recvd += buf
recvd = recvd[:-len(PROMPT)]
if recvd.startswith(QUESTION):
recvd = recvd[len(QUESTION):]
stdout.write(recvd)
raw = stdin.readline() or b'n'
else:
stdout.write(recvd)
raw = b''
try:
raw = rinput(PROMPT.decode('utf-8'))
except (EOFError, KeyboardInterrupt):
pass
else:
if not isinstance(raw, bytes):
raw = raw.encode('utf-8')
raw += b'\n'
if not raw:
raw = b'quit\n'
eintr_retry_call(sock.send, raw)
except KeyboardInterrupt:
pass
