def env(self): env = os.environ.copy() env[native_string_type('CALIBRE_WORKER')] = environ_item('1') td = as_hex_unicode(msgpack_dumps(base_dir())) env[native_string_type('CALIBRE_WORKER_TEMP_DIR')] = environ_item(td) env.update(self._env) return env
def create_listener(authkey, backlog=4): prefix = os.path.join(base_dir(), 'ipc-socket-%d-%%d' % os.getpid()) max_tries = 20 while max_tries > 0: max_tries -= 1 address = prefix % next(_name_counter) if not isinstance(address, bytes): address = address.encode('utf-8') # multiprocessing needs bytes in python 2 try: return address, Listener(address=address, authkey=authkey, backlog=backlog) except EnvironmentError as err: if max_tries < 1: raise if err.errno == errno.ENOENT: # Some OS X machines have software that deletes temp # files/dirs after prolonged inactivity. See for # example, https://bugs.launchpad.net/bugs/1541356 try: os.makedirs(os.path.dirname(prefix)) except EnvironmentError as e: if e.errno != errno.EEXIST: raise continue if err.errno != errno.EADDRINUSE: raise
def launch_worker(self, gui=False, redirect_output=None, job_name=None): start = time.monotonic() id = next(self.launched_worker_counter) fd, rfile = tempfile.mkstemp(prefix='ipc_result_%d_%d_' % (self.id, id), dir=base_dir(), suffix='.pickle') os.close(fd) if redirect_output is None: redirect_output = not gui env = { 'CALIBRE_WORKER_ADDRESS': environ_item(as_hex_unicode(msgpack_dumps(self.address))), 'CALIBRE_WORKER_KEY': environ_item(as_hex_unicode(self.auth_key)), 'CALIBRE_WORKER_RESULT': environ_item(as_hex_unicode(rfile)), } cw = self.do_launch(env, gui, redirect_output, rfile, job_name=job_name) if isinstance(cw, string_or_bytes): raise CriticalError('Failed to launch worker process:\n' + force_unicode(cw)) if DEBUG: print( 'Worker Launch took: {:.2f} seconds'.format(time.monotonic() - start)) return cw
def launch_worker(self, gui=False, redirect_output=None, job_name=None): start = time.time() with self._worker_launch_lock: self.launched_worker_count += 1 id = self.launched_worker_count fd, rfile = tempfile.mkstemp(prefix=u'ipc_result_%d_%d_' % (self.id, id), dir=base_dir(), suffix=u'.pickle') os.close(fd) if redirect_output is None: redirect_output = not gui env = { 'CALIBRE_WORKER_ADDRESS': environ_item(hexlify(msgpack_dumps(self.listener.address))), 'CALIBRE_WORKER_KEY': environ_item(hexlify(self.auth_key)), 'CALIBRE_WORKER_RESULT': environ_item(hexlify(rfile.encode('utf-8'))), } cw = self.do_launch(env, gui, redirect_output, rfile, job_name=job_name) if isinstance(cw, string_or_bytes): raise CriticalError('Failed to launch worker process:\n' + cw) if DEBUG: print('Worker Launch took:', time.time() - start) return cw
def create_listener(authkey, backlog=4): prefix = os.path.join(base_dir(), 'ipc-socket-%d-%%d' % os.getpid()) max_tries = 20 while max_tries > 0: max_tries -= 1 address = prefix % next(_name_counter) try: return address, Listener(address=address, authkey=authkey, backlog=backlog) except EnvironmentError as err: if max_tries < 1: raise if err.errno == errno.ENOENT: # Some OS X machines have software that deletes temp # files/dirs after prolonged inactivity. See for # example, https://bugs.launchpad.net/bugs/1541356 try: os.makedirs(os.path.dirname(prefix)) except EnvironmentError as e: if e.errno != errno.EEXIST: raise continue if err.errno != errno.EADDRINUSE: raise
def env(self): if ispy3: env = os.environ.copy() else: # We use this inefficient method of copying the environment variables # because of non ascii env vars on windows. See https://bugs.launchpad.net/bugs/811191 env = {} for key in os.environ: try: val = os.environ[key] if isinstance(val, unicode_type): # On windows subprocess cannot handle unicode env vars try: val = val.encode(filesystem_encoding) except ValueError: val = val.encode('utf-8') if isinstance(key, unicode_type): key = key.encode('ascii') env[key] = val except: pass env[native_string_type('CALIBRE_WORKER')] = environ_item('1') td = as_hex_unicode(msgpack_dumps(base_dir())) env[native_string_type('CALIBRE_WORKER_TEMP_DIR')] = environ_item(td) env.update(self._env) return env
def consolidate_log(self): logs = [self.log.html, self.log.plain_text] bdir = base_dir() log_dir = os.path.join(bdir, 'threaded_job_logs') if not os.path.exists(log_dir): os.makedirs(log_dir) fd, path = tempfile.mkstemp(suffix='.json', prefix='log-', dir=log_dir) with os.fdopen(fd, 'wb') as f: f.write( json.dumps(logs, ensure_ascii=False, indent=2).encode('utf-8')) self.consolidated_log = path self.log = None
def consolidate_log(self): logs = [self.log.html, self.log.plain_text] bdir = base_dir() log_dir = os.path.join(bdir, 'threaded_job_logs') if not os.path.exists(log_dir): os.makedirs(log_dir) fd, path = tempfile.mkstemp(suffix='.json', prefix='log-', dir=log_dir) with os.fdopen(fd, 'wb') as f: f.write(json.dumps(logs, ensure_ascii=False, indent=2).encode('utf-8')) self.consolidated_log = path self.log = None
def launch_worker(self, gui=False, redirect_output=None, job_name=None): start = time.monotonic() id = next(self.launched_worker_counter) fd, rfile = tempfile.mkstemp(prefix='ipc_result_%d_%d_' % (self.id, id), dir=base_dir(), suffix='.pickle') os.close(fd) if redirect_output is None: redirect_output = not gui cw = self.do_launch(gui, redirect_output, rfile, job_name=job_name) if isinstance(cw, string_or_bytes): raise CriticalError('Failed to launch worker process:\n' + force_unicode(cw)) if DEBUG: print( f'Worker Launch took: {time.monotonic() - start:.2f} seconds') return cw
def launch_worker(self, gui=False, redirect_output=None, job_name=None): start = time.time() with self._worker_launch_lock: self.launched_worker_count += 1 id = self.launched_worker_count fd, rfile = tempfile.mkstemp(prefix=u'ipc_result_%d_%d_'%(self.id, id), dir=base_dir(), suffix=u'.pickle') os.close(fd) if redirect_output is None: redirect_output = not gui env = { 'CALIBRE_WORKER_ADDRESS' : environ_item(as_hex_unicode(msgpack_dumps(self.address))), 'CALIBRE_WORKER_KEY' : environ_item(as_hex_unicode(self.auth_key)), 'CALIBRE_WORKER_RESULT' : environ_item(as_hex_unicode(rfile)), } cw = self.do_launch(env, gui, redirect_output, rfile, job_name=job_name) if isinstance(cw, string_or_bytes): raise CriticalError('Failed to launch worker process:\n'+cw) if DEBUG: print('Worker Launch took:', time.time() - start) return cw
def launch_worker(self, gui=False, redirect_output=None, job_name=None): start = time.time() with self._worker_launch_lock: self.launched_worker_count += 1 id = self.launched_worker_count fd, rfile = tempfile.mkstemp(prefix=u'ipc_result_%d_%d_'%(self.id, id), dir=base_dir(), suffix=u'.pickle') os.close(fd) if redirect_output is None: redirect_output = not gui env = { 'CALIBRE_WORKER_ADDRESS' : hexlify(cPickle.dumps(self.listener.address, -1)), 'CALIBRE_WORKER_KEY' : hexlify(self.auth_key), 'CALIBRE_WORKER_RESULT' : hexlify(rfile.encode('utf-8')), } cw = self.do_launch(env, gui, redirect_output, rfile, job_name=job_name) if isinstance(cw, basestring): raise CriticalError('Failed to launch worker process:\n'+cw) if DEBUG: print 'Worker Launch took:', time.time() - start return cw
def env(self): # We use this inefficient method of copying the environment variables # because of non ascii env vars on windows. See https://bugs.launchpad.net/bugs/811191 env = {} for key in os.environ: try: val = os.environ[key] if isinstance(val, unicode): # On windows subprocess cannot handle unicode env vars try: val = val.encode(filesystem_encoding) except ValueError: val = val.encode('utf-8') if isinstance(key, unicode): key = key.encode('ascii') env[key] = val except: pass env[b'CALIBRE_WORKER'] = b'1' td = binascii.hexlify(cPickle.dumps(base_dir())) env[b'CALIBRE_WORKER_TEMP_DIR'] = bytes(td) env.update(self._env) return env
_run_once = True ################################################################################ # Platform specific modules if iswindows: winutil, winutilerror = plugins['winutil'] if not winutil: raise RuntimeError('Failed to load the winutil plugin: %s' % winutilerror) if len(sys.argv) > 1 and not isinstance(sys.argv[1], unicode): sys.argv[1:] = winutil.argv()[1 - len(sys.argv):] ################################################################################ # Ensure that all temp files/dirs are created under a calibre tmp dir from calibre.ptempfile import base_dir base_dir() ################################################################################ # Convert command line arguments to unicode enc = preferred_encoding if isosx: # Newer versions of OS X seem to use UTF-8 try: [x.decode('utf-8') for x in sys.argv[1:]] enc = 'utf-8' except: pass for i in range(1, len(sys.argv)): if not isinstance(sys.argv[i], unicode): sys.argv[i] = sys.argv[i].decode(enc, 'replace')
sys.meta_path.insert(0, PyQt4Ban()) # # Platform specific modules if iswindows: winutil, winutilerror = plugins['winutil'] if not winutil: raise RuntimeError('Failed to load the winutil plugin: %s'%winutilerror) if len(sys.argv) > 1 and not isinstance(sys.argv[1], unicode): sys.argv[1:] = winutil.argv()[1-len(sys.argv):] # # Ensure that all temp files/dirs are created under a calibre tmp dir from calibre.ptempfile import base_dir base_dir() # # Convert command line arguments to unicode enc = preferred_encoding if isosx: # Newer versions of OS X seem to use UTF-8 try: [x.decode('utf-8') for x in sys.argv[1:]] enc = 'utf-8' except: pass for i in range(1, len(sys.argv)): if not isinstance(sys.argv[i], unicode): sys.argv[i] = sys.argv[i].decode(enc, 'replace')
def pdftohtml_extract_pdf_issn(pdf_path): output_dir= base_dir() output_dir = output_dir.replace(os.sep,'/') if DEBUG: print("Current output working directory ( output_dir= base_dir() ) for temporary files is: ", output_dir) pdftohtml(output_dir, pdf_path, no_images=True, as_xml = False) return output_dir
def format(self, book_id, fmt, as_file=False, as_path=False, preserve_filename=False): ''' Return the ebook format as a bytestring or `None` if the format doesn't exist, or we don't have permission to write to the ebook file. :param as_file: If True the ebook format is returned as a file object. Note that the file object is a SpooledTemporaryFile, so if what you want to do is copy the format to another file, use :method:`copy_format_to` instead for performance. :param as_path: Copies the format file to a temp file and returns the path to the temp file :param preserve_filename: If True and returning a path the filename is the same as that used in the library. Note that using this means that repeated calls yield the same temp file (which is re-created each time) ''' ext = ('.' + fmt.lower()) if fmt else '' if as_path: if preserve_filename: with self.read_lock: try: fname = self.fields['formats'].format_fname( book_id, fmt) except: return None fname += ext bd = base_dir() d = os.path.join(bd, 'format_abspath') try: os.makedirs(d) except: pass ret = os.path.join(d, fname) try: self.copy_format_to(book_id, fmt, ret) except NoSuchFormat: return None else: with PersistentTemporaryFile(ext) as pt: try: self.copy_format_to(book_id, fmt, pt) except NoSuchFormat: return None ret = pt.name elif as_file: with self.read_lock: try: fname = self.fields['formats'].format_fname(book_id, fmt) except: return None fname += ext ret = SpooledTemporaryFile(SPOOL_SIZE) try: self.copy_format_to(book_id, fmt, ret) except NoSuchFormat: return None ret.seek(0) # Various bits of code try to use the name as the default # title when reading metadata, so set it ret.name = fname else: buf = BytesIO() try: self.copy_format_to(book_id, fmt, buf) except NoSuchFormat: return None ret = buf.getvalue() return ret
def initialize_calibre(): if hasattr(initialize_calibre, 'initialized'): return initialize_calibre.initialized = True # Ensure that all temp files/dirs are created under a calibre tmp dir from calibre.ptempfile import base_dir try: base_dir() except OSError: pass # Ignore this error during startup, so we can show a better error message to the user later. # # Ensure that the max number of open files is at least 1024 if iswindows: # See https://msdn.microsoft.com/en-us/library/6e3b887c.aspx from calibre_extensions import winutil winutil.setmaxstdio(max(1024, winutil.getmaxstdio())) else: import resource soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) if soft < 1024: try: resource.setrlimit(resource.RLIMIT_NOFILE, (min(1024, hard), hard)) except Exception: if DEBUG: import traceback traceback.print_exc() # # Fix multiprocessing from multiprocessing import spawn, util def get_command_line(**kwds): prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)' prog %= ', '.join('%s=%r' % item for item in kwds.items()) return get_debug_executable() + ['--fix-multiprocessing', '--', prog] spawn.get_command_line = get_command_line orig_spawn_passfds = util.spawnv_passfds def spawnv_passfds(path, args, passfds): try: idx = args.index('-c') except ValueError: return orig_spawn_passfds(args[0], args, passfds) patched_args = get_debug_executable() + [ '--fix-multiprocessing', '--' ] + args[idx + 1:] return orig_spawn_passfds(patched_args[0], patched_args, passfds) util.spawnv_passfds = spawnv_passfds # # Setup resources import calibre.utils.resources as resources resources # # Setup translations from calibre.utils.localization import set_translators set_translators() # # Initialize locale # Import string as we do not want locale specific # string.whitespace/printable, on windows especially, this causes problems. # Before the delay load optimizations, string was loaded before this point # anyway, so we preserve the old behavior explicitly. import string string try: locale.setlocale(locale.LC_ALL, '') # set the locale to the user's default locale except: dl = locale.getdefaultlocale() try: if dl: locale.setlocale(locale.LC_ALL, dl[0]) except: pass builtins.__dict__['lopen'] = open # legacy compatibility from calibre.utils.icu import title_case, lower as icu_lower, upper as icu_upper builtins.__dict__['icu_lower'] = icu_lower builtins.__dict__['icu_upper'] = icu_upper builtins.__dict__['icu_title'] = title_case def connect_lambda(bound_signal, self, func, **kw): import weakref r = weakref.ref(self) del self num_args = func.__code__.co_argcount - 1 if num_args < 0: raise TypeError('lambda must take at least one argument') def slot(*args): ctx = r() if ctx is not None: if len(args) != num_args: args = args[:num_args] func(ctx, *args) bound_signal.connect(slot, **kw) builtins.__dict__['connect_lambda'] = connect_lambda if islinux or ismacos or isfreebsd: # Name all threads at the OS level created using the threading module, see # http://bugs.python.org/issue15500 import threading from calibre_extensions import speedup orig_start = threading.Thread.start def new_start(self): orig_start(self) try: name = self.name if not name or name.startswith('Thread-'): name = self.__class__.__name__ if name == 'Thread': name = self.name if name: if isinstance(name, str): name = name.encode('ascii', 'replace').decode('ascii') speedup.set_thread_name(name[:15]) except Exception: pass # Don't care about failure to set name threading.Thread.start = new_start
def format(self, book_id, fmt, as_file=False, as_path=False, preserve_filename=False): ''' Return the ebook format as a bytestring or `None` if the format doesn't exist, or we don't have permission to write to the ebook file. :param as_file: If True the ebook format is returned as a file object. Note that the file object is a SpooledTemporaryFile, so if what you want to do is copy the format to another file, use :method:`copy_format_to` instead for performance. :param as_path: Copies the format file to a temp file and returns the path to the temp file :param preserve_filename: If True and returning a path the filename is the same as that used in the library. Note that using this means that repeated calls yield the same temp file (which is re-created each time) ''' ext = ('.'+fmt.lower()) if fmt else '' if as_path: if preserve_filename: with self.read_lock: try: fname = self.fields['formats'].format_fname(book_id, fmt) except: return None fname += ext bd = base_dir() d = os.path.join(bd, 'format_abspath') try: os.makedirs(d) except: pass ret = os.path.join(d, fname) try: self.copy_format_to(book_id, fmt, ret) except NoSuchFormat: return None else: with PersistentTemporaryFile(ext) as pt: try: self.copy_format_to(book_id, fmt, pt) except NoSuchFormat: return None ret = pt.name elif as_file: with self.read_lock: try: fname = self.fields['formats'].format_fname(book_id, fmt) except: return None fname += ext ret = SpooledTemporaryFile(SPOOL_SIZE) try: self.copy_format_to(book_id, fmt, ret) except NoSuchFormat: return None ret.seek(0) # Various bits of code try to use the name as the default # title when reading metadata, so set it ret.name = fname else: buf = BytesIO() try: self.copy_format_to(book_id, fmt, buf) except NoSuchFormat: return None ret = buf.getvalue() return ret
def __init__(self, args, force_calibre_style=False, override_program_name=None, headless=False, color_prefs=gprefs): self.file_event_hook = None if override_program_name: args = [override_program_name] + args[1:] if headless: if not args: args = sys.argv[:1] args.extend(['-platformpluginpath', sys.extensions_location, '-platform', 'headless']) self.headless = headless qargs = [i.encode('utf-8') if isinstance(i, unicode) else i for i in args] self.pi = plugins['progress_indicator'][0] if not isosx and not headless: # On OS X high dpi scaling is turned on automatically by the OS, so we dont need to set it explicitly setup_hidpi() QApplication.setOrganizationName('calibre-ebook.com') QApplication.setOrganizationDomain(QApplication.organizationName()) QApplication.setApplicationVersion(__version__) QApplication.setApplicationName(APP_UID) QApplication.__init__(self, qargs) self.setAttribute(Qt.AA_UseHighDpiPixmaps) self.setAttribute(Qt.AA_SynthesizeTouchForUnhandledMouseEvents, False) try: base_dir() except EnvironmentError as err: if not headless: show_temp_dir_error(err) raise SystemExit('Failed to create temporary directory') if DEBUG and not headless: prints('devicePixelRatio:', self.devicePixelRatio()) s = self.primaryScreen() if s: prints('logicalDpi:', s.logicalDotsPerInchX(), 'x', s.logicalDotsPerInchY()) prints('physicalDpi:', s.physicalDotsPerInchX(), 'x', s.physicalDotsPerInchY()) if not iswindows: self.setup_unix_signals() if islinux or isbsd: self.setAttribute(Qt.AA_DontUseNativeMenuBar, 'CALIBRE_NO_NATIVE_MENUBAR' in os.environ) self.setup_styles(force_calibre_style) self.setup_ui_font() if not self.using_calibre_style and self.style().objectName() == 'fusion': # Since Qt is using the fusion style anyway, specialize it self.load_calibre_style() fi = gprefs['font'] if fi is not None: font = QFont(*(fi[:4])) s = gprefs.get('font_stretch', None) if s is not None: font.setStretch(s) QApplication.setFont(font) self.line_height = max(12, QFontMetrics(self.font()).lineSpacing()) dl = QLocale(get_lang()) if unicode(dl.bcp47Name()) != u'C': QLocale.setDefault(dl) global gui_thread, qt_app gui_thread = QThread.currentThread() self._translator = None self.load_translations() qt_app = self self._file_open_paths = [] self._file_open_lock = RLock() if not isosx: # OS X uses a native color dialog that does not support custom # colors self.color_prefs = color_prefs self.read_custom_colors() self.lastWindowClosed.connect(self.save_custom_colors) if isxp: error_dialog(None, _('Windows XP not supported'), '<p>' + _( 'calibre versions newer than 2.0 do not run on Windows XP. This is' ' because the graphics toolkit calibre uses (Qt 5) crashes a lot' ' on Windows XP. We suggest you stay with <a href="%s">calibre 1.48</a>' ' which works well on Windows XP.') % 'http://download.calibre-ebook.com/1.48.0/', show=True) raise SystemExit(1) if iswindows: # On windows the highlighted colors for inactive widgets are the # same as non highlighted colors. This is a regression from Qt 4. # https://bugreports.qt-project.org/browse/QTBUG-41060 p = self.palette() for role in (p.Highlight, p.HighlightedText, p.Base, p.AlternateBase): p.setColor(p.Inactive, role, p.color(p.Active, role)) self.setPalette(p) # Prevent text copied to the clipboard from being lost on quit due to # Qt 5 bug: https://bugreports.qt-project.org/browse/QTBUG-41125 self.aboutToQuit.connect(self.flush_clipboard)