global _mt_inited

import mimetypes

mimetypes.init([P('mime.types')])

import mimetypes

if not _mt_inited:

_init_mimetypes()

import mimetypes

if not _mt_inited:

_init_mimetypes()

import mimetypes

if not _mt_inited:

_init_mimetypes()

ext = mimetypes.guess_extension(*args, **kwargs)

if not ext and args and args[0] == 'application/x-palmreader':

ext = '.pdb'

import mimetypes

if not _mt_inited:

_init_mimetypes()

p = os.path

def d(x):

return p.normcase(p.normpath(p.realpath(p.normpath(x))))

if not p1 or not p2:

return False

if isinstance(path, bytes):

path = path.decode(filesystem_encoding)

if abs:

path = os.path.abspath(path)

if ismacos:

import platform

src = platform.mac_ver()[0]

m = re.match(r'(\d+)\.(\d+)\.(\d+)', src)

if m:

'''

Sanitize the filename `name`. All invalid characters are replaced by `substitute`.

The set of invalid characters is the union of the invalid characters in Windows,

macOS and Linux. Also removes leading and trailing whitespace.

**WARNING:** This function also replaces path separators, so only pass file names

and not full paths to it.

'''

if isbytestring(name):

name = name.decode(filesystem_encoding, 'replace')

if isbytestring(substitute):

substitute = substitute.decode(filesystem_encoding, 'replace')

chars = (substitute if c in _filename_sanitize_unicode else c for c in name)

one = ''.join(chars)

one = re.sub(r'\s', ' ', one).strip()

bname, ext = os.path.splitext(one)

one = re.sub(r'^\.+$', '_', bname)

one = one.replace('..', substitute)

one += ext

# Windows doesn't like path components that end with a period or space

if one and one[-1] in ('.', ' '):

one = one[:-1]+'_'

# Names starting with a period are hidden on Unix

if one.startswith('.'):

one = '_' + one[1:]

import logging

if hasenv('CALIBRE_WORKER') and logger.handlers:

return

logger.setLevel(level)

if level == logging.WARNING:

handler = logging.StreamHandler(sys.stdout)

handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))

handler.setLevel(logging.WARNING)

elif level == logging.INFO:

handler = logging.StreamHandler(sys.stdout)

handler.setFormatter(logging.Formatter())

handler.setLevel(logging.INFO)

elif level == logging.DEBUG:

handler = logging.StreamHandler(sys.stderr)

handler.setLevel(logging.DEBUG)

handler.setFormatter(logging.Formatter('[%(levelname)s] %(filename)s:%(lineno)s: %(message)s'))

if iswindows:

return cdll.LoadLibrary(name)

if ismacos:

name += '.dylib'

if hasattr(sys, 'frameworks_dir'):

return cdll.LoadLibrary(os.path.join(getattr(sys, 'frameworks_dir'), name))

return cdll.LoadLibrary(name)

extractor = None

# First use the file header to identify its type

with open(path, 'rb') as f:

id_ = f.read(3)

if id_ == b'Rar':

from calibre.utils.unrar import extract as rarextract

extractor = rarextract

elif id_.startswith(b'PK'):

from calibre.libunzip import extract as zipextract

extractor = zipextract

elif id_.startswith(b'7z'):

from calibre.utils.seven_zip import extract as seven_extract

extractor = seven_extract

if extractor is None:

# Fallback to file extension

ext = os.path.splitext(path)[1][1:].lower()

if ext in ('zip', 'cbz', 'epub', 'oebzip'):

from calibre.libunzip import extract as zipextract

extractor = zipextract

elif ext in ('cbr', 'rar'):

from calibre.utils.unrar import extract as rarextract

extractor = rarextract

elif ext in ('cb7', '7z'):

from calibre.utils.seven_zip import extract as seven_extract

extractor = seven_extract

if extractor is None:

raise Exception('Unknown archive type')

from polyglot.urllib import getproxies

proxies = getproxies()

for key, proxy in list(proxies.items()):

if not proxy or '..' in proxy or key == 'auto':

del proxies[key]

continue

if proxy.startswith(key+'://'):

proxy = proxy[len(key)+3:]

if key == 'https' and proxy.startswith('http://'):

proxy = proxy[7:]

if proxy.endswith('/'):

proxy = proxy[:-1]

if len(proxy) > 4:

proxies[key] = proxy

else:

prints('Removing invalid', key, 'proxy:', proxy)

del proxies[key]

if proxies and debug:

prints('Using proxies:', proxies)

proxies = get_proxies(debug)

proxy = proxies.get(typ, None)

if proxy:

pattern = re.compile((

'(?:ptype://)?'

'(?:(?P<user>\\w+):(?P<pass>.*)@)?'

'(?P<host>[\\w\\-\\.]+)'

'(?::(?P<port>\\d+))?').replace('ptype', typ)

)

match = pattern.match(proxies[typ])

if match:

try:

ans = {

'host' : match.group('host'),

'port' : match.group('port'),

'user' : match.group('user'),

'pass' : match.group('pass')

}

if ans['port']:

ans['port'] = int(ans['port'])

except:

if debug:

import traceback

traceback.print_exc()

else:

if debug:

prints('Using http proxy', str(ans))

'''

Parse all proxy information from a proxy string (as returned by

get_proxies). The returned dict will have members set to None when the info

is not available in the string. If an exception occurs parsing the string

this method returns None.

'''

from polyglot.urllib import urlparse

try:

proxy_url = '%s://%s'%(proxy_scheme, proxy_string)

urlinfo = urlparse(proxy_url)

ans = {

'scheme': urlinfo.scheme,

'hostname': urlinfo.hostname,

'port': urlinfo.port,

'username': urlinfo.username,

'password': urlinfo.password,

}

except Exception:

return None

from calibre.utils.random_ua import common_user_agents, choose_randomly_by_popularity

ua_list = common_user_agents()

ua_list = tuple(x for x in ua_list if not is_mobile_ua(x))

if not allow_ie:

ua_list = tuple(x for x in ua_list if 'Trident/' not in x)

if choose is not None:

return ua_list[choose]

'''

Create a mechanize browser for web scraping. The browser handles cookies,

refresh requests and ignores robots.txt. Also uses proxy if available.

:param honor_time: If True honors pause time in refresh requests

:param max_time: Maximum time in seconds to wait during a refresh request

:param verify_ssl_certificates: If false SSL certificates errors are ignored

'''

from calibre.utils.browser import Browser

opener = Browser(verify_ssl=verify_ssl_certificates)

opener.set_handle_refresh(handle_refresh, max_time=max_time, honor_time=honor_time)

opener.set_handle_robots(False)

if user_agent is None:

user_agent = random_user_agent(0, allow_ie=False)

elif user_agent == 'common_words/based':

from calibre.utils.random_ua import common_english_word_ua

user_agent = common_english_word_ua()

opener.addheaders = [('User-agent', user_agent)]

proxies = get_proxies()

to_add = {}

http_proxy = proxies.get('http', None)

if http_proxy:

to_add['http'] = http_proxy

https_proxy = proxies.get('https', None)

if https_proxy:

to_add['https'] = https_proxy

if to_add:

opener.set_proxies(to_add)

'''

Fit image in box of width pwidth and height pheight.

@param width: Width of image

@param height: Height of image

@param pwidth: Width of box

@param pheight: Height of box

@return: scaled, new_width, new_height. scaled is True iff new_width and/or new_height is different from width or height.

'''

if height < 1 or width < 1:

return False, int(width), int(height)

scaled = height > pheight or width > pwidth

if height > pheight:

corrf = pheight / float(height)

width, height = floor(corrf*width), pheight

if width > pwidth:

corrf = pwidth / float(width)

width, height = pwidth, floor(corrf*height)

if height > pheight:

corrf = pheight / float(height)

width, height = floor(corrf*width), pheight

def __init__(self, path):

self.path = path

self.cwd = None

def __enter__(self, *args):

self.cwd = os.getcwd()

os.chdir(self.path)

return self.cwd

def __exit__(self, *args):

try:

os.chdir(self.cwd)

except OSError:

# The previous CWD no longer exists

global _ncpus

if _ncpus is None:

_ncpus = max(1, os.cpu_count() or 1)

''' A nice interface to os.walk '''

for record in os.walk(dir):

for f in record[-1]:

''' A version of strftime that returns unicode strings and tries to handle dates

before 1900 '''

if not fmt:

return ''

if t is None:

t = time.localtime()

if hasattr(t, 'timetuple'):

t = t.timetuple()

early_year = t[0] < 1900

if early_year:

replacement = 1900 if t[0]%4 == 0 else 1901

fmt = fmt.replace('%Y', '_early year hack##')

t = list(t)

orig_year = t[0]

t[0] = replacement

t = time.struct_time(t)

ans = None

if isinstance(fmt, bytes):

fmt = fmt.decode('mbcs' if iswindows else 'utf-8', 'replace')

ans = time.strftime(fmt, t)

if early_year:

ans = ans.replace('_early year hack##', str(orig_year))

try:

return safe_chr(num)

except (ValueError, OverflowError):

result_exceptions={}):

'''

:param match: A match object such that '&'+match.group(1)';' is the entity.

:param exceptions: A list of entities to not convert (Each entry is the name of the entity, e.g. 'apos' or '#1234'

:param encoding: The encoding to use to decode numeric entities between 128 and 256.

If None, the Unicode UCS encoding is used. A common encoding is cp1252.

:param result_exceptions: A mapping of characters to entities. If the result

is in result_exceptions, result_exception[result] is returned instead.

Convenient way to specify exception for things like < or > that can be

specified by various actual entities.

'''

def check(ch):

return result_exceptions.get(ch, ch)

ent = match.group(1)

if ent in exceptions:

return '&'+ent+';'

if ent in {'apos', 'squot'}: # squot is generated by some broken CMS software

return check("'")

if ent == 'hellips':

ent = 'hellip'

if ent.startswith('#'):

try:

if ent[1] in ('x', 'X'):

num = int(ent[2:], 16)

else:

num = int(ent[1:])

except:

return '&'+ent+';'

if encoding is None or num > 255:

return check(my_unichr(num))

try:

return check(bytes(bytearray((num,))).decode(encoding))

except UnicodeDecodeError:

return check(my_unichr(num))

from calibre.ebooks.html_entities import html5_entities

try:

return check(html5_entities[ent])

except KeyError:

pass

from polyglot.html_entities import name2codepoint

try:

return check(my_unichr(name2codepoint[ent]))

except KeyError:

'"' : '&quot;',

"'" : '&apos;',

'<' : '&lt;',

'>' : '&gt;',

raw = _ent_pat.sub(entity_to_unicode, raw)

raw = raw.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')

if attribute:

raw = raw.replace('"', '&quot;').replace("'", '&apos;')

if isbytestring(obj):

try:

obj = obj.decode(enc)

except Exception:

try:

obj = obj.decode(filesystem_encoding if enc ==

preferred_encoding else preferred_encoding)

except Exception:

try:

obj = obj.decode('utf-8')

except Exception:

obj = repr(obj)

if isbytestring(obj):

obj = obj.decode('utf-8')

if not isbytestring(obj):

try:

obj = str(obj)

except Exception:

try:

obj = native_string_type(obj)

except Exception:

obj = repr(obj)

'''

Removes redundant /'s from url's.

'''

""" Convert a size in bytes into a human readable form """

divisor, suffix = 1, "B"

for i, candidate in enumerate(('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')):

if size < (1 << ((i + 1) * 10)):

divisor, suffix = (1 << (i * 10)), candidate

break

size = str(float(size)/divisor)

if size.find(".") > -1:

size = size[:size.find(".")+2]

if size.endswith('.0'):

size = size[:-2]

fileobj.flush()

os.fsync(fileobj.fileno())

if islinux and getattr(fileobj, 'name', None):

# On Linux kernels after 5.1.9 and 4.19.50 using fsync without any

# following activity causes Kindles to eject. Instead of fixing this in

# the obvious way, which is to have the kernel send some harmless

# filesystem activity after the FSYNC, the kernel developers seem to

# think the correct solution is to disable FSYNC using a mount flag

# which users will have to turn on manually. So instead we create some

# harmless filesystem activity, and who cares about performance.

# See https://bugs.launchpad.net/calibre/+bug/1834641

# and https://bugzilla.kernel.org/show_bug.cgi?id=203973

# To check for the existence of the bug, simply run:

# python -c "p = '/run/media/kovid/Kindle/driveinfo.calibre'; f = open(p, 'r+b'); os.fsync(f.fileno());"

# this will cause the Kindle to disconnect.

try:

os.utime(fileobj.name, None)

except Exception:

import traceback