def collect_data(self, ids, tdir):
from calibre.ebooks.metadata.opf2 import metadata_to_opf
data = {}
for i in set(ids):
mi = self.db.get_metadata(i, index_is_id=True, get_cover=True,
cover_as_data=True)
opf = metadata_to_opf(mi)
if isbytestring(opf):
opf = opf.decode('utf-8')
cpath = None
if mi.cover_data and mi.cover_data[1]:
cpath = os.path.join(tdir, 'cover_%s.jpg'%i)
with lopen(cpath, 'wb') as f:
f.write(mi.cover_data[1])
if isbytestring(cpath):
cpath = cpath.decode(filesystem_encoding)
formats = {}
if mi.formats:
for fmt in mi.formats:
fpath = os.path.join(tdir, 'fmt_%s.%s'%(i, fmt.lower()))
with lopen(fpath, 'wb') as f:
try:
self.db.copy_format_to(i, fmt, f, index_is_id=True)
except NoSuchFormat:
continue
else:
if isbytestring(fpath):
fpath = fpath.decode(filesystem_encoding)
formats[fmt.lower()] = fpath
data[i] = [opf, cpath, formats, mi.last_modified.isoformat()]
return data
def pynocase(one, two, encoding='utf-8'):
if isbytestring(one):
try:
one = one.decode(encoding, 'replace')
except:
pass
if isbytestring(two):
try:
two = two.decode(encoding, 'replace')
except:
pass
return cmp(one.lower(), two.lower())
def __init__(self, path):
self.handle_map = {}
import win32file, winerror
from pywintypes import error
from collections import defaultdict
if isbytestring(path):
path = path.decode(filesystem_encoding)
if not os.path.exists(path):
return
names = os.listdir(path)
name_to_fileid = {x:windows_get_fileid(os.path.join(path, x)) for x in names}
fileid_to_names = defaultdict(set)
for name, fileid in name_to_fileid.iteritems():
fileid_to_names[fileid].add(name)
for x in names:
f = os.path.normcase(os.path.abspath(os.path.join(path, x)))
if not os.path.isfile(f):
continue
try:
# Ensure the file is not read-only
win32file.SetFileAttributes(f, win32file.FILE_ATTRIBUTE_NORMAL)
except:
pass
try:
h = win32file.CreateFile(f, win32file.GENERIC_READ,
win32file.FILE_SHARE_DELETE, None,
win32file.OPEN_EXISTING, win32file.FILE_FLAG_SEQUENTIAL_SCAN, 0)
except error as e:
if getattr(e, 'winerror', 0) == winerror.ERROR_SHARING_VIOLATION:
# The file could be a hardlink to an already opened file,
# in which case we use the same handle for both files
fileid = name_to_fileid[x]
found = False
if fileid is not None:
for other in fileid_to_names[fileid]:
other = os.path.normcase(os.path.abspath(os.path.join(path, other)))
if other in self.handle_map:
self.handle_map[f] = self.handle_map[other]
found = True
break
if found:
continue
self.close_handles()
if getattr(e, 'winerror', 0) == winerror.ERROR_SHARING_VIOLATION:
err = IOError(errno.EACCES,
_('File is open in another process'))
err.filename = f
raise err
raise
except:
self.close_handles()
raise
self.handle_map[f] = h
def edit_file(self, name, syntax, use_template=None):
editor = editors.get(name, None)
if editor is None:
if use_template is None:
data = current_container().raw_data(name)
if isbytestring(data) and syntax in {
'html', 'css', 'text', 'xml'
}:
try:
data = data.decode('utf-8')
except UnicodeDecodeError:
return error_dialog(
self.gui,
_('Cannot decode'),
_('Cannot edit %s as it appears to be in an unknown character encoding'
) % name,
show=True)
else:
data = use_template
editor = editors[name] = editor_from_syntax(
syntax, self.gui.editor_tabs)
self.init_editor(name,
editor,
data,
use_template=bool(use_template))
self.show_editor(name)
return editor
def __init__(self, path):
self.handle_map = {}
import win32file, winerror
from pywintypes import error
from collections import defaultdict
if isbytestring(path):
path = path.decode(filesystem_encoding)
if not os.path.exists(path):
return
names = os.listdir(path)
name_to_fileid = {x:windows_get_fileid(os.path.join(path, x)) for x in names}
fileid_to_names = defaultdict(set)
for name, fileid in name_to_fileid.iteritems():
fileid_to_names[fileid].add(name)
for x in names:
f = os.path.normcase(os.path.abspath(os.path.join(path, x)))
if not os.path.isfile(f):
continue
try:
# Ensure the file is not read-only
win32file.SetFileAttributes(f, win32file.FILE_ATTRIBUTE_NORMAL)
except:
pass
try:
h = win32file.CreateFile(f, win32file.GENERIC_READ,
win32file.FILE_SHARE_DELETE, None,
win32file.OPEN_EXISTING, win32file.FILE_FLAG_SEQUENTIAL_SCAN, 0)
except error as e:
if getattr(e, 'winerror', 0) == winerror.ERROR_SHARING_VIOLATION:
# The file could be a hardlink to an already opened file,
# in which case we use the same handle for both files
fileid = name_to_fileid[x]
found = False
if fileid is not None:
for other in fileid_to_names[fileid]:
other = os.path.normcase(os.path.abspath(os.path.join(path, other)))
if other in self.handle_map:
self.handle_map[f] = self.handle_map[other]
found = True
break
if found:
continue
self.close_handles()
if getattr(e, 'winerror', 0) == winerror.ERROR_SHARING_VIOLATION:
err = IOError(errno.EACCES,
_('File is open in another process'))
err.filename = f
raise err
raise
except:
self.close_handles()
raise
self.handle_map[f] = h
def ajax_search(self, query='', sort='title', offset=0, num=25,
sort_order='asc'):
'''
Return the books (as list of ids) matching the specified search query.
'''
try:
num = int(num)
except:
raise cherrypy.HTTPError(404, "Invalid num: %r"%num)
try:
offset = int(offset)
except:
raise cherrypy.HTTPError(404, "Invalid offset: %r"%offset)
sfield = self.db.data.sanitize_sort_field_name(sort)
if sfield not in self.db.field_metadata.sortable_field_keys():
raise cherrypy.HTTPError(404, '%s is not a valid sort field'%sort)
if isbytestring(query):
query = query.decode('UTF-8')
ids = list(self.search_for_books(query))
self.db.data.multisort(fields=[(sfield, sort_order == 'asc')], subsort=True,
only_ids=ids)
total_num = len(ids)
ids = ids[offset:offset+num]
return {
'total_num': total_num, 'sort_order':sort_order,
'offset':offset, 'num':len(ids), 'sort':sort,
'base_url':absurl(self.opts.url_prefix, '/ajax/search'),
'query': query,
'book_ids':ids
}
def create_mail2(from_, to, subject, text=None, attachment_data=None, attachment_type=None, attachment_name=None):
assert text or attachment_data
from email.mime.multipart import MIMEMultipart
from email.utils import formatdate
from email import encoders
mail = MIMEMultipart()
mail['Subject'] = subject
mail['To'] = to
mail['From'] = from_
mail['Date'] = formatdate(localtime=True)
mail.preamble = 'You will not see this in a MIME-aware mail reader.\n'
if text is not None:
if isbytestring(text):
msg = MIMEText(text)
else:
msg = MIMEText(text, 'plain', 'utf-8')
mail.attach(msg)
if attachment_data is not None:
assert attachment_data and attachment_name
name = Header(attachment_name, 'utf-8').encode()
subtype = 'octet-stream;\n\tcharset="utf-8";\n\tname=%s' % name
msg = MIMEBase('application', subtype)
msg.set_payload(attachment_data)
encoders.encode_base64(msg)
msg.add_header('Content-Disposition', 'attachment',
filename=Header(attachment_name, 'utf-8').encode())
mail.attach(msg)
return mail.as_string()
def split_txt(txt, epub_split_size_kb=0):
'''
Ensure there are split points for converting
to EPUB. A mis-detected paragraph type can
result in the entire document being one giant
paragraph. In this case the EPUB parser will not
be able to determine where to split the file
to accommodate the EPUB file size limitation
and will fail.
'''
# Takes care if there is no point to split
if epub_split_size_kb > 0:
if isinstance(txt, unicode_type):
txt = txt.encode('utf-8')
if len(txt) > epub_split_size_kb * 1024:
chunk_size = max(16, epub_split_size_kb - 32) * 1024
# if there are chunks with a superior size then go and break
parts = txt.split(b'\n\n')
if parts and max(map(len, parts)) > chunk_size:
txt = b'\n\n'.join(
split_string_separator(line, chunk_size) for line in parts
)
if isbytestring(txt):
txt = txt.decode('utf-8')
return txt
def __init__(self, library_path, db):
if isbytestring(library_path):
library_path = library_path.decode(filesystem_encoding)
self.src_library_path = os.path.abspath(library_path)
self.db = db
self.is_case_sensitive = db.is_case_sensitive
self.all_authors = frozenset([x[1] for x in db.all_authors()])
self.all_ids = frozenset([id_ for id_ in db.all_ids()])
self.all_dbpaths = frozenset(self.dbpath(id_) for id_ in self.all_ids)
self.all_lc_dbpaths = frozenset([f.lower() for f in self.all_dbpaths])
self.db_id_regexp = re.compile(r'^.* \((\d+)\)$')
self.dirs = []
self.book_dirs = []
self.potential_authors = {}
self.invalid_authors = []
self.extra_authors = []
self.invalid_titles = []
self.extra_titles = []
self.unknown_book_files = []
self.missing_formats = []
self.extra_formats = []
self.extra_files = []
self.missing_covers = []
self.extra_covers = []
self.failed_folders = []
def __init__(self, library_path, db):
if isbytestring(library_path):
library_path = library_path.decode(filesystem_encoding)
self.src_library_path = os.path.abspath(library_path)
self.db = db
self.is_case_sensitive = db.is_case_sensitive
self.all_authors = frozenset([x[1] for x in db.all_authors()])
self.all_ids = frozenset([id_ for id_ in db.all_ids()])
self.all_dbpaths = frozenset(self.dbpath(id_) for id_ in self.all_ids)
self.all_lc_dbpaths = frozenset([f.lower() for f in self.all_dbpaths])
self.db_id_regexp = re.compile(r'^.* \((\d+)\)$')
self.dirs = []
self.book_dirs = []
self.potential_authors = {}
self.invalid_authors = []
self.extra_authors = []
self.invalid_titles = []
self.extra_titles = []
self.unknown_book_files = []
self.missing_formats = []
self.extra_formats = []
self.extra_files = []
self.missing_covers = []
self.extra_covers = []
self.failed_folders = []
def get_library_path(gui):
# gets the current library path from a gui
db = gui.library_view.model().db
lp = db.library_path
if isbytestring(lp):
lp = lp.decode(filesystem_encoding)
return lp
def library_name(self):
db = self.gui.library_view.model().db
path = db.library_path
if isbytestring(path):
path = path.decode(filesystem_encoding)
path = path.replace(os.sep, '/')
return self.stats.pretty(path)
def _generate(self, article, style=None, extra_css=None):
content = article.content if article.content else ''
summary = article.summary if article.summary else ''
text = content if len(content) > len(summary) else summary
head = HEAD(TITLE(article.title))
if style:
head.append(STYLE(style, type='text/css'))
if extra_css:
head.append(STYLE(extra_css, type='text/css'))
if isbytestring(text):
text = text.decode('utf-8', 'replace')
elements = html.fragments_fromstring(text)
self.root = HTML(head,
BODY(H2(article.title), DIV()))
div = self.root.find('body').find('div')
if elements and isinstance(elements[0], unicode_type):
div.text = elements[0]
elements = list(elements)[1:]
for elem in elements:
if hasattr(elem, 'getparent'):
elem.getparent().remove(elem)
else:
elem = SPAN(elem)
div.append(elem)
def _generate(self, article, style=None, extra_css=None):
content = article.content if article.content else ''
summary = article.summary if article.summary else ''
text = content if len(content) > len(summary) else summary
head = HEAD(TITLE(article.title))
if style:
head.append(STYLE(style, type='text/css'))
if extra_css:
head.append(STYLE(extra_css, type='text/css'))
if isbytestring(text):
text = text.decode('utf-8', 'replace')
elements = html.fragments_fromstring(text)
self.root = HTML(head,
BODY(H2(article.title), DIV()))
div = self.root.find('body').find('div')
if elements and isinstance(elements[0], unicode_type):
div.text = elements[0]
elements = list(elements)[1:]
for elem in elements:
if hasattr(elem, 'getparent'):
elem.getparent().remove(elem)
else:
elem = SPAN(elem)
div.append(elem)
def object_to_unicode(obj, enc=preferred_encoding):
def dec(x):
return x.decode(enc, 'replace')
if isbytestring(obj):
return dec(obj)
if isinstance(obj, (list, tuple)):
return [dec(x) if isbytestring(x) else x for x in obj]
if isinstance(obj, dict):
ans = {}
for k, v in obj.items():
k = object_to_unicode(k)
v = object_to_unicode(v)
ans[k] = v
return ans
return obj
def clean_txt(txt):
'''
Run transformations on the text to put it into
consistent state.
'''
if isbytestring(txt):
txt = txt.decode('utf-8', 'replace')
# Strip whitespace from the end of the line. Also replace
# all line breaks with \n.
txt = '\n'.join([line.rstrip() for line in txt.splitlines()])
# Replace whitespace at the beginning of the line with
txt = re.sub('(?m)(?<=^)([ ]{2,}|\t+)(?=.)', ' ' * 4, txt)
# Condense redundant spaces
txt = re.sub('[ ]{2,}', ' ', txt)
# Remove blank space from the beginning and end of the document.
txt = re.sub('^\s+(?=.)', '', txt)
txt = re.sub('(?<=.)\s+$', '', txt)
# Remove excessive line breaks.
txt = re.sub('\n{5,}', '\n\n\n\n', txt)
# remove ASCII invalid chars : 0 to 8 and 11-14 to 24
txt = clean_ascii_chars(txt)
return txt
def object_to_unicode(obj, enc=preferred_encoding):
def dec(x):
return x.decode(enc, "replace")
if isbytestring(obj):
return dec(obj)
if isinstance(obj, (list, tuple)):
return [dec(x) if isbytestring(x) else x for x in obj]
if isinstance(obj, dict):
ans = {}
for k, v in obj.items():
k = object_to_unicode(k)
v = object_to_unicode(v)
ans[k] = v
return ans
return obj
def ajax_search(self, query='', sort='title', offset=0, num=25,
sort_order='asc'):
'''
Return the books (as list of ids) matching the specified search query.
'''
try:
num = int(num)
except:
raise cherrypy.HTTPError(404, "Invalid num: %r"%num)
try:
offset = int(offset)
except:
raise cherrypy.HTTPError(404, "Invalid offset: %r"%offset)
sfield = self.db.data.sanitize_sort_field_name(sort)
if sfield not in self.db.field_metadata.sortable_field_keys():
raise cherrypy.HTTPError(404, '%s is not a valid sort field'%sort)
if isbytestring(query):
query = query.decode('UTF-8')
ids = list(self.search_for_books(query))
self.db.data.multisort(fields=[(sfield, sort_order == 'asc')], subsort=True,
only_ids=ids)
total_num = len(ids)
ids = ids[offset:offset+num]
return {
'total_num': total_num, 'sort_order':sort_order,
'offset':offset, 'num':len(ids), 'sort':sort,
'base_url':absurl(self.opts.url_prefix, '/ajax/search'),
'query': query,
'book_ids':ids
}
def split_txt(txt, epub_split_size_kb=0):
'''
Ensure there are split points for converting
to EPUB. A misdetected paragraph type can
result in the entire document being one giant
paragraph. In this case the EPUB parser will not
be able to determine where to split the file
to accomidate the EPUB file size limitation
and will fail.
'''
# Takes care if there is no point to split
if epub_split_size_kb > 0:
if isinstance(txt, unicode):
txt = txt.encode('utf-8')
length_byte = len(txt)
# Calculating the average chunk value for easy splitting as EPUB (+2 as a safe margin)
chunk_size = long(length_byte / (int(length_byte / (epub_split_size_kb * 1024)) + 2))
# if there are chunks with a superior size then go and break
if (len(filter(lambda x: len(x) > chunk_size, txt.split('\n\n')))) :
txt = '\n\n'.join([split_string_separator(line, chunk_size)
for line in txt.split('\n\n')])
if isbytestring(txt):
txt = txt.decode('utf-8')
return txt
def library_name(self):
db = self.gui.library_view.model().db
path = db.library_path
if isbytestring(path):
path = path.decode(filesystem_encoding)
path = path.replace(os.sep, '/')
return self.stats.pretty(path)
def clean_txt(txt):
'''
Run transformations on the text to put it into
consistent state.
'''
if isbytestring(txt):
txt = txt.decode('utf-8', 'replace')
# Strip whitespace from the end of the line. Also replace
# all line breaks with \n.
txt = '\n'.join([line.rstrip() for line in txt.splitlines()])
# Replace whitespace at the beginning of the line with
txt = re.sub('(?m)(?<=^)([ ]{2,}|\t+)(?=.)', ' ' * 4, txt)
# Condense redundant spaces
txt = re.sub('[ ]{2,}', ' ', txt)
# Remove blank space from the beginning and end of the document.
txt = re.sub('^\s+(?=.)', '', txt)
txt = re.sub('(?<=.)\s+$', '', txt)
# Remove excessive line breaks.
txt = re.sub('\n{5,}', '\n\n\n\n', txt)
# remove ASCII invalid chars : 0 to 8 and 11-14 to 24
txt = clean_ascii_chars(txt)
return txt
def split_txt(txt, epub_split_size_kb=0):
'''
Ensure there are split points for converting
to EPUB. A misdetected paragraph type can
result in the entire document being one giant
paragraph. In this case the EPUB parser will not
be able to determine where to split the file
to accomidate the EPUB file size limitation
and will fail.
'''
# Takes care if there is no point to split
if epub_split_size_kb > 0:
if isinstance(txt, unicode):
txt = txt.encode('utf-8')
length_byte = len(txt)
# Calculating the average chunk value for easy splitting as EPUB (+2 as a safe margin)
chunk_size = long(length_byte / (int(length_byte /
(epub_split_size_kb * 1024)) + 2))
# if there are chunks with a superior size then go and break
if (len(filter(lambda x: len(x) > chunk_size, txt.split('\n\n')))):
txt = '\n\n'.join([
split_string_separator(line, chunk_size)
for line in txt.split('\n\n')
])
if isbytestring(txt):
txt = txt.decode('utf-8')
return txt
def add_system_font(self, path):
if isbytestring(path):
path = path.decode(filesystem_encoding)
path = os.path.abspath(path)
ret = self.w.add_system_font(path)
if ret > 0:
atexit.register(self.remove_system_font, path)
return ret
def plugin_tweaks_string(self):
ans = []
for key, val in self.plugin_tweaks.iteritems():
ans.extend(['%s = %r'%(key, val), '', ''])
ans = '\n'.join(ans)
if isbytestring(ans):
ans = ans.decode('utf-8')
return ans
def plugin_tweaks_string(self):
ans = []
for key, val in self.plugin_tweaks.iteritems():
ans.extend(['%s = %r' % (key, val), '', ''])
ans = '\n'.join(ans)
if isbytestring(ans):
ans = ans.decode('utf-8')
return ans
def windows_get_fileid(path):
''' The fileid uniquely identifies actual file contents (it is the same for
all hardlinks to a file). Similar to inode number on linux. '''
from calibre_extensions.winutil import get_file_id
if isbytestring(path):
path = path.decode(filesystem_encoding)
with suppress(OSError):
return get_file_id(path)
def windows_get_fileid(path):
''' The fileid uniquely identifies actual file contents (it is the same for
all hardlinks to a file). Similar to inode number on linux. '''
get_file_id = plugins['winutil'][0].get_file_id
if isbytestring(path):
path = path.decode(filesystem_encoding)
with suppress(OSError):
return get_file_id(path)
def generate(self, *args, **kwargs):
if "style" not in kwargs:
kwargs["style"] = ""
for key in kwargs.keys():
if isbytestring(kwargs[key]):
kwargs[key] = kwargs[key].decode("utf-8", "replace")
if kwargs[key] is None:
kwargs[key] = u""
args = list(args)
for i in range(len(args)):
if isbytestring(args[i]):
args[i] = args[i].decode("utf-8", "replace")
if args[i] is None:
args[i] = u""
self._generate(*args, **kwargs)
return self
def generate(self, *args, **kwargs):
if 'style' not in kwargs:
kwargs['style'] = ''
for key in kwargs.keys():
if isbytestring(kwargs[key]):
kwargs[key] = kwargs[key].decode('utf-8', 'replace')
if kwargs[key] is None:
kwargs[key] = u''
args = list(args)
for i in range(len(args)):
if isbytestring(args[i]):
args[i] = args[i].decode('utf-8', 'replace')
if args[i] is None:
args[i] = u''
self._generate(*args, **kwargs)
return self
def get_fileid(x):
if isbytestring(x): x = x.decode(filesystem_encoding)
try:
h = win32file.CreateFile(x, 0, 0, None, win32file.OPEN_EXISTING,
win32file.FILE_FLAG_BACKUP_SEMANTICS, 0)
data = win32file.GetFileInformationByHandle(h)
except (error, EnvironmentError):
return None
return (data[4], data[8], data[9])
def generate(self, *args, **kwargs):
if 'style' not in kwargs:
kwargs['style'] = ''
for key in kwargs.keys():
if isbytestring(kwargs[key]):
kwargs[key] = kwargs[key].decode('utf-8', 'replace')
if kwargs[key] is None:
kwargs[key] = u''
args = list(args)
for i in range(len(args)):
if isbytestring(args[i]):
args[i] = args[i].decode('utf-8', 'replace')
if args[i] is None:
args[i] = u''
self._generate(*args, **kwargs)
return self
def browse_template(self, sort, category=True, initial_search=''):
if not hasattr(self, '__browse_template__') or \
self.opts.develop:
self.__browse_template__ = \
P('content_server/browse/browse.html', data=True).decode('utf-8')
ans = self.__browse_template__
scn = 'calibre_browse_server_sort_'
if category:
sort_opts = [('rating', _('Average rating')), ('name', _('Name')),
('popularity', _('Popularity'))]
scn += 'category'
else:
scn += 'list'
fm = self.db.field_metadata
sort_opts, added = [], set([])
displayed_custom_fields = custom_fields_to_display(self.db)
for x in fm.sortable_field_keys():
if x in ('ondevice', 'formats', 'sort'):
continue
if fm.is_ignorable_field(
x) and x not in displayed_custom_fields:
continue
if x == 'comments' or fm[x]['datatype'] == 'comments':
continue
n = fm[x]['name']
if n not in added:
added.add(n)
sort_opts.append((x, n))
ans = ans.replace('{sort_select_label}', xml(_('Sort by') + ':'))
ans = ans.replace('{sort_cookie_name}', scn)
ans = ans.replace('{prefix}', self.opts.url_prefix)
ans = ans.replace('{library}', _('library'))
ans = ans.replace('{home}', _('home'))
ans = ans.replace('{Search}', _('Search'))
opts = [
'<option %svalue="%s">%s</option>' % (
'selected="selected" ' if k == sort else '',
xml(k),
xml(nl),
) for k, nl in sorted(
sort_opts, key=lambda x: sort_key(operator.itemgetter(1)(x)))
if k and nl
]
ans = ans.replace('{sort_select_options}',
('\n' + ' ' * 20).join(opts))
lp = self.db.library_path
if isbytestring(lp):
lp = force_unicode(lp, filesystem_encoding)
ans = ans.replace('{library_name}', xml(os.path.basename(lp)))
ans = ans.replace('{library_path}', xml(lp, True))
ans = ans.replace('{initial_search}',
xml(initial_search, attribute=True))
return ans
def command_check_library(args, dbpath):
from calibre.library.check_library import CheckLibrary, CHECKS
parser = check_library_option_parser()
opts, args = parser.parse_args(args)
if len(args) != 0:
parser.print_help()
return 1
if opts.library_path is not None:
dbpath = opts.library_path
if isbytestring(dbpath):
dbpath = dbpath.decode(preferred_encoding)
if opts.report is None:
checks = CHECKS
else:
checks = []
for r in opts.report.split(','):
found = False
for c in CHECKS:
if c[0] == r:
checks.append(c)
found = True
break
if not found:
print _('Unknown report check'), r
return 1
if opts.names is None:
names = []
else:
names = [f.strip() for f in opts.names.split(',') if f.strip()]
if opts.exts is None:
exts = []
else:
exts = [f.strip() for f in opts.exts.split(',') if f.strip()]
def print_one(checker, check):
attr = check[0]
list = getattr(checker, attr, None)
if list is None:
return
if opts.csv:
for i in list:
print check[1] + ',' + i[0] + ',' + i[1]
else:
print check[1]
for i in list:
print ' %-40.40s - %-40.40s'%(i[0], i[1])
db = LibraryDatabase2(dbpath)
checker = CheckLibrary(dbpath, db)
checker.scan_library(names, exts)
for check in checks:
print_one(checker, check)
def command_check_library(args, dbpath):
from calibre.library.check_library import CheckLibrary, CHECKS
parser = check_library_option_parser()
opts, args = parser.parse_args(args)
if len(args) != 0:
parser.print_help()
return 1
if opts.library_path is not None:
dbpath = opts.library_path
if isbytestring(dbpath):
dbpath = dbpath.decode(preferred_encoding)
if opts.report is None:
checks = CHECKS
else:
checks = []
for r in opts.report.split(','):
found = False
for c in CHECKS:
if c[0] == r:
checks.append(c)
found = True
break
if not found:
print _('Unknown report check'), r
return 1
if opts.names is None:
names = []
else:
names = [f.strip() for f in opts.names.split(',') if f.strip()]
if opts.exts is None:
exts = []
else:
exts = [f.strip() for f in opts.exts.split(',') if f.strip()]
def print_one(checker, check):
attr = check[0]
list = getattr(checker, attr, None)
if list is None:
return
if opts.csv:
for i in list:
print check[1] + ',' + i[0] + ',' + i[1]
else:
print check[1]
for i in list:
print ' %-40.40s - %-40.40s'%(i[0], i[1])
db = LibraryDatabase2(dbpath)
checker = CheckLibrary(dbpath, db)
checker.scan_library(names, exts)
for check in checks:
print_one(checker, check)
def windows_nlinks(path):
import win32file
dwFlagsAndAttributes = win32file.FILE_FLAG_BACKUP_SEMANTICS if os.path.isdir(path) else 0
if isbytestring(path):
path = path.decode(filesystem_encoding)
handle = win32file.CreateFileW(path, win32file.GENERIC_READ, win32file.FILE_SHARE_READ, None, win32file.OPEN_EXISTING, dwFlagsAndAttributes, None)
try:
return win32file.GetFileInformationByHandle(handle)[7]
finally:
handle.Close()
def get_fileid(x):
if isbytestring(x): x = x.decode(filesystem_encoding)
try:
h = win32file.CreateFile(x, 0, 0, None, win32file.OPEN_EXISTING,
win32file.FILE_FLAG_BACKUP_SEMANTICS, 0)
handles.append(h)
data = win32file.GetFileInformationByHandle(h)
except (error, EnvironmentError):
return None
return (data[4], data[8], data[9])
def browse_category_group(self, category=None, group=None, sort=None):
if sort == 'null':
sort = None
if sort not in ('rating', 'name', 'popularity'):
sort = 'name'
try:
category = unhexlify(category)
if isbytestring(category):
category = category.decode('utf-8')
except:
raise cherrypy.HTTPError(404, 'invalid category')
categories = self.categories_cache()
if category not in categories:
raise cherrypy.HTTPError(404, 'category not found')
category_meta = self.db.field_metadata
try:
datatype = category_meta[category]['datatype']
except KeyError:
datatype = 'text'
try:
group = unhexlify(group)
if isbytestring(group):
group = group.decode('utf-8')
except:
raise cherrypy.HTTPError(404, 'invalid group')
items = categories[category]
entries = []
getter = lambda x: unicode(getattr(x, 'sort', None) or x.name)
for x in items:
val = getter(x)
if not val:
val = u'A'
if val.upper().startswith(group):
entries.append(x)
sort = self.browse_sort_categories(entries, sort)
entries = get_category_items(category, entries,
datatype, self.opts.url_prefix)
return json.dumps(entries, ensure_ascii=True)
def browse_category_group(self, category=None, group=None, sort=None):
if sort == 'null':
sort = None
if sort not in ('rating', 'name', 'popularity'):
sort = 'name'
try:
category = unhexlify(category)
if isbytestring(category):
category = category.decode('utf-8')
except:
raise cherrypy.HTTPError(404, 'invalid category')
categories = self.categories_cache()
if category not in categories:
raise cherrypy.HTTPError(404, 'category not found')
category_meta = self.db.field_metadata
try:
datatype = category_meta[category]['datatype']
except KeyError:
datatype = 'text'
try:
group = unhexlify(group)
if isbytestring(group):
group = group.decode('utf-8')
except:
raise cherrypy.HTTPError(404, 'invalid group')
items = categories[category]
entries = []
getter = lambda x: unicode(getattr(x, 'sort', None) or x.name)
for x in items:
val = getter(x)
if not val:
val = u'A'
if val.upper().startswith(group):
entries.append(x)
sort = self.browse_sort_categories(entries, sort)
entries = get_category_items(category, entries,
datatype, self.opts.url_prefix)
return json.dumps(entries, ensure_ascii=True)
def has_book(self, mi):
title = mi.title
if title:
if isbytestring(title):
title = title.decode(preferred_encoding, 'replace')
q = icu_lower(title)
for title in self.fields['title'].table.book_col_map.itervalues():
if q == icu_lower(title):
return True
return False
def windows_nlinks(path):
import win32file
dwFlagsAndAttributes = win32file.FILE_FLAG_BACKUP_SEMANTICS if os.path.isdir(path) else 0
if isbytestring(path):
path = path.decode(filesystem_encoding)
handle = win32file.CreateFileW(path, win32file.GENERIC_READ, win32file.FILE_SHARE_READ, None, win32file.OPEN_EXISTING, dwFlagsAndAttributes, None)
try:
return win32file.GetFileInformationByHandle(handle)[7]
finally:
handle.Close()
def create_book_entry(self, mi, cover=None, add_duplicates=True, force_id=None, apply_import_tags=True, preserve_uuid=False):
if mi.tags:
mi.tags = list(mi.tags)
if apply_import_tags:
_add_newbook_tag(mi)
if not add_duplicates and self._has_book(mi):
return
series_index = (self._get_next_series_num_for(mi.series) if mi.series_index is None else mi.series_index)
if not mi.authors:
mi.authors = (_('Unknown'),)
aus = mi.author_sort if mi.author_sort else self._author_sort_from_authors(mi.authors)
mi.title = mi.title or _('Unknown')
if isbytestring(aus):
aus = aus.decode(preferred_encoding, 'replace')
if isbytestring(mi.title):
mi.title = mi.title.decode(preferred_encoding, 'replace')
conn = self.backend.conn
if force_id is None:
conn.execute('INSERT INTO books(title, series_index, author_sort) VALUES (?, ?, ?)',
(mi.title, series_index, aus))
else:
conn.execute('INSERT INTO books(id, title, series_index, author_sort) VALUES (?, ?, ?, ?)',
(force_id, mi.title, series_index, aus))
book_id = conn.last_insert_rowid()
mi.timestamp = utcnow() if mi.timestamp is None else mi.timestamp
mi.pubdate = UNDEFINED_DATE if mi.pubdate is None else mi.pubdate
if cover is not None:
mi.cover, mi.cover_data = None, (None, cover)
self._set_metadata(book_id, mi, ignore_errors=True)
if preserve_uuid and mi.uuid:
self._set_field('uuid', {book_id:mi.uuid})
# Update the caches for fields from the books table
self.fields['size'].table.book_col_map[book_id] = 0
row = next(conn.execute('SELECT sort, series_index, author_sort, uuid, has_cover FROM books WHERE id=?', (book_id,)))
for field, val in zip(('sort', 'series_index', 'author_sort', 'uuid', 'cover'), row):
if field == 'cover':
val = bool(val)
elif field == 'uuid':
self.fields[field].table.uuid_to_id_map[val] = book_id
self.fields[field].table.book_col_map[book_id] = val
return book_id
def cleanup_tags(tags):
tags = [x.strip().replace(',', ';') for x in tags if x.strip()]
tags = [x.decode(preferred_encoding, 'replace')
if isbytestring(x) else x for x in tags]
tags = [u' '.join(x.split()) for x in tags]
ans, seen = [], set([])
for tag in tags:
if tag.lower() not in seen:
seen.add(tag.lower())
ans.append(tag)
return ans
def cleanup_tags(tags):
tags = [x.strip().replace(',', ';') for x in tags if x.strip()]
tags = [x.decode(preferred_encoding, 'replace')
if isbytestring(x) else x for x in tags]
tags = [u' '.join(x.split()) for x in tags]
ans, seen = [], set([])
for tag in tags:
if tag.lower() not in seen:
seen.add(tag.lower())
ans.append(tag)
return ans
def browse_search(self, query="", list_sort=None):
if isbytestring(query):
query = query.decode("UTF-8")
ids = self.search_for_books(query)
items = [self.db.data.tablerow_for_id(x) for x in ids]
sort = self.browse_sort_book_list(items, list_sort)
ids = [x[0] for x in items]
html = render_book_list(ids, self.opts.url_prefix, suffix=_("in search") + ": " + xml(query))
return self.browse_template(sort, category=False, initial_search=query).format(
title=_("Matching books"), script="search_result();", main=html
)
def browse_search(self, query='', list_sort=None):
if isbytestring(query):
query = query.decode('UTF-8')
ids = self.db.search_getting_ids(query.strip(), self.search_restriction)
items = [self.db.data._data[x] for x in ids]
sort = self.browse_sort_book_list(items, list_sort)
ids = [x[0] for x in items]
html = render_book_list(ids, self.opts.url_prefix,
suffix=_('in search')+': '+xml(query))
return self.browse_template(sort, category=False, initial_search=query).format(
title=_('Matching books'),
script='search_result();', main=html)
def add_system_font(self, path):
'''
WARNING: The file you are adding must have execute permissions or
windows will fail to add it. (ls -l in cygwin to check)
'''
if isbytestring(path):
path = path.decode(filesystem_encoding)
path = os.path.abspath(path)
ret = self.w.add_system_font(path)
if ret > 0:
atexit.register(self.remove_system_font, path)
return ret
def add_system_font(self, path):
'''
WARNING: The file you are adding must have execute permissions or
windows will fail to add it. (ls -l in cygwin to check)
'''
if isbytestring(path):
path = path.decode(filesystem_encoding)
path = os.path.abspath(path)
ret = self.w.add_system_font(path)
if ret > 0:
atexit.register(self.remove_system_font, path)
return ret
def browse_search(self, query='', list_sort=None):
if isbytestring(query):
query = query.decode('UTF-8')
ids = self.search_for_books(query)
items = [self.db.data.tablerow_for_id(x) for x in ids]
sort = self.browse_sort_book_list(items, list_sort)
ids = [x[0] for x in items]
html = render_book_list(ids, self.opts.url_prefix,
suffix=_('in search')+': '+xml(query))
return self.browse_template(sort, category=False, initial_search=query).format(
title=_('Matching books'),
main=html)
def windows_get_size(path):
''' On windows file sizes are only accurately stored in the actual file,
not in the directory entry (which could be out of date). So we open the
file, and get the actual size. '''
from calibre_extensions import winutil
if isbytestring(path):
path = path.decode(filesystem_encoding)
with closing(
winutil.create_file(
path, 0, winutil.FILE_SHARE_READ | winutil.FILE_SHARE_WRITE
| winutil.FILE_SHARE_DELETE, winutil.OPEN_EXISTING, 0)) as h:
return winutil.get_file_size(h)
def windows_get_fileid(path):
''' The fileid uniquely identifies actual file contents (it is the same for
all hardlinks to a file). Similar to inode number on linux. '''
try:
get_file_id = plugins['winutil'][0].get_file_id
except AttributeError:
# running from source without updating binary
return old_windows_get_fileid(path)
if isbytestring(path):
path = path.decode(filesystem_encoding)
with suppress(OSError):
return get_file_id(path)
def browse_search(self, query='', list_sort=None):
if isbytestring(query):
query = query.decode('UTF-8')
ids = self.search_for_books(query)
items = [self.db.data.tablerow_for_id(x) for x in ids]
sort = self.browse_sort_book_list(items, list_sort)
ids = [x[0] for x in items]
html = render_book_list(ids, self.opts.url_prefix,
suffix=_('in search')+': '+xml(query))
return self.browse_template(sort, category=False, initial_search=query).format(
title=_('Matching books'),
script='search_result();', main=html)
def apsw_connect_to_library(self):
my_db = self.gui.library_view.model().db
self.lib_path = my_db.library_path
self.lib_path = self.lib_path.replace(os.sep, '/')
if isbytestring(self.lib_path):
self.lib_path = self.lib_path.decode(filesystem_encoding)
path = my_db.library_path
if isbytestring(path):
path = path.decode(filesystem_encoding)
path = path.replace(os.sep, '/')
path = os.path.join(path, 'metadata.db')
path = path.replace(os.sep, '/')
if isbytestring(path):
path = path.decode(filesystem_encoding)
if path.endswith("/"):
path = path[0:-1]
if path.count("metadata.db") == 0:
path = path + "/metadata.db"
try:
my_db = apsw.Connection(path)
is_valid = True
except Exception as e:
if DEBUG: print("path to metadata.db is: ", path)
if DEBUG: print("error: ", as_unicode(e))
is_valid = False
return None, None, is_valid
my_cursor = my_db.cursor()
mysql = "PRAGMA main.busy_timeout = 5000;" #PRAGMA busy_timeout = milliseconds;
my_cursor.execute(mysql)
return my_db, my_cursor, is_valid
def browse_category_group(self, category=None, group=None, sort=None):
if sort == "null":
sort = None
if sort not in ("rating", "name", "popularity"):
sort = "name"
try:
category = unhexlify(category)
if isbytestring(category):
category = category.decode("utf-8")
except:
raise cherrypy.HTTPError(404, "invalid category")
categories = self.categories_cache()
if category not in categories:
raise cherrypy.HTTPError(404, "category not found")
category_meta = self.db.field_metadata
datatype = category_meta[category]["datatype"]
try:
group = unhexlify(group)
if isbytestring(group):
group = group.decode("utf-8")
except:
raise cherrypy.HTTPError(404, "invalid group")
items = categories[category]
entries = []
getter = lambda x: unicode(getattr(x, "sort", x.name))
for x in items:
val = getter(x)
if not val:
val = u"A"
if val.upper().startswith(group):
entries.append(x)
sort = self.browse_sort_categories(entries, sort)
entries = get_category_items(category, entries, datatype, self.opts.url_prefix)
return json.dumps(entries, ensure_ascii=True)
def browse_template(self, sort, category=True, initial_search=''):
if not hasattr(self, '__browse_template__') or \
self.opts.develop:
self.__browse_template__ = \
P('content_server/browse/browse.html', data=True).decode('utf-8')
ans = self.__browse_template__
scn = 'calibre_browse_server_sort_'
if category:
sort_opts = [('rating', _('Average rating')), ('name',
_('Name')), ('popularity', _('Popularity'))]
scn += 'category'
else:
scn += 'list'
fm = self.db.field_metadata
sort_opts, added = [], set([])
displayed_custom_fields = custom_fields_to_display(self.db)
for x in fm.sortable_field_keys():
if x in ('ondevice', 'formats', 'sort'):
continue
if fm.is_ignorable_field(x) and x not in displayed_custom_fields:
continue
if x == 'comments' or fm[x]['datatype'] == 'comments':
continue
n = fm[x]['name']
if n not in added:
added.add(n)
sort_opts.append((x, n))
ans = ans.replace('{sort_select_label}', xml(_('Sort by')+':'))
ans = ans.replace('{sort_cookie_name}', scn)
ans = ans.replace('{prefix}', self.opts.url_prefix)
ans = ans.replace('{library}', _('library'))
ans = ans.replace('{home}', _('home'))
ans = ans.replace('{Search}', _('Search'))
ans = ans.replace('{nav}', self.nav)
opts = ['<option %svalue="%s">%s</option>' % (
'selected="selected" ' if k==sort else '',
xml(k), xml(nl), ) for k, nl in
sorted(sort_opts, key=lambda x: sort_key(operator.itemgetter(1)(x))) if k and nl]
ans = ans.replace('{sort_select_options}', ('\n'+' '*20).join(opts))
lp = self.db.library_path
if isbytestring(lp):
lp = force_unicode(lp, filesystem_encoding)
ans = ans.replace('{library_name}', xml(os.path.basename(lp)))
ans = ans.replace('{library_path}', xml(lp, True))
ans = ans.replace('{initial_search}', xml(initial_search, attribute=True))
return ans
def windows_get_size(path):
''' On windows file sizes are only accurately stored in the actual file,
not in the directory entry (which could be out of date). So we open the
file, and get the actual size. '''
import win32file
if isbytestring(path):
path = path.decode(filesystem_encoding)
h = win32file.CreateFileW(
path, 0, win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE | win32file.FILE_SHARE_DELETE,
None, win32file.OPEN_EXISTING, 0, None)
try:
return win32file.GetFileSize(h)
finally:
win32file.CloseHandle(h)
def create_mail(from_,
to,
subject,
text=None,
attachment_data=None,
attachment_type=None,
attachment_name=None):
assert text or attachment_data
from email.mime.multipart import MIMEMultipart
from email.utils import formatdate
from email import encoders
import uuid
outer = MIMEMultipart()
outer['Subject'] = subject
outer['To'] = to
outer['From'] = from_
outer['Date'] = formatdate(localtime=True)
outer['Message-Id'] = "<{}@{}>".format(uuid.uuid4(),
get_msgid_domain(from_))
outer.preamble = 'You will not see this in a MIME-aware mail reader.\n'
if text is not None:
from email.mime.text import MIMEText
if isbytestring(text):
msg = MIMEText(text)
else:
msg = MIMEText(text, 'plain', 'utf-8')
outer.attach(msg)
if attachment_data is not None:
from email.mime.base import MIMEBase
from email.header import Header
assert attachment_data and attachment_name
try:
maintype, subtype = attachment_type.split('/', 1)
except AttributeError:
maintype, subtype = 'application', 'octet-stream'
msg = MIMEBase(maintype,
subtype,
name=Header(attachment_name, 'utf-8').encode())
msg.set_payload(attachment_data)
encoders.encode_base64(msg)
msg.add_header('Content-Disposition',
'attachment',
filename=Header(attachment_name, 'utf-8').encode())
outer.attach(msg)
return outer.as_string()
def windows_get_size(path):
''' On windows file sizes are only accurately stored in the actual file,
not in the directory entry (which could be out of date). So we open the
file, and get the actual size. '''
import win32file
if isbytestring(path):
path = path.decode(filesystem_encoding)
h = win32file.CreateFileW(
path, 0, win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE | win32file.FILE_SHARE_DELETE,
None, win32file.OPEN_EXISTING, 0, None)
try:
return win32file.GetFileSize(h)
finally:
win32file.CloseHandle(h)
def command_backup_metadata(args, dbpath):
parser = backup_metadata_option_parser()
opts, args = parser.parse_args(args)
if len(args) != 0:
parser.print_help()
return 1
if opts.library_path is not None:
dbpath = opts.library_path
if isbytestring(dbpath):
dbpath = dbpath.decode(preferred_encoding)
db = LibraryDatabase2(dbpath)
book_ids = None
if opts.all:
book_ids = db.all_ids()
db.dump_metadata(book_ids=book_ids, callback=BackupProgress())
def __init__(self, db, callback, parent):
QDialog.__init__(self, parent)
self.setupUi(self)
self.db = db
self.new_db = None
self.callback = callback
self.location.initialize('choose_library_dialog')
lp = db.library_path
if isbytestring(lp):
lp = lp.decode(filesystem_encoding)
loc = unicode_type(self.old_location.text()).format(lp)
self.old_location.setText(loc)
self.browse_button.clicked.connect(self.choose_loc)
self.empty_library.toggled.connect(self.empty_library_toggled)
self.copy_structure.setEnabled(False)
def windows_get_fileid(path):
''' The fileid uniquely identifies actual file contents (it is the same for
all hardlinks to a file). Similar to inode number on linux. '''
import win32file
from pywintypes import error
if isbytestring(path):
path = path.decode(filesystem_encoding)
try:
h = win32file.CreateFileW(path, 0, 0, None, win32file.OPEN_EXISTING,
win32file.FILE_FLAG_BACKUP_SEMANTICS, 0)
try:
data = win32file.GetFileInformationByHandle(h)
finally:
win32file.CloseHandle(h)
except (error, EnvironmentError):
return None
return data[4], data[8], data[9]
