def copy_cover_to(self, path, dest, windows_atomic_move=None, use_hardlink=False):
path = os.path.abspath(os.path.join(self.library_path, path, 'cover.jpg'))
if windows_atomic_move is not None:
if not isinstance(dest, basestring):
raise Exception("Error, you must pass the dest as a path when"
" using windows_atomic_move")
if os.access(path, os.R_OK) and dest and not samefile(dest, path):
windows_atomic_move.copy_path_to(path, dest)
return True
else:
if os.access(path, os.R_OK):
try:
f = lopen(path, 'rb')
except (IOError, OSError):
time.sleep(0.2)
f = lopen(path, 'rb')
with f:
if hasattr(dest, 'write'):
shutil.copyfileobj(f, dest)
if hasattr(dest, 'flush'):
dest.flush()
return True
elif dest and not samefile(dest, path):
if use_hardlink:
try:
hardlink_file(path, dest)
return True
except:
pass
with lopen(dest, 'wb') as d:
shutil.copyfileobj(f, d)
return True
return False
def copy_format_to(self,
book_id,
fmt,
fname,
path,
dest,
windows_atomic_move=None,
use_hardlink=False):
path = self.format_abspath(book_id, fmt, fname, path)
if path is None:
return False
if windows_atomic_move is not None:
if not isinstance(dest, basestring):
raise Exception("Error, you must pass the dest as a path when"
" using windows_atomic_move")
if dest:
if samefile(dest, path):
# Ensure that the file has the same case as dest
try:
if path != dest:
os.rename(path, dest)
except:
pass # Nothing too catastrophic happened, the cases mismatch, that's all
else:
windows_atomic_move.copy_path_to(path, dest)
else:
if hasattr(dest, 'write'):
with lopen(path, 'rb') as f:
shutil.copyfileobj(f, dest)
if hasattr(dest, 'flush'):
dest.flush()
elif dest:
if samefile(dest, path):
if not self.is_case_sensitive and path != dest:
# Ensure that the file has the same case as dest
try:
os.rename(path, dest)
except:
pass # Nothing too catastrophic happened, the cases mismatch, that's all
else:
if use_hardlink:
try:
hardlink_file(path, dest)
return True
except:
pass
with lopen(path, 'rb') as f, lopen(dest, 'wb') as d:
shutil.copyfileobj(f, d)
return True
def run_import_plugins(paths, group_id, tdir):
final_paths = []
for path in paths:
if not os.access(path, os.R_OK):
continue
try:
nfp = run_plugins_on_import(path)
except Exception:
nfp = None
import traceback
traceback.print_exc()
if nfp and os.access(nfp, os.R_OK) and not samefile(nfp, path):
# Ensure that the filename is preserved so that
# reading metadata from filename is not broken
name = os.path.splitext(os.path.basename(path))[0]
ext = os.path.splitext(nfp)[1]
path = os.path.join(tdir, '%s' % group_id, name + ext)
try:
os.mkdir(os.path.dirname(path))
except EnvironmentError as err:
if err.errno != errno.EEXIST:
raise
try:
os.rename(nfp, path)
except EnvironmentError:
shutil.copyfile(nfp, path)
final_paths.append(path)
return final_paths
def container_diff(left, right):
left_names, right_names = set(left.name_path_map), set(right.name_path_map)
if left.cloned or right.cloned:
# Since containers are often clones of each other, as a performance
# optimization, discard identical names that point to the same physical
# file, without needing to read the file's contents.
# First commit dirtied names
for c in (left, right):
Container.commit(c, keep_parsed=True)
samefile_names = {name for name in left_names & right_names if samefile(
left.name_path_map[name], right.name_path_map[name])}
left_names -= samefile_names
right_names -= samefile_names
cache, changed_names, renamed_names, removed_names, added_names = changed_files(
left_names, right_names, left.raw_data, right.raw_data)
def syntax(container, name):
mt = container.mime_map[name]
return syntax_from_mime(name, mt)
syntax_map = {name:syntax(left, name) for name in changed_names}
syntax_map.update({name:syntax(left, name) for name in renamed_names})
syntax_map.update({name:syntax(right, name) for name in added_names})
syntax_map.update({name:syntax(left, name) for name in removed_names})
return cache, syntax_map, changed_names, renamed_names, removed_names, added_names
def container_diff(left, right):
left_names, right_names = set(left.name_path_map), set(right.name_path_map)
if left.cloned or right.cloned:
# Since containers are often clones of each other, as a performance
# optimization, discard identical names that point to the same physical
# file, without needing to read the file's contents.
# First commit dirtied names
for c in (left, right):
Container.commit(c, keep_parsed=True)
samefile_names = {
name
for name in left_names & right_names
if samefile(left.name_path_map[name], right.name_path_map[name])
}
left_names -= samefile_names
right_names -= samefile_names
cache, changed_names, renamed_names, removed_names, added_names = changed_files(
left_names, right_names, left.raw_data, right.raw_data)
def syntax(container, name):
mt = container.mime_map[name]
return syntax_from_mime(name, mt)
syntax_map = {name: syntax(left, name) for name in changed_names}
syntax_map.update({name: syntax(left, name) for name in renamed_names})
syntax_map.update({name: syntax(right, name) for name in added_names})
syntax_map.update({name: syntax(left, name) for name in removed_names})
return cache, syntax_map, changed_names, renamed_names, removed_names, added_names
def run_import_plugins(paths, group_id, tdir):
final_paths = []
for path in paths:
if not os.access(path, os.R_OK):
continue
try:
nfp = run_plugins_on_import(path)
except Exception:
nfp = None
import traceback
traceback.print_exc()
if nfp and os.access(nfp, os.R_OK) and not samefile(nfp, path):
# Ensure that the filename is preserved so that
# reading metadata from filename is not broken
name = os.path.splitext(os.path.basename(path))[0]
ext = os.path.splitext(nfp)[1]
path = os.path.join(tdir, '%s' % group_id, name + ext)
try:
os.mkdir(os.path.dirname(path))
except EnvironmentError as err:
if err.errno != errno.EEXIST:
raise
try:
os.rename(nfp, path)
except EnvironmentError:
shutil.copyfile(nfp, path)
final_paths.append(path)
return final_paths
def copy_format_to(self, book_id, fmt, fname, path, dest,
windows_atomic_move=None, use_hardlink=False):
path = self.format_abspath(book_id, fmt, fname, path)
if path is None:
return False
if windows_atomic_move is not None:
if not isinstance(dest, basestring):
raise Exception("Error, you must pass the dest as a path when"
" using windows_atomic_move")
if dest:
if samefile(dest, path):
# Ensure that the file has the same case as dest
try:
if path != dest:
os.rename(path, dest)
except:
pass # Nothing too catastrophic happened, the cases mismatch, that's all
else:
windows_atomic_move.copy_path_to(path, dest)
else:
if hasattr(dest, 'write'):
with lopen(path, 'rb') as f:
shutil.copyfileobj(f, dest)
if hasattr(dest, 'flush'):
dest.flush()
elif dest:
if samefile(dest, path):
if not self.is_case_sensitive and path != dest:
# Ensure that the file has the same case as dest
try:
os.rename(path, dest)
except:
pass # Nothing too catastrophic happened, the cases mismatch, that's all
else:
if use_hardlink:
try:
hardlink_file(path, dest)
return True
except:
pass
with lopen(path, 'rb') as f, lopen(dest, 'wb') as d:
shutil.copyfileobj(f, d)
return True
def copy_cover_to(self,
path,
dest,
windows_atomic_move=None,
use_hardlink=False):
path = os.path.abspath(
os.path.join(self.library_path, path, 'cover.jpg'))
if windows_atomic_move is not None:
if not isinstance(dest, basestring):
raise Exception("Error, you must pass the dest as a path when"
" using windows_atomic_move")
if os.access(path, os.R_OK) and dest and not samefile(dest, path):
windows_atomic_move.copy_path_to(path, dest)
return True
else:
if os.access(path, os.R_OK):
try:
f = lopen(path, 'rb')
except (IOError, OSError):
time.sleep(0.2)
f = lopen(path, 'rb')
with f:
if hasattr(dest, 'write'):
shutil.copyfileobj(f, dest)
if hasattr(dest, 'flush'):
dest.flush()
return True
elif dest and not samefile(dest, path):
if use_hardlink:
try:
hardlink_file(path, dest)
return True
except:
pass
with lopen(dest, 'wb') as d:
shutil.copyfileobj(f, d)
return True
return False
def all_known_libraries():
from calibre.gui2 import gprefs
lus = gprefs.get('library_usage_stats', {})
paths = set(lus)
if prefs['library_path']:
paths.add(prefs['library_path'])
added = {}
for path in paths:
mdb = os.path.join(path, 'metadata.db')
if os.path.exists(mdb):
for c in added:
if samefile(mdb, os.path.join(c, 'metadata.db')):
break
else:
added[path] = lus.get(path, 1)
return added
def all_known_libraries():
from calibre.gui2 import gprefs
lus = gprefs.get('library_usage_stats', {})
paths = set(lus)
if prefs['library_path']:
paths.add(prefs['library_path'])
added = {}
for path in paths:
mdb = os.path.join(path, 'metadata.db')
if os.path.exists(mdb):
for c in added:
if samefile(mdb, os.path.join(c, 'metadata.db')):
break
else:
added[path] = lus.get(path, 1)
return added
def add_format(self, book_id, fmt, stream, title, author, path):
fname = self.construct_file_name(book_id, title, author)
path = os.path.join(self.library_path, path)
fmt = ('.' + fmt.lower()) if fmt else ''
dest = os.path.join(path, fname + fmt)
if not os.path.exists(path):
os.makedirs(path)
size = 0
if (not getattr(stream, 'name', False) or not samefile(dest, stream.name)):
with lopen(dest, 'wb') as f:
shutil.copyfileobj(stream, f)
size = f.tell()
elif os.path.exists(dest):
size = os.path.getsize(dest)
return size, fname
def actual_case_for_name(container, name):
from calibre.utils.filenames import samefile
if not container.exists(name):
raise ValueError('Cannot get actual case for %s as it does not exist' % name)
parts = name.split('/')
base = ''
ans = []
for i, x in enumerate(parts):
base = '/'.join(ans + [x])
path = container.name_to_abspath(base)
pdir = os.path.dirname(path)
candidates = {os.path.join(pdir, q) for q in os.listdir(pdir)}
if x in candidates:
correctx = x
else:
for q in candidates:
if samefile(q, path):
correctx = os.path.basename(q)
break
else:
raise RuntimeError('Something bad happened')
ans.append(correctx)
return '/'.join(ans)
def actual_case_for_name(container, name):
from calibre.utils.filenames import samefile
if not container.exists(name):
raise ValueError('Cannot get actual case for %s as it does not exist' % name)
parts = name.split('/')
base = ''
ans = []
for i, x in enumerate(parts):
base = '/'.join(ans + [x])
path = container.name_to_abspath(base)
pdir = os.path.dirname(path)
candidates = {os.path.join(pdir, q) for q in os.listdir(pdir)}
if x in candidates:
correctx = x
else:
for q in candidates:
if samefile(q, path):
correctx = os.path.basename(q)
break
else:
raise RuntimeError('Something bad happened')
ans.append(correctx)
return '/'.join(ans)
def run_import_plugins(paths, group_id, tdir):
final_paths = []
for path in paths:
if not os.access(path, os.R_OK):
continue
try:
nfp = run_plugins_on_import(path)
except Exception:
nfp = None
import traceback
traceback.print_exc()
if nfp and os.access(nfp, os.R_OK) and not samefile(nfp, path):
# Ensure that the filename is preserved so that
# reading metadata from filename is not broken
name = os.path.splitext(os.path.basename(path))[0]
ext = os.path.splitext(nfp)[1]
path = os.path.join(tdir, str(group_id), name + ext)
os.makedirs(os.path.dirname(path), exist_ok=True)
try:
os.replace(nfp, path)
except OSError:
shutil.copyfile(nfp, path)
final_paths.append(path)
return final_paths
def update_path(self, book_id, title, author, path_field, formats_field):
path = self.construct_path_name(book_id, title, author)
current_path = path_field.for_book(book_id)
formats = formats_field.for_book(book_id, default_value=())
fname = self.construct_file_name(book_id, title, author)
# Check if the metadata used to construct paths has changed
changed = False
for fmt in formats:
name = formats_field.format_fname(book_id, fmt)
if name and name != fname:
changed = True
break
if path == current_path and not changed:
return
spath = os.path.join(self.library_path, *current_path.split('/'))
tpath = os.path.join(self.library_path, *path.split('/'))
source_ok = current_path and os.path.exists(spath)
wam = WindowsAtomicFolderMove(
spath) if iswindows and source_ok else None
try:
if not os.path.exists(tpath):
os.makedirs(tpath)
if source_ok: # Migrate existing files
dest = os.path.join(tpath, 'cover.jpg')
self.copy_cover_to(current_path,
dest,
windows_atomic_move=wam,
use_hardlink=True)
for fmt in formats:
dest = os.path.join(tpath, fname + '.' + fmt.lower())
self.copy_format_to(book_id,
fmt,
formats_field.format_fname(
book_id, fmt),
current_path,
dest,
windows_atomic_move=wam,
use_hardlink=True)
# Update db to reflect new file locations
for fmt in formats:
formats_field.table.set_fname(book_id, fmt, fname, self)
path_field.table.set_path(book_id, path, self)
# Delete not needed directories
if source_ok:
if os.path.exists(spath) and not samefile(spath, tpath):
if wam is not None:
wam.delete_originals()
self.rmtree(spath, permanent=True)
parent = os.path.dirname(spath)
if len(os.listdir(parent)) == 0:
self.rmtree(parent, permanent=True)
finally:
if wam is not None:
wam.close_handles()
curpath = self.library_path
c1, c2 = current_path.split('/'), path.split('/')
if not self.is_case_sensitive and len(c1) == len(c2):
# On case-insensitive systems, title and author renames that only
# change case don't cause any changes to the directories in the file
# system. This can lead to having the directory names not match the
# title/author, which leads to trouble when libraries are copied to
# a case-sensitive system. The following code attempts to fix this
# by checking each segment. If they are different because of case,
# then rename the segment. Note that the code above correctly
# handles files in the directories, so no need to do them here.
for oldseg, newseg in zip(c1, c2):
if oldseg.lower() == newseg.lower() and oldseg != newseg:
try:
os.rename(os.path.join(curpath, oldseg),
os.path.join(curpath, newseg))
except:
break # Fail silently since nothing catastrophic has happened
curpath = os.path.join(curpath, newseg)
def update_path(self, book_id, title, author, path_field, formats_field):
path = self.construct_path_name(book_id, title, author)
current_path = path_field.for_book(book_id)
formats = formats_field.for_book(book_id, default_value=())
fname = self.construct_file_name(book_id, title, author)
# Check if the metadata used to construct paths has changed
changed = False
for fmt in formats:
name = formats_field.format_fname(book_id, fmt)
if name and name != fname:
changed = True
break
if path == current_path and not changed:
return
spath = os.path.join(self.library_path, *current_path.split('/'))
tpath = os.path.join(self.library_path, *path.split('/'))
source_ok = current_path and os.path.exists(spath)
wam = WindowsAtomicFolderMove(spath) if iswindows and source_ok else None
try:
if not os.path.exists(tpath):
os.makedirs(tpath)
if source_ok: # Migrate existing files
dest = os.path.join(tpath, 'cover.jpg')
self.copy_cover_to(current_path, dest,
windows_atomic_move=wam, use_hardlink=True)
for fmt in formats:
dest = os.path.join(tpath, fname+'.'+fmt.lower())
self.copy_format_to(book_id, fmt, formats_field.format_fname(book_id, fmt), current_path,
dest, windows_atomic_move=wam, use_hardlink=True)
# Update db to reflect new file locations
for fmt in formats:
formats_field.table.set_fname(book_id, fmt, fname, self)
path_field.table.set_path(book_id, path, self)
# Delete not needed directories
if source_ok:
if os.path.exists(spath) and not samefile(spath, tpath):
if wam is not None:
wam.delete_originals()
self.rmtree(spath, permanent=True)
parent = os.path.dirname(spath)
if len(os.listdir(parent)) == 0:
self.rmtree(parent, permanent=True)
finally:
if wam is not None:
wam.close_handles()
curpath = self.library_path
c1, c2 = current_path.split('/'), path.split('/')
if not self.is_case_sensitive and len(c1) == len(c2):
# On case-insensitive systems, title and author renames that only
# change case don't cause any changes to the directories in the file
# system. This can lead to having the directory names not match the
# title/author, which leads to trouble when libraries are copied to
# a case-sensitive system. The following code attempts to fix this
# by checking each segment. If they are different because of case,
# then rename the segment. Note that the code above correctly
# handles files in the directories, so no need to do them here.
for oldseg, newseg in zip(c1, c2):
if oldseg.lower() == newseg.lower() and oldseg != newseg:
try:
os.rename(os.path.join(curpath, oldseg),
os.path.join(curpath, newseg))
except:
break # Fail silently since nothing catastrophic has happened
curpath = os.path.join(curpath, newseg)