Пример #1
0
 def copy_cover_to(self, path, dest, windows_atomic_move=None, use_hardlink=False):
     path = os.path.abspath(os.path.join(self.library_path, path, 'cover.jpg'))
     if windows_atomic_move is not None:
         if not isinstance(dest, basestring):
             raise Exception("Error, you must pass the dest as a path when"
                     " using windows_atomic_move")
         if os.access(path, os.R_OK) and dest and not samefile(dest, path):
             windows_atomic_move.copy_path_to(path, dest)
             return True
     else:
         if os.access(path, os.R_OK):
             try:
                 f = lopen(path, 'rb')
             except (IOError, OSError):
                 time.sleep(0.2)
             f = lopen(path, 'rb')
             with f:
                 if hasattr(dest, 'write'):
                     shutil.copyfileobj(f, dest)
                     if hasattr(dest, 'flush'):
                         dest.flush()
                     return True
                 elif dest and not samefile(dest, path):
                     if use_hardlink:
                         try:
                             hardlink_file(path, dest)
                             return True
                         except:
                             pass
                     with lopen(dest, 'wb') as d:
                         shutil.copyfileobj(f, d)
                     return True
     return False
Пример #2
0
 def copy_format_to(self,
                    book_id,
                    fmt,
                    fname,
                    path,
                    dest,
                    windows_atomic_move=None,
                    use_hardlink=False):
     path = self.format_abspath(book_id, fmt, fname, path)
     if path is None:
         return False
     if windows_atomic_move is not None:
         if not isinstance(dest, basestring):
             raise Exception("Error, you must pass the dest as a path when"
                             " using windows_atomic_move")
         if dest:
             if samefile(dest, path):
                 # Ensure that the file has the same case as dest
                 try:
                     if path != dest:
                         os.rename(path, dest)
                 except:
                     pass  # Nothing too catastrophic happened, the cases mismatch, that's all
             else:
                 windows_atomic_move.copy_path_to(path, dest)
     else:
         if hasattr(dest, 'write'):
             with lopen(path, 'rb') as f:
                 shutil.copyfileobj(f, dest)
             if hasattr(dest, 'flush'):
                 dest.flush()
         elif dest:
             if samefile(dest, path):
                 if not self.is_case_sensitive and path != dest:
                     # Ensure that the file has the same case as dest
                     try:
                         os.rename(path, dest)
                     except:
                         pass  # Nothing too catastrophic happened, the cases mismatch, that's all
             else:
                 if use_hardlink:
                     try:
                         hardlink_file(path, dest)
                         return True
                     except:
                         pass
                 with lopen(path, 'rb') as f, lopen(dest, 'wb') as d:
                     shutil.copyfileobj(f, d)
     return True
Пример #3
0
def run_import_plugins(paths, group_id, tdir):
    final_paths = []
    for path in paths:
        if not os.access(path, os.R_OK):
            continue
        try:
            nfp = run_plugins_on_import(path)
        except Exception:
            nfp = None
            import traceback
            traceback.print_exc()
        if nfp and os.access(nfp, os.R_OK) and not samefile(nfp, path):
            # Ensure that the filename is preserved so that
            # reading metadata from filename is not broken
            name = os.path.splitext(os.path.basename(path))[0]
            ext = os.path.splitext(nfp)[1]
            path = os.path.join(tdir, '%s' % group_id, name + ext)
            try:
                os.mkdir(os.path.dirname(path))
            except EnvironmentError as err:
                if err.errno != errno.EEXIST:
                    raise
            try:
                os.rename(nfp, path)
            except EnvironmentError:
                shutil.copyfile(nfp, path)
        final_paths.append(path)
    return final_paths
Пример #4
0
def container_diff(left, right):
    left_names, right_names = set(left.name_path_map), set(right.name_path_map)
    if left.cloned or right.cloned:
        # Since containers are often clones of each other, as a performance
        # optimization, discard identical names that point to the same physical
        # file, without needing to read the file's contents.

        # First commit dirtied names
        for c in (left, right):
            Container.commit(c, keep_parsed=True)

        samefile_names = {name for name in left_names & right_names if samefile(
            left.name_path_map[name], right.name_path_map[name])}
        left_names -= samefile_names
        right_names -= samefile_names

    cache, changed_names, renamed_names, removed_names, added_names = changed_files(
        left_names, right_names, left.raw_data, right.raw_data)

    def syntax(container, name):
        mt = container.mime_map[name]
        return syntax_from_mime(name, mt)

    syntax_map = {name:syntax(left, name) for name in changed_names}
    syntax_map.update({name:syntax(left, name) for name in renamed_names})
    syntax_map.update({name:syntax(right, name) for name in added_names})
    syntax_map.update({name:syntax(left, name) for name in removed_names})
    return cache, syntax_map, changed_names, renamed_names, removed_names, added_names
Пример #5
0
def container_diff(left, right):
    left_names, right_names = set(left.name_path_map), set(right.name_path_map)
    if left.cloned or right.cloned:
        # Since containers are often clones of each other, as a performance
        # optimization, discard identical names that point to the same physical
        # file, without needing to read the file's contents.

        # First commit dirtied names
        for c in (left, right):
            Container.commit(c, keep_parsed=True)

        samefile_names = {
            name
            for name in left_names & right_names
            if samefile(left.name_path_map[name], right.name_path_map[name])
        }
        left_names -= samefile_names
        right_names -= samefile_names

    cache, changed_names, renamed_names, removed_names, added_names = changed_files(
        left_names, right_names, left.raw_data, right.raw_data)

    def syntax(container, name):
        mt = container.mime_map[name]
        return syntax_from_mime(name, mt)

    syntax_map = {name: syntax(left, name) for name in changed_names}
    syntax_map.update({name: syntax(left, name) for name in renamed_names})
    syntax_map.update({name: syntax(right, name) for name in added_names})
    syntax_map.update({name: syntax(left, name) for name in removed_names})
    return cache, syntax_map, changed_names, renamed_names, removed_names, added_names
Пример #6
0
def run_import_plugins(paths, group_id, tdir):
    final_paths = []
    for path in paths:
        if not os.access(path, os.R_OK):
            continue
        try:
            nfp = run_plugins_on_import(path)
        except Exception:
            nfp = None
            import traceback
            traceback.print_exc()
        if nfp and os.access(nfp, os.R_OK) and not samefile(nfp, path):
            # Ensure that the filename is preserved so that
            # reading metadata from filename is not broken
            name = os.path.splitext(os.path.basename(path))[0]
            ext = os.path.splitext(nfp)[1]
            path = os.path.join(tdir, '%s' % group_id, name + ext)
            try:
                os.mkdir(os.path.dirname(path))
            except EnvironmentError as err:
                if err.errno != errno.EEXIST:
                    raise
            try:
                os.rename(nfp, path)
            except EnvironmentError:
                shutil.copyfile(nfp, path)
        final_paths.append(path)
    return final_paths
Пример #7
0
 def copy_format_to(self, book_id, fmt, fname, path, dest,
                    windows_atomic_move=None, use_hardlink=False):
     path = self.format_abspath(book_id, fmt, fname, path)
     if path is None:
         return False
     if windows_atomic_move is not None:
         if not isinstance(dest, basestring):
             raise Exception("Error, you must pass the dest as a path when"
                     " using windows_atomic_move")
         if dest:
             if samefile(dest, path):
                 # Ensure that the file has the same case as dest
                 try:
                     if path != dest:
                         os.rename(path, dest)
                 except:
                     pass  # Nothing too catastrophic happened, the cases mismatch, that's all
             else:
                 windows_atomic_move.copy_path_to(path, dest)
     else:
         if hasattr(dest, 'write'):
             with lopen(path, 'rb') as f:
                 shutil.copyfileobj(f, dest)
             if hasattr(dest, 'flush'):
                 dest.flush()
         elif dest:
             if samefile(dest, path):
                 if not self.is_case_sensitive and path != dest:
                     # Ensure that the file has the same case as dest
                     try:
                         os.rename(path, dest)
                     except:
                         pass  # Nothing too catastrophic happened, the cases mismatch, that's all
             else:
                 if use_hardlink:
                     try:
                         hardlink_file(path, dest)
                         return True
                     except:
                         pass
                 with lopen(path, 'rb') as f, lopen(dest, 'wb') as d:
                     shutil.copyfileobj(f, d)
     return True
Пример #8
0
 def copy_cover_to(self,
                   path,
                   dest,
                   windows_atomic_move=None,
                   use_hardlink=False):
     path = os.path.abspath(
         os.path.join(self.library_path, path, 'cover.jpg'))
     if windows_atomic_move is not None:
         if not isinstance(dest, basestring):
             raise Exception("Error, you must pass the dest as a path when"
                             " using windows_atomic_move")
         if os.access(path, os.R_OK) and dest and not samefile(dest, path):
             windows_atomic_move.copy_path_to(path, dest)
             return True
     else:
         if os.access(path, os.R_OK):
             try:
                 f = lopen(path, 'rb')
             except (IOError, OSError):
                 time.sleep(0.2)
             f = lopen(path, 'rb')
             with f:
                 if hasattr(dest, 'write'):
                     shutil.copyfileobj(f, dest)
                     if hasattr(dest, 'flush'):
                         dest.flush()
                     return True
                 elif dest and not samefile(dest, path):
                     if use_hardlink:
                         try:
                             hardlink_file(path, dest)
                             return True
                         except:
                             pass
                     with lopen(dest, 'wb') as d:
                         shutil.copyfileobj(f, d)
                     return True
     return False
Пример #9
0
def all_known_libraries():
    from calibre.gui2 import gprefs
    lus = gprefs.get('library_usage_stats', {})
    paths = set(lus)
    if prefs['library_path']:
        paths.add(prefs['library_path'])
    added = {}
    for path in paths:
        mdb = os.path.join(path, 'metadata.db')
        if os.path.exists(mdb):
            for c in added:
                if samefile(mdb, os.path.join(c, 'metadata.db')):
                    break
            else:
                added[path] = lus.get(path, 1)
    return added
Пример #10
0
def all_known_libraries():
    from calibre.gui2 import gprefs
    lus = gprefs.get('library_usage_stats', {})
    paths = set(lus)
    if prefs['library_path']:
        paths.add(prefs['library_path'])
    added = {}
    for path in paths:
        mdb = os.path.join(path, 'metadata.db')
        if os.path.exists(mdb):
            for c in added:
                if samefile(mdb, os.path.join(c, 'metadata.db')):
                    break
            else:
                added[path] = lus.get(path, 1)
    return added
Пример #11
0
    def add_format(self, book_id, fmt, stream, title, author, path):
        fname = self.construct_file_name(book_id, title, author)
        path = os.path.join(self.library_path, path)
        fmt = ('.' + fmt.lower()) if fmt else ''
        dest = os.path.join(path, fname + fmt)
        if not os.path.exists(path):
            os.makedirs(path)
        size = 0

        if (not getattr(stream, 'name', False) or not samefile(dest, stream.name)):
            with lopen(dest, 'wb') as f:
                shutil.copyfileobj(stream, f)
                size = f.tell()
        elif os.path.exists(dest):
            size = os.path.getsize(dest)

        return size, fname
Пример #12
0
def actual_case_for_name(container, name):
    from calibre.utils.filenames import samefile
    if not container.exists(name):
        raise ValueError('Cannot get actual case for %s as it does not exist' % name)
    parts = name.split('/')
    base = ''
    ans = []
    for i, x in enumerate(parts):
        base = '/'.join(ans + [x])
        path = container.name_to_abspath(base)
        pdir = os.path.dirname(path)
        candidates = {os.path.join(pdir, q) for q in os.listdir(pdir)}
        if x in candidates:
            correctx = x
        else:
            for q in candidates:
                if samefile(q, path):
                    correctx = os.path.basename(q)
                    break
            else:
                raise RuntimeError('Something bad happened')
        ans.append(correctx)
    return '/'.join(ans)
Пример #13
0
def actual_case_for_name(container, name):
    from calibre.utils.filenames import samefile
    if not container.exists(name):
        raise ValueError('Cannot get actual case for %s as it does not exist' % name)
    parts = name.split('/')
    base = ''
    ans = []
    for i, x in enumerate(parts):
        base = '/'.join(ans + [x])
        path = container.name_to_abspath(base)
        pdir = os.path.dirname(path)
        candidates = {os.path.join(pdir, q) for q in os.listdir(pdir)}
        if x in candidates:
            correctx = x
        else:
            for q in candidates:
                if samefile(q, path):
                    correctx = os.path.basename(q)
                    break
            else:
                raise RuntimeError('Something bad happened')
        ans.append(correctx)
    return '/'.join(ans)
Пример #14
0
def run_import_plugins(paths, group_id, tdir):
    final_paths = []
    for path in paths:
        if not os.access(path, os.R_OK):
            continue
        try:
            nfp = run_plugins_on_import(path)
        except Exception:
            nfp = None
            import traceback
            traceback.print_exc()
        if nfp and os.access(nfp, os.R_OK) and not samefile(nfp, path):
            # Ensure that the filename is preserved so that
            # reading metadata from filename is not broken
            name = os.path.splitext(os.path.basename(path))[0]
            ext = os.path.splitext(nfp)[1]
            path = os.path.join(tdir, str(group_id), name + ext)
            os.makedirs(os.path.dirname(path), exist_ok=True)
            try:
                os.replace(nfp, path)
            except OSError:
                shutil.copyfile(nfp, path)
        final_paths.append(path)
    return final_paths
Пример #15
0
    def update_path(self, book_id, title, author, path_field, formats_field):
        path = self.construct_path_name(book_id, title, author)
        current_path = path_field.for_book(book_id)
        formats = formats_field.for_book(book_id, default_value=())
        fname = self.construct_file_name(book_id, title, author)
        # Check if the metadata used to construct paths has changed
        changed = False
        for fmt in formats:
            name = formats_field.format_fname(book_id, fmt)
            if name and name != fname:
                changed = True
                break
        if path == current_path and not changed:
            return
        spath = os.path.join(self.library_path, *current_path.split('/'))
        tpath = os.path.join(self.library_path, *path.split('/'))

        source_ok = current_path and os.path.exists(spath)
        wam = WindowsAtomicFolderMove(
            spath) if iswindows and source_ok else None
        try:
            if not os.path.exists(tpath):
                os.makedirs(tpath)

            if source_ok:  # Migrate existing files
                dest = os.path.join(tpath, 'cover.jpg')
                self.copy_cover_to(current_path,
                                   dest,
                                   windows_atomic_move=wam,
                                   use_hardlink=True)
                for fmt in formats:
                    dest = os.path.join(tpath, fname + '.' + fmt.lower())
                    self.copy_format_to(book_id,
                                        fmt,
                                        formats_field.format_fname(
                                            book_id, fmt),
                                        current_path,
                                        dest,
                                        windows_atomic_move=wam,
                                        use_hardlink=True)
            # Update db to reflect new file locations
            for fmt in formats:
                formats_field.table.set_fname(book_id, fmt, fname, self)
            path_field.table.set_path(book_id, path, self)

            # Delete not needed directories
            if source_ok:
                if os.path.exists(spath) and not samefile(spath, tpath):
                    if wam is not None:
                        wam.delete_originals()
                    self.rmtree(spath, permanent=True)
                    parent = os.path.dirname(spath)
                    if len(os.listdir(parent)) == 0:
                        self.rmtree(parent, permanent=True)
        finally:
            if wam is not None:
                wam.close_handles()

        curpath = self.library_path
        c1, c2 = current_path.split('/'), path.split('/')
        if not self.is_case_sensitive and len(c1) == len(c2):
            # On case-insensitive systems, title and author renames that only
            # change case don't cause any changes to the directories in the file
            # system. This can lead to having the directory names not match the
            # title/author, which leads to trouble when libraries are copied to
            # a case-sensitive system. The following code attempts to fix this
            # by checking each segment. If they are different because of case,
            # then rename the segment. Note that the code above correctly
            # handles files in the directories, so no need to do them here.
            for oldseg, newseg in zip(c1, c2):
                if oldseg.lower() == newseg.lower() and oldseg != newseg:
                    try:
                        os.rename(os.path.join(curpath, oldseg),
                                  os.path.join(curpath, newseg))
                    except:
                        break  # Fail silently since nothing catastrophic has happened
                curpath = os.path.join(curpath, newseg)
Пример #16
0
    def update_path(self, book_id, title, author, path_field, formats_field):
        path = self.construct_path_name(book_id, title, author)
        current_path = path_field.for_book(book_id)
        formats = formats_field.for_book(book_id, default_value=())
        fname = self.construct_file_name(book_id, title, author)
        # Check if the metadata used to construct paths has changed
        changed = False
        for fmt in formats:
            name = formats_field.format_fname(book_id, fmt)
            if name and name != fname:
                changed = True
                break
        if path == current_path and not changed:
            return
        spath = os.path.join(self.library_path, *current_path.split('/'))
        tpath = os.path.join(self.library_path, *path.split('/'))

        source_ok = current_path and os.path.exists(spath)
        wam = WindowsAtomicFolderMove(spath) if iswindows and source_ok else None
        try:
            if not os.path.exists(tpath):
                os.makedirs(tpath)

            if source_ok:  # Migrate existing files
                dest = os.path.join(tpath, 'cover.jpg')
                self.copy_cover_to(current_path, dest,
                        windows_atomic_move=wam, use_hardlink=True)
                for fmt in formats:
                    dest = os.path.join(tpath, fname+'.'+fmt.lower())
                    self.copy_format_to(book_id, fmt, formats_field.format_fname(book_id, fmt), current_path,
                                        dest, windows_atomic_move=wam, use_hardlink=True)
            # Update db to reflect new file locations
            for fmt in formats:
                formats_field.table.set_fname(book_id, fmt, fname, self)
            path_field.table.set_path(book_id, path, self)

            # Delete not needed directories
            if source_ok:
                if os.path.exists(spath) and not samefile(spath, tpath):
                    if wam is not None:
                        wam.delete_originals()
                    self.rmtree(spath, permanent=True)
                    parent = os.path.dirname(spath)
                    if len(os.listdir(parent)) == 0:
                        self.rmtree(parent, permanent=True)
        finally:
            if wam is not None:
                wam.close_handles()

        curpath = self.library_path
        c1, c2 = current_path.split('/'), path.split('/')
        if not self.is_case_sensitive and len(c1) == len(c2):
            # On case-insensitive systems, title and author renames that only
            # change case don't cause any changes to the directories in the file
            # system. This can lead to having the directory names not match the
            # title/author, which leads to trouble when libraries are copied to
            # a case-sensitive system. The following code attempts to fix this
            # by checking each segment. If they are different because of case,
            # then rename the segment. Note that the code above correctly
            # handles files in the directories, so no need to do them here.
            for oldseg, newseg in zip(c1, c2):
                if oldseg.lower() == newseg.lower() and oldseg != newseg:
                    try:
                        os.rename(os.path.join(curpath, oldseg),
                                os.path.join(curpath, newseg))
                    except:
                        break  # Fail silently since nothing catastrophic has happened
                curpath = os.path.join(curpath, newseg)