def filesystem_cache(self):
if self._filesystem_cache is None:
debug('Loading filesystem metadata...')
st = time.time()
from calibre.devices.mtp.filesystem_cache import FilesystemCache
ts = self.total_space()
all_storage = []
items = []
for storage_id, capacity in zip([self._main_id, self._carda_id,
self._cardb_id], ts):
if storage_id is None: continue
name = _('Unknown')
for s in self.dev.data['storage']:
if s['id'] == storage_id:
name = s['name']
break
storage = {'id':storage_id, 'size':capacity, 'name':name,
'is_folder':True, 'can_delete':False, 'is_system':True}
self._currently_getting_sid = unicode(storage_id)
id_map = self.dev.get_filesystem(storage_id,
self._filesystem_callback)
for x in id_map.itervalues(): x['storage_id'] = storage_id
all_storage.append(storage)
items.append(id_map.itervalues())
self._filesystem_cache = FilesystemCache(all_storage, chain(*items))
debug('Filesystem metadata loaded in %g seconds (%d objects)'%(
time.time()-st, len(self._filesystem_cache)))
return self._filesystem_cache
def filesystem_cache(self):
if self._filesystem_cache is None:
st = time.time()
debug('Loading filesystem metadata...')
from calibre.devices.mtp.filesystem_cache import FilesystemCache
with self.lock:
storage, all_items, all_errs = [], [], []
for sid, capacity in zip([self._main_id, self._carda_id,
self._cardb_id], self.total_space()):
if sid is None: continue
name = _('Unknown')
for x in self.dev.storage_info:
if x['id'] == sid:
name = x['name']
break
storage.append({'id':sid, 'size':capacity,
'is_folder':True, 'name':name, 'can_delete':False,
'is_system':True})
items, errs = self.dev.get_filesystem(sid,
self._filesystem_callback)
all_items.extend(items), all_errs.extend(errs)
if not all_items and all_errs:
raise DeviceError(
'Failed to read filesystem from %s with errors: %s'
%(self.current_friendly_name,
self.format_errorstack(all_errs)))
if all_errs:
prints('There were some errors while getting the '
' filesystem from %s: %s'%(
self.current_friendly_name,
self.format_errorstack(all_errs)))
self._filesystem_cache = FilesystemCache(storage, all_items)
debug('Filesystem metadata loaded in %g seconds (%d objects)'%(
time.time()-st, len(self._filesystem_cache)))
return self._filesystem_cache
def sync_booklists(self, booklists, end_session=True):
debug('sync_booklists() called')
for bl in booklists:
if getattr(bl, 'storage_id', None) is None:
continue
storage = self.filesystem_cache.storage(bl.storage_id)
if storage is None:
continue
self.write_metadata_cache(storage, bl)
debug('sync_booklists() ended')
def osx_is_device_mtp(self, d, debug=None):
if not d.serial:
ans = False
else:
try:
ans = self.usbobserver.is_mtp_device(d.vendor_id, d.product_id, d.bcd, d.serial)
except Exception:
if debug is not None:
import traceback
traceback.print_stack()
return False
if debug is not None and ans:
debug('Device {0} claims to be an MTP device in the IOKit registry'.format(d))
return bool(ans)
def _filesystem_callback(self, fs_map, entry, level):
name = entry.get('name', '')
self.filesystem_callback(_('Found object: %s')%name)
fs_map[entry.get('id', null)] = entry
path = [name]
pid = entry.get('parent_id', 0)
while pid != 0 and pid in fs_map:
parent = fs_map[pid]
path.append(parent.get('name', ''))
pid = parent.get('parent_id', 0)
if fs_map.get(pid, None) is parent:
break # An object is its own parent
path = tuple(reversed(path))
ok = not self.is_folder_ignored(self._currently_getting_sid, path)
if not ok:
debug('Ignored object: %s' % '/'.join(path))
return ok
def upload_books(self, files, names, on_card=None, end_session=True,
metadata=None):
debug('upload_books() called')
from calibre.devices.utils import sanity_check
sanity_check(on_card, files, self.card_prefix(), self.free_space())
prefix = self.prefix_for_location(on_card)
sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(on_card,
self._main_id)
bl_idx = {'carda':1, 'cardb':2}.get(on_card, 0)
storage = self.filesystem_cache.storage(sid)
ans = []
self.report_progress(0, _('Transferring books to device...'))
i, total = 0, len(files)
routing = {fmt:dest for fmt,dest in self.get_pref('rules')}
for infile, fname, mi in izip(files, names, metadata):
path = self.create_upload_path(prefix, mi, fname, routing)
if path and self.is_folder_ignored(storage, path):
raise MTPInvalidSendPathError('/'.join(path))
parent = self.ensure_parent(storage, path)
if hasattr(infile, 'read'):
pos = infile.tell()
infile.seek(0, 2)
sz = infile.tell()
infile.seek(pos)
stream = infile
close = False
else:
sz = os.path.getsize(infile)
stream = lopen(infile, 'rb')
close = True
try:
mtp_file = self.put_file(parent, path[-1], stream, sz)
finally:
if close:
stream.close()
ans.append((mtp_file, bl_idx))
i += 1
self.report_progress(i/total, _('Transferred %s to device')%mi.title)
self.report_progress(1, _('Transfer to device finished...'))
debug('upload_books() ended')
return ans
def add_books_to_metadata(self, mtp_files, metadata, booklists):
debug('add_books_to_metadata() called')
from calibre.devices.mtp.books import Book
i, total = 0, len(mtp_files)
self.report_progress(0, _('Adding books to device metadata listing...'))
for x, mi in izip(mtp_files, metadata):
mtp_file, bl_idx = x
bl = booklists[bl_idx]
book = Book(mtp_file.storage_id, '/'.join(mtp_file.mtp_relpath),
other=mi)
book = bl.add_book(book, replace_metadata=True)
if book is not None:
book.size = mtp_file.size
book.datetime = mtp_file.last_modified.timetuple()
book.path = mtp_file.mtp_id_path
i += 1
self.report_progress(i/total, _('Added %s')%mi.title)
self.report_progress(1, _('Adding complete'))
debug('add_books_to_metadata() ended')
def add_books_to_metadata(self, mtp_files, metadata, booklists):
debug('add_books_to_metadata() called')
from calibre.devices.mtp.books import Book
i, total = 0, len(mtp_files)
self.report_progress(0, _('Adding books to device metadata listing...'))
for x, mi in zip(mtp_files, metadata):
mtp_file, bl_idx = x
bl = booklists[bl_idx]
book = Book(mtp_file.storage_id, '/'.join(mtp_file.mtp_relpath),
other=mi)
book = bl.add_book(book, replace_metadata=True)
if book is not None:
book.size = mtp_file.size
book.datetime = mtp_file.last_modified.timetuple()
book.path = mtp_file.mtp_id_path
i += 1
self.report_progress(i/total, _('Added %s')%mi.title)
self.report_progress(1, _('Adding complete'))
debug('add_books_to_metadata() ended')
def filesystem_cache(self):
if self._filesystem_cache is None:
debug('Loading filesystem metadata...')
st = time.time()
from calibre.devices.mtp.filesystem_cache import FilesystemCache
ts = self.total_space()
all_storage = []
items = []
for storage_id, capacity in zip(
[self._main_id, self._carda_id, self._cardb_id], ts):
if storage_id is None:
continue
name = _('Unknown')
for s in self.dev.data['storage']:
if s['id'] == storage_id:
name = s['name']
break
storage = {
'id': storage_id,
'size': capacity,
'name': name,
'is_folder': True,
'can_delete': False,
'is_system': True
}
self._currently_getting_sid = unicode_type(storage_id)
id_map = self.dev.get_filesystem(
storage_id, partial(self._filesystem_callback, {}))
for x in itervalues(id_map):
x['storage_id'] = storage_id
all_storage.append(storage)
items.append(itervalues(id_map))
self._filesystem_cache = FilesystemCache(all_storage,
chain(*items))
debug('Filesystem metadata loaded in %g seconds (%d objects)' %
(time.time() - st, len(self._filesystem_cache)))
return self._filesystem_cache
def filesystem_cache(self):
if self._filesystem_cache is None:
st = time.time()
debug('Loading filesystem metadata...')
from calibre.devices.mtp.filesystem_cache import FilesystemCache
with self.lock:
storage, all_items, all_errs = [], [], []
for sid, capacity in zip([self._main_id, self._carda_id,
self._cardb_id], self.total_space()):
if sid is None:
continue
name = _('Unknown')
for x in self.dev.storage_info:
if x['id'] == sid:
name = x['name']
break
storage.append({'id':sid, 'size':capacity,
'is_folder':True, 'name':name, 'can_delete':False,
'is_system':True})
self._currently_getting_sid = unicode_type(sid)
items, errs = self.dev.get_filesystem(sid,
partial(self._filesystem_callback, {}))
all_items.extend(items), all_errs.extend(errs)
if not all_items and all_errs:
raise DeviceError(
'Failed to read filesystem from %s with errors: %s'
%(self.current_friendly_name,
self.format_errorstack(all_errs)))
if all_errs:
prints('There were some errors while getting the '
' filesystem from %s: %s'%(
self.current_friendly_name,
self.format_errorstack(all_errs)))
self._filesystem_cache = FilesystemCache(storage, all_items)
debug('Filesystem metadata loaded in %g seconds (%d objects)'%(
time.time()-st, len(self._filesystem_cache)))
return self._filesystem_cache
def books(self, oncard=None, end_session=True):
from calibre.devices.mtp.books import JSONCodec
from calibre.devices.mtp.books import BookList, Book
self.report_progress(0, _('Listing files, this can take a while'))
self.get_driveinfo() # Ensure driveinfo is loaded
sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(oncard,
self._main_id)
if sid is None:
return BookList(None)
bl = BookList(sid)
# If True then there is a mismatch between the ebooks on the device and
# the metadata cache
need_sync = False
all_books = list(self.filesystem_cache.iterebooks(sid))
steps = len(all_books) + 2
count = 0
self.report_progress(0, _('Reading e-book metadata'))
# Read the cache if it exists
storage = self.filesystem_cache.storage(sid)
cache = storage.find_path(self.calibre_file_paths['metadata'].split('/'))
if cache is not None:
json_codec = JSONCodec()
try:
stream = self.get_mtp_file(cache)
json_codec.decode_from_file(stream, bl, Book, sid)
except:
need_sync = True
relpath_cache = {b.mtp_relpath:i for i, b in enumerate(bl)}
for mtp_file in all_books:
count += 1
relpath = mtp_file.mtp_relpath
idx = relpath_cache.get(relpath, None)
if idx is not None:
cached_metadata = bl[idx]
del relpath_cache[relpath]
if cached_metadata.size == mtp_file.size:
cached_metadata.datetime = mtp_file.last_modified.timetuple()
cached_metadata.path = mtp_file.mtp_id_path
debug('Using cached metadata for',
'/'.join(mtp_file.full_path))
continue # No need to update metadata
book = cached_metadata
else:
book = Book(sid, '/'.join(relpath))
bl.append(book)
need_sync = True
self.report_progress(count/steps, _('Reading metadata from %s')%
('/'.join(relpath)))
try:
book.smart_update(self.read_file_metadata(mtp_file))
debug('Read metadata for', '/'.join(mtp_file.full_path))
except:
prints('Failed to read metadata from',
'/'.join(mtp_file.full_path))
traceback.print_exc()
book.size = mtp_file.size
book.datetime = mtp_file.last_modified.timetuple()
book.path = mtp_file.mtp_id_path
# Remove books in the cache that no longer exist
for idx in sorted(relpath_cache.itervalues(), reverse=True):
del bl[idx]
need_sync = True
if need_sync:
self.report_progress(count/steps, _('Updating metadata cache on device'))
self.write_metadata_cache(storage, bl)
self.report_progress(1, _('Finished reading metadata from device'))
return bl
