def _clean_snapshot(request, device):
"""remove calibre book ids from snapshot uploads"""
# the snapshot upload tells the upstream server all the books that are on the device
# but it includes all books, not only the ones it thinks it got from the Amazon service
# so we update the set of books that we know the device has
# but don't update the list of books it should remove from the device if they are not found in the calibre library
was_updated = False
books_on_device = set()
lines = []
for line in request.body_text().splitlines(True):
if line.startswith(b'Type=EBOK,Key=') and len(line) == 50 + (line[-1] == ord('\n')):
asin = str(line[14:50], 'ascii') # SHOULD be uuid
if is_uuid(asin, 'EBOK'):
books_on_device.add(asin)
if features.scrub_uploads:
was_updated = True
continue
elif line.startswith(b'Type=PDOC,Key=') and len(line) == 46 + (line[-1] == ord('\n')):
asin = str(line[14:46], 'ascii')
if is_uuid(asin, 'PDOC'):
books_on_device.add(asin)
if features.scrub_uploads:
was_updated = True
continue
lines.append(line)
if books_on_device:
postprocess.enqueue(_process_books_on_device, device, books_on_device)
if was_updated and not request.is_signed():
request.update_body(b''.join(lines))
return was_updated
def _process_xml(request, doc, device):
books_on_device = set()
book_nodes = []
was_modified = False
x_annotations = qxml.get_child(doc, 'annotations')
for x_book in qxml.list_children(x_annotations, 'book'):
asin = x_book.getAttribute('key')
if is_uuid(asin, x_book.getAttribute('type')):
books_on_device.add(asin)
x_annotations.removeChild(x_book)
book_nodes.append(x_book)
was_modified = True
for x_collection in qxml.iter_children(x_annotations, 'collection'):
for x_book in qxml.iter_children(x_collection, 'book'):
asin = x_book.getAttribute('id')
if is_uuid(asin, x_book.getAttribute('type')) and 'add' == x_book.getAttribute('action'):
books_on_device.add(asin)
if books_on_device or book_nodes:
postprocess.enqueue(_process_sidecar_upload, device, books_on_device, book_nodes)
if was_modified:
qxml.remove_whitespace(x_annotations)
if not len(x_annotations.childNodes):
doc.removeChild(x_annotations)
if not len(doc.childNodes):
# there's no more content relevant to Amazon, just reply with an empty 200
raise ExceptionResponse()
return was_modified
def _process_item(device,
action=None,
cde_type=None,
key=None,
complete_status=None,
**extra):
if key.startswith('KSP.') or cde_type.startswith('KSP.'):
# KSP internal stuff, not relevant upstream
return True
if not features.allow_logs_upload and action == 'SND' and cde_type == 'CMND' and key.endswith(
':SYSLOG:UPLOAD'):
return True
if action in ('GET', 'DOWNLOAD') and cde_type in ('EBOK', 'PDOC',
'APNX') and is_uuid(
key, cde_type):
book = calibre.book(key)
if complete_status == 'COMPLETED':
if book:
book.mark_downloaded_by(device)
else:
logging.warn("%s successfully updated unknown book %s", device,
key)
elif complete_status == 'FAILED':
logging.warn("%s failed to update book %s", device, book or key)
else:
logging.warn("%s: unknown downloaded status %s for %s", device,
complete_status, book or key)
return True
if action == 'UPD_LPRD' and is_uuid(key, cde_type):
if complete_status == 'COMPLETED':
annotations.last_read_updated(device, key)
elif complete_status == 'FAILED':
logging.warn("%s failed to update last_read for book %s", device,
key)
else:
logging.warn("%s: unknown UPD_LPRD status %s for book %s", device,
complete_status, key)
return True
if action == 'UPD_ANOT' and is_uuid(key, cde_type):
if complete_status == 'COMPLETED':
annotations.annotations_updated(device, key)
elif complete_status == 'FAILED':
logging.warn("%s failed to update last_read for book %s", device,
key)
else:
logging.warn("%s: unknown UPD_LPRD status %s for book %s", device,
complete_status, key)
return True
return False
def call(self, request, device):
if device.is_provisional():
return None
if request.command == 'GET':
q = request.get_query_params()
asin = q.get('key')
if is_uuid(asin, q.get('type')):
book = calibre.book(asin)
if not book:
logging.warn("tried to download sidecar for unknown book %s", asin)
return None
sidecar_data = formats.sidecar(book)
if sidecar_data:
content_type, data = sidecar_data
return DummyResponse(headers = { 'Content-Type': content_type }, data = data)
return None
elif request.command == 'POST':
q = request.get_query_params()
lto = q.get('device_lto', -1)
if lto != -1:
try: device.lto = int(lto)
except: pass
with minidom.parseString(request.body_text()) as doc:
if _process_xml(request, doc, device):
if not request.is_signed():
# drats
xml = doc.toxml('UTF-8')
request.update_body(xml)
return self.call_upstream(request, device)
def call(self, request, device):
q = query_params(request.body_text())
if is_uuid(q.get('key'), q.get('type')):
logging.warn("sharing annotations for Calibre books is not supported")
return 200
return self.call_upstream(request, device)
def _clean_snapshot(request, device):
"""remove calibre book ids from snapshot uploads"""
# the snapshot upload tells the upstream server all the books that are on the device
# but it includes all books, not only the ones it thinks it got from the Amazon service
# so we update the set of books that we know the device has
# but don't update the list of books it should remove from the device if they are not found in the calibre library
was_updated = False
books_on_device = set()
lines = []
for line in request.body_text().splitlines(True):
if line.startswith(b'Type=EBOK,Key='):
asin = str(line[14:].strip(), 'ascii')
if is_uuid(asin, 'EBOK'):
books_on_device.add(asin)
if features.scrub_uploads:
was_updated = True
continue
elif line.startswith(b'Type=PDOC,Key='):
asin = str(line[14:].strip(), 'ascii')
if is_uuid(asin, 'PDOC'):
books_on_device.add(asin)
if features.scrub_uploads:
was_updated = True
continue
elif line.startswith(b'ASIN='): # Android
asin = str(line[5:].strip(), 'ascii')
if is_uuid(asin):
books_on_device.add(asin)
if features.scrub_uploads:
was_updated = True
continue
lines.append(line)
if books_on_device:
postprocess.enqueue(_process_books_on_device, device, books_on_device)
if was_updated and not request.is_signed():
request.update_body(b''.join(lines))
return was_updated
def _process_item(device, action = None, cde_type = None, key = None, complete_status = None, **extra):
if key.startswith('KSP.') or cde_type.startswith('KSP.'):
# KSP internal stuff, not relevant upstream
return True
if not features.allow_logs_upload and action == 'SND' and cde_type == 'CMND' and key.endswith(':SYSLOG:UPLOAD'):
return True
if action in ('GET', 'DOWNLOAD') and cde_type in ('EBOK', 'PDOC', 'APNX') and is_uuid(key, cde_type):
book = calibre.book(key)
if complete_status == 'COMPLETED':
if book:
book.mark_downloaded_by(device)
else:
logging.warn("%s successfully updated unknown book %s", device, key)
elif complete_status == 'FAILED':
logging.warn("%s failed to update book %s", device, book or key)
else:
logging.warn("%s: unknown downloaded status %s for %s", device, complete_status, book or key)
return True
if action == 'UPD_LPRD' and is_uuid(key, cde_type):
if complete_status == 'COMPLETED':
annotations.last_read_updated(device, key)
elif complete_status == 'FAILED':
logging.warn("%s failed to update last_read for book %s", device, key)
else:
logging.warn("%s: unknown UPD_LPRD status %s for book %s", device, complete_status, key)
return True
if action == 'UPD_ANOT' and is_uuid(key, cde_type):
if complete_status == 'COMPLETED':
annotations.annotations_updated(device, key)
elif complete_status == 'FAILED':
logging.warn("%s failed to update last_read for book %s", device, key)
else:
logging.warn("%s: unknown UPD_LPRD status %s for book %s", device, complete_status, key)
return True
return False
def _filter_item(x_items, x_item):
action = x_item.getAttribute('action')
item_type = x_item.getAttribute('type')
if action == 'UPLOAD':
if item_type in ['MESG', 'LOGS'] and not features.allow_logs_upload:
x_items.removeChild(x_item)
return True
item_url = x_item.getAttribute('url')
new_url = _rewrite_url(item_url)
if new_url != item_url:
logging.warn("rewrote url %s => %s", item_url, new_url)
x_item.setAttribute('url', new_url)
return True
return False
if action == 'DOWNLOAD':
item_key = x_item.getAttribute('key')
item_url = x_item.getAttribute('url')
if item_url and (item_type == 'CRED' or is_uuid(item_key)):
new_url = _rewrite_url(item_url)
if new_url != item_url:
logging.warn("rewrote url for %s: %s => %s", item_key, item_url, new_url)
x_item.setAttribute('url', new_url)
return True
logging.warn("not rewriting url %s for %s", item_url, item_key)
return False
if action == 'GET':
if item_type == 'FWUP' and not features.allow_firmware_updates:
x_items.removeChild(x_items)
return True
if action == 'SND' and item_type == 'CMND':
item_key = x_item.getAttribute('key')
if item_key and item_key.endswith(':SYSLOG:UPLOAD') and not features.allow_logs_upload:
# not sure if this is smart, ignoring these items appears to queue them up at amazon
x_items.removeChild(x_item)
return True
# very unlikely for these to change upstream for books not downloaded from Amazon...
# if action == 'UPD_ANOT' or action == 'UPD_LPRD':
# # annotations and LPRD (last position read?)
# item_key = x_item.getAttribute('key')
# if is_uuid(item_key):
# x_items.removeChild(x_item)
# return True
return False
def call(self, request, device):
if device.is_provisional():
return None
q = request.get_query_params()
asin = q.get('ASIN')
if not asin:
return 400
if is_uuid(asin): # yay
return process(self, request, device)
del request.headers['Referer']
request.headers['Referer'] = 'https://www.amazon.com'
return self.call_upstream(request, device)
def call(self, request, device):
q = request.get_query_params()
asin = q.get('key')
if q.get('type') == 'EBOK' and is_uuid(asin, 'EBOK'):
book = calibre.book(asin)
apnx_path = annotations.apnx_path(book)
# logging.debug("looking for apnx of %s, found %s", book, apnx_path)
if apnx_path:
# APNX files are usually small (a few Ks at most),
# so there's no need to do stream copying
apnx_data = None
with open(apnx_path, 'rb') as apnx:
apnx_data = apnx.read()
if apnx_data:
return DummyResponse(headers=_HEADERS, data=apnx_data)
def call(self, request, device):
q = request.get_query_params()
cde_type = q.get('type')
if 'key' in q and cde_type in ('EBOK', 'PDOC'):
key = q['key']
if is_uuid(key, cde_type): # very likely comes from our library
return self.book_response(key, device, request.headers['Range'])
if device.is_provisional():
return None
if request.is_signed():
redirect_header = { 'Location': 'https://cde-ta-g7g.amazon.com' + request.path }
else:
redirect_header = { 'Location': 'https://cde-g7g.amazon.com' + request.path }
return DummyResponse(302, redirect_header)
def call(self, request, device):
q = request.get_query_params()
asin = q.get('key')
if is_uuid(asin, q.get('type')):
kind = q.get('filter')
book = calibre.book(asin)
if not book:
logging.warn("book not found %s", asin)
return None
if kind == 'last_read':
return _last_read(book, device.serial)
logging.warn("don't know how to filter annotations of kind %s", kind)
return None
return self.call_upstream(request, device)
def call(self, request, device):
q = request.get_query_params()
asin = q.get('key')
if is_uuid(asin, q.get('type')):
kind = q.get('filter')
book = calibre.book(asin)
if not book:
logging.warn("book not found %s", asin)
return None
if kind == 'last_read':
return _last_read(book, device.serial)
logging.warn("don't know how to filter annotations of kind %s",
kind)
return None
return self.call_upstream(request, device)
def _filter_item(request, x_items, x_item):
action = x_item.getAttribute('action')
item_type = x_item.getAttribute('type')
if action == 'UPLOAD':
if item_type in ['MESG', 'LOGS'] and not features.allow_logs_upload:
x_items.removeChild(x_item)
return True
item_url = x_item.getAttribute('url')
new_url = _rewrite_url(request, item_url)
if new_url != item_url:
logging.warn("rewrote url %s => %s", item_url, new_url)
x_item.setAttribute('url', new_url)
return True
return False
if action == 'DOWNLOAD':
item_key = x_item.getAttribute('key')
item_url = x_item.getAttribute('url')
if item_url and (item_type == 'CRED' or is_uuid(item_key)):
new_url = _rewrite_url(request, item_url)
if new_url != item_url:
logging.warn("rewrote url for %s: %s => %s", item_key,
item_url, new_url)
x_item.setAttribute('url', new_url)
return True
logging.warn("not rewriting url %s for %s", item_url, item_key)
return False
if action == 'GET':
if item_type == 'FWUP' and not features.allow_firmware_updates:
x_items.removeChild(x_items)
return True
if action == 'SND' and item_type == 'CMND':
item_key = x_item.getAttribute('key')
if item_key and item_key.endswith(
':SYSLOG:UPLOAD') and not features.allow_logs_upload:
# not sure if this is smart, ignoring these items appears to queue them up at amazon
x_items.removeChild(x_item)
return True
return False
def call(self, request, device):
if request.path.startswith('/images/P/'):
asin = asin[10:asin.find('.', 11)]
if is_uuid(asin):
book = calibre.book(asin)
# logging.debug("looking for cover of %s %s", asin, book)
if book and book.file_path:
image_path = os.path.join(os.path.dirname(book.file_path), "cover.jpg")
image_data = None
if os.path.isfile(image_path):
try:
with open(image_path, 'rb') as fi:
image_data = fi.read()
except:
logging.exception("failed to read cover for %s", book)
if image_data:
return DummyResponse(headers = _HEADERS, data = image_data)
return 404 # forces Kindle 4 PC to use the cover from the .mobi file, if any
return DummyResponse(303, headers = { 'Location: http://' + _ECX + request.path })
def call(self, request, device):
q = request.get_query_params()
cde_type = q.get('type')
if 'key' in q and cde_type in ('EBOK', 'PDOC'):
key = q['key']
if is_uuid(key, cde_type): # very likely comes from our library
return self.book_response(key, device, request.headers['Range'])
if device.is_provisional():
return None
if request.is_secure():
if request.is_signed():
redirect_header = { 'Location': 'https://cde-ta-g7g.amazon.com' + request.path }
else:
redirect_header = { 'Location': 'https://cde-g7g.amazon.com' + request.path }
return DummyResponse(302, redirect_header)
# the request was made over http, we'll have to download the file ourselves
return self.call_upstream(request, device)
def _filter_item(x_items, x_item):
action = x_item.getAttribute('action')
item_type = x_item.getAttribute('type')
if action == 'UPLOAD':
if item_type in ['MESG', 'LOGS'] and not features.allow_logs_upload:
x_items.removeChild(x_item)
return True
item_url = x_item.getAttribute('url')
new_url = _rewrite_url(item_url)
if new_url != item_url:
logging.warn("rewrote url %s => %s", item_url, new_url)
x_item.setAttribute('url', new_url)
return True
return False
if action == 'DOWNLOAD':
item_key = x_item.getAttribute('key')
item_url = x_item.getAttribute('url')
if item_url and (item_type == 'CRED' or is_uuid(item_key)):
new_url = _rewrite_url(item_url)
if new_url != item_url:
logging.warn("rewrote url for %s: %s => %s", item_key, item_url, new_url)
x_item.setAttribute('url', new_url)
return True
logging.warn("not rewriting url %s for %s", item_url, item_key)
return False
if action == 'GET':
if item_type == 'FWUP' and not features.allow_firmware_updates:
x_items.removeChild(x_items)
return True
if action == 'SND' and item_type == 'CMND':
item_key = x_item.getAttribute('key')
if item_key and item_key.endswith(':SYSLOG:UPLOAD') and not features.allow_logs_upload:
# not sure if this is smart, ignoring these items appears to queue them up at amazon
x_items.removeChild(x_item)
return True
return False
def call(self, request, device):
if request.path.startswith('/images/P/'):
asin = request.path[10:]
asin = asin[:asin.find('.', 11)]
if is_uuid(asin):
book = calibre.book(asin)
# logging.debug("looking for cover of %s %s", asin, book)
if book and book.file_path:
image_path = os.path.join(os.path.dirname(book.file_path),
"cover.jpg")
image_data = None
if os.path.isfile(image_path):
try:
with open(image_path, 'rb') as fi:
image_data = fi.read()
except:
logging.exception("failed to read cover for %s",
book)
if image_data:
return DummyResponse(headers=_HEADERS, data=image_data)
return 404 # forces Kindle 4 PC to use the cover from the .mobi file, if any
return DummyResponse(
303, headers={'Location: http://' + _ECX + request.path})