Beispiel #1
0
def _clean_snapshot(request, device):
	"""remove calibre book ids from snapshot uploads"""

	# the snapshot upload tells the upstream server all the books that are on the device
	# but it includes all books, not only the ones it thinks it got from the Amazon service
	# so we update the set of books that we know the device has
	# but don't update the list of books it should remove from the device if they are not found in the calibre library

	was_updated = False
	books_on_device = set()

	lines = []
	for line in request.body_text().splitlines(True):
		if line.startswith(b'Type=EBOK,Key=') and len(line) == 50 + (line[-1] == ord('\n')):
			asin = str(line[14:50], 'ascii') # SHOULD be uuid
			if is_uuid(asin, 'EBOK'):
				books_on_device.add(asin)
				if features.scrub_uploads:
					was_updated = True
					continue
		elif line.startswith(b'Type=PDOC,Key=') and len(line) == 46 + (line[-1] == ord('\n')):
			asin = str(line[14:46], 'ascii')
			if is_uuid(asin, 'PDOC'):
				books_on_device.add(asin)
				if features.scrub_uploads:
					was_updated = True
					continue
		lines.append(line)

	if books_on_device:
		postprocess.enqueue(_process_books_on_device, device, books_on_device)

	if was_updated and not request.is_signed():
		request.update_body(b''.join(lines))
	return was_updated
Beispiel #2
0
def _process_xml(request, doc, device):
	books_on_device = set()
	book_nodes = []
	was_modified = False

	x_annotations = qxml.get_child(doc, 'annotations')

	for x_book in qxml.list_children(x_annotations, 'book'):
		asin = x_book.getAttribute('key')
		if is_uuid(asin, x_book.getAttribute('type')):
			books_on_device.add(asin)
			x_annotations.removeChild(x_book)
			book_nodes.append(x_book)
			was_modified = True

	for x_collection in qxml.iter_children(x_annotations, 'collection'):
		for x_book in qxml.iter_children(x_collection, 'book'):
			asin = x_book.getAttribute('id')
			if is_uuid(asin, x_book.getAttribute('type')) and 'add' == x_book.getAttribute('action'):
				books_on_device.add(asin)

	if books_on_device or book_nodes:
		postprocess.enqueue(_process_sidecar_upload, device, books_on_device, book_nodes)

	if was_modified:
		qxml.remove_whitespace(x_annotations)
		if not len(x_annotations.childNodes):
			doc.removeChild(x_annotations)
		if not len(doc.childNodes):
			# there's no more content relevant to Amazon, just reply with an empty 200
			raise ExceptionResponse()

	return was_modified
Beispiel #3
0
def _process_item(device,
                  action=None,
                  cde_type=None,
                  key=None,
                  complete_status=None,
                  **extra):
    if key.startswith('KSP.') or cde_type.startswith('KSP.'):
        # KSP internal stuff, not relevant upstream
        return True

    if not features.allow_logs_upload and action == 'SND' and cde_type == 'CMND' and key.endswith(
            ':SYSLOG:UPLOAD'):
        return True

    if action in ('GET', 'DOWNLOAD') and cde_type in ('EBOK', 'PDOC',
                                                      'APNX') and is_uuid(
                                                          key, cde_type):
        book = calibre.book(key)
        if complete_status == 'COMPLETED':
            if book:
                book.mark_downloaded_by(device)
            else:
                logging.warn("%s successfully updated unknown book %s", device,
                             key)
        elif complete_status == 'FAILED':
            logging.warn("%s failed to update book %s", device, book or key)
        else:
            logging.warn("%s: unknown downloaded status %s for %s", device,
                         complete_status, book or key)
        return True

    if action == 'UPD_LPRD' and is_uuid(key, cde_type):
        if complete_status == 'COMPLETED':
            annotations.last_read_updated(device, key)
        elif complete_status == 'FAILED':
            logging.warn("%s failed to update last_read for book %s", device,
                         key)
        else:
            logging.warn("%s: unknown UPD_LPRD status %s for book %s", device,
                         complete_status, key)
        return True

    if action == 'UPD_ANOT' and is_uuid(key, cde_type):
        if complete_status == 'COMPLETED':
            annotations.annotations_updated(device, key)
        elif complete_status == 'FAILED':
            logging.warn("%s failed to update last_read for book %s", device,
                         key)
        else:
            logging.warn("%s: unknown UPD_LPRD status %s for book %s", device,
                         complete_status, key)
        return True

    return False
Beispiel #4
0
	def call(self, request, device):
		if device.is_provisional():
			return None

		if request.command == 'GET':
			q = request.get_query_params()
			asin = q.get('key')
			if is_uuid(asin, q.get('type')):
				book = calibre.book(asin)
				if not book:
					logging.warn("tried to download sidecar for unknown book %s", asin)
					return None
				sidecar_data = formats.sidecar(book)
				if sidecar_data:
					content_type, data = sidecar_data
					return DummyResponse(headers = { 'Content-Type': content_type }, data = data)
				return None
		elif request.command == 'POST':
			q = request.get_query_params()
			lto = q.get('device_lto', -1)
			if lto != -1:
				try: device.lto = int(lto)
				except: pass

			with minidom.parseString(request.body_text()) as doc:
				if _process_xml(request, doc, device):
					if not request.is_signed():
						# drats
						xml = doc.toxml('UTF-8')
						request.update_body(xml)

		return self.call_upstream(request, device)
Beispiel #5
0
	def call(self, request, device):
		q = query_params(request.body_text())
		if is_uuid(q.get('key'), q.get('type')):
			logging.warn("sharing annotations for Calibre books is not supported")
			return 200

		return self.call_upstream(request, device)
Beispiel #6
0
def _clean_snapshot(request, device):
    """remove calibre book ids from snapshot uploads"""

    # the snapshot upload tells the upstream server all the books that are on the device
    # but it includes all books, not only the ones it thinks it got from the Amazon service
    # so we update the set of books that we know the device has
    # but don't update the list of books it should remove from the device if they are not found in the calibre library

    was_updated = False
    books_on_device = set()

    lines = []
    for line in request.body_text().splitlines(True):
        if line.startswith(b'Type=EBOK,Key='):
            asin = str(line[14:].strip(), 'ascii')
            if is_uuid(asin, 'EBOK'):
                books_on_device.add(asin)
                if features.scrub_uploads:
                    was_updated = True
                    continue
        elif line.startswith(b'Type=PDOC,Key='):
            asin = str(line[14:].strip(), 'ascii')
            if is_uuid(asin, 'PDOC'):
                books_on_device.add(asin)
                if features.scrub_uploads:
                    was_updated = True
                    continue
        elif line.startswith(b'ASIN='):  # Android
            asin = str(line[5:].strip(), 'ascii')
            if is_uuid(asin):
                books_on_device.add(asin)
                if features.scrub_uploads:
                    was_updated = True
                    continue
        lines.append(line)

    if books_on_device:
        postprocess.enqueue(_process_books_on_device, device, books_on_device)

    if was_updated and not request.is_signed():
        request.update_body(b''.join(lines))
    return was_updated
Beispiel #7
0
def _process_item(device, action = None, cde_type = None, key = None, complete_status = None, **extra):
	if key.startswith('KSP.') or cde_type.startswith('KSP.'):
		# KSP internal stuff, not relevant upstream
		return True

	if not features.allow_logs_upload and action == 'SND' and cde_type == 'CMND' and key.endswith(':SYSLOG:UPLOAD'):
		return True

	if action in ('GET', 'DOWNLOAD') and cde_type in ('EBOK', 'PDOC', 'APNX') and is_uuid(key, cde_type):
		book = calibre.book(key)
		if complete_status == 'COMPLETED':
			if book:
				book.mark_downloaded_by(device)
			else:
				logging.warn("%s successfully updated unknown book %s", device, key)
		elif complete_status == 'FAILED':
			logging.warn("%s failed to update book %s", device, book or key)
		else:
			logging.warn("%s: unknown downloaded status %s for %s", device, complete_status, book or key)
		return True

	if action == 'UPD_LPRD' and is_uuid(key, cde_type):
		if complete_status == 'COMPLETED':
			annotations.last_read_updated(device, key)
		elif complete_status == 'FAILED':
			logging.warn("%s failed to update last_read for book %s", device, key)
		else:
			logging.warn("%s: unknown UPD_LPRD status %s for book %s", device, complete_status, key)
		return True

	if action == 'UPD_ANOT' and is_uuid(key, cde_type):
		if complete_status == 'COMPLETED':
			annotations.annotations_updated(device, key)
		elif complete_status == 'FAILED':
			logging.warn("%s failed to update last_read for book %s", device, key)
		else:
			logging.warn("%s: unknown UPD_LPRD status %s for book %s", device, complete_status, key)
		return True

	return False
Beispiel #8
0
def _filter_item(x_items, x_item):
	action = x_item.getAttribute('action')
	item_type = x_item.getAttribute('type')

	if action == 'UPLOAD':
		if item_type in ['MESG', 'LOGS'] and not features.allow_logs_upload:
			x_items.removeChild(x_item)
			return True
		item_url = x_item.getAttribute('url')
		new_url = _rewrite_url(item_url)
		if new_url != item_url:
			logging.warn("rewrote url %s => %s", item_url, new_url)
			x_item.setAttribute('url', new_url)
			return True
		return False

	if action == 'DOWNLOAD':
		item_key = x_item.getAttribute('key')
		item_url = x_item.getAttribute('url')
		if item_url and (item_type == 'CRED' or is_uuid(item_key)):
			new_url = _rewrite_url(item_url)
			if new_url != item_url:
				logging.warn("rewrote url for %s: %s => %s", item_key, item_url, new_url)
				x_item.setAttribute('url', new_url)
				return True
		logging.warn("not rewriting url %s for %s", item_url, item_key)
		return False

	if action == 'GET':
		if item_type == 'FWUP' and not features.allow_firmware_updates:
			x_items.removeChild(x_items)
			return True

	if action == 'SND' and item_type == 'CMND':
		item_key = x_item.getAttribute('key')
		if item_key and item_key.endswith(':SYSLOG:UPLOAD') and not features.allow_logs_upload:
			# not sure if this is smart, ignoring these items appears to queue them up at amazon
			x_items.removeChild(x_item)
			return True

	# very unlikely for these to change upstream for books not downloaded from Amazon...
	# if action == 'UPD_ANOT' or action == 'UPD_LPRD':
	# 	# annotations and LPRD (last position read?)
	# 	item_key = x_item.getAttribute('key')
	# 	if is_uuid(item_key):
	# 		x_items.removeChild(x_item)
	# 		return True

	return False
Beispiel #9
0
	def call(self, request, device):
		if device.is_provisional():
			return None

		q = request.get_query_params()
		asin = q.get('ASIN')
		if not asin:
			return 400

		if is_uuid(asin): # yay
			return process(self, request, device)

		del request.headers['Referer']
		request.headers['Referer'] = 'https://www.amazon.com'
		return self.call_upstream(request, device)
Beispiel #10
0
 def call(self, request, device):
     q = request.get_query_params()
     asin = q.get('key')
     if q.get('type') == 'EBOK' and is_uuid(asin, 'EBOK'):
         book = calibre.book(asin)
         apnx_path = annotations.apnx_path(book)
         # logging.debug("looking for apnx of %s, found %s", book, apnx_path)
         if apnx_path:
             # APNX files are usually small (a few Ks at most),
             # so there's no need to do stream copying
             apnx_data = None
             with open(apnx_path, 'rb') as apnx:
                 apnx_data = apnx.read()
             if apnx_data:
                 return DummyResponse(headers=_HEADERS, data=apnx_data)
Beispiel #11
0
	def call(self, request, device):
		q = request.get_query_params()
		cde_type = q.get('type')
		if 'key' in q and cde_type in ('EBOK', 'PDOC'):
			key = q['key']
			if is_uuid(key, cde_type): # very likely comes from our library
				return self.book_response(key, device, request.headers['Range'])

		if device.is_provisional():
			return None

		if request.is_signed():
			redirect_header = { 'Location': 'https://cde-ta-g7g.amazon.com' + request.path }
		else:
			redirect_header = { 'Location': 'https://cde-g7g.amazon.com' + request.path }
		return DummyResponse(302, redirect_header)
Beispiel #12
0
	def call(self, request, device):
		q = request.get_query_params()
		asin = q.get('key')

		if is_uuid(asin, q.get('type')):
			kind = q.get('filter')
			book = calibre.book(asin)
			if not book:
				logging.warn("book not found %s", asin)
				return None
			if kind == 'last_read':
				return _last_read(book, device.serial)
			logging.warn("don't know how to filter annotations of kind %s", kind)
			return None

		return self.call_upstream(request, device)
Beispiel #13
0
    def call(self, request, device):
        q = request.get_query_params()
        asin = q.get('key')

        if is_uuid(asin, q.get('type')):
            kind = q.get('filter')
            book = calibre.book(asin)
            if not book:
                logging.warn("book not found %s", asin)
                return None
            if kind == 'last_read':
                return _last_read(book, device.serial)
            logging.warn("don't know how to filter annotations of kind %s",
                         kind)
            return None

        return self.call_upstream(request, device)
Beispiel #14
0
def _filter_item(request, x_items, x_item):
    action = x_item.getAttribute('action')
    item_type = x_item.getAttribute('type')

    if action == 'UPLOAD':
        if item_type in ['MESG', 'LOGS'] and not features.allow_logs_upload:
            x_items.removeChild(x_item)
            return True
        item_url = x_item.getAttribute('url')
        new_url = _rewrite_url(request, item_url)
        if new_url != item_url:
            logging.warn("rewrote url %s => %s", item_url, new_url)
            x_item.setAttribute('url', new_url)
            return True
        return False

    if action == 'DOWNLOAD':
        item_key = x_item.getAttribute('key')
        item_url = x_item.getAttribute('url')
        if item_url and (item_type == 'CRED' or is_uuid(item_key)):
            new_url = _rewrite_url(request, item_url)
            if new_url != item_url:
                logging.warn("rewrote url for %s: %s => %s", item_key,
                             item_url, new_url)
                x_item.setAttribute('url', new_url)
                return True
        logging.warn("not rewriting url %s for %s", item_url, item_key)
        return False

    if action == 'GET':
        if item_type == 'FWUP' and not features.allow_firmware_updates:
            x_items.removeChild(x_items)
            return True

    if action == 'SND' and item_type == 'CMND':
        item_key = x_item.getAttribute('key')
        if item_key and item_key.endswith(
                ':SYSLOG:UPLOAD') and not features.allow_logs_upload:
            # not sure if this is smart, ignoring these items appears to queue them up at amazon
            x_items.removeChild(x_item)
            return True

    return False
Beispiel #15
0
	def call(self, request, device):
		if request.path.startswith('/images/P/'):
			asin = asin[10:asin.find('.', 11)]
			if is_uuid(asin):
				book = calibre.book(asin)
				# logging.debug("looking for cover of %s %s", asin, book)
				if book and book.file_path:
					image_path = os.path.join(os.path.dirname(book.file_path), "cover.jpg")
					image_data = None
					if os.path.isfile(image_path):
						try:
							with open(image_path, 'rb') as fi:
								image_data = fi.read()
						except:
							logging.exception("failed to read cover for %s", book)
					if image_data:
						return DummyResponse(headers = _HEADERS, data = image_data)
				return 404 # forces Kindle 4 PC to use the cover from the .mobi file, if any

		return DummyResponse(303, headers = { 'Location: http://' + _ECX + request.path })
Beispiel #16
0
	def call(self, request, device):
		q = request.get_query_params()
		cde_type = q.get('type')
		if 'key' in q and cde_type in ('EBOK', 'PDOC'):
			key = q['key']
			if is_uuid(key, cde_type): # very likely comes from our library
				return self.book_response(key, device, request.headers['Range'])

		if device.is_provisional():
			return None

		if request.is_secure():
			if request.is_signed():
				redirect_header = { 'Location': 'https://cde-ta-g7g.amazon.com' + request.path }
			else:
				redirect_header = { 'Location': 'https://cde-g7g.amazon.com' + request.path }
			return DummyResponse(302, redirect_header)

		# the request was made over http, we'll have to download the file ourselves
		return self.call_upstream(request, device)
Beispiel #17
0
def _filter_item(x_items, x_item):
	action = x_item.getAttribute('action')
	item_type = x_item.getAttribute('type')

	if action == 'UPLOAD':
		if item_type in ['MESG', 'LOGS'] and not features.allow_logs_upload:
			x_items.removeChild(x_item)
			return True
		item_url = x_item.getAttribute('url')
		new_url = _rewrite_url(item_url)
		if new_url != item_url:
			logging.warn("rewrote url %s => %s", item_url, new_url)
			x_item.setAttribute('url', new_url)
			return True
		return False

	if action == 'DOWNLOAD':
		item_key = x_item.getAttribute('key')
		item_url = x_item.getAttribute('url')
		if item_url and (item_type == 'CRED' or is_uuid(item_key)):
			new_url = _rewrite_url(item_url)
			if new_url != item_url:
				logging.warn("rewrote url for %s: %s => %s", item_key, item_url, new_url)
				x_item.setAttribute('url', new_url)
				return True
		logging.warn("not rewriting url %s for %s", item_url, item_key)
		return False

	if action == 'GET':
		if item_type == 'FWUP' and not features.allow_firmware_updates:
			x_items.removeChild(x_items)
			return True

	if action == 'SND' and item_type == 'CMND':
		item_key = x_item.getAttribute('key')
		if item_key and item_key.endswith(':SYSLOG:UPLOAD') and not features.allow_logs_upload:
			# not sure if this is smart, ignoring these items appears to queue them up at amazon
			x_items.removeChild(x_item)
			return True

	return False
Beispiel #18
0
    def call(self, request, device):
        if request.path.startswith('/images/P/'):
            asin = request.path[10:]
            asin = asin[:asin.find('.', 11)]
            if is_uuid(asin):
                book = calibre.book(asin)
                # logging.debug("looking for cover of %s %s", asin, book)
                if book and book.file_path:
                    image_path = os.path.join(os.path.dirname(book.file_path),
                                              "cover.jpg")
                    image_data = None
                    if os.path.isfile(image_path):
                        try:
                            with open(image_path, 'rb') as fi:
                                image_data = fi.read()
                        except:
                            logging.exception("failed to read cover for %s",
                                              book)
                    if image_data:
                        return DummyResponse(headers=_HEADERS, data=image_data)
                return 404  # forces Kindle 4 PC to use the cover from the .mobi file, if any

        return DummyResponse(
            303, headers={'Location: http://' + _ECX + request.path})