Beispiel #1
0
def download_all_served_ebooks(ip, port):
    # ask for available ebooks
    code, mimetype, result = download_file(ip, port, url(ip, port, "index"))
    if code == requests.codes.ok and mimetype == "text/plain":
        epubs = result.split("|")
        # download them all
        for epub in epubs:
            log(LIBRARIAN_SYNC, "download",
                "D/L: %s." % os.path.basename(epub))
            code, mimetype, result = download_file(ip, port,
                                                   url(ip, port, epub))
            if code == requests.codes.not_found:
                log(LIBRARIAN_SYNC, "download", "%s not found." % epub, "W")
            elif code == requests.codes.ok:
                if mimetype not in ebook_mimetypes:
                    raise Exception("what?")
                else:
                    log(LIBRARIAN_SYNC,
                        "download",
                        "Downloaded: %s." % os.path.basename(epub),
                        display=False)
        # ask for the associated collections
        code, mimetype, result = download_file(
            ip, port, url(ip, port, "collections.json"))
        # shutdown the server, all done
        code, mimetype, result = download_file(
            ip, port, url(ip, port, "LibrarianServer::shutdown"))
        if code == requests.codes.ok and mimetype == "text/plain":
            log(LIBRARIAN_SYNC, "shutdown", result)
    else:
        log(LIBRARIAN_SYNC, "retrieve_index", "Could not retrieve index.")
Beispiel #2
0
def download_file(ip, port, url):
    r = requests.get(url, stream=True)

    if r.headers['content-type'] in ebook_mimetypes or\
       r.headers['content-type'] == "application/json":

        filename = url.split(SERVER_HTTP % (ip, port))[1]
        if filename == "collections.json":
            local_filename = os.path.join(COLLECTIONS_DIR, filename)
        else:
            local_filename = os.path.join(DESTINATION_DIR, filename)
        if not os.path.exists(os.path.dirname(local_filename)):
            log(LIBRARIAN_SYNC, "download_file",
                "Creating %s" % os.path.dirname(local_filename))
            os.makedirs(os.path.dirname(local_filename))
        with open(local_filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                if chunk:  # filter out keep-alive new chunks
                    f.write(chunk)
                    f.flush()
        return r.status_code, r.headers['content-type'], local_filename
    elif r.headers['content-type'] == "text/plain":
        r.encoding = "utf8"
        return r.status_code, r.headers['content-type'], r.text
    else:
        return r.status_code, r.headers['content-type'], None
Beispiel #3
0
 def execute(self):
     if not self.commands:
         log(LIBRARIAN_SYNC, "cc_update", "Nothing to update.")
     else:
         log(LIBRARIAN_SYNC, "cc_update", "Sending commands...")
         full_command = {
             "commands": self.commands,
             "type": "ChangeRequest",
             "id": 1
         }
         # When WiFi's enabled, we inherit the WhisperSync proxy, which we *cannot* go through,
         # since we're talking to a local service. So make sure we do *NOT* use any proxies.
         # Turns out that this is *slightly* tricky to achieve with requests,
         # c.f., https://github.com/requests/requests/issues/879#issuecomment-10001977
         os.environ['no_proxy'] = '127.0.0.1,localhost'
         r = requests.post("http://127.0.0.1:9101/change",
                           data=json.dumps(full_command),
                           headers={
                               'AuthToken': self.session_token,
                               'content-type': 'application/json'
                           },
                           proxies={'no': 'pass'})
         if r.status_code == requests.codes.ok:
             log(LIBRARIAN_SYNC, "cc_update", "Success.")
         else:
             log(LIBRARIAN_SYNC, "cc_update", "Oh, no. It failed.", "E")
Beispiel #4
0
 def build_legacy_hashes_list(self):
     hashes_list = []
     for e in self.original_ebooks:
         # Guard against NULL cdeKeys, which should never happen for books, but have been seen in the wild w/ manually sideloaded stuff...
         if e.cdekey:
             if e.cdekey.startswith('*'):
                 # No ASIN set, we don't care about the cdeType, use it as-is
                 hashes_list.append(e.cdekey)
             else:
                 # Proper or fake ASIN set, build the hash
                 hashes_list.append('#{}^{}'.format(e.cdekey, e.cdetype))
         else:
             log(LIBRARIAN_SYNC, "legacy hash building",
                 "Book %s has no cdeKey?! Skipping it."
                 "(sideloaded book?)" % e.location,
                 "W", display=False)
     return hashes_list
Beispiel #5
0
 def build_legacy_hashes_list(self):
     hashes_list = []
     for e in self.original_ebooks:
         # Guard against NULL cdeKeys, which should never happen for books, but have been seen in the wild w/ manually sideloaded stuff...
         if e.cdekey:
             if e.cdekey.startswith('*'):
                 # No ASIN set, we don't care about the cdeType, use it as-is
                 hashes_list.append(e.cdekey)
             else:
                 # Proper or fake ASIN set, build the hash
                 hashes_list.append('#{}^{}'.format(e.cdekey, e.cdetype))
         else:
             log(LIBRARIAN_SYNC, "legacy hash building",
                 "Book %s has no cdeKey?! Skipping it."
                 "(sideloaded book?)" % e.location,
                 "W", display=False)
     return hashes_list
Beispiel #6
0
def parse_entries(cursor, ignore_empty_collections=False):
    db_ebooks = []
    db_collections = []

    cursor.execute(SELECT_COLLECTION_ENTRIES)
    for (c_uuid, label) in cursor.fetchall():
        db_collections.append(Collection(c_uuid, label))

    cursor.execute(SELECT_EBOOK_ENTRIES)
    for (e_uuid, location, cdekey, cdetype) in cursor.fetchall():
        # only consider user ebooks
        if location is not None and KINDLE_EBOOKS_ROOT in location:
            db_ebooks.append(Ebook(e_uuid, location, cdekey, cdetype))

    cursor.execute(SELECT_EXISTING_COLLECTIONS)
    for (collection_uuid, ebook_uuid) in cursor.fetchall():
        collection_idx = find_collection(db_collections, collection_uuid)
        ebook_idx_list = find_ebook(db_ebooks, ebook_uuid)
        if collection_idx != -1 and ebook_idx_list != []:
            for ebook_idx in ebook_idx_list:
                db_collections[collection_idx].add_ebook(
                    db_ebooks[ebook_idx], True)
                db_ebooks[ebook_idx].add_collection(
                    db_collections[collection_idx], True)
        else:
            log(LIBRARIAN_SYNC,
                "parse_entries",
                "Skipping collection {} (collection_idx: {}, ebook_uuid: {})".
                format(collection_uuid, collection_idx, ebook_uuid),
                "W",
                display=False)

    # remove empty collections:
    if ignore_empty_collections:
        db_collections = [
            c for c in db_collections if len(c.original_ebooks) != 0
        ]

    return db_ebooks, db_collections
Beispiel #7
0
 def execute(self):
     if not self.commands:
         log(LIBRARIAN_SYNC, "cc_update", "Nothing to update.")
     else:
         log(LIBRARIAN_SYNC, "cc_update", "Sending commands...")
         full_command = {"commands": self.commands, "type": "ChangeRequest", "id": 1}
         r = requests.post(
             "http://localhost:9101/change",
             data=json.dumps(full_command),
             headers={"content-type": "application/json"},
         )
         if r.json()[u"ok"]:
             log(LIBRARIAN_SYNC, "cc_update", "Success.")
         else:
             log(LIBRARIAN_SYNC, "cc_update", "Oh, no. It failed.", "E")
Beispiel #8
0
def update_lists_from_calibre_plugin_json(db_ebooks, db_collections,
                                          collection_contents):

    for (collection_label, ebook_hashes_list) in collection_contents.items():
        # find collection by label
        collection_idx = find_collection(db_collections, collection_label)
        if collection_idx == -1:
            # creating new collection object
            db_collections.append(
                Collection(uuid.uuid4(), collection_label, is_new=True))
            collection_idx = len(db_collections) - 1

        for ebook_hash in ebook_hashes_list:
            cdekey, cdetype = parse_legacy_hash(ebook_hash)
            # NOTE: We don't actually use the cdeType. We shouldn't need to,
            # unless we run into the extremely unlikely case of two items with
            # the same cdeKey, but different cdeTypes
            # find ebook by cdeKey
            ebook_idx_list = find_ebook(db_ebooks, cdekey)
            if ebook_idx_list == []:
                log(LIBRARIAN_SYNC,
                    "update calibre",
                    "Couldn't match a db uuid to cdeKey %s"
                    "(book not on device?)" % cdekey,
                    "W",
                    display=False)
                continue  # invalid
            for ebook_idx in ebook_idx_list:
                # update ebook
                db_ebooks[ebook_idx].add_collection(
                    db_collections[collection_idx])
                # update collection
                db_collections[collection_idx].add_ebook(db_ebooks[ebook_idx])

    # remove empty collections:
    db_collections = [c for c in db_collections if len(c.ebooks) != 0]

    return db_ebooks, db_collections
Beispiel #9
0
def update_lists_from_librarian_json(db_ebooks, db_collections,
                                     collection_contents):

    for (ebook_location,
         ebook_collection_labels_list) in collection_contents.items():
        # find ebook by location
        if ebook_location.startswith("re:"):
            ebook_idx_list = find_ebook(db_ebooks, ebook_location, regexp=True)
        else:
            ebook_idx_list = find_ebook(
                db_ebooks, os.path.join(KINDLE_EBOOKS_ROOT, ebook_location))
        if ebook_idx_list == []:
            log(LIBRARIAN_SYNC,
                "update librarian",
                "Invalid location: %s" % ebook_location.encode("utf8"),
                "W",
                display=False)
            continue  # invalid
        for collection_label in ebook_collection_labels_list:
            # find collection by label
            collection_idx = find_collection(db_collections, collection_label)
            if collection_idx == -1:
                # creating new collection object
                db_collections.append(
                    Collection(uuid.uuid4(), collection_label, is_new=True))
                collection_idx = len(db_collections) - 1
            for ebook_idx in ebook_idx_list:
                # udpate ebook
                db_ebooks[ebook_idx].add_collection(
                    db_collections[collection_idx])
                # update collection
                db_collections[collection_idx].add_ebook(db_ebooks[ebook_idx])

    # remove empty collections:
    db_collections = [c for c in db_collections if len(c.ebooks) != 0]

    return db_ebooks, db_collections
def update_lists_from_librarian_json(db_ebooks, db_collections,
                                     collection_contents):

    for (ebook_location,
         ebook_collection_labels_list) in collection_contents.items():
        # find ebook by location
        if ebook_location.startswith("re:"):
            ebook_idx_list = find_ebook(db_ebooks, ebook_location, regexp=True)
        else:
            ebook_idx_list = find_ebook(db_ebooks, os.path.join(
                KINDLE_EBOOKS_ROOT, ebook_location))
        if ebook_idx_list == []:
            log(LIBRARIAN_SYNC, "update librarian",
                "Invalid location: %s" % ebook_location.encode("utf8"),
                "W", display=False)
            continue  # invalid
        for collection_label in ebook_collection_labels_list:
            # find collection by label
            collection_idx = find_collection(db_collections, collection_label)
            if collection_idx == -1:
                # creating new collection object
                db_collections.append(Collection(uuid.uuid4(),
                                                 collection_label,
                                                 is_new=True))
                collection_idx = len(db_collections)-1
            for ebook_idx in ebook_idx_list:
                # udpate ebook
                db_ebooks[ebook_idx].add_collection(
                    db_collections[collection_idx])
                # update collection
                db_collections[collection_idx].add_ebook(db_ebooks[ebook_idx])

    # remove empty collections:
    db_collections = [c for c in db_collections if len(c.ebooks) != 0]

    return db_ebooks, db_collections
def parse_entries(cursor, ignore_empty_collections=False):
    db_ebooks = []
    db_collections = []

    cursor.execute(SELECT_COLLECTION_ENTRIES)
    for (c_uuid, label) in cursor.fetchall():
        db_collections.append(Collection(c_uuid, label))

    cursor.execute(SELECT_EBOOK_ENTRIES)
    for (e_uuid, location, cdekey, cdetype) in cursor.fetchall():
        # only consider user ebooks
        if location is not None and KINDLE_EBOOKS_ROOT in location:
            db_ebooks.append(Ebook(e_uuid, location, cdekey, cdetype))

    cursor.execute(SELECT_EXISTING_COLLECTIONS)
    for (collection_uuid, ebook_uuid) in cursor.fetchall():
        collection_idx = find_collection(db_collections, collection_uuid)
        ebook_idx_list = find_ebook(db_ebooks, ebook_uuid)
        if collection_idx != -1 and ebook_idx_list != []:
            for ebook_idx in ebook_idx_list:
                db_collections[collection_idx].add_ebook(db_ebooks[ebook_idx],
                                                        True)
                db_ebooks[ebook_idx].add_collection(
                    db_collections[collection_idx], True)
        else:
            log(LIBRARIAN_SYNC, "parse_entries",
                "Skipping collection {} (collection_idx: {}, ebook_uuid: {})"
                .format(collection_uuid, collection_idx, ebook_uuid),
                "W", display=False)

    # remove empty collections:
    if ignore_empty_collections:
        db_collections = [c for c in db_collections
                          if len(c.original_ebooks) != 0]

    return db_ebooks, db_collections
def update_lists_from_calibre_plugin_json(db_ebooks, db_collections,
                                          collection_contents):

    for (collection_label, ebook_hashes_list) in collection_contents.items():
        # find collection by label
        collection_idx = find_collection(db_collections, collection_label)
        if collection_idx == -1:
            # creating new collection object
            db_collections.append(Collection(uuid.uuid4(), collection_label,
                                             is_new=True))
            collection_idx = len(db_collections)-1

        for ebook_hash in ebook_hashes_list:
            cdekey, cdetype = parse_legacy_hash(ebook_hash)
            # NOTE: We don't actually use the cdeType. We shouldn't need to,
            # unless we run into the extremely unlikely case of two items with
            # the same cdeKey, but different cdeTypes
            # find ebook by cdeKey
            ebook_idx_list = find_ebook(db_ebooks, cdekey)
            if ebook_idx_list == []:
                log(LIBRARIAN_SYNC, "update calibre",
                    "Couldn't match a db uuid to cdeKey %s"
                    "(book not on device?)" % cdekey,
                    "W", display=False)
                continue  # invalid
            for ebook_idx in ebook_idx_list:
                # update ebook
                db_ebooks[ebook_idx].add_collection(
                    db_collections[collection_idx])
                # update collection
                db_collections[collection_idx].add_ebook(db_ebooks[ebook_idx])

    # remove empty collections:
    db_collections = [c for c in db_collections if len(c.ebooks) != 0]

    return db_ebooks, db_collections
def download_all_served_ebooks(ip, port):
    # ask for available ebooks
    code, mimetype, result = download_file(ip, port, url(ip, port, "index"))
    if code == requests.codes.ok and mimetype == "text/plain":
        epubs = result.split("|")
        # download them all
        for epub in epubs:
            log(LIBRARIAN_SYNC, "download", "D/L: %s." % os.path.basename(epub))
            code, mimetype, result = download_file(ip, port, url(ip, port, epub))
            if code == requests.codes.not_found:
                log(LIBRARIAN_SYNC, "download", "%s not found." % epub, "W")
            elif code == requests.codes.ok:
                if mimetype not in ebook_mimetypes:
                    raise Exception("what?")
                else:
                    log(LIBRARIAN_SYNC, "download", "Downloaded: %s." % os.path.basename(epub), display=False)
        # ask for the associated collections
        code, mimetype, result = download_file(ip, port, url(ip, port, "collections.json"))
        # shutdown the server, all done
        code, mimetype, result = download_file(ip, port, url(ip, port, "LibrarianServer::shutdown"))
        if code == requests.codes.ok and mimetype == "text/plain":
            log(LIBRARIAN_SYNC, "shutdown", result)
    else:
        log(LIBRARIAN_SYNC, "retrieve_index", "Could not retrieve index.")
Beispiel #14
0
 def execute(self):
     if not self.commands:
         log(LIBRARIAN_SYNC, "cc_update", "Nothing to update.")
     else:
         log(LIBRARIAN_SYNC, "cc_update", "Sending commands...")
         full_command = {
             "commands": self.commands,
             "type": "ChangeRequest",
             "id": 1
         }
         r = requests.post("http://localhost:9101/change",
                           data=json.dumps(full_command),
                           headers={'content-type': 'application/json'})
         if r.json()[u"ok"]:
             log(LIBRARIAN_SYNC, "cc_update", "Success.")
         else:
             log(LIBRARIAN_SYNC, "cc_update", "Oh, no. It failed.", "E")
def download_file(ip, port, url):
    r = requests.get(url, stream=True)

    if r.headers["content-type"] in ebook_mimetypes or r.headers["content-type"] == "application/json":

        filename = url.split(SERVER_HTTP % (ip, port))[1]
        if filename == "collections.json":
            local_filename = os.path.join(COLLECTIONS_DIR, filename)
        else:
            local_filename = os.path.join(DESTINATION_DIR, filename)
        if not os.path.exists(os.path.dirname(local_filename)):
            log(LIBRARIAN_SYNC, "download_file", "Creating %s" % os.path.dirname(local_filename))
            os.makedirs(os.path.dirname(local_filename))
        with open(local_filename, "wb") as f:
            for chunk in r.iter_content(chunk_size=1024):
                if chunk:  # filter out keep-alive new chunks
                    f.write(chunk)
                    f.flush()
        return r.status_code, r.headers["content-type"], local_filename
    elif r.headers["content-type"] == "text/plain":
        r.encoding = "utf8"
        return r.status_code, r.headers["content-type"], r.text
    else:
        return r.status_code, r.headers["content-type"], None
                    raise Exception("what?")
                else:
                    log(LIBRARIAN_SYNC, "download", "Downloaded: %s." % os.path.basename(epub), display=False)
        # ask for the associated collections
        code, mimetype, result = download_file(ip, port, url(ip, port, "collections.json"))
        # shutdown the server, all done
        code, mimetype, result = download_file(ip, port, url(ip, port, "LibrarianServer::shutdown"))
        if code == requests.codes.ok and mimetype == "text/plain":
            log(LIBRARIAN_SYNC, "shutdown", result)
    else:
        log(LIBRARIAN_SYNC, "retrieve_index", "Could not retrieve index.")


if __name__ == "__main__":
    start = time.time()
    log(LIBRARIAN_SYNC, "download", "Starting...")
    try:
        config = ConfigParser.ConfigParser()
        config.read("librarian_download.ini")
        IPs = config.get("server", "IP", "localhost").split("|")
        port = config.get("server", "port", 13699)
    except:
        log(LIBRARIAN_SYNC, "download", "Missing or incorrect configuration file.")
    failed = 0
    for ip in IPs:
        try:
            download_all_served_ebooks(ip, port)
        except requests.packages.urllib3.exceptions.ProtocolError as e:
            err, code = e
            failed += 1
            log(LIBRARIAN_SYNC, "connect", "%s : %s" % (ip, code), display=False)
Beispiel #17
0
                        help='rebuild collections from librarian.')
    parser.add_argument('--update-calibre',
                        dest='update_calibre',
                        action='store_true',
                        default=False,
                        help='update collections from calibre kindle plugin.')
    parser.add_argument('--rebuild-calibre',
                        dest='rebuild_calibre',
                        action='store_true',
                        default=False,
                        help='rebuild collections from calibre kindle plugin.')

    args = parser.parse_args()

    start = time.time()
    log(LIBRARIAN_SYNC, "main", "Starting...")
    try:
        with sqlite3.connect(KINDLE_DB_PATH) as cc_db:
            c = cc_db.cursor()
            if args.rebuild:
                log(LIBRARIAN_SYNC, "rebuild",
                    "Rebuilding collections (librarian)...")
                update_cc_db(c, complete_rebuild=True, source="librarian")
            elif args.update:
                log(LIBRARIAN_SYNC, "update",
                    "Updating collections (librarian)...")
                update_cc_db(c, complete_rebuild=False, source="librarian")
            elif args.folders:
                log(LIBRARIAN_SYNC, "rebuild_from_folders",
                    "Rebuilding collections (folders)...")
                update_cc_db(c, complete_rebuild=True, source="folders")
                        action='store_true', default=False,
                        help='update collections from librarian.')
    parser.add_argument('-r', '--rebuild',   dest='rebuild',
                        action='store_true', default=False,
                        help='rebuild collections from librarian.')
    parser.add_argument('--update-calibre',  dest='update_calibre',
                        action='store_true', default=False,
                        help='update collections from calibre kindle plugin.')
    parser.add_argument('--rebuild-calibre', dest='rebuild_calibre',
                        action='store_true', default=False,
                        help='rebuild collections from calibre kindle plugin.')

    args = parser.parse_args()

    start = time.time()
    log(LIBRARIAN_SYNC, "main", "Starting...")
    try:
        with sqlite3.connect(KINDLE_DB_PATH) as cc_db:
            c = cc_db.cursor()
            if args.rebuild:
                log(LIBRARIAN_SYNC, "rebuild",
                    "Rebuilding collections (librarian)...")
                update_cc_db(c, complete_rebuild=True,
                             source="librarian")
            elif args.update:
                log(LIBRARIAN_SYNC, "update",
                    "Updating collections (librarian)...")
                update_cc_db(c, complete_rebuild=False,
                             source="librarian")
            elif args.folders:
                log(LIBRARIAN_SYNC, "rebuild_from_folders",
Beispiel #19
0
                        display=False)
        # ask for the associated collections
        code, mimetype, result = download_file(
            ip, port, url(ip, port, "collections.json"))
        # shutdown the server, all done
        code, mimetype, result = download_file(
            ip, port, url(ip, port, "LibrarianServer::shutdown"))
        if code == requests.codes.ok and mimetype == "text/plain":
            log(LIBRARIAN_SYNC, "shutdown", result)
    else:
        log(LIBRARIAN_SYNC, "retrieve_index", "Could not retrieve index.")


if __name__ == "__main__":
    start = time.time()
    log(LIBRARIAN_SYNC, "download", "Starting...")
    try:
        config = ConfigParser.ConfigParser()
        config.read("librarian_download.ini")
        IPs = config.get("server", "IP", "localhost").split("|")
        port = config.get("server", "port", 13699)
    except:
        log(LIBRARIAN_SYNC, "download",
            "Missing or incorrect configuration file.")
    failed = 0
    for ip in IPs:
        try:
            download_all_served_ebooks(ip, port)
        except requests.packages.urllib3.exceptions.ProtocolError as e:
            err, code = e
            failed += 1