def download_all_served_ebooks(ip, port):
# ask for available ebooks
code, mimetype, result = download_file(ip, port, url(ip, port, "index"))
if code == requests.codes.ok and mimetype == "text/plain":
epubs = result.split("|")
# download them all
for epub in epubs:
log(LIBRARIAN_SYNC, "download",
"D/L: %s." % os.path.basename(epub))
code, mimetype, result = download_file(ip, port,
url(ip, port, epub))
if code == requests.codes.not_found:
log(LIBRARIAN_SYNC, "download", "%s not found." % epub, "W")
elif code == requests.codes.ok:
if mimetype not in ebook_mimetypes:
raise Exception("what?")
else:
log(LIBRARIAN_SYNC,
"download",
"Downloaded: %s." % os.path.basename(epub),
display=False)
# ask for the associated collections
code, mimetype, result = download_file(
ip, port, url(ip, port, "collections.json"))
# shutdown the server, all done
code, mimetype, result = download_file(
ip, port, url(ip, port, "LibrarianServer::shutdown"))
if code == requests.codes.ok and mimetype == "text/plain":
log(LIBRARIAN_SYNC, "shutdown", result)
else:
log(LIBRARIAN_SYNC, "retrieve_index", "Could not retrieve index.")
def download_file(ip, port, url):
r = requests.get(url, stream=True)
if r.headers['content-type'] in ebook_mimetypes or\
r.headers['content-type'] == "application/json":
filename = url.split(SERVER_HTTP % (ip, port))[1]
if filename == "collections.json":
local_filename = os.path.join(COLLECTIONS_DIR, filename)
else:
local_filename = os.path.join(DESTINATION_DIR, filename)
if not os.path.exists(os.path.dirname(local_filename)):
log(LIBRARIAN_SYNC, "download_file",
"Creating %s" % os.path.dirname(local_filename))
os.makedirs(os.path.dirname(local_filename))
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
return r.status_code, r.headers['content-type'], local_filename
elif r.headers['content-type'] == "text/plain":
r.encoding = "utf8"
return r.status_code, r.headers['content-type'], r.text
else:
return r.status_code, r.headers['content-type'], None
def execute(self):
if not self.commands:
log(LIBRARIAN_SYNC, "cc_update", "Nothing to update.")
else:
log(LIBRARIAN_SYNC, "cc_update", "Sending commands...")
full_command = {
"commands": self.commands,
"type": "ChangeRequest",
"id": 1
}
# When WiFi's enabled, we inherit the WhisperSync proxy, which we *cannot* go through,
# since we're talking to a local service. So make sure we do *NOT* use any proxies.
# Turns out that this is *slightly* tricky to achieve with requests,
# c.f., https://github.com/requests/requests/issues/879#issuecomment-10001977
os.environ['no_proxy'] = '127.0.0.1,localhost'
r = requests.post("http://127.0.0.1:9101/change",
data=json.dumps(full_command),
headers={
'AuthToken': self.session_token,
'content-type': 'application/json'
},
proxies={'no': 'pass'})
if r.status_code == requests.codes.ok:
log(LIBRARIAN_SYNC, "cc_update", "Success.")
else:
log(LIBRARIAN_SYNC, "cc_update", "Oh, no. It failed.", "E")
def build_legacy_hashes_list(self):
hashes_list = []
for e in self.original_ebooks:
# Guard against NULL cdeKeys, which should never happen for books, but have been seen in the wild w/ manually sideloaded stuff...
if e.cdekey:
if e.cdekey.startswith('*'):
# No ASIN set, we don't care about the cdeType, use it as-is
hashes_list.append(e.cdekey)
else:
# Proper or fake ASIN set, build the hash
hashes_list.append('#{}^{}'.format(e.cdekey, e.cdetype))
else:
log(LIBRARIAN_SYNC, "legacy hash building",
"Book %s has no cdeKey?! Skipping it."
"(sideloaded book?)" % e.location,
"W", display=False)
return hashes_list
def build_legacy_hashes_list(self):
hashes_list = []
for e in self.original_ebooks:
# Guard against NULL cdeKeys, which should never happen for books, but have been seen in the wild w/ manually sideloaded stuff...
if e.cdekey:
if e.cdekey.startswith('*'):
# No ASIN set, we don't care about the cdeType, use it as-is
hashes_list.append(e.cdekey)
else:
# Proper or fake ASIN set, build the hash
hashes_list.append('#{}^{}'.format(e.cdekey, e.cdetype))
else:
log(LIBRARIAN_SYNC, "legacy hash building",
"Book %s has no cdeKey?! Skipping it."
"(sideloaded book?)" % e.location,
"W", display=False)
return hashes_list
def parse_entries(cursor, ignore_empty_collections=False):
db_ebooks = []
db_collections = []
cursor.execute(SELECT_COLLECTION_ENTRIES)
for (c_uuid, label) in cursor.fetchall():
db_collections.append(Collection(c_uuid, label))
cursor.execute(SELECT_EBOOK_ENTRIES)
for (e_uuid, location, cdekey, cdetype) in cursor.fetchall():
# only consider user ebooks
if location is not None and KINDLE_EBOOKS_ROOT in location:
db_ebooks.append(Ebook(e_uuid, location, cdekey, cdetype))
cursor.execute(SELECT_EXISTING_COLLECTIONS)
for (collection_uuid, ebook_uuid) in cursor.fetchall():
collection_idx = find_collection(db_collections, collection_uuid)
ebook_idx_list = find_ebook(db_ebooks, ebook_uuid)
if collection_idx != -1 and ebook_idx_list != []:
for ebook_idx in ebook_idx_list:
db_collections[collection_idx].add_ebook(
db_ebooks[ebook_idx], True)
db_ebooks[ebook_idx].add_collection(
db_collections[collection_idx], True)
else:
log(LIBRARIAN_SYNC,
"parse_entries",
"Skipping collection {} (collection_idx: {}, ebook_uuid: {})".
format(collection_uuid, collection_idx, ebook_uuid),
"W",
display=False)
# remove empty collections:
if ignore_empty_collections:
db_collections = [
c for c in db_collections if len(c.original_ebooks) != 0
]
return db_ebooks, db_collections
def execute(self):
if not self.commands:
log(LIBRARIAN_SYNC, "cc_update", "Nothing to update.")
else:
log(LIBRARIAN_SYNC, "cc_update", "Sending commands...")
full_command = {"commands": self.commands, "type": "ChangeRequest", "id": 1}
r = requests.post(
"http://localhost:9101/change",
data=json.dumps(full_command),
headers={"content-type": "application/json"},
)
if r.json()[u"ok"]:
log(LIBRARIAN_SYNC, "cc_update", "Success.")
else:
log(LIBRARIAN_SYNC, "cc_update", "Oh, no. It failed.", "E")
def update_lists_from_calibre_plugin_json(db_ebooks, db_collections,
collection_contents):
for (collection_label, ebook_hashes_list) in collection_contents.items():
# find collection by label
collection_idx = find_collection(db_collections, collection_label)
if collection_idx == -1:
# creating new collection object
db_collections.append(
Collection(uuid.uuid4(), collection_label, is_new=True))
collection_idx = len(db_collections) - 1
for ebook_hash in ebook_hashes_list:
cdekey, cdetype = parse_legacy_hash(ebook_hash)
# NOTE: We don't actually use the cdeType. We shouldn't need to,
# unless we run into the extremely unlikely case of two items with
# the same cdeKey, but different cdeTypes
# find ebook by cdeKey
ebook_idx_list = find_ebook(db_ebooks, cdekey)
if ebook_idx_list == []:
log(LIBRARIAN_SYNC,
"update calibre",
"Couldn't match a db uuid to cdeKey %s"
"(book not on device?)" % cdekey,
"W",
display=False)
continue # invalid
for ebook_idx in ebook_idx_list:
# update ebook
db_ebooks[ebook_idx].add_collection(
db_collections[collection_idx])
# update collection
db_collections[collection_idx].add_ebook(db_ebooks[ebook_idx])
# remove empty collections:
db_collections = [c for c in db_collections if len(c.ebooks) != 0]
return db_ebooks, db_collections
def update_lists_from_librarian_json(db_ebooks, db_collections,
collection_contents):
for (ebook_location,
ebook_collection_labels_list) in collection_contents.items():
# find ebook by location
if ebook_location.startswith("re:"):
ebook_idx_list = find_ebook(db_ebooks, ebook_location, regexp=True)
else:
ebook_idx_list = find_ebook(
db_ebooks, os.path.join(KINDLE_EBOOKS_ROOT, ebook_location))
if ebook_idx_list == []:
log(LIBRARIAN_SYNC,
"update librarian",
"Invalid location: %s" % ebook_location.encode("utf8"),
"W",
display=False)
continue # invalid
for collection_label in ebook_collection_labels_list:
# find collection by label
collection_idx = find_collection(db_collections, collection_label)
if collection_idx == -1:
# creating new collection object
db_collections.append(
Collection(uuid.uuid4(), collection_label, is_new=True))
collection_idx = len(db_collections) - 1
for ebook_idx in ebook_idx_list:
# udpate ebook
db_ebooks[ebook_idx].add_collection(
db_collections[collection_idx])
# update collection
db_collections[collection_idx].add_ebook(db_ebooks[ebook_idx])
# remove empty collections:
db_collections = [c for c in db_collections if len(c.ebooks) != 0]
return db_ebooks, db_collections
def update_lists_from_librarian_json(db_ebooks, db_collections,
collection_contents):
for (ebook_location,
ebook_collection_labels_list) in collection_contents.items():
# find ebook by location
if ebook_location.startswith("re:"):
ebook_idx_list = find_ebook(db_ebooks, ebook_location, regexp=True)
else:
ebook_idx_list = find_ebook(db_ebooks, os.path.join(
KINDLE_EBOOKS_ROOT, ebook_location))
if ebook_idx_list == []:
log(LIBRARIAN_SYNC, "update librarian",
"Invalid location: %s" % ebook_location.encode("utf8"),
"W", display=False)
continue # invalid
for collection_label in ebook_collection_labels_list:
# find collection by label
collection_idx = find_collection(db_collections, collection_label)
if collection_idx == -1:
# creating new collection object
db_collections.append(Collection(uuid.uuid4(),
collection_label,
is_new=True))
collection_idx = len(db_collections)-1
for ebook_idx in ebook_idx_list:
# udpate ebook
db_ebooks[ebook_idx].add_collection(
db_collections[collection_idx])
# update collection
db_collections[collection_idx].add_ebook(db_ebooks[ebook_idx])
# remove empty collections:
db_collections = [c for c in db_collections if len(c.ebooks) != 0]
return db_ebooks, db_collections
def parse_entries(cursor, ignore_empty_collections=False):
db_ebooks = []
db_collections = []
cursor.execute(SELECT_COLLECTION_ENTRIES)
for (c_uuid, label) in cursor.fetchall():
db_collections.append(Collection(c_uuid, label))
cursor.execute(SELECT_EBOOK_ENTRIES)
for (e_uuid, location, cdekey, cdetype) in cursor.fetchall():
# only consider user ebooks
if location is not None and KINDLE_EBOOKS_ROOT in location:
db_ebooks.append(Ebook(e_uuid, location, cdekey, cdetype))
cursor.execute(SELECT_EXISTING_COLLECTIONS)
for (collection_uuid, ebook_uuid) in cursor.fetchall():
collection_idx = find_collection(db_collections, collection_uuid)
ebook_idx_list = find_ebook(db_ebooks, ebook_uuid)
if collection_idx != -1 and ebook_idx_list != []:
for ebook_idx in ebook_idx_list:
db_collections[collection_idx].add_ebook(db_ebooks[ebook_idx],
True)
db_ebooks[ebook_idx].add_collection(
db_collections[collection_idx], True)
else:
log(LIBRARIAN_SYNC, "parse_entries",
"Skipping collection {} (collection_idx: {}, ebook_uuid: {})"
.format(collection_uuid, collection_idx, ebook_uuid),
"W", display=False)
# remove empty collections:
if ignore_empty_collections:
db_collections = [c for c in db_collections
if len(c.original_ebooks) != 0]
return db_ebooks, db_collections
def update_lists_from_calibre_plugin_json(db_ebooks, db_collections,
collection_contents):
for (collection_label, ebook_hashes_list) in collection_contents.items():
# find collection by label
collection_idx = find_collection(db_collections, collection_label)
if collection_idx == -1:
# creating new collection object
db_collections.append(Collection(uuid.uuid4(), collection_label,
is_new=True))
collection_idx = len(db_collections)-1
for ebook_hash in ebook_hashes_list:
cdekey, cdetype = parse_legacy_hash(ebook_hash)
# NOTE: We don't actually use the cdeType. We shouldn't need to,
# unless we run into the extremely unlikely case of two items with
# the same cdeKey, but different cdeTypes
# find ebook by cdeKey
ebook_idx_list = find_ebook(db_ebooks, cdekey)
if ebook_idx_list == []:
log(LIBRARIAN_SYNC, "update calibre",
"Couldn't match a db uuid to cdeKey %s"
"(book not on device?)" % cdekey,
"W", display=False)
continue # invalid
for ebook_idx in ebook_idx_list:
# update ebook
db_ebooks[ebook_idx].add_collection(
db_collections[collection_idx])
# update collection
db_collections[collection_idx].add_ebook(db_ebooks[ebook_idx])
# remove empty collections:
db_collections = [c for c in db_collections if len(c.ebooks) != 0]
return db_ebooks, db_collections
def download_all_served_ebooks(ip, port):
# ask for available ebooks
code, mimetype, result = download_file(ip, port, url(ip, port, "index"))
if code == requests.codes.ok and mimetype == "text/plain":
epubs = result.split("|")
# download them all
for epub in epubs:
log(LIBRARIAN_SYNC, "download", "D/L: %s." % os.path.basename(epub))
code, mimetype, result = download_file(ip, port, url(ip, port, epub))
if code == requests.codes.not_found:
log(LIBRARIAN_SYNC, "download", "%s not found." % epub, "W")
elif code == requests.codes.ok:
if mimetype not in ebook_mimetypes:
raise Exception("what?")
else:
log(LIBRARIAN_SYNC, "download", "Downloaded: %s." % os.path.basename(epub), display=False)
# ask for the associated collections
code, mimetype, result = download_file(ip, port, url(ip, port, "collections.json"))
# shutdown the server, all done
code, mimetype, result = download_file(ip, port, url(ip, port, "LibrarianServer::shutdown"))
if code == requests.codes.ok and mimetype == "text/plain":
log(LIBRARIAN_SYNC, "shutdown", result)
else:
log(LIBRARIAN_SYNC, "retrieve_index", "Could not retrieve index.")
def execute(self):
if not self.commands:
log(LIBRARIAN_SYNC, "cc_update", "Nothing to update.")
else:
log(LIBRARIAN_SYNC, "cc_update", "Sending commands...")
full_command = {
"commands": self.commands,
"type": "ChangeRequest",
"id": 1
}
r = requests.post("http://localhost:9101/change",
data=json.dumps(full_command),
headers={'content-type': 'application/json'})
if r.json()[u"ok"]:
log(LIBRARIAN_SYNC, "cc_update", "Success.")
else:
log(LIBRARIAN_SYNC, "cc_update", "Oh, no. It failed.", "E")
def download_file(ip, port, url):
r = requests.get(url, stream=True)
if r.headers["content-type"] in ebook_mimetypes or r.headers["content-type"] == "application/json":
filename = url.split(SERVER_HTTP % (ip, port))[1]
if filename == "collections.json":
local_filename = os.path.join(COLLECTIONS_DIR, filename)
else:
local_filename = os.path.join(DESTINATION_DIR, filename)
if not os.path.exists(os.path.dirname(local_filename)):
log(LIBRARIAN_SYNC, "download_file", "Creating %s" % os.path.dirname(local_filename))
os.makedirs(os.path.dirname(local_filename))
with open(local_filename, "wb") as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
return r.status_code, r.headers["content-type"], local_filename
elif r.headers["content-type"] == "text/plain":
r.encoding = "utf8"
return r.status_code, r.headers["content-type"], r.text
else:
return r.status_code, r.headers["content-type"], None
raise Exception("what?")
else:
log(LIBRARIAN_SYNC, "download", "Downloaded: %s." % os.path.basename(epub), display=False)
# ask for the associated collections
code, mimetype, result = download_file(ip, port, url(ip, port, "collections.json"))
# shutdown the server, all done
code, mimetype, result = download_file(ip, port, url(ip, port, "LibrarianServer::shutdown"))
if code == requests.codes.ok and mimetype == "text/plain":
log(LIBRARIAN_SYNC, "shutdown", result)
else:
log(LIBRARIAN_SYNC, "retrieve_index", "Could not retrieve index.")
if __name__ == "__main__":
start = time.time()
log(LIBRARIAN_SYNC, "download", "Starting...")
try:
config = ConfigParser.ConfigParser()
config.read("librarian_download.ini")
IPs = config.get("server", "IP", "localhost").split("|")
port = config.get("server", "port", 13699)
except:
log(LIBRARIAN_SYNC, "download", "Missing or incorrect configuration file.")
failed = 0
for ip in IPs:
try:
download_all_served_ebooks(ip, port)
except requests.packages.urllib3.exceptions.ProtocolError as e:
err, code = e
failed += 1
log(LIBRARIAN_SYNC, "connect", "%s : %s" % (ip, code), display=False)
help='rebuild collections from librarian.')
parser.add_argument('--update-calibre',
dest='update_calibre',
action='store_true',
default=False,
help='update collections from calibre kindle plugin.')
parser.add_argument('--rebuild-calibre',
dest='rebuild_calibre',
action='store_true',
default=False,
help='rebuild collections from calibre kindle plugin.')
args = parser.parse_args()
start = time.time()
log(LIBRARIAN_SYNC, "main", "Starting...")
try:
with sqlite3.connect(KINDLE_DB_PATH) as cc_db:
c = cc_db.cursor()
if args.rebuild:
log(LIBRARIAN_SYNC, "rebuild",
"Rebuilding collections (librarian)...")
update_cc_db(c, complete_rebuild=True, source="librarian")
elif args.update:
log(LIBRARIAN_SYNC, "update",
"Updating collections (librarian)...")
update_cc_db(c, complete_rebuild=False, source="librarian")
elif args.folders:
log(LIBRARIAN_SYNC, "rebuild_from_folders",
"Rebuilding collections (folders)...")
update_cc_db(c, complete_rebuild=True, source="folders")
action='store_true', default=False,
help='update collections from librarian.')
parser.add_argument('-r', '--rebuild', dest='rebuild',
action='store_true', default=False,
help='rebuild collections from librarian.')
parser.add_argument('--update-calibre', dest='update_calibre',
action='store_true', default=False,
help='update collections from calibre kindle plugin.')
parser.add_argument('--rebuild-calibre', dest='rebuild_calibre',
action='store_true', default=False,
help='rebuild collections from calibre kindle plugin.')
args = parser.parse_args()
start = time.time()
log(LIBRARIAN_SYNC, "main", "Starting...")
try:
with sqlite3.connect(KINDLE_DB_PATH) as cc_db:
c = cc_db.cursor()
if args.rebuild:
log(LIBRARIAN_SYNC, "rebuild",
"Rebuilding collections (librarian)...")
update_cc_db(c, complete_rebuild=True,
source="librarian")
elif args.update:
log(LIBRARIAN_SYNC, "update",
"Updating collections (librarian)...")
update_cc_db(c, complete_rebuild=False,
source="librarian")
elif args.folders:
log(LIBRARIAN_SYNC, "rebuild_from_folders",
display=False)
# ask for the associated collections
code, mimetype, result = download_file(
ip, port, url(ip, port, "collections.json"))
# shutdown the server, all done
code, mimetype, result = download_file(
ip, port, url(ip, port, "LibrarianServer::shutdown"))
if code == requests.codes.ok and mimetype == "text/plain":
log(LIBRARIAN_SYNC, "shutdown", result)
else:
log(LIBRARIAN_SYNC, "retrieve_index", "Could not retrieve index.")
if __name__ == "__main__":
start = time.time()
log(LIBRARIAN_SYNC, "download", "Starting...")
try:
config = ConfigParser.ConfigParser()
config.read("librarian_download.ini")
IPs = config.get("server", "IP", "localhost").split("|")
port = config.get("server", "port", 13699)
except:
log(LIBRARIAN_SYNC, "download",
"Missing or incorrect configuration file.")
failed = 0
for ip in IPs:
try:
download_all_served_ebooks(ip, port)
except requests.packages.urllib3.exceptions.ProtocolError as e:
err, code = e
failed += 1
