class UploadWorker(QObject):
readyForNext = pyqtSignal(int)
uploadProgress = pyqtSignal(int, int, int)
uploaded = pyqtSignal(int)
updated = pyqtSignal(int)
skipped = pyqtSignal(int)
failed = pyqtSignal(int, str)
aborted = pyqtSignal(str)
def __init__(self, index, reupload, db, logger):
QObject.__init__(self)
self.index = index
self.reupload = reupload
self.db = db
self.logger = logger
self.api_key = prefs['api_key']
self.reply = None
self.canceled = False
self.retries = 0
def start(self):
self.network = QNetworkAccessManager()
self.network.authenticationRequired.connect(self.auth)
self.readyForNext.emit(self.index)
def cancel(self):
self.canceled = True
if self.reply:
self.reply.abort()
def sync(self, book_id, file_path):
self.book_id = book_id
self.file_path = file_path
self.check()
def auth(self, reply, authenticator):
if not authenticator.user():
authenticator.setUser(self.api_key)
authenticator.setPassword('')
def check(self):
self.digest = None
identifiers = self.db.get_proxy_metadata(self.book_id).identifiers
if identifiers.get('bookfusion'):
self.is_search_req = False
self.req = api.build_request('/uploads/' +
identifiers['bookfusion'])
self.log_info('Upload check: bookfusion={}'.format(
identifiers['bookfusion']))
elif identifiers.get('isbn'):
self.is_search_req = True
self.req = api.build_request('/uploads',
{'isbn': identifiers['isbn']})
self.log_info('Upload check: isbn={}'.format(identifiers['isbn']))
else:
self.calculate_digest()
self.is_search_req = False
self.req = api.build_request('/uploads/' + self.digest)
self.log_info('Upload check: digest={}'.format(self.digest))
self.reply = self.network.get(self.req)
self.reply.finished.connect(self.complete_check)
def complete_check(self):
abort = False
skip = False
update = False
result = None
error = self.reply.error()
if error == QNetworkReply.AuthenticationRequiredError:
abort = True
self.aborted.emit('Invalid API key.')
self.log_info('Upload check: AuthenticationRequiredError')
elif error == QNetworkReply.NoError:
resp = self.reply.readAll()
self.log_info('Upload check response: {}'.format(resp))
if self.is_search_req:
results = json.loads(resp.data())
if len(results) > 0:
result = results[0]
else:
result = json.loads(resp.data())
if result is not None:
self.set_bookfusion_id(result['id'])
update = True
elif error == QNetworkReply.ContentNotFoundError:
self.log_info('Upload check: ContentNotFoundError')
elif error == QNetworkReply.InternalServerError:
self.log_info('Upload check: InternalServerError')
resp = self.reply.readAll()
self.log_info('Upload check response: {}'.format(resp))
elif error == QNetworkReply.UnknownServerError:
self.log_info('Upload check: UnknownServerError')
resp = self.reply.readAll()
self.log_info('Upload check response: {}'.format(resp))
elif error == QNetworkReply.OperationCanceledError:
abort = True
self.log_info('Upload check: OperationCanceledError')
else:
abort = True
self.aborted.emit('Error {}.'.format(error))
self.log_info('Upload check error: {}'.format(error))
self.reply.deleteLater()
self.reply = None
if not abort:
if skip:
self.readyForNext.emit(self.index)
else:
self.metadata_digest = self.get_metadata_digest()
if not result is None and self.metadata_digest == result[
'calibre_metadata_digest'] and not self.reupload:
self.skipped.emit(self.book_id)
self.readyForNext.emit(self.index)
else:
if update:
self.update()
else:
self.init_upload()
def init_upload(self):
self.calculate_digest()
self.req = api.build_request('/uploads/init')
self.req_body = QHttpMultiPart(QHttpMultiPart.FormDataType)
self.req_body.append(
self.build_req_part('filename', path.basename(self.file_path)))
self.req_body.append(self.build_req_part('digest', self.digest))
self.reply = self.network.post(self.req, self.req_body)
self.reply.finished.connect(self.complete_init_upload)
def complete_init_upload(self):
resp, retry, abort = self.complete_req('Upload init', return_json=True)
if retry:
self.init_upload()
return
if abort:
return
if resp is not None:
self.upload_url = resp['url']
self.upload_params = resp['params']
self.upload()
else:
self.readyForNext.emit(self.index)
def upload(self):
self.file = QFile(self.file_path)
self.file.open(QIODevice.ReadOnly)
self.req = QNetworkRequest(QUrl(self.upload_url))
self.req_body = QHttpMultiPart(QHttpMultiPart.FormDataType)
for key, value in self.upload_params.items():
self.log_info('{}={}'.format(key, value))
self.req_body.append(self.build_req_part(key, value))
self.req_body.append(self.build_req_part('file', self.file))
self.reply = self.network.post(self.req, self.req_body)
self.reply.finished.connect(self.complete_upload)
self.reply.uploadProgress.connect(self.upload_progress)
def complete_upload(self):
if self.file:
self.file.close()
resp, retry, abort = self.complete_req('Upload')
if retry:
self.upload()
return
if abort:
return
if resp is not None:
self.finalize_upload()
else:
self.readyForNext.emit(self.index)
def finalize_upload(self):
self.req = api.build_request('/uploads/finalize')
self.req_body = QHttpMultiPart(QHttpMultiPart.FormDataType)
self.req_body.append(
self.build_req_part('key', self.upload_params['key']))
self.req_body.append(self.build_req_part('digest', self.digest))
self.append_metadata_req_parts()
self.reply = self.network.post(self.req, self.req_body)
self.reply.finished.connect(self.complete_finalize_upload)
def complete_finalize_upload(self):
self.clean_metadata_req()
resp, retry, abort = self.complete_req('Upload finalize',
return_json=True)
if retry:
self.finalize_upload()
return
if abort:
return
if resp is not None:
self.set_bookfusion_id(resp['id'])
self.uploaded.emit(self.book_id)
self.readyForNext.emit(self.index)
def update(self):
if not prefs['update_metadata'] and not self.reupload:
self.skipped.emit(self.book_id)
self.readyForNext.emit(self.index)
return
identifiers = self.db.get_proxy_metadata(self.book_id).identifiers
if not identifiers.get('bookfusion') and not self.reupload:
self.skipped.emit(self.book_id)
self.readyForNext.emit(self.index)
return
self.req = api.build_request('/uploads/' + identifiers['bookfusion'])
self.req_body = QHttpMultiPart(QHttpMultiPart.FormDataType)
if self.reupload:
self.file = QFile(self.file_path)
self.file.open(QIODevice.ReadOnly)
self.req_body.append(self.build_req_part('file', self.file))
self.append_metadata_req_parts()
self.reply = self.network.put(self.req, self.req_body)
self.reply.finished.connect(self.complete_update)
def complete_update(self):
self.clean_metadata_req()
resp, retry, abort = self.complete_req('Update')
if retry:
self.update()
return
if abort:
return
if resp is not None:
self.updated.emit(self.book_id)
self.readyForNext.emit(self.index)
def upload_progress(self, sent, total):
self.uploadProgress.emit(self.book_id, sent, total)
def log_info(self, msg):
self.logger.info('[worker-{}] {}'.format(self.index, msg))
def get_metadata_digest(self):
metadata = self.db.get_proxy_metadata(self.book_id)
h = sha256()
language = next(iter(metadata.languages), None)
summary = metadata.comments
isbn = metadata.isbn
issued_on = metadata.pubdate.date().isoformat()
if issued_on == '0101-01-01':
issued_on = None
h.update(metadata.title.encode('utf-8'))
if summary:
h.update(summary.encode('utf-8'))
if language:
h.update(language.encode('utf-8'))
if isbn:
h.update(isbn.encode('utf-8'))
if issued_on:
h.update(issued_on.encode('utf-8'))
for series_item in self.get_series(metadata):
h.update(series_item['title'].encode('utf-8'))
if series_item['index'] is not None:
h.update(str(series_item['index']).encode('utf-8'))
for author in metadata.authors:
h.update(author.encode('utf-8'))
for tag in metadata.tags:
h.update(tag.encode('utf-8'))
bookshelves = self.get_bookshelves(metadata)
if bookshelves is not None:
for bookshelf in bookshelves:
h.update(bookshelf.encode('utf-8'))
cover_path = self.db.cover(self.book_id, as_path=True)
if cover_path:
h.update(bytes(path.getsize(cover_path)))
h.update(b'\0')
with open(cover_path, 'rb') as file:
block = file.read(65536)
while len(block) > 0:
h.update(block)
block = file.read(65536)
return h.hexdigest()
def append_metadata_req_parts(self):
metadata = self.db.get_proxy_metadata(self.book_id)
language = next(iter(metadata.languages), None)
summary = metadata.comments
isbn = metadata.isbn
issued_on = metadata.pubdate.date().isoformat()
if issued_on == '0101-01-01':
issued_on = None
self.req_body.append(
self.build_req_part('metadata[calibre_metadata_digest]',
self.metadata_digest))
self.req_body.append(
self.build_req_part('metadata[title]', metadata.title))
if summary:
self.req_body.append(
self.build_req_part('metadata[summary]', summary))
if language:
self.req_body.append(
self.build_req_part('metadata[language]', language))
if isbn:
self.req_body.append(self.build_req_part('metadata[isbn]', isbn))
if issued_on:
self.req_body.append(
self.build_req_part('metadata[issued_on]', issued_on))
for series_item in self.get_series(metadata):
self.req_body.append(
self.build_req_part('metadata[series][][title]',
series_item['title']))
if series_item['index'] is not None:
self.req_body.append(
self.build_req_part('metadata[series][][index]',
str(series_item['index'])))
for author in metadata.authors:
self.req_body.append(
self.build_req_part('metadata[author_list][]', author))
for tag in metadata.tags:
self.req_body.append(
self.build_req_part('metadata[tag_list][]', tag))
bookshelves = self.get_bookshelves(metadata)
if bookshelves is not None:
self.req_body.append(
self.build_req_part('metadata[bookshelves][]', ''))
for bookshelf in bookshelves:
self.req_body.append(
self.build_req_part('metadata[bookshelves][]', bookshelf))
cover_path = self.db.cover(self.book_id, as_path=True)
if cover_path:
self.cover = QFile(cover_path)
self.cover.open(QIODevice.ReadOnly)
self.req_body.append(
self.build_req_part('metadata[cover]', self.cover))
else:
self.cover = None
def clean_metadata_req(self):
if self.cover:
self.cover.remove()
def build_req_part(self, name, value):
part = QHttpPart()
part.setHeader(QNetworkRequest.ContentTypeHeader, None)
if isinstance(value, QFile):
filename = QFileInfo(value).fileName()
part.setHeader(
QNetworkRequest.ContentDispositionHeader,
'form-data; name="{}"; filename="{}"'.format(
self.escape_quotes(name), self.escape_quotes(filename)))
part.setBodyDevice(value)
else:
part.setHeader(
QNetworkRequest.ContentDispositionHeader,
'form-data; name="{}"'.format(self.escape_quotes(name)))
part.setBody(value.encode('utf-8'))
return part
def complete_req(self, tag, return_json=False):
retry = False
abort = False
if self.canceled:
abort = True
error = self.reply.error()
resp = None
if error == QNetworkReply.AuthenticationRequiredError:
abort = True
self.aborted.emit('Invalid API key.')
self.log_info('{}: AuthenticationRequiredError'.format(tag))
elif error == QNetworkReply.NoError:
resp = self.reply.readAll()
self.log_info('{} response: {}'.format(tag, resp))
if return_json:
try:
resp = json.loads(resp.data())
except ValueError as e:
resp = None
self.log_info('{}: {}'.format(tag, e))
self.failed.emit(self.book_id,
'Cannot parse the server response')
elif error == QNetworkReply.UnknownContentError:
if self.reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute) == 422:
err_resp = self.reply.readAll()
self.log_info('{} response: {}'.format(tag, err_resp))
msg = json.loads(err_resp.data())['error']
self.failed.emit(self.book_id, msg)
else:
self.log_info('{}: UnknownContentError'.format(tag))
elif error == QNetworkReply.InternalServerError:
self.log_info('{}: InternalServerError'.format(tag))
err_resp = self.reply.readAll()
self.log_info('{} response: {}'.format(tag, err_resp))
elif error == QNetworkReply.UnknownServerError:
self.log_info('{}: UnknownServerError'.format(tag))
err_resp = self.reply.readAll()
self.log_info('{} response: {}'.format(tag, err_resp))
elif error == QNetworkReply.ConnectionRefusedError or \
error == QNetworkReply.RemoteHostClosedError or \
error == QNetworkReply.HostNotFoundError or \
error == QNetworkReply.TimeoutError or \
error == QNetworkReply.TemporaryNetworkFailureError:
retry = True
self.log_info('{}: {}'.format(tag, error))
elif error == QNetworkReply.OperationCanceledError:
abort = True
self.log_info('{}: OperationCanceledError'.format(tag))
else:
abort = True
self.aborted.emit('Error {}.'.format(error))
self.log_info('{} error: {}'.format(tag, error))
self.reply.deleteLater()
self.reply = None
if retry:
self.retries += 1
if self.retries > 2:
self.retries = 0
self.aborted.emit('Error {}.'.format(error))
retry = False
else:
abort = False
else:
self.retries = 0
return (resp, retry, abort)
def calculate_digest(self):
if self.digest is not None:
return
h = sha256()
h.update(bytes(path.getsize(self.file_path)))
h.update(b'\0')
with open(self.file_path, 'rb') as file:
block = file.read(65536)
while len(block) > 0:
h.update(block)
block = file.read(65536)
self.digest = h.hexdigest()
def escape_quotes(self, value):
return value.replace('"', '\\"')
def set_bookfusion_id(self, bookfusion_id):
identifiers = self.db.get_proxy_metadata(self.book_id).identifiers
identifiers['bookfusion'] = str(bookfusion_id)
self.db.set_field('identifiers', {self.book_id: identifiers})
def get_bookshelves(self, metadata):
bookshelves_custom_column = prefs['bookshelves_custom_column']
if bookshelves_custom_column:
try:
bookshelves = getattr(metadata, bookshelves_custom_column)
except AttributeError:
return None
if bookshelves is None:
return []
if isinstance(bookshelves, list):
return bookshelves
else:
return [bookshelves]
else:
return None
def get_series(self, metadata):
series_items = []
if metadata.series:
series_items.append({
'title': metadata.series,
'index': metadata.series_index
})
for key, meta in self.db.field_metadata.custom_iteritems():
if meta['datatype'] == 'series':
title = getattr(metadata, key)
if title:
found = False
for series_item in series_items:
if series_item['title'].lower() == title.lower():
found = True
if not found:
index = getattr(metadata, key + '_index')
series_items.append({'title': title, 'index': index})
return series_items
class CheckWorker(QObject):
finished = pyqtSignal()
aborted = pyqtSignal(str)
readyToRunCheck = pyqtSignal()
progress = pyqtSignal(int)
limitsAvailable = pyqtSignal(dict)
resultsAvailable = pyqtSignal(int, list)
def __init__(self, db, logger, book_ids):
QObject.__init__(self)
self.db = db
self.logger = logger
self.book_ids = book_ids
self.api_key = prefs['api_key']
self.reply = None
self.canceled = False
def start(self):
self.network = QNetworkAccessManager()
self.network.authenticationRequired.connect(self.auth)
self.readyToRunCheck.connect(self.run_check)
self.pending_book_ids = self.book_ids
self.count = 0
self.books_count = 0
self.valid_ids = []
self.fetch_limits()
def cancel(self):
self.canceled = True
if self.reply:
self.reply.abort()
self.finished.emit()
def auth(self, reply, authenticator):
if not authenticator.user():
authenticator.setUser(self.api_key)
authenticator.setPassword('')
def fetch_limits(self):
self.req = api.build_request('/limits')
self.reply = self.network.get(self.req)
self.reply.finished.connect(self.finish_fetch_limits)
def finish_fetch_limits(self):
if self.canceled:
return
abort = False
error = self.reply.error()
if error == QNetworkReply.AuthenticationRequiredError:
abort = True
self.aborted.emit('Invalid API key.')
self.logger.info('Fetch limits: AuthenticationRequiredError')
elif error == QNetworkReply.NoError:
resp = self.reply.readAll()
self.logger.info('Fetch limits response: {}'.format(resp))
self.limits = json.loads(resp.data())
self.limitsAvailable.emit(self.limits)
elif error == QNetworkReply.OperationCanceledError:
abort = True
self.logger.info('Fetch limits: OperationCanceledError')
else:
abort = True
self.aborted.emit('Error {}.'.format(error))
self.logger.info('Fetch limits error: {}'.format(error))
self.reply.deleteLater()
self.reply = None
if abort:
self.finished.emit()
else:
self.readyToRunCheck.emit()
def run_check(self):
for book_id in self.pending_book_ids:
if self.canceled:
return
self.progress.emit(self.count)
self.count += 1
self.logger.info('File: book_id={}'.format(book_id))
book_format = BookFormat(self.db, book_id)
if book_format.file_path:
self.books_count += 1
if getsize(book_format.file_path) <= self.limits['filesize']:
self.valid_ids.append(book_id)
self.logger.info('File ok: book_id={}'.format(book_id))
else:
self.logger.info(
'Filesize exceeded: book_id={}'.format(book_id))
else:
self.logger.info(
'Unsupported format: book_id={}'.format(book_id))
self.resultsAvailable.emit(self.books_count, self.valid_ids)
self.finished.emit()
