def run(self):
tmp_directory = app.config.get('BASE_DIR', '')
for user_id in os.listdir(tmp_directory):
subdirectory = os.path.join(tmp_directory, user_id)
if not os.path.isdir(subdirectory):
log.error("Non-directory in base directory. Please delete before trying again.")
return
user = User.query.filter_by(id=user_id).first()
if user is None:
log.info("User {user_id} doesn't exist.".format(user_id=user_id))
filesystem.clear_directory(subdirectory)
else:
kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id, blocking=False)
if kindlebox_lock is None:
continue
send_books_lock = acquire_send_books_lock(user_id, blocking=False)
if send_books_lock is None:
kindlebox_lock.release()
continue
log.info("Clearing user directory id {user_id}.".format(user_id=user_id))
filesystem.clear_directory(subdirectory)
send_books_lock.release()
kindlebox_lock.release()
def run(self):
tmp_directory = app.config.get('BASE_DIR', '')
for user_id in os.listdir(tmp_directory):
subdirectory = os.path.join(tmp_directory, user_id)
if not os.path.isdir(subdirectory):
log.error(
"Non-directory in base directory. Please delete before trying again."
)
return
user = User.query.filter_by(id=user_id).first()
if user is None:
log.info(
"User {user_id} doesn't exist.".format(user_id=user_id))
filesystem.clear_directory(subdirectory)
else:
kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id,
blocking=False)
if kindlebox_lock is None:
continue
send_books_lock = acquire_send_books_lock(user_id,
blocking=False)
if send_books_lock is None:
kindlebox_lock.release()
continue
log.info("Clearing user directory id {user_id}.".format(
user_id=user_id))
filesystem.clear_directory(subdirectory)
send_books_lock.release()
kindlebox_lock.release()
def send_books(user_id, min_book_id=0, blocking=True):
"""
Task to send any books associated with the given user ID that are marked as
`unsent`. Sends a batch of at most `ATTACHMENTS_LIMIT` books, all with
Book.id greater than or equal to the given `min_book_id`. Books are
downloaded if necessary. Files that need conversion are converted before
sending.
Before finishing, the task queues another `send_books` task for the next
batch of (distinct) books.
"""
send_lock = acquire_send_books_lock(user_id, blocking)
if send_lock is None:
return
# Only resend books for active users.
user = User.query.filter_by(id=user_id, active=True).first()
if user is None:
return
# Get the next batch of books that haven't been sent yet and are still
# under the maximum number of send attempts.
unsent_books_query = (user.books.filter_by(unsent=True)
.filter(Book.num_attempts < MAX_SEND_ATTEMPTS)
.order_by(Book.id))
unsent_books = (unsent_books_query.filter(Book.id >= min_book_id)
.limit(ATTACHMENTS_LIMIT).all())
if len(unsent_books) == 0:
send_lock.release()
return
log_string = ['{' + str(i) + '}' for i in range(len(unsent_books))]
log_string = ' '.join(log_string).format(*[book.id for book in unsent_books])
log.info("Processing book resend for user id {0}, book ids {1}".format(user_id, log_string))
# Re-download and convert books that failed to send before. Check that
# hashes match.
try:
_send_books(user, unsent_books)
for book in unsent_books:
book.num_attempts += 1
db.session.commit()
except:
log.error("Failed to resend books for user id {0}".format(user_id),
exc_info=True)
finally:
# If there are any more books to send after this batch, requeue them.
next_unsent_book = unsent_books_query.filter(Book.id > unsent_books[-1].id).first()
send_lock.release()
if next_unsent_book is None:
kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id)
# Dropbox may have registered more books, so don't clear them yet.
if kindlebox_lock is None:
return
filesystem.clear_directory(user.get_directory())
kindlebox_lock.release()
else:
send_books.delay(user_id, min_book_id=next_unsent_book.id)
def clear_user_files(user_id, task):
"""
Clears as many temporary files as possible for the given `user_id` and
celery `task`. If `task` is not one of 'kindlebox' or 'send_books', does
nothing.
"""
if task == u'kindlebox':
acquire_method = acquire_send_books_lock
elif task == u'send_books':
acquire_method = acquire_kindlebox_lock
else:
return
task_directory = filesystem.get_user_directory(user_id, task)
filesystem.clear_directory(task_directory)
# May be downloading books to send, so don't clear the upper-level
# directory yet.
lock = acquire_method(user_id)
if lock is not None:
user_directory = filesystem.get_user_directory(user_id)
filesystem.clear_empty_directory(user_directory)
lock.release()
def clear_user_files(user_id, task):
"""
Clears as many temporary files as possible for the given `user_id` and
celery `task`. If `task` is not one of 'kindlebox' or 'send_books', does
nothing.
"""
if task == u'kindlebox':
acquire_method = acquire_send_books_lock
elif task == u'send_books':
acquire_method = acquire_kindlebox_lock
else:
return
task_directory = filesystem.get_user_directory(user_id, task)
filesystem.clear_directory(task_directory)
# May be downloading books to send, so don't clear the upper-level
# directory yet.
lock = acquire_method(user_id)
if lock is not None:
user_directory = filesystem.get_user_directory(user_id)
filesystem.clear_empty_directory(user_directory)
lock.release()
def send_books(user_id, min_book_id=0, convert=False):
"""
Task to send any books associated with the given user ID that are marked as
`unsent`. Sends a batch of at most `ATTACHMENTS_LIMIT` books, all with
Book.id greater than or equal to the given `min_book_id`. Books are
downloaded if necessary. Download (and convert) books that need conversion
if and only if `convert` is True.
Before finishing, the task queues another `send_books` task for the next
batch of (distinct) books.
"""
send_lock = acquire_send_books_lock(user_id)
if send_lock is None:
return
# Only resend books for active users.
user = User.query.filter_by(id=user_id, active=True).first()
if user is None:
return
# Get the next batch of books that haven't been sent yet and are still
# under the maximum number of send attempts.
unsent_books_query = (user.books.filter_by(unsent=True)
.filter(Book.num_attempts < MAX_SEND_ATTEMPTS)
.order_by(Book.id))
unsent_books = unsent_books_query.filter(Book.id >= min_book_id).all()
# Only short-circuit if there are no new books at all to send, not just
# ones that don't need conversion.
if len(unsent_books) == 0 and min_book_id == 0:
send_lock.release()
return
# Send either books that need conversion or books that don't.
compatible_books, convertible_books = [], []
for book in unsent_books:
if convert_to_mobi_path(book.pathname) is None:
compatible_books.append(book)
else:
convertible_books.append(book)
if convert:
unsent_books = convertible_books[:CONVERTIBLE_ATTACHMENTS_LIMIT]
else:
unsent_books = compatible_books[:ATTACHMENTS_LIMIT]
log_string = ['{' + str(i) + '}' for i in range(len(unsent_books))]
if len(unsent_books) > 0:
log_string = ' '.join(log_string).format(*[book.id for book in unsent_books])
if convert:
log_string += ', with conversion'
log.info(u"Processing book resend for user id {0}, book ids {1}".format(user_id, log_string))
# Re-download and convert books that failed to send before.
try:
_send_books(user, unsent_books)
# TODO: Reset all attempts to 0 before release.
for book in unsent_books:
book.num_attempts += 1
db.session.commit()
except:
log.error(u"Failed to resend books for user id {0}".format(user_id),
exc_info=True)
next_unsent_book = None
if len(unsent_books) > 0:
# If there are any more books to send after this batch, requeue them.
next_unsent_book = unsent_books_query.filter(Book.id > unsent_books[-1].id).first()
send_lock.release()
# For some reason, calibre is leaving a lot of garbage files...
filesystem.clear_calibre_files()
if next_unsent_book is None:
# If we've finished sending all books, including ones that need
# conversion, clear all the books from the filesystem and finish
# the task.
if convert:
kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id)
# Dropbox may have registered more books, so don't clear them yet.
if kindlebox_lock is None:
return
filesystem.clear_directory(user.get_directory())
kindlebox_lock.release()
# Else, start sending the books that need conversion.
else:
send_books.apply_async((user_id, ),
{'convert': True},
queue='conversion')
else:
queue_kwarg = {}
if convert:
queue_kwarg['queue'] = 'conversion'
send_books.apply_async((user_id, ),
{
'min_book_id': next_unsent_book.id,
'convert': convert,
},
**queue_kwarg)
def send_books(user_id, min_book_id=0, blocking=True):
"""
Task to send any books associated with the given user ID that are marked as
`unsent`. Sends a batch of at most `ATTACHMENTS_LIMIT` books, all with
Book.id greater than or equal to the given `min_book_id`. Books are
downloaded if necessary. Files that need conversion are converted before
sending.
Before finishing, the task queues another `send_books` task for the next
batch of (distinct) books.
"""
send_lock = acquire_send_books_lock(user_id, blocking)
if send_lock is None:
return
# Only resend books for active users.
user = User.query.filter_by(id=user_id, active=True).first()
if user is None:
return
# Get the next batch of books that haven't been sent yet and are still
# under the maximum number of send attempts.
unsent_books_query = (user.books.filter_by(unsent=True).filter(
Book.num_attempts < MAX_SEND_ATTEMPTS).order_by(Book.id))
unsent_books = (unsent_books_query.filter(
Book.id >= min_book_id).limit(ATTACHMENTS_LIMIT).all())
if len(unsent_books) == 0:
send_lock.release()
return
log_string = ['{' + str(i) + '}' for i in range(len(unsent_books))]
log_string = ' '.join(log_string).format(
*[book.id for book in unsent_books])
log.info("Processing book resend for user id {0}, book ids {1}".format(
user_id, log_string))
# Re-download and convert books that failed to send before. Check that
# hashes match.
try:
_send_books(user, unsent_books)
for book in unsent_books:
book.num_attempts += 1
db.session.commit()
except:
log.error("Failed to resend books for user id {0}".format(user_id),
exc_info=True)
finally:
# If there are any more books to send after this batch, requeue them.
next_unsent_book = unsent_books_query.filter(
Book.id > unsent_books[-1].id).first()
send_lock.release()
# For some reason, calibre is leaving a lot of garbage files...
filesystem.clear_calibre_files()
if next_unsent_book is None:
kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id)
# Dropbox may have registered more books, so don't clear them yet.
if kindlebox_lock is None:
return
filesystem.clear_directory(user.get_directory())
kindlebox_lock.release()
else:
send_books.delay(user_id, min_book_id=next_unsent_book.id)