def run(self): tmp_directory = app.config.get('BASE_DIR', '') for user_id in os.listdir(tmp_directory): subdirectory = os.path.join(tmp_directory, user_id) if not os.path.isdir(subdirectory): log.error("Non-directory in base directory. Please delete before trying again.") return user = User.query.filter_by(id=user_id).first() if user is None: log.info("User {user_id} doesn't exist.".format(user_id=user_id)) filesystem.clear_directory(subdirectory) else: kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id, blocking=False) if kindlebox_lock is None: continue send_books_lock = acquire_send_books_lock(user_id, blocking=False) if send_books_lock is None: kindlebox_lock.release() continue log.info("Clearing user directory id {user_id}.".format(user_id=user_id)) filesystem.clear_directory(subdirectory) send_books_lock.release() kindlebox_lock.release()
def run(self): tmp_directory = app.config.get('BASE_DIR', '') for user_id in os.listdir(tmp_directory): subdirectory = os.path.join(tmp_directory, user_id) if not os.path.isdir(subdirectory): log.error( "Non-directory in base directory. Please delete before trying again." ) return user = User.query.filter_by(id=user_id).first() if user is None: log.info( "User {user_id} doesn't exist.".format(user_id=user_id)) filesystem.clear_directory(subdirectory) else: kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id, blocking=False) if kindlebox_lock is None: continue send_books_lock = acquire_send_books_lock(user_id, blocking=False) if send_books_lock is None: kindlebox_lock.release() continue log.info("Clearing user directory id {user_id}.".format( user_id=user_id)) filesystem.clear_directory(subdirectory) send_books_lock.release() kindlebox_lock.release()
def send_books(user_id, min_book_id=0, blocking=True): """ Task to send any books associated with the given user ID that are marked as `unsent`. Sends a batch of at most `ATTACHMENTS_LIMIT` books, all with Book.id greater than or equal to the given `min_book_id`. Books are downloaded if necessary. Files that need conversion are converted before sending. Before finishing, the task queues another `send_books` task for the next batch of (distinct) books. """ send_lock = acquire_send_books_lock(user_id, blocking) if send_lock is None: return # Only resend books for active users. user = User.query.filter_by(id=user_id, active=True).first() if user is None: return # Get the next batch of books that haven't been sent yet and are still # under the maximum number of send attempts. unsent_books_query = (user.books.filter_by(unsent=True) .filter(Book.num_attempts < MAX_SEND_ATTEMPTS) .order_by(Book.id)) unsent_books = (unsent_books_query.filter(Book.id >= min_book_id) .limit(ATTACHMENTS_LIMIT).all()) if len(unsent_books) == 0: send_lock.release() return log_string = ['{' + str(i) + '}' for i in range(len(unsent_books))] log_string = ' '.join(log_string).format(*[book.id for book in unsent_books]) log.info("Processing book resend for user id {0}, book ids {1}".format(user_id, log_string)) # Re-download and convert books that failed to send before. Check that # hashes match. try: _send_books(user, unsent_books) for book in unsent_books: book.num_attempts += 1 db.session.commit() except: log.error("Failed to resend books for user id {0}".format(user_id), exc_info=True) finally: # If there are any more books to send after this batch, requeue them. next_unsent_book = unsent_books_query.filter(Book.id > unsent_books[-1].id).first() send_lock.release() if next_unsent_book is None: kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id) # Dropbox may have registered more books, so don't clear them yet. if kindlebox_lock is None: return filesystem.clear_directory(user.get_directory()) kindlebox_lock.release() else: send_books.delay(user_id, min_book_id=next_unsent_book.id)
def clear_user_files(user_id, task): """ Clears as many temporary files as possible for the given `user_id` and celery `task`. If `task` is not one of 'kindlebox' or 'send_books', does nothing. """ if task == u'kindlebox': acquire_method = acquire_send_books_lock elif task == u'send_books': acquire_method = acquire_kindlebox_lock else: return task_directory = filesystem.get_user_directory(user_id, task) filesystem.clear_directory(task_directory) # May be downloading books to send, so don't clear the upper-level # directory yet. lock = acquire_method(user_id) if lock is not None: user_directory = filesystem.get_user_directory(user_id) filesystem.clear_empty_directory(user_directory) lock.release()
def send_books(user_id, min_book_id=0, convert=False): """ Task to send any books associated with the given user ID that are marked as `unsent`. Sends a batch of at most `ATTACHMENTS_LIMIT` books, all with Book.id greater than or equal to the given `min_book_id`. Books are downloaded if necessary. Download (and convert) books that need conversion if and only if `convert` is True. Before finishing, the task queues another `send_books` task for the next batch of (distinct) books. """ send_lock = acquire_send_books_lock(user_id) if send_lock is None: return # Only resend books for active users. user = User.query.filter_by(id=user_id, active=True).first() if user is None: return # Get the next batch of books that haven't been sent yet and are still # under the maximum number of send attempts. unsent_books_query = (user.books.filter_by(unsent=True) .filter(Book.num_attempts < MAX_SEND_ATTEMPTS) .order_by(Book.id)) unsent_books = unsent_books_query.filter(Book.id >= min_book_id).all() # Only short-circuit if there are no new books at all to send, not just # ones that don't need conversion. if len(unsent_books) == 0 and min_book_id == 0: send_lock.release() return # Send either books that need conversion or books that don't. compatible_books, convertible_books = [], [] for book in unsent_books: if convert_to_mobi_path(book.pathname) is None: compatible_books.append(book) else: convertible_books.append(book) if convert: unsent_books = convertible_books[:CONVERTIBLE_ATTACHMENTS_LIMIT] else: unsent_books = compatible_books[:ATTACHMENTS_LIMIT] log_string = ['{' + str(i) + '}' for i in range(len(unsent_books))] if len(unsent_books) > 0: log_string = ' '.join(log_string).format(*[book.id for book in unsent_books]) if convert: log_string += ', with conversion' log.info(u"Processing book resend for user id {0}, book ids {1}".format(user_id, log_string)) # Re-download and convert books that failed to send before. try: _send_books(user, unsent_books) # TODO: Reset all attempts to 0 before release. for book in unsent_books: book.num_attempts += 1 db.session.commit() except: log.error(u"Failed to resend books for user id {0}".format(user_id), exc_info=True) next_unsent_book = None if len(unsent_books) > 0: # If there are any more books to send after this batch, requeue them. next_unsent_book = unsent_books_query.filter(Book.id > unsent_books[-1].id).first() send_lock.release() # For some reason, calibre is leaving a lot of garbage files... filesystem.clear_calibre_files() if next_unsent_book is None: # If we've finished sending all books, including ones that need # conversion, clear all the books from the filesystem and finish # the task. if convert: kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id) # Dropbox may have registered more books, so don't clear them yet. if kindlebox_lock is None: return filesystem.clear_directory(user.get_directory()) kindlebox_lock.release() # Else, start sending the books that need conversion. else: send_books.apply_async((user_id, ), {'convert': True}, queue='conversion') else: queue_kwarg = {} if convert: queue_kwarg['queue'] = 'conversion' send_books.apply_async((user_id, ), { 'min_book_id': next_unsent_book.id, 'convert': convert, }, **queue_kwarg)
def send_books(user_id, min_book_id=0, blocking=True): """ Task to send any books associated with the given user ID that are marked as `unsent`. Sends a batch of at most `ATTACHMENTS_LIMIT` books, all with Book.id greater than or equal to the given `min_book_id`. Books are downloaded if necessary. Files that need conversion are converted before sending. Before finishing, the task queues another `send_books` task for the next batch of (distinct) books. """ send_lock = acquire_send_books_lock(user_id, blocking) if send_lock is None: return # Only resend books for active users. user = User.query.filter_by(id=user_id, active=True).first() if user is None: return # Get the next batch of books that haven't been sent yet and are still # under the maximum number of send attempts. unsent_books_query = (user.books.filter_by(unsent=True).filter( Book.num_attempts < MAX_SEND_ATTEMPTS).order_by(Book.id)) unsent_books = (unsent_books_query.filter( Book.id >= min_book_id).limit(ATTACHMENTS_LIMIT).all()) if len(unsent_books) == 0: send_lock.release() return log_string = ['{' + str(i) + '}' for i in range(len(unsent_books))] log_string = ' '.join(log_string).format( *[book.id for book in unsent_books]) log.info("Processing book resend for user id {0}, book ids {1}".format( user_id, log_string)) # Re-download and convert books that failed to send before. Check that # hashes match. try: _send_books(user, unsent_books) for book in unsent_books: book.num_attempts += 1 db.session.commit() except: log.error("Failed to resend books for user id {0}".format(user_id), exc_info=True) finally: # If there are any more books to send after this batch, requeue them. next_unsent_book = unsent_books_query.filter( Book.id > unsent_books[-1].id).first() send_lock.release() # For some reason, calibre is leaving a lot of garbage files... filesystem.clear_calibre_files() if next_unsent_book is None: kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id) # Dropbox may have registered more books, so don't clear them yet. if kindlebox_lock is None: return filesystem.clear_directory(user.get_directory()) kindlebox_lock.release() else: send_books.delay(user_id, min_book_id=next_unsent_book.id)