Exemplo n.º 1
0
    def run(self):
        tmp_directory = app.config.get('BASE_DIR', '')
        for user_id in os.listdir(tmp_directory):
            subdirectory = os.path.join(tmp_directory, user_id)
            if not os.path.isdir(subdirectory):
                log.error("Non-directory in base directory. Please delete before trying again.")
                return

            user = User.query.filter_by(id=user_id).first()
            if user is None:
                log.info("User {user_id} doesn't exist.".format(user_id=user_id))
                filesystem.clear_directory(subdirectory)
            else:
                kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id, blocking=False)
                if kindlebox_lock is None:
                    continue

                send_books_lock = acquire_send_books_lock(user_id, blocking=False)
                if send_books_lock is None:
                    kindlebox_lock.release()
                    continue

                log.info("Clearing user directory id {user_id}.".format(user_id=user_id))
                filesystem.clear_directory(subdirectory)
                send_books_lock.release()
                kindlebox_lock.release()
Exemplo n.º 2
0
    def run(self):
        tmp_directory = app.config.get('BASE_DIR', '')
        for user_id in os.listdir(tmp_directory):
            subdirectory = os.path.join(tmp_directory, user_id)
            if not os.path.isdir(subdirectory):
                log.error(
                    "Non-directory in base directory. Please delete before trying again."
                )
                return

            user = User.query.filter_by(id=user_id).first()
            if user is None:
                log.info(
                    "User {user_id} doesn't exist.".format(user_id=user_id))
                filesystem.clear_directory(subdirectory)
            else:
                kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id,
                                                        blocking=False)
                if kindlebox_lock is None:
                    continue

                send_books_lock = acquire_send_books_lock(user_id,
                                                          blocking=False)
                if send_books_lock is None:
                    kindlebox_lock.release()
                    continue

                log.info("Clearing user directory id {user_id}.".format(
                    user_id=user_id))
                filesystem.clear_directory(subdirectory)
                send_books_lock.release()
                kindlebox_lock.release()
Exemplo n.º 3
0
def send_books(user_id, min_book_id=0, blocking=True):
    """
    Task to send any books associated with the given user ID that are marked as
    `unsent`. Sends a batch of at most `ATTACHMENTS_LIMIT` books, all with
    Book.id greater than or equal to the given `min_book_id`. Books are
    downloaded if necessary. Files that need conversion are converted before
    sending.

    Before finishing, the task queues another `send_books` task for the next
    batch of (distinct) books.
    """
    send_lock = acquire_send_books_lock(user_id, blocking)
    if send_lock is None:
        return

    # Only resend books for active users.
    user = User.query.filter_by(id=user_id, active=True).first()
    if user is None:
        return

    # Get the next batch of books that haven't been sent yet and are still
    # under the maximum number of send attempts.
    unsent_books_query = (user.books.filter_by(unsent=True)
                                    .filter(Book.num_attempts < MAX_SEND_ATTEMPTS)
                                    .order_by(Book.id))
    unsent_books = (unsent_books_query.filter(Book.id >= min_book_id)
                                      .limit(ATTACHMENTS_LIMIT).all())
    if len(unsent_books) == 0:
        send_lock.release()
        return

    log_string = ['{' + str(i) + '}' for i in range(len(unsent_books))]
    log_string = ' '.join(log_string).format(*[book.id for book in unsent_books])
    log.info("Processing book resend for user id {0}, book ids {1}".format(user_id, log_string))

    # Re-download and convert books that failed to send before. Check that
    # hashes match.
    try:
        _send_books(user, unsent_books)
        for book in unsent_books:
            book.num_attempts += 1
        db.session.commit()
    except:
        log.error("Failed to resend books for user id {0}".format(user_id),
                  exc_info=True)
    finally:
        # If there are any more books to send after this batch, requeue them.
        next_unsent_book = unsent_books_query.filter(Book.id > unsent_books[-1].id).first()

        send_lock.release()

        if next_unsent_book is None:
            kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id)
            # Dropbox may have registered more books, so don't clear them yet.
            if kindlebox_lock is None:
                return
            filesystem.clear_directory(user.get_directory())
            kindlebox_lock.release()
        else:
            send_books.delay(user_id, min_book_id=next_unsent_book.id)
Exemplo n.º 4
0
def clear_user_files(user_id, task):
    """
    Clears as many temporary files as possible for the given `user_id` and
    celery `task`. If `task` is not one of 'kindlebox' or 'send_books', does
    nothing.
    """
    if task == u'kindlebox':
        acquire_method = acquire_send_books_lock
    elif task == u'send_books':
        acquire_method = acquire_kindlebox_lock
    else:
        return

    task_directory = filesystem.get_user_directory(user_id, task)
    filesystem.clear_directory(task_directory)

    # May be downloading books to send, so don't clear the upper-level
    # directory yet.
    lock = acquire_method(user_id)
    if lock is not None:
        user_directory = filesystem.get_user_directory(user_id)
        filesystem.clear_empty_directory(user_directory)
        lock.release()
Exemplo n.º 5
0
def clear_user_files(user_id, task):
    """
    Clears as many temporary files as possible for the given `user_id` and
    celery `task`. If `task` is not one of 'kindlebox' or 'send_books', does
    nothing.
    """
    if task == u'kindlebox':
        acquire_method = acquire_send_books_lock
    elif task == u'send_books':
        acquire_method = acquire_kindlebox_lock
    else:
        return

    task_directory = filesystem.get_user_directory(user_id, task)
    filesystem.clear_directory(task_directory)

    # May be downloading books to send, so don't clear the upper-level
    # directory yet.
    lock = acquire_method(user_id)
    if lock is not None:
        user_directory = filesystem.get_user_directory(user_id)
        filesystem.clear_empty_directory(user_directory)
        lock.release()
Exemplo n.º 6
0
def send_books(user_id, min_book_id=0, convert=False):
    """
    Task to send any books associated with the given user ID that are marked as
    `unsent`. Sends a batch of at most `ATTACHMENTS_LIMIT` books, all with
    Book.id greater than or equal to the given `min_book_id`. Books are
    downloaded if necessary. Download (and convert) books that need conversion
    if and only if `convert` is True.

    Before finishing, the task queues another `send_books` task for the next
    batch of (distinct) books.
    """
    send_lock = acquire_send_books_lock(user_id)
    if send_lock is None:
        return

    # Only resend books for active users.
    user = User.query.filter_by(id=user_id, active=True).first()
    if user is None:
        return

    # Get the next batch of books that haven't been sent yet and are still
    # under the maximum number of send attempts.
    unsent_books_query = (user.books.filter_by(unsent=True)
                                    .filter(Book.num_attempts < MAX_SEND_ATTEMPTS)
                                    .order_by(Book.id))
    unsent_books = unsent_books_query.filter(Book.id >= min_book_id).all()

    # Only short-circuit if there are no new books at all to send, not just
    # ones that don't need conversion.
    if len(unsent_books) == 0 and min_book_id == 0:
        send_lock.release()
        return

    # Send either books that need conversion or books that don't.
    compatible_books, convertible_books = [], []
    for book in unsent_books:
        if convert_to_mobi_path(book.pathname) is None:
            compatible_books.append(book)
        else:
            convertible_books.append(book)
    if convert:
        unsent_books = convertible_books[:CONVERTIBLE_ATTACHMENTS_LIMIT]
    else:
        unsent_books = compatible_books[:ATTACHMENTS_LIMIT]

    log_string = ['{' + str(i) + '}' for i in range(len(unsent_books))]
    if len(unsent_books) > 0:
        log_string = ' '.join(log_string).format(*[book.id for book in unsent_books])
        if convert:
            log_string += ', with conversion'
    log.info(u"Processing book resend for user id {0}, book ids {1}".format(user_id, log_string))

    # Re-download and convert books that failed to send before.
    try:
        _send_books(user, unsent_books)
        # TODO: Reset all attempts to 0 before release.
        for book in unsent_books:
            book.num_attempts += 1
        db.session.commit()
    except:
        log.error(u"Failed to resend books for user id {0}".format(user_id),
                  exc_info=True)

    next_unsent_book = None
    if len(unsent_books) > 0:
        # If there are any more books to send after this batch, requeue them.
        next_unsent_book = unsent_books_query.filter(Book.id > unsent_books[-1].id).first()

    send_lock.release()

    # For some reason, calibre is leaving a lot of garbage files...
    filesystem.clear_calibre_files()

    if next_unsent_book is None:
        # If we've finished sending all books, including ones that need
        # conversion, clear all the books from the filesystem and finish
        # the task.
        if convert:
            kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id)
            # Dropbox may have registered more books, so don't clear them yet.
            if kindlebox_lock is None:
                return
            filesystem.clear_directory(user.get_directory())
            kindlebox_lock.release()
        # Else, start sending the books that need conversion.
        else:
            send_books.apply_async((user_id, ),
                                   {'convert': True},
                                   queue='conversion')
    else:
        queue_kwarg = {}
        if convert:
            queue_kwarg['queue'] = 'conversion'
        send_books.apply_async((user_id, ),
                               {
                                   'min_book_id': next_unsent_book.id,
                                   'convert': convert,
                               },
                               **queue_kwarg)
Exemplo n.º 7
0
def send_books(user_id, min_book_id=0, blocking=True):
    """
    Task to send any books associated with the given user ID that are marked as
    `unsent`. Sends a batch of at most `ATTACHMENTS_LIMIT` books, all with
    Book.id greater than or equal to the given `min_book_id`. Books are
    downloaded if necessary. Files that need conversion are converted before
    sending.

    Before finishing, the task queues another `send_books` task for the next
    batch of (distinct) books.
    """
    send_lock = acquire_send_books_lock(user_id, blocking)
    if send_lock is None:
        return

    # Only resend books for active users.
    user = User.query.filter_by(id=user_id, active=True).first()
    if user is None:
        return

    # Get the next batch of books that haven't been sent yet and are still
    # under the maximum number of send attempts.
    unsent_books_query = (user.books.filter_by(unsent=True).filter(
        Book.num_attempts < MAX_SEND_ATTEMPTS).order_by(Book.id))
    unsent_books = (unsent_books_query.filter(
        Book.id >= min_book_id).limit(ATTACHMENTS_LIMIT).all())
    if len(unsent_books) == 0:
        send_lock.release()
        return

    log_string = ['{' + str(i) + '}' for i in range(len(unsent_books))]
    log_string = ' '.join(log_string).format(
        *[book.id for book in unsent_books])
    log.info("Processing book resend for user id {0}, book ids {1}".format(
        user_id, log_string))

    # Re-download and convert books that failed to send before. Check that
    # hashes match.
    try:
        _send_books(user, unsent_books)
        for book in unsent_books:
            book.num_attempts += 1
        db.session.commit()
    except:
        log.error("Failed to resend books for user id {0}".format(user_id),
                  exc_info=True)
    finally:
        # If there are any more books to send after this batch, requeue them.
        next_unsent_book = unsent_books_query.filter(
            Book.id > unsent_books[-1].id).first()

        send_lock.release()

        # For some reason, calibre is leaving a lot of garbage files...
        filesystem.clear_calibre_files()

        if next_unsent_book is None:
            kindlebox_lock = acquire_kindlebox_lock(user.dropbox_id)
            # Dropbox may have registered more books, so don't clear them yet.
            if kindlebox_lock is None:
                return
            filesystem.clear_directory(user.get_directory())
            kindlebox_lock.release()
        else:
            send_books.delay(user_id, min_book_id=next_unsent_book.id)