Example #1
0
def updatestats():
    """
    Get the changes since last time from the Pocket API
    """
    logger = get_logger()
    session = get_db_connection()

    last_time = get_last_update()
    debug_print('Previous update: ' + datetimeutil.unix_to_string(last_time))

    previously_unread = nr_unread(session)

    report = updatestats_since_last(logger, session, last_time)

    debug_print(report.pretty_print())

    if report.net_result > 0:
        debug_print('More items added than read or deleted')
    elif report.net_result == 0:
        debug_print('Stagnating')
    else:
        # Calculate number of days it will take to finish the backlog at this rate
        #timedelta
        #days = last_time
        debug_print('Slowly but surely reading away your backlog')

    debug_print('\n' + get_read_progressbar(session))

    debug_print(report.print_changed_articles(session))
    logger.info(report)
Example #2
0
def updatestats():
    """
    Get the changes since last time from the Pocket API
    """
    logger = get_logger()
    session = get_db_connection()

    last_time = get_last_update()
    debug_print('Previous update: ' + datetimeutil.unix_to_string(last_time))

    previously_unread = nr_unread(session)

    report = updatestats_since_last(logger, session, last_time)

    debug_print(report.pretty_print())

    if report.net_result > 0:
        debug_print('More items added than read or deleted')
    elif report.net_result == 0:
        debug_print('Stagnating')
    else:
        # Calculate number of days it will take to finish the backlog at this rate
        #timedelta
        #days = last_time
        debug_print('Slowly but surely reading away your backlog')

        last_datetime = datetimeutil.unix_to_python(last_time)
        days = datetime.datetime.now() - last_datetime
        hours = days.total_seconds()//3600.0
        items_read = report.net_result * -1
        items_per_hour = float(items_read) / hours
        total_unread = nr_unread(session)
        debug_print('\nHours: {}, days: {}, items per hour: {} \n'.format(hours, days, items_per_hour))
        debug_print('\nAt this rate of ' + str(items_read) + ' per ' + str(hours) + ' hours it takes ' + str(round(float(total_unread) / (items_per_hour * 24.0), 1)) + ' days to read the ' + str(total_unread) + ' remaining items')

    debug_print('\n' + get_read_progressbar(session))

    debug_print(report.print_changed_articles(session))
    logger.info(report)
Example #3
0
def updatestats_since_last(logger, session, last_time):
    """
    Get the changes since last time from the Pocket API
    """
    pocket_instance = get_pocket_instance()
    if last_time:
        items = pocket_instance.get(since=last_time, state='all', detailType='complete')
    else:
        if DEBUG:
            # When debugging, limit to 20 items
            items = pocket_instance.get(count=20, state='all', detailType='complete')
        else:
            items = pocket_instance.get(state='all', detailType='complete')
    debug_print('Number of items in reponse: ' + str(len(items[0]['list'])))
    logger.debug('Number of items in response: ' + str(len(items[0]['list'])))

    now = datetime.datetime.now()
    report = Report(time_updated=now)
    nr_added = 0
    nr_read = 0
    nr_deleted = 0
    nr_favourited = 0
    nr_updated = 0
    changed_articles = {'added': [], 'read': [], 'deleted': [], 'favourited': [], 'updated': []}
    report.time_since = datetimeutil.unix_to_python(items[0]['since'])
    report.time_since_unix = items[0]['since']
    report.status = items[0]['status']
    report.complete = items[0]['complete']
    report.error = items[0]['error']
    report.total_response = len(items[0]['list'])

    for item_id in items[0]['list']:
        item = items[0]['list'][item_id]
        existing_item = get_existing_item(session, item_id)
        if not existing_item:
            #article = Article(sort_id=item['sort_id'], item_id=item['item_id'])
            article = Article(item_id=item['item_id'])
            logger.debug('Existing item NOT found for ' + item_id)
        else:
            article = existing_item
            logger.debug('Existing item found for ' + item_id)

        if existing_item:
            previous_status = existing_item.status
        # 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
        article.status = item['status']
        try:
            if article.status == '0' and not existing_item:
                nr_added += 1
                changed_articles['added'].append(item['item_id'])
            elif article.status == '1' and not existing_item:
                nr_added += 1
                nr_read += 1
                changed_articles['added'].append(item['item_id'])
                changed_articles['read'].append(item['item_id'])
            elif article.status == '1':
                nr_read += 1
                changed_articles['read'].append(item['item_id'])
            elif article.status == '2' and not existing_item:
                nr_added += 1
                nr_deleted += 1
                changed_articles['added'].append(item['item_id'])
                changed_articles['deleted'].append(item['item_id'])
            elif article.status == '2':
                nr_deleted += 1
                changed_articles['deleted'].append(item['item_id'])
        except KeyError:
            logger.info('No resolved_id found')

        #if not existing_item and not 'resolved_id' in item:
        if 'resolved_id' not in item:
            # Item was added and immediately deleted, or at least before we saw it
            logger.debug(stringutil.safe_unicode(item['status']) + ' ' + stringutil.safe_unicode(item['item_id']) + ' deleted')

            article.item_id = item['item_id']
            article.firstseen_status = item['status']
            article.firstseen_time = now
            try:
                article.firstseen_time_updated = datetimeutil.unix_to_python(item['time_updated'])
            except KeyError:
                pass

            # If item didn't exist yet, add it (otherwise it's updated automagically)
            session.add(article)
            # Skip the rest of the loop
            continue

        logger.debug(stringutil.safe_unicode(item['status']) + ' ' + stringutil.safe_unicode(item['item_id']) + ' ' + stringutil.safe_unicode(item['resolved_id']) + ' ' + datetimeutil.unix_to_string(item['time_added']) + ' ' + datetimeutil.unix_to_string(item['time_updated']) + ' ' + stringutil.safe_unicode(item['resolved_url']))
        article.resolved_id = item['resolved_id']
        article.sort_id = item['sort_id']
        article.given_url = item['given_url']
        article.resolved_url = item['resolved_url']
        article.given_title = item['given_title']
        article.resolved_title = item['resolved_title']
        if existing_item and existing_item.favorite == 0 and item['favorite'] == '1':
            nr_favourited += 1
            changed_articles['favourited'].append(item['item_id'])
        elif not existing_item and item['favorite'] == '1':
            nr_favourited += 1
            changed_articles['favourited'].append(item['item_id'])
        article.favorite = item['favorite']

        article.excerpt = item['excerpt']
        article.is_article = item['is_article']
        article.has_image = item['has_image']
        article.has_video = item['has_video']
        article.word_count = item['word_count']
        if 'tags' in item:
            article.tags = json.dumps(item['tags'])
        if 'authors' in item:
            article.authors = json.dumps(item['authors'])
        if 'images' in item:
            article.images = json.dumps(item['images'])
        if 'videos' in item:
            article.videos = json.dumps(item['videos'])
        if existing_item and existing_item.time_updated != datetimeutil.unix_to_python(item['time_updated']):
            nr_updated += 1
            changed_articles['updated'].append(item['item_id'])
        article.time_updated = datetimeutil.unix_to_python(item['time_updated'])
        article.time_favorited = datetimeutil.unix_to_python(item['time_favorited'])
        article.time_read = datetimeutil.unix_to_python(item['time_read'])
        if not existing_item:
            article.firstseen_status = item['status']
            article.firstseen_time = now
            article.firstseen_time_updated = datetimeutil.unix_to_python(item['time_updated'])

            # If item didn't exist yet, add it (otherwise it's updated automagically)
            session.add(article)

    report.nr_added = nr_added
    report.nr_read = nr_read
    report.nr_favourited = nr_favourited
    report.nr_deleted = nr_deleted
    report.nr_updated = nr_updated
    report.changed_articles = json.dumps(changed_articles)
    #debug_print(report.changed_articles)
    session.add(report)

    # Check what's pending
    #logger.debug('About to commit to DB:')
    #logger.debug(session.new)

    # Save to DB
    session.commit()

    return report
Example #4
0
def updatestats_since_last(logger, session, last_time):
    """
    Get the changes since last time from the Pocket API
    """
    pocket_instance = get_pocket_instance()
    if last_time:
        items = pocket_instance.get(since=last_time, state='all', detailType='complete')
    else:
        if DEBUG:
            # When debugging, limit to 20 items
            items = pocket_instance.get(count=20, state='all', detailType='complete')
        else:
            items = pocket_instance.get(state='all', detailType='complete')
    debug_print('Number of items in reponse: ' + str(len(items[0]['list'])))
    logger.debug('Number of items in response: ' + str(len(items[0]['list'])))

    now = datetime.datetime.now()
    report = Report(time_updated=now)
    nr_added = 0
    nr_read = 0
    nr_deleted = 0
    nr_favourited = 0
    nr_updated = 0
    changed_articles = {'added': [], 'read': [], 'deleted': [], 'favourited': [], 'updated': []}
    report.time_since = datetimeutil.unix_to_python(items[0]['since'])
    report.time_since_unix = items[0]['since']
    report.status = items[0]['status']
    report.complete = items[0]['complete']
    report.error = items[0]['error']
    report.total_response = len(items[0]['list'])

    for item_id in items[0]['list']:
        item = items[0]['list'][item_id]
        existing_item = get_existing_item(session, item_id)
        if not existing_item:
            #article = Article(sort_id=item['sort_id'], item_id=item['item_id'])
            article = Article(item_id=item['item_id'])
            logger.debug('Existing item NOT found for ' + item_id)
        else:
            article = existing_item
            logger.debug('Existing item found for ' + item_id)

        if existing_item:
            previous_status = existing_item.status
        # 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
        article.status = item['status']
        try:
            if article.status == '0' and not existing_item:
                nr_added += 1
                changed_articles['added'].append(item['item_id'])
            elif article.status == '1' and not existing_item:
                nr_added += 1
                nr_read += 1
                changed_articles['added'].append(item['item_id'])
                changed_articles['read'].append(item['item_id'])
            elif article.status == '1':
                nr_read += 1
                changed_articles['read'].append(item['item_id'])
            elif article.status == '2' and not existing_item:
                nr_added += 1
                nr_deleted += 1
                changed_articles['added'].append(item['item_id'])
                changed_articles['deleted'].append(item['item_id'])
            elif article.status == '2':
                nr_deleted += 1
                changed_articles['deleted'].append(item['item_id'])
        except KeyError:
            logger.info('No resolved_id found')

        #if not existing_item and not 'resolved_id' in item:
        if 'resolved_id' not in item:
            # Item was added and immediately deleted, or at least before we saw it
            logger.debug(stringutil.safe_unicode(item['status']) + ' ' + stringutil.safe_unicode(item['item_id']) + ' deleted')

            article.item_id = item['item_id']
            article.firstseen_status = item['status']
            article.firstseen_time = now
            try:
                article.firstseen_time_updated = datetimeutil.unix_to_python(item['time_updated'])
            except KeyError:
                pass

            # If item didn't exist yet, add it (otherwise it's updated automagically)
            session.add(article)
            # Skip the rest of the loop
            continue

        logger.debug(stringutil.safe_unicode(item['status']) + ' ' + stringutil.safe_unicode(item['item_id']) + ' ' + stringutil.safe_unicode(item['resolved_id']) + ' ' + datetimeutil.unix_to_string(item['time_added']) + ' ' + datetimeutil.unix_to_string(item['time_updated']) + ' ' + stringutil.safe_unicode(item['resolved_url']))
        article.resolved_id = item['resolved_id']
        article.sort_id = item['sort_id']
        article.given_url = item['given_url']
        article.resolved_url = item['resolved_url']
        article.given_title = item['given_title']
        article.resolved_title = item['resolved_title']
        if existing_item and existing_item.favorite == 0 and item['favorite'] == '1':
            nr_favourited += 1
            changed_articles['favourited'].append(item['item_id'])
        elif not existing_item and item['favorite'] == '1':
            nr_favourited += 1
            changed_articles['favourited'].append(item['item_id'])
        article.favorite = item['favorite']

        article.excerpt = item['excerpt']
        article.is_article = item['is_article']
        article.has_image = item['has_image']
        article.has_video = item['has_video']
        article.word_count = item['word_count']
        if 'tags' in item:
            article.tags = json.dumps(item['tags'])
        if 'authors' in item:
            article.authors = json.dumps(item['authors'])
        if 'images' in item:
            article.images = json.dumps(item['images'])
        if 'videos' in item:
            article.videos = json.dumps(item['videos'])
        if existing_item and existing_item.time_updated != datetimeutil.unix_to_python(item['time_updated']):
            nr_updated += 1
            changed_articles['updated'].append(item['item_id'])
        article.time_updated = datetimeutil.unix_to_python(item['time_updated'])
        article.time_favorited = datetimeutil.unix_to_python(item['time_favorited'])
        article.time_read = datetimeutil.unix_to_python(item['time_read'])
        if not existing_item:
            article.firstseen_status = item['status']
            article.firstseen_time = now
            article.firstseen_time_updated = datetimeutil.unix_to_python(item['time_updated'])

            # If item didn't exist yet, add it (otherwise it's updated automagically)
            session.add(article)

    report.nr_added = nr_added
    report.nr_read = nr_read
    report.nr_favourited = nr_favourited
    report.nr_deleted = nr_deleted
    report.nr_updated = nr_updated
    report.changed_articles = json.dumps(changed_articles)
    #debug_print(report.changed_articles)
    session.add(report)

    # Check what's pending
    #logger.debug('About to commit to DB:')
    #logger.debug(session.new)

    # Save to DB
    session.commit()

    return report