def updatestats(): """ Get the changes since last time from the Pocket API """ logger = get_logger() session = get_db_connection() last_time = get_last_update() debug_print('Previous update: ' + datetimeutil.unix_to_string(last_time)) previously_unread = nr_unread(session) report = updatestats_since_last(logger, session, last_time) debug_print(report.pretty_print()) if report.net_result > 0: debug_print('More items added than read or deleted') elif report.net_result == 0: debug_print('Stagnating') else: # Calculate number of days it will take to finish the backlog at this rate #timedelta #days = last_time debug_print('Slowly but surely reading away your backlog') debug_print('\n' + get_read_progressbar(session)) debug_print(report.print_changed_articles(session)) logger.info(report)
def updatestats(): """ Get the changes since last time from the Pocket API """ logger = get_logger() session = get_db_connection() last_time = get_last_update() debug_print('Previous update: ' + datetimeutil.unix_to_string(last_time)) previously_unread = nr_unread(session) report = updatestats_since_last(logger, session, last_time) debug_print(report.pretty_print()) if report.net_result > 0: debug_print('More items added than read or deleted') elif report.net_result == 0: debug_print('Stagnating') else: # Calculate number of days it will take to finish the backlog at this rate #timedelta #days = last_time debug_print('Slowly but surely reading away your backlog') last_datetime = datetimeutil.unix_to_python(last_time) days = datetime.datetime.now() - last_datetime hours = days.total_seconds()//3600.0 items_read = report.net_result * -1 items_per_hour = float(items_read) / hours total_unread = nr_unread(session) debug_print('\nHours: {}, days: {}, items per hour: {} \n'.format(hours, days, items_per_hour)) debug_print('\nAt this rate of ' + str(items_read) + ' per ' + str(hours) + ' hours it takes ' + str(round(float(total_unread) / (items_per_hour * 24.0), 1)) + ' days to read the ' + str(total_unread) + ' remaining items') debug_print('\n' + get_read_progressbar(session)) debug_print(report.print_changed_articles(session)) logger.info(report)
def updatestats_since_last(logger, session, last_time): """ Get the changes since last time from the Pocket API """ pocket_instance = get_pocket_instance() if last_time: items = pocket_instance.get(since=last_time, state='all', detailType='complete') else: if DEBUG: # When debugging, limit to 20 items items = pocket_instance.get(count=20, state='all', detailType='complete') else: items = pocket_instance.get(state='all', detailType='complete') debug_print('Number of items in reponse: ' + str(len(items[0]['list']))) logger.debug('Number of items in response: ' + str(len(items[0]['list']))) now = datetime.datetime.now() report = Report(time_updated=now) nr_added = 0 nr_read = 0 nr_deleted = 0 nr_favourited = 0 nr_updated = 0 changed_articles = {'added': [], 'read': [], 'deleted': [], 'favourited': [], 'updated': []} report.time_since = datetimeutil.unix_to_python(items[0]['since']) report.time_since_unix = items[0]['since'] report.status = items[0]['status'] report.complete = items[0]['complete'] report.error = items[0]['error'] report.total_response = len(items[0]['list']) for item_id in items[0]['list']: item = items[0]['list'][item_id] existing_item = get_existing_item(session, item_id) if not existing_item: #article = Article(sort_id=item['sort_id'], item_id=item['item_id']) article = Article(item_id=item['item_id']) logger.debug('Existing item NOT found for ' + item_id) else: article = existing_item logger.debug('Existing item found for ' + item_id) if existing_item: previous_status = existing_item.status # 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted article.status = item['status'] try: if article.status == '0' and not existing_item: nr_added += 1 changed_articles['added'].append(item['item_id']) elif article.status == '1' and not existing_item: nr_added += 1 nr_read += 1 changed_articles['added'].append(item['item_id']) changed_articles['read'].append(item['item_id']) elif article.status == '1': nr_read += 1 changed_articles['read'].append(item['item_id']) elif article.status == '2' and not existing_item: nr_added += 1 nr_deleted += 1 changed_articles['added'].append(item['item_id']) changed_articles['deleted'].append(item['item_id']) elif article.status == '2': nr_deleted += 1 changed_articles['deleted'].append(item['item_id']) except KeyError: logger.info('No resolved_id found') #if not existing_item and not 'resolved_id' in item: if 'resolved_id' not in item: # Item was added and immediately deleted, or at least before we saw it logger.debug(stringutil.safe_unicode(item['status']) + ' ' + stringutil.safe_unicode(item['item_id']) + ' deleted') article.item_id = item['item_id'] article.firstseen_status = item['status'] article.firstseen_time = now try: article.firstseen_time_updated = datetimeutil.unix_to_python(item['time_updated']) except KeyError: pass # If item didn't exist yet, add it (otherwise it's updated automagically) session.add(article) # Skip the rest of the loop continue logger.debug(stringutil.safe_unicode(item['status']) + ' ' + stringutil.safe_unicode(item['item_id']) + ' ' + stringutil.safe_unicode(item['resolved_id']) + ' ' + datetimeutil.unix_to_string(item['time_added']) + ' ' + datetimeutil.unix_to_string(item['time_updated']) + ' ' + stringutil.safe_unicode(item['resolved_url'])) article.resolved_id = item['resolved_id'] article.sort_id = item['sort_id'] article.given_url = item['given_url'] article.resolved_url = item['resolved_url'] article.given_title = item['given_title'] article.resolved_title = item['resolved_title'] if existing_item and existing_item.favorite == 0 and item['favorite'] == '1': nr_favourited += 1 changed_articles['favourited'].append(item['item_id']) elif not existing_item and item['favorite'] == '1': nr_favourited += 1 changed_articles['favourited'].append(item['item_id']) article.favorite = item['favorite'] article.excerpt = item['excerpt'] article.is_article = item['is_article'] article.has_image = item['has_image'] article.has_video = item['has_video'] article.word_count = item['word_count'] if 'tags' in item: article.tags = json.dumps(item['tags']) if 'authors' in item: article.authors = json.dumps(item['authors']) if 'images' in item: article.images = json.dumps(item['images']) if 'videos' in item: article.videos = json.dumps(item['videos']) if existing_item and existing_item.time_updated != datetimeutil.unix_to_python(item['time_updated']): nr_updated += 1 changed_articles['updated'].append(item['item_id']) article.time_updated = datetimeutil.unix_to_python(item['time_updated']) article.time_favorited = datetimeutil.unix_to_python(item['time_favorited']) article.time_read = datetimeutil.unix_to_python(item['time_read']) if not existing_item: article.firstseen_status = item['status'] article.firstseen_time = now article.firstseen_time_updated = datetimeutil.unix_to_python(item['time_updated']) # If item didn't exist yet, add it (otherwise it's updated automagically) session.add(article) report.nr_added = nr_added report.nr_read = nr_read report.nr_favourited = nr_favourited report.nr_deleted = nr_deleted report.nr_updated = nr_updated report.changed_articles = json.dumps(changed_articles) #debug_print(report.changed_articles) session.add(report) # Check what's pending #logger.debug('About to commit to DB:') #logger.debug(session.new) # Save to DB session.commit() return report
def updatestats_since_last(logger, session, last_time): """ Get the changes since last time from the Pocket API """ pocket_instance = get_pocket_instance() if last_time: items = pocket_instance.get(since=last_time, state='all', detailType='complete') else: if DEBUG: # When debugging, limit to 20 items items = pocket_instance.get(count=20, state='all', detailType='complete') else: items = pocket_instance.get(state='all', detailType='complete') debug_print('Number of items in reponse: ' + str(len(items[0]['list']))) logger.debug('Number of items in response: ' + str(len(items[0]['list']))) now = datetime.datetime.now() report = Report(time_updated=now) nr_added = 0 nr_read = 0 nr_deleted = 0 nr_favourited = 0 nr_updated = 0 changed_articles = {'added': [], 'read': [], 'deleted': [], 'favourited': [], 'updated': []} report.time_since = datetimeutil.unix_to_python(items[0]['since']) report.time_since_unix = items[0]['since'] report.status = items[0]['status'] report.complete = items[0]['complete'] report.error = items[0]['error'] report.total_response = len(items[0]['list']) for item_id in items[0]['list']: item = items[0]['list'][item_id] existing_item = get_existing_item(session, item_id) if not existing_item: #article = Article(sort_id=item['sort_id'], item_id=item['item_id']) article = Article(item_id=item['item_id']) logger.debug('Existing item NOT found for ' + item_id) else: article = existing_item logger.debug('Existing item found for ' + item_id) if existing_item: previous_status = existing_item.status # 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted article.status = item['status'] try: if article.status == '0' and not existing_item: nr_added += 1 changed_articles['added'].append(item['item_id']) elif article.status == '1' and not existing_item: nr_added += 1 nr_read += 1 changed_articles['added'].append(item['item_id']) changed_articles['read'].append(item['item_id']) elif article.status == '1': nr_read += 1 changed_articles['read'].append(item['item_id']) elif article.status == '2' and not existing_item: nr_added += 1 nr_deleted += 1 changed_articles['added'].append(item['item_id']) changed_articles['deleted'].append(item['item_id']) elif article.status == '2': nr_deleted += 1 changed_articles['deleted'].append(item['item_id']) except KeyError: logger.info('No resolved_id found') #if not existing_item and not 'resolved_id' in item: if 'resolved_id' not in item: # Item was added and immediately deleted, or at least before we saw it logger.debug(stringutil.safe_unicode(item['status']) + ' ' + stringutil.safe_unicode(item['item_id']) + ' deleted') article.item_id = item['item_id'] article.firstseen_status = item['status'] article.firstseen_time = now try: article.firstseen_time_updated = datetimeutil.unix_to_python(item['time_updated']) except KeyError: pass # If item didn't exist yet, add it (otherwise it's updated automagically) session.add(article) # Skip the rest of the loop continue logger.debug(stringutil.safe_unicode(item['status']) + ' ' + stringutil.safe_unicode(item['item_id']) + ' ' + stringutil.safe_unicode(item['resolved_id']) + ' ' + datetimeutil.unix_to_string(item['time_added']) + ' ' + datetimeutil.unix_to_string(item['time_updated']) + ' ' + stringutil.safe_unicode(item['resolved_url'])) article.resolved_id = item['resolved_id'] article.sort_id = item['sort_id'] article.given_url = item['given_url'] article.resolved_url = item['resolved_url'] article.given_title = item['given_title'] article.resolved_title = item['resolved_title'] if existing_item and existing_item.favorite == 0 and item['favorite'] == '1': nr_favourited += 1 changed_articles['favourited'].append(item['item_id']) elif not existing_item and item['favorite'] == '1': nr_favourited += 1 changed_articles['favourited'].append(item['item_id']) article.favorite = item['favorite'] article.excerpt = item['excerpt'] article.is_article = item['is_article'] article.has_image = item['has_image'] article.has_video = item['has_video'] article.word_count = item['word_count'] if 'tags' in item: article.tags = json.dumps(item['tags']) if 'authors' in item: article.authors = json.dumps(item['authors']) if 'images' in item: article.images = json.dumps(item['images']) if 'videos' in item: article.videos = json.dumps(item['videos']) if existing_item and existing_item.time_updated != datetimeutil.unix_to_python(item['time_updated']): nr_updated += 1 changed_articles['updated'].append(item['item_id']) article.time_updated = datetimeutil.unix_to_python(item['time_updated']) article.time_favorited = datetimeutil.unix_to_python(item['time_favorited']) article.time_read = datetimeutil.unix_to_python(item['time_read']) if not existing_item: article.firstseen_status = item['status'] article.firstseen_time = now article.firstseen_time_updated = datetimeutil.unix_to_python(item['time_updated']) # If item didn't exist yet, add it (otherwise it's updated automagically) session.add(article) report.nr_added = nr_added report.nr_read = nr_read report.nr_favourited = nr_favourited report.nr_deleted = nr_deleted report.nr_updated = nr_updated report.changed_articles = json.dumps(changed_articles) #debug_print(report.changed_articles) session.add(report) # Check what's pending #logger.debug('About to commit to DB:') #logger.debug(session.new) # Save to DB session.commit() return report