def updatestats():
"""
Get the changes since last time from the Pocket API
"""
logger = get_logger()
session = get_db_connection()
last_time = get_last_update()
debug_print('Previous update: ' + datetimeutil.unix_to_string(last_time))
previously_unread = nr_unread(session)
report = updatestats_since_last(logger, session, last_time)
debug_print(report.pretty_print())
if report.net_result > 0:
debug_print('More items added than read or deleted')
elif report.net_result == 0:
debug_print('Stagnating')
else:
# Calculate number of days it will take to finish the backlog at this rate
#timedelta
#days = last_time
debug_print('Slowly but surely reading away your backlog')
debug_print('\n' + get_read_progressbar(session))
debug_print(report.print_changed_articles(session))
logger.info(report)
def updatestats():
"""
Get the changes since last time from the Pocket API
"""
logger = get_logger()
session = get_db_connection()
last_time = get_last_update()
debug_print('Previous update: ' + datetimeutil.unix_to_string(last_time))
previously_unread = nr_unread(session)
report = updatestats_since_last(logger, session, last_time)
debug_print(report.pretty_print())
if report.net_result > 0:
debug_print('More items added than read or deleted')
elif report.net_result == 0:
debug_print('Stagnating')
else:
# Calculate number of days it will take to finish the backlog at this rate
#timedelta
#days = last_time
debug_print('Slowly but surely reading away your backlog')
last_datetime = datetimeutil.unix_to_python(last_time)
days = datetime.datetime.now() - last_datetime
hours = days.total_seconds()//3600.0
items_read = report.net_result * -1
items_per_hour = float(items_read) / hours
total_unread = nr_unread(session)
debug_print('\nHours: {}, days: {}, items per hour: {} \n'.format(hours, days, items_per_hour))
debug_print('\nAt this rate of ' + str(items_read) + ' per ' + str(hours) + ' hours it takes ' + str(round(float(total_unread) / (items_per_hour * 24.0), 1)) + ' days to read the ' + str(total_unread) + ' remaining items')
debug_print('\n' + get_read_progressbar(session))
debug_print(report.print_changed_articles(session))
logger.info(report)
def updatestats_since_last(logger, session, last_time):
"""
Get the changes since last time from the Pocket API
"""
pocket_instance = get_pocket_instance()
if last_time:
items = pocket_instance.get(since=last_time, state='all', detailType='complete')
else:
if DEBUG:
# When debugging, limit to 20 items
items = pocket_instance.get(count=20, state='all', detailType='complete')
else:
items = pocket_instance.get(state='all', detailType='complete')
debug_print('Number of items in reponse: ' + str(len(items[0]['list'])))
logger.debug('Number of items in response: ' + str(len(items[0]['list'])))
now = datetime.datetime.now()
report = Report(time_updated=now)
nr_added = 0
nr_read = 0
nr_deleted = 0
nr_favourited = 0
nr_updated = 0
changed_articles = {'added': [], 'read': [], 'deleted': [], 'favourited': [], 'updated': []}
report.time_since = datetimeutil.unix_to_python(items[0]['since'])
report.time_since_unix = items[0]['since']
report.status = items[0]['status']
report.complete = items[0]['complete']
report.error = items[0]['error']
report.total_response = len(items[0]['list'])
for item_id in items[0]['list']:
item = items[0]['list'][item_id]
existing_item = get_existing_item(session, item_id)
if not existing_item:
#article = Article(sort_id=item['sort_id'], item_id=item['item_id'])
article = Article(item_id=item['item_id'])
logger.debug('Existing item NOT found for ' + item_id)
else:
article = existing_item
logger.debug('Existing item found for ' + item_id)
if existing_item:
previous_status = existing_item.status
# 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
article.status = item['status']
try:
if article.status == '0' and not existing_item:
nr_added += 1
changed_articles['added'].append(item['item_id'])
elif article.status == '1' and not existing_item:
nr_added += 1
nr_read += 1
changed_articles['added'].append(item['item_id'])
changed_articles['read'].append(item['item_id'])
elif article.status == '1':
nr_read += 1
changed_articles['read'].append(item['item_id'])
elif article.status == '2' and not existing_item:
nr_added += 1
nr_deleted += 1
changed_articles['added'].append(item['item_id'])
changed_articles['deleted'].append(item['item_id'])
elif article.status == '2':
nr_deleted += 1
changed_articles['deleted'].append(item['item_id'])
except KeyError:
logger.info('No resolved_id found')
#if not existing_item and not 'resolved_id' in item:
if 'resolved_id' not in item:
# Item was added and immediately deleted, or at least before we saw it
logger.debug(stringutil.safe_unicode(item['status']) + ' ' + stringutil.safe_unicode(item['item_id']) + ' deleted')
article.item_id = item['item_id']
article.firstseen_status = item['status']
article.firstseen_time = now
try:
article.firstseen_time_updated = datetimeutil.unix_to_python(item['time_updated'])
except KeyError:
pass
# If item didn't exist yet, add it (otherwise it's updated automagically)
session.add(article)
# Skip the rest of the loop
continue
logger.debug(stringutil.safe_unicode(item['status']) + ' ' + stringutil.safe_unicode(item['item_id']) + ' ' + stringutil.safe_unicode(item['resolved_id']) + ' ' + datetimeutil.unix_to_string(item['time_added']) + ' ' + datetimeutil.unix_to_string(item['time_updated']) + ' ' + stringutil.safe_unicode(item['resolved_url']))
article.resolved_id = item['resolved_id']
article.sort_id = item['sort_id']
article.given_url = item['given_url']
article.resolved_url = item['resolved_url']
article.given_title = item['given_title']
article.resolved_title = item['resolved_title']
if existing_item and existing_item.favorite == 0 and item['favorite'] == '1':
nr_favourited += 1
changed_articles['favourited'].append(item['item_id'])
elif not existing_item and item['favorite'] == '1':
nr_favourited += 1
changed_articles['favourited'].append(item['item_id'])
article.favorite = item['favorite']
article.excerpt = item['excerpt']
article.is_article = item['is_article']
article.has_image = item['has_image']
article.has_video = item['has_video']
article.word_count = item['word_count']
if 'tags' in item:
article.tags = json.dumps(item['tags'])
if 'authors' in item:
article.authors = json.dumps(item['authors'])
if 'images' in item:
article.images = json.dumps(item['images'])
if 'videos' in item:
article.videos = json.dumps(item['videos'])
if existing_item and existing_item.time_updated != datetimeutil.unix_to_python(item['time_updated']):
nr_updated += 1
changed_articles['updated'].append(item['item_id'])
article.time_updated = datetimeutil.unix_to_python(item['time_updated'])
article.time_favorited = datetimeutil.unix_to_python(item['time_favorited'])
article.time_read = datetimeutil.unix_to_python(item['time_read'])
if not existing_item:
article.firstseen_status = item['status']
article.firstseen_time = now
article.firstseen_time_updated = datetimeutil.unix_to_python(item['time_updated'])
# If item didn't exist yet, add it (otherwise it's updated automagically)
session.add(article)
report.nr_added = nr_added
report.nr_read = nr_read
report.nr_favourited = nr_favourited
report.nr_deleted = nr_deleted
report.nr_updated = nr_updated
report.changed_articles = json.dumps(changed_articles)
#debug_print(report.changed_articles)
session.add(report)
# Check what's pending
#logger.debug('About to commit to DB:')
#logger.debug(session.new)
# Save to DB
session.commit()
return report
def updatestats_since_last(logger, session, last_time):
"""
Get the changes since last time from the Pocket API
"""
pocket_instance = get_pocket_instance()
if last_time:
items = pocket_instance.get(since=last_time, state='all', detailType='complete')
else:
if DEBUG:
# When debugging, limit to 20 items
items = pocket_instance.get(count=20, state='all', detailType='complete')
else:
items = pocket_instance.get(state='all', detailType='complete')
debug_print('Number of items in reponse: ' + str(len(items[0]['list'])))
logger.debug('Number of items in response: ' + str(len(items[0]['list'])))
now = datetime.datetime.now()
report = Report(time_updated=now)
nr_added = 0
nr_read = 0
nr_deleted = 0
nr_favourited = 0
nr_updated = 0
changed_articles = {'added': [], 'read': [], 'deleted': [], 'favourited': [], 'updated': []}
report.time_since = datetimeutil.unix_to_python(items[0]['since'])
report.time_since_unix = items[0]['since']
report.status = items[0]['status']
report.complete = items[0]['complete']
report.error = items[0]['error']
report.total_response = len(items[0]['list'])
for item_id in items[0]['list']:
item = items[0]['list'][item_id]
existing_item = get_existing_item(session, item_id)
if not existing_item:
#article = Article(sort_id=item['sort_id'], item_id=item['item_id'])
article = Article(item_id=item['item_id'])
logger.debug('Existing item NOT found for ' + item_id)
else:
article = existing_item
logger.debug('Existing item found for ' + item_id)
if existing_item:
previous_status = existing_item.status
# 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
article.status = item['status']
try:
if article.status == '0' and not existing_item:
nr_added += 1
changed_articles['added'].append(item['item_id'])
elif article.status == '1' and not existing_item:
nr_added += 1
nr_read += 1
changed_articles['added'].append(item['item_id'])
changed_articles['read'].append(item['item_id'])
elif article.status == '1':
nr_read += 1
changed_articles['read'].append(item['item_id'])
elif article.status == '2' and not existing_item:
nr_added += 1
nr_deleted += 1
changed_articles['added'].append(item['item_id'])
changed_articles['deleted'].append(item['item_id'])
elif article.status == '2':
nr_deleted += 1
changed_articles['deleted'].append(item['item_id'])
except KeyError:
logger.info('No resolved_id found')
#if not existing_item and not 'resolved_id' in item:
if 'resolved_id' not in item:
# Item was added and immediately deleted, or at least before we saw it
logger.debug(stringutil.safe_unicode(item['status']) + ' ' + stringutil.safe_unicode(item['item_id']) + ' deleted')
article.item_id = item['item_id']
article.firstseen_status = item['status']
article.firstseen_time = now
try:
article.firstseen_time_updated = datetimeutil.unix_to_python(item['time_updated'])
except KeyError:
pass
# If item didn't exist yet, add it (otherwise it's updated automagically)
session.add(article)
# Skip the rest of the loop
continue
logger.debug(stringutil.safe_unicode(item['status']) + ' ' + stringutil.safe_unicode(item['item_id']) + ' ' + stringutil.safe_unicode(item['resolved_id']) + ' ' + datetimeutil.unix_to_string(item['time_added']) + ' ' + datetimeutil.unix_to_string(item['time_updated']) + ' ' + stringutil.safe_unicode(item['resolved_url']))
article.resolved_id = item['resolved_id']
article.sort_id = item['sort_id']
article.given_url = item['given_url']
article.resolved_url = item['resolved_url']
article.given_title = item['given_title']
article.resolved_title = item['resolved_title']
if existing_item and existing_item.favorite == 0 and item['favorite'] == '1':
nr_favourited += 1
changed_articles['favourited'].append(item['item_id'])
elif not existing_item and item['favorite'] == '1':
nr_favourited += 1
changed_articles['favourited'].append(item['item_id'])
article.favorite = item['favorite']
article.excerpt = item['excerpt']
article.is_article = item['is_article']
article.has_image = item['has_image']
article.has_video = item['has_video']
article.word_count = item['word_count']
if 'tags' in item:
article.tags = json.dumps(item['tags'])
if 'authors' in item:
article.authors = json.dumps(item['authors'])
if 'images' in item:
article.images = json.dumps(item['images'])
if 'videos' in item:
article.videos = json.dumps(item['videos'])
if existing_item and existing_item.time_updated != datetimeutil.unix_to_python(item['time_updated']):
nr_updated += 1
changed_articles['updated'].append(item['item_id'])
article.time_updated = datetimeutil.unix_to_python(item['time_updated'])
article.time_favorited = datetimeutil.unix_to_python(item['time_favorited'])
article.time_read = datetimeutil.unix_to_python(item['time_read'])
if not existing_item:
article.firstseen_status = item['status']
article.firstseen_time = now
article.firstseen_time_updated = datetimeutil.unix_to_python(item['time_updated'])
# If item didn't exist yet, add it (otherwise it's updated automagically)
session.add(article)
report.nr_added = nr_added
report.nr_read = nr_read
report.nr_favourited = nr_favourited
report.nr_deleted = nr_deleted
report.nr_updated = nr_updated
report.changed_articles = json.dumps(changed_articles)
#debug_print(report.changed_articles)
session.add(report)
# Check what's pending
#logger.debug('About to commit to DB:')
#logger.debug(session.new)
# Save to DB
session.commit()
return report