def backfill(group_name, date=None):
log.info('{}: Backfilling group...'.format(group_name))
server = Server()
_, count, first, last, _ = server.group(group_name)
if date:
target_article = server.day_to_post(group_name, server.days_old(date))
else:
target_article = server.day_to_post(group_name, config.site['backfill_days'])
group = db.groups.find_one({'name': group_name})
if group:
# if the group hasn't been updated before, quit
if not group['first']:
log.error('{}: Need to run a normal update prior to backfilling group.'.format(group_name))
if server.connection:
server.connection.quit()
return False
log.info('{0}: Server has {1:d} - {2:d} or ~{3:d} days.'
.format(group_name, first, last, server.days_old(server.post_date(group_name, first)))
)
# if the first article we have is lower than the target
if target_article >= group['first']:
log.info('{}: Nothing to do, we already have the target post.'.format(group_name))
if server.connection:
server.connection.quit()
return True
# or if the target is below the server's first
if target_article < first:
log.warning(
'{}: Backfill target is older than the server\'s retention. Setting target to the first possible article.'.format(
group_name))
target_article = first
total = group['first'] - target_article
end = group['first'] - 1
start = end - MESSAGE_LIMIT + 1
if target_article > start:
start = target_article
while True:
messages = server.scan(group_name, start, end)
if messages:
if parts.save_all(messages):
db.groups.update({
'_id': group['_id']
},
{
'$set': {
'first': start
}
})
pass
else:
log.error('{}: Failed while saving parts.'.format(group_name))
if server.connection:
server.connection.quit()
return False
if start == target_article:
if server.connection:
server.connection.quit()
return True
else:
end = start - 1
start = end - MESSAGE_LIMIT + 1
if target_article > start:
start = target_article
else:
log.error('{}: Group doesn\'t exist in db.'.format(group_name))
if server.connection:
server.connection.quit()
return False
def backfill(group_name, date=None):
log.info('group: {}: backfilling group'.format(group_name))
server = Server()
_, count, first, last, _ = server.group(group_name)
if date:
target_article = server.day_to_post(group_name, server.days_old(date))
else:
target_article = server.day_to_post(group_name, config.scan.get('backfill_days', 10))
group = db.groups.find_one({'name': group_name})
if group:
# if the group hasn't been updated before, quit
if not group['first']:
log.error('group: {}: run a normal update prior to backfilling'.format(group_name))
if server.connection:
server.connection.quit()
return False
# if the first article we have is lower than the target
if target_article >= group['first']:
log.info('group: {}: Nothing to do, we already have the target post.'.format(group_name))
if server.connection:
server.connection.quit()
return True
# or if the target is below the server's first
if target_article < first:
target_article = first
total = group['first'] - target_article
end = group['first'] - 1
start = end - MESSAGE_LIMIT + 1
if target_article > start:
start = target_article
retries = 0
while True:
messages = server.scan(group_name, start, end)
if messages:
if parts.save_all(messages):
db.groups.update({
'_id': group['_id']
},
{
'$set': {
'first': start
}
})
retries = 0
else:
log.error('group: {}: failed while saving parts'.format(group_name))
if server.connection:
server.connection.quit()
return False
else:
log.error('group: {}: problem updating group - trying again'.format(group_name))
retries += 1
# keep trying the same block 3 times, then skip
if retries <= 3:
continue
if start == target_article:
if server.connection:
server.connection.quit()
return True
else:
end = start - 1
start = end - MESSAGE_LIMIT + 1
if target_article > start:
start = target_article
else:
log.error('group: {}: group doesn\'t exist in db.'.format(group_name))
if server.connection:
server.connection.quit()
return False
