def backfill(group_name, date=None): log.info('{}: Backfilling group...'.format(group_name)) server = Server() _, count, first, last, _ = server.group(group_name) if date: target_article = server.day_to_post(group_name, server.days_old(date)) else: target_article = server.day_to_post(group_name, config.site['backfill_days']) group = db.groups.find_one({'name': group_name}) if group: # if the group hasn't been updated before, quit if not group['first']: log.error('{}: Need to run a normal update prior to backfilling group.'.format(group_name)) if server.connection: server.connection.quit() return False log.info('{0}: Server has {1:d} - {2:d} or ~{3:d} days.' .format(group_name, first, last, server.days_old(server.post_date(group_name, first))) ) # if the first article we have is lower than the target if target_article >= group['first']: log.info('{}: Nothing to do, we already have the target post.'.format(group_name)) if server.connection: server.connection.quit() return True # or if the target is below the server's first if target_article < first: log.warning( '{}: Backfill target is older than the server\'s retention. Setting target to the first possible article.'.format( group_name)) target_article = first total = group['first'] - target_article end = group['first'] - 1 start = end - MESSAGE_LIMIT + 1 if target_article > start: start = target_article while True: messages = server.scan(group_name, start, end) if messages: if parts.save_all(messages): db.groups.update({ '_id': group['_id'] }, { '$set': { 'first': start } }) pass else: log.error('{}: Failed while saving parts.'.format(group_name)) if server.connection: server.connection.quit() return False if start == target_article: if server.connection: server.connection.quit() return True else: end = start - 1 start = end - MESSAGE_LIMIT + 1 if target_article > start: start = target_article else: log.error('{}: Group doesn\'t exist in db.'.format(group_name)) if server.connection: server.connection.quit() return False
def backfill(group_name, date=None): log.info('group: {}: backfilling group'.format(group_name)) server = Server() _, count, first, last, _ = server.group(group_name) if date: target_article = server.day_to_post(group_name, server.days_old(date)) else: target_article = server.day_to_post(group_name, config.scan.get('backfill_days', 10)) group = db.groups.find_one({'name': group_name}) if group: # if the group hasn't been updated before, quit if not group['first']: log.error('group: {}: run a normal update prior to backfilling'.format(group_name)) if server.connection: server.connection.quit() return False # if the first article we have is lower than the target if target_article >= group['first']: log.info('group: {}: Nothing to do, we already have the target post.'.format(group_name)) if server.connection: server.connection.quit() return True # or if the target is below the server's first if target_article < first: target_article = first total = group['first'] - target_article end = group['first'] - 1 start = end - MESSAGE_LIMIT + 1 if target_article > start: start = target_article retries = 0 while True: messages = server.scan(group_name, start, end) if messages: if parts.save_all(messages): db.groups.update({ '_id': group['_id'] }, { '$set': { 'first': start } }) retries = 0 else: log.error('group: {}: failed while saving parts'.format(group_name)) if server.connection: server.connection.quit() return False else: log.error('group: {}: problem updating group - trying again'.format(group_name)) retries += 1 # keep trying the same block 3 times, then skip if retries <= 3: continue if start == target_article: if server.connection: server.connection.quit() return True else: end = start - 1 start = end - MESSAGE_LIMIT + 1 if target_article > start: start = target_article else: log.error('group: {}: group doesn\'t exist in db.'.format(group_name)) if server.connection: server.connection.quit() return False