def parse_log(logfile):
    (loghash, logtime) = FILE_REGEX.match(logfile).group(1, 2)
    logtime = datetime.datetime.fromtimestamp(int(logtime) / 1000)

    fail = False
    logfile_full = log_dir + '/' + logfile
    try:
        logtext = gzip.open(logfile_full, 'rt', encoding='utf-8').read()
    except:
        try:
            logtext = open(logfile_full, 'rt', encoding='utf-8').read()
        except:
            logger.warn('Cannot read file: ' + logfile_full)
            failed[logfile] = sys.exc_info()
            failed_logtime[logfile] = logtime
            fail = True
    if not fail:
        try:
            game = gokoparse.parse_goko_log(logtext)
            game.logfile = logfile
            game.time = logtime
            games.append(game)
            if len(games) % 100 == 0:
                logger.info('Parsed %d' % len(games))
        except:
            failed[logfile] = sys.exc_info()
            failed_logtime[logfile] = logtime
def parse_log(logfile):
    (loghash, logtime) = FILE_REGEX.match(logfile).group(1, 2)
    logtime = datetime.datetime.fromtimestamp(int(logtime) / 1000)

    fail = False
    logfile_full = log_dir + '/' + logfile
    try:
        logtext = gzip.open(logfile_full, 'rt', encoding='utf-8').read()
    except:
        try:
            logtext = open(logfile_full, 'rt', encoding='utf-8').read()
        except:
            logger.warn('Cannot read file: ' + logfile_full)
            failed[logfile] = sys.exc_info()
            failed_logtime[logfile] = logtime
            fail = True
    if not fail:
        try:
            game = gokoparse.parse_goko_log(logtext)
            game.logfile = logfile
            game.time = logtime
            games.append(game)
            if len(games) % 100 == 0:
                logger.info('Parsed %d' % len(games))
        except:
            failed[logfile] = sys.exc_info()
            failed_logtime[logfile] = logtime
def parse_new_logs(date):

    # Determine which logs we have downloaded but not yet parsed
    log_dir = '%s/%s' % (LOG_DIR, date.strftime('%Y%m%d'))
    local_logs = os.listdir(log_dir)
    dblogs = db_manager.search_daily_log_filenames(date)
    not_parsed = set(local_logs) - set(dblogs)
    logger.info('Found %d downloaded logs to be parsed' % len(not_parsed))

    # Parse the unparsed logs
    games = []
    failed = {}
    failed_logtime = {}
    for logfile in not_parsed:
        (loghash, logtime) = FILE_REGEX.match(logfile).group(1, 2)
        logtime = datetime.datetime.fromtimestamp(int(logtime) / 1000)

        fail = False
        logfile_full = log_dir + '/' + logfile
        try:
            logtext = gzip.open(logfile_full, 'rt', encoding='utf-8').read()
        except:
            try:
                logtext = open(logfile_full, 'rt', encoding='utf-8').read()
            except:
                logger.warn('Cannot read file: ' + logfile_full)
                failed[logfile] = sys.exc_info()
                failed_logtime[logfile] = logtime
                fail = True
        if not fail:
            try:
                game = gokoparse.parse_goko_log(logtext)
                game.logfile = logfile
                game.time = logtime
                games.append(game)
                if len(games) % 100 == 0:
                    logger.info('Parsed %d' % len(games))
            except:
                failed[logfile] = sys.exc_info()
                failed_logtime[logfile] = logtime

    logger.info('Finished parsing')
    logger.info('%d games parsed' % len(games))
    logger.info('%d games failed' % len(failed))

    # Insert parsed games into database, 100 at a time
    count = 0
    total_inserted = 0
    try:
        db_manager.inserts(games)
        total_inserted += len(games)
    except:
        logger.info(sys.exc_info())

    logger.info('inserted %d parsed games into DB' % total_inserted)

    # Notify of failed logs.  Record in database.
    for f in failed:
        logger.warn('Failed to parse: %s in %s' % (failed[f][0].__name__, f))
        for line in traceback.format_tb(failed[f][2]):
            logger.warn(line)
        time = failed_logtime[f]
        db_manager.insert_parsefail(time, f, failed[f][0].__name__)

    return total_inserted
def parse_new_logs(date):

    # Determine which logs we have downloaded but not yet parsed
    log_dir = '%s/%s' % (LOG_DIR, date.strftime('%Y%m%d'))
    local_logs = os.listdir(log_dir)
    dblogs = db_manager.search_daily_log_filenames(date)
    not_parsed = set(local_logs) - set(dblogs)
    logger.info('Found %d downloaded logs to be parsed' % len(not_parsed))

    # Parse the unparsed logs
    games = []
    failed = {}
    failed_logtime = {}
    for logfile in not_parsed:
        (loghash, logtime) = FILE_REGEX.match(logfile).group(1, 2)
        logtime = datetime.datetime.fromtimestamp(int(logtime) / 1000)

        fail = False
        logfile_full = log_dir + '/' + logfile
        try:
            logtext = gzip.open(logfile_full, 'rt', encoding='utf-8').read()
        except:
            try:
                logtext = open(logfile_full, 'rt', encoding='utf-8').read()
            except:
                logger.warn('Cannot read file: ' + logfile_full)
                failed[logfile] = sys.exc_info()
                failed_logtime[logfile] = logtime
                fail = True
        if not fail:
            try:
                game = gokoparse.parse_goko_log(logtext)
                game.logfile = logfile
                game.time = logtime
                games.append(game)
                if len(games) % 100 == 0:
                    logger.info('Parsed %d' % len(games))
            except:
                failed[logfile] = sys.exc_info()
                failed_logtime[logfile] = logtime

    logger.info('Finished parsing')
    logger.info('%d games parsed' % len(games))
    logger.info('%d games failed' % len(failed))

    # Insert parsed games into database, 100 at a time
    count = 0
    total_inserted = 0
    try:
        db_manager.inserts(games)
        total_inserted += len(games)
    except:
        logger.info(sys.exc_info())

    logger.info('inserted %d parsed games into DB' % total_inserted)

    # Notify of failed logs.  Record in database.
    for f in failed:
        logger.warn('Failed to parse: %s in %s' % (failed[f][0].__name__, f))
        for line in traceback.format_tb(failed[f][2]):
            logger.warn(line)
        time = failed_logtime[f]
        db_manager.insert_parsefail(time, f, failed[f][0].__name__)

    return total_inserted
def parse_new_logs(date):

    # Determine which logs we have downloaded but not yet parsed
    log_dir = '/mnt/raid/media/dominion/logs/%s' % date.strftime('%Y%m%d')
    local_logs = os.listdir(log_dir)
    dblogs = db_manager.search_daily_log_filenames(date)
    not_parsed = set(local_logs) - set(dblogs)
    logging.info('Found %d downloaded logs to be parsed' % len(not_parsed))

    # Parse the unparsed logs
    games = []
    failed = {}
    for logfile in not_parsed:
        (loghash, logtime) = FILE_REGEX.match(logfile).group(1, 2)
        logtime = datetime.datetime.fromtimestamp(int(logtime) / 1000)

        fail = False
        logfile_full = log_dir + '/' + logfile
        try:
            logtext = gzip.open(logfile_full, 'rt', encoding='utf-8').read()
        except:
            try:
                logtext = open(logfile_full, 'rt', encoding='utf-8').read()
            except:
                logging.warn('Cannot read file: ' + logfile_full)
                failed[logfile] = sys.exc_info()
                fail = True
        if not fail:
            try:
                game = gokoparse.parse_goko_log(logtext)
                game.logfile = logfile
                game.time = logtime
                games.append(game)
                if len(games) % 100 == 0:
                    logging.info('Parsed %d' % len(games))
            except:
                failed[logfile] = sys.exc_info()

    logging.info('Finished parsing')
    logging.info('%d games parsed' % len(games))
    logging.info('%d games failed' % len(failed))

    # Insert parsed games into database, 100 at a time
    # TODO: Logs are read in file order, not chronological order.  This
    # is a (minor) violation of the TrueSkill algorithm: a game played at
    # 11:59 PM might be processed before one played at 12:00 AM on the
    # same day (almost 24 hours earlier).
    count = 0
    total_inserted = 0
    try:
        db_manager.inserts(games)
        # Update ratings
        count = -1
        while count != 0:
          x = db_manager.get_last_rated_game()
          if x:
            (t, l) = x
          else:
            (t, l) = (None, None)

          count = ts.record_ratings(100, t, l, ts.isodominion_env)
          total_inserted += count
    except:
        logging.info(sys.exc_info())

    logging.info('inserted %d parsed games into DB' % total_inserted)

    # Notify of failed logs.  Record in database.
    for f in failed:
        logging.warn('Failed to parse: %s in %s' % (failed[f][0].__name__, f))
        for line in traceback.format_tb(failed[f][2]):
            logging.warn(line)
        #TODO: record failures
        db_manager.insert_parsefail(f)

    return total_inserted