def evaluate_fit(trained_model, test_sessions, test_queries):
    """
    Evaluate the model's fit to the observed data - i.e. whether C==0 or C==1 for every
    item/session.

    There are two measures:

    - Log likelihood goes from negative infinity (bad) to 0 (good)
    - It measures the likelihood of observing the clicks in all the test sessions if the model is correct
    - Perplexity goes from 1 (good) to 2 (bad).
    - It's a measure of how surprised we are about all clicks and non-clicks in all of the test sessions if the model is correct.
    - When comparing models you can use perplexity gain (pB - pA) / (pB - 1)
    - It can be computed at individual ranks, or averaged across all ranks. Perplexity is normally higher for higher ranks.
    """
    print("-------------------------------")
    print("Testing on %d search sessions (%d unique queries)." % (len(test_sessions), len(test_queries)))
    print("-------------------------------")

    loglikelihood = LogLikelihood()
    perplexity = Perplexity()

    start = time.time()
    ll_value = loglikelihood.evaluate(trained_model, test_sessions)
    end = time.time()
    print("\tlog-likelihood: %f; time: %i secs" % (ll_value, end - start))

    start = time.time()
    perp_value = perplexity.evaluate(trained_model, test_sessions)[0]
    end = time.time()
    print("\tperplexity: %f; time: %i secs" % (perp_value, end - start))
Beispiel #2
0
    test_queries = Utils.get_unique_queries(test_sessions)

    print("===============================")
    print("Training on %d search sessions (%d unique queries)." %
          (len(train_sessions), len(train_queries)))
    print("===============================")

    start = time.time()
    click_model.train(train_sessions)
    end = time.time()
    print("\tTrained %s click model in %i secs:\n%r" %
          (click_model.__class__.__name__, end - start, click_model))

    print("-------------------------------")
    print("Testing on %d search sessions (%d unique queries)." %
          (len(test_sessions), len(test_queries)))
    print("-------------------------------")

    loglikelihood = LogLikelihood()
    perplexity = Perplexity()

    start = time.time()
    ll_value = loglikelihood.evaluate(click_model, test_sessions)
    end = time.time()
    print("\tlog-likelihood: %f; time: %i secs" % (ll_value, end - start))

    start = time.time()
    perp_value = perplexity.evaluate(click_model, test_sessions)[0]
    end = time.time()
    print("\tperplexity: %f; time: %i secs" % (perp_value, end - start))
Beispiel #3
0
    end = time.time()
    logger.info('Finish training. Time consumed: {} seconds'.format(end -
                                                                    start))

    # Log likelihood
    logger.info('Computing log likelihood')
    loglikelihood = LogLikelihood()
    start = time.time()
    ll_value = loglikelihood.evaluate(click_model, test_dataset)
    end = time.time()
    logger.info('Log likelihood: {}. Time consumed: {} seconds'.format(
        ll_value, end - start))

    # Perplexity
    logger.info('Computing perplexity')
    perplexity = Perplexity()
    start = time.time()
    perp_value = perplexity.evaluate(click_model, test_dataset)[0]
    end = time.time()
    logger.info('Perplexity: {}. Time consumed: {} seconds'.format(
        perp_value, end - start))

    # NDCG
    logger.info('Computing NDCG@k')
    RelevanceEstimation = RankingPerformance(args)
    start = time.time()
    ndcg_version1 = {}
    ndcg_version2 = {}
    ks = [1, 3, 5, 10]
    for k in ks:
        ndcg_version1[k], ndcg_version2[k] = RelevanceEstimation.evaluate(
Beispiel #4
0
        sys.exit(1)

    click_model = globals()[sys.argv[1]]()
    search_sessions_path = sys.argv[2]
    search_sessions_num = int(sys.argv[3])

    search_sessions = YandexRelPredChallengeParser().parse(search_sessions_path, search_sessions_num)

    train_test_split = int(len(search_sessions) * 0.75)
    train_sessions = search_sessions[:train_test_split]
    train_queries = Utils.get_unique_queries(train_sessions)

    test_sessions = Utils.filter_sessions(search_sessions[train_test_split:], train_queries)
    test_queries = Utils.get_unique_queries(test_sessions)

    print "-------------------------------"
    print "Training on %d search sessions (%d unique queries)." % (len(train_sessions), len(train_queries))
    print "-------------------------------"

    click_model.train(train_sessions)
    print "\tTrained %s click model:\n%r" % (click_model.__class__.__name__, click_model)

    print "-------------------------------"
    print "Testing on %d search sessions (%d unique queries)." % (len(test_sessions), len(test_queries))
    print "-------------------------------"

    loglikelihood = LogLikelihood()
    print "\tlog-likelihood: %f" % loglikelihood.evaluate(click_model, test_sessions)
    perplexity = Perplexity()
    print "\tperplexity: %f" % perplexity.evaluate(click_model, test_sessions)[0]
Beispiel #5
0
        search_sessions_path, search_sessions_num)

    train_test_split = int(len(search_sessions) * 0.75)
    train_sessions = search_sessions[:train_test_split]
    train_queries = Utils.get_unique_queries(train_sessions)

    test_sessions = Utils.filter_sessions(search_sessions[train_test_split:],
                                          train_queries)
    test_queries = Utils.get_unique_queries(test_sessions)

    print "-------------------------------"
    print "Training on %d search sessions (%d unique queries)." % (
        len(train_sessions), len(train_queries))
    print "-------------------------------"

    click_model.train(train_sessions)
    print "\tTrained %s click model:\n%r" % (click_model.__class__.__name__,
                                             click_model)

    print "-------------------------------"
    print "Testing on %d search sessions (%d unique queries)." % (
        len(test_sessions), len(test_queries))
    print "-------------------------------"

    loglikelihood = LogLikelihood()
    print "\tlog-likelihood: %f" % loglikelihood.evaluate(
        click_model, test_sessions)
    perplexity = Perplexity()
    print "\tperplexity: %f" % perplexity.evaluate(click_model,
                                                   test_sessions)[0]
Beispiel #6
0
        sys.exit(1)

    click_model = globals()[sys.argv[1]]()
    search_sessions_path = sys.argv[2]
    search_sessions_num = int(sys.argv[3])

    search_sessions = YandexRelPredChallengeParser().parse(search_sessions_path, search_sessions_num)

    train_test_split = int(len(search_sessions) * 0.75)
    train_sessions = search_sessions[:train_test_split]
    train_queries = Utils.get_unique_queries(train_sessions)

    test_sessions = Utils.filter_sessions(search_sessions[train_test_split:], train_queries)
    test_queries = Utils.get_unique_queries(test_sessions)

    print "-------------------------------"
    print "Training on %d search sessions (%d unique queries)." % (len(train_sessions), len(train_queries))
    print "-------------------------------"

    click_model.train(search_sessions)
    #print "\tTrained %s click model:\n%r" % (click_model.__class__.__name__, click_model)

    print "-------------------------------"
    print "Testing on %d search sessions (%d unique queries)." % (len(test_sessions), len(test_queries))
    print "-------------------------------"

    loglikelihood = LogLikelihood()
    print "\tlog-likelihood: %f" % loglikelihood.evaluate(click_model, test_queries)
    perplexity = Perplexity()
    print "\tperplexity: %f" % perplexity.evaluate(click_model, test_queries)[0]