def assign_clusters():
    '''
    This function does the actual donor assignment, assigning unique donor IDs in the contribution table
    based on clusters of contributions that appear to have the same donor. It works pretty much the same
    as the mark_matches function above.
    '''
    # Again, instantiate and train our classifier
    clf = RandomForestClassifier(n_estimators=10, random_state=0)
    clf = clf.fit([eval(t.features) for t in TRAINING_DATA],
                  [int(t.same) for t in TRAINING_DATA])

    # Loop through the last name groups
    print 'Processing groups ...'
    for g in Contribution.objects.all().values('group_id').distinct():
        if not g['group_id']: continue
        toupdate = []
        G = nx.Graph(
        )  # Create an empty network graph for each last name group
        # We're using a simple hash function to help generate unique donor IDs
        nameid = hashlib.sha224(str(g['group_id'])).hexdigest()
        # For each match in a last name group
        for m in Match.objects.filter(c1__group_id=g['group_id']):
            # Do the two contributions have the same donor? Same as above.
            edge = clf.predict_proba(eval(m.features))
            if edge[0][1] > edge[0][0]:
                # If they do, add an edge between those contributions in the network graph we created
                # a few steps ago. This process is outlined in the steps here:
                # https://github.com/cjdd3b/fec-standardizer/wiki/Matching-donors
                G.add_edge(m.c1, m.c2)

        # Now we want to go through the graph we created and basically find all the contributions that are
        # connected. If the contributions were connected in the step above, that means they're probably from
        # the same donor. So a donor is basically defined by small networks of connected contributions. This
        # is described further here: https://github.com/cjdd3b/fec-standardizer/wiki/Defining-donor-clusters
        ccs = connected_components(G)

        # Now loop through each of the donor clusters generated by the connected_components function
        for c in enumerate(ccs):
            donor_id = c[0]
            for i in c[1]:
                # Create a donor ID based on our group hash above and the enumerated cluster number
                classifier_id = '%s%s' % (donor_id, nameid)
                i.classifier_id = classifier_id[:12]
                toupdate.append(i)
        # Bulk save the donor IDs to the contribution table
        commit_saves(toupdate)

    print 'Cleaning up the leftovers ...'
    tocleanup = []
    for record in Contribution.objects.filter(classifier_id__isnull=True):
        if not record.match_repr: continue
        classifier_id = '99%s' % hashlib.sha224(record.match_repr).hexdigest()
        record.classifier_id = classifier_id[:12]
        tocleanup.append(record)
    commit_saves(tocleanup)
    return
def assign_clusters():
    '''
    This function does the actual donor assignment, assigning unique donor IDs in the contribution table
    based on clusters of contributions that appear to have the same donor. It works pretty much the same
    as the mark_matches function above.
    '''
    # Again, instantiate and train our classifier
    clf = RandomForestClassifier(n_estimators=10, random_state=0)
    clf = clf.fit([eval(t.features) for t in TRAINING_DATA], [int(t.same) for t in TRAINING_DATA])

    # Loop through the last name groups
    print 'Processing groups ...'
    for g in Contribution.objects.all().values('group_id').distinct():
        if not g['group_id']: continue
        toupdate = []
        G = nx.Graph() # Create an empty network graph for each last name group
        # We're using a simple hash function to help generate unique donor IDs
        nameid = hashlib.sha224(str(g['group_id'])).hexdigest()
        # For each match in a last name group
        for m in Match.objects.filter(c1__group_id=g['group_id']):
            # Do the two contributions have the same donor? Same as above.
            edge = clf.predict_proba(eval(m.features))
            if edge[0][1] > edge[0][0]:
                # If they do, add an edge between those contributions in the network graph we created
                # a few steps ago. This process is outlined in the steps here:
                # https://github.com/cjdd3b/fec-standardizer/wiki/Matching-donors
                G.add_edge(m.c1, m.c2)

        # Now we want to go through the graph we created and basically find all the contributions that are
        # connected. If the contributions were connected in the step above, that means they're probably from
        # the same donor. So a donor is basically defined by small networks of connected contributions. This
        # is described further here: https://github.com/cjdd3b/fec-standardizer/wiki/Defining-donor-clusters
        ccs = connected_components(G)

        # Now loop through each of the donor clusters generated by the connected_components function
        for c in enumerate(ccs):
            donor_id = c[0]
            for i in c[1]:
                # Create a donor ID based on our group hash above and the enumerated cluster number
                classifier_id = '%s%s' % (donor_id, nameid)
                i.classifier_id = classifier_id[:12]
                toupdate.append(i)
        # Bulk save the donor IDs to the contribution table
        commit_saves(toupdate)

    print 'Cleaning up the leftovers ...'
    tocleanup = []
    for record in Contribution.objects.filter(classifier_id__isnull=True):
        if not record.match_repr: continue
        classifier_id = '99%s' % hashlib.sha224(record.match_repr).hexdigest()
        record.classifier_id = classifier_id[:12]
        tocleanup.append(record)
    commit_saves(tocleanup)
    return
def mark_matches():
    ''' 
    This function is mostly just here for testing whether the classifier and CRP agree at the match
    level. It doesn't do anything in terms of grouping donors, unlike the assign_clusters
    function below. You can run it or not -- it doesn't matter.
    '''
    # Prepare our Random Forest classifier and train using the training set above
    clf = RandomForestClassifier(n_estimators=10, random_state=0)
    # Training the Random Forest takes two inputs: a list of feature vectors and a list of correct
    # classifications associated with those vectors. More info here:
    # http://scikit-learn.org/dev/modules/generated/sklearn.ensemble.RandomForestClassifier.html
    clf = clf.fit([eval(t.features) for t in TRAINING_DATA],
                  [int(t.same) for t in TRAINING_DATA])

    # Loop through each of the initial groups we created
    for g in Contribution.objects.all().values('group_id').distinct():
        toupdate = []
        # Now go through all the contribution pairs within a given group
        for m in Match.objects.filter(c1__group_id=g['group_id']):
            # Use the classifier to predict, based on the match feature vector, whether the donors
            # are the same. This returns a probability score of yes vs. no
            edge = clf.predict_proba(eval(m.features))
            # If the yes probability outweighs the no
            if edge[0][1] > edge[0][0]:
                # Mark them as the same and record confidence score
                m.classifier_same = True
                m.score = edge[0][1]
            else:
                # Or else mark them as a non-match and record the score
                m.classifier_same = False
                m.score = edge[0][0]
            # Now compare our classifier's judgment vs. the ground truth CRP data and mark accordingly
            m.match_status = get_match_status(m)
            toupdate.append(m)
        # Using manual transaction management to speed things up. See utils/db.py
        commit_saves(toupdate)
    return
def mark_matches():
    ''' 
    This function is mostly just here for testing whether the classifier and CRP agree at the match
    level. It doesn't do anything in terms of grouping donors, unlike the assign_clusters
    function below. You can run it or not -- it doesn't matter.
    '''
    # Prepare our Random Forest classifier and train using the training set above
    clf = RandomForestClassifier(n_estimators=10, random_state=0)
    # Training the Random Forest takes two inputs: a list of feature vectors and a list of correct
    # classifications associated with those vectors. More info here:
    # http://scikit-learn.org/dev/modules/generated/sklearn.ensemble.RandomForestClassifier.html
    clf = clf.fit([eval(t.features) for t in TRAINING_DATA], [int(t.same) for t in TRAINING_DATA])

    # Loop through each of the initial groups we created
    for g in Contribution.objects.all().values('group_id').distinct():
        toupdate = []
        # Now go through all the contribution pairs within a given group
        for m in Match.objects.filter(c1__group_id=g['group_id']):
            # Use the classifier to predict, based on the match feature vector, whether the donors
            # are the same. This returns a probability score of yes vs. no
            edge = clf.predict_proba(eval(m.features))
            # If the yes probability outweighs the no
            if edge[0][1] > edge[0][0]:
                # Mark them as the same and record confidence score
                m.classifier_same = True
                m.score = edge[0][1]
            else:
                # Or else mark them as a non-match and record the score
                m.classifier_same = False
                m.score = edge[0][0]
            # Now compare our classifier's judgment vs. the ground truth CRP data and mark accordingly
            m.match_status = get_match_status(m)
            toupdate.append(m)
        # Using manual transaction management to speed things up. See utils/db.py
        commit_saves(toupdate)
    return