def past_timesteps_experiment_50_50_split(ndims, improper_classification,
                                          num_previous, avg_variant,
                                          date_of_wordvecs,
                                          subtract_temporal_average_bool):
    if avg_variant == 'avg':
        print "AVG version is not supported for previous timestep mode"
        return
    elif avg_variant == 'pca':
        fmris = load_avg_pca(ndims, 1)
        for mask in fmris.keys():
            fmri = fmris[mask]
            print "size of fmri PCA: " + str(fmri.shape)
            # assume that time = #cols
            num_time_steps = fmri.shape[1]
            fmri_tr = fmri[:, xrange(num_time_steps / 2 + 1)]
            fmri_tst = fmri[:, xrange(num_time_steps / 2 + 1, num_time_steps)]
            fmri_tr = add_prev_time_steps(fmri_tr, num_previous)
            fmri_tst = add_prev_time_steps(fmri_tst, num_previous)
            new_fmri = np.c_[fmri_tr, fmri_tst]
            fmris[mask] = new_fmri

    elif avg_variant == 'srm' or avg_variant == 'srmica':
        # here, srm means we don't do the testing over T iterations
        # we just use the average of all of them without error bars
        if avg_variant == 'srm':
            srms = load_srm(ndims, 1)
        elif avg_variant == 'srmica':
            srms = load_srmica(ndims, 1)
        # calculate the average for each mask, and replace with that
        fmris = {}
        for mask in srms.keys():
            #if mask != 'erez_dmna_network':
            #	continue
            srm = srms[mask]
            Ws = srm[0]  # the map
            S = srm[1]  # averaged SRM-projected training data
            num_training_time_steps = S.shape[1]
            tst_data_list = srm[2]  # test portion of the data, pre-selected
            #print "tst_data_list length = " + str(len(tst_data_list))
            num_subjs = len(tst_data_list)
            avg_srm_tst_data = None
            for i in xrange(num_subjs):
                #print "Subject #"+ str(i)
                tst_data = tst_data_list[i]
                #print "Test data shape = " + str(tst_data.shape)
                transformed = np.dot(Ws[i, :, :].T, tst_data)
                if i == 0:
                    avg_srm_tst_data = transformed / (0. + num_subjs)
                else:
                    avg_srm_tst_data += transformed / (0. + num_subjs)
            # add previous time steps
            # note that we add previous time steps SEPARATELY to training and test
            # this means that there is NO OVERLAP between training and test in the previous time steps.
            S_prev_times = add_prev_time_steps(S, num_previous)
            avg_srm_prev_times = add_prev_time_steps(avg_srm_tst_data,
                                                     num_previous)
            #print "shape of S_prev_times = " + str(S_prev_times.shape)
            #print "shape of avg_srm_prev_times = " + str(avg_srm_prev_times.shape)
            fmri_mask = np.c_[S_prev_times, avg_srm_prev_times]
            fmris[mask] = fmri_mask

    else:
        print 'avg_variant is wrong'
        return
    if date_of_wordvecs == 'may15':
        semantic_vecs = load_may15_annotation_vecs()
    elif date_of_wordvecs == 'sep12weighted':
        semantic_vecs = load_sep12_weighted_word_vecs_annotations()
    elif date_of_wordvecs == 'sep12unweighted':
        semantic_vecs = load_sep12_unweighted_word_vecs_annotations()
    else:
        # default
        semantic_vecs = load_sep12_weighted_word_vecs_annotations()
    print "Semantic vector shape = " + str(semantic_vecs.shape)
    num_sem_vecs = semantic_vecs.shape[1]  # assume time is # cols
    # don't subtract out average of all semantic vectors for all time points
    # only calculate average for training, and subtract that average out of the test
    # CHANGE MADE: sep 19 12:40 AM
    '''
	# subtract out average if we're doing this case
	if subtract_temporal_average_bool == True:
		semantic_vecs, avg_semantic_vec = subtract_column_mean(semantic_vecs)
	'''

    #print "num semantic vectors = " + str(num_sem_vecs)
    word_tr = semantic_vecs[:, xrange(num_sem_vecs / 2 + 1)]
    #print "word_tr.shape = " + str(word_tr.shape)
    word_tst = semantic_vecs[:, xrange(num_sem_vecs / 2 + 1, num_sem_vecs)]
    #print "word_tst.shape = " + str(word_tst.shape)
    #print "word_tr.shape = " + str(word_tr.shape)
    #print "word_tst.shape = " + str(word_tst.shape)

    # don't subtract out average of all semantic vectors for all time points
    # only calculate average for training, and subtract that average out of the test
    # CHANGE MADE: sep 19 12:40 AM
    if subtract_temporal_average_bool == True:
        word_tr, avg_tr_word_vec = subtract_column_mean(word_tr)
        word_tst = word_tst - avg_tr_word_vec[:, np.newaxis]

    # add previous time steps to semantic stuff
    word_tr = add_prev_time_steps(word_tr, num_previous)
    word_tst = add_prev_time_steps(word_tst, num_previous)

    mask_results = {}
    for mask in fmris.keys():
        fmri = fmris[mask]

        num_total_time_steps = fmri.shape[1]
        #print "fmri.shape = " + str(fmri.shape)
        fmri_tr = fmri[:, xrange(num_sem_vecs / 2 + 1)]
        fmri_tst = fmri[:, xrange(num_sem_vecs / 2 + 1, num_sem_vecs)]
        print "fmri_tr.shape = " + str(fmri_tr.shape)
        print "fmri_tst.shape = " + str(fmri_tst.shape)
        print "Training linear maps..."
        if improper_classification == None:
            # truncate one side before learning linear map (for each direction)
            # modify fmri_tr, word_tr (aren't used again)
            long_fmri_tr = fmri_tr.copy()
            long_word_tr = word_tr.copy()
            short_fmri_tr = keep_only_first_time_step(fmri_tr, num_previous)
            short_word_tr = keep_only_first_time_step(word_tr, num_previous)
            # learn long fmri -> short text
            Wridge_ft, useless1, Wpro_ft, useless2 = learn_linear_maps(
                xrange(num_sem_vecs / 2 + 1), [], long_fmri_tr,
                mask + avg_variant, short_word_tr, date_of_wordvecs)
            # learn long text -> short fmri
            useless1, Wridge_tf, useless2, Wpro_tf = learn_linear_maps(
                xrange(num_sem_vecs / 2 + 1), [], short_fmri_tr,
                mask + avg_variant, long_word_tr, date_of_wordvecs)
            #clear memory
            useless1 = None
            useless2 = None
            long_fmri_tr = None
            long_word_tr = None
            short_fmri_tr = None
            short_word_tr = None

        else:
            Wridge_ft, Wridge_tf, Wpro_ft, Wpro_tf = learn_linear_maps(
                xrange(num_sem_vecs / 2 + 1), [], fmri_tr, mask + avg_variant,
                word_tr, date_of_wordvecs)
        print "Done training linear maps..."
        print "Wridge_ft.shape = " + str(Wridge_ft[0].shape)
        print "Wridge_tf.shape = " + str(Wridge_tf[0].shape)
        print "Wpro_ft.shape = " + str(Wpro_ft[0].shape)
        print "Wpro_tf.shape = " + str(Wpro_tf[0].shape)

        TF_class_scores = []
        TF_rank_scores = []
        FT_class_scores = []
        FT_rank_scores = []

        num_time_steps = fmri_tst.shape[
            1]  # assume time steps are # of columns
        chunk_array = []
        index = 0
        num_chunks = 25  # CHANGED JAN 11
        while index < num_time_steps:
            chunk_array.append(index)
            index += num_time_steps / num_chunks
            if index > num_time_steps:
                index = num_time_steps
        k = 1  #-> 4% chance rate

        if improper_classification == True or improper_classification == False:
            # comparisons in fMRI space (i.e. text -> fMRI) (best was ridge, use that map first)
            # truth is fmri_tst
            # prediction is Wridge_tf*word_tst
            # flipped is using procrustes instead
            TF_prediction = np.dot(Wridge_tf, word_tst)
            TF_prediction_flipped = np.dot(Wpro_tf, word_tst)
            # comparisons in Semantic space (i.e. fMRI -> text) (best was procrustes, use that map first)
            # truth is word_tst
            # prediction is Wpro_ft*fmri_tst
            # flipped is using ridge instead
            FT_prediction = np.dot(Wpro_ft, fmri_tst)
            FT_prediction_flipped = np.dot(Wridge_ft, fmri_tst)

            # fix shape
            TF_prediction = TF_prediction[0, :, :]
            TF_prediction_flipped = TF_prediction_flipped[0, :, :]
            FT_prediction = FT_prediction[0, :, :]
            FT_prediction_flipped = FT_prediction_flipped[0, :, :]

        elif improper_classification == None:
            long_fmri_tst = fmri_tst
            long_word_tst = word_tst

            # these linear maps will already be in the short space
            # so the predictions don't need to be truncated, they're already the right size
            TF_prediction = np.dot(Wridge_tf, long_word_tst)
            TF_prediction_flipped = np.dot(Wpro_tf, long_word_tst)

            FT_prediction = np.dot(Wpro_ft, long_fmri_tst)
            FT_prediction_flipped = np.dot(Wridge_ft, long_fmri_tst)

            # fix shape
            TF_prediction = TF_prediction[0, :, :]
            TF_prediction_flipped = TF_prediction_flipped[0, :, :]
            FT_prediction = FT_prediction[0, :, :]
            FT_prediction_flipped = FT_prediction_flipped[0, :, :]

            TF_truth = keep_only_first_time_step(fmri_tst, num_previous)
            FT_truth = keep_only_first_time_step(word_tst, num_previous)

        Wridge_tf = None
        Wpro_tf = None
        Wpro_ft = None
        Wridge_ft = None

        if improper_classification == True:
            # no truncation
            TF_truth = fmri_tst.copy()
            FT_truth = word_tst.copy()

        # Truncate after learning linear maps btwn long and long
        elif improper_classification == False:
            # evaluate in fMRI space
            TF_prediction = keep_only_first_time_step(TF_prediction,
                                                      num_previous)
            TF_prediction_flipped = keep_only_first_time_step(
                TF_prediction_flipped, num_previous)

            # evaluate in text space
            FT_prediction = keep_only_first_time_step(FT_prediction,
                                                      num_previous)
            FT_prediction_flipped = keep_only_first_time_step(
                FT_prediction_flipped, num_previous)

            TF_truth = keep_only_first_time_step(fmri_tst, num_previous)
            FT_truth = keep_only_first_time_step(word_tst, num_previous)

        # check everything is the right size
        print "TF_prediction.shape = " + str(TF_prediction.shape)
        print "TF_prediction_flipped.shape = " + str(
            TF_prediction_flipped.shape)
        print "TF truth shape: " + str(TF_truth.shape)
        assert (TF_truth.shape == TF_prediction.shape)
        assert (TF_truth.shape == TF_prediction_flipped.shape)

        print "FT_prediction.shape = " + str(FT_prediction.shape)
        print "FT_prediction_flipped.shape = " + str(
            FT_prediction_flipped.shape)
        print "FT truth shape: " + str(FT_truth.shape)
        assert (FT_truth.shape == FT_prediction.shape)
        assert (FT_truth.shape == FT_prediction_flipped.shape)

        # CALCULATE SCORES

        TF_classification_score = scene_classification(TF_truth, TF_prediction,
                                                       chunk_array, k)
        TF_classification_score_flipped = scene_classification(
            TF_truth, TF_prediction_flipped, chunk_array, k)
        TF_rank_score, TF_ranks = scene_ranking(TF_truth, TF_prediction,
                                                chunk_array)
        TF_rank_score_flipped, TF_ranks_flipped = scene_ranking(
            TF_truth, TF_prediction_flipped, chunk_array)
        TF_rank_score = TF_rank_score / (num_chunks + 0.)
        TF_rank_score_flipped = TF_rank_score_flipped / (num_chunks + 0.)

        FT_classification_score = scene_classification(FT_truth, FT_prediction,
                                                       chunk_array, k)
        FT_classification_score_flipped = scene_classification(
            FT_truth, FT_prediction_flipped, chunk_array, k)
        FT_rank_score, FT_ranks = scene_ranking(FT_truth, FT_prediction,
                                                chunk_array)
        FT_rank_score_flipped, FT_ranks_flipped = scene_ranking(
            FT_truth, FT_prediction_flipped, chunk_array)
        FT_rank_score = FT_rank_score / (num_chunks + 0.)
        FT_rank_score_flipped = FT_rank_score_flipped / (num_chunks + 0.)

        # DISPLAY SCORES

        print "---------------------------------------------------------------------------"
        print "Mask: " + mask
        print "Using Ridge for Text -> fMRI and Procrustes for fMRI -> Text"
        print "Text -> fMRI (Ridge) scene classification (" + str(
            float(k) * 100 /
            num_chunks) + "% chance) avg = " + str(TF_classification_score)
        print "Text -> fMRI (Ridge) scene ranking (50% chance) avg = " + str(
            TF_rank_score)
        print "fMRI -> Text (Procrustes) scene classification (" + str(
            float(k) * 100 /
            num_chunks) + "% chance) avg = " + str(FT_classification_score)
        print "fMRI -> Text (Procrustes) scene ranking (50% chance) avg = " + str(
            FT_rank_score)
        print "---------------------------------------------------------------------------"
        print "Using Procrustes for Text -> fMRI and Ridge for fMRI -> Text"
        print "Text -> fMRI (Procrustes) scene classification (" + str(
            float(k) * 100 / num_chunks) + "% chance) avg = " + str(
                TF_classification_score_flipped)
        print "Text -> fMRI (Procrustes) scene ranking (50% chance) avg = " + str(
            TF_rank_score_flipped)
        print "fMRI -> Text (Ridge) scene classification (" + str(
            float(k) * 100 / num_chunks) + "% chance) avg = " + str(
                FT_classification_score_flipped)
        print "fMRI -> Text (Ridge) scene ranking (50% chance) avg = " + str(
            FT_rank_score_flipped)
        print "---------------------------------------------------------------------------"
        print "///////////////////////////////////////////////////////////////////////////"

        mask_results[mask] = (TF_classification_score, TF_rank_score,
                              FT_classification_score, FT_rank_score,
                              TF_classification_score_flipped,
                              TF_rank_score_flipped,
                              FT_classification_score_flipped,
                              FT_rank_score_flipped)
    return mask_results
def experiment_50_50_split(ndims, avg_variant, date_of_wordvecs,
                           subtract_temporal_average_bool):
    if avg_variant == 'avg':
        fmris = load_avg(1)  # 1 because there's no averaging (see load_fmri)
    elif avg_variant == 'pca':
        fmris = load_avg_pca(ndims, 1)
    elif avg_variant == 'srm' or avg_variant == 'srmica':
        # here, srm means we don't do the testing over T iterations
        # we just use the average of all of them without error bars
        if avg_variant == 'srm':
            srms = load_srm(ndims, 1)
        elif avg_variant == 'srmica':
            srms = load_srmica(ndims, 1)
        # calculate the average for each mask, and replace with that
        fmris = {}
        for mask in srms.keys():
            #if mask != 'erez_dmna_network':
            #	continue
            srm = srms[mask]
            Ws = srm[0]  # the map
            S = srm[1]  # averaged SRM-projected training data
            num_training_time_steps = S.shape[1]
            tst_data_list = srm[2]  # test portion of the data, pre-selected
            #print "tst_data_list length = " + str(len(tst_data_list))
            num_subjs = len(tst_data_list)
            avg_srm_tst_data = None
            for i in xrange(num_subjs):
                #print "Subject #"+ str(i)
                tst_data = tst_data_list[i]
                #print "Test data shape = " + str(tst_data.shape)
                transformed = np.dot(Ws[i, :, :].T, tst_data)
                if i == 0:
                    avg_srm_tst_data = transformed / (0. + num_subjs)
                else:
                    avg_srm_tst_data += transformed / (0. + num_subjs)
            fmri_mask = np.c_[S, avg_srm_tst_data]
            fmris[mask] = fmri_mask

    else:
        print 'avg_variant is wrong'
        return
    if date_of_wordvecs == 'may15':
        semantic_vecs = load_may15_annotation_vecs()
    elif date_of_wordvecs == 'sep12weighted':
        semantic_vecs = load_sep12_weighted_word_vecs_annotations()
    elif date_of_wordvecs == 'sep12unweighted':
        semantic_vecs = load_sep12_unweighted_word_vecs_annotations()
    else:
        # default
        semantic_vecs = load_sep12_weighted_word_vecs_annotations()
    print "Semantic vector shape = " + str(semantic_vecs.shape)
    num_sem_vecs = semantic_vecs.shape[1]  # assume time is # cols
    # don't subtract out average of all semantic vectors for all time points
    # only calculate average for training, and subtract that average out of the test
    # CHANGE MADE: sep 19 12:40 AM
    '''
	# subtract out average if we're doing this case
	if subtract_temporal_average_bool == True:
		semantic_vecs, avg_semantic_vec = subtract_column_mean(semantic_vecs)
	'''

    #print "num semantic vectors = " + str(num_sem_vecs)
    word_tr = semantic_vecs[:, xrange(num_sem_vecs / 2 + 1)]
    #print "word_tr.shape = " + str(word_tr.shape)
    word_tst = semantic_vecs[:, xrange(num_sem_vecs / 2 + 1, num_sem_vecs)]
    #print "word_tst.shape = " + str(word_tst.shape)
    #print "word_tr.shape = " + str(word_tr.shape)
    #print "word_tst.shape = " + str(word_tst.shape)

    # don't subtract out average of all semantic vectors for all time points
    # only calculate average for training, and subtract that average out of the test
    # CHANGE MADE: sep 19 12:40 AM
    if subtract_temporal_average_bool == True:
        word_tr, avg_tr_word_vec = subtract_column_mean(word_tr)
        word_tst = word_tst - avg_tr_word_vec[:, np.newaxis]

    mask_results = {}
    for mask in fmris.keys():
        fmri = fmris[mask]

        num_total_time_steps = fmri.shape[1]
        #print "fmri.shape = " + str(fmri.shape)
        fmri_tr = fmri[:, xrange(num_sem_vecs / 2 + 1)]
        fmri_tst = fmri[:, xrange(num_sem_vecs / 2 + 1, num_sem_vecs)]
        #print "fmri_tr.shape = " + str(fmri_tr.shape)
        #print "fmri_tst.shape = " + str(fmri_tst.shape)
        #print "Training linear maps..."
        Wridge_ft, Wridge_tf, Wpro_ft, Wpro_tf = learn_linear_maps(
            xrange(num_sem_vecs / 2 + 1), [], fmri_tr, mask + avg_variant,
            word_tr, date_of_wordvecs)
        #print "Done training linear maps..."

        TF_class_scores = []
        TF_rank_scores = []
        FT_class_scores = []
        FT_rank_scores = []

        num_time_steps = fmri_tst.shape[
            1]  # assume time steps are # of columns
        chunk_array = []
        index = 0
        num_chunks = 25  ### CHANGED JAN 11
        while index < num_time_steps:
            chunk_array.append(index)
            index += num_time_steps / num_chunks
            if index > num_time_steps:
                index = num_time_steps
        k = 1  #-> 4% chance rate ## CHANGED FEB 4
        # comparisons in fMRI space (i.e. text -> fMRI) (best was ridge, use that map first)
        # truth is fmri_tst
        # prediction is Wridge_tf*word_tst
        # flipped is using procrustes instead
        TF_truth = fmri_tst
        TF_prediction = np.dot(Wridge_tf, word_tst)
        TF_prediction_flipped = np.dot(Wpro_tf, word_tst)
        TF_prediction = TF_prediction[0, :, :]
        TF_prediction_flipped = TF_prediction_flipped[0, :, :]
        #print "TF pred shape: " + str(TF_prediction.shape)
        #print "TF pred flipped shape: " + str(TF_prediction_flipped.shape)
        #print "TF truth shape: " + str(TF_truth.shape)
        assert (TF_truth.shape == TF_prediction.shape)
        assert (TF_truth.shape == TF_prediction_flipped.shape)

        TF_classification_score = scene_classification(TF_truth, TF_prediction,
                                                       chunk_array, k)
        TF_classification_score_flipped = scene_classification(
            TF_truth, TF_prediction_flipped, chunk_array, k)
        TF_rank_score, TF_ranks = scene_ranking(TF_truth, TF_prediction,
                                                chunk_array)
        TF_rank_score_flipped, TF_ranks_flipped = scene_ranking(
            TF_truth, TF_prediction_flipped, chunk_array)
        TF_rank_score = TF_rank_score / (num_chunks + 0.)
        TF_rank_score_flipped = TF_rank_score_flipped / (num_chunks + 0.)

        # comparisons in Semantic space (i.e. fMRI -> text) (best was procrustes, use that map first)
        # truth is word_tst
        # prediction is Wpro_ft*fmri_tst
        # flipped is using ridge instead
        FT_truth = word_tst
        FT_prediction = np.dot(Wpro_ft, fmri_tst)
        FT_prediction_flipped = np.dot(Wridge_ft, fmri_tst)
        FT_prediction = FT_prediction[0, :, :]
        FT_prediction_flipped = FT_prediction_flipped[0, :, :]
        #print "FT pred shape: " + str(FT_prediction.shape)
        #print "FT pred flipped shape: " + str(FT_prediction_flipped.shape)
        #print "FT truth shape: " + str(FT_truth.shape)
        assert (FT_truth.shape == FT_prediction.shape)
        assert (FT_truth.shape == FT_prediction_flipped.shape)

        FT_classification_score = scene_classification(FT_truth, FT_prediction,
                                                       chunk_array, k)
        FT_classification_score_flipped = scene_classification(
            FT_truth, FT_prediction_flipped, chunk_array, k)
        FT_rank_score, FT_ranks = scene_ranking(FT_truth, FT_prediction,
                                                chunk_array)
        FT_rank_score_flipped, FT_ranks_flipped = scene_ranking(
            FT_truth, FT_prediction_flipped, chunk_array)
        FT_rank_score = FT_rank_score / (num_chunks + 0.)
        FT_rank_score_flipped = FT_rank_score_flipped / (num_chunks + 0.)

        print "Mask: " + mask
        print "Using Ridge for Text -> fMRI and Procrustes for fMRI -> Text"
        print "Text -> fMRI (Ridge) scene classification (" + str(
            float(k) /
            num_chunks) + "% chance) avg = " + str(TF_classification_score)
        print "Text -> fMRI (Ridge) scene ranking (50% chance) avg = " + str(
            TF_rank_score)
        print "fMRI -> Text (Procrustes) scene classification (" + str(
            float(k) /
            num_chunks) + "% chance) avg = " + str(FT_classification_score)
        print "fMRI -> Text (Procrustes) scene ranking (50% chance) avg = " + str(
            FT_rank_score)
        print "---------------------------------------------------------------------------"
        print "Using Procrustes for Text -> fMRI and Ridge for fMRI -> Text"
        print "Text -> fMRI (Procrustes) scene classification (" + str(
            float(k) / num_chunks) + "% chance) avg = " + str(
                TF_classification_score_flipped)
        print "Text -> fMRI (Procrustes) scene ranking (50% chance) avg = " + str(
            TF_rank_score_flipped)
        print "fMRI -> Text (Ridge) scene classification (" + str(
            float(k) / num_chunks) + "% chance) avg = " + str(
                FT_classification_score_flipped)
        print "fMRI -> Text (Ridge) scene ranking (50% chance) avg = " + str(
            FT_rank_score_flipped)
        print "///////////////////////////////////////////////////////////////////////////"

        mask_results[mask] = (TF_classification_score, TF_rank_score,
                              FT_classification_score, FT_rank_score,
                              TF_classification_score_flipped,
                              TF_rank_score_flipped,
                              FT_classification_score_flipped,
                              FT_rank_score_flipped)
    return mask_results
def semantic_time_weighting_experiment_50_50_split(
        fmri_or_text_time_weights, timestep_weighting, learn_weights,
        avg_variant, date_of_wordvecs, subtract_temporal_average_bool):
    # if we're learning the weights, then timestep_weighting is irrelevant
    if learn_weights == True:
        assert (timestep_weighting == None)
    assert ((fmri_or_text_time_weights == "fmri")
            or (fmri_or_text_time_weights == "text"))
    if avg_variant == 'avg':
        fmris = load_avg(1)  # 1 because there's no averaging (see load_fmri)
    elif avg_variant == 'pca':
        fmris = load_avg_pca(1)
    elif avg_variant == 'srm' or avg_variant == 'srmica':
        # here, srm means we don't do the testing over T iterations
        # we just use the average of all of them without error bars
        if avg_variant == 'srm':
            srms = load_srm(1)
        elif avg_variant == 'srmica':
            srms = load_srmica(1)
        # calculate the average for each mask, and replace with that
        fmris = {}
        for mask in srms.keys():
            #if mask != 'erez_dmna_network':
            #	continue
            srm = srms[mask]
            Ws = srm[0]  # the map
            S = srm[1]  # averaged SRM-projected training data
            num_training_time_steps = S.shape[1]
            tst_data_list = srm[2]  # test portion of the data, pre-selected
            #print "tst_data_list length = " + str(len(tst_data_list))
            num_subjs = len(tst_data_list)
            avg_srm_tst_data = None
            for i in xrange(num_subjs):
                #print "Subject #"+ str(i)
                tst_data = tst_data_list[i]
                #print "Test data shape = " + str(tst_data.shape)
                transformed = np.dot(Ws[i, :, :].T, tst_data)
                if i == 0:
                    avg_srm_tst_data = transformed / (0. + num_subjs)
                else:
                    avg_srm_tst_data += transformed / (0. + num_subjs)
            fmri_mask = np.c_[S, avg_srm_tst_data]
            fmris[mask] = fmri_mask

    else:
        print 'avg_variant is wrong'
        return
    if date_of_wordvecs == 'may15':
        semantic_vecs = load_may15_annotation_vecs()
    elif date_of_wordvecs == 'sep12weighted':
        semantic_vecs = load_sep12_weighted_word_vecs_annotations()
    elif date_of_wordvecs == 'sep12unweighted':
        semantic_vecs = load_sep12_unweighted_word_vecs_annotations()
    else:
        # default
        semantic_vecs = load_sep12_weighted_word_vecs_annotations()
    print "Semantic vector shape = " + str(semantic_vecs.shape)
    num_sem_vecs = semantic_vecs.shape[1]  # assume time is # cols
    # don't subtract out average of all semantic vectors for all time points
    # only calculate average for training, and subtract that average out of the test
    # CHANGE MADE: sep 19 12:40 AM
    '''
	# subtract out average if we're doing this case
	if subtract_temporal_average_bool == True:
		semantic_vecs, avg_semantic_vec = subtract_column_mean(semantic_vecs)
	'''

    #print "num semantic vectors = " + str(num_sem_vecs)
    word_tr = semantic_vecs[:, xrange(num_sem_vecs / 2 + 1)]
    #print "word_tr.shape = " + str(word_tr.shape)
    word_tst = semantic_vecs[:, xrange(num_sem_vecs / 2 + 1, num_sem_vecs)]
    #print "word_tst.shape = " + str(word_tst.shape)
    #print "word_tr.shape = " + str(word_tr.shape)
    #print "word_tst.shape = " + str(word_tst.shape)

    # don't subtract out average of all semantic vectors for all time points
    # only calculate average for training, and subtract that average out of the test
    # CHANGE MADE: sep 19 12:40 AM
    if subtract_temporal_average_bool == True:
        word_tr, avg_tr_word_vec = subtract_column_mean(word_tr)
        word_tst = word_tst - avg_tr_word_vec[:, np.newaxis]

    mask_results = {}
    for mask in fmris.keys():
        fmri = fmris[mask]

        num_total_time_steps = fmri.shape[1]
        #print "fmri.shape = " + str(fmri.shape)
        fmri_tr = fmri[:, xrange(num_sem_vecs / 2 + 1)]
        fmri_tst = fmri[:, xrange(num_sem_vecs / 2 + 1, num_sem_vecs)]
        #print "fmri_tr.shape = " + str(fmri_tr.shape)
        #print "fmri_tst.shape = " + str(fmri_tst.shape)
        print "Training linear maps..."
        # all procrustes
        if learn_weights == True:
            if fmri_or_text_time_weights == "fmri":
                W_tf, W_ft, C_weights = learn_matrix_factorization_for_time_text_weights(
                    xrange(num_sem_vecs / 2 + 1), [], fmri_tr,
                    mask + avg_variant, word_tr, date_of_wordvecs)
            elif fmri_or_text_time_weights == "text":
                W_tf, W_ft, C_weights = learn_matrix_factorization_for_time_text_weights(
                    xrange(num_sem_vecs / 2 + 1), [], word_tr,
                    date_of_wordvecs, fmri_tr, mask + avg_variant)

        else:
            C_weights = timestep_weighting
            num_times = len(xrange(num_sem_vecs / 2 + 1))
            C = create_k_diag_conv_mat_from_weights(
                np.zeros((num_times, num_times)), C_weights, num_times)

            if fmri_or_text_time_weights == "fmri":
                X = fmri_tr[:, xrange(num_sem_vecs / 2 + 1)]
                Y = word_tr[:, xrange(num_sem_vecs / 2 + 1)]
                YC = np.dot(Y, C)
                # text -> fmri
                W_tf = procrustes_fit([X, YC])
                W_tf = W_tf[0]
                W_ft = W_tf.T
            elif fmri_or_text_time_weights == "text":
                Y = fmri_tr[:, xrange(num_sem_vecs / 2 + 1)]
                X = word_tr[:, xrange(num_sem_vecs / 2 + 1)]
                YC = np.dot(Y, C)
                # text -> fmri
                W_tf = procrustes_fit([YC, X])
                W_tf = W_tf[0]
                W_ft = W_tf.T

        print "Done training linear maps..."

        TF_class_scores = []
        TF_rank_scores = []
        FT_class_scores = []
        FT_rank_scores = []

        num_time_steps = fmri_tst.shape[
            1]  # assume time steps are # of columns
        chunk_array = []
        index = 0
        num_chunks = 25  ### CHANGED JAN 11
        while index < num_time_steps:
            chunk_array.append(index)
            index += num_time_steps / num_chunks
            if index > num_time_steps:
                index = num_time_steps
        k = 1  #-> 4% chance rate ## CHANGED FEB 4

        # weighting matrices for semantic vectors
        C = create_k_diag_conv_mat_from_weights(
            np.zeros((num_time_steps, num_time_steps)), C_weights,
            num_time_steps)
        time_weighted_word_tst = np.dot(word_tst, C)

        # comparisons in fMRI space (i.e. text -> fMRI)
        # truth is fmri_tst
        # prediction is W_tf*time_weighted_word_tst
        TF_truth = fmri_tst

        TF_prediction = np.dot(W_tf, time_weighted_word_tst)
        #TF_prediction = TF_prediction[0, :, :]
        #print "TF pred shape: " + str(TF_prediction.shape)
        #print "TF truth shape: " + str(TF_truth.shape)
        assert (TF_truth.shape == TF_prediction.shape)

        TF_classification_score = scene_classification(TF_truth, TF_prediction,
                                                       chunk_array, k)
        TF_rank_score, TF_ranks = scene_ranking(TF_truth, TF_prediction,
                                                chunk_array)
        TF_rank_score = TF_rank_score / (num_chunks + 0.)

        # comparisons in Semantic space (i.e. fMRI -> text) (time-weighted_averages)
        # truth is time_weighted_word_tst
        # prediction is W_ft*fmri_tst
        FT_truth = time_weighted_word_tst
        FT_prediction = np.dot(W_ft, fmri_tst)
        #FT_prediction = FT_prediction[0, :, :]
        #print "FT pred shape: " + str(FT_prediction.shape)
        #print "FT truth shape: " + str(FT_truth.shape)
        assert (FT_truth.shape == FT_prediction.shape)

        FT_classification_score = scene_classification(FT_truth, FT_prediction,
                                                       chunk_array, k)
        FT_rank_score, FT_ranks = scene_ranking(FT_truth, FT_prediction,
                                                chunk_array)
        FT_rank_score = FT_rank_score / (num_chunks + 0.)

        print "Mask: " + mask
        print "---------------------------------------------------------------------------"
        print "fMRI -> Text (Procrustes) scene classification (" + str(
            float(k) /
            num_chunks) + "% chance) avg = " + str(FT_classification_score)
        print "fMRI -> Text (Procrustes) scene ranking (50% chance) avg = " + str(
            FT_rank_score)
        print "---------------------------------------------------------------------------"
        print "Text -> fMRI (Procrustes) scene classification (" + str(
            float(k) /
            num_chunks) + "% chance) avg = " + str(TF_classification_score)
        print "Text -> fMRI (Procrustes) scene ranking (50% chance) avg = " + str(
            TF_rank_score)
        print "---------------------------------------------------------------------------"
        print "///////////////////////////////////////////////////////////////////////////"

        mask_results[mask] = (C_weights, TF_classification_score,
                              TF_rank_score, FT_classification_score,
                              FT_rank_score)
    return mask_results