Esempio n. 1
0
def predict_example():

    # UNRESOLVED ISSUES WITH PREDICTION

    # --------------------------------------------------------------------------
    # Preparing the feature matrix
    # --------------------------------------------------------------------------
    df_test = pd.read_hdf('../class_photoz/data/DR7DR14Q_flux_cat.hdf5',
                          'data')
    # df_train = pd.read_hdf('../class_photoz/data/DR7DR14Q_flux_cat.hdf5','data')
    df_train = pd.read_hdf('../class_photoz/data/brightqsos_2.hdf5', 'data')
    passband_names = [\
            'SDSS_u','SDSS_g','SDSS_r','SDSS_i','SDSS_z', \
            # 'TMASS_j','TMASS_h','TMASS_ks', \
            'WISE_w1','WISE_w2', \
            # 'WISE_w3' \
            ]
    # Try a fraction of the whole datafile first
    df_train = df_train.sample(frac=1.0)

    # df_test.query('Z > 1.1',inplace=True)
    # df_train.query('z > 1.1',inplace=True)

    for name in passband_names:
        df_train.rename(columns={'obsFlux_' + name: name}, inplace=True)
        df_train.rename(columns={'obsFluxErr_' + name: 'sigma_' + name},
                        inplace=True)

    df_test.replace(np.inf, np.nan, inplace=True)
    df_train.replace(np.inf, np.nan, inplace=True)

    df_test, features = qs.prepare_flux_ratio_catalog(df_test, passband_names)
    df_train, features = qs.prepare_flux_ratio_catalog(df_train,
                                                       passband_names)

    print df_test.shape, df_train.shape
    # --------------------------------------------------------------------------
    # Random Forest Regression Grid Search
    # --------------------------------------------------------------------------

    features = ['SDSS_i', 'WISE_w1', 'ug', 'gr', 'ri', 'iz', 'zw1', 'w1w2']
    # features = ['SDSS_i','WISE_w1','TMASS_j','ug','gr','ri','iz','zj','jh', 'hks', 'ksw1', 'w1w2']
    label = 'z'

    params = {
        'kernel': 'rbf',
        'C': 1.0,
        'gamma': 0.001,
        'epsilon': 0.2,
        'cache_size': 1200
    }

    df_test = svr.svm_reg_predict(df_train, df_test, features, label, params,
                                  'svm_photoz')
    print df_test['svm_photoz'].describe()
    ml_an.evaluate_regression(df_test['Z'], df_test['svm_photoz'])
    pz_an.plot_redshifts(df_test['Z'], df_test['svm_photoz'])
    pz_an.plot_error_hist(df_test['Z'], df_test['svm_photoz'])
    plt.show()
Esempio n. 2
0
def simqso_predict_dr7dr12():
    # --------------------------------------------------------------------------
    # Preparing the feature matrix
    # --------------------------------------------------------------------------
    df_test = pd.read_hdf('../class_photoz/data/DR7DR12Q_clean_flux_cat.hdf5',
                          'data')
    df_train = pd.read_hdf('../class_photoz/data/brightqsos_sim_2k_new.hdf5',
                           'data')
    passband_names = [\
            'SDSS_u','SDSS_g','SDSS_r','SDSS_i','SDSS_z', \
            # 'TMASS_j','TMASS_h','TMASS_ks', \
            'WISE_w1','WISE_w2', \
            # 'WISE_w3' \
            ]

    df_test = df_test.query('0.3 < Z_VI < 5.5')
    # df_train.query('obsMag_SDSS_i <= 18.5',inplace=True)
    # df_test.query('SDSS_mag_i <= 18.5',inplace=True)
    # df_train.query('z > 1.1',inplace=True)

    for name in passband_names:
        df_train.rename(columns={'obsFlux_' + name: name}, inplace=True)
        df_train.rename(columns={'obsFluxErr_' + name: 'sigma_' + name},
                        inplace=True)

    df_test.replace(np.inf, np.nan, inplace=True)
    df_train.replace(np.inf, np.nan, inplace=True)

    df_test, features = qs.prepare_flux_ratio_catalog(df_test, passband_names)
    df_train, features = qs.prepare_flux_ratio_catalog(df_train,
                                                       passband_names)

    print df_test.shape, df_train.shape
    # --------------------------------------------------------------------------
    # Random Forest Regression Grid Search
    # --------------------------------------------------------------------------

    # features = ['SDSS_u','SDSS_i','SDSS_r','SDSS_z','SDSS_g','WISE_w1','WISE_w2']
    #  features = ['SDSS_i','WISE_w1','ug','gr','ri','iz','zw1','w1w2']
    features = ['ug', 'gr', 'ri', 'iz', 'zw1', 'w1w2']
    # features = ['SDSS_i','WISE_w1','TMASS_j','ug','gr','ri','iz','zj','jh', 'hks', 'ksw1', 'w1w2']
    label = 'z'
    rand_state = 1

    params = {
        'n_estimators': 300,
        'max_depth': 30,
        'min_samples_split': 4,
        'n_jobs': 2,
        'random_state': rand_state
    }

    df_test = rf.rf_reg_predict(df_train, df_test, features, label, params,
                                'rf_photoz')

    ml_an.evaluate_regression(df_test['Z_VI'], df_test['rf_photoz'])
    pz_an.plot_redshifts(df_test['Z_VI'], df_test['rf_photoz'])
    pz_an.plot_error_hist(df_test['Z_VI'], df_test['rf_photoz'])
    plt.show()
def dr7dr12_predict_simqso():
    # --------------------------------------------------------------------------
    # Preparing the feature matrix
    # --------------------------------------------------------------------------
    df_train = pd.read_hdf('../class_photoz/data/DR7DR12Q_clean_flux_cat.hdf5',
                           'data')
    df_test = pd.read_hdf('../class_photoz/data/brightqsos_sim_2k_new.hdf5',
                          'data')
    passband_names = [\
            'SDSS_u','SDSS_g','SDSS_r','SDSS_i','SDSS_z', \
            # 'TMASS_j','TMASS_h','TMASS_ks', \
            'WISE_w1','WISE_w2', \
            # 'WISE_w3' \
            ]

    df_train = df_train.query('0.3 < Z_VI < 5.5')
    df_test.query('obsMag_SDSS_i <= 18.5', inplace=True)
    df_train.query('SDSS_mag_i <= 18.5', inplace=True)
    # df_train.query('z > 1.1',inplace=True)

    for name in passband_names:
        df_test.rename(columns={'obsFlux_' + name: name}, inplace=True)
        df_test.rename(columns={'obsFluxErr_' + name: 'sigma_' + name},
                       inplace=True)

    df_test.replace(np.inf, np.nan, inplace=True)
    df_train.replace(np.inf, np.nan, inplace=True)

    df_test, features = qs.prepare_flux_ratio_catalog(df_test, passband_names)
    df_train, features = qs.prepare_flux_ratio_catalog(df_train,
                                                       passband_names)

    print df_test.shape, df_train.shape
    # --------------------------------------------------------------------------
    # Random Forest Regression Grid Search
    # --------------------------------------------------------------------------

    # features = ['SDSS_u','SDSS_i','SDSS_r','SDSS_z','SDSS_g','WISE_w1','WISE_w2']
    features = ['SDSS_i', 'WISE_w1', 'ug', 'gr', 'ri', 'iz', 'zw1', 'w1w2']
    # features = ['SDSS_i','WISE_w1','TMASS_j','ug','gr','ri','iz','zj','jh', 'hks', 'ksw1', 'w1w2']
    label = 'Z_VI'
    rand_state = 1

    params = {
        'kernel': 'rbf',
        'C': 1.0,
        'gamma': 0.001,
        'epsilon': 0.2,
        'cache_size': 1200
    }

    df_test = svr.svm_reg_predict(df_train, df_test, features, label, params,
                                  'svm_photoz')

    ml_an.evaluate_regression(df_test['z'], df_test['svm_photoz'])
    pz_an.plot_redshifts(df_test['z'], df_test['svm_photoz'])
    pz_an.plot_error_hist(df_test['z'], df_test['svm_photoz'])
    plt.show()
Esempio n. 4
0
def photoz_analysis(df_pred, z_label_pred, z_label_true):

    df = df_pred.copy(deep=True)
    df = df.query('bin_class_true == "QSO"')

    z_true = df[z_label_true].values
    z_pred = df[z_label_pred].values

    print("The r2 score for the Photometric Redshift Estimation is:\t"
          ), met.r2_score(z_true, z_pred)

    pz_an.plot_redshifts(z_true, z_pred)
    pz_an.plot_error_hist(z_true, z_pred)

    plt.show()