コード例 #1
0
def create_wine(data_to_create=WINE_RED):
    red_file = "wine/winequality-red.csv"
    white_file = "wine/winequality-white.csv"
    field_names, red_data = load_csv(red_file, delim=";")
    white_data = load_csv(white_file, delim=";")[1]

    if data_to_create == WINE_TRANSFER:
        red_ids = np.zeros((red_data.shape[0], 1))
        white_ids = np.ones((white_data.shape[0], 1))
        red_data = np.hstack((red_data, red_ids))
        white_data = np.hstack((white_data, white_ids))
        wine_data = np.vstack((red_data, white_data))

        ids = wine_data[:, -1]
        x = wine_data[:, :-2]
        y = wine_data[:, -2]
        used_field_names = field_names[:-1]
        viz = True
        if viz:
            learner = make_learner()
            # learner = None
            viz_features(x, y, ids, used_field_names, alpha=0.01, learner=learner)
        suffix = "transfer"
    else:
        if data_to_create == WINE_RED:
            wine_data = red_data
            suffix = "red"
        elif data_to_create == WINE_WHITE:
            wine_data = white_data
            suffix = "white"
        else:
            assert False

        ids = None
        x = wine_data[:, :-1]
        y = wine_data[:, -1]
        used_field_names = field_names[:-1]
    data = data_class.Data()
    data.x = data.x = array_functions.standardize(x)
    if data_to_create == WINE_TRANSFER:
        pass
        # feat_idx = 1
        # data.x = array_functions.vec_to_2d(x[:,feat_idx])

    data.y = y
    data.set_train()
    data.set_target()
    data.set_true_y()
    data.data_set_ids = ids
    data.is_regression = True
    """
    data = data.rand_sample(.25, data.data_set_ids == 0)
    data = data.rand_sample(.1, data.data_set_ids == 1)
    s = wine_file % ('-small-' + str(data.p))
    """
    s = wine_file % ("-" + suffix)
    helper_functions.save_object(s, data)
コード例 #2
0
def create_energy():
    file = 'energy/ENB2012_data.csv'
    field_names, energy_data = load_csv(file)
    domain_ids = energy_data[:, 4]
    x = energy_data
    y = energy_data[:, -2]
    from methods import method
    #learner = method.NadarayaWatsonMethod()
    learner = None
    viz_features(x, y, domain_ids, field_names, learner=learner)
    pass
コード例 #3
0
def create_energy():
    file = "energy/ENB2012_data.csv"
    field_names, energy_data = load_csv(file)
    domain_ids = energy_data[:, 4]
    x = energy_data
    y = energy_data[:, -2]
    from methods import method

    # learner = method.NadarayaWatsonMethod()
    learner = None
    viz_features(x, y, domain_ids, field_names, learner=learner)
    pass
コード例 #4
0
def create_pair(i, j, y_col):
    file = pair_file(i, j)
    data_i = load_pair(i)
    data_j = load_pair(j)
    data_all = np.vstack((data_i, data_j))
    x = data_all[:, 1 - y_col]
    y = data_all[:, y_col]
    domain_ids = np.zeros(data_i.shape[0] + data_j.shape[0])
    domain_ids[data_i.shape[0] :] = 1
    # viz_features(x,y,domain_ids,learner=make_learner())
    viz_features(x, y, domain_ids, learner=None)
    create_and_save_data(x, y, domain_ids, file)
コード例 #5
0
def create_pair(i, j, y_col):
    file = pair_file(i, j)
    data_i = load_pair(i)
    data_j = load_pair(j)
    data_all = np.vstack((data_i, data_j))
    x = data_all[:, 1 - y_col]
    y = data_all[:, y_col]
    domain_ids = np.zeros(data_i.shape[0] + data_j.shape[0])
    domain_ids[data_i.shape[0]:] = 1
    #viz_features(x,y,domain_ids,learner=make_learner())
    viz_features(x, y, domain_ids, learner=None)
    create_and_save_data(x, y, domain_ids, file)
コード例 #6
0
def create_forest_fires():
    months = {
        'jan': 1,
        'feb': 2,
        'mar': 3,
        'apr': 4,
        'may': 5,
        'jun': 6,
        'jul': 7,
        'aug': 8,
        'sep': 9,
        'oct': 10,
        'nov': 11,
        'dec': 12
    }
    days = {
        'sun': 1,
        'mon': 2,
        'tue': 3,
        'wed': 4,
        'thu': 5,
        'fri': 6,
        'sat': 7
    }
    #month_to_season = lambda x : (months[x]-1)/3
    month_to_season = lambda x: months[x]
    day_to_int = lambda x: days[x]
    file = 'forest_fires/forestfires.csv'
    converters = {2: month_to_season, 3: day_to_int}
    field_names, forest_data = load_csv(file,
                                        dtype='float',
                                        converters=converters)
    x = forest_data
    y = forest_data[:, -1]
    i = field_names == 'month'
    domain_ids = forest_data[:, i]
    months_to_use = np.asarray([6, 7, 8])
    #months_to_use = np.asarray([1,2,3,4,5,6,7,8,9,10,11,12])
    to_use = array_functions.find_set(domain_ids, months_to_use)
    x = x[to_use, :]
    y = y[to_use]
    domain_ids = domain_ids[to_use]
    x = x[:, 4:]
    field_names = field_names[4:]
    I = (y > 0) & (y < 700)
    x = x[I, :]
    y = y[I]
    domain_ids = domain_ids[I]

    from methods import method
    learner = method.NadarayaWatsonMethod()
    viz_features(x, y, domain_ids, field_names, learner=learner)
    pass
コード例 #7
0
def create_mpg():
    file = "mpg/auto-mpg.data.txt"
    # field_names, mpg_data = load_csv(file,has_field_names=False,dtype='string',delim=' ')
    data = pd.read_csv(file, skiprows=0, delim_whitespace=True, dtype="string")
    data = np.asarray(data)[:, 0:-1]
    has_missing_values = (data == "?").any(1)
    data = data[~has_missing_values, :]
    data = data.astype("float")
    domain_ids = data[:, 1]
    x = data
    y = data[:, 0]
    viz_features(x, y, domain_ids)
    pass
コード例 #8
0
def create_mpg():
    file = 'mpg/auto-mpg.data.txt'
    #field_names, mpg_data = load_csv(file,has_field_names=False,dtype='string',delim=' ')
    data = pd.read_csv(file, skiprows=0, delim_whitespace=True, dtype='string')
    data = np.asarray(data)[:, 0:-1]
    has_missing_values = (data == '?').any(1)
    data = data[~has_missing_values, :]
    data = data.astype('float')
    domain_ids = data[:, 1]
    x = data
    y = data[:, 0]
    viz_features(x, y, domain_ids)
    pass
コード例 #9
0
def create_concrete(transfer=False):
    file = 'concrete/Concrete_Data.csv'
    used_field_names, concrete_data = load_csv(file)

    data = data_class.Data()
    t = ''
    if transfer:
        feat_ind = 0
        domain_ind = (used_field_names == 'age').nonzero()[0][0]
        ages = concrete_data[:, domain_ind]
        domain_ids = np.zeros(ages.shape)
        domain_ids[ages < 10] = 1
        domain_ids[(ages >= 10) & (ages <= 28)] = 2
        domain_ids[ages > 75] = 3
        data.x = concrete_data[:, 0:(concrete_data.shape[1] - 2)]
        #0,3,5
        #data.x = preprocessing.scale(data.x)
        if concrete_num_feats == 1:
            data.x = array_functions.vec_to_2d(data.x[:, feat_ind])
            t = '-feat=' + str(feat_ind)
        elif concrete_num_feats >= data.x.shape[1]:
            t = '-' + str(min(data.x.shape[1], concrete_num_feats))
        else:
            assert False
        data.data_set_ids = domain_ids
    else:
        data.x = concrete_data[:, 0:-1]

    data.y = concrete_data[:, -1]
    data.set_train()
    data.set_target()
    data.set_true_y()
    data.is_regression = True

    viz = False
    if viz:
        to_use = domain_ids > 0
        domain_ids = domain_ids[to_use]
        concrete_data = concrete_data[to_use, :]
        np.delete(concrete_data, domain_ind, 1)
        viz_features(concrete_data, concrete_data[:, -1], domain_ids,
                     used_field_names)

        return
    data.x = array_functions.standardize(data.x)
    #viz_features(data.x,data.y,data.data_set_ids)

    s = concrete_file % t
    helper_functions.save_object(s, data)
コード例 #10
0
def create_concrete(transfer=False):
    file = "concrete/Concrete_Data.csv"
    used_field_names, concrete_data = load_csv(file)

    data = data_class.Data()
    t = ""
    if transfer:
        feat_ind = 0
        domain_ind = (used_field_names == "age").nonzero()[0][0]
        ages = concrete_data[:, domain_ind]
        domain_ids = np.zeros(ages.shape)
        domain_ids[ages < 10] = 1
        domain_ids[(ages >= 10) & (ages <= 28)] = 2
        domain_ids[ages > 75] = 3
        data.x = concrete_data[:, 0 : (concrete_data.shape[1] - 2)]
        # 0,3,5
        # data.x = preprocessing.scale(data.x)
        if concrete_num_feats == 1:
            data.x = array_functions.vec_to_2d(data.x[:, feat_ind])
            t = "-feat=" + str(feat_ind)
        elif concrete_num_feats >= data.x.shape[1]:
            t = "-" + str(min(data.x.shape[1], concrete_num_feats))
        else:
            assert False
        data.data_set_ids = domain_ids
    else:
        data.x = concrete_data[:, 0:-1]

    data.y = concrete_data[:, -1]
    data.set_train()
    data.set_target()
    data.set_true_y()
    data.is_regression = True

    viz = False
    if viz:
        to_use = domain_ids > 0
        domain_ids = domain_ids[to_use]
        concrete_data = concrete_data[to_use, :]
        np.delete(concrete_data, domain_ind, 1)
        viz_features(concrete_data, concrete_data[:, -1], domain_ids, used_field_names)

        return
    data.x = array_functions.standardize(data.x)
    # viz_features(data.x,data.y,data.data_set_ids)

    s = concrete_file % t
    helper_functions.save_object(s, data)
コード例 #11
0
def create_forest_fires():
    months = {
        "jan": 1,
        "feb": 2,
        "mar": 3,
        "apr": 4,
        "may": 5,
        "jun": 6,
        "jul": 7,
        "aug": 8,
        "sep": 9,
        "oct": 10,
        "nov": 11,
        "dec": 12,
    }
    days = {"sun": 1, "mon": 2, "tue": 3, "wed": 4, "thu": 5, "fri": 6, "sat": 7}
    # month_to_season = lambda x : (months[x]-1)/3
    month_to_season = lambda x: months[x]
    day_to_int = lambda x: days[x]
    file = "forest_fires/forestfires.csv"
    converters = {2: month_to_season, 3: day_to_int}
    field_names, forest_data = load_csv(file, dtype="float", converters=converters)
    x = forest_data
    y = forest_data[:, -1]
    i = field_names == "month"
    domain_ids = forest_data[:, i]
    months_to_use = np.asarray([6, 7, 8])
    # months_to_use = np.asarray([1,2,3,4,5,6,7,8,9,10,11,12])
    to_use = array_functions.find_set(domain_ids, months_to_use)
    x = x[to_use, :]
    y = y[to_use]
    domain_ids = domain_ids[to_use]
    x = x[:, 4:]
    field_names = field_names[4:]
    I = (y > 0) & (y < 700)
    x = x[I, :]
    y = y[I]
    domain_ids = domain_ids[I]

    from methods import method

    learner = method.NadarayaWatsonMethod()
    viz_features(x, y, domain_ids, field_names, learner=learner)
    pass
コード例 #12
0
def create_bike_sharing():
    file = "bike_sharing/day.csv"
    columns = [0] + range(2, 16)
    all_field_names = pd.read_csv(file, nrows=1, dtype="string")
    all_field_names = np.asarray(all_field_names.keys())
    used_field_names = all_field_names[columns]
    bike_data = np.loadtxt(file, skiprows=1, delimiter=",", usecols=columns)
    domain_ind = used_field_names == "yr"
    domain_ids = np.squeeze(bike_data[:, domain_ind])
    # inds_to_keep = (used_field_names == 'temp') | (used_field_names == 'atemp')
    # bike_data = bike_data[:,inds_to_keep]
    # used_field_names = used_field_names[inds_to_keep]

    viz = True
    to_use = np.asarray([8, 9, 10, 11])
    x = bike_data[:, to_use]
    used_field_names = used_field_names[to_use]
    y = bike_data[:, -1]
    if viz:
        # learner = make_learner()
        learner = None
        viz_features(x, y, domain_ids, used_field_names, learner=learner)
    field_to_use = 1
    x = x[:, field_to_use]

    data = data_class.Data()
    data.is_regression = True
    data.x = array_functions.vec_to_2d(x)
    data.x = array_functions.standardize(data.x)
    data.y = y
    data.y = array_functions.normalize(data.y)
    data.set_defaults()
    data.data_set_ids = domain_ids

    s = bike_file % ("-feat=" + str(field_to_use))
    helper_functions.save_object(s, data)

    pass
コード例 #13
0
def create_bike_sharing():
    file = 'bike_sharing/day.csv'
    columns = [0] + range(2, 16)
    all_field_names = pd.read_csv(file, nrows=1, dtype='string')
    all_field_names = np.asarray(all_field_names.keys())
    used_field_names = all_field_names[columns]
    bike_data = np.loadtxt(file, skiprows=1, delimiter=',', usecols=columns)
    domain_ind = used_field_names == 'yr'
    domain_ids = np.squeeze(bike_data[:, domain_ind])
    #inds_to_keep = (used_field_names == 'temp') | (used_field_names == 'atemp')
    #bike_data = bike_data[:,inds_to_keep]
    #used_field_names = used_field_names[inds_to_keep]

    viz = True
    to_use = np.asarray([8, 9, 10, 11])
    x = bike_data[:, to_use]
    used_field_names = used_field_names[to_use]
    y = bike_data[:, -1]
    if viz:
        #learner = make_learner()
        learner = None
        viz_features(x, y, domain_ids, used_field_names, learner=learner)
    field_to_use = 1
    x = x[:, field_to_use]

    data = data_class.Data()
    data.is_regression = True
    data.x = array_functions.vec_to_2d(x)
    data.x = array_functions.standardize(data.x)
    data.y = y
    data.y = array_functions.normalize(data.y)
    data.set_defaults()
    data.data_set_ids = domain_ids

    s = bike_file % ('-feat=' + str(field_to_use))
    helper_functions.save_object(s, data)

    pass
コード例 #14
0
def create_wine(data_to_create=WINE_RED):
    red_file = 'wine/winequality-red.csv'
    white_file = 'wine/winequality-white.csv'
    field_names, red_data = load_csv(red_file, delim=';')
    white_data = load_csv(white_file, delim=';')[1]

    if data_to_create == WINE_TRANSFER:
        red_ids = np.zeros((red_data.shape[0], 1))
        white_ids = np.ones((white_data.shape[0], 1))
        red_data = np.hstack((red_data, red_ids))
        white_data = np.hstack((white_data, white_ids))
        wine_data = np.vstack((red_data, white_data))

        ids = wine_data[:, -1]
        x = wine_data[:, :-2]
        y = wine_data[:, -2]
        used_field_names = field_names[:-1]
        viz = True
        if viz:
            learner = make_learner()
            #learner = None
            viz_features(x,
                         y,
                         ids,
                         used_field_names,
                         alpha=.01,
                         learner=learner)
        suffix = 'transfer'
    else:
        if data_to_create == WINE_RED:
            wine_data = red_data
            suffix = 'red'
        elif data_to_create == WINE_WHITE:
            wine_data = white_data
            suffix = 'white'
        else:
            assert False

        ids = None
        x = wine_data[:, :-1]
        y = wine_data[:, -1]
        used_field_names = field_names[:-1]
    data = data_class.Data()
    data.x = data.x = array_functions.standardize(x)
    if data_to_create == WINE_TRANSFER:
        pass
        #feat_idx = 1
        #data.x = array_functions.vec_to_2d(x[:,feat_idx])

    data.y = y
    data.set_train()
    data.set_target()
    data.set_true_y()
    data.data_set_ids = ids
    data.is_regression = True
    '''
    data = data.rand_sample(.25, data.data_set_ids == 0)
    data = data.rand_sample(.1, data.data_set_ids == 1)
    s = wine_file % ('-small-' + str(data.p))
    '''
    s = wine_file % ('-' + suffix)
    helper_functions.save_object(s, data)