def map_str_to_digit(data,
features,
no_map_features,
only_map_features=" ",
label=" "):
no_map_features_index = get_known_features_index(features, no_map_features)
features_map_info = dict()
fixed_str_features = np.array(load_result("str_features.csv"))[0]
fixed_str_features_index = get_known_features_index(
features, fixed_str_features)
only_map_features_index = range(len(features))
if not only_map_features == " ":
only_map_features_index = get_known_features_index(
features, only_map_features)
for fea_pos in range(1, len(features)):
if not fea_pos in no_map_features_index and fea_pos in only_map_features_index:
map_info = OrderedDict()
#feature_map_info = OrderedDict()
fea_val_cla = feature_value_class(data, fea_pos, label,
fixed_str_features_index)
# if this feature is a string value, just convert it to value
if fea_val_cla["str_feature"]:
data, map_info = map_str_feature_to_value(
data, fea_pos, fea_val_cla)
features_map_info[features[fea_pos]] = map_info
#features_map_info[].append([feature_map_info])
digited_data = convert_to_numerical(data, features)
return digited_data, features_map_info
def new_UserInfo_18(data, features):
solved_features = ["UserInfo_18"]
fea_indexs = get_known_features_index(features, solved_features)
feature_name = "UserInfo_18_bined"
new_add_feature = np.array([feature_name])
new_features = np.concatenate((features, new_add_feature))
feature_data = np.zeros((len(data), 1))
for user in range(data.shape[0]):
user_age = data[user, fea_indexs]
if user_age < "22":
feature_data[user, 0] = 0
elif user_age < "30":
feature_data[user, 0] = 1
elif user_age < "40":
feature_data[user, 0] = 2
elif user_age < "50":
feature_data[user, 0] = 3
else:
feature_data[user, 0] = 4
new_data = np.concatenate((data, feature_data), axis=1)
new_data, new_features, deleted = delete_features(new_data, new_features, \
delete_feas_list = solved_features)
print(deleted)
return new_data, new_features
def new_UserInfo_differ(data,
features,
key_features,
feature_name,
deleted_all=True):
fea_indexs = get_known_features_index(features, key_features)
new_add_feature = np.array([feature_name])
feature_data = np.zeros((len(data), 1))
new_features = np.concatenate((features, new_add_feature))
for user in range(data.shape[0]):
if not len(set(list(data[user, fea_indexs]))) == 1:
feature_data[user, 0] = 1
new_data = np.concatenate((data, feature_data), axis=1)
delete_feas = key_features[1:]
if deleted_all:
delete_feas = key_features
new_data, new_features, deleted = delete_features(new_data, new_features, \
delete_feas_list = delete_feas)
print(deleted)
return new_data, new_features
def thirdParty_one_period_info(data, features, label, period_number): based_feature_name = "ThirdParty_Info_Period" solved_features = list() for i in range(1, 17): solved_feature = based_feature_name + period_number[0] + "_" + str(i) solved_features.append(solved_feature) indexs = get_known_features_index(features, solved_features) one_period_info = OrderedDict() one_period_info["missing count"] = 0 one_period_info["missing contain positive count"] = 0 one_period_info["missing indexs"] = list() one_period_info["missing indexs label"] = list() for user in range(data.shape[0]): sat_data = list(data[user, indexs]) if -1 in sat_data: one_period_info["missing count"] += 1 if label[user] == 1: one_period_info["missing contain positive count"] += 1 one_period_info["missing indexs"].append(user) one_period_info["missing indexs label"].append(label[user]) return one_period_info
def new_UserInfo_18(data, features): solved_features = ["UserInfo_18"] fea_indexs = get_known_features_index(features, solved_features) feature_name = "UserInfo_18_bined" new_add_feature = np.array([feature_name]) new_features = np.concatenate((features, new_add_feature)) feature_data = np.zeros((len(data), 1)) for user in range(data.shape[0]): user_age = data[user, fea_indexs] if user_age < "22": feature_data[user, 0] = 0 elif user_age < "30": feature_data[user, 0] = 1 elif user_age < "40": feature_data[user, 0] = 2 elif user_age < "50": feature_data[user, 0] = 3 else: feature_data[user, 0] = 4 new_data = np.concatenate((data, feature_data), axis = 1) new_data, new_features, deleted = delete_features(new_data, new_features, \ delete_feas_list = solved_features) print(deleted) return new_data, new_features
def correlation_between_properties(data, features):
fixed_str_features = np.array(load_result("str_features.csv"))[0]
indexs = get_known_features_index(features, fixed_str_features)
title = list()
title.append("features1")
title.append("features2")
title.append("calculate_method")
title.append("cor")
title.append("pval")
save_result(title, "pearsonr_spearmanr_results.csv")
save_result(title, "pearsonr_spearmanr_Strong_correlation.csv")
for fea_pos in range(len(features)):
for fea_pos_add in range(fea_pos + 1, len(features)):
info_result = list()
info_result.append(features[fea_pos])
info_result.append(features[fea_pos_add])
a1 = data[:, fea_pos]
a2 = data[:, fea_pos_add]
# they are all not str style features
if fea_pos not in indexs and fea_pos_add not in indexs:
info_result.append("pearsonr")
cor, pval = stats.pearsonr(a1, a2)
else: # one of them or all of them are str style features
info_result.append("spearmanr")
cor, pval = stats.spearmanr(a1, a2)
cor = round(cor, 3)
info_result.append(cor)
info_result.append(pval)
if abs(cor) >= 0.2:
save_result(info_result, "pearsonr_spearmanr_results.csv", style = "a+")
if abs(cor) >= 0.86:
save_result(info_result, "pearsonr_spearmanr_Strong_correlation.csv", \
style = "a+")
def new_EI_5_6_7_8(data, features): key_features = ["Education_Info5", "Education_Info6", "Education_Info7", "Education_Info8"] fea_indexs = get_known_features_index(features, key_features) feature_name = "combine_EI_5_6_7_8" new_add_feature = np.array([feature_name]) new_features = np.concatenate((features, new_add_feature)) ##### map rules #### map_to_zero = [["1", "AQ", "-1", "T"], ["1", "AQ", "-1", "80"], ["1", "U", "-1", "-1"], ["1", "AQ", "-1", "-1"], ["1", "B", "-1", "-1"], ["1", "A", "-1", "-1"], ["1", "AM", "-1", "80"], ["1", "A", "-1", "F"], ["1", "B", "-1", "AE"], ["1", "U", "-1", "AE"], ["1", "AQ", "-1", "V"], ["1", "AM", "-1", "V"]] map_to_one = [["1", "A", "-1", "T"], ["1", "AQ", "-1", "F"], ["1", "AM", "-1", "-1"], ["1", "AM", "-1", "-1"], ["1", "AM", "-1", "F"], ["1", "AM", "-1", "T"]] map_to_two = [["0", "E", "E", "E"]] feature_data = np.ones((len(data), 1)) for user in range(data.shape[0]): EI_5_6_7_8 = list(data[user, fea_indexs]) if EI_5_6_7_8 in map_to_zero: feature_data[user, 0] = 0 elif EI_5_6_7_8 in map_to_one: feature_data[user, 0] = 1 elif EI_5_6_7_8 in map_to_two: feature_data[user, 0] = 2 else: feature_data[user, 0] = 3 new_data = np.concatenate((data, feature_data), axis = 1) new_data, new_features, deleted = delete_features(new_data, new_features, \ delete_feas_list = key_features[1:]) print(deleted) return new_data, new_features
def new_EI_8(data, features):
solved_features = ["Education_Info8"]
fea_indexs = get_known_features_index(features, solved_features)
feature_name = "Education_Info8_info_(cat)"
new_add_feature = np.array([feature_name])
new_features = np.concatenate((features, new_add_feature))
feature_data = np.zeros((len(data), 1))
for user in range(data.shape[0]):
if data[user, fea_indexs[0]] == "V" or data[user, fea_indexs[0]] == "AE":
feature_data[user, 0] = 0
elif data[user, fea_indexs[0]] == "80":
feature_data[user, 0] = 1
elif data[user, fea_indexs[0]] == "F":
feature_data[user, 0] = 2
elif data[user, fea_indexs[0]] == "T":
feature_data[user, 0] = 3
elif data[user, fea_indexs[0]] == "-1":
feature_data[user, 0] = 4
elif data[user, fea_indexs[0]] == "E":
feature_data[user, 0] = 5
else:
print("error in Education_Info8")
new_data = np.concatenate((data, feature_data), axis = 1)
print("Education_Info8 solved")
return new_data, new_features
def new_WI_19(data, features):
solved_features = ["WeblogInfo_19"]
fea_indexs = get_known_features_index(features, solved_features)
feature_name = "WeblogInfo_19_info_(cat)"
new_add_feature = np.array([feature_name])
new_features = np.concatenate((features, new_add_feature))
feature_data = np.zeros((len(data), 1))
for user in range(data.shape[0]):
if data[user, fea_indexs[0]] == "H":
feature_data[user, 0] = 0
elif data[user, fea_indexs[0]] == "G":
feature_data[user, 0] = 1
elif data[user, fea_indexs[0]] == "J":
feature_data[user, 0] = 2
elif data[user, fea_indexs[0]] == "E":
feature_data[user, 0] = 3
elif data[user, fea_indexs[0]] == "F":
feature_data[user, 0] = 4
elif data[user, fea_indexs[0]] == "D":
feature_data[user, 0] = 5
else:
feature_data[user, 0] = 6
new_data = np.concatenate((data, feature_data), axis = 1)
new_data, new_features, deleted = delete_features(new_data, new_features, \
delete_feas_list = solved_features)
print("WeblogInfo_19 solved")
print(deleted)
return new_data, new_features
def new_WI_19(data, features):
solved_features = ["WeblogInfo_19"]
fea_indexs = get_known_features_index(features, solved_features)
feature_name = "WeblogInfo_19_info_(cat)"
new_add_feature = np.array([feature_name])
new_features = np.concatenate((features, new_add_feature))
feature_data = np.zeros((len(data), 1))
for user in range(data.shape[0]):
if data[user, fea_indexs[0]] == "H":
feature_data[user, 0] = 0
elif data[user, fea_indexs[0]] == "G":
feature_data[user, 0] = 1
elif data[user, fea_indexs[0]] == "J":
feature_data[user, 0] = 2
elif data[user, fea_indexs[0]] == "E":
feature_data[user, 0] = 3
elif data[user, fea_indexs[0]] == "F":
feature_data[user, 0] = 4
elif data[user, fea_indexs[0]] == "D":
feature_data[user, 0] = 5
else:
feature_data[user, 0] = 6
new_data = np.concatenate((data, feature_data), axis=1)
new_data, new_features, deleted = delete_features(new_data, new_features, \
delete_feas_list = solved_features)
print("WeblogInfo_19 solved")
print(deleted)
return new_data, new_features
def new_EI_1_2_3_4(data, features):
key_features = ["Education_Info1", "Education_Info2", "Education_Info3", "Education_Info4"]
fea_indexs = get_known_features_index(features, key_features)
feature_name = "combine_EI_1_2_3_4"
new_add_feature = np.array([feature_name])
new_features = np.concatenate((features, new_add_feature))
##### map rules ####
map_to_zero = [["1", "AQ", "毕业", "T"], ["1", "A", "毕业", "V"], ["1", "AN", "结业", "T"],
["1", "AM", "结业", "T"], ["1", "B", "毕业", "AE"], ["1", "A", "结业", "T"],
["1", "A", "毕业", "AR"]]
map_to_one = [["1", "U", "毕业", "AE"], ["1", "AM", "毕业", "AR"], ["1", "AM", "毕业", "V"],
["1", "AQ", "毕业", "F"], ["1", "A", "毕业", "F"], ["1", "AN", "毕业", "T"],
["1", "AQ", "毕业", "V"]]
map_to_two = [["1", "AM", "毕业", "T"], ["1", "A", "毕业", "T"], ["1", "AM", "毕业", "F"]]
map_to_three = [["0", "E", "E", "E"]]
feature_data = np.ones((len(data), 1))
for user in range(data.shape[0]):
EI_1_2_3_4 = list(data[user, fea_indexs])
if EI_1_2_3_4 in map_to_zero:
feature_data[user, 0] = 0
elif EI_1_2_3_4 in map_to_one:
feature_data[user, 0] = 1
elif EI_1_2_3_4 in map_to_two:
feature_data[user, 0] = 2
elif EI_1_2_3_4 in map_to_three:
feature_data[user, 0] = 3
else:
print("error!!!!")
new_data = np.concatenate((data, feature_data), axis = 1)
new_data, new_features, deleted = delete_features(new_data, new_features, \
delete_feas_list = key_features[1:])
print(deleted)
return new_data, new_features
def new_EI_8(data, features):
solved_features = ["Education_Info8"]
fea_indexs = get_known_features_index(features, solved_features)
feature_name = "Education_Info8_info_(cat)"
new_add_feature = np.array([feature_name])
new_features = np.concatenate((features, new_add_feature))
feature_data = np.zeros((len(data), 1))
for user in range(data.shape[0]):
if data[user, fea_indexs[0]] == "V" or data[user,
fea_indexs[0]] == "AE":
feature_data[user, 0] = 0
elif data[user, fea_indexs[0]] == "80":
feature_data[user, 0] = 1
elif data[user, fea_indexs[0]] == "F":
feature_data[user, 0] = 2
elif data[user, fea_indexs[0]] == "T":
feature_data[user, 0] = 3
elif data[user, fea_indexs[0]] == "-1":
feature_data[user, 0] = 4
elif data[user, fea_indexs[0]] == "E":
feature_data[user, 0] = 5
else:
print("error in Education_Info8")
new_data = np.concatenate((data, feature_data), axis=1)
print("Education_Info8 solved")
return new_data, new_features
def new_WI_20_by_present(data, features):
solved_features = ["WeblogInfo_20"]
fea_indexs = get_known_features_index(features, solved_features)
feature_name = "WeblogInfo_20_present_info_(cat)"
new_add_feature = np.array([feature_name])
new_features = np.concatenate((features, new_add_feature))
none_finded_combine = OrderedDict()
feature_data = np.zeros((len(data), 1))
map_to_zero = [
'F3', 'C39', 'F5', 'F2', 'F9', 'F12', 'I8', 'C38', 'F10', 'F11', 'O',
'C13', 'I6', 'C16', 'I7', 'I10'
]
map_to_one = ['F14', 'F15', 'F13', 'C17', 'C12', 'C18', 'C15']
map_to_two = ['F16', 'C11', 'C1', 'C20', 'C19']
map_to_three = ['I3', 'U', 'C21', 'I4']
map_to_four = ['I5']
map_to_five = ['-1']
for user in range(data.shape[0]):
fea_value = data[user, fea_indexs[0]]
if fea_value in map_to_zero:
feature_data[user, 0] = 0
elif fea_value in map_to_one:
feature_data[user, 0] = 1
elif fea_value in map_to_two:
feature_data[user, 0] = 2
elif fea_value in map_to_three:
feature_data[user, 0] = 3
elif fea_value in map_to_four:
feature_data[user, 0] = 4
elif fea_value in map_to_five:
feature_data[user, 0] = 5
else:
# print("error")
# print(fea_value)
if fea_value not in none_finded_combine.keys():
none_finded_combine[fea_value] = list()
none_finded_combine[fea_value].append(user)
for fea_value, users in none_finded_combine.items():
if fea_value[0] == "-1":
feature_data[users, 0] = 5
if len(users) < 20:
feature_data[users, 0] = 0
elif len(users) < 100:
feature_data[users, 0] = 1
elif len(users) < 1000:
feature_data[users, 0] = 2
elif len(users) < 5000:
feature_data[users, 0] = 3
else:
feature_data[users, 0] = 4
new_data = np.concatenate((data, feature_data), axis=1)
new_data, new_features, deleted = delete_features(new_data, new_features, \
delete_feas_list = solved_features)
print("WeblogInfo_20 solved present")
print(deleted)
return new_data, new_features
def new_WI_20_by_present(data, features):
solved_features = ["WeblogInfo_20"]
fea_indexs = get_known_features_index(features, solved_features)
feature_name = "WeblogInfo_20_present_info_(cat)"
new_add_feature = np.array([feature_name])
new_features = np.concatenate((features, new_add_feature))
none_finded_combine = OrderedDict()
feature_data = np.zeros((len(data), 1))
map_to_zero = ['F3', 'C39', 'F5', 'F2', 'F9', 'F12', 'I8', 'C38', 'F10', 'F11', 'O', 'C13', 'I6', 'C16', 'I7', 'I10']
map_to_one = ['F14', 'F15', 'F13', 'C17', 'C12', 'C18', 'C15']
map_to_two = ['F16', 'C11', 'C1', 'C20', 'C19']
map_to_three = ['I3', 'U', 'C21', 'I4']
map_to_four = ['I5']
map_to_five = ['-1']
for user in range(data.shape[0]):
fea_value = data[user, fea_indexs[0]]
if fea_value in map_to_zero:
feature_data[user, 0] = 0
elif fea_value in map_to_one:
feature_data[user, 0] = 1
elif fea_value in map_to_two:
feature_data[user, 0] = 2
elif fea_value in map_to_three:
feature_data[user, 0] = 3
elif fea_value in map_to_four:
feature_data[user, 0] = 4
elif fea_value in map_to_five:
feature_data[user, 0] = 5
else:
# print("error")
# print(fea_value)
if fea_value not in none_finded_combine.keys():
none_finded_combine[fea_value] = list()
none_finded_combine[fea_value].append(user)
for fea_value, users in none_finded_combine.items():
if fea_value[0] == "-1":
feature_data[users, 0] = 5
if len(users) < 20:
feature_data[users, 0] = 0
elif len(users) < 100:
feature_data[users, 0] = 1
elif len(users) < 1000:
feature_data[users, 0] = 2
elif len(users) < 5000:
feature_data[users, 0] = 3
else:
feature_data[users, 0] = 4
new_data = np.concatenate((data, feature_data), axis = 1)
new_data, new_features, deleted = delete_features(new_data, new_features, \
delete_feas_list = solved_features)
print("WeblogInfo_20 solved present")
print(deleted)
return new_data, new_features
def extract_data_by_features(data, features, needed_features):
needed_features_index = get_known_features_index(features, needed_features)
new_data = np.ones((data.shape[0], len(needed_features)), dtype=np.int64)
print(len(needed_features_index))
print(new_data.shape)
for i in range(len(needed_features)):
new_data[:, i] = data[:, needed_features_index[i]]
new_features = needed_features
return new_data, new_features
def extract_data_by_features(data, features, needed_features): needed_features_index = get_known_features_index(features, needed_features) new_data = np.ones((data.shape[0], len(needed_features)), dtype=np.int64) print(len(needed_features_index)) print(new_data.shape) for i in range(len(needed_features)): new_data[:, i] = data[:, needed_features_index[i]] new_features = needed_features return new_data, new_features
def map_str_to_digit_with_experience(data, features, digited_special_str_features, \
contain_special_features):
map_experience = load_result(FEATURES_MAP_INFO_FILE_NAME, \
dir_name = "resultData/features_map")
print(map_experience)
fixed_str_features = np.array(load_result("str_features.csv"))[0]
fixed_str_features_index = get_known_features_index(
features, fixed_str_features)
digited_special_str_features_index = get_known_features_index(features, \
digited_special_str_features)
contain_special_features_index = get_known_features_index(features, \
contain_special_features)
remember = list()
for fea_pos in range(1, len(features)):
# str style features + str .. but not digited + the str we want to digit
if fea_pos in fixed_str_features_index and \
fea_pos not in digited_special_str_features_index and \
fea_pos in contain_special_features_index:
# the ListingInfo may be reverse !!!
if features[fea_pos] == "ListingInfo" and int(
data[0, fea_pos].split("/")[0]) < 1000:
data = reverse_date(data, fea_pos)
if features[fea_pos] in map_experience.keys():
for i in range(len(data)):
if data[i, fea_pos] == "-1":
continue
try:
data[i, fea_pos] = map_experience[features[fea_pos]][
data[i, fea_pos]]
except:
if i < 50:
print(features[fea_pos])
print(map_experience[features[fea_pos]])
print(map_experience[features[fea_pos]][data[
i, fea_pos]])
remember.append(i) # this is a error value
#print(remember)
data = np.delete(data, remember, 0)
digited_data = convert_to_numerical(data, features)
return digited_data
def compare_features_info2(data, features, key_features): fea_indexs = get_known_features_index(features, key_features) compare_result = OrderedDict() for user in range(data.shape[0]): # user_id = data[user, 0] combine_data = reduce(lambda x,y: str(x) + '_' + str(y), list(data[user, fea_indexs])) if combine_data not in compare_result.keys(): compare_result[combine_data] = 0 compare_result[combine_data] += 1 return compare_result
def one_features_info2(data, features, key_feature): fea_index = get_known_features_index(features, key_feature) user_value_info = dict() for user in range(data.shape[0]): value = list(data[user, fea_index])[0] if not value in user_value_info.keys(): user_value_info[value] = 0 user_value_info[value] += 1 return user_value_info
def one_features_info(data, features, label, key_feature): fea_index = get_known_features_index(features, key_feature) user_value_info = dict() for user in range(data.shape[0]): value = list(data[user, fea_index])[0] if not value in user_value_info.keys(): user_value_info[value] = [0, 0] user_value_info[value][0] += 1 if label[user] == 1: user_value_info[value][1] += 1 return user_value_info
def fill_all_missing(data, features, label = None):
fixed_str_features = np.array(load_result("str_features.csv"))[0]
indexs = get_known_features_index(features, fixed_str_features)
#!start from range(1,...) is because the first line of the feature is the id, useless
for fea_pos in range(1, len(features)):
fea_val_cla = feature_value_class(data, fea_pos, label, indexs)
if not fea_val_cla[-1]._present_num == 0:
if fea_pos == 5:
print(fea_val_cla)
data = fill_the_missing(data, fea_pos, fea_val_cla, label)
#write_to_deleted_features_area(np.array(deleted_feas))
return data, features
def according_coefficient_variation_delete(data, features):
waiting_to_delete = np.array(load_result("complex_value_features.csv"))
waiting_to_delete = waiting_to_delete.reshape((waiting_to_delete.size,))
#print(waiting_to_delete)
indexs = get_known_features_index(features, waiting_to_delete)
coefficient_variation_info = OrderedDict()
for fea_pos in indexs:
try:
coefficient_variation_fea = stats.variation(data[:, fea_pos])
coefficient_variation_info[features[fea_pos]] = coefficient_variation_fea
except:
pass
return coefficient_variation_info
def map_str_to_digit_with_experience(data, features, digited_special_str_features, \
contain_special_features):
map_experience = load_result(FEATURES_MAP_INFO_FILE_NAME, \
dir_name = "resultData/features_map")
print(map_experience)
fixed_str_features = np.array(load_result("str_features.csv"))[0]
fixed_str_features_index = get_known_features_index(features, fixed_str_features)
digited_special_str_features_index = get_known_features_index(features, \
digited_special_str_features)
contain_special_features_index = get_known_features_index(features, \
contain_special_features)
remember = list()
for fea_pos in range(1, len(features)):
# str style features + str .. but not digited + the str we want to digit
if fea_pos in fixed_str_features_index and \
fea_pos not in digited_special_str_features_index and \
fea_pos in contain_special_features_index:
# the ListingInfo may be reverse !!!
if features[fea_pos] == "ListingInfo" and int(data[0, fea_pos].split("/")[0]) < 1000:
data = reverse_date(data, fea_pos)
if features[fea_pos] in map_experience.keys():
for i in range(len(data)):
if data[i, fea_pos] == "-1":
continue
try:
data[i, fea_pos] = map_experience[features[fea_pos]][data[i, fea_pos]]
except:
if i < 50:
print(features[fea_pos])
print(map_experience[features[fea_pos]])
print(map_experience[features[fea_pos]][data[i, fea_pos]])
remember.append(i) # this is a error value
#print(remember)
data = np.delete(data, remember, 0)
digited_data = convert_to_numerical(data, features)
return digited_data
def according_coefficient_variation_delete(data, features):
waiting_to_delete = np.array(load_result("complex_value_features.csv"))
waiting_to_delete = waiting_to_delete.reshape((waiting_to_delete.size, ))
#print(waiting_to_delete)
indexs = get_known_features_index(features, waiting_to_delete)
coefficient_variation_info = OrderedDict()
for fea_pos in indexs:
try:
coefficient_variation_fea = stats.variation(data[:, fea_pos])
coefficient_variation_info[
features[fea_pos]] = coefficient_variation_fea
except:
pass
return coefficient_variation_info
def missed_instances_info(data, features, label, key_feature = None): missed_infos = OrderedDict() users_miss_info = OrderedDict() if key_feature == None: indexs = range(len(features)) key_data = data key_feature = features else: indexs = get_known_features_index(features, key_feature) key_data = data[:, indexs] print(key_data.shape) print(key_data.shape[0]) print(key_data.shape[1]) missed_infos["missed_instances_sum"] = 0 missed_infos["positive_sum"] = 0 missed_infos["negitive_sum"] = 0 for i in range(key_data.shape[0]): user_miss_count = 0 missed_features_index = list() missed_features = list() flag = 0 user_idx = data[i, 0] users_miss_info[user_idx] = OrderedDict() for j in range(key_data.shape[1]): if key_data[i, j] == -1: flag = 1 user_miss_count += 1 missed_features_index.append(indexs[j]) missed_features.append(key_feature[j]) # exist miss in this line if flag == 1: missed_infos["missed_instances_sum"] += 1 if label[i] == 1: missed_infos["positive_sum"] += 1 else: missed_infos["negitive_sum"] += 1 # if this uer missed, statistic information if user_miss_count: users_miss_info[user_idx]["missed_count"] = user_miss_count users_miss_info[user_idx]["miss_features_indexs"] = missed_features_index users_miss_info[user_idx]["missed_features"] = missed_features users_miss_info[user_idx]["label"] = label[i] return missed_infos, users_miss_info
def map_str_to_digit(data, features, no_map_features, only_map_features = " ", label = " "):
no_map_features_index = get_known_features_index(features, no_map_features)
features_map_info = dict()
fixed_str_features = np.array(load_result("str_features.csv"))[0]
fixed_str_features_index = get_known_features_index(features, fixed_str_features)
only_map_features_index = range(len(features))
if not only_map_features == " ":
only_map_features_index = get_known_features_index(features, only_map_features)
for fea_pos in range(1, len(features)):
if not fea_pos in no_map_features_index and fea_pos in only_map_features_index:
map_info = OrderedDict()
#feature_map_info = OrderedDict()
fea_val_cla = feature_value_class(data, fea_pos, label, fixed_str_features_index)
# if this feature is a string value, just convert it to value
if fea_val_cla["str_feature"]:
data, map_info = map_str_feature_to_value(data, fea_pos, fea_val_cla)
features_map_info[features[fea_pos]] = map_info
#features_map_info[].append([feature_map_info])
digited_data = convert_to_numerical(data, features)
return digited_data, features_map_info
def sta_thirdParty_info(data, features, type_number, label = None):
based_feature_name = "ThirdParty_Info_Period"
solved_features = list()
for i in range(1, 7):
solved_feature = based_feature_name + str(i) + "_" + type_number[0]
solved_features.append(solved_feature)
indexs = get_known_features_index(features, solved_features)
# print(solved_features)
# print(indexs)
users_sta_name = ["average", "Standard deviation", "coefficient of variation", "max", "min"]
sta_name = "ms_type" + type_number[0]
#print("sta_name: ", sta_name)
users_stability = OrderedDict()
for user in range(data.shape[0]):
users_stability[user] = OrderedDict()
if not label == None:
users_stability[user]["label"] = label[user]
calculate_data = data[user, indexs]
users_stability[user]["value"] = list(calculate_data)
users_stability[user][sta_name] = dict()
fea_average = round(float(np.mean(calculate_data)), 3)
fea_std = round(float(np.std(calculate_data)), 3)
if fea_average == 0:
fea_cv = 0
else:
fea_cv = round(float(fea_std / fea_average), 3)
max_v = np.amax(calculate_data)
min_v = np.amin(calculate_data)
users_stability[user][sta_name]["average"] = fea_average
users_stability[user][sta_name]["Standard deviation"] = fea_std
users_stability[user][sta_name]["cv"] = fea_cv
users_stability[user][sta_name]["max"] = max_v
users_stability[user][sta_name]["min"] = min_v
return users_stability
def new_UserInfo_19_20(data, features):
solved_features = ["UserInfo_19", "UserInfo_20"]
fea_indexs = get_known_features_index(features, solved_features)
feature_name = "UserInfo_19_20_wrong_province_city"
new_add_feature = np.array([feature_name])
feature_data = np.zeros((len(data), 1))
new_features = np.concatenate((features, new_add_feature))
for user in range(data.shape[0]):
if str(-1) in list(data[user, fea_indexs]):
feature_data[user, 0] = 1
new_data = np.concatenate((data, feature_data), axis=1)
new_data, new_features, deleted = delete_features(new_data, new_features, \
delete_feas_list = solved_features)
print(deleted)
return new_data, new_features
def new_UserInfo_19_20(data, features): solved_features = ["UserInfo_19", "UserInfo_20"] fea_indexs = get_known_features_index(features, solved_features) feature_name = "UserInfo_19_20_wrong_province_city" new_add_feature = np.array([feature_name]) feature_data = np.zeros((len(data), 1)) new_features = np.concatenate((features, new_add_feature)) for user in range(data.shape[0]): if str(-1) in list(data[user, fea_indexs]): feature_data[user, 0] = 1 new_data = np.concatenate((data, feature_data), axis = 1) new_data, new_features, deleted = delete_features(new_data, new_features, \ delete_feas_list = solved_features) print(deleted) return new_data, new_features
def new_EI_1_2_3_4(data, features):
key_features = [
"Education_Info1", "Education_Info2", "Education_Info3",
"Education_Info4"
]
fea_indexs = get_known_features_index(features, key_features)
feature_name = "combine_EI_1_2_3_4"
new_add_feature = np.array([feature_name])
new_features = np.concatenate((features, new_add_feature))
##### map rules ####
map_to_zero = [["1", "AQ", "毕业", "T"], ["1", "A", "毕业", "V"],
["1", "AN", "结业", "T"], ["1", "AM", "结业", "T"],
["1", "B", "毕业", "AE"], ["1", "A", "结业", "T"],
["1", "A", "毕业", "AR"]]
map_to_one = [["1", "U", "毕业", "AE"], ["1", "AM", "毕业", "AR"],
["1", "AM", "毕业", "V"], ["1", "AQ", "毕业", "F"],
["1", "A", "毕业", "F"], ["1", "AN", "毕业", "T"],
["1", "AQ", "毕业", "V"]]
map_to_two = [["1", "AM", "毕业", "T"], ["1", "A", "毕业", "T"],
["1", "AM", "毕业", "F"]]
map_to_three = [["0", "E", "E", "E"]]
feature_data = np.ones((len(data), 1))
for user in range(data.shape[0]):
EI_1_2_3_4 = list(data[user, fea_indexs])
if EI_1_2_3_4 in map_to_zero:
feature_data[user, 0] = 0
elif EI_1_2_3_4 in map_to_one:
feature_data[user, 0] = 1
elif EI_1_2_3_4 in map_to_two:
feature_data[user, 0] = 2
elif EI_1_2_3_4 in map_to_three:
feature_data[user, 0] = 3
else:
print("error!!!!")
new_data = np.concatenate((data, feature_data), axis=1)
new_data, new_features, deleted = delete_features(new_data, new_features, \
delete_feas_list = key_features[1:])
print(deleted)
return new_data, new_features
def compare_features_info(data, features, label, key_features): fea_indexs = get_known_features_index(features, key_features) compare_result = OrderedDict() compare_result["num_differ"] = 0 compare_result["different_combine_info"] = dict() compare_result["num_differ_positive"] = 0 compare_result["num_same"] = 0 compare_result["same_combine_info"] = dict() compare_result["num_same_positive"] = 0 compare_result["num_same_miss"] = 0 for user in range(data.shape[0]): # user_id = data[user, 0] combine_data = reduce(lambda x,y: str(x) + '_' + str(y), list(data[user, fea_indexs])) province = list(data[user, fea_indexs])[0] pro_info = list(data[user, fea_indexs])[1] if not len(set(list(data[user, fea_indexs]))) == 1: compare_result["num_differ"] += 1 if not combine_data in compare_result["different_combine_info"].keys(): compare_result["different_combine_info"][combine_data] = [0, 0] compare_result["different_combine_info"][combine_data][0] += 1 if label[user] == 1: compare_result["num_differ_positive"] += 1 compare_result["different_combine_info"][combine_data][1] += 1 else: compare_result["num_same"] += 1 if not combine_data in compare_result["same_combine_info"].keys(): compare_result["same_combine_info"][combine_data] = [0, 0] compare_result["same_combine_info"][combine_data][0] += 1 if label[user] == 1: compare_result["num_same_positive"] += 1 compare_result["same_combine_info"][combine_data][1] += 1 if str(-1) == list(data[user, fea_indexs])[0]: compare_result["num_same_miss"] += 1 return compare_result
def new_UserInfo_11_12_13(data, features):
solved_features = ["UserInfo_11", "UserInfo_12", "UserInfo_13"]
fea_indexs = get_known_features_index(features, solved_features)
feature_name = "UserInfo_11_12_13_info"
new_add_feature = np.array([feature_name])
feature_data = np.zeros((len(data), 1))
new_features = np.concatenate((features, new_add_feature))
for user in range(data.shape[0]):
combine_data = list(map(int, list(data[user, fea_indexs])))
add = reduce(lambda x, y: x * 2 + y, combine_data)
if add > 0:
feature_data[user, 0] = add
else:
feature_data[user, 0] = 0
new_data = np.concatenate((data, feature_data), axis = 1)
print("extract from UserInfo 11 12 13")
return new_data, new_features
def new_UserInfo_11_12_13(data, features):
solved_features = ["UserInfo_11", "UserInfo_12", "UserInfo_13"]
fea_indexs = get_known_features_index(features, solved_features)
feature_name = "UserInfo_11_12_13_info"
new_add_feature = np.array([feature_name])
feature_data = np.zeros((len(data), 1))
new_features = np.concatenate((features, new_add_feature))
for user in range(data.shape[0]):
combine_data = list(map(int, list(data[user, fea_indexs])))
add = reduce(lambda x, y: x * 2 + y, combine_data)
if add > 0:
feature_data[user, 0] = add
else:
feature_data[user, 0] = 0
new_data = np.concatenate((data, feature_data), axis=1)
print("extract from UserInfo 11 12 13")
return new_data, new_features
def correlation_between_properties(data, features):
fixed_str_features = np.array(load_result("str_features.csv"))[0]
indexs = get_known_features_index(features, fixed_str_features)
title = list()
title.append("features1")
title.append("features2")
title.append("calculate_method")
title.append("cor")
title.append("pval")
save_result(title, "pearsonr_spearmanr_results.csv")
save_result(title, "pearsonr_spearmanr_Strong_correlation.csv")
for fea_pos in range(len(features)):
for fea_pos_add in range(fea_pos + 1, len(features)):
info_result = list()
info_result.append(features[fea_pos])
info_result.append(features[fea_pos_add])
a1 = data[:, fea_pos]
a2 = data[:, fea_pos_add]
# they are all not str style features
if fea_pos not in indexs and fea_pos_add not in indexs:
info_result.append("pearsonr")
cor, pval = stats.pearsonr(a1, a2)
else: # one of them or all of them are str style features
info_result.append("spearmanr")
cor, pval = stats.spearmanr(a1, a2)
cor = round(cor, 3)
info_result.append(cor)
info_result.append(pval)
if abs(cor) >= 0.2:
save_result(info_result,
"pearsonr_spearmanr_results.csv",
style="a+")
if abs(cor) >= 0.86:
save_result(info_result, "pearsonr_spearmanr_Strong_correlation.csv", \
style = "a+")
def new_UserInfo_differ(data, features, key_features, feature_name, deleted_all = True): fea_indexs = get_known_features_index(features, key_features) new_add_feature = np.array([feature_name]) feature_data = np.zeros((len(data), 1)) new_features = np.concatenate((features, new_add_feature)) for user in range(data.shape[0]): if not len(set(list(data[user, fea_indexs]))) == 1: feature_data[user, 0] = 1 new_data = np.concatenate((data, feature_data), axis = 1) delete_feas = key_features[1:] if deleted_all: delete_feas = key_features new_data, new_features, deleted = delete_features(new_data, new_features, \ delete_feas_list = delete_feas) print(deleted) return new_data, new_features
def save_features_info(data,
features,
label,
file_name,
dir_name="resultData"):
file_path = os.path.join(os.getcwd(), dir_name, file_name)
first_line = np.array(['features_name', 'str_feature', \
'num_values', \
'average|most_presentS',
'postitive(average|most_present)', \
'negitive(average|most_present)', \
'num_positive', \
'num_negitive', 'feature_value_info'])
fixed_str_features = np.array(load_result("str_features.csv"))[0]
indexs = get_known_features_index(features, fixed_str_features)
with open(file_path, "w", newline='') as csv_file:
spamwriter = csv.writer(csv_file)
spamwriter.writerow(first_line)
from solve_data import feature_value_class, FeatureInData
from collections import OrderedDict
for fea_pos in range(1, len(features)):
feature_info = list()
feature_info.append(features[fea_pos])
fea_val_cla = feature_value_class(data,
fea_pos,
label,
fixed_str_features_index=indexs)
feature_info.append(fea_val_cla["str_feature"])
feature_info.append(fea_val_cla["num_of_value"])
if fea_val_cla["str_feature"]:
feature_info.append(fea_val_cla["most_presentS"])
try:
l = label[0, 0]
feature_info.append(fea_val_cla["most_presentS_positive"])
feature_info.append(fea_val_cla["most_presentS_negitive"])
except:
feature_info.append("None")
feature_info.append("None")
else:
feature_info.append(fea_val_cla["average"])
try:
l = label[0, 0]
feature_info.append(fea_val_cla["average_positive"])
feature_info.append(fea_val_cla["average_negitive"])
except:
feature_info.append("None")
feature_info.append("None")
try:
l = label[0, 0]
feature_info.append(fea_val_cla["num_positive"])
feature_info.append(fea_val_cla["num_negitive"])
except:
feature_info.append("None")
feature_info.append("None")
for k, v in fea_val_cla.items():
if isinstance(v, FeatureInData):
value_info = OrderedDict()
value_info["value"] = k
value_info["present_num"] = v._present_num
try:
l = label[0, 0]
value_info[
"respond_positive_num"] = v._respond_positive_num
value_info[
"respond_negitive_num"] = v._respond_negitive_num
except:
pass
feature_info.append(value_info)
with open(file_path, "a+", newline='') as csv_file:
spamwriter = csv.writer(csv_file)
spamwriter.writerow(feature_info)
def new_UserInfo_22_23_combine2(data, features):
key_features = ["UserInfo_22", "UserInfo_23"]
print("combine2")
fea_indexs = get_known_features_index(features, key_features)
feature_name = "UserInfo_combine2_by_present_22_23"
new_add_feature = np.array([feature_name])
new_features = np.concatenate((features, new_add_feature))
##### map rules ####
map_to_zero = [['未婚', 'AE'], ['已婚', 'AH'], ['再婚', 'M'], ['未婚', 'X'],
['未婚', 'AJ'], ['-1', 'AC'], ['离婚', 'K'], ['已婚', 'AF'],
['未婚', 'AP'], ['再婚', 'G'], ['未婚', 'R'], ['已婚', 'AL'],
['离婚', '专科毕业'], ['离婚', 'G'], ['离婚', 'AC'], ['未婚', 'AD'],
['-1', 'M'], ['离婚', 'AB'], ['已婚', 'AJ'], ['-1', 'R'],
['已婚', 'Y'], ['离婚', 'H'], ['未婚', 'Q'], ['离婚', 'P'],
['已婚', 'Z'], ['初婚', 'G'], ['-1', 'K'], ['再婚', 'O'],
['-1', 'AI'], ['离婚', '-1'], ['已婚', '-1'], ['再婚', 'H'],
['未婚', 'AH'], ['离婚', '大学本科（简称“大学'], ['离婚', 'M'],
['-1', 'P'], ['已婚', 'AE'], ['-1', '专科毕业'], ['-1', 'AH'],
['已婚', 'P'], ['已婚', 'AI'], ['离婚', 'AH'], ['离婚', 'O'],
['已婚', 'AC'], ['-1', 'H'], ['未婚', 'AC'], ['-1', 'AK']]
map_to_one = [['已婚', 'K'], ['未婚', 'K'], ['未婚', 'W'], ['-1', '大学本科（简称“大学'],
['已婚', '专科毕业']]
map_to_two = [['未婚', 'Y'], ['已婚', 'AB'], ['未婚', '专科毕业'],
['已婚', '大学本科（简称“大学'], ['已婚', 'M'], ['-1', 'Y'], ['未婚', 'P'],
['-1', 'O'], ['已婚', 'AK'], ['未婚', 'AI'], ['未婚', 'M'],
['未婚', '-1'], ['-1', 'G'], ['未婚', 'H'], ['已婚', 'H'],
['-1', 'AB'], ['未婚', 'AK'], ['已婚', 'O']]
map_to_three = [['未婚', '大学本科（简称“大学'], ['已婚', 'G'], ['未婚', 'O'],
['未婚', 'AB'], ['未婚', 'G']]
map_to_four = [['D', 'D']]
map_to_five = [['-1', '-1']]
none_finded_combine = OrderedDict()
feature_data = np.ones((len(data), 1))
for user in range(data.shape[0]):
EI_22_23 = list(data[user, fea_indexs])
if EI_22_23 in map_to_zero:
feature_data[user, 0] = 0
elif EI_22_23 in map_to_one:
feature_data[user, 0] = 1
elif EI_22_23 in map_to_two:
feature_data[user, 0] = 2
elif EI_22_23 in map_to_three:
feature_data[user, 0] = 3
elif EI_22_23 in map_to_four:
feature_data[user, 0] = 4
elif EI_22_23 in map_to_five:
feature_data[user, 0] = 5
else:
EI_22_23_str = reduce(lambda x, y: x + "_" + y, EI_22_23)
if EI_22_23_str not in none_finded_combine.keys():
none_finded_combine[EI_22_23_str] = list()
none_finded_combine[EI_22_23_str].append(user)
for EI_combine, users in none_finded_combine.items():
EI_combine = EI_combine.split("_")
if EI_combine[0] == "-1" and EI_combine[1] == "-1":
feature_data[users, 0] = 5
if len(users) < 10:
feature_data[users, 0] = 0
elif len(users) < 20:
feature_data[users, 0] = 1
elif len(users) < 100:
feature_data[users, 0] = 2
elif len(users) < 1000:
feature_data[users, 0] = 3
else:
feature_data[users, 0] = 4
new_data = np.concatenate((data, feature_data), axis=1)
new_data, new_features, deleted = delete_features(new_data, new_features, \
delete_feas_list = key_features)
print(deleted)
return new_data, new_features
def save_features_info(data, features, label, file_name, dir_name = "resultData"):
file_path = os.path.join(os.getcwd(), dir_name, file_name)
first_line = np.array(['features_name', 'str_feature', \
'num_values', \
'average|most_presentS',
'postitive(average|most_present)', \
'negitive(average|most_present)', \
'num_positive', \
'num_negitive', 'feature_value_info'])
fixed_str_features = np.array(load_result("str_features.csv"))[0]
indexs = get_known_features_index(features, fixed_str_features)
with open(file_path, "w", newline='') as csv_file:
spamwriter = csv.writer(csv_file)
spamwriter.writerow(first_line)
from solve_data import feature_value_class, FeatureInData
from collections import OrderedDict
for fea_pos in range(1, len(features)):
feature_info = list()
feature_info.append(features[fea_pos])
fea_val_cla = feature_value_class(data, fea_pos, label, fixed_str_features_index = indexs)
feature_info.append(fea_val_cla["str_feature"])
feature_info.append(fea_val_cla["num_of_value"])
if fea_val_cla["str_feature"]:
feature_info.append(fea_val_cla["most_presentS"])
try:
l = label[0, 0]
feature_info.append(fea_val_cla["most_presentS_positive"])
feature_info.append(fea_val_cla["most_presentS_negitive"])
except:
feature_info.append("None")
feature_info.append("None")
else:
feature_info.append(fea_val_cla["average"])
try:
l = label[0, 0]
feature_info.append(fea_val_cla["average_positive"])
feature_info.append(fea_val_cla["average_negitive"])
except:
feature_info.append("None")
feature_info.append("None")
try:
l = label[0, 0]
feature_info.append(fea_val_cla["num_positive"])
feature_info.append(fea_val_cla["num_negitive"])
except:
feature_info.append("None")
feature_info.append("None")
for k, v in fea_val_cla.items():
if isinstance(v, FeatureInData):
value_info = OrderedDict()
value_info["value"] = k
value_info["present_num"] = v._present_num
try:
l = label[0, 0]
value_info["respond_positive_num"] = v._respond_positive_num
value_info["respond_negitive_num"] = v._respond_negitive_num
except:
pass
feature_info.append(value_info)
with open(file_path, "a+", newline='') as csv_file:
spamwriter = csv.writer(csv_file)
spamwriter.writerow(feature_info)
def new_UserInfo_22_23_combine2(data, features):
key_features = ["UserInfo_22", "UserInfo_23"]
print("combine2")
fea_indexs = get_known_features_index(features, key_features)
feature_name = "UserInfo_combine2_by_present_22_23"
new_add_feature = np.array([feature_name])
new_features = np.concatenate((features, new_add_feature))
##### map rules ####
map_to_zero = [['未婚', 'AE'], ['已婚', 'AH'], ['再婚', 'M'], ['未婚', 'X'], ['未婚', 'AJ'],
['-1', 'AC'], ['离婚', 'K'], ['已婚', 'AF'], ['未婚', 'AP'], ['再婚', 'G'],
['未婚', 'R'], ['已婚', 'AL'], ['离婚', '专科毕业'], ['离婚', 'G'], ['离婚', 'AC'],
['未婚', 'AD'], ['-1', 'M'], ['离婚', 'AB'], ['已婚', 'AJ'], ['-1', 'R'],
['已婚', 'Y'], ['离婚', 'H'], ['未婚', 'Q'], ['离婚', 'P'], ['已婚', 'Z'],
['初婚', 'G'], ['-1', 'K'], ['再婚', 'O'], ['-1', 'AI'], ['离婚', '-1'],
['已婚', '-1'], ['再婚', 'H'], ['未婚', 'AH'], ['离婚', '大学本科（简称“大学'],
['离婚', 'M'], ['-1', 'P'], ['已婚', 'AE'], ['-1', '专科毕业'], ['-1', 'AH'],
['已婚', 'P'], ['已婚', 'AI'], ['离婚', 'AH'], ['离婚', 'O'], ['已婚', 'AC'],
['-1', 'H'], ['未婚', 'AC'], ['-1', 'AK']]
map_to_one = [['已婚', 'K'], ['未婚', 'K'], ['未婚', 'W'], ['-1', '大学本科（简称“大学'], ['已婚', '专科毕业']]
map_to_two = [['未婚', 'Y'], ['已婚', 'AB'], ['未婚', '专科毕业'], ['已婚', '大学本科（简称“大学'],
['已婚', 'M'], ['-1', 'Y'], ['未婚', 'P'], ['-1', 'O'], ['已婚', 'AK'],
['未婚', 'AI'], ['未婚', 'M'], ['未婚', '-1'], ['-1', 'G'], ['未婚', 'H'],
['已婚', 'H'], ['-1', 'AB'], ['未婚', 'AK'], ['已婚', 'O']]
map_to_three = [['未婚', '大学本科（简称“大学'], ['已婚', 'G'], ['未婚', 'O'], ['未婚', 'AB'], ['未婚', 'G']]
map_to_four = [['D', 'D']]
map_to_five = [['-1', '-1']]
none_finded_combine = OrderedDict()
feature_data = np.ones((len(data), 1))
for user in range(data.shape[0]):
EI_22_23 = list(data[user, fea_indexs])
if EI_22_23 in map_to_zero:
feature_data[user, 0] = 0
elif EI_22_23 in map_to_one:
feature_data[user, 0] = 1
elif EI_22_23 in map_to_two:
feature_data[user, 0] = 2
elif EI_22_23 in map_to_three:
feature_data[user, 0] = 3
elif EI_22_23 in map_to_four:
feature_data[user, 0] = 4
elif EI_22_23 in map_to_five:
feature_data[user, 0] = 5
else:
EI_22_23_str = reduce(lambda x, y: x + "_" + y, EI_22_23)
if EI_22_23_str not in none_finded_combine.keys():
none_finded_combine[EI_22_23_str] = list()
none_finded_combine[EI_22_23_str].append(user)
for EI_combine, users in none_finded_combine.items():
EI_combine = EI_combine.split("_")
if EI_combine[0] == "-1" and EI_combine[1] == "-1":
feature_data[users, 0] = 5
if len(users) < 10:
feature_data[users, 0] = 0
elif len(users) < 20:
feature_data[users, 0] = 1
elif len(users) < 100:
feature_data[users, 0] = 2
elif len(users) < 1000:
feature_data[users, 0] = 3
else:
feature_data[users, 0] = 4
new_data = np.concatenate((data, feature_data), axis = 1)
new_data, new_features, deleted = delete_features(new_data, new_features, \
delete_feas_list = key_features)
print(deleted)
return new_data, new_features