def run_diagnostics_and_transformations(data):
#This looks for any mostly empty rows
#print diagnostic_tools.count_empty(data)
#Looking at this data it is clear that the last two rows ought to be removed
#This code removes the 2 empty rows and returns the data
data = cleaning_tools.remove_empties(data)
#This data confirms that the data was removed
#print diagnostic_tools.count_empty(data)
#Converts the list data into dictionary data for easier analysis
#ID row is set 1 because the respondent ID is 1 in this case
data_dict = dictionary_conversion.create_dictionary(data,id_row=1)
return data_dict
def cluster(final_data_dict, cluster_range, list_or_dict):
final_data_list= clustering_module.convert_to_list(final_data_dict)
respondent_IDs = np.array(map(int, final_data_dict.keys()))
feature_names = final_data_dict.values()[0].keys()
final_data_list_imputed = clustering_module.preprocess(final_data_list)
Scaler = MinMaxScaler()
final_data_list_scaled = Scaler.fit_transform(final_data_list_imputed)
#Transformed is distance of each respondent from each cluster center
#Predicted is the cluster membership of each respondent
merging_list = clustering_module.convert_to_list(final_data_dict,remove_NaN=0 )
data = list(merging_list)
ignore_set_added = set(['ids'])
for num_clusters in cluster_range:
transformed, predicted, score = clustering_module.clustering(final_data_list_scaled, num_clusters)
cluster_name = "%s_clusters" % num_clusters
ignore_set_added.add(cluster_name)
data, feature_names = clustering_module.add_new_data_to_rows(predicted, data, feature_names, [cluster_name])
data, feature_names = clustering_module.add_new_data_to_rows(respondent_IDs, data, feature_names, ["ids"], "before")
if list_or_dict == "dict":
temp = dictionary_conversion.create_dictionary(data, feature_names)
num_converted = dictionary_conversion.convert_values_to_int(temp)
#Set of features that should be different due to being categorical
ignore_set_changed = set(['busgrn', 'peopgrn', 'sex', 'race', 'topprob1', 'topprob2'])
verdict = compare_respondent_dicts(respondent_IDs, num_converted, final_data_dict, ignore_set_changed, ignore_set_added)
return num_converted, verdict
elif list_or_dict == "list":
return data, feature_names
