def selectAttibutesGA(): matrix = csv.readCsvRaw(csv.headered_name_pca_corr) num_attributes = len(matrix[0])-1 if False: num_subset = 5 num_trials = max(100, num_attributes*2) results = findBestOfSize(matrix, num_subset, num_trials) order = orderByResults(results,num_attributes) if True: sort_order = [i for i in range(num_attributes)] for num_subset in range(5, num_attributes, 5): num_trials = max(100, num_attributes*2) csv_matrix_name = csv.makeCsvPath('subset.matrix' +('%03d'%num_subset)) csv_results_name = csv.makePath('subset.results'+('%03d'%num_subset)) csv_best_name = csv.makeCsvPath('subset.best' +('%03d'%num_subset)) csv_summary_name = csv.makeCsvPath('subset.summary'+('%03d'%num_subset)) ordered_matrix = pca.reorderMatrix(matrix, sort_order) csv.writeCsv(csv_matrix_name, ordered_matrix) results = findBestOfSize(ordered_matrix, num_subset, num_trials, csv_summary_name) sort_order = orderByResults(results,num_attributes) #c_x = results[0].columns + [-1] # include outcome #sub_matrix = [[row[i] for i in c_x] for row in ordered_matrix] #csv.writeCsv(csv_best_name,sub_matrix, ) if not is_testing: shutil.copyfile(results[0]['csv'],csv_best_name) shutil.copyfile(results[0]['results'],csv_results_name)
def rankByCorrelationWithOutcomes(in_fn): "Rank each attribute by its correlation with the outcome" print 'rankByCorrelationWithOutcomes:', in_fn in_cells = csv.readCsvRaw(in_fn) csv.validateMatrix(in_cells) name_map = {'nonad.':0.0, 'ad.':1.0} def strToFloat(s): return name_map[s.strip()] #last column is ad categories. normalize other columns in_data = [[float(e) for e in row[:-1]] for row in in_cells[1:]] print 'in_data', len(in_data), len(in_data[0]) raw_outcomes = [strToFloat(row[-1]) for row in in_cells[1:]] print 'outcomes', len(raw_outcomes) #,len(raw_outcomes[0]) values = array(in_data) outcomes = array(raw_outcomes) def correlationWithOutcome(column): return correlation(column, outcomes) # http://www.scipy.org/Numpy_Example_List#head-528347f2f13004fc0081dce432e81b87b3726a33 corr_with_outcomes = apply_along_axis(correlationWithOutcome,0,values) # print 'corr_with_outcomes', corr_with_outcomes corr_index = [(i,c) for i,c in enumerate(corr_with_outcomes)] # print corr_index corr_index.sort(key = lambda x: -abs(x[1])) # print corr_index sort_order = [x[0] for x in corr_index] #print sort_order return (sort_order, corr_index)
def wekaToPredict(weka_filename): """ Convert a Weka formatted .cvs file to a Google Predict .csv file by moving class from last column to first """ parts = os.path.splitext(weka_filename) predict_filename = parts[0] + '.gp' + parts[1] print 'wekaToPredict:', weka_filename,'=>', predict_filename weka = csv.readCsvRaw(weka_filename) predict = [[w[-1]] + w[:-1] for w in weka] csv.writeCsv(predict_filename, predict)
def preprocess(raw_name, headered_name, headered_name_pp): """ Add headers and pre-process the raw Kushmerick data. This needs to be done once. - raw_name is the Kushmerick data that is input - headered_name is the name of CSV file with headers that is created - headered_name_pp is the named a file created by preprocessing header name that is created """ print 'preprocess', raw_name, '=>', headered_name, '=>', headered_name_pp header = csv.makeHeader() data = csv.readCsvRaw(raw_name) hdata = [header] + data assert(len(hdata)==len(data)+1) csv.validateMatrix(hdata) #swapMatrixColumn(data, 3, -1) csv.writeCsv(headered_name, hdata) h2data = csv.readCsvRaw(headered_name) csv.replaceMissingValues(hdata) csv.writeCsv(headered_name_pp, hdata)
def testBySize(incrementing_hidden): "Test MLP results on matrix by number of left side columns" start_num_columns = 30 delta_num_columns = 10 opts = '-M 0.5 -L 0.3 -x 4 -H ' num_hidden = 13 csv_matrix_name = csv.makeCsvPath('subset.matrix035') base_name = 'number.attributes' if incrementing_hidden: base_name = base_name + '.inc' csv_results_name = csv.makePath(base_name + '.results') csv_summary_name = csv.makeCsvPath(base_name + '.summary') csv_best_name = csv.makeCsvPath(base_name + '.best') matrix = csv.readCsvRaw(csv_matrix_name) num_attribs = len(matrix[0])-1 # last column is category print 'testBySize', len(matrix), start_num_columns, delta_num_columns, len(matrix[0]) best_accuracy = 0.0 summary = [] csv_summary = file(csv_summary_name, 'w') for num_columns in range(start_num_columns, len(matrix[0]), delta_num_columns): columns = [i for i in range(num_columns)] if incrementing_hidden: num_hidden = int(float(num_columns)*13.0/30.0) accuracy, temp_csv, temp_results, duration = testMatrixMLP(matrix, columns, opts + str(num_hidden)) r = {'num':num_columns, 'accuracy':accuracy, 'csv':temp_csv, 'results':temp_results, 'duration':duration} summary.append(r) summary.sort(key = lambda r: -r['accuracy']) if True: print num_columns, ':', accuracy, len(results), int(duration), 'seconds' for i in range(min(3,len(results))): rr = results[i] print ' ',i, ':', rr['accuracy'], rr['num'], int(rr['duration']) summary_row = [num_columns, accuracy, duration, temp_csv, temp_results] csv_line = ','.join([str(e) for e in summary_row]) csv_summary.write(csv_line + '\n') csv_summary.flush() if accuracy > best_accuracy: best_accuracy = accuracy shutil.copyfile(temp_csv, csv_best_name) shutil.copyfile(temp_results, csv_results_name) return results
def normalizeData(in_fn, out_fn): """ Normalize ad data to equal std dev in_fn : read input data from this csv file out_fn : write output data to this csv fuile """ print 'normalizeData:', in_fn, '=>', out_fn in_cells = csv.readCsvRaw(in_fn) csv.validateMatrix2(in_cells) # Remove header row on top and category row on right in_data = [[float(e.strip()) for e in row[:-1]] for row in in_cells[1:3280]] print 'data', len(in_data), len(in_data[0]) out_data = normalizeMatrix(in_data) print 'out_data', len(out_data), len(out_data[0]) out_cells = [in_cells[0]] + [out_data[i-1] + [in_cells[i][-1]] for i in range(1,len(in_cells))] csv.writeCsv(out_fn, out_cells)
def testByNumberHidden(csv_matrix_name, output_basename, num_columns, num_cv = 4): """Test MLP results on matrix by number of neurons in hidden layer num_columns is number of leftmost columns of matrix to test num_cv is the number of cross-validation rounds """ start_num_hidden = min(10, num_columns-1) delta_num_hidden = 10 results_name = csv.makePath(output_basename + '.results') model_name = csv.makePath(output_basename + '.model') csv_summary_name = csv.makeCsvPath(output_basename + '.summary') csv_best_name = csv.makeCsvPath(output_basename + '.best') matrix = csv.readCsvRaw(csv_matrix_name) num_attribs = len(matrix[0])-1 # last column is category print 'testByNumberHidden', len(matrix), start_num_hidden, delta_num_hidden, num_columns best_accuracy = 0.0 results = [] csv_summary = file(csv_summary_name, 'w') for num_hidden in range(start_num_hidden, num_columns, delta_num_hidden): columns = [i for i in range(num_columns)] accuracy, temp_csv, temp_results, duration = testMatrixMLP(matrix, columns, makeWekaOptions(0.3, 0.5, num_hidden, num_cv)) r = {'num':num_hidden, 'accuracy':accuracy, 'csv':temp_csv, 'results':temp_results, 'duration':duration} results.append(r) results.sort(key = lambda r: -r['accuracy']) if True: print num_hidden, ':', accuracy, len(results), int(duration), 'seconds' for i in range(min(5,len(results))): rr = results[i] print ' ',i, ':', rr['accuracy'], rr['num'], int(rr['duration']) summary = [num_hidden, accuracy, duration, temp_csv, temp_results] csv_line = ','.join([str(e) for e in summary]) csv_summary.write(csv_line + '\n') csv_summary.flush() if accuracy > best_accuracy: best_accuracy = accuracy shutil.copyfile(temp_csv, csv_best_name) shutil.copyfile(temp_results, results_name) shutil.copyfile(outnameToModelname(temp_results), model_name) return {'summary':csv_summary_name, 'best':csv_best_name, 'results':results_name, 'model':model_name}
def testCostMatrix(num_columns, num_cv = 4): """Test MLP results with a range of false positive costs """ num_hidden = 5 csv_matrix_name = csv.makeCsvPath('subset.matrix035') base_name = 'cost.col' + str(num_columns) + '.x' + str(num_cv) csv_results_name = csv.makePath(base_name + '.results') csv_summary_name = csv.makeCsvPath(base_name + '.summary') csv_best_name = csv.makeCsvPath(base_name + '.best') matrix = csv.readCsvRaw(csv_matrix_name) num_attribs = len(matrix[0])-1 # last column is category print 'testCostMatrix', len(matrix), num_hidden, num_columns best_accuracy = 0.0 results = [] csv_results = file(csv_summary_name, 'w') for false_positive_cost in range(1, 11, 2): columns = [i for i in range(num_columns)] costs_map = {'True':1.0, 'False':float(false_positive_cost)} accuracy, temp_csv, temp_results, duration = testMatrixMLP(matrix, columns, makeWekaOptions(0.3, 0.5, num_hidden, num_cv, costs_map)) r = {'cost':false_positive_cost, 'accuracy':accuracy, 'csv':temp_csv, 'results':temp_results, 'duration':duration} results.append(r) results.sort(key = lambda r: -r['accuracy']) if True: print false_positive_cost, ':', accuracy, len(results), int(duration), 'seconds' for i in range(min(5,len(results))): rr = results[i] print ' ',i, ':', rr['accuracy'], rr['cost'], int(rr['duration']) summary = [num_hidden, accuracy, duration, temp_csv, temp_results] csv_line = ','.join([str(e) for e in summary]) csv_results.write(csv_line + '\n') csv_results.flush() if accuracy > best_accuracy: best_accuracy = accuracy shutil.copyfile(temp_csv, csv_best_name) shutil.copyfile(temp_results, csv_results_name) return results
def pcaAdData(theshold_variance, in_filename, out_filename): """ Run PCA on the Kushmerick ad data Stop when there are sufficient PCA components to explain threshold_variance Project input data onto these PCA components - in_filename : input data read from this CSV file - out_filename : output data written to this CSV file """ h2data = csv.readCsvRaw(in_filename) csv.validateMatrix(h2data) # Boolean data are columns 3 to second last bool_data = [[float(e) for e in v[3:-1]] for v in h2data[1:]] print 'bool_data', len(bool_data), len(bool_data[0]) x = array(bool_data) # Find the output dimension (#basis vectors) required to explain # threshold_variance print 'output_dim, explained_variance, time(sec)' for odim in range(50, len(x[0]), 50): start_time = time.clock() pcanode = mdp.nodes.PCANode(svd=True, output_dim = odim, dtype='float64') pcanode.train(x) p = pcanode.get_projmatrix() d = pcanode.output_dim print '%10d' % d, ',', v = pcanode.explained_variance print '%15.03f' % v, ',', print '%6.1f' % (time.clock() - start_time) if v >= theshold_variance: break #print '-----------------------------1' print 'p', len(p), len(p[0]) #print '-----------------------------2' # Project out data onto PCA components xfd = dot(x, p) pca = [[x for x in row] for row in xfd] print 'pca', len(pca), len(pca[0]) pca_header = ['pca_%03d' % i for i in range(len(pca[0]))] header = h2data[0][:3] + pca_header + [h2data[0][-1]] num_data = [h2data[i+1][:3] + pca[i] + [h2data[i+1][-1]] for i in range(len(h2data)-1)] data = [header] + num_data csv.writeCsv(out_filename, data)
def preprocessSoybeanData(): """ Pre-process the Soybean data set downloaded from http://archive.ics.uci.edu/ml/machine-learning-databases/soybean/ """ """ Read the data files """ training_data = csv.readCsvRaw(os.path.join(dir, training_file)) test_data = csv.readCsvRaw(os.path.join(dir, test_file)) """ Combined data file """ combined_data = test_data + training_data print 'combined data', len(combined_data), len(combined_data[0]) """ Random data file where the percentage of each class and attribute matches the combined data """ random_data = getRandomData(combined_data) """ Find the duplicate instances in each data set The number of duplicates in random_data provides an estimate of the number of duplicates that would occur in the real data sets by pure chance """ training_duplicates = getDuplicates(training_data) print 'training_duplicates =', len(training_duplicates) test_duplicates = getDuplicates(test_data) print 'test_duplicates =', len(test_duplicates) combined_duplicates = getDuplicates(combined_data) print 'combined_duplicates =', len(combined_duplicates) random_duplicates = getDuplicates(random_data) duplicates_warning = '*** Data files should not contain duplicates!' if len(random_duplicates) == 0 else '' print 'random_duplicates =', len(random_duplicates), duplicates_warning """ Remove duplicate instances within each data set We know removing duplicates is valid if len(random_duplicates) is zero """ filtered_training_data = removeDuplicates(training_data, training_duplicates, False) filtered_test_data = removeDuplicates(test_data, test_duplicates, False) filtered_combined_data = removeDuplicates(combined_data, combined_duplicates, False) filtered_random_data = removeDuplicates(random_data, random_duplicates, False) """ Remove the instances in duplicate-free test data that duplicate instances in duplicate-free training data """ all_duplicates = getDuplicates(filtered_training_data + filtered_test_data) filtered_test_data = removeDuplicates(filtered_test_data, all_duplicates, True) """ Sanity check """ assert(len(filtered_test_data) + len(filtered_training_data) + len(combined_duplicates) == len(combined_data)) """ Write out the intermediate .csv files with duplicates marked for debugging """ csv.writeCsv(appendDescription(dir, training_file, 'sorted'), markDuplicates(training_data)) csv.writeCsv(appendDescription(dir, test_file, 'sorted'), markDuplicates(test_data)) csv.writeCsv(appendDescription(dir, combined_file, 'sorted'), markDuplicates(combined_data)) csv.writeCsv(appendDescription(dir, random_file, 'sorted'), markDuplicates(random_data)) """ Read the names of the classes and attributes from downloaded files """ classes = parseClasses(os.path.join(dir, classes_file)) attrs = parseAttrs(os.path.join(dir, attrs_file)) """ Add class and attribute names to original data, for comparison with filter data """ original_named_training_data = applyAttrs(training_data, attrs) original_named_test_data = applyAttrs(test_data, attrs) original_named_combined_data = applyAttrs(combined_data, attrs) """ Add class and attribute names to filtered data """ named_training_data = applyAttrs(filtered_training_data, attrs) named_test_data = applyAttrs(filtered_test_data, attrs) named_combined_data = applyAttrs(filtered_combined_data, attrs) named_random_data = applyAttrs(filtered_random_data, attrs) """ Get the class distribution """ class_distribution_training = getClassDistribution(named_training_data) class_distribution_test = getClassDistribution(named_test_data) class_distribution_combined = getClassDistribution(named_combined_data) named_training_data = removeClassesWithFewerInstances(named_training_data, class_distribution_training,2) """ Create a header row for the .csv file """ header = makeHeaderRow(attrs) """ Write out the .csv files """ csv.writeCsv(appendDescription(dir, training_file, 'distribution'), dictToMatrix(class_distribution_training), ['Class', 'Number']) csv.writeCsv(appendDescription(dir, training_file, 'orig'), named_training_data, header) csv.writeCsv(appendDescription(dir, test_file, 'orig'), named_test_data, header) csv.writeCsv(appendDescription(dir, combined_file, 'orig'), named_combined_data, header) csv.writeCsv(appendDescription(dir, training_file, 'named'), original_named_training_data, header) csv.writeCsv(appendDescription(dir, test_file, 'named'), original_named_test_data, header) csv.writeCsv(appendDescription(dir, combined_file, 'named'), original_named_combined_data, header) """ Write out the .arff files """ writeArff(buildPath(dir, training_file, '.arff', 'orig'), 'soybean', classes, attrs, original_named_training_data) writeArff(buildPath(dir, test_file, '.arff', 'orig'), 'soybean', classes, attrs, original_named_test_data) writeArff(buildPath(dir, combined_file, '.arff', 'orig'), 'soybean', classes, attrs, original_named_combined_data) writeArff(buildPath(dir, training_file, '.arff'), 'soybean', classes, attrs, named_training_data) writeArff(buildPath(dir, test_file, '.arff'), 'soybean', classes, attrs, named_test_data) writeArff(buildPath(dir, combined_file, '.arff'), 'soybean', classes, attrs, named_combined_data) writeArff(buildPath(dir, random_file, '.arff'), 'soybean', classes, attrs, named_random_data)