def get_full_transcripts(): list_of_dicts = gd.get_data_list_of_dicts() final = {} identifiers = [] for entry in list_of_dicts: if not entry["Identifier"] in identifiers: identifiers.append(entry["Identifier"]) final[entry["Identifier"]] = [entry] else: final[entry["Identifier"]].append(entry) for i in final.keys(): final[i] = sorted(final[i], key= lambda k: k['Title']) transcripts = {} for entry in final.keys(): temp_string = "" for item in final[entry]: temp_string += item["Transcript"] transcripts[entry] = temp_string return transcripts
def makeTrainingData(wordList): list_of_dicts = gd.get_data_list_of_dicts() wordCounts = [] probability = 3 full = [] partial = [] added = { "0-10": False, "10-20": False, "20-30": False, "30-40": False, "40-50": False, "50-60": False, "60-70": False, "70-80": False, "80-90": False, "unkown": False } for entry in list_of_dicts: tempDict = {} decade = "unkown" if len(entry["Age of Author"]) > 3 and len(entry["Age of Author"]) < 9: decade = entry["Age of Author"] tempDict["class"] = decade t = entry["Transcript"] t = t.replace("<br>", " ") t = t.replace("COMMA", " ") t = re.sub('\W', ' ', t) t = t.split(" ") for word in t: if not word == "" and word in wordList: word = word.lower() try: tempDict[word] += 1 except: tempDict[word] = 1 full.append(tempDict) temp = randint(0, 10) if temp > probability or not added[decade]: added[decade] = True partial.append(tempDict) filename1 = "trainingData.csv" filename2 = "fullData.csv" wordList.append("class") headers = wordList gd.write_data_dicts(filename1, headers, partial) gd.write_data_dicts(filename2, headers, full)
def makeTrainingData(wordList): list_of_dicts = gd.get_data_list_of_dicts() wordCounts = [] probability = 3 full = [] partial = [] added = {"0-10":False, "10-20":False, "20-30":False, "30-40":False, "40-50":False, "50-60":False, "60-70":False, "70-80":False, "80-90":False, "unkown":False } for entry in list_of_dicts: tempDict = {} decade = "unkown" if len(entry["Age of Author"]) > 3 and len(entry["Age of Author"]) < 9: decade = entry["Age of Author"] tempDict["class"] = decade t = entry["Transcript"] t = t.replace("<br>", " ") t = t.replace("COMMA", " ") t = re.sub('\W',' ',t) t = t.split(" ") for word in t: if not word == "" and word in wordList: word = word.lower() try: tempDict[word] += 1 except: tempDict[word] = 1 full.append(tempDict) temp = randint(0,10) if temp > probability or not added[decade]: added[decade] = True partial.append(tempDict) filename1 = "trainingData.csv" filename2 = "fullData.csv" wordList.append("class") headers = wordList gd.write_data_dicts(filename1,headers,partial) gd.write_data_dicts(filename2,headers,full)
def normalize_data(): list_of_dicts = gd.get_data_list_of_dicts() headers = gd.get_headers() final = [] for y in range(len(headers)): final.append([]) for x in range(len(headers)): col = gd.get_data_slice(headers[x], list_of_dicts) colFloat = convert_colFloat(col) final[x].append(np.std(colFloat)) final[x].append(np.mean(colFloat)) return final
def remove_dup_cols(): list_of_dicts = gd.get_data_list_of_dicts() headers = gd.get_headers() cols_2remove = [] for x in range(len(headers)-15): col1 = gd.get_data_slice(headers[x],list_of_dicts) col1 = replace_missing(col1) for y in range(len(headers)-15): col2 = gd.get_data_slice(headers[y],list_of_dicts) col2 = replace_missing(col2) r = find_r(col1,col2) pairIn = exist_same_pair(cols_2remove, x, y) if r>=.99 and r<=1.002 and not x ==y and not pairIn: cols_2remove.append([y,x, r[0], headers[x], headers[y]])
def make_points(): rows_list_of_lists = gd.get_data_list_of_dicts() rows_list_of_lists = rows_list_of_lists[1:] points = [] for row in rows_list_of_lists: temp = [] for entry in row.keys(): temp.append(0) for entry in row.keys(): index = get_index(entry) temp[index] = float(row[entry]) temp_point = Point(temp) temp_point.normalize() points.append(temp_point) return points
import get_data as gd import re import operator list_of_dicts = gd.get_data_list_of_dicts() decades_list = [{ "decade": "1800", "words": {}, "letters": 0, "wordCount": 0 }, { "decade": "1810", "words": {}, "letters": 0, "wordCount": 0 }, { "decade": "1820", "words": {}, "letters": 0, "wordCount": 0 }, { "decade": "1830", "words": {}, "letters": 0, "wordCount": 0 }, { "decade": "1840", "words": {}, "letters": 0, "wordCount": 0
final = final else: final = final + i print "entry: " + final new_headers.append(final) return new_headers def change_to_list_of_lists(list_of_dicts): headers = gd.get_headers() final = [] for x in range(len(list_of_dicts)): final.append([]) for entry in headers: col = gd.get_data_slice(entry, list_of_dicts) for y in range(len(list_of_dicts)): final[y].append(col[y]) return final filename = "cleaned_headers.csv" list_of_dicts = gd.get_data_list_of_dicts() new_headers = cleaned_headers() rows_list_of_lists = change_to_list_of_lists(list_of_dicts) gd.write_data(filename, new_headers, rows_list_of_lists)