def convert_stock_data_to_stock_files(): timer = Timer() timer.start_timer() only_files = [f for f in listdir(temporary_zipping_location_path) if isfile(join(temporary_zipping_location_path, f))] start_symbol = set() good_stocks = set() bad_stocks = set() file_index = 0 for file_name in only_files: print("File name: " + file_name + "\t" + str(file_index)) file_index += 1 with open(temporary_zipping_location_path + file_name, "r") as csv_file: data_reader = csv.reader(csv_file) for row in data_reader: purged_stock_name = stock.get_purged_name(row[0]) if purged_stock_name not in good_stocks and purged_stock_name not in bad_stocks: good_stocks.add(purged_stock_name) new_file = open((all_stocks_path + purged_stock_name + '.csv'), 'w') new_file.close() start_symbol.add(row[0][0]) for start in start_symbol: print(start) timer.end_timer() timer.print_time()
def fill_stock_files_with_information(): timer = Timer() timer.start_timer() only_files = [f for f in listdir(temporary_zipping_location_path) if isfile(join(temporary_zipping_location_path, f))] all_stock_names = [f for f in listdir(all_stocks_path) if isfile(join(all_stocks_path, f))] date_list = [] for file_name in only_files: x = file_name[1:].replace('.csv', '') month = x[0] + x[1] day = x[2] + x[3] year = x[4] + x[5] date_list.append([year, month, day]) file_index = 0 total_file_index = 0 all_stock_information_as_dictionary = {} ignore_list = get_stock_ignore_list() for date in reversed(sorted(date_list)): temp_file_name = 's' + date[1] + date[2] + date[0] + '.csv' print("Opening: " + temp_file_name + "\tthis is file #: " + str(file_index)) file_index += 1 total_file_index += 1 with open(temporary_zipping_location_path + temp_file_name, "r") as csv_file: data_reader = csv.reader(csv_file) for row in data_reader: purged_stock_name = stock.get_purged_name(row[0]) if purged_stock_name not in ignore_list: file_name_to_append_to = all_stocks_path + purged_stock_name + '.csv' if file_name_to_append_to in all_stock_information_as_dictionary: all_stock_information_as_dictionary[file_name_to_append_to].append(DayData(row[1], row[2], row[3], row[4], row[5], row[6])) else: all_stock_information_as_dictionary[file_name_to_append_to] = [DayData(row[1], row[2], row[3], row[4], row[5], row[6])] if file_index >= 200: print('Dumping Information') for key in all_stock_information_as_dictionary: #print("Stock file to append to: " + key) with open(key, 'a') as apple: writer = csv.writer(apple) for r in all_stock_information_as_dictionary[key]: writer.writerow( [r.get_date(), r.get_open(), r.get_high(), r.get_low(), r.get_close(), r.get_volume()]) all_stock_information_as_dictionary.clear() print('Dumping finished.\ton file ' + str(total_file_index)) file_index = 0 timer.end_timer() timer.print_time()
def get_stock_ignore_list(): print("Getting stock ignore list.") only_files = [f for f in listdir(temporary_zipping_location_path) if isfile(join(temporary_zipping_location_path, f))] bad_stocks = set() file_index = 0 for file_name in only_files: print("File name: " + file_name + "\t" + str(file_index)) file_index += 1 with open(temporary_zipping_location_path + file_name, "r") as csv_file: data_reader = csv.reader(csv_file) net_data = 0 net_volume = 0 has_2015 = False for row in data_reader: purged_stock_name = stock.get_purged_name(row[0]) # Check has 2015. if row[1].split('/')[2] == '2015': has_2015 = True # Check volume. if float(row[6]) <= 1000.0: net_volume -= 1 else: net_volume += 1 # Check price. if float(row[2]) <= 5.0: net_data -= 1 else: net_data += 1 if net_data <= 0 or not has_2015 or net_volume <= 0: bad_stocks.add(purged_stock_name) print("Finished getting stock ignore list.") return bad_stocks