def batch_parser(): NEW_DATA_DIR = '/home/liulc/csi300/new_data/' stockfiles = [NEW_DATA_DIR+fn for fn in os.listdir(NEW_DATA_DIR) if fn.endswith('txt.gz')] stockfiles = sorted(stockfiles) #print stockfiles; exit() for filename in stockfiles: print 'Batch parsing ' + filename + '...' #np.save(filename[-21:-7]+'.npy',format_stock_data(filename)) #cache_dict = np.load(filename[-21:-7]+'.npy').tolist() cache_dict = format_stock_data(filename) insert_file_parser(filename, cache_dict, monday=True) cache_hs300 = format_hs300_data(filename) insert_hs300_parser(filename, cache_hs300)
def daily_parser(): NEW_DATA_DIR = '/home/liulc/csi300/new_data/' #today_date = datetime.today().date().strftime("%Y%m%d") today_date = raw_input('Please input the date you want to update(e.g. 20131010):\n') #print today_date; exit() stockfile = 'stock_' + today_date + '.txt.gz' stockfile = os.path.join(NEW_DATA_DIR, stockfile) if not os.path.isfile(stockfile): print '%s does not exist, please check, exiting...' %stockfile exit() #print stockfile; exit() checker = raw_input('You are going to update the data \ of %s to the raw_data, are you sure(Y/N)?\n' %today_date) if checker == 'N': print 'Please go ahead checking, exiting...' exit() elif checker == 'Y': cache_dict = format_stock_data(stockfile) insert_file_parser(stockfile, cache_dict, monday=True) cache_hs300 = format_hs300_data(stockfile) insert_hs300_parser(stockfile, cache_hs300)