date_list[df_daily_compressed_cnt] = prev_date
last_list[df_daily_compressed_cnt] = prev_last
volume_list[df_daily_compressed_cnt] = prev_volume
df_daily_compressed_cnt += 1
prev_volume = curr_volume
prev_date = curr_date
prev_last = curr_last
date_list[df_daily_compressed_cnt] = curr_date
last_list[df_daily_compressed_cnt] = curr_last
volume_list[df_daily_compressed_cnt] = curr_volume
df_daily_compressed = DataFrame({'Date': date_list, 'Last': last_list, 'Volume': volume_list})
df_compressed = df_compressed.append(df_daily_compressed)
# increment to next day
start_stamp_utc += Day()
# if start date is Thursday 5PM CST jump to Sunday 5PM CST
if start_stamp_utc.weekday() == 4:
start_stamp_utc += 2*Day()
futures_db.upload_dataframe_to_table(df_compressed, compressed_table_name)
futures_db.create_table_index(compressed_table_name, "Date")
futures_db.close_database_connection()
volume_list[df_daily_compressed_cnt] = prev_volume
df_daily_compressed_cnt += 1
prev_volume = curr_volume
prev_date = curr_date
prev_last = curr_last
date_list[df_daily_compressed_cnt] = curr_date
last_list[df_daily_compressed_cnt] = curr_last
volume_list[df_daily_compressed_cnt] = curr_volume
df_daily_compressed = DataFrame({
'Date': date_list,
'Last': last_list,
'Volume': volume_list
})
df_compressed = df_compressed.append(df_daily_compressed)
# increment to next day
start_stamp_utc += Day()
# if start date is Thursday 5PM CST jump to Sunday 5PM CST
if start_stamp_utc.weekday() == 4:
start_stamp_utc += 2 * Day()
futures_db.upload_dataframe_to_table(df_compressed, compressed_table_name)
futures_db.create_table_index(compressed_table_name, "Date")
futures_db.close_database_connection()
instrument_list = ['GC', 'CL', 'ZB']
futures_db = FuturesDatabase()
for instrument in instrument_list:
table_name = instrument + '_LAST'
futures_db.drop_table_if_exist(table_name)
futures_db.create_historical_table(table_name)
rootPath = "/home/aouyang1/NinjaTrader/TickData/" + instrument
folders = os.listdir(rootPath)
fnames = os.listdir(rootPath)
for fileNames in fnames:
print fileNames
df = pd.read_csv(rootPath + '/' + fileNames,
delimiter=";",
names=['Date', 'Last', 'Volume'],
parse_dates=[0],
date_parser=lambda x: datetime.datetime.strptime(
x, '%Y%m%d %H%M%S'))
futures_db.upload_dataframe_to_table(df, table_name)
futures_db.create_table_index(table_name, "Date")
futures_db.close_database_connection()
import pandas as pd
from util.futuresdatabase import FuturesDatabase
instrument_list = ['GC', 'CL', 'ZB']
futures_db = FuturesDatabase()
for instrument in instrument_list:
table_name = instrument + '_LAST'
futures_db.drop_table_if_exist(table_name)
futures_db.create_historical_table(table_name)
rootPath = "/home/aouyang1/NinjaTrader/TickData/" + instrument
folders = os.listdir(rootPath)
fnames = os.listdir(rootPath)
for fileNames in fnames:
print fileNames
df = pd.read_csv(rootPath + '/' + fileNames, delimiter=";", names=['Date', 'Last', 'Volume'], parse_dates=[0],
date_parser=lambda x: datetime.datetime.strptime(x, '%Y%m%d %H%M%S'))
futures_db.upload_dataframe_to_table(df, table_name)
futures_db.create_table_index(table_name, "Date")
futures_db.close_database_connection()
for fname in filelist:
try:
df = pd.read_csv(rootPath + '/' + fname, parse_dates=[8, 9],
date_parser=lambda x: datetime.datetime.strptime(x, '%m/%d/%Y %I:%M:%S %p'))
except:
df = pd.read_csv(rootPath + '/' + fname, parse_dates=[8, 9],
date_parser=lambda x: datetime.datetime.strptime(x, '%m/%d/%Y %I:%M'))
print '{} in PL {} has a different time format...'.format(fname, PL)
try:
df.drop('Unnamed: 19', axis=1, inplace=True)
except:
print '{} in PL {} has no Unnamed: 19 column...'.format(fname, PL)
df['PL'] = PL
cols = df.columns.tolist()
cols = cols[-1:] + cols[:-1]
df = df[cols]
df.columns = col_names
fdb.upload_dataframe_to_table(df, table_name)
stdout.write("\r%s" % table_name + " " + str(PL) + "/" + str(max(plrange)))
stdout.flush()
stdout.write("\n")
fdb.create_table_index(table_name, "PL")
print "---------------------"
fdb.close_database_connection()
