TICK_SIZE = 1/32.0
init_day = '2013-09-01 17:00:00'
final_day = '2014-11-30 16:59:59'
table_name = instr_name + '_LAST'
compressed_table_name = table_name + '_COMPRESSED'
start_stamp = pd.Timestamp(init_day).tz_localize('US/Central')
start_stamp_utc = start_stamp.tz_convert('utc')
final_stamp = pd.Timestamp(final_day).tz_localize('US/Central')
final_stamp_utc = final_stamp.tz_convert('utc')
futures_db = FuturesDatabase()
futures_db.drop_table_if_exist(compressed_table_name)
futures_db.create_historical_table(compressed_table_name)
df_compressed = DataFrame(columns=['Date', 'Last', 'Volume'])
while start_stamp_utc < final_stamp_utc:
start_date = timestamp_to_SQLstring(start_stamp_utc)
# get end of day timestamp
end_stamp_utc = start_stamp_utc + Day() - 45*Minute()
end_date = timestamp_to_SQLstring(end_stamp_utc)
df = futures_db.fetch_between_dates(table_name=table_name,
start_date=start_date,
TICK_SIZE = 1 / 32.0
init_day = '2013-09-01 17:00:00'
final_day = '2014-11-30 16:59:59'
table_name = instr_name + '_LAST'
compressed_table_name = table_name + '_COMPRESSED'
start_stamp = pd.Timestamp(init_day).tz_localize('US/Central')
start_stamp_utc = start_stamp.tz_convert('utc')
final_stamp = pd.Timestamp(final_day).tz_localize('US/Central')
final_stamp_utc = final_stamp.tz_convert('utc')
futures_db = FuturesDatabase()
futures_db.drop_table_if_exist(compressed_table_name)
futures_db.create_historical_table(compressed_table_name)
df_compressed = DataFrame(columns=['Date', 'Last', 'Volume'])
while start_stamp_utc < final_stamp_utc:
start_date = timestamp_to_SQLstring(start_stamp_utc)
# get end of day timestamp
end_stamp_utc = start_stamp_utc + Day() - 45 * Minute()
end_date = timestamp_to_SQLstring(end_stamp_utc)
df = futures_db.fetch_between_dates(table_name=table_name,
start_date=start_date,
import os
import datetime
import pandas as pd
from util.futuresdatabase import FuturesDatabase
instrument_list = ['GC', 'CL', 'ZB']
futures_db = FuturesDatabase()
for instrument in instrument_list:
table_name = instrument + '_LAST'
futures_db.drop_table_if_exist(table_name)
futures_db.create_historical_table(table_name)
rootPath = "/home/aouyang1/NinjaTrader/TickData/" + instrument
folders = os.listdir(rootPath)
fnames = os.listdir(rootPath)
for fileNames in fnames:
print fileNames
df = pd.read_csv(rootPath + '/' + fileNames,
delimiter=";",
names=['Date', 'Last', 'Volume'],
parse_dates=[0],
date_parser=lambda x: datetime.datetime.strptime(
x, '%Y%m%d %H%M%S'))
import os
import datetime
import pandas as pd
from util.futuresdatabase import FuturesDatabase
instrument_list = ['GC', 'CL', 'ZB']
futures_db = FuturesDatabase()
for instrument in instrument_list:
table_name = instrument + '_LAST'
futures_db.drop_table_if_exist(table_name)
futures_db.create_historical_table(table_name)
rootPath = "/home/aouyang1/NinjaTrader/TickData/" + instrument
folders = os.listdir(rootPath)
fnames = os.listdir(rootPath)
for fileNames in fnames:
print fileNames
df = pd.read_csv(rootPath + '/' + fileNames, delimiter=";", names=['Date', 'Last', 'Volume'], parse_dates=[0],
date_parser=lambda x: datetime.datetime.strptime(x, '%Y%m%d %H%M%S'))
futures_db.upload_dataframe_to_table(df, table_name)
futures_db.create_table_index(table_name, "Date")
"/home/aouyang1/Dropbox/Futures Trading/FT_QUICKY_v3/GC/CON1/PL",
"/home/aouyang1/Dropbox/Futures Trading/FT_QUICKY_v3/GC/BASE/PL"]
plrange_list = [range(13, 22, 2),
range(11, 41),
range(11, 41)]
col_names = ['PL', 'TradeNum', 'Instrument', 'Account', 'Strategy', 'Market_pos', 'Quantity',
'Entry_price', 'Exit_price', 'Entry_time', 'Exit_time', 'Entry_name', 'Exit_name',
'Profit', 'Cum_profit', 'Commission', 'MAE', 'MFE', 'ETD', 'Bars']
fdb = FuturesDatabase()
for table_name, rootname, plrange in zip(table_name_list, rootname_list, plrange_list):
#ipdb.set_trace()
fdb.drop_table_if_exist(table_name)
fdb.create_PL_table(table_name)
for PL in plrange:
rootPath = rootname + str(PL)
filelist = os.listdir(rootPath)
for fname in filelist:
try:
df = pd.read_csv(rootPath + '/' + fname, parse_dates=[8, 9],
date_parser=lambda x: datetime.datetime.strptime(x, '%m/%d/%Y %I:%M:%S %p'))
except:
df = pd.read_csv(rootPath + '/' + fname, parse_dates=[8, 9],
date_parser=lambda x: datetime.datetime.strptime(x, '%m/%d/%Y %I:%M'))
print '{} in PL {} has a different time format...'.format(fname, PL)
