volume_list[df_daily_compressed_cnt] = prev_volume
df_daily_compressed_cnt += 1
prev_volume = curr_volume
prev_date = curr_date
prev_last = curr_last
date_list[df_daily_compressed_cnt] = curr_date
last_list[df_daily_compressed_cnt] = curr_last
volume_list[df_daily_compressed_cnt] = curr_volume
df_daily_compressed = DataFrame({
'Date': date_list,
'Last': last_list,
'Volume': volume_list
})
df_compressed = df_compressed.append(df_daily_compressed)
# increment to next day
start_stamp_utc += Day()
# if start date is Thursday 5PM CST jump to Sunday 5PM CST
if start_stamp_utc.weekday() == 4:
start_stamp_utc += 2 * Day()
futures_db.upload_dataframe_to_table(df_compressed, compressed_table_name)
futures_db.create_table_index(compressed_table_name, "Date")
futures_db.close_database_connection()
date_list[df_daily_compressed_cnt] = prev_date
last_list[df_daily_compressed_cnt] = prev_last
volume_list[df_daily_compressed_cnt] = prev_volume
df_daily_compressed_cnt += 1
prev_volume = curr_volume
prev_date = curr_date
prev_last = curr_last
date_list[df_daily_compressed_cnt] = curr_date
last_list[df_daily_compressed_cnt] = curr_last
volume_list[df_daily_compressed_cnt] = curr_volume
df_daily_compressed = DataFrame({'Date': date_list, 'Last': last_list, 'Volume': volume_list})
df_compressed = df_compressed.append(df_daily_compressed)
# increment to next day
start_stamp_utc += Day()
# if start date is Thursday 5PM CST jump to Sunday 5PM CST
if start_stamp_utc.weekday() == 4:
start_stamp_utc += 2*Day()
futures_db.upload_dataframe_to_table(df_compressed, compressed_table_name)
futures_db.create_table_index(compressed_table_name, "Date")
futures_db.close_database_connection()
try:
df = pd.read_csv(rootPath + '/' + fname, parse_dates=[8, 9],
date_parser=lambda x: datetime.datetime.strptime(x, '%m/%d/%Y %I:%M:%S %p'))
except:
df = pd.read_csv(rootPath + '/' + fname, parse_dates=[8, 9],
date_parser=lambda x: datetime.datetime.strptime(x, '%m/%d/%Y %I:%M'))
print '{} in PL {} has a different time format...'.format(fname, PL)
try:
df.drop('Unnamed: 19', axis=1, inplace=True)
except:
print '{} in PL {} has no Unnamed: 19 column...'.format(fname, PL)
df['PL'] = PL
cols = df.columns.tolist()
cols = cols[-1:] + cols[:-1]
df = df[cols]
df.columns = col_names
fdb.upload_dataframe_to_table(df, table_name)
stdout.write("\r%s" % table_name + " " + str(PL) + "/" + str(max(plrange)))
stdout.flush()
stdout.write("\n")
fdb.create_table_index(table_name, "PL")
print "---------------------"
fdb.close_database_connection()
