def generate_order_list(df_events, ldt_timestamps, order_name='orders.csv'):
writer = csv.writer(open(order_name, 'wb'), delimiter=',')
trade_day = []
trade_order = []
trade_qty = []
trade_symbol = []
for symbol in list(df_events.keys()):
idx = np.where(df_events[symbol].notnull())[0]
if len(idx) > 0:
for day_idx in idx:
trade_day.append(ldt_timestamps[day_idx])
trade_order.append('BUY')
trade_qty.append(100)
trade_symbol.append(symbol)
trade_day.append(du.getNYSEoffset(ldt_timestamps[day_idx] ,5))
trade_order.append('SELL')
trade_qty.append(100)
trade_symbol.append(symbol)
d = {'SYMBOL':trade_symbol,
'ACTION': trade_order,
'QTY': trade_qty }
orders_df = pd.DataFrame(d,index=trade_day)
orders_df = orders_df.sort()
for idx, data in orders_df.iterrows():
writer.writerow([idx.year,
idx.month,
idx.day,
data['SYMBOL'],
data['ACTION'],
data['QTY'], ''])
def write_order(timestamp, symbol):
order_array.append([
timestamp.year, timestamp.month, timestamp.day, symbol, "BUY", 100
])
sell_date = du.getNYSEoffset(timestamp, 5)
order_array.append([
sell_date.year, sell_date.month, sell_date.day, symbol, "SELL", 100
])
def convert_events_to_orders(events):
logging.debug('Converting events into orders..')
orders = []
for ts, row in events.iterrows():
for stock, has_event_occured in row.iteritems():
if stock != 'SPY' and has_event_occured == EVENT_OCCURED:
buy_order = (ts.date(), stock, OrderSide.Buy, 100)
sell_date = du.getNYSEoffset(ts, 5).date()
sell_order = (sell_date, stock, OrderSide.Sell, 100)
orders.extend((buy_order, sell_order))
orders.sort(key=lambda row: row[0])
return [(order[0].year, order[0].month, order[0].day) + order[1:] for order in orders]
def find_events(ls_symbols, d_data):
''' Finding the event dataframe '''
df_close = d_data['close']
ts_market = df_close['SPY']
print "Finding Events"
# Creating an empty dataframe
df_events = copy.deepcopy(df_close)
df_events = df_events * np.NAN
# Time stamps for the event range
ldt_timestamps = df_close.index
spy_bollinger = (df_close['SPY'] - pd.rolling_mean(df_close['SPY'],20)) / pd.rolling_std(df_close['SPY'],20)
for s_sym in ls_symbols:
#if( s_sym == 'GOOG' ):
# print pd.rolling_mean(df_close[s_sym], 20)
#if( s_sym == 'SPY' ):
# spy_bollinger = (df_close[s_sym] - pd.rolling_mean(df_close[s_sym],20)) / pd.rolling_std(df_close[s_sym],20)
#else:
# equity_bollinger = (df_close[s_sym] - pd.rolling_mean(df_close[s_sym],20)) / pd.rolling_std(df_close[s_sym],20)
equity_bollinger = (df_close[s_sym] - pd.rolling_mean(df_close[s_sym],20)) / pd.rolling_std(df_close[s_sym],20)
for i in range(1, len(ldt_timestamps)):
# Calculating the returns for this timestamp
# f_symprice_today = df_close[s_sym].ix[ldt_timestamps[i]]
# f_symprice_yest = df_close[s_sym].ix[ldt_timestamps[i - 1]]
# f_marketprice_today = ts_market.ix[ldt_timestamps[i]]
# f_marketprice_yest = ts_market.ix[ldt_timestamps[i - 1]]
# f_symreturn_today = (f_symprice_today / f_symprice_yest) - 1
# f_marketreturn_today = (f_marketprice_today / f_marketprice_yest) - 1
# Event is found if the symbol is down more then 3% while the
# market is up more then 2%
#f_equity_today = (d_data['close'] - pd.rolling_mean(d_data['close'], int(sys.argv[4]))) / pd.rolling_std(d_data['close'], int(sys.argv[4]))
equity_bollinger_today = equity_bollinger.loc[ldt_timestamps[i]]
equity_bollinger_yest = equity_bollinger.loc[ldt_timestamps[i-1]]
spy_bollinger_today = spy_bollinger.loc[ldt_timestamps[i]]
if equity_bollinger_today < -2.0 and equity_bollinger_yest >= -2.0 and spy_bollinger_today >= 1.5:
buy_data = [ldt_timestamps[i].year, ldt_timestamps[i].month, ldt_timestamps[i].day, s_sym, 'Buy', 100]
#print buy_data
write_csv(buy_data)
sell_date = du.getNYSEoffset(ldt_timestamps[i],5)
sell_data = [sell_date.year, sell_date.month, sell_date.day, s_sym, 'Sell', 100]
#print sell_data
write_csv(sell_data)
#df_events[s_sym].ix[ldt_timestamps[i]] = 1
return df_events
def find_events(ls_symbols, d_data, key, limit, days, output):
''' Finding the event dataframe '''
df_close = d_data[key]
ts_market = df_close['SPY']
print "Finding Events"
# Creating an empty dataframe
df_events = copy.deepcopy(df_close)
df_events = df_events * np.NAN
# Time stamps for the event range
ldt_timestamps = df_close.index
resultFile = open(output,'wb')
wr = csv.writer(resultFile, dialect='excel')
for s_sym in ls_symbols:
for i in range(1, len(ldt_timestamps)):
# Calculating the returns for this timestamp
f_symprice_today = df_close[s_sym].ix[ldt_timestamps[i]]
f_symprice_yest = df_close[s_sym].ix[ldt_timestamps[i - 1]]
f_marketprice_today = ts_market.ix[ldt_timestamps[i]]
f_marketprice_yest = ts_market.ix[ldt_timestamps[i - 1]]
f_symreturn_today = (f_symprice_today / f_symprice_yest) - 1
f_marketreturn_today = (f_marketprice_today / f_marketprice_yest) - 1
if f_symprice_today < limit and f_symprice_yest >= limit:
df_events[s_sym].ix[ldt_timestamps[i]] = 1
date = ldt_timestamps[i]
row = [date.year, date.month, date.day, s_sym,'Buy',100,'']
wr.writerow(row)
sellDate = du.getNYSEoffset(date, 5)
if (sellDate > ldt_timestamps[-1]):
sellDate = ldt_timestamps[-1]
row = [sellDate.year, sellDate.month, sellDate.day, s_sym,'Sell',100,'']
wr.writerow(row)
# Event is found if the symbol is down more then 3% while the
# market is up more then 2%
#if f_symreturn_today <= -0.03 and f_marketreturn_today >= 0.02:
# df_events[s_sym].ix[ldt_timestamps[i]] = 1
return df_events
def find_events(ls_symbols, d_data):
''' Finding the event dataframe '''
df_close = d_data['actual_close']
# ts_market = df_close['SPY']
print "Finding Events"
# Creating an empty dataframe
df_events = copy.deepcopy(df_close)
df_events = df_events * np.NAN
# Time stamps for the event range
ldt_timestamps = df_close.index
#Erase the existing file
file = open('orders.csv', 'w')
writer = csv.writer(file, delimiter=',')
file.close()
for s_sym in ls_symbols:
for i in range(1, len(ldt_timestamps)):
# Calculating the returns for this timestamp
f_symprice_today = df_close[s_sym].ix[ldt_timestamps[i]]
f_symprice_yest = df_close[s_sym].ix[ldt_timestamps[i - 1]]
# f_marketprice_today = ts_market.ix[ldt_timestamps[i]]
# f_marketprice_yest = ts_market.ix[ldt_timestamps[i - 1]]
# f_symreturn_today = (f_symprice_today / f_symprice_yest) - 1
# f_marketreturn_today = (f_marketprice_today / f_marketprice_yest) - 1
# Event is found if the symbol is down more then 3% while the
# market is up more then 2%
if f_symprice_today < 6.0 and f_symprice_yest >= 6.0:
buy_data = [ldt_timestamps[i].year, ldt_timestamps[i].month, ldt_timestamps[i].day, s_sym, 'Buy', 100]
print buy_data
write_csv(buy_data)
sell_date = du.getNYSEoffset(ldt_timestamps[i],5)
sell_data = [sell_date.year, sell_date.month, sell_date.day, s_sym, 'Sell', 100]
print sell_data
write_csv(sell_data)
#df_events[s_sym].ix[ldt_timestamps[i]] = 1
return df_events
def process_events(events, day_delta, number_of_shares, dt_last_day):
print "Writing orders.txt"
with open('orders.csv', 'w') as orders_file:
for event_row in events.iterrows():
index = 0
for column in events.columns:
if not math.isnan(event_row[1][index]):
values = []
values.append(event_row[0].year)
values.append(event_row[0].month)
values.append(event_row[0].day)
values.append(column)
values.append('Buy')
values.append(number_of_shares)
csv_row = ','.join(str(v) for v in values)
csv_row = csv_row + ',' + '\n'
orders_file.write(csv_row)
sell_date = du.getNYSEoffset(event_row[0], day_delta)
if sell_date > dt_last_day:
sell_date = dt_last_day
values = []
values.append(sell_date.year)
values.append(sell_date.month)
values.append(sell_date.day)
values.append(column)
values.append('Sell')
values.append(number_of_shares)
csv_row = ','.join(str(v) for v in values)
csv_row = csv_row + ',' + '\n'
orders_file.write(csv_row)
index = index + 1
print "Done Writing orders.txt"
def testName(self): date = dt.datetime(2010,10,17,16) d = du.getNYSEoffset(date, 5) self.assertEqual(date, d)
def process_events():
dt_start = dt.datetime(2008, 1, 1)
dt_end = dt.datetime(2009, 12, 31)
ldt_timestamps = du.getNYSEdays(dt_start, dt_end, dt.timedelta(hours=16))
dataobj = da.DataAccess('Yahoo')
ls_symbols = dataobj.get_symbols_from_list('sp5002012')
# Limit number of stocks for testing
# ls_symbols = ls_symbols[:15]
# ls_symbols.append('SPY')
ls_keys = ['actual_close']
print "Reading Data"
ldf_data = dataobj.get_data(ldt_timestamps, ls_symbols, ls_keys)
d_data = dict(zip(ls_keys, ldf_data))
d_data['close'] = d_data['actual_close']
d_data['close'] = d_data['close'].fillna(method='ffill')
d_data['close'] = d_data['close'].fillna(method='bfill')
d_data['close'] = d_data['close'].fillna(1.0)
print "Finding Events"
df_events = find_events(ls_symbols, d_data)
print "Number of raw events " + str(df_events.sum(0).sum(0))
print "Writing orders.txt"
with open('orders.csv', 'w') as orders_file:
dt_last_day = dt.datetime(2009, 12, 31)
for event_row in df_events.iterrows():
index = 0
for column in df_events.columns:
if not math.isnan(event_row[1][index]):
values = []
values.append(event_row[0].year)
values.append(event_row[0].month)
values.append(event_row[0].day)
values.append(column)
values.append('Buy')
values.append(100)
csv_row = ','.join(str(v) for v in values)
csv_row = csv_row + ',' + '\n'
orders_file.write(csv_row)
sell_date = du.getNYSEoffset(event_row[0], 5)
if sell_date > dt_last_day:
sell_date = dt_last_day
values = []
values.append(sell_date.year)
values.append(sell_date.month)
values.append(sell_date.day)
values.append(column)
values.append('Sell')
values.append(100)
csv_row = ','.join(str(v) for v in values)
csv_row = csv_row + ',' + '\n'
orders_file.write(csv_row)
index = index + 1
print "Done"
def write_order(timestamp, symbol):
order_array.append([timestamp.year, timestamp.month, timestamp.day,
symbol, "BUY", 100])
sell_date = du.getNYSEoffset(timestamp, 5)
order_array.append([sell_date.year, sell_date.month, sell_date.day,
symbol, "SELL", 100])
def find_events(ls_symbols, d_data, key, limit, days, output, times):
""" Finding the event dataframe """
df_close = d_data[key]
ts_market = df_close["SPY"]
print "Finding Events"
# Creating an empty dataframe
df_events = copy.deepcopy(df_close)
df_events = df_events * np.NAN
print "DF_EVENTS", type(df_events)
p = df_close
loopback = 20
mean = pd.rolling_mean(p, loopback)
std = pd.rolling_std(p, loopback)
upper = mean + times * std
low = mean - times * std
bollinger = (df_close - mean) / std
# Time stamps for the event range
ldt_timestamps = df_close.index
resultFile = open(output, "wb")
wr = csv.writer(resultFile, dialect="excel")
for s_sym in ls_symbols:
for i in range(1, len(ldt_timestamps)):
# Calculating the returns for this timestamp
f_symprice_today = bollinger[s_sym].ix[ldt_timestamps[i]]
f_symprice_yest = bollinger[s_sym].ix[ldt_timestamps[i - 1]]
f_marketprice_today = bollinger["SPY"].ix[ldt_timestamps[i]] # ts_market.ix[ldt_timestamps[i]]
f_marketprice_yest = bollinger["SPY"].ix[ldt_timestamps[i - 1]] # ts_market.ix[ldt_timestamps[i - 1]]
f_symreturn_today = (f_symprice_today / f_symprice_yest) - 1
f_marketreturn_today = (f_marketprice_today / f_marketprice_yest) - 1
if f_symprice_today < limit and f_symprice_yest >= limit and f_marketprice_today >= 1.2:
df_events[s_sym].ix[ldt_timestamps[i]] = 1
date = ldt_timestamps[i]
row = [date.year, date.month, date.day, s_sym, "Buy", 100, ""]
wr.writerow(row)
sellDate = du.getNYSEoffset(date, 5)
if sellDate > ldt_timestamps[-1]:
sellDate = ldt_timestamps[-1]
row = [sellDate.year, sellDate.month, sellDate.day, s_sym, "Sell", 100, ""]
wr.writerow(row)
"""f_symprice_today = df_close[s_sym].ix[ldt_timestamps[i]]
f_symprice_yest = df_close[s_sym].ix[ldt_timestamps[i - 1]]
f_marketprice_today = ts_market.ix[ldt_timestamps[i]]
f_marketprice_yest = ts_market.ix[ldt_timestamps[i - 1]]
f_symreturn_today = (f_symprice_today / f_symprice_yest) - 1
f_marketreturn_today = (f_marketprice_today / f_marketprice_yest) - 1
if f_symprice_today < limit and f_symprice_yest >= limit:
df_events[s_sym].ix[ldt_timestamps[i]] = 1
date = ldt_timestamps[i]
row = [date.year, date.month, date.day, s_sym,'Buy',100,'']
wr.writerow(row)
sellDate = du.getNYSEoffset(date, 5)
if (sellDate > ldt_timestamps[-1]):
sellDate = ldt_timestamps[-1]
row = [sellDate.year, sellDate.month, sellDate.day, s_sym,'Sell',100,'']
wr.writerow(row)
"""
# Event is found if the symbol is down more then 3% while the
# market is up more then 2%
# if f_symreturn_today <= -0.03 and f_marketreturn_today >= 0.02:
# df_events[s_sym].ix[ldt_timestamps[i]] = 1
return df_events
def analysis(mkt, dt_date):
plot_chart = False
'''Main Function'''
#ls_symbols = ["G"]
# We need closing prices so the timestamp should be hours=16.
dt_timeofday = dt.timedelta(hours=16)
# Start and End date of the charts
#dt_start = dt.datetime.strptime(sys.argv[1], "%Y/%m/%d") + dt.timedelta(hours=16)
dt_start = dt.datetime(2016, 1, 1) + dt_timeofday
dt_end = dt_date #dt.datetime.strptime(datestr, "%Y/%m/%d") + dt_timeofday
# Get a list of trading days between the start and the end.
ldt_timestamps = qdu.getNYSEdays(dt_start, dt_end, dt_timeofday, mkt)
dt_start = qdu.getNYSEoffset(dt_end, -199, mkt)
ldt_timestamps = qdu.getNYSEdays(dt_start, dt_end, dt_timeofday, mkt)
# Creating an object of the dataaccess class with Yahoo as the source.
c_dataobj = da.DataAccess(mkt + 'Yahoo')
# Keys to be read from the data, it is good to read everything in one go.
ls_keys = ['open', 'high', 'low', 'close', 'volume', 'actual_close']
# Reading the data, now d_data is a dictionary with the keys above.
# Timestamps and symbols are the ones that were specified before.
ldf_data = c_dataobj.get_data(ldt_timestamps, ls_symbols, ls_keys)
d_data = dict(zip(ls_keys, ldf_data))
# Copying close price into separate dataframe to find rets
df_close = d_data['close']
df_actual_close = d_data['actual_close']
#SMA
df_sma50 = pd.rolling_mean(d_data['actual_close'], 50)
df_sma200 = pd.rolling_mean(d_data['actual_close'], 200)
#Bollinger Bands
df_mean = pd.rolling_mean(d_data['actual_close'], 20)
df_std = pd.rolling_std(d_data['actual_close'], 20)
upper_bband = df_mean + (2 * df_std)
lower_bband = df_mean - (2 * df_std)
df_bollinger = (df_actual_close - df_mean) / (2 * df_std)
#MACD
emaslow = pd.ewma(d_data['actual_close'], span=26)
emafast = pd.ewma(d_data['actual_close'], span=12)
macd = emafast - emaslow
ema9 = pd.ewma(macd, span=9)
str_date = dt.datetime.strftime(dt_end, "%Y%m%d")
f = open(analysisPath + 'analysis-' + str_date + '.csv', 'w')
f.write(
'symbol,close,sma50,sma200,sma50/sma200,upper_bband,lower_bband,bollinger,macd,ema9,macd-ema9\n'
)
#print df_close.tail()
#print df_mean.tail()
#print df_std.tail()
#print df_bollinger.tail()
# Plotting the prices with x-axis=timestamps
for ls_symbol in ls_symbols:
#print ls_symbol
#print ls_symbol + ",{},{},{},{}".format(df_close[ls_symbol][dt_end],df_sma50[ls_symbol][dt_end],df_sma200[ls_symbol][dt_end], df_bollinger[ls_symbol][dt_end])
f.write(ls_symbol + ",{},{},{},{},{},{},{},{},{},{}".format(
df_close[ls_symbol][dt_end], df_sma50[ls_symbol][dt_end],
df_sma200[ls_symbol][dt_end], df_sma50[ls_symbol][dt_end] /
df_sma200[ls_symbol][dt_end], upper_bband[ls_symbol][dt_end],
lower_bband[ls_symbol][dt_end], df_bollinger[ls_symbol][dt_end],
macd[ls_symbol][dt_end], ema9[ls_symbol][dt_end],
macd[ls_symbol][dt_end] - ema9[ls_symbol][dt_end]) + '\n')
#print df_bollinger[ls_symbol].tail(1)
if plot_chart:
if df_bollinger[ls_symbol].tail(1) <= -0.75:
plt.clf()
plt.subplot(211)
plt.plot(ldt_timestamps, df_close[ls_symbol], label=ls_symbol)
plt.legend()
plt.ylabel('Price')
plt.xlabel('Date')
plt.xticks(size='xx-small')
plt.xlim(ldt_timestamps[0], ldt_timestamps[-1])
plt.subplot(212)
plt.plot(ldt_timestamps,
df_bollinger[ls_symbol],
label=ls_symbol + '-Bollinger')
plt.axhline(1.0, color='r')
plt.axhline(-1.0, color='r')
plt.legend()
plt.ylabel('Bollinger')
plt.xlabel('Date')
plt.xticks(size='xx-small')
plt.xlim(ldt_timestamps[0], ldt_timestamps[-1])
plt.savefig(ls_symbol + '.pdf', format='pdf')
f.close()
