def generate_order_list(df_events, ldt_timestamps, order_name='orders.csv'): writer = csv.writer(open(order_name, 'wb'), delimiter=',') trade_day = [] trade_order = [] trade_qty = [] trade_symbol = [] for symbol in list(df_events.keys()): idx = np.where(df_events[symbol].notnull())[0] if len(idx) > 0: for day_idx in idx: trade_day.append(ldt_timestamps[day_idx]) trade_order.append('BUY') trade_qty.append(100) trade_symbol.append(symbol) trade_day.append(du.getNYSEoffset(ldt_timestamps[day_idx] ,5)) trade_order.append('SELL') trade_qty.append(100) trade_symbol.append(symbol) d = {'SYMBOL':trade_symbol, 'ACTION': trade_order, 'QTY': trade_qty } orders_df = pd.DataFrame(d,index=trade_day) orders_df = orders_df.sort() for idx, data in orders_df.iterrows(): writer.writerow([idx.year, idx.month, idx.day, data['SYMBOL'], data['ACTION'], data['QTY'], ''])
def write_order(timestamp, symbol): order_array.append([ timestamp.year, timestamp.month, timestamp.day, symbol, "BUY", 100 ]) sell_date = du.getNYSEoffset(timestamp, 5) order_array.append([ sell_date.year, sell_date.month, sell_date.day, symbol, "SELL", 100 ])
def convert_events_to_orders(events): logging.debug('Converting events into orders..') orders = [] for ts, row in events.iterrows(): for stock, has_event_occured in row.iteritems(): if stock != 'SPY' and has_event_occured == EVENT_OCCURED: buy_order = (ts.date(), stock, OrderSide.Buy, 100) sell_date = du.getNYSEoffset(ts, 5).date() sell_order = (sell_date, stock, OrderSide.Sell, 100) orders.extend((buy_order, sell_order)) orders.sort(key=lambda row: row[0]) return [(order[0].year, order[0].month, order[0].day) + order[1:] for order in orders]
def find_events(ls_symbols, d_data): ''' Finding the event dataframe ''' df_close = d_data['close'] ts_market = df_close['SPY'] print "Finding Events" # Creating an empty dataframe df_events = copy.deepcopy(df_close) df_events = df_events * np.NAN # Time stamps for the event range ldt_timestamps = df_close.index spy_bollinger = (df_close['SPY'] - pd.rolling_mean(df_close['SPY'],20)) / pd.rolling_std(df_close['SPY'],20) for s_sym in ls_symbols: #if( s_sym == 'GOOG' ): # print pd.rolling_mean(df_close[s_sym], 20) #if( s_sym == 'SPY' ): # spy_bollinger = (df_close[s_sym] - pd.rolling_mean(df_close[s_sym],20)) / pd.rolling_std(df_close[s_sym],20) #else: # equity_bollinger = (df_close[s_sym] - pd.rolling_mean(df_close[s_sym],20)) / pd.rolling_std(df_close[s_sym],20) equity_bollinger = (df_close[s_sym] - pd.rolling_mean(df_close[s_sym],20)) / pd.rolling_std(df_close[s_sym],20) for i in range(1, len(ldt_timestamps)): # Calculating the returns for this timestamp # f_symprice_today = df_close[s_sym].ix[ldt_timestamps[i]] # f_symprice_yest = df_close[s_sym].ix[ldt_timestamps[i - 1]] # f_marketprice_today = ts_market.ix[ldt_timestamps[i]] # f_marketprice_yest = ts_market.ix[ldt_timestamps[i - 1]] # f_symreturn_today = (f_symprice_today / f_symprice_yest) - 1 # f_marketreturn_today = (f_marketprice_today / f_marketprice_yest) - 1 # Event is found if the symbol is down more then 3% while the # market is up more then 2% #f_equity_today = (d_data['close'] - pd.rolling_mean(d_data['close'], int(sys.argv[4]))) / pd.rolling_std(d_data['close'], int(sys.argv[4])) equity_bollinger_today = equity_bollinger.loc[ldt_timestamps[i]] equity_bollinger_yest = equity_bollinger.loc[ldt_timestamps[i-1]] spy_bollinger_today = spy_bollinger.loc[ldt_timestamps[i]] if equity_bollinger_today < -2.0 and equity_bollinger_yest >= -2.0 and spy_bollinger_today >= 1.5: buy_data = [ldt_timestamps[i].year, ldt_timestamps[i].month, ldt_timestamps[i].day, s_sym, 'Buy', 100] #print buy_data write_csv(buy_data) sell_date = du.getNYSEoffset(ldt_timestamps[i],5) sell_data = [sell_date.year, sell_date.month, sell_date.day, s_sym, 'Sell', 100] #print sell_data write_csv(sell_data) #df_events[s_sym].ix[ldt_timestamps[i]] = 1 return df_events
def find_events(ls_symbols, d_data, key, limit, days, output): ''' Finding the event dataframe ''' df_close = d_data[key] ts_market = df_close['SPY'] print "Finding Events" # Creating an empty dataframe df_events = copy.deepcopy(df_close) df_events = df_events * np.NAN # Time stamps for the event range ldt_timestamps = df_close.index resultFile = open(output,'wb') wr = csv.writer(resultFile, dialect='excel') for s_sym in ls_symbols: for i in range(1, len(ldt_timestamps)): # Calculating the returns for this timestamp f_symprice_today = df_close[s_sym].ix[ldt_timestamps[i]] f_symprice_yest = df_close[s_sym].ix[ldt_timestamps[i - 1]] f_marketprice_today = ts_market.ix[ldt_timestamps[i]] f_marketprice_yest = ts_market.ix[ldt_timestamps[i - 1]] f_symreturn_today = (f_symprice_today / f_symprice_yest) - 1 f_marketreturn_today = (f_marketprice_today / f_marketprice_yest) - 1 if f_symprice_today < limit and f_symprice_yest >= limit: df_events[s_sym].ix[ldt_timestamps[i]] = 1 date = ldt_timestamps[i] row = [date.year, date.month, date.day, s_sym,'Buy',100,''] wr.writerow(row) sellDate = du.getNYSEoffset(date, 5) if (sellDate > ldt_timestamps[-1]): sellDate = ldt_timestamps[-1] row = [sellDate.year, sellDate.month, sellDate.day, s_sym,'Sell',100,''] wr.writerow(row) # Event is found if the symbol is down more then 3% while the # market is up more then 2% #if f_symreturn_today <= -0.03 and f_marketreturn_today >= 0.02: # df_events[s_sym].ix[ldt_timestamps[i]] = 1 return df_events
def find_events(ls_symbols, d_data): ''' Finding the event dataframe ''' df_close = d_data['actual_close'] # ts_market = df_close['SPY'] print "Finding Events" # Creating an empty dataframe df_events = copy.deepcopy(df_close) df_events = df_events * np.NAN # Time stamps for the event range ldt_timestamps = df_close.index #Erase the existing file file = open('orders.csv', 'w') writer = csv.writer(file, delimiter=',') file.close() for s_sym in ls_symbols: for i in range(1, len(ldt_timestamps)): # Calculating the returns for this timestamp f_symprice_today = df_close[s_sym].ix[ldt_timestamps[i]] f_symprice_yest = df_close[s_sym].ix[ldt_timestamps[i - 1]] # f_marketprice_today = ts_market.ix[ldt_timestamps[i]] # f_marketprice_yest = ts_market.ix[ldt_timestamps[i - 1]] # f_symreturn_today = (f_symprice_today / f_symprice_yest) - 1 # f_marketreturn_today = (f_marketprice_today / f_marketprice_yest) - 1 # Event is found if the symbol is down more then 3% while the # market is up more then 2% if f_symprice_today < 6.0 and f_symprice_yest >= 6.0: buy_data = [ldt_timestamps[i].year, ldt_timestamps[i].month, ldt_timestamps[i].day, s_sym, 'Buy', 100] print buy_data write_csv(buy_data) sell_date = du.getNYSEoffset(ldt_timestamps[i],5) sell_data = [sell_date.year, sell_date.month, sell_date.day, s_sym, 'Sell', 100] print sell_data write_csv(sell_data) #df_events[s_sym].ix[ldt_timestamps[i]] = 1 return df_events
def process_events(events, day_delta, number_of_shares, dt_last_day): print "Writing orders.txt" with open('orders.csv', 'w') as orders_file: for event_row in events.iterrows(): index = 0 for column in events.columns: if not math.isnan(event_row[1][index]): values = [] values.append(event_row[0].year) values.append(event_row[0].month) values.append(event_row[0].day) values.append(column) values.append('Buy') values.append(number_of_shares) csv_row = ','.join(str(v) for v in values) csv_row = csv_row + ',' + '\n' orders_file.write(csv_row) sell_date = du.getNYSEoffset(event_row[0], day_delta) if sell_date > dt_last_day: sell_date = dt_last_day values = [] values.append(sell_date.year) values.append(sell_date.month) values.append(sell_date.day) values.append(column) values.append('Sell') values.append(number_of_shares) csv_row = ','.join(str(v) for v in values) csv_row = csv_row + ',' + '\n' orders_file.write(csv_row) index = index + 1 print "Done Writing orders.txt"
def testName(self): date = dt.datetime(2010,10,17,16) d = du.getNYSEoffset(date, 5) self.assertEqual(date, d)
def process_events(): dt_start = dt.datetime(2008, 1, 1) dt_end = dt.datetime(2009, 12, 31) ldt_timestamps = du.getNYSEdays(dt_start, dt_end, dt.timedelta(hours=16)) dataobj = da.DataAccess('Yahoo') ls_symbols = dataobj.get_symbols_from_list('sp5002012') # Limit number of stocks for testing # ls_symbols = ls_symbols[:15] # ls_symbols.append('SPY') ls_keys = ['actual_close'] print "Reading Data" ldf_data = dataobj.get_data(ldt_timestamps, ls_symbols, ls_keys) d_data = dict(zip(ls_keys, ldf_data)) d_data['close'] = d_data['actual_close'] d_data['close'] = d_data['close'].fillna(method='ffill') d_data['close'] = d_data['close'].fillna(method='bfill') d_data['close'] = d_data['close'].fillna(1.0) print "Finding Events" df_events = find_events(ls_symbols, d_data) print "Number of raw events " + str(df_events.sum(0).sum(0)) print "Writing orders.txt" with open('orders.csv', 'w') as orders_file: dt_last_day = dt.datetime(2009, 12, 31) for event_row in df_events.iterrows(): index = 0 for column in df_events.columns: if not math.isnan(event_row[1][index]): values = [] values.append(event_row[0].year) values.append(event_row[0].month) values.append(event_row[0].day) values.append(column) values.append('Buy') values.append(100) csv_row = ','.join(str(v) for v in values) csv_row = csv_row + ',' + '\n' orders_file.write(csv_row) sell_date = du.getNYSEoffset(event_row[0], 5) if sell_date > dt_last_day: sell_date = dt_last_day values = [] values.append(sell_date.year) values.append(sell_date.month) values.append(sell_date.day) values.append(column) values.append('Sell') values.append(100) csv_row = ','.join(str(v) for v in values) csv_row = csv_row + ',' + '\n' orders_file.write(csv_row) index = index + 1 print "Done"
def write_order(timestamp, symbol): order_array.append([timestamp.year, timestamp.month, timestamp.day, symbol, "BUY", 100]) sell_date = du.getNYSEoffset(timestamp, 5) order_array.append([sell_date.year, sell_date.month, sell_date.day, symbol, "SELL", 100])
def find_events(ls_symbols, d_data, key, limit, days, output, times): """ Finding the event dataframe """ df_close = d_data[key] ts_market = df_close["SPY"] print "Finding Events" # Creating an empty dataframe df_events = copy.deepcopy(df_close) df_events = df_events * np.NAN print "DF_EVENTS", type(df_events) p = df_close loopback = 20 mean = pd.rolling_mean(p, loopback) std = pd.rolling_std(p, loopback) upper = mean + times * std low = mean - times * std bollinger = (df_close - mean) / std # Time stamps for the event range ldt_timestamps = df_close.index resultFile = open(output, "wb") wr = csv.writer(resultFile, dialect="excel") for s_sym in ls_symbols: for i in range(1, len(ldt_timestamps)): # Calculating the returns for this timestamp f_symprice_today = bollinger[s_sym].ix[ldt_timestamps[i]] f_symprice_yest = bollinger[s_sym].ix[ldt_timestamps[i - 1]] f_marketprice_today = bollinger["SPY"].ix[ldt_timestamps[i]] # ts_market.ix[ldt_timestamps[i]] f_marketprice_yest = bollinger["SPY"].ix[ldt_timestamps[i - 1]] # ts_market.ix[ldt_timestamps[i - 1]] f_symreturn_today = (f_symprice_today / f_symprice_yest) - 1 f_marketreturn_today = (f_marketprice_today / f_marketprice_yest) - 1 if f_symprice_today < limit and f_symprice_yest >= limit and f_marketprice_today >= 1.2: df_events[s_sym].ix[ldt_timestamps[i]] = 1 date = ldt_timestamps[i] row = [date.year, date.month, date.day, s_sym, "Buy", 100, ""] wr.writerow(row) sellDate = du.getNYSEoffset(date, 5) if sellDate > ldt_timestamps[-1]: sellDate = ldt_timestamps[-1] row = [sellDate.year, sellDate.month, sellDate.day, s_sym, "Sell", 100, ""] wr.writerow(row) """f_symprice_today = df_close[s_sym].ix[ldt_timestamps[i]] f_symprice_yest = df_close[s_sym].ix[ldt_timestamps[i - 1]] f_marketprice_today = ts_market.ix[ldt_timestamps[i]] f_marketprice_yest = ts_market.ix[ldt_timestamps[i - 1]] f_symreturn_today = (f_symprice_today / f_symprice_yest) - 1 f_marketreturn_today = (f_marketprice_today / f_marketprice_yest) - 1 if f_symprice_today < limit and f_symprice_yest >= limit: df_events[s_sym].ix[ldt_timestamps[i]] = 1 date = ldt_timestamps[i] row = [date.year, date.month, date.day, s_sym,'Buy',100,''] wr.writerow(row) sellDate = du.getNYSEoffset(date, 5) if (sellDate > ldt_timestamps[-1]): sellDate = ldt_timestamps[-1] row = [sellDate.year, sellDate.month, sellDate.day, s_sym,'Sell',100,''] wr.writerow(row) """ # Event is found if the symbol is down more then 3% while the # market is up more then 2% # if f_symreturn_today <= -0.03 and f_marketreturn_today >= 0.02: # df_events[s_sym].ix[ldt_timestamps[i]] = 1 return df_events
def analysis(mkt, dt_date): plot_chart = False '''Main Function''' #ls_symbols = ["G"] # We need closing prices so the timestamp should be hours=16. dt_timeofday = dt.timedelta(hours=16) # Start and End date of the charts #dt_start = dt.datetime.strptime(sys.argv[1], "%Y/%m/%d") + dt.timedelta(hours=16) dt_start = dt.datetime(2016, 1, 1) + dt_timeofday dt_end = dt_date #dt.datetime.strptime(datestr, "%Y/%m/%d") + dt_timeofday # Get a list of trading days between the start and the end. ldt_timestamps = qdu.getNYSEdays(dt_start, dt_end, dt_timeofday, mkt) dt_start = qdu.getNYSEoffset(dt_end, -199, mkt) ldt_timestamps = qdu.getNYSEdays(dt_start, dt_end, dt_timeofday, mkt) # Creating an object of the dataaccess class with Yahoo as the source. c_dataobj = da.DataAccess(mkt + 'Yahoo') # Keys to be read from the data, it is good to read everything in one go. ls_keys = ['open', 'high', 'low', 'close', 'volume', 'actual_close'] # Reading the data, now d_data is a dictionary with the keys above. # Timestamps and symbols are the ones that were specified before. ldf_data = c_dataobj.get_data(ldt_timestamps, ls_symbols, ls_keys) d_data = dict(zip(ls_keys, ldf_data)) # Copying close price into separate dataframe to find rets df_close = d_data['close'] df_actual_close = d_data['actual_close'] #SMA df_sma50 = pd.rolling_mean(d_data['actual_close'], 50) df_sma200 = pd.rolling_mean(d_data['actual_close'], 200) #Bollinger Bands df_mean = pd.rolling_mean(d_data['actual_close'], 20) df_std = pd.rolling_std(d_data['actual_close'], 20) upper_bband = df_mean + (2 * df_std) lower_bband = df_mean - (2 * df_std) df_bollinger = (df_actual_close - df_mean) / (2 * df_std) #MACD emaslow = pd.ewma(d_data['actual_close'], span=26) emafast = pd.ewma(d_data['actual_close'], span=12) macd = emafast - emaslow ema9 = pd.ewma(macd, span=9) str_date = dt.datetime.strftime(dt_end, "%Y%m%d") f = open(analysisPath + 'analysis-' + str_date + '.csv', 'w') f.write( 'symbol,close,sma50,sma200,sma50/sma200,upper_bband,lower_bband,bollinger,macd,ema9,macd-ema9\n' ) #print df_close.tail() #print df_mean.tail() #print df_std.tail() #print df_bollinger.tail() # Plotting the prices with x-axis=timestamps for ls_symbol in ls_symbols: #print ls_symbol #print ls_symbol + ",{},{},{},{}".format(df_close[ls_symbol][dt_end],df_sma50[ls_symbol][dt_end],df_sma200[ls_symbol][dt_end], df_bollinger[ls_symbol][dt_end]) f.write(ls_symbol + ",{},{},{},{},{},{},{},{},{},{}".format( df_close[ls_symbol][dt_end], df_sma50[ls_symbol][dt_end], df_sma200[ls_symbol][dt_end], df_sma50[ls_symbol][dt_end] / df_sma200[ls_symbol][dt_end], upper_bband[ls_symbol][dt_end], lower_bband[ls_symbol][dt_end], df_bollinger[ls_symbol][dt_end], macd[ls_symbol][dt_end], ema9[ls_symbol][dt_end], macd[ls_symbol][dt_end] - ema9[ls_symbol][dt_end]) + '\n') #print df_bollinger[ls_symbol].tail(1) if plot_chart: if df_bollinger[ls_symbol].tail(1) <= -0.75: plt.clf() plt.subplot(211) plt.plot(ldt_timestamps, df_close[ls_symbol], label=ls_symbol) plt.legend() plt.ylabel('Price') plt.xlabel('Date') plt.xticks(size='xx-small') plt.xlim(ldt_timestamps[0], ldt_timestamps[-1]) plt.subplot(212) plt.plot(ldt_timestamps, df_bollinger[ls_symbol], label=ls_symbol + '-Bollinger') plt.axhline(1.0, color='r') plt.axhline(-1.0, color='r') plt.legend() plt.ylabel('Bollinger') plt.xlabel('Date') plt.xticks(size='xx-small') plt.xlim(ldt_timestamps[0], ldt_timestamps[-1]) plt.savefig(ls_symbol + '.pdf', format='pdf') f.close()