def get_following_vix(self, from_date=None, to_date=None): from_date = from_date or TradeTime.get_latest_trade_date( ) - datetime.timedelta(30) to_date = to_date or TradeTime.get_latest_trade_date() #self.logger.info('today=%s, from_date=%s, to_date=%s'%(datetime.datetime.today(), from_date, to_date)) symbols = VIX.get_vix_symbol_list(from_date, to_date, 3) #records_index = self.get_vix_price_by_symbol('VIY00') symbol_dic = {} for symbol in symbols: symbol_dic[symbol] = list_to_hash( self.get_vix_price_by_symbol_and_date(symbol, from_date, to_date)) days = (to_date - from_date).days + 1 records_f1 = [] records_f2 = [] records_f3 = [] for i in range(days): date = from_date + datetime.timedelta(days=i) if TradeTime.is_trade_day(date): symbol_f1 = VIX.get_f1_by_date(date) symbol_f2 = VIX.get_f2_by_date(date) symbol_f3 = VIX.get_f3_by_date(date) records_f1.append( [date, symbol_dic[symbol_f1].get(date), symbol_f1]) records_f2.append( [date, symbol_dic[symbol_f2].get(date), symbol_f2]) records_f3.append( [date, symbol_dic[symbol_f3].get(date), symbol_f3]) # self.logger.info([records_f1[-1], records_f2[-1]], records_f3[-1]]) return (records_f1, records_f2, records_f3)
def run_schedule_only(schedule_functions, start_date, end_date, logger): for date in TradeTime.generate_dates(start_date, end_date): sf_dt_list = map(lambda x: [x, x.time_rule.get_datetime(date)], schedule_functions) sf_dt_list.sort(key=lambda x: x[1]) for sf_dt in sf_dt_list: if Container.context.terminate_p: return schedule_function = sf_dt[0] dt = sf_dt[1] if schedule_function.date_rule.validate(dt): Container.data.set_datetime(dt) logger.set_dt(dt) try: start = datetime.datetime.now() if TradeTime.is_half_trade_day(dt.date()) and schedule_function.half_days is False: pass else: schedule_function.my_func() end = datetime.datetime.now() # logger.info('Spend time for schedule function: %s seconds' % (end - start).seconds) except Exception as e: logger.error('Trace: ' + traceback.format_exc(), False) logger.error('Error: get action arguments failed: %s, %s' % (str(e), traceback.format_exc())) Container.data.set_datetime(datetime.datetime(date.year, date.month, date.day, 16, 0, 0)) Container.analysis.add_portfolio_trace(date, Container.api.portfolio)
def GET(self, symbol, days): try: days = int(days) except Exception: days = 63 from_date = TradeTime.get_latest_trade_date() - datetime.timedelta(days) equity_records = EquityDAO().get_all_equity_price_by_symbol(symbol, from_date) if datetime.date.today() > TradeTime.get_latest_trade_date(): new_spy_price = YahooScraper.get_data_by_symbol(symbol) equity_records.append([datetime.date.today(), new_spy_price]) dates = map(lambda x: x[0], equity_records) equity_prices = map(lambda x: x[1], equity_records) fig = Figure(figsize=[16, 4]) ax = fig.add_axes([.1, .1, .8, .9]) ax.plot(dates, equity_prices, label='price') ax.legend(loc='upper left') ax.grid() ax.xaxis.set_major_formatter(DateFormatter('%y%m%d')) ax.set_xticks(dates) for tick in ax.get_xticklabels(): tick.set_rotation(45) # conver to canvas canvas = FigureCanvasAgg(fig) buf = cStringIO.StringIO() canvas.print_png(buf) data = buf.getvalue() return data
def validate_integrity_for_real_time_data(self, symbol='SVXY', ): us_dt = datetime.datetime.now(tz=pytz.timezone('US/Eastern')) now = datetime.datetime(us_dt.year, us_dt.month, us_dt.day, us_dt.hour, us_dt.minute, us_dt.second) integrity_p = True latest_date = TradeTime.get_latest_trade_date() start_time = datetime.datetime(latest_date.year, latest_date.month, latest_date.day, 9, 30, 0) if TradeTime.is_half_trade_day(latest_date): default_end_time = datetime.datetime(latest_date.year, latest_date.month, latest_date.day, 13, 0, 0) else: default_end_time = datetime.datetime(latest_date.year, latest_date.month, latest_date.day, 16, 0, 0) end_time = min(now, default_end_time) minutes_count = range((end_time - start_time).seconds / 60 + 1) trade_minutes = map(lambda x: start_time + datetime.timedelta(minutes=x), minutes_count) # print trade_minutes rows = self.get_min_time_and_price(symbol, start_time, end_time) # print rows j = 0 # self.logger.info('rows = %s, \n trade_minutes = %s' %(rows[-2:], trade_minutes[-2:])) for i, time in enumerate(trade_minutes): if j >= len(rows): break # rows length may less than trade_minutes for 1 elements. if rows[j][0].minute > time.minute or rows[j][0].hour > time.hour: integrity_p = False else: j = j + 1 return integrity_p, rows
def _generate_hitorical_min(self, symbol, window, current_date_time): datetimes = TradeTime.generate_trade_datetimes_by_window(window, current_date_time) start = datetimes[0] end = datetimes[-1] all_datetimes = TradeTime.generate_datetimes(start.date(), end.date()) prices = MontCarloSimulator.simulate_min(symbol, start, end, len(all_datetimes), 1) rows = map(lambda x, y: [x, y], datetimes, prices[0]) return rows
def is_month_start_date(date): if not TradeTime.is_trade_day(date): return False # if find any date before this day is trade day, return False. for i in range(date.day-1): date_i = date - timedelta(i + 1) if TradeTime.is_trade_day(date_i): return False return True
def _generate_historical_daily(self, symbol, window, current_date): dates = TradeTime.generate_trade_dates_by_window(window + 1, current_date) start_date = dates[0] # end_date = current_date end_date = TradeTime.get_latest_trade_date() all_dates = TradeTime.generate_dates(start_date, TradeTime.get_latest_trade_date()) prices = MontCarloSimulator.simulate_daily(symbol, start_date, end_date, len(all_dates), 1) rows = map(lambda x, y: [x, y], all_dates, prices[0]) return rows
def is_week_end_date(date): if not TradeTime.is_trade_day(date): return False left_days = 4 - date.weekday() # if find any date after this day is trade day, return False. for i in range(left_days): date_i = date + timedelta(i + 1) if TradeTime.is_trade_day(date_i): return False return True
def is_week_start_date(date): if not TradeTime.is_trade_day(date): return False weekday = date.weekday() # if find any date before this day is trade day, return False. for i in range(weekday): date_i = date - timedelta(i+1) if TradeTime.is_trade_day(date_i): return False return True
def validate(self, dt): if TradeTime.is_trade_day(dt.date()): if TradeTime.is_half_trade_day(dt.date()) and self.hours * 60 + self.minutes >= 3*60 + 30: return False else: passed_minutes = dt.hour * 60 + dt.minute - (9*60 + 30) if passed_minutes == self.hours * 60 + self.minutes: return True else: return False return False
def is_month_end_date(date): if not TradeTime.is_trade_day(date): return False next_month = date.replace(day=28) + timedelta(days=4) last_date = next_month - timedelta(days=next_month.day) left_days = last_date.day - date.day # if find any date after this day is trade day, return False. for i in range(left_days): date_i = date + timedelta(i + 1) if TradeTime.is_trade_day(date_i): return False return True
def get_date_values(self): expiration_date = self.get_expiration_date() latest_date = min(TradeTime.get_latest_trade_date(), expiration_date) days = (latest_date - self.start_date).days for i in range(days + 1): date = self.start_date + datetime.timedelta(days=i) if TradeTime.is_trade_day(date): value = DataProvider.get_price_by_date(self.option_symbol, date) #print 'date=%s,value=%s'%(date,value) if value is not None: if self.long_short.lower() == 'short': value = -value yield [date, self.quantity * value]
def filter_liquidity_symbols(self, current_date=None, window=30, count=50, ignore_symbols=['BIL', 'IEF', 'XIV']): if current_date is None: current_date = TradeTime.get_latest_trade_date() from_date = TradeTime.get_from_date_by_window(window, current_date) ignore_symbols_sql = ','.join( map(lambda x: '\'%s\'' % x, ignore_symbols)) sql_template = """SELECT symbol, avg(adjClosePrice * volume) as liquidity FROM tradehero.yahoo_equity where tradeDate > '{}' and tradeDate <='{}' and symbol not like '^%' and symbol not like '%.SS' and symbol not in ({}) group by symbol order by liquidity desc;""" sql = sql_template.format(from_date, current_date, ignore_symbols_sql) rows = self.select(sql) return map(lambda x: x[0], rows[:count])
def get_vix_price_by_symbol_and_date(self, symbol, from_date=datetime.datetime( 1993, 1, 1), to_date=None, remove_invalid_date=True): to_date = to_date or TradeTime.get_latest_trade_date() query_template = """select dailyDate1dAgo, dailyLastPrice from vix where symbol = '{}' and dailyDate1dAgo >= str_to_date('{}', '%Y-%m-%d') and dailyDate1dAgo <= str_to_date('{}', '%Y-%m-%d') order by dailyDate1dAgo""" query = BaseDAO.mysql_format(query_template, symbol, from_date.strftime('%Y-%m-%d'), to_date.strftime('%Y-%m-%d')) rows = self.select(query) if remove_invalid_date: rows = filter(lambda x: TradeTime.is_trade_day(x[0]), rows) return rows
def history_min(self, symbol, window): url_template = 'http://ondemand.websol.barchart.com/getHistory.json?apikey={}&symbol={}.BZ&type=formTMinutes&startDate={}00' days_window = window / 391 + 2 from_date = TradeTime.get_from_date_by_window(days_window) start_time = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0) url = url_template.format(self.api_key, symbol, start_time.strftime('%Y%m%d%M%S')) content = HttpHelper.http_get(url) data = json.loads(content) if data['status']['code'] != 200: raise Exception('http response unexcepted, the the content is: %s' % content) else: rows = map(lambda x: [datetime.datetime.strptime(x['timestamp'][:-6], '%Y-%m-%dT%H:%M:%S'), x['close']], data['results']) rows = filter(lambda x: TradeTime.is_valid_trade_time(x[0]), rows) rows.sort(key=lambda x: x[0]) return rows[-window:]
def add_missing_data_in_real_time( self, symbol='SVXY', ): us_dt = datetime.datetime.now(tz=pytz.timezone('US/Eastern')) now = datetime.datetime(us_dt.year, us_dt.month, us_dt.day, us_dt.hour, us_dt.minute, us_dt.second) if TradeTime.is_trade_day(now.date()): default_start_time = datetime.datetime(now.year, now.month, now.day, 9, 30, 0) start_time = max( default_start_time, datetime.datetime(now.year, now.month, now.day, now.hour - 1, now.minute, 0)) if now > start_time: if TradeTime.is_half_trade_day(now.date()): default_end_time = datetime.datetime( now.year, now.month, now.day, 13, 0, 0) else: default_end_time = datetime.datetime( now.year, now.month, now.day, 16, 0, 0) end_time = min(now, default_end_time) if end_time > start_time: minutes_count = range((end_time - start_time).seconds / 60 + 1) trade_minutes = map( lambda x: start_time + datetime.timedelta(minutes=x), minutes_count) # print trade_minutes rows = self.get_time_and_price(symbol, start_time, end_time) # print rows j = 0 missing_records = [] for i, time in enumerate(trade_minutes): if rows[j][0] > time: if j > 0: price = rows[j - 1][1] else: price = rows[0][1] missing_records.append( Equity(symbol, time, price, price, price, price, 0, 0)) else: j = j + 1 if len(missing_records) > 0: self.insert(missing_records) return len(missing_records)
def parse_for_option(bigcharts_option_symbol, underlying_symbol, last_price): if last_price == '': return None content = bigcharts_option_symbol[len(underlying_symbol):] option_type, month = BigChartsScraper.find_type_and_month(content[0]) date = int(content[1:3]) year = int(content[3:5]) expiration_date = datetime.date(year + 2000, month, date) strike_price = int(content[6:]) / pow(10, int(content[5])) option_symbol = '%s%s%s%08d' % (underlying_symbol, expiration_date.strftime('%y%m%d'), option_type[0], strike_price * 1000) option = Option() option.optionType = option_type option.tradeTime = TradeTime.get_latest_trade_date() option.symbol = option_symbol option.expirationDate = expiration_date if '^' + underlying_symbol in Symbols.indexes: option.underlingSymbol = '^' + underlying_symbol else: option.underlingSymbol = underlying_symbol option.strikePrice = strike_price option.daysToExpiration = (expiration_date - option.tradeTime).days option.lastPrice = float(last_price) return option
def get_daily_price(self, symbol): # return self.history(symbol, window=1)[0] date = self.specified_date_time.date() if symbol in self.provider.cache.keys() and date >= self.provider.cache[symbol][0]: dic = self.provider.cache[symbol][1] for i in range(15): if date not in dic: date = date - datetime.timedelta(days=1) else: return dic[date] else: window = len(TradeTime.generate_dates(date, TradeTime.get_latest_trade_date())) rows = self.provider.history(symbol, 'price', window, TradeTime.get_latest_trade_date()) dic = list_to_hash(rows) self.provider.cache[symbol] = [rows[0][0], dic] return dic[date]
def get_vix_price_by_symbol(self, symbol, remove_invalid_date=True): query_template = """select dailyDate1dAgo, dailyLastPrice from vix where symbol = '{}' order by dailyDate1dAgo""" query = BaseDAO.mysql_format(query_template, symbol) rows = self.select(query) if remove_invalid_date: rows = filter(lambda x: TradeTime.is_trade_day(x[0]), rows) return rows
def __init__(self): self.circle = 20 self.option_dao = OptionDAO() # get the equity records from 100 date ago. #from_date_str = (datetime.date.today() - datetime.timedelta(100)).strftime('%Y-%m-%d') from_date = (datetime.date.today() - datetime.timedelta(100)) #self.spy_records = YahooEquityDAO().get_all_equity_price_by_symbol('SPY', from_date_str) self.spy_records = EquityDAO().get_all_equity_price_by_symbol( 'SPY', from_date) self.hv_spy = OptionCalculater.get_year_history_volatility_list( self.spy_records, self.circle) self.spy_delta_records = self.get_delta_records( 'SPY', self.spy_records) from_date = TradeTime.get_latest_trade_date() - datetime.timedelta(50) self.vix_index_records = VIXDAO().get_vix_price_by_symbol_and_date( 'VIY00', from_date=from_date) (records_f1, records_f2, records_f3) = VIXDAO().get_following_vix(from_date) self.vixf1_records = records_f1 self.vix_delta_records = map(lambda x, y: [x[0], y[1] - x[1]], self.vix_index_records, self.vixf1_records) self.hv_vix = list(self.calculate_f1_volatilities()) #vxx_records = YahooEquityDAO().get_all_equity_price_by_symbol('VXX', from_date_str) vxx_records = EquityDAO().get_all_equity_price_by_symbol( 'VXX', from_date) self.vxx_delta_records = self.get_delta_records('VXX', vxx_records)
def get3vix(self, date_str=None): date_str = date_str or TradeTime.get_latest_trade_date().strftime( '%Y-%m-%d') following_symbols = list(VIX.get_following_symbols(date_str)) symbols = ['VIY00'] symbols.extend(following_symbols[0:3]) return map(lambda x: self.get_vix_price_by_symbol(x), symbols)
def GET(self): low_rate = 1.25 * np.sqrt(21)/100/np.sqrt(252) high_rate = 1.75 * np.sqrt(21)/100/np.sqrt(252) spy_low = map(lambda x, y: x * (1 + low_rate*y), self.spy_values, self.vix_values) spy_high = map(lambda x, y: x * (1 + high_rate*y), self.spy_values, self.vix_values) append_dates = TradeTime.generate_dates(self.dates[-1], self.dates[-1] + datetime.timedelta(days=50)) shifted_dates = self.dates[21:] + (append_dates[0:21]) append_spy_values = [self.spy_values[-1]] * 21 shifted_spy_values = self.spy_values[21:] + append_spy_values fig = Figure(figsize=[60, 10]) ax = fig.add_axes([.1, .1, .8, .8]) ax.plot(shifted_dates, shifted_spy_values, label='spy', color='black') ax.plot(shifted_dates, spy_low, label='spy low', color='blue') # line1.set_dashes([2, 4]) ax.plot(shifted_dates, spy_high, label='spy high', color='skyblue') # line2.set_dashes([8, 4, 2, 4, 2, 4]) # autodates = AutoDateLocator() # ax.xaxis.set_major_locator(autodates) yearsFmt = DateFormatter('%Y-%m-%d') ax.xaxis.set_major_formatter(yearsFmt) ax.set_xticks(shifted_dates) # ax.xticks(shifted_dates, rotation=90) for tick in ax.get_xticklabels(): tick.set_rotation(45) ax.legend(loc='upper left') ax.grid() canvas = FigureCanvasAgg(fig) buf = cStringIO.StringIO() canvas.print_png(buf) data = buf.getvalue() return data
def get_all_unexpired_dates(self, equity_symbol, from_date=None, cursor = None): from_date = from_date or TradeTime.get_latest_trade_date() query_template = """select distinct(expirationDate) from option_data where underlingSymbol = '{}' and expirationDate > str_to_date('{}', '%Y-%m-%d') order by expirationDate""" query = query_template.format(equity_symbol, from_date.strftime('%Y-%m-%d')) rows = self.select(query, cursor) return map(lambda x: x[0], rows)
def get_datetime(self, date): if TradeTime.is_half_trade_day(date): passed_minutes = 13 * 60 - self.hours * 60 - self.minutes else: passed_minutes = 16 * 60 - self.hours * 60 - self.minutes return datetime.datetime(date.year, date.month, date.day, int(math.floor(passed_minutes / 60)), passed_minutes % 60, 0)
def history_min(self, symbol, window, current_date_time): # us_dt = datetime.datetime.now(tz=pytz.timezone('US/Eastern')) # end_time = datetime.datetime(us_dt.year, us_dt.month, us_dt.day, us_dt.hour, us_dt.minute, us_dt.second) end_time = current_date_time days_window = window/391 + 2 from_date = TradeTime.get_from_date_by_window(days_window, current_date_time.date()) start_time = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0) rows = YahooEquityDAO().get_min_time_and_price(symbol, start_time, end_time) return rows[-window:]
def history_min(self, symbol, window): # yahoo_symbol = Symbols.get_mapped_symbol(symbol, Symbols.YahooSymbolMapping) us_dt = datetime.datetime.now(tz=pytz.timezone('US/Eastern')) end_time = datetime.datetime(us_dt.year, us_dt.month, us_dt.day, us_dt.hour, us_dt.minute, us_dt.second) days_window = window/391 + 2 from_date = TradeTime.get_from_date_by_window(days_window) start_time = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0) rows = YahooEquityDAO().get_min_time_and_price(symbol, start_time, end_time) return rows[-window:]
def GET(self): records = YahooEquityDAO().get_start_end_date_by_symbols() last_trade_date = TradeTime.get_latest_trade_date() error = False for record in records: if record[2] < last_trade_date: error = True dic = Symbols.get_yahoo_mapping_dic() return render.start_end_date(records, last_trade_date, error, dic)
def get_following_expirationDate(self, equity_symbol, from_date=None): from_date = from_date or TradeTime.get_latest_trade_date() dates = self.get_all_unexpired_dates(equity_symbol, from_date) if all_number_p(equity_symbol): return dates[0] else: for d in dates: if d.weekday() == 4 and 14 < d.day < 22: return d
def run(self, current_time): dt = convert_to_us_east_dt(current_time) if TradeTime.is_half_trade_day(dt.date()) and self.half_days is False: return # if you want to run the schedule function immediately, # you would need to add below 2 line code, given the correct time interval. # dt += datetime.timedelta(hours=12, minutes=33) # print dt if self.date_rule.validate(dt) and self.time_rule.validate(dt): self.my_func()
def validate(self, dt): if TradeTime.is_trade_day(dt.date()): if TradeTime.is_half_trade_day(dt.date()): if self.hours * 60 + self.minutes >= 3.5 * 60: return False else: left_minutes = 13 * 60 - (dt.hour * 60 + dt.minute) if left_minutes == self.hours * 60 + self.minutes: return True else: return False else: left_minutes = 16 * 60 - (dt.hour * 60 + dt.minute) if left_minutes == self.hours * 60 + self.minutes: return True else: return False return False