def get_following_vix(self, from_date=None, to_date=None):
from_date = from_date or TradeTime.get_latest_trade_date(
) - datetime.timedelta(30)
to_date = to_date or TradeTime.get_latest_trade_date()
#self.logger.info('today=%s, from_date=%s, to_date=%s'%(datetime.datetime.today(), from_date, to_date))
symbols = VIX.get_vix_symbol_list(from_date, to_date, 3)
#records_index = self.get_vix_price_by_symbol('VIY00')
symbol_dic = {}
for symbol in symbols:
symbol_dic[symbol] = list_to_hash(
self.get_vix_price_by_symbol_and_date(symbol, from_date,
to_date))
days = (to_date - from_date).days + 1
records_f1 = []
records_f2 = []
records_f3 = []
for i in range(days):
date = from_date + datetime.timedelta(days=i)
if TradeTime.is_trade_day(date):
symbol_f1 = VIX.get_f1_by_date(date)
symbol_f2 = VIX.get_f2_by_date(date)
symbol_f3 = VIX.get_f3_by_date(date)
records_f1.append(
[date, symbol_dic[symbol_f1].get(date), symbol_f1])
records_f2.append(
[date, symbol_dic[symbol_f2].get(date), symbol_f2])
records_f3.append(
[date, symbol_dic[symbol_f3].get(date), symbol_f3])
# self.logger.info([records_f1[-1], records_f2[-1]], records_f3[-1]])
return (records_f1, records_f2, records_f3)
def run_schedule_only(schedule_functions, start_date, end_date, logger):
for date in TradeTime.generate_dates(start_date, end_date):
sf_dt_list = map(lambda x: [x, x.time_rule.get_datetime(date)], schedule_functions)
sf_dt_list.sort(key=lambda x: x[1])
for sf_dt in sf_dt_list:
if Container.context.terminate_p:
return
schedule_function = sf_dt[0]
dt = sf_dt[1]
if schedule_function.date_rule.validate(dt):
Container.data.set_datetime(dt)
logger.set_dt(dt)
try:
start = datetime.datetime.now()
if TradeTime.is_half_trade_day(dt.date()) and schedule_function.half_days is False:
pass
else:
schedule_function.my_func()
end = datetime.datetime.now()
# logger.info('Spend time for schedule function: %s seconds' % (end - start).seconds)
except Exception as e:
logger.error('Trace: ' + traceback.format_exc(), False)
logger.error('Error: get action arguments failed: %s, %s' % (str(e), traceback.format_exc()))
Container.data.set_datetime(datetime.datetime(date.year, date.month, date.day, 16, 0, 0))
Container.analysis.add_portfolio_trace(date, Container.api.portfolio)
def GET(self, symbol, days):
try:
days = int(days)
except Exception:
days = 63
from_date = TradeTime.get_latest_trade_date() - datetime.timedelta(days)
equity_records = EquityDAO().get_all_equity_price_by_symbol(symbol, from_date)
if datetime.date.today() > TradeTime.get_latest_trade_date():
new_spy_price = YahooScraper.get_data_by_symbol(symbol)
equity_records.append([datetime.date.today(), new_spy_price])
dates = map(lambda x: x[0], equity_records)
equity_prices = map(lambda x: x[1], equity_records)
fig = Figure(figsize=[16, 4])
ax = fig.add_axes([.1, .1, .8, .9])
ax.plot(dates, equity_prices, label='price')
ax.legend(loc='upper left')
ax.grid()
ax.xaxis.set_major_formatter(DateFormatter('%y%m%d'))
ax.set_xticks(dates)
for tick in ax.get_xticklabels():
tick.set_rotation(45)
# conver to canvas
canvas = FigureCanvasAgg(fig)
buf = cStringIO.StringIO()
canvas.print_png(buf)
data = buf.getvalue()
return data
def validate_integrity_for_real_time_data(self, symbol='SVXY', ):
us_dt = datetime.datetime.now(tz=pytz.timezone('US/Eastern'))
now = datetime.datetime(us_dt.year, us_dt.month, us_dt.day, us_dt.hour, us_dt.minute, us_dt.second)
integrity_p = True
latest_date = TradeTime.get_latest_trade_date()
start_time = datetime.datetime(latest_date.year, latest_date.month, latest_date.day, 9, 30, 0)
if TradeTime.is_half_trade_day(latest_date):
default_end_time = datetime.datetime(latest_date.year, latest_date.month, latest_date.day, 13, 0, 0)
else:
default_end_time = datetime.datetime(latest_date.year, latest_date.month, latest_date.day, 16, 0, 0)
end_time = min(now, default_end_time)
minutes_count = range((end_time - start_time).seconds / 60 + 1)
trade_minutes = map(lambda x: start_time + datetime.timedelta(minutes=x), minutes_count)
# print trade_minutes
rows = self.get_min_time_and_price(symbol, start_time, end_time)
# print rows
j = 0
# self.logger.info('rows = %s, \n trade_minutes = %s' %(rows[-2:], trade_minutes[-2:]))
for i, time in enumerate(trade_minutes):
if j >= len(rows):
break # rows length may less than trade_minutes for 1 elements.
if rows[j][0].minute > time.minute or rows[j][0].hour > time.hour:
integrity_p = False
else:
j = j + 1
return integrity_p, rows
def _generate_hitorical_min(self, symbol, window, current_date_time):
datetimes = TradeTime.generate_trade_datetimes_by_window(window, current_date_time)
start = datetimes[0]
end = datetimes[-1]
all_datetimes = TradeTime.generate_datetimes(start.date(), end.date())
prices = MontCarloSimulator.simulate_min(symbol, start, end, len(all_datetimes), 1)
rows = map(lambda x, y: [x, y], datetimes, prices[0])
return rows
def is_month_start_date(date):
if not TradeTime.is_trade_day(date):
return False
# if find any date before this day is trade day, return False.
for i in range(date.day-1):
date_i = date - timedelta(i + 1)
if TradeTime.is_trade_day(date_i):
return False
return True
def _generate_historical_daily(self, symbol, window, current_date):
dates = TradeTime.generate_trade_dates_by_window(window + 1, current_date)
start_date = dates[0]
# end_date = current_date
end_date = TradeTime.get_latest_trade_date()
all_dates = TradeTime.generate_dates(start_date, TradeTime.get_latest_trade_date())
prices = MontCarloSimulator.simulate_daily(symbol, start_date, end_date, len(all_dates), 1)
rows = map(lambda x, y: [x, y], all_dates, prices[0])
return rows
def is_week_end_date(date):
if not TradeTime.is_trade_day(date):
return False
left_days = 4 - date.weekday()
# if find any date after this day is trade day, return False.
for i in range(left_days):
date_i = date + timedelta(i + 1)
if TradeTime.is_trade_day(date_i):
return False
return True
def is_week_start_date(date):
if not TradeTime.is_trade_day(date):
return False
weekday = date.weekday()
# if find any date before this day is trade day, return False.
for i in range(weekday):
date_i = date - timedelta(i+1)
if TradeTime.is_trade_day(date_i):
return False
return True
def validate(self, dt):
if TradeTime.is_trade_day(dt.date()):
if TradeTime.is_half_trade_day(dt.date()) and self.hours * 60 + self.minutes >= 3*60 + 30:
return False
else:
passed_minutes = dt.hour * 60 + dt.minute - (9*60 + 30)
if passed_minutes == self.hours * 60 + self.minutes:
return True
else:
return False
return False
def is_month_end_date(date):
if not TradeTime.is_trade_day(date):
return False
next_month = date.replace(day=28) + timedelta(days=4)
last_date = next_month - timedelta(days=next_month.day)
left_days = last_date.day - date.day
# if find any date after this day is trade day, return False.
for i in range(left_days):
date_i = date + timedelta(i + 1)
if TradeTime.is_trade_day(date_i):
return False
return True
def get_date_values(self):
expiration_date = self.get_expiration_date()
latest_date = min(TradeTime.get_latest_trade_date(), expiration_date)
days = (latest_date - self.start_date).days
for i in range(days + 1):
date = self.start_date + datetime.timedelta(days=i)
if TradeTime.is_trade_day(date):
value = DataProvider.get_price_by_date(self.option_symbol,
date)
#print 'date=%s,value=%s'%(date,value)
if value is not None:
if self.long_short.lower() == 'short':
value = -value
yield [date, self.quantity * value]
def filter_liquidity_symbols(self,
current_date=None,
window=30,
count=50,
ignore_symbols=['BIL', 'IEF', 'XIV']):
if current_date is None:
current_date = TradeTime.get_latest_trade_date()
from_date = TradeTime.get_from_date_by_window(window, current_date)
ignore_symbols_sql = ','.join(
map(lambda x: '\'%s\'' % x, ignore_symbols))
sql_template = """SELECT symbol, avg(adjClosePrice * volume) as liquidity FROM tradehero.yahoo_equity where tradeDate > '{}' and tradeDate <='{}' and symbol not like '^%' and symbol not like '%.SS' and symbol not in ({}) group by symbol order by liquidity desc;"""
sql = sql_template.format(from_date, current_date, ignore_symbols_sql)
rows = self.select(sql)
return map(lambda x: x[0], rows[:count])
def get_vix_price_by_symbol_and_date(self,
symbol,
from_date=datetime.datetime(
1993, 1, 1),
to_date=None,
remove_invalid_date=True):
to_date = to_date or TradeTime.get_latest_trade_date()
query_template = """select dailyDate1dAgo, dailyLastPrice from vix where symbol = '{}' and dailyDate1dAgo >= str_to_date('{}', '%Y-%m-%d') and dailyDate1dAgo <= str_to_date('{}', '%Y-%m-%d') order by dailyDate1dAgo"""
query = BaseDAO.mysql_format(query_template, symbol,
from_date.strftime('%Y-%m-%d'),
to_date.strftime('%Y-%m-%d'))
rows = self.select(query)
if remove_invalid_date:
rows = filter(lambda x: TradeTime.is_trade_day(x[0]), rows)
return rows
def history_min(self, symbol, window):
url_template = 'http://ondemand.websol.barchart.com/getHistory.json?apikey={}&symbol={}.BZ&type=formTMinutes&startDate={}00'
days_window = window / 391 + 2
from_date = TradeTime.get_from_date_by_window(days_window)
start_time = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0)
url = url_template.format(self.api_key, symbol, start_time.strftime('%Y%m%d%M%S'))
content = HttpHelper.http_get(url)
data = json.loads(content)
if data['status']['code'] != 200:
raise Exception('http response unexcepted, the the content is: %s' % content)
else:
rows = map(lambda x: [datetime.datetime.strptime(x['timestamp'][:-6], '%Y-%m-%dT%H:%M:%S'), x['close']], data['results'])
rows = filter(lambda x: TradeTime.is_valid_trade_time(x[0]), rows)
rows.sort(key=lambda x: x[0])
return rows[-window:]
def add_missing_data_in_real_time(
self,
symbol='SVXY',
):
us_dt = datetime.datetime.now(tz=pytz.timezone('US/Eastern'))
now = datetime.datetime(us_dt.year, us_dt.month, us_dt.day, us_dt.hour,
us_dt.minute, us_dt.second)
if TradeTime.is_trade_day(now.date()):
default_start_time = datetime.datetime(now.year, now.month,
now.day, 9, 30, 0)
start_time = max(
default_start_time,
datetime.datetime(now.year, now.month, now.day, now.hour - 1,
now.minute, 0))
if now > start_time:
if TradeTime.is_half_trade_day(now.date()):
default_end_time = datetime.datetime(
now.year, now.month, now.day, 13, 0, 0)
else:
default_end_time = datetime.datetime(
now.year, now.month, now.day, 16, 0, 0)
end_time = min(now, default_end_time)
if end_time > start_time:
minutes_count = range((end_time - start_time).seconds /
60 + 1)
trade_minutes = map(
lambda x: start_time + datetime.timedelta(minutes=x),
minutes_count)
# print trade_minutes
rows = self.get_time_and_price(symbol, start_time,
end_time)
# print rows
j = 0
missing_records = []
for i, time in enumerate(trade_minutes):
if rows[j][0] > time:
if j > 0:
price = rows[j - 1][1]
else:
price = rows[0][1]
missing_records.append(
Equity(symbol, time, price, price, price,
price, 0, 0))
else:
j = j + 1
if len(missing_records) > 0:
self.insert(missing_records)
return len(missing_records)
def parse_for_option(bigcharts_option_symbol, underlying_symbol,
last_price):
if last_price == '':
return None
content = bigcharts_option_symbol[len(underlying_symbol):]
option_type, month = BigChartsScraper.find_type_and_month(content[0])
date = int(content[1:3])
year = int(content[3:5])
expiration_date = datetime.date(year + 2000, month, date)
strike_price = int(content[6:]) / pow(10, int(content[5]))
option_symbol = '%s%s%s%08d' % (underlying_symbol,
expiration_date.strftime('%y%m%d'),
option_type[0], strike_price * 1000)
option = Option()
option.optionType = option_type
option.tradeTime = TradeTime.get_latest_trade_date()
option.symbol = option_symbol
option.expirationDate = expiration_date
if '^' + underlying_symbol in Symbols.indexes:
option.underlingSymbol = '^' + underlying_symbol
else:
option.underlingSymbol = underlying_symbol
option.strikePrice = strike_price
option.daysToExpiration = (expiration_date - option.tradeTime).days
option.lastPrice = float(last_price)
return option
def get_daily_price(self, symbol):
# return self.history(symbol, window=1)[0]
date = self.specified_date_time.date()
if symbol in self.provider.cache.keys() and date >= self.provider.cache[symbol][0]:
dic = self.provider.cache[symbol][1]
for i in range(15):
if date not in dic:
date = date - datetime.timedelta(days=1)
else:
return dic[date]
else:
window = len(TradeTime.generate_dates(date, TradeTime.get_latest_trade_date()))
rows = self.provider.history(symbol, 'price', window, TradeTime.get_latest_trade_date())
dic = list_to_hash(rows)
self.provider.cache[symbol] = [rows[0][0], dic]
return dic[date]
def get_vix_price_by_symbol(self, symbol, remove_invalid_date=True):
query_template = """select dailyDate1dAgo, dailyLastPrice from vix where symbol = '{}' order by dailyDate1dAgo"""
query = BaseDAO.mysql_format(query_template, symbol)
rows = self.select(query)
if remove_invalid_date:
rows = filter(lambda x: TradeTime.is_trade_day(x[0]), rows)
return rows
def __init__(self):
self.circle = 20
self.option_dao = OptionDAO()
# get the equity records from 100 date ago.
#from_date_str = (datetime.date.today() - datetime.timedelta(100)).strftime('%Y-%m-%d')
from_date = (datetime.date.today() - datetime.timedelta(100))
#self.spy_records = YahooEquityDAO().get_all_equity_price_by_symbol('SPY', from_date_str)
self.spy_records = EquityDAO().get_all_equity_price_by_symbol(
'SPY', from_date)
self.hv_spy = OptionCalculater.get_year_history_volatility_list(
self.spy_records, self.circle)
self.spy_delta_records = self.get_delta_records(
'SPY', self.spy_records)
from_date = TradeTime.get_latest_trade_date() - datetime.timedelta(50)
self.vix_index_records = VIXDAO().get_vix_price_by_symbol_and_date(
'VIY00', from_date=from_date)
(records_f1, records_f2,
records_f3) = VIXDAO().get_following_vix(from_date)
self.vixf1_records = records_f1
self.vix_delta_records = map(lambda x, y: [x[0], y[1] - x[1]],
self.vix_index_records,
self.vixf1_records)
self.hv_vix = list(self.calculate_f1_volatilities())
#vxx_records = YahooEquityDAO().get_all_equity_price_by_symbol('VXX', from_date_str)
vxx_records = EquityDAO().get_all_equity_price_by_symbol(
'VXX', from_date)
self.vxx_delta_records = self.get_delta_records('VXX', vxx_records)
def get3vix(self, date_str=None):
date_str = date_str or TradeTime.get_latest_trade_date().strftime(
'%Y-%m-%d')
following_symbols = list(VIX.get_following_symbols(date_str))
symbols = ['VIY00']
symbols.extend(following_symbols[0:3])
return map(lambda x: self.get_vix_price_by_symbol(x), symbols)
def GET(self):
low_rate = 1.25 * np.sqrt(21)/100/np.sqrt(252)
high_rate = 1.75 * np.sqrt(21)/100/np.sqrt(252)
spy_low = map(lambda x, y: x * (1 + low_rate*y), self.spy_values, self.vix_values)
spy_high = map(lambda x, y: x * (1 + high_rate*y), self.spy_values, self.vix_values)
append_dates = TradeTime.generate_dates(self.dates[-1], self.dates[-1] + datetime.timedelta(days=50))
shifted_dates = self.dates[21:] + (append_dates[0:21])
append_spy_values = [self.spy_values[-1]] * 21
shifted_spy_values = self.spy_values[21:] + append_spy_values
fig = Figure(figsize=[60, 10])
ax = fig.add_axes([.1, .1, .8, .8])
ax.plot(shifted_dates, shifted_spy_values, label='spy', color='black')
ax.plot(shifted_dates, spy_low, label='spy low', color='blue')
# line1.set_dashes([2, 4])
ax.plot(shifted_dates, spy_high, label='spy high', color='skyblue')
# line2.set_dashes([8, 4, 2, 4, 2, 4])
# autodates = AutoDateLocator()
# ax.xaxis.set_major_locator(autodates)
yearsFmt = DateFormatter('%Y-%m-%d')
ax.xaxis.set_major_formatter(yearsFmt)
ax.set_xticks(shifted_dates)
# ax.xticks(shifted_dates, rotation=90)
for tick in ax.get_xticklabels():
tick.set_rotation(45)
ax.legend(loc='upper left')
ax.grid()
canvas = FigureCanvasAgg(fig)
buf = cStringIO.StringIO()
canvas.print_png(buf)
data = buf.getvalue()
return data
def get_all_unexpired_dates(self, equity_symbol, from_date=None, cursor = None):
from_date = from_date or TradeTime.get_latest_trade_date()
query_template = """select distinct(expirationDate) from option_data
where underlingSymbol = '{}' and expirationDate > str_to_date('{}', '%Y-%m-%d')
order by expirationDate"""
query = query_template.format(equity_symbol, from_date.strftime('%Y-%m-%d'))
rows = self.select(query, cursor)
return map(lambda x: x[0], rows)
def get_datetime(self, date):
if TradeTime.is_half_trade_day(date):
passed_minutes = 13 * 60 - self.hours * 60 - self.minutes
else:
passed_minutes = 16 * 60 - self.hours * 60 - self.minutes
return datetime.datetime(date.year, date.month, date.day,
int(math.floor(passed_minutes / 60)),
passed_minutes % 60, 0)
def history_min(self, symbol, window, current_date_time):
# us_dt = datetime.datetime.now(tz=pytz.timezone('US/Eastern'))
# end_time = datetime.datetime(us_dt.year, us_dt.month, us_dt.day, us_dt.hour, us_dt.minute, us_dt.second)
end_time = current_date_time
days_window = window/391 + 2
from_date = TradeTime.get_from_date_by_window(days_window, current_date_time.date())
start_time = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0)
rows = YahooEquityDAO().get_min_time_and_price(symbol, start_time, end_time)
return rows[-window:]
def history_min(self, symbol, window):
# yahoo_symbol = Symbols.get_mapped_symbol(symbol, Symbols.YahooSymbolMapping)
us_dt = datetime.datetime.now(tz=pytz.timezone('US/Eastern'))
end_time = datetime.datetime(us_dt.year, us_dt.month, us_dt.day, us_dt.hour, us_dt.minute, us_dt.second)
days_window = window/391 + 2
from_date = TradeTime.get_from_date_by_window(days_window)
start_time = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0)
rows = YahooEquityDAO().get_min_time_and_price(symbol, start_time, end_time)
return rows[-window:]
def GET(self):
records = YahooEquityDAO().get_start_end_date_by_symbols()
last_trade_date = TradeTime.get_latest_trade_date()
error = False
for record in records:
if record[2] < last_trade_date:
error = True
dic = Symbols.get_yahoo_mapping_dic()
return render.start_end_date(records, last_trade_date, error, dic)
def get_following_expirationDate(self, equity_symbol, from_date=None):
from_date = from_date or TradeTime.get_latest_trade_date()
dates = self.get_all_unexpired_dates(equity_symbol, from_date)
if all_number_p(equity_symbol):
return dates[0]
else:
for d in dates:
if d.weekday() == 4 and 14 < d.day < 22:
return d
def run(self, current_time):
dt = convert_to_us_east_dt(current_time)
if TradeTime.is_half_trade_day(dt.date()) and self.half_days is False:
return
# if you want to run the schedule function immediately,
# you would need to add below 2 line code, given the correct time interval.
# dt += datetime.timedelta(hours=12, minutes=33)
# print dt
if self.date_rule.validate(dt) and self.time_rule.validate(dt):
self.my_func()
def validate(self, dt):
if TradeTime.is_trade_day(dt.date()):
if TradeTime.is_half_trade_day(dt.date()):
if self.hours * 60 + self.minutes >= 3.5 * 60:
return False
else:
left_minutes = 13 * 60 - (dt.hour * 60 + dt.minute)
if left_minutes == self.hours * 60 + self.minutes:
return True
else:
return False
else:
left_minutes = 16 * 60 - (dt.hour * 60 + dt.minute)
if left_minutes == self.hours * 60 + self.minutes:
return True
else:
return False
return False
