def run_schedule_only(schedule_functions, start_date, end_date, logger):
for date in TradeTime.generate_dates(start_date, end_date):
sf_dt_list = map(lambda x: [x, x.time_rule.get_datetime(date)], schedule_functions)
sf_dt_list.sort(key=lambda x: x[1])
for sf_dt in sf_dt_list:
if Container.context.terminate_p:
return
schedule_function = sf_dt[0]
dt = sf_dt[1]
if schedule_function.date_rule.validate(dt):
Container.data.set_datetime(dt)
logger.set_dt(dt)
try:
start = datetime.datetime.now()
if TradeTime.is_half_trade_day(dt.date()) and schedule_function.half_days is False:
pass
else:
schedule_function.my_func()
end = datetime.datetime.now()
# logger.info('Spend time for schedule function: %s seconds' % (end - start).seconds)
except Exception as e:
logger.error('Trace: ' + traceback.format_exc(), False)
logger.error('Error: get action arguments failed: %s, %s' % (str(e), traceback.format_exc()))
Container.data.set_datetime(datetime.datetime(date.year, date.month, date.day, 16, 0, 0))
Container.analysis.add_portfolio_trace(date, Container.api.portfolio)
def GET(self):
low_rate = 1.25 * np.sqrt(21)/100/np.sqrt(252)
high_rate = 1.75 * np.sqrt(21)/100/np.sqrt(252)
spy_low = map(lambda x, y: x * (1 + low_rate*y), self.spy_values, self.vix_values)
spy_high = map(lambda x, y: x * (1 + high_rate*y), self.spy_values, self.vix_values)
append_dates = TradeTime.generate_dates(self.dates[-1], self.dates[-1] + datetime.timedelta(days=50))
shifted_dates = self.dates[21:] + (append_dates[0:21])
append_spy_values = [self.spy_values[-1]] * 21
shifted_spy_values = self.spy_values[21:] + append_spy_values
fig = Figure(figsize=[60, 10])
ax = fig.add_axes([.1, .1, .8, .8])
ax.plot(shifted_dates, shifted_spy_values, label='spy', color='black')
ax.plot(shifted_dates, spy_low, label='spy low', color='blue')
# line1.set_dashes([2, 4])
ax.plot(shifted_dates, spy_high, label='spy high', color='skyblue')
# line2.set_dashes([8, 4, 2, 4, 2, 4])
# autodates = AutoDateLocator()
# ax.xaxis.set_major_locator(autodates)
yearsFmt = DateFormatter('%Y-%m-%d')
ax.xaxis.set_major_formatter(yearsFmt)
ax.set_xticks(shifted_dates)
# ax.xticks(shifted_dates, rotation=90)
for tick in ax.get_xticklabels():
tick.set_rotation(45)
ax.legend(loc='upper left')
ax.grid()
canvas = FigureCanvasAgg(fig)
buf = cStringIO.StringIO()
canvas.print_png(buf)
data = buf.getvalue()
return data
def _generate_historical_daily(self, symbol, window, current_date):
dates = TradeTime.generate_trade_dates_by_window(window + 1, current_date)
start_date = dates[0]
# end_date = current_date
end_date = TradeTime.get_latest_trade_date()
all_dates = TradeTime.generate_dates(start_date, TradeTime.get_latest_trade_date())
prices = MontCarloSimulator.simulate_daily(symbol, start_date, end_date, len(all_dates), 1)
rows = map(lambda x, y: [x, y], all_dates, prices[0])
return rows
def get_daily_price(self, symbol):
# return self.history(symbol, window=1)[0]
date = self.specified_date_time.date()
if symbol in self.provider.cache.keys() and date >= self.provider.cache[symbol][0]:
dic = self.provider.cache[symbol][1]
for i in range(15):
if date not in dic:
date = date - datetime.timedelta(days=1)
else:
return dic[date]
else:
window = len(TradeTime.generate_dates(date, TradeTime.get_latest_trade_date()))
rows = self.provider.history(symbol, 'price', window, TradeTime.get_latest_trade_date())
dic = list_to_hash(rows)
self.provider.cache[symbol] = [rows[0][0], dic]
return dic[date]
def GET(self, from_date_str=None):
default_from_date = TradeTime.get_latest_trade_date() - datetime.timedelta(60)
if from_date_str is None or from_date_str == '':
from_date = default_from_date
else:
try:
input_from_date = datetime.datetime.strptime(from_date_str, '%Y-%m-%d').date()
from_date = TradeTime.get_from_date_by_window(22, input_from_date)
except Exception:
from_date = default_from_date
records_svxy = YahooEquityDAO().get_all_equity_price_by_symbol('SVXY', from_date.strftime('%Y-%m-%d'), 'Closeprice')
dates = map(lambda x: x[0], records_svxy)
price_svxy = map(lambda x: x[1], records_svxy)
# shift
append_dates = TradeTime.generate_dates(dates[-1], dates[-1] + datetime.timedelta(days=50))
dates = dates[21:] + append_dates[1:22]
price_svxy = price_svxy[21:] + [price_svxy[-1]] * 21
if from_date < default_from_date:
dates = dates[:42]
price_svxy = price_svxy[:42]
fig = Figure(figsize=[24, 4])
ax = fig.add_axes([.1, .1, .8, .8])
ax.plot(dates, price_svxy, label='SVXY')
ax.legend(loc='upper left')
ax.grid()
ax.xaxis.set_major_formatter(DateFormatter('%y%m%d'))
ax.set_xticks(dates)
for tick in ax.get_xticklabels():
tick.set_rotation(45)
canvas = FigureCanvasAgg(fig)
buf = cStringIO.StringIO()
canvas.print_png(buf)
data = buf.getvalue()
return data
break
if rows[j][0] > time:
if j > 0:
price = rows[j - 1][1]
else:
price = rows[0][1]
if not (time.hour == 9 and time.minute == 30):
missing_time.append(time)
else:
j = j + 1
if len(missing_time) > 0:
integrity_p = False
return integrity_p, missing_time
def remove_market_open_records(self):
sql = """delete from equity_min where tradetime like '%9:30:00'"""
self.execute_query(sql)
if __name__ == '__main__':
# EquityMinDAO().add_missing_data()
# EquityMinDAO().add_missing_data(validate_date=datetime.date(2018, 1, 19))
# EquityMinDAO().add_missing_data_in_real_time('XIV')
# print EquityMinDAO().save_to_csv()
# print EquityMinDAO().validate_integrity_for_min_data('UBT')
# for trade_date in TradeTime.generate_dates(datetime.date(2009, 1, 30), TradeTime.get_latest_trade_date()):
# print 'Add missing data for %s' % trade_date
# EquityMinDAO().add_missing_data('VXX', trade_date)
for trade_date in TradeTime.generate_dates(
datetime.date(2018, 5, 30), TradeTime.get_latest_trade_date()):
EquityMinDAO().add_missing_data('SPY', trade_date)
sub_records.append(record)
combined_record = AGG30Min.combine_records(symbol, sub_records)
combined_records.append(combined_record)
sub_records = []
Equity30MinDAO().insert(combined_records)
@staticmethod
def agg1mtodaily(symbol, start_time=datetime.datetime(1971, 1, 1, 0, 0, 0), end_time=datetime.datetime(9999, 1, 1, 0, 0, 0)):
records = EquityMinDAO().get_records(symbol, start_time, end_time)
sub_records = []
combined_records = []
for record in records:
sub_records.append(record)
if record[0].hour == 15:
sub_records.append(record)
combined_record = AGG30Min.combine_records(symbol, sub_records)
combined_records.append(combined_record)
sub_records = []
YahooEquityDAO().save_from_equities(combined_records)
if __name__ == '__main__':
# AGG30Min.agg1to30('510050')
# AGG30Min.agg1mtodaily('510050')
from common.tradetime import TradeTime
for trade_date in TradeTime.generate_dates(datetime.date(1999, 3, 10), TradeTime.get_latest_trade_date()):
from_datetime = datetime.datetime(trade_date.year, trade_date.month, trade_date.day, 0, 0, 0)
to_datetime = from_datetime + datetime.timedelta(days=1)
print 'Generate 30 min data for %s' % trade_date
AGG30Min.agg1to30('QQQ', from_datetime, to_datetime)
def GET(self, from_date_str=None):
default_from_date = TradeTime.get_latest_trade_date() - datetime.timedelta(60)
if from_date_str is None or from_date_str == '':
from_date = default_from_date
else:
try:
input_from_date = datetime.datetime.strptime(from_date_str, '%Y-%m-%d').date()
from_date = TradeTime.get_from_date_by_window(22, input_from_date)
except Exception:
from_date = default_from_date
# records_index = VIXDAO().get_vix_price_by_symbol_and_date('VIY00', from_date=from_date)
records_index = YahooEquityDAO().get_all_equity_price_by_symbol('^VIX', from_date.strftime('%Y-%m-%d'),'Closeprice')
(records_f1, records_f2, records_f3) = VIXDAO().get_following_vix(from_date)
new_spy_price = None
if datetime.date.today() > TradeTime.get_latest_trade_date():
try:
new_spy_price = YahooScraper.get_data_by_symbol('SPY')
except Exception as e:
new_spy_price = CNBCScraper.get_data_by_symbol('SPY')
try:
new_vix_price = YahooScraper.get_data_by_symbol('VIX')
except Exception as e:
new_vix_price = CNBCScraper.get_data_by_symbol('VIX')
new_vix_features = CBOEScraper.get_vix_future()
[new_f1, new_f2, new_f3] = [new_vix_features[0][2], new_vix_features[1][2], new_vix_features[2][2]]
records_index.append([datetime.date.today(), new_vix_price])
records_f1.append([datetime.date.today(), new_f1])
records_f2.append([datetime.date.today(), new_f2])
records_f3.append([datetime.date.today(), new_f3])
dates = map(lambda x: x[0], records_index)
price_index = map(lambda x: x[1], records_index)
price_f1 = map(lambda x: x[1], records_f1)
price_f2 = map(lambda x: x[1], records_f2)
price_f3 = map(lambda x: x[1], records_f3)
hv_records = self.get_historical_volatility(new_spy_price, from_date)[-len(dates):]
hv_prices = map(lambda x: x[1]*100, hv_records)
spy_prices = self.get_equity_prices('SPY', from_date, new_spy_price)
[spy_low, spy_high] = self.get_low_and_upper(spy_prices, price_index)
# shift
append_dates = TradeTime.generate_dates(dates[-1], dates[-1] + datetime.timedelta(days=50))
dates = dates[21:] + (append_dates[0:21])
hv_prices = self.shift_values(hv_prices)
spy_prices = self.shift_values(spy_prices)
price_index = self.shift_values(price_index)
price_f1 = self.shift_values(price_f1)
price_f2 = self.shift_values(price_f2)
price_f3 = self.shift_values(price_f3)
if from_date < default_from_date:
dates = dates[:42]
hv_prices = hv_prices[:42]
spy_prices = spy_prices[:42]
spy_low = spy_low[:42]
price_index = price_index[:42]
price_f1 = price_f1[:42]
price_f2 = price_f2[:42]
price_f3 = price_f3[:42]
fig = Figure(figsize=[24, 8])
ax1 = fig.add_axes([.1, .1, .8, .8])
ax2 = ax1.twinx()
ax1.plot(dates, hv_prices, label='historical volatility', color='black')
ax1.plot(dates, price_index, label='vix index', color='blue')
ax1.plot(dates, price_f1, label='vix first month', color='deepskyblue')
ax1.plot(dates, price_f2, label='vix second month', color='lightskyblue')
ax1.plot(dates, price_f3, label='vix third month', color='lightblue')
ax1.legend(loc='upper left')
ax2.plot(dates, spy_prices, 'red', label='SPY')
ax2.plot(dates, spy_low, 'orange', label='spy_low')
# ax2.plot(dates, spy_high, 'yellow', label='spy_high')
ax2.legend(loc='upper right')
ax1.grid()
ax1.xaxis.set_major_formatter(DateFormatter('%y%m%d'))
ax1.set_xticks(dates)
for tick in ax1.get_xticklabels():
tick.set_rotation(45)
canvas = FigureCanvasAgg(fig)
buf = cStringIO.StringIO()
canvas.print_png(buf)
data = buf.getvalue()
return data
