def gen_all(self):
for symbol in Symbols.get_option_symbols():
self.logger.info('generate data for %s ...'%symbol)
self.gen_all_for_symbol(symbol)
self.logger.info('generate VIX data...')
self.gen_vix_data()
self.logger.info('Data ingestion completed...')
def get_start_end_date_by_symbols(self):
reversed_yahoo_symbol_mapping = Symbols.get_reversed_yahoo_symbol_mapping(
)
conn = self.get_connection()
cursor = conn.cursor()
records = []
for symbol in Symbols.get_all_symbols():
start_date = self.get_start_date_by_symbol(symbol, cursor)
end_date = self.get_end_date_by_symbol(symbol, cursor)
records.append([
Symbols.get_mapped_symbol(symbol,
reversed_yahoo_symbol_mapping),
start_date, end_date
])
records.sort()
conn.close()
return records
def load_all(self):
for symbol in Symbols.get_option_symbols():
equity = self.load_equity_data_by_symbol(symbol)
self.equity_records.append(equity)
option_list = list(
self.load_option_data_by_symbol(symbol, equity.tradeTime))
self.option_records.extend(option_list)
self.vix_records = list(self.load_vix_data_by())
def GET(self):
records = YahooEquityDAO().get_start_end_date_by_symbols()
last_trade_date = TradeTime.get_latest_trade_date()
error = False
for record in records:
if record[2] < last_trade_date:
error = True
dic = Symbols.get_yahoo_mapping_dic()
return render.start_end_date(records, last_trade_date, error, dic)
def ingest_all_historical_etf(date_from = '1993-01-29', date_to=None, symbols=None):
if symbols is None:
symbols = Symbols.get_all_symbols()
date_to = date_to or datetime.date.today().strftime("%Y-%m-%d")
logger = Logger(__name__, PathMgr.get_log_path())
for symbol in symbols:
logger.info('ingest for %s...' % symbol)
path = PathMgr.get_historical_etf_path(symbol)
content = YahooScraper.download_quote2(symbol, date_from, date_to)
write_to_file(path, content)
time.sleep(1)
def ingest_all_options(symbols=Symbols.get_option_symbols()):
logger = Logger(__name__, PathMgr.get_log_path())
for symbol in symbols:
logger.info('ingest option data for %s...' % symbol)
date_values = YahooScraper.get_option_expirations(symbol)
for date_value in date_values:
path = PathMgr.get_yahoo_option_path(symbol, date_value)
content = YahooScraper.ingest_option(symbol, date_value)
write_to_file(path, content)
time.sleep(1)
logger.info('ingest option data completed..')
def get_missing_records_symbols(self):
conn = self.get_connection()
cursor = conn.cursor()
missing_symbols = []
last_trade_date = TradeTime.get_latest_trade_date()
for symbol in Symbols.get_all_symbols():
end_date = self.get_end_date_by_symbol(symbol, cursor)
if end_date < last_trade_date:
missing_symbols.append(symbol)
conn.close()
return missing_symbols
def get_current_data(self, symbol):
yahoo_symbol = Symbols.get_mapped_symbol(symbol)
url = 'https://finance.yahoo.com/quote/%s/' % yahoo_symbol
content = HttpHelper.http_get(url)
try:
sub_content = string_fetch(content, 'Currency in USD', 'At close:')
sub_content = string_fetch(sub_content, 'react-text', 'react-text')
value = string_fetch(sub_content, '-->', '<!--')
return float(value.replace(',', ''))
except Exception:
sub_content = string_fetch(content, '\"close\":', ',')
value = round(float(sub_content), 2)
return value
def get_missing_option_data(self):
for symbol in Symbols.get_option_symbols():
expiration_file_path = os.path.join(self.expiration_date_dir, '{}.json'.format(symbol))
with open(expiration_file_path) as fs:
json_data = json.load(fs)
try:
expirations = json_data['meta']['expirations']
except Exception:
expirations = []
for expiration in expirations:
option_file_path = os.path.join(self.option_data_dir, '{}{}.json'.format(symbol, expiration))
if not os.path.exists(option_file_path):
yield [symbol, expiration]
def get_data_by_symbol(symbol):
logger = Logger(__name__, PathMgr.get_log_path())
yahoo_symbol = Symbols.get_mapped_symbol(symbol)
url = 'https://finance.yahoo.com/quote/%s/' % yahoo_symbol
logger.info('Http request to: %s' % url, False)
content = HttpHelper.http_get(url)
try:
sub_content = string_fetch(content, 'Currency in USD', 'At close:')
sub_content = string_fetch(sub_content, 'react-text', 'react-text')
value = string_fetch(sub_content, '-->', '<!--')
return float(value.replace(',', ''))
except Exception:
sub_content = string_fetch(content, '\"close\":', ',')
value = round(float(sub_content), 2)
return value
def get_lack_of_liquity_symbols(self, window=100):
from dateutil.relativedelta import relativedelta
result = []
today = datetime.date.today()
current = datetime.date(2011, 1, 1)
tradable_symbols = Symbols.get_all_tradeable_symbols()
set_symbols = set(
['BIL', 'IEF', 'XIV', 'VIX', 'GLD', 'SLV', 'TLT', 'ZIV'])
while current <= today:
result.append(current)
current += relativedelta(months=1)
for date in result:
print date
symbols = self.filter_liquidity_symbols(date, window=window)
set_symbols = set_symbols.union(set(symbols))
# print 'set_symbols = %s'%set_symbols
return filter(lambda x: x not in set_symbols, tradable_symbols)
def history(self, symbol, field, window):
fields_dic = {
'open': 'openPrice',
'close': 'adjclosePrice',
'high': 'highPrice',
'low': 'lowPrice',
'price': 'adjclosePrice'
}
fields = fields_dic.keys()
if field.lower() not in field:
raise Exception('the field should be in %s...' % fields)
price_field = fields_dic[field]
yahoo_symbol = Symbols.get_mapped_symbol(symbol)
from_date = TradeTime.get_from_date_by_window(window)
rows = YahooEquityDAO().get_all_equity_price_by_symbol(
yahoo_symbol, from_date.strftime('%Y-%m-%d'), price_field)
return rows
def save_all(self, symbols=Symbols.get_all_symbols()):
for symbol in symbols:
self.logger.info('save data for %s...' % symbol)
path = PathMgr.get_historical_etf_path(symbol)
df = pd.read_csv(path)
self.save(symbol, df)
def insert_all(self):
for symbol in Symbols.get_all_symbols():
self.logger.info('insert data for %s...' % symbol)
path = PathMgr.get_historical_etf_path(symbol)
df = pd.read_csv(path)
self.insert(symbol, df)
def get_symbols_mapping():
symbols = Symbols.get_all_tradeable_symbols()
items = map(lambda x: '\'%s\':symbol(\'%s\')' % (x, x), symbols)
return '{%s}' % ','.join(items)
def load_all(self):
for symbol in Symbols.get_option_symbols():
self.parse_option_data_by_symbol(symbol)
vix.openPrice = row['openPrice']
vix.highPrice = row['highPrice']
vix.lowPrice = row['lowPrice']
vix.previousPrice = row['previousPrice']
vix.volume = row['volume']
vix.tradeTime = row['tradeTime']
vix.dailyLastPrice = row['dailyLastPrice']
vix.dailyPriceChange = row['dailyPriceChange']
vix.dailyOpenPrice = row['dailyOpenPrice']
vix.dailyHighPrice = row['dailyHighPrice']
vix.dailyLowPrice = row['dailyLowPrice']
vix.dailyPreviousPrice = row['dailyPreviousPrice']
vix.dailyVolume = row['dailyVolume']
vix.dailyDate1dAgo = row['dailyDate1dAgo']
self.vix_records.append(vix)
if __name__ == '__main__':
option_parser = H5DataOptionParser(
'/Users/tradehero/python-projects/stock-data/Barchart_Options_Data/ETF_options_data_04-20-17.h5'
)
option_parser.load_all()
print option_parser.equity_records
vix_parser = H5DataVixParser(
'/Users/tradehero/python-projects/stock-data/Barchart_VIXFutures_Data/ETF_VIXFutures_data_07-16-17.h5'
)
vix_parser.parse_vix_data()
print len(Symbols.get_option_symbols())
#print vix_parser.vix_records
def get_missing_expiration_data(self):
for symbol in Symbols.get_option_symbols():
file_path = os.path.join(self.expiration_date_dir, '{}.json'.format(symbol))
if not os.path.exists(file_path):
yield symbol
def get_symbols(self):
symbols = Symbols.get_option_symbols()
symbols.append('^VIX')
symbols.append('510050')
return symbols