def gen_all(self):
for symbol in Symbols.get_option_symbols():
self.logger.info('generate data for %s ...'%symbol)
self.gen_all_for_symbol(symbol)
self.logger.info('generate VIX data...')
self.gen_vix_data()
self.logger.info('Data ingestion completed...')
def load_all(self):
for symbol in Symbols.get_option_symbols():
equity = self.load_equity_data_by_symbol(symbol)
self.equity_records.append(equity)
option_list = list(
self.load_option_data_by_symbol(symbol, equity.tradeTime))
self.option_records.extend(option_list)
self.vix_records = list(self.load_vix_data_by())
def ingest_all_options(symbols=Symbols.get_option_symbols()):
logger = Logger(__name__, PathMgr.get_log_path())
for symbol in symbols:
logger.info('ingest option data for %s...' % symbol)
date_values = YahooScraper.get_option_expirations(symbol)
for date_value in date_values:
path = PathMgr.get_yahoo_option_path(symbol, date_value)
content = YahooScraper.ingest_option(symbol, date_value)
write_to_file(path, content)
time.sleep(1)
logger.info('ingest option data completed..')
def get_missing_option_data(self):
for symbol in Symbols.get_option_symbols():
expiration_file_path = os.path.join(self.expiration_date_dir, '{}.json'.format(symbol))
with open(expiration_file_path) as fs:
json_data = json.load(fs)
try:
expirations = json_data['meta']['expirations']
except Exception:
expirations = []
for expiration in expirations:
option_file_path = os.path.join(self.option_data_dir, '{}{}.json'.format(symbol, expiration))
if not os.path.exists(option_file_path):
yield [symbol, expiration]
def load_all(self):
for symbol in Symbols.get_option_symbols():
self.parse_option_data_by_symbol(symbol)
vix.openPrice = row['openPrice']
vix.highPrice = row['highPrice']
vix.lowPrice = row['lowPrice']
vix.previousPrice = row['previousPrice']
vix.volume = row['volume']
vix.tradeTime = row['tradeTime']
vix.dailyLastPrice = row['dailyLastPrice']
vix.dailyPriceChange = row['dailyPriceChange']
vix.dailyOpenPrice = row['dailyOpenPrice']
vix.dailyHighPrice = row['dailyHighPrice']
vix.dailyLowPrice = row['dailyLowPrice']
vix.dailyPreviousPrice = row['dailyPreviousPrice']
vix.dailyVolume = row['dailyVolume']
vix.dailyDate1dAgo = row['dailyDate1dAgo']
self.vix_records.append(vix)
if __name__ == '__main__':
option_parser = H5DataOptionParser(
'/Users/tradehero/python-projects/stock-data/Barchart_Options_Data/ETF_options_data_04-20-17.h5'
)
option_parser.load_all()
print option_parser.equity_records
vix_parser = H5DataVixParser(
'/Users/tradehero/python-projects/stock-data/Barchart_VIXFutures_Data/ETF_VIXFutures_data_07-16-17.h5'
)
vix_parser.parse_vix_data()
print len(Symbols.get_option_symbols())
#print vix_parser.vix_records
def get_symbols(self):
symbols = Symbols.get_option_symbols()
symbols.append('^VIX')
symbols.append('510050')
return symbols
def get_missing_expiration_data(self):
for symbol in Symbols.get_option_symbols():
file_path = os.path.join(self.expiration_date_dir, '{}.json'.format(symbol))
if not os.path.exists(file_path):
yield symbol