def __init__(self):
self.logger = Logger(__name__, PathMgr.get_log_path('minutes'))
self.alpha_vantage = AlphaVantage()
self.equity_min_dao = EquityMinDAO()
self.symbols = [
'SVXY', 'SPY', 'SPX', 'VIX', 'UVXY', 'QQQ', 'QLD', 'SSO', 'TLT',
'UBT'
]
def ingest_all_options(symbols=Symbols.get_option_symbols()):
logger = Logger(__name__, PathMgr.get_log_path())
for symbol in symbols:
logger.info('ingest option data for %s...' % symbol)
date_values = YahooScraper.get_option_expirations(symbol)
for date_value in date_values:
path = PathMgr.get_yahoo_option_path(symbol, date_value)
content = YahooScraper.ingest_option(symbol, date_value)
write_to_file(path, content)
time.sleep(1)
logger.info('ingest option data completed..')
def ingest_all_historical_etf(date_from = '1993-01-29', date_to=None, symbols=None):
if symbols is None:
symbols = Symbols.get_all_symbols()
date_to = date_to or datetime.date.today().strftime("%Y-%m-%d")
logger = Logger(__name__, PathMgr.get_log_path())
for symbol in symbols:
logger.info('ingest for %s...' % symbol)
path = PathMgr.get_historical_etf_path(symbol)
content = YahooScraper.download_quote2(symbol, date_from, date_to)
write_to_file(path, content)
time.sleep(1)
def __init__(self, raw_data_path = None):
self.date = datetime.date.today()
self.raw_data_path = raw_data_path or PathMgr.get_raw_data_path()
self.daily_path = os.path.join(self.raw_data_path, str(self.date))
self.expiration_date_dir = os.path.join(self.daily_path, 'expiration_date')
self.equity_dir = os.path.join(self.daily_path, 'equity_data')
self.option_data_dir = os.path.join(self.daily_path, 'option_data')
self.vix_data_dir = os.path.join(self.daily_path, 'vix_data')
ensure_dir_exists(self.daily_path)
ensure_dir_exists(self.expiration_date_dir)
ensure_dir_exists(self.equity_dir)
ensure_dir_exists(self.option_data_dir)
ensure_dir_exists(self.vix_data_dir)
self.logger = Logger(__name__, PathMgr.get_log_path())
def get_data_by_symbol(symbol):
logger = Logger(__name__, PathMgr.get_log_path())
yahoo_symbol = Symbols.get_mapped_symbol(symbol)
url = 'https://finance.yahoo.com/quote/%s/' % yahoo_symbol
logger.info('Http request to: %s' % url, False)
content = HttpHelper.http_get(url)
try:
sub_content = string_fetch(content, 'Currency in USD', 'At close:')
sub_content = string_fetch(sub_content, 'react-text', 'react-text')
value = string_fetch(sub_content, '-->', '<!--')
return float(value.replace(',', ''))
except Exception:
sub_content = string_fetch(content, '\"close\":', ',')
value = round(float(sub_content), 2)
return value
def get_logger():
return DailyLoggerFactory.get_logger(__name__, PathMgr.get_log_path())
def __init__(self):
self.logger = Logger(__name__, PathMgr.get_log_path())
def notify(subject, log_file = None):
if log_file is None:
log_file = PathMgr.get_log_path('%s.log'%str(datetime.date.today()))
mail_config = ConfigMgr.get_mail_config()
sendmail(mail_config['smtp_server'], mail_config['sender'], mail_config['sender_password'], mail_config['receiver'], subject, log_file)
def get_logger(strategy_name):
return DailyBackTestLoggerFactory.get_logger(
PathMgr.get_log_path('BackTest/%s' % strategy_name))
def __init__(self):
# BaseDAO.__init__(self)
# super(BaseDAO, self).__init__()
self.logger = LoggerFactory.create_daily_logger(
__name__, PathMgr.get_log_path())
def __init__(self, apikey='JW72YXW7G33OWE5S'):
self.logger = Logger(__name__, PathMgr.get_log_path('minutes'))
self.apikey = apikey
def __init__(self):
self.logger = Logger(__name__, PathMgr.get_log_path())
self.historical_data_provider = DBProvider()
def __init__(self):
self.logger = Logger(self.__class__.__name__ or __name__,
PathMgr.get_log_path())
def __init__(self, daily_raw_path = None):
if daily_raw_path is None:
daily_raw_path = PathMgr.get_raw_data_path(str(datetime.date.today()))
self.logger = Logger(__name__, PathMgr.get_log_path())
self.parser = RawDataParser(daily_raw_path)
self.parser.load_all()
def __init__(self):
self.logger = Logger(__name__, PathMgr.get_log_path('minutes'))
self.alpha_vantage = AlphaVantage()
self.equity_min_dao = EquityMinDAO()
def __init__(self, process_type, processes):
self.process_type = process_type
self.processDao = ProcessDAO()
self.processes_info = ProcessesInfo(processes, self.update_process)
self.logger = Logger(__name__, PathMgr.get_log_path())
self.start_time = None
def logger(self):
return LoggerFactory.create_daily_logger(
__name__, PathMgr.get_log_path('realtime'))
def __init__(self, daily_raw_path=None):
self.logger = Logger(__name__, PathMgr.get_log_path())
self.config = ConfigMgr.get_output_config()
self.output_dir = self.config['output_dir']
from ingestion.dailyingestor import DailyIngestor
from ingestion.yahooscraper import YahooScraper
from ingestion.nyseingestor import NYSEIngestor
from ingestion.bigchartsingestor import BigChartsScraper
from dataaccess.rawfilemgr import RawFileMgr
from dataaccess.raw2db import RawToDB
from dataaccess.equitymindao import EquityMinDAO
from dataaccess.equityrealtimedao import EquityRealTimeDAO
from dataaccess.yahooequitydao import YahooEquityDAO
from dataaccess.nysecreditdao import NYSECreditDAO
from dataaccess.yahoooptionparser import YahooOptionParser
from aggregation.agg_spyvixhedge import AGGSPYVIXHedge
from processman import ProcessMan
from validation import Validator
logger = Logger(__name__, PathMgr.get_log_path())
def clean_obsoleted_data():
RawFileMgr().clean_obsoleted_data()
def process_for_ingesting_barchart_data():
logger.info('daily ingestion...')
daily_ingestor = DailyIngestor()
daily_ingestor.gen_all()
if daily_ingestor.validate():
RawToDB().push_to_db()
#exporter = DataExporter()
#exporter.export_skew()
#exporter.export_vix()
def get_logger(strategy_name):
if strategy_name not in Container._strategies_logger_dic.keys():
logger = Logger(strategy_name, PathMgr.get_log_path(strategy_name),
True)
Container._strategies_logger_dic[strategy_name] = logger
return Container._strategies_logger_dic[strategy_name]