def zip_raw_data(folder_name=None, logger=Logger(None, None, True)):
folder_name = folder_name or str(datetime.date.today())
folder_path = PathMgr.get_raw_data_path(folder_name)
file_path = os.path.join(PathMgr.get_raw_data_path(),
folder_name + '.zip')
logger.info('zip file form {} to {}...'.format(folder_path, file_path))
Shell.zip(folder_path, file_path, logger)
return file_path
def get_option_content(symbol, url_template):
#url = 'http://bigcharts.marketwatch.com/quickchart/options.asp?symb={}&showAll=True'.format(symbol)
url = url_template.format(symbol)
content = HttpHelper.http_get(url)
if 'showAll' in url:
file_path = PathMgr.get_bigcharts_option_symbol_path(symbol + '2')
else:
file_path = PathMgr.get_bigcharts_option_symbol_path(symbol + '1')
write_to_file(file_path, content)
return content
def ingest_all_options(symbols=Symbols.get_option_symbols()):
logger = Logger(__name__, PathMgr.get_log_path())
for symbol in symbols:
logger.info('ingest option data for %s...' % symbol)
date_values = YahooScraper.get_option_expirations(symbol)
for date_value in date_values:
path = PathMgr.get_yahoo_option_path(symbol, date_value)
content = YahooScraper.ingest_option(symbol, date_value)
write_to_file(path, content)
time.sleep(1)
logger.info('ingest option data completed..')
def ingest_all_historical_etf(date_from = '1993-01-29', date_to=None, symbols=None):
if symbols is None:
symbols = Symbols.get_all_symbols()
date_to = date_to or datetime.date.today().strftime("%Y-%m-%d")
logger = Logger(__name__, PathMgr.get_log_path())
for symbol in symbols:
logger.info('ingest for %s...' % symbol)
path = PathMgr.get_historical_etf_path(symbol)
content = YahooScraper.download_quote2(symbol, date_from, date_to)
write_to_file(path, content)
time.sleep(1)
def __init__(self, raw_data_path = None):
self.date = datetime.date.today()
self.raw_data_path = raw_data_path or PathMgr.get_raw_data_path()
self.daily_path = os.path.join(self.raw_data_path, str(self.date))
self.expiration_date_dir = os.path.join(self.daily_path, 'expiration_date')
self.equity_dir = os.path.join(self.daily_path, 'equity_data')
self.option_data_dir = os.path.join(self.daily_path, 'option_data')
self.vix_data_dir = os.path.join(self.daily_path, 'vix_data')
ensure_dir_exists(self.daily_path)
ensure_dir_exists(self.expiration_date_dir)
ensure_dir_exists(self.equity_dir)
ensure_dir_exists(self.option_data_dir)
ensure_dir_exists(self.vix_data_dir)
self.logger = Logger(__name__, PathMgr.get_log_path())
def to_csv(self, symbol, date):
file_name = '%s%s.log' % (symbol, date.strftime('%Y%m%d'))
path = PathMgr.get_data_path('quantopian_daily_min/%s' % file_name)
content = read_file_to_string(path)
lines = content.split('\n')
filtered_lines = filter(lambda x: len(x) > 100, lines)
lines = map(lambda x: string_fetch(x, 'PRINT ', ''), filtered_lines)
close_list_str = ','.join(lines)
# print close_list_str
prices_list = map(float, close_list_str.split(','))
datetimes = TradeTime.generate_datetimes(date, date)
new_lines = map(lambda x, y: '%s,%s' % (x, y), datetimes, prices_list)
new_content = '\n'.join(new_lines)
write_path = PathMgr.get_data_path('quantopian_daily_min/%s%s.csv' %
(symbol, date.strftime('%Y%m%d')))
write_to_file(write_path, new_content)
def __init__(self):
self.logger = Logger(__name__, PathMgr.get_log_path('minutes'))
self.alpha_vantage = AlphaVantage()
self.equity_min_dao = EquityMinDAO()
self.symbols = [
'SVXY', 'SPY', 'SPX', 'VIX', 'UVXY', 'QQQ', 'QLD', 'SSO', 'TLT',
'UBT'
]
def get_vxmt_daily():
url = 'http://www.cboe.com/publish/ScheduledTask/MktData/datahouse/vxmtdailyprices.csv'
content = HttpHelper.http_get(url)
records = content.split('\r\n')[3:-1]
yahoo_records = ['Date,Open,High,Low,Close,Adj Close,Volume'] + map(CBOEScraper.to_yahoo_format, records)
yahoo_content = '\r\n'.join(yahoo_records)
path = PathMgr.get_historical_etf_path('^VXMT')
write_to_file(path, yahoo_content)
def save_portfolio_info(strategy_name):
date = datetime.datetime.now(
tz=pytz.timezone('US/Eastern')).strftime('%Y-%m-%d')
dic = PortfolioDAO.read_portfolio_info(strategy_name)
dic[date] = API().get_portfolio_info().to_dict()
file_path = PathMgr.get_strategies_portfolio_file(strategy_name)
ensure_parent_dir_exists(file_path)
write_to_file(file_path, json.dumps(dic, indent=4, sort_keys=True))
def parse_raw_data(subpath=None):
subpath = subpath or datetime.date.today().strftime('%Y-%m-%d')
dir = PathMgr.get_yahoo_option_symbol_dir('^VIX', subpath)
files = get_sub_files(dir, 'html')
options = []
for file in files:
options.extend(list(YahooOptionParser.parse_raw_file(file)))
return options
def read_portfolio_info(strategy_name):
file_path = PathMgr.get_strategies_portfolio_file(strategy_name)
if os.path.exists(file_path):
content = read_file_to_string(file_path)
dic = json.loads(content)
# for key in dic.keys():
# dic[key] = Portfolio.from_dict(dic[key])
return dic
else:
return {}
def write_trade_trace(strategy_name, asset, amount, style):
file_path = PathMgr.get_strategies_tradetrace_file(strategy_name)
time = datetime.datetime.now(
tz=pytz.timezone('US/Eastern')).strftime('%Y-%m-%d %H:%M:%S')
if style == OrderStyle.MarketOrder:
price = Data().current(asset)
else:
price = style[1]
content = ','.join(map(str, [time, asset, amount, price]))
append_to_file(file_path, '%s\r\n' % content)
def run_cmd(self, cmd_name, args=[]):
file_path = PathMgr.get_command_file_path(cmd_name)
lst = map(str, [file_path] + args)
cmd = 'python \"{}\"'.format('\" \"'.join(lst))
get_logger().info('run command: %s' % cmd)
output = Shell.run_cmd(cmd, True)
if 'errorCode=502' in output:
raise Exception(output)
get_logger().info('output: %s' % output, False)
return output
def realtimedata_to_csv():
records = \
EquityRealTimeDAO().get_min_time_and_price('SVXY',
datetime.datetime(2018, 05, 29, 0, 0, 0),
datetime.datetime(2018, 05, 30, 0, 0, 0),
)
lines = map(lambda x: '%s,%s' % (x[0], x[1]), records[1:])
content = '\n'.join(lines)
write_path = PathMgr.get_data_path(
'quantopian_daily_min/realtime_%s%s.csv' % ('SVXY', '20180529'))
write_to_file(write_path, content)
def clean_obsoleted_data(self, hold_days=3):
self.logger.info('remove raw files {} days agao ...'.format(hold_days))
date = datetime.datetime.now() - datetime.timedelta(hold_days)
start_date = date.date().strftime('%Y-%m-%d')
data_path = PathMgr.get_raw_data_path()
sub_dir_names = get_sub_dir_names(data_path)
for dir_name in sub_dir_names:
if dir_name < start_date:
dir_path = os.path.join(data_path, dir_name)
self.logger.info(
'Remove obsoleted data on: {}'.format(dir_path))
shutil.rmtree(dir_path)
def load_log(self, symbol, date):
file_name = '%s%s.log' % (symbol, date.strftime('%Y%m%d'))
path = PathMgr.get_data_path('quantopian_daily_min/%s' % file_name)
content = read_file_to_string(path)
lines = content.split('\n')
filtered_lines = filter(lambda x: len(x) > 100, lines)
lines = map(lambda x: string_fetch(x, 'PRINT ', ''), filtered_lines)
close_list_str = ','.join(lines)
prices_list = map(float, close_list_str.split(','))
datetimes = TradeTime.generate_datetimes(date, date)
equities = map(lambda x, y: Equity(symbol, x, y, y, y, y), datetimes,
prices_list)
EquityMinDAO().insert(equities)
def get_data_by_symbol(symbol):
logger = Logger(__name__, PathMgr.get_log_path())
yahoo_symbol = Symbols.get_mapped_symbol(symbol)
url = 'https://finance.yahoo.com/quote/%s/' % yahoo_symbol
logger.info('Http request to: %s' % url, False)
content = HttpHelper.http_get(url)
try:
sub_content = string_fetch(content, 'Currency in USD', 'At close:')
sub_content = string_fetch(sub_content, 'react-text', 'react-text')
value = string_fetch(sub_content, '-->', '<!--')
return float(value.replace(',', ''))
except Exception:
sub_content = string_fetch(content, '\"close\":', ',')
value = round(float(sub_content), 2)
return value
def save_to_csv(self, trade_date=None):
if trade_date is None:
trade_date = TradeTime.get_latest_trade_date()
start_time = datetime.datetime(trade_date.year, trade_date.month, trade_date.day, 9, 30, 0)
end_time = datetime.datetime(trade_date.year, trade_date.month, trade_date.day, 16, 0, 0)
query = """select * from equity_realtime where tradeTime >= '{}' and tradeTime <= '{}'""".format(start_time, end_time)
rows = self.select(query)
if rows is not None and len(rows) > 0:
records = map(lambda x: ','.join(map(str, x[1:])), rows)
content = '\n'.join(records)
raw_daily_path = PathMgr.get_raw_data_path(datetime.date.today().strftime('%Y-%m-%d'))
realtime_dir = os.path.join(raw_daily_path, 'realtime')
ensure_dir_exists(realtime_dir)
file_path = os.path.join(realtime_dir, '%s.csv' % trade_date.strftime('%Y-%m-%d'))
write_to_file(file_path, content)
def __init__(self, strategy_name):
self.name = strategy_name
self.file_path = PathMgr.get_strategies_config_file(strategy_name)
def __init__(self, daily_raw_path = None):
if daily_raw_path is None:
daily_raw_path = PathMgr.get_raw_data_path(str(datetime.date.today()))
self.logger = Logger(__name__, PathMgr.get_log_path())
self.parser = RawDataParser(daily_raw_path)
self.parser.load_all()
from common.pathmgr import PathMgr
from dataaccess.rawdataparser import RawDataParser
from dataaccess.equitydao import EquityDAO
from dataaccess.optiondao import OptionDAO
from dataaccess.vixdao import VIXDAO
class RawToDB(object):
def __init__(self, daily_raw_path = None):
if daily_raw_path is None:
daily_raw_path = PathMgr.get_raw_data_path(str(datetime.date.today()))
self.logger = Logger(__name__, PathMgr.get_log_path())
self.parser = RawDataParser(daily_raw_path)
self.parser.load_all()
def push_to_db(self):
self.logger.info('Push equity data to db...')
EquityDAO().insert(self.parser.equity_records)
self.logger.info('Push option data to db...')
OptionDAO().insert(self.parser.option_records)
self.logger.info('Push vix data to db...')
VIXDAO().insert(self.parser.vix_records)
if __name__ == '__main__':
# RawToDB(PathMgr.get_raw_data_path()).push_to_db()
#RawToDB(PathMgr.get_raw_data_path('2017-07-25')).push_to_db()
RawToDB(PathMgr.get_raw_data_path('2018-07-03')).push_to_db()
from ingestion.dailyingestor import DailyIngestor
from ingestion.yahooscraper import YahooScraper
from ingestion.nyseingestor import NYSEIngestor
from ingestion.bigchartsingestor import BigChartsScraper
from dataaccess.rawfilemgr import RawFileMgr
from dataaccess.raw2db import RawToDB
from dataaccess.equitymindao import EquityMinDAO
from dataaccess.equityrealtimedao import EquityRealTimeDAO
from dataaccess.yahooequitydao import YahooEquityDAO
from dataaccess.nysecreditdao import NYSECreditDAO
from dataaccess.yahoooptionparser import YahooOptionParser
from aggregation.agg_spyvixhedge import AGGSPYVIXHedge
from processman import ProcessMan
from validation import Validator
logger = Logger(__name__, PathMgr.get_log_path())
def clean_obsoleted_data():
RawFileMgr().clean_obsoleted_data()
def process_for_ingesting_barchart_data():
logger.info('daily ingestion...')
daily_ingestor = DailyIngestor()
daily_ingestor.gen_all()
if daily_ingestor.validate():
RawToDB().push_to_db()
#exporter = DataExporter()
#exporter.export_skew()
#exporter.export_vix()
file_path = os.path.join(self.daily_path, 'vix_data', 'vix.json')
with open(file_path) as fs:
json_data = json.load(fs)
for record in json_data['data']:
vix = VIX.loads(record)
yield vix
def load_all(self):
for symbol in Symbols.get_option_symbols():
equity = self.load_equity_data_by_symbol(symbol)
self.equity_records.append(equity)
option_list = list(
self.load_option_data_by_symbol(symbol, equity.tradeTime))
self.option_records.extend(option_list)
self.vix_records = list(self.load_vix_data_by())
if __name__ == '__main__':
#parser = RawDataParser(PathMgr.get_data_path(str(datetime.date.today())));
parser = RawDataParser(PathMgr.get_raw_data_path('2017-11-25'))
#parser.load_equity_data_by_symbol('UNG')
#parser.load_option_data_by_symbol('UNG')
#vix_list = list(parser.load_vix_data_by_symbol())
#print len(vix_list)
#print vix_list[8].to_json()
#parser.load_all()
#print parser.vix_records
records = list(parser.load_vix_data_by())
from dataaccess.vixdao import VIXDAO
VIXDAO().insert(records)
def get_logger():
return DailyLoggerFactory.get_logger(__name__, PathMgr.get_log_path())
def __init__(self):
self.logger = Logger(__name__, PathMgr.get_log_path())
def get_logger(strategy_name):
return DailyBackTestLoggerFactory.get_logger(
PathMgr.get_log_path('BackTest/%s' % strategy_name))
def notify(subject, log_file = None):
if log_file is None:
log_file = PathMgr.get_log_path('%s.log'%str(datetime.date.today()))
mail_config = ConfigMgr.get_mail_config()
sendmail(mail_config['smtp_server'], mail_config['sender'], mail_config['sender_password'], mail_config['receiver'], subject, log_file)
def get_order_id():
order_file_path = PathMgr.get_data_file_path('orderid.txt')
order_id = int(read_file_to_string(order_file_path))
write_to_file(order_file_path, str(order_id + 1))
return order_id
def __init__(self):
# BaseDAO.__init__(self)
# super(BaseDAO, self).__init__()
self.logger = LoggerFactory.create_daily_logger(
__name__, PathMgr.get_log_path())
def __init__(self, apikey='JW72YXW7G33OWE5S'):
self.logger = Logger(__name__, PathMgr.get_log_path('minutes'))
self.apikey = apikey