def zip_raw_data(folder_name=None, logger=Logger(None, None, True)):
folder_name = folder_name or str(datetime.date.today())
folder_path = PathMgr.get_raw_data_path(folder_name)
file_path = os.path.join(PathMgr.get_raw_data_path(),
folder_name + '.zip')
logger.info('zip file form {} to {}...'.format(folder_path, file_path))
Shell.zip(folder_path, file_path, logger)
return file_path
def clean_obsoleted_data(self, hold_days=3):
self.logger.info('remove raw files {} days agao ...'.format(hold_days))
date = datetime.datetime.now() - datetime.timedelta(hold_days)
start_date = date.date().strftime('%Y-%m-%d')
data_path = PathMgr.get_raw_data_path()
sub_dir_names = get_sub_dir_names(data_path)
for dir_name in sub_dir_names:
if dir_name < start_date:
dir_path = os.path.join(data_path, dir_name)
self.logger.info(
'Remove obsoleted data on: {}'.format(dir_path))
shutil.rmtree(dir_path)
def __init__(self, raw_data_path = None):
self.date = datetime.date.today()
self.raw_data_path = raw_data_path or PathMgr.get_raw_data_path()
self.daily_path = os.path.join(self.raw_data_path, str(self.date))
self.expiration_date_dir = os.path.join(self.daily_path, 'expiration_date')
self.equity_dir = os.path.join(self.daily_path, 'equity_data')
self.option_data_dir = os.path.join(self.daily_path, 'option_data')
self.vix_data_dir = os.path.join(self.daily_path, 'vix_data')
ensure_dir_exists(self.daily_path)
ensure_dir_exists(self.expiration_date_dir)
ensure_dir_exists(self.equity_dir)
ensure_dir_exists(self.option_data_dir)
ensure_dir_exists(self.vix_data_dir)
self.logger = Logger(__name__, PathMgr.get_log_path())
def save_to_csv(self, trade_date=None):
if trade_date is None:
trade_date = TradeTime.get_latest_trade_date()
start_time = datetime.datetime(trade_date.year, trade_date.month, trade_date.day, 9, 30, 0)
end_time = datetime.datetime(trade_date.year, trade_date.month, trade_date.day, 16, 0, 0)
query = """select * from equity_realtime where tradeTime >= '{}' and tradeTime <= '{}'""".format(start_time, end_time)
rows = self.select(query)
if rows is not None and len(rows) > 0:
records = map(lambda x: ','.join(map(str, x[1:])), rows)
content = '\n'.join(records)
raw_daily_path = PathMgr.get_raw_data_path(datetime.date.today().strftime('%Y-%m-%d'))
realtime_dir = os.path.join(raw_daily_path, 'realtime')
ensure_dir_exists(realtime_dir)
file_path = os.path.join(realtime_dir, '%s.csv' % trade_date.strftime('%Y-%m-%d'))
write_to_file(file_path, content)
file_path = os.path.join(self.daily_path, 'vix_data', 'vix.json')
with open(file_path) as fs:
json_data = json.load(fs)
for record in json_data['data']:
vix = VIX.loads(record)
yield vix
def load_all(self):
for symbol in Symbols.get_option_symbols():
equity = self.load_equity_data_by_symbol(symbol)
self.equity_records.append(equity)
option_list = list(
self.load_option_data_by_symbol(symbol, equity.tradeTime))
self.option_records.extend(option_list)
self.vix_records = list(self.load_vix_data_by())
if __name__ == '__main__':
#parser = RawDataParser(PathMgr.get_data_path(str(datetime.date.today())));
parser = RawDataParser(PathMgr.get_raw_data_path('2017-11-25'))
#parser.load_equity_data_by_symbol('UNG')
#parser.load_option_data_by_symbol('UNG')
#vix_list = list(parser.load_vix_data_by_symbol())
#print len(vix_list)
#print vix_list[8].to_json()
#parser.load_all()
#print parser.vix_records
records = list(parser.load_vix_data_by())
from dataaccess.vixdao import VIXDAO
VIXDAO().insert(records)
def __init__(self, daily_raw_path = None):
if daily_raw_path is None:
daily_raw_path = PathMgr.get_raw_data_path(str(datetime.date.today()))
self.logger = Logger(__name__, PathMgr.get_log_path())
self.parser = RawDataParser(daily_raw_path)
self.parser.load_all()
from common.pathmgr import PathMgr
from dataaccess.rawdataparser import RawDataParser
from dataaccess.equitydao import EquityDAO
from dataaccess.optiondao import OptionDAO
from dataaccess.vixdao import VIXDAO
class RawToDB(object):
def __init__(self, daily_raw_path = None):
if daily_raw_path is None:
daily_raw_path = PathMgr.get_raw_data_path(str(datetime.date.today()))
self.logger = Logger(__name__, PathMgr.get_log_path())
self.parser = RawDataParser(daily_raw_path)
self.parser.load_all()
def push_to_db(self):
self.logger.info('Push equity data to db...')
EquityDAO().insert(self.parser.equity_records)
self.logger.info('Push option data to db...')
OptionDAO().insert(self.parser.option_records)
self.logger.info('Push vix data to db...')
VIXDAO().insert(self.parser.vix_records)
if __name__ == '__main__':
# RawToDB(PathMgr.get_raw_data_path()).push_to_db()
#RawToDB(PathMgr.get_raw_data_path('2017-07-25')).push_to_db()
RawToDB(PathMgr.get_raw_data_path('2018-07-03')).push_to_db()