def load_data_from_file(instruments_id=None, granularities=None):
files_name = list()
if instruments_id is not None and granularities is not None:
for instrument_id in instruments_id.split(','):
for granularity in granularities.split(','):
file_name = '_'.join(
[instrument_id,
(60 * int(granularity)).__str__()]) + '.json'
if os.path.isfile(
os.path.join(config['data_source_dir'], file_name)):
files_name.append(file_name)
else:
for file_name in os.listdir(config['data_source_dir']):
files_name.append(file_name)
for file_name in files_name:
p = instrument_id_interval_pattern.match(file_name)
if p is not None:
fields = p.groups()
if fields[0] not in DEPOSITARY_OF_KLINE:
DEPOSITARY_OF_KLINE[fields[0]] = dict()
if fields[1] not in DEPOSITARY_OF_KLINE[fields[0]]:
DEPOSITARY_OF_KLINE[fields[0]][fields[1]] = {
'path': os.path.join(config['data_source_dir'], file_name),
'data': list(),
'MA': dict(),
'MAC': dict()
}
for step in config['ma_steps']:
str_step = step.__str__()
if str_step not in DEPOSITARY_OF_KLINE[fields[0]][
fields[1]]['MA']:
DEPOSITARY_OF_KLINE[fields[0]][
fields[1]]['MA'][str_step] = dict()
DEPOSITARY_OF_KLINE[fields[0]][
fields[1]]['MA'][str_step]['pump'] = MAPump(
step=step)
DEPOSITARY_OF_KLINE[fields[0]][
fields[1]]['MA'][str_step]['data'] = list()
for mac in config['macs']:
if mac not in DEPOSITARY_OF_KLINE[fields[0]][
fields[1]]['MAC']:
DEPOSITARY_OF_KLINE[fields[0]][
fields[1]]['MAC'][mac] = dict()
DEPOSITARY_OF_KLINE[fields[0]][
fields[1]]['MAC'][mac]['data'] = list()
for k, v in list(DEPOSITARY_OF_KLINE.items()):
for _k, _v in list(v.items()):
with open(_v['path'], 'r') as f:
for line in f:
json_k_line = json.loads(line.strip())
DEPOSITARY_OF_KLINE[k][_k]['data'].append(json_k_line)
for ma_k, ma_v in list(_v['MA'].items()):
ma_ret = DEPOSITARY_OF_KLINE[k][_k]['MA'][ma_k][
'pump'].process_data(json_k_line)
DEPOSITARY_OF_KLINE[k][_k]['MA'][ma_k]['data'].append(
ma_ret)
for mac_k, mac_v in list(_v['MAC'].items()):
mac = mac_k.lower().split('c')
data = {
'date_time':
DEPOSITARY_OF_KLINE[k][_k]['MA'][
mac[0]]['data'][-1]['date_time'],
'up_crossing':
None,
'a':
DEPOSITARY_OF_KLINE[k][_k]['MA'][
mac[0]]['data'][-1]['avg'],
'b':
DEPOSITARY_OF_KLINE[k][_k]['MA'][mac[1]]['data']
[-1]['avg']
}
last_mac = {'up_crossing': data['a'] >= data['b']}
if DEPOSITARY_OF_KLINE[k][_k]['MAC'][mac_k][
'data'].__len__() > 0:
last_mac = DEPOSITARY_OF_KLINE[k][_k]['MAC'][
mac_k]['data'][-1]
if data['a'] > data['b']:
data['up_crossing'] = True
elif data['a'] < data['b']:
data['up_crossing'] = False
else:
data['up_crossing'] = None
if DEPOSITARY_OF_KLINE[k][_k]['MAC'][mac_k]['data'].__len__() > 0 and \
last_mac['up_crossing'] == data['up_crossing']:
data['up_crossing'] = None
DEPOSITARY_OF_KLINE[k][_k]['MAC'][mac_k][
'data'].append(data)
q_macs.put({
'instrument_id': k,
'granularity': _k,
'mac_k': mac_k,
'data': data
})
def sewing_data_to_file_and_depositary(depth_market_data=None):
for key in ['InstrumentID', 'LastPrice', 'ActionDay', 'UpdateTime']:
if not hasattr(depth_market_data, key):
return
instrument_id = depth_market_data.InstrumentID
contract_code = contract_code_pattern.match(instrument_id).group()
workdays = TradingPeriod.get_workdays(begin=config['begin'], end='2019-12-31')
workdays_exchange_trading_period_by_ts = \
TradingPeriod.get_workdays_exchange_trading_period(
_workdays=workdays, exchange_trading_period=EXCHANGE_TRADING_PERIOD)
date = '-'.join([depth_market_data.ActionDay[:4], depth_market_data.ActionDay[4:6],
depth_market_data.ActionDay[6:]])
date_time = ' '.join([date, depth_market_data.UpdateTime])
# if not trading_time_filter(
# date_time=date_time, contract_code=contract_code,
# exchange_trading_period_by_ts=workdays_exchange_trading_period_by_ts[date]):
# return
formatted_depth_market_data = dict()
formatted_depth_market_data['trading_day'] = date.replace('-', '')
formatted_depth_market_data['update_time'] = depth_market_data.UpdateTime
formatted_depth_market_data['instrument_id'] = instrument_id
if isinstance(depth_market_data.LastPrice, basestring):
if depth_market_data.LastPrice.isdigit():
formatted_depth_market_data['last_price'] = int(depth_market_data.LastPrice)
else:
try:
formatted_depth_market_data['last_price'] = float('%0.2f' % float(depth_market_data.LastPrice))
except ValueError:
return
else:
formatted_depth_market_data['last_price'] = depth_market_data.LastPrice
if instrument_id not in DEPOSITARY_OF_KLINE:
DEPOSITARY_OF_KLINE[instrument_id] = dict()
for granularity in config['granularities']:
interval = 60 * granularity
str_interval = str(interval)
if str_interval not in DEPOSITARY_OF_KLINE[instrument_id]:
file_name = '_'.join([instrument_id, str_interval]) + '.json'
DEPOSITARY_OF_KLINE[instrument_id][str_interval] = {
'path': os.path.join(config['data_source_dir'], file_name),
'data': list()
}
DEPOSITARY_OF_KLINE[instrument_id][str_interval]['k_line_pump'] = KLinePump()
DEPOSITARY_OF_KLINE[instrument_id][str_interval]['k_line_pump'].interval = interval
for k, v in DEPOSITARY_OF_KLINE[instrument_id].items():
DEPOSITARY_OF_KLINE[instrument_id][k]['k_line_pump'].process_data(
depth_market_data=formatted_depth_market_data, save_path=DEPOSITARY_OF_KLINE[instrument_id][k]['path'])
if DEPOSITARY_OF_KLINE[instrument_id][k]['k_line_pump'].str_k_line is not None:
json_k_line = json.loads(DEPOSITARY_OF_KLINE[instrument_id][k]['k_line_pump'].str_k_line)
DEPOSITARY_OF_KLINE[instrument_id][k]['data'].append(json_k_line)
DEPOSITARY_OF_KLINE[instrument_id][k]['k_line_pump'].str_k_line = None
if 'MA' not in DEPOSITARY_OF_KLINE[instrument_id][k]:
DEPOSITARY_OF_KLINE[instrument_id][k]['MA'] = dict()
for step in config['ma_steps']:
str_step = step.__str__()
if str_step not in DEPOSITARY_OF_KLINE[instrument_id][k]['MA']:
DEPOSITARY_OF_KLINE[instrument_id][k]['MA'][str_step] = dict()
DEPOSITARY_OF_KLINE[instrument_id][k]['MA'][str_step]['pump'] = MAPump(step=step)
DEPOSITARY_OF_KLINE[instrument_id][k]['MA'][str_step]['data'] = list()
for ma_k, ma_v in DEPOSITARY_OF_KLINE[instrument_id][k]['MA'].items():
ma_ret = DEPOSITARY_OF_KLINE[instrument_id][k]['MA'][ma_k]['pump'].process_data(json_k_line)
DEPOSITARY_OF_KLINE[instrument_id][k]['MA'][ma_k]['data'].append(ma_ret)
if 'MAC' not in DEPOSITARY_OF_KLINE[instrument_id][k]:
DEPOSITARY_OF_KLINE[instrument_id][k]['MAC'] = dict()
for mac in config['macs']:
if mac not in DEPOSITARY_OF_KLINE[instrument_id][k]['MAC']:
DEPOSITARY_OF_KLINE[instrument_id][k]['MAC'][mac] = dict()
DEPOSITARY_OF_KLINE[instrument_id][k]['MAC'][mac]['data'] = list()
for mac_k, mac_v in DEPOSITARY_OF_KLINE[instrument_id][k]['MAC'].items():
mac = mac_k.lower().split('c')
data = {
'date_time': DEPOSITARY_OF_KLINE[instrument_id][k]['MA'][mac[0]]['data'][-1]['date_time'],
'up_crossing': None,
'a': DEPOSITARY_OF_KLINE[instrument_id][k]['MA'][mac[0]]['data'][-1]['avg'],
'b': DEPOSITARY_OF_KLINE[instrument_id][k]['MA'][mac[1]]['data'][-1]['avg']
}
last_mac = {
'up_crossing': data['a'] >= data['b']
}
if DEPOSITARY_OF_KLINE[instrument_id][k]['MAC'][mac_k]['data'].__len__() > 0:
last_mac = DEPOSITARY_OF_KLINE[instrument_id][k]['MAC'][mac_k]['data'][-1]
if data['a'] > data['b']:
data['up_crossing'] = True
elif data['a'] < data['b']:
data['up_crossing'] = False
else:
data['up_crossing'] = None
if DEPOSITARY_OF_KLINE[instrument_id][k]['MAC'][mac_k]['data'].__len__() > 0 and \
last_mac['up_crossing'] == data['up_crossing']:
data['up_crossing'] = None
DEPOSITARY_OF_KLINE[instrument_id][k]['MAC'][mac_k]['data'].append(data)
q_macs.put({'instrument_id': instrument_id, 'granularity': k, 'mac_k': mac_k, 'data': data})