def on_time(self, timestamp):
if is_same_date(buy_timestamp, timestamp):
self.buy(due_timestamp=buy_timestamp, happen_timestamp=buy_timestamp,
entity_ids=['stock_sz_000338'])
self.has_buy = True
self.buy(due_timestamp=buy_timestamp, happen_timestamp=buy_timestamp,
entity_ids=['stock_sh_601318'])
if is_same_date(sell_timestamp, timestamp):
self.sell(due_timestamp=sell_timestamp, happen_timestamp=sell_timestamp,
entity_ids=['stock_sz_000338', 'stock_sh_601318'])
def persist(self, security_item, domain_list):
if domain_list:
first_timestamp = domain_list[0].timestamp
last_timestamp = domain_list[-1].timestamp
self.logger.info(
"recording {} for security_id:{},level:{},first_timestamp:{},last_timestamp:{}"
.format(self.data_schema, security_item.id, self.level,
first_timestamp, last_timestamp))
current_timestamp = now_pd_timestamp()
saving_datas = domain_list
# FIXME:remove this logic
# FIXME:should remove unfinished data when recording,always set it to False now
if is_same_date(current_timestamp,
last_timestamp) and self.contain_unfinished_data:
close_hour, close_minute = get_close_time(security_item.id)
if current_timestamp.hour >= close_hour and current_timestamp.minute >= close_minute + 2:
# after the closing time of the day,we think the last data is finished
saving_datas = domain_list
else:
# ignore unfinished kdata
saving_datas = domain_list[:-1]
self.logger.info(
"ignore kdata for security_id:{},level:{},timestamp:{},current_timestamp"
.format(security_item.id, self.level, last_timestamp,
current_timestamp))
self.session.add_all(saving_datas)
self.session.commit()
def run(self):
# 按不同类别抓取
# 编码 基金运作方式
# 401001 开放式基金
# 401002 封闭式基金
# 401003 QDII
# 401004 FOF
# 401005 ETF
# 401006 LOF
for operate_mode_id in (401001, 401002, 401005):
year_count = 2
while True:
latest = Fund.query_data(
filters=[Fund.operate_mode_id == operate_mode_id],
order=Fund.timestamp.desc(),
limit=1,
return_type="domain",
)
start_timestamp = "2000-01-01"
if latest:
start_timestamp = latest[0].timestamp
end_timestamp = min(
next_date(start_timestamp, 365 * year_count),
now_pd_timestamp())
df = run_query(
table="finance.FUND_MAIN_INFO",
conditions=
f"operate_mode_id#=#{operate_mode_id}&start_date#>=#{to_time_str(start_timestamp)}&start_date#<=#{to_time_str(end_timestamp)}",
parse_dates=["start_date", "end_date"],
dtype={"main_code": str},
)
if not pd_is_not_null(df) or (df["start_date"].max().year <
end_timestamp.year):
year_count = year_count + 1
if pd_is_not_null(df):
df.rename(columns={"start_date": "timestamp"},
inplace=True)
df["timestamp"] = pd.to_datetime(df["timestamp"])
df["list_date"] = df["timestamp"]
df["end_date"] = pd.to_datetime(df["end_date"])
df["code"] = df["main_code"]
df["entity_id"] = df["code"].apply(
lambda x: to_entity_id(entity_type="fund", jq_code=x))
df["id"] = df["entity_id"]
df["entity_type"] = "fund"
df["exchange"] = "sz"
df_to_db(df,
data_schema=Fund,
provider=self.provider,
force_update=self.force_update)
self.logger.info(
f"persist fund {operate_mode_id} list success {start_timestamp} to {end_timestamp}"
)
if is_same_date(end_timestamp, now_pd_timestamp()):
break
def on_trading_open(self, timestamp):
self.logger.info('on_trading_open:{}'.format(timestamp))
if is_same_date(timestamp, self.start_timestamp):
return
self.account = self.load_account()
self.logger.info('on_trading_open:{},current_account:{}'.format(
timestamp, self.account))
def run(self):
# 按不同类别抓取
# 编码 基金运作方式
# 401001 开放式基金
# 401002 封闭式基金
# 401003 QDII
# 401004 FOF
# 401005 ETF
# 401006 LOF
for operate_mode_id in (401001, 401002, 401005):
year_count = 2
while True:
latest = Fund.query_data(
region=self.region,
filters=[Fund.operate_mode_id == operate_mode_id],
order=Fund.timestamp.desc(),
limit=1,
return_type='domain')
start_timestamp = '2000-01-01'
if latest:
start_timestamp = latest[0].timestamp
end_timestamp = min(
next_date(start_timestamp, 365 * year_count),
now_pd_timestamp(self.region))
df = jq_run_query(
table='finance.FUND_MAIN_INFO',
conditions=
f'operate_mode_id#=#{operate_mode_id}&start_date#>=#{to_time_str(start_timestamp)}&start_date#<=#{to_time_str(end_timestamp)}',
parse_dates=['start_date', 'end_date'],
dtype={'main_code': str})
if not pd_is_not_null(df) or (df['start_date'].max().year <
end_timestamp.year):
year_count = year_count + 1
if pd_is_not_null(df):
df.rename(columns={'start_date': 'timestamp'},
inplace=True)
df['timestamp'] = pd.to_datetime(df['timestamp'])
df['list_date'] = df['timestamp']
df['end_date'] = pd.to_datetime(df['end_date'])
df['code'] = df['main_code']
df['entity_id'] = df['code'].apply(
lambda x: to_entity_id(entity_type='fund', jq_code=x))
df['id'] = df['entity_id']
df['entity_type'] = 'fund'
df['exchange'] = 'sz'
df_to_db(df,
ref_df=None,
region=self.region,
data_schema=Fund,
provider=self.provider)
self.logger.info(
f'persist fund {operate_mode_id} list success {start_timestamp} to {end_timestamp}'
)
if is_same_date(end_timestamp, now_pd_timestamp(self.region)):
break
def on_trading_open(self, timestamp):
self.logger.info('on_trading_open:{}'.format(timestamp))
if is_same_date(timestamp, self.start_timestamp):
return
# get the account for trading at the date
accounts = get_account(session=self.session,
trader_name=self.trader_name,
return_type='domain',
end_timestamp=to_time_str(timestamp),
limit=1,
order=SimAccount.timestamp.desc())
if accounts:
account = accounts[0]
else:
return
positions = []
# FIXME:dump all directly
for position_domain in account.positions:
position_dict = position_schema.dump(position_domain).data
self.logger.info('current position:{}'.format(position_dict))
del position_dict['sim_account']
positions.append(position_dict)
self.latest_account = sim_account_schema.dump(account).data
self.latest_account['positions'] = positions
self.logger.info('on_trading_open:{},latest_account:{}'.format(
timestamp, self.latest_account))
def test_split_time_interval():
first = None
last = None
start = '2020-01-01'
end = '2021-01-01'
for interval in split_time_interval(start, end, interval=30):
if first is None:
first = interval
last = interval
print(first)
print(last)
assert is_same_date(first[0], start)
assert is_same_date(first[-1], '2020-01-31')
assert is_same_date(last[-1], end)
def evaluate_start_end_size_timestamps(self, entity):
start, end, size, timestamps = super().evaluate_start_end_size_timestamps(entity)
if start:
trade_day = StockTradeDay.query_data(limit=1, order=StockTradeDay.timestamp.desc(), return_type='domain')
if trade_day:
if is_same_date(trade_day[0].timestamp, start):
size = 0
return start, end, size, timestamps
def test_split_time_interval_month():
first = None
last = None
start = "2020-01-01"
end = "2021-01-01"
for interval in split_time_interval(start, end, method="month"):
if first is None:
first = interval
last = interval
print(first)
print(last)
assert is_same_date(first[0], start)
assert is_same_date(first[-1], "2020-01-31")
assert is_same_date(last[0], "2021-01-01")
assert is_same_date(last[-1], "2021-01-01")
def is_in_trading(security_type, exchange, timestamp):
current = now_pd_timestamp()
timestamp = to_pd_timestamp(timestamp)
if is_same_date(current, timestamp):
for start, end in get_trading_intervals(security_type=security_type,
exchange=exchange):
if current > date_and_time(
current, start) and current < date_and_time(current, end):
return True
return False
def record(self, entity, start, end, size, timestamps, http_session):
the_quarters = get_year_quarters(start, now_pd_timestamp(Region.CHN))
if not is_same_date(entity.timestamp, start) and len(the_quarters) > 1:
the_quarters = the_quarters[1:]
param = {
'security_item': entity,
'quarters': the_quarters,
'level': self.level.value
}
security_item = param['security_item']
quarters = param['quarters']
level = param['level']
result_df = pd.DataFrame()
for year, quarter in quarters:
query_url = self.url.format(security_item.code, year, quarter)
response = request_get(http_session, query_url)
response.encoding = 'gbk'
try:
dfs = pd.read_html(response.text)
except ValueError as error:
self.logger.error(
f'skip ({year}-{quarter:02d}){security_item.code}{security_item.name}({error})'
)
self.sleep()
continue
if len(dfs) < 5:
self.sleep()
continue
df = dfs[4].copy()
df = df.iloc[1:]
df.columns = [
'timestamp', 'open', 'high', 'close', 'low', 'volume',
'turnover'
]
df['name'] = security_item.name
df['level'] = level
df['timestamp'] = pd.to_datetime(df['timestamp'])
df['provider'] = Provider.Sina
result_df = pd.concat([result_df, df])
self.logger.info(
f'({security_item.code}{security_item.name})({year}-{quarter:02d})'
)
self.sleep()
result_df = result_df.sort_values(by='timestamp')
return result_df.to_dict(orient='records')
def record(self, entity, start, end, size, timestamps):
the_quarters = get_year_quarters(start)
if not is_same_date(entity.timestamp, start) and len(the_quarters) > 1:
the_quarters = the_quarters[1:]
param = {
"security_item": entity,
"quarters": the_quarters,
"level": self.level.value
}
security_item = param["security_item"]
quarters = param["quarters"]
level = param["level"]
result_df = pd.DataFrame()
for year, quarter in quarters:
query_url = self.url.format(security_item.code, year, quarter)
response = requests.get(query_url)
response.encoding = "gbk"
try:
dfs = pd.read_html(response.text)
except ValueError as error:
self.logger.error(
f"skip ({year}-{quarter:02d}){security_item.code}{security_item.name}({error})"
)
time.sleep(10.0)
continue
if len(dfs) < 5:
time.sleep(10.0)
continue
df = dfs[4].copy()
df = df.iloc[1:]
df.columns = [
"timestamp", "open", "high", "close", "low", "volume",
"turnover"
]
df["name"] = security_item.name
df["level"] = level
df["timestamp"] = pd.to_datetime(df["timestamp"])
df["provider"] = "sina"
result_df = pd.concat([result_df, df])
self.logger.info(
f"({security_item.code}{security_item.name})({year}-{quarter:02d})"
)
time.sleep(10.0)
result_df = result_df.sort_values(by="timestamp")
return result_df.to_dict(orient="records")
def generate_request_param(self, security_item, start, end, size,
timestamp):
the_quarters = get_year_quarters(start)
if not is_same_date(security_item.timestamp,
start) and len(the_quarters) > 1:
the_quarters = the_quarters[1:]
return {
'security_item': security_item,
'quarters': the_quarters,
'level': self.level.value
}
def record(self, entity, start, end, size, timestamps, http_session):
the_quarters = get_year_quarters(start, now_pd_timestamp(Region.CHN))
if not is_same_date(entity.timestamp, start) and len(the_quarters) > 1:
the_quarters = the_quarters[1:]
param = {
'security_item': entity,
'quarters': the_quarters,
'level': self.level.value
}
security_item = param['security_item']
quarters = param['quarters']
level = param['level']
result_df = pd.DataFrame()
for year, quarter in quarters:
query_url = self.url.format(security_item.code, year, quarter)
text = sync_get(http_session,
query_url,
encoding='gbk',
return_type='text')
if text is None:
continue
try:
dfs = pd.read_html(text)
except ValueError as error:
self.logger.error(
f'skip ({year}-{quarter:02d}){security_item.code}{security_item.name}({error})'
)
self.sleep()
continue
if len(dfs) < 5:
self.sleep()
continue
df = dfs[4].copy()
df = df.iloc[1:]
df.columns = [
'timestamp', 'open', 'high', 'close', 'low', 'volume',
'turnover'
]
result_df = pd.concat([result_df, df])
self.sleep()
if pd_is_not_null(result_df):
result_df['level'] = level
return result_df
return None
def evaluate_start_end_size_timestamps(self, security_item):
"""
evaluate the size for recording data
:param security_item:
:type security_item: str
:return:the start,end,size need to recording,size=0 means finish recording
:rtype:(pd.Timestamp,pd.Timestamp,int)
"""
# get latest record
latest_record = get_data(security_id=security_item.id,
provider=self.provider,
data_schema=self.data_schema, level=self.level.value,
order=self.data_schema.timestamp.desc(), limit=1,
return_type='domain',
session=self.session)
if latest_record:
latest_timestamp = latest_record[0].timestamp
else:
latest_timestamp = security_item.timestamp
if not latest_timestamp:
return latest_timestamp, None, self.default_size, None
current_time = pd.Timestamp.now()
time_delta = current_time - latest_timestamp
if self.level == TradingLevel.LEVEL_1DAY:
if is_same_date(current_time, latest_timestamp):
return latest_timestamp, None, 0, None
return latest_timestamp, None, time_delta.days + 1, None
close_hour, close_minute = get_close_time(security_item.id)
# to today,check closing time
if time_delta.days == 0:
if latest_timestamp.hour == close_hour and latest_timestamp.minute == close_minute:
return latest_timestamp, None, 0, None
if self.level == TradingLevel.LEVEL_5MIN:
if time_delta.days > 0:
minutes = (time_delta.days + 1) * get_one_day_trading_minutes(security_item.id)
return latest_timestamp, None, int(math.ceil(minutes / 5)) + 1, None
else:
return latest_timestamp, None, int(math.ceil(time_delta.total_seconds() / (5 * 60))) + 1, None
if self.level == TradingLevel.LEVEL_1HOUR:
if time_delta.days > 0:
minutes = (time_delta.days + 1) * get_one_day_trading_minutes(security_item.id)
return latest_timestamp, None, int(math.ceil(minutes / 60)) + 1, None
else:
return latest_timestamp, None, int(math.ceil(time_delta.total_seconds() / (60 * 60))) + 1, None
def test_month_start_end_range():
start = "2020-01-01"
end = "2021-01-01"
ranges = month_start_end_ranges(start_date=start, end_date=end)
print(ranges)
assert is_same_date(ranges[0][0], "2020-01-01")
assert is_same_date(ranges[0][1], "2020-01-31")
assert is_same_date(ranges[-1][0], "2020-12-01")
assert is_same_date(ranges[-1][1], "2020-12-31")
start = "2020-01-01"
end = "2021-01-31"
ranges = month_start_end_ranges(start_date=start, end_date=end)
print(ranges)
assert is_same_date(ranges[0][0], "2020-01-01")
assert is_same_date(ranges[0][1], "2020-01-31")
assert is_same_date(ranges[-1][0], "2021-01-01")
assert is_same_date(ranges[-1][1], "2021-01-31")
def test_month_start_end_range():
start = '2020-01-01'
end = '2021-01-01'
ranges = month_start_end_ranges(start_date=start, end_date=end)
print(ranges)
assert is_same_date(ranges[0][0], '2020-01-01')
assert is_same_date(ranges[0][1], '2020-01-31')
assert is_same_date(ranges[-1][0], '2020-12-01')
assert is_same_date(ranges[-1][1], '2020-12-31')
start = '2020-01-01'
end = '2021-01-31'
ranges = month_start_end_ranges(start_date=start, end_date=end)
print(ranges)
assert is_same_date(ranges[0][0], '2020-01-01')
assert is_same_date(ranges[0][1], '2020-01-31')
assert is_same_date(ranges[-1][0], '2021-01-01')
assert is_same_date(ranges[-1][1], '2021-01-31')
def eval_fetch_timestamps(self, entity, ref_record, http_session):
latest_timestamp = None
try:
if pd_is_not_null(ref_record):
time_field = self.get_evaluated_time_field()
latest_timestamp = ref_record[time_field].max(axis=0)
except Exception as e:
self.logger.warning(f'get ref record failed with error: {e}')
if not latest_timestamp:
latest_timestamp = entity.timestamp
if not latest_timestamp:
return self.start_timestamp, self.end_timestamp, self.default_size, None
now = now_pd_timestamp(self.region)
now_end = now.replace(hour=18, minute=0, second=0)
trade_day_index = 0
if len(self.trade_day) > 0:
if is_same_date(self.trade_day[trade_day_index], now) and now < now_end:
trade_day_index = 1
end = self.trade_day[trade_day_index]
else:
end = now
start_timestamp = next_date(latest_timestamp)
start = max(self.start_timestamp, start_timestamp) if self.start_timestamp else start_timestamp
if start >= end:
return start, end, 0, None
size = eval_size_of_timestamp(start_timestamp=start,
end_timestamp=end,
level=self.level,
one_day_trading_minutes=self.one_day_trading_minutes)
return start, end, size, None
def evaluate_start_end_size_timestamps(self, security_item):
"""
evaluate the size for recording data
:param security_item:
:type security_item: str
:return:the start,end,size need to recording,size=0 means finish recording
:rtype:(pd.Timestamp,pd.Timestamp,int)
"""
# get latest record
latest_record = get_data(security_id=security_item.id,
provider=self.provider,
data_schema=self.data_schema,
level=self.level.value,
order=self.data_schema.timestamp.desc(),
limit=1,
return_type='domain',
session=self.session)
if latest_record:
latest_timestamp = latest_record[0].timestamp
else:
latest_timestamp = security_item.timestamp
if not latest_timestamp:
return latest_timestamp, None, self.default_size, None
current_time = pd.Timestamp.now()
time_delta = current_time - latest_timestamp
if self.level == TradingLevel.LEVEL_1DAY:
if is_same_date(current_time, latest_timestamp):
return latest_timestamp, None, 0, None
return latest_timestamp, None, time_delta.days + 1, None
close_hour, close_minute = get_close_time(security_item.id)
# to today,check closing time
# 0,0 means never stop,e.g,coin
if (close_hour != 0 and close_minute != 0) and time_delta.days == 0:
if latest_timestamp.hour == close_hour and latest_timestamp.minute == close_minute:
return latest_timestamp, None, 0, None
if self.kdata_use_begin_time:
touching_timestamp = latest_timestamp + pd.Timedelta(
seconds=self.level.to_second())
else:
touching_timestamp = latest_timestamp
waiting_seconds, size = self.level.count_from_timestamp(
touching_timestamp,
one_day_trading_minutes=get_one_day_trading_minutes(
security_item.id))
if not self.one_shot and waiting_seconds and (waiting_seconds > 30):
t = waiting_seconds / 2
self.logger.info(
'level:{},recorded_time:{},touching_timestamp:{},current_time:{},next_ok_time:{},just sleep:{} seconds'
.format(
self.level.value, latest_timestamp, touching_timestamp,
current_time, touching_timestamp +
pd.Timedelta(seconds=self.level.to_second()), t))
time.sleep(t)
return latest_timestamp, None, size, None
def record(self, security_item, start, end, size, timestamps):
the_quarters = get_year_quarters(start)
# treat has recorded the season if contains some date
if not is_same_date(security_item.timestamp,
start) and len(the_quarters) > 1:
the_quarters = the_quarters[1:]
for year, quarter in the_quarters:
kdatas = []
for fuquan in ['bfq', 'hfq']:
the_url = self.get_kdata_url(security_item.code, year, quarter,
fuquan)
resp = requests.get(the_url)
trs = Selector(text=resp.text).xpath(
'//*[@id="FundHoldSharesTable"]/tr[position()>1 and position()<=last()]'
).extract()
for idx, tr in enumerate(trs):
tds = Selector(text=tr).xpath('//td//text()').extract()
tds = [x.strip() for x in tds if x.strip()]
open = tds[1]
high = tds[2]
close = tds[3]
low = tds[4]
volume = tds[5]
turnover = tds[6]
if fuquan == 'hfq':
factor = tds[7]
the_timestamp = to_pd_timestamp(tds[0])
the_id = generate_kdata_id(security_id=security_item.id,
timestamp=the_timestamp,
level=self.level)
if fuquan == 'hfq':
# we got bfq at first and then update hfq data
existed = [
item for item in kdatas if item['id'] == the_id
]
if existed:
kdata = existed[0]
else:
self.logger.error(
"bfq not got for:{}".format(the_id))
kdata = {
'id': the_id,
'timestamp': the_timestamp,
'name': security_item.name,
'level': self.level.value,
'open': to_float(open) / to_float(factor),
'close': to_float(close) / to_float(factor),
'high': to_float(high) / to_float(factor),
'low': to_float(low) / to_float(factor),
'volume': to_float(volume),
'turnover': to_float(turnover)
}
kdatas.append(kdata)
kdata['hfq_open'] = to_float(open)
kdata['hfq_high'] = to_float(high)
kdata['hfq_close'] = to_float(close)
kdata['hfq_low'] = to_float(low)
kdata['factor'] = to_float(factor)
self.latest_factors[security_item.id] = to_float(
factor)
else:
kdatas.append({
'id': the_id,
'timestamp': the_timestamp,
'name': security_item.name,
'level': self.level.value,
'open': to_float(open),
'close': to_float(close),
'high': to_float(high),
'low': to_float(low),
'volume': to_float(volume),
'turnover': to_float(turnover)
})
return kdatas
def run(self):
now = now_pd_timestamp(self.region)
# iterate timestamp of the min level,e.g,9:30,9:35,9.40...for 5min level
# timestamp represents the timestamp in kdata
for timestamp in self.entity_schema.get_interval_timestamps(
start_date=self.start_timestamp,
end_date=self.end_timestamp,
level=self.level):
if not self.in_trading_date(timestamp=timestamp):
continue
waiting_seconds = 0
if self.level == IntervalLevel.LEVEL_1DAY:
if is_same_date(timestamp, now):
while True:
self.logger.info(
f'time is:{now},just smoke for minutes')
time.sleep(60)
if now.hour >= 19:
waiting_seconds = 20
break
elif self.real_time:
# all selector move on to handle the coming data
if self.kdata_use_begin_time:
real_end_timestamp = timestamp + pd.Timedelta(
seconds=self.level.to_second())
else:
real_end_timestamp = timestamp
seconds = (now - real_end_timestamp).total_seconds()
waiting_seconds = self.level.to_second() - seconds
# meaning the future kdata not ready yet,we could move on to check
if waiting_seconds > 0:
# iterate the selector from min to max which in finished timestamp kdata
for level in self.trading_level_asc:
if self.entity_schema.is_finished_kdata_timestamp(
timestamp=timestamp, level=level):
for selector in self.selectors:
if selector.level == level:
selector.move_on(timestamp,
self.kdata_use_begin_time,
timeout=waiting_seconds + 20)
# on_trading_open to setup the account
if self.level >= IntervalLevel.LEVEL_1DAY or (
self.level != IntervalLevel.LEVEL_1DAY
and self.entity_schema.is_open_timestamp(timestamp)):
self.on_trading_open(timestamp)
self.on_time(timestamp=timestamp)
if self.selectors:
for level in self.trading_level_asc:
# in every cycle, all level selector do its job in its time
if self.entity_schema.is_finished_kdata_timestamp(
timestamp=timestamp, level=level):
all_long_targets = []
all_short_targets = []
for selector in self.selectors:
if selector.level == level:
long_targets = selector.get_open_long_targets(
timestamp=timestamp)
long_targets = self.filter_selector_long_targets(
timestamp=timestamp,
selector=selector,
long_targets=long_targets)
short_targets = selector.get_open_short_targets(
timestamp=timestamp)
short_targets = self.filter_selector_short_targets(
timestamp=timestamp,
selector=selector,
short_targets=short_targets)
if long_targets:
all_long_targets += long_targets
if short_targets:
all_short_targets += short_targets
if all_long_targets:
self.set_long_targets_by_level(
level, all_long_targets)
if all_short_targets:
self.set_short_targets_by_level(
level, all_short_targets)
# the time always move on by min level step and we could check all targets of levels
# 1)the targets is generated for next interval
# 2)the acceptable price is next interval prices,you could buy it at current price
# if the time is before the timestamp(due_timestamp) when trading signal received
# 3)the suggest price the the close price for generating the signal(happen_timestamp)
due_timestamp = timestamp + pd.Timedelta(
seconds=self.level.to_second())
if level == self.level:
long_selected = self.select_long_targets_from_levels(
timestamp)
short_selected = self.select_short_targets_from_levels(
timestamp)
self.logger.debug(
'timestamp:{},long_selected:{}'.format(
due_timestamp, long_selected))
self.logger.debug(
'timestamp:{},short_selected:{}'.format(
due_timestamp, short_selected))
self.trade_the_targets(
due_timestamp=due_timestamp,
happen_timestamp=timestamp,
long_selected=long_selected,
short_selected=short_selected)
if self.trading_signals:
self.on_trading_signals(self.trading_signals)
# clear
self.trading_signals = []
# on_trading_close to calculate date account
if self.level >= IntervalLevel.LEVEL_1DAY or (
self.level != IntervalLevel.LEVEL_1DAY
and self.entity_schema.is_close_timestamp(timestamp)):
self.on_trading_close(timestamp)
self.on_finish(timestamp)
def evaluate_start_end_size_timestamps(self, now, entity, trade_day, stock_detail, http_session):
# not to list date yet
# step1 = time.time()
trade_index = 0
if entity.timestamp and (entity.timestamp >= now):
trade = trade_day[trade_index] if len(trade_day) > 0 else None
return entity.timestamp, None, trade, 0, None
# get latest record
latest_saved_record = self.get_latest_saved_record(entity=entity)
# self.logger.info("step 1: get latest save record: {}".format(time.time()-step1))
if latest_saved_record:
# the latest saved timestamp
latest_saved_timestamp = latest_saved_record.timestamp
else:
# the list date
latest_saved_timestamp = entity.timestamp
# print("step 3: latest_saved_timestamp:{}".format(latest_saved_timestamp))
# print("step 4: start_timestamp:{}, end_timestamp:{}".format(self.start_timestamp, self.end_timestamp))
if not latest_saved_timestamp:
trade = trade_day[trade_index] if len(trade_day) > 0 else None
return None, None, trade, self.default_size, None
# self.logger.info("latest_saved_timestamp:{}, tradedays:{}".format(latest_saved_timestamp, trade_day[:2]))
if trade_day is not None and len(trade_day) > 0:
count_mins = count_mins_before_close_time(now, self.close_hour, self.close_minute)
if count_mins > 0 and is_same_date(trade_day[0], now):
trade_index = 1
# self.logger.info("step 2: get trade index: {}".format(time.time()-step1))
try:
end_date = stock_detail.loc[entity.id].at['end_date']
days = date_delta(now, end_date)
except Exception as e:
# self.logger.warning("can't find stock in stock detail:{}".format(e))
days = -1
# self.logger.info("step 3: get end date: {}".format(time.time()-step1))
if days > 0:
try:
trade_index = trade_day.index(end_date)
# self.logger.info("entity:{}, index:{}, out of market at date:{}, index_day:{}".format(entity.id, trade_index, end_date, trade_day[trade_index]))
except Exception as _:
try:
trade_index = trade_day.index[trade_day.index < end_date].index[0]
except Exception as e:
self.logger.warning("can't find timestamp:{} between trade_day:{}".format(end_date, e))
size = evaluate_size_from_timestamp(start_timestamp=latest_saved_timestamp,
end_timestamp=now,
level=self.level,
one_day_trading_minutes=self.one_day_trading_minutes,
trade_day=trade_day[trade_index:])
# self.logger.info("step 4: evaluate: {}".format(time.time()-step1))
trade = trade_day[trade_index] if len(trade_day) > 0 else None
return latest_saved_timestamp, None, trade, size, None
def run(self):
# iterate timestamp of the min level,e.g,9:30,9:35,9.40...for 5min level
# timestamp represents the timestamp in kdata
for timestamp in self.entity_schema.get_interval_timestamps(start_date=self.start_timestamp,
end_date=self.end_timestamp, level=self.level):
if not self.in_trading_date(timestamp=timestamp):
continue
waiting_seconds = 0
if self.level == IntervalLevel.LEVEL_1DAY:
if is_same_date(timestamp, now_pd_timestamp()):
while True:
self.logger.info(f'time is:{now_pd_timestamp()},just smoke for minutes')
time.sleep(60)
current = now_pd_timestamp()
if current.hour >= 19:
waiting_seconds = 20
break
elif self.real_time:
# all selector move on to handle the coming data
if self.kdata_use_begin_time:
real_end_timestamp = timestamp + pd.Timedelta(seconds=self.level.to_second())
else:
real_end_timestamp = timestamp
seconds = (now_pd_timestamp() - real_end_timestamp).total_seconds()
waiting_seconds = self.level.to_second() - seconds
# meaning the future kdata not ready yet,we could move on to check
if waiting_seconds > 0:
# iterate the selector from min to max which in finished timestamp kdata
for level in self.trading_level_asc:
if self.entity_schema.is_finished_kdata_timestamp(timestamp=timestamp, level=level):
for selector in self.selectors:
if selector.level == level:
selector.move_on(timestamp, self.kdata_use_begin_time, timeout=waiting_seconds + 20)
# on_trading_open to setup the account
if self.level >= IntervalLevel.LEVEL_1DAY or (
self.level != IntervalLevel.LEVEL_1DAY and self.entity_schema.is_open_timestamp(timestamp)):
self.on_trading_open(timestamp)
self.on_time(timestamp=timestamp)
# 一般来说selector(factors)计算 多标的 历史数据比较快，多级别的计算也比较方便，常用于全市场标的粗过滤
# 更细节的控制可以在on_targets_filtered里进一步处理
# 也可以在on_time里面设计一些自己的逻辑配合过滤
if self.selectors:
# 多级别的遍历算法要点:
# 1)计算各级别的 标的，通过 on_targets_filtered 过滤，缓存在level_map_long_targets，level_map_short_targets
# 2)在最小的level通过 on_targets_selected_from_levels 根据多级别的缓存标的，生成最终的选中标的
# 这里需要注意的是，小级别拿到上一个周期的大级别的标的，这是合理的
for level in self.trading_level_asc:
# in every cycle, all level selector do its job in its time
if self.entity_schema.is_finished_kdata_timestamp(timestamp=timestamp, level=level):
all_long_targets = []
all_short_targets = []
# 从该level的selector中过滤targets
for selector in self.selectors:
if selector.level == level:
long_targets = selector.get_open_long_targets(timestamp=timestamp)
short_targets = selector.get_open_short_targets(timestamp=timestamp)
if long_targets or short_targets:
long_targets, short_targets = self.on_targets_filtered(timestamp=timestamp,
level=level,
selector=selector,
long_targets=long_targets,
short_targets=short_targets)
if long_targets:
all_long_targets += long_targets
if short_targets:
all_short_targets += short_targets
# 将各级别的targets缓存在level_map_long_targets，level_map_short_targets
self.update_targets_by_level(level, all_long_targets, all_short_targets)
# the time always move on by min level step and we could check all targets of levels
# 1)the targets is generated for next interval
# 2)the acceptable price is next interval prices,you could buy it at current price
# if the time is before the timestamp(due_timestamp) when trading signal received
# 3)the suggest price the the close price for generating the signal(happen_timestamp)
due_timestamp = timestamp + pd.Timedelta(seconds=self.level.to_second())
# 在最小level生成最终的 交易信号
if level == self.level:
long_selected, short_selected = self.on_targets_selected_from_levels(timestamp)
# 处理 止赢 止损
passive_short, _ = self.on_profit_control()
if passive_short:
if not short_selected:
short_selected = passive_short
else:
short_selected = list(set(short_selected) | set(passive_short))
self.logger.debug('timestamp:{},long_selected:{}'.format(due_timestamp, long_selected))
self.logger.debug('timestamp:{},short_selected:{}'.format(due_timestamp, short_selected))
if long_selected or short_selected:
self.trade_the_targets(due_timestamp=due_timestamp, happen_timestamp=timestamp,
long_selected=long_selected, short_selected=short_selected)
if self.trading_signals:
self.on_trading_signals(self.trading_signals)
# clear
self.trading_signals = []
# on_trading_close to calculate date account
if self.level >= IntervalLevel.LEVEL_1DAY or (
self.level != IntervalLevel.LEVEL_1DAY and self.entity_schema.is_close_timestamp(timestamp)):
self.on_trading_close(timestamp)
self.on_finish(timestamp)
def on_trading_open(self, timestamp):
self.logger.info("on_trading_open:{}".format(timestamp))
if is_same_date(timestamp, self.start_timestamp):
return
self.account = self.load_account()
