def on_time(self, timestamp): if is_same_date(buy_timestamp, timestamp): self.buy(due_timestamp=buy_timestamp, happen_timestamp=buy_timestamp, entity_ids=['stock_sz_000338']) self.has_buy = True self.buy(due_timestamp=buy_timestamp, happen_timestamp=buy_timestamp, entity_ids=['stock_sh_601318']) if is_same_date(sell_timestamp, timestamp): self.sell(due_timestamp=sell_timestamp, happen_timestamp=sell_timestamp, entity_ids=['stock_sz_000338', 'stock_sh_601318'])
def persist(self, security_item, domain_list): if domain_list: first_timestamp = domain_list[0].timestamp last_timestamp = domain_list[-1].timestamp self.logger.info( "recording {} for security_id:{},level:{},first_timestamp:{},last_timestamp:{}" .format(self.data_schema, security_item.id, self.level, first_timestamp, last_timestamp)) current_timestamp = now_pd_timestamp() saving_datas = domain_list # FIXME:remove this logic # FIXME:should remove unfinished data when recording,always set it to False now if is_same_date(current_timestamp, last_timestamp) and self.contain_unfinished_data: close_hour, close_minute = get_close_time(security_item.id) if current_timestamp.hour >= close_hour and current_timestamp.minute >= close_minute + 2: # after the closing time of the day,we think the last data is finished saving_datas = domain_list else: # ignore unfinished kdata saving_datas = domain_list[:-1] self.logger.info( "ignore kdata for security_id:{},level:{},timestamp:{},current_timestamp" .format(security_item.id, self.level, last_timestamp, current_timestamp)) self.session.add_all(saving_datas) self.session.commit()
def run(self): # 按不同类别抓取 # 编码 基金运作方式 # 401001 开放式基金 # 401002 封闭式基金 # 401003 QDII # 401004 FOF # 401005 ETF # 401006 LOF for operate_mode_id in (401001, 401002, 401005): year_count = 2 while True: latest = Fund.query_data( filters=[Fund.operate_mode_id == operate_mode_id], order=Fund.timestamp.desc(), limit=1, return_type="domain", ) start_timestamp = "2000-01-01" if latest: start_timestamp = latest[0].timestamp end_timestamp = min( next_date(start_timestamp, 365 * year_count), now_pd_timestamp()) df = run_query( table="finance.FUND_MAIN_INFO", conditions= f"operate_mode_id#=#{operate_mode_id}&start_date#>=#{to_time_str(start_timestamp)}&start_date#<=#{to_time_str(end_timestamp)}", parse_dates=["start_date", "end_date"], dtype={"main_code": str}, ) if not pd_is_not_null(df) or (df["start_date"].max().year < end_timestamp.year): year_count = year_count + 1 if pd_is_not_null(df): df.rename(columns={"start_date": "timestamp"}, inplace=True) df["timestamp"] = pd.to_datetime(df["timestamp"]) df["list_date"] = df["timestamp"] df["end_date"] = pd.to_datetime(df["end_date"]) df["code"] = df["main_code"] df["entity_id"] = df["code"].apply( lambda x: to_entity_id(entity_type="fund", jq_code=x)) df["id"] = df["entity_id"] df["entity_type"] = "fund" df["exchange"] = "sz" df_to_db(df, data_schema=Fund, provider=self.provider, force_update=self.force_update) self.logger.info( f"persist fund {operate_mode_id} list success {start_timestamp} to {end_timestamp}" ) if is_same_date(end_timestamp, now_pd_timestamp()): break
def on_trading_open(self, timestamp): self.logger.info('on_trading_open:{}'.format(timestamp)) if is_same_date(timestamp, self.start_timestamp): return self.account = self.load_account() self.logger.info('on_trading_open:{},current_account:{}'.format( timestamp, self.account))
def run(self): # 按不同类别抓取 # 编码 基金运作方式 # 401001 开放式基金 # 401002 封闭式基金 # 401003 QDII # 401004 FOF # 401005 ETF # 401006 LOF for operate_mode_id in (401001, 401002, 401005): year_count = 2 while True: latest = Fund.query_data( region=self.region, filters=[Fund.operate_mode_id == operate_mode_id], order=Fund.timestamp.desc(), limit=1, return_type='domain') start_timestamp = '2000-01-01' if latest: start_timestamp = latest[0].timestamp end_timestamp = min( next_date(start_timestamp, 365 * year_count), now_pd_timestamp(self.region)) df = jq_run_query( table='finance.FUND_MAIN_INFO', conditions= f'operate_mode_id#=#{operate_mode_id}&start_date#>=#{to_time_str(start_timestamp)}&start_date#<=#{to_time_str(end_timestamp)}', parse_dates=['start_date', 'end_date'], dtype={'main_code': str}) if not pd_is_not_null(df) or (df['start_date'].max().year < end_timestamp.year): year_count = year_count + 1 if pd_is_not_null(df): df.rename(columns={'start_date': 'timestamp'}, inplace=True) df['timestamp'] = pd.to_datetime(df['timestamp']) df['list_date'] = df['timestamp'] df['end_date'] = pd.to_datetime(df['end_date']) df['code'] = df['main_code'] df['entity_id'] = df['code'].apply( lambda x: to_entity_id(entity_type='fund', jq_code=x)) df['id'] = df['entity_id'] df['entity_type'] = 'fund' df['exchange'] = 'sz' df_to_db(df, ref_df=None, region=self.region, data_schema=Fund, provider=self.provider) self.logger.info( f'persist fund {operate_mode_id} list success {start_timestamp} to {end_timestamp}' ) if is_same_date(end_timestamp, now_pd_timestamp(self.region)): break
def on_trading_open(self, timestamp): self.logger.info('on_trading_open:{}'.format(timestamp)) if is_same_date(timestamp, self.start_timestamp): return # get the account for trading at the date accounts = get_account(session=self.session, trader_name=self.trader_name, return_type='domain', end_timestamp=to_time_str(timestamp), limit=1, order=SimAccount.timestamp.desc()) if accounts: account = accounts[0] else: return positions = [] # FIXME:dump all directly for position_domain in account.positions: position_dict = position_schema.dump(position_domain).data self.logger.info('current position:{}'.format(position_dict)) del position_dict['sim_account'] positions.append(position_dict) self.latest_account = sim_account_schema.dump(account).data self.latest_account['positions'] = positions self.logger.info('on_trading_open:{},latest_account:{}'.format( timestamp, self.latest_account))
def test_split_time_interval(): first = None last = None start = '2020-01-01' end = '2021-01-01' for interval in split_time_interval(start, end, interval=30): if first is None: first = interval last = interval print(first) print(last) assert is_same_date(first[0], start) assert is_same_date(first[-1], '2020-01-31') assert is_same_date(last[-1], end)
def evaluate_start_end_size_timestamps(self, entity): start, end, size, timestamps = super().evaluate_start_end_size_timestamps(entity) if start: trade_day = StockTradeDay.query_data(limit=1, order=StockTradeDay.timestamp.desc(), return_type='domain') if trade_day: if is_same_date(trade_day[0].timestamp, start): size = 0 return start, end, size, timestamps
def test_split_time_interval_month(): first = None last = None start = "2020-01-01" end = "2021-01-01" for interval in split_time_interval(start, end, method="month"): if first is None: first = interval last = interval print(first) print(last) assert is_same_date(first[0], start) assert is_same_date(first[-1], "2020-01-31") assert is_same_date(last[0], "2021-01-01") assert is_same_date(last[-1], "2021-01-01")
def is_in_trading(security_type, exchange, timestamp): current = now_pd_timestamp() timestamp = to_pd_timestamp(timestamp) if is_same_date(current, timestamp): for start, end in get_trading_intervals(security_type=security_type, exchange=exchange): if current > date_and_time( current, start) and current < date_and_time(current, end): return True return False
def record(self, entity, start, end, size, timestamps, http_session): the_quarters = get_year_quarters(start, now_pd_timestamp(Region.CHN)) if not is_same_date(entity.timestamp, start) and len(the_quarters) > 1: the_quarters = the_quarters[1:] param = { 'security_item': entity, 'quarters': the_quarters, 'level': self.level.value } security_item = param['security_item'] quarters = param['quarters'] level = param['level'] result_df = pd.DataFrame() for year, quarter in quarters: query_url = self.url.format(security_item.code, year, quarter) response = request_get(http_session, query_url) response.encoding = 'gbk' try: dfs = pd.read_html(response.text) except ValueError as error: self.logger.error( f'skip ({year}-{quarter:02d}){security_item.code}{security_item.name}({error})' ) self.sleep() continue if len(dfs) < 5: self.sleep() continue df = dfs[4].copy() df = df.iloc[1:] df.columns = [ 'timestamp', 'open', 'high', 'close', 'low', 'volume', 'turnover' ] df['name'] = security_item.name df['level'] = level df['timestamp'] = pd.to_datetime(df['timestamp']) df['provider'] = Provider.Sina result_df = pd.concat([result_df, df]) self.logger.info( f'({security_item.code}{security_item.name})({year}-{quarter:02d})' ) self.sleep() result_df = result_df.sort_values(by='timestamp') return result_df.to_dict(orient='records')
def record(self, entity, start, end, size, timestamps): the_quarters = get_year_quarters(start) if not is_same_date(entity.timestamp, start) and len(the_quarters) > 1: the_quarters = the_quarters[1:] param = { "security_item": entity, "quarters": the_quarters, "level": self.level.value } security_item = param["security_item"] quarters = param["quarters"] level = param["level"] result_df = pd.DataFrame() for year, quarter in quarters: query_url = self.url.format(security_item.code, year, quarter) response = requests.get(query_url) response.encoding = "gbk" try: dfs = pd.read_html(response.text) except ValueError as error: self.logger.error( f"skip ({year}-{quarter:02d}){security_item.code}{security_item.name}({error})" ) time.sleep(10.0) continue if len(dfs) < 5: time.sleep(10.0) continue df = dfs[4].copy() df = df.iloc[1:] df.columns = [ "timestamp", "open", "high", "close", "low", "volume", "turnover" ] df["name"] = security_item.name df["level"] = level df["timestamp"] = pd.to_datetime(df["timestamp"]) df["provider"] = "sina" result_df = pd.concat([result_df, df]) self.logger.info( f"({security_item.code}{security_item.name})({year}-{quarter:02d})" ) time.sleep(10.0) result_df = result_df.sort_values(by="timestamp") return result_df.to_dict(orient="records")
def generate_request_param(self, security_item, start, end, size, timestamp): the_quarters = get_year_quarters(start) if not is_same_date(security_item.timestamp, start) and len(the_quarters) > 1: the_quarters = the_quarters[1:] return { 'security_item': security_item, 'quarters': the_quarters, 'level': self.level.value }
def record(self, entity, start, end, size, timestamps, http_session): the_quarters = get_year_quarters(start, now_pd_timestamp(Region.CHN)) if not is_same_date(entity.timestamp, start) and len(the_quarters) > 1: the_quarters = the_quarters[1:] param = { 'security_item': entity, 'quarters': the_quarters, 'level': self.level.value } security_item = param['security_item'] quarters = param['quarters'] level = param['level'] result_df = pd.DataFrame() for year, quarter in quarters: query_url = self.url.format(security_item.code, year, quarter) text = sync_get(http_session, query_url, encoding='gbk', return_type='text') if text is None: continue try: dfs = pd.read_html(text) except ValueError as error: self.logger.error( f'skip ({year}-{quarter:02d}){security_item.code}{security_item.name}({error})' ) self.sleep() continue if len(dfs) < 5: self.sleep() continue df = dfs[4].copy() df = df.iloc[1:] df.columns = [ 'timestamp', 'open', 'high', 'close', 'low', 'volume', 'turnover' ] result_df = pd.concat([result_df, df]) self.sleep() if pd_is_not_null(result_df): result_df['level'] = level return result_df return None
def evaluate_start_end_size_timestamps(self, security_item): """ evaluate the size for recording data :param security_item: :type security_item: str :return:the start,end,size need to recording,size=0 means finish recording :rtype:(pd.Timestamp,pd.Timestamp,int) """ # get latest record latest_record = get_data(security_id=security_item.id, provider=self.provider, data_schema=self.data_schema, level=self.level.value, order=self.data_schema.timestamp.desc(), limit=1, return_type='domain', session=self.session) if latest_record: latest_timestamp = latest_record[0].timestamp else: latest_timestamp = security_item.timestamp if not latest_timestamp: return latest_timestamp, None, self.default_size, None current_time = pd.Timestamp.now() time_delta = current_time - latest_timestamp if self.level == TradingLevel.LEVEL_1DAY: if is_same_date(current_time, latest_timestamp): return latest_timestamp, None, 0, None return latest_timestamp, None, time_delta.days + 1, None close_hour, close_minute = get_close_time(security_item.id) # to today,check closing time if time_delta.days == 0: if latest_timestamp.hour == close_hour and latest_timestamp.minute == close_minute: return latest_timestamp, None, 0, None if self.level == TradingLevel.LEVEL_5MIN: if time_delta.days > 0: minutes = (time_delta.days + 1) * get_one_day_trading_minutes(security_item.id) return latest_timestamp, None, int(math.ceil(minutes / 5)) + 1, None else: return latest_timestamp, None, int(math.ceil(time_delta.total_seconds() / (5 * 60))) + 1, None if self.level == TradingLevel.LEVEL_1HOUR: if time_delta.days > 0: minutes = (time_delta.days + 1) * get_one_day_trading_minutes(security_item.id) return latest_timestamp, None, int(math.ceil(minutes / 60)) + 1, None else: return latest_timestamp, None, int(math.ceil(time_delta.total_seconds() / (60 * 60))) + 1, None
def test_month_start_end_range(): start = "2020-01-01" end = "2021-01-01" ranges = month_start_end_ranges(start_date=start, end_date=end) print(ranges) assert is_same_date(ranges[0][0], "2020-01-01") assert is_same_date(ranges[0][1], "2020-01-31") assert is_same_date(ranges[-1][0], "2020-12-01") assert is_same_date(ranges[-1][1], "2020-12-31") start = "2020-01-01" end = "2021-01-31" ranges = month_start_end_ranges(start_date=start, end_date=end) print(ranges) assert is_same_date(ranges[0][0], "2020-01-01") assert is_same_date(ranges[0][1], "2020-01-31") assert is_same_date(ranges[-1][0], "2021-01-01") assert is_same_date(ranges[-1][1], "2021-01-31")
def test_month_start_end_range(): start = '2020-01-01' end = '2021-01-01' ranges = month_start_end_ranges(start_date=start, end_date=end) print(ranges) assert is_same_date(ranges[0][0], '2020-01-01') assert is_same_date(ranges[0][1], '2020-01-31') assert is_same_date(ranges[-1][0], '2020-12-01') assert is_same_date(ranges[-1][1], '2020-12-31') start = '2020-01-01' end = '2021-01-31' ranges = month_start_end_ranges(start_date=start, end_date=end) print(ranges) assert is_same_date(ranges[0][0], '2020-01-01') assert is_same_date(ranges[0][1], '2020-01-31') assert is_same_date(ranges[-1][0], '2021-01-01') assert is_same_date(ranges[-1][1], '2021-01-31')
def eval_fetch_timestamps(self, entity, ref_record, http_session): latest_timestamp = None try: if pd_is_not_null(ref_record): time_field = self.get_evaluated_time_field() latest_timestamp = ref_record[time_field].max(axis=0) except Exception as e: self.logger.warning(f'get ref record failed with error: {e}') if not latest_timestamp: latest_timestamp = entity.timestamp if not latest_timestamp: return self.start_timestamp, self.end_timestamp, self.default_size, None now = now_pd_timestamp(self.region) now_end = now.replace(hour=18, minute=0, second=0) trade_day_index = 0 if len(self.trade_day) > 0: if is_same_date(self.trade_day[trade_day_index], now) and now < now_end: trade_day_index = 1 end = self.trade_day[trade_day_index] else: end = now start_timestamp = next_date(latest_timestamp) start = max(self.start_timestamp, start_timestamp) if self.start_timestamp else start_timestamp if start >= end: return start, end, 0, None size = eval_size_of_timestamp(start_timestamp=start, end_timestamp=end, level=self.level, one_day_trading_minutes=self.one_day_trading_minutes) return start, end, size, None
def evaluate_start_end_size_timestamps(self, security_item): """ evaluate the size for recording data :param security_item: :type security_item: str :return:the start,end,size need to recording,size=0 means finish recording :rtype:(pd.Timestamp,pd.Timestamp,int) """ # get latest record latest_record = get_data(security_id=security_item.id, provider=self.provider, data_schema=self.data_schema, level=self.level.value, order=self.data_schema.timestamp.desc(), limit=1, return_type='domain', session=self.session) if latest_record: latest_timestamp = latest_record[0].timestamp else: latest_timestamp = security_item.timestamp if not latest_timestamp: return latest_timestamp, None, self.default_size, None current_time = pd.Timestamp.now() time_delta = current_time - latest_timestamp if self.level == TradingLevel.LEVEL_1DAY: if is_same_date(current_time, latest_timestamp): return latest_timestamp, None, 0, None return latest_timestamp, None, time_delta.days + 1, None close_hour, close_minute = get_close_time(security_item.id) # to today,check closing time # 0,0 means never stop,e.g,coin if (close_hour != 0 and close_minute != 0) and time_delta.days == 0: if latest_timestamp.hour == close_hour and latest_timestamp.minute == close_minute: return latest_timestamp, None, 0, None if self.kdata_use_begin_time: touching_timestamp = latest_timestamp + pd.Timedelta( seconds=self.level.to_second()) else: touching_timestamp = latest_timestamp waiting_seconds, size = self.level.count_from_timestamp( touching_timestamp, one_day_trading_minutes=get_one_day_trading_minutes( security_item.id)) if not self.one_shot and waiting_seconds and (waiting_seconds > 30): t = waiting_seconds / 2 self.logger.info( 'level:{},recorded_time:{},touching_timestamp:{},current_time:{},next_ok_time:{},just sleep:{} seconds' .format( self.level.value, latest_timestamp, touching_timestamp, current_time, touching_timestamp + pd.Timedelta(seconds=self.level.to_second()), t)) time.sleep(t) return latest_timestamp, None, size, None
def record(self, security_item, start, end, size, timestamps): the_quarters = get_year_quarters(start) # treat has recorded the season if contains some date if not is_same_date(security_item.timestamp, start) and len(the_quarters) > 1: the_quarters = the_quarters[1:] for year, quarter in the_quarters: kdatas = [] for fuquan in ['bfq', 'hfq']: the_url = self.get_kdata_url(security_item.code, year, quarter, fuquan) resp = requests.get(the_url) trs = Selector(text=resp.text).xpath( '//*[@id="FundHoldSharesTable"]/tr[position()>1 and position()<=last()]' ).extract() for idx, tr in enumerate(trs): tds = Selector(text=tr).xpath('//td//text()').extract() tds = [x.strip() for x in tds if x.strip()] open = tds[1] high = tds[2] close = tds[3] low = tds[4] volume = tds[5] turnover = tds[6] if fuquan == 'hfq': factor = tds[7] the_timestamp = to_pd_timestamp(tds[0]) the_id = generate_kdata_id(security_id=security_item.id, timestamp=the_timestamp, level=self.level) if fuquan == 'hfq': # we got bfq at first and then update hfq data existed = [ item for item in kdatas if item['id'] == the_id ] if existed: kdata = existed[0] else: self.logger.error( "bfq not got for:{}".format(the_id)) kdata = { 'id': the_id, 'timestamp': the_timestamp, 'name': security_item.name, 'level': self.level.value, 'open': to_float(open) / to_float(factor), 'close': to_float(close) / to_float(factor), 'high': to_float(high) / to_float(factor), 'low': to_float(low) / to_float(factor), 'volume': to_float(volume), 'turnover': to_float(turnover) } kdatas.append(kdata) kdata['hfq_open'] = to_float(open) kdata['hfq_high'] = to_float(high) kdata['hfq_close'] = to_float(close) kdata['hfq_low'] = to_float(low) kdata['factor'] = to_float(factor) self.latest_factors[security_item.id] = to_float( factor) else: kdatas.append({ 'id': the_id, 'timestamp': the_timestamp, 'name': security_item.name, 'level': self.level.value, 'open': to_float(open), 'close': to_float(close), 'high': to_float(high), 'low': to_float(low), 'volume': to_float(volume), 'turnover': to_float(turnover) }) return kdatas
def run(self): now = now_pd_timestamp(self.region) # iterate timestamp of the min level,e.g,9:30,9:35,9.40...for 5min level # timestamp represents the timestamp in kdata for timestamp in self.entity_schema.get_interval_timestamps( start_date=self.start_timestamp, end_date=self.end_timestamp, level=self.level): if not self.in_trading_date(timestamp=timestamp): continue waiting_seconds = 0 if self.level == IntervalLevel.LEVEL_1DAY: if is_same_date(timestamp, now): while True: self.logger.info( f'time is:{now},just smoke for minutes') time.sleep(60) if now.hour >= 19: waiting_seconds = 20 break elif self.real_time: # all selector move on to handle the coming data if self.kdata_use_begin_time: real_end_timestamp = timestamp + pd.Timedelta( seconds=self.level.to_second()) else: real_end_timestamp = timestamp seconds = (now - real_end_timestamp).total_seconds() waiting_seconds = self.level.to_second() - seconds # meaning the future kdata not ready yet,we could move on to check if waiting_seconds > 0: # iterate the selector from min to max which in finished timestamp kdata for level in self.trading_level_asc: if self.entity_schema.is_finished_kdata_timestamp( timestamp=timestamp, level=level): for selector in self.selectors: if selector.level == level: selector.move_on(timestamp, self.kdata_use_begin_time, timeout=waiting_seconds + 20) # on_trading_open to setup the account if self.level >= IntervalLevel.LEVEL_1DAY or ( self.level != IntervalLevel.LEVEL_1DAY and self.entity_schema.is_open_timestamp(timestamp)): self.on_trading_open(timestamp) self.on_time(timestamp=timestamp) if self.selectors: for level in self.trading_level_asc: # in every cycle, all level selector do its job in its time if self.entity_schema.is_finished_kdata_timestamp( timestamp=timestamp, level=level): all_long_targets = [] all_short_targets = [] for selector in self.selectors: if selector.level == level: long_targets = selector.get_open_long_targets( timestamp=timestamp) long_targets = self.filter_selector_long_targets( timestamp=timestamp, selector=selector, long_targets=long_targets) short_targets = selector.get_open_short_targets( timestamp=timestamp) short_targets = self.filter_selector_short_targets( timestamp=timestamp, selector=selector, short_targets=short_targets) if long_targets: all_long_targets += long_targets if short_targets: all_short_targets += short_targets if all_long_targets: self.set_long_targets_by_level( level, all_long_targets) if all_short_targets: self.set_short_targets_by_level( level, all_short_targets) # the time always move on by min level step and we could check all targets of levels # 1)the targets is generated for next interval # 2)the acceptable price is next interval prices,you could buy it at current price # if the time is before the timestamp(due_timestamp) when trading signal received # 3)the suggest price the the close price for generating the signal(happen_timestamp) due_timestamp = timestamp + pd.Timedelta( seconds=self.level.to_second()) if level == self.level: long_selected = self.select_long_targets_from_levels( timestamp) short_selected = self.select_short_targets_from_levels( timestamp) self.logger.debug( 'timestamp:{},long_selected:{}'.format( due_timestamp, long_selected)) self.logger.debug( 'timestamp:{},short_selected:{}'.format( due_timestamp, short_selected)) self.trade_the_targets( due_timestamp=due_timestamp, happen_timestamp=timestamp, long_selected=long_selected, short_selected=short_selected) if self.trading_signals: self.on_trading_signals(self.trading_signals) # clear self.trading_signals = [] # on_trading_close to calculate date account if self.level >= IntervalLevel.LEVEL_1DAY or ( self.level != IntervalLevel.LEVEL_1DAY and self.entity_schema.is_close_timestamp(timestamp)): self.on_trading_close(timestamp) self.on_finish(timestamp)
def evaluate_start_end_size_timestamps(self, now, entity, trade_day, stock_detail, http_session): # not to list date yet # step1 = time.time() trade_index = 0 if entity.timestamp and (entity.timestamp >= now): trade = trade_day[trade_index] if len(trade_day) > 0 else None return entity.timestamp, None, trade, 0, None # get latest record latest_saved_record = self.get_latest_saved_record(entity=entity) # self.logger.info("step 1: get latest save record: {}".format(time.time()-step1)) if latest_saved_record: # the latest saved timestamp latest_saved_timestamp = latest_saved_record.timestamp else: # the list date latest_saved_timestamp = entity.timestamp # print("step 3: latest_saved_timestamp:{}".format(latest_saved_timestamp)) # print("step 4: start_timestamp:{}, end_timestamp:{}".format(self.start_timestamp, self.end_timestamp)) if not latest_saved_timestamp: trade = trade_day[trade_index] if len(trade_day) > 0 else None return None, None, trade, self.default_size, None # self.logger.info("latest_saved_timestamp:{}, tradedays:{}".format(latest_saved_timestamp, trade_day[:2])) if trade_day is not None and len(trade_day) > 0: count_mins = count_mins_before_close_time(now, self.close_hour, self.close_minute) if count_mins > 0 and is_same_date(trade_day[0], now): trade_index = 1 # self.logger.info("step 2: get trade index: {}".format(time.time()-step1)) try: end_date = stock_detail.loc[entity.id].at['end_date'] days = date_delta(now, end_date) except Exception as e: # self.logger.warning("can't find stock in stock detail:{}".format(e)) days = -1 # self.logger.info("step 3: get end date: {}".format(time.time()-step1)) if days > 0: try: trade_index = trade_day.index(end_date) # self.logger.info("entity:{}, index:{}, out of market at date:{}, index_day:{}".format(entity.id, trade_index, end_date, trade_day[trade_index])) except Exception as _: try: trade_index = trade_day.index[trade_day.index < end_date].index[0] except Exception as e: self.logger.warning("can't find timestamp:{} between trade_day:{}".format(end_date, e)) size = evaluate_size_from_timestamp(start_timestamp=latest_saved_timestamp, end_timestamp=now, level=self.level, one_day_trading_minutes=self.one_day_trading_minutes, trade_day=trade_day[trade_index:]) # self.logger.info("step 4: evaluate: {}".format(time.time()-step1)) trade = trade_day[trade_index] if len(trade_day) > 0 else None return latest_saved_timestamp, None, trade, size, None
def run(self): # iterate timestamp of the min level,e.g,9:30,9:35,9.40...for 5min level # timestamp represents the timestamp in kdata for timestamp in self.entity_schema.get_interval_timestamps(start_date=self.start_timestamp, end_date=self.end_timestamp, level=self.level): if not self.in_trading_date(timestamp=timestamp): continue waiting_seconds = 0 if self.level == IntervalLevel.LEVEL_1DAY: if is_same_date(timestamp, now_pd_timestamp()): while True: self.logger.info(f'time is:{now_pd_timestamp()},just smoke for minutes') time.sleep(60) current = now_pd_timestamp() if current.hour >= 19: waiting_seconds = 20 break elif self.real_time: # all selector move on to handle the coming data if self.kdata_use_begin_time: real_end_timestamp = timestamp + pd.Timedelta(seconds=self.level.to_second()) else: real_end_timestamp = timestamp seconds = (now_pd_timestamp() - real_end_timestamp).total_seconds() waiting_seconds = self.level.to_second() - seconds # meaning the future kdata not ready yet,we could move on to check if waiting_seconds > 0: # iterate the selector from min to max which in finished timestamp kdata for level in self.trading_level_asc: if self.entity_schema.is_finished_kdata_timestamp(timestamp=timestamp, level=level): for selector in self.selectors: if selector.level == level: selector.move_on(timestamp, self.kdata_use_begin_time, timeout=waiting_seconds + 20) # on_trading_open to setup the account if self.level >= IntervalLevel.LEVEL_1DAY or ( self.level != IntervalLevel.LEVEL_1DAY and self.entity_schema.is_open_timestamp(timestamp)): self.on_trading_open(timestamp) self.on_time(timestamp=timestamp) # 一般来说selector(factors)计算 多标的 历史数据比较快,多级别的计算也比较方便,常用于全市场标的粗过滤 # 更细节的控制可以在on_targets_filtered里进一步处理 # 也可以在on_time里面设计一些自己的逻辑配合过滤 if self.selectors: # 多级别的遍历算法要点: # 1)计算各级别的 标的,通过 on_targets_filtered 过滤,缓存在level_map_long_targets,level_map_short_targets # 2)在最小的level通过 on_targets_selected_from_levels 根据多级别的缓存标的,生成最终的选中标的 # 这里需要注意的是,小级别拿到上一个周期的大级别的标的,这是合理的 for level in self.trading_level_asc: # in every cycle, all level selector do its job in its time if self.entity_schema.is_finished_kdata_timestamp(timestamp=timestamp, level=level): all_long_targets = [] all_short_targets = [] # 从该level的selector中过滤targets for selector in self.selectors: if selector.level == level: long_targets = selector.get_open_long_targets(timestamp=timestamp) short_targets = selector.get_open_short_targets(timestamp=timestamp) if long_targets or short_targets: long_targets, short_targets = self.on_targets_filtered(timestamp=timestamp, level=level, selector=selector, long_targets=long_targets, short_targets=short_targets) if long_targets: all_long_targets += long_targets if short_targets: all_short_targets += short_targets # 将各级别的targets缓存在level_map_long_targets,level_map_short_targets self.update_targets_by_level(level, all_long_targets, all_short_targets) # the time always move on by min level step and we could check all targets of levels # 1)the targets is generated for next interval # 2)the acceptable price is next interval prices,you could buy it at current price # if the time is before the timestamp(due_timestamp) when trading signal received # 3)the suggest price the the close price for generating the signal(happen_timestamp) due_timestamp = timestamp + pd.Timedelta(seconds=self.level.to_second()) # 在最小level生成最终的 交易信号 if level == self.level: long_selected, short_selected = self.on_targets_selected_from_levels(timestamp) # 处理 止赢 止损 passive_short, _ = self.on_profit_control() if passive_short: if not short_selected: short_selected = passive_short else: short_selected = list(set(short_selected) | set(passive_short)) self.logger.debug('timestamp:{},long_selected:{}'.format(due_timestamp, long_selected)) self.logger.debug('timestamp:{},short_selected:{}'.format(due_timestamp, short_selected)) if long_selected or short_selected: self.trade_the_targets(due_timestamp=due_timestamp, happen_timestamp=timestamp, long_selected=long_selected, short_selected=short_selected) if self.trading_signals: self.on_trading_signals(self.trading_signals) # clear self.trading_signals = [] # on_trading_close to calculate date account if self.level >= IntervalLevel.LEVEL_1DAY or ( self.level != IntervalLevel.LEVEL_1DAY and self.entity_schema.is_close_timestamp(timestamp)): self.on_trading_close(timestamp) self.on_finish(timestamp)
def on_trading_open(self, timestamp): self.logger.info("on_trading_open:{}".format(timestamp)) if is_same_date(timestamp, self.start_timestamp): return self.account = self.load_account()