def stock_kdata_to_es(start='000001', end='666666', force=False): for _, security_item in get_security_list(start=start, end=end).iterrows(): # 创建索引 index_name = get_es_kdata_index(security_item['id']) es_index_mapping(index_name, StockKData) start_date = None if not force: latest_record = es_get_latest_record(index_name) logger.info("latest_record:{}".format(latest_record)) if latest_record: start_date = latest_record['timestamp'] for _, kdata_item in get_kdata(security_item, start_date=start_date).iterrows(): if start_date and is_same_date(start_date, kdata_item['timestamp']): continue try: id = '{}_{}'.format(kdata_item['securityId'], kdata_item['timestamp']) kdata = StockKData(meta={'id': id}, id=id) fill_doc_type(kdata, json.loads(kdata_item.to_json())) kdata.save(index=index_name) except Exception as e: logger.warn("wrong KdataDay:{},error:{}", kdata_item, e)
def stock_kdata_to_es(start='000001', end='666666', codes=US_STOCK_CODES, force=False): for _, security_item in get_security_list(start=start, end=end, exchanges=['sh', 'sz', 'nasdaq'], codes=codes).iterrows(): # 创建索引 index_name = get_es_kdata_index(security_item['type'], security_item['exchange']) es_index_mapping(index_name, StockKData) start_date = None if not force: query = { "term": {"securityId": ""} } query["term"]["securityId"] = security_item["id"] latest_record = es_get_latest_record(index_name, query=query) logger.info("latest_record:{}".format(latest_record)) if latest_record: start_date = latest_record['timestamp'] actions = [] for _, kdata_item in get_kdata(security_item, start_date=start_date).iterrows(): if start_date and is_same_date(start_date, kdata_item['timestamp']): continue try: id = '{}_{}'.format(kdata_item['securityId'], kdata_item['timestamp']) kdata = StockKData(meta={'id': id}, id=id) kdata.meta['index'] = index_name fill_doc_type(kdata, json.loads(kdata_item.to_json())) # kdata.save(index=index_name) actions.append(kdata.to_dict(include_meta=True)) except Exception as e: logger.warn("wrong KdataDay:{},error:{}", kdata_item, e) if actions: resp = elasticsearch.helpers.bulk(es, actions) logger.info(resp)
def stock_kdata_to_es(start='000001', end='666666', force=False): for _, security_item in get_security_list(start=start, end=end).iterrows(): # 创建索引 index_name = get_es_kdata_index(security_item['type'], security_item['exchange']) es_index_mapping(index_name, StockKData) start_date = None if not force: query = {"term": {"securityId": ""}} query["term"]["securityId"] = security_item["id"] latest_record = es_get_latest_record(index_name, query=query) logger.info("latest_record:{}".format(latest_record)) if latest_record: start_date = latest_record['timestamp'] actions = [] for _, kdata_item in get_kdata(security_item, start_date=start_date).iterrows(): if start_date and is_same_date(start_date, kdata_item['timestamp']): continue try: id = '{}_{}'.format(kdata_item['securityId'], kdata_item['timestamp']) kdata = StockKData(meta={'id': id}, id=id) fill_doc_type(kdata, json.loads(kdata_item.to_json())) # kdata.save(index=index_name) actions.append(kdata.to_dict(include_meta=True)) except Exception as e: logger.warn("wrong KdataDay:{},error:{}", kdata_item, e) if actions: resp = elasticsearch.helpers.bulk(es, actions) logger.info(resp)
def kdata_to_es(security_type='stock', start_code=None, end_code=None, force=False): if security_type == 'stock': doc_type = StockKData elif security_type == 'index': doc_type = IndexKData elif security_type == 'cryptocurrency': doc_type = CryptoCurrencyKData for _, security_item in get_security_list(security_type=security_type, start_code=start_code, end_code=end_code).iterrows(): index_name = get_es_kdata_index(security_item['type'], security_item['exchange']) query = None if not force: query = {"term": {"securityId": ""}} query["term"]["securityId"] = security_item["id"] df = get_kdata(security_item, generate_id=True) df_to_es(df, doc_type=doc_type, index_name=index_name, query=query, force=force)
def kdata_to_es(security_type='stock', start_code=None, end_code=None, force=False): codes = None if security_type == 'stock': doc_type = StockKData elif security_type == 'index': doc_type = IndexKData elif security_type == 'cryptocurrency': doc_type = CryptoCurrencyKData codes = CRYPTOCURRENCY_CODE for _, security_item in get_security_list(security_type=security_type, start_code=start_code, end_code=end_code, codes=codes).iterrows(): index_name = get_es_kdata_index(security_item['type'], security_item['exchange']) df = get_kdata(security_item, generate_id=True) df_to_es(df, doc_type=doc_type, index_name=index_name, security_item=security_item, force=force)
def es_get_kdata(security_item, the_date=None, start_date=None, end_date=None, level='day', fields=None, from_idx=0, size=10): """ get kdata. Parameters ---------- security_item : SecurityItem or str the security item,id or code the_date : TimeStamp str or TimeStamp get the kdata for the exact date start_date : TimeStamp str or TimeStamp start date end_date : TimeStamp str or TimeStamp end date level : str or int the kdata level,{1,5,15,30,60,'day','week','month'},default : 'day' fields : filed list for es _source if not set,would use the default fields for the security type from_idx : int pagination start offset size : int pagination return size Returns ------- JSON """ security_item = to_security_item(security_item) index = get_es_kdata_index(security_type=security_item['type'], exchange=security_item['exchange'], level=level) if not fields: if security_item['type'] == 'stock': fields = KDATA_STOCK_COL elif security_item['type'] == 'future': fields = KDATA_FUTURE_COL elif security_item['type'] == 'index': fields = KDATA_INDEX_COL else: fields = KDATA_COMMON_COL # 单日的日k线直接按id获取 if level == 'day' and the_date: doc_id = '{}_{}'.format(security_item['id'], to_time_str(the_date)) return es_client.get_source(index=index, doc_type='doc', id=doc_id, _source_include=fields) elif start_date and end_date: s = Search(using=es_client, index=index, doc_type='doc') \ .source(include=fields) \ .filter('term', code=security_item['code']) \ .filter('range', timestamp={'gte': start_date, 'lte': end_date}) \ .sort({"timestamp": {"order": "asc"}}) resp = s[from_idx:from_idx + size].execute() return resp['hits'].to_dict()
def kdata_to_es(start=None, end=None, security_type='stock', exchanges=['sh', 'sz'], force=False): if security_type == 'stock': doc_type = StockKData elif security_type == 'index': doc_type = IndexKData elif security_type == 'cryptocurrency': doc_type = CryptoCurrencyKData for _, security_item in get_security_list(security_type=security_type, exchanges=exchanges, start=start, end=end).iterrows(): # 创建索引 index_name = get_es_kdata_index(security_item['type'], security_item['exchange']) es_index_mapping(index_name, doc_type) start_date = None if not force: query = {"term": {"securityId": ""}} query["term"]["securityId"] = security_item["id"] latest_record = es_get_latest_record(index_name, query=query) logger.info("latest_record:{}".format(latest_record)) if latest_record: start_date = latest_record['timestamp'] actions = [] df_kdata = get_kdata(security_item, start_date=start_date) for _, kdata_item in df_kdata.iterrows(): if start_date and is_same_date(start_date, kdata_item['timestamp']): continue try: id = '{}_{}'.format(kdata_item['securityId'], kdata_item['timestamp']) kdata = doc_type(meta={'id': id}, id=id) kdata.meta['index'] = index_name kdata_json = json.loads(kdata_item.to_json()) fill_doc_type(kdata, kdata_json) # kdata.save(index=index_name) actions.append(kdata.to_dict(include_meta=True)) except Exception as e: logger.warn("wrong KdataDay:{},error:{}", kdata_item, e) if actions: resp = elasticsearch.helpers.bulk(es_client, actions) logger.info(resp)
# -*- coding: utf-8 -*- from datetime import timedelta, datetime import pandas as pd from fooltrader.bot.bot import NotifyEventBot from fooltrader.contract.es_contract import get_es_kdata_index, get_es_statistic_index from fooltrader.domain.data.es_quote import CommonKData, CommonStatistic from fooltrader.settings import TIME_FORMAT_MICRO from fooltrader.utils.es_utils import es_get_latest_timestamp, es_get_latest_record, es_index_mapping from fooltrader.utils.utils import to_timestamp, to_time_str, fill_doc_type, is_same_date, is_same_time statistic_index_name = get_es_statistic_index(security_type='cryptocurrency', exchange='contract') kdata_index_name = get_es_kdata_index(security_type='cryptocurrency', exchange='contract', level='1min') es_index_mapping(statistic_index_name, CommonStatistic) es_index_mapping(kdata_index_name, CommonKData) class EosStatisticBot(NotifyEventBot): BIG_ORDER = 2000 * 10000 MIDDLE_ORDER = 500 * 10000 def on_init(self): super().on_init() self.security_id = 'cryptocurrency_contract_RAM-EOS' query = { "term": {"securityId": ""} } query["term"]["securityId"] = self.security_id