def stock_kdata_to_es(start='000001', end='666666', force=False):
for _, security_item in get_security_list(start=start, end=end).iterrows():
# 创建索引
index_name = get_es_kdata_index(security_item['id'])
es_index_mapping(index_name, StockKData)
start_date = None
if not force:
latest_record = es_get_latest_record(index_name)
logger.info("latest_record:{}".format(latest_record))
if latest_record:
start_date = latest_record['timestamp']
for _, kdata_item in get_kdata(security_item,
start_date=start_date).iterrows():
if start_date and is_same_date(start_date,
kdata_item['timestamp']):
continue
try:
id = '{}_{}'.format(kdata_item['securityId'],
kdata_item['timestamp'])
kdata = StockKData(meta={'id': id}, id=id)
fill_doc_type(kdata, json.loads(kdata_item.to_json()))
kdata.save(index=index_name)
except Exception as e:
logger.warn("wrong KdataDay:{},error:{}", kdata_item, e)
def stock_kdata_to_es(start='000001', end='666666', codes=US_STOCK_CODES, force=False):
for _, security_item in get_security_list(start=start, end=end, exchanges=['sh', 'sz', 'nasdaq'],
codes=codes).iterrows():
# 创建索引
index_name = get_es_kdata_index(security_item['type'], security_item['exchange'])
es_index_mapping(index_name, StockKData)
start_date = None
if not force:
query = {
"term": {"securityId": ""}
}
query["term"]["securityId"] = security_item["id"]
latest_record = es_get_latest_record(index_name, query=query)
logger.info("latest_record:{}".format(latest_record))
if latest_record:
start_date = latest_record['timestamp']
actions = []
for _, kdata_item in get_kdata(security_item, start_date=start_date).iterrows():
if start_date and is_same_date(start_date, kdata_item['timestamp']):
continue
try:
id = '{}_{}'.format(kdata_item['securityId'], kdata_item['timestamp'])
kdata = StockKData(meta={'id': id}, id=id)
kdata.meta['index'] = index_name
fill_doc_type(kdata, json.loads(kdata_item.to_json()))
# kdata.save(index=index_name)
actions.append(kdata.to_dict(include_meta=True))
except Exception as e:
logger.warn("wrong KdataDay:{},error:{}", kdata_item, e)
if actions:
resp = elasticsearch.helpers.bulk(es, actions)
logger.info(resp)
def stock_kdata_to_es(start='000001', end='666666', force=False):
for _, security_item in get_security_list(start=start, end=end).iterrows():
# 创建索引
index_name = get_es_kdata_index(security_item['type'],
security_item['exchange'])
es_index_mapping(index_name, StockKData)
start_date = None
if not force:
query = {"term": {"securityId": ""}}
query["term"]["securityId"] = security_item["id"]
latest_record = es_get_latest_record(index_name, query=query)
logger.info("latest_record:{}".format(latest_record))
if latest_record:
start_date = latest_record['timestamp']
actions = []
for _, kdata_item in get_kdata(security_item,
start_date=start_date).iterrows():
if start_date and is_same_date(start_date,
kdata_item['timestamp']):
continue
try:
id = '{}_{}'.format(kdata_item['securityId'],
kdata_item['timestamp'])
kdata = StockKData(meta={'id': id}, id=id)
fill_doc_type(kdata, json.loads(kdata_item.to_json()))
# kdata.save(index=index_name)
actions.append(kdata.to_dict(include_meta=True))
except Exception as e:
logger.warn("wrong KdataDay:{},error:{}", kdata_item, e)
if actions:
resp = elasticsearch.helpers.bulk(es, actions)
logger.info(resp)
def kdata_to_es(security_type='stock',
start_code=None,
end_code=None,
force=False):
if security_type == 'stock':
doc_type = StockKData
elif security_type == 'index':
doc_type = IndexKData
elif security_type == 'cryptocurrency':
doc_type = CryptoCurrencyKData
for _, security_item in get_security_list(security_type=security_type,
start_code=start_code,
end_code=end_code).iterrows():
index_name = get_es_kdata_index(security_item['type'],
security_item['exchange'])
query = None
if not force:
query = {"term": {"securityId": ""}}
query["term"]["securityId"] = security_item["id"]
df = get_kdata(security_item, generate_id=True)
df_to_es(df,
doc_type=doc_type,
index_name=index_name,
query=query,
force=force)
def kdata_to_es(security_type='stock',
start_code=None,
end_code=None,
force=False):
codes = None
if security_type == 'stock':
doc_type = StockKData
elif security_type == 'index':
doc_type = IndexKData
elif security_type == 'cryptocurrency':
doc_type = CryptoCurrencyKData
codes = CRYPTOCURRENCY_CODE
for _, security_item in get_security_list(security_type=security_type,
start_code=start_code,
end_code=end_code,
codes=codes).iterrows():
index_name = get_es_kdata_index(security_item['type'],
security_item['exchange'])
df = get_kdata(security_item, generate_id=True)
df_to_es(df,
doc_type=doc_type,
index_name=index_name,
security_item=security_item,
force=force)
def es_get_kdata(security_item, the_date=None, start_date=None, end_date=None, level='day', fields=None,
from_idx=0, size=10):
"""
get kdata.
Parameters
----------
security_item : SecurityItem or str
the security item,id or code
the_date : TimeStamp str or TimeStamp
get the kdata for the exact date
start_date : TimeStamp str or TimeStamp
start date
end_date : TimeStamp str or TimeStamp
end date
level : str or int
the kdata level,{1,5,15,30,60,'day','week','month'},default : 'day'
fields : filed list for es _source
if not set,would use the default fields for the security type
from_idx : int
pagination start offset
size : int
pagination return size
Returns
-------
JSON
"""
security_item = to_security_item(security_item)
index = get_es_kdata_index(security_type=security_item['type'], exchange=security_item['exchange'],
level=level)
if not fields:
if security_item['type'] == 'stock':
fields = KDATA_STOCK_COL
elif security_item['type'] == 'future':
fields = KDATA_FUTURE_COL
elif security_item['type'] == 'index':
fields = KDATA_INDEX_COL
else:
fields = KDATA_COMMON_COL
# 单日的日k线直接按id获取
if level == 'day' and the_date:
doc_id = '{}_{}'.format(security_item['id'], to_time_str(the_date))
return es_client.get_source(index=index, doc_type='doc', id=doc_id, _source_include=fields)
elif start_date and end_date:
s = Search(using=es_client, index=index, doc_type='doc') \
.source(include=fields) \
.filter('term', code=security_item['code']) \
.filter('range', timestamp={'gte': start_date, 'lte': end_date}) \
.sort({"timestamp": {"order": "asc"}})
resp = s[from_idx:from_idx + size].execute()
return resp['hits'].to_dict()
def kdata_to_es(start=None,
end=None,
security_type='stock',
exchanges=['sh', 'sz'],
force=False):
if security_type == 'stock':
doc_type = StockKData
elif security_type == 'index':
doc_type = IndexKData
elif security_type == 'cryptocurrency':
doc_type = CryptoCurrencyKData
for _, security_item in get_security_list(security_type=security_type,
exchanges=exchanges,
start=start,
end=end).iterrows():
# 创建索引
index_name = get_es_kdata_index(security_item['type'],
security_item['exchange'])
es_index_mapping(index_name, doc_type)
start_date = None
if not force:
query = {"term": {"securityId": ""}}
query["term"]["securityId"] = security_item["id"]
latest_record = es_get_latest_record(index_name, query=query)
logger.info("latest_record:{}".format(latest_record))
if latest_record:
start_date = latest_record['timestamp']
actions = []
df_kdata = get_kdata(security_item, start_date=start_date)
for _, kdata_item in df_kdata.iterrows():
if start_date and is_same_date(start_date,
kdata_item['timestamp']):
continue
try:
id = '{}_{}'.format(kdata_item['securityId'],
kdata_item['timestamp'])
kdata = doc_type(meta={'id': id}, id=id)
kdata.meta['index'] = index_name
kdata_json = json.loads(kdata_item.to_json())
fill_doc_type(kdata, kdata_json)
# kdata.save(index=index_name)
actions.append(kdata.to_dict(include_meta=True))
except Exception as e:
logger.warn("wrong KdataDay:{},error:{}", kdata_item, e)
if actions:
resp = elasticsearch.helpers.bulk(es_client, actions)
logger.info(resp)
# -*- coding: utf-8 -*-
from datetime import timedelta, datetime
import pandas as pd
from fooltrader.bot.bot import NotifyEventBot
from fooltrader.contract.es_contract import get_es_kdata_index, get_es_statistic_index
from fooltrader.domain.data.es_quote import CommonKData, CommonStatistic
from fooltrader.settings import TIME_FORMAT_MICRO
from fooltrader.utils.es_utils import es_get_latest_timestamp, es_get_latest_record, es_index_mapping
from fooltrader.utils.utils import to_timestamp, to_time_str, fill_doc_type, is_same_date, is_same_time
statistic_index_name = get_es_statistic_index(security_type='cryptocurrency', exchange='contract')
kdata_index_name = get_es_kdata_index(security_type='cryptocurrency', exchange='contract', level='1min')
es_index_mapping(statistic_index_name, CommonStatistic)
es_index_mapping(kdata_index_name, CommonKData)
class EosStatisticBot(NotifyEventBot):
BIG_ORDER = 2000 * 10000
MIDDLE_ORDER = 500 * 10000
def on_init(self):
super().on_init()
self.security_id = 'cryptocurrency_contract_RAM-EOS'
query = {
"term": {"securityId": ""}
}
query["term"]["securityId"] = self.security_id