def df_to_es(df,
doc_type,
index_name=None,
timestamp_filed='timestamp',
security_item=None,
force=False):
if not index_name:
index_name = doc_type().meta.index
es_index_mapping(index_name, doc_type)
if not force:
query = None
if not force:
query = {"term": {"securityId": ""}}
query["term"]["securityId"] = security_item["id"]
start_date = es_get_latest_timestamp(index=index_name,
query=query,
time_field=timestamp_filed)
logger.info("{} latest timestamp:{}".format(index_name, start_date))
if start_date:
df = df.loc[start_date:, :]
actions = []
for _, item in df.iterrows():
try:
es_data = doc_type(meta={'id': item['id'], 'index': index_name})
item_json = json.loads(item.to_json())
fill_doc_type(es_data, item_json)
actions.append(es_data.to_dict(include_meta=True))
except Exception as e:
logger.exception("wrong item:{},error:{}".format(item, e))
if actions:
resp = elasticsearch.helpers.bulk(es_client, actions)
logger.info("index to {} success:{} failed:{}".format(
index_name, resp[0], len(resp[1])))
if resp[1]:
logger.error("index to {} error:{}".format(index_name, resp[1]))
def finance_sheet_to_es(sheet_type=None,
start_code=None,
end_code=None,
force=False):
if sheet_type is None:
sheet_types = [
'balance_sheet', 'income_statement', 'cash_flow_statement'
]
else:
sheet_types = [sheet_type]
for sheet_type in sheet_types:
if sheet_type == 'balance_sheet':
doc_type = BalanceSheet
elif sheet_type == 'income_statement':
doc_type = IncomeStatement
elif sheet_type == 'cash_flow_statement':
doc_type = CashFlowStatement
es_index_mapping(sheet_type, doc_type)
for _, security_item in get_security_list(
start_code=start_code, end_code=end_code).iterrows():
try:
if sheet_type == 'balance_sheet':
items = get_balance_sheet_items(security_item)
elif sheet_type == 'income_statement':
items = get_income_statement_items(security_item)
elif sheet_type == 'cash_flow_statement':
items = get_cash_flow_statement_items(security_item)
df = pd.DataFrame(items)
df = index_df_with_time(df, index='reportPeriod')
df_to_es(df,
doc_type=doc_type,
timestamp_filed='reportPeriod',
security_item=security_item,
force=force)
except Exception as e:
logger.exception(
"index {} {} failed".format(security_item['code'],
sheet_type), e)
def __init__(self,
bot_name,
timestamp,
base_capital=1000000,
buy_cost=0.001,
sell_cost=0.001,
slippage=0.001,
stock_fuquan='hfq'):
self.logger = logging.getLogger(__name__)
self.base_capital = base_capital
self.buy_cost = buy_cost
self.sell_cost = sell_cost
self.slippage = slippage
self.stock_fuquan = stock_fuquan
self.bot_name = bot_name
account = es_get_latest_record(index='account',
query={"term": {
"botName": bot_name
}})
if account:
self.logger.warning(
"bot:{} has run before,old result would be deleted".format(
bot_name))
es_delete(index='account', query={"term": {"botName": bot_name}})
es_index_mapping('account', Account)
self.account = Account()
self.account.botName = bot_name
self.account.cash = self.base_capital
self.account.positions = []
self.account.value = self.base_capital
self.account.timestamp = timestamp
self.account.save()
def finance_sheet_to_es(sheet_type='balance_sheet',
start_code=None,
end_code=None,
force=False):
if sheet_type == 'balance_sheet':
doc_type = BalanceSheet
elif sheet_type == 'income_statement':
doc_type = IncomeStatement
elif sheet_type == 'cash_flow_statement':
doc_type = CashFlowStatement
es_index_mapping(sheet_type, doc_type)
for _, security_item in get_security_list(start_code=start_code,
end_code=end_code).iterrows():
query = None
if not force:
query = {"term": {"securityId": ""}}
query["term"]["securityId"] = security_item["id"]
if sheet_type == 'balance_sheet':
items = get_balance_sheet_items(security_item)
elif sheet_type == 'income_statement':
items = get_income_statement_items(security_item)
elif sheet_type == 'cash_flow_statement':
items = get_cash_flow_statement_items(security_item)
df = pd.DataFrame(items)
df = index_df_with_time(df, index='reportPeriod')
df_to_es(df,
doc_type=doc_type,
timestamp_filed='reportPeriod',
query=query,
force=force)
# -*- coding: utf-8 -*-
from datetime import timedelta, datetime
import pandas as pd
from fooltrader.bot.bot import NotifyEventBot
from fooltrader.contract.es_contract import get_es_kdata_index, get_es_statistic_index
from fooltrader.domain.data.es_quote import CommonKData, CommonStatistic
from fooltrader.settings import TIME_FORMAT_MICRO
from fooltrader.utils.es_utils import es_get_latest_timestamp, es_get_latest_record, es_index_mapping
from fooltrader.utils.utils import to_timestamp, to_time_str, fill_doc_type, is_same_date, is_same_time
statistic_index_name = get_es_statistic_index(security_type='cryptocurrency', exchange='contract')
kdata_index_name = get_es_kdata_index(security_type='cryptocurrency', exchange='contract', level='1min')
es_index_mapping(statistic_index_name, CommonStatistic)
es_index_mapping(kdata_index_name, CommonKData)
class EosStatisticBot(NotifyEventBot):
BIG_ORDER = 2000 * 10000
MIDDLE_ORDER = 500 * 10000
def on_init(self):
super().on_init()
self.security_id = 'cryptocurrency_contract_RAM-EOS'
query = {
"term": {"securityId": ""}
}
query["term"]["securityId"] = self.security_id
from pymongo import MongoClient
from fooltrader import fill_doc_type, es_client
from fooltrader.domain.data.es_quote import EosAccount
from fooltrader.settings import EOS_MONGODB_URL
from fooltrader.utils.es_utils import es_index_mapping
from fooltrader.utils.utils import to_time_str
logger = logging.getLogger(__name__)
client = MongoClient(EOS_MONGODB_URL)
db = client['eosMain']
db_rs = client['local']
es_index_mapping("eos_account", EosAccount)
def eos_acount_update_to_es():
pass
def eos_account_to_es():
account = db.accounts
count = account.count()
logger.info("current account size:{}".format(count))
actions = []
# {
# -*- coding: utf-8 -*-
from flask import Flask, Config
from fooltrader import FOOLTRADER_STORE_PATH
from fooltrader.domain.business.es_subscription import PriceSubscription, SubscriptionTriggered
from fooltrader.utils.es_utils import es_index_mapping
es_index_mapping('price_subscription', PriceSubscription)
es_index_mapping('subscription_triggered', SubscriptionTriggered)
app = Flask(__name__)
app.debug = True
app.config.from_object(Config(root_path=FOOLTRADER_STORE_PATH))
app.config['JSON_AS_ASCII'] = False
from fooltrader.rest.controller.security import *
from fooltrader.rest.controller.subscription import *
from fooltrader.rest.controller.tech import *
import pandas as pd
from fooltrader import es_client
from fooltrader.api.esapi.esapi import es_get_user_statistic, es_get_latest_daily_user_statistic
from fooltrader.bot.bot import NotifyEventBot
from fooltrader.contract.es_contract import get_cryptocurrency_user_statistic_index, \
get_cryptocurrency_daily_user_statistic_index
from fooltrader.domain.data.es_quote import EosUserStatistic
from fooltrader.settings import TIME_FORMAT_MICRO
from fooltrader.utils.es_utils import es_get_latest_record, es_index_mapping
from fooltrader.utils.utils import to_timestamp, to_time_str, is_same_date
user_statistic_index_name = get_cryptocurrency_user_statistic_index()
daily_user_statistic_index_name = get_cryptocurrency_daily_user_statistic_index()
es_index_mapping(user_statistic_index_name, EosUserStatistic)
es_index_mapping(daily_user_statistic_index_name, EosUserStatistic)
class EosUserStatisticBot(NotifyEventBot):
def on_init(self):
super().on_init()
self.security_id = 'cryptocurrency_contract_RAM-EOS'
query = {
"term": {"securityId": ""}
}
query["term"]["securityId"] = self.security_id
# get latest user statistic timestamp
latest_eos_user_statistic_record = es_get_latest_record(index=user_statistic_index_name,