def concurrent_engine_quotes(num=4):
engine = Engine(best_ip=True, thread_num=num)
engine.connect()
engine.stock_list.index.tolist()
now = datetime.now()
engine.stock_quotes()
return (datetime.now() - now).total_seconds()
def original_engine_quotes():
engine = Engine(best_ip=True)
engine.connect()
engine.stock_list.index.tolist()
now = datetime.now()
engine.stock_quotes()
return (datetime.now() - now).total_seconds()
def transactions():
eg = Engine(best_ip=True)
eg.connect()
m1 = eg.get_security_bars('000001', '1m')
df = eg.time_and_price('000001')
ohlcv = df.price.resample('1 Min', label='right', closed='left').ohlc()
ohlcv['volume'] = df.vol.resample('1 Min', label='right',
closed='left').sum()
def tdx_bundle(assets,
ingest_minute, # whether to ingest minute data, default False
environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir):
eg = Engine(auto_retry=True, multithread=True, best_ip=True, thread_num=8)
eg.connect()
symbols = fetch_symbols(eg, assets)
metas = []
def gen_symbols_data(symbol_map, freq='1d'):
for index, symbol in symbol_map.iteritems():
data = reindex_to_calendar(
calendar,
fetch_single_equity(eg, symbol, freq),
freq=freq,
)
if freq == '1d':
metas.append(get_meta_from_bars(data))
yield int(symbol), data
symbol_map = symbols.symbol
assets = set([int(s) for s in symbol_map])
daily_bar_writer.write(gen_symbols_data(symbol_map, freq="1d"), assets=assets, show_progress=show_progress)
if ingest_minute:
with click.progressbar(gen_symbols_data(symbol_map, freq="1m"),
label="Merging minute equity files:",
length=len(assets),
item_show_func=lambda e: e if e is None else str(e[0]),
) as bar:
minute_bar_writer.write(bar, show_progress=False)
symbols = pd.concat([symbols, pd.DataFrame(data=metas)], axis=1)
splits, dividends = fetch_splits_and_dividends(eg, symbols)
symbols.set_index('symbol', drop=False, inplace=True)
asset_db_writer.write(symbols)
adjustment_writer.write(
splits=splits,
dividends=dividends
)
eg.exit()
def test_data():
eg = Engine(auto_retry=True, multithread=True, thread_num=8)
with eg.connect():
symbols = fetch_symbols(eg)
symbols = symbols[:5]
metas = []
def gen_symbols_data(symbol_map, freq='1d'):
for index, symbol in symbol_map.iteritems():
data = fetch_single_equity(eg, symbol, freq)
if freq == '1d':
metas.append(get_meta_from_bars(data))
assert data is not None
yield int(symbol), data
symbol_map = symbols.symbol
assets = set([int(s) for s in symbol_map])
gen_symbols_data(symbol_map, freq="1d")
gen_symbols_data(symbol_map, freq="1m")
symbols = pd.concat([symbols, pd.DataFrame(data=metas)], axis=1)
splits, dividends = fetch_splits_and_dividends(eg, symbols)
symbols.set_index('symbol', drop=False, inplace=True)
assert symbols is not None
assert splits is not None
assert dividends is not None
def test_security_list():
engine = Engine(best_ip=True)
engine.connect()
code = engine.stock_list.index.tolist()
api = TdxHq_API()
api.connect()
best_ip = engine.best_ip
print("security list: ({},{})".format(concurrent_api(2),
original_api()))
print("concurrent quotes ({},{})".format(
concurrent_quotes(code, best_ip, 2),
original_quotes(code, best_ip)))
print("concurrent engine quotes ({},{})".format(
concurrent_engine_quotes(2), original_engine_quotes()))
def test_transaction():
engine = Engine(best_ip=True, thread_num=1)
with engine.connect():
df = engine.get_k_data('000001', '20170601', '20171231', '1m')
df = engine.get_security_bars(['000001', '000521'],
'1d',
start=pd.to_datetime('20180102'))
def engine_func(best_ip, thread_num):
engine = Engine(best_ip=best_ip, thread_num=thread_num)
with engine.connect():
assert engine.best_ip is not None
assert engine.gbbq is not None
assert engine.security_list is not None
assert engine.stock_quotes() is not None
assert engine.customer_block is not None
assert engine.quotes('000001') is not None
assert engine.get_security_bars('000001', '1m') is not None
assert engine.get_security_bars('000001', '1d') is not None
assert engine.get_security_bars('000300', '1m', index=True) is not None
assert engine.get_security_bars('000300', '1d', index=True) is not None
assert engine.concept is not None
assert engine.fengge is not None
assert engine.index is not None
assert engine.stock_list is not None
class TdxCatsBroker(TdxShipaneBroker):
"""
中信Cats系统CSV下单模块封装
"""
def __init__(self, cats_client=None):
"""
:param cat_client:
:type cat_client: CatsTrade
"""
self._shipane_client = cats_client
self._orders = {}
self.currency = 'RMB'
self._subscribed_assets = []
self._bars = {}
self._bars_update_dt = None
self._bars_update_interval = pd.tslib.Timedelta('5 S')
self._mkt_client = Engine(auto_retry=True, best_ip=True)
self._mkt_client.connect()
# super(TdxShipaneBroker, self).__init__(tdx_uri, account_id)
def tdx_order_to_zipline_order(self, order):
"""
status
0 已报
1 部分成交
2 全部成交
3 部分撤单
4 全部撤单
5 交易所拒单
6 柜台未接受
:param order:
:return:
"""
if order.status == "3" or '4' == order.status:
zp_status = ZP_ORDER_STATUS.CANCELLED
elif order.status == "0":
zp_status = ZP_ORDER_STATUS.OPEN
elif order.status == "1":
zp_status = ZP_ORDER_STATUS.FILLED
elif order.status == "2":
zp_status = ZP_ORDER_STATUS.HELD
elif order.status == "5" or order.status == "6":
zp_status = ZP_ORDER_STATUS.REJECTED
zp_order_id = self._tdx_to_zp_order_id(order.order_id)
od = ZPOrder(
dt=order.dt,
asset=symbol(order.symbol),
amount=order.amount,
filled=order.filled,
stop=None,
limit=order.price, # TODO 市价单和限价单
id=zp_order_id,
)
od.broker_order_id = order.order_id
od.status = zp_status
return od
pass
calendar = get_calendar('SHSZ')
if start:
if not calendar.is_session(start):
start = calendar.all_sessions[searchsorted(calendar.all_sessions,
start)]
bundles.register('tdx',
partial(tdx_bundle, assets, minute, fundamental),
'SHSZ',
start,
end,
minutes_per_day=240)
bundles.register('tdx',
partial(tdx_bundle, None, False, False),
minutes_per_day=240)
if __name__ == '__main__':
eg = Engine(auto_retry=True, multithread=True, thread_num=8)
with eg.connect():
symbols = fetch_symbols(eg)
symbols = symbols[:3]
data = []
metas = []
for symbol in symbols.symbol:
data.append((int(symbol), fetch_single_equity(eg, symbol)))
metas.append(get_meta_from_bars(data[-1][1]))
symbols = pd.concat([symbols, pd.DataFrame(data=metas)], axis=1)
splits, dividends = fetch_splits_and_dividends(eg, symbols)
class TdxBroker(Broker):
def __init__(self, tdx_uri, account_id=None):
self._orders = {}
if tdx_uri.startswith('tcp'):
self._client = zerorpc.Client()
self._client.connect(tdx_uri)
elif platform.architecture()[0] == '32bit':
self._client = TdxClient(tdx_uri)
self._client.login()
else:
raise Exception(
"please use 32bit python to use local client directly, or use tcp client"
)
self.currency = 'RMB'
self._subscribed_assets = []
self._bars = {}
self._bars_update_dt = None
self._bars_update_interval = pd.tslib.Timedelta('5 S')
self._mkt_client = Engine(auto_retry=True, best_ip=True)
self._mkt_client.connect()
super(TdxBroker, self).__init__()
def subscribe_to_market_data(self, asset):
# TODO fix me subcribe_to_market_data
if asset not in self.subscribed_assets:
# remove str() cast to have a fun debugging journey
# self._client.subscribe_to_market_data(str(asset.symbol))
self._subscribed_assets.append(asset)
self._bars_update_dt = None
def _update_bars(self):
now = pd.to_datetime('now')
if self._bars_update_dt and (now - self._bars_update_dt <
self._bars_update_interval):
return
for code in self.subscribed_assets:
bars = self._mkt_client.time_and_price(code.symbol)
bars.index = bars.index.tz_localize('Asia/Shanghai').tz_convert(
'UTC')
self._bars[code.symbol] = bars
self._bars_update_dt = now
@property
def subscribed_assets(self):
return self._subscribed_assets
@property
def positions(self):
now = datetime.datetime.now()
z_positions = protocol.Positions()
for pos in self._client.positions():
if isinstance(pos, list):
pos = TdxPosition(*pos)
sid = pos.sid
available = pos.available
z_position = protocol.Position(symbol(sid))
z_position.amount = pos.amount
z_position.cost_basis = pos.cost_basis
z_position.last_sale_price = pos.last_sale_price
z_position.last_sale_date = now
z_positions[symbol(sid)] = z_position
return z_positions
@property
def portfolio(self):
z_portfolio = protocol.Portfolio()
pfo = self._client.portfolio()
if isinstance(pfo, list):
pfo = TdxPortfolio(*pfo)
z_portfolio.capital_used = None # TODO
z_portfolio.starting_cash = None
z_portfolio.portfolio_value = pfo.portfolio_value
z_portfolio.pnl = None
z_portfolio.returns = None
z_portfolio.cash = pfo.cash
z_portfolio.start_date = None
z_portfolio.positions = self.positions
z_portfolio.positions_value = pfo.positions_value
z_portfolio.position_exposure = z_portfolio.positions_value / (
z_portfolio.positions_value + z_portfolio.cash)
return z_portfolio
@property
def account(self):
z_account = protocol.Account()
z_account.settled_cash = self.portfolio.cash
z_account.accrued_interest = None
z_account.buying_power = self.portfolio.portfolio_value
z_account.equity_with_loan = self.portfolio.portfolio_value
z_account.total_positions_value = self.portfolio.positions_value
z_account.total_positions_exposure = z_account.total_positions_value / (
z_account.total_positions_value + z_account.settled_cash)
return z_account
@property
def time_skew(self):
return pd.Timedelta('1 S')
def order(self, asset, amount, style):
code = asset.symbol
if amount > 0:
action = BUY
else:
action = SELL
is_busy = (amount > 0)
if isinstance(style, MarketOrder):
order_type = FIVE_LEVEL_MARKET_ORDER
price = 0.0
elif isinstance(style, LimitOrder):
order_type = LIMIT_CHARGE
price = style.get_limit_price(is_busy)
elif isinstance(style, StopOrder):
raise Exception("stop order is not supported")
elif isinstance(style, StopLimitOrder):
raise Exception("stop limit order is not supported")
data, err = self._client.order(code, abs(amount), price, action,
order_type)
if isinstance(data, list):
data = OrderRt(*data)
order_id = str(data.order_id)
zp_order = self._get_or_create_zp_order(order_id)
log.info("Placing order-{order_id}: "
"{action} {qty} {symbol} with {order_type}, price = {price}".
format(order_id=order_id,
action=action,
qty=amount,
symbol=code,
order_type=order_type,
price=price))
return zp_order
@property
def orders(self):
self._update_orders()
return self._orders
def _get_or_create_zp_order(self, order_id, order=None):
zp_order_id = self._tdx_to_zp_order_id(order_id)
if zp_order_id in self._orders:
return self._orders[zp_order_id]
if not order:
order = self._client.orders()[order_id]
if isinstance(order,
list): # handle rpc response for namedtuple object
order = TdxOrder(*order)
self._orders[zp_order_id] = self.tdx_order_to_zipline_order(order)
return self._orders[zp_order_id]
# amount 可正数可负数
def _create_zp_order(self, order_id, asset, price, amount, order_type):
zp_order_id = self._tdx_to_zp_order_id(order_id)
if zp_order_id in self._orders:
return self._orders[zp_order_id]
dt = pd.to_datetime("now", utc=True)
self._orders[zp_order_id] = ZPOrder(
dt=dt,
asset=asset,
amount=amount,
stop=None,
limit=price if order_type is LIMIT_CHARGE else None,
id=zp_order_id,
broker_order_id=order_id)
return self._orders[zp_order_id]
def _tdx_to_zp_order_id(self, order_id):
return "TDX-{date}-{account_id}-{order_id}".format(
date=str(pd.to_datetime('today').date()),
account_id=self._client.account_id(),
order_id=order_id)
def tdx_order_to_zipline_order(self, order):
if order.status == '已撤':
zp_status = ZP_ORDER_STATUS.CANCELLED
elif order.filled == 0:
zp_status = ZP_ORDER_STATUS.OPEN
else:
zp_status = ZP_ORDER_STATUS.FILLED
zp_order_id = self._tdx_to_zp_order_id(order.order_id)
od = ZPOrder(
dt=pd.to_datetime(order.dt),
asset=symbol(order.symbol),
amount=order.amount,
filled=order.filled,
stop=None,
limit=order.price, # TODO 市价单和限价单
id=zp_order_id,
)
od.broker_order_id = order.order_id
od.status = zp_status
return od
def _update_orders(self):
ods = self._client.orders()
for tdx_order_id, tdx_order in iteritems(ods):
if isinstance(tdx_order,
list): # handle rpc response for namedtuple object
tdx_order = TdxOrder(*tdx_order)
zp_order_id = self._tdx_to_zp_order_id(tdx_order_id)
self._orders[zp_order_id] = self.tdx_order_to_zipline_order(
tdx_order)
def _tdx_transaction_to_zipline(self, transaction):
return ZPTransaction(
asset=symbol(transaction.asset),
amount=transaction.amount,
dt=pd.to_datetime(
transaction.dt).tz_localize('Asia/Shanghai').tz_convert('UTC'),
price=transaction.price,
order_id=self._tdx_to_zp_order_id(transaction.order_id),
commission=transaction.commission,
)
@property
def transactions(self):
t = self._client.transactions()
rt = {}
for exec_id, transaction in t.items():
try:
if isinstance(
transaction,
list): # handle rpc response for namedtuple object
transaction = TdxTransaction(*transaction)
rt[exec_id] = self._tdx_transaction_to_zipline(transaction)
except SymbolNotFound as e:
log.warning(e.message)
return rt
def cancel_order(self, order_id):
if order_id not in self.orders: # order become transaction, can't cancel
return
tdx_order_id = self.orders[order_id].broker_order_id
broker_id = self._client.get_stock_type("{:6d}".format(
self.orders[order_id].asset.sid))
self._client.cancel_orders(broker_id, tdx_order_id)
def get_last_traded_dt(self, asset):
self.subscribe_to_market_data(asset)
self._update_bars()
return self._bars[str(asset.symbol)].index[-1]
def get_spot_value(self, assets, field, dt, data_frequency):
symbol = str(assets.symbol)
self.subscribe_to_market_data(assets)
self._update_bars()
bars = self._bars[symbol]
last_event_time = bars.index[-1]
minute_start = (last_event_time - pd.Timedelta('1 min')) \
.time()
minute_end = last_event_time.time()
if bars.empty:
return pd.NaT if field == 'last_traded' else np.NaN
else:
if field == 'price':
return bars.price.iloc[-1]
elif field == 'last_traded':
return last_event_time or pd.NaT
minute_df = bars.between_time(minute_start,
minute_end,
include_start=True,
include_end=True)
if minute_df.empty:
return np.NaN
else:
if field == 'open':
return minute_df.price.iloc[0]
elif field == 'close':
return minute_df.price.iloc[-1]
elif field == 'high':
return minute_df.price.max()
elif field == 'low':
return minute_df.price.min()
elif field == 'volume':
return minute_df.vol.sum()
def get_realtime_bars(self, assets, frequency):
if frequency == '1m':
resample_freq = 'Min'
elif frequency == '1d':
resample_freq = '24 H'
else:
raise ValueError("Invalid frequency specified: %s" % frequency)
df = pd.DataFrame()
for asset in assets:
symbol = str(asset.symbol)
self.subscribe_to_market_data(asset)
self._update_bars()
trade_prices = self._bars[symbol]['price']
trade_sizes = self._bars[symbol]['vol']
ohlcv = trade_prices.resample(resample_freq,
label='right',
closed='left').ohlc()
ohlcv['volume'] = trade_sizes.resample(resample_freq,
label='right',
closed='left').sum()
ohlcv.columns = pd.MultiIndex.from_product([[
asset,
], ohlcv.columns])
df = pd.concat([df, ohlcv], axis=1)
return df
eg.exit()
def register_tdx(assets=None, minute=False, start=None, fundamental=False, end=None):
try:
bundles.unregister('tdx')
except bundles.UnknownBundle:
pass
calendar = get_calendar('SHSZ')
if start:
if not calendar.is_session(start):
start = calendar.all_sessions[searchsorted(calendar.all_sessions, start)]
bundles.register('tdx', partial(tdx_bundle, assets, minute, fundamental), 'SHSZ', start, end,
minutes_per_day=240)
bundles.register('tdx', partial(tdx_bundle, None, False, False), minutes_per_day=240)
if __name__ == '__main__':
eg = Engine(auto_retry=True, multithread=True, thread_num=8)
with eg.connect():
symbols = fetch_symbols(eg)
symbols = symbols[:3]
data = []
metas = []
for symbol in symbols.symbol:
data.append((int(symbol), fetch_single_equity(eg, symbol)))
metas.append(get_meta_from_bars(data[-1][1]))
symbols = pd.concat([symbols, pd.DataFrame(data=metas)], axis=1)
splits, dividends = fetch_splits_and_dividends(eg, symbols)
def transactions():
eg = Engine(best_ip=True, auto_retry=True)
eg.connect()
m1 = eg.get_k_data('000001', '20170101', '20180101', '1m')
def main():
logbook.StderrHandler().push_application()
engine = Engine(best_ip=True, thread_num=1)
with engine.connect():
engine.get_k_data('000002', '20100921', '20100930', '1m')
def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day):
"""
Ensure we have benchmark data for `symbol` from `first_date` to `last_date`
Parameters
----------
symbol : str
The symbol for the benchmark to load.
first_date : pd.Timestamp
First required date for the cache.
last_date : pd.Timestamp
Last required date for the cache.
now : pd.Timestamp
The current time. This is used to prevent repeated attempts to
re-download data that isn't available due to scheduling quirks or other
failures.
trading_day : pd.CustomBusinessDay
A trading day delta. Used to find the day before first_date so we can
get the close of the day prior to first_date.
We attempt to download data unless we already have data stored at the data
cache for `symbol` whose first entry is before or on `first_date` and whose
last entry is on or after `last_date`.
If we perform a download and the cache criteria are not satisfied, we wait
at least one hour before attempting a redownload. This is determined by
comparing the current time to the result of os.path.getmtime on the cache
path.
"""
path = get_data_filepath(get_benchmark_filename(symbol))
# If the path does not exist, it means the first download has not happened
# yet, so don't try to read from 'path'.
if os.path.exists(path):
try:
data = pd.Series.from_csv(path).tz_localize('UTC')
if has_data_for_dates(data, first_date, last_date):
return data
# Don't re-download if we've successfully downloaded and written a
# file in the last hour.
last_download_time = last_modified_time(path)
if (now - last_download_time) <= ONE_HOUR:
logger.warn(
"Refusing to download new benchmark data because a "
"download succeeded at %s." % last_download_time
)
return data
except (OSError, IOError, ValueError) as e:
# These can all be raised by various versions of pandas on various
# classes of malformed input. Treat them all as cache misses.
logger.info(
"Loading data for {path} failed with error [{error}].".format(
path=path, error=e,
)
)
logger.info(
"Cache at {path} does not have data from {start} to {end}.\n"
"Downloading benchmark data for '{symbol}'.",
start=first_date,
end=last_date,
symbol=symbol,
path=path,
)
engine = Engine(auto_retry=True, multithread=True, thread_num=8, best_ip=True)
engine.connect()
data = engine.get_security_bars(symbol, '1d',index=True)
data = data['close'].sort_index().tz_localize('UTC').pct_change(1).iloc[1:]
data.index = data.index.normalize() # change datetime at 15:00 to midnight
data.to_csv(path)
return data
print(grouped.sort_values('up_limit', ascending=False))
def minute_time_data():
stock_list = engine.stock_list.index.tolist()
now = datetime.datetime.now()
for stock in stock_list:
fs = engine.api.to_df(
engine.api.get_minute_time_data(stock[0], stock[1]))
# print(fs)
print((datetime.datetime.now() - now).total_seconds())
def quotes():
start_dt = datetime.datetime.now()
quote = engine.stock_quotes()
print(datetime.datetime.now() - start_dt).total_seconds()
process_quotes(quote)
if __name__ == '__main__':
engine = Engine(best_ip=True)
with engine.connect():
print(
engine.get_security_bars('002920', '1d',
pd.to_datetime('20170701')))
class TdxBroker(Broker):
def __init__(self, tdx_uri, account_id=None):
self._orders = {}
if tdx_uri.startswith('tcp'):
self._client = zerorpc.Client()
self._client.connect(tdx_uri)
elif platform.architecture()[0] == '32bit':
self._client = TdxClient(tdx_uri)
self._client.login()
else:
raise Exception("please use 32bit python to use local client directly, or use tcp client")
self.currency = 'RMB'
self._subscribed_assets = []
self._bars = {}
self._bars_update_dt = None
self._bars_update_interval = pd.tslib.Timedelta('5 S')
self._mkt_client = Engine(auto_retry=True, best_ip=True)
self._mkt_client.connect()
super(TdxBroker, self).__init__()
def subscribe_to_market_data(self, asset):
# TODO fix me subcribe_to_market_data
if asset not in self.subscribed_assets:
# remove str() cast to have a fun debugging journey
# self._client.subscribe_to_market_data(str(asset.symbol))
self._subscribed_assets.append(asset)
self._bars_update_dt = None
def _update_bars(self):
now = pd.to_datetime('now')
if self._bars_update_dt and (now - self._bars_update_dt < self._bars_update_interval):
return
for code in self.subscribed_assets:
bars = self._mkt_client.time_and_price(code.symbol)
if bars.empty:
return
bars.index = bars.index.tz_localize('Asia/Shanghai').tz_convert('UTC')
self._bars[code.symbol] = bars
self._bars_update_dt = now
@property
def subscribed_assets(self):
return self._subscribed_assets
def is_alive(self):
return True
@property
def positions(self):
now = datetime.datetime.now()
z_positions = protocol.Positions()
for pos in self._client.positions():
if isinstance(pos, list):
pos = TdxPosition(*pos)
sid = pos.sid
available = pos.available
z_position = protocol.Position(symbol(sid))
z_position.amount = pos.amount
z_position.cost_basis = pos.cost_basis
z_position.last_sale_price = pos.last_sale_price
z_position.last_sale_date = now
z_positions[symbol(sid)] = z_position
return z_positions
@property
def portfolio(self):
z_portfolio = protocol.Portfolio()
pfo = self._client.portfolio()
if isinstance(pfo, list):
pfo = TdxPortfolio(*pfo)
z_portfolio.capital_used = None # TODO
z_portfolio.starting_cash = None
z_portfolio.portfolio_value = pfo.portfolio_value
z_portfolio.pnl = None
z_portfolio.returns = None
z_portfolio.cash = pfo.cash
z_portfolio.start_date = None
z_portfolio.positions = self.positions
z_portfolio.positions_value = pfo.positions_value
z_portfolio.position_exposure = z_portfolio.positions_value / (z_portfolio.positions_value + z_portfolio.cash)
return z_portfolio
@property
def account(self):
z_account = protocol.Account()
z_account.settled_cash = self.portfolio.cash
z_account.accrued_interest = None
z_account.buying_power = self.portfolio.portfolio_value
z_account.equity_with_loan = self.portfolio.portfolio_value
z_account.total_positions_value = self.portfolio.positions_value
z_account.total_positions_exposure = z_account.total_positions_value / (
z_account.total_positions_value + z_account.settled_cash)
return z_account
@property
def time_skew(self):
return pd.Timedelta('1 S')
def order(self, asset, amount, style):
code = asset.symbol
if amount > 0:
action = BUY
else:
action = SELL
is_busy = (amount > 0)
if isinstance(style, MarketOrder):
order_type = FIVE_LEVEL_MARKET_ORDER
price = 0.0
elif isinstance(style, LimitOrder):
order_type = LIMIT_CHARGE
price = style.get_limit_price(is_busy)
elif isinstance(style, StopOrder):
raise Exception("stop order is not supported")
elif isinstance(style, StopLimitOrder):
raise Exception("stop limit order is not supported")
data, err = self._client.order(code, abs(amount), price, action, order_type)
if isinstance(data,list):
data = OrderRt(*data)
order_id = str(data.order_id)
zp_order = self._get_or_create_zp_order(order_id)
log.info("Placing order-{order_id}: "
"{action} {qty} {symbol} with {order_type}, price = {price}".format(
order_id=order_id,
action=action,
qty=amount,
symbol=code,
order_type=order_type,
price=price
))
return zp_order
@property
def orders(self):
self._update_orders()
return self._orders
def _get_or_create_zp_order(self, order_id, order=None):
zp_order_id = self._tdx_to_zp_order_id(order_id)
if zp_order_id in self._orders:
return self._orders[zp_order_id]
if not order:
order = self._client.orders()[order_id]
if isinstance(order, list): # handle rpc response for namedtuple object
order = TdxOrder(*order)
self._orders[zp_order_id] = self.tdx_order_to_zipline_order(order)
return self._orders[zp_order_id]
# amount 可正数可负数
def _create_zp_order(self, order_id, asset, price, amount, order_type):
zp_order_id = self._tdx_to_zp_order_id(order_id)
if zp_order_id in self._orders:
return self._orders[zp_order_id]
dt = pd.to_datetime("now", utc=True)
self._orders[zp_order_id] = ZPOrder(
dt=dt,
asset=asset,
amount=amount,
stop=None,
limit=price if order_type is LIMIT_CHARGE else None,
id=zp_order_id,
broker_order_id=order_id
)
return self._orders[zp_order_id]
def _tdx_to_zp_order_id(self, order_id):
return "TDX-{date}-{account_id}-{order_id}".format(
date=str(pd.to_datetime('today').date()),
account_id=self._client.account_id(),
order_id=order_id
)
def tdx_order_to_zipline_order(self, order):
if order.status == '已撤':
zp_status = ZP_ORDER_STATUS.CANCELLED
elif order.filled == 0:
zp_status = ZP_ORDER_STATUS.OPEN
else:
zp_status = ZP_ORDER_STATUS.FILLED
zp_order_id = self._tdx_to_zp_order_id(order.order_id)
od = ZPOrder(
dt=pd.to_datetime(order.dt),
asset=symbol(order.symbol),
amount=order.amount,
filled=order.filled,
stop=None,
limit=order.price, # TODO 市价单和限价单
id=zp_order_id,
)
od.broker_order_id = order.order_id
od.status = zp_status
return od
def _update_orders(self):
ods = self._client.orders()
for tdx_order_id, tdx_order in iteritems(ods):
if isinstance(tdx_order, list): # handle rpc response for namedtuple object
tdx_order = TdxOrder(*tdx_order)
zp_order_id = self._tdx_to_zp_order_id(tdx_order_id)
self._orders[zp_order_id] = self.tdx_order_to_zipline_order(tdx_order)
def _tdx_transaction_to_zipline(self, transaction):
return ZPTransaction(
asset=symbol(transaction.asset),
amount=transaction.amount,
dt=pd.to_datetime(transaction.dt).tz_localize('Asia/Shanghai').tz_convert('UTC'),
price=transaction.price,
order_id=self._tdx_to_zp_order_id(transaction.order_id),
commission=transaction.commission,
)
@property
def transactions(self):
t = self._client.transactions()
rt = {}
for exec_id, transaction in t.items():
try:
if isinstance(transaction, list): # handle rpc response for namedtuple object
transaction = TdxTransaction(*transaction)
rt[exec_id] = self._tdx_transaction_to_zipline(transaction)
except SymbolNotFound as e:
log.warning(e.message)
return rt
def cancel_order(self, order_id):
if order_id not in self.orders: # order become transaction, can't cancel
return
tdx_order_id = self.orders[order_id].broker_order_id
broker_id = self._client.get_stock_type("{:6d}".format(self.orders[order_id].asset.sid))
self._client.cancel_orders(broker_id, tdx_order_id)
def get_last_traded_dt(self, asset):
self.subscribe_to_market_data(asset)
self._update_bars()
return self._bars[str(asset.symbol)].index[-1]
def get_spot_value(self, assets, field, dt, data_frequency):
symbol = str(assets.symbol)
self.subscribe_to_market_data(assets)
self._update_bars()
if symbol not in self._bars:
return pd.NaT if field == 'last_traded' else np.NaN
bars = self._bars[symbol]
last_event_time = bars.index[-1]
minute_start = (last_event_time - pd.Timedelta('1 min')) \
.time()
minute_end = last_event_time.time()
if bars.empty:
return pd.NaT if field == 'last_traded' else np.NaN
else:
if field == 'price':
return bars.price.iloc[-1]
elif field == 'last_traded':
return last_event_time or pd.NaT
minute_df = bars.between_time(minute_start, minute_end,
include_start=True, include_end=True)
if minute_df.empty:
return np.NaN
else:
if field == 'open':
return minute_df.price.iloc[0]
elif field == 'close':
return minute_df.price.iloc[-1]
elif field == 'high':
return minute_df.price.max()
elif field == 'low':
return minute_df.price.min()
elif field == 'volume':
return minute_df.vol.sum()
def get_realtime_bars(self, assets, frequency):
if frequency == '1m':
resample_freq = 'Min'
elif frequency == '1d':
resample_freq = 'd'
else:
raise ValueError("Invalid frequency specified: %s" % frequency)
df = pd.DataFrame()
for asset in assets:
symbol = str(asset.symbol)
self.subscribe_to_market_data(asset)
self._update_bars()
trade_prices = self._bars[symbol]['price']
trade_sizes = self._bars[symbol]['vol']
ohlcv = trade_prices.resample(resample_freq,
label='left',
closed='left').ohlc()
ohlcv['volume'] = trade_sizes.resample(resample_freq,
label='left',
closed='left').sum()
ohlcv.columns = pd.MultiIndex.from_product([[asset, ],
ohlcv.columns])
df = pd.concat([df, ohlcv], axis=1)
return df
def tdx_bundle(assets,
ingest_minute, # whether to ingest minute data, default False
fundamental, # whether to ingest fundamental data, default False
environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
fundamental_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir):
# eg = Engine(auto_retry=True, multithread=True, best_ip=True, thread_num=1)
eg = Engine(auto_retry=True, multithread=True, best_ip=True, thread_num=1)
eg.connect()
symbols = fetch_symbols(eg, assets)
metas = []
today = pd.to_datetime('today', utc=True)
distance = calendar.session_distance(start_session, today)
dates_path = join(output_dir, DATE_DIR)
if os.path.isfile(dates_path):
with open(dates_path, 'r') as f:
dates_json = json.load(f)
else:
dates_json = {
'1d': {},
'1m': {}
}
session_bars = create_engine('sqlite:///' + join(output_dir, SESSION_BAR_DB))
def gen_symbols_data(symbol_map, freq='1d'):
if not session_bars.has_table(SESSION_BAR_TABLE):
Base.metadata.create_all(session_bars.connect(), checkfirst=True,
tables=[Base.metadata.tables[SESSION_BAR_TABLE]])
func = partial(fetch_single_equity, eg)
now = pd.to_datetime('now', utc=True)
if end_session >= now.normalize():
end = now.normalize()
if now.tz_convert('Asia/Shanghai').time() < datetime.time(15, 5):
end = end - pd.Timedelta('1 D')
else:
end = end_session
end_idx = calendar.all_sessions.searchsorted(end)
if calendar.all_sessions[end_idx] > end:
end = calendar.all_sessions[end_idx -1]
for index, symbol in symbol_map.iteritems():
try:
start = pd.to_datetime(dates_json[freq][symbol], utc=True) + pd.Timedelta('1 D')
start = calendar.all_sessions[calendar.all_sessions.searchsorted(start)]
if start > end:
if freq == '1d'and symbol in dates_json[freq]:
data = pd.read_sql(
"select * from {} where id = {} order by day ASC ".format(SESSION_BAR_TABLE, int(symbol)),
session_bars, index_col='day')
data.index = pd.to_datetime(data.index)
yield int(symbol), data
else:
yield int(symbol), pd.DataFrame()
continue
except KeyError:
start = start_session
if freq == '1m':
single_distance = calendar.session_distance(start, end)
if single_distance >= 100:
func = eg.get_k_data
data = reindex_to_calendar(
calendar,
func(symbol, start, end, freq),
start_session=start, end_session=end,
freq=freq,
)
if data is None or data.empty:
if freq == '1d'and symbol in dates_json[freq]:
data = pd.read_sql(
"select * from {} where id = {} order by day ASC ".format(SESSION_BAR_TABLE, int(symbol)),
session_bars, index_col='day')
data.index = pd.to_datetime(data.index)
yield int(symbol), data
continue
if freq == '1d':
if data.close.isnull()[0]: # padding fill error if the first is NaN
data2 = pd.read_sql(
"select * from {} where id = {} order by day desc limit 1 ".format(SESSION_BAR_TABLE, int(symbol)),
session_bars, index_col='day')
if data2.empty:
data = data[data.close.notnull()]
else:
data["close"][0] = data2["close"][0]
fillna(data)
data.to_sql(SESSION_BAR_TABLE, session_bars.connect(), if_exists='append', index_label='day')
if symbol in dates_json[freq]:
data = pd.read_sql(
"select * from {} where id = {} order by day ASC ".format(SESSION_BAR_TABLE, int(symbol)),
session_bars, index_col='day')
data.index = pd.to_datetime(data.index)
dates_json[freq][symbol] = data.index[-1].strftime('%Y%m%d')
yield int(symbol), data
with open(dates_path, 'w') as f:
json.dump(dates_json, f)
symbol_map = symbols.symbol
assets = set([int(s) for s in symbol_map])
daily_bar_writer.write(gen_symbols_data(symbol_map, freq="1d"), assets=assets, show_progress=show_progress)
splits, dividends, shares = fetch_splits_and_dividends(eg, symbols, start_session, end_session)
metas = pd.read_sql("select id as symbol,min(day) as start_date,max(day) as end_date from bars group by id;",
session_bars,
parse_dates=['start_date','end_date']
)
metas['symbol'] = metas['symbol'].apply(lambda x:format(x,'06'))
metas['first_traded'] = metas['start_date']
metas['auto_close_date'] = metas['end_date']
symbols = symbols.set_index('symbol', drop=False).join(metas.set_index('symbol'), how='inner')
asset_db_writer.write(symbols)
adjustment_writer.write(
splits=splits,
dividends=dividends,
shares=shares
)
if fundamental:
logger.info("writing fundamental data:")
try:
fundamental_writer.write(start_session, end_session)
except Exception as e:
pass
if ingest_minute:
with click.progressbar(gen_symbols_data(symbol_map, freq="1m"),
label="Merging minute equity files:",
length=len(assets),
item_show_func=lambda e: e if e is None else str(e[0]),
) as bar:
minute_bar_writer.write(bar, show_progress=False)
eg.exit()
class TdxShipaneBroker(TdxBroker):
def __init__(self, tdx_uri, shipane_client, account_id=None):
self._shipane_client = shipane_client
self._orders = {}
self.currency = 'RMB'
self._subscribed_assets = []
self._bars = {}
self._bars_update_dt = None
self._bars_update_interval = pd.tslib.Timedelta('5 S')
self._mkt_client = Engine(auto_retry=True, best_ip=True)
self._mkt_client.connect()
# super(TdxShipaneBroker, self).__init__(tdx_uri, account_id)
@property
def positions(self):
now = datetime.datetime.now()
z_positions = protocol.Positions()
for pos in self._shipane_client.positions():
if isinstance(pos, list):
pos = TdxPosition(*pos)
sid = pos.sid
z_position = protocol.Position(symbol(sid))
z_position.available = pos.available
z_position.amount = pos.amount
z_position.cost_basis = pos.cost_basis
z_position.last_sale_price = pos.last_sale_price
z_position.last_sale_date = now
z_positions[symbol(sid)] = z_position
return z_positions
@property
def portfolio(self):
z_portfolio = protocol.Portfolio()
pfo = self._shipane_client.portfolio()
if isinstance(pfo, list):
pfo = TdxPortfolio(*pfo)
z_portfolio.capital_used = None # TODO
z_portfolio.starting_cash = None
z_portfolio.portfolio_value = pfo.portfolio_value
z_portfolio.pnl = None
z_portfolio.returns = None
z_portfolio.cash = pfo.cash
z_portfolio.start_date = None
z_portfolio.positions = self.positions
z_portfolio.positions_value = pfo.positions_value
z_portfolio.position_exposure = z_portfolio.positions_value / (z_portfolio.positions_value + z_portfolio.cash)
return z_portfolio
def order(self, asset, amount, style):
code = asset.symbol
if amount > 0:
action = BUY
else:
action = SELL
is_busy = (amount > 0)
if isinstance(style, MarketOrder):
order_type = FIVE_LEVEL_MARKET_ORDER
price = 0.0
elif isinstance(style, LimitOrder):
order_type = LIMIT_CHARGE
price = style.get_limit_price(is_busy)
elif isinstance(style, StopOrder):
raise Exception("stop order is not supported")
elif isinstance(style, StopLimitOrder):
raise Exception("stop limit order is not supported")
data, err = self._shipane_client.order(code, abs(amount), price, action, order_type)
if isinstance(data,list):
if len(data) > 0:
data = OrderRt(*data)
else:
return None
order_id = str(data.order_id)
zp_order = self._get_or_create_zp_order(order_id)
log.info("Placing order-{order_id}: "
"{action} {qty} {symbol} with {order_type}, price = {price}".format(
order_id=order_id,
action=action,
qty=amount,
symbol=code,
order_type=order_type,
price=price
))
return zp_order
@property
def orders(self):
self._update_orders()
return self._orders
def _get_or_create_zp_order(self, order_id):
zp_order_id = self._tdx_to_zp_order_id(order_id)
if zp_order_id in self.orders:
return self._orders[zp_order_id]
if isinstance(order_id, list): # handle rpc response for namedtuple object
order = TdxOrder(*order_id)
self._orders[zp_order_id] = self.tdx_order_to_zipline_order(order)
return self._orders[zp_order_id]
# amount 可正数可负数
def _create_zp_order(self, order_id, asset, price, amount, order_type):
zp_order_id = self._tdx_to_zp_order_id(order_id)
if zp_order_id in self._orders:
return self._orders[zp_order_id]
dt = pd.to_datetime("now", utc=True)
self._orders[zp_order_id] = ZPOrder(
dt=dt,
asset=asset,
amount=amount,
stop=None,
limit=price if order_type is LIMIT_CHARGE else None,
id=zp_order_id,
broker_order_id=order_id
)
return self._orders[zp_order_id]
def _tdx_to_zp_order_id(self, order_id):
return "TDX-{date}-{account_id}-{order_id}".format(
date=str(pd.to_datetime('today').date()),
account_id="XXX",
order_id=order_id
)
def tdx_order_to_zipline_order(self, order):
if order.status is not None and 'CANCEL' == order.status:
zp_status = ZP_ORDER_STATUS.CANCELLED
elif order.filled == 0:
zp_status = ZP_ORDER_STATUS.OPEN
else:
zp_status = ZP_ORDER_STATUS.FILLED
zp_order_id = self._tdx_to_zp_order_id(order.order_id)
od = ZPOrder(
dt=pd.to_datetime(order.dt),
asset=symbol(order.symbol),
amount=order.amount,
filled=order.filled,
stop=None,
limit=order.price, # TODO 市价单和限价单
id=zp_order_id,
)
od.broker_order_id = order.order_id
od.status = zp_status
return od
def _update_orders(self):
ods = self._shipane_client.orders()
if ods is None or len(ods) <= 0:
return
for tdx_order_id, tdx_order in iteritems(ods):
if isinstance(tdx_order, list): # handle rpc response for namedtuple object
tdx_order = TdxOrder(*tdx_order)
zp_order_id = self._tdx_to_zp_order_id(tdx_order_id)
self._orders[zp_order_id] = self.tdx_order_to_zipline_order(tdx_order)
@property
def transactions(self):
# TODO: do we need the tx record now?
# t = self._client.transactions()
rt = {}
return rt
def cancel_order(self, order_id):
if order_id not in self.orders: # order become transaction, can't cancel
return
tdx_order_id = self.orders[order_id].broker_order_id
self._shipane_client.cancel_orders(0, tdx_order_id)
def main():
engine = Engine(best_ip=True, thread_num=1)
with engine.connect():
engine.get_k_data('000001', '20161201', '20171231', '1m')
def tdx_bundle(
assets,
ingest_minute, # whether to ingest minute data, default False
fundamental, # whether to ingest fundamental data, default False
environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
fundamental_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir):
eg = Engine(auto_retry=True, multithread=True, best_ip=True, thread_num=8)
eg.connect()
symbols = fetch_symbols(eg, assets)
metas = []
today = pd.to_datetime('today', utc=True)
distance = calendar.session_distance(start_session, today)
dates_path = join(output_dir, DATE_DIR)
if os.path.isfile(dates_path):
with open(dates_path, 'r') as f:
dates_json = json.load(f)
else:
dates_json = {'1d': {}, '1m': {}}
session_bars = create_engine('sqlite:///' +
join(output_dir, SESSION_BAR_DB))
def gen_symbols_data(symbol_map, freq='1d'):
if not session_bars.has_table(SESSION_BAR_TABLE):
Base.metadata.create_all(
session_bars.connect(),
checkfirst=True,
tables=[Base.metadata.tables[SESSION_BAR_TABLE]])
func = partial(fetch_single_equity, eg)
now = pd.to_datetime('now', utc=True)
if end_session >= now.normalize():
end = now.normalize()
if now.tz_convert('Asia/Shanghai').time() < datetime.time(15, 5):
end = end - pd.Timedelta('1 D')
else:
end = end_session
if freq == '1m':
if distance >= 100:
func = eg.get_k_data
for index, symbol in symbol_map.iteritems():
try:
start = pd.to_datetime(dates_json[freq][symbol],
utc=True) + pd.Timedelta('1 D')
if start >= end:
continue
except KeyError:
start = start_session
data = reindex_to_calendar(
calendar,
func(symbol, start, end, freq),
freq=freq,
)
if freq == '1d':
data.to_sql(SESSION_BAR_TABLE,
session_bars.connect(),
if_exists='append',
index_label='day')
if symbol in dates_json[freq]:
data = pd.read_sql(
"select * from {} where id = {} order by day ASC ".
format(SESSION_BAR_TABLE, int(symbol)),
session_bars,
index_col='day')
data.index = pd.to_datetime(data.index)
dates_json[freq][symbol] = end.strftime('%Y%m%d')
yield int(symbol), data
with open(dates_path, 'w') as f:
json.dump(dates_json, f)
symbol_map = symbols.symbol
assets = set([int(s) for s in symbol_map])
daily_bar_writer.write(gen_symbols_data(symbol_map, freq="1d"),
assets=assets,
show_progress=show_progress)
if ingest_minute:
with click.progressbar(
gen_symbols_data(symbol_map, freq="1m"),
label="Merging minute equity files:",
length=len(assets),
item_show_func=lambda e: e if e is None else str(e[0]),
) as bar:
minute_bar_writer.write(bar, show_progress=False)
splits, dividends, shares = fetch_splits_and_dividends(
eg, symbols, start_session, end_session)
metas = pd.read_sql(
"select id as symbol,min(day) as start_date,max(day) as end_date from bars group by id;",
session_bars,
parse_dates=['start_date', 'end_date'])
metas['symbol'] = metas['symbol'].apply(lambda x: format(x, '06'))
metas['first_traded'] = metas['start_date']
metas['auto_close_date'] = metas['end_date']
symbols = symbols.set_index('symbol',
drop=False).join(metas.set_index('symbol'),
how='inner')
asset_db_writer.write(symbols)
adjustment_writer.write(splits=splits, dividends=dividends, shares=shares)
if fundamental:
logger.info("writing fundamental data:")
try:
fundamental_writer.write(start_session, end_session)
except Exception as e:
pass
eg.exit()
def tdx_bundle(assets,
ingest_minute, # whether to ingest minute data, default False
overwrite,
environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir):
eg = Engine(auto_retry=True, multithread=True, best_ip=True, thread_num=8)
eg.connect()
symbols = fetch_symbols(eg, assets)
metas = []
today = pd.to_datetime('today',utc=True)
distance = calendar.session_distance(start_session, today)
if ingest_minute and not overwrite and (start_session < today - pd.DateOffset(years=3)):
minute_start = calendar.all_sessions[searchsorted(calendar.all_sessions, today - pd.DateOffset(years=3))]
logger.warning(
"overwrite start_session for minute bars to {}(3 years),"
" to fetch minute data before that, please add '--overwrite True'".format(minute_start))
else:
minute_start = start_session
def gen_symbols_data(symbol_map, freq='1d'):
func = partial(fetch_single_equity, eg)
start = start_session
end = end_session
if freq == '1m':
if distance >= 100:
func = eg.get_k_data
start = minute_start
for index, symbol in symbol_map.iteritems():
data = reindex_to_calendar(
calendar,
func(symbol, start, end, freq),
freq=freq,
)
if freq == '1d':
metas.append(get_meta_from_bars(data))
yield int(symbol), data
symbol_map = symbols.symbol
assets = set([int(s) for s in symbol_map])
daily_bar_writer.write(gen_symbols_data(symbol_map, freq="1d"), assets=assets, show_progress=show_progress)
if ingest_minute:
with click.progressbar(gen_symbols_data(symbol_map, freq="1m"),
label="Merging minute equity files:",
length=len(assets),
item_show_func=lambda e: e if e is None else str(e[0]),
) as bar:
minute_bar_writer.write(bar, show_progress=False)
symbols = pd.concat([symbols, pd.DataFrame(data=metas)], axis=1)
splits, dividends = fetch_splits_and_dividends(eg, symbols)
symbols.set_index('symbol', drop=False, inplace=True)
asset_db_writer.write(symbols)
adjustment_writer.write(
splits=splits,
dividends=dividends
)
eg.exit()
def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day):
"""
Ensure we have benchmark data for `symbol` from `first_date` to `last_date`
Parameters
----------
symbol : str
The symbol for the benchmark to load.
first_date : pd.Timestamp
First required date for the cache.
last_date : pd.Timestamp
Last required date for the cache.
now : pd.Timestamp
The current time. This is used to prevent repeated attempts to
re-download data that isn't available due to scheduling quirks or other
failures.
trading_day : pd.CustomBusinessDay
A trading day delta. Used to find the day before first_date so we can
get the close of the day prior to first_date.
We attempt to download data unless we already have data stored at the data
cache for `symbol` whose first entry is before or on `first_date` and whose
last entry is on or after `last_date`.
If we perform a download and the cache criteria are not satisfied, we wait
at least one hour before attempting a redownload. This is determined by
comparing the current time to the result of os.path.getmtime on the cache
path.
"""
path = get_data_filepath(get_benchmark_filename(symbol))
# If the path does not exist, it means the first download has not happened
# yet, so don't try to read from 'path'.
if os.path.exists(path):
try:
data = pd.Series.from_csv(path).tz_localize('UTC')
if has_data_for_dates(data, first_date, last_date):
return data
# Don't re-download if we've successfully downloaded and written a
# file in the last hour.
last_download_time = last_modified_time(path)
if (now - last_download_time) <= ONE_HOUR:
logger.warn(
"Refusing to download new benchmark data because a "
"download succeeded at %s." % last_download_time)
return data
except (OSError, IOError, ValueError) as e:
# These can all be raised by various versions of pandas on various
# classes of malformed input. Treat them all as cache misses.
logger.info(
"Loading data for {path} failed with error [{error}].".format(
path=path,
error=e,
))
logger.info(
"Cache at {path} does not have data from {start} to {end}.\n"
"Downloading benchmark data for '{symbol}'.",
start=first_date,
end=last_date,
symbol=symbol,
path=path,
)
engine = Engine(auto_retry=True, multithread=True, thread_num=8)
engine.connect()
data = engine.get_security_bars(symbol, '1d', index=True)
data = data['close'].sort_index().tz_localize('UTC').pct_change(1).iloc[1:]
data.index = data.index.shift(-15,
'1H') # change datetime at 15:00 to midnight
data.to_csv(path)
return data
class TdxShipaneBroker(TdxBroker):
def __init__(self, tdx_uri, shipane_client, account_id=None):
self._shipane_client = shipane_client
self._orders = {}
self.currency = 'RMB'
self._subscribed_assets = []
self._bars = {}
self._bars_update_dt = None
self._bars_update_interval = pd.tslib.Timedelta('5 S')
self._mkt_client = Engine(auto_retry=True, best_ip=True)
self._mkt_client.connect()
# super(TdxShipaneBroker, self).__init__(tdx_uri, account_id)
@property
def positions(self):
now = datetime.datetime.now()
z_positions = protocol.Positions()
for pos in self._shipane_client.positions():
if isinstance(pos, list):
pos = TdxPosition(*pos)
sid = pos.sid
z_position = protocol.Position(symbol(sid))
z_position.available = pos.available
z_position.amount = pos.amount
z_position.cost_basis = pos.cost_basis
z_position.last_sale_price = pos.last_sale_price
z_position.last_sale_date = now
z_positions[symbol(sid)] = z_position
return z_positions
@property
def portfolio(self):
z_portfolio = protocol.Portfolio()
pfo = self._shipane_client.portfolio()
if isinstance(pfo, list):
pfo = TdxPortfolio(*pfo)
z_portfolio.capital_used = None # TODO
z_portfolio.starting_cash = None
z_portfolio.portfolio_value = pfo.portfolio_value
z_portfolio.pnl = None
z_portfolio.returns = None
z_portfolio.cash = pfo.cash
z_portfolio.start_date = None
z_portfolio.positions = self.positions
z_portfolio.positions_value = pfo.positions_value
z_portfolio.position_exposure = z_portfolio.positions_value / (z_portfolio.positions_value + z_portfolio.cash)
return z_portfolio
def order(self, asset, amount, style):
code = asset.symbol
if amount > 0:
action = BUY
else:
action = SELL
is_busy = (amount > 0)
if isinstance(style, MarketOrder):
order_type = FIVE_LEVEL_MARKET_ORDER
price = 0.0
elif isinstance(style, LimitOrder):
order_type = LIMIT_CHARGE
price = style.get_limit_price(is_busy)
elif isinstance(style, StopOrder):
raise Exception("stop order is not supported")
elif isinstance(style, StopLimitOrder):
raise Exception("stop limit order is not supported")
data, err = self._shipane_client.order(code, abs(amount), price, action, order_type)
if isinstance(data,list):
if len(data) > 0:
data = OrderRt(*data)
else:
return None
order_id = str(data.order_id)
zp_order = self._get_or_create_zp_order(order_id)
log.info("Placing order-{order_id}: "
"{action} {qty} {symbol} with {order_type}, price = {price}".format(
order_id=order_id,
action=action,
qty=amount,
symbol=code,
order_type=order_type,
price=price
))
return zp_order
@property
def orders(self):
self._update_orders()
return self._orders
def _get_or_create_zp_order(self, order_id):
zp_order_id = self._tdx_to_zp_order_id(order_id)
if zp_order_id in self.orders:
return self._orders[zp_order_id]
if isinstance(order, list): # handle rpc response for namedtuple object
order = TdxOrder(*order)
self._orders[zp_order_id] = self.tdx_order_to_zipline_order(order)
return self._orders[zp_order_id]
# amount 可正数可负数
def _create_zp_order(self, order_id, asset, price, amount, order_type):
zp_order_id = self._tdx_to_zp_order_id(order_id)
if zp_order_id in self._orders:
return self._orders[zp_order_id]
dt = pd.to_datetime("now", utc=True)
self._orders[zp_order_id] = ZPOrder(
dt=dt,
asset=asset,
amount=amount,
stop=None,
limit=price if order_type is LIMIT_CHARGE else None,
id=zp_order_id,
broker_order_id=order_id
)
return self._orders[zp_order_id]
def _tdx_to_zp_order_id(self, order_id):
return "TDX-{date}-{account_id}-{order_id}".format(
date=str(pd.to_datetime('today').date()),
account_id="XXX",
order_id=order_id
)
def tdx_order_to_zipline_order(self, order):
if order.status is not None and 'CANCEL' == order.status:
zp_status = ZP_ORDER_STATUS.CANCELLED
elif order.filled == 0:
zp_status = ZP_ORDER_STATUS.OPEN
else:
zp_status = ZP_ORDER_STATUS.FILLED
zp_order_id = self._tdx_to_zp_order_id(order.order_id)
od = ZPOrder(
dt=pd.to_datetime(order.dt),
asset=symbol(order.symbol),
amount=order.amount,
filled=order.filled,
stop=None,
limit=order.price, # TODO 市价单和限价单
id=zp_order_id,
)
od.broker_order_id = order.order_id
od.status = zp_status
return od
def _update_orders(self):
ods = self._shipane_client.orders()
if ods is None or len(ods) <= 0:
return
for tdx_order_id, tdx_order in iteritems(ods):
if isinstance(tdx_order, list): # handle rpc response for namedtuple object
tdx_order = TdxOrder(*tdx_order)
zp_order_id = self._tdx_to_zp_order_id(tdx_order_id)
self._orders[zp_order_id] = self.tdx_order_to_zipline_order(tdx_order)
@property
def transactions(self):
# TODO: do we need the tx record now?
# t = self._client.transactions()
rt = {}
return rt
def cancel_order(self, order_id):
if order_id not in self.orders: # order become transaction, can't cancel
return
tdx_order_id = self.orders[order_id].broker_order_id
self._shipane_client.cancel_orders(0, tdx_order_id)