def original_engine_quotes():
engine = Engine(best_ip=True)
engine.connect()
engine.stock_list.index.tolist()
now = datetime.now()
engine.stock_quotes()
return (datetime.now() - now).total_seconds()
def concurrent_engine_quotes(num=4):
engine = Engine(best_ip=True, thread_num=num)
engine.connect()
engine.stock_list.index.tolist()
now = datetime.now()
engine.stock_quotes()
return (datetime.now() - now).total_seconds()
def test_data():
eg = Engine(auto_retry=True, multithread=True, thread_num=8)
with eg.connect():
symbols = fetch_symbols(eg)
symbols = symbols[:5]
metas = []
def gen_symbols_data(symbol_map, freq='1d'):
for index, symbol in symbol_map.iteritems():
data = fetch_single_equity(eg, symbol, freq)
if freq == '1d':
metas.append(get_meta_from_bars(data))
assert data is not None
yield int(symbol), data
symbol_map = symbols.symbol
assets = set([int(s) for s in symbol_map])
gen_symbols_data(symbol_map, freq="1d")
gen_symbols_data(symbol_map, freq="1m")
symbols = pd.concat([symbols, pd.DataFrame(data=metas)], axis=1)
splits, dividends = fetch_splits_and_dividends(eg, symbols)
symbols.set_index('symbol', drop=False, inplace=True)
assert symbols is not None
assert splits is not None
assert dividends is not None
def transactions():
eg = Engine(best_ip=True)
eg.connect()
m1 = eg.get_security_bars('000001', '1m')
df = eg.time_and_price('000001')
ohlcv = df.price.resample('1 Min', label='right', closed='left').ohlc()
ohlcv['volume'] = df.vol.resample('1 Min', label='right',
closed='left').sum()
def test_transaction():
engine = Engine(best_ip=True, thread_num=1)
with engine.connect():
df = engine.get_k_data('000001', '20170601', '20171231', '1m')
df = engine.get_security_bars(['000001', '000521'],
'1d',
start=pd.to_datetime('20180102'))
def __init__(self, tdx_uri, shipane_client, account_id=None):
self._shipane_client = shipane_client
self._orders = {}
self.currency = 'RMB'
self._subscribed_assets = []
self._bars = {}
self._bars_update_dt = None
self._bars_update_interval = pd.tslib.Timedelta('5 S')
self._mkt_client = Engine(auto_retry=True, best_ip=True)
self._mkt_client.connect()
def tdx_bundle(assets,
ingest_minute, # whether to ingest minute data, default False
environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir):
eg = Engine(auto_retry=True, multithread=True, best_ip=True, thread_num=8)
eg.connect()
symbols = fetch_symbols(eg, assets)
metas = []
def gen_symbols_data(symbol_map, freq='1d'):
for index, symbol in symbol_map.iteritems():
data = reindex_to_calendar(
calendar,
fetch_single_equity(eg, symbol, freq),
freq=freq,
)
if freq == '1d':
metas.append(get_meta_from_bars(data))
yield int(symbol), data
symbol_map = symbols.symbol
assets = set([int(s) for s in symbol_map])
daily_bar_writer.write(gen_symbols_data(symbol_map, freq="1d"), assets=assets, show_progress=show_progress)
if ingest_minute:
with click.progressbar(gen_symbols_data(symbol_map, freq="1m"),
label="Merging minute equity files:",
length=len(assets),
item_show_func=lambda e: e if e is None else str(e[0]),
) as bar:
minute_bar_writer.write(bar, show_progress=False)
symbols = pd.concat([symbols, pd.DataFrame(data=metas)], axis=1)
splits, dividends = fetch_splits_and_dividends(eg, symbols)
symbols.set_index('symbol', drop=False, inplace=True)
asset_db_writer.write(symbols)
adjustment_writer.write(
splits=splits,
dividends=dividends
)
eg.exit()
def __init__(self, cats_client=None):
"""
:param cat_client:
:type cat_client: CatsTrade
"""
self._shipane_client = cats_client
self._orders = {}
self.currency = 'RMB'
self._subscribed_assets = []
self._bars = {}
self._bars_update_dt = None
self._bars_update_interval = pd.tslib.Timedelta('5 S')
self._mkt_client = Engine(auto_retry=True, best_ip=True)
self._mkt_client.connect()
def test_security_list():
engine = Engine(best_ip=True)
engine.connect()
code = engine.stock_list.index.tolist()
api = TdxHq_API()
api.connect()
best_ip = engine.best_ip
print("security list: ({},{})".format(concurrent_api(2),
original_api()))
print("concurrent quotes ({},{})".format(
concurrent_quotes(code, best_ip, 2),
original_quotes(code, best_ip)))
print("concurrent engine quotes ({},{})".format(
concurrent_engine_quotes(2), original_engine_quotes()))
def engine_func(best_ip, thread_num):
engine = Engine(best_ip=best_ip, thread_num=thread_num)
with engine.connect():
assert engine.best_ip is not None
assert engine.gbbq is not None
assert engine.security_list is not None
assert engine.stock_quotes() is not None
assert engine.customer_block is not None
assert engine.quotes('000001') is not None
assert engine.get_security_bars('000001', '1m') is not None
assert engine.get_security_bars('000001', '1d') is not None
assert engine.get_security_bars('000300', '1m', index=True) is not None
assert engine.get_security_bars('000300', '1d', index=True) is not None
assert engine.concept is not None
assert engine.fengge is not None
assert engine.index is not None
assert engine.stock_list is not None
def __init__(self, tdx_uri, account_id=None):
self._orders = {}
if tdx_uri.startswith('tcp'):
self._client = zerorpc.Client()
self._client.connect(tdx_uri)
elif platform.architecture()[0] == '32bit':
self._client = TdxClient(tdx_uri)
self._client.login()
else:
raise Exception("please use 32bit python to use local client directly, or use tcp client")
self.currency = 'RMB'
self._subscribed_assets = []
self._bars = {}
self._bars_update_dt = None
self._bars_update_interval = pd.tslib.Timedelta('5 S')
self._mkt_client = Engine(auto_retry=True, best_ip=True)
self._mkt_client.connect()
super(self.__class__, self).__init__()
#!/usr/bin/env python3
# coding: utf-8
import pymongo
from tdx.engine import Engine
def get_stock_list(engine):
with engine.connect():
return engine.stock_list
if __name__ == '__main__':
conn = pymongo.MongoClient('192.168.0.114', 27016)
stock_tick_db = conn.stock_tick
code_finishing_post = stock_tick_db.code_finishing
eg = Engine(auto_retry=True,
multithread=True,
best_ip=True,
thread_num=1,
raise_exception=True)
stock_list = get_stock_list(eg)
index = 1
for code in stock_list.code:
code_finishing_post.insert({'code': code, 'index': index})
index = index + 1
pass
calendar = get_calendar('SHSZ')
if start:
if not calendar.is_session(start):
start = calendar.all_sessions[searchsorted(calendar.all_sessions,
start)]
bundles.register('tdx',
partial(tdx_bundle, assets, minute, fundamental),
'SHSZ',
start,
end,
minutes_per_day=240)
bundles.register('tdx',
partial(tdx_bundle, None, False, False),
minutes_per_day=240)
if __name__ == '__main__':
eg = Engine(auto_retry=True, multithread=True, thread_num=8)
with eg.connect():
symbols = fetch_symbols(eg)
symbols = symbols[:3]
data = []
metas = []
for symbol in symbols.symbol:
data.append((int(symbol), fetch_single_equity(eg, symbol)))
metas.append(get_meta_from_bars(data[-1][1]))
symbols = pd.concat([symbols, pd.DataFrame(data=metas)], axis=1)
splits, dividends = fetch_splits_and_dividends(eg, symbols)
def tdx_bundle(
assets,
ingest_minute, # whether to ingest minute data, default False
fundamental, # whether to ingest fundamental data, default False
environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
fundamental_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir):
eg = Engine(auto_retry=True, multithread=True, best_ip=True, thread_num=8)
eg.connect()
symbols = fetch_symbols(eg, assets)
metas = []
today = pd.to_datetime('today', utc=True)
distance = calendar.session_distance(start_session, today)
dates_path = join(output_dir, DATE_DIR)
if os.path.isfile(dates_path):
with open(dates_path, 'r') as f:
dates_json = json.load(f)
else:
dates_json = {'1d': {}, '1m': {}}
session_bars = create_engine('sqlite:///' +
join(output_dir, SESSION_BAR_DB))
def gen_symbols_data(symbol_map, freq='1d'):
if not session_bars.has_table(SESSION_BAR_TABLE):
Base.metadata.create_all(
session_bars.connect(),
checkfirst=True,
tables=[Base.metadata.tables[SESSION_BAR_TABLE]])
func = partial(fetch_single_equity, eg)
now = pd.to_datetime('now', utc=True)
if end_session >= now.normalize():
end = now.normalize()
if now.tz_convert('Asia/Shanghai').time() < datetime.time(15, 5):
end = end - pd.Timedelta('1 D')
else:
end = end_session
if freq == '1m':
if distance >= 100:
func = eg.get_k_data
for index, symbol in symbol_map.iteritems():
try:
start = pd.to_datetime(dates_json[freq][symbol],
utc=True) + pd.Timedelta('1 D')
if start >= end:
continue
except KeyError:
start = start_session
data = reindex_to_calendar(
calendar,
func(symbol, start, end, freq),
freq=freq,
)
if freq == '1d':
data.to_sql(SESSION_BAR_TABLE,
session_bars.connect(),
if_exists='append',
index_label='day')
if symbol in dates_json[freq]:
data = pd.read_sql(
"select * from {} where id = {} order by day ASC ".
format(SESSION_BAR_TABLE, int(symbol)),
session_bars,
index_col='day')
data.index = pd.to_datetime(data.index)
dates_json[freq][symbol] = end.strftime('%Y%m%d')
yield int(symbol), data
with open(dates_path, 'w') as f:
json.dump(dates_json, f)
symbol_map = symbols.symbol
assets = set([int(s) for s in symbol_map])
daily_bar_writer.write(gen_symbols_data(symbol_map, freq="1d"),
assets=assets,
show_progress=show_progress)
if ingest_minute:
with click.progressbar(
gen_symbols_data(symbol_map, freq="1m"),
label="Merging minute equity files:",
length=len(assets),
item_show_func=lambda e: e if e is None else str(e[0]),
) as bar:
minute_bar_writer.write(bar, show_progress=False)
splits, dividends, shares = fetch_splits_and_dividends(
eg, symbols, start_session, end_session)
metas = pd.read_sql(
"select id as symbol,min(day) as start_date,max(day) as end_date from bars group by id;",
session_bars,
parse_dates=['start_date', 'end_date'])
metas['symbol'] = metas['symbol'].apply(lambda x: format(x, '06'))
metas['first_traded'] = metas['start_date']
metas['auto_close_date'] = metas['end_date']
symbols = symbols.set_index('symbol',
drop=False).join(metas.set_index('symbol'),
how='inner')
asset_db_writer.write(symbols)
adjustment_writer.write(splits=splits, dividends=dividends, shares=shares)
if fundamental:
logger.info("writing fundamental data:")
try:
fundamental_writer.write(start_session, end_session)
except Exception as e:
pass
eg.exit()
def main():
logbook.StderrHandler().push_application()
engine = Engine(best_ip=True, thread_num=1)
with engine.connect():
engine.get_k_data('000002', '20100921', '20100930', '1m')
print(grouped.sort_values('up_limit', ascending=False))
def minute_time_data():
stock_list = engine.stock_list.index.tolist()
now = datetime.datetime.now()
for stock in stock_list:
fs = engine.api.to_df(
engine.api.get_minute_time_data(stock[0], stock[1]))
# print(fs)
print((datetime.datetime.now() - now).total_seconds())
def quotes():
start_dt = datetime.datetime.now()
quote = engine.stock_quotes()
print(datetime.datetime.now() - start_dt).total_seconds()
process_quotes(quote)
if __name__ == '__main__':
engine = Engine(best_ip=True)
with engine.connect():
print(
engine.get_security_bars('002920', '1d',
pd.to_datetime('20170701')))
def transactions():
eg = Engine(best_ip=True, auto_retry=True)
eg.connect()
m1 = eg.get_k_data('000001', '20170101', '20180101', '1m')
def main():
engine = Engine(best_ip=True, thread_num=1)
with engine.connect():
engine.get_k_data('000001', '20161201', '20171231', '1m')
def test_transaction():
engine = Engine(best_ip=True, thread_num=1)
with engine.connect():
df = engine.get_k_data('000001', '20170601', '20171231', '1m')
df = engine.get_security_bars(['000001', '000521'],
'1d',
start=pd.to_datetime('20180102'))
if __name__ == '__main__':
engine = Engine(best_ip=True, thread_num=1)
with engine.connect():
print(engine.api.get_security_count(0))
print(engine.api.get_security_count(1))
lists = engine.stock_list
print(
engine.get_security_bars('300737', '1d',
pd.to_datetime('20161201'),
pd.to_datetime('20171231')))
print(engine.get_k_data('300737', '20161201', '20171231', '1d'))
print(timeit.timeit(test_transaction, number=1))
print(timeit.timeit(main, number=1))
def main():
engine = Engine(best_ip=True, thread_num=1)
with engine.connect():
engine.get_k_data('000001', '20161201', '20171231', '1m')
def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day):
"""
Ensure we have benchmark data for `symbol` from `first_date` to `last_date`
Parameters
----------
symbol : str
The symbol for the benchmark to load.
first_date : pd.Timestamp
First required date for the cache.
last_date : pd.Timestamp
Last required date for the cache.
now : pd.Timestamp
The current time. This is used to prevent repeated attempts to
re-download data that isn't available due to scheduling quirks or other
failures.
trading_day : pd.CustomBusinessDay
A trading day delta. Used to find the day before first_date so we can
get the close of the day prior to first_date.
We attempt to download data unless we already have data stored at the data
cache for `symbol` whose first entry is before or on `first_date` and whose
last entry is on or after `last_date`.
If we perform a download and the cache criteria are not satisfied, we wait
at least one hour before attempting a redownload. This is determined by
comparing the current time to the result of os.path.getmtime on the cache
path.
"""
path = get_data_filepath(get_benchmark_filename(symbol))
# If the path does not exist, it means the first download has not happened
# yet, so don't try to read from 'path'.
if os.path.exists(path):
try:
data = pd.Series.from_csv(path).tz_localize('UTC')
if has_data_for_dates(data, first_date, last_date):
return data
# Don't re-download if we've successfully downloaded and written a
# file in the last hour.
last_download_time = last_modified_time(path)
if (now - last_download_time) <= ONE_HOUR:
logger.warn(
"Refusing to download new benchmark data because a "
"download succeeded at %s." % last_download_time)
return data
except (OSError, IOError, ValueError) as e:
# These can all be raised by various versions of pandas on various
# classes of malformed input. Treat them all as cache misses.
logger.info(
"Loading data for {path} failed with error [{error}].".format(
path=path,
error=e,
))
logger.info(
"Cache at {path} does not have data from {start} to {end}.\n"
"Downloading benchmark data for '{symbol}'.",
start=first_date,
end=last_date,
symbol=symbol,
path=path,
)
engine = Engine(auto_retry=True, multithread=True, thread_num=8)
engine.connect()
data = engine.get_security_bars(symbol, '1d', index=True)
data = data['close'].sort_index().tz_localize('UTC').pct_change(1).iloc[1:]
data.index = data.index.shift(-15,
'1H') # change datetime at 15:00 to midnight
data.to_csv(path)
return data
def tdx_bundle(assets,
ingest_minute, # whether to ingest minute data, default False
overwrite,
environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir):
eg = Engine(auto_retry=True, multithread=True, best_ip=True, thread_num=8)
eg.connect()
symbols = fetch_symbols(eg, assets)
metas = []
today = pd.to_datetime('today',utc=True)
distance = calendar.session_distance(start_session, today)
if ingest_minute and not overwrite and (start_session < today - pd.DateOffset(years=3)):
minute_start = calendar.all_sessions[searchsorted(calendar.all_sessions, today - pd.DateOffset(years=3))]
logger.warning(
"overwrite start_session for minute bars to {}(3 years),"
" to fetch minute data before that, please add '--overwrite True'".format(minute_start))
else:
minute_start = start_session
def gen_symbols_data(symbol_map, freq='1d'):
func = partial(fetch_single_equity, eg)
start = start_session
end = end_session
if freq == '1m':
if distance >= 100:
func = eg.get_k_data
start = minute_start
for index, symbol in symbol_map.iteritems():
data = reindex_to_calendar(
calendar,
func(symbol, start, end, freq),
freq=freq,
)
if freq == '1d':
metas.append(get_meta_from_bars(data))
yield int(symbol), data
symbol_map = symbols.symbol
assets = set([int(s) for s in symbol_map])
daily_bar_writer.write(gen_symbols_data(symbol_map, freq="1d"), assets=assets, show_progress=show_progress)
if ingest_minute:
with click.progressbar(gen_symbols_data(symbol_map, freq="1m"),
label="Merging minute equity files:",
length=len(assets),
item_show_func=lambda e: e if e is None else str(e[0]),
) as bar:
minute_bar_writer.write(bar, show_progress=False)
symbols = pd.concat([symbols, pd.DataFrame(data=metas)], axis=1)
splits, dividends = fetch_splits_and_dividends(eg, symbols)
symbols.set_index('symbol', drop=False, inplace=True)
asset_db_writer.write(symbols)
adjustment_writer.write(
splits=splits,
dividends=dividends
)
eg.exit()
