def save(market, dir):
# 写入交易日
param = {
"market": market,
"stock": "FFFFFF",
"metaId": 2,
"period": 86400000L,
"timeTag": 0x1F3FFFFF830L,
"compress": False
}
response = request(ReaderAddress, "requestByCount", param)
writeToFile(response, "%s/%s_tradeDate" % (dir, market))
# 写入股票列表
if len(response["C"]) > 1:
lastDate = response["C"][-1]["G"]
print lastDate
param = {
"market": market,
"stock": "XXXXXX",
"metaId": 2000,
"period": 86400000L,
"timeTag": lastDate,
"seq": 0L,
"compress": False
}
response = request(ReaderAddress, "requestByDaily", param)
writeToFile(response, "%s/%s_stockList" % (dir, market))
def run(self):
while True:
try:
data, addr = self.com.recvfrom(1024)
request("127.0.0.1:9000", "recv", {"data": data}, 0)
except Exception as err:
print(err)
def req_send(self):
#data = dict(symbol=self.symbol,feedtype=self.feedtype,port=self.port,status=self.status)
data = {
'symbol': self.symbol,
'feedtype': self.feedtype,
'port': self.port,
'status': self.status
}
# while True:
try:
request("127.0.0.1:9001", "req_data", data)
except Exception as err:
print('error', err)
def dispatchFromNet(market):
# 请求交易日
param = {
"market": market,
"stock": "FFFFFF",
"metaId": 2,
"period": 86400000L,
"timeTag": 0x1F3FFFFF830L,
"compress": False
}
response = request(ReaderAddress, "requestByCount", param)
# 写入交易日信息
address = WriterAddress.split(":")
client = RPCClient(address[0], int(address[1]), 100)
for x in response["C"]:
param = {"F": {"M": market, "S": "FFFFFF"}, "C": [{"C": x}]}
r = client.request("writeData", param)
#print "write data ", param
# 更新市场状态
if len(response["C"]) > 1:
r = client.request(
"update", {
u"path": u"marketStatus/%s" % market,
u"data": {
u"date": response["C"][-1]["G"],
u"status": 2
}
})
print "update market status", param
# 写入日数据信息
if len(response["C"]) > 1:
lastDate = response["C"][-1]["G"]
print lastDate
param = {
"market": "SF",
"stock": "XXXXXX",
"metaId": 2000,
"period": 86400000L,
"timeTag": lastDate,
"seq": 0L,
"compress": False
}
response = request(ReaderAddress, "requestByDaily", param)
print "request by daily ", len(response)
param = {"M": market, "G": lastDate, "C": response}
response = request(WriterAddress, "writeDaily", param)
print "write daily ", response
def dispatchFromFile(market, dir):
# 请求交易日
fileName = "%s/%s_tradeDate" % (dir, market)
response = loadFromFile(fileName)
# 写入交易日信息
address = WriterAddress.split(":")
client = RPCClient(address[0], int(address[1]), 100)
for x in response["C"]:
param = {"F": {"M": market, "S": "FFFFFF"}, "C": [{"C": x}]}
r = client.request("writeData", param)
# print "write data ", param
# 更新市场状态
if len(response["C"]) > 1:
r = client.request(
"update", {u"path": "marketStatus/%s" % market, u"data": {u"date": response["C"][-1]["G"], u"status": 2}}
)
print "update market status", param
# 写入日数据信息
if len(response["C"]) > 1:
lastDate = response["C"][-1]["G"]
print lastDate
fileName = "%s/%s_stockList" % (dir, market)
response = loadFromFile(fileName)
print "request by daily ", len(response)
param = {"M": market, "G": lastDate, "C": response}
response = request(WriterAddress, "writeDaily", param)
print "write daily ", response
def dispatchFromFile(market, dir):
# 请求交易日
fileName = "%s/%s_tradeDate" % (dir, market)
response = loadFromFile(fileName)
# 写入交易日信息
address = WriterAddress.split(":")
client = RPCClient(address[0], int(address[1]), 100)
for x in response["C"]:
param = {"F": {"M": market, "S": "FFFFFF"}, "C": [{"C": x}]}
r = client.request("writeData", param)
#print "write data ", param
# 更新市场状态
if len(response["C"]) > 1:
r = client.request(
"update", {
u"path": "marketStatus/%s" % market,
u"data": {
u"date": response["C"][-1]["G"],
u"status": 2
}
})
print "update market status", param
# 写入日数据信息
if len(response["C"]) > 1:
lastDate = response["C"][-1]["G"]
print lastDate
fileName = "%s/%s_stockList" % (dir, market)
response = loadFromFile(fileName)
print "request by daily ", len(response)
param = {"M": market, "G": lastDate, "C": response}
response = request(WriterAddress, "writeDaily", param)
print "write daily ", response
def _load_minute_file(self, field, sid, dt):
symbol = sid.symbol
sid = int(sid)
try:
carray = self._carrays[field][sid]
except KeyError:
from net.RPCClient import request
_dict = {}
_dt = pd.Timestamp(dt.value).normalize()
_dt_next = _dt + pd.DateOffset(days=1)
self._last_get_value_dt = _dt_next.to_datetime64().astype(np.int64)
d = request(
"123.56.77.52:10030", "Kline",
{
"symbol": symbol,
"period": "m",
"s": _dt.strftime("%Y%m%d"),
"e": _dt_next.strftime("%Y%m%d")
},
)
df = pd.DataFrame(d)
df.index = pd.to_datetime(df.index, utc=True) - pd.Timedelta(
hours=8)
for col in df.columns:
if col == "amount":
continue
self._carrays[col][sid] = df[col]
carray = self._carrays[field][sid]
return carray
def run(self):
while True:
try:
data, addr = self.com.recvfrom(1024)
res = request("113.108.181.146:29001", "recv", {"data": data},
0)
except Exception as err:
print(err)
def remove():
account = {
"m_nBrokerType": 2,
"m_nPlatformID": 11001, # 目前主要用于区别不同的行情, 根据此来选择对应行情
"m_strBrokerID": "111000",
"m_nAccountType": 49,
"m_strAccountID": "37000109",
}
response = request(ADDRESS, "remove", {"req": account})
def get_sector(symbol, date=None):
symbols = request(
"123.56.77.52:10030",
"Sector",
{
"symbol": "000300.SH",
# "date": date,
}
)
return symbols
def correct():
account = {
"m_nBrokerType": 2,
"m_nPlatformID": 21002, # 目前主要用于区别不同的行情, 根据此来选择对应行情
"m_strBrokerID": "0",
"m_nAccountType": -1,
"m_strAccountID": "10001"
}
response = request(ADDRESS, "correct", {"req": account})
print response
def load_data(self, fields, date, sid):
from net.RPCClient import request
_dict = {}
start = date.normlize()
end = start + pd.DateOffset(days=1)
d = request(
"123.56.77.52:10030", "Kline",
{
"symbol": sid.symbol,
"period": "m",
"s": start.strftime("%Y%m%d"),
"e": end.strftime("%y%m%d")
},
)
return
def get_data(symbol, start, end):
print symbol
data = request(
"123.56.77.52:10030",
"Kline", {
"symbol": symbol,
"start": start,
"end": end
}
)
if end is None:
end = pd.Timestamp("today")
df = pd.DataFrame(data)
df.index.name = "Date"
df.index = pd.to_datetime(df.index)
return df.ix[(df.index >= start)&(df.index <= end)]
def req_send(self):
data = {
'symbol': self.symbol,
'feedtype': self.feedtype,
'port': self.port,
'status': self.status
}
# while True:
try:
# data_res = request("113.108.181.146:29001", "req_data", data)
status = request("218.77.104.29:8000", "req_data",
data) # call utils
print(status) # return status
except Exception as err:
print('error', err)
def queryStockData():
account = {
"m_nBrokerType": 2,
"m_nPlatformID": 11001, # 目前主要用于区别不同的行情, 根据此来选择对应行情
"m_strBrokerID": "111000",
"m_nAccountType": 49,
"m_strAccountID": "37000149",
}
#ids = ["37000034", "37000178", "37000179", "37000025", "37000024", "37000139", "37000144", "37000080", "37000184", "37000086", "37000234"]
#ids = ["37000045", "37000048", "37000053", "37000098", "37000058", "37000164", "37000149", "37000024", "37000167", "37000156", "37000212", "37000020","37000121", "37000147"]
ids = ["37000149"]
for id in ids:
account["m_strAccountID"] = id
response = request(ADDRESS, "queryData", {"req": {"metaId": 543, "account": account}})
#print response["content"][0]
print response
def queryFutureData():
account = {
"m_nBrokerType": 1,
"m_nPlatformID": 21001, # 目前主要用于区别不同的行情, 根据此来选择对应行情
"m_strBrokerID": "1001",
"m_nAccountType": -1,
"m_strAccountID": "00000170",
"m_strSubAccount": "00000170_02"
}
#ids = ["00000174", "00000222", "00000223", "00000224", "00000225", "00000228", "00000233", "00000234", "00000236", "00000237", "00000238", "00000241", "00000242", "00000243", "00000244"]
#ids = ["37000045", "37000048", "37000053", "37000098", "37000058", "37000164", "37000149", "37000024", "37000167", "37000156", "37000212", "37000020","37000121", "37000147"]
ids = ["00000170"]
for id in ids:
account["m_strAccountID"] = id
response = request(ADDRESS, "queryData", {"req": {"metaType": 543, "account": account}})
print id, response
def get(self, seq, params, session):
res = request("123.56.77.52:10030", "Kline",
{"symbol": "000001.SZ", "period":"m",
"s":"20170505","e":"20170506"},)
df = pd.DataFrame(res)
df.index.name = "Date"
df.index = pd.to_datetime(df.index)
pd.Timestamp("today")
import time
for _index in df.index:
if session.connected:
print _index
time.sleep(10)
session.sendResponse(seq, [_index])
else:
print "connect lost"
break
def queryRawPosition(account):
response = request(ADDRESS, "queryData", {"req": {"metaType": 545, "account": account}})
return response
def load_raw_arrays(self, fields, start_dt, end_dt, sids):
"""
Parameters
----------
fields : list of str
'open', 'high', 'low', 'close', or 'volume'
start_dt: Timestamp
Beginning of the window range.
end_dt: Timestamp
End of the window range.
sids : list of int
The asset identifiers in the window.
Returns
-------
list of np.ndarray
A list with an entry per field of ndarrays with shape
(minutes in range, sids) with a dtype of float64, containing the
values for the respective field over start and end dt range.
"""
from net.RPCClient import request
_dict = {}
for sid in sids:
d = request(
"123.56.77.52:10030", "Kline",
{
"symbol": sid.symbol,
"period": "m",
"s": start_dt.strftime("%Y%m%d"),
"e": end_dt
},
)
df = pd.DataFrame(d).set_index(['date'])
df.index = pd.to_datetime(df.index - 28800, unit="s")
_dict[sid] =df.loc[(df.index>=start_dt) & (df.index<=end_dt)]
panel = pd.Panel(_dict)
indexs = panel.major_axis
# start_idx = self._find_position_of_minute(start_dt)
# end_idx = self._find_position_of_minute(end_dt)
_index = indexs[(indexs>=start_dt)&(indexs<=end_dt)]
# num_minutes = len()
# num_minutes =
#
results = []
#
# indices_to_exclude = self._exclusion_indices_for_range(
# start_idx, end_idx)
# if indices_to_exclude is not None:
# for excl_start, excl_stop in indices_to_exclude:
# length = excl_stop - excl_start + 1
# num_minutes -= length
#
# shape = num_minutes, len(sids)
#
for field in fields:
out = panel[:,_index, field].values
if field != 'volume':
pass
else:
out = out.astype(np.uint32)
# out = np.zeros(shape, dtype=np.uint32)
# for i, sid in enumerate(sids):
# carray = self._open_minute_file(field, sid)
# values = carray[start_idx:end_idx + 1]
# if indices_to_exclude is not None:
# for excl_start, excl_stop in indices_to_exclude[::-1]:
# excl_slice = np.s_[
# excl_start - start_idx:excl_stop - start_idx + 1]
# values = np.delete(values, excl_slice)
#
# where = values != 0
# # first slice down to len(where) because we might not have
# # written data for all the minutes requested
# if field != 'volume':
# out[:len(where), i][where] = (
# values[where] * self._ohlc_ratio_inverse_for_sid(sid))
# else:
# out[:len(where), i][where] = values[where]
#
results.append(out)
return results
__author__ = "rzrk001"
# encoding:utf-8
from net.RPCClient import RPCClient, request
if __name__ == "__main__":
strPath = r"d:\Xtservice.dat"
param = {"strPath": strPath}
try:
ret = request(u"192.168.1.150:5379", u"addDgbmFile", param)
except Exception, ex:
print ex
ret = ""
strDb = "Xtservice"
param = {"strDb": strDb}
try:
ret = request(u"192.168.1.150:5379", u"getAllKeys", param)
except Exception, ex:
print ex
ret = ""
strDb = "Xtservice"
key = "key1"
param = {"strDb": strDb, "strKey": key}
try:
ret = request(u"192.168.1.150:5379", u"readData", param)
except Exception, ex:
print ex
ret = ""
response["error"] = "系统异常![%s]" % str(e)
logger.error(response["error"] + getTraceBack())
return HttpResponse(json.dumps(response), mimetype="application/json")
def dispatchFromNet(market):
# 请求交易日
param = {
"market": market,
"stock": "FFFFFF",
"metaId": 2,
"period": 86400000L,
"timeTag": 0x1F3FFFFF830L,
"compress": False,
}
response = request(ReaderAddress, "requestByCount", param)
# 写入交易日信息
address = WriterAddress.split(":")
client = RPCClient(address[0], int(address[1]), 100)
for x in response["C"]:
param = {"F": {"M": market, "S": "FFFFFF"}, "C": [{"C": x}]}
r = client.request("writeData", param)
# print "write data ", param
# 更新市场状态
if len(response["C"]) > 1:
r = client.request(
"update", {u"path": u"marketStatus/%s" % market, u"data": {u"date": response["C"][-1]["G"], u"status": 2}}
)
print "update market status", param
__author__ = 'rzrk001'
#encoding:utf-8
from net.RPCClient import RPCClient, request
if __name__ == "__main__":
strPath = r'd:\Xtservice.dat'
param = {"strPath": strPath}
try:
ret = request(u"192.168.1.150:5379", u"addDgbmFile", param)
except Exception, ex:
print ex
ret = ""
strDb = 'Xtservice'
param = {"strDb": strDb}
try:
ret = request(u"192.168.1.150:5379", u"getAllKeys", param)
except Exception, ex:
print ex
ret = ""
strDb = 'Xtservice'
key = 'key1'
param = {"strDb": strDb, "strKey": key}
try:
ret = request(u"192.168.1.150:5379", u"readData", param)
except Exception, ex:
print ex
ret = ""
print ret
def send(data):
try:
request("127.0.0.1:9000", "recv", {"data": data})
except Exception as err:
print('error', err)
def ingest(environ,
asset_db_writer,
minute_bar_writer, # unused
daily_bar_writer,
adjustment_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir,
# pass these as defaults to make them 'nonlocal' in py2
start=start,
end=end):
if start is None:
start = start_session
if end is None:
end = None
metadata = pd.DataFrame(np.empty(len(symbols), dtype=[
('start_date', 'datetime64[ns]'),
('end_date', 'datetime64[ns]'),
('auto_close_date', 'datetime64[ns]'),
('symbol', 'object'),
]))
trading_days = get_calendar('SH').all_sessions
trading_days = trading_days.astype("datetime64[ns]")
def _pricing_iter():
sid = 0
with maybe_show_progress(
symbols,
show_progress,
label='Downloading Yahoo pricing data: ') as it, \
requests.Session() as session:
for symbol in it:
print symbol
path = _cachpath(symbol, 'ohlcv')
try:
df = cache[path]
except KeyError:
df = cache[path] = get_data(
symbol,
start,
end
)
# df = cache[path] = DataReader(
# symbol,
# 'yahoo',
# start,
# end,
# session=session,
# ).sort_index()
# the start date is the date of the first trade and
# the end date is the date of the last trade
df = df[df.Volume>0]
start_date = df.index[0]
end_date = df.index[-1]
df = df.reindex(trading_days[(trading_days>=start_date)])
df.Volume = df.Volume.fillna(0)
df = df.ffill()
# The auto_close date is the day after the last trade.
ac_date = end_date + pd.Timedelta(days=1)
metadata.iloc[sid] = start_date, end_date, ac_date, symbol
df.rename(
columns={
'Open': 'open',
'High': 'high',
'Low': 'low',
'Close': 'close',
'Volume': 'volume',
},
inplace=True,
)
yield sid, df
sid += 1
daily_bar_writer.write(_pricing_iter(), show_progress=show_progress)
symbol_map = pd.Series(metadata.symbol.index, metadata.symbol)
# Hardcode the exchange to "YAHOO" for all assets and (elsewhere)
# register "YAHOO" to resolve to the NYSE calendar, because these are
# all equities and thus can use the NYSE calendar.
metadata['exchange'] = "hs300"
asset_db_writer.write(equities=metadata)
adjustments = []
with maybe_show_progress(
symbols,
show_progress,
label='Downloading Yahoo adjustment data: ') as it, \
requests.Session() as session:
for symbol in it:
path = _cachpath(symbol, 'adjustment')
try:
df = cache[path]
except KeyError:
data = cache[path] = request(
"123.56.77.52:10030",
"Divid",
{"symbol": symbol}
)
df = pd.DataFrame(data).sort_index()
# print df
# df = cache[path] = DataReader(
# symbol,
# 'yahoo-actions',
# start,
# end,
# session=session,
# ).sort_index()
df['sid'] = symbol_map[symbol]
adjustments.append(df)
adj_df = pd.concat(adjustments)
adj_df.index.name = 'date'
adj_df.reset_index(inplace=True)
adj_df.date = pd.to_datetime(adj_df.date)
adj_df = adj_df[adj_df.date > pd.Timestamp("2010-01-01")]
splits = adj_df[adj_df.action == 'SPLIT']
splits = splits.rename(
columns={'value': 'ratio', 'date': 'effective_date'},
)
splits.drop('action', axis=1, inplace=True)
dividends = adj_df[adj_df.action == 'DIVIDEND']
dividends = dividends.rename(
columns={'value': 'amount', 'date': 'ex_date'},
)
dividends.drop('action', axis=1, inplace=True)
# we do not have this data in the yahoo dataset
dividends['record_date'] = pd.NaT
dividends['declared_date'] = pd.NaT
dividends['pay_date'] = pd.NaT
adjustment_writer.write(splits=splits, dividends=dividends)
def getAccountDetails(account):
response = request(ADDRESS, "getAccountDetails", {"accounts": [account]})
return response
def queryRawAccountDetail(account):
response = request(ADDRESS, "queryData", {"req": {"metaType": 543, "account": account}})
return response
