def I(self, func: typing.Callable, *args) -> np.ndarray:
value = func(*args)
value = np.asarray(value)
utils.assert_msg(value.shape[-1] == len(self._data.Close),
"指示器长度必须和data长度相同")
self._indicators.append(value)
return value
async def create_job_for_analysis(ws,
http_client,
base_url,
analysis_uuid,
token=None):
job_uuid = _get_uuid_str()
job_url = f"{base_url}/api/jobs/{job_uuid}/"
job_data = {
"job": {
"analysis": analysis_uuid,
}
}
job_url = add_token(job_url, token)
async with http_client.put(job_url, json=job_data) as resp:
print(await resp.text())
assert resp.status == 200
resp_json = await resp.json()
assert resp_json['status'] == "ok"
msg = json.loads(await ws.recv())
assert_msg(msg, 'JOB_STARTED')
assert msg['job'] == job_uuid
assert msg['analysis'] == analysis_uuid
assert msg['details']['id'] == job_uuid
return job_uuid, job_url
async def create_analysis(ws,
http_client,
base_url,
ds_uuid,
ca_uuid,
details=None):
analysis_uuid = _get_uuid_str()
analysis_url = "{}/api/compoundAnalyses/{}/analyses/{}/".format(
base_url, ca_uuid, analysis_uuid)
if details is None:
details = {"analysisType": "SUM_FRAMES", "parameters": {}}
else:
assert "analysisType" in details
assert "parameters" in details
analysis_data = {
"dataset": ds_uuid,
"details": details,
}
async with http_client.put(analysis_url, json=analysis_data) as resp:
print(await resp.text())
assert resp.status == 200
resp_json = await resp.json()
assert resp_json['status'] == "ok"
msg = json.loads(await ws.recv())
assert_msg(msg, 'ANALYSIS_CREATED')
assert msg['dataset'] == ds_uuid
assert msg['analysis'] == analysis_uuid
assert msg['details'] == details
return analysis_uuid, analysis_url
async def update_analysis(
ws,
http_client,
base_url,
ds_uuid,
ca_uuid,
analysis_uuid,
details,
token=None,
):
analysis_url = "{}/api/compoundAnalyses/{}/analyses/{}/".format(
base_url, ca_uuid, analysis_uuid)
analysis_url = add_token(analysis_url, token)
assert "analysisType" in details
assert "parameters" in details
analysis_data = {
"dataset": ds_uuid,
"details": details,
}
async with http_client.put(analysis_url, json=analysis_data) as resp:
print(await resp.text())
assert resp.status == 200
resp_json = await resp.json()
assert resp_json['status'] == "ok"
msg = json.loads(await ws.recv())
assert_msg(msg, 'ANALYSIS_UPDATED')
assert msg['dataset'] == ds_uuid
assert msg['analysis'] == analysis_uuid
assert msg['details'] == details
async def test_cancel_unknown_job(default_raw, base_url, http_client,
server_port):
conn_url = "{}/api/config/connection/".format(base_url)
conn_details = {
'connection': {
'type': 'local',
'numWorkers': 2,
}
}
async with http_client.put(conn_url, json=conn_details) as response:
assert response.status == 200
assert (await response.json())['status'] == 'ok'
# connect to ws endpoint:
ws_url = "ws://127.0.0.1:{}/api/events/".format(server_port)
async with websockets.connect(ws_url) as ws:
initial_msg = json.loads(await ws.recv())
assert_msg(initial_msg, 'INITIAL_STATE')
job_uuid = "un-kn-ow-n"
job_url = "{}/api/jobs/{}/".format(base_url, job_uuid)
# try to cancel unknown job:
async with http_client.delete(job_url) as resp:
assert resp.status == 200
assert_msg(await resp.json(), 'CANCEL_JOB_FAILED', status='error')
async def test_detect_failed(default_raw, base_url, http_client, server_port):
conn_url = "{}/api/config/connection/".format(base_url)
conn_details = {
'connection': {
'type': 'local',
'numWorkers': 2,
}
}
async with http_client.put(conn_url, json=conn_details) as response:
assert response.status == 200
# connect to ws endpoint:
ws_url = "ws://127.0.0.1:{}/api/events/".format(server_port)
async with websockets.connect(ws_url) as ws:
initial_msg = json.loads(await ws.recv())
assert_msg(initial_msg, 'INITIAL_STATE')
assert initial_msg['datasets'] == []
assert initial_msg['jobs'] == []
path = default_raw._path
detect_url = "{}/api/datasets/detect/".format(base_url)
async with http_client.get(detect_url, params={"path": path}) as resp:
assert resp.status == 200
resp_json = await resp.json()
assert_msg(resp_json, 'DATASET_DETECTION_FAILED', status='error')
async def test_load_raw_fail(default_raw, base_url, http_client):
conn_url = "{}/api/config/connection/".format(base_url)
conn_details = {
'connection': {
'type': 'local',
'numWorkers': 2,
}
}
async with http_client.put(conn_url, json=conn_details) as response:
assert response.status == 200
raw_path = default_raw._path
uuid = "ae5d23bd-1f2a-4c57-bab2-dfc59a1219f3"
ds_url = "{}/api/datasets/{}/".format(base_url, uuid)
ds_data = _get_raw_params(raw_path)
ds_data["dataset"]["params"]["scan_size"] = [
32, 32
] # too large, should cause error
async with http_client.put(ds_url, json=ds_data) as resp:
assert resp.status == 200
resp_json = await resp.json()
assert_msg(resp_json, 'CREATE_DATASET_ERROR', status='error')
assert resp_json['dataset'] == uuid
assert resp_json['msg'].startswith('invalid dataset: ')
def __init__(self,
data: pd.DataFrame,
strategy_type: type(Strategy),
broker_type: type(ExchangeAPI),
cash: float = 1000,
commission: float = .0):
utils.assert_msg(issubclass(strategy_type, Strategy),
"strategy_type 不是一个Strategy类型")
utils.assert_msg(issubclass(broker_type, ExchangeAPI),
"broker_type 不是一个ExchangeAPI类型")
utils.assert_msg(isinstance(commission, numbers.Number),
"commission 不是浮点型")
data = data.copy(False)
if "Volume" not in data:
data["Volume"] = np.nan
utils.assert_msg(
len(data.columns
& {"Open", "High", "Low", "Close", "Volume"}) == 5,
"输入的 `data` 格式不正确")
utils.assert_msg(
not data[["Open", "High", "Low", "Close", "Volume"
]].max().isnull().any(), "部分 OHLC 包含缺失值 ")
if not data.index.is_monotonic_increasing:
data = data.sort_index()
self._data = data
self._broker = broker_type(data, cash, commission)
self._strategy = strategy_type(self._broker, self._data)
self._results = None
def I(self, func: Callable, *args) -> np.ndarray:
value = func(*args)
value = np.asarray(value)
assert_msg(value.shape[-1] == len(self._data.Close),
'The length of indicator must equal to that of data!')
self._indicators.append(value)
return value
async def test_cancel_unknown_job(
default_raw,
base_url,
http_client,
server_port,
local_cluster_url,
default_token,
):
await create_connection(
base_url,
http_client,
scheduler_url=local_cluster_url,
token=default_token,
)
ws_url = f"ws://127.0.0.1:{server_port}/api/events/?token={default_token}"
async with websockets.connect(ws_url) as ws:
initial_msg = json.loads(await ws.recv())
assert_msg(initial_msg, 'INITIAL_STATE')
job_uuid = "un-kn-ow-n"
job_url = f"{base_url}/api/jobs/{job_uuid}/?token={default_token}"
# try to cancel unknown job:
async with http_client.delete(job_url) as resp:
assert resp.status == 200
assert_msg(await resp.json(), 'CANCEL_JOB_FAILED', status='error')
async def test_detect_hdf5(hdf5, base_url, http_client, server_port):
conn_url = "{}/api/config/connection/".format(base_url)
conn_details = {
'connection': {
'type': 'local',
'numWorkers': 2,
}
}
async with http_client.put(conn_url, json=conn_details) as response:
assert response.status == 200
# connect to ws endpoint:
ws_url = "ws://127.0.0.1:{}/api/events/".format(server_port)
async with websockets.connect(ws_url) as ws:
initial_msg = json.loads(await ws.recv())
assert_msg(initial_msg, 'INITIAL_STATE')
assert initial_msg['datasets'] == []
assert initial_msg['jobs'] == []
path = hdf5.filename
detect_url = "{}/api/datasets/detect/".format(base_url)
async with http_client.get(detect_url, params={"path": path}) as resp:
assert resp.status == 200
resp_json = await resp.json()
assert_msg(resp_json, 'DATASET_DETECTED')
assert "datasetParams" in resp_json
params = resp_json["datasetParams"]
assert params['type'] == 'HDF5'
assert params['path'] == path
assert params['ds_path'] == "data"
assert "tileshape" in params
def __init__(self, data, cash, commission):
assert_msg(0 < cash, "初始现金数量大于0，输入的现金数量：{}".format(cash))
assert_msg(0 <= commission <= 0.05, "合理的手续费率一般不会超过5%，输入的费率：{}".format(commission))
self._initial_cash = cash
self._data = data
self._commission = commission
self._position = 0
self._cash = cash
self._i = 0
def __init__(self, data, cash, commission):
assert_msg(0 < cash, "initial cash deposite amount must be bigger than 0，please type the amount：{}".format(cash))
assert_msg(0 <= commission <= 0.05, "reasonable commission fee should usually be less than 5%，please type the rate：{}".format(commission))
self._inital_cash = cash
self._data = data
self._commission = commission
self._position = 0
self._cash = cash
self._i = 0
async def test_shutdown(base_url, http_client, server_port):
await create_connection(base_url, http_client)
print("checkpoint 1")
url = f"ws://127.0.0.1:{server_port}/api/shutdown/"
async with http_client.delete(url) as resp:
print("checkpoint 2")
assert resp.status == 200
assert_msg(await resp.json(), "SERVER_SHUTDOWN")
def __init__(self,
data: pd.DataFrame,
strategy_type: type(Strategy),
broker_type: type(ExchangeAPI),
cash: float = 10000,
commission: float = .0):
"""
构造回测对象。需要的参数包括：历史数据，策略对象，初始资金数量，手续费率等。
初始化过程包括检测输入类型，填充数据空值等。
参数：
：param data: pd.DataFrame pandas Dataframe格式的历史OHLCV数据
:param broker_type: type(ExchangeAPI) 交易所API类型，负责执行买卖操作以及账户状态的维护
:param strategy_type: type(Strategy) 策略类型
:param cash: float 初始资金数量
:param commission: float 每次交易手续费率。如2%的手续费此处为0.02
"""
assert_msg(issubclass(strategy_type, Strategy),
'strategy_type不是一个Strategy类型')
assert_msg(issubclass(broker_type, ExchangeAPI),
'broker_type不是一个ExchangeAPI类型')
assert_msg(isinstance(commission, float), 'commission不是浮点数值类型')
self._strategy_value = []
self._strategy_return = []
data = data.copy(False)
# 如果没有Volumn列，填充NaN
if 'Volume' not in data:
data['Volume'] = np.nan
# 验证OHLC数据格式
assert_msg(
len(data.columns
& {'Open', 'High', 'Low', 'Close', 'Volume'}) == 5,
("输入的`data`格式不正确，至少需要包含这些列："
""
"'Open', 'High', 'Low', 'Close'"))
# 检查缺失值
assert_msg(
not data[['Open', 'High', 'Low', 'Close']].max().isnull().any(),
('部分OHLC包含缺失值，请去掉那些行或者通过差值填充. '))
# 如果行情数据没有按照时间排序，重新排序一下
if not data.index.is_monotonic_increasing:
data = data.sort_index()
# 利用数据，初始化交易所对象和策略对象。
self._data = data # type: pd.DataFrame
self._broker = broker_type(data, cash, commission)
self._strategy = strategy_type(self._broker, self._data)
self._results = None
def __init__(self, data, cash, commission):
utils.assert_msg(0 < cash, "初始资金大于0，输入的现金数量：{}".format(cash))
utils.assert_msg(0 <= commission <= 0.05,
"合理的手续费不超过5%，输入的费率：{}".format(commission))
self._init_cash = cash
self._data = data
self._commission = commission
self._position = 0
self._cash = cash
self._i = 0
def __init__(self, data, cash, commission):
assert_msg(cash > 0, 'Initial cash must be greater than 0!')
assert_msg(
0 <= commission <= 0.05,
'Commission must be a positive integer which is usually smaller that 0.05'
)
self._initial_cash = cash
self._cash = cash
self._position = 0
self._data = data
self._commission = commission
self._i = 0
def __init__(self, data, cash, commission):
utils.assert_msg(0 < cash,
"初始现金数量须大于0，输入现金数量: {}".format(cash))
utils.assert_msg(0 <= commission <= 0.05,
"合理的手续费率一般不会超过5%，输入的费率:{"
"}".format(commission))
self._initial_cash = cash
self._data = data
self._commission = commission
self._position = 0
self._cash = cash
self._i = 0
async def test_cluster_connect_error(base_url, http_client):
url = "{}/api/config/connection/".format(base_url)
conn_details = {
'connection': {
'type': 'TCP',
'address': 'tcp://invalid',
}
}
async with http_client.put(url, json=conn_details) as response:
assert response.status == 200
conn_resp = await response.json()
assert_msg(conn_resp, 'CLUSTER_CONN_ERROR', status='error')
async def test_initial_state_w_existing_ds(
default_raw, base_url, http_client, server_port, local_cluster_url, default_token,
):
conn_url = f"{base_url}/api/config/connection/?token={default_token}"
conn_details = {
'connection': {
'type': 'tcp',
'address': local_cluster_url,
}
}
async with http_client.put(conn_url, json=conn_details) as response:
assert response.status == 200
# first connect has empty list of datasets:
ws_url = f"ws://127.0.0.1:{server_port}/api/events/?token={default_token}"
async with websockets.connect(ws_url) as ws:
initial_msg = json.loads(await ws.recv())
assert initial_msg['messageType'] == "INITIAL_STATE"
assert initial_msg['status'] == "ok"
assert initial_msg['datasets'] == []
raw_path = default_raw._path
ds_uuid = "ae5d23bd-1f2a-4c57-bab2-dfc59a1219f3"
ds_url = "{}/api/datasets/{}/?token={}".format(
base_url, ds_uuid, default_token,
)
ds_params = _get_raw_params(raw_path)
async with http_client.put(ds_url, json=ds_params) as resp:
assert resp.status == 200
resp_json = await resp.json()
assert_msg(resp_json, 'CREATE_DATASET')
for k in ds_params['dataset']['params']:
assert ds_params['dataset']['params'][k] == resp_json['details']['params'][k]
# second connect has one dataset:
async with websockets.connect(ws_url) as ws:
initial_msg = json.loads(await ws.recv())
assert initial_msg['messageType'] == "INITIAL_STATE"
assert initial_msg['status'] == "ok"
assert len(initial_msg['datasets']) == 1
assert initial_msg['datasets'][0]["id"] == ds_uuid
assert initial_msg['datasets'][0]["params"] == {
"type": "RAW",
"path": raw_path,
"dtype": "float32",
"sig_shape": [128, 128],
"enable_direct": False,
"nav_shape": [16, 16],
"shape": [16, 16, 128, 128],
"sync_offset": 0
}
def __init__(self,
data: pd.DataFrame,
strategy_type: type(Strategy),
broker_type: type(ExchangeAPI),
cash: float = 10000,
commission: float = .0):
"""
Construct a backtest object.
Parameters：
:param data: pd.DataFrame OHLCV data in pandas Dataframe format
:param broker_type: type(ExchangeAPI) API type of the exchange
:param strategy_type: type(Strategy) Type of the strategy
:param cash: float Initial cash amount
:param commission: float Process fee amount percentage
"""
assert_msg(issubclass(strategy_type, Strategy),
'strategy_type is not an instance of Strategy class')
assert_msg(issubclass(broker_type, ExchangeAPI),
'strategy_type is not an instance of Strategy class')
assert_msg(isinstance(commission, Number),
'commission is not an instance of float class')
data = data.copy(False)
if 'Volume' not in data:
data['Volume'] = np.nan
# validate OHLC
assert_msg(
len(data.columns
& {'Open', 'High', 'Low', 'Close', 'Volume'}) == 5,
("The input data is not in the correct format. It should at least include the following columns："
"'Open', 'High', 'Low', 'Close'"))
# check for null values
assert_msg(
not data[['Open', 'High', 'Low', 'Close']].max().isnull().any(),
('Part of OHLC data has null values, please remove or interpolate those. '
))
# sort based on time
if not data.index.is_monotonic_increasing:
data = data.sort_index()
# initialize exchange and strategy
self._data = data # type: pd.DataFrame
self._broker = broker_type(data, cash, commission)
self._strategy = strategy_type(self._broker, self._data)
self._results = None
async def test_load_raw_success(default_raw, base_url, http_client):
await create_connection(base_url, http_client)
raw_path = default_raw._path
uuid = "ae5d23bd-1f2a-4c57-bab2-dfc59a1219f3"
ds_url = "{}/api/datasets/{}/".format(base_url, uuid)
ds_data = _get_raw_params(raw_path)
async with http_client.put(ds_url, json=ds_data) as resp:
assert resp.status == 200
resp_json = await resp.json()
assert_msg(resp_json, 'CREATE_DATASET')
for k in ds_data['dataset']['params']:
assert ds_data['dataset']['params'][k] == resp_json['details'][
'params'][k]
async def create_default_dataset(default_raw, ws, http_client, base_url):
ds_uuid = _get_uuid_str()
ds_url = "{}/api/datasets/{}/".format(base_url, ds_uuid)
ds_data = _get_raw_params(default_raw._path)
async with http_client.put(ds_url, json=ds_data) as resp:
assert resp.status == 200
resp_json = await resp.json()
assert_msg(resp_json, 'CREATE_DATASET')
# same msg via ws:
msg = json.loads(await ws.recv())
assert_msg(msg, 'CREATE_DATASET')
return ds_uuid, ds_url
async def test_load_raw_fail(default_raw, base_url, http_client):
await create_connection(base_url, http_client)
uuid = "ae5d23bd-1f2a-4c57-bab2-dfc59a1219f3"
ds_url = "{}/api/datasets/{}/".format(base_url, uuid)
ds_data = _get_raw_params("/does/not/exist/")
async with http_client.put(ds_url, json=ds_data) as resp:
assert resp.status == 200
resp_json = await resp.json()
assert_msg(resp_json, 'CREATE_DATASET_ERROR', status='error')
assert resp_json['dataset'] == uuid
assert ("No such file or directory" in resp_json['msg']
or "The system cannot find the path specified"
in resp_json['msg'])
def I(self, func: Callable, *args) -> np.ndarray:
"""
计算买卖指标向量。买卖指标向量是一个数组，长度和历史数据对应；
用于判定这个时间点上需要进行"买"还是"卖"。
例如计算滑动平均：
def init():
self.sma = self.I(utils.SMA, self.data.Close, N)
"""
value = func(*args)
value = np.asarray(value)
assert_msg(value.shape[-1] == len(self._data.Close), '指标长度必须和data长度相同')
self._indicators.append(value)
return value
async def create_update_compound_analysis(
ws,
http_client,
base_url,
ds_uuid,
details=None,
ca_uuid=None,
token=None,
):
if ca_uuid is None:
ca_uuid = _get_uuid_str()
creating = True
else:
creating = False
ca_url = f"{base_url}/api/compoundAnalyses/{ca_uuid}/"
ca_url = add_token(ca_url, token)
if details is None:
details = {
"mainType": "APPLY_RING_MASK",
"analyses": [],
}
else:
assert "analyses" in details
assert "mainType" in details
ca_data = {
"dataset": ds_uuid,
"details": details,
}
print("dispatching PUT request...")
async with http_client.put(ca_url, json=ca_data) as resp:
print(await resp.text())
assert resp.status == 200
resp_json = await resp.json()
assert resp_json['status'] == "ok"
msg = json.loads(await ws.recv())
if creating:
assert_msg(msg, 'COMPOUND_ANALYSIS_CREATED')
else:
assert_msg(msg, 'COMPOUND_ANALYSIS_UPDATED')
assert msg['dataset'] == ds_uuid
assert msg['compoundAnalysis'] == ca_uuid
assert msg['details'] == details
return ca_uuid, ca_url
def __init__(self,
data: pd.DataFrame, # pandas Dataframe格式的历史OHLCV数据
strategy_type: type(Strategy), # 交易所API类型，负责执行买卖操作以及账户状态的维护
broker_type: type(ExchangeAPI), # 策略类型
cash: float = 10000, # 初始资金数量
commission: float = .0): # 每次交易手续费率。如2%的手续费此处为0.02
utils.assert_msg(issubclass(strategy_type, Strategy),
"strategy_type不是一个Strategy类型")
utils.assert_msg(issubclass(broker_type, ExchangeAPI),
"broker_type不是一个ExchangeAPI类型")
utils.assert_msg(isinstance(commission, Number),
"commission不是浮点类型数值")
# False代表浅复制，没有内存拷贝
data = data.copy(False)
# 如果没有Volume列，填充NaN
if "Volume" not in data:
data["Volume"] = numpy.nan
# 验证OHLC数据格式
utils.assert_msg(len(data.columns &
{"Open", "High", "Low", "Close", "Volume"}) == 5,
"部分OHLC包含缺失值，请去掉那些行或者通过差值填充.")
# 如果行情数据没有按时间排序，重新排序一下
if not data.index.is_monotonic_increasing:
data = data.sort_index()
# 利用数据，初始化交易所对象和策略对象
self._data = data
self._broker = broker_type(data, cash, commission)
self._strategy = strategy_type(self._broker, self._data)
self._results = None
def __init__(self, data, cash, commission):
assert_msg(
0 < cash,
"Initial cash amount should be bigger than zero, the input is:：{}".
format(cash))
assert_msg(
0 <= commission <= 0.05,
"A reasonbable process fee is typically lower than 5%, the input is: {}"
.format(commission))
self._inital_cash = cash
self._data = data
self._commission = commission
self._position = 0
self._cash = cash
self._i = 0
async def test_cancel_unknown_job(default_raw, base_url, http_client, server_port):
await create_connection(base_url, http_client)
ws_url = "ws://127.0.0.1:{}/api/events/".format(server_port)
async with websockets.connect(ws_url) as ws:
initial_msg = json.loads(await ws.recv())
assert_msg(initial_msg, 'INITIAL_STATE')
job_uuid = "un-kn-ow-n"
job_url = "{}/api/jobs/{}/".format(base_url, job_uuid)
# try to cancel unknown job:
async with http_client.delete(job_url) as resp:
assert resp.status == 200
assert_msg(await resp.json(), 'CANCEL_JOB_FAILED', status='error')
def __exchange(self, symbol, amount, price, side, Extype):
"""
:param symbol: string Example: 'btcusd"
:param amount: string Example: '1'
:param price: string Example: '9400'
:param side: string Example: 'buy' or 'side'
:param type: string Example: 'exchange limit'
:return:
"""
assert_msg(type(symbol) == str, 'symbol不是一个str类型')
assert_msg(type(amount) == str, 'amount不是一个str类型')
assert_msg(type(price) == str, 'price不是一个str类型')
assert_msg(type(side) == str, 'side不是一个str类型')
assert_msg(type(Extype) == str, 'Extype不是一个str类型')
t = datetime.datetime.now()
payload_nonce = str(int(time.mktime(t.timetuple())))
time.sleep(1)
url = self._base_url + self._exendpoint
payload = {
"request": "/v1/order/new",
"nonce": payload_nonce,
"symbol": symbol,
"amount": amount,
"price": price,
"side": side,
"type": Extype,
}
encoded_payload = json.dumps(payload).encode()
b64 = base64.b64encode(encoded_payload)
signature = hmac.new(self.__gemini_api_secret, b64,
hashlib.sha384).hexdigest()
request_headers = {
'Content-Type': "text/plain",
'Content-Length': "0",
'X-GEMINI-APIKEY': self.__gemini_api_key,
'X-GEMINI-PAYLOAD': b64,
'X-GEMINI-SIGNATURE': signature,
'Cache-Control': "no-cache"
}
response = requests.post(url, data=None, headers=request_headers)
#assert_msg(type(response) == list, response)
new_clearing_order = response.json()