def estimate_gradient_upper(y, eps, x_minus, x_plus, y_minus, y_plus):
y1 = y - eps
y2, z = find_y2(x_minus, x_plus, y_minus, y_plus, y1)
a, b, c = get_abc_upper(y1, y2, z, x_minus, x_plus, y_minus, y_plus)
volume1 = utils.get_volume(a, b, c, x_minus, x_plus, y_minus, y_plus)
y1 = y + eps
y2, z = find_y2(x_minus, x_plus, y_minus, y_plus, y1)
a, b, c = get_abc_upper(y1, y2, z, x_minus, x_plus, y_minus, y_plus)
volume2 = utils.get_volume(a, b, c, x_minus, x_plus, y_minus, y_plus)
gradient = (volume2 - volume1) / (2 * eps)
return gradient
def estimate_gradient_lower(x, eps, x_minus, x_plus, y_minus, y_plus):
x1 = x - eps
x2, z = find_x2(x_minus, x_plus, y_minus, y_plus, x1)
a, b, c = get_abc_lower(x1, x2, z, x_minus, x_plus, y_minus, y_plus)
volume1 = utils.get_volume(a, b, c, x_minus, x_plus, y_minus, y_plus)
x1 = x + eps
x2, z = find_x2(x_minus, x_plus, y_minus, y_plus, x1)
a, b, c = get_abc_lower(x1, x2, z, x_minus, x_plus, y_minus, y_plus)
volume2 = utils.get_volume(a, b, c, x_minus, x_plus, y_minus, y_plus)
gradient = (volume2 - volume1) / (2 * eps)
return gradient
def main_lower(x_minus, x_plus, y_minus, y_plus, plot=False, num=0):
# x1 = (x_minus + x_plus) / 2
# x1 = x_minus
x1 = binary_search_x1(x_minus, x_plus, y_minus, y_plus)
# y1 = binary_search_y1(x_minus, x_plus, y_minus, y_plus)
x2, z = find_x2(x_minus, x_plus, y_minus, y_plus, x1)
a, b, c = get_abc_lower(x1, x2, z, x_minus, x_plus, y_minus, y_plus)
volume = utils.get_volume(a, b, c, x_minus, x_plus, y_minus, y_plus)
if plot:
utils.plot_surface(x_minus[num], x_plus[num], y_minus[num],
y_plus[num], a[num], b[num], c[num])
# x = torch.linspace(x_minus.item(), x_plus.item(),100)
# v = torch.zeros(x.shape)
# g = torch.zeros(x.shape)
# for i in range(len(x)):
# x2,z = find_x2(x_minus, x_plus, y_minus, y_plus, torch.Tensor([x[i]]))
# a,b,c = get_abc_lower(x[i],x2,z,x_minus, x_plus, y_minus, y_plus)
# v[i] = utils.get_volume(a,b,c,x_minus, x_plus, y_minus, y_plus)
# g[i] = estimate_gradient_lower(torch.Tensor([x[i]]), 1e-3, x_minus, x_plus, y_minus, y_plus)
# # v =
# plt.figure()
# plt.plot(x.numpy(),v.numpy())
# plt.figure()
# plt.plot(x.numpy(),g.numpy())
return a, b, c, volume, x1, x2
def train_lower(x0,
y0,
x_minus,
x_plus,
y_minus,
y_plus,
lr=1e-3,
max_iter=100,
print_info=True):
# search over (x,y) \in (x_minus, x_plus) * (y_minus, y_plus)
# the plane z = ax + by + c is the tangent plane of the surface tanh(x) sigmoid(y) at point (x,y)
x = x0.data.clone()
x.requires_grad = True
y = y0.data.clone()
y.requires_grad = True
a_best = torch.zeros(x_minus.shape, device=x_minus.device)
b_best = torch.zeros(x_minus.shape, device=x_minus.device)
c_best = torch.zeros(x_minus.shape, device=x_minus.device)
x_best = torch.zeros(x_minus.shape, device=x_minus.device)
y_best = torch.zeros(x_minus.shape, device=x_minus.device)
optimizer = optim.Adam([x, y], lr=lr)
cubic = torch.abs((x_plus - x_minus) * (y_plus - y_minus) *
torch.tanh(x_minus) * torch.sigmoid(y_plus))
cubic = torch.clamp(cubic, min=1e-3)
v_best = -torch.ones(x_minus.shape, device=x_minus.device) * 1000
#we want to maximize the volume
for i in range(max_iter):
q_loss, valid = qualification_loss_lower(x, y, x_minus, x_plus,
y_minus, y_plus)
a, b, c = plane(x, y)
v_loss = get_volume(a, b, c, x_minus, x_plus, y_minus, y_plus) / cubic
best = (v_loss > v_best) * valid
a_best[best] = a[best]
b_best[best] = b[best]
c_best[best] = c[best]
x_best[best] = x[best]
y_best[best] = y[best]
v_best[best] = v_loss[best]
# print('volume', v_loss)
if print_info:
print('2l q loss: %.4f volume: %.4f' %
(q_loss.mean().item(), v_loss.mean().item()))
loss = q_loss - (valid.float() + 0.1) * v_loss
loss = loss.mean()
optimizer.zero_grad()
loss.backward()
optimizer.step()
# print('x,y',x,y)
return a_best, b_best, c_best, x_best, y_best
def main_upper(x_minus, x_plus, y_minus, y_plus, plot=False, num=0):
y1 = binary_search_y1(x_minus, x_plus, y_minus, y_plus)
y2, z = find_y2(x_minus, x_plus, y_minus, y_plus, y1)
a, b, c = get_abc_upper(y1, y2, z, x_minus, x_plus, y_minus, y_plus)
volume = utils.get_volume(a, b, c, x_minus, x_plus, y_minus, y_plus)
if plot:
utils.plot_surface(x_minus[num], x_plus[num], y_minus[num],
y_plus[num], a[num], b[num], c[num])
return a, b, c, volume, y1, y2
def coin_market_price(self, currency, disable_cache=False):
"""
https://coinmarketcap.com/currencies/<currency>/#markets
:param currency: 虚拟货币的名称 eg：bitcoin
:param disable_cache: 禁用缓存默认为
:return:
"""
endpoint = u'currencies/{}/#markets'.format(currency)
response = self.client.raw_request(self.__CMC_BASE_URL, endpoint, None,
disable_cache)
soup = bs(response, u'html.parser')
table_body = soup.find(u'table', {
u'id': u'markets-table'
}).find(u'tbody')
rows = table_body.find_all(u'tr')
items = []
for row in rows:
tds = row.find_all(u'td')
item = {
u'exchange': tds[1][u'data-sort'],
u'pair': tds[2][u'data-sort'],
u'volume': utils.get_volume(tds[3]),
u'price': utils.get_price(tds[4]),
u'percentage': tds[5][u'data-sort']
}
items.append(item)
resp = {
u'data': items,
u"metadata": {
u"num_prices": len(items),
u"error": None
}
}
return resp
def train_upper(z10,
z20,
z30,
x_minus,
x_plus,
y_minus,
y_plus,
qualification_loss_upper,
message,
lr=1e-3,
max_iter=100,
print_info=True):
# the initial position of the plane z = ax + by + c is determined by the three points
# (x_minus, y_minus, z10), (x_minus, y_plus, z20), (x_plus, y_plus, z30)
# a,b,c can be determined by (x_minus, y_minus, z1), (x_minus, y_plus, z2), (x_plus, y_plus, z3)
# z10,z20,z30,x_minus, x_plus, y_minus, y_plus must be tensors of the same shape
# qualification_loss_upper(x_minus, x_plus, y_minus, y_plus, a,b,c, confidence=-0.0)
# this function indicates whether the plane is valid, namely, plane z = ax + by + c is above the surface z = tanh(x) sigmoid(y)
# in the rectangular area [x_minus, x_plus] * [y_minus， y_plus]
# the plane is valid if and only if qualification_loss_upper <= 0
# This function search over z1,z2,z3 to minimize the volume between the plane z = ax + by + c and the z = 0 plane
# with the constraint qualification_loss_upper <= 0
z1 = z10.data.clone() #(x_minus, y_minus)
z1.requires_grad = True
z2 = z20.data.clone() #(x_minus, y_plus)
z2.requires_grad = True
z3 = z30.data.clone() #(x_plus, y_plus)
z3.requires_grad = True
ones = torch.ones(x_minus.shape, device=x_minus.device)
z1_best = torch.zeros(x_minus.shape, device=x_minus.device)
z2_best = torch.zeros(x_minus.shape, device=x_minus.device)
z3_best = torch.zeros(x_minus.shape, device=x_minus.device)
optimizer = optim.Adam([z1, z2, z3], lr=lr)
cubic = (x_plus - x_minus) * (y_plus - y_minus)
cubic = torch.clamp(cubic, min=1e-3)
v_best = 100000 * torch.ones(x_minus.shape, device=x_minus.device)
for i in range(max_iter):
#x * a + y * b + c = z
a, b, c = get_abc(x_minus, y_minus, ones, z1, x_minus, y_plus, ones,
z2, x_plus, y_plus, ones, z3)
q_loss, valid = qualification_loss_upper(x_minus,
x_plus,
y_minus,
y_plus,
a,
b,
c,
confidence=-0.0)
v_loss = get_volume(a, b, c, x_minus, x_plus, y_minus, y_plus) / cubic
best = (v_loss < v_best) * valid
z1_best[best] = z1[best]
z2_best[best] = z2[best]
z3_best[best] = z3[best]
v_best[best] = v_loss[best]
# print('volume', v_loss)
if print_info:
print(message + ' q loss: %.4f volume: %.4f' %
(q_loss.mean().item(), v_loss.mean().item()))
loss = q_loss + (valid.float() + 0.1) * v_loss
loss = loss.mean()
optimizer.zero_grad()
loss.backward()
z1.grad[z1.grad != z1.grad] = 0
z2.grad[z2.grad != z2.grad] = 0
z3.grad[z3.grad != z3.grad] = 0
optimizer.step()
a_best, b_best, c_best = get_abc(x_minus, y_minus, ones, z1_best, x_minus,
y_plus, ones, z2_best, x_plus, y_plus,
ones, z3_best)
return a_best, b_best, c_best
def train_lower(u0, v0, ka0, kb0, x_minus, x_plus, y_minus, y_plus,
lr_x = 1e-3, lr_k=1e-2, max_iter = 100, print_info = True):
device = x_minus.device
x_best = torch.zeros(x_minus.shape).to(device)
y_best = torch.zeros(x_minus.shape).to(device)
a_best = torch.zeros(x_minus.shape).to(device)
b_best = torch.zeros(x_minus.shape).to(device)
c_best = torch.zeros(x_minus.shape).to(device)
ka_best = torch.zeros(x_minus.shape).to(device)
kb_best = torch.zeros(x_minus.shape).to(device)
cubic = -(x_plus-x_minus) * (y_plus-y_minus) * torch.tanh(x_minus) * torch.sigmoid(y_plus)
cubic = torch.clamp(cubic, min=1e-3)
v_best = -cubic/cubic * 10000
# eps = 0.1
u = u0.data.clone()#torch.clamp(x_plus.data.clone(), max=3)#torch.rand(x_plus.shape)#torch.Tensor([3])
v = v0.data.clone()#torch.clamp(y_plus.data.clone(), max=3)#torch.rand(y_plus.shape)#torch.Tensor([3])
ka = ka0.data.clone()#torch.Tensor([1])
kb = kb0.data.clone()#torch.Tensor([1])
u.requires_grad = True
v.requires_grad = True
ka.requires_grad = True
kb.requires_grad = True
# optimizer = optim.SGD([u,v,ka,kb], lr=lr, momentum=momentum)
optimizer_x = optim.Adam([u,v], lr=lr_x)
optimizer_k = optim.Adam([ka,kb], lr=lr_k)
max_iter = max_iter
# tanh_l_min = tanh_lmin(x_minus, y_minus)
# sigmoid_l_min = sigmoid_lmin(x_plus, y_minus)
for i in range(max_iter):
#x: 0 to x_minus <=0
#y: 0 to y_plus
slop = 0.01
u_minus = -F.leaky_relu(-u, negative_slope=slop)
#this makes u_minus grows much slower when u>=0
idx_v = (v>=y_minus).float()
v_plus = v * idx_v + (1-idx_v)*(slop*(v-y_minus)+y_minus)
#this makes v_plus decrease slower when v< y_minus
idx_x = (u>=x_minus).float()
x = u_minus * idx_x + (1-idx_x)*(slop*(u_minus-x_minus)+x_minus)
#make x decrease slower when u<x_minus
idx_y = (v<=y_plus).float()
y = v_plus * idx_y + (1-idx_y)*(slop*(v_plus-y_plus)+y_plus)
#make y grows slower when v>y_plus
a,b,c = plane(x,y)
idx = (x<=x_minus).float()
a = a - F.leaky_relu(ka, negative_slope=slop) * idx
c = c + F.leaky_relu(ka, negative_slope=slop) * x * idx
#ka = ka * idx
#if x<=x_minus, we keep its original value
#if x>x_minus, we reset it to 0
idx = (y>=y_plus).float()
b = b + F.leaky_relu(kb, negative_slope=slop) * idx
c = c - F.leaky_relu(kb, negative_slope=slop) * y * idx
# q_loss, valid = qualification_loss_lower(a,b,c,x_minus, x_plus, y_minus, y_plus,
# tanh_l_min,
# sigmoid_l_min, confidence=-0)
q_loss, valid = qualification_loss(x_minus, x_plus, y_minus, y_plus,
a, b ,c, confidence=-1e-3)
# print('q_loss:',q_loss)
v_loss = get_volume(a,b,c,x_minus, x_plus, y_minus, y_plus)
v_loss = v_loss/cubic
# print('volume', v_loss)
if print_info:
print('12l q loss: %.4f volume: %.4f' % (q_loss.mean().item(), v_loss.mean().item()))
loss = (q_loss - v_loss*(valid.float()+0.01)).mean() #we want to maximize the volume
best = (v_loss > v_best) * (valid)
v_best[best] = v_loss[best]
x_best[best] = x[best]
y_best[best] = y[best]
ka_best[best] = ka[best]
kb_best[best] = kb[best]
a_best[best] = a[best]
b_best[best] = b[best]
c_best[best] = c[best]
optimizer_x.zero_grad()
optimizer_k.zero_grad()
loss.backward()
idx = (y>y_plus) * (kb>0)
#if y>y_plus and kb>0, we don't move v
v.grad = v.grad * (1-idx.float())
idx = (x<x_minus) * (ka>0)
#if x<x_minus and ka>0, we don't move u
u.grad = u.grad * (1-idx.float())
# if y>=y_plus and kb >=0:
# v.grad = v.grad * 0
# if x<=x_plus and ka >=0:
# u.grad = u.grad * 0
optimizer_x.step()
optimizer_k.step()
# print('u,v:',u,v)
return x_best,y_best,ka_best,kb_best,a_best,b_best,c_best,v_best
def train_lower(u0,
v0,
ka0,
kb0,
x_minus,
x_plus,
y_minus,
y_plus,
lr_x=1e-3,
lr_k=1e-2,
max_iter=100,
print_info=True):
device = x_minus.device
x_best = torch.zeros(x_minus.shape).to(device)
y_best = torch.zeros(x_minus.shape).to(device)
a_best = torch.zeros(x_minus.shape).to(device)
b_best = torch.zeros(x_minus.shape).to(device)
c_best = torch.zeros(x_minus.shape).to(device)
ka_best = torch.zeros(x_minus.shape).to(device)
kb_best = torch.zeros(x_minus.shape).to(device)
cubic = (x_plus - x_minus) * (y_plus - y_minus)
cubic = torch.clamp(cubic, min=1e-4)
v_best = -torch.ones(x_minus.shape).to(device)
# eps = 0.1
u = u0 #torch.clamp(x_plus.data.clone(), max=3)#torch.rand(x_plus.shape)#torch.Tensor([3])
v = v0 #torch.clamp(y_plus.data.clone(), max=3)#torch.rand(y_plus.shape)#torch.Tensor([3])
ka = ka0 #torch.Tensor([1])
kb = kb0 #torch.Tensor([1])
u.requires_grad = True
v.requires_grad = True
ka.requires_grad = True
kb.requires_grad = True
# optimizer = optim.SGD([u,v,ka,kb], lr=lr, momentum=momentum)
optimizer_x = optim.Adam([u, v], lr=lr_x)
optimizer_k = optim.Adam([ka, kb], lr=lr_k)
max_iter = max_iter
for i in range(max_iter):
#x: 0 to x_minus <=0
#y: 0 to y_plus
slop = 0.01
u_minus = -F.leaky_relu(-u, negative_slope=0.01)
v_plus = F.leaky_relu(v, negative_slope=0.01)
idx_x = (u >= x_minus).float()
x = u_minus * idx_x + (1 - idx_x) * (slop *
(u_minus - x_minus) + x_minus)
idx_y = (v <= y_plus).float()
y = v_plus * idx_y + (1 - idx_y) * (slop * (v_plus - y_plus) + y_plus)
a, b, c = plane(x, y)
idx = (x <= x_minus).float()
a = a - F.leaky_relu(ka, negative_slope=0.01) * idx
c = c + F.leaky_relu(ka, negative_slope=0.01) * x * idx
#ka = ka * idx
#if x<=x_minus, we keep its original value
#if x>x_minus, we reset it to 0
idx = (y >= y_plus).float()
b = b + F.leaky_relu(kb, negative_slope=0.01) * idx
c = c - F.leaky_relu(kb, negative_slope=0.01) * y * idx
q_loss, valid = qualification_loss_lower(a,
b,
c,
x_minus,
x_plus,
y_minus,
y_plus,
confidence=-0)
# print('q_loss:',q_loss)
v_loss = get_volume(a, b, c, x_minus, x_plus, y_minus, y_plus)
v_loss = v_loss / cubic
# print('cubic', cubic.min())
# print('u,v', u.max(), u.min(), v.max(), v.min())
# print('ka,kb', ka.max(), ka.min(), ka.max(), ka.min())
# # print('volume', v_loss)
# print('a,b,c',a.max(),b.max(),c.max(), a.min,b.min,c.min)
if print_info:
print('all 4l q_loss %.4f, volume %.4f' %
(q_loss.mean().item(), v_loss.mean().item()))
loss = q_loss - (v_loss * (valid.float() + 0.1)).mean()
best = (v_loss > v_best) * (valid)
v_best[best] = v_loss[best]
x_best[best] = x[best]
y_best[best] = y[best]
ka_best[best] = ka[best]
kb_best[best] = kb[best]
a_best[best] = a[best]
b_best[best] = b[best]
c_best[best] = c[best]
optimizer_x.zero_grad()
optimizer_k.zero_grad()
loss.backward()
idx = (y > y_plus) * (kb > 0)
v.grad = v.grad * (1 - idx.float())
idx = (x < x_minus) * (ka > 0)
u.grad = u.grad * (1 - idx.float())
# print('u grad', u.grad)
# print('v grad', v.grad)
# nan = (u.grad != u.grad)
# print('not a number', u0[nan], v0[nan], ka0[nan], kb0[nan],
# x_minus[nan], x_plus[nan], y_minus[nan], y_plus[nan])
# print('u', u)
# print('v', v)
# if y>=y_plus and kb >=0:
# v.grad = v.grad * 0
# if x<=x_plus and ka >=0:
# u.grad = u.grad * 0
u.grad[u.grad != u.grad] = 0
v.grad[v.grad != v.grad] = 0
ka.grad[ka.grad != ka.grad] = 0
kb.grad[kb.grad != kb.grad] = 0
optimizer_x.step()
optimizer_k.step()
# print('u,v:',u,v)
return x_best, y_best, ka_best, kb_best, a_best, b_best, c_best, v_best