def d2_pi_sigmaj(i, j, p, mu, sigma, x, proportional=None):
if i == j:
diff = ratio(num=[np.dot(p, dnorm(x, mu, sigma)) * dnorm(x, mu[i], sigma[i]) -
p[i] * dnorm(x, mu[i], sigma[i] ** 2)],
den=[np.dot(p, dnorm(x, mu, sigma)) ** 2])
num = [(mu[i] - x) ** 2 / (sigma[i] ** 2) - 1, diff]
den = [sigma[i]]
else:
num = [- p[j], (x - mu[j]) ** 2 / sigma[j] ** 2 - 1, dnorm(x, mu[i], sigma[i]), dnorm(x, mu[j], sigma[j])]
den = [sigma[j], (np.dot(p, dnorm(x, mu, sigma)) ** 2)]
return ratio(num, den)
def d2_mui_muj(i, j, p, mu, sigma, x, proportional=None):
if i == j:
#diff = 1 - ratio([p[i] * dnorm(x, mu[i], sigma[i])], [np.dot(p, dnorm(x, mu, sigma))])
num1 = [p[i], dnorm(x, mu[i], sigma[i]), ((x - mu[i]) / sigma[i] ** 2)**2 - 1/(sigma[i]**2)]
den = [np.dot(p, dnorm(x, mu, sigma))]
num2=[p[i],dnorm(x, mu[i], sigma[i]), (x-mu[i])/(sigma[i]**2)]
ratio1=ratio(num1,den)
ratio2=ratio(num2,den)**2
return(ratio1-ratio2)
else:
# TODO : Check this derivative
num = [- p[i], p[j], x - mu[i], x - mu[j], dnorm(x, mu[i], sigma[i]), dnorm(x, mu[j], sigma[j])]
den = [sigma[i] ** 2, sigma[j] ** 2, np.dot(p, dnorm(x, mu, sigma)) ** 2]
return ratio(num, den)
def insert(self, total_data):
# data order:
# 公司代码 公司简称 董监高姓名 职务 股票种类 货币种类 本次变动前持股数
# 变动数 本次变动平均价格 变动后持股数 变动原因 变动日期 填报日期
number = (lambda num: num.replace(',', ''))
for item in total_data:
currency = data_by_table_type({'zhsname': item[5]}, [('code',)], 'curr')
secu_code, orgid = data_by_table_type({'tick': item[0]}, [('code', ), ('org', 'id')], 'stock')
query_name_en = {'name.szh': item[2], 'orgid': orgid}
name_en, pid = data_by_table_type(query_name_en, [('name', 'en'), ('pid',)], 'exec')
to_ratio, cir_ratio = ratio(number(item[7]), secu_code, 'stock', 'vary')
uid_uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, ''.join(item).encode('u8')))
data = {
'secu': secu_code or item[0], 'org': orgid, 'scp': {'szh': '', 'en': ''}, 'uuid': uid_uuid,
'name': {'szh': item[2], 'en': name_en}, 'relation': '', 'pid': pid,
'change': item[7], 'after': item[9], 'cur': currency, 'cause': item[10], 'cd': item[11],
'rd': item[12], 'stat': 1, 'price': item[8], 'upt': datetime.now(), 'upu': 'system',
'torat': to_ratio, 'cirrat': cir_ratio, 'typ': 'sha'
}
if not self._coll_in.get({'uuid': uid_uuid}, {'secu': 1}):
self._coll_in.insert(data)
else:
print 'uuid existed:', uid_uuid
self._coll_in.disconnect()
def d2_sigmai_sigmaj(i, j, p, mu, sigma, x, proportional=None):
# if proportional:
# num = [(p[i] ** 2 / sigma[i] ** 2), (((x - mu[i]) ** 2 / sigma[i] ** 2 - 1) * dnorm(x, mu[i], sigma[i])) ** 2]
# den = [np.dot(p, dnorm(x, mu, sigma))]
num = [(p[i] * p[j]), ((x - mu[i]) ** 2 / sigma[i] ** 2 - 1), dnorm(x, mu[i], sigma[i]),
((x - mu[j]) ** 2 / sigma[j] ** 2 - 1), dnorm(x, mu[j], sigma[j])]
den = [(sigma[i] * sigma[j]), np.dot(p, dnorm(x, mu, sigma))]
return ratio(num, den)
def d2_mui_sigmaj(i, j, p, mu, sigma, x, proportional=None):
if i == j:
num = [p[i] ** 2, (x - mu[i]), ((x - mu[i]) ** 2 / sigma[i] ** 2 - 1), dnorm(x, mu[i], sigma[i]) ** 2]
den = [sigma[i] ** 3, np.dot(p, dnorm(x, mu, sigma))]
else:
num = [(p[i] * p[j]), (x - mu[i]), ((x - mu[j]) ** 2 / sigma[j] ** 2 - 1), dnorm(x, mu[j], sigma[j]),
dnorm(x, mu[i], sigma[i])]
den = [(sigma[i] * sigma[j]), (sigma[i] ** 2 * sigma[j]), np.dot(p, dnorm(x, mu, sigma))]
return ratio(num, den)
def d2_pi_muj(i, j, p, mu, sigma, x, proportional=None):
if i == j:
num = [(x - mu[i]), dnorm(x, mu[i], sigma[i]),
(p[i] * dnorm(x, mu[i], sigma[i]) - np.dot(p, dnorm(x, mu, sigma)))]
den = [sigma[i] ** 2, np.dot(p, dnorm(x, mu, sigma)) ** 2]
else:
num = [p[j], (mu[j] - x), dnorm(x, mu[j], sigma[j]), dnorm(x, mu[i], sigma[i])]
den = [sigma[j] ** 2, np.dot(p, dnorm(x, mu, sigma)) ** 2]
return ratio(num=num, den=den)
def main(self, multi_pool_page=None):
assert multi_pool_page is None or isinstance(multi_pool_page, int), '`multi_pool_page` must None or int.'
if multi_pool_page is None:
start_page, end_page = 1, self.crawl_pages
else:
start_page, end_page = multi_pool_page, multi_pool_page
for page in range(start_page, end_page + 1):
url = self._base_url + self._query_string.format(page)
secus, changes, afters, prices, scps, ncdrs = self.parse_data(url)
for i in range(len(secus)):
# Getting secu_code, orgid with `tick` from coll_stock table
# Getting name_en, pid with `董监高姓名` from coll_exec table
# Getting to_ratio, cir_ratio with `变动股份数量` and `secu_code` from coll_stock and coll_vary table
# update data change_date greater than self._latest_cd_data
cp_flag = self._latest_cd_data < ncdrs[i][1]
secu_code, orgid = data_by_table_type({'tick': secus[i]}, [('code',), ('org', 'id')], 'stock')
query_name_en__pid = {'name.szh': ncdrs[i][0], 'orgid': orgid}
name_en, pid = data_by_table_type(query_name_en__pid, [('name', 'en'), ('pid',)], 'exec')
to_ratio, cir_ratio = ratio(changes[i], secu_code, 'stock', 'vary')
# uuid is unique identifier
name = ''.join([secus[i], ncdrs[i][0], ncdrs[i][1], changes[i], prices[i], ncdrs[i][2],
afters[i], scps[i].replace('\\', ''), ncdrs[i][3]])
uid_uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, name))
data = {
'secu': secu_code or secus[i], 'org': orgid, 'change': changes[i], 'after': afters[i],
'price': prices[i], 'cd': ncdrs[i][1], 'cause': ncdrs[i][2], 'relation': ncdrs[i][3],
'stat': 1, 'name': {'szh': ncdrs[i][0], 'en': name_en},
'scp': {
'szh': MarkReplace(secu_code, scps[i].replace('\\', '')).replace_mark(),
'en': data_by_table_type({'name.szh': scps[i]}, [('name', 'en')], 'exec')
},
'cur': 'CNY', 'rd': '', 'pid': pid, 'uuid': uid_uuid, 'typ': 'szx',
'upu': 'system', 'upt': datetime.now(), 'torat': to_ratio, 'cirrat': cir_ratio
}
if not cp_flag and not self._coll_in.get({'uuid': uid_uuid}, {'secu': 1}):
self._coll_in.insert(data)
elif cp_flag:
self._coll_in.insert(data)
print 'page: [{0}] done!'.format(page)
self._coll_in.disconnect()
def d2_pi_pj(i, j, p, mu, sigma, x, proportional=None):
num = [- dnorm(x, mu[i], sigma[i]), dnorm(x, mu[j], sigma[j])]
den = [np.dot(p, dnorm(x, mu, sigma)) ** 2]
return ratio(num=num, den=den)
from tools import ratio num = [4, 0.0000001] den = [0.00000000000000000000000000000000000000000000000000001, 1] rat = ratio(num, den, log=True) print(rat)
def main(self, multi_pool_page=None):
assert multi_pool_page is None or isinstance(
multi_pool_page, int), '`multi_pool_page` must None or int.'
if multi_pool_page is None:
start_page, end_page = 1, self.crawl_pages
else:
start_page, end_page = multi_pool_page, multi_pool_page
for page in range(start_page, end_page + 1):
url = self._base_url + self._query_string.format(page)
secus, changes, afters, prices, scps, ncdrs = self.parse_data(url)
for i in range(len(secus)):
# Getting secu_code, orgid with `tick` from coll_stock table
# Getting name_en, pid with `董监高姓名` from coll_exec table
# Getting to_ratio, cir_ratio with `变动股份数量` and `secu_code` from coll_stock and coll_vary table
# update data change_date greater than self._latest_cd_data
cp_flag = self._latest_cd_data < ncdrs[i][1]
secu_code, orgid = data_by_table_type({'tick': secus[i]},
[('code', ),
('org', 'id')], 'stock')
query_name_en__pid = {'name.szh': ncdrs[i][0], 'orgid': orgid}
name_en, pid = data_by_table_type(query_name_en__pid,
[('name', 'en'),
('pid', )], 'exec')
to_ratio, cir_ratio = ratio(changes[i], secu_code, 'stock',
'vary')
# uuid is unique identifier
name = ''.join([
secus[i], ncdrs[i][0], ncdrs[i][1], changes[i], prices[i],
ncdrs[i][2], afters[i], scps[i].replace('\\',
''), ncdrs[i][3]
])
uid_uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, name))
data = {
'secu': secu_code or secus[i],
'org': orgid,
'change': changes[i],
'after': afters[i],
'price': prices[i],
'cd': ncdrs[i][1],
'cause': ncdrs[i][2],
'relation': ncdrs[i][3],
'stat': 1,
'name': {
'szh': ncdrs[i][0],
'en': name_en
},
'scp': {
'szh':
MarkReplace(secu_code,
scps[i].replace('\\', '')).replace_mark(),
'en':
data_by_table_type({'name.szh': scps[i]},
[('name', 'en')], 'exec')
},
'cur': 'CNY',
'rd': '',
'pid': pid,
'uuid': uid_uuid,
'typ': 'szx',
'upu': 'system',
'upt': datetime.now(),
'torat': to_ratio,
'cirrat': cir_ratio
}
if not cp_flag and not self._coll_in.get({'uuid': uid_uuid},
{'secu': 1}):
self._coll_in.insert(data)
elif cp_flag:
self._coll_in.insert(data)
print 'page: [{0}] done!'.format(page)
self._coll_in.disconnect()