def FillItemList(table):
rows = table["body"]
rows = rows[1:]
if 11 != len(rows[0]):
return None
table["body"] = rows
arr = CreateEmptyItemList(table)
for row in rows:
key = row[0]
rowIndex = 1
while rowIndex < len(arr):
if key != "筹码集中度" and True != CHECKER.IsDate(row[rowIndex]):
arr[rowIndex - 1][key] = CONVERT.UnitStrToFloat(row[rowIndex])
elif "筹码集中度" == key:
arr[rowIndex - 1][key] = row[rowIndex]
else:
if 11 == len(row[rowIndex]):
"20" + row[rowIndex]
arr[rowIndex - 1][key + "Tag"] = CONVERT.DateToInt(
row[rowIndex])
arr[rowIndex - 1][key] = row[rowIndex]
arr[rowIndex - 1]["DateTag"] = CONVERT.DateToInt(row[rowIndex])
rowIndex += 1
return arr
def SaveData北向成交明细ToRedis(code):
jsonStr = HtmlConvertor.GetInst().LoadFromString(
html=chrome.page_source).ConvertToJson()
jsonData = json.loads(jsonStr)
if ("Tables" in jsonData and [] != jsonData["Tables"]):
tables = jsonData["Tables"]
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 9 == len(row):
日期Tag = CONVERT.DateToInt(row[0])
item = {
"Code": code,
"日期": row[0],
"日期Tag": CONVERT.DateToInt(row[0]),
"当日排名": CONVERT.StrToInt(row[2]),
"收盘价": CONVERT.StrToFloat(row[3]),
"涨跌幅": CONVERT.UnitStrToFloat(row[4]),
"沪深股通净买额": CONVERT.UnitStrToFloat(row[5]),
"沪深股通买入金额": CONVERT.UnitStrToFloat(row[6]),
"沪深股通卖出金额": CONVERT.UnitStrToFloat(row[7]),
"沪深股通成交金额": CONVERT.UnitStrToFloat(row[8])
}
print(item)
qName北向成交 = "Stock:BXCJMX:%s" % code
r.SortDictSave(qName北向成交, item, 日期Tag)
def SaveData北向成交明细ToRedis(code):
jsonStr = HtmlConvertor.GetInst().LoadFromString(
html=chrome.page_source).ConvertToJson()
jsonData = json.loads(jsonStr)
if ("Tables" in jsonData and [] != jsonData["Tables"]):
tables = jsonData["Tables"]
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 9 == len(row):
日期Tag = CONVERT.DateToInt(row[0])
item = {
"Code": code,
"日期": row[0],
"日期Tag": CONVERT.DateToInt(row[0]),
"当日排名": CONVERT.StrToInt(row[2]),
"收盘价": CONVERT.StrToFloat(row[3]),
"涨跌幅": CONVERT.UnitStrToFloat(row[4]),
"沪深股通净买额": CONVERT.UnitStrToFloat(row[5]),
"沪深股通买入金额": CONVERT.UnitStrToFloat(row[6]),
"沪深股通卖出金额": CONVERT.UnitStrToFloat(row[7]),
"沪深股通成交金额": CONVERT.UnitStrToFloat(row[8])
}
print("北向成交明细 Task%s %s" % (taskID, item))
qName北向成交 = "Stock:BXCJMX:%s" % code
targetNameSpace = qName北向成交
r.SortDictSave(qName北向成交, item, 日期Tag)
SyncNotice.SendSyncNotice(
targetNameSpace, {
"namespace": targetNameSpace,
"code": code,
"score": 日期Tag,
"value": item,
"type": "SortedSet"
})
def AddToArray(jgCode, jgName, targetArr):
jsonStr = HtmlConvertor.GetInst().LoadFromString(
chrome.page_source).ConvertToJson()
jsonData = json.loads(jsonStr)
tables = jsonData["Tables"]
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 12 == len(row):
持股日期Tag = CONVERT.DateToInt(row[0])
item = {
"持股日期": row[0],
"持股日期Tag": 持股日期Tag,
"Code": row[1],
"股票简称": row[2],
"当日收盘价": CONVERT.StrToFloat(row[3]),
"当日涨跌幅": CONVERT.StrToFloat(row[4]),
"机构名称": jgName,
"jgCode": jgCode,
"持股数量": CONVERT.UnitStrToFloat(row[6]),
"持股市值": CONVERT.UnitStrToFloat(row[7]),
"持股数量占A股百分比": CONVERT.StrToFloat(row[8]),
"持股市值变化1日": CONVERT.UnitStrToFloat(row[9]),
"持股市值变化5日": CONVERT.UnitStrToFloat(row[10]),
"持股市值变化10日": CONVERT.UnitStrToFloat(row[11])
}
targetArr.append(item)
return targetArr
def ProcWebData():
taskId = "Task0"
serverUrl = "http://122.51.159.248:5000/YunStock2Service?keyName=JGDYLB&taskId=%s" % taskId
#serverUrl = "http://127.0.0.1:5000/YunStock2Service?keyName=JGDYLB&taskId=%s" % taskId
count = 0
while True:
try:
res = requests.get(serverUrl)
print("接收 %s \r\n --------\r\n" % res.text)
res = json.loads(res.text)
if (res["success"] == True):
data = res["data"]
if 10 <= len(data):
data = json.loads(data)
url = data["Url"]
code = data["Code"]
resArr = []
jsonStr = spider.LoadWeb(url, "机构调研列表").GetDataFromWeb()
jsonData = json.loads(jsonStr)
if "Tables" in jsonData:
tables = jsonData["Tables"]
if 0 < len(tables):
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 10 == len(row):
持股日期Tag = CONVERT.DateToInt(row[0])
item = {
"Code": code,
"接待机构数量": CONVERT.StrToInt(row[4]),
"接待方式": row[5],
"接待人员": row[6],
"接待地点": row[7],
"接待日期": row[8],
"公告日期": row[9],
"接待日期Tag": CONVERT.DateToInt(row[8]),
"公告日期Tag": CONVERT.DateToInt(row[9])
}
resArr.append(item)
post_data = {
"taskId": taskId,
"keyName": "JGDYLB",
"jsonReq": json.dumps(data, ensure_ascii=False),
"jsonRes": json.dumps(resArr, ensure_ascii=False)
}
res2 = requests.post(serverUrl, data=post_data)
print("POST %s \r\n RES %s \r\n --------- \r\n" %
(post_data, res2.text))
else:
time.sleep(3)
except BaseException as e:
print(" 异常 %s " % e)
time.sleep(20)
pass
def callback(qName, input):
print(input)
data = json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
if 0 == len(tables):
return None
table = data["Tables"][0]
rows = table["body"]
for row in rows:
if (10 == len(row)):
序号 = row[0]
代码 = row[1]
名称 = row[2]
接待机构数量 = CONVERT.StrToInt(row[4])
接待方式 = row[5]
接待人员 = row[6]
接待地点 = row[7]
接待日期 = row[8]
接待日期Tag = 0
if (10 == len(接待日期)):
接待日期Tag = CONVERT.DateToInt(接待日期)
公告日期 = row[9]
公告日期Tag = 0
if 10 != len(公告日期) and 10 == len(接待日期):
公告日期 = 接待日期.split("-")[0] + "/" + 公告日期
公告日期Tag = CONVERT.DateToInt(公告日期)
item = {
"Code": code,
"Name": name,
"序号": 序号,
"代码": 代码,
"名称": 名称,
"接待机构数量": 接待机构数量,
"接待方式": 接待方式,
"接待人员": 接待人员,
"接待地点": 接待地点,
"接待日期": 接待日期,
"接待日期Tag": 接待日期Tag,
"公告日期": 公告日期,
"公告日期Tag": 公告日期Tag
}
dictName = "PreData:股票:%s:机构调研" % code
r.SortDictSave(dictName, json.dumps(item, ensure_ascii=False),
接待日期Tag)
print(item)
pass
def ProcWebData():
taskId = "Task0"
serverUrl = "http://122.51.159.248/YunStock2Service?keyName=BXCJMX&taskId=%s" % taskId
#serverUrl="http://127.0.0.1:80/YunStock2Service?keyName=BXCJMX&taskId=%s"%taskId
while True:
try:
res = requests.get(serverUrl)
print("接收 %s \r\n --------\r\n" % res.text)
res = json.loads(res.text)
if (res["success"] == True):
data = res["data"]
if 10 <= len(data):
data = json.loads(data)
code = data["Code"]
url = data["Url"]
resArr = []
jsonStr = spider.LoadWeb(url, "北向成交明细").GetDataFromWeb()
jsonData = json.loads(jsonStr)
if ("Tables" in jsonData and [] != jsonData["Tables"]):
tables = jsonData["Tables"]
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 9 == len(row):
日期Tag = CONVERT.DateToInt(row[0])
item = {
"Code": code,
"日期": row[0],
"日期Tag": CONVERT.DateToInt(row[0]),
"当日排名": CONVERT.StrToInt(row[2]),
"收盘价": CONVERT.StrToFloat(row[3]),
"涨跌幅": CONVERT.UnitStrToFloat(row[4]),
"沪深股通净买额": CONVERT.UnitStrToFloat(row[5]),
"沪深股通买入金额": CONVERT.UnitStrToFloat(row[6]),
"沪深股通卖出金额": CONVERT.UnitStrToFloat(row[7]),
"沪深股通成交金额": CONVERT.UnitStrToFloat(row[8])
}
resArr.append(item)
post_data = {
"keyName": "BXCJMX",
"taskId": taskId,
"method": "SaveBXCJMX",
"jsonReq": json.dumps(data, ensure_ascii=False),
"jsonRes": json.dumps(resArr, ensure_ascii=False)
}
res2 = requests.post(serverUrl, data=post_data)
print("POST %s \r\n RES %s \r\n --------- \r\n" %
(post_data, res2.text))
else:
time.sleep(10)
except BaseException as e:
print(" 异常 %s " % e)
time.sleep(20)
pass
def callback(qName,input):
#input = input.encode(encoding= "gbk").decode(encoding="utf-8")
#print(input)
data=json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
if 3!=len(tables):
return None
table=data["Tables"][2]
rows=table["body"]
for row in rows:
if 13== len(row):
序号 =CONVERT.StrToInt (row[0])
交易标的 =row[1]
买方 = row[2]
卖方 = row[3]
交易金额 = CONVERT.UnitStrToFloat(row[4])
币种 =row[5]
股权转让比例 = CONVERT.StrToFloat(row[6])
标的类型 = row[7]
并购方式= row[8]
最新进展= row[9]
披露日期= row[10]
披露日期Tag= CONVERT.DateToInt(row[10])
最新公告日= row[11]
最新公告日Tag= CONVERT.DateToInt(row[11])
item={
"Code":code
,"Name":name
,"序号":序号
,"交易标的":交易标的
,"买方":买方
,"卖方":卖方
,"交易金额":交易金额
,"币种":币种
,"股权转让比例":股权转让比例
,"标的类型":标的类型
,"并购方式":并购方式
,"最新进展":最新进展
,"披露日期":披露日期
,"披露日期Tag":披露日期Tag
,"最新公告日":最新公告日
,"最新公告日Tag":最新公告日Tag
}
dictName="PreData:股票:%s:并购重组"%code
r.SortDictSave(dictName,json.dumps(item,ensure_ascii=False),最新公告日Tag)
print(item)
pass
def 最新盈利能力指标(arr, count=8):
currentDateTag = CONVERT.GetCurrentDateTag()
arr = sorted(arr, key=lambda k: k["盈利能力指标ag"], reverse=True)
size = len(arr)
count = count if count < size else size
subArr = arr[:count]
subArr = list(filter(lambda k: k["盈利能力指标Tag"] >= 20190930,
subArr)) #时间段要改进
if 0 == len(subArr):
return False
subArr加权净资产收益率 = list(map(lambda x: x["加权净资产收益率(%)"], subArr))
subArr主力净流入净占比中位数 = median(subArr主力净流入净占比)
find = True and 0 < len(subArr)
for item in subArr:
#print("%s %s %s"%(arr[0]["Code"],arr[0]["Name"],item["股东户数增减比例"]))
if item["加权净资产收益率(%)"] > 0:
find = find and False
break
elif item["主力净流入净占比"] == 0:
find = find and False
break
elif item["主力净流入净占比"] < 0 and subArr主力净流入净占比中位数 < 0:
find = find and True
return find
pass
def callback(qName, input):
#print(input)
data = json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
dateTag = CONVERT.StrToInt(data["DateTag"])
article = data["Article"]
uri = urllib.parse.urlparse(data["Uri"].strip())
hostname = uri.hostname
scheme = uri.scheme
#print(article)
points = article.split("\n")
for point in points:
point = point.strip()
if (5 <= len(point) and CHECKER.HasHanZi(point)):
print(point)
item = {
"Code": code,
"Name": name,
"Section": point,
"DateTag": dateTag
}
dictName = "PreData:股票:%s:核心题材" % code
r.SortDictSave(dictName, item, dateTag)
pass
def 连续N次股东户数减少(arr, count=1):
currentDateTag = CONVERT.GetCurrentDateTag()
arr = sorted(arr, key=lambda k: k["股东户数统计截止日Tag"], reverse=True)
size = len(arr)
count = count if count < size else size
subArr = arr[:count]
subArr = list(
filter(lambda k: k["股东户数统计截止日Tag"] >= currentDateTag - 100,
subArr)) #时间段要改进
if 0 == len(subArr):
return False
subArr股东户数增减比例 = list(map(lambda x: x["股东户数增减比例"], subArr))
subArr股东户数增减比例中位数 = median(subArr股东户数增减比例)
find = True and 0 < len(subArr)
for item in subArr:
#print("%s %s %s"%(arr[0]["Code"],arr[0]["Name"],item["股东户数增减比例"]))
if item["股东户数增减比例"] > 0:
find = find and False
break
elif item["股东户数增减比例"] == 0:
find = find and False
break
elif item["股东户数增减比例"] < 0 and subArr股东户数增减比例中位数 < 0:
find = find and True
return find
pass
def 最新N月内机构调研(arr,count=3):
arr = list(filter(lambda k:"接待日期Tag" in k,arr)) #
currentDateTag = CONVERT.GetCurrentDateTag()
arr = sorted(arr, key = lambda k:k["接待日期Tag"],reverse=True)
size = len(arr)
count = count if count<size else size
subArr = arr[:count]
subArr = list(filter(lambda k:k["接待日期Tag"]>= 20191101,subArr)) #时间段要改进
if 0== len(subArr):
return False
subArr接待机构数量=list(map(lambda x:x["接待机构数量"],subArr))
subArr接待机构数量中位数 = median(subArr接待机构数量)
find=True and 0<len(subArr)
for item in subArr:
#print("%s %s %s"%(arr[0]["Code"],arr[0]["Name"],item["股东户数增减比例"]))
if 8<=item["接待机构数量"] and 8<=subArr接待机构数量中位数:
find=find and True
elif item["接待机构数量"]==0:
find=find and False
break
elif item["接待机构数量"]<0:
find=find and False
break
else:
return False
return find
pass
def callback(qName, input):
#print(input)
data = json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
uri = urllib.parse.urlparse(data["Uri"].strip())
hostname = uri.hostname
scheme = uri.scheme
links = data["Links"]
table = data["Tables"][2]
rows = table["body"]
for row in rows:
title = row[0].strip()
_type = row[1].strip()
date = row[2].strip()
dateTag = CONVERT.DateToInt(row[2].strip())
item = {
"Code": code,
"Name": name,
"Title": title,
"Type": _type,
"Date": date,
"DateTag": dateTag,
"Url": ""
}
if type([]) == type(links):
for link in links:
if link["Text"].strip() == title:
item["Url"] = "%s://%s/%s" % (scheme, hostname,
link["Href"])
dictName = "PreData:股票:%s:公告大全" % code
r.SortDictSave(dictName, item, dateTag)
pass
def 最新盈利能力指标(arr, count=3):
arr = list(filter(lambda k: "每股指标Tag" in k, arr)) #
currentDateTag = CONVERT.GetCurrentDateTag()
arr = sorted(arr, key=lambda k: k["每股指标Tag"], reverse=True)
size = len(arr)
count = count if count < size else size
subArr = arr[:count]
subArr = list(filter(lambda k: k["每股指标Tag"] >= 20190331, subArr)) #时间段要改进
if 0 == len(subArr):
return False
subArr加权净资产收益率 = list(map(lambda x: x["加权净资产收益率(%)"], subArr))
subArr加权净资产收益率中位数 = median(subArr加权净资产收益率)
find = True and 0 < len(subArr)
for item in subArr:
#print("%s %s %s"%(arr[0]["Code"],arr[0]["Name"],item["股东户数增减比例"]))
if 6 < item["加权净资产收益率(%)"] and 0.5 <= item["每股经营现金流(元)"] and item[
"资产负债率(%)"] <= 50:
find = find and True
elif item["加权净资产收益率(%)"] == 0:
find = find and False
break
elif item["加权净资产收益率(%)"] < 0:
find = find and False
break
else:
return False
return find
pass
def callback(qName, input):
#print(input)
data = json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
if 1 != len(tables):
return None
table = data["Tables"][0]
rows = table["body"]
for row in rows:
if 17 == len(row):
最新价 = CONVERT.StrToFloat(row[3])
跌涨幅 = CONVERT.PercentStrToFloat(row[4])
股东名称 = row[5]
持股变动信息增减 = row[6]
持股变动信息变动数量 = CONVERT.StrToFloat(row[7])
持股变动信息占总股本比例 = CONVERT.PercentStrToFloat(row[8])
持股变动信息占流通股比例 = CONVERT.PercentStrToFloat(row[9])
变动后持股情况持股总数 = CONVERT.StrToFloat(row[10])
变动后持股情况占总股本比例 = CONVERT.PercentStrToFloat(row[11])
变动后持股情况持流通股数 = CONVERT.StrToFloat(row[12])
变动后持股情况占流通股比例 = CONVERT.PercentStrToFloat(row[13])
变动开始日 = row[14]
变动截止日 = row[15]
公告日 = row[16]
公告日Tag = 0
item = {
"Code": code,
"Name": name,
"最新价": 最新价,
"跌涨幅": 跌涨幅,
"股东名称": 股东名称,
"持股变动信息增减": 持股变动信息增减,
"持股变动信息变动数量": 持股变动信息变动数量,
"持股变动信息占总股本比例": 持股变动信息占总股本比例,
"持股变动信息占流通股比例": 持股变动信息占流通股比例,
"变动后持股情况持股总数": 变动后持股情况持股总数,
"变动后持股情况占总股本比例": 变动后持股情况占总股本比例,
"变动后持股情况持流通股数": 变动后持股情况持流通股数,
"变动后持股情况占流通股比例": 变动后持股情况占流通股比例,
"变动开始日": 变动开始日,
"变动截止日": 变动截止日,
"公告日": 公告日,
"公告日Tag": 公告日Tag
}
dictName = "PreData:股票:%s:股东增减持" % code
r.SortDictSave(dictName, json.dumps(item, ensure_ascii=False),
公告日Tag)
print(item)
pass
def CreateTask北向成交明细(setName, item):
code = item[0]
url = "http://data.eastmoney.com/hsgt/%s.html" % code #北向持股明细列表
task = {"Code": code, "Url": url, "RetryCount": 3}
taskID = CONVERT.StrToInt(code[len(code) - 1]) % psutil.cpu_count(True)
qName = "Stock:Task:BXCJMX:Task%s" % taskID
r.QueueEn(qName, json.dumps(task, ensure_ascii=False))
pass
def CreateTask资金流向(setName, item):
code = item[0]
url = "http://data.eastmoney.com/zjlx/%s.html" % code
threadCount = psutil.cpu_count(True)
taskID = CONVERT.StrToInt(code[len(code) - 1]) % threadCount
task = {"Code": code, "Url": url, "RetryCount": 3}
qName = "Stock:Task:ZJLX:Task%d" % taskID
r.QueueEn(qName, json.dumps(task, ensure_ascii=False))
print("CreateTask资金流向 %s" % code)
pass
def CreateTask北向成交明细(dictName, key, val, pageIndex, pageCount, pageSize,
curIndex, total):
code = key
url = "http://data.eastmoney.com/hsgt/%s.html" % code #北向持股明细列表
task = {"Code": code, "Url": url, "RetryCount": 3}
taskID = CONVERT.StrToInt(code[len(code) - 1]) % 4
qName = "Stock:Task:BXCJMX:Task%s" % taskID
r.QueueEn(qName, json.dumps(task, ensure_ascii=False))
print(curIndex)
pass
def FillItemList(table):
arr = CreateEmptyItemList(table)
rows = table["body"]
for row in rows:
key = row[0]
rowIndex = 1
while rowIndex < len(arr):
if True != CHECKER.IsDate(row[rowIndex]):
arr[rowIndex - 1][key] = CONVERT.UnitStrToFloat(row[rowIndex],unit=None)
else:
if 8 == len(row[rowIndex]):
row[rowIndex] = "20" + row[rowIndex]
arr[rowIndex - 1][key + "Tag"] = CONVERT.DateToInt(row[rowIndex])
arr[rowIndex - 1][key] = row[rowIndex]
arr[rowIndex - 1]["DateTag"] = CONVERT.DateToInt(row[rowIndex])
rowIndex+=1
return arr
def CreateTask新财务分析(setName, item):
code = item[0]
code2 = "SH" + code if code[0] == "6" else "SZ" + code
url = "http://f10.eastmoney.com/f10_v2/FinanceAnalysis.aspx?code=%s" % code2 #核心题材
threadCount = psutil.cpu_count(True)
taskID = CONVERT.StrToInt(code[len(code) - 1]) % threadCount
task = {"Code": code, "Url": url, "RetryCount": 3}
qName = "Stock:Task:CWFX:Task%d" % taskID
r.QueueEn(qName, json.dumps(task, ensure_ascii=False))
print("CreateTask新财务分析 %s" % code)
pass
def ProcWebData():
taskId = "Task0"
#serverUrl="http://122.51.159.248:5000/YunStock2Service?keyName=GJDCGGK&taskId=%s" % taskId
serverUrl = "http://127.0.0.1:5000/YunStock2Service?keyName=GJDCGGK&taskId=%s" % taskId
count = 0
while True:
try:
res = requests.get(serverUrl)
print("接收 %s \r\n --------\r\n" % res.text)
res = json.loads(res.text)
if (res["success"] == True):
data = res["data"]
if 10 <= len(data):
data = json.loads(data)
url = data["Url"]
params = urllib.parse.parse_qs(
urllib.parse.urlparse(url).query)
resArr = []
jsonStr = spider.LoadWeb(url, "国家队持股概况").GetDataFromWeb()
jsonData = json.loads(jsonStr)
if "Tables" in jsonData:
tables = jsonData["Tables"]
if 0 < len(tables):
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 8 == len(row):
报告期Tag = CONVERT.DateToInt(row[0])
item = {
"报告期": row[0],
"报告期Tag": 报告期Tag,
"持股数量": CONVERT.StrToInt(row[2]),
"持股市值": CONVERT.UnitStrToFloat(row[3]),
"持股占市场总和比例":
CONVERT.UnitStrToFloat(row[4]),
"持有个股数量": CONVERT.StrToInt(row[5]),
"沪深300指数":
CONVERT.UnitStrToFloat(row[6]),
"涨跌幅": CONVERT.UnitStrToFloat(row[7])
}
resArr.append(item)
post_data = {
"taskId": taskId,
"keyName": "GJDCGGK",
"jsonReq": json.dumps(data, ensure_ascii=False),
"jsonRes": json.dumps(resArr, ensure_ascii=False)
}
res2 = requests.post(serverUrl, data=post_data)
print("POST %s \r\n RES %s \r\n --------- \r\n" %
(post_data, res2.text))
else:
time.sleep(3)
except BaseException as e:
print(" 异常 %s " % e)
time.sleep(20)
pass
def DownloadAllCode():
taskId = "Task0"
#serverUrl="http://122.51.159.248:5000/YunStock2Service?keyName=GJDCGGGMX&taskId=%s" % taskId
serverUrl = "http://127.0.0.1:5000/YunStock2Service?keyName=GJDCGGGMX&taskId=%s" % taskId
url = "http://data.eastmoney.com/gjdcg/mingxi.aspx?date=2020-03-31" #沪深A股
chrome.get(url)
linkBtn = chrome.find_element_by_link_text("下一页")
clsVal = linkBtn.get_attribute('class')
pageIndex = 0
while True != CHECKER.Contains(clsVal, "nolink"):
chrome.switch_to_window(chrome.window_handles[0])
linkBtn = chrome.find_element_by_link_text("下一页")
chrome.execute_script("arguments[0].scrollIntoView(false);", linkBtn)
clsVal = linkBtn.get_attribute('class')
jsonStr = HtmlConvertor.GetInst().LoadFromString(
html=chrome.page_source).ConvertToJson()
jsonData = json.loads(jsonStr)
if ("Tables" in jsonData and [] != jsonData["Tables"]):
tables = jsonData["Tables"]
rows = tables[len(tables) - 1]["body"]
postArr = []
for row in rows:
if (11 == len(row)):
item = {
"Code": row[1],
"股票简称": row[2],
"持股数量合计": CONVERT.UnitStrToFloat(row[4]),
"持股数量变动": CONVERT.UnitStrToFloat(row[5]),
"持股市值合计": CONVERT.UnitStrToFloat(row[6]),
"持股市值变动": CONVERT.UnitStrToFloat(row[7]),
"持股比例合计": CONVERT.StrToFloat(row[8]),
"持股比例变动": CONVERT.StrToFloat(row[9]),
"公告日期": row[10],
"公告日期Tag": CONVERT.DateToInt(row[10]),
}
postArr.append(item)
post_data = {
"keyName": "GJDCGGGMX",
"jsonReq": json.dumps({}, ensure_ascii=False),
"jsonRes": json.dumps(postArr, ensure_ascii=False)
}
print(post_data)
#res2 = requests.post(serverUrl,data=post_data)
#print("POST %s \r\n RES %s \r\n --------- \r\n"%(post_data,res2.text))
else:
print("%d 页数据未能加载" % pageIndex)
time.sleep(20)
pass
pageIndex += 1
linkBtn = chrome.find_element_by_link_text("下一页")
linkBtn.click()
time.sleep(random.uniform(3, 5))
chrome.quit()
def 北向持股明细任务处理(qName, qItem):
task = json.loads(qItem)
code = task["Code"]
url = task["Url"]
retryCount = task["RetryCount"]
try:
jsonStr = spider.LoadWeb(url, "北向持股明细").GetDataFromWeb()
jsonData = json.loads(jsonStr)
if "Tables" in jsonData:
tables = jsonData["Tables"]
if 0 < len(tables):
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 12 == len(row):
持股日期Tag = CONVERT.DateToInt(row[0])
item = {
"Code": code,
"持股日期": row[0],
"持股日期Tag": 持股日期Tag,
"股票代码": row[1],
"股票简称": row[2],
"当日收盘价": CONVERT.StrToFloat(row[3]),
"当日涨跌幅": CONVERT.StrToFloat(row[4]),
"机构名称": row[5],
"持股数量": CONVERT.UnitStrToFloat(row[6]),
"持股市值": CONVERT.UnitStrToFloat(row[7]),
"持股数量占A股百分比": CONVERT.StrToFloat(row[8]),
"持股市值变化1日": CONVERT.UnitStrToFloat(row[9]),
"持股市值变化5日": CONVERT.UnitStrToFloat(row[10]),
"持股市值变化10日": CONVERT.UnitStrToFloat(row[11])
}
qName北向持股 = "Stock:北向持股:%s" % code
r.SortDictSave(qName北向持股, item, 持股日期Tag)
print(item)
except BaseException as e:
retryCount = retryCount - 1
if 0 < retryCount:
task = {"Code": code, "Url": url, "RetryCount": retryCount}
r.QueueEn(qName, json.dumps(task, ensure_ascii=False))
print("%s %s" % (qItem, e))
time.sleep(60)
pass
def callback(qName, input):
#print(input)
data = json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
if 2 != len(tables):
return None
table = data["Tables"][1]
rows = table["body"]
for row in rows:
if 14 == len(row):
日期 = row[0]
日期Tag = CONVERT.DateToInt(row[0])
代码 = row[1]
名称 = row[2]
变动人 = row[3]
变动股数 = CONVERT.UnitStrToFloat(row[4])
成交均价 = CONVERT.StrToFloat(row[5])
变动金额 = CONVERT.StrToFloat(row[6])
变动原因 = row[7]
变动比例 = CONVERT.StrToFloat(row[8])
变动后持股数 = CONVERT.UnitStrToFloat(row[6])
持股种类 = row[10]
董监高人员姓名 = row[11]
职务 = row[12]
变动人与董监高的关系 = row[13]
item = {
"Code": code,
"Name": name,
"日期": 日期,
"日期Tag": 日期Tag,
"代码": 代码,
"名称": 名称,
"变动人": 变动人,
"变动股数": 变动股数,
"成交均价": 成交均价,
"变动金额": 变动金额,
"变动原因": 变动原因,
"变动比例": 变动比例,
"变动后持股数": 变动后持股数,
"持股种类": 持股种类,
"董监高人员姓名": 董监高人员姓名,
"职务": 职务,
"变动人与董监高的关系": 变动人与董监高的关系
}
dictName = "PreData:股票:%s:高管持股" % code
r.SortDictSave(dictName, json.dumps(item, ensure_ascii=False),
日期Tag)
print(item)
pass
def callback(qName, input):
#input = input.encode(encoding= "gbk").decode(encoding="utf-8")
#print(input)
data = json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
#if 9!=len(tables):
# return None
tables = data["Tables"]
for table in tables:
rows = table["body"]
rowIndex = 0
item = {}
head = rows[0]
rows = rows[1:]
for row in rows:
if 9 == len(row):
日期Tag = CONVERT.DateToInt(head[0])
item = {
"Code": code,
"Name": name,
"日期": head[0],
"日期Tag": 日期Tag,
"分类": row[0],
"主营构成": row[1],
"主营收入": CONVERT.UnitStrToFloat(row[2]),
"收入比例": CONVERT.PercentStrToFloat(row[3]),
"主营成本": CONVERT.UnitStrToFloat(row[4]),
"成本比例": CONVERT.PercentStrToFloat(row[5]),
"主营利润": CONVERT.UnitStrToFloat(row[6]),
"利润比例": CONVERT.PercentStrToFloat(row[7]),
"毛利率": CONVERT.PercentStrToFloat(row[8])
}
dictName = "PreData:股票:%s:经营分析" % code
r.SortDictSave(dictName, json.dumps(item, ensure_ascii=False),
日期Tag)
print(item)
pass
def ProcWebData():
taskId = "Task0"
#serverUrl="http://122.51.159.248:5000/YunStock2Service?keyName=ARTICLE&taskId=%s" % taskId
serverUrl = "http://127.0.0.1:5000/YunStock2Service?keyName=ARTICLE&taskId=%s" % taskId
count = 0
while True:
try:
res = requests.get(serverUrl)
print("接收 %s \r\n --------\r\n" % res.text)
res = json.loads(res.text)
if (res["success"] == True):
data = res["data"]
if 10 <= len(data):
data = json.loads(data)
url = data["Url"]
resArr = []
jsonStr = spider.LoadWeb(url, "文章收藏").GetDataFromWeb()
if jsonStr != None:
jsonData = json.loads(jsonStr)
title = jsonData["Title"]
content = jsonData["Content"]
publishTime = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
dateTag = CONVERT.DateToInt(publishTime)
item = {
"Title": title,
"Content": content,
"PublishTime": publishTime
}
#post_data = {"taskId":taskId,"keyName":"JGDYLB","jsonReq": json.dumps(data,ensure_ascii=False),"jsonRes": json.dumps(resArr,ensure_ascii=False)}
#res2 = requests.post(serverUrl,data=post_data)
#print("POST %s \r\n RES %s \r\n --------- \r\n" % (post_data,res2.text))
else:
time.sleep(3)
except BaseException as e:
print(" 异常 %s " % e)
time.sleep(20)
pass
def 涨跌幅P1大于一半(arr, count=20):
currentDateTag = CONVERT.GetCurrentDateTag()
arr = sorted(arr, key=lambda k: k["日期Tag"], reverse=True)
size = len(arr)
count = count if count < size else size
subArr = arr[:count]
subArr = list(filter(lambda k: k["日期Tag"] >= 20191215, subArr)) #时间段要改进
if 0 == len(subArr):
return False
subArr涨跌幅 = list(map(lambda x: x["涨跌幅"], subArr))
subArr涨幅绝对值大于P1小于P4 = list(
filter(lambda x: abs(x["涨跌幅"]) > 1.0 and abs(x["涨跌幅"]) <= 4.0, subArr))
subArr涨幅绝对值大于P1小于P4中位数 = median(
list(map(lambda x: x["涨跌幅"], subArr涨幅绝对值大于P1小于P4)))
if 0.7 <= (
len(subArr涨幅绝对值大于P1小于P4) / len(subArr涨跌幅)
) and 0 < subArr涨幅绝对值大于P1小于P4中位数 and subArr涨幅绝对值大于P1小于P4中位数 < 0.2:
return True
return False
pass
def callback(qName,input):
#print(input)
data=json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
if 3!=len(tables):
return None
table=data["Tables"][2]
rows=table["body"]
for row in rows:
if 8== len(row):
序号 =CONVERT.StrToInt (row[0])
机构名称 = row[1]
日期Tag = CONVERT.GetCurrentDateTag()
机构属性 = row[3]
持股总数 =CONVERT.StrToFloat(row[4])
持股市值 = CONVERT.StrToFloat(row[5])
占总股本比例 = CONVERT.StrToFloat( row[6])
占流通股本比例= CONVERT.StrToFloat(row[7])
item={
"Code":code
,"Name":name
,"序号":序号
,"日期Tag":日期Tag
,"机构属性":机构属性
,"持股总数":持股总数
,"持股市值":持股市值
,"占总股本比例":占总股本比例
,"占流通股本比例":占流通股本比例
}
dictName="PreData:股票:%s:主力持仓"%code
r.SortDictSave(dictName,json.dumps(item,ensure_ascii=False),日期Tag)
print(item)
pass
def callback(qName,input):
#input = input.encode(encoding= "gbk").decode(encoding="utf-8")
#print(input)
data=json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
if 2!=len(tables):
return None
table=data["Tables"][1]
rows=table["body"]
for row in rows:
if 16== len(row):
序号 = CONVERT.StrToInt( row[0])
日期 =row[1]
日期Tag =CONVERT.DateToInt (row[1])
收盘价 = CONVERT.StrToFloat(row[3])
涨跌幅 = CONVERT.PercentStrToFloat(row[4])
后1日涨跌幅 = CONVERT.PercentStrToFloat(row[5])
后2日涨跌幅 =CONVERT.PercentStrToFloat(row[6])
后3日涨跌幅 = CONVERT.PercentStrToFloat(row[7])
后5日涨跌幅 = CONVERT.PercentStrToFloat( row[8])
后10日涨跌幅= CONVERT.PercentStrToFloat(row[9])
后20日涨跌幅= CONVERT.PercentStrToFloat(row[10])
后30日涨跌幅= CONVERT.PercentStrToFloat(row[11])
上榜营业部买入合计= CONVERT.StrToFloat(row[12])
上榜营业部卖出合计= CONVERT.StrToFloat(row[13])
上榜营业部买卖净额= CONVERT.StrToFloat(row[14])
上榜原因= row[15]
item={
"Code":code
,"Name":name
,"序号":序号
,"日期":日期
,"日期Tag":日期Tag
,"收盘价":收盘价
,"涨跌幅":涨跌幅
,"后1日涨跌幅":后1日涨跌幅
,"后2日涨跌幅":后2日涨跌幅
,"后3日涨跌幅":后3日涨跌幅
,"后5日涨跌幅":后5日涨跌幅
,"后10日涨跌幅":后10日涨跌幅
,"后20日涨跌幅":后20日涨跌幅
,"后30日涨跌幅":后30日涨跌幅
,"上榜营业部买入合计":上榜营业部买入合计
,"上榜营业部卖出合计":上榜营业部卖出合计
,"上榜营业部买卖净额":上榜营业部买卖净额
,"上榜原因":上榜原因
}
dictName="PreData:股票:%s:龙虎榜单历次上榜"%code
r.SortDictSave(dictName,json.dumps(item,ensure_ascii=False),日期Tag)
print(item)
pass
def callback(qName, input):
#print(input)
data = json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
if 2 != len(tables):
return None
table = data["Tables"][1]
rows = table["body"]
for row in rows:
if 13 == len(row):
股东户数统计截止日 = row[0]
股东户数统计截止日Tag = CONVERT.DateToInt(股东户数统计截止日)
区间涨跌幅 = CONVERT.StrToFloat(row[1])
股东户数本次 = CONVERT.StrToFloat(row[2])
股东户数上次 = CONVERT.StrToFloat(row[3])
股东户数增减 = CONVERT.StrToFloat(row[4])
股东户数增减比例 = CONVERT.StrToFloat(row[5])
户均持股市值 = CONVERT.StrToFloat(row[6])
户均持股数量 = CONVERT.StrToFloat(row[7])
总市值 = CONVERT.StrToFloat(row[8])
总股本 = CONVERT.StrToFloat(row[9])
股本变动 = CONVERT.UnitStrToFloat(row[10])
股本变动原因 = row[11]
股东户数公告日期 = row[12]
股东户数公告日期Tag = CONVERT.DateToInt(股东户数公告日期)
item = {
"Code": code,
"Name": name,
"股东户数统计截止日": 股东户数统计截止日,
"股东户数统计截止日Tag": 股东户数统计截止日Tag,
"区间涨跌幅": 区间涨跌幅,
"股东户数本次": 股东户数本次,
"股东户数上次": 股东户数上次,
"股东户数增减": 股东户数增减,
"股东户数增减比例": 股东户数增减比例,
"户均持股市值": 户均持股市值,
"户均持股数量": 户均持股数量,
"总市值": 总市值,
"总股本": 总股本,
"股本变动": 股本变动,
"股本变动原因": 股本变动原因,
"股东户数公告日期": 股东户数公告日期,
"股东户数公告日期Tag ": 股东户数公告日期Tag
}
dictName = "PreData:股票:%s:股东户数" % code
r.SortDictSave(dictName, json.dumps(item, ensure_ascii=False),
股东户数统计截止日Tag)
print(item)
pass
