def SaveData北向成交明细ToRedis(code):
jsonStr = HtmlConvertor.GetInst().LoadFromString(
html=chrome.page_source).ConvertToJson()
jsonData = json.loads(jsonStr)
if ("Tables" in jsonData and [] != jsonData["Tables"]):
tables = jsonData["Tables"]
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 9 == len(row):
日期Tag = CONVERT.DateToInt(row[0])
item = {
"Code": code,
"日期": row[0],
"日期Tag": CONVERT.DateToInt(row[0]),
"当日排名": CONVERT.StrToInt(row[2]),
"收盘价": CONVERT.StrToFloat(row[3]),
"涨跌幅": CONVERT.UnitStrToFloat(row[4]),
"沪深股通净买额": CONVERT.UnitStrToFloat(row[5]),
"沪深股通买入金额": CONVERT.UnitStrToFloat(row[6]),
"沪深股通卖出金额": CONVERT.UnitStrToFloat(row[7]),
"沪深股通成交金额": CONVERT.UnitStrToFloat(row[8])
}
print(item)
qName北向成交 = "Stock:BXCJMX:%s" % code
r.SortDictSave(qName北向成交, item, 日期Tag)
def SaveData北向成交明细ToRedis(code):
jsonStr = HtmlConvertor.GetInst().LoadFromString(
html=chrome.page_source).ConvertToJson()
jsonData = json.loads(jsonStr)
if ("Tables" in jsonData and [] != jsonData["Tables"]):
tables = jsonData["Tables"]
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 9 == len(row):
日期Tag = CONVERT.DateToInt(row[0])
item = {
"Code": code,
"日期": row[0],
"日期Tag": CONVERT.DateToInt(row[0]),
"当日排名": CONVERT.StrToInt(row[2]),
"收盘价": CONVERT.StrToFloat(row[3]),
"涨跌幅": CONVERT.UnitStrToFloat(row[4]),
"沪深股通净买额": CONVERT.UnitStrToFloat(row[5]),
"沪深股通买入金额": CONVERT.UnitStrToFloat(row[6]),
"沪深股通卖出金额": CONVERT.UnitStrToFloat(row[7]),
"沪深股通成交金额": CONVERT.UnitStrToFloat(row[8])
}
print("北向成交明细 Task%s %s" % (taskID, item))
qName北向成交 = "Stock:BXCJMX:%s" % code
targetNameSpace = qName北向成交
r.SortDictSave(qName北向成交, item, 日期Tag)
SyncNotice.SendSyncNotice(
targetNameSpace, {
"namespace": targetNameSpace,
"code": code,
"score": 日期Tag,
"value": item,
"type": "SortedSet"
})
def AddToArray(jgCode, jgName, targetArr):
jsonStr = HtmlConvertor.GetInst().LoadFromString(
chrome.page_source).ConvertToJson()
jsonData = json.loads(jsonStr)
tables = jsonData["Tables"]
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 12 == len(row):
持股日期Tag = CONVERT.DateToInt(row[0])
item = {
"持股日期": row[0],
"持股日期Tag": 持股日期Tag,
"Code": row[1],
"股票简称": row[2],
"当日收盘价": CONVERT.StrToFloat(row[3]),
"当日涨跌幅": CONVERT.StrToFloat(row[4]),
"机构名称": jgName,
"jgCode": jgCode,
"持股数量": CONVERT.UnitStrToFloat(row[6]),
"持股市值": CONVERT.UnitStrToFloat(row[7]),
"持股数量占A股百分比": CONVERT.StrToFloat(row[8]),
"持股市值变化1日": CONVERT.UnitStrToFloat(row[9]),
"持股市值变化5日": CONVERT.UnitStrToFloat(row[10]),
"持股市值变化10日": CONVERT.UnitStrToFloat(row[11])
}
targetArr.append(item)
return targetArr
def DownloadAllCode():
taskId = "Task0"
#serverUrl="http://122.51.159.248:5000/YunStock2Service?keyName=GJDCGGGMX&taskId=%s" % taskId
serverUrl = "http://127.0.0.1:5000/YunStock2Service?keyName=GJDCGGGMX&taskId=%s" % taskId
url = "http://data.eastmoney.com/gjdcg/mingxi.aspx?date=2020-03-31" #沪深A股
chrome.get(url)
linkBtn = chrome.find_element_by_link_text("下一页")
clsVal = linkBtn.get_attribute('class')
pageIndex = 0
while True != CHECKER.Contains(clsVal, "nolink"):
chrome.switch_to_window(chrome.window_handles[0])
linkBtn = chrome.find_element_by_link_text("下一页")
chrome.execute_script("arguments[0].scrollIntoView(false);", linkBtn)
clsVal = linkBtn.get_attribute('class')
jsonStr = HtmlConvertor.GetInst().LoadFromString(
html=chrome.page_source).ConvertToJson()
jsonData = json.loads(jsonStr)
if ("Tables" in jsonData and [] != jsonData["Tables"]):
tables = jsonData["Tables"]
rows = tables[len(tables) - 1]["body"]
postArr = []
for row in rows:
if (11 == len(row)):
item = {
"Code": row[1],
"股票简称": row[2],
"持股数量合计": CONVERT.UnitStrToFloat(row[4]),
"持股数量变动": CONVERT.UnitStrToFloat(row[5]),
"持股市值合计": CONVERT.UnitStrToFloat(row[6]),
"持股市值变动": CONVERT.UnitStrToFloat(row[7]),
"持股比例合计": CONVERT.StrToFloat(row[8]),
"持股比例变动": CONVERT.StrToFloat(row[9]),
"公告日期": row[10],
"公告日期Tag": CONVERT.DateToInt(row[10]),
}
postArr.append(item)
post_data = {
"keyName": "GJDCGGGMX",
"jsonReq": json.dumps({}, ensure_ascii=False),
"jsonRes": json.dumps(postArr, ensure_ascii=False)
}
print(post_data)
#res2 = requests.post(serverUrl,data=post_data)
#print("POST %s \r\n RES %s \r\n --------- \r\n"%(post_data,res2.text))
else:
print("%d 页数据未能加载" % pageIndex)
time.sleep(20)
pass
pageIndex += 1
linkBtn = chrome.find_element_by_link_text("下一页")
linkBtn.click()
time.sleep(random.uniform(3, 5))
chrome.quit()
def ProcWebData():
taskId = "Task0"
#serverUrl="http://122.51.159.248:5000/YunStock2Service?keyName=GJDCGGK&taskId=%s" % taskId
serverUrl = "http://127.0.0.1:5000/YunStock2Service?keyName=GJDCGGK&taskId=%s" % taskId
count = 0
while True:
try:
res = requests.get(serverUrl)
print("接收 %s \r\n --------\r\n" % res.text)
res = json.loads(res.text)
if (res["success"] == True):
data = res["data"]
if 10 <= len(data):
data = json.loads(data)
url = data["Url"]
params = urllib.parse.parse_qs(
urllib.parse.urlparse(url).query)
resArr = []
jsonStr = spider.LoadWeb(url, "国家队持股概况").GetDataFromWeb()
jsonData = json.loads(jsonStr)
if "Tables" in jsonData:
tables = jsonData["Tables"]
if 0 < len(tables):
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 8 == len(row):
报告期Tag = CONVERT.DateToInt(row[0])
item = {
"报告期": row[0],
"报告期Tag": 报告期Tag,
"持股数量": CONVERT.StrToInt(row[2]),
"持股市值": CONVERT.UnitStrToFloat(row[3]),
"持股占市场总和比例":
CONVERT.UnitStrToFloat(row[4]),
"持有个股数量": CONVERT.StrToInt(row[5]),
"沪深300指数":
CONVERT.UnitStrToFloat(row[6]),
"涨跌幅": CONVERT.UnitStrToFloat(row[7])
}
resArr.append(item)
post_data = {
"taskId": taskId,
"keyName": "GJDCGGK",
"jsonReq": json.dumps(data, ensure_ascii=False),
"jsonRes": json.dumps(resArr, ensure_ascii=False)
}
res2 = requests.post(serverUrl, data=post_data)
print("POST %s \r\n RES %s \r\n --------- \r\n" %
(post_data, res2.text))
else:
time.sleep(3)
except BaseException as e:
print(" 异常 %s " % e)
time.sleep(20)
pass
def ProcWebData():
taskId = "Task0"
serverUrl = "http://122.51.159.248/YunStock2Service?keyName=BXCJMX&taskId=%s" % taskId
#serverUrl="http://127.0.0.1:80/YunStock2Service?keyName=BXCJMX&taskId=%s"%taskId
while True:
try:
res = requests.get(serverUrl)
print("接收 %s \r\n --------\r\n" % res.text)
res = json.loads(res.text)
if (res["success"] == True):
data = res["data"]
if 10 <= len(data):
data = json.loads(data)
code = data["Code"]
url = data["Url"]
resArr = []
jsonStr = spider.LoadWeb(url, "北向成交明细").GetDataFromWeb()
jsonData = json.loads(jsonStr)
if ("Tables" in jsonData and [] != jsonData["Tables"]):
tables = jsonData["Tables"]
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 9 == len(row):
日期Tag = CONVERT.DateToInt(row[0])
item = {
"Code": code,
"日期": row[0],
"日期Tag": CONVERT.DateToInt(row[0]),
"当日排名": CONVERT.StrToInt(row[2]),
"收盘价": CONVERT.StrToFloat(row[3]),
"涨跌幅": CONVERT.UnitStrToFloat(row[4]),
"沪深股通净买额": CONVERT.UnitStrToFloat(row[5]),
"沪深股通买入金额": CONVERT.UnitStrToFloat(row[6]),
"沪深股通卖出金额": CONVERT.UnitStrToFloat(row[7]),
"沪深股通成交金额": CONVERT.UnitStrToFloat(row[8])
}
resArr.append(item)
post_data = {
"keyName": "BXCJMX",
"taskId": taskId,
"method": "SaveBXCJMX",
"jsonReq": json.dumps(data, ensure_ascii=False),
"jsonRes": json.dumps(resArr, ensure_ascii=False)
}
res2 = requests.post(serverUrl, data=post_data)
print("POST %s \r\n RES %s \r\n --------- \r\n" %
(post_data, res2.text))
else:
time.sleep(10)
except BaseException as e:
print(" 异常 %s " % e)
time.sleep(20)
pass
def callback(qName, input):
这个有问题
#input = input.encode(encoding= "gbk").decode(encoding="utf-8")
#print(input)
test = u'\u7528\u4f8b\u540d\u79f0'.encode().decode('unicode-escape')
data = json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
if 4 != len(tables):
return None
table = data["Tables"][2]
rows = table["body"]
for row in rows:
if 14 == len(row):
序号 = CONVERT.StrToInt(row[0])
解禁时间 = row[1]
解禁时间Tag = CONVERT.DateToInt(row[1])
解禁股东数 = CONVERT.StrToInt(row[2].encode('utf-8').decode('gbk'))
解禁数量 = CONVERT.UnitStrToFloat(row[4].encode('utf-8').decode('gbk'))
实际解禁数量 = CONVERT.UnitStrToFloat(
row[5].encode('utf-8').decode('gbk'))
未解禁数量 = CONVERT.UnitStrToFloat(
row[6].encode('utf-8').decode('gbk'))
实际解禁市值 = CONVERT.UnitStrToFloat(
row[7].encode('utf-8').decode('gbk'))
占总市值比例 = CONVERT.StrToFloat(row[8])
占流通市值比例 = CONVERT.StrToFloat(row[9])
解禁前一日收盘价 = CONVERT.StrToFloat(row[10])
限售股类型 = row[11]
解禁前20日涨跌幅 = CONVERT.StrToFloat(row[12])
解禁后20日涨跌幅 = CONVERT.StrToFloat(row[13])
item = {
"Code": code,
"Name": name,
"序号": 序号,
"解禁时间": 解禁时间,
"解禁时间Tag": 解禁时间Tag,
"解禁股东数": 解禁股东数,
"解禁数量": 解禁数量,
"实际解禁数量": 实际解禁数量,
"未解禁数量": 未解禁数量,
"实际解禁市值": 实际解禁市值,
"占总市值比例": 占总市值比例,
"占流通市值比例": 占流通市值比例,
"解禁前一日收盘价": 解禁前一日收盘价,
"限售股类型": 限售股类型,
"解禁前20日涨跌幅": 解禁前20日涨跌幅,
"解禁后20日涨跌幅": 解禁后20日涨跌幅
}
dictName = "PreData:股票:%s:限售解禁" % code
#r.SortDictSave(dictName,json.dumps(item,ensure_ascii=False),日期Tag)
print(item)
pass
def callback(qName, input):
#print(input)
data = json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
if 2 != len(tables):
return None
table = data["Tables"][1]
rows = table["body"]
for row in rows:
if 14 == len(row):
日期 = row[0]
日期Tag = CONVERT.DateToInt(row[0])
代码 = row[1]
名称 = row[2]
变动人 = row[3]
变动股数 = CONVERT.UnitStrToFloat(row[4])
成交均价 = CONVERT.StrToFloat(row[5])
变动金额 = CONVERT.StrToFloat(row[6])
变动原因 = row[7]
变动比例 = CONVERT.StrToFloat(row[8])
变动后持股数 = CONVERT.UnitStrToFloat(row[6])
持股种类 = row[10]
董监高人员姓名 = row[11]
职务 = row[12]
变动人与董监高的关系 = row[13]
item = {
"Code": code,
"Name": name,
"日期": 日期,
"日期Tag": 日期Tag,
"代码": 代码,
"名称": 名称,
"变动人": 变动人,
"变动股数": 变动股数,
"成交均价": 成交均价,
"变动金额": 变动金额,
"变动原因": 变动原因,
"变动比例": 变动比例,
"变动后持股数": 变动后持股数,
"持股种类": 持股种类,
"董监高人员姓名": 董监高人员姓名,
"职务": 职务,
"变动人与董监高的关系": 变动人与董监高的关系
}
dictName = "PreData:股票:%s:高管持股" % code
r.SortDictSave(dictName, json.dumps(item, ensure_ascii=False),
日期Tag)
print(item)
pass
def callback(qName, input):
#input = input.encode(encoding= "gbk").decode(encoding="utf-8")
#print(input)
data = json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
if 3 != len(tables):
return None
table = data["Tables"][2]
rows = table["body"]
for row in rows:
if 13 == len(row):
日期 = row[0]
日期Tag = CONVERT.DateToInt(row[0])
收盘价 = CONVERT.StrToFloat(row[1])
涨跌幅 = CONVERT.PercentStrToFloat(row[2])
主力净流入净额 = CONVERT.UnitStrToFloat(row[3])
主力净流入净占比 = CONVERT.PercentStrToFloat(row[4])
超大单净流入净额 = CONVERT.UnitStrToFloat(row[5])
超大单净流入净占比 = CONVERT.PercentStrToFloat(row[6])
大单净流入净额 = CONVERT.UnitStrToFloat(row[7])
大单净流入净占比 = CONVERT.PercentStrToFloat(row[8])
中单净流入净额 = CONVERT.UnitStrToFloat(row[9])
中单净流入净占比 = CONVERT.PercentStrToFloat(row[10])
小单净流入净额 = CONVERT.UnitStrToFloat(row[11])
小单净流入净占比 = CONVERT.PercentStrToFloat(row[12])
item = {
"Code": code,
"Name": name,
"日期": 日期,
"日期Tag": 日期Tag,
"收盘价": 收盘价,
"涨跌幅": 涨跌幅,
"主力净流入净额": 主力净流入净额,
"主力净流入净占比": 主力净流入净占比,
"超大单净流入净额": 超大单净流入净额,
"超大单净流入净占比": 超大单净流入净占比,
"大单净流入净额": 大单净流入净额,
"大单净流入净占比": 大单净流入净占比,
"中单净流入净额": 中单净流入净额,
"中单净流入净占比": 中单净流入净占比,
"小单净流入净额": 小单净流入净额,
"小单净流入净占比": 小单净流入净占比
}
dictName = "PreData:股票:%s:资金流向" % code
r.SortDictSave(dictName, json.dumps(item, ensure_ascii=False),
日期Tag)
print(item)
pass
def FillItemList(table):
rows = table["body"]
rows = rows[1:]
if 11 != len(rows[0]):
return None
table["body"] = rows
arr = CreateEmptyItemList(table)
for row in rows:
key = row[0]
rowIndex = 1
while rowIndex < len(arr):
if key != "筹码集中度" and True != CHECKER.IsDate(row[rowIndex]):
arr[rowIndex - 1][key] = CONVERT.UnitStrToFloat(row[rowIndex])
elif "筹码集中度" == key:
arr[rowIndex - 1][key] = row[rowIndex]
else:
if 11 == len(row[rowIndex]):
"20" + row[rowIndex]
arr[rowIndex - 1][key + "Tag"] = CONVERT.DateToInt(
row[rowIndex])
arr[rowIndex - 1][key] = row[rowIndex]
arr[rowIndex - 1]["DateTag"] = CONVERT.DateToInt(row[rowIndex])
rowIndex += 1
return arr
def 北向持股明细任务处理(qName, qItem):
task = json.loads(qItem)
code = task["Code"]
url = task["Url"]
retryCount = task["RetryCount"]
try:
jsonStr = spider.LoadWeb(url, "北向持股明细").GetDataFromWeb()
jsonData = json.loads(jsonStr)
if "Tables" in jsonData:
tables = jsonData["Tables"]
if 0 < len(tables):
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 12 == len(row):
持股日期Tag = CONVERT.DateToInt(row[0])
item = {
"Code": code,
"持股日期": row[0],
"持股日期Tag": 持股日期Tag,
"股票代码": row[1],
"股票简称": row[2],
"当日收盘价": CONVERT.StrToFloat(row[3]),
"当日涨跌幅": CONVERT.StrToFloat(row[4]),
"机构名称": row[5],
"持股数量": CONVERT.UnitStrToFloat(row[6]),
"持股市值": CONVERT.UnitStrToFloat(row[7]),
"持股数量占A股百分比": CONVERT.StrToFloat(row[8]),
"持股市值变化1日": CONVERT.UnitStrToFloat(row[9]),
"持股市值变化5日": CONVERT.UnitStrToFloat(row[10]),
"持股市值变化10日": CONVERT.UnitStrToFloat(row[11])
}
qName北向持股 = "Stock:北向持股:%s" % code
r.SortDictSave(qName北向持股, item, 持股日期Tag)
print(item)
except BaseException as e:
retryCount = retryCount - 1
if 0 < retryCount:
task = {"Code": code, "Url": url, "RetryCount": retryCount}
r.QueueEn(qName, json.dumps(task, ensure_ascii=False))
print("%s %s" % (qItem, e))
time.sleep(60)
pass
def callback(qName, input):
#input = input.encode(encoding= "gbk").decode(encoding="utf-8")
#print(input)
data = json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
#if 9!=len(tables):
# return None
tables = data["Tables"]
for table in tables:
rows = table["body"]
rowIndex = 0
item = {}
head = rows[0]
rows = rows[1:]
for row in rows:
if 9 == len(row):
日期Tag = CONVERT.DateToInt(head[0])
item = {
"Code": code,
"Name": name,
"日期": head[0],
"日期Tag": 日期Tag,
"分类": row[0],
"主营构成": row[1],
"主营收入": CONVERT.UnitStrToFloat(row[2]),
"收入比例": CONVERT.PercentStrToFloat(row[3]),
"主营成本": CONVERT.UnitStrToFloat(row[4]),
"成本比例": CONVERT.PercentStrToFloat(row[5]),
"主营利润": CONVERT.UnitStrToFloat(row[6]),
"利润比例": CONVERT.PercentStrToFloat(row[7]),
"毛利率": CONVERT.PercentStrToFloat(row[8])
}
dictName = "PreData:股票:%s:经营分析" % code
r.SortDictSave(dictName, json.dumps(item, ensure_ascii=False),
日期Tag)
print(item)
pass
def callback(qName, input):
#print(input)
data = json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
if 2 != len(tables):
return None
table = data["Tables"][1]
rows = table["body"]
for row in rows:
if 13 == len(row):
股东户数统计截止日 = row[0]
股东户数统计截止日Tag = CONVERT.DateToInt(股东户数统计截止日)
区间涨跌幅 = CONVERT.StrToFloat(row[1])
股东户数本次 = CONVERT.StrToFloat(row[2])
股东户数上次 = CONVERT.StrToFloat(row[3])
股东户数增减 = CONVERT.StrToFloat(row[4])
股东户数增减比例 = CONVERT.StrToFloat(row[5])
户均持股市值 = CONVERT.StrToFloat(row[6])
户均持股数量 = CONVERT.StrToFloat(row[7])
总市值 = CONVERT.StrToFloat(row[8])
总股本 = CONVERT.StrToFloat(row[9])
股本变动 = CONVERT.UnitStrToFloat(row[10])
股本变动原因 = row[11]
股东户数公告日期 = row[12]
股东户数公告日期Tag = CONVERT.DateToInt(股东户数公告日期)
item = {
"Code": code,
"Name": name,
"股东户数统计截止日": 股东户数统计截止日,
"股东户数统计截止日Tag": 股东户数统计截止日Tag,
"区间涨跌幅": 区间涨跌幅,
"股东户数本次": 股东户数本次,
"股东户数上次": 股东户数上次,
"股东户数增减": 股东户数增减,
"股东户数增减比例": 股东户数增减比例,
"户均持股市值": 户均持股市值,
"户均持股数量": 户均持股数量,
"总市值": 总市值,
"总股本": 总股本,
"股本变动": 股本变动,
"股本变动原因": 股本变动原因,
"股东户数公告日期": 股东户数公告日期,
"股东户数公告日期Tag ": 股东户数公告日期Tag
}
dictName = "PreData:股票:%s:股东户数" % code
r.SortDictSave(dictName, json.dumps(item, ensure_ascii=False),
股东户数统计截止日Tag)
print(item)
pass
def callback(qName,input):
#input = input.encode(encoding= "gbk").decode(encoding="utf-8")
#print(input)
data=json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
if 3!=len(tables):
return None
table=data["Tables"][2]
rows=table["body"]
for row in rows:
if 13== len(row):
序号 =CONVERT.StrToInt (row[0])
交易标的 =row[1]
买方 = row[2]
卖方 = row[3]
交易金额 = CONVERT.UnitStrToFloat(row[4])
币种 =row[5]
股权转让比例 = CONVERT.StrToFloat(row[6])
标的类型 = row[7]
并购方式= row[8]
最新进展= row[9]
披露日期= row[10]
披露日期Tag= CONVERT.DateToInt(row[10])
最新公告日= row[11]
最新公告日Tag= CONVERT.DateToInt(row[11])
item={
"Code":code
,"Name":name
,"序号":序号
,"交易标的":交易标的
,"买方":买方
,"卖方":卖方
,"交易金额":交易金额
,"币种":币种
,"股权转让比例":股权转让比例
,"标的类型":标的类型
,"并购方式":并购方式
,"最新进展":最新进展
,"披露日期":披露日期
,"披露日期Tag":披露日期Tag
,"最新公告日":最新公告日
,"最新公告日Tag":最新公告日Tag
}
dictName="PreData:股票:%s:并购重组"%code
r.SortDictSave(dictName,json.dumps(item,ensure_ascii=False),最新公告日Tag)
print(item)
pass
def FillItemList(table):
arr = CreateEmptyItemList(table)
rows = table["body"]
for row in rows:
key = row[0]
rowIndex = 1
while rowIndex < len(arr):
if True != CHECKER.IsDate(row[rowIndex]):
arr[rowIndex - 1][key] = CONVERT.UnitStrToFloat(row[rowIndex],unit=None)
else:
if 8 == len(row[rowIndex]):
row[rowIndex] = "20" + row[rowIndex]
arr[rowIndex - 1][key + "Tag"] = CONVERT.DateToInt(row[rowIndex])
arr[rowIndex - 1][key] = row[rowIndex]
arr[rowIndex - 1]["DateTag"] = CONVERT.DateToInt(row[rowIndex])
rowIndex+=1
return arr
def ProcWebData():
taskId = "Task0"
serverUrl = "http://122.51.159.248/YunStock2Service?keyName=RZRQ&taskId=%s" % taskId
#serverUrl="http://127.0.0.1:80/YunStock2Service?keyName=RZRQ&taskId=%s"%taskId
while True:
try:
res = requests.get(serverUrl)
print("接收 %s \r\n --------\r\n" % res.text)
res = json.loads(res.text)
if (res["success"] == True):
data = res["data"]
if 10 <= len(data):
data = json.loads(data)
code = data["Code"]
url = data["Url"]
resArr = []
jsonStr = spider.LoadWeb(url, "融资融券").GetDataFromWeb()
jsonData = json.loads(jsonStr)
if ("Tables" in jsonData and [] != jsonData["Tables"]):
tables = jsonData["Tables"]
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 15 == len(row):
交易日期Tag = CONVERT.DateToInt(row[0])
item = {
"Code": code,
"交易日期": row[0],
"交易日期Tag": 交易日期Tag,
"收盘价": CONVERT.StrToFloat(row[1]),
"涨跌幅": CONVERT.UnitStrToFloat(row[2]),
"融资余额": CONVERT.UnitStrToFloat(row[3]),
"融资余额占流通市值比":
CONVERT.UnitStrToFloat(row[4]),
"融资买入额": CONVERT.UnitStrToFloat(row[5]),
"融资偿还额": CONVERT.UnitStrToFloat(row[6]),
"融资净买入": CONVERT.UnitStrToFloat(row[7]),
"融券余额": CONVERT.UnitStrToFloat(row[8]),
"融券余量": CONVERT.UnitStrToFloat(row[9]),
"融券卖出量": CONVERT.UnitStrToFloat(row[10]),
"融券偿还量": CONVERT.UnitStrToFloat(row[11]),
"融券净卖出": CONVERT.UnitStrToFloat(row[12]),
"融资融券余额": CONVERT.UnitStrToFloat(row[13]),
"融资融券余额差值": CONVERT.UnitStrToFloat(row[14])
}
resArr.append(item)
post_data = {
"keyName": "RZRQ",
"taskId": taskId,
"method": "SaveProcData",
"jsonReq": json.dumps(data, ensure_ascii=False),
"jsonRes": json.dumps(resArr, ensure_ascii=False)
}
res2 = requests.post(serverUrl, data=post_data)
print("POST %s \r\n RES %s \r\n --------- \r\n" %
(post_data, res2.text))
else:
time.sleep(5)
except BaseException as e:
print(" 异常 %s " % e)
time.sleep(60)
pass
def 北向持股明细任务处理(qName, qItem):
task = json.loads(qItem)
code = task["Code"]
url = task["Url"]
retryCount = task["RetryCount"]
taskID = qName.split(":")[3]
try:
jsonStr = spider.LoadWeb(url, "北向持股明细").GetDataFromWeb()
jsonData = json.loads(jsonStr)
if "Tables" in jsonData:
tables = jsonData["Tables"]
if 0 < len(tables):
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 12 == len(row):
持股日期Tag = CONVERT.DateToInt(row[0])
item = {
"Code": code,
"持股日期": row[0],
"持股日期Tag": 持股日期Tag,
"股票代码": row[1],
"股票简称": row[2],
"当日收盘价": CONVERT.StrToFloat(row[3]),
"当日涨跌幅": CONVERT.StrToFloat(row[4]),
"机构名称": row[5],
"持股数量": CONVERT.UnitStrToFloat(row[6]),
"持股市值": CONVERT.UnitStrToFloat(row[7]),
"持股数量占A股百分比": CONVERT.StrToFloat(row[8]),
"持股市值变化1日": CONVERT.UnitStrToFloat(row[9]),
"持股市值变化5日": CONVERT.UnitStrToFloat(row[10]),
"持股市值变化10日": CONVERT.UnitStrToFloat(row[11])
}
qName北向持股 = "Stock:BXCGMX:%s" % code
targetNameSpace = qName北向持股
r.SortDictSave(qName北向持股, item, 持股日期Tag)
SyncNotice.SendSyncNotice(
targetNameSpace, {
"namespace": targetNameSpace,
"code": code,
"score": 持股日期Tag,
"value": item,
"type": "SortedSet"
})
print("%s %s" % (taskID, item))
r.DictSave(
"Stock:Task:BXCGMX:Status", "%s" % taskID, {
"StartTime":
startTime.strftime('%Y-%m-%d %H:%M:%S'),
"UpdateTime":
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
"Running":
(datetime.datetime.now() - startTime).seconds
})
except BaseException as e:
retryCount = retryCount - 1
if 0 < retryCount:
task = {"Code": code, "Url": url, "RetryCount": retryCount}
r.QueueEn(qName, json.dumps(task, ensure_ascii=False))
print("%s %s" % (qItem, e))
time.sleep(120)
pass
def ProcWebData():
taskId = "Task1"
serverUrl = "http://122.51.159.248/YunStockService?taskId=%s" % taskId
#serverUrl="http://127.0.0.1:5000/YunStockService?taskId=%s"%taskId
while True:
try:
res = requests.get(serverUrl)
print("接收 %s \r\n --------\r\n" % res.text)
res = json.loads(res.text)
if (res["success"] == True):
data = res["data"]
if 10 <= len(data):
data = json.loads(data)
code = data["Code"]
url = data["Url"]
resArr = []
jsonStr = spider.LoadWeb(url, "北向持股明细").GetDataFromWeb()
jsonData = json.loads(jsonStr)
if "Tables" in jsonData:
tables = jsonData["Tables"]
if 0 < len(tables):
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 12 == len(row):
持股日期Tag = CONVERT.DateToInt(row[0])
item = {
"Code":
code,
"持股日期":
row[0],
"持股日期Tag":
持股日期Tag,
"股票代码":
row[1],
"股票简称":
row[2],
"当日收盘价":
CONVERT.StrToFloat(row[3]),
"当日涨跌幅":
CONVERT.StrToFloat(row[4]),
"机构名称":
row[5],
"持股数量":
CONVERT.UnitStrToFloat(row[6]),
"持股市值":
CONVERT.UnitStrToFloat(row[7]),
"持股数量占A股百分比":
CONVERT.StrToFloat(row[8]),
"持股市值变化1日":
CONVERT.UnitStrToFloat(row[9]),
"持股市值变化5日":
CONVERT.UnitStrToFloat(row[10]),
"持股市值变化10日":
CONVERT.UnitStrToFloat(row[11])
}
resArr.append(item)
post_data = {
"taskId": taskId,
"method": "SaveBXCGMX",
"jsonReq": json.dumps(data, ensure_ascii=False),
"jsonRes": json.dumps(resArr, ensure_ascii=False)
}
res2 = requests.post(serverUrl, data=post_data)
print("POST %s \r\n RES %s \r\n --------- \r\n" %
(post_data, res2.text))
else:
time.sleep(3)
except BaseException as e:
print(" 异常 %s " % e)
time.sleep(20)
pass
def ProcWebData():
taskId = "Task0"
serverUrl = "http://122.51.159.248:5000/YunStock2Service?keyName=DPZJLX&taskId=%s" % taskId
#serverUrl = "http://127.0.0.1:5000/YunStock2Service?keyName=DPZJLX&taskId=%s" % taskId
count = 0
while True:
try:
res = requests.get(serverUrl)
print("接收 %s \r\n --------\r\n" % res.text)
res = json.loads(res.text)
if (res["success"] == True):
data = res["data"]
if 10 <= len(data):
data = json.loads(data)
url = data["Url"]
params = urllib.parse.parse_qs(
urllib.parse.urlparse(url).query)
resArr = []
jsonStr = spider.LoadWeb(url, "大盘资金流向").GetDataFromWeb()
jsonData = json.loads(jsonStr)
if "Tables" in jsonData:
tables = jsonData["Tables"]
if 0 < len(tables):
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 15 == len(row):
日期Tag = CONVERT.DateToInt(row[0])
item = {
"日期": row[0],
"日期Tag": 日期Tag,
"上证收盘价": CONVERT.StrToFloat(row[1]),
"上证涨跌幅":
CONVERT.UnitStrToFloat(row[2]),
"深证收盘价": CONVERT.StrToFloat(row[3]),
"深证涨跌幅":
CONVERT.UnitStrToFloat(row[4]),
"主力净流入净额":
CONVERT.UnitStrToFloat(row[5]),
"主力净流入净占比":
CONVERT.UnitStrToFloat(row[6]),
"超大单净流入净额":
CONVERT.UnitStrToFloat(row[7]),
"超大单净流入净占比":
CONVERT.UnitStrToFloat(row[8]),
"大单净流入净额":
CONVERT.UnitStrToFloat(row[9]),
"大单净流入净占比":
CONVERT.UnitStrToFloat(row[10]),
"中单净流入净额":
CONVERT.UnitStrToFloat(row[11]),
"中单净流入净占比":
CONVERT.UnitStrToFloat(row[12]),
"小单净流入净额": CONVERT.UnitStrToFloat(
row[13]),
"小单净流入净占比": CONVERT.UnitStrToFloat(
row[14])
}
resArr.append(item)
post_data = {
"taskId": taskId,
"keyName": "DPZJLX",
"jsonReq": json.dumps(data, ensure_ascii=False),
"jsonRes": json.dumps(resArr, ensure_ascii=False)
}
res2 = requests.post(serverUrl, data=post_data)
print("POST %s \r\n RES %s \r\n --------- \r\n" %
(post_data, res2.text))
else:
time.sleep(3)
except BaseException as e:
print(" 异常 %s " % e)
time.sleep(20)
pass
def callback(qName,input):
#print(input)
data=json.loads(input)
code = data["Code"].strip()
name = data["Name"].strip()
tables = data["Tables"]
if 3!=len(tables):
return None
table=data["Tables"][2]
rows=table["body"]
for row in rows:
if 15 == len(row):
交易日期 = row[0]
交易日期Tag = CONVERT.DateToInt(交易日期)
收盘价 = CONVERT.StrToFloat(row[1])
涨跌幅 = CONVERT.StrToFloat( row[2])
融资余额 = CONVERT.UnitStrToFloat( row[3])
融资余额占流通市值比 = CONVERT.PercentStrToFloat( row[4])
融资买入额 = CONVERT.UnitStrToFloat(row[5])
融资偿还额 = CONVERT.UnitStrToFloat(row[6])
融资净买入 = CONVERT.UnitStrToFloat(row[7] )
融券余额 = CONVERT.UnitStrToFloat(row[8] )
融券余量 = CONVERT.UnitStrToFloat(row[9] )
融券卖出量 = CONVERT.UnitStrToFloat(row[10] )
融券偿还量 =CONVERT.UnitStrToFloat( row[11] )
融券净卖出 =CONVERT.UnitStrToFloat( row[12] )
融资融券余额 =CONVERT.UnitStrToFloat( row[13] )
融资融券余额差值 =CONVERT.UnitStrToFloat( row[14] )
item={
"Code":code
,"Name":name
,"交易日期":交易日期
,"交易日期Tag":交易日期Tag
,"收盘价":收盘价
,"涨跌幅":涨跌幅
,"融资余额":融资余额
,"融资余额占流通市值比":融资余额占流通市值比
,"融资买入额":融资买入额
,"融资偿还额":融资偿还额
,"融资净买入":融资净买入
,"融券余额":融券余额
,"融券余量":融券余量
,"融券卖出量":融券卖出量
,"融券偿还量":融券偿还量
,"融券净卖出":融券净卖出
,"融资融券余额":融资融券余额
,"融资融券余额差值":融资融券余额差值
}
dictName="PreData:股票:%s:融资融券"%code
r.SortDictSave(dictName,json.dumps(item,ensure_ascii=False),交易日期Tag)
print(item)
pass
def SaveData融资融券ToRedis(code):
jsonStr = HtmlConvertor.GetInst().LoadFromString(
html=chrome.page_source).ConvertToJson()
jsonData = json.loads(jsonStr)
if ("Tables" in jsonData and [] != jsonData["Tables"]):
tables = jsonData["Tables"]
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 15 == len(row):
交易日期Tag = CONVERT.DateToInt(row[0])
item = {
"Code": code,
"交易日期": row[0],
"交易日期Tag": 交易日期Tag,
"收盘价": CONVERT.StrToFloat(row[1]),
"涨跌幅": CONVERT.UnitStrToFloat(row[2]),
"融资余额": CONVERT.UnitStrToFloat(row[3]),
"融资余额占流通市值比": CONVERT.UnitStrToFloat(row[4]),
"融资买入额": CONVERT.UnitStrToFloat(row[5]),
"融资偿还额": CONVERT.UnitStrToFloat(row[6]),
"融资净买入": CONVERT.UnitStrToFloat(row[7]),
"融券余额": CONVERT.UnitStrToFloat(row[8]),
"融券余量": CONVERT.UnitStrToFloat(row[9]),
"融券卖出量": CONVERT.UnitStrToFloat(row[10]),
"融券偿还量": CONVERT.UnitStrToFloat(row[11]),
"融券净卖出": CONVERT.UnitStrToFloat(row[12]),
"融资融券余额": CONVERT.UnitStrToFloat(row[13]),
"融资融券余额差值": CONVERT.UnitStrToFloat(row[14])
}
print(item)
qName融资融券 = "Stock:RZRQ:%s" % code
r.SortDictSave(qName融资融券, item, 交易日期Tag)
def SaveData资金流向ToRedis(code):
jsonStr = HtmlConvertor.GetInst().LoadFromString(
html=chrome.page_source).ConvertToJson()
jsonData = json.loads(jsonStr)
if ("Tables" in jsonData and [] != jsonData["Tables"]):
tables = jsonData["Tables"]
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 13 == len(row):
日期Tag = CONVERT.DateToInt(row[0])
item = {
"Code": code,
"日期": row[0],
"日期Tag": 日期Tag,
"收盘价": CONVERT.StrToFloat(row[1]),
"涨跌幅": CONVERT.UnitStrToFloat(row[2]),
"主力净流入净额": CONVERT.UnitStrToFloat(row[3]),
"主力净流入净占比": CONVERT.UnitStrToFloat(row[4]),
"超大单净流入净额": CONVERT.UnitStrToFloat(row[5]),
"超大单净流入净占比": CONVERT.UnitStrToFloat(row[6]),
"大单净流入净额": CONVERT.UnitStrToFloat(row[7]),
"大单净流入净占比": CONVERT.UnitStrToFloat(row[8]),
"中单净流入净额": CONVERT.UnitStrToFloat(row[9]),
"中单净流入净占比": CONVERT.UnitStrToFloat(row[10]),
"小单净流入净额": CONVERT.UnitStrToFloat(row[11]),
"小单净流入净占比": CONVERT.UnitStrToFloat(row[12])
}
print(item)
qName资金流向 = "Stock:ZJLX:%s" % code
targetNameSpace = qName资金流向
r.SortDictSave(qName资金流向, item, 日期Tag)
SyncNotice.SendSyncNotice(
targetNameSpace, {
"namespace": targetNameSpace,
"code": code,
"score": 日期Tag,
"value": item,
"type": "SortedSet"
})
def ProcWebData():
taskId = "Task0"
serverUrl = "http://122.51.159.248:5000/YunStock2Service?keyName=BXCGTJ&taskId=%s" % taskId
#serverUrl = "http://127.0.0.1:5000/YunStock2Service?keyName=BXCGTJ&taskId=%s" % taskId
count = 0
while True:
try:
res = requests.get(serverUrl)
print("接收 %s \r\n --------\r\n" % res.text)
res = json.loads(res.text)
if (res["success"] == True):
data = res["data"]
if 10 <= len(data):
data = json.loads(data)
url = data["Url"]
params = urllib.parse.parse_qs(
urllib.parse.urlparse(url).query)
jgCode = params["jgCode"][0]
resArr = []
jsonStr = spider.LoadWeb(url, "北向持股统计").GetDataFromWeb()
jsonData = json.loads(jsonStr)
if "Tables" in jsonData:
tables = jsonData["Tables"]
if 0 < len(tables):
rows = tables[len(tables) - 1]["body"]
for row in rows:
if 8 == len(row):
持股日期Tag = CONVERT.DateToInt(row[0])
item = {
"jgCode": jgCode,
"持股日期": row[0],
"持股日期Tag": 持股日期Tag,
"机构名称": row[1],
"持股只数": CONVERT.StrToInt(row[3]),
"持股市值": CONVERT.UnitStrToFloat(row[4]),
"持股市值变化1日":
CONVERT.UnitStrToFloat(row[5]),
"持股市值变化5日":
CONVERT.UnitStrToFloat(row[6]),
"持股市值变化10日":
CONVERT.UnitStrToFloat(row[7])
}
resArr.append(item)
post_data = {
"taskId": taskId,
"keyName": "BXCGTJ",
"jsonReq": json.dumps(data, ensure_ascii=False),
"jsonRes": json.dumps(resArr, ensure_ascii=False)
}
res2 = requests.post(serverUrl, data=post_data)
print("POST %s \r\n RES %s \r\n --------- \r\n" %
(post_data, res2.text))
if "Links" in jsonData:
links = jsonData["Links"]
links = list(
filter(
lambda x: CHECKER.StartWith(
x["Href"],
"/hsgtcg/InstitutionHdDetail.aspx\?jgCode="
) and x["Text"] == "详细", links))
links = list(
map(
lambda x: "http://data.eastmoney.com%s" % x[
"Href"], links))
items = []
for link in links:
params = urllib.parse.parse_qs(
urllib.parse.urlparse(link).query)
date = params["date"][0]
dateTag = CONVERT.DateToInt(date)
jgCode = params["jgCode"][0]
jgName = urllib.parse.unquote(
params["jgName"][0].replace(
'%u', '\\u').encode('utf-8').decode(
'unicode-escape')).strip()
item = {
"DateTag": dateTag,
"Date": date,
"jgCode": jgCode,
"jgName": jgName,
"Url": link,
"RetryCount": 3
}
items.append(item)
post_data2 = {
"keyName": "BXCGMXURLTASK",
"jsonReq": json.dumps(data, ensure_ascii=False),
"jsonRes": json.dumps(items, ensure_ascii=False)
}
res3 = requests.post(serverUrl, data=post_data2)
print("POST %s \r\n RES %s \r\n --------- \r\n" %
(post_data2, res3.text))
count += 1
print("已处理 %s" % count)
else:
time.sleep(3)
except BaseException as e:
print(" 异常 %s " % e)
time.sleep(20)
pass