def ProcWebData(): taskId = "Task0" #serverUrl="http://122.51.159.248:5000/YunStock2Service?keyName=GJDCGGK&taskId=%s" % taskId serverUrl = "http://127.0.0.1:5000/YunStock2Service?keyName=GJDCGGK&taskId=%s" % taskId count = 0 while True: try: res = requests.get(serverUrl) print("接收 %s \r\n --------\r\n" % res.text) res = json.loads(res.text) if (res["success"] == True): data = res["data"] if 10 <= len(data): data = json.loads(data) url = data["Url"] params = urllib.parse.parse_qs( urllib.parse.urlparse(url).query) resArr = [] jsonStr = spider.LoadWeb(url, "国家队持股概况").GetDataFromWeb() jsonData = json.loads(jsonStr) if "Tables" in jsonData: tables = jsonData["Tables"] if 0 < len(tables): rows = tables[len(tables) - 1]["body"] for row in rows: if 8 == len(row): 报告期Tag = CONVERT.DateToInt(row[0]) item = { "报告期": row[0], "报告期Tag": 报告期Tag, "持股数量": CONVERT.StrToInt(row[2]), "持股市值": CONVERT.UnitStrToFloat(row[3]), "持股占市场总和比例": CONVERT.UnitStrToFloat(row[4]), "持有个股数量": CONVERT.StrToInt(row[5]), "沪深300指数": CONVERT.UnitStrToFloat(row[6]), "涨跌幅": CONVERT.UnitStrToFloat(row[7]) } resArr.append(item) post_data = { "taskId": taskId, "keyName": "GJDCGGK", "jsonReq": json.dumps(data, ensure_ascii=False), "jsonRes": json.dumps(resArr, ensure_ascii=False) } res2 = requests.post(serverUrl, data=post_data) print("POST %s \r\n RES %s \r\n --------- \r\n" % (post_data, res2.text)) else: time.sleep(3) except BaseException as e: print(" 异常 %s " % e) time.sleep(20) pass
def callback(qName, input): 这个有问题 #input = input.encode(encoding= "gbk").decode(encoding="utf-8") #print(input) test = u'\u7528\u4f8b\u540d\u79f0'.encode().decode('unicode-escape') data = json.loads(input) code = data["Code"].strip() name = data["Name"].strip() tables = data["Tables"] if 4 != len(tables): return None table = data["Tables"][2] rows = table["body"] for row in rows: if 14 == len(row): 序号 = CONVERT.StrToInt(row[0]) 解禁时间 = row[1] 解禁时间Tag = CONVERT.DateToInt(row[1]) 解禁股东数 = CONVERT.StrToInt(row[2].encode('utf-8').decode('gbk')) 解禁数量 = CONVERT.UnitStrToFloat(row[4].encode('utf-8').decode('gbk')) 实际解禁数量 = CONVERT.UnitStrToFloat( row[5].encode('utf-8').decode('gbk')) 未解禁数量 = CONVERT.UnitStrToFloat( row[6].encode('utf-8').decode('gbk')) 实际解禁市值 = CONVERT.UnitStrToFloat( row[7].encode('utf-8').decode('gbk')) 占总市值比例 = CONVERT.StrToFloat(row[8]) 占流通市值比例 = CONVERT.StrToFloat(row[9]) 解禁前一日收盘价 = CONVERT.StrToFloat(row[10]) 限售股类型 = row[11] 解禁前20日涨跌幅 = CONVERT.StrToFloat(row[12]) 解禁后20日涨跌幅 = CONVERT.StrToFloat(row[13]) item = { "Code": code, "Name": name, "序号": 序号, "解禁时间": 解禁时间, "解禁时间Tag": 解禁时间Tag, "解禁股东数": 解禁股东数, "解禁数量": 解禁数量, "实际解禁数量": 实际解禁数量, "未解禁数量": 未解禁数量, "实际解禁市值": 实际解禁市值, "占总市值比例": 占总市值比例, "占流通市值比例": 占流通市值比例, "解禁前一日收盘价": 解禁前一日收盘价, "限售股类型": 限售股类型, "解禁前20日涨跌幅": 解禁前20日涨跌幅, "解禁后20日涨跌幅": 解禁后20日涨跌幅 } dictName = "PreData:股票:%s:限售解禁" % code #r.SortDictSave(dictName,json.dumps(item,ensure_ascii=False),日期Tag) print(item) pass
def SaveData北向成交明细ToRedis(code): jsonStr = HtmlConvertor.GetInst().LoadFromString( html=chrome.page_source).ConvertToJson() jsonData = json.loads(jsonStr) if ("Tables" in jsonData and [] != jsonData["Tables"]): tables = jsonData["Tables"] rows = tables[len(tables) - 1]["body"] for row in rows: if 9 == len(row): 日期Tag = CONVERT.DateToInt(row[0]) item = { "Code": code, "日期": row[0], "日期Tag": CONVERT.DateToInt(row[0]), "当日排名": CONVERT.StrToInt(row[2]), "收盘价": CONVERT.StrToFloat(row[3]), "涨跌幅": CONVERT.UnitStrToFloat(row[4]), "沪深股通净买额": CONVERT.UnitStrToFloat(row[5]), "沪深股通买入金额": CONVERT.UnitStrToFloat(row[6]), "沪深股通卖出金额": CONVERT.UnitStrToFloat(row[7]), "沪深股通成交金额": CONVERT.UnitStrToFloat(row[8]) } print("北向成交明细 Task%s %s" % (taskID, item)) qName北向成交 = "Stock:BXCJMX:%s" % code targetNameSpace = qName北向成交 r.SortDictSave(qName北向成交, item, 日期Tag) SyncNotice.SendSyncNotice( targetNameSpace, { "namespace": targetNameSpace, "code": code, "score": 日期Tag, "value": item, "type": "SortedSet" })
def callback(qName, input): #print(input) data = json.loads(input) code = data["Code"].strip() name = data["Name"].strip() dateTag = CONVERT.StrToInt(data["DateTag"]) article = data["Article"] uri = urllib.parse.urlparse(data["Uri"].strip()) hostname = uri.hostname scheme = uri.scheme #print(article) points = article.split("\n") for point in points: point = point.strip() if (5 <= len(point) and CHECKER.HasHanZi(point)): print(point) item = { "Code": code, "Name": name, "Section": point, "DateTag": dateTag } dictName = "PreData:股票:%s:核心题材" % code r.SortDictSave(dictName, item, dateTag) pass
def SaveData北向成交明细ToRedis(code): jsonStr = HtmlConvertor.GetInst().LoadFromString( html=chrome.page_source).ConvertToJson() jsonData = json.loads(jsonStr) if ("Tables" in jsonData and [] != jsonData["Tables"]): tables = jsonData["Tables"] rows = tables[len(tables) - 1]["body"] for row in rows: if 9 == len(row): 日期Tag = CONVERT.DateToInt(row[0]) item = { "Code": code, "日期": row[0], "日期Tag": CONVERT.DateToInt(row[0]), "当日排名": CONVERT.StrToInt(row[2]), "收盘价": CONVERT.StrToFloat(row[3]), "涨跌幅": CONVERT.UnitStrToFloat(row[4]), "沪深股通净买额": CONVERT.UnitStrToFloat(row[5]), "沪深股通买入金额": CONVERT.UnitStrToFloat(row[6]), "沪深股通卖出金额": CONVERT.UnitStrToFloat(row[7]), "沪深股通成交金额": CONVERT.UnitStrToFloat(row[8]) } print(item) qName北向成交 = "Stock:BXCJMX:%s" % code r.SortDictSave(qName北向成交, item, 日期Tag)
def CreateTask北向成交明细(setName, item): code = item[0] url = "http://data.eastmoney.com/hsgt/%s.html" % code #北向持股明细列表 task = {"Code": code, "Url": url, "RetryCount": 3} taskID = CONVERT.StrToInt(code[len(code) - 1]) % psutil.cpu_count(True) qName = "Stock:Task:BXCJMX:Task%s" % taskID r.QueueEn(qName, json.dumps(task, ensure_ascii=False)) pass
def ProcWebData(): taskId = "Task0" serverUrl = "http://122.51.159.248/YunStock2Service?keyName=BXCJMX&taskId=%s" % taskId #serverUrl="http://127.0.0.1:80/YunStock2Service?keyName=BXCJMX&taskId=%s"%taskId while True: try: res = requests.get(serverUrl) print("接收 %s \r\n --------\r\n" % res.text) res = json.loads(res.text) if (res["success"] == True): data = res["data"] if 10 <= len(data): data = json.loads(data) code = data["Code"] url = data["Url"] resArr = [] jsonStr = spider.LoadWeb(url, "北向成交明细").GetDataFromWeb() jsonData = json.loads(jsonStr) if ("Tables" in jsonData and [] != jsonData["Tables"]): tables = jsonData["Tables"] rows = tables[len(tables) - 1]["body"] for row in rows: if 9 == len(row): 日期Tag = CONVERT.DateToInt(row[0]) item = { "Code": code, "日期": row[0], "日期Tag": CONVERT.DateToInt(row[0]), "当日排名": CONVERT.StrToInt(row[2]), "收盘价": CONVERT.StrToFloat(row[3]), "涨跌幅": CONVERT.UnitStrToFloat(row[4]), "沪深股通净买额": CONVERT.UnitStrToFloat(row[5]), "沪深股通买入金额": CONVERT.UnitStrToFloat(row[6]), "沪深股通卖出金额": CONVERT.UnitStrToFloat(row[7]), "沪深股通成交金额": CONVERT.UnitStrToFloat(row[8]) } resArr.append(item) post_data = { "keyName": "BXCJMX", "taskId": taskId, "method": "SaveBXCJMX", "jsonReq": json.dumps(data, ensure_ascii=False), "jsonRes": json.dumps(resArr, ensure_ascii=False) } res2 = requests.post(serverUrl, data=post_data) print("POST %s \r\n RES %s \r\n --------- \r\n" % (post_data, res2.text)) else: time.sleep(10) except BaseException as e: print(" 异常 %s " % e) time.sleep(20) pass
def CreateTask资金流向(setName, item): code = item[0] url = "http://data.eastmoney.com/zjlx/%s.html" % code threadCount = psutil.cpu_count(True) taskID = CONVERT.StrToInt(code[len(code) - 1]) % threadCount task = {"Code": code, "Url": url, "RetryCount": 3} qName = "Stock:Task:ZJLX:Task%d" % taskID r.QueueEn(qName, json.dumps(task, ensure_ascii=False)) print("CreateTask资金流向 %s" % code) pass
def CreateTask北向成交明细(dictName, key, val, pageIndex, pageCount, pageSize, curIndex, total): code = key url = "http://data.eastmoney.com/hsgt/%s.html" % code #北向持股明细列表 task = {"Code": code, "Url": url, "RetryCount": 3} taskID = CONVERT.StrToInt(code[len(code) - 1]) % 4 qName = "Stock:Task:BXCJMX:Task%s" % taskID r.QueueEn(qName, json.dumps(task, ensure_ascii=False)) print(curIndex) pass
def ProcWebData(): taskId = "Task0" serverUrl = "http://122.51.159.248:5000/YunStock2Service?keyName=JGDYLB&taskId=%s" % taskId #serverUrl = "http://127.0.0.1:5000/YunStock2Service?keyName=JGDYLB&taskId=%s" % taskId count = 0 while True: try: res = requests.get(serverUrl) print("接收 %s \r\n --------\r\n" % res.text) res = json.loads(res.text) if (res["success"] == True): data = res["data"] if 10 <= len(data): data = json.loads(data) url = data["Url"] code = data["Code"] resArr = [] jsonStr = spider.LoadWeb(url, "机构调研列表").GetDataFromWeb() jsonData = json.loads(jsonStr) if "Tables" in jsonData: tables = jsonData["Tables"] if 0 < len(tables): rows = tables[len(tables) - 1]["body"] for row in rows: if 10 == len(row): 持股日期Tag = CONVERT.DateToInt(row[0]) item = { "Code": code, "接待机构数量": CONVERT.StrToInt(row[4]), "接待方式": row[5], "接待人员": row[6], "接待地点": row[7], "接待日期": row[8], "公告日期": row[9], "接待日期Tag": CONVERT.DateToInt(row[8]), "公告日期Tag": CONVERT.DateToInt(row[9]) } resArr.append(item) post_data = { "taskId": taskId, "keyName": "JGDYLB", "jsonReq": json.dumps(data, ensure_ascii=False), "jsonRes": json.dumps(resArr, ensure_ascii=False) } res2 = requests.post(serverUrl, data=post_data) print("POST %s \r\n RES %s \r\n --------- \r\n" % (post_data, res2.text)) else: time.sleep(3) except BaseException as e: print(" 异常 %s " % e) time.sleep(20) pass
def callback(qName,input): #input = input.encode(encoding= "gbk").decode(encoding="utf-8") #print(input) data=json.loads(input) code = data["Code"].strip() name = data["Name"].strip() tables = data["Tables"] if 2!=len(tables): return None table=data["Tables"][1] rows=table["body"] for row in rows: if 16== len(row): 序号 = CONVERT.StrToInt( row[0]) 日期 =row[1] 日期Tag =CONVERT.DateToInt (row[1]) 收盘价 = CONVERT.StrToFloat(row[3]) 涨跌幅 = CONVERT.PercentStrToFloat(row[4]) 后1日涨跌幅 = CONVERT.PercentStrToFloat(row[5]) 后2日涨跌幅 =CONVERT.PercentStrToFloat(row[6]) 后3日涨跌幅 = CONVERT.PercentStrToFloat(row[7]) 后5日涨跌幅 = CONVERT.PercentStrToFloat( row[8]) 后10日涨跌幅= CONVERT.PercentStrToFloat(row[9]) 后20日涨跌幅= CONVERT.PercentStrToFloat(row[10]) 后30日涨跌幅= CONVERT.PercentStrToFloat(row[11]) 上榜营业部买入合计= CONVERT.StrToFloat(row[12]) 上榜营业部卖出合计= CONVERT.StrToFloat(row[13]) 上榜营业部买卖净额= CONVERT.StrToFloat(row[14]) 上榜原因= row[15] item={ "Code":code ,"Name":name ,"序号":序号 ,"日期":日期 ,"日期Tag":日期Tag ,"收盘价":收盘价 ,"涨跌幅":涨跌幅 ,"后1日涨跌幅":后1日涨跌幅 ,"后2日涨跌幅":后2日涨跌幅 ,"后3日涨跌幅":后3日涨跌幅 ,"后5日涨跌幅":后5日涨跌幅 ,"后10日涨跌幅":后10日涨跌幅 ,"后20日涨跌幅":后20日涨跌幅 ,"后30日涨跌幅":后30日涨跌幅 ,"上榜营业部买入合计":上榜营业部买入合计 ,"上榜营业部卖出合计":上榜营业部卖出合计 ,"上榜营业部买卖净额":上榜营业部买卖净额 ,"上榜原因":上榜原因 } dictName="PreData:股票:%s:龙虎榜单历次上榜"%code r.SortDictSave(dictName,json.dumps(item,ensure_ascii=False),日期Tag) print(item) pass
def callback(qName, input): #print(input) data = json.loads(input) code = data["Code"].strip() name = data["Name"].strip() tables = data["Tables"] if 3 != len(tables): return None table = data["Tables"][2] rows = table["body"] for row in rows: if 15 == len(row): 序号 = CONVERT.StrToInt(row[0]) 交易日期 = row[1] 交易日期Tag = CONVERT.DateToInt(row[1]) 涨跌幅 = CONVERT.StrToFloat(row[2]) 收盘价 = CONVERT.StrToFloat(row[3]) 成交价 = CONVERT.StrToFloat(row[4]) 折溢率 = CONVERT.StrToFloat(row[5]) 成交量 = CONVERT.StrToFloat(row[6]) 成交额 = CONVERT.StrToFloat(row[7]) 成交额流通市值比例 = CONVERT.PercentStrToFloat(row[8]) 买方营业部 = row[9] 卖方营业部 = row[10] 上榜后涨跌幅1日 = CONVERT.StrToFloat(row[11]) 上榜后涨跌幅5日 = CONVERT.StrToFloat(row[12]) 上榜后涨跌幅10日 = CONVERT.StrToFloat(row[13]) 上榜后涨跌幅20日 = CONVERT.StrToFloat(row[14]) item = { "Code": code, "Name": name, "序号": 序号, "交易日期": 交易日期, "交易日期Tag": 交易日期Tag, "涨跌幅": 涨跌幅, "收盘价": 收盘价, "成交价": 成交价, "折溢率": 折溢率, "成交量": 成交量, "成交额": 成交额, "成交额流通市值比例": 成交额流通市值比例, "买方营业部": 买方营业部, "卖方营业部": 卖方营业部, "上榜后涨跌幅1日": 上榜后涨跌幅1日, "上榜后涨跌幅5日": 上榜后涨跌幅5日, "上榜后涨跌幅10日": 上榜后涨跌幅10日, "上榜后涨跌幅20日": 上榜后涨跌幅20日 } dictName = "PreData:股票:%s:大宗交易" % code r.SortDictSave(dictName, json.dumps(item, ensure_ascii=False), 交易日期Tag) print(item) pass
def CreateTask新财务分析(setName, item): code = item[0] code2 = "SH" + code if code[0] == "6" else "SZ" + code url = "http://f10.eastmoney.com/f10_v2/FinanceAnalysis.aspx?code=%s" % code2 #核心题材 threadCount = psutil.cpu_count(True) taskID = CONVERT.StrToInt(code[len(code) - 1]) % threadCount task = {"Code": code, "Url": url, "RetryCount": 3} qName = "Stock:Task:CWFX:Task%d" % taskID r.QueueEn(qName, json.dumps(task, ensure_ascii=False)) print("CreateTask新财务分析 %s" % code) pass
def callback(qName, input): print(input) data = json.loads(input) code = data["Code"].strip() name = data["Name"].strip() tables = data["Tables"] if 0 == len(tables): return None table = data["Tables"][0] rows = table["body"] for row in rows: if (10 == len(row)): 序号 = row[0] 代码 = row[1] 名称 = row[2] 接待机构数量 = CONVERT.StrToInt(row[4]) 接待方式 = row[5] 接待人员 = row[6] 接待地点 = row[7] 接待日期 = row[8] 接待日期Tag = 0 if (10 == len(接待日期)): 接待日期Tag = CONVERT.DateToInt(接待日期) 公告日期 = row[9] 公告日期Tag = 0 if 10 != len(公告日期) and 10 == len(接待日期): 公告日期 = 接待日期.split("-")[0] + "/" + 公告日期 公告日期Tag = CONVERT.DateToInt(公告日期) item = { "Code": code, "Name": name, "序号": 序号, "代码": 代码, "名称": 名称, "接待机构数量": 接待机构数量, "接待方式": 接待方式, "接待人员": 接待人员, "接待地点": 接待地点, "接待日期": 接待日期, "接待日期Tag": 接待日期Tag, "公告日期": 公告日期, "公告日期Tag": 公告日期Tag } dictName = "PreData:股票:%s:机构调研" % code r.SortDictSave(dictName, json.dumps(item, ensure_ascii=False), 接待日期Tag) print(item) pass
def callback(qName,input): #input = input.encode(encoding= "gbk").decode(encoding="utf-8") #print(input) data=json.loads(input) code = data["Code"].strip() name = data["Name"].strip() tables = data["Tables"] if 3!=len(tables): return None table=data["Tables"][2] rows=table["body"] for row in rows: if 13== len(row): 序号 =CONVERT.StrToInt (row[0]) 交易标的 =row[1] 买方 = row[2] 卖方 = row[3] 交易金额 = CONVERT.UnitStrToFloat(row[4]) 币种 =row[5] 股权转让比例 = CONVERT.StrToFloat(row[6]) 标的类型 = row[7] 并购方式= row[8] 最新进展= row[9] 披露日期= row[10] 披露日期Tag= CONVERT.DateToInt(row[10]) 最新公告日= row[11] 最新公告日Tag= CONVERT.DateToInt(row[11]) item={ "Code":code ,"Name":name ,"序号":序号 ,"交易标的":交易标的 ,"买方":买方 ,"卖方":卖方 ,"交易金额":交易金额 ,"币种":币种 ,"股权转让比例":股权转让比例 ,"标的类型":标的类型 ,"并购方式":并购方式 ,"最新进展":最新进展 ,"披露日期":披露日期 ,"披露日期Tag":披露日期Tag ,"最新公告日":最新公告日 ,"最新公告日Tag":最新公告日Tag } dictName="PreData:股票:%s:并购重组"%code r.SortDictSave(dictName,json.dumps(item,ensure_ascii=False),最新公告日Tag) print(item) pass
def callback(qName,input): #print(input) data=json.loads(input) code = data["Code"].strip() name = data["Name"].strip() tables = data["Tables"] if 3!=len(tables): return None table=data["Tables"][2] rows=table["body"] for row in rows: if 8== len(row): 序号 =CONVERT.StrToInt (row[0]) 机构名称 = row[1] 日期Tag = CONVERT.GetCurrentDateTag() 机构属性 = row[3] 持股总数 =CONVERT.StrToFloat(row[4]) 持股市值 = CONVERT.StrToFloat(row[5]) 占总股本比例 = CONVERT.StrToFloat( row[6]) 占流通股本比例= CONVERT.StrToFloat(row[7]) item={ "Code":code ,"Name":name ,"序号":序号 ,"日期Tag":日期Tag ,"机构属性":机构属性 ,"持股总数":持股总数 ,"持股市值":持股市值 ,"占总股本比例":占总股本比例 ,"占流通股本比例":占流通股本比例 } dictName="PreData:股票:%s:主力持仓"%code r.SortDictSave(dictName,json.dumps(item,ensure_ascii=False),日期Tag) print(item) pass
def ProcWebData(): taskId = "Task0" serverUrl = "http://122.51.159.248:5000/YunStock2Service?keyName=BXCGTJ&taskId=%s" % taskId #serverUrl = "http://127.0.0.1:5000/YunStock2Service?keyName=BXCGTJ&taskId=%s" % taskId count = 0 while True: try: res = requests.get(serverUrl) print("接收 %s \r\n --------\r\n" % res.text) res = json.loads(res.text) if (res["success"] == True): data = res["data"] if 10 <= len(data): data = json.loads(data) url = data["Url"] params = urllib.parse.parse_qs( urllib.parse.urlparse(url).query) jgCode = params["jgCode"][0] resArr = [] jsonStr = spider.LoadWeb(url, "北向持股统计").GetDataFromWeb() jsonData = json.loads(jsonStr) if "Tables" in jsonData: tables = jsonData["Tables"] if 0 < len(tables): rows = tables[len(tables) - 1]["body"] for row in rows: if 8 == len(row): 持股日期Tag = CONVERT.DateToInt(row[0]) item = { "jgCode": jgCode, "持股日期": row[0], "持股日期Tag": 持股日期Tag, "机构名称": row[1], "持股只数": CONVERT.StrToInt(row[3]), "持股市值": CONVERT.UnitStrToFloat(row[4]), "持股市值变化1日": CONVERT.UnitStrToFloat(row[5]), "持股市值变化5日": CONVERT.UnitStrToFloat(row[6]), "持股市值变化10日": CONVERT.UnitStrToFloat(row[7]) } resArr.append(item) post_data = { "taskId": taskId, "keyName": "BXCGTJ", "jsonReq": json.dumps(data, ensure_ascii=False), "jsonRes": json.dumps(resArr, ensure_ascii=False) } res2 = requests.post(serverUrl, data=post_data) print("POST %s \r\n RES %s \r\n --------- \r\n" % (post_data, res2.text)) if "Links" in jsonData: links = jsonData["Links"] links = list( filter( lambda x: CHECKER.StartWith( x["Href"], "/hsgtcg/InstitutionHdDetail.aspx\?jgCode=" ) and x["Text"] == "详细", links)) links = list( map( lambda x: "http://data.eastmoney.com%s" % x[ "Href"], links)) items = [] for link in links: params = urllib.parse.parse_qs( urllib.parse.urlparse(link).query) date = params["date"][0] dateTag = CONVERT.DateToInt(date) jgCode = params["jgCode"][0] jgName = urllib.parse.unquote( params["jgName"][0].replace( '%u', '\\u').encode('utf-8').decode( 'unicode-escape')).strip() item = { "DateTag": dateTag, "Date": date, "jgCode": jgCode, "jgName": jgName, "Url": link, "RetryCount": 3 } items.append(item) post_data2 = { "keyName": "BXCGMXURLTASK", "jsonReq": json.dumps(data, ensure_ascii=False), "jsonRes": json.dumps(items, ensure_ascii=False) } res3 = requests.post(serverUrl, data=post_data2) print("POST %s \r\n RES %s \r\n --------- \r\n" % (post_data2, res3.text)) count += 1 print("已处理 %s" % count) else: time.sleep(3) except BaseException as e: print(" 异常 %s " % e) time.sleep(20) pass