def get_data(): gctype = OrderedDict([("勘察设计", "001"), ("监理", "002"), ("施工", "003"), ("货物", "004"), ("其它", "005")]) ggtype = OrderedDict([("zhaobiao", "001"), ("zhongbiao", "002"), ("biangeng", "003")]) xs = OrderedDict([("市本级", "001"), ("莱城区", "002"), ("钢城区", "003"), ("高新区", "004"), ("雪野旅游区", "005"), ("经济开发区", "006"), ("其它", "007")]) ggtype2 = OrderedDict([("zhaobiao", "001"), ("zhongbiao", "003"), ("biangeng", "002")]) data = [] for w1 in ggtype.keys(): for w2 in gctype.keys(): p1 = "044001%s" % (ggtype[w1]) p2 = "044001%s%s" % (ggtype[w1], gctype[w2]) href = "http://ggzy.laiwu.gov.cn/lwwznew/jyxx/044001/%s/%s/MoreInfo.aspx?CategoryNum=%s" % ( p1, p2, p2) tmp = [ "gcjs_%s_gctype%s_gg" % (w1, gctype[w2]), href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"gctype": w2}), f2 ] data.append(tmp) for w1 in ggtype2.keys(): for w2 in xs.keys(): p1 = "044002%s" % (ggtype2[w1]) p2 = "044002%s%s" % (ggtype2[w1], xs[w2]) href = "http://ggzy.laiwu.gov.cn/lwwznew/jyxx/044002/%s/%s/MoreInfo.aspx?CategoryNum=%s" % ( p1, p2, p2) tmp = [ "zfcg_%s_xs%s_gg" % (w1, xs[w2]), href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"diqu": w2}), f2 ] data.append(tmp) for w1 in ggtype.keys(): p1 = "044004%s" % (ggtype2[w1]) href = "http://ggzy.laiwu.gov.cn/lwwznew/jyxx/044004/%s/MoreInfo.aspx?CategoryNum=%s" % ( p1, p1) tmp = [ "yiliao_%s_gg" % w1, href, ["name", "ggstart_time", "href", "info"], f1, f2 ] data.append(tmp) remove_arr = [ "gcjs_biangeng_gctype004_gg", "gcjs_biangeng_gctype005_gg", "gcjs_biangeng_gctype005_gg" ] data1 = data.copy() for w in data: if w[0] in remove_arr: data1.remove(w) return data1
def get_data(): data = [] ggtype1 = OrderedDict([("zhaobiao", "001"), ("zhongbiaohx", "003"), ("zhongbiao", "004")]) ggtype2 = OrderedDict([("zhaobiao", "001"), ("yucai", "002"), ("zhongbiao", "004")]) ggtype3 = OrderedDict([("zhaobiao", "001"), ("biangeng", "002"), ("zhongbiao", "004")]) adtype1 = OrderedDict([('本级', '1'), ("宣州", "2"), ("郎溪", "3"), ("广德", "4"), ("宁国", "5"), ('泾县', '6'), ('绩溪', '7'), ('旌德', '8')]) for w1 in ggtype1.keys(): for w2 in adtype1.keys(): href = "http://ggzyjy.xuancheng.gov.cn/XCTPFront/jsgc/01100{0}/01100{1}{2}/?Paging=1".format( adtype1[w2], adtype1[w2], ggtype1[w1]) tmp = [ "gcjs_%s_diqu%s_gg" % (w1, adtype1[w2]), href, ["name", "ggstart_time", "href", 'info'], add_info(f1, {"diqu": w2}), f2 ] data.append(tmp) for w1 in ggtype2.keys(): for w2 in adtype1.keys(): href = "http://ggzyjy.xuancheng.gov.cn/XCTPFront/zfcg/01200{0}/01200{1}{2}/?Paging=1".format( adtype1[w2], adtype1[w2], ggtype2[w1]) tmp = [ "zfcg_%s_diqu%s_gg" % (w1, adtype1[w2]), href, ["name", "ggstart_time", "href", 'info'], add_info(f1, {"diqu": w2}), f2 ] data.append(tmp) for w1 in ggtype3.keys(): for w2 in adtype1.keys(): href = "http://ggzyjy.xuancheng.gov.cn/XCTPFront/xaxm/02200{0}/02200{1}{2}/?Paging=1".format( adtype1[w2], adtype1[w2], ggtype3[w1]) tmp = [ "qsydw_%s_diqu%s_gg" % (w1, adtype1[w2]), href, ["name", "ggstart_time", "href", 'info'], add_info(f1, {"diqu": w2}), f2 ] data.append(tmp) data1 = data.copy() return data1
def get_data(): data = [] ggtype = OrderedDict([("zhaobiao", "1"), ("biangengliubiao", "3"), ("zhongbiao", "2")]) gctype = OrderedDict([('工程', '2'), ("货物", "1"), ("服务", "3"), ("询价", "4")]) adtype = OrderedDict([('东营市', '5'), ("东营区", "6"), ("河口区", "7"), ("广饶县", "8"), ("垦利县", "9"), ("利津县", "10"), ("开发区", "11"), ("东营港", "12"), ("农高区", "13")]) for w1 in ggtype.keys(): for w2 in adtype.keys(): for w3 in gctype.keys(): href = "http://dycg.dongying.gov.cn/BigClassList.aspx?BigClass={gc}&Zone={ad}&Type={gg}".format( gc=gctype[w3], gg=ggtype[w1], ad=adtype[w2]) tmp = [ "zfcg_%s_diqu%s_type%s_gg" % (w1, adtype[w2], gctype[w3]), href, ["name", "ggstart_time", "href", 'info'], add_info(f1, { "jy_type": w3, "diqu": w2 }), f2 ] data.append(tmp) remove_arr = ["zfcg_biangengliubiao_diqu9_type2_gg"] data1 = data.copy() for w in data: if w[0] in remove_arr: data1.remove(w) return data1
def get_data(): data = [] ggtype1 = OrderedDict([("zhaobiao", "2"), ("bumen", "5"), ("zhongbiao", "3")]) adtype1 = OrderedDict([('市本级','1'),("章贡区", "20"), ("赣县", "2"), ("南康", "10"), ("信丰", "9"), ('大余','4'),('上犹','11'),('崇义','7'),('安远','13'),('龙南','6'), ('全南','15'),('定南','16'),('兴国','17'),('宁都','5'),('于都','3'), ('瑞金','12'),('会昌','18'),('寻乌','8'),('石城','14')]) adtype2 = OrderedDict([('市本级', '1'), ("章贡区", "20"), ("赣县", "2"), ("南康", "10"), ("信丰", "9"), ('大余', '4'), ('上犹', '11'), ('崇义', '7'), ('安远', '13'), ('龙南', '6'), ('全南', '15'), ('定南', '16'), ('兴国', '17'), ('宁都', '5'), ('于都', '3'), ('瑞金', '12'), ('会昌', '18'), ('寻乌', '8'), ('石城', '14'),('开发区','19')]) for w1 in adtype1.keys(): href = "http://gcjs.gzzbtbzx.com:88/zbgg/more_ze.asp?keyword=&dq={}&cut=&page=1".format(adtype1[w1]) tmp = ["gcjs_%s_diqu%s_gg" % ('zhaobiao', adtype1[w1]), href, ["name","ggstart_time","href",'info'], add_info(f1, {"diqu": w1}), f2] data.append(tmp) for w1 in adtype1.keys(): href = "http://www.gzzbtbzx.com/more.asp?id=13&city=%s&page=1" % (adtype1[w1]) tmp = ["gcjs_%s_diqu%s_gg" % ('zhongbiaohx', adtype1[w1]), href, ["name","ggstart_time","href",'info'], add_info(f1, {"diqu": w1}), f2] data.append(tmp) for w1 in ggtype1.keys(): for w2 in adtype2.keys(): href="http://www.gzzbtbzx.com/more.asp?id=%s&city=%s&page=1"%(ggtype1[w1],adtype2[w2]) tmp=["zfcg_%s_diqu%s_gg"%(w1,adtype2[w2]),href,["name","ggstart_time","href",'info'],add_info(f1,{"diqu":w2}),f2] data.append(tmp) remove_arr = ["gcjs_zhaobiao_diqu1_gg","zfcg_bumen_diqu4_gg",'zfcg_bumen_diqu7_gg','zfcg_bumen_diqu13_gg', 'zfcg_bumen_diqu15_gg','zfcg_bumen_diqu5_gg','zfcg_bumen_diqu12_gg','zfcg_bumen_diqu18_gg', 'zfcg_bumen_diqu8_gg'] data1 = data.copy() for w in data: if w[0] in remove_arr: data1.remove(w) data1.append(["gcjs_zhaobiao_diqu1_gg", "http://www.gzzbtbzx.com/more.asp?id=12&city=1&page=1", ["name", "ggstart_time", "href", 'info'], add_info(f1, {"diqu": '市本级'}), f2], ) return data1
def get_data(): xs = OrderedDict([("东营市", "001"), ("东营区", "002"), ("河口区", "003"), ("广饶县", "004"), ("垦利区", "005"), ("利津县", "006"), ("开发区", "007"), ("东营港", "008"), ("农高区", "009")]) ggtype1 = OrderedDict([("zhaobiao", "001"), ("zhongbiao", "003"), ("biangeng", "004"), ("biangen", "005")]) ggtype2 = OrderedDict([("zhaobiao", "001"), ("biangen", "002"), ("zhongbiao", "003")]) data = [] for w1 in ggtype1.keys(): for w2 in xs.keys(): p1 = "004001%s" % (ggtype1[w1]) p2 = "004001%s%s" % (ggtype1[w1], xs[w2]) href = "http://www.dyggzyjy.gov.cn/dysite/004/004001/%s/%s/MoreInfo.aspx?CategoryNum=%s" % ( p1, p2, p2) tmp = [ "gcjs_%s_diqu%s_gg" % (w1, xs[w2]), href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"diqu": w2}), f2 ] data.append(tmp) for w1 in ggtype2.keys(): for w2 in xs.keys(): p1 = "004002%s" % (ggtype2[w1]) p2 = "004002%s%s" % (ggtype2[w1], xs[w2]) href = "http://www.dyggzyjy.gov.cn/dysite/004/004002/%s/%s/MoreInfo.aspx?CategoryNum=%s" % ( p1, p2, p2) tmp = [ "zfcg_%s_diqu%s_gg" % (w1, xs[w2]), href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"diqu": w2}), f2 ] data.append(tmp) data1 = data.copy() arr = [] for w in data: if w[0] in arr: data1.remove(w) return data1
page = driver.page_source soup = BeautifulSoup(page, 'html.parser') div = soup.find('td', id="TDContent") return div data = [ [ "gcjs_zhaobiao_diqu1_gg", "http://www.xnggzy.gov.cn/xnweb/jsgc/012001/012001001/?Paging=1", ["name", "href", "ggstart_time", "info"], add_info(f1, {"diqu": '市本级'}), f2 ], [ "gcjs_biangen_diqu1_gg", "http://www.xnggzy.gov.cn/xnweb/jsgc/012002/012002001/?Paging=1", ["name", "href", "ggstart_time", "info"], add_info(f1, {"diqu": '市本级'}), f2 ], [ "gcjs_zhongbiao_diqu1_gg", "http://www.xnggzy.gov.cn/xnweb/jsgc/012003/012003001/?Paging=1", ["name", "href", "ggstart_time", "info"], add_info(f1, {"diqu": '市本级'}), f2 ], [ "gcjs_liubiao_diqu1_gg",
i += 1 if i > 5: break page = driver.page_source soup = BeautifulSoup(page, 'lxml') div = soup.find('div', class_='ewb-text-content ewb-row') return div data = [ [ "zfcg_zhaobiao_diqu2_gg", "http://www.ccggzy.gov.cn/qxxxgk/003001/003001001/CountyZfcgNotice.html", ['gg_type', "name", "ggstart_time", "href", "info"], add_info(f1, {"diqu2": '区县'}), f2 ], [ "zfcg_biangen_diqu2_gg", "http://www.ccggzy.gov.cn/qxxxgk/003001/003001003/CountyZfcgNotice.html", ['gg_type', "name", "ggstart_time", "href", "info"], add_info(f1, {"diqu2": '区县'}), f2 ], [ "zfcg_zhongbiao_diqu2_gg", "http://www.ccggzy.gov.cn/qxxxgk/003001/003001004/CountyZfcgNotice.html", ['gg_type', "name", "ggstart_time", "href", "info"], add_info(f1, {"diqu2": '区县'}), f2 ], [ "gcjs_zhaobiao_diqu2_gg",
return wrap # time.sleep(1) # locator=(By.XPATH,"//span[@class='ui-dialog-loading'][2]") # WebDriverWait(driver,10).until(EC.invisibility_of_element_located(locator)) data=[ ["gcjs_zhaobiao_fangwu_gg","http://www.sqggzy.com/spweb/HNSQ/TradeCenter/tradeList.do?Deal_Type=Deal_Type1" ,["name","ggstart_time","href","info"],add_info(switchto(f1,"房建","招标"),{"gctype":"房建市政"}),f2], ["gcjs_zhaobiao_fangwu_gg","http://www.sqggzy.com/spweb/HNSQ/TradeCenter/tradeList.do?Deal_Type=Deal_Type1" ,["name","ggstart_time","href","info"],add_info(switchto(f1,"交通","招标"),{"gctype":"交通"}),f2], ["gcjs_zhaobiao_shuili_gg","http://www.sqggzy.com/spweb/HNSQ/TradeCenter/tradeList.do?Deal_Type=Deal_Type1" ,["name","ggstart_time","href","info"],add_info(switchto(f1,"水利","招标"),{"gctype":"水利"}),f2], ["zfcg_zhaobiao_gg","http://www.sqggzy.com/spweb/HNSQ/TradeCenter/tradeList.do?Deal_Type=Deal_Type1" ,["name","ggstart_time","href","info"],switchto(f1,"政府采购","招标"),f2], ["zfcg_zhongbiao_gg","http://www.sqggzy.com/spweb/HNSQ/TradeCenter/tradeList.do?Deal_Type=Deal_Type4" ,["name","ggstart_time","href","info"],switchto(f1,"政府采购","结果公告"),f2], ["zfcg_gqita_gg","http://www.sqggzy.com/spweb/HNSQ/TradeCenter/tradeList.do?Deal_Type=Deal_Type1"
ggtype = OrderedDict([("zhaobiao", "001"), ("biangeng", "002"), ("zhongbiao", "003"), ("zhongbiaohx", "004")]) data = [] for w1 in ggtype.keys(): for w2 in dztype.keys(): p1 = "002001%s" % ggtype[w1] p2 = "002001%s%s" % (ggtype[w1], dztype[w2]) href = "http://www.zkggzyjy.gov.cn/zhoukou/jyxx/002001/%s/%s/?Paging=1" % ( p1, p2) tmp = [ "gcjs_%s_diqu%s_gg" % (w1, dztype[w2]), href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"quyu": dztype[w2]}), f2 ] data.append(tmp) ggtype1 = OrderedDict([("zhaobiao", "001"), ("biangeng", "002"), ("zhongbiao", "003")]) for w1 in ggtype1.keys(): for w2 in dztype.keys(): p1 = "002002%s" % ggtype1[w1] p2 = ["002002%s%s" % (ggtype1[w1], dztype[w2])] href = "http://www.zkggzyjy.gov.cn/TPFront/jyxx/002002/%s/%s/?Pagin=1" % ( p1, p2) tmp = [ "zfcg_%s_diqu%s_gg" % (w1, dztype[w2]), href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"quyu": dztype[w2]}), f2
"zfcg_biangeng_diqu2_gg", "http://cgb.yantai.gov.cn/col/col14664/index.html?uid=36257&pageNum=1", ["name", "ggstart_time", "href", 'info'], f1, f2 ], [ "zfcg_yanshou_diqu2_gg", "http://cgb.yantai.gov.cn/col/col14666/index.html?uid=36260&pageNum=1", ["name", "ggstart_time", "href", 'info'], f1, f2 ], ####一次请求整个区县招标数量太大,无法请求,需拆分成县区 [ "zfcg_zhaobiao_diqu3_gg", "http://cgb.yantai.gov.cn/col/col12531/index.html?uid=9401&pageNum=1", ["name", "ggstart_time", "href", 'info'], add_info(f1, {"diqu": "芝罘区"}), f2 ], [ "zfcg_zhaobiao_diqu4_gg", "http://cgb.yantai.gov.cn/col/col12537/index.html?uid=9401&pageNum=1", ["name", "ggstart_time", "href", 'info'], add_info(f1, {"diqu": "莱山区"}), f2 ], [ "zfcg_zhaobiao_diqu5_gg", "http://cgb.yantai.gov.cn/col/col12543/index.html?uid=9401&pageNum=1", ["name", "ggstart_time", "href", 'info'], add_info(f1, {"diqu": "福山区"}), f2 ], [ "zfcg_zhaobiao_diqu6_gg",
def get_data(): data = [] ggtype1 = OrderedDict([("zhaobiao", "001"), ("biangeng", "002"), ("zhongbiao", "003")]) dwtype = OrderedDict([("住建局", "001"), ("公路局", "002"), ("园林局", "003"), ("水利局", "004"), ("交通局", "005"), ("其它", "006")]) for w1 in dwtype.keys(): p1 = "074001001%s" % dwtype[w1] href = "http://ggzyjy.linyi.gov.cn/TPFront/jyxx/074001/074001001/%s" % p1 tmp = [ "gcjs_zhaobiao_dw%s_gg" % dwtype[w1], href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"dwtype": w1}), f2 ] data.append(tmp) tmp = [ "gcjs_biangeng_gg", "http://ggzyjy.linyi.gov.cn/TPFront/jyxx/074001/074001002/", ["name", "ggstart_time", "href", "info"], f1, f2 ] data.append(tmp) dwtype1 = OrderedDict([("住建局", "001"), ("交通局", "005"), ("其它", "006")]) for w1 in dwtype1.keys(): p1 = "074001003001%s" % dwtype1[w1] href = "http://ggzyjy.linyi.gov.cn/TPFront/jyxx/074001/074001003/074001003001/%s" % p1 tmp = [ "gcjs_zgys_dw%s_gg" % dwtype1[w1], href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"dwtype": w1}), f2 ] data.append(tmp) dwtype2 = OrderedDict([("住建局", "001"), ("公路局", "002"), ("水利局", "004"), ("交通局", "005"), ("其它", "006")]) for w1 in dwtype2.keys(): p1 = "074001003002%s" % dwtype2[w1] href = "http://ggzyjy.linyi.gov.cn/TPFront/jyxx/074001/074001003/074001003002/%s" % p1 tmp = [ "gcjs_zhongbiaohx_dw%s_gg" % dwtype2[w1], href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"dwtype": w1}), f2 ] data.append(tmp) dwtype3 = OrderedDict([("住建局", "001"), ("公路局", "002"), ("园林局", "003"), ("水利局", "004"), ("交通局", "005"), ("其它", "006")]) for w1 in dwtype3.keys(): p1 = "074001003003%s" % dwtype3[w1] href = "http://ggzyjy.linyi.gov.cn/TPFront/jyxx/074001/074001003/074001003003/%s" % p1 tmp = [ "gcjs_zhongbiao_dw%s_gg" % dwtype3[w1], href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"dwtype": w1}), f2 ] data.append(tmp) ggtype = OrderedDict([("yucai", "001"), ("zhaobiao", "002"), ("zhongbiao", "004")]) zbfs = OrderedDict([("公开招标", "001"), ("竞争性谈判", "002"), ("邀请招标", "003"), ("单一来源", "004"), ("询价", "005"), ("协议采购", "006"), ("竞争性磋商", "007")]) for w1 in ggtype.keys(): for w2 in zbfs.keys(): p1 = "074002%s" % ggtype[w1] p2 = "074002%s%s" % (ggtype[w1], zbfs[w2]) href = "http://ggzyjy.linyi.gov.cn/TPFront/jyxx/074002/074002001/074002001001/" tmp = [ "zfcg_%s_zbfs%s_gg" % (w1, zbfs[w2]), href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"zbfs": w2}), f2 ] data.append(tmp) tmp = [ "zfcg_biangeng_gg", "http://ggzyjy.linyi.gov.cn/TPFront/jyxx/074002/074002003/", ["name", "ggstart_time", "href", "info"], f1, f2 ] data.append(tmp) ggtype2 = OrderedDict([("zhaobiao", "001"), ("biangeng", "002"), ("zhongbiao", "003")]) for w1 in ggtype2.keys(): p1 = "074006%s" % ggtype2[w1] href = "http://ggzyjy.linyi.gov.cn/TPFront/jyxx/074006/%s/" % p1 tmp = [ "zfcg_%s_gg" % w1, href, ["name", "ggstart_time", "href", "info"], f1, f2 ] data.append(tmp) data1 = data.copy() arr = ["gcjs_zhongbiao_dw006_gg"] for w in data: if w[0] in arr: data1.remove(w) return data1
def get_data(): data = [] # 工程建设部分 xs = OrderedDict([("市本级", "001"), ("广安区", "002"), ("前锋区", "003"), ("岳池县", "004"), ("武胜县", "005"), ("邻水县", "006"), ("华蓥市", "007")]) # "http://www.bzggzyjy.gov.cn/bzweb/002/002004/002004001/" ggtype = OrderedDict([("zhaobiao", "001"), ("biangeng", "002"), ("zishenjieguo", "003"), ("zhongbiaohx", "004"), ("zhongbiaohxbiangen", "005"), ("zhaobiao_jingzheng", "007")]) for w2 in xs.keys(): for w1 in ggtype.keys(): p1 = "009%s" % (xs[w2]) p2 = "009%s%s" % (xs[w2], ggtype[w1]) href = "http://www.gasggzy.com/gasggzy/gcjs/%s/%s/MoreInfo.aspx?CategoryNum=%s" % ( p1, p2, p2) tb = "gcjs_%s_diqu%s_gg" % (w1, xs[w2]) if tb == 'gcjs_zhongbiaohxbiangen_diqu003_gg': href = "http://www.gasggzy.com/gasggzy/gcjs/009003/009003006/MoreInfo.aspx?CategoryNum=009003006" if tb == 'gcjs_zhaobiao_jingzheng_diqu003_gg': href = 'http://www.gasggzy.com/gasggzy/gcjs/009003/009003008/MoreInfo.aspx?CategoryNum=009003008' col = ["name", "ggstart_time", "href", "info"] tmp = [tb, href, col, add_info(f1, {"diqu": w2}), f2] data.append(tmp) # 政府采购部分 # 招标 zbfs = OrderedDict([("zhaobiao", "002"), ("jieguo", "003"), ("biangen", "004"), ("qita", "005")]) for w2 in xs.keys(): for w1 in zbfs.keys(): p1 = "010%s" % (xs[w2]) p2 = "010%s%s" % (xs[w2], zbfs[w1]) href = "http://www.gasggzy.com/gasggzy/zfcg/%s/%s/MoreInfo.aspx?CategoryNum=%s" % ( p1, p2, p2) col = ["name", "ggstart_time", "href", "info"] tb = "zfcg_%s_diqu%s_gg" % (w1, xs[w2]) tmp = [tb, href, col, add_info(f1, {"diqu": w2, "zbfs": w1}), f2] data.append(tmp) # 国有企业 ggtype2 = OrderedDict([("zhaobiao", "001"), ("biangen", "002"), ("jieguo", "003"), ("qita", "004")]) gq = OrderedDict([("工程建设", "001"), ("企业采购", "002")]) for w2 in gq.keys(): for w1 in ggtype2.keys(): p1 = "012%s" % (gq[w2]) p2 = "012%s%s" % (gq[w2], ggtype2[w1]) href = "http://www.gasggzy.com/gasggzy/gyqy/%s/%s/MoreInfo.aspx?CategoryNum=%s" % ( p1, p2, p2) col = ["name", "ggstart_time", "href", "info"] tb = "qsydw_%s_lx%s_gg" % (w1, gq[w2]) tmp = [tb, href, col, add_info(f1, {"leixing": w2}), f2] data.append(tmp) data1 = data.copy() return data1
def get_data(): dztype=OrderedDict([("驻马店市","001"),("西平县","002"),("上蔡县","003"),("平舆县","004"),("正阳县","005"),("确山县","006") ,("泌阳县","007"),("汝南县","008"),("遂平县","009")]) ggtype=OrderedDict([("zhaobiao","001"),("zgys","002"),("biangeng","003"),("liubiao","004"),("zhongbiaohx","005")]) data=[] for w1 in ggtype.keys(): for w2 in dztype.keys(): p1="003001%s"%ggtype[w1] p2="003001%s%s"%(ggtype[w1],dztype[w2]) href="http://www.zmdggzy.gov.cn/TPFront/jyxx/003001/%s/%s/"%(p1,p2) tmp=["gcjs_%s_diqu%s_gg"%(w1,dztype[w2]),href,["name","ggstart_time","href","info"],add_info(f1,{"quyu":dztype[w2]}),f2] data.append(tmp) ggtype1=OrderedDict([("zhaobiao","001"),("yucai","002"),("biangeng","003"),("zhongbiao","004"),("liubiao","005")]) for w1 in ggtype1.keys(): for w2 in dztype.keys(): p1="003002%s"%ggtype1[w1] p2=["003002%s%s"%(ggtype1[w1],dztype[w2])] href="http://www.zmdggzy.gov.cn/TPFront/jyxx/003002/%s/%s/"%(p1,p2) tmp=["zfcg_%s_diqu%s_gg"%(w1,dztype[w2]),href,["name","ggstart_time","href","info"],add_info(f1,{"quyu":dztype[w2]}),f2] data1=data.copy() arr=["gcjs_zgys_diqu005_gg","gcjs_zgys_diqu009_gg"] for w in data: if w[0] in arr:data1.remove(w) return data1
time.sleep(0.1) after = len(driver.page_source) i += 1 if i > 5: break page = driver.page_source soup = BeautifulSoup(page, 'lxml') div = soup.find('div',class_='ewb-text-content ewb-row') return div data=[ ["zfcg_zhaobiao_diqu2_gg","http://www.ccggzy.gov.cn/qxxxgk/003001/003001001/CountyZfcgNotice.html",['gg_type',"name","ggstart_time","href","info"],add_info(f1,{"diqu2":'区县'}),f2], ["zfcg_biangen_diqu2_gg","http://www.ccggzy.gov.cn/qxxxgk/003001/003001003/CountyZfcgNotice.html",['gg_type',"name","ggstart_time","href","info"],add_info(f1,{"diqu2":'区县'}),f2], ["zfcg_zhongbiao_diqu2_gg","http://www.ccggzy.gov.cn/qxxxgk/003001/003001004/CountyZfcgNotice.html",['gg_type',"name","ggstart_time","href","info"],add_info(f1,{"diqu2":'区县'}),f2], ["gcjs_zhaobiao_diqu2_gg","http://www.ccggzy.gov.cn/qxxxgk/003002/003002001/CountyPurhcaseNotice.html",["name","ggstart_time","href","info"],add_info(f1,{"diqu2":'区县'}),f2], ["gcjs_zhongbiaohx_diqu2_gg","http://www.ccggzy.gov.cn/qxxxgk/003002/003002002/CountyPurhcaseNotice.html",["name","ggstart_time","href","info"],add_info(f1,{"diqu2":'区县'}),f2], ["gcjs_biangen_diqu2_gg","http://www.ccggzy.gov.cn/qxxxgk/003002/003002003/CountyPurhcaseNotice.html",["name","ggstart_time","href","info"],add_info(f1,{"diqu2":'区县'}),f2], ["gcjs_zhongbiao_diqu2_gg","http://www.ccggzy.gov.cn/qxxxgk/003002/003002004/CountyPurhcaseNotice.html",["name","ggstart_time","href","info"],add_info(f1,{"diqu2":'区县'}),f2], ["zfcg_yucai_diqu1_gg","http://www.ccggzy.gov.cn/sjxxgk/002001/002001002/CityZfcgNotice.html",["ggtype_","name","ggstart_time","href","info"],add_info(f4,{"diqu1":'市级'}),f5], ["zfcg_zhaobiao_diqu1_gg","http://www.ccggzy.gov.cn/sjxxgk/002001/002001001/CityZfcgNotice.html",['ggtype_',"name","ggstart_time","href","info"],add_info(f4,{"diqu1":'市级'}),f5], ["zfcg_biangen_diqu1_gg","http://www.ccggzy.gov.cn/sjxxgk/002001/002001003/CityZfcgNotice.html",['ggtype_',"name","ggstart_time","href","info"],add_info(f4,{"diqu1":'市级'}),f5], ["zfcg_jieguo_diqu1_gg","http://www.ccggzy.gov.cn/sjxxgk/002001/002001004/CityZfcgNotice.html",['ggtype_',"name","ggstart_time","href","info"],add_info(f4,{"diqu1":'市级'}),f5],
def get_data(): data = [] #工程建设部分 xs = OrderedDict([("市本级", "001"), ("滨城区", "002"), ("沾化区", "003"), ("惠民县", "004"), ("阳信县", "005"), ("无棣县", "006"), ("博兴县", "007"), ("邹平县", "008"), ("北海新区", "009"), ("开发区", "010"), ("高新区", "011")]) #"http://www.bzggzyjy.gov.cn/bzweb/002/002004/002004001/" ggtype = OrderedDict([("zhaobiao", "001"), ("zhongbiaohx", "002"), ("biangeng", "003")]) for w1 in ggtype.keys(): for w2 in xs.keys(): p1 = "002004%s" % (ggtype[w1]) p2 = "002004%s%s" % (ggtype[w1], xs[w2]) href = "http://www.bzggzyjy.gov.cn/bzweb/002/002004/%s/%s/MoreInfo.aspx?CategoryNum=%s" % ( p1, p2, p2) tb = "gcjs_%s_diqu%s_gg" % (w1, xs[w2]) col = ["name", "ggstart_time", "href", "info"] tmp = [tb, href, col, add_info(f1, {"diqu": w2}), f2] data.append(tmp) #政府采购部分 #招标 zbfs = OrderedDict([("公开招标", "001"), ("网上竞价", "002"), ("竞争性谈判", "003"), ("询价", "004"), ("邀请招标", "005"), ("定点采购", "007"), ("竞争性磋商", "008")]) for w1 in zbfs.keys(): for w2 in xs.keys(): p1 = "002005001%s" % (zbfs[w1]) p2 = "002005001%s%s" % (zbfs[w1], xs[w2]) href = "http://www.bzggzyjy.gov.cn/bzweb/002/002005/002005001/%s/%s/MoreInfo.aspx?CategoryNum=%s" % ( p1, p2, p2) tmp = [ 'zfcg_zhaobiao_diqu%s_zbfs%s_gg' % (xs[w2], zbfs[w1]), href, ["name", "ggstart_time", "href", "info"], add_info(f1, { "diqu": w2, "zbfs": w1 }), f2 ] data.append(tmp) #变更和yucai 中标 yanshou feibiao ggtype2 = OrderedDict([("biangeng", "002"), ("yucai", "004"), ("zhongbiaohx", "003"), ("yanshou", "006"), ("liubiao", "007"), ("hetong", "005")]) for w1 in ggtype2.keys(): for w2 in xs.keys(): p1 = "002005%s" % (ggtype2[w1]) p2 = "002005%s%s" % (ggtype2[w1], xs[w2]) href = "http://www.bzggzyjy.gov.cn/bzweb/002/002005/%s/%s/MoreInfo.aspx?CategoryNum=%s" % ( p1, p2, p2) tmp = [ 'zfcg_%s_diqu%s_gg' % (w1, xs[w2]), href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"diqu": w2}), f2 ] data.append(tmp) data1 = data.copy() remove_arr = [ "zfcg_zhaobiao_diqu004_zbfs002_gg", "zfcg_zhaobiao_diqu007_zbfs002_gg", "zfcg_zhaobiao_diqu007_zbfs004_gg", "zfcg_zhaobiao_diqu009_zbfs005_gg", "zfcg_zhaobiao_diqu008_zbfs005_gg", "zfcg_zhaobiao_diqu009_zbfs005_gg", "zfcg_zhaobiao_diqu011_zbfs005_gg", "zfcg_zhaobiao_diqu003_zbfs007_gg", "zfcg_zhaobiao_diqu003_zbfs007_gg", "zfcg_zhaobiao_diqu006_zbfs007_gg", "zfcg_zhaobiao_diqu009_zbfs007_gg", "zfcg_zhaobiao_diqu009_zbfs005_gg", "zfcg_zhaobiao_diqu009_zbfs005_gg" ] for w in data: if w[0] in remove_arr: data1.remove(w) return data1
page = driver.page_source soup = BeautifulSoup(page, 'lxml') div = soup.find('div', class_='detail-main') #div=div.find_all('div',class_='ewb-article')[0] return div data = [ [ "gcjs_shigong_zhaobiao_gg", "http://ggzyjy.yichang.gov.cn/TPFront/jyxx/003001/003001001/003001001001/", ["name", "ggstart_time", "href", "info"], add_info(f1, {"gctype": "施工"}), f2 ], [ "gcjs_jianli_zhaobiao_gg", "http://ggzyjy.yichang.gov.cn/TPFront/jyxx/003001/003001001/003001001002/", ["name", "ggstart_time", "href", "info"], add_info(f1, {"gctype": "监理"}), f2 ], [ "gcjs_kancha_zhaobiao_gg", "http://ggzyjy.yichang.gov.cn/TPFront/jyxx/003001/003001001/003001001003/", ["name", "ggstart_time", "href", "info"], add_info(f1, {"gctype": "勘察"}), f2 ], [ "gcjs_gcqita_zhaobiao_gg",
def get_data(): data = [] ggtype1 = OrderedDict([("zhaobiao", "001"), ("biangeng", "002"), ("zhongbiaohx", "003")]) sx = OrderedDict([("市本级", "001"), ("张店区", "002"), ("高新区", "003"), ("文昌湖区", "004"), ("高青县", "005"), ("临淄区", "006"), ("桓台县", "007"), ("博山区", "008")]) hrefs = [ "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001001/002001001001/MoreInfo.aspx?CategoryNum=268698113", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001001/002001001002/MoreInfo.aspx?CategoryNum=268698114", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001001/002001001004/MoreInfo.aspx?CategoryNum=268698116", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001001/002001001005/MoreInfo.aspx?CategoryNum=268698117", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001001/002001001006/MoreInfo.aspx?CategoryNum=268698118", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001001/002001001007/MoreInfo.aspx?CategoryNum=268698119", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001001/002001001008/MoreInfo.aspx?CategoryNum=2001001008", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001001/002001001009/MoreInfo.aspx?CategoryNum=2001001009", #shanchu "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001002/002001002001/MoreInfo.aspx?CategoryNum=268698625", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001002/002001002002/MoreInfo.aspx?CategoryNum=268698626", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001002/002001002004/MoreInfo.aspx?CategoryNum=268698628", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001002/002001002005/MoreInfo.aspx?CategoryNum=268698629", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001002/002001002006/MoreInfo.aspx?CategoryNum=268698630", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001002/002001002007/MoreInfo.aspx?CategoryNum=268698631", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001002/002001002008/MoreInfo.aspx?CategoryNum=2001002008", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001002/002001002009/MoreInfo.aspx?CategoryNum=2001002009", #xx "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001003/002001003001/MoreInfo.aspx?CategoryNum=268699137", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001003/002001003002/MoreInfo.aspx?CategoryNum=268699138", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001003/002001003004/MoreInfo.aspx?CategoryNum=268699140", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001003/002001003005/MoreInfo.aspx?CategoryNum=268699141", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001003/002001003006/MoreInfo.aspx?CategoryNum=268699142", #xx "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001003/002001003007/MoreInfo.aspx?CategoryNum=268699143", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001003/002001003008/MoreInfo.aspx?CategoryNum=2001003008", #xx "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002001/002001003/002001003009/MoreInfo.aspx?CategoryNum=2001003009", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002001/002002001001/MoreInfo.aspx?CategoryNum=268960257", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002001/002002001002/MoreInfo.aspx?CategoryNum=268960258", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002001/002002001002/MoreInfo.aspx?CategoryNum=268960260", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002001/002002001002/MoreInfo.aspx?CategoryNum=268960261", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002001/002002001002/MoreInfo.aspx?CategoryNum=268960262", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002001/002002001002/MoreInfo.aspx?CategoryNum=268960263", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002001/002002001008/MoreInfo.aspx?CategoryNum=2002001008", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002001/002002001009/MoreInfo.aspx?CategoryNum=2002001009", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002001/002002001010/MoreInfo.aspx?CategoryNum=268960264", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002002/002002002001/MoreInfo.aspx?CategoryNum=268960769", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002002/002002002002/MoreInfo.aspx?CategoryNum=268960770", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002002/002002002004/MoreInfo.aspx?CategoryNum=268960772", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002002/002002002007/MoreInfo.aspx?CategoryNum=268960773", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002002/002002002007/MoreInfo.aspx?CategoryNum=268960774", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002002/002002002007/MoreInfo.aspx?CategoryNum=268960775", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002002/002002002008/MoreInfo.aspx?CategoryNum=2002002008", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002002/002002002009/MoreInfo.aspx?CategoryNum=2002002009", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002002/002002002010/MoreInfo.aspx?CategoryNum=268960776", # "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002003/002002003001/MoreInfo.aspx?CategoryNum=268961281", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002003/002002003002/MoreInfo.aspx?CategoryNum=268961282", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002003/002002003004/MoreInfo.aspx?CategoryNum=268961284", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002003/002002003005/MoreInfo.aspx?CategoryNum=268961285", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002003/002002003006/MoreInfo.aspx?CategoryNum=268961286", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002003/002002003007/MoreInfo.aspx?CategoryNum=268961287", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002003/002002003008/MoreInfo.aspx?CategoryNum=2002003008", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002003/002002003009/MoreInfo.aspx?CategoryNum=2002003009", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002003/002002003010/MoreInfo.aspx?CategoryNum=268961288", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002004/002002004001/MoreInfo.aspx?CategoryNum=268961793", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002004/002002004002/MoreInfo.aspx?CategoryNum=268961794", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002004/002002004004/MoreInfo.aspx?CategoryNum=268961796", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002004/002002004006/MoreInfo.aspx?CategoryNum=268961797", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002004/002002004006/MoreInfo.aspx?CategoryNum=268961798", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002004/002002004007/MoreInfo.aspx?CategoryNum=268961799", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002004/002002004008/MoreInfo.aspx?CategoryNum=2002004008", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002004/002002004009/MoreInfo.aspx?CategoryNum=2002004009", "http://ggzyjy.zibo.gov.cn/TPFront/jyxx/002002/002002004/002002004010/MoreInfo.aspx?CategoryNum=268961800" ] i = 0 for w1 in ggtype1.keys(): for w2 in sx.keys(): href = hrefs[i] i += 1 tmp = [ "gcjs_%s_diqu%s_gg" % (w1, sx[w2]), href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"diqu": w2}), f2 ] data.append(tmp) ggtype2 = OrderedDict([("zhaobiao", "001"), ("biangeng", "002"), ("zhongbiaohx", "003"), ("yucai", "004")]) sx2 = OrderedDict([("市本级", "001"), ("张店区", "002"), ("高新区", "003"), ("文昌湖区", "004"), ("沂源县", "005"), ("博山区", "006"), ("高青县", "007"), ("临淄区", "008"), ("桓台县", "009")]) for w1 in ggtype2.keys(): for w2 in sx.keys(): href = hrefs[i] i += 1 tmp = [ "zfcg_%s_diqu%s_gg" % (w1, sx[w2]), href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"diqu": w2}), f2 ] data.append(tmp) data1 = data.copy() arr = [ "gcjs_zhaobiao_diqu008_gg", "gcjs_biangeng_diqu008_gg", "gcjs_zhongbiaohx_diqu005_gg", "gcjs_zhongbiaohx_diqu007_gg", "gcjs_zhongbiaohx_diqu008_gg", "zfcg_zhongbiaohx_diqu002_gg" ] for w in data: if w[0] in arr: data1.remove(w) return data1
def get_data(): data = [] ggtype1 = OrderedDict([("zhaobiao", "001"), ("biangeng", "002"), ("zhongbiao", "003"), ("yucai", "004")]) ggtype2 = OrderedDict([("zhaobiao", "001"), ("biangeng", "002"), ("zhongbiao", "003"), ("liubiao", "004"), ("yucai", "006")]) ggtype3 = OrderedDict([("zhaobiao", "001"), ("biangeng", "002"), ("zhongbiao", "003")]) ggtype4 = OrderedDict([("zhaobiao", "001"), ("biangeng", "002"), ("zhongbiao", "003"), ("liubiao", "004"), ("yucai", "005")]) gctype = OrderedDict([("勘察设计", "001"), ("施工", "002"), ("监理", "003"), ("专业工程", "004")]) zbfs = OrderedDict([("公开招标", "001"), ("邀请招标", "002"), ("竞争性磋商", "003"), ("竞争性谈判", "004"), ("询价", "005"), ("单一来源公示", "006")]) for w1 in ggtype1.keys(): for w2 in gctype.keys(): p1 = "079001%s" % (ggtype1[w1]) p2 = "079001%s%s" % (ggtype1[w1], gctype[w2]) href = "http://ggzy.linqing.gov.cn/lqweb/jyxx/079001/%s/%s" % (p1, p2) tmp = [ "gcjs_%s_gctype%s_gg" % (w1, gctype[w2]), href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"gctype": w2}), f2 ] data.append(tmp) for w1 in ggtype2.keys(): for w2 in zbfs.keys(): p1 = "079002%s" % (ggtype2[w1]) p2 = "079002%s%s" % (ggtype2[w1], zbfs[w2]) href = "http://ggzy.linqing.gov.cn/lqweb/jyxx/079002/%s/%s" % (p1, p2) tmp = [ "zfcg_%s_zbfs%s_gg" % (w1, zbfs[w2]), href, ["name", "ggstart_time", "href", "info"], add_info(f1, {"zbfs": w2}), f2 ] data.append(tmp) for w1 in ggtype3.keys(): p1 = "079005%s" % (ggtype3[w1]) href = "http://ggzy.linqing.gov.cn/lqweb/jyxx/079005/%s" % p1 tmp = [ "yiliao_%s_gg" % (w1), href, ["name", "ggstart_time", "href", "info"], f1, f2 ] data.append(tmp) for w1 in ggtype4.keys(): p1 = "079006%s" % (ggtype4[w1]) href = "http://ggzy.linqing.gov.cn/lqweb/jyxx/079006/%s" % p1 tmp = [ "qsydw_%s_gg" % (w1), href, ["name", "ggstart_time", "href", "info"], f1, f2 ] data.append(tmp) remove_arr = [ "gcjs_biangeng_gctype001_gg", "gcjs_biangeng_gctype004_gg", "gcjs_yucai_gctype004_gg", "gcjs_yucai_gctype003_gg", "zfcg_zhaobiao_zbfs002_gg", "zfcg_biangeng_zbfs002_gg", "zfcg_biangeng_zbfs006_gg", "zfcg_liaobiao_zbfs006_gg", "zfcg_liubiao_zbfs002_gg", "zfcg_liubiao_zbfs005_gg", "zfcg_liubiao_zbfs006_gg", "zfcg_yucai_zbfs003_gg", "zfcg_yucai_zbfs004_gg" ] data1 = data.copy() for w in data: if w[0] in remove_arr: data1.remove(w) return data1