Esempio n. 1
0
 def qyzz(self):
     # 企业资质
     try:
         qy_zz = f'http://jzsc.mohurd.gov.cn/api/webApi/dataservice/query/comp/caDetailList?qyId={self.qyid1}&pg=0&pgsz=500'
         # qy_zz = f'http://jzsc.mohurd.gov.cn/api/webApi/dataservice/query/comp/caDetailList?qyId=0F0F0E060E0E0D0C0D0E0A0E0D0B09070B09&pg=0'
         resp2 = requests.post(url=qy_zz,
                               headers=self.headers,
                               proxies=self.ip,
                               timeout=10)
         if resp2.text.find('服务器繁忙,请稍后重试') != -1:
             print('服务器繁忙,请稍后重试')
         else:
             asddd4 = self.jd_nx(data=f'{resp2.text}')
             if asddd4['code'] != 200:
                 # self.hq_token(qyid=self.qyid1, name=self.z)  # 调用selenuim获得token值
                 Mysql.dele_token(token=self.jichu)
                 print('token删除成功')
                 Mysql.token(token=self.jichu)
                 self.jichu = self.jichutoken()
                 self.ip = {
                     "http": "http://" + self.jichutoken()[1],
                     "https": "https://" + self.jichutoken()[1]
                 }
             else:
                 if asddd4['data'] == None:
                     self.gx_qyid()
                 else:
                     a123 = asddd4['data']['pageList']['list']
                     if len(a123) != 0:
                         p1 = 0
                         for resp in a123:
                             p1 += 1
                             print(f'一共{len(a123)}个企业资质现在是第{p1}个企业资质')
                             qyxx.qyzz(resp=resp, qyid=self.qyid)
                         if p1 == len(a123):
                             Mysql.updatet_qyzzzt(qyzzzt='0', eid=self.qyid)
                             return '0'
                     else:
                         print('没有企业资质')
                         Mysql.updatet_qyzzzt(qyzzzt='404', eid=self.qyid)
                         return '404'
     except Exception as e:
         qq = str(e)
         if qq.find("HTTPConnectionPool") != -1:
             print('ip失效')
             Mysql.dele_token(token=self.jichu)
             print('token删除成功')
             Mysql.token(token=self.jichu)
             self.jichu = self.jichutoken()[0]
             self.ip = {
                 "http": "http://" + self.jichutoken()[1],
                 "https": "https://" + self.jichutoken()[1]
             }
         else:
             print('不存在')
             print(e, '资质信息错误')
Esempio n. 2
0
 def jichu12(self):
     # 基础信息
     try:
         qy_jichu = f'http://jzsc.mohurd.gov.cn/api/webApi/dataservice/query/comp/compDetail?compId={self.qyid1}'
         resp1 = requests.get(url=qy_jichu,
                              headers=self.headers,
                              proxies=self.ip,
                              timeout=100)
         if resp1.text.find('服务器繁忙,请稍后重试') != -1:
             print('服务器繁忙,请稍后重试')
         else:
             asddd2 = self.jd_nx(data=f'{resp1.text}')
             if asddd2['code'] != 200:
                 # self.hq_token(qyid=self.qyid1, name=self.z)#调用selenuim获得token值
                 Mysql.dele_token(token=self.jichu)
                 print('token删除成功')
                 Mysql.token(token=self.jichu)
                 self.jichu = self.jichutoken()[0]
                 self.ip = {
                     "http": "http://" + self.jichutoken()[1],
                     "https": "https://" + self.jichutoken()[1]
                 }
             else:
                 if asddd2['data'] == None:
                     self.gx_qyid()
                 else:
                     qyxx.qyjichu(asddd2['data']['compMap'], qyid=self.qyid)
                     Mysql.update_qyjcxx(qy_jcxx_zt='0', eid=self.qyid)
                     return '0'
     except Exception as e:
         qq = str(e)
         if qq.find("HTTPConnectionPool") != -1:
             print('ip失效')
             Mysql.dele_token(token=self.jichu)
             print('token删除成功')
             Mysql.token(token=self.jichu)
             self.jichu = self.jichutoken()[0]
             self.ip = {
                 "http": "http://" + self.jichutoken()[1],
                 "https": "https://" + self.jichutoken()[1]
             }
         else:
             print('不存在')
             print(e, '基础信息错误')
Esempio n. 3
0
 def gx_qyid(self):
     try:
         print('开始更新企业id')
         qyurl = f'http://jzsc.mohurd.gov.cn/api/webApi/dataservice/query/comp/list?complexname={self.z}&pg=0&pgsz=15&total=0'
         resp1 = requests.get(url=qyurl,
                              headers=self.headers,
                              proxies=self.ipz(),
                              timeout=10)
         ew = resp1.text
         if str(ew).find('服务器繁忙,请稍后重试') != -1:
             print('服务器繁忙,请稍后重试')
         else:
             asddd2 = self.jd_nx(data=f'{resp1.text}')
             if len(asddd2['data']['list']) == 0:
                 print('没有这个公司异常')
                 Mysql.gxqy_fupa(cx_state='3', eid=self.qyid)
             else:
                 qyid = asddd2['data']['list'][0]['QY_ID']
                 Mysql.update_qyid(qyurl=qyid, eid=self.qyid)  #更新企业id
                 print('企业更新完毕')
                 return qyid
     except Exception as e:
         qq = str(e)
         if qq.find("HTTPConnectionPool") != -1:
             print('ip失效')
             Mysql.dele_token(token=self.jichu)
             print('token删除成功')
             Mysql.token(token=self.jichu)
             self.jichu = self.jichutoken()[0]
             self.ip = {
                 "http": "http://" + self.jichutoken()[1],
                 "https": "https://" + self.jichutoken()[1]
             }
         else:
             print('不存在')
             print(e, '基础信息错误')
Esempio n. 4
0
 def qy_user(self):
     # 企业人员信息的爬取
     try:
         ry_url = f'http://jzsc.mohurd.gov.cn/api/webApi/dataservice/query/comp/regStaffList?qyId={self.qyid1}&pg=0&pgsz=5000'
         # print(ry_url)
         resp2 = requests.get(url=ry_url,
                              headers=self.headers,
                              proxies=self.ip,
                              timeout=10)
         if resp2.text.find('服务器繁忙,请稍后重试') != -1:
             print('服务器繁忙,请稍后重试')
         else:
             asd4 = self.jd_nx(data=f'{resp2.text}')
             if asd4['data'] == None:
                 self.gx_qyid()
             else:
                 ry_urlte = asd4['data']['pageList']['list']
                 if len(ry_urlte) == 0:
                     print('该公司没有人员信息')
                     Mysql.updateryzt(ryzt='404', eid=self.qyid)
                     return '404'
                 else:
                     # asd=Mysql.selecttbl_qyname(eid=self.qyid)
                     # Mysql.delete_tbl_user(qyid=self.qyid)  # 删除人员的基础信息
                     # Mysql.deletetbl_user_zcxx1(qyid=self.z)
                     a1 = Mysql.selectryurl_ys1(eid=self.qyid)
                     if a1 == None:
                         a2 = 0
                         po = 1
                         Mysql.delete_tbl_user(qyid=self.qyid)
                         Mysql.deletetbl_user_zcxx1(qyid=self.qyid)
                         print('删除成功')
                     else:
                         a2 = int(a1) - 1
                         po = int(a1)
                     # print(ry_urlte[int(a2)-1:])
                     for res in ry_urlte[a2:]:
                         print(
                             f'--------------------------一共{len(ry_urlte)}个人++++第{po}个人--------------------------------'
                         )
                         userid = uuid4()
                         user = res['RY_ID']
                         username = res['RY_NAME']
                         ry_xinxi = f'http://jzsc.mohurd.gov.cn/api/webApi/dataservice/query/staff/staffDetail?staffId={user}'
                         resp3 = requests.get(url=ry_xinxi,
                                              headers=self.headers,
                                              proxies=self.ip,
                                              timeout=10)
                         if resp3.text.find('服务器繁忙,请稍后重试') != -1:
                             print('服务器繁忙,请稍后重试')
                         else:
                             asd6 = self.jd_nx(data=f'{resp3.text}')
                             print(asd6)
                             if asd6['code'] != 200:
                                 print(asd6)
                                 Mysql.dele_token(token=self.jichu)
                                 print('token删除成功')
                                 Mysql.token(token=self.jichu)
                                 self.jichu = self.jichutoken()[0]
                                 self.ip = {
                                     "http":
                                     "http://" + self.jichutoken()[1],
                                     "https":
                                     "https://" + self.jichutoken()[1]
                                 }
                                 break
                             else:
                                 qyxx.ryxx(resp=asd6['data']['staffMap'],
                                           qyid=self.qyid,
                                           user=userid)
                                 asd7 = asd6['data']['regCertList']
                                 x = 1
                                 for res in asd7:
                                     print(
                                         f'{username}一共有{len(asd7)}个注册信息正在爬取第{x}个注册信息'
                                     )
                                     qyxx.ryxx_xinxi(resp=res,
                                                     user=userid,
                                                     zc_dwid=self.qyid)
                                     x += 1
                                 if len(asd7) == x - 1:
                                     Mysql.updatery_page_zd(
                                         ry_page_zd=po,
                                         eid=self.qyid)  # 实时更新爬取的页数
                                     po += 1
                                 else:
                                     pass
                                 if po - 1 == len(ry_urlte):
                                     Mysql.updateryzt(
                                         ryzt='0', eid=self.qyid)  #更新爬取状态
                                     return '0'
                                 else:
                                     pass
     except Exception as e:
         qq = str(e)
         if qq.find("HTTPConnectionPool") != -1:
             print('ip失效')
             Mysql.dele_token(token=self.jichu)
             print('token删除成功')
             Mysql.token(token=self.jichu)
             self.jichu = self.jichutoken()[0]
             self.ip = {
                 "http": "http://" + self.jichutoken()[1],
                 "https": "https://" + self.jichutoken()[1]
             }
         else:
             print('不存在')
             print(e, '人员信息错误')