def __scheduleProjectInfo__(self, cycle=SPIDER_CYCLE):
"""
定时爬取公司项目信息
:param cycle:
:return:
"""
try:
conn = RedisClient()
spider = ProjectInfoSpider()
print('开始获取公司项目信息')
print('当前以获取项目ID量为：', conn.count(rediskey='ProjectInfoID'))
list_id = list(set(conn.all(rediskey='TempProjectListID')) - set(conn.all(rediskey='ProjectInfoID')))
spider.run(list_id)
except Exception as e:
print("Error Spider project list", e)
def __scheduleCompInfo__(self, cycle=SPIDER_CYCLE):
"""
定时爬取公司信息
:param cycle:
:return:
"""
try:
conn = RedisClient()
spider = CompanyInfoSpider()
print('开始获取公司信息')
print('当前以获取公司信息量为：', conn.count(rediskey='CompInfoID'))
list_id = list(set(conn.all(rediskey='TempCompInfoID')) - set(conn.all(rediskey='CompInfoID')))
# new_list = self.__divList__(list_id=list_id)
spider.run(list_id)
except Exception as e:
print("Error Spider comp info", e)
def __scheduleBuildLicencePerson__(self, cycle=SPIDER_CYCLE):
"""
定时爬取公司项目施工许可人员信息
:param cycle:
:return:
"""
try:
conn = RedisClient()
spider = BuildLicencePersonSpider()
print('开始获取公司项目施工许可人员信息')
print('当前以获取项目施工许可人员ID量为：', conn.count(rediskey='BuildLicencePersonID'))
list_id = list(
set(conn.all(rediskey='BuildLicenceInfoID')) - set(conn.all(rediskey='BuildLicencePersonID')))
spider.run(list_id)
except Exception as e:
print("Error Spider project list", e)
def __schedulePersonInfo__(self, cycle=SPIDER_CYCLE):
"""
定时爬取公司人员信息
:param cycle:
:return:
"""
try:
conn = RedisClient()
spider = PersonSpider()
print('开始获取公司人员信息')
print('当前以获取公司人员信息量为：', conn.count(rediskey='PersonInfoID'))
list_id = list(set(conn.all(rediskey='CompInfoID')) - set(conn.all(rediskey='PersonInfoID')))
# new_list = self.__divList__(list_id=list_id)
spider.run(list_id)
time.sleep(cycle)
except Exception as e:
print("Error Spider Staff info", e)
def __scheduleProjectCorpInfo__(self):
try:
conn = RedisClient()
spider = ProjectCorpInfo()
print('开始获取项目')
key = 'TempProjectCorpInfoID'
list_id = list(set(conn.all(rediskey=key)))
spider.run(list_id, main_url=[PROJECTCORPINFO], key=key)
except Exception as e:
print("Error Spider project list", e)
def __scheduleProjectFinishInfo__(self, cycle=SPIDER_CYCLE):
"""
定时爬取公司项目竣工信息
:param cycle:
:return:
"""
try:
conn = RedisClient()
print('开始获取公司项目竣工信息')
spider = ProjectFinishListSpider()
key = 'TempProFinishListID'
list_id = list(set(conn.all(rediskey=key)))
spider.run(list_id, main_url=[FINISHMANAGE], key=key)
except Exception as e:
print("Error Spider project list", e)
def __scheduleBuildLicenceInfo__(self, cycle=SPIDER_CYCLE):
"""
定时爬取公司项目施工许可信息
:param cycle:
:return:
"""
try:
conn = RedisClient()
print('开始获取公司项目施工许可信息')
spider = BuildLicenceListSpider()
key = 'TempBuildLicenceListID'
list_id = list(set(conn.all(rediskey=key)))
spider.run(list_id, main_url=[LICENCEMANAGE], key=key)
except Exception as e:
print("Error Spider project list", e)
def __scheduleContractRecordInfo__(self, cycle=SPIDER_CYCLE):
"""
定时爬取公司项目合同登记信息
:param cycle:
:return:
"""
try:
conn = RedisClient()
print('开始获取公司项目合同登记信息')
spider = ContractRecordListSpider()
key = 'TempContractListID'
list_id = list(set(conn.all(rediskey=key)))
spider.run(list_id, main_url=[CONTRACTRECORD], key=key)
except Exception as e:
print("Error Spider project list", e)
def __scheduleTenderInfo__(self, cycle=SPIDER_CYCLE):
"""
定时爬取项目招标信息
:param cycle:
:return:
"""
try:
conn = RedisClient()
print('开始获取公司项目招标信息')
spider = TenderListSpider()
key = 'TempTenderListID'
list_id = list(set(conn.all(rediskey=key)))
spider.run(list_id, main_url=[TENDERAPI], key=key)
except Exception as e:
print("Error Spider project list", e)
def __scheduleWithinProjectList__(self, cycle=SPIDER_CYCLE, main_url=None, TempList=None):
"""
获取项目内的各个类型list
:param cycle:
:param main_url:
:param TempList:
:return:
"""
try:
conn = RedisClient()
key = 'Temp' + TempList
list_id = list(set(conn.all(rediskey=key)))
spider = SaveWithinProjectSpider()
spider.run(list_id, main_url, TempList)
except Exception as e:
print("Error Spider project list", e)