def __scheduleProjectInfo__(self, cycle=SPIDER_CYCLE):
"""
定时爬取公司项目信息
:param cycle:
:return:
"""
try:
conn = RedisClient()
spider = ProjectInfoSpider()
print('开始获取公司项目信息')
print('当前以获取项目ID量为：', conn.count(rediskey='ProjectInfoID'))
list_id = list(set(conn.all(rediskey='TempProjectListID')) - set(conn.all(rediskey='ProjectInfoID')))
spider.run(list_id)
except Exception as e:
print("Error Spider project list", e)
def __scheduleCompInfo__(self, cycle=SPIDER_CYCLE):
"""
定时爬取公司信息
:param cycle:
:return:
"""
try:
conn = RedisClient()
spider = CompanyInfoSpider()
print('开始获取公司信息')
print('当前以获取公司信息量为：', conn.count(rediskey='CompInfoID'))
list_id = list(set(conn.all(rediskey='TempCompInfoID')) - set(conn.all(rediskey='CompInfoID')))
# new_list = self.__divList__(list_id=list_id)
spider.run(list_id)
except Exception as e:
print("Error Spider comp info", e)
def __scheduleBuildLicencePerson__(self, cycle=SPIDER_CYCLE):
"""
定时爬取公司项目施工许可人员信息
:param cycle:
:return:
"""
try:
conn = RedisClient()
spider = BuildLicencePersonSpider()
print('开始获取公司项目施工许可人员信息')
print('当前以获取项目施工许可人员ID量为：', conn.count(rediskey='BuildLicencePersonID'))
list_id = list(
set(conn.all(rediskey='BuildLicenceInfoID')) - set(conn.all(rediskey='BuildLicencePersonID')))
spider.run(list_id)
except Exception as e:
print("Error Spider project list", e)
def __schedulePersonInfo__(self, cycle=SPIDER_CYCLE):
"""
定时爬取公司人员信息
:param cycle:
:return:
"""
try:
conn = RedisClient()
spider = PersonSpider()
print('开始获取公司人员信息')
print('当前以获取公司人员信息量为：', conn.count(rediskey='PersonInfoID'))
list_id = list(set(conn.all(rediskey='CompInfoID')) - set(conn.all(rediskey='PersonInfoID')))
# new_list = self.__divList__(list_id=list_id)
spider.run(list_id)
time.sleep(cycle)
except Exception as e:
print("Error Spider Staff info", e)