예제 #1
0
파일: agent.py 프로젝트: strongant/gsorglz
def checkAllLz(filePath, taskResultId):

    taskResult = TaskResult.getOne(TaskResult.taskResultId == taskResultId)
    delayDay = taskResult.taskId.intervalDay
    #根据当前任务编号目录获取该目录下的所有xml文件
    print '目录路径:',filePath
    fnames=os.listdir(filePath)
    for name in fnames:
        xmlName = filePath+'/'+name
        print 'xmlName:',xmlName
        data = parseXMLFile(xmlName, 'CheckItem')
        print 'data:',data
        # 将需要检查的信息入库
        for d in data:
            cname = d.get('cname')
            url = d.get('url')
            if url != '':
                if url[-1] == '/':
                    url = url.replace('http://', '')[0:-1].replace(' ','')
            if cname!='':
                cname = cname.replace(' ','')
            webArea = d.get('area')
            webtype = d.get('WebType')
            # 检查更新company
            if Company.getOne(Company.coname == cname) is None:
                c = Company()
                c.coname = cname
                c.save(force_insert=True)
            # 检查更新website
            if Website.getOne(Website.domain == url) is not None:
                q = Website.update(domain=url, type=webtype, area=webArea).where(Website.domain == url)
                q.execute()
            else:
                com = Company.getOne(Company.coname == cname)
                w = Website()
                w.regID = com
                w.domain = url
                w.area = webArea
                w.type = webtype
                w.save(force_insert=True)
            updateWeb = Website.getOne(Website.domain == url)
            subTask = TaskInfo()
            subTask.taskResultId = taskResult
            subTask.webId = updateWeb
            subTask.state = '1'
            subTask.save(force_insert=True)

    taskResultId = str(taskResultId)
    if delayDay > 0:
        # 需要周期执行的任务
        executeMultiTaskInfo(taskResultId)
    else:
        #logger.debug("开始调用单次任务")
        # 单次执行的任务
        executeSingleTaskInfo(taskResultId)
예제 #2
0
파일: agent.py 프로젝트: strongant/gsorglz
def isAlreadyExistsWeb(companyName, currentTime, expired, shortUrl, subTaskId, url, isExistsWebId):
    subTask = TaskInfo.getOne(TaskInfo.id == subTaskId)
    taskResult = subTask.taskResultId
    bigTask = taskResult.taskId
    bigTaskId = bigTask.taskId
    # 获取当前网站上一次的更新时间
    websiteResult = Website.getOne(Website.webId == isExistsWebId)
    webUpdateTime = websiteResult.updateDate
    if webUpdateTime is None:
        diffDay = expired + 1
    else:
        diffDay = (currentTime - webUpdateTime).days
    if diffDay > expired:
        print '时间已经过期'
        # 网站信息过期,需要重新抓取并检测
        fetchWebsite(companyName, bigTaskId, subTaskId, url)
    else:
        # 判断是否有当前网站信息是否存在亮照编号
        if websiteResult.licID != '':
            #logger.debug('已经亮照:', shortUrl)
            q = TaskInfo.update(state='2').where(TaskInfo.id == subTaskId)
            q.execute()
        else:
            #logger.debug('未亮照:', shortUrl)
            q = TaskInfo.update(state='3').where(TaskInfo.id == subTaskId)
            q.execute()
예제 #3
0
파일: agent.py 프로젝트: strongant/gsorglz
def checkLz(subtaskId):
    try:
        # 根据任务结果编号获取本次需要检查的任务记录
        subTask = TaskInfo.getOne(TaskInfo.id == subtaskId)
        checkTaskResult = subTask.taskResultId
        bigTaskId = checkTaskResult.taskId.taskId
        if subTask is not None:
            companyName = subTask.webId.regID.coname
            url = subTask.webId.domain
            if (validateUrl(url)):
                # 从任务表中获取是否已经进行检查过了(根据网站有没有更新时间进行判断)
                isExistsTask = TaskInfo.getOne(
                        TaskInfo.webId == Website.getOne((Website.domain == url) & (Website.updateDate.is_null(False))))
                subId = subTask.id
                # 数据库中无此task对应的网站记录
                if isExistsTask is None:
                    # 抓取检测
                    fetchWebsite(companyName, bigTaskId,subId, url)
                else:
                    isExistsWebId = isExistsTask.webId.webId
                    # 如果当前网站更新时间小于过期时间,说明不用重新进行抓取并检查
                    expired = Configs.getOne(Configs.type == 'update').expired
                    # 获取当前时间
                    currentTime = datetime.datetime.now()
                    isAlreadyExistsWeb(companyName, currentTime, expired, url, subId,
                                       url,
                                       isExistsWebId)
    except Exception:
        q = TaskInfo.update(state='-1')
        q.execute()
예제 #4
0
파일: MakeLz.py 프로젝트: strongant/gsorglz
def judgeLzResult(com, web, shortUrl, subTask):
    # 更新网站信息和公司信息
    impCompanyInfo(com)
    # impWebsite(web)
    # nWeb = Website.getOne(Website.domain == web.domain)

    # judgeWeb = Website.getOne(Website.domain ** str.format("%{}%", shortUrl))

    # 如果查询网址与抓取亮照后的网址不匹配
    if shortUrl != '':
        shortUrl = shortUrl.replace('http://', '').replace(' ', '')
    print  'shortUrl:', shortUrl
    print "web.domain:", web.domain
    com = Company.getOne(Company.coname == com.coname)
    if shortUrl != web.domain:
        # 当已经存在跳转关系记录时,不再操作
        existsJumpWeb = Website.getOne((Website.domain == web.domain))
        if existsJumpWeb is None:
            dt = format(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S')
            web.updateDate = dt
            web.regID = com
            web.save(force_insert=True)
            # 更新网站跳转地址
            q = Website.update(jump=web).where(Website.webId == subTask.webId.webId)
            q.execute()
        else:
            impWebsite(web)
            # 更新网站跳转地址
            q = Website.update(jump=existsJumpWeb).where(Website.webId == subTask.webId.webId)
            q.execute()
    else:
        #更新网站信息
        impWebsite(web)

    count = Website.select().where(
            (Website.licID != '') & (Website.regID == com) & (Website.domain == shortUrl)
    ).count()
    if count == 0:
        onlyCname = Website.select().where(
                (Website.licID != '') & (Website.regID == com)).count()
        onlyDomain = Website.select().where(
                (Website.licID != '') & (Website.domain == shortUrl)).count()
        if onlyCname > 0:
            q = TaskInfo.update(state='9').where(TaskInfo.id == subTask.id)
            q.execute()
        elif onlyDomain > 0:
            q = TaskInfo.update(state='8').where(TaskInfo.id == subTask.id)
            q.execute()
        else:
            q = TaskInfo.update(state='4').where(TaskInfo.id == subTask.id)
            q.execute()
    else:
        q = TaskInfo.update(state='2').where(TaskInfo.id == subTask.id)
        q.execute()