def getInfo3(url):
    # http://club.xywy.com/familyDoctor/pay/43983196?info=1&page=2#name2 对应的用户评价具体内容
    try:
        # 当第一次访问页面时,除了获取评论信息,也要获取全部的评论页的总数
        html = NetworkIO().requestHtml(url)
        if html is not None:
            evaluateBlock = html.findall(
                './/div[@class="User_eval lh180 btn-a f14 fwei mt10"]')
            for index, block in enumerate(evaluateBlock):
                uName = block.findtext('.//span[@class="mr10 fl"]').strip()
                evalAtti = block.findtext(
                    './/span[@class="fl colbd mr10"]').strip()
                evalScore = block.findtext(
                    './/span[@class="colClass01 fl"]').strip()
                evalText = block.findtext('.//div[@class="pt5"]').strip()
                evalTime = block.findtext(
                    './/span[@class="colbd f12 db pt10"]').strip()
                dbInfo = (url + '#' + str(index), uName, evalAtti, evalScore,
                          evalText,
                          datetime.strptime(evalTime, '%Y-%m-%d %H:%M:%S'))
                MySQL().saveDoctorEvaluationText(dbInfo)
            # 评价共有多少页
            totalPageInfo = html.find(
                './/div[@class="mt20 HomeFen f14"]/span[@class="mr5"]')
            totalPageInfo = 1 if totalPageInfo is None else totalPageInfo.text.strip(
            )[1:-3]
            # 目前评价页的索引值
            tmpIndex = url.find('page=') + 5
            currentPageIndex = url[tmpIndex:-6]
            # 获取当前页以后的评论页的评论信息
            if int(currentPageIndex) < int(totalPageInfo):
                for pageIndex in range(
                        int(currentPageIndex) + 1,
                        int(totalPageInfo) + 1):
                    url = url[:int(tmpIndex)] + str(pageIndex) + '#name2'
                    html = NetworkIO().requestHtml(url)
                    if html is not None:
                        evaluateBlock = html.findall(
                            './/div[@class="User_eval lh180 btn-a f14 fwei mt10"]'
                        )
                        for index, block in enumerate(evaluateBlock):
                            uName = block.findtext(
                                './/span[@class="mr10 fl"]').strip()
                            evalAtti = block.findtext(
                                './/span[@class="fl colbd mr10"]').strip()
                            evalScore = block.findtext(
                                './/span[@class="colClass01 fl"]').strip()
                            evalText = block.findtext(
                                './/div[@class="pt5"]').strip()
                            evalTime = block.findtext(
                                './/span[@class="colbd f12 db pt10"]').strip()
                            dbInfo = (url + '#' + str(index), uName, evalAtti,
                                      evalScore, evalText,
                                      datetime.strptime(
                                          evalTime, '%Y-%m-%d %H:%M:%S'))
                            MySQL().saveDoctorEvaluationText(dbInfo)
    except:
        doExpt('url3', url, 'url3')
def getQInfo(url, elem):
    sectionBlock = elem.findall('./p[@class="pt10 pb10 lh180 znblue normal-a"]/a')
    # 一级科室
    keshi1 = getPureText(sectionBlock[2].text)
    # 二级科室
    keshi2 = None
    if len(sectionBlock) >= 4:
        keshi2 = getPureText(sectionBlock[3].text)
    qInfoBlock = elem.find('./div/div[@class="User_askcon clearfix pr"]')
    # 问题题目
    qTitle = getPureText(qInfoBlock.find('.//p[@class="fl dib fb"]').text)
    userInfoBlock = qInfoBlock.findall('./div[@class="f12 graydeep Userinfo clearfix pl29"]/span')
    # 提问者姓名
    uName = None
    # 提问者性别
    uSex = None
    # 提问者年龄
    uAge = None
    # 问题发表时间
    qDatetime = None
    if len(userInfoBlock) >= 7:
        uName = getPureText(userInfoBlock[0].text)
        uSex = getPureText(userInfoBlock[2].text)
        uAge = getPureText(userInfoBlock[4].text)
        qDatetime = getPureText(userInfoBlock[6].text)
        qDatetime = (datetime.strptime(qDatetime, '%Y-%m-%d %H:%M:%S') if qDatetime is not None
                     else datetime.strptime('2000-01-01 00:00', '%Y-%m-%d %H:%M:%S'))
    # 问题内容
    qBodyBlock = qInfoBlock.find('./div/div[@id="qdetailc"]')
    qBody = ''
    for tmpText in qBodyBlock.itertext():
        subText = getPureText(tmpText)
        qBody = qBody + subText if subText is not None else ''
    MySQL().saveQInfo((url, qTitle, qBody, qDatetime, keshi1, keshi2, uName, uSex, uAge))
def getInfo2(url):
    # http://club.xywy.com/familyDoctor/pay/43983196?info=1&page=2#name2 对应页面总的用户评价相关信息
    try:
        html = NetworkIO().requestHtml(url)
        if html is not None:
            evaluateScore = html.findtext(
                './/h4[@class="f30 colClass01 fWei tc"]').strip()
            evaluateStat = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0}
            evaluateStatBlock = html.findall(
                './/div[@class="HomSptop_Ri fWei f14 mt20 fl"]/span')
            for index, item in enumerate(evaluateStatBlock):
                tmptext = item.text
                evaluateStat[index] = 0 if len(tmptext) == 0 else int(
                    tmptext[tmptext.find('(') + 1:tmptext.find(')')])
            dbInfo = (url, evaluateScore, evaluateStat[0], evaluateStat[1],
                      evaluateStat[2], evaluateStat[3], evaluateStat[4],
                      evaluateStat[5], evaluateStat[6], evaluateStat[7])
            MySQL().saveDoctorEvaluation(dbInfo)
    except:
        doExpt('url2', url, 'url2')
def getReply2Info(reply1Id, elems):
    reply2Index = 1
    for reply2InfoBlock in elems:
        # 二级回复中,是哪一方在回复:值为0表示提问者追问,值为1表示医生回复
        whoReply = reply2InfoBlock.find('.//span[@class="fl dib fb Doc_bla"]')
        whoReply = getPureText(whoReply.text if whoReply is not None else None)
        if whoReply is not None:
            if '追问' in whoReply:
                whoReply = 0
            else:
                whoReply = 1
        # 二级回复的内容
        reply2Body = reply2InfoBlock.find('.//*[@class="fl w390"]')
        reply2Body = getPureText(reply2Body.text)
        # 二级回复的时间
        reply2Datetime = reply2InfoBlock.find('./p[@class="tr col99 f12"]/span')
        reply2Datetime = getPureText(reply2Datetime.text)
        reply2Datetime = (datetime.strptime(reply2Datetime, '%Y-%m-%d %H:%M:%S') if reply2Datetime is not None
                          else datetime.strptime('2000-01-01 00:00', '%Y-%m-%d %H:%M:%S'))
        reply2Id = reply1Id + '_' + str(reply2Index)
        reply2Index += 1
        MySQL().saveReply2Info((reply2Id, reply2Body, whoReply, reply2Datetime))
def getReplyInfo(url, elem):
    # 医生回复是否被采纳:0--未采纳;1--采纳
    accepted = elem.find('./div[@class="t9999 questnew_icon Quest_askh2 pa"]')
    accepted = getPureText(accepted.text) if accepted is not None else None
    accepted = 1 if accepted == '最佳答案' else 0
    # reply1Block = None
    if accepted:
        reply1Block = elem.findall('./div[@class="docall clearfix Bestbg"]')
    else:
        reply1Block = elem.findall('./div[@class="docall clearfix "]')
    reply1Index = 1
    for block in reply1Block:
        # 回复医生的个人url
        doctorUrl = block.find('.//a[@class="f14 fb Doc_bla"]')
        doctorUrl = doctorUrl.get('href') if doctorUrl is not None else None
        # 医生回复的具体内容
        reply1Body = block.find('.//div[@class="pt15 f14 graydeep  pl20 pr20"]')
        reply1Body = getAllText(reply1Body) if reply1Body is not None else None
        # 回复的时间
        reply1Datetime = block.find('.//p[@class="col99 tr clearfix pr20"]/span')
        reply1Datetime = getPureText(reply1Datetime.text)
        reply1Datetime = (datetime.strptime(reply1Datetime, '%Y-%m-%d %H:%M:%S') if reply1Datetime is not None
                          else datetime.strptime('2000-01-01 00:00', '%Y-%m-%d %H:%M:%S'))
        if accepted:
            puIndex = block.find('.//div[@class="clearfix pb10  pl20 pr20 ballc"]//b[@class="gratenum"]')
        else:
            puIndex = block.find('.//div[@class="clearfix pb10  pl20 pr20 ballc pr"]//b[@class="gratenum"]')
        puIndex = getPureText(puIndex.text) if puIndex is not None else None
        reply1Id = url + '#' + str(reply1Index)
        reply1Index += 1
        MySQL().saveReply1Info((reply1Id, doctorUrl, reply1Body, reply1Datetime, puIndex, accepted))

        # 获取二级回复信息
        reply2InfoBlock = block.findall('.//div[@class="appdoc appxian ml20 mr20 mt15 pb10 clearfix f14"]'
                                        '/div[@class="usezw pt10 clearfix"]')
        getReply2Info(reply1Id, reply2InfoBlock)
        if rawText == '':
            rawText = None
    return rawText


def doExpt(password, tb, url, logIdentifier):
    if password is not None:
        UrlClient.saveUrl(password, tb, url)
    else:
        Redis().saveUrl(tb, url)  # 与redis在同一台主机上时
    FileIO.handleExpt(traceback.format_exc(), url, logIdentifier)


if __name__ == '__main__':
    tmpPwd = input('请输入用于认证的密码:').strip()
    tmpYear = input('请输入数据所在队列键名:').strip()
    MySQL().createTables()
    print('数据库中相应数据表已准备完成...')
    # threadList = []
    # for i in range(5):
    #     tmpThread = threading.Thread(target=getQPageInfo, args=(tmpYear, None if tmpPwd == '' else tmpPwd))
    #     threadList.append(tmpThread)
    # for tmpThread in threadList:
    #     tmpThread.start()
    # for tmpThread in threadList:
    #     tmpThread.join()
    jobs = []
    for i in range(5):
        jobs.append(gevent.spawn(getQPageInfo, tmpYear, (None if tmpPwd == '' else tmpPwd)))
    gevent.joinall(jobs)
def getInfo(url):
    # http://club.xywy.com/familyDoctor/pay/43983196 对应的页面信息
    try:
        html = NetworkIO().requestHtml(url)
        if html is not None:
            # 医生姓名
            doctorName = html.findtext('.//i[@class="fwei fl"]')
            doctorName = doctorName[:-6] if doctorName is not None and len(
                doctorName) > 6 else None
            # 医生职称和医院科室
            doctorRankAndHosp = html.find('.//div[@class=" lh200 pt10 f14"]')
            doctorRank = doctorRankAndHosp.text
            doctorHosp = doctorRankAndHosp.find('./br')
            # 获取医生的勋章
            medalsBlock = html.findall('.//div[@class="HomePth"]/span')
            medals = ''
            for medal in medalsBlock:
                medals += medal.get('data-th') + ','
            # 医生的寄语
            sendWord = html.find(
                './/div[@class="f12 graydeep club_home_icon HomePj"]/span'
            ).tail
            # 医生的服务类型
            serviceTypeBlock = {
                0: html.find('.//div[@class="fl pr"]'),
                1: None
            }
            if serviceTypeBlock[0] is None:
                serviceTypeBlock[1] = html.find('.//div[@class="fl f14"]')
            serviceTypes = {0: '', 1: ''}
            oldServiceTypes = {0: '', 1: ''}
            if serviceTypeBlock[0] is not None:
                serviceTypeBlock2 = serviceTypeBlock[0].findall('.//a[@cate]')
                for index, item in enumerate(serviceTypeBlock2):
                    for text in item.itertext():
                        serviceTypes[index] += text.strip()
            elif serviceTypeBlock[1] is not None:
                # 各服务原始价格
                serviceTypeBlock2 = serviceTypeBlock[1].findall('.//a[@cate]')
                for index, item in enumerate(serviceTypeBlock2):
                    for text in item.itertext():
                        serviceTypes[index] += text.strip()
                serviceTypeBlock2 = serviceTypeBlock[1].findall(
                    './/span[@class="f14 col99 ml10"]')
                for index, item in enumerate(serviceTypeBlock2):
                    for text in item.itertext():
                        oldServiceTypes[index] += text.strip()
            # 用户评分(放到用户评价界面抓取)
            # evaluateScore = html.findtext('.//span[@class="fl colClass01 fwei"]')
            # 签约家庭和帮助用户
            helpedInfo = {0: None, 1: None}
            helpedInfoBlock = html.findall('.//span[@class="fb f16 ml5"]')
            for index, item in enumerate(helpedInfoBlock):
                helpedInfo[index] = item.text
            # 擅长、简介以及荣誉
            infos = {0: '', 1: '', 2: ''}
            infoBlock = html.findall('.//div[@class="HomeJie f14 fwei pt20"]')
            for item in infoBlock:
                tmp = item.findtext('./h4')
                textblock = item.find('./div')
                tmptext = ''
                for text in textblock.itertext():
                    tmptext += text.strip()
                if '擅长' in tmp:
                    infos[0] = tmptext
                elif '简介' in tmp:
                    infos[1] = tmptext
                elif '荣誉' in tmp:
                    infos[2] = tmptext
            dbInfo = (url, doctorName, doctorRank, doctorHosp.tail, medals,
                      sendWord, serviceTypes[0], serviceTypes[1],
                      oldServiceTypes[0], oldServiceTypes[1], helpedInfo[0],
                      helpedInfo[1], infos[0], infos[1], infos[2])
            MySQL().saveDoctorInfo(dbInfo)
    except:
        doExpt('url1', url, 'url1')
def getInfo4(url):
    # http://club.xywy.com/familyDoctor/pay/43983196?info=2&page=2#name3 对应的服务购买信息
    try:
        html = NetworkIO().requestHtml(url)
        if html is not None:
            serviceBuyBlock = html.findall('.//div[@class="HomBone fwei f14"]')
            for index, block in enumerate(serviceBuyBlock):
                uName = block.findtext('.//span[@class="w100"]').strip()
                serviceType = 1 if '包月' in block.findtext(
                    './/span[@class="w200 tl"]').strip() else 0
                serviceCount = block.findtext(
                    './/span[@class="w60 tc"]').strip()
                servicePrice = block.findtext(
                    './/span[@class="colClass01 fb w80 tc"]').strip()
                serviceStatus = block.findtext(
                    './/span[@class="club_home_icon HomBsuc"]').strip()
                serviceTime = block.findtext(
                    './/span[@class="col99 ml20 tc"]').strip()
                dbInfo = (url + '#' + str(index), uName, serviceType,
                          serviceCount, servicePrice, serviceStatus,
                          serviceTime)
                MySQL().saveServiceInfo(dbInfo)
            # 评价共有多少页
            totalPageInfo = html.find(
                './/div[@class="mt20 HomeFen f14"]/span[@class="mr5"]')
            totalPageInfo = 1 if totalPageInfo is None else totalPageInfo.text.strip(
            )[1:-3]
            # 目前评价页的索引值
            tmpIndex = url.find('page=') + 5
            currentPageIndex = url[tmpIndex:-6]
            # 获取当前页以后的评论页的评论信息
            if int(currentPageIndex) < int(totalPageInfo):
                for pageIndex in range(
                        int(currentPageIndex) + 1,
                        int(totalPageInfo) + 1):
                    url = url[:int(tmpIndex)] + str(pageIndex) + '#name3'
                    html = NetworkIO().requestHtml(url)
                    if html is not None:
                        serviceBuyBlock = html.findall(
                            './/div[@class="HomBone fwei f14"]')
                        for index, block in enumerate(serviceBuyBlock):
                            uName = block.findtext(
                                './/span[@class="w100"]').strip()
                            serviceType = 1 if '包月' in block.findtext(
                                './/span[@class="w200 tl"]').strip() else 0
                            serviceCount = block.findtext(
                                './/span[@class="w60 tc"]').strip()
                            servicePrice = block.findtext(
                                './/span[@class="colClass01 fb w80 tc"]'
                            ).strip()
                            serviceStatus = block.findtext(
                                './/span[@class="club_home_icon HomBsuc"]'
                            ).strip()
                            serviceTime = block.findtext(
                                './/span[@class="col99 ml20 tc"]').strip()
                            dbInfo = (url + '#' + str(index), uName,
                                      serviceType, serviceCount, servicePrice,
                                      serviceStatus, serviceTime)
                            MySQL().saveServiceInfo(dbInfo)
    except:
        doExpt('url4', url, 'url4')