コード例 #1
0
ファイル: client.py プロジェクト: stone02111/company-crawler
    def search(keyword: str) -> list:
        results = list()
        if keyword:
            payload = {
                "searchKey": keyword,
                "token": TOKEN,
                "pageIndex": 1,  # 每个关键字默认获取第一页数据共20条
                "searchType": 0,
                "isSortAsc": False
            }
            http_result = httpclient.get(url=QICHACHA.SEARCH_API,
                                         params=payload,
                                         headers=REQUEST_HEADERS)
            sleep(2)

            if http_result.status_code == 200:
                api_result = json.loads(http_result.text)
                if api_result.get('status') == 200:
                    try:
                        results = http_result.json().get('result', dict()).get(
                            'Result', dict())
                    except JSONDecodeError:
                        pass
                else:
                    log.info(str(api_result))

        return results
コード例 #2
0
    def search(keyword: str):
        if not keyword:
            return None

        payload = {
            "searchKey": keyword,
            "token": TOKEN,
            "pageIndex": 1,  # 每个关键字默认获取第一页数据共20条
            "searchType": 0,
            "isSortAsc": False
        }
        httpret = httpclient.get(url=QCC_SEARCH_API,
                                 params=payload,
                                 headers=REQUEST_HEADERS)
        sleep(2)

        message, code = httpret.reason, httpret.status_code

        if code != 200:
            log.warning('http error, %s-%s' % (code, message))
            return None

        try:
            apiret = httpret.json()
        except JSONDecodeError:
            log.error('解析httpret失败')
            return None

        return apiret.get('result')
コード例 #3
0
ファイル: client.py プロジェクト: stone02111/company-crawler
 def search(cls, keyword: str) -> list:
     """
     根据关键字搜索相关企业信息
     :param keyword: 关键字
     :return:
     """
     payload = {"pageNum": 1, "pageSize": 20, "sortType": 0}
     url = TIANYANCHA.SEARCH_API + "/" + url_encoder.quote(keyword)
     http_result = httpclient.get(url=url,
                                  params=payload,
                                  headers=REQUEST_HEADERS)
     time.sleep(2)
     if http_result.status_code == 200:
         try:
             api_result = http_result.json()  # api响应数据
             if api_result.get('state') == 'ok':
                 return api_result.get('data',
                                       dict()).get('companyList', list())
             else:
                 log.info(str(api_result))
         except JSONDecodeError as error:
             pass
     else:
         log.info(str(http_result.text))
     return list()
コード例 #4
0
ファイル: crawler.py プロジェクト: yinluobing/dev.crawler
    def search_detail(company_id: int):
        url = DETAIL_API + "/" + str(company_id)
        http_result = httpclient.get(url=url,
                                     params=None,
                                     headers=REQUEST_HEADERS)
        time.sleep(2)

        ok, message, code = http_result.ok, http_result.reason, http_result.status_code
        if not ok or code != 200:
            log.error('%s-%s-%s' % (time.strftime(
                '%Y-%m-%d %H:%M:%S', time.localtime()), code, message))
        try:
            api_result = http_result.json()  # api响应数据
        except RuntimeError as error:
            log.error('unboxing error, error: %s' % error)
            return None

        api_message, api_state = api_result.get('message'), api_result.get(
            'state')
        if api_state != 'ok':
            log.error('[tyc]api error, %s-%s' % (api_state, api_message))
            return None

        company_detail = api_result.get('data')
        return company_detail
コード例 #5
0
ファイル: crawler.py プロジェクト: yinluobing/dev.crawler
    def search(key: str):
        """
        根据关键字搜索相关企业信息
        :param key: 关键字
        :return:
        """
        payload = {"pageNum": 1, "pageSize": 20, "sortType": 0}
        url = SEARCH_API + "/" + url_encoder.quote(key)
        http_result = httpclient.get(url=url,
                                     params=payload,
                                     headers=REQUEST_HEADERS)
        time.sleep(2)

        ok, message, code = http_result.ok, http_result.reason, http_result.status_code
        if not ok or code != 200:
            log.error('%s-%s-%s' % (time.strftime(
                '%Y-%m-%d %H:%M:%S', time.localtime()), code, message))
            return None

        try:
            api_result = http_result.json()  # api响应数据
        except RuntimeError as error:
            log.error('unboxing error, error: %s' % error)
            return None

        api_message, api_state = api_result.get('message'), api_result.get(
            'state')
        if api_state != 'ok':
            log.error('[tyc]api error, %s-%s' % (api_state, api_message))
            return None

        companies = api_result.get('data').get(
            'companyList')  # 搜索公司结果json array
        return companies
コード例 #6
0
ファイル: client.py プロジェクト: stone02111/company-crawler
    def search_detail(key_no):
        detail = dict()
        if key_no:
            payload = {"token": TOKEN, "unique": key_no}
            http_result = httpclient.get(url=QICHACHA.COMPANY_DETAIL_API,
                                         params=payload,
                                         headers=REQUEST_HEADERS)
            sleep(2)

            if http_result.status_code == 200:
                api_result = json.loads(http_result.text)
                if api_result.get('status') == 200:
                    try:
                        detail = http_result.json().get('result', dict()).get(
                            'Company', dict())
                    except JSONDecodeError:
                        pass
                else:
                    log.info(str(api_result))
        return detail
コード例 #7
0
ファイル: client.py プロジェクト: stone02111/company-crawler
 def search_detail(cls, company_id: int):
     """
     根据公司ID查询公司信息详情
     :param company_id:
     :return: 公司详情json结果
     """
     url = TIANYANCHA.DETAIL_API + "/" + str(company_id)
     http_result = httpclient.get(url=url,
                                  params=None,
                                  headers=REQUEST_HEADERS)
     time.sleep(2)
     if http_result.status_code == 200:
         try:
             api_result = http_result.json()  # api响应数据
             if api_result.get('state') == 'ok':
                 return api_result.get('data', dict())
             else:
                 log.info(str(api_result))
         except JSONDecodeError as error:
             pass
     else:
         log.info(str(http_result.text))
     return dict()
コード例 #8
0
    def search_detail(key_no):
        if not key_no:
            log.info('company id null')
            return None
        payload = {"token": TOKEN, "unique": key_no}

        httpresult = httpclient.get(url=QCC_SEARCH_DETAIL_API,
                                    params=payload,
                                    headers=REQUEST_HEADERS)
        sleep(2)

        message, code = httpresult.reason, httpresult.status_code

        if code != 200:
            log.warning('http error, %s-%s' % (code, message))
            return None

        try:
            apiret = httpresult.json()
        except JSONDecodeError:
            log.error('解析httpret失败')
            return None

        return apiret.get('result')