Python get_crawl_interal示例

示例#1

import requests

from config import headers
from logger import crawler
from login import get_cookies
from db.dao import LoginInfoOper
from utils import (send_email, getip)
from db.redis_db import (Urls, Cookies)
from page_parse import (is_403, is_404, is_complete)
from decorators import (timeout_decorator, timeout)
from config import (get_timeout, get_crawl_interal, get_excp_interal,
                    get_max_retries)

TIME_OUT = get_timeout()
INTERAL = get_crawl_interal()
MAX_RETRIES = get_max_retries()
EXCP_INTERAL = get_excp_interal()
COOKIES = get_cookies()

# Instead of disable warning, why not use it as docs suggested
# https://stackoverflow.com/questions/42982143/python-requests-how-to-use-system-ca-certificates-debian-ubuntu
os.environ['REQUESTS_CA_BUNDLE'] = os.path.join(os.sep, '/etc/ssl/certs',
                                                'ca-certificates.crt')


def is_banned(url):
    if 'unfreeze' in url or 'accessdeny' in url or 'userblock' in url or 'verifybmobile' in url:
        return True
    return False

示例#2

文件： basic.py 项目： ResolveWang/WeiboSpider

from logger import crawler
from login import get_cookies
from db.dao import LoginInfoOper
from utils import (send_email, getip)
from db.redis_db import (
    Urls, Cookies)
from page_parse import (
    is_403, is_404, is_complete)
from decorators import (
    timeout_decorator, timeout)
from config import (
    get_timeout, get_crawl_interal, get_excp_interal, get_max_retries)


TIME_OUT = get_timeout()
INTERAL = get_crawl_interal()
MAX_RETRIES = get_max_retries()
EXCP_INTERAL = get_excp_interal()
COOKIES = get_cookies()


# Disable annoying InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)


def is_banned(url):
    if 'unfreeze' in url or 'accessdeny' in url or 'userblock' in url or 'verifybmobile' in url:
        return True
    return False