def image_open(url, timeout=30): session = requests.Session() session.max_redirects = 2 resp = Response() resp.url = url try: headers = {'User-Agent': _DEFAULT_UA, 'Accept': _DEFAULT_ACCEPT} headers['Connection'] = 'close' r = session.head(url=url, headers=headers, timeout=timeout) content_type = r.headers.get('content-type', '') r = session.get(url=url, headers=headers, timeout=timeout) resp.status_code = r.status_code resp.url = r.url for chunk in r.iter_content(chunk_size=40960): resp.content += chunk except requests.exceptions.ConnectionError as e: resp.status_code = Response.CONNECTION_ERROR resp.explain = str(e) except requests.exceptions.ProxyError as e: resp.status_code = Response.PROXY_ERROR resp.explain = str(e) except requests.exceptions.SSLError as e: resp.status_code = Response.SSL_ERROR resp.explain = str(e) except requests.exceptions.Timeout as e: resp.status_code = Response.TIMEOUT resp.explain = str(e) except Exception as e: resp.status_code = Response.ERROR resp.explain = str(e) return resp
def url_open(url, use_proxy=0, timeout=30): session = requests.Session() session.max_redirects = 2 resp = Response() resp.url = url try: headers = {'User-Agent': _DEFAULT_UA} headers['Accept'] = 'text/html' if use_proxy: mp = get_mayi_proxy(use_proxy) headers['Connection'] = 'close' headers['Proxy-Authorization'] = mp.header r = session.get(url=url, proxies=mp.proxies, headers=headers, timeout=timeout) else: headers['Connection'] = 'close' r = session.head(url=url, headers=headers, timeout=timeout) content_type = r.headers.get('content-type', '') if content_type and content_type.find('text/html') == -1: resp.status_code = Response.NOT_HTML return resp r = session.get(url=url, headers=headers, timeout=timeout) resp.status_code = r.status_code resp.url = r.url for chunk in r.iter_content(chunk_size=40960): resp.content += chunk except requests.exceptions.ConnectionError as e: resp.status_code = Response.CONNECTION_ERROR resp.explain = str(e) except requests.exceptions.ProxyError as e: resp.status_code = Response.PROXY_ERROR resp.explain = str(e) except requests.exceptions.SSLError as e: resp.status_code = Response.SSL_ERROR resp.explain = str(e) except requests.exceptions.Timeout as e: resp.status_code = Response.TIMEOUT resp.explain = str(e) except Exception as e: resp.status_code = Response.ERROR resp.explain = str(e) return resp