def elastic(path): proxy_request_headers = {"kbn-xsrf": "reporting"} requests_response = None response = Response() session = requests_session() session.headers["Connection"] = "close" auth = AWSAuth(config.aws_credentials, config.aws_region) query_string = urlencode(parse_qs(request.query_string.decode("utf-8"), keep_blank_values=True), quote_via=quote, doseq=True) endpoint = config.aws_endpoint + "/" + path + "?" + query_string for header, value in request.headers.items(): if header.lower() in PROXY_REQ_HEADERS_WHITELIST: proxy_request_headers[header] = value if request.method == "HEAD": session_response = session.head(endpoint, auth=auth, cookies=request.cookies, headers=proxy_request_headers) elif request.method == "GET": session_response = session.get(endpoint, auth=auth, cookies=request.cookies, headers=proxy_request_headers) elif request.method == "POST": session_response = session.post(endpoint, cookies=request.cookies, auth=auth, data=request.get_data(), headers=proxy_request_headers) elif request.method == "PUT": session_response = session.put(endpoint, cookies=request.cookies, auth=auth, data=request.get_data(), headers=proxy_request_headers) elif request.method == "DELETE": session_response = session.delete(endpoint, auth=auth, cookies=request.cookies, headers=proxy_request_headers) else: return "Method is not allowed!" if request.method != "HEAD": response.set_data(session_response.content) response.status_code = session_response.status_code for header, value in session_response.headers.items(): if header.lower() not in PROXY_RESP_HEADERS_BLACKLIST: response.headers[header] = value return response
def request(self, method, path, **kwargs): url = self.url if path: url = utils.urlparse.urljoin(self.url, path) if self._session is None: self._session = requests_session() more = dict(self._settings) more.update(kwargs) return utils.request(method, url, session=self._session, **more)
def request(self, method, path, data=None, headers=None, is_subpath=True): url = self.url if path: url = utils.urlparse.urljoin(self.url, path) assert url.startswith(self.url) or not is_subpath if self._session is None: self._session = requests_session() self._check_dav_header() return utils.request(method, url, data=data, headers=headers, session=self._session, **self._settings)
def run(self): self.check_browser() self.set_urls() self.client = requests_session() self.ping_server() self.server_configuration_check() sessions_to_create = [] session_config_name = self.session_config_name if session_config_name: if session_config_name not in SESSION_CONFIGS_DICT: raise ValueError( 'No session config named "{}"'.format(session_config_name) ) session_config_names = [session_config_name] else: # default to all session configs session_config_names = SESSION_CONFIGS_DICT.keys() self.max_name_length = max( len(config_name) for config_name in session_config_names ) for session_config_name in session_config_names: session_config = SESSION_CONFIGS_DICT[session_config_name] num_bot_cases = session_config.get_num_bot_cases() for case_number in range(num_bot_cases): num_participants = ( self.num_participants or session_config['num_demo_participants'] ) sessions_to_create.append( { 'session_config_name': session_config_name, 'num_participants': num_participants, 'case_number': case_number, } ) total_time_spent = 0 # run in a separate loop, because we want to validate upfront # that the session configs are valid, etc, # rather than the command failing halfway through for session_to_create in sessions_to_create: total_time_spent += self.run_session(**session_to_create) print('Total: {} seconds'.format(round(total_time_spent, 1)))
def _empty_ip_pools(self): ''' 清空ip池 :return: ''' if self.type == ip_proxy_pool: base_url = 'http://127.0.0.1:8000' with requests_session() as _s: result = _s.get(base_url).json() delete_url = 'http://127.0.0.1:8000/delete?ip=' for item in result: if item[2] < 11: delete_info = _s.get(delete_url + item[0]) print(delete_info.text) elif self.type == fz_ip_pool \ or self.type == sesame_ip_pool: self.redis_cli.set(self.h_key, '') return None
def _get_coordinates_for(address): formatted_address = re_sub(ADDRESS_CLEANER_PATTERN, "", address.strip()) city = formatted_address.split(" ")[0] if not re_sub("(%s|\W)" % city, "", formatted_address): return params = { "address": formatted_address, "sensor": "false", "key": google_api_key } url = BASE_GEOCODE_URL + urlencode(params) session = requests_session() session.headers["Connection"] = "close" response = session.get(url) if response.status_code == 200: try: return response.json()["results"][0]["geometry"]["location"] except LookupError: return None
from requests import session as requests_session from grequests import request as async_request from time import sleep import warnings token_type = '' access_token = '' headers = {'X-API-VERSION': '1.0', 'content-type': 'application/json'} vadmin_api_session = requests_session() vadmin_api_session.hooks = { 'response': lambda r, *args, **kwargs: r.raise_for_status() } class BrokenPlanError(Exception): """ Error occures in case if user is trying to start the broken plan """ class VAdminAPI: def __init__(self, new_vadmin_api_host, database_id): self.host = new_vadmin_api_host self.database_id = database_id def get_token(self, username, password): """ Returns token by username&password given. Sets token_type and access_token attributes. Adds Authorization header. """ url = self.host + "/idsrv/connect/token"
import concurrent.futures # Contains a map with links at each level (root page is level 0) as such: # 0: [root] # 1: [link 1, link 2] # 2: [link 3, link 6, link 7] # This dictionary does not allow duplicates so that the same page isn't downloaded multiple times (e.g. home page is present on multiple pages) linksDepthMap = {} # Root folder of each page, this will contain all the sub pages including the root one outputWebFolder = '' # Used to prevent saving the links of the next level while parsing a page on the last level totalDepth = 0; requestsSession = requests_session() def crawl_website(url, crawl_depth): # The given URL should be the root linksDepthMap[0] = [url] # Compute the root output folder for each application run global outputWebFolder outputWebFolder = Utils.compute_alpha_numerical_string(url) global totalDepth totalDepth = crawl_depth # Each link from each level is parsed and saved on a different thread # The application waits for each level to be completely finished before moving to the next one for current_depth_level in range(crawl_depth):
def get_proxy_ip_from_ip_pool(self): ''' 从代理ip池中获取到对应ip :return: dict类型 {'http': ['http://183.136.218.253:80', ...]} ''' proxy_list = [] if self.type == ip_proxy_pool: if self.high_conceal: base_url = 'http://127.0.0.1:8000/?types=0' # types: 0高匿|1匿名|2透明 else: base_url = 'http://127.0.0.1:8000' try: with requests_session() as _s: result = _s.get(base_url).json() except Exception as e: print(e) return {'http': None} with requests_session() as _s: for item in result: if item[2] > 7: tmp_url = 'http://{}:{}'.format(item[0], item[1]) proxy_list.append(tmp_url) else: delete_url = 'http://127.0.0.1:8000/delete?ip=' delete_info = _s.get(delete_url + item[0]) elif self.type == fz_ip_pool: base_url = 'http://127.0.0.1:8002/get_all' # mac 内网读取树莓派的服务(本地不开, ip池白名单冲突!) # base_url = 'http://192.168.2.112:8001/get_all' try: with requests_session() as _s: res = _s.get(base_url).json() assert res != [], 'res为空list!' except Exception as e: print(e) return {'https': None} proxy_list = ['http://{}:{}'.format(item['ip'], item['port']) for item in res] elif self.type == sesame_ip_pool: _ = json_2_dict(self.redis_cli.get(name=self.h_key) or dumps([]), default_res=[]) proxy_list = [] for i in _: if datetime_to_timestamp(string_to_datetime(i.get('expire_time', ''))) \ > datetime_to_timestamp(get_shanghai_time()) + 15: proxy_list.append('http://{}:{}'.format(i.get('ip', ''), i.get('port', ''))) elif self.type == tri_ip_pool: base_url = 'http://127.0.0.1:8001/get_all' # mac 内网读取树莓派的服务(本地不开, ip池白名单冲突!) # base_url = 'http://192.168.2.112:8001/get_all' try: with requests_session() as _s: res = _s.get(base_url).json() assert res != [], 'res为空list!' except Exception as e: print(e) return {'https': None} proxy_list = ['http://{}:{}'.format(item['ip'], item['port']) for item in res] return { 'https': proxy_list, } else: raise ValueError('type值异常, 请检查!') return { 'http': proxy_list, }