def elastic(path):
proxy_request_headers = {"kbn-xsrf": "reporting"}
requests_response = None
response = Response()
session = requests_session()
session.headers["Connection"] = "close"
auth = AWSAuth(config.aws_credentials, config.aws_region)
query_string = urlencode(parse_qs(request.query_string.decode("utf-8"),
keep_blank_values=True),
quote_via=quote,
doseq=True)
endpoint = config.aws_endpoint + "/" + path + "?" + query_string
for header, value in request.headers.items():
if header.lower() in PROXY_REQ_HEADERS_WHITELIST:
proxy_request_headers[header] = value
if request.method == "HEAD":
session_response = session.head(endpoint,
auth=auth,
cookies=request.cookies,
headers=proxy_request_headers)
elif request.method == "GET":
session_response = session.get(endpoint,
auth=auth,
cookies=request.cookies,
headers=proxy_request_headers)
elif request.method == "POST":
session_response = session.post(endpoint,
cookies=request.cookies,
auth=auth,
data=request.get_data(),
headers=proxy_request_headers)
elif request.method == "PUT":
session_response = session.put(endpoint,
cookies=request.cookies,
auth=auth,
data=request.get_data(),
headers=proxy_request_headers)
elif request.method == "DELETE":
session_response = session.delete(endpoint,
auth=auth,
cookies=request.cookies,
headers=proxy_request_headers)
else:
return "Method is not allowed!"
if request.method != "HEAD":
response.set_data(session_response.content)
response.status_code = session_response.status_code
for header, value in session_response.headers.items():
if header.lower() not in PROXY_RESP_HEADERS_BLACKLIST:
response.headers[header] = value
return response
def request(self, method, path, **kwargs):
url = self.url
if path:
url = utils.urlparse.urljoin(self.url, path)
if self._session is None:
self._session = requests_session()
more = dict(self._settings)
more.update(kwargs)
return utils.request(method, url, session=self._session, **more)
def request(self, method, path, **kwargs):
url = self.url
if path:
url = utils.urlparse.urljoin(self.url, path)
if self._session is None:
self._session = requests_session()
more = dict(self._settings)
more.update(kwargs)
return utils.request(method, url, session=self._session, **more)
def request(self, method, path, data=None, headers=None,
is_subpath=True):
url = self.url
if path:
url = utils.urlparse.urljoin(self.url, path)
assert url.startswith(self.url) or not is_subpath
if self._session is None:
self._session = requests_session()
self._check_dav_header()
return utils.request(method, url, data=data, headers=headers,
session=self._session, **self._settings)
def run(self):
self.check_browser()
self.set_urls()
self.client = requests_session()
self.ping_server()
self.server_configuration_check()
sessions_to_create = []
session_config_name = self.session_config_name
if session_config_name:
if session_config_name not in SESSION_CONFIGS_DICT:
raise ValueError(
'No session config named "{}"'.format(session_config_name)
)
session_config_names = [session_config_name]
else:
# default to all session configs
session_config_names = SESSION_CONFIGS_DICT.keys()
self.max_name_length = max(
len(config_name) for config_name in session_config_names
)
for session_config_name in session_config_names:
session_config = SESSION_CONFIGS_DICT[session_config_name]
num_bot_cases = session_config.get_num_bot_cases()
for case_number in range(num_bot_cases):
num_participants = (
self.num_participants or session_config['num_demo_participants']
)
sessions_to_create.append(
{
'session_config_name': session_config_name,
'num_participants': num_participants,
'case_number': case_number,
}
)
total_time_spent = 0
# run in a separate loop, because we want to validate upfront
# that the session configs are valid, etc,
# rather than the command failing halfway through
for session_to_create in sessions_to_create:
total_time_spent += self.run_session(**session_to_create)
print('Total: {} seconds'.format(round(total_time_spent, 1)))
def _empty_ip_pools(self):
'''
清空ip池
:return:
'''
if self.type == ip_proxy_pool:
base_url = 'http://127.0.0.1:8000'
with requests_session() as _s:
result = _s.get(base_url).json()
delete_url = 'http://127.0.0.1:8000/delete?ip='
for item in result:
if item[2] < 11:
delete_info = _s.get(delete_url + item[0])
print(delete_info.text)
elif self.type == fz_ip_pool \
or self.type == sesame_ip_pool:
self.redis_cli.set(self.h_key, '')
return None
def _get_coordinates_for(address):
formatted_address = re_sub(ADDRESS_CLEANER_PATTERN, "", address.strip())
city = formatted_address.split(" ")[0]
if not re_sub("(%s|\W)" % city, "", formatted_address):
return
params = {
"address": formatted_address,
"sensor": "false",
"key": google_api_key
}
url = BASE_GEOCODE_URL + urlencode(params)
session = requests_session()
session.headers["Connection"] = "close"
response = session.get(url)
if response.status_code == 200:
try:
return response.json()["results"][0]["geometry"]["location"]
except LookupError:
return None
from requests import session as requests_session
from grequests import request as async_request
from time import sleep
import warnings
token_type = ''
access_token = ''
headers = {'X-API-VERSION': '1.0', 'content-type': 'application/json'}
vadmin_api_session = requests_session()
vadmin_api_session.hooks = {
'response': lambda r, *args, **kwargs: r.raise_for_status()
}
class BrokenPlanError(Exception):
""" Error occures in case if user is trying to start the broken plan """
class VAdminAPI:
def __init__(self, new_vadmin_api_host, database_id):
self.host = new_vadmin_api_host
self.database_id = database_id
def get_token(self, username, password):
"""
Returns token by username&password given.
Sets token_type and access_token attributes.
Adds Authorization header.
"""
url = self.host + "/idsrv/connect/token"
import concurrent.futures
# Contains a map with links at each level (root page is level 0) as such:
# 0: [root]
# 1: [link 1, link 2]
# 2: [link 3, link 6, link 7]
# This dictionary does not allow duplicates so that the same page isn't downloaded multiple times (e.g. home page is present on multiple pages)
linksDepthMap = {}
# Root folder of each page, this will contain all the sub pages including the root one
outputWebFolder = ''
# Used to prevent saving the links of the next level while parsing a page on the last level
totalDepth = 0;
requestsSession = requests_session()
def crawl_website(url, crawl_depth):
# The given URL should be the root
linksDepthMap[0] = [url]
# Compute the root output folder for each application run
global outputWebFolder
outputWebFolder = Utils.compute_alpha_numerical_string(url)
global totalDepth
totalDepth = crawl_depth
# Each link from each level is parsed and saved on a different thread
# The application waits for each level to be completely finished before moving to the next one
for current_depth_level in range(crawl_depth):
def get_proxy_ip_from_ip_pool(self):
'''
从代理ip池中获取到对应ip
:return: dict类型 {'http': ['http://183.136.218.253:80', ...]}
'''
proxy_list = []
if self.type == ip_proxy_pool:
if self.high_conceal:
base_url = 'http://127.0.0.1:8000/?types=0' # types: 0高匿|1匿名|2透明
else:
base_url = 'http://127.0.0.1:8000'
try:
with requests_session() as _s:
result = _s.get(base_url).json()
except Exception as e:
print(e)
return {'http': None}
with requests_session() as _s:
for item in result:
if item[2] > 7:
tmp_url = 'http://{}:{}'.format(item[0], item[1])
proxy_list.append(tmp_url)
else:
delete_url = 'http://127.0.0.1:8000/delete?ip='
delete_info = _s.get(delete_url + item[0])
elif self.type == fz_ip_pool:
base_url = 'http://127.0.0.1:8002/get_all'
# mac 内网读取树莓派的服务(本地不开, ip池白名单冲突!)
# base_url = 'http://192.168.2.112:8001/get_all'
try:
with requests_session() as _s:
res = _s.get(base_url).json()
assert res != [], 'res为空list!'
except Exception as e:
print(e)
return {'https': None}
proxy_list = ['http://{}:{}'.format(item['ip'], item['port']) for item in res]
elif self.type == sesame_ip_pool:
_ = json_2_dict(self.redis_cli.get(name=self.h_key) or dumps([]), default_res=[])
proxy_list = []
for i in _:
if datetime_to_timestamp(string_to_datetime(i.get('expire_time', ''))) \
> datetime_to_timestamp(get_shanghai_time()) + 15:
proxy_list.append('http://{}:{}'.format(i.get('ip', ''), i.get('port', '')))
elif self.type == tri_ip_pool:
base_url = 'http://127.0.0.1:8001/get_all'
# mac 内网读取树莓派的服务(本地不开, ip池白名单冲突!)
# base_url = 'http://192.168.2.112:8001/get_all'
try:
with requests_session() as _s:
res = _s.get(base_url).json()
assert res != [], 'res为空list!'
except Exception as e:
print(e)
return {'https': None}
proxy_list = ['http://{}:{}'.format(item['ip'], item['port']) for item in res]
return {
'https': proxy_list,
}
else:
raise ValueError('type值异常, 请检查!')
return {
'http': proxy_list,
}
