def request(method, url, **kwargs):
"""Constructs and sends a :class:`Request <Request>`.
:param method: method for the new :class:`Request` object: ``GET``, ``OPTIONS``, ``HEAD``, ``POST``, ``PUT``, ``PATCH``, or ``DELETE``.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary, list of tuples or bytes to send
in the query string for the :class:`Request`.
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the :class:`Request`.
:param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
:param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload.
``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')``
or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string
defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers
to add for the file.
:param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) How many seconds to wait for the server to send data
before giving up, as a float, or a :ref:`(connect timeout, read
timeout) <timeouts>` tuple.
:type timeout: float or tuple
:param allow_redirects: (optional) Boolean. Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``True``.
:type allow_redirects: bool
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
:param verify: (optional) Either a boolean, in which case it controls whether we verify
the server's TLS certificate, or a string, in which case it must be a path
to a CA bundle to use. Defaults to ``True``.
:param stream: (optional) if ``False``, the response content will be immediately downloaded.
:param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
:return: :class:`Response <Response>` object
:rtype: requests.Response
Usage::
>>> import requests
>>> req = requests.request('GET', 'https://httpbin.org/get')
>>> req
<Response [200]>
"""
# By using the 'with' statement we are sure the session is closed, thus we
# avoid leaving sockets open which can trigger a ResourceWarning in some
# cases, and look like a memory leak in others.
if config.proxy:
proxies = {
'http': "{}:{}".format(config.proxy_host, config.proxy_port),
'https': "{}:{}".format(config.proxy_host, config.proxy_port)
}
with sessions.Session() as session:
return session.request(method=method,
url=url,
proxies=proxies,
**kwargs)
else:
with sessions.Session() as session:
return session.request(method=method, url=url, **kwargs)
def main():
args = parse_arguments()
config = yaml.load(open(args.config))
with sessions.Session() as session:
rocket = RocketChat(
user_id=config["user_id"],
auth_token=config["auth_token"],
server_url=config["server"],
session=session,
)
fields = get_rocketchat_fields(args)
fields_string = json.dumps(fields)
print(fields_string)
users = get_all_users(rocket, fields_string)
if args.add_roles:
add_roles(users, rocket)
df = pd.DataFrame(users)
if "emails" in df:
df["email"] = df["emails"].apply(join_emails)
if "email" not in df and not args.no_email:
print("WARN: emails not retrieved; do you have permission?")
if "lastLogin" not in df and not args.no_lastlogin:
print("WARN: last login not retrieved; do you have permission?")
df.to_csv("rocketchat-user-details.csv", index=False)
def main():
args = parse_arguments()
config = yaml.load(open(args.config), Loader=yaml.SafeLoader)
with sessions.Session() as session:
rocket = RocketChat(
user_id=config["user_id"],
auth_token=config["auth_token"],
server_url=config["server"],
session=session,
)
channels_params = get_params(args.filter_featured, args.regexp)
count = 50 # number of channels per page (default 50)
channels_list: List[Dict] = []
channels_json = rocket.channels_list(count=count, **channels_params).json()
channels_list.extend(channels_json["channels"])
total = channels_json["total"]
print(f"Found {total} channels")
for offset in tqdm(range(count, total, count)):
channels_json = rocket.channels_list(
offset=offset, count=count, **channels_params
).json()
channels_list.extend(channels_json["channels"])
channels_df = postprocess(pd.DataFrame(channels_list))
if args.add_owners:
print("Adding owners")
channels_df = add_owners(channels_df, rocket)
channels_df.to_csv(args.output_file, index=False)
def main():
args = parse_arguments()
config = yaml.load(open(args.config))
with sessions.Session() as session:
rocket = RocketChat(
user_id=config["user_id"],
auth_token=config["auth_token"],
server_url=config["server"],
session=session,
)
channel_dump = pd.read_csv(args.channel_dump)
with open(args.input) as f:
destination_channels = [channel.strip() for channel in f]
destination_channels = [
channel for channel in destination_channels if channel != ""
]
nonexistent_channels = set(destination_channels) - set(
channel_dump["name"])
assert len(
nonexistent_channels
) == 0, "Some channels not found: " + str(nonexistent_channels)
channel_ids = channel_dump[channel_dump["name"].isin(
destination_channels)]["_id"]
for channel_name, channel in zip(destination_channels, channel_ids):
if not args.test:
rocket.channels_add_all(channel)
print('Added all users to channel "{}"'.format(channel_name))
def login():
sina = "http://www.sina.com"
portal = "https://portalnew.dhu.edu.cn/"
headers = {
'Accept':
'text/html,application/xhtml+xm…plication/xml;q=0.9,*/*;q=0.8',
'Accept-Encoding':
'gzip, deflate',
'Accept-Language':
'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Connection':
'keep-alive',
'Host':
'www.sina.com',
'Upgrade-Insecure-Requests':
'1',
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; W…) Gecko/20100101 Firefox/57.0'
}
s = sessions.Session()
sina = s.get(sina, allow_redirects=False)
sina.encoding = 'UTF-8-SIG'
# print(sina.text)
if match(".*html.*", sina.text) == None:
switch_hm1 = s.get(sina.headers['Location'], allow_redirects=False)
# print(switch_hm1.headers)
switchphp = s.get(portal + switch_hm1.headers['Location'],
allow_redirects=False)
# print(switchphp.cookies.get_dict())
ck = switchphp.cookies
# print(ck)
loginInfo = {'username': username, 'password': password}
#ck = {"PHPSESSID":"t25bao5qm3219mndrhdltgo455"}
# print(username)
# print(password)
resp = s.post('https://portalnew.dhu.edu.cn/post.php',
data=loginInfo,
cookies=ck,
allow_redirects=False)
resp.encoding = 'UTF-8'
# print(resp.text)
# print(resp.headers)
h = resp.headers
h = dict(h)
# print(h)
# print(type(h))
# print(h)
# print(h['location'])
query = resp.headers['Location']
params = urlparse.parse_qs(query.split('?')[1], True)
# print(params)
if (params['p'][0] == 'status'):
print("Success! ")
else:
print(params['error'][0])
else:
print("Already connect to Internet!")
def __init__(self,
auth_key: str,
sleep_time_on_rate_limit: Optional[float] = None,
print_on_rate_limit: bool = False,
base_url=None,
do_initial_ping=True):
"""
:param auth_key: authentication key for API calls.
:param sleep_time_on_rate_limit: seconds to wait after being rate limited.
Default value is calculated depending on patron type.
:param print_on_rate_limit: whether or not to print upon rate limits.
"""
self.auth_key = auth_key
self._session = sessions.Session()
self.steam_name = None
self.steam_id = None
self.patron_type = None
self.rate_limit_count = 0
self.base_url = DEFAULT_URL if base_url is None else base_url
if do_initial_ping:
self.ping()
if sleep_time_on_rate_limit is None:
self.sleep_time_on_rate_limit = {
"regular": 3600 / 1000,
"gold": 3600 / 2000,
"diamond": 3600 / 5000,
"champion": 1 / 8,
"gc": 1 / 16
}.get(self.patron_type or "regular")
else:
self.sleep_time_on_rate_limit = sleep_time_on_rate_limit
self.print_on_rate_limit = print_on_rate_limit
def query(endpoint, page_number=1, page_size=None, **parameters):
"""
Send a query request to Intrinio API for a dataset page including page
count and other metadata using optional query parameters.
Args:
endpoint: Intrinio endpoint, for example: companies
page_number: Optional page number where 1 is the first page (default 1)
page_size: Optional page size (default max page size for the endpoint)
parameters: Optional query parameters
Returns:
Intrinio endpoint response as a tree of dictionaries, values and lists
"""
if page_size is None:
page_size = get_page_size(endpoint)
url = '{}/{}'.format(api_base_url, endpoint)
parameters['page_number'] = page_number
parameters['page_size'] = page_size
auth = (username, password)
with sessions.Session() as session:
response = session.request('GET', url, params=parameters, auth=auth,
verify=True)
if not response.ok:
response.raise_for_status()
return json.loads(response.content.decode('utf-8'))
def _vt_default_request(req):
'''
if use requests streaming, we cannot return r.content direct,
we should return response instance.
'''
import requests
from requests import sessions
if not req:
return None
er = None
request_retry = req.get(u'request_retry', 1)
request_kwargs = req.get(u'requests_kwargs')
with sessions.Session() as ses:
for _ in range(request_retry):
try:
return ses.request(method=req.get(u'method', u'get')
, url=req.get(u'url')
, **request_kwargs)
except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout) as er1:
t = request_kwargs.get(u'timeout', 0)
request_kwargs.update({u'timeout': t + 5})
er = er1
except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as er2:
er = er2
raise er
def create_rocketchat_channels(channel_names):
with open(ROCKETCHAT_KEY, "r") as f:
config = yaml.safe_load(f)
today = datetime.datetime.today() + datetime.timedelta(days=10)
today = today.strftime("%Y-%m-%dT%H:%M:%S")
oldest = datetime.datetime.today() + datetime.timedelta(days=-10)
oldest = oldest.strftime("%Y-%m-%dT%H:%M:%S")
with sessions.Session() as session:
rocket = connect_rocket_API(config, session)
for paper in channel_names:
channel_name = channel_names[paper]["channel_name"]
created = rocket.channels_create(channel_name).json()
if created["success"] == False: ## Code to handle when API Limit is hit
print("API rate limit hit, pausing for 1 minute")
sleep_session(60)
try:
created = rocket.channels_create(channel_name).json()
except:
rocket = connect_rocket_API(config, session)
created = rocket.channels_create(channel_name).json()
print(channel_name, created)
channel_id = rocket.channels_info(channel=channel_name).json()["channel"][
"_id"
]
rocket.channels_set_topic(channel_id, channel_names[paper]["topic"]).json()
rocket.channels_set_description(
channel_id, channel_names[paper]["description"]
).json()
# pdb.set_trace()
# rocket.rooms_info(room_name=channel_name)
# rocket.rooms_clean_history(room_id = channel_id,latest=today,oldest=oldest)
print(
"Creating " + channel_name + " topic " + channel_names[paper]["topic"]
)
def __init__(self, url, timeout=5, username=None, password=None):
super(RestClient, self).__init__()
self.url = url
self.timeout = timeout
self.session = sessions.Session()
if username is not None and password is not None:
self.session.auth = (username, password)
def listener(self, event):
if event['type'] == 'intrinio_request':
with sessions.Session() as sess:
endpoint = event['endpoint'] if event['endpoint'].endswith(
'.csv') else event['endpoint'] + '.csv'
if 'parameters' in event:
result = get_csv(sess,
endpoint=endpoint,
**event['parameters'])
else:
result = get_csv(sess, endpoint=endpoint)
if 'dataframe' in event:
result = to_dataframe(result)
self.listeners({'type': 'intrinio_request_result', 'data': result})
elif event['type'] == 'intrinio_historical_data':
data = event['data'] if isinstance(event['data'],
list) else event['data']
result = get_historical_data(
data,
threads=event['threads'] if 'threads' in event else 1,
async=event['async'] if 'async' in event else False)
self.listeners({
'type': 'intrinio_historical_data_result',
'data': result
})
def _handle_requests(
self,
tasks: Iterable[AzureLogRecord],
exception_handler: Callable = None,
timeout: int = 5,
) -> None:
"""Concurrently handles a collection of AzureLogRecords to convert the requests to responses.
:param tasks: a collection of AzureLogRecord objects.
:param exception_handler: Callback function, called when exception occured. Params: Request, Exception
:param timeout: Request timeout in seconds.
"""
tasks = list(tasks)
if not tasks:
return
with sessions.Session() as session:
for record in tasks:
request = session.prepare_request(record.log_request)
record.log_response = session.send(request, timeout=timeout)
if record.log_response:
continue
if exception_handler and hasattr(record.log_request,
"exception"):
record.log_response = exception_handler(
record.log_request, record.log_request.exception)
else:
record.log_response = None
def __makeRequest(self, url, method="GET", data=None, auth=None, files=None):
headers = {}
if auth is None:
if self.auth:
auth = self.auth
elif self.api_key:
headers = {'Authorization': 'Bearer {}'.format(self.api_key)}
requestObj = Request(method=method, url=url, headers=headers, data=data, auth=auth, files=files)
prep_req = requestObj.prepare()
with sessions.Session() as session:
response = session.send(prep_req)
if response.status_code == 200:
if (response.content and hasattr(response.content, "decode")):
return response.content.decode("utf-8")
return response.content
if response.status_code == 401:
raise SplitwiseUnauthorizedException("Please check your token or consumer id and secret", response=response)
if response.status_code == 403:
raise SplitwiseNotAllowedException("You are not allowed to perform this operation", response=response)
if response.status_code == 400:
raise SplitwiseBadRequestException("Please check your request", response=response)
if response.status_code == 404:
raise SplitwiseNotFoundException("Required resource is not found", response)
raise SplitwiseException("Unknown error happened", response)
def request(self):
with sessions.Session() as session:
try:
response = session.request(method=self.method, url=self.url, data=self.params, timeout=2)
return response
except Exception as e:
self.log(f"请求错误:{e}")
return None
def proxy(self, request):
url = self.get_request_url(request)
params = self.get_request_params(request)
data = self.get_request_data(request)
files = self.get_request_files(request)
headers = self.get_headers(request)
try:
if files:
"""
By default requests library uses chunked upload for files
but it is much more easier for servers to handle streamed
uploads.
This new implementation is also lightweight as files are not
read entirely into memory.
"""
boundary = generate_boundary()
headers[
'Content-Type'] = 'multipart/form-data; boundary=%s' % boundary
body = StreamingMultipart(data, files, boundary)
session = sessions.Session()
session.mount('http://', StreamingHTTPAdapter())
session.mount('https://', StreamingHTTPAdapter())
response = session.request(request.method,
url,
params=params,
data=body,
headers=headers,
timeout=self.proxy_settings.TIMEOUT)
else:
response = requests.request(
request.method,
url,
params=params,
data=data,
files=files,
headers=headers,
timeout=self.proxy_settings.TIMEOUT)
except (ConnectionError, SSLError):
status = requests.status_codes.codes.bad_gateway
return self.create_error_response(
{
'code': status,
'error': 'Bad gateway',
}, status)
except (Timeout):
status = requests.status_codes.codes.gateway_timeout
return self.create_error_response(
{
'code': status,
'error': 'Gateway timed out',
}, status)
return self.create_response(response)
def _get_instance(self):
inst_remote = mock.MagicMock()
inst_remote.get_http_client.return_value = sessions.Session()
inst_remote.__enter__.return_value = inst_remote
inst = mock.MagicMock()
inst.remote.return_value = inst_remote
return inst
def calls(method, url, params, data, header):
try:
with sessions.Session() as session:
res = session.request(method=method,
url=url,
params=params,
data=data,
headers=header)
return response(res)
except Exception as e:
return e
def __third_party_lib(ticker):
from requests import sessions
session = sessions.Session()
ticker_sa = ticker + '.sa'
try:
from yahooquery import Ticker
preco = Ticker(ticker_sa, session=session).price[ticker_sa]['regularMarketPrice']
finally:
session.close()
return preco
def __init__(self, url, username, password, timeout):
super(OpenDaylightRestClient, self).__init__()
self.url = url
self.timeout = timeout
self.session = sessions.Session()
self.session.auth = (username, password)
# NOTE(knasim-wrs): Set the audit report within the underlying
# REST Client driver. This is because the L3RouterPlugin will
# independantly instantiate the client driver and needs to be
# audited as well.
self.audit_report = self.get_audit_report()
def get_json(
url: str,
params: Optional[dict] = None,
cache: Optional[JsonResponseCache] = None,
dangerous_encoded_params: bool = False,
**kwargs,
) -> Union[dict, list]:
"""
If `params` is not a dict, `dangerous_encoded_params` must also be True to avoid re-encoding by requests.Request.prepare().
"""
req = requests.Request(
"GET",
url,
headers=settings.HTTP_REQUEST_HEADERS_JSON,
params=params,
)
r = req.prepare()
if dangerous_encoded_params is True and isinstance(params, str):
encoded_url = urlparse(r.url)
r.url = encoded_url.geturl().replace(encoded_url.query, params)
encoded_url = r.url
if cache:
cached = cache.get_json(encoded_url)
if cached:
log.info(f"[cached] {encoded_url}")
return cached
else:
log.warning(f"No cache specified for call to '{url}'")
log.info(f"[using network] {encoded_url}")
with sessions.Session() as session:
response = session.send(r)
rate_limit()
if response.status_code == status.HTTP_204_NO_CONTENT:
raise HttpNoContent
elif response.status_code >= 500:
raise HttpServerError(response.status_code)
elif response.status_code >= 400:
raise HttpClientError(response.status_code)
response.encoding = "utf-8-sig"
data = response.json()
if cache:
cache.remember(encoded_url, data)
return data
def __init__(self, url, key, cert=True, reanalyze=True):
self.url = url
self.key = key
if cert and os.path.isfile(cert):
self.cert = cert
else:
self.cert = False
self.reanalyze = reanalyze
self.headers = self._prepare_headers()
self.session = sessions.Session()
self.session.headers = self.headers
self.session.verify = self.cert
def __init__(self, name, conf):
session = sessions.Session()
session.mount('http', HTTPAdapter(pool_maxsize=25))
session.mount('https', HTTPAdapter(pool_maxsize=25))
self.session = session
self.platform = name
self.conf = conf
self.roe = conf.roe
self.scale = conf.scale
self.currency = conf.currency
self.choices = set(conf.choices)
LOG.info('%s roe is %f' % (self.name, self.roe))
def call(method: str,
url: str,
data: dict,
headers: dict,
retries: int = 10) -> Response:
global response
session: Session = sessions.Session()
retry_count: int = 0
status_code: int = 0
while status_code != 200 and retry_count < retries:
response: Response = session.request(method, url, data, headers)
status_code = response.status_code
retry_count += 1
return response
def __init__(self):
self.sess = sessions.Session()
self.sess.proxies = {
'http': 'http://*****:*****@114.119.39.130:18888/',
'https': 'https://*****:*****@114.119.39.130:18888/'
}
self.headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:42.0) Gecko/20100101 Firefox/42.0'
}
# self.pauth = ("ipin", "ipin1234")
self.sess.get("https://passport.zhaopin.com/org/login")
self.ocr = OnlineOCR("zhilian")
self.cansearch = 0
self.index = 0
self.count = 0
def __init__(self,
url,
key,
recursive_sample_limit=10,
reanalyze=True,
verify=True,
**optional_parameters):
self.url = url
self.key = key
self.reanalyze = reanalyze
self.recursive_sample_limit = recursive_sample_limit
self.headers = self._prepare_headers()
self.session = sessions.Session()
self.session.headers = self.headers
self.session.verify = verify
self.optional_parameters = optional_parameters
def __set_up_http_session(self):
if self.__http_session:
return
self.__http_session = sessions.Session()
windows_auth_url = self.__get_windows_authentication_url(
self.__http_session)
qps_session_cookie = authenticator.Authenticator \
.get_qps_session_cookie_windows_auth(
ad_domain=self.__ad_domain,
username=self.__username,
password=self.__password,
auth_url=windows_auth_url)
self.__http_session.cookies.set_cookie(qps_session_cookie)
logging.debug('QPS session cookie issued for the QRS API: %s',
qps_session_cookie)
def __third_party_lib(ticker):
from requests import sessions
session = sessions.Session()
ticker_sa = ticker + '.sa'
try:
from yahooquery import Ticker
ticker = Ticker(ticker_sa,
validate=True,
status_forcelist=[404],
backoff_factor=5,
retry=10,
progress=True,
session=session)
preco = ticker.price[ticker_sa]['regularMarketPrice']
finally:
session.close()
return preco
class RequestUtil:
session = sessions.Session()
@classmethod
def do_request(cls, request, load_json=True, stream=False):
response = cls.session.request(
method=request.method,
url=request.url,
headers=request.headers,
params=request.params,
cookies=request.cookies,
stream=stream,
timeout=10)
return cls.build_response(request, response, load_json)
@classmethod
def build_response(cls, request, raw_response, load_json):
return Response(request=request, raw_response=raw_response, load_json=load_json)
def construct_connection(url: str, headers: str,
timeout: int) -> sessions.Session:
"""
Creates a requests.session.Session() object with a handle to the neo4j DB
Arguments:
url {str} -- Neo4j URL to connect to
headers {str} -- JSON string containing the necessary authorization header and content type
timeout {int} -- HTTP request timeout in seconds
Returns:
requests.sessions.Session -- requests.sessions.Session() object for the given connection
"""
s = sessions.Session()
s.url = url
s.headers = headers
s.timeout = timeout
return s
def main():
"""
Set roles to users, according to a CSV with this format:
user,roles
username1, "role1, role2, ..."
"""
args = parse_arguments()
config = yaml.load(open(args.config))
with sessions.Session() as session:
rocket = RocketChat(
user_id=config["user_id"],
auth_token=config["auth_token"],
server_url=config["server"],
session=session,
)
df = pd.read_csv(args.role_file)
# This is awful, but I have to do it as it is not implemented in
# RocketChat API, and also to avoid pylint's complaining
def add_role(user, role):
boundmethod = getattr(rocket, "_RocketChat__call_api_post")
return boundmethod(method="roles.addUserToRole",
roleName=role,
username=user).json()
for _, row in df.iterrows():
roles = row.roles.split(",")
user = row.user
for role in roles:
resp = add_role(user, role)
if not resp["success"]:
print(
f"Couldn't set {row.user} as {role} -- permission issue? does role exist?"
)