def request(method, url, **kwargs): """Constructs and sends a :class:`Request <Request>`. :param method: method for the new :class:`Request` object: ``GET``, ``OPTIONS``, ``HEAD``, ``POST``, ``PUT``, ``PATCH``, or ``DELETE``. :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary, list of tuples or bytes to send in the query string for the :class:`Request`. :param data: (optional) Dictionary, list of tuples, bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. ``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')`` or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers to add for the file. :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. :param timeout: (optional) How many seconds to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple :param allow_redirects: (optional) Boolean. Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``True``. :type allow_redirects: bool :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use. Defaults to ``True``. :param stream: (optional) if ``False``, the response content will be immediately downloaded. :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. :return: :class:`Response <Response>` object :rtype: requests.Response Usage:: >>> import requests >>> req = requests.request('GET', 'https://httpbin.org/get') >>> req <Response [200]> """ # By using the 'with' statement we are sure the session is closed, thus we # avoid leaving sockets open which can trigger a ResourceWarning in some # cases, and look like a memory leak in others. if config.proxy: proxies = { 'http': "{}:{}".format(config.proxy_host, config.proxy_port), 'https': "{}:{}".format(config.proxy_host, config.proxy_port) } with sessions.Session() as session: return session.request(method=method, url=url, proxies=proxies, **kwargs) else: with sessions.Session() as session: return session.request(method=method, url=url, **kwargs)
def main(): args = parse_arguments() config = yaml.load(open(args.config)) with sessions.Session() as session: rocket = RocketChat( user_id=config["user_id"], auth_token=config["auth_token"], server_url=config["server"], session=session, ) fields = get_rocketchat_fields(args) fields_string = json.dumps(fields) print(fields_string) users = get_all_users(rocket, fields_string) if args.add_roles: add_roles(users, rocket) df = pd.DataFrame(users) if "emails" in df: df["email"] = df["emails"].apply(join_emails) if "email" not in df and not args.no_email: print("WARN: emails not retrieved; do you have permission?") if "lastLogin" not in df and not args.no_lastlogin: print("WARN: last login not retrieved; do you have permission?") df.to_csv("rocketchat-user-details.csv", index=False)
def main(): args = parse_arguments() config = yaml.load(open(args.config), Loader=yaml.SafeLoader) with sessions.Session() as session: rocket = RocketChat( user_id=config["user_id"], auth_token=config["auth_token"], server_url=config["server"], session=session, ) channels_params = get_params(args.filter_featured, args.regexp) count = 50 # number of channels per page (default 50) channels_list: List[Dict] = [] channels_json = rocket.channels_list(count=count, **channels_params).json() channels_list.extend(channels_json["channels"]) total = channels_json["total"] print(f"Found {total} channels") for offset in tqdm(range(count, total, count)): channels_json = rocket.channels_list( offset=offset, count=count, **channels_params ).json() channels_list.extend(channels_json["channels"]) channels_df = postprocess(pd.DataFrame(channels_list)) if args.add_owners: print("Adding owners") channels_df = add_owners(channels_df, rocket) channels_df.to_csv(args.output_file, index=False)
def main(): args = parse_arguments() config = yaml.load(open(args.config)) with sessions.Session() as session: rocket = RocketChat( user_id=config["user_id"], auth_token=config["auth_token"], server_url=config["server"], session=session, ) channel_dump = pd.read_csv(args.channel_dump) with open(args.input) as f: destination_channels = [channel.strip() for channel in f] destination_channels = [ channel for channel in destination_channels if channel != "" ] nonexistent_channels = set(destination_channels) - set( channel_dump["name"]) assert len( nonexistent_channels ) == 0, "Some channels not found: " + str(nonexistent_channels) channel_ids = channel_dump[channel_dump["name"].isin( destination_channels)]["_id"] for channel_name, channel in zip(destination_channels, channel_ids): if not args.test: rocket.channels_add_all(channel) print('Added all users to channel "{}"'.format(channel_name))
def login(): sina = "http://www.sina.com" portal = "https://portalnew.dhu.edu.cn/" headers = { 'Accept': 'text/html,application/xhtml+xm…plication/xml;q=0.9,*/*;q=0.8', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2', 'Connection': 'keep-alive', 'Host': 'www.sina.com', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; W…) Gecko/20100101 Firefox/57.0' } s = sessions.Session() sina = s.get(sina, allow_redirects=False) sina.encoding = 'UTF-8-SIG' # print(sina.text) if match(".*html.*", sina.text) == None: switch_hm1 = s.get(sina.headers['Location'], allow_redirects=False) # print(switch_hm1.headers) switchphp = s.get(portal + switch_hm1.headers['Location'], allow_redirects=False) # print(switchphp.cookies.get_dict()) ck = switchphp.cookies # print(ck) loginInfo = {'username': username, 'password': password} #ck = {"PHPSESSID":"t25bao5qm3219mndrhdltgo455"} # print(username) # print(password) resp = s.post('https://portalnew.dhu.edu.cn/post.php', data=loginInfo, cookies=ck, allow_redirects=False) resp.encoding = 'UTF-8' # print(resp.text) # print(resp.headers) h = resp.headers h = dict(h) # print(h) # print(type(h)) # print(h) # print(h['location']) query = resp.headers['Location'] params = urlparse.parse_qs(query.split('?')[1], True) # print(params) if (params['p'][0] == 'status'): print("Success! ") else: print(params['error'][0]) else: print("Already connect to Internet!")
def __init__(self, auth_key: str, sleep_time_on_rate_limit: Optional[float] = None, print_on_rate_limit: bool = False, base_url=None, do_initial_ping=True): """ :param auth_key: authentication key for API calls. :param sleep_time_on_rate_limit: seconds to wait after being rate limited. Default value is calculated depending on patron type. :param print_on_rate_limit: whether or not to print upon rate limits. """ self.auth_key = auth_key self._session = sessions.Session() self.steam_name = None self.steam_id = None self.patron_type = None self.rate_limit_count = 0 self.base_url = DEFAULT_URL if base_url is None else base_url if do_initial_ping: self.ping() if sleep_time_on_rate_limit is None: self.sleep_time_on_rate_limit = { "regular": 3600 / 1000, "gold": 3600 / 2000, "diamond": 3600 / 5000, "champion": 1 / 8, "gc": 1 / 16 }.get(self.patron_type or "regular") else: self.sleep_time_on_rate_limit = sleep_time_on_rate_limit self.print_on_rate_limit = print_on_rate_limit
def query(endpoint, page_number=1, page_size=None, **parameters): """ Send a query request to Intrinio API for a dataset page including page count and other metadata using optional query parameters. Args: endpoint: Intrinio endpoint, for example: companies page_number: Optional page number where 1 is the first page (default 1) page_size: Optional page size (default max page size for the endpoint) parameters: Optional query parameters Returns: Intrinio endpoint response as a tree of dictionaries, values and lists """ if page_size is None: page_size = get_page_size(endpoint) url = '{}/{}'.format(api_base_url, endpoint) parameters['page_number'] = page_number parameters['page_size'] = page_size auth = (username, password) with sessions.Session() as session: response = session.request('GET', url, params=parameters, auth=auth, verify=True) if not response.ok: response.raise_for_status() return json.loads(response.content.decode('utf-8'))
def _vt_default_request(req): ''' if use requests streaming, we cannot return r.content direct, we should return response instance. ''' import requests from requests import sessions if not req: return None er = None request_retry = req.get(u'request_retry', 1) request_kwargs = req.get(u'requests_kwargs') with sessions.Session() as ses: for _ in range(request_retry): try: return ses.request(method=req.get(u'method', u'get') , url=req.get(u'url') , **request_kwargs) except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout) as er1: t = request_kwargs.get(u'timeout', 0) request_kwargs.update({u'timeout': t + 5}) er = er1 except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as er2: er = er2 raise er
def create_rocketchat_channels(channel_names): with open(ROCKETCHAT_KEY, "r") as f: config = yaml.safe_load(f) today = datetime.datetime.today() + datetime.timedelta(days=10) today = today.strftime("%Y-%m-%dT%H:%M:%S") oldest = datetime.datetime.today() + datetime.timedelta(days=-10) oldest = oldest.strftime("%Y-%m-%dT%H:%M:%S") with sessions.Session() as session: rocket = connect_rocket_API(config, session) for paper in channel_names: channel_name = channel_names[paper]["channel_name"] created = rocket.channels_create(channel_name).json() if created["success"] == False: ## Code to handle when API Limit is hit print("API rate limit hit, pausing for 1 minute") sleep_session(60) try: created = rocket.channels_create(channel_name).json() except: rocket = connect_rocket_API(config, session) created = rocket.channels_create(channel_name).json() print(channel_name, created) channel_id = rocket.channels_info(channel=channel_name).json()["channel"][ "_id" ] rocket.channels_set_topic(channel_id, channel_names[paper]["topic"]).json() rocket.channels_set_description( channel_id, channel_names[paper]["description"] ).json() # pdb.set_trace() # rocket.rooms_info(room_name=channel_name) # rocket.rooms_clean_history(room_id = channel_id,latest=today,oldest=oldest) print( "Creating " + channel_name + " topic " + channel_names[paper]["topic"] )
def __init__(self, url, timeout=5, username=None, password=None): super(RestClient, self).__init__() self.url = url self.timeout = timeout self.session = sessions.Session() if username is not None and password is not None: self.session.auth = (username, password)
def listener(self, event): if event['type'] == 'intrinio_request': with sessions.Session() as sess: endpoint = event['endpoint'] if event['endpoint'].endswith( '.csv') else event['endpoint'] + '.csv' if 'parameters' in event: result = get_csv(sess, endpoint=endpoint, **event['parameters']) else: result = get_csv(sess, endpoint=endpoint) if 'dataframe' in event: result = to_dataframe(result) self.listeners({'type': 'intrinio_request_result', 'data': result}) elif event['type'] == 'intrinio_historical_data': data = event['data'] if isinstance(event['data'], list) else event['data'] result = get_historical_data( data, threads=event['threads'] if 'threads' in event else 1, async=event['async'] if 'async' in event else False) self.listeners({ 'type': 'intrinio_historical_data_result', 'data': result })
def _handle_requests( self, tasks: Iterable[AzureLogRecord], exception_handler: Callable = None, timeout: int = 5, ) -> None: """Concurrently handles a collection of AzureLogRecords to convert the requests to responses. :param tasks: a collection of AzureLogRecord objects. :param exception_handler: Callback function, called when exception occured. Params: Request, Exception :param timeout: Request timeout in seconds. """ tasks = list(tasks) if not tasks: return with sessions.Session() as session: for record in tasks: request = session.prepare_request(record.log_request) record.log_response = session.send(request, timeout=timeout) if record.log_response: continue if exception_handler and hasattr(record.log_request, "exception"): record.log_response = exception_handler( record.log_request, record.log_request.exception) else: record.log_response = None
def __makeRequest(self, url, method="GET", data=None, auth=None, files=None): headers = {} if auth is None: if self.auth: auth = self.auth elif self.api_key: headers = {'Authorization': 'Bearer {}'.format(self.api_key)} requestObj = Request(method=method, url=url, headers=headers, data=data, auth=auth, files=files) prep_req = requestObj.prepare() with sessions.Session() as session: response = session.send(prep_req) if response.status_code == 200: if (response.content and hasattr(response.content, "decode")): return response.content.decode("utf-8") return response.content if response.status_code == 401: raise SplitwiseUnauthorizedException("Please check your token or consumer id and secret", response=response) if response.status_code == 403: raise SplitwiseNotAllowedException("You are not allowed to perform this operation", response=response) if response.status_code == 400: raise SplitwiseBadRequestException("Please check your request", response=response) if response.status_code == 404: raise SplitwiseNotFoundException("Required resource is not found", response) raise SplitwiseException("Unknown error happened", response)
def request(self): with sessions.Session() as session: try: response = session.request(method=self.method, url=self.url, data=self.params, timeout=2) return response except Exception as e: self.log(f"请求错误:{e}") return None
def proxy(self, request): url = self.get_request_url(request) params = self.get_request_params(request) data = self.get_request_data(request) files = self.get_request_files(request) headers = self.get_headers(request) try: if files: """ By default requests library uses chunked upload for files but it is much more easier for servers to handle streamed uploads. This new implementation is also lightweight as files are not read entirely into memory. """ boundary = generate_boundary() headers[ 'Content-Type'] = 'multipart/form-data; boundary=%s' % boundary body = StreamingMultipart(data, files, boundary) session = sessions.Session() session.mount('http://', StreamingHTTPAdapter()) session.mount('https://', StreamingHTTPAdapter()) response = session.request(request.method, url, params=params, data=body, headers=headers, timeout=self.proxy_settings.TIMEOUT) else: response = requests.request( request.method, url, params=params, data=data, files=files, headers=headers, timeout=self.proxy_settings.TIMEOUT) except (ConnectionError, SSLError): status = requests.status_codes.codes.bad_gateway return self.create_error_response( { 'code': status, 'error': 'Bad gateway', }, status) except (Timeout): status = requests.status_codes.codes.gateway_timeout return self.create_error_response( { 'code': status, 'error': 'Gateway timed out', }, status) return self.create_response(response)
def _get_instance(self): inst_remote = mock.MagicMock() inst_remote.get_http_client.return_value = sessions.Session() inst_remote.__enter__.return_value = inst_remote inst = mock.MagicMock() inst.remote.return_value = inst_remote return inst
def calls(method, url, params, data, header): try: with sessions.Session() as session: res = session.request(method=method, url=url, params=params, data=data, headers=header) return response(res) except Exception as e: return e
def __third_party_lib(ticker): from requests import sessions session = sessions.Session() ticker_sa = ticker + '.sa' try: from yahooquery import Ticker preco = Ticker(ticker_sa, session=session).price[ticker_sa]['regularMarketPrice'] finally: session.close() return preco
def __init__(self, url, username, password, timeout): super(OpenDaylightRestClient, self).__init__() self.url = url self.timeout = timeout self.session = sessions.Session() self.session.auth = (username, password) # NOTE(knasim-wrs): Set the audit report within the underlying # REST Client driver. This is because the L3RouterPlugin will # independantly instantiate the client driver and needs to be # audited as well. self.audit_report = self.get_audit_report()
def get_json( url: str, params: Optional[dict] = None, cache: Optional[JsonResponseCache] = None, dangerous_encoded_params: bool = False, **kwargs, ) -> Union[dict, list]: """ If `params` is not a dict, `dangerous_encoded_params` must also be True to avoid re-encoding by requests.Request.prepare(). """ req = requests.Request( "GET", url, headers=settings.HTTP_REQUEST_HEADERS_JSON, params=params, ) r = req.prepare() if dangerous_encoded_params is True and isinstance(params, str): encoded_url = urlparse(r.url) r.url = encoded_url.geturl().replace(encoded_url.query, params) encoded_url = r.url if cache: cached = cache.get_json(encoded_url) if cached: log.info(f"[cached] {encoded_url}") return cached else: log.warning(f"No cache specified for call to '{url}'") log.info(f"[using network] {encoded_url}") with sessions.Session() as session: response = session.send(r) rate_limit() if response.status_code == status.HTTP_204_NO_CONTENT: raise HttpNoContent elif response.status_code >= 500: raise HttpServerError(response.status_code) elif response.status_code >= 400: raise HttpClientError(response.status_code) response.encoding = "utf-8-sig" data = response.json() if cache: cache.remember(encoded_url, data) return data
def __init__(self, url, key, cert=True, reanalyze=True): self.url = url self.key = key if cert and os.path.isfile(cert): self.cert = cert else: self.cert = False self.reanalyze = reanalyze self.headers = self._prepare_headers() self.session = sessions.Session() self.session.headers = self.headers self.session.verify = self.cert
def __init__(self, name, conf): session = sessions.Session() session.mount('http', HTTPAdapter(pool_maxsize=25)) session.mount('https', HTTPAdapter(pool_maxsize=25)) self.session = session self.platform = name self.conf = conf self.roe = conf.roe self.scale = conf.scale self.currency = conf.currency self.choices = set(conf.choices) LOG.info('%s roe is %f' % (self.name, self.roe))
def call(method: str, url: str, data: dict, headers: dict, retries: int = 10) -> Response: global response session: Session = sessions.Session() retry_count: int = 0 status_code: int = 0 while status_code != 200 and retry_count < retries: response: Response = session.request(method, url, data, headers) status_code = response.status_code retry_count += 1 return response
def __init__(self): self.sess = sessions.Session() self.sess.proxies = { 'http': 'http://*****:*****@114.119.39.130:18888/', 'https': 'https://*****:*****@114.119.39.130:18888/' } self.headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:42.0) Gecko/20100101 Firefox/42.0' } # self.pauth = ("ipin", "ipin1234") self.sess.get("https://passport.zhaopin.com/org/login") self.ocr = OnlineOCR("zhilian") self.cansearch = 0 self.index = 0 self.count = 0
def __init__(self, url, key, recursive_sample_limit=10, reanalyze=True, verify=True, **optional_parameters): self.url = url self.key = key self.reanalyze = reanalyze self.recursive_sample_limit = recursive_sample_limit self.headers = self._prepare_headers() self.session = sessions.Session() self.session.headers = self.headers self.session.verify = verify self.optional_parameters = optional_parameters
def __set_up_http_session(self): if self.__http_session: return self.__http_session = sessions.Session() windows_auth_url = self.__get_windows_authentication_url( self.__http_session) qps_session_cookie = authenticator.Authenticator \ .get_qps_session_cookie_windows_auth( ad_domain=self.__ad_domain, username=self.__username, password=self.__password, auth_url=windows_auth_url) self.__http_session.cookies.set_cookie(qps_session_cookie) logging.debug('QPS session cookie issued for the QRS API: %s', qps_session_cookie)
def __third_party_lib(ticker): from requests import sessions session = sessions.Session() ticker_sa = ticker + '.sa' try: from yahooquery import Ticker ticker = Ticker(ticker_sa, validate=True, status_forcelist=[404], backoff_factor=5, retry=10, progress=True, session=session) preco = ticker.price[ticker_sa]['regularMarketPrice'] finally: session.close() return preco
class RequestUtil: session = sessions.Session() @classmethod def do_request(cls, request, load_json=True, stream=False): response = cls.session.request( method=request.method, url=request.url, headers=request.headers, params=request.params, cookies=request.cookies, stream=stream, timeout=10) return cls.build_response(request, response, load_json) @classmethod def build_response(cls, request, raw_response, load_json): return Response(request=request, raw_response=raw_response, load_json=load_json)
def construct_connection(url: str, headers: str, timeout: int) -> sessions.Session: """ Creates a requests.session.Session() object with a handle to the neo4j DB Arguments: url {str} -- Neo4j URL to connect to headers {str} -- JSON string containing the necessary authorization header and content type timeout {int} -- HTTP request timeout in seconds Returns: requests.sessions.Session -- requests.sessions.Session() object for the given connection """ s = sessions.Session() s.url = url s.headers = headers s.timeout = timeout return s
def main(): """ Set roles to users, according to a CSV with this format: user,roles username1, "role1, role2, ..." """ args = parse_arguments() config = yaml.load(open(args.config)) with sessions.Session() as session: rocket = RocketChat( user_id=config["user_id"], auth_token=config["auth_token"], server_url=config["server"], session=session, ) df = pd.read_csv(args.role_file) # This is awful, but I have to do it as it is not implemented in # RocketChat API, and also to avoid pylint's complaining def add_role(user, role): boundmethod = getattr(rocket, "_RocketChat__call_api_post") return boundmethod(method="roles.addUserToRole", roleName=role, username=user).json() for _, row in df.iterrows(): roles = row.roles.split(",") user = row.user for role in roles: resp = add_role(user, role) if not resp["success"]: print( f"Couldn't set {row.user} as {role} -- permission issue? does role exist?" )