def get_response(url): """ HTTP Request header example: { "Origin": "https://developer.riotgames.com", "Accept-Charset": "application/x-www-form-urlencoded; charset=UTF-8", "X-Riot-Token": "api-key", "Accept-Language": "es-ES,es;q=0.8,en;q=0.6", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36" } """ key_file = os.path.join(os.path.dirname(__file__), "api-key") key = open(key_file).readline()[:-1] headers = {"X-Riot-Token": key} data = requests.get(url, headers=headers) if data.status_code == 200: return data.json() else: raise requests.HTTPError(data)
def request(self, method, url, **kwargs): log.debug('%s: %s %s', method, url, repr(kwargs) if kwargs else '') kwargs.setdefault('timeout', 60) kwargs.setdefault('allow_redirects', False) for _ in range(self.max_attempts): try: resp = super().request(method=method, url=url, **kwargs) except (requests.ConnectionError, requests.Timeout, requests.exceptions.ChunkedEncodingError): continue if 200 <= resp.status_code < 300: return resp elif 300 <= resp.status_code < 400: raise requests.HTTPError( 'Redirect attempted with url: {}'.format(url)) elif 400 <= resp.status_code < 600: continue raise requests.ConnectionError( 'Se ha excedido el máximo de reintentos con la url: {}'.format( url))
def test_get_domain_command_non_404_request_exception(mocker): """ Given: - list of domains When: - A non 404 http error is returned Then: - raise error """ with pytest.raises(RequestException): error = requests.HTTPError() error.response = requests.Response() error.response.status_code = 403 mocker.patch.object(demisto, 'args', return_value={'domain': ["bad1.com", "bad2.com"]}) mocker.patch.object(Cisco_umbrella_investigate, 'get_whois_for_domain', side_effect=error) Cisco_umbrella_investigate.get_domain_command()
def __asyncSendData(self, url, data=None, cb=None, method="POST", **kwargs): while not self.rateCheck(): time.sleep(self.getRemainingRate() + self.rateOff) r = res = None try: r = requests.request(method, url, json=data, **kwargs) if r.status_code == 200: res = r.json() else: res = { "error": "Http error", "message": r.reason, "exception": requests.HTTPError(response=r) } except json.JSONDecodeError as e: res = { "error": "Error parsing data", "message": e.__doc__, "exception": e } except Exception as e: res = { "error": "Error sending data", "message": e.__doc__, "exception": e } if cb and res: res["response"] = r self.lock.acquire() cb(res) self.lock.release()
def _query_grlc(self, params, endpoint): """Query the nanopub server grlc endpoint. Query a nanopub grlc server endpoint (for example: find_text). Try several of the nanopub garlic servers. """ headers = {"Accept": "application/json"} r = None random.shuffle(self.grlc_urls) # To balance load across servers for grlc_url in self.grlc_urls: url = grlc_url + endpoint r = requests.get(url, params=params, headers=headers) if r.status_code == 502: # Server is likely down warnings.warn(f'Could not get response from {grlc_url}, trying other servers') else: r.raise_for_status() # For non-502 errors we don't want to try other servers return r raise requests.HTTPError(f'Could not get response from any of the nanopub grlc ' f'endpoints, last response: {r.status_code}:{r.reason}')
def test_get_domain_command_no_valid_domains(mocker): """ Given: - list of domains When: - All of the domains cannot be found by whois Then: - return an empty list """ error = requests.HTTPError() error.response = requests.Response() error.response.status_code = 404 mocker.patch.object(demisto, 'args', return_value={'domain': ["bad1.com", "bad2.com"]}) mocker.patch.object(Cisco_umbrella_investigate, 'get_whois_for_domain', side_effect=error) assert len(Cisco_umbrella_investigate.get_domain_command()) == 2
def index(profile_name, url): import sys cj = http.cookiejar.CookieJar() ff_cookies = "./{nos}/cookies.sqlite" cookies = get_cookies(cj, ff_cookies.format(nos=profile_name)) s = requests.Session() s.cookies = cj req = s.get(url, verify=False, headers=explore_headers, cookies=cookies) if req.status_code == 200: tree = html.fromstring(req.content) t = tree.xpath(x_path) data = t[0].attrib['data-count'] if data != 0: send_to_discord(data, profile_name) else: pass else: raise requests.HTTPError(str(req.status_code)) return True
def fetch_item(self, code_or_item): """Returns a unique item object. code_or_item should be a the code of the desired item, or an item with the proper code.""" code = code_or_item if not isinstance(code_or_item, str): # if code_or_item is item, then fetch the code code = self.get_code(code_or_item) logger.debug(self._endpoint) url = urljoin(self._endpoint, code) logger.debug(url) r = self._session.get(url) if r.status_code != 200: raise requests.HTTPError("The item {0} doesn't exit: {1}".format( code, r.status_code)) logger.debug(r.status_code) logger.debug(r.text) return json.loads(r.text) # returns item as a dict
def get_lines(self): retry_times = 0 while True: try: resp = requests.get(self.query_api) if resp.status_code == 200: return resp.text.split('\n') else: raise requests.HTTPError("Invalid response code %d" % resp.status_code) except Exception as e: if retry_times >= self.max_retry: self.logger.error( "Having retried %d times, now giving up exception %s, message <%s>..." % (retry_times, e, e.message)) return [] self.logger.error( "Request exception %s, message <%s>, retrying..." % (e, e.message)) retry_times += 1
def download_file(url, local_filename, file_size): if os.path.exists(local_filename): print(f'{local_filename} already exists.') return file_size = sizeof_fmt(file_size) print(f'Downloading {local_filename} ({file_size})', end=' ... ') sys.stdout.flush() token = get_jwt_token() with requests.get(url, stream=True, params={'access_token': token}) as r: r.raise_for_status() with open(local_filename, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): if b'<!doctype html>' in chunk or b'"errorCode"' in chunk: raise requests.HTTPError('Got an error page.') f.write(chunk) print('Done')
def get_res_json_for_single(req_id, session, url): """ like get_res_json_for_list() 实现异常检测 """ re_request_times = 0 r_status, json_res = False, '' while not r_status: if re_request_times == constants.MAX_RE_REQUEST_TIMES: raise requests.HTTPError('reach max_request_times') if re_request_times > 0: print('req_id: {0}, re_request_times: {1}'.format(req_id, re_request_times)) time.sleep(constants.TIME_SLEEP_FOR_REQ) try: res = session.get(url, headers={'User-Agent': constants.ua.random}, timeout=constants.REQ_TIME_OUT) r_status, json_res = check_r(res) if not r_status: re_request_times += 1 except: re_request_times += 1 time.sleep(constants.NET_TEST_INTEVAL * 2) return json_res
def get_web_session(): """Logs in as admin user and returns the valid requests.Session object""" sat_session = requests.Session() url = f'https://{settings.server.hostname}' init_request = sat_session.get(url, verify=False) login_request = sat_session.post( f'{url}/users/login', data={ 'authenticity_token': extract_ui_token(init_request.text), 'login[login]': settings.server.admin_username, 'login[password]': settings.server.admin_password, 'commit': 'Log In', }, verify=False, ) login_request.raise_for_status() if 'users/login' in login_request.history[0].headers.get('Location'): raise requests.HTTPError('Failed to authenticate using the given credentials') return sat_session
def get_search_page(url): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36' } try: page = requests.get(url, headers=headers, auth=HTTPBasicAuth(Email, Password)) page.raise_for_status() except Exception: raise requests.HTTPError(Exception) if page.status_code == 200: return page else: return -1
def test_calculate_suites_fallback(self): n_tests = 100 response = Mock() response.status_code = requests.codes.SERVICE_UNAVAILABLE evg = Mock() evg.test_stats_by_project.side_effect = requests.HTTPError(response=response) main = grt.Main(evg) main.options = Mock() main.options.execution_time_minutes = 10 main.config_options = self.get_mock_options() main.list_tests = Mock(return_value=["test{}.js".format(i) for i in range(n_tests)]) suites = main.calculate_suites(_DATE, _DATE) self.assertEqual(main.config_options.fallback_num_sub_suites, len(suites)) for suite in suites: self.assertEqual(50, len(suite.tests)) self.assertEqual(n_tests, len(main.test_list))
def login(): global SESSION_ID data = { 'user': CREDENTIALS.get('identifier'), 'password': CREDENTIALS.get('password') } if SESSION_ID is None: url = '/j/rest/v1/access/login/json/' try: res = http_request('POST', url, data=data) if res.get('error') is not None: raise requests.HTTPError('Failed to login: {}'.format( res['error'])) SESSION_ID = res.get('uid') except requests.exceptions.RequestException as e: # noqa return_error( 'Demisto has encounter a connection error, ' 'please check the server_url and credentials parameters')
def create_new_sptfy_playlist_with_id(access_token, playlist_name): sptfy_user_id = secrets['spotify_user_id'] playlist_endpoint = f'https://api.spotify.com/v1/users/{sptfy_user_id}/playlists' playlist_data = { 'name': playlist_name } playlist_header = { 'Authorization': f'Bearer {access_token}', 'Content-Type': 'application/json' } playlist_data_j = json.dumps(playlist_data) r = requests.post(playlist_endpoint, headers=playlist_header, data=playlist_data_j) r_playlist_data = r.json() valid_request = r.status_code is 200 or 201 if not valid_request: raise requests.HTTPError(f'''Playlist creation request failed with HTTP error {r.status_code}. Expected status code 200-201''') playlist_id = r_playlist_data['id'] return playlist_id
def get_all_members(api_key): url = 'https://actionnetwork.org/api/v2/people' headers = {'OSDI-API-Token': api_key} members = [] while True: response = requests.get(url, headers=headers) status_code = response.status_code if status_code != 200: raise requests.HTTPError(response=response) content = response.json() new_members = content['_embedded']['osdi:people'] members.extend(new_members) try: url = content['_links']['next']['href'] except KeyError: # end querying when there is no more data left break return members
def authenticate(self): """Sends request to authenticate user. Returns: object: authentication data (token and after what time it expires) """ try: data = { 'name': self.config['username'], 'secret': self.config['password'] } content_type = 'application/json' response = self.post('/authenticate', data, content_type) if response.status_code == 201: data = response.json() return data else: raise requests.HTTPError(response.status_code) except Exception as ex: raise ex
def _generic_request(self, reqfunction, rest_query, data=None): # if the data is a dictionary, it has to be converted to json text if isinstance(data, dict): data = json.dumps(data) # makes sure the base url is the one from the instance rest_query = rest_query.replace(self._base_url, '') if not rest_query.startswith('/'): rest_query = '/' + rest_query r = reqfunction(self._base_url + rest_query, stream=True, headers=self._headers, data=data) if r and r.status_code in (200, 201): return json.loads(r.text) elif r and r.status_code == 204: return else: logger_.error(self._base_url + rest_query) logger_.error(r.text) raise (requests.HTTPError(r))
def test_client_graphql_retries_if_token_needs_refreshing(monkeypatch): error = requests.HTTPError() error.response = MagicMock(status_code=401) # unauthorized post = MagicMock( return_value=MagicMock( raise_for_status=MagicMock(side_effect=error), json=MagicMock(return_value=dict(token="new-token")), ) ) monkeypatch.setattr("requests.post", post) with set_temporary_config( {"cloud.graphql": "http://my-cloud.foo", "cloud.auth_token": "secret_token"} ): client = Client() with pytest.raises(requests.HTTPError) as exc: result = client.graphql("{}") assert exc.value is error assert post.call_count == 3 # first call -> refresh token -> last call assert post.call_args[0][0] == "http://my-cloud.foo" assert client.token == "new-token"
def test_calculate_suites_fallback(self): response = Mock() response.status_code = requests.codes.SERVICE_UNAVAILABLE evg = Mock() evg.test_stats.side_effect = requests.HTTPError(response=response) main = grs.Main(evg) main.options = Mock() main.options.execution_time_minutes = 10 main.config_options = grs.ConfigOptions(2, 15, "project", "", 1, True, "task", "suite", "variant") main.list_tests = Mock( return_value=["test{}.js".format(i) for i in range(100)]) suites = main.calculate_suites(_DATE, _DATE) self.assertEqual(main.config_options.fallback_num_sub_suites, len(suites)) for suite in suites: self.assertEqual(50, len(suite.tests))
def _check_response(self, response): """Check our responses for a good response code. Parameters ---------- response : str API call response Raises ------- requests.HTTPError Raises this error if we get a response status code that is not in the 200 range. """ if response.status_code // 100 == 2: return True else: raise requests.HTTPError('{} ==> {}'.format( response.status_code, response.text))
def pins(swlat, swlng, nelat, nelng, pintypes='stop', *, raw=False): """ DVB Map Pins (GET https://www.dvb.de/apps/map/pins) :param swlat: South-West Bounding Box Latitude :param swlng: South-West Bounding Box Longitude :param nelat: North-East Bounding Box Latitude :param nelng: North-East Bounding Box Longitude :param pintypes: Types to search for, defaults to 'stop' :param raw: Return raw response :return: """ try: swlat, swlng = wgs_to_gk4(swlat, swlng) nelat, nelng = wgs_to_gk4(nelat, nelng) r = requests.get( url='https://www.dvb.de/apps/map/pins', params={ 'showlines': 'true', 'swlat': swlat, 'swlng': swlng, 'nelat': nelat, 'nelng': nelng, 'pintypes': pintypes, }, ) if r.status_code == 200: response = json.loads(r.content.decode('utf-8')) else: raise requests.HTTPError('HTTP Status: {}'.format(r.status_code)) except requests.RequestException as e: print('Failed to access DVB map pins app. Request Exception', e) response = None if response is None: return None return response if raw else [ pins_return_results(line, pintypes) for line in response ]
def fetch_items(cls): """ Fetch a list of items via the game's npc_items.txt Fetches npc_items.txt as JSON via Dotabuff's d2vpk repository, and parses it for data. Falls back to data stored on the file-system in case of a HTTPError or KeyError. Only retrieves ID and token for now, but there's a ton more data available should we ever need it. Returns: An array of Item objects. """ try: req = requests.get(ITEM_DATA_URL) # Raise HTTPError if we don't get HTTP OK if req.status_code != requests.codes.ok: raise requests.HTTPError("Response not HTTP OK") # Fetch relevant pieces of data from JSON data input_items = req.json()['DOTAAbilities'] output_items = [] # Iterate through heries, create an instance of this class for each. for key, item in input_items.items(): # Skip these keys - they're not item definitions if key in ["Version"]: continue output_items.append(cls(_id=int(item.get('ID')), token=key)) return output_items except (steam.api.HTTPError, KeyError): sentry.captureMessage('Item.fetch_items failed', exc_info=sys.exc_info) # Try to get data from existing cache entry data = fs_cache.cache.get(cls._CACHE_KEY, ignore_expiry=True) # Return data if we have any, else return an empty list() return data or list()
def __call__(self, method, path, params=None, json=None, check=True): """ :type check: bool | dict[int:Exception] """ r = self._session.request(method, '{}/repos/{}/{}'.format( self._url, self._repo, path), params=params, json=json) if check: if isinstance(check, collections.Mapping): exc = check.get(r.status_code) if exc: raise exc(r.text) if r.status_code >= 400: headers = '\n'.join('\t%s: %s' % (h, v) for h, v in r.headers.items()) if r.headers.get('content-type', '').startswith( ('application/json', 'application/javascript')): body = r.json() elif r.encoding is not None: body = utils.shorten(r.text, 200) else: body = utils.shorten(r.content, 200) _logger.error( "%(method)s /%(repo)s/%(path)s\n=> %(status)d %(reason)s\n%(headers)s\n\n\t%(body)r\n====================", { 'status': r.status_code, 'reason': r.reason, 'method': method, 'repo': self._repo, 'path': path, 'headers': headers, 'body': body }) if not isinstance(body, (bytes, str)): raise requests.HTTPError(json_.dumps(body, indent=4), response=r) r.raise_for_status() return r
def execute(self, document, variable_values=None, timeout=None): # type: (Node, dict, int) -> ExecutionResult """Execute the provided document AST against the configured remote server. This uses the requests library to perform a HTTP POST request to the remote server. :param document: GraphQL query as AST Node object. :param variable_values: Dictionary of input parameters (Default: None). :param timeout: Specifies a default timeout for requests (Default: None). :return: The result of execution. `data` is the result of executing the query, `errors` is null if no errors occurred, and is a non-empty array if an error occurred. """ query_str = print_ast(document) payload = {'query': query_str, 'variables': variable_values or {}} data_key = 'json' if self.use_json else 'data' post_args = { 'headers': self.headers, 'auth': self.auth, 'cookies': self.cookies, 'timeout': timeout or self.default_timeout, 'verify': self.verify, data_key: payload } # Pass kwargs to requests post method post_args.update(self.kwargs) response = requests.post(self.url, **post_args) try: result = response.json() if not isinstance(result, dict): raise ValueError except ValueError: result = {} if 'errors' not in result and 'data' not in result: response.raise_for_status() raise requests.HTTPError("Server did not return a GraphQL result", response=response) return ExecutionResult(errors=result.get('errors'), data=result.get('data'))
def nominatim_query(url, *, q): if url[-1] == '/': url = url[:-1] res = start = end = None # Avoids warnings for i in range(5): if i > 0: time.sleep(1) PROM_NOMINATIM_REQS.inc() # Count all requests start = time.perf_counter() if isinstance(q, (tuple, list)): # Batch query res = _nominatim_session.get( url + '/search?' + urlencode({ 'batch': json.dumps([{ 'q': qe } for qe in q]), 'format': 'jsonv2', }), ) else: # Normal query res = _nominatim_session.get( url + '/search?' + urlencode({ 'q': q, 'format': 'jsonv2' }), ) end = time.perf_counter() if res.status_code not in (502, 503, 504): break res.raise_for_status() # Record time for successful request PROM_NOMINATIM_REQ_TIME.observe(end - start) if not res.headers['Content-Type'].startswith('application/json'): raise requests.HTTPError( "Response is not JSON for URL: %s" % res.url, response=res, ) if isinstance(q, (tuple, list)): return res.json()['batch'] else: return res.json()
def import_gists_to_database(db, username, commit=True): r = requests.get('https://api.github.com/users/{}/gists'.format(username)) if r.status_code != 200: raise requests.HTTPError() request_data = r.json() #convert request data into json format #create db connetion for gist in request_data: github_id = gist["id"] html_url = gist["html_url"] git_pull_url = gist["git_pull_url"] git_push_url = gist["git_push_url"] commits_url = gist["commits_url"] forks_url = gist["forks_url"] public = gist["public"] created_at = gist["created_at"] updated_at = gist["updated_at"] comments = gist["comments"] comments_url = gist["comments_url"] query = """INSERT INTO gists (github_id, html_url, git_pull_url, git_push_url, commits_url, forks_url, public, created_at, updated_at, comments, comments_url) VALUES (:github_id, :html_url, :git_pull_url, :git_push_url, :commits_url, :forks_url, :public, :created_at, :updated_at, :comments, :comments_url)""" cursor = db.execute( query, { 'github_id': github_id, 'html_url': html_url, 'git_pull_url': git_pull_url, 'git_push_url': git_push_url, 'commits_url': commits_url, 'forks_url': forks_url, 'public': public, 'created_at': created_at, 'updated_at': updated_at, 'comments': comments, 'comments_url': comments_url })
def wait_until_task_is_finished(host: str, task_name: str, repo: str, state: str = 'active', task_type: str = 'running_one_off', **kwargs): """Wait until a task is finished Args: - host (str): the host for the backup service. - task_name (str): the task we are looking for. - task_type [running_one_off, running_tasks]: The type of task we are waiting for. - state ([active, imported, archived]): The repository state. - kwargs: - timeout (int): request timeout. - retries (int): how many times to poll the endpoint at a maximum. - user (str): The user to use for the requests. - passwrod (str): The password to use for the requests. """ timeout = int(kwargs.get('timeout', 60)) retries = int(kwargs.get('retries', 10)) user, password = get_user_and_password(**kwargs) if task_type not in ['running_one_off', 'running_tasks']: raise ValueError('taks type must be one off [running_one_off, running_tasks]') for retry in range(retries): logger.debug(f'Checking if task {task_type} {task_name} is still running. Attempt {retry}') res = requests.get(f'{host}/api/v1/cluster/self/repository/{state}/{repo}', auth=(user, password), timeout=timeout) if res.status_code != 200: raise requests.HTTPError(f'Unexpected error code: {res.status_code} {res.json()}') repository = res.json() logger.debug(f'/api/v1/cluster/self/repository/{state}/{repo} response: {repository}') if task_type not in repository or task_name not in repository[task_type]: if retry == 0: logger.debug('Task finished before first run') # if no running one offs assume task is finished return time.sleep(1 * (retries + 1)) continue raise AssertionError('Task is not finished')
def _route_call(aircraft_id): """ Make a call to the BlueBird LISTROUTE endpoint. Parameters ---------- aircraft_id: str A string aircraft identifier. For the BlueSky simulator, this has to be at least three characters. Returns ------- dict : A dictionary with keys: ``"callsign"`` A string aircraft identifier. ``"next_waypoint"`` A string. Name of waypoint the aircraft is currently headed toward. ``"route_name"`` A string name of the route. ``"route_waypoints"`` A list of strings. All the waypoints on the route. Notes ----- If the aircraft has no route information, a dictionary with just the callsign is returned. """ url = construct_endpoint_url(endpoint) resp = requests.get( url, params={config_param("query_aircraft_id"): aircraft_id}) if resp.status_code == 200: return json.loads(resp.text) elif response.status == config_param("status_code_aircraft_has_no_route"): return {config_param("query_aircraft_id"): aircraft_id} else: raise requests.HTTPError(resp.text)