def get_response(url):
"""
HTTP Request header example:
{
"Origin": "https://developer.riotgames.com",
"Accept-Charset": "application/x-www-form-urlencoded; charset=UTF-8",
"X-Riot-Token": "api-key",
"Accept-Language": "es-ES,es;q=0.8,en;q=0.6",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36"
}
"""
key_file = os.path.join(os.path.dirname(__file__), "api-key")
key = open(key_file).readline()[:-1]
headers = {"X-Riot-Token": key}
data = requests.get(url, headers=headers)
if data.status_code == 200:
return data.json()
else:
raise requests.HTTPError(data)
def request(self, method, url, **kwargs):
log.debug('%s: %s %s', method, url, repr(kwargs) if kwargs else '')
kwargs.setdefault('timeout', 60)
kwargs.setdefault('allow_redirects', False)
for _ in range(self.max_attempts):
try:
resp = super().request(method=method, url=url, **kwargs)
except (requests.ConnectionError, requests.Timeout,
requests.exceptions.ChunkedEncodingError):
continue
if 200 <= resp.status_code < 300:
return resp
elif 300 <= resp.status_code < 400:
raise requests.HTTPError(
'Redirect attempted with url: {}'.format(url))
elif 400 <= resp.status_code < 600:
continue
raise requests.ConnectionError(
'Se ha excedido el máximo de reintentos con la url: {}'.format(
url))
def test_get_domain_command_non_404_request_exception(mocker):
"""
Given:
- list of domains
When:
- A non 404 http error is returned
Then:
- raise error
"""
with pytest.raises(RequestException):
error = requests.HTTPError()
error.response = requests.Response()
error.response.status_code = 403
mocker.patch.object(demisto,
'args',
return_value={'domain': ["bad1.com", "bad2.com"]})
mocker.patch.object(Cisco_umbrella_investigate,
'get_whois_for_domain',
side_effect=error)
Cisco_umbrella_investigate.get_domain_command()
def __asyncSendData(self, url, data=None, cb=None, method="POST", **kwargs):
while not self.rateCheck():
time.sleep(self.getRemainingRate() + self.rateOff)
r = res = None
try:
r = requests.request(method, url, json=data, **kwargs)
if r.status_code == 200: res = r.json()
else: res = { "error": "Http error", "message": r.reason, "exception": requests.HTTPError(response=r) }
except json.JSONDecodeError as e:
res = { "error": "Error parsing data", "message": e.__doc__, "exception": e }
except Exception as e:
res = { "error": "Error sending data", "message": e.__doc__, "exception": e }
if cb and res:
res["response"] = r
self.lock.acquire()
cb(res)
self.lock.release()
def _query_grlc(self, params, endpoint):
"""Query the nanopub server grlc endpoint.
Query a nanopub grlc server endpoint (for example: find_text). Try several of the nanopub
garlic servers.
"""
headers = {"Accept": "application/json"}
r = None
random.shuffle(self.grlc_urls) # To balance load across servers
for grlc_url in self.grlc_urls:
url = grlc_url + endpoint
r = requests.get(url, params=params, headers=headers)
if r.status_code == 502: # Server is likely down
warnings.warn(f'Could not get response from {grlc_url}, trying other servers')
else:
r.raise_for_status() # For non-502 errors we don't want to try other servers
return r
raise requests.HTTPError(f'Could not get response from any of the nanopub grlc '
f'endpoints, last response: {r.status_code}:{r.reason}')
def test_get_domain_command_no_valid_domains(mocker):
"""
Given:
- list of domains
When:
- All of the domains cannot be found by whois
Then:
- return an empty list
"""
error = requests.HTTPError()
error.response = requests.Response()
error.response.status_code = 404
mocker.patch.object(demisto,
'args',
return_value={'domain': ["bad1.com", "bad2.com"]})
mocker.patch.object(Cisco_umbrella_investigate,
'get_whois_for_domain',
side_effect=error)
assert len(Cisco_umbrella_investigate.get_domain_command()) == 2
def index(profile_name, url):
import sys
cj = http.cookiejar.CookieJar()
ff_cookies = "./{nos}/cookies.sqlite"
cookies = get_cookies(cj, ff_cookies.format(nos=profile_name))
s = requests.Session()
s.cookies = cj
req = s.get(url, verify=False, headers=explore_headers, cookies=cookies)
if req.status_code == 200:
tree = html.fromstring(req.content)
t = tree.xpath(x_path)
data = t[0].attrib['data-count']
if data != 0:
send_to_discord(data, profile_name)
else:
pass
else:
raise requests.HTTPError(str(req.status_code))
return True
def fetch_item(self, code_or_item):
"""Returns a unique item object. code_or_item should be a the code
of the desired item, or an item with the proper code."""
code = code_or_item
if not isinstance(code_or_item, str):
# if code_or_item is item, then fetch the code
code = self.get_code(code_or_item)
logger.debug(self._endpoint)
url = urljoin(self._endpoint, code)
logger.debug(url)
r = self._session.get(url)
if r.status_code != 200:
raise requests.HTTPError("The item {0} doesn't exit: {1}".format(
code, r.status_code))
logger.debug(r.status_code)
logger.debug(r.text)
return json.loads(r.text) # returns item as a dict
def get_lines(self):
retry_times = 0
while True:
try:
resp = requests.get(self.query_api)
if resp.status_code == 200:
return resp.text.split('\n')
else:
raise requests.HTTPError("Invalid response code %d" %
resp.status_code)
except Exception as e:
if retry_times >= self.max_retry:
self.logger.error(
"Having retried %d times, now giving up exception %s, message <%s>..."
% (retry_times, e, e.message))
return []
self.logger.error(
"Request exception %s, message <%s>, retrying..." %
(e, e.message))
retry_times += 1
def download_file(url, local_filename, file_size):
if os.path.exists(local_filename):
print(f'{local_filename} already exists.')
return
file_size = sizeof_fmt(file_size)
print(f'Downloading {local_filename} ({file_size})', end=' ... ')
sys.stdout.flush()
token = get_jwt_token()
with requests.get(url, stream=True, params={'access_token': token}) as r:
r.raise_for_status()
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
if b'<!doctype html>' in chunk or b'"errorCode"' in chunk:
raise requests.HTTPError('Got an error page.')
f.write(chunk)
print('Done')
def get_res_json_for_single(req_id, session, url):
""" like get_res_json_for_list() 实现异常检测 """
re_request_times = 0
r_status, json_res = False, ''
while not r_status:
if re_request_times == constants.MAX_RE_REQUEST_TIMES:
raise requests.HTTPError('reach max_request_times')
if re_request_times > 0:
print('req_id: {0}, re_request_times: {1}'.format(req_id, re_request_times))
time.sleep(constants.TIME_SLEEP_FOR_REQ)
try:
res = session.get(url, headers={'User-Agent': constants.ua.random}, timeout=constants.REQ_TIME_OUT)
r_status, json_res = check_r(res)
if not r_status:
re_request_times += 1
except:
re_request_times += 1
time.sleep(constants.NET_TEST_INTEVAL * 2)
return json_res
def get_web_session():
"""Logs in as admin user and returns the valid requests.Session object"""
sat_session = requests.Session()
url = f'https://{settings.server.hostname}'
init_request = sat_session.get(url, verify=False)
login_request = sat_session.post(
f'{url}/users/login',
data={
'authenticity_token': extract_ui_token(init_request.text),
'login[login]': settings.server.admin_username,
'login[password]': settings.server.admin_password,
'commit': 'Log In',
},
verify=False,
)
login_request.raise_for_status()
if 'users/login' in login_request.history[0].headers.get('Location'):
raise requests.HTTPError('Failed to authenticate using the given credentials')
return sat_session
def get_search_page(url):
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36'
}
try:
page = requests.get(url,
headers=headers,
auth=HTTPBasicAuth(Email, Password))
page.raise_for_status()
except Exception:
raise requests.HTTPError(Exception)
if page.status_code == 200:
return page
else:
return -1
def test_calculate_suites_fallback(self):
n_tests = 100
response = Mock()
response.status_code = requests.codes.SERVICE_UNAVAILABLE
evg = Mock()
evg.test_stats_by_project.side_effect = requests.HTTPError(response=response)
main = grt.Main(evg)
main.options = Mock()
main.options.execution_time_minutes = 10
main.config_options = self.get_mock_options()
main.list_tests = Mock(return_value=["test{}.js".format(i) for i in range(n_tests)])
suites = main.calculate_suites(_DATE, _DATE)
self.assertEqual(main.config_options.fallback_num_sub_suites, len(suites))
for suite in suites:
self.assertEqual(50, len(suite.tests))
self.assertEqual(n_tests, len(main.test_list))
def login():
global SESSION_ID
data = {
'user': CREDENTIALS.get('identifier'),
'password': CREDENTIALS.get('password')
}
if SESSION_ID is None:
url = '/j/rest/v1/access/login/json/'
try:
res = http_request('POST', url, data=data)
if res.get('error') is not None:
raise requests.HTTPError('Failed to login: {}'.format(
res['error']))
SESSION_ID = res.get('uid')
except requests.exceptions.RequestException as e: # noqa
return_error(
'Demisto has encounter a connection error, '
'please check the server_url and credentials parameters')
def create_new_sptfy_playlist_with_id(access_token, playlist_name):
sptfy_user_id = secrets['spotify_user_id']
playlist_endpoint = f'https://api.spotify.com/v1/users/{sptfy_user_id}/playlists'
playlist_data = {
'name': playlist_name
}
playlist_header = {
'Authorization': f'Bearer {access_token}',
'Content-Type': 'application/json'
}
playlist_data_j = json.dumps(playlist_data)
r = requests.post(playlist_endpoint, headers=playlist_header, data=playlist_data_j)
r_playlist_data = r.json()
valid_request = r.status_code is 200 or 201
if not valid_request:
raise requests.HTTPError(f'''Playlist creation request failed with
HTTP error {r.status_code}.
Expected status code 200-201''')
playlist_id = r_playlist_data['id']
return playlist_id
def get_all_members(api_key):
url = 'https://actionnetwork.org/api/v2/people'
headers = {'OSDI-API-Token': api_key}
members = []
while True:
response = requests.get(url, headers=headers)
status_code = response.status_code
if status_code != 200:
raise requests.HTTPError(response=response)
content = response.json()
new_members = content['_embedded']['osdi:people']
members.extend(new_members)
try:
url = content['_links']['next']['href']
except KeyError: # end querying when there is no more data left
break
return members
def authenticate(self):
"""Sends request to authenticate user.
Returns:
object: authentication data (token and after what time it expires)
"""
try:
data = {
'name': self.config['username'],
'secret': self.config['password']
}
content_type = 'application/json'
response = self.post('/authenticate', data, content_type)
if response.status_code == 201:
data = response.json()
return data
else:
raise requests.HTTPError(response.status_code)
except Exception as ex:
raise ex
def _generic_request(self, reqfunction, rest_query, data=None):
# if the data is a dictionary, it has to be converted to json text
if isinstance(data, dict):
data = json.dumps(data)
# makes sure the base url is the one from the instance
rest_query = rest_query.replace(self._base_url, '')
if not rest_query.startswith('/'):
rest_query = '/' + rest_query
r = reqfunction(self._base_url + rest_query,
stream=True,
headers=self._headers,
data=data)
if r and r.status_code in (200, 201):
return json.loads(r.text)
elif r and r.status_code == 204:
return
else:
logger_.error(self._base_url + rest_query)
logger_.error(r.text)
raise (requests.HTTPError(r))
def test_client_graphql_retries_if_token_needs_refreshing(monkeypatch):
error = requests.HTTPError()
error.response = MagicMock(status_code=401) # unauthorized
post = MagicMock(
return_value=MagicMock(
raise_for_status=MagicMock(side_effect=error),
json=MagicMock(return_value=dict(token="new-token")),
)
)
monkeypatch.setattr("requests.post", post)
with set_temporary_config(
{"cloud.graphql": "http://my-cloud.foo", "cloud.auth_token": "secret_token"}
):
client = Client()
with pytest.raises(requests.HTTPError) as exc:
result = client.graphql("{}")
assert exc.value is error
assert post.call_count == 3 # first call -> refresh token -> last call
assert post.call_args[0][0] == "http://my-cloud.foo"
assert client.token == "new-token"
def test_calculate_suites_fallback(self):
response = Mock()
response.status_code = requests.codes.SERVICE_UNAVAILABLE
evg = Mock()
evg.test_stats.side_effect = requests.HTTPError(response=response)
main = grs.Main(evg)
main.options = Mock()
main.options.execution_time_minutes = 10
main.config_options = grs.ConfigOptions(2, 15, "project", "", 1, True,
"task", "suite", "variant")
main.list_tests = Mock(
return_value=["test{}.js".format(i) for i in range(100)])
suites = main.calculate_suites(_DATE, _DATE)
self.assertEqual(main.config_options.fallback_num_sub_suites,
len(suites))
for suite in suites:
self.assertEqual(50, len(suite.tests))
def _check_response(self, response):
"""Check our responses for a good response code.
Parameters
----------
response : str
API call response
Raises
-------
requests.HTTPError
Raises this error if we get a response status code that is not in
the 200 range.
"""
if response.status_code // 100 == 2:
return True
else:
raise requests.HTTPError('{} ==> {}'.format(
response.status_code, response.text))
def pins(swlat, swlng, nelat, nelng, pintypes='stop', *, raw=False):
"""
DVB Map Pins
(GET https://www.dvb.de/apps/map/pins)
:param swlat: South-West Bounding Box Latitude
:param swlng: South-West Bounding Box Longitude
:param nelat: North-East Bounding Box Latitude
:param nelng: North-East Bounding Box Longitude
:param pintypes: Types to search for, defaults to 'stop'
:param raw: Return raw response
:return:
"""
try:
swlat, swlng = wgs_to_gk4(swlat, swlng)
nelat, nelng = wgs_to_gk4(nelat, nelng)
r = requests.get(
url='https://www.dvb.de/apps/map/pins',
params={
'showlines': 'true',
'swlat': swlat,
'swlng': swlng,
'nelat': nelat,
'nelng': nelng,
'pintypes': pintypes,
},
)
if r.status_code == 200:
response = json.loads(r.content.decode('utf-8'))
else:
raise requests.HTTPError('HTTP Status: {}'.format(r.status_code))
except requests.RequestException as e:
print('Failed to access DVB map pins app. Request Exception', e)
response = None
if response is None:
return None
return response if raw else [
pins_return_results(line, pintypes) for line in response
]
def fetch_items(cls):
""" Fetch a list of items via the game's npc_items.txt
Fetches npc_items.txt as JSON via Dotabuff's d2vpk repository, and parses it for data.
Falls back to data stored on the file-system in case of a HTTPError or KeyError.
Only retrieves ID and token for now, but there's a ton more data available should we ever need it.
Returns:
An array of Item objects.
"""
try:
req = requests.get(ITEM_DATA_URL)
# Raise HTTPError if we don't get HTTP OK
if req.status_code != requests.codes.ok:
raise requests.HTTPError("Response not HTTP OK")
# Fetch relevant pieces of data from JSON data
input_items = req.json()['DOTAAbilities']
output_items = []
# Iterate through heries, create an instance of this class for each.
for key, item in input_items.items():
# Skip these keys - they're not item definitions
if key in ["Version"]:
continue
output_items.append(cls(_id=int(item.get('ID')), token=key))
return output_items
except (steam.api.HTTPError, KeyError):
sentry.captureMessage('Item.fetch_items failed',
exc_info=sys.exc_info)
# Try to get data from existing cache entry
data = fs_cache.cache.get(cls._CACHE_KEY, ignore_expiry=True)
# Return data if we have any, else return an empty list()
return data or list()
def __call__(self, method, path, params=None, json=None, check=True):
"""
:type check: bool | dict[int:Exception]
"""
r = self._session.request(method,
'{}/repos/{}/{}'.format(
self._url, self._repo, path),
params=params,
json=json)
if check:
if isinstance(check, collections.Mapping):
exc = check.get(r.status_code)
if exc:
raise exc(r.text)
if r.status_code >= 400:
headers = '\n'.join('\t%s: %s' % (h, v)
for h, v in r.headers.items())
if r.headers.get('content-type', '').startswith(
('application/json', 'application/javascript')):
body = r.json()
elif r.encoding is not None:
body = utils.shorten(r.text, 200)
else:
body = utils.shorten(r.content, 200)
_logger.error(
"%(method)s /%(repo)s/%(path)s\n=> %(status)d %(reason)s\n%(headers)s\n\n\t%(body)r\n====================",
{
'status': r.status_code,
'reason': r.reason,
'method': method,
'repo': self._repo,
'path': path,
'headers': headers,
'body': body
})
if not isinstance(body, (bytes, str)):
raise requests.HTTPError(json_.dumps(body, indent=4),
response=r)
r.raise_for_status()
return r
def execute(self, document, variable_values=None, timeout=None):
# type: (Node, dict, int) -> ExecutionResult
"""Execute the provided document AST against the configured remote server.
This uses the requests library to perform a HTTP POST request to the remote server.
:param document: GraphQL query as AST Node object.
:param variable_values: Dictionary of input parameters (Default: None).
:param timeout: Specifies a default timeout for requests (Default: None).
:return: The result of execution. `data` is the result of executing the query, `errors` is null if no errors
occurred, and is a non-empty array if an error occurred.
"""
query_str = print_ast(document)
payload = {'query': query_str, 'variables': variable_values or {}}
data_key = 'json' if self.use_json else 'data'
post_args = {
'headers': self.headers,
'auth': self.auth,
'cookies': self.cookies,
'timeout': timeout or self.default_timeout,
'verify': self.verify,
data_key: payload
}
# Pass kwargs to requests post method
post_args.update(self.kwargs)
response = requests.post(self.url, **post_args)
try:
result = response.json()
if not isinstance(result, dict):
raise ValueError
except ValueError:
result = {}
if 'errors' not in result and 'data' not in result:
response.raise_for_status()
raise requests.HTTPError("Server did not return a GraphQL result",
response=response)
return ExecutionResult(errors=result.get('errors'),
data=result.get('data'))
def nominatim_query(url, *, q):
if url[-1] == '/':
url = url[:-1]
res = start = end = None # Avoids warnings
for i in range(5):
if i > 0:
time.sleep(1)
PROM_NOMINATIM_REQS.inc() # Count all requests
start = time.perf_counter()
if isinstance(q, (tuple, list)):
# Batch query
res = _nominatim_session.get(
url + '/search?' +
urlencode({
'batch': json.dumps([{
'q': qe
} for qe in q]),
'format': 'jsonv2',
}), )
else:
# Normal query
res = _nominatim_session.get(
url + '/search?' + urlencode({
'q': q,
'format': 'jsonv2'
}), )
end = time.perf_counter()
if res.status_code not in (502, 503, 504):
break
res.raise_for_status()
# Record time for successful request
PROM_NOMINATIM_REQ_TIME.observe(end - start)
if not res.headers['Content-Type'].startswith('application/json'):
raise requests.HTTPError(
"Response is not JSON for URL: %s" % res.url,
response=res,
)
if isinstance(q, (tuple, list)):
return res.json()['batch']
else:
return res.json()
def import_gists_to_database(db, username, commit=True):
r = requests.get('https://api.github.com/users/{}/gists'.format(username))
if r.status_code != 200:
raise requests.HTTPError()
request_data = r.json() #convert request data into json format
#create db connetion
for gist in request_data:
github_id = gist["id"]
html_url = gist["html_url"]
git_pull_url = gist["git_pull_url"]
git_push_url = gist["git_push_url"]
commits_url = gist["commits_url"]
forks_url = gist["forks_url"]
public = gist["public"]
created_at = gist["created_at"]
updated_at = gist["updated_at"]
comments = gist["comments"]
comments_url = gist["comments_url"]
query = """INSERT INTO gists (github_id, html_url, git_pull_url, git_push_url, commits_url, forks_url, public,
created_at, updated_at, comments, comments_url) VALUES (:github_id, :html_url, :git_pull_url, :git_push_url,
:commits_url, :forks_url, :public, :created_at, :updated_at, :comments, :comments_url)"""
cursor = db.execute(
query, {
'github_id': github_id,
'html_url': html_url,
'git_pull_url': git_pull_url,
'git_push_url': git_push_url,
'commits_url': commits_url,
'forks_url': forks_url,
'public': public,
'created_at': created_at,
'updated_at': updated_at,
'comments': comments,
'comments_url': comments_url
})
def wait_until_task_is_finished(host: str, task_name: str, repo: str, state: str = 'active',
task_type: str = 'running_one_off', **kwargs):
"""Wait until a task is finished
Args:
- host (str): the host for the backup service.
- task_name (str): the task we are looking for.
- task_type [running_one_off, running_tasks]: The type of task we are waiting for.
- state ([active, imported, archived]): The repository state.
- kwargs:
- timeout (int): request timeout.
- retries (int): how many times to poll the endpoint at a maximum.
- user (str): The user to use for the requests.
- passwrod (str): The password to use for the requests.
"""
timeout = int(kwargs.get('timeout', 60))
retries = int(kwargs.get('retries', 10))
user, password = get_user_and_password(**kwargs)
if task_type not in ['running_one_off', 'running_tasks']:
raise ValueError('taks type must be one off [running_one_off, running_tasks]')
for retry in range(retries):
logger.debug(f'Checking if task {task_type} {task_name} is still running. Attempt {retry}')
res = requests.get(f'{host}/api/v1/cluster/self/repository/{state}/{repo}', auth=(user, password),
timeout=timeout)
if res.status_code != 200:
raise requests.HTTPError(f'Unexpected error code: {res.status_code} {res.json()}')
repository = res.json()
logger.debug(f'/api/v1/cluster/self/repository/{state}/{repo} response: {repository}')
if task_type not in repository or task_name not in repository[task_type]:
if retry == 0:
logger.debug('Task finished before first run')
# if no running one offs assume task is finished
return
time.sleep(1 * (retries + 1))
continue
raise AssertionError('Task is not finished')
def _route_call(aircraft_id):
"""
Make a call to the BlueBird LISTROUTE endpoint.
Parameters
----------
aircraft_id: str
A string aircraft identifier. For the BlueSky simulator, this has to be
at least three characters.
Returns
-------
dict :
A dictionary with keys:
``"callsign"``
A string aircraft identifier.
``"next_waypoint"``
A string. Name of waypoint the aircraft is currently headed toward.
``"route_name"``
A string name of the route.
``"route_waypoints"``
A list of strings. All the waypoints on the route.
Notes
-----
If the aircraft has no route information, a dictionary with just
the callsign is returned.
"""
url = construct_endpoint_url(endpoint)
resp = requests.get(
url, params={config_param("query_aircraft_id"): aircraft_id})
if resp.status_code == 200:
return json.loads(resp.text)
elif response.status == config_param("status_code_aircraft_has_no_route"):
return {config_param("query_aircraft_id"): aircraft_id}
else:
raise requests.HTTPError(resp.text)
