def sanityCheck(args):
import requests
from urllib.parse import urlparse
if not Scrap.checkUrlValid(args.url):
raise requests.ConnectionError(f"invalid url {args.url}")
if args.navdepth < -1:
raise OverflowError("depth range should be [-1, ... ).")
main(args=args)
def connect(self):
logging.info("Connecting to neviweb")
login_parameters = {'email': self._email,
'password': self._password,
'stayConnected': "0"}
r = requests.post("https://neviweb.com/api/login",
data=login_parameters,
headers=self._headers,
timeout=self.timeout)
response = r.json()
logging.debug(response)
if "code" in response:
raise requests.ConnectionError(response["code"] + ": " + response.get("message", "None"))
self._session_id = response.get('session', None)
if self._session_id:
self._headers['Session-Id'] = self._session_id
else:
raise requests.ConnectionError("Invalid response")
def test_objects_load_screen_nested(terminal, use_ascii):
terminal.config['ascii'] = use_ascii
with terminal.loader(message='Outer'):
with terminal.loader(message='Inner'):
raise requests.ConnectionError()
assert False # Should never be reached
assert isinstance(terminal.loader.exception, requests.ConnectionError)
assert terminal.loader.depth == 0
assert not terminal.loader._is_running
assert not terminal.loader._animator.is_alive()
def test_objects_load_screen_exception_handled(terminal, stdscr, use_ascii):
terminal.config['ascii'] = use_ascii
# Raising a handled exception should get stored on the loaders
with terminal.loader(delay=0):
assert terminal.loader._animator.is_alive()
raise requests.ConnectionError()
assert not terminal.loader._is_running
assert not terminal.loader._animator.is_alive()
assert isinstance(terminal.loader.exception, requests.ConnectionError)
error_message = 'ConnectionError'.encode('ascii' if use_ascii else 'utf-8')
stdscr.subwin.addstr.assert_called_with(1, 1, error_message)
def mocked_get(url, params, **options):
calls.append(url)
if len(calls) < 3:
raise requests.ConnectionError('unable to connect')
return Response({
'bugs': [{
'id': 123456789,
'status': 'NEW',
'resolution': '',
'summary': 'Some Summary',
}],
})
def build_iterator(self,
service,
indicator_type,
risk_rule: Optional[str] = None):
"""Retrieves all entries from the feed.
Args:
service (str): The service from recorded future. Can be 'connectApi' or 'fusion'
indicator_type (str): The indicator type. Can be 'domain', 'ip', 'hash' or 'url'
risk_rule (str): A risk rule that limits the fetched indicators
Returns:
list of feed dictionaries.
"""
_session = requests.Session()
prepared_request = self._build_request(service, indicator_type,
risk_rule)
# this is to honour the proxy environment variables
rkwargs = _session.merge_environment_settings(
prepared_request.url,
{},
None,
None,
None # defaults
)
rkwargs['stream'] = True
rkwargs['verify'] = self._verify
rkwargs['timeout'] = self.polling_timeout
try:
response = _session.send(prepared_request, **rkwargs)
except requests.ConnectionError as e:
raise requests.ConnectionError(
f'Failed to establish a new connection: {str(e)}')
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
if "Insufficient credits" in response.text:
return_error(
"'Insufficient Credits' error was returned from Recorded Future. \n"
"Try increasing the integration's fetch interval in order to decrease the amount of API"
" requests made to Recorded Future. ")
else:
return_error('{} - exception in request: {} {}'.format(
self.SOURCE_NAME, response.status_code, response.content))
if service == 'connectApi':
response_content = gzip.decompress(response.content)
response_content = response_content.decode('utf-8')
with open("response.txt", "w") as f:
f.write(response_content)
else:
with open("response.txt", "w") as f:
f.write(response.text)
def piwik_UsersManager_setUserAccess(piwik_url, piwik_api_token, userLogin,
access, idSites):
params = {"module": "API", "format": "JSON", "token_auth": piwik_api_token}
params["method"] = "UsersManager.setUserAccess"
params["userLogin"] = userLogin
params["access"] = access
params["idSites"] = idSites
result = requests.get(piwik_url, params=params, timeout=15)
if result.status_code != 200:
raise requests.ConnectionError()
content = result.json(
) # Content should be either an empty array, or an array like [{"idsite":"44"}]
# print "piwik_UsersManager_setUserAccess", content
if not content:
raise requests.ConnectionError()
user_access_is_set = ("result" in content
and content["result"] == "success")
return user_access_is_set
def _sync_users_and_groups(self, users_and_groups, apply_changes=True, remove_deleted=False):
"""
Syncs users and groups.
:param users_and_groups: List of users and groups to sync.
:type users_and_groups: UsersAndGroups
:param apply_changes: If true, changes will be applied. If not, then it just says what will happen.
:type apply_changes: bool
:param remove_deleted: Flag to removed deleted users. If true, delete. Cannot be used with batch_size.
:type remove_deleted: bool
:returns: The response from the sync.
"""
is_valid = users_and_groups.is_valid()
if not is_valid[0]:
# print("Invalid user and group structure.")
raise Exception("Invalid users and groups")
url = self.format_url(SyncUserAndGroups.SYNC_ALL_URL)
logging.debug("calling %s" % url)
json_str = users_and_groups.to_json()
logging.info("%s" % json_str)
json.loads(json_str) # do a load to see if it breaks due to bad JSON.
tmp_file = "/tmp/ug.json.%d" % time.time()
with open(tmp_file, "w") as out:
out.write(json_str)
params = {
"principals": (tmp_file, open(tmp_file, "rb"), "text/json"),
"applyChanges": json.dumps(apply_changes),
"removeDeleted": json.dumps(remove_deleted),
}
if self.global_password:
params["password"] = self.global_password
response = self.session.post(url, files=params, cookies=self.cookies)
if response.status_code == 200:
logging.info("Successfully synced users and groups.")
logging.info(response.text.encode("utf-8"))
return response
else:
logging.error("Failed synced users and groups.")
logging.info(response.text.encode("utf-8"))
with open("ts_users_and_groups.json", "w") as outfile:
outfile.write(json_str.encode("utf-8"))
raise requests.ConnectionError(
"Error syncing users and groups (%d)" % response.status_code,
response.text,
)
def piwik_UsersManager_getUserByEmail(piwik_url, piwik_api_token, userEmail):
params = {"module": "API", "format": "JSON", "token_auth": piwik_api_token}
params["method"] = "UsersManager.getUserByEmail"
params[
"userEmail"] = userEmail # Piwik has two different fields for login and email, but a user can have the same value as login and email
result = requests.get(piwik_url, params=params, timeout=15)
if result.status_code != 200:
raise requests.ConnectionError()
content = result.json(
) # returns something like [{"login":"******","email":"*****@*****.**"}] or {"result":"error","message":"L'utilisateur '*****@*****.**' est inexistant."}
# print "piwik_UsersManager_getUserByEmail", content
if not content:
raise requests.ConnectionError()
if "result" in content and content["result"] == "error":
return False
elif not isinstance(content, list):
raise requests.ConnectionError()
else:
return content
def get_ip_address():
"""Tries to get the computer's public IP."""
try:
request_output = requests.request('GET',
'https://api.ipify.org',
timeout=REQUEST_TIMEOUT_SECONDS)
ip_address = ipaddress.ip_address(request_output.text)
public_ip = str(ip_address)
except:
raise requests.ConnectionError('Unable to get public IP')
return public_ip
def test_retry_automatically_connection_error(self):
# Deliveries that received a connection error are automatically
# retried until 24 hours after the initial attempt.
job, reqs = self.makeAndRunJob(
raises=requests.ConnectionError('Connection refused'))
self.assertTrue(job.retry_automatically)
job.json_data['date_first_sent'] = (
job.date_first_sent - timedelta(hours=23)).isoformat()
self.assertTrue(job.retry_automatically)
job.json_data['date_first_sent'] = (
job.date_first_sent - timedelta(hours=24)).isoformat()
self.assertFalse(job.retry_automatically)
def testConnectionErrorRecovery(self):
instrumentor = RequestsInstrumentor()
# When we can't connect at all (server not listening), we get a
# requests.exceptions.ConnectionError but the response object is None.
err_response = requests.ConnectionError("Error", response=None)
instrumentor.responses = [err_response, _make_200("Good")]
with instrumentor.instrument():
manager = MockHTTPManager()
result = manager.OpenServerEndpoint("control")
self.assertEqual(result.data, "Good")
def test_connection_error(self):
# Attempts that fail to connect have a connection_error rather
# than a response.
[call], result = self.sendToWebhook(
body=requests.ConnectionError('Connection refused'))
self.assertThat(
result,
MatchesDict({
'request': self.request_matcher,
'connection_error': Equals('Connection refused'),
}))
self.assertIsInstance(call.response, requests.ConnectionError)
def get_privileges_for_group(self, group_name):
"""
Gets the current privileges for a given group.
:param group_name: Name of the group to get privileges for.
:returns: A list of privileges.
:rtype: list of str
"""
url = self.format_url(
SetGroupPrivilegesAPI.METADATA_LIST_URL) + "&pattern=" + group_name
response = self.session.get(url, cookies=self.cookies)
if response.status_code == 200: # success
results = json.loads(response.text)
try:
group_id = results[0][
"id"] # should always be present, but might want to add try / catch.
detail_url = SetGroupPrivilegesAPI.METADATA_DETAIL_URL.format(
guid=group_id)
detail_url = self.format_url(detail_url)
detail_response = self.session.get(detail_url,
cookies=self.cookies)
if detail_response.status_code == 200: # success
privileges = json.loads(detail_response.text)["privileges"]
return privileges
else:
logging.error("Failed to get privileges for group %s" %
group_name)
raise requests.ConnectionError(
"Error (%d) setting privileges for group %s. %s" %
(response.status_code, group_name, response.text))
except Exception:
logging.error("Error getting group details.")
raise
else:
logging.error("Failed to get privileges for group %s" % group_name)
raise requests.ConnectionError(
"Error (%d) setting privileges for group %s. %s" %
(response.status_code, group_name, response.text))
def get_comment_from_url(url):
global STABLE_PROXY, PROXY_TIMEOUT_TIMER, IS_SURE_USE_PROXY
try:
response = get_proxy_request(url)
if response.status_code == 200:
if PROXY_TIMEOUT_TIMER: PROXY_TIMEOUT_TIMER = None
datas = response.json()
if datas["code"] == "100000":
print("URL: %s" % (url))
html = datas["data"]["html"]
try:
comment_list = parse_html_to_content(html)
writer_queue(comment_list)
except:
raise ValueError("解析失败 %s skip.." %
(traceback.format_exc()))
else:
totalpage = datas["data"]["page"]["totalpage"]
return len(comment_list), totalpage
else:
raise IOError("错误的响应码: %s cause: %s" %
(datas["code"], datas["msg"]))
elif response.status_code == 414:
STABLE_PROXY = None
print(requests.ConnectionError("请求失败 (414: IP被禁)-切换: %s" % (url)))
IS_SURE_USE_PROXY = not IS_SURE_USE_PROXY
PROXY_TIMEOUT_TIMER = time.time()
elif response.status_code == 400:
print(requests.ConnectionError("无效的代理 (400): %s" % (url)))
STABLE_PROXY = None
# 判断无效的代理是否超过5min, 微博5min内会解禁被封的IP-可以切回原来的IP
swith_proxy()
except AttributeError as ex:
pass
except requests.exceptions.ProxyError:
swith_proxy()
except BaseException as ex:
pass
def getCuries(curies_location):
# FIXME this will 'fail' silently ...
# probably need to warn?
try:
if curies_location is None:
raise TypeError('curies should always fail over to '
'.config/pyontutils/curie_map.yaml '
'what have you done!?')
# TODO staleness check
with open(curies_location, 'rt') as f:
curie_map = yaml.safe_load(f)
return curie_map
except (FileNotFoundError, NotADirectoryError) as e:
# retrieving stuff over the net is bad
# but having stale curies seems worse?
# current options for retrieval
# github, best, but rate limit problems
# ontquery CURIE_MAP
# ../nifstd/scigraph/curie_map.yaml
# Cypher()
# strong coupling between the version
# in this repo at this commit is what
# causes the issue, github is the best
# solution, so write once to a known location
if curies_location == devconfig.curies.default:
master_blob = 'https://github.com/tgbugs/pyontutils/blob/master/'
raw_path = '/nifstd/scigraph/curie_map.yaml?raw=true'
curies_url = master_blob + raw_path
resp = requests.get(curies_url)
if resp.ok:
clp = Path(curies_location)
if not clp.parent.exists():
clp.parent.mkdir()
with open(clp, 'wt') as f:
f.write(resp.text)
print(f'Wrote {clp} from {curies_url}')
return getCuries(curies_location)
else:
raise requests.ConnectionError(resp.request, resp)
else:
raise TypeError(
f'{curies_location} does not exist and '
f'is not at the default {devconfig.curies.default} '
'so we will not write to it. You can update it '
'manually if you want to keep it at that location.')
def start(self):
"""Establishes a connect to the Supervisor, and then uses this to
establish a connection with a running robot. It then initialises all
connections, ensuring API-side callbacks are accessible.
"""
# Establish a connection to the supervisor (throw an error on failure)
print("Waiting to establish connection to a running supervisor ... ",
end='')
sys.stdout.flush()
start_time = time.time()
connected = False
while not connected and time.time() - start_time < TIMEOUT_SUPERVISOR:
try:
self._query("/", BenchBot.RouteType.EXPLICIT)
connected = True
except:
pass
time.sleep(3)
if not connected:
raise requests.ConnectionError(
"Could not find a BenchBot supervisor @ '%s'. "
"Are you sure it is available?" % self.supervisor_address)
print("Connected!")
# Wait until the robot is running
print("Waiting to establish connection to a running robot ... ",
end='')
sys.stdout.flush()
while (not self._query("is_running",
BenchBot.RouteType.ROBOT)['is_running']):
time.sleep(0.1)
print("Connected!")
# Get references to all of the API callbacks in robot config
self._connection_callbacks = {
k: BenchBot._attempt_connection_imports(v)
for k, v in self._query('robot', BenchBot.RouteType.CONFIG)
['connections'].items()
}
# Ensure we are starting in a clean robot state
if (self._query('selected_environment',
BenchBot.RouteType.ROBOT)['number'] != 0):
print(
"Robot detected not to be in the first scene. "
"Performing restart ... ",
end='')
sys.stdout.flush()
self._query('restart', BenchBot.RouteType.ROBOT)
print("Done.")
else:
self.reset()
def connect(self):
"""
Establish a connection with QMMF IPC webserver on the camera.
This API also sets the `session_token` attribute from the token
obtained from the camera.
Returns
-------
bool
True if the connection was successful.
False on failure.
Raises
------
Timeout
When the request times out on the connect request.
RequestException
The request is not correctly formed.
"""
with requests.session() as mysession:
try:
payload = "{\"username\": \"%s\", \"userpwd\": \"%s\"}" % (
self.username, self.password)
url = "http://" + self.ip_address + ":" + self._port + "/login"
#self.logger.info("API: " + url + ",data: " + json.dumps(payload,sort_keys=True))
print("URL: " + url + " data: " + payload +
"Sent from IPCprovider.connect")
#response = mysession.post(url, data=json.dumps(payload, sort_keys= True))
response = mysession.post(url, data=payload)
print("Response: " + response.text)
self.logger.info("LOGIN RESPONSE: " + response.text)
json_resp = json.loads(response.text)
if json_resp['status']:
self._session_token = response.headers['Set-Cookie']
print("connection established with session token: [%s]" %
self._session_token)
return True
else:
raise requests.ConnectionError(
"Failed to connect. Server returned status=False")
except requests.exceptions.Timeout:
# TODO: user should have a way to figure out if required services are running or not? maybe some simple URL
self._show_error(
"Timeout: Please check that the device is up and running and the IPC service is available"
)
raise
except requests.exceptions.RequestException as e:
self.logger.exception(e.strerror)
raise
def test_create_new_domain_url_path_connection_fails(self):
auth_entity = testutil.FakeAuthEntity(
id='x',
user_json=json_dumps({'urls': [{
'value': 'http://flaky/foo'
}]}))
self.expect_requests_get('http://flaky').AndRaise(
requests.ConnectionError('DNS lookup failed for URL: http://bad/'))
self.mox.ReplayAll()
source = FakeSource.create_new(self.handler, auth_entity=auth_entity)
self.assertEqual(['http://flaky/foo'], source.domain_urls)
self.assertEqual(['flaky'], source.domains)
def _api_parse(self, r):
"""if JSON parsing failed - raising requests.ConnectionError
Override to parse JSON specific errors."""
try:
j = r.json()
except Exception as e: # TO JSON ERROR?
j = {}
self.logger.warn('request error - %s: %r', e.__class__.__name__, e)
# will wait and try again
raise requests.ConnectionError(
'Error converting server reponse to JSON...')
else:
return j
def test_network_failure(
head: mock.MagicMock,
run: mock.MagicMock,
):
head.side_effect = requests.ConnectionError()
with pytest.raises(requests.ConnectionError):
login.Auth()._run_server()
run.assert_not_called()
head.side_effect = requests.RequestException()
with pytest.raises(requests.RequestException):
login.Auth()._run_server()
run.assert_not_called()
def test_get_local_slave_state_connection_error(
mock_getfqdn,
mock_requests_get,
):
fake_request = requests.Request('GET', url='doesnt_matter')
mock_getfqdn.return_value = 'fake_hostname'
mock_requests_get.side_effect = requests.ConnectionError(
'fake_message',
request=fake_request,
)
with raises(mesos_tools.MesosSlaveConnectionError):
mesos_tools.get_local_slave_state()
def submit_result(self, **kwargs):
if 'id' in kwargs:
raise ValueError("Keyword argument with name 'id' not allowed.")
token = kwargs.pop('token')
data = kwargs.copy()
data['id'] = self.id
r = requests.post(url=self._add_result_url(),
json=data,
headers={'Authorization': f'Bearer {token}'})
if r.status_code != 201:
raise requests.ConnectionError(
"Expected status code 201, but got {}".format(r.status_code))
return r
def test_result_eventual_success(mock_civis):
# Test that we can get a result back from a succeeded job,
# even if we need to retry a few times to succeed with the download.
callback = mock.MagicMock()
exc = requests.ConnectionError()
se = make_to_file_mock('spam', max_n_err=2, exc=exc)
mock_civis.io.civis_to_file.side_effect = se
fut = ContainerFuture(1, 2, client=mock.MagicMock())
fut.set_result(Response({'state': 'success'}))
res = civis.parallel._CivisBackendResult(fut, callback)
assert res.get() == 'spam'
assert callback.call_count == 1
def getTrades(self, market, timestamp=None):
if timestamp == None:
timestamp = time.time(
) - 60 * 60 * 24 # se solicita por defecto las últimas 24 horas
url = self.createUrl('/markets/' + market + '/trades',
{'timestamp': timestamp})
self.response = self.consume(url)
body = self.response.json()
if self.response.status_code != 200:
raise requests.ConnectionError(
'No fue posible obtener los intercambios del mercado ' +
market + ': ' + body['message'])
return body['trades']
def test_registration_check_conn_error(test_connection, get_proxies,
_init_session, _):
'''
Can't connect, run connection test
Returns False
'''
config = Mock(legacy_upload=True, base_url='example.com')
conn = InsightsConnection(config)
conn.get = MagicMock()
conn.get.side_effect = requests.ConnectionError()
assert conn.api_registration_check() is False
test_connection.assert_called_once()
def check_http_response(response, expected_status_code):
""" validate http status code """
status = response.status_code
if status == expected_status_code:
return True
if status == 401: # unauthorized
msg = "ERROR: status = %s - Invalid credentials?" % status
raise requests.ConnectionError(msg)
if status in [404, 500]: # unknown resource, operation cannot be performed
""" response 404 cannot be jsonified """
error = "Operation cannot be performed, retry with valid parameters"
raise IOError(error)
return False
def close(self):
"""
Logout of the current session.
"""
if self.authToken is None:
raise requests.ConnectionError('No session token found.')
url = self.sisenseUrl + "/api/v1/authentication/logout"
headers = {'Authorization': self.authToken}
response = requests.request("GET", url, headers=headers).json()
if response['success']:
self.authToken = None
def test_persistent_connection_error(client):
for _i in range(6):
load_response(
RegisteredResponse(
path="https://foo.api.globus.org/bar",
body=requests.ConnectionError("foo-err"),
))
load_response(
RegisteredResponse(path="https://foo.api.globus.org/bar",
json={"baz": 1}))
with pytest.raises(globus_sdk.GlobusConnectionError):
client.get("/bar")
def __perform_crumb_call(self, session):
crumb_response = session.get(url=self.api_settings.crumb_url,
auth=self.api_settings.auth)
if crumb_response.status_code != 200:
logger.error("url=%s\nheaders=%s\nbody=%s\n",
crumb_response.request.url,
crumb_response.request.headers,
crumb_response.request.body)
logger.error(self.api_settings.auth)
logger.error(crumb_response.text)
raise requests.ConnectionError('Could not issue Jenkins crumb.',
response=crumb_response)
return crumb_response.json()
