def sanityCheck(args): import requests from urllib.parse import urlparse if not Scrap.checkUrlValid(args.url): raise requests.ConnectionError(f"invalid url {args.url}") if args.navdepth < -1: raise OverflowError("depth range should be [-1, ... ).") main(args=args)
def connect(self): logging.info("Connecting to neviweb") login_parameters = {'email': self._email, 'password': self._password, 'stayConnected': "0"} r = requests.post("https://neviweb.com/api/login", data=login_parameters, headers=self._headers, timeout=self.timeout) response = r.json() logging.debug(response) if "code" in response: raise requests.ConnectionError(response["code"] + ": " + response.get("message", "None")) self._session_id = response.get('session', None) if self._session_id: self._headers['Session-Id'] = self._session_id else: raise requests.ConnectionError("Invalid response")
def test_objects_load_screen_nested(terminal, use_ascii): terminal.config['ascii'] = use_ascii with terminal.loader(message='Outer'): with terminal.loader(message='Inner'): raise requests.ConnectionError() assert False # Should never be reached assert isinstance(terminal.loader.exception, requests.ConnectionError) assert terminal.loader.depth == 0 assert not terminal.loader._is_running assert not terminal.loader._animator.is_alive()
def test_objects_load_screen_exception_handled(terminal, stdscr, use_ascii): terminal.config['ascii'] = use_ascii # Raising a handled exception should get stored on the loaders with terminal.loader(delay=0): assert terminal.loader._animator.is_alive() raise requests.ConnectionError() assert not terminal.loader._is_running assert not terminal.loader._animator.is_alive() assert isinstance(terminal.loader.exception, requests.ConnectionError) error_message = 'ConnectionError'.encode('ascii' if use_ascii else 'utf-8') stdscr.subwin.addstr.assert_called_with(1, 1, error_message)
def mocked_get(url, params, **options): calls.append(url) if len(calls) < 3: raise requests.ConnectionError('unable to connect') return Response({ 'bugs': [{ 'id': 123456789, 'status': 'NEW', 'resolution': '', 'summary': 'Some Summary', }], })
def build_iterator(self, service, indicator_type, risk_rule: Optional[str] = None): """Retrieves all entries from the feed. Args: service (str): The service from recorded future. Can be 'connectApi' or 'fusion' indicator_type (str): The indicator type. Can be 'domain', 'ip', 'hash' or 'url' risk_rule (str): A risk rule that limits the fetched indicators Returns: list of feed dictionaries. """ _session = requests.Session() prepared_request = self._build_request(service, indicator_type, risk_rule) # this is to honour the proxy environment variables rkwargs = _session.merge_environment_settings( prepared_request.url, {}, None, None, None # defaults ) rkwargs['stream'] = True rkwargs['verify'] = self._verify rkwargs['timeout'] = self.polling_timeout try: response = _session.send(prepared_request, **rkwargs) except requests.ConnectionError as e: raise requests.ConnectionError( f'Failed to establish a new connection: {str(e)}') try: response.raise_for_status() except requests.exceptions.HTTPError: if "Insufficient credits" in response.text: return_error( "'Insufficient Credits' error was returned from Recorded Future. \n" "Try increasing the integration's fetch interval in order to decrease the amount of API" " requests made to Recorded Future. ") else: return_error('{} - exception in request: {} {}'.format( self.SOURCE_NAME, response.status_code, response.content)) if service == 'connectApi': response_content = gzip.decompress(response.content) response_content = response_content.decode('utf-8') with open("response.txt", "w") as f: f.write(response_content) else: with open("response.txt", "w") as f: f.write(response.text)
def piwik_UsersManager_setUserAccess(piwik_url, piwik_api_token, userLogin, access, idSites): params = {"module": "API", "format": "JSON", "token_auth": piwik_api_token} params["method"] = "UsersManager.setUserAccess" params["userLogin"] = userLogin params["access"] = access params["idSites"] = idSites result = requests.get(piwik_url, params=params, timeout=15) if result.status_code != 200: raise requests.ConnectionError() content = result.json( ) # Content should be either an empty array, or an array like [{"idsite":"44"}] # print "piwik_UsersManager_setUserAccess", content if not content: raise requests.ConnectionError() user_access_is_set = ("result" in content and content["result"] == "success") return user_access_is_set
def _sync_users_and_groups(self, users_and_groups, apply_changes=True, remove_deleted=False): """ Syncs users and groups. :param users_and_groups: List of users and groups to sync. :type users_and_groups: UsersAndGroups :param apply_changes: If true, changes will be applied. If not, then it just says what will happen. :type apply_changes: bool :param remove_deleted: Flag to removed deleted users. If true, delete. Cannot be used with batch_size. :type remove_deleted: bool :returns: The response from the sync. """ is_valid = users_and_groups.is_valid() if not is_valid[0]: # print("Invalid user and group structure.") raise Exception("Invalid users and groups") url = self.format_url(SyncUserAndGroups.SYNC_ALL_URL) logging.debug("calling %s" % url) json_str = users_and_groups.to_json() logging.info("%s" % json_str) json.loads(json_str) # do a load to see if it breaks due to bad JSON. tmp_file = "/tmp/ug.json.%d" % time.time() with open(tmp_file, "w") as out: out.write(json_str) params = { "principals": (tmp_file, open(tmp_file, "rb"), "text/json"), "applyChanges": json.dumps(apply_changes), "removeDeleted": json.dumps(remove_deleted), } if self.global_password: params["password"] = self.global_password response = self.session.post(url, files=params, cookies=self.cookies) if response.status_code == 200: logging.info("Successfully synced users and groups.") logging.info(response.text.encode("utf-8")) return response else: logging.error("Failed synced users and groups.") logging.info(response.text.encode("utf-8")) with open("ts_users_and_groups.json", "w") as outfile: outfile.write(json_str.encode("utf-8")) raise requests.ConnectionError( "Error syncing users and groups (%d)" % response.status_code, response.text, )
def piwik_UsersManager_getUserByEmail(piwik_url, piwik_api_token, userEmail): params = {"module": "API", "format": "JSON", "token_auth": piwik_api_token} params["method"] = "UsersManager.getUserByEmail" params[ "userEmail"] = userEmail # Piwik has two different fields for login and email, but a user can have the same value as login and email result = requests.get(piwik_url, params=params, timeout=15) if result.status_code != 200: raise requests.ConnectionError() content = result.json( ) # returns something like [{"login":"******","email":"*****@*****.**"}] or {"result":"error","message":"L'utilisateur '*****@*****.**' est inexistant."} # print "piwik_UsersManager_getUserByEmail", content if not content: raise requests.ConnectionError() if "result" in content and content["result"] == "error": return False elif not isinstance(content, list): raise requests.ConnectionError() else: return content
def get_ip_address(): """Tries to get the computer's public IP.""" try: request_output = requests.request('GET', 'https://api.ipify.org', timeout=REQUEST_TIMEOUT_SECONDS) ip_address = ipaddress.ip_address(request_output.text) public_ip = str(ip_address) except: raise requests.ConnectionError('Unable to get public IP') return public_ip
def test_retry_automatically_connection_error(self): # Deliveries that received a connection error are automatically # retried until 24 hours after the initial attempt. job, reqs = self.makeAndRunJob( raises=requests.ConnectionError('Connection refused')) self.assertTrue(job.retry_automatically) job.json_data['date_first_sent'] = ( job.date_first_sent - timedelta(hours=23)).isoformat() self.assertTrue(job.retry_automatically) job.json_data['date_first_sent'] = ( job.date_first_sent - timedelta(hours=24)).isoformat() self.assertFalse(job.retry_automatically)
def testConnectionErrorRecovery(self): instrumentor = RequestsInstrumentor() # When we can't connect at all (server not listening), we get a # requests.exceptions.ConnectionError but the response object is None. err_response = requests.ConnectionError("Error", response=None) instrumentor.responses = [err_response, _make_200("Good")] with instrumentor.instrument(): manager = MockHTTPManager() result = manager.OpenServerEndpoint("control") self.assertEqual(result.data, "Good")
def test_connection_error(self): # Attempts that fail to connect have a connection_error rather # than a response. [call], result = self.sendToWebhook( body=requests.ConnectionError('Connection refused')) self.assertThat( result, MatchesDict({ 'request': self.request_matcher, 'connection_error': Equals('Connection refused'), })) self.assertIsInstance(call.response, requests.ConnectionError)
def get_privileges_for_group(self, group_name): """ Gets the current privileges for a given group. :param group_name: Name of the group to get privileges for. :returns: A list of privileges. :rtype: list of str """ url = self.format_url( SetGroupPrivilegesAPI.METADATA_LIST_URL) + "&pattern=" + group_name response = self.session.get(url, cookies=self.cookies) if response.status_code == 200: # success results = json.loads(response.text) try: group_id = results[0][ "id"] # should always be present, but might want to add try / catch. detail_url = SetGroupPrivilegesAPI.METADATA_DETAIL_URL.format( guid=group_id) detail_url = self.format_url(detail_url) detail_response = self.session.get(detail_url, cookies=self.cookies) if detail_response.status_code == 200: # success privileges = json.loads(detail_response.text)["privileges"] return privileges else: logging.error("Failed to get privileges for group %s" % group_name) raise requests.ConnectionError( "Error (%d) setting privileges for group %s. %s" % (response.status_code, group_name, response.text)) except Exception: logging.error("Error getting group details.") raise else: logging.error("Failed to get privileges for group %s" % group_name) raise requests.ConnectionError( "Error (%d) setting privileges for group %s. %s" % (response.status_code, group_name, response.text))
def get_comment_from_url(url): global STABLE_PROXY, PROXY_TIMEOUT_TIMER, IS_SURE_USE_PROXY try: response = get_proxy_request(url) if response.status_code == 200: if PROXY_TIMEOUT_TIMER: PROXY_TIMEOUT_TIMER = None datas = response.json() if datas["code"] == "100000": print("URL: %s" % (url)) html = datas["data"]["html"] try: comment_list = parse_html_to_content(html) writer_queue(comment_list) except: raise ValueError("解析失败 %s skip.." % (traceback.format_exc())) else: totalpage = datas["data"]["page"]["totalpage"] return len(comment_list), totalpage else: raise IOError("错误的响应码: %s cause: %s" % (datas["code"], datas["msg"])) elif response.status_code == 414: STABLE_PROXY = None print(requests.ConnectionError("请求失败 (414: IP被禁)-切换: %s" % (url))) IS_SURE_USE_PROXY = not IS_SURE_USE_PROXY PROXY_TIMEOUT_TIMER = time.time() elif response.status_code == 400: print(requests.ConnectionError("无效的代理 (400): %s" % (url))) STABLE_PROXY = None # 判断无效的代理是否超过5min, 微博5min内会解禁被封的IP-可以切回原来的IP swith_proxy() except AttributeError as ex: pass except requests.exceptions.ProxyError: swith_proxy() except BaseException as ex: pass
def getCuries(curies_location): # FIXME this will 'fail' silently ... # probably need to warn? try: if curies_location is None: raise TypeError('curies should always fail over to ' '.config/pyontutils/curie_map.yaml ' 'what have you done!?') # TODO staleness check with open(curies_location, 'rt') as f: curie_map = yaml.safe_load(f) return curie_map except (FileNotFoundError, NotADirectoryError) as e: # retrieving stuff over the net is bad # but having stale curies seems worse? # current options for retrieval # github, best, but rate limit problems # ontquery CURIE_MAP # ../nifstd/scigraph/curie_map.yaml # Cypher() # strong coupling between the version # in this repo at this commit is what # causes the issue, github is the best # solution, so write once to a known location if curies_location == devconfig.curies.default: master_blob = 'https://github.com/tgbugs/pyontutils/blob/master/' raw_path = '/nifstd/scigraph/curie_map.yaml?raw=true' curies_url = master_blob + raw_path resp = requests.get(curies_url) if resp.ok: clp = Path(curies_location) if not clp.parent.exists(): clp.parent.mkdir() with open(clp, 'wt') as f: f.write(resp.text) print(f'Wrote {clp} from {curies_url}') return getCuries(curies_location) else: raise requests.ConnectionError(resp.request, resp) else: raise TypeError( f'{curies_location} does not exist and ' f'is not at the default {devconfig.curies.default} ' 'so we will not write to it. You can update it ' 'manually if you want to keep it at that location.')
def start(self): """Establishes a connect to the Supervisor, and then uses this to establish a connection with a running robot. It then initialises all connections, ensuring API-side callbacks are accessible. """ # Establish a connection to the supervisor (throw an error on failure) print("Waiting to establish connection to a running supervisor ... ", end='') sys.stdout.flush() start_time = time.time() connected = False while not connected and time.time() - start_time < TIMEOUT_SUPERVISOR: try: self._query("/", BenchBot.RouteType.EXPLICIT) connected = True except: pass time.sleep(3) if not connected: raise requests.ConnectionError( "Could not find a BenchBot supervisor @ '%s'. " "Are you sure it is available?" % self.supervisor_address) print("Connected!") # Wait until the robot is running print("Waiting to establish connection to a running robot ... ", end='') sys.stdout.flush() while (not self._query("is_running", BenchBot.RouteType.ROBOT)['is_running']): time.sleep(0.1) print("Connected!") # Get references to all of the API callbacks in robot config self._connection_callbacks = { k: BenchBot._attempt_connection_imports(v) for k, v in self._query('robot', BenchBot.RouteType.CONFIG) ['connections'].items() } # Ensure we are starting in a clean robot state if (self._query('selected_environment', BenchBot.RouteType.ROBOT)['number'] != 0): print( "Robot detected not to be in the first scene. " "Performing restart ... ", end='') sys.stdout.flush() self._query('restart', BenchBot.RouteType.ROBOT) print("Done.") else: self.reset()
def connect(self): """ Establish a connection with QMMF IPC webserver on the camera. This API also sets the `session_token` attribute from the token obtained from the camera. Returns ------- bool True if the connection was successful. False on failure. Raises ------ Timeout When the request times out on the connect request. RequestException The request is not correctly formed. """ with requests.session() as mysession: try: payload = "{\"username\": \"%s\", \"userpwd\": \"%s\"}" % ( self.username, self.password) url = "http://" + self.ip_address + ":" + self._port + "/login" #self.logger.info("API: " + url + ",data: " + json.dumps(payload,sort_keys=True)) print("URL: " + url + " data: " + payload + "Sent from IPCprovider.connect") #response = mysession.post(url, data=json.dumps(payload, sort_keys= True)) response = mysession.post(url, data=payload) print("Response: " + response.text) self.logger.info("LOGIN RESPONSE: " + response.text) json_resp = json.loads(response.text) if json_resp['status']: self._session_token = response.headers['Set-Cookie'] print("connection established with session token: [%s]" % self._session_token) return True else: raise requests.ConnectionError( "Failed to connect. Server returned status=False") except requests.exceptions.Timeout: # TODO: user should have a way to figure out if required services are running or not? maybe some simple URL self._show_error( "Timeout: Please check that the device is up and running and the IPC service is available" ) raise except requests.exceptions.RequestException as e: self.logger.exception(e.strerror) raise
def test_create_new_domain_url_path_connection_fails(self): auth_entity = testutil.FakeAuthEntity( id='x', user_json=json_dumps({'urls': [{ 'value': 'http://flaky/foo' }]})) self.expect_requests_get('http://flaky').AndRaise( requests.ConnectionError('DNS lookup failed for URL: http://bad/')) self.mox.ReplayAll() source = FakeSource.create_new(self.handler, auth_entity=auth_entity) self.assertEqual(['http://flaky/foo'], source.domain_urls) self.assertEqual(['flaky'], source.domains)
def _api_parse(self, r): """if JSON parsing failed - raising requests.ConnectionError Override to parse JSON specific errors.""" try: j = r.json() except Exception as e: # TO JSON ERROR? j = {} self.logger.warn('request error - %s: %r', e.__class__.__name__, e) # will wait and try again raise requests.ConnectionError( 'Error converting server reponse to JSON...') else: return j
def test_network_failure( head: mock.MagicMock, run: mock.MagicMock, ): head.side_effect = requests.ConnectionError() with pytest.raises(requests.ConnectionError): login.Auth()._run_server() run.assert_not_called() head.side_effect = requests.RequestException() with pytest.raises(requests.RequestException): login.Auth()._run_server() run.assert_not_called()
def test_get_local_slave_state_connection_error( mock_getfqdn, mock_requests_get, ): fake_request = requests.Request('GET', url='doesnt_matter') mock_getfqdn.return_value = 'fake_hostname' mock_requests_get.side_effect = requests.ConnectionError( 'fake_message', request=fake_request, ) with raises(mesos_tools.MesosSlaveConnectionError): mesos_tools.get_local_slave_state()
def submit_result(self, **kwargs): if 'id' in kwargs: raise ValueError("Keyword argument with name 'id' not allowed.") token = kwargs.pop('token') data = kwargs.copy() data['id'] = self.id r = requests.post(url=self._add_result_url(), json=data, headers={'Authorization': f'Bearer {token}'}) if r.status_code != 201: raise requests.ConnectionError( "Expected status code 201, but got {}".format(r.status_code)) return r
def test_result_eventual_success(mock_civis): # Test that we can get a result back from a succeeded job, # even if we need to retry a few times to succeed with the download. callback = mock.MagicMock() exc = requests.ConnectionError() se = make_to_file_mock('spam', max_n_err=2, exc=exc) mock_civis.io.civis_to_file.side_effect = se fut = ContainerFuture(1, 2, client=mock.MagicMock()) fut.set_result(Response({'state': 'success'})) res = civis.parallel._CivisBackendResult(fut, callback) assert res.get() == 'spam' assert callback.call_count == 1
def getTrades(self, market, timestamp=None): if timestamp == None: timestamp = time.time( ) - 60 * 60 * 24 # se solicita por defecto las últimas 24 horas url = self.createUrl('/markets/' + market + '/trades', {'timestamp': timestamp}) self.response = self.consume(url) body = self.response.json() if self.response.status_code != 200: raise requests.ConnectionError( 'No fue posible obtener los intercambios del mercado ' + market + ': ' + body['message']) return body['trades']
def test_registration_check_conn_error(test_connection, get_proxies, _init_session, _): ''' Can't connect, run connection test Returns False ''' config = Mock(legacy_upload=True, base_url='example.com') conn = InsightsConnection(config) conn.get = MagicMock() conn.get.side_effect = requests.ConnectionError() assert conn.api_registration_check() is False test_connection.assert_called_once()
def check_http_response(response, expected_status_code): """ validate http status code """ status = response.status_code if status == expected_status_code: return True if status == 401: # unauthorized msg = "ERROR: status = %s - Invalid credentials?" % status raise requests.ConnectionError(msg) if status in [404, 500]: # unknown resource, operation cannot be performed """ response 404 cannot be jsonified """ error = "Operation cannot be performed, retry with valid parameters" raise IOError(error) return False
def close(self): """ Logout of the current session. """ if self.authToken is None: raise requests.ConnectionError('No session token found.') url = self.sisenseUrl + "/api/v1/authentication/logout" headers = {'Authorization': self.authToken} response = requests.request("GET", url, headers=headers).json() if response['success']: self.authToken = None
def test_persistent_connection_error(client): for _i in range(6): load_response( RegisteredResponse( path="https://foo.api.globus.org/bar", body=requests.ConnectionError("foo-err"), )) load_response( RegisteredResponse(path="https://foo.api.globus.org/bar", json={"baz": 1})) with pytest.raises(globus_sdk.GlobusConnectionError): client.get("/bar")
def __perform_crumb_call(self, session): crumb_response = session.get(url=self.api_settings.crumb_url, auth=self.api_settings.auth) if crumb_response.status_code != 200: logger.error("url=%s\nheaders=%s\nbody=%s\n", crumb_response.request.url, crumb_response.request.headers, crumb_response.request.body) logger.error(self.api_settings.auth) logger.error(crumb_response.text) raise requests.ConnectionError('Could not issue Jenkins crumb.', response=crumb_response) return crumb_response.json()