def test_run_next_exception(self, mock_get_kube_client, mock_kubernetes_job_watcher): # When a quota is exceeded this is the ApiException we get r = HTTPResponse() r.body = { "kind": "Status", "apiVersion": "v1", "metadata": {}, "status": "Failure", "message": "pods \"podname\" is forbidden: " + "exceeded quota: compute-resources, " + "requested: limits.memory=4Gi, " + "used: limits.memory=6508Mi, " + "limited: limits.memory=10Gi", "reason": "Forbidden", "details": { "name": "podname", "kind": "pods" }, "code": 403 }, r.status = 403 r.reason = "Forbidden" # A mock kube_client that throws errors when making a pod mock_kube_client = mock.patch('kubernetes.client.CoreV1Api', autospec=True) mock_kube_client.create_namespaced_pod = mock.MagicMock( side_effect=ApiException(http_resp=r)) mock_get_kube_client.return_value = mock_kube_client kubernetesExecutor = KubernetesExecutor() kubernetesExecutor.start() # Execute a task while the Api Throws errors try_number = 1 kubernetesExecutor.execute_async(key=('dag', 'task', datetime.utcnow(), try_number), command='command', executor_config={}) kubernetesExecutor.sync() kubernetesExecutor.sync() assert mock_kube_client.create_namespaced_pod.called self.assertFalse(kubernetesExecutor.task_queue.empty()) # Disable the ApiException mock_kube_client.create_namespaced_pod.side_effect = None # Execute the task without errors should empty the queue kubernetesExecutor.sync() assert mock_kube_client.create_namespaced_pod.called self.assertTrue(kubernetesExecutor.task_queue.empty())
def _download_file_stream(self, response: HTTPResponse, file_path: str) -> bool: """ Stream the requested file to file_path. This way, even large files can be downloaded. # Arguments response (#HttpResponse): the http response of the server. Should contain an output stream. file_path (string): the path where the file will be saved # Returns True if the download was successful, False otherwise """ if not os.path.exists(os.path.dirname(file_path)): os.makedirs(os.path.dirname(file_path)) try: # set chunk_size to same size as server's output size chunk_size = 16384 downloaded = 0 total_length = int( response.headers['Content-Disposition'].split('size=')[1]) desc = "Downloading " + text_utils.truncate_middle( os.path.basename(file_path), 20) pbar = tqdm(total=total_length, initial=downloaded, mininterval=0.3, unit='B', unit_scale=True, desc=desc, file=sys.stdout) with open(file_path, 'wb') as out: while True: data = response.read(chunk_size) downloaded += len(data) if not data: break out.write(data) pbar.update(len(data)) pbar.close() response.release_conn() if total_length != 0 and downloaded != total_length: print("Failed to download full file") except Exception as e: self.log.warning("Could not stream the data, so write it in once", e) try: with open(file_path, 'wb') as f: f.write(response.data) except: return False return True
def _provide_ssl_auth_required(self): """ Provide ssl auth response :return: urllib3.HTTPResponse """ response = HTTPResponse() response.status = self.DEFAULT_SSL_CERT_REQUIRED_STATUSES response.__setattr__('_body', ' ') return response
def test_run_next_exception(self, mock_get_kube_client, mock_kubernetes_job_watcher): import sys path = sys.path[ 0] + '/tests/kubernetes/pod_generator_base_with_secrets.yaml' # When a quota is exceeded this is the ApiException we get response = HTTPResponse( body= '{"kind": "Status", "apiVersion": "v1", "metadata": {}, "status": "Failure", ' '"message": "pods \\"podname\\" is forbidden: exceeded quota: compute-resources, ' 'requested: limits.memory=4Gi, used: limits.memory=6508Mi, limited: limits.memory=10Gi", ' '"reason": "Forbidden", "details": {"name": "podname", "kind": "pods"}, "code": 403}' ) response.status = 403 response.reason = "Forbidden" # A mock kube_client that throws errors when making a pod mock_kube_client = mock.patch('kubernetes.client.CoreV1Api', autospec=True) mock_kube_client.create_namespaced_pod = mock.MagicMock( side_effect=ApiException(http_resp=response)) mock_get_kube_client.return_value = mock_kube_client mock_api_client = mock.MagicMock() mock_api_client.sanitize_for_serialization.return_value = {} mock_kube_client.api_client = mock_api_client config = { ('kubernetes', 'pod_template_file'): path, } with conf_vars(config): kubernetes_executor = self.kubernetes_executor kubernetes_executor.start() # Execute a task while the Api Throws errors try_number = 1 kubernetes_executor.execute_async( key=('dag', 'task', datetime.utcnow(), try_number), queue=None, command=['airflow', 'tasks', 'run', 'true', 'some_parameter'], ) kubernetes_executor.sync() kubernetes_executor.sync() assert mock_kube_client.create_namespaced_pod.called assert not kubernetes_executor.task_queue.empty() # Disable the ApiException mock_kube_client.create_namespaced_pod.side_effect = None # Execute the task without errors should empty the queue kubernetes_executor.sync() assert mock_kube_client.create_namespaced_pod.called assert kubernetes_executor.task_queue.empty()
def test_message_get_retry_after(self): response = HTTPResponse(reason="too many requests") response.headers.add('Retry-After', '63') influx_db_error = InfluxDBError(response=response) self.assertEqual("too many requests", str(influx_db_error)) self.assertEqual("63", influx_db_error.retry_after) influx_db_error = InfluxDBError(response=HTTPResponse( reason="too many requests")) self.assertEqual("too many requests", str(influx_db_error)) self.assertEqual(None, influx_db_error.retry_after)
def test_cache(self, tmpdir, send_returns): """ L{IntersphinxCache.get} caches responses to the file system. """ url = u"https://cache.example/objects.inv" content = b'content' send_returns( HTTPResponse( body=ClosingBytesIO(content), headers={ 'date': 'Sun, 06 Nov 1994 08:49:37 GMT', }, status=200, preload_content=False, decode_content=False, ), ) loadsCache = sphinx.IntersphinxCache.fromParameters( sessionFactory=requests.Session, cachePath=str(tmpdir), maxAgeDictionary={"weeks": 1} ) assert loadsCache.get(url) == content # Now the response contains different data that will not be # returned when the cache is enabled. send_returns( HTTPResponse( body=ClosingBytesIO(content * 2), headers={ 'date': 'Sun, 06 Nov 1994 08:49:37 GMT', }, status=200, preload_content=False, decode_content=False, ), ) assert loadsCache.get(url) == content readsCacheFromFileSystem = sphinx.IntersphinxCache.fromParameters( sessionFactory=requests.Session, cachePath=str(tmpdir), maxAgeDictionary={"weeks": 1} ) assert readsCacheFromFileSystem.get(url) == content
def test_get_offers_on_device_decisioning_emits_notifications(self): get_offers_opts = { "request": EXECUTE_REQUEST, "session_id": "dummy_session" } client_opts = dict(CONFIG) client_opts["decisioning_method"] = DecisioningMethod.ON_DEVICE.value client_opts["target_location_hint"] = "28" # send_notifications call with patch.object(DeliveryApi, "execute", return_value=create_delivery_response({})) \ as mock_delivery_api: with patch("target_decisioning_engine.artifact_provider.urllib3.PoolManager") as mock_artifact_provider: artifact_instance = mock_artifact_provider.return_value artifact_response = HTTPResponse(status=OK, body=json.dumps(ARTIFACT_AB_SIMPLE)) artifact_instance.request.return_value = artifact_response client = TargetClient.create(client_opts) self.assertEqual(mock_artifact_provider.call_count, 1) result = client.get_offers(get_offers_opts) self.assertIsNotNone(result.get("response")) self.assertEqual(result.get("response").status, OK) self.assertIsNotNone(result.get("response").execute) time.sleep(1) # notifications sent async self.assertEqual(mock_delivery_api.call_count, 1) notification_request = to_dict(mock_delivery_api.call_args[0][2]) expect_to_match_object(notification_request, EXPECTED_NOTIFICATION_REQUEST)
def get_socket_http_response(sock): charset = 'utf-8' _UNKNOWN = 'UNKNOWN' http_response = None # 接收html字节数据 r = client.HTTPResponse(sock) try: try: r.begin() except ConnectionError as ce: logger.error('拉取数据异常:%s', ce) will_close = r.will_close http_response = HTTPResponse.from_httplib(r) if will_close and will_close != _UNKNOWN: # logger.debug('数据已接收,主机关闭了连接') sock.close() except Exception as e: logger.error('数据接收异常:%s', e) finally: r.close() # print('response:') # print(response.decode(charset)) # 保持连接 if http_response is not None: setattr(http_response, "body", http_response.data.decode(charset)) return http_response else: return None
def get_object_side_effect(bucket, object_name): file = ( file_data.unsupported if object_name.endswith(".unsupported") else file_data.png ) return HTTPResponse(body=BytesIO(file), preload_content=False,)
def test_logging(self): response = HTTPResponse( body= '{"code":"too many requests","message":"org 04014de4ed590000 has exceeded limited_write plan limit"}' ) response.headers.add('Retry-After', '63') with self.assertLogs('influxdb_client.client.write.retry', level='WARNING') as cm: WritesRetry(total=5, backoff_factor=1, max_retry_delay=15) \ .increment(response=response) \ .increment(error=Exception("too many requests")) \ .increment(url='http://localhost:9999') self.assertEqual( "WARNING:influxdb_client.client.write.retry:The retriable error occurred during request. " "Reason: 'org 04014de4ed590000 has exceeded limited_write plan limit'. Retry in 63s.", cm.output[0]) self.assertEqual( "WARNING:influxdb_client.client.write.retry:The retriable error occurred during request. " "Reason: 'too many requests'.", cm.output[1]) self.assertEqual( "WARNING:influxdb_client.client.write.retry:The retriable error occurred during request. " "Reason: 'Failed request to: http://localhost:9999'.", cm.output[2])
def stub_get_namespaced_multus_network(**args): with open( resource_root + '/stub_results/stub_get_namespaced_multus_network.txt', 'rb') as infile: body = infile.read() return_http_response = HTTPResponse(body=body, status=200) return return_http_response
def test_run_next_exception(self, mock_get_kube_client, mock_kubernetes_job_watcher): # When a quota is exceeded this is the ApiException we get r = HTTPResponse() r.body = { "kind": "Status", "apiVersion": "v1", "metadata": {}, "status": "Failure", "message": "pods \"podname\" is forbidden: " + "exceeded quota: compute-resources, " + "requested: limits.memory=4Gi, " + "used: limits.memory=6508Mi, " + "limited: limits.memory=10Gi", "reason": "Forbidden", "details": {"name": "podname", "kind": "pods"}, "code": 403}, r.status = 403 r.reason = "Forbidden" # A mock kube_client that throws errors when making a pod mock_kube_client = mock.patch('kubernetes.client.CoreV1Api', autospec=True) mock_kube_client.create_namespaced_pod = mock.MagicMock( side_effect=ApiException(http_resp=r)) mock_get_kube_client.return_value = mock_kube_client kubernetesExecutor = KubernetesExecutor() kubernetesExecutor.start() # Execute a task while the Api Throws errors try_number = 1 kubernetesExecutor.execute_async(key=('dag', 'task', datetime.utcnow(), try_number), command='command', executor_config={}) kubernetesExecutor.sync() kubernetesExecutor.sync() mock_kube_client.create_namespaced_pod.assert_called() self.assertFalse(kubernetesExecutor.task_queue.empty()) # Disable the ApiException mock_kube_client.create_namespaced_pod.side_effect = None # Execute the task without errors should empty the queue kubernetesExecutor.sync() mock_kube_client.create_namespaced_pod.assert_called() self.assertTrue(kubernetesExecutor.task_queue.empty())
def test_get_retry_after_jitter(self): response = HTTPResponse() response.headers.add('Retry-After', '5') retry = WritesRetry(jitter_interval=2) retry_after = retry.get_retry_after(response) self.assertGreater(retry_after, 5) self.assertLessEqual(retry_after, 7)
def http_response_mock(body="", headers=None, status=200): if headers is None: headers = {} if not six.PY2: body = body.encode("UTF-8") return HTTPResponse(body, HTTPHeaderDict(headers), status=status)
def _parse_to_tables(data: str): fp = BytesIO(str.encode(data)) _parser = FluxCsvParser( response=HTTPResponse(fp, preload_content=False), serialization_mode=FluxSerializationMode.tables) list(_parser.generator()) tables = _parser.tables return tables
def urllib3_http_response_from_exception(exception: Exception) -> HTTPResponse: """On exceptions, we want to create our own HTTPResponse with exception's message to set it to response.raw to be read later.""" return HTTPResponse( body=io.BytesIO(str(exception).encode('utf-8', errors='replace')), # https://github.com/requests/requests/issues/2635#issuecomment-112270117 preload_content=False, )
def response(): response = requests.Response() response._content = b'{"id": 5}' response.status_code = 201 response.headers["Content-Type"] = "application/json" response.raw = HTTPResponse( body=io.BytesIO(response._content), status=response.status_code, headers=response.headers ) response.request = requests.PreparedRequest() response.request.prepare(method="POST", url="http://example.com", headers={"Content-Type": "application/json"}) return response
def __init__(self, response: HTTPResponse = None, message: str = None): """Initialize the InfluxDBError handler.""" if response is not None: self.response = response self.message = self._get_message(response) self.retry_after = response.getheader('Retry-After') else: self.response = None self.message = message or 'no response' self.retry_after = None super().__init__(self.message)
def test_client_recovers_if_location_hint_request_fails(self): get_offers_opts = { "request": TARGET_REQUEST, "session_id": "dummy_session" } client_opts = dict(CONFIG) client_opts["decisioning_method"] = DecisioningMethod.ON_DEVICE.value with patch("delivery_api_client.rest.urllib3.PoolManager") as mock_delivery_api: delivery_instance = mock_delivery_api.return_value delivery_response = HTTPResponse(status=503) delivery_instance.request.return_value = delivery_response with patch("target_decisioning_engine.artifact_provider.urllib3.PoolManager") as mock_artifact_provider: artifact_instance = mock_artifact_provider.return_value artifact_response = HTTPResponse(status=OK, body=json.dumps(ARTIFACT_AB_SIMPLE)) artifact_instance.request.return_value = artifact_response client = TargetClient.create(client_opts) result = client.get_offers(get_offers_opts) # on-device self.assertIsNone(result.get("target_location_hint_cookie"))
def _response_from_bytes(data: bytes) -> Urllib3HTTPResponse: class BytesIOSocket: def __init__(self, content): self.handle = BytesIO(content) def makefile(self, mode) -> BytesIO: return self.handle sock = BytesIOSocket(data) response = HttpHTTPResponse(sock) response.begin() return Urllib3HTTPResponse.from_httplib(response)
def responses(): iterator = iter(snapshots) while True: items = take(page_size, iterator) yield HTTPResponse(status=200, body=json.dumps({ 'total': len(snapshots), 'filteredTotal': len(snapshots), 'items': list(items) })) if not items: break
def test_get_offers_hybrid_does_remote_request_if_necessary(self): get_offers_opts = { "request": TARGET_REQUEST_DICT, "session_id": "dummy_session" } get_offers_opts["request"]["prefetch"] = { "mboxes": [{ "name": "mbox-feature-flags", "index": 1 }, { "name": "remote-only-mbox-a", "index": 2 }, { "name": "remote-only-mbox-b", "index": 2 }] } get_offers_opts["request"] = create_delivery_request( get_offers_opts["request"]) client_opts = dict(CONFIG) client_opts["decisioning_method"] = DecisioningMethod.HYBRID.value with patch.object(DeliveryApi, "execute", return_value=create_delivery_response(DELIVERY_RESPONSE)) \ as mock_delivery_api: with patch( "target_decisioning_engine.artifact_provider.urllib3.PoolManager" ) as mock_artifact_provider: artifact_instance = mock_artifact_provider.return_value artifact_response = HTTPResponse( status=OK, body=json.dumps(FEATURE_FLAG_ARTIFACT)) artifact_instance.request.return_value = artifact_response client = TargetClient.create(client_opts) self.assertEqual(mock_artifact_provider.call_count, 1) result = client.get_offers(get_offers_opts) self.assertEqual(mock_delivery_api.call_count, 1) self.assertEqual(result["response"].status, OK) self.assertIsNotNone(result["response"].prefetch) self.assertEqual(result["meta"]["decisioning_method"], DecisioningMethod.HYBRID.value) self.assertSetEqual( set(result["meta"]["remote_mboxes"]), set(["remote-only-mbox-a", "remote-only-mbox-b"])) self.assertEqual(result["meta"]["remote_views"], [])
def test_create_decisioning_method_on_device(self): client_ready_mock = Mock() client_opts = dict(CONFIG) client_opts["decisioning_method"] = DecisioningMethod.ON_DEVICE.value client_opts["events"] = { "client_ready": client_ready_mock } with patch("target_decisioning_engine.artifact_provider.urllib3.PoolManager") as mock_pool_manager: instance = mock_pool_manager.return_value mock_response = HTTPResponse(status=OK, body=json.dumps(ARTIFACT_BLANK)) instance.request.return_value = mock_response with patch.object(DeliveryApi, "execute", return_value=create_delivery_response(LOCATION_HINT_RESPONSE)): client = TargetClient.create(client_opts) self.assertIsNotNone(client.decisioning_engine) self.assertTrue(callable(client.decisioning_engine.get_offers)) self.assertEqual(client_ready_mock.call_count, 1)
def setUp(self): self.headers = { "x-geo-latitude": 37.773972, "x-geo-longitude": -122.431297, "x-geo-country-code": "US", "x-geo-region-code": "CA", "x-geo-city": "SANFRANCISCO" } self.config = DecisioningConfig("myClient", "myOrgId") geo_values = { "x-geo-longitude": -122.4, "x-geo-latitude": 37.75, "x-geo-city": "SAN FRANCISCO", "x-geo-region-code": "CA", "x-geo-country-code": "US" } self.mock_geo_response = HTTPResponse(body=json.dumps(geo_values), status=OK)
def _mock_urlopen(http_client, method, url, *, headers, **kwargs): nonlocal called self.assertEqual(method, 'GET') self.assertEqual(str(furl(url).remove(query=True)), tdr_client._repository_endpoint('snapshots')) headers = {k.capitalize(): v for k, v in headers.items()} token = headers['Authorization'].split('Bearer ').pop() response = HTTPResponse( status=200, body=json.dumps({ 'total': 1, 'filteredTotal': 1, 'items': [] if called else self._mock_snapshots(token) })) called = True return response
def test_get_offers_on_device_partial_returns_206_and_remote_mbox_names( self): get_offers_opts = { "request": TARGET_REQUEST_DICT, "session_id": "dummy_session" } get_offers_opts["request"]["prefetch"] = { "mboxes": [{ "name": "mbox-feature-flags", "index": 1 }, { "name": "remote-only-mbox-a", "index": 2 }, { "name": "remote-only-mbox-b", "index": 2 }] } get_offers_opts["request"] = create_delivery_request( get_offers_opts["request"]) client_opts = dict(CONFIG) client_opts["decisioning_method"] = DecisioningMethod.ON_DEVICE.value with patch( "target_decisioning_engine.artifact_provider.urllib3.PoolManager" ) as mock_artifact_provider: artifact_instance = mock_artifact_provider.return_value artifact_response = HTTPResponse( status=OK, body=json.dumps(FEATURE_FLAG_ARTIFACT)) artifact_instance.request.return_value = artifact_response client = TargetClient.create(client_opts) self.assertEqual(mock_artifact_provider.call_count, 1) result = client.get_offers(get_offers_opts) self.assertEqual(result["response"].status, PARTIAL_CONTENT) self.assertIsNotNone(result["response"].prefetch) self.assertEqual(result["meta"]["decisioning_method"], DecisioningMethod.ON_DEVICE.value) self.assertSetEqual( set(result["meta"]["remote_mboxes"]), set(["remote-only-mbox-a", "remote-only-mbox-b"])) self.assertEqual(result["meta"]["remote_views"], [])
def test_get_offers_valid_on_device_decisioning_response(self): get_offers_opts = { "request": PREFETCH_REQUEST, "session_id": "dummy_session" } client_opts = dict(CONFIG) client_opts["decisioning_method"] = DecisioningMethod.ON_DEVICE.value client_opts["target_location_hint"] = "28" with patch("target_decisioning_engine.artifact_provider.urllib3.PoolManager") as mock_artifact_provider: artifact_instance = mock_artifact_provider.return_value artifact_response = HTTPResponse(status=OK, body=json.dumps(ARTIFACT_AB_SIMPLE)) artifact_instance.request.return_value = artifact_response with patch.object(DeliveryApi, "execute", return_value=create_delivery_response(LOCATION_HINT_RESPONSE)): client = TargetClient.create(client_opts) result = client.get_offers(get_offers_opts) result_dict = to_dict(result) expect_to_match_object(result_dict, EXPECTED_PREFETCH_RESULT)
def test_top_level_request_with_timeout(self) -> None: with mock.patch("urllib3.poolmanager.RequestMethods.request") as mockRequest: mockRequest.return_value = HTTPResponse(status=200) r = request("GET", f"{self.base_url}/redirect", timeout=2.5) assert r.status == 200 mockRequest.assert_called_with( "GET", f"{self.base_url}/redirect", body=None, fields=None, headers=None, preload_content=True, decode_content=True, redirect=True, retries=None, timeout=2.5, json=None, )
def test_valid_geo_request_non_200_response(self): artifact = deepcopy(ARTIFACT_BLANK) artifact["geoTargetingEnabled"] = True geo_provider = GeoProvider(self.config, artifact) mock_bad_response = HTTPResponse(body="Bad Request", status=BAD_REQUEST) with patch.object(geo_provider.pool_manager, "request", return_value=mock_bad_response) as mock_http_call: geo_input = Geo(ip_address="12.21.1.40") result = geo_provider.valid_geo_request_context(geo_input) time.sleep(1) self.assertEqual(result, None) self.assertEqual(mock_http_call.call_count, 1) self.assertEqual(mock_http_call.call_args[0][1], "https://assets.adobetarget.com/v1/geo") self.assertEqual( mock_http_call.call_args[1].get("headers").get( HTTP_HEADER_FORWARDED_FOR), "12.21.1.40")
def test_client_preemptively_fetches_target_location_hint(self): get_offers_opts = { "request": TARGET_REQUEST, "session_id": "dummy_session" } client_opts = dict(CONFIG) client_opts["decisioning_method"] = DecisioningMethod.ON_DEVICE.value with patch("target_decisioning_engine.artifact_provider.urllib3.PoolManager") as mock_artifact_provider: artifact_instance = mock_artifact_provider.return_value artifact_response = HTTPResponse(status=OK, body=json.dumps(ARTIFACT_AB_SIMPLE)) artifact_instance.request.return_value = artifact_response with patch.object(DeliveryApi, "execute", return_value=create_delivery_response(LOCATION_HINT_RESPONSE)): client = TargetClient.create(client_opts) result = client.get_offers(get_offers_opts) # on-device self.assertEqual(result.get("target_location_hint_cookie"), { "name": "mboxEdgeCluster", "value": "28", "maxAge": 1860 })
def test_client_uses_location_hint_from_config(self): get_offers_opts = { "request": TARGET_REQUEST, "session_id": "dummy_session" } client_opts = dict(CONFIG) client_opts["decisioning_method"] = DecisioningMethod.ON_DEVICE.value client_opts["target_location_hint"] = "28" with patch("target_decisioning_engine.artifact_provider.urllib3.PoolManager") as mock_artifact_provider: artifact_instance = mock_artifact_provider.return_value artifact_response = HTTPResponse(status=OK, body=json.dumps(ARTIFACT_AB_SIMPLE)) artifact_instance.request.return_value = artifact_response client = TargetClient.create(client_opts) result = client.get_offers(get_offers_opts) self.assertEqual(result.get("target_location_hint_cookie"), { "name": "mboxEdgeCluster", "value": "28", "maxAge": 1860 })