class Contract(object): def __init__(self, network_name): self.name = network_name @retry(wait=wait_random_exponential(multiplier=1, max=10)) def ensure_network_is_running(self): api = API() if not api.is_network_running(self.name): api.start_network(self.name) if not api.is_network_running(self.name): raise FailedToStartException(self.name) @retry(wait=wait_random_exponential(multiplier=1, max=10)) def ensure_network_is_stopped(self): api = API() if api.is_network_running(self.name): api.stop_network(self.name) if api.is_network_running(self.name): raise FailedToStopException(self.name) @retry(wait=wait_random_exponential(multiplier=2, max=10), stop=stop_after_attempt(3), reraise=True) def check_network_health(self): logger.info("Checking '{}' network health...".format(self.name)) api = API() anchors = api.get_anchors(self.name) degraded_state_errors = [] if len(anchors) == 0: degraded_state_errors.append("No anchors found in CUWB network") for a in anchors: anchor_name = a.get("name", "UNKNOWN") connectivity = a.get("connectivity_state", "") synchronization = a.get("synchronization_state", "") if not connectivity.lower() == "ethernet and uwb connected": degraded_state_errors.append( "Anchor '{}' reporting bad connectivity - `{}`".format( anchor_name, connectivity)) if not synchronization.lower() == "rx and tx synced": degraded_state_errors.append( "Anchor '{}' reporting bad synchronization - `{}`".format( anchor_name, synchronization)) if len(degraded_state_errors) > 0: logger.error("'{}' network health degraded: {}".format( self.name, degraded_state_errors)) raise AnchorsDegradedException(self.name, degraded_state_errors)
def run_compute_singles(run, site, processed_output_path, database): """Compute the singles (underlying uncorrelated) rate.""" path_prefix = os.path.join(processed_output_path, f'EH{site}') ads = common.dets_for(site, run) update_db = True iteration = 0 extra_cut = '1' for ad in ads: infile = os.path.join( path_prefix, f'hadded_ad{ad}/out_ad{ad}_{run}.root', ) # Ideally should check to see if rate has been computed before. # But, this is so fast that I will just re-compute every time. for attempt in tenacity.Retrying( reraise=True, wait=tenacity.wait_random_exponential(max=60), retry=tenacity.retry_if_exception_type(sqlite3.Error), before_sleep=tenacity.before_sleep_log(logging, logging.DEBUG), ): with attempt: compute_singles.main( infile, database, GENERAL_LABEL, update_db, iteration, extra_cut, ) return
def sync(cnxt, entity_id, current_traversal, is_update, propagate, predecessors, new_data): # Retry waits up to 60 seconds at most, with exponentially increasing # amounts of jitter per resource still outstanding wait_strategy = tenacity.wait_random_exponential(max=60) def init_jitter(existing_input_data): nconflicts = max(0, len(predecessors) - len(existing_input_data) - 1) # 10ms per potential conflict, up to a max of 10s in total return min(nconflicts, 1000) * 0.01 @tenacity.retry( retry=tenacity.retry_if_result(lambda r: r is None), wait=wait_strategy ) def _sync(): sync_point = get(cnxt, entity_id, current_traversal, is_update) input_data = deserialize_input_data(sync_point.input_data) wait_strategy.multiplier = init_jitter(input_data) input_data.update(new_data) rows_updated = update_input_data( cnxt, entity_id, current_traversal, is_update, sync_point.atomic_key, serialize_input_data(input_data)) return input_data if rows_updated else None input_data = _sync() waiting = predecessors - set(input_data) key = make_key(entity_id, current_traversal, is_update) if waiting: LOG.debug('[%s] Waiting %s: Got %s; still need %s', key, entity_id, _dump_list(input_data), _dump_list(waiting)) else: LOG.debug('[%s] Ready %s: Got %s', key, entity_id, _dump_list(input_data)) propagate(entity_id, serialize_input_data(input_data))
def sync_to_db(self, session: Optional[Session] = None): """Save attributes about list of DAG to the DB.""" # To avoid circular import - airflow.models.dagbag -> airflow.models.dag -> airflow.models.dagbag from airflow.models.dag import DAG from airflow.models.serialized_dag import SerializedDagModel # Retry 'DAG.bulk_write_to_db' & 'SerializedDagModel.bulk_sync_to_db' in case # of any Operational Errors # In case of failures, provide_session handles rollback for attempt in tenacity.Retrying( retry=tenacity.retry_if_exception_type( exception_types=OperationalError), wait=tenacity.wait_random_exponential(multiplier=0.5, max=5), stop=tenacity.stop_after_attempt(settings.MAX_DB_RETRIES), before_sleep=tenacity.before_sleep_log(self.log, logging.DEBUG), reraise=True): with attempt: self.log.debug( "Running dagbag.sync_to_db with retries. Try %d of %d", attempt.retry_state.attempt_number, settings.MAX_DB_RETRIES) self.log.debug("Calling the DAG.bulk_sync_to_db method") try: DAG.bulk_write_to_db(self.dags.values(), session=session) # Write Serialized DAGs to DB self.log.debug( "Calling the SerializedDagModel.bulk_sync_to_db method" ) SerializedDagModel.bulk_sync_to_db(self.dags.values(), session=session) except OperationalError: session.rollback() raise
def sync(cnxt, entity_id, current_traversal, is_update, propagate, predecessors, new_data): # Retry waits up to 60 seconds at most, with exponentially increasing # amounts of jitter per resource still outstanding wait_strategy = tenacity.wait_random_exponential(max=60) def init_jitter(existing_input_data): nconflicts = max(0, len(predecessors) - len(existing_input_data) - 1) # 10ms per potential conflict, up to a max of 10s in total return min(nconflicts, 1000) * 0.01 @tenacity.retry(retry=tenacity.retry_if_result(lambda r: r is None), wait=wait_strategy) def _sync(): sync_point = get(cnxt, entity_id, current_traversal, is_update) input_data = deserialize_input_data(sync_point.input_data) wait_strategy.multiplier = init_jitter(input_data) input_data.update(new_data) rows_updated = update_input_data(cnxt, entity_id, current_traversal, is_update, sync_point.atomic_key, serialize_input_data(input_data)) return input_data if rows_updated else None input_data = _sync() waiting = predecessors - set(input_data) key = make_key(entity_id, current_traversal, is_update) if waiting: LOG.debug('[%s] Waiting %s: Got %s; still need %s', key, entity_id, _dump_list(input_data), _dump_list(waiting)) else: LOG.debug('[%s] Ready %s: Got %s', key, entity_id, _dump_list(input_data)) propagate(entity_id, serialize_input_data(input_data))
def batch_wait(self, tasks, timeout=300, wait_exp_multiplier=0.05, wait_exp_max=1.0): """ Wait until a list of task are completed. Expires after 'timeout' seconds. Returns a tuple of list (pending_tasks, success_tasks, error_tasks). Each list contains a couple (original_position, task) sorted by original_position asc original_position gives the original index in the input tasks list parameter. This helps to keep the order. """ try: positions = {} pending_tasks = [] for pos, task in enumerate(tasks): positions[task.pk] = pos pending_tasks.append((pos, task)) success_tasks = [] error_tasks = [] retryer = Retrying(wait=wait_random_exponential(multiplier=wait_exp_multiplier, max=wait_exp_max), stop=stop_after_delay(timeout), retry=retry_if_result(has_pending_tasks), before=before_log(logger, logging.DEBUG), after=after_log(logger, logging.DEBUG)) retryer(self._refresh_tasks_status, pending_tasks, success_tasks, error_tasks, positions) except RetryError: pass return (sorted(pending_tasks, key=lambda v: v[0]), sorted(success_tasks, key=lambda v: v[0]), sorted(error_tasks, key=lambda v: v[0]))
def retry_request(method, prepared_req, session, max_retries=10): retry_conditions = None def _make_request(req, sess): """send the prepared session request""" response = sess.send(req) return response def _return_last_value(retry_state): """return the result of the last call attempt and let code pick up the error""" return retry_state.outcome.result() if method.upper() == 'POST': retry_conditions = (retry_if_result( lambda res: res.status_code in civis.civis.POST_RETRY_CODES)) elif method.upper() in civis.civis.RETRY_VERBS: retry_conditions = (retry_if_result( lambda res: res.status_code in civis.civis.RETRY_CODES)) if retry_conditions: retry_config = Retrying( retry=retry_conditions, wait=wait_for_retry_after_header( fallback=wait_random_exponential(multiplier=2, max=60)), stop=(stop_after_delay(600) | stop_after_attempt(max_retries)), retry_error_callback=_return_last_value, ) response = retry_config(_make_request, prepared_req, session) return response response = _make_request(prepared_req, session) return response
class PushManager: """ Manager around the push notification sending that handles retrying """ def __init__(self, quit_event): self._quit_event = quit_event @retry( stop=stop_after_attempt(5), wait=wait_random_exponential(), retry=retry_if_exception_type(get_pushing_error_classes()), ) def push(self, notification, push_sent_event): """ Send a push notification and retry in cases of failures. This function will try sending up to 5 times, and uses an exponential backoff strategy, where a random waiting time is chosen between 0 and a max value that doubles every retry, to avoid all sender processes retrying at the same time. """ if self._quit_event.is_set(): raise GracefulExit() push_notification(notification, push_sent_event) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass
def __attrs_post_init__(self): if self.max_retry_delay is None: self.max_retry_delay = self.timeout * 3 self._retry_upload = tenacity.Retrying( # Retry after 1s, 2s, 4s, 8s with some randomness wait=tenacity.wait_random_exponential(multiplier=0.5), stop=tenacity.stop_after_delay(self.max_retry_delay), retry_error_cls=UploadFailed, retry=tenacity.retry_if_exception_type((http_client.HTTPException, OSError, IOError)), ) tags = { k: six.ensure_binary(v) for k, v in itertools.chain( parse_tags_str(os.environ.get("DD_TAGS")).items(), parse_tags_str(os.environ.get("DD_PROFILING_TAGS")).items(), ) } tags.update({k: six.ensure_binary(v) for k, v in self.tags.items()}) tags.update( { "host": HOSTNAME.encode("utf-8"), "language": b"python", "runtime": PYTHON_IMPLEMENTATION, "runtime_version": PYTHON_VERSION, "profiler_version": ddtrace.__version__.encode("ascii"), } ) if self.version: tags["version"] = self.version.encode("utf-8") if self.env: tags["env"] = self.env.encode("utf-8") self.tags = tags
def __init__(self, configuration=None, header_name=None, header_value=None, cookie=None, pool_threads=1): if configuration is None: configuration = Configuration() self.configuration = configuration self.pool_threads = pool_threads self.retrying = tenacity.Retrying( stop=tenacity.stop_after_attempt(configuration.retry_count), wait=tenacity.wait_random_exponential( multiplier=configuration.back_off, max=configuration.retry_max_delay, min=configuration.retry_delay), retry=(tenacity.retry_if_result(self.is_retry_enabled) and ((tenacity.retry_if_exception_type(RetryableException)) | (tenacity.retry_if_exception_type(HTTPError))))) self.rest_client = rest.RESTClientObject(configuration, retrying=self.retrying) self.default_headers = {} if header_name is not None: self.default_headers[header_name] = header_value self.cookie = cookie # Set default User-Agent. self.user_agent = 'opsgenie-sdk-python-2.0.2' # init metric publishers self.http_metric_publisher = self.rest_client.http_metric self.api_metric_publisher = metrics.ApiMetric('ApiMetricPublisher') self.sdk_metric_publisher = metrics.SdkMetric('SdkMetricPublisher')
def run_shell_expbackoff(command, debug=False, timeout=60.0, close_fds=True, redirect_stdout=subprocess.PIPE, redirect_stderr=subprocess.PIPE, return_outerr=False, raise_exceptions=False, retries=4, retry_multiplier=1.0, retry_maxtime=30.0): '''This runs a subprocess with shell=True, a configured timeout for the initial run, and a configurable number of retries with an exponential backoff. ''' @retry(stop=stop_after_attempt(retries), wait=wait_random_exponential(multiplier=retry_multiplier, max=retry_maxtime), reraise=raise_exceptions) def runner(command, debug, timeout, close_fds, return_outerr, redirect_stdout, redirect_stderr): return run_shell(command, debug=debug, redirect_stdout=redirect_stdout, redirect_stderr=redirect_stderr, timeout=timeout, close_fds=close_fds, return_outerr=return_outerr) return runner(command, debug, timeout, close_fds, return_outerr, redirect_stdout, redirect_stderr)
class TestPushMq(object): """mq测试用例""" uuid = '367dab7a-2a1a-11ea-90ca-acde48001122' @retry(stop=stop_after_attempt(5), wait=wait_random_exponential(2, max=5)) def db_retry(self, uuid, od): """异步数据校验,重试查询数据库""" res = od.tsms_select('sms_send', 'consume,status,mobile', uuid) assert res[0].get("status") == "success" def test_push_001(self, mq, od): """消息发送MQ生产消费功能测试""" data = { "uid": self.uuid, "phone": "18535941072", "content": "【hellokitty】验证码为:1234" } #将测试数据状态改为failed od.tsms_update('sms_send', 'status', 'failed', uuid=self.uuid) res = od.tsms_select('sms_send', 'status', uuid=self.uuid) assert res[0].get("status") == "failed" #写MQ,将字典data通过json序列化成json字符串 mq.push_direct_secure(json.dumps(data)) #查数据库 self.db_retry(self.uuid)
def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult: stream_descriptor = BaseIngestClient._prepare_stream(stream_descriptor, ingestion_properties) stream = stream_descriptor.stream buffered_stream = read_until_size_or_end(stream, self.MAX_STREAMING_SIZE_IN_BYTES + 1) if len(buffered_stream.getbuffer()) > self.MAX_STREAMING_SIZE_IN_BYTES: stream_descriptor.stream = chain_streams([buffered_stream, stream]) return self.queued_client.ingest_from_stream(stream_descriptor, ingestion_properties) stream_descriptor.stream = buffered_stream try: for attempt in Retrying( stop=stop_after_attempt(self._num_of_attempts), wait=wait_random_exponential(max=self._max_seconds_per_retry), reraise=True ): with attempt: stream.seek(0, SEEK_SET) client_request_id = ManagedStreamingIngestClient._get_request_id(stream_descriptor.source_id, attempt.retry_state.attempt_number - 1) return self.streaming_client._ingest_from_stream_with_client_request_id(stream_descriptor, ingestion_properties, client_request_id) except KustoApiError as ex: error = ex.get_api_error() if error.permanent: raise return self.queued_client.ingest_from_stream(stream_descriptor, ingestion_properties)
def retry(retry_param: Optional[Type[Exception]], interval: int = 1, retries: int = 3, backoff_rate: int = 2, wait_random: bool = False, retry=tenacity.retry_if_exception_type) -> Callable: if retries < 1: raise ValueError('Retries must be greater than or ' 'equal to 1 (received: %s). ' % retries) if wait_random: wait = tenacity.wait_random_exponential(multiplier=interval) else: wait = tenacity.wait_exponential(multiplier=interval, min=0, exp_base=backoff_rate) def _decorator(f: Callable) -> Callable: @functools.wraps(f) def _wrapper(*args, **kwargs): r = tenacity.Retrying(sleep=tenacity.nap.sleep, before_sleep=tenacity.before_sleep_log( LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), stop=tenacity.stop_after_attempt(retries), reraise=True, retry=retry(retry_param), wait=wait) return r.call(f, *args, **kwargs) return _wrapper return _decorator
def retry(exceptions, interval=1, retries=3, backoff_rate=2, wait_random=False): if retries < 1: raise ValueError('Retries must be greater than or ' 'equal to 1 (received: %s). ' % retries) if wait_random: wait = tenacity.wait_random_exponential(multiplier=interval) else: wait = tenacity.wait_exponential(multiplier=interval, min=0, exp_base=backoff_rate) def _decorator(f): @six.wraps(f) def _wrapper(*args, **kwargs): r = tenacity.Retrying( before_sleep=tenacity.before_sleep_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), stop=tenacity.stop_after_attempt(retries), reraise=True, retry=tenacity.retry_if_exception_type(exceptions), wait=wait) return r.call(f, *args, **kwargs) return _wrapper return _decorator
class GoogleAPIResource(Resource): # Names of the get and update methods. Most are the same but override in # the Resource if necessary resource_property = "" get_method = "get" update_method = "update" def __init__(self, resource_data, **kwargs): full_resource_path = "{}.{}".format(self.service_name, self.resource_path) self.service = build_subresource(full_resource_path, self.version, **kwargs) self.resource_data = resource_data @staticmethod def factory(resource_data, **kargs): resource_type_map = { 'bigquery.datasets': GcpBigqueryDataset, 'compute.instances': GcpComputeInstance, 'sqladmin.instances': GcpSqlInstance, 'storage.buckets': GcpStorageBucket, 'storage.buckets.iam': GcpStorageBucketIamPolicy } resource_type = resource_data.get('resource_type') if not resource_type: assert 0, 'Unrecognized resource' if resource_type not in resource_type_map: assert 0, 'Unrecognized resource' cls = resource_type_map.get(resource_type) return cls(resource_data, **kargs) def type(self): type_components = ["gcp", self.service_name, self.resource_path] # Things like IAM policy are not separate resources, but rather # properties of a resource. We may want to evaluate policy on these # properties, so we represent them as resources and need to distinguish # them in the resource type. if self.resource_property: type_components.append(self.resource_property) return ".".join(type_components) def get(self): method = getattr(self.service, self.get_method) return method(**self._get_request_args()).execute() @tenacity.retry(retry=tenacity.retry_if_exception(is_retryable_exception), wait=tenacity.wait_random_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(10)) def update(self, body): method = getattr(self.service, self.update_method) return method(**self._update_request_args(body)).execute()
class TestTsmsMqProduce(object): uuid = "a1e2ff2e-23c5-11ea-a694-acde48001122" @retry(stop=stop_after_attempt(5), wait=wait_random_exponential(2, max=5)) def db_retry(self, uuid): """ 异步数据校验,重试查询数据库 """ res = td.tsms_select("send", "consume,status,mobile", uuid=uuid) assert res["status"] == "success" def test_pro_01(self, mq, td): data = { "uid": self.uuid, "phone": "17134198056", "content": "【hellokitty】验证码为:123" } # 改为failed td.tsms_update("send", "status", "failed", uuid=self.uuid) res = td.tsms_select("send", "status", uuid=self.uuid) assert res.get("status") == "failed" # 写mq mq.push_direct_secure(json.dumps(data)) # 再查数据库 self.db_retry(self.uuid)
def __init__(self, *args, **kwargs): """ Parameters ---------- **kwargs ``amqp_uri`` : str Uri to be used to communicate with the AMQP broker. ``queue_name`` : str Name of the queue that should be used in the AMQP broker, to store the messages. ``alertable_event`` : threading.Event Event to be used to determine when the `self._retrying_policy` must stop re-trying. """ super().__init__() self._amqp_uri: str = kwargs.get("amqp_uri") self._queue_name: str = kwargs.get("queue_name") self._alertable_event = kwargs.get("alertable_event") self._cnx: typing.Optional[pika.BlockingConnection] = None self._channel: typing.Optional[pika.channel.Channel] = None # Define re-trying policy to be used, when fails to successfully communicate with the AMQP broker: self._retrying_policy = tenacity.Retrying( wait=tenacity.wait_random_exponential( multiplier=0.5, max=30), # Random exponential back-off before retry retry=tenacity.retry_if_exception_type( pika.exceptions.AMQPConnectionError), stop=tenacity.stop.stop_when_event_set(self._alertable_event), after=lambda _, __, ___: self._log.warning( f"Unable to connect to AMQP server with uri: {self._amqp_uri}") ) self._log.debug( f"{self.__class__.__name__}.__init__(queue_name={self._queue_name})" )
def retry_random_upon_exception(exc, delay=0.5, max_delay=5, max_attempts=DEFAULT_MAX_ATTEMPTS): return tenacity.retry(reraise=True, retry=tenacity.retry_if_exception_type(exc), wait=tenacity.wait_random_exponential( multiplier=delay, max=max_delay), stop=tenacity.stop_after_attempt(max_attempts), before=_log_before_retry, after=_log_after_retry)
def requests_retry(self): self.retry = Retrying( stop=stop_after_attempt(RETRY), retry=retry_if_exception_type(SLRateLimitExceededError), wait=wait_random_exponential(multiplier=BACK_OFF_FACTOR, max=MAX_INTERVAL), reraise=True) return self.retry
def retry_on_conflict(func): wrapper = tenacity.retry(stop=tenacity.stop_after_attempt(11), wait=tenacity.wait_random_exponential( multiplier=0.5, max=60), retry=tenacity.retry_if_exception_type( exception.ConcurrentTransaction), reraise=True) return wrapper(func)
def retry_random_upon_exception(exc, delay=0.5, max_delay=5, max_attempts=DEFAULT_MAX_ATTEMPTS): return tenacity.retry(reraise=True, retry=tenacity.retry_if_exception_type(exc), wait=tenacity.wait_random_exponential( multiplier=delay, max=max_delay), stop=tenacity.stop_after_attempt(max_attempts), before=_log_before_retry, after=_log_after_retry)
def sync_to_db(self, session: Optional[Session] = None): """Save attributes about list of DAG to the DB.""" # To avoid circular import - airflow.models.dagbag -> airflow.models.dag -> airflow.models.dagbag from airflow.models.dag import DAG from airflow.models.serialized_dag import SerializedDagModel def _serialze_dag_capturing_errors(dag, session): """ Try to serialize the dag to the DB, but make a note of any errors. We can't place them directly in import_errors, as this may be retried, and work the next time """ if dag.is_subdag: return [] try: # We cant use bulk_write_to_db as we want to capture each error individually SerializedDagModel.write_dag( dag, min_update_interval=settings.MIN_SERIALIZED_DAG_UPDATE_INTERVAL, session=session, ) return [] except OperationalError: raise except Exception: # pylint: disable=broad-except return [(dag.fileloc, traceback.format_exc(limit=-self.dagbag_import_error_traceback_depth))] # Retry 'DAG.bulk_write_to_db' & 'SerializedDagModel.bulk_sync_to_db' in case # of any Operational Errors # In case of failures, provide_session handles rollback for attempt in tenacity.Retrying( retry=tenacity.retry_if_exception_type(exception_types=OperationalError), wait=tenacity.wait_random_exponential(multiplier=0.5, max=5), stop=tenacity.stop_after_attempt(settings.MAX_DB_RETRIES), before_sleep=tenacity.before_sleep_log(self.log, logging.DEBUG), reraise=True, ): with attempt: serialize_errors = [] self.log.debug( "Running dagbag.sync_to_db with retries. Try %d of %d", attempt.retry_state.attempt_number, settings.MAX_DB_RETRIES, ) self.log.debug("Calling the DAG.bulk_sync_to_db method") try: # Write Serialized DAGs to DB, capturing errors for dag in self.dags.values(): serialize_errors.extend(_serialze_dag_capturing_errors(dag, session)) DAG.bulk_write_to_db(self.dags.values(), session=session) except OperationalError: session.rollback() raise # Only now we are "complete" do we update import_errors - don't want to record errors from # previous failed attempts self.import_errors.update(dict(serialize_errors))
class OpenPolicyAgent(Engine): def __init__(self, opa_base_url): self.opa_base_url = opa_base_url @tenacity.retry( retry=tenacity.retry_if_exception(is_retryable_exception), wait=tenacity.wait_random_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(5), ) def _opa_request(self, path, method="GET", data=None): url = "{}/{}".format(self.opa_base_url, path) headers = {"Content-type": "application/json"} req = request.Request(url, data=json.dumps(data).encode("utf-8"), method=method, headers=headers) with request.urlopen(req) as resp: decoded_resp = resp.read().decode("utf-8") deserialized_resp = json.loads(decoded_resp) if "result" not in deserialized_resp: err = "Endpoint {} not found on the OPA server.".format(url) raise NoSuchEndpoint(err) return deserialized_resp["result"] # Perform an evaluation on a given resource def evaluate(self, resource): input = { "input": resource.get(), } evals = self._opa_request("rpe/evaluate", method="POST", data=input) return [ Evaluation(engine=self, resource=resource, **ev) for ev in evals ] def policies(self): """ Returns: A list of all configured policies, optionally filtered by a resource_type """ policies = self._opa_request("rpe/policies") return [Policy(engine=self, **p) for p in policies] def remediate(self, resource, policy_id): rem_path = "rpe/policy/{}/remediate".format(policy_id) input = {"input": resource.get()} remediation = self._opa_request(rem_path, method="POST", data=input) if remediation: resource.remediate(remediation) else: raise NoPossibleRemediation( "Remediation is not supported for this resource/policy")
def __set_throttling_settings(self, num_of_attempts: int = 4, max_seconds_per_retry: float = 30): self._retryer = Retrying( wait=wait_random_exponential(max=max_seconds_per_retry), retry=retry_if_exception_type(KustoThrottlingError), stop=stop_after_attempt(num_of_attempts), reraise=True, )
class _Streaming(): """ await streaming.start(server=queue_uri, client_name="service-spawner", loop=loop) """ def __init__(self) -> None: self._nc: NATS = None self._sc: STAN = None self._status = False self._subscription: Dict = {} self.service_group: Optional[str] = None @retry(wait=wait_random_exponential(multiplier=1, max=10)) async def start(self, server: str, client_name: str, service_group: str, loop: asyncio.AbstractEventLoop=None): """Start connection with the streams.""" if self._status is False: loop = loop or asyncio.get_event_loop() self.service_group = service_group self._nc = NATS() await self._nc.connect(servers=[server], io_loop=loop) # Start session with NATS Streaming cluster. self._sc = STAN() await self._sc.connect("test-cluster", client_name, nats=self._nc) self._status = True log.info("Streaming connected.") else: log.info("Streaming already connected.") async def publish(self, name: str, data: Dict) -> None: """Publish a message inside a queue.""" if self._status: body = msgpack.packb(data) await self._sc.publish(name, body) log.info(f"Event {data} published inside {name}") else: raise RuntimeError("Streaming is not active.") async def subscribe(self, name: str, callback: Union[Callable, Awaitable]): """Subscribe to a given channel.""" self._subscription[name] = await self._sc.subscribe( name, queue=self.service_group, durable_name="durable", cb=callback) async def unsubscribe(self, name: str) -> None: """Unsubscribe from a given channel.""" if name in self._subscription: await self._subscription[name].unsubscribe() async def stop(self) -> None: """Close all connections.""" log.warning("Closing connections....") for subsciption in self._subscription.values(): await subsciption.unsubscribe() await self._sc.close() await self._nc.close() self._status = False
def __attrs_post_init__(self): if self.max_retry_delay is None: self.max_retry_delay = self.timeout * 3 self._retry_upload = tenacity.Retrying( # Retry after 1s, 2s, 4s, 8s with some randomness wait=tenacity.wait_random_exponential(multiplier=0.5), stop=tenacity.stop_after_delay(self.max_retry_delay), retry_error_cls=UploadFailed, retry=tenacity.retry_if_exception_type((http_client.HTTPException, OSError, IOError)), )
def __init__(self): # throttling self.throttling_wait = wait_chain( # always wait 20-40s first wait_fixed(20) + wait_random(0, 20), # wait 20-40s again wait_fixed(20) + wait_random(0, 20), # wait from 30 to 630s, with full jitter and exponentially # increasing max wait time wait_fixed(30) + wait_random_exponential(multiplier=1, max=600) ) # connection errors, other client and server failures self.network_wait = ( # wait from 3s to ~1m wait_random(3, 7) + wait_random_exponential(multiplier=1, max=55) ) self.server_wait = self.network_wait
def export(self, events, start_time_ns, end_time_ns): """Export events to an HTTP endpoint. :param events: The event dictionary from a `ddtrace.profiling.recorder.Recorder`. :param start_time_ns: The start time of recording. :param end_time_ns: The end time of recording. """ if not self.endpoint: raise InvalidEndpoint("Endpoint is empty") common_headers = { "DD-API-KEY": self.api_key.encode(), } profile = super(PprofHTTPExporter, self).export(events, start_time_ns, end_time_ns) s = six.BytesIO() with gzip.GzipFile(fileobj=s, mode="wb") as gz: gz.write(profile.SerializeToString()) fields = { "runtime-id": runtime.get_runtime_id().encode("ascii"), "recording-start": ( datetime.datetime.utcfromtimestamp(start_time_ns / 1e9).replace(microsecond=0).isoformat() + "Z" ).encode(), "recording-end": ( datetime.datetime.utcfromtimestamp(end_time_ns / 1e9).replace(microsecond=0).isoformat() + "Z" ).encode(), "runtime": PYTHON_IMPLEMENTATION, "format": b"pprof", "type": b"cpu+alloc+exceptions", "chunk-data": s.getvalue(), } service_name = self.service_name or os.path.basename(profile.string_table[profile.mapping[0].filename]) content_type, body = self._encode_multipart_formdata(fields, tags=self._get_tags(service_name),) headers = common_headers.copy() headers["Content-Type"] = content_type # urllib uses `POST` if `data` is supplied (Python 2 version does not handle `method` kwarg) req = request.Request(self.endpoint, data=body, headers=headers) retry = tenacity.Retrying( # Retry after 1s, 2s, 4s, 8s with some randomness wait=tenacity.wait_random_exponential(multiplier=0.5), stop=tenacity.stop_after_delay(self.max_retry_delay), retry=tenacity.retry_if_exception_type( (error.HTTPError, error.URLError, http_client.HTTPException, OSError, IOError) ), ) try: retry(request.urlopen, req, timeout=self.timeout) except tenacity.RetryError as e: raise UploadFailed(e.last_attempt.exception())
class DataBaseConnection: @retry(wait=wait_random_exponential(multiplier=1, max=60)) def connect(self) -> None: hostname = os.getenv("DB_HOSTNAME", "db") self.client = AsyncIOMotorClient(host=hostname) self.db = DataBase(self.client) print("Connected to database") def disconnect(self) -> None: self.client.close() print("Disconnected from database")
def retry_upon_none_result(max_attempts, delay=0.5, max_delay=2, random=False): if random: wait_func = tenacity.wait_exponential( multiplier=delay, max=max_delay) else: wait_func = tenacity.wait_random_exponential( multiplier=delay, max=max_delay) return tenacity.retry(reraise=True, retry=tenacity.retry_if_result(lambda x: x is None), wait=wait_func, stop=tenacity.stop_after_attempt(max_attempts), before=_log_before_retry, after=_log_after_retry)
def retry_upon_none_result(max_attempts, delay=0.5, max_delay=2, random=False): if random: wait_func = tenacity.wait_exponential(multiplier=delay, max=max_delay) else: wait_func = tenacity.wait_random_exponential(multiplier=delay, max=max_delay) return tenacity.retry(reraise=True, retry=tenacity.retry_if_result(lambda x: x is None), wait=wait_func, stop=tenacity.stop_after_attempt(max_attempts), before=_log_before_retry, after=_log_after_retry)
def retry_429(func): @retry(wait=wait_random_exponential(multiplier=EXPONENTIAL_BACKOFF_MULTIPLIER, max=MAX_SECONDS_WAIT), retry=retry_if_exception_type(RateLimitException), stop=stop_after_attempt(MAX_RETRY_ATTEMPTS), reraise=True, before_sleep=before_sleep_on_429) def wrapped_function(*args, **kwargs): try: return func(*args, **kwargs) except HTTPError as e: if e.response.status_code == 429: raise RateLimitException("429 Too Many Requests") raise e return wrapped_function
def test_wait_random_exponential(self): fn = tenacity.wait_random_exponential(0.5, 60.0) for _ in six.moves.range(1000): self._assert_inclusive_range(fn(make_retry_state(1, 0)), 0, 1.0) self._assert_inclusive_range(fn(make_retry_state(2, 0)), 0, 2.0) self._assert_inclusive_range(fn(make_retry_state(3, 0)), 0, 4.0) self._assert_inclusive_range(fn(make_retry_state(4, 0)), 0, 8.0) self._assert_inclusive_range(fn(make_retry_state(5, 0)), 0, 16.0) self._assert_inclusive_range(fn(make_retry_state(6, 0)), 0, 32.0) self._assert_inclusive_range(fn(make_retry_state(7, 0)), 0, 60.0) self._assert_inclusive_range(fn(make_retry_state(8, 0)), 0, 60.0) self._assert_inclusive_range(fn(make_retry_state(9, 0)), 0, 60.0) fn = tenacity.wait_random_exponential(10, 5) for _ in six.moves.range(1000): self._assert_inclusive_range(fn(make_retry_state(1, 0)), 0.00, 5.00) # Default arguments exist fn = tenacity.wait_random_exponential() fn(0, 0)
def wait(self, timeout=60, wait_exp_multiplier=0.05, wait_exp_max=1.0): """ Wait until task is completed. Expires after 'timeout' seconds. """ try: retryer = Retrying(wait=wait_random_exponential(multiplier=wait_exp_multiplier, max=wait_exp_max), stop=stop_after_delay(timeout), retry=retry_if_result(is_pending_status), before=before_log(logger, logging.DEBUG), after=after_log(logger, logging.DEBUG)) retryer(self._refresh_status) except RetryError: raise TaskTimeout(self.data()) if is_error_status(self['status']): raise TaskError(self.data()) return self