class Contract(object):
def __init__(self, network_name):
self.name = network_name
@retry(wait=wait_random_exponential(multiplier=1, max=10))
def ensure_network_is_running(self):
api = API()
if not api.is_network_running(self.name):
api.start_network(self.name)
if not api.is_network_running(self.name):
raise FailedToStartException(self.name)
@retry(wait=wait_random_exponential(multiplier=1, max=10))
def ensure_network_is_stopped(self):
api = API()
if api.is_network_running(self.name):
api.stop_network(self.name)
if api.is_network_running(self.name):
raise FailedToStopException(self.name)
@retry(wait=wait_random_exponential(multiplier=2, max=10),
stop=stop_after_attempt(3),
reraise=True)
def check_network_health(self):
logger.info("Checking '{}' network health...".format(self.name))
api = API()
anchors = api.get_anchors(self.name)
degraded_state_errors = []
if len(anchors) == 0:
degraded_state_errors.append("No anchors found in CUWB network")
for a in anchors:
anchor_name = a.get("name", "UNKNOWN")
connectivity = a.get("connectivity_state", "")
synchronization = a.get("synchronization_state", "")
if not connectivity.lower() == "ethernet and uwb connected":
degraded_state_errors.append(
"Anchor '{}' reporting bad connectivity - `{}`".format(
anchor_name, connectivity))
if not synchronization.lower() == "rx and tx synced":
degraded_state_errors.append(
"Anchor '{}' reporting bad synchronization - `{}`".format(
anchor_name, synchronization))
if len(degraded_state_errors) > 0:
logger.error("'{}' network health degraded: {}".format(
self.name, degraded_state_errors))
raise AnchorsDegradedException(self.name, degraded_state_errors)
def run_compute_singles(run, site, processed_output_path, database):
"""Compute the singles (underlying uncorrelated) rate."""
path_prefix = os.path.join(processed_output_path, f'EH{site}')
ads = common.dets_for(site, run)
update_db = True
iteration = 0
extra_cut = '1'
for ad in ads:
infile = os.path.join(
path_prefix,
f'hadded_ad{ad}/out_ad{ad}_{run}.root',
)
# Ideally should check to see if rate has been computed before.
# But, this is so fast that I will just re-compute every time.
for attempt in tenacity.Retrying(
reraise=True,
wait=tenacity.wait_random_exponential(max=60),
retry=tenacity.retry_if_exception_type(sqlite3.Error),
before_sleep=tenacity.before_sleep_log(logging, logging.DEBUG),
):
with attempt:
compute_singles.main(
infile,
database,
GENERAL_LABEL,
update_db,
iteration,
extra_cut,
)
return
def sync(cnxt, entity_id, current_traversal, is_update, propagate,
predecessors, new_data):
# Retry waits up to 60 seconds at most, with exponentially increasing
# amounts of jitter per resource still outstanding
wait_strategy = tenacity.wait_random_exponential(max=60)
def init_jitter(existing_input_data):
nconflicts = max(0, len(predecessors) - len(existing_input_data) - 1)
# 10ms per potential conflict, up to a max of 10s in total
return min(nconflicts, 1000) * 0.01
@tenacity.retry(
retry=tenacity.retry_if_result(lambda r: r is None),
wait=wait_strategy
)
def _sync():
sync_point = get(cnxt, entity_id, current_traversal, is_update)
input_data = deserialize_input_data(sync_point.input_data)
wait_strategy.multiplier = init_jitter(input_data)
input_data.update(new_data)
rows_updated = update_input_data(
cnxt, entity_id, current_traversal, is_update,
sync_point.atomic_key, serialize_input_data(input_data))
return input_data if rows_updated else None
input_data = _sync()
waiting = predecessors - set(input_data)
key = make_key(entity_id, current_traversal, is_update)
if waiting:
LOG.debug('[%s] Waiting %s: Got %s; still need %s',
key, entity_id, _dump_list(input_data), _dump_list(waiting))
else:
LOG.debug('[%s] Ready %s: Got %s',
key, entity_id, _dump_list(input_data))
propagate(entity_id, serialize_input_data(input_data))
def sync_to_db(self, session: Optional[Session] = None):
"""Save attributes about list of DAG to the DB."""
# To avoid circular import - airflow.models.dagbag -> airflow.models.dag -> airflow.models.dagbag
from airflow.models.dag import DAG
from airflow.models.serialized_dag import SerializedDagModel
# Retry 'DAG.bulk_write_to_db' & 'SerializedDagModel.bulk_sync_to_db' in case
# of any Operational Errors
# In case of failures, provide_session handles rollback
for attempt in tenacity.Retrying(
retry=tenacity.retry_if_exception_type(
exception_types=OperationalError),
wait=tenacity.wait_random_exponential(multiplier=0.5, max=5),
stop=tenacity.stop_after_attempt(settings.MAX_DB_RETRIES),
before_sleep=tenacity.before_sleep_log(self.log,
logging.DEBUG),
reraise=True):
with attempt:
self.log.debug(
"Running dagbag.sync_to_db with retries. Try %d of %d",
attempt.retry_state.attempt_number,
settings.MAX_DB_RETRIES)
self.log.debug("Calling the DAG.bulk_sync_to_db method")
try:
DAG.bulk_write_to_db(self.dags.values(), session=session)
# Write Serialized DAGs to DB
self.log.debug(
"Calling the SerializedDagModel.bulk_sync_to_db method"
)
SerializedDagModel.bulk_sync_to_db(self.dags.values(),
session=session)
except OperationalError:
session.rollback()
raise
def sync(cnxt, entity_id, current_traversal, is_update, propagate,
predecessors, new_data):
# Retry waits up to 60 seconds at most, with exponentially increasing
# amounts of jitter per resource still outstanding
wait_strategy = tenacity.wait_random_exponential(max=60)
def init_jitter(existing_input_data):
nconflicts = max(0, len(predecessors) - len(existing_input_data) - 1)
# 10ms per potential conflict, up to a max of 10s in total
return min(nconflicts, 1000) * 0.01
@tenacity.retry(retry=tenacity.retry_if_result(lambda r: r is None),
wait=wait_strategy)
def _sync():
sync_point = get(cnxt, entity_id, current_traversal, is_update)
input_data = deserialize_input_data(sync_point.input_data)
wait_strategy.multiplier = init_jitter(input_data)
input_data.update(new_data)
rows_updated = update_input_data(cnxt, entity_id, current_traversal,
is_update, sync_point.atomic_key,
serialize_input_data(input_data))
return input_data if rows_updated else None
input_data = _sync()
waiting = predecessors - set(input_data)
key = make_key(entity_id, current_traversal, is_update)
if waiting:
LOG.debug('[%s] Waiting %s: Got %s; still need %s', key, entity_id,
_dump_list(input_data), _dump_list(waiting))
else:
LOG.debug('[%s] Ready %s: Got %s', key, entity_id,
_dump_list(input_data))
propagate(entity_id, serialize_input_data(input_data))
def batch_wait(self, tasks, timeout=300, wait_exp_multiplier=0.05, wait_exp_max=1.0):
"""
Wait until a list of task are completed. Expires after 'timeout' seconds.
Returns a tuple of list (pending_tasks, success_tasks, error_tasks).
Each list contains a couple (original_position, task) sorted by original_position asc
original_position gives the original index in the input tasks list parameter. This helps to keep the order.
"""
try:
positions = {}
pending_tasks = []
for pos, task in enumerate(tasks):
positions[task.pk] = pos
pending_tasks.append((pos, task))
success_tasks = []
error_tasks = []
retryer = Retrying(wait=wait_random_exponential(multiplier=wait_exp_multiplier, max=wait_exp_max),
stop=stop_after_delay(timeout),
retry=retry_if_result(has_pending_tasks),
before=before_log(logger, logging.DEBUG),
after=after_log(logger, logging.DEBUG))
retryer(self._refresh_tasks_status, pending_tasks, success_tasks, error_tasks, positions)
except RetryError:
pass
return (sorted(pending_tasks, key=lambda v: v[0]),
sorted(success_tasks, key=lambda v: v[0]),
sorted(error_tasks, key=lambda v: v[0]))
def retry_request(method, prepared_req, session, max_retries=10):
retry_conditions = None
def _make_request(req, sess):
"""send the prepared session request"""
response = sess.send(req)
return response
def _return_last_value(retry_state):
"""return the result of the last call attempt
and let code pick up the error"""
return retry_state.outcome.result()
if method.upper() == 'POST':
retry_conditions = (retry_if_result(
lambda res: res.status_code in civis.civis.POST_RETRY_CODES))
elif method.upper() in civis.civis.RETRY_VERBS:
retry_conditions = (retry_if_result(
lambda res: res.status_code in civis.civis.RETRY_CODES))
if retry_conditions:
retry_config = Retrying(
retry=retry_conditions,
wait=wait_for_retry_after_header(
fallback=wait_random_exponential(multiplier=2, max=60)),
stop=(stop_after_delay(600) | stop_after_attempt(max_retries)),
retry_error_callback=_return_last_value,
)
response = retry_config(_make_request, prepared_req, session)
return response
response = _make_request(prepared_req, session)
return response
class PushManager:
"""
Manager around the push notification sending that handles retrying
"""
def __init__(self, quit_event):
self._quit_event = quit_event
@retry(
stop=stop_after_attempt(5),
wait=wait_random_exponential(),
retry=retry_if_exception_type(get_pushing_error_classes()),
)
def push(self, notification, push_sent_event):
"""
Send a push notification and retry in cases of failures.
This function will try sending up to 5 times, and uses an exponential
backoff strategy, where a random waiting time is chosen between 0 and a max
value that doubles every retry, to avoid all sender processes retrying at the
same time.
"""
if self._quit_event.is_set():
raise GracefulExit()
push_notification(notification, push_sent_event)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass
def __attrs_post_init__(self):
if self.max_retry_delay is None:
self.max_retry_delay = self.timeout * 3
self._retry_upload = tenacity.Retrying(
# Retry after 1s, 2s, 4s, 8s with some randomness
wait=tenacity.wait_random_exponential(multiplier=0.5),
stop=tenacity.stop_after_delay(self.max_retry_delay),
retry_error_cls=UploadFailed,
retry=tenacity.retry_if_exception_type((http_client.HTTPException, OSError, IOError)),
)
tags = {
k: six.ensure_binary(v)
for k, v in itertools.chain(
parse_tags_str(os.environ.get("DD_TAGS")).items(),
parse_tags_str(os.environ.get("DD_PROFILING_TAGS")).items(),
)
}
tags.update({k: six.ensure_binary(v) for k, v in self.tags.items()})
tags.update(
{
"host": HOSTNAME.encode("utf-8"),
"language": b"python",
"runtime": PYTHON_IMPLEMENTATION,
"runtime_version": PYTHON_VERSION,
"profiler_version": ddtrace.__version__.encode("ascii"),
}
)
if self.version:
tags["version"] = self.version.encode("utf-8")
if self.env:
tags["env"] = self.env.encode("utf-8")
self.tags = tags
def __init__(self,
configuration=None,
header_name=None,
header_value=None,
cookie=None,
pool_threads=1):
if configuration is None:
configuration = Configuration()
self.configuration = configuration
self.pool_threads = pool_threads
self.retrying = tenacity.Retrying(
stop=tenacity.stop_after_attempt(configuration.retry_count),
wait=tenacity.wait_random_exponential(
multiplier=configuration.back_off,
max=configuration.retry_max_delay,
min=configuration.retry_delay),
retry=(tenacity.retry_if_result(self.is_retry_enabled) and
((tenacity.retry_if_exception_type(RetryableException)) |
(tenacity.retry_if_exception_type(HTTPError)))))
self.rest_client = rest.RESTClientObject(configuration,
retrying=self.retrying)
self.default_headers = {}
if header_name is not None:
self.default_headers[header_name] = header_value
self.cookie = cookie
# Set default User-Agent.
self.user_agent = 'opsgenie-sdk-python-2.0.2'
# init metric publishers
self.http_metric_publisher = self.rest_client.http_metric
self.api_metric_publisher = metrics.ApiMetric('ApiMetricPublisher')
self.sdk_metric_publisher = metrics.SdkMetric('SdkMetricPublisher')
def run_shell_expbackoff(command,
debug=False,
timeout=60.0,
close_fds=True,
redirect_stdout=subprocess.PIPE,
redirect_stderr=subprocess.PIPE,
return_outerr=False,
raise_exceptions=False,
retries=4,
retry_multiplier=1.0,
retry_maxtime=30.0):
'''This runs a subprocess with shell=True, a configured timeout
for the initial run, and a configurable number of retries
with an exponential backoff.
'''
@retry(stop=stop_after_attempt(retries),
wait=wait_random_exponential(multiplier=retry_multiplier,
max=retry_maxtime),
reraise=raise_exceptions)
def runner(command, debug, timeout, close_fds, return_outerr,
redirect_stdout, redirect_stderr):
return run_shell(command,
debug=debug,
redirect_stdout=redirect_stdout,
redirect_stderr=redirect_stderr,
timeout=timeout,
close_fds=close_fds,
return_outerr=return_outerr)
return runner(command, debug, timeout, close_fds, return_outerr,
redirect_stdout, redirect_stderr)
class TestPushMq(object):
"""mq测试用例"""
uuid = '367dab7a-2a1a-11ea-90ca-acde48001122'
@retry(stop=stop_after_attempt(5), wait=wait_random_exponential(2, max=5))
def db_retry(self, uuid, od):
"""异步数据校验，重试查询数据库"""
res = od.tsms_select('sms_send', 'consume,status,mobile', uuid)
assert res[0].get("status") == "success"
def test_push_001(self, mq, od):
"""消息发送MQ生产消费功能测试"""
data = {
"uid": self.uuid,
"phone": "18535941072",
"content": "【hellokitty】验证码为:1234"
}
#将测试数据状态改为failed
od.tsms_update('sms_send', 'status', 'failed', uuid=self.uuid)
res = od.tsms_select('sms_send', 'status', uuid=self.uuid)
assert res[0].get("status") == "failed"
#写MQ,将字典data通过json序列化成json字符串
mq.push_direct_secure(json.dumps(data))
#查数据库
self.db_retry(self.uuid)
def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult:
stream_descriptor = BaseIngestClient._prepare_stream(stream_descriptor, ingestion_properties)
stream = stream_descriptor.stream
buffered_stream = read_until_size_or_end(stream, self.MAX_STREAMING_SIZE_IN_BYTES + 1)
if len(buffered_stream.getbuffer()) > self.MAX_STREAMING_SIZE_IN_BYTES:
stream_descriptor.stream = chain_streams([buffered_stream, stream])
return self.queued_client.ingest_from_stream(stream_descriptor, ingestion_properties)
stream_descriptor.stream = buffered_stream
try:
for attempt in Retrying(
stop=stop_after_attempt(self._num_of_attempts), wait=wait_random_exponential(max=self._max_seconds_per_retry), reraise=True
):
with attempt:
stream.seek(0, SEEK_SET)
client_request_id = ManagedStreamingIngestClient._get_request_id(stream_descriptor.source_id, attempt.retry_state.attempt_number - 1)
return self.streaming_client._ingest_from_stream_with_client_request_id(stream_descriptor, ingestion_properties, client_request_id)
except KustoApiError as ex:
error = ex.get_api_error()
if error.permanent:
raise
return self.queued_client.ingest_from_stream(stream_descriptor, ingestion_properties)
def retry(retry_param: Optional[Type[Exception]],
interval: int = 1,
retries: int = 3,
backoff_rate: int = 2,
wait_random: bool = False,
retry=tenacity.retry_if_exception_type) -> Callable:
if retries < 1:
raise ValueError('Retries must be greater than or '
'equal to 1 (received: %s). ' % retries)
if wait_random:
wait = tenacity.wait_random_exponential(multiplier=interval)
else:
wait = tenacity.wait_exponential(multiplier=interval,
min=0,
exp_base=backoff_rate)
def _decorator(f: Callable) -> Callable:
@functools.wraps(f)
def _wrapper(*args, **kwargs):
r = tenacity.Retrying(sleep=tenacity.nap.sleep,
before_sleep=tenacity.before_sleep_log(
LOG, logging.DEBUG),
after=tenacity.after_log(LOG, logging.DEBUG),
stop=tenacity.stop_after_attempt(retries),
reraise=True,
retry=retry(retry_param),
wait=wait)
return r.call(f, *args, **kwargs)
return _wrapper
return _decorator
def retry(exceptions,
interval=1,
retries=3,
backoff_rate=2,
wait_random=False):
if retries < 1:
raise ValueError('Retries must be greater than or '
'equal to 1 (received: %s). ' % retries)
if wait_random:
wait = tenacity.wait_random_exponential(multiplier=interval)
else:
wait = tenacity.wait_exponential(multiplier=interval,
min=0,
exp_base=backoff_rate)
def _decorator(f):
@six.wraps(f)
def _wrapper(*args, **kwargs):
r = tenacity.Retrying(
before_sleep=tenacity.before_sleep_log(LOG, logging.DEBUG),
after=tenacity.after_log(LOG, logging.DEBUG),
stop=tenacity.stop_after_attempt(retries),
reraise=True,
retry=tenacity.retry_if_exception_type(exceptions),
wait=wait)
return r.call(f, *args, **kwargs)
return _wrapper
return _decorator
class GoogleAPIResource(Resource):
# Names of the get and update methods. Most are the same but override in
# the Resource if necessary
resource_property = ""
get_method = "get"
update_method = "update"
def __init__(self, resource_data, **kwargs):
full_resource_path = "{}.{}".format(self.service_name,
self.resource_path)
self.service = build_subresource(full_resource_path, self.version,
**kwargs)
self.resource_data = resource_data
@staticmethod
def factory(resource_data, **kargs):
resource_type_map = {
'bigquery.datasets': GcpBigqueryDataset,
'compute.instances': GcpComputeInstance,
'sqladmin.instances': GcpSqlInstance,
'storage.buckets': GcpStorageBucket,
'storage.buckets.iam': GcpStorageBucketIamPolicy
}
resource_type = resource_data.get('resource_type')
if not resource_type:
assert 0, 'Unrecognized resource'
if resource_type not in resource_type_map:
assert 0, 'Unrecognized resource'
cls = resource_type_map.get(resource_type)
return cls(resource_data, **kargs)
def type(self):
type_components = ["gcp", self.service_name, self.resource_path]
# Things like IAM policy are not separate resources, but rather
# properties of a resource. We may want to evaluate policy on these
# properties, so we represent them as resources and need to distinguish
# them in the resource type.
if self.resource_property:
type_components.append(self.resource_property)
return ".".join(type_components)
def get(self):
method = getattr(self.service, self.get_method)
return method(**self._get_request_args()).execute()
@tenacity.retry(retry=tenacity.retry_if_exception(is_retryable_exception),
wait=tenacity.wait_random_exponential(multiplier=1,
max=10),
stop=tenacity.stop_after_attempt(10))
def update(self, body):
method = getattr(self.service, self.update_method)
return method(**self._update_request_args(body)).execute()
class TestTsmsMqProduce(object):
uuid = "a1e2ff2e-23c5-11ea-a694-acde48001122"
@retry(stop=stop_after_attempt(5), wait=wait_random_exponential(2, max=5))
def db_retry(self, uuid):
"""
异步数据校验，重试查询数据库
"""
res = td.tsms_select("send", "consume,status,mobile", uuid=uuid)
assert res["status"] == "success"
def test_pro_01(self, mq, td):
data = {
"uid": self.uuid,
"phone": "17134198056",
"content": "【hellokitty】验证码为：123"
}
# 改为failed
td.tsms_update("send", "status", "failed", uuid=self.uuid)
res = td.tsms_select("send", "status", uuid=self.uuid)
assert res.get("status") == "failed"
# 写mq
mq.push_direct_secure(json.dumps(data))
# 再查数据库
self.db_retry(self.uuid)
def __init__(self, *args, **kwargs):
"""
Parameters
----------
**kwargs
``amqp_uri`` : str
Uri to be used to communicate with the AMQP broker.
``queue_name`` : str
Name of the queue that should be used in the AMQP broker, to store the messages.
``alertable_event`` : threading.Event
Event to be used to determine when the `self._retrying_policy` must stop re-trying.
"""
super().__init__()
self._amqp_uri: str = kwargs.get("amqp_uri")
self._queue_name: str = kwargs.get("queue_name")
self._alertable_event = kwargs.get("alertable_event")
self._cnx: typing.Optional[pika.BlockingConnection] = None
self._channel: typing.Optional[pika.channel.Channel] = None
# Define re-trying policy to be used, when fails to successfully communicate with the AMQP broker:
self._retrying_policy = tenacity.Retrying(
wait=tenacity.wait_random_exponential(
multiplier=0.5,
max=30), # Random exponential back-off before retry
retry=tenacity.retry_if_exception_type(
pika.exceptions.AMQPConnectionError),
stop=tenacity.stop.stop_when_event_set(self._alertable_event),
after=lambda _, __, ___: self._log.warning(
f"Unable to connect to AMQP server with uri: {self._amqp_uri}")
)
self._log.debug(
f"{self.__class__.__name__}.__init__(queue_name={self._queue_name})"
)
def retry_random_upon_exception(exc, delay=0.5, max_delay=5,
max_attempts=DEFAULT_MAX_ATTEMPTS):
return tenacity.retry(reraise=True,
retry=tenacity.retry_if_exception_type(exc),
wait=tenacity.wait_random_exponential(
multiplier=delay, max=max_delay),
stop=tenacity.stop_after_attempt(max_attempts),
before=_log_before_retry, after=_log_after_retry)
def requests_retry(self):
self.retry = Retrying(
stop=stop_after_attempt(RETRY),
retry=retry_if_exception_type(SLRateLimitExceededError),
wait=wait_random_exponential(multiplier=BACK_OFF_FACTOR,
max=MAX_INTERVAL),
reraise=True)
return self.retry
def retry_on_conflict(func):
wrapper = tenacity.retry(stop=tenacity.stop_after_attempt(11),
wait=tenacity.wait_random_exponential(
multiplier=0.5, max=60),
retry=tenacity.retry_if_exception_type(
exception.ConcurrentTransaction),
reraise=True)
return wrapper(func)
def retry_random_upon_exception(exc, delay=0.5, max_delay=5,
max_attempts=DEFAULT_MAX_ATTEMPTS):
return tenacity.retry(reraise=True,
retry=tenacity.retry_if_exception_type(exc),
wait=tenacity.wait_random_exponential(
multiplier=delay, max=max_delay),
stop=tenacity.stop_after_attempt(max_attempts),
before=_log_before_retry, after=_log_after_retry)
def sync_to_db(self, session: Optional[Session] = None):
"""Save attributes about list of DAG to the DB."""
# To avoid circular import - airflow.models.dagbag -> airflow.models.dag -> airflow.models.dagbag
from airflow.models.dag import DAG
from airflow.models.serialized_dag import SerializedDagModel
def _serialze_dag_capturing_errors(dag, session):
"""
Try to serialize the dag to the DB, but make a note of any errors.
We can't place them directly in import_errors, as this may be retried, and work the next time
"""
if dag.is_subdag:
return []
try:
# We cant use bulk_write_to_db as we want to capture each error individually
SerializedDagModel.write_dag(
dag,
min_update_interval=settings.MIN_SERIALIZED_DAG_UPDATE_INTERVAL,
session=session,
)
return []
except OperationalError:
raise
except Exception: # pylint: disable=broad-except
return [(dag.fileloc, traceback.format_exc(limit=-self.dagbag_import_error_traceback_depth))]
# Retry 'DAG.bulk_write_to_db' & 'SerializedDagModel.bulk_sync_to_db' in case
# of any Operational Errors
# In case of failures, provide_session handles rollback
for attempt in tenacity.Retrying(
retry=tenacity.retry_if_exception_type(exception_types=OperationalError),
wait=tenacity.wait_random_exponential(multiplier=0.5, max=5),
stop=tenacity.stop_after_attempt(settings.MAX_DB_RETRIES),
before_sleep=tenacity.before_sleep_log(self.log, logging.DEBUG),
reraise=True,
):
with attempt:
serialize_errors = []
self.log.debug(
"Running dagbag.sync_to_db with retries. Try %d of %d",
attempt.retry_state.attempt_number,
settings.MAX_DB_RETRIES,
)
self.log.debug("Calling the DAG.bulk_sync_to_db method")
try:
# Write Serialized DAGs to DB, capturing errors
for dag in self.dags.values():
serialize_errors.extend(_serialze_dag_capturing_errors(dag, session))
DAG.bulk_write_to_db(self.dags.values(), session=session)
except OperationalError:
session.rollback()
raise
# Only now we are "complete" do we update import_errors - don't want to record errors from
# previous failed attempts
self.import_errors.update(dict(serialize_errors))
class OpenPolicyAgent(Engine):
def __init__(self, opa_base_url):
self.opa_base_url = opa_base_url
@tenacity.retry(
retry=tenacity.retry_if_exception(is_retryable_exception),
wait=tenacity.wait_random_exponential(multiplier=1, max=10),
stop=tenacity.stop_after_attempt(5),
)
def _opa_request(self, path, method="GET", data=None):
url = "{}/{}".format(self.opa_base_url, path)
headers = {"Content-type": "application/json"}
req = request.Request(url,
data=json.dumps(data).encode("utf-8"),
method=method,
headers=headers)
with request.urlopen(req) as resp:
decoded_resp = resp.read().decode("utf-8")
deserialized_resp = json.loads(decoded_resp)
if "result" not in deserialized_resp:
err = "Endpoint {} not found on the OPA server.".format(url)
raise NoSuchEndpoint(err)
return deserialized_resp["result"]
# Perform an evaluation on a given resource
def evaluate(self, resource):
input = {
"input": resource.get(),
}
evals = self._opa_request("rpe/evaluate", method="POST", data=input)
return [
Evaluation(engine=self, resource=resource, **ev) for ev in evals
]
def policies(self):
"""
Returns:
A list of all configured policies, optionally filtered by a resource_type
"""
policies = self._opa_request("rpe/policies")
return [Policy(engine=self, **p) for p in policies]
def remediate(self, resource, policy_id):
rem_path = "rpe/policy/{}/remediate".format(policy_id)
input = {"input": resource.get()}
remediation = self._opa_request(rem_path, method="POST", data=input)
if remediation:
resource.remediate(remediation)
else:
raise NoPossibleRemediation(
"Remediation is not supported for this resource/policy")
def __set_throttling_settings(self,
num_of_attempts: int = 4,
max_seconds_per_retry: float = 30):
self._retryer = Retrying(
wait=wait_random_exponential(max=max_seconds_per_retry),
retry=retry_if_exception_type(KustoThrottlingError),
stop=stop_after_attempt(num_of_attempts),
reraise=True,
)
class _Streaming():
"""
await streaming.start(server=queue_uri, client_name="service-spawner", loop=loop)
"""
def __init__(self) -> None:
self._nc: NATS = None
self._sc: STAN = None
self._status = False
self._subscription: Dict = {}
self.service_group: Optional[str] = None
@retry(wait=wait_random_exponential(multiplier=1, max=10))
async def start(self, server: str, client_name: str, service_group: str, loop: asyncio.AbstractEventLoop=None):
"""Start connection with the streams."""
if self._status is False:
loop = loop or asyncio.get_event_loop()
self.service_group = service_group
self._nc = NATS()
await self._nc.connect(servers=[server], io_loop=loop)
# Start session with NATS Streaming cluster.
self._sc = STAN()
await self._sc.connect("test-cluster", client_name, nats=self._nc)
self._status = True
log.info("Streaming connected.")
else:
log.info("Streaming already connected.")
async def publish(self, name: str, data: Dict) -> None:
"""Publish a message inside a queue."""
if self._status:
body = msgpack.packb(data)
await self._sc.publish(name, body)
log.info(f"Event {data} published inside {name}")
else:
raise RuntimeError("Streaming is not active.")
async def subscribe(self, name: str, callback: Union[Callable, Awaitable]):
"""Subscribe to a given channel."""
self._subscription[name] = await self._sc.subscribe(
name, queue=self.service_group, durable_name="durable", cb=callback)
async def unsubscribe(self, name: str) -> None:
"""Unsubscribe from a given channel."""
if name in self._subscription:
await self._subscription[name].unsubscribe()
async def stop(self) -> None:
"""Close all connections."""
log.warning("Closing connections....")
for subsciption in self._subscription.values():
await subsciption.unsubscribe()
await self._sc.close()
await self._nc.close()
self._status = False
def __attrs_post_init__(self):
if self.max_retry_delay is None:
self.max_retry_delay = self.timeout * 3
self._retry_upload = tenacity.Retrying(
# Retry after 1s, 2s, 4s, 8s with some randomness
wait=tenacity.wait_random_exponential(multiplier=0.5),
stop=tenacity.stop_after_delay(self.max_retry_delay),
retry_error_cls=UploadFailed,
retry=tenacity.retry_if_exception_type((http_client.HTTPException, OSError, IOError)),
)
def __init__(self):
# throttling
self.throttling_wait = wait_chain(
# always wait 20-40s first
wait_fixed(20) + wait_random(0, 20),
# wait 20-40s again
wait_fixed(20) + wait_random(0, 20),
# wait from 30 to 630s, with full jitter and exponentially
# increasing max wait time
wait_fixed(30) + wait_random_exponential(multiplier=1, max=600)
)
# connection errors, other client and server failures
self.network_wait = (
# wait from 3s to ~1m
wait_random(3, 7) + wait_random_exponential(multiplier=1, max=55)
)
self.server_wait = self.network_wait
def export(self, events, start_time_ns, end_time_ns):
"""Export events to an HTTP endpoint.
:param events: The event dictionary from a `ddtrace.profiling.recorder.Recorder`.
:param start_time_ns: The start time of recording.
:param end_time_ns: The end time of recording.
"""
if not self.endpoint:
raise InvalidEndpoint("Endpoint is empty")
common_headers = {
"DD-API-KEY": self.api_key.encode(),
}
profile = super(PprofHTTPExporter, self).export(events, start_time_ns, end_time_ns)
s = six.BytesIO()
with gzip.GzipFile(fileobj=s, mode="wb") as gz:
gz.write(profile.SerializeToString())
fields = {
"runtime-id": runtime.get_runtime_id().encode("ascii"),
"recording-start": (
datetime.datetime.utcfromtimestamp(start_time_ns / 1e9).replace(microsecond=0).isoformat() + "Z"
).encode(),
"recording-end": (
datetime.datetime.utcfromtimestamp(end_time_ns / 1e9).replace(microsecond=0).isoformat() + "Z"
).encode(),
"runtime": PYTHON_IMPLEMENTATION,
"format": b"pprof",
"type": b"cpu+alloc+exceptions",
"chunk-data": s.getvalue(),
}
service_name = self.service_name or os.path.basename(profile.string_table[profile.mapping[0].filename])
content_type, body = self._encode_multipart_formdata(fields, tags=self._get_tags(service_name),)
headers = common_headers.copy()
headers["Content-Type"] = content_type
# urllib uses `POST` if `data` is supplied (Python 2 version does not handle `method` kwarg)
req = request.Request(self.endpoint, data=body, headers=headers)
retry = tenacity.Retrying(
# Retry after 1s, 2s, 4s, 8s with some randomness
wait=tenacity.wait_random_exponential(multiplier=0.5),
stop=tenacity.stop_after_delay(self.max_retry_delay),
retry=tenacity.retry_if_exception_type(
(error.HTTPError, error.URLError, http_client.HTTPException, OSError, IOError)
),
)
try:
retry(request.urlopen, req, timeout=self.timeout)
except tenacity.RetryError as e:
raise UploadFailed(e.last_attempt.exception())
class DataBaseConnection:
@retry(wait=wait_random_exponential(multiplier=1, max=60))
def connect(self) -> None:
hostname = os.getenv("DB_HOSTNAME", "db")
self.client = AsyncIOMotorClient(host=hostname)
self.db = DataBase(self.client)
print("Connected to database")
def disconnect(self) -> None:
self.client.close()
print("Disconnected from database")
def retry_upon_none_result(max_attempts, delay=0.5, max_delay=2, random=False):
if random:
wait_func = tenacity.wait_exponential(
multiplier=delay, max=max_delay)
else:
wait_func = tenacity.wait_random_exponential(
multiplier=delay, max=max_delay)
return tenacity.retry(reraise=True,
retry=tenacity.retry_if_result(lambda x: x is None),
wait=wait_func,
stop=tenacity.stop_after_attempt(max_attempts),
before=_log_before_retry, after=_log_after_retry)
def retry_upon_none_result(max_attempts, delay=0.5, max_delay=2, random=False):
if random:
wait_func = tenacity.wait_exponential(multiplier=delay, max=max_delay)
else:
wait_func = tenacity.wait_random_exponential(multiplier=delay,
max=max_delay)
return tenacity.retry(reraise=True,
retry=tenacity.retry_if_result(lambda x: x is None),
wait=wait_func,
stop=tenacity.stop_after_attempt(max_attempts),
before=_log_before_retry,
after=_log_after_retry)
def retry_429(func):
@retry(wait=wait_random_exponential(multiplier=EXPONENTIAL_BACKOFF_MULTIPLIER,
max=MAX_SECONDS_WAIT), retry=retry_if_exception_type(RateLimitException),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS), reraise=True,
before_sleep=before_sleep_on_429)
def wrapped_function(*args, **kwargs):
try:
return func(*args, **kwargs)
except HTTPError as e:
if e.response.status_code == 429:
raise RateLimitException("429 Too Many Requests")
raise e
return wrapped_function
def test_wait_random_exponential(self):
fn = tenacity.wait_random_exponential(0.5, 60.0)
for _ in six.moves.range(1000):
self._assert_inclusive_range(fn(make_retry_state(1, 0)), 0, 1.0)
self._assert_inclusive_range(fn(make_retry_state(2, 0)), 0, 2.0)
self._assert_inclusive_range(fn(make_retry_state(3, 0)), 0, 4.0)
self._assert_inclusive_range(fn(make_retry_state(4, 0)), 0, 8.0)
self._assert_inclusive_range(fn(make_retry_state(5, 0)), 0, 16.0)
self._assert_inclusive_range(fn(make_retry_state(6, 0)), 0, 32.0)
self._assert_inclusive_range(fn(make_retry_state(7, 0)), 0, 60.0)
self._assert_inclusive_range(fn(make_retry_state(8, 0)), 0, 60.0)
self._assert_inclusive_range(fn(make_retry_state(9, 0)), 0, 60.0)
fn = tenacity.wait_random_exponential(10, 5)
for _ in six.moves.range(1000):
self._assert_inclusive_range(fn(make_retry_state(1, 0)), 0.00,
5.00)
# Default arguments exist
fn = tenacity.wait_random_exponential()
fn(0, 0)
def wait(self, timeout=60, wait_exp_multiplier=0.05, wait_exp_max=1.0):
"""
Wait until task is completed. Expires after 'timeout' seconds.
"""
try:
retryer = Retrying(wait=wait_random_exponential(multiplier=wait_exp_multiplier, max=wait_exp_max),
stop=stop_after_delay(timeout),
retry=retry_if_result(is_pending_status),
before=before_log(logger, logging.DEBUG),
after=after_log(logger, logging.DEBUG))
retryer(self._refresh_status)
except RetryError:
raise TaskTimeout(self.data())
if is_error_status(self['status']):
raise TaskError(self.data())
return self
