def test_wait_combine(self):
r = Retrying(wait=tenacity.wait_combine(tenacity.wait_random(0, 3),
tenacity.wait_fixed(5)))
# Test it a few time since it's random
for i in six.moves.range(1000):
w = r.wait(1, 5)
self.assertLess(w, 8)
self.assertGreaterEqual(w, 5)
def test_exponential(self):
r = Retrying(wait=tenacity.wait_exponential(max=100))
self.assertEqual(r.wait(1, 0), 2)
self.assertEqual(r.wait(2, 0), 4)
self.assertEqual(r.wait(3, 0), 8)
self.assertEqual(r.wait(4, 0), 16)
self.assertEqual(r.wait(5, 0), 32)
self.assertEqual(r.wait(6, 0), 64)
def test_wait_triple_sum(self):
r = Retrying(wait=tenacity.wait_fixed(1) + tenacity.wait_random(0, 3) +
tenacity.wait_fixed(5))
# Test it a few time since it's random
for i in six.moves.range(1000):
w = r.wait(1, 5)
self.assertLess(w, 9)
self.assertGreaterEqual(w, 6)
def test_stop_backward_compat(self):
r = Retrying(stop=lambda attempt, delay: attempt == delay)
with reports_deprecation_warning():
self.assertFalse(r.stop(make_retry_state(1, 3)))
with reports_deprecation_warning():
self.assertFalse(r.stop(make_retry_state(100, 99)))
with reports_deprecation_warning():
self.assertTrue(r.stop(make_retry_state(101, 101)))
def test_stop_func_with_retry_state(self):
def stop_func(retry_state):
rs = retry_state
return rs.attempt_number == rs.seconds_since_start
r = Retrying(stop=stop_func)
self.assertFalse(r.stop(make_retry_state(1, 3)))
self.assertFalse(r.stop(make_retry_state(100, 99)))
self.assertTrue(r.stop(make_retry_state(101, 101)))
async def test_upgrading_rules_provider_also_updates_rule_files(
ops_test, prometheus_tester_charm):
"""Ensure scrape alert rules can be updated.
This test upgrades the metrics provider charm and checks that
updates to alert rules are propagated correctly.
"""
await ops_test.model.deploy(prometheus_tester_charm,
resources=tester_resources,
application_name=TESTER_APP_NAME)
await ops_test.model.wait_for_idle(apps=[TESTER_APP_NAME], status="active")
await ops_test.model.block_until(
lambda: len(ops_test.model.applications[TESTER_APP_NAME].units) > 0)
assert ops_test.model.applications[TESTER_APP_NAME].units[
0].workload_status == "active"
await ops_test.model.add_relation(PROMETHEUS_APP_NAME, TESTER_APP_NAME)
await ops_test.model.wait_for_idle(
apps=[PROMETHEUS_APP_NAME, TESTER_APP_NAME], status="active")
# Check only one alert rule exists
tester_rules = []
for attempt in Retrying(retry=retry_if_exception_type(AssertionError),
stop=stop_after_attempt(3)):
try:
with attempt:
rules_with_relation = await get_prometheus_rules(
ops_test, PROMETHEUS_APP_NAME, 0)
tester_rules = get_rules_for(TESTER_APP_NAME,
rules_with_relation)
assert len(tester_rules) == 1
except RetryError:
pass
# Add new alert rule, rebuild and refresh prometheus tester charm
write_tester_alert_rule_file(MISSING_TARGET_RULE, "target_missing.rule")
tester_charm = await rebuild_prometheus_tester(ops_test)
await ops_test.model.applications[TESTER_APP_NAME].refresh(
path=tester_charm, resources=tester_resources)
remove_tester_alert_rule_file("target_missing.rule")
await ops_test.model.wait_for_idle(
apps=[PROMETHEUS_APP_NAME, TESTER_APP_NAME], status="active")
# Check there are now two alert rules
tester_rules = []
for attempt in Retrying(retry=retry_if_exception_type(AssertionError),
stop=stop_after_attempt(3)):
try:
with attempt:
rules_with_relation = await get_prometheus_rules(
ops_test, PROMETHEUS_APP_NAME, 0)
tester_rules = get_rules_for(TESTER_APP_NAME,
rules_with_relation)
assert len(tester_rules) == 2
except RetryError:
pass
def test_wait_arbitrary_sum(self):
r = Retrying(wait=sum([tenacity.wait_fixed(1),
tenacity.wait_random(0, 3),
tenacity.wait_fixed(5),
tenacity.wait_none()]))
# Test it a few time since it's random
for i in six.moves.range(1000):
w = r.wait(1, 5)
self.assertLess(w, 9)
self.assertGreaterEqual(w, 6)
def __init__(self, strand_api_client):
self.strand_api_client = strand_api_client
self.logger = get_logger('StrandApiClientWrapper')
self.standard_retrier = Retrying(
reraise=True,
wait=wait_fixed(2),
stop=stop_after_attempt(5),
after=after_log(logger=self.logger,
log_level=self.logger.getEffectiveLevel()),
retry=retry_if_exception_type(StrandApiClientException))
def __init__(self, SlackClientClass):
self.SlackClientClass = SlackClientClass
self.logger = get_logger('SlackClientWrapper')
self.standard_retrier = Retrying(
reraise=True,
wait=wait_fixed(2),
stop=stop_after_attempt(5),
after=after_log(logger=self.logger,
log_level=self.logger.getEffectiveLevel()),
retry=(retry_if_exception_type(ConnectionError)))
def test_random_sleep(self):
r = Retrying(wait=tenacity.wait_random(min=1, max=20))
times = set()
for x in six.moves.range(1000):
times.add(r.wait(1, 6546))
# this is kind of non-deterministic...
self.assertTrue(len(times) > 1)
for t in times:
self.assertTrue(t >= 1)
self.assertTrue(t < 20)
def test_alibi_detect_cifar10_rclone(self, namespace):
spec = "../resources/adserver-cifar10-od-rclone.yaml"
name = "cifar10-od-server-rclone"
vs_prefix = name
retry_run(f"kubectl apply -f {spec} -n {namespace}")
wait_for_deployment(name, namespace)
time.sleep(AFTER_WAIT_SLEEP)
with open(self.truck_json) as f:
data = json.load(f)
for attempt in Retrying(
wait=wait_fixed(TENACITY_WAIT),
stop=stop_after_attempt(TENACITY_STOP_AFTER_ATTEMPT),
):
with attempt:
r = requests.post(
f"http://localhost:8004/{vs_prefix}/",
json=data,
headers=self.HEADERS,
)
j = r.json()
assert j["data"]["is_outlier"][0] == 0
assert j["meta"]["name"] == "OutlierVAE"
assert j["meta"]["detector_type"] == "offline"
assert j["meta"]["data_type"] == "image"
with open(self.truck_json_outlier) as f:
data = json.load(f)
for attempt in Retrying(
wait=wait_fixed(TENACITY_WAIT),
stop=stop_after_attempt(TENACITY_STOP_AFTER_ATTEMPT),
):
with attempt:
r = requests.post(
f"http://localhost:8004/{vs_prefix}/",
json=data,
headers=self.HEADERS,
)
j = r.json()
assert j["data"]["is_outlier"][0] == 1
assert j["meta"]["name"] == "OutlierVAE"
assert j["meta"]["detector_type"] == "offline"
assert j["meta"]["data_type"] == "image"
run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
def test_retry_function_object(self):
"""Test that six.wraps doesn't cause problems with callable objects.
It raises an error upon trying to wrap it in Py2, because __name__
attribute is missing. It's fixed in Py3 but was never backported.
"""
class Hello(object):
def __call__(self):
return "Hello"
retrying = Retrying(wait=tenacity.wait_fixed(0.01),
stop=tenacity.stop_after_attempt(3))
h = retrying.wraps(Hello())
self.assertEqual(h(), "Hello")
def get_engagement(self, url):
r = Retrying(
retry=retry_if_exception_type((
RateLimitError,
OSError,
requests.exceptions.ConnectionError,
urllib3.exceptions.MaxRetryError,
urllib3.exceptions.NewConnectionError,
)),
before_sleep=self._switch_token,
wait=wait_fixed(5),
)
return r.call(self.api_call, url)
def test_random_sleep_without_min(self):
r = Retrying(wait=tenacity.wait_random(max=2))
times = set()
times.add(r.wait(1, 6546))
times.add(r.wait(1, 6546))
times.add(r.wait(1, 6546))
times.add(r.wait(1, 6546))
# this is kind of non-deterministic...
self.assertTrue(len(times) > 1)
for t in times:
self.assertTrue(t >= 0)
self.assertTrue(t <= 2)
def test_wait_chain(self):
r = Retrying(wait=tenacity.wait_chain(
*[tenacity.wait_fixed(1) for i in six.moves.range(2)] +
[tenacity.wait_fixed(4) for i in six.moves.range(2)] +
[tenacity.wait_fixed(8) for i in six.moves.range(1)]))
for i in six.moves.range(10):
w = r.wait(i + 1, 1)
if i < 2:
self._assert_range(w, 1, 2)
elif i < 4:
self._assert_range(w, 4, 5)
else:
self._assert_range(w, 8, 9)
def _wait_for_ready_cni(self, timeout=900):
self.logging.info(
"Waiting up to %.2f minutes for ready CNI on the Windows agents",
timeout / 60.0)
win_node_addresses = self.deployer.windows_private_addresses
local_script_path = os.path.join(self.e2e_runner_dir,
"scripts/confirm-ready-cni.ps1")
remote_script_path = "/tmp/confirm-ready-cni.ps1"
self._upload_to(local_script_path, remote_script_path,
win_node_addresses)
for attempt in Retrying(stop=stop_after_delay(timeout),
wait=wait_exponential(max=30),
retry=retry_if_exception_type(AssertionError),
reraise=True):
with attempt:
cni_ready = True
for node_address in win_node_addresses:
try:
stdout = subprocess.check_output(
["ssh", node_address, remote_script_path],
timeout=30)
except Exception:
cni_ready = False
break
cni_ready = strtobool(stdout.decode().strip())
if not cni_ready:
break
assert cni_ready
self.logging.info("The CNI is ready on all the Windows agents")
def test_alibi_explain_anchor_image_tensorflow_protocol(self, namespace):
spec = "../resources/tf_cifar_anchor_image_explainer.yaml"
name = "cifar10-default-explainer"
vs_prefix = (f"seldon/{namespace}/cifar10-explainer/default/v1/models/"
f"cifar10-classifier:explain")
retry_run(f"kubectl apply -f {spec} -n {namespace}")
wait_for_deployment(name, namespace)
# note: we add a batch dimension but it should be really one image
test_data = np.random.randn(1, 32, 32, 3)
inference_request = {"instances": test_data.tolist()}
for attempt in Retrying(
wait=wait_fixed(TENACITY_WAIT),
stop=stop_after_attempt(TENACITY_STOP_AFTER_ATTEMPT),
):
with attempt:
r = requests.post(
f"http://localhost:8004/{vs_prefix}",
json=inference_request,
)
explanation = r.json()
assert explanation["meta"]["name"] == "AnchorImage"
assert "anchor" in explanation["data"]
assert "precision" in explanation["data"]
assert "coverage" in explanation["data"]
run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
def test_change_batch_qty_leading_to_reallocation():
'''
测试改变批次数量 导致重新分配
'''
orderid, sku = random_orderid(), random_sku()
earlier_batch, later_batch = random_batchref('old'), random_batchref('new')
api_client.post_to_add_batch(earlier_batch, sku, 10, str(today))
api_client.post_to_add_batch(later_batch, sku, 10, str(tomorrow))
api_client.post_to_allocte(orderid, sku, 10)
r = api_client.get_allocation(orderid)
assert r.json()[0]['batchref'] == earlier_batch
subscription = redis_client.subscribe_to('line_allocated')
# 发布一个命令 终端的事件消费者接收到 调用hander 这个逻辑里 成功完成分配订单 发布了一个line_allocated
redis_client.publish_message('change_batch_quantity', {
'batchref': earlier_batch,
'qty': 5
})
messages = []
for attempt in Retrying(stop=stop_after_delay(3), reraise=True):
with attempt:
message = subscription.get_message(timeout=1)
if message:
messages.append(message)
print(message)
data = json.loads(messages[-1]['data'])
assert data['orderid'] == orderid
assert data['batchref'] == later_batch
def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult:
stream_descriptor = BaseIngestClient._prepare_stream(stream_descriptor, ingestion_properties)
stream = stream_descriptor.stream
buffered_stream = read_until_size_or_end(stream, self.MAX_STREAMING_SIZE_IN_BYTES + 1)
if len(buffered_stream.getbuffer()) > self.MAX_STREAMING_SIZE_IN_BYTES:
stream_descriptor.stream = chain_streams([buffered_stream, stream])
return self.queued_client.ingest_from_stream(stream_descriptor, ingestion_properties)
stream_descriptor.stream = buffered_stream
try:
for attempt in Retrying(
stop=stop_after_attempt(self._num_of_attempts), wait=wait_random_exponential(max=self._max_seconds_per_retry), reraise=True
):
with attempt:
stream.seek(0, SEEK_SET)
client_request_id = ManagedStreamingIngestClient._get_request_id(stream_descriptor.source_id, attempt.retry_state.attempt_number - 1)
return self.streaming_client._ingest_from_stream_with_client_request_id(stream_descriptor, ingestion_properties, client_request_id)
except KustoApiError as ex:
error = ex.get_api_error()
if error.permanent:
raise
return self.queued_client.ingest_from_stream(stream_descriptor, ingestion_properties)
def create_flavors(nova_client=None):
"""Create basic flavors.
:param nova_client: Authenticated nova client
:type nova_client: novaclient.v2.client.Client
"""
if not nova_client:
keystone_session = openstack_utils.get_overcloud_keystone_session()
nova_client = openstack_utils.get_nova_session_client(keystone_session)
cli_utils.setup_logging()
for attempt in Retrying(stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=1, min=2,
max=10)):
with attempt:
existing_flavors = nova_client.flavors.list()
names = [flavor.name for flavor in existing_flavors]
for flavor in nova_utils.FLAVORS.keys():
if flavor not in names:
nova_flavor = nova_client.flavors.create(
name=flavor,
ram=nova_utils.FLAVORS[flavor]['ram'],
vcpus=nova_utils.FLAVORS[flavor]['vcpus'],
disk=nova_utils.FLAVORS[flavor]['disk'],
flavorid=nova_utils.FLAVORS[flavor]['flavorid'])
if 'extra-specs' in nova_utils.FLAVORS[flavor]:
nova_flavor.set_keys(nova_utils.FLAVORS[flavor]['extra-specs'])
def test_retry_child_class_with_override_backward_compat(self):
class MyStop(tenacity.stop_after_attempt):
def __init__(self):
super(MyStop, self).__init__(1)
def __call__(self, attempt_number, seconds_since_start):
return super(MyStop, self).__call__(attempt_number,
seconds_since_start)
retrying = Retrying(wait=tenacity.wait_fixed(0.01), stop=MyStop())
def failing():
raise NotImplementedError()
with pytest.raises(RetryError):
retrying.call(failing)
def test_change_batch_quantity_leading_to_reallocation():
# start with two batches and an order allocated to one of them
orderid, sku = random_orderid(), random_sku()
earlier_batch, later_batch = random_batchref("old"), random_batchref(
"newer")
api_client.post_to_add_batch(earlier_batch, sku, qty=10, eta="2011-01-02")
api_client.post_to_add_batch(later_batch, sku, qty=10, eta="2011-01-03")
r = api_client.post_to_allocate(orderid, sku, 10)
assert r.ok
response = api_client.get_allocation(orderid)
assert response.json()[0]["batchref"] == earlier_batch
subscription = redis_client.subscribe_to("line_allocated")
# change quantity on allocated batch so it's less than our order
redis_client.publish_message("change_batch_quantity", {
"batchref": earlier_batch,
"qty": 5
})
# wait until we see a message saying the order has been reallocated
messages = []
for attempt in Retrying(stop=stop_after_delay(3), reraise=True):
with attempt:
message = subscription.get_message(timeout=1)
if message:
messages.append(message)
print(messages)
data = json.loads(messages[-1]["data"])
assert data["orderid"] == orderid
assert data["batchref"] == later_batch
def test_alibi_explain_anchor_tabular(self, namespace):
spec = "../resources/iris_anchor_tabular_explainer.yaml"
name = "iris-default-explainer"
vs_prefix = f"seldon/{namespace}/iris-explainer/default/api/v1.0/explain"
inference_request = {
"data": {
"names": ["text"],
"ndarray": [[5.964, 4.006, 2.081, 1.031]],
}
}
retry_run(f"kubectl apply -f {spec} -n {namespace}")
wait_for_deployment(name, namespace)
time.sleep(AFTER_WAIT_SLEEP)
for attempt in Retrying(
wait=wait_fixed(TENACITY_WAIT),
stop=stop_after_attempt(TENACITY_STOP_AFTER_ATTEMPT),
):
with attempt:
r = requests.post(
f"http://localhost:8004/{vs_prefix}",
json=inference_request,
)
explanation = r.json()
assert explanation["meta"]["name"] == "AnchorTabular"
assert "anchor" in explanation["data"]
assert "precision" in explanation["data"]
assert "coverage" in explanation["data"]
run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
def test_change_batch_quantity_leading_to_reallocation():
# start with two batches and an order allocated to one of them
orderid, sku = random_orderid(), random_sku()
earlier_batch, later_batch = random_batchref('old'), random_batchref(
'newer')
api_client.post_to_add_batch(earlier_batch, sku, qty=10, eta='2011-01-01')
api_client.post_to_add_batch(later_batch, sku, qty=10, eta='2011-01-02')
response = api_client.post_to_allocate(orderid, sku, 10)
assert response.ok
response = api_client.get_allocation(orderid)
assert response.json()[0]['batchref'] == earlier_batch
subscription = redis_client.subscribe_to('line_allocated')
# change quantity on allocated batch so it's less than our order
redis_client.publish_message('change_batch_quantity', {
'batchref': earlier_batch,
'qty': 5
})
# wait until we see a message saying the order has been reallocated
messages = []
for attempt in Retrying(stop=stop_after_delay(3), reraise=True):
with attempt:
message = subscription.get_message(timeout=1)
if message:
messages.append(message)
logger.info("%s", messages)
data = json.loads(messages[-1]['data'])
assert data['orderid'] == orderid
assert data['batchref'] == later_batch
def tearDown(cls):
"""Remove test resources."""
logging.info('Running teardown')
for attempt in Retrying(
stop=stop_after_attempt(8),
wait=wait_exponential(multiplier=1, min=2, max=60)):
with attempt:
volumes = list(cls.cinder_client.volumes.list())
snapped_volumes = [v for v in volumes
if v.name.endswith("-from-snap")]
if snapped_volumes:
logging.info("Removing volumes from snapshot")
cls._remove_volumes(snapped_volumes)
volumes = list(cls.cinder_client.volumes.list())
snapshots = list(cls.cinder_client.volume_snapshots.list())
if snapshots:
logging.info("tearDown - snapshots: {}".format(
", ".join(s.name for s in snapshots)))
cls._remove_snapshots(snapshots)
if volumes:
logging.info("tearDown - volumes: {}".format(
", ".join(v.name for v in volumes)))
cls._remove_volumes(volumes)
def background_event_listening(pubsub, event_name) -> None:
for attempt in Retrying(stop=stop_after_delay(5), reraise=True):
with attempt:
msg = pubsub.get_message()
if msg:
data = parse_encoded_message(msg)["data"]
assert data["name"] == event_name
def minio_service(minio_config: Dict[str, str]) -> Iterator[Minio]:
client = Minio(**minio_config["client"])
for attempt in Retrying(
wait=tenacity.wait_fixed(5),
stop=tenacity.stop_after_attempt(60),
before_sleep=tenacity.before_sleep_log(log, logging.WARNING),
reraise=True,
):
with attempt:
# TODO: improve as https://docs.min.io/docs/minio-monitoring-guide.html
if not client.bucket_exists("pytest"):
client.make_bucket("pytest")
client.remove_bucket("pytest")
bucket_name = minio_config["bucket_name"]
# cleans up in case a failing tests left this bucket
_ensure_remove_bucket(client, bucket_name)
client.make_bucket(bucket_name)
assert client.bucket_exists(bucket_name)
yield client
# cleanup upon tear-down
_ensure_remove_bucket(client, bucket_name)
def driver(request, driver_class, driver_kwargs):
"""Returns a WebDriver instance based on options and capabilities"""
retries = int(request.config.getini("max_driver_init_attempts"))
for retry in Retrying(stop=stop_after_attempt(retries),
wait=wait_exponential(),
reraise=True):
with retry:
LOGGER.info(
f"Driver init, attempt {retry.retry_state.attempt_number}/{retries}"
)
driver = driver_class(**driver_kwargs)
event_listener = request.config.getoption("event_listener")
if event_listener is not None:
# Import the specified event listener and wrap the driver instance
mod_name, class_name = event_listener.rsplit(".", 1)
mod = __import__(mod_name, fromlist=[class_name])
event_listener = getattr(mod, class_name)
if not isinstance(driver, EventFiringWebDriver):
driver = EventFiringWebDriver(driver, event_listener())
request.node._driver = driver
yield driver
driver.quit()
def sign_up(
self,
email: str,
password: str,
phone_number: str = None,
name: str = None,
photo_url: str = None,
user_id: str = None,
project_id: str = None,
):
if phone_number and not phone_number.startswith("+"):
phone_number = f"+{phone_number}"
data = {
"email": email,
"password": password,
"displayName": name,
"photoUrl": photo_url,
"disabled": False,
"localId": user_id,
"phoneNumber": phone_number,
"targetProjectId": project_id or self.project_id,
}
self._execute(method=self.client.accounts().signUp, body=data)
for attempt in Retrying(stop=stop_after_attempt(3), wait=wait_fixed(1)):
with attempt:
return self.find(email=email, project_id=project_id)
def exists():
count = max_attempts
if not is_retry:
count = 1
def air_retry_exists():
return loop_find(self.airtest_ui,
timeout=UI_TIME_OUT,
**kwargs)
def poco_retry_exists():
if self.poco_ui is None:
self.get_ui()
return False
return self.poco_ui.attr('visible')
r = Retrying(retry=retry_if_exception_type(TargetNotFoundError),
wait=wait_fixed(sleeps),
stop=stop_after_attempt(count),
reraise=True)
try:
res = r(
r,
air_retry_exists if self.is_airtest else poco_retry_exists)
except (PocoTargetRemovedException, PocoNoSuchNodeException,
TargetNotFoundError):
res = False
return res
