def perform_pull(worker_state: WorkerState, sfm_queue: Queue,
subscriber_client: SubscriberClient, subscription_path: str):
pull_request = PullRequest()
pull_request.max_messages = PROCESSING_WORKER_PULL_REQUEST_MAX_MESSAGES
pull_request.subscription = subscription_path
response: PullResponse = subscriber_client.pull(pull_request)
for received_message in response.received_messages:
# print(f"Received: {received_message.message.data}.")
message_job = _process_message(sfm_queue, received_message)
if not message_job or message_job.bytes_size > REQUEST_BODY_MAX_SIZE - 2:
worker_state.ack_ids.append(received_message.ack_id)
continue
if worker_state.should_flush(message_job):
perform_flush(worker_state, sfm_queue, subscriber_client,
subscription_path)
worker_state.add_job(message_job, received_message.ack_id)
# check if should flush because of time
if worker_state.should_flush():
perform_flush(worker_state, sfm_queue, subscriber_client,
subscription_path)
def test_flush_manager(options: Dict[str, Any], emulator: str, web: str):
print("starting test")
api = CoreV1Api()
# max number of loops to run when waiting for kube actions to complete
max_wait_loops = 20 if options["cluster"] is None else 60
# server has invalid PUBSUB_EMULATOR, so that only flush can deliver messages
static_pvs = options["cluster"] is None
if static_pvs:
create_static_pvs(api)
print("waiting for pods to be healthy")
for _ in range(max_wait_loops):
if all(pod.status.phase == "Running"
for pod in api.list_namespaced_pod("default").items):
break
time.sleep(1)
else:
assert False, "pods did not become healthy"
# create a subscription to the defined topic
print("creating pubsub subscription")
os.environ["PUBSUB_EMULATOR_HOST"] = emulator
sub_client = SubscriberClient()
topic_path = "projects/{project}/topics/{topic}".format(**options)
subscription_path = "projects/{project}/subscriptions/{topic}".format(
**options)
try:
sub_client.create_subscription(subscription_path,
topic_path,
retry=None)
except AlreadyExists:
pass
print("posting message 0")
requests.post(web, headers={
"host": "web"
}, json={
"id": 0
}).raise_for_status()
print("setting up race condition: attached pvc is also deleted")
delete_pvcs(api)
print("setting up race condition: pod unschedulable due to missing pvc")
with pytest.raises(ApiException) as excinfo:
restart_web_pods(api)
assert excinfo.value.reason == "Conflict"
print("posting message 1")
with pytest.raises(requests.exceptions.ConnectionError):
requests.post(web, headers={
"host": "web"
}, json={
"id": 1
}).raise_for_status()
print("starting flush-manager")
# TODO optionally run flush-manager via subprocess.Popen, to ensure testing
# current code and enable code coverage
_sa = kube_resource("kube/flush-manager.sa.yml", **options)
_cluster_role = kube_resource("kube/flush-manager.clusterrole.yml",
**options)
_cluster_role_binding = kube_resource(
"kube/flush-manager.clusterrolebinding.yml", **options)
_role = kube_resource("kube/flush-manager.role.yml", **options)
_role_binding = kube_resource("kube/flush-manager.rolebinding.yml",
**options)
_deploy = kube_resource("kube/flush-manager.deploy.yml", **options)
with _sa, _cluster_role, _cluster_role_binding, _role, _role_binding, _deploy:
print("posting message 2 until successful")
for i in range(max_wait_loops):
try:
requests.post(web, headers={
"host": "web"
}, json={
"id": 2
}).raise_for_status()
except requests.exceptions.ConnectionError:
if i > 0 and static_pvs:
create_static_pvs(api)
else:
break
time.sleep(1)
else:
assert False, "pod did not recover"
# scale to 0 pods
print("scaling web to 0 pods")
AppsV1Api().patch_namespaced_stateful_set_scale(
name="web",
namespace="default",
body=V1StatefulSet(api_version="apps/v1",
kind="StatefulSet",
spec=dict(replicas=0)),
)
# wait for no pvcs
print("waiting for cleanup to complete")
for _ in range(max_wait_loops):
if not api.list_persistent_volume().items:
break
time.sleep(1)
else:
print("pvs were not cleaned up")
assert [] == api.list_persistent_volume().items
# assert jobs and pvcs also deleted
assert [] == list_pvcs(api)
assert [] == BatchV1Api().list_namespaced_job("default").items
# assert received message id 0 and 2
assert [b'{"id": 0}', b'{"id": 2}'] == [
element.message.data
for element in sub_client.pull(subscription_path, 2).received_messages
]
