def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.client = AivenClient(
base_url=os.environ.get("AIVEN_WEB_URL", "https://api.aiven.io"))
self.client.set_auth_token(os.environ["AIVEN_AUTH_TOKEN"])
self.project = os.environ.get("AIVEN_TEST_PROJECT", "test")
self.service_specifications = self._discover_required_services()
class AivenExampleTest(unittest.TestCase):
required_services = tuple()
services = defaultdict(dict)
_project_ca = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.client = AivenClient(
base_url=os.environ.get("AIVEN_WEB_URL", "https://api.aiven.io"))
self.client.set_auth_token(os.environ["AIVEN_AUTH_TOKEN"])
self.project = os.environ.get("AIVEN_TEST_PROJECT", "test")
def execute(self, command, **kwargs):
return subprocess.run(shlex.split(command),
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
universal_newlines=True,
**kwargs)
@property
def project_ca(self):
if self._project_ca is None:
self._project_ca = self.client.get_project_ca(
self.project)['certificate']
return self._project_ca
def main():
logger = logging.getLogger(os.path.basename(__file__))
# Setup Aiven SDK
logger.info("Setting up Aiven SDK")
client = AivenClient("https://api.aiven.io")
client.set_auth_token(os.environ["AIVEN_TOKEN"])
# Lookup the target service
logger.info("Looking up the target Aiven Kafka Service")
service = client.get_service(project=os.environ["AIVEN_PROJECT"],
service=os.environ["AIVEN_SERVICE"])
if not service:
raise SystemExit("Failed to look up the target service")
# Store credentials on disk. This is using the main access certificates (avnadmin).
logger.info("Storing Aiven service access credentials")
with open("client.crt", "w") as fh:
fh.write(service["connection_info"]["kafka_access_cert"])
with open("client.key", "w") as fh:
fh.write(service["connection_info"]["kafka_access_key"])
# Project CA certificate
logger.info("Fetching project CA certificate")
result = client.get_project_ca(project=os.environ["AIVEN_PROJECT"])
with open("ca.crt", "w") as fh:
fh.write(result["certificate"])
# write out properties for rdkafka benchmark tool
logger.info("Create properties file for rdkafka_performance")
with open("producer.properties", "w") as fh:
fh.write(
rdkafka_props_template.format(
aiven_service_uri=service["service_uri"]))
# Start rdkafka_performance tool
logger.info("Start load generation, break with CTRL-C")
subprocess.run([
"/root/librdkafka/examples/rdkafka_performance",
"-X",
"file=producer.properties",
"-P",
"-s",
"512",
"-t",
os.environ["AIVEN_TOPIC"],
])
def main():
logger = logging.getLogger(os.path.basename(__file__))
# Setup Aiven SDK
logger.info("Setting up Aiven SDK")
client = AivenClient("https://api.aiven.io")
client.set_auth_token(os.environ["AIVEN_TOKEN"])
# Lookup the target service
logger.info("Looking up the target Aiven Kafka Service")
service = client.get_service(project=os.environ["AIVEN_PROJECT"],
service=os.environ["AIVEN_SERVICE"])
if not service:
raise SystemExit("Failed to look up the target service")
# Store credentials on disk. This is using the main access certificates (avnadmin).
logger.info("Storing Aiven service access credentials")
with open("client.crt", "w") as fh:
fh.write(service["connection_info"]["kafka_access_cert"])
with open("client.key", "w") as fh:
fh.write(service["connection_info"]["kafka_access_key"])
# Project CA certificate
logger.info("Fetching project CA certificate")
result = client.get_project_ca(project=os.environ["AIVEN_PROJECT"])
with open("ca.crt", "w") as fh:
fh.write(result["certificate"])
# Initialize Kafka client
kafka_client = KafkaConsumer(
bootstrap_servers=service["service_uri"],
security_protocol="SSL",
ssl_cafile="ca.crt",
ssl_certfile="client.crt",
ssl_keyfile="client.key",
)
partitions = kafka_client.partitions_for_topic(os.environ["AIVEN_TOPIC"])
tps = [
TopicPartition(os.environ["AIVEN_TOPIC"], partition)
for partition in partitions
]
last_timestamp = time.monotonic()
last_offsets = {}
logger.info("Start result collection loop, break with CTRL-C")
readings = []
while True:
delta = 0
result = kafka_client.end_offsets(tps)
timenow = time.monotonic()
for tp, offset in result.items():
if tp in last_offsets:
delta += offset - last_offsets[tp]
last_offsets[tp] = offset
messages_per_second = int(delta / (timenow - last_timestamp))
readings.append(messages_per_second)
readings = readings[-30:]
logger.info("%d messages/s, 30 sample average %d messages/s",
messages_per_second,
sum(readings) / len(readings))
last_timestamp = timenow
time.sleep(2)
class ServiceManager:
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.client = AivenClient(
base_url=os.environ.get("AIVEN_WEB_URL", "https://api.aiven.io"))
self.client.set_auth_token(os.environ["AIVEN_AUTH_TOKEN"])
self.project = os.environ.get("AIVEN_TEST_PROJECT", "test")
self.service_specifications = self._discover_required_services()
def spin_up_services(self):
start = time.monotonic()
with ThreadPool(processes=len(self.service_specifications)) as pool:
services = pool.map(self._create_service,
self.service_specifications)
# If we created new services, sleep for a minute to allow them to settle.
# Attempting to use them immediately after they transition to running
# results in many transient errors caused by nodes not quite ready, DNS not
# propagated, etc.
new_services = time.monotonic() - start > 60
if new_services:
time.sleep(60)
for service in services:
service_name, service_type = service['service_name'], service[
'service_type']
AivenExampleTest.services[service_type][service_name] = service
def tear_down_services(self):
for service in self.service_specifications:
self.client.delete_service(self.project, service.name)
def _discover_required_services(self):
subclasses = self._gather_subclasses(AivenExampleTest)
service_specifications = set()
for subclass in subclasses:
service_specifications.update(subclass.required_services)
return service_specifications
def _gather_subclasses(self, cls):
all_subclasses = []
for subclass in cls.__subclasses__():
all_subclasses.append(subclass)
all_subclasses.extend(self._gather_subclasses(subclass))
return all_subclasses
def _create_service(self, service_spec, timeout=600):
start = time.monotonic()
while time.monotonic() - start < timeout:
service = self._get_service(self.project, service_spec)
if service:
return service
time.sleep(5)
raise TimeoutError(
"Creating service timed out after {} seconds".format(timeout))
def _get_service(self, project, service_spec):
try:
service = self.client.get_service(project=project,
service=service_spec.name)
if service['state'] == 'RUNNING':
return service
except Error as e:
if e.status != 404:
raise
self.client.create_service(self.project,
service_spec.name,
service_spec.type,
'default',
service_spec.plan,
user_config=None,
cloud='google-europe-west1')
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.client = AivenClient(
base_url=os.environ.get("AIVEN_WEB_URL", "https://api.aiven.io"))
self.client.set_auth_token(os.environ["AIVEN_AUTH_TOKEN"])
self.project = os.environ.get("AIVEN_TEST_PROJECT", "test")