def wait_cloud_init_completes(remoter: RemoteCmdRunnerBase, instance: VmInstance):
"""Connects to VM with SSH and waits for cloud-init to complete. Verify if everything went ok.
"""
LOGGER.info("Waiting for cloud-init to complete on node %s...", instance.name)
errors_found = False
remoter.is_up(60 * 5)
result = remoter.sudo("cloud-init status --wait")
status = result.stdout
LOGGER.debug("cloud-init status: %s", status)
if "done" not in status:
LOGGER.error("Some error during cloud-init %s", status)
errors_found = True
scripts_errors_found = log_user_data_scripts_errors(remoter=remoter)
if errors_found or scripts_errors_found:
raise CloudInitError("Errors during cloud-init provisioning phase. See logs for errors.")
def _journal_thread(self):
self._remoter = RemoteCmdRunnerBase.create_remoter(**self._remoter_params)
read_from_timestamp = None
while not self._termination_event.is_set():
self._wait_ssh_up(verbose=False)
self._retrieve_journal(since=read_from_timestamp)
read_from_timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
def provision_instances_with_fallback(
provisioner: Provisioner, definitions: List[InstanceDefinition],
pricing_model: PricingModel,
fallback_on_demand: bool) -> List[VmInstance]:
try:
provision_with_retry(provisioner,
definitions=definitions,
pricing_model=pricing_model)
except ProvisionError:
if pricing_model.is_spot() and fallback_on_demand:
provision_with_retry(provisioner,
definitions=definitions,
pricing_model=PricingModel.ON_DEMAND)
else:
raise
provisioned_instances = provisioner.get_or_create_instances(
definitions=definitions)
for v_m in provisioned_instances:
ssh_login_info = {
'hostname': v_m.public_ip_address,
'user': v_m.user_name,
'key_file': f"~/.ssh/{v_m.ssh_key_name}"
}
remoter = RemoteCmdRunnerBase.create_remoter(**ssh_login_info)
wait_cloud_init_completes(remoter=remoter, instance=v_m)
# todo: wait for scylla-machine-image service to complete if instance is scylla-db?
# todo: download cloud-init logs
return provisioned_instances
def get_remoter(self, host):
self._ssh_pkey_file = tempfile.NamedTemporaryFile(mode="w",
delete=False)
self._ssh_pkey_file.write(self.key_pair().private_key.decode())
self._ssh_pkey_file.flush()
return RemoteCmdRunnerBase.create_remoter(
hostname=host,
user=self.LOGIN_USER,
key_file=self._ssh_pkey_file.name)
def __init__(self):
self.ssh_login_info = {
'hostname': '34.253.205.91',
'user': '******',
'key_file': '~/.ssh/scylla-qa-ec2'
}
self.remoter = RemoteCmdRunnerBase.create_remoter(
**self.ssh_login_info)
self.ip_address = '34.253.205.91'
self.cassandra_stress_version = '3.11'
def get_remoter(
self,
host,
connect_timeout: Optional[float] = None) -> RemoteCmdRunnerBase:
self._ssh_pkey_file = tempfile.NamedTemporaryFile(mode="w",
delete=False) # pylint: disable=consider-using-with
self._ssh_pkey_file.write(self.key_pair.private_key.decode())
self._ssh_pkey_file.flush()
return RemoteCmdRunnerBase.create_remoter(
hostname=host,
user=self.LOGIN_USER,
key_file=self._ssh_pkey_file.name,
connect_timeout=connect_timeout)
def log_user_data_scripts_errors(remoter: RemoteCmdRunnerBase) -> bool:
errors_found = False
result = remoter.run(f"ls {CLOUD_INIT_SCRIPTS_PATH}")
ls_errors = result.stderr
if ls_errors:
LOGGER.error("Error listing generated scripts: %s", ls_errors)
errors_found = True
files_list = result.stdout
if not files_list:
LOGGER.error("No user data scripts were generated.")
errors_found = True
elif ".failed" in files_list:
LOGGER.error("Some user data scripts have failed: %s", files_list)
errors_found = True
elif "done" not in files_list:
LOGGER.error("User data scripts were not executed at all.")
errors_found = True
return errors_found
def fake_remoter():
RemoteCmdRunnerBase.set_default_remoter_class(FakeRemoter)
return FakeRemoter