def builder_start_cmd(_):
instance = BuilderInstance.instance()
if instance.status() == 'stopped':
print("Starting builder instance...")
instance.start()
for _ in range(60):
if instance.status() == 'running':
break
time.sleep(1)
else:
raise RuntimeError(
"Unable to start instance, still in state: {}".format(
instance.status()))
for _ in range(60):
try:
r = exec_remote(instance, ["echo", "hello"])
if r.strip() == "hello":
break
except subprocess.CalledProcessError as e:
print("Still waiting for SSH: got: {}".format(e))
time.sleep(1)
else:
raise RuntimeError("Unable to get SSH access")
res = exec_remote(instance, [
"bash", "-c", "cd infra && git pull && sudo ./setup-builder-startup.sh"
])
print(res)
print("Builder started OK")
def builder_start_cmd(args):
instance = BuilderInstance.instance()
if instance.status() == 'stopped':
print("Starting builder instance...")
instance.start()
for i in range(60):
if instance.status() == 'running':
break
time.sleep(1)
else:
raise RuntimeError("Unable to start instance, still in state: {}".format(instance.status()))
for i in range(60):
try:
r = exec_remote(instance, ["echo", "hello"])
if r.strip() == "hello":
break
except Exception as e:
print("Still waiting for SSH: got: {}".format(e))
pass
time.sleep(1)
else:
raise RuntimeError("Unable to get SSH access")
res = exec_remote(instance, ["bash", "-c", "cd compiler-explorer-image && git pull && sudo ./setup-builder.sh"])
print(res)
print("Builder started OK")
def update(self, health=None):
if not health:
health = elb_client.describe_target_health(
TargetGroupArn=self.group_arn,
Targets=[{
'Id': self.instance.instance_id
}])['TargetHealthDescriptions'][0]
self.instance.load()
self.elb_health = health['TargetHealth']['State']
if can_ssh_to(self):
try:
self.service_status = {
key: value
for key, value in (
s.split("=", 1)
for s in exec_remote(self, [
'sudo', 'systemctl', 'show', 'compiler-explorer'
]).split("\n") if "=" in s)
}
self.running_version = exec_remote(self, [
'bash', '-c', 'if [[ -f /infra/.deploy/s3_key ]]; '
'then cat /infra/.deploy/s3_key; fi'
]).strip()
except subprocess.CalledProcessError as e:
logger.warning("Failed to execute on remote host: %s", e)
def restart_one_instance(as_group_name, instance, modified_groups):
instance_id = instance.instance.instance_id
logger.info("Enabling instance protection for {}".format(instance))
as_client.set_instance_protection(AutoScalingGroupName=as_group_name,
InstanceIds=[instance_id],
ProtectedFromScaleIn=True)
as_group = get_autoscaling_group(as_group_name)
adjustment_required = as_group['DesiredCapacity'] == as_group['MinSize']
if adjustment_required:
logger.info("Group '{}' needs to be adjusted to keep enough nodes".format(as_group_name))
modified_groups[as_group['AutoScalingGroupName']] = as_group['DesiredCapacity']
logger.info("Putting {} into standby".format(instance))
as_client.enter_standby(
InstanceIds=[instance_id],
AutoScalingGroupName=as_group_name,
ShouldDecrementDesiredCapacity=not adjustment_required)
wait_for_autoscale_state(instance, 'Standby')
logger.info("Restarting service on {}".format(instance))
restart_response = exec_remote(instance, ['sudo', 'systemctl', 'restart', 'compiler-explorer'])
if restart_response:
logger.warn("Restart gave some output: {}".format(restart_response))
wait_for_healthok(instance)
logger.info("Moving {} out of standby".format(instance))
as_client.exit_standby(
InstanceIds=[instance_id],
AutoScalingGroupName=as_group_name)
wait_for_autoscale_state(instance, 'InService')
wait_for_elb_state(instance, 'healthy')
logger.info("Disabling instance protection for {}".format(instance))
as_client.set_instance_protection(AutoScalingGroupName=as_group_name,
InstanceIds=[instance_id],
ProtectedFromScaleIn=False)
logger.info("Instance restarted ok")
def runner_start():
"""Start the runner instance."""
instance = RunnerInstance.instance()
if instance.status() == 'stopped':
print("Starting runner instance...")
instance.start()
for _ in range(60):
if instance.status() == 'running':
break
time.sleep(5)
else:
raise RuntimeError(
"Unable to start instance, still in state: {}".format(
instance.status()))
for _ in range(60):
try:
r = exec_remote(instance, ["echo", "hello"])
if r.strip() == "hello":
break
except Exception as e: # pylint: disable=broad-except
print("Still waiting for SSH: got: {}".format(e))
time.sleep(5)
else:
raise RuntimeError("Unable to get SSH access")
print("Runner started OK")
def is_everything_awesome(instance):
try:
response = exec_remote(
instance,
['curl', '-s', '--max-time', '2', 'http://127.0.0.1/healthcheck'])
return response.strip() == "Everything is awesome"
except subprocess.CalledProcessError:
return False
def wait_for_healthok(instance):
logger.info("Waiting for instance to be Online {}".format(instance))
response = ""
sys.stdout.write('Waiting')
while response != "Everything is awesome":
try:
response = exec_remote(instance, ['curl', '-s', 'http://127.0.0.1/healthcheck'])
except Exception:
sys.stdout.write('.')
# Flush stdout so tmux updates
sys.stdout.flush()
pass
time.sleep(10)
print("Ok, Everything is awesome!")
def conan_reloadwww_cmd(_):
instance = ConanInstance.instance()
exec_remote(
instance,
["sudo", "git", "-C", "/home/ubuntu/ceconan/conanproxy", "pull"])
def conan_restart_cmd(_):
instance = ConanInstance.instance()
exec_remote(instance, ["sudo", "service", "ce-conan", "restart"])
def conan_reloadwww():
"""Reload the conan web."""
instance = ConanInstance.instance()
exec_remote(
instance,
["sudo", "git", "-C", "/home/ubuntu/ceconan/conanproxy", "pull"])
def conan_restart():
"""Restart the conan instance."""
instance = ConanInstance.instance()
exec_remote(instance, ["sudo", "service", "ce-conan", "restart"])
