def destroy(group_name=None):
if os.path.exists(CLOUD_CONFIG_FPATH):
creds_data = read_json(CLOUD_CONFIG_FPATH)
else:
print("Cloud credentials not found at %s" % CLOUD_CONFIG_FPATH)
return 1
creds = AWSCredentials(creds_data["aws"]["key"],
creds_data["aws"]["secret"])
if not os.path.exists(CLOUD_STATE_FPATH):
print("No saved cloud state info")
return 1
vms_info = read_json(CLOUD_STATE_FPATH)
to_destroy = []
if group_name:
print("Destroying hosts in the '%s' group" % group_name)
if not group_name.startswith("@"):
group_name = "@" + group_name
if group_name not in vms_info:
print("Group '%s' not found" % group_name)
return 1
region = vms_info[group_name]["meta"]["region"]
driver = get_cloud_driver(Providers.AWS, creds, region)
for name, vm_info in vms_info[group_name].items():
if name == "meta":
continue
vm_uuid = vm_info["uuid"]
vm = VM.get_by_uuid(vm_uuid, driver)
if vm is not None:
to_destroy.append(vm)
else:
print("VM '%s' not found in the clouds" % vm_uuid)
del vms_info[group_name]
else:
print("Destroying all hosts")
for group_name in [
key for key in vms_info.keys() if key.startswith("@")
]:
region = vms_info[group_name]["meta"]["region"]
driver = get_cloud_driver(Providers.AWS, creds, region)
for name, vm_info in vms_info[group_name].items():
if name == "meta":
continue
vm_uuid = vm_info["uuid"]
vm = VM.get_by_uuid(vm_uuid, driver)
if vm is not None:
to_destroy.append(vm)
else:
print("VM '%s' not found in the clouds" % vm_uuid)
del vms_info[group_name]
destroy_vms(to_destroy)
write_json(CLOUD_STATE_FPATH, vms_info)
return 0
def spawn(platform, count, role, group_name, provider=Providers.AWS, region=None):
if os.path.exists(CLOUD_CONFIG_FPATH):
creds_data = read_json(CLOUD_CONFIG_FPATH)
else:
print("Cloud credentials not found at %s" % CLOUD_CONFIG_FPATH)
return 1
if os.path.exists(CLOUD_STATE_FPATH):
vms_info = read_json(CLOUD_STATE_FPATH)
else:
vms_info = dict()
group_key = "@%s" % group_name
if group_key in vms_info:
print("Group '%s' already exists!" % group_key)
return 1
try:
creds = AWSCredentials(creds_data["aws"]["key"], creds_data["aws"]["secret"])
sec_groups = creds_data["aws"]["security_groups"]
key_pair = creds_data["aws"]["key_pair"]
except KeyError:
print("Incomplete AWS credential info") # TODO: report missing keys
return 1
region = region or creds_data["aws"].get("region", "eu-west-1")
requests = []
for i in range(count):
requests.append(VMRequest(platform=platform,
name=(platform + role + str(i)),
size=None))
print("Spawning VMs...", end="")
sys.stdout.flush()
vms = spawn_vms(requests, creds, region, key_pair,
security_groups=sec_groups,
provider=provider,
role=role)
print("DONE")
if not all(vm.public_ips for vm in vms):
print("Waiting for VMs to get IP addresses...", end="")
sys.stdout.flush() # STDOUT is line-buffered
while not all(vm.public_ips for vm in vms):
time.sleep(1)
print(".", end="")
sys.stdout.flush() # STDOUT is line-buffered
print("DONE")
vms_info[group_key] = dump_vms_info(vms)
write_json(CLOUD_STATE_FPATH, vms_info)
print("Details about the spawned VMs can be found in %s" % CLOUD_STATE_FPATH)
return 0
def get_json(url):
r = requests.get(url)
assert r.status_code >= 200 and r.status_code < 300
data = r.json()
filename = os.path.basename(url)
dir = cf_remote_dir("json")
path = os.path.join(dir, filename)
log.debug("Saving '{}' to '{}'".format(url, path))
write_json(path, data)
return data
def get_json(url):
r = requests.get(url)
assert r.status_code >= 200 and r.status_code < 300
data = r.json()
filename = os.path.basename(url)
dir = cf_remote_dir("json")
path = os.path.join(dir, filename)
log.debug("Saving '{}' to '{}'".format(url, path))
write_json(path, data)
return data
def init_cloud_config():
if os.path.exists(CLOUD_CONFIG_FPATH):
print("File %s already exists" % CLOUD_CONFIG_FPATH)
return 1
empty_config = {
"aws": {
"key": "TBD",
"secret": "TBD",
"key_pair": "TBD",
"security_groups": ["TBD"],
"region": "OPTIONAL (DEFAULT: eu-west-1)",
},
}
write_json(CLOUD_CONFIG_FPATH, empty_config)
print("Config file %s created, please complete the configuration in it." % CLOUD_CONFIG_FPATH)
def destroy(group_name=None):
if os.path.exists(CLOUD_CONFIG_FPATH):
creds_data = read_json(CLOUD_CONFIG_FPATH)
else:
print("Cloud credentials not found at %s" % CLOUD_CONFIG_FPATH)
return 1
aws_creds = None
try:
aws_creds = AWSCredentials(creds_data["aws"]["key"],
creds_data["aws"]["secret"])
except KeyError:
# missing/incomplete AWS credentials, may not be needed, though
pass
gcp_creds = None
try:
gcp_creds = GCPCredentials(creds_data["gcp"]["project_id"],
creds_data["gcp"]["service_account_id"],
creds_data["gcp"]["key_path"])
except KeyError:
# missing/incomplete GCP credentials, may not be needed, though
pass
if not os.path.exists(CLOUD_STATE_FPATH):
print("No saved cloud state info")
return 1
vms_info = read_json(CLOUD_STATE_FPATH)
to_destroy = []
if group_name:
print("Destroying hosts in the '%s' group" % group_name)
if not group_name.startswith("@"):
group_name = "@" + group_name
if group_name not in vms_info:
print("Group '%s' not found" % group_name)
return 1
region = vms_info[group_name]["meta"]["region"]
provider = vms_info[group_name]["meta"]["provider"]
if provider == "aws":
if aws_creds is None:
user_error("Missing/incomplete AWS credentials")
return 1
driver = get_cloud_driver(Providers.AWS, aws_creds, region)
if provider == "gcp":
if gcp_creds is None:
user_error("Missing/incomplete GCP credentials")
return 1
driver = get_cloud_driver(Providers.GCP, gcp_creds, region)
nodes = driver.list_nodes()
for name, vm_info in vms_info[group_name].items():
if name == "meta":
continue
vm_uuid = vm_info["uuid"]
vm = VM.get_by_uuid(vm_uuid, nodes=nodes)
if vm is not None:
to_destroy.append(vm)
else:
print("VM '%s' not found in the clouds" % vm_uuid)
del vms_info[group_name]
else:
print("Destroying all hosts")
for group_name in [
key for key in vms_info.keys() if key.startswith("@")
]:
region = vms_info[group_name]["meta"]["region"]
provider = vms_info[group_name]["meta"]["provider"]
if provider == "aws":
if aws_creds is None:
user_error("Missing/incomplete AWS credentials")
return 1
driver = get_cloud_driver(Providers.AWS, aws_creds, region)
if provider == "gcp":
if gcp_creds is None:
user_error("Missing/incomplete GCP credentials")
return 1
driver = get_cloud_driver(Providers.GCP, gcp_creds, region)
nodes = driver.list_nodes()
for name, vm_info in vms_info[group_name].items():
if name == "meta":
continue
vm_uuid = vm_info["uuid"]
vm = VM.get_by_uuid(vm_uuid, nodes=nodes)
if vm is not None:
to_destroy.append(vm)
else:
print("VM '%s' not found in the clouds" % vm_uuid)
del vms_info[group_name]
destroy_vms(to_destroy)
write_json(CLOUD_STATE_FPATH, vms_info)
return 0
def spawn(platform,
count,
role,
group_name,
provider=Providers.AWS,
region=None,
size=None,
network=None,
public_ip=True,
extend_group=False):
if os.path.exists(CLOUD_CONFIG_FPATH):
creds_data = read_json(CLOUD_CONFIG_FPATH)
else:
print("Cloud credentials not found at %s" % CLOUD_CONFIG_FPATH)
return 1
if os.path.exists(CLOUD_STATE_FPATH):
vms_info = read_json(CLOUD_STATE_FPATH)
else:
vms_info = dict()
group_key = "@%s" % group_name
group_exists = group_key in vms_info
if not extend_group and group_exists:
print("Group '%s' already exists!" % group_key)
return 1
creds = None
sec_groups = None
key_pair = None
if provider == Providers.AWS:
try:
creds = AWSCredentials(creds_data["aws"]["key"],
creds_data["aws"]["secret"])
sec_groups = creds_data["aws"]["security_groups"]
key_pair = creds_data["aws"]["key_pair"]
except KeyError:
print(
"Incomplete AWS credential info") # TODO: report missing keys
return 1
region = region or creds_data["aws"].get("region", "eu-west-1")
elif provider == Providers.GCP:
try:
creds = GCPCredentials(creds_data["gcp"]["project_id"],
creds_data["gcp"]["service_account_id"],
creds_data["gcp"]["key_path"])
except KeyError:
print(
"Incomplete GCP credential info") # TODO: report missing keys
return 1
region = region or creds_data["gcp"].get("region", "europe-west1-b")
# TODO: Do we have to complicate this instead of just assuming existing VMs
# were created by this code and thus follow the naming pattern from this
# code?
if group_exists:
range_start = len(
[key for key in vms_info[group_key].keys() if key != "meta"])
else:
range_start = 0
requests = []
for i in range(range_start, range_start + count):
vm_name = whoami()[0:2] + group_name + "-" + platform + role + str(i)
requests.append(
VMRequest(platform=platform,
name=vm_name,
size=size,
public_ip=public_ip))
print("Spawning VMs...", end="")
sys.stdout.flush()
vms = spawn_vms(requests,
creds,
region,
key_pair,
security_groups=sec_groups,
provider=provider,
network=network,
role=role,
spawned_cb=print_progress_dot)
print("DONE")
if public_ip and (not all(vm.public_ips for vm in vms)):
print("Waiting for VMs to get IP addresses...", end="")
sys.stdout.flush() # STDOUT is line-buffered
while not all(vm.public_ips for vm in vms):
time.sleep(1)
print_progress_dot()
print("DONE")
if group_exists:
vms_info[group_key].update(dump_vms_info(vms))
else:
vms_info[group_key] = dump_vms_info(vms)
write_json(CLOUD_STATE_FPATH, vms_info)
print("Details about the spawned VMs can be found in %s" %
CLOUD_STATE_FPATH)
return 0
