def destroy(group_name=None): if os.path.exists(CLOUD_CONFIG_FPATH): creds_data = read_json(CLOUD_CONFIG_FPATH) else: print("Cloud credentials not found at %s" % CLOUD_CONFIG_FPATH) return 1 creds = AWSCredentials(creds_data["aws"]["key"], creds_data["aws"]["secret"]) if not os.path.exists(CLOUD_STATE_FPATH): print("No saved cloud state info") return 1 vms_info = read_json(CLOUD_STATE_FPATH) to_destroy = [] if group_name: print("Destroying hosts in the '%s' group" % group_name) if not group_name.startswith("@"): group_name = "@" + group_name if group_name not in vms_info: print("Group '%s' not found" % group_name) return 1 region = vms_info[group_name]["meta"]["region"] driver = get_cloud_driver(Providers.AWS, creds, region) for name, vm_info in vms_info[group_name].items(): if name == "meta": continue vm_uuid = vm_info["uuid"] vm = VM.get_by_uuid(vm_uuid, driver) if vm is not None: to_destroy.append(vm) else: print("VM '%s' not found in the clouds" % vm_uuid) del vms_info[group_name] else: print("Destroying all hosts") for group_name in [ key for key in vms_info.keys() if key.startswith("@") ]: region = vms_info[group_name]["meta"]["region"] driver = get_cloud_driver(Providers.AWS, creds, region) for name, vm_info in vms_info[group_name].items(): if name == "meta": continue vm_uuid = vm_info["uuid"] vm = VM.get_by_uuid(vm_uuid, driver) if vm is not None: to_destroy.append(vm) else: print("VM '%s' not found in the clouds" % vm_uuid) del vms_info[group_name] destroy_vms(to_destroy) write_json(CLOUD_STATE_FPATH, vms_info) return 0
def spawn(platform, count, role, group_name, provider=Providers.AWS, region=None): if os.path.exists(CLOUD_CONFIG_FPATH): creds_data = read_json(CLOUD_CONFIG_FPATH) else: print("Cloud credentials not found at %s" % CLOUD_CONFIG_FPATH) return 1 if os.path.exists(CLOUD_STATE_FPATH): vms_info = read_json(CLOUD_STATE_FPATH) else: vms_info = dict() group_key = "@%s" % group_name if group_key in vms_info: print("Group '%s' already exists!" % group_key) return 1 try: creds = AWSCredentials(creds_data["aws"]["key"], creds_data["aws"]["secret"]) sec_groups = creds_data["aws"]["security_groups"] key_pair = creds_data["aws"]["key_pair"] except KeyError: print("Incomplete AWS credential info") # TODO: report missing keys return 1 region = region or creds_data["aws"].get("region", "eu-west-1") requests = [] for i in range(count): requests.append(VMRequest(platform=platform, name=(platform + role + str(i)), size=None)) print("Spawning VMs...", end="") sys.stdout.flush() vms = spawn_vms(requests, creds, region, key_pair, security_groups=sec_groups, provider=provider, role=role) print("DONE") if not all(vm.public_ips for vm in vms): print("Waiting for VMs to get IP addresses...", end="") sys.stdout.flush() # STDOUT is line-buffered while not all(vm.public_ips for vm in vms): time.sleep(1) print(".", end="") sys.stdout.flush() # STDOUT is line-buffered print("DONE") vms_info[group_key] = dump_vms_info(vms) write_json(CLOUD_STATE_FPATH, vms_info) print("Details about the spawned VMs can be found in %s" % CLOUD_STATE_FPATH) return 0
def get_json(url): r = requests.get(url) assert r.status_code >= 200 and r.status_code < 300 data = r.json() filename = os.path.basename(url) dir = cf_remote_dir("json") path = os.path.join(dir, filename) log.debug("Saving '{}' to '{}'".format(url, path)) write_json(path, data) return data
def init_cloud_config(): if os.path.exists(CLOUD_CONFIG_FPATH): print("File %s already exists" % CLOUD_CONFIG_FPATH) return 1 empty_config = { "aws": { "key": "TBD", "secret": "TBD", "key_pair": "TBD", "security_groups": ["TBD"], "region": "OPTIONAL (DEFAULT: eu-west-1)", }, } write_json(CLOUD_CONFIG_FPATH, empty_config) print("Config file %s created, please complete the configuration in it." % CLOUD_CONFIG_FPATH)
def destroy(group_name=None): if os.path.exists(CLOUD_CONFIG_FPATH): creds_data = read_json(CLOUD_CONFIG_FPATH) else: print("Cloud credentials not found at %s" % CLOUD_CONFIG_FPATH) return 1 aws_creds = None try: aws_creds = AWSCredentials(creds_data["aws"]["key"], creds_data["aws"]["secret"]) except KeyError: # missing/incomplete AWS credentials, may not be needed, though pass gcp_creds = None try: gcp_creds = GCPCredentials(creds_data["gcp"]["project_id"], creds_data["gcp"]["service_account_id"], creds_data["gcp"]["key_path"]) except KeyError: # missing/incomplete GCP credentials, may not be needed, though pass if not os.path.exists(CLOUD_STATE_FPATH): print("No saved cloud state info") return 1 vms_info = read_json(CLOUD_STATE_FPATH) to_destroy = [] if group_name: print("Destroying hosts in the '%s' group" % group_name) if not group_name.startswith("@"): group_name = "@" + group_name if group_name not in vms_info: print("Group '%s' not found" % group_name) return 1 region = vms_info[group_name]["meta"]["region"] provider = vms_info[group_name]["meta"]["provider"] if provider == "aws": if aws_creds is None: user_error("Missing/incomplete AWS credentials") return 1 driver = get_cloud_driver(Providers.AWS, aws_creds, region) if provider == "gcp": if gcp_creds is None: user_error("Missing/incomplete GCP credentials") return 1 driver = get_cloud_driver(Providers.GCP, gcp_creds, region) nodes = driver.list_nodes() for name, vm_info in vms_info[group_name].items(): if name == "meta": continue vm_uuid = vm_info["uuid"] vm = VM.get_by_uuid(vm_uuid, nodes=nodes) if vm is not None: to_destroy.append(vm) else: print("VM '%s' not found in the clouds" % vm_uuid) del vms_info[group_name] else: print("Destroying all hosts") for group_name in [ key for key in vms_info.keys() if key.startswith("@") ]: region = vms_info[group_name]["meta"]["region"] provider = vms_info[group_name]["meta"]["provider"] if provider == "aws": if aws_creds is None: user_error("Missing/incomplete AWS credentials") return 1 driver = get_cloud_driver(Providers.AWS, aws_creds, region) if provider == "gcp": if gcp_creds is None: user_error("Missing/incomplete GCP credentials") return 1 driver = get_cloud_driver(Providers.GCP, gcp_creds, region) nodes = driver.list_nodes() for name, vm_info in vms_info[group_name].items(): if name == "meta": continue vm_uuid = vm_info["uuid"] vm = VM.get_by_uuid(vm_uuid, nodes=nodes) if vm is not None: to_destroy.append(vm) else: print("VM '%s' not found in the clouds" % vm_uuid) del vms_info[group_name] destroy_vms(to_destroy) write_json(CLOUD_STATE_FPATH, vms_info) return 0
def spawn(platform, count, role, group_name, provider=Providers.AWS, region=None, size=None, network=None, public_ip=True, extend_group=False): if os.path.exists(CLOUD_CONFIG_FPATH): creds_data = read_json(CLOUD_CONFIG_FPATH) else: print("Cloud credentials not found at %s" % CLOUD_CONFIG_FPATH) return 1 if os.path.exists(CLOUD_STATE_FPATH): vms_info = read_json(CLOUD_STATE_FPATH) else: vms_info = dict() group_key = "@%s" % group_name group_exists = group_key in vms_info if not extend_group and group_exists: print("Group '%s' already exists!" % group_key) return 1 creds = None sec_groups = None key_pair = None if provider == Providers.AWS: try: creds = AWSCredentials(creds_data["aws"]["key"], creds_data["aws"]["secret"]) sec_groups = creds_data["aws"]["security_groups"] key_pair = creds_data["aws"]["key_pair"] except KeyError: print( "Incomplete AWS credential info") # TODO: report missing keys return 1 region = region or creds_data["aws"].get("region", "eu-west-1") elif provider == Providers.GCP: try: creds = GCPCredentials(creds_data["gcp"]["project_id"], creds_data["gcp"]["service_account_id"], creds_data["gcp"]["key_path"]) except KeyError: print( "Incomplete GCP credential info") # TODO: report missing keys return 1 region = region or creds_data["gcp"].get("region", "europe-west1-b") # TODO: Do we have to complicate this instead of just assuming existing VMs # were created by this code and thus follow the naming pattern from this # code? if group_exists: range_start = len( [key for key in vms_info[group_key].keys() if key != "meta"]) else: range_start = 0 requests = [] for i in range(range_start, range_start + count): vm_name = whoami()[0:2] + group_name + "-" + platform + role + str(i) requests.append( VMRequest(platform=platform, name=vm_name, size=size, public_ip=public_ip)) print("Spawning VMs...", end="") sys.stdout.flush() vms = spawn_vms(requests, creds, region, key_pair, security_groups=sec_groups, provider=provider, network=network, role=role, spawned_cb=print_progress_dot) print("DONE") if public_ip and (not all(vm.public_ips for vm in vms)): print("Waiting for VMs to get IP addresses...", end="") sys.stdout.flush() # STDOUT is line-buffered while not all(vm.public_ips for vm in vms): time.sleep(1) print_progress_dot() print("DONE") if group_exists: vms_info[group_key].update(dump_vms_info(vms)) else: vms_info[group_key] = dump_vms_info(vms) write_json(CLOUD_STATE_FPATH, vms_info) print("Details about the spawned VMs can be found in %s" % CLOUD_STATE_FPATH) return 0