def test_name_reset(self): HEADING() n = Name() Benchmark.Start() n.reset() Benchmark.Stop() assert n.counter == 1
def setup(self): banner("setup", c="-") self.user = Config()["cloudmesh.profile.user"] self.key_path = path_expand( Config()["cloudmesh"]["profile"]["publickey"]) f = open(self.key_path, 'r') self.key_val = f.read() self.clouduser = '******' self.name_generator = Name(schema=f"{self.user}-vm", counter=1) self.name = str(self.name_generator) self.name_generator.incr() self.new_name = str(self.name_generator) self.p = GCloudProvider(name="google") self.secgroupname = "CM4TestSecGroup" self.secgrouprule = { "ip_protocol": "tcp", "from_port": 8080, "to_port": 8088, "ip_range": "129.79.0.0/16" } self.testnode = None
def setup(self): banner("setup", c="-") self.user = Config()["cloudmesh"]["profile"]["user"] self.clouduser = '******' self.name_generator = Name(experiment="exp", group="grp", user=self.user, kind="vm", counter=1) self.name = str(self.name_generator) self.name_generator.incr() self.new_name = str(self.name_generator) self.p = Provider(name="azure") self.secgroupname = "CM4TestSecGroup" self.secgrouprule = { "ip_protocol": "tcp", "from_port": 8080, "to_port": 8088, "ip_range": "129.79.0.0/16" } self.testnode = None
def new_name(self, experiment=None, group=None, user=None): """ Generate a VM name with the format `experiment-group-name-<counter>` where `counter` represents a running count of VMs created. Defaults can be modified in the cloudmesh4.yaml file. :param experiment: :param group: :param user: :return: The generated name. """ experiment = experiment or self.config["default"]["experiment"] group = group or self.config["default"]["group"] user = user or getpass.getuser() counter = Counter() count = counter.get() name = Name() name_format = { 'experiment': experiment, 'group': group, 'user': user, 'counter': count } name.set_schema('instance') counter.incr() return name.get(name_format)
def setup(self): print() self.user = Config()["cloudmesh"]["profile"]["user"] self.clouduser = '******' self.name_generator = Name(experiment="exp", group="grp", user=self.user, kind="vm", counter=1) self.name = str(self.name_generator) self.name_generator.incr() self.new_name = str(self.name_generator) variables = Variables() clouds = Parameter.expand(variables['cloud']) cloud = clouds[0] self.p = Provider(name=cloud) self.secgroupname = "CM4TestSecGroup" self.secgrouprule = { "ip_protocol": "tcp", "from_port": 8080, "to_port": 8088, "ip_range": "129.79.0.0/16" } self.testnode = None print("\n")
def entry(): name = Name() print(name) d = {"cmid": str(name), "cmcounter": name.counter, "name": "albert"} name.incr() pprint(d) return d
def test_name_incr(self): HEADING() n = Name() Benchmark.Start() n.incr() Benchmark.Stop() print(n) assert str(n) == f"{username}-vm-2"
def setup(self): self.database = CmDatabase() self.name = Name(experiment="exp", group="grp", user="******", kind="vm", counter=1)
def test_define(self): HEADING() Benchmark.Start() n = Name(**data) Benchmark.Stop() VERBOSE(data) VERBOSE(n.dict()) assert dict(data) == n.dict()
def setup(self): banner("setup", c="-") self.user = Config()["cloudmesh.profile.user"] self.name_generator = Name(schema=f"{self.user}-vm", counter=1) self.name = str(self.name_generator) self.name_generator.incr() self.new_name = str(self.name_generator) self.p = Provider(name=self.cloud)
def test_aws_aws_boot(self): HEADING() name = Name() name.incr() vm_name = str(name) provider = Provider(name=CLOUD) vm = provider.create(name=vm_name, image="ami-0c929bde1796e1484", size="t2.micro") print(vm) print(provider.list())
def test_name_dict(self): HEADING() n = Name() pprint(n.dict()) Benchmark.Start() data = n.dict() Benchmark.Stop() assert data == dict({ 'schema': '{user}-{kind}-{counter}', 'counter': 1, 'user': username, 'kind': 'vm'})
def create(self, queue_name, cluster_name, policy, charge=None, unit=None): """ This method is used to create a queue :param queue_name: name of the queue to create :param cluster_name: slurm cluster on which the job is gonna run :param policy: policy of the queue :param charge: charge of the queue :param unit: unit of the charge for the queue :return: """ name = Name(order=["cloud", "name"], cloud=cluster_name, name=queue_name) uid = name.id(cloud=cluster_name, name=queue_name) print(uid) self.info = Munch({ 'uid': uid, "cloud": cluster_name, "kind": "batch-queue", "name": queue_name, "cm": { "cloud": cluster_name, "kind": "batch-queue", "name": queue_name, "cluster": self.cm_config.get('cloudmesh').get('cluster')[cluster_name] }, "queue": { 'policy': policy, 'status': 'EMPTY', 'active': False, 'charge': charge, 'unit': unit, "numJobs": 0, "numRunningJobs": 0, } }) self.policyFunctionMap = Munch({ 'FIFO': self.popFIFO, 'FILO': self.popFILO }) # list of parameters that can be set self.settable_params = ['policy', 'charge', 'unit'] if self.database.exists(self.info)[0]: Console.error("Queue already exists") return return [self.info]
def setup(self): banner("setup", c="-") self.user = Config()["cloudmesh.profile.user"] self.name_generator = Name(experiment="exp", group="grp", user=self.user, kind="vm", counter=1) self.name = str(self.name_generator) self.name_generator.incr() self.new_name = str(self.name_generator) self.p = Provider(name=self.cloud)
def test_name_dict(self): n = Name() pprint(n.dict()) Benchmark.Start() data = n.dict() Benchmark.Stop() assert data == dict({ 'counter': 1, 'kind': 'vm', 'path': '/Users/grey/.cloudmesh/name.yaml', 'schema': '{user}-{kind}-{counter}', 'user': '******' })
class TestName: os = "ubuntu" version = "18.04" def next_name(self): self.name_generator.incr() self.new_name = str(self.name_generator) return self.new_name def setup(self): banner("setup", c="-") self.config = Config() self.user = self.config["cloudmesh.profile.user"] self.config = self.config["cloudmesh.cloud.docker"] self.name_generator = Name(experiment="exp", group="grp", user=self.user, kind="vm", counter=1) self.name = str(self.name_generator) self.name_generator.incr() self.new_name = str(self.name_generator) self.p = Provider(name="docker") def test_01_config(self): HEADING() pprint(self.config) def test_02_version(self): HEADING() version = self.p.version() pprint(version) assert "18.09.2" == version["docker"]["version"] assert 6247962 <= int(version["docker"]["build"]) assert "18.09.2" == version.docker.version assert 6247962 <= int(version.docker.build) ''' def test_03_terminal(self): HEADING() t1 = Shell.terminal() t2 = Shell.terminal(command="pwd") ''' def test_03_images(self): HEADING images = self.p.images() pprint(images) """
def __init__(self, name=None, configuration="~/.cloudmesh/cloudmesh.yaml"): """ Initializes the provider. The default parameters are read from the configutation file that is defined in yaml format. :param name: The name of the provider as defined in the yaml file :param configuration: The location of the yaml configuration filw """ HEADING(c=".") conf = Config(configuration)["cloudmesh"] #self.user = conf["profile"] self.user = Config()["cloudmesh"]["profile"]["user"] self.spec = conf["cloud"][name] self.cloud = name cred = self.spec["credentials"] self.cloudtype = self.spec["cm"]["kind"] super().__init__(name, conf) self.name_generator = Name( experiment="exp", group="grp", user = "******", kind="vm", counter=1) self.name = str(self.name_generator) self.name_generator.incr() self.name = str(self.name_generator) self.key_path = path_expand(Config()["cloudmesh"]["profile"]["publickey"]) f = open(self.key_path, 'r') self.key_val = f.read() self.testnode = None if self.cloudtype in Provider.ProviderMapper: self.driver = get_driver( Provider.ProviderMapper[self.cloudtype]) if self.cloudtype == 'google': self.cloudman = self.driver( cred["client_email"], cred["path_to_json_file"], project = cred["project"] ) else: print("Specified provider not available") self.cloudman = None self.default_image = None self.default_size = None self.public_key_path = conf["profile"]["publickey"]
def create_name(): """ Create volume name if name is not specified :return: string """ config = Config() n = VolumeName(user=config["cloudmesh.profile.user"], kind="volume", path=f"{config.location}/volume.yaml", schema="{user}-volume-{counter}") n.incr() return n
def test_10_fancy(self): HEADING() counter = 1 n = Name(experiment="exp", group="grp", user="******", kind="vm", counter=counter) print(n) entries = [{ "cmcounter": counter, "cmid": str(n), "name": "gregor", "phone": "android" }] self.database.update(entries, replace=True) r = self.database.find() pprint(r) assert len(r) == 4
def test_name_print(self): HEADING() n = Name() Benchmark.Start() print(n) Benchmark.Start() assert str(n) == f"{username}-vm-1"
def test_provider_vm_terminate(self): HEADING() name = str(Name()) Benchmark.Start() data = provider.destroy(name=name) Benchmark.Stop() pprint(data) termination_timeout = 360 time = 0 while time <= termination_timeout: sleep(5) time += 5 if cloud == 'chameleon' and len(provider.info(name=name)) == 0: break elif cloud == 'google': break elif cloud == 'aws' and (len( provider.info(name=name)) == 0 or provider.info( name=name)[0]["cm"]["status"] in ['TERMINATED']): break elif cloud == 'azure': try: provider.info(name=name) except Exception: # if there is an exception that means the group has been # deleted break # print(provider.info(name=name)) if cloud == 'chameleon': assert len(provider.info(name=name)) == 0 elif cloud == 'google': cm = CmDatabase() vm = cm.find_name(name, kind="vm")[0] assert 'status' in vm and vm['status'] == 'DELETED' elif cloud == 'aws': assert len(data) == 0 if data else True \ or (data[0]["cm"]["status"] in [ 'BOOTING', 'TERMINATED'] if data and data[0].get('cm', None) is not None else True) elif cloud == 'azure': try: provider.info(name=name) except Exception: # if there is an exception that means the group has been # deleted pass elif cloud == 'oracle': info = provider.info(name) assert info is None or info[0]['_lifecycle_state'] in [ 'TERMINATED' ] else: raise NotImplementedError
def test_provider_vm_wait(self): HEADING() name = str(Name()) Benchmark.Start() cm = CmDatabase() vm = cm.find_name(name, kind="vm")[0] assert provider.wait(vm=vm), "cms wait timed out ..." Benchmark.Stop()
def test_define_new(self): HEADING() os.remove(path) Benchmark.Start() n = Name(schema="{user}-{kind}-{counter}", counter="3", user=username, kind="vm") Benchmark.Stop() data = n.dict() pprint(data) assert data == dict({ 'schema': '{user}-{kind}-{counter}', 'counter': 3, 'user': username, 'kind': 'vm'})
class TestName: def setup(self): print() self.user = Config()["cloudmesh"]["profile"]["user"] self.clouduser = '******' self.name_generator = Name(experiment="exp", group="grp", user=self.user, kind="vm", counter=1) self.name = str(self.name_generator) self.name_generator.incr() self.new_name = str(self.name_generator) variables = Variables() cloud = variables['cloud'] self.p = Provider(name=cloud) self.secgroupname = "CM4TestSecGroup" self.secgrouprule = { "ip_protocol": "tcp", "from_port": 8080, "to_port": 8088, "ip_range": "129.79.0.0/16" } self.testnode = None print("\n") def test_01_list_flavors(self): HEADING() flavors = self.p.flavors() def test_02_list_images(self): HEADING() flavors = self.p.images() def test_03_list_keys(self): HEADING() self.keys = self.p.keys() def test_03_list_nodes(self): HEADING() self.keys = self.p.list()
def test_define_new(self): os.remove(path) Benchmark.Start() n = Name(schema="{user}-{kind}-{counter}", counter="3", user="******", kind="vm") Benchmark.Stop() data = n.dict() pprint(data) assert data == dict({ 'counter': 3, 'kind': 'vm', 'path': '/Users/grey/.cloudmesh/name.yaml', 'schema': '{user}-{kind}-{counter}', 'user': '******' })
def test_provider_vm_info(self): # This is just a dry run, status test actually uses info() in all # provider HEADING() Benchmark.Start() name = str(Name()) data = provider.info(name=name) print("dry run info():") pprint(data) Benchmark.Stop()
def test_vm_status(self): HEADING() name = str(Name()) Benchmark.Start() data = provider.status(name=name) if type(data) == list: data = data[0] print(data) Benchmark.Stop() assert data["cm.status"] in ['ACTIVE', 'BOOTING', 'TERMINATED', 'STOPPED']
def create(self, queue_name, cluster_name, policy, charge=None, unit=None): """ This method is used to create a job for running on remote slurm cluster :param queue_name: name of the queue to create :param cluster_name: slurm cluster on which the job is gonna run :return: """ name = Name(order=["cloud", "name"], cloud=cluster_name, name=queue_name) uid = name.id(cloud=cluster_name, name=queue_name) print(uid) self.queue = { 'uid': uid, "cloud": cluster_name, "kind": "batch-queue", "name": queue_name, "cm": { "cloud": cluster_name, "kind": "batch-queue", "name": queue_name, "cluster": self.cm_config.get('cloudmesh').get('cluster')[cluster_name] }, "queue": { 'policy': policy, 'status': 'Empty', 'active': False, 'charge': charge, 'unit': unit, "numJobs": 0, "numRunningJobs": 0, } } if self.database.exists(self.queue)[0]: Console.error("Queue already exists") return return [self.queue]
def findQueue(self, cloud_name, queue_name): ''' finds a queue in the database based on the name :param name: name of the queue :return: ''' # if self.database.exists(self.info)[0]: # Console.error("Queue already exists") name = Name(order=["cloud", "name"], cloud=cloud_name, name=queue_name) uid = name.id(cloud=cloud_name, name=queue_name) queue = self.database.find_by_KeyValue( collection_name="{cloud}-{kind}".format(cloud=cloud_name, kind='batch-queue'), KeyValue={'uid': uid}) if type(queue) is cursor.Cursor: self.info = munch.munchify(queue[0]) return True # # queue found elif type(queue) is list and len(queue) == 0: return False # queue not found
def test_provider_vm_ssh(self): HEADING() name = str(Name()) Benchmark.Start() cm = CmDatabase() vm = cm.find_name(name, kind="vm")[0] data = provider.ssh(vm=vm, command='\"echo IAmAlive\"') print(data) assert 'IAmAlive' in data Benchmark.Stop() VERBOSE(data)