def test_name_reset(self):
HEADING()
n = Name()
Benchmark.Start()
n.reset()
Benchmark.Stop()
assert n.counter == 1
def setup(self):
banner("setup", c="-")
self.user = Config()["cloudmesh.profile.user"]
self.key_path = path_expand(
Config()["cloudmesh"]["profile"]["publickey"])
f = open(self.key_path, 'r')
self.key_val = f.read()
self.clouduser = '******'
self.name_generator = Name(schema=f"{self.user}-vm", counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.new_name = str(self.name_generator)
self.p = GCloudProvider(name="google")
self.secgroupname = "CM4TestSecGroup"
self.secgrouprule = {
"ip_protocol": "tcp",
"from_port": 8080,
"to_port": 8088,
"ip_range": "129.79.0.0/16"
}
self.testnode = None
def setup(self):
banner("setup", c="-")
self.user = Config()["cloudmesh"]["profile"]["user"]
self.clouduser = '******'
self.name_generator = Name(experiment="exp",
group="grp",
user=self.user,
kind="vm",
counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.new_name = str(self.name_generator)
self.p = Provider(name="azure")
self.secgroupname = "CM4TestSecGroup"
self.secgrouprule = {
"ip_protocol": "tcp",
"from_port": 8080,
"to_port": 8088,
"ip_range": "129.79.0.0/16"
}
self.testnode = None
def new_name(self, experiment=None, group=None, user=None):
"""
Generate a VM name with the format `experiment-group-name-<counter>` where `counter`
represents a running count of VMs created.
Defaults can be modified in the cloudmesh4.yaml file.
:param experiment:
:param group:
:param user:
:return: The generated name.
"""
experiment = experiment or self.config["default"]["experiment"]
group = group or self.config["default"]["group"]
user = user or getpass.getuser()
counter = Counter()
count = counter.get()
name = Name()
name_format = {
'experiment': experiment,
'group': group,
'user': user,
'counter': count
}
name.set_schema('instance')
counter.incr()
return name.get(name_format)
def setup(self):
print()
self.user = Config()["cloudmesh"]["profile"]["user"]
self.clouduser = '******'
self.name_generator = Name(experiment="exp",
group="grp",
user=self.user,
kind="vm",
counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.new_name = str(self.name_generator)
variables = Variables()
clouds = Parameter.expand(variables['cloud'])
cloud = clouds[0]
self.p = Provider(name=cloud)
self.secgroupname = "CM4TestSecGroup"
self.secgrouprule = {
"ip_protocol": "tcp",
"from_port": 8080,
"to_port": 8088,
"ip_range": "129.79.0.0/16"
}
self.testnode = None
print("\n")
def entry():
name = Name()
print(name)
d = {"cmid": str(name), "cmcounter": name.counter, "name": "albert"}
name.incr()
pprint(d)
return d
def test_name_incr(self):
HEADING()
n = Name()
Benchmark.Start()
n.incr()
Benchmark.Stop()
print(n)
assert str(n) == f"{username}-vm-2"
def setup(self):
self.database = CmDatabase()
self.name = Name(experiment="exp",
group="grp",
user="******",
kind="vm",
counter=1)
def test_define(self):
HEADING()
Benchmark.Start()
n = Name(**data)
Benchmark.Stop()
VERBOSE(data)
VERBOSE(n.dict())
assert dict(data) == n.dict()
def setup(self):
banner("setup", c="-")
self.user = Config()["cloudmesh.profile.user"]
self.name_generator = Name(schema=f"{self.user}-vm", counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.new_name = str(self.name_generator)
self.p = Provider(name=self.cloud)
def test_aws_aws_boot(self):
HEADING()
name = Name()
name.incr()
vm_name = str(name)
provider = Provider(name=CLOUD)
vm = provider.create(name=vm_name, image="ami-0c929bde1796e1484",
size="t2.micro")
print(vm)
print(provider.list())
def test_name_dict(self):
HEADING()
n = Name()
pprint(n.dict())
Benchmark.Start()
data = n.dict()
Benchmark.Stop()
assert data == dict({
'schema': '{user}-{kind}-{counter}',
'counter': 1,
'user': username,
'kind': 'vm'})
def create(self, queue_name, cluster_name, policy, charge=None, unit=None):
"""
This method is used to create a queue
:param queue_name: name of the queue to create
:param cluster_name: slurm cluster on which the job is gonna run
:param policy: policy of the queue
:param charge: charge of the queue
:param unit: unit of the charge for the queue
:return:
"""
name = Name(order=["cloud", "name"],
cloud=cluster_name,
name=queue_name)
uid = name.id(cloud=cluster_name, name=queue_name)
print(uid)
self.info = Munch({
'uid': uid,
"cloud": cluster_name,
"kind": "batch-queue",
"name": queue_name,
"cm": {
"cloud":
cluster_name,
"kind":
"batch-queue",
"name":
queue_name,
"cluster":
self.cm_config.get('cloudmesh').get('cluster')[cluster_name]
},
"queue": {
'policy': policy,
'status': 'EMPTY',
'active': False,
'charge': charge,
'unit': unit,
"numJobs": 0,
"numRunningJobs": 0,
}
})
self.policyFunctionMap = Munch({
'FIFO': self.popFIFO,
'FILO': self.popFILO
})
# list of parameters that can be set
self.settable_params = ['policy', 'charge', 'unit']
if self.database.exists(self.info)[0]:
Console.error("Queue already exists")
return
return [self.info]
def setup(self):
banner("setup", c="-")
self.user = Config()["cloudmesh.profile.user"]
self.name_generator = Name(experiment="exp",
group="grp",
user=self.user,
kind="vm",
counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.new_name = str(self.name_generator)
self.p = Provider(name=self.cloud)
def test_name_dict(self):
n = Name()
pprint(n.dict())
Benchmark.Start()
data = n.dict()
Benchmark.Stop()
assert data == dict({
'counter': 1,
'kind': 'vm',
'path': '/Users/grey/.cloudmesh/name.yaml',
'schema': '{user}-{kind}-{counter}',
'user': '******'
})
class TestName:
os = "ubuntu"
version = "18.04"
def next_name(self):
self.name_generator.incr()
self.new_name = str(self.name_generator)
return self.new_name
def setup(self):
banner("setup", c="-")
self.config = Config()
self.user = self.config["cloudmesh.profile.user"]
self.config = self.config["cloudmesh.cloud.docker"]
self.name_generator = Name(experiment="exp",
group="grp",
user=self.user,
kind="vm",
counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.new_name = str(self.name_generator)
self.p = Provider(name="docker")
def test_01_config(self):
HEADING()
pprint(self.config)
def test_02_version(self):
HEADING()
version = self.p.version()
pprint(version)
assert "18.09.2" == version["docker"]["version"]
assert 6247962 <= int(version["docker"]["build"])
assert "18.09.2" == version.docker.version
assert 6247962 <= int(version.docker.build)
'''
def test_03_terminal(self):
HEADING()
t1 = Shell.terminal()
t2 = Shell.terminal(command="pwd")
'''
def test_03_images(self):
HEADING
images = self.p.images()
pprint(images)
"""
def __init__(self, name=None, configuration="~/.cloudmesh/cloudmesh.yaml"):
"""
Initializes the provider. The default parameters are read from the configutation
file that is defined in yaml format.
:param name: The name of the provider as defined in the yaml file
:param configuration: The location of the yaml configuration filw
"""
HEADING(c=".")
conf = Config(configuration)["cloudmesh"]
#self.user = conf["profile"]
self.user = Config()["cloudmesh"]["profile"]["user"]
self.spec = conf["cloud"][name]
self.cloud = name
cred = self.spec["credentials"]
self.cloudtype = self.spec["cm"]["kind"]
super().__init__(name, conf)
self.name_generator = Name(
experiment="exp",
group="grp",
user = "******",
kind="vm",
counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.name = str(self.name_generator)
self.key_path = path_expand(Config()["cloudmesh"]["profile"]["publickey"])
f = open(self.key_path, 'r')
self.key_val = f.read()
self.testnode = None
if self.cloudtype in Provider.ProviderMapper:
self.driver = get_driver(
Provider.ProviderMapper[self.cloudtype])
if self.cloudtype == 'google':
self.cloudman = self.driver(
cred["client_email"],
cred["path_to_json_file"],
project = cred["project"]
)
else:
print("Specified provider not available")
self.cloudman = None
self.default_image = None
self.default_size = None
self.public_key_path = conf["profile"]["publickey"]
def create_name():
"""
Create volume name if name is not specified
:return: string
"""
config = Config()
n = VolumeName(user=config["cloudmesh.profile.user"],
kind="volume",
path=f"{config.location}/volume.yaml",
schema="{user}-volume-{counter}")
n.incr()
return n
def test_10_fancy(self):
HEADING()
counter = 1
n = Name(experiment="exp",
group="grp",
user="******",
kind="vm",
counter=counter)
print(n)
entries = [{
"cmcounter": counter,
"cmid": str(n),
"name": "gregor",
"phone": "android"
}]
self.database.update(entries, replace=True)
r = self.database.find()
pprint(r)
assert len(r) == 4
def test_name_print(self):
HEADING()
n = Name()
Benchmark.Start()
print(n)
Benchmark.Start()
assert str(n) == f"{username}-vm-1"
def test_provider_vm_terminate(self):
HEADING()
name = str(Name())
Benchmark.Start()
data = provider.destroy(name=name)
Benchmark.Stop()
pprint(data)
termination_timeout = 360
time = 0
while time <= termination_timeout:
sleep(5)
time += 5
if cloud == 'chameleon' and len(provider.info(name=name)) == 0:
break
elif cloud == 'google':
break
elif cloud == 'aws' and (len(
provider.info(name=name)) == 0 or provider.info(
name=name)[0]["cm"]["status"] in ['TERMINATED']):
break
elif cloud == 'azure':
try:
provider.info(name=name)
except Exception:
# if there is an exception that means the group has been
# deleted
break
# print(provider.info(name=name))
if cloud == 'chameleon':
assert len(provider.info(name=name)) == 0
elif cloud == 'google':
cm = CmDatabase()
vm = cm.find_name(name, kind="vm")[0]
assert 'status' in vm and vm['status'] == 'DELETED'
elif cloud == 'aws':
assert len(data) == 0 if data else True \
or (data[0]["cm"]["status"] in [
'BOOTING', 'TERMINATED']
if data and data[0].get('cm',
None) is not None
else True)
elif cloud == 'azure':
try:
provider.info(name=name)
except Exception:
# if there is an exception that means the group has been
# deleted
pass
elif cloud == 'oracle':
info = provider.info(name)
assert info is None or info[0]['_lifecycle_state'] in [
'TERMINATED'
]
else:
raise NotImplementedError
def test_provider_vm_wait(self):
HEADING()
name = str(Name())
Benchmark.Start()
cm = CmDatabase()
vm = cm.find_name(name, kind="vm")[0]
assert provider.wait(vm=vm), "cms wait timed out ..."
Benchmark.Stop()
def test_define_new(self):
HEADING()
os.remove(path)
Benchmark.Start()
n = Name(schema="{user}-{kind}-{counter}",
counter="3",
user=username,
kind="vm")
Benchmark.Stop()
data = n.dict()
pprint(data)
assert data == dict({
'schema': '{user}-{kind}-{counter}',
'counter': 3,
'user': username,
'kind': 'vm'})
class TestName:
def setup(self):
print()
self.user = Config()["cloudmesh"]["profile"]["user"]
self.clouduser = '******'
self.name_generator = Name(experiment="exp",
group="grp",
user=self.user,
kind="vm",
counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.new_name = str(self.name_generator)
variables = Variables()
cloud = variables['cloud']
self.p = Provider(name=cloud)
self.secgroupname = "CM4TestSecGroup"
self.secgrouprule = {
"ip_protocol": "tcp",
"from_port": 8080,
"to_port": 8088,
"ip_range": "129.79.0.0/16"
}
self.testnode = None
print("\n")
def test_01_list_flavors(self):
HEADING()
flavors = self.p.flavors()
def test_02_list_images(self):
HEADING()
flavors = self.p.images()
def test_03_list_keys(self):
HEADING()
self.keys = self.p.keys()
def test_03_list_nodes(self):
HEADING()
self.keys = self.p.list()
def test_define_new(self):
os.remove(path)
Benchmark.Start()
n = Name(schema="{user}-{kind}-{counter}",
counter="3",
user="******",
kind="vm")
Benchmark.Stop()
data = n.dict()
pprint(data)
assert data == dict({
'counter': 3,
'kind': 'vm',
'path': '/Users/grey/.cloudmesh/name.yaml',
'schema': '{user}-{kind}-{counter}',
'user': '******'
})
def test_provider_vm_info(self):
# This is just a dry run, status test actually uses info() in all
# provider
HEADING()
Benchmark.Start()
name = str(Name())
data = provider.info(name=name)
print("dry run info():")
pprint(data)
Benchmark.Stop()
def test_vm_status(self):
HEADING()
name = str(Name())
Benchmark.Start()
data = provider.status(name=name)
if type(data) == list:
data = data[0]
print(data)
Benchmark.Stop()
assert data["cm.status"] in ['ACTIVE', 'BOOTING', 'TERMINATED', 'STOPPED']
def create(self, queue_name, cluster_name, policy, charge=None, unit=None):
"""
This method is used to create a job for running on remote slurm cluster
:param queue_name: name of the queue to create
:param cluster_name: slurm cluster on which the job is gonna run
:return:
"""
name = Name(order=["cloud", "name"],
cloud=cluster_name,
name=queue_name)
uid = name.id(cloud=cluster_name, name=queue_name)
print(uid)
self.queue = {
'uid': uid,
"cloud": cluster_name,
"kind": "batch-queue",
"name": queue_name,
"cm": {
"cloud":
cluster_name,
"kind":
"batch-queue",
"name":
queue_name,
"cluster":
self.cm_config.get('cloudmesh').get('cluster')[cluster_name]
},
"queue": {
'policy': policy,
'status': 'Empty',
'active': False,
'charge': charge,
'unit': unit,
"numJobs": 0,
"numRunningJobs": 0,
}
}
if self.database.exists(self.queue)[0]:
Console.error("Queue already exists")
return
return [self.queue]
def findQueue(self, cloud_name, queue_name):
'''
finds a queue in the database based on the name
:param name: name of the queue
:return:
'''
# if self.database.exists(self.info)[0]:
# Console.error("Queue already exists")
name = Name(order=["cloud", "name"], cloud=cloud_name, name=queue_name)
uid = name.id(cloud=cloud_name, name=queue_name)
queue = self.database.find_by_KeyValue(
collection_name="{cloud}-{kind}".format(cloud=cloud_name,
kind='batch-queue'),
KeyValue={'uid': uid})
if type(queue) is cursor.Cursor:
self.info = munch.munchify(queue[0])
return True # # queue found
elif type(queue) is list and len(queue) == 0:
return False # queue not found
def test_provider_vm_ssh(self):
HEADING()
name = str(Name())
Benchmark.Start()
cm = CmDatabase()
vm = cm.find_name(name, kind="vm")[0]
data = provider.ssh(vm=vm, command='\"echo IAmAlive\"')
print(data)
assert 'IAmAlive' in data
Benchmark.Stop()
VERBOSE(data)
