def setup(self):
self.status_id = "test-id-1"
config = Config().data["cloudmesh"]["data"]["mongo"]
self.mongo = MongoDB(host=config["MONGO_HOST"],
username=config["MONGO_USERNAME"],
password=config["MONGO_PASSWORD"],
port=config["MONGO_PORT"])
def __init__(self):
config = Config().data['cloudmesh']
self.private_key_file = config['cloud']['aws']['credentials']['EC2_PRIVATE_KEY_FILE_PATH']
self.mongo = MongoDB(host=config['data']['mongo']['MONGO_HOST'],
username=config['data']['mongo']['MONGO_USERNAME'],
password=config['data']['mongo']['MONGO_PASSWORD'],
port=config['data']['mongo']['MONGO_PORT'])
class TestMongo:
def setup(self):
self.status_id = "test-id-1"
config = Config().data["cloudmesh"]["data"]["mongo"]
self.mongo = MongoDB(host=config["MONGO_HOST"],
username=config["MONGO_USERNAME"],
password=config["MONGO_PASSWORD"],
port=config["MONGO_PORT"])
def test_01_insert_status(self):
HEADING(myself())
status = self.mongo.status_document(self.status_id, "running",
"job-id-1", "history")
post_id = self.mongo.insert_status_collection(status)
assert post_id is not None
def test_02_find(self):
HEADING(myself())
doc = self.mongo.find_document("status", "id", self.status_id)
assert doc is not None
def test_03_delete(self):
HEADING(myself())
old_doc = self.mongo.delete_document("status", "id", self.status_id)
assert old_doc is not None
deleted_doc = self.mongo.find_document("status", "id", self.status_id)
assert deleted_doc is None
def __init__(self, cloud):
self.config = Config().data["cloudmesh"]
self.provider = Vmprovider().get_provider(cloud)
self.mongo = MongoDB(
host=self.config["data"]["mongo"]["MONGO_HOST"],
username=self.config["data"]["mongo"]["MONGO_USERNAME"],
password=self.config["data"]["mongo"]["MONGO_PASSWORD"],
port=self.config["data"]["mongo"]["MONGO_PORT"])
def __init__(self, **kwargs):
"""
initialize the mongodb connection, ger required information to access nodes in AWS
:param kwargs: port, username, password for mongodb
"""
self.database = MongoDB()
self.database.set_port(kwargs['port'])
self.database.set_username(kwargs['username'])
self.database.set_password(kwargs['password'])
self.database.connect_db()
credential = self.database.find_document('cloud', 'kind', 'aws')
self.controller = AWSController(credential['EC2_ACCESS_ID'],
credential['EC2_SECRET_KEY'], credential['EC2_REGION'])
self.ssh_key = credential['EC2_PRIVATE_KEY_FILE']
def __init__(self, cloud):
self.mongo = MongoDB()
self.config = Config().data["cloudmesh"]
self.public_key_path = self.config["profile"]["key"]["public"]
self.kind = self.config["cloud"][cloud]["cm"]["kind"]
if self.kind == 'azure':
self.provider = AzureProvider(self.config)
elif self.kind == 'aws':
self.provider = AwsProvider(self.config)
elif self.kind == 'openstack':
self.provider = OpenstackCM("chameleon")
elif self.kind == 'vbox':
raise NotImplementedError
else:
raise NotImplementedError(f"Cloud `{self.kind}` not supported.")
from flask import request, jsonify
from cm4.mongo.mongoDB import MongoDB
from cm4.configuration.config import Config
from cm4.vm.Vm import Vm
config = Config()
db = MongoDB(config.get('data.mongo.MONGO_DBNAME'),
config.get('data.mongo.MONGO_USERNAME'),
config.get('data.mongo.MONGO_PASSWORD'),
config.get('data.mongo.MONGO_PORT'))
db.connect_db()
def vm_list():
cloud = request.args.get('cloud')
if cloud:
rep = Vm(cloud).list()
return 'No node is found on {}!\n'.format(cloud) if not rep else \
jsonify(**{'records': [db.var_to_json(x.__dict__) for x in rep]})
else:
return jsonify(**{'records': [db.var_to_json(x) for x in db.db['cloud'].find()]})
class Vm:
def __init__(self, cloud):
self.config = Config().data["cloudmesh"]
self.provider = Vmprovider().get_provider(cloud)
self.mongo = MongoDB(
host=self.config["data"]["mongo"]["MONGO_HOST"],
username=self.config["data"]["mongo"]["MONGO_USERNAME"],
password=self.config["data"]["mongo"]["MONGO_PASSWORD"],
port=self.config["data"]["mongo"]["MONGO_PORT"])
def start(self, name):
"""
start the node based on the id
:param name:
:return: VM document
"""
info = self.info(name)
if info.state != 'running':
self.provider.ex_start_node(info)
Thread(self, 'test', name, 'running').start()
document = self.mongo.find_document('cloud', 'name', name)
return document
else:
document = self.mongo.find_document('cloud', 'name', name)
return document
def stop(self, name, deallocate=True):
"""
stop the node based on the ide
:param name:
:param deallocate:
:return: VM document
"""
info = self.info(name)
if info.state != 'stopped':
self.provider.ex_stop_node(info, deallocate)
Thread(self, 'test', name, 'stopped').start()
document = self.mongo.find_document('cloud', 'name', name)
return document
else:
document = self.mongo.find_document('cloud', 'name', name)
return document
def resume(self, name):
"""
start the node based on id
:param name:
"""
return self.start(name)
def suspend(self, name):
"""
stop the node based on id
:param name:
"""
return self.stop(name, False)
def destroy(self, name):
"""
delete the node based on id
:param name:
:return: True/False
"""
result = self.provider.destroy_node(self.info(name))
self.mongo.delete_document('cloud', 'name', name)
return result
def create(self, name):
"""
create a new node
:param name: the name for the new node
:return:
"""
node = self.provider.create_node(name)
self.mongo.insert_cloud_document(vars(node))
Thread(self, 'test', name, 'running').start()
return node
def list(self):
"""
list existed nodes
:return: all nodes' information
"""
result = self.provider.list_nodes()
return result
def status(self, name):
"""
show node information based on id
:param name:
:return: all information about one node
"""
self.info(name)
status = self.mongo.find_document('cloud', 'name', name)['state']
return status
def info(self, name):
"""
show node information based on id
:param name:
:return: all information about one node
"""
nodes = self.list()
for i in nodes:
if i.name == name:
document = vars(i)
if self.mongo.find_document('cloud', 'name', name):
self.mongo.update_document('cloud', 'name', name, document)
else:
self.mongo.insert_cloud_document(document)
return i
raise ValueError('Node: ' + name + ' does not exist!')
def new_name(self, experiment=None, group=None, user=None):
"""
Generate a VM name with the format `experiment-group-name-<counter>` where `counter`
represents a running count of VMs created.
Defaults can be modified in the cloudmesh4.yaml file.
:param experiment:
:param group:
:param user:
:return: The generated name.
"""
experiment = experiment or self.config["default"]["experiment"]
group = group or self.config["default"]["group"]
user = user or getpass.getuser()
counter = Counter()
count = counter.get()
name = Name()
name_format = {
'experiment': experiment,
'group': group,
'user': user,
'counter': count
}
name.set_schema('instance')
counter.incr()
return name.get(name_format)
def get_public_ips(self, name=None):
"""
Returns all the public ips available if a name is not given.
If a name is provided, the ip of the vm name would be returned.
:param name: name of the VM.
:return: Dictionary of VMs with their public ips
"""
if name is None:
filters = {"$exists": True, "$not": {"$size": 0}}
documents = self.mongo.find('cloud', 'public_ips', filters)
if documents is None:
return None
else:
result = {}
for document in documents:
result[document['name']] = document['public_ips']
return result
else:
public_ips = self.mongo.find_document('cloud', 'name',
name)['public_ips']
if not public_ips:
return None
else:
return {name: public_ips}
def set_public_ip(self, name, public_ip):
"""
Assign the given public ip to the given VM.
:param name: name of the VM
:param public_ip: public ip to be assigned.
"""
if name is not None and public_ip is not None:
self.provider.set_public_ip(name, public_ip)
def remove_public_ip(self, name):
"""
Deletes the public ip of the given VM.
:param name: name of the VM
"""
if name is not None:
self.provider.remove_public_ip(name)
class AWSCommand(ProcessManagerABC):
def __init__(self, **kwargs):
"""
initialize the mongodb connection, ger required information to access nodes in AWS
:param kwargs: port, username, password for mongodb
"""
self.database = MongoDB()
self.database.set_port(kwargs['port'])
self.database.set_username(kwargs['username'])
self.database.set_password(kwargs['password'])
self.database.connect_db()
credential = self.database.find_document('cloud', 'kind', 'aws')
self.controller = AWSController(credential['EC2_ACCESS_ID'],
credential['EC2_SECRET_KEY'], credential['EC2_REGION'])
self.ssh_key = credential['EC2_PRIVATE_KEY_FILE']
def check_running_node(self, list_id):
"""
user inputs list of node ids, check the node is running or not
:param list_id: list of required node id
:return: stopped node id
"""
list_instance_status = self.controller.ls()
stopped_instance = []
for i in list_id:
for id, name, state in list_instance_status:
if i == id and state != 'running':
stopped_instance.append([i, state])
return stopped_instance
def find_node_DNS(self, name_id):
"""
based on AWS design, we need to get the DNS of the required nodes
:param name_id: node is
:return: the DNS
"""
nodes = self.controller.ls()
required_id = ''
for i in nodes:
if i.name == name_id or i.id == name_id:
required_id = i.id
DNS = self.controller.info(required_id)['extra']['dns_name']
return DNS
def read_script(self, script):
"""
read the script file
:param script: the script that would be run in instance
:return content: the content of the script
"""
content = open(script, "r").read()
return content
def run_command(self, command, vm_list):
"""
running command in nodes, update the job and status document in MongoDB
:param command: the input command
:param vm_list: list of node id
:return: the result of each node
"""
output = []
for i in vm_list:
username = '******' + self.find_node_DNS(i)
job_id = self.start_job('Null', command, i)
self.update_status(i, job_id)
temp = subprocess.check_output(['ssh', '-i', self.ssh_key, username,
command]).decode("utf-8")
self.end_job(i, temp)
self.update_status(i, job_id)
output.append('Running command: ' + command + ' in Instance(name or id) ' + i + ':\n' + temp)
return output
def run_script(self, script, vm_list):
"""
running script in nodes, update the job and status document in MongoDB
:param script: the input script
:param vm_list: list of node id
:return: the result of each node
"""
content = self.read_script(script)
output = []
for i in vm_list:
username = '******' + self.find_node_DNS(i)
job_id = self.start_job(content, 'Null', i)
self.update_status(i, job_id)
temp = subprocess.check_output(['ssh', '-i', self.ssh_key, username, content]).decode("utf-8")
self.end_job(i, temp)
self.update_status(i, job_id)
output.append('Running command: ' + script + ' in Instance(name or id) ' + i + ':\n' + temp)
return output
def parallel(self):
print('working on it')
def start_job(self, script, command, vm):
"""
create new job document in MongoDB, status is processing
:param script: the running script
:param command: the input command
:param vm: the node id
:return: the job document id
"""
job = self.database.job_document(vm, 'processing', script, 'Null', 'Null', command)
return self.database.insert_job_collection(job)
def end_job(self, ID, output):
"""
jod is done, update the information into job collection in MongoDB, status is done
:param ID: the job document id
:param output: the result
:return: True/False
"""
return self.database.update_document('job', ID, dict(status='done', output=output))
def update_status(self, vm, job_id):
"""
for each node, it has its own status. when job is running, it will update its currentJob. When the jon id one,
currentJob is Null, and history will be update
:param vm: node id
:param job_id: jod id
"""
status = self.database.find_document('status', '_id', vm)
history = status['history']
if status['currentJob'] == job_id:
self.database.update_document('status', vm, dict(currentJob='Null', history=history.append(job_id)))
else:
self.database.update_document('status', vm, dict(currentJob=job_id))
def disconnect(self):
"""
disconnect from mongodb
"""
self.database.close_client()
class Vm(CloudManagerABC):
def __init__(self, cloud):
self.mongo = MongoDB()
self.config = Config().data["cloudmesh"]
self.public_key_path = self.config["profile"]["key"]["public"]
self.kind = self.config["cloud"][cloud]["cm"]["kind"]
if self.kind == 'azure':
self.provider = AzureProvider(self.config)
elif self.kind == 'aws':
self.provider = AwsProvider(self.config)
elif self.kind == 'openstack':
self.provider = OpenstackCM("chameleon")
elif self.kind == 'vbox':
raise NotImplementedError
else:
raise NotImplementedError(f"Cloud `{self.kind}` not supported.")
def start(self, name):
"""
start the node based on the id
:param name:
:return: VM document
"""
if self.kind in ["vbox"]:
raise NotImplementedError
else:
info = self.info(name)
if info.state != 'running':
self.provider.start(**info)
Thread(self, 'test', name, 'running').start()
document = self.mongo.find_document('cloud', 'name', name)
return document
else:
document = self.mongo.find_document('cloud', 'name', name)
return document
def stop(self, name=None):
"""
stop the node based on the ide
:param name:
:return: VM document
"""
info = self.info(name)
if info.state != 'stopped':
self.provider.stop(name)
Thread(self, 'test', name, 'stopped').start()
document = self.mongo.find_document('cloud', 'name', name)
return document
else:
document = self.mongo.find_document('cloud', 'name', name)
return document
def resume(self, name=None):
"""
start the node based on id
:param name:
"""
return self.start(name)
def suspend(self, name=None):
"""
stop the node based on id
:param name:
"""
return self.stop(name)
def destroy(self, name=None):
"""
delete the node based on id
:param name:
:return: True/False
"""
result = self.provider.destroy(name)
self.mongo.delete_document('cloud', 'name', name)
return result
def create(self, name=None):
"""
create a new node
:param name: the name for the new node
:return:
"""
name = name or self.new_name()
node = self.provider.create(name=name)
self.mongo.insert_cloud_document(vars(node))
Thread(self, 'test', name, 'running').start()
return node
def nodes(self):
return self.provider.nodes()
def status(self, name):
"""
show node information based on id
:param name:
:return: all information about one node
"""
self.info(name)
status = self.mongo.find_document('cloud', 'name', name)['state']
return status
def info(self, name=None):
"""
show node information based on id
:param name:
:return: all information about one node
"""
nodes = self.nodes()
for i in nodes:
if i.name == name:
document = vars(i)
if self.mongo.find_document('cloud', 'name', name):
self.mongo.update_document('cloud', 'name', name, document)
else:
self.mongo.insert_cloud_document(document)
return i
raise ValueError(f"Node: {name} does not exist!")
def new_name(self, experiment=None, group=None, user=None):
"""
Generate a VM name with the format `experiment-group-name-<counter>` where `counter`
represents a running count of VMs created.
Defaults can be modified in the cloudmesh4.yaml file.
:param experiment:
:param group:
:param user:
:return: The generated name.
"""
experiment = experiment or self.config["default"]["experiment"]
group = group or self.config["default"]["group"]
user = user or getpass.getuser()
counter = Counter()
count = counter.get()
name = Name()
name_format = {
'experiment': experiment,
'group': group,
'user': user,
'counter': count
}
name.set_schema('instance')
counter.incr()
return name.get(name_format)
def get_public_ips(self, name=None):
"""
Returns all the public ips available if a name is not given.
If a name is provided, the ip of the vm name would be returned.
:param name: name of the VM.
:return: Dictionary of VMs with their public ips
"""
if name is None:
filters = {"$exists": True, "$not": {"$size": 0}}
documents = self.mongo.find('cloud', 'public_ips', filters)
if documents is None:
return None
else:
result = {}
for document in documents:
result[document['name']] = document['public_ips']
return result
else:
public_ips = self.mongo.find_document('cloud', 'name',
name)['public_ips']
if not public_ips:
return None
else:
return {name: public_ips}
def set_public_ip(self, name, public_ip):
"""
Assign the given public ip to the given VM.
:param name: name of the VM
:param public_ip: public ip to be assigned.
"""
if name is not None and public_ip is not None:
self.provider.set_public_ip(name, public_ip)
def remove_public_ip(self, name):
"""
Deletes the public ip of the given VM.
:param name: name of the VM
"""
if name is not None:
self.provider.remove_public_ip(name)
class CommandAWS(object):
def __init__(self):
config = Config().data['cloudmesh']
self.private_key_file = config['cloud']['aws']['credentials']['EC2_PRIVATE_KEY_FILE_PATH']
self.mongo = MongoDB(host=config['data']['mongo']['MONGO_HOST'],
username=config['data']['mongo']['MONGO_USERNAME'],
password=config['data']['mongo']['MONGO_PASSWORD'],
port=config['data']['mongo']['MONGO_PORT'])
def find_node_dns(self, vm_name):
"""
based on AWS design, we need to get the DNS of the required nodes
:param vm_name: the name of vm
:return: the DNS
"""
dns = self.mongo.find_document('cloud', 'name', vm_name).get('extra').get('dns_name')
return dns
@staticmethod
def read_script(script):
"""
read the script file
:param script: the script that would be run in instance
:return content: the content of the script
"""
content = open(script, "r").read()
return content
def run_command(self, command, vm_name):
"""
run command in the vm, and save the job information into MongoDB
:param command: the command string
:param vm_name: the vm name
:return: the result of command
"""
username = '******' + self.find_node_dns(vm_name)
job_id = self.job_start_update_mongo('Null', command, vm_name)
self.update_instance_job_status(vm_name, job_id)
temp = subprocess.check_output(['ssh', '-i', self.private_key_file, username, command]).decode("utf-8")
self.job_end_update_mongo(job_id, temp)
self.update_instance_job_status(vm_name, job_id)
output = 'Running command ' + command + 'in Instance ' + vm_name + ':\n' + temp
return output
def run_script(self, script, vm_name):
"""
run script in the vm, and save the job information into MongoDB
:param script: the raw shell script file
:param vm_name: the vm name
:return: the result of script
"""
username = '******' + self.find_node_dns(vm_name)
content = self.read_script(script)
job_id = self.job_start_update_mongo(content, 'Null', vm_name)
self.update_instance_job_status(vm_name, job_id)
temp = subprocess.check_output(['ssh', '-i', self.private_key_file, username, content]).decode("utf-8")
self.job_end_update_mongo(job_id, temp)
self.update_instance_job_status(vm_name, job_id)
output = 'Running command ' + script + 'in Instance ' + vm_name + ':\n' + temp
return output
def job_start_update_mongo(self, script, command, vm_name):
"""
create new job document in MongoDB, status is processing
:param script: the running script
:param command: the input command
:param vm_name: the vm name
:return: the job document id
"""
job = self.mongo.job_document(vm_name, 'processing', script, 'Null', command, 'single job')
return self.mongo.insert_job_collection(job)
def job_end_update_mongo(self, document_id, output):
"""
jod is done, update the information into job collection in MongoDB, status is done
:param document_id: the job document id
:param output: the result
:return: True/False
"""
var = dict(status='done', output=output)
return self.mongo.update_document('job', '_id', ObjectId(document_id), var)
def update_instance_job_status(self, vm_name, job_id):
"""
for each node, it has its own status. when job is running, it will update its currentJob. When the jon id one,
currentJob is Null, and history will be update
:param vm_name: the name of vm
:param job_id: jod id
"""
status = self.mongo.find_document('status', 'id', vm_name)
if status is None:
document = self.mongo.status_document(vm_name, 'processing', job_id, [])
self.mongo.insert_status_collection(document)
else:
history = self.mongo.find_document('status', 'id', vm_name)['history']
if status['status'] == 'processing':
history.append(str(job_id))
var = dict(status='No Job', currentJob='Null', history=history)
self.mongo.update_document('status', 'id', vm_name, var)
else:
var = dict(status='processing', currentJob=job_id)
self.mongo.update_document('status', 'id', vm_name, var)
def disconnect(self):
"""
disconnect from mongodb
"""
self.mongo.close_client()