def delete_package(name):
banner("CLEAN PREVIOUS {0} INSTALLS".format(name))
r = int(local("pip freeze |fgrep {0} | wc -l".format(name), capture=True))
while r > 0:
local('echo "y" | pip uninstall {0}'.format(name))
r = int(
local("pip freeze |fgrep {0} | wc -l".format(name), capture=True))
def run(self):
auto_create_version("cmd3", version)
os.system("make clean")
os.system("python setup.py install")
os.system("python setup.py bdist_wheel")
banner("Build Distribution")
os.system("python setup.py sdist --format=bztar,zip upload")
def run(self):
auto_create_version("cloudmesh_pbs", version)
auto_create_requirements(requirements)
os.system("make clean")
os.system("python setup.py install")
banner("Build Distribution")
os.system("python setup.py sdist --format=bztar,zip upload")
def test_002(self):
"""flavors"""
HEADING()
banner("get_flavors()")
pprint(self.cloud.get_flavors())
banner("flavors")
pprint(self.cloud.flavors)
def run(self):
banner("Regenerate the python code ")
python_files = ['user', 'project']
for item in python_files:
SourceCode.parse(class_type=item)
def setup_inventory(self):
banner("Read Dicts")
self.sample_user = ConfigDict(filename=config_file("/me.yaml"))
self.portalname = self.sample_user.get("portalname")
print("PORTALNAME", self.portalname)
print("SAMPLE USER", self.sample_user)
banner("create user from template, duplicates cm init generate me")
t = cm_template(config_file("/etc/cloudmesh.yaml"))
pprint(set(t.variables()))
self.config = t.replace(kind="dict", values=self.sample_user)
print(type(self.config))
print(self.config)
#
# BUG?
#
self.idp = cm_userLDAP()
self.idp.connect("fg-ldap", "ldap")
self.idp.refresh()
ldap_info = self.idp.get(self.portalname)
print(ldap_info)
print(type(self.config))
self.config['cloudmesh']['projects'] = ldap_info['projects']
self.config['cloudmesh']['keys'] = ldap_info['keys']
try:
self.config['cloudmesh']['projects'][
'deafult'] = ldap_info['projects']['active'][0]
except:
print("ERROR: you have no projects")
def run(self):
banner("Install Cloudmesh Management")
obj = Mongo()
obj.check_mongo()
get_mongo_db("manage", DBConnFactory.TYPE_MONGOENGINE)
install.run(self)
banner("Adding Super User")
users = Users()
found = User.objects(username='******')
if not found:
data = User(
status="approved",
title="None",
firstname="Super",
lastname="User",
email="*****@*****.**",
username="******",
active=True,
password=sha256_crypt.encrypt("MyPassword"),
phone="555-555-5555",
department="IT",
institution="IU",
institutionrole="Other",
address="IU",
country="United States(US)",
citizenship="United States(US)",
bio="Manage Project Committee",
url="http://cloudmesh.github.io/cloudmesh.html",
advisor="None",
confirm=sha256_crypt.encrypt("MyPassword"),
projects=[],
)
users.add(data)
def test_job_submit(self):
HEADING()
xmkdir(self.manager, "~/scripts/test")
self.pbs.jobid_incr()
banner('qsub')
jobname = "job-" + self.pbs.jobid + ".pbs"
print(jobname)
print (self.host)
script_template = self.pbs.read_script("etc/job.pbs")
r = self.pbs.qsub(jobname, self.host, 'echo "Hello"', template=script_template)
pprint(r)
#banner('variable list')
#pprint(self.pbs.variable_list(r))
assert len(r.keys()) == 1
banner('status')
jobid = self.pbs.getid(r)
print (jobid)
r = self.pbs.jobstatus(self.host, jobid)
print(r)
def boot(auth=True):
# wipe mongo
wipe()
time.sleep(1)
PROGRESS.next()
# start mongo without auth
start(auth=False)
time.sleep(5)
PROGRESS.next()
if isyes(auth):
# create users
admin()
time.sleep(2)
PROGRESS.next()
# restart with auth
kill()
time.sleep(10)
PROGRESS.next()
start(auth=auth)
time.sleep(1)
PROGRESS.next()
config = cm_config_server().get("cloudmesh.server.mongo")
path = path_expand(config["path"])
banner(path)
print Shell.ls(path)
banner("PROCESS")
with settings(warn_only=True):
local("ps -ax | fgrep mongo")
PROGRESS.next()
def execute_command(msg, command, debug):
_capture = not debug
if debug:
banner(msg, debug=debug)
else:
PROGRESS.next()
local(command, capture=_capture)
def delete(self, name, grouponly=False):
"""
deletes the named cluster
::param grouponly: in default, when deleting a cluster, the group object along
with its VMs will be deleted, if grouponly=True, then the
VMs will be preserved
"""
from cloudmesh.experiment.group_usage import remove_vm_from_group_while_deleting
vms = self.vms(name)[name]
if not grouponly:
clouds = []
for vmname, value in vms.iteritems():
cloudname = value['cm_cloud']
id_ = value['id']
banner("Deleting vm->{0} on cloud->{1}".format(vmname, cloudname))
result = self.mongodb.vm_delete(cloudname, id_, self.username)
print(result)
if cloudname not in clouds:
clouds.append(cloudname)
remove_vm_from_group_while_deleting(self.username, vmname)
time.sleep(5)
for cloud in clouds:
self.mongodb.release_unused_public_ips(cloud, self.username)
self.mongodb.refresh(cm_user_id=self.username, types=['servers'])
banner("Deleting group->{0}".format(name))
self.GroupManage.delete_group(name)
def test_gregor(self):
banner("ME")
id = ConfigDict(filename=config_file("/me.yaml")).get("portalname")
user = cm_user()
result = user.info(id)
pprint(result)
pass
def test_007(self):
"""testing cm group list --cloud=juno --format table"""
HEADING()
banner("cm group list --cloud=juno --format=table")
result = run("cm group list --cloud=juno --format=table")
assert "groupA" in result
return
def compile():
for package in cloudmesh_dirs:
banner("Installing and Building: " + package)
if not os.path.isdir("../{0}".format(package)):
local("cd ..; git clone [email protected]:cloudmesh/{0}.git".format(package))
else:
#local("cd ..; git pull")
local("cd ../{0}; git pull".format(package))
def quick(server="server", browser='yes'):
""" starts in dir webgui the program server.py and displays a browser on the given port and link"""
banner("INSTALL CLOUDMESH")
local("python setup.py install")
banner("START WEB SERVER")
local("cd cloudmesh_web; python {0}.py &".format(server))
def test_004(self):
"""testing cm group merge groupA groupB groupC"""
HEADING()
banner("cm group merge groupA groupB groupC")
result = run("cm group merge groupA groupB groupC")
assert "successful!" in result
return
def test_005(self):
"""testing cm group list --cloud=kilo groupA"""
HEADING()
banner("cm group list --cloud=kilo groupA")
result = run("cm group list --cloud=kilo groupA")
assert "groupA" in result
return
def test_006(self):
"""testing cm group list --cloud=kilo --format json groupA"""
HEADING()
banner("cm group list --cloud=kilo --format=json groupA")
result = run("cm group list --cloud=kilo --format=json groupA")
assert "groupA" in result
return
def test_001(self):
"""images"""
HEADING()
banner("get_images()")
pprint(self.cloud.get_images())
banner("images")
pprint(self.cloud.images)
assert True
def test_003(self):
"""testing cm list --cloud trial --user fake default"""
HEADING()
banner("cm list --cloud trial --user fake default")
result = run("cm list --cloud trial --user fake default")
assert "No" in result
return
def all():
"""clean the dis and uninstall cloudmesh"""
dir()
cmd3()
banner("CLEAN PREVIOUS CLOUDMESH INSTALLS")
r = int(local("pip freeze |fgrep cloudmesh | wc -l", capture=True))
while r > 0:
local('echo "y\n" | pip uninstall cloudmesh')
r = int(local("pip freeze |fgrep cloudmesh | wc -l", capture=True))
def test_003(self):
"""testing cm secgroup rules-add --cloud india --tenant fg479 test-group 80 80 tcp 0.0.0.0/0"""
HEADING()
banner("cm secgroup rules-add --cloud india "
"--tenant fg479 test-group 80 80 tcp 0.0.0.0/0")
result = run("cm secgroup rules-add --cloud india "
"--tenant fg479 test-group 80 80 tcp 0.0.0.0/0")
assert "Added rule" in result
def test_002(self):
"""testing cm secgroup list --cloud india --tenant fg479"""
HEADING()
banner("cm secgroup list --cloud india --tenant fg479")
result = run("cm secgroup list --cloud india --tenant fg479")
assert "default" in result
return
def test_003(self):
"""testing cm group copy groupA groupB"""
HEADING()
banner("cm group copy groupA groupB")
result = run("cm group copy groupA groupB")
assert "[groupB]" in result
return
def all():
"""clean the dis and uninstall cloudmesh"""
dir()
cmd3()
banner("CLEAN PREVIOUS CLOUDMESH INSTALLS")
delete_package("cloudmesh")
delete_package("cloudmesh_cmd3")
delete_package("cloudmesh_common")
delete_package("cloudmesh_install")
def test_001(self):
"""testing cm secgroup create --cloud india --tenant fg479 test-group"""
HEADING()
banner("cm secgroup create --cloud india "
"--tenant fg479 test-group")
result = run("cm secgroup create --cloud india "
"--tenant fg479 test-group")
assert "Created a new security group [test-group]" in result
return
def test_003(self):
"""vms"""
HEADING()
banner("get_vms()")
pprint(self.cloud.get_servers())
banner("servers")
print("TTTT")
r = self.cloud.servers
pprint(r)
def test_006(self):
"""cm secgroup rules-delete --cloud india --tenant fg479 test-group 80 80 tcp 0.0.0.0/0"""
HEADING()
banner("cm secgroup rules-delete --cloud india "
"--tenant fg479 test-group 80 80 tcp 0.0.0.0/0")
result = run("cm secgroup rules-delete --cloud india "
"--tenant fg479 test-group 80 80 tcp 0.0.0.0/0")
assert "Rule [80 | 80 | tcp | 0.0.0.0/0] deleted" in result
return
def test_005(self):
"""cm secgroup rules-list --cloud india --tenant fg479 test-group"""
HEADING()
banner("cm secgroup rules-list --cloud india "
"--tenant fg479 test-group")
result = run("cm secgroup rules-list --cloud india "
"--tenant fg479 test-group")
assert "test-group | 80" in result
return
def dir():
"""clean the dirs"""
banner("CLEAN DIR")
local("rm -rf *.egg")
local('find . -name "*~" -exec rm {} \; ')
local('find . -name "*.pyc" -exec rm {} \; ')
local("rm -rf build dist *.egg-info *~ #*")
#local("cd docs; make clean")
local("rm -rf *.egg-info")
local("rm -f celeryd@*")
local("rm -f *.dump")
def insert(self,
job_name,
input="",
output="",
program="",
parameters="",
job_group=None,
job_label=None,
job_id=None, # possibly same as name ?
host=None,
start_time=str(datetime.datetime.now()),
end_time=None,
update_time=str(datetime.datetime.now()),
job_status="C_DEFINED", # see what we wrote in paper
):
"""
inserts a job with specific attributes into the database.
:param name: the name of the job
:param input: an array of input files
:param output: an array of output files
:param program: the executable
:param parameters: an array of parameters
:param job_group: the group of the job
:param job_label: a label for the job
:param job_id: a unique id for the job
:param host: the host on which the job is to be run.
:param start_time: the start time when the job is to be run
:param end_time: the end time when the job is to be run
:param update_time: the time the job record has been updated
:param job_status: the status of the job
:return: returns the job object
"""
if self.database is not None:
job = {
"_id": job_name,
"job_name": job_name,
"program": program,
"job_id": job_name,
"job_group": job_group,
"job_label": job_label,
"job_status": job_status,
"host": host,
"input": input, # must be array
"output": output, # must be array
"start_time": start_time,
"end_time": end_time,
"update_time": update_time,
}
banner("insert job", c=".")
pprint(job)
db_job_object = self.jobs.save(job)
# print (db_job_object)
# db_job_id = db_job_object
# print (db_job_id)
return db_job_object
else:
Console.error("Please connect to the database first")
return -1
def cm_manage():
"""Usage:
cm-manage config projects list
cm-manage config projects
cm-manage config [-f FILE] [-o OUT] [-p PROJECT] cloud NAME [-]
cm-manage config dump [--format=(yaml|dict)]
cm-manage config init [-o OUT] [-u USER]
cm-manage config list
cm-manage config password NAME
cm-manage config show passwords
cm-manage config fetch [-u USER] [-r HOST]
cm-manage --version
cm-manage --help
Arguments:
NAME name of the cloud
Options:
-h --help show this help message and exit
-v --version show version and exit
-f NAME --file=NAME the Name of the cloud to be specified,
if ? a selection is presented
-o OUT --out=OUT writes the result in the specifide file
-p PROJECT --project=PROJECT selects a project (e.g. for eucalyptus
which has project-specific environments)
-u USER --user=USER the user (login) name
-r HOST --remote=HOST the host machine on which the yaml file is
located in the CONFIG directory
[default: india.futuregrid.org]
-d --debug debug
- this option is a - at the end of the command.
If data is written to a file it is also put out to stdout
Description:
Command to generate rc files from our cloudmesh configuration files.
This program generates form a YAML file containing the login
information for a cloud an rc file that can be used to later source
it.
Example:
we assume the yaml file has an entry india-openstack::
cm-manage config -o novarc india-openstack
source novarc
This will create a novarc file and than you can source it::
cm-manage config ? -
Presents a selction of cloud choices and writes the choice into a
file called CONFIG/novarc
"""
default_path = config_file_prefix + '/novarc'
arguments = docopt(cm_manage.__doc__)
DEBUG("arguments", arguments)
home = os.environ['HOME']
# DEBUG("home", home)
#
# This secion deals with handeling "cm config" related commands
######################################################################
is_config = arguments['config'] is not None
if is_config:
# DEBUG('Arguments', arguments)
file = arguments['--file']
try:
config = cm_config(file)
# DEBUG("config", config)
except IOError:
print("{0}: Configuration file '{1}' not found".format("CM ERROR", file))
sys.exit(1)
except (yaml.scanner.ScannerError, yaml.parser.ParserError) as yamlerror:
print("{0}: YAML error: {1}, in configuration file '{2}'".format("CM ERROR", yamlerror, file))
sys.exit(1)
except:
print("Unexpected error:", sys.exc_info()[0])
sys.exit(1)
name = arguments['NAME']
#
# NOT TESTED
#
if arguments['fetch'] or name == 'fetch':
DEBUG('Arguments', arguments)
# get user
var = {}
var['user'] = arguments['--user']
var['host'] = arguments['--remote']
#
# BUG should be
#
var['file'] = config_file_prefix() + "/cloudmesh.yaml"
if var['user'] is None:
var['user'] = getpass.getuser()
from_location = "%(user)s@%(host)s:%(file)s" % var
to_location = config_file("/%(file)s" % var)
if os.path.isfile(to_location):
print("WARNING: The file %s exists" % to_location)
if not yn_choice("Would you like to replace the file", default='y'):
sys.exit(1)
print(from_location)
print(to_location)
print("Copy cloudmesh file from %s to %s" % (from_location, to_location))
result = scp(from_location, to_location)
print(result)
sys.exit(0)
#
# ok
#
# if (arguments['projects'] and arguments['list']) :
if arguments['projects'] and arguments['list']:
projects = config.get('cloudmesh.projects')
print(yaml.dump(projects, default_flow_style=False, indent=4))
sys.exit(0)
#
# OK, needs setting
#
if arguments['projects']:
projects = config.projects('active')
print("Please select from the following:")
print("0 - Cancel")
selected = False
choices = []
while not selected:
counter = 1
for name in projects:
print(counter, "-" "%20s" % name)
choices.append(name)
counter += 1
print("Please select:")
input = int(sys.stdin.readline())
if input == 0:
print("Selection terminated")
sys.exit(0)
selected = (input > 0) and (input < counter)
print("Selected: ", input)
name = choices[input - 1]
print(name)
print("ERROR: THIS JUST SELECTS A PROJECT ID BUT DOES NOT SET IT")
sys.exit(0)
if arguments['init'] or name == 'init':
output = arguments['--out']
username = arguments['--user'] or os.getenv('USER')
location = path_expand(output)
new_yaml_file = open(location, 'w+')
user_yaml = cm_user().generate_yaml(username, 'cloudmesh')
print(yaml.dump(
user_yaml, default_flow_style=False), file=new_yaml_file)
new_yaml_file.close()
print("Written new yaml file in " + location)
sys.exit(0)
#
# OK
#
if arguments['list'] or name == 'list':
for name in config.cloudnames():
if 'cm_type' in config.cloud(name):
print(name, "(%s)" % config.cloud(name)['cm_type'])
sys.exit(0)
#
# NOT TESTED
#
if arguments['password']:
oldpass = getpass("Current password: "******"New password: "******"New password (again): ")
# TODO: some kind of password strength checking?
if newpass1 == newpass2:
config.change_own_password(name, oldpass, newpass1)
else:
print("New passwords did not match; password not changed.")
sys.exit(0)
#
# OK, but does not display the username
#
if arguments['show'] or name == 'show' and arguments['passwords']:
warning = "Your passwords will appear on the screen. Continue?"
if yn_choice(warning, 'n'):
me = ConfigDict(filename=config_file("/me.yaml"))
banner("PASSWORDS")
for name in me['password']:
print("{0}: {1}".format(name, me['password'][name]))
sys.exit(0)
#
# OK
#
if arguments['dump'] or name == 'dump':
format = arguments['--format']
if format == 'yaml':
d = dict(config)
print(yaml.dump(d, default_flow_style=False))
elif format == 'dict' or format is None:
print(config)
sys.exit(0)
#
# NOT TESTED
#
if name in ['?', 'x']:
if file is None:
arguments['--out'] = "%s/%s" % (home, default_path)
print("Please select from the following:")
print("0 - Cancel")
selected = False
choices = []
while not selected:
counter = 1
for name in config.cloudnames():
if 'cm_type' in config.cloud(name):
print("{0} - {1:<30} ({2})".format(counter, name, config.cloud(name)['cm_type']))
choices.append(name)
counter += 1
print("Please select:")
input = int(sys.stdin.readline())
if input == 0:
print("Selection terminated")
sys.exit(0)
selected = (input > 0) and (input < counter)
print("Selected: ", input)
name = choices[input - 1]
output = arguments['--out']
#
# OK
#
if name is not None:
cloud = config.cloud(name)
if not cloud:
print("%s: The cloud '%s' is not defined." % ("CM ERROR", name))
print("Try instead:")
for keyname in config.cloudnames():
print(" ", keyname)
sys.exit(1)
if cloud['cm_type'] == 'eucalyptus':
if arguments['--project']:
project = arguments['--project']
if not project in cloud:
print("No such project %s defined in cloud %s." % (project, name))
sys.exit(1)
else:
project = config.cloud_default(
name, 'project') or config.projects('default')
if not project in cloud:
print("Default project %s not defined in cloud %s." % (project, name))
sys.exit(1)
rc_func = lambda name: config.rc_euca(name, project)
else:
rc_func = config.rc
result = rc_func(name)
#
# OK
#
if arguments["-"]:
print(result)
else:
if output is None:
arguments['--out'] = "%s/%s" % (home, default_path)
output = arguments['--out']
out = False
if yn_choice("Would you like to review the information", default="y"):
banner("WARNING: FIle will be written to " + output)
print(result)
print(banner(""))
try:
# First we try to open the file assuming it doesn't exist
out = os.open(
output, os.O_CREAT | os.O_EXCL | os.O_WRONLY, stat.S_IRUSR | stat.S_IWUSR)
except OSError as oserr:
# If file exists, offer to replace it
if oserr.errno == 17:
delete = raw_input(
"'%s' exists; Overwrite it [N|y]? " % output)
if delete.strip().lower() == 'y':
out = os.open(output, os.O_TRUNC | os.O_WRONLY)
if out:
os.write(out, result)
os.close(out)
def delete_vm(username,
cloudname,
server_id_list=None,
preview=False,
refresh=False):
'''
delete vms of a cloud for a user
this function accepts a list of VM ids and delete them
TODO: what if fail to delete, how to acknowledge it;
it looks like even though delete a vm and return {msg: seccess},
sometimes refresh after 5 sec, it might be still there
:param username: user name
:type username: string
:param cloudname: cloud name
:type cloudname: string
:param server_id_list: the list of VM ids to delete
:type server_id_list: list
:param preview: True if the user wants to preview and confirm before start
to delete
:type preview: boolean
:param refresh: True to refresh the database before processing, the database
has to be refreshed before start processing here. Since
there are many
places that refreshes the database, so if there is a place
in the program refreshes shortly before calling this
funtion, to avoid redundant work, make this argument False
:type refresh: boolean
:return: False if error occurs
:rtype: boolean
'''
# changed the scope of this import
# Benefit: other functions are not affected with this import
# drawback: hard to see which module is being loaded in this file
# Hyungro Lee 12/01/2014
from cloudmesh.experiment.group_usage import remove_vm_from_group_while_deleting
try:
mongo = cm_mongo()
except:
Console.error("There is a problem with the mongo server")
return False
if refresh:
mongo.activate(cm_user_id=username, names=[cloudname])
mongo.refresh(cm_user_id=username,
names=[cloudname],
types=['servers'])
serverdata = mongo.servers(
clouds=[cloudname], cm_user_id=username)[cloudname]
# -------------------------
# preview and confirm
confirm_deletion = True
if preview:
if server_id_list == []:
Console.warning("no vm meets the condition")
return False
else:
resserverdata = {}
for i in server_id_list:
resserverdata[i] = serverdata[i]
cloudobj = CloudManage()
itemkeys = {"openstack":
[
['name', 'name'],
['status', 'status'],
['addresses', 'addresses'],
['id', 'id'],
['flavor', 'flavor', 'id'],
['image', 'image', 'id'],
['user_id', 'cm_user_id'],
['metadata', 'metadata'],
['key_name', 'key_name'],
['created', 'created'],
['cloud', 'cm_cloud']
],
"ec2":
[
["name", "id"],
["status", "extra", "status"],
["addresses", "public_ips"],
["flavor", "extra", "instance_type"],
['id', 'id'],
['image', 'extra', 'imageId'],
["user_id", 'user_id'],
["metadata", "metadata"],
["key_name", "extra", "key_name"],
["created", "extra", "launch_time"]
],
"aws":
[
["name", "name"],
["status", "extra", "status"],
["addresses", "public_ips"],
["flavor", "extra", "instance_type"],
['id', 'id'],
['image', 'extra', 'image_id'],
["user_id", "user_id"],
["metadata", "metadata"],
["key_name", "extra", "key_name"],
["created", "extra", "launch_time"]
],
"azure":
[
['name', 'name'],
['status', 'status'],
['addresses', 'vip'],
['flavor', 'flavor', 'id'],
['id', 'id'],
['image', 'image', 'id'],
['user_id', 'user_id'],
['metadata', 'metadata'],
['key_name', 'key_name'],
['created', 'created'],
]
}
cloudobj.print_cloud_servers(username=username,
cloudname=cloudname,
itemkeys=itemkeys,
refresh=False,
output=False,
serverdata=resserverdata)
if yn_choice("confirm to delete these vms?", default='n', tries=3):
pass
else:
confirm_deletion = False
# -------------------------
# deleting
if confirm_deletion:
if server_id_list == []:
return
watch = time.time()
useQueue = False
if useQueue:
# not functioning
cloudmanager = mongo.clouds[username][cloudname]["manager"]
cm_type = mongo.get_cloud_info(username, cloudname)['cm_type']
package = "cloudmesh.iaas.%s.queue" % cm_type
name = "tasks"
imported = getattr(__import__(package, fromlist=[name]), name)
queue_name = "%s-%s" % (cm_type, "servers")
for i in server_id_list:
tempservername = serverdata[i]['name'].encode("ascii")
banner(
"Deleting vm->{0} on cloud->{1}".format(tempservername, cloudname))
result = imported.vm_delete.apply_async(
(cloudname, i, username), queue=queue_name)
print("job status:", result.state)
try:
remove_vm_from_group_while_deleting(username, tempservername)
except Exception, err:
Console.error(str(err))
return
# print result.traceback #########
imported.wait.apply_async(
args=None, kwargs={'t': 10}, queue=queue_name)
handleip = imported.release_unused_public_ips.apply_async(
(cloudname, username), queue=queue_name)
handlerefresh = imported.refresh.apply_async(args=None,
kwargs={'cm_user_id': username,
'names': [cloudname],
'types': ['servers']},
queue=queue_name)
# print handleip.state
# print handleip.traceback
# print handlerefresh.state
# print handlerefresh.traceback
if preview:
print("to check realtime vm status: list vm --refresh")
else:
for i in server_id_list:
tempservername = serverdata[i]['name'].encode("ascii")
banner(
"Deleting vm->{0} on cloud->{1}".format(tempservername, cloudname))
result = mongo.vm_delete(cloudname, i, username)
pprint(result)
try:
remove_vm_from_group_while_deleting(username, tempservername)
except Exception, err:
Console.error(str(err))
return
time.sleep(5)
mongo.release_unused_public_ips(cloudname, username)
mongo.refresh(username, names=[cloudname], types=['servers'])
def run(self):
banner("Installing Requirements for Cloudmesh " + package_name)
os.system("pip install -r requirements.txt")
banner("Installing Cloudmesh " + package_name)
install.run(self)
def run(self):
banner("Register with Pypi")
# os.system("python shell_plugins.py register")
print ("not implemented")
def run(self):
banner("Register with Pypi")
os.system("python setup.py.in register")
import os
package_name = "cloudmesh_spark"
try:
from cloudmesh_base.util import banner
except:
os.system("pip install cloudmesh_base")
from cloudmesh_base.util import banner
from cloudmesh_base.util import path_expand
from cloudmesh_base.Shell import Shell
from cloudmesh_base.util import auto_create_version
from cloudmesh_base.util import auto_create_requirements
banner("Installing Cloudmesh " + package_name)
home = os.path.expanduser("~")
auto_create_version(package_name, version)
auto_create_requirements(requirements)
class UploadToPypi(install):
"""Upload the package to pypi."""
def run(self):
os.system("Make clean Install")
os.system("python setup.py.in install")
banner("Build Distribution")
os.system("python setup.py.in sdist --format=bztar,zip upload")
try:
import cloudmesh_base
print("Using cloudmesh_base version:", cloudmesh_base.__version__)
except:
# os.system("pip install cloudmesh_base")
os.system("pip install git+https://github.com/cloudmesh/base.git")
from cloudmesh_base.util import banner
from cloudmesh_base.util import path_expand
from cloudmesh_base.Shell import Shell
from cloudmesh_base.util import auto_create_version
from cloudmesh_base.setup import parse_requirements, os_execute, get_version_from_git
version = get_version_from_git()
banner("Installing Cmd3 {:}".format(version))
requirements = parse_requirements('requirements.txt')
auto_create_version("cmd3", version, filename="version.py")
class UploadToPypi(install):
"""Upload the package to pypi. -- only for Maintainers."""
description = __doc__
def run(self):
auto_create_version("cmd3", version, filename="version.py")
os.system("make clean")
commands = """
def _start_mongo(self):
"""
start the mongodb service in the location as specified in
cloudmesh_server.yaml
"""
banner("Starting mongod")
config = cm_config_server().get("cloudmesh.server.mongo")
path = path_expand(config["path"])
port = config["port"]
# pprint(config)
# print(path)
# print(port)
banner("creating dir")
if not os.path.exists(path):
print("Creating mongodb directory in {0}".format(path))
sh.mkdir("-p", path)
banner("check")
# lines = str(sh.grep(sh.ps("-ax"), "mongod", "*port {0}".format(port)))
# need to get rid of fabric local later
with settings(warn_only=True):
with hide('output', 'running', 'warnings'):
lines = local(
"ps -ax |grep '[m]ongod.*port {0}'".format(port), capture=True) \
.split("\n")
print("search result:")
# print(type(lines))
print(lines)
if lines != ['']:
pid = lines[0].split(" ")[0]
print("NO ACTION: mongo already running in pid "
"{0} for port {1}".format(pid, port))
return
print("ACTION: Starting mongod")
print()
print("NOTE: the preparation of mongo may take a few minutes")
print(" please do not interrupt this program.")
print()
print(" Please be patient!")
print()
'''
Shell.mongod("--auth",
"--bind_ip", "127.0.0.1"
"--fork",
"--dbpath", path,
"--logpath", "{0}/mongodb.log".format(path),
"--port", port,
_bg=True)
'''
# need to get rid of fabric local later
local(
'mongod --auth --bind_ip 127.0.0.1 '
'--fork --dbpath {0} '
'--logpath {0}/mongodb.log '
'--port {1}'.format(path, port))
def test_qinfo(self):
for host in self.hosts:
banner(host)
self.get_qinfo(host)
def _start_web_server(self):
# from cloudmesh_web import server as cloudmesh_web_server_start
banner("start the web server")
os.system("cd cloudmesh_web; python server.py &")
time.sleep(4)
def install_command(args):
"""
Usage:
install -h | --help
install --version
install cloudmesh [--force]
install delete_yaml
install system
install query
install new [--force]
install vagrant
install enable admin [--username=<username>]
"""
# This is a debuging message
# print "IIIII<" + args + ">"
arguments = docopt(install_command.__doc__, args)
# print arguments
if arguments["cloudmesh"]:
force = arguments['--force']
deploy(force=force)
elif arguments["new"]:
force = arguments["--force"]
new_cloudmesh_yaml(force)
elif arguments["delete_yaml"]:
answer = yn_choice(
"THIS COMMAND IS REAL DANGEROUS AND WILL DELETE ALL YOUR YAML FILE. Proceed",
default='y')
if answer:
# TODO: cp is not imported, defined
print("You fool we just deleted your yaml files")
cp("etc/*.yaml", config_file_prefix())
else:
print "puuh you interrupted"
pass
elif arguments["system"]:
banner("Installing Ubuntu System Requirements")
if is_ubuntu():
ubuntu()
elif is_osx():
osx()
elif is_centos():
centos()
elif arguments["query"]:
import platform
print "System: ", platform.system()
# print "Uname: ", platform.uname()
print "Machine: ", platform.machine()
print "Processor: ", platform.processor()
print "Platform: ", platform.platform()
print "Python: ", platform.python_version()
print "Virtualenv:", hasattr(sys, 'real_prefix')
elif arguments["vagrant"]:
vagrant()
elif arguments["enable"] and arguments["admin"]:
enable_admin_page(arguments['--username'])
'''
class CredentialFromMongo(UserBaseClass):
def __init__(self, user, cloud, datasource=None):
"""data source is a collectionname in cloudmesh_server.yaml"""
"""if day=tasource is none than use the default on which is ?"""
raise NotImplementedError()
'''
if __name__ == "__main__":
# -------------------------------------------------------------------------
banner("YAML read test")
# -------------------------------------------------------------------------
user = UserFromYaml("gvonlasz")
pprint(user)
store = UserDictStore(UserFromYaml, password="******")
store.set("gvonlasz", "~/.cloudmesh/cloudmesh.yaml", password="******")
banner("credentialstore")
pprint(store)
# banner("gvonlasz - hp")
# -------------------------------------------------------------------------
# cred = store.credential("gvonlasz", "hp", password="******")
# pprint (cred)
for email in emails:
stats[email] = {'percentage': [
stats[email]["fileschanged"] / float(sums["fileschanged"]),
stats[email]["inserted"] / float(sums["inserted"]),
stats[email]["deleted"] / float(sums["deleted"]),
stats[email]["lineschanged"] / float(sums["lineschanged"])
]}
return stats
if __name__ == "__main__":
gitinfo = GitInfo()
# print gitinfo.version()
banner("a")
print(gitinfo.authors())
banner("b")
pprint(gitinfo.authors("dict"))
banner("c")
pprint(gitinfo.emails())
banner("d")
pprint(gitinfo.emails("dict"))
banner("e")
pprint(gitinfo.info())
banner("f")
group='group1',
user='******',
state='defined',
ssh_config='india',
queue=queue)
job.save()
# print (Job.status(name))
# jobs = Job.objects()
Job.list()
Job.info(name)
j = Job.find(name)
j.name = 'a'
j.save()
Job.list()
# # BUG
# Job.rename(name, 'job2')
# Job.info('job2')
banner("PBS")
pbs = PBS("gvonlasz", "india.futuregrid.org")
# pprint (pbs.qinfo())
pprint(pbs.qstat())
def install():
sphinx_updates()
banner("cloudmesh python install")
local("python setup.py install")
def run(self):
os.system("Make clean Install")
os.system("python setup.py.in install")
banner("Build Distribution")
os.system("python setup.py.in sdist --format=bztar,zip upload")
def fetchrc(userid=None, outdir=None):
banner("download rcfiles (novarc, eucarc, etc) from IaaS platforms")
print ""
# Task 1. list portal user id
'''
try:
from cloudmesh_base.ConfigDict import ConfigDict
except Exception, e:
print "ERROR: your have not yet configured cloudmesh completely. "
print " Have you called"
print
print " ./install cloudmesh"
print
sys.exit(1)
dir = config_file("")
config = ConfigDict(dir + "/me.yaml")
userid = config["portalname"]
'''
if not userid:
userid = getpass.getuser()
userid = raw_input("Please enter your portal user id [default: %s]: " %
userid) or userid
# Task 2. list hostnames to get access. In FutureSystems, india is default
# TEMPORARY
host_ids = ["india"]
# user input is disabled
# host_ids = raw_input("Please enter host identifications [default: %s]: "
# % ", ".join(host_ids)) or host_ids
if isinstance(host_ids, str):
host_ids = map(lambda x: x.strip(), host_ids.split(","))
# domain_name = ".futuregrid.org"
# hostnames = map(lambda x: x.split("_")[0] + domain_name, host_ids)
# key_path = "~/.ssh/id_rsa"
# private key path is disabled
# key_path = raw_input("Please enter a path of the ssh private key to" + \
# " login the hosts [default: %s]: " % key_path) or \
# key_path
try:
download_rc_files(userid)
update_permission("~/.cloudmesh/clouds")
"""
# cmd = "fab rcfile.download:userid='%s',host_ids='%s',key_path='%s'" \
cmd = "fab -H %s -u %s -i %s rcfile.download:'%s','%s'" \
% (",".join(hostnames), userid, key_path,
"\,".join(host_ids), outdir)
print "CMD", cmd
os.system(cmd)
"""
except:
print sys.exc_info()
sys.exit(1)
def run(self):
banner("Installing Cloudmesh " + package_name)
install.run(self)
def shell_commands_dict_output(
username,
d,
# choose format manually
print_format=None,
# choose table format if needed
table_format=None,
# more specific table arguments
# --------------------------
# for table format: "key_list"
indexed=False,
# --------------------------
firstheader=None,
header=None,
oneitem=False,
vertical_table=False,
title=None,
count=False):
'''
some shell commands output a dict or table, this function should be
called when commands are printing in such way.
if none of parameters json and table is given, the funtion will try
to find the default printing form from db_defaults
if db_defaults has no item 'shell_print_format', the function will
set 'shell_print_format' = table
param username:: user id
param d:: data to print
param print_format:: print format: table, json, csv
param table_format:: choose table format if needed, DESCRIPTIONS:
type1: key_list:
accept a dict in the form:
{key1: [list1],
key2: [list2],
.......
=>
| key1 | key2 |
| l
| i
| s
| t
acceptable args: indexed: provide index
param firstheader:: designed for table, provide a attribute name for the
first item of each row, since the dict doesn't provide
it
param header:: designed for table(and used to filter input dict), a list of lists,
provides column order, e.g.
[[a,b], [c, d], ...
where 'a' is the printing column name and 'b' is the attribute name
in the dict
if you don't want to change the header name but want to keep the
header order or filter the items to list, for each item in the list
you may provide a string or a list with one item instead of a list of
two items
param oneitem:: designed for table, normally the input dict should be in such
form:
{a: {...}, b:{...}}, where each subitem is a row
if there is only one item, input dict is the {...} of the subitem
above
param title: provide a title for the table
param count: provide count info at the end of the table
'''
format_type = None
if print_format:
format_type = print_format
else:
format_type = get_default_print_format(username)
if format_type not in ALLOWED_PRINT_FORMAT:
Console.error(
"wrong print format: {0}. (allowed print format: {1})".format(
format_type, ", ".join(ALLOWED_PRINT_FORMAT)))
return False
headers = None
order = None
if header:
headers = []
order = []
for i in header:
if isinstance(i, basestring):
headers.append(i)
order.append(i)
elif isinstance(i, list):
if len(i) == 1:
headers.append(i[0])
order.append(i[0])
else:
headers.append(i[0])
order.append(i[1])
else:
print("ERROR: header info is not correct")
return False
# --------------------------------------------------------------------------
# filter the input dict
# --------------------------------------------------------------------------
if order:
new_d = {}
if oneitem:
for item in order:
if item in d:
new_d[item] = d[item]
else:
for i in d:
new_d[i] = {}
for item in order:
if item in d[i]:
new_d[i][item] = d[i][item]
d = new_d
# --------------------------------------------------------------------------
if format_type == "json":
if title:
banner(title)
print(json.dumps(d, indent=4))
elif format_type == "csv":
with open(".temp.csv", "wb") as f:
w = csv.DictWriter(f, d.keys())
w.writeheader()
w.writerow(d)
elif format_type == "table":
if table_format == "key_list":
print(dict_key_list_table_printer(d, indexed=indexed))
return
if title:
print("+" + "-" * (len(title) - 2) + "+")
print(title)
print_data = []
if oneitem:
print_data = [d]
else:
for k in sorted(d):
d[k][' '] = k
print_data.append(d[k])
if header:
if firstheader:
headers = [firstheader] + headers
order = [' '] + order
if vertical_table:
print(
array_dict_table_printer(print_data,
order=order,
header=headers,
vertical=True))
else:
print(
array_dict_table_printer(print_data,
order=order,
header=headers))
if count:
c = len(print_data)
sentence = "count: {0}".format(c)
print(sentence)
print("+" + "-" * (len(sentence) - 2) + "+")
from cloudmesh_base.util import auto_create_version
from cloudmesh_base.util import path_expand
from cloudmesh_base.setup import parse_requirements, os_execute, get_version_from_git, check_pip
import shutil
import sys
from cloudmesh_base.gitinfo import GitInfo
check_pip()
version = get_version_from_git()
requirements = parse_requirements('requirements.txt')
banner("Installing Cloudmesh Base")
home = os.path.expanduser("~")
#
# MANAGE VERSION NUMBER
#
auto_create_version("cloudmesh_base", version, "version.py")
# banner("Install Cloudmesh Base Requirements")
# os.system("pip install -r requirements.txt")
class Make(object):
# values = yaml.safe_load(Template(result).substitute(os.environ))
# print json.dumps(values, indent=4)
except Exception, e:
print "ERROR: There is an error in the yaml file", e
sys.exit(1)
for cloud in values['clouds']:
values['clouds'][cloud]['default'] = {}
values['clouds'][cloud]['default']['image'] = None
values['clouds'][cloud]['default']['flavor'] = None
file_from_template(cloudmesh_template, cloudmesh_out, values)
print "# Created: {0}".format(me_file)
banner(c="-")
# sys.exit()
#
# format = "yaml"
# if format in ["json"]:
# result = json.dumps(values, indent=4)
# elif format in ["yaml", "yml"]:
# result = yaml.dump(values, default_flow_style=False)
# banner("done", c="-")
# print "# Template: {0}".format(filename_template)
# print "# Values : {0}".format(filename_values)
# print "# Backup : {0}".format(filename_bak)
def run(self):
banner("Requirements")
Make.install_requirements()
banner("Install Cloudmesh Base")
# os.system("pip install pip -U")
install.run(self)
def cmd3():
banner("CLEAN CMD3")
local("rm -rf {0}".format(config_file("/cmd3local")))
script = """
uname -a
"""
jobname = "job-" + pbs.jobid
job_script = pbs.create_script(jobname, script, script_template)
print(job_script)
# print(pbs.jobid)
# pbs.jobid_set(100)
# print(pbs.jobid)
pbs.jobid_incr()
banner('qsub')
jobname = "job-" + pbs.jobid + ".pbs"
r = pbs.qsub(jobname, host, 'echo "Hello"', template=script_template)
pprint(r)
banner('variable list')
pprint(OpenPBS.variable_list(r))
banner('status')
jobid = pbs.getid(r)
r = pbs.jobstatus(host, jobid)
pprint(r)
r = pbs.qstat("india")
credentials = get_credentials(hosts)
task = {}
watch = StopWatch()
for execute in [Sequential, Parallel]:
# for execute in [Sequential]:
# for execute in [Parallel]:
name = execute.__name__
banner(name)
watch.start(name)
result = execute(credentials, cm_ssh, command="qstat")
watch.stop(name)
pprint(result)
banner("PRINT")
for host in result:
print result[host]["output"]
for timer in watch.keys():
print timer, watch.get(timer), "s"
def start_vm(username,
cloudname,
count=1,
flavorname=None,
flavorid=None,
imagename=None,
imageid=None,
groupname=None,
servername=None):
'''
create a VM on a cloud for a user
If flavor or image is provided, the function will check the flavor, image
existence;
it's better to check cloud active status before use this function
TODO: this function should be relocated into a class;
what if fail to create, how to acknowledge it. no return now because
of using celery;
missing security group
:param username: user name
:type username: string
:param cloudname: cloud name
:type cloudname: string
:param count: give the number of vms to start at one call of this function,
the default is 1
:type count: int
:param flavorname: flavor name, if flavor name or id is not provided, the
function will try to find the user specified default
flavor
:type flavorname: string
:param flavorid: flavor id, if flavor name or id is not provided, the
function will try to find the user specified default
flavor
:type flavorid: string
:param imagename: image name, if image name or id is not provided, the
function will try to find the user specified default
image
:type imagename: string
:param imageid: image id, if image name or id is not provided, the
function will try to find the user specified default
image
:type imageid: string
:param groupname: user can specify groupname of the VM, this groupname will
be included in the metadata of the VM
:type groupname: string
:param servername: user can provide the name of the VM, which will replace
the default prefix+index form
:type servername: string
:return: False if error occurs
:rtype: boolean
'''
# Changed scope of this import - hyungro lee 12/01/2014
from cloudmesh.experiment.group_usage import add_vm_to_group_while_creating
mongo = cm_mongo()
userobj = cm_user()
cloudobj = CloudManage()
mongo.activate(cm_user_id=username, names=[cloudname])
userinfo = userobj.info(username)
key = None
vm_image_id = None
vm_flavor_id = None
error = ''
# -------------------------
# refresh server flavor or image
to_refresh = ["servers"]
if flavorname is not None or flavorid is not None:
to_refresh.append("flavors")
if imagename is not None or imageid is not None:
to_refresh.append("images")
if to_refresh != []:
mongo.refresh(username, names=[cloudname], types=to_refresh)
# -------------------------
# get exist VM names list, to prevent names duplicate
serverdata = mongo.servers(
clouds=[cloudname],
cm_user_id=username)[cloudname]
servers_names_list = []
for k, v in serverdata.iteritems():
servers_names_list.append(v['name'])
# -------------------------
# flavor handler
if flavorname is not None or flavorid is not None:
flavordata = mongo.flavors(
clouds=[cloudname], cm_user_id=username)[cloudname]
same_name_count = 0
for k, v in flavordata.iteritems():
if flavorname is not None:
if flavorname == v['name']:
vm_flavor_id = k
same_name_count = same_name_count + 1
else:
if flavorid == k:
vm_flavor_id = k
break
if vm_flavor_id is None:
error = error + "The flavor you provide doesn't exist. "
if same_name_count > 1:
error = error + "There are more than one flavor with the name you provide" \
"please use flavorid instead or select one by command cloud" \
"set flavor [CLOUD]. "
else:
try:
vm_flavor_id = userinfo["defaults"]["flavors"][cloudname]
except:
pass
if vm_flavor_id in [None, 'none']:
error = error + \
"Please specify a default flavor(command: cloud set flavor [CLOUD]). "
# -------------------------
# image handler
if imagename is not None or imageid is not None:
imagedata = mongo.images(
clouds=[cloudname], cm_user_id=username)[cloudname]
same_name_count = 0
for k, v in imagedata.iteritems():
if imagename is not None:
if imagename == v['name']:
vm_image_id = k
same_name_count = same_name_count + 1
else:
if imageid == k:
vm_image_id = k
break
if vm_image_id is None:
error = error + "The image you provide doesn't exist. "
if same_name_count > 1:
error = error + "There are more than one image with the name you provide" \
"please use imageid instead or select one by command cloud" \
"set image [CLOUD]. "
else:
try:
vm_image_id = userinfo["defaults"]["images"][cloudname]
except:
pass
if vm_image_id in [None, 'none']:
error = error + \
"Please specify a default image(command: cloud set flavor [CLOUD]). "
# -------------------------
# key handler
if "key" in userinfo["defaults"]:
key = userinfo["defaults"]["key"]
elif len(userinfo["keys"]["keylist"].keys()) > 0:
key = userinfo["keys"]["keylist"].keys()[0]
if key:
keycontent = userinfo["keys"]["keylist"][key]
if keycontent.startswith('key '):
keycontent = keycontent[4:]
cm_keys_mongo(username).check_register_key(username, cloudname, key, keycontent)
keynamenew = _keyname_sanitation(username, key)
else:
error = error + \
"No sshkey found. Please Upload one"
# -------------------------
if error != '':
Console.error(error)
return False
# -------------------------
metadata = {'cm_owner': username}
if groupname:
metadata['cm_group'] = groupname
tmpnamefl = cloudobj.get_flavors(
cloudname=cloudname, getone=True, id=vm_flavor_id)['name']
tmpnameim = cloudobj.get_images(
cloudname=cloudname, getone=True, id=vm_image_id)['name']
while count > 0:
userinfo = userobj.info(username)
if servername:
prefix = ''
index = ''
givenvmname = servername
tmpnameser = servername
else:
prefix = userinfo["defaults"]["prefix"]
index = userinfo["defaults"]["index"]
givenvmname = None
tmpnameser = prefix + '_' + str(index)
# ------------------------
# do not allow server name duplicate
if tmpnameser in servers_names_list:
Console.error("vm name '{0}' exists, please "
"use other names or delete it first".format(tmpnameser))
if not servername:
userobj.set_default_attribute(username, "index", int(index) + 1)
count = count - 1
continue
# ------------------------
# vm start procedure
banner("Starting vm->{0} on cloud->{1} using image->{2}, flavor->{3}, key->{4}"
.format(tmpnameser, cloudname, tmpnameim, tmpnamefl, keynamenew))
useQueue = False
if useQueue:
result = mongo.vm_create_queue(cloudname,
prefix,
index,
vm_flavor_id,
vm_image_id,
keynamenew,
meta=metadata,
cm_user_id=username,
givenvmname=givenvmname)
print("job status:", result.state)
else:
result = mongo.vm_create(cloudname,
prefix,
index,
vm_flavor_id,
vm_image_id,
keynamenew,
meta=metadata,
cm_user_id=username,
givenvmname=givenvmname)
if "server" in result and "adminPass" in result["server"]:
result["server"]["adminPass"] = "******"
pprint(result)
# ------------------------
# add it to the group in database if groupname provided
if groupname:
try:
add_vm_to_group_while_creating(username, groupname, tmpnameser)
except Exception, err:
Console.error(str(err))
return
# ------------------------
# increase index if it is used
if not servername:
userobj.set_default_attribute(username, "index", int(index) + 1)
# ------------------------
servers_names_list.append(tmpnameser)
count = count - 1
if __name__ == "__main__":
qsub = False
db = DbPBS()
db.clear()
db.info()
db.update(host="india", user=False)
print(db.list(output="table"))
print(db.list(output="csv"))
print(db.list(output="dict"))
print(db.list(output="yaml"))
banner("user")
db.clear()
db.update(host="india")
print(db.list(output="table"))
if qsub:
banner('qsub')
pbs = OpenPBS()
jobname = "job-" + pbs.jobid + ".pbs"
host = "india"
script_template = pbs.read_script("etc/job.pbs")
print(script_template)
r = db.qsub(jobname, host, 'echo "Hello"', template=script_template)
import os
try:
import cloudmesh_base
print("Using cloudmesh_base version:", cloudmesh_base.__version__)
except:
# os.system("pip install cloudmesh_base")
os.system("pip install git+https://github.com/cloudmesh/base.git")
from cloudmesh_base.util import banner
from cloudmesh_base.setup import os_execute
from cloudmesh_portal.version import __version__
banner("Installing Cloudmesh_portal {:}".format(__version__))
requirements = [
"cloudmesh_client", "cloudmesh_workflow", "jinja2schema",
"django-rest-swagger", "django-bootstrap3", "django-admin-bootstrapped",
"django-bootstrap-themes", "django-jinja", "djangorestframework",
"markdown", "django-filter", "aldjemy", "nwdiag", "pygal"
]
class UploadToPypi(install):
"""Upload the package to pypi. -- only for Maintainers."""
description = __doc__
def run(self):
def test_all(self):
HEADING()
for host in self.hosts:
banner(host)
self.get_qstat(host)
