def delete_package(name): banner("CLEAN PREVIOUS {0} INSTALLS".format(name)) r = int(local("pip freeze |fgrep {0} | wc -l".format(name), capture=True)) while r > 0: local('echo "y" | pip uninstall {0}'.format(name)) r = int( local("pip freeze |fgrep {0} | wc -l".format(name), capture=True))
def execute_command(msg, command, debug): _capture = not debug if debug: banner(msg, debug=debug) else: PROGRESS.next() local(command, capture=_capture)
def quick(server="server", browser='yes'): """ starts in dir webgui the program server.py and displays a browser on the given port and link""" banner("INSTALL CLOUDMESH") local("python setup.py install") banner("START WEB SERVER") local("cd cloudmesh_web; python {0}.py &".format(server))
def all(): """clean the dis and uninstall cloudmesh""" dir() cmd3() banner("CLEAN PREVIOUS CLOUDMESH INSTALLS") delete_package("cloudmesh") delete_package("cloudmesh_cmd3") delete_package("cloudmesh_common") delete_package("cloudmesh_install")
def all(): """clean the dis and uninstall cloudmesh""" dir() cmd3() banner("CLEAN PREVIOUS CLOUDMESH INSTALLS") r = int(local("pip freeze |fgrep cloudmesh | wc -l", capture=True)) while r > 0: local('echo "y\n" | pip uninstall cloudmesh') r = int(local("pip freeze |fgrep cloudmesh | wc -l", capture=True))
def dir(): """clean the dirs""" banner("CLEAN DIR") local("rm -rf *.egg") local('find . -name "*~" -exec rm {} \; ') local('find . -name "*.pyc" -exec rm {} \; ') local("rm -rf build dist *.egg-info *~ #*") # local("cd docs; make clean") local("rm -rf *.egg-info") local("rm -f *.dump")
def cat(filename): """prints the contents of a file with the given name. :param filename: name of the file, which can include ~ and $ environment variables :type: string """ location = path_expand(filename) banner(filename) with open(location, 'r') as f: print(f.read())
def dir(): """clean the dirs""" banner("CLEAN DIR") local("rm -rf *.egg") local('find . -name "*~" -exec rm {} \; ') local('find . -name "*.pyc" -exec rm {} \; ') local("rm -rf build dist *.egg-info *~ #*") #local("cd docs; make clean") local("rm -rf *.egg-info") local("rm -f celeryd@*") local("rm -f *.dump")
def HEADING(txt=None): """ Prints a message to stdout with #### surrounding it. This is useful for nosetests to better distinguish them. :param txt: a text message to be printed :type txt: string """ if txt is None: txt = inspect.getouterframes(inspect.currentframe())[1][3] banner(txt)
def dir(): """clean the dirs""" banner("STOPPING SERVER") server.stop() banner("CLEAN DIR") local("rm -rf *.egg") local('find . -name "*~" -exec rm {} \; ') local('find . -name "*.pyc" -exec rm {} \; ') local("rm -rf build dist *.egg-info *~ #*") # local("cd docs; make clean") local("rm -rf *.egg-info") local("rm -f celeryd@*") local("rm -f *.dump")
def delete_defaults(): filename = config_file("/cloudmesh.yaml") banner("reading data from {0}".format(filename)) config = cm_config(filename=filename) username = config.get("cloudmesh.hpc.username") print username user = cm_user() user.set_defaults(username, {}) # user.set_default_attribute(username, 'images', {}) user.info(username)
def login(): error = None form = LoginForm() if request.method == 'POST' and form.validate_on_submit(): form.error = None try: # idp = cm_userLDAP () # idp.connect("fg-ldap", "ldap") # user = idp.find_one({'cm_user_id': form.username.data}) # print "MONGO USER" # banner("LDAPUSER") # pprint (user) banner("CM_USER") user_obj = cm_user() user = user_obj.info(form.username.data) except Exception, e: print traceback.format_exc() error = "LDAP server not reachable" error += str(e) return render_template('error.html', form=form, type="Can not reach LDAP", msg="") if user is None: form.error = 'Login Invalid' elif 'cm_user_id' not in user: form.error = 'Login Invalid' elif user['cm_user_id'] != form.username.data: form.error = 'Login Invalid' elif user_obj.authenticate(form.username.data, form.password.data): print "LOGIN USER" g.user = load_user(form.username.data) ret = login_user(g.user) identity_changed.send(current_app._get_current_object(), identity=Identity(g.user.id)) return redirect(request.args.get('next') or '/') else: form.error = 'Login Invalid'
def replace(self, kind='text', values=None): try: template = Template(self.content) if kind == "text": self.result = template.render(**values) elif kind == "dict": self.result = yaml.safe_load(template.render(**values)) else: log.error( "kind='dict' or 'text' parameter missing in template replace") raise RuntimeError return self.result except UndefinedError, e: banner("ERROR: Undefined variable in template") print(e)
def start(server="server", browser='yes', debug=False): """ starts in dir webgui the program server.py and displays a browser on the given port and link""" # pprint (fabric.state.output) """ 'aborts': True, 'debug': False, 'running': True, 'status': True, 'stderr': True, 'stdout': True, 'user': True, 'warnings': True } """ # banner(debug) banner("KILL THE SERVER", debug=debug) kill(debug=debug) if not debug: PROGRESS.next() execute_command("INSTALL CLOUDMESH", "python setup.py install", debug=debug) mongo.start() # execute_command("START MONGO", # "fab mongo.start", # debug) queue.start() # execute_command("START RABITMQ", # "fab queue.start", debug) queue.flower_server() # execute_command("START FLOWER", # "fab queue.flower_server", # debug) fabric.state.output.stdout = True fabric.state.output.stderr = True execute_command("START WEB SERVER", "cd cloudmesh_web; python {0}.py &".format(server), True) # view(link) PROGRESS.finish()
def sphinx_updates(): # *mktemp -d* should be applied to get a unique directory name to a user # who runs this function. # Otherwise, if there are ohter users who run this command as well, # permission conflict will occur when it trys to write or delete # the directory # TODO: the use of mktemp was wrong as we need to pass a template banner("install sphinx autorun", c="-") user = getpass.getuser() dirname = local("mktemp -d /tmp/{0}_cloudmesh.XXXXX".format(user), capture=True) dirname = dirname + "/install-cloudmesh" local('rm -rf %s' % dirname) local('mkdir -p %s' % dirname) local('cd %s; hg clone http://bitbucket.org/birkenfeld/sphinx-contrib/' % dirname) local('cd %s/sphinx-contrib/autorun; python setup.py install' % dirname) banner("insall autorun ok")
def start(auth=True): ''' start the mongod service in the location as specified in cloudmesh_server.yaml ''' banner("Starting mongod") config = cm_config_server().get("cloudmesh.server.mongo") path = path_expand(config["path"]) port = config["port"] if not os.path.exists(path): print "Creating mongodb directory in", path local("mkdir -p {0}".format(path)) with settings(warn_only=True): with hide('output', 'running', 'warnings'): lines = local( "ps -ax |grep '[m]ongod.*port {0}'".format(port), capture=True)\ .split("\n") if lines != ['']: pid = lines[0].split(" ")[0] print "NO ACTION: mongo already running in pid {0} for port {1}"\ .format(pid, port) else: print "ACTION: Starting mongod" print print "NOTE: the preparation of mongo may take a few minutes" print " please do not interrupt this program." print print " Please be patient!" print with_auth = "" if isyes(auth): with_auth = "--auth" local('mongod {2} --bind_ip 127.0.0.1 ' '--fork --dbpath {0} ' '--logpath {0}/mongodb.log ' '--port {1}'.format(path, port, with_auth))
def sphinx_updates(): # *mktemp -d* should be applied to get a unique directory name to a user # who runs this function. # Otherwise, if there are ohter users who run this command as well, # permission conflict will occur when it trys to write or delete # the directory # TODO: the use of mktemp was wrong as we need to pass a template banner("install sphinx autorun", c="-") user = getpass.getuser() dirname = local( "mktemp -d /tmp/{0}_cloudmesh.XXXXX".format(user), capture=True) dirname = dirname + "/install-cloudmesh" local('rm -rf %s' % dirname) local('mkdir -p %s' % dirname) local('cd %s; hg clone http://bitbucket.org/birkenfeld/sphinx-contrib/' % dirname) local('cd %s/sphinx-contrib/autorun; python setup.py install' % dirname) banner("insall autorun ok")
def admin(): """creates a password protected user for mongo""" banner("create auth user") config = cm_config_server().get("cloudmesh.server.mongo") user = config["username"] password = config["password"] # # setting up the list of dbs # dbs = set() # print config["collections"] for collection in config["collections"]: dbs.add(config['collections'][collection]['db']) # setting the admin user script = [] script.append('db.addUser("{0}", "{1}");'.format(user, password)) script.append('db.auth("{0}", "{1}");'.format(user, password)) # setting a password for each db for db in dbs: script.append('db = db.getSiblingDB("{0}");'.format(db)) script.append('db.addUser("{0}", "{1}");'.format(user, password)) script.append("use admin;") script.append('db.addUser("{0}", "{1}");'.format(user, password)) script.append('db.auth("{0}", "{1}");'.format(user, password)) script.append('db.shutdownServer();') mongo_script = '\n'.join(script) # print mongo_script command = "echo -e '{0}' | mongo".format(mongo_script) print command banner("Executing js") os.system(command)
def boot(auth=True): # wipe mongo wipe() time.sleep(1) PROGRESS.next() # start mongo without auth start(auth=False) time.sleep(5) PROGRESS.next() if isyes(auth): # create users admin() time.sleep(2) PROGRESS.next() # restart with auth kill() time.sleep(10) PROGRESS.next() start(auth=auth) time.sleep(1) PROGRESS.next() config = cm_config_server().get("cloudmesh.server.mongo") path = path_expand(config["path"]) banner(path) print ls(path) banner("PROCESS") with settings(warn_only=True): local("ps -ax | fgrep mongo") PROGRESS.next()
def reset(password=None): banner("initiating mongo") boot() banner("initiating user data to mongo") PROGRESS.next() print if password is None: local("fab user.mongo") else: local("fab user.mongo:passwd={0}".format(password)) PROGRESS.next() banner("refreshing cloud info") simple() local("reset")
def wipe(): """wipes out all traces from mongo""" kill() config = cm_config_server().get("cloudmesh.server.mongo") path = path_expand(config["path"]) banner("{0}".format(path)) local("mkdir -p {0}".format(path)) result = str(ls(path)) banner(path, "-") print result print 70 * "-" if result != "": if yn_choice("deleting the directory", default="n"): local("rm -rf {0}".format(path)) local("mkdir -p {0}".format(path)) banner("{0}".format(path)) local("ls {0}".format(path))
def install_command(args): """ Usage: install -h | --help install --version install cloudmesh install delete_yaml install system install query install new [--force] install vagrant install enable admin [--username=<username>] """ # This is a debuging message # print "IIIII<" + args + ">" arguments = docopt(install_command.__doc__, args) print arguments if arguments["cloudmesh"]: deploy() elif arguments["new"]: force = arguments["--force"] new_cloudmesh_yaml(force) elif arguments["delete_yaml"]: answer = yn_choice( "THIS COMMAND IS REAL DANGEROUS AND WILL DELETE ALL YOUR YAML FILE. Proceed", default='y') if answer: # TODO: cp is not imported, defined print("You fool we just deleted your yaml files") cp("etc/*.yaml", config_file_prefix()) else: print "puuh you interrupted" pass elif arguments["system"]: banner("Installing Ubuntu System Requirements") if is_ubuntu(): ubuntu() elif is_osx(): osx() elif is_centos(): centos() elif arguments["query"]: import platform print "System: ", platform.system() # print "Uname: ", platform.uname() print "Machine: ", platform.machine() print "Processor: ", platform.processor() print "Platform: ", platform.platform() print "Python: ", platform.python_version() print "Virtualenv:", hasattr(sys, 'real_prefix') elif arguments["vagrant"]: vagrant() elif arguments["enable"] and arguments["admin"]: enable_admin_page(arguments['--username'])
def cmd3(): banner("CLEAN CMD3") local("rm -rf {0}".format(config_file("/cmd3local")))
def fetchrc(userid=None, outdir=None): banner("download rcfiles (novarc, eucarc, etc) from IaaS platforms") print "" # Task 1. list portal user id ''' try: from cloudmesh.config.ConfigDict import ConfigDict except Exception, e: print "ERROR: your have not yet configured cloudmesh completely. " print " Have you called" print print " ./install cloudmesh" print sys.exit(1) dir = config_file("") config = ConfigDict(dir + "/me.yaml") userid = config["portalname"] ''' if not userid: userid = getpass.getuser() userid = raw_input("Please enter your portal user id [default: %s]: " % userid) or userid # Task 2. list hostnames to get access. In FutureSystems, india is default # TEMPORARY host_ids = ["india"] # user input is disabled # host_ids = raw_input("Please enter host identifications [default: %s]: " # % ", ".join(host_ids)) or host_ids if isinstance(host_ids, str): host_ids = map(lambda x: x.strip(), host_ids.split(",")) # domain_name = ".futuregrid.org" # hostnames = map(lambda x: x.split("_")[0] + domain_name, host_ids) # key_path = "~/.ssh/id_rsa" # private key path is disabled # key_path = raw_input("Please enter a path of the ssh private key to" + \ # " login the hosts [default: %s]: " % key_path) or \ # key_path try: download_rc_files(userid) update_permission("~/.cloudmesh/clouds") """ # cmd = "fab rcfile.download:userid='%s',host_ids='%s',key_path='%s'" \ cmd = "fab -H %s -u %s -i %s rcfile.download:'%s','%s'" \ % (",".join(hostnames), userid, key_path, "\,".join(host_ids), outdir) print "CMD", cmd os.system(cmd) """ except: print sys.exc_info() sys.exit(1)
from flask.ext.principal import Permission, RoleNeed # from cloudmesh.experiment.cm_experiment_db import cm_experiment_db import os # # disable RST PAGES dues to sphinx incompatibility # try: RSTPAGES = os.environ['RSTPAGES'] RSTPAGES = (RSTPAGES == None) or RSTPAGES.lower() == 'true' except: RSTPAGES = True msg = 'RST PAGES ' + str(RSTPAGES) banner(msg) if RSTPAGES: from flask.ext.rstpages import RSTPages from pprint import pprint import cloudmesh import pkg_resources import requests import traceback import types import webbrowser admin_permission = Permission(RoleNeed('admin'))
def init_shell_command(arguments): """ :: Usage: init [--force] generate yaml init [--force] generate me init [--force] generate none init [--force] generate FILENAME init list [KIND] [--json] init list clouds [--file=FILENAME] [--json] init inspect --file=FILENAME init fill --file=FILENAME [VALUES] Initializes cloudmesh from a yaml file Arguments: generate generates a yaml file yaml specifies if a yaml file is used for generation the file is located at me.yaml me same as yaml none specifies if a yaml file is used for generation the file is located at CONFIG/etc/none.yaml FILENAME The filename to be generated or from which to read information. VALUES yaml file with the velues to be sed in the FILENAME KIND The kind of the yaml file. Options: --force force mode does not ask. This may be dangerous as it overwrites the CONFIG/cloudmesh.yaml file --file=FILENAME The file --json make the output format json -v verbose mode Description: init list [KIND] [--json] list the versions and types of the yaml files in the CONFIG and CONFIG/etc directories. init list clouds [--file=FILENAME] Lists the available clouds in the configuration yaml file. init inspect --file=FILENAME print the variables in the yaml template """ if arguments["inspect"]: filename = arguments['--file'] if filename is None: filename = config_file('/cloudmesh.yaml') content = open(filename, 'r').read() t = cm_template(filename) sorted_vars = sorted(set(t.variables())) print("\n".join(sorted_vars)) # banner("PARSER") # env = Environment() # ast = env.parse(content) # for v in meta.find_undeclared_variables(ast): # print v if arguments["list"] and not arguments["clouds"]: dirs = [ path_expand(config_dir + '/*.yaml'), path_expand(config_dir + '/etc/*.yaml') ] file_list = [] for directory in dirs: file_list.extend(glob.glob(directory)) vector = {} vector['kind'] = [] vector['yaml_version'] = [] vector['meta'] = [] vector['filename'] = [] for filename in file_list: values = {'kind': "-", 'yaml_version': "-", 'meta': "-"} head_of_file = find_meta(filename) values = {'kind': "-", 'yaml_version': "-", 'meta': "-"} for line in head_of_file: if ":" in line: (attribute, value) = line.strip().split(":") if attribute in ["kind", "yaml_version"]: values[attribute] = value.strip() if attribute in ["meta"]: values[attribute] = "+" if arguments["KIND"] is None or \ values['kind'] == arguments['KIND']: for attribute in values.keys(): vector[attribute].append(values[attribute]) vector['filename'].append(filename) vector['Kind'] = vector.pop('kind') vector['Version'] = vector.pop('yaml_version') vector['Meta'] = vector.pop('meta') vector['Filename'] = vector.pop('filename') banner("Configuration Directory: {0}".format(config_dir), c="-") print(column_table(vector)) # print filename, values["kind"], values["version"] if arguments["list"] and arguments["clouds"]: filename = arguments['--file'] if filename is None: filename = config_file('/cloudmesh.yaml') config = cm_config(filename) data = {} data['Clouds'] = config.cloudnames() data['Labels'] = [] data['Type'] = [] data['Version'] = [] for cloud_key in data['Clouds']: data['Labels'].append(config.cloud(cloud_key)['cm_label']) data['Type'].append(config.cloud(cloud_key)['cm_type']) data['Version'].append(config.cloud(cloud_key)['cm_type_version']) if arguments["--json"]: print(json.dumps(data, sort_keys=True, indent=4)) else: print(column_table(data, ['Labels', 'Clouds', 'Type', 'Version'])) if arguments["fill"]: filename_template = arguments['--file'] if filename_template is None: filename_template = config_file('/etc/cloudmesh.yaml') filename_template = path_expand(filename_template) filename_values = arguments['VALUES'] if filename_values is None: filename_values = config_file('/me.yaml') content = open(filename_template, 'r').read() t = cm_template(filename_template) sorted_vars = sorted(set(t.variables())) try: values = ConfigDict(filename=filename_values) except Exception, e: print("ERROR: There is an error in the yaml file", e) for cloud in values['clouds']: values['clouds'][cloud]['default'] = {} values['clouds'][cloud]['default']['image'] = None values['clouds'][cloud]['default']['flavor'] = None banner("%s -> %s" % (filename_values, filename_template)) env = Environment(undefined=IgnoreUndefined) template = env.from_string(content) result = template.render(values) print(result)
def web(server="server", browser='yes'): banner("START WEB SERVER") local("cd cloudmesh_web; python {0}.py &".format(server))
def install(): sphinx_updates() banner("cloudmesh python install") local("python setup.py install")
# values = yaml.safe_load(Template(result).substitute(os.environ)) # print json.dumps(values, indent=4) except Exception, e: print "ERROR: There is an error in the yaml file", e sys.exit(1) for cloud in values['clouds']: values['clouds'][cloud]['default'] = {} values['clouds'][cloud]['default']['image'] = None values['clouds'][cloud]['default']['flavor'] = None file_from_template(cloudmesh_template, cloudmesh_out, values) print "# Created: {0}".format(me_file) banner(c="-") # sys.exit() # # format = "yaml" # if format in ["json"]: # result = json.dumps(values, indent=4) # elif format in ["yaml", "yml"]: # result = yaml.dump(values, default_flow_style=False) # banner("done", c="-") # print "# Template: {0}".format(filename_template) # print "# Values : {0}".format(filename_values) # print "# Backup : {0}".format(filename_bak)
def lsl(): print pbs_queue banner("Dict") pprint(l_queue.__dict__) i = l_queue.control.inspect() c = l_queue.control banner("Active Queues") pprint(i.active_queues()) banner("Registered") pprint(i.registered()) banner("Active") pprint(i.active()) banner("Scheduled") pprint(i.scheduled()) banner("Reserved") pprint(i.reserved()) # banner("Revoked") # pprint (i.resoked()) banner("Stats") pprint(i.stats()) banner("Ping") pprint(c.ping(timeout=0.5))
result = t.replace(kind="dict", values=user_config) location = path_expand(out_file) yaml_file = open(location, 'w+') print(yaml.dump(result, default_flow_style=False), file=yaml_file) yaml_file.close() log.info("Written new yaml file in " + location) if __name__ == "__main__": cloudmesh_yaml = config_file("/etc/cloudmesh.yaml") user_config = ConfigDict(filename=config_file("/me.yaml")) t = cm_template(cloudmesh_yaml) banner("VARIABLES") s = set(t.variables()) print(("\n".join(s))) banner("GREP") s = t.grep() print(("\n".join(s))) # banner("YAML FILE") # result = t.replace(kind="dict", values=user_config) # print yaml.dump(result, default_flow_style=False) # location = config_file('/cloudmesh-new.yaml') # yaml_file = open(location, 'w+') # print >> yaml_file, yaml.dump(result, default_flow_style=False) # yaml_file.close() # print "Written new yaml file in " + location