def __deepen(self, depth): cmd(self.__bare_path(), ["git", "fetch", "--deepen", str(depth), self.cache.repository, self.__ref()])
def check_debian(): arch = cmd('dpkg --print-architecture',output=True).strip() mach = cmd('uname -m',output=True).strip() if arch == 'i386' and mach == 'x86_64': log.warning('''Your kernel claims to be x86_64 but your user space claims to be i386. This will greatly confuse the various build systems. To work around this issue, either install an i686 kernel or run "linux32 /bin/bash" before installing or running the code.''') return False return True
def hsPreprocess_btn_clicked(a): cmd("mkdir -p output/hsTmp") for m in range(120): hs_array_time2 = [] for j in range(109): fo = open(opdir + 'hs', "r+") datalines1 = fo.readlines()[9 + j + 231 * m] p1 = datalines1.split() for i in range(1, 152): hs_array_time2.append(p1[i]) fo = open(opdir + 'hs', "r+") datalines2 = fo.readlines()[122 + j + 231 * m] p2 = datalines2.split() for i in range(1, 31): hs_array_time2.append(p2[i]) fo.close() hsfilename = opdir + 'hsTmp/hs_' + str(m + 1) hs_file_object_time2 = open(hsfilename, 'w') for ip in hs_array_time2: if (ip == "-900."): ip = "NaN" hs_file_object_time2.write(ip) hs_file_object_time2.write('\n') hs_file_object_time2.close() fig_size = [] fig_size.append(15) fig_size.append(8)
def build(): 'Build CMT in previously unpacked cmt.srcdir().' log.info('building cmt') fs.goto(os.path.join(srcdir(),'mgr/')) from command import cmd,make,source def stupid_hack(): # quick hack for v1r22. this really doesn't belong here! fp = open("../source/cmt_std.h") for line in fp: if 'climits' in line: return continue fp.close() fp = open("../source/cmt_std.h","a") fp.write("#include <climits>\n") fp.close() return stupid_hack() # always run this in case user does something silly like move the # installation somewhere else cmd('./INSTALL') environ = env() cmtexe = '%s/%s/cmt'%(srcdir(),environ['CMTCONFIG']) if os.path.exists(cmtexe): log.info('CMT appears to already have been built: %s'%cmtexe) else: log.info('CMT rebuild, not found at: %s'%cmtexe) make(env=environ) return cmtexe
def abort_btn_clicked(a): g = re.match(r'^\S+',jobSelect.value) with logOp: if g: jobid = g.group(0) rcmd = "jobs-stop "+jobid cmd(rcmd)
def _unpack_tar(self): tarcmd = "tar -x" if 'gz' in self.tarfilename: tarcmd += "z" tarcmd += "f %s"%self.tarfilename from command import cmd cmd(tarcmd, dir=self.cli.opts.unpack_directory) return
def make_archive(self): if not self.__exists(): log.warning("Fetching tag '{}' from {}".format( self.tag, self.cache.repository )) with self.cache.lock(): bare_path = self.__bare_path() ref = "refs/tags/{}".format(self.tag) setup_bare(bare_path) try: cmd(bare_path, ["git", "fetch", "--depth=1", self.cache.repository, ref]) except subprocess.CalledProcessError as e: log.debug( "Did not find tag {}, instead got output: {}".format( self.tag, e.output ) ) raise NixError( "tag-not-found", "Tag {} not found".format(self.tag) ) archive_ref(bare_path, "FETCH_HEAD", self.__archive_path()) return self.__archive_path()
def load(reload=False, trace=False): if not reload: try: with open("app-data.pickle", "rb") as fd: return pickle.load(fd) except: pass # Get the username with open(os.environ["HOME"] + "/.agave/current", "r") as fd: jd = json.loads(fd.read()) user = jd["username"] all_apps = {} out = cmd('apps-list', show=False, trace=trace) for line in out["stdout"]: app = line if not re.match(r'crcollab', app): continue # Get permissions for the app pout = cmd("apps-pems-list -v -u " + user + " " + app, show=False, trace=trace) pdata = json.loads(" ".join(pout["stdout"])) perm = pdata["permission"] perms = "" if perm["read"]: perms += "R" if perm["write"]: perms += "W" if perm["execute"]: perms += "X" appdata = cmd('apps-list -V ' + app, show=False, trace=True) jtxt = " ".join(appdata["stdout"]) jd = json.loads(jtxt) jd = jd["result"] exec_sys = jd["executionSystem"] storage_sys = jd["deploymentSystem"] execdata = cmd('systems-list -v ' + exec_sys, show=False, trace=trace) jd2 = json.loads(" ".join(execdata["stdout"])) queues = [] for q in range(len(jd2["queues"])): name = jd2["queues"][q]["name"] ppn = jd2["queues"][q]["maxProcessorsPerNode"] queues += [{"name": name, "ppn": ppn}] all_apps[app] = { "perm": perms, "exec_sys": exec_sys, "storage_sys": storage_sys, "queues": queues } with open("app-data.pickle", "wb") as fd: pickle.dump(all_apps, fd) return all_apps
def _fix_cmt(self): import os, cmt from command import cmd newdir = os.path.join(self.dstdir,'external/CMT',cmt.ver(),'mgr') cmd('./INSTALL',env=None,dir=newdir) import shutil shutil.copy(os.path.join(newdir,'setup.sh'), os.path.join(self.dstdir,'projects/setup/00_cmt.sh')) shutil.copy(os.path.join(newdir,'setup.csh'), os.path.join(self.dstdir,'projects/setup/00_cmt.csh')) return
def __fetch_all(self): try: cmd(self.__bare_path(), ["git", "fetch", "--unshallow", self.cache.repository, self.__ref()]) except subprocess.CalledProcessError: # We may already have unshallowed this repository pass
def __commit_exists(self): try: cmd(self.__bare_path(), ["git", "cat-file", "-e", "{}^{{commit}}".format(self.revision)]) return True except subprocess.CalledProcessError as e: log.debug( "Did not find commit {} and instead got output: {}".format( self.revision, e.output ) ) return False
def configure2(agave_username, exec_machine, storage_name, project_name): setvar(""" AGAVE_USERNAME=""" + agave_username + """ APP_NAME=""" + project_name + """ AGAVE_JSON_PARSER=jq AGAVE_CACHE_DIR=$HOME/.agave PATH=$HOME/agave-model/cli/bin:$PATH STORAGE_MACHINE=""" + storage_name + """ DEPLOYMENT_PATH=agave-deployment EXEC_MACHINE=""" + exec_machine + """ QUEUE=checkpt """) readpass("AGAVE_PASSWD") readpass("PBTOK") cmd("tenants-init -t agave.prod") cmd("clients-delete -u $AGAVE_USERNAME -p $AGAVE_PASSWD $APP_NAME", show=False) cmd("clients-create -p $AGAVE_PASSWD -S -N $APP_NAME -u $AGAVE_USERNAME", show=False) cmd("auth-tokens-create -u $AGAVE_USERNAME -p $AGAVE_PASSWD", show=False) setvar("[email protected]") print("Successfully configured Agave")
def cmd_inrel(self, cmdstr): 'Run a command in the release package. No-op if none defined.' relpkg = self.rel_pkg() if not relpkg: err = 'Project %s has no release package defined'%self.name log.warning(err) raise CommandFailure, err import fs relpkgcmt = os.path.join(fs.projects(), self.name, relpkg, 'cmt') if not os.path.exists(relpkgcmt): err = 'Project %s has release package defined, but no dir: %s'%(self.name, relpkgcmt) log.warning(err) raise CommandFailure, err import command command.cmd(cmdstr, env=self.env(), dir=relpkgcmt)
def template_on_change(change): inputTmp = '' if change['type'] == 'change' and change['name'] == 'value': if(change['new'] == 'Choose Input Template'): tab_nest.children[0].children[2].children = [] return if(change['new'] == 'Basic Template'): with logOp: cmd("tar -xvf input_" + cur_model + ".tgz") inputTmp = 'input_' + cur_model + '/basic_template.txt' if(change['new'] == 'HDF5 Template'): with logOp: cmd("tar -xvf input_" + cur_model + ".tgz") inputTmp = 'input_' + cur_model + '/hdf5_template.txt' tab_nest.children[0].children[2].children = generatePara(inputTmp)
def archive_ref(source, ref, destination): try: os.makedirs(destination) except FileExistsError: pass cmd(source, [ "bash", # Bash will run the following safe bash, and accept the defs # of $1 and $2 via the following -s "-c", "git archive --format=tar \"$1\" | tar -x -C \"$2\"", # Passing ref and destination this way is much safer than # using interpolation "-s", ref, destination ])
def jobHis_btn_clicked(a): g = re.match(r'^\S+', jobSelect.value) with logOp: if g: jobid = g.group(0) rcmd = "jobs-history -V " + jobid cout = cmd(rcmd) out1 = cout["stdout"] jobHisSelect.options = out1
def jobOutput_btn_clicked(a): g = re.match(r'^\S+',jobSelect.value) with logOp: if g: jobid = g.group(0) rcmd = "jobs-output-list "+ jobid cout = cmd(rcmd) out1 = cout["stdout"] outputSelect.options = out1
def isjobexist(): query_terms = 'model='+modelTitle.value+'&model_ver='+modelVersionDd.value+'&mpich='+mpiDd.value+'&hdf5='+h5Dd.value+'&hypre='+hypreDd.value query_cmd = "jobs-search 'status=FINISHED' 'parameters.like={*\"versions\":\""+query_terms+"\"*}'" print (query_cmd) out = cmd(query_cmd,show=False,trace=False) result = out['stdout'][0] if not result: return False return True
def download_btn_clicked(a): with logOp: try: g = re.match(r'^\S+',jobSelect.value) jobid = g.group(0) if(outputSelect.value.find('.')==-1): rcmd = "jobs-output-get -r "+ jobid +" "+ outputSelect.value else: rcmd = "jobs-output-get "+ jobid +" "+ outputSelect.value print (rcmd) os.system(rcmd) if(outputSelect.value == 'output.tar.gz'): cmd("rm -fr output") cmd("mkdir -p output_" + jobid) cmd('tar -xvf output.tar.gz -C ' + jobid) elif(re.match(r'.*\.(txt|out|err|ipcexe)',outputSelect.value)): with open(outputSelect.value,'r') as fd: for line in fd.readlines(): print(line,end='') else: print('value=(',outputSelect.value,')',sep='') except Exception as ex: print("DIED!",ex)
def build(): "Build git in previously unpacked git.srcdir()." log.info("building git") fs.goto(srcdir()) from command import cmd, make instpath = prefix() if not os.path.exists("config.status"): log.info("configuring git") cmd("./configure --prefix=%s" % instpath) if os.path.exists("git"): log.info("git appears to already have been built") else: make() if os.path.exists(instpath): log.info("git appears to already have been installed to %s" % instpath) else: make("install") return
def pack_garpi(self): ''' Add garpi to the pack. ''' import command files = [] out = command.cmd("find garpi -type f -print", output=True) for path in out.split('\n'): if not self._keeper(path): continue if path[:2] == './': path = path[2:] files.append(path) continue self.pack_files(files) return
def cmt(cmdstr='', environ=None, dir=None, output=False): """Run 'cmt [cmdstr]'. By default the environment will be that provided by cmt.env(pkgdir=dir) unless an explicity environment is given via the environ arg. The dir and output options are passed to command.cmd.""" if not environ: environ = env() from command import cmd, source try: ret = cmd('cmt '+cmdstr, env=environ, dir=dir, output=output) except OSError,msg: print msg print 'PATH:' for p in environ['PATH'].split(':'): print '\t',p #for k,v in environ.iteritems(): # print '"%s" = "%s"'%(k,v) raise
def update_btn_clicked(a): if (tab_nest.children[0].children[1].value == True): with logOp: cmd("rm -f input.tgz") cmd("rm -fr input") cmd("cp -f ../input_" + cur_model + ".tgz input.tgz") return newInput = 'input_'+ cur_model + '/input_tmp.txt' inputTmp = '' if(tab_nest.children[0].children[0].value == 'Basic Template'): inputTmp = 'input_' + cur_model + '/basic_template.txt' if(tab_nest.children[0].children[0].value == 'HDF5 Template'): inputTmp = 'input_' + cur_model + '/basic_template.txt' updatePara(inputTmp, newInput, tab_nest.children[0].children[2]) tab_nest.children[0].children[4].value = open(newInput, 'r').read()
def ofUpInput_btn_clicked(a): if (ofCbox.value == True): with logOp: cmd("rm -f input.tgz") cmd("rm -fr input") cmd("cp -f input_openfoam/foam_run/" + ofOwnCaseName.value + " input.tgz") return if not ofCaseName.value == "Select Input Case": with open("input_openfoam/tutorials/" + ofCaseName.value[:-1] + "/system/decomposeParDict", "r") as fd: contents = fd.read() ofInputArea.value = contents sp = r'(?:\s|//.*)' pat = re.sub(r'\\s',sp,r'coeffs\s*{\s*n\s+\(\s*(\d+)\s+(\d+)\s+(\d+)\s*\)\s*;\s*}') g = re.search(pat, contents) if g: numXSlider.value = g.group(1) numYSlider.value = g.group(2) numZSlider.value = g.group(3) fd.close()
def buildBtn_clicked(a): model_ver = modelTitle.value.upper() + "_VER" mpi_ver = '' if mpiDd.value is "None" else mpiDd.value h5_ver = '' if h5Dd.value is "None" else h5Dd.value hypre_ver = '' if hypreDd.value is "None" else hypreDd.value with logOp: setvar("MPICH_VER=" + mpiDd.value) setvar("HDF5_VER=" + h5Dd.value) setvar("HYPRE_VER=" + hypreDd.value) setvar("MODEL_VER=" + modelVersionDd.value) writefile("env_setting.txt",""" #!/bin/bash export """+model_ver+"""="""+modelVersionDd.value+""" export MPICH_VER="""+mpi_ver+""" export H5_VER="""+h5_ver+""" export HYPRE_VER="""+hypre_ver+""" """) # cmd("rm -fr build_input") # cmd("mkdir build_input") # cmd("cp env_setting build_input/") exist = isjobexist() if exist is True: with msgOut: print ("The job exist, no build job would be submitted!") else: setvar("INPUT_DIR=${AGAVE_USERNAME}_$(date +%Y-%m-%d_%H-%M-%S)") cmd("files-mkdir -S ${STORAGE_MACHINE} -N inputs/${INPUT_DIR}") cmd("files-upload -F env_setting.txt -S ${STORAGE_MACHINE} inputs/${INPUT_DIR}/") submitBuildJob(machines.value, queues.value) # set job permissions to users users = ['ysboss','tg457049','lzhu','nanw','reza'] for user in users: cmd('jobs-pems-update -u '+user+' -p READ ${JOB_ID}')
def configure(agave_username, machine_username, machine_name, project_name): g = re.match(r'(\w+)\.(.*)', machine_name) os.environ["MACHINE"] = g.group(1) os.environ["DOMAIN"] = g.group(2) os.environ["MACHINE_FULL"] = machine_name setvar(""" AGAVE_USERNAME=""" + agave_username + """ MACHINE_USERNAME=""" + machine_username + """ BASE_APP_NAME=crcollaboratory PORT=22 ALLOCATION=hpc_startup_funwave WORK_DIR=/work/${MACHINE_USERNAME} HOME_DIR=/home/${MACHINE_USERNAME} SCRATCH_DIR=/work/${MACHINE_USERNAME} DEPLOYMENT_PATH=agave-deployment AGAVE_JSON_PARSER=jq PATH=$HOME/agave-model/cli/bin:$PATH """) readpass("MACHINE_PASSWD") readpass("AGAVE_PASSWD") readpass("PBTOK") setvar("APP_NAME=${BASE_APP_NAME}-${MACHINE}-${AGAVE_USERNAME}") #cmd("git clone https://bitbucket.org/agaveapi/cli.git") cmd("tenants-init -t agave.prod") cmd("clients-delete -u $AGAVE_USERNAME -p $AGAVE_PASSWD $APP_NAME", show=False) cmd("clients-create -p $AGAVE_PASSWD -S -N $APP_NAME -u $AGAVE_USERNAME", show=False) cmd("auth-tokens-create -u $AGAVE_USERNAME -p $AGAVE_PASSWD", show=False) cmd("auth-tokens-refresh") setvar("STORAGE_MACHINE=${MACHINE}-storage-${AGAVE_USERNAME}3") writefile( "${STORAGE_MACHINE}.txt", """{ "id": "${f }", "name": "${MACHINE} storage (${MACHINE_USERNAME})", "description": "The ${MACHINE} computer", "site": "${DOMAIN}", "type": "STORAGE", "storage": { "host": "${MACHINE_FULL}", "port": ${PORT}, "protocol": "SFTP", "rootDir": "/", "homeDir": "${HOME_DIR}", "auth": { "username" : "${MACHINE_USERNAME}", "password" : "${MACHINE_PASSWD}", "type" : "PASSWORD" } } } """) cmd("systems-addupdate -F ${STORAGE_MACHINE}.txt") #cmd("sshpass -f MACHINE_PASSWD.txt ssh -o StrictHostKeyChecking=no ${MACHINE_USERNAME}@${MACHINE_FULL} -p ${PORT} (sinfo || qstat -q)") setvar(""" # Gather info about the machine # Executing this cell is essential MAX_TIME=72:00:00 # Max duration of a job # We figure out the number of processes automatically. # This assumes the head node and compute nodes have # the same number of procs. CPUINFO=$(sshpass -f MACHINE_PASSWD.txt ssh -o StrictHostKeyChecking=no ${MACHINE_USERNAME}@${MACHINE_FULL} -p ${PORT} lscpu) QUEUE=checkpt # Name of default queue NODES=42 # Number of nodes in queue """) g = re.search(r'(?m)CPU\(s\):\s*(\d+)', os.environ["CPUINFO"]) os.environ["PROCS"] = g.group(1) print(repvar("PROCS=$PROCS")) setvar("EXEC_MACHINE=${MACHINE}-exec-${AGAVE_USERNAME}") os.environ["DIRECTIVES"] = re.sub( "\n\\s*", r"\\n", """ #PBS -A ${ALLOCATION} #PBS -l cput=\${AGAVE_JOB_MAX_RUNTIME} #PBS -l walltime=\${AGAVE_JOB_MAX_RUNTIME} #PBS -q \${AGAVE_JOB_BATCH_QUEUE} #PBS -l nodes=\${AGAVE_JOB_NODE_COUNT}:ppn=16 """.strip()) writefile( "${EXEC_MACHINE}.txt", """ { "id": "${EXEC_MACHINE}", "name": "${MACHINE} (${MACHINE_USERNAME})", "description": "The ${MACHINE} computer", "site": "${DOMAIN}", "public": false, "status": "UP", "type": "EXECUTION", "executionType": "HPC", "scheduler" : "CUSTOM_TORQUE", "environment": null, "scratchDir" : "${SCRATCH_DIR}", "queues": [ { "customDirectives" : "${DIRECTIVES}", "name": "${QUEUE}", "default": true, "maxJobs": 10, "maxUserJobs": 10, "maxNodes": ${NODES}, "maxProcessorsPerNode": ${PROCS}, "minProcessorsPerNode": 1, "maxRequestedTime": "${MAX_TIME}" } ], "login": { "auth": { "username" : "${MACHINE_USERNAME}", "password" : "${MACHINE_PASSWD}", "type" : "PASSWORD" }, "host": "${MACHINE_FULL}", "port": ${PORT}, "protocol": "SSH" }, "maxSystemJobs": 50, "maxSystemJobsPerUser": 50, "storage": { "host": "${MACHINE_FULL}", "port": ${PORT}, "protocol": "SFTP", "rootDir": "/", "homeDir": "${HOME_DIR}", "auth": { "username" : "${MACHINE_USERNAME}", "password" : "${MACHINE_PASSWD}", "type" : "PASSWORD" } } }, "workDir": "${WORK_DIR}" }""") cmd("systems-addupdate -F ${EXEC_MACHINE}.txt") writefile( "${APP_NAME}-wrapper.txt", """ #!/bin/bash handle_trap() { rc=\$? if [ "\$rc" != 0 ] then \$(\${AGAVE_JOB_CALLBACK_FAILURE}) fi } trap 'handle_trap' ERR EXIT echo 'running \${simagename} model' # Setting the x flag will echo every # command onto stderr. This is # for debugging, so we can see what's # going on. set -x set -e echo ==PWD============= # We also print out the execution # directory. Again, for debugging purposes. pwd echo ==JOB============= if [ "\${PBS_NODEFILE}" = "" ] then # When running on a system managed by Torque # this variable should be set. If it's not, # that's a problem. echo "The PBS_NODEFILE was not set" \$(\${AGAVE_JOB_CALLBACK_FAILURE}) exit 2 fi # By default, the PBS_NODEFILE lists nodes multiple # times, once for each MPI process that should run # there. We only want one MPI process per node, so # we create a new file with "sort -u". LOCAL_NODEFILE=nodefile.txt sort -u < \${PBS_NODEFILE} > \${LOCAL_NODEFILE} PROCS=\$(wc -l < \${LOCAL_NODEFILE}) if [ "\${PROCS}" = "" ] then echo "PROCS was not set" \$(\${AGAVE_JOB_CALLBACK_FAILURE}) exit 3 fi # Prepare the nodes to run the image export SING_OPTS="--bind \$PWD:/workdir \$SING_OPTS" for host in \$(cat nodefile.txt) do hostfile="\$HOME/.bash.\${host}.sh" echo "export SING_IMAGE=/project/sbrandt/chemora/images/\${simagename}.simg" > \$hostfile echo "export SING_OPTS='\$SING_OPTS'" >> \$hostfile done # Create a nodefile that matches our choices at submit time touch nodes.txt for i in \$(seq 1 \${AGAVE_JOB_PROCESSORS_PER_NODE}) do cat nodefile.txt >> nodes.txt done export NP=\$(wc -l < nodes.txt) tar xzvf input.tgz mkdir -p output /project/singularity/bin/singularity exec \$SING_OPTS /project/sbrandt/chemora/images/\${simagename}.simg bash /usr/local/bin/runapp.sh mv input output rm -f output/PRINT* tar cvzf output.tar.gz output """) cmd("files-mkdir -S ${STORAGE_MACHINE} -N ${DEPLOYMENT_PATH}") cmd("files-mkdir -S ${STORAGE_MACHINE} -N inputs") cmd("files-upload -F ${APP_NAME}-wrapper.txt -S ${STORAGE_MACHINE} ${DEPLOYMENT_PATH}/" ) writefile("test.txt", """ parfile="input.txt" ${APP_NAME}-wrapper.txt """) cmd("files-mkdir -S ${STORAGE_MACHINE} -N ${DEPLOYMENT_PATH}") cmd("files-upload -F test.txt -S ${STORAGE_MACHINE} ${DEPLOYMENT_PATH}/") writefile( "${APP_NAME}.txt", """ { "name":"${APP_NAME}", "version":"2.0", "label":"${APP_NAME}", "shortDescription":"Run ISAAC app", "longDescription":"", "deploymentSystem":"${STORAGE_MACHINE}", "deploymentPath":"${DEPLOYMENT_PATH}", "templatePath":"${APP_NAME}-wrapper.txt", "testPath":"test.txt", "executionSystem":"${EXEC_MACHINE}", "executionType":"HPC", "parallelism":"PARALLEL", "allocation":"${ALLOCATION}", "modules":[], "inputs":[ { "id":"input tarball", "details":{ "label":"input tarball", "description":"", "argument":null, "showArgument":false }, "value":{ "default":"", "order":0, "required":false, "validator":"", "visible":true } } ], "parameters":[ { "id": "simagename", "value": { "visible": true, "required": false, "type": "string", "order": 0, "enquote": false, "default": "python", "validator": null }, "details": { "label": "Singularity Image", "description": "The Singularity image to run: swan, funwave", "argument": null, "showArgument": false, "repeatArgument": false }, "semantics": { "minCardinality": 0, "maxCardinality": 1, "ontology": [] } } ], "outputs":[ { "id":"Output", "details":{ "description":"The output", "label":"tables" }, "value":{ "default":"", "validator":"" } } ] } """) cmd("apps-addupdate -F ${APP_NAME}.txt") setvar("APP_NAME=${APP_NAME}-2.0") cmd("files-upload -F input.txt -S ${STORAGE_MACHINE}/") setvar("[email protected]") print("Successfully configured Agave")
pass if option == 1: param_id = int(raw_input('Type in ID: ')) name, value, data = receive_msg(mavfile, 'PARAM_VALUE', True, param_id = param_id) print('----------\n' + value[0] + ': ' + str(value[1]) + '\n' + '----------') elif option == 2 or option == '': param_name = raw_input('Type in parameter name: ') inv_dic = {v: k for k, v in dic.items()} if '_' not in param_name: param_name_splitted = param_name.split() param_name = '' for word in param_name_splitted: param_name += word.upper()+'_' param_name = param_name[:-1] #removing the last _ from the string print('Changed input to: ' + param_name) else: pass param_id = int(inv_dic[param_name]) name, value, data = receive_msg(mavfile, 'PARAM_VALUE', True, param_id = param_id) print('----------\n' + value[0] + ': ' + str(value[1]) + '\n' + '----------') elif option == 3: get_pos_param(mavfile, 'print') else: print('Option not defined') cmd_get_param = command.cmd('get_param', get_param) cmd_get_param.execute_cmd()
import command def change_data_stream(mavfile): ''' Data Stream is deprecated. Using message intervall instead (pymavlink 1..61) ''' #target_system = mavfile.target_system #target_component = mavfile.target_component mavfile.mav.message_interval_send(0,-1) #req_stream_id = int(raw_input('Stream_id? 0-all, 1-raw_sensors, 2-ext_status, 3-rc_channels, 4-raw_controller, 6-position: ')) #req_message_rate = int(raw_input('Stream rate (in Hz): ')) #start_stop = int(raw_input('start/stop (1/0)')) #mavfile.mav.request_data_stream_send(target_system, target_component, req_stream_id, req_message_rate, start_stop) cds = command.cmd('change data stream', change_data_stream) cds.execute_cmd()
def _project_files(self): import os, command, fs cmtconfig = os.getenv('CMTCONFIG') project_area = fs.projects() basedir = os.path.dirname(project_area) snip = len(basedir)+1 grist = [] for proj in self.project_objects: projdir = os.path.join(project_area, proj.name) out = command.cmd("find . -type f -print", dir=projdir, output=True) for path in out.split('\n'): if not self._keeper(path): continue if path[:2] == './': path = path[2:] path = os.path.join(projdir, path) path = path[snip:] # Handle InstallArea special if '/InstallArea/' in path: if path[-7:] != '.cmtref': grist.append(path) continue # If not companion symlink, then give up and just add the file linkname = path[:-7] if not os.path.islink(path): grist.append(path) continue # Resolve symlink and save both pathname = readlink(linkname) grist.append((linkname, pathname)) continue if cmtconfig in path: continue # ignore some common cruft if '/cmt/' in path: if '.root' == path[-5:]: continue if '.log' == path[-4:]: continue if '.out' == path[-4:]: continue filename = path.split('/')[-1] if 'setup' in filename: continue if 'cleanup' in filename: continue if 'Makefile' == filename: continue grist.append(path) continue continue # some extra by hand projdir = fs.projects()[snip:] grist.append(os.path.join(projdir, 'setup')) grist.append(os.path.join(projdir, 'setup.sh')) grist.append(os.path.join(projdir, 'setup.csh')) return grist
def cvscmd(cmdstr): return cmd('%s %s'%(cvsexe(),cmdstr),output=True)
def setup_bare(path): if not os.path.isdir(path): log.debug("About to init a bare repository at {}".format(path)) cmd(None, ["git", "init", "--bare", path])
def svncmd(cmdstr): return cmd('%s %s'%(svnexe(),cmdstr),output=True)
def gitcmd(cmdstr): from command import source env = source(fs.projects() + "/setup.sh") return cmd("%s %s" % (gitexe(), cmdstr), env=env, output=True)
try: action_autopilot = raw_input('For autopilot: Nothing (0), Reboot (1), Shutdown (2) ') try: action_autopilot = int(action_autopilot) if action_autopilot < 0 or action_autopilot > 2: print('The input has to be 0, 1 or 2') except: print('Input invalid. Doing nothing') action_autopilot = 0 except: print('Input invalid. Doing nothing') action_autopilot = 0 try: action_computer = raw_input('For computer: Nothing (0), Reboot (1), Shutdown (2) ') try: action_computer = int(action_computer) if action_computer < 0 or action_computer > 2: print('The input has to be 0, 1 or 2') except: print('Input invalid. Doing nothing') action_computer = 0 except: print('Input invalid. Doing nothing') action_computer = 0 mavfile.mav.command_long_send(mavfile.target_system, mavfile.target_component,command_id , confirmation,action_autopilot,action_computer,0,0,0,0,0) cmd_reboot_pxh = command.cmd('reboot_pxh', reboot_pxh) cmd_reboot_pxh.execute_cmd()
#!/usr/bin/env python """ Testing the connection and bandwidth """ import command def test_bw(mavfile): mavfile.mav.command_long_send(1, 0, 92, 1, 1, 0, 0, 0, 0, 0, 0) cmd_test_connection = command.cmd("test_connection", test_bw) cmd_test_connection.execute_cmd()
mavfile.mav.total_packets_received-rec_prev, (mavfile.mav.total_packets_received-rec_prev)/(t_now - t_start), 0.001*(mavfile.mav.total_bytes_received-bytes_recv)/(t_now - t_start))) break #print("%u sent, %u received, %u errors bwin=%.1f kB/s bwout=%.1f kB/s" % ( #mavfile.mav.total_packets_sent, #mavfile.mav.total_packets_received, #mavfile.mav.total_receive_errors, #0.001*(mavfile.mav.total_bytes_received-bytes_recv)/(t2-t1), #0.001*(mavfile.mav.total_bytes_sent-bytes_sent)/(t2-t1))) #bytes_sent = mavfile.mav.total_bytes_sent #bytes_recv = mavfile.mav.total_bytes_received #rec_list.append(mavfile.mav.total_packets_received - rec_prev) #rec_prev = mavfile.mav.total_packets_received #t1 = t2 #time_now = time.time() #t_diff = time_now - time_start #if (t_diff) > secs: #print("4 seconds passed. Ending testcon") #rec_list = rec_list[1:] #print(rec_list) #print(np.mean(rec_list)) #break cmd_test_connection = command.cmd('test_connection', test_bw) cmd_test_connection.execute_cmd()
name, value, data = receive_msg(mavfile, msg_type, time) else: pass if log == True: #write_log_number_line(cmt) log_data(name, path) log_data(value, path) else: pass for line in data: print(line) while not command.end_entered() and run: new_name, new_value, new_data = receive_msg(mavfile, msg_type, time) #Only print further data if it differs from the first one if not matching(new_value,value, tol, tol_value): if log == True: log_data(new_value, path) else: pass print('Values changed:') for line in new_data: print(line) value= new_value if time_now()-time_start > 120: break cmd_print_out_received_data = command.cmd('print_out_received_data', print_out_received_data) cmd_print_out_received_data.execute_cmd()
#!/usr/bin/env python ''' Moving the Servo from Left to Right ''' import command import util_usr from time import time, sleep def servo_move_left_right(mavfile): unused = 0xFFFF t_start = time() pwm = 1000 # Run for 60 seconds while time()-t_start < 60: if pwm <= 2000: pwm += 50 else: pwm = 1000 sleep(1) mavfile.mav.rc_channels_override_send(mavfile.target_system,mavfile.target_component,unused,unused,unused,pwm,unused,unused,unused,unused) sleep(0.01) sleep(1) mavfile.mav.rc_channels_override_send(0,0,0,0,0,1500,0,0,0,0) mavfile.mav.rc_channels_override_send(0,0,0,0,0,0,0,0,0,0) cmd_servo_move_left_right = command.cmd('servo_move_left_right', servo_move_left_right) cmd_servo_move_left_right.execute_cmd()
def runfun_btn_clicked(a): exec_sys = machines.value app = exec_to_app[exec_sys] app_data = all_apps[app] queue = queues.value with logOp: setvar("APP_NAME=%s" % app) setvar("STORAGE_MACHINE=%s" % app_data["storage_sys"]) setvar("EXEC_MACHINE=%s" % exec_sys) setvar("QUEUE=%s" % queue) ppn = 1 for i in range(len(app_data["queues"])): if app_data["queues"][i]["name"] == queue: ppn = int(app_data["queues"][i]["ppn"]) procs = get_procs() nodes = procs[0]//ppn if procs[0] % ppn != 0: nodes += 1 with logOp: if (tab_nest.children[0].children[1].value == False): cmd("rm -fr input") cmd("mkdir input") if (cur_model == "swan"): cmd("mv input_" + cur_model + "/input_tmp.txt input_" + cur_model + "/INPUT") cmd("cp -rT input_" + cur_model + " input") if (cur_model == "funwave" or cur_model == "cactus"): cmd("mv input_" + cur_model + "/input_tmp.txt input_" + cur_model + "/input.txt") modInput(procs[0], "input_funwave/input.txt") cmd("cp input_" + cur_model + "/input.txt input") cmd("cp input_" + cur_model + "/depth.txt input") if (cur_model == "openfoam"): cmd("cp -a input_" + cur_model + "/tutorials/"+ofCaseName.value[:-1]+"/. input") modify_openfoam(ofCaseName.value) cmd("tar cvzf input.tgz input") setvar("INPUT_DIR=${AGAVE_USERNAME}_$(date +%Y-%m-%d_%H-%M-%S)") #cmd("files-mkdir -S ${STORAGE_MACHINE} -N inputs/${INPUT_DIR}") #cmd("files-upload -F input.tgz -S ${STORAGE_MACHINE} inputs/${INPUT_DIR}/") cmd("files-mkdir agave://${STORAGE_MACHINE}/inputs/${INPUT_DIR}") cmd("files-cp input.tgz agave://${STORAGE_MACHINE}/inputs/${INPUT_DIR}/") submitJob(nodes, procs[0], cur_model, jobNameText.value, machines.value, queues.value)
# handle args parser = argparse.ArgumentParser(description="import router config") parser.add_argument('configs', metavar='F', type=str, nargs='+', help='a config file to import') args = parser.parse_args() # get config config = [] for file_name in args.configs: with open(file_name, 'r') as file: config.extend(file.readlines()) print "config", config # parse config def parseConfig(config): command = None for line in config: if line.startswith('/'): if command: yield "".join(command) command = [] command.extend(line) yield "".join(command) # send config over ssh for command in parseConfig(config): print "Asfd" print command cmd(command)
def jobList_btn_clicked(a): with logOp: cout = cmd("jobs-list -l 10") out1 = cout["stdout"] jobSelect.options = out1
while not command.end_entered(): pwm_ch3 = raw_input('PWM value (range:1000-2000, safety and test purposes limited 1400-1600) for channel 3 or end/0 : ') if pwm_ch3 == 'end': set_motor_pwm(mavfile, 1500) print('Module override motor ended. Releasing the channel') release_channel(mavfile) break try: pwm_ch3 = int(pwm_ch3) except: print('couldnt handle input') pass if pwm_ch3 >= 1000 and pwm_ch3 <= 2000 and isinstance(pwm_ch3, int): set_motor_pwm(mavfile, pwm_ch3) elif pwm_ch3 == 0: #First 2 parameters are target_system, target_component, last 8 are to release override #mavfile.mav.rc_channels_override_send(0,0,0,0,0,0,0,0,0,0) set_motor_pwm(mavfile, 1500) print('Motor in idle mode: PWM 1500') pass else: print('PWM has to be integer between 1400 and 1550') cmd_set_pwm_ch3 = command.cmd('set_pwm_ch3', set_pwm_ch3) cmd_set_pwm_ch3.execute_cmd()
import requests, re, os from command import cmd import json jsonurl = "https://sourceforge.net/projects/swanmodel/best_release.json" r = requests.get(jsonurl) jd = json.loads(r.text) url = jd["platform_releases"]["linux"]["url"] filename = jd["platform_releases"]["linux"]["filename"] ftar = re.sub(r'.*/','',filename) fbase = re.sub(r'.tar.gz$','',ftar) cmd("curl -kL %s -o %s" % (url,fbase)) cmd("tar xzvf %s" % fbase) os.chdir(fbase) cmd("make config") cmd("make mpi") cmd("cp swan.exe /usr/local/bin")
set_motor(mavfile,1530) sleep(t_sleep) print('Turning off motor') set_motor(mavfile, convert_speed(v = 0)) sleep(1) print('idle') set_servo(mavfile, 1500) release_channel(mavfile) print('done') elif mode == 'line': print('Forward starting') set_motor(mavfile, 1530) sleep(t_sleep) print('Stopping for 2 seconds') set_motor(mavfile, convert_speed(v = 0)) sleep(2) print('Backward starting') set_motor(mavfile, 1470) sleep(t_sleep) print('idle') set_motor(mavfile, convert_speed(v = 0)) release_channel(mavfile) elif mode == 'idle': print('doing nothing') else: print('Mode not supported (yet)') start_autopilot = command.cmd('autopilot', autopilot) start_autopilot.execute_cmd()
while not command.end_entered(): pwm_ch4 = raw_input('PWM value for channel 4 or end : ') #pwm_ch4 = int(pwm_ch4) #mavfile.mav.rc_channels_override_send(mavfile.target_system,mavfile.target_component,unused,unused,unused,pwm_ch4,unused,unused,unused,unused) if pwm_ch4 == 'end': set_servo_pwm(mavfile, 1500) time.sleep(1) print('Releasing channel. Ending now') release_channel(mavfile) break try: pwm_ch4 = int(pwm_ch4) except: print('couldnt handle input') pass if pwm_ch4 >= 1000 and pwm_ch4 <= 20000 and isinstance(pwm_ch4, int): print("OK") #print(mavfile.target_system) #print(mavfile.target_component) #mavfile.mav.rc_channels_override_send(1,1,unused,unused,unused,pwm_ch4,unused,unused,unused,unused) #mavfile.mav.rc_channels_override_send(mavfile.target_system,mavfile.target_component,unused,unused,unused,pwm_ch4,unused,unused,unused,unused) set_servo_pwm(mavfile, pwm_ch4) else: print('PWM has to be integer between 1000 and 2000') cmd_set_pwm_ch4 = command.cmd('set_pwm_ch4', set_pwm_ch4) cmd_set_pwm_ch4.execute_cmd()
##################################### OpenFoam Input tab ###################################### ofCaseName = Dropdown() ofOwnCaseName = Dropdown(options=["Select Input Case of Own"]) ofCbox = Checkbox(value = False, description = "Use Own Input") casesTutorials = ["Select Input Case of Tutorials"] casesOwn = ["Select Input Case of Own"] with logStash: cmd("tar -xvf input_openfoam.tgz") with open("input_openfoam/tutorials/cases.txt", 'r') as fr: lines = fr.readlines() for line in lines: casesTutorials.append(line) ofCaseName.options = casesTutorials def ofCbox_change(change): if change['type'] == 'change' and change['name'] == 'value': if(change['new'] == True): ownfiles = os.listdir("input_openfoam/foam_run/") ofOwnCaseName.options = ownfiles if(change['new'] == False): ofOwnCaseName.options = ["Select Input Case of Own"] ofCbox.observe(ofCbox_change)
import command, os, shutil def clear_log(mavfile): confirm = raw_input('Are you sure to clear the log? (y/n)') if confirm == 'y': log = open('log_received_data.txt', 'w') log.write('log_number: 1') else: pass cmd_clear_log = command.cmd('clear_log', clear_log) cmd_clear_log.execute_cmd()
print(csv_obj) else: print('Input not accepted') elif chose == 'plot': path = get_log_path() if path == 'end': break else: pass x_val, y_val = get_column_from_path(path) plt.figure(1) print('Implement title here') plt.title('to be implemented') ts2 = pd.Series(y_val, x_val) plt.axis([min(x_val), max(x_val), min(y_val), max(y_val)]) ts2.plot() plt.show() x_min = min(x_val) x_max = max(x_val) y_min = min(y_val) y_max = max(y_val) print('x_min: ' + str(x_min) + ' x_max: ' + str(x_max) + ' x_diff: ' + str(x_max - x_min)) print('y_min: ' + str(y_min) + ' y_max: ' + str(y_max) + ' y_diff: ' + str(y_max - y_min)) else: print('Chose plot or print (lower case)') cmd_open_log = command.cmd('open_log', open_log) cmd_open_log.execute_cmd()