def __deepen(self, depth):
cmd(self.__bare_path(), ["git",
"fetch",
"--deepen",
str(depth),
self.cache.repository,
self.__ref()])
def check_debian():
arch = cmd('dpkg --print-architecture',output=True).strip()
mach = cmd('uname -m',output=True).strip()
if arch == 'i386' and mach == 'x86_64':
log.warning('''Your kernel claims to be x86_64 but your user space claims to be i386. This will greatly confuse the various build systems. To work around this issue, either install an i686 kernel or run "linux32 /bin/bash" before installing or running the code.''')
return False
return True
def hsPreprocess_btn_clicked(a):
cmd("mkdir -p output/hsTmp")
for m in range(120):
hs_array_time2 = []
for j in range(109):
fo = open(opdir + 'hs', "r+")
datalines1 = fo.readlines()[9 + j + 231 * m]
p1 = datalines1.split()
for i in range(1, 152):
hs_array_time2.append(p1[i])
fo = open(opdir + 'hs', "r+")
datalines2 = fo.readlines()[122 + j + 231 * m]
p2 = datalines2.split()
for i in range(1, 31):
hs_array_time2.append(p2[i])
fo.close()
hsfilename = opdir + 'hsTmp/hs_' + str(m + 1)
hs_file_object_time2 = open(hsfilename, 'w')
for ip in hs_array_time2:
if (ip == "-900."):
ip = "NaN"
hs_file_object_time2.write(ip)
hs_file_object_time2.write('\n')
hs_file_object_time2.close()
fig_size = []
fig_size.append(15)
fig_size.append(8)
def build():
'Build CMT in previously unpacked cmt.srcdir().'
log.info('building cmt')
fs.goto(os.path.join(srcdir(),'mgr/'))
from command import cmd,make,source
def stupid_hack():
# quick hack for v1r22. this really doesn't belong here!
fp = open("../source/cmt_std.h")
for line in fp:
if 'climits' in line: return
continue
fp.close()
fp = open("../source/cmt_std.h","a")
fp.write("#include <climits>\n")
fp.close()
return
stupid_hack()
# always run this in case user does something silly like move the
# installation somewhere else
cmd('./INSTALL')
environ = env()
cmtexe = '%s/%s/cmt'%(srcdir(),environ['CMTCONFIG'])
if os.path.exists(cmtexe):
log.info('CMT appears to already have been built: %s'%cmtexe)
else:
log.info('CMT rebuild, not found at: %s'%cmtexe)
make(env=environ)
return cmtexe
def abort_btn_clicked(a):
g = re.match(r'^\S+',jobSelect.value)
with logOp:
if g:
jobid = g.group(0)
rcmd = "jobs-stop "+jobid
cmd(rcmd)
def _unpack_tar(self):
tarcmd = "tar -x"
if 'gz' in self.tarfilename: tarcmd += "z"
tarcmd += "f %s"%self.tarfilename
from command import cmd
cmd(tarcmd, dir=self.cli.opts.unpack_directory)
return
def make_archive(self):
if not self.__exists():
log.warning("Fetching tag '{}' from {}".format(
self.tag,
self.cache.repository
))
with self.cache.lock():
bare_path = self.__bare_path()
ref = "refs/tags/{}".format(self.tag)
setup_bare(bare_path)
try:
cmd(bare_path, ["git",
"fetch",
"--depth=1",
self.cache.repository,
ref])
except subprocess.CalledProcessError as e:
log.debug(
"Did not find tag {}, instead got output: {}".format(
self.tag, e.output
)
)
raise NixError(
"tag-not-found",
"Tag {} not found".format(self.tag)
)
archive_ref(bare_path, "FETCH_HEAD", self.__archive_path())
return self.__archive_path()
def load(reload=False, trace=False):
if not reload:
try:
with open("app-data.pickle", "rb") as fd:
return pickle.load(fd)
except:
pass
# Get the username
with open(os.environ["HOME"] + "/.agave/current", "r") as fd:
jd = json.loads(fd.read())
user = jd["username"]
all_apps = {}
out = cmd('apps-list', show=False, trace=trace)
for line in out["stdout"]:
app = line
if not re.match(r'crcollab', app):
continue
# Get permissions for the app
pout = cmd("apps-pems-list -v -u " + user + " " + app,
show=False,
trace=trace)
pdata = json.loads(" ".join(pout["stdout"]))
perm = pdata["permission"]
perms = ""
if perm["read"]:
perms += "R"
if perm["write"]:
perms += "W"
if perm["execute"]:
perms += "X"
appdata = cmd('apps-list -V ' + app, show=False, trace=True)
jtxt = " ".join(appdata["stdout"])
jd = json.loads(jtxt)
jd = jd["result"]
exec_sys = jd["executionSystem"]
storage_sys = jd["deploymentSystem"]
execdata = cmd('systems-list -v ' + exec_sys, show=False, trace=trace)
jd2 = json.loads(" ".join(execdata["stdout"]))
queues = []
for q in range(len(jd2["queues"])):
name = jd2["queues"][q]["name"]
ppn = jd2["queues"][q]["maxProcessorsPerNode"]
queues += [{"name": name, "ppn": ppn}]
all_apps[app] = {
"perm": perms,
"exec_sys": exec_sys,
"storage_sys": storage_sys,
"queues": queues
}
with open("app-data.pickle", "wb") as fd:
pickle.dump(all_apps, fd)
return all_apps
def _fix_cmt(self):
import os, cmt
from command import cmd
newdir = os.path.join(self.dstdir,'external/CMT',cmt.ver(),'mgr')
cmd('./INSTALL',env=None,dir=newdir)
import shutil
shutil.copy(os.path.join(newdir,'setup.sh'), os.path.join(self.dstdir,'projects/setup/00_cmt.sh'))
shutil.copy(os.path.join(newdir,'setup.csh'), os.path.join(self.dstdir,'projects/setup/00_cmt.csh'))
return
def __fetch_all(self):
try:
cmd(self.__bare_path(), ["git",
"fetch",
"--unshallow",
self.cache.repository,
self.__ref()])
except subprocess.CalledProcessError:
# We may already have unshallowed this repository
pass
def __commit_exists(self):
try:
cmd(self.__bare_path(), ["git", "cat-file", "-e",
"{}^{{commit}}".format(self.revision)])
return True
except subprocess.CalledProcessError as e:
log.debug(
"Did not find commit {} and instead got output: {}".format(
self.revision, e.output
)
)
return False
def configure2(agave_username, exec_machine, storage_name, project_name):
setvar("""
AGAVE_USERNAME=""" + agave_username + """
APP_NAME=""" + project_name + """
AGAVE_JSON_PARSER=jq
AGAVE_CACHE_DIR=$HOME/.agave
PATH=$HOME/agave-model/cli/bin:$PATH
STORAGE_MACHINE=""" + storage_name + """
DEPLOYMENT_PATH=agave-deployment
EXEC_MACHINE=""" + exec_machine + """
QUEUE=checkpt
""")
readpass("AGAVE_PASSWD")
readpass("PBTOK")
cmd("tenants-init -t agave.prod")
cmd("clients-delete -u $AGAVE_USERNAME -p $AGAVE_PASSWD $APP_NAME",
show=False)
cmd("clients-create -p $AGAVE_PASSWD -S -N $APP_NAME -u $AGAVE_USERNAME",
show=False)
cmd("auth-tokens-create -u $AGAVE_USERNAME -p $AGAVE_PASSWD", show=False)
setvar("[email protected]")
print("Successfully configured Agave")
def cmd_inrel(self, cmdstr):
'Run a command in the release package. No-op if none defined.'
relpkg = self.rel_pkg()
if not relpkg:
err = 'Project %s has no release package defined'%self.name
log.warning(err)
raise CommandFailure, err
import fs
relpkgcmt = os.path.join(fs.projects(), self.name, relpkg, 'cmt')
if not os.path.exists(relpkgcmt):
err = 'Project %s has release package defined, but no dir: %s'%(self.name, relpkgcmt)
log.warning(err)
raise CommandFailure, err
import command
command.cmd(cmdstr, env=self.env(), dir=relpkgcmt)
def template_on_change(change):
inputTmp = ''
if change['type'] == 'change' and change['name'] == 'value':
if(change['new'] == 'Choose Input Template'):
tab_nest.children[0].children[2].children = []
return
if(change['new'] == 'Basic Template'):
with logOp:
cmd("tar -xvf input_" + cur_model + ".tgz")
inputTmp = 'input_' + cur_model + '/basic_template.txt'
if(change['new'] == 'HDF5 Template'):
with logOp:
cmd("tar -xvf input_" + cur_model + ".tgz")
inputTmp = 'input_' + cur_model + '/hdf5_template.txt'
tab_nest.children[0].children[2].children = generatePara(inputTmp)
def archive_ref(source, ref, destination):
try:
os.makedirs(destination)
except FileExistsError:
pass
cmd(source,
[
"bash",
# Bash will run the following safe bash, and accept the defs
# of $1 and $2 via the following -s
"-c", "git archive --format=tar \"$1\" | tar -x -C \"$2\"",
# Passing ref and destination this way is much safer than
# using interpolation
"-s", ref, destination
])
def jobHis_btn_clicked(a):
g = re.match(r'^\S+', jobSelect.value)
with logOp:
if g:
jobid = g.group(0)
rcmd = "jobs-history -V " + jobid
cout = cmd(rcmd)
out1 = cout["stdout"]
jobHisSelect.options = out1
def jobOutput_btn_clicked(a):
g = re.match(r'^\S+',jobSelect.value)
with logOp:
if g:
jobid = g.group(0)
rcmd = "jobs-output-list "+ jobid
cout = cmd(rcmd)
out1 = cout["stdout"]
outputSelect.options = out1
def isjobexist():
query_terms = 'model='+modelTitle.value+'&model_ver='+modelVersionDd.value+'&mpich='+mpiDd.value+'&hdf5='+h5Dd.value+'&hypre='+hypreDd.value
query_cmd = "jobs-search 'status=FINISHED' 'parameters.like={*\"versions\":\""+query_terms+"\"*}'"
print (query_cmd)
out = cmd(query_cmd,show=False,trace=False)
result = out['stdout'][0]
if not result:
return False
return True
def download_btn_clicked(a):
with logOp:
try:
g = re.match(r'^\S+',jobSelect.value)
jobid = g.group(0)
if(outputSelect.value.find('.')==-1):
rcmd = "jobs-output-get -r "+ jobid +" "+ outputSelect.value
else:
rcmd = "jobs-output-get "+ jobid +" "+ outputSelect.value
print (rcmd)
os.system(rcmd)
if(outputSelect.value == 'output.tar.gz'):
cmd("rm -fr output")
cmd("mkdir -p output_" + jobid)
cmd('tar -xvf output.tar.gz -C ' + jobid)
elif(re.match(r'.*\.(txt|out|err|ipcexe)',outputSelect.value)):
with open(outputSelect.value,'r') as fd:
for line in fd.readlines():
print(line,end='')
else:
print('value=(',outputSelect.value,')',sep='')
except Exception as ex:
print("DIED!",ex)
def build():
"Build git in previously unpacked git.srcdir()."
log.info("building git")
fs.goto(srcdir())
from command import cmd, make
instpath = prefix()
if not os.path.exists("config.status"):
log.info("configuring git")
cmd("./configure --prefix=%s" % instpath)
if os.path.exists("git"):
log.info("git appears to already have been built")
else:
make()
if os.path.exists(instpath):
log.info("git appears to already have been installed to %s" % instpath)
else:
make("install")
return
def pack_garpi(self):
'''
Add garpi to the pack.
'''
import command
files = []
out = command.cmd("find garpi -type f -print", output=True)
for path in out.split('\n'):
if not self._keeper(path):
continue
if path[:2] == './': path = path[2:]
files.append(path)
continue
self.pack_files(files)
return
def cmt(cmdstr='', environ=None, dir=None, output=False):
"""Run 'cmt [cmdstr]'. By default the environment will be that
provided by cmt.env(pkgdir=dir) unless an explicity environment is
given via the environ arg. The dir and output options are passed
to command.cmd."""
if not environ:
environ = env()
from command import cmd, source
try:
ret = cmd('cmt '+cmdstr, env=environ, dir=dir, output=output)
except OSError,msg:
print msg
print 'PATH:'
for p in environ['PATH'].split(':'):
print '\t',p
#for k,v in environ.iteritems():
# print '"%s" = "%s"'%(k,v)
raise
def update_btn_clicked(a):
if (tab_nest.children[0].children[1].value == True):
with logOp:
cmd("rm -f input.tgz")
cmd("rm -fr input")
cmd("cp -f ../input_" + cur_model + ".tgz input.tgz")
return
newInput = 'input_'+ cur_model + '/input_tmp.txt'
inputTmp = ''
if(tab_nest.children[0].children[0].value == 'Basic Template'):
inputTmp = 'input_' + cur_model + '/basic_template.txt'
if(tab_nest.children[0].children[0].value == 'HDF5 Template'):
inputTmp = 'input_' + cur_model + '/basic_template.txt'
updatePara(inputTmp, newInput, tab_nest.children[0].children[2])
tab_nest.children[0].children[4].value = open(newInput, 'r').read()
def ofUpInput_btn_clicked(a):
if (ofCbox.value == True):
with logOp:
cmd("rm -f input.tgz")
cmd("rm -fr input")
cmd("cp -f input_openfoam/foam_run/" + ofOwnCaseName.value + " input.tgz")
return
if not ofCaseName.value == "Select Input Case":
with open("input_openfoam/tutorials/" + ofCaseName.value[:-1] + "/system/decomposeParDict", "r") as fd:
contents = fd.read()
ofInputArea.value = contents
sp = r'(?:\s|//.*)'
pat = re.sub(r'\\s',sp,r'coeffs\s*{\s*n\s+\(\s*(\d+)\s+(\d+)\s+(\d+)\s*\)\s*;\s*}')
g = re.search(pat, contents)
if g:
numXSlider.value = g.group(1)
numYSlider.value = g.group(2)
numZSlider.value = g.group(3)
fd.close()
def buildBtn_clicked(a):
model_ver = modelTitle.value.upper() + "_VER"
mpi_ver = '' if mpiDd.value is "None" else mpiDd.value
h5_ver = '' if h5Dd.value is "None" else h5Dd.value
hypre_ver = '' if hypreDd.value is "None" else hypreDd.value
with logOp:
setvar("MPICH_VER=" + mpiDd.value)
setvar("HDF5_VER=" + h5Dd.value)
setvar("HYPRE_VER=" + hypreDd.value)
setvar("MODEL_VER=" + modelVersionDd.value)
writefile("env_setting.txt","""
#!/bin/bash
export """+model_ver+"""="""+modelVersionDd.value+"""
export MPICH_VER="""+mpi_ver+"""
export H5_VER="""+h5_ver+"""
export HYPRE_VER="""+hypre_ver+"""
""")
# cmd("rm -fr build_input")
# cmd("mkdir build_input")
# cmd("cp env_setting build_input/")
exist = isjobexist()
if exist is True:
with msgOut:
print ("The job exist, no build job would be submitted!")
else:
setvar("INPUT_DIR=${AGAVE_USERNAME}_$(date +%Y-%m-%d_%H-%M-%S)")
cmd("files-mkdir -S ${STORAGE_MACHINE} -N inputs/${INPUT_DIR}")
cmd("files-upload -F env_setting.txt -S ${STORAGE_MACHINE} inputs/${INPUT_DIR}/")
submitBuildJob(machines.value, queues.value)
# set job permissions to users
users = ['ysboss','tg457049','lzhu','nanw','reza']
for user in users:
cmd('jobs-pems-update -u '+user+' -p READ ${JOB_ID}')
def configure(agave_username, machine_username, machine_name, project_name):
g = re.match(r'(\w+)\.(.*)', machine_name)
os.environ["MACHINE"] = g.group(1)
os.environ["DOMAIN"] = g.group(2)
os.environ["MACHINE_FULL"] = machine_name
setvar("""
AGAVE_USERNAME=""" + agave_username + """
MACHINE_USERNAME=""" + machine_username + """
BASE_APP_NAME=crcollaboratory
PORT=22
ALLOCATION=hpc_startup_funwave
WORK_DIR=/work/${MACHINE_USERNAME}
HOME_DIR=/home/${MACHINE_USERNAME}
SCRATCH_DIR=/work/${MACHINE_USERNAME}
DEPLOYMENT_PATH=agave-deployment
AGAVE_JSON_PARSER=jq
PATH=$HOME/agave-model/cli/bin:$PATH
""")
readpass("MACHINE_PASSWD")
readpass("AGAVE_PASSWD")
readpass("PBTOK")
setvar("APP_NAME=${BASE_APP_NAME}-${MACHINE}-${AGAVE_USERNAME}")
#cmd("git clone https://bitbucket.org/agaveapi/cli.git")
cmd("tenants-init -t agave.prod")
cmd("clients-delete -u $AGAVE_USERNAME -p $AGAVE_PASSWD $APP_NAME",
show=False)
cmd("clients-create -p $AGAVE_PASSWD -S -N $APP_NAME -u $AGAVE_USERNAME",
show=False)
cmd("auth-tokens-create -u $AGAVE_USERNAME -p $AGAVE_PASSWD", show=False)
cmd("auth-tokens-refresh")
setvar("STORAGE_MACHINE=${MACHINE}-storage-${AGAVE_USERNAME}3")
writefile(
"${STORAGE_MACHINE}.txt", """{
"id": "${f
}",
"name": "${MACHINE} storage (${MACHINE_USERNAME})",
"description": "The ${MACHINE} computer",
"site": "${DOMAIN}",
"type": "STORAGE",
"storage": {
"host": "${MACHINE_FULL}",
"port": ${PORT},
"protocol": "SFTP",
"rootDir": "/",
"homeDir": "${HOME_DIR}",
"auth": {
"username" : "${MACHINE_USERNAME}",
"password" : "${MACHINE_PASSWD}",
"type" : "PASSWORD"
}
}
}
""")
cmd("systems-addupdate -F ${STORAGE_MACHINE}.txt")
#cmd("sshpass -f MACHINE_PASSWD.txt ssh -o StrictHostKeyChecking=no ${MACHINE_USERNAME}@${MACHINE_FULL} -p ${PORT} (sinfo || qstat -q)")
setvar("""
# Gather info about the machine
# Executing this cell is essential
MAX_TIME=72:00:00 # Max duration of a job
# We figure out the number of processes automatically.
# This assumes the head node and compute nodes have
# the same number of procs.
CPUINFO=$(sshpass -f MACHINE_PASSWD.txt ssh -o StrictHostKeyChecking=no ${MACHINE_USERNAME}@${MACHINE_FULL} -p ${PORT} lscpu)
QUEUE=checkpt # Name of default queue
NODES=42 # Number of nodes in queue
""")
g = re.search(r'(?m)CPU\(s\):\s*(\d+)', os.environ["CPUINFO"])
os.environ["PROCS"] = g.group(1)
print(repvar("PROCS=$PROCS"))
setvar("EXEC_MACHINE=${MACHINE}-exec-${AGAVE_USERNAME}")
os.environ["DIRECTIVES"] = re.sub(
"\n\\s*", r"\\n", """
#PBS -A ${ALLOCATION}
#PBS -l cput=\${AGAVE_JOB_MAX_RUNTIME}
#PBS -l walltime=\${AGAVE_JOB_MAX_RUNTIME}
#PBS -q \${AGAVE_JOB_BATCH_QUEUE}
#PBS -l nodes=\${AGAVE_JOB_NODE_COUNT}:ppn=16
""".strip())
writefile(
"${EXEC_MACHINE}.txt", """
{
"id": "${EXEC_MACHINE}",
"name": "${MACHINE} (${MACHINE_USERNAME})",
"description": "The ${MACHINE} computer",
"site": "${DOMAIN}",
"public": false,
"status": "UP",
"type": "EXECUTION",
"executionType": "HPC",
"scheduler" : "CUSTOM_TORQUE",
"environment": null,
"scratchDir" : "${SCRATCH_DIR}",
"queues": [
{
"customDirectives" : "${DIRECTIVES}",
"name": "${QUEUE}",
"default": true,
"maxJobs": 10,
"maxUserJobs": 10,
"maxNodes": ${NODES},
"maxProcessorsPerNode": ${PROCS},
"minProcessorsPerNode": 1,
"maxRequestedTime": "${MAX_TIME}"
}
],
"login": {
"auth": {
"username" : "${MACHINE_USERNAME}",
"password" : "${MACHINE_PASSWD}",
"type" : "PASSWORD"
},
"host": "${MACHINE_FULL}",
"port": ${PORT},
"protocol": "SSH"
},
"maxSystemJobs": 50,
"maxSystemJobsPerUser": 50,
"storage": {
"host": "${MACHINE_FULL}",
"port": ${PORT},
"protocol": "SFTP",
"rootDir": "/",
"homeDir": "${HOME_DIR}",
"auth": {
"username" : "${MACHINE_USERNAME}",
"password" : "${MACHINE_PASSWD}",
"type" : "PASSWORD"
}
}
},
"workDir": "${WORK_DIR}"
}""")
cmd("systems-addupdate -F ${EXEC_MACHINE}.txt")
writefile(
"${APP_NAME}-wrapper.txt", """
#!/bin/bash
handle_trap() {
rc=\$?
if [ "\$rc" != 0 ]
then
\$(\${AGAVE_JOB_CALLBACK_FAILURE})
fi
}
trap 'handle_trap' ERR EXIT
echo 'running \${simagename} model'
# Setting the x flag will echo every
# command onto stderr. This is
# for debugging, so we can see what's
# going on.
set -x
set -e
echo ==PWD=============
# We also print out the execution
# directory. Again, for debugging purposes.
pwd
echo ==JOB=============
if [ "\${PBS_NODEFILE}" = "" ]
then
# When running on a system managed by Torque
# this variable should be set. If it's not,
# that's a problem.
echo "The PBS_NODEFILE was not set"
\$(\${AGAVE_JOB_CALLBACK_FAILURE})
exit 2
fi
# By default, the PBS_NODEFILE lists nodes multiple
# times, once for each MPI process that should run
# there. We only want one MPI process per node, so
# we create a new file with "sort -u".
LOCAL_NODEFILE=nodefile.txt
sort -u < \${PBS_NODEFILE} > \${LOCAL_NODEFILE}
PROCS=\$(wc -l < \${LOCAL_NODEFILE})
if [ "\${PROCS}" = "" ]
then
echo "PROCS was not set"
\$(\${AGAVE_JOB_CALLBACK_FAILURE})
exit 3
fi
# Prepare the nodes to run the image
export SING_OPTS="--bind \$PWD:/workdir \$SING_OPTS"
for host in \$(cat nodefile.txt)
do
hostfile="\$HOME/.bash.\${host}.sh"
echo "export SING_IMAGE=/project/sbrandt/chemora/images/\${simagename}.simg" > \$hostfile
echo "export SING_OPTS='\$SING_OPTS'" >> \$hostfile
done
# Create a nodefile that matches our choices at submit time
touch nodes.txt
for i in \$(seq 1 \${AGAVE_JOB_PROCESSORS_PER_NODE})
do
cat nodefile.txt >> nodes.txt
done
export NP=\$(wc -l < nodes.txt)
tar xzvf input.tgz
mkdir -p output
/project/singularity/bin/singularity exec \$SING_OPTS /project/sbrandt/chemora/images/\${simagename}.simg bash /usr/local/bin/runapp.sh
mv input output
rm -f output/PRINT*
tar cvzf output.tar.gz output
""")
cmd("files-mkdir -S ${STORAGE_MACHINE} -N ${DEPLOYMENT_PATH}")
cmd("files-mkdir -S ${STORAGE_MACHINE} -N inputs")
cmd("files-upload -F ${APP_NAME}-wrapper.txt -S ${STORAGE_MACHINE} ${DEPLOYMENT_PATH}/"
)
writefile("test.txt", """
parfile="input.txt"
${APP_NAME}-wrapper.txt
""")
cmd("files-mkdir -S ${STORAGE_MACHINE} -N ${DEPLOYMENT_PATH}")
cmd("files-upload -F test.txt -S ${STORAGE_MACHINE} ${DEPLOYMENT_PATH}/")
writefile(
"${APP_NAME}.txt", """
{
"name":"${APP_NAME}",
"version":"2.0",
"label":"${APP_NAME}",
"shortDescription":"Run ISAAC app",
"longDescription":"",
"deploymentSystem":"${STORAGE_MACHINE}",
"deploymentPath":"${DEPLOYMENT_PATH}",
"templatePath":"${APP_NAME}-wrapper.txt",
"testPath":"test.txt",
"executionSystem":"${EXEC_MACHINE}",
"executionType":"HPC",
"parallelism":"PARALLEL",
"allocation":"${ALLOCATION}",
"modules":[],
"inputs":[
{
"id":"input tarball",
"details":{
"label":"input tarball",
"description":"",
"argument":null,
"showArgument":false
},
"value":{
"default":"",
"order":0,
"required":false,
"validator":"",
"visible":true
}
}
],
"parameters":[
{
"id": "simagename",
"value": {
"visible": true,
"required": false,
"type": "string",
"order": 0,
"enquote": false,
"default": "python",
"validator": null
},
"details": {
"label": "Singularity Image",
"description": "The Singularity image to run: swan, funwave",
"argument": null,
"showArgument": false,
"repeatArgument": false
},
"semantics": {
"minCardinality": 0,
"maxCardinality": 1,
"ontology": []
}
}
],
"outputs":[
{
"id":"Output",
"details":{
"description":"The output",
"label":"tables"
},
"value":{
"default":"",
"validator":""
}
}
]
}
""")
cmd("apps-addupdate -F ${APP_NAME}.txt")
setvar("APP_NAME=${APP_NAME}-2.0")
cmd("files-upload -F input.txt -S ${STORAGE_MACHINE}/")
setvar("[email protected]")
print("Successfully configured Agave")
pass
if option == 1:
param_id = int(raw_input('Type in ID: '))
name, value, data = receive_msg(mavfile, 'PARAM_VALUE', True, param_id = param_id)
print('----------\n' + value[0] + ': ' + str(value[1]) + '\n' + '----------')
elif option == 2 or option == '':
param_name = raw_input('Type in parameter name: ')
inv_dic = {v: k for k, v in dic.items()}
if '_' not in param_name:
param_name_splitted = param_name.split()
param_name = ''
for word in param_name_splitted:
param_name += word.upper()+'_'
param_name = param_name[:-1] #removing the last _ from the string
print('Changed input to: ' + param_name)
else:
pass
param_id = int(inv_dic[param_name])
name, value, data = receive_msg(mavfile, 'PARAM_VALUE', True, param_id = param_id)
print('----------\n' + value[0] + ': ' + str(value[1]) + '\n' + '----------')
elif option == 3:
get_pos_param(mavfile, 'print')
else:
print('Option not defined')
cmd_get_param = command.cmd('get_param', get_param)
cmd_get_param.execute_cmd()
import command
def change_data_stream(mavfile):
'''
Data Stream is deprecated. Using message intervall instead (pymavlink 1..61)
'''
#target_system = mavfile.target_system
#target_component = mavfile.target_component
mavfile.mav.message_interval_send(0,-1)
#req_stream_id = int(raw_input('Stream_id? 0-all, 1-raw_sensors, 2-ext_status, 3-rc_channels, 4-raw_controller, 6-position: '))
#req_message_rate = int(raw_input('Stream rate (in Hz): '))
#start_stop = int(raw_input('start/stop (1/0)'))
#mavfile.mav.request_data_stream_send(target_system, target_component, req_stream_id, req_message_rate, start_stop)
cds = command.cmd('change data stream', change_data_stream)
cds.execute_cmd()
def _project_files(self):
import os, command, fs
cmtconfig = os.getenv('CMTCONFIG')
project_area = fs.projects()
basedir = os.path.dirname(project_area)
snip = len(basedir)+1
grist = []
for proj in self.project_objects:
projdir = os.path.join(project_area, proj.name)
out = command.cmd("find . -type f -print",
dir=projdir, output=True)
for path in out.split('\n'):
if not self._keeper(path): continue
if path[:2] == './': path = path[2:]
path = os.path.join(projdir, path)
path = path[snip:]
# Handle InstallArea special
if '/InstallArea/' in path:
if path[-7:] != '.cmtref':
grist.append(path)
continue
# If not companion symlink, then give up and just add the file
linkname = path[:-7]
if not os.path.islink(path):
grist.append(path)
continue
# Resolve symlink and save both
pathname = readlink(linkname)
grist.append((linkname, pathname))
continue
if cmtconfig in path: continue
# ignore some common cruft
if '/cmt/' in path:
if '.root' == path[-5:]: continue
if '.log' == path[-4:]: continue
if '.out' == path[-4:]: continue
filename = path.split('/')[-1]
if 'setup' in filename: continue
if 'cleanup' in filename: continue
if 'Makefile' == filename: continue
grist.append(path)
continue
continue
# some extra by hand
projdir = fs.projects()[snip:]
grist.append(os.path.join(projdir, 'setup'))
grist.append(os.path.join(projdir, 'setup.sh'))
grist.append(os.path.join(projdir, 'setup.csh'))
return grist
def cvscmd(cmdstr):
return cmd('%s %s'%(cvsexe(),cmdstr),output=True)
def setup_bare(path):
if not os.path.isdir(path):
log.debug("About to init a bare repository at {}".format(path))
cmd(None, ["git", "init", "--bare", path])
def svncmd(cmdstr):
return cmd('%s %s'%(svnexe(),cmdstr),output=True)
def gitcmd(cmdstr):
from command import source
env = source(fs.projects() + "/setup.sh")
return cmd("%s %s" % (gitexe(), cmdstr), env=env, output=True)
try:
action_autopilot = raw_input('For autopilot: Nothing (0), Reboot (1), Shutdown (2) ')
try:
action_autopilot = int(action_autopilot)
if action_autopilot < 0 or action_autopilot > 2:
print('The input has to be 0, 1 or 2')
except:
print('Input invalid. Doing nothing')
action_autopilot = 0
except:
print('Input invalid. Doing nothing')
action_autopilot = 0
try:
action_computer = raw_input('For computer: Nothing (0), Reboot (1), Shutdown (2) ')
try:
action_computer = int(action_computer)
if action_computer < 0 or action_computer > 2:
print('The input has to be 0, 1 or 2')
except:
print('Input invalid. Doing nothing')
action_computer = 0
except:
print('Input invalid. Doing nothing')
action_computer = 0
mavfile.mav.command_long_send(mavfile.target_system, mavfile.target_component,command_id , confirmation,action_autopilot,action_computer,0,0,0,0,0)
cmd_reboot_pxh = command.cmd('reboot_pxh', reboot_pxh)
cmd_reboot_pxh.execute_cmd()
#!/usr/bin/env python
"""
Testing the connection and bandwidth
"""
import command
def test_bw(mavfile):
mavfile.mav.command_long_send(1, 0, 92, 1, 1, 0, 0, 0, 0, 0, 0)
cmd_test_connection = command.cmd("test_connection", test_bw)
cmd_test_connection.execute_cmd()
mavfile.mav.total_packets_received-rec_prev,
(mavfile.mav.total_packets_received-rec_prev)/(t_now - t_start),
0.001*(mavfile.mav.total_bytes_received-bytes_recv)/(t_now - t_start)))
break
#print("%u sent, %u received, %u errors bwin=%.1f kB/s bwout=%.1f kB/s" % (
#mavfile.mav.total_packets_sent,
#mavfile.mav.total_packets_received,
#mavfile.mav.total_receive_errors,
#0.001*(mavfile.mav.total_bytes_received-bytes_recv)/(t2-t1),
#0.001*(mavfile.mav.total_bytes_sent-bytes_sent)/(t2-t1)))
#bytes_sent = mavfile.mav.total_bytes_sent
#bytes_recv = mavfile.mav.total_bytes_received
#rec_list.append(mavfile.mav.total_packets_received - rec_prev)
#rec_prev = mavfile.mav.total_packets_received
#t1 = t2
#time_now = time.time()
#t_diff = time_now - time_start
#if (t_diff) > secs:
#print("4 seconds passed. Ending testcon")
#rec_list = rec_list[1:]
#print(rec_list)
#print(np.mean(rec_list))
#break
cmd_test_connection = command.cmd('test_connection', test_bw)
cmd_test_connection.execute_cmd()
name, value, data = receive_msg(mavfile, msg_type, time)
else:
pass
if log == True:
#write_log_number_line(cmt)
log_data(name, path)
log_data(value, path)
else:
pass
for line in data:
print(line)
while not command.end_entered() and run:
new_name, new_value, new_data = receive_msg(mavfile, msg_type, time)
#Only print further data if it differs from the first one
if not matching(new_value,value, tol, tol_value):
if log == True:
log_data(new_value, path)
else:
pass
print('Values changed:')
for line in new_data:
print(line)
value= new_value
if time_now()-time_start > 120:
break
cmd_print_out_received_data = command.cmd('print_out_received_data', print_out_received_data)
cmd_print_out_received_data.execute_cmd()
#!/usr/bin/env python
'''
Moving the Servo from Left to Right
'''
import command
import util_usr
from time import time, sleep
def servo_move_left_right(mavfile):
unused = 0xFFFF
t_start = time()
pwm = 1000
# Run for 60 seconds
while time()-t_start < 60:
if pwm <= 2000:
pwm += 50
else:
pwm = 1000
sleep(1)
mavfile.mav.rc_channels_override_send(mavfile.target_system,mavfile.target_component,unused,unused,unused,pwm,unused,unused,unused,unused)
sleep(0.01)
sleep(1)
mavfile.mav.rc_channels_override_send(0,0,0,0,0,1500,0,0,0,0)
mavfile.mav.rc_channels_override_send(0,0,0,0,0,0,0,0,0,0)
cmd_servo_move_left_right = command.cmd('servo_move_left_right', servo_move_left_right)
cmd_servo_move_left_right.execute_cmd()
def runfun_btn_clicked(a):
exec_sys = machines.value
app = exec_to_app[exec_sys]
app_data = all_apps[app]
queue = queues.value
with logOp:
setvar("APP_NAME=%s" % app)
setvar("STORAGE_MACHINE=%s" % app_data["storage_sys"])
setvar("EXEC_MACHINE=%s" % exec_sys)
setvar("QUEUE=%s" % queue)
ppn = 1
for i in range(len(app_data["queues"])):
if app_data["queues"][i]["name"] == queue:
ppn = int(app_data["queues"][i]["ppn"])
procs = get_procs()
nodes = procs[0]//ppn
if procs[0] % ppn != 0:
nodes += 1
with logOp:
if (tab_nest.children[0].children[1].value == False):
cmd("rm -fr input")
cmd("mkdir input")
if (cur_model == "swan"):
cmd("mv input_" + cur_model + "/input_tmp.txt input_" + cur_model + "/INPUT")
cmd("cp -rT input_" + cur_model + " input")
if (cur_model == "funwave" or cur_model == "cactus"):
cmd("mv input_" + cur_model + "/input_tmp.txt input_" + cur_model + "/input.txt")
modInput(procs[0], "input_funwave/input.txt")
cmd("cp input_" + cur_model + "/input.txt input")
cmd("cp input_" + cur_model + "/depth.txt input")
if (cur_model == "openfoam"):
cmd("cp -a input_" + cur_model + "/tutorials/"+ofCaseName.value[:-1]+"/. input")
modify_openfoam(ofCaseName.value)
cmd("tar cvzf input.tgz input")
setvar("INPUT_DIR=${AGAVE_USERNAME}_$(date +%Y-%m-%d_%H-%M-%S)")
#cmd("files-mkdir -S ${STORAGE_MACHINE} -N inputs/${INPUT_DIR}")
#cmd("files-upload -F input.tgz -S ${STORAGE_MACHINE} inputs/${INPUT_DIR}/")
cmd("files-mkdir agave://${STORAGE_MACHINE}/inputs/${INPUT_DIR}")
cmd("files-cp input.tgz agave://${STORAGE_MACHINE}/inputs/${INPUT_DIR}/")
submitJob(nodes, procs[0], cur_model, jobNameText.value, machines.value, queues.value)
# handle args
parser = argparse.ArgumentParser(description="import router config")
parser.add_argument('configs', metavar='F', type=str, nargs='+',
help='a config file to import')
args = parser.parse_args()
# get config
config = []
for file_name in args.configs:
with open(file_name, 'r') as file:
config.extend(file.readlines())
print "config", config
# parse config
def parseConfig(config):
command = None
for line in config:
if line.startswith('/'):
if command:
yield "".join(command)
command = []
command.extend(line)
yield "".join(command)
# send config over ssh
for command in parseConfig(config):
print "Asfd"
print command
cmd(command)
def jobList_btn_clicked(a):
with logOp:
cout = cmd("jobs-list -l 10")
out1 = cout["stdout"]
jobSelect.options = out1
while not command.end_entered():
pwm_ch3 = raw_input('PWM value (range:1000-2000, safety and test purposes limited 1400-1600) for channel 3 or end/0 : ')
if pwm_ch3 == 'end':
set_motor_pwm(mavfile, 1500)
print('Module override motor ended. Releasing the channel')
release_channel(mavfile)
break
try:
pwm_ch3 = int(pwm_ch3)
except:
print('couldnt handle input')
pass
if pwm_ch3 >= 1000 and pwm_ch3 <= 2000 and isinstance(pwm_ch3, int):
set_motor_pwm(mavfile, pwm_ch3)
elif pwm_ch3 == 0:
#First 2 parameters are target_system, target_component, last 8 are to release override
#mavfile.mav.rc_channels_override_send(0,0,0,0,0,0,0,0,0,0)
set_motor_pwm(mavfile, 1500)
print('Motor in idle mode: PWM 1500')
pass
else:
print('PWM has to be integer between 1400 and 1550')
cmd_set_pwm_ch3 = command.cmd('set_pwm_ch3', set_pwm_ch3)
cmd_set_pwm_ch3.execute_cmd()
import requests, re, os
from command import cmd
import json
jsonurl = "https://sourceforge.net/projects/swanmodel/best_release.json"
r = requests.get(jsonurl)
jd = json.loads(r.text)
url = jd["platform_releases"]["linux"]["url"]
filename = jd["platform_releases"]["linux"]["filename"]
ftar = re.sub(r'.*/','',filename)
fbase = re.sub(r'.tar.gz$','',ftar)
cmd("curl -kL %s -o %s" % (url,fbase))
cmd("tar xzvf %s" % fbase)
os.chdir(fbase)
cmd("make config")
cmd("make mpi")
cmd("cp swan.exe /usr/local/bin")
set_motor(mavfile,1530)
sleep(t_sleep)
print('Turning off motor')
set_motor(mavfile, convert_speed(v = 0))
sleep(1)
print('idle')
set_servo(mavfile, 1500)
release_channel(mavfile)
print('done')
elif mode == 'line':
print('Forward starting')
set_motor(mavfile, 1530)
sleep(t_sleep)
print('Stopping for 2 seconds')
set_motor(mavfile, convert_speed(v = 0))
sleep(2)
print('Backward starting')
set_motor(mavfile, 1470)
sleep(t_sleep)
print('idle')
set_motor(mavfile, convert_speed(v = 0))
release_channel(mavfile)
elif mode == 'idle':
print('doing nothing')
else:
print('Mode not supported (yet)')
start_autopilot = command.cmd('autopilot', autopilot)
start_autopilot.execute_cmd()
while not command.end_entered():
pwm_ch4 = raw_input('PWM value for channel 4 or end : ')
#pwm_ch4 = int(pwm_ch4)
#mavfile.mav.rc_channels_override_send(mavfile.target_system,mavfile.target_component,unused,unused,unused,pwm_ch4,unused,unused,unused,unused)
if pwm_ch4 == 'end':
set_servo_pwm(mavfile, 1500)
time.sleep(1)
print('Releasing channel. Ending now')
release_channel(mavfile)
break
try:
pwm_ch4 = int(pwm_ch4)
except:
print('couldnt handle input')
pass
if pwm_ch4 >= 1000 and pwm_ch4 <= 20000 and isinstance(pwm_ch4, int):
print("OK")
#print(mavfile.target_system)
#print(mavfile.target_component)
#mavfile.mav.rc_channels_override_send(1,1,unused,unused,unused,pwm_ch4,unused,unused,unused,unused)
#mavfile.mav.rc_channels_override_send(mavfile.target_system,mavfile.target_component,unused,unused,unused,pwm_ch4,unused,unused,unused,unused)
set_servo_pwm(mavfile, pwm_ch4)
else:
print('PWM has to be integer between 1000 and 2000')
cmd_set_pwm_ch4 = command.cmd('set_pwm_ch4', set_pwm_ch4)
cmd_set_pwm_ch4.execute_cmd()
##################################### OpenFoam Input tab ######################################
ofCaseName = Dropdown()
ofOwnCaseName = Dropdown(options=["Select Input Case of Own"])
ofCbox = Checkbox(value = False, description = "Use Own Input")
casesTutorials = ["Select Input Case of Tutorials"]
casesOwn = ["Select Input Case of Own"]
with logStash:
cmd("tar -xvf input_openfoam.tgz")
with open("input_openfoam/tutorials/cases.txt", 'r') as fr:
lines = fr.readlines()
for line in lines:
casesTutorials.append(line)
ofCaseName.options = casesTutorials
def ofCbox_change(change):
if change['type'] == 'change' and change['name'] == 'value':
if(change['new'] == True):
ownfiles = os.listdir("input_openfoam/foam_run/")
ofOwnCaseName.options = ownfiles
if(change['new'] == False):
ofOwnCaseName.options = ["Select Input Case of Own"]
ofCbox.observe(ofCbox_change)
import command, os, shutil
def clear_log(mavfile):
confirm = raw_input('Are you sure to clear the log? (y/n)')
if confirm == 'y':
log = open('log_received_data.txt', 'w')
log.write('log_number: 1')
else:
pass
cmd_clear_log = command.cmd('clear_log', clear_log)
cmd_clear_log.execute_cmd()
print(csv_obj)
else:
print('Input not accepted')
elif chose == 'plot':
path = get_log_path()
if path == 'end':
break
else:
pass
x_val, y_val = get_column_from_path(path)
plt.figure(1)
print('Implement title here')
plt.title('to be implemented')
ts2 = pd.Series(y_val, x_val)
plt.axis([min(x_val), max(x_val), min(y_val), max(y_val)])
ts2.plot()
plt.show()
x_min = min(x_val)
x_max = max(x_val)
y_min = min(y_val)
y_max = max(y_val)
print('x_min: ' + str(x_min) + ' x_max: ' + str(x_max) + ' x_diff: ' + str(x_max - x_min))
print('y_min: ' + str(y_min) + ' y_max: ' + str(y_max) + ' y_diff: ' + str(y_max - y_min))
else:
print('Chose plot or print (lower case)')
cmd_open_log = command.cmd('open_log', open_log)
cmd_open_log.execute_cmd()
