def __launch(self, command):
"""launch mechanism for prepared launch command"""
executable = command.split("|")[-1].split()[0]
from pox import which
if not which(executable):
raise IOError("launch failed: %s not found" % executable)
return Popen([command], shell=True) #FIXME: shell=True is insecure
def __launch(self, command):
"""launch mechanism for prepared launch command"""
executable = command.split("|")[-1].split()[0]
from pox import which
if not which(executable):
raise IOError, "launch failed: %s not found" % executable
return Popen([command], shell=True) #FIXME: shell=True is insecure
def executable_exist(module, prog):
try:
assert which(prog)
# process = Popen([prog, '--help'], stderr=STDOUT, stdout=PIPE)
# process.wait()
return True
except (OSError, AssertionError):
from sys import exc_info
print("%s:: Executable '%s' not found" % (module, prog))
#print("%s:: %s" % (prog, exc_info()[1]))
return False
def executable_exist(module, prog):
try:
assert which(prog)
# process = Popen([prog, '--help'], stderr=STDOUT, stdout=PIPE)
# process.wait()
return True
except (OSError, AssertionError):
from sys import exc_info
print("%s:: Executable '%s' not found" % (module, prog))
#print("%s:: %s" % (prog, exc_info()[1]))
return False
def which_strategy(scatter=True, lazy=False, fullpath=True):
"""try to autodetect an available strategy (scatter or pool)"""
target = 'ezscatter.py' if scatter else 'ezpool.py'
import sys
if (sys.platform[:3] == 'win'): lazy=False
if lazy: target = "`which %s`" % target
# lookup full path
elif not lazy and fullpath:
from pox import which
target = which(target, ignore_errors=True)
if not target: target = None #XXX: better None or "" ?
return target
def which_strategy(scatter=True, lazy=False, fullpath=True):
"""try to autodetect an available strategy (scatter or pool)"""
target = 'ezscatter' if scatter else 'ezpool'
import sys
if (sys.platform[:3] == 'win'): lazy = False
if lazy:
target = "`which %s`" % target
# lookup full path
elif not lazy and fullpath:
from pox import which
target = which(target, ignore_errors=True)
if not target: target = None #XXX: better None or "" ?
return target
def which_mpirun(mpich=None, fullpath=False):
"""try to autodetect an available mpi launcher
if mpich=True only look for mpich, if False only look for openmpi"""
import os
from pox import which
progs = ['mpiexec', 'mpirun', 'mpiexec-mpich-mp', 'mpiexec-openmpi-mp', 'mpirun-mpich-mp', 'mpirun-openmpi-mp']
if mpich == True: pop = 'openmpi'
elif mpich == False: pop = 'mpich'
else: pop = 'THIS IS NOT THE MPI YOU ARE LOOKING FOR'
progs = (i for i in progs if pop not in i)
mpi = None
for prog in progs:
mpi = which(prog, ignore_errors=True)
if mpi: break
if mpi and not fullpath:
mpi = os.path.split(mpi)[-1]
return mpi
def which_mpirun(mpich=None, fullpath=False):
"""try to autodetect an available mpi launcher
if mpich=True only look for mpich, if False only look for openmpi"""
import os
from pox import which
progs = [
'mpiexec', 'mpirun', 'mpiexec-mpich-mp', 'mpiexec-openmpi-mp',
'mpirun-mpich-mp', 'mpirun-openmpi-mp'
]
if mpich == True: pop = 'openmpi'
elif mpich == False: pop = 'mpich'
else: pop = 'THIS IS NOT THE MPI YOU ARE LOOKING FOR'
progs = (i for i in progs if pop not in i)
mpi = None
for prog in progs:
mpi = which(prog, ignore_errors=True)
if mpi: break
if mpi and not fullpath:
mpi = os.path.split(mpi)[-1]
return mpi
def pickport(rhost):
'''select a open port on a remote host
Inputs:
rhost -- hostname on which to select a open port
'''
from pox import which, getSEP
file = which('portpicker.py') #file = 'util.py'
dest = '~' #FIXME: *nix only
from time import sleep
delay = 0.0
# copy over the port selector to remote host
print 'executing {scp %s %s:%s}' % (file,rhost,dest)
copy(file,rhost,dest)
file = file.split(getSEP())[-1] # save only the filename
sleep(delay)
# get an available remote port number
command = 'python %s' % file
print 'executing {ssh %s "%s"}' % (rhost,command)
try:
rport = int(run(command,rhost,bg=False))
except:
from Tunnel import TunnelException
raise TunnelException, "failure to pick remote port"
#print rport
sleep(delay)
# remove temporary remote file (i.e. the port selector file)
dest = dest+'/'+file #FIXME: *nix only
command = 'rm -f %s' % dest #FIXME: *nix only
print 'executing {ssh %s "%s"}' % (rhost,command)
run(command,rhost)
# return remote port number
return rport
def pickport(rhost):
'''select a open port on a remote host
Inputs:
rhost -- hostname on which to select a open port
'''
from pox import which, getSEP
file = which('portpicker.py') #file = 'util.py'
dest = '~' #FIXME: *nix only
from time import sleep
delay = 0.0
# copy over the port selector to remote host
print 'executing {scp %s %s:%s}' % (file, rhost, dest)
copy(file, rhost, dest)
file = file.split(getSEP())[-1] # save only the filename
sleep(delay)
# get an available remote port number
command = 'python %s' % file
print 'executing {ssh %s "%s"}' % (rhost, command)
try:
rport = int(run(command, rhost, bg=False))
except:
from Tunnel import TunnelException
raise TunnelException, "failure to pick remote port"
#print rport
sleep(delay)
# remove temporary remote file (i.e. the port selector file)
dest = dest + '/' + file #FIXME: *nix only
command = 'rm -f %s' % dest #FIXME: *nix only
print 'executing {ssh %s "%s"}' % (rhost, command)
run(command, rhost)
# return remote port number
return rport
def test():
'''test(); script to test all functions'''
from pox import shelltype, homedir, rootdir, sep, mkdir, walk, where, \
username, minpath, env, which, find, shellsub, expandvars
#print('testing shelltype...')
shell = shelltype()
try:
assert shell in [
'bash', 'sh', 'csh', 'zsh', 'tcsh', 'ksh', 'rc', 'es', 'cmd'
]
except AssertionError:
print("Warning: non-standard shell type")
assert isinstance(shell, str)
#print('testing username...')
#print(username())
#print('testing homedir...')
#print(homedir())
assert homedir().rstrip(sep()).endswith(username())
#print('testing rootdir...')
#print(rootdir())
assert homedir().startswith(rootdir())
#print('testing sep...')
#print(sep())
#print(sep('ext'))
# print(sep('foo'))
#print('testing mkdir...')
newdir = sep().join(['xxxtest', 'testxxx'])
assert mkdir(newdir).rstrip(sep()).endswith(newdir)
#print('cleaning up...')
os.removedirs(newdir)
#print('testing walk...')
#print(walk('/usr/local','*',recurse=False,folders=True,files=False))
folders = walk(rootdir(), '*', recurse=False, folders=True, files=False)
assert len(folders) > 0
assert all(not os.path.isfile(folder) for folder in folders)
home = walk(homedir() + sep() + os.pardir, username(), False, True)[0]
assert home == homedir()
#print('testing where...')
shells = walk(home, '.bashrc', recurse=0)
bashrc = where('.bashrc', home)
if bashrc:
assert bashrc in shells
else:
assert not shells
#print(bashrc)
#print('testing minpath...')
#print(minpath(os.path.expandvars('$PATH')))
path = expandvars('$PATH')
assert minpath(path).count(sep('path')) <= path.count(sep('path'))
#print('testing env...')
assert env('ACSDAGHQSBFCASDCOMAOCMQOMCQWMOCQOMCOMQRCVOMQOCMQORMCQ') == {}
assert env('HOME', all=False) or env('USERPROFILE', all=False) == homedir()
pathdict = env('*PATH*', minimal=True)
assert len(pathdict) > 0
assert all('PATH' in key for key in pathdict)
#print('testing which...')
assert which('python').endswith(('python', 'python.exe'))
assert which('python') in which('python', all=True)
#print('testing find...')
#print(find('python','/usr/local',type='l'))
#print(find('*py;*txt'))
assert find('test_*', '.', False, 'f') == find('*py;*txt', recurse=False)
#print('testing shellsub...')
command = '${HOME}/bin/which foo("bar")'
#print(repr(command))
#print(repr(shellsub(command)))
assert shellsub(command) == '\\${HOME}/bin/which foo\\(\\"bar\\"\\)'
return
def test_shutils():
'''script to test all shutils functions'''
from pox import shelltype, homedir, rootdir, sep, mkdir, walk, where, env, \
username, minpath, which, which_python, find, shellsub, \
expandvars, __version__ as version
#print('testing shelltype...')
shell = shelltype()
try:
assert shell in [
'bash', 'sh', 'csh', 'zsh', 'tcsh', 'ksh', 'rc', 'es', 'cmd'
]
except AssertionError:
if shell:
print("Warning: non-standard shell type")
assert isinstance(shell, str)
else:
print("Warning: could not determine shell type")
assert shell is None
#print('testing username...')
#print(username())
#print('testing homedir...')
#print(homedir())
assert homedir().rstrip(sep()).endswith(username())
#print('testing rootdir...')
#print(rootdir())
assert homedir().startswith(rootdir())
#print('testing sep...')
#print(sep())
#print(sep('ext'))
# print(sep('foo'))
#print('testing mkdir...')
newdir = sep().join(['xxxtest', 'testxxx'])
assert mkdir(newdir).rstrip(sep()).endswith(newdir)
#print('cleaning up...')
os.removedirs(newdir)
#print('testing walk...')
#print(walk('/usr/local','*',recurse=False,folders=True,files=False))
folders = walk(rootdir(), '*', recurse=False, folders=True, files=False)
assert len(folders) > 0
### assert all(not os.path.isfile(folder) for folder in folders)
home = walk(homedir() + sep() + os.pardir, username(), False, True)[0]
assert home == homedir()
#print('testing where...')
shells = walk(home, '.bashrc', recurse=0)
bashrc = where('.bashrc', home)
if bashrc:
assert bashrc in shells
else:
assert not shells
#print(bashrc)
#print('testing minpath...')
#print(minpath(os.path.expandvars('$PATH')))
path = expandvars('$PATH')
assert minpath(path).count(sep('path')) <= path.count(sep('path'))
#print('testing env...')
assert env('ACSDAGHQSBFCASDCOMAOCMQOMCQWMOCQOMCOMQRCVOMQOCMQORMCQ') == {}
if 'HOME' not in os.environ:
os.environ['HOME'] = homedir()
assert env('HOME', all=False) or env('USERPROFILE', all=False) == homedir()
pathdict = env('*PATH*', minimal=True)
assert len(pathdict) > 0
assert all('PATH' in key for key in pathdict)
#print('testing which...')
assert which('python').endswith(('python', 'python.exe'))
assert which('python') in which('python', all=True)
#print('testing find...')
#print(find('python','/usr/local',type='l'))
#print(find('*py;*txt'))
x = os.path.dirname(__file__)
if not x: # this file is not found
x = which('pox;pox_launcher.py')
if x: # if executable found, then navigate to the test directory
p = which_python(fullpath=False, version=True)
x = os.sep.join((x.rsplit(os.sep, 2)[0], 'lib', p))
x = [
p for p in find('test_shutils.py', x, True, 'f')
if version in p
]
x = x[0] if x else ''
if x:
assert set(find('__init__*;__main__*;test_*', x, False,
'f')) == set(find('*py;*pyc', x, recurse=False))
#print('testing shellsub...')
command = '${HOME}/bin/which foo("bar")'
#print(repr(command))
#print(repr(shellsub(command)))
assert shellsub(command) == '\\${HOME}/bin/which foo\\(\\"bar\\"\\)'
return
def test_shutils():
"""script to test all shutils functions"""
from pox import (
shelltype,
homedir,
rootdir,
sep,
mkdir,
walk,
where,
username,
minpath,
env,
which,
find,
shellsub,
expandvars,
)
# print('testing shelltype...')
shell = shelltype()
try:
assert shell in ["bash", "sh", "csh", "zsh", "tcsh", "ksh", "rc", "es", "cmd"]
except AssertionError:
if shell:
print("Warning: non-standard shell type")
assert isinstance(shell, str)
else:
print("Warning: could not determine shell type")
assert shell is None
# print('testing username...')
# print(username())
# print('testing homedir...')
# print(homedir())
assert homedir().rstrip(sep()).endswith(username())
# print('testing rootdir...')
# print(rootdir())
assert homedir().startswith(rootdir())
# print('testing sep...')
# print(sep())
# print(sep('ext'))
# print(sep('foo'))
# print('testing mkdir...')
newdir = sep().join(["xxxtest", "testxxx"])
assert mkdir(newdir).rstrip(sep()).endswith(newdir)
# print('cleaning up...')
os.removedirs(newdir)
# print('testing walk...')
# print(walk('/usr/local','*',recurse=False,folders=True,files=False))
folders = walk(rootdir(), "*", recurse=False, folders=True, files=False)
assert len(folders) > 0
assert all(not os.path.isfile(folder) for folder in folders)
home = walk(homedir() + sep() + os.pardir, username(), False, True)[0]
assert home == homedir()
# print('testing where...')
shells = walk(home, ".bashrc", recurse=0)
bashrc = where(".bashrc", home)
if bashrc:
assert bashrc in shells
else:
assert not shells
# print(bashrc)
# print('testing minpath...')
# print(minpath(os.path.expandvars('$PATH')))
path = expandvars("$PATH")
assert minpath(path).count(sep("path")) <= path.count(sep("path"))
# print('testing env...')
assert env("ACSDAGHQSBFCASDCOMAOCMQOMCQWMOCQOMCOMQRCVOMQOCMQORMCQ") == {}
if "HOME" not in os.environ:
os.environ["HOME"] = homedir()
assert env("HOME", all=False) or env("USERPROFILE", all=False) == homedir()
pathdict = env("*PATH*", minimal=True)
assert len(pathdict) > 0
assert all("PATH" in key for key in pathdict)
# print('testing which...')
assert which("python").endswith(("python", "python.exe"))
assert which("python") in which("python", all=True)
# print('testing find...')
# print(find('python','/usr/local',type='l'))
# print(find('*py;*txt'))
x = "tests" if find("setup.py", recurse=False) else "."
assert find("__init__*;test_*", x, False, "f") == find("*py", x, recurse=False)
# print('testing shellsub...')
command = '${HOME}/bin/which foo("bar")'
# print(repr(command))
# print(repr(shellsub(command)))
assert shellsub(command) == '\\${HOME}/bin/which foo\\(\\"bar\\"\\)'
return
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 2018-2022 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/ppft/blob/master/LICENSE
from __future__ import print_function
import glob
import os
try:
import pox
python = pox.which_python(fullpath=False)
if not python:
python = 'python'
elif not pox.which(python):
python = pox.which_python(fullpath=False, version=True)
except ImportError:
python = 'python'
import subprocess as sp
from sys import platform
shell = platform[:3] == 'win'
suite = os.path.dirname(__file__) or os.path.curdir
tests = glob.glob(suite + os.path.sep + '*.py')
tests = [f for f in tests if not os.path.basename(f).startswith('__')]
if __name__ == '__main__':
for test in tests:
def test():
'''test(); script to test all functions'''
from pox import shelltype, homedir, rootdir, sep, mkdir, walk, where, \
username, minpath, env, which, find, shellsub, expandvars
#print('testing shelltype...')
shell = shelltype()
try:
assert shell in ['bash','sh','csh','zsh','tcsh','ksh','rc','es','cmd']
except AssertionError:
print("Warning: non-standard shell type")
assert isinstance(shell, str)
#print('testing username...')
#print(username())
#print('testing homedir...')
#print(homedir())
assert homedir().rstrip(sep()).endswith(username())
#print('testing rootdir...')
#print(rootdir())
assert homedir().startswith(rootdir())
#print('testing sep...')
#print(sep())
#print(sep('ext'))
# print(sep('foo'))
#print('testing mkdir...')
newdir = sep().join(['xxxtest','testxxx'])
assert mkdir(newdir).rstrip(sep()).endswith(newdir)
#print('cleaning up...')
os.removedirs(newdir)
#print('testing walk...')
#print(walk('/usr/local','*',recurse=False,folders=True,files=False))
folders = walk(rootdir(),'*',recurse=False,folders=True,files=False)
assert len(folders) > 0
assert all(not os.path.isfile(folder) for folder in folders)
home = walk(homedir()+sep()+os.pardir, username(), False, True)[0]
assert home == homedir()
#print('testing where...')
shells = walk(home,'.bashrc',recurse=0)
bashrc = where('.bashrc',home)
if bashrc:
assert bashrc in shells
else:
assert not shells
#print(bashrc)
#print('testing minpath...')
#print(minpath(os.path.expandvars('$PATH')))
path = expandvars('$PATH')
assert minpath(path).count(sep('path')) <= path.count(sep('path'))
#print('testing env...')
assert env('ACSDAGHQSBFCASDCOMAOCMQOMCQWMOCQOMCOMQRCVOMQOCMQORMCQ') == {}
assert env('HOME',all=False) == homedir()
pathdict = env('*PATH*',minimal=True)
assert len(pathdict) > 0
assert all('PATH' in key for key in pathdict)
#print('testing which...')
assert which('python').endswith('python')
assert which('python') in which('python',all=True)
#print('testing find...')
#print(find('python','/usr/local',type='l'))
#print(find('*py;*txt'))
assert find('test_*','.',False,'f') == find('*py;*txt',recurse=False)
#print('testing shellsub...')
command = '${HOME}/bin/which foo("bar")'
#print(repr(command))
#print(repr(shellsub(command)))
assert shellsub(command) == '\\${HOME}/bin/which foo\\(\\"bar\\"\\)'
return
except AssertionError:
print("%s:: Version >= %s is required" % (module, version))
returns += 1
# check required executables
try:
from pox import which
#from subprocess import Popen, STDOUT, PIPE#, call
except ImportError:
sys.exit(returns)
for module,executables in run.items():
for prog in reversed(executables):
try:
assert which(prog)
# process = Popen([prog, '--help'], stderr=STDOUT, stdout=PIPE)
# process.wait()
if isinstance(executables, list): break # just requires one
except (OSError, AssertionError):
if isinstance(executables, list) and \
prog != executables[0]: pass
from sys import exc_info
print("%s:: Executable '%s' not found" % (module, prog))
#print("%s:: %s" % (prog, exc_info()[1]))
returns += 1
# final report
if not returns:
print('OK.')
except AttributeError:
pass # can't version-check non-standard packages...
except AssertionError:
print("%s:: Version >= %s is required" % (module, version))
returns += 1
# check required executables
try:
from pox import which
#from subprocess import Popen, STDOUT, PIPE#, call
except ImportError:
sys.exit(returns)
for module, executables in run.items():
for prog in reversed(executables):
try:
assert which(prog)
# process = Popen([prog, '--help'], stderr=STDOUT, stdout=PIPE)
# process.wait()
if isinstance(executables, list): break # just requires one
except (OSError, AssertionError):
if isinstance(executables, list) and \
prog != executables[0]:
pass
from sys import exc_info
print("%s:: Executable '%s' not found" % (module, prog))
#print("%s:: %s" % (prog, exc_info()[1]))
returns += 1
# final report
if not returns:
print('OK.')