Example #1
0
 def __launch(self, command):
     """launch mechanism for prepared launch command"""
     executable = command.split("|")[-1].split()[0]
     from pox import which
     if not which(executable):
         raise IOError("launch failed: %s not found" % executable)
     return Popen([command], shell=True)  #FIXME: shell=True is insecure
Example #2
0
 def __launch(self, command):
     """launch mechanism for prepared launch command"""
     executable = command.split("|")[-1].split()[0]
     from pox import which
     if not which(executable):
         raise IOError, "launch failed: %s not found" % executable
     return Popen([command], shell=True) #FIXME: shell=True is insecure
def executable_exist(module, prog):
    try:
        assert which(prog)
        #           process = Popen([prog, '--help'], stderr=STDOUT, stdout=PIPE)
        #           process.wait()
        return True
    except (OSError, AssertionError):
        from sys import exc_info
        print("%s:: Executable '%s' not found" % (module, prog))
        #print("%s:: %s" % (prog, exc_info()[1]))
        return False
def executable_exist(module, prog):
    try:
        assert which(prog)
#           process = Popen([prog, '--help'], stderr=STDOUT, stdout=PIPE)
#           process.wait()
        return True
    except (OSError, AssertionError):
        from sys import exc_info
        print("%s:: Executable '%s' not found" % (module, prog))
        #print("%s:: %s" % (prog, exc_info()[1]))
        return False
Example #5
0
def which_strategy(scatter=True, lazy=False, fullpath=True):
    """try to autodetect an available strategy (scatter or pool)"""
    target = 'ezscatter.py' if scatter else 'ezpool.py'
    import sys
    if (sys.platform[:3] == 'win'): lazy=False
    if lazy: target = "`which %s`" % target
    # lookup full path
    elif not lazy and fullpath:
        from pox import which
        target = which(target, ignore_errors=True)
    if not target: target = None #XXX: better None or "" ?
    return target
Example #6
0
def which_strategy(scatter=True, lazy=False, fullpath=True):
    """try to autodetect an available strategy (scatter or pool)"""
    target = 'ezscatter' if scatter else 'ezpool'
    import sys
    if (sys.platform[:3] == 'win'): lazy = False
    if lazy:
        target = "`which %s`" % target
        # lookup full path
    elif not lazy and fullpath:
        from pox import which
        target = which(target, ignore_errors=True)
    if not target: target = None  #XXX: better None or "" ?
    return target
Example #7
0
def which_mpirun(mpich=None, fullpath=False):
    """try to autodetect an available mpi launcher

if mpich=True only look for mpich, if False only look for openmpi"""
    import os
    from pox import which
    progs = ['mpiexec', 'mpirun', 'mpiexec-mpich-mp', 'mpiexec-openmpi-mp', 'mpirun-mpich-mp', 'mpirun-openmpi-mp']
    if mpich == True: pop = 'openmpi'
    elif mpich == False: pop = 'mpich'
    else: pop = 'THIS IS NOT THE MPI YOU ARE LOOKING FOR'
    progs = (i for i in progs if pop not in i)
    mpi = None
    for prog in progs:
        mpi = which(prog, ignore_errors=True)
        if mpi: break
    if mpi and not fullpath:
        mpi = os.path.split(mpi)[-1]
    return mpi
Example #8
0
def which_mpirun(mpich=None, fullpath=False):
    """try to autodetect an available mpi launcher

if mpich=True only look for mpich, if False only look for openmpi"""
    import os
    from pox import which
    progs = [
        'mpiexec', 'mpirun', 'mpiexec-mpich-mp', 'mpiexec-openmpi-mp',
        'mpirun-mpich-mp', 'mpirun-openmpi-mp'
    ]
    if mpich == True: pop = 'openmpi'
    elif mpich == False: pop = 'mpich'
    else: pop = 'THIS IS NOT THE MPI YOU ARE LOOKING FOR'
    progs = (i for i in progs if pop not in i)
    mpi = None
    for prog in progs:
        mpi = which(prog, ignore_errors=True)
        if mpi: break
    if mpi and not fullpath:
        mpi = os.path.split(mpi)[-1]
    return mpi
Example #9
0
def pickport(rhost):
  '''select a open port on a remote host

Inputs:
    rhost -- hostname on which to select a open port
  '''
  from pox import which, getSEP
  file = which('portpicker.py') #file = 'util.py'
  dest = '~' #FIXME: *nix only

  from time import sleep
  delay = 0.0

  # copy over the port selector to remote host
  print 'executing {scp %s %s:%s}' % (file,rhost,dest)
  copy(file,rhost,dest)
  file = file.split(getSEP())[-1]   # save only the filename
  sleep(delay)

  # get an available remote port number
  command = 'python %s' % file
  print 'executing {ssh %s "%s"}' % (rhost,command)
  try:
    rport = int(run(command,rhost,bg=False))
  except:
    from Tunnel import TunnelException
    raise TunnelException, "failure to pick remote port"
  #print rport
  sleep(delay)

  # remove temporary remote file (i.e. the port selector file)
  dest = dest+'/'+file  #FIXME: *nix only
  command = 'rm -f %s' % dest #FIXME: *nix only
  print 'executing {ssh %s "%s"}' % (rhost,command)
  run(command,rhost)

  # return remote port number
  return rport
Example #10
0
File: core.py Project: yuhjay/vmaf
def pickport(rhost):
    '''select a open port on a remote host

Inputs:
    rhost -- hostname on which to select a open port
  '''
    from pox import which, getSEP
    file = which('portpicker.py')  #file = 'util.py'
    dest = '~'  #FIXME: *nix only

    from time import sleep
    delay = 0.0

    # copy over the port selector to remote host
    print 'executing {scp %s %s:%s}' % (file, rhost, dest)
    copy(file, rhost, dest)
    file = file.split(getSEP())[-1]  # save only the filename
    sleep(delay)

    # get an available remote port number
    command = 'python %s' % file
    print 'executing {ssh %s "%s"}' % (rhost, command)
    try:
        rport = int(run(command, rhost, bg=False))
    except:
        from Tunnel import TunnelException
        raise TunnelException, "failure to pick remote port"
    #print rport
    sleep(delay)

    # remove temporary remote file (i.e. the port selector file)
    dest = dest + '/' + file  #FIXME: *nix only
    command = 'rm -f %s' % dest  #FIXME: *nix only
    print 'executing {ssh %s "%s"}' % (rhost, command)
    run(command, rhost)

    # return remote port number
    return rport
Example #11
0
def test():
    '''test(); script to test all functions'''
    from pox import shelltype, homedir, rootdir, sep, mkdir, walk, where, \
                    username, minpath, env, which, find, shellsub, expandvars

    #print('testing shelltype...')
    shell = shelltype()
    try:
        assert shell in [
            'bash', 'sh', 'csh', 'zsh', 'tcsh', 'ksh', 'rc', 'es', 'cmd'
        ]
    except AssertionError:
        print("Warning: non-standard shell type")
        assert isinstance(shell, str)

#print('testing username...')
#print(username())

#print('testing homedir...')
#print(homedir())
    assert homedir().rstrip(sep()).endswith(username())

    #print('testing rootdir...')
    #print(rootdir())
    assert homedir().startswith(rootdir())

    #print('testing sep...')
    #print(sep())
    #print(sep('ext'))
    #   print(sep('foo'))

    #print('testing mkdir...')
    newdir = sep().join(['xxxtest', 'testxxx'])
    assert mkdir(newdir).rstrip(sep()).endswith(newdir)
    #print('cleaning up...')
    os.removedirs(newdir)

    #print('testing walk...')
    #print(walk('/usr/local','*',recurse=False,folders=True,files=False))
    folders = walk(rootdir(), '*', recurse=False, folders=True, files=False)
    assert len(folders) > 0
    assert all(not os.path.isfile(folder) for folder in folders)
    home = walk(homedir() + sep() + os.pardir, username(), False, True)[0]
    assert home == homedir()

    #print('testing where...')
    shells = walk(home, '.bashrc', recurse=0)
    bashrc = where('.bashrc', home)
    if bashrc:
        assert bashrc in shells
    else:
        assert not shells

#print(bashrc)

#print('testing minpath...')
#print(minpath(os.path.expandvars('$PATH')))
    path = expandvars('$PATH')
    assert minpath(path).count(sep('path')) <= path.count(sep('path'))

    #print('testing env...')
    assert env('ACSDAGHQSBFCASDCOMAOCMQOMCQWMOCQOMCOMQRCVOMQOCMQORMCQ') == {}
    assert env('HOME', all=False) or env('USERPROFILE', all=False) == homedir()
    pathdict = env('*PATH*', minimal=True)
    assert len(pathdict) > 0
    assert all('PATH' in key for key in pathdict)

    #print('testing which...')
    assert which('python').endswith(('python', 'python.exe'))
    assert which('python') in which('python', all=True)

    #print('testing find...')
    #print(find('python','/usr/local',type='l'))
    #print(find('*py;*txt'))
    assert find('test_*', '.', False, 'f') == find('*py;*txt', recurse=False)

    #print('testing shellsub...')
    command = '${HOME}/bin/which foo("bar")'
    #print(repr(command))
    #print(repr(shellsub(command)))
    assert shellsub(command) == '\\${HOME}/bin/which foo\\(\\"bar\\"\\)'

    return
Example #12
0
def test_shutils():
    '''script to test all shutils functions'''
    from pox import shelltype, homedir, rootdir, sep, mkdir, walk, where, env, \
                    username, minpath, which, which_python, find, shellsub, \
                    expandvars, __version__ as version

    #print('testing shelltype...')
    shell = shelltype()
    try:
        assert shell in [
            'bash', 'sh', 'csh', 'zsh', 'tcsh', 'ksh', 'rc', 'es', 'cmd'
        ]
    except AssertionError:
        if shell:
            print("Warning: non-standard shell type")
            assert isinstance(shell, str)
        else:
            print("Warning: could not determine shell type")
            assert shell is None

#print('testing username...')
#print(username())

#print('testing homedir...')
#print(homedir())
    assert homedir().rstrip(sep()).endswith(username())

    #print('testing rootdir...')
    #print(rootdir())
    assert homedir().startswith(rootdir())

    #print('testing sep...')
    #print(sep())
    #print(sep('ext'))
    #   print(sep('foo'))

    #print('testing mkdir...')
    newdir = sep().join(['xxxtest', 'testxxx'])
    assert mkdir(newdir).rstrip(sep()).endswith(newdir)
    #print('cleaning up...')
    os.removedirs(newdir)

    #print('testing walk...')
    #print(walk('/usr/local','*',recurse=False,folders=True,files=False))
    folders = walk(rootdir(), '*', recurse=False, folders=True, files=False)
    assert len(folders) > 0
    ### assert all(not os.path.isfile(folder) for folder in folders)
    home = walk(homedir() + sep() + os.pardir, username(), False, True)[0]
    assert home == homedir()

    #print('testing where...')
    shells = walk(home, '.bashrc', recurse=0)
    bashrc = where('.bashrc', home)
    if bashrc:
        assert bashrc in shells
    else:
        assert not shells

#print(bashrc)

#print('testing minpath...')
#print(minpath(os.path.expandvars('$PATH')))
    path = expandvars('$PATH')
    assert minpath(path).count(sep('path')) <= path.count(sep('path'))

    #print('testing env...')
    assert env('ACSDAGHQSBFCASDCOMAOCMQOMCQWMOCQOMCOMQRCVOMQOCMQORMCQ') == {}
    if 'HOME' not in os.environ:
        os.environ['HOME'] = homedir()
    assert env('HOME', all=False) or env('USERPROFILE', all=False) == homedir()
    pathdict = env('*PATH*', minimal=True)
    assert len(pathdict) > 0
    assert all('PATH' in key for key in pathdict)

    #print('testing which...')
    assert which('python').endswith(('python', 'python.exe'))
    assert which('python') in which('python', all=True)

    #print('testing find...')
    #print(find('python','/usr/local',type='l'))
    #print(find('*py;*txt'))
    x = os.path.dirname(__file__)
    if not x:  # this file is not found
        x = which('pox;pox_launcher.py')
        if x:  # if executable found, then navigate to the test directory
            p = which_python(fullpath=False, version=True)
            x = os.sep.join((x.rsplit(os.sep, 2)[0], 'lib', p))
            x = [
                p for p in find('test_shutils.py', x, True, 'f')
                if version in p
            ]
            x = x[0] if x else ''
    if x:
        assert set(find('__init__*;__main__*;test_*', x, False,
                        'f')) == set(find('*py;*pyc', x, recurse=False))

    #print('testing shellsub...')
    command = '${HOME}/bin/which foo("bar")'
    #print(repr(command))
    #print(repr(shellsub(command)))
    assert shellsub(command) == '\\${HOME}/bin/which foo\\(\\"bar\\"\\)'

    return
Example #13
0
def test_shutils():
    """script to test all shutils functions"""
    from pox import (
        shelltype,
        homedir,
        rootdir,
        sep,
        mkdir,
        walk,
        where,
        username,
        minpath,
        env,
        which,
        find,
        shellsub,
        expandvars,
    )

    # print('testing shelltype...')
    shell = shelltype()
    try:
        assert shell in ["bash", "sh", "csh", "zsh", "tcsh", "ksh", "rc", "es", "cmd"]
    except AssertionError:
        if shell:
            print("Warning: non-standard shell type")
            assert isinstance(shell, str)
        else:
            print("Warning: could not determine shell type")
            assert shell is None

    # print('testing username...')
    # print(username())

    # print('testing homedir...')
    # print(homedir())
    assert homedir().rstrip(sep()).endswith(username())

    # print('testing rootdir...')
    # print(rootdir())
    assert homedir().startswith(rootdir())

    # print('testing sep...')
    # print(sep())
    # print(sep('ext'))
    #   print(sep('foo'))

    # print('testing mkdir...')
    newdir = sep().join(["xxxtest", "testxxx"])
    assert mkdir(newdir).rstrip(sep()).endswith(newdir)
    # print('cleaning up...')
    os.removedirs(newdir)

    # print('testing walk...')
    # print(walk('/usr/local','*',recurse=False,folders=True,files=False))
    folders = walk(rootdir(), "*", recurse=False, folders=True, files=False)
    assert len(folders) > 0
    assert all(not os.path.isfile(folder) for folder in folders)
    home = walk(homedir() + sep() + os.pardir, username(), False, True)[0]
    assert home == homedir()

    # print('testing where...')
    shells = walk(home, ".bashrc", recurse=0)
    bashrc = where(".bashrc", home)
    if bashrc:
        assert bashrc in shells
    else:
        assert not shells
    # print(bashrc)

    # print('testing minpath...')
    # print(minpath(os.path.expandvars('$PATH')))
    path = expandvars("$PATH")
    assert minpath(path).count(sep("path")) <= path.count(sep("path"))

    # print('testing env...')
    assert env("ACSDAGHQSBFCASDCOMAOCMQOMCQWMOCQOMCOMQRCVOMQOCMQORMCQ") == {}
    if "HOME" not in os.environ:
        os.environ["HOME"] = homedir()
    assert env("HOME", all=False) or env("USERPROFILE", all=False) == homedir()
    pathdict = env("*PATH*", minimal=True)
    assert len(pathdict) > 0
    assert all("PATH" in key for key in pathdict)

    # print('testing which...')
    assert which("python").endswith(("python", "python.exe"))
    assert which("python") in which("python", all=True)

    # print('testing find...')
    # print(find('python','/usr/local',type='l'))
    # print(find('*py;*txt'))
    x = "tests" if find("setup.py", recurse=False) else "."
    assert find("__init__*;test_*", x, False, "f") == find("*py", x, recurse=False)

    # print('testing shellsub...')
    command = '${HOME}/bin/which foo("bar")'
    # print(repr(command))
    # print(repr(shellsub(command)))
    assert shellsub(command) == '\\${HOME}/bin/which foo\\(\\"bar\\"\\)'

    return
Example #14
0
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 2018-2022 The Uncertainty Quantification Foundation.
# License: 3-clause BSD.  The full license text is available at:
#  - https://github.com/uqfoundation/ppft/blob/master/LICENSE

from __future__ import print_function
import glob
import os
try:
    import pox
    python = pox.which_python(fullpath=False)
    if not python:
        python = 'python'
    elif not pox.which(python):
        python = pox.which_python(fullpath=False, version=True)
except ImportError:
    python = 'python'
import subprocess as sp
from sys import platform
shell = platform[:3] == 'win'

suite = os.path.dirname(__file__) or os.path.curdir
tests = glob.glob(suite + os.path.sep + '*.py')
tests = [f for f in tests if not os.path.basename(f).startswith('__')]


if __name__ == '__main__':

    for test in tests:
Example #15
0
def test():
    '''test(); script to test all functions'''
    from pox import shelltype, homedir, rootdir, sep, mkdir, walk, where, \
                    username, minpath, env, which, find, shellsub, expandvars

   #print('testing shelltype...')
    shell = shelltype()
    try:
        assert shell in ['bash','sh','csh','zsh','tcsh','ksh','rc','es','cmd']
    except AssertionError:
        print("Warning: non-standard shell type")
        assert isinstance(shell, str)

   #print('testing username...')
   #print(username())

   #print('testing homedir...')
   #print(homedir())
    assert homedir().rstrip(sep()).endswith(username())

   #print('testing rootdir...')
   #print(rootdir())
    assert homedir().startswith(rootdir())

   #print('testing sep...')
   #print(sep())
   #print(sep('ext'))
#   print(sep('foo'))

   #print('testing mkdir...')
    newdir = sep().join(['xxxtest','testxxx'])
    assert mkdir(newdir).rstrip(sep()).endswith(newdir)
   #print('cleaning up...')
    os.removedirs(newdir)

   #print('testing walk...')
   #print(walk('/usr/local','*',recurse=False,folders=True,files=False))
    folders = walk(rootdir(),'*',recurse=False,folders=True,files=False)
    assert len(folders) > 0
    assert all(not os.path.isfile(folder) for folder in folders)
    home = walk(homedir()+sep()+os.pardir, username(), False, True)[0]
    assert home == homedir()

   #print('testing where...')
    shells = walk(home,'.bashrc',recurse=0)
    bashrc = where('.bashrc',home)
    if bashrc:
        assert bashrc in shells
    else:
        assert not shells
   #print(bashrc)

   #print('testing minpath...')
   #print(minpath(os.path.expandvars('$PATH')))
    path = expandvars('$PATH')
    assert minpath(path).count(sep('path')) <= path.count(sep('path'))

   #print('testing env...')
    assert env('ACSDAGHQSBFCASDCOMAOCMQOMCQWMOCQOMCOMQRCVOMQOCMQORMCQ') == {}
    assert env('HOME',all=False) == homedir()
    pathdict = env('*PATH*',minimal=True)
    assert len(pathdict) > 0
    assert all('PATH' in key for key in pathdict)

   #print('testing which...')
    assert which('python').endswith('python')
    assert which('python') in which('python',all=True)

   #print('testing find...')
   #print(find('python','/usr/local',type='l'))
   #print(find('*py;*txt'))
    assert find('test_*','.',False,'f') == find('*py;*txt',recurse=False)

   #print('testing shellsub...')
    command = '${HOME}/bin/which foo("bar")'
   #print(repr(command))
   #print(repr(shellsub(command)))
    assert shellsub(command) == '\\${HOME}/bin/which foo\\(\\"bar\\"\\)'

    return
Example #16
0
    except AssertionError:
        print("%s:: Version >= %s is required" % (module, version))
        returns += 1



# check required executables
try:
    from pox import which
   #from subprocess import Popen, STDOUT, PIPE#, call
except ImportError:
    sys.exit(returns)
for module,executables in run.items():
    for prog in reversed(executables):
        try:
            assert which(prog)
#           process = Popen([prog, '--help'], stderr=STDOUT, stdout=PIPE)
#           process.wait()
            if isinstance(executables, list): break  # just requires one
        except (OSError, AssertionError):
            if isinstance(executables, list) and \
               prog != executables[0]: pass
            from sys import exc_info
            print("%s:: Executable '%s' not found" % (module, prog))
           #print("%s:: %s" % (prog, exc_info()[1]))
            returns += 1


# final report
if not returns:
    print('OK.')
Example #17
0
    except AttributeError:
        pass  # can't version-check non-standard packages...
    except AssertionError:
        print("%s:: Version >= %s is required" % (module, version))
        returns += 1

# check required executables
try:
    from pox import which
#from subprocess import Popen, STDOUT, PIPE#, call
except ImportError:
    sys.exit(returns)
for module, executables in run.items():
    for prog in reversed(executables):
        try:
            assert which(prog)
            #           process = Popen([prog, '--help'], stderr=STDOUT, stdout=PIPE)
            #           process.wait()
            if isinstance(executables, list): break  # just requires one
        except (OSError, AssertionError):
            if isinstance(executables, list) and \
               prog != executables[0]:
                pass
            from sys import exc_info
            print("%s:: Executable '%s' not found" % (module, prog))
            #print("%s:: %s" % (prog, exc_info()[1]))
            returns += 1

# final report
if not returns:
    print('OK.')