def remote_build(srcdirs=(),
destdir=None,
build_type='bdist_egg',
py=None,
remote_dir=None,
debug=False,
**kwargs):
"""Take the python distribution in the given directory, tar it up,
ship it over to host, build it, and bring it back, placing it
in the specified destination directory.
"""
from fabric.api import run, put, get
locdistbld = os.path.join(os.path.dirname(__file__), 'locdistbld.py')
pkgs = []
if remote_dir is None:
remote_dir = get_tmp_user_dir()
clean_remotedir = True
else:
clean_remotedir = False
for srcdir in srcdirs:
put_dir(srcdir, os.path.join(remote_dir, os.path.basename(srcdir)))
pkgname = os.path.basename(srcdir)
pkgdir = os.path.join(remote_dir, pkgname)
remotebuilder = os.path.join(pkgdir, 'locdistbld.py')
put(locdistbld, remotebuilder)
remtmp = remote_tmpdir()
result = run('%s %s -s %s -d %s -b "%s"' %
(py, remotebuilder, pkgdir, remtmp, build_type))
#print result
pkg = remote_listdir(remtmp)[
0] # should only have one file in directory
pkgpath = os.path.join(destdir, pkg)
print 'retrieving built distribution %s' % pkgpath
get(os.path.join(remtmp, pkg), pkgpath)
pkgs.append(pkgpath)
if debug:
print 'removing %s' % remtmp
rm_remote_tree(remtmp)
if clean_remotedir:
if debug:
print 'removing %s' % remote_dir
rm_remote_tree(remote_dir)
return 0
def _remote_build_and_test(fname=None,
pyversion='python',
keep=False,
branch=None,
testargs='',
hostname='',
**kwargs):
if fname is None:
raise RuntimeError("_remote_build_and_test: missing arg 'fname'")
remotedir = get_tmp_user_dir()
remote_mkdir(remotedir)
locbldtstfile = os.path.join(os.path.dirname(__file__), 'loc_bld_tst.py')
pushfiles = [locbldtstfile]
build_type = 'release' if fname.endswith('.py') else 'dev'
if os.path.isfile(fname):
pushfiles.append(fname)
remoteargs = ['-f', os.path.basename(fname)]
elif os.path.isdir(fname):
put_dir(fname, os.path.join(remotedir, os.path.basename(fname)))
if sys.platform.startswith('win'):
vername = 'latest' #readlink doesn't work on windows, so try 'latest'
else:
vername = os.readlink(os.path.join(fname, 'downloads', 'latest'))
remoteargs = [
'-f',
os.path.join(os.path.basename(fname), 'downloads', vername,
'go-openmdao.py')
]
else:
remoteargs = ['-f', fname]
if branch:
remoteargs.append('--branch=%s' % branch)
if testargs:
remoteargs.append('--testargs="%s"' % testargs)
try:
result = push_and_run(pushfiles,
runner=pyversion,
remotedir=remotedir,
args=remoteargs)
return result.return_code
finally:
if not keep:
print "removing remote directory: %s" % remotedir
rm_remote_tree(remotedir)
def remote_build(srcdirs=(), destdir=None, build_type='bdist_egg',
py=None, remote_dir=None, debug=False, **kwargs):
"""Take the Python distribution in the given directory, tar it up,
ship it over to host, build it, and bring it back, placing it
in the specified destination directory.
"""
from fabric.api import run, put, get
locdistbld = os.path.join(os.path.dirname(__file__), 'locdistbld.py')
pkgs = []
if remote_dir is None:
remote_dir = get_tmp_user_dir()
clean_remotedir = True
else:
clean_remotedir = False
for srcdir in srcdirs:
put_dir(srcdir, os.path.join(remote_dir,
os.path.basename(srcdir)))
pkgname = os.path.basename(srcdir)
pkgdir = os.path.join(remote_dir, pkgname)
remotebuilder = os.path.join(pkgdir, 'locdistbld.py')
put(locdistbld, remotebuilder)
remtmp = remote_tmpdir()
result = run('%s %s -s %s -d %s -b "%s"' % (py,
remotebuilder,
pkgdir,
remtmp,
build_type))
#print result
pkg = remote_listdir(remtmp)[0] # should only have one file in directory
pkgpath = os.path.join(destdir, pkg)
print 'retrieving built distribution %s' % pkgpath
get(os.path.join(remtmp, pkg), pkgpath)
pkgs.append(pkgpath)
if debug:
print 'removing %s' % remtmp
rm_remote_tree(remtmp)
if clean_remotedir:
if debug:
print 'removing %s' % remote_dir
rm_remote_tree(remote_dir)
return 0
def _remote_build_and_test(fname=None, pyversion='python', keep=False,
branch=None, testargs='', hostname='',
**kwargs):
if fname is None:
raise RuntimeError("_remote_build_and_test: missing arg 'fname'")
remotedir = get_tmp_user_dir()
remote_mkdir(remotedir)
locbldtstfile = os.path.join(os.path.dirname(__file__), 'loc_bld_tst.py')
pushfiles = [locbldtstfile]
build_type = 'release' if fname.endswith('.py') else 'dev'
if os.path.isfile(fname):
pushfiles.append(fname)
remoteargs = ['-f', os.path.basename(fname)]
elif os.path.isdir(fname):
put_dir(fname, os.path.join(remotedir, os.path.basename(fname)))
if sys.platform.startswith('win'):
vername = 'latest' #readlink doesn't work on windows, so try 'latest'
else:
vername = os.readlink(os.path.join(fname,
'downloads',
'latest'))
remoteargs = ['-f', os.path.join(os.path.basename(fname),
'downloads',
vername,
'go-openmdao.py')]
else:
remoteargs = ['-f', fname]
if branch:
remoteargs.append('--branch=%s' % branch)
if testargs:
remoteargs.append('--testargs="%s"' % testargs)
try:
result = push_and_run(pushfiles, runner=pyversion,
remotedir=remotedir,
args=remoteargs)
return result.return_code
finally:
if not keep:
print "removing remote directory: %s" % remotedir
rm_remote_tree(remotedir)
def test_branch(parser, options, args=None):
if args:
print_sub_help(parser, 'test_branch')
return -1
atexit.register(fabric_cleanup, True)
paramiko.util.log_to_file('paramiko.log')
options.filters = ['test_branch==true']
config, conn, ec2_hosts = process_options(options)
if not options.hosts:
parser.print_help()
print "nothing to do - no hosts specified"
sys.exit(0)
if options.fname is None: # assume we're testing the current repo
print 'creating tar file of current branch: ',
options.fname = os.path.join(os.getcwd(),
'OpenMDAO-Framework-testbranch.tar')
ziptarname = options.fname + '.gz'
cleanup(ziptarname) # clean up the old tar file
make_git_archive(options.fname,
prefix='OpenMDAO-OpenMDAO-Framework-testbranch/')
subprocess.check_call(['gzip', options.fname])
options.fname = os.path.abspath(ziptarname)
print options.fname
cleanup_tar = True
else:
cleanup_tar = False
fname = options.fname
if not (fname.startswith('http') or \
fname.startswith('git:') or fname.startswith('git@')):
fname = os.path.abspath(os.path.expanduser(options.fname))
if fname.endswith('.tar.gz') or fname.endswith('.tar'):
if not os.path.isfile(fname):
print "can't find file '%s'" % fname
sys.exit(-1)
elif fname.endswith('.git') or \
(fname.startswith('http') and os.path.splitext(fname)[1]==''):
pass
else:
parser.print_help()
print "\nfilename '%s' must specify a tar file or git repository" % fname
sys.exit(-1)
testargs = '-v ' if options.verbose else ''
testargs += '--gui ' if options.gui else ''
testargs += '--skip-gui ' if options.skip_gui else ''
testargs += options.testargs
funct_kwargs = {'keep': options.keep,
'testargs': testargs,
'fname': fname,
'remotedir': get_tmp_user_dir(),
'branch': options.branch,
'cfg': config
}
try:
retcode = run_host_processes(config, conn, ec2_hosts, options,
funct=_remote_build_and_test,
funct_kwargs=funct_kwargs,
done_functs=[print_host_codes])
finally:
if cleanup_tar:
cleanup(ziptarname)
if retcode == 0:
cleanup('paramiko.log')
return retcode
def _remote_build_and_test(fname=None, pyversion='python', keep=False,
branch=None, testargs='', hostname='', cfg=None,
**kwargs):
if fname is None:
raise RuntimeError("_remote_build_and_test: missing arg 'fname'")
remotedir = get_tmp_user_dir()
remote_mkdir(remotedir)
locbldtstfile = os.path.join(os.path.dirname(__file__), 'loc_bld_tst.py')
pushfiles = [locbldtstfile]
build_type = 'release' if fname.endswith('.py') else 'dev'
if cfg and cfg.has_option(hostname, 'pull_docs')and build_type == 'dev':
pull_docs = cfg.getboolean(hostname, 'pull_docs')
else:
pull_docs = False
if os.path.isfile(fname):
pushfiles.append(fname)
remoteargs = ['-f', os.path.basename(fname)]
elif os.path.isdir(fname):
put_dir(fname, os.path.join(remotedir, os.path.basename(fname)))
if sys.platform.startswith('win'):
vername = 'latest' # readlink doesn't work on windows, so try 'latest'
else:
vername = os.readlink(os.path.join(fname,
'downloads',
'latest'))
remoteargs = ['-f', os.path.join(os.path.basename(fname),
'downloads',
vername,
'go-openmdao-{}.py'.format(vername))]
else:
remoteargs = ['-f', fname]
if branch:
remoteargs.append('--branch=%s' % branch)
if testargs:
remoteargs.append('--testargs="%s"' % testargs)
try:
result = push_and_run(pushfiles, runner=pyversion,
remotedir=remotedir,
args=remoteargs)
if pull_docs:
print "pulling docs from %s" % hostname
retrieve_docs(os.path.join('~', remotedir))
print "doc retrieval successful"
else:
print "not pulling docs from %s because pull_docs is False" % hostname
return result.return_code
finally:
if build_type == 'dev':
print "pulling any pngs from %s" % hostname
try:
retrieve_pngs(os.path.join('~', remotedir))
print "png retrieval successful"
except Exception as exc:
print "png retrieval failed:", exc
if not keep:
print "removing remote directory: %s" % remotedir
rm_remote_tree(remotedir)
def test_branch(parser, options, args=None):
if args:
print_sub_help(parser, 'test_branch')
return -1
atexit.register(fabric_cleanup, True)
paramiko.util.log_to_file('paramiko.log')
options.filters = ['test_branch==true']
config, conn, ec2_hosts = process_options(options)
if not options.hosts:
parser.print_help()
print "nothing to do - no hosts specified"
sys.exit(0)
if options.fname is None: # assume we're testing the current repo
print 'creating tar file of current branch: ',
options.fname = os.path.join(os.getcwd(),
'OpenMDAO-Framework-testbranch.tar')
ziptarname = options.fname + '.gz'
cleanup(ziptarname) # clean up the old tar file
make_git_archive(options.fname,
prefix='OpenMDAO-OpenMDAO-Framework-testbranch/')
subprocess.check_call(['gzip', options.fname])
options.fname = os.path.abspath(ziptarname)
print options.fname
cleanup_tar = True
else:
cleanup_tar = False
fname = options.fname
if not (fname.startswith('http') or \
fname.startswith('git:') or fname.startswith('git@')):
fname = os.path.abspath(os.path.expanduser(options.fname))
if fname.endswith('.tar.gz') or fname.endswith('.tar'):
if not os.path.isfile(fname):
print "can't find file '%s'" % fname
sys.exit(-1)
elif fname.endswith('.git') or \
(fname.startswith('http') and os.path.splitext(fname)[1]==''):
pass
else:
parser.print_help()
print "\nfilename '%s' must specify a tar file or git repository" % fname
sys.exit(-1)
testargs = '-v ' if options.verbose else ''
testargs += '--gui ' if options.gui else ''
testargs += '--skip-gui ' if options.skip_gui else ''
testargs += options.testargs
funct_kwargs = {
'keep': options.keep,
'testargs': testargs,
'fname': fname,
'remotedir': get_tmp_user_dir(),
'branch': options.branch,
'cfg': config
}
try:
retcode = run_host_processes(config,
conn,
ec2_hosts,
options,
funct=_remote_build_and_test,
funct_kwargs=funct_kwargs,
done_functs=[print_host_codes])
finally:
if cleanup_tar:
cleanup(ziptarname)
if retcode == 0:
cleanup('paramiko.log')
return retcode
def _remote_build_and_test(fname=None,
pyversion='python',
keep=False,
branch=None,
testargs='',
hostname='',
cfg=None,
**kwargs):
if fname is None:
raise RuntimeError("_remote_build_and_test: missing arg 'fname'")
remotedir = get_tmp_user_dir()
remote_mkdir(remotedir)
locbldtstfile = os.path.join(os.path.dirname(__file__), 'loc_bld_tst.py')
pushfiles = [locbldtstfile]
build_type = 'release' if fname.endswith('.py') else 'dev'
if cfg and cfg.has_option(hostname, 'pull_docs') and build_type == 'dev':
pull_docs = cfg.getboolean(hostname, 'pull_docs')
else:
pull_docs = False
if os.path.isfile(fname):
pushfiles.append(fname)
remoteargs = ['-f', os.path.basename(fname)]
elif os.path.isdir(fname):
put_dir(fname, os.path.join(remotedir, os.path.basename(fname)))
if sys.platform.startswith('win'):
vername = 'latest' # readlink doesn't work on windows, so try 'latest'
else:
vername = os.readlink(os.path.join(fname, 'downloads', 'latest'))
remoteargs = [
'-f',
os.path.join(os.path.basename(fname), 'downloads', vername,
'go-openmdao.py')
]
else:
remoteargs = ['-f', fname]
if branch:
remoteargs.append('--branch=%s' % branch)
if testargs:
remoteargs.append('--testargs="%s"' % testargs)
try:
result = push_and_run(pushfiles,
runner=pyversion,
remotedir=remotedir,
args=remoteargs)
if pull_docs:
print "pulling docs from %s" % hostname
retrieve_docs(os.path.join('~', remotedir))
print "doc retrieval successful"
else:
print "not pulling docs from %s because pull_docs is False" % hostname
return result.return_code
finally:
if build_type == 'dev':
print "pulling any pngs from %s" % hostname
try:
retrieve_pngs(os.path.join('~', remotedir))
print "png retrieval successful"
except Exception as exc:
print "png retrieval failed:", exc
if not keep:
print "removing remote directory: %s" % remotedir
rm_remote_tree(remotedir)
def test_branch(argv=None):
atexit.register(fabric_cleanup, True)
paramiko.util.log_to_file('paramiko.log')
if argv is None:
argv = sys.argv[1:]
parser = ArgumentParser()
add_config_options(parser)
parser.add_argument("-k","--keep", action="store_true", dest='keep',
help="Don't delete the temporary build directory. "
"If testing on EC2 stop the instance instead of terminating it.")
parser.add_argument("-f","--file", action="store", type=str,
dest='fname',
help="Pathname of a tarfile or URL of a git repo. "
"Defaults to the current repo.")
parser.add_argument("-b","--branch", action="store", type=str,
dest='branch',
help="If file is a git repo, supply branch name here")
parser.add_argument("--testargs", action="store", type=str, dest='testargs',
default='',
help="args to be passed to openmdao test")
options = parser.parse_args()
options.filters = ['test_branch==true']
config, conn, ec2_hosts = process_options(options)
if not options.hosts:
parser.print_help()
print "nothing to do - no hosts specified"
sys.exit(0)
startdir = os.getcwd()
if options.fname is None: # assume we're testing the current repo
print 'creating tar file of current branch: ',
options.fname = os.path.join(os.getcwd(), 'testbranch.tar')
ziptarname = options.fname+'.gz'
cleanup(ziptarname) # clean up the old tar file
make_git_archive(options.fname)
subprocess.check_call(['gzip', options.fname])
options.fname = os.path.abspath(ziptarname)
print options.fname
cleanup_tar = True
else:
cleanup_tar = False
fname = options.fname
if not (fname.startswith('http') or fname.startswith('git:') or fname.startswith('git@')):
fname = os.path.abspath(os.path.expanduser(options.fname))
if fname.endswith('.tar.gz') or fname.endswith('.tar'):
if not os.path.isfile(fname):
print "can't find file '%s'" % fname
sys.exit(-1)
elif fname.endswith('.git') or (fname.startswith('http') and os.path.splitext(fname)[1]==''):
pass
else:
parser.print_help()
print "\nfilename '%s' must specify a tar file or git repository" % fname
sys.exit(-1)
funct_kwargs = { 'keep': options.keep,
'testargs': options.testargs,
'fname': fname,
'remotedir': get_tmp_user_dir(),
'branch': options.branch,
}
try:
retcode = run_host_processes(config, conn, ec2_hosts, options,
funct=_remote_build_and_test,
funct_kwargs=funct_kwargs,
done_functs=[print_host_codes])
finally:
if cleanup_tar:
cleanup(ziptarname)
if retcode == 0:
cleanup('paramiko.log')
return retcode
def test_branch(argv=None):
atexit.register(fabric_cleanup, True)
paramiko.util.log_to_file('paramiko.log')
if argv is None:
argv = sys.argv[1:]
parser = CfgOptionParser(usage="%prog [OPTIONS] -- [options to openmdao_test]")
parser.add_option("-k","--keep", action="store_true", dest='keep',
help="Don't delete the temporary build directory. "
"If testing on EC2 stop the instance instead of terminating it.")
parser.add_option("-f","--file", action="store", type='string',
dest='fname',
help="Pathname of a tarfile or URL of a git repo. "
"Defaults to the current repo.")
parser.add_option("-b","--branch", action="store", type='string',
dest='branch',
help="If file is a git repo, supply branch name here")
(options, args) = parser.parse_args(argv)
config, conn, ec2_hosts = process_options(options, parser)
if not options.hosts:
parser.print_help()
print "nothing to do - no hosts specified"
sys.exit(0)
startdir = os.getcwd()
if options.fname is None: # assume we're testing the current repo
print 'creating tar file of current branch: ',
options.fname = os.path.join(os.getcwd(), 'testbranch.tar')
ziptarname = options.fname+'.gz'
cleanup(ziptarname) # clean up the old tar file
make_git_archive(options.fname)
subprocess.check_call(['gzip', options.fname])
options.fname = os.path.abspath(ziptarname)
print options.fname
cleanup_tar = True
else:
cleanup_tar = False
fname = os.path.abspath(os.path.expanduser(options.fname))
if fname.endswith('.tar.gz') or fname.endswith('.tar'):
if not os.path.isfile(fname):
print "can't find file '%s'" % fname
sys.exit(-1)
elif fname.endswith('.git'):
pass
else:
parser.print_help()
print "\nfilename must end in '.tar.gz', '.tar', or '.git'"
sys.exit(-1)
funct_kwargs = { 'keep': options.keep,
'testargs': args,
'fname': fname,
'remotedir': get_tmp_user_dir(),
'branch': options.branch,
}
try:
retcode = run_host_processes(config, conn, ec2_hosts, options,
funct=_remote_build_and_test,
funct_kwargs=funct_kwargs)
finally:
if cleanup_tar:
cleanup(ziptarname)
if retcode == 0:
cleanup('paramiko.log')
return retcode
def test_branch(argv=None):
atexit.register(fabric_cleanup, True)
paramiko.util.log_to_file('paramiko.log')
if argv is None:
argv = sys.argv[1:]
parser = ArgumentParser()
add_config_options(parser)
parser.add_argument(
"-k",
"--keep",
action="store_true",
dest='keep',
help="Don't delete the temporary build directory. "
"If testing on EC2 stop the instance instead of terminating it.")
parser.add_argument("-f",
"--file",
action="store",
type=str,
dest='fname',
help="Pathname of a tarfile or URL of a git repo. "
"Defaults to the current repo.")
parser.add_argument("-b",
"--branch",
action="store",
type=str,
dest='branch',
help="If file is a git repo, supply branch name here")
parser.add_argument("--testargs",
action="store",
type=str,
dest='testargs',
default='',
help="args to be passed to openmdao test")
options = parser.parse_args()
options.filters = ['test_branch==true']
config, conn, ec2_hosts = process_options(options)
if not options.hosts:
parser.print_help()
print "nothing to do - no hosts specified"
sys.exit(0)
startdir = os.getcwd()
if options.fname is None: # assume we're testing the current repo
print 'creating tar file of current branch: ',
options.fname = os.path.join(os.getcwd(), 'testbranch.tar')
ziptarname = options.fname + '.gz'
cleanup(ziptarname) # clean up the old tar file
make_git_archive(options.fname)
subprocess.check_call(['gzip', options.fname])
options.fname = os.path.abspath(ziptarname)
print options.fname
cleanup_tar = True
else:
cleanup_tar = False
fname = options.fname
if not (fname.startswith('http') or fname.startswith('git:')
or fname.startswith('git@')):
fname = os.path.abspath(os.path.expanduser(options.fname))
if fname.endswith('.tar.gz') or fname.endswith('.tar'):
if not os.path.isfile(fname):
print "can't find file '%s'" % fname
sys.exit(-1)
elif fname.endswith('.git') or (fname.startswith('http')
and os.path.splitext(fname)[1] == ''):
pass
else:
parser.print_help()
print "\nfilename '%s' must specify a tar file or git repository" % fname
sys.exit(-1)
funct_kwargs = {
'keep': options.keep,
'testargs': options.testargs,
'fname': fname,
'remotedir': get_tmp_user_dir(),
'branch': options.branch,
}
try:
retcode = run_host_processes(config,
conn,
ec2_hosts,
options,
funct=_remote_build_and_test,
funct_kwargs=funct_kwargs,
done_functs=[print_host_codes])
finally:
if cleanup_tar:
cleanup(ziptarname)
if retcode == 0:
cleanup('paramiko.log')
return retcode