def remote_build(srcdirs=(), destdir=None, build_type='bdist_egg', py=None, remote_dir=None, debug=False, **kwargs): """Take the python distribution in the given directory, tar it up, ship it over to host, build it, and bring it back, placing it in the specified destination directory. """ from fabric.api import run, put, get locdistbld = os.path.join(os.path.dirname(__file__), 'locdistbld.py') pkgs = [] if remote_dir is None: remote_dir = get_tmp_user_dir() clean_remotedir = True else: clean_remotedir = False for srcdir in srcdirs: put_dir(srcdir, os.path.join(remote_dir, os.path.basename(srcdir))) pkgname = os.path.basename(srcdir) pkgdir = os.path.join(remote_dir, pkgname) remotebuilder = os.path.join(pkgdir, 'locdistbld.py') put(locdistbld, remotebuilder) remtmp = remote_tmpdir() result = run('%s %s -s %s -d %s -b "%s"' % (py, remotebuilder, pkgdir, remtmp, build_type)) #print result pkg = remote_listdir(remtmp)[ 0] # should only have one file in directory pkgpath = os.path.join(destdir, pkg) print 'retrieving built distribution %s' % pkgpath get(os.path.join(remtmp, pkg), pkgpath) pkgs.append(pkgpath) if debug: print 'removing %s' % remtmp rm_remote_tree(remtmp) if clean_remotedir: if debug: print 'removing %s' % remote_dir rm_remote_tree(remote_dir) return 0
def _remote_build_and_test(fname=None, pyversion='python', keep=False, branch=None, testargs='', hostname='', **kwargs): if fname is None: raise RuntimeError("_remote_build_and_test: missing arg 'fname'") remotedir = get_tmp_user_dir() remote_mkdir(remotedir) locbldtstfile = os.path.join(os.path.dirname(__file__), 'loc_bld_tst.py') pushfiles = [locbldtstfile] build_type = 'release' if fname.endswith('.py') else 'dev' if os.path.isfile(fname): pushfiles.append(fname) remoteargs = ['-f', os.path.basename(fname)] elif os.path.isdir(fname): put_dir(fname, os.path.join(remotedir, os.path.basename(fname))) if sys.platform.startswith('win'): vername = 'latest' #readlink doesn't work on windows, so try 'latest' else: vername = os.readlink(os.path.join(fname, 'downloads', 'latest')) remoteargs = [ '-f', os.path.join(os.path.basename(fname), 'downloads', vername, 'go-openmdao.py') ] else: remoteargs = ['-f', fname] if branch: remoteargs.append('--branch=%s' % branch) if testargs: remoteargs.append('--testargs="%s"' % testargs) try: result = push_and_run(pushfiles, runner=pyversion, remotedir=remotedir, args=remoteargs) return result.return_code finally: if not keep: print "removing remote directory: %s" % remotedir rm_remote_tree(remotedir)
def remote_build(srcdirs=(), destdir=None, build_type='bdist_egg', py=None, remote_dir=None, debug=False, **kwargs): """Take the Python distribution in the given directory, tar it up, ship it over to host, build it, and bring it back, placing it in the specified destination directory. """ from fabric.api import run, put, get locdistbld = os.path.join(os.path.dirname(__file__), 'locdistbld.py') pkgs = [] if remote_dir is None: remote_dir = get_tmp_user_dir() clean_remotedir = True else: clean_remotedir = False for srcdir in srcdirs: put_dir(srcdir, os.path.join(remote_dir, os.path.basename(srcdir))) pkgname = os.path.basename(srcdir) pkgdir = os.path.join(remote_dir, pkgname) remotebuilder = os.path.join(pkgdir, 'locdistbld.py') put(locdistbld, remotebuilder) remtmp = remote_tmpdir() result = run('%s %s -s %s -d %s -b "%s"' % (py, remotebuilder, pkgdir, remtmp, build_type)) #print result pkg = remote_listdir(remtmp)[0] # should only have one file in directory pkgpath = os.path.join(destdir, pkg) print 'retrieving built distribution %s' % pkgpath get(os.path.join(remtmp, pkg), pkgpath) pkgs.append(pkgpath) if debug: print 'removing %s' % remtmp rm_remote_tree(remtmp) if clean_remotedir: if debug: print 'removing %s' % remote_dir rm_remote_tree(remote_dir) return 0
def _remote_build_and_test(fname=None, pyversion='python', keep=False, branch=None, testargs='', hostname='', **kwargs): if fname is None: raise RuntimeError("_remote_build_and_test: missing arg 'fname'") remotedir = get_tmp_user_dir() remote_mkdir(remotedir) locbldtstfile = os.path.join(os.path.dirname(__file__), 'loc_bld_tst.py') pushfiles = [locbldtstfile] build_type = 'release' if fname.endswith('.py') else 'dev' if os.path.isfile(fname): pushfiles.append(fname) remoteargs = ['-f', os.path.basename(fname)] elif os.path.isdir(fname): put_dir(fname, os.path.join(remotedir, os.path.basename(fname))) if sys.platform.startswith('win'): vername = 'latest' #readlink doesn't work on windows, so try 'latest' else: vername = os.readlink(os.path.join(fname, 'downloads', 'latest')) remoteargs = ['-f', os.path.join(os.path.basename(fname), 'downloads', vername, 'go-openmdao.py')] else: remoteargs = ['-f', fname] if branch: remoteargs.append('--branch=%s' % branch) if testargs: remoteargs.append('--testargs="%s"' % testargs) try: result = push_and_run(pushfiles, runner=pyversion, remotedir=remotedir, args=remoteargs) return result.return_code finally: if not keep: print "removing remote directory: %s" % remotedir rm_remote_tree(remotedir)
def test_branch(parser, options, args=None): if args: print_sub_help(parser, 'test_branch') return -1 atexit.register(fabric_cleanup, True) paramiko.util.log_to_file('paramiko.log') options.filters = ['test_branch==true'] config, conn, ec2_hosts = process_options(options) if not options.hosts: parser.print_help() print "nothing to do - no hosts specified" sys.exit(0) if options.fname is None: # assume we're testing the current repo print 'creating tar file of current branch: ', options.fname = os.path.join(os.getcwd(), 'OpenMDAO-Framework-testbranch.tar') ziptarname = options.fname + '.gz' cleanup(ziptarname) # clean up the old tar file make_git_archive(options.fname, prefix='OpenMDAO-OpenMDAO-Framework-testbranch/') subprocess.check_call(['gzip', options.fname]) options.fname = os.path.abspath(ziptarname) print options.fname cleanup_tar = True else: cleanup_tar = False fname = options.fname if not (fname.startswith('http') or \ fname.startswith('git:') or fname.startswith('git@')): fname = os.path.abspath(os.path.expanduser(options.fname)) if fname.endswith('.tar.gz') or fname.endswith('.tar'): if not os.path.isfile(fname): print "can't find file '%s'" % fname sys.exit(-1) elif fname.endswith('.git') or \ (fname.startswith('http') and os.path.splitext(fname)[1]==''): pass else: parser.print_help() print "\nfilename '%s' must specify a tar file or git repository" % fname sys.exit(-1) testargs = '-v ' if options.verbose else '' testargs += '--gui ' if options.gui else '' testargs += '--skip-gui ' if options.skip_gui else '' testargs += options.testargs funct_kwargs = {'keep': options.keep, 'testargs': testargs, 'fname': fname, 'remotedir': get_tmp_user_dir(), 'branch': options.branch, 'cfg': config } try: retcode = run_host_processes(config, conn, ec2_hosts, options, funct=_remote_build_and_test, funct_kwargs=funct_kwargs, done_functs=[print_host_codes]) finally: if cleanup_tar: cleanup(ziptarname) if retcode == 0: cleanup('paramiko.log') return retcode
def _remote_build_and_test(fname=None, pyversion='python', keep=False, branch=None, testargs='', hostname='', cfg=None, **kwargs): if fname is None: raise RuntimeError("_remote_build_and_test: missing arg 'fname'") remotedir = get_tmp_user_dir() remote_mkdir(remotedir) locbldtstfile = os.path.join(os.path.dirname(__file__), 'loc_bld_tst.py') pushfiles = [locbldtstfile] build_type = 'release' if fname.endswith('.py') else 'dev' if cfg and cfg.has_option(hostname, 'pull_docs')and build_type == 'dev': pull_docs = cfg.getboolean(hostname, 'pull_docs') else: pull_docs = False if os.path.isfile(fname): pushfiles.append(fname) remoteargs = ['-f', os.path.basename(fname)] elif os.path.isdir(fname): put_dir(fname, os.path.join(remotedir, os.path.basename(fname))) if sys.platform.startswith('win'): vername = 'latest' # readlink doesn't work on windows, so try 'latest' else: vername = os.readlink(os.path.join(fname, 'downloads', 'latest')) remoteargs = ['-f', os.path.join(os.path.basename(fname), 'downloads', vername, 'go-openmdao-{}.py'.format(vername))] else: remoteargs = ['-f', fname] if branch: remoteargs.append('--branch=%s' % branch) if testargs: remoteargs.append('--testargs="%s"' % testargs) try: result = push_and_run(pushfiles, runner=pyversion, remotedir=remotedir, args=remoteargs) if pull_docs: print "pulling docs from %s" % hostname retrieve_docs(os.path.join('~', remotedir)) print "doc retrieval successful" else: print "not pulling docs from %s because pull_docs is False" % hostname return result.return_code finally: if build_type == 'dev': print "pulling any pngs from %s" % hostname try: retrieve_pngs(os.path.join('~', remotedir)) print "png retrieval successful" except Exception as exc: print "png retrieval failed:", exc if not keep: print "removing remote directory: %s" % remotedir rm_remote_tree(remotedir)
def test_branch(parser, options, args=None): if args: print_sub_help(parser, 'test_branch') return -1 atexit.register(fabric_cleanup, True) paramiko.util.log_to_file('paramiko.log') options.filters = ['test_branch==true'] config, conn, ec2_hosts = process_options(options) if not options.hosts: parser.print_help() print "nothing to do - no hosts specified" sys.exit(0) if options.fname is None: # assume we're testing the current repo print 'creating tar file of current branch: ', options.fname = os.path.join(os.getcwd(), 'OpenMDAO-Framework-testbranch.tar') ziptarname = options.fname + '.gz' cleanup(ziptarname) # clean up the old tar file make_git_archive(options.fname, prefix='OpenMDAO-OpenMDAO-Framework-testbranch/') subprocess.check_call(['gzip', options.fname]) options.fname = os.path.abspath(ziptarname) print options.fname cleanup_tar = True else: cleanup_tar = False fname = options.fname if not (fname.startswith('http') or \ fname.startswith('git:') or fname.startswith('git@')): fname = os.path.abspath(os.path.expanduser(options.fname)) if fname.endswith('.tar.gz') or fname.endswith('.tar'): if not os.path.isfile(fname): print "can't find file '%s'" % fname sys.exit(-1) elif fname.endswith('.git') or \ (fname.startswith('http') and os.path.splitext(fname)[1]==''): pass else: parser.print_help() print "\nfilename '%s' must specify a tar file or git repository" % fname sys.exit(-1) testargs = '-v ' if options.verbose else '' testargs += '--gui ' if options.gui else '' testargs += '--skip-gui ' if options.skip_gui else '' testargs += options.testargs funct_kwargs = { 'keep': options.keep, 'testargs': testargs, 'fname': fname, 'remotedir': get_tmp_user_dir(), 'branch': options.branch, 'cfg': config } try: retcode = run_host_processes(config, conn, ec2_hosts, options, funct=_remote_build_and_test, funct_kwargs=funct_kwargs, done_functs=[print_host_codes]) finally: if cleanup_tar: cleanup(ziptarname) if retcode == 0: cleanup('paramiko.log') return retcode
def _remote_build_and_test(fname=None, pyversion='python', keep=False, branch=None, testargs='', hostname='', cfg=None, **kwargs): if fname is None: raise RuntimeError("_remote_build_and_test: missing arg 'fname'") remotedir = get_tmp_user_dir() remote_mkdir(remotedir) locbldtstfile = os.path.join(os.path.dirname(__file__), 'loc_bld_tst.py') pushfiles = [locbldtstfile] build_type = 'release' if fname.endswith('.py') else 'dev' if cfg and cfg.has_option(hostname, 'pull_docs') and build_type == 'dev': pull_docs = cfg.getboolean(hostname, 'pull_docs') else: pull_docs = False if os.path.isfile(fname): pushfiles.append(fname) remoteargs = ['-f', os.path.basename(fname)] elif os.path.isdir(fname): put_dir(fname, os.path.join(remotedir, os.path.basename(fname))) if sys.platform.startswith('win'): vername = 'latest' # readlink doesn't work on windows, so try 'latest' else: vername = os.readlink(os.path.join(fname, 'downloads', 'latest')) remoteargs = [ '-f', os.path.join(os.path.basename(fname), 'downloads', vername, 'go-openmdao.py') ] else: remoteargs = ['-f', fname] if branch: remoteargs.append('--branch=%s' % branch) if testargs: remoteargs.append('--testargs="%s"' % testargs) try: result = push_and_run(pushfiles, runner=pyversion, remotedir=remotedir, args=remoteargs) if pull_docs: print "pulling docs from %s" % hostname retrieve_docs(os.path.join('~', remotedir)) print "doc retrieval successful" else: print "not pulling docs from %s because pull_docs is False" % hostname return result.return_code finally: if build_type == 'dev': print "pulling any pngs from %s" % hostname try: retrieve_pngs(os.path.join('~', remotedir)) print "png retrieval successful" except Exception as exc: print "png retrieval failed:", exc if not keep: print "removing remote directory: %s" % remotedir rm_remote_tree(remotedir)
def test_branch(argv=None): atexit.register(fabric_cleanup, True) paramiko.util.log_to_file('paramiko.log') if argv is None: argv = sys.argv[1:] parser = ArgumentParser() add_config_options(parser) parser.add_argument("-k","--keep", action="store_true", dest='keep', help="Don't delete the temporary build directory. " "If testing on EC2 stop the instance instead of terminating it.") parser.add_argument("-f","--file", action="store", type=str, dest='fname', help="Pathname of a tarfile or URL of a git repo. " "Defaults to the current repo.") parser.add_argument("-b","--branch", action="store", type=str, dest='branch', help="If file is a git repo, supply branch name here") parser.add_argument("--testargs", action="store", type=str, dest='testargs', default='', help="args to be passed to openmdao test") options = parser.parse_args() options.filters = ['test_branch==true'] config, conn, ec2_hosts = process_options(options) if not options.hosts: parser.print_help() print "nothing to do - no hosts specified" sys.exit(0) startdir = os.getcwd() if options.fname is None: # assume we're testing the current repo print 'creating tar file of current branch: ', options.fname = os.path.join(os.getcwd(), 'testbranch.tar') ziptarname = options.fname+'.gz' cleanup(ziptarname) # clean up the old tar file make_git_archive(options.fname) subprocess.check_call(['gzip', options.fname]) options.fname = os.path.abspath(ziptarname) print options.fname cleanup_tar = True else: cleanup_tar = False fname = options.fname if not (fname.startswith('http') or fname.startswith('git:') or fname.startswith('git@')): fname = os.path.abspath(os.path.expanduser(options.fname)) if fname.endswith('.tar.gz') or fname.endswith('.tar'): if not os.path.isfile(fname): print "can't find file '%s'" % fname sys.exit(-1) elif fname.endswith('.git') or (fname.startswith('http') and os.path.splitext(fname)[1]==''): pass else: parser.print_help() print "\nfilename '%s' must specify a tar file or git repository" % fname sys.exit(-1) funct_kwargs = { 'keep': options.keep, 'testargs': options.testargs, 'fname': fname, 'remotedir': get_tmp_user_dir(), 'branch': options.branch, } try: retcode = run_host_processes(config, conn, ec2_hosts, options, funct=_remote_build_and_test, funct_kwargs=funct_kwargs, done_functs=[print_host_codes]) finally: if cleanup_tar: cleanup(ziptarname) if retcode == 0: cleanup('paramiko.log') return retcode
def test_branch(argv=None): atexit.register(fabric_cleanup, True) paramiko.util.log_to_file('paramiko.log') if argv is None: argv = sys.argv[1:] parser = CfgOptionParser(usage="%prog [OPTIONS] -- [options to openmdao_test]") parser.add_option("-k","--keep", action="store_true", dest='keep', help="Don't delete the temporary build directory. " "If testing on EC2 stop the instance instead of terminating it.") parser.add_option("-f","--file", action="store", type='string', dest='fname', help="Pathname of a tarfile or URL of a git repo. " "Defaults to the current repo.") parser.add_option("-b","--branch", action="store", type='string', dest='branch', help="If file is a git repo, supply branch name here") (options, args) = parser.parse_args(argv) config, conn, ec2_hosts = process_options(options, parser) if not options.hosts: parser.print_help() print "nothing to do - no hosts specified" sys.exit(0) startdir = os.getcwd() if options.fname is None: # assume we're testing the current repo print 'creating tar file of current branch: ', options.fname = os.path.join(os.getcwd(), 'testbranch.tar') ziptarname = options.fname+'.gz' cleanup(ziptarname) # clean up the old tar file make_git_archive(options.fname) subprocess.check_call(['gzip', options.fname]) options.fname = os.path.abspath(ziptarname) print options.fname cleanup_tar = True else: cleanup_tar = False fname = os.path.abspath(os.path.expanduser(options.fname)) if fname.endswith('.tar.gz') or fname.endswith('.tar'): if not os.path.isfile(fname): print "can't find file '%s'" % fname sys.exit(-1) elif fname.endswith('.git'): pass else: parser.print_help() print "\nfilename must end in '.tar.gz', '.tar', or '.git'" sys.exit(-1) funct_kwargs = { 'keep': options.keep, 'testargs': args, 'fname': fname, 'remotedir': get_tmp_user_dir(), 'branch': options.branch, } try: retcode = run_host_processes(config, conn, ec2_hosts, options, funct=_remote_build_and_test, funct_kwargs=funct_kwargs) finally: if cleanup_tar: cleanup(ziptarname) if retcode == 0: cleanup('paramiko.log') return retcode
def test_branch(argv=None): atexit.register(fabric_cleanup, True) paramiko.util.log_to_file('paramiko.log') if argv is None: argv = sys.argv[1:] parser = ArgumentParser() add_config_options(parser) parser.add_argument( "-k", "--keep", action="store_true", dest='keep', help="Don't delete the temporary build directory. " "If testing on EC2 stop the instance instead of terminating it.") parser.add_argument("-f", "--file", action="store", type=str, dest='fname', help="Pathname of a tarfile or URL of a git repo. " "Defaults to the current repo.") parser.add_argument("-b", "--branch", action="store", type=str, dest='branch', help="If file is a git repo, supply branch name here") parser.add_argument("--testargs", action="store", type=str, dest='testargs', default='', help="args to be passed to openmdao test") options = parser.parse_args() options.filters = ['test_branch==true'] config, conn, ec2_hosts = process_options(options) if not options.hosts: parser.print_help() print "nothing to do - no hosts specified" sys.exit(0) startdir = os.getcwd() if options.fname is None: # assume we're testing the current repo print 'creating tar file of current branch: ', options.fname = os.path.join(os.getcwd(), 'testbranch.tar') ziptarname = options.fname + '.gz' cleanup(ziptarname) # clean up the old tar file make_git_archive(options.fname) subprocess.check_call(['gzip', options.fname]) options.fname = os.path.abspath(ziptarname) print options.fname cleanup_tar = True else: cleanup_tar = False fname = options.fname if not (fname.startswith('http') or fname.startswith('git:') or fname.startswith('git@')): fname = os.path.abspath(os.path.expanduser(options.fname)) if fname.endswith('.tar.gz') or fname.endswith('.tar'): if not os.path.isfile(fname): print "can't find file '%s'" % fname sys.exit(-1) elif fname.endswith('.git') or (fname.startswith('http') and os.path.splitext(fname)[1] == ''): pass else: parser.print_help() print "\nfilename '%s' must specify a tar file or git repository" % fname sys.exit(-1) funct_kwargs = { 'keep': options.keep, 'testargs': options.testargs, 'fname': fname, 'remotedir': get_tmp_user_dir(), 'branch': options.branch, } try: retcode = run_host_processes(config, conn, ec2_hosts, options, funct=_remote_build_and_test, funct_kwargs=funct_kwargs, done_functs=[print_host_codes]) finally: if cleanup_tar: cleanup(ziptarname) if retcode == 0: cleanup('paramiko.log') return retcode