Exemplo n.º 1
0
 def run_cleanup_in_subprocess(self):
     """ Runs the cleanup job in a subprocess. Necessary as setup.py is often
         changing the state of the virtualenv as it goes, and working_set in
         memory might not reflect the real state of the world
     """
     # Using the entry pount here instead of the module. This is because the module may have
     # been moved by the time we go to run it, eg if we just updated pkglib itself.
     cmd = [sys.executable, os.path.join(sys.exec_prefix, 'bin', 'pycleanup')]
     for arg in ['-v', '-n', '--verbose', '--dry-run']:
         if arg in sys.argv:
             cmd.append(arg)
     cmdline.run(cmd, capture_stdout=False)
Exemplo n.º 2
0
def run_graph_easy(entries, renderer, outfile=None):
    """ Given the path edge entries, run the graphing tools and produce the
        output.

        Parameters
        ----------
        entries :    `list`
            Path edges
        renderer :   `str`
            One of 'ascii', 'boxart' or 'graphviz'
        outfile :   `str`
            File to save to, only for graphviz. If None, it will delete the
            generated file.
    """
    if not CONFIG.graph_easy:
        log.warn("Graph-Easy not configured, please set graph_easy variable in pkglib config")
        return

    from path import path
    if renderer == 'graphviz' and not os.getenv('DISPLAY'):
        log.info("No DISPLAY set, using ascii renderer")
        renderer = 'ascii'

    instream = '\n'.join(entries)
    if os.path.isfile(CONFIG.graph_easy / 'bin' / 'graph-easy'):
        with chdir(CONFIG.graph_easy / 'lib'):
            if renderer == 'graphviz':
                delete = False
                if not outfile:
                    tmpdir = path(tempfile.mkdtemp())
                    outfile = tmpdir / 'depgraph.png'
                    delete = True
                outfile = path(outfile)
                outfile = outfile.abspath()

                run(['-c', ('../bin/graph-easy --as={0} | '
                            '/usr/bin/dot -Tpng -o {1}'
                            .format(renderer, outfile))],
                    capture_stdout=False, stdin=instream, shell=True)
                if not outfile.isfile:
                    log.error("Failed to create image file.")
                    return
                webbrowser.open('file://%s' % outfile)
                if delete:
                    time.sleep(5)
                    shutil.rmtree(tmpdir)
                else:
                    log.info("Created graph at %s" % outfile)
            else:
                run(['../bin/graph-easy', '--as=%s' % renderer],
                    capture_stdout=False, stdin=instream)
        return
    log.warn("Can't find graphing tool at %s" % CONFIG.graph_easy)
Exemplo n.º 3
0
    def mirror_eggs(self, file_root, target_host, target_root,
                    target_packages=None, subprocesses=10):
        """  Mirrors egg files from this PyPi instance to a target host and
             path. Used for filling out a cache that can be used by
             CONFIG.installer_search_path.

             Parameters
             ----------
             file_root : `str`
                 filesystem path to the root of the file store
             target_host : `str`
                 host to mirror to
             target_root : `str`
                 filesystem path to mirror to on target host
             target_packages : `list` or None
                 list of packages to mirror. Use None for all.
             subprocesses : `int`
                 number of subprocesses to spawn when doing the mirror
        """
        from path import path
        file_root = path(file_root)
        target_root = path(target_root)

        pkg_dirs, target_dirs = self.get_mirror_targets(file_root, target_root,
                                                        target_packages)

        print "Creating target root dirs"
        run(['/usr/bin/ssh', target_host, 'mkdir -p ' + ' '.join(target_dirs)])

        work = []
        for pkg in pkg_dirs:
            # Filter non-egg and dev packages out, as this is a site-packages
            # mirror which won't work with source packages.
            files = [i for i in pkg.files()
                     if i.basename().endswith('egg')
                     and not 'dev' in i.basename()]
            print "Found %s (%d files)" % (pkg.basename(), len(files))
            if files:
                cmd = (['/usr/bin/rsync', '-av', '--ignore-existing'] +
                       [i.abspath().strip() for i in files] +
                       [target_host + ':' + target_root /
                        self.get_mirror_dirname(pkg.basename())]
                      )
                work.append(cmd)

        # Using multiprocessing here to multiplex the transfers
        if subprocesses > 1:
            pool = Pool(processes=subprocesses)
            pool.map(run, work)
        else:
            map(run, work)

        self.unpack_eggs(files, target_host, target_root)
Exemplo n.º 4
0
def run_with_coverage(cmd, pytestconfig, coverage='coverage', cd=None, **kwargs):
    """
    Run a given command with coverage enabled. This won't make any sense
    if the command isn't a python script.

    This must be run within a pytest session that has been setup with
    the '--cov=xxx' options, and therefore requires the pytestconfig
    argument that can be retrieved from the standard pytest funcarg
    of the same name.

    Parameters
    ----------
    cmd: `List`
        Command to run
    pytestconfig: `pytest._config.Config`
        Pytest configuration object
    coverage: `str`
        Path to the coverage executable
    cd: `str`
        If not None, will change to this directory before running the cmd.
        This is the directory that the coverage files will be created in.
    kwargs: keyword arguments
        Any extra arguments to pass to `pkglib.cmdline.run`

    Returns
    -------
    `str` standard output

    Examples
    --------

    >>> def test_example(pytestconfig):
    ...   cmd = ['python','myscript.py']
    ...   run_with_coverage(cmd, pytestconfig)
    """
    if isinstance(cmd, str):
        cmd = [cmd]

    args = [coverage, 'run', '-p']
    if pytestconfig.option.cov_source:
        source_dirs = ",".join(pytestconfig.option.cov_source)
        args += ['--source=%s' % source_dirs]
    args += cmd
    if cd:
        with manage.chdir(cd):
            return run(args, **kwargs)
    return run(args, **kwargs)
Exemplo n.º 5
0
 def get_open_files(self):
     """ Returns open files under our site-packages
     """
     # It's far cheaper to run lsof for all files and search later than running it with
     # the +D option to only return results under a certain directory
     cmd = "/usr/sbin/lsof 2>/dev/null | grep %s | awk '{ print $2 \" \" $9 }'" % \
           self.site_packages
     return [i.split() for i in cmdline.run(cmd, capture_stdout=True, check_rc=False, shell=True).split('\n') if i]
Exemplo n.º 6
0
 def get_open_files(self):
     """ Returns open files under our site-packages
     """
     # It's far cheaper to run lsof for all files and search later than
     # running it with the +D option to only return results under a certain
     # directory
     # TODO: this might not be on the path, and be hidden by the >/dev/null
     cmd = ("lsof 2>/dev/null | grep {} |"
            "awk '{{ print $2 \" \" $9 }}'").format(self.site_packages)
     return [i.split() for i in
             cmdline.run(cmd, capture_stdout=True, check_rc=False,
                         shell=True).split('\n') if i]
Exemplo n.º 7
0
    def unpack_eggs(self, files, target_host, target_root):
        """ Unpacks all eggs on the target host and root
        """
        print "Unpacking eggs: %r" % files

        target_eggs = [target_root / self.get_mirror_dirname(f.parent.basename()) \
                       / f.name for f in files]
        cmd = """set -x
            for EGG in %s; do
                if [ -f $EGG ]; then
                    echo Unzipping $EGG
                    ZIPFILE=./.tmp.`basename $EGG`
                    mv $EGG $ZIPFILE &&  \
                    mkdir $EGG &&  \
                    unzip -q $ZIPFILE -d $EGG && \
                    rm $ZIPFILE &&  \
                    chmod -R 555 $EGG
                fi
            done""" % ' '.join(target_eggs)
        print "Running cmd on %s" % target_host
        print cmd
        run(['/usr/bin/ssh', target_host, cmd])