Example #1
0
    def test_non_existing_dep(self):
        requested = {
            REPO_PYPI: [get_req('dep1 == 1000')]
        }
        interpreter = 'python3'
        is_current = True
        options = {'virtualenv_options': [],
                   'pyvenv_options': [],
                   }
        pip_options = []

        with patch.object(envbuilder._FadesEnvBuilder, 'create_env') as mock_create:
            with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
                mock_create.return_value = ('env_path', 'env_bin_path', 'pip_installed')
                mock_mgr_c.return_value = self.FailInstallManager()
                with patch.object(envbuilder, 'destroy_venv', spec=True) as mock_destroy:
                    with self.assertRaises(FadesError) as cm:
                        envbuilder.create_venv(
                            requested,
                            interpreter,
                            is_current,
                            options,
                            pip_options)
                    self.assertEqual(str(cm.exception), 'Dependency installation failed')
                    mock_destroy.assert_called_once_with('env_path')

        self.assertLoggedDebug("Installation Step failed, removing virtualenv")
Example #2
0
    def test_non_existing_dep(self):
        requested = {REPO_PYPI: [get_req('dep1 == 1000')]}
        interpreter = 'python3'
        is_current = True
        avoid_pip_upgrade = False
        options = {
            'virtualenv_options': [],
            'pyvenv_options': [],
        }
        pip_options = []

        with patch.object(envbuilder._FadesEnvBuilder,
                          'create_env') as mock_create:
            with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
                mock_create.return_value = ('env_path', 'env_bin_path',
                                            'pip_installed')
                mock_mgr_c.return_value = self.FailInstallManager()
                with patch.object(envbuilder, 'destroy_venv',
                                  spec=True) as mock_destroy:
                    with self.assertRaises(FadesError) as cm:
                        envbuilder.create_venv(requested, interpreter,
                                               is_current, options,
                                               pip_options, avoid_pip_upgrade)
                    self.assertEqual(str(cm.exception),
                                     'Dependency installation failed')
                    mock_destroy.assert_called_once_with('env_path')

        self.assertLoggedDebug("Installation Step failed, removing virtualenv")
Example #3
0
    def test_unknown_repo(self):
        requested = {
            'unknown': {'dep': ''}
        }
        interpreter = 'python3'
        is_current = True
        with patch.object(envbuilder.FadesEnvBuilder, 'create_env') as mock_create:
            with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
                mock_create.return_value = ('env_path', 'env_bin_path', 'pip_installed')
                mock_mgr_c.return_value = self.FakeManager()
                envbuilder.create_venv(requested, interpreter, is_current)

        self.assertLoggedWarning("Install from 'unknown' not implemented")
Example #4
0
    def test_unknown_repo(self):
        requested = {'unknown': {'dep': ''}}
        interpreter = 'python3'
        is_current = True
        with patch.object(envbuilder.FadesEnvBuilder,
                          'create_env') as mock_create:
            with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
                mock_create.return_value = ('env_path', 'env_bin_path',
                                            'pip_installed')
                mock_mgr_c.return_value = self.FakeManager()
                envbuilder.create_venv(requested, interpreter, is_current)

        self.assertLoggedWarning("Install from 'unknown' not implemented")
Example #5
0
    def test_different_versions(self):
        requested = {REPO_PYPI: [get_req('dep1 == v1'), get_req('dep2 == v2')]}
        interpreter = 'python3'
        is_current = True
        avoid_pip_upgrade = False
        options = {
            "virtualenv_options": [],
            "pyvenv_options": [],
        }
        pip_options = []
        with patch.object(envbuilder._FadesEnvBuilder,
                          'create_env') as mock_create:
            with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
                mock_create.return_value = ('env_path', 'env_bin_path',
                                            'pip_installed')
                mock_mgr_c.return_value = fake_manager = self.FakeManager()
                fake_manager.really_installed = {'dep1': 'vX', 'dep2': 'v2'}
                _, installed = envbuilder.create_venv(requested, interpreter,
                                                      is_current, options,
                                                      pip_options,
                                                      avoid_pip_upgrade)

        self.assertEqual(installed,
                         {REPO_PYPI: {
                             'dep1': 'vX',
                             'dep2': 'v2',
                         }})
Example #6
0
    def test_create_simple(self):
        requested = {
            REPO_PYPI: [get_req('dep1 == v1'), get_req('dep2 == v2')]
        }
        interpreter = 'python3'
        is_current = True
        options = {"virtualenv_options": [],
                   "pyvenv_options": [],
                   }
        pip_options = []
        with patch.object(envbuilder._FadesEnvBuilder, 'create_env') as mock_create:
            with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
                mock_create.return_value = ('env_path', 'env_bin_path', 'pip_installed')
                mock_mgr_c.return_value = fake_manager = self.FakeManager()
                fake_manager.really_installed = {'dep1': 'v1', 'dep2': 'v2'}
                venv_data, installed = envbuilder.create_venv(requested, interpreter, is_current,
                                                              options, pip_options)

        self.assertEqual(venv_data, {
            'env_bin_path': 'env_bin_path',
            'env_path': 'env_path',
            'pip_installed': 'pip_installed',
        })
        self.assertDictEqual(installed, {
            REPO_PYPI: {
                'dep1': 'v1',
                'dep2': 'v2',
            }
        })
Example #7
0
    def test_create_simple(self):
        requested = {REPO_PYPI: [get_req('dep1 == v1'), get_req('dep2 == v2')]}
        interpreter = 'python3'
        is_current = True
        with patch.object(envbuilder.FadesEnvBuilder,
                          'create_env') as mock_create:
            with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
                mock_create.return_value = ('env_path', 'env_bin_path',
                                            'pip_installed')
                mock_mgr_c.return_value = fake_manager = self.FakeManager()
                fake_manager.really_installed = {'dep1': 'v1', 'dep2': 'v2'}
                venv_data, installed = envbuilder.create_venv(
                    requested, interpreter, is_current)

        self.assertEqual(
            venv_data, {
                'env_bin_path': 'env_bin_path',
                'env_path': 'env_path',
                'pip_installed': 'pip_installed',
            })
        self.assertDictEqual(installed,
                             {REPO_PYPI: {
                                 'dep1': 'v1',
                                 'dep2': 'v2',
                             }})
Example #8
0
    def test_unknown_repo(self):
        requested = {'unknown': {'dep': ''}}
        interpreter = 'python3'
        is_current = True
        avoid_pip_upgrade = False
        options = {
            "virtualenv_options": [],
            "pyvenv_options": [],
        }
        pip_options = []
        with patch.object(envbuilder._FadesEnvBuilder,
                          'create_env') as mock_create:
            with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
                mock_create.return_value = ('env_path', 'env_bin_path',
                                            'pip_installed')
                mock_mgr_c.return_value = self.FakeManager()
                envbuilder.create_venv(requested, interpreter, is_current,
                                       options, pip_options, avoid_pip_upgrade)

        self.assertLoggedWarning("Install from 'unknown' not implemented")
Example #9
0
    def test_different_versions(self):
        requested = {
            REPO_PYPI: [get_req('dep1 == v1'), get_req('dep2 == v2')]
        }
        interpreter = 'python3'
        is_current = True
        with patch.object(envbuilder.FadesEnvBuilder, 'create_env') as mock_create:
            with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
                mock_create.return_value = ('env_path', 'env_bin_path', 'pip_installed')
                mock_mgr_c.return_value = fake_manager = self.FakeManager()
                fake_manager.really_installed = {'dep1': 'vX', 'dep2': 'v2'}
                _, installed = envbuilder.create_venv(requested, interpreter, is_current)

        self.assertEqual(installed, {
            REPO_PYPI: {
                'dep1': 'vX',
                'dep2': 'v2',
            }
        })
Example #10
0
    def test_create_vcs(self):
        requested = {
            REPO_VCS: [parsing.VCSDependency("someurl")]
        }
        interpreter = 'python3'
        is_current = True
        options = {"virtualenv_options": [], "pyvenv_options": []}
        pip_options = []
        with patch.object(envbuilder._FadesEnvBuilder, 'create_env') as mock_create:
            with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
                mock_create.return_value = ('env_path', 'env_bin_path', 'pip_installed')
                mock_mgr_c.return_value = self.FakeManager()
                venv_data, installed = envbuilder.create_venv(requested, interpreter, is_current,
                                                              options, pip_options)

        self.assertEqual(venv_data, {
            'env_bin_path': 'env_bin_path',
            'env_path': 'env_path',
            'pip_installed': 'pip_installed',
        })
        self.assertDictEqual(installed, {REPO_VCS: {'someurl': None}})
Example #11
0
    def test_create_vcs(self):
        requested = {
            REPO_VCS: [parsing.VCSDependency("someurl")]
        }
        interpreter = 'python3'
        is_current = True
        options = {"virtualenv_options": [], "pyvenv_options": []}
        pip_options = []
        with patch.object(envbuilder._FadesEnvBuilder, 'create_env') as mock_create:
            with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
                mock_create.return_value = ('env_path', 'env_bin_path', 'pip_installed')
                mock_mgr_c.return_value = self.FakeManager()
                venv_data, installed = envbuilder.create_venv(requested, interpreter, is_current,
                                                              options, pip_options)

        self.assertEqual(venv_data, {
            'env_bin_path': 'env_bin_path',
            'env_path': 'env_path',
            'pip_installed': 'pip_installed',
        })
        self.assertDictEqual(installed, {REPO_VCS: {'someurl': None}})
Example #12
0
    def test_create_simple(self):
        requested = {REPO_PYPI: [get_req('dep1 == v1'), get_req('dep2 == v2')]}
        interpreter = 'python3'
        is_current = True
        avoid_pip_upgrade = False
        options = {
            "virtualenv_options": [],
            "pyvenv_options": [],
        }
        pip_options = []
        with patch.object(envbuilder._FadesEnvBuilder,
                          'create_env') as mock_create:
            with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
                mock_create.return_value = ('env_path', 'env_bin_path',
                                            'pip_installed')
                mock_mgr_c.return_value = fake_manager = self.FakeManager()
                fake_manager.really_installed = {'dep1': 'v1', 'dep2': 'v2'}
                venv_data, installed = envbuilder.create_venv(
                    requested, interpreter, is_current, options, pip_options,
                    avoid_pip_upgrade)

        self.assertEqual(
            venv_data, {
                'env_bin_path': 'env_bin_path',
                'env_path': 'env_path',
                'pip_installed': 'pip_installed',
            })
        self.assertDictEqual(installed,
                             {REPO_PYPI: {
                                 'dep1': 'v1',
                                 'dep2': 'v2',
                             }})
        expected_pipmanager_call = call('env_bin_path',
                                        pip_installed='pip_installed',
                                        options=[],
                                        avoid_pip_upgrade=avoid_pip_upgrade)
        self.assertEqual(mock_mgr_c.call_args, expected_pipmanager_call)
Example #13
0
def go():
    """Make the magic happen."""
    parser = argparse.ArgumentParser(
        prog='PROG',
        epilog=help_epilog,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument(
        '-V',
        '--version',
        action='store_true',
        help="show version and info about the system, and exit")
    parser.add_argument(
        '-v',
        '--verbose',
        action='store_true',
        help="send all internal debugging lines to stderr, which may be very "
        "useful to debug any problem that may arise.")
    parser.add_argument(
        '-q',
        '--quiet',
        action='store_true',
        help="don't show anything (unless it has a real problem), so the "
        "original script stderr is not polluted at all.")
    parser.add_argument(
        '-d',
        '--dependency',
        action='append',
        help="specify dependencies through command line (this option can be "
        "used multiple times)")
    parser.add_argument(
        '-r',
        '--requirement',
        action='append',
        help="indicate files to read dependencies from (this option can be "
        "used multiple times)")
    parser.add_argument('-p',
                        '--python',
                        action='store',
                        help=("Specify the Python interpreter to use.\n"
                              " Default is: %s") % (sys.executable, ))
    parser.add_argument(
        '-x',
        '--exec',
        dest='executable',
        action='store_true',
        help=("Indicate that the child_program should be looked up in the "
              "virtualenv."))
    parser.add_argument('-i',
                        '--ipython',
                        action='store_true',
                        help="use IPython shell.")
    parser.add_argument('-m',
                        '--module',
                        action='store',
                        help="Run library module as a script")
    parser.add_argument('--system-site-packages',
                        action='store_true',
                        default=False,
                        help=("Give the virtual environment access to the "
                              "system site-packages dir."))
    parser.add_argument(
        '--virtualenv-options',
        action='append',
        default=[],
        help=(
            "Extra options to be supplied to virtualenv. (this option can be "
            "used multiple times)"))
    parser.add_argument('--check-updates',
                        action='store_true',
                        help=("check for packages updates"))
    parser.add_argument(
        '--no-precheck-availability',
        action='store_true',
        help=("Don't check if the packages exists in PyPI before actually try "
              "to install them."))
    parser.add_argument(
        '--pip-options',
        action='append',
        default=[],
        help=("Extra options to be supplied to pip. (this option can be "
              "used multiple times)"))
    parser.add_argument(
        '--python-options',
        action='append',
        default=[],
        help=("Extra options to be supplied to python. (this option can be "
              "used multiple times)"))
    parser.add_argument(
        '--rm',
        dest='remove',
        metavar='UUID',
        help=("Remove a virtualenv by UUID. See --get-venv-dir option to "
              "easily find out the UUID."))
    parser.add_argument(
        '--clean-unused-venvs',
        action='store',
        help=("This option remove venvs that haven't been used for more than "
              "CLEAN_UNUSED_VENVS days. Appart from that, will compact usage "
              "stats file.\n"
              "When this option is present, the cleaning takes place at the "
              "beginning of the execution."))
    parser.add_argument(
        '--get-venv-dir',
        action='store_true',
        help=("Show the virtualenv base directory (which includes the "
              "virtualenv UUID) and quit."))
    parser.add_argument('child_program', nargs='?', default=None)
    parser.add_argument('child_options', nargs=argparse.REMAINDER)

    cli_args = _get_normalized_args(parser)

    # update args from config file (if needed).
    args = file_options.options_from_file(cli_args)

    # validate input, parameters, and support some special options
    if args.version:
        print("Running 'fades' version", fades.__version__)
        print("    Python:", sys.version_info)
        print("    System:", platform.platform())
        return 0

    # set up logger and dump basic version info
    logger = fades_logger.set_up(args.verbose, args.quiet)
    logger.debug("Running Python %s on %r", sys.version_info,
                 platform.platform())
    logger.debug("Starting fades v. %s", fades.__version__)
    logger.debug("Arguments: %s", args)

    # verify that the module is NOT being used from a virtualenv
    if detect_inside_virtualenv(sys.prefix, getattr(sys, 'real_prefix', None),
                                getattr(sys, 'base_prefix', None)):
        logger.error(
            "fades is running from inside a virtualenv (%r), which is not supported",
            sys.prefix)
        raise FadesError("Cannot run from a virtualenv")

    if args.verbose and args.quiet:
        logger.warning("Overriding 'quiet' option ('verbose' also requested)")

    # start the virtualenvs manager
    venvscache = cache.VEnvsCache(
        os.path.join(helpers.get_basedir(), 'venvs.idx'))
    # start usage manager
    usage_manager = envbuilder.UsageManager(
        os.path.join(helpers.get_basedir(), 'usage_stats'), venvscache)

    if args.clean_unused_venvs:
        try:
            max_days_to_keep = int(args.clean_unused_venvs)
        except ValueError:
            logger.error("clean_unused_venvs must be an integer.")
            raise FadesError('clean_unused_venvs not an integer')

        usage_manager.clean_unused_venvs(max_days_to_keep)
        return 0

    uuid = args.remove
    if uuid:
        venv_data = venvscache.get_venv(uuid=uuid)
        if venv_data:
            # remove this venv from the cache
            env_path = venv_data.get('env_path')
            if env_path:
                envbuilder.destroy_venv(env_path, venvscache)
            else:
                logger.warning(
                    "Invalid 'env_path' found in virtualenv metadata: %r. "
                    "Not removing virtualenv.", env_path)
        else:
            logger.warning('No virtualenv found with uuid: %s.', uuid)
        return 0

    # decided which the child program really is
    analyzable_child_program, child_program = decide_child_program(
        args.executable, args.child_program)

    # Group and merge dependencies
    indicated_deps = consolidate_dependencies(args.ipython,
                                              analyzable_child_program,
                                              args.requirement,
                                              args.dependency)

    # Check for packages updates
    if args.check_updates:
        helpers.check_pypi_updates(indicated_deps)

    # get the interpreter version requested for the child_program
    interpreter, is_current = helpers.get_interpreter_version(args.python)

    # options
    pip_options = args.pip_options  # pip_options mustn't store.
    python_options = args.python_options
    options = {}
    options['pyvenv_options'] = []
    options['virtualenv_options'] = args.virtualenv_options
    if args.system_site_packages:
        options['virtualenv_options'].append("--system-site-packages")
        options['pyvenv_options'] = ["--system-site-packages"]

    create_venv = False
    venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options)
    if venv_data:
        env_path = venv_data['env_path']
        # A venv was found in the cache check if its valid or re-generate it.
        if not os.path.exists(env_path):
            logger.warning(
                "Missing directory (the virtualenv will be re-created): %r",
                env_path)
            venvscache.remove(env_path)
            create_venv = True
    else:
        create_venv = True

    if create_venv:
        # Check if the requested packages exists in pypi.
        if not args.no_precheck_availability and indicated_deps.get('pypi'):
            logger.info(
                "Checking the availabilty of dependencies in PyPI. "
                "You can use '--no-precheck-availability' to avoid it.")
            if not helpers.check_pypi_exists(indicated_deps):
                logger.error("An indicated dependency doesn't exist. Exiting")
                raise FadesError("Required dependency does not exist")

        # Create a new venv
        venv_data, installed = envbuilder.create_venv(indicated_deps,
                                                      args.python, is_current,
                                                      options, pip_options)
        # store this new venv in the cache
        venvscache.store(installed, venv_data, interpreter, options)

    if args.get_venv_dir:
        # all it was requested is the virtualenv's path, show it and quit (don't run anything)
        print(venv_data['env_path'])
        return 0

    # run forest run!!
    python_exe = 'ipython' if args.ipython else 'python'
    python_exe = os.path.join(venv_data['env_bin_path'], python_exe)

    # add the virtualenv /bin path to the child PATH.
    environ_path = venv_data['env_bin_path']
    if 'PATH' in os.environ:
        environ_path += os.pathsep + os.environ['PATH']
    os.environ['PATH'] = environ_path

    # store usage information
    usage_manager.store_usage_stat(venv_data, venvscache)
    if args.module:
        logger.debug("Executing module %r", args.module)
        module_option = ["-m"] + args.module.split()
        cmd = [python_exe] + python_options + module_option
        p = subprocess.Popen(cmd)
    elif child_program is None:
        interactive = True
        logger.debug(
            "Calling the interactive Python interpreter with arguments %r",
            python_options)
        cmd = [python_exe] + python_options
        p = subprocess.Popen(cmd)
    else:
        interactive = False
        if args.executable:
            cmd = [os.path.join(venv_data['env_bin_path'], child_program)]
            logger.debug("Calling child program %r with options %s",
                         child_program, args.child_options)
        else:
            cmd = [python_exe] + python_options + [child_program]
            logger.debug(
                "Calling Python interpreter with arguments %s to execute the child program"
                " %r with options %s", python_options, child_program,
                args.child_options)

        try:
            p = subprocess.Popen(cmd + args.child_options)
        except FileNotFoundError:
            logger.error("Command not found: %s", child_program)
            raise FadesError("Command not found")

    def _signal_handler(signum, _):
        """Handle signals received by parent process, send them to child.

        The only exception is CTRL-C, that is generated *from* the interactive
        interpreter (it's a keyboard combination!), so we swallow it for the
        interpreter to not see it twice.
        """
        if interactive and signum == signal.SIGINT:
            logger.debug("Swallowing signal %s", signum)
        else:
            logger.debug("Redirecting signal %s to child", signum)
            os.kill(p.pid, signum)

    # redirect the useful signals
    for s in REDIRECTED_SIGNALS:
        signal.signal(s, _signal_handler)

    # wait child to finish, end
    rc = p.wait()
    if rc:
        logger.debug("Child process not finished correctly: returncode=%d", rc)
    return rc
Example #14
0
def go():
    """Make the magic happen."""
    parser = argparse.ArgumentParser(
        prog='fades',
        epilog=HELP_EPILOG,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument(
        '-V',
        '--version',
        action='store_true',
        help="show version and info about the system, and exit")
    parser.add_argument(
        '-d',
        '--dependency',
        action='append',
        help=
        "specify dependencies through command line (this option can be used multiple times)"
    )
    parser.add_argument(
        '-r',
        '--requirement',
        action='append',
        help=
        "indicate files to read dependencies from (this option can be used multiple times)"
    )
    parser.add_argument(
        '-p',
        '--python',
        action='store',
        help="specify the Python interpreter to use; the default is: {}".
        format(sys.executable))
    parser.add_argument('-i',
                        '--ipython',
                        action='store_true',
                        help="use IPython shell when in interactive mode")
    parser.add_argument(
        '--system-site-packages',
        action='store_true',
        default=False,
        help=
        "give the virtual environment access to the system site-packages dir.")
    parser.add_argument(
        '--virtualenv-options',
        action='append',
        default=[],
        help=
        "extra options to be supplied to virtualenv (this option can be used multiple times)"
    )
    parser.add_argument('--check-updates',
                        action='store_true',
                        help="check for packages updates")
    parser.add_argument(
        '--no-precheck-availability',
        action='store_true',
        help=
        "don't check if the packages exists in PyPI before actually try to install them"
    )
    parser.add_argument(
        '--pip-options',
        action='append',
        default=[],
        help=
        "extra options to be supplied to pip (this option can be used multiple times)"
    )
    parser.add_argument(
        '--python-options',
        action='append',
        default=[],
        help=
        "extra options to be supplied to python (this option can be used multiple times)"
    )
    parser.add_argument(
        '--rm',
        dest='remove',
        metavar='UUID',
        help=
        "remove a virtualenv by UUID; see --get-venv-dir option to easily find out the UUID"
    )
    parser.add_argument(
        '--clean-unused-venvs',
        action='store',
        help=
        "remove venvs that haven't been used for more than the indicated days and compact "
        "usage stats file (all this takes place at the beginning of the execution)"
    )
    parser.add_argument(
        '--get-venv-dir',
        action='store_true',
        help=
        "show the virtualenv base directory (including the venv's UUID) and quit"
    )
    parser.add_argument(
        '-a',
        '--autoimport',
        action='store_true',
        help=
        "automatically import the specified dependencies in the interactive mode "
        "(ignored otherwise).")
    parser.add_argument(
        '--freeze',
        action='store',
        metavar='FILEPATH',
        help=
        "dump all the dependencies and its versions to the specified filepath "
        "(operating normally beyond that)")
    parser.add_argument(
        '--avoid-pip-upgrade',
        action='store_true',
        help=
        "disable the automatic pip upgrade that happens after the virtualenv is created "
        "and before the dependencies begin to be installed.")

    mutexg = parser.add_mutually_exclusive_group()
    mutexg.add_argument(
        '-v',
        '--verbose',
        action='store_true',
        help="send all internal debugging lines to stderr, which may be very "
        "useful to debug any problem that may arise")
    mutexg.add_argument(
        '-q',
        '--quiet',
        action='store_true',
        help="don't show anything (unless it has a real problem), so the "
        "original script stderr is not polluted at all")

    mutexg = parser.add_mutually_exclusive_group()
    mutexg.add_argument(
        '-x',
        '--exec',
        dest='executable',
        action='store_true',
        help=
        "execute the child_program (must be present) in the context of the virtualenv"
    )
    mutexg.add_argument(
        '-m',
        '--module',
        action='store_true',
        help=
        "run library module as a script (same behaviour than Python's -m parameter)"
    )

    parser.add_argument('child_program', nargs='?', default=None)
    parser.add_argument('child_options', nargs=argparse.REMAINDER)

    cli_args = parser.parse_args()

    # update args from config file (if needed).
    args = file_options.options_from_file(cli_args)

    # validate input, parameters, and support some special options
    if args.version:
        print("Running 'fades' version", fades.__version__)
        print("    Python:", sys.version_info)
        print("    System:", platform.platform())
        return 0

    # The --exec and --module flags needs child_program to exist (this is not handled at
    # argparse level because it's easier to collect the executable as the
    # normal child_program, so everything after that are parameteres
    # considered for the executable itself, not for fades).
    if args.executable and not args.child_program:
        parser.print_usage()
        print(
            "fades: error: argument -x/--exec needs child_program to be present"
        )
        return -1
    if args.module and not args.child_program:
        parser.print_usage()
        print(
            "fades: error: argument -m/--module needs child_program (module) to be present"
        )
        return -1

    # set up the logger and dump basic version info
    logger_set_up(args.verbose, args.quiet)
    logger.debug("Running Python %s on %r", sys.version_info,
                 platform.platform())
    logger.debug("Starting fades v. %s", fades.__version__)
    logger.debug("Arguments: %s", args)

    # verify that the module is NOT being used from a virtualenv
    _real_prefix = getattr(sys, 'real_prefix', None)
    _base_prefix = getattr(sys, 'base_prefix', None)
    if detect_inside_virtualenv(sys.prefix, _real_prefix, _base_prefix):
        logger.error(
            "fades is running from inside a virtualenv (%r), which is not supported",
            sys.prefix)
        raise FadesError("Cannot run from a virtualenv")

    if args.verbose and args.quiet:
        logger.warning("Overriding 'quiet' option ('verbose' also requested)")

    # start the virtualenvs manager
    venvscache = cache.VEnvsCache(
        os.path.join(helpers.get_basedir(), 'venvs.idx'))
    # start usage manager
    usage_manager = envbuilder.UsageManager(
        os.path.join(helpers.get_basedir(), 'usage_stats'), venvscache)

    if args.clean_unused_venvs:
        try:
            max_days_to_keep = int(args.clean_unused_venvs)
        except ValueError:
            logger.error("clean_unused_venvs must be an integer.")
            raise FadesError('clean_unused_venvs not an integer')

        usage_manager.clean_unused_venvs(max_days_to_keep)
        return 0

    uuid = args.remove
    if uuid:
        venv_data = venvscache.get_venv(uuid=uuid)
        if venv_data:
            # remove this venv from the cache
            env_path = venv_data.get('env_path')
            if env_path:
                envbuilder.destroy_venv(env_path, venvscache)
            else:
                logger.warning(
                    "Invalid 'env_path' found in virtualenv metadata: %r. "
                    "Not removing virtualenv.", env_path)
        else:
            logger.warning('No virtualenv found with uuid: %s.', uuid)
        return 0

    # decided which the child program really is
    analyzable_child_program, child_program = decide_child_program(
        args.executable, args.module, args.child_program)

    # Group and merge dependencies
    indicated_deps = consolidate_dependencies(args.ipython,
                                              analyzable_child_program,
                                              args.requirement,
                                              args.dependency)

    # Check for packages updates
    if args.check_updates:
        helpers.check_pypi_updates(indicated_deps)

    # get the interpreter version requested for the child_program
    interpreter, is_current = helpers.get_interpreter_version(args.python)

    # options
    pip_options = args.pip_options  # pip_options mustn't store.
    python_options = args.python_options
    options = {}
    options['pyvenv_options'] = []
    options['virtualenv_options'] = args.virtualenv_options
    if args.system_site_packages:
        options['virtualenv_options'].append("--system-site-packages")
        options['pyvenv_options'] = ["--system-site-packages"]

    create_venv = False
    venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options)
    if venv_data:
        env_path = venv_data['env_path']
        # A venv was found in the cache check if its valid or re-generate it.
        if not os.path.exists(env_path):
            logger.warning(
                "Missing directory (the virtualenv will be re-created): %r",
                env_path)
            venvscache.remove(env_path)
            create_venv = True
    else:
        create_venv = True

    if create_venv:
        # Check if the requested packages exists in pypi.
        if not args.no_precheck_availability and indicated_deps.get('pypi'):
            logger.info(
                "Checking the availabilty of dependencies in PyPI. "
                "You can use '--no-precheck-availability' to avoid it.")
            if not helpers.check_pypi_exists(indicated_deps):
                logger.error("An indicated dependency doesn't exist. Exiting")
                raise FadesError("Required dependency does not exist")

        # Create a new venv
        venv_data, installed = envbuilder.create_venv(indicated_deps,
                                                      args.python, is_current,
                                                      options, pip_options,
                                                      args.avoid_pip_upgrade)
        # store this new venv in the cache
        venvscache.store(installed, venv_data, interpreter, options)

    if args.get_venv_dir:
        # all it was requested is the virtualenv's path, show it and quit (don't run anything)
        print(venv_data['env_path'])
        return 0

    if args.freeze:
        # beyond all the rest of work, dump the dependencies versions to a file
        mgr = pipmanager.PipManager(venv_data['env_bin_path'])
        mgr.freeze(args.freeze)

    # run forest run!!
    python_exe = 'ipython' if args.ipython else 'python'
    python_exe = os.path.join(venv_data['env_bin_path'], python_exe)

    # add the virtualenv /bin path to the child PATH.
    environ_path = venv_data['env_bin_path']
    if 'PATH' in os.environ:
        environ_path += os.pathsep + os.environ['PATH']
    os.environ['PATH'] = environ_path

    # store usage information
    usage_manager.store_usage_stat(venv_data, venvscache)

    if child_program is None:
        interactive = True
        cmd = [python_exe] + python_options

        # get possible extra python options and environement for auto import
        if indicated_deps and args.autoimport:
            temp_scriptpath = get_autoimport_scriptname(
                indicated_deps, args.ipython)
            cmd += ['-i', temp_scriptpath]

        logger.debug("Calling the interactive Python interpreter: %s", cmd)
        proc = subprocess.Popen(cmd)
    else:
        interactive = False
        if args.executable:
            # Build the exec path relative to 'bin' dir; note that if child_program's path
            # is absolute (starting with '/') the resulting exec_path will be just it,
            # which is something fades supports
            exec_path = os.path.join(venv_data['env_bin_path'], child_program)
            cmd = [exec_path]
        elif args.module:
            cmd = [python_exe, '-m'] + python_options + [child_program]
        else:
            cmd = [python_exe] + python_options + [child_program]

        # Incorporate the child options, always at the end, log and run.
        cmd += args.child_options
        logger.debug("Calling %s", cmd)

        try:
            proc = subprocess.Popen(cmd)
        except FileNotFoundError:
            logger.error("Command not found: %s", child_program)
            raise FadesError("Command not found")

    def _signal_handler(signum, _):
        """Handle signals received by parent process, send them to child.

        The only exception is CTRL-C, that is generated *from* the interactive
        interpreter (it's a keyboard combination!), so we swallow it for the
        interpreter to not see it twice.
        """
        if interactive and signum == signal.SIGINT:
            logger.debug("Swallowing signal %s", signum)
        else:
            logger.debug("Redirecting signal %s to child", signum)
            os.kill(proc.pid, signum)

    # redirect the useful signals
    for s in REDIRECTED_SIGNALS:
        signal.signal(s, _signal_handler)

    # wait child to finish, end
    rc = proc.wait()
    if rc:
        logger.debug("Child process not finished correctly: returncode=%d", rc)
    return rc
Example #15
0
def go(version, argv):
    """Make the magic happen."""
    parser = argparse.ArgumentParser(
        prog='PROG',
        epilog=help_epilog,
        usage=help_usage,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument(
        '-V',
        '--version',
        action='store_true',
        help="show version and info about the system, and exit")
    parser.add_argument(
        '-v',
        '--verbose',
        action='store_true',
        help="send all internal debugging lines to stderr, which may be very "
        "useful to debug any problem that may arise.")
    parser.add_argument(
        '-q',
        '--quiet',
        action='store_true',
        help="don't show anything (unless it has a real problem), so the "
        "original script stderr is not polluted at all.")
    parser.add_argument(
        '-d',
        '--dependency',
        action='append',
        help="specify dependencies through command line (this option can be "
        "used multiple times)")
    parser.add_argument('-r',
                        '--requirement',
                        help="indicate from which file read the dependencies")
    parser.add_argument('-p',
                        '--python',
                        action='store',
                        help=("Specify the Python interpreter to use.\n"
                              " Default is: %s") % (sys.executable, ))
    parser.add_argument('child_program', nargs='?', default=None)
    parser.add_argument('child_options', nargs=argparse.REMAINDER)

    # support the case when executed from a shell-bang, where all the
    # parameters come in sys.argv[1] in a single string separated
    # by spaces (in this case, the third parameter is what is being
    # executed)
    if len(sys.argv) > 1 and " " in sys.argv[1]:
        real_args = sys.argv[1].split() + [sys.argv[2]]
        args = parser.parse_args(real_args)
    else:
        args = parser.parse_args()

    # validate input, parameters, and support some special options
    if args.version:
        print("Running 'fades' version", version)
        print("    Python:", sys.version_info)
        print("    System:", sys.platform)
        sys.exit()

    if args.verbose:
        log_level = logging.DEBUG
    elif args.quiet:
        log_level = logging.WARNING
    else:
        log_level = logging.INFO

    # set up logger and dump basic version info
    l = logger.set_up(log_level)
    l.debug("Running Python %s on %r", sys.version_info, sys.platform)
    l.debug("Starting fades v. %s", version)
    l.debug("Arguments: %s", args)

    if args.verbose and args.quiet:
        l.warning("Overriding 'quiet' option ('verbose' also requested)")

    # parse file and get deps
    indicated_deps = parsing.parse_srcfile(args.child_program)
    l.debug("Dependencies from source file: %s", indicated_deps)
    reqfile_deps = parsing.parse_reqfile(args.requirement)
    l.debug("Dependencies from requirements file: %s", reqfile_deps)
    manual_deps = parsing.parse_manual(args.dependency)
    l.debug("Dependencies from parameters: %s", manual_deps)
    indicated_deps = _merge_deps(indicated_deps, reqfile_deps, manual_deps)

    # get the interpreter version requested for the child_program
    interpreter, is_current = helpers.get_interpreter_version(args.python)

    # start the virtualenvs manager
    venvscache = cache.VEnvsCache(
        os.path.join(helpers.get_basedir(), 'venvs.idx'))
    venv_data = venvscache.get_venv(indicated_deps, interpreter)
    if venv_data is None:
        venv_data, installed = envbuilder.create_venv(indicated_deps,
                                                      interpreter, is_current)
        # store this new venv in the cache
        venvscache.store(installed, venv_data, interpreter)

    # run forest run!!
    python_exe = os.path.join(venv_data['env_bin_path'], 'python')
    if args.child_program is None:
        l.debug("Calling the interactive Python interpreter")
        p = subprocess.Popen([python_exe])

    else:
        l.debug("Calling the child Python program %r with options %s",
                args.child_program, args.child_options)
        p = subprocess.Popen([python_exe, args.child_program] +
                             args.child_options)

        def _signal_handler(signum, _):
            """Handle signals received by parent process, send them to child."""
            l.debug("Redirecting signal %s to child", signum)
            os.kill(p.pid, signum)

        # redirect these signals
        for s in REDIRECTED_SIGNALS:
            signal.signal(s, _signal_handler)

    # wait child to finish, end
    rc = p.wait()
    if rc:
        l.debug("Child process not finished correctly: returncode=%d", rc)
Example #16
0
def go(version, argv):
    """Make the magic happen."""
    parser = argparse.ArgumentParser(prog='PROG', epilog=help_epilog,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('-V', '--version', action='store_true',
                        help="show version and info about the system, and exit")
    parser.add_argument('-v', '--verbose', action='store_true',
                        help="send all internal debugging lines to stderr, which may be very "
                             "useful to debug any problem that may arise.")
    parser.add_argument('-q', '--quiet', action='store_true',
                        help="don't show anything (unless it has a real problem), so the "
                             "original script stderr is not polluted at all.")
    parser.add_argument('-d', '--dependency', action='append',
                        help="specify dependencies through command line (this option can be "
                             "used multiple times)")
    parser.add_argument('-r', '--requirement',
                        help="indicate from which file read the dependencies")
    parser.add_argument('child_program', nargs='?', default=None)
    parser.add_argument('child_options', nargs=argparse.REMAINDER)
    args = parser.parse_args()

    # validate input, parameters, and support some special options
    if args.version:
        print("Running 'fades' version", version)
        print("    Python:", sys.version_info)
        print("    System:", sys.platform)
        sys.exit()

    if args.verbose:
        log_level = logging.DEBUG
    elif args.quiet:
        log_level = logging.WARNING
    else:
        log_level = logging.INFO

    # set up logger and dump basic version info
    l = logger.set_up(log_level)
    l.debug("Running Python %s on %r", sys.version_info, sys.platform)
    l.debug("Starting fades v. %s", version)
    l.debug("Arguments: %s", args)

    if args.verbose and args.quiet:
        l.warning("Overriding 'quiet' option ('verbose' also requested)")

    # parse file and get deps
    indicated_deps = parsing.parse_srcfile(args.child_program)
    l.debug("Dependencies from source file: %s", indicated_deps)
    reqfile_deps = parsing.parse_reqfile(args.requirement)
    l.debug("Dependencies from requirements file: %s", reqfile_deps)
    manual_deps = parsing.parse_manual(args.dependency)
    l.debug("Dependencies from parameters: %s", manual_deps)
    indicated_deps = _merge_deps(indicated_deps, reqfile_deps, manual_deps)

    # start the virtualenvs manager
    venvscache = cache.VEnvsCache(os.path.join(helpers.get_basedir(), 'venvs.idx'))
    venv_data = venvscache.get_venv(indicated_deps)
    if venv_data is None:
        venv_data, installed = envbuilder.create_venv(indicated_deps)
        # store this new venv in the cache
        venvscache.store(installed, venv_data)

    # run forest run!!
    python_exe = os.path.join(venv_data['env_bin_path'], "python3")
    if args.child_program is None:
        l.debug("Calling the interactive Python interpreter")
        p = subprocess.Popen([python_exe])

    else:
        l.debug("Calling the child Python program %r with options %s",
                args.child_program, args.child_options)
        p = subprocess.Popen([python_exe, args.child_program] + args.child_options)

        def _signal_handler(signum, _):
            """Handle signals received by parent process, send them to child."""
            l.debug("Redirecting signal %s to child", signum)
            os.kill(p.pid, signum)

        # redirect these signals
        for s in REDIRECTED_SIGNALS:
            signal.signal(s, _signal_handler)

    # wait child to finish, end
    rc = p.wait()
    if rc:
        l.debug("Child process not finished correctly: returncode=%d", rc)
Example #17
0
def go(argv):
    """Make the magic happen."""
    parser = argparse.ArgumentParser(prog='PROG', epilog=help_epilog, usage=help_usage,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('-V', '--version', action='store_true',
                        help="show version and info about the system, and exit")
    parser.add_argument('-v', '--verbose', action='store_true',
                        help="send all internal debugging lines to stderr, which may be very "
                             "useful to debug any problem that may arise.")
    parser.add_argument('-q', '--quiet', action='store_true',
                        help="don't show anything (unless it has a real problem), so the "
                             "original script stderr is not polluted at all.")
    parser.add_argument('-d', '--dependency', action='append',
                        help="specify dependencies through command line (this option can be "
                             "used multiple times)")
    parser.add_argument('-r', '--requirement',
                        help="indicate from which file read the dependencies")
    parser.add_argument('-p', '--python', action='store',
                        help=("Specify the Python interpreter to use.\n"
                              " Default is: %s") % (sys.executable,))
    parser.add_argument('-x', '--exec', dest='executable', action='store_true',
                        help=("Indicate that the child_program should be looked up in the "
                              "virtualenv."))
    parser.add_argument('-i', '--ipython', action='store_true', help="use IPython shell.")
    parser.add_argument('--system-site-packages', action='store_true', default=False,
                        help=("Give the virtual environment access to the "
                              "system site-packages dir."))
    parser.add_argument('--virtualenv-options', action='append', default=[],
                        help=("Extra options to be supplied to virtualenv. (this option can be "
                              "used multiple times)"))
    parser.add_argument('--check-updates', action='store_true',
                        help=("check for packages updates"))
    parser.add_argument('--pip-options', action='append', default=[],
                        help=("Extra options to be supplied to pip. (this option can be "
                              "used multiple times)"))
    parser.add_argument('--rm', dest='remove', metavar='UUID',
                        help=("Remove a virtualenv by UUID."))
    parser.add_argument('--clean-unused-venvs', action='store',
                        help=("This option remove venvs that haven't been used for more than "
                              "CLEAN_UNUSED_VENVS days. Appart from that, will compact usage "
                              "stats file.\n"
                              "When this option is present, the cleaning takes place at the "
                              "beginning of the execution."))
    parser.add_argument('child_program', nargs='?', default=None)
    parser.add_argument('child_options', nargs=argparse.REMAINDER)

    # support the case when executed from a shell-bang, where all the
    # parameters come in sys.argv[1] in a single string separated
    # by spaces (in this case, the third parameter is what is being
    # executed)
    if len(sys.argv) > 1 and " " in sys.argv[1]:
        real_args = sys.argv[1].split() + sys.argv[2:]
        cli_args = parser.parse_args(real_args)
    else:
        cli_args = parser.parse_args()

    # update args from config file (if needed).
    args = file_options.options_from_file(cli_args)

    # validate input, parameters, and support some special options
    if args.version:
        print("Running 'fades' version", fades.__version__)
        print("    Python:", sys.version_info)
        print("    System:", sys.platform)
        sys.exit()

    # set up logger and dump basic version info
    l = logger.set_up(args.verbose, args.quiet)
    l.debug("Running Python %s on %r", sys.version_info, sys.platform)
    l.debug("Starting fades v. %s", fades.__version__)
    l.debug("Arguments: %s", args)

    if args.verbose and args.quiet:
        l.warning("Overriding 'quiet' option ('verbose' also requested)")

    # start the virtualenvs manager
    venvscache = cache.VEnvsCache(os.path.join(helpers.get_basedir(), 'venvs.idx'))
    # start usage manager
    usage_manager = envbuilder.UsageManager(os.path.join(helpers.get_basedir(), 'usage_stats'),
                                            venvscache)

    if args.clean_unused_venvs:
        try:
            max_days_to_keep = int(args.clean_unused_venvs)
            usage_manager.clean_unused_venvs(max_days_to_keep)
        except:
            l.debug("CLEAN_UNUSED_VENVS must be an integer.")
            raise
        finally:
            sys.exit()

    uuid = args.remove
    if uuid:
        venv_data = venvscache.get_venv(uuid=uuid)
        if venv_data:
            # remove this venv from the cache
            env_path = venv_data.get('env_path')
            if env_path:
                envbuilder.destroy_venv(env_path, venvscache)
            else:
                l.warning("Invalid 'env_path' found in virtualenv metadata: %r. "
                          "Not removing virtualenv.", env_path)
        else:
            l.warning('No virtualenv found with uuid: %s.', uuid)
        return

    # parse file and get deps
    if args.ipython:
        l.debug("Adding ipython dependency because --ipython was detected")
        ipython_dep = parsing.parse_manual(['ipython'])
    else:
        ipython_dep = {}

    if args.executable:
        indicated_deps = {}
        docstring_deps = {}
    else:
        indicated_deps = parsing.parse_srcfile(args.child_program)
        l.debug("Dependencies from source file: %s", indicated_deps)
        docstring_deps = parsing.parse_docstring(args.child_program)
        l.debug("Dependencies from docstrings: %s", docstring_deps)
    reqfile_deps = parsing.parse_reqfile(args.requirement)
    l.debug("Dependencies from requirements file: %s", reqfile_deps)
    manual_deps = parsing.parse_manual(args.dependency)
    l.debug("Dependencies from parameters: %s", manual_deps)
    indicated_deps = _merge_deps(ipython_dep, indicated_deps, docstring_deps,
                                 reqfile_deps, manual_deps)

    # Check for packages updates
    if args.check_updates:
        helpers.check_pypi_updates(indicated_deps)

    # get the interpreter version requested for the child_program
    interpreter, is_current = helpers.get_interpreter_version(args.python)

    # options
    pip_options = args.pip_options  # pip_options mustn't store.
    options = {}
    options['pyvenv_options'] = []
    options['virtualenv_options'] = args.virtualenv_options
    if args.system_site_packages:
        options['virtualenv_options'].append("--system-site-packages")
        options['pyvenv_options'] = ["--system-site-packages"]

    create_venv = False
    venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options)
    if venv_data:
        env_path = venv_data['env_path']
        # A venv was found in the cache check if its valid or re-generate it.
        if not os.path.exists(env_path):
            l.warning("Missing directory (the virtualenv will be re-created): %r", env_path)
            venvscache.remove(env_path)
            create_venv = True
    else:
        create_venv = True

    if create_venv:
        # Create a new venv
        venv_data, installed = envbuilder.create_venv(indicated_deps, args.python, is_current,
                                                      options, pip_options)
        # store this new venv in the cache
        venvscache.store(installed, venv_data, interpreter, options)

    # run forest run!!
    python_exe = 'ipython' if args.ipython else 'python'
    python_exe = os.path.join(venv_data['env_bin_path'], python_exe)

    # store usage information
    usage_manager.store_usage_stat(venv_data, venvscache)

    if args.child_program is None:
        l.debug("Calling the interactive Python interpreter")
        p = subprocess.Popen([python_exe])

    else:
        if args.executable:
            cmd = [os.path.join(venv_data['env_bin_path'], args.child_program)]
        else:
            cmd = [python_exe, args.child_program]
        l.debug("Calling the child program %r with options %s",
                args.child_program, args.child_options)
        p = subprocess.Popen(cmd + args.child_options)

        def _signal_handler(signum, _):
            """Handle signals received by parent process, send them to child."""
            l.debug("Redirecting signal %s to child", signum)
            os.kill(p.pid, signum)

        # redirect these signals
        for s in REDIRECTED_SIGNALS:
            signal.signal(s, _signal_handler)

    # wait child to finish, end
    rc = p.wait()
    if rc:
        l.debug("Child process not finished correctly: returncode=%d", rc)
    sys.exit(rc)
Example #18
0
def go(version, argv):
    """Make the magic happen."""
    parser = argparse.ArgumentParser(
        prog="PROG", epilog=help_epilog, usage=help_usage, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument("-V", "--version", action="store_true", help="show version and info about the system, and exit")
    parser.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        help="send all internal debugging lines to stderr, which may be very "
        "useful to debug any problem that may arise.",
    )
    parser.add_argument(
        "-q",
        "--quiet",
        action="store_true",
        help="don't show anything (unless it has a real problem), so the "
        "original script stderr is not polluted at all.",
    )
    parser.add_argument(
        "-d",
        "--dependency",
        action="append",
        help="specify dependencies through command line (this option can be " "used multiple times)",
    )
    parser.add_argument("-r", "--requirement", help="indicate from which file read the dependencies")
    parser.add_argument(
        "-p",
        "--python",
        action="store",
        help=("Specify the Python interpreter to use.\n" " Default is: %s") % (sys.executable,),
    )
    parser.add_argument(
        "-x",
        "--exec",
        dest="executable",
        action="store_true",
        help=("Indicate that the child_program should be looked up in the " "virtualenv."),
    )
    parser.add_argument("-i", "--ipython", action="store_true", help="use IPython shell.")
    parser.add_argument("child_program", nargs="?", default=None)
    parser.add_argument("child_options", nargs=argparse.REMAINDER)

    # support the case when executed from a shell-bang, where all the
    # parameters come in sys.argv[1] in a single string separated
    # by spaces (in this case, the third parameter is what is being
    # executed)
    if len(sys.argv) > 1 and " " in sys.argv[1]:
        real_args = sys.argv[1].split() + [sys.argv[2]]
        args = parser.parse_args(real_args)
    else:
        args = parser.parse_args()

    # validate input, parameters, and support some special options
    if args.version:
        print("Running 'fades' version", version)
        print("    Python:", sys.version_info)
        print("    System:", sys.platform)
        sys.exit()

    if args.verbose:
        log_level = logging.DEBUG
    elif args.quiet:
        log_level = logging.WARNING
    else:
        log_level = logging.INFO

    # set up logger and dump basic version info
    l = logger.set_up(log_level)
    l.debug("Running Python %s on %r", sys.version_info, sys.platform)
    l.debug("Starting fades v. %s", version)
    l.debug("Arguments: %s", args)

    if args.verbose and args.quiet:
        l.warning("Overriding 'quiet' option ('verbose' also requested)")

    # parse file and get deps
    if args.ipython:
        l.debug("Adding ipython dependency because --ipython was detected")
        ipython_dep = parsing.parse_manual(["ipython"])
    else:
        ipython_dep = {}
    if args.executable:
        indicated_deps = {}
    else:
        indicated_deps = parsing.parse_srcfile(args.child_program)
        l.debug("Dependencies from source file: %s", indicated_deps)
    reqfile_deps = parsing.parse_reqfile(args.requirement)
    l.debug("Dependencies from requirements file: %s", reqfile_deps)
    manual_deps = parsing.parse_manual(args.dependency)
    l.debug("Dependencies from parameters: %s", manual_deps)
    indicated_deps = _merge_deps(ipython_dep, indicated_deps, reqfile_deps, manual_deps)

    # get the interpreter version requested for the child_program
    interpreter, is_current = helpers.get_interpreter_version(args.python)

    # start the virtualenvs manager
    venvscache = cache.VEnvsCache(os.path.join(helpers.get_basedir(), "venvs.idx"))
    venv_data = venvscache.get_venv(indicated_deps, interpreter)
    if venv_data is None:
        venv_data, installed = envbuilder.create_venv(indicated_deps, interpreter, is_current)
        # store this new venv in the cache
        venvscache.store(installed, venv_data, interpreter)

    # run forest run!!
    python_exe = "ipython" if args.ipython else "python"
    python_exe = os.path.join(venv_data["env_bin_path"], python_exe)
    if args.child_program is None:
        l.debug("Calling the interactive Python interpreter")
        p = subprocess.Popen([python_exe])

    else:
        if args.executable:
            cmd = [os.path.join(venv_data["env_bin_path"], args.child_program)]
        else:
            cmd = [python_exe, args.child_program]
        l.debug("Calling the child program %r with options %s", args.child_program, args.child_options)
        p = subprocess.Popen(cmd + args.child_options)

        def _signal_handler(signum, _):
            """Handle signals received by parent process, send them to child."""
            l.debug("Redirecting signal %s to child", signum)
            os.kill(p.pid, signum)

        # redirect these signals
        for s in REDIRECTED_SIGNALS:
            signal.signal(s, _signal_handler)

    # wait child to finish, end
    rc = p.wait()
    if rc:
        l.debug("Child process not finished correctly: returncode=%d", rc)
Example #19
0
def go(argv):
    """Make the magic happen."""
    parser = argparse.ArgumentParser(
        prog='PROG',
        epilog=help_epilog,
        usage=help_usage,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument(
        '-V',
        '--version',
        action='store_true',
        help="show version and info about the system, and exit")
    parser.add_argument(
        '-v',
        '--verbose',
        action='store_true',
        help="send all internal debugging lines to stderr, which may be very "
        "useful to debug any problem that may arise.")
    parser.add_argument(
        '-q',
        '--quiet',
        action='store_true',
        help="don't show anything (unless it has a real problem), so the "
        "original script stderr is not polluted at all.")
    parser.add_argument(
        '-d',
        '--dependency',
        action='append',
        help="specify dependencies through command line (this option can be "
        "used multiple times)")
    parser.add_argument('-r',
                        '--requirement',
                        help="indicate from which file read the dependencies")
    parser.add_argument('-p',
                        '--python',
                        action='store',
                        help=("Specify the Python interpreter to use.\n"
                              " Default is: %s") % (sys.executable, ))
    parser.add_argument(
        '-x',
        '--exec',
        dest='executable',
        action='store_true',
        help=("Indicate that the child_program should be looked up in the "
              "virtualenv."))
    parser.add_argument('-i',
                        '--ipython',
                        action='store_true',
                        help="use IPython shell.")
    parser.add_argument('--system-site-packages',
                        action='store_true',
                        default=False,
                        help=("Give the virtual environment access to the "
                              "system site-packages dir."))
    parser.add_argument(
        '--virtualenv-options',
        action='append',
        default=[],
        help=(
            "Extra options to be supplied to virtualenv. (this option can be "
            "used multiple times)"))
    parser.add_argument('--check-updates',
                        action='store_true',
                        help=("check for packages updates"))
    parser.add_argument(
        '--pip-options',
        action='append',
        default=[],
        help=("Extra options to be supplied to pip. (this option can be "
              "used multiple times)"))
    parser.add_argument('--rm',
                        dest='remove',
                        metavar='UUID',
                        help=("Remove a virtualenv by UUID."))
    parser.add_argument(
        '--clean-unused-venvs',
        action='store',
        help=("This option remove venvs that haven't been used for more than "
              "CLEAN_UNUSED_VENVS days. Appart from that, will compact usage "
              "stats file.\n"
              "When this option is present, the cleaning takes place at the "
              "beginning of the execution."))
    parser.add_argument('child_program', nargs='?', default=None)
    parser.add_argument('child_options', nargs=argparse.REMAINDER)

    # support the case when executed from a shell-bang, where all the
    # parameters come in sys.argv[1] in a single string separated
    # by spaces (in this case, the third parameter is what is being
    # executed)
    if len(sys.argv) > 1 and " " in sys.argv[1]:
        real_args = sys.argv[1].split() + sys.argv[2:]
        cli_args = parser.parse_args(real_args)
    else:
        cli_args = parser.parse_args()

    # update args from config file (if needed).
    args = file_options.options_from_file(cli_args)

    # validate input, parameters, and support some special options
    if args.version:
        print("Running 'fades' version", fades.__version__)
        print("    Python:", sys.version_info)
        print("    System:", sys.platform)
        sys.exit(0)

    # set up logger and dump basic version info
    l = logger.set_up(args.verbose, args.quiet)
    l.debug("Running Python %s on %r", sys.version_info, sys.platform)
    l.debug("Starting fades v. %s", fades.__version__)
    l.debug("Arguments: %s", args)

    # verify that the module is NOT being used from a virtualenv
    if detect_inside_virtualenv(sys.prefix, getattr(sys, 'real_prefix', None),
                                getattr(sys, 'base_prefix', None)):
        l.warning(
            "fades is running from a virtualenv (%r), which is not supported",
            sys.prefix)

    if args.verbose and args.quiet:
        l.warning("Overriding 'quiet' option ('verbose' also requested)")

    # start the virtualenvs manager
    venvscache = cache.VEnvsCache(
        os.path.join(helpers.get_basedir(), 'venvs.idx'))
    # start usage manager
    usage_manager = envbuilder.UsageManager(
        os.path.join(helpers.get_basedir(), 'usage_stats'), venvscache)

    rc = 0
    if args.clean_unused_venvs:
        try:
            max_days_to_keep = int(args.clean_unused_venvs)
            usage_manager.clean_unused_venvs(max_days_to_keep)
        except:
            rc = 1
            l.debug("CLEAN_UNUSED_VENVS must be an integer.")
            raise
        finally:
            sys.exit(rc)

    uuid = args.remove
    if uuid:
        venv_data = venvscache.get_venv(uuid=uuid)
        if venv_data:
            # remove this venv from the cache
            env_path = venv_data.get('env_path')
            if env_path:
                envbuilder.destroy_venv(env_path, venvscache)
            else:
                l.warning(
                    "Invalid 'env_path' found in virtualenv metadata: %r. "
                    "Not removing virtualenv.", env_path)
        else:
            l.warning('No virtualenv found with uuid: %s.', uuid)
        return

    # parse file and get deps
    if args.ipython:
        l.debug("Adding ipython dependency because --ipython was detected")
        ipython_dep = parsing.parse_manual(['ipython'])
    else:
        ipython_dep = {}

    if args.executable:
        indicated_deps = {}
        docstring_deps = {}
    else:
        indicated_deps = parsing.parse_srcfile(args.child_program)
        l.debug("Dependencies from source file: %s", indicated_deps)
        docstring_deps = parsing.parse_docstring(args.child_program)
        l.debug("Dependencies from docstrings: %s", docstring_deps)
    reqfile_deps = parsing.parse_reqfile(args.requirement)
    l.debug("Dependencies from requirements file: %s", reqfile_deps)
    manual_deps = parsing.parse_manual(args.dependency)
    l.debug("Dependencies from parameters: %s", manual_deps)
    indicated_deps = _merge_deps(ipython_dep, indicated_deps, docstring_deps,
                                 reqfile_deps, manual_deps)

    # Check for packages updates
    if args.check_updates:
        helpers.check_pypi_updates(indicated_deps)

    # get the interpreter version requested for the child_program
    interpreter, is_current = helpers.get_interpreter_version(args.python)

    # options
    pip_options = args.pip_options  # pip_options mustn't store.
    options = {}
    options['pyvenv_options'] = []
    options['virtualenv_options'] = args.virtualenv_options
    if args.system_site_packages:
        options['virtualenv_options'].append("--system-site-packages")
        options['pyvenv_options'] = ["--system-site-packages"]

    create_venv = False
    venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options)
    if venv_data:
        env_path = venv_data['env_path']
        # A venv was found in the cache check if its valid or re-generate it.
        if not os.path.exists(env_path):
            l.warning(
                "Missing directory (the virtualenv will be re-created): %r",
                env_path)
            venvscache.remove(env_path)
            create_venv = True
    else:
        create_venv = True

    if create_venv:
        # Create a new venv
        venv_data, installed = envbuilder.create_venv(indicated_deps,
                                                      args.python, is_current,
                                                      options, pip_options)
        # store this new venv in the cache
        venvscache.store(installed, venv_data, interpreter, options)

    # run forest run!!
    python_exe = 'ipython' if args.ipython else 'python'
    python_exe = os.path.join(venv_data['env_bin_path'], python_exe)

    # store usage information
    usage_manager.store_usage_stat(venv_data, venvscache)

    if args.child_program is None:
        interactive = True
        l.debug("Calling the interactive Python interpreter")
        p = subprocess.Popen([python_exe])

    else:
        interactive = False
        if args.executable:
            cmd = [os.path.join(venv_data['env_bin_path'], args.child_program)]
        else:
            cmd = [python_exe, args.child_program]
        l.debug("Calling the child program %r with options %s",
                args.child_program, args.child_options)
        p = subprocess.Popen(cmd + args.child_options)

    def _signal_handler(signum, _):
        """Handle signals received by parent process, send them to child.

        The only exception is CTRL-C, that is generated *from* the interactive
        interpreter (it's a keyboard combination!), so we swallow it for the
        interpreter to not see it twice.
        """
        if interactive and signum == signal.SIGINT:
            l.debug("Swallowing signal %s", signum)
        else:
            l.debug("Redirecting signal %s to child", signum)
            os.kill(p.pid, signum)

    # redirect the useful signals
    for s in REDIRECTED_SIGNALS:
        signal.signal(s, _signal_handler)

    # wait child to finish, end
    rc = p.wait()
    if rc:
        l.debug("Child process not finished correctly: returncode=%d", rc)
    sys.exit(rc)
Example #20
0
File: main.py Project: PyAr/fades
def go():
    """Make the magic happen."""
    parser = argparse.ArgumentParser(prog='PROG', epilog=help_epilog,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('-V', '--version', action='store_true',
                        help="show version and info about the system, and exit")
    parser.add_argument('-v', '--verbose', action='store_true',
                        help="send all internal debugging lines to stderr, which may be very "
                             "useful to debug any problem that may arise.")
    parser.add_argument('-q', '--quiet', action='store_true',
                        help="don't show anything (unless it has a real problem), so the "
                             "original script stderr is not polluted at all.")
    parser.add_argument('-d', '--dependency', action='append',
                        help="specify dependencies through command line (this option can be "
                             "used multiple times)")
    parser.add_argument('-r', '--requirement', action='append',
                        help="indicate files to read dependencies from (this option can be "
                             "used multiple times)")
    parser.add_argument('-p', '--python', action='store',
                        help=("Specify the Python interpreter to use.\n"
                              " Default is: %s") % (sys.executable,))
    parser.add_argument('-x', '--exec', dest='executable', action='store_true',
                        help=("Indicate that the child_program should be looked up in the "
                              "virtualenv."))
    parser.add_argument('-i', '--ipython', action='store_true', help="use IPython shell.")
    parser.add_argument('--system-site-packages', action='store_true', default=False,
                        help=("Give the virtual environment access to the "
                              "system site-packages dir."))
    parser.add_argument('--virtualenv-options', action='append', default=[],
                        help=("Extra options to be supplied to virtualenv. (this option can be "
                              "used multiple times)"))
    parser.add_argument('--check-updates', action='store_true',
                        help=("check for packages updates"))
    parser.add_argument('--no-precheck-availability', action='store_true',
                        help=("Don't check if the packages exists in PyPI before actually try "
                              "to install them."))
    parser.add_argument('--pip-options', action='append', default=[],
                        help=("Extra options to be supplied to pip. (this option can be "
                              "used multiple times)"))
    parser.add_argument('--python-options', action='append', default=[],
                        help=("Extra options to be supplied to python. (this option can be "
                              "used multiple times)"))
    parser.add_argument('--rm', dest='remove', metavar='UUID',
                        help=("Remove a virtualenv by UUID. See --get-venv-dir option to "
                              "easily find out the UUID."))
    parser.add_argument('--clean-unused-venvs', action='store',
                        help=("This option remove venvs that haven't been used for more than "
                              "CLEAN_UNUSED_VENVS days. Appart from that, will compact usage "
                              "stats file.\n"
                              "When this option is present, the cleaning takes place at the "
                              "beginning of the execution."))
    parser.add_argument('--get-venv-dir', action='store_true',
                        help=("Show the virtualenv base directory (which includes the "
                              "virtualenv UUID) and quit."))
    parser.add_argument('child_program', nargs='?', default=None)
    parser.add_argument('child_options', nargs=argparse.REMAINDER)

    cli_args = _get_normalized_args(parser)

    # update args from config file (if needed).
    args = file_options.options_from_file(cli_args)

    # validate input, parameters, and support some special options
    if args.version:
        print("Running 'fades' version", fades.__version__)
        print("    Python:", sys.version_info)
        print("    System:", platform.platform())
        return 0

    # set up logger and dump basic version info
    logger = fades_logger.set_up(args.verbose, args.quiet)
    logger.debug("Running Python %s on %r", sys.version_info, platform.platform())
    logger.debug("Starting fades v. %s", fades.__version__)
    logger.debug("Arguments: %s", args)

    # verify that the module is NOT being used from a virtualenv
    if detect_inside_virtualenv(sys.prefix, getattr(sys, 'real_prefix', None),
                                getattr(sys, 'base_prefix', None)):
        logger.error(
            "fades is running from inside a virtualenv (%r), which is not supported", sys.prefix)
        raise FadesError("Cannot run from a virtualenv")

    if args.verbose and args.quiet:
        logger.warning("Overriding 'quiet' option ('verbose' also requested)")

    # start the virtualenvs manager
    venvscache = cache.VEnvsCache(os.path.join(helpers.get_basedir(), 'venvs.idx'))
    # start usage manager
    usage_manager = envbuilder.UsageManager(os.path.join(helpers.get_basedir(), 'usage_stats'),
                                            venvscache)

    if args.clean_unused_venvs:
        try:
            max_days_to_keep = int(args.clean_unused_venvs)
        except ValueError:
            logger.error("clean_unused_venvs must be an integer.")
            raise FadesError('clean_unused_venvs not an integer')

        usage_manager.clean_unused_venvs(max_days_to_keep)
        return 0

    uuid = args.remove
    if uuid:
        venv_data = venvscache.get_venv(uuid=uuid)
        if venv_data:
            # remove this venv from the cache
            env_path = venv_data.get('env_path')
            if env_path:
                envbuilder.destroy_venv(env_path, venvscache)
            else:
                logger.warning("Invalid 'env_path' found in virtualenv metadata: %r. "
                               "Not removing virtualenv.", env_path)
        else:
            logger.warning('No virtualenv found with uuid: %s.', uuid)
        return 0

    # decided which the child program really is
    analyzable_child_program, child_program = decide_child_program(
        args.executable, args.child_program)

    # Group and merge dependencies
    indicated_deps = consolidate_dependencies(args.ipython,
                                              analyzable_child_program,
                                              args.requirement,
                                              args.dependency)

    # Check for packages updates
    if args.check_updates:
        helpers.check_pypi_updates(indicated_deps)

    # get the interpreter version requested for the child_program
    interpreter, is_current = helpers.get_interpreter_version(args.python)

    # options
    pip_options = args.pip_options  # pip_options mustn't store.
    python_options = args.python_options
    options = {}
    options['pyvenv_options'] = []
    options['virtualenv_options'] = args.virtualenv_options
    if args.system_site_packages:
        options['virtualenv_options'].append("--system-site-packages")
        options['pyvenv_options'] = ["--system-site-packages"]

    create_venv = False
    venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options)
    if venv_data:
        env_path = venv_data['env_path']
        # A venv was found in the cache check if its valid or re-generate it.
        if not os.path.exists(env_path):
            logger.warning("Missing directory (the virtualenv will be re-created): %r", env_path)
            venvscache.remove(env_path)
            create_venv = True
    else:
        create_venv = True

    if create_venv:
        # Check if the requested packages exists in pypi.
        if not args.no_precheck_availability and indicated_deps.get('pypi'):
            logger.info("Checking the availabilty of dependencies in PyPI. "
                        "You can use '--no-precheck-availability' to avoid it.")
            if not helpers.check_pypi_exists(indicated_deps):
                logger.error("An indicated dependency doesn't exist. Exiting")
                raise FadesError("Required dependency does not exist")

        # Create a new venv
        venv_data, installed = envbuilder.create_venv(indicated_deps, args.python, is_current,
                                                      options, pip_options)
        # store this new venv in the cache
        venvscache.store(installed, venv_data, interpreter, options)

    if args.get_venv_dir:
        # all it was requested is the virtualenv's path, show it and quit (don't run anything)
        print(venv_data['env_path'])
        return 0

    # run forest run!!
    python_exe = 'ipython' if args.ipython else 'python'
    python_exe = os.path.join(venv_data['env_bin_path'], python_exe)

    # add the virtualenv /bin path to the child PATH.
    environ_path = venv_data['env_bin_path']
    if 'PATH' in os.environ:
        environ_path += os.pathsep + os.environ['PATH']
    os.environ['PATH'] = environ_path

    # store usage information
    usage_manager.store_usage_stat(venv_data, venvscache)
    if child_program is None:
        interactive = True
        logger.debug(
            "Calling the interactive Python interpreter with arguments %r", python_options)
        cmd = [python_exe] + python_options
        p = subprocess.Popen(cmd)
    else:
        interactive = False
        if args.executable:
            cmd = [os.path.join(venv_data['env_bin_path'], child_program)]
            logger.debug("Calling child program %r with options %s",
                         child_program, args.child_options)
        else:
            cmd = [python_exe] + python_options + [child_program]
            logger.debug(
                "Calling Python interpreter with arguments %s to execute the child program"
                " %r with options %s", python_options, child_program, args.child_options)

        try:
            p = subprocess.Popen(cmd + args.child_options)
        except FileNotFoundError:
            logger.error("Command not found: %s", child_program)
            raise FadesError("Command not found")

    def _signal_handler(signum, _):
        """Handle signals received by parent process, send them to child.

        The only exception is CTRL-C, that is generated *from* the interactive
        interpreter (it's a keyboard combination!), so we swallow it for the
        interpreter to not see it twice.
        """
        if interactive and signum == signal.SIGINT:
            logger.debug("Swallowing signal %s", signum)
        else:
            logger.debug("Redirecting signal %s to child", signum)
            os.kill(p.pid, signum)

    # redirect the useful signals
    for s in REDIRECTED_SIGNALS:
        signal.signal(s, _signal_handler)

    # wait child to finish, end
    rc = p.wait()
    if rc:
        logger.debug("Child process not finished correctly: returncode=%d", rc)
    return rc