def test_error_hitting_pypi(self): dependency = parsing.parse_manual(["foo"]) with self.assertRaises(Exception): with patch('urllib.request.urlopen') as mock_urlopen: mock_urlopen.side_effect = ValueError("cabum!!") helpers.check_pypi_exists(dependency)
def test_error_hitting_pypi(self): dependency = parsing.parse_manual(["foo"]) with self.assertRaises(Exception): with patch('urllib.request.urlopen') as mock_urlopen: mock_urlopen.side_effect = ValueError("cabum!!") helpers.check_pypi_exists(dependency)
def test_status_code_error(self): dependency = parsing.parse_manual(["foo"]) with self.assertRaises(Exception): with patch('urllib.request.urlopen') as mock_urlopen: mock_http_error = HTTPError("url", 400, "mgs", {}, io.BytesIO()) mock_urlopen.side_effect = mock_http_error helpers.check_pypi_exists(dependency)
def test_status_code_error(self): dependency = parsing.parse_manual(["foo"]) with self.assertRaises(Exception): with patch('urllib.request.urlopen') as mock_urlopen: mock_http_error = HTTPError("url", 400, "mgs", {}, io.BytesIO()) mock_urlopen.side_effect = mock_http_error helpers.check_pypi_exists(dependency)
def test_error_hitting_pypi(self): dependency = parsing.parse_manual(["foo"]) with patch('sys.exit') as mocked_exit: with patch('urllib.request.urlopen') as mock_urlopen: mock_urlopen.side_effect = ValueError("cabum!!") helpers.check_pypi_exists(dependency) self.assertTrue(mocked_exit.called) mocked_exit.assert_called_once_with(1)
def test_status_code_error(self): dependency = parsing.parse_manual(["foo"]) with patch('sys.exit') as mocked_exit: with patch('urllib.request.urlopen') as mock_urlopen: mock_http_error = HTTPError("url", 400, "mgs", {}, io.BytesIO()) mock_urlopen.side_effect = mock_http_error helpers.check_pypi_exists(dependency) self.assertTrue(mocked_exit.called) mocked_exit.assert_called_once_with(1)
def test_redirect_response(self): deps = parsing.parse_manual(["foo"]) with patch('urllib.request.urlopen') as mock_urlopen: with patch('http.client.HTTPResponse') as mock_http_response: mock_http_response.status = 302 # redirect mock_urlopen.return_value = mock_http_response exists = helpers.check_pypi_exists(deps) self.assertTrue(exists) self.assertLoggedWarning("Got a (unexpected) HTTP_STATUS")
def test_doesnt_exists(self): dependency = parsing.parse_manual(["foo"]) with patch('urllib.request.urlopen') as mock_urlopen: mock_http_error = HTTPError("url", HTTP_STATUS_NOT_FOUND, "mgs", {}, io.BytesIO()) mock_urlopen.side_effect = mock_http_error exists = helpers.check_pypi_exists(dependency) self.assertFalse(exists) self.assertLoggedError("foo doesn't exists in PyPI.")
def test_redirect_response(self): deps = parsing.parse_manual(["foo"]) with patch('urllib.request.urlopen') as mock_urlopen: with patch('http.client.HTTPResponse') as mock_http_response: mock_http_response.status = 302 # redirect mock_urlopen.return_value = mock_http_response exists = helpers.check_pypi_exists(deps) self.assertTrue(exists) self.assertLoggedWarning("Got a (unexpected) HTTP_STATUS")
def test_doesnt_exists(self): dependency = parsing.parse_manual(["foo"]) with patch('urllib.request.urlopen') as mock_urlopen: mock_http_error = HTTPError("url", HTTP_STATUS_NOT_FOUND, "mgs", {}, io.BytesIO()) mock_urlopen.side_effect = mock_http_error exists = helpers.check_pypi_exists(dependency) self.assertFalse(exists) self.assertLoggedError("foo doesn't exists in PyPI.")
def test_exists(self): deps = parsing.parse_manual(["foo"]) with patch('urllib.request.urlopen') as mock_urlopen: with patch('http.client.HTTPResponse') as mock_http_response: mock_http_response.status = HTTP_STATUS_OK mock_urlopen.return_value = mock_http_response exists = helpers.check_pypi_exists(deps) self.assertTrue(exists) self.assertLogged("exists in PyPI")
def test_all_exists(self): dependencies = parsing.parse_manual(['foo', 'bar', 'baz']) with patch('urllib.request.urlopen') as mock_urlopen: with patch('http.client.HTTPResponse') as mock_http_response: mock_http_response.status = HTTP_STATUS_OK mock_urlopen.side_effect = [mock_http_response] * 3 exists = helpers.check_pypi_exists(dependencies) self.assertTrue(exists) self.assertLogged("exists in PyPI")
def test_exists(self): deps = parsing.parse_manual(["foo"]) with patch('urllib.request.urlopen') as mock_urlopen: with patch('http.client.HTTPResponse') as mock_http_response: mock_http_response.status = HTTP_STATUS_OK mock_urlopen.return_value = mock_http_response exists = helpers.check_pypi_exists(deps) self.assertTrue(exists) self.assertLogged("exists in PyPI")
def test_all_exists(self): dependencies = parsing.parse_manual(['foo', 'bar', 'baz']) with patch('urllib.request.urlopen') as mock_urlopen: with patch('http.client.HTTPResponse') as mock_http_response: mock_http_response.status = HTTP_STATUS_OK mock_urlopen.side_effect = [mock_http_response] * 3 exists = helpers.check_pypi_exists(dependencies) self.assertTrue(exists) self.assertLogged("exists in PyPI")
def test_one_doesnt_exists(self): dependencies = parsing.parse_manual(["foo", "bar"]) with patch('urllib.request.urlopen') as mock_urlopen: with patch('http.client.HTTPResponse') as mock_http_response: mock_http_error = HTTPError("url", HTTP_STATUS_NOT_FOUND, "mgs", {}, io.BytesIO()) mock_http_response.status = HTTP_STATUS_OK mock_urlopen.side_effect = [mock_http_response, mock_http_error] exists = helpers.check_pypi_exists(dependencies) self.assertFalse(exists) self.assertLoggedError("bar doesn't exists in PyPI.")
def test_one_doesnt_exists(self): dependencies = parsing.parse_manual(["foo", "bar"]) with patch('urllib.request.urlopen') as mock_urlopen: with patch('http.client.HTTPResponse') as mock_http_response: mock_http_error = HTTPError("url", HTTP_STATUS_NOT_FOUND, "mgs", {}, io.BytesIO()) mock_http_response.status = HTTP_STATUS_OK mock_urlopen.side_effect = [mock_http_response, mock_http_error] exists = helpers.check_pypi_exists(dependencies) self.assertFalse(exists) self.assertLoggedError("bar doesn't exists in PyPI.")
env_path = venv_data['env_path'] # A venv was found in the cache check if its valid or re-generate it. if not os.path.exists(env_path): logger.warning("Missing directory (the virtualenv will be re-created): %r", env_path) venvscache.remove(env_path) create_venv = True else: create_venv = True if create_venv: # Check if the requested packages exists in pypi. if not args.no_precheck_availability and indicated_deps.get('pypi'): logger.info( "Checking the availabilty of dependencies in PyPI. " "You can use '--no-precheck-availability' to avoid it.") if not helpers.check_pypi_exists(indicated_deps): logger.error("An indicated dependency doesn't exist. Exiting") raise FadesError("Required dependency does not exist") # Create a new venv venv_data, installed = envbuilder.create_venv( indicated_deps, args.python, is_current, options, pip_options, args.avoid_pip_upgrade) # store this new venv in the cache venvscache.store(installed, venv_data, interpreter, options) if args.get_venv_dir: # all it was requested is the virtualenv's path, show it and quit (don't run anything) print(venv_data['env_path']) return 0 if args.freeze:
def go(): """Make the magic happen.""" parser = argparse.ArgumentParser(prog='PROG', epilog=help_epilog, usage=help_usage, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('-V', '--version', action='store_true', help="show version and info about the system, and exit") parser.add_argument('-v', '--verbose', action='store_true', help="send all internal debugging lines to stderr, which may be very " "useful to debug any problem that may arise.") parser.add_argument('-q', '--quiet', action='store_true', help="don't show anything (unless it has a real problem), so the " "original script stderr is not polluted at all.") parser.add_argument('-d', '--dependency', action='append', help="specify dependencies through command line (this option can be " "used multiple times)") parser.add_argument('-r', '--requirement', help="indicate from which file read the dependencies") parser.add_argument('-p', '--python', action='store', help=("Specify the Python interpreter to use.\n" " Default is: %s") % (sys.executable,)) parser.add_argument('-x', '--exec', dest='executable', action='store_true', help=("Indicate that the child_program should be looked up in the " "virtualenv.")) parser.add_argument('-i', '--ipython', action='store_true', help="use IPython shell.") parser.add_argument('--system-site-packages', action='store_true', default=False, help=("Give the virtual environment access to the " "system site-packages dir.")) parser.add_argument('--virtualenv-options', action='append', default=[], help=("Extra options to be supplied to virtualenv. (this option can be " "used multiple times)")) parser.add_argument('--check-updates', action='store_true', help=("check for packages updates")) parser.add_argument('--no-precheck-availability', action='store_true', help=("Don't check if the packages exists in PyPI.")) parser.add_argument('--pip-options', action='append', default=[], help=("Extra options to be supplied to pip. (this option can be " "used multiple times)")) parser.add_argument('--python-options', action='append', default=[], help=("Extra options to be supplied to python. (this option can be " "used multiple times)")) parser.add_argument('--rm', dest='remove', metavar='UUID', help=("Remove a virtualenv by UUID.")) parser.add_argument('--clean-unused-venvs', action='store', help=("This option remove venvs that haven't been used for more than " "CLEAN_UNUSED_VENVS days. Appart from that, will compact usage " "stats file.\n" "When this option is present, the cleaning takes place at the " "beginning of the execution.")) parser.add_argument('child_program', nargs='?', default=None) parser.add_argument('child_options', nargs=argparse.REMAINDER) # support the case when executed from a shell-bang, where all the # parameters come in sys.argv[1] in a single string separated # by spaces (in this case, the third parameter is what is being # executed) if len(sys.argv) > 1 and " " in sys.argv[1]: real_args = sys.argv[1].split() + sys.argv[2:] cli_args = parser.parse_args(real_args) else: cli_args = parser.parse_args() # update args from config file (if needed). args = file_options.options_from_file(cli_args) # validate input, parameters, and support some special options if args.version: print("Running 'fades' version", fades.__version__) print(" Python:", sys.version_info) print(" System:", platform.platform()) sys.exit(0) # set up logger and dump basic version info logger = fades_logger.set_up(args.verbose, args.quiet) logger.debug("Running Python %s on %r", sys.version_info, platform.platform()) logger.debug("Starting fades v. %s", fades.__version__) logger.debug("Arguments: %s", args) # verify that the module is NOT being used from a virtualenv if detect_inside_virtualenv(sys.prefix, getattr(sys, 'real_prefix', None), getattr(sys, 'base_prefix', None)): logger.error( "fades is running from inside a virtualenv (%r), which is not supported", sys.prefix) sys.exit(-1) if args.verbose and args.quiet: logger.warning("Overriding 'quiet' option ('verbose' also requested)") # start the virtualenvs manager venvscache = cache.VEnvsCache(os.path.join(helpers.get_basedir(), 'venvs.idx')) # start usage manager usage_manager = envbuilder.UsageManager(os.path.join(helpers.get_basedir(), 'usage_stats'), venvscache) rc = 0 if args.clean_unused_venvs: try: max_days_to_keep = int(args.clean_unused_venvs) usage_manager.clean_unused_venvs(max_days_to_keep) except Exception: rc = 1 logger.debug("CLEAN_UNUSED_VENVS must be an integer.") raise finally: sys.exit(rc) uuid = args.remove if uuid: venv_data = venvscache.get_venv(uuid=uuid) if venv_data: # remove this venv from the cache env_path = venv_data.get('env_path') if env_path: envbuilder.destroy_venv(env_path, venvscache) else: logger.warning("Invalid 'env_path' found in virtualenv metadata: %r. " "Not removing virtualenv.", env_path) else: logger.warning('No virtualenv found with uuid: %s.', uuid) return # parse file and get deps if args.ipython: logger.debug("Adding ipython dependency because --ipython was detected") ipython_dep = parsing.parse_manual(['ipython']) else: ipython_dep = {} if args.executable: indicated_deps = {} docstring_deps = {} else: indicated_deps = parsing.parse_srcfile(args.child_program) logger.debug("Dependencies from source file: %s", indicated_deps) docstring_deps = parsing.parse_docstring(args.child_program) logger.debug("Dependencies from docstrings: %s", docstring_deps) reqfile_deps = parsing.parse_reqfile(args.requirement) logger.debug("Dependencies from requirements file: %s", reqfile_deps) manual_deps = parsing.parse_manual(args.dependency) logger.debug("Dependencies from parameters: %s", manual_deps) indicated_deps = _merge_deps(ipython_dep, indicated_deps, docstring_deps, reqfile_deps, manual_deps) # Check for packages updates if args.check_updates: helpers.check_pypi_updates(indicated_deps) # get the interpreter version requested for the child_program interpreter, is_current = helpers.get_interpreter_version(args.python) # options pip_options = args.pip_options # pip_options mustn't store. python_options = args.python_options options = {} options['pyvenv_options'] = [] options['virtualenv_options'] = args.virtualenv_options if args.system_site_packages: options['virtualenv_options'].append("--system-site-packages") options['pyvenv_options'] = ["--system-site-packages"] create_venv = False venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options) if venv_data: env_path = venv_data['env_path'] # A venv was found in the cache check if its valid or re-generate it. if not os.path.exists(env_path): logger.warning("Missing directory (the virtualenv will be re-created): %r", env_path) venvscache.remove(env_path) create_venv = True else: create_venv = True if create_venv: # Check if the requested packages exists in pypi. if not args.no_precheck_availability: logger.info("Checking the availabilty of dependencies in PyPI. " "You can use '--no-precheck-availability' to avoid it.") if not helpers.check_pypi_exists(indicated_deps): logger.error("An indicated dependency doesn't exists. Exiting") sys.exit(1) # Create a new venv venv_data, installed = envbuilder.create_venv(indicated_deps, args.python, is_current, options, pip_options) # store this new venv in the cache venvscache.store(installed, venv_data, interpreter, options) # run forest run!! python_exe = 'ipython' if args.ipython else 'python' python_exe = os.path.join(venv_data['env_bin_path'], python_exe) # add the virtualenv /bin path to the child PATH. environ_path = venv_data['env_bin_path'] if 'PATH' in os.environ: environ_path += os.pathsep + os.environ['PATH'] os.environ['PATH'] = environ_path # store usage information usage_manager.store_usage_stat(venv_data, venvscache) if args.child_program is None: interactive = True logger.debug( "Calling the interactive Python interpreter with arguments %r", python_options) cmd = [python_exe] + python_options p = subprocess.Popen(cmd) else: interactive = False if args.executable: cmd = [os.path.join(venv_data['env_bin_path'], args.child_program)] logger.debug("Calling child program %r with options %s", args.child_program, args.child_options) else: cmd = [python_exe] + python_options + [args.child_program] logger.debug( "Calling Python interpreter with arguments %s to execute the child program" " %r with options %s", python_options, args.child_program, args.child_options) try: p = subprocess.Popen(cmd + args.child_options) except FileNotFoundError: logger.error("Command not found: %s", args.child_program) sys.exit(1) def _signal_handler(signum, _): """Handle signals received by parent process, send them to child. The only exception is CTRL-C, that is generated *from* the interactive interpreter (it's a keyboard combination!), so we swallow it for the interpreter to not see it twice. """ if interactive and signum == signal.SIGINT: logger.debug("Swallowing signal %s", signum) else: logger.debug("Redirecting signal %s to child", signum) os.kill(p.pid, signum) # redirect the useful signals for s in REDIRECTED_SIGNALS: signal.signal(s, _signal_handler) # wait child to finish, end rc = p.wait() if rc: logger.debug("Child process not finished correctly: returncode=%d", rc) sys.exit(rc)
def go(): """Make the magic happen.""" parser = argparse.ArgumentParser( prog='fades', epilog=HELP_EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( '-V', '--version', action='store_true', help="show version and info about the system, and exit") parser.add_argument( '-d', '--dependency', action='append', help= "specify dependencies through command line (this option can be used multiple times)" ) parser.add_argument( '-r', '--requirement', action='append', help= "indicate files to read dependencies from (this option can be used multiple times)" ) parser.add_argument( '-p', '--python', action='store', help="specify the Python interpreter to use; the default is: {}". format(sys.executable)) parser.add_argument('-i', '--ipython', action='store_true', help="use IPython shell when in interactive mode") parser.add_argument( '--system-site-packages', action='store_true', default=False, help= "give the virtual environment access to the system site-packages dir.") parser.add_argument( '--virtualenv-options', action='append', default=[], help= "extra options to be supplied to virtualenv (this option can be used multiple times)" ) parser.add_argument('--check-updates', action='store_true', help="check for packages updates") parser.add_argument( '--no-precheck-availability', action='store_true', help= "don't check if the packages exists in PyPI before actually try to install them" ) parser.add_argument( '--pip-options', action='append', default=[], help= "extra options to be supplied to pip (this option can be used multiple times)" ) parser.add_argument( '--python-options', action='append', default=[], help= "extra options to be supplied to python (this option can be used multiple times)" ) parser.add_argument( '--rm', dest='remove', metavar='UUID', help= "remove a virtualenv by UUID; see --get-venv-dir option to easily find out the UUID" ) parser.add_argument( '--clean-unused-venvs', action='store', help= "remove venvs that haven't been used for more than the indicated days and compact " "usage stats file (all this takes place at the beginning of the execution)" ) parser.add_argument( '--get-venv-dir', action='store_true', help= "show the virtualenv base directory (including the venv's UUID) and quit" ) parser.add_argument( '-a', '--autoimport', action='store_true', help= "automatically import the specified dependencies in the interactive mode " "(ignored otherwise).") parser.add_argument( '--freeze', action='store', metavar='FILEPATH', help= "dump all the dependencies and its versions to the specified filepath " "(operating normally beyond that)") parser.add_argument( '--avoid-pip-upgrade', action='store_true', help= "disable the automatic pip upgrade that happens after the virtualenv is created " "and before the dependencies begin to be installed.") mutexg = parser.add_mutually_exclusive_group() mutexg.add_argument( '-v', '--verbose', action='store_true', help="send all internal debugging lines to stderr, which may be very " "useful to debug any problem that may arise") mutexg.add_argument( '-q', '--quiet', action='store_true', help="don't show anything (unless it has a real problem), so the " "original script stderr is not polluted at all") mutexg = parser.add_mutually_exclusive_group() mutexg.add_argument( '-x', '--exec', dest='executable', action='store_true', help= "execute the child_program (must be present) in the context of the virtualenv" ) mutexg.add_argument( '-m', '--module', action='store_true', help= "run library module as a script (same behaviour than Python's -m parameter)" ) parser.add_argument('child_program', nargs='?', default=None) parser.add_argument('child_options', nargs=argparse.REMAINDER) cli_args = parser.parse_args() # update args from config file (if needed). args = file_options.options_from_file(cli_args) # validate input, parameters, and support some special options if args.version: print("Running 'fades' version", fades.__version__) print(" Python:", sys.version_info) print(" System:", platform.platform()) return 0 # The --exec and --module flags needs child_program to exist (this is not handled at # argparse level because it's easier to collect the executable as the # normal child_program, so everything after that are parameteres # considered for the executable itself, not for fades). if args.executable and not args.child_program: parser.print_usage() print( "fades: error: argument -x/--exec needs child_program to be present" ) return -1 if args.module and not args.child_program: parser.print_usage() print( "fades: error: argument -m/--module needs child_program (module) to be present" ) return -1 # set up the logger and dump basic version info logger_set_up(args.verbose, args.quiet) logger.debug("Running Python %s on %r", sys.version_info, platform.platform()) logger.debug("Starting fades v. %s", fades.__version__) logger.debug("Arguments: %s", args) # verify that the module is NOT being used from a virtualenv _real_prefix = getattr(sys, 'real_prefix', None) _base_prefix = getattr(sys, 'base_prefix', None) if detect_inside_virtualenv(sys.prefix, _real_prefix, _base_prefix): logger.error( "fades is running from inside a virtualenv (%r), which is not supported", sys.prefix) raise FadesError("Cannot run from a virtualenv") if args.verbose and args.quiet: logger.warning("Overriding 'quiet' option ('verbose' also requested)") # start the virtualenvs manager venvscache = cache.VEnvsCache( os.path.join(helpers.get_basedir(), 'venvs.idx')) # start usage manager usage_manager = envbuilder.UsageManager( os.path.join(helpers.get_basedir(), 'usage_stats'), venvscache) if args.clean_unused_venvs: try: max_days_to_keep = int(args.clean_unused_venvs) except ValueError: logger.error("clean_unused_venvs must be an integer.") raise FadesError('clean_unused_venvs not an integer') usage_manager.clean_unused_venvs(max_days_to_keep) return 0 uuid = args.remove if uuid: venv_data = venvscache.get_venv(uuid=uuid) if venv_data: # remove this venv from the cache env_path = venv_data.get('env_path') if env_path: envbuilder.destroy_venv(env_path, venvscache) else: logger.warning( "Invalid 'env_path' found in virtualenv metadata: %r. " "Not removing virtualenv.", env_path) else: logger.warning('No virtualenv found with uuid: %s.', uuid) return 0 # decided which the child program really is analyzable_child_program, child_program = decide_child_program( args.executable, args.module, args.child_program) # Group and merge dependencies indicated_deps = consolidate_dependencies(args.ipython, analyzable_child_program, args.requirement, args.dependency) # Check for packages updates if args.check_updates: helpers.check_pypi_updates(indicated_deps) # get the interpreter version requested for the child_program interpreter, is_current = helpers.get_interpreter_version(args.python) # options pip_options = args.pip_options # pip_options mustn't store. python_options = args.python_options options = {} options['pyvenv_options'] = [] options['virtualenv_options'] = args.virtualenv_options if args.system_site_packages: options['virtualenv_options'].append("--system-site-packages") options['pyvenv_options'] = ["--system-site-packages"] create_venv = False venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options) if venv_data: env_path = venv_data['env_path'] # A venv was found in the cache check if its valid or re-generate it. if not os.path.exists(env_path): logger.warning( "Missing directory (the virtualenv will be re-created): %r", env_path) venvscache.remove(env_path) create_venv = True else: create_venv = True if create_venv: # Check if the requested packages exists in pypi. if not args.no_precheck_availability and indicated_deps.get('pypi'): logger.info( "Checking the availabilty of dependencies in PyPI. " "You can use '--no-precheck-availability' to avoid it.") if not helpers.check_pypi_exists(indicated_deps): logger.error("An indicated dependency doesn't exist. Exiting") raise FadesError("Required dependency does not exist") # Create a new venv venv_data, installed = envbuilder.create_venv(indicated_deps, args.python, is_current, options, pip_options, args.avoid_pip_upgrade) # store this new venv in the cache venvscache.store(installed, venv_data, interpreter, options) if args.get_venv_dir: # all it was requested is the virtualenv's path, show it and quit (don't run anything) print(venv_data['env_path']) return 0 if args.freeze: # beyond all the rest of work, dump the dependencies versions to a file mgr = pipmanager.PipManager(venv_data['env_bin_path']) mgr.freeze(args.freeze) # run forest run!! python_exe = 'ipython' if args.ipython else 'python' python_exe = os.path.join(venv_data['env_bin_path'], python_exe) # add the virtualenv /bin path to the child PATH. environ_path = venv_data['env_bin_path'] if 'PATH' in os.environ: environ_path += os.pathsep + os.environ['PATH'] os.environ['PATH'] = environ_path # store usage information usage_manager.store_usage_stat(venv_data, venvscache) if child_program is None: interactive = True cmd = [python_exe] + python_options # get possible extra python options and environement for auto import if indicated_deps and args.autoimport: temp_scriptpath = get_autoimport_scriptname( indicated_deps, args.ipython) cmd += ['-i', temp_scriptpath] logger.debug("Calling the interactive Python interpreter: %s", cmd) proc = subprocess.Popen(cmd) else: interactive = False if args.executable: # Build the exec path relative to 'bin' dir; note that if child_program's path # is absolute (starting with '/') the resulting exec_path will be just it, # which is something fades supports exec_path = os.path.join(venv_data['env_bin_path'], child_program) cmd = [exec_path] elif args.module: cmd = [python_exe, '-m'] + python_options + [child_program] else: cmd = [python_exe] + python_options + [child_program] # Incorporate the child options, always at the end, log and run. cmd += args.child_options logger.debug("Calling %s", cmd) try: proc = subprocess.Popen(cmd) except FileNotFoundError: logger.error("Command not found: %s", child_program) raise FadesError("Command not found") def _signal_handler(signum, _): """Handle signals received by parent process, send them to child. The only exception is CTRL-C, that is generated *from* the interactive interpreter (it's a keyboard combination!), so we swallow it for the interpreter to not see it twice. """ if interactive and signum == signal.SIGINT: logger.debug("Swallowing signal %s", signum) else: logger.debug("Redirecting signal %s to child", signum) os.kill(proc.pid, signum) # redirect the useful signals for s in REDIRECTED_SIGNALS: signal.signal(s, _signal_handler) # wait child to finish, end rc = proc.wait() if rc: logger.debug("Child process not finished correctly: returncode=%d", rc) return rc
def go(): """Make the magic happen.""" parser = argparse.ArgumentParser(prog='PROG', epilog=help_epilog, usage=help_usage, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('-V', '--version', action='store_true', help="show version and info about the system, and exit") parser.add_argument('-v', '--verbose', action='store_true', help="send all internal debugging lines to stderr, which may be very " "useful to debug any problem that may arise.") parser.add_argument('-q', '--quiet', action='store_true', help="don't show anything (unless it has a real problem), so the " "original script stderr is not polluted at all.") parser.add_argument('-d', '--dependency', action='append', help="specify dependencies through command line (this option can be " "used multiple times)") parser.add_argument('-r', '--requirement', help="indicate from which file read the dependencies") parser.add_argument('-p', '--python', action='store', help=("Specify the Python interpreter to use.\n" " Default is: %s") % (sys.executable,)) parser.add_argument('-x', '--exec', dest='executable', action='store_true', help=("Indicate that the child_program should be looked up in the " "virtualenv.")) parser.add_argument('-i', '--ipython', action='store_true', help="use IPython shell.") parser.add_argument('--system-site-packages', action='store_true', default=False, help=("Give the virtual environment access to the " "system site-packages dir.")) parser.add_argument('--virtualenv-options', action='append', default=[], help=("Extra options to be supplied to virtualenv. (this option can be " "used multiple times)")) parser.add_argument('--check-updates', action='store_true', help=("check for packages updates")) parser.add_argument('--no-precheck-availability', action='store_true', help=("Don't check if the packages exists in PyPI.")) parser.add_argument('--pip-options', action='append', default=[], help=("Extra options to be supplied to pip. (this option can be " "used multiple times)")) parser.add_argument('--python-options', action='append', default=[], help=("Extra options to be supplied to python. (this option can be " "used multiple times)")) parser.add_argument('--rm', dest='remove', metavar='UUID', help=("Remove a virtualenv by UUID.")) parser.add_argument('--clean-unused-venvs', action='store', help=("This option remove venvs that haven't been used for more than " "CLEAN_UNUSED_VENVS days. Appart from that, will compact usage " "stats file.\n" "When this option is present, the cleaning takes place at the " "beginning of the execution.")) parser.add_argument('child_program', nargs='?', default=None) parser.add_argument('child_options', nargs=argparse.REMAINDER) # support the case when executed from a shell-bang, where all the # parameters come in sys.argv[1] in a single string separated # by spaces (in this case, the third parameter is what is being # executed) if len(sys.argv) > 1 and " " in sys.argv[1]: real_args = sys.argv[1].split() + sys.argv[2:] cli_args = parser.parse_args(real_args) else: cli_args = parser.parse_args() # update args from config file (if needed). args = file_options.options_from_file(cli_args) # validate input, parameters, and support some special options if args.version: print("Running 'fades' version", fades.__version__) print(" Python:", sys.version_info) print(" System:", platform.platform()) return 0 # set up logger and dump basic version info logger = fades_logger.set_up(args.verbose, args.quiet) logger.debug("Running Python %s on %r", sys.version_info, platform.platform()) logger.debug("Starting fades v. %s", fades.__version__) logger.debug("Arguments: %s", args) # verify that the module is NOT being used from a virtualenv if detect_inside_virtualenv(sys.prefix, getattr(sys, 'real_prefix', None), getattr(sys, 'base_prefix', None)): logger.error( "fades is running from inside a virtualenv (%r), which is not supported", sys.prefix) raise FadesError("Cannot run from a virtualenv") if args.verbose and args.quiet: logger.warning("Overriding 'quiet' option ('verbose' also requested)") # start the virtualenvs manager venvscache = cache.VEnvsCache(os.path.join(helpers.get_basedir(), 'venvs.idx')) # start usage manager usage_manager = envbuilder.UsageManager(os.path.join(helpers.get_basedir(), 'usage_stats'), venvscache) if args.clean_unused_venvs: try: max_days_to_keep = int(args.clean_unused_venvs) except ValueError: logger.error("clean_unused_venvs must be an integer.") raise FadesError('clean_unused_venvs not an integer') usage_manager.clean_unused_venvs(max_days_to_keep) return 0 uuid = args.remove if uuid: venv_data = venvscache.get_venv(uuid=uuid) if venv_data: # remove this venv from the cache env_path = venv_data.get('env_path') if env_path: envbuilder.destroy_venv(env_path, venvscache) else: logger.warning("Invalid 'env_path' found in virtualenv metadata: %r. " "Not removing virtualenv.", env_path) else: logger.warning('No virtualenv found with uuid: %s.', uuid) return 0 # parse file and get deps if args.ipython: logger.debug("Adding ipython dependency because --ipython was detected") ipython_dep = parsing.parse_manual(['ipython']) else: ipython_dep = {} if args.executable: indicated_deps = {} docstring_deps = {} else: indicated_deps = parsing.parse_srcfile(args.child_program) logger.debug("Dependencies from source file: %s", indicated_deps) docstring_deps = parsing.parse_docstring(args.child_program) logger.debug("Dependencies from docstrings: %s", docstring_deps) reqfile_deps = parsing.parse_reqfile(args.requirement) logger.debug("Dependencies from requirements file: %s", reqfile_deps) manual_deps = parsing.parse_manual(args.dependency) logger.debug("Dependencies from parameters: %s", manual_deps) indicated_deps = _merge_deps(ipython_dep, indicated_deps, docstring_deps, reqfile_deps, manual_deps) # Check for packages updates if args.check_updates: helpers.check_pypi_updates(indicated_deps) # get the interpreter version requested for the child_program interpreter, is_current = helpers.get_interpreter_version(args.python) # options pip_options = args.pip_options # pip_options mustn't store. python_options = args.python_options options = {} options['pyvenv_options'] = [] options['virtualenv_options'] = args.virtualenv_options if args.system_site_packages: options['virtualenv_options'].append("--system-site-packages") options['pyvenv_options'] = ["--system-site-packages"] create_venv = False venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options) if venv_data: env_path = venv_data['env_path'] # A venv was found in the cache check if its valid or re-generate it. if not os.path.exists(env_path): logger.warning("Missing directory (the virtualenv will be re-created): %r", env_path) venvscache.remove(env_path) create_venv = True else: create_venv = True if create_venv: # Check if the requested packages exists in pypi. if not args.no_precheck_availability: logger.info("Checking the availabilty of dependencies in PyPI. " "You can use '--no-precheck-availability' to avoid it.") if not helpers.check_pypi_exists(indicated_deps): logger.error("An indicated dependency doesn't exist. Exiting") raise FadesError("Required dependency does not exist") # Create a new venv venv_data, installed = envbuilder.create_venv(indicated_deps, args.python, is_current, options, pip_options) # store this new venv in the cache venvscache.store(installed, venv_data, interpreter, options) # run forest run!! python_exe = 'ipython' if args.ipython else 'python' python_exe = os.path.join(venv_data['env_bin_path'], python_exe) # add the virtualenv /bin path to the child PATH. environ_path = venv_data['env_bin_path'] if 'PATH' in os.environ: environ_path += os.pathsep + os.environ['PATH'] os.environ['PATH'] = environ_path # store usage information usage_manager.store_usage_stat(venv_data, venvscache) if args.child_program is None: interactive = True logger.debug( "Calling the interactive Python interpreter with arguments %r", python_options) cmd = [python_exe] + python_options p = subprocess.Popen(cmd) else: interactive = False if args.executable: cmd = [os.path.join(venv_data['env_bin_path'], args.child_program)] logger.debug("Calling child program %r with options %s", args.child_program, args.child_options) else: cmd = [python_exe] + python_options + [args.child_program] logger.debug( "Calling Python interpreter with arguments %s to execute the child program" " %r with options %s", python_options, args.child_program, args.child_options) try: p = subprocess.Popen(cmd + args.child_options) except FileNotFoundError: logger.error("Command not found: %s", args.child_program) raise FadesError("Command not found") def _signal_handler(signum, _): """Handle signals received by parent process, send them to child. The only exception is CTRL-C, that is generated *from* the interactive interpreter (it's a keyboard combination!), so we swallow it for the interpreter to not see it twice. """ if interactive and signum == signal.SIGINT: logger.debug("Swallowing signal %s", signum) else: logger.debug("Redirecting signal %s to child", signum) os.kill(p.pid, signum) # redirect the useful signals for s in REDIRECTED_SIGNALS: signal.signal(s, _signal_handler) # wait child to finish, end rc = p.wait() if rc: logger.debug("Child process not finished correctly: returncode=%d", rc) return rc