def test_non_existing_dep(self):
requested = {
REPO_PYPI: [get_req('dep1 == 1000')]
}
interpreter = 'python3'
is_current = True
options = {'virtualenv_options': [],
'pyvenv_options': [],
}
pip_options = []
with patch.object(envbuilder._FadesEnvBuilder, 'create_env') as mock_create:
with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
mock_create.return_value = ('env_path', 'env_bin_path', 'pip_installed')
mock_mgr_c.return_value = self.FailInstallManager()
with patch.object(envbuilder, 'destroy_venv', spec=True) as mock_destroy:
with self.assertRaises(FadesError) as cm:
envbuilder.create_venv(
requested,
interpreter,
is_current,
options,
pip_options)
self.assertEqual(str(cm.exception), 'Dependency installation failed')
mock_destroy.assert_called_once_with('env_path')
self.assertLoggedDebug("Installation Step failed, removing virtualenv")
def test_non_existing_dep(self):
requested = {REPO_PYPI: [get_req('dep1 == 1000')]}
interpreter = 'python3'
is_current = True
avoid_pip_upgrade = False
options = {
'virtualenv_options': [],
'pyvenv_options': [],
}
pip_options = []
with patch.object(envbuilder._FadesEnvBuilder,
'create_env') as mock_create:
with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
mock_create.return_value = ('env_path', 'env_bin_path',
'pip_installed')
mock_mgr_c.return_value = self.FailInstallManager()
with patch.object(envbuilder, 'destroy_venv',
spec=True) as mock_destroy:
with self.assertRaises(FadesError) as cm:
envbuilder.create_venv(requested, interpreter,
is_current, options,
pip_options, avoid_pip_upgrade)
self.assertEqual(str(cm.exception),
'Dependency installation failed')
mock_destroy.assert_called_once_with('env_path')
self.assertLoggedDebug("Installation Step failed, removing virtualenv")
def test_unknown_repo(self):
requested = {
'unknown': {'dep': ''}
}
interpreter = 'python3'
is_current = True
with patch.object(envbuilder.FadesEnvBuilder, 'create_env') as mock_create:
with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
mock_create.return_value = ('env_path', 'env_bin_path', 'pip_installed')
mock_mgr_c.return_value = self.FakeManager()
envbuilder.create_venv(requested, interpreter, is_current)
self.assertLoggedWarning("Install from 'unknown' not implemented")
def test_unknown_repo(self):
requested = {'unknown': {'dep': ''}}
interpreter = 'python3'
is_current = True
with patch.object(envbuilder.FadesEnvBuilder,
'create_env') as mock_create:
with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
mock_create.return_value = ('env_path', 'env_bin_path',
'pip_installed')
mock_mgr_c.return_value = self.FakeManager()
envbuilder.create_venv(requested, interpreter, is_current)
self.assertLoggedWarning("Install from 'unknown' not implemented")
def test_different_versions(self):
requested = {REPO_PYPI: [get_req('dep1 == v1'), get_req('dep2 == v2')]}
interpreter = 'python3'
is_current = True
avoid_pip_upgrade = False
options = {
"virtualenv_options": [],
"pyvenv_options": [],
}
pip_options = []
with patch.object(envbuilder._FadesEnvBuilder,
'create_env') as mock_create:
with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
mock_create.return_value = ('env_path', 'env_bin_path',
'pip_installed')
mock_mgr_c.return_value = fake_manager = self.FakeManager()
fake_manager.really_installed = {'dep1': 'vX', 'dep2': 'v2'}
_, installed = envbuilder.create_venv(requested, interpreter,
is_current, options,
pip_options,
avoid_pip_upgrade)
self.assertEqual(installed,
{REPO_PYPI: {
'dep1': 'vX',
'dep2': 'v2',
}})
def test_create_simple(self):
requested = {
REPO_PYPI: [get_req('dep1 == v1'), get_req('dep2 == v2')]
}
interpreter = 'python3'
is_current = True
options = {"virtualenv_options": [],
"pyvenv_options": [],
}
pip_options = []
with patch.object(envbuilder._FadesEnvBuilder, 'create_env') as mock_create:
with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
mock_create.return_value = ('env_path', 'env_bin_path', 'pip_installed')
mock_mgr_c.return_value = fake_manager = self.FakeManager()
fake_manager.really_installed = {'dep1': 'v1', 'dep2': 'v2'}
venv_data, installed = envbuilder.create_venv(requested, interpreter, is_current,
options, pip_options)
self.assertEqual(venv_data, {
'env_bin_path': 'env_bin_path',
'env_path': 'env_path',
'pip_installed': 'pip_installed',
})
self.assertDictEqual(installed, {
REPO_PYPI: {
'dep1': 'v1',
'dep2': 'v2',
}
})
def test_create_simple(self):
requested = {REPO_PYPI: [get_req('dep1 == v1'), get_req('dep2 == v2')]}
interpreter = 'python3'
is_current = True
with patch.object(envbuilder.FadesEnvBuilder,
'create_env') as mock_create:
with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
mock_create.return_value = ('env_path', 'env_bin_path',
'pip_installed')
mock_mgr_c.return_value = fake_manager = self.FakeManager()
fake_manager.really_installed = {'dep1': 'v1', 'dep2': 'v2'}
venv_data, installed = envbuilder.create_venv(
requested, interpreter, is_current)
self.assertEqual(
venv_data, {
'env_bin_path': 'env_bin_path',
'env_path': 'env_path',
'pip_installed': 'pip_installed',
})
self.assertDictEqual(installed,
{REPO_PYPI: {
'dep1': 'v1',
'dep2': 'v2',
}})
def test_unknown_repo(self):
requested = {'unknown': {'dep': ''}}
interpreter = 'python3'
is_current = True
avoid_pip_upgrade = False
options = {
"virtualenv_options": [],
"pyvenv_options": [],
}
pip_options = []
with patch.object(envbuilder._FadesEnvBuilder,
'create_env') as mock_create:
with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
mock_create.return_value = ('env_path', 'env_bin_path',
'pip_installed')
mock_mgr_c.return_value = self.FakeManager()
envbuilder.create_venv(requested, interpreter, is_current,
options, pip_options, avoid_pip_upgrade)
self.assertLoggedWarning("Install from 'unknown' not implemented")
def test_different_versions(self):
requested = {
REPO_PYPI: [get_req('dep1 == v1'), get_req('dep2 == v2')]
}
interpreter = 'python3'
is_current = True
with patch.object(envbuilder.FadesEnvBuilder, 'create_env') as mock_create:
with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
mock_create.return_value = ('env_path', 'env_bin_path', 'pip_installed')
mock_mgr_c.return_value = fake_manager = self.FakeManager()
fake_manager.really_installed = {'dep1': 'vX', 'dep2': 'v2'}
_, installed = envbuilder.create_venv(requested, interpreter, is_current)
self.assertEqual(installed, {
REPO_PYPI: {
'dep1': 'vX',
'dep2': 'v2',
}
})
def test_create_vcs(self):
requested = {
REPO_VCS: [parsing.VCSDependency("someurl")]
}
interpreter = 'python3'
is_current = True
options = {"virtualenv_options": [], "pyvenv_options": []}
pip_options = []
with patch.object(envbuilder._FadesEnvBuilder, 'create_env') as mock_create:
with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
mock_create.return_value = ('env_path', 'env_bin_path', 'pip_installed')
mock_mgr_c.return_value = self.FakeManager()
venv_data, installed = envbuilder.create_venv(requested, interpreter, is_current,
options, pip_options)
self.assertEqual(venv_data, {
'env_bin_path': 'env_bin_path',
'env_path': 'env_path',
'pip_installed': 'pip_installed',
})
self.assertDictEqual(installed, {REPO_VCS: {'someurl': None}})
def test_create_vcs(self):
requested = {
REPO_VCS: [parsing.VCSDependency("someurl")]
}
interpreter = 'python3'
is_current = True
options = {"virtualenv_options": [], "pyvenv_options": []}
pip_options = []
with patch.object(envbuilder._FadesEnvBuilder, 'create_env') as mock_create:
with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
mock_create.return_value = ('env_path', 'env_bin_path', 'pip_installed')
mock_mgr_c.return_value = self.FakeManager()
venv_data, installed = envbuilder.create_venv(requested, interpreter, is_current,
options, pip_options)
self.assertEqual(venv_data, {
'env_bin_path': 'env_bin_path',
'env_path': 'env_path',
'pip_installed': 'pip_installed',
})
self.assertDictEqual(installed, {REPO_VCS: {'someurl': None}})
def test_create_simple(self):
requested = {REPO_PYPI: [get_req('dep1 == v1'), get_req('dep2 == v2')]}
interpreter = 'python3'
is_current = True
avoid_pip_upgrade = False
options = {
"virtualenv_options": [],
"pyvenv_options": [],
}
pip_options = []
with patch.object(envbuilder._FadesEnvBuilder,
'create_env') as mock_create:
with patch.object(envbuilder, 'PipManager') as mock_mgr_c:
mock_create.return_value = ('env_path', 'env_bin_path',
'pip_installed')
mock_mgr_c.return_value = fake_manager = self.FakeManager()
fake_manager.really_installed = {'dep1': 'v1', 'dep2': 'v2'}
venv_data, installed = envbuilder.create_venv(
requested, interpreter, is_current, options, pip_options,
avoid_pip_upgrade)
self.assertEqual(
venv_data, {
'env_bin_path': 'env_bin_path',
'env_path': 'env_path',
'pip_installed': 'pip_installed',
})
self.assertDictEqual(installed,
{REPO_PYPI: {
'dep1': 'v1',
'dep2': 'v2',
}})
expected_pipmanager_call = call('env_bin_path',
pip_installed='pip_installed',
options=[],
avoid_pip_upgrade=avoid_pip_upgrade)
self.assertEqual(mock_mgr_c.call_args, expected_pipmanager_call)
def go():
"""Make the magic happen."""
parser = argparse.ArgumentParser(
prog='PROG',
epilog=help_epilog,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
'-V',
'--version',
action='store_true',
help="show version and info about the system, and exit")
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help="send all internal debugging lines to stderr, which may be very "
"useful to debug any problem that may arise.")
parser.add_argument(
'-q',
'--quiet',
action='store_true',
help="don't show anything (unless it has a real problem), so the "
"original script stderr is not polluted at all.")
parser.add_argument(
'-d',
'--dependency',
action='append',
help="specify dependencies through command line (this option can be "
"used multiple times)")
parser.add_argument(
'-r',
'--requirement',
action='append',
help="indicate files to read dependencies from (this option can be "
"used multiple times)")
parser.add_argument('-p',
'--python',
action='store',
help=("Specify the Python interpreter to use.\n"
" Default is: %s") % (sys.executable, ))
parser.add_argument(
'-x',
'--exec',
dest='executable',
action='store_true',
help=("Indicate that the child_program should be looked up in the "
"virtualenv."))
parser.add_argument('-i',
'--ipython',
action='store_true',
help="use IPython shell.")
parser.add_argument('-m',
'--module',
action='store',
help="Run library module as a script")
parser.add_argument('--system-site-packages',
action='store_true',
default=False,
help=("Give the virtual environment access to the "
"system site-packages dir."))
parser.add_argument(
'--virtualenv-options',
action='append',
default=[],
help=(
"Extra options to be supplied to virtualenv. (this option can be "
"used multiple times)"))
parser.add_argument('--check-updates',
action='store_true',
help=("check for packages updates"))
parser.add_argument(
'--no-precheck-availability',
action='store_true',
help=("Don't check if the packages exists in PyPI before actually try "
"to install them."))
parser.add_argument(
'--pip-options',
action='append',
default=[],
help=("Extra options to be supplied to pip. (this option can be "
"used multiple times)"))
parser.add_argument(
'--python-options',
action='append',
default=[],
help=("Extra options to be supplied to python. (this option can be "
"used multiple times)"))
parser.add_argument(
'--rm',
dest='remove',
metavar='UUID',
help=("Remove a virtualenv by UUID. See --get-venv-dir option to "
"easily find out the UUID."))
parser.add_argument(
'--clean-unused-venvs',
action='store',
help=("This option remove venvs that haven't been used for more than "
"CLEAN_UNUSED_VENVS days. Appart from that, will compact usage "
"stats file.\n"
"When this option is present, the cleaning takes place at the "
"beginning of the execution."))
parser.add_argument(
'--get-venv-dir',
action='store_true',
help=("Show the virtualenv base directory (which includes the "
"virtualenv UUID) and quit."))
parser.add_argument('child_program', nargs='?', default=None)
parser.add_argument('child_options', nargs=argparse.REMAINDER)
cli_args = _get_normalized_args(parser)
# update args from config file (if needed).
args = file_options.options_from_file(cli_args)
# validate input, parameters, and support some special options
if args.version:
print("Running 'fades' version", fades.__version__)
print(" Python:", sys.version_info)
print(" System:", platform.platform())
return 0
# set up logger and dump basic version info
logger = fades_logger.set_up(args.verbose, args.quiet)
logger.debug("Running Python %s on %r", sys.version_info,
platform.platform())
logger.debug("Starting fades v. %s", fades.__version__)
logger.debug("Arguments: %s", args)
# verify that the module is NOT being used from a virtualenv
if detect_inside_virtualenv(sys.prefix, getattr(sys, 'real_prefix', None),
getattr(sys, 'base_prefix', None)):
logger.error(
"fades is running from inside a virtualenv (%r), which is not supported",
sys.prefix)
raise FadesError("Cannot run from a virtualenv")
if args.verbose and args.quiet:
logger.warning("Overriding 'quiet' option ('verbose' also requested)")
# start the virtualenvs manager
venvscache = cache.VEnvsCache(
os.path.join(helpers.get_basedir(), 'venvs.idx'))
# start usage manager
usage_manager = envbuilder.UsageManager(
os.path.join(helpers.get_basedir(), 'usage_stats'), venvscache)
if args.clean_unused_venvs:
try:
max_days_to_keep = int(args.clean_unused_venvs)
except ValueError:
logger.error("clean_unused_venvs must be an integer.")
raise FadesError('clean_unused_venvs not an integer')
usage_manager.clean_unused_venvs(max_days_to_keep)
return 0
uuid = args.remove
if uuid:
venv_data = venvscache.get_venv(uuid=uuid)
if venv_data:
# remove this venv from the cache
env_path = venv_data.get('env_path')
if env_path:
envbuilder.destroy_venv(env_path, venvscache)
else:
logger.warning(
"Invalid 'env_path' found in virtualenv metadata: %r. "
"Not removing virtualenv.", env_path)
else:
logger.warning('No virtualenv found with uuid: %s.', uuid)
return 0
# decided which the child program really is
analyzable_child_program, child_program = decide_child_program(
args.executable, args.child_program)
# Group and merge dependencies
indicated_deps = consolidate_dependencies(args.ipython,
analyzable_child_program,
args.requirement,
args.dependency)
# Check for packages updates
if args.check_updates:
helpers.check_pypi_updates(indicated_deps)
# get the interpreter version requested for the child_program
interpreter, is_current = helpers.get_interpreter_version(args.python)
# options
pip_options = args.pip_options # pip_options mustn't store.
python_options = args.python_options
options = {}
options['pyvenv_options'] = []
options['virtualenv_options'] = args.virtualenv_options
if args.system_site_packages:
options['virtualenv_options'].append("--system-site-packages")
options['pyvenv_options'] = ["--system-site-packages"]
create_venv = False
venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options)
if venv_data:
env_path = venv_data['env_path']
# A venv was found in the cache check if its valid or re-generate it.
if not os.path.exists(env_path):
logger.warning(
"Missing directory (the virtualenv will be re-created): %r",
env_path)
venvscache.remove(env_path)
create_venv = True
else:
create_venv = True
if create_venv:
# Check if the requested packages exists in pypi.
if not args.no_precheck_availability and indicated_deps.get('pypi'):
logger.info(
"Checking the availabilty of dependencies in PyPI. "
"You can use '--no-precheck-availability' to avoid it.")
if not helpers.check_pypi_exists(indicated_deps):
logger.error("An indicated dependency doesn't exist. Exiting")
raise FadesError("Required dependency does not exist")
# Create a new venv
venv_data, installed = envbuilder.create_venv(indicated_deps,
args.python, is_current,
options, pip_options)
# store this new venv in the cache
venvscache.store(installed, venv_data, interpreter, options)
if args.get_venv_dir:
# all it was requested is the virtualenv's path, show it and quit (don't run anything)
print(venv_data['env_path'])
return 0
# run forest run!!
python_exe = 'ipython' if args.ipython else 'python'
python_exe = os.path.join(venv_data['env_bin_path'], python_exe)
# add the virtualenv /bin path to the child PATH.
environ_path = venv_data['env_bin_path']
if 'PATH' in os.environ:
environ_path += os.pathsep + os.environ['PATH']
os.environ['PATH'] = environ_path
# store usage information
usage_manager.store_usage_stat(venv_data, venvscache)
if args.module:
logger.debug("Executing module %r", args.module)
module_option = ["-m"] + args.module.split()
cmd = [python_exe] + python_options + module_option
p = subprocess.Popen(cmd)
elif child_program is None:
interactive = True
logger.debug(
"Calling the interactive Python interpreter with arguments %r",
python_options)
cmd = [python_exe] + python_options
p = subprocess.Popen(cmd)
else:
interactive = False
if args.executable:
cmd = [os.path.join(venv_data['env_bin_path'], child_program)]
logger.debug("Calling child program %r with options %s",
child_program, args.child_options)
else:
cmd = [python_exe] + python_options + [child_program]
logger.debug(
"Calling Python interpreter with arguments %s to execute the child program"
" %r with options %s", python_options, child_program,
args.child_options)
try:
p = subprocess.Popen(cmd + args.child_options)
except FileNotFoundError:
logger.error("Command not found: %s", child_program)
raise FadesError("Command not found")
def _signal_handler(signum, _):
"""Handle signals received by parent process, send them to child.
The only exception is CTRL-C, that is generated *from* the interactive
interpreter (it's a keyboard combination!), so we swallow it for the
interpreter to not see it twice.
"""
if interactive and signum == signal.SIGINT:
logger.debug("Swallowing signal %s", signum)
else:
logger.debug("Redirecting signal %s to child", signum)
os.kill(p.pid, signum)
# redirect the useful signals
for s in REDIRECTED_SIGNALS:
signal.signal(s, _signal_handler)
# wait child to finish, end
rc = p.wait()
if rc:
logger.debug("Child process not finished correctly: returncode=%d", rc)
return rc
def go():
"""Make the magic happen."""
parser = argparse.ArgumentParser(
prog='fades',
epilog=HELP_EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
'-V',
'--version',
action='store_true',
help="show version and info about the system, and exit")
parser.add_argument(
'-d',
'--dependency',
action='append',
help=
"specify dependencies through command line (this option can be used multiple times)"
)
parser.add_argument(
'-r',
'--requirement',
action='append',
help=
"indicate files to read dependencies from (this option can be used multiple times)"
)
parser.add_argument(
'-p',
'--python',
action='store',
help="specify the Python interpreter to use; the default is: {}".
format(sys.executable))
parser.add_argument('-i',
'--ipython',
action='store_true',
help="use IPython shell when in interactive mode")
parser.add_argument(
'--system-site-packages',
action='store_true',
default=False,
help=
"give the virtual environment access to the system site-packages dir.")
parser.add_argument(
'--virtualenv-options',
action='append',
default=[],
help=
"extra options to be supplied to virtualenv (this option can be used multiple times)"
)
parser.add_argument('--check-updates',
action='store_true',
help="check for packages updates")
parser.add_argument(
'--no-precheck-availability',
action='store_true',
help=
"don't check if the packages exists in PyPI before actually try to install them"
)
parser.add_argument(
'--pip-options',
action='append',
default=[],
help=
"extra options to be supplied to pip (this option can be used multiple times)"
)
parser.add_argument(
'--python-options',
action='append',
default=[],
help=
"extra options to be supplied to python (this option can be used multiple times)"
)
parser.add_argument(
'--rm',
dest='remove',
metavar='UUID',
help=
"remove a virtualenv by UUID; see --get-venv-dir option to easily find out the UUID"
)
parser.add_argument(
'--clean-unused-venvs',
action='store',
help=
"remove venvs that haven't been used for more than the indicated days and compact "
"usage stats file (all this takes place at the beginning of the execution)"
)
parser.add_argument(
'--get-venv-dir',
action='store_true',
help=
"show the virtualenv base directory (including the venv's UUID) and quit"
)
parser.add_argument(
'-a',
'--autoimport',
action='store_true',
help=
"automatically import the specified dependencies in the interactive mode "
"(ignored otherwise).")
parser.add_argument(
'--freeze',
action='store',
metavar='FILEPATH',
help=
"dump all the dependencies and its versions to the specified filepath "
"(operating normally beyond that)")
parser.add_argument(
'--avoid-pip-upgrade',
action='store_true',
help=
"disable the automatic pip upgrade that happens after the virtualenv is created "
"and before the dependencies begin to be installed.")
mutexg = parser.add_mutually_exclusive_group()
mutexg.add_argument(
'-v',
'--verbose',
action='store_true',
help="send all internal debugging lines to stderr, which may be very "
"useful to debug any problem that may arise")
mutexg.add_argument(
'-q',
'--quiet',
action='store_true',
help="don't show anything (unless it has a real problem), so the "
"original script stderr is not polluted at all")
mutexg = parser.add_mutually_exclusive_group()
mutexg.add_argument(
'-x',
'--exec',
dest='executable',
action='store_true',
help=
"execute the child_program (must be present) in the context of the virtualenv"
)
mutexg.add_argument(
'-m',
'--module',
action='store_true',
help=
"run library module as a script (same behaviour than Python's -m parameter)"
)
parser.add_argument('child_program', nargs='?', default=None)
parser.add_argument('child_options', nargs=argparse.REMAINDER)
cli_args = parser.parse_args()
# update args from config file (if needed).
args = file_options.options_from_file(cli_args)
# validate input, parameters, and support some special options
if args.version:
print("Running 'fades' version", fades.__version__)
print(" Python:", sys.version_info)
print(" System:", platform.platform())
return 0
# The --exec and --module flags needs child_program to exist (this is not handled at
# argparse level because it's easier to collect the executable as the
# normal child_program, so everything after that are parameteres
# considered for the executable itself, not for fades).
if args.executable and not args.child_program:
parser.print_usage()
print(
"fades: error: argument -x/--exec needs child_program to be present"
)
return -1
if args.module and not args.child_program:
parser.print_usage()
print(
"fades: error: argument -m/--module needs child_program (module) to be present"
)
return -1
# set up the logger and dump basic version info
logger_set_up(args.verbose, args.quiet)
logger.debug("Running Python %s on %r", sys.version_info,
platform.platform())
logger.debug("Starting fades v. %s", fades.__version__)
logger.debug("Arguments: %s", args)
# verify that the module is NOT being used from a virtualenv
_real_prefix = getattr(sys, 'real_prefix', None)
_base_prefix = getattr(sys, 'base_prefix', None)
if detect_inside_virtualenv(sys.prefix, _real_prefix, _base_prefix):
logger.error(
"fades is running from inside a virtualenv (%r), which is not supported",
sys.prefix)
raise FadesError("Cannot run from a virtualenv")
if args.verbose and args.quiet:
logger.warning("Overriding 'quiet' option ('verbose' also requested)")
# start the virtualenvs manager
venvscache = cache.VEnvsCache(
os.path.join(helpers.get_basedir(), 'venvs.idx'))
# start usage manager
usage_manager = envbuilder.UsageManager(
os.path.join(helpers.get_basedir(), 'usage_stats'), venvscache)
if args.clean_unused_venvs:
try:
max_days_to_keep = int(args.clean_unused_venvs)
except ValueError:
logger.error("clean_unused_venvs must be an integer.")
raise FadesError('clean_unused_venvs not an integer')
usage_manager.clean_unused_venvs(max_days_to_keep)
return 0
uuid = args.remove
if uuid:
venv_data = venvscache.get_venv(uuid=uuid)
if venv_data:
# remove this venv from the cache
env_path = venv_data.get('env_path')
if env_path:
envbuilder.destroy_venv(env_path, venvscache)
else:
logger.warning(
"Invalid 'env_path' found in virtualenv metadata: %r. "
"Not removing virtualenv.", env_path)
else:
logger.warning('No virtualenv found with uuid: %s.', uuid)
return 0
# decided which the child program really is
analyzable_child_program, child_program = decide_child_program(
args.executable, args.module, args.child_program)
# Group and merge dependencies
indicated_deps = consolidate_dependencies(args.ipython,
analyzable_child_program,
args.requirement,
args.dependency)
# Check for packages updates
if args.check_updates:
helpers.check_pypi_updates(indicated_deps)
# get the interpreter version requested for the child_program
interpreter, is_current = helpers.get_interpreter_version(args.python)
# options
pip_options = args.pip_options # pip_options mustn't store.
python_options = args.python_options
options = {}
options['pyvenv_options'] = []
options['virtualenv_options'] = args.virtualenv_options
if args.system_site_packages:
options['virtualenv_options'].append("--system-site-packages")
options['pyvenv_options'] = ["--system-site-packages"]
create_venv = False
venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options)
if venv_data:
env_path = venv_data['env_path']
# A venv was found in the cache check if its valid or re-generate it.
if not os.path.exists(env_path):
logger.warning(
"Missing directory (the virtualenv will be re-created): %r",
env_path)
venvscache.remove(env_path)
create_venv = True
else:
create_venv = True
if create_venv:
# Check if the requested packages exists in pypi.
if not args.no_precheck_availability and indicated_deps.get('pypi'):
logger.info(
"Checking the availabilty of dependencies in PyPI. "
"You can use '--no-precheck-availability' to avoid it.")
if not helpers.check_pypi_exists(indicated_deps):
logger.error("An indicated dependency doesn't exist. Exiting")
raise FadesError("Required dependency does not exist")
# Create a new venv
venv_data, installed = envbuilder.create_venv(indicated_deps,
args.python, is_current,
options, pip_options,
args.avoid_pip_upgrade)
# store this new venv in the cache
venvscache.store(installed, venv_data, interpreter, options)
if args.get_venv_dir:
# all it was requested is the virtualenv's path, show it and quit (don't run anything)
print(venv_data['env_path'])
return 0
if args.freeze:
# beyond all the rest of work, dump the dependencies versions to a file
mgr = pipmanager.PipManager(venv_data['env_bin_path'])
mgr.freeze(args.freeze)
# run forest run!!
python_exe = 'ipython' if args.ipython else 'python'
python_exe = os.path.join(venv_data['env_bin_path'], python_exe)
# add the virtualenv /bin path to the child PATH.
environ_path = venv_data['env_bin_path']
if 'PATH' in os.environ:
environ_path += os.pathsep + os.environ['PATH']
os.environ['PATH'] = environ_path
# store usage information
usage_manager.store_usage_stat(venv_data, venvscache)
if child_program is None:
interactive = True
cmd = [python_exe] + python_options
# get possible extra python options and environement for auto import
if indicated_deps and args.autoimport:
temp_scriptpath = get_autoimport_scriptname(
indicated_deps, args.ipython)
cmd += ['-i', temp_scriptpath]
logger.debug("Calling the interactive Python interpreter: %s", cmd)
proc = subprocess.Popen(cmd)
else:
interactive = False
if args.executable:
# Build the exec path relative to 'bin' dir; note that if child_program's path
# is absolute (starting with '/') the resulting exec_path will be just it,
# which is something fades supports
exec_path = os.path.join(venv_data['env_bin_path'], child_program)
cmd = [exec_path]
elif args.module:
cmd = [python_exe, '-m'] + python_options + [child_program]
else:
cmd = [python_exe] + python_options + [child_program]
# Incorporate the child options, always at the end, log and run.
cmd += args.child_options
logger.debug("Calling %s", cmd)
try:
proc = subprocess.Popen(cmd)
except FileNotFoundError:
logger.error("Command not found: %s", child_program)
raise FadesError("Command not found")
def _signal_handler(signum, _):
"""Handle signals received by parent process, send them to child.
The only exception is CTRL-C, that is generated *from* the interactive
interpreter (it's a keyboard combination!), so we swallow it for the
interpreter to not see it twice.
"""
if interactive and signum == signal.SIGINT:
logger.debug("Swallowing signal %s", signum)
else:
logger.debug("Redirecting signal %s to child", signum)
os.kill(proc.pid, signum)
# redirect the useful signals
for s in REDIRECTED_SIGNALS:
signal.signal(s, _signal_handler)
# wait child to finish, end
rc = proc.wait()
if rc:
logger.debug("Child process not finished correctly: returncode=%d", rc)
return rc
def go(version, argv):
"""Make the magic happen."""
parser = argparse.ArgumentParser(
prog='PROG',
epilog=help_epilog,
usage=help_usage,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
'-V',
'--version',
action='store_true',
help="show version and info about the system, and exit")
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help="send all internal debugging lines to stderr, which may be very "
"useful to debug any problem that may arise.")
parser.add_argument(
'-q',
'--quiet',
action='store_true',
help="don't show anything (unless it has a real problem), so the "
"original script stderr is not polluted at all.")
parser.add_argument(
'-d',
'--dependency',
action='append',
help="specify dependencies through command line (this option can be "
"used multiple times)")
parser.add_argument('-r',
'--requirement',
help="indicate from which file read the dependencies")
parser.add_argument('-p',
'--python',
action='store',
help=("Specify the Python interpreter to use.\n"
" Default is: %s") % (sys.executable, ))
parser.add_argument('child_program', nargs='?', default=None)
parser.add_argument('child_options', nargs=argparse.REMAINDER)
# support the case when executed from a shell-bang, where all the
# parameters come in sys.argv[1] in a single string separated
# by spaces (in this case, the third parameter is what is being
# executed)
if len(sys.argv) > 1 and " " in sys.argv[1]:
real_args = sys.argv[1].split() + [sys.argv[2]]
args = parser.parse_args(real_args)
else:
args = parser.parse_args()
# validate input, parameters, and support some special options
if args.version:
print("Running 'fades' version", version)
print(" Python:", sys.version_info)
print(" System:", sys.platform)
sys.exit()
if args.verbose:
log_level = logging.DEBUG
elif args.quiet:
log_level = logging.WARNING
else:
log_level = logging.INFO
# set up logger and dump basic version info
l = logger.set_up(log_level)
l.debug("Running Python %s on %r", sys.version_info, sys.platform)
l.debug("Starting fades v. %s", version)
l.debug("Arguments: %s", args)
if args.verbose and args.quiet:
l.warning("Overriding 'quiet' option ('verbose' also requested)")
# parse file and get deps
indicated_deps = parsing.parse_srcfile(args.child_program)
l.debug("Dependencies from source file: %s", indicated_deps)
reqfile_deps = parsing.parse_reqfile(args.requirement)
l.debug("Dependencies from requirements file: %s", reqfile_deps)
manual_deps = parsing.parse_manual(args.dependency)
l.debug("Dependencies from parameters: %s", manual_deps)
indicated_deps = _merge_deps(indicated_deps, reqfile_deps, manual_deps)
# get the interpreter version requested for the child_program
interpreter, is_current = helpers.get_interpreter_version(args.python)
# start the virtualenvs manager
venvscache = cache.VEnvsCache(
os.path.join(helpers.get_basedir(), 'venvs.idx'))
venv_data = venvscache.get_venv(indicated_deps, interpreter)
if venv_data is None:
venv_data, installed = envbuilder.create_venv(indicated_deps,
interpreter, is_current)
# store this new venv in the cache
venvscache.store(installed, venv_data, interpreter)
# run forest run!!
python_exe = os.path.join(venv_data['env_bin_path'], 'python')
if args.child_program is None:
l.debug("Calling the interactive Python interpreter")
p = subprocess.Popen([python_exe])
else:
l.debug("Calling the child Python program %r with options %s",
args.child_program, args.child_options)
p = subprocess.Popen([python_exe, args.child_program] +
args.child_options)
def _signal_handler(signum, _):
"""Handle signals received by parent process, send them to child."""
l.debug("Redirecting signal %s to child", signum)
os.kill(p.pid, signum)
# redirect these signals
for s in REDIRECTED_SIGNALS:
signal.signal(s, _signal_handler)
# wait child to finish, end
rc = p.wait()
if rc:
l.debug("Child process not finished correctly: returncode=%d", rc)
def go(version, argv):
"""Make the magic happen."""
parser = argparse.ArgumentParser(prog='PROG', epilog=help_epilog,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-V', '--version', action='store_true',
help="show version and info about the system, and exit")
parser.add_argument('-v', '--verbose', action='store_true',
help="send all internal debugging lines to stderr, which may be very "
"useful to debug any problem that may arise.")
parser.add_argument('-q', '--quiet', action='store_true',
help="don't show anything (unless it has a real problem), so the "
"original script stderr is not polluted at all.")
parser.add_argument('-d', '--dependency', action='append',
help="specify dependencies through command line (this option can be "
"used multiple times)")
parser.add_argument('-r', '--requirement',
help="indicate from which file read the dependencies")
parser.add_argument('child_program', nargs='?', default=None)
parser.add_argument('child_options', nargs=argparse.REMAINDER)
args = parser.parse_args()
# validate input, parameters, and support some special options
if args.version:
print("Running 'fades' version", version)
print(" Python:", sys.version_info)
print(" System:", sys.platform)
sys.exit()
if args.verbose:
log_level = logging.DEBUG
elif args.quiet:
log_level = logging.WARNING
else:
log_level = logging.INFO
# set up logger and dump basic version info
l = logger.set_up(log_level)
l.debug("Running Python %s on %r", sys.version_info, sys.platform)
l.debug("Starting fades v. %s", version)
l.debug("Arguments: %s", args)
if args.verbose and args.quiet:
l.warning("Overriding 'quiet' option ('verbose' also requested)")
# parse file and get deps
indicated_deps = parsing.parse_srcfile(args.child_program)
l.debug("Dependencies from source file: %s", indicated_deps)
reqfile_deps = parsing.parse_reqfile(args.requirement)
l.debug("Dependencies from requirements file: %s", reqfile_deps)
manual_deps = parsing.parse_manual(args.dependency)
l.debug("Dependencies from parameters: %s", manual_deps)
indicated_deps = _merge_deps(indicated_deps, reqfile_deps, manual_deps)
# start the virtualenvs manager
venvscache = cache.VEnvsCache(os.path.join(helpers.get_basedir(), 'venvs.idx'))
venv_data = venvscache.get_venv(indicated_deps)
if venv_data is None:
venv_data, installed = envbuilder.create_venv(indicated_deps)
# store this new venv in the cache
venvscache.store(installed, venv_data)
# run forest run!!
python_exe = os.path.join(venv_data['env_bin_path'], "python3")
if args.child_program is None:
l.debug("Calling the interactive Python interpreter")
p = subprocess.Popen([python_exe])
else:
l.debug("Calling the child Python program %r with options %s",
args.child_program, args.child_options)
p = subprocess.Popen([python_exe, args.child_program] + args.child_options)
def _signal_handler(signum, _):
"""Handle signals received by parent process, send them to child."""
l.debug("Redirecting signal %s to child", signum)
os.kill(p.pid, signum)
# redirect these signals
for s in REDIRECTED_SIGNALS:
signal.signal(s, _signal_handler)
# wait child to finish, end
rc = p.wait()
if rc:
l.debug("Child process not finished correctly: returncode=%d", rc)
def go(argv):
"""Make the magic happen."""
parser = argparse.ArgumentParser(prog='PROG', epilog=help_epilog, usage=help_usage,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-V', '--version', action='store_true',
help="show version and info about the system, and exit")
parser.add_argument('-v', '--verbose', action='store_true',
help="send all internal debugging lines to stderr, which may be very "
"useful to debug any problem that may arise.")
parser.add_argument('-q', '--quiet', action='store_true',
help="don't show anything (unless it has a real problem), so the "
"original script stderr is not polluted at all.")
parser.add_argument('-d', '--dependency', action='append',
help="specify dependencies through command line (this option can be "
"used multiple times)")
parser.add_argument('-r', '--requirement',
help="indicate from which file read the dependencies")
parser.add_argument('-p', '--python', action='store',
help=("Specify the Python interpreter to use.\n"
" Default is: %s") % (sys.executable,))
parser.add_argument('-x', '--exec', dest='executable', action='store_true',
help=("Indicate that the child_program should be looked up in the "
"virtualenv."))
parser.add_argument('-i', '--ipython', action='store_true', help="use IPython shell.")
parser.add_argument('--system-site-packages', action='store_true', default=False,
help=("Give the virtual environment access to the "
"system site-packages dir."))
parser.add_argument('--virtualenv-options', action='append', default=[],
help=("Extra options to be supplied to virtualenv. (this option can be "
"used multiple times)"))
parser.add_argument('--check-updates', action='store_true',
help=("check for packages updates"))
parser.add_argument('--pip-options', action='append', default=[],
help=("Extra options to be supplied to pip. (this option can be "
"used multiple times)"))
parser.add_argument('--rm', dest='remove', metavar='UUID',
help=("Remove a virtualenv by UUID."))
parser.add_argument('--clean-unused-venvs', action='store',
help=("This option remove venvs that haven't been used for more than "
"CLEAN_UNUSED_VENVS days. Appart from that, will compact usage "
"stats file.\n"
"When this option is present, the cleaning takes place at the "
"beginning of the execution."))
parser.add_argument('child_program', nargs='?', default=None)
parser.add_argument('child_options', nargs=argparse.REMAINDER)
# support the case when executed from a shell-bang, where all the
# parameters come in sys.argv[1] in a single string separated
# by spaces (in this case, the third parameter is what is being
# executed)
if len(sys.argv) > 1 and " " in sys.argv[1]:
real_args = sys.argv[1].split() + sys.argv[2:]
cli_args = parser.parse_args(real_args)
else:
cli_args = parser.parse_args()
# update args from config file (if needed).
args = file_options.options_from_file(cli_args)
# validate input, parameters, and support some special options
if args.version:
print("Running 'fades' version", fades.__version__)
print(" Python:", sys.version_info)
print(" System:", sys.platform)
sys.exit()
# set up logger and dump basic version info
l = logger.set_up(args.verbose, args.quiet)
l.debug("Running Python %s on %r", sys.version_info, sys.platform)
l.debug("Starting fades v. %s", fades.__version__)
l.debug("Arguments: %s", args)
if args.verbose and args.quiet:
l.warning("Overriding 'quiet' option ('verbose' also requested)")
# start the virtualenvs manager
venvscache = cache.VEnvsCache(os.path.join(helpers.get_basedir(), 'venvs.idx'))
# start usage manager
usage_manager = envbuilder.UsageManager(os.path.join(helpers.get_basedir(), 'usage_stats'),
venvscache)
if args.clean_unused_venvs:
try:
max_days_to_keep = int(args.clean_unused_venvs)
usage_manager.clean_unused_venvs(max_days_to_keep)
except:
l.debug("CLEAN_UNUSED_VENVS must be an integer.")
raise
finally:
sys.exit()
uuid = args.remove
if uuid:
venv_data = venvscache.get_venv(uuid=uuid)
if venv_data:
# remove this venv from the cache
env_path = venv_data.get('env_path')
if env_path:
envbuilder.destroy_venv(env_path, venvscache)
else:
l.warning("Invalid 'env_path' found in virtualenv metadata: %r. "
"Not removing virtualenv.", env_path)
else:
l.warning('No virtualenv found with uuid: %s.', uuid)
return
# parse file and get deps
if args.ipython:
l.debug("Adding ipython dependency because --ipython was detected")
ipython_dep = parsing.parse_manual(['ipython'])
else:
ipython_dep = {}
if args.executable:
indicated_deps = {}
docstring_deps = {}
else:
indicated_deps = parsing.parse_srcfile(args.child_program)
l.debug("Dependencies from source file: %s", indicated_deps)
docstring_deps = parsing.parse_docstring(args.child_program)
l.debug("Dependencies from docstrings: %s", docstring_deps)
reqfile_deps = parsing.parse_reqfile(args.requirement)
l.debug("Dependencies from requirements file: %s", reqfile_deps)
manual_deps = parsing.parse_manual(args.dependency)
l.debug("Dependencies from parameters: %s", manual_deps)
indicated_deps = _merge_deps(ipython_dep, indicated_deps, docstring_deps,
reqfile_deps, manual_deps)
# Check for packages updates
if args.check_updates:
helpers.check_pypi_updates(indicated_deps)
# get the interpreter version requested for the child_program
interpreter, is_current = helpers.get_interpreter_version(args.python)
# options
pip_options = args.pip_options # pip_options mustn't store.
options = {}
options['pyvenv_options'] = []
options['virtualenv_options'] = args.virtualenv_options
if args.system_site_packages:
options['virtualenv_options'].append("--system-site-packages")
options['pyvenv_options'] = ["--system-site-packages"]
create_venv = False
venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options)
if venv_data:
env_path = venv_data['env_path']
# A venv was found in the cache check if its valid or re-generate it.
if not os.path.exists(env_path):
l.warning("Missing directory (the virtualenv will be re-created): %r", env_path)
venvscache.remove(env_path)
create_venv = True
else:
create_venv = True
if create_venv:
# Create a new venv
venv_data, installed = envbuilder.create_venv(indicated_deps, args.python, is_current,
options, pip_options)
# store this new venv in the cache
venvscache.store(installed, venv_data, interpreter, options)
# run forest run!!
python_exe = 'ipython' if args.ipython else 'python'
python_exe = os.path.join(venv_data['env_bin_path'], python_exe)
# store usage information
usage_manager.store_usage_stat(venv_data, venvscache)
if args.child_program is None:
l.debug("Calling the interactive Python interpreter")
p = subprocess.Popen([python_exe])
else:
if args.executable:
cmd = [os.path.join(venv_data['env_bin_path'], args.child_program)]
else:
cmd = [python_exe, args.child_program]
l.debug("Calling the child program %r with options %s",
args.child_program, args.child_options)
p = subprocess.Popen(cmd + args.child_options)
def _signal_handler(signum, _):
"""Handle signals received by parent process, send them to child."""
l.debug("Redirecting signal %s to child", signum)
os.kill(p.pid, signum)
# redirect these signals
for s in REDIRECTED_SIGNALS:
signal.signal(s, _signal_handler)
# wait child to finish, end
rc = p.wait()
if rc:
l.debug("Child process not finished correctly: returncode=%d", rc)
sys.exit(rc)
def go(version, argv):
"""Make the magic happen."""
parser = argparse.ArgumentParser(
prog="PROG", epilog=help_epilog, usage=help_usage, formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument("-V", "--version", action="store_true", help="show version and info about the system, and exit")
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="send all internal debugging lines to stderr, which may be very "
"useful to debug any problem that may arise.",
)
parser.add_argument(
"-q",
"--quiet",
action="store_true",
help="don't show anything (unless it has a real problem), so the "
"original script stderr is not polluted at all.",
)
parser.add_argument(
"-d",
"--dependency",
action="append",
help="specify dependencies through command line (this option can be " "used multiple times)",
)
parser.add_argument("-r", "--requirement", help="indicate from which file read the dependencies")
parser.add_argument(
"-p",
"--python",
action="store",
help=("Specify the Python interpreter to use.\n" " Default is: %s") % (sys.executable,),
)
parser.add_argument(
"-x",
"--exec",
dest="executable",
action="store_true",
help=("Indicate that the child_program should be looked up in the " "virtualenv."),
)
parser.add_argument("-i", "--ipython", action="store_true", help="use IPython shell.")
parser.add_argument("child_program", nargs="?", default=None)
parser.add_argument("child_options", nargs=argparse.REMAINDER)
# support the case when executed from a shell-bang, where all the
# parameters come in sys.argv[1] in a single string separated
# by spaces (in this case, the third parameter is what is being
# executed)
if len(sys.argv) > 1 and " " in sys.argv[1]:
real_args = sys.argv[1].split() + [sys.argv[2]]
args = parser.parse_args(real_args)
else:
args = parser.parse_args()
# validate input, parameters, and support some special options
if args.version:
print("Running 'fades' version", version)
print(" Python:", sys.version_info)
print(" System:", sys.platform)
sys.exit()
if args.verbose:
log_level = logging.DEBUG
elif args.quiet:
log_level = logging.WARNING
else:
log_level = logging.INFO
# set up logger and dump basic version info
l = logger.set_up(log_level)
l.debug("Running Python %s on %r", sys.version_info, sys.platform)
l.debug("Starting fades v. %s", version)
l.debug("Arguments: %s", args)
if args.verbose and args.quiet:
l.warning("Overriding 'quiet' option ('verbose' also requested)")
# parse file and get deps
if args.ipython:
l.debug("Adding ipython dependency because --ipython was detected")
ipython_dep = parsing.parse_manual(["ipython"])
else:
ipython_dep = {}
if args.executable:
indicated_deps = {}
else:
indicated_deps = parsing.parse_srcfile(args.child_program)
l.debug("Dependencies from source file: %s", indicated_deps)
reqfile_deps = parsing.parse_reqfile(args.requirement)
l.debug("Dependencies from requirements file: %s", reqfile_deps)
manual_deps = parsing.parse_manual(args.dependency)
l.debug("Dependencies from parameters: %s", manual_deps)
indicated_deps = _merge_deps(ipython_dep, indicated_deps, reqfile_deps, manual_deps)
# get the interpreter version requested for the child_program
interpreter, is_current = helpers.get_interpreter_version(args.python)
# start the virtualenvs manager
venvscache = cache.VEnvsCache(os.path.join(helpers.get_basedir(), "venvs.idx"))
venv_data = venvscache.get_venv(indicated_deps, interpreter)
if venv_data is None:
venv_data, installed = envbuilder.create_venv(indicated_deps, interpreter, is_current)
# store this new venv in the cache
venvscache.store(installed, venv_data, interpreter)
# run forest run!!
python_exe = "ipython" if args.ipython else "python"
python_exe = os.path.join(venv_data["env_bin_path"], python_exe)
if args.child_program is None:
l.debug("Calling the interactive Python interpreter")
p = subprocess.Popen([python_exe])
else:
if args.executable:
cmd = [os.path.join(venv_data["env_bin_path"], args.child_program)]
else:
cmd = [python_exe, args.child_program]
l.debug("Calling the child program %r with options %s", args.child_program, args.child_options)
p = subprocess.Popen(cmd + args.child_options)
def _signal_handler(signum, _):
"""Handle signals received by parent process, send them to child."""
l.debug("Redirecting signal %s to child", signum)
os.kill(p.pid, signum)
# redirect these signals
for s in REDIRECTED_SIGNALS:
signal.signal(s, _signal_handler)
# wait child to finish, end
rc = p.wait()
if rc:
l.debug("Child process not finished correctly: returncode=%d", rc)
def go(argv):
"""Make the magic happen."""
parser = argparse.ArgumentParser(
prog='PROG',
epilog=help_epilog,
usage=help_usage,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
'-V',
'--version',
action='store_true',
help="show version and info about the system, and exit")
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help="send all internal debugging lines to stderr, which may be very "
"useful to debug any problem that may arise.")
parser.add_argument(
'-q',
'--quiet',
action='store_true',
help="don't show anything (unless it has a real problem), so the "
"original script stderr is not polluted at all.")
parser.add_argument(
'-d',
'--dependency',
action='append',
help="specify dependencies through command line (this option can be "
"used multiple times)")
parser.add_argument('-r',
'--requirement',
help="indicate from which file read the dependencies")
parser.add_argument('-p',
'--python',
action='store',
help=("Specify the Python interpreter to use.\n"
" Default is: %s") % (sys.executable, ))
parser.add_argument(
'-x',
'--exec',
dest='executable',
action='store_true',
help=("Indicate that the child_program should be looked up in the "
"virtualenv."))
parser.add_argument('-i',
'--ipython',
action='store_true',
help="use IPython shell.")
parser.add_argument('--system-site-packages',
action='store_true',
default=False,
help=("Give the virtual environment access to the "
"system site-packages dir."))
parser.add_argument(
'--virtualenv-options',
action='append',
default=[],
help=(
"Extra options to be supplied to virtualenv. (this option can be "
"used multiple times)"))
parser.add_argument('--check-updates',
action='store_true',
help=("check for packages updates"))
parser.add_argument(
'--pip-options',
action='append',
default=[],
help=("Extra options to be supplied to pip. (this option can be "
"used multiple times)"))
parser.add_argument('--rm',
dest='remove',
metavar='UUID',
help=("Remove a virtualenv by UUID."))
parser.add_argument(
'--clean-unused-venvs',
action='store',
help=("This option remove venvs that haven't been used for more than "
"CLEAN_UNUSED_VENVS days. Appart from that, will compact usage "
"stats file.\n"
"When this option is present, the cleaning takes place at the "
"beginning of the execution."))
parser.add_argument('child_program', nargs='?', default=None)
parser.add_argument('child_options', nargs=argparse.REMAINDER)
# support the case when executed from a shell-bang, where all the
# parameters come in sys.argv[1] in a single string separated
# by spaces (in this case, the third parameter is what is being
# executed)
if len(sys.argv) > 1 and " " in sys.argv[1]:
real_args = sys.argv[1].split() + sys.argv[2:]
cli_args = parser.parse_args(real_args)
else:
cli_args = parser.parse_args()
# update args from config file (if needed).
args = file_options.options_from_file(cli_args)
# validate input, parameters, and support some special options
if args.version:
print("Running 'fades' version", fades.__version__)
print(" Python:", sys.version_info)
print(" System:", sys.platform)
sys.exit(0)
# set up logger and dump basic version info
l = logger.set_up(args.verbose, args.quiet)
l.debug("Running Python %s on %r", sys.version_info, sys.platform)
l.debug("Starting fades v. %s", fades.__version__)
l.debug("Arguments: %s", args)
# verify that the module is NOT being used from a virtualenv
if detect_inside_virtualenv(sys.prefix, getattr(sys, 'real_prefix', None),
getattr(sys, 'base_prefix', None)):
l.warning(
"fades is running from a virtualenv (%r), which is not supported",
sys.prefix)
if args.verbose and args.quiet:
l.warning("Overriding 'quiet' option ('verbose' also requested)")
# start the virtualenvs manager
venvscache = cache.VEnvsCache(
os.path.join(helpers.get_basedir(), 'venvs.idx'))
# start usage manager
usage_manager = envbuilder.UsageManager(
os.path.join(helpers.get_basedir(), 'usage_stats'), venvscache)
rc = 0
if args.clean_unused_venvs:
try:
max_days_to_keep = int(args.clean_unused_venvs)
usage_manager.clean_unused_venvs(max_days_to_keep)
except:
rc = 1
l.debug("CLEAN_UNUSED_VENVS must be an integer.")
raise
finally:
sys.exit(rc)
uuid = args.remove
if uuid:
venv_data = venvscache.get_venv(uuid=uuid)
if venv_data:
# remove this venv from the cache
env_path = venv_data.get('env_path')
if env_path:
envbuilder.destroy_venv(env_path, venvscache)
else:
l.warning(
"Invalid 'env_path' found in virtualenv metadata: %r. "
"Not removing virtualenv.", env_path)
else:
l.warning('No virtualenv found with uuid: %s.', uuid)
return
# parse file and get deps
if args.ipython:
l.debug("Adding ipython dependency because --ipython was detected")
ipython_dep = parsing.parse_manual(['ipython'])
else:
ipython_dep = {}
if args.executable:
indicated_deps = {}
docstring_deps = {}
else:
indicated_deps = parsing.parse_srcfile(args.child_program)
l.debug("Dependencies from source file: %s", indicated_deps)
docstring_deps = parsing.parse_docstring(args.child_program)
l.debug("Dependencies from docstrings: %s", docstring_deps)
reqfile_deps = parsing.parse_reqfile(args.requirement)
l.debug("Dependencies from requirements file: %s", reqfile_deps)
manual_deps = parsing.parse_manual(args.dependency)
l.debug("Dependencies from parameters: %s", manual_deps)
indicated_deps = _merge_deps(ipython_dep, indicated_deps, docstring_deps,
reqfile_deps, manual_deps)
# Check for packages updates
if args.check_updates:
helpers.check_pypi_updates(indicated_deps)
# get the interpreter version requested for the child_program
interpreter, is_current = helpers.get_interpreter_version(args.python)
# options
pip_options = args.pip_options # pip_options mustn't store.
options = {}
options['pyvenv_options'] = []
options['virtualenv_options'] = args.virtualenv_options
if args.system_site_packages:
options['virtualenv_options'].append("--system-site-packages")
options['pyvenv_options'] = ["--system-site-packages"]
create_venv = False
venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options)
if venv_data:
env_path = venv_data['env_path']
# A venv was found in the cache check if its valid or re-generate it.
if not os.path.exists(env_path):
l.warning(
"Missing directory (the virtualenv will be re-created): %r",
env_path)
venvscache.remove(env_path)
create_venv = True
else:
create_venv = True
if create_venv:
# Create a new venv
venv_data, installed = envbuilder.create_venv(indicated_deps,
args.python, is_current,
options, pip_options)
# store this new venv in the cache
venvscache.store(installed, venv_data, interpreter, options)
# run forest run!!
python_exe = 'ipython' if args.ipython else 'python'
python_exe = os.path.join(venv_data['env_bin_path'], python_exe)
# store usage information
usage_manager.store_usage_stat(venv_data, venvscache)
if args.child_program is None:
interactive = True
l.debug("Calling the interactive Python interpreter")
p = subprocess.Popen([python_exe])
else:
interactive = False
if args.executable:
cmd = [os.path.join(venv_data['env_bin_path'], args.child_program)]
else:
cmd = [python_exe, args.child_program]
l.debug("Calling the child program %r with options %s",
args.child_program, args.child_options)
p = subprocess.Popen(cmd + args.child_options)
def _signal_handler(signum, _):
"""Handle signals received by parent process, send them to child.
The only exception is CTRL-C, that is generated *from* the interactive
interpreter (it's a keyboard combination!), so we swallow it for the
interpreter to not see it twice.
"""
if interactive and signum == signal.SIGINT:
l.debug("Swallowing signal %s", signum)
else:
l.debug("Redirecting signal %s to child", signum)
os.kill(p.pid, signum)
# redirect the useful signals
for s in REDIRECTED_SIGNALS:
signal.signal(s, _signal_handler)
# wait child to finish, end
rc = p.wait()
if rc:
l.debug("Child process not finished correctly: returncode=%d", rc)
sys.exit(rc)
def go():
"""Make the magic happen."""
parser = argparse.ArgumentParser(prog='PROG', epilog=help_epilog,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-V', '--version', action='store_true',
help="show version and info about the system, and exit")
parser.add_argument('-v', '--verbose', action='store_true',
help="send all internal debugging lines to stderr, which may be very "
"useful to debug any problem that may arise.")
parser.add_argument('-q', '--quiet', action='store_true',
help="don't show anything (unless it has a real problem), so the "
"original script stderr is not polluted at all.")
parser.add_argument('-d', '--dependency', action='append',
help="specify dependencies through command line (this option can be "
"used multiple times)")
parser.add_argument('-r', '--requirement', action='append',
help="indicate files to read dependencies from (this option can be "
"used multiple times)")
parser.add_argument('-p', '--python', action='store',
help=("Specify the Python interpreter to use.\n"
" Default is: %s") % (sys.executable,))
parser.add_argument('-x', '--exec', dest='executable', action='store_true',
help=("Indicate that the child_program should be looked up in the "
"virtualenv."))
parser.add_argument('-i', '--ipython', action='store_true', help="use IPython shell.")
parser.add_argument('--system-site-packages', action='store_true', default=False,
help=("Give the virtual environment access to the "
"system site-packages dir."))
parser.add_argument('--virtualenv-options', action='append', default=[],
help=("Extra options to be supplied to virtualenv. (this option can be "
"used multiple times)"))
parser.add_argument('--check-updates', action='store_true',
help=("check for packages updates"))
parser.add_argument('--no-precheck-availability', action='store_true',
help=("Don't check if the packages exists in PyPI before actually try "
"to install them."))
parser.add_argument('--pip-options', action='append', default=[],
help=("Extra options to be supplied to pip. (this option can be "
"used multiple times)"))
parser.add_argument('--python-options', action='append', default=[],
help=("Extra options to be supplied to python. (this option can be "
"used multiple times)"))
parser.add_argument('--rm', dest='remove', metavar='UUID',
help=("Remove a virtualenv by UUID. See --get-venv-dir option to "
"easily find out the UUID."))
parser.add_argument('--clean-unused-venvs', action='store',
help=("This option remove venvs that haven't been used for more than "
"CLEAN_UNUSED_VENVS days. Appart from that, will compact usage "
"stats file.\n"
"When this option is present, the cleaning takes place at the "
"beginning of the execution."))
parser.add_argument('--get-venv-dir', action='store_true',
help=("Show the virtualenv base directory (which includes the "
"virtualenv UUID) and quit."))
parser.add_argument('child_program', nargs='?', default=None)
parser.add_argument('child_options', nargs=argparse.REMAINDER)
cli_args = _get_normalized_args(parser)
# update args from config file (if needed).
args = file_options.options_from_file(cli_args)
# validate input, parameters, and support some special options
if args.version:
print("Running 'fades' version", fades.__version__)
print(" Python:", sys.version_info)
print(" System:", platform.platform())
return 0
# set up logger and dump basic version info
logger = fades_logger.set_up(args.verbose, args.quiet)
logger.debug("Running Python %s on %r", sys.version_info, platform.platform())
logger.debug("Starting fades v. %s", fades.__version__)
logger.debug("Arguments: %s", args)
# verify that the module is NOT being used from a virtualenv
if detect_inside_virtualenv(sys.prefix, getattr(sys, 'real_prefix', None),
getattr(sys, 'base_prefix', None)):
logger.error(
"fades is running from inside a virtualenv (%r), which is not supported", sys.prefix)
raise FadesError("Cannot run from a virtualenv")
if args.verbose and args.quiet:
logger.warning("Overriding 'quiet' option ('verbose' also requested)")
# start the virtualenvs manager
venvscache = cache.VEnvsCache(os.path.join(helpers.get_basedir(), 'venvs.idx'))
# start usage manager
usage_manager = envbuilder.UsageManager(os.path.join(helpers.get_basedir(), 'usage_stats'),
venvscache)
if args.clean_unused_venvs:
try:
max_days_to_keep = int(args.clean_unused_venvs)
except ValueError:
logger.error("clean_unused_venvs must be an integer.")
raise FadesError('clean_unused_venvs not an integer')
usage_manager.clean_unused_venvs(max_days_to_keep)
return 0
uuid = args.remove
if uuid:
venv_data = venvscache.get_venv(uuid=uuid)
if venv_data:
# remove this venv from the cache
env_path = venv_data.get('env_path')
if env_path:
envbuilder.destroy_venv(env_path, venvscache)
else:
logger.warning("Invalid 'env_path' found in virtualenv metadata: %r. "
"Not removing virtualenv.", env_path)
else:
logger.warning('No virtualenv found with uuid: %s.', uuid)
return 0
# decided which the child program really is
analyzable_child_program, child_program = decide_child_program(
args.executable, args.child_program)
# Group and merge dependencies
indicated_deps = consolidate_dependencies(args.ipython,
analyzable_child_program,
args.requirement,
args.dependency)
# Check for packages updates
if args.check_updates:
helpers.check_pypi_updates(indicated_deps)
# get the interpreter version requested for the child_program
interpreter, is_current = helpers.get_interpreter_version(args.python)
# options
pip_options = args.pip_options # pip_options mustn't store.
python_options = args.python_options
options = {}
options['pyvenv_options'] = []
options['virtualenv_options'] = args.virtualenv_options
if args.system_site_packages:
options['virtualenv_options'].append("--system-site-packages")
options['pyvenv_options'] = ["--system-site-packages"]
create_venv = False
venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options)
if venv_data:
env_path = venv_data['env_path']
# A venv was found in the cache check if its valid or re-generate it.
if not os.path.exists(env_path):
logger.warning("Missing directory (the virtualenv will be re-created): %r", env_path)
venvscache.remove(env_path)
create_venv = True
else:
create_venv = True
if create_venv:
# Check if the requested packages exists in pypi.
if not args.no_precheck_availability and indicated_deps.get('pypi'):
logger.info("Checking the availabilty of dependencies in PyPI. "
"You can use '--no-precheck-availability' to avoid it.")
if not helpers.check_pypi_exists(indicated_deps):
logger.error("An indicated dependency doesn't exist. Exiting")
raise FadesError("Required dependency does not exist")
# Create a new venv
venv_data, installed = envbuilder.create_venv(indicated_deps, args.python, is_current,
options, pip_options)
# store this new venv in the cache
venvscache.store(installed, venv_data, interpreter, options)
if args.get_venv_dir:
# all it was requested is the virtualenv's path, show it and quit (don't run anything)
print(venv_data['env_path'])
return 0
# run forest run!!
python_exe = 'ipython' if args.ipython else 'python'
python_exe = os.path.join(venv_data['env_bin_path'], python_exe)
# add the virtualenv /bin path to the child PATH.
environ_path = venv_data['env_bin_path']
if 'PATH' in os.environ:
environ_path += os.pathsep + os.environ['PATH']
os.environ['PATH'] = environ_path
# store usage information
usage_manager.store_usage_stat(venv_data, venvscache)
if child_program is None:
interactive = True
logger.debug(
"Calling the interactive Python interpreter with arguments %r", python_options)
cmd = [python_exe] + python_options
p = subprocess.Popen(cmd)
else:
interactive = False
if args.executable:
cmd = [os.path.join(venv_data['env_bin_path'], child_program)]
logger.debug("Calling child program %r with options %s",
child_program, args.child_options)
else:
cmd = [python_exe] + python_options + [child_program]
logger.debug(
"Calling Python interpreter with arguments %s to execute the child program"
" %r with options %s", python_options, child_program, args.child_options)
try:
p = subprocess.Popen(cmd + args.child_options)
except FileNotFoundError:
logger.error("Command not found: %s", child_program)
raise FadesError("Command not found")
def _signal_handler(signum, _):
"""Handle signals received by parent process, send them to child.
The only exception is CTRL-C, that is generated *from* the interactive
interpreter (it's a keyboard combination!), so we swallow it for the
interpreter to not see it twice.
"""
if interactive and signum == signal.SIGINT:
logger.debug("Swallowing signal %s", signum)
else:
logger.debug("Redirecting signal %s to child", signum)
os.kill(p.pid, signum)
# redirect the useful signals
for s in REDIRECTED_SIGNALS:
signal.signal(s, _signal_handler)
# wait child to finish, end
rc = p.wait()
if rc:
logger.debug("Child process not finished correctly: returncode=%d", rc)
return rc