def main():
""" Execute the test.
"""
@contextmanager
def tmpdir():
""" Enter a self-deleting temporary directory. """
cwd = getcwd()
tmp = mkdtemp()
try:
chdir(tmp)
yield tmp
finally:
rmtree(tmp)
chdir(cwd)
return
template = dirname(dirname(abspath(__file__)))
defaults = load(open(join(template, "cookiecutter.json")))
with tmpdir():
cookiecutter(template, no_input=True)
chdir(defaults["project_slug"])
create("venv", with_pip=True)
path = join("venv", "bin")
pip = which("pip", path=path) or "pip" # Travis CI workaround
install = "{:s} install .".format(pip)
for req in (join(root, "requirements.txt") for root in (".", "test")):
# Add each requirements file to the install.
install = " ".join((install, "--requirement={:s}".format(req)))
pytest = which("pytest", path=path) or "pytest" # Travis CI workaround
test = "{:s} --verbose tests/".format(pytest)
check_call(split(test))
return 0
def create(self, name, *, system_site_packages=False, symlinks=False,
with_pip=True):
"""Create a virtual environment in $VIRTUALENV_HOME with python3's ``venv``.
Parameters
----------
name : str
Virtual environment name
system_site_packages : bool
If True, the system (global) site-packages dir is available to
created environments.
symlinks : bool
If True, attempt to symlink rather than copy files into virtual
environment.
with_pip : bool
If True, ensure pip is installed in the virtual environment. (Default is True)
"""
# NOTE: clear=True is the same as delete then create.
# NOTE: upgrade=True is its own method
env_path = os.path.join(self.venvdir, name)
venv.create(
env_path,
system_site_packages=system_site_packages, symlinks=symlinks,
with_pip=with_pip)
events.vox_on_create.fire(name)
def _setup_venv(self):
from stoqlib.lib.osutils import get_application_dir
import venv
stoqdir = get_application_dir("stoq")
env_dir = os.path.join(stoqdir, 'venv')
if not os.path.exists(env_dir):
log.info('creating venv at %s', env_dir)
if platform.system() == 'Windows':
# On windows, pip will be included as an egg
venv.create(env_dir, system_site_packages=True)
else:
venv.create(env_dir, system_site_packages=True, with_pip=True)
log.info('creating venv done')
# This is exactly what activate_this.py does
old_os_path = os.environ.get('PATH', '')
os.environ['PATH'] = os.path.join(env_dir, 'bin') + os.pathsep + old_os_path
if sys.platform == 'win32':
site_packages = os.path.join(env_dir, 'Lib', 'site-packages')
else:
site_packages = os.path.join(env_dir, 'lib', 'python%s' % sys.version[:3],
'site-packages')
prev_sys_path = list(sys.path)
import site
site.addsitedir(site_packages)
sys.real_prefix = sys.prefix
sys.prefix = env_dir
# Move the added items to the front of the path:
new_sys_path = []
for item in list(sys.path):
if item not in prev_sys_path:
new_sys_path.append(item)
sys.path.remove(item)
sys.path[:0] = new_sys_path
def install(package: str, upgrade: bool):
"""Install a package"""
click.echo('Installing {}'.format(package))
if package_dir(package).exists() and not upgrade:
click.echo('Application {} already installed, to upgrade use --upgrade'.format(package))
exit(1)
# Create venv
venv_dir = package_dir(package)
venv.create(str(venv_dir), clear=True, with_pip=True)
# Ask pip to install the package
try:
call_pip(package, ['install', package])
except subprocess.CalledProcessError as err:
click.echo(err.output)
# TODO remove package so far if it fails here
exit(1)
# Install entry-point launchers
for entry_point_group, entry_point in get_entry_points(package):
python_path = package_dir(package) / 'bin' / 'python' # TODO ask setuptools for this info?
launcher = create_launcher_text(
package=package,
version=entry_point.dist.version,
entry_point=entry_point.name,
entry_point_group=entry_point_group,
python_path=str(python_path)
)
launcher_path = config['bindir'] / entry_point.name
install_launcher(launcher_path, launcher)
def create(self, name, *, system_site_packages=False, symlinks=False,
with_pip=True):
"""Create a virtual environment in $VIRTUALENV_HOME with python3's ``venv``.
Parameters
----------
name : str
Virtual environment name
system_site_packages : bool
If True, the system (global) site-packages dir is available to
created environments.
symlinks : bool
If True, attempt to symlink rather than copy files into virtual
environment.
with_pip : bool
If True, ensure pip is installed in the virtual environment. (Default is True)
"""
# NOTE: clear=True is the same as delete then create.
# NOTE: upgrade=True is its own method
if isinstance(name, PathLike):
env_path = fspath(name)
else:
env_path = os.path.join(self.venvdir, name)
if not self._check_reserved(env_path):
raise ValueError("venv can't contain reserved names ({})".format(', '.join(_subdir_names())))
venv.create(
env_path,
system_site_packages=system_site_packages, symlinks=symlinks,
with_pip=with_pip)
events.vox_on_create.fire(name=name)
def ensure_venv(self):
"""
Find the local venv.
If it does not exist, create it and install requirements.
"""
if not os.path.exists(self.venv_path):
os.mkdir(self.venv_path)
venv.create(self.venv_path, with_pip=True)
self.install_requirements()
def do_pyvenv(path, system_site_packages):
try:
import venv
except ImportError:
error("Standard Python 'venv' module not found", ERROR_EXCEPTION)
# In Python >= 3.4 venv.create() has a new parameter with_pip=False
# that allows to automatically install setuptools and pip with the module
# ensurepip. Unfortunately, we cannot use this parameter and have to
# bootstrap these packages ourselves, since some distributions of CPython
# on Ubuntu don't include ensurepip.
venv.create(path, system_site_packages=system_site_packages)
def _create_virtualenv(self):
"""Create virtualenv to install packages"""
Pip2Pkgbuild.log.info("Preparing virtualenv")
if os.path.exists(VENV_DIR):
return
venv.create(VENV_DIR,
with_pip=True)
# upgrade pip
Pip2Pkgbuild.log.info('checking for pip upgrade')
self._exec(subprocess.check_call, [VENV_PIP, 'install', '-U', 'pip'])
def create_env(name):
"""Create a virtual environment in $VIRTUALENV_HOME with python3's ``venv``.
Parameters
----------
name : str
Virtual environment name
"""
env_path = os.path.join(builtins.__xonsh_env__['VIRTUALENV_HOME'], name)
print('Creating environment...')
venv.create(env_path, with_pip=True)
msg = 'Environment {0!r} created. Activate it with "vox activate {0}".\n'
print(msg.format(name))
def package_source_tgz():
venv.create('build/pkgenv', clear=True, with_pip=True)
print_and_do('./build/pkgenv/bin/pip install -r requirements.txt')
print_and_do('./build/pkgenv/bin/pip freeze > build/requirements.freeze')
app_version = MoneyGuru.VERSION
name = 'moneyguru-src-{}.tar'.format(app_version)
dest = op.join('build', name)
print_and_do('git archive -o {} HEAD'.format(dest))
print("Adding requirements.freeze and wrapping up")
os.chdir('build')
print_and_do('tar -rf {} requirements.freeze'.format(name))
print_and_do('gzip {}'.format(name))
os.chdir('..')
def install(self):
"""Installer"""
options = self.options
path = [p for p in sys.path if 'parts' not in p]
del sys.modules['site']
sys.path[:] = path
env = os.environ.copy()
env['PYTHONPATH'] = ':'.join(path)
key = 'tox-install-dir'
if key in self.buildout['buildout']:
install_dir = self.buildout['buildout'][key]
elif key in self.options:
install_dir = self.options[key]
else:
install_dir = join(self.buildout['buildout']['parts-directory'],
self.name)
bin_bir = join(install_dir, 'bin')
tox = join(bin_bir, 'tox')
if not os.path.isfile(tox):
if sys.version_info[:2] >= (3,4):
import venv
venv.create(install_dir, with_pip=True)
del env['PYTHONPATH']
subprocess.call([join(bin_bir, 'pip'), 'install', 'tox'], env=env)
else:
import virtualenv
subprocess.call([sys.executable, virtualenv.__file__[:-1],
'-q', '--distribute', install_dir], env=env)
del env['PYTHONPATH']
subprocess.call([join(bin_bir, 'easy_install'), 'tox'],
env=env)
from zc.recipe.egg import Scripts
options['eggs'] = 'virtualenv'
options['scripts'] = 'tox'
options['entry-points'] = 'tox=os:execve'
options['arguments'] = (
'%(tox)r, [%(tox)r] + sys.argv[1:], os.environ'
) % dict(tox=tox)
options['initialization'] = '\n'.join([
'import os',
"os.environ['PYTHONPATH'] = ''",
])
script = Scripts(self.buildout, self.name, self.options)
script.install()
return tuple()
def create_env(name):
"""Create a virtual environment in $VIRTUALENV_HOME with python3's ``venv``.
Parameters
----------
name : str
Virtual environment name
"""
env_path = join(builtins.__xonsh_env__['VIRTUALENV_HOME'], name)
print('Creating environment...')
venv.create(env_path, with_pip=True)
print('Environment "%s" created. Activate it with "vox activate %s".\n' % (name, name))
def test_overwrite_existing(self):
"""
Test creating environment in an existing directory.
"""
self.create_contents(self.ENV_SUBDIRS, 'foo')
venv.create(self.env_dir)
for subdirs in self.ENV_SUBDIRS:
fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
self.assertTrue(os.path.exists(fn))
with open(fn, 'rb') as f:
self.assertEqual(f.read(), b'Still here?')
builder = venv.EnvBuilder(clear=True)
builder.create(self.env_dir)
for subdirs in self.ENV_SUBDIRS:
fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
self.assertFalse(os.path.exists(fn))
def virtualenv(location, virtualenv, cmds=None):
""" create a virtualenv with the given dependencies in the target location
{
"location": "target/path",
"virtualenv": ["dep1", "dep2"]
}
"""
if not os.path.exists(location):
venv.create(location, with_pip=True)
vpython = os.path.join(location, 'bin', 'python')
pip_install = [vpython, '-m', 'pip', 'install', '--upgrade']
run(pip_install + ['pip'])
dependencies = virtualenv
run(pip_install + dependencies)
_exec_cmds(location, cmds)
def create_virtualenv(env_dir):
notify('\nCreating virtualenv')
res = venv.create(env_dir, system_site_packages=False, with_pip=True)
if not res:
proc = subprocess.run(
['pip', 'install', '--upgrade', 'pip'],
env=os.environ.copy())
if proc.returncode:
sys.exit(proc.returncode)
status_ok('Done')
return res
def make_virtualenv(env):
""" Create a virtualenv """
if sys.version_info.major == 2:
from urllib import urlretrieve
if find_executable("virtualenv") is not None:
cmd = ["virtualenv"] + [env]
subprocess.check_call(cmd)
else:
# Otherwise, download virtualenv from pypi
path = urlretrieve(VENV_URL)[0]
subprocess.check_call(["tar", "xzf", path])
subprocess.check_call(
[sys.executable, "virtualenv-%s/virtualenv.py" % VENV_VERSION, env]
)
os.unlink(path)
shutil.rmtree("virtualenv-%s" % VENV_VERSION)
else:
import venv
venv.create(env, with_pip=True)
def upgrade(self, name, *, symlinks=False, with_pip=True):
"""Create a virtual environment in $VIRTUALENV_HOME with python3's ``venv``.
WARNING: If a virtual environment was created with symlinks or without PIP, you must
specify these options again on upgrade.
Parameters
----------
name : str
Virtual environment name
symlinks : bool
If True, attempt to symlink rather than copy files into virtual
environment.
with_pip : bool
If True, ensure pip is installed in the virtual environment.
"""
# venv doesn't reload this, so we have to do it ourselves.
# Is there a bug for this in Python? There should be.
env_path, bin_path = self[name]
cfgfile = os.path.join(env_path, 'pyvenv.cfg')
cfgops = {}
with open(cfgfile) as cfgfile:
for l in cfgfile:
l = l.strip()
if '=' not in l:
continue
k, v = l.split('=', 1)
cfgops[k.strip()] = v.strip()
flags = {
'system_site_packages': cfgops['include-system-site-packages'] == 'true',
'symlinks': symlinks,
'with_pip': with_pip,
}
# END things we shouldn't be doing.
# Ok, do what we came here to do.
venv.create(env_path, upgrade=True, **flags)
def setup_venv(c, venv_dir, requirements):
venv.create(venv_dir, with_pip='True')
pip = os.path.join(venv_dir, 'bin', 'pip')
c.run(f'{pip} install -r {requirements}')
def main():
# Get info from user
project_name = input("Project name (example: my_awesome_app): ")
project_desc = input("Short description: ")
target_loc = Path(input("Project location (example: /home/john/Documents): "))
app_name = input("App name (optional): ") # name for your source dir
app_name = project_name if not app_name else app_name
if not target_loc.exists():
sys.exit(f"Target location not found: {target_loc}")
# Sanitize app name
delim = re.compile("[ -]")
app_name = delim.sub("_", app_name)
# Directories
proj_dir = Path(f"{target_loc}/{project_name}").resolve()
src_dir = proj_dir/app_name
test_dir = proj_dir/"test"
# Files
readme = proj_dir/"README.md"
setup_py = proj_dir/"setup.py"
app_py = src_dir/f"{app_name}.py"
test_py = test_dir/f"test_{delim.sub('_', project_name)}.py"
init_py = "__init__.py"
# Create directories and files
proj_dir.mkdir(exist_ok=False, parents=True)
src_dir.mkdir(exist_ok=False, parents=True)
test_dir.mkdir(exist_ok=False, parents=True)
readme.touch()
setup_py.touch()
app_py.touch()
test_py.touch()
(src_dir/init_py).touch()
(test_dir/init_py).touch()
# Update file content
with readme.open("w") as fh:
fh.write(f"# {project_name}\n\n{project_desc}\n")
with setup_py.open("w") as fh:
fh.write(file_content.setup(app_name, project_desc))
with test_py.open("w") as fh:
fh.write(file_content.testpy(app_name))
# Create virtual environment
v = input("Do you want to create a virtual environment? (y/n)")
if v.lower() != "y":
print("No virtual environment will be created.")
else:
try:
import venv
venv.create(proj_dir/"env", with_pip=True, clear=True)
print("Activate the environemnt by entering 'source ./env/bin/activate'")
print("You can upgrade pip by executing 'python -m pip install --upgrade pip'")
print("Recommended updates: 'pip install --upgrade setuptools wheel'")
print("Recommended package: 'sudo apt install python3-dev'")
except ImportError:
print("Unable to import venv, do you have python3-venv installed?")
except Exception as e:
print(f"Unable to create a virtual environment: {e}")
# Initialize a git repo
g = input("Do you want to initialize a local git repository? (y/n)")
if g.lower() != "y":
print("No git repo will be created.")
else:
try:
subprocess.run(["git", "init", str(proj_dir)])
print("Don't forget to update git config!")
except Exception as e:
print(f"Unable to initialize a git repository: {e}")
# Create .gitignore
ign = proj_dir/".gitignore"
ign.touch(exist_ok=True)
with ign.open("w") as fh:
fh.write(file_content.GITIGNORE)
def spec_venv(root, target=None):
root_only(root)
venv.create('spec-venv', with_pip=True, clear=True)
pip = 'spec-venv\Scripts\pip' if platform.system(
) == 'Windows' else 'spec-venv/bin/pip'
shell('%s install --quiet -r requirements.txt' % pip)
def run_stubtest(dist: Path, *, verbose: bool = False) -> bool:
with open(dist / "METADATA.toml") as f:
metadata = dict(tomli.loads(f.read()))
print(f"{dist.name}... ", end="")
stubtest_meta = metadata.get("tool", {}).get("stubtest", {})
if stubtest_meta.get("skip", False):
print(colored("skipping", "yellow"))
return True
with tempfile.TemporaryDirectory() as tmp:
venv_dir = Path(tmp)
venv.create(venv_dir, with_pip=True, clear=True)
pip_exe = str(venv_dir / "bin" / "pip")
python_exe = str(venv_dir / "bin" / "python")
dist_version = metadata["version"]
assert isinstance(dist_version, str)
dist_req = f"{dist.name}=={dist_version}"
# If @tests/requirements-stubtest.txt exists, run "pip install" on it.
req_path = dist / "@tests" / "requirements-stubtest.txt"
if req_path.exists():
try:
pip_cmd = [pip_exe, "install", "-r", str(req_path)]
subprocess.run(pip_cmd, check=True, capture_output=True)
except subprocess.CalledProcessError as e:
print_command_failure("Failed to install requirements", e)
return False
# We need stubtest to be able to import the package, so install mypy into the venv
# Hopefully mypy continues to not need too many dependencies
# TODO: Maybe find a way to cache these in CI
dists_to_install = [dist_req, get_mypy_req()]
dists_to_install.extend(metadata.get("requires", []))
pip_cmd = [pip_exe, "install"] + dists_to_install
try:
subprocess.run(pip_cmd, check=True, capture_output=True)
except subprocess.CalledProcessError as e:
print_command_failure("Failed to install", e)
return False
ignore_missing_stub = ["--ignore-missing-stub"] if stubtest_meta.get(
"ignore_missing_stub", True) else []
packages_to_check = [
d.name for d in dist.iterdir()
if d.is_dir() and d.name.isidentifier()
]
modules_to_check = [
d.stem for d in dist.iterdir()
if d.is_file() and d.suffix == ".pyi"
]
stubtest_cmd = [
python_exe,
"-m",
"mypy.stubtest",
# Use --custom-typeshed-dir in case we make linked changes to stdlib or _typeshed
"--custom-typeshed-dir",
str(dist.parent.parent),
*ignore_missing_stub,
*packages_to_check,
*modules_to_check,
]
allowlist_path = dist / "@tests/stubtest_allowlist.txt"
if allowlist_path.exists():
stubtest_cmd.extend(["--allowlist", str(allowlist_path)])
try:
subprocess.run(stubtest_cmd,
env={
"MYPYPATH": str(dist),
"MYPY_FORCE_COLOR": "1"
},
check=True,
capture_output=True)
except subprocess.CalledProcessError as e:
print_error("fail")
print_commands(dist, pip_cmd, stubtest_cmd)
print_command_output(e)
print("Ran with the following environment:", file=sys.stderr)
ret = subprocess.run([pip_exe, "freeze"], capture_output=True)
print_command_output(ret)
if allowlist_path.exists():
print(
f'To fix "unused allowlist" errors, remove the corresponding entries from {allowlist_path}',
file=sys.stderr)
print(file=sys.stderr)
else:
print(
f"Re-running stubtest with --generate-allowlist.\nAdd the following to {allowlist_path}:",
file=sys.stderr)
ret = subprocess.run(stubtest_cmd + ["--generate-allowlist"],
env={"MYPYPATH": str(dist)},
capture_output=True)
print_command_output(ret)
return False
else:
print_success_msg()
if verbose:
print_commands(dist, pip_cmd, stubtest_cmd)
return True
def create_venv():
path = pathlib.Path().joinpath(pathlib.Path().absolute(), '/lime' )
venv.create(pathlib.Path().absolute())
def create_venv(venv_path):
venv.create(venv_path, clear=True, with_pip=True)
def run(params):
env_dir = pathlib.Path(os.path.dirname(os.path.realpath(__file__))) / 'env'
if not os.path.exists(env_dir):
import venv
env_dir.mkdir(parents=False, exist_ok=True)
venv.create(env_dir, system_site_packages=True)
exec_path = env_dir / 'Scripts' / 'python.exe'
requirement_dir = pathlib.Path(os.path.dirname(os.path.realpath(__file__)))
file1 = open(requirement_dir / 'requirements.txt', 'r')
Lines = file1.readlines()
Lines = {l.strip() for l in Lines}
subprocess.check_call(
[str(exec_path), '-m', 'pip', 'install', *Lines])
# Check if input image exists
inputImagePath_ = params['inputImagePath']
if not os.path.exists(inputImagePath_):
raise ValueError('Error: {inputImagePath_} does not exist')
# Get Z count and T count
z_count, t_count = [int(params[f'{s}Count']) for s in ['Z', 'T']]
# Get the path of the folder that contains this python script
parentFolder = str(Path(__file__).parent)
# Get the path of python executable in the virtual environment
pythonExec_ = parentFolder + '\\env\\Scripts\\python.exe'
# Get the path of the python script to run under the virtual environment
scrptPath_ = parentFolder + '\\Data\\Cellpose_venv.py'
# Get input, output, and parameters as strings
zCount_ = str(z_count)
tCount_ = str(t_count)
diameter_ = params['diameter']
model_type_ = params['modelType']
conf_map_path_ = params['confMapPath']
mask_path_ = params['maskPath']
cellprob_threshold_ = params['cellThreshold']
flow_threshold_ = params['flowThreshold']
# Display input, output, and parameters
print('------------------------------------------')
print(' Cellpose Python Recipe')
print('------------------------------------------')
print(f' pythonExec_= {pythonExec_}')
print(f' scrptPath_= {scrptPath_}')
print(f' inputImagePath_= {inputImagePath_}')
print(f' zCount_= {zCount_}')
print(f' tCount_= {tCount_}')
print(f' diameter_= {diameter_}')
print(f' model_type_= {model_type_}')
print(f' conf_map_path_= {conf_map_path_}')
print(f' mask_path_= {mask_path_}')
print(f'cellprob_threshold_= {cellprob_threshold_}')
print(f' flow_threshold_= {flow_threshold_}')
print('------------------------------------------')
# Run the script under the virtual environment
proc = subprocess.Popen(
[pythonExec_, scrptPath_, inputImagePath_, zCount_, tCount_,
diameter_, model_type_, conf_map_path_, mask_path_,
cellprob_threshold_, flow_threshold_],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
# Write sub process outputs
for line in proc.stdout:
print(line.rstrip())
proc.wait()
def do_pyvenv(path, system_site_packages):
try:
import venv
except ImportError:
error("Standard Python 'venv' module not found", ERROR_EXCEPTION)
venv.create(path, system_site_packages=system_site_packages)
def run(venv_name='venv', requirements_file='requirements.txt'):
# Detect flags intended for autovenv and remove them from sys.argv
flags = '--no-autovenv --remove-venv'
flags = {f:not sys.argv.remove(f) if f in sys.argv else False for f in flags.split()}
# Do nothing if this is the second stage run where an environment
# variable has already been set, or if the user disabled autovenv
if flags['--no-autovenv'] or 'AUTOVENV_IS_RUNNING' in os.environ:
return
os.environ['AUTOVENV_IS_RUNNING'] = __version__
# Find the __main__ module which called this function and look for
# or create a virtualenv in that module's containing directory
caller = [f[1] for f in inspect.stack() if f[0].f_locals.get('__name__') == '__main__'][0]
calling_script = os.path.realpath(caller)
calling_script_dir = os.path.dirname(calling_script)
venv_dir = os.path.join(calling_script_dir, venv_name)
venv_python = os.path.join(venv_dir, 'bin', 'python')
# Show the disclaimer
log('*', "Autovenv is bootstrapping a virtual environment using " + requirements_file,
"\n --no-autovenv Don't auto activate or install a virtualenv",
"\n --remove-venv Remove old virtualenv so a fresh one can be installed",
"\n")
log('+', 'Running', calling_script, '\n ', len('Running '+calling_script)*'-')
# Remove the bad virtualenv
if flags['--remove-venv']:
shutil.rmtree(venv_dir, ignore_errors=True)
log('i', 'Removed existing virtualenv', error=1)
# Handle the case of the nonexistant virtualenv by creating it
if not os.path.isfile(venv_python):
log('i', 'No virtualenv found')
# Run with working directory of the calling script
original_working_dir = os.getcwd()
os.chdir(calling_script_dir)
log('i', 'Changed working directory to', calling_script_dir)
# Create the virtualenv
venv.create(venv_dir, with_pip=True)
log('+', 'Created virtualenv', venv_dir)
# Call pip to install the requirements file
log('i', 'Installing required packages (this may take some time)')
if subprocess.call([venv_python, "-m", "pip", "install", "-r", requirements_file]):
# A nonzero return code means something went wrong
log('-', 'Installing required packages failed!')
shutil.rmtree(venv_dir)
log('i', 'Removed the incomplete or broken virtualenv in', venv_dir)
log('darn', 'sucks. is your', requirements_file, 'file any good?');
log('+google?', 'do you maybe need a "-dev" package or some compiler?', error=1);
log('+', 'Installation was successful!')
# Return to the original working directory
os.chdir(original_working_dir)
log('i', 'Restored working directory to', os.getcwd())
# If it exists, we assume the whole thing works; no warranties
log('i', 'Found virtualenv', venv_dir)
# Pass the python path twice to convince its silly little brain
os.execl(venv_python, venv_python, calling_script, *sys.argv[1:])
"""Python virtual environments.
See also <http://www.python.org/dev/peps/pep-0405/>.
"""
import venv
venv.create('pyvenv')
#import markdown
#print(markdown.markdown('**hello**'))
def create_python(self):
print("Creating a python specific project")
self.create_directories()
pathVenv = os.path.join(self._path, self._name + "Env")
print("Creating virtual enviroment...")
venv.create(pathVenv, with_pip=True)
def create(self, clear=False):
venv.create(self.dst, clear=clear, with_pip=True)
if sys.prefix == sys.base_prefix:
# Check if virtual environment is available
venvPython = os.path.join(os.getcwd(), venvDir, binDir, python)
if not os.path.exists(venvPython):
# Installing virtual environment
# https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/
try:
import venv
except:
pip3InstallModule("virtualenv")
#subprocess.Popen([python, "-m", "venv", venvDir])
print("Setting up environment ...")
try:
if not "venv" in sys.modules:
import venv
venv.create(env_dir=venvDir, with_pip=True)
except:
pass
# Run main.py
if platform.system() == "Windows":
if python.endswith(".exe"):
python = python[:-4]
# Create a .bat for application shortcut
shortcutSh = os.path.join(os.getcwd(), "UniqueBibleApp.bat")
with open(shortcutSh, "w") as fileObj:
fileObj.write('{0} "{1}"'.format(python, thisFile))
# Activate virtual environment
activator = os.path.join(os.getcwd(), venvDir, binDir, "activate")
# Run main.py
mainPy = "main.py {0}".format(
def make_venv_34_plus(env_dir):
"""Create a virtual environment on 3.4+."""
import venv
venv.create(env_dir, with_pip=True, clear=True)
return True
def make_virtualenv_and_rerun(location: Path) -> None:
"""
This is a hail mary, when we cannot install to the already existing virtualenv because
someone created it as root or another user than the user running this process
Creates a .venv dir in project root (which is gitignored already)
if found SC will restart automatically using tthe environment in .venv
"""
location = location.resolve()
current_interpreter = Path(sys.executable).resolve()
result = 0 # Ok
if str(location) == str(sys.prefix):
if IS_VIRTUALENV:
logger.info(
f"Unable to install to the existing virtual environment located at {sys.prefix}"
)
logger.info(
"Please check the permissions, and that it does not include global site packages"
)
result = 126 # Command invoked cannot execute
else:
if not location.is_dir():
logger.info(
f"Because of the above errors, we will try creating a new virtualenvironment in {location}"
)
try:
import venv
venv.create(location,
system_site_packages=False,
clear=True,
symlinks=os.name != "nt",
with_pip=True)
logger.info(
f"Created new virtualenvironment in {location} using venv module!"
)
except:
if check_installed("virtualenv"):
result = subprocess_call([
f"{sys.executable}", "-m", "virtualenv", "-p",
f"{sys.executable}", f"{location}"
])
if result != 0: # Not Ok
logger.info(
"Due to the above error, we cannot continue! Exiting"
)
else:
logger.info(
f"Created new virtualenvironment in {location}")
else:
logger.info(
"virtualenv module not found, getting a portable one to use temporarily"
)
tfd = download_to_temp_file(
"https://bootstrap.pypa.io/virtualenv.pyz")
result = subprocess_call([
f"{sys.executable}", f"{tfd.name}", "-p",
f"{sys.executable}", f"{location}"
])
os.unlink(tfd.name)
if result != 0: # Not Ok
logger.info(
"Due to the above error, we cannot continue! Exiting"
)
else:
logger.info(
f"Created new virtualenvironment in {location}")
if location.is_dir() and result == 0: # Ok
locations_to_check = []
# append the bin/python.ext to the new venv path
check = location
for part in current_interpreter.parts[-2:]:
if sys.platform == "win32" and part == "tools":
part = "Scripts"
check /= part
locations_to_check.append(check)
locations_to_check.append(location / "bin" /
current_interpreter.parts[-1])
locations_to_check.append(location / "Scripts" /
current_interpreter.parts[-1])
locations_to_check.extend(x for x in location.rglob("*python3.?")
if x.is_file())
locations_to_check.extend(x for x in location.rglob("*python3")
if x.is_file())
locations_to_check.extend(x for x in location.rglob("*python")
if x.is_file())
for place in locations_to_check:
if place.is_file() and place.stat().st_mode & os.X_OK:
# add original arguments to this re-call
new_argv = [str(place)] + sys.argv
logger.info(f"Restarting SickChill with {new_argv}")
return os.execvp(new_argv[0], new_argv)
logger.info(
f"Something weird happend when creating the virtualenv, Could not find the bin dir or executable in {location}. Exiting"
)
os._exit(result)
def setup(self, reinstall=False):
"""
Begins setup of the app.
"""
loc = self.configuration["install_loc"]
cprint("deploy.py: beginning app setup", "green")
cprint("deploy.py: obtaining sudo access", "green")
cprint("deploy.py: please enter your password if appropriate below.", "cyan")
# Attempt sudo access
status = call(["sudo", "true"])
if status != 0:
cprint("deploy.py: error: unable to access root", "red")
cprint("deploy.py: this may hamper your ability to run some things later on", "red")
has_root = False
else:
has_root = True
# continue
if not reinstall:
cprint("deploy.py: creating app directory", "green")
try:
os.makedirs(loc)
except PermissionError:
if not has_root:
cprint("deploy.py: unable to make project directory. exiting.", "red")
sys.exit(1)
call("sudo mkdir -v {}".format(loc))
call("sudo chown -v {}:{} {}".format(os.geteuid(), os.getegid(), loc))
except FileExistsError:
# oh well
call("sudo chown -v {}:{} {}".format(os.geteuid(), os.getegid(), loc))
cprint("deploy.py: changing working directory...", "green")
original_cwd = os.getcwd()
# change cwd
os.chdir(loc)
cprint("deploy.py: new cwd is {}".format(loc), "cyan")
cprint("deploy.py: getting app sources", "green")
if self.configuration["project_download_type"] == "git":
if not reinstall:
call("git init")
call("git remote add deploy {}".format(self.configuration["git_url"]))
call("git fetch deploy")
res = call("git checkout deploy/{}".format(self.configuration["git_branch"]))
if res:
cprint("deploy.py: error: unable to clone project files. exiting.", "red")
sys.exit(1)
cprint("deploy.py: app sources downloaded, creating virtual environment")
venv.create("./.venv/", with_pip=True)
cprint("deploy.py: pretending we're in that virtualenv", "green")
cprint("deploy.py: this may break things", "yellow")
# Ungodly os.path.join.
sys.path.insert(0, os.path.join("lib", "python" + '.'.join(map(str, sys.version_info[0:2])), "site-packages"))
sys.executable = os.path.abspath("./.venv/bin/python")
cprint("deploy.py: overriding $PATH", "yellow")
os.environ["PATH"] = os.path.abspath("./.venv/bin/") + ":" + os.environ["PATH"]
cprint("deploy.py: copying deployment files to app directory...")
shutil.copy(os.path.join(original_cwd, "deploy.py"), os.path.join(os.getcwd(), "deploy.py"))
if not os.path.exists(os.path.join(os.getcwd(), "deployconf.py")):
shutil.copy(os.path.join(original_cwd, "deployconf.py"), os.path.join(os.getcwd(), "deployconf.py"))
cprint("deploy.py: switching to new interpreter...", "green")
cprint("deploy.py: this will attempt to pick up where we left off.")
executable = os.path.join(os.getcwd(), ".venv/bin/python")
print()
print()
subprocess.original_call([executable, "deploy.py", "--run-hooks"] + (["--reinstall"] if reinstall else []))
def make_venv(self, folder):
venv.create(Path(self.wd, folder))
def run_stubtest(dist: Path) -> bool:
with open(dist / "METADATA.toml") as f:
metadata = dict(tomli.loads(f.read()))
if not run_stubtest_for(metadata, dist):
print(f"Skipping stubtest for {dist.name}\n\n")
return True
with tempfile.TemporaryDirectory() as tmp:
venv_dir = Path(tmp)
venv.create(venv_dir, with_pip=True, clear=True)
pip_exe = str(venv_dir / "bin" / "pip")
python_exe = str(venv_dir / "bin" / "python")
dist_version = metadata["version"]
assert isinstance(dist_version, str)
dist_req = f"{dist.name}=={dist_version}"
# If @tests/requirements-stubtest.txt exists, run "pip install" on it.
req_path = dist / "@tests" / "requirements-stubtest.txt"
if req_path.exists():
try:
pip_cmd = [pip_exe, "install", "-r", str(req_path)]
subprocess.run(pip_cmd, check=True, capture_output=True)
except subprocess.CalledProcessError as e:
print(f"Failed to install requirements for {dist.name}",
file=sys.stderr)
print(e.stdout.decode(), file=sys.stderr)
print(e.stderr.decode(), file=sys.stderr)
return False
# We need stubtest to be able to import the package, so install mypy into the venv
# Hopefully mypy continues to not need too many dependencies
# TODO: Maybe find a way to cache these in CI
dists_to_install = [dist_req, get_mypy_req()]
dists_to_install.extend(metadata.get("requires", []))
pip_cmd = [pip_exe, "install"] + dists_to_install
print(" ".join(pip_cmd), file=sys.stderr)
try:
subprocess.run(pip_cmd, check=True, capture_output=True)
except subprocess.CalledProcessError as e:
print(f"Failed to install {dist.name}", file=sys.stderr)
print(e.stdout.decode(), file=sys.stderr)
print(e.stderr.decode(), file=sys.stderr)
return False
packages_to_check = [
d.name for d in dist.iterdir()
if d.is_dir() and d.name.isidentifier()
]
modules_to_check = [
d.stem for d in dist.iterdir()
if d.is_file() and d.suffix == ".pyi"
]
cmd = [
python_exe,
"-m",
"mypy.stubtest",
# Use --ignore-missing-stub, because if someone makes a correct addition, they'll need to
# also make a allowlist change and if someone makes an incorrect addition, they'll run into
# false negatives.
"--ignore-missing-stub",
# Use --custom-typeshed-dir in case we make linked changes to stdlib or _typeshed
"--custom-typeshed-dir",
str(dist.parent.parent),
*packages_to_check,
*modules_to_check,
]
allowlist_path = dist / "@tests/stubtest_allowlist.txt"
if allowlist_path.exists():
cmd.extend(["--allowlist", str(allowlist_path)])
try:
print(f"MYPYPATH={dist}", " ".join(cmd), file=sys.stderr)
subprocess.run(cmd,
env={
"MYPYPATH": str(dist),
"MYPY_FORCE_COLOR": "1"
},
check=True)
except subprocess.CalledProcessError:
print(f"stubtest failed for {dist.name}", file=sys.stderr)
print("\n\n", file=sys.stderr)
if allowlist_path.exists():
print(
f'To fix "unused allowlist" errors, remove the corresponding entries from {allowlist_path}',
file=sys.stderr)
else:
print(
f"Re-running stubtest with --generate-allowlist.\nAdd the following to {allowlist_path}:",
file=sys.stderr)
subprocess.run(cmd + ["--generate-allowlist"],
env={"MYPYPATH": str(dist)})
print("\n\n", file=sys.stderr)
return False
else:
print(f"stubtest succeeded for {dist.name}", file=sys.stderr)
print("\n\n", file=sys.stderr)
return True
res[0].decode('utf-8'))
if pip:
print("Found Pip for {} in {}".format(pip[3], pip[2]))
found = True
python_exe = python
break
else:
print("Pip not found, can't setup test environment")
if not found:
print("Pip not found, can't setup test environment")
exit()
# generate virtual environment
try:
venv.create(VENV_PATH, clear=True, with_pip=True)
except CalledProcessError:
print("Error Creating virtual environment")
venv.create(VENV_PATH, system_site_packages=True, with_pip=False)
print("Creating virtual environment for testing")
requirements_file = os.path.join(CALIBRE_WEB_PATH, 'requirements.txt')
p = process_open(
[VENV_PYTHON, "-m", "pip", "install", "-r", requirements_file], (0, 5))
if os.name == 'nt':
while p.poll() == None:
p.stdout.readline()
else:
p.wait()
environment.init_environment(VENV_PYTHON, sub_dependencies)
def _stage1_create_venv():
# Create it in our cwd + '.venv'.
venv.create(venv_path, with_pip=True)
else:
value = binascii.hexlify(os.urandom(20)).decode('utf-8')
return value
SECRET_FILES = ('development-secrets.ini', )
SECRET_VARS = (
(r'%cookiecutter.authentication_random%', compat_token_hex()),
(r'%cookiecutter.authomatic_random%', compat_token_hex()),
(r'%cookiecutter.session_random%', compat_token_hex()),
)
if VIRTUALENV_AVAILABLE:
try:
venv.create('env', with_pip=True)
proc = subprocess.Popen(
['env/bin/pip', 'install', '-r', 'requirements.txt'],
shell=sys.platform.startswith('win'),
cwd='.')
proc.wait()
except subprocess.CalledProcessError:
print(
'It was not possible to create the virtualenv. Maybe inside tox?')
except FileNotFoundError as e:
print(subprocess.check_output(['ls', './env/bin/']), str(e))
for filename in SECRET_FILES:
path = './{{ cookiecutter.namespace }}/{{ cookiecutter.package_name }}/conf/{filename}'.format(
filename=filename)
with open(path, 'r+') as fh:
def build_virtualenv(
name, directory, python_interpreter=None, user=None, verbose=False):
"""
Build a virtualenv in a directory
Parameters
----------
name : str
Name of the virtualenv to create
directory : str
Directory to create the virtualenv in
python_interpreter : str, optional
Python interpreter to provide in the virtualenv, defaults to the
interpreter that is running the virtualenv command
verbose : bool
If True, provides status output while running.
Returns
-------
str
Full path to the root of the virtualenv directory
Raises
------
BuildException
The Virtualenv build failed
"""
# if not python_interpreter:
# if not hasattr(sys, 'frozen'):
# python_interpreter = sys.executable
# logger.debug('Python interpreter is: %s' % sys.executable)
cwd = os.getcwd()
if not os.path.isdir(directory):
os.makedirs(directory)
virtualenv_dir = os.path.join(directory, name)
user_uid = None
user_gid = None
if user:
user_uid = getpwnam(user).pw_uid
user_gid = getpwnam(user).pw_gid
if False and not python_interpreter and BUILTIN_VENV and \
not hasattr(sys, 'frozen'):
logger.debug(
'Building virtualenv %r using the built in venv module',
virtualenv_dir
)
venv.create(virtualenv_dir, with_pip=True)
else:
os.chdir(directory)
command = [virtualenv_command()]
if python_interpreter:
command += ['-p', python_interpreter]
command += [name]
logger.debug(
'Building virtualenv using external command %r', ' '.join(command)
)
try:
output = subprocess.check_output(
command,
stderr=subprocess.STDOUT,
# preexec_fn=change_uid_gid(user_uid=user_uid),
)
if verbose:
print(output.decode().strip())
except subprocess.CalledProcessError as error:
if verbose:
print(error.output.decode().strip())
logger.debug(error.output.decode().strip())
logger.exception(
'Virtualenv create command %r failed', ' '.join(command))
remove_virtualenv(name, directory)
os.chdir(cwd)
raise BuildException('Virtualenv create failed')
os.chdir(cwd)
for directory in ['conf', 'logs']:
filename = os.path.join(virtualenv_dir, directory)
if not os.path.exists(filename):
logger.debug('Creating %r directory', filename)
os.makedirs(filename)
return virtualenv_dir
def new_projet(projet_name,virenv=True,git=True,github=True):
""" style Google
Args:
projet_name: Name of New project
virenv: virtual environement creation ( True or False)
git: git link creation ( True or False)
github: github repo creation ( True or False)
Returns:
create folder with the new project name
create sub folder for the application project
create virtual environement
create first git link and commit
create new github repo and remote it
"""
with open("config.json", "r") as variable:
data = json.load(variable)
working_directory = data['path']
loggin = data['githublogin']
password = data['githubpass']
PROJET_PATH = os.path.join(working_directory, projet_name)
APP_PATH = os.path.join(PROJET_PATH, "app")
APP_FILE = os.path.join(APP_PATH,"app.py")
README_FILE = os.path.join(PROJET_PATH,"README.md")
ENV_PATH = os.path.join(PROJET_PATH,"env")
#BIN_PATH = os.path.join(ENV_PATH, "bin")
if not os.path.exists(PROJET_PATH):
os.mkdir(PROJET_PATH)
if not os.path.exists(APP_PATH):
os.mkdir(APP_PATH)
if not os.path.exists(APP_FILE):
with open(APP_FILE, "a"):
os.utime(APP_FILE, None)
if not os.path.exists(README_FILE):
with open(README_FILE, "a"):
os.utime(README_FILE, None)
if virenv == True:
if not os.path.exists(ENV_PATH):
venv.create(ENV_PATH,with_pip=True)
time.sleep(2)
os.chdir(PROJET_PATH)
cmd = 'code .'
os.system(cmd)
time.sleep(2)
if git == True:
gitignore_origine = "/home/kilann/Documents/Dev_Learning/New_Project_Creator/.gitignore_default"
gitignore_dest = os.path.join(PROJET_PATH, '.gitignore')
if not os.path.exists(gitignore_dest):
with open(gitignore_dest, "a"):
os.utime(APP_FILE, None)
shutil.copy2(gitignore_origine, gitignore_dest)
subprocess.Popen("git init", stdout=subprocess.PIPE, shell=True)
os.chdir(APP_PATH)
time.sleep(1)
subprocess.Popen("git add app.py", stdout=subprocess.PIPE, shell=True)
time.sleep(1)
subprocess.Popen("git commit -m 'first commit'", stdout=subprocess.PIPE, shell=True)
if github == True:
name = projet_name
browser = webdriver.Chrome('/home/kilann/Documents/mes_modules/chromedriver')
browser.get('https://github.com/new')
time.sleep(1)
user_name = browser.find_elements_by_xpath("//*[@name='login']")[0]
user_name.send_keys(loggin)
user_name = browser.find_elements_by_xpath("//*[@name='password']")[0]
user_name.send_keys(password)
user_name = browser.find_elements_by_xpath("//*[@name='commit']")[0]
user_name.click()
time.sleep(1)
user_name = browser.find_elements_by_xpath("//*[@name='repository[name]']")[0]
user_name.send_keys(name)
time.sleep(1)
user_name = browser.find_elements_by_xpath("//*[@data-disable-with='Creating repository…']")[0]
user_name.click()
time.sleep(1)
git_path = browser.find_element_by_xpath("/html/body/div[4]/div/main/div[3]/div/div[1]/div[1]/div/div[3]/div/span/input").get_attribute("value")
browser.close()
print(git_path)
time.sleep(1)
subprocess.Popen("git remote add origin "+ git_path, stdout=subprocess.PIPE, shell=True)
time.sleep(1)
subprocess.Popen("git remote -v", stdout=subprocess.PIPE, shell=True)
""" Enable venv and show convenience message """
import subprocess
import sys
from textwrap import dedent
try:
# python 3.2+
import venv
VIRTUALENV_AVAILABLE = True
except ImportError:
VIRTUALENV_AVAILABLE = False
if VIRTUALENV_AVAILABLE:
venv.create('.', with_pip=True)
proc = subprocess.Popen(
['bin/pip', 'install', '--upgrade', 'pip', 'setuptools'],
shell=sys.platform.startswith('win'),
cwd='.'
)
proc.wait()
proc = subprocess.Popen(
['bin/pip', 'install', '-e', '.'],
shell=sys.platform.startswith('win'),
cwd='.'
)
proc.wait()
separator = "=" * 79
msg = dedent(
def ensure_venv(target):
if os.path.exists(os.path.join(target, "bin", "pip3")):
# XXX Support probing the target whether it works properly and rebuild
# if necessary
return
if os.path.exists(target):
print("Deleting unclean target)")
cmd("rm -rf {target}".format(target=target))
version = sys.version.split()[0]
python_maj_min = ".".join(str(x) for x in sys.version_info[:2])
print("Creating venv ...")
venv.create(target, with_pip=False)
try:
# This is trying to detect whether we're on a proper Python stdlib
# or on a broken Debian. See various StackOverflow questions about
# this.
import distutils.util # noqa: F401 imported but unused
import ensurepip # noqa: F401 imported but unused
except ImportError:
# Okay, lets repair this, if we can. May need privilege escalation
# at some point.
# We could do: apt-get -y -q install python3-distutils python3-venv
# on some systems but it requires root and is specific to Debian.
# I decided to go a more sledge hammer route.
# XXX we can speed this up by storing this in ~/.appenv/overlay instead
# of doing the download for every venv we manage
print("Activating broken distutils/ensurepip stdlib workaround ...")
tmp_base = tempfile.mkdtemp()
try:
download = os.path.join(tmp_base, "download.tar.gz")
with open(download, mode="wb") as f:
get("www.python.org",
"/ftp/python/{v}/Python-{v}.tgz".format(v=version), f)
cmd("tar xf {} -C {}".format(download, tmp_base))
assert os.path.exists(
os.path.join(tmp_base, "Python-{}".format(version)))
for module in ["ensurepip", "distutils"]:
print(module)
shutil.copytree(
os.path.join(tmp_base, "Python-{}".format(version), "Lib",
module),
os.path.join(target, "lib",
"python{}.{}".format(*sys.version_info[:2]),
"site-packages", module))
# (always) prepend the site packages so we can actually have a
# fixed distutils installation.
site_packages = os.path.abspath(
os.path.join(target, "lib", "python" + python_maj_min,
"site-packages"))
with open(os.path.join(site_packages, "batou.pth"), "w") as f:
f.write("import sys; sys.path.insert(0, '{}')\n".format(
site_packages))
finally:
shutil.rmtree(tmp_base)
print("Ensuring pip ...")
cmd("{target}/bin/python -m ensurepip --default-pip".format(target=target))
cmd("{target}/bin/python -m pip install --upgrade pip".format(
target=target))
def do_pyvenv(path, system_site_packages):
try:
import venv
except ImportError:
error("Standard Python 'venv' module not found", ERROR_EXCEPTION)
venv.create(path, system_site_packages=system_site_packages)
def create_virtualenv():
import venv
venv.create('env', clear=True, with_pip=True)
def create_venv(parent_path: Path) -> Path:
venv_path = parent_path / 'package-smoke-test'
venv.create(venv_path, with_pip=True)
subprocess.run([venv_path / 'bin' / 'pip', 'install', '-U', 'pip', 'setuptools'], check=True)
return venv_path
print(
"Encountered error(s) with exit code {0} while running: {1}\nTerminating..."
.format(status, cmd))
sys.exit(1)
test_suites = [
"pyrosetta.tests.bindings.core.test_pose",
"pyrosetta.tests.distributed.test_concurrency",
"pyrosetta.tests.distributed.test_dask",
"pyrosetta.tests.distributed.test_gil",
"pyrosetta.tests.distributed.test_smoke",
"pyrosetta.tests.distributed.test_viewer",
"pyrosetta.tests.numeric.test_alignment"
]
with tempfile.TemporaryDirectory(prefix="tmp_pyrosetta_env") as venv_dir:
venv.create(venv_dir,
clear=True,
system_site_packages=False,
with_pip=True)
packages = "blosc dask distributed jupyter numpy pandas py3Dmol scipy traitlets"
e("source {0}/bin/activate && {0}/bin/pip install {1}".format(
venv_dir, packages))
for test_suite in test_suites:
e("source {0}/bin/activate && {0}/bin/python -m unittest {1}".format(
venv_dir, test_suite))
