def skeleton(self):
ATTRIBURES_BY_SECTION = {'project': ['name', 'namespace_packages', 'install_requires', 'version_file',
'description', 'long_description', 'console_scripts', 'upgrade_code',
'package_data']}
DEFAULTS = {'namespace_packages': [], 'install_requires': [], 'console_scripts': [], 'package_data': []}
from infi.projector.helper.utils import open_buildout_configfile
if not self.arguments.get("update"):
logger.error("Not implemented")
raise SystemExit(1)
logger.info("Starting skeleton update")
with open_buildout_configfile() as original:
backup = {}
logger.info("Backing up buildout sections")
for section in ATTRIBURES_BY_SECTION.keys():
backup[section] = {key: get(original, section, key, DEFAULTS.get(key, None))
for key in ATTRIBURES_BY_SECTION[section]}
logger.info("Writing skeleton files")
self.overwrite_update_files()
self.safe_append_to_gitignore("get-pip.py")
if self.arguments.get("--remove-deprecated-files", False):
logger.info("Removing deprecated files")
self.remove_deprecated_files()
with open_buildout_configfile(write_on_exit=True) as update:
logger.info("Writing buildout.cfg")
for section, attributes in backup.items():
for key, value in attributes.items():
update.set(section, key, value)
if self.arguments.get("--commit-changes", False):
logger.info("Committing changes")
from gitpy import LocalRepository
from os import curdir
repository = LocalRepository(curdir)
message = "updated project files from skeleton"
repository.commit(message, commitAll=True)
def _get_bootstrap_command(self):
from os.path import exists
from os import environ
if not exists("bootstrap.py"):
logger.error("bootsrap.py does not exist")
raise SystemExit(1)
cmd = "bootstrap.py"
additional_optional_args = {"PROJECTOR_BOOTSTRAP_DOWNLOAD_BASE": "--download-base",
"PROJECTOR_BOOTSTRAP_SETUP_SOURCE": "--setup-source",
"PROJECTOR_BOOTSTRAP_INDEX_URL": "--index-url",
}
for key, cmd_option in additional_optional_args.items():
option_value = environ.get(key, None)
if option_value:
cmd += ' {}={}'.format(cmd_option, option_value)
if not environ.get("PROJECTOR_BOOTSTRAP_INDEX_URL", None):
# we want to extract the index-url from pydistutils.cfg
cmd += ' --index-url={}'.format(self._get_pypi_index_url())
# in case dependencies are frozen, we need to use the frozen version of setuptools
with utils.open_buildout_configfile() as buildout:
if buildout.has_option("versions", "setuptools"):
cmd += ' --setuptools-version={}'.format(buildout.get("versions", "setuptools"))
if buildout.has_option("versions", "zc.buildout"):
cmd += ' --version={}'.format(buildout.get("versions", "zc.buildout"))
return cmd
def get_isolated_python_section_name(self):
with utils.open_buildout_configfile() as buildout:
sections = [section for section in buildout.sections()
if buildout.has_option(section, "recipe") and \
buildout.get(section, "recipe").startswith("infi.recipe.python")\
and not buildout.get(section, "recipe").endswith(":pack")]
return sections[0]
def install_sections_by_recipe(self, recipe):
with utils.open_buildout_configfile() as buildout:
sections_to_install = [section for section in buildout.sections()
if buildout.has_option(section, "recipe") and \
buildout.get(section, "recipe") == recipe]
if sections_to_install:
utils.execute_with_buildout("install {}".format(' '.join(sections_to_install)))
def install_sections_by_recipe(self, recipe, stripped=True):
with utils.open_buildout_configfile() as buildout:
sections_to_install = [section for section in buildout.sections()
if buildout.has_option(section, "recipe") and
buildout.get(section, "recipe").startswith(recipe)]
if sections_to_install:
utils.execute_with_buildout("install {}".format(' '.join(sections_to_install)), stripped=stripped)
def relocate(self):
relative_paths = self.arguments.get("--relative", False)
with utils.open_buildout_configfile(write_on_exit=True) as buildout:
buildout.set("buildout", "relative-paths", 'true' if relative_paths else 'false')
if self.arguments.get("--commit-changes", False):
commit_message = "Changing shebang to {} paths".format("relative" if relative_paths else "absolute")
utils.commit_changes_to_buildout(commit_message)
logger.info("Configuration changed. Run `projector devenv build [--use-isolated-python]`.")
def create_cache_directories(self):
from os import makedirs
from os.path import join, exists
with utils.open_buildout_configfile() as buildout:
cachedir = buildout.get("buildout", "download-cache")
cache_dist = join(cachedir, "dist")
if not exists(cache_dist):
makedirs(cache_dist)
def _apply_close_on_upgrade_or_removal(self, value):
from gitpy import LocalRepository
from infi.recipe.application_packager.utils.buildout import open_buildout_configfile
with open_buildout_configfile(write_on_exit=True) as buildout:
buildout.set("pack", "close-on-upgrade-or-removal", value)
repository = LocalRepository('.')
repository.add('buildout.cfg')
repository.commit('HOSTDEV-1922 testing close-on-upgrade-or-removal={}'.format(value))
def get_section(cls):
with open_buildout_configfile() as buildout:
sections = [
section
for section in buildout.sections()
if buildout.has_option(section, "recipe")
and buildout.get(section, "recipe") == "infi.recipe.console_scripts"
]
return sections[0]
def create_console_scripts():
from infi.execute import execute_assert_success
from infi.projector.helper.utils import open_buildout_configfile
with open_buildout_configfile(filepath="buildout.cfg", write_on_exit=True) as buildout:
buildout.set("project", "post_install_script_name", "post_install")
buildout.set("project", "pre_uninstall_script_name", "pre_uninstall")
for name in CONSOLE_SCRIPTS:
with open_buildout_configfile(filepath="buildout.cfg", write_on_exit=True) as buildout:
scripts = buildout.get("pack", "scripts").split() \
if buildout.has_section("pack") and buildout.has_option("pack", "scripts") \
else []
scripts.append(name)
buildout.set("pack", "scripts", "\n".join(scripts))
execute_assert_success([os.path.join('bin', 'projector'),
"console-scripts", "add", name,
"infi.recipe.application_packager.scripts:{0}".format(name),
"--commit-changes"])
execute_assert_success([os.path.join('bin', 'projector'),
"devenv", "build", "--no-scripts"])
def remove(self):
with open_buildout_configfile(write_on_exit=True) as buildout:
name = self.arguments.get("<name>")
if name in self.get_submodule_sections():
buildout.remove_section(name)
where_to_look_for_setup_py = set(buildout.get("buildout", "develop").split())
where_to_look_for_setup_py.discard(name)
buildout.set("buildout", "develop", ' '.join(where_to_look_for_setup_py))
if self.arguments.get("--commit-changes", False):
commit_message = "Removing git submodule {}".format(name)
commit_changes_to_buildout(commit_message)
def _get_default_remote_path(self):
from infi.projector.helper.utils import open_buildout_configfile
is_windows = self.arguments.get("<remote-user>") == "Administrator"
basedir = "/cygdrive/c/Program Files/" if is_windows else "/opt"
with open_buildout_configfile() as buildout:
get = buildout.get
if is_windows:
return "/".join([basedir, get("project", "company"), get("project", "product_name")])
else:
return "/".join([basedir, get("project", "company").lower(),
get("project", "product_name").replace(' ', '-').replace('_', '-').lower()])
def _remove_setuptools_egg_link(self):
# HOSTDEV-1130
# https://bugs.launchpad.net/zc.buildout/+bug/1210996
import os
with utils.open_buildout_configfile() as buildout:
try:
develop_eggs_dir = buildout.get("buildout", "develop-eggs-directory")
except (configparser.NoSectionError, configparser.NoOptionError):
develop_eggs_dir = "develop-eggs"
setuptools_egg_link = os.path.join(develop_eggs_dir, "setuptools.egg-link")
if os.path.exists(setuptools_egg_link):
os.remove(setuptools_egg_link)
def test_build_with_frozen_setuptools_and_zc_buildout_versions(self):
with self.temporary_directory_context():
self.projector("repository init a.b.c none short long")
with utils.open_buildout_configfile(write_on_exit=True) as buildout:
buildout.add_section("versions")
buildout.set("versions", "setuptools", "2.2")
buildout.set("versions", "zc.buildout", "2.2.1")
self.projector("devenv build --use-isolated-python")
self.assertTrue(path.exists(path.join("parts", "python")))
self.assert_scripts_were_generated_by_buildout()
self.assert_specific_setuptools_version_is_being_used("2.2")
self.assert_specific_zc_buildout_version_is_being_used("2.2.1")
def set_buildout_config(self):
from infi.projector.helper.utils import open_buildout_configfile
project_name = self.get_project_name()
with open_buildout_configfile(write_on_exit=True) as buildout:
buildout.set('project', 'name', project_name)
buildout.set('project', 'namespace_packages', str(get_package_namespace(project_name)))
buildout.set('project', 'version_file',
'/'.join(['src'] + project_name.split('.') + ['__version__.py']))
buildout.set('project', 'description', self.arguments.get("<short_description>"))
buildout.set('project', 'long_description', indent(self.arguments.get("<long_description>")))
buildout.set('project', 'upgrade_code', generate_package_code())
buildout.set('project', 'product_name', project_name)
def test_build_with_frozen_setuptools_version(self):
with self.temporary_directory_context():
self.projector("repository init a.b.c none short long")
with utils.open_buildout_configfile(write_on_exit=True) as buildout:
buildout.add_section("versions")
buildout.set("versions", "setuptools", "34.3.2")
# ipython>4 depends on simplegeneric>0.8, and setuptools 8.1 fails to parse this dependency correctly
# this is fixed in setuptools 8.4, so if anyone bumps setuptools, remove the following set:
buildout.set("versions", "ipython", "3.2.1")
self.projector("devenv build --use-isolated-python")
self.assertTrue(path.exists(path.join("parts", "python")))
self.assert_scripts_were_generated_by_buildout()
self.assert_specific_setuptools_version_is_being_used("34.3.2")
def _install_setuptools_and_zc_buildout(self):
from os.path import join, exists
from os import environ, remove
with utils.open_buildout_configfile() as buildout:
cachedir = buildout.get("buildout", "download-cache")
cache_dist = join(cachedir, "dist")
cmd = []
packages = []
# in case dependencies are frozen, we need to use the frozen version of setuptools and zc.buildout
with utils.open_buildout_configfile() as buildout:
for package in ['setuptools', 'zc.buildout', 'pip']:
if buildout.has_option("versions", package):
packages += ['{}=={}'.format(package, buildout.get("versions", package))]
else:
packages += [package]
env = environ.copy()
env['PYTHONPATH'] = ''
utils.execute_assert_success([utils.get_isolated_executable('python'), 'get-pip.py', '--prefix=%s' % join('parts', 'python')] + packages, env=env)
remove('get-pip.py')
utils.execute_assert_success([utils.get_isolated_executable('python'), '-m', 'pip', 'download', '--dest', cache_dist] + packages, env=env)
def python_version(self):
with open_buildout_configfile(write_on_exit=self.arguments.get("set")) as buildout:
sections = [section for section in buildout.sections()
if buildout.has_option(section, "recipe") and \
buildout.get(section, "recipe") == "infi.recipe.python"]
if not sections: # pragma: no cover
logger.error("isolated python section not found in buildout.cfg")
raise SystemExit(1)
if self.arguments.get("get"):
print buildout.get(sections[0], "version")
elif self.arguments.get("set"):
version = self.arguments.get("<version>")
buildout.set(sections[0], "version", version)
if self.arguments.get("--commit-changes", False):
commit_message = "changed isolated python version to {}".format(version)
commit_changes_to_buildout(commit_message)
def add(self):
with open_buildout_configfile(write_on_exit=True) as buildout:
name = self.arguments.get("<name>")
if name not in self.get_submodule_sections():
buildout.add_section(name)
repository = self.arguments.get("<repository>")
rev = self.arguments.get("<rev>")
buildout.set(name, "recipe", "zerokspot.recipe.git")
buildout.set(name, "repository", repository)
buildout.set(name, "rev", rev)
buildout.set(name, "newest", "true")
if self.arguments.get("use-setup-py"):
where_to_look_for_setup_py = set(buildout.get("buildout", "develop").split())
where_to_look_for_setup_py.add(name)
buildout.set("buildout", "develop", ' '.join(where_to_look_for_setup_py))
if self.arguments.get("--commit-changes", False):
commit_message = "Adding git submodule {}".format(name)
commit_changes_to_buildout(commit_message)
def add(self):
with open_buildout_configfile(write_on_exit=True) as buildout:
name = self.arguments.get("<name>")
if name not in self.get_submodule_sections():
buildout.add_section(name)
repository = self.arguments.get("<repository>")
rev = self.arguments.get("<rev>")
buildout.set(name, "recipe", "zerokspot.recipe.git")
buildout.set(name, "repository", repository)
buildout.set(name, "rev", rev)
buildout.set(name, "newest", "true")
if self.arguments.get("use-setup-py"):
where_to_look_for_setup_py = set(
buildout.get("buildout", "develop").split())
where_to_look_for_setup_py.add(name)
buildout.set("buildout", "develop",
' '.join(where_to_look_for_setup_py))
if self.arguments.get("--commit-changes", False):
commit_message = "Adding git submodule {}".format(name)
commit_changes_to_buildout(commit_message)
def add(self):
with open_buildout_configfile(write_on_exit=True) as buildout_cfg:
if not buildout_cfg.has_section("js-requirements"):
buildout_cfg.add_section("js-requirements")
buildout_cfg.set("js-requirements", "recipe",
"infi.recipe.js_requirements")
buildout_cfg.set("js-requirements", "js-directory", "")
buildout_cfg.set("js-requirements", "symlink-to-directory",
"parts/js")
buildout_cfg.set("js-requirements", "javascript-packages",
"[]")
package_set = self.get_package_set()
requirements = package_set.get()
requirement = self.arguments.get('<requirement>')
if requirement not in requirements:
requirements.add(requirement)
package_set.set(requirements)
if self.arguments.get("--commit-changes", False):
commit_message = "adding {} to js-requirements".format(requirement)
commit_changes_to_buildout(commit_message)
def python_version(self):
with open_buildout_configfile(
write_on_exit=self.arguments.get("set")) as buildout:
sections = [section for section in buildout.sections()
if buildout.has_option(section, "recipe") and \
buildout.get(section, "recipe") == "infi.recipe.python"]
if not sections: # pragma: no cover
logger.error(
"isolated python section not found in buildout.cfg")
raise SystemExit(1)
if self.arguments.get("get"):
print buildout.get(sections[0], "version")
elif self.arguments.get("set"):
version = self.arguments.get("<version>")
if not version.startswith("v"):
version = "v" + version
buildout.set(sections[0], "version", version)
if self.arguments.get("--commit-changes", False):
commit_message = "changed isolated python version to {}".format(
version)
commit_changes_to_buildout(commit_message)
def freeze(self):
from gitpy import LocalRepository
from os import curdir
import json
# Read/write the buildout.cfg
with open_buildout_configfile(write_on_exit=True) as buildout_cfg:
if not buildout_cfg.has_section('js-requirements'):
print("Missing js-requirements section")
return
packages_path = buildout_cfg.get(
'js-requirements', 'js-directory') or self.DEFAULT_DIRECTORY
try:
with open(os.path.join(packages_path, '.package-lock.json'),
'r') as pljson:
selected_versions = json.load(pljson)
if buildout_cfg.has_section("js_versions"):
buildout_cfg.remove_section("js_versions")
buildout_cfg.add_section("js_versions")
for key in sorted(selected_versions.keys(),
key=lambda s: s.lower()):
buildout_cfg.set("js_versions", key,
selected_versions[key])
buildout_cfg.set('buildout', 'js_versions', 'True')
except IOError as e:
import errno
print(str(e))
if hasattr(e, 'errno') and e.errno == errno.ENOENT:
print(
'.package-lock.json file is missing, try running projector devenv build to create the file'
)
# Git operations
repository = LocalRepository(curdir)
if self.arguments.get("--commit-changes", False):
repository.add("buildout.cfg")
repository.commit("Freezing javascript dependencies")
push_changes = self.arguments.get("--push-changes", False)
if push_changes:
repository._executeGitCommandAssertSuccess("git push")
def get_value(self):
with open_buildout_configfile() as buildout_cfg:
return buildout_cfg.get("project", self.get_attribute())
def set_value(self, value):
with open_buildout_configfile(write_on_exit=True) as buildout_cfg:
return buildout_cfg.set("project", self.get_attribute(), value)
def get(self):
with open_buildout_configfile(self.filepath) as cfg:
return self.from_value(cfg)
def get_dependencies():
from infi.projector.helper.utils import open_buildout_configfile
with open_buildout_configfile() as buildout:
exec 'dependencies = {0}'.format(
buildout.get("project", "install_requires"))
return dependencies
def get_dependencies():
from infi.projector.helper.utils import open_buildout_configfile
with open_buildout_configfile() as buildout:
exec 'dependencies = {0}'.format(buildout.get("project", "install_requires"))
return dependencies
def set_value(self, value):
with open_buildout_configfile(write_on_exit=True) as buildout_cfg:
return buildout_cfg.set("project", self.get_attribute(), value)
def get(self):
with open_buildout_configfile() as buildout_cfg:
return self.from_value(
buildout_cfg.get(self.section_name, self.attribute_name))
def set(self, package_set):
with open_buildout_configfile(write_on_exit=True) as buildout_cfg:
buildout_cfg.set(self.section_name, self.attribute_name,
self.to_value(package_set))
def get_section(cls):
with open_buildout_configfile() as buildout:
sections = [section for section in buildout.sections()
if buildout.has_option(section, "recipe") and \
buildout.get(section, "recipe") == "infi.recipe.console_scripts"]
return sections[0]
def get(self):
with open_buildout_configfile(self.filepath) as cfg:
return self.from_value(cfg)
def get(self):
with open_buildout_configfile() as buildout_cfg:
return self.from_value(buildout_cfg.get(self.section_name, self.attribute_name))
def get_submodule_sections(self):
with open_buildout_configfile() as buildout:
sections = [section for section in buildout.sections()
if buildout.has_option(section, "recipe") and \
buildout.get(section, "recipe") == "zerokspot.recipe.git"]
return sections
def set(self, package_set):
with open_buildout_configfile(write_on_exit=True) as buildout_cfg:
buildout_cfg.set(self.section_name, self.attribute_name, self.to_value(package_set))
def get_dependencies():
from infi.projector.helper.utils import open_buildout_configfile
with open_buildout_configfile() as buildout:
exec 'dependencies = {0}'.format(buildout.get("project", "install_requires"))
return dependencies # noqa # flake8 doesn't recognize the exec, thinks 'dependencies' is undefined
def get_value(self):
with open_buildout_configfile() as buildout_cfg:
return buildout_cfg.get("project", self.get_attribute())
def get_dependencies():
from infi.projector.helper.utils import open_buildout_configfile
with open_buildout_configfile() as buildout:
exec("dependencies = {0}".format(buildout.get("project", "install_requires")))
return dependencies # noqa # flake8 doesn't recognize the exec, thinks 'dependencies' is undefined
def get_submodule_sections(self):
with open_buildout_configfile() as buildout:
sections = [section for section in buildout.sections()
if buildout.has_option(section, "recipe") and \
buildout.get(section, "recipe") == "zerokspot.recipe.git"]
return sections
