def _store_git_repos(self, git_repo, git_branch):
"""Clone and or update all git repositories.
:param git_repo: URL for git repo
:type git_repo: ``str``
:param git_branch: Branch for git repo
:type git_branch: ``str``
"""
# Set the repo name to the base name of the git_repo variable.
repo_name = os.path.basename(git_repo)
# Set the git repo path.
repo_path_name = os.path.join(self.args['git_repo_path'], repo_name)
# If there is no .git dir remove the target and re-clone
if not os.path.isdir(os.path.join(repo_path_name, '.git')):
# If the directory exists update
if os.path.isdir(repo_path_name):
utils.remove_dirs(directory=repo_path_name)
# Clone the main repos
self._run_clone(git_repo, repo_path_name)
# Temporarily change the directory to the repo path.
with utils.ChangeDir(target_dir=repo_path_name):
self._run_update(git_repo=git_repo, git_branch=git_branch)
def build_wheels(self, packages, clean_first=False, force_iterate=False):
"""Create python wheels from a list of packages.
This method will build all of the wheels from a list of packages. Once
the loop is completed the wheels items will be moved to the storage
pool location. Upon the completion of the method the ``build_output``
directory will be removed.
:param packages: List of packages to build.
:type packages: ``list``
:param clean_first: Enable a search and clean for existing package
:type clean_first: ``bol``
:param force_iterate: Force package iteration.
:type force_iterate: ``bol``
"""
try:
if clean_first and self.args["link_dir"]:
files = utils.get_file_names(self.args["link_dir"])
for package in packages:
self._package_clean(package=package, files=files)
if self.args["pip_bulk_operation"] and not force_iterate:
req_file = os.path.join(self.args["link_dir"], "build_reqs.txt")
LOG.info('Requirement file being written: "%s"', req_file)
self.shell_cmds.mkdir_p(path=os.path.dirname(req_file))
with open(req_file, "wb") as f:
f.writelines(["%s\n" % i for i in packages])
self._pip_build_wheels(packages_file=req_file)
else:
for package in packages:
self._setup_build_wheels(package=package)
self._store_pool()
finally:
utils.remove_dirs(directory=self.args["build_output"])
def build_wheels(self, packages, clean_first=False, force_iterate=False):
"""Create python wheels from a list of packages.
This method will build all of the wheels from a list of packages. Once
the loop is completed the wheels items will be moved to the storage
pool location. Upon the completion of the method the ``build_output``
directory will be removed.
:param packages: List of packages to build.
:type packages: ``list``
:param clean_first: Enable a search and clean for existing package
:type clean_first: ``bol``
:param force_iterate: Force package iteration.
:type force_iterate: ``bol``
"""
try:
if clean_first and self.args['link_dir']:
files = utils.get_file_names(self.args['link_dir'])
for package in packages:
self._package_clean(package=package, files=files)
if self.args['pip_bulk_operation'] and not force_iterate:
req_file = os.path.join(self.args['link_dir'],
'build_reqs.txt')
LOG.info('Requirement file being written: "%s"', req_file)
self.shell_cmds.mkdir_p(path=os.path.dirname(req_file))
with open(req_file, 'wb') as f:
f.writelines(['%s\n' % i for i in packages])
self._pip_build_wheels(packages_file=req_file)
else:
for package in packages:
self._setup_build_wheels(package=package)
self._store_pool()
finally:
utils.remove_dirs(directory=self.args['build_output'])
def _setup_build_wheels(self, package):
"""Create a Python wheel using a git with subdirectories.
The method will clone the source into place, move to the designated
sub directory and create a python wheel from the designated
subdirectory.
:param package: Name of a particular package to build.
:type package: ``str``
"""
package_full_link = package.split("git+")[1]
if "#" in package_full_link:
package_link, extra_data = package_full_link.split("#")
else:
package_link, extra_data = package_full_link, None
package_link, branch = package_link.split("@")
# Split the branches to see if there are multiple checkouts/branches
git_branches, int_branch = self.split_git_branches(git_branch=branch)
if len(git_branches) > 1 or "refs/changes" in branch:
branch = int_branch
package_name = utils.git_pip_link_parse(repo=package)[0]
repo_location = os.path.join(self.args["git_repo_path"], package_name)
if extra_data and "subdirectory" in extra_data:
package_subdir = extra_data.split("subdirectory=")[1].split("&")[0]
git_package_location = os.path.join(repo_location, package_subdir)
else:
git_package_location = repo_location
# Check that setuptools is available
setup_py = "setup.py"
setup_file = os.path.join(git_package_location, setup_py)
remove_extra_setup = False
if os.path.isfile(setup_file):
with open(setup_file, "r") as f:
setup_file_contents = [i for i in f.readlines() if i if not i.startswith("#")]
for i in setup_file_contents:
if "setuptools" in i:
break
else:
for line in setup_file_contents:
if line.startswith("import"):
pass
elif line.startswith("from"):
pass
else:
index = setup_file_contents.index(line)
break
else:
index = 0
setup_file_contents.insert(index, "import setuptools")
setup_py = "%s2" % setup_file
remove_extra_setup = True
with open(setup_py, "w") as sf:
sf.writelines(setup_file_contents)
try:
with utils.ChangeDir(git_package_location):
# Checkout the given branch
checkout_command = ["git", "checkout", "'%s'" % branch]
self._run_command(command=checkout_command)
# Build the wheel using `python setup.py`
build_command = [
"python",
setup_py,
"bdist_wheel",
"--dist-dir",
self.args["build_output"],
"--bdist-dir",
self.args["build_dir"],
]
self._run_command(command=build_command)
LOG.debug('Build Success for: "%s"', package)
finally:
utils.remove_dirs(directory=self.args["build_dir"])
if remove_extra_setup:
os.remove(setup_py)
def _pip_build_wheels(self, package=None, packages_file=None, no_links=False, retry=False):
"""Create a python wheel.
The base command will timeout in 120 seconds and will create the wheels
within a defined build output directory, will allow external packages,
and will source the build output directory as the first link directory
to look through for already build wheels.
the argument options will enable no dependencies build, setting a build
directory otherwise a temporary directory will be used, setting an
additional link directory, setting extra link directories, changing the
defaul pip index URL, adding an extra pip index URL, and enabling
verbose mode.
:param package: Name of a particular package to build.
:type package: ``str``
:param packages_file: $PATH to the file which contains a list of
packages to build.
:type packages_file: ``str``
:param no_links: Enable / Disable add on links when building the wheel.
:type no_links: ``bol``
:param retry: Enable retry mode.
:type retry: ``bol``
"""
command = ["pip", "wheel", "--timeout", "120", "--wheel-dir", self.args["build_output"], "--allow-all-external"]
if self.args["pip_pre"]:
command.append("--pre")
if not no_links:
if self.args["pip_extra_link_dirs"]:
for link in self.args["pip_extra_link_dirs"]:
command.extend(["--find-links", link])
if self.args["pip_no_deps"]:
command.append("--no-deps")
else:
if self.args["pip_index"]:
command.extend(["--index-url", self.args["pip_index"]])
domain = urlparse.urlparse(self.args["pip_index"])
command.extend(["--trusted-host", domain.hostname])
if self.args["pip_extra_index"]:
command.extend(["--extra-index-url", self.args["pip_extra_index"]])
domain = urlparse.urlparse(self.args["pip_extra_index"])
command.extend(["--trusted-host", domain.hostname])
if self.args["pip_no_index"]:
command.append("--no-index")
if self.args["build_dir"]:
build_dir = self.args["build_dir"]
command.extend(["--build", build_dir])
else:
build_dir = tempfile.mkstemp(prefix="orb_")
command.extend(["--build", build_dir])
if self.args["debug"] is True:
command.append("--verbose")
if packages_file:
command.extend(["--requirement", packages_file])
else:
command.append('"%s"' % utils.stip_quotes(item=package))
try:
self._run_command(command=command)
except (IOError, OSError) as exp:
# If retry mode is enabled and there's an exception fail
if retry:
raise utils.AError(
'Failed to process wheel build: "%s", other data: "%s"', package or packages_file, str(exp)
)
LOG.warn(
'Failed to process wheel build: "%s", other data: "%s"' " Trying again without defined link lookups.",
package or packages_file,
str(exp),
)
# Remove the build directory when failed.
utils.remove_dirs(build_dir)
# Rerun wheel builder in retry mode without links
build_wheel_args = {"no_links": True, "retry": True}
if package:
build_wheel_args["package"] = package
else:
build_wheel_args["packages_file"] = packages_file
self._pip_build_wheels(**build_wheel_args)
else:
LOG.debug('Build Success for: "%s"', package or packages_file)
finally:
utils.remove_dirs(directory=build_dir)
def _setup_build_wheels(self, package):
"""Create a Python wheel using a git with subdirectories.
The method will clone the source into place, move to the designated
sub directory and create a python wheel from the designated
subdirectory.
:param package: Name of a particular package to build.
:type package: ``str``
"""
try:
package_full_link = package.split('git+')[1]
except IndexError:
return self._pip_build_wheels(package=package)
if '#' in package_full_link:
package_link, extra_data = package_full_link.split('#')
else:
package_link, extra_data = package_full_link, None
package_link, branch = package_link.split('@')
# Split the branches to see if there are multiple checkouts/branches
git_branches, int_branch = self.split_git_branches(git_branch=branch)
if len(git_branches) > 1 or 'refs/changes' in branch:
branch = int_branch
package_name = utils.git_pip_link_parse(repo=package)[0]
repo_location = os.path.join(
self.args['git_repo_path'], package_name
)
if extra_data and 'subdirectory' in extra_data:
package_subdir = extra_data.split('subdirectory=')[1].split('&')[0]
git_package_location = os.path.join(
repo_location,
package_subdir
)
else:
git_package_location = repo_location
try:
with utils.ChangeDir(git_package_location):
# Checkout the given branch
checkout_command = ['git', 'checkout', "'%s'" % branch]
self._run_command(command=checkout_command)
try:
LOG.debug('Build for: "%s"', package)
self._pip_build_wheels(
package=git_package_location,
no_links=True,
constraint_file=os.path.join(
git_package_location,
'constraints.txt'
)
)
except SystemExit:
# Build the wheel using `python setup.py`
LOG.warn(
'Running subdir package build for "%s" in fall back mode',
package
)
build_command = [
'python',
'setup.py',
'bdist_wheel',
'--dist-dir',
self.args['build_output'],
'--bdist-dir',
self.args['build_dir']
]
with utils.ChangeDir(git_package_location):
self._run_command(command=build_command)
finally:
utils.remove_dirs(directory=self.args['build_dir'])
def _pip_build_wheels(self, package=None, packages_file=None,
no_links=False, retry=False, constraint_file=None):
"""Create a python wheel.
The base command will timeout in 120 seconds and will create the wheels
within a defined build output directory, will allow external packages,
and will source the build output directory as the first link directory
to look through for already build wheels.
the argument options will enable no dependencies build, setting a build
directory otherwise a temporary directory will be used, setting an
additional link directory, setting extra link directories, changing the
defaul pip index URL, adding an extra pip index URL, and enabling
verbose mode.
:param package: Name of a particular package to build.
:type package: ``str``
:param packages_file: $PATH to the file which contains a list of
packages to build.
:type packages_file: ``str``
:param no_links: Enable / Disable add on links when building the wheel.
:type no_links: ``bol``
:param retry: Enable retry mode.
:type retry: ``bol``
:param constraint_file: Full path to a file containing project
constraints for packages.
:type constraint_file: ``str``
"""
command = [
'pip',
'wheel',
'--timeout',
'120',
'--wheel-dir',
self.args['build_output'],
'--allow-all-external'
]
if constraint_file and os.path.isfile(constraint_file):
command.extend(['--constraint', constraint_file])
if self.args['pip_pre']:
command.append('--pre')
if not no_links:
if self.args['pip_extra_link_dirs']:
for link in self.args['pip_extra_link_dirs']:
command.extend(['--find-links', link])
if self.args['pip_no_deps']:
command.append('--no-deps')
else:
if self.args['pip_index']:
command.extend(['--index-url', self.args['pip_index']])
domain = urlparse.urlparse(self.args['pip_index'])
command.extend(['--trusted-host', domain.hostname])
if self.args['pip_extra_index']:
command.extend(
['--extra-index-url', self.args['pip_extra_index']]
)
domain = urlparse.urlparse(self.args['pip_extra_index'])
command.extend(['--trusted-host', domain.hostname])
if self.args['pip_no_index']:
command.append('--no-index')
if self.args['build_dir']:
build_dir = self.args['build_dir']
command.extend(['--build', build_dir])
else:
build_dir = tempfile.mkstemp(prefix='orb_')
command.extend(['--build', build_dir])
if self.args['debug'] is True:
command.append('--verbose')
if packages_file:
command.extend(['--requirement', packages_file])
else:
command.append('"%s"' % utils.stip_quotes(item=package))
try:
self._run_command(command=command)
except (IOError, OSError) as exp:
# If retry mode is enabled and there's an exception fail
if retry:
raise utils.AError(
'Failed to process wheel build: "%s", other data: "%s"',
package or packages_file,
str(exp)
)
LOG.warn(
'Failed to process wheel build: "%s", other data: "%s"'
' Trying again without defined link lookups.',
package or packages_file,
str(exp)
)
# Remove the build directory when failed.
utils.remove_dirs(build_dir)
# Rerun wheel builder in retry mode without links
build_wheel_args = {'no_links': True, 'retry': True}
if package:
build_wheel_args['package'] = package
else:
build_wheel_args['packages_file'] = packages_file
self._pip_build_wheels(**build_wheel_args)
else:
LOG.debug('Build Success for: "%s"', package or packages_file)
finally:
utils.remove_dirs(directory=build_dir)
def _setup_build_wheels(self, package):
"""Create a Python wheel using a git with subdirectories.
The method will clone the source into place, move to the designated
sub directory and create a python wheel from the designated
subdirectory.
:param package: Name of a particular package to build.
:type package: ``str``
"""
package_full_link = package.split('git+')[1]
if '#' in package_full_link:
package_link, extra_data = package_full_link.split('#')
else:
package_link, extra_data = package_full_link, None
package_link, branch = package_link.split('@')
# Split the branches to see if there are multiple checkouts/branches
git_branches, int_branch = self.split_git_branches(git_branch=branch)
if len(git_branches) > 1 or 'refs/changes' in branch:
branch = int_branch
package_name = utils.git_pip_link_parse(repo=package)[0]
repo_location = os.path.join(self.args['git_repo_path'], package_name)
if extra_data and 'subdirectory' in extra_data:
package_subdir = extra_data.split('subdirectory=')[1].split('&')[0]
git_package_location = os.path.join(repo_location, package_subdir)
else:
git_package_location = repo_location
# Check that setuptools is available
setup_py = 'setup.py'
setup_file = os.path.join(git_package_location, setup_py)
remove_extra_setup = False
if os.path.isfile(setup_file):
with open(setup_file, 'r') as f:
setup_file_contents = [
i for i in f.readlines() if i if not i.startswith('#')
]
for i in setup_file_contents:
if 'setuptools' in i:
break
else:
for line in setup_file_contents:
if line.startswith('import'):
pass
elif line.startswith('from'):
pass
else:
index = setup_file_contents.index(line)
break
else:
index = 0
setup_file_contents.insert(index, 'import setuptools')
setup_py = '%s2' % setup_file
remove_extra_setup = True
with open(setup_py, 'w') as sf:
sf.writelines(setup_file_contents)
try:
with utils.ChangeDir(git_package_location):
# Checkout the given branch
checkout_command = ['git', 'checkout', "'%s'" % branch]
self._run_command(command=checkout_command)
# Build the wheel using `python setup.py`
build_command = [
'python', setup_py, 'bdist_wheel', '--dist-dir',
self.args['build_output'], '--bdist-dir',
self.args['build_dir']
]
self._run_command(command=build_command)
LOG.debug('Build Success for: "%s"', package)
finally:
utils.remove_dirs(directory=self.args['build_dir'])
if remove_extra_setup:
os.remove(setup_py)
def _pip_build_wheels(self,
package=None,
packages_file=None,
no_links=False,
retry=False):
"""Create a python wheel.
The base command will timeout in 120 seconds and will create the wheels
within a defined build output directory, will allow external packages,
and will source the build output directory as the first link directory
to look through for already build wheels.
the argument options will enable no dependencies build, setting a build
directory otherwise a temporary directory will be used, setting an
additional link directory, setting extra link directories, changing the
defaul pip index URL, adding an extra pip index URL, and enabling
verbose mode.
:param package: Name of a particular package to build.
:type package: ``str``
:param packages_file: $PATH to the file which contains a list of
packages to build.
:type packages_file: ``str``
:param no_links: Enable / Disable add on links when building the wheel.
:type no_links: ``bol``
:param retry: Enable retry mode.
:type retry: ``bol``
"""
command = [
'pip', 'wheel', '--timeout', '120', '--wheel-dir',
self.args['build_output'], '--allow-all-external'
]
if self.args['pip_pre']:
command.append('--pre')
if not no_links:
if self.args['pip_extra_link_dirs']:
for link in self.args['pip_extra_link_dirs']:
command.extend(['--find-links', link])
if self.args['pip_no_deps']:
command.append('--no-deps')
else:
if self.args['pip_index']:
command.extend(['--index-url', self.args['pip_index']])
domain = urlparse.urlparse(self.args['pip_index'])
command.extend(['--trusted-host', domain.hostname])
if self.args['pip_extra_index']:
command.extend(
['--extra-index-url', self.args['pip_extra_index']])
domain = urlparse.urlparse(self.args['pip_extra_index'])
command.extend(['--trusted-host', domain.hostname])
if self.args['pip_no_index']:
command.append('--no-index')
if self.args['build_dir']:
build_dir = self.args['build_dir']
command.extend(['--build', build_dir])
else:
build_dir = tempfile.mkstemp(prefix='orb_')
command.extend(['--build', build_dir])
if self.args['debug'] is True:
command.append('--verbose')
if packages_file:
command.extend(['--requirement', packages_file])
else:
command.append('"%s"' % utils.stip_quotes(item=package))
try:
self._run_command(command=command)
except (IOError, OSError) as exp:
# If retry mode is enabled and there's an exception fail
if retry:
raise utils.AError(
'Failed to process wheel build: "%s", other data: "%s"',
package or packages_file, str(exp))
LOG.warn(
'Failed to process wheel build: "%s", other data: "%s"'
' Trying again without defined link lookups.', package
or packages_file, str(exp))
# Remove the build directory when failed.
utils.remove_dirs(build_dir)
# Rerun wheel builder in retry mode without links
build_wheel_args = {'no_links': True, 'retry': True}
if package:
build_wheel_args['package'] = package
else:
build_wheel_args['packages_file'] = packages_file
self._pip_build_wheels(**build_wheel_args)
else:
LOG.debug('Build Success for: "%s"', package or packages_file)
finally:
utils.remove_dirs(directory=build_dir)