def _store_git_repos(self, git_repo, git_branch): """Clone and or update all git repositories. :param git_repo: URL for git repo :type git_repo: ``str`` :param git_branch: Branch for git repo :type git_branch: ``str`` """ # Set the repo name to the base name of the git_repo variable. repo_name = os.path.basename(git_repo) # Set the git repo path. repo_path_name = os.path.join(self.args['git_repo_path'], repo_name) # If there is no .git dir remove the target and re-clone if not os.path.isdir(os.path.join(repo_path_name, '.git')): # If the directory exists update if os.path.isdir(repo_path_name): utils.remove_dirs(directory=repo_path_name) # Clone the main repos self._run_clone(git_repo, repo_path_name) # Temporarily change the directory to the repo path. with utils.ChangeDir(target_dir=repo_path_name): self._run_update(git_repo=git_repo, git_branch=git_branch)
def build_wheels(self, packages, clean_first=False, force_iterate=False): """Create python wheels from a list of packages. This method will build all of the wheels from a list of packages. Once the loop is completed the wheels items will be moved to the storage pool location. Upon the completion of the method the ``build_output`` directory will be removed. :param packages: List of packages to build. :type packages: ``list`` :param clean_first: Enable a search and clean for existing package :type clean_first: ``bol`` :param force_iterate: Force package iteration. :type force_iterate: ``bol`` """ try: if clean_first and self.args["link_dir"]: files = utils.get_file_names(self.args["link_dir"]) for package in packages: self._package_clean(package=package, files=files) if self.args["pip_bulk_operation"] and not force_iterate: req_file = os.path.join(self.args["link_dir"], "build_reqs.txt") LOG.info('Requirement file being written: "%s"', req_file) self.shell_cmds.mkdir_p(path=os.path.dirname(req_file)) with open(req_file, "wb") as f: f.writelines(["%s\n" % i for i in packages]) self._pip_build_wheels(packages_file=req_file) else: for package in packages: self._setup_build_wheels(package=package) self._store_pool() finally: utils.remove_dirs(directory=self.args["build_output"])
def build_wheels(self, packages, clean_first=False, force_iterate=False): """Create python wheels from a list of packages. This method will build all of the wheels from a list of packages. Once the loop is completed the wheels items will be moved to the storage pool location. Upon the completion of the method the ``build_output`` directory will be removed. :param packages: List of packages to build. :type packages: ``list`` :param clean_first: Enable a search and clean for existing package :type clean_first: ``bol`` :param force_iterate: Force package iteration. :type force_iterate: ``bol`` """ try: if clean_first and self.args['link_dir']: files = utils.get_file_names(self.args['link_dir']) for package in packages: self._package_clean(package=package, files=files) if self.args['pip_bulk_operation'] and not force_iterate: req_file = os.path.join(self.args['link_dir'], 'build_reqs.txt') LOG.info('Requirement file being written: "%s"', req_file) self.shell_cmds.mkdir_p(path=os.path.dirname(req_file)) with open(req_file, 'wb') as f: f.writelines(['%s\n' % i for i in packages]) self._pip_build_wheels(packages_file=req_file) else: for package in packages: self._setup_build_wheels(package=package) self._store_pool() finally: utils.remove_dirs(directory=self.args['build_output'])
def _setup_build_wheels(self, package): """Create a Python wheel using a git with subdirectories. The method will clone the source into place, move to the designated sub directory and create a python wheel from the designated subdirectory. :param package: Name of a particular package to build. :type package: ``str`` """ package_full_link = package.split("git+")[1] if "#" in package_full_link: package_link, extra_data = package_full_link.split("#") else: package_link, extra_data = package_full_link, None package_link, branch = package_link.split("@") # Split the branches to see if there are multiple checkouts/branches git_branches, int_branch = self.split_git_branches(git_branch=branch) if len(git_branches) > 1 or "refs/changes" in branch: branch = int_branch package_name = utils.git_pip_link_parse(repo=package)[0] repo_location = os.path.join(self.args["git_repo_path"], package_name) if extra_data and "subdirectory" in extra_data: package_subdir = extra_data.split("subdirectory=")[1].split("&")[0] git_package_location = os.path.join(repo_location, package_subdir) else: git_package_location = repo_location # Check that setuptools is available setup_py = "setup.py" setup_file = os.path.join(git_package_location, setup_py) remove_extra_setup = False if os.path.isfile(setup_file): with open(setup_file, "r") as f: setup_file_contents = [i for i in f.readlines() if i if not i.startswith("#")] for i in setup_file_contents: if "setuptools" in i: break else: for line in setup_file_contents: if line.startswith("import"): pass elif line.startswith("from"): pass else: index = setup_file_contents.index(line) break else: index = 0 setup_file_contents.insert(index, "import setuptools") setup_py = "%s2" % setup_file remove_extra_setup = True with open(setup_py, "w") as sf: sf.writelines(setup_file_contents) try: with utils.ChangeDir(git_package_location): # Checkout the given branch checkout_command = ["git", "checkout", "'%s'" % branch] self._run_command(command=checkout_command) # Build the wheel using `python setup.py` build_command = [ "python", setup_py, "bdist_wheel", "--dist-dir", self.args["build_output"], "--bdist-dir", self.args["build_dir"], ] self._run_command(command=build_command) LOG.debug('Build Success for: "%s"', package) finally: utils.remove_dirs(directory=self.args["build_dir"]) if remove_extra_setup: os.remove(setup_py)
def _pip_build_wheels(self, package=None, packages_file=None, no_links=False, retry=False): """Create a python wheel. The base command will timeout in 120 seconds and will create the wheels within a defined build output directory, will allow external packages, and will source the build output directory as the first link directory to look through for already build wheels. the argument options will enable no dependencies build, setting a build directory otherwise a temporary directory will be used, setting an additional link directory, setting extra link directories, changing the defaul pip index URL, adding an extra pip index URL, and enabling verbose mode. :param package: Name of a particular package to build. :type package: ``str`` :param packages_file: $PATH to the file which contains a list of packages to build. :type packages_file: ``str`` :param no_links: Enable / Disable add on links when building the wheel. :type no_links: ``bol`` :param retry: Enable retry mode. :type retry: ``bol`` """ command = ["pip", "wheel", "--timeout", "120", "--wheel-dir", self.args["build_output"], "--allow-all-external"] if self.args["pip_pre"]: command.append("--pre") if not no_links: if self.args["pip_extra_link_dirs"]: for link in self.args["pip_extra_link_dirs"]: command.extend(["--find-links", link]) if self.args["pip_no_deps"]: command.append("--no-deps") else: if self.args["pip_index"]: command.extend(["--index-url", self.args["pip_index"]]) domain = urlparse.urlparse(self.args["pip_index"]) command.extend(["--trusted-host", domain.hostname]) if self.args["pip_extra_index"]: command.extend(["--extra-index-url", self.args["pip_extra_index"]]) domain = urlparse.urlparse(self.args["pip_extra_index"]) command.extend(["--trusted-host", domain.hostname]) if self.args["pip_no_index"]: command.append("--no-index") if self.args["build_dir"]: build_dir = self.args["build_dir"] command.extend(["--build", build_dir]) else: build_dir = tempfile.mkstemp(prefix="orb_") command.extend(["--build", build_dir]) if self.args["debug"] is True: command.append("--verbose") if packages_file: command.extend(["--requirement", packages_file]) else: command.append('"%s"' % utils.stip_quotes(item=package)) try: self._run_command(command=command) except (IOError, OSError) as exp: # If retry mode is enabled and there's an exception fail if retry: raise utils.AError( 'Failed to process wheel build: "%s", other data: "%s"', package or packages_file, str(exp) ) LOG.warn( 'Failed to process wheel build: "%s", other data: "%s"' " Trying again without defined link lookups.", package or packages_file, str(exp), ) # Remove the build directory when failed. utils.remove_dirs(build_dir) # Rerun wheel builder in retry mode without links build_wheel_args = {"no_links": True, "retry": True} if package: build_wheel_args["package"] = package else: build_wheel_args["packages_file"] = packages_file self._pip_build_wheels(**build_wheel_args) else: LOG.debug('Build Success for: "%s"', package or packages_file) finally: utils.remove_dirs(directory=build_dir)
def _setup_build_wheels(self, package): """Create a Python wheel using a git with subdirectories. The method will clone the source into place, move to the designated sub directory and create a python wheel from the designated subdirectory. :param package: Name of a particular package to build. :type package: ``str`` """ try: package_full_link = package.split('git+')[1] except IndexError: return self._pip_build_wheels(package=package) if '#' in package_full_link: package_link, extra_data = package_full_link.split('#') else: package_link, extra_data = package_full_link, None package_link, branch = package_link.split('@') # Split the branches to see if there are multiple checkouts/branches git_branches, int_branch = self.split_git_branches(git_branch=branch) if len(git_branches) > 1 or 'refs/changes' in branch: branch = int_branch package_name = utils.git_pip_link_parse(repo=package)[0] repo_location = os.path.join( self.args['git_repo_path'], package_name ) if extra_data and 'subdirectory' in extra_data: package_subdir = extra_data.split('subdirectory=')[1].split('&')[0] git_package_location = os.path.join( repo_location, package_subdir ) else: git_package_location = repo_location try: with utils.ChangeDir(git_package_location): # Checkout the given branch checkout_command = ['git', 'checkout', "'%s'" % branch] self._run_command(command=checkout_command) try: LOG.debug('Build for: "%s"', package) self._pip_build_wheels( package=git_package_location, no_links=True, constraint_file=os.path.join( git_package_location, 'constraints.txt' ) ) except SystemExit: # Build the wheel using `python setup.py` LOG.warn( 'Running subdir package build for "%s" in fall back mode', package ) build_command = [ 'python', 'setup.py', 'bdist_wheel', '--dist-dir', self.args['build_output'], '--bdist-dir', self.args['build_dir'] ] with utils.ChangeDir(git_package_location): self._run_command(command=build_command) finally: utils.remove_dirs(directory=self.args['build_dir'])
def _pip_build_wheels(self, package=None, packages_file=None, no_links=False, retry=False, constraint_file=None): """Create a python wheel. The base command will timeout in 120 seconds and will create the wheels within a defined build output directory, will allow external packages, and will source the build output directory as the first link directory to look through for already build wheels. the argument options will enable no dependencies build, setting a build directory otherwise a temporary directory will be used, setting an additional link directory, setting extra link directories, changing the defaul pip index URL, adding an extra pip index URL, and enabling verbose mode. :param package: Name of a particular package to build. :type package: ``str`` :param packages_file: $PATH to the file which contains a list of packages to build. :type packages_file: ``str`` :param no_links: Enable / Disable add on links when building the wheel. :type no_links: ``bol`` :param retry: Enable retry mode. :type retry: ``bol`` :param constraint_file: Full path to a file containing project constraints for packages. :type constraint_file: ``str`` """ command = [ 'pip', 'wheel', '--timeout', '120', '--wheel-dir', self.args['build_output'], '--allow-all-external' ] if constraint_file and os.path.isfile(constraint_file): command.extend(['--constraint', constraint_file]) if self.args['pip_pre']: command.append('--pre') if not no_links: if self.args['pip_extra_link_dirs']: for link in self.args['pip_extra_link_dirs']: command.extend(['--find-links', link]) if self.args['pip_no_deps']: command.append('--no-deps') else: if self.args['pip_index']: command.extend(['--index-url', self.args['pip_index']]) domain = urlparse.urlparse(self.args['pip_index']) command.extend(['--trusted-host', domain.hostname]) if self.args['pip_extra_index']: command.extend( ['--extra-index-url', self.args['pip_extra_index']] ) domain = urlparse.urlparse(self.args['pip_extra_index']) command.extend(['--trusted-host', domain.hostname]) if self.args['pip_no_index']: command.append('--no-index') if self.args['build_dir']: build_dir = self.args['build_dir'] command.extend(['--build', build_dir]) else: build_dir = tempfile.mkstemp(prefix='orb_') command.extend(['--build', build_dir]) if self.args['debug'] is True: command.append('--verbose') if packages_file: command.extend(['--requirement', packages_file]) else: command.append('"%s"' % utils.stip_quotes(item=package)) try: self._run_command(command=command) except (IOError, OSError) as exp: # If retry mode is enabled and there's an exception fail if retry: raise utils.AError( 'Failed to process wheel build: "%s", other data: "%s"', package or packages_file, str(exp) ) LOG.warn( 'Failed to process wheel build: "%s", other data: "%s"' ' Trying again without defined link lookups.', package or packages_file, str(exp) ) # Remove the build directory when failed. utils.remove_dirs(build_dir) # Rerun wheel builder in retry mode without links build_wheel_args = {'no_links': True, 'retry': True} if package: build_wheel_args['package'] = package else: build_wheel_args['packages_file'] = packages_file self._pip_build_wheels(**build_wheel_args) else: LOG.debug('Build Success for: "%s"', package or packages_file) finally: utils.remove_dirs(directory=build_dir)
def _setup_build_wheels(self, package): """Create a Python wheel using a git with subdirectories. The method will clone the source into place, move to the designated sub directory and create a python wheel from the designated subdirectory. :param package: Name of a particular package to build. :type package: ``str`` """ package_full_link = package.split('git+')[1] if '#' in package_full_link: package_link, extra_data = package_full_link.split('#') else: package_link, extra_data = package_full_link, None package_link, branch = package_link.split('@') # Split the branches to see if there are multiple checkouts/branches git_branches, int_branch = self.split_git_branches(git_branch=branch) if len(git_branches) > 1 or 'refs/changes' in branch: branch = int_branch package_name = utils.git_pip_link_parse(repo=package)[0] repo_location = os.path.join(self.args['git_repo_path'], package_name) if extra_data and 'subdirectory' in extra_data: package_subdir = extra_data.split('subdirectory=')[1].split('&')[0] git_package_location = os.path.join(repo_location, package_subdir) else: git_package_location = repo_location # Check that setuptools is available setup_py = 'setup.py' setup_file = os.path.join(git_package_location, setup_py) remove_extra_setup = False if os.path.isfile(setup_file): with open(setup_file, 'r') as f: setup_file_contents = [ i for i in f.readlines() if i if not i.startswith('#') ] for i in setup_file_contents: if 'setuptools' in i: break else: for line in setup_file_contents: if line.startswith('import'): pass elif line.startswith('from'): pass else: index = setup_file_contents.index(line) break else: index = 0 setup_file_contents.insert(index, 'import setuptools') setup_py = '%s2' % setup_file remove_extra_setup = True with open(setup_py, 'w') as sf: sf.writelines(setup_file_contents) try: with utils.ChangeDir(git_package_location): # Checkout the given branch checkout_command = ['git', 'checkout', "'%s'" % branch] self._run_command(command=checkout_command) # Build the wheel using `python setup.py` build_command = [ 'python', setup_py, 'bdist_wheel', '--dist-dir', self.args['build_output'], '--bdist-dir', self.args['build_dir'] ] self._run_command(command=build_command) LOG.debug('Build Success for: "%s"', package) finally: utils.remove_dirs(directory=self.args['build_dir']) if remove_extra_setup: os.remove(setup_py)
def _pip_build_wheels(self, package=None, packages_file=None, no_links=False, retry=False): """Create a python wheel. The base command will timeout in 120 seconds and will create the wheels within a defined build output directory, will allow external packages, and will source the build output directory as the first link directory to look through for already build wheels. the argument options will enable no dependencies build, setting a build directory otherwise a temporary directory will be used, setting an additional link directory, setting extra link directories, changing the defaul pip index URL, adding an extra pip index URL, and enabling verbose mode. :param package: Name of a particular package to build. :type package: ``str`` :param packages_file: $PATH to the file which contains a list of packages to build. :type packages_file: ``str`` :param no_links: Enable / Disable add on links when building the wheel. :type no_links: ``bol`` :param retry: Enable retry mode. :type retry: ``bol`` """ command = [ 'pip', 'wheel', '--timeout', '120', '--wheel-dir', self.args['build_output'], '--allow-all-external' ] if self.args['pip_pre']: command.append('--pre') if not no_links: if self.args['pip_extra_link_dirs']: for link in self.args['pip_extra_link_dirs']: command.extend(['--find-links', link]) if self.args['pip_no_deps']: command.append('--no-deps') else: if self.args['pip_index']: command.extend(['--index-url', self.args['pip_index']]) domain = urlparse.urlparse(self.args['pip_index']) command.extend(['--trusted-host', domain.hostname]) if self.args['pip_extra_index']: command.extend( ['--extra-index-url', self.args['pip_extra_index']]) domain = urlparse.urlparse(self.args['pip_extra_index']) command.extend(['--trusted-host', domain.hostname]) if self.args['pip_no_index']: command.append('--no-index') if self.args['build_dir']: build_dir = self.args['build_dir'] command.extend(['--build', build_dir]) else: build_dir = tempfile.mkstemp(prefix='orb_') command.extend(['--build', build_dir]) if self.args['debug'] is True: command.append('--verbose') if packages_file: command.extend(['--requirement', packages_file]) else: command.append('"%s"' % utils.stip_quotes(item=package)) try: self._run_command(command=command) except (IOError, OSError) as exp: # If retry mode is enabled and there's an exception fail if retry: raise utils.AError( 'Failed to process wheel build: "%s", other data: "%s"', package or packages_file, str(exp)) LOG.warn( 'Failed to process wheel build: "%s", other data: "%s"' ' Trying again without defined link lookups.', package or packages_file, str(exp)) # Remove the build directory when failed. utils.remove_dirs(build_dir) # Rerun wheel builder in retry mode without links build_wheel_args = {'no_links': True, 'retry': True} if package: build_wheel_args['package'] = package else: build_wheel_args['packages_file'] = packages_file self._pip_build_wheels(**build_wheel_args) else: LOG.debug('Build Success for: "%s"', package or packages_file) finally: utils.remove_dirs(directory=build_dir)