Exemplo n.º 1
0
    def _store_git_repos(self, git_repo, git_branch):
        """Clone and or update all git repositories.

        :param git_repo: URL for git repo
        :type git_repo: ``str``
        :param git_branch: Branch for git repo
        :type git_branch: ``str``
        """
        # Set the repo name to the base name of the git_repo variable.
        repo_name = os.path.basename(git_repo)

        # Set the git repo path.
        repo_path_name = os.path.join(self.args['git_repo_path'], repo_name)

        # If there is no .git dir remove the target and re-clone
        if not os.path.isdir(os.path.join(repo_path_name, '.git')):
            # If the directory exists update
            if os.path.isdir(repo_path_name):
                utils.remove_dirs(directory=repo_path_name)

            # Clone the main repos
            self._run_clone(git_repo, repo_path_name)

        # Temporarily change the directory to the repo path.
        with utils.ChangeDir(target_dir=repo_path_name):
            self._run_update(git_repo=git_repo, git_branch=git_branch)
Exemplo n.º 2
0
    def build_wheels(self, packages, clean_first=False, force_iterate=False):
        """Create python wheels from a list of packages.

        This method will build all of the wheels from a list of packages. Once
        the loop is completed the wheels items will be moved to the storage
        pool location. Upon the completion of the method the ``build_output``
        directory will be removed.

        :param packages: List of packages to build.
        :type packages: ``list``
        :param clean_first: Enable a search and clean for existing package
        :type clean_first: ``bol``
        :param force_iterate: Force package iteration.
        :type force_iterate: ``bol``
        """
        try:
            if clean_first and self.args["link_dir"]:
                files = utils.get_file_names(self.args["link_dir"])
                for package in packages:
                    self._package_clean(package=package, files=files)

            if self.args["pip_bulk_operation"] and not force_iterate:
                req_file = os.path.join(self.args["link_dir"], "build_reqs.txt")
                LOG.info('Requirement file being written: "%s"', req_file)
                self.shell_cmds.mkdir_p(path=os.path.dirname(req_file))
                with open(req_file, "wb") as f:
                    f.writelines(["%s\n" % i for i in packages])

                self._pip_build_wheels(packages_file=req_file)
            else:
                for package in packages:
                    self._setup_build_wheels(package=package)
            self._store_pool()
        finally:
            utils.remove_dirs(directory=self.args["build_output"])
Exemplo n.º 3
0
    def build_wheels(self, packages, clean_first=False, force_iterate=False):
        """Create python wheels from a list of packages.

        This method will build all of the wheels from a list of packages. Once
        the loop is completed the wheels items will be moved to the storage
        pool location. Upon the completion of the method the ``build_output``
        directory will be removed.

        :param packages: List of packages to build.
        :type packages: ``list``
        :param clean_first: Enable a search and clean for existing package
        :type clean_first: ``bol``
        :param force_iterate: Force package iteration.
        :type force_iterate: ``bol``
        """
        try:
            if clean_first and self.args['link_dir']:
                files = utils.get_file_names(self.args['link_dir'])
                for package in packages:
                    self._package_clean(package=package, files=files)

            if self.args['pip_bulk_operation'] and not force_iterate:
                req_file = os.path.join(self.args['link_dir'],
                                        'build_reqs.txt')
                LOG.info('Requirement file being written: "%s"', req_file)
                self.shell_cmds.mkdir_p(path=os.path.dirname(req_file))
                with open(req_file, 'wb') as f:
                    f.writelines(['%s\n' % i for i in packages])

                self._pip_build_wheels(packages_file=req_file)
            else:
                for package in packages:
                    self._setup_build_wheels(package=package)
            self._store_pool()
        finally:
            utils.remove_dirs(directory=self.args['build_output'])
Exemplo n.º 4
0
    def _setup_build_wheels(self, package):
        """Create a Python wheel using a git with subdirectories.

        The method will clone the source into place, move to the designated
        sub directory and create a python wheel from the designated
        subdirectory.

        :param package: Name of a particular package to build.
        :type package: ``str``
        """
        package_full_link = package.split("git+")[1]

        if "#" in package_full_link:
            package_link, extra_data = package_full_link.split("#")
        else:
            package_link, extra_data = package_full_link, None

        package_link, branch = package_link.split("@")

        # Split the branches to see if there are multiple checkouts/branches
        git_branches, int_branch = self.split_git_branches(git_branch=branch)
        if len(git_branches) > 1 or "refs/changes" in branch:
            branch = int_branch

        package_name = utils.git_pip_link_parse(repo=package)[0]
        repo_location = os.path.join(self.args["git_repo_path"], package_name)
        if extra_data and "subdirectory" in extra_data:
            package_subdir = extra_data.split("subdirectory=")[1].split("&")[0]
            git_package_location = os.path.join(repo_location, package_subdir)
        else:
            git_package_location = repo_location

        # Check that setuptools is available
        setup_py = "setup.py"
        setup_file = os.path.join(git_package_location, setup_py)
        remove_extra_setup = False
        if os.path.isfile(setup_file):
            with open(setup_file, "r") as f:
                setup_file_contents = [i for i in f.readlines() if i if not i.startswith("#")]
                for i in setup_file_contents:
                    if "setuptools" in i:
                        break
                else:
                    for line in setup_file_contents:
                        if line.startswith("import"):
                            pass
                        elif line.startswith("from"):
                            pass
                        else:
                            index = setup_file_contents.index(line)
                            break
                    else:
                        index = 0

                    setup_file_contents.insert(index, "import setuptools")
                    setup_py = "%s2" % setup_file
                    remove_extra_setup = True
                    with open(setup_py, "w") as sf:
                        sf.writelines(setup_file_contents)
        try:
            with utils.ChangeDir(git_package_location):
                # Checkout the given branch
                checkout_command = ["git", "checkout", "'%s'" % branch]
                self._run_command(command=checkout_command)

                # Build the wheel using `python setup.py`
                build_command = [
                    "python",
                    setup_py,
                    "bdist_wheel",
                    "--dist-dir",
                    self.args["build_output"],
                    "--bdist-dir",
                    self.args["build_dir"],
                ]
                self._run_command(command=build_command)
            LOG.debug('Build Success for: "%s"', package)
        finally:
            utils.remove_dirs(directory=self.args["build_dir"])
            if remove_extra_setup:
                os.remove(setup_py)
Exemplo n.º 5
0
    def _pip_build_wheels(self, package=None, packages_file=None, no_links=False, retry=False):
        """Create a python wheel.

        The base command will timeout in 120 seconds and will create the wheels
        within a defined build output directory, will allow external packages,
        and will source the build output directory as the first link directory
        to look through for already build wheels.

        the argument options will enable no dependencies build, setting a build
        directory otherwise a temporary directory will be used, setting an
        additional link directory, setting extra link directories, changing the
        defaul pip index URL, adding an extra pip index URL, and enabling
        verbose mode.

        :param package: Name of a particular package to build.
        :type package: ``str``
        :param packages_file: $PATH to the file which contains a list of
                              packages to build.
        :type packages_file: ``str``
        :param no_links: Enable / Disable add on links when building the wheel.
        :type no_links: ``bol``
        :param retry: Enable retry mode.
        :type retry: ``bol``
        """
        command = ["pip", "wheel", "--timeout", "120", "--wheel-dir", self.args["build_output"], "--allow-all-external"]

        if self.args["pip_pre"]:
            command.append("--pre")

        if not no_links:
            if self.args["pip_extra_link_dirs"]:
                for link in self.args["pip_extra_link_dirs"]:
                    command.extend(["--find-links", link])

        if self.args["pip_no_deps"]:
            command.append("--no-deps")
        else:
            if self.args["pip_index"]:
                command.extend(["--index-url", self.args["pip_index"]])
                domain = urlparse.urlparse(self.args["pip_index"])
                command.extend(["--trusted-host", domain.hostname])

            if self.args["pip_extra_index"]:
                command.extend(["--extra-index-url", self.args["pip_extra_index"]])
                domain = urlparse.urlparse(self.args["pip_extra_index"])
                command.extend(["--trusted-host", domain.hostname])

        if self.args["pip_no_index"]:
            command.append("--no-index")

        if self.args["build_dir"]:
            build_dir = self.args["build_dir"]
            command.extend(["--build", build_dir])
        else:
            build_dir = tempfile.mkstemp(prefix="orb_")
            command.extend(["--build", build_dir])

        if self.args["debug"] is True:
            command.append("--verbose")

        if packages_file:
            command.extend(["--requirement", packages_file])
        else:
            command.append('"%s"' % utils.stip_quotes(item=package))
        try:
            self._run_command(command=command)
        except (IOError, OSError) as exp:
            # If retry mode is enabled and there's an exception fail
            if retry:
                raise utils.AError(
                    'Failed to process wheel build: "%s", other data: "%s"', package or packages_file, str(exp)
                )

            LOG.warn(
                'Failed to process wheel build: "%s", other data: "%s"' " Trying again without defined link lookups.",
                package or packages_file,
                str(exp),
            )

            # Remove the build directory when failed.
            utils.remove_dirs(build_dir)

            # Rerun wheel builder in retry mode without links
            build_wheel_args = {"no_links": True, "retry": True}
            if package:
                build_wheel_args["package"] = package
            else:
                build_wheel_args["packages_file"] = packages_file
            self._pip_build_wheels(**build_wheel_args)
        else:
            LOG.debug('Build Success for: "%s"', package or packages_file)
        finally:
            utils.remove_dirs(directory=build_dir)
Exemplo n.º 6
0
    def _setup_build_wheels(self, package):
        """Create a Python wheel using a git with subdirectories.

        The method will clone the source into place, move to the designated
        sub directory and create a python wheel from the designated
        subdirectory.

        :param package: Name of a particular package to build.
        :type package: ``str``
        """
        try:
            package_full_link = package.split('git+')[1]
        except IndexError:
            return self._pip_build_wheels(package=package)

        if '#' in package_full_link:
            package_link, extra_data = package_full_link.split('#')
        else:
            package_link, extra_data = package_full_link, None

        package_link, branch = package_link.split('@')

        # Split the branches to see if there are multiple checkouts/branches
        git_branches, int_branch = self.split_git_branches(git_branch=branch)
        if len(git_branches) > 1 or 'refs/changes' in branch:
            branch = int_branch

        package_name = utils.git_pip_link_parse(repo=package)[0]
        repo_location = os.path.join(
            self.args['git_repo_path'], package_name
        )
        if extra_data and 'subdirectory' in extra_data:
            package_subdir = extra_data.split('subdirectory=')[1].split('&')[0]
            git_package_location = os.path.join(
                repo_location,
                package_subdir
            )
        else:
            git_package_location = repo_location

        try:
            with utils.ChangeDir(git_package_location):
                # Checkout the given branch
                checkout_command = ['git', 'checkout', "'%s'" % branch]
                self._run_command(command=checkout_command)

            try:
                LOG.debug('Build for: "%s"', package)
                self._pip_build_wheels(
                    package=git_package_location,
                    no_links=True,
                    constraint_file=os.path.join(
                        git_package_location,
                        'constraints.txt'
                    )
                )
            except SystemExit:
                # Build the wheel using `python setup.py`
                LOG.warn(
                    'Running subdir package build for "%s" in fall back mode',
                    package
                )
                build_command = [
                    'python',
                    'setup.py',
                    'bdist_wheel',
                    '--dist-dir',
                    self.args['build_output'],
                    '--bdist-dir',
                    self.args['build_dir']
                ]
                with utils.ChangeDir(git_package_location):
                    self._run_command(command=build_command)
        finally:
            utils.remove_dirs(directory=self.args['build_dir'])
Exemplo n.º 7
0
    def _pip_build_wheels(self, package=None, packages_file=None,
                          no_links=False, retry=False, constraint_file=None):
        """Create a python wheel.

        The base command will timeout in 120 seconds and will create the wheels
        within a defined build output directory, will allow external packages,
        and will source the build output directory as the first link directory
        to look through for already build wheels.

        the argument options will enable no dependencies build, setting a build
        directory otherwise a temporary directory will be used, setting an
        additional link directory, setting extra link directories, changing the
        defaul pip index URL, adding an extra pip index URL, and enabling
        verbose mode.

        :param package: Name of a particular package to build.
        :type package: ``str``
        :param packages_file: $PATH to the file which contains a list of
                              packages to build.
        :type packages_file: ``str``
        :param no_links: Enable / Disable add on links when building the wheel.
        :type no_links: ``bol``
        :param retry: Enable retry mode.
        :type retry: ``bol``
        :param constraint_file: Full path to a file containing project
                                constraints for packages.
        :type constraint_file: ``str``
        """
        command = [
            'pip',
            'wheel',
            '--timeout',
            '120',
            '--wheel-dir',
            self.args['build_output'],
            '--allow-all-external'
        ]

        if constraint_file and os.path.isfile(constraint_file):
            command.extend(['--constraint', constraint_file])

        if self.args['pip_pre']:
            command.append('--pre')

        if not no_links:
            if self.args['pip_extra_link_dirs']:
                for link in self.args['pip_extra_link_dirs']:
                    command.extend(['--find-links', link])

        if self.args['pip_no_deps']:
            command.append('--no-deps')
        else:
            if self.args['pip_index']:
                command.extend(['--index-url', self.args['pip_index']])
                domain = urlparse.urlparse(self.args['pip_index'])
                command.extend(['--trusted-host', domain.hostname])

            if self.args['pip_extra_index']:
                command.extend(
                    ['--extra-index-url', self.args['pip_extra_index']]
                )
                domain = urlparse.urlparse(self.args['pip_extra_index'])
                command.extend(['--trusted-host', domain.hostname])

        if self.args['pip_no_index']:
            command.append('--no-index')

        if self.args['build_dir']:
            build_dir = self.args['build_dir']
            command.extend(['--build', build_dir])
        else:
            build_dir = tempfile.mkstemp(prefix='orb_')
            command.extend(['--build', build_dir])

        if self.args['debug'] is True:
            command.append('--verbose')

        if packages_file:
            command.extend(['--requirement', packages_file])
        else:
            command.append('"%s"' % utils.stip_quotes(item=package))
        try:
            self._run_command(command=command)
        except (IOError, OSError) as exp:
            # If retry mode is enabled and there's an exception fail
            if retry:
                raise utils.AError(
                    'Failed to process wheel build: "%s", other data: "%s"',
                    package or packages_file,
                    str(exp)
                )

            LOG.warn(
                'Failed to process wheel build: "%s", other data: "%s"'
                ' Trying again without defined link lookups.',
                package or packages_file,
                str(exp)
            )

            # Remove the build directory when failed.
            utils.remove_dirs(build_dir)

            # Rerun wheel builder in retry mode without links
            build_wheel_args = {'no_links': True, 'retry': True}
            if package:
                build_wheel_args['package'] = package
            else:
                build_wheel_args['packages_file'] = packages_file
            self._pip_build_wheels(**build_wheel_args)
        else:
            LOG.debug('Build Success for: "%s"', package or packages_file)
        finally:
            utils.remove_dirs(directory=build_dir)
Exemplo n.º 8
0
    def _setup_build_wheels(self, package):
        """Create a Python wheel using a git with subdirectories.

        The method will clone the source into place, move to the designated
        sub directory and create a python wheel from the designated
        subdirectory.

        :param package: Name of a particular package to build.
        :type package: ``str``
        """
        package_full_link = package.split('git+')[1]

        if '#' in package_full_link:
            package_link, extra_data = package_full_link.split('#')
        else:
            package_link, extra_data = package_full_link, None

        package_link, branch = package_link.split('@')

        # Split the branches to see if there are multiple checkouts/branches
        git_branches, int_branch = self.split_git_branches(git_branch=branch)
        if len(git_branches) > 1 or 'refs/changes' in branch:
            branch = int_branch

        package_name = utils.git_pip_link_parse(repo=package)[0]
        repo_location = os.path.join(self.args['git_repo_path'], package_name)
        if extra_data and 'subdirectory' in extra_data:
            package_subdir = extra_data.split('subdirectory=')[1].split('&')[0]
            git_package_location = os.path.join(repo_location, package_subdir)
        else:
            git_package_location = repo_location

        # Check that setuptools is available
        setup_py = 'setup.py'
        setup_file = os.path.join(git_package_location, setup_py)
        remove_extra_setup = False
        if os.path.isfile(setup_file):
            with open(setup_file, 'r') as f:
                setup_file_contents = [
                    i for i in f.readlines() if i if not i.startswith('#')
                ]
                for i in setup_file_contents:
                    if 'setuptools' in i:
                        break
                else:
                    for line in setup_file_contents:
                        if line.startswith('import'):
                            pass
                        elif line.startswith('from'):
                            pass
                        else:
                            index = setup_file_contents.index(line)
                            break
                    else:
                        index = 0

                    setup_file_contents.insert(index, 'import setuptools')
                    setup_py = '%s2' % setup_file
                    remove_extra_setup = True
                    with open(setup_py, 'w') as sf:
                        sf.writelines(setup_file_contents)
        try:
            with utils.ChangeDir(git_package_location):
                # Checkout the given branch
                checkout_command = ['git', 'checkout', "'%s'" % branch]
                self._run_command(command=checkout_command)

                # Build the wheel using `python setup.py`
                build_command = [
                    'python', setup_py, 'bdist_wheel', '--dist-dir',
                    self.args['build_output'], '--bdist-dir',
                    self.args['build_dir']
                ]
                self._run_command(command=build_command)
            LOG.debug('Build Success for: "%s"', package)
        finally:
            utils.remove_dirs(directory=self.args['build_dir'])
            if remove_extra_setup:
                os.remove(setup_py)
Exemplo n.º 9
0
    def _pip_build_wheels(self,
                          package=None,
                          packages_file=None,
                          no_links=False,
                          retry=False):
        """Create a python wheel.

        The base command will timeout in 120 seconds and will create the wheels
        within a defined build output directory, will allow external packages,
        and will source the build output directory as the first link directory
        to look through for already build wheels.

        the argument options will enable no dependencies build, setting a build
        directory otherwise a temporary directory will be used, setting an
        additional link directory, setting extra link directories, changing the
        defaul pip index URL, adding an extra pip index URL, and enabling
        verbose mode.

        :param package: Name of a particular package to build.
        :type package: ``str``
        :param packages_file: $PATH to the file which contains a list of
                              packages to build.
        :type packages_file: ``str``
        :param no_links: Enable / Disable add on links when building the wheel.
        :type no_links: ``bol``
        :param retry: Enable retry mode.
        :type retry: ``bol``
        """
        command = [
            'pip', 'wheel', '--timeout', '120', '--wheel-dir',
            self.args['build_output'], '--allow-all-external'
        ]

        if self.args['pip_pre']:
            command.append('--pre')

        if not no_links:
            if self.args['pip_extra_link_dirs']:
                for link in self.args['pip_extra_link_dirs']:
                    command.extend(['--find-links', link])

        if self.args['pip_no_deps']:
            command.append('--no-deps')
        else:
            if self.args['pip_index']:
                command.extend(['--index-url', self.args['pip_index']])
                domain = urlparse.urlparse(self.args['pip_index'])
                command.extend(['--trusted-host', domain.hostname])

            if self.args['pip_extra_index']:
                command.extend(
                    ['--extra-index-url', self.args['pip_extra_index']])
                domain = urlparse.urlparse(self.args['pip_extra_index'])
                command.extend(['--trusted-host', domain.hostname])

        if self.args['pip_no_index']:
            command.append('--no-index')

        if self.args['build_dir']:
            build_dir = self.args['build_dir']
            command.extend(['--build', build_dir])
        else:
            build_dir = tempfile.mkstemp(prefix='orb_')
            command.extend(['--build', build_dir])

        if self.args['debug'] is True:
            command.append('--verbose')

        if packages_file:
            command.extend(['--requirement', packages_file])
        else:
            command.append('"%s"' % utils.stip_quotes(item=package))
        try:
            self._run_command(command=command)
        except (IOError, OSError) as exp:
            # If retry mode is enabled and there's an exception fail
            if retry:
                raise utils.AError(
                    'Failed to process wheel build: "%s", other data: "%s"',
                    package or packages_file, str(exp))

            LOG.warn(
                'Failed to process wheel build: "%s", other data: "%s"'
                ' Trying again without defined link lookups.', package
                or packages_file, str(exp))

            # Remove the build directory when failed.
            utils.remove_dirs(build_dir)

            # Rerun wheel builder in retry mode without links
            build_wheel_args = {'no_links': True, 'retry': True}
            if package:
                build_wheel_args['package'] = package
            else:
                build_wheel_args['packages_file'] = packages_file
            self._pip_build_wheels(**build_wheel_args)
        else:
            LOG.debug('Build Success for: "%s"', package or packages_file)
        finally:
            utils.remove_dirs(directory=build_dir)