def _pop_branches(self, release): """Remove requirement items that are within a branch list. :param release: name of link that is pip installable. :type release: ``str`` """ name = utils.git_pip_link_parse(repo=release)[0] self._pop_items( found_repos=[i for i in self.branches if utils.git_pip_link_parse(repo=i)[0] == name], list_items="branches" )
def _pop_branches(self, release): """Remove requirement items that are within a branch list. :param release: name of link that is pip installable. :type release: ``str`` """ name = utils.git_pip_link_parse(repo=release)[0] self._pop_items(found_repos=[ i for i in self.branches if utils.git_pip_link_parse(repo=i)[0] == name ], list_items='branches')
def organize_data(args): """Return a package building report. :param args: Parsed arguments in dictionary format. :type args: ``dict`` :return: ``dict`` """ repo_data = dict() if args["parsed_command"] == "create-report": git_repos = package_processing(args, repo_data) else: git_repos = processing_report(args) LOG.debug("Git repos: %s", git_repos) for repo in [i for i in git_repos if i]: name, branch, plugin_path, url, o_data = utils.git_pip_link_parse(repo) if plugin_path: name = "%s_plugin_pkg_%s" % (name, os.path.basename(plugin_path)) _repo_data = repo_data[name] = dict() _repo_data["name"] = name _repo_data["branch"] = branch _repo_data["plugin_path"] = plugin_path _repo_data["git_url"] = url _repo_data["original_data"] = o_data return repo_data
def organize_data(args): """Return a package building report. :param args: Parsed arguments in dictionary format. :type args: ``dict`` :return: ``dict`` """ repo_data = dict() if args['parsed_command'] == 'create-report': git_repos = package_processing(args, repo_data) else: git_repos = processing_report(args) LOG.debug('Git repos: %s', git_repos) for repo in [i for i in git_repos if i]: name, branch, plugin_path, url, o_data = utils.git_pip_link_parse(repo) if plugin_path: name = '%s_plugin_pkg_%s' % (name, os.path.basename(plugin_path)) _repo_data = repo_data[name] = dict() _repo_data['name'] = name _repo_data['branch'] = branch _repo_data['plugin_path'] = plugin_path _repo_data['git_url'] = url _repo_data['original_data'] = o_data return repo_data
def _pop_requirements(self, release): """Remove requirement items that are within a requirements list. :param release: name of link that is pip installable. :type release: ``str`` """ name = utils.git_pip_link_parse(repo=release)[0] self._pop_items( found_repos=[ i for i in self.requirements if self._requirement_name(i)[0] == name if self._requirement_name(i)[0] == name.split('.git')[0] ], list_items='requirements' )
def define_new_repo(self, repo): """From a repo entry return a dict object with its data. :param repo: repository string :type repo: ``str`` :returns: ``dict`` """ name, branch, plugin_path, url, o_data = utils.git_pip_link_parse(repo) if plugin_path: name = '%s_plugin_pkg_%s' % (name, os.path.basename(plugin_path)) # Process the new requirement item return { 'name': name, 'branch': branch, 'plugin_path': plugin_path, 'git_url': url, 'original_data': o_data }
def _package_clean(self, package, files): """Remove links for a given package name if found. This method will index the provided link directory and remove any items that match the name of the package. :param package: Name of a particular package to build. :type package: ``str`` """ # Set the name of the package from an expected type, git+ or string. if "git+" in package: name = utils.git_pip_link_parse(repo=package)[0] else: name = self._requirement_name(package)[0] name = name.replace("-", "_").lower() LOG.debug("Checking for package name [ %s ] in link directory.", name) for file_name in files: base_file_name = os.path.basename(file_name).split("-")[0].lower() if name == base_file_name: LOG.info('Removed link item from cleanup "%s"', file_name) os.remove(file_name) files.remove(file_name)
def _package_clean(self, package, files): """Remove links for a given package name if found. This method will index the provided link directory and remove any items that match the name of the package. :param package: Name of a particular package to build. :type package: ``str`` """ # Set the name of the package from an expected type, git+ or string. if 'git+' in package: name = utils.git_pip_link_parse(repo=package)[0] else: name = self._requirement_name(package)[0] name = name.replace('-', '_').lower() LOG.debug('Checking for package name [ %s ] in link directory.', name) for file_name in files: base_file_name = os.path.basename(file_name).split('-')[0].lower() if name == base_file_name: LOG.info('Removed link item from cleanup "%s"', file_name) os.remove(file_name) files.remove(file_name)
def _setup_build_wheels(self, package): """Create a Python wheel using a git with subdirectories. The method will clone the source into place, move to the designated sub directory and create a python wheel from the designated subdirectory. :param package: Name of a particular package to build. :type package: ``str`` """ package_full_link = package.split("git+")[1] if "#" in package_full_link: package_link, extra_data = package_full_link.split("#") else: package_link, extra_data = package_full_link, None package_link, branch = package_link.split("@") # Split the branches to see if there are multiple checkouts/branches git_branches, int_branch = self.split_git_branches(git_branch=branch) if len(git_branches) > 1 or "refs/changes" in branch: branch = int_branch package_name = utils.git_pip_link_parse(repo=package)[0] repo_location = os.path.join(self.args["git_repo_path"], package_name) if extra_data and "subdirectory" in extra_data: package_subdir = extra_data.split("subdirectory=")[1].split("&")[0] git_package_location = os.path.join(repo_location, package_subdir) else: git_package_location = repo_location # Check that setuptools is available setup_py = "setup.py" setup_file = os.path.join(git_package_location, setup_py) remove_extra_setup = False if os.path.isfile(setup_file): with open(setup_file, "r") as f: setup_file_contents = [i for i in f.readlines() if i if not i.startswith("#")] for i in setup_file_contents: if "setuptools" in i: break else: for line in setup_file_contents: if line.startswith("import"): pass elif line.startswith("from"): pass else: index = setup_file_contents.index(line) break else: index = 0 setup_file_contents.insert(index, "import setuptools") setup_py = "%s2" % setup_file remove_extra_setup = True with open(setup_py, "w") as sf: sf.writelines(setup_file_contents) try: with utils.ChangeDir(git_package_location): # Checkout the given branch checkout_command = ["git", "checkout", "'%s'" % branch] self._run_command(command=checkout_command) # Build the wheel using `python setup.py` build_command = [ "python", setup_py, "bdist_wheel", "--dist-dir", self.args["build_output"], "--bdist-dir", self.args["build_dir"], ] self._run_command(command=build_command) LOG.debug('Build Success for: "%s"', package) finally: utils.remove_dirs(directory=self.args["build_dir"]) if remove_extra_setup: os.remove(setup_py)
def _setup_build_wheels(self, package): """Create a Python wheel using a git with subdirectories. The method will clone the source into place, move to the designated sub directory and create a python wheel from the designated subdirectory. :param package: Name of a particular package to build. :type package: ``str`` """ try: package_full_link = package.split('git+')[1] except IndexError: return self._pip_build_wheels(package=package) if '#' in package_full_link: package_link, extra_data = package_full_link.split('#') else: package_link, extra_data = package_full_link, None package_link, branch = package_link.split('@') # Split the branches to see if there are multiple checkouts/branches git_branches, int_branch = self.split_git_branches(git_branch=branch) if len(git_branches) > 1 or 'refs/changes' in branch: branch = int_branch package_name = utils.git_pip_link_parse(repo=package)[0] repo_location = os.path.join( self.args['git_repo_path'], package_name ) if extra_data and 'subdirectory' in extra_data: package_subdir = extra_data.split('subdirectory=')[1].split('&')[0] git_package_location = os.path.join( repo_location, package_subdir ) else: git_package_location = repo_location try: with utils.ChangeDir(git_package_location): # Checkout the given branch checkout_command = ['git', 'checkout', "'%s'" % branch] self._run_command(command=checkout_command) try: LOG.debug('Build for: "%s"', package) self._pip_build_wheels( package=git_package_location, no_links=True, constraint_file=os.path.join( git_package_location, 'constraints.txt' ) ) except SystemExit: # Build the wheel using `python setup.py` LOG.warn( 'Running subdir package build for "%s" in fall back mode', package ) build_command = [ 'python', 'setup.py', 'bdist_wheel', '--dist-dir', self.args['build_output'], '--bdist-dir', self.args['build_dir'] ] with utils.ChangeDir(git_package_location): self._run_command(command=build_command) finally: utils.remove_dirs(directory=self.args['build_dir'])
def _process_repo_requirements(self, repo_data, base_report_data): """Parse and populate requirements from within branches. This method will populate the dictionary items that are within the ``base_branches``. While there is nothing being returned within this method, the modifications made to the base branches will be available to the calling method. :param repo_data: Repository data :type repo_data: ``dict`` :param base_report_data: Dictionary items of branches that will be populated with information parsed within this method. :type base_report_data: ``dict`` """ name = utils.git_pip_link_parse(repo=repo_data['original_data'])[0] repo_path = os.path.join(self.args['git_repo_path'], name) if repo_data['plugin_path']: repo_path = os.path.join(repo_path, repo_data['plugin_path']) with utils.ChangeDir(repo_path): LOG.debug( 'Discovered branch "%s" for repo "%s"', repo_data['branch'], repo_data['name'] ) git_branches, int_branch = self.split_git_branches( git_branch=repo_data['branch'] ) patched_from = None if len(git_branches) > 1 or 'refs/changes' in repo_data['branch']: repo_data['branch'] = int_branch patched_from = True self._run_command(command=['git', 'checkout', repo_data['branch']]) branch_data = base_report_data[repo_data['branch']] = dict() # Record the items that make up a patched branch if patched_from: branch_data['patched_from'] = git_branches branch_reqs = branch_data['requirements'] = dict() if 'yaprtignorerequirements=true' in repo_data['original_data']: requirement_files = list() else: requirement_files = yaprt.REQUIREMENTS_FILE_TYPES for type_name, file_name in requirement_files: file_path = os.path.join(repo_path, file_name) if os.path.isfile(file_path): repo_data['file'] = file_name with open(file_path, 'r') as f: _file_requirements = f.readlines() # If the requirement file has a -e item within it treat # it like a local subdirectory plugin and process it. _requirements = list() for item in _file_requirements: requirement = item.split('#')[0].strip() if requirement.startswith('-e'): if requirement.endswith('.'): # skip if "-e ." continue elif 'git+' in item: repo_str = item.split('-e')[-1].strip() self.process_repo( repo=self.define_new_repo( repo=repo_str ) ) else: self._process_sub_plugin( requirement=requirement, repo_data=repo_data ) else: _requirements.append(requirement) _requirements = [ i.split('#')[0].strip() for i in _requirements if not i.startswith('#') if i.strip() ] LOG.debug('Found requirements: %s', _requirements) if _requirements: branch_reqs[type_name] = sorted(_requirements) setup_file_path = os.path.join(repo_path, 'setup.py') if os.path.isfile(setup_file_path): branch_data['pip_install_url'] = repo_data['original_data']
def _setup_build_wheels(self, package): """Create a Python wheel using a git with subdirectories. The method will clone the source into place, move to the designated sub directory and create a python wheel from the designated subdirectory. :param package: Name of a particular package to build. :type package: ``str`` """ package_full_link = package.split('git+')[1] if '#' in package_full_link: package_link, extra_data = package_full_link.split('#') else: package_link, extra_data = package_full_link, None package_link, branch = package_link.split('@') # Split the branches to see if there are multiple checkouts/branches git_branches, int_branch = self.split_git_branches(git_branch=branch) if len(git_branches) > 1 or 'refs/changes' in branch: branch = int_branch package_name = utils.git_pip_link_parse(repo=package)[0] repo_location = os.path.join(self.args['git_repo_path'], package_name) if extra_data and 'subdirectory' in extra_data: package_subdir = extra_data.split('subdirectory=')[1].split('&')[0] git_package_location = os.path.join(repo_location, package_subdir) else: git_package_location = repo_location # Check that setuptools is available setup_py = 'setup.py' setup_file = os.path.join(git_package_location, setup_py) remove_extra_setup = False if os.path.isfile(setup_file): with open(setup_file, 'r') as f: setup_file_contents = [ i for i in f.readlines() if i if not i.startswith('#') ] for i in setup_file_contents: if 'setuptools' in i: break else: for line in setup_file_contents: if line.startswith('import'): pass elif line.startswith('from'): pass else: index = setup_file_contents.index(line) break else: index = 0 setup_file_contents.insert(index, 'import setuptools') setup_py = '%s2' % setup_file remove_extra_setup = True with open(setup_py, 'w') as sf: sf.writelines(setup_file_contents) try: with utils.ChangeDir(git_package_location): # Checkout the given branch checkout_command = ['git', 'checkout', "'%s'" % branch] self._run_command(command=checkout_command) # Build the wheel using `python setup.py` build_command = [ 'python', setup_py, 'bdist_wheel', '--dist-dir', self.args['build_output'], '--bdist-dir', self.args['build_dir'] ] self._run_command(command=build_command) LOG.debug('Build Success for: "%s"', package) finally: utils.remove_dirs(directory=self.args['build_dir']) if remove_extra_setup: os.remove(setup_py)
def _process_repo_requirements(self, repo_data, base_report_data): """Parse and populate requirements from within branches. This method will populate the dictionary items that are within the ``base_branches``. While there is nothing being returned within this method, the modifications made to the base branches will be available to the calling method. :param repo_data: Repository data :type repo_data: ``dict`` :param base_report_data: Dictionary items of branches that will be populated with information parsed within this method. :type base_report_data: ``dict`` """ name = utils.git_pip_link_parse(repo=repo_data['original_data'])[0] repo_path = os.path.join(self.args['git_repo_path'], name) if repo_data['plugin_path']: repo_path = os.path.join(repo_path, repo_data['plugin_path']) with utils.ChangeDir(repo_path): LOG.debug( 'Discovered branch "%s" for repo "%s"', repo_data['branch'], repo_data['name'] ) git_branches, int_branch = self.split_git_branches( git_branch=repo_data['branch'] ) patched_from = None if len(git_branches) > 1 or 'refs/changes' in repo_data['branch']: repo_data['branch'] = int_branch patched_from = True self._run_command(command=['git', 'checkout', repo_data['branch']]) branch_data = base_report_data[repo_data['branch']] = dict() # Record the items that make up a patched branch if patched_from: branch_data['patched_from'] = git_branches branch_reqs = branch_data['requirements'] = dict() if 'yaprtignorerequirements=true' in repo_data['original_data']: requirement_files = list() else: requirement_files = yaprt.REQUIREMENTS_FILE_TYPES for type_name, file_name in requirement_files: file_path = os.path.join(repo_path, file_name) if os.path.isfile(file_path): repo_data['file'] = file_name with open(file_path, 'r') as f: _file_requirements = f.readlines() # If the requirement file has a -e item within it treat # it like a local subdirectory plugin and process it. _requirements = list() for item in _file_requirements: requirement = item.split('#')[0].strip() if requirement.startswith('-e'): if requirement.endswith('.'): # skip if "-e ." continue elif 'git+' in item: repo_str = item.split('-e')[-1].strip() self.process_repo( repo=self.define_new_repo( repo=repo_str ) ) else: self._process_sub_plugin( requirement=requirement, repo_data=repo_data ) else: _requirements.append(requirement) _requirements = [ i.split('#')[0].strip() for i in _requirements if not i.startswith('#') if i.strip() ] LOG.debug('Found requirements: %s', _requirements) if _requirements: branch_reqs[type_name] = sorted(_requirements) setup_file_path = os.path.join(repo_path, 'setup.py') if os.path.isfile(setup_file_path): branch_data['pip_install_url'] = repo_data['original_data']