def _pop_branches(self, release):
"""Remove requirement items that are within a branch list.
:param release: name of link that is pip installable.
:type release: ``str``
"""
name = utils.git_pip_link_parse(repo=release)[0]
self._pop_items(
found_repos=[i for i in self.branches if utils.git_pip_link_parse(repo=i)[0] == name], list_items="branches"
)
def _pop_branches(self, release):
"""Remove requirement items that are within a branch list.
:param release: name of link that is pip installable.
:type release: ``str``
"""
name = utils.git_pip_link_parse(repo=release)[0]
self._pop_items(found_repos=[
i for i in self.branches
if utils.git_pip_link_parse(repo=i)[0] == name
],
list_items='branches')
def organize_data(args):
"""Return a package building report.
:param args: Parsed arguments in dictionary format.
:type args: ``dict``
:return: ``dict``
"""
repo_data = dict()
if args["parsed_command"] == "create-report":
git_repos = package_processing(args, repo_data)
else:
git_repos = processing_report(args)
LOG.debug("Git repos: %s", git_repos)
for repo in [i for i in git_repos if i]:
name, branch, plugin_path, url, o_data = utils.git_pip_link_parse(repo)
if plugin_path:
name = "%s_plugin_pkg_%s" % (name, os.path.basename(plugin_path))
_repo_data = repo_data[name] = dict()
_repo_data["name"] = name
_repo_data["branch"] = branch
_repo_data["plugin_path"] = plugin_path
_repo_data["git_url"] = url
_repo_data["original_data"] = o_data
return repo_data
def organize_data(args):
"""Return a package building report.
:param args: Parsed arguments in dictionary format.
:type args: ``dict``
:return: ``dict``
"""
repo_data = dict()
if args['parsed_command'] == 'create-report':
git_repos = package_processing(args, repo_data)
else:
git_repos = processing_report(args)
LOG.debug('Git repos: %s', git_repos)
for repo in [i for i in git_repos if i]:
name, branch, plugin_path, url, o_data = utils.git_pip_link_parse(repo)
if plugin_path:
name = '%s_plugin_pkg_%s' % (name, os.path.basename(plugin_path))
_repo_data = repo_data[name] = dict()
_repo_data['name'] = name
_repo_data['branch'] = branch
_repo_data['plugin_path'] = plugin_path
_repo_data['git_url'] = url
_repo_data['original_data'] = o_data
return repo_data
def _pop_requirements(self, release):
"""Remove requirement items that are within a requirements list.
:param release: name of link that is pip installable.
:type release: ``str``
"""
name = utils.git_pip_link_parse(repo=release)[0]
self._pop_items(
found_repos=[
i for i in self.requirements
if self._requirement_name(i)[0] == name
if self._requirement_name(i)[0] == name.split('.git')[0]
],
list_items='requirements'
)
def define_new_repo(self, repo):
"""From a repo entry return a dict object with its data.
:param repo: repository string
:type repo: ``str``
:returns: ``dict``
"""
name, branch, plugin_path, url, o_data = utils.git_pip_link_parse(repo)
if plugin_path:
name = '%s_plugin_pkg_%s' % (name, os.path.basename(plugin_path))
# Process the new requirement item
return {
'name': name,
'branch': branch,
'plugin_path': plugin_path,
'git_url': url,
'original_data': o_data
}
def _package_clean(self, package, files):
"""Remove links for a given package name if found.
This method will index the provided link directory and remove any items
that match the name of the package.
:param package: Name of a particular package to build.
:type package: ``str``
"""
# Set the name of the package from an expected type, git+ or string.
if "git+" in package:
name = utils.git_pip_link_parse(repo=package)[0]
else:
name = self._requirement_name(package)[0]
name = name.replace("-", "_").lower()
LOG.debug("Checking for package name [ %s ] in link directory.", name)
for file_name in files:
base_file_name = os.path.basename(file_name).split("-")[0].lower()
if name == base_file_name:
LOG.info('Removed link item from cleanup "%s"', file_name)
os.remove(file_name)
files.remove(file_name)
def _package_clean(self, package, files):
"""Remove links for a given package name if found.
This method will index the provided link directory and remove any items
that match the name of the package.
:param package: Name of a particular package to build.
:type package: ``str``
"""
# Set the name of the package from an expected type, git+ or string.
if 'git+' in package:
name = utils.git_pip_link_parse(repo=package)[0]
else:
name = self._requirement_name(package)[0]
name = name.replace('-', '_').lower()
LOG.debug('Checking for package name [ %s ] in link directory.', name)
for file_name in files:
base_file_name = os.path.basename(file_name).split('-')[0].lower()
if name == base_file_name:
LOG.info('Removed link item from cleanup "%s"', file_name)
os.remove(file_name)
files.remove(file_name)
def _setup_build_wheels(self, package):
"""Create a Python wheel using a git with subdirectories.
The method will clone the source into place, move to the designated
sub directory and create a python wheel from the designated
subdirectory.
:param package: Name of a particular package to build.
:type package: ``str``
"""
package_full_link = package.split("git+")[1]
if "#" in package_full_link:
package_link, extra_data = package_full_link.split("#")
else:
package_link, extra_data = package_full_link, None
package_link, branch = package_link.split("@")
# Split the branches to see if there are multiple checkouts/branches
git_branches, int_branch = self.split_git_branches(git_branch=branch)
if len(git_branches) > 1 or "refs/changes" in branch:
branch = int_branch
package_name = utils.git_pip_link_parse(repo=package)[0]
repo_location = os.path.join(self.args["git_repo_path"], package_name)
if extra_data and "subdirectory" in extra_data:
package_subdir = extra_data.split("subdirectory=")[1].split("&")[0]
git_package_location = os.path.join(repo_location, package_subdir)
else:
git_package_location = repo_location
# Check that setuptools is available
setup_py = "setup.py"
setup_file = os.path.join(git_package_location, setup_py)
remove_extra_setup = False
if os.path.isfile(setup_file):
with open(setup_file, "r") as f:
setup_file_contents = [i for i in f.readlines() if i if not i.startswith("#")]
for i in setup_file_contents:
if "setuptools" in i:
break
else:
for line in setup_file_contents:
if line.startswith("import"):
pass
elif line.startswith("from"):
pass
else:
index = setup_file_contents.index(line)
break
else:
index = 0
setup_file_contents.insert(index, "import setuptools")
setup_py = "%s2" % setup_file
remove_extra_setup = True
with open(setup_py, "w") as sf:
sf.writelines(setup_file_contents)
try:
with utils.ChangeDir(git_package_location):
# Checkout the given branch
checkout_command = ["git", "checkout", "'%s'" % branch]
self._run_command(command=checkout_command)
# Build the wheel using `python setup.py`
build_command = [
"python",
setup_py,
"bdist_wheel",
"--dist-dir",
self.args["build_output"],
"--bdist-dir",
self.args["build_dir"],
]
self._run_command(command=build_command)
LOG.debug('Build Success for: "%s"', package)
finally:
utils.remove_dirs(directory=self.args["build_dir"])
if remove_extra_setup:
os.remove(setup_py)
def _setup_build_wheels(self, package):
"""Create a Python wheel using a git with subdirectories.
The method will clone the source into place, move to the designated
sub directory and create a python wheel from the designated
subdirectory.
:param package: Name of a particular package to build.
:type package: ``str``
"""
try:
package_full_link = package.split('git+')[1]
except IndexError:
return self._pip_build_wheels(package=package)
if '#' in package_full_link:
package_link, extra_data = package_full_link.split('#')
else:
package_link, extra_data = package_full_link, None
package_link, branch = package_link.split('@')
# Split the branches to see if there are multiple checkouts/branches
git_branches, int_branch = self.split_git_branches(git_branch=branch)
if len(git_branches) > 1 or 'refs/changes' in branch:
branch = int_branch
package_name = utils.git_pip_link_parse(repo=package)[0]
repo_location = os.path.join(
self.args['git_repo_path'], package_name
)
if extra_data and 'subdirectory' in extra_data:
package_subdir = extra_data.split('subdirectory=')[1].split('&')[0]
git_package_location = os.path.join(
repo_location,
package_subdir
)
else:
git_package_location = repo_location
try:
with utils.ChangeDir(git_package_location):
# Checkout the given branch
checkout_command = ['git', 'checkout', "'%s'" % branch]
self._run_command(command=checkout_command)
try:
LOG.debug('Build for: "%s"', package)
self._pip_build_wheels(
package=git_package_location,
no_links=True,
constraint_file=os.path.join(
git_package_location,
'constraints.txt'
)
)
except SystemExit:
# Build the wheel using `python setup.py`
LOG.warn(
'Running subdir package build for "%s" in fall back mode',
package
)
build_command = [
'python',
'setup.py',
'bdist_wheel',
'--dist-dir',
self.args['build_output'],
'--bdist-dir',
self.args['build_dir']
]
with utils.ChangeDir(git_package_location):
self._run_command(command=build_command)
finally:
utils.remove_dirs(directory=self.args['build_dir'])
def _process_repo_requirements(self, repo_data, base_report_data):
"""Parse and populate requirements from within branches.
This method will populate the dictionary items that are within the
``base_branches``. While there is nothing being returned within this
method, the modifications made to the base branches will be available
to the calling method.
:param repo_data: Repository data
:type repo_data: ``dict``
:param base_report_data: Dictionary items of branches that will be
populated with information parsed within this
method.
:type base_report_data: ``dict``
"""
name = utils.git_pip_link_parse(repo=repo_data['original_data'])[0]
repo_path = os.path.join(self.args['git_repo_path'], name)
if repo_data['plugin_path']:
repo_path = os.path.join(repo_path, repo_data['plugin_path'])
with utils.ChangeDir(repo_path):
LOG.debug(
'Discovered branch "%s" for repo "%s"',
repo_data['branch'],
repo_data['name']
)
git_branches, int_branch = self.split_git_branches(
git_branch=repo_data['branch']
)
patched_from = None
if len(git_branches) > 1 or 'refs/changes' in repo_data['branch']:
repo_data['branch'] = int_branch
patched_from = True
self._run_command(command=['git', 'checkout', repo_data['branch']])
branch_data = base_report_data[repo_data['branch']] = dict()
# Record the items that make up a patched branch
if patched_from:
branch_data['patched_from'] = git_branches
branch_reqs = branch_data['requirements'] = dict()
if 'yaprtignorerequirements=true' in repo_data['original_data']:
requirement_files = list()
else:
requirement_files = yaprt.REQUIREMENTS_FILE_TYPES
for type_name, file_name in requirement_files:
file_path = os.path.join(repo_path, file_name)
if os.path.isfile(file_path):
repo_data['file'] = file_name
with open(file_path, 'r') as f:
_file_requirements = f.readlines()
# If the requirement file has a -e item within it treat
# it like a local subdirectory plugin and process it.
_requirements = list()
for item in _file_requirements:
requirement = item.split('#')[0].strip()
if requirement.startswith('-e'):
if requirement.endswith('.'): # skip if "-e ."
continue
elif 'git+' in item:
repo_str = item.split('-e')[-1].strip()
self.process_repo(
repo=self.define_new_repo(
repo=repo_str
)
)
else:
self._process_sub_plugin(
requirement=requirement,
repo_data=repo_data
)
else:
_requirements.append(requirement)
_requirements = [
i.split('#')[0].strip() for i in _requirements
if not i.startswith('#')
if i.strip()
]
LOG.debug('Found requirements: %s', _requirements)
if _requirements:
branch_reqs[type_name] = sorted(_requirements)
setup_file_path = os.path.join(repo_path, 'setup.py')
if os.path.isfile(setup_file_path):
branch_data['pip_install_url'] = repo_data['original_data']
def _setup_build_wheels(self, package):
"""Create a Python wheel using a git with subdirectories.
The method will clone the source into place, move to the designated
sub directory and create a python wheel from the designated
subdirectory.
:param package: Name of a particular package to build.
:type package: ``str``
"""
package_full_link = package.split('git+')[1]
if '#' in package_full_link:
package_link, extra_data = package_full_link.split('#')
else:
package_link, extra_data = package_full_link, None
package_link, branch = package_link.split('@')
# Split the branches to see if there are multiple checkouts/branches
git_branches, int_branch = self.split_git_branches(git_branch=branch)
if len(git_branches) > 1 or 'refs/changes' in branch:
branch = int_branch
package_name = utils.git_pip_link_parse(repo=package)[0]
repo_location = os.path.join(self.args['git_repo_path'], package_name)
if extra_data and 'subdirectory' in extra_data:
package_subdir = extra_data.split('subdirectory=')[1].split('&')[0]
git_package_location = os.path.join(repo_location, package_subdir)
else:
git_package_location = repo_location
# Check that setuptools is available
setup_py = 'setup.py'
setup_file = os.path.join(git_package_location, setup_py)
remove_extra_setup = False
if os.path.isfile(setup_file):
with open(setup_file, 'r') as f:
setup_file_contents = [
i for i in f.readlines() if i if not i.startswith('#')
]
for i in setup_file_contents:
if 'setuptools' in i:
break
else:
for line in setup_file_contents:
if line.startswith('import'):
pass
elif line.startswith('from'):
pass
else:
index = setup_file_contents.index(line)
break
else:
index = 0
setup_file_contents.insert(index, 'import setuptools')
setup_py = '%s2' % setup_file
remove_extra_setup = True
with open(setup_py, 'w') as sf:
sf.writelines(setup_file_contents)
try:
with utils.ChangeDir(git_package_location):
# Checkout the given branch
checkout_command = ['git', 'checkout', "'%s'" % branch]
self._run_command(command=checkout_command)
# Build the wheel using `python setup.py`
build_command = [
'python', setup_py, 'bdist_wheel', '--dist-dir',
self.args['build_output'], '--bdist-dir',
self.args['build_dir']
]
self._run_command(command=build_command)
LOG.debug('Build Success for: "%s"', package)
finally:
utils.remove_dirs(directory=self.args['build_dir'])
if remove_extra_setup:
os.remove(setup_py)
def _process_repo_requirements(self, repo_data, base_report_data):
"""Parse and populate requirements from within branches.
This method will populate the dictionary items that are within the
``base_branches``. While there is nothing being returned within this
method, the modifications made to the base branches will be available
to the calling method.
:param repo_data: Repository data
:type repo_data: ``dict``
:param base_report_data: Dictionary items of branches that will be
populated with information parsed within this
method.
:type base_report_data: ``dict``
"""
name = utils.git_pip_link_parse(repo=repo_data['original_data'])[0]
repo_path = os.path.join(self.args['git_repo_path'], name)
if repo_data['plugin_path']:
repo_path = os.path.join(repo_path, repo_data['plugin_path'])
with utils.ChangeDir(repo_path):
LOG.debug(
'Discovered branch "%s" for repo "%s"',
repo_data['branch'],
repo_data['name']
)
git_branches, int_branch = self.split_git_branches(
git_branch=repo_data['branch']
)
patched_from = None
if len(git_branches) > 1 or 'refs/changes' in repo_data['branch']:
repo_data['branch'] = int_branch
patched_from = True
self._run_command(command=['git', 'checkout', repo_data['branch']])
branch_data = base_report_data[repo_data['branch']] = dict()
# Record the items that make up a patched branch
if patched_from:
branch_data['patched_from'] = git_branches
branch_reqs = branch_data['requirements'] = dict()
if 'yaprtignorerequirements=true' in repo_data['original_data']:
requirement_files = list()
else:
requirement_files = yaprt.REQUIREMENTS_FILE_TYPES
for type_name, file_name in requirement_files:
file_path = os.path.join(repo_path, file_name)
if os.path.isfile(file_path):
repo_data['file'] = file_name
with open(file_path, 'r') as f:
_file_requirements = f.readlines()
# If the requirement file has a -e item within it treat
# it like a local subdirectory plugin and process it.
_requirements = list()
for item in _file_requirements:
requirement = item.split('#')[0].strip()
if requirement.startswith('-e'):
if requirement.endswith('.'): # skip if "-e ."
continue
elif 'git+' in item:
repo_str = item.split('-e')[-1].strip()
self.process_repo(
repo=self.define_new_repo(
repo=repo_str
)
)
else:
self._process_sub_plugin(
requirement=requirement,
repo_data=repo_data
)
else:
_requirements.append(requirement)
_requirements = [
i.split('#')[0].strip() for i in _requirements
if not i.startswith('#')
if i.strip()
]
LOG.debug('Found requirements: %s', _requirements)
if _requirements:
branch_reqs[type_name] = sorted(_requirements)
setup_file_path = os.path.join(repo_path, 'setup.py')
if os.path.isfile(setup_file_path):
branch_data['pip_install_url'] = repo_data['original_data']
