def _get_source_from_conda_package(pkg_dir):
"""
Download the source for a conda package
"""
#pylint: disable=import-outside-toplevel
import conda_build.source
source_folder = os.path.join(utils.TMP_LICENSE_DIR,
os.path.basename(pkg_dir))
if not os.path.exists(source_folder):
os.makedirs(source_folder)
# Find the recipe's meta.yaml file and download the values within the "source" field.
with open(os.path.join(pkg_dir, "info", "recipe",
"meta.yaml")) as file_stream:
recipe_data = open_ce.yaml_utils.load(file_stream)
if not recipe_data.get("source"):
return source_folder
sources = recipe_data["source"]
if not isinstance(sources, list):
sources = [sources]
for source in sources:
# If the source comes from a url, use conda-build's download_to_cache function.
if source.get("url"):
try:
local_path, _ = conda_build.source.download_to_cache(
source_folder, pkg_dir, source, False)
_extract(local_path, source_folder)
except RuntimeError:
print("Unable to download source for " +
os.path.basename(pkg_dir))
elif source.get("git_url"):
git_url = source["git_url"]
try:
utils.git_clone(git_url, source.get("git_rev"),
source_folder)
except OpenCEError as error:
try:
# If the URL is from a private GIT server, try and use an equivalent URL on GitHub
parsed_url = urllib.parse.urlsplit(git_url)
netloc = parsed_url.netloc.split(".")
if netloc[0] == "git":
parsed_url = parsed_url._replace(
netloc="github.com")
parsed_url = parsed_url._replace(path=os.path.join(
netloc[1], os.path.basename(parsed_url.path)))
git_url = urllib.parse.urlunsplit(parsed_url)
utils.git_clone(git_url, source.get("git_rev"),
source_folder)
else:
raise error
except OpenCEError:
print("Unable to clone source for " +
os.path.basename(pkg_dir))
return source_folder
def add_licenses_from_info_file(self, license_file):
"""
Add all of the licensing information from an info file.
"""
if not os.path.exists(license_file):
return
with open(license_file) as file_stream:
license_data = yaml.safe_load(file_stream)
utils.validate_dict_schema(license_data, _OPEN_CE_INFO_SCHEMA)
for package in license_data.get(Key.third_party_packages.name, []):
source_folder = os.path.join(utils.TMP_LICENSE_DIR,
package[Key.name.name] + "-" + str(package[Key.version.name]))
if not os.path.exists(source_folder):
os.makedirs(source_folder)
urls = [package[Key.license_url.name]] if Key.license_url.name in package else package[Key.url.name]
# Download the source from each URL
for url in urls:
if url.endswith(".git"):
try:
utils.git_clone(url, package[Key.version.name], source_folder)
except OpenCEError:
print("Unable to clone source for " + package[Key.name.name])
else:
try:
res = requests.get(url)
local_path = os.path.join(source_folder, os.path.basename(url))
with open(local_path, 'wb') as file_stream:
file_stream.write(res.content)
_extract(local_path, source_folder)
#pylint: disable=broad-except
except Exception:
print("Unable to download source for " + package[Key.name.name])
# Find every license file within the downloaded source
license_files = _find_license_files(source_folder)
# Get copyright information from the downloaded source (unless the copyright string is provided)
if Key.copyright_string.name in package:
copyright_string = [package[Key.copyright_string.name]]
else:
copyright_string = _get_copyrights_from_files(license_files)
info = LicenseGenerator.LicenseInfo(package[Key.name.name],
package[Key.version.name],
package[Key.url.name],
package[Key.license.name],
copyright_string)
self._licenses.add(info)
def _clone_repo(self, git_url, repo_dir, env_config_data, package):
"""
Clone the git repo at repository.
"""
# Priority is given to command line specified tag, if it is not
# specified then package specific tag, and when even that is not specified
# then top level git tag specified for env in the env file. And if nothing is
# at all specified then fall back to default branch of the repo.
git_tag = self._git_tag_for_env
git_tag_for_package = None
if git_tag is None:
git_tag_for_package = package.get(env_config.Key.git_tag.name,
None) if package else None
if git_tag_for_package:
git_tag = git_tag_for_package
else:
git_tag = env_config_data.get(
env_config.Key.git_tag_for_env.name,
None) if env_config_data else None
clone_successful = utils.git_clone(
git_url, git_tag, repo_dir, self._git_up_to_date
and not git_tag_for_package)
if clone_successful:
patches = package.get(env_config.Key.patches.name,
[]) if package else []
if len(patches) > 0:
cur_dir = os.getcwd()
os.chdir(repo_dir)
for patch in patches:
if os.path.isabs(patch) and os.path.exists(patch):
patch_file = patch
else:
# Look for patch relative to where the Open-CE environment file is
patch_file = os.path.join(
os.path.dirname(
env_config_data.get(
env_config.Key.opence_env_file_path.name)),
patch)
if utils.is_url(patch_file):
patch_file = utils.download_file(patch_file)
patch_apply_cmd = "git apply {}".format(patch_file)
log.info("Patch apply command: %s", patch_apply_cmd)
patch_apply_res = os.system(patch_apply_cmd)
if patch_apply_res != 0:
os.chdir(cur_dir)
shutil.rmtree(repo_dir)
raise OpenCEError(
Error.PATCH_APPLICATION, patch,
package[env_config.Key.feedstock.name])
os.chdir(cur_dir)
def _get_licenses_from_info_file_helper(self, info):
if info in self._licenses:
return None
source_folder = os.path.join(utils.TMP_LICENSE_DIR,
info.name + "-" + str(info.version))
if not os.path.exists(source_folder):
os.makedirs(source_folder)
# Download the source from each URL
for url in info.url:
if url.endswith(".git"):
try:
utils.git_clone(url, info.version, source_folder)
except OpenCEError:
show_warning(Error.UNABLE_CLONE_SOURCE, info.name)
else:
try:
res = requests.get(url)
local_path = os.path.join(source_folder,
os.path.basename(url))
with open(local_path, 'wb') as file_stream:
file_stream.write(res.content)
_extract(local_path, source_folder)
#pylint: disable=broad-except
except Exception:
show_warning(Error.UNABLE_DOWNLOAD_SOURCE, info.name)
# Find every license file within the downloaded source
info.license_files = _find_license_files(source_folder,
info.license_files)
# Get copyright information from the downloaded source (unless the copyright string is provided)
if not info.copyrights:
info.copyrights = _get_copyrights_from_files(info.license_files)
return info
def clone_repo(git_url, repo_dir, git_tag=None):
'''Clone a repo to the given location.'''
utils.git_clone(git_url, git_tag, repo_dir)