def gather_files(folder):
file_dict = {}
symlinks = {}
keep_python = get_env("CONAN_KEEP_PYTHON_FILES", False)
for root, dirs, files in walk(folder):
dirs[:] = [d for d in dirs
if d != "__pycache__"] # Avoid recursing pycache
for d in dirs:
abs_path = os.path.join(root, d)
if os.path.islink(abs_path):
rel_path = abs_path[len(folder) + 1:].replace("\\", "/")
symlinks[rel_path] = os.readlink(abs_path)
for f in files:
if discarded_file(f, keep_python):
continue
abs_path = os.path.join(root, f)
rel_path = abs_path[len(folder) + 1:].replace("\\", "/")
if os.path.exists(abs_path):
file_dict[rel_path] = abs_path
else:
if not get_env("CONAN_SKIP_BROKEN_SYMLINKS_CHECK", False):
raise ConanException(
"The file is a broken symlink, verify that "
"you are packaging the needed destination files: '%s'."
"You can skip this check adjusting the "
"'general.skip_broken_symlinks_check' at the conan.conf "
"file." % abs_path)
return file_dict, symlinks
def _process_folder(config, folder, cache, output):
if config.source_folder:
folder = os.path.join(folder, config.source_folder)
for root, dirs, files in walk(folder):
dirs[:] = [d for d in dirs if d != ".git"]
if ".git" in root:
continue
for f in files:
if f == "settings.yml":
output.info("Installing settings.yml")
settings_path = cache.settings_path
shutil.copy(os.path.join(root, f), settings_path)
elif f == "conan.conf":
output.info("Processing conan.conf")
_handle_conan_conf(cache.config, os.path.join(root, f))
elif f == "remotes.txt":
output.info("Defining remotes from remotes.txt")
_handle_remotes(cache, os.path.join(root, f))
else:
# This is ugly, should be removed in Conan 2.0
if root == folder and f in ("README.md", "LICENSE.txt"):
output.info("Skip %s" % f)
continue
relpath = os.path.relpath(root, folder)
if config.target_folder:
target_folder = os.path.join(cache.conan_folder,
config.target_folder, relpath)
else:
target_folder = os.path.join(cache.conan_folder, relpath)
mkdir(target_folder)
output.info("Copying file %s to %s" % (f, target_folder))
shutil.copy(os.path.join(root, f), target_folder)
def _process_folder(folder, client_cache, output):
for root, dirs, files in walk(folder):
for f in files:
if f == "settings.yml":
output.info("Installing settings.yml")
settings_path = client_cache.settings_path
shutil.copy(os.path.join(root, f), settings_path)
elif f == "conan.conf":
output.info("Processing conan.conf")
conan_conf = client_cache.conan_config
_handle_conan_conf(conan_conf, os.path.join(root, f))
elif f == "remotes.txt":
output.info("Defining remotes")
registry_path = client_cache.registry
_handle_remotes(registry_path, os.path.join(root, f), output)
else:
relpath = os.path.relpath(root, folder)
target_folder = os.path.join(client_cache.conan_folder, relpath)
mkdir(target_folder)
output.info("Copying file %s to %s" % (f, target_folder))
shutil.copy(os.path.join(root, f), target_folder)
for d in dirs:
if d == "profiles":
output.info("Installing profiles")
profiles_path = client_cache.profiles_path
_handle_profiles(os.path.join(root, d), profiles_path, output)
break
dirs[:] = [d for d in dirs if d not in ("profiles", ".git")]
def merge_directories(src, dst, excluded=None, symlinks=True):
dst = os.path.normpath(dst)
excluded = excluded or []
excluded = [os.path.normpath(entry) for entry in excluded]
def is_excluded(origin_path):
if origin_path == dst:
return True
rel_path = os.path.normpath(os.path.relpath(origin_path, src))
if rel_path in excluded:
return True
return False
for src_dir, dirs, files in walk(src, followlinks=True):
if is_excluded(src_dir):
dirs[:] = []
continue
# Overwriting the dirs will prevents walk to get into them
files[:] = [
d for d in files if not is_excluded(os.path.join(src_dir, d))
]
dst_dir = os.path.normpath(
os.path.join(dst, os.path.relpath(src_dir, src)))
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
for file_ in files:
src_file = os.path.join(src_dir, file_)
dst_file = os.path.join(dst_dir, file_)
if os.path.islink(src_file) and symlinks:
linkto = os.readlink(src_file)
os.symlink(linkto, dst_file)
else:
shutil.copy2(src_file, dst_file)
def excluded_files(self):
ret = []
try:
file_paths = [
os.path.normpath(
os.path.join(os.path.relpath(folder, self.folder),
el)).replace("\\", "/")
for folder, dirpaths, fs in walk(self.folder)
for el in fs + dirpaths
]
if file_paths:
p = subprocess.Popen(['git', 'check-ignore', '--stdin'],
stdout=PIPE,
stdin=PIPE,
stderr=STDOUT,
cwd=self.folder)
paths = to_file_bytes("\n".join(file_paths))
grep_stdout = decode_text(p.communicate(input=paths)[0])
ret = grep_stdout.splitlines()
except (CalledProcessError, FileNotFoundError) as e:
if self._output:
self._output.warn("Error checking excluded git files: %s. "
"Ignoring excluded files" % e)
ret = []
return ret
def _filter_files(self, src, pattern, links, excludes, ignore_case):
""" return a list of the files matching the patterns
The list will be relative path names wrt to the root src folder
"""
filenames = []
linked_folders = []
if excludes:
if not isinstance(excludes, (tuple, list)):
excludes = (excludes, )
if ignore_case:
excludes = [e.lower() for e in excludes]
else:
excludes = []
for root, subfolders, files in walk(src, followlinks=True):
if root in self._excluded:
subfolders[:] = []
continue
if links and os.path.islink(root):
linked_folders.append(os.path.relpath(root, src))
subfolders[:] = []
continue
basename = os.path.basename(root)
# Skip git or svn subfolders
if basename in [".git", ".svn"]:
subfolders[:] = []
continue
if basename == "test_package": # DO NOT export test_package/build folder
try:
subfolders.remove("build")
except:
pass
relative_path = os.path.relpath(root, src)
for exclude in excludes:
if fnmatch.fnmatch(relative_path, exclude):
subfolders[:] = []
files = []
break
for f in files:
relative_name = os.path.normpath(os.path.join(
relative_path, f))
filenames.append(relative_name)
if ignore_case:
filenames = {f.lower(): f for f in filenames}
pattern = pattern.lower()
files_to_copy = fnmatch.filter(filenames, pattern)
for exclude in excludes:
files_to_copy = [
f for f in files_to_copy if not fnmatch.fnmatch(f, exclude)
]
if ignore_case:
files_to_copy = [filenames[f] for f in files_to_copy]
return files_to_copy, linked_folders
def gather_files(folder):
file_dict = {}
symlinks = {}
for root, dirs, files in walk(folder):
dirs[:] = [d for d in dirs
if d != "__pycache__"] # Avoid recursing pycache
for d in dirs:
abs_path = os.path.join(root, d)
if os.path.islink(abs_path):
rel_path = abs_path[len(folder) + 1:].replace("\\", "/")
symlinks[rel_path] = os.readlink(abs_path)
for f in files:
if discarded_file(f):
continue
abs_path = os.path.join(root, f)
rel_path = abs_path[len(folder) + 1:].replace("\\", "/")
if os.path.exists(abs_path):
file_dict[rel_path] = abs_path
else:
raise ConanException(
"The file is a broken symlink, verify that "
"you are packaging the needed destination files: '%s'" %
abs_path)
return file_dict, symlinks
def _process_folder(folder, cache, output):
for root, dirs, files in walk(folder):
for f in files:
if f == "settings.yml":
output.info("Installing settings.yml")
settings_path = cache.settings_path
shutil.copy(os.path.join(root, f), settings_path)
elif f == "conan.conf":
output.info("Processing conan.conf")
conan_conf = cache.conan_config
_handle_conan_conf(conan_conf, os.path.join(root, f))
elif f == "remotes.txt":
output.info("Defining remotes from remotes.txt")
_handle_remotes(cache, os.path.join(root, f))
elif f == "README.md":
output.info("Skip README.md")
elif f == "LICENSE.txt":
output.info("Skip LICENSE.txt")
else:
relpath = os.path.relpath(root, folder)
target_folder = os.path.join(cache.conan_folder, relpath)
mkdir(target_folder)
output.info("Copying file %s to %s" % (f, target_folder))
shutil.copy(os.path.join(root, f), target_folder)
for d in dirs:
if d == "profiles":
output.info("Installing profiles:")
profiles_path = cache.profiles_path
_handle_profiles(os.path.join(root, d), profiles_path, output)
elif d == "hooks" and ".git" not in root: # Avoid git hooks
output.info("Installing hooks:")
src_hooks_path = os.path.join(root, d)
dst_hooks_path = cache.hooks_path
_handle_hooks(src_hooks_path, dst_hooks_path, output)
dirs[:] = [d for d in dirs if d not in ("profiles", ".git", "hooks")]
def _handle_hooks(src_hooks_path, dst_hooks_path, output):
"""
Copies files to the hooks folder overwriting the files that are in the same path
(shutil.copytree fails on doing this), skips git related files (.git, .gitmodule...) and outputs
the copied files
:param src_hooks_path: Folder where the hooks come from
:param dst_hooks_path: Folder where the hooks should finally go
:param output: Output to indicate the files copied
"""
hooks_dirs = []
for root, dirs, files in walk(src_hooks_path):
if root == src_hooks_path:
hooks_dirs = dirs
else:
copied_files = False
relpath = os.path.relpath(root, src_hooks_path)
for f in files:
if ".git" not in f:
dst = os.path.join(dst_hooks_path, relpath)
mkdir(dst)
shutil.copy(os.path.join(root, f), dst)
copied_files = True
if copied_files and relpath in hooks_dirs:
output.info(" - %s" % relpath)
def merge_directories(src, dst, excluded=None):
src = os.path.normpath(src)
dst = os.path.normpath(dst)
excluded = excluded or []
excluded = [os.path.normpath(entry) for entry in excluded]
def is_excluded(origin_path):
if origin_path == dst:
return True
rel_path = os.path.normpath(os.path.relpath(origin_path, src))
if rel_path in excluded:
return True
return False
def link_to_rel(pointer_src):
linkto = os.readlink(pointer_src)
if not os.path.isabs(linkto):
linkto = os.path.join(os.path.dirname(pointer_src), linkto)
# Check if it is outside the sources
out_of_source = os.path.relpath(linkto,
os.path.realpath(src)).startswith(".")
if out_of_source:
# May warn about out of sources symlink
return
# Create the symlink
linkto_rel = os.path.relpath(linkto, os.path.dirname(pointer_src))
pointer_dst = os.path.normpath(
os.path.join(dst, os.path.relpath(pointer_src, src)))
os.symlink(linkto_rel, pointer_dst)
for src_dir, dirs, files in walk(src, followlinks=True):
if is_excluded(src_dir):
dirs[:] = []
continue
if os.path.islink(src_dir):
link_to_rel(src_dir)
dirs[:] = [] # Do not enter subdirectories
continue
# Overwriting the dirs will prevents walk to get into them
files[:] = [
d for d in files if not is_excluded(os.path.join(src_dir, d))
]
dst_dir = os.path.normpath(
os.path.join(dst, os.path.relpath(src_dir, src)))
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
for file_ in files:
src_file = os.path.join(src_dir, file_)
dst_file = os.path.join(dst_dir, file_)
if os.path.islink(src_file):
link_to_rel(src_file)
else:
shutil.copy2(src_file, dst_file)
def walk_encoding_test(self):
badfilename = "\xE3\x81\x82badfile.txt"
folder = temp_folder()
filepath = os.path.join(folder, badfilename)
save(to_file_bytes(filepath), "contents")
if six.PY2:
folder = unicode(folder)
a_file = [f[0] for _, _, f in walk(folder)][0]
self.assertTrue(a_file.endswith("badfile.txt"))
def _process_folder(config, folder, cache, output):
if not os.path.isdir(folder):
raise ConanException("No such directory: '%s'" % str(folder))
if config.source_folder:
folder = os.path.join(folder, config.source_folder)
for root, dirs, files in walk(folder):
dirs[:] = [d for d in dirs if d != ".git"]
for f in files:
_process_file(root, f, config, cache, output, folder)
def _handle_profiles(source_folder, target_folder, output):
mkdir(target_folder)
for root, _, files in walk(source_folder):
relative_path = os.path.relpath(root, source_folder)
if relative_path == ".":
relative_path = ""
for f in files:
profile = os.path.join(relative_path, f)
output.info(" Installing profile %s" % profile)
shutil.copy(os.path.join(root, f), os.path.join(target_folder, profile))
def _process_folder(config, folder, cache, output):
if not os.path.isdir(folder):
raise ConanException("No such directory: '%s'" % str(folder))
if config.source_folder:
folder = os.path.join(folder, config.source_folder)
for root, dirs, files in walk(folder):
dirs[:] = [d for d in dirs if d != ".git"]
if ".git" in root:
continue
for f in files:
if f == "settings.yml":
output.info("Installing settings.yml")
_filecopy(root, f, cache.cache_folder)
elif f == "conan.conf":
output.info("Processing conan.conf")
_handle_conan_conf(cache.config, os.path.join(root, f))
elif f == "remotes.txt":
output.info("Defining remotes from remotes.txt")
_handle_remotes(cache, os.path.join(root, f))
elif f in ("registry.txt", "registry.json"):
try:
os.remove(cache.remotes_path)
except OSError:
pass
finally:
_filecopy(root, f, cache.cache_folder)
migrate_registry_file(cache, output)
elif f == "remotes.json":
# Fix for Conan 2.0
raise ConanException(
"remotes.json install is not supported yet. Use 'remotes.txt'"
)
else:
# This is ugly, should be removed in Conan 2.0
if root == folder and f in ("README.md", "LICENSE.txt"):
output.info("Skip %s" % f)
continue
relpath = os.path.relpath(root, folder)
if config.target_folder:
target_folder = os.path.join(cache.cache_folder,
config.target_folder, relpath)
else:
target_folder = os.path.join(cache.cache_folder, relpath)
mkdir(target_folder)
output.info("Copying file %s to %s" % (f, target_folder))
_filecopy(root, f, target_folder)
def excluded_files(self):
ret = []
try:
file_paths = [os.path.normpath(
os.path.join(
os.path.relpath(folder, self.folder), el)).replace("\\", "/")
for folder, dirpaths, fs in walk(self.folder)
for el in fs + dirpaths]
if file_paths:
paths = to_file_bytes("\n".join(file_paths))
out = input_runner(['git', 'check-ignore', '--stdin'], paths, self.folder)
grep_stdout = decode_text(out)
ret = grep_stdout.splitlines()
except (CalledProcessError, IOError, OSError) as e:
if self._output:
self._output.warn("Error checking excluded git files: %s. "
"Ignoring excluded files" % e)
ret = []
return ret
def excluded_files(self):
try:
file_paths = [
os.path.normpath(
os.path.join(os.path.relpath(folder, self.folder),
el)).replace("\\", "/")
for folder, dirpaths, fs in walk(self.folder)
for el in fs + dirpaths
]
p = subprocess.Popen(['git', 'check-ignore', '--stdin'],
stdout=PIPE,
stdin=PIPE,
stderr=STDOUT,
cwd=self.folder)
paths = to_file_bytes("\n".join(file_paths))
grep_stdout = decode_text(p.communicate(input=paths)[0])
tmp = grep_stdout.splitlines()
except CalledProcessError:
tmp = []
return tmp
def patch_config_paths(self):
"""
changes references to the absolute path of the installed package and its dependencies in
exported cmake config files to the appropriate conan variable. This makes
most (sensible) cmake config files portable.
For example, if a package foo installs a file called "fooConfig.cmake" to
be used by cmake's find_package method, normally this file will contain
absolute paths to the installed package folder, for example it will contain
a line such as:
SET(Foo_INSTALL_DIR /home/developer/.conan/data/Foo/1.0.0/...)
This will cause cmake find_package() method to fail when someone else
installs the package via conan.
This function will replace such mentions to
SET(Foo_INSTALL_DIR ${CONAN_FOO_ROOT})
which is a variable that is set by conanbuildinfo.cmake, so that find_package()
now correctly works on this conan package.
For dependent packages, if a package foo installs a file called "fooConfig.cmake" to
be used by cmake's find_package method and if it depends to a package bar,
normally this file will contain absolute paths to the bar package folder,
for example it will contain a line such as:
SET_TARGET_PROPERTIES(foo PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES
"/home/developer/.conan/data/Bar/1.0.0/user/channel/id/include")
This function will replace such mentions to
SET_TARGET_PROPERTIES(foo PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES
"${CONAN_BAR_ROOT}/include")
If the install() method of the CMake object in the conan file is used, this
function should be called _after_ that invocation. For example:
def build(self):
cmake = CMake(self)
cmake.configure()
cmake.build()
cmake.install()
cmake.patch_config_paths()
"""
if not self._conanfile.should_install:
return
if not self._conanfile.name:
raise ConanException(
"cmake.patch_config_paths() can't work without package name. "
"Define name in your recipe")
pf = self.definitions.get(cmake_install_prefix_var_name)
replstr = "${CONAN_%s_ROOT}" % self._conanfile.name.upper()
allwalk = chain(walk(self._conanfile.build_folder),
walk(self._conanfile.package_folder))
# We don't want warnings printed because there is no replacement of the abs path.
# there could be MANY cmake files in the package and the normal thing is to not find
# the abs paths
_null_out = ConanOutput(StringIO())
for root, _, files in allwalk:
for f in files:
if f.endswith(".cmake") and not f.startswith("conan"):
path = os.path.join(root, f)
tools.replace_path_in_file(path,
pf,
replstr,
strict=False,
output=_null_out)
# patch paths of dependent packages that are found in any cmake files of the
# current package
for dep in self._conanfile.deps_cpp_info.deps:
from_str = self._conanfile.deps_cpp_info[dep].rootpath
dep_str = "${CONAN_%s_ROOT}" % dep.upper()
ret = tools.replace_path_in_file(path,
from_str,
dep_str,
strict=False,
output=_null_out)
if ret:
self._conanfile.output.info(
"Patched paths for %s: %s to %s" %
(dep, from_str, dep_str))