def unzip_output_test(self):
tmp_dir = temp_folder()
file_path = os.path.join(tmp_dir, "example.txt")
save(file_path, "Hello world!")
zip_path = os.path.join(tmp_dir, 'example.zip')
zipf = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
for root, _, files in os.walk(tmp_dir):
for f in files:
zipf.write(os.path.join(root, f), f)
zipf.close()
output_dir = os.path.join(tmp_dir, "output_dir")
new_out = StringIO()
old_out = sys.stdout
try:
sys.stdout = new_out
tools.unzip(zip_path, output_dir)
finally:
sys.stdout = old_out
output = new_out.getvalue()
self.assertRegexpMatches(output, "Unzipping [\d]+ bytes, this can take a while")
content = load(os.path.join(output_dir, "example.txt"))
self.assertEqual(content, "Hello world!")
def _get_file_conf(self, section, varname=None):
"""Gets the section from config file or raises an exception"""
try:
if not os.path.exists(self.config_filename):
jwt_random_secret = ''.join(random.choice(string.ascii_letters) for _ in range(24))
updown_random_secret = ''.join(random.choice(string.ascii_letters) for _ in range(24))
server_conf = default_server_conf.format(jwt_secret=jwt_random_secret,
updown_secret=updown_random_secret)
save(self.config_filename, server_conf)
if not self._loaded:
self._loaded = True
self.read(self.config_filename)
if varname:
section = dict(self.items(section))
return section[varname]
else:
return self.items(section)
except NoSectionError as exc:
raise ConanException("No section '%s' found" % section)
except Exception as exc:
logger.debug(exc)
raise ConanException("Invalid configuration, "
"missing %s: %s" % (section, varname))
def _handle_system_requirements(self, conan_ref, package_reference, conan_file, coutput):
""" check first the system_reqs/system_requirements.txt existence, if not existing
check package/sha1/
"""
if "system_requirements" not in type(conan_file).__dict__:
return
system_reqs_path = self._client_cache.system_reqs(conan_ref)
system_reqs_package_path = self._client_cache.system_reqs_package(package_reference)
if os.path.exists(system_reqs_path) or os.path.exists(system_reqs_package_path):
return
try:
output = conan_file.system_requirements()
except Exception as e:
coutput.error("while executing system_requirements(): %s" % str(e))
raise ConanException("Error in system requirements")
try:
output = str(output or "")
except:
coutput.warn("System requirements didn't return a string")
output = ""
if getattr(conan_file, "global_system_requirements", None):
save(system_reqs_path, output)
else:
save(system_reqs_package_path, output)
def write_generators(conanfile, path, output):
""" produces auxiliary files, required to build a project or a package.
"""
from conans.model.build_info import CppInfo
available_generators = {"txt": (TXTGenerator, BUILD_INFO),
"gcc": (GCCGenerator, BUILD_INFO_GCC),
"cmake": (CMakeGenerator, BUILD_INFO_CMAKE),
"qmake": (QmakeGenerator, BUILD_INFO_QMAKE),
"visual_studio": (VisualStudioGenerator, BUILD_INFO_VISUAL_STUDIO),
"xcode": (XCodeGenerator, BUILD_INFO_XCODE),
"ycm": (YouCompleteMeGenerator, BUILD_INFO_YCM)}
conanfile.cpp_info = CppInfo(path)
conanfile.cpp_info.dependencies = []
conanfile.package_info()
for generator in conanfile.generators:
if generator not in available_generators:
output.warn("Invalid generator '%s'. Available options: %s" %
(generator, ", ".join(available_generators.keys())))
else:
generator_class, filename = available_generators[generator]
generator = generator_class(conanfile.deps_cpp_info, conanfile.cpp_info)
output.info("Generated %s" % filename)
save(os.path.join(path, filename), generator.content)
def cmd_profile_delete_key(profile_name, key, cache_profiles_path):
first_key, rest_key = _get_profile_keys(key)
profile, _ = read_profile(profile_name, get_cwd(), cache_profiles_path)
try:
package, name = rest_key.split(":")
except ValueError:
package = None
name = rest_key
try:
if first_key == "settings":
del profile.settings[rest_key]
elif first_key == "options":
profile.options.remove(name, package)
elif first_key == "env":
profile.env_values.remove(name, package)
elif first_key == "build_requires":
raise ConanException("Edit the profile manually to delete a build_require")
except KeyError:
raise ConanException("Profile key '%s' doesn't exist" % key)
contents = profile.dumps()
profile_path = get_profile_path(profile_name, cache_profiles_path, get_cwd())
save(profile_path, contents)
def _retrieve_remote_conan_file(self, conan_reference):
export_files = self._remote_manager.get_conanfile(conan_reference, self._remote)
export_folder = self._paths.export(conan_reference)
# TODO: Download only the CONANFILE file and only download the rest of files
# in install if needed (not found remote package)
for file_name, content in export_files: # export_files is a generator
save(os.path.join(export_folder, file_name), content)
def __init__(self, base_folder=None, current_folder=None,
servers=None, users=None, client_version=CLIENT_VERSION,
min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION):
"""
storage_folder: Local storage path
current_folder: Current execution folder
servers: dict of {remote_name: TestServer}
logins is a list of (user, password) for auto input in order
if required==> [("lasote", "mypass"), ("other", "otherpass")]
"""
self.users = users or {"default":
[(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]}
self.servers = servers or {}
self.client_version = Version(str(client_version))
self.min_server_compatible_version = Version(str(min_server_compatible_version))
self.base_folder = base_folder or temp_folder()
# Define storage_folder, if not, it will be read from conf file & pointed to real user home
self.storage_folder = os.path.join(self.base_folder, ".conan", "data")
self.paths = ConanPaths(self.base_folder, self.storage_folder, TestBufferConanOutput())
self.default_settings(get_env("CONAN_COMPILER", "gcc"),
get_env("CONAN_COMPILER_VERSION", "4.8"))
self.init_dynamic_vars()
save(self.paths.registry, "")
registry = RemoteRegistry(self.paths.registry, TestBufferConanOutput())
for name, server in self.servers.items():
registry.add(name, server.fake_url)
logger.debug("Client storage = %s" % self.storage_folder)
self.current_folder = current_folder or temp_folder()
def uncompress_files(files, folder, name):
for file_name, content in files:
if os.path.basename(file_name) != name:
save(os.path.join(folder, file_name), content)
else:
# Unzip the file
tar_extract(BytesIO(content), folder)
def setUp(self):
self.conan_reference = ConanFileReference.loads("openssl/2.0.3@lasote/testing")
self.package_reference = PackageReference(self.conan_reference, "123123123")
self.tmp_dir = temp_folder()
read_perms = [("*/*@*/*", "*")]
write_perms = []
authorizer = BasicAuthorizer(read_perms, write_perms)
self.fake_url = "http://url"
updown_auth_manager = JWTUpDownAuthManager("secret",
timedelta(seconds=200))
adapter = ServerDiskAdapter(self.fake_url, self.tmp_dir, updown_auth_manager)
self.paths = SimplePaths(self.tmp_dir)
self.file_manager = FileManager(self.paths, adapter)
search_adapter = DiskSearchAdapter()
self.search_manager = DiskSearchManager(self.paths, search_adapter)
self.service = ConanService(authorizer, self.file_manager, "lasote")
self.search_service = SearchService(authorizer, self.search_manager, "lasote")
files = hello_source_files("test")
save_files(self.paths.export(self.conan_reference), files)
self.conan_digest = FileTreeManifest.create(self.paths.export(self.conan_reference))
conan_digest_path = os.path.join(self.paths.export(self.conan_reference), CONAN_MANIFEST)
save(conan_digest_path, str(self.conan_digest))
files = hello_source_files("package")
save_files(self.paths.package(self.package_reference), files)
def _create_profile(self, name, settings, scopes=None, env=None):
profile = Profile()
profile.settings = settings or {}
if scopes:
profile.scopes = Scopes.from_list(["%s=%s" % (key, value) for key, value in scopes.items()])
profile.env = env or {}
save(self.client.client_cache.profile_path(name), profile.dumps())
def test_clean_sh_path(self):
if platform.system() != "Windows":
return
os.environ["PATH"] = os.environ.get("PATH", "") + os.pathsep + self.tempdir
save(os.path.join(self.tempdir, "sh.exe"), "Fake sh")
conanfile = ConanFileMock()
settings = Settings.loads(default_settings_yml)
settings.os = "Windows"
settings.compiler = "Visual Studio"
settings.compiler.version = "12"
settings.arch = "x86"
conanfile.settings = settings
cmake = CMake(conanfile)
cmake.configure()
self.assertIn(self.tempdir, conanfile.path)
cmake.generator = "MinGW Makefiles"
cmake.configure()
self.assertNotIn(self.tempdir, conanfile.path)
# Automatic gcc
settings = Settings.loads(default_settings_yml)
settings.os = "Windows"
settings.compiler = "gcc"
settings.compiler.version = "5.4"
settings.arch = "x86"
conanfile.settings = settings
cmake = CMake(conanfile)
cmake.configure()
self.assertNotIn(self.tempdir, conanfile.path)
def shortener(path, short_paths):
""" short_paths is 4-state:
False: Never shorten the path
True: Always shorten the path, create link if not existing
None: Use shorten path only if already exists, not create
Other: Integrity check. Consumer knows it should be short, but it isn't
"""
if short_paths is False:
return path
link = os.path.join(path, ".conan_link")
if os.path.exists(link):
return load(link)
elif short_paths is None:
return path
elif short_paths is not True:
raise ConanException("This path should be short, but it isn't: %s\n"
"Try to remove these packages and re-build them" % path)
drive = os.path.splitdrive(path)[0]
short_path = drive + "/.conan"
try:
os.makedirs(short_path)
except:
pass
redirect = tempfile.mkdtemp(dir=short_path)
save(link, redirect)
return redirect
def test_package_settings(self):
# CREATE A CONANFILE TO LOAD
tmp_dir = temp_folder()
conanfile_path = os.path.join(tmp_dir, "conanfile.py")
conanfile = """from conans import ConanFile
class MyTest(ConanFile):
requires = {}
name = "MyPackage"
version = "1.0"
settings = "os"
"""
save(conanfile_path, conanfile)
# Apply windows for MyPackage
profile = Profile()
profile.package_settings = {"MyPackage": OrderedDict([("os", "Windows")])}
loader = ConanFileLoader(None, Settings({"os": ["Windows", "Linux"]}), profile)
recipe = loader.load_conan(conanfile_path, None)
self.assertEquals(recipe.settings.os, "Windows")
# Apply Linux for MyPackage
profile.package_settings = {"MyPackage": OrderedDict([("os", "Linux")])}
loader = ConanFileLoader(None, Settings({"os": ["Windows", "Linux"]}), profile)
recipe = loader.load_conan(conanfile_path, None)
self.assertEquals(recipe.settings.os, "Linux")
# If the package name is different from the conanfile one, it wont apply
profile.package_settings = {"OtherPACKAGE": OrderedDict([("os", "Linux")])}
loader = ConanFileLoader(None, Settings({"os": ["Windows", "Linux"]}), profile)
recipe = loader.load_conan(conanfile_path, None)
self.assertIsNone(recipe.settings.os.value)
def _shortener(path, short_paths):
""" short_paths is 4-state:
False: Never shorten the path
True: Always shorten the path, create link if not existing
None: Use shorten path only if already exists, not create
Other: Integrity check. Consumer knows it should be short, but it isn't
"""
if short_paths is False:
return path
link = os.path.join(path, CONAN_LINK)
if os.path.exists(link):
return load(link)
elif short_paths is None:
return path
elif short_paths is not True:
raise ConanException("This path should be short, but it isn't: %s\n"
"Try to remove these packages and re-build them" % path)
short_home = os.getenv("CONAN_USER_HOME_SHORT")
if not short_home:
drive = os.path.splitdrive(path)[0]
short_home = drive + "/.conan"
try:
os.makedirs(short_home)
except:
pass
redirect = tempfile.mkdtemp(dir=short_home, prefix="")
# This "1" is the way to have a non-existing directory, so commands like
# shutil.copytree() to it, works. It can be removed without compromising the
# temp folder generator and conan-links consistency
redirect = os.path.join(redirect, "1")
save(link, redirect)
return redirect
def package_files_test(self):
if platform.system() == "Windows":
return
client = TestClient()
conanfile = """
from conans import ConanFile
class TestConan(ConanFile):
name = "Hello"
version = "0.1"
def package(self):
self.copy("*", symlinks=True)
"""
client.save({"recipe/conanfile.py": conanfile})
file1 = os.path.join(client.current_folder, "file1.txt")
file2 = os.path.join(client.current_folder, "version1/file2.txt")
file11 = os.path.join(client.current_folder, "file1.txt.1")
latest = os.path.join(client.current_folder, "latest")
save(file1, "Hello1")
os.symlink("file1.txt", file11)
save(file2, "Hello2")
os.symlink("version1", latest)
client.run("export-pkg ./recipe Hello/0.1@lasote/stable")
ref = PackageReference.loads("Hello/0.1@lasote/stable:"
"5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9")
self._check(client, ref, build=False)
def no_configuration_test(self):
dummy = """GlobalSection
EndGlobalSection
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Win32 = Debug|Win32
Debug|x64 = Debug|x64
Release|Win32 = Release|Win32
Release|x64 = Release|x64
EndGlobalSection
"""
folder = temp_folder()
path = os.path.join(folder, "dummy.sln")
save(path, dummy)
new_out = StringIO()
tools.set_global_instances(ConanOutput(new_out), None)
command = build_sln_command(Settings({}), sln_path=path, targets=None, upgrade_project=False,
build_type='Debug', arch="x86", parallel=False)
self.assertIn('/p:Configuration=Debug /p:Platform="x86"', command)
self.assertIn("WARN: ***** The configuration Debug|x86 does not exist in this solution *****",
new_out.getvalue())
# use platforms
new_out = StringIO()
tools.set_global_instances(ConanOutput(new_out), None)
command = build_sln_command(Settings({}), sln_path=path, targets=None, upgrade_project=False,
build_type='Debug', arch="x86", parallel=False, platforms={"x86": "Win32"})
self.assertIn('/p:Configuration=Debug /p:Platform="Win32"', command)
self.assertNotIn("WARN", new_out.getvalue())
self.assertNotIn("ERROR", new_out.getvalue())
def __init__(self, requester, client_cache, timeout):
self.proxies = client_cache.conan_config.proxies or {}
self._no_proxy_match = [el.strip() for el in
self.proxies.pop("no_proxy_match", "").split(",")]
self._timeout_seconds = timeout
# Retrocompatibility with deprecated no_proxy
# Account for the requests NO_PROXY env variable, not defined as a proxy like http=
no_proxy = self.proxies.pop("no_proxy", None)
if no_proxy:
os.environ["NO_PROXY"] = no_proxy
self._requester = requester
self._client_cache = client_cache
if not os.path.exists(self._client_cache.cacert_path):
from conans.client.rest.cacert import cacert
save(self._client_cache.cacert_path, cacert)
if not os.path.exists(client_cache.client_cert_path):
self._client_certificates = None
else:
if os.path.exists(client_cache.client_cert_key_path):
# Requests can accept a tuple with cert and key, or just an string with a
# file having both
self._client_certificates = (client_cache.client_cert_path,
client_cache.client_cert_key_path)
else:
self._client_certificates = client_cache.client_cert_path
def _upload_recipe(self, conan_reference, base_files=None, retry=1, retry_wait=0):
files = hello_source_files(3, [1, 12])
if base_files:
files.update(base_files)
content = """
from conans import ConanFile
class MyConan(ConanFile):
name = "%s"
version = "%s"
settings = arch, compiler, os
""" % (conan_reference.name, conan_reference.version)
files[CONANFILE] = content
files_md5s = {filename: md5(content) for filename, content in files.items()}
conan_digest = FileTreeManifest(123123123, files_md5s)
files[CONAN_MANIFEST] = str(conan_digest)
tmp_dir = temp_folder()
abs_paths = {}
for filename, content in files.items():
abs_path = os.path.join(tmp_dir, filename)
save(abs_path, content)
abs_paths[filename] = abs_path
self.api.upload_recipe(conan_reference, abs_paths, retry, retry_wait, False)
def upload_newer_recipe_test(self):
servers = {}
test_server = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")],
users={"lasote": "mypass"})
servers["default"] = test_server
client = TestClient(servers=servers, users={"default": [("lasote", "mypass")]})
files = cpp_hello_conan_files("Hello0", "1.2.1", build=False)
client.save(files)
client.run("export frodo/stable")
client.run("install Hello0/1.2.1@frodo/stable --build=missing")
client.run("upload Hello0/1.2.1@frodo/stable --all")
client2 = TestClient(servers=servers, users={"default": [("lasote", "mypass")]})
client2.save(files)
client2.run("export frodo/stable")
ref = ConanFileReference.loads("Hello0/1.2.1@frodo/stable")
manifest = client2.client_cache.load_manifest(ref)
manifest.time += 10
save(client2.client_cache.digestfile_conanfile(ref), str(manifest))
client2.run("install Hello0/1.2.1@frodo/stable --build=missing")
client2.run("upload Hello0/1.2.1@frodo/stable --all")
self.assertNotIn("conanfile", client2.user_io.out)
self.assertIn("Package is up to date.", client2.user_io.out)
self.assertIn("Uploading conanmanifest.txt", client2.user_io.out)
# Now try again with the other client, which timestamp is older
client.run("upload Hello0/1.2.1@frodo/stable --all")
self.assertNotIn("conanfile", client2.user_io.out)
self.assertIn("Package is up to date.", client2.user_io.out)
self.assertIn("Uploading conanmanifest.txt", client.user_io.out)
def short_paths_unzip_output_test(self):
if platform.system() != "Windows":
return
tmp_dir = temp_folder()
file_path = os.path.join(tmp_dir, "src/"*40, "example.txt")
save(file_path, "Hello world!")
zip_path = os.path.join(tmp_dir, 'example.zip')
zipf = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
for root, _, files in os.walk(tmp_dir):
for f in files:
zipf.write(os.path.join(root, f), os.path.join("src/"*20, f))
zipf.close()
output_dir = os.path.join(tmp_dir, "dst/"*40, "output_dir")
new_out = StringIO()
old_out = sys.stdout
try:
import conans
conans.tools.set_global_instances(ConanOutput(new_out), None)
tools.unzip(zip_path, output_dir)
finally:
conans.tools.set_global_instances(ConanOutput(old_out), None)
output = new_out.getvalue()
self.assertIn("ERROR: Error extract src/src", output)
def test_tree_manifest(self):
tmp_dir = temp_folder()
files = {"one.ext": "aalakjshdlkjahsdlkjahsdljkhsadljkhasljkdhlkjashd",
"path/to/two.txt": "asdas13123",
"two.txt": "asdasdasdasdasdasd",
"folder/damn.pyc": "binarythings",
"folder/damn.pyo": "binarythings2",
"pythonfile.pyc": "binarythings3"}
for filename, content in files.items():
save(os.path.join(tmp_dir, filename), content)
manifest = FileTreeManifest.create(tmp_dir)
save(os.path.join(tmp_dir, "THEMANIFEST.txt"), str(manifest))
readed_manifest = FileTreeManifest.loads(load(os.path.join(tmp_dir, "THEMANIFEST.txt")))
self.assertEqual(readed_manifest.time, manifest.time)
self.assertEqual(readed_manifest, manifest)
# Not included the pycs or pyo
self.assertEquals(set(manifest.file_sums.keys()),
set(["one.ext", "path/to/two.txt", "two.txt"]))
for filepath, md5readed in manifest.file_sums.items():
content = files[filepath]
self.assertEquals(md5(content), md5readed)
def unzip_output_test(self):
tmp_dir = temp_folder()
file_path = os.path.join(tmp_dir, "example.txt")
save(file_path, "Hello world!")
zip_path = os.path.join(tmp_dir, 'example.zip')
zipf = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
for root, _, files in os.walk(tmp_dir):
for f in files:
zipf.write(os.path.join(root, f), f)
zipf.close()
output_dir = os.path.join(tmp_dir, "output_dir")
new_out = StringIO()
old_out = sys.stdout
try:
import requests
import conans
conans.tools.set_global_instances(ConanOutput(new_out), requests)
tools.unzip(zip_path, output_dir)
finally:
conans.tools.set_global_instances(ConanOutput(old_out), requests)
output = new_out.getvalue()
self.assertRegexpMatches(output, "Unzipping [\d]+B")
content = load(os.path.join(output_dir, "example.txt"))
self.assertEqual(content, "Hello world!")
def env_vars_test_inheritance(self):
tmp_dir = temp_folder()
p1 = '''[env]\nVAR=1'''
p2 = '''include(p1)\n[env]\nVAR=2'''
save(os.path.join(tmp_dir, "p1"), p1)
new_profile, _ = _load_profile(p2, tmp_dir, tmp_dir)
self.assertEquals(new_profile.env_values.data[None]["VAR"], "2")
def run():
parser = argparse.ArgumentParser(description='Extracts build-info from a specified '
'conan trace log and return a valid JSON')
parser.add_argument('trace_path', help='Path to the conan trace log file e.j: '
'/tmp/conan_trace.log')
parser.add_argument("--output", default=False,
help='Optional file to output the JSON contents, if not specified the JSON'
' will be printed to stdout')
args = parser.parse_args()
if not os.path.exists(args.trace_path):
print("Error, conan trace log not found! '%s'" % args.trace_path)
exit(1)
if args.output and not os.path.exists(os.path.dirname(args.output)):
print("Error, output file directory not found! '%s'" % args.trace_path)
exit(1)
try:
info = get_build_info(args.trace_path)
the_json = json.dumps(info.serialize())
if args.output:
save(args.output, the_json)
else:
print(the_json)
except Exception as exc:
print(exc)
exit(1)
def test_patch_from_file(self, strip):
if strip:
file_content = base_conanfile + '''
def build(self):
patch(patch_file="file.patch", strip=%s)
''' % strip
patch_content = '''--- %s/text.txt\t2016-01-25 17:57:11.452848309 +0100
+++ %s/text_new.txt\t2016-01-25 17:57:28.839869950 +0100
@@ -1 +1 @@
-ONE TWO THREE
+ONE TWO FOUR''' % ("old_path", "new_path")
else:
file_content = base_conanfile + '''
def build(self):
patch(patch_file="file.patch")
'''
patch_content = '''--- text.txt\t2016-01-25 17:57:11.452848309 +0100
+++ text_new.txt\t2016-01-25 17:57:28.839869950 +0100
@@ -1 +1 @@
-ONE TWO THREE
+ONE TWO FOUR'''
tmp_dir, file_path, text_file = self._save_files(file_content)
patch_file = os.path.join(tmp_dir, "file.patch")
save(patch_file, patch_content)
self._build_and_check(tmp_dir, file_path, text_file, "ONE TWO FOUR")
def write_generators(conanfile, path, output):
""" produces auxiliary files, required to build a project or a package.
"""
for generator_name in conanfile.generators:
if generator_name not in registered_generators:
output.warn("Invalid generator '%s'. Available types: %s" %
(generator_name, ", ".join(registered_generators.available)))
else:
generator_class = registered_generators[generator_name]
try:
generator = generator_class(conanfile)
except TypeError:
# To allow old-style generator packages to work (e.g. premake)
output.warn("Generator %s failed with new __init__(), trying old one")
generator = generator_class(conanfile.deps_cpp_info, conanfile.cpp_info)
try:
content = generator.content
if isinstance(content, dict):
if generator.filename:
output.warn("Generator %s is multifile. Property 'filename' not used"
% (generator_name,))
for k, v in content.items():
v = normalize(v)
output.info("Generated %s created %s" % (generator_name, k))
save(join(path, k), v)
else:
content = normalize(content)
output.info("Generated %s created %s" % (generator_name, generator.filename))
save(join(path, generator.filename), content)
except Exception as e:
output.error("Generator %s(file:%s) failed\n%s"
% (generator_name, generator.filename, str(e)))
def _save_files(self, file_content):
tmp_dir = temp_folder()
file_path = os.path.join(tmp_dir, "conanfile.py")
text_file = os.path.join(tmp_dir, "text.txt")
save(file_path, file_content)
save(text_file, "ONE TWO THREE")
return tmp_dir, file_path, text_file
def test_replace_in_file(self):
file_content = '''
from conans import ConanFile
from conans.tools import download, unzip, replace_in_file
import os
class ConanFileToolsTest(ConanFile):
name = "test"
version = "1.9.10"
settings = []
def source(self):
pass
def build(self):
replace_in_file("otherfile.txt", "ONE TWO THREE", "FOUR FIVE SIX")
'''
tmp_dir = temp_folder()
file_path = os.path.join(tmp_dir, "conanfile.py")
other_file = os.path.join(tmp_dir, "otherfile.txt")
save(file_path, file_content)
save(other_file, "ONE TWO THREE")
loader = ConanFileLoader(None, None, Settings(), OptionsValues.loads(""))
ret = loader.load_conan(file_path)
curdir = os.path.abspath(os.curdir)
os.chdir(tmp_dir)
try:
ret.build()
finally:
os.chdir(curdir)
content = load(other_file)
self.assertEquals(content, "FOUR FIVE SIX")
def testMd5Name(self):
files = {
"one_file.txt": {
"contents": b"The contents",
"mode": 0o777
},
"Two_file.txt": {
"contents": b"Two contents",
"mode": 0o777
}
}
new_files = compress_files(files, PACKAGE_TGZ_NAME, excluded=[])
folder = temp_folder()
file_path = os.path.join(folder, PACKAGE_TGZ_NAME)
save(file_path, new_files[PACKAGE_TGZ_NAME])
md5_a = md5sum(file_path)
time.sleep(1) # Timestamps change
new_files = compress_files(files, PACKAGE_TGZ_NAME, excluded=[])
folder = temp_folder()
file_path = os.path.join(folder, PACKAGE_TGZ_NAME)
save(file_path, new_files[PACKAGE_TGZ_NAME])
md5_b = md5sum(file_path)
self.assertEquals(md5_a, md5_b)
def test_md5_compress(self):
folder = temp_folder()
save(os.path.join(folder, "one_file.txt"), b"The contents")
save(os.path.join(folder, "Two_file.txt"), b"Two contents")
files = {
"one_file.txt": os.path.join(folder, "one_file.txt"),
"Two_file.txt": os.path.join(folder, "Two_file.txt"),
}
compress_files(files, {}, PACKAGE_TGZ_NAME, dest_dir=folder)
file_path = os.path.join(folder, PACKAGE_TGZ_NAME)
md5_a = md5sum(file_path)
self.assertEqual(md5_a, "df220cfbc0652e8992a89a77666c03b5")
time.sleep(1) # Timestamps change
folder = temp_folder()
compress_files(files, {}, PACKAGE_TGZ_NAME, dest_dir=folder)
file_path = os.path.join(folder, PACKAGE_TGZ_NAME)
md5_b = md5sum(file_path)
self.assertEquals(md5_a, md5_b)
def _replace_with_separator(filepath, sep):
tmp = load(filepath)
ret = sep.join(tmp.splitlines())
if tmp.endswith("\n"):
ret += sep
save(filepath, ret)
def setUp(self):
self.client = self._get_client()
self.ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable#%s" %
DEFAULT_REVISION_V1)
self.pref = PackageReference(self.ref, "myfakeid", DEFAULT_REVISION_V1)
reg_folder = self.client.cache.package_layout(self.ref).export()
self.client.run('upload %s' % str(self.ref), assert_error=True)
self.assertIn("ERROR: Recipe not found: '%s'" % str(self.ref),
self.client.out)
files = hello_source_files()
fake_metadata = PackageMetadata()
fake_metadata.recipe.revision = DEFAULT_REVISION_V1
fake_metadata.packages[self.pref.id].revision = DEFAULT_REVISION_V1
self.client.save({"metadata.json": fake_metadata.dumps()},
path=self.client.cache.package_layout(
self.ref).base_folder())
self.client.save(files, path=reg_folder)
self.client.save(
{
CONANFILE: myconan1,
"include/math/lib1.h": "//copy",
"my_lib/debug/libd.a": "//copy",
"my_data/readme.txt": "//copy",
"my_bin/executable": "//copy"
},
path=reg_folder)
mkdir(self.client.cache.package_layout(self.ref).export_sources())
manifest = FileTreeManifest.create(reg_folder)
manifest.time = '123123123'
manifest.save(reg_folder)
self.test_server.server_store.update_last_revision(self.ref)
self.server_pack_folder = self.test_server.server_store.package(
self.pref)
package_folder = self.client.cache.package_layout(self.ref).package(
self.pref)
save(os.path.join(package_folder, "include", "lib1.h"), "//header")
save(os.path.join(package_folder, "lib", "my_lib", "libd.a"), "//lib")
save(os.path.join(package_folder, "res", "shares", "readme.txt"),
"//res")
save(os.path.join(package_folder, "bin", "my_bin", "executable"),
"//bin")
save(os.path.join(package_folder, CONANINFO),
"""[recipe_hash]\n%s""" % manifest.summary_hash)
FileTreeManifest.create(package_folder).save(package_folder)
self.test_server.server_store.update_last_package_revision(self.pref)
os.chmod(
os.path.join(package_folder, "bin", "my_bin", "executable"),
os.stat(os.path.join(package_folder, "bin", "my_bin",
"executable")).st_mode | stat.S_IRWXU)
expected_manifest = FileTreeManifest.create(package_folder)
expected_manifest.save(package_folder)
self.server_reg_folder = self.test_server.server_store.export(self.ref)
self.assertFalse(os.path.exists(self.server_reg_folder))
self.assertFalse(os.path.exists(self.server_pack_folder))
def generate_manifest(package_folder):
# Create the digest for the package
digest = FileTreeManifest.create(package_folder)
save(os.path.join(package_folder, CONAN_MANIFEST), str(digest))
def unzip(filename,
destination=".",
keep_permissions=False,
pattern=None,
output=None):
"""
Unzip a zipped file
:param filename: Path to the zip file
:param destination: Destination folder (or file for .gz files)
:param keep_permissions: Keep the zip permissions. WARNING: Can be
dangerous if the zip was not created in a NIX system, the bits could
produce undefined permission schema. Use this option only if you are sure
that the zip was created correctly.
:param pattern: Extract only paths matching the pattern. This should be a
Unix shell-style wildcard, see fnmatch documentation for more details.
:param output: output
:return:
"""
output = default_output(output, 'conans.client.tools.files.unzip')
if (filename.endswith(".tar.gz") or filename.endswith(".tgz")
or filename.endswith(".tbz2") or filename.endswith(".tar.bz2")
or filename.endswith(".tar")):
return untargz(filename, destination, pattern)
if filename.endswith(".gz"):
import gzip
with gzip.open(filename, 'rb') as f:
file_content = f.read()
target_name = filename[:-3] if destination == "." else destination
save(target_name, file_content)
return
if filename.endswith(".tar.xz") or filename.endswith(".txz"):
if six.PY2:
raise ConanException(
"XZ format not supported in Python 2. Use Python 3 instead")
return untargz(filename, destination, pattern)
import zipfile
full_path = os.path.normpath(os.path.join(get_cwd(), destination))
if hasattr(sys.stdout, "isatty") and sys.stdout.isatty():
def print_progress(the_size, uncomp_size):
the_size = (the_size * 100.0 /
uncomp_size) if uncomp_size != 0 else 0
txt_msg = "Unzipping %d %%"
if the_size > print_progress.last_size + 1:
output.rewrite_line(txt_msg % the_size)
print_progress.last_size = the_size
if int(the_size) == 99:
output.rewrite_line(txt_msg % 100)
output.writeln("")
else:
def print_progress(_, __):
pass
with zipfile.ZipFile(filename, "r") as z:
if not pattern:
zip_info = z.infolist()
else:
zip_info = [
zi for zi in z.infolist() if fnmatch(zi.filename, pattern)
]
uncompress_size = sum((file_.file_size for file_ in zip_info))
if uncompress_size > 100000:
output.info("Unzipping %s, this can take a while" %
human_size(uncompress_size))
else:
output.info("Unzipping %s" % human_size(uncompress_size))
extracted_size = 0
print_progress.last_size = -1
if platform.system() == "Windows":
for file_ in zip_info:
extracted_size += file_.file_size
print_progress(extracted_size, uncompress_size)
try:
z.extract(file_, full_path)
except Exception as e:
output.error("Error extract %s\n%s" %
(file_.filename, str(e)))
else: # duplicated for, to avoid a platform check for each zipped file
for file_ in zip_info:
extracted_size += file_.file_size
print_progress(extracted_size, uncompress_size)
try:
z.extract(file_, full_path)
if keep_permissions:
# Could be dangerous if the ZIP has been created in a non nix system
# https://bugs.python.org/issue15795
perm = file_.external_attr >> 16 & 0xFFF
os.chmod(os.path.join(full_path, file_.filename), perm)
except Exception as e:
output.error("Error extract %s\n%s" %
(file_.filename, str(e)))
def save(self, filename):
ret = {"remotes": [{"name": r, "url": u, "verify_ssl": v}
for r, (_, u, v) in self._remotes.items()]}
save(filename, json.dumps(ret, indent=True))
def patch(base_path=None,
patch_file=None,
patch_string=None,
strip=0,
output=None):
"""Applies a diff from file (patch_file) or string (patch_string)
in base_path directory or current dir if None"""
class PatchLogHandler(logging.Handler):
def __init__(self):
logging.Handler.__init__(self, logging.DEBUG)
self.output = output or ConanOutput(
sys.stdout, sys.stderr, color=True)
self.patchname = patch_file if patch_file else "patch"
def emit(self, record):
logstr = self.format(record)
if record.levelno == logging.WARN:
self.output.warn("%s: %s" % (self.patchname, logstr))
else:
self.output.info("%s: %s" % (self.patchname, logstr))
patchlog = logging.getLogger("patch")
if patchlog:
patchlog.handlers = []
patchlog.addHandler(PatchLogHandler())
if not patch_file and not patch_string:
return
if patch_file:
patchset = fromfile(patch_file)
else:
patchset = fromstring(patch_string.encode())
if not patchset:
raise ConanException("Failed to parse patch: %s" %
(patch_file if patch_file else "string"))
def decode_clean(path, prefix):
path = path.decode("utf-8").replace("\\", "/")
if path.startswith(prefix):
path = path[2:]
return path
def strip_path(path):
tokens = path.split("/")
if len(tokens) > 1:
tokens = tokens[strip:]
path = "/".join(tokens)
if base_path:
path = os.path.join(base_path, path)
return path
# account for new and deleted files, upstream dep won't fix them
items = []
for p in patchset:
source = decode_clean(p.source, "a/")
target = decode_clean(p.target, "b/")
if "dev/null" in source:
target = strip_path(target)
hunks = [s.decode("utf-8") for s in p.hunks[0].text]
new_file = "".join(hunk[1:] for hunk in hunks)
save(target, new_file)
elif "dev/null" in target:
source = strip_path(source)
os.unlink(source)
else:
items.append(p)
patchset.items = items
if not patchset.apply(root=base_path, strip=strip):
raise ConanException("Failed to apply patch: %s" % patch_file)
