def unzip_output_test(self): tmp_dir = temp_folder() file_path = os.path.join(tmp_dir, "example.txt") save(file_path, "Hello world!") zip_path = os.path.join(tmp_dir, 'example.zip') zipf = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) for root, _, files in os.walk(tmp_dir): for f in files: zipf.write(os.path.join(root, f), f) zipf.close() output_dir = os.path.join(tmp_dir, "output_dir") new_out = StringIO() old_out = sys.stdout try: sys.stdout = new_out tools.unzip(zip_path, output_dir) finally: sys.stdout = old_out output = new_out.getvalue() self.assertRegexpMatches(output, "Unzipping [\d]+ bytes, this can take a while") content = load(os.path.join(output_dir, "example.txt")) self.assertEqual(content, "Hello world!")
def _get_file_conf(self, section, varname=None): """Gets the section from config file or raises an exception""" try: if not os.path.exists(self.config_filename): jwt_random_secret = ''.join(random.choice(string.ascii_letters) for _ in range(24)) updown_random_secret = ''.join(random.choice(string.ascii_letters) for _ in range(24)) server_conf = default_server_conf.format(jwt_secret=jwt_random_secret, updown_secret=updown_random_secret) save(self.config_filename, server_conf) if not self._loaded: self._loaded = True self.read(self.config_filename) if varname: section = dict(self.items(section)) return section[varname] else: return self.items(section) except NoSectionError as exc: raise ConanException("No section '%s' found" % section) except Exception as exc: logger.debug(exc) raise ConanException("Invalid configuration, " "missing %s: %s" % (section, varname))
def _handle_system_requirements(self, conan_ref, package_reference, conan_file, coutput): """ check first the system_reqs/system_requirements.txt existence, if not existing check package/sha1/ """ if "system_requirements" not in type(conan_file).__dict__: return system_reqs_path = self._client_cache.system_reqs(conan_ref) system_reqs_package_path = self._client_cache.system_reqs_package(package_reference) if os.path.exists(system_reqs_path) or os.path.exists(system_reqs_package_path): return try: output = conan_file.system_requirements() except Exception as e: coutput.error("while executing system_requirements(): %s" % str(e)) raise ConanException("Error in system requirements") try: output = str(output or "") except: coutput.warn("System requirements didn't return a string") output = "" if getattr(conan_file, "global_system_requirements", None): save(system_reqs_path, output) else: save(system_reqs_package_path, output)
def write_generators(conanfile, path, output): """ produces auxiliary files, required to build a project or a package. """ from conans.model.build_info import CppInfo available_generators = {"txt": (TXTGenerator, BUILD_INFO), "gcc": (GCCGenerator, BUILD_INFO_GCC), "cmake": (CMakeGenerator, BUILD_INFO_CMAKE), "qmake": (QmakeGenerator, BUILD_INFO_QMAKE), "visual_studio": (VisualStudioGenerator, BUILD_INFO_VISUAL_STUDIO), "xcode": (XCodeGenerator, BUILD_INFO_XCODE), "ycm": (YouCompleteMeGenerator, BUILD_INFO_YCM)} conanfile.cpp_info = CppInfo(path) conanfile.cpp_info.dependencies = [] conanfile.package_info() for generator in conanfile.generators: if generator not in available_generators: output.warn("Invalid generator '%s'. Available options: %s" % (generator, ", ".join(available_generators.keys()))) else: generator_class, filename = available_generators[generator] generator = generator_class(conanfile.deps_cpp_info, conanfile.cpp_info) output.info("Generated %s" % filename) save(os.path.join(path, filename), generator.content)
def cmd_profile_delete_key(profile_name, key, cache_profiles_path): first_key, rest_key = _get_profile_keys(key) profile, _ = read_profile(profile_name, get_cwd(), cache_profiles_path) try: package, name = rest_key.split(":") except ValueError: package = None name = rest_key try: if first_key == "settings": del profile.settings[rest_key] elif first_key == "options": profile.options.remove(name, package) elif first_key == "env": profile.env_values.remove(name, package) elif first_key == "build_requires": raise ConanException("Edit the profile manually to delete a build_require") except KeyError: raise ConanException("Profile key '%s' doesn't exist" % key) contents = profile.dumps() profile_path = get_profile_path(profile_name, cache_profiles_path, get_cwd()) save(profile_path, contents)
def _retrieve_remote_conan_file(self, conan_reference): export_files = self._remote_manager.get_conanfile(conan_reference, self._remote) export_folder = self._paths.export(conan_reference) # TODO: Download only the CONANFILE file and only download the rest of files # in install if needed (not found remote package) for file_name, content in export_files: # export_files is a generator save(os.path.join(export_folder, file_name), content)
def __init__(self, base_folder=None, current_folder=None, servers=None, users=None, client_version=CLIENT_VERSION, min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION): """ storage_folder: Local storage path current_folder: Current execution folder servers: dict of {remote_name: TestServer} logins is a list of (user, password) for auto input in order if required==> [("lasote", "mypass"), ("other", "otherpass")] """ self.users = users or {"default": [(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]} self.servers = servers or {} self.client_version = Version(str(client_version)) self.min_server_compatible_version = Version(str(min_server_compatible_version)) self.base_folder = base_folder or temp_folder() # Define storage_folder, if not, it will be read from conf file & pointed to real user home self.storage_folder = os.path.join(self.base_folder, ".conan", "data") self.paths = ConanPaths(self.base_folder, self.storage_folder, TestBufferConanOutput()) self.default_settings(get_env("CONAN_COMPILER", "gcc"), get_env("CONAN_COMPILER_VERSION", "4.8")) self.init_dynamic_vars() save(self.paths.registry, "") registry = RemoteRegistry(self.paths.registry, TestBufferConanOutput()) for name, server in self.servers.items(): registry.add(name, server.fake_url) logger.debug("Client storage = %s" % self.storage_folder) self.current_folder = current_folder or temp_folder()
def uncompress_files(files, folder, name): for file_name, content in files: if os.path.basename(file_name) != name: save(os.path.join(folder, file_name), content) else: # Unzip the file tar_extract(BytesIO(content), folder)
def setUp(self): self.conan_reference = ConanFileReference.loads("openssl/2.0.3@lasote/testing") self.package_reference = PackageReference(self.conan_reference, "123123123") self.tmp_dir = temp_folder() read_perms = [("*/*@*/*", "*")] write_perms = [] authorizer = BasicAuthorizer(read_perms, write_perms) self.fake_url = "http://url" updown_auth_manager = JWTUpDownAuthManager("secret", timedelta(seconds=200)) adapter = ServerDiskAdapter(self.fake_url, self.tmp_dir, updown_auth_manager) self.paths = SimplePaths(self.tmp_dir) self.file_manager = FileManager(self.paths, adapter) search_adapter = DiskSearchAdapter() self.search_manager = DiskSearchManager(self.paths, search_adapter) self.service = ConanService(authorizer, self.file_manager, "lasote") self.search_service = SearchService(authorizer, self.search_manager, "lasote") files = hello_source_files("test") save_files(self.paths.export(self.conan_reference), files) self.conan_digest = FileTreeManifest.create(self.paths.export(self.conan_reference)) conan_digest_path = os.path.join(self.paths.export(self.conan_reference), CONAN_MANIFEST) save(conan_digest_path, str(self.conan_digest)) files = hello_source_files("package") save_files(self.paths.package(self.package_reference), files)
def _create_profile(self, name, settings, scopes=None, env=None): profile = Profile() profile.settings = settings or {} if scopes: profile.scopes = Scopes.from_list(["%s=%s" % (key, value) for key, value in scopes.items()]) profile.env = env or {} save(self.client.client_cache.profile_path(name), profile.dumps())
def test_clean_sh_path(self): if platform.system() != "Windows": return os.environ["PATH"] = os.environ.get("PATH", "") + os.pathsep + self.tempdir save(os.path.join(self.tempdir, "sh.exe"), "Fake sh") conanfile = ConanFileMock() settings = Settings.loads(default_settings_yml) settings.os = "Windows" settings.compiler = "Visual Studio" settings.compiler.version = "12" settings.arch = "x86" conanfile.settings = settings cmake = CMake(conanfile) cmake.configure() self.assertIn(self.tempdir, conanfile.path) cmake.generator = "MinGW Makefiles" cmake.configure() self.assertNotIn(self.tempdir, conanfile.path) # Automatic gcc settings = Settings.loads(default_settings_yml) settings.os = "Windows" settings.compiler = "gcc" settings.compiler.version = "5.4" settings.arch = "x86" conanfile.settings = settings cmake = CMake(conanfile) cmake.configure() self.assertNotIn(self.tempdir, conanfile.path)
def shortener(path, short_paths): """ short_paths is 4-state: False: Never shorten the path True: Always shorten the path, create link if not existing None: Use shorten path only if already exists, not create Other: Integrity check. Consumer knows it should be short, but it isn't """ if short_paths is False: return path link = os.path.join(path, ".conan_link") if os.path.exists(link): return load(link) elif short_paths is None: return path elif short_paths is not True: raise ConanException("This path should be short, but it isn't: %s\n" "Try to remove these packages and re-build them" % path) drive = os.path.splitdrive(path)[0] short_path = drive + "/.conan" try: os.makedirs(short_path) except: pass redirect = tempfile.mkdtemp(dir=short_path) save(link, redirect) return redirect
def test_package_settings(self): # CREATE A CONANFILE TO LOAD tmp_dir = temp_folder() conanfile_path = os.path.join(tmp_dir, "conanfile.py") conanfile = """from conans import ConanFile class MyTest(ConanFile): requires = {} name = "MyPackage" version = "1.0" settings = "os" """ save(conanfile_path, conanfile) # Apply windows for MyPackage profile = Profile() profile.package_settings = {"MyPackage": OrderedDict([("os", "Windows")])} loader = ConanFileLoader(None, Settings({"os": ["Windows", "Linux"]}), profile) recipe = loader.load_conan(conanfile_path, None) self.assertEquals(recipe.settings.os, "Windows") # Apply Linux for MyPackage profile.package_settings = {"MyPackage": OrderedDict([("os", "Linux")])} loader = ConanFileLoader(None, Settings({"os": ["Windows", "Linux"]}), profile) recipe = loader.load_conan(conanfile_path, None) self.assertEquals(recipe.settings.os, "Linux") # If the package name is different from the conanfile one, it wont apply profile.package_settings = {"OtherPACKAGE": OrderedDict([("os", "Linux")])} loader = ConanFileLoader(None, Settings({"os": ["Windows", "Linux"]}), profile) recipe = loader.load_conan(conanfile_path, None) self.assertIsNone(recipe.settings.os.value)
def _shortener(path, short_paths): """ short_paths is 4-state: False: Never shorten the path True: Always shorten the path, create link if not existing None: Use shorten path only if already exists, not create Other: Integrity check. Consumer knows it should be short, but it isn't """ if short_paths is False: return path link = os.path.join(path, CONAN_LINK) if os.path.exists(link): return load(link) elif short_paths is None: return path elif short_paths is not True: raise ConanException("This path should be short, but it isn't: %s\n" "Try to remove these packages and re-build them" % path) short_home = os.getenv("CONAN_USER_HOME_SHORT") if not short_home: drive = os.path.splitdrive(path)[0] short_home = drive + "/.conan" try: os.makedirs(short_home) except: pass redirect = tempfile.mkdtemp(dir=short_home, prefix="") # This "1" is the way to have a non-existing directory, so commands like # shutil.copytree() to it, works. It can be removed without compromising the # temp folder generator and conan-links consistency redirect = os.path.join(redirect, "1") save(link, redirect) return redirect
def package_files_test(self): if platform.system() == "Windows": return client = TestClient() conanfile = """ from conans import ConanFile class TestConan(ConanFile): name = "Hello" version = "0.1" def package(self): self.copy("*", symlinks=True) """ client.save({"recipe/conanfile.py": conanfile}) file1 = os.path.join(client.current_folder, "file1.txt") file2 = os.path.join(client.current_folder, "version1/file2.txt") file11 = os.path.join(client.current_folder, "file1.txt.1") latest = os.path.join(client.current_folder, "latest") save(file1, "Hello1") os.symlink("file1.txt", file11) save(file2, "Hello2") os.symlink("version1", latest) client.run("export-pkg ./recipe Hello/0.1@lasote/stable") ref = PackageReference.loads("Hello/0.1@lasote/stable:" "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9") self._check(client, ref, build=False)
def no_configuration_test(self): dummy = """GlobalSection EndGlobalSection GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 Debug|x64 = Debug|x64 Release|Win32 = Release|Win32 Release|x64 = Release|x64 EndGlobalSection """ folder = temp_folder() path = os.path.join(folder, "dummy.sln") save(path, dummy) new_out = StringIO() tools.set_global_instances(ConanOutput(new_out), None) command = build_sln_command(Settings({}), sln_path=path, targets=None, upgrade_project=False, build_type='Debug', arch="x86", parallel=False) self.assertIn('/p:Configuration=Debug /p:Platform="x86"', command) self.assertIn("WARN: ***** The configuration Debug|x86 does not exist in this solution *****", new_out.getvalue()) # use platforms new_out = StringIO() tools.set_global_instances(ConanOutput(new_out), None) command = build_sln_command(Settings({}), sln_path=path, targets=None, upgrade_project=False, build_type='Debug', arch="x86", parallel=False, platforms={"x86": "Win32"}) self.assertIn('/p:Configuration=Debug /p:Platform="Win32"', command) self.assertNotIn("WARN", new_out.getvalue()) self.assertNotIn("ERROR", new_out.getvalue())
def __init__(self, requester, client_cache, timeout): self.proxies = client_cache.conan_config.proxies or {} self._no_proxy_match = [el.strip() for el in self.proxies.pop("no_proxy_match", "").split(",")] self._timeout_seconds = timeout # Retrocompatibility with deprecated no_proxy # Account for the requests NO_PROXY env variable, not defined as a proxy like http= no_proxy = self.proxies.pop("no_proxy", None) if no_proxy: os.environ["NO_PROXY"] = no_proxy self._requester = requester self._client_cache = client_cache if not os.path.exists(self._client_cache.cacert_path): from conans.client.rest.cacert import cacert save(self._client_cache.cacert_path, cacert) if not os.path.exists(client_cache.client_cert_path): self._client_certificates = None else: if os.path.exists(client_cache.client_cert_key_path): # Requests can accept a tuple with cert and key, or just an string with a # file having both self._client_certificates = (client_cache.client_cert_path, client_cache.client_cert_key_path) else: self._client_certificates = client_cache.client_cert_path
def _upload_recipe(self, conan_reference, base_files=None, retry=1, retry_wait=0): files = hello_source_files(3, [1, 12]) if base_files: files.update(base_files) content = """ from conans import ConanFile class MyConan(ConanFile): name = "%s" version = "%s" settings = arch, compiler, os """ % (conan_reference.name, conan_reference.version) files[CONANFILE] = content files_md5s = {filename: md5(content) for filename, content in files.items()} conan_digest = FileTreeManifest(123123123, files_md5s) files[CONAN_MANIFEST] = str(conan_digest) tmp_dir = temp_folder() abs_paths = {} for filename, content in files.items(): abs_path = os.path.join(tmp_dir, filename) save(abs_path, content) abs_paths[filename] = abs_path self.api.upload_recipe(conan_reference, abs_paths, retry, retry_wait, False)
def upload_newer_recipe_test(self): servers = {} test_server = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")], users={"lasote": "mypass"}) servers["default"] = test_server client = TestClient(servers=servers, users={"default": [("lasote", "mypass")]}) files = cpp_hello_conan_files("Hello0", "1.2.1", build=False) client.save(files) client.run("export frodo/stable") client.run("install Hello0/1.2.1@frodo/stable --build=missing") client.run("upload Hello0/1.2.1@frodo/stable --all") client2 = TestClient(servers=servers, users={"default": [("lasote", "mypass")]}) client2.save(files) client2.run("export frodo/stable") ref = ConanFileReference.loads("Hello0/1.2.1@frodo/stable") manifest = client2.client_cache.load_manifest(ref) manifest.time += 10 save(client2.client_cache.digestfile_conanfile(ref), str(manifest)) client2.run("install Hello0/1.2.1@frodo/stable --build=missing") client2.run("upload Hello0/1.2.1@frodo/stable --all") self.assertNotIn("conanfile", client2.user_io.out) self.assertIn("Package is up to date.", client2.user_io.out) self.assertIn("Uploading conanmanifest.txt", client2.user_io.out) # Now try again with the other client, which timestamp is older client.run("upload Hello0/1.2.1@frodo/stable --all") self.assertNotIn("conanfile", client2.user_io.out) self.assertIn("Package is up to date.", client2.user_io.out) self.assertIn("Uploading conanmanifest.txt", client.user_io.out)
def short_paths_unzip_output_test(self): if platform.system() != "Windows": return tmp_dir = temp_folder() file_path = os.path.join(tmp_dir, "src/"*40, "example.txt") save(file_path, "Hello world!") zip_path = os.path.join(tmp_dir, 'example.zip') zipf = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) for root, _, files in os.walk(tmp_dir): for f in files: zipf.write(os.path.join(root, f), os.path.join("src/"*20, f)) zipf.close() output_dir = os.path.join(tmp_dir, "dst/"*40, "output_dir") new_out = StringIO() old_out = sys.stdout try: import conans conans.tools.set_global_instances(ConanOutput(new_out), None) tools.unzip(zip_path, output_dir) finally: conans.tools.set_global_instances(ConanOutput(old_out), None) output = new_out.getvalue() self.assertIn("ERROR: Error extract src/src", output)
def test_tree_manifest(self): tmp_dir = temp_folder() files = {"one.ext": "aalakjshdlkjahsdlkjahsdljkhsadljkhasljkdhlkjashd", "path/to/two.txt": "asdas13123", "two.txt": "asdasdasdasdasdasd", "folder/damn.pyc": "binarythings", "folder/damn.pyo": "binarythings2", "pythonfile.pyc": "binarythings3"} for filename, content in files.items(): save(os.path.join(tmp_dir, filename), content) manifest = FileTreeManifest.create(tmp_dir) save(os.path.join(tmp_dir, "THEMANIFEST.txt"), str(manifest)) readed_manifest = FileTreeManifest.loads(load(os.path.join(tmp_dir, "THEMANIFEST.txt"))) self.assertEqual(readed_manifest.time, manifest.time) self.assertEqual(readed_manifest, manifest) # Not included the pycs or pyo self.assertEquals(set(manifest.file_sums.keys()), set(["one.ext", "path/to/two.txt", "two.txt"])) for filepath, md5readed in manifest.file_sums.items(): content = files[filepath] self.assertEquals(md5(content), md5readed)
def unzip_output_test(self): tmp_dir = temp_folder() file_path = os.path.join(tmp_dir, "example.txt") save(file_path, "Hello world!") zip_path = os.path.join(tmp_dir, 'example.zip') zipf = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) for root, _, files in os.walk(tmp_dir): for f in files: zipf.write(os.path.join(root, f), f) zipf.close() output_dir = os.path.join(tmp_dir, "output_dir") new_out = StringIO() old_out = sys.stdout try: import requests import conans conans.tools.set_global_instances(ConanOutput(new_out), requests) tools.unzip(zip_path, output_dir) finally: conans.tools.set_global_instances(ConanOutput(old_out), requests) output = new_out.getvalue() self.assertRegexpMatches(output, "Unzipping [\d]+B") content = load(os.path.join(output_dir, "example.txt")) self.assertEqual(content, "Hello world!")
def env_vars_test_inheritance(self): tmp_dir = temp_folder() p1 = '''[env]\nVAR=1''' p2 = '''include(p1)\n[env]\nVAR=2''' save(os.path.join(tmp_dir, "p1"), p1) new_profile, _ = _load_profile(p2, tmp_dir, tmp_dir) self.assertEquals(new_profile.env_values.data[None]["VAR"], "2")
def run(): parser = argparse.ArgumentParser(description='Extracts build-info from a specified ' 'conan trace log and return a valid JSON') parser.add_argument('trace_path', help='Path to the conan trace log file e.j: ' '/tmp/conan_trace.log') parser.add_argument("--output", default=False, help='Optional file to output the JSON contents, if not specified the JSON' ' will be printed to stdout') args = parser.parse_args() if not os.path.exists(args.trace_path): print("Error, conan trace log not found! '%s'" % args.trace_path) exit(1) if args.output and not os.path.exists(os.path.dirname(args.output)): print("Error, output file directory not found! '%s'" % args.trace_path) exit(1) try: info = get_build_info(args.trace_path) the_json = json.dumps(info.serialize()) if args.output: save(args.output, the_json) else: print(the_json) except Exception as exc: print(exc) exit(1)
def test_patch_from_file(self, strip): if strip: file_content = base_conanfile + ''' def build(self): patch(patch_file="file.patch", strip=%s) ''' % strip patch_content = '''--- %s/text.txt\t2016-01-25 17:57:11.452848309 +0100 +++ %s/text_new.txt\t2016-01-25 17:57:28.839869950 +0100 @@ -1 +1 @@ -ONE TWO THREE +ONE TWO FOUR''' % ("old_path", "new_path") else: file_content = base_conanfile + ''' def build(self): patch(patch_file="file.patch") ''' patch_content = '''--- text.txt\t2016-01-25 17:57:11.452848309 +0100 +++ text_new.txt\t2016-01-25 17:57:28.839869950 +0100 @@ -1 +1 @@ -ONE TWO THREE +ONE TWO FOUR''' tmp_dir, file_path, text_file = self._save_files(file_content) patch_file = os.path.join(tmp_dir, "file.patch") save(patch_file, patch_content) self._build_and_check(tmp_dir, file_path, text_file, "ONE TWO FOUR")
def write_generators(conanfile, path, output): """ produces auxiliary files, required to build a project or a package. """ for generator_name in conanfile.generators: if generator_name not in registered_generators: output.warn("Invalid generator '%s'. Available types: %s" % (generator_name, ", ".join(registered_generators.available))) else: generator_class = registered_generators[generator_name] try: generator = generator_class(conanfile) except TypeError: # To allow old-style generator packages to work (e.g. premake) output.warn("Generator %s failed with new __init__(), trying old one") generator = generator_class(conanfile.deps_cpp_info, conanfile.cpp_info) try: content = generator.content if isinstance(content, dict): if generator.filename: output.warn("Generator %s is multifile. Property 'filename' not used" % (generator_name,)) for k, v in content.items(): v = normalize(v) output.info("Generated %s created %s" % (generator_name, k)) save(join(path, k), v) else: content = normalize(content) output.info("Generated %s created %s" % (generator_name, generator.filename)) save(join(path, generator.filename), content) except Exception as e: output.error("Generator %s(file:%s) failed\n%s" % (generator_name, generator.filename, str(e)))
def _save_files(self, file_content): tmp_dir = temp_folder() file_path = os.path.join(tmp_dir, "conanfile.py") text_file = os.path.join(tmp_dir, "text.txt") save(file_path, file_content) save(text_file, "ONE TWO THREE") return tmp_dir, file_path, text_file
def test_replace_in_file(self): file_content = ''' from conans import ConanFile from conans.tools import download, unzip, replace_in_file import os class ConanFileToolsTest(ConanFile): name = "test" version = "1.9.10" settings = [] def source(self): pass def build(self): replace_in_file("otherfile.txt", "ONE TWO THREE", "FOUR FIVE SIX") ''' tmp_dir = temp_folder() file_path = os.path.join(tmp_dir, "conanfile.py") other_file = os.path.join(tmp_dir, "otherfile.txt") save(file_path, file_content) save(other_file, "ONE TWO THREE") loader = ConanFileLoader(None, None, Settings(), OptionsValues.loads("")) ret = loader.load_conan(file_path) curdir = os.path.abspath(os.curdir) os.chdir(tmp_dir) try: ret.build() finally: os.chdir(curdir) content = load(other_file) self.assertEquals(content, "FOUR FIVE SIX")
def testMd5Name(self): files = { "one_file.txt": { "contents": b"The contents", "mode": 0o777 }, "Two_file.txt": { "contents": b"Two contents", "mode": 0o777 } } new_files = compress_files(files, PACKAGE_TGZ_NAME, excluded=[]) folder = temp_folder() file_path = os.path.join(folder, PACKAGE_TGZ_NAME) save(file_path, new_files[PACKAGE_TGZ_NAME]) md5_a = md5sum(file_path) time.sleep(1) # Timestamps change new_files = compress_files(files, PACKAGE_TGZ_NAME, excluded=[]) folder = temp_folder() file_path = os.path.join(folder, PACKAGE_TGZ_NAME) save(file_path, new_files[PACKAGE_TGZ_NAME]) md5_b = md5sum(file_path) self.assertEquals(md5_a, md5_b)
def test_md5_compress(self): folder = temp_folder() save(os.path.join(folder, "one_file.txt"), b"The contents") save(os.path.join(folder, "Two_file.txt"), b"Two contents") files = { "one_file.txt": os.path.join(folder, "one_file.txt"), "Two_file.txt": os.path.join(folder, "Two_file.txt"), } compress_files(files, {}, PACKAGE_TGZ_NAME, dest_dir=folder) file_path = os.path.join(folder, PACKAGE_TGZ_NAME) md5_a = md5sum(file_path) self.assertEqual(md5_a, "df220cfbc0652e8992a89a77666c03b5") time.sleep(1) # Timestamps change folder = temp_folder() compress_files(files, {}, PACKAGE_TGZ_NAME, dest_dir=folder) file_path = os.path.join(folder, PACKAGE_TGZ_NAME) md5_b = md5sum(file_path) self.assertEquals(md5_a, md5_b)
def _replace_with_separator(filepath, sep): tmp = load(filepath) ret = sep.join(tmp.splitlines()) if tmp.endswith("\n"): ret += sep save(filepath, ret)
def setUp(self): self.client = self._get_client() self.ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable#%s" % DEFAULT_REVISION_V1) self.pref = PackageReference(self.ref, "myfakeid", DEFAULT_REVISION_V1) reg_folder = self.client.cache.package_layout(self.ref).export() self.client.run('upload %s' % str(self.ref), assert_error=True) self.assertIn("ERROR: Recipe not found: '%s'" % str(self.ref), self.client.out) files = hello_source_files() fake_metadata = PackageMetadata() fake_metadata.recipe.revision = DEFAULT_REVISION_V1 fake_metadata.packages[self.pref.id].revision = DEFAULT_REVISION_V1 self.client.save({"metadata.json": fake_metadata.dumps()}, path=self.client.cache.package_layout( self.ref).base_folder()) self.client.save(files, path=reg_folder) self.client.save( { CONANFILE: myconan1, "include/math/lib1.h": "//copy", "my_lib/debug/libd.a": "//copy", "my_data/readme.txt": "//copy", "my_bin/executable": "//copy" }, path=reg_folder) mkdir(self.client.cache.package_layout(self.ref).export_sources()) manifest = FileTreeManifest.create(reg_folder) manifest.time = '123123123' manifest.save(reg_folder) self.test_server.server_store.update_last_revision(self.ref) self.server_pack_folder = self.test_server.server_store.package( self.pref) package_folder = self.client.cache.package_layout(self.ref).package( self.pref) save(os.path.join(package_folder, "include", "lib1.h"), "//header") save(os.path.join(package_folder, "lib", "my_lib", "libd.a"), "//lib") save(os.path.join(package_folder, "res", "shares", "readme.txt"), "//res") save(os.path.join(package_folder, "bin", "my_bin", "executable"), "//bin") save(os.path.join(package_folder, CONANINFO), """[recipe_hash]\n%s""" % manifest.summary_hash) FileTreeManifest.create(package_folder).save(package_folder) self.test_server.server_store.update_last_package_revision(self.pref) os.chmod( os.path.join(package_folder, "bin", "my_bin", "executable"), os.stat(os.path.join(package_folder, "bin", "my_bin", "executable")).st_mode | stat.S_IRWXU) expected_manifest = FileTreeManifest.create(package_folder) expected_manifest.save(package_folder) self.server_reg_folder = self.test_server.server_store.export(self.ref) self.assertFalse(os.path.exists(self.server_reg_folder)) self.assertFalse(os.path.exists(self.server_pack_folder))
def generate_manifest(package_folder): # Create the digest for the package digest = FileTreeManifest.create(package_folder) save(os.path.join(package_folder, CONAN_MANIFEST), str(digest))
def unzip(filename, destination=".", keep_permissions=False, pattern=None, output=None): """ Unzip a zipped file :param filename: Path to the zip file :param destination: Destination folder (or file for .gz files) :param keep_permissions: Keep the zip permissions. WARNING: Can be dangerous if the zip was not created in a NIX system, the bits could produce undefined permission schema. Use this option only if you are sure that the zip was created correctly. :param pattern: Extract only paths matching the pattern. This should be a Unix shell-style wildcard, see fnmatch documentation for more details. :param output: output :return: """ output = default_output(output, 'conans.client.tools.files.unzip') if (filename.endswith(".tar.gz") or filename.endswith(".tgz") or filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or filename.endswith(".tar")): return untargz(filename, destination, pattern) if filename.endswith(".gz"): import gzip with gzip.open(filename, 'rb') as f: file_content = f.read() target_name = filename[:-3] if destination == "." else destination save(target_name, file_content) return if filename.endswith(".tar.xz") or filename.endswith(".txz"): if six.PY2: raise ConanException( "XZ format not supported in Python 2. Use Python 3 instead") return untargz(filename, destination, pattern) import zipfile full_path = os.path.normpath(os.path.join(get_cwd(), destination)) if hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): def print_progress(the_size, uncomp_size): the_size = (the_size * 100.0 / uncomp_size) if uncomp_size != 0 else 0 txt_msg = "Unzipping %d %%" if the_size > print_progress.last_size + 1: output.rewrite_line(txt_msg % the_size) print_progress.last_size = the_size if int(the_size) == 99: output.rewrite_line(txt_msg % 100) output.writeln("") else: def print_progress(_, __): pass with zipfile.ZipFile(filename, "r") as z: if not pattern: zip_info = z.infolist() else: zip_info = [ zi for zi in z.infolist() if fnmatch(zi.filename, pattern) ] uncompress_size = sum((file_.file_size for file_ in zip_info)) if uncompress_size > 100000: output.info("Unzipping %s, this can take a while" % human_size(uncompress_size)) else: output.info("Unzipping %s" % human_size(uncompress_size)) extracted_size = 0 print_progress.last_size = -1 if platform.system() == "Windows": for file_ in zip_info: extracted_size += file_.file_size print_progress(extracted_size, uncompress_size) try: z.extract(file_, full_path) except Exception as e: output.error("Error extract %s\n%s" % (file_.filename, str(e))) else: # duplicated for, to avoid a platform check for each zipped file for file_ in zip_info: extracted_size += file_.file_size print_progress(extracted_size, uncompress_size) try: z.extract(file_, full_path) if keep_permissions: # Could be dangerous if the ZIP has been created in a non nix system # https://bugs.python.org/issue15795 perm = file_.external_attr >> 16 & 0xFFF os.chmod(os.path.join(full_path, file_.filename), perm) except Exception as e: output.error("Error extract %s\n%s" % (file_.filename, str(e)))
def save(self, filename): ret = {"remotes": [{"name": r, "url": u, "verify_ssl": v} for r, (_, u, v) in self._remotes.items()]} save(filename, json.dumps(ret, indent=True))
def patch(base_path=None, patch_file=None, patch_string=None, strip=0, output=None): """Applies a diff from file (patch_file) or string (patch_string) in base_path directory or current dir if None""" class PatchLogHandler(logging.Handler): def __init__(self): logging.Handler.__init__(self, logging.DEBUG) self.output = output or ConanOutput( sys.stdout, sys.stderr, color=True) self.patchname = patch_file if patch_file else "patch" def emit(self, record): logstr = self.format(record) if record.levelno == logging.WARN: self.output.warn("%s: %s" % (self.patchname, logstr)) else: self.output.info("%s: %s" % (self.patchname, logstr)) patchlog = logging.getLogger("patch") if patchlog: patchlog.handlers = [] patchlog.addHandler(PatchLogHandler()) if not patch_file and not patch_string: return if patch_file: patchset = fromfile(patch_file) else: patchset = fromstring(patch_string.encode()) if not patchset: raise ConanException("Failed to parse patch: %s" % (patch_file if patch_file else "string")) def decode_clean(path, prefix): path = path.decode("utf-8").replace("\\", "/") if path.startswith(prefix): path = path[2:] return path def strip_path(path): tokens = path.split("/") if len(tokens) > 1: tokens = tokens[strip:] path = "/".join(tokens) if base_path: path = os.path.join(base_path, path) return path # account for new and deleted files, upstream dep won't fix them items = [] for p in patchset: source = decode_clean(p.source, "a/") target = decode_clean(p.target, "b/") if "dev/null" in source: target = strip_path(target) hunks = [s.decode("utf-8") for s in p.hunks[0].text] new_file = "".join(hunk[1:] for hunk in hunks) save(target, new_file) elif "dev/null" in target: source = strip_path(source) os.unlink(source) else: items.append(p) patchset.items = items if not patchset.apply(root=base_path, strip=strip): raise ConanException("Failed to apply patch: %s" % patch_file)