def test_tree_manifest(self): tmp_dir = temp_folder() files = {"one.ext": "aalakjshdlkjahsdlkjahsdljkhsadljkhasljkdhlkjashd", "path/to/two.txt": "asdas13123", "two.txt": "asdasdasdasdasdasd", "folder/damn.pyc": "binarythings", "folder/damn.pyo": "binarythings2", "pythonfile.pyc": "binarythings3"} for filename, content in files.items(): save(os.path.join(tmp_dir, filename), content) manifest = FileTreeManifest.create(tmp_dir) save(os.path.join(tmp_dir, "THEMANIFEST.txt"), str(manifest)) readed_manifest = FileTreeManifest.loads(load(os.path.join(tmp_dir, "THEMANIFEST.txt"))) self.assertEqual(readed_manifest.time, manifest.time) self.assertEqual(readed_manifest, manifest) # Not included the pycs or pyo self.assertEquals(set(manifest.file_sums.keys()), set(["one.ext", "path/to/two.txt", "two.txt"])) for filepath, md5readed in manifest.file_sums.items(): content = files[filepath] self.assertEquals(md5(content), md5readed)
def _upload_recipe(self, conan_reference, base_files=None, retry=1, retry_wait=0): files = hello_source_files(3, [1, 12]) if base_files: files.update(base_files) content = """ from conans import ConanFile class MyConan(ConanFile): name = "%s" version = "%s" settings = arch, compiler, os """ % (conan_reference.name, conan_reference.version) files[CONANFILE] = content files_md5s = {filename: md5(content) for filename, content in files.items()} conan_digest = FileTreeManifest(123123123, files_md5s) tmp_dir = temp_folder() abs_paths = {} for filename, content in files.items(): abs_path = os.path.join(tmp_dir, filename) save(abs_path, content) abs_paths[filename] = abs_path abs_paths[CONAN_MANIFEST] = os.path.join(tmp_dir, CONAN_MANIFEST) conan_digest.save(tmp_dir) self.api.upload_recipe(conan_reference, abs_paths, retry, retry_wait, False, None)
def valid_digest(self, digest_path): if not os.path.exists(digest_path): return False expected_digest = FileTreeManifest.create(os.path.dirname(digest_path)) del expected_digest.file_sums[CONAN_MANIFEST] # Exclude the MANIFEST itself expected_digest.file_sums.pop(CONANFILE + "c", None) # Exclude the CONANFILE.pyc expected_digest.file_sums.pop(".DS_Store", None) # Exclude tmp in mac readed_digest = FileTreeManifest.loads(load(digest_path)) return readed_digest.file_sums == expected_digest.file_sums
def _report_save_manifest(copied_files, output, dest_folder, manifest_name): report_copied_files(copied_files, output) if copied_files: date = calendar.timegm(time.gmtime()) file_dict = {} for f in copied_files: abs_path = os.path.join(dest_folder, f) file_dict[f] = md5sum(abs_path) manifest = FileTreeManifest(date, file_dict) manifest.save(dest_folder, manifest_name)
def test_export_a_new_version(self): self._create_packages_and_builds() # Export an update of the same conans conan2 = TestClient(self.conan.base_folder) files2 = cpp_hello_conan_files("Hello0", "0.1") files2[CONANFILE] = "# insert comment\n %s" % files2[CONANFILE] conan2.save(files2) conan2.run("export lasote/stable") reg_path3 = conan2.paths.export(self.conan_ref) digest3 = FileTreeManifest.loads(load(conan2.paths.digestfile_conanfile(self.conan_ref))) self.assertIn('A new conanfile.py version was exported', conan2.user_io.out) # self.assertIn('All the previous packs were cleaned', conan2.user_io.out) self.assertIn('conanfile.py exported as %s in %s' % (self.conan_ref, reg_path3), conan2.user_io.out) self.assertTrue(os.path.exists(reg_path3)) for name in files2.keys(): self.assertTrue(os.path.exists(os.path.join(reg_path3, name))) expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'main.cpp': '0479f3c223c9a656a718f3148e044124', 'CMakeLists.txt': '310d65b60943b879286cf8839cf9ba08', 'conanfile.py': '5a2c6e2d6b39638b7d8560786d7935a3', 'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'} self.assertEqual(expected_sums, digest3.file_sums)
def test_export_the_same_code(self): file_list = self._create_packages_and_builds() # Export the same conans conan2 = TestClient(self.conan.base_folder) files2 = cpp_hello_conan_files("Hello0", "0.1") conan2.save(files2) conan2.run("export lasote/stable") reg_path2 = conan2.paths.export(self.conan_ref) digest2 = FileTreeManifest.loads(load(conan2.paths.digestfile_conanfile(self.conan_ref))) self.assertNotIn('A new Conan version was exported', conan2.user_io.out) self.assertNotIn('Cleaning the old builds ...', conan2.user_io.out) self.assertNotIn('Cleaning the old packs ...', conan2.user_io.out) self.assertNotIn('All the previous packs were cleaned', conan2.user_io.out) self.assertIn('conanfile.py exported as %s in %s' % (self.conan_ref, reg_path2), conan2.user_io.out) self.assertTrue(os.path.exists(reg_path2)) for name in files2.keys(): self.assertTrue(os.path.exists(os.path.join(reg_path2, name))) expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'main.cpp': '0479f3c223c9a656a718f3148e044124', 'CMakeLists.txt': '310d65b60943b879286cf8839cf9ba08', 'conanfile.py': '4df4b5266ad6d5d55cbdd0a8b12c9d1a', 'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'} self.assertEqual(expected_sums, digest2.file_sums) for f in file_list: self.assertTrue(os.path.exists(f))
def setUp(self): self.conan_reference = ConanFileReference.loads("openssl/2.0.3@lasote/testing") self.package_reference = PackageReference(self.conan_reference, "123123123") self.tmp_dir = temp_folder() read_perms = [("*/*@*/*", "*")] write_perms = [] authorizer = BasicAuthorizer(read_perms, write_perms) self.fake_url = "http://url" updown_auth_manager = JWTUpDownAuthManager("secret", timedelta(seconds=200)) adapter = ServerDiskAdapter(self.fake_url, self.tmp_dir, updown_auth_manager) self.paths = SimplePaths(self.tmp_dir) self.file_manager = FileManager(self.paths, adapter) search_adapter = DiskSearchAdapter() self.search_manager = DiskSearchManager(self.paths, search_adapter) self.service = ConanService(authorizer, self.file_manager, "lasote") self.search_service = SearchService(authorizer, self.search_manager, "lasote") files = hello_source_files("test") save_files(self.paths.export(self.conan_reference), files) self.conan_digest = FileTreeManifest.create(self.paths.export(self.conan_reference)) conan_digest_path = os.path.join(self.paths.export(self.conan_reference), CONAN_MANIFEST) save(conan_digest_path, str(self.conan_digest)) files = hello_source_files("package") save_files(self.paths.package(self.package_reference), files)
def test_export_the_same_code(self): file_list = self._create_packages_and_builds() # Export the same conans conan2 = TestClient(self.conan.base_folder) files2 = cpp_hello_conan_files("Hello0", "0.1") conan2.save(files2) conan2.run("export lasote/stable") reg_path2 = conan2.paths.export(self.conan_ref) digest2 = FileTreeManifest.loads(load(conan2.paths.digestfile_conanfile(self.conan_ref))) self.assertNotIn('A new Conan version was exported', conan2.user_io.out) self.assertNotIn('Cleaning the old builds ...', conan2.user_io.out) self.assertNotIn('Cleaning the old packs ...', conan2.user_io.out) self.assertNotIn('All the previous packs were cleaned', conan2.user_io.out) self.assertIn('%s: conanfile.py exported to local storage' % str(self.conan_ref), self.conan.user_io.out) self.assertIn('%s: Folder: %s' % (str(self.conan_ref), reg_path2), self.conan.user_io.out) self.assertTrue(os.path.exists(reg_path2)) for name in files2.keys(): self.assertTrue(os.path.exists(os.path.join(reg_path2, name))) expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'main.cpp': '0479f3c223c9a656a718f3148e044124', 'CMakeLists.txt': 'bc3405da4bb0b51a3b9f05aca71e58c8', 'conanfile.py': 'f21a98d974e9294b0d709070042c6e78', 'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'} self.assertEqual(expected_sums, digest2.file_sums) for f in file_list: self.assertTrue(os.path.exists(f))
def test_basic(self): current_folder = temp_folder() source_folder = os.path.join(current_folder, "source") client = TestClient(current_folder=current_folder) files = cpp_hello_conan_files("Hello0", "0.1") conan_ref = ConanFileReference("Hello0", "0.1", "lasote", "stable") client.save(files, path=source_folder) client.run("export lasote/stable --path=source") reg_path = client.paths.export(conan_ref) manif = FileTreeManifest.loads(load(client.paths.digestfile_conanfile(conan_ref))) self.assertIn('%s: A new conanfile.py version was exported' % str(conan_ref), client.user_io.out) self.assertIn('%s: Folder: %s' % (str(conan_ref), reg_path), client.user_io.out) self.assertTrue(os.path.exists(reg_path)) for name in list(files.keys()): self.assertTrue(os.path.exists(os.path.join(reg_path, name))) expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'main.cpp': '0479f3c223c9a656a718f3148e044124', 'CMakeLists.txt': 'bc3405da4bb0b51a3b9f05aca71e58c8', 'conanfile.py': '5632cf850a7161388ab24f42b9bdb3fd', 'executable': '68b329da9893e34099c7d8ad5cb9c940', 'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'} self.assertEqual(expected_sums, manif.file_sums)
def test_path(self): base_folder = temp_folder() source_folder = os.path.join(base_folder, "source") current_folder = os.path.join(base_folder, "current") client = TestClient(current_folder=current_folder) files = cpp_hello_conan_files("Hello0", "0.1") conan_ref = ConanFileReference("Hello0", "0.1", "lasote", "stable") conanfile = files.pop("conanfile.py") client.save(files, path=source_folder) conanfile = conanfile.replace("exports = '*'", 'exports = "../source*"') client.save({"conanfile.py": conanfile}) client.run("export lasote/stable") reg_path = client.paths.export(conan_ref) manif = FileTreeManifest.loads(load(client.paths.digestfile_conanfile(conan_ref))) self.assertIn('%s: A new conanfile.py version was exported' % str(conan_ref), client.user_io.out) self.assertIn('%s: Folder: %s' % (str(conan_ref), reg_path), client.user_io.out) self.assertTrue(os.path.exists(reg_path)) for name in ['conanfile.py', 'conanmanifest.txt', 'source/main.cpp', 'source/executable', 'source/hello.cpp', 'source/CMakeLists.txt', 'source/helloHello0.h']: self.assertTrue(os.path.exists(os.path.join(reg_path, name))) expected_sums = {'source/hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'source/main.cpp': '0479f3c223c9a656a718f3148e044124', 'source/CMakeLists.txt': '52546396c42f16be3daf72ecf7ab7143', 'conanfile.py': '3ac566eb5b2e4df4417003f0e606e237', 'source/executable': '68b329da9893e34099c7d8ad5cb9c940', 'source/helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'} self.assertEqual(expected_sums, manif.file_sums)
def test_export_a_new_version(self): self._create_packages_and_builds() # Export an update of the same conans conan2 = TestClient(self.conan.base_folder) files2 = cpp_hello_conan_files("Hello0", "0.1") files2[CONANFILE] = "# insert comment\n %s" % files2[CONANFILE] conan2.save(files2) conan2.run("export lasote/stable") reg_path3 = conan2.paths.export(self.conan_ref) digest3 = FileTreeManifest.loads(load(conan2.paths.digestfile_conanfile(self.conan_ref))) self.assertIn('A new conanfile.py version was exported', conan2.user_io.out) self.assertIn('%s: conanfile.py exported to local storage' % str(self.conan_ref), self.conan.user_io.out) self.assertIn('%s: Folder: %s' % (str(self.conan_ref), reg_path3), self.conan.user_io.out) self.assertTrue(os.path.exists(reg_path3)) for name in files2.keys(): self.assertTrue(os.path.exists(os.path.join(reg_path3, name))) expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'main.cpp': '0479f3c223c9a656a718f3148e044124', 'CMakeLists.txt': 'bc3405da4bb0b51a3b9f05aca71e58c8', 'conanfile.py': 'c5889ea6485599c7a0cae02b54270b35', 'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'} self.assertEqual(expected_sums, digest3.file_sums)
def test_export_a_new_version(self): self._create_packages_and_builds() # Export an update of the same conans conan2 = TestClient(self.conan.base_folder) files2 = cpp_hello_conan_files("Hello0", "0.1") files2[CONANFILE] = "# insert comment\n %s" % files2[CONANFILE] conan2.save(files2) conan2.run("export lasote/stable") reg_path3 = conan2.paths.export(self.conan_ref) digest3 = FileTreeManifest.loads(load(conan2.paths.digestfile_conanfile(self.conan_ref))) self.assertIn('%s: A new conanfile.py version was exported' % str(self.conan_ref), self.conan.user_io.out) self.assertIn('%s: Folder: %s' % (str(self.conan_ref), reg_path3), self.conan.user_io.out) self.assertTrue(os.path.exists(reg_path3)) for name in list(files2.keys()): self.assertTrue(os.path.exists(os.path.join(reg_path3, name))) expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'main.cpp': '0479f3c223c9a656a718f3148e044124', 'CMakeLists.txt': '52546396c42f16be3daf72ecf7ab7143', 'conanfile.py': 'ad17cf00b3142728b03ac37782b9acd9', 'executable': '68b329da9893e34099c7d8ad5cb9c940', 'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'} self.assertEqual(expected_sums, digest3.file_sums)
def undo_imports(current_path, output): manifest_path = os.path.join(current_path, IMPORTS_MANIFESTS) try: manifest_content = load(manifest_path) except: raise ConanException("Cannot load file %s" % manifest_path) try: manifest = FileTreeManifest.loads(manifest_content) except: raise ConanException("Wrong manifest file format %s" % manifest_path) not_removed = 0 files = manifest.files() for filepath in files: if not os.path.exists(filepath): output.warn("File doesn't exist: %s" % filepath) continue try: os.remove(filepath) except: output.error("Cannot remove file (open or busy): %s" % filepath) not_removed += 1 if not_removed: raise ConanException("Cannot remove %s or more imported files" % not_removed) output.success("Removed %s imported files" % (len(files))) try: os.remove(manifest_path) output.success("Removed imports manifest file: %s" % manifest_path) except: raise ConanException("Cannot remove manifest file (open or busy): %s" % manifest_path)
def test_export_the_same_code(self): file_list = self._create_packages_and_builds() # Export the same conans conan2 = TestClient(self.conan.base_folder) files2 = cpp_hello_conan_files("Hello0", "0.1") conan2.save(files2) conan2.run("export lasote/stable") reg_path2 = conan2.paths.export(self.conan_ref) digest2 = FileTreeManifest.loads(load(conan2.paths.digestfile_conanfile(self.conan_ref))) self.assertNotIn('A new Conan version was exported', conan2.user_io.out) self.assertNotIn('Cleaning the old builds ...', conan2.user_io.out) self.assertNotIn('Cleaning the old packs ...', conan2.user_io.out) self.assertNotIn('All the previous packs were cleaned', conan2.user_io.out) self.assertIn('%s: A new conanfile.py version was exported' % str(self.conan_ref), self.conan.user_io.out) self.assertIn('%s: Folder: %s' % (str(self.conan_ref), reg_path2), self.conan.user_io.out) self.assertTrue(os.path.exists(reg_path2)) for name in list(files2.keys()): self.assertTrue(os.path.exists(os.path.join(reg_path2, name))) expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'main.cpp': '0479f3c223c9a656a718f3148e044124', 'CMakeLists.txt': '52546396c42f16be3daf72ecf7ab7143', 'conanfile.py': '355949fbf0b4fc32b8f1c5a338dfe1ae', 'executable': '68b329da9893e34099c7d8ad5cb9c940', 'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'} self.assertEqual(expected_sums, digest2.file_sums) for f in file_list: self.assertTrue(os.path.exists(f))
def test_rel_path(self): base_folder = temp_folder() source_folder = os.path.join(base_folder, "source") current_folder = os.path.join(base_folder, "current") os.makedirs(current_folder) client = TestClient(current_folder=current_folder) files = cpp_hello_conan_files("Hello0", "0.1") conan_ref = ConanFileReference("Hello0", "0.1", "lasote", "stable") client.save(files, path=source_folder) client.run("export lasote/stable --path=../source") reg_path = client.paths.export(conan_ref) manif = FileTreeManifest.loads(load(client.paths.digestfile_conanfile(conan_ref))) self.assertIn('%s: A new conanfile.py version was exported' % str(conan_ref), client.user_io.out) self.assertIn('%s: Folder: %s' % (str(conan_ref), reg_path), client.user_io.out) self.assertTrue(os.path.exists(reg_path)) for name in list(files.keys()): self.assertTrue(os.path.exists(os.path.join(reg_path, name))) expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'main.cpp': '0479f3c223c9a656a718f3148e044124', 'CMakeLists.txt': '52546396c42f16be3daf72ecf7ab7143', 'conanfile.py': '355949fbf0b4fc32b8f1c5a338dfe1ae', 'executable': '68b329da9893e34099c7d8ad5cb9c940', 'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'} self.assertEqual(expected_sums, manif.file_sums)
def _check_manifest(self): manifest_content = load(os.path.join(self.client.current_folder, IMPORTS_MANIFESTS)) manifest = FileTreeManifest.loads(manifest_content) self.assertEqual(manifest.file_sums, {os.path.join(self.client.current_folder, "file1.txt"): "8b1a9953c4611296a827abf8c47804d7", os.path.join(self.client.current_folder, "file2.txt"): "f5a7924e621e84c9280a9a27e1bcb7f6"})
def setUp(self): hello_files = cpp_hello_conan_files("Hello") test_conanfile_contents = hello_files[CONANFILE] self.server_folder = temp_folder() test_server = TestServer(users={"fenix": "mypass"}, base_path=self.server_folder) # exported users and passwords self.server = test_server servers = {"default": test_server} client = TestClient(servers=servers, users={"default": [("fenix", "mypass")]}) # Conans with and without packages created self.root_folder = {"H1": 'Hello/1.4.10/fenix/testing', "H2": 'Hello/2.4.11/fenix/testing', "B": 'Bye/0.14/fenix/testing', "O": 'Other/1.2/fenix/testing'} files = {} pack_refs = [] for key, folder in self.root_folder.items(): ref = ConanFileReference.loads(folder) files["%s/%s/conanfile.py" % (folder, EXPORT_FOLDER)] = test_conanfile_contents files["%s/%s/conanmanifest.txt" % (folder, EXPORT_FOLDER)] = "" files["%s/%s/conans.txt" % (folder, SRC_FOLDER)] = "" for pack_id in (1, 2): i = pack_id pack_id = "%s_%s" % (pack_id, key) pack_refs.append(PackageReference(ref, str(pack_id))) files["%s/%s/%s/conans.txt" % (folder, BUILD_FOLDER, pack_id)] = "" files["%s/%s/%s/conans.txt" % (folder, PACKAGES_FOLDER, pack_id)] = "" files["%s/%s/%s/%s" % (folder, PACKAGES_FOLDER, pack_id, CONANINFO)] = conaninfo % str(i) files["%s/%s/%s/%s" % (folder, PACKAGES_FOLDER, pack_id, CONAN_MANIFEST)] = "" exports_sources_dir = os.path.join(client.client_cache.export(ref), EXPORT_SOURCES_DIR) os.makedirs(exports_sources_dir) client.save(files, client.client_cache.store) # Create the manifests to be able to upload for pack_ref in pack_refs: digest_path = client.client_cache.digestfile_package(pack_ref) expected_manifest = FileTreeManifest.create(os.path.dirname(digest_path)) files["%s/%s/%s/%s" % ("/".join(pack_ref.conan), PACKAGES_FOLDER, pack_ref.package_id, CONAN_MANIFEST)] = str(expected_manifest) client.save(files, client.client_cache.store) self.client = client for folder in self.root_folder.values(): client.run("upload %s --all" % folder.replace("/fenix", "@fenix")) self.assert_folders({"H1": [1, 2], "H2": [1, 2], "B": [1, 2], "O": [1, 2]}, {"H1": [1, 2], "H2": [1, 2], "B": [1, 2], "O": [1, 2]}, {"H1": [1, 2], "H2": [1, 2], "B": [1, 2], "O": [1, 2]}, {"H1": True, "H2": True, "B": True, "O": True})
def force_test(self): '''Tries to upload a conans exported after than remote version.''' # Upload all conans and packages self.client.run('upload %s --all' % str(self.conan_ref)) self.assertTrue(os.path.exists(self.server_reg_folder)) self.assertTrue(os.path.exists(self.server_pack_folder)) # Fake datetime from exported date and upload again old_digest = self.client.paths.load_manifest(self.conan_ref) old_digest.file_sums["new_file"] = "012345" fake_digest = FileTreeManifest(2, old_digest.file_sums) fake_digest.save(self.client.paths.export(self.conan_ref)) self.client.run('upload %s' % str(self.conan_ref), ignore_error=True) self.assertIn("Remote recipe is newer than local recipe", self.client.user_io.out) self.client.run('upload %s --force' % str(self.conan_ref)) self.assertIn("Uploading %s" % str(self.conan_ref), self.client.user_io.out)
def setUp(self): self.client = self._get_client() conan_digest = FileTreeManifest('123123123', {}) self.conan_ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable") reg_folder = self.client.paths.export(self.conan_ref) self.client.run('upload %s' % str(self.conan_ref), ignore_error=True) self.assertIn("There is no local conanfile exported as %s" % str(self.conan_ref), self.client.user_io.out) files = hello_source_files() self.client.save(files, path=reg_folder) self.client.save({CONANFILE: myconan1, "include/math/lib1.h": "//copy", "my_lib/debug/libd.a": "//copy", "my_data/readme.txt": "//copy", "my_bin/executable": "//copy"}, path=reg_folder) conan_digest.save(reg_folder) mkdir(self.client.client_cache.export_sources(self.conan_ref)) self.package_ref = PackageReference(self.conan_ref, "myfakeid") self.server_pack_folder = self.test_server.paths.package(self.package_ref) package_folder = self.client.paths.package(self.package_ref) save(os.path.join(package_folder, "include", "lib1.h"), "//header") save(os.path.join(package_folder, "lib", "my_lib", "libd.a"), "//lib") save(os.path.join(package_folder, "res", "shares", "readme.txt"), "//res") save(os.path.join(package_folder, "bin", "my_bin", "executable"), "//bin") save(os.path.join(package_folder, CONANINFO), "info") save(os.path.join(package_folder, CONAN_MANIFEST), "manifest") os.chmod(os.path.join(package_folder, "bin", "my_bin", "executable"), os.stat(os.path.join(package_folder, "bin", "my_bin", "executable")).st_mode | stat.S_IRWXU) digest_path = self.client.client_cache.digestfile_package(self.package_ref) expected_manifest = FileTreeManifest.create(os.path.dirname(digest_path)) expected_manifest.save(package_folder) self.server_reg_folder = self.test_server.paths.export(self.conan_ref) self.assertFalse(os.path.exists(self.server_reg_folder)) self.assertFalse(os.path.exists(self.server_pack_folder))
def test_tree_manifest(self): tmp_dir = temp_folder() files = {"one.ext": "aalakjshdlkjahsdlkjahsdljkhsadljkhasljkdhlkjashd", "path/to/two.txt": "asdas13123", "two.txt": "asdasdasdasdasdasd"} for filename, content in files.iteritems(): save(os.path.join(tmp_dir, filename), content) manifest = FileTreeManifest.create(tmp_dir) save(os.path.join(tmp_dir, "THEMANIFEST.txt"), str(manifest)) readed_manifest = FileTreeManifest.loads(load(os.path.join(tmp_dir, "THEMANIFEST.txt"))) self.assertEquals(readed_manifest, manifest) for filepath, md5readed in manifest.file_sums.iteritems(): content = files[filepath] self.assertEquals(md5(content), md5readed)
def already_pyc_in_manifest_test(self): tmp_dir = temp_folder() save(os.path.join(tmp_dir, "man.txt"), "1478122267\nconanfile.pyc: " "2bcac725a0e6843ef351f4d18cf867ec\n" "conanfile.py: 2bcac725a0e6843ef351f4d18cf867ec", "conanfile.pyo: 2bcac725a0e6843ef351f4d18cf867ec") read_manifest = FileTreeManifest.loads(load(os.path.join(tmp_dir, "man.txt"))) # Not included the pycs or pyo self.assertEquals(set(read_manifest.file_sums.keys()), set(["conanfile.py"]))
def get_conan_digest(self, conan_reference): """Gets a FileTreeManifest from conans""" # Obtain the URLs url = "%s/conans/%s/digest" % (self._remote_api_url, "/".join(conan_reference)) urls = self._get_json(url) # Get the digest contents = self.download_files(urls) contents = {key: decode_text(value) for key, value in dict(contents).items()} # Unroll generator and decode shas (plain text) return FileTreeManifest.loads(contents[CONAN_MANIFEST])
def get_conan_digest(self, conan_reference): """Gets a FileTreeManifest from conans""" # Obtain the URLs url = "%s/conans/%s/digest" % (self._remote_api_url, "/".join(conan_reference)) urls = self._get_json(url) # Get the digest contents = self.download_files(urls) contents = dict(contents) # Unroll generator return FileTreeManifest.loads(contents[CONAN_MANIFEST])
def setUp(self): self.servers = {"local": TestServer(server_capabilities=[]), "search_able": TestServer(server_capabilities=[COMPLEX_SEARCH_CAPABILITY])} self.client = TestClient(servers=self.servers) # No conans created self.client.run("search") output = self.client.user_io.out self.assertIn('There are no packages', output) # Conans with and without packages created self.root_folder1 = 'Hello/1.4.10/fenix/testing' root_folder2 = 'helloTest/1.4.10/fenix/stable' root_folder3 = 'Bye/0.14/fenix/testing' root_folder4 = 'NodeInfo/1.0.2/fenix/stable' root_folder5 = 'MissFile/1.0.2/fenix/stable' root_folder11 = 'Hello/1.4.11/fenix/testing' root_folder12 = 'Hello/1.4.12/fenix/testing' self.client.save({"Empty/1.10/fake/test/reg/fake.txt": "//", "%s/%s/WindowsPackageSHA/%s" % (self.root_folder1, PACKAGES_FOLDER, CONANINFO): conan_vars1, "%s/%s/WindowsPackageSHA/%s" % (root_folder11, PACKAGES_FOLDER, CONANINFO): conan_vars1, "%s/%s/WindowsPackageSHA/%s" % (root_folder12, PACKAGES_FOLDER, CONANINFO): conan_vars1, "%s/%s/PlatformIndependantSHA/%s" % (self.root_folder1, PACKAGES_FOLDER, CONANINFO): conan_vars1b, "%s/%s/LinuxPackageSHA/%s" % (self.root_folder1, PACKAGES_FOLDER, CONANINFO): conan_vars1c, "%s/%s/a44f541cd44w57/%s" % (root_folder2, PACKAGES_FOLDER, CONANINFO): conan_vars2, "%s/%s/e4f7vdwcv4w55d/%s" % (root_folder3, PACKAGES_FOLDER, CONANINFO): conan_vars3, "%s/%s/e4f7vdwcv4w55d/%s" % (root_folder4, PACKAGES_FOLDER, CONANINFO): conan_vars4, "%s/%s/e4f7vdwcv4w55d/%s" % (root_folder5, PACKAGES_FOLDER, "hello.txt"): "Hello"}, self.client.paths.store) # Fake some manifests to be able to calculate recipe hash fake_manifest = FileTreeManifest(1212, {}) fake_manifest.save(os.path.join(self.client.paths.store, self.root_folder1, EXPORT_FOLDER)) fake_manifest.save(os.path.join(self.client.paths.store, root_folder2, EXPORT_FOLDER)) fake_manifest.save(os.path.join(self.client.paths.store, root_folder3, EXPORT_FOLDER)) fake_manifest.save(os.path.join(self.client.paths.store, root_folder4, EXPORT_FOLDER)) fake_manifest.save(os.path.join(self.client.paths.store, root_folder5, EXPORT_FOLDER))
def upload_same_package_dont_compress_test(self): # Create a manifest for the faked package pack_path = self.client.paths.package(self.package_ref) digest_path = self.client.client_cache.digestfile_package(self.package_ref) expected_manifest = FileTreeManifest.create(os.path.dirname(digest_path)) save(os.path.join(pack_path, CONAN_MANIFEST), str(expected_manifest)) self.client.run("upload %s --all" % str(self.conan_ref), ignore_error=False) self.assertIn("Compressing package", str(self.client.user_io.out)) self.client.run("upload %s --all" % str(self.conan_ref), ignore_error=False) self.assertNotIn("Compressing package", str(self.client.user_io.out)) self.assertIn("Package is up to date", str(self.client.user_io.out))
def _check(self, reference, manifest, remote, path): if os.path.exists(path): existing_manifest = FileTreeManifest.loads(load(path)) if existing_manifest.file_sums == manifest.file_sums: self._log.append("Manifest for '%s': OK" % str(reference)) return if self._verify: raise ConanException("Modified or new manifest '%s' detected.\n" "Remote: %s\nProject manifest doesn't match installed one" % (str(reference), remote)) self._handle_add(reference, remote, manifest, path)
def _init_export_folder(destination_folder): previous_digest = None try: if os.path.exists(destination_folder): if os.path.exists(os.path.join(destination_folder, CONAN_MANIFEST)): manifest_content = load(os.path.join(destination_folder, CONAN_MANIFEST)) previous_digest = FileTreeManifest.loads(manifest_content) # Maybe here we want to invalidate cache rmdir(destination_folder) os.makedirs(destination_folder) except Exception as e: raise ConanException("Unable to create folder %s\n%s" % (destination_folder, str(e))) return previous_digest
def get_package_manifest(self, package_reference): """Gets a FileTreeManifest from a package""" # Obtain the URLs url = "%s/conans/%s/packages/%s/digest" % (self._remote_api_url, "/".join(package_reference.conan), package_reference.package_id) urls = self._get_file_to_url_dict(url) # Get the digest contents = self.download_files(urls) # Unroll generator and decode shas (plain text) contents = {key: decode_text(value) for key, value in dict(contents).items()} return FileTreeManifest.loads(contents[CONAN_MANIFEST])
def _create_aux_files(build_folder, package_folder): """ auxiliary method that creates CONANINFO in the package_folder """ try: logger.debug("Creating config files to %s" % package_folder) shutil.copy(os.path.join(build_folder, CONANINFO), package_folder) # Create the digest for the package digest = FileTreeManifest.create(package_folder) save(os.path.join(package_folder, CONAN_MANIFEST), str(digest)) except IOError: raise ConanException("%s does not exist inside of your % folder. Try to re-build it again" " to solve it." % (CONANINFO, build_folder))
def setUp(self): servers = {} # All can write (for avoid authentication until we mock user_io) self.test_server = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")], users={"lasote": "mypass"}) servers["default"] = self.test_server conan_digest = FileTreeManifest('123123123', {}) self.client = TestClient(servers=servers, users={"default": [("lasote", "mypass")]}) self.conan_ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable") reg_folder = self.client.paths.export(self.conan_ref) self.client.run('upload %s' % str(self.conan_ref), ignore_error=True) self.assertIn("There is no local conanfile exported as %s" % str(self.conan_ref), self.client.user_io.out) files = hello_source_files() self.client.save(files, path=reg_folder) self.client.save({CONANFILE: myconan1, CONAN_MANIFEST: str(conan_digest), "include/math/lib1.h": "//copy", "my_lib/debug/libd.a": "//copy", "my_data/readme.txt": "//copy", "my_bin/executable": "//copy"}, path=reg_folder) self.package_ref = PackageReference(self.conan_ref, "myfakeid") self.server_pack_folder = self.test_server.paths.package(self.package_ref) package_folder = self.client.paths.package(self.package_ref) save(os.path.join(package_folder, "include", "lib1.h"), "//header") save(os.path.join(package_folder, "lib", "my_lib", "libd.a"), "//lib") save(os.path.join(package_folder, "res", "shares", "readme.txt"), "//res") save(os.path.join(package_folder, "bin", "my_bin", "executable"), "//bin") save(os.path.join(package_folder, CONANINFO), "info") save(os.path.join(package_folder, CONAN_MANIFEST), "manifest") os.chmod(os.path.join(package_folder, "bin", "my_bin", "executable"), os.stat(os.path.join(package_folder, "bin", "my_bin", "executable")).st_mode | stat.S_IRWXU) digest_path = self.client.client_cache.digestfile_package(self.package_ref) expected_manifest = FileTreeManifest.create(os.path.dirname(digest_path)) save(os.path.join(package_folder, CONAN_MANIFEST), str(expected_manifest)) self.server_reg_folder = self.test_server.paths.export(self.conan_ref) self.assertFalse(os.path.exists(self.server_reg_folder)) self.assertFalse(os.path.exists(self.server_pack_folder))
def complete_test(self): """ basic installation of a new conans """ servers = {} # All can write (for avoid authentication until we mock user_io) test_server = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")]) servers["default"] = test_server conan_digest = FileTreeManifest(123123123, {}) client = TestClient(servers=servers) client.init_dynamic_vars() conan_ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable") reg_folder = client.paths.export(conan_ref) files = hello_source_files() client.save(files, path=reg_folder) client.save( { CONANFILE: myconan1, CONAN_MANIFEST: str(conan_digest), "include/math/lib1.h": "//copy", "my_lib/debug/libd.a": "//copy", "my_data/readme.txt": "//copy" }, path=reg_folder) package_ref = PackageReference(conan_ref, "fakeid") package_folder = client.paths.package(package_ref) save(os.path.join(package_folder, "include", "lib1.h"), "//header") save(os.path.join(package_folder, "lib", "my_lib", "libd.a"), "//lib") save(os.path.join(package_folder, "res", "shares", "readme.txt"), "//res") client.remote_manager.upload_conan(conan_ref) client.remote_manager.upload_package(package_ref) client2 = TestClient(servers=servers) client2.init_dynamic_vars() loader = ConanFileLoader(TestBufferConanOutput(), None, Settings(), OptionsValues.loads("")) installer = ConanInstaller(client2.paths, client2.user_io, loader, client2.remote_manager, "default") installer.retrieve_conanfile(conan_ref) installer._retrieve_remote_package(package_ref) reg_path = client2.paths.export( ConanFileReference.loads("Hello/1.2.1/frodo/stable")) pack_folder = client2.paths.package(package_ref) # Test the file in the downloaded conans files = [ 'CMakeLists.txt', 'my_lib/debug/libd.a', 'hello.cpp', 'hello0.h', CONANFILE, CONAN_MANIFEST, 'main.cpp', 'include/math/lib1.h', 'my_data/readme.txt' ] for _file in files: self.assertTrue(os.path.exists(os.path.join(reg_path, _file))) self.assertTrue(os.path.exists(pack_folder)) # Test the file in the downloaded package self.assertTrue(os.path.exists(pack_folder)) self.assertTrue( os.path.exists(os.path.join(pack_folder, "include", "lib1.h"))) self.assertTrue( os.path.exists(os.path.join(pack_folder, "lib", "my_lib/libd.a"))) self.assertTrue( os.path.exists( os.path.join(pack_folder, "res", "shares/readme.txt")))
def recipe_manifest(self): return FileTreeManifest.load(self.export())
def setUp(self): hello_files = cpp_hello_conan_files("Hello") test_conanfile_contents = hello_files[CONANFILE] self.server_folder = temp_folder() test_server = TestServer( users={"fenix": "mypass"}, base_path=self.server_folder) # exported users and passwords self.server = test_server servers = {"default": test_server} client = TestClient(servers=servers, users={"default": [("fenix", "mypass")]}) # Conans with and without packages created self.root_folder = { "H1": 'Hello/1.4.10/fenix/testing', "H2": 'Hello/2.4.11/fenix/testing', "B": 'Bye/0.14/fenix/testing', "O": 'Other/1.2/fenix/testing' } files = {} pack_refs = [] for key, folder in self.root_folder.items(): ref = ConanFileReference.loads(folder) files["%s/%s/conanfile.py" % (folder, EXPORT_FOLDER)] = test_conanfile_contents files["%s/%s/conanmanifest.txt" % (folder, EXPORT_FOLDER)] = "" files["%s/%s/conans.txt" % (folder, SRC_FOLDER)] = "" for pack_id in (1, 2): i = pack_id pack_id = "%s_%s" % (pack_id, key) pack_refs.append(PackageReference(ref, str(pack_id))) files["%s/%s/%s/conans.txt" % (folder, BUILD_FOLDER, pack_id)] = "" files["%s/%s/%s/conans.txt" % (folder, PACKAGES_FOLDER, pack_id)] = "" files["%s/%s/%s/%s" % (folder, PACKAGES_FOLDER, pack_id, CONANINFO)] = conaninfo % str(i) files["%s/%s/%s/%s" % (folder, PACKAGES_FOLDER, pack_id, CONAN_MANIFEST)] = "" exports_sources_dir = client.client_cache.export_sources(ref) os.makedirs(exports_sources_dir) client.save(files, client.client_cache.store) # Create the manifests to be able to upload for pack_ref in pack_refs: digest_path = client.client_cache.digestfile_package(pack_ref) expected_manifest = FileTreeManifest.create( os.path.dirname(digest_path)) files["%s/%s/%s/%s" % ("/".join( pack_ref.conan), PACKAGES_FOLDER, pack_ref.package_id, CONAN_MANIFEST)] = str(expected_manifest) client.save(files, client.client_cache.store) self.client = client for folder in self.root_folder.values(): client.run("upload %s --all" % folder.replace("/fenix", "@fenix")) self.assert_folders( { "H1": [1, 2], "H2": [1, 2], "B": [1, 2], "O": [1, 2] }, { "H1": [1, 2], "H2": [1, 2], "B": [1, 2], "O": [1, 2] }, { "H1": [1, 2], "H2": [1, 2], "B": [1, 2], "O": [1, 2] }, { "H1": True, "H2": True, "B": True, "O": True })
def load_package_manifest(self, package_reference): """conan_id = sha(zip file)""" package_folder = self.package(package_reference, short_paths=None) return FileTreeManifest.load(package_folder)
def _get_recipe(self, reference, check_updates, update, remote_name, recorder): output = ScopedOutput(str(reference), self._out) # check if it is in disk conanfile_path = self._client_cache.conanfile(reference) # NOT in disk, must be retrieved from remotes if not os.path.exists(conanfile_path): remote, new_ref = self._download_recipe(reference, output, remote_name, recorder) status = RECIPE_DOWNLOADED return conanfile_path, status, remote, new_ref metadata = self._client_cache.load_metadata(reference) cur_revision = metadata.recipe.revision remote = self._registry.refs.get(reference) named_remote = self._registry.remotes.get( remote_name) if remote_name else None update_remote = named_remote or remote # Check if we have a revision different from the requested one revisions_enabled = get_env("CONAN_CLIENT_REVISIONS_ENABLED", False) if revisions_enabled and reference.revision and cur_revision != reference.revision: output.info( "Different revision requested, removing current local recipe..." ) DiskRemover(self._client_cache).remove_recipe(reference) output.info("Retrieving from remote '%s'..." % update_remote.name) new_ref = self._remote_manager.get_recipe(reference, update_remote) self._registry.refs.set(new_ref, update_remote.name) status = RECIPE_UPDATED return conanfile_path, status, update_remote, new_ref check_updates = check_updates or update # Recipe exists in disk, but no need to check updates if not check_updates: status = RECIPE_INCACHE ref = reference.copy_with_rev(cur_revision) return conanfile_path, status, remote, ref if not update_remote: status = RECIPE_NO_REMOTE ref = reference.copy_with_rev(cur_revision) return conanfile_path, status, None, ref try: # get_conan_manifest can fail, not in server upstream_manifest = self._remote_manager.get_conan_manifest( reference, update_remote) except NotFoundException: status = RECIPE_NOT_IN_REMOTE ref = reference.copy_with_rev(cur_revision) return conanfile_path, status, update_remote, ref export = self._client_cache.export(reference) read_manifest = FileTreeManifest.load(export) if upstream_manifest != read_manifest: if upstream_manifest.time > read_manifest.time: if update: DiskRemover(self._client_cache).remove_recipe(reference) output.info("Retrieving from remote '%s'..." % update_remote.name) new_ref = self._remote_manager.get_recipe( reference, update_remote) self._registry.refs.set(new_ref, update_remote.name) status = RECIPE_UPDATED return conanfile_path, status, update_remote, new_ref else: status = RECIPE_UPDATEABLE else: status = RECIPE_NEWER else: status = RECIPE_INCACHE ref = reference.copy_with_rev(cur_revision) return conanfile_path, status, update_remote, ref
def cmd_export(app, conanfile_path, name, version, user, channel, keep_source, export=True, graph_lock=None): """ Export the recipe param conanfile_path: the original source directory of the user containing a conanfile.py """ loader, cache, hook_manager, output = app.loader, app.cache, app.hook_manager, app.out revisions_enabled = app.config.revisions_enabled conanfile = loader.load_export(conanfile_path, name, version, user, channel) # FIXME: Conan 2.0, deprecate CONAN_USER AND CONAN_CHANNEL and remove this try excepts # Take the default from the env vars if they exist to not break behavior try: user = conanfile.user except ConanException: user = None try: channel = conanfile.channel except ConanException: channel = None ref = ConanFileReference(conanfile.name, conanfile.version, user, channel) # If we receive lock information, python_requires could have been locked if graph_lock: node_id = graph_lock.get_node(ref) python_requires = graph_lock.python_requires(node_id) # TODO: check that the locked python_requires are different from the loaded ones # FIXME: private access, will be improved when api collaborators are improved loader._python_requires._range_resolver.output # invalidate previous version range output conanfile = loader.load_export(conanfile_path, conanfile.name, conanfile.version, conanfile.user, conanfile.channel, python_requires) check_casing_conflict(cache=cache, ref=ref) package_layout = cache.package_layout(ref, short_paths=conanfile.short_paths) if not export: metadata = package_layout.load_metadata() recipe_revision = metadata.recipe.revision ref = ref.copy_with_rev(recipe_revision) if graph_lock: graph_lock.update_exported_ref(node_id, ref) return ref hook_manager.execute("pre_export", conanfile=conanfile, conanfile_path=conanfile_path, reference=package_layout.ref) logger.debug("EXPORT: %s" % conanfile_path) output.highlight("Exporting package recipe") conan_linter(conanfile_path, output) output = conanfile.output # Get previous digest try: previous_manifest = FileTreeManifest.load(package_layout.export()) except IOError: previous_manifest = None finally: _recreate_folders(package_layout.export(), package_layout.export_sources()) # Copy sources to target folders with package_layout.conanfile_write_lock(output=output): origin_folder = os.path.dirname(conanfile_path) export_recipe(conanfile, origin_folder, package_layout.export()) export_source(conanfile, origin_folder, package_layout.export_sources()) shutil.copy2(conanfile_path, package_layout.conanfile()) _capture_export_scm_data(conanfile, os.path.dirname(conanfile_path), package_layout.export(), output, scm_src_file=package_layout.scm_folder()) # Execute post-export hook before computing the digest hook_manager.execute("post_export", conanfile=conanfile, reference=package_layout.ref, conanfile_path=package_layout.conanfile()) # Compute the new digest manifest = FileTreeManifest.create(package_layout.export(), package_layout.export_sources()) modified_recipe = not previous_manifest or previous_manifest != manifest if modified_recipe: output.success('A new %s version was exported' % CONANFILE) output.info('Folder: %s' % package_layout.export()) else: output.info("The stored package has not changed") manifest = previous_manifest # Use the old one, keep old timestamp manifest.save(package_layout.export()) # Compute the revision for the recipe revision = _update_revision_in_metadata( package_layout=package_layout, revisions_enabled=revisions_enabled, output=output, path=os.path.dirname(conanfile_path), manifest=manifest, revision_mode=conanfile.revision_mode) # FIXME: Conan 2.0 Clear the registry entry if the recipe has changed source_folder = package_layout.source() if os.path.exists(source_folder): try: if is_dirty(source_folder): output.info("Source folder is corrupted, forcing removal") rmdir(source_folder) elif modified_recipe and not keep_source: output.info( "Package recipe modified in export, forcing source folder removal" ) output.info("Use the --keep-source, -k option to skip it") rmdir(source_folder) except BaseException as e: output.error( "Unable to delete source folder. Will be marked as corrupted for deletion" ) output.warn(str(e)) set_dirty(source_folder) # When revisions enabled, remove the packages not matching the revision if revisions_enabled: packages = search_packages(package_layout, query=None) metadata = package_layout.load_metadata() recipe_revision = metadata.recipe.revision to_remove = [ pid for pid in packages if metadata.packages.get(pid) and metadata.packages.get(pid).recipe_revision != recipe_revision ] if to_remove: output.info( "Removing the local binary packages from different recipe revisions" ) remover = DiskRemover() remover.remove_packages(package_layout, ids_filter=to_remove) ref = ref.copy_with_rev(revision) output.info("Exported revision: %s" % revision) if graph_lock: graph_lock.update_exported_ref(node_id, ref) return ref
def setUp(self): self.servers = { "local": TestServer(server_capabilities=[]), "search_able": TestServer(server_capabilities=[COMPLEX_SEARCH_CAPABILITY]) } self.client = TestClient(servers=self.servers) # No conans created self.client.run("search") output = self.client.user_io.out self.assertIn('There are no packages', output) # Conans with and without packages created self.root_folder1 = 'Hello/1.4.10/fenix/testing' root_folder2 = 'helloTest/1.4.10/fenix/stable' root_folder3 = 'Bye/0.14/fenix/testing' root_folder4 = 'NodeInfo/1.0.2/fenix/stable' root_folder5 = 'MissFile/1.0.2/fenix/stable' root_folder11 = 'Hello/1.4.11/fenix/testing' root_folder12 = 'Hello/1.4.12/fenix/testing' self.client.save( { "Empty/1.10/fake/test/reg/fake.txt": "//", "%s/%s/WindowsPackageSHA/%s" % (self.root_folder1, PACKAGES_FOLDER, CONANINFO): conan_vars1, "%s/%s/WindowsPackageSHA/%s" % (root_folder11, PACKAGES_FOLDER, CONANINFO): conan_vars1, "%s/%s/WindowsPackageSHA/%s" % (root_folder12, PACKAGES_FOLDER, CONANINFO): conan_vars1, "%s/%s/PlatformIndependantSHA/%s" % (self.root_folder1, PACKAGES_FOLDER, CONANINFO): conan_vars1b, "%s/%s/LinuxPackageSHA/%s" % (self.root_folder1, PACKAGES_FOLDER, CONANINFO): conan_vars1c, "%s/%s/a44f541cd44w57/%s" % (root_folder2, PACKAGES_FOLDER, CONANINFO): conan_vars2, "%s/%s/e4f7vdwcv4w55d/%s" % (root_folder3, PACKAGES_FOLDER, CONANINFO): conan_vars3, "%s/%s/e4f7vdwcv4w55d/%s" % (root_folder4, PACKAGES_FOLDER, CONANINFO): conan_vars4, "%s/%s/e4f7vdwcv4w55d/%s" % (root_folder5, PACKAGES_FOLDER, "hello.txt"): "Hello" }, self.client.paths.store) # Fake some manifests to be able to calculate recipe hash fake_manifest = FileTreeManifest(1212, {}) fake_manifest.save( os.path.join(self.client.paths.store, self.root_folder1, EXPORT_FOLDER)) fake_manifest.save( os.path.join(self.client.paths.store, root_folder2, EXPORT_FOLDER)) fake_manifest.save( os.path.join(self.client.paths.store, root_folder3, EXPORT_FOLDER)) fake_manifest.save( os.path.join(self.client.paths.store, root_folder4, EXPORT_FOLDER)) fake_manifest.save( os.path.join(self.client.paths.store, root_folder5, EXPORT_FOLDER))
def deploy_test(self, deploy_to_abs): client = TestClient() libconanfile = """from conans import ConanFile from conans.tools import save class Lib(ConanFile): exports_sources = "*" def build(self): save("mylib.dll", "mydll") def package(self): self.copy("*") def deploy(self): self.output.info("Lib deploy()") """ client.save({ "conanfile.py": libconanfile, "License.md": "lib license", "otherfile": "" }) client.run("create Lib/0.1@user/testing") self.assertNotIn("Lib deploy()", client.out) if deploy_to_abs: dll_folder = temp_folder() mkdir(dll_folder) else: dll_folder = "" conanfile = """from conans import ConanFile from conans.tools import save class Pkg(ConanFile): requires = "Lib/0.1@user/testing" def build(self): save("myapp.exe", "myexe") def package(self): self.copy("*") def deploy(self): self.output.info("Pkg deploy()") self.copy("*.exe") self.copy_deps("*.dll", dst="%s") """ % dll_folder.replace("\\", "/") client.save({"conanfile.py": conanfile}) client.run("create Pkg/0.1@user/testing") self.assertNotIn("deploy()", client.out) client.current_folder = temp_folder() client.run("install Pkg/0.1@user/testing") self.assertIn( "Pkg/0.1@user/testing deploy(): Copied 1 '.dll' files: mylib.dll", client.out) self.assertIn( "Pkg/0.1@user/testing deploy(): Copied 1 '.exe' files: myapp.exe", client.out) deploy_manifest = FileTreeManifest.loads( load(os.path.join(client.current_folder, "deploy_manifest.txt"))) app = os.path.join(client.current_folder, "myapp.exe") if deploy_to_abs: lib = os.path.join(dll_folder, "mylib.dll") else: lib = os.path.join(client.current_folder, "mylib.dll") self.assertEqual(sorted([app, lib]), sorted(deploy_manifest.file_sums.keys())) self.assertEqual(load(app), "myexe") self.assertEqual(load(lib), "mydll")
def cmd_export(conanfile_path, name, version, user, channel, keep_source, revisions_enabled, output, hook_manager, loader, cache, export=True): """ Export the recipe param conanfile_path: the original source directory of the user containing a conanfile.py """ conanfile = loader.load_export(conanfile_path, name, version, user, channel) ref = ConanFileReference(conanfile.name, conanfile.version, conanfile.user, conanfile.channel) check_casing_conflict(cache=cache, ref=ref) package_layout = cache.package_layout(ref, short_paths=conanfile.short_paths) if not export: metadata = package_layout.load_metadata() recipe_revision = metadata.recipe.revision ref = ref.copy_with_rev(recipe_revision) return ref hook_manager.execute("pre_export", conanfile=conanfile, conanfile_path=conanfile_path, reference=package_layout.ref) logger.debug("EXPORT: %s" % conanfile_path) output.highlight("Exporting package recipe") conan_linter(conanfile_path, output) output = conanfile.output # Get previous digest try: previous_digest = FileTreeManifest.load(package_layout.export()) except IOError: previous_digest = None finally: _recreate_folders(package_layout.export(), package_layout.export_sources()) # Copy sources to target folders with package_layout.conanfile_write_lock(output=output): origin_folder = os.path.dirname(conanfile_path) export_recipe(conanfile, origin_folder, package_layout.export()) export_source(conanfile, origin_folder, package_layout.export_sources()) shutil.copy2(conanfile_path, package_layout.conanfile()) _capture_export_scm_data(conanfile, os.path.dirname(conanfile_path), package_layout.export(), output, scm_src_file=package_layout.scm_folder()) # Execute post-export hook before computing the digest hook_manager.execute("post_export", conanfile=conanfile, reference=package_layout.ref, conanfile_path=package_layout.conanfile()) # Compute the new digest digest = FileTreeManifest.create(package_layout.export(), package_layout.export_sources()) modified_recipe = not previous_digest or previous_digest != digest if modified_recipe: output.success('A new %s version was exported' % CONANFILE) output.info('Folder: %s' % package_layout.export()) else: output.info("The stored package has not changed") digest = previous_digest # Use the old one, keep old timestamp digest.save(package_layout.export()) # Compute the revision for the recipe revision = _update_revision_in_metadata( package_layout=package_layout, revisions_enabled=revisions_enabled, output=output, path=os.path.dirname(conanfile_path), digest=digest, revision_mode=conanfile.revision_mode) # FIXME: Conan 2.0 Clear the registry entry if the recipe has changed source_folder = package_layout.source() if os.path.exists(source_folder): try: if is_dirty(source_folder): output.info("Source folder is corrupted, forcing removal") rmdir(source_folder) elif modified_recipe and not keep_source: output.info( "Package recipe modified in export, forcing source folder removal" ) output.info("Use the --keep-source, -k option to skip it") rmdir(source_folder) except BaseException as e: output.error( "Unable to delete source folder. Will be marked as corrupted for deletion" ) output.warn(str(e)) set_dirty(source_folder) # When revisions enabled, remove the packages not matching the revision if revisions_enabled: packages = search_packages(package_layout, query=None) metadata = package_layout.load_metadata() recipe_revision = metadata.recipe.revision to_remove = [ pid for pid in packages if metadata.packages.get(pid) and metadata.packages.get(pid).recipe_revision != recipe_revision ] if to_remove: output.info( "Removing the local binary packages from different recipe revisions" ) remover = DiskRemover() remover.remove_packages(package_layout, ids_filter=to_remove) ref = ref.copy_with_rev(revision) return ref
def _digests(digest_path, exports_sources_folder=None): readed_digest = FileTreeManifest.loads(load(digest_path)) expected_digest = FileTreeManifest.create(os.path.dirname(digest_path), exports_sources_folder) return readed_digest, expected_digest
def _create_manifest(self, package_reference): # Create the manifest to be able to upload the package pack_path = self.client.paths.package(package_reference) digest_path = self.client.client_cache.digestfile_package(package_reference) expected_manifest = FileTreeManifest.create(os.path.dirname(digest_path)) save(os.path.join(pack_path, CONAN_MANIFEST), str(expected_manifest))
def inc_package_manifest_timestamp(cache, package_reference, inc_time): pref = PackageReference.loads(package_reference) path = cache.package_layout(pref.ref).package(pref) manifest = FileTreeManifest.load(path) manifest.time += inc_time manifest.save(path)
def bump_time(inc_time): path = client.client_cache.export( ConanFileReference.loads("Pkg/0.1@lasote/testing")) manifest = FileTreeManifest.load(path) manifest.time += inc_time manifest.save(path)
def load_digest(self, conan_reference): '''conan_id = sha(zip file)''' filename = os.path.join(self.export(conan_reference), CONAN_MANIFEST) return FileTreeManifest.loads(load(filename))
def _digests(self, digest_path): if not path_exists(digest_path, self.store): return None, None readed_digest = FileTreeManifest.loads(load(digest_path)) expected_digest = FileTreeManifest.create(os.path.dirname(digest_path)) return readed_digest, expected_digest
def load_manifest(self, conan_reference): '''conan_id = sha(zip file)''' filename = self.digestfile_conanfile(conan_reference) return FileTreeManifest.loads(load(filename))
def setUp(self): self.client = self._get_client() self.ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable#%s" % DEFAULT_REVISION_V1) self.pref = PackageReference(self.ref, "myfakeid", DEFAULT_REVISION_V1) reg_folder = self.client.cache.package_layout(self.ref).export() self.client.run('upload %s' % str(self.ref), assert_error=True) self.assertIn("ERROR: Recipe not found: '%s'" % str(self.ref), self.client.out) files = hello_source_files() fake_metadata = PackageMetadata() fake_metadata.recipe.revision = DEFAULT_REVISION_V1 fake_metadata.packages[self.pref.id].revision = DEFAULT_REVISION_V1 self.client.save({"metadata.json": fake_metadata.dumps()}, path=self.client.cache.package_layout( self.ref).base_folder()) self.client.save(files, path=reg_folder) self.client.save( { CONANFILE: myconan1, "include/math/lib1.h": "//copy", "my_lib/debug/libd.a": "//copy", "my_data/readme.txt": "//copy", "my_bin/executable": "//copy" }, path=reg_folder) mkdir(self.client.cache.package_layout(self.ref).export_sources()) manifest = FileTreeManifest.create(reg_folder) manifest.time = '123123123' manifest.save(reg_folder) self.test_server.server_store.update_last_revision(self.ref) self.server_pack_folder = self.test_server.server_store.package( self.pref) package_folder = self.client.cache.package_layout(self.ref).package( self.pref) save(os.path.join(package_folder, "include", "lib1.h"), "//header") save(os.path.join(package_folder, "lib", "my_lib", "libd.a"), "//lib") save(os.path.join(package_folder, "res", "shares", "readme.txt"), "//res") save(os.path.join(package_folder, "bin", "my_bin", "executable"), "//bin") save(os.path.join(package_folder, CONANINFO), """[recipe_hash]\n%s""" % manifest.summary_hash) FileTreeManifest.create(package_folder).save(package_folder) self.test_server.server_store.update_last_package_revision(self.pref) os.chmod( os.path.join(package_folder, "bin", "my_bin", "executable"), os.stat(os.path.join(package_folder, "bin", "my_bin", "executable")).st_mode | stat.S_IRWXU) expected_manifest = FileTreeManifest.create(package_folder) expected_manifest.save(package_folder) self.server_reg_folder = self.test_server.server_store.export(self.ref) self.assertFalse(os.path.exists(self.server_reg_folder)) self.assertFalse(os.path.exists(self.server_pack_folder))
def load_package_manifest(self, package_reference): '''conan_id = sha(zip file)''' filename = self.digestfile_package(package_reference, short_paths=None) return FileTreeManifest.loads(load(filename))
def load_manifest(self, conan_reference): """conan_id = sha(zip file)""" assert isinstance(conan_reference, ConanFileReference) export_folder = self.export(conan_reference) check_ref_case(conan_reference, export_folder, self.store) return FileTreeManifest.load(export_folder)
def _digests(self, digest_path): readed_digest = FileTreeManifest.loads(load(digest_path)) expected_digest = FileTreeManifest.create(os.path.dirname(digest_path)) return readed_digest, expected_digest
def _digests(folder, exports_sources_folder=None): readed_digest = FileTreeManifest.load(folder) expected_digest = FileTreeManifest.create(folder, exports_sources_folder) return readed_digest, expected_digest
def setUp(self): hello_files = cpp_hello_conan_files("Hello") test_conanfile_contents = hello_files[CONANFILE] self.server_folder = temp_folder() test_server = TestServer( users={"myuser": "******"}, base_path=self.server_folder) # exported users and passwords self.server = test_server servers = {"default": test_server} client = TestClient(servers=servers, users={"default": [("myuser", "mypass")]}) # Conans with and without packages created self.root_folder = { "H1": 'Hello/1.4.10@myuser/testing', "H2": 'Hello/2.4.11@myuser/testing', "B": 'Bye/0.14@myuser/testing', "O": 'Other/1.2@myuser/testing' } files = {} prefs = [] for key, folder in self.root_folder.items(): ref = ConanFileReference.loads(folder) folder = folder.replace("@", "/") fake_metadata = PackageMetadata() fake_metadata.recipe.revision = DEFAULT_REVISION_V1 files["%s/%s/conanfile.py" % (folder, EXPORT_FOLDER)] = test_conanfile_contents files["%s/%s/conanmanifest.txt" % (folder, EXPORT_FOLDER )] = "%s\nconanfile.py: 234234234" % fake_recipe_hash files["%s/%s/conans.txt" % (folder, SRC_FOLDER)] = "" for pack_id in (1, 2): i = pack_id pack_id = "%s_%s" % (pack_id, key) fake_metadata.packages[pack_id].revision = DEFAULT_REVISION_V1 prefs.append(PackageReference(ref, str(pack_id))) files["%s/%s/%s/conans.txt" % (folder, BUILD_FOLDER, pack_id)] = "" files["%s/%s/%s/conans.txt" % (folder, PACKAGES_FOLDER, pack_id)] = "" files["%s/%s/%s/%s" % ( folder, PACKAGES_FOLDER, pack_id, CONANINFO )] = conaninfo % str(i) + "905eefe3570dd09a8453b30b9272bb44" files["%s/%s/%s/%s" % (folder, PACKAGES_FOLDER, pack_id, CONAN_MANIFEST)] = "" files["%s/metadata.json" % folder] = fake_metadata.dumps() exports_sources_dir = client.cache.package_layout( ref).export_sources() os.makedirs(exports_sources_dir) client.save(files, client.cache.store) # Create the manifests to be able to upload for pref in prefs: pkg_folder = client.cache.package_layout(pref.ref).package(pref) expected_manifest = FileTreeManifest.create(pkg_folder) files["%s/%s/%s/%s" % (pref.ref.dir_repr(), PACKAGES_FOLDER, pref.id, CONAN_MANIFEST)] = repr(expected_manifest) client.save(files, client.cache.store) self.client = client for folder in self.root_folder.values(): client.run("upload %s --all" % folder) self.assert_folders( { "H1": [1, 2], "H2": [1, 2], "B": [1, 2], "O": [1, 2] }, { "H1": [1, 2], "H2": [1, 2], "B": [1, 2], "O": [1, 2] }, { "H1": [1, 2], "H2": [1, 2], "B": [1, 2], "O": [1, 2] }, { "H1": True, "H2": True, "B": True, "O": True })
def _create_manifest(self, client, package_reference): # Create the manifest to be able to upload the package pack_path = client.cache.package(package_reference) expected_manifest = FileTreeManifest.create(pack_path) expected_manifest.save(pack_path)
def _get_recipe(self, ref, check_updates, update, remotes, recorder): output = ScopedOutput(str(ref), self._out) # check if it is in disk conanfile_path = self._cache.package_layout(ref).conanfile() # NOT in disk, must be retrieved from remotes if not os.path.exists(conanfile_path): remote, new_ref = self._download_recipe(ref, output, remotes, remotes.selected, recorder) status = RECIPE_DOWNLOADED return conanfile_path, status, remote, new_ref metadata = self._cache.package_layout(ref).load_metadata() cur_revision = metadata.recipe.revision cur_remote = metadata.recipe.remote cur_remote = remotes[cur_remote] if cur_remote else None selected_remote = remotes.selected or cur_remote check_updates = check_updates or update # Recipe exists in disk, but no need to check updates requested_different_revision = ( ref.revision is not None) and cur_revision != ref.revision if requested_different_revision and not check_updates: raise NotFoundException( "The recipe in the local cache doesn't match the specified " "revision. Use '--update' to check in the remote.") if not requested_different_revision: if not check_updates: status = RECIPE_INCACHE ref = ref.copy_with_rev(cur_revision) return conanfile_path, status, cur_remote, ref if not selected_remote: status = RECIPE_NO_REMOTE ref = ref.copy_with_rev(cur_revision) return conanfile_path, status, None, ref else: # Requested different revision and with --update remote, new_ref = self._download_recipe(ref, output, remotes, selected_remote, recorder) status = RECIPE_DOWNLOADED return conanfile_path, status, remote, new_ref try: # get_recipe_manifest can fail, not in server upstream_manifest, ref = self._remote_manager.get_recipe_manifest( ref, selected_remote) except NotFoundException: status = RECIPE_NOT_IN_REMOTE ref = ref.copy_with_rev(cur_revision) return conanfile_path, status, selected_remote, ref export = self._cache.package_layout(ref).export() read_manifest = FileTreeManifest.load(export) if upstream_manifest != read_manifest: if upstream_manifest.time > read_manifest.time: if update: DiskRemover().remove_recipe( self._cache.package_layout(ref), output=output) output.info("Retrieving from remote '%s'..." % selected_remote.name) self._download_recipe(ref, output, remotes, selected_remote, recorder) with self._cache.package_layout( ref).update_metadata() as metadata: metadata.recipe.remote = selected_remote.name status = RECIPE_UPDATED return conanfile_path, status, selected_remote, ref else: status = RECIPE_UPDATEABLE else: status = RECIPE_NEWER else: status = RECIPE_INCACHE ref = ref.copy_with_rev(cur_revision) return conanfile_path, status, selected_remote, ref
def inc_recipe_manifest_timestamp(cache, reference, inc_time): ref = ConanFileReference.loads(reference) path = cache.package_layout(ref).export() manifest = FileTreeManifest.load(path) manifest.time += inc_time manifest.save(path)
def package_manifests(self, pref): package_folder = self.package(pref) readed_manifest = FileTreeManifest.load(package_folder) expected_manifest = FileTreeManifest.create(package_folder) return readed_manifest, expected_manifest
def _create_manifest(client, pref): # Create the manifest to be able to upload the package pack_path = client.cache.package_layout(pref.ref).package(pref) expected_manifest = FileTreeManifest.create(pack_path) expected_manifest.save(pack_path)
def inc_package_manifest_timestamp(client_cache, package_ref, inc_time): pkg_ref = PackageReference.loads(str(package_ref)) path = client_cache.package(pkg_ref) manifest = FileTreeManifest.load(path) manifest.time += inc_time manifest.save(path)
def generate_manifest(package_folder): # Create the digest for the package digest = FileTreeManifest.create(package_folder) save(os.path.join(package_folder, CONAN_MANIFEST), str(digest))
def inc_recipe_manifest_timestamp(client_cache, conan_ref, inc_time): conan_ref = ConanFileReference.loads(str(conan_ref)) path = client_cache.export(conan_ref) manifest = FileTreeManifest.load(path) manifest.time += inc_time manifest.save(path)