def test_tree_manifest(self): tmp_dir = temp_folder() files = {"one.ext": "aalakjshdlkjahsdlkjahsdljkhsadljkhasljkdhlkjashd", "path/to/two.txt": "asdas13123", "two.txt": "asdasdasdasdasdasd", "folder/damn.pyc": "binarythings", "folder/damn.pyo": "binarythings2", "pythonfile.pyc": "binarythings3"} for filename, content in files.items(): save(os.path.join(tmp_dir, filename), content) manifest = FileTreeManifest.create(tmp_dir) manifest.save(tmp_dir) readed_manifest = FileTreeManifest.load(tmp_dir) self.assertEqual(readed_manifest.time, manifest.time) self.assertEqual(readed_manifest, manifest) # Not included the pycs or pyo self.assertEquals(set(manifest.file_sums.keys()), set(["one.ext", "path/to/two.txt", "two.txt"])) for filepath, md5readed in manifest.file_sums.items(): content = files[filepath] self.assertEquals(md5(content), md5readed)
def _remove_if_outdated(package_folder, package_ref, proxy, output): if os.path.exists(package_folder): try: # get_conan_digest can fail, not in server # FIXME: This can iterate remotes to get and associate in registry upstream_manifest = proxy.get_package_manifest(package_ref) except NotFoundException: output.warn("Can't update, no package in remote") except NoRemoteAvailable: output.warn("Can't update, no remote defined") else: read_manifest = FileTreeManifest.load(package_folder) if upstream_manifest != read_manifest: if upstream_manifest.time > read_manifest.time: output.warn("Current package is older than remote upstream one") output.warn("Removing it to retrieve or build an updated one") rmdir(package_folder) else: output.warn("Current package is newer than remote upstream one")
def test_export_the_same_code(self): file_list = self._create_packages_and_builds() # Export the same conans # Do not adjust cpu_count, it is reusing a cache client2 = TestClient(self.client.cache_folder, cpu_count=False) files2 = { "conanfile.py": GenConanfile("Hello0", "0.1").with_exports("*"), "main.cpp": "MyMain", "CMakeLists.txt": "MyCmake", "executable": "myexe" } client2.save(files2) client2.run("export . lasote/stable") reg_path2 = client2.cache.package_layout(self.ref).export() digest2 = FileTreeManifest.load( client2.cache.package_layout(self.ref).export()) self.assertNotIn('A new Conan version was exported', client2.out) self.assertNotIn('Cleaning the old builds ...', client2.out) self.assertNotIn('Cleaning the old packs ...', client2.out) self.assertNotIn('All the previous packs were cleaned', client2.out) self.assertIn( '%s: A new conanfile.py version was exported' % str(self.ref), self.client.out) self.assertIn('%s: Folder: %s' % (str(self.ref), reg_path2), self.client.out) self.assertTrue(os.path.exists(reg_path2)) for name in list(files2.keys()): self.assertTrue(os.path.exists(os.path.join(reg_path2, name))) expected_sums = { 'CMakeLists.txt': '3cf710785270c7e98a30d4a90ea66492', 'conanfile.py': '73eb512b9f02ac18278823a217cfff79', 'executable': 'db299d5f0d82f113fad627a21f175e59', 'main.cpp': 'd9c03c934a4b3b1670775c17c26f39e9' } self.assertEqual(expected_sums, digest2.file_sums) for f in file_list: self.assertTrue(os.path.exists(f))
def _failed_package_verify(self, reference, remote="local cache"): self.client.run("install %s --build missing --manifests" % str(reference)) self.assertIn("Installed manifest for 'Hello/0.1@lasote/stable' from %s" % remote, self.client.user_io.out) self.assertIn("Installed manifest for 'Hello/0.1@lasote/stable:" "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9' from %s" % remote, self.client.user_io.out) output_folder = os.path.join(self.client.current_folder, ".conan_manifests") paths = SimplePaths(output_folder) self.assertTrue(os.path.exists(os.path.join(paths.export(self.reference), CONAN_MANIFEST))) package_reference = PackageReference.loads("Hello/0.1@lasote/stable:" "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9") self.assertTrue(os.path.exists(os.path.join(paths.package(package_reference), CONAN_MANIFEST))) client = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]}) client.save(self.files) client.run("export . lasote/stable") client.run("install Hello/0.1@lasote/stable --build=missing") info = os.path.join(client.paths.package(package_reference), "conaninfo.txt") info_content = load(info) info_content += "# Dummy string" save(info, info_content) package_folder = client.paths.package(package_reference) manifest = FileTreeManifest.load(package_folder) manifest.file_sums["conaninfo.txt"] = md5(info_content) manifest.save(package_folder) client.run("upload %s --all" % str(self.reference)) # now verify, with update self.client.run("remove Hello/0.1@lasote/stable -f") self.client.run("install %s --build missing --verify" % str(self.reference), ignore_error=True) self.assertNotIn("Manifest for 'Hello/0.1@lasote/stable': OK", self.client.user_io.out) self.assertNotIn("Manifest for '%s': OK" % str(package_reference), self.client.user_io.out) self.assertIn("Modified or new manifest '%s' detected" % str(package_reference), self.client.user_io.out)
def _failed_package_verify(self, reference, remote="local cache"): self.client.run("install %s --build missing --manifests" % str(reference)) self.assertIn("Installed manifest for 'Hello/0.1@lasote/stable' from %s" % remote, self.client.user_io.out) self.assertIn("Installed manifest for 'Hello/0.1@lasote/stable:" "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9' from %s" % remote, self.client.user_io.out) output_folder = os.path.join(self.client.current_folder, ".conan_manifests") paths = SimplePaths(output_folder) self.assertTrue(os.path.exists(paths.digestfile_conanfile(self.reference))) package_reference = PackageReference.loads("Hello/0.1@lasote/stable:" "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9") self.assertTrue(os.path.exists(paths.digestfile_package(package_reference))) client = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]}) client.save(self.files) client.run("export . lasote/stable") client.run("install Hello/0.1@lasote/stable --build=missing") info = os.path.join(client.paths.package(package_reference), "conaninfo.txt") info_content = load(info) info_content += "# Dummy string" save(info, info_content) package_folder = client.paths.package(package_reference) manifest = FileTreeManifest.load(package_folder) manifest.file_sums["conaninfo.txt"] = md5(info_content) manifest.save(package_folder) client.run("upload %s --all" % str(self.reference)) # now verify, with update self.client.run("remove Hello/0.1@lasote/stable -f") self.client.run("install %s --build missing --verify" % str(self.reference), ignore_error=True) self.assertNotIn("Manifest for 'Hello/0.1@lasote/stable': OK", self.client.user_io.out) self.assertNotIn("Manifest for '%s': OK" % str(package_reference), self.client.user_io.out) self.assertIn("Modified or new manifest '%s' detected" % str(package_reference), self.client.user_io.out)
def _init_export_folder(destination_folder, destination_src_folder): previous_digest = None try: if os.path.exists(destination_folder): if os.path.exists(os.path.join(destination_folder, CONAN_MANIFEST)): previous_digest = FileTreeManifest.load(destination_folder) # Maybe here we want to invalidate cache rmdir(destination_folder) os.makedirs(destination_folder) except Exception as e: raise ConanException("Unable to create folder %s\n%s" % (destination_folder, str(e))) try: if os.path.exists(destination_src_folder): rmdir(destination_src_folder) os.makedirs(destination_src_folder) except Exception as e: raise ConanException("Unable to create folder %s\n%s" % (destination_src_folder, str(e))) return previous_digest
def test_basic(self): """ simple registration of a new conans """ reg_path = self.client.cache.package_layout(self.ref).export() manif = FileTreeManifest.load(self.client.cache.package_layout(self.ref).export()) self.assertIn('%s: A new conanfile.py version was exported' % str(self.ref), self.client.out) self.assertIn('%s: Folder: %s' % (str(self.ref), reg_path), self.client.out) self.assertTrue(os.path.exists(reg_path)) for name in list(self.files.keys()): self.assertTrue(os.path.exists(os.path.join(reg_path, name))) expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'main.cpp': '0479f3c223c9a656a718f3148e044124', 'CMakeLists.txt': '10d907c160c360b28f6991397a5aa9b4', 'conanfile.py': '355949fbf0b4fc32b8f1c5a338dfe1ae', 'executable': '68b329da9893e34099c7d8ad5cb9c940', 'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'} self.assertEqual(expected_sums, manif.file_sums)
def test_path(self): base_folder = temp_folder() source_folder = os.path.join(base_folder, "source") current_folder = os.path.join(base_folder, "current") client = TestClient(current_folder=current_folder) files = cpp_hello_conan_files("Hello0", "0.1") conan_ref = ConanFileReference("Hello0", "0.1", "lasote", "stable") conanfile = files.pop("conanfile.py") client.save(files, path=source_folder) conanfile = conanfile.replace("exports = '*'", 'exports = "../source*"') client.save({"conanfile.py": conanfile}) client.run("export . lasote/stable") reg_path = client.client_cache.export(conan_ref) manif = FileTreeManifest.load(client.client_cache.export(conan_ref)) self.assertIn( '%s: A new conanfile.py version was exported' % str(conan_ref), client.user_io.out) self.assertIn('%s: Folder: %s' % (str(conan_ref), reg_path), client.user_io.out) self.assertTrue(os.path.exists(reg_path)) for name in [ 'conanfile.py', 'conanmanifest.txt', 'source/main.cpp', 'source/executable', 'source/hello.cpp', 'source/CMakeLists.txt', 'source/helloHello0.h' ]: self.assertTrue(os.path.exists(os.path.join(reg_path, name))) expected_sums = { 'source/hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'source/main.cpp': '0479f3c223c9a656a718f3148e044124', 'source/CMakeLists.txt': '10d907c160c360b28f6991397a5aa9b4', 'conanfile.py': '3ac566eb5b2e4df4417003f0e606e237', 'source/executable': '68b329da9893e34099c7d8ad5cb9c940', 'source/helloHello0.h': '9448df034392fc8781a47dd03ae71bdd' } self.assertEqual(expected_sums, manif.file_sums)
def test_export_the_same_code(self): file_list = self._create_packages_and_builds() # Export the same conans # Do not adjust cpu_count, it is reusing a cache conan2 = TestClient(self.conan.base_folder, cpu_count=False) files2 = cpp_hello_conan_files("Hello0", "0.1") conan2.save(files2) conan2.run("export . lasote/stable") reg_path2 = conan2.client_cache.export(self.conan_ref) digest2 = FileTreeManifest.load( conan2.client_cache.export(self.conan_ref)) self.assertNotIn('A new Conan version was exported', conan2.user_io.out) self.assertNotIn('Cleaning the old builds ...', conan2.user_io.out) self.assertNotIn('Cleaning the old packs ...', conan2.user_io.out) self.assertNotIn('All the previous packs were cleaned', conan2.user_io.out) self.assertIn( '%s: A new conanfile.py version was exported' % str(self.conan_ref), self.conan.user_io.out) self.assertIn('%s: Folder: %s' % (str(self.conan_ref), reg_path2), self.conan.user_io.out) self.assertTrue(os.path.exists(reg_path2)) for name in list(files2.keys()): self.assertTrue(os.path.exists(os.path.join(reg_path2, name))) expected_sums = { 'hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'main.cpp': '0479f3c223c9a656a718f3148e044124', 'CMakeLists.txt': '10d907c160c360b28f6991397a5aa9b4', 'conanfile.py': '355949fbf0b4fc32b8f1c5a338dfe1ae', 'executable': '68b329da9893e34099c7d8ad5cb9c940', 'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd' } self.assertEqual(expected_sums, digest2.file_sums) for f in file_list: self.assertTrue(os.path.exists(f))
def test_path(self): base_folder = temp_folder() source_folder = os.path.join(base_folder, "source") current_folder = os.path.join(base_folder, "current") client = TestClient(current_folder=current_folder) files = cpp_hello_conan_files("Hello0", "0.1") ref = ConanFileReference("Hello0", "0.1", "lasote", "stable") conanfile = files.pop("conanfile.py") client.save(files, path=source_folder) conanfile = conanfile.replace("exports = '*'", 'exports = "../source*"') client.save({"conanfile.py": conanfile}) client.run("export . lasote/stable") reg_path = client.cache.package_layout(ref).export() manif = FileTreeManifest.load( client.cache.package_layout(ref).export()) self.assertIn('%s: A new conanfile.py version was exported' % str(ref), client.out) self.assertIn('%s: Folder: %s' % (str(ref), reg_path), client.out) self.assertTrue(os.path.exists(reg_path)) for name in [ 'conanfile.py', 'conanmanifest.txt', 'source/main.cpp', 'source/executable', 'source/hello.cpp', 'source/CMakeLists.txt', 'source/helloHello0.h' ]: self.assertTrue(os.path.exists(os.path.join(reg_path, name))) expected_sums = { 'source/hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'source/main.cpp': '0479f3c223c9a656a718f3148e044124', 'source/CMakeLists.txt': '10d907c160c360b28f6991397a5aa9b4', 'conanfile.py': '073cdeb3d07fc62180ba510ad7b4794d', 'source/executable': '68b329da9893e34099c7d8ad5cb9c940', 'source/helloHello0.h': 'd0a6868b5df17a6ae6e61ebddb0c9eb3' } self.assertEqual(expected_sums, manif.file_sums)
def test_basic(self): reg_path = self.client.cache.package_layout(self.ref).export() manif = FileTreeManifest.load( self.client.cache.package_layout(self.ref).export()) self.assertIn( '%s: A new conanfile.py version was exported' % str(self.ref), self.client.out) self.assertIn('%s: Folder: %s' % (str(self.ref), reg_path), self.client.out) self.assertTrue(os.path.exists(reg_path)) for name in list(self.files.keys()): self.assertTrue(os.path.exists(os.path.join(reg_path, name))) expected_sums = { 'CMakeLists.txt': '3cf710785270c7e98a30d4a90ea66492', 'conanfile.py': '73eb512b9f02ac18278823a217cfff79', 'executable': 'db299d5f0d82f113fad627a21f175e59', 'main.cpp': 'd9c03c934a4b3b1670775c17c26f39e9' } self.assertEqual(expected_sums, manif.file_sums)
def test_export_the_same_code(self): file_list = self._create_packages_and_builds() # Export the same conans # Do not adjust cpu_count, it is reusing a cache client2 = TestClient(self.client.cache_folder, cpu_count=False) files2 = cpp_hello_conan_files("Hello0", "0.1") client2.save(files2) client2.run("export . lasote/stable") reg_path2 = client2.cache.package_layout(self.ref).export() digest2 = FileTreeManifest.load( client2.cache.package_layout(self.ref).export()) self.assertNotIn('A new Conan version was exported', client2.out) self.assertNotIn('Cleaning the old builds ...', client2.out) self.assertNotIn('Cleaning the old packs ...', client2.out) self.assertNotIn('All the previous packs were cleaned', client2.out) self.assertIn( '%s: A new conanfile.py version was exported' % str(self.ref), self.client.out) self.assertIn('%s: Folder: %s' % (str(self.ref), reg_path2), self.client.out) self.assertTrue(os.path.exists(reg_path2)) for name in list(files2.keys()): self.assertTrue(os.path.exists(os.path.join(reg_path2, name))) expected_sums = { 'hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'main.cpp': '0479f3c223c9a656a718f3148e044124', 'CMakeLists.txt': '10d907c160c360b28f6991397a5aa9b4', 'conanfile.py': '1ad1f4b995ae7ffdb00d53ff49e1366f', 'executable': '68b329da9893e34099c7d8ad5cb9c940', 'helloHello0.h': 'd0a6868b5df17a6ae6e61ebddb0c9eb3' } self.assertEqual(expected_sums, digest2.file_sums) for f in file_list: self.assertTrue(os.path.exists(f))
def test_path(relative_path): client = TestClient() if relative_path: client.save({"conan/conanfile.py": GenConanfile().with_exports("../source*"), "source/main.cpp": "mymain"}) with client.chdir("current"): client.run("export ../conan hello/0.1@lasote/stable") else: client.save({"current/conanfile.py": GenConanfile().with_exports("../source*"), "source/main.cpp": "mymain"}) with client.chdir("current"): client.run("export . hello/0.1@lasote/stable") ref = ConanFileReference("hello", "0.1", "lasote", "stable") reg_path = client.cache.package_layout(ref).export() manif = FileTreeManifest.load(reg_path) for name in ['conanfile.py', 'conanmanifest.txt', 'source/main.cpp']: assert os.path.exists(os.path.join(reg_path, name)) expected_sums = {'conanfile.py': '718d94ef217b17297e10d60f1132ccf5', 'source/main.cpp': '76c0a7a9d385266e27d69d3875f6ac19'} assert expected_sums == manif.file_sums
def update_test(self, mode): self._create_code(mode) self.client.run("export . lasote/testing") self.client.run("install Hello/0.1@lasote/testing --build=missing") self.client.run("upload Hello/0.1@lasote/testing --all") self.client.run('remove Hello/0.1@lasote/testing -f') self.client.run("install Hello/0.1@lasote/testing") # upload to remote again, the folder remains as installed self.client.run("install Hello/0.1@lasote/testing --update") self.assertIn("Hello/0.1@lasote/testing: Already installed!", self.client.user_io.out) self._check_export_installed_folder(mode) server_path = self.server.paths.export(self.reference) save(os.path.join(server_path, "license.txt"), "mylicense") manifest = FileTreeManifest.load(server_path) manifest.time += 1 manifest.file_sums["license.txt"] = md5sum(os.path.join(server_path, "license.txt")) manifest.save(server_path) self.client.run("install Hello/0.1@lasote/testing --update") self._check_export_installed_folder(mode, updated=True)
def update_test(self, mode): self._create_code(mode) self.client.run("export . lasote/testing") self.client.run("install Hello/0.1@lasote/testing --build=missing") self.client.run("upload Hello/0.1@lasote/testing --all") self.client.run('remove Hello/0.1@lasote/testing -f') self.client.run("install Hello/0.1@lasote/testing") # upload to remote again, the folder remains as installed self.client.run("install Hello/0.1@lasote/testing --update") self.assertIn("Hello/0.1@lasote/testing: Already installed!", self.client.user_io.out) self._check_export_installed_folder(mode) server_path = self.server.paths.export(self.reference) save(os.path.join(server_path, "license.txt"), "mylicense") manifest = FileTreeManifest.load(server_path) manifest.time += 1 manifest.file_sums["license.txt"] = md5sum(os.path.join(server_path, "license.txt")) manifest.save(server_path) self.client.run("install Hello/0.1@lasote/testing --update") self._check_export_installed_folder(mode, updated=True)
def test_basic(self): reg_path = self.client.cache.package_layout(self.ref).export() manif = FileTreeManifest.load( self.client.cache.package_layout(self.ref).export()) self.assertIn( '%s: A new conanfile.py version was exported' % str(self.ref), self.client.out) self.assertIn('%s: Folder: %s' % (str(self.ref), reg_path), self.client.out) self.assertTrue(os.path.exists(reg_path)) for name in list(self.files.keys()): self.assertTrue(os.path.exists(os.path.join(reg_path, name))) expected_sums = { 'hello.cpp': '4f005274b2fdb25e6113b69774dac184', 'main.cpp': '0479f3c223c9a656a718f3148e044124', 'CMakeLists.txt': '10d907c160c360b28f6991397a5aa9b4', 'conanfile.py': '1ad1f4b995ae7ffdb00d53ff49e1366f', 'executable': '68b329da9893e34099c7d8ad5cb9c940', 'helloHello0.h': 'd0a6868b5df17a6ae6e61ebddb0c9eb3' } self.assertEqual(expected_sums, manif.file_sums)
def bump_time(inc_time): path = client.client_cache.export( ConanFileReference.loads("Pkg/0.1@lasote/testing")) manifest = FileTreeManifest.load(path) manifest.time += inc_time manifest.save(path)
def cmd_export(app, conanfile_path, name, version, user, channel, keep_source, export=True, graph_lock=None): """ Export the recipe param conanfile_path: the original source directory of the user containing a conanfile.py """ loader, cache, hook_manager, output = app.loader, app.cache, app.hook_manager, app.out revisions_enabled = app.config.revisions_enabled conanfile = loader.load_export(conanfile_path, name, version, user, channel) # FIXME: Conan 2.0, deprecate CONAN_USER AND CONAN_CHANNEL and remove this try excepts # Take the default from the env vars if they exist to not break behavior try: user = conanfile.user except ConanException: user = None try: channel = conanfile.channel except ConanException: channel = None ref = ConanFileReference(conanfile.name, conanfile.version, user, channel) # If we receive lock information, python_requires could have been locked if graph_lock: node_id = graph_lock.get_node(ref) python_requires = graph_lock.python_requires(node_id) # TODO: check that the locked python_requires are different from the loaded ones # FIXME: private access, will be improved when api collaborators are improved loader._python_requires._range_resolver.output # invalidate previous version range output conanfile = loader.load_export(conanfile_path, conanfile.name, conanfile.version, conanfile.user, conanfile.channel, python_requires) check_casing_conflict(cache=cache, ref=ref) package_layout = cache.package_layout(ref, short_paths=conanfile.short_paths) if not export: metadata = package_layout.load_metadata() recipe_revision = metadata.recipe.revision ref = ref.copy_with_rev(recipe_revision) if graph_lock: graph_lock.update_exported_ref(node_id, ref) return ref hook_manager.execute("pre_export", conanfile=conanfile, conanfile_path=conanfile_path, reference=package_layout.ref) logger.debug("EXPORT: %s" % conanfile_path) output.highlight("Exporting package recipe") conan_linter(conanfile_path, output) output = conanfile.output # Get previous digest try: previous_manifest = FileTreeManifest.load(package_layout.export()) except IOError: previous_manifest = None finally: _recreate_folders(package_layout.export(), package_layout.export_sources()) # Copy sources to target folders with package_layout.conanfile_write_lock(output=output): origin_folder = os.path.dirname(conanfile_path) export_recipe(conanfile, origin_folder, package_layout.export()) export_source(conanfile, origin_folder, package_layout.export_sources()) shutil.copy2(conanfile_path, package_layout.conanfile()) _capture_export_scm_data(conanfile, os.path.dirname(conanfile_path), package_layout.export(), output, scm_src_file=package_layout.scm_folder()) # Execute post-export hook before computing the digest hook_manager.execute("post_export", conanfile=conanfile, reference=package_layout.ref, conanfile_path=package_layout.conanfile()) # Compute the new digest manifest = FileTreeManifest.create(package_layout.export(), package_layout.export_sources()) modified_recipe = not previous_manifest or previous_manifest != manifest if modified_recipe: output.success('A new %s version was exported' % CONANFILE) output.info('Folder: %s' % package_layout.export()) else: output.info("The stored package has not changed") manifest = previous_manifest # Use the old one, keep old timestamp manifest.save(package_layout.export()) # Compute the revision for the recipe revision = _update_revision_in_metadata( package_layout=package_layout, revisions_enabled=revisions_enabled, output=output, path=os.path.dirname(conanfile_path), manifest=manifest, revision_mode=conanfile.revision_mode) # FIXME: Conan 2.0 Clear the registry entry if the recipe has changed source_folder = package_layout.source() if os.path.exists(source_folder): try: if is_dirty(source_folder): output.info("Source folder is corrupted, forcing removal") rmdir(source_folder) elif modified_recipe and not keep_source: output.info( "Package recipe modified in export, forcing source folder removal" ) output.info("Use the --keep-source, -k option to skip it") rmdir(source_folder) except BaseException as e: output.error( "Unable to delete source folder. Will be marked as corrupted for deletion" ) output.warn(str(e)) set_dirty(source_folder) # When revisions enabled, remove the packages not matching the revision if revisions_enabled: packages = search_packages(package_layout, query=None) metadata = package_layout.load_metadata() recipe_revision = metadata.recipe.revision to_remove = [ pid for pid in packages if metadata.packages.get(pid) and metadata.packages.get(pid).recipe_revision != recipe_revision ] if to_remove: output.info( "Removing the local binary packages from different recipe revisions" ) remover = DiskRemover() remover.remove_packages(package_layout, ids_filter=to_remove) ref = ref.copy_with_rev(revision) output.info("Exported revision: %s" % revision) if graph_lock: graph_lock.update_exported_ref(node_id, ref) return ref
def package_summary_hash(self, package_ref): package_folder = self.package(package_ref, short_paths=None) readed_digest = FileTreeManifest.load(package_folder) return readed_digest.summary_hash
def _get_recipe(self, ref, check_updates, update, remotes, recorder): output = ScopedOutput(str(ref), self._out) # check if it is in disk conanfile_path = self._cache.package_layout(ref).conanfile() # NOT in disk, must be retrieved from remotes if not os.path.exists(conanfile_path): remote, new_ref = self._download_recipe(ref, output, remotes, remotes.selected, recorder) status = RECIPE_DOWNLOADED return conanfile_path, status, remote, new_ref metadata = self._cache.package_layout(ref).load_metadata() cur_revision = metadata.recipe.revision cur_remote = metadata.recipe.remote cur_remote = remotes[cur_remote] if cur_remote else None selected_remote = remotes.selected or cur_remote check_updates = check_updates or update # Recipe exists in disk, but no need to check updates requested_different_revision = ( ref.revision is not None) and cur_revision != ref.revision if requested_different_revision and not check_updates: raise NotFoundException( "The recipe in the local cache doesn't match the specified " "revision. Use '--update' to check in the remote.") if not requested_different_revision: if not check_updates: status = RECIPE_INCACHE ref = ref.copy_with_rev(cur_revision) return conanfile_path, status, cur_remote, ref if not selected_remote: status = RECIPE_NO_REMOTE ref = ref.copy_with_rev(cur_revision) return conanfile_path, status, None, ref else: # Requested different revision and with --update remote, new_ref = self._download_recipe(ref, output, remotes, selected_remote, recorder) status = RECIPE_DOWNLOADED return conanfile_path, status, remote, new_ref try: # get_recipe_manifest can fail, not in server upstream_manifest, ref = self._remote_manager.get_recipe_manifest( ref, selected_remote) except NotFoundException: status = RECIPE_NOT_IN_REMOTE ref = ref.copy_with_rev(cur_revision) return conanfile_path, status, selected_remote, ref export = self._cache.package_layout(ref).export() read_manifest = FileTreeManifest.load(export) if upstream_manifest != read_manifest: if upstream_manifest.time > read_manifest.time: if update: DiskRemover().remove_recipe( self._cache.package_layout(ref), output=output) output.info("Retrieving from remote '%s'..." % selected_remote.name) self._download_recipe(ref, output, remotes, selected_remote, recorder) with self._cache.package_layout( ref).update_metadata() as metadata: metadata.recipe.remote = selected_remote.name status = RECIPE_UPDATED return conanfile_path, status, selected_remote, ref else: status = RECIPE_UPDATEABLE else: status = RECIPE_NEWER else: status = RECIPE_INCACHE ref = ref.copy_with_rev(cur_revision) return conanfile_path, status, selected_remote, ref
def inc_package_manifest_timestamp(client_cache, package_ref, inc_time): pkg_ref = PackageReference.loads(str(package_ref)) path = client_cache.package(pkg_ref) manifest = FileTreeManifest.load(path) manifest.time += inc_time manifest.save(path)
def inc_recipe_manifest_timestamp(client_cache, conan_ref, inc_time): conan_ref = ConanFileReference.loads(str(conan_ref)) path = client_cache.export(conan_ref) manifest = FileTreeManifest.load(path) manifest.time += inc_time manifest.save(path)
def export_pkg(cache, graph_manager, hook_manager, recorder, output, ref, source_folder, build_folder, package_folder, install_folder, graph_info, force): conan_file_path = cache.conanfile(ref) if not os.path.exists(conan_file_path): raise ConanException("Package recipe '%s' does not exist" % str(ref)) # The graph has to be loaded with build_mode=[ref.name], so that node is not tried # to be downloaded from remotes deps_graph, _ = graph_manager.load_graph(ref, None, graph_info=graph_info, build_mode=[ref.name], check_updates=False, update=False, remote_name=None, recorder=recorder, apply_build_requires=False) # this is a bit tricky, but works. The root (virtual), has only 1 neighbor, # which is the exported pkg nodes = deps_graph.root.neighbors() conanfile = nodes[0].conanfile from conans.client.conan_api import existing_info_files if install_folder and existing_info_files(install_folder): load_deps_info(install_folder, conanfile, required=True) package_id = nodes[0].package_id output.info("Packaging to %s" % package_id) pref = PackageReference(ref, package_id) dest_package_folder = cache.package(pref, short_paths=conanfile.short_paths) if os.path.exists(dest_package_folder): if force: rmdir(dest_package_folder) else: raise ConanException( "Package already exists. Please use --force, -f to " "overwrite it") recipe_hash = cache.package_layout(ref).recipe_manifest().summary_hash conanfile.info.recipe_hash = recipe_hash conanfile.develop = True if package_folder: packager.export_pkg(conanfile, package_id, package_folder, dest_package_folder, hook_manager, conan_file_path, ref) else: with get_env_context_manager(conanfile): packager.create_package(conanfile, package_id, source_folder, build_folder, dest_package_folder, install_folder, hook_manager, conan_file_path, ref, local=True) readed_manifest = FileTreeManifest.load(dest_package_folder) pref = PackageReference(pref.ref, pref.id, readed_manifest.summary_hash) output.info("Package revision %s" % pref.revision) with cache.package_layout(ref).update_metadata() as metadata: metadata.packages[package_id].revision = pref.revision metadata.packages[ package_id].recipe_revision = metadata.recipe.revision recorder.package_exported(pref)
def inc_recipe_manifest_timestamp(cache, reference, inc_time): ref = ConanFileReference.loads(reference) path = cache.package_layout(ref).export() manifest = FileTreeManifest.load(path) manifest.time += inc_time manifest.save(path)
def load_package_manifest(self, package_reference): """conan_id = sha(zip file)""" package_folder = self.package(package_reference, short_paths=None) return FileTreeManifest.load(package_folder)
def _get_recipe(self, reference, check_updates, update, remote_name, recorder): output = ScopedOutput(str(reference), self._out) # check if it is in disk conanfile_path = self._client_cache.conanfile(reference) # NOT in disk, must be retrieved from remotes if not os.path.exists(conanfile_path): remote, new_ref = self._download_recipe(reference, output, remote_name, recorder) status = RECIPE_DOWNLOADED return conanfile_path, status, remote, new_ref remote = self._registry.get_recipe_remote(reference) check_updates = check_updates or update # Recipe exists in disk, but no need to check updates if not check_updates: status = RECIPE_INCACHE log_recipe_got_from_local_cache(reference) recorder.recipe_fetched_from_cache(reference) return conanfile_path, status, remote, reference named_remote = self._registry.remote( remote_name) if remote_name else None update_remote = named_remote or remote if not update_remote: status = RECIPE_NO_REMOTE log_recipe_got_from_local_cache(reference) recorder.recipe_fetched_from_cache(reference) return conanfile_path, status, None, reference try: # get_conan_manifest can fail, not in server upstream_manifest = self._remote_manager.get_conan_manifest( reference, update_remote) except NotFoundException: status = RECIPE_NOT_IN_REMOTE log_recipe_got_from_local_cache(reference) recorder.recipe_fetched_from_cache(reference) return conanfile_path, status, update_remote, reference export = self._client_cache.export(reference) read_manifest = FileTreeManifest.load(export) if upstream_manifest != read_manifest: if upstream_manifest.time > read_manifest.time: if update: DiskRemover(self._client_cache).remove_recipe(reference) output.info("Retrieving from remote '%s'..." % update_remote.name) reference = self._remote_manager.get_recipe( reference, update_remote) self._registry.set_ref(reference, update_remote.name) status = RECIPE_UPDATED else: status = RECIPE_UPDATEABLE else: status = RECIPE_NEWER else: status = RECIPE_INCACHE log_recipe_got_from_local_cache(reference) recorder.recipe_fetched_from_cache(reference) return conanfile_path, status, update_remote, reference
def cmd_export(conanfile_path, name, version, user, channel, keep_source, revisions_enabled, output, hook_manager, loader, cache, export=True): """ Export the recipe param conanfile_path: the original source directory of the user containing a conanfile.py """ conanfile = loader.load_export(conanfile_path, name, version, user, channel) ref = ConanFileReference(conanfile.name, conanfile.version, conanfile.user, conanfile.channel) check_casing_conflict(cache=cache, ref=ref) package_layout = cache.package_layout(ref, short_paths=conanfile.short_paths) if not export: metadata = package_layout.load_metadata() recipe_revision = metadata.recipe.revision ref = ref.copy_with_rev(recipe_revision) return ref hook_manager.execute("pre_export", conanfile=conanfile, conanfile_path=conanfile_path, reference=package_layout.ref) logger.debug("EXPORT: %s" % conanfile_path) output.highlight("Exporting package recipe") conan_linter(conanfile_path, output) output = conanfile.output # Get previous digest try: previous_digest = FileTreeManifest.load(package_layout.export()) except IOError: previous_digest = None finally: _recreate_folders(package_layout.export(), package_layout.export_sources()) # Copy sources to target folders with package_layout.conanfile_write_lock(output=output): origin_folder = os.path.dirname(conanfile_path) export_recipe(conanfile, origin_folder, package_layout.export()) export_source(conanfile, origin_folder, package_layout.export_sources()) shutil.copy2(conanfile_path, package_layout.conanfile()) _capture_export_scm_data(conanfile, os.path.dirname(conanfile_path), package_layout.export(), output, scm_src_file=package_layout.scm_folder()) # Execute post-export hook before computing the digest hook_manager.execute("post_export", conanfile=conanfile, reference=package_layout.ref, conanfile_path=package_layout.conanfile()) # Compute the new digest digest = FileTreeManifest.create(package_layout.export(), package_layout.export_sources()) modified_recipe = not previous_digest or previous_digest != digest if modified_recipe: output.success('A new %s version was exported' % CONANFILE) output.info('Folder: %s' % package_layout.export()) else: output.info("The stored package has not changed") digest = previous_digest # Use the old one, keep old timestamp digest.save(package_layout.export()) # Compute the revision for the recipe revision = _update_revision_in_metadata( package_layout=package_layout, revisions_enabled=revisions_enabled, output=output, path=os.path.dirname(conanfile_path), digest=digest, revision_mode=conanfile.revision_mode) # FIXME: Conan 2.0 Clear the registry entry if the recipe has changed source_folder = package_layout.source() if os.path.exists(source_folder): try: if is_dirty(source_folder): output.info("Source folder is corrupted, forcing removal") rmdir(source_folder) elif modified_recipe and not keep_source: output.info( "Package recipe modified in export, forcing source folder removal" ) output.info("Use the --keep-source, -k option to skip it") rmdir(source_folder) except BaseException as e: output.error( "Unable to delete source folder. Will be marked as corrupted for deletion" ) output.warn(str(e)) set_dirty(source_folder) # When revisions enabled, remove the packages not matching the revision if revisions_enabled: packages = search_packages(package_layout, query=None) metadata = package_layout.load_metadata() recipe_revision = metadata.recipe.revision to_remove = [ pid for pid in packages if metadata.packages.get(pid) and metadata.packages.get(pid).recipe_revision != recipe_revision ] if to_remove: output.info( "Removing the local binary packages from different recipe revisions" ) remover = DiskRemover() remover.remove_packages(package_layout, ids_filter=to_remove) ref = ref.copy_with_rev(revision) return ref
def _get_recipe(self, reference, check_updates, update, remote_name, recorder): output = ScopedOutput(str(reference), self._out) # check if it is in disk conanfile_path = self._client_cache.conanfile(reference) # NOT in disk, must be retrieved from remotes if not os.path.exists(conanfile_path): remote, new_ref = self._download_recipe(reference, output, remote_name, recorder) status = RECIPE_DOWNLOADED return conanfile_path, status, remote, new_ref metadata = self._client_cache.load_metadata(reference) cur_revision = metadata.recipe.revision remote = self._registry.refs.get(reference) named_remote = self._registry.remotes.get( remote_name) if remote_name else None update_remote = named_remote or remote # Check if we have a revision different from the requested one revisions_enabled = get_env("CONAN_CLIENT_REVISIONS_ENABLED", False) if revisions_enabled and reference.revision and cur_revision != reference.revision: output.info( "Different revision requested, removing current local recipe..." ) DiskRemover(self._client_cache).remove_recipe(reference) output.info("Retrieving from remote '%s'..." % update_remote.name) new_ref = self._remote_manager.get_recipe(reference, update_remote) self._registry.refs.set(new_ref, update_remote.name) status = RECIPE_UPDATED return conanfile_path, status, update_remote, new_ref check_updates = check_updates or update # Recipe exists in disk, but no need to check updates if not check_updates: status = RECIPE_INCACHE ref = reference.copy_with_rev(cur_revision) return conanfile_path, status, remote, ref if not update_remote: status = RECIPE_NO_REMOTE ref = reference.copy_with_rev(cur_revision) return conanfile_path, status, None, ref try: # get_conan_manifest can fail, not in server upstream_manifest = self._remote_manager.get_conan_manifest( reference, update_remote) except NotFoundException: status = RECIPE_NOT_IN_REMOTE ref = reference.copy_with_rev(cur_revision) return conanfile_path, status, update_remote, ref export = self._client_cache.export(reference) read_manifest = FileTreeManifest.load(export) if upstream_manifest != read_manifest: if upstream_manifest.time > read_manifest.time: if update: DiskRemover(self._client_cache).remove_recipe(reference) output.info("Retrieving from remote '%s'..." % update_remote.name) new_ref = self._remote_manager.get_recipe( reference, update_remote) self._registry.refs.set(new_ref, update_remote.name) status = RECIPE_UPDATED return conanfile_path, status, update_remote, new_ref else: status = RECIPE_UPDATEABLE else: status = RECIPE_NEWER else: status = RECIPE_INCACHE ref = reference.copy_with_rev(cur_revision) return conanfile_path, status, update_remote, ref
def load_manifest(self, conan_reference): """conan_id = sha(zip file)""" assert isinstance(conan_reference, ConanFileReference) export_folder = self.export(conan_reference) check_ref_case(conan_reference, export_folder, self.store) return FileTreeManifest.load(export_folder)
def recipe_manifest(self): return FileTreeManifest.load(self.export())
def _digests(folder, exports_sources_folder=None): readed_digest = FileTreeManifest.load(folder) expected_digest = FileTreeManifest.create(folder, exports_sources_folder) return readed_digest, expected_digest
def package_manifests(self, pref): package_folder = self.package(pref) readed_manifest = FileTreeManifest.load(package_folder) expected_manifest = FileTreeManifest.create(package_folder) return readed_manifest, expected_manifest
def inc_package_manifest_timestamp(cache, package_reference, inc_time): pref = PackageReference.loads(package_reference) path = cache.package_layout(pref.ref).package(pref) manifest = FileTreeManifest.load(path) manifest.time += inc_time manifest.save(path)
def mocked_post_export(*args, **kwargs): # There shouldn't be a digest yet with self.assertRaisesRegexp(IOError, "No such file or directory"): FileTreeManifest.load(package_layout.export()) self.assertFalse(os.path.exists(os.path.join(package_layout.export(), CONAN_MANIFEST)))