def _handle_node_cache(self, node, keep_build, processed_package_references): pref = node.pref conan_file = node.conanfile output = conan_file.output package_folder = self._cache.package(pref, conan_file.short_paths) with self._cache.package_lock(pref): if pref not in processed_package_references: processed_package_references.add(pref) if node.binary == BINARY_BUILD: set_dirty(package_folder) self._build_package(node, pref, output, keep_build) clean_dirty(package_folder) elif node.binary in (BINARY_UPDATE, BINARY_DOWNLOAD): if not self._node_concurrently_installed(node, package_folder): set_dirty(package_folder) assert pref.revision is not None, "Installer should receive #PREV always" self._remote_manager.get_package(pref, package_folder, node.binary_remote, output, self._recorder) self._registry.prefs.set(pref, node.binary_remote.name) clean_dirty(package_folder) else: output.success('Download skipped. Probable concurrent download') log_package_got_from_local_cache(pref) self._recorder.package_fetched_from_cache(pref) elif node.binary == BINARY_CACHE: output.success('Already installed!') log_package_got_from_local_cache(pref) self._recorder.package_fetched_from_cache(pref) # Call the info method self._call_package_info(conan_file, package_folder) self._recorder.package_cpp_info(pref, conan_file.cpp_info)
def compress_files(files, symlinks, name, dest_dir): t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) set_dirty(tgz_path) with open(tgz_path, "wb") as tgz_handle: # tgz_contents = BytesIO() tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle) for filename, dest in sorted(symlinks.items()): info = tarfile.TarInfo(name=filename) info.type = tarfile.SYMTYPE info.linkname = dest tgz.addfile(tarinfo=info) mask = ~(stat.S_IWOTH | stat.S_IWGRP) for filename, abs_path in sorted(files.items()): info = tarfile.TarInfo(name=filename) info.size = os.stat(abs_path).st_size info.mode = os.stat(abs_path).st_mode & mask if os.path.islink(abs_path): info.type = tarfile.SYMTYPE info.linkname = os.readlink(abs_path) # @UndefinedVariable tgz.addfile(tarinfo=info) else: with open(abs_path, 'rb') as file_handler: tgz.addfile(tarinfo=info, fileobj=file_handler) tgz.close() clean_dirty(tgz_path) duration = time.time() - t1 log_compressed_files(files, duration, tgz_path) return tgz_path
def _build(self, nodes_to_process, deps_graph, skip_nodes, profile_build_requires, keep_build, root_node, update): """ The build assumes an input of conans ordered by degree, first level should be independent from each other, the next-second level should have dependencies only to first level conans. param nodes_by_level: list of lists [[nodeA, nodeB], [nodeC], [nodeD, ...], ...] """ inverse_levels = deps_graph.inverse_levels() for level in inverse_levels: level[:] = [n for n in level if n not in skip_nodes] for node, package_id, build_needed in nodes_to_process: conan_ref, conan_file = node.conan_ref, node.conanfile output = ScopedOutput(str(conan_ref), self._out) package_ref = PackageReference(conan_ref, package_id) package_folder = self._client_cache.package(package_ref, conan_file.short_paths) with self._client_cache.package_lock(package_ref): set_dirty(package_folder) if build_needed and (conan_ref, package_id) not in self._built_packages: self._build_package(node, package_id, package_ref, output, keep_build, profile_build_requires, inverse_levels, deps_graph, update) else: self._get_existing_package(conan_file, package_ref, output, package_folder, update) self._propagate_info(node, inverse_levels, deps_graph) # Call the info method self._call_package_info(conan_file, package_folder) clean_dirty(package_folder) # Finally, propagate information to root node (conan_ref=None) self._propagate_info(root_node, inverse_levels, deps_graph)
def upload_dirty_test(self): test_server = TestServer([], users={"lasote": "mypass"}) client = TestClient(servers={"default": test_server}, users={"default": [("lasote", "mypass")]}) client.save({"conanfile.py": str(TestConanFile())}) client.run("export . lasote/testing") ref = ConanFileReference.loads("Hello/0.1@lasote/testing") pkg_ref = PackageReference(ref, "12345") package_folder = client.client_cache.package(pkg_ref) recipe_rev = client.get_revision(ref) p_rev = client.get_package_revision(pkg_ref) with client.client_cache.update_metadata(pkg_ref.conan) as metadata: metadata.packages[pkg_ref.package_id].revision = p_rev metadata.packages[pkg_ref.package_id].recipe_revision = recipe_rev save(os.path.join(package_folder, "conanmanifest.txt"), "888") set_dirty(package_folder) client.run("upload * --all --confirm", assert_error=True) self.assertIn( "ERROR: Package Hello/0.1@lasote/testing:12345 is corrupted, aborting upload", client.out) self.assertIn( "Remove it with 'conan remove Hello/0.1@lasote/testing -p=12345'", client.out) client.run("remove Hello/0.1@lasote/testing -p=12345 -f") client.run("upload * --all --confirm")
def _handle_node_cache(self, node, package_ref, keep_build, processed_package_references): conan_ref, conan_file = node.conan_ref, node.conanfile output = ScopedOutput(str(conan_ref), self._out) package_folder = self._client_cache.package(package_ref, conan_file.short_paths) with self._client_cache.package_lock(package_ref): if package_ref not in processed_package_references: processed_package_references.add(package_ref) set_dirty(package_folder) if node.binary == BINARY_BUILD: self._build_package(node, package_ref, output, keep_build) elif node.binary in (BINARY_UPDATE, BINARY_DOWNLOAD): self._remote_manager.get_package(package_ref, package_folder, node.binary_remote, output, self._recorder) if node.binary_remote != node.remote: self._registry.set_ref(conan_ref, node.binary_remote.name) elif node.binary == BINARY_CACHE: output.success('Already installed!') log_package_got_from_local_cache(package_ref) self._recorder.package_fetched_from_cache(package_ref) clean_dirty(package_folder) # Call the info method self._call_package_info(conan_file, package_folder)
def build_package(self, node, keep_build, recorder, remotes): t1 = time.time() conanfile = node.conanfile pref = node.pref package_layout = self._cache.package_layout(pref.ref, conanfile.short_paths) source_folder = package_layout.source() conanfile_path = package_layout.conanfile() package_folder = package_layout.package(pref) build_folder, skip_build = self._get_build_folder( conanfile, package_layout, pref, keep_build, recorder) # PREPARE SOURCES if not skip_build: with package_layout.conanfile_write_lock(self._output): set_dirty(build_folder) self._prepare_sources(conanfile, pref, package_layout, conanfile_path, source_folder, build_folder, remotes) # BUILD & PACKAGE with package_layout.conanfile_read_lock(self._output): _remove_folder_raising(package_folder) mkdir(build_folder) with tools.chdir(build_folder): self._output.info('Building your package in %s' % build_folder) try: if getattr(conanfile, 'no_copy_source', False): conanfile.source_folder = source_folder else: conanfile.source_folder = build_folder if not skip_build: with get_env_context_manager(conanfile): conanfile.build_folder = build_folder conanfile.package_folder = package_folder # In local cache, install folder always is build_folder conanfile.install_folder = build_folder self._build(conanfile, pref, build_folder) clean_dirty(build_folder) prev = self._package(conanfile, pref, package_layout, conanfile_path, build_folder, package_folder) assert prev node.prev = prev log_file = os.path.join(build_folder, RUN_LOG_NAME) log_file = log_file if os.path.exists(log_file) else None log_package_built(pref, time.time() - t1, log_file) recorder.package_built(pref) except ConanException as exc: recorder.package_install_error(pref, INSTALL_ERROR_BUILDING, str(exc), remote_name=None) raise exc return node.pref
def upload_dirty_test(self): test_server = TestServer([], users={"lasote": "mypass"}) client = TestClient(servers={"default": test_server}, users={"default": [("lasote", "mypass")]}) client.save({ "conanfile.py": GenConanfile().with_name("Hello").with_version("0.1") }) client.run("create . lasote/testing") ref = ConanFileReference.loads("Hello/0.1@lasote/testing") pref = PackageReference(ref, NO_SETTINGS_PACKAGE_ID) package_folder = client.cache.package_layout(pref.ref).package(pref) set_dirty(package_folder) client.run("upload * --all --confirm", assert_error=True) self.assertIn( "ERROR: Package %s is corrupted, aborting upload" % str(pref), client.out) self.assertIn( "Remove it with 'conan remove Hello/0.1@lasote/testing -p=%s'" % NO_SETTINGS_PACKAGE_ID, client.out) client.run("remove Hello/0.1@lasote/testing -p=%s -f" % NO_SETTINGS_PACKAGE_ID) client.run("upload * --all --confirm")
def test_clean_dirty(self): """ Dirty flag must be cleaned by clean_dirty """ set_dirty(self.temp_folder) self.assertTrue(os.path.exists(self.dirty_folder)) clean_dirty(self.temp_folder) self.assertFalse(os.path.exists(self.dirty_folder))
def _export_conanfile(conanfile_path, output, client_cache, conanfile, conan_ref, keep_source): exports_folder = client_cache.export(conan_ref) exports_source_folder = client_cache.export_sources( conan_ref, conanfile.short_paths) previous_digest = _init_export_folder(exports_folder, exports_source_folder) origin_folder = os.path.dirname(conanfile_path) export_recipe(conanfile, origin_folder, exports_folder, output) export_source(conanfile, origin_folder, exports_source_folder, output) shutil.copy2(conanfile_path, os.path.join(exports_folder, CONANFILE)) scm_data, captured_revision = _capture_export_scm_data( conanfile, os.path.dirname(conanfile_path), exports_folder, output, client_cache, conan_ref) digest = FileTreeManifest.create(exports_folder, exports_source_folder) if previous_digest and previous_digest == digest: output.info("The stored package has not changed") modified_recipe = False digest = previous_digest # Use the old one, keep old timestamp else: output.success('A new %s version was exported' % CONANFILE) output.info('Folder: %s' % exports_folder) modified_recipe = True digest.save(exports_folder) revision = scm_data.revision if scm_data and captured_revision else digest.summary_hash with client_cache.update_metadata(conan_ref) as metadata: # Note that there is no time set, the time will come from the remote metadata.recipe.revision = revision # FIXME: Conan 2.0 Clear the registry entry if the recipe has changed source = client_cache.source(conan_ref, conanfile.short_paths) remove = False if is_dirty(source): output.info("Source folder is corrupted, forcing removal") remove = True elif modified_recipe and not keep_source and os.path.exists(source): output.info( "Package recipe modified in export, forcing source folder removal") output.info("Use the --keep-source, -k option to skip it") remove = True if remove: output.info( "Removing 'source' folder, this can take a while for big packages") try: # remove only the internal rmdir(source) except BaseException as e: output.error("Unable to delete source folder. " "Will be marked as corrupted for deletion") output.warn(str(e)) set_dirty(source)
def compress_files(files, symlinks, name, dest_dir, output=None): t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) set_dirty(tgz_path) with open(tgz_path, "wb") as tgz_handle: # tgz_contents = BytesIO() tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle) for filename, dest in sorted(symlinks.items()): info = tarfile.TarInfo(name=filename) info.type = tarfile.SYMTYPE info.linkname = dest info.size = 0 # A symlink shouldn't have size tgz.addfile(tarinfo=info) mask = ~(stat.S_IWOTH | stat.S_IWGRP) i_file = 0 n_files = len(files) last_progress = None if output and n_files > 1 and not output.is_terminal: output.write("[") for filename, abs_path in sorted(files.items()): info = tarfile.TarInfo(name=filename) info.size = os.stat(abs_path).st_size info.mode = os.stat(abs_path).st_mode & mask if os.path.islink(abs_path): info.type = tarfile.SYMTYPE info.size = 0 # A symlink shouldn't have size info.linkname = os.readlink(abs_path) # @UndefinedVariable tgz.addfile(tarinfo=info) else: with open(abs_path, 'rb') as file_handler: tgz.addfile(tarinfo=info, fileobj=file_handler) if output and n_files > 1: i_file = i_file + 1 units = min(50, int(50 * i_file / n_files)) if last_progress != units: # Avoid screen refresh if nothing has change if output.is_terminal: text = "%s/%s files" % (i_file, n_files) output.rewrite_line("[%s%s] %s" % ('=' * units, ' ' * (50 - units), text)) else: output.write('=' * (units - (last_progress or 0))) last_progress = units if output and n_files > 1: if output.is_terminal: output.writeln("") else: output.writeln("]") tgz.close() clean_dirty(tgz_path) duration = time.time() - t1 log_compressed_files(files, duration, tgz_path) return tgz_path
def build_package(self, node, keep_build, recorder, remotes): t1 = time.time() conanfile = node.conanfile pref = node.pref package_layout = self._cache.package_layout(pref.ref, conanfile.short_paths) base_source = package_layout.source() conanfile_path = package_layout.conanfile() base_package = package_layout.package(pref) base_build, skip_build = self._get_build_folder( conanfile, package_layout, pref, keep_build, recorder) # PREPARE SOURCES if not skip_build: with package_layout.conanfile_write_lock(self._output): set_dirty(base_build) self._prepare_sources(conanfile, pref, package_layout, remotes) self._copy_sources(conanfile, base_source, base_build) # BUILD & PACKAGE with package_layout.conanfile_read_lock(self._output): self._output.info('Building your package in %s' % base_build) try: if getattr(conanfile, 'no_copy_source', False): conanfile.folders.set_base_source(base_source) else: conanfile.folders.set_base_source(base_build) conanfile.folders.set_base_build(base_build) conanfile.folders.set_base_imports(base_build) conanfile.folders.set_base_package(base_package) if not skip_build: # In local cache, generators folder always in build_folder conanfile.folders.set_base_generators(base_build) # In local cache, install folder always is build_folder conanfile.folders.set_base_install(base_build) self._build(conanfile, pref) clean_dirty(base_build) prev = self._package(conanfile, pref, package_layout, conanfile_path) assert prev node.prev = prev log_file = os.path.join(base_build, RUN_LOG_NAME) log_file = log_file if os.path.exists(log_file) else None log_package_built(pref, time.time() - t1, log_file) recorder.package_built(pref) except ConanException as exc: recorder.package_install_error(pref, INSTALL_ERROR_BUILDING, str(exc), remote_name=None) raise exc return node.pref
def config_source(export_folder, export_source_folder, src_folder, conanfile, output, conanfile_path, reference, hook_manager, cache): """ Implements the sources configuration when a package is going to be built in the local cache. """ def remove_source(raise_error=True): output.warn("This can take a while for big packages") try: rmdir(src_folder) except BaseException as e_rm: set_dirty(src_folder) msg = str(e_rm) if six.PY2: msg = str(e_rm).decode( "latin1") # Windows prints some chars in latin1 output.error("Unable to remove source folder %s\n%s" % (src_folder, msg)) output.warn("**** Please delete it manually ****") if raise_error or isinstance(e_rm, KeyboardInterrupt): raise ConanException("Unable to remove source folder") sources_pointer = cache.package_layout(reference).scm_folder() local_sources_path = load(sources_pointer) if os.path.exists( sources_pointer) else None if is_dirty(src_folder): output.warn("Trying to remove corrupted source folder") remove_source() elif conanfile.build_policy_always: output.warn( "Detected build_policy 'always', trying to remove source folder") remove_source() elif local_sources_path and os.path.exists(local_sources_path): output.warn( "Detected 'scm' auto in conanfile, trying to remove source folder") remove_source() if not os.path.exists(src_folder): # No source folder, need to get it set_dirty(src_folder) mkdir(src_folder) def get_sources_from_exports(): # so self exported files have precedence over python_requires ones merge_directories(export_folder, src_folder) # Now move the export-sources to the right location merge_directories(export_source_folder, src_folder) _run_source(conanfile, conanfile_path, src_folder, hook_manager, reference, cache, local_sources_path, get_sources_from_exports=get_sources_from_exports) clean_dirty(src_folder) # Everything went well, remove DIRTY flag
def _build_package(self, node, package_ref, output, keep_build): conan_ref, conan_file = node.conan_ref, node.conanfile t1 = time.time() # It is necessary to complete the sources of python requires, which might be used for python_require in conan_file.python_requires: complete_recipe_sources(self._remote_manager, self._client_cache, conan_file, python_require.conan_ref) builder = _ConanPackageBuilder(conan_file, package_ref, self._client_cache, output, self._hook_manager) if is_dirty(builder.build_folder): output.warn("Build folder is dirty, removing it: %s" % builder.build_folder) rmdir(builder.build_folder) skip_build = conan_file.develop and keep_build if skip_build: output.info("Won't be built as specified by --keep-build") if skip_build: if not os.path.exists(builder.build_folder): msg = "--keep-build specified, but build folder not found" self._recorder.package_install_error( package_ref, INSTALL_ERROR_MISSING_BUILD_FOLDER, msg, remote_name=None) raise ConanException(msg) else: with self._client_cache.conanfile_write_lock(conan_ref): set_dirty(builder.build_folder) complete_recipe_sources(self._remote_manager, self._client_cache, conan_file, conan_ref) builder.prepare_build() with self._client_cache.conanfile_read_lock(conan_ref): try: if not skip_build: builder.build() clean_dirty(builder.build_folder) builder.package() except ConanException as exc: self._recorder.package_install_error(package_ref, INSTALL_ERROR_BUILDING, str(exc), remote_name=None) raise exc else: # Log build self._log_built_package(builder.build_folder, package_ref.copy_clear_rev(), time.time() - t1)
def _handle_node_cache(self, node, keep_build, processed_package_references, remotes): pref = node.pref assert pref.id, "Package-ID without value" conanfile = node.conanfile output = conanfile.output package_folder = self._cache.package_layout( pref.ref, conanfile.short_paths).package(pref) with self._cache.package_layout(pref.ref).package_lock(pref): if pref not in processed_package_references: processed_package_references.add(pref) if node.binary == BINARY_BUILD: assert node.prev is None, "PREV for %s to be built should be None" % str( pref) set_dirty(package_folder) pref = self._build_package(node, pref, output, keep_build, remotes) clean_dirty(package_folder) assert node.prev is not None, "PREV for %s to be built is None" % str( pref) assert pref.revision is not None, "PREV for %s to be built is None" % str( pref) elif node.binary in (BINARY_UPDATE, BINARY_DOWNLOAD): assert node.prev, "PREV for %s is None" % str(pref) if not self._node_concurrently_installed( node, package_folder): set_dirty(package_folder) assert pref.revision is not None, "Installer should receive #PREV always" self._remote_manager.get_package( pref, package_folder, node.binary_remote, output, self._recorder) output.info("Downloaded package revision %s" % pref.revision) with self._cache.package_layout( pref.ref).update_metadata() as metadata: metadata.packages[ pref.id].remote = node.binary_remote.name clean_dirty(package_folder) else: output.success( 'Download skipped. Probable concurrent download') log_package_got_from_local_cache(pref) self._recorder.package_fetched_from_cache(pref) elif node.binary == BINARY_CACHE: assert node.prev, "PREV for %s is None" % str(pref) output.success('Already installed!') log_package_got_from_local_cache(pref) self._recorder.package_fetched_from_cache(pref) # Call the info method self._call_package_info(conanfile, package_folder, ref=pref.ref) self._recorder.package_cpp_info(pref, conanfile.cpp_info)
def remove_source(raise_error=True): output.warn("This can take a while for big packages") try: rmdir(src_folder) except BaseException as e_rm: set_dirty(src_folder) msg = str(e_rm) if six.PY2: msg = str(e_rm).decode("latin1") # Windows prints some chars in latin1 output.error("Unable to remove source folder %s\n%s" % (src_folder, msg)) output.warn("**** Please delete it manually ****") if raise_error or isinstance(e_rm, KeyboardInterrupt): raise ConanException("Unable to remove source folder")
def download(self, url, file_path=None, md5=None, sha1=None, sha256=None, **kwargs): """ compatible interface of FileDownloader + checksum """ checksum = sha256 or sha1 or md5 # If it is a user download, it must contain a checksum assert (not self._user_download) or (self._user_download and checksum) h = self._get_hash(url, checksum) with self._lock(h): cached_path = os.path.join(self._cache_folder, h) if is_dirty(cached_path): if os.path.exists(cached_path): os.remove(cached_path) clean_dirty(cached_path) if os.path.exists(cached_path): # If exists but it is corrupted, it is removed. Note that v2 downloads # do not have checksums, this only works for user downloads try: check_checksum(cached_path, md5, sha1, sha256) except ConanException: logger.error("Cached file corrupt, redownloading") remove(cached_path) if not os.path.exists(cached_path): set_dirty(cached_path) self._file_downloader.download(url=url, file_path=cached_path, md5=md5, sha1=sha1, sha256=sha256, **kwargs) clean_dirty(cached_path) if file_path is not None: file_path = os.path.abspath(file_path) mkdir(os.path.dirname(file_path)) shutil.copy2(cached_path, file_path) else: with open(cached_path, 'rb') as handle: tmp = handle.read() return tmp
def _export_conanfile(conanfile_path, output, paths, conanfile, conan_ref, keep_source): exports_folder = paths.export(conan_ref) exports_source_folder = paths.export_sources(conan_ref, conanfile.short_paths) previous_digest = _init_export_folder(exports_folder, exports_source_folder) _execute_export(conanfile_path, conanfile, exports_folder, exports_source_folder, output) shutil.copy2(conanfile_path, os.path.join(exports_folder, CONANFILE)) _capture_export_scm_data(conanfile, os.path.dirname(conanfile_path), exports_folder, output, paths, conan_ref) digest = FileTreeManifest.create(exports_folder, exports_source_folder) if previous_digest and previous_digest == digest: output.info("The stored package has not changed") modified_recipe = False digest = previous_digest # Use the old one, keep old timestamp else: output.success('A new %s version was exported' % CONANFILE) output.info('Folder: %s' % exports_folder) modified_recipe = True digest.save(exports_folder) source = paths.source(conan_ref, conanfile.short_paths) remove = False if is_dirty(source): output.info("Source folder is corrupted, forcing removal") remove = True elif modified_recipe and not keep_source and os.path.exists(source): output.info( "Package recipe modified in export, forcing source folder removal") output.info("Use the --keep-source, -k option to skip it") remove = True if remove: output.info( "Removing 'source' folder, this can take a while for big packages") try: # remove only the internal rmdir(source) except BaseException as e: output.error("Unable to delete source folder. " "Will be marked as corrupted for deletion") output.warn(str(e)) set_dirty(source)
def test_upload_dirty(self): client = TestClient(default_server_user=True) client.save({"conanfile.py": GenConanfile("Hello", "0.1")}) client.run("create . lasote/testing") ref = ConanFileReference.loads("Hello/0.1@lasote/testing") pref = PackageReference(ref, NO_SETTINGS_PACKAGE_ID) layout = client.cache.package_layout(pref.ref) pkg_folder = os.path.join(layout.base_folder(), PACKAGES_FOLDER, pref.id) set_dirty(pkg_folder) client.run("upload * --all --confirm", assert_error=True) self.assertIn("ERROR: Hello/0.1@lasote/testing:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9: " "Upload package to 'default' failed: Package %s is corrupted, aborting upload" % str(pref), client.out) self.assertIn("Remove it with 'conan remove Hello/0.1@lasote/testing -p=%s'" % NO_SETTINGS_PACKAGE_ID, client.out) client.run("remove Hello/0.1@lasote/testing -p=%s -f" % NO_SETTINGS_PACKAGE_ID) client.run("upload * --all --confirm")
def upload_dirty_test(self): test_server = TestServer([], users={"lasote": "mypass"}) client = TestClient(servers={"default": test_server}, users={"default": [("lasote", "mypass")]}) client.save({"conanfile.py": str(TestConanFile())}) client.run("export . lasote/testing") ref = ConanFileReference.loads("Hello/0.1@lasote/testing") pkg_ref = PackageReference(ref, "12345") package_folder = client.client_cache.package(pkg_ref) save(os.path.join(package_folder, "conaninfo.txt"), "") save(os.path.join(package_folder, "conanmanifest.txt"), "") set_dirty(package_folder) error = client.run("upload * --all --confirm", ignore_error=True) self.assertTrue(error) self.assertIn("ERROR: Package Hello/0.1@lasote/testing:12345 is corrupted, aborting upload", client.out) self.assertIn("Remove it with 'conan remove Hello/0.1@lasote/testing -p=12345'", client.out) client.run("remove Hello/0.1@lasote/testing -p=12345 -f") client.run("upload * --all --confirm")
def test_dirty_download(self): # https://github.com/conan-io/conan/issues/8578 client = TestClient(default_server_user=True) cache_folder = temp_folder() client.run('config set storage.download_cache="%s"' % cache_folder) client.save({ "conanfile.py": GenConanfile().with_package_file("file.txt", "content") }) client.run("create . pkg/0.1@") client.run("upload * --all -c") client.run("remove * -f") client.run("install pkg/0.1@") for f in os.listdir(cache_folder): # damage the file path = os.path.join(cache_folder, f) if os.path.isfile(path): save(path, "broken!") set_dirty(path) client.run("remove * -f") client.run("install pkg/0.1@") assert "pkg/0.1: Downloaded package" in client.out
def _build(self, nodes_to_process, deps_graph, skip_nodes, profile_build_requires, keep_build, root_node, update): """ The build assumes an input of conans ordered by degree, first level should be independent from each other, the next-second level should have dependencies only to first level conans. param nodes_by_level: list of lists [[nodeA, nodeB], [nodeC], [nodeD, ...], ...] """ inverse = deps_graph.inverse_levels() flat = [] for level in inverse: level = sorted(level, key=lambda x: x.conan_ref) flat.extend(n for n in level if n not in skip_nodes) for node, package_id, build_needed in nodes_to_process: conan_ref, conan_file = node.conan_ref, node.conanfile output = ScopedOutput(str(conan_ref), self._out) package_ref = PackageReference(conan_ref, package_id) package_folder = self._client_cache.package(package_ref, conan_file.short_paths) with self._client_cache.package_lock(package_ref): set_dirty(package_folder) if build_needed and (conan_ref, package_id) not in self._built_packages: self._build_package(node, package_id, package_ref, output, keep_build, profile_build_requires, flat, deps_graph, update) else: self._get_existing_package(conan_file, package_ref, output, package_folder, update) self._propagate_info(node, flat, deps_graph) # Call the info method self._call_package_info(conan_file, package_folder) clean_dirty(package_folder) # Finally, propagate information to root node (conan_ref=None) self._propagate_info(root_node, flat, deps_graph)
def _handle_node_cache(self, node, package_id, build_needed, keep_build, profile_build_requires, inverse_levels, deps_graph, update): conan_ref, conan_file = node.conan_ref, node.conanfile output = ScopedOutput(str(conan_ref), self._out) package_ref = PackageReference(conan_ref, package_id) package_folder = self._client_cache.package(package_ref, conan_file.short_paths) with self._client_cache.package_lock(package_ref): set_dirty(package_folder) if build_needed and (conan_ref, package_id) not in self._built_packages: self._build_package(node, package_id, package_ref, output, keep_build, profile_build_requires, inverse_levels, deps_graph, update) else: self._get_existing_package(conan_file, package_ref, output, package_folder, update) self._propagate_info(node, inverse_levels, deps_graph) # Call the info method self._call_package_info(conan_file, package_folder) clean_dirty(package_folder)
def download(self, url, file_path=None, md5=None, sha1=None, sha256=None, **kwargs): """ compatible interface of FileDownloader + checksum """ checksum = sha256 or sha1 or md5 # If it is a user download, it must contain a checksum assert (not self._user_download) or (self._user_download and checksum) h = self._get_hash(url, checksum) with self._lock(h): cached_path = os.path.join(self._cache_folder, h) if is_dirty(cached_path): if os.path.exists(cached_path): os.remove(cached_path) clean_dirty(cached_path) if not os.path.exists(cached_path): set_dirty(cached_path) self._file_downloader.download(url=url, file_path=cached_path, md5=md5, sha1=sha1, sha256=sha256, **kwargs) clean_dirty(cached_path) else: # specific check for corrupted cached files, will raise, but do nothing more # user can report it or "rm -rf cache_folder/path/to/file" try: check_checksum(cached_path, md5, sha1, sha256) except ConanException as e: raise ConanException("%s\nCached downloaded file corrupted: %s" % (str(e), cached_path)) if file_path is not None: file_path = os.path.abspath(file_path) mkdir(os.path.dirname(file_path)) shutil.copy2(cached_path, file_path) else: with open(cached_path, 'rb') as handle: tmp = handle.read() return tmp
def cmd_export(conanfile_path, name, version, user, channel, keep_source, revisions_enabled, output, hook_manager, loader, cache, export=True): """ Export the recipe param conanfile_path: the original source directory of the user containing a conanfile.py """ conanfile = loader.load_export(conanfile_path, name, version, user, channel) ref = ConanFileReference(conanfile.name, conanfile.version, conanfile.user, conanfile.channel) check_casing_conflict(cache=cache, ref=ref) package_layout = cache.package_layout(ref, short_paths=conanfile.short_paths) if not export: metadata = package_layout.load_metadata() recipe_revision = metadata.recipe.revision ref = ref.copy_with_rev(recipe_revision) return ref hook_manager.execute("pre_export", conanfile=conanfile, conanfile_path=conanfile_path, reference=package_layout.ref) logger.debug("EXPORT: %s" % conanfile_path) output.highlight("Exporting package recipe") conan_linter(conanfile_path, output) output = conanfile.output # Get previous digest try: previous_digest = FileTreeManifest.load(package_layout.export()) except IOError: previous_digest = None finally: _recreate_folders(package_layout.export(), package_layout.export_sources()) # Copy sources to target folders with package_layout.conanfile_write_lock(output=output): origin_folder = os.path.dirname(conanfile_path) export_recipe(conanfile, origin_folder, package_layout.export()) export_source(conanfile, origin_folder, package_layout.export_sources()) shutil.copy2(conanfile_path, package_layout.conanfile()) _capture_export_scm_data(conanfile, os.path.dirname(conanfile_path), package_layout.export(), output, scm_src_file=package_layout.scm_folder()) # Execute post-export hook before computing the digest hook_manager.execute("post_export", conanfile=conanfile, reference=package_layout.ref, conanfile_path=package_layout.conanfile()) # Compute the new digest digest = FileTreeManifest.create(package_layout.export(), package_layout.export_sources()) modified_recipe = not previous_digest or previous_digest != digest if modified_recipe: output.success('A new %s version was exported' % CONANFILE) output.info('Folder: %s' % package_layout.export()) else: output.info("The stored package has not changed") digest = previous_digest # Use the old one, keep old timestamp digest.save(package_layout.export()) # Compute the revision for the recipe revision = _update_revision_in_metadata( package_layout=package_layout, revisions_enabled=revisions_enabled, output=output, path=os.path.dirname(conanfile_path), digest=digest, revision_mode=conanfile.revision_mode) # FIXME: Conan 2.0 Clear the registry entry if the recipe has changed source_folder = package_layout.source() if os.path.exists(source_folder): try: if is_dirty(source_folder): output.info("Source folder is corrupted, forcing removal") rmdir(source_folder) elif modified_recipe and not keep_source: output.info( "Package recipe modified in export, forcing source folder removal" ) output.info("Use the --keep-source, -k option to skip it") rmdir(source_folder) except BaseException as e: output.error( "Unable to delete source folder. Will be marked as corrupted for deletion" ) output.warn(str(e)) set_dirty(source_folder) # When revisions enabled, remove the packages not matching the revision if revisions_enabled: packages = search_packages(package_layout, query=None) metadata = package_layout.load_metadata() recipe_revision = metadata.recipe.revision to_remove = [ pid for pid in packages if metadata.packages.get(pid) and metadata.packages.get(pid).recipe_revision != recipe_revision ] if to_remove: output.info( "Removing the local binary packages from different recipe revisions" ) remover = DiskRemover() remover.remove_packages(package_layout, ids_filter=to_remove) ref = ref.copy_with_rev(revision) return ref
def test_set_dirty(self): """ Dirty flag must be created by set_dirty """ set_dirty(self.temp_folder) self.assertTrue(os.path.exists(self.dirty_folder))
def set_dirty_context_manager(self, pref): pkg_folder = os.path.join(self._base_folder, PACKAGES_FOLDER, pref.id) set_dirty(pkg_folder) yield clean_dirty(pkg_folder)
def config_source(export_folder, export_source_folder, local_sources_path, src_folder, conan_file, output, force=False): """ creates src folder and retrieve, calling source() from conanfile the necessary source code """ def remove_source(raise_error=True): output.warn("This can take a while for big packages") try: rmdir(src_folder) except BaseException as e_rm: set_dirty(src_folder) msg = str(e_rm) if six.PY2: msg = str(e_rm).decode( "latin1") # Windows prints some chars in latin1 output.error("Unable to remove source folder %s\n%s" % (src_folder, msg)) output.warn("**** Please delete it manually ****") if raise_error or isinstance(e_rm, KeyboardInterrupt): raise ConanException("Unable to remove source folder") if force: output.warn("Forced removal of source folder") remove_source() elif is_dirty(src_folder): output.warn("Trying to remove corrupted source folder") remove_source() elif conan_file.build_policy_always: output.warn( "Detected build_policy 'always', trying to remove source folder") remove_source() if not os.path.exists(src_folder): output.info('Configuring sources in %s' % src_folder) shutil.copytree(export_folder, src_folder, symlinks=True) # Now move the export-sources to the right location merge_directories(export_source_folder, src_folder) _clean_source_folder(src_folder) try: shutil.rmtree(os.path.join(src_folder, "__pycache__")) except OSError: pass set_dirty(src_folder) os.chdir(src_folder) conan_file.source_folder = src_folder try: with get_env_context_manager(conan_file): with conanfile_exception_formatter(str(conan_file), "source"): conan_file.build_folder = None conan_file.package_folder = None scm = get_scm(conan_file, src_folder) if scm: # scm.capture_origin before exporting if local_sources_path and os.path.exists( local_sources_path): output.info("Getting sources from folder: %s" % local_sources_path) merge_directories(local_sources_path, src_folder) _clean_source_folder(src_folder) else: output.info("Getting sources from url: '%s'" % scm.url) scm.clone() scm.checkout() conan_file.source() clean_dirty(src_folder) # Everything went well, remove DIRTY flag except Exception as e: os.chdir(export_folder) # in case source() fails (user error, typically), remove the src_folder # and raise to interrupt any other processes (build, package) output.warn("Trying to remove corrupted source folder") remove_source(raise_error=False) if isinstance(e, ConanExceptionInUserConanfileMethod): raise e raise ConanException(e)
def config_source(export_folder, export_source_folder, local_sources_path, src_folder, conanfile, output, conanfile_path, reference, hook_manager, client_cache): """ creates src folder and retrieve, calling source() from conanfile the necessary source code """ def remove_source(raise_error=True): output.warn("This can take a while for big packages") try: rmdir(src_folder) except BaseException as e_rm: set_dirty(src_folder) msg = str(e_rm) if six.PY2: msg = str(e_rm).decode( "latin1") # Windows prints some chars in latin1 output.error("Unable to remove source folder %s\n%s" % (src_folder, msg)) output.warn("**** Please delete it manually ****") if raise_error or isinstance(e_rm, KeyboardInterrupt): raise ConanException("Unable to remove source folder") if is_dirty(src_folder): output.warn("Trying to remove corrupted source folder") remove_source() elif conanfile.build_policy_always: output.warn( "Detected build_policy 'always', trying to remove source folder") remove_source() elif local_sources_path and os.path.exists(local_sources_path): output.warn( "Detected 'scm' auto in conanfile, trying to remove source folder") remove_source() if not os.path.exists(src_folder): set_dirty(src_folder) mkdir(src_folder) os.chdir(src_folder) conanfile.source_folder = src_folder try: with conanfile_exception_formatter(str(conanfile), "source"): with get_env_context_manager(conanfile): conanfile.build_folder = None conanfile.package_folder = None hook_manager.execute("pre_source", conanfile=conanfile, conanfile_path=conanfile_path, reference=reference) output.info('Configuring sources in %s' % src_folder) scm_data = get_scm_data(conanfile) if scm_data: dest_dir = os.path.normpath( os.path.join(src_folder, scm_data.subfolder)) captured = local_sources_path and os.path.exists( local_sources_path) local_sources_path = local_sources_path if captured else None _fetch_scm(scm_data, dest_dir, local_sources_path, output) # Files from python requires are obtained before the self files from conans.client.cmd.export import export_source for python_require in conanfile.python_requires: src = client_cache.export_sources( python_require.conan_ref) export_source(conanfile, src, src_folder, output) # so self exported files have precedence over python_requires ones merge_directories(export_folder, src_folder) # Now move the export-sources to the right location merge_directories(export_source_folder, src_folder) _clean_source_folder(src_folder) try: shutil.rmtree(os.path.join(src_folder, "__pycache__")) except OSError: pass conanfile.source() hook_manager.execute("post_source", conanfile=conanfile, conanfile_path=conanfile_path, reference=reference) clean_dirty(src_folder) # Everything went well, remove DIRTY flag except Exception as e: os.chdir(export_folder) # in case source() fails (user error, typically), remove the src_folder # and raise to interrupt any other processes (build, package) output.warn("Trying to remove corrupted source folder") remove_source(raise_error=False) if isinstance(e, ConanExceptionInUserConanfileMethod): raise e raise ConanException(e)
def cmd_export(app, conanfile_path, name, version, user, channel, keep_source, export=True, graph_lock=None): """ Export the recipe param conanfile_path: the original source directory of the user containing a conanfile.py """ loader, cache, hook_manager, output = app.loader, app.cache, app.hook_manager, app.out revisions_enabled = app.config.revisions_enabled conanfile = loader.load_export(conanfile_path, name, version, user, channel) # FIXME: Conan 2.0, deprecate CONAN_USER AND CONAN_CHANNEL and remove this try excepts # Take the default from the env vars if they exist to not break behavior try: user = conanfile.user except ConanException: user = None try: channel = conanfile.channel except ConanException: channel = None ref = ConanFileReference(conanfile.name, conanfile.version, user, channel) # If we receive lock information, python_requires could have been locked if graph_lock: node_id = graph_lock.get_node(ref) python_requires = graph_lock.python_requires(node_id) # TODO: check that the locked python_requires are different from the loaded ones # FIXME: private access, will be improved when api collaborators are improved loader._python_requires._range_resolver.output # invalidate previous version range output conanfile = loader.load_export(conanfile_path, conanfile.name, conanfile.version, conanfile.user, conanfile.channel, python_requires) check_casing_conflict(cache=cache, ref=ref) package_layout = cache.package_layout(ref, short_paths=conanfile.short_paths) if not export: metadata = package_layout.load_metadata() recipe_revision = metadata.recipe.revision ref = ref.copy_with_rev(recipe_revision) if graph_lock: graph_lock.update_exported_ref(node_id, ref) return ref hook_manager.execute("pre_export", conanfile=conanfile, conanfile_path=conanfile_path, reference=package_layout.ref) logger.debug("EXPORT: %s" % conanfile_path) output.highlight("Exporting package recipe") conan_linter(conanfile_path, output) output = conanfile.output # Get previous digest try: previous_manifest = FileTreeManifest.load(package_layout.export()) except IOError: previous_manifest = None finally: _recreate_folders(package_layout.export(), package_layout.export_sources()) # Copy sources to target folders with package_layout.conanfile_write_lock(output=output): origin_folder = os.path.dirname(conanfile_path) export_recipe(conanfile, origin_folder, package_layout.export()) export_source(conanfile, origin_folder, package_layout.export_sources()) shutil.copy2(conanfile_path, package_layout.conanfile()) _capture_export_scm_data(conanfile, os.path.dirname(conanfile_path), package_layout.export(), output, scm_src_file=package_layout.scm_folder()) # Execute post-export hook before computing the digest hook_manager.execute("post_export", conanfile=conanfile, reference=package_layout.ref, conanfile_path=package_layout.conanfile()) # Compute the new digest manifest = FileTreeManifest.create(package_layout.export(), package_layout.export_sources()) modified_recipe = not previous_manifest or previous_manifest != manifest if modified_recipe: output.success('A new %s version was exported' % CONANFILE) output.info('Folder: %s' % package_layout.export()) else: output.info("The stored package has not changed") manifest = previous_manifest # Use the old one, keep old timestamp manifest.save(package_layout.export()) # Compute the revision for the recipe revision = _update_revision_in_metadata( package_layout=package_layout, revisions_enabled=revisions_enabled, output=output, path=os.path.dirname(conanfile_path), manifest=manifest, revision_mode=conanfile.revision_mode) # FIXME: Conan 2.0 Clear the registry entry if the recipe has changed source_folder = package_layout.source() if os.path.exists(source_folder): try: if is_dirty(source_folder): output.info("Source folder is corrupted, forcing removal") rmdir(source_folder) elif modified_recipe and not keep_source: output.info( "Package recipe modified in export, forcing source folder removal" ) output.info("Use the --keep-source, -k option to skip it") rmdir(source_folder) except BaseException as e: output.error( "Unable to delete source folder. Will be marked as corrupted for deletion" ) output.warn(str(e)) set_dirty(source_folder) # When revisions enabled, remove the packages not matching the revision if revisions_enabled: packages = search_packages(package_layout, query=None) metadata = package_layout.load_metadata() recipe_revision = metadata.recipe.revision to_remove = [ pid for pid in packages if metadata.packages.get(pid) and metadata.packages.get(pid).recipe_revision != recipe_revision ] if to_remove: output.info( "Removing the local binary packages from different recipe revisions" ) remover = DiskRemover() remover.remove_packages(package_layout, ids_filter=to_remove) ref = ref.copy_with_rev(revision) output.info("Exported revision: %s" % revision) if graph_lock: graph_lock.update_exported_ref(node_id, ref) return ref
def cmd_export(app, conanfile_path, name, version, user, channel, keep_source, export=True, graph_lock=None, ignore_dirty=False): """ Export the recipe param conanfile_path: the original source directory of the user containing a conanfile.py """ loader, cache, hook_manager, output = app.loader, app.cache, app.hook_manager, app.out revisions_enabled = app.config.revisions_enabled scm_to_conandata = app.config.scm_to_conandata conanfile = loader.load_export(conanfile_path, name, version, user, channel) # FIXME: Conan 2.0, deprecate CONAN_USER AND CONAN_CHANNEL and remove this try excepts # Take the default from the env vars if they exist to not break behavior try: user = conanfile.user except ConanV2Exception: raise except ConanException: user = None try: channel = conanfile.channel except ConanV2Exception: raise except ConanException: channel = None ref = ConanFileReference(conanfile.name, conanfile.version, user, channel) # If we receive lock information, python_requires could have been locked if graph_lock: node_id = graph_lock.get_consumer(ref) python_requires = graph_lock.python_requires(node_id) # TODO: check that the locked python_requires are different from the loaded ones app.range_resolver.clear_output( ) # invalidate previous version range output conanfile = loader.load_export(conanfile_path, conanfile.name, conanfile.version, ref.user, ref.channel, python_requires) check_casing_conflict(cache=cache, ref=ref) package_layout = cache.package_layout(ref, short_paths=conanfile.short_paths) if not export: metadata = package_layout.load_metadata() recipe_revision = metadata.recipe.revision ref = ref.copy_with_rev(recipe_revision) if graph_lock: graph_lock.update_exported_ref(node_id, ref) return ref _check_settings_for_warnings(conanfile, output) hook_manager.execute("pre_export", conanfile=conanfile, conanfile_path=conanfile_path, reference=package_layout.ref) logger.debug("EXPORT: %s" % conanfile_path) output.highlight("Exporting package recipe") output = conanfile.output # Copy sources to target folders with package_layout.conanfile_write_lock(output=output): # Get previous manifest try: previous_manifest = package_layout.recipe_manifest() except IOError: previous_manifest = None package_layout.export_remove() export_folder = package_layout.export() export_src_folder = package_layout.export_sources() mkdir(export_folder) mkdir(export_src_folder) origin_folder = os.path.dirname(conanfile_path) export_recipe(conanfile, origin_folder, export_folder) export_source(conanfile, origin_folder, export_src_folder) shutil.copy2(conanfile_path, package_layout.conanfile()) # Calculate the "auto" values and replace in conanfile.py scm_data, local_src_folder = _capture_scm_auto_fields( conanfile, os.path.dirname(conanfile_path), package_layout, output, ignore_dirty, scm_to_conandata) # Clear previous scm_folder modified_recipe = False scm_sources_folder = package_layout.scm_sources() if local_src_folder and not keep_source: # Copy the local scm folder to scm_sources in the cache mkdir(scm_sources_folder) _export_scm(scm_data, local_src_folder, scm_sources_folder, output) # https://github.com/conan-io/conan/issues/5195#issuecomment-551840597 # It will cause the source folder to be removed (needed because the recipe still has # the "auto" with uncommitted changes) modified_recipe = True # Execute post-export hook before computing the digest hook_manager.execute("post_export", conanfile=conanfile, reference=package_layout.ref, conanfile_path=package_layout.conanfile()) # Compute the new digest manifest = FileTreeManifest.create(export_folder, export_src_folder) modified_recipe |= not previous_manifest or previous_manifest != manifest if modified_recipe: output.success('A new %s version was exported' % CONANFILE) output.info('Folder: %s' % export_folder) else: output.info("The stored package has not changed") manifest = previous_manifest # Use the old one, keep old timestamp manifest.save(export_folder) # Compute the revision for the recipe revision = _update_revision_in_metadata( package_layout=package_layout, revisions_enabled=revisions_enabled, output=output, path=os.path.dirname(conanfile_path), manifest=manifest, revision_mode=conanfile.revision_mode) # FIXME: Conan 2.0 Clear the registry entry if the recipe has changed source_folder = package_layout.source() if os.path.exists(source_folder): try: if is_dirty(source_folder): output.info("Source folder is corrupted, forcing removal") rmdir(source_folder) clean_dirty(source_folder) elif modified_recipe and not keep_source: output.info( "Package recipe modified in export, forcing source folder removal" ) output.info("Use the --keep-source, -k option to skip it") rmdir(source_folder) except BaseException as e: output.error( "Unable to delete source folder. Will be marked as corrupted for deletion" ) output.warn(str(e)) set_dirty(source_folder) # When revisions enabled, remove the packages not matching the revision if revisions_enabled: packages = search_packages(package_layout, query=None) metadata = package_layout.load_metadata() recipe_revision = metadata.recipe.revision to_remove = [ pid for pid in packages if metadata.packages.get(pid) and metadata.packages.get(pid).recipe_revision != recipe_revision ] if to_remove: output.info( "Removing the local binary packages from different recipe revisions" ) remover = DiskRemover() remover.remove_packages(package_layout, ids_filter=to_remove) ref = ref.copy_with_rev(revision) output.info("Exported revision: %s" % revision) if graph_lock: graph_lock.update_exported_ref(node_id, ref) return ref
def config_source(export_folder, export_source_folder, src_folder, conan_file, output, force=False): """ creates src folder and retrieve, calling source() from conanfile the necessary source code """ def remove_source(raise_error=True): output.warn("This can take a while for big packages") try: rmdir(src_folder) except BaseException as e_rm: set_dirty(src_folder) msg = str(e_rm) if six.PY2: msg = str(e_rm).decode( "latin1") # Windows prints some chars in latin1 output.error("Unable to remove source folder %s\n%s" % (src_folder, msg)) output.warn("**** Please delete it manually ****") if raise_error or isinstance(e_rm, KeyboardInterrupt): raise ConanException("Unable to remove source folder") if force: output.warn("Forced removal of source folder") remove_source() elif is_dirty(src_folder): output.warn("Trying to remove dirty source folder") remove_source() elif conan_file.build_policy_always: output.warn( "Detected build_policy 'always', trying to remove source folder") remove_source() if not os.path.exists(src_folder): output.info('Configuring sources in %s' % src_folder) shutil.copytree(export_folder, src_folder, symlinks=True) # Now move the export-sources to the right location merge_directories(export_source_folder, src_folder) for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE + "c", CONANFILE + "o", CONANFILE, CONAN_MANIFEST): try: os.remove(os.path.join(src_folder, f)) except OSError: pass try: shutil.rmtree(os.path.join(src_folder, "__pycache__")) except OSError: pass set_dirty(src_folder) os.chdir(src_folder) conan_file.source_folder = src_folder try: with tools.environment_append(conan_file.env): with conanfile_exception_formatter(str(conan_file), "source"): conan_file.build_folder = None conan_file.package_folder = None conan_file.source() clean_dirty(src_folder) # Everything went well, remove DIRTY flag except Exception as e: os.chdir(export_folder) # in case source() fails (user error, typically), remove the src_folder # and raise to interrupt any other processes (build, package) output.warn("Trying to remove dirty source folder") remove_source(raise_error=False) if isinstance(e, ConanExceptionInUserConanfileMethod): raise e raise ConanException(e)
def config_source(export_folder, export_source_folder, src_folder, conan_file, output, force=False): """ creates src folder and retrieve, calling source() from conanfile the necessary source code """ def remove_source(raise_error=True): output.warn("This can take a while for big packages") try: rmdir(src_folder) except BaseException as e_rm: set_dirty(src_folder) msg = str(e_rm) if six.PY2: msg = str(e_rm).decode("latin1") # Windows prints some chars in latin1 output.error("Unable to remove source folder %s\n%s" % (src_folder, msg)) output.warn("**** Please delete it manually ****") if raise_error or isinstance(e_rm, KeyboardInterrupt): raise ConanException("Unable to remove source folder") if force: output.warn("Forced removal of source folder") remove_source() elif is_dirty(src_folder): output.warn("Trying to remove corrupted source folder") remove_source() elif conan_file.build_policy_always: output.warn("Detected build_policy 'always', trying to remove source folder") remove_source() if not os.path.exists(src_folder): output.info('Configuring sources in %s' % src_folder) shutil.copytree(export_folder, src_folder, symlinks=True) # Now move the export-sources to the right location merge_directories(export_source_folder, src_folder) for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE+"c", CONANFILE+"o", CONANFILE, CONAN_MANIFEST): try: os.remove(os.path.join(src_folder, f)) except OSError: pass try: shutil.rmtree(os.path.join(src_folder, "__pycache__")) except OSError: pass set_dirty(src_folder) os.chdir(src_folder) conan_file.source_folder = src_folder try: with get_env_context_manager(conan_file): with conanfile_exception_formatter(str(conan_file), "source"): conan_file.build_folder = None conan_file.package_folder = None conan_file.source() clean_dirty(src_folder) # Everything went well, remove DIRTY flag except Exception as e: os.chdir(export_folder) # in case source() fails (user error, typically), remove the src_folder # and raise to interrupt any other processes (build, package) output.warn("Trying to remove corrupted source folder") remove_source(raise_error=False) if isinstance(e, ConanExceptionInUserConanfileMethod): raise e raise ConanException(e)