def test_hardlink_or_copy_cross_device_should_copy(self): content = b'hello' # Mock os.link to throw an CROSS-DEVICE error with unittest.mock.patch('os.link') as os_mock: err = OSError() err.errno = errno.EXDEV os_mock.side_effect = err with temporary_dir() as src_dir, temporary_file() as dst: dst.write(content) dst.close() src_path = os.path.join(src_dir, 'src') safe_hardlink_or_copy(dst.name, src_path) with open(src_path, 'rb') as f: self.assertEqual(content, f.read()) # Make sure it's not symlink self.assertFalse(os.path.islink(dst.name)) # Make sure they are separate copies self.assertFalse(self._is_hard_link(dst.name, src_path))
def test_hardlink_or_copy_cross_device_should_copy(self): content = b'hello' # Mock os.link to throw an CROSS-DEVICE error with mock.patch('os.link') as os_mock: err = OSError() err.errno = errno.EXDEV os_mock.side_effect = err with temporary_dir() as src_dir, temporary_file() as dst: dst.write(content) dst.close() src_path = os.path.join(src_dir, 'src') safe_hardlink_or_copy(dst.name, src_path) with open(src_path, 'rb') as f: self.assertEqual(content, f.read()) # Make sure it's not symlink self.assertFalse(os.path.islink(dst.name)) # Make sure they are separate copies self.assertFalse(self._is_hard_link(dst.name, src_path))
def _compile_compiler_bridge(self, context): """Compile the compiler bridge to be used by zinc, using our scala bootstrapper. It will compile and cache the jar, and materialize it if not already there. :param context: The context of the task trying to compile the bridge. This is mostly needed to use its scheduler to create digests of the relevant jars. :return: The absolute path to the compiled scala-compiler-bridge jar. """ bridge_jar_name = "scala-compiler-bridge.jar" bridge_jar = os.path.join(self._compiler_bridge_cache_dir, bridge_jar_name) global_bridge_cache_dir = os.path.join( self._zinc_factory.get_options().pants_bootstrapdir, fast_relpath(self._compiler_bridge_cache_dir, self._workdir()), ) globally_cached_bridge_jar = os.path.join(global_bridge_cache_dir, bridge_jar_name) # Workaround to avoid recompiling the bridge for every integration test # We check the bootstrapdir (.cache) for the bridge. # If it exists, we make a copy to the buildroot. # # TODO Remove when action caches are implemented. if os.path.exists(globally_cached_bridge_jar): # Cache the bridge jar under buildroot, to allow snapshotting safe_mkdir(self._relative_to_buildroot(self._compiler_bridge_cache_dir)) safe_hardlink_or_copy(globally_cached_bridge_jar, bridge_jar) if not os.path.exists(bridge_jar): res = self._run_bootstrapper(bridge_jar, context) context._scheduler.materialize_directory( DirectoryToMaterialize(res.output_directory_digest) ) # For the workaround above to work, we need to store a copy of the bridge in # the bootstrapdir cache (.cache). safe_mkdir(global_bridge_cache_dir) safe_hardlink_or_copy(bridge_jar, globally_cached_bridge_jar) return ClasspathEntry(bridge_jar, res.output_directory_digest) else: bridge_jar_snapshot = context._scheduler.capture_snapshots( ( PathGlobsAndRoot( PathGlobs((self._relative_to_buildroot(bridge_jar),)), get_buildroot() ), ) )[0] bridge_jar_digest = bridge_jar_snapshot.directory_digest return ClasspathEntry(bridge_jar, bridge_jar_digest)
def test_hardlink_or_copy(self): content = b'hello' with temporary_dir() as src_dir, temporary_file() as dst: dst.write(content) dst.close() src_path = os.path.join(src_dir, 'src') safe_hardlink_or_copy(dst.name, src_path) with open(src_path, 'rb') as f: self.assertEqual(content, f.read()) # Make sure it's not symlink self.assertFalse(os.path.islink(dst.name)) # Make sure they point to the same node self.assertTrue(self._is_hard_link(dst.name, src_path))
def test_hardlink_or_copy(self): content = b'hello' with temporary_dir() as src_dir, temporary_file() as dst: dst.write(content) dst.close() src_path = os.path.join(src_dir, 'src') safe_hardlink_or_copy(dst.name, src_path) with open(src_path, 'rb') as f: self.assertEqual(content, f.read()) # Make sure it's not symlink self.assertFalse(os.path.islink(dst.name)) # Make sure they point to the same node self.assertTrue(self._is_hard_link(dst.name, src_path))
def compile_compiler_bridge(self, context): """Compile the compiler bridge to be used by zinc, using our scala bootstrapper. It will compile and cache the jar, and materialize it if not already there. :param context: The context of the task trying to compile the bridge. This is mostly needed to use its scheduler to create digests of the relevant jars. :return: The absolute path to the compiled scala-compiler-bridge jar. """ bridge_jar_name = 'scala-compiler-bridge.jar' bridge_jar = os.path.join(self._compiler_bridge_cache_dir, bridge_jar_name) global_bridge_cache_dir = os.path.join(self._zinc_factory.get_options().pants_bootstrapdir, fast_relpath(self._compiler_bridge_cache_dir, self._workdir())) globally_cached_bridge_jar = os.path.join(global_bridge_cache_dir, bridge_jar_name) # Workaround to avoid recompiling the bridge for every integration test # We check the bootstrapdir (.cache) for the bridge. # If it exists, we make a copy to the buildroot. # # TODO Remove when action caches are implemented. if os.path.exists(globally_cached_bridge_jar): # Cache the bridge jar under buildroot, to allow snapshotting safe_mkdir(self._relative_to_buildroot(self._compiler_bridge_cache_dir)) safe_hardlink_or_copy(globally_cached_bridge_jar, bridge_jar) if not os.path.exists(bridge_jar): res = self._run_bootstrapper(bridge_jar, context) context._scheduler.materialize_directories(( DirectoryToMaterialize(get_buildroot(), res.output_directory_digest), )) # For the workaround above to work, we need to store a copy of the bridge in # the bootstrapdir cache (.cache). safe_mkdir(global_bridge_cache_dir) safe_hardlink_or_copy(bridge_jar, globally_cached_bridge_jar) return ClasspathEntry(bridge_jar, res.output_directory_digest) else: bridge_jar_snapshot = context._scheduler.capture_snapshots((PathGlobsAndRoot( PathGlobs((self._relative_to_buildroot(bridge_jar),)), text_type(get_buildroot()) ),))[0] bridge_jar_digest = bridge_jar_snapshot.directory_digest return ClasspathEntry(bridge_jar, bridge_jar_digest)
def _hardlink_cachepath(cls, ivy_repository_cache_dir, inpath, hardlink_dir, outpath): """hardlinks all paths listed in inpath that are under ivy_repository_cache_dir into hardlink_dir. If there is an existing hardlink for a file under inpath, it is used rather than creating a new hardlink. Preserves all other paths. Writes the resulting paths to outpath. Returns a map of path -> hardlink to that path. """ safe_mkdir(hardlink_dir) # The ivy_repository_cache_dir might itself be a hardlink. In this case, ivy may return paths that # reference the realpath of the .jar file after it is resolved in the cache dir. To handle # this case, add both the hardlink'ed path and the realpath to the jar to the hardlink map. real_ivy_cache_dir = os.path.realpath(ivy_repository_cache_dir) hardlink_map = OrderedDict() inpaths = cls._load_classpath_from_cachepath(inpath) paths = OrderedSet([os.path.realpath(path) for path in inpaths]) for path in paths: if path.startswith(real_ivy_cache_dir): hardlink_map[path] = os.path.join(hardlink_dir, os.path.relpath(path, real_ivy_cache_dir)) else: # This path is outside the cache. We won't hardlink it. hardlink_map[path] = path # Create hardlinks for paths in the ivy cache dir. for path, hardlink in six.iteritems(hardlink_map): if path == hardlink: # Skip paths that aren't going to be hardlinked. continue safe_mkdir(os.path.dirname(hardlink)) safe_hardlink_or_copy(path, hardlink) # (re)create the classpath with all of the paths with safe_open(outpath, 'w') as outfile: outfile.write(':'.join(OrderedSet(hardlink_map.values()))) return dict(hardlink_map)
def _map_coord_to_resolved_jars(cls, result, coursier_cache_path, pants_jar_path_base): """ Map resolved files to each org:name:version Example: { "conflict_resolution": {}, "dependencies": [ { "coord": "a", "dependencies": ["b", "c"], "file": "a.jar" }, { "coord": "b", "dependencies": [], "file": "b.jar" }, { "coord": "c", "dependencies": [], "file": "c.jar" }, { "coord": "a:sources", "dependencies": ["b", "c"], "file": "a-sources.jar" }, ] } Should return: { M2Coordinate("a", ...): ResolvedJar(classifier='', path/cache_path="a.jar"), M2Coordinate("a", ..., classifier="sources"): ResolvedJar(classifier='sources', path/cache_path="a-sources.jar"), M2Coordinate("b", ...): ResolvedJar(classifier='', path/cache_path="b.jar"), M2Coordinate("c", ...): ResolvedJar(classifier='', path/cache_path="c.jar"), } :param result: coursier json output :param coursier_cache_path: coursier cache location :param pants_jar_path_base: location under pants workdir to store the hardlink to the coursier cache :return: a map from maven coordinate to a resolved jar. """ coord_to_resolved_jars = dict() for dep in result['dependencies']: coord = dep['coord'] jar_path = dep.get('file', None) if not jar_path: # NB: Not all coordinates will have associated files. # This is fine. Some coordinates will just have dependencies. continue if not os.path.exists(jar_path): raise CoursierResultNotFound( "Jar path not found: {}".format(jar_path)) pants_path = cls._get_path_to_jar(coursier_cache_path, pants_jar_path_base, jar_path) if not os.path.exists(pants_path): safe_mkdir(os.path.dirname(pants_path)) safe_hardlink_or_copy(jar_path, pants_path) coord = cls.to_m2_coord(coord) resolved_jar = ResolvedJar(coord, cache_path=jar_path, pants_path=pants_path) coord_to_resolved_jars[coord] = resolved_jar return coord_to_resolved_jars
def _map_coord_to_resolved_jars(cls, result, coursier_cache_path, pants_jar_path_base): """ Map resolved files to each org:name:version Example: { "conflict_resolution": {}, "dependencies": [ { "coord": "a", "dependencies": ["b", "c"], "file": "a.jar" }, { "coord": "b", "dependencies": [], "file": "b.jar" }, { "coord": "c", "dependencies": [], "file": "c.jar" }, { "coord": "a:sources", "dependencies": ["b", "c"], "file": "a-sources.jar" }, ] } Should return: { M2Coordinate("a", ...): ResolvedJar(classifier='', path/cache_path="a.jar"), M2Coordinate("a", ..., classifier="sources"): ResolvedJar(classifier='sources', path/cache_path="a-sources.jar"), M2Coordinate("b", ...): ResolvedJar(classifier='', path/cache_path="b.jar"), M2Coordinate("c", ...): ResolvedJar(classifier='', path/cache_path="c.jar"), } :param result: coursier json output :param coursier_cache_path: coursier cache location :param pants_jar_path_base: location under pants workdir to store the hardlink to the coursier cache :return: a map from maven coordinate to a resolved jar. """ coord_to_resolved_jars = dict() for dep in result['dependencies']: coord = dep['coord'] jar_path = dep.get('file', None) if not jar_path: # NB: Not all coordinates will have associated files. # This is fine. Some coordinates will just have dependencies. continue if not os.path.exists(jar_path): raise CoursierResultNotFound("Jar path not found: {}".format(jar_path)) pants_path = cls._get_path_to_jar(coursier_cache_path, pants_jar_path_base, jar_path) if not os.path.exists(pants_path): safe_mkdir(os.path.dirname(pants_path)) safe_hardlink_or_copy(jar_path, pants_path) coord = cls.to_m2_coord(coord) resolved_jar = ResolvedJar(coord, cache_path=jar_path, pants_path=pants_path) coord_to_resolved_jars[coord] = resolved_jar return coord_to_resolved_jars