def test_hardlink_or_copy_cross_device_should_copy(self):
content = b'hello'
# Mock os.link to throw an CROSS-DEVICE error
with unittest.mock.patch('os.link') as os_mock:
err = OSError()
err.errno = errno.EXDEV
os_mock.side_effect = err
with temporary_dir() as src_dir, temporary_file() as dst:
dst.write(content)
dst.close()
src_path = os.path.join(src_dir, 'src')
safe_hardlink_or_copy(dst.name, src_path)
with open(src_path, 'rb') as f:
self.assertEqual(content, f.read())
# Make sure it's not symlink
self.assertFalse(os.path.islink(dst.name))
# Make sure they are separate copies
self.assertFalse(self._is_hard_link(dst.name, src_path))
def test_hardlink_or_copy_cross_device_should_copy(self):
content = b'hello'
# Mock os.link to throw an CROSS-DEVICE error
with mock.patch('os.link') as os_mock:
err = OSError()
err.errno = errno.EXDEV
os_mock.side_effect = err
with temporary_dir() as src_dir, temporary_file() as dst:
dst.write(content)
dst.close()
src_path = os.path.join(src_dir, 'src')
safe_hardlink_or_copy(dst.name, src_path)
with open(src_path, 'rb') as f:
self.assertEqual(content, f.read())
# Make sure it's not symlink
self.assertFalse(os.path.islink(dst.name))
# Make sure they are separate copies
self.assertFalse(self._is_hard_link(dst.name, src_path))
def _compile_compiler_bridge(self, context):
"""Compile the compiler bridge to be used by zinc, using our scala bootstrapper. It will
compile and cache the jar, and materialize it if not already there.
:param context: The context of the task trying to compile the bridge.
This is mostly needed to use its scheduler to create digests of the relevant jars.
:return: The absolute path to the compiled scala-compiler-bridge jar.
"""
bridge_jar_name = "scala-compiler-bridge.jar"
bridge_jar = os.path.join(self._compiler_bridge_cache_dir, bridge_jar_name)
global_bridge_cache_dir = os.path.join(
self._zinc_factory.get_options().pants_bootstrapdir,
fast_relpath(self._compiler_bridge_cache_dir, self._workdir()),
)
globally_cached_bridge_jar = os.path.join(global_bridge_cache_dir, bridge_jar_name)
# Workaround to avoid recompiling the bridge for every integration test
# We check the bootstrapdir (.cache) for the bridge.
# If it exists, we make a copy to the buildroot.
#
# TODO Remove when action caches are implemented.
if os.path.exists(globally_cached_bridge_jar):
# Cache the bridge jar under buildroot, to allow snapshotting
safe_mkdir(self._relative_to_buildroot(self._compiler_bridge_cache_dir))
safe_hardlink_or_copy(globally_cached_bridge_jar, bridge_jar)
if not os.path.exists(bridge_jar):
res = self._run_bootstrapper(bridge_jar, context)
context._scheduler.materialize_directory(
DirectoryToMaterialize(res.output_directory_digest)
)
# For the workaround above to work, we need to store a copy of the bridge in
# the bootstrapdir cache (.cache).
safe_mkdir(global_bridge_cache_dir)
safe_hardlink_or_copy(bridge_jar, globally_cached_bridge_jar)
return ClasspathEntry(bridge_jar, res.output_directory_digest)
else:
bridge_jar_snapshot = context._scheduler.capture_snapshots(
(
PathGlobsAndRoot(
PathGlobs((self._relative_to_buildroot(bridge_jar),)), get_buildroot()
),
)
)[0]
bridge_jar_digest = bridge_jar_snapshot.directory_digest
return ClasspathEntry(bridge_jar, bridge_jar_digest)
def test_hardlink_or_copy(self):
content = b'hello'
with temporary_dir() as src_dir, temporary_file() as dst:
dst.write(content)
dst.close()
src_path = os.path.join(src_dir, 'src')
safe_hardlink_or_copy(dst.name, src_path)
with open(src_path, 'rb') as f:
self.assertEqual(content, f.read())
# Make sure it's not symlink
self.assertFalse(os.path.islink(dst.name))
# Make sure they point to the same node
self.assertTrue(self._is_hard_link(dst.name, src_path))
def test_hardlink_or_copy(self):
content = b'hello'
with temporary_dir() as src_dir, temporary_file() as dst:
dst.write(content)
dst.close()
src_path = os.path.join(src_dir, 'src')
safe_hardlink_or_copy(dst.name, src_path)
with open(src_path, 'rb') as f:
self.assertEqual(content, f.read())
# Make sure it's not symlink
self.assertFalse(os.path.islink(dst.name))
# Make sure they point to the same node
self.assertTrue(self._is_hard_link(dst.name, src_path))
def compile_compiler_bridge(self, context):
"""Compile the compiler bridge to be used by zinc, using our scala bootstrapper.
It will compile and cache the jar, and materialize it if not already there.
:param context: The context of the task trying to compile the bridge.
This is mostly needed to use its scheduler to create digests of the relevant jars.
:return: The absolute path to the compiled scala-compiler-bridge jar.
"""
bridge_jar_name = 'scala-compiler-bridge.jar'
bridge_jar = os.path.join(self._compiler_bridge_cache_dir, bridge_jar_name)
global_bridge_cache_dir = os.path.join(self._zinc_factory.get_options().pants_bootstrapdir, fast_relpath(self._compiler_bridge_cache_dir, self._workdir()))
globally_cached_bridge_jar = os.path.join(global_bridge_cache_dir, bridge_jar_name)
# Workaround to avoid recompiling the bridge for every integration test
# We check the bootstrapdir (.cache) for the bridge.
# If it exists, we make a copy to the buildroot.
#
# TODO Remove when action caches are implemented.
if os.path.exists(globally_cached_bridge_jar):
# Cache the bridge jar under buildroot, to allow snapshotting
safe_mkdir(self._relative_to_buildroot(self._compiler_bridge_cache_dir))
safe_hardlink_or_copy(globally_cached_bridge_jar, bridge_jar)
if not os.path.exists(bridge_jar):
res = self._run_bootstrapper(bridge_jar, context)
context._scheduler.materialize_directories((
DirectoryToMaterialize(get_buildroot(), res.output_directory_digest),
))
# For the workaround above to work, we need to store a copy of the bridge in
# the bootstrapdir cache (.cache).
safe_mkdir(global_bridge_cache_dir)
safe_hardlink_or_copy(bridge_jar, globally_cached_bridge_jar)
return ClasspathEntry(bridge_jar, res.output_directory_digest)
else:
bridge_jar_snapshot = context._scheduler.capture_snapshots((PathGlobsAndRoot(
PathGlobs((self._relative_to_buildroot(bridge_jar),)),
text_type(get_buildroot())
),))[0]
bridge_jar_digest = bridge_jar_snapshot.directory_digest
return ClasspathEntry(bridge_jar, bridge_jar_digest)
def _hardlink_cachepath(cls, ivy_repository_cache_dir, inpath, hardlink_dir, outpath):
"""hardlinks all paths listed in inpath that are under ivy_repository_cache_dir into hardlink_dir.
If there is an existing hardlink for a file under inpath, it is used rather than creating
a new hardlink. Preserves all other paths. Writes the resulting paths to outpath.
Returns a map of path -> hardlink to that path.
"""
safe_mkdir(hardlink_dir)
# The ivy_repository_cache_dir might itself be a hardlink. In this case, ivy may return paths that
# reference the realpath of the .jar file after it is resolved in the cache dir. To handle
# this case, add both the hardlink'ed path and the realpath to the jar to the hardlink map.
real_ivy_cache_dir = os.path.realpath(ivy_repository_cache_dir)
hardlink_map = OrderedDict()
inpaths = cls._load_classpath_from_cachepath(inpath)
paths = OrderedSet([os.path.realpath(path) for path in inpaths])
for path in paths:
if path.startswith(real_ivy_cache_dir):
hardlink_map[path] = os.path.join(hardlink_dir, os.path.relpath(path, real_ivy_cache_dir))
else:
# This path is outside the cache. We won't hardlink it.
hardlink_map[path] = path
# Create hardlinks for paths in the ivy cache dir.
for path, hardlink in six.iteritems(hardlink_map):
if path == hardlink:
# Skip paths that aren't going to be hardlinked.
continue
safe_mkdir(os.path.dirname(hardlink))
safe_hardlink_or_copy(path, hardlink)
# (re)create the classpath with all of the paths
with safe_open(outpath, 'w') as outfile:
outfile.write(':'.join(OrderedSet(hardlink_map.values())))
return dict(hardlink_map)
def _map_coord_to_resolved_jars(cls, result, coursier_cache_path,
pants_jar_path_base):
"""
Map resolved files to each org:name:version
Example:
{
"conflict_resolution": {},
"dependencies": [
{
"coord": "a",
"dependencies": ["b", "c"],
"file": "a.jar"
},
{
"coord": "b",
"dependencies": [],
"file": "b.jar"
},
{
"coord": "c",
"dependencies": [],
"file": "c.jar"
},
{
"coord": "a:sources",
"dependencies": ["b", "c"],
"file": "a-sources.jar"
},
]
}
Should return:
{
M2Coordinate("a", ...): ResolvedJar(classifier='', path/cache_path="a.jar"),
M2Coordinate("a", ..., classifier="sources"): ResolvedJar(classifier='sources', path/cache_path="a-sources.jar"),
M2Coordinate("b", ...): ResolvedJar(classifier='', path/cache_path="b.jar"),
M2Coordinate("c", ...): ResolvedJar(classifier='', path/cache_path="c.jar"),
}
:param result: coursier json output
:param coursier_cache_path: coursier cache location
:param pants_jar_path_base: location under pants workdir to store the hardlink to the coursier cache
:return: a map from maven coordinate to a resolved jar.
"""
coord_to_resolved_jars = dict()
for dep in result['dependencies']:
coord = dep['coord']
jar_path = dep.get('file', None)
if not jar_path:
# NB: Not all coordinates will have associated files.
# This is fine. Some coordinates will just have dependencies.
continue
if not os.path.exists(jar_path):
raise CoursierResultNotFound(
"Jar path not found: {}".format(jar_path))
pants_path = cls._get_path_to_jar(coursier_cache_path,
pants_jar_path_base, jar_path)
if not os.path.exists(pants_path):
safe_mkdir(os.path.dirname(pants_path))
safe_hardlink_or_copy(jar_path, pants_path)
coord = cls.to_m2_coord(coord)
resolved_jar = ResolvedJar(coord,
cache_path=jar_path,
pants_path=pants_path)
coord_to_resolved_jars[coord] = resolved_jar
return coord_to_resolved_jars
def _map_coord_to_resolved_jars(cls, result, coursier_cache_path, pants_jar_path_base):
"""
Map resolved files to each org:name:version
Example:
{
"conflict_resolution": {},
"dependencies": [
{
"coord": "a",
"dependencies": ["b", "c"],
"file": "a.jar"
},
{
"coord": "b",
"dependencies": [],
"file": "b.jar"
},
{
"coord": "c",
"dependencies": [],
"file": "c.jar"
},
{
"coord": "a:sources",
"dependencies": ["b", "c"],
"file": "a-sources.jar"
},
]
}
Should return:
{
M2Coordinate("a", ...): ResolvedJar(classifier='', path/cache_path="a.jar"),
M2Coordinate("a", ..., classifier="sources"): ResolvedJar(classifier='sources', path/cache_path="a-sources.jar"),
M2Coordinate("b", ...): ResolvedJar(classifier='', path/cache_path="b.jar"),
M2Coordinate("c", ...): ResolvedJar(classifier='', path/cache_path="c.jar"),
}
:param result: coursier json output
:param coursier_cache_path: coursier cache location
:param pants_jar_path_base: location under pants workdir to store the hardlink to the coursier cache
:return: a map from maven coordinate to a resolved jar.
"""
coord_to_resolved_jars = dict()
for dep in result['dependencies']:
coord = dep['coord']
jar_path = dep.get('file', None)
if not jar_path:
# NB: Not all coordinates will have associated files.
# This is fine. Some coordinates will just have dependencies.
continue
if not os.path.exists(jar_path):
raise CoursierResultNotFound("Jar path not found: {}".format(jar_path))
pants_path = cls._get_path_to_jar(coursier_cache_path, pants_jar_path_base, jar_path)
if not os.path.exists(pants_path):
safe_mkdir(os.path.dirname(pants_path))
safe_hardlink_or_copy(jar_path, pants_path)
coord = cls.to_m2_coord(coord)
resolved_jar = ResolvedJar(coord,
cache_path=jar_path,
pants_path=pants_path)
coord_to_resolved_jars[coord] = resolved_jar
return coord_to_resolved_jars
