def test_java_compile_reads_resource_mapping(self):
# Ensure that if an annotation processor produces a resource-mapping,
# the artifact contains that resource mapping.
with temporary_dir() as cache_dir:
config = {"cache.compile.rsc": {"write_to": [cache_dir]}}
self.assert_success(
self.run_pants(
[
"compile",
"testprojects/src/java/org/pantsbuild/testproject/annotation/main",
],
config=config,
))
base_artifact_dir = self.get_cache_subdir(cache_dir)
artifact_dir = os.path.join(
base_artifact_dir,
"testprojects.src.java.org.pantsbuild.testproject.annotation.main.main",
)
self.assertTrue(os.path.exists(artifact_dir))
artifacts = os.listdir(artifact_dir)
self.assertEqual(len(artifacts), 1)
single_artifact = artifacts[0]
with temporary_dir() as extract_dir:
artifact_path = os.path.join(artifact_dir, single_artifact)
archiver_for_path(artifact_path).extract(
artifact_path, extract_dir)
all_files = set()
for dirpath, dirs, files in safe_walk(extract_dir):
for name in files:
path = os.path.join(dirpath, name)
all_files.add(path)
# Locate the report file on the classpath.
report_file_name = "deprecation_report.txt"
reports = [
f for f in all_files if f.endswith(report_file_name)
]
self.assertEqual(
1,
len(reports),
"Expected exactly one {} file; got: {}".format(
report_file_name, all_files),
)
with open(reports[0], "r") as fp:
annotated_classes = [
line.rstrip() for line in fp.read().splitlines()
]
self.assertEqual(
{
"org.pantsbuild.testproject.annotation.main.Main",
"org.pantsbuild.testproject.annotation.main.Main$TestInnerClass",
},
set(annotated_classes),
)
def resolve_target(self, node_task, target, results_dir, node_paths):
self._copy_sources(target, results_dir)
with temporary_dir() as temp_dir:
archive_file_name = urllib_parse.urlsplit(
target.dependencies_archive_url).path.split('/')[-1]
if not archive_file_name:
raise TaskError(
'Could not determine archive file name for {target} from {url}'
.format(target=target.address.reference(),
url=target.dependencies_archive_url))
download_path = os.path.join(temp_dir, archive_file_name)
logger.info(
'Downloading archive {archive_file_name} from '
'{dependencies_archive_url} to {path}'.format(
archive_file_name=archive_file_name,
dependencies_archive_url=target.dependencies_archive_url,
path=download_path))
try:
with closing(
urllib_request.urlopen(target.dependencies_archive_url)
) as opened_archive_url:
with safe_open(download_path, 'wb') as downloaded_archive:
downloaded_archive.write(opened_archive_url.read())
except (IOError, urllib_error.HTTPError, urllib_error.URLError,
ValueError) as error:
raise TaskError(
'Failed to fetch preinstalled node_modules for {target} from '
'{dependencies_archive_url}: {error}'.format(
target=target.address.reference(),
url=target.dependencies_archive_url,
error=error))
logger.info(
'Fetched archive {archive_file_name} from {dependencies_archive_url} to {path}'
.format(
archive_file_name=archive_file_name,
dependencies_archive_url=target.dependencies_archive_url,
path=download_path))
archiver_for_path(archive_file_name).extract(
download_path, temp_dir)
extracted_node_modules = os.path.join(temp_dir, 'node_modules')
if not os.path.isdir(extracted_node_modules):
raise TaskError(
'Did not find an extracted node_modules directory for {target} '
'inside {dependencies_archive_url}'.format(
target=target.address.reference(),
dependencies_archive_url=target.
dependencies_archive_url))
shutil.move(extracted_node_modules,
os.path.join(results_dir, 'node_modules'))
def resolve_target(self, node_task, target, results_dir, node_paths):
self._copy_sources(target, results_dir)
with temporary_dir() as temp_dir:
archive_file_name = urllib_parse.urlsplit(
target.dependencies_archive_url).path.split('/')[-1]
if not archive_file_name:
raise TaskError(
'Could not determine archive file name for {target} from {url}'
.format(target=target.address.reference(),
url=target.dependencies_archive_url))
download_path = os.path.join(temp_dir, archive_file_name)
logger.info(
'Downloading archive {archive_file_name} from '
'{dependencies_archive_url} to {path}'.format(
archive_file_name=archive_file_name,
dependencies_archive_url=target.dependencies_archive_url,
path=download_path))
try:
Fetcher(get_buildroot()).download(
target.dependencies_archive_url,
listener=Fetcher.ProgressListener(),
path_or_fd=download_path,
timeout_secs=self.get_options().fetch_timeout_secs)
except Fetcher.Error as error:
raise TaskError(
'Failed to fetch preinstalled node_modules for {target} from {url}: {error}'
.format(target=target.address.reference(),
url=target.dependencies_archive_url,
error=error))
logger.info(
'Fetched archive {archive_file_name} from {dependencies_archive_url} to {path}'
.format(
archive_file_name=archive_file_name,
dependencies_archive_url=target.dependencies_archive_url,
path=download_path))
archiver_for_path(archive_file_name).extract(
download_path, temp_dir)
extracted_node_modules = os.path.join(temp_dir, 'node_modules')
if not os.path.isdir(extracted_node_modules):
raise TaskError(
'Did not find an extracted node_modules directory for {target} '
'inside {dependencies_archive_url}'.format(
target=target.address.reference(),
dependencies_archive_url=target.
dependencies_archive_url))
shutil.move(extracted_node_modules,
os.path.join(results_dir, 'node_modules'))
def resolve_target(self, node_task, target, results_dir, node_paths, resolve_locally=False, **kwargs):
if not resolve_locally:
self._copy_sources(target, results_dir)
with temporary_dir() as temp_dir:
archive_file_name = urllib_parse.urlsplit(target.dependencies_archive_url).path.split('/')[-1]
if not archive_file_name:
raise TaskError('Could not determine archive file name for {target} from {url}'
.format(target=target.address.reference(),
url=target.dependencies_archive_url))
download_path = os.path.join(temp_dir, archive_file_name)
node_task.context.log.info(
'Downloading archive {archive_file_name} from '
'{dependencies_archive_url} to {path}'
.format(archive_file_name=archive_file_name,
dependencies_archive_url=target.dependencies_archive_url,
path=download_path))
try:
Fetcher(get_buildroot()).download(target.dependencies_archive_url,
listener=Fetcher.ProgressListener(),
path_or_fd=download_path,
timeout_secs=self.get_options().fetch_timeout_secs)
except Fetcher.Error as error:
raise TaskError('Failed to fetch preinstalled node_modules for {target} from {url}: {error}'
.format(target=target.address.reference(),
url=target.dependencies_archive_url,
error=error))
node_task.context.log.info(
'Fetched archive {archive_file_name} from {dependencies_archive_url} to {path}'
.format(archive_file_name=archive_file_name,
dependencies_archive_url=target.dependencies_archive_url,
path=download_path))
archiver_for_path(archive_file_name).extract(download_path, temp_dir)
extracted_node_modules = os.path.join(temp_dir, 'node_modules')
if not os.path.isdir(extracted_node_modules):
raise TaskError('Did not find an extracted node_modules directory for {target} '
'inside {dependencies_archive_url}'
.format(target=target.address.reference(),
dependencies_archive_url=target.dependencies_archive_url))
# shutil.move doesn't handle directory collision nicely. This is mainly to address
# installing within the source directory for local resolves.
node_modules_path = os.path.join(results_dir, 'node_modules')
safe_rmtree(node_modules_path)
shutil.move(extracted_node_modules, node_modules_path)
def select(argv):
# Parse positional arguments to the script.
args = _create_bootstrap_binary_arg_parser().parse_args(argv[1:])
# Resolve bootstrap options with a fake empty command line.
options_bootstrapper = OptionsBootstrapper.create(args=[argv[0]])
subsystems = (GlobalOptionsRegistrar, BinaryUtil.Factory)
known_scope_infos = reduce(set.union,
(ss.known_scope_infos() for ss in subsystems),
set())
options = options_bootstrapper.get_full_options(known_scope_infos)
# Initialize Subsystems.
Subsystem.set_options(options)
# If the filename provided ends in a known archive extension (such as ".tar.gz"), then we get the
# appropriate Archiver to pass to BinaryUtil.
archiver_for_current_binary = None
filename = args.filename or args.util_name
try:
archiver_for_current_binary = archiver_for_path(filename)
# BinaryRequest requires the `name` field to be provided without an extension, as it appends the
# archiver's extension if one is provided, so we have to remove it here.
filename = filename[:-(len(archiver_for_current_binary.extension) + 1)]
except ValueError:
pass
binary_util = BinaryUtil.Factory.create()
binary_request = BinaryRequest(supportdir='bin/{}'.format(args.util_name),
version=args.version,
name=filename,
platform_dependent=True,
external_url_generator=None,
archiver=archiver_for_current_binary)
return binary_util.select(binary_request)
def fetch(self, import_path, dest, rev=None):
match, url_info = self._matcher(import_path)
pkg = GoRemoteLibrary.remote_package_path(self.root(import_path), import_path)
archive_url = match.expand(url_info.url_format).format(rev=url_info.rev(rev), pkg=pkg)
try:
archiver = archiver_for_path(archive_url)
except ValueError:
raise self.FetchError("Don't know how to unpack archive at url {}".format(archive_url))
with self._fetch(archive_url) as archive:
if url_info.strip_level == 0:
archiver.extract(archive, dest)
else:
with temporary_dir() as scratch:
archiver.extract(archive, scratch)
for dirpath, dirnames, filenames in os.walk(scratch, topdown=True):
if dirpath != scratch:
relpath = os.path.relpath(dirpath, scratch)
relpath_components = relpath.split(os.sep)
if len(relpath_components) == url_info.strip_level and (dirnames or filenames):
for path in dirnames + filenames:
src = os.path.join(dirpath, path)
dst = os.path.join(dest, path)
shutil.move(src, dst)
del dirnames[:] # Stops the walk.
def fetch_archive(self, archive_url, strip_level, dest):
try:
archiver = archiver_for_path(archive_url)
except ValueError:
raise FetchError(
f"Don't know how to unpack archive at url {archive_url}")
with self._fetch(archive_url) as archive:
if strip_level == 0:
archiver.extract(archive, dest)
else:
with temporary_dir() as scratch:
archiver.extract(archive, scratch)
for dirpath, dirnames, filenames in os.walk(scratch,
topdown=True):
if dirpath != scratch:
relpath = os.path.relpath(dirpath, scratch)
relpath_components = relpath.split(os.sep)
if len(relpath_components) == strip_level and (
dirnames or filenames):
for path in dirnames + filenames:
src = os.path.join(dirpath, path)
dst = os.path.join(dest, path)
shutil.move(src, dst)
del dirnames[:] # Stops the walk.
def select(argv):
# Parse positional arguments to the script.
args = _create_bootstrap_binary_arg_parser().parse_args(argv[1:])
# Resolve bootstrap options with a fake empty command line.
options_bootstrapper = OptionsBootstrapper.create(args=[argv[0]])
subsystems = (GlobalOptionsRegistrar, BinaryUtil.Factory)
known_scope_infos = reduce(set.union, (ss.known_scope_infos() for ss in subsystems), set())
options = options_bootstrapper.get_full_options(known_scope_infos)
# Initialize Subsystems.
Subsystem.set_options(options)
# If the filename provided ends in a known archive extension (such as ".tar.gz"), then we get the
# appropriate Archiver to pass to BinaryUtil.
archiver_for_current_binary = None
filename = args.filename or args.util_name
try:
archiver_for_current_binary = archiver_for_path(filename)
# BinaryRequest requires the `name` field to be provided without an extension, as it appends the
# archiver's extension if one is provided, so we have to remove it here.
filename = filename[:-(len(archiver_for_current_binary.extension) + 1)]
except ValueError:
pass
binary_util = BinaryUtil.Factory.create()
binary_request = BinaryRequest(
supportdir='bin/{}'.format(args.util_name),
version=args.version,
name=filename,
platform_dependent=True,
external_url_generator=None,
archiver=archiver_for_current_binary)
return binary_util.select(binary_request)
def test_java_compile_reads_resource_mapping(self):
# Ensure that if an annotation processor produces a resource-mapping,
# the artifact contains that resource mapping.
with temporary_dir() as cache_dir:
config = {'cache.compile.zinc': {'write_to': [cache_dir]}}
self.assert_success(self.run_pants([
'compile',
'testprojects/src/java/org/pantsbuild/testproject/annotation/main',
], config=config))
base_artifact_dir = self.get_cache_subdir(cache_dir)
artifact_dir = os.path.join(
base_artifact_dir,
'testprojects.src.java.org.pantsbuild.testproject.annotation.main.main',
)
self.assertTrue(os.path.exists(artifact_dir))
artifacts = os.listdir(artifact_dir)
self.assertEqual(len(artifacts), 1)
single_artifact = artifacts[0]
with temporary_dir() as extract_dir:
artifact_path = os.path.join(artifact_dir, single_artifact)
archiver_for_path(artifact_path).extract(artifact_path, extract_dir)
all_files = set()
for dirpath, dirs, files in safe_walk(extract_dir):
for name in files:
path = os.path.join(dirpath, name)
all_files.add(path)
# Locate the report file on the classpath.
report_file_name = 'deprecation_report.txt'
reports = [f for f in all_files if f.endswith(report_file_name)]
self.assertEqual(1, len(reports),
'Expected exactly one {} file; got: {}'.format(report_file_name,
all_files))
with open(reports[0], 'r') as fp:
annotated_classes = [line.rstrip() for line in fp.read().splitlines()]
self.assertEqual(
{'org.pantsbuild.testproject.annotation.main.Main',
'org.pantsbuild.testproject.annotation.main.Main$TestInnerClass'},
set(annotated_classes))
def resolve_target(self, node_task, target, results_dir, node_paths):
self._copy_sources(target, results_dir)
with temporary_dir() as temp_dir:
archive_file_name = urllib_parse.urlsplit(target.dependencies_archive_url).path.split('/')[-1]
if not archive_file_name:
raise TaskError('Could not determine archive file name for {target} from {url}'
.format(target=target.address.reference(),
url=target.dependencies_archive_url))
download_path = os.path.join(temp_dir, archive_file_name)
logger.info('Downloading archive {archive_file_name} from '
'{dependencies_archive_url} to {path}'
.format(archive_file_name=archive_file_name,
dependencies_archive_url=target.dependencies_archive_url,
path=download_path))
try:
Fetcher().download(target.dependencies_archive_url,
listener=Fetcher.ProgressListener(),
path_or_fd=download_path,
timeout_secs=self.get_options().fetch_timeout_secs)
except Fetcher.Error as error:
raise TaskError('Failed to fetch preinstalled node_modules for {target} from {url}: {error}'
.format(target=target.address.reference(),
url=target.dependencies_archive_url,
error=error))
logger.info('Fetched archive {archive_file_name} from {dependencies_archive_url} to {path}'
.format(archive_file_name=archive_file_name,
dependencies_archive_url=target.dependencies_archive_url,
path=download_path))
archiver_for_path(archive_file_name).extract(download_path, temp_dir)
extracted_node_modules = os.path.join(temp_dir, 'node_modules')
if not os.path.isdir(extracted_node_modules):
raise TaskError('Did not find an extracted node_modules directory for {target} '
'inside {dependencies_archive_url}'
.format(target=target.address.reference(),
dependencies_archive_url=target.dependencies_archive_url))
shutil.move(extracted_node_modules, os.path.join(results_dir, 'node_modules'))
def _extract_archive(self, archive_path):
with temporary_dir() as temp_dir:
_, extension = os.path.splitext(archive_path)
print (extension)
if extension == '.jar':
extraction_archiver = archiver('zip')
else:
extraction_archiver = archiver_for_path(os.path.basename(archive_path))
extraction_archiver.extract(archive_path, temp_dir)
yield temp_dir
def resolve_target(self, node_task, target, results_dir, node_paths):
self._copy_sources(target, results_dir)
with temporary_dir() as temp_dir:
archive_file_name = urllib_parse.urlsplit(target.dependencies_archive_url).path.split('/')[-1]
if not archive_file_name:
raise TaskError('Could not determine archive file name for {target} from {url}'
.format(target=target.address.reference(),
url=target.dependencies_archive_url))
download_path = os.path.join(temp_dir, archive_file_name)
logger.info('Downloading archive {archive_file_name} from '
'{dependencies_archive_url} to {path}'
.format(archive_file_name=archive_file_name,
dependencies_archive_url=target.dependencies_archive_url,
path=download_path))
try:
with closing(urllib_request.urlopen(target.dependencies_archive_url)) as opened_archive_url:
with safe_open(download_path, 'wb') as downloaded_archive:
downloaded_archive.write(opened_archive_url.read())
except (IOError, urllib_error.HTTPError, urllib_error.URLError, ValueError) as error:
raise TaskError('Failed to fetch preinstalled node_modules for {target} from '
'{dependencies_archive_url}: {error}'
.format(target=target.address.reference(),
url=target.dependencies_archive_url, error=error))
logger.info('Fetched archive {archive_file_name} from {dependencies_archive_url} to {path}'
.format(archive_file_name=archive_file_name,
dependencies_archive_url=target.dependencies_archive_url,
path=download_path))
archiver_for_path(archive_file_name).extract(download_path, temp_dir)
extracted_node_modules = os.path.join(temp_dir, 'node_modules')
if not os.path.isdir(extracted_node_modules):
raise TaskError('Did not find an extracted node_modules directory for {target} '
'inside {dependencies_archive_url}'
.format(target=target.address.reference(),
dependencies_archive_url=target.dependencies_archive_url))
shutil.move(extracted_node_modules, os.path.join(results_dir, 'node_modules'))
def _extract_archive(self, archive_path):
with temporary_dir() as temp_dir:
_, extension = os.path.splitext(archive_path)
print(extension)
if extension == '.jar':
extraction_archiver = create_archiver('zip')
else:
extraction_archiver = archiver_for_path(os.path.basename(archive_path))
extraction_archiver.extract(archive_path, temp_dir)
yield temp_dir
def _construct_path(self, context=None):
fetched = self.select(context)
if not self._extract:
return fetched
unpacked_dir = os.path.dirname(fetched)
outdir = os.path.join(unpacked_dir, 'unpacked')
if not os.path.exists(outdir):
with temporary_dir(root_dir=unpacked_dir) as tmp_root:
# This is an upstream lever that pattern matches the filepath to an archive type.
archiver = archiver_for_path(fetched)
archiver.extract(fetched, tmp_root)
os.rename(tmp_root, outdir)
return os.path.join(outdir)
def _construct_path(self, context=None):
fetched = self.select(context)
if not self._extract:
return fetched
unpacked_dir = os.path.dirname(fetched)
outdir = os.path.join(unpacked_dir, 'unpacked')
if not os.path.exists(outdir):
with temporary_dir(root_dir=unpacked_dir) as tmp_root:
# This is an upstream lever that pattern matches the filepath to an archive type.
archiver = archiver_for_path(fetched)
archiver.extract(fetched, tmp_root)
os.rename(tmp_root, outdir)
return os.path.join(outdir)
def _construct_path(self):
fetched = self.remote_source_util.select_binary(self._relpath, self.version, self._filename)
if not self._extract:
return fetched
unpacked_dir = os.path.dirname(fetched)
outdir = os.path.join(unpacked_dir, 'unpacked')
if not os.path.exists(outdir):
with temporary_dir(root_dir=unpacked_dir) as tmp_root:
# This is an upstream lever that pattern matches the filepath to an archive type.
archiver = archiver_for_path(fetched)
archiver.extract(fetched, tmp_root)
os.rename(tmp_root, outdir)
return os.path.join(outdir)
def fetch_archive(self, archive_url, strip_level, dest):
try:
archiver = archiver_for_path(archive_url)
except ValueError:
raise FetchError("Don't know how to unpack archive at url {}".format(archive_url))
with self._fetch(archive_url) as archive:
if strip_level == 0:
archiver.extract(archive, dest)
else:
with temporary_dir() as scratch:
archiver.extract(archive, scratch)
for dirpath, dirnames, filenames in os.walk(scratch, topdown=True):
if dirpath != scratch:
relpath = os.path.relpath(dirpath, scratch)
relpath_components = relpath.split(os.sep)
if len(relpath_components) == strip_level and (dirnames or filenames):
for path in dirnames + filenames:
src = os.path.join(dirpath, path)
dst = os.path.join(dest, path)
shutil.move(src, dst)
del dirnames[:] # Stops the walk.
def resolve_target(self,
node_task,
target,
results_dir,
node_paths,
resolve_locally=False,
**kwargs):
if not resolve_locally:
self._copy_sources(target, results_dir)
with temporary_dir() as temp_dir:
archive_file_name = urllib.parse.urlsplit(
target.dependencies_archive_url).path.split("/")[-1]
if not archive_file_name:
raise TaskError(
"Could not determine archive file name for {target} from {url}"
.format(target=target.address.reference(),
url=target.dependencies_archive_url))
download_path = os.path.join(temp_dir, archive_file_name)
node_task.context.log.info(
"Downloading archive {archive_file_name} from "
"{dependencies_archive_url} to {path}".format(
archive_file_name=archive_file_name,
dependencies_archive_url=target.dependencies_archive_url,
path=download_path,
))
try:
Fetcher(get_buildroot()).download(
target.dependencies_archive_url,
listener=Fetcher.ProgressListener(),
path_or_fd=download_path,
timeout_secs=self.get_options().fetch_timeout_secs,
)
except Fetcher.Error as error:
raise TaskError(
"Failed to fetch preinstalled node_modules for {target} from {url}: {error}"
.format(
target=target.address.reference(),
url=target.dependencies_archive_url,
error=error,
))
node_task.context.log.info(
"Fetched archive {archive_file_name} from {dependencies_archive_url} to {path}"
.format(
archive_file_name=archive_file_name,
dependencies_archive_url=target.dependencies_archive_url,
path=download_path,
))
archiver_for_path(archive_file_name).extract(
download_path, temp_dir)
extracted_node_modules = os.path.join(temp_dir, "node_modules")
if not os.path.isdir(extracted_node_modules):
raise TaskError(
"Did not find an extracted node_modules directory for {target} "
"inside {dependencies_archive_url}".format(
target=target.address.reference(),
dependencies_archive_url=target.
dependencies_archive_url,
))
# shutil.move doesn't handle directory collision nicely. This is mainly to address
# installing within the source directory for local resolves.
node_modules_path = os.path.join(results_dir, "node_modules")
safe_rmtree(node_modules_path)
shutil.move(extracted_node_modules, node_modules_path)
