def symlink_cachepath(ivy_home, inpath, symlink_dir, outpath):
"""Symlinks all paths listed in inpath that are under ivy_home into symlink_dir.
Preserves all other paths. Writes the resulting paths to outpath.
Returns a map of path -> symlink to that path.
"""
safe_mkdir(symlink_dir)
with safe_open(inpath, 'r') as infile:
paths = filter(None, infile.read().strip().split(os.pathsep))
new_paths = []
for path in paths:
if not path.startswith(ivy_home):
new_paths.append(path)
continue
symlink = os.path.join(symlink_dir, os.path.relpath(path, ivy_home))
try:
os.makedirs(os.path.dirname(symlink))
except OSError as e:
if e.errno != errno.EEXIST:
raise
# Note: The try blocks cannot be combined. It may be that the dir exists but the link doesn't.
try:
os.symlink(path, symlink)
except OSError as e:
# We don't delete and recreate the symlink, as this may break concurrently executing code.
if e.errno != errno.EEXIST:
raise
new_paths.append(symlink)
with safe_open(outpath, 'w') as outfile:
outfile.write(':'.join(new_paths))
symlink_map = dict(zip(paths, new_paths))
return symlink_map
def execute(self, targets):
java_targets = filter(JavaCompile._is_java, targets)
if java_targets:
with self.context.state('classpath', []) as cp:
for conf in self._confs:
cp.insert(0, (conf, self._output_dir))
with self.changed(java_targets, invalidate_dependants=True) as changed:
bases, sources_by_target, processors, fingerprint = self.calculate_sources(changed)
if sources_by_target:
classpath = [jar for conf, jar in cp if conf in self._confs]
result = self.compile(classpath, bases, sources_by_target, fingerprint)
if result != 0:
raise TaskError('%s returned %d' % (self._main, result))
if processors:
if os.path.exists(self._processor_service_info_file):
with safe_open(self._processor_service_info_file, 'r') as f:
for processor in f:
processors.add(processor.strip())
with safe_open(self._processor_service_info_file, 'w') as f:
for processor in processors:
f.write('%s\n' % processor)
if self.context.products.isrequired('classes'):
genmap = self.context.products.get('classes')
classes_by_target = SunCompiler.findclasses(self._output_dir, targets)
for target, classes in classes_by_target.items():
genmap.add(target, self._output_dir, classes)
def _spawn_nailgun_server(self, workunit):
self.context.log.debug("No ng server found, spawning...")
with _safe_open(self._ng_out, "w"):
pass # truncate
pid = os.fork()
if pid != 0:
# In the parent tine - block on ng being up for connections
return self._await_nailgun_server(workunit)
# NOTE: Don't use self.context.log or self.context.new_workunit here.
# They use threadlocal state, which interacts poorly with fork().
os.setsid()
in_fd = open("/dev/null", "w")
out_fd = safe_open(self._ng_out, "w")
err_fd = safe_open(self._ng_err, "w")
args = ["java"]
if self._ng_server_args:
args.extend(self._ng_server_args)
args.append(NailgunTask.PANTS_NG_ARG)
args.append(self._identifier_arg)
ng_classpath = os.pathsep.join(binary_util.profile_classpath(self._nailgun_profile))
args.extend(["-cp", ng_classpath, "com.martiansoftware.nailgun.NGServer", ":0"])
s = " ".join(args)
with binary_util.safe_classpath():
subprocess.Popen(args, stdin=in_fd, stdout=out_fd, stderr=err_fd, close_fds=True, cwd=get_buildroot())
# Prevents finally blocks being executed, unlike sys.exit(). We don't want to execute finally
# blocks because we might, e.g., clean up tempfiles that the parent still needs.
os._exit(0)
def setUpClass(cls):
cls.origin = safe_mkdtemp()
with pushd(cls.origin):
subprocess.check_call(['git', 'init', '--bare'])
cls.gitdir = safe_mkdtemp()
cls.worktree = safe_mkdtemp()
cls.readme_file = os.path.join(cls.worktree, 'README')
with environment_as(GIT_DIR=cls.gitdir, GIT_WORK_TREE=cls.worktree):
cls.init_repo('depot', cls.origin)
touch(cls.readme_file)
subprocess.check_call(['git', 'add', 'README'])
subprocess.check_call(['git', 'commit', '-am', 'initial commit with decode -> \x81b'])
subprocess.check_call(['git', 'tag', 'first'])
subprocess.check_call(['git', 'push', '--tags', 'depot', 'master'])
subprocess.check_call(['git', 'branch', '--set-upstream', 'master', 'depot/master'])
with safe_open(cls.readme_file, 'w') as readme:
readme.write('Hello World.')
subprocess.check_call(['git', 'commit', '-am', 'Update README.'])
cls.clone2 = safe_mkdtemp()
with pushd(cls.clone2):
cls.init_repo('origin', cls.origin)
subprocess.check_call(['git', 'pull', '--tags', 'origin', 'master:master'])
with safe_open(os.path.realpath('README'), 'a') as readme:
readme.write('--')
subprocess.check_call(['git', 'commit', '-am', 'Update README 2.'])
subprocess.check_call(['git', 'push', '--tags', 'origin', 'master'])
cls.git = Git(gitdir=cls.gitdir, worktree=cls.worktree)
def process(self, outdir, base, source, standalone, url_builder, get_config, css=None):
def parse_url(spec):
match = MarkdownToHtml.PANTS_LINK.match(spec)
if match:
page = Target.get(Address.parse(get_buildroot(), match.group(1)))
if not page:
raise TaskError('Invalid link %s' % match.group(1))
alias, url = url_builder(page, config=get_config(page))
return alias, url
else:
return spec, spec
def build_url(label):
components = label.split('|', 1)
if len(components) == 1:
return parse_url(label.strip())
else:
alias, link = components
_, url = parse_url(link.strip())
return alias, url
wikilinks = WikilinksExtension(build_url)
path, ext = os.path.splitext(source)
with safe_open(os.path.join(outdir, path + '.html'), 'w') as output:
with open(os.path.join(get_buildroot(), base, source), 'r') as input:
md_html = markdown.markdown(
input.read(),
extensions=['codehilite', 'extra', 'toc', wikilinks]
)
if standalone:
if css:
css_relpath = os.path.relpath(css, outdir)
out_relpath = os.path.dirname(source)
link_relpath = os.path.relpath(css_relpath, out_relpath)
css = '<link rel="stylesheet" type="text/css" href="%s"/>' % link_relpath
html = textwrap.dedent('''
<html>
<head>
%s
</head>
<body>
<!-- generated by pants! -->
%s
</body>
</html>
''').strip() % (css or '', md_html)
output.write(html)
else:
if css:
with safe_open(css) as fd:
output.write(textwrap.dedent('''
<style type="text/css">
%s
</style>
''').strip() % fd.read())
output.write('\n')
output.write(md_html)
return output.name
def _prepare_fork(self):
user, current_user = self._getpwuid()
uid, gid = user.pw_uid, user.pw_gid
self._fork_time = self._platform.clock().time()
self._setup_ckpt()
self._stdout = safe_open(self._pathspec.with_filename('stdout').getpath('process_logdir'), "w")
self._stderr = safe_open(self._pathspec.with_filename('stderr').getpath('process_logdir'), "w")
os.chown(self._stdout.name, user.pw_uid, user.pw_gid)
os.chown(self._stderr.name, user.pw_uid, user.pw_gid)
def _prepare_fork(self):
user, current_user = self._getpwuid()
if self._user:
if user != current_user and os.geteuid() != 0:
raise self.PermissionError('Must be root to run processes as other users!')
uid, gid = user.pw_uid, user.pw_gid
self._fork_time = self._platform.clock().time()
self._setup_ckpt()
self._stdout = safe_open(self._pathspec.with_filename('stdout').getpath('process_logdir'), "a")
self._stderr = safe_open(self._pathspec.with_filename('stderr').getpath('process_logdir'), "a")
os.chown(self._stdout.name, user.pw_uid, user.pw_gid)
os.chown(self._stderr.name, user.pw_uid, user.pw_gid)
def stage_artifacts(target, jar, version, changelog, confs=None, synth_target=None):
def artifact_path(name=None, suffix='', extension='jar', artifact_ext=''):
return os.path.join(self.outdir, jar.org, jar.name + artifact_ext,
'%s%s-%s%s.%s' % (
(name or jar.name),
artifact_ext if name != 'ivy' else '',
version,
suffix,
extension
))
def get_pushdb(target):
return get_db(target)[0]
with safe_open(artifact_path(suffix='-CHANGELOG', extension='txt'), 'w') as changelog_file:
changelog_file.write(changelog)
ivyxml = artifact_path(name='ivy', extension='xml')
IvyWriter(get_pushdb).write(target, ivyxml, confs)
PomWriter(get_pushdb).write(target, artifact_path(extension='pom'))
idl_ivyxml = None
if synth_target:
changelog_path = artifact_path(suffix='-CHANGELOG', extension='txt', artifact_ext='-only')
with safe_open(changelog_path, 'w') as changelog_file:
changelog_file.write(changelog)
idl_ivyxml = artifact_path(name='ivy', extension='xml', artifact_ext='-only')
# use idl publication spec in ivy for idl artifact
IvyWriter(get_pushdb).write(synth_target, idl_ivyxml, ['idl'], synth=True)
PomWriter(get_pushdb).write(synth_target,
artifact_path(extension='pom', artifact_ext='-only'),
synth=True)
def copy(tgt, typename, suffix='', artifact_ext=''):
genmap = self.context.products.get(typename)
mapping = genmap.get(tgt)
if not mapping:
print('no mapping for %s' % tgt)
else:
for basedir, jars in mapping.items():
for artifact in jars:
path = artifact_path(suffix=suffix, artifact_ext=artifact_ext)
shutil.copy(os.path.join(basedir, artifact), path)
copy(target, typename='jars')
copy(target, typename='source_jars', suffix='-sources')
if (synth_target):
copy(synth_target, typename='idl_jars', suffix='-idl', artifact_ext='-only')
if is_java(target):
copy(target, typename='javadoc_jars', suffix='-javadoc')
return ivyxml, idl_ivyxml
def execute(self):
pages = []
targets = self.context.targets()
for target in targets:
if isinstance(target, Page):
for wiki_artifact in target.payload.provides:
pages.append((target, wiki_artifact))
urls = list()
genmap = self.context.products.get('wiki_html')
for page, wiki_artifact in pages:
html_info = genmap.get((wiki_artifact, page))
if len(html_info) > 1:
raise TaskError('Unexpected resources for %s: %s' % (page, html_info))
basedir, htmls = html_info.items()[0]
if len(htmls) != 1:
raise TaskError('Unexpected resources for %s: %s' % (page, htmls))
with safe_open(os.path.join(basedir, htmls[0])) as contents:
url = self.publish_page(
page.address,
wiki_artifact.config['space'],
wiki_artifact.config['title'],
contents.read(),
# Default to none if not present in the hash.
parent=wiki_artifact.config.get('parent')
)
if url:
urls.append(url)
self.context.log.info('Published %s to %s' % (page, url))
if self.open and urls:
binary_util.ui_open(*urls)
def execute_single_compilation(self, versioned_targets, cp):
# TODO: Use the artifact cache.
depfile = self.create_depfile_path(versioned_targets.targets)
if not versioned_targets.valid:
self.merge_depfile(versioned_targets) # Get what we can from previous builds.
self.context.log.info('Compiling targets %s' % str(versioned_targets.targets))
sources_by_target, processors, fingerprint = self.calculate_sources(versioned_targets.targets)
if sources_by_target:
sources = reduce(lambda all, sources: all.union(sources), sources_by_target.values())
if not sources:
self.context.log.warn('Skipping java compile for targets with no sources:\n %s' %
'\n '.join(str(t) for t in sources_by_target.keys()))
else:
classpath = [jar for conf, jar in cp if conf in self._confs]
result = self.compile(classpath, sources, fingerprint, depfile)
if result != 0:
default_message = 'Unexpected error - %s returned %d' % (_JMAKE_MAIN, result)
raise TaskError(_JMAKE_ERROR_CODES.get(result, default_message))
if processors and not self.dry_run:
# Produce a monolithic apt processor service info file for further compilation rounds
# and the unit test classpath.
processor_info_file = os.path.join(self._classes_dir, _PROCESSOR_INFO_FILE)
if os.path.exists(processor_info_file):
with safe_open(processor_info_file, 'r') as f:
for processor in f:
processors.add(processor.strip())
self.write_processor_info(processor_info_file, processors)
self.post_process(versioned_targets)
def _get_nailgun_endpoint(self):
if os.path.exists(self._pidfile):
with _safe_open(self._pidfile, "r") as pidfile:
contents = pidfile.read()
def invalid_pidfile():
log.warn("Invalid ng pidfile %s contained: %s" % (self._pidfile, contents))
return None
endpoint = contents.split(":")
if len(endpoint) != 2:
return invalid_pidfile()
pid, port = endpoint
try:
return int(pid.strip()), int(port.strip())
except ValueError:
return invalid_pidfile()
elif NailgunTask._find:
pid_port = NailgunTask._find(self._pidfile)
if pid_port:
self.context.log.info("found ng server @ pid:%d port:%d" % pid_port)
with safe_open(self._pidfile, "w") as pidfile:
pidfile.write("%d:%d\n" % pid_port)
return pid_port
return None
def write(self, target, path, confs=None):
def as_jar(internal_target):
jar, _, _, _ = self.get_db(internal_target).as_jar_with_version(internal_target)
return jar
# TODO(John Sirois): a dict is used here to de-dup codegen targets which have both the original
# codegen target - say java_thrift_library - and the synthetic generated target (java_library)
# Consider reworking codegen tasks to add removal of the original codegen targets when rewriting
# the graph
dependencies = OrderedDict()
internal_codegen = {}
for dep in target_internal_dependencies(target):
jar = as_jar(dep)
dependencies[(jar.org, jar.name)] = self.internaldep(jar, dep)
if dep.is_codegen:
internal_codegen[jar.name] = jar.name
for jar in target.jar_dependencies:
if jar.rev:
dependencies[(jar.org, jar.name)] = self.jardep(jar)
target_jar = self.internaldep(as_jar(target)).extend(dependencies=dependencies.values())
template_kwargs = self.templateargs(target_jar, confs)
with safe_open(path, 'w') as output:
template = pkgutil.get_data(__name__, self.template_relpath)
Generator(template, **template_kwargs).write(output)
def from_file(filename, **kw):
try:
with safe_open(filename) as fp:
task = Task.json_load(fp)
return ThermosTaskWrapper(task, **kw)
except Exception as e:
return None
def _await_nailgun_server(self, stdout, stderr):
nailgun_timeout_seconds = 5
max_socket_connect_attempts = 10
nailgun = None
port_parse_start = time.time()
with safe_open(self._ng_out, 'r') as ng_out:
while not nailgun:
started = ng_out.readline()
if started:
port = self._parse_nailgun_port(started)
nailgun = self._create_ngclient(port, stdout, stderr)
log.debug('Detected ng server up on port %d' % port)
elif time.time() - port_parse_start > nailgun_timeout_seconds:
raise NailgunClient.NailgunError('Failed to read ng output after'
' %s seconds' % nailgun_timeout_seconds)
attempt = 0
while nailgun:
sock = nailgun.try_connect()
if sock:
sock.close()
endpoint = self._get_nailgun_endpoint()
if endpoint:
log.debug('Connected to ng server with fingerprint %s pid: %d @ port: %d' % endpoint)
else:
raise NailgunClient.NailgunError('Failed to connect to ng server.')
return nailgun
elif attempt > max_socket_connect_attempts:
raise nailgun.NailgunError('Failed to connect to ng output after %d connect attempts'
% max_socket_connect_attempts)
attempt += 1
log.debug('Failed to connect on attempt %d' % attempt)
time.sleep(0.1)
def generate_reports():
args = [
'report',
'-in', self.coverage_metadata_file,
'-in', self.coverage_file,
'-exit'
]
source_bases = set(t.target_base for t in targets)
for source_base in source_bases:
args.extend(['-sp', source_base])
sorting = ['-Dreport.sort', '+name,+class,+method,+block']
if self.coverage_report_console:
args.extend(['-r', 'txt',
'-Dreport.txt.out.file=%s' % self.coverage_console_file] + sorting)
if self.coverage_report_xml:
args.extend(['-r', 'xml','-Dreport.xml.out.file=%s' % self.coverage_xml_file])
if self.coverage_report_html:
args.extend(['-r', 'html',
'-Dreport.html.out.file=%s' % self.coverage_html_file,
'-Dreport.out.encoding=UTF-8'] + sorting)
result = runjava(
classpath=emma_classpath,
main='emma',
args=args
)
if result != 0:
raise TaskError('Failed to emma generate code coverage reports: %d' % result)
if self.coverage_report_console:
with safe_open(self.coverage_console_file) as console_report:
sys.stdout.write(console_report.read())
if self.coverage_report_html_open:
binary_utils.open(self.coverage_html_file)
def execute(self, targets):
try:
wiki = Confluence.login(self.url)
except ConfluenceError as e:
raise TaskError("Failed to login to confluence: %s" % e)
urls = list()
genmap = self.context.products.get("markdown_html")
for page in filter(lambda t: isinstance(t, Page), targets):
wikiconfig = page.wiki_config(self.wiki())
if wikiconfig:
html_info = genmap.get((self.wiki(), page))
if len(html_info) > 1:
raise TaskError("Unexpected resources for %s: %s" % (page, html_info))
basedir, htmls = html_info.items()[0]
if len(htmls) != 1:
raise TaskError("Unexpected resources for %s: %s" % (page, htmls))
with safe_open(os.path.join(basedir, htmls[0])) as contents:
url = self.publish_page(
wiki, wikiconfig["space"], wikiconfig["title"], contents.read(), parent=wikiconfig.get("parent")
)
if url:
urls.append(url)
self.context.log.info("Published %s to %s" % (page, url))
if self.open and urls:
binary_utils.open(*urls)
def select_binary(base_path, version, name, config=None):
"""Selects a binary matching the current os and architecture.
:raises: :class:`pants.binary_util.BinaryUtil.BinaryNotFound` if no binary of the given version
and name could be found.
"""
# TODO(John Sirois): finish doc of the path structure expexcted under base_path
config = config or Config.load()
bootstrap_dir = config.getdefault('pants_bootstrapdir')
binary_path = select_binary_base_path(base_path, version, name)
bootstrapped_binary_path = os.path.join(bootstrap_dir, binary_path)
if not os.path.exists(bootstrapped_binary_path):
downloadpath = bootstrapped_binary_path + '~'
try:
with select_binary_stream(base_path, version, name, config) as stream:
with safe_open(downloadpath, 'wb') as bootstrapped_binary:
bootstrapped_binary.write(stream())
os.rename(downloadpath, bootstrapped_binary_path)
chmod_plus_x(bootstrapped_binary_path)
finally:
safe_delete(downloadpath)
log.debug('Selected {binary} binary bootstrapped to: {path}'
.format(binary=name, path=bootstrapped_binary_path))
return bootstrapped_binary_path
def execute(self, targets):
java_targets = filter(_is_java, targets)
if java_targets:
safe_mkdir(self._classes_dir)
safe_mkdir(self._depfile_dir)
egroups = self.context.products.get_data('exclusives_groups')
group_id = egroups.get_group_key_for_target(java_targets[0])
for conf in self._confs:
egroups.update_compatible_classpaths(group_id, [(conf, self._resources_dir)])
egroups.update_compatible_classpaths(group_id, [(conf, self._classes_dir)])
with self.invalidated(java_targets, invalidate_dependents=True,
partition_size_hint=self._partition_size_hint) as invalidation_check:
for vt in invalidation_check.invalid_vts_partitioned:
# Compile, using partitions for efficiency.
exclusives_classpath = egroups.get_classpath_for_group(group_id)
self.execute_single_compilation(vt, exclusives_classpath)
if not self.dry_run:
vt.update()
for vt in invalidation_check.all_vts:
depfile = self.create_depfile_path(vt.targets)
if not self.dry_run and os.path.exists(depfile):
# Read in the deps created either just now or by a previous run on these targets.
deps = Dependencies(self._classes_dir)
deps.load(depfile)
self._deps.merge(deps)
if not self.dry_run:
if self.context.products.isrequired('classes'):
genmap = self.context.products.get('classes')
# Map generated classes to the owning targets and sources.
for target, classes_by_source in self._deps.findclasses(java_targets).items():
for source, classes in classes_by_source.items():
genmap.add(source, self._classes_dir, classes)
genmap.add(target, self._classes_dir, classes)
# TODO(John Sirois): Map target.resources in the same way
# 'Map' (rewrite) annotation processor service info files to the owning targets.
for target in java_targets:
if is_apt(target) and target.processors:
basedir = os.path.join(self._resources_dir, Target.maybe_readable_identify([target]))
processor_info_file = os.path.join(basedir, _PROCESSOR_INFO_FILE)
self.write_processor_info(processor_info_file, target.processors)
genmap.add(target, basedir, [_PROCESSOR_INFO_FILE])
# Produce a monolithic apt processor service info file for further compilation rounds
# and the unit test classpath.
all_processors = set()
for target in java_targets:
if is_apt(target) and target.processors:
all_processors.update(target.processors)
processor_info_file = os.path.join(self._classes_dir, _PROCESSOR_INFO_FILE)
if os.path.exists(processor_info_file):
with safe_open(processor_info_file, 'r') as f:
for processor in f:
all_processors.add(processor.strip())
self.write_processor_info(processor_info_file, all_processors)
def execute(self, targets):
pages = []
for target in targets:
if isinstance(target, Page):
wikiconfig = target.wiki_config(self.wiki())
if wikiconfig:
pages.append((target, wikiconfig))
urls = list()
genmap = self.context.products.get('wiki_html')
for page, wikiconfig in pages:
html_info = genmap.get((self.wiki(), page))
if len(html_info) > 1:
raise TaskError('Unexpected resources for %s: %s' % (page, html_info))
basedir, htmls = html_info.items()[0]
if len(htmls) != 1:
raise TaskError('Unexpected resources for %s: %s' % (page, htmls))
with safe_open(os.path.join(basedir, htmls[0])) as contents:
url = self.publish_page(
page.address,
wikiconfig['space'],
wikiconfig['title'],
contents.read(),
parent=wikiconfig.get('parent')
)
if url:
urls.append(url)
self.context.log.info('Published %s to %s' % (page, url))
if self.open and urls:
binary_util.ui_open(*urls)
def checkstyle(self, sources, targets):
egroups = self.context.products.get_data('exclusives_groups')
etag = egroups.get_group_key_for_target(targets[0])
classpath = self._jvm_tool_bootstrapper.get_jvm_tool_classpath(self._checkstyle_bootstrap_key)
cp = egroups.get_classpath_for_group(etag)
classpath.extend(jar for conf, jar in cp if conf in self._confs)
args = [
'-c', self._configuration_file,
'-f', 'plain'
]
if self._properties:
properties_file = os.path.join(self._work_dir, 'checkstyle.properties')
with safe_open(properties_file, 'w') as pf:
for k, v in self._properties.items():
pf.write('%s=%s\n' % (k, v))
args.extend(['-p', properties_file])
# We've hit known cases of checkstyle command lines being too long for the system so we guard
# with Xargs since checkstyle does not accept, for example, @argfile style arguments.
def call(xargs):
return self.runjava(classpath=classpath, main=CHECKSTYLE_MAIN,
args=args + xargs, workunit_name='checkstyle')
checks = Xargs(call)
return checks.execute(sources)
def from_file(cls, filename, **kw):
try:
with safe_open(filename) as fp:
task = Task.json_load(fp)
return cls(task, **kw)
except Exception:
return None
def stage_artifacts(target, jar, version, changelog, confs=None):
def artifact_path(name=None, suffix='', extension='jar'):
return os.path.join(self.outdir, jar.org, jar.name,
'%s-%s%s.%s' % ((name or jar.name), version, suffix, extension))
with safe_open(artifact_path(suffix='-CHANGELOG', extension='txt'), 'w') as changelog_file:
changelog_file.write(changelog)
def get_pushdb(target):
return get_db(target)[0]
PomWriter(get_pushdb).write(target, artifact_path(extension='pom'))
ivyxml = artifact_path(name='ivy', extension='xml')
IvyWriter(get_pushdb).write(target, ivyxml, confs)
def copy(typename, suffix=''):
genmap = self.context.products.get(typename)
for basedir, jars in genmap.get(target).items():
for artifact in jars:
shutil.copy(os.path.join(basedir, artifact), artifact_path(suffix=suffix))
copy('jars')
if is_java(target):
copy('javadoc_jars', '-javadoc')
copy('source_jars', '-sources')
return ivyxml
def _get_file(self, name):
if self._mode == LoggerMode.STANDARD:
return safe_open(self._get_log_path(name), mode='a')
if self._mode == LoggerMode.ROTATE:
log_size = int(self._rotate_log_size.as_(Data.BYTES))
return RotatingFileHandler(self._get_log_path(name),
log_size,
self._rotate_log_backups)
def execute(self, targets):
if not self._flatten and len(targets) > 1:
topologically_sorted_targets = filter(JavaCompile._is_java, reversed(InternalTarget.sort_targets(targets)))
for target in topologically_sorted_targets:
self.execute([target])
return
self.context.log.info('Compiling targets %s' % str(targets))
java_targets = filter(JavaCompile._is_java, targets)
if java_targets:
with self.context.state('classpath', []) as cp:
for conf in self._confs:
cp.insert(0, (conf, self._resources_dir))
cp.insert(0, (conf, self._classes_dir))
with self.changed(java_targets, invalidate_dependants=True) as changed:
sources_by_target, processors, fingerprint = self.calculate_sources(changed)
if sources_by_target:
sources = reduce(lambda all, sources: all.union(sources), sources_by_target.values())
if not sources:
self.context.log.warn('Skipping java compile for targets with no sources:\n %s' %
'\n '.join(str(t) for t in sources_by_target.keys()))
else:
classpath = [jar for conf, jar in cp if conf in self._confs]
result = self.compile(classpath, sources, fingerprint)
if result != 0:
default_message = 'Unexpected error - %s returned %d' % (_JMAKE_MAIN, result)
raise TaskError(_JMAKE_ERROR_CODES.get(result, default_message))
if processors:
# Produce a monolithic apt processor service info file for further compilation rounds
# and the unit test classpath.
processor_info_file = os.path.join(self._classes_dir, _PROCESSOR_INFO_FILE)
if os.path.exists(processor_info_file):
with safe_open(processor_info_file, 'r') as f:
for processor in f:
processors.add(processor.strip())
self.write_processor_info(processor_info_file, processors)
if self.context.products.isrequired('classes'):
genmap = self.context.products.get('classes')
# Map generated classes to the owning targets and sources.
dependencies = Dependencies(self._classes_dir, self._dependencies_file)
for target, classes_by_source in dependencies.findclasses(targets).items():
for source, classes in classes_by_source.items():
genmap.add(source, self._classes_dir, classes)
genmap.add(target, self._classes_dir, classes)
# TODO(John Sirois): Map target.resources in the same way
# 'Map' (rewrite) annotation processor service info files to the owning targets.
for target in targets:
if is_apt(target) and target.processors:
basedir = os.path.join(self._resources_dir, target.id)
processor_info_file = os.path.join(basedir, _PROCESSOR_INFO_FILE)
self.write_processor_info(processor_info_file, target.processors)
genmap.add(target, basedir, [_PROCESSOR_INFO_FILE])
def download_fp(_path_or_fd):
if _path_or_fd and not isinstance(_path_or_fd, Compatibility.string):
yield _path_or_fd, _path_or_fd.name
else:
if not _path_or_fd:
fd, _path_or_fd = tempfile.mkstemp()
os.close(fd)
with safe_open(_path_or_fd, 'w') as fp:
yield fp, _path_or_fd
def _do_in_context(self, work, path=None):
# TODO(John Sirois): eliminate the need for all the gymanstics needed to synthesize a target
build_dir = path or self.config.getdefault('pants_workdir')
build_path = os.path.join(build_dir, 'BUILD.pants')
if not os.path.exists(build_path):
with safe_open(build_path, 'w') as build_file:
build_file.write('# dummy BUILD file generated by pants\n')
return ParseContext(BuildFile(get_buildroot(), build_path)).do_in_context(work)
def create_file(cls, relpath, contents='', mode='w'):
"""Writes to a file under the buildroot.
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
with safe_open(os.path.join(cls.build_root, relpath), mode=mode) as fp:
fp.write(contents)
def write(self, target, path):
dependencies = [self.internaldep(dep) for dep in target.internal_dependencies]
dependencies.extend(PomWriter.jardep(dep) for dep in target.jar_dependencies if dep.rev)
target_jar = self.internaldep(target).extend(dependencies=dependencies)
with safe_open(path, 'w') as output:
generator = Generator(pkgutil.get_data(__name__, os.path.join('jar_publish', 'pom.mk')),
artifact=target_jar)
generator.write(output)
def write_gen_file_map_for_target(self, gen_file_map, target, outdir):
def calc_srcs(target):
_, srcs = calculate_compile_sources([target], self.is_gentarget)
return srcs
with safe_open(self.gen_file_map_path_for_target(target, outdir), 'w') as f:
for src in sorted(calc_srcs(target)):
clss = gen_file_map[src]
for cls in sorted(clss):
print('%s -> %s' % (src, os.path.join(outdir, cls)), file=f)
def write_plugin_info(self, target):
basedir = os.path.join(self._resources_dir, target.id)
with safe_open(os.path.join(basedir, _PLUGIN_INFO_FILE), 'w') as f:
f.write(textwrap.dedent('''
<plugin>
<name>%s</name>
<classname>%s</classname>
</plugin>
''' % (target.plugin, target.classname)).strip())
return basedir
def safe_args(args,
max_args=None,
config=None,
argfile=None,
delimiter='\n',
quoter=None,
delete=True):
"""
Yields args if there are less than a limit otherwise writes args to an argfile and yields an
argument list with one argument formed from the path of the argfile.
:args The args to work with.
:max_args The maximum number of args to let though without writing an argfile. If not specified
then the maximum will be loaded from config.
:config Used to lookup the configured maximum number of args that can be passed to a subprocess;
defaults to the default config and looks for key 'max_subprocess_args' in the DEFAULTS.
:argfile The file to write args to when there are too many; defaults to a temporary file.
:delimiter The delimiter to insert between args written to the argfile, defaults to '\n'
:quoter A function that can take the argfile path and return a single argument value;
defaults to:
<code>lambda f: '@' + f<code>
:delete If True deletes any arg files created upon exit from this context; defaults to True.
"""
max_args = max_args or (config or Config.load()).getdefault(
'max_subprocess_args', int, 10)
if len(args) > max_args:
def create_argfile(fp):
fp.write(delimiter.join(args))
fp.close()
return [quoter(fp.name) if quoter else '@%s' % fp.name]
if argfile:
try:
with safe_open(argfile, 'w') as fp:
yield create_argfile(fp)
finally:
if delete and os.path.exists(argfile):
os.unlink(argfile)
else:
with temporary_file(cleanup=delete) as fp:
yield create_argfile(fp)
else:
yield args
def run_tests(classpath, main, jvmargs=None):
if self.only_write_cmd_line is None:
with safe_args(tests) as all_tests:
result = runjava(
jvmargs=(jvmargs or []) + self.java_args,
classpath=classpath,
main=main,
args=self.flags + all_tests,
)
else:
with safe_open(self.only_write_cmd_line, 'w') as fd:
result = runjava(jvmargs=(jvmargs or []) +
self.java_args,
classpath=classpath,
main=main,
args=self.flags + tests,
only_write_cmd_line_to=fd)
if result != 0:
raise TaskError()
def _get_handler(self, name):
"""
Constructs correct handler or file object based on the provided configuration.
"""
# On no destination write logs to /dev/null
if self._destination == LoggerDestination.NONE:
return StreamHandler(safe_open(os.devnull, 'w'))
# Streamed logs to predefined outputs
if self._destination == LoggerDestination.CONSOLE:
return sys.stdout if name == self.STDOUT else sys.stderr
# Streaming AND file logs are required
if self._destination == LoggerDestination.BOTH:
return TeeHandler(self._get_stream(name), self._get_file(name))
# File only logs are required
return self._get_file(name)
def maybe_locally_cache(dist, cache_dir):
from pkg_resources import PathMetadata, Distribution
from twitter.common.dirutil import safe_rmtree, safe_open, safe_mkdir
egg_name = os.path.join(cache_dir, dist.egg_name() + '.egg')
safe_mkdir(cache_dir)
if not os.path.exists(egg_name):
egg_tmp_path = tempfile.mkdtemp(dir=cache_dir,
prefix=dist.egg_name())
for fn, content in DistributionHelper.walk(dist):
with safe_open(os.path.join(egg_tmp_path, fn), 'wb') as fp:
fp.write(content)
try:
os.rename(egg_tmp_path, egg_name)
except OSError as e:
# Handle the race condition of other people trying to write into the target cache.
if e.errno == errno.ENOTEMPTY:
safe_rmtree(egg_tmp_path)
metadata = PathMetadata(egg_name, os.path.join(egg_name, 'EGG-INFO'))
return Distribution.from_filename(egg_name, metadata=metadata)
def execute(self, targets):
# The called binary may block for a while, allow concurrent pants activity during this pants
# idle period.
#
# TODO(John Sirois): refactor lock so that I can do:
# with self.context.lock.yield():
# - blocking code
#
# Currently re-acquiring the lock requires a path argument that was set up by the goal
# execution engine. I do not want task code to learn the lock location.
# http://jira.local.twitter.com/browse/AWESOME-1317
self.context.lock.release()
# Run the first target that is a binary.
binaries = filter(is_binary, targets)
if len(binaries) > 0: # We only run the first one.
main = binaries[0].main
# TODO(John Sirois): Since --dry-run is plumbed throughout the Task infra it seems like we
# should just be using that. Ask Benjy why this particular task uses a custom file.
def run_binary(dryrun=False):
def run_workunit_factory(name, labels=list(), cmd=''):
return self.context.new_workunit(name=name, labels=[WorkUnit.RUN] + labels, cmd=cmd)
result = runjava_indivisible(
jvmargs=self.jvm_args,
classpath=(self.classpath(confs=self.confs)),
main=main,
args=self.args,
dryrun=dryrun,
workunit_factory=run_workunit_factory,
workunit_name='run'
)
if dryrun:
return result
if result != 0:
raise TaskError()
result = run_binary(dryrun=self.only_write_cmd_line)
if self.only_write_cmd_line:
with safe_open(self.only_write_cmd_line, 'w') as fd:
fd.write(result)
def __init__(self, filename, max_bytes, max_backups, mode='w'):
"""
required:
filename = The file name.
max_bytes = The maximum size of an individual log file.
max_backups = The maximum number of log file backups to create.
optional:
mode = Mode to open the file in.
"""
if max_bytes > 0 and max_backups <= 0:
raise ValueError(
'A positive value for max_backups must be specified if max_bytes > 0.'
)
self._max_bytes = max_bytes
self._max_backups = max_backups
self.file = safe_open(filename, mode=mode)
self.filename = filename
self.mode = mode
self.closed = False
def checkstyle(self, sources):
classpath = self.profile_classpath(self._profile)
with self.context.state('classpath', []) as cp:
classpath.extend(jar for conf, jar in cp if conf in self._confs)
opts = ['-c', self._configuration_file, '-f', 'plain']
if self._properties:
properties_file = os.path.join(self._work_dir,
'checkstyle.properties')
with safe_open(properties_file, 'w') as pf:
for k, v in self._properties.items():
pf.write('%s=%s\n' % (k, v))
opts.extend(['-p', properties_file])
return self.runjava(CHECKSTYLE_MAIN,
classpath=classpath,
opts=opts,
args=sources,
workunit_name='checkstyle')
def report(self, targets, tests, junit_classpath):
emma_classpath = self._task_exports.tool_classpath(self._emma_bootstrap_key)
args = [
'report',
'-in', self._coverage_metadata_file,
'-in', self._coverage_file,
'-exit'
]
source_bases = set()
def collect_source_base(target):
if self.is_coverage_target(target):
source_bases.add(target.target_base)
for target in self._test_target_candidates(targets):
target.walk(collect_source_base)
for source_base in source_bases:
args.extend(['-sp', source_base])
sorting = ['-Dreport.sort', '+name,+class,+method,+block']
if self._coverage_report_console:
args.extend(['-r', 'txt',
'-Dreport.txt.out.file=%s' % self._coverage_console_file] + sorting)
if self._coverage_report_xml:
args.extend(['-r', 'xml', '-Dreport.xml.out.file=%s' % self._coverage_xml_file])
if self._coverage_report_html:
args.extend(['-r', 'html',
'-Dreport.html.out.file=%s' % self._coverage_html_file,
'-Dreport.out.encoding=UTF-8'] + sorting)
main = 'emma'
result = execute_java(classpath=emma_classpath, main=main, args=args,
workunit_factory=self._context.new_workunit,
workunit_name='emma-report')
if result != 0:
raise TaskError("java %s ... exited non-zero (%i)"
" 'failed to generate code coverage reports'" % (main, result))
if self._coverage_report_console:
with safe_open(self._coverage_console_file) as console_report:
sys.stdout.write(console_report.read())
if self._coverage_report_html_open:
binary_util.ui_open(self._coverage_html_file)
def find_java_home():
# A kind-of-insane hack to find the effective java home. On some platforms there are so
# many hard and symbolic links into the JRE dirs that it's actually quite hard to
# establish what path to use as the java home, e.g., for the purpose of rebasing.
# In practice, this seems to work fine.
#
# TODO: In the future we should probably hermeticize the Java enivronment rather than relying
# on whatever's on the shell's PATH. E.g., you either specify a path to the Java home via a
# cmd-line flag or .pantsrc, or we infer one with this method but verify that it's of a
# supported version.
with temporary_dir() as tmpdir:
with safe_open(os.path.join(tmpdir, 'X.java'), 'w') as srcfile:
srcfile.write(
'class X { public static void main(String[] argv) { '
'System.out.println(System.getProperty("java.home")); } }')
subprocess.Popen(['javac', '-d', tmpdir, srcfile.name],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
return subprocess.Popen(['java', '-cp', tmpdir, 'X'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()[0]
def checkstyle(self, sources, targets):
egroups = self.context.products.get_data('exclusives_groups')
etag = egroups.get_group_key_for_target(targets[0])
classpath = self.profile_classpath(self._profile)
cp = egroups.get_classpath_for_group(etag)
classpath.extend(jar for conf, jar in cp if conf in self._confs)
opts = [
'-c', self._configuration_file,
'-f', 'plain'
]
if self._properties:
properties_file = os.path.join(self._work_dir, 'checkstyle.properties')
with safe_open(properties_file, 'w') as pf:
for k, v in self._properties.items():
pf.write('%s=%s\n' % (k, v))
opts.extend(['-p', properties_file])
return self.runjava(CHECKSTYLE_MAIN, classpath=classpath, opts=opts, args=sources,
workunit_name='checkstyle')
def execute(self, targets):
# Run the first target that is a binary.
self.context.lock.release()
binaries = filter(is_binary, targets)
if len(binaries) > 0: # We only run the first one.
main = binaries[0].main
def run_binary(only_write_cmd_line_to):
result = runjava(jvmargs=self.jvm_args,
classpath=(self.classpath(confs=self.confs)),
main=main,
args=self.args,
only_write_cmd_line_to=only_write_cmd_line_to)
if result != 0:
raise TaskError()
if self.only_write_cmd_line is None:
run_binary(None)
else:
with safe_open(self.only_write_cmd_line, 'w') as fd:
run_binary(fd)
def execute(self, targets):
# The called binary may block for a while, allow concurrent pants activity during this pants
# idle period.
#
# TODO(John Sirois): refactor lock so that I can do:
# with self.context.lock.yield():
# - blocking code
#
# Currently re-acquiring the lock requires a path argument that was set up by the goal
# execution engine. I do not want task code to learn the lock location.
# http://jira.local.twitter.com/browse/AWESOME-1317
self.context.lock.release()
# Run the first target that is a binary.
binaries = filter(is_binary, targets)
if len(binaries) > 0: # We only run the first one.
main = binaries[0].main
egroups = self.context.products.get_data('exclusives_groups')
group_key = egroups.get_group_key_for_target(binaries[0])
group_classpath = egroups.get_classpath_for_group(group_key)
executor = CommandLineGrabber(
) if self.only_write_cmd_line else None
result = execute_java(classpath=(self.classpath(
confs=self.confs, exclusives_classpath=group_classpath)),
main=main,
executor=executor,
jvm_options=self.jvm_args,
args=self.args,
workunit_factory=self.context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnit.RUN])
if self.only_write_cmd_line:
with safe_open(self.only_write_cmd_line, 'w') as outfile:
outfile.write(' '.join(executor.cmd))
elif result != 0:
raise TaskError('java %s ... exited non-zero (%i)' %
(main, result),
exit_code=result)
def upload_sync(self, stats):
try:
last_modified = self.collect_host_env_info(stats)
if not last_modified:
last_modified = int(os.path.getmtime(self._pants_stat_file))
with safe_open(self._pants_stat_file, 'r') as stats_file:
lines = stats_file.readlines()
#Just want to make sure, we do not wait for MAX_RECORDS but also upload when
#the last time we uploaded is less than configured value in the pants.ini
last_uploaded = Amount(int(time.time()) - last_modified, Time.SECONDS)
if (self.force_stats_upload or len(lines) >= MAX_RECORDS or last_uploaded > self._max_delay):
#Put the file in the right place.
dirutil.safe_mkdir(self._stats_dir)
with temporary_file(self._stats_dir, False) as stats_uploader_tmpfile:
os.rename(self._pants_stat_file, stats_uploader_tmpfile.name)
self._stats_http_client.process_stats_file()
#Merge Logs so that user /tmp is not cluttered with too many log files for each run.
self.merge_logs()
sys.exit(0)
except OSError as e:
log.debug("Error manipulating stats files for upload %s" % e)
def _await_nailgun_server(self, stdout, stderr):
nailgun_timeout_seconds = 5
max_socket_connect_attempts = 10
nailgun = None
port_parse_start = time.time()
with safe_open(self._ng_out, 'r') as ng_out:
while not nailgun:
started = ng_out.readline()
if started:
port = self._parse_nailgun_port(started)
nailgun = self._create_ngclient(port, stdout, stderr)
log.debug('Detected ng server up on port %d' % port)
elif time.time() - port_parse_start > nailgun_timeout_seconds:
raise NailgunClient.NailgunError(
'Failed to read ng output after'
' %s seconds' % nailgun_timeout_seconds)
attempt = 0
while nailgun:
sock = nailgun.try_connect()
if sock:
sock.close()
endpoint = self._get_nailgun_endpoint()
if endpoint:
log.debug(
'Connected to ng server with fingerprint %s pid: %d @ port: %d'
% endpoint)
else:
raise NailgunClient.NailgunError(
'Failed to connect to ng server.')
return nailgun
elif attempt > max_socket_connect_attempts:
raise nailgun.NailgunError(
'Failed to connect to ng output after %d connect attempts'
% max_socket_connect_attempts)
attempt += 1
log.debug('Failed to connect on attempt %d' % attempt)
time.sleep(0.1)
def stage_artifact(tgt,
jar,
version,
changelog,
confs=None,
artifact_ext=''):
def path(name=None, suffix='', extension='jar'):
return artifact_path(jar,
version,
name=name,
suffix=suffix,
extension=extension,
artifact_ext=artifact_ext)
with safe_open(path(suffix='-CHANGELOG', extension='txt'),
'w') as changelog_file:
changelog_file.write(changelog)
ivyxml = path(name='ivy', extension='xml')
IvyWriter(get_pushdb).write(tgt, ivyxml, confs=confs)
PomWriter(get_pushdb).write(tgt, path(extension='pom'))
return ivyxml
def _get_nailgun_endpoint(self):
if os.path.exists(self._pidfile):
with _safe_open(self._pidfile, 'r') as pidfile:
contents = pidfile.read()
def invalid_pidfile():
log.warn('Invalid ng pidfile %s contained: %s' % (self._pidfile, contents))
return None
endpoint = contents.split(':')
if len(endpoint) != 2:
return invalid_pidfile()
pid, port = endpoint
try:
return int(pid.strip()), int(port.strip())
except ValueError:
return invalid_pidfile()
elif NailgunTask._find:
pid_port = NailgunTask._find(self._pidfile)
if pid_port:
self.context.log.info('found ng server @ pid:%d port:%d' % pid_port)
with safe_open(self._pidfile, 'w') as pidfile:
pidfile.write('%d:%d\n' % pid_port)
return pid_port
return None
def write(self, target, path, confs=None):
def as_jar(internal_target):
jar, _, _, _ = self.get_db(internal_target).as_jar_with_version(
internal_target)
return jar
# TODO(John Sirois): a dict is used here to de-dup codegen targets which have both the original
# codegen target - say java_thrift_library - and the synthetic generated target (java_library)
# Consider reworking codegen tasks to add removal of the original codegen targets when rewriting
# the graph
dependencies = OrderedDict()
for dep in target.internal_dependencies:
jar = as_jar(dep)
dependencies[(jar.org, jar.name)] = self.internaldep(jar)
for jar in target.jar_dependencies:
if jar.rev:
dependencies[(jar.org, jar.name)] = self.jardep(jar)
target_jar = self.internaldep(
as_jar(target)).extend(dependencies=dependencies.values())
template_kwargs = self.templateargs(target_jar, confs)
with safe_open(path, 'w') as output:
template = pkgutil.get_data(__name__, self.template_relpath)
Generator(template, **template_kwargs).write(output)
def select_binary(base_path, version, name, config=None):
"""Selects a binary matching the current os and architecture.
Raises TaskError if no binary of the given version and name could be found.
"""
# TODO(John Sirois): finish doc of the path structure expexcted under base_path
config = config or Config.load()
cachedir = config.getdefault('pants_cachedir', default=os.path.expanduser('~/.pants.d'))
baseurl = config.getdefault('pants_support_baseurl')
timeout_secs = config.getdefault('pants_support_fetch_timeout_secs', type=int, default=30)
sysname, _, release, _, machine = os.uname()
os_id = _ID_BY_OS[sysname.lower()]
if os_id:
middle_path = _PATH_BY_ID[os_id(release, machine)]
if middle_path:
binary_path = os.path.join(base_path, *(middle_path + [version, name]))
cached_binary_path = os.path.join(cachedir, binary_path)
if not os.path.exists(cached_binary_path):
url = posixpath.join(baseurl, binary_path)
log.info('Fetching %s binary from: %s' % (name, url))
downloadpath = cached_binary_path + '~'
try:
with closing(urllib_request.urlopen(url, timeout=timeout_secs)) as binary:
with safe_open(downloadpath, 'wb') as cached_binary:
cached_binary.write(binary.read())
os.rename(downloadpath, cached_binary_path)
chmod_plus_x(cached_binary_path)
except (IOError, urllib_error.HTTPError, urllib_error.URLError) as e:
raise TaskError('Failed to fetch binary from %s: %s' % (url, e))
finally:
safe_delete(downloadpath)
log.debug('Selected %s binary cached at: %s' % (name, cached_binary_path))
return cached_binary_path
raise TaskError('No %s binary found for: %s' % (name, (sysname, release, machine)))
def _cachepath(self, file):
with safe_open(file, 'r') as cp:
yield (path.strip() for path in cp.read().split(os.pathsep)
if path.strip())
def test(self):
self.assertEqual(set(), self.git.changed_files())
self.assertEqual(set(['README']),
self.git.changed_files(from_commit='HEAD^'))
tip_sha = self.git.commit_id
self.assertTrue(tip_sha)
self.assertTrue(tip_sha in self.git.changelog())
self.assertTrue(self.git.tag_name.startswith('first-'),
msg='un-annotated tags should be found')
self.assertEqual('master', self.git.branch_name)
def edit_readme():
with open(self.readme_file, 'a') as readme:
readme.write('More data.')
edit_readme()
with open(os.path.join(self.worktree, 'INSTALL'), 'w') as untracked:
untracked.write('make install')
self.assertEqual(set(['README']), self.git.changed_files())
self.assertEqual(set(['README', 'INSTALL']),
self.git.changed_files(include_untracked=True))
try:
# These changes should be rejected because our branch point from origin is 1 commit behind
# the changes pushed there in clone 2.
self.git.commit('API Changes.')
except Scm.RemoteException:
with environment_as(GIT_DIR=self.gitdir,
GIT_WORK_TREE=self.worktree):
subprocess.check_call(
['git', 'reset', '--hard', 'depot/master'])
self.git.refresh()
edit_readme()
self.git.commit('''API '"' " Changes.''')
self.git.tag('second', message='''Tagged ' " Changes''')
with temporary_dir() as clone:
with pushd(clone):
self.init_repo('origin', self.origin)
subprocess.check_call(
['git', 'pull', '--tags', 'origin', 'master:master'])
with open(os.path.realpath('README')) as readme:
self.assertEqual('--More data.', readme.read())
git = Git()
# Check that we can pick up committed and uncommitted changes.
with safe_open(os.path.realpath('CHANGES'), 'w') as changes:
changes.write('none')
subprocess.check_call(['git', 'add', 'CHANGES'])
self.assertEqual(set(['README', 'CHANGES']),
git.changed_files(from_commit='first'))
self.assertEqual('master', git.branch_name)
self.assertEqual('second',
git.tag_name,
msg='annotated tags should be found')
def emit_codehighlight_css(path, style):
with safe_open(path, 'w') as css:
css.write((HtmlFormatter(style=style)).get_style_defs('.codehilite'))
return path
def process(self, outdir, base, source, fragmented, url_builder, get_config, css=None):
def parse_url(spec):
match = MarkdownToHtml.PANTS_LINK.match(spec)
if match:
page = Target.get(Address.parse(get_buildroot(), match.group(1)))
anchor = match.group(2) or ''
if not page:
raise TaskError('Invalid link %s' % match.group(1))
alias, url = url_builder(page, config=get_config(page))
return alias, url + anchor
else:
return spec, spec
def build_url(label):
components = label.split('|', 1)
if len(components) == 1:
return parse_url(label.strip())
else:
alias, link = components
_, url = parse_url(link.strip())
return alias, url
wikilinks = WikilinksExtension(build_url)
path, ext = os.path.splitext(source)
output_path = os.path.join(outdir, path + '.html')
safe_mkdir(os.path.dirname(output_path))
with codecs.open(output_path, 'w', 'utf-8') as output:
with codecs.open(os.path.join(get_buildroot(), base, source), 'r', 'utf-8') as input:
md_html = markdown.markdown(
input.read(),
extensions=['codehilite(guess_lang=False)', 'extra', 'tables', 'toc', wikilinks],
)
if fragmented:
if css:
with safe_open(css) as fd:
output.write(textwrap.dedent('''
<style type="text/css">
%s
</style>
''').strip() % fd.read())
output.write('\n')
output.write(md_html)
else:
if css:
css_relpath = os.path.relpath(css, outdir)
out_relpath = os.path.dirname(source)
link_relpath = os.path.relpath(css_relpath, out_relpath)
css = '<link rel="stylesheet" type="text/css" href="%s"/>' % link_relpath
html = textwrap.dedent('''
<html>
<head>
<meta charset="utf-8">
%s
</head>
<body>
<!-- generated by pants! -->
%s
</body>
</html>
''').strip() % (css or '', md_html)
output.write(html)
return output.name
def _write_processor_info(self, processor_info_file, processors):
with safe_open(processor_info_file, 'w') as f:
for processor in processors:
f.write('%s\n' % processor.strip())
def write(self, path, contents):
with safe_open(path, 'w') as fp:
fp.write(contents)
return path
def generate_project(self, project):
def linked_folder_id(source_set):
return source_set.source_base.replace(os.path.sep, '.')
def base_path(source_set):
return os.path.join(source_set.root_dir, source_set.source_base)
def create_source_base_template(source_set):
source_base = base_path(source_set)
return source_base, TemplateData(id=linked_folder_id(source_set),
path=source_base)
source_bases = dict(map(create_source_base_template, project.sources))
if project.has_python:
source_bases.update(
map(create_source_base_template, project.py_sources))
source_bases.update(
map(create_source_base_template, project.py_libs))
def create_source_template(base_id, includes=None, excludes=None):
return TemplateData(
base=base_id,
includes='|'.join(OrderedSet(includes)) if includes else None,
excludes='|'.join(OrderedSet(excludes)) if excludes else None,
)
def create_sourcepath(base_id, sources):
def normalize_path_pattern(path):
return '%s/' % path if not path.endswith('/') else path
includes = [
normalize_path_pattern(src_set.path) for src_set in sources
if src_set.path
]
excludes = []
for source_set in sources:
excludes.extend(
normalize_path_pattern(exclude)
for exclude in source_set.excludes)
return create_source_template(base_id, includes, excludes)
pythonpaths = []
if project.has_python:
for source_set in project.py_sources:
pythonpaths.append(
create_source_template(linked_folder_id(source_set)))
for source_set in project.py_libs:
lib_path = source_set.path if source_set.path.endswith(
'.egg') else '%s/' % source_set.path
pythonpaths.append(
create_source_template(linked_folder_id(source_set),
includes=[lib_path]))
configured_project = TemplateData(
name=self.project_name,
java=TemplateData(jdk=self.java_jdk,
language_level=('1.%d' %
self.java_language_level)),
python=project.has_python,
scala=project.has_scala and not project.skip_scala,
source_bases=source_bases.values(),
pythonpaths=pythonpaths,
debug_port=project.debug_port,
)
outdir = os.path.abspath(os.path.join(self.work_dir, 'bin'))
safe_mkdir(outdir)
source_sets = defaultdict(OrderedSet) # base_id -> source_set
for source_set in project.sources:
source_sets[linked_folder_id(source_set)].add(source_set)
sourcepaths = [
create_sourcepath(base_id, sources)
for base_id, sources in source_sets.items()
]
libs = []
def add_jarlibs(classpath_entries):
for classpath_entry in classpath_entries:
libs.append((classpath_entry.jar, classpath_entry.source_jar))
add_jarlibs(project.internal_jars)
add_jarlibs(project.external_jars)
configured_classpath = TemplateData(
sourcepaths=sourcepaths,
has_tests=project.has_tests,
libs=libs,
scala=project.has_scala,
# Eclipse insists the outdir be a relative path unlike other paths
outdir=os.path.relpath(outdir, get_buildroot()),
)
def apply_template(output_path, template_relpath, **template_data):
with safe_open(output_path, 'w') as output:
Generator(pkgutil.get_data(__name__, template_relpath),
**template_data).write(output)
apply_template(self.project_filename,
self.project_template,
project=configured_project)
apply_template(self.classpath_filename,
self.classpath_template,
classpath=configured_classpath)
apply_template(os.path.join(
self.work_dir, 'Debug on port %d.launch' % project.debug_port),
self.debug_template,
project=configured_project)
apply_template(self.coreprefs_filename,
self.coreprefs_template,
project=configured_project)
for resource in _SETTINGS:
with safe_open(os.path.join(self.cwd, '.settings', resource),
'w') as prefs:
prefs.write(
pkgutil.get_data(
__name__, os.path.join('files', 'eclipse', resource)))
factorypath = TemplateData(
project_name=self.project_name,
# The easiest way to make sure eclipse sees all annotation processors is to put all libs on
# the apt factorypath - this does not seem to hurt eclipse performance in any noticeable way.
jarpaths=libs)
apply_template(self.apt_filename,
self.apt_template,
factorypath=factorypath)
if project.has_python:
apply_template(self.pydev_filename,
self.pydev_template,
project=configured_project)
else:
safe_delete(self.pydev_filename)
print('\nGenerated project at %s%s' % (self.work_dir, os.sep))
def execute(self, targets):
if not self._flatten and len(targets) > 1:
topologically_sorted_targets = filter(
JavaCompile._is_java,
reversed(InternalTarget.sort_targets(targets)))
for target in topologically_sorted_targets:
self.execute([target])
return
self.context.log.info('Compiling targets %s' % str(targets))
java_targets = filter(JavaCompile._is_java, targets)
if java_targets:
with self.context.state('classpath', []) as cp:
for conf in self._confs:
cp.insert(0, (conf, self._resources_dir))
cp.insert(0, (conf, self._classes_dir))
with self.changed(java_targets,
invalidate_dependants=True) as changed:
sources_by_target, processors, fingerprint = self.calculate_sources(
changed)
if sources_by_target:
sources = reduce(
lambda all, sources: all.union(sources),
sources_by_target.values())
if not sources:
self.context.log.warn(
'Skipping java compile for targets with no sources:\n %s'
% '\n '.join(
str(t) for t in sources_by_target.keys()))
else:
classpath = [
jar for conf, jar in cp if conf in self._confs
]
result = self.compile(classpath, sources,
fingerprint)
if result != 0:
default_message = 'Unexpected error - %s returned %d' % (
_JMAKE_MAIN, result)
raise TaskError(
_JMAKE_ERROR_CODES.get(
result, default_message))
if processors:
# Produce a monolithic apt processor service info file for further compilation rounds
# and the unit test classpath.
processor_info_file = os.path.join(
self._classes_dir, _PROCESSOR_INFO_FILE)
if os.path.exists(processor_info_file):
with safe_open(processor_info_file, 'r') as f:
for processor in f:
processors.add(processor.strip())
self.write_processor_info(processor_info_file,
processors)
if self.context.products.isrequired('classes'):
genmap = self.context.products.get('classes')
# Map generated classes to the owning targets and sources.
dependencies = Dependencies(self._classes_dir,
self._dependencies_file)
for target, classes_by_source in dependencies.findclasses(
targets).items():
for source, classes in classes_by_source.items():
genmap.add(source, self._classes_dir, classes)
genmap.add(target, self._classes_dir, classes)
# TODO(John Sirois): Map target.resources in the same way
# 'Map' (rewrite) annotation processor service info files to the owning targets.
for target in targets:
if is_apt(target) and target.processors:
basedir = os.path.join(self._resources_dir, target.id)
processor_info_file = os.path.join(
basedir, _PROCESSOR_INFO_FILE)
self.write_processor_info(processor_info_file,
target.processors)
genmap.add(target, basedir, [_PROCESSOR_INFO_FILE])
def create_product(product):
with safe_open(os.path.join(outdir, product), mode='w') as fp:
fp.write(product)
return product
def create_product(product):
abspath = os.path.join(outdir, product)
with safe_open(abspath, mode='w') as fp:
fp.write(product)
return abspath
def __enter__(self): self._file = safe_open(self._path, *self._args, **self._kwargs) return self._file
def changed(self, targets, only_buildfiles=False, invalidate_dependants=False):
"""
Yields an iterable over the targets that have changed since the last check to a with block.
If no exceptions are thrown by work in the block, the cache is updated for the targets,
otherwise if a TargetError is thrown by the work in the block all targets except those in the
TargetError are cached.
:targets The targets to check for changes.
:only_buildfiles If True, then just the target's BUILD files are checked for changes.
:invalidate_dependants If True then any targets depending on changed targets are invalidated
:returns: the subset of targets that have changed
"""
safe_mkdir(self._basedir)
cache_manager = Task.CacheManager(BuildCache(self._basedir), targets, only_buildfiles)
check = self.invalidate_for()
if check is not None:
with safe_open(self._extradata, 'w') as pickled:
pickle.dump(check, pickled)
cache_key = cache_manager.check_content(Task.EXTRA_DATA, [self._extradata])
if cache_key:
self.context.log.debug('invalidating all targets for %s' % self.__class__.__name__)
for target in targets:
cache_manager.invalidate(target, cache_key)
for target in targets:
cache_manager.check(target)
if invalidate_dependants and cache_manager.changed:
for target in (self.context.dependants(lambda t: t in cache_manager.changed.keys())).keys():
cache_manager.invalidate(target)
if invalidate_dependants:
if cache_manager.foreign_invalidated_targets:
self.context.log.info('Invalidated %d dependant targets '
'for the next round' % cache_manager.foreign_invalidated_targets)
if cache_manager.changed_files:
msg = 'Operating on %d files in %d changed targets' % (
cache_manager.changed_files,
len(cache_manager.changed)
)
if cache_manager.invalidated_files:
msg += ' and %d files in %d invalidated dependant targets' % (
cache_manager.invalidated_files,
cache_manager.invalidated_targets
)
self.context.log.info(msg)
elif cache_manager.changed_files:
self.context.log.info('Operating on %d files in %d changed targets' % (
cache_manager.changed_files,
len(cache_manager.changed)
))
try:
yield cache_manager.changed.keys()
for cache_keys in cache_manager.changed.values():
for cache_key in cache_keys:
cache_manager.update(cache_key)
except TargetError as e:
for target, cache_keys in cache_manager.changed.items():
if target not in e.targets:
for cache_key in cache_keys:
cache_manager.update(cache_key)
