def execute_antlr_test(self, expected_package):
context = self.create_context()
task = self.execute(context)
# get the synthetic target from the private graph
task_outdir = os.path.join(task.workdir, 'isolated', self.get_antlr_target().id)
# verify that the synthetic target's list of sources match what are actually created
def re_relativize(p):
"""Take a path relative to task_outdir, and make it relative to the build_root"""
return os.path.relpath(os.path.join(task_outdir, p), self.build_root)
actual_sources = [re_relativize(s) for s in Fileset.rglobs('*.java', root=task_outdir)]
expected_sources = self.get_antlr_syn_target(task).sources_relative_to_buildroot()
self.assertEquals(set(expected_sources), set(actual_sources))
# and that the synthetic target has a valid source root and the generated sources have the
# expected java package
def get_package(path):
with open(path) as fp:
for line in fp:
match = self.PACKAGE_RE.match(line)
if match:
return match.group('package_name')
return None
syn_target = self.get_antlr_syn_target(task)
for source in syn_target.sources_relative_to_source_root():
source_path = os.path.join(task_outdir, source)
self.assertTrue(os.path.isfile(source_path),
"{0} is not the source root for {1}".format(task_outdir, source))
self.assertEqual(expected_package, get_package(source_path))
self.assertIn(syn_target, context.targets())
def execute_antlr_test(self, expected_package, target_workdir_fun=None):
target = self.get_antlr_target()
context = self.create_context()
task = self.prepare_execute(context)
target_workdir_fun = target_workdir_fun or (lambda x: safe_mkdtemp(dir=x))
target_workdir = target_workdir_fun(task.workdir)
# Generate code, then create a synthetic target.
task.execute_codegen(target, target_workdir)
syn_target = task._inject_synthetic_target(target, target_workdir)
actual_sources = [s for s in Fileset.rglobs('*.java', root=target_workdir)]
expected_sources = syn_target.sources_relative_to_source_root()
self.assertEquals(set(expected_sources), set(actual_sources))
# and that the synthetic target has a valid source root and the generated sources have the
# expected java package
def get_package(path):
with open(path) as fp:
for line in fp:
match = self.PACKAGE_RE.match(line)
if match:
return match.group('package_name')
return None
for source in syn_target.sources_relative_to_source_root():
source_path = os.path.join(target_workdir, source)
self.assertTrue(os.path.isfile(source_path),
"{0} is not the source root for {1}".format(target_workdir, source))
self.assertEqual(expected_package, get_package(source_path))
self.assertIn(syn_target, context.targets())
return syn_target
def execute_antlr4_test(self, expected_package):
context = self.create_context()
task = self.execute(context)
# get the synthetic target from the private graph
task_outdir = os.path.join(task.workdir, 'antlr4', 'gen-java')
syn_sourceroot = os.path.join(task_outdir, self.PARTS['srcroot'])
syn_target_name = ('{srcroot}/{dir}.{name}'.format(**self.PARTS)).replace('/', '.')
syn_address = SyntheticAddress(spec_path=os.path.relpath(syn_sourceroot, self.build_root),
target_name=syn_target_name)
syn_target = context.build_graph.get_target(syn_address)
# verify that the synthetic target's list of sources match what are actually created
def re_relativize(p):
"""Take a path relative to task_outdir, and make it relative to the build_root"""
return os.path.relpath(os.path.join(task_outdir, p), self.build_root)
actual_sources = [re_relativize(s) for s in Fileset.rglobs('*.java', root=task_outdir)]
self.assertEquals(set(syn_target.sources_relative_to_buildroot()), set(actual_sources))
# and that the synthetic target has a valid source root and the generated sources have the
# expected java package
def get_package(path):
with open(path) as fp:
for line in fp:
match = self.PACKAGE_RE.match(line)
if match:
return match.group('package_name')
return None
for source in syn_target.sources_relative_to_source_root():
source_path = os.path.join(syn_sourceroot, source)
self.assertTrue(os.path.isfile(source_path),
"{0} is not the source root for {1}".format(syn_sourceroot, source))
self.assertEqual(expected_package, get_package(source_path))
def path_iterator(args, options):
for path in args:
if os.path.isdir(path):
for filename in Fileset.rglobs('*.py', root=path)():
yield os.path.join(path, filename), None
elif os.path.isfile(path):
yield path, None
def execute_antlr_test(self, expected_package, target_workdir_fun=None):
target = self.get_antlr_target()
context = self.create_context()
task = self.prepare_execute(context)
target_workdir_fun = target_workdir_fun or (lambda x: safe_mkdtemp(dir=x))
# Do not use task.workdir here, because when we calculating hash for synthetic target
# we need persistent source paths in terms of relative position to build root.
target_workdir = target_workdir_fun(self.build_root)
vt = DummyVersionedTarget(target, target_workdir)
# Generate code, then create a synthetic target.
task.execute_codegen(target, target_workdir)
sources = task._capture_sources((vt,))[0]
syn_target = task._inject_synthetic_target(vt, sources)
actual_sources = [s for s in Fileset.rglobs('*.java', root=target_workdir)]
expected_sources = syn_target.sources_relative_to_source_root()
self.assertEqual(set(expected_sources), set(actual_sources))
# Check that the synthetic target has a valid source root and the generated sources have the
# expected java package
def get_package(path):
with open(path, 'r') as fp:
for line in fp:
match = self.PACKAGE_RE.match(line)
if match:
return match.group('package_name')
return None
for source in syn_target.sources_relative_to_source_root():
source_path = os.path.join(target_workdir, source)
self.assertTrue(os.path.isfile(source_path),
"{0} is not the source root for {1}".format(target_workdir, source))
self.assertEqual(expected_package, get_package(source_path))
self.assertIn(syn_target, context.targets())
# Check that the output file locations match the package
if expected_package is not None:
expected_path_prefix = expected_package.replace('.', os.path.sep) + os.path.sep
for source in syn_target.sources_relative_to_source_root():
self.assertTrue(source.startswith(expected_path_prefix),
"{0} does not start with {1}".format(source, expected_path_prefix))
# Check that empty directories have been removed
for root, dirs, files in os.walk(target_workdir):
for d in dirs:
full_dir = os.path.join(root, d)
self.assertTrue(os.listdir(full_dir),
"Empty directories should have been removed ({0})".format(full_dir))
return syn_target
def process_stats_file(self):
for filename in Fileset.walk(self._stats_dir):
try :
with open(os.path.join(self._stats_dir, filename), 'r') as stats_file:
lines = stats_file.readlines()
tmp_str = ",".join(lines)
tmp_str.strip(',')
self.push_stats("[" + tmp_str + "]")
os.remove(os.path.join(self._stats_dir, filename))
except httplib.HTTPException as e:
log.debug("HTTPException %s" % e)
except OSError as e:
log.debug("Error reading or deleting a stats file %s" % e)
def test_antlr4(self):
parts = {'srcroot': 'src/antlr',
'dir': 'this/is/a/directory',
'name': 'smoke',
'package': 'this.is.a.package',
'prefix': 'SMOKE'}
self.create_file(relpath='%(srcroot)s/%(dir)s/%(prefix)s.g4' % parts,
contents=dedent('''
grammar %(prefix)s;
options { language=Java; }
////////////////////
start : letter EOF ;
letter : LETTER ;
////////////////////
fragment LETTER : [a-zA-Z] ;
''' % parts))
self.add_to_build_file('%(srcroot)s/%(dir)s/BUILD' % parts, dedent('''
java_antlr_library(
name='%(name)s',
compiler='antlr4',
package='%(package)s',
sources=['%(prefix)s.g4'],
)
''' % parts))
# generate a context to contain the build graph for the input target, then execute
context = self.context(target_roots=[self.target('%(srcroot)s/%(dir)s:%(name)s' % parts)])
task = self.execute(context, AntlrGen)
# get the synthetic target from the private graph
task_outdir = os.path.join(task.workdir, 'antlr4', 'gen-java')
syn_sourceroot = os.path.join(task_outdir, parts['srcroot'])
syn_target_name = ('%(srcroot)s/%(dir)s.%(name)s' % parts).replace('/', '.')
syn_address = SyntheticAddress(spec_path=os.path.relpath(syn_sourceroot, self.build_root),
target_name=syn_target_name)
syn_target = context.build_graph.get_target(syn_address)
# verify that the synthetic target's list of sources match what are actually created
def re_relativize(p):
"""Take a path relative to task_outdir, and make it relative to the build_root"""
# TODO: is there a way to do this directly with rglobs?
return os.path.relpath(os.path.join(task_outdir, p), self.build_root)
actual_sources = [re_relativize(s) for s in Fileset.rglobs('*.java', root=task_outdir)]
self.assertEquals(set(syn_target.sources_relative_to_buildroot()), set(actual_sources))
# and that the synthetic target has a valid sourceroot
for source in syn_target.sources_relative_to_source_root():
self.assertTrue(os.path.isfile(os.path.join(syn_sourceroot, source)),
"%s is not the sourceroot for %s" % (syn_sourceroot, source))
def __init__(self, address=None, payload=None, **kwargs):
# TODO(John Sirois): Make pants.backend.core.wrapped_globs.Globs in the core backend
# constructable with just a rel_path. Right now it violates the Law of Demeter and
# fundamentally takes a ParseContext, which it doesn't actually need and which other backend
# consumers should not need to know about or create.
# Here we depend on twitter/commons which is less than ideal in core pants and even worse in a
# plugin. We depend on it here to ensure the globbing is lazy and skipped if the target is
# never fingerprinted (eg: when running `./pants list`).
sources = Fileset.globs('*.go', root=os.path.join(get_buildroot(), address.spec_path))
payload = payload or Payload()
payload.add_fields({
'sources': self.create_sources_field(sources=sources,
sources_rel_path=address.spec_path,
key_arg='sources'),
})
super(GoLocalSource, self).__init__(address=address, payload=payload, **kwargs)
def test_antlr_py_gen(self):
self.create_file(
relpath='foo/bar/baz/Baz.g',
contents=dedent("""
grammar Baz;
options {
language=Python;
output=template;
}
a : ID INT
-> template(id={$ID.text}, int={$INT.text})
"id=<id>, int=<int>"
;
ID : 'a'..'z'+;
INT : '0'..'9'+;
WS : (' '|'\n') {$channel=HIDDEN;} ;
"""))
self.add_to_build_file('foo/bar/baz/BUILD', dedent("""
python_antlr_library(
name='baz',
module='foo.bar.baz',
sources=['Baz.g'],
)
"""))
target = self.target('foo/bar/baz')
context = self.context(target_roots=[target])
task = self.prepare_execute(context)
target_workdir = self.test_workdir
# Generate code, then create a synthetic target.
task.execute_codegen(target, target_workdir)
actual_sources = {s for s in Fileset.rglobs('*.py', root=target_workdir)}
self.assertSetEqual({
'foo/__init__.py',
'foo/bar/__init__.py',
'foo/bar/baz/__init__.py',
'foo/bar/baz/BazParser.py',
'foo/bar/baz/BazLexer.py',
}, actual_sources)
def zglobs_following_symlinked_dirs_by_default(*globspecs, **kw):
if 'follow_links' not in kw:
kw['follow_links'] = True
return Fileset.zglobs(*globspecs, **kw)
def rglobs_following_symlinked_dirs_by_default(*globspecs, **kw):
if "follow_links" not in kw:
kw["follow_links"] = True
return Fileset.rglobs(*globspecs, **kw)
def rglobs_following_symlinked_dirs_by_default(*globspecs, **kw):
if 'follow_links' not in kw:
kw['follow_links'] = True
return Fileset.rglobs(*globspecs, **kw)
def test_antlr4(self):
parts = {
'srcroot': 'testprojects/src/antlr',
'dir': 'this/is/a/directory',
'name': 'smoke',
'package': 'this.is.a.package',
'prefix': 'SMOKE'
}
self.create_file(relpath='%(srcroot)s/%(dir)s/%(prefix)s.g4' % parts,
contents=dedent('''
grammar %(prefix)s;
options { language=Java; }
////////////////////
start : letter EOF ;
letter : LETTER ;
////////////////////
fragment LETTER : [a-zA-Z] ;
''' % parts))
self.add_to_build_file(
'%(srcroot)s/%(dir)s/BUILD' % parts,
dedent('''
java_antlr_library(
name='%(name)s',
compiler='antlr4',
package='%(package)s',
sources=['%(prefix)s.g4'],
)
''' % parts))
# generate a context to contain the build graph for the input target, then execute
context = self.context(
target_roots=[self.target('%(srcroot)s/%(dir)s:%(name)s' % parts)])
task = self.execute(context, AntlrGen)
# get the synthetic target from the private graph
task_outdir = os.path.join(task.workdir, 'antlr4', 'gen-java')
syn_sourceroot = os.path.join(task_outdir, parts['srcroot'])
syn_target_name = ('%(srcroot)s/%(dir)s.%(name)s' % parts).replace(
'/', '.')
syn_address = SyntheticAddress(spec_path=os.path.relpath(
syn_sourceroot, self.build_root),
target_name=syn_target_name)
syn_target = context.build_graph.get_target(syn_address)
# verify that the synthetic target's list of sources match what are actually created
def re_relativize(p):
"""Take a path relative to task_outdir, and make it relative to the build_root"""
# TODO: is there a way to do this directly with rglobs?
return os.path.relpath(os.path.join(task_outdir, p),
self.build_root)
actual_sources = [
re_relativize(s)
for s in Fileset.rglobs('*.java', root=task_outdir)
]
self.assertEquals(set(syn_target.sources_relative_to_buildroot()),
set(actual_sources))
# and that the synthetic target has a valid sourceroot
for source in syn_target.sources_relative_to_source_root():
self.assertTrue(
os.path.isfile(os.path.join(syn_sourceroot, source)),
"%s is not the sourceroot for %s" % (syn_sourceroot, source))
def execute_antlr_test(self, expected_package, target_workdir_fun=None):
target = self.get_antlr_target()
context = self.create_context()
task = self.prepare_execute(context)
target_workdir_fun = target_workdir_fun or (
lambda x: safe_mkdtemp(dir=x))
# Do not use task.workdir here, because when we calculating hash for synthetic target
# we need persistent source paths in terms of relative position to build root.
target_workdir = target_workdir_fun(self.build_root)
vt = DummyVersionedTarget(target, target_workdir)
# Generate code, then create a synthetic target.
task.execute_codegen(target, target_workdir)
sources = task._capture_sources((vt, ))[0]
syn_target = task._inject_synthetic_target(vt, sources)
actual_sources = [
s for s in Fileset.rglobs('*.java', root=target_workdir)
]
expected_sources = syn_target.sources_relative_to_source_root()
self.assertEqual(set(expected_sources), set(actual_sources))
# Check that the synthetic target has a valid source root and the generated sources have the
# expected java package
def get_package(path):
with open(path, 'r') as fp:
for line in fp:
match = self.PACKAGE_RE.match(line)
if match:
return match.group('package_name')
return None
for source in syn_target.sources_relative_to_source_root():
source_path = os.path.join(target_workdir, source)
self.assertTrue(
os.path.isfile(source_path),
"{0} is not the source root for {1}".format(
target_workdir, source))
self.assertEqual(expected_package, get_package(source_path))
self.assertIn(syn_target, context.targets())
# Check that the output file locations match the package
if expected_package is not None:
expected_path_prefix = expected_package.replace(
'.', os.path.sep) + os.path.sep
for source in syn_target.sources_relative_to_source_root():
self.assertTrue(
source.startswith(expected_path_prefix),
"{0} does not start with {1}".format(
source, expected_path_prefix))
# Check that empty directories have been removed
for root, dirs, files in os.walk(target_workdir):
for d in dirs:
full_dir = os.path.join(root, d)
self.assertTrue(
os.listdir(full_dir),
"Empty directories should have been removed ({0})".format(
full_dir))
return syn_target