def test_bser_encoding_failure(self):
build_file_processor = self.create_build_file_processor(extra_funcs=[foo_rule])
fake_stdout = StringIO.StringIO()
build_file = ProjectFile(
self.project_root,
path='BUCK',
contents=(
'foo_rule(',
' name="foo",'
' srcs=[object()],'
')'
))
self.write_file(build_file)
with build_file_processor.with_builtins(__builtin__.__dict__):
process_with_diagnostics(
{
'buildFile': self.build_file_name,
'watchRoot': '',
'projectPrefix': self.project_root,
},
build_file_processor,
fake_stdout)
result = fake_stdout.getvalue()
decoded_result = bser.loads(result)
self.assertEqual(
[],
decoded_result['values'])
self.assertEqual(
'fatal',
decoded_result['diagnostics'][0]['level'])
self.assertEqual(
'parse',
decoded_result['diagnostics'][0]['source'])
def test_bserInput(self):
sockname = self.getSockPath()
watchman_cmd = bser.dumps(['get-sockname'])
cli_cmd = [
'watchman',
'--sockname={0}'.format(sockname),
'--logfile=/BOGUS',
'--statefile=/BOGUS',
'--no-spawn',
'--no-local',
'-j',
]
proc = subprocess.Popen(cli_cmd,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
stdout, stderr = proc.communicate(input=watchman_cmd)
self.assertEqual(proc.poll(), 0, stderr)
# the response should be bser to match our input
result = bser.loads(stdout)
result_sockname = result['sockname']
if compat.PYTHON3:
result_sockname = encoding.decode_local(result_sockname)
self.assertEqual(result_sockname, sockname,
binascii.hexlify(stdout).decode('ascii'))
def test_bserInput(self):
sockpath = self.getSockPath()
watchman_cmd = bser.dumps(["get-sockname"])
cli_cmd = [
os.environ.get("WATCHMAN_BINARY", "watchman"),
"--unix-listener-path={0}".format(sockpath.unix_domain),
"--named-pipe-path={0}".format(sockpath.named_pipe),
"--logfile=/BOGUS",
"--statefile=/BOGUS",
"--no-spawn",
"--no-local",
"-j",
]
proc = subprocess.Popen(
cli_cmd,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
)
stdout, stderr = proc.communicate(input=watchman_cmd)
self.assertEqual(proc.poll(), 0, stderr)
# the response should be bser to match our input
result = bser.loads(stdout)
result_sockname = result["unix_domain"]
if compat.PYTHON3:
result_sockname = encoding.decode_local(result_sockname)
self.assertEqual(
result_sockname,
sockpath.unix_domain,
binascii.hexlify(stdout).decode("ascii"),
)
def _resolve_sockname_helper():
# if invoked via a trigger, watchman will set this env var; we
# should use it unless explicitly set otherwise
path = os.getenv('WATCHMAN_SOCK')
if path:
return path
cmd = ['watchman', '--output-encoding=bser', 'get-sockname']
try:
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=os.name != 'nt')
except OSError as e:
raise WatchmanError('"watchman" executable not in PATH (%s)', e)
stdout, stderr = p.communicate()
exitcode = p.poll()
if exitcode:
raise WatchmanError('watchman exited with code %d' % exitcode)
result = bser.loads(stdout)
if 'error' in result:
raise WatchmanError(str(result['error']))
return result['sockname']
def test_bser_encoding_failure(self):
build_file_processor = self.create_build_file_processor(extra_funcs=[foo_rule])
fake_stdout = StringIO.StringIO()
build_file = ProjectFile(
self.project_root,
path='BUCK',
contents=(
'foo_rule(',
' name="foo",'
' srcs=[object()],'
')'
))
self.write_file(build_file)
with build_file_processor.with_builtins(__builtin__.__dict__):
process_with_diagnostics(
{
'buildFile': self.build_file_name,
'watchRoot': '',
'projectPrefix': self.project_root,
},
build_file_processor,
fake_stdout)
result = fake_stdout.getvalue()
decoded_result = bser.loads(result)
self.assertEqual(
[],
decoded_result['values'])
self.assertEqual(
'fatal',
decoded_result['diagnostics'][0]['level'])
self.assertEqual(
'parse',
decoded_result['diagnostics'][0]['source'])
def test_bserInput(self):
sockname = self.getSockPath()
watchman_cmd = bser.dumps(['get-sockname'])
cli_cmd = [
'watchman',
'--sockname={}'.format(sockname),
'--logfile=/BOGUS',
'--statefile=/BOGUS',
'--no-spawn',
'--no-local',
'-j',
]
proc = subprocess.Popen(cli_cmd,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
stdout, stderr = proc.communicate(input=watchman_cmd)
self.assertEqual(proc.poll(), 0, stderr)
# the response should be bser to match our input
result = bser.loads(stdout)
if PY2:
self.assertEqual(result[b'sockname'], sockname, stdout.encode('hex'))
if PY3:
self.assertEqual(result[b'sockname'], sockname, binascii.hexlify(stdout))
def _resolve_sockname_helper():
# if invoked via a trigger, watchman will set this env var; we
# should use it unless explicitly set otherwise
path = os.getenv('WATCHMAN_SOCK')
if path:
return path
cmd = ['watchman', '--output-encoding=bser', 'get-sockname']
try:
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=os.name != 'nt')
except OSError as e:
raise WatchmanError('"watchman" executable not in PATH (%s)', e)
stdout, stderr = p.communicate()
exitcode = p.poll()
if exitcode:
raise WatchmanError('watchman exited with code %d' % exitcode)
result = bser.loads(stdout)
if 'error' in result:
raise WatchmanError(str(result['error']))
return result['sockname']
def test_bserInput(self):
sockname = self.getSockPath()
watchman_cmd = bser.dumps(["get-sockname"])
cli_cmd = [
"watchman",
"--sockname={0}".format(sockname),
"--logfile=/BOGUS",
"--statefile=/BOGUS",
"--no-spawn",
"--no-local",
"-j",
]
proc = subprocess.Popen(
cli_cmd,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
)
stdout, stderr = proc.communicate(input=watchman_cmd)
self.assertEqual(proc.poll(), 0, stderr)
# the response should be bser to match our input
result = bser.loads(stdout)
result_sockname = result["sockname"]
if compat.PYTHON3:
result_sockname = encoding.decode_local(result_sockname)
self.assertEqual(result_sockname, sockname,
binascii.hexlify(stdout).decode("ascii"))
def munged(self, val, munged):
enc = bser.dumps(val)
if isinstance(val, unicode):
print "# %s --> %s" % (val.encode('utf-8'), enc.encode('hex'))
else:
print "# %s --> %s" % (val, enc.encode('hex'))
dec = bser.loads(enc)
self.assertEquals(munged, dec)
def _loads(self, response):
""" Parse the BSER packet """
return bser.loads(
response,
True,
value_encoding=encoding.get_local_encoding(),
value_errors=encoding.default_local_errors,
)
def munged(self, val, munged):
enc = bser.dumps(val)
if isinstance(val, unicode):
print "# %s --> %s" % (val.encode('utf-8'), enc.encode('hex'))
else:
print "# %s --> %s" % (val, enc.encode('hex'))
dec = bser.loads(enc)
self.assertEquals(munged, dec)
def _loads(self, response):
""" Parse the BSER packet """
return bser.loads(
response,
True,
value_encoding=encoding.get_local_encoding(),
value_errors=encoding.default_local_errors,
)
def test_bserInput(self):
sockname = self.getSockPath()
watchman_cmd = bser.dumps(["get-sockname"])
cli_cmd = ["watchman", "--sockname={}".format(sockname), "--no-spawn", "--no-local", "-j"]
proc = subprocess.Popen(cli_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
stdout, stderr = proc.communicate(input=watchman_cmd)
self.assertEqual(proc.poll(), 0, stderr)
# the response should be bser to match our input
result = bser.loads(stdout)
self.assertEqual(result["sockname"], sockname, stdout.encode("hex"))
def test_watchman_glob_failure_raises_diagnostic_with_stack(self):
class FakeWatchmanClient:
def __init__(self):
self.query_invoked = False
def query(self, *args):
self.query_invoked = True
raise WatchmanError("Nobody watches the watchmen")
def close(self):
pass
self.watchman_client = FakeWatchmanClient()
build_file = ProjectFile(
self.project_root,
path='BUCK',
contents=(
'foo_rule(',
' name="foo",'
' srcs=glob(["*.java"]),',
')'
))
java_file = ProjectFile(self.project_root, path='Foo.java', contents=())
self.write_files(build_file, java_file)
build_file_processor = self.create_build_file_processor(extra_funcs=[foo_rule])
diagnostics = []
rules = []
fake_stdout = StringIO.StringIO()
with build_file_processor.with_builtins(__builtin__.__dict__):
self.assertRaises(
WatchmanError,
process_with_diagnostics,
{
'buildFile': self.build_file_name,
'watchRoot': '',
'projectPrefix': self.project_root,
},
build_file_processor,
fake_stdout)
self.assertTrue(self.watchman_client.query_invoked)
result = fake_stdout.getvalue()
decoded_result = bser.loads(result)
self.assertEqual([], decoded_result['values'])
self.assertEqual(1, len(decoded_result['diagnostics']))
diagnostic = decoded_result['diagnostics'][0]
self.assertEqual('fatal', diagnostic['level'])
self.assertEqual('parse', diagnostic['source'])
self.assertEqual('Nobody watches the watchmen', diagnostic['message'])
exception = diagnostic['exception']
self.assertEqual('WatchmanError', exception['type'])
self.assertEqual('Nobody watches the watchmen', exception['value'])
self.assertTrue(len(exception['traceback']) > 0)
def test_template(self):
# since we can't generate the template bser output, here's a
# a blob from the C test suite in watchman
templ = "\x00\x01\x03\x28" + \
"\x0b\x00\x03\x02\x02\x03\x04\x6e\x61\x6d\x65\x02" + \
"\x03\x03\x61\x67\x65\x03\x03\x02\x03\x04\x66\x72" + \
"\x65\x64\x03\x14\x02\x03\x04\x70\x65\x74\x65\x03" + \
"\x1e\x0c\x03\x19"
dec = bser.loads(templ)
exp = [
{"name": "fred", "age": 20},
{"name": "pete", "age": 30},
{"age": 25}
]
self.assertEquals(exp, dec)
def test_template(self):
# since we can't generate the template bser output, here's a
# a blob from the C test suite in watchman
templ = "\x00\x01\x03\x28" + \
"\x0b\x00\x03\x02\x02\x03\x04\x6e\x61\x6d\x65\x02" + \
"\x03\x03\x61\x67\x65\x03\x03\x02\x03\x04\x66\x72" + \
"\x65\x64\x03\x14\x02\x03\x04\x70\x65\x74\x65\x03" + \
"\x1e\x0c\x03\x19"
dec = bser.loads(templ)
exp = [
{"name": "fred", "age": 20},
{"name": "pete", "age": 30},
{"age": 25}
]
self.assertEquals(exp, dec)
def test_watchman_glob_failure_raises_diagnostic_with_stack(self):
class FakeWatchmanClient:
def __init__(self):
self.query_invoked = False
def query(self, *args):
self.query_invoked = True
raise WatchmanError("Nobody watches the watchmen")
def close(self):
pass
self.watchman_client = FakeWatchmanClient()
build_file = ProjectFile(self.project_root,
path='BUCK',
contents=('foo_rule(', ' name="foo",'
' srcs=glob(["*.java"]),', ')'))
java_file = ProjectFile(self.project_root,
path='Foo.java',
contents=())
self.write_files(build_file, java_file)
build_file_processor = self.create_build_file_processor(
extra_funcs=[foo_rule])
diagnostics = []
rules = []
fake_stdout = StringIO.StringIO()
with build_file_processor.with_builtins(__builtin__.__dict__):
self.assertRaises(
WatchmanError, process_with_diagnostics, {
'buildFile': self.build_file_name,
'watchRoot': '',
'projectPrefix': self.project_root,
}, build_file_processor, fake_stdout)
self.assertTrue(self.watchman_client.query_invoked)
result = fake_stdout.getvalue()
decoded_result = bser.loads(result)
self.assertEqual([], decoded_result['values'])
self.assertEqual(1, len(decoded_result['diagnostics']))
diagnostic = decoded_result['diagnostics'][0]
self.assertEqual('fatal', diagnostic['level'])
self.assertEqual('parse', diagnostic['source'])
self.assertEqual('Nobody watches the watchmen', diagnostic['message'])
exception = diagnostic['exception']
self.assertEqual('WatchmanError', exception['type'])
self.assertEqual('Nobody watches the watchmen', exception['value'])
self.assertTrue(len(exception['traceback']) > 0)
def test_bser_encoding_failure(self):
build_file_processor = self.create_build_file_processor(extra_funcs=[foo_rule])
build_file_processor.install_builtins(__builtin__.__dict__)
fake_stdout = StringIO.StringIO()
build_file = ProjectFile(
self.project_root, path="BUCK", contents=("foo_rule(", ' name="foo",' " srcs=[object()]," ")")
)
self.write_file(build_file)
process_with_diagnostics(
{"buildFile": self.build_file_name, "watchRoot": "", "projectPrefix": self.project_root},
build_file_processor,
fake_stdout,
)
result = fake_stdout.getvalue()
decoded_result = bser.loads(result)
self.assertEqual([], decoded_result["values"])
self.assertEqual("fatal", decoded_result["diagnostics"][0]["level"])
self.assertEqual("parse", decoded_result["diagnostics"][0]["source"])
def test_bserInput(self):
sockname = self.getSockPath()
watchman_cmd = bser.dumps(['get-sockname'])
cli_cmd = [
'watchman',
'--sockname={}'.format(sockname),
'--no-spawn',
'--no-local',
'-j',
]
proc = subprocess.Popen(cli_cmd,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
stdout, stderr = proc.communicate(input=watchman_cmd)
self.assertEqual(proc.poll(), 0, stderr)
# the response should be bser to match our input
result = bser.loads(stdout)
self.assertEqual(result['sockname'], sockname, stdout.encode('hex'))
def roundtrip(self, val):
enc = bser.dumps(val)
print "# %s --> %s" % (val, enc.encode('hex'))
dec = bser.loads(enc)
self.assertEquals(val, dec)
def main():
# Our parent expects to read BSER from our stdout, so if anyone
# uses print, buck will complain with a helpful "but I wanted an
# array!" message and quit. Redirect stdout to stderr so that
# doesn't happen. Actually dup2 the file handle so that writing
# to file descriptor 1, os.system, and so on work as expected too.
to_parent = os.fdopen(os.dup(sys.stdout.fileno()), 'ab')
os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
parser = optparse.OptionParser()
parser.add_option(
'--project_root',
action='store',
type='string',
dest='project_root')
parser.add_option(
'--build_file_name',
action='store',
type='string',
dest="build_file_name")
parser.add_option(
'--allow_empty_globs',
action='store_true',
dest='allow_empty_globs',
help='Tells the parser not to raise an error when glob returns no results.')
parser.add_option(
'--use_watchman_glob',
action='store_true',
dest='use_watchman_glob',
help='Invokes `watchman query` to get lists of files instead of globbing in-process.')
parser.add_option(
'--watchman_watch_root',
action='store',
type='string',
dest='watchman_watch_root',
help='Path to root of watchman watch as returned by `watchman watch-project`.')
parser.add_option(
'--watchman_socket_path',
action='store',
type='string',
dest='watchman_socket_path',
help='Path to Unix domain socket/named pipe as returned by `watchman get-sockname`.')
parser.add_option(
'--watchman_project_prefix',
action='store',
type='string',
dest='watchman_project_prefix',
help='Relative project prefix as returned by `watchman watch-project`.')
parser.add_option(
'--watchman_query_timeout_ms',
action='store',
type='int',
dest='watchman_query_timeout_ms',
help='Maximum time in milliseconds to wait for watchman query to respond.')
parser.add_option(
'--include',
action='append',
dest='include')
parser.add_option(
'--config',
help='BuckConfig settings available at parse time.')
parser.add_option(
'--quiet',
action='store_true',
dest='quiet',
help='Stifles exception backtraces printed to stderr during parsing.')
parser.add_option(
'--ignore_buck_autodeps_files',
action='store_true',
help='Profile every buck file execution')
parser.add_option(
'--profile',
action='store_true',
help='Profile every buck file execution')
(options, args) = parser.parse_args()
# Even though project_root is absolute path, it may not be concise. For
# example, it might be like "C:\project\.\rule".
#
# Under cygwin, the project root will be invoked from buck as C:\path, but
# the cygwin python uses UNIX-style paths. They can be converted using
# cygpath, which is necessary because abspath will treat C:\path as a
# relative path.
options.project_root = cygwin_adjusted_path(options.project_root)
project_root = os.path.abspath(options.project_root)
watchman_client = None
watchman_error = None
if options.use_watchman_glob:
import pywatchman
client_args = {}
if options.watchman_query_timeout_ms is not None:
# pywatchman expects a timeout as a nonnegative floating-point
# value in seconds.
client_args['timeout'] = max(0.0, options.watchman_query_timeout_ms / 1000.0)
if options.watchman_socket_path is not None:
client_args['sockpath'] = options.watchman_socket_path
client_args['transport'] = 'local'
watchman_client = pywatchman.client(**client_args)
watchman_error = pywatchman.WatchmanError
configs = {}
if options.config is not None:
with open(options.config, 'rb') as f:
for section, contents in bser.loads(f.read()).iteritems():
for field, value in contents.iteritems():
configs[(section, field)] = value
buildFileProcessor = BuildFileProcessor(
project_root,
options.watchman_watch_root,
options.watchman_project_prefix,
options.build_file_name,
options.allow_empty_globs,
options.ignore_buck_autodeps_files,
watchman_client,
watchman_error,
implicit_includes=options.include or [],
configs=configs)
buildFileProcessor.install_builtins(__builtin__.__dict__)
# While processing, we'll write exceptions as diagnostic messages
# to the parent then re-raise them to crash the process. While
# doing so, we don't want Python's default unhandled exception
# behavior of writing to stderr.
orig_excepthook = None
if options.quiet:
orig_excepthook = sys.excepthook
sys.excepthook = silent_excepthook
for build_file in args:
process_with_diagnostics(build_file, buildFileProcessor, to_parent,
should_profile=options.profile)
# "for ... in sys.stdin" in Python 2.x hangs until stdin is closed.
for build_file in iter(sys.stdin.readline, ''):
process_with_diagnostics(build_file, buildFileProcessor, to_parent,
should_profile=options.profile)
if options.quiet:
sys.excepthook = orig_excepthook
# Python tries to flush/close stdout when it quits, and if there's a dead
# pipe on the other end, it will spit some warnings to stderr. This breaks
# tests sometimes. Prevent that by explicitly catching the error.
try:
to_parent.close()
except IOError:
pass
def main():
# Our parent expects to read BSER from our stdout, so if anyone
# uses print, buck will complain with a helpful "but I wanted an
# array!" message and quit. Redirect stdout to stderr so that
# doesn't happen. Actually dup2 the file handle so that writing
# to file descriptor 1, os.system, and so on work as expected too.
to_parent = os.fdopen(os.dup(sys.stdout.fileno()), 'ab')
os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
parser = optparse.OptionParser()
parser.add_option('--project_root',
action='store',
type='string',
dest='project_root')
parser.add_option('--build_file_name',
action='store',
type='string',
dest="build_file_name")
parser.add_option(
'--allow_empty_globs',
action='store_true',
dest='allow_empty_globs',
help=
'Tells the parser not to raise an error when glob returns no results.')
parser.add_option(
'--use_watchman_glob',
action='store_true',
dest='use_watchman_glob',
help=
'Invokes `watchman query` to get lists of files instead of globbing in-process.'
)
parser.add_option(
'--watchman_watch_root',
action='store',
type='string',
dest='watchman_watch_root',
help=
'Path to root of watchman watch as returned by `watchman watch-project`.'
)
parser.add_option(
'--watchman_socket_path',
action='store',
type='string',
dest='watchman_socket_path',
help=
'Path to Unix domain socket/named pipe as returned by `watchman get-sockname`.'
)
parser.add_option(
'--watchman_project_prefix',
action='store',
type='string',
dest='watchman_project_prefix',
help='Relative project prefix as returned by `watchman watch-project`.'
)
parser.add_option(
'--watchman_query_timeout_ms',
action='store',
type='int',
dest='watchman_query_timeout_ms',
help=
'Maximum time in milliseconds to wait for watchman query to respond.')
parser.add_option('--include', action='append', dest='include')
parser.add_option('--config',
help='BuckConfig settings available at parse time.')
parser.add_option('--ignore_paths', help='Paths that should be ignored.')
parser.add_option(
'--quiet',
action='store_true',
dest='quiet',
help='Stifles exception backtraces printed to stderr during parsing.')
parser.add_option('--ignore_buck_autodeps_files',
action='store_true',
help='Profile every buck file execution')
parser.add_option('--profile',
action='store_true',
help='Profile every buck file execution')
(options, args) = parser.parse_args()
# Even though project_root is absolute path, it may not be concise. For
# example, it might be like "C:\project\.\rule".
#
# Under cygwin, the project root will be invoked from buck as C:\path, but
# the cygwin python uses UNIX-style paths. They can be converted using
# cygpath, which is necessary because abspath will treat C:\path as a
# relative path.
options.project_root = cygwin_adjusted_path(options.project_root)
project_root = os.path.abspath(options.project_root)
watchman_client = None
watchman_error = None
if options.use_watchman_glob:
import pywatchman
client_args = {}
if options.watchman_query_timeout_ms is not None:
# pywatchman expects a timeout as a nonnegative floating-point
# value in seconds.
client_args['timeout'] = max(
0.0, options.watchman_query_timeout_ms / 1000.0)
if options.watchman_socket_path is not None:
client_args['sockpath'] = options.watchman_socket_path
client_args['transport'] = 'local'
watchman_client = pywatchman.client(**client_args)
watchman_error = pywatchman.WatchmanError
configs = {}
if options.config is not None:
with open(options.config, 'rb') as f:
for section, contents in bser.loads(f.read()).iteritems():
for field, value in contents.iteritems():
configs[(section, field)] = value
ignore_paths = None
if options.ignore_paths is not None:
with open(options.ignore_paths, 'rb') as f:
ignore_paths = [make_glob(i) for i in bser.loads(f.read())]
buildFileProcessor = BuildFileProcessor(project_root,
options.watchman_watch_root,
options.watchman_project_prefix,
options.build_file_name,
options.allow_empty_globs,
options.ignore_buck_autodeps_files,
watchman_client,
watchman_error,
implicit_includes=options.include
or [],
configs=configs,
ignore_paths=ignore_paths)
buildFileProcessor.install_builtins(__builtin__.__dict__)
# While processing, we'll write exceptions as diagnostic messages
# to the parent then re-raise them to crash the process. While
# doing so, we don't want Python's default unhandled exception
# behavior of writing to stderr.
orig_excepthook = None
if options.quiet:
orig_excepthook = sys.excepthook
sys.excepthook = silent_excepthook
for build_file in args:
process_with_diagnostics(build_file,
buildFileProcessor,
to_parent,
should_profile=options.profile)
# "for ... in sys.stdin" in Python 2.x hangs until stdin is closed.
for build_file in iter(sys.stdin.readline, ''):
process_with_diagnostics(build_file,
buildFileProcessor,
to_parent,
should_profile=options.profile)
if options.quiet:
sys.excepthook = orig_excepthook
# Python tries to flush/close stdout when it quits, and if there's a dead
# pipe on the other end, it will spit some warnings to stderr. This breaks
# tests sometimes. Prevent that by explicitly catching the error.
try:
to_parent.close()
except IOError:
pass
def roundtrip(self, val):
enc = bser.dumps(val)
print "# %s --> %s" % (val, enc.encode('hex'))
dec = bser.loads(enc)
self.assertEquals(val, dec)