def test_bser_encoding_failure(self): build_file_processor = self.create_build_file_processor(extra_funcs=[foo_rule]) fake_stdout = StringIO.StringIO() build_file = ProjectFile( self.project_root, path='BUCK', contents=( 'foo_rule(', ' name="foo",' ' srcs=[object()],' ')' )) self.write_file(build_file) with build_file_processor.with_builtins(__builtin__.__dict__): process_with_diagnostics( { 'buildFile': self.build_file_name, 'watchRoot': '', 'projectPrefix': self.project_root, }, build_file_processor, fake_stdout) result = fake_stdout.getvalue() decoded_result = bser.loads(result) self.assertEqual( [], decoded_result['values']) self.assertEqual( 'fatal', decoded_result['diagnostics'][0]['level']) self.assertEqual( 'parse', decoded_result['diagnostics'][0]['source'])
def test_bserInput(self): sockname = self.getSockPath() watchman_cmd = bser.dumps(['get-sockname']) cli_cmd = [ 'watchman', '--sockname={0}'.format(sockname), '--logfile=/BOGUS', '--statefile=/BOGUS', '--no-spawn', '--no-local', '-j', ] proc = subprocess.Popen(cli_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE) stdout, stderr = proc.communicate(input=watchman_cmd) self.assertEqual(proc.poll(), 0, stderr) # the response should be bser to match our input result = bser.loads(stdout) result_sockname = result['sockname'] if compat.PYTHON3: result_sockname = encoding.decode_local(result_sockname) self.assertEqual(result_sockname, sockname, binascii.hexlify(stdout).decode('ascii'))
def test_bserInput(self): sockpath = self.getSockPath() watchman_cmd = bser.dumps(["get-sockname"]) cli_cmd = [ os.environ.get("WATCHMAN_BINARY", "watchman"), "--unix-listener-path={0}".format(sockpath.unix_domain), "--named-pipe-path={0}".format(sockpath.named_pipe), "--logfile=/BOGUS", "--statefile=/BOGUS", "--no-spawn", "--no-local", "-j", ] proc = subprocess.Popen( cli_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE, ) stdout, stderr = proc.communicate(input=watchman_cmd) self.assertEqual(proc.poll(), 0, stderr) # the response should be bser to match our input result = bser.loads(stdout) result_sockname = result["unix_domain"] if compat.PYTHON3: result_sockname = encoding.decode_local(result_sockname) self.assertEqual( result_sockname, sockpath.unix_domain, binascii.hexlify(stdout).decode("ascii"), )
def _resolve_sockname_helper(): # if invoked via a trigger, watchman will set this env var; we # should use it unless explicitly set otherwise path = os.getenv('WATCHMAN_SOCK') if path: return path cmd = ['watchman', '--output-encoding=bser', 'get-sockname'] try: p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=os.name != 'nt') except OSError as e: raise WatchmanError('"watchman" executable not in PATH (%s)', e) stdout, stderr = p.communicate() exitcode = p.poll() if exitcode: raise WatchmanError('watchman exited with code %d' % exitcode) result = bser.loads(stdout) if 'error' in result: raise WatchmanError(str(result['error'])) return result['sockname']
def test_bserInput(self): sockname = self.getSockPath() watchman_cmd = bser.dumps(['get-sockname']) cli_cmd = [ 'watchman', '--sockname={}'.format(sockname), '--logfile=/BOGUS', '--statefile=/BOGUS', '--no-spawn', '--no-local', '-j', ] proc = subprocess.Popen(cli_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE) stdout, stderr = proc.communicate(input=watchman_cmd) self.assertEqual(proc.poll(), 0, stderr) # the response should be bser to match our input result = bser.loads(stdout) if PY2: self.assertEqual(result[b'sockname'], sockname, stdout.encode('hex')) if PY3: self.assertEqual(result[b'sockname'], sockname, binascii.hexlify(stdout))
def test_bserInput(self): sockname = self.getSockPath() watchman_cmd = bser.dumps(["get-sockname"]) cli_cmd = [ "watchman", "--sockname={0}".format(sockname), "--logfile=/BOGUS", "--statefile=/BOGUS", "--no-spawn", "--no-local", "-j", ] proc = subprocess.Popen( cli_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE, ) stdout, stderr = proc.communicate(input=watchman_cmd) self.assertEqual(proc.poll(), 0, stderr) # the response should be bser to match our input result = bser.loads(stdout) result_sockname = result["sockname"] if compat.PYTHON3: result_sockname = encoding.decode_local(result_sockname) self.assertEqual(result_sockname, sockname, binascii.hexlify(stdout).decode("ascii"))
def munged(self, val, munged): enc = bser.dumps(val) if isinstance(val, unicode): print "# %s --> %s" % (val.encode('utf-8'), enc.encode('hex')) else: print "# %s --> %s" % (val, enc.encode('hex')) dec = bser.loads(enc) self.assertEquals(munged, dec)
def _loads(self, response): """ Parse the BSER packet """ return bser.loads( response, True, value_encoding=encoding.get_local_encoding(), value_errors=encoding.default_local_errors, )
def test_bserInput(self): sockname = self.getSockPath() watchman_cmd = bser.dumps(["get-sockname"]) cli_cmd = ["watchman", "--sockname={}".format(sockname), "--no-spawn", "--no-local", "-j"] proc = subprocess.Popen(cli_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE) stdout, stderr = proc.communicate(input=watchman_cmd) self.assertEqual(proc.poll(), 0, stderr) # the response should be bser to match our input result = bser.loads(stdout) self.assertEqual(result["sockname"], sockname, stdout.encode("hex"))
def test_watchman_glob_failure_raises_diagnostic_with_stack(self): class FakeWatchmanClient: def __init__(self): self.query_invoked = False def query(self, *args): self.query_invoked = True raise WatchmanError("Nobody watches the watchmen") def close(self): pass self.watchman_client = FakeWatchmanClient() build_file = ProjectFile( self.project_root, path='BUCK', contents=( 'foo_rule(', ' name="foo",' ' srcs=glob(["*.java"]),', ')' )) java_file = ProjectFile(self.project_root, path='Foo.java', contents=()) self.write_files(build_file, java_file) build_file_processor = self.create_build_file_processor(extra_funcs=[foo_rule]) diagnostics = [] rules = [] fake_stdout = StringIO.StringIO() with build_file_processor.with_builtins(__builtin__.__dict__): self.assertRaises( WatchmanError, process_with_diagnostics, { 'buildFile': self.build_file_name, 'watchRoot': '', 'projectPrefix': self.project_root, }, build_file_processor, fake_stdout) self.assertTrue(self.watchman_client.query_invoked) result = fake_stdout.getvalue() decoded_result = bser.loads(result) self.assertEqual([], decoded_result['values']) self.assertEqual(1, len(decoded_result['diagnostics'])) diagnostic = decoded_result['diagnostics'][0] self.assertEqual('fatal', diagnostic['level']) self.assertEqual('parse', diagnostic['source']) self.assertEqual('Nobody watches the watchmen', diagnostic['message']) exception = diagnostic['exception'] self.assertEqual('WatchmanError', exception['type']) self.assertEqual('Nobody watches the watchmen', exception['value']) self.assertTrue(len(exception['traceback']) > 0)
def test_template(self): # since we can't generate the template bser output, here's a # a blob from the C test suite in watchman templ = "\x00\x01\x03\x28" + \ "\x0b\x00\x03\x02\x02\x03\x04\x6e\x61\x6d\x65\x02" + \ "\x03\x03\x61\x67\x65\x03\x03\x02\x03\x04\x66\x72" + \ "\x65\x64\x03\x14\x02\x03\x04\x70\x65\x74\x65\x03" + \ "\x1e\x0c\x03\x19" dec = bser.loads(templ) exp = [ {"name": "fred", "age": 20}, {"name": "pete", "age": 30}, {"age": 25} ] self.assertEquals(exp, dec)
def test_watchman_glob_failure_raises_diagnostic_with_stack(self): class FakeWatchmanClient: def __init__(self): self.query_invoked = False def query(self, *args): self.query_invoked = True raise WatchmanError("Nobody watches the watchmen") def close(self): pass self.watchman_client = FakeWatchmanClient() build_file = ProjectFile(self.project_root, path='BUCK', contents=('foo_rule(', ' name="foo",' ' srcs=glob(["*.java"]),', ')')) java_file = ProjectFile(self.project_root, path='Foo.java', contents=()) self.write_files(build_file, java_file) build_file_processor = self.create_build_file_processor( extra_funcs=[foo_rule]) diagnostics = [] rules = [] fake_stdout = StringIO.StringIO() with build_file_processor.with_builtins(__builtin__.__dict__): self.assertRaises( WatchmanError, process_with_diagnostics, { 'buildFile': self.build_file_name, 'watchRoot': '', 'projectPrefix': self.project_root, }, build_file_processor, fake_stdout) self.assertTrue(self.watchman_client.query_invoked) result = fake_stdout.getvalue() decoded_result = bser.loads(result) self.assertEqual([], decoded_result['values']) self.assertEqual(1, len(decoded_result['diagnostics'])) diagnostic = decoded_result['diagnostics'][0] self.assertEqual('fatal', diagnostic['level']) self.assertEqual('parse', diagnostic['source']) self.assertEqual('Nobody watches the watchmen', diagnostic['message']) exception = diagnostic['exception'] self.assertEqual('WatchmanError', exception['type']) self.assertEqual('Nobody watches the watchmen', exception['value']) self.assertTrue(len(exception['traceback']) > 0)
def test_bser_encoding_failure(self): build_file_processor = self.create_build_file_processor(extra_funcs=[foo_rule]) build_file_processor.install_builtins(__builtin__.__dict__) fake_stdout = StringIO.StringIO() build_file = ProjectFile( self.project_root, path="BUCK", contents=("foo_rule(", ' name="foo",' " srcs=[object()]," ")") ) self.write_file(build_file) process_with_diagnostics( {"buildFile": self.build_file_name, "watchRoot": "", "projectPrefix": self.project_root}, build_file_processor, fake_stdout, ) result = fake_stdout.getvalue() decoded_result = bser.loads(result) self.assertEqual([], decoded_result["values"]) self.assertEqual("fatal", decoded_result["diagnostics"][0]["level"]) self.assertEqual("parse", decoded_result["diagnostics"][0]["source"])
def test_bserInput(self): sockname = self.getSockPath() watchman_cmd = bser.dumps(['get-sockname']) cli_cmd = [ 'watchman', '--sockname={}'.format(sockname), '--no-spawn', '--no-local', '-j', ] proc = subprocess.Popen(cli_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE) stdout, stderr = proc.communicate(input=watchman_cmd) self.assertEqual(proc.poll(), 0, stderr) # the response should be bser to match our input result = bser.loads(stdout) self.assertEqual(result['sockname'], sockname, stdout.encode('hex'))
def roundtrip(self, val): enc = bser.dumps(val) print "# %s --> %s" % (val, enc.encode('hex')) dec = bser.loads(enc) self.assertEquals(val, dec)
def main(): # Our parent expects to read BSER from our stdout, so if anyone # uses print, buck will complain with a helpful "but I wanted an # array!" message and quit. Redirect stdout to stderr so that # doesn't happen. Actually dup2 the file handle so that writing # to file descriptor 1, os.system, and so on work as expected too. to_parent = os.fdopen(os.dup(sys.stdout.fileno()), 'ab') os.dup2(sys.stderr.fileno(), sys.stdout.fileno()) parser = optparse.OptionParser() parser.add_option( '--project_root', action='store', type='string', dest='project_root') parser.add_option( '--build_file_name', action='store', type='string', dest="build_file_name") parser.add_option( '--allow_empty_globs', action='store_true', dest='allow_empty_globs', help='Tells the parser not to raise an error when glob returns no results.') parser.add_option( '--use_watchman_glob', action='store_true', dest='use_watchman_glob', help='Invokes `watchman query` to get lists of files instead of globbing in-process.') parser.add_option( '--watchman_watch_root', action='store', type='string', dest='watchman_watch_root', help='Path to root of watchman watch as returned by `watchman watch-project`.') parser.add_option( '--watchman_socket_path', action='store', type='string', dest='watchman_socket_path', help='Path to Unix domain socket/named pipe as returned by `watchman get-sockname`.') parser.add_option( '--watchman_project_prefix', action='store', type='string', dest='watchman_project_prefix', help='Relative project prefix as returned by `watchman watch-project`.') parser.add_option( '--watchman_query_timeout_ms', action='store', type='int', dest='watchman_query_timeout_ms', help='Maximum time in milliseconds to wait for watchman query to respond.') parser.add_option( '--include', action='append', dest='include') parser.add_option( '--config', help='BuckConfig settings available at parse time.') parser.add_option( '--quiet', action='store_true', dest='quiet', help='Stifles exception backtraces printed to stderr during parsing.') parser.add_option( '--ignore_buck_autodeps_files', action='store_true', help='Profile every buck file execution') parser.add_option( '--profile', action='store_true', help='Profile every buck file execution') (options, args) = parser.parse_args() # Even though project_root is absolute path, it may not be concise. For # example, it might be like "C:\project\.\rule". # # Under cygwin, the project root will be invoked from buck as C:\path, but # the cygwin python uses UNIX-style paths. They can be converted using # cygpath, which is necessary because abspath will treat C:\path as a # relative path. options.project_root = cygwin_adjusted_path(options.project_root) project_root = os.path.abspath(options.project_root) watchman_client = None watchman_error = None if options.use_watchman_glob: import pywatchman client_args = {} if options.watchman_query_timeout_ms is not None: # pywatchman expects a timeout as a nonnegative floating-point # value in seconds. client_args['timeout'] = max(0.0, options.watchman_query_timeout_ms / 1000.0) if options.watchman_socket_path is not None: client_args['sockpath'] = options.watchman_socket_path client_args['transport'] = 'local' watchman_client = pywatchman.client(**client_args) watchman_error = pywatchman.WatchmanError configs = {} if options.config is not None: with open(options.config, 'rb') as f: for section, contents in bser.loads(f.read()).iteritems(): for field, value in contents.iteritems(): configs[(section, field)] = value buildFileProcessor = BuildFileProcessor( project_root, options.watchman_watch_root, options.watchman_project_prefix, options.build_file_name, options.allow_empty_globs, options.ignore_buck_autodeps_files, watchman_client, watchman_error, implicit_includes=options.include or [], configs=configs) buildFileProcessor.install_builtins(__builtin__.__dict__) # While processing, we'll write exceptions as diagnostic messages # to the parent then re-raise them to crash the process. While # doing so, we don't want Python's default unhandled exception # behavior of writing to stderr. orig_excepthook = None if options.quiet: orig_excepthook = sys.excepthook sys.excepthook = silent_excepthook for build_file in args: process_with_diagnostics(build_file, buildFileProcessor, to_parent, should_profile=options.profile) # "for ... in sys.stdin" in Python 2.x hangs until stdin is closed. for build_file in iter(sys.stdin.readline, ''): process_with_diagnostics(build_file, buildFileProcessor, to_parent, should_profile=options.profile) if options.quiet: sys.excepthook = orig_excepthook # Python tries to flush/close stdout when it quits, and if there's a dead # pipe on the other end, it will spit some warnings to stderr. This breaks # tests sometimes. Prevent that by explicitly catching the error. try: to_parent.close() except IOError: pass
def main(): # Our parent expects to read BSER from our stdout, so if anyone # uses print, buck will complain with a helpful "but I wanted an # array!" message and quit. Redirect stdout to stderr so that # doesn't happen. Actually dup2 the file handle so that writing # to file descriptor 1, os.system, and so on work as expected too. to_parent = os.fdopen(os.dup(sys.stdout.fileno()), 'ab') os.dup2(sys.stderr.fileno(), sys.stdout.fileno()) parser = optparse.OptionParser() parser.add_option('--project_root', action='store', type='string', dest='project_root') parser.add_option('--build_file_name', action='store', type='string', dest="build_file_name") parser.add_option( '--allow_empty_globs', action='store_true', dest='allow_empty_globs', help= 'Tells the parser not to raise an error when glob returns no results.') parser.add_option( '--use_watchman_glob', action='store_true', dest='use_watchman_glob', help= 'Invokes `watchman query` to get lists of files instead of globbing in-process.' ) parser.add_option( '--watchman_watch_root', action='store', type='string', dest='watchman_watch_root', help= 'Path to root of watchman watch as returned by `watchman watch-project`.' ) parser.add_option( '--watchman_socket_path', action='store', type='string', dest='watchman_socket_path', help= 'Path to Unix domain socket/named pipe as returned by `watchman get-sockname`.' ) parser.add_option( '--watchman_project_prefix', action='store', type='string', dest='watchman_project_prefix', help='Relative project prefix as returned by `watchman watch-project`.' ) parser.add_option( '--watchman_query_timeout_ms', action='store', type='int', dest='watchman_query_timeout_ms', help= 'Maximum time in milliseconds to wait for watchman query to respond.') parser.add_option('--include', action='append', dest='include') parser.add_option('--config', help='BuckConfig settings available at parse time.') parser.add_option('--ignore_paths', help='Paths that should be ignored.') parser.add_option( '--quiet', action='store_true', dest='quiet', help='Stifles exception backtraces printed to stderr during parsing.') parser.add_option('--ignore_buck_autodeps_files', action='store_true', help='Profile every buck file execution') parser.add_option('--profile', action='store_true', help='Profile every buck file execution') (options, args) = parser.parse_args() # Even though project_root is absolute path, it may not be concise. For # example, it might be like "C:\project\.\rule". # # Under cygwin, the project root will be invoked from buck as C:\path, but # the cygwin python uses UNIX-style paths. They can be converted using # cygpath, which is necessary because abspath will treat C:\path as a # relative path. options.project_root = cygwin_adjusted_path(options.project_root) project_root = os.path.abspath(options.project_root) watchman_client = None watchman_error = None if options.use_watchman_glob: import pywatchman client_args = {} if options.watchman_query_timeout_ms is not None: # pywatchman expects a timeout as a nonnegative floating-point # value in seconds. client_args['timeout'] = max( 0.0, options.watchman_query_timeout_ms / 1000.0) if options.watchman_socket_path is not None: client_args['sockpath'] = options.watchman_socket_path client_args['transport'] = 'local' watchman_client = pywatchman.client(**client_args) watchman_error = pywatchman.WatchmanError configs = {} if options.config is not None: with open(options.config, 'rb') as f: for section, contents in bser.loads(f.read()).iteritems(): for field, value in contents.iteritems(): configs[(section, field)] = value ignore_paths = None if options.ignore_paths is not None: with open(options.ignore_paths, 'rb') as f: ignore_paths = [make_glob(i) for i in bser.loads(f.read())] buildFileProcessor = BuildFileProcessor(project_root, options.watchman_watch_root, options.watchman_project_prefix, options.build_file_name, options.allow_empty_globs, options.ignore_buck_autodeps_files, watchman_client, watchman_error, implicit_includes=options.include or [], configs=configs, ignore_paths=ignore_paths) buildFileProcessor.install_builtins(__builtin__.__dict__) # While processing, we'll write exceptions as diagnostic messages # to the parent then re-raise them to crash the process. While # doing so, we don't want Python's default unhandled exception # behavior of writing to stderr. orig_excepthook = None if options.quiet: orig_excepthook = sys.excepthook sys.excepthook = silent_excepthook for build_file in args: process_with_diagnostics(build_file, buildFileProcessor, to_parent, should_profile=options.profile) # "for ... in sys.stdin" in Python 2.x hangs until stdin is closed. for build_file in iter(sys.stdin.readline, ''): process_with_diagnostics(build_file, buildFileProcessor, to_parent, should_profile=options.profile) if options.quiet: sys.excepthook = orig_excepthook # Python tries to flush/close stdout when it quits, and if there's a dead # pipe on the other end, it will spit some warnings to stderr. This breaks # tests sometimes. Prevent that by explicitly catching the error. try: to_parent.close() except IOError: pass