def start(self):
args = [
'watchman',
'--foreground',
'--log-level=2',
]
args.extend(self.get_state_args())
env = os.environ.copy()
env["WATCHMAN_CONFIG_FILE"] = self.cfg_file
self.proc = subprocess.Popen(args,
env=env,
stdin=None,
stdout=self.cli_log_file,
stderr=self.cli_log_file)
# wait for it to come up
deadline = time.time() + self.start_timeout
while time.time() < deadline:
try:
client = pywatchman.client(sockpath=self.sock_file)
self.pid = client.query('get-pid')['pid']
break
except Exception as e:
t, val, tb = sys.exc_info()
time.sleep(0.1)
finally:
client.close()
if self.pid is None:
pywatchman.compat.reraise(t, val, tb)
def start(self):
args = [
'watchman',
'--foreground',
'--sockname={0}'.format(self.sock_file),
'--logfile={0}'.format(self.log_file_name),
'--statefile={0}'.format(self.state_file),
'--log-level=2',
]
env = os.environ.copy()
env["WATCHMAN_CONFIG_FILE"] = self.cfg_file
self.proc = subprocess.Popen(args,
env=env,
stdin=None,
stdout=self.log_file,
stderr=self.log_file)
# wait for it to come up
last_err = None
for i in range(1, 10):
try:
client = pywatchman.client(sockpath=self.sock_file)
self.pid = client.query('get-pid')['pid']
break
except Exception as e:
t, val, tb = sys.exc_info()
last_err = ''.join(traceback.format_exception(t, val, tb))
time.sleep(0.1)
finally:
client.close()
if self.pid is None:
raise Exception(last_err)
def start(self):
args = [
'watchman',
'--foreground',
'--sockname={}'.format(self.sock_file),
'--logfile={}'.format(self.log_file_name),
'--statefile={}'.format(self.state_file),
'--log-level=2',
]
env = os.environ.copy()
env["WATCHMAN_CONFIG_FILE"] = self.cfg_file
self.proc = subprocess.Popen(args,
env=env,
stdin=None,
stdout=self.log_file,
stderr=self.log_file)
# wait for it to come up
last_err = None
for i in xrange(1, 10):
try:
client = pywatchman.client(sockpath=self.sock_file)
self.pid = client.query('get-pid')['pid']
break
except Exception as e:
last_err = e
time.sleep(0.1)
if not self.pid:
raise last_err
def start(self):
args = [self.watchmanBinary(), "--foreground", "--log-level=2"]
args.extend(self.get_state_args())
env = os.environ.copy()
env["WATCHMAN_CONFIG_FILE"] = self.cfg_file
with open(self.cli_log_file_name, "w+") as cli_log_file:
self.proc = subprocess.Popen(
args, env=env, stdin=None, stdout=cli_log_file, stderr=cli_log_file
)
if self.debug_watchman:
print("Watchman instance PID: " + str(self.proc.pid))
if pywatchman.compat.PYTHON3:
user_input = input
else:
user_input = raw_input # noqa:F821
user_input("Press Enter to continue...")
# wait for it to come up
deadline = time.time() + self.start_timeout
while time.time() < deadline:
try:
client = pywatchman.client(sockpath=self.sock_file)
self.pid = client.query("get-pid")["pid"]
break
except pywatchman.SocketConnectError:
t, val, tb = sys.exc_info()
time.sleep(0.1)
finally:
client.close()
if self.pid is None:
# self.proc didn't come up: wait for it to die
self.stop()
pywatchman.compat.reraise(t, val, tb)
def test_client(self):
# verify that we can talk to the instance set up by the harness
# we're just verifying that the socket set in the environment works
# and that we can understand the result
sock = os.getenv('WATCHMAN_SOCK')
c = pywatchman.client()
res = c.query('get-sockname')
self.assertEquals(sock, res['sockname'])
def getClient(self):
if not hasattr(self, 'client'):
self.client = pywatchman.client(
transport=self.transport,
sendEncoding=self.encoding,
recvEncoding=self.encoding,
sockpath=WatchmanInstance.getSharedInstance().getSockPath())
return self.client
def getClient(self):
if not hasattr(self, 'client'):
self.client = pywatchman.client(
# ASAN-enabled builds can be slower enough that we hit timeouts
# with the default of 1 second
timeout=3.0,
transport=self.transport,
sendEncoding=self.encoding,
recvEncoding=self.encoding,
sockpath=WatchmanInstance.getSharedInstance().getSockPath())
return self.client
def getClient(self, inst=None, replace_cached=False, no_cache=False):
if inst or not hasattr(self, "client") or no_cache:
client = pywatchman.client(
timeout=self.socketTimeout,
transport=self.transport,
sendEncoding=self.encoding,
recvEncoding=self.encoding,
sockpath=(inst or WatchmanInstance.getSharedInstance()).getSockPath(),
)
if (not inst or replace_cached) and not no_cache:
# only cache the client if it points to the shared instance
self.client = client
self.addCleanup(lambda: self.__clearClient())
return client
return self.client
def check_availability(cls):
if not pywatchman:
raise WatchmanUnavailable('pywatchman not installed.')
client = pywatchman.client(timeout=0.1)
try:
result = client.capabilityCheck()
except Exception:
# The service is down?
raise WatchmanUnavailable('Cannot connect to the watchman service.')
version = get_version_tuple(result['version'])
# Watchman 4.9 includes multiple improvements to watching project
# directories as well as case insensitive filesystems.
logger.debug('Watchman version %s', version)
if version < (4, 9):
raise WatchmanUnavailable('Watchman 4.9 or later is required.')
def getClient(self, inst=None):
if inst or not hasattr(self, 'client'):
client = pywatchman.client(
# ASAN-enabled builds can be slower enough that we hit timeouts
# with the default of 1 second
timeout=3.0,
transport=self.transport,
sendEncoding=self.encoding,
recvEncoding=self.encoding,
sockpath=(inst or
WatchmanInstance.getSharedInstance()).getSockPath())
if not inst:
# only cache the client if it points to the shared instance
self.client = client
return client
return self.client
def test_transport_error(self):
buf = '{"foo":"bar"}'
failAfterBytesRead = 5
class FakeFailingTransport(Transport):
def __init__(self, sockpath, timeout):
self.readBuf = buf
self.readBufPos = 0
self.writeBuf = []
self.closed = False
def close(self):
self.closed = True
def readBytes(self, size):
readEnd = self.readBufPos + size
if readEnd > failAfterBytesRead:
raise IOError(23, "fnord")
elif readEnd > len(self.readBuf):
return ""
read = self.readBuf[self.readBufPos : self.readBufPos + size]
self.readBufPos += size
return read
def write(self, buf):
self.writeBuf.extend(buf)
c = client(
sockpath="",
transport=FakeFailingTransport,
sendEncoding="json",
recvEncoding="json",
)
try:
c.query("foobarbaz")
self.assertTrue(False, "expected a WatchmanError")
except WatchmanError as e:
self.assertIn(
"I/O error communicating with watchman daemon: "
+ "errno=23 errmsg=fnord, while executing "
+ "('foobarbaz',)",
str(e),
)
except Exception as e:
self.assertTrue(False, "expected a WatchmanError, but got " + str(e))
def start(self):
args = [
self.watchmanBinary(),
'--foreground',
'--log-level=2',
]
args.extend(self.get_state_args())
env = os.environ.copy()
env["WATCHMAN_CONFIG_FILE"] = self.cfg_file
self.proc = subprocess.Popen(args,
env=env,
stdin=None,
stdout=self.cli_log_file,
stderr=self.cli_log_file)
if self.debug_watchman:
print('Watchman instance PID: ' + str(self.proc.pid))
if pywatchman.compat.PYTHON3:
user_input = input
else:
user_input = raw_input
user_input('Press Enter to continue...')
# wait for it to come up
deadline = time.time() + self.start_timeout
while time.time() < deadline:
try:
client = pywatchman.client(sockpath=self.sock_file)
self.pid = client.query('get-pid')['pid']
break
except Exception as e:
t, val, tb = sys.exc_info()
time.sleep(0.1)
finally:
client.close()
if self.pid is None:
# self.proc didn't come up: wait for it to die
self.stop()
pywatchman.compat.reraise(t, val, tb)
def client(self):
return pywatchman.client(timeout=self.client_timeout)
def runner():
global results_queue
global tests_queue
broken = False
try:
# Start up a shared watchman instance for the tests.
inst = WatchmanInstance.Instance({"watcher": args.watcher},
debug_watchman=args.debug_watchman)
inst.start()
# Allow tests to locate this default instance
WatchmanInstance.setSharedInstance(inst)
if has_asyncio:
# Each thread will have its own event loop
asyncio.set_event_loop(asyncio.new_event_loop())
except Exception as e:
print("while starting watchman: %s" % str(e))
traceback.print_exc()
broken = True
while not broken:
test = tests_queue.get()
try:
if test == "terminate":
break
if Interrupt.wasInterrupted() or broken:
continue
result = None
for attempt in range(0, args.retry_flaky + 1):
# Check liveness of the server
try:
client = pywatchman.client(timeout=3.0,
sockpath=inst.getSockPath())
client.query("version")
client.close()
except Exception as exc:
print(
"Failed to connect to watchman server: %s; starting a new one"
% exc)
try:
inst.stop()
except Exception:
pass
try:
inst = WatchmanInstance.Instance(
{"watcher": args.watcher},
debug_watchman=args.debug_watchman,
)
inst.start()
# Allow tests to locate this default instance
WatchmanInstance.setSharedInstance(inst)
except Exception as e:
print("while starting watchman: %s" % str(e))
traceback.print_exc()
broken = True
continue
try:
result = Result()
result.setAttemptNumber(attempt)
if hasattr(test, "setAttemptNumber"):
test.setAttemptNumber(attempt)
test.run(result)
if hasattr(
test,
"setAttemptNumber") and not result.wasSuccessful():
# Facilitate retrying this possibly flaky test
continue
break
except Exception as e:
print(e)
if hasattr(
test,
"setAttemptNumber") and not result.wasSuccessful():
# Facilitate retrying this possibly flaky test
continue
if (not result.wasSuccessful() and "TRAVIS" in os.environ
and hasattr(test, "dumpLogs")):
test.dumpLogs()
results_queue.put(result)
finally:
tests_queue.task_done()
if not broken:
inst.stop()
def input_changes(self, verbose=True):
'''
Return an iterator of `FasterBuildChange` instances as inputs
to the faster build system change.
'''
# TODO: provide the debug diagnostics we want: this print is
# not immediately before the watch.
if verbose:
print_line('watch', 'Connecting to watchman')
# TODO: figure out why a large timeout is required for the
# client, and a robust strategy for retrying timed out
# requests.
self.client = pywatchman.client(timeout=5.0)
try:
if verbose:
print_line('watch', 'Checking watchman capabilities')
# TODO: restrict these capabilities to the minimal set.
self.client.capabilityCheck(required=[
'clock-sync-timeout',
'cmd-watch-project',
'term-dirname',
'wildmatch',
])
if verbose:
print_line('watch', 'Subscribing to {}'.format(self.config_environment.topsrcdir))
self.subscribe_to_topsrcdir()
if verbose:
print_line('watch', 'Watching {}'.format(self.config_environment.topsrcdir))
input_to_outputs = self.file_copier.input_to_outputs_tree()
for input, outputs in input_to_outputs.items():
if not outputs:
raise Exception("Refusing to watch input ({}) with no outputs".format(input))
while True:
try:
_watch_result = self.client.receive()
changed = self.changed_files()
if not changed:
continue
result = FasterBuildChange()
for change in changed:
if change in input_to_outputs:
result.input_to_outputs[change] = set(input_to_outputs[change])
else:
result.unrecognized.add(change)
for input, outputs in result.input_to_outputs.items():
for output in outputs:
if output not in result.output_to_inputs:
result.output_to_inputs[output] = set()
result.output_to_inputs[output].add(input)
yield result
except pywatchman.SocketTimeout:
# Let's check to see if we're still functional.
_version = self.client.query('version')
except pywatchman.CommandError as e:
# Abstract away pywatchman errors.
raise FasterBuildException(e, 'Command error using pywatchman to watch {}'.format(
self.config_environment.topsrcdir))
except pywatchman.SocketTimeout as e:
# Abstract away pywatchman errors.
raise FasterBuildException(e, 'Socket timeout using pywatchman to watch {}'.format(
self.config_environment.topsrcdir))
finally:
self.client.close()
def getClient(self):
if not hasattr(self, 'client'):
self.client = pywatchman.client()
return self.client
def client(self):
return pywatchman.client(timeout=self.client_timeout)
def main():
# Our parent expects to read BSER from our stdout, so if anyone
# uses print, buck will complain with a helpful "but I wanted an
# array!" message and quit. Redirect stdout to stderr so that
# doesn't happen. Actually dup2 the file handle so that writing
# to file descriptor 1, os.system, and so on work as expected too.
to_parent = os.fdopen(os.dup(sys.stdout.fileno()), 'ab')
os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
parser = optparse.OptionParser()
parser.add_option('--project_root',
action='store',
type='string',
dest='project_root')
parser.add_option('--build_file_name',
action='store',
type='string',
dest="build_file_name")
parser.add_option(
'--allow_empty_globs',
action='store_true',
dest='allow_empty_globs',
help=
'Tells the parser not to raise an error when glob returns no results.')
parser.add_option(
'--use_watchman_glob',
action='store_true',
dest='use_watchman_glob',
help=
'Invokes `watchman query` to get lists of files instead of globbing in-process.'
)
parser.add_option(
'--watchman_watch_root',
action='store',
type='string',
dest='watchman_watch_root',
help=
'Path to root of watchman watch as returned by `watchman watch-project`.'
)
parser.add_option(
'--watchman_socket_path',
action='store',
type='string',
dest='watchman_socket_path',
help=
'Path to Unix domain socket/named pipe as returned by `watchman get-sockname`.'
)
parser.add_option(
'--watchman_project_prefix',
action='store',
type='string',
dest='watchman_project_prefix',
help='Relative project prefix as returned by `watchman watch-project`.'
)
parser.add_option(
'--watchman_query_timeout_ms',
action='store',
type='int',
dest='watchman_query_timeout_ms',
help=
'Maximum time in milliseconds to wait for watchman query to respond.')
parser.add_option('--include', action='append', dest='include')
parser.add_option('--config',
help='BuckConfig settings available at parse time.')
parser.add_option(
'--quiet',
action='store_true',
dest='quiet',
help='Stifles exception backtraces printed to stderr during parsing.')
parser.add_option('--ignore_buck_autodeps_files',
action='store_true',
help='Profile every buck file execution')
parser.add_option('--profile',
action='store_true',
help='Profile every buck file execution')
(options, args) = parser.parse_args()
# Even though project_root is absolute path, it may not be concise. For
# example, it might be like "C:\project\.\rule".
#
# Under cygwin, the project root will be invoked from buck as C:\path, but
# the cygwin python uses UNIX-style paths. They can be converted using
# cygpath, which is necessary because abspath will treat C:\path as a
# relative path.
options.project_root = cygwin_adjusted_path(options.project_root)
project_root = os.path.abspath(options.project_root)
watchman_client = None
watchman_error = None
if options.use_watchman_glob:
import pywatchman
client_args = {}
if options.watchman_query_timeout_ms is not None:
# pywatchman expects a timeout as a nonnegative floating-point
# value in seconds.
client_args['timeout'] = max(
0.0, options.watchman_query_timeout_ms / 1000.0)
if options.watchman_socket_path is not None:
client_args['sockpath'] = options.watchman_socket_path
client_args['transport'] = 'local'
watchman_client = pywatchman.client(**client_args)
watchman_error = pywatchman.WatchmanError
configs = {}
if options.config is not None:
with open(options.config, 'rb') as f:
for section, contents in bser.loads(f.read()).iteritems():
for field, value in contents.iteritems():
configs[(section, field)] = value
buildFileProcessor = BuildFileProcessor(project_root,
options.watchman_watch_root,
options.watchman_project_prefix,
options.build_file_name,
options.allow_empty_globs,
options.ignore_buck_autodeps_files,
watchman_client,
watchman_error,
implicit_includes=options.include
or [],
configs=configs)
buildFileProcessor.install_builtins(__builtin__.__dict__)
# While processing, we'll write exceptions as diagnostic messages
# to the parent then re-raise them to crash the process. While
# doing so, we don't want Python's default unhandled exception
# behavior of writing to stderr.
orig_excepthook = None
if options.quiet:
orig_excepthook = sys.excepthook
sys.excepthook = silent_excepthook
for build_file in args:
process_with_diagnostics(build_file,
buildFileProcessor,
to_parent,
should_profile=options.profile)
# "for ... in sys.stdin" in Python 2.x hangs until stdin is closed.
for build_file in iter(sys.stdin.readline, ''):
process_with_diagnostics(build_file,
buildFileProcessor,
to_parent,
should_profile=options.profile)
if options.quiet:
sys.excepthook = orig_excepthook
# Python tries to flush/close stdout when it quits, and if there's a dead
# pipe on the other end, it will spit some warnings to stderr. This breaks
# tests sometimes. Prevent that by explicitly catching the error.
try:
to_parent.close()
except IOError:
pass
def main():
# Our parent expects to read JSON from our stdout, so if anyone
# uses print, buck will complain with a helpful "but I wanted an
# array!" message and quit. Redirect stdout to stderr so that
# doesn't happen. Actually dup2 the file handle so that writing
# to file descriptor 1, os.system, and so on work as expected too.
to_parent = os.fdopen(os.dup(sys.stdout.fileno()), 'a')
os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
parser = optparse.OptionParser()
parser.add_option(
'--project_root',
action='store',
type='string',
dest='project_root')
parser.add_option(
'--build_file_name',
action='store',
type='string',
dest="build_file_name")
parser.add_option(
'--allow_empty_globs',
action='store_true',
dest='allow_empty_globs',
help='Tells the parser not to raise an error when glob returns no results.')
parser.add_option(
'--use_watchman_glob',
action='store_true',
dest='use_watchman_glob',
help='Invokes `watchman query` to get lists of files instead of globbing in-process.')
parser.add_option(
'--watchman_watch_root',
action='store',
type='string',
dest='watchman_watch_root',
help='Path to root of watchman watch as returned by `watchman watch-project`.')
parser.add_option(
'--watchman_project_prefix',
action='store',
type='string',
dest='watchman_project_prefix',
help='Relative project prefix as returned by `watchman watch-project`.')
parser.add_option(
'--watchman_query_timeout_ms',
action='store',
type='int',
dest='watchman_query_timeout_ms',
help='Maximum time in milliseconds to wait for watchman query to respond.')
parser.add_option(
'--include',
action='append',
dest='include')
(options, args) = parser.parse_args()
# Even though project_root is absolute path, it may not be concise. For
# example, it might be like "C:\project\.\rule".
#
# Under cygwin, the project root will be invoked from buck as C:\path, but
# the cygwin python uses UNIX-style paths. They can be converted using
# cygpath, which is necessary because abspath will treat C:\path as a
# relative path.
options.project_root = cygwin_adjusted_path(options.project_root)
project_root = os.path.abspath(options.project_root)
watchman_client = None
watchman_error = None
output_format = 'JSON'
output_encode = lambda val: json.dumps(val, sort_keys=True)
if options.use_watchman_glob:
import pywatchman
client_args = {}
if options.watchman_query_timeout_ms is not None:
# pywatchman expects a timeout as a nonnegative floating-point
# value in seconds.
client_args['timeout'] = max(0.0, options.watchman_query_timeout_ms / 1000.0)
watchman_client = pywatchman.client(**client_args)
watchman_error = pywatchman.WatchmanError
try:
import pywatchman.bser as bser
except ImportError, e:
import pywatchman.pybser as bser
output_format = 'BSER'
output_encode = lambda val: bser.dumps(val)
def main():
# Our parent expects to read BSER from our stdout, so if anyone
# uses print, buck will complain with a helpful "but I wanted an
# array!" message and quit. Redirect stdout to stderr so that
# doesn't happen. Actually dup2 the file handle so that writing
# to file descriptor 1, os.system, and so on work as expected too.
to_parent = os.fdopen(os.dup(sys.stdout.fileno()), "a")
os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
parser = optparse.OptionParser()
parser.add_option("--project_root", action="store", type="string", dest="project_root")
parser.add_option("--build_file_name", action="store", type="string", dest="build_file_name")
parser.add_option(
"--allow_empty_globs",
action="store_true",
dest="allow_empty_globs",
help="Tells the parser not to raise an error when glob returns no results.",
)
parser.add_option(
"--use_watchman_glob",
action="store_true",
dest="use_watchman_glob",
help="Invokes `watchman query` to get lists of files instead of globbing in-process.",
)
parser.add_option(
"--watchman_watch_root",
action="store",
type="string",
dest="watchman_watch_root",
help="Path to root of watchman watch as returned by `watchman watch-project`.",
)
parser.add_option(
"--watchman_project_prefix",
action="store",
type="string",
dest="watchman_project_prefix",
help="Relative project prefix as returned by `watchman watch-project`.",
)
parser.add_option(
"--watchman_query_timeout_ms",
action="store",
type="int",
dest="watchman_query_timeout_ms",
help="Maximum time in milliseconds to wait for watchman query to respond.",
)
parser.add_option("--include", action="append", dest="include")
(options, args) = parser.parse_args()
# Even though project_root is absolute path, it may not be concise. For
# example, it might be like "C:\project\.\rule".
#
# Under cygwin, the project root will be invoked from buck as C:\path, but
# the cygwin python uses UNIX-style paths. They can be converted using
# cygpath, which is necessary because abspath will treat C:\path as a
# relative path.
options.project_root = cygwin_adjusted_path(options.project_root)
project_root = os.path.abspath(options.project_root)
watchman_client = None
watchman_error = None
if options.use_watchman_glob:
import pywatchman
client_args = {}
if options.watchman_query_timeout_ms is not None:
# pywatchman expects a timeout as a nonnegative floating-point
# value in seconds.
client_args["timeout"] = max(0.0, options.watchman_query_timeout_ms / 1000.0)
watchman_client = pywatchman.client(**client_args)
watchman_error = pywatchman.WatchmanError
buildFileProcessor = BuildFileProcessor(
project_root,
options.watchman_watch_root,
options.watchman_project_prefix,
options.build_file_name,
options.allow_empty_globs,
watchman_client,
watchman_error,
implicit_includes=options.include or [],
)
buildFileProcessor.install_builtins(__builtin__.__dict__)
for build_file in args:
build_file = cygwin_adjusted_path(build_file)
values = buildFileProcessor.process(build_file)
to_parent.write(bser.dumps(values))
to_parent.flush()
# "for ... in sys.stdin" in Python 2.x hangs until stdin is closed.
for build_file in iter(sys.stdin.readline, ""):
build_file = cygwin_adjusted_path(build_file)
values = buildFileProcessor.process(build_file.rstrip())
to_parent.write(bser.dumps(values))
to_parent.flush()
# Python tries to flush/close stdout when it quits, and if there's a dead
# pipe on the other end, it will spit some warnings to stderr. This breaks
# tests sometimes. Prevent that by explicitly catching the error.
try:
to_parent.close()
except IOError:
pass
def runner():
global results_queue
global tests_queue
broken = False
try:
# Start up a shared watchman instance for the tests.
inst = WatchmanInstance.Instance(
{"watcher": args.watcher}, debug_watchman=args.debug_watchman
)
inst.start()
# Allow tests to locate this default instance
WatchmanInstance.setSharedInstance(inst)
if has_asyncio:
# Each thread will have its own event loop
asyncio.set_event_loop(asyncio.new_event_loop())
except Exception as e:
print("while starting watchman: %s" % str(e))
traceback.print_exc()
broken = True
while not broken:
test = tests_queue.get()
try:
if test == "terminate":
break
if Interrupt.wasInterrupted() or broken:
continue
result = None
for attempt in range(0, args.retry_flaky + 1):
# Check liveness of the server
try:
client = pywatchman.client(timeout=3.0, sockpath=inst.getSockPath())
client.query("version")
client.close()
except Exception as exc:
print(
"Failed to connect to watchman server: %s; starting a new one"
% exc
)
try:
inst.stop()
except Exception:
pass
try:
inst = WatchmanInstance.Instance(
{"watcher": args.watcher},
debug_watchman=args.debug_watchman,
)
inst.start()
# Allow tests to locate this default instance
WatchmanInstance.setSharedInstance(inst)
except Exception as e:
print("while starting watchman: %s" % str(e))
traceback.print_exc()
broken = True
continue
try:
result = Result()
result.setAttemptNumber(attempt)
if hasattr(test, "setAttemptNumber"):
test.setAttemptNumber(attempt)
test.run(result)
if hasattr(test, "setAttemptNumber") and not result.wasSuccessful():
# Facilitate retrying this possibly flaky test
continue
break
except Exception as e:
print(e)
if hasattr(test, "setAttemptNumber") and not result.wasSuccessful():
# Facilitate retrying this possibly flaky test
continue
if (
not result.wasSuccessful()
and "TRAVIS" in os.environ
and hasattr(test, "dumpLogs")
):
test.dumpLogs()
results_queue.put(result)
finally:
tests_queue.task_done()
if not broken:
inst.stop()
def client(self):
return pywatchman.client()
def client(self):
return pywatchman.client()
def main():
# Our parent expects to read JSON from our stdout, so if anyone
# uses print, buck will complain with a helpful "but I wanted an
# array!" message and quit. Redirect stdout to stderr so that
# doesn't happen. Actually dup2 the file handle so that writing
# to file descriptor 1, os.system, and so on work as expected too.
to_parent = os.fdopen(os.dup(sys.stdout.fileno()), "a")
os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
parser = optparse.OptionParser()
parser.add_option("--project_root", action="store", type="string", dest="project_root")
parser.add_option("--build_file_name", action="store", type="string", dest="build_file_name")
parser.add_option(
"--allow_empty_globs",
action="store_true",
dest="allow_empty_globs",
help="Tells the parser not to raise an error when glob returns no results.",
)
parser.add_option(
"--use_watchman_glob",
action="store_true",
dest="use_watchman_glob",
help="Invokes `watchman query` to get lists of files instead of globbing in-process.",
)
parser.add_option(
"--watchman_watch_root",
action="store",
type="string",
dest="watchman_watch_root",
help="Path to root of watchman watch as returned by `watchman watch-project`.",
)
parser.add_option(
"--watchman_project_prefix",
action="store",
type="string",
dest="watchman_project_prefix",
help="Relative project prefix as returned by `watchman watch-project`.",
)
parser.add_option(
"--watchman_query_timeout_ms",
action="store",
type="int",
dest="watchman_query_timeout_ms",
help="Maximum time in milliseconds to wait for watchman query to respond.",
)
parser.add_option("--include", action="append", dest="include")
(options, args) = parser.parse_args()
# Even though project_root is absolute path, it may not be concise. For
# example, it might be like "C:\project\.\rule".
#
# Under cygwin, the project root will be invoked from buck as C:\path, but
# the cygwin python uses UNIX-style paths. They can be converted using
# cygpath, which is necessary because abspath will treat C:\path as a
# relative path.
options.project_root = cygwin_adjusted_path(options.project_root)
project_root = os.path.abspath(options.project_root)
watchman_client = None
watchman_error = None
output_format = "JSON"
output_encode = lambda val: json.dumps(val, sort_keys=True)
if options.use_watchman_glob:
try:
# pywatchman may not be built, so fall back to non-watchman
# in that case.
import pywatchman
client_args = {}
if options.watchman_query_timeout_ms is not None:
# pywatchman expects a timeout as a nonnegative floating-point
# value in seconds.
client_args["timeout"] = max(0.0, options.watchman_query_timeout_ms / 1000.0)
watchman_client = pywatchman.client(**client_args)
watchman_error = pywatchman.WatchmanError
output_format = "BSER"
output_encode = lambda val: pywatchman.bser.dumps(val)
except ImportError, e:
# TODO(agallagher): Restore this when the PEX builds pywatchman.
# print >> sys.stderr, \
# 'Could not import pywatchman (sys.path {}): {}'.format(
# sys.path,
# repr(e))
pass
def main():
# Our parent expects to read JSON from our stdout, so if anyone
# uses print, buck will complain with a helpful "but I wanted an
# array!" message and quit. Redirect stdout to stderr so that
# doesn't happen. Actually dup2 the file handle so that writing
# to file descriptor 1, os.system, and so on work as expected too.
to_parent = os.fdopen(os.dup(sys.stdout.fileno()), 'a')
os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
parser = optparse.OptionParser()
parser.add_option(
'--project_root',
action='store',
type='string',
dest='project_root')
parser.add_option(
'--build_file_name',
action='store',
type='string',
dest="build_file_name")
parser.add_option(
'--allow_empty_globs',
action='store_true',
dest='allow_empty_globs',
help='Tells the parser not to raise an error when glob returns no results.')
parser.add_option(
'--use_watchman_glob',
action='store_true',
dest='use_watchman_glob',
help='Invokes `watchman query` to get lists of files instead of globbing in-process.')
parser.add_option(
'--watchman_watch_root',
action='store',
type='string',
dest='watchman_watch_root',
help='Path to root of watchman watch as returned by `watchman watch-project`.')
parser.add_option(
'--watchman_project_prefix',
action='store',
type='string',
dest='watchman_project_prefix',
help='Relative project prefix as returned by `watchman watch-project`.')
parser.add_option(
'--include',
action='append',
dest='include')
(options, args) = parser.parse_args()
# Even though project_root is absolute path, it may not be concise. For
# example, it might be like "C:\project\.\rule".
#
# Under cygwin, the project root will be invoked from buck as C:\path, but
# the cygwin python uses UNIX-style paths. They can be converted using
# cygpath, which is necessary because abspath will treat C:\path as a
# relative path.
options.project_root = cygwin_adjusted_path(options.project_root)
project_root = os.path.abspath(options.project_root)
watchman_client = None
output_format = 'JSON'
output_encode = lambda val: json.dumps(val, sort_keys=True)
if options.use_watchman_glob:
try:
# pywatchman may not be built, so fall back to non-watchman
# in that case.
import pywatchman
watchman_client = pywatchman.client()
output_format = 'BSER'
output_encode = lambda val: pywatchman.bser.dumps(val)
except ImportError, e:
# TODO(agallagher): Restore this when the PEX builds pywatchman.
# print >> sys.stderr, \
# 'Could not import pywatchman (sys.path {}): {}'.format(
# sys.path,
# repr(e))
pass
def main():
# Our parent expects to read BSER from our stdout, so if anyone
# uses print, buck will complain with a helpful "but I wanted an
# array!" message and quit. Redirect stdout to stderr so that
# doesn't happen. Actually dup2 the file handle so that writing
# to file descriptor 1, os.system, and so on work as expected too.
to_parent = os.fdopen(os.dup(sys.stdout.fileno()), 'ab')
os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
parser = optparse.OptionParser()
parser.add_option(
'--project_root',
action='store',
type='string',
dest='project_root')
parser.add_option(
'--build_file_name',
action='store',
type='string',
dest="build_file_name")
parser.add_option(
'--allow_empty_globs',
action='store_true',
dest='allow_empty_globs',
help='Tells the parser not to raise an error when glob returns no results.')
parser.add_option(
'--use_watchman_glob',
action='store_true',
dest='use_watchman_glob',
help='Invokes `watchman query` to get lists of files instead of globbing in-process.')
parser.add_option(
'--watchman_watch_root',
action='store',
type='string',
dest='watchman_watch_root',
help='Path to root of watchman watch as returned by `watchman watch-project`.')
parser.add_option(
'--watchman_socket_path',
action='store',
type='string',
dest='watchman_socket_path',
help='Path to Unix domain socket/named pipe as returned by `watchman get-sockname`.')
parser.add_option(
'--watchman_project_prefix',
action='store',
type='string',
dest='watchman_project_prefix',
help='Relative project prefix as returned by `watchman watch-project`.')
parser.add_option(
'--watchman_query_timeout_ms',
action='store',
type='int',
dest='watchman_query_timeout_ms',
help='Maximum time in milliseconds to wait for watchman query to respond.')
parser.add_option(
'--include',
action='append',
dest='include')
parser.add_option(
'--config',
help='BuckConfig settings available at parse time.')
parser.add_option(
'--quiet',
action='store_true',
dest='quiet',
help='Stifles exception backtraces printed to stderr during parsing.')
parser.add_option(
'--ignore_buck_autodeps_files',
action='store_true',
help='Profile every buck file execution')
parser.add_option(
'--profile',
action='store_true',
help='Profile every buck file execution')
(options, args) = parser.parse_args()
# Even though project_root is absolute path, it may not be concise. For
# example, it might be like "C:\project\.\rule".
#
# Under cygwin, the project root will be invoked from buck as C:\path, but
# the cygwin python uses UNIX-style paths. They can be converted using
# cygpath, which is necessary because abspath will treat C:\path as a
# relative path.
options.project_root = cygwin_adjusted_path(options.project_root)
project_root = os.path.abspath(options.project_root)
watchman_client = None
watchman_error = None
if options.use_watchman_glob:
import pywatchman
client_args = {}
if options.watchman_query_timeout_ms is not None:
# pywatchman expects a timeout as a nonnegative floating-point
# value in seconds.
client_args['timeout'] = max(0.0, options.watchman_query_timeout_ms / 1000.0)
if options.watchman_socket_path is not None:
client_args['sockpath'] = options.watchman_socket_path
client_args['transport'] = 'local'
watchman_client = pywatchman.client(**client_args)
watchman_error = pywatchman.WatchmanError
configs = {}
if options.config is not None:
with open(options.config, 'rb') as f:
for section, contents in bser.loads(f.read()).iteritems():
for field, value in contents.iteritems():
configs[(section, field)] = value
buildFileProcessor = BuildFileProcessor(
project_root,
options.watchman_watch_root,
options.watchman_project_prefix,
options.build_file_name,
options.allow_empty_globs,
options.ignore_buck_autodeps_files,
watchman_client,
watchman_error,
implicit_includes=options.include or [],
configs=configs)
buildFileProcessor.install_builtins(__builtin__.__dict__)
# While processing, we'll write exceptions as diagnostic messages
# to the parent then re-raise them to crash the process. While
# doing so, we don't want Python's default unhandled exception
# behavior of writing to stderr.
orig_excepthook = None
if options.quiet:
orig_excepthook = sys.excepthook
sys.excepthook = silent_excepthook
for build_file in args:
process_with_diagnostics(build_file, buildFileProcessor, to_parent,
should_profile=options.profile)
# "for ... in sys.stdin" in Python 2.x hangs until stdin is closed.
for build_file in iter(sys.stdin.readline, ''):
process_with_diagnostics(build_file, buildFileProcessor, to_parent,
should_profile=options.profile)
if options.quiet:
sys.excepthook = orig_excepthook
# Python tries to flush/close stdout when it quits, and if there's a dead
# pipe on the other end, it will spit some warnings to stderr. This breaks
# tests sometimes. Prevent that by explicitly catching the error.
try:
to_parent.close()
except IOError:
pass
def main():
# Our parent expects to read JSON from our stdout, so if anyone
# uses print, buck will complain with a helpful "but I wanted an
# array!" message and quit. Redirect stdout to stderr so that
# doesn't happen. Actually dup2 the file handle so that writing
# to file descriptor 1, os.system, and so on work as expected too.
to_parent = os.fdopen(os.dup(sys.stdout.fileno()), 'a')
os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
parser = optparse.OptionParser()
parser.add_option(
'--project_root',
action='store',
type='string',
dest='project_root')
parser.add_option(
'--build_file_name',
action='store',
type='string',
dest="build_file_name")
parser.add_option(
'--allow_empty_globs',
action='store_true',
dest='allow_empty_globs',
help='Tells the parser not to raise an error when glob returns no results.')
parser.add_option(
'--use_watchman_glob',
action='store_true',
dest='use_watchman_glob',
help='Invokes `watchman query` to get lists of files instead of globbing in-process.')
parser.add_option(
'--watchman_watch_root',
action='store',
type='string',
dest='watchman_watch_root',
help='Path to root of watchman watch as returned by `watchman watch-project`.')
parser.add_option(
'--watchman_project_prefix',
action='store',
type='string',
dest='watchman_project_prefix',
help='Relative project prefix as returned by `watchman watch-project`.')
parser.add_option(
'--include',
action='append',
dest='include')
(options, args) = parser.parse_args()
# Even though project_root is absolute path, it may not be concise. For
# example, it might be like "C:\project\.\rule".
#
# Under cygwin, the project root will be invoked from buck as C:\path, but
# the cygwin python uses UNIX-style paths. They can be converted using
# cygpath, which is necessary because abspath will treat C:\path as a
# relative path.
options.project_root = cygwin_adjusted_path(options.project_root)
project_root = os.path.abspath(options.project_root)
watchman_client = None
output_format = 'JSON'
output_encode = lambda val: json.dumps(val, sort_keys=True)
if options.use_watchman_glob:
try:
# pywatchman may not be built, so fall back to non-watchman
# in that case.
import pywatchman
watchman_client = pywatchman.client()
output_format = 'BSER'
output_encode = lambda val: pywatchman.bser.dumps(val)
except ImportError, e:
# TODO(agallagher): Restore this when the PEX builds pywatchman.
# print >> sys.stderr, \
# 'Could not import pywatchman (sys.path {}): {}'.format(
# sys.path,
# repr(e))
pass