def runner():
global results_queue
global tests_queue
try:
# Start up a shared watchman instance for the tests.
inst = WatchmanInstance.Instance()
inst.start()
# Allow tests to locate this default instance
WatchmanInstance.setSharedInstance(inst)
except Exception as e:
print('This is going to suck:', e)
return
while True:
test = tests_queue.get()
try:
if test == 'terminate':
break
if Interrupt.wasInterrupted():
continue
try:
result = Result()
test.run(result)
results_queue.put(result)
except Exception as e:
print(e)
finally:
tests_queue.task_done()
inst.stop()
def runner():
global results_queue
global tests_queue
try:
# Start up a shared watchman instance for the tests.
inst = WatchmanInstance.Instance()
inst.start()
# Allow tests to locate this default instance
WatchmanInstance.setSharedInstance(inst)
except Exception as e:
print('This is going to suck:', e)
return
while True:
test = tests_queue.get()
try:
if test == 'terminate':
break
if Interrupt.wasInterrupted():
continue
try:
result = Result()
test.run(result)
results_queue.put(result)
except Exception as e:
print(e)
finally:
tests_queue.task_done()
inst.stop()
def _async_runner(self, test_core):
wminst = WatchmanInstance.Instance()
wminst.start()
try:
return asyncio.new_event_loop().run_until_complete(test_core(wminst))
finally:
wminst.stop()
def test_scmHgRequestId(self):
if not is_hg_installed():
self.skipTest("Hg not installed")
self.skipIfNoHgRequestIdSupport()
root = self.mkdtemp()
# In this test, the repo does not necessarily need fsmonitor enabled,
# since watchman calls HGREQUESTID=... hg status and that would also
# have request_id logged without fsmonitor.
env = os.environ.copy()
env["HGPLAIN"] = "1"
env["WATCHMAN_SOCK"] = (WatchmanInstance.getSharedInstance().
getSockPath().legacy_sockpath())
subprocess.call(["hg", "init"], env=env, cwd=root)
subprocess.call(
[
"hg",
"commit",
"-mempty",
"-utest",
"-d0 0",
"--config=ui.allowemptycommit=1",
],
env=env,
cwd=root,
)
commit_hash = subprocess.check_output(["hg", "log", "-r.", "-T{node}"],
env=env,
cwd=root).decode("utf-8")
# Must watch the directory after it's an HG repo to perform scm-aware
# queries.
self.watchmanCommand("watch", root)
request_id = "4c05a798ea1acc7c97b75e61fec5f640d90f8209"
params = {
"fields": ["name"],
"request_id": request_id,
"since": {
"scm": {
"mergebase-with": commit_hash
}
},
}
self.watchmanCommand("query", root, params)
blackbox_path = os.path.join(root, ".hg", "blackbox.log")
def try_read_blackbox():
try:
with open(blackbox_path) as f:
return f.read()
except IOError:
return ""
self.assertWaitFor(
lambda: request_id in try_read_blackbox(),
message="request_id passed to and logged by hg",
)
def test_Illegal(self):
config = {
"illegal_fstypes": [
# This should include any/all fs types. If this test fails on
# your platform, look in /tmp/watchman-test.log for a line like:
# "path /var/tmp/a3osdzvzqnco0sok is on filesystem type zfs"
# then add the type name to this list, in sorted order
"NTFS",
"apfs",
"cifs",
"hfs",
"nfs",
"smb",
"tmpfs",
"ufs",
"unknown",
"zfs",
],
"illegal_fstypes_advice":
"just cos",
}
with WatchmanInstance.Instance(config=config) as inst:
inst.start()
client = self.getClient(inst)
d = self.mkdtemp()
with self.assertRaises(pywatchman.WatchmanError) as ctx:
client.query("watch", d)
self.assertIn(
("filesystem and is disallowed by global config" +
" illegal_fstypes: just cos"),
str(ctx.exception),
)
def runTest(self):
env = os.environ.copy()
env['WATCHMAN_SOCK'] = WatchmanInstance.getSharedInstance().getSockPath()
dotted = os.path.normpath(self.id()).replace(os.sep, '.').replace(
'tests.integration.', '').replace('.php', '')
env['TMPDIR'] = os.path.join(TempDir.get_temp_dir().get_dir(), dotted)
os.mkdir(env['TMPDIR'])
# build the node module with npm
node_dir = os.path.join(env['TMPDIR'], 'fb-watchman')
shutil.copytree(os.path.join(WATCHMAN_SRC_DIR, 'node'), node_dir)
subprocess.check_call(['npm', 'install'], cwd=node_dir)
env['TMP'] = env['TMPDIR']
env['TEMP'] = env['TMPDIR']
env['IN_PYTHON_HARNESS'] = '1'
env['NODE_PATH'] = '%s:%s' % (env['TMPDIR'], env.get('NODE_PATH', ''))
proc = subprocess.Popen(
self.getCommandArgs(),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = proc.communicate()
status = proc.poll()
if status == -signal.SIGINT:
Interrupt.setInterrupted()
self.fail('Interrupted by SIGINT')
return
if status != 0:
self.fail("Exit status %d\n%s\n%s\n" %
(status, stdout.decode('utf-8'), stderr.decode('utf-8')))
return
self.assertTrue(True, self.getCommandArgs())
def runWatchTests(self, config, expect):
with WatchmanInstance.Instance(config=config) as inst:
inst.start()
client = self.getClient(inst)
for filetype, name, expect_pass in expect:
for watch_type in ["watch", "watch-project"]:
# encode the test criteria in the dirname so that we can
# figure out which test scenario failed more easily
d = self.mkdtemp(suffix="-%s-%s-%s-%s" %
(filetype, name, expect_pass, watch_type))
if filetype == "directory":
os.mkdir(os.path.join(d, name))
elif filetype == "file":
self.touchRelative(d, name)
assert_functions = {
(True, "watch"):
self.assertWatchSucceeds,
(True, "watch-project"):
self.assertWatchProjectSucceeds,
(False, "watch"):
self.assertWatchIsRestricted,
(False, "watch-project"):
self.assertWatchProjectIsRestricted,
}
assert_function = assert_functions[(expect_pass,
watch_type)]
assert_function(inst, client, d)
def getServerLogContents(self):
'''
Returns the contents of the server log file as an array
that has already been split by line.
'''
return WatchmanInstance.getSharedInstance().\
getServerLogContents().split('\n')
def runTest(self):
env = os.environ.copy()
env["WATCHMAN_SOCK"] = WatchmanInstance.getSharedInstance().getSockPath()
env["TMPDIR"] = self.tempdir
# build the node module with npm
node_dir = os.path.join(env["TMPDIR"], "fb-watchman")
shutil.copytree(os.path.join(WATCHMAN_SRC_DIR, "node"), node_dir)
subprocess.check_call(["npm", "install"], cwd=node_dir)
env["TMP"] = env["TMPDIR"]
env["TEMP"] = env["TMPDIR"]
env["IN_PYTHON_HARNESS"] = "1"
env["NODE_PATH"] = "%s:%s" % (env["TMPDIR"], env.get("NODE_PATH", ""))
proc = subprocess.Popen(
self.getCommandArgs(),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
(stdout, stderr) = proc.communicate()
status = proc.poll()
if status == -signal.SIGINT:
Interrupt.setInterrupted()
self.fail("Interrupted by SIGINT")
return
if status != 0:
self.fail(
"Exit status %d\n%s\n%s\n"
% (status, stdout.decode("utf-8"), stderr.decode("utf-8"))
)
return
self.assertTrue(True, self.getCommandArgs())
def runTest(self):
env = os.environ.copy()
env['WATCHMAN_SOCK'] = WatchmanInstance.getSharedInstance(
).getSockPath()
dotted = os.path.normpath(self.id()).replace(os.sep, '.').replace(
'tests.integration.', '').replace('.php', '')
env['TMPDIR'] = os.path.join(tempfile.tempdir, dotted)
os.mkdir(env['TMPDIR'])
env['TMP'] = env['TMPDIR']
env['TEMP'] = env['TMPDIR']
env['IN_PYTHON_HARNESS'] = '1'
proc = subprocess.Popen(self.getCommandArgs(),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = proc.communicate()
status = proc.poll()
if status == -signal.SIGINT:
Interrupt.setInterrupted()
self.fail('Interrupted by SIGINT')
return
if status != 0:
self.fail("Exit status %d\n%s\n%s\n" % (status, stdout, stderr))
return
self.assertTrue(True, self.jsfile)
def setUp(self):
super(WatchmanEdenTestCase, self).setUp()
# The test EdenFS instance.
# We let it create and manage its own temporary test directory, rather than
# using the one scoped to the test because we have very real length limits on
# the socket path name that we're likely to hit otherwise.
self.eden = edenclient.EdenFS()
self.addCleanup(lambda: self.cleanUpEden())
# where we'll mount the eden client(s)
self.mounts_dir = self.mkdtemp(prefix="eden_mounts")
# Watchman needs to start up with the same HOME as eden, otherwise
# it won't be able to locate the eden socket
self.save_home = os.environ["HOME"]
os.environ["HOME"] = str(self.eden.home_dir)
self.addCleanup(lambda: self._restoreHome())
self.system_hgrc = None
self.eden_watchman = WatchmanInstance.Instance()
self.eden_watchman.start()
self.addCleanup(self.cleanUpWatchman)
self.client = self.getClient(self.eden_watchman)
# chg can interfere with eden, so disable it up front
os.environ["CHGDISABLE"] = "1"
# Start the EdenFS instance
self.eden.start()
def setUp(self):
# the eden home directory. We use the global dir for the test runner
# rather than one scoped to the test because we have very real length
# limits on the socket path name that we're likely to hit otherwise.
# fake a home dir so that eden is isolated from the settings
# of the user running these tests.
self.eden_home = tempfile.mkdtemp(prefix='eden_home')
self.eden_dir = os.path.join(self.eden_home, 'local/.eden')
os.makedirs(self.eden_dir)
self.etc_eden_dir = os.path.join(self.eden_home, 'etc-eden')
os.mkdir(self.etc_eden_dir)
# The directory holding the system configuration files
self.system_config_dir = os.path.join(self.etc_eden_dir, 'config.d')
os.mkdir(self.system_config_dir)
# where we'll mount the eden client(s)
self.mounts_dir = self.mkdtemp(prefix='eden_mounts')
self.save_home = os.environ['HOME']
os.environ['HOME'] = self.eden_home
self.eden = edenclient.EdenFS(self.eden_dir,
etc_eden_dir=self.etc_eden_dir,
home_dir=self.eden_home)
self.eden.start()
# Watchman also needs to start up with the same HOME, otherwise
# it won't be able to locate the eden socket
self.eden_watchman = WatchmanInstance.Instance()
self.eden_watchman.start()
self.client = self.getClient(self.eden_watchman)
def test_sock_name(self):
resp = self.watchmanCommand("get-sockname")
self.assertEqual(
resp["sockname"],
WatchmanInstance.getSharedInstance().getSockPath().legacy_sockpath(
),
)
def hg(self, args, cwd=None):
env = dict(os.environ)
env["HGPLAIN"] = "1"
env["HGUSER"] = "******"
env["NOSCMLOG"] = "1" # disable some instrumentation at FB
sockpath = WatchmanInstance.getSharedInstance().getSockPath()
env["WATCHMAN_SOCK"] = sockpath.legacy_sockpath()
p = subprocess.Popen(
[
env.get("EDEN_HG_BINARY", "hg"),
# we force the extension on. This is a soft error for
# mercurial if it is not available, so we also employ
# the skipIfNoFSMonitor() test above to make sure the
# environment is sane.
"--config",
"extensions.fsmonitor=",
# Deployed versions of mercurial regressed and stopped
# respecting the WATCHMAN_SOCK environment override, so
# we have to reach in and force their hardcoded sockpath here.
"--config",
"fsmonitor.sockpath=%s" % sockpath.legacy_sockpath(),
] + args,
env=env,
cwd=cwd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out, err = p.communicate()
if p.returncode != 0:
raise Exception("hg %r failed: %s, %s" % (args, out, err))
return out, err
def test_rel_root(self):
root = self.mkdtemp()
a_dir = os.path.join(root, 'a')
os.mkdir(a_dir)
b_dir = os.path.join(root, 'b')
os.mkdir(b_dir)
wi = WatchmanWaitInstance.Instance(
sock_path=WatchmanInstance.getSharedInstance().getSockPath()
)
wi.start(paths=[a_dir, b_dir], cmdArgs=['--relative', b_dir])
self.assertSyncWatchmanWait(wi, a_dir)
self.touchRelative(a_dir, 'afoo')
self.touchRelative(b_dir, 'bfoo')
self.assertWaitedFileList(
wi, ['../a/afoo', 'bfoo'], msg="wait, relative create files")
a_sub_dir = os.path.join(a_dir, 'asub')
os.mkdir(a_sub_dir)
b_sub_dir = os.path.join(b_dir, 'bsub')
os.mkdir(b_sub_dir)
self.assertWaitedFileList(
wi, ['../a/asub', 'bsub'], msg="wait, relative create directories")
def runTest(self):
env = os.environ.copy()
env['WATCHMAN_SOCK'] = WatchmanInstance.getSharedInstance().getSockPath()
dotted = os.path.normpath(self.id()).replace(os.sep, '.').replace(
'tests.integration.', '').replace('.php', '')
env['TMPDIR'] = os.path.join(tempfile.tempdir, dotted)
os.mkdir(env['TMPDIR'])
env['TMP'] = env['TMPDIR']
env['TEMP'] = env['TMPDIR']
env['IN_PYTHON_HARNESS'] = '1'
proc = subprocess.Popen(
self.getCommandArgs(),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = proc.communicate()
status = proc.poll()
if status == -signal.SIGINT:
Interrupt.setInterrupted()
self.fail('Interrupted by SIGINT')
return
if status != 0:
self.fail("Exit status %d\n%s\n%s\n" %
(status, stdout.decode('utf-8'), stderr.decode('utf-8')))
return
self.assertTrue(True, self.jsfile)
def test_Illegal(self):
config = {
'illegal_fstypes': [
# This should include any/all fs types. If this test fails on
# your platform, look in /tmp/watchman-test.log for a line like:
# "path /var/tmp/a3osdzvzqnco0sok is on filesystem type zfs"
# then add the type name to this list, in sorted order
'NTFS',
'apfs',
'cifs',
'hfs',
'nfs',
'smb',
'tmpfs',
'ufs',
'unknown',
'zfs',
],
'illegal_fstypes_advice':
'just cos',
}
inst = WatchmanInstance.Instance(config=config)
try:
inst.start()
client = self.getClient(inst)
d = self.mkdtemp()
with self.assertRaises(pywatchman.WatchmanError) as ctx:
client.query('watch', d)
self.assertIn(('filesystem and is disallowed by global config' +
' illegal_fstypes: just cos'), str(ctx.exception))
finally:
inst.stop()
def runTest(self):
env = os.environ.copy()
env['WATCHMAN_SOCK'] = WatchmanInstance.getSharedInstance(
).getSockPath()
dotted = os.path.normpath(self.id()).replace(os.sep, '.').replace(
'tests.integration.', '').replace('.php', '')
env['TMPDIR'] = os.path.join(tempfile.tempdir, dotted)
os.mkdir(env['TMPDIR'])
# build the node module with npm
node_dir = os.path.join(env['TMPDIR'], 'fb-watchman')
shutil.copytree('node', node_dir)
subprocess.check_call(['npm', 'install'], cwd=node_dir)
env['TMP'] = env['TMPDIR']
env['TEMP'] = env['TMPDIR']
env['IN_PYTHON_HARNESS'] = '1'
env['NODE_PATH'] = '%s:%s' % (env['TMPDIR'], env.get('NODE_PATH', ''))
proc = subprocess.Popen(self.getCommandArgs(),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = proc.communicate()
status = proc.poll()
if status == -signal.SIGINT:
Interrupt.setInterrupted()
self.fail('Interrupted by SIGINT')
return
if status != 0:
self.fail("Exit status %d\n%s\n%s\n" %
(status, stdout.decode('utf-8'), stderr.decode('utf-8')))
return
self.assertTrue(True, self.jsfile)
def hg(self, args=None, cwd=None):
env = dict(os.environ)
env["HGPLAIN"] = "1"
env["HGUSER"] = "******"
env["NOSCMLOG"] = "1" # disable some instrumentation at FB
env["WATCHMAN_SOCK"] = WatchmanInstance.getSharedInstance(
).getSockPath()
p = subprocess.Popen(
# we force the extension on. This is a soft error for
# mercurial if it is not available, so we also employ
# the skipIfNoFSMonitor() test above to make sure the
# environment is sane.
[
env.get("EDEN_HG_BINARY", "hg"), "--config",
"extensions.fsmonitor="
] + args,
env=env,
cwd=cwd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out, err = p.communicate()
if p.returncode != 0:
raise Exception("hg %r failed: %s, %s" % (args, out, err))
return out, err
def test_cppclient(self):
env = os.environ.copy()
env["WATCHMAN_SOCK"] = (WatchmanInstance.getSharedInstance().
getSockPath().legacy_sockpath())
proc = subprocess.Popen(
TEST_BINARY,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=self.tmpDir,
)
(stdout, stderr) = proc.communicate()
status = proc.poll()
if status == -signal.SIGINT:
Interrupt.setInterrupted()
self.fail("Interrupted by SIGINT")
return
if status != 0:
self.fail("Exit status %d\n%s\n%s\n" %
(status, stdout.decode("utf-8"), stderr.decode("utf-8")))
return
self.assertTrue(True, TEST_BINARY)
def test_wait(self):
root = self.mkdtemp()
self.touchRelative(root, 'foo')
a_dir = os.path.join(root, 'a')
os.mkdir(a_dir)
self.touchRelative(a_dir, 'foo')
wi = WatchmanWaitInstance.Instance(
sock_path=WatchmanInstance.getSharedInstance().getSockPath()
)
wi.start(paths=[root], cmdArgs=['--relative', root])
self.assertSyncWatchmanWait(wi, root)
self.touchRelative(root, 'bar')
self.removeRelative(root, 'foo')
self.assertWaitedFileList(wi, ['foo', 'bar'], msg="wait, create files")
self.touchRelative(a_dir, 'bar')
self.removeRelative(a_dir, 'foo')
self.assertWaitedFileList(
wi, ['a/bar', 'a/foo', 'a'], msg="wait, create dir")
b_dir = os.path.join(root, 'b')
os.mkdir(b_dir)
self.touchRelative(b_dir, 'foo')
self.assertWaitedFileList(wi, ['b', 'b/foo'], msg=None)
def getServerLogContents(self):
"""
Returns the contents of the server log file as an array
that has already been split by line.
"""
return WatchmanInstance.getSharedInstance().getServerLogContents(
).split("\n")
def runTest(self):
env = os.environ.copy()
env["WATCHMAN_SOCK"] = WatchmanInstance.getSharedInstance(
).getSockPath()
env["TMPDIR"] = self.tempdir
# build the node module with npm
node_dir = os.path.join(env["TMPDIR"], "fb-watchman")
shutil.copytree(os.path.join(WATCHMAN_SRC_DIR, "node"), node_dir)
subprocess.check_call(["npm", "install"], cwd=node_dir)
env["TMP"] = env["TMPDIR"]
env["TEMP"] = env["TMPDIR"]
env["IN_PYTHON_HARNESS"] = "1"
env["NODE_PATH"] = "%s:%s" % (env["TMPDIR"], env.get("NODE_PATH", ""))
proc = subprocess.Popen(
self.getCommandArgs(),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
(stdout, stderr) = proc.communicate()
status = proc.poll()
if status == -signal.SIGINT:
Interrupt.setInterrupted()
self.fail("Interrupted by SIGINT")
return
if status != 0:
self.fail("Exit status %d\n%s\n%s\n" %
(status, stdout.decode("utf-8"), stderr.decode("utf-8")))
return
self.assertTrue(True, self.getCommandArgs())
def getClient(self):
if not hasattr(self, 'client'):
self.client = pywatchman.client(
transport=self.transport,
sendEncoding=self.encoding,
recvEncoding=self.encoding,
sockpath=WatchmanInstance.getSharedInstance().getSockPath())
return self.client
def getClient(self):
if not hasattr(self, 'client'):
self.client = pywatchman.client(
transport=self.transport,
sendEncoding=self.encoding,
recvEncoding=self.encoding,
sockpath=WatchmanInstance.getSharedInstance().getSockPath())
return self.client
def dumpLogs(self):
''' used in travis CI to show the hopefully relevant log snippets '''
inst = WatchmanInstance.getSharedInstance()
def tail(logstr, n):
lines = logstr.split('\n')[-n:]
return '\n'.join(lines)
print(self.getLogSample())
def _new_instance(self, config, expect_success=True):
if expect_success:
start_timeout = 1.0
else:
# If the instance is going to fail anyway then there's no point
# waiting so long
start_timeout = 0.2
return WatchmanInstance.InstanceWithStateDir(
config=config, start_timeout=start_timeout)
def dumpLogs(self):
''' used in travis CI to show the hopefully relevant log snippets '''
inst = WatchmanInstance.getSharedInstance()
def tail(logstr, n):
lines = logstr.split('\n')[-n:]
return '\n'.join(lines)
print(self.getLogSample())
def test_failing_to_start_when_nice(self):
if sys.platform == "darwin":
self.skipTest("launchd renders this test invalid on macOS")
inst = WatchmanInstance.Instance()
stdout, stderr = inst.commandViaCLI(["version"], prefix=["nice"])
print("stdout", stdout)
print("stderr", stderr)
stderr = stderr.decode("ascii")
self.assertEqual(b"", stdout)
self.assertRegex(stderr, "refusing to start")
def getClient(self):
if not hasattr(self, 'client'):
self.client = pywatchman.client(
# ASAN-enabled builds can be slower enough that we hit timeouts
# with the default of 1 second
timeout=3.0,
transport=self.transport,
sendEncoding=self.encoding,
recvEncoding=self.encoding,
sockpath=WatchmanInstance.getSharedInstance().getSockPath())
return self.client
def test_spawner(self):
config = {"spawn_watchman_service": os.path.join(THIS_DIR, "site_spawn.py")}
inst = WatchmanInstance.Instance(config=config)
stdout, stderr = inst.commandViaCLI(["version"])
parsed = json.loads(stdout.decode("ascii"))
self.assertTrue("version" in parsed)
# Shut down that process, as we have no automatic way to deal with it
inst.commandViaCLI(["--no-spawn", "--no-local", "shutdown-server"])
def runProjectTests(self, config, expect, touch_watchmanconfig=False):
inst = WatchmanInstance.Instance(config=config)
try:
inst.start()
client = self.getClient(inst)
for touch, expect_watch, expect_rel, expect_pass in expect:
# encode the test criteria in the dirname so that we can
# figure out which test scenario failed more easily
suffix = "-%s-%s-%s-%s" % (touch, expect_watch, expect_rel,
expect_pass)
suffix = suffix.replace("/", "Z")
d = self.mkdtemp(suffix=suffix)
dir_to_watch = os.path.join(d, "a", "b", "c")
os.makedirs(dir_to_watch, 0o777)
dir_to_watch = self.normAbsolutePath(dir_to_watch)
self.touchRelative(d, touch)
if touch_watchmanconfig:
self.touchRelative(d, ".watchmanconfig")
if expect_watch:
expect_watch = os.path.join(d, expect_watch)
else:
expect_watch = d
if expect_pass:
res = client.query("watch-project", dir_to_watch)
self.assertEqual(
self.normAbsolutePath(os.path.join(d, expect_watch)),
self.normAbsolutePath(res["watch"]),
)
if not expect_rel:
self.assertEqual(None, res.get("relative_path"))
else:
self.assertEqual(
self.normRelativePath(expect_rel),
self.normRelativePath(res.get("relative_path")),
)
else:
with self.assertRaises(pywatchman.WatchmanError) as ctx:
client.query("watch-project", dir_to_watch)
self.assertIn(
("None of the files listed in global config " +
"root_files are present in path `" + dir_to_watch +
"` or any of its parent directories. " +
"root_files is defined by the"),
str(ctx.exception),
)
finally:
inst.stop()
def test_bulkstat_off(self):
config = {"_use_bulkstat": False}
with WatchmanInstance.Instance(config=config) as inst:
inst.start()
self.getClient(inst, replace_cached=True)
root = self.mkdtemp()
self.client.query("watch", root)
self.touchRelative(root, "foo")
self.touchRelative(root, "bar")
self.assertFileList(root, ["foo", "bar"])
def dumpLogs(self):
''' used in travis CI to show the hopefully relevant log snippets '''
inst = WatchmanInstance.getSharedInstance()
def tail(logstr, n):
lines = logstr.split('\n')[-n:]
return '\n'.join(lines)
print('CLI logs')
print(tail(inst.getCLILogContents(), 500))
print('Server logs')
print(tail(inst.getServerLogContents(), 500))
def test_rootRestrict(self):
config = {"root_restrict_files": [".git", ".hg", ".foo", ".bar"]}
inst = WatchmanInstance.Instance(config=config)
try:
inst.start()
client = self.getClient(inst)
expect = [
("directory", ".git", True),
("directory", ".hg", True),
("file", ".foo", True),
("file", ".bar", True),
("directory", ".bar", True),
(None, None, False),
("directory", ".svn", False),
("file", "baz", False),
]
for filetype, name, expect_pass in expect:
# encode the test criteria in the dirname so that we can
# figure out which test scenario failed more easily
d = self.mkdtemp(suffix="-%s-%s-%s" %
(filetype, name, expect_pass))
if filetype == "directory":
os.mkdir(os.path.join(d, name))
elif filetype == "file":
self.touchRelative(d, name)
if expect_pass:
client.query("watch", d)
else:
with self.assertRaises(pywatchman.WatchmanError) as ctx:
client.query("watch", d)
self.assertRegex(
str(ctx.exception),
("unable to resolve root .*" +
": Your watchman administrator has configured watchman "
+ "to prevent watching path `.*`. " +
"None of the files listed in global config root_files "
+
"are present and enforce_root_files is set to true. "
+
"root_files is defined by the `.*` config file and " +
"includes `.watchmanconfig`, `.git`, `.hg`, `.foo`, "
+ "and `.bar`. One or more of these files must be " +
"present in order to allow a watch. Try pulling " +
"and checking out a newer version of the project?"),
)
finally:
inst.stop()
def runProjectTests(self, config, expect, touch_watchmanconfig=False):
inst = WatchmanInstance.Instance(config=config)
try:
inst.start()
client = self.getClient(inst)
for touch, expect_watch, expect_rel, expect_pass in expect:
# encode the test criteria in the dirname so that we can
# figure out which test scenario failed more easily
suffix = '-%s-%s-%s-%s' % (
touch, expect_watch, expect_rel, expect_pass
)
suffix = suffix.replace('/', 'Z')
d = self.mkdtemp(suffix=suffix)
dir_to_watch = os.path.join(d, 'a', 'b', 'c')
os.makedirs(dir_to_watch, 0o777)
self.touchRelative(d, touch)
if touch_watchmanconfig:
self.touchRelative(d, '.watchmanconfig')
if expect_watch:
expect_watch = os.path.join(d, expect_watch)
else:
expect_watch = d
if expect_pass:
res = client.query('watch-project', dir_to_watch)
self.assertEqual(
self.normAbsolutePath(os.path.join(d, expect_watch)),
self.normAbsolutePath(res['watch'])
)
if not expect_rel:
self.assertEqual(None, res.get('relative_path'))
else:
self.assertEqual(self.normRelativePath(expect_rel),
self.normRelativePath(res.get('relative_path')))
else:
with self.assertRaises(pywatchman.WatchmanError) as ctx:
client.query('watch-project', dir_to_watch)
self.assertIn(
(
'none of the files listed in global config ' +
'root_files are present in path `' + os.path.join(
d, 'a', 'b', 'c'
) + ' or any of its parent directories'
), str(ctx.exception)
)
finally:
inst.stop()
def runner():
global results_queue
global tests_queue
broken = False
try:
# Start up a shared watchman instance for the tests.
inst = WatchmanInstance.Instance({
"watcher": args.watcher
}, debug_watchman=args.debug_watchman)
inst.start()
# Allow tests to locate this default instance
WatchmanInstance.setSharedInstance(inst)
except Exception as e:
print('while starting watchman: %s' % str(e))
traceback.print_exc()
broken = True
while not broken:
test = tests_queue.get()
try:
if test == 'terminate':
break
if Interrupt.wasInterrupted() or broken:
continue
try:
result = Result()
test.run(result)
results_queue.put(result)
except Exception as e:
print(e)
finally:
tests_queue.task_done()
if not broken:
inst.stop()
def getLogSample(self):
''' used in CI to show the hopefully relevant log snippets '''
inst = WatchmanInstance.getSharedInstance()
def tail(logstr, n):
lines = logstr.split('\n')[-n:]
return '\n'.join(lines)
return '\n'.join([
'CLI logs',
tail(inst.getCLILogContents(), 500),
'Server logs',
tail(inst.getServerLogContents(), 500),
])
def getClient(self, inst=None, replace_cached=False, no_cache=False):
if inst or not hasattr(self, "client") or no_cache:
client = pywatchman.client(
timeout=self.socketTimeout,
transport=self.transport,
sendEncoding=self.encoding,
recvEncoding=self.encoding,
sockpath=(inst or WatchmanInstance.getSharedInstance()).getSockPath(),
)
if (not inst or replace_cached) and not no_cache:
# only cache the client if it points to the shared instance
self.client = client
self.addCleanup(lambda: self.__clearClient())
return client
return self.client
def getClient(self, inst=None):
if inst or not hasattr(self, 'client'):
client = pywatchman.client(
# ASAN-enabled builds can be slower enough that we hit timeouts
# with the default of 1 second
timeout=3.0,
transport=self.transport,
sendEncoding=self.encoding,
recvEncoding=self.encoding,
sockpath=(inst or
WatchmanInstance.getSharedInstance()).getSockPath())
if not inst:
# only cache the client if it points to the shared instance
self.client = client
return client
return self.client
def getLogSample(self):
""" used in CI to show the hopefully relevant log snippets """
inst = WatchmanInstance.getSharedInstance()
def tail(logstr, n):
lines = logstr.split("\n")[-n:]
return "\n".join(lines)
return "\n".join(
[
"CLI logs",
tail(inst.getCLILogContents(), 500),
"Server logs",
tail(inst.getServerLogContents(), 500),
]
)
def spawnWatchmanWait(self, cmdArgs):
wait_script = os.environ.get("WATCHMAN_WAIT_PATH")
if wait_script:
args = [wait_script]
else:
args = [
sys.executable,
os.path.join(os.environ["WATCHMAN_PYTHON_BIN"], "watchman-wait"),
]
args.extend(cmdArgs)
env = os.environ.copy()
sock_path = WatchmanInstance.getSharedInstance().getSockPath()
env["WATCHMAN_SOCK"] = sock_path
env["PYTHONPATH"] = env["PYWATCHMAN_PATH"]
return subprocess.Popen(
args, env=env, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
def test_cppclient(self):
env = os.environ.copy()
env["WATCHMAN_SOCK"] = WatchmanInstance.getSharedInstance().getSockPath()
proc = subprocess.Popen(
TEST_BINARY, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
(stdout, stderr) = proc.communicate()
status = proc.poll()
if status == -signal.SIGINT:
Interrupt.setInterrupted()
self.fail("Interrupted by SIGINT")
return
if status != 0:
self.fail(
"Exit status %d\n%s\n%s\n"
% (status, stdout.decode("utf-8"), stderr.decode("utf-8"))
)
return
self.assertTrue(True, TEST_BINARY)
def hg(self, args=None, cwd=None):
env = dict(os.environ)
env['HGPLAIN'] = '1'
env['HGUSER'] = '******'
env['NOSCMLOG'] = '1' # disable some instrumentation at FB
env['WATCHMAN_SOCK'] = \
WatchmanInstance.getSharedInstance().getSockPath()
p = subprocess.Popen(
# we force the extension on. This is a soft error for
# mercurial if it is not available, so we also employ
# the skipIfNoFSMonitor() test above to make sure the
# environment is sane.
['hg', '--config', 'extensions.fsmonitor='] + args,
env=env,
cwd=cwd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
if p.returncode != 0:
raise Exception("hg %r failed: %s, %s" % (args, out, err))
return out, err
def hg(self, args=None, cwd=None):
env = dict(os.environ)
env["HGPLAIN"] = "1"
env["HGUSER"] = "******"
env["NOSCMLOG"] = "1" # disable some instrumentation at FB
env["WATCHMAN_SOCK"] = WatchmanInstance.getSharedInstance().getSockPath()
p = subprocess.Popen(
# we force the extension on. This is a soft error for
# mercurial if it is not available, so we also employ
# the skipIfNoFSMonitor() test above to make sure the
# environment is sane.
[env.get("EDEN_HG_BINARY", "hg"), "--config", "extensions.fsmonitor="]
+ args,
env=env,
cwd=cwd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out, err = p.communicate()
if p.returncode != 0:
raise Exception("hg %r failed: %s, %s" % (args, out, err))
return out, err
def suspendWatchman(self):
WatchmanInstance.getSharedInstance().suspend()
def resumeWatchman(self):
WatchmanInstance.getSharedInstance().resume()
def getSockPath(self):
return WatchmanInstance.getSharedInstance().getSockPath()
def runTest(self):
env = os.environ.copy()
env['WATCHMAN_SOCK'] = WatchmanInstance.getSharedInstance().getSockPath()
def clean_file_name(name):
name = name.replace(os.sep, '')
name = name.replace('tests.integration', '')
name = name.replace('.php', '')
name = name.replace('_php', '')
name = name.replace('.runTest', '')
return name
dotted = clean_file_name(os.path.normpath(self.id()))
env['TMPDIR'] = os.path.join(TempDir.get_temp_dir().get_dir(), dotted)
if os.name != 'nt' and len(env['TMPDIR']) > 94:
self.fail('temp dir name %s is too long for unix domain sockets' %
env['TMPDIR'])
os.mkdir(env['TMPDIR'])
env['TMP'] = env['TMPDIR']
env['TEMP'] = env['TMPDIR']
env['IN_PYTHON_HARNESS'] = '1'
env['WATCHMAN_PYTHON_BINARY'] = sys.executable
proc = subprocess.Popen(
self.getCommandArgs(),
cwd=env.get('WATCHMAN_SRC_DIR', os.getcwd()),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = proc.communicate()
status = proc.poll()
if status == -signal.SIGINT:
Interrupt.setInterrupted()
self.fail('Interrupted by SIGINT')
return
if status != 0:
self.fail("Exit status %d\n%s\n%s\n" % (status, stdout, stderr))
return
res_pat = re.compile(b'^(not )?ok (\d+) (.*)$')
diag_pat = re.compile(b'^# (.*)$')
plan_pat = re.compile(b'^1\.\.(\d+)$')
# Now parse the TAP output
lines = stdout.replace(b'\r\n', b'\n').split(b'\n')
last_test = 0
diags = None
plan = None
for line in lines:
res = plan_pat.match(line)
if res:
plan = int(res.group(1))
continue
res = res_pat.match(line)
if res:
this_test = int(res.group(2))
if this_test != last_test + 1:
print(stdout, stderr)
self.fail('Expected test number %d, got %d' % (
last_test + 1,
this_test))
last_test = this_test
if res.group(1) == b'not ':
# Failed
msg = line
if diags is not None:
msg = msg + b'\n' + b'\n'.join(diags)
self.fail(msg.decode('utf-8'))
failed
diags = None
continue
res = diag_pat.match(line)
if res:
if diags is None:
diags = []
diags.append(res.group(1))
continue
if line != b'':
print('Invalid tap output from %s: %s' %
(self.id(), line))
if plan is None:
self.fail('no plan was observed')
else:
self.assertEqual(last_test, plan,
'%s planned %d but executed %s tests' % (
self.id(),
plan,
last_test))
def getServerLogContents(self):
"""
Returns the contents of the server log file as an array
that has already been split by line.
"""
return WatchmanInstance.getSharedInstance().getServerLogContents().split("\n")
def runner():
global results_queue
global tests_queue
broken = False
try:
# Start up a shared watchman instance for the tests.
inst = WatchmanInstance.Instance(
{"watcher": args.watcher}, debug_watchman=args.debug_watchman
)
inst.start()
# Allow tests to locate this default instance
WatchmanInstance.setSharedInstance(inst)
if has_asyncio:
# Each thread will have its own event loop
asyncio.set_event_loop(asyncio.new_event_loop())
except Exception as e:
print("while starting watchman: %s" % str(e))
traceback.print_exc()
broken = True
while not broken:
test = tests_queue.get()
try:
if test == "terminate":
break
if Interrupt.wasInterrupted() or broken:
continue
result = None
for attempt in range(0, args.retry_flaky + 1):
# Check liveness of the server
try:
client = pywatchman.client(timeout=3.0, sockpath=inst.getSockPath())
client.query("version")
client.close()
except Exception as exc:
print(
"Failed to connect to watchman server: %s; starting a new one"
% exc
)
try:
inst.stop()
except Exception:
pass
try:
inst = WatchmanInstance.Instance(
{"watcher": args.watcher},
debug_watchman=args.debug_watchman,
)
inst.start()
# Allow tests to locate this default instance
WatchmanInstance.setSharedInstance(inst)
except Exception as e:
print("while starting watchman: %s" % str(e))
traceback.print_exc()
broken = True
continue
try:
result = Result()
result.setAttemptNumber(attempt)
if hasattr(test, "setAttemptNumber"):
test.setAttemptNumber(attempt)
test.run(result)
if hasattr(test, "setAttemptNumber") and not result.wasSuccessful():
# Facilitate retrying this possibly flaky test
continue
break
except Exception as e:
print(e)
if hasattr(test, "setAttemptNumber") and not result.wasSuccessful():
# Facilitate retrying this possibly flaky test
continue
if (
not result.wasSuccessful()
and "TRAVIS" in os.environ
and hasattr(test, "dumpLogs")
):
test.dumpLogs()
results_queue.put(result)
finally:
tests_queue.task_done()
if not broken:
inst.stop()
def runTest(self):
env = os.environ.copy()
env['WATCHMAN_SOCK'] = WatchmanInstance.getSharedInstance().getSockPath()
proc = subprocess.Popen(
self.getCommandArgs(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = proc.communicate()
status = proc.poll()
if status == -signal.SIGINT:
Interrupt.setInterrupted()
self.fail('Interrupted by SIGINT')
return
if status != 0:
self.fail("Exit status %d\n%s\n%s\n" % (status, stdout, stderr))
return
res_pat = re.compile('^(not )?ok (\d+) (.*)$')
diag_pat = re.compile('^# (.*)$')
# Now parse the TAP output
lines = stdout.split('\n')
# first line is the plan
try:
plan = int(lines.pop(0).split('..')[1])
except Exception as e:
self.fail(stdout + '\n' + stderr)
return
last_test = 0
diags = None
for line in lines:
res = res_pat.match(line)
if res:
this_test = int(res.group(2))
if this_test != last_test + 1:
print(stdout, stderr)
self.fail('Expected test number %d, got %d' % (
last_test + 1,
this_test))
last_test = this_test
if res.group(1) == 'not ':
# Failed
msg = line
if diags is not None:
msg = msg + '\n'.join(diags)
self.fail(msg)
failed
diags = None
continue
res = diag_pat.match(line)
if res:
if diags is None:
diags = []
diags.append(res.group(1))
continue
if line != '':
print('Invalid tap output from %s: %s' %
(self.id(), line))
self.assertEqual(last_test, plan,
'%s planned %d but executed %d tests' % (
self.id(),
plan,
last_test))
def setUp(self):
self.loop = asyncio.get_event_loop()
sockpath = WatchmanInstance.getSharedInstance().getSockPath()
self.client = self.loop.run_until_complete(WatchmanClient.from_socket(sockpath))
def runTest(self):
env = os.environ.copy()
env["WATCHMAN_SOCK"] = WatchmanInstance.getSharedInstance().getSockPath()
dotted = os.path.normpath(self.id()).replace(os.sep, ".").replace("tests.integration.", "").replace(".php", "")
env["TMPDIR"] = os.path.join(tempfile.tempdir, dotted)
if os.name != "nt" and len(env["TMPDIR"]) > 94:
self.fail("temp dir name %s is too long for unix domain sockets" % env["TMPDIR"])
os.mkdir(env["TMPDIR"])
env["IN_PYTHON_HARNESS"] = "1"
proc = subprocess.Popen(self.getCommandArgs(), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(stdout, stderr) = proc.communicate()
status = proc.poll()
if status == -signal.SIGINT:
Interrupt.setInterrupted()
self.fail("Interrupted by SIGINT")
return
if status != 0:
self.fail("Exit status %d\n%s\n%s\n" % (status, stdout, stderr))
return
res_pat = re.compile("^(not )?ok (\d+) (.*)$")
diag_pat = re.compile("^# (.*)$")
plan_pat = re.compile("^1\.\.(\d+)$")
# Now parse the TAP output
lines = stdout.split("\n")
last_test = 0
diags = None
plan = None
for line in lines:
res = plan_pat.match(line)
if res:
plan = int(res.group(1))
continue
res = res_pat.match(line)
if res:
this_test = int(res.group(2))
if this_test != last_test + 1:
print(stdout, stderr)
self.fail("Expected test number %d, got %d" % (last_test + 1, this_test))
last_test = this_test
if res.group(1) == "not ":
# Failed
msg = line
if diags is not None:
msg = msg + "\n".join(diags)
self.fail(msg)
failed
diags = None
continue
res = diag_pat.match(line)
if res:
if diags is None:
diags = []
diags.append(res.group(1))
continue
if line != "":
print("Invalid tap output from %s: %s" % (self.id(), line))
self.assertEqual(last_test, plan, "%s planned %d but executed %s tests" % (self.id(), plan, last_test))
