def test_remote_fs_observer(remote_fs_observer, tmpdir):
from robocorp_ls_core import watchdog_wrapper
from robocorp_ls_core.watchdog_wrapper import PathInfo
from robocorp_ls_core.unittest_tools.fixtures import wait_for_test_condition
from robocorp_ls_core.watchdog_wrapper import IFSObserver
tmpdir.join("dir_not_rec").mkdir()
tmpdir.join("dir_rec").mkdir()
found = []
def on_change(filepath, *args):
found.append(filepath)
assert args == ("foo", "bar")
notifier = watchdog_wrapper.create_notifier(on_change, timeout=0.1)
observer: IFSObserver = remote_fs_observer
watch = observer.notify_on_any_change(
[
PathInfo(tmpdir.join("dir_not_rec"), False),
PathInfo(tmpdir.join("dir_rec"), True),
],
notifier.on_change,
call_args=("foo", "bar"),
)
try:
tmpdir.join("dir_not_rec").join("mya.txt").write("foo")
tmpdir.join("dir_not_rec").join("mya.libspec").write("foo")
tmpdir.join("dir_rec").join("myb.txt").write("foo")
tmpdir.join("dir_rec").join("myb.libspec").write("foo")
def collect_basenames():
import os.path
basenames = [os.path.basename(x) for x in found]
return set(basenames)
def check1():
expected = {"myb.txt", "mya.libspec", "myb.libspec", "mya.txt"}
return collect_basenames().issuperset(expected)
wait_for_test_condition(
check1, msg=lambda: f"Basenames found: {collect_basenames()}"
)
finally:
watch.stop_tracking()
notifier.dispose()
observer.dispose()
def __init__(self, builtin_libspec_dir=None, user_libspec_dir=None):
"""
:param __internal_libspec_dir__:
Only to be used in tests (to regenerate the builtins)!
"""
from robocorp_ls_core import watchdog_wrapper
self._libspec_failures_cache: Dict[tuple, bool] = {}
self._main_thread = threading.current_thread()
watch_impl = os.environ.get("ROBOTFRAMEWORK_LS_WATCH_IMPL", "watchdog")
if watch_impl not in ("watchdog", "fsnotify"):
log.info(
f"ROBOTFRAMEWORK_LS_WATCH_IMPL should be 'watchdog' or 'fsnotify'. Found: {watch_impl} (falling back to fsnotify)"
)
# i.e.: the default is watchdog, so, if a different one is set,
# presumably the default is not ok, so, fall back to watchdog.
watch_impl = "fsnotify"
self._observer = watchdog_wrapper.create_observer(
watch_impl, (".py", ".libspec")
)
self._file_changes_notifier = watchdog_wrapper.create_notifier(
self._on_file_changed, timeout=0.5
)
self._libspec_dir = self.get_internal_libspec_dir()
self._user_libspec_dir = user_libspec_dir or os.path.join(
self._libspec_dir, "user"
)
self._builtins_libspec_dir = (
builtin_libspec_dir
or self.get_internal_builtins_libspec_dir(self._libspec_dir)
)
log.debug("User libspec dir: %s", self._user_libspec_dir)
log.debug("Builtins libspec dir: %s", self._builtins_libspec_dir)
try:
os.makedirs(self._user_libspec_dir)
except:
# Ignore exception if it's already created.
pass
try:
os.makedirs(self._builtins_libspec_dir)
except:
# Ignore exception if it's already created.
pass
# Spec info found in the workspace
self._workspace_folder_uri_to_folder_info = {}
self._additional_pythonpath_folder_to_folder_info = {}
# Spec info found in the pythonpath
pythonpath_folder_to_folder_info = {}
for path in sys.path:
if path and os.path.isdir(path):
pythonpath_folder_to_folder_info[path] = _FolderInfo(
path, recursive=False
)
self._pythonpath_folder_to_folder_info = pythonpath_folder_to_folder_info
# Spec info found in internal dirs (autogenerated)
self._internal_folder_to_folder_info = {
self._user_libspec_dir: _FolderInfo(
self._user_libspec_dir, recursive=False
),
self._builtins_libspec_dir: _FolderInfo(
self._builtins_libspec_dir, recursive=False
),
}
# Must be set from the outside world when needed.
self.config = None
log.debug("Generating builtin libraries.")
self._gen_builtin_libraries()
log.debug("Synchronizing internal caches.")
self._synchronize()
log.debug("Finished initializing LibspecManager.")
def test_watchdog_extensions(tmpdir):
from robocorp_ls_core import watchdog_wrapper
from robocorp_ls_core.watchdog_wrapper import PathInfo
from robocorp_ls_core.unittest_tools.fixtures import wait_for_test_condition
tmpdir.join("dir_not_rec").mkdir()
tmpdir.join("dir_rec").mkdir()
found = []
def on_change(filepath, *args):
found.append(filepath)
assert args == ("foo", "bar")
notifier = watchdog_wrapper.create_notifier(on_change, timeout=0.1)
observer = watchdog_wrapper.create_observer()
watch = observer.notify_on_extensions_change(
[
PathInfo(tmpdir.join("dir_not_rec"), False),
PathInfo(tmpdir.join("dir_rec"), True),
],
["libspec"],
notifier.on_change,
call_args=("foo", "bar"),
)
try:
tmpdir.join("my.txt").write("foo")
tmpdir.join("my.libspec").write("foo")
tmpdir.join("dir_not_rec").join("mya.txt").write("foo")
tmpdir.join("dir_not_rec").join("mya.libspec").write("foo")
tmpdir.join("dir_rec").join("myb.txt").write("foo")
tmpdir.join("dir_rec").join("myb.libspec").write("foo")
def check1():
found_my_a = False
found_my_b = False
for filepath in found:
if not filepath.endswith(".libspec"):
raise AssertionError(
"Expected only libspec files to be tracked.")
if filepath.endswith("my.libspec"):
raise AssertionError("Wrong folder tracked.")
found_my_a = found_my_a or "mya.libspec" in filepath
found_my_b = found_my_b or "myb.libspec" in filepath
return found_my_a and found_my_b
wait_for_test_condition(
check1,
msg=lambda:
"Expected to find mya.libspec and myb.libspec. Found:\n%s" %
("\n".join(found), ),
)
# not listened
tmpdir.join("dir_not_rec").join("another").mkdir()
tmpdir.join("dir_not_rec").join("another").join("myc.txt").write("foo")
tmpdir.join("dir_not_rec").join("another").join("myc.libspec").write(
"foo")
# listened
tmpdir.join("dir_rec").join("another").mkdir()
tmpdir.join("dir_rec").join("another").join("myd.txt").write("foo")
tmpdir.join("dir_rec").join("another").join("myd.libspec").write("foo")
del found[:]
def check2():
found_my_d = False
for filepath in found:
if not filepath.endswith(".libspec"):
raise AssertionError(
"Expected only libspec files to be tracked.")
if filepath.endswith("myc.libspec"):
raise AssertionError("Wrong folder tracked.")
found_my_d = found_my_d or "myd.libspec" in filepath
return found_my_d
wait_for_test_condition(
check2,
msg=lambda: "Expected to find myd.libspec. Found:\n%s" %
("\n".join(found), ),
)
watch.stop_tracking()
del found[:]
tmpdir.join("dir_rec").join("mye.txt").write("foo")
tmpdir.join("dir_rec").join("mye.libspec").write("foo")
# Give time to check if some change arrives.
time.sleep(1)
assert not found
finally:
notifier.dispose()
observer.dispose()
def __init__(self, builtin_libspec_dir=None, user_libspec_dir=None):
"""
:param __internal_libspec_dir__:
Only to be used in tests (to regenerate the builtins)!
"""
from robocorp_ls_core import watchdog_wrapper
self._libspec_failures_cache: Dict[tuple, bool] = {}
self._main_thread = threading.current_thread()
self._observer = watchdog_wrapper.create_observer()
self._file_changes_notifier = watchdog_wrapper.create_notifier(
self._on_file_changed, timeout=0.5)
self._root_uri = None
self._libspec_dir = self.get_internal_libspec_dir()
self._user_libspec_dir = user_libspec_dir or os.path.join(
self._libspec_dir, "user")
self._builtins_libspec_dir = (
builtin_libspec_dir
or self.get_internal_builtins_libspec_dir(self._libspec_dir))
log.debug("User libspec dir: %s", self._user_libspec_dir)
log.debug("Builtins libspec dir: %s", self._builtins_libspec_dir)
try:
os.makedirs(self._user_libspec_dir)
except BaseException:
# Ignore exception if it's already created.
pass
try:
os.makedirs(self._builtins_libspec_dir)
except BaseException:
# Ignore exception if it's already created.
pass
self.libspec_errors: Dict[LibspecErrorEntry, str] = {}
self.libspec_warnings: Dict[LibspecErrorEntry, str] = {}
# Spec info found in the workspace
self._workspace_folder_uri_to_folder_info = {}
self._additional_pythonpath_folder_to_folder_info = {}
# Spec info found in the pythonpath
pythonpath_folder_to_folder_info = {}
for path in sys.path:
if path and os.path.isdir(path):
pythonpath_folder_to_folder_info[path] = _FolderInfo(
path, recursive=False)
self._pythonpath_folder_to_folder_info = pythonpath_folder_to_folder_info
# Spec info found in internal dirs (autogenerated)
self._internal_folder_to_folder_info = {
self._user_libspec_dir:
_FolderInfo(self._user_libspec_dir, recursive=False),
self._builtins_libspec_dir:
_FolderInfo(self._builtins_libspec_dir, recursive=False),
}
# Must be set from the outside world when needed.
self.config = None
self.process_pool = ProcessPoolExecutor()
# we have to submit a dummy process to start the process pool,
# because it does not start correctly if we do it from another
# thread
self.process_pool.submit(dummy_process).result()
self.thread_pool = ThreadPoolExecutor(
thread_name_prefix="libspec_manager_")
log.debug("Generating builtin libraries.")
self._gen_builtin_libraries()
log.debug("Synchronizing internal caches.")
self._synchronize()
log.debug("Finished initializing LibspecManager.")
def __init__(self, builtin_libspec_dir=None, user_libspec_dir=None):
"""
:param __internal_libspec_dir__:
Only to be used in tests (to regenerate the builtins)!
"""
from robocorp_ls_core import watchdog_wrapper
self._main_thread = threading.current_thread()
self._observer = watchdog_wrapper.create_observer()
self._spec_changes_notifier = watchdog_wrapper.create_notifier(
self._on_spec_file_changed, timeout=0.5)
self._libspec_dir = self.get_internal_libspec_dir()
self._user_libspec_dir = user_libspec_dir or os.path.join(
self._libspec_dir, "user")
self._builtins_libspec_dir = (
builtin_libspec_dir
or self.get_internal_builtins_libspec_dir(self._libspec_dir))
log.debug("User libspec dir: %s", self._user_libspec_dir)
log.debug("Builtins libspec dir: %s", self._builtins_libspec_dir)
try:
os.makedirs(self._user_libspec_dir)
except:
# Ignore exception if it's already created.
pass
try:
os.makedirs(self._builtins_libspec_dir)
except:
# Ignore exception if it's already created.
pass
# Spec info found in the workspace
self._workspace_folder_uri_to_folder_info = {}
self._additional_pythonpath_folder_to_folder_info = {}
# Spec info found in the pythonpath
pythonpath_folder_to_folder_info = {}
for path in sys.path:
if path and os.path.isdir(path):
pythonpath_folder_to_folder_info[path] = _FolderInfo(
path, recursive=False)
self._pythonpath_folder_to_folder_info = pythonpath_folder_to_folder_info
# Spec info found in internal dirs (autogenerated)
self._internal_folder_to_folder_info = {
self._user_libspec_dir:
_FolderInfo(self._user_libspec_dir, recursive=False),
self._builtins_libspec_dir:
_FolderInfo(self._builtins_libspec_dir, recursive=False),
}
# Must be set from the outside world when needed.
self.config = None
log.debug("Generating builtin libraries.")
self._gen_builtin_libraries()
log.debug("Synchronizing internal caches.")
self._synchronize()
log.debug("Finished initializing LibspecManager.")
def __init__(self, builtin_libspec_dir=None, user_libspec_dir=None):
"""
:param __internal_libspec_dir__:
Only to be used in tests (to regenerate the builtins)!
"""
from robocorp_ls_core import watchdog_wrapper
self._libspec_failures_cache: Dict[tuple, bool] = {}
self._main_thread = threading.current_thread()
watch_impl = os.environ.get("ROBOTFRAMEWORK_LS_WATCH_IMPL", "auto")
if watch_impl not in ("watchdog", "fsnotify", "auto"):
log.info(
f"ROBOTFRAMEWORK_LS_WATCH_IMPL should be 'auto', 'watchdog' or 'fsnotify'. Found: {watch_impl} (falling back to auto)"
)
watch_impl = "auto"
if watch_impl == "auto":
# In auto mode we use watchdog for windows and fsnotify (polling)
# for Linux and Mac. The reason for that is that on Linux and Mac
# if big folders are watched the system may complain due to the
# lack of resources, which may prevent the extension from working
# properly.
#
# If users want to opt-in, they can change to watchdog (and
# ideally install it to their env to get native extensions).
if sys.platform == "win32":
watch_impl = "watchdog"
else:
watch_impl = "fsnotify"
self._observer = watchdog_wrapper.create_observer(
watch_impl, (".py", ".libspec")
)
self._file_changes_notifier = watchdog_wrapper.create_notifier(
self._on_file_changed, timeout=0.5
)
self._libspec_dir = self.get_internal_libspec_dir()
self._user_libspec_dir = user_libspec_dir or os.path.join(
self._libspec_dir, "user"
)
self._builtins_libspec_dir = (
builtin_libspec_dir
or self.get_internal_builtins_libspec_dir(self._libspec_dir)
)
log.debug("User libspec dir: %s", self._user_libspec_dir)
log.debug("Builtins libspec dir: %s", self._builtins_libspec_dir)
try:
os.makedirs(self._user_libspec_dir)
except:
# Ignore exception if it's already created.
pass
try:
os.makedirs(self._builtins_libspec_dir)
except:
# Ignore exception if it's already created.
pass
# Spec info found in the workspace
self._workspace_folder_uri_to_folder_info = {}
self._additional_pythonpath_folder_to_folder_info = {}
# Spec info found in the pythonpath
pythonpath_folder_to_folder_info = {}
for path in sys.path:
if path and os.path.isdir(path):
pythonpath_folder_to_folder_info[path] = _FolderInfo(
path, recursive=False
)
self._pythonpath_folder_to_folder_info = pythonpath_folder_to_folder_info
# Spec info found in internal dirs (autogenerated)
self._internal_folder_to_folder_info = {
self._user_libspec_dir: _FolderInfo(
self._user_libspec_dir, recursive=False
),
self._builtins_libspec_dir: _FolderInfo(
self._builtins_libspec_dir, recursive=False
),
}
# Must be set from the outside world when needed.
self.config = None
log.debug("Generating builtin libraries.")
self._gen_builtin_libraries()
log.debug("Synchronizing internal caches.")
self._synchronize()
log.debug("Finished initializing LibspecManager.")