def download_rcc(location: str, force: bool = False) -> None:
"""
Downloads rcc to the given location. Note that we don't overwrite it if it
already exists (unless force == True).
:param location:
The location to store the rcc executable in the filesystem.
:param force:
Whether we should overwrite an existing installation.
"""
from robocorp_ls_core.system_mutex import timed_acquire_mutex
if not os.path.exists(location) or force:
with timed_acquire_mutex("robocorp_get_rcc", timeout=120):
if not os.path.exists(location) or force:
import platform
import urllib.request
machine = platform.machine()
is_64 = not machine or "64" in machine
if sys.platform == "win32":
if is_64:
url = "https://downloads.code.robocorp.com/rcc/v2/windows64/rcc.exe"
else:
url = "https://downloads.code.robocorp.com/rcc/v2/windows32/rcc.exe"
elif sys.platform == "darwin":
url = "https://downloads.code.robocorp.com/rcc/v2/macos64/rcc"
else:
if is_64:
url = "https://downloads.code.robocorp.com/rcc/v2/linux64/rcc"
else:
url = "https://downloads.code.robocorp.com/rcc/v2/linux32/rcc"
log.info(f"Downloading rcc from: {url} to: {location}.")
response = urllib.request.urlopen(url)
# Put it all in memory before writing (i.e. just write it if
# we know we downloaded everything).
data = response.read()
try:
os.makedirs(os.path.dirname(location))
except Exception:
pass # Error expected if the parent dir already exists.
try:
with open(location, "wb") as stream:
stream.write(data)
os.chmod(location, 0x744)
except Exception:
log.exception(
"Error writing to: %s.\nParent dir exists: %s",
location,
os.path.dirname(location),
)
raise
def _gen_builtin_libraries(self):
"""
Generates .libspec files for the libraries builtin (if needed).
"""
try:
import time
from concurrent import futures
from robotframework_ls.impl import robot_constants
from robocorp_ls_core.system_mutex import timed_acquire_mutex
from robocorp_ls_core.system_mutex import generate_mutex_name
from robotframework_ls.impl.robot_constants import RESERVED_LIB
initial_time = time.time()
wait_for = []
max_workers = min(10, (os.cpu_count() or 1) + 4)
thread_pool = futures.ThreadPoolExecutor(max_workers=max_workers)
try:
log.debug("Waiting for mutex to generate builtins.")
with timed_acquire_mutex(
generate_mutex_name(
_norm_filename(self._builtins_libspec_dir),
prefix="gen_builtins_",
),
timeout=100,
):
log.debug("Obtained mutex to generate builtins.")
for libname in robot_constants.STDLIBS:
if libname == RESERVED_LIB:
continue
builtins_libspec_dir = self._builtins_libspec_dir
if not os.path.exists(
os.path.join(builtins_libspec_dir, f"{libname}.libspec")
):
wait_for.append(
thread_pool.submit(
self._create_libspec, libname, is_builtin=True
)
)
for future in wait_for:
future.result()
if wait_for:
log.debug(
"Total time to generate builtins: %.2fs"
% (time.time() - initial_time)
)
self.synchronize_internal_libspec_folders()
finally:
thread_pool.shutdown(wait=False)
except:
log.exception("Error creating builtin libraries.")
finally:
log.info("Finished creating builtin libraries.")
def _gen_builtin_libraries(self):
"""
Generates .libspec files for the libraries builtin (if needed).
"""
try:
import time
from robotframework_ls.impl import robot_constants
from robocorp_ls_core.system_mutex import timed_acquire_mutex
from robocorp_ls_core.system_mutex import generate_mutex_name
initial_time = time.time()
wait_for = []
log.debug("Waiting for mutex to generate builtins.")
with timed_acquire_mutex(
generate_mutex_name(
_norm_filename(self._builtins_libspec_dir),
prefix="gen_builtins_",
),
timeout=100,
):
log.debug("Obtained mutex to generate builtins.")
def get_builtins():
try:
import robot.libraries
return robot.libraries.STDLIBS
except:
pass
return robot_constants.STDLIBS
for libname in get_builtins():
builtins_libspec_dir = self._builtins_libspec_dir
if not os.path.exists(
os.path.join(builtins_libspec_dir,
f"{libname}.libspec")):
wait_for.append(
self.thread_pool.submit(self._create_libspec,
libname,
is_builtin=True))
for future in wait_for:
future.result()
if wait_for:
log.debug("Total time to generate builtins: %.2fs" %
(time.time() - initial_time))
self.synchronize_internal_libspec_folders()
except BaseException:
log.exception("Error creating builtin libraries.")
finally:
log.info("Finished creating builtin libraries.")
def make_numbered_dir(root: Path, prefix: str) -> Path:
"""create a directory with an increased number as suffix for the given prefix"""
from robocorp_ls_core.system_mutex import generate_mutex_name
from robocorp_ls_core.system_mutex import timed_acquire_mutex
with timed_acquire_mutex(generate_mutex_name(f"generate_numbered{root}")):
max_existing = max(map(parse_num, find_suffixes(root, prefix)),
default=-1)
new_number = max_existing + 1
new_path = root.joinpath("{}{}".format(prefix, new_number))
new_path.mkdir()
return new_path
def _load_library_doc_and_mtime(spec_filename, obtain_mutex=True):
"""
:param obtain_mutex:
Should be False if this is part of a bigger operation that already
has the spec_filename mutex.
"""
from robotframework_ls.impl import robot_specbuilder
from robocorp_ls_core.system_mutex import timed_acquire_mutex
if obtain_mutex:
ctx = timed_acquire_mutex(_get_libspec_mutex_name(spec_filename))
else:
ctx = NULL
with ctx:
# We must load it with a mutex to avoid conflicts between generating/reading.
builder = robot_specbuilder.SpecDocBuilder()
try:
mtime = os.path.getmtime(spec_filename)
libdoc = builder.build(spec_filename)
return libdoc, mtime
except Exception:
log.exception("Error when loading spec info from: %s", spec_filename)
return None
def _cached_create_libspec(
self,
libname,
is_builtin: bool,
target_file: Optional[str],
*,
_internal_force_text=False, # Should only be set from within this function.
):
"""
:param str libname:
:raise Exception: if unable to create the library.
"""
import time
from robotframework_ls.impl import robot_constants
from robocorp_ls_core.subprocess_wrapper import subprocess
from robocorp_ls_core.system_mutex import timed_acquire_mutex
if _internal_force_text:
# In this case this is a recursive call and we already have the lock.
timed_acquire_mutex = NULL
additional_path = None
additional_path_exists = False
log_time = True
cwd = None
if target_file is not None:
additional_path = os.path.dirname(target_file)
additional_path_exists = os.path.exists(additional_path)
if additional_path and additional_path_exists:
cwd = additional_path
libname = os.path.basename(libname)
if libname.lower().endswith((".py", ".class", ".java")):
libname = os.path.splitext(libname)[0]
curtime = time.time()
try:
try:
call = [sys.executable]
major_version = self.get_robot_major_version()
if major_version < 4:
call.extend("-m robot.libdoc --format XML:HTML".split())
else:
# Use default values for libspec (--format XML:HTML is deprecated).
call.extend("-m robot.libdoc".split())
if additional_path and additional_path_exists:
call.extend(["-P", additional_path])
if _internal_force_text:
call.append("--docformat")
call.append("text")
additional_pythonpath_entries = list(
self._additional_pythonpath_folder_to_folder_info.keys()
)
for entry in list(additional_pythonpath_entries):
if os.path.exists(entry):
call.extend(["-P", entry])
call.append(libname)
libspec_dir = self._user_libspec_dir
if libname in robot_constants.STDLIBS:
libspec_dir = self._builtins_libspec_dir
if target_file:
import hashlib
digest = hashlib.sha256(
target_file.encode("utf-8", "replace")
).hexdigest()[:8]
libspec_filename = os.path.join(libspec_dir, digest + ".libspec")
else:
libspec_filename = os.path.join(libspec_dir, libname + ".libspec")
log.debug(f"Obtaining mutex to generate libspec: {libspec_filename}.")
with timed_acquire_mutex(
_get_libspec_mutex_name(libspec_filename)
): # Could fail.
log.debug(
f"Obtained mutex to generate libspec: {libspec_filename}."
)
call.append(libspec_filename)
mtime: float = -1
try:
mtime = os.path.getmtime(libspec_filename)
except:
pass
log.debug(
"Generating libspec for: %s.\nCwd:%s\nCommand line:\n%s",
libname,
cwd,
" ".join(call),
)
try:
try:
# Note: stdout is always subprocess.PIPE in this call.
# Note: the env is always inherited (the process which has
# the LibspecManager must be the target env already).
self._subprocess_check_output(
call,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
cwd=cwd,
)
except OSError as e:
log.exception("Error calling: %s", call)
# We may have something as: Ignore OSError: [WinError 6] The handle is invalid,
# give the result based on whether the file changed on disk.
try:
if mtime != os.path.getmtime(libspec_filename):
_dump_spec_filename_additional_info(
libspec_filename,
is_builtin=is_builtin,
obtain_mutex=False,
)
return True
except:
pass
log.debug("Not retrying after OSError failure.")
return False
except subprocess.CalledProcessError as e:
if not _internal_force_text:
if (
b"reST format requires 'docutils' module to be installed"
in e.output
):
return self._cached_create_libspec(
libname,
is_builtin,
target_file,
_internal_force_text=True,
)
log.exception(
"Error creating libspec: %s.\nReturn code: %s\nOutput:\n%s",
libname,
e.returncode,
e.output,
)
return False
_dump_spec_filename_additional_info(
libspec_filename, is_builtin=is_builtin, obtain_mutex=False
)
return True
except Exception:
log.exception("Error creating libspec: %s", libname)
return False
finally:
if log_time:
delta = time.time() - curtime
log.debug("Took: %.2fs to generate info for: %s" % (delta, libname))
def _run_rcc(
self,
args: List[str],
timeout: float = 30,
error_msg: str = "",
mutex_name=None,
cwd: Optional[str] = None,
log_errors=True,
stderr=Sentinel.SENTINEL,
) -> ActionResult[str]:
"""
Returns an ActionResult where the result is the stdout of the executed command.
:param log_errors:
If false, errors won't be logged (i.e.: should be false when errors
are expected).
"""
from robocorp_ls_core.basic import build_subprocess_kwargs
from subprocess import check_output
from robocorp_ls_core.subprocess_wrapper import subprocess
if stderr is Sentinel.SENTINEL:
stderr = subprocess.PIPE
rcc_location = self.get_rcc_location()
env = os.environ.copy()
env.pop("PYTHONPATH", "")
env.pop("PYTHONHOME", "")
env.pop("VIRTUAL_ENV", "")
env["PYTHONIOENCODING"] = "utf-8"
env["PYTHONUNBUFFERED"] = "1"
robocorp_home = self._get_robocorp_home()
if robocorp_home:
env["ROBOCORP_HOME"] = robocorp_home
kwargs: dict = build_subprocess_kwargs(cwd, env, stderr=stderr)
args = [rcc_location] + args + ["--controller", "RobocorpCode"]
cmdline = " ".join([str(x) for x in args])
try:
if mutex_name:
from robocorp_ls_core.system_mutex import timed_acquire_mutex
else:
timed_acquire_mutex = NULL
with timed_acquire_mutex(mutex_name, timeout=15):
boutput: bytes = check_output(args, timeout=timeout, **kwargs)
except CalledProcessError as e:
stdout = as_str(e.stdout)
stderr = as_str(e.stderr)
msg = f"Error running: {cmdline}.\nROBOCORP_HOME: {robocorp_home}\n\nStdout: {stdout}\nStderr: {stderr}"
if log_errors:
log.exception(msg)
if not error_msg:
return ActionResult(success=False, message=msg)
else:
additional_info = [error_msg]
if stdout or stderr:
if stdout and stderr:
additional_info.append("\nDetails: ")
additional_info.append("\nStdout")
additional_info.append(stdout)
additional_info.append("\nStderr")
additional_info.append(stderr)
elif stdout:
additional_info.append("\nDetails: ")
additional_info.append(stdout)
elif stderr:
additional_info.append("\nDetails: ")
additional_info.append(stderr)
return ActionResult(success=False,
message="".join(additional_info))
except Exception:
msg = f"Error running: {args}"
log.exception(msg)
return ActionResult(success=False, message=msg)
output = boutput.decode("utf-8", "replace")
log.debug("Output from: %s:\n%s", cmdline, output)
return ActionResult(success=True, message=None, result=output)
def _cached_create_libspec(self, libname, env, log_time, cwd,
additional_path, is_builtin, arguments, alias,
current_doc_uri):
"""
:param str libname:
:raise Exception: if unable to create the library.
"""
import time
from robotframework_ls.impl import robot_constants
from robocorp_ls_core.system_mutex import timed_acquire_mutex
from multiprocessing import Process
from robotframework_ls.impl.generate_libdoc import run_doc
curtime = time.time()
try:
try:
additional_pythonpath_entries = list([
x.folder_path for x in
self._additional_pythonpath_folder_to_folder_info.values()
])
libargs = "::".join(
arguments
) if arguments is not None and len(arguments) > 0 else None
libspec_dir = self._user_libspec_dir
if libname in robot_constants.STDLIBS:
libspec_dir = self._builtins_libspec_dir
encoded_libname = libname
if libargs is not None:
digest = hashlib.sha256(libargs.encode()).hexdigest()[:8]
encoded_libname += f"__{digest}"
libspec_filename = os.path.join(libspec_dir,
encoded_libname + ".libspec")
libspec_error_entry = LibspecErrorEntry(
libname, arguments, alias, current_doc_uri)
if libspec_error_entry in self.libspec_errors:
del self.libspec_errors[libspec_error_entry]
if libspec_error_entry in self.libspec_warnings:
del self.libspec_warnings[libspec_error_entry]
log.debug(
f"Obtaining mutex to generate libpsec: {libspec_filename}."
)
# TODO define builtin vars
variables = {
"${CURDIR}": additional_path,
"${TEMPDIR}": os.path.abspath(tempfile.gettempdir()),
"${EXECDIR}": os.path.abspath("."),
"${/}": os.sep,
"${:}": os.pathsep,
"${\\n}": os.linesep,
'${SPACE}': ' ',
'${True}': True,
'${False}': False,
'${None}': None,
'${null}': None,
"${TEST NAME}": None,
"@{TEST TAGS}": [],
"${TEST DOCUMENTATION}": None,
"${TEST STATUS}": None,
"${TEST MESSAGE}": None,
"${PREV TEST NAME}": None,
"${PREV TEST STATUS}": None,
"${PREV TEST MESSAGE}": None,
"${SUITE NAME}": None,
"${SUITE SOURCE}": None,
"${SUITE DOCUMENTATION}": None,
"&{SUITE METADATA}": {},
"${SUITE STATUS}": None,
"${SUITE MESSAGE}": None,
"${KEYWORD STATUS}": None,
"${KEYWORD MESSAGE}": None,
"${LOG LEVEL}": None,
"${OUTPUT FILE}": None,
"${LOG FILE}": None,
"${REPORT FILE}": None,
"${DEBUG FILE}": None,
"${OUTPUT DIR}": None,
}
with timed_acquire_mutex(
_get_libspec_mutex_name(libspec_filename)):
try:
# remove old
if os.path.exists(libspec_filename):
log.info("remove old spec file %s",
libspec_filename)
os.remove(libspec_filename)
additional_libspec_filename = _get_additional_info_filename(
libspec_filename)
if os.path.exists(additional_libspec_filename):
os.remove(additional_libspec_filename)
future = self.process_pool.submit(
run_doc,
f"{libname}{f'::{libargs}' if libargs else ''}",
libspec_filename, additional_path,
additional_pythonpath_entries, variables)
_, error, warning = future.result(100)
if warning is not None:
self.libspec_warnings[
libspec_error_entry] = warning
if error is not None:
self.libspec_errors[libspec_error_entry] = error
else:
_dump_spec_filename_additional_info(
libspec_filename,
is_builtin=is_builtin,
obtain_mutex=False,
arguments=arguments,
alias=alias)
except BaseException as e:
self.libspec_errors[libspec_error_entry] = str(e)
raise
return True
except Exception as e:
log.exception("Error creating libspec: %s\n%s", libname, e)
return False
finally:
if log_time:
delta = time.time() - curtime
log.debug("Took: %.2fs to generate info for: %s" %
(delta, libname))
def _run_rcc(
self,
args: List[str],
timeout: float = 30,
expect_ok=True,
error_msg: str = "",
mutex_name=None,
cwd: Optional[str] = None,
) -> ActionResult[str]:
"""
Returns an ActionResult where the result is the stdout of the executed command.
"""
from robocorp_ls_core.basic import build_subprocess_kwargs
from subprocess import check_output
from robocorp_ls_core.subprocess_wrapper import subprocess
rcc_location = self.get_rcc_location()
env = os.environ.copy()
env.pop("PYTHONPATH", "")
env.pop("PYTHONHOME", "")
env.pop("VIRTUAL_ENV", "")
env["PYTHONIOENCODING"] = "utf-8"
env["PYTHONUNBUFFERED"] = "1"
kwargs: dict = build_subprocess_kwargs(cwd, env, stderr=subprocess.PIPE)
args = [rcc_location] + args
cmdline = " ".join([str(x) for x in args])
try:
if mutex_name:
from robocorp_ls_core.system_mutex import timed_acquire_mutex
else:
timed_acquire_mutex = NULL
with timed_acquire_mutex(mutex_name, timeout=15):
boutput: bytes = check_output(args, timeout=timeout, **kwargs)
except CalledProcessError as e:
stdout = as_str(e.stdout)
stderr = as_str(e.stderr)
msg = f"Error running: {cmdline}.\nStdout: {stdout}\nStderr: {stderr}"
log.exception(msg)
if not error_msg:
return ActionResult(success=False, message=msg)
else:
additional_info = [error_msg]
if stdout or stderr:
if stdout and stderr:
additional_info.append("\nDetails: ")
additional_info.append("\nStdout")
additional_info.append(stdout)
additional_info.append("\nStderr")
additional_info.append(stderr)
elif stdout:
additional_info.append("\nDetails: ")
additional_info.append(stdout)
elif stderr:
additional_info.append("\nDetails: ")
additional_info.append(stderr)
return ActionResult(success=False, message="".join(additional_info))
except Exception:
msg = f"Error running: {args}"
log.exception(msg)
return ActionResult(success=False, message=msg)
output = boutput.decode("utf-8", "replace")
log.debug(f"Output from: {cmdline}:\n{output}")
if expect_ok:
if "OK." in output:
return ActionResult(success=True, message=None, result=output)
else:
return ActionResult(success=True, message=None, result=output)
return ActionResult(
success=False, message="OK. not found in message", result=output
)
def _create_libspec(
self,
libname,
env=None,
log_time=True,
cwd=None,
additional_path=None,
is_builtin=False,
):
"""
:param str libname:
:raise Exception: if unable to create the library.
"""
import time
from robotframework_ls.impl import robot_constants
from robocorp_ls_core.subprocess_wrapper import subprocess
from robocorp_ls_core.system_mutex import timed_acquire_mutex
curtime = time.time()
try:
try:
call = [sys.executable]
call.extend("-m robot.libdoc --format XML:HTML".split())
if additional_path:
if os.path.exists(additional_path):
call.extend(["-P", additional_path])
additional_pythonpath_entries = list(
self._additional_pythonpath_folder_to_folder_info.keys())
for entry in list(additional_pythonpath_entries):
if os.path.exists(entry):
call.extend(["-P", entry])
call.append(libname)
libspec_dir = self._user_libspec_dir
if libname in robot_constants.STDLIBS:
libspec_dir = self._builtins_libspec_dir
libspec_filename = os.path.join(libspec_dir,
libname + ".libspec")
log.debug(
f"Obtaining mutex to generate libpsec: {libspec_filename}."
)
with timed_acquire_mutex(
_get_libspec_mutex_name(
libspec_filename)): # Could fail.
log.debug(
f"Obtained mutex to generate libpsec: {libspec_filename}."
)
call.append(libspec_filename)
mtime = -1
try:
mtime = os.path.getmtime(libspec_filename)
except:
pass
log.debug(
"Generating libspec for: %s.\nCwd:%s\nCommand line:\n%s",
libname,
cwd,
" ".join(call),
)
try:
try:
# Note: stdout is always subprocess.PIPE in this call.
subprocess.check_output(
call,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
env=env,
cwd=cwd,
)
except OSError as e:
log.exception("Error calling: %s", call)
# We may have something as: Ignore OSError: [WinError 6] The handle is invalid,
# give the result based on whether the file changed on disk.
try:
if mtime != os.path.getmtime(libspec_filename):
_dump_spec_filename_additional_info(
libspec_filename,
is_builtin=is_builtin,
obtain_mutex=False,
)
return True
except:
pass
log.debug("Not retrying after OSError failure.")
return False
except subprocess.CalledProcessError as e:
log.exception(
"Error creating libspec: %s. Output:\n%s", libname,
e.output)
return False
_dump_spec_filename_additional_info(libspec_filename,
is_builtin=is_builtin,
obtain_mutex=False)
return True
except Exception:
log.exception("Error creating libspec: %s", libname)
return False
finally:
if log_time:
delta = time.time() - curtime
log.debug("Took: %.2fs to generate info for: %s" %
(delta, libname))
def test_system_mutex():
from robocorp_ls_core.system_mutex import SystemMutex
from robocorp_ls_core.system_mutex import timed_acquire_mutex
from robocorp_ls_core.subprocess_wrapper import subprocess
import sys
import pytest
import time
import threading
import weakref
mutex_name = "mutex_name_test_system_mutex"
system_mutex = SystemMutex(mutex_name)
assert system_mutex.get_mutex_aquired()
class Check2Thread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.worked = False
def run(self):
mutex2 = SystemMutex(mutex_name)
assert not mutex2.get_mutex_aquired()
self.worked = True
t = Check2Thread()
t.start()
t.join()
assert t.worked
assert not system_mutex.disposed
system_mutex.release_mutex()
assert system_mutex.disposed
mutex3 = SystemMutex(mutex_name)
assert not mutex3.disposed
assert mutex3.get_mutex_aquired()
mutex3 = weakref.ref(mutex3) # Garbage-collected
# Calling release more times should not be an error
system_mutex.release_mutex()
mutex4 = SystemMutex(mutex_name)
assert mutex4.get_mutex_aquired()
with pytest.raises(AssertionError):
SystemMutex("mutex/") # Invalid name
time_to_release_mutex = 1
def release_mutex():
time.sleep(time_to_release_mutex)
mutex4.release_mutex()
t = threading.Thread(target=release_mutex)
t.start()
initial_time = time.time()
with timed_acquire_mutex(
mutex_name, check_reentrant=False
): # The current mutex will be released in a thread, so, check_reentrant=False.
acquired_time = time.time()
# Should timeout as the lock is already acquired.
with pytest.raises(RuntimeError) as exc:
with timed_acquire_mutex(mutex_name, timeout=1):
pass
assert "not a reentrant mutex" in str(exc)
# Must also fail from another process.
code = """
from robocorp_ls_core.system_mutex import timed_acquire_mutex
mutex_name = "mutex_name_test_system_mutex"
with timed_acquire_mutex(mutex_name, timeout=1):
pass
"""
with pytest.raises(subprocess.CalledProcessError):
subprocess.check_call([sys.executable, "-c", code], stderr=subprocess.PIPE)
assert acquired_time - initial_time > time_to_release_mutex
