def download_rcc(location: str, force: bool = False) -> None: """ Downloads rcc to the given location. Note that we don't overwrite it if it already exists (unless force == True). :param location: The location to store the rcc executable in the filesystem. :param force: Whether we should overwrite an existing installation. """ from robocorp_ls_core.system_mutex import timed_acquire_mutex if not os.path.exists(location) or force: with timed_acquire_mutex("robocorp_get_rcc", timeout=120): if not os.path.exists(location) or force: import platform import urllib.request machine = platform.machine() is_64 = not machine or "64" in machine if sys.platform == "win32": if is_64: url = "https://downloads.code.robocorp.com/rcc/v2/windows64/rcc.exe" else: url = "https://downloads.code.robocorp.com/rcc/v2/windows32/rcc.exe" elif sys.platform == "darwin": url = "https://downloads.code.robocorp.com/rcc/v2/macos64/rcc" else: if is_64: url = "https://downloads.code.robocorp.com/rcc/v2/linux64/rcc" else: url = "https://downloads.code.robocorp.com/rcc/v2/linux32/rcc" log.info(f"Downloading rcc from: {url} to: {location}.") response = urllib.request.urlopen(url) # Put it all in memory before writing (i.e. just write it if # we know we downloaded everything). data = response.read() try: os.makedirs(os.path.dirname(location)) except Exception: pass # Error expected if the parent dir already exists. try: with open(location, "wb") as stream: stream.write(data) os.chmod(location, 0x744) except Exception: log.exception( "Error writing to: %s.\nParent dir exists: %s", location, os.path.dirname(location), ) raise
def _gen_builtin_libraries(self): """ Generates .libspec files for the libraries builtin (if needed). """ try: import time from concurrent import futures from robotframework_ls.impl import robot_constants from robocorp_ls_core.system_mutex import timed_acquire_mutex from robocorp_ls_core.system_mutex import generate_mutex_name from robotframework_ls.impl.robot_constants import RESERVED_LIB initial_time = time.time() wait_for = [] max_workers = min(10, (os.cpu_count() or 1) + 4) thread_pool = futures.ThreadPoolExecutor(max_workers=max_workers) try: log.debug("Waiting for mutex to generate builtins.") with timed_acquire_mutex( generate_mutex_name( _norm_filename(self._builtins_libspec_dir), prefix="gen_builtins_", ), timeout=100, ): log.debug("Obtained mutex to generate builtins.") for libname in robot_constants.STDLIBS: if libname == RESERVED_LIB: continue builtins_libspec_dir = self._builtins_libspec_dir if not os.path.exists( os.path.join(builtins_libspec_dir, f"{libname}.libspec") ): wait_for.append( thread_pool.submit( self._create_libspec, libname, is_builtin=True ) ) for future in wait_for: future.result() if wait_for: log.debug( "Total time to generate builtins: %.2fs" % (time.time() - initial_time) ) self.synchronize_internal_libspec_folders() finally: thread_pool.shutdown(wait=False) except: log.exception("Error creating builtin libraries.") finally: log.info("Finished creating builtin libraries.")
def _gen_builtin_libraries(self): """ Generates .libspec files for the libraries builtin (if needed). """ try: import time from robotframework_ls.impl import robot_constants from robocorp_ls_core.system_mutex import timed_acquire_mutex from robocorp_ls_core.system_mutex import generate_mutex_name initial_time = time.time() wait_for = [] log.debug("Waiting for mutex to generate builtins.") with timed_acquire_mutex( generate_mutex_name( _norm_filename(self._builtins_libspec_dir), prefix="gen_builtins_", ), timeout=100, ): log.debug("Obtained mutex to generate builtins.") def get_builtins(): try: import robot.libraries return robot.libraries.STDLIBS except: pass return robot_constants.STDLIBS for libname in get_builtins(): builtins_libspec_dir = self._builtins_libspec_dir if not os.path.exists( os.path.join(builtins_libspec_dir, f"{libname}.libspec")): wait_for.append( self.thread_pool.submit(self._create_libspec, libname, is_builtin=True)) for future in wait_for: future.result() if wait_for: log.debug("Total time to generate builtins: %.2fs" % (time.time() - initial_time)) self.synchronize_internal_libspec_folders() except BaseException: log.exception("Error creating builtin libraries.") finally: log.info("Finished creating builtin libraries.")
def make_numbered_dir(root: Path, prefix: str) -> Path: """create a directory with an increased number as suffix for the given prefix""" from robocorp_ls_core.system_mutex import generate_mutex_name from robocorp_ls_core.system_mutex import timed_acquire_mutex with timed_acquire_mutex(generate_mutex_name(f"generate_numbered{root}")): max_existing = max(map(parse_num, find_suffixes(root, prefix)), default=-1) new_number = max_existing + 1 new_path = root.joinpath("{}{}".format(prefix, new_number)) new_path.mkdir() return new_path
def _load_library_doc_and_mtime(spec_filename, obtain_mutex=True): """ :param obtain_mutex: Should be False if this is part of a bigger operation that already has the spec_filename mutex. """ from robotframework_ls.impl import robot_specbuilder from robocorp_ls_core.system_mutex import timed_acquire_mutex if obtain_mutex: ctx = timed_acquire_mutex(_get_libspec_mutex_name(spec_filename)) else: ctx = NULL with ctx: # We must load it with a mutex to avoid conflicts between generating/reading. builder = robot_specbuilder.SpecDocBuilder() try: mtime = os.path.getmtime(spec_filename) libdoc = builder.build(spec_filename) return libdoc, mtime except Exception: log.exception("Error when loading spec info from: %s", spec_filename) return None
def _cached_create_libspec( self, libname, is_builtin: bool, target_file: Optional[str], *, _internal_force_text=False, # Should only be set from within this function. ): """ :param str libname: :raise Exception: if unable to create the library. """ import time from robotframework_ls.impl import robot_constants from robocorp_ls_core.subprocess_wrapper import subprocess from robocorp_ls_core.system_mutex import timed_acquire_mutex if _internal_force_text: # In this case this is a recursive call and we already have the lock. timed_acquire_mutex = NULL additional_path = None additional_path_exists = False log_time = True cwd = None if target_file is not None: additional_path = os.path.dirname(target_file) additional_path_exists = os.path.exists(additional_path) if additional_path and additional_path_exists: cwd = additional_path libname = os.path.basename(libname) if libname.lower().endswith((".py", ".class", ".java")): libname = os.path.splitext(libname)[0] curtime = time.time() try: try: call = [sys.executable] major_version = self.get_robot_major_version() if major_version < 4: call.extend("-m robot.libdoc --format XML:HTML".split()) else: # Use default values for libspec (--format XML:HTML is deprecated). call.extend("-m robot.libdoc".split()) if additional_path and additional_path_exists: call.extend(["-P", additional_path]) if _internal_force_text: call.append("--docformat") call.append("text") additional_pythonpath_entries = list( self._additional_pythonpath_folder_to_folder_info.keys() ) for entry in list(additional_pythonpath_entries): if os.path.exists(entry): call.extend(["-P", entry]) call.append(libname) libspec_dir = self._user_libspec_dir if libname in robot_constants.STDLIBS: libspec_dir = self._builtins_libspec_dir if target_file: import hashlib digest = hashlib.sha256( target_file.encode("utf-8", "replace") ).hexdigest()[:8] libspec_filename = os.path.join(libspec_dir, digest + ".libspec") else: libspec_filename = os.path.join(libspec_dir, libname + ".libspec") log.debug(f"Obtaining mutex to generate libspec: {libspec_filename}.") with timed_acquire_mutex( _get_libspec_mutex_name(libspec_filename) ): # Could fail. log.debug( f"Obtained mutex to generate libspec: {libspec_filename}." ) call.append(libspec_filename) mtime: float = -1 try: mtime = os.path.getmtime(libspec_filename) except: pass log.debug( "Generating libspec for: %s.\nCwd:%s\nCommand line:\n%s", libname, cwd, " ".join(call), ) try: try: # Note: stdout is always subprocess.PIPE in this call. # Note: the env is always inherited (the process which has # the LibspecManager must be the target env already). self._subprocess_check_output( call, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, cwd=cwd, ) except OSError as e: log.exception("Error calling: %s", call) # We may have something as: Ignore OSError: [WinError 6] The handle is invalid, # give the result based on whether the file changed on disk. try: if mtime != os.path.getmtime(libspec_filename): _dump_spec_filename_additional_info( libspec_filename, is_builtin=is_builtin, obtain_mutex=False, ) return True except: pass log.debug("Not retrying after OSError failure.") return False except subprocess.CalledProcessError as e: if not _internal_force_text: if ( b"reST format requires 'docutils' module to be installed" in e.output ): return self._cached_create_libspec( libname, is_builtin, target_file, _internal_force_text=True, ) log.exception( "Error creating libspec: %s.\nReturn code: %s\nOutput:\n%s", libname, e.returncode, e.output, ) return False _dump_spec_filename_additional_info( libspec_filename, is_builtin=is_builtin, obtain_mutex=False ) return True except Exception: log.exception("Error creating libspec: %s", libname) return False finally: if log_time: delta = time.time() - curtime log.debug("Took: %.2fs to generate info for: %s" % (delta, libname))
def _run_rcc( self, args: List[str], timeout: float = 30, error_msg: str = "", mutex_name=None, cwd: Optional[str] = None, log_errors=True, stderr=Sentinel.SENTINEL, ) -> ActionResult[str]: """ Returns an ActionResult where the result is the stdout of the executed command. :param log_errors: If false, errors won't be logged (i.e.: should be false when errors are expected). """ from robocorp_ls_core.basic import build_subprocess_kwargs from subprocess import check_output from robocorp_ls_core.subprocess_wrapper import subprocess if stderr is Sentinel.SENTINEL: stderr = subprocess.PIPE rcc_location = self.get_rcc_location() env = os.environ.copy() env.pop("PYTHONPATH", "") env.pop("PYTHONHOME", "") env.pop("VIRTUAL_ENV", "") env["PYTHONIOENCODING"] = "utf-8" env["PYTHONUNBUFFERED"] = "1" robocorp_home = self._get_robocorp_home() if robocorp_home: env["ROBOCORP_HOME"] = robocorp_home kwargs: dict = build_subprocess_kwargs(cwd, env, stderr=stderr) args = [rcc_location] + args + ["--controller", "RobocorpCode"] cmdline = " ".join([str(x) for x in args]) try: if mutex_name: from robocorp_ls_core.system_mutex import timed_acquire_mutex else: timed_acquire_mutex = NULL with timed_acquire_mutex(mutex_name, timeout=15): boutput: bytes = check_output(args, timeout=timeout, **kwargs) except CalledProcessError as e: stdout = as_str(e.stdout) stderr = as_str(e.stderr) msg = f"Error running: {cmdline}.\nROBOCORP_HOME: {robocorp_home}\n\nStdout: {stdout}\nStderr: {stderr}" if log_errors: log.exception(msg) if not error_msg: return ActionResult(success=False, message=msg) else: additional_info = [error_msg] if stdout or stderr: if stdout and stderr: additional_info.append("\nDetails: ") additional_info.append("\nStdout") additional_info.append(stdout) additional_info.append("\nStderr") additional_info.append(stderr) elif stdout: additional_info.append("\nDetails: ") additional_info.append(stdout) elif stderr: additional_info.append("\nDetails: ") additional_info.append(stderr) return ActionResult(success=False, message="".join(additional_info)) except Exception: msg = f"Error running: {args}" log.exception(msg) return ActionResult(success=False, message=msg) output = boutput.decode("utf-8", "replace") log.debug("Output from: %s:\n%s", cmdline, output) return ActionResult(success=True, message=None, result=output)
def _cached_create_libspec(self, libname, env, log_time, cwd, additional_path, is_builtin, arguments, alias, current_doc_uri): """ :param str libname: :raise Exception: if unable to create the library. """ import time from robotframework_ls.impl import robot_constants from robocorp_ls_core.system_mutex import timed_acquire_mutex from multiprocessing import Process from robotframework_ls.impl.generate_libdoc import run_doc curtime = time.time() try: try: additional_pythonpath_entries = list([ x.folder_path for x in self._additional_pythonpath_folder_to_folder_info.values() ]) libargs = "::".join( arguments ) if arguments is not None and len(arguments) > 0 else None libspec_dir = self._user_libspec_dir if libname in robot_constants.STDLIBS: libspec_dir = self._builtins_libspec_dir encoded_libname = libname if libargs is not None: digest = hashlib.sha256(libargs.encode()).hexdigest()[:8] encoded_libname += f"__{digest}" libspec_filename = os.path.join(libspec_dir, encoded_libname + ".libspec") libspec_error_entry = LibspecErrorEntry( libname, arguments, alias, current_doc_uri) if libspec_error_entry in self.libspec_errors: del self.libspec_errors[libspec_error_entry] if libspec_error_entry in self.libspec_warnings: del self.libspec_warnings[libspec_error_entry] log.debug( f"Obtaining mutex to generate libpsec: {libspec_filename}." ) # TODO define builtin vars variables = { "${CURDIR}": additional_path, "${TEMPDIR}": os.path.abspath(tempfile.gettempdir()), "${EXECDIR}": os.path.abspath("."), "${/}": os.sep, "${:}": os.pathsep, "${\\n}": os.linesep, '${SPACE}': ' ', '${True}': True, '${False}': False, '${None}': None, '${null}': None, "${TEST NAME}": None, "@{TEST TAGS}": [], "${TEST DOCUMENTATION}": None, "${TEST STATUS}": None, "${TEST MESSAGE}": None, "${PREV TEST NAME}": None, "${PREV TEST STATUS}": None, "${PREV TEST MESSAGE}": None, "${SUITE NAME}": None, "${SUITE SOURCE}": None, "${SUITE DOCUMENTATION}": None, "&{SUITE METADATA}": {}, "${SUITE STATUS}": None, "${SUITE MESSAGE}": None, "${KEYWORD STATUS}": None, "${KEYWORD MESSAGE}": None, "${LOG LEVEL}": None, "${OUTPUT FILE}": None, "${LOG FILE}": None, "${REPORT FILE}": None, "${DEBUG FILE}": None, "${OUTPUT DIR}": None, } with timed_acquire_mutex( _get_libspec_mutex_name(libspec_filename)): try: # remove old if os.path.exists(libspec_filename): log.info("remove old spec file %s", libspec_filename) os.remove(libspec_filename) additional_libspec_filename = _get_additional_info_filename( libspec_filename) if os.path.exists(additional_libspec_filename): os.remove(additional_libspec_filename) future = self.process_pool.submit( run_doc, f"{libname}{f'::{libargs}' if libargs else ''}", libspec_filename, additional_path, additional_pythonpath_entries, variables) _, error, warning = future.result(100) if warning is not None: self.libspec_warnings[ libspec_error_entry] = warning if error is not None: self.libspec_errors[libspec_error_entry] = error else: _dump_spec_filename_additional_info( libspec_filename, is_builtin=is_builtin, obtain_mutex=False, arguments=arguments, alias=alias) except BaseException as e: self.libspec_errors[libspec_error_entry] = str(e) raise return True except Exception as e: log.exception("Error creating libspec: %s\n%s", libname, e) return False finally: if log_time: delta = time.time() - curtime log.debug("Took: %.2fs to generate info for: %s" % (delta, libname))
def _run_rcc( self, args: List[str], timeout: float = 30, expect_ok=True, error_msg: str = "", mutex_name=None, cwd: Optional[str] = None, ) -> ActionResult[str]: """ Returns an ActionResult where the result is the stdout of the executed command. """ from robocorp_ls_core.basic import build_subprocess_kwargs from subprocess import check_output from robocorp_ls_core.subprocess_wrapper import subprocess rcc_location = self.get_rcc_location() env = os.environ.copy() env.pop("PYTHONPATH", "") env.pop("PYTHONHOME", "") env.pop("VIRTUAL_ENV", "") env["PYTHONIOENCODING"] = "utf-8" env["PYTHONUNBUFFERED"] = "1" kwargs: dict = build_subprocess_kwargs(cwd, env, stderr=subprocess.PIPE) args = [rcc_location] + args cmdline = " ".join([str(x) for x in args]) try: if mutex_name: from robocorp_ls_core.system_mutex import timed_acquire_mutex else: timed_acquire_mutex = NULL with timed_acquire_mutex(mutex_name, timeout=15): boutput: bytes = check_output(args, timeout=timeout, **kwargs) except CalledProcessError as e: stdout = as_str(e.stdout) stderr = as_str(e.stderr) msg = f"Error running: {cmdline}.\nStdout: {stdout}\nStderr: {stderr}" log.exception(msg) if not error_msg: return ActionResult(success=False, message=msg) else: additional_info = [error_msg] if stdout or stderr: if stdout and stderr: additional_info.append("\nDetails: ") additional_info.append("\nStdout") additional_info.append(stdout) additional_info.append("\nStderr") additional_info.append(stderr) elif stdout: additional_info.append("\nDetails: ") additional_info.append(stdout) elif stderr: additional_info.append("\nDetails: ") additional_info.append(stderr) return ActionResult(success=False, message="".join(additional_info)) except Exception: msg = f"Error running: {args}" log.exception(msg) return ActionResult(success=False, message=msg) output = boutput.decode("utf-8", "replace") log.debug(f"Output from: {cmdline}:\n{output}") if expect_ok: if "OK." in output: return ActionResult(success=True, message=None, result=output) else: return ActionResult(success=True, message=None, result=output) return ActionResult( success=False, message="OK. not found in message", result=output )
def _create_libspec( self, libname, env=None, log_time=True, cwd=None, additional_path=None, is_builtin=False, ): """ :param str libname: :raise Exception: if unable to create the library. """ import time from robotframework_ls.impl import robot_constants from robocorp_ls_core.subprocess_wrapper import subprocess from robocorp_ls_core.system_mutex import timed_acquire_mutex curtime = time.time() try: try: call = [sys.executable] call.extend("-m robot.libdoc --format XML:HTML".split()) if additional_path: if os.path.exists(additional_path): call.extend(["-P", additional_path]) additional_pythonpath_entries = list( self._additional_pythonpath_folder_to_folder_info.keys()) for entry in list(additional_pythonpath_entries): if os.path.exists(entry): call.extend(["-P", entry]) call.append(libname) libspec_dir = self._user_libspec_dir if libname in robot_constants.STDLIBS: libspec_dir = self._builtins_libspec_dir libspec_filename = os.path.join(libspec_dir, libname + ".libspec") log.debug( f"Obtaining mutex to generate libpsec: {libspec_filename}." ) with timed_acquire_mutex( _get_libspec_mutex_name( libspec_filename)): # Could fail. log.debug( f"Obtained mutex to generate libpsec: {libspec_filename}." ) call.append(libspec_filename) mtime = -1 try: mtime = os.path.getmtime(libspec_filename) except: pass log.debug( "Generating libspec for: %s.\nCwd:%s\nCommand line:\n%s", libname, cwd, " ".join(call), ) try: try: # Note: stdout is always subprocess.PIPE in this call. subprocess.check_output( call, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, env=env, cwd=cwd, ) except OSError as e: log.exception("Error calling: %s", call) # We may have something as: Ignore OSError: [WinError 6] The handle is invalid, # give the result based on whether the file changed on disk. try: if mtime != os.path.getmtime(libspec_filename): _dump_spec_filename_additional_info( libspec_filename, is_builtin=is_builtin, obtain_mutex=False, ) return True except: pass log.debug("Not retrying after OSError failure.") return False except subprocess.CalledProcessError as e: log.exception( "Error creating libspec: %s. Output:\n%s", libname, e.output) return False _dump_spec_filename_additional_info(libspec_filename, is_builtin=is_builtin, obtain_mutex=False) return True except Exception: log.exception("Error creating libspec: %s", libname) return False finally: if log_time: delta = time.time() - curtime log.debug("Took: %.2fs to generate info for: %s" % (delta, libname))
def test_system_mutex(): from robocorp_ls_core.system_mutex import SystemMutex from robocorp_ls_core.system_mutex import timed_acquire_mutex from robocorp_ls_core.subprocess_wrapper import subprocess import sys import pytest import time import threading import weakref mutex_name = "mutex_name_test_system_mutex" system_mutex = SystemMutex(mutex_name) assert system_mutex.get_mutex_aquired() class Check2Thread(threading.Thread): def __init__(self): threading.Thread.__init__(self) self.worked = False def run(self): mutex2 = SystemMutex(mutex_name) assert not mutex2.get_mutex_aquired() self.worked = True t = Check2Thread() t.start() t.join() assert t.worked assert not system_mutex.disposed system_mutex.release_mutex() assert system_mutex.disposed mutex3 = SystemMutex(mutex_name) assert not mutex3.disposed assert mutex3.get_mutex_aquired() mutex3 = weakref.ref(mutex3) # Garbage-collected # Calling release more times should not be an error system_mutex.release_mutex() mutex4 = SystemMutex(mutex_name) assert mutex4.get_mutex_aquired() with pytest.raises(AssertionError): SystemMutex("mutex/") # Invalid name time_to_release_mutex = 1 def release_mutex(): time.sleep(time_to_release_mutex) mutex4.release_mutex() t = threading.Thread(target=release_mutex) t.start() initial_time = time.time() with timed_acquire_mutex( mutex_name, check_reentrant=False ): # The current mutex will be released in a thread, so, check_reentrant=False. acquired_time = time.time() # Should timeout as the lock is already acquired. with pytest.raises(RuntimeError) as exc: with timed_acquire_mutex(mutex_name, timeout=1): pass assert "not a reentrant mutex" in str(exc) # Must also fail from another process. code = """ from robocorp_ls_core.system_mutex import timed_acquire_mutex mutex_name = "mutex_name_test_system_mutex" with timed_acquire_mutex(mutex_name, timeout=1): pass """ with pytest.raises(subprocess.CalledProcessError): subprocess.check_call([sys.executable, "-c", code], stderr=subprocess.PIPE) assert acquired_time - initial_time > time_to_release_mutex