def __install_solr(dist_directory: str = MC_DIST_DIR, solr_version: str = MC_SOLR_VERSION) -> None: """Install Solr to distribution directory; lock directory before installing and unlock afterwards.""" if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): raise McSolrRunException( "Solr %s is already installed in distribution directory '%s'." % (solr_version, dist_directory)) solr_path = __solr_path(dist_directory=dist_directory, solr_version=solr_version) log.info("Creating Solr directory...") mkdir_p(solr_path) installing_file_path = __solr_installing_file_path( dist_directory=dist_directory, solr_version=solr_version) log.info("Locking Solr directory for installation...") lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT) # Waited for concurrent installation to finish? if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): log.info( "While waiting for Solr directory to unlock, Solr got installed to said directory." ) return solr_dist_url = __solr_dist_url(solr_version=solr_version) log.info("Downloading Solr %s from %s..." % (solr_version, solr_dist_url)) solr_tarball_dest_path = download_file_to_temp_path(solr_dist_url) log.info("Extracting %s to %s..." % (solr_tarball_dest_path, solr_path)) extract_tarball_to_directory(archive_file=solr_tarball_dest_path, dest_directory=solr_path, strip_root=True) # Solr needs its .war extracted first before ZkCLI is usable jetty_home_path = __jetty_home_path(dist_directory=dist_directory, solr_version=solr_version) solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp") if not os.path.exists(os.path.join(solr_war_dest_dir, "index.html")): raise McSolrRunException("Solr's .war is not extracted at path %s" % solr_war_dest_dir) log.info("Creating 'installed' file...") installed_file_path = __solr_installed_file_path( dist_directory=dist_directory, solr_version=solr_version) lock_file(installed_file_path) log.info("Removing lock file...") unlock_file(installing_file_path) if not __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): raise McSolrRunException( "I've done everything but Solr is still not installed.")
def __install_zookeeper(dist_directory: str = MC_DIST_DIR, zookeeper_version: str = MC_ZOOKEEPER_VERSION) -> None: """Install ZooKeeper to distribution directory; lock directory before installing and unlock afterwards.""" if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): raise McZooKeeperRunException( "ZooKeeper %s is already installed in distribution directory '%s'." % (zookeeper_version, dist_directory)) zookeeper_path = __zookeeper_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version) log.info("Creating ZooKeeper directory...") mkdir_p(zookeeper_path) installing_file_path = __zookeeper_installing_file_path( dist_directory=dist_directory, zookeeper_version=zookeeper_version) log.info("Locking ZooKeeper directory for installation...") lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT) # Waited for concurrent installation to finish? if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): log.info( "While waiting for ZooKeeper directory to unlock, ZooKeeper got installed to said directory." ) return zookeeper_dist_url = __zookeeper_dist_url( zookeeper_version=zookeeper_version) log.info("Downloading ZooKeeper %s from %s..." % (zookeeper_version, zookeeper_dist_url)) zookeeper_tarball_dest_path = download_file_to_temp_path( source_url=zookeeper_dist_url) log.info("Extracting %s to %s..." % (zookeeper_tarball_dest_path, zookeeper_path)) extract_tarball_to_directory(archive_file=zookeeper_tarball_dest_path, dest_directory=zookeeper_path, strip_root=True) log.info("Creating 'installed' file...") installed_file_path = __zookeeper_installed_file_path( dist_directory=dist_directory, zookeeper_version=zookeeper_version) lock_file(installed_file_path) log.info("Removing lock file...") unlock_file(installing_file_path) if not __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): raise McZooKeeperRunException( "I've done everything but ZooKeeper is still not installed.")
def __install_solr(dist_directory: str = MC_DIST_DIR, solr_version: str = MC_SOLR_VERSION) -> None: """Install Solr to distribution directory; lock directory before installing and unlock afterwards.""" if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): raise Exception("Solr %s is already installed in distribution directory '%s'." % (solr_version, dist_directory)) solr_path = __solr_path(dist_directory=dist_directory, solr_version=solr_version) l.info("Creating Solr directory...") mkdir_p(solr_path) installing_file_path = __solr_installing_file_path(dist_directory=dist_directory, solr_version=solr_version) l.info("Locking Solr directory for installation...") lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT) # Waited for concurrent installation to finish? if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): l.info("While waiting for Solr directory to unlock, Solr got installed to said directory.") return solr_dist_url = __solr_dist_url(solr_version=solr_version) l.info("Downloading Solr %s from %s..." % (solr_version, solr_dist_url)) solr_tarball_dest_path = download_file_to_temp_path(solr_dist_url) l.info("Extracting %s to %s..." % (solr_tarball_dest_path, solr_path)) extract_tarball_to_directory(archive_file=solr_tarball_dest_path, dest_directory=solr_path, strip_root=True) # Solr needs its .war extracted first before ZkCLI is usable jetty_home_path = __jetty_home_path(dist_directory=dist_directory, solr_version=solr_version) solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp") # Solr 5.5.2+ already has the .war extracted if not os.path.exists(os.path.join(solr_war_dest_dir, "index.html")): solr_war_path = os.path.join(jetty_home_path, "webapps", "solr.war") if not os.path.isfile(solr_war_path): raise Exception("Solr's .war file does not exist at path %s" % solr_war_path) solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp") l.info("Extracting solr.war at '%s' to '%s'..." % (solr_war_path, solr_war_dest_dir)) mkdir_p(solr_war_dest_dir) extract_zip_to_directory(archive_file=solr_war_path, dest_directory=solr_war_dest_dir) l.info("Creating 'installed' file...") installed_file_path = __solr_installed_file_path(dist_directory=dist_directory, solr_version=solr_version) lock_file(installed_file_path) l.info("Removing lock file...") unlock_file(installing_file_path) if not __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version): raise Exception("I've done everything but Solr is still not installed.")
def __remove_run_alone_lock_file(signum: int = None, frame: int = None) -> None: global __run_alone_function_lock_file if __run_alone_function_lock_file is not None: l.info("Caught SIGTERM, unlocking '%s'..." % __run_alone_function_lock_file) try: unlock_file(__run_alone_function_lock_file) except McUnlockFileException as exception: # Not critical, the lock file might have been removed by some other process l.warning("Unlocking file failed: %s" % str(exception)) else: l.debug("Nothing to unlock.") sys.exit(signum)
def __install_zookeeper(dist_directory: str = MC_DIST_DIR, zookeeper_version: str = MC_ZOOKEEPER_VERSION) -> None: """Install ZooKeeper to distribution directory; lock directory before installing and unlock afterwards.""" if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): raise McZooKeeperRunException("ZooKeeper %s is already installed in distribution directory '%s'." % ( zookeeper_version, dist_directory )) zookeeper_path = __zookeeper_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version) log.info("Creating ZooKeeper directory...") mkdir_p(zookeeper_path) installing_file_path = __zookeeper_installing_file_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version) log.info("Locking ZooKeeper directory for installation...") lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT) # Waited for concurrent installation to finish? if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): log.info("While waiting for ZooKeeper directory to unlock, ZooKeeper got installed to said directory.") return zookeeper_dist_url = __zookeeper_dist_url(zookeeper_version=zookeeper_version) log.info("Downloading ZooKeeper %s from %s..." % (zookeeper_version, zookeeper_dist_url)) zookeeper_tarball_dest_path = download_file_to_temp_path(source_url=zookeeper_dist_url) log.info("Extracting %s to %s..." % (zookeeper_tarball_dest_path, zookeeper_path)) extract_tarball_to_directory(archive_file=zookeeper_tarball_dest_path, dest_directory=zookeeper_path, strip_root=True) log.info("Creating 'installed' file...") installed_file_path = __zookeeper_installed_file_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version) lock_file(installed_file_path) log.info("Removing lock file...") unlock_file(installing_file_path) if not __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version): raise McZooKeeperRunException("I've done everything but ZooKeeper is still not installed.")
def __remove_run_alone_lock_file(signum: int = None, frame: int = None, no_exception: bool = False) -> None: global __run_alone_function_lock_file if __run_alone_function_lock_file is not None: log.info("Caught SIGTERM, unlocking '%s'..." % __run_alone_function_lock_file) try: unlock_file(__run_alone_function_lock_file) except McUnlockFileException as exception: # Not critical, the lock file might have been removed by some other process log.warning("Unlocking file failed: %s" % str(exception)) else: log.debug("Nothing to unlock.") if no_exception: # noinspection PyProtectedMember os._exit(signum) else: sys.exit(signum)
def test_lock_unlock_file(): temp_dir = tempfile.mkdtemp() lock_file_path = os.path.join(temp_dir, 'test.lock') assert os.path.isfile(lock_file_path) is False mc_paths.lock_file(lock_file_path) assert os.path.isfile(lock_file_path) is True mc_paths.unlock_file(lock_file_path) assert os.path.isfile(lock_file_path) is False # Try locking twice, with timeout assert os.path.isfile(lock_file_path) is False mc_paths.lock_file(lock_file_path) assert os.path.isfile(lock_file_path) is True with pytest.raises(mc_paths.McLockFileException): mc_paths.lock_file(lock_file_path, 2) assert os.path.isfile(lock_file_path) is True mc_paths.unlock_file(lock_file_path) assert os.path.isfile(lock_file_path) is False # Try unlocking nonexistent file assert os.path.isfile(lock_file_path) is False with pytest.raises(mc_paths.McUnlockFileException): mc_paths.unlock_file(lock_file_path) assert os.path.isfile(lock_file_path) is False
def run_alone(isolated_function: Callable, *args, **kwargs) -> Any: """Run function while making sure that only a single instance of it is running.""" global __run_alone_function_lock_file try: function_unique_id = __function_unique_id(isolated_function) except Exception as ex: raise McUnableToDetermineCaller( "Unable to determine caller script: %s" % str(ex)) timeout = 5 log.debug("Function unique ID: %s" % function_unique_id) function_unique_id_hash = hashlib.sha256(bytes(function_unique_id, 'utf-8')).hexdigest() log.debug("Function unique ID hash: %s" % function_unique_id_hash) # Catch SIGINTs and SIGTERMs while running the function to be able to remove lock file afterwards original_sigint_handler = signal.getsignal(signal.SIGINT) original_sigterm_handler = signal.getsignal(signal.SIGTERM) signal.signal(signal.SIGINT, __remove_run_alone_lock_file) signal.signal(signal.SIGTERM, __remove_run_alone_lock_file) atexit.register(__remove_run_alone_lock_file, signum=0, no_exception=True) try: if sys.platform.lower() == 'darwin': # OS X -- /var/run is not world-writable by default lock_file_path = '/var/tmp' else: # Linux -- keep lock files in '/var/run/lock' as they will be removed after reboot lock_file_path = '/var/run/lock' if not os.path.exists(lock_file_path): raise McRunAloneException( 'Lock file location "%s" does not exist.' % lock_file_path) if not os.access(lock_file_path, os.W_OK): raise McRunAloneException( 'Lock file location "%s" exists but is not writable.' % lock_file_path) function_lock_file = os.path.join(lock_file_path, function_unique_id_hash) try: lock_file(path=function_lock_file, timeout=timeout) __run_alone_function_lock_file = function_lock_file except McLockFileException as ex: raise McScriptInstanceIsAlreadyRunning( "Instance of %s is already running: %s" % (str(isolated_function), str(ex))) # noinspection PyCallingNonCallable return_value = isolated_function(*args, **kwargs) try: unlock_file(__run_alone_function_lock_file) except McUnlockFileException as exc: # Not critical, the lock file might have been removed by some other process log.warning("Unlocking file failed: %s" % str(exc)) except Exception as ex: raise ex # We want to reset signal handlers no matter what because if they remain set, weird things happen (e.g. pytest # doesn't exit(1) on failures) finally: atexit.unregister(__remove_run_alone_lock_file) signal.signal(signal.SIGINT, original_sigint_handler) signal.signal(signal.SIGTERM, original_sigterm_handler) __run_alone_function_lock_file = None return return_value
def run_alone(isolated_function: Callable, *args, **kwargs) -> Any: """Run function while making sure that only a single instance of it is running.""" global __run_alone_function_lock_file try: function_unique_id = __function_unique_id(isolated_function) except Exception as ex: raise McUnableToDetermineCaller("Unable to determine caller script: %s" % str(ex)) timeout = 5 log.debug("Function unique ID: %s" % function_unique_id) function_unique_id_hash = hashlib.sha256(bytes(function_unique_id, 'utf-8')).hexdigest() log.debug("Function unique ID hash: %s" % function_unique_id_hash) # Catch SIGINTs and SIGTERMs while running the function to be able to remove lock file afterwards original_sigint_handler = signal.getsignal(signal.SIGINT) original_sigterm_handler = signal.getsignal(signal.SIGTERM) signal.signal(signal.SIGINT, __remove_run_alone_lock_file) signal.signal(signal.SIGTERM, __remove_run_alone_lock_file) atexit.register(__remove_run_alone_lock_file, signum=0, no_exception=True) try: if sys.platform.lower() == 'darwin': # OS X -- /var/run is not world-writable by default lock_file_path = '/var/tmp' else: # Linux -- keep lock files in '/var/run/lock' as they will be removed after reboot lock_file_path = '/var/run/lock' if not os.path.exists(lock_file_path): raise McRunAloneException( 'Lock file location "%s" does not exist.' % lock_file_path ) if not os.access(lock_file_path, os.W_OK): raise McRunAloneException( 'Lock file location "%s" exists but is not writable.' % lock_file_path ) function_lock_file = os.path.join(lock_file_path, function_unique_id_hash) try: lock_file(path=function_lock_file, timeout=timeout) __run_alone_function_lock_file = function_lock_file except McLockFileException as ex: raise McScriptInstanceIsAlreadyRunning( "Instance of %s is already running: %s" % (str(isolated_function), str(ex)) ) # noinspection PyCallingNonCallable return_value = isolated_function(*args, **kwargs) try: unlock_file(__run_alone_function_lock_file) except McUnlockFileException as exc: # Not critical, the lock file might have been removed by some other process log.warning("Unlocking file failed: %s" % str(exc)) except Exception as ex: raise ex # We want to reset signal handlers no matter what because if they remain set, weird things happen (e.g. pytest # doesn't exit(1) on failures) finally: atexit.unregister(__remove_run_alone_lock_file) signal.signal(signal.SIGINT, original_sigint_handler) signal.signal(signal.SIGTERM, original_sigterm_handler) __run_alone_function_lock_file = None return return_value
def run_alone(function: Callable, *args, **kwargs) -> Any: """Run function while making sure that only a single instance of it is running.""" global __run_alone_function_lock_file def __function_unique_id(func: Callable) -> str: """Return unique signature of the function, consisting of its absolute path and a name. Retry locking for 5 seconds before giving up.""" module = inspect.getmodule(func) signature = inspect.signature(func) module_path = module.__file__ if not os.path.isfile(module_path): raise Exception("Module '%s' for function '%s' does not exist." % (module_path, str(func))) function_name = func.__qualname__ if function_name is None or len(function_name) == 0: raise Exception("Unable to determine function name: %s" % str(func)) parameters_type = str(signature.parameters) return_value_type = str(signature.return_annotation) unique_id = '%(module_path)s:%(function_name)s:%(parameters)s:%(return_value)s' % { 'module_path': module_path, 'function_name': function_name, 'parameters': parameters_type, 'return_value': return_value_type, } return unique_id # noinspection PyUnusedLocal def __remove_run_alone_lock_file(signum: int = None, frame: int = None) -> None: global __run_alone_function_lock_file if __run_alone_function_lock_file is not None: l.info("Caught SIGTERM, unlocking '%s'..." % __run_alone_function_lock_file) try: unlock_file(__run_alone_function_lock_file) except McUnlockFileException as exception: # Not critical, the lock file might have been removed by some other process l.warning("Unlocking file failed: %s" % str(exception)) else: l.debug("Nothing to unlock.") sys.exit(signum) try: function_unique_id = __function_unique_id(function) except Exception as ex: raise McUnableToDetermineCaller( "Unable to determine caller script: %s" % str(ex)) timeout = 5 l.debug("Function unique ID: %s" % function_unique_id) function_unique_id_hash = hashlib.sha256(bytes(function_unique_id, 'utf-8')).hexdigest() l.debug("Function unique ID hash: %s" % function_unique_id_hash) # Catch SIGINTs and SIGTERMs while running the function to be able to remove lock file afterwards original_sigint_handler = signal.getsignal(signal.SIGINT) original_sigterm_handler = signal.getsignal(signal.SIGTERM) signal.signal(signal.SIGINT, __remove_run_alone_lock_file) signal.signal(signal.SIGTERM, __remove_run_alone_lock_file) atexit.register(__remove_run_alone_lock_file) function_lock_file = os.path.join("/var", "tmp", function_unique_id_hash) try: lock_file(path=function_lock_file, timeout=timeout) __run_alone_function_lock_file = function_lock_file except McLockFileException as ex: raise McScriptInstanceIsAlreadyRunning( "Instance of %s is already running: %s" % (str(function), str(ex))) # noinspection PyCallingNonCallable return_value = function(*args, **kwargs) try: unlock_file(__run_alone_function_lock_file) except McUnlockFileException as exc: # Not critical, the lock file might have been removed by some other process l.warning("Unlocking file failed: %s" % str(exc)) # Reset signal handlers atexit.unregister(__remove_run_alone_lock_file) signal.signal(signal.SIGINT, original_sigint_handler) signal.signal(signal.SIGTERM, original_sigterm_handler) __run_alone_function_lock_file = None return return_value
def run_alone(isolated_function: Callable, *args, **kwargs) -> Any: """Run function while making sure that only a single instance of it is running.""" global __run_alone_function_lock_file def __function_unique_id(func: Callable) -> str: """Return unique signature of the function, consisting of its absolute path and a name. Retry locking for 5 seconds before giving up.""" function_module = inspect.getmodule(func) function_signature = inspect.signature(func) module_path = function_module.__file__ if not os.path.isfile(module_path): raise Exception("Module '%s' for function '%s' does not exist." % (module_path, str(func))) function_name = func.__qualname__ if function_name is None or len(function_name) == 0: raise Exception("Unable to determine function name: %s" % str(func)) parameters_type = str(function_signature.parameters) return_value_type = str(function_signature.return_annotation) unique_id = '%(module_path)s:%(function_name)s:%(parameters)s:%(return_value)s' % { 'module_path': module_path, 'function_name': function_name, 'parameters': parameters_type, 'return_value': return_value_type, } return unique_id # noinspection PyUnusedLocal def __remove_run_alone_lock_file(signum: int = None, frame: int = None, no_exception: bool = False) -> None: global __run_alone_function_lock_file if __run_alone_function_lock_file is not None: log.info("Caught SIGTERM, unlocking '%s'..." % __run_alone_function_lock_file) try: unlock_file(__run_alone_function_lock_file) except McUnlockFileException as exception: # Not critical, the lock file might have been removed by some other process log.warning("Unlocking file failed: %s" % str(exception)) else: log.debug("Nothing to unlock.") if no_exception: # noinspection PyProtectedMember os._exit(signum) else: sys.exit(signum) try: function_unique_id = __function_unique_id(isolated_function) except Exception as ex: raise McUnableToDetermineCaller( "Unable to determine caller script: %s" % str(ex)) timeout = 5 log.debug("Function unique ID: %s" % function_unique_id) function_unique_id_hash = hashlib.sha256(bytes(function_unique_id, 'utf-8')).hexdigest() log.debug("Function unique ID hash: %s" % function_unique_id_hash) # Catch SIGINTs and SIGTERMs while running the function to be able to remove lock file afterwards original_sigint_handler = signal.getsignal(signal.SIGINT) original_sigterm_handler = signal.getsignal(signal.SIGTERM) signal.signal(signal.SIGINT, __remove_run_alone_lock_file) signal.signal(signal.SIGTERM, __remove_run_alone_lock_file) atexit.register(__remove_run_alone_lock_file, signum=0, no_exception=True) try: if sys.platform.lower() == 'darwin': # OS X -- /var/run is not world-writable by default lock_file_path = '/var/tmp' else: # Linux -- keep lock files in '/var/run/lock' as they will be removed after reboot lock_file_path = '/var/run/lock' if not os.path.exists(lock_file_path): raise McRunAloneException( 'Lock file location "%s" does not exist.' % lock_file_path) if not os.access(lock_file_path, os.W_OK): raise McRunAloneException( 'Lock file location "%s" exists but is not writable.' % lock_file_path) function_lock_file = os.path.join(lock_file_path, function_unique_id_hash) try: lock_file(path=function_lock_file, timeout=timeout) __run_alone_function_lock_file = function_lock_file except McLockFileException as ex: raise McScriptInstanceIsAlreadyRunning( "Instance of %s is already running: %s" % (str(isolated_function), str(ex))) # noinspection PyCallingNonCallable return_value = isolated_function(*args, **kwargs) try: unlock_file(__run_alone_function_lock_file) except McUnlockFileException as exc: # Not critical, the lock file might have been removed by some other process log.warning("Unlocking file failed: %s" % str(exc)) except Exception as ex: raise ex # We want to reset signal handlers no matter what because if they remain set, weird things happen (e.g. pytest # doesn't exit(1) on failures) finally: atexit.unregister(__remove_run_alone_lock_file) signal.signal(signal.SIGINT, original_sigint_handler) signal.signal(signal.SIGTERM, original_sigterm_handler) __run_alone_function_lock_file = None return return_value