def __install_solr(dist_directory: str = MC_DIST_DIR,
solr_version: str = MC_SOLR_VERSION) -> None:
"""Install Solr to distribution directory; lock directory before installing and unlock afterwards."""
if __solr_is_installed(dist_directory=dist_directory,
solr_version=solr_version):
raise McSolrRunException(
"Solr %s is already installed in distribution directory '%s'." %
(solr_version, dist_directory))
solr_path = __solr_path(dist_directory=dist_directory,
solr_version=solr_version)
log.info("Creating Solr directory...")
mkdir_p(solr_path)
installing_file_path = __solr_installing_file_path(
dist_directory=dist_directory, solr_version=solr_version)
log.info("Locking Solr directory for installation...")
lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT)
# Waited for concurrent installation to finish?
if __solr_is_installed(dist_directory=dist_directory,
solr_version=solr_version):
log.info(
"While waiting for Solr directory to unlock, Solr got installed to said directory."
)
return
solr_dist_url = __solr_dist_url(solr_version=solr_version)
log.info("Downloading Solr %s from %s..." % (solr_version, solr_dist_url))
solr_tarball_dest_path = download_file_to_temp_path(solr_dist_url)
log.info("Extracting %s to %s..." % (solr_tarball_dest_path, solr_path))
extract_tarball_to_directory(archive_file=solr_tarball_dest_path,
dest_directory=solr_path,
strip_root=True)
# Solr needs its .war extracted first before ZkCLI is usable
jetty_home_path = __jetty_home_path(dist_directory=dist_directory,
solr_version=solr_version)
solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp")
if not os.path.exists(os.path.join(solr_war_dest_dir, "index.html")):
raise McSolrRunException("Solr's .war is not extracted at path %s" %
solr_war_dest_dir)
log.info("Creating 'installed' file...")
installed_file_path = __solr_installed_file_path(
dist_directory=dist_directory, solr_version=solr_version)
lock_file(installed_file_path)
log.info("Removing lock file...")
unlock_file(installing_file_path)
if not __solr_is_installed(dist_directory=dist_directory,
solr_version=solr_version):
raise McSolrRunException(
"I've done everything but Solr is still not installed.")
def __install_zookeeper(dist_directory: str = MC_DIST_DIR,
zookeeper_version: str = MC_ZOOKEEPER_VERSION) -> None:
"""Install ZooKeeper to distribution directory; lock directory before installing and unlock afterwards."""
if __zookeeper_is_installed(dist_directory=dist_directory,
zookeeper_version=zookeeper_version):
raise McZooKeeperRunException(
"ZooKeeper %s is already installed in distribution directory '%s'."
% (zookeeper_version, dist_directory))
zookeeper_path = __zookeeper_path(dist_directory=dist_directory,
zookeeper_version=zookeeper_version)
log.info("Creating ZooKeeper directory...")
mkdir_p(zookeeper_path)
installing_file_path = __zookeeper_installing_file_path(
dist_directory=dist_directory, zookeeper_version=zookeeper_version)
log.info("Locking ZooKeeper directory for installation...")
lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT)
# Waited for concurrent installation to finish?
if __zookeeper_is_installed(dist_directory=dist_directory,
zookeeper_version=zookeeper_version):
log.info(
"While waiting for ZooKeeper directory to unlock, ZooKeeper got installed to said directory."
)
return
zookeeper_dist_url = __zookeeper_dist_url(
zookeeper_version=zookeeper_version)
log.info("Downloading ZooKeeper %s from %s..." %
(zookeeper_version, zookeeper_dist_url))
zookeeper_tarball_dest_path = download_file_to_temp_path(
source_url=zookeeper_dist_url)
log.info("Extracting %s to %s..." %
(zookeeper_tarball_dest_path, zookeeper_path))
extract_tarball_to_directory(archive_file=zookeeper_tarball_dest_path,
dest_directory=zookeeper_path,
strip_root=True)
log.info("Creating 'installed' file...")
installed_file_path = __zookeeper_installed_file_path(
dist_directory=dist_directory, zookeeper_version=zookeeper_version)
lock_file(installed_file_path)
log.info("Removing lock file...")
unlock_file(installing_file_path)
if not __zookeeper_is_installed(dist_directory=dist_directory,
zookeeper_version=zookeeper_version):
raise McZooKeeperRunException(
"I've done everything but ZooKeeper is still not installed.")
def __install_solr(dist_directory: str = MC_DIST_DIR, solr_version: str = MC_SOLR_VERSION) -> None:
"""Install Solr to distribution directory; lock directory before installing and unlock afterwards."""
if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version):
raise Exception("Solr %s is already installed in distribution directory '%s'." % (solr_version, dist_directory))
solr_path = __solr_path(dist_directory=dist_directory, solr_version=solr_version)
l.info("Creating Solr directory...")
mkdir_p(solr_path)
installing_file_path = __solr_installing_file_path(dist_directory=dist_directory, solr_version=solr_version)
l.info("Locking Solr directory for installation...")
lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT)
# Waited for concurrent installation to finish?
if __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version):
l.info("While waiting for Solr directory to unlock, Solr got installed to said directory.")
return
solr_dist_url = __solr_dist_url(solr_version=solr_version)
l.info("Downloading Solr %s from %s..." % (solr_version, solr_dist_url))
solr_tarball_dest_path = download_file_to_temp_path(solr_dist_url)
l.info("Extracting %s to %s..." % (solr_tarball_dest_path, solr_path))
extract_tarball_to_directory(archive_file=solr_tarball_dest_path, dest_directory=solr_path, strip_root=True)
# Solr needs its .war extracted first before ZkCLI is usable
jetty_home_path = __jetty_home_path(dist_directory=dist_directory, solr_version=solr_version)
solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp")
# Solr 5.5.2+ already has the .war extracted
if not os.path.exists(os.path.join(solr_war_dest_dir, "index.html")):
solr_war_path = os.path.join(jetty_home_path, "webapps", "solr.war")
if not os.path.isfile(solr_war_path):
raise Exception("Solr's .war file does not exist at path %s" % solr_war_path)
solr_war_dest_dir = os.path.join(jetty_home_path, "solr-webapp", "webapp")
l.info("Extracting solr.war at '%s' to '%s'..." % (solr_war_path, solr_war_dest_dir))
mkdir_p(solr_war_dest_dir)
extract_zip_to_directory(archive_file=solr_war_path, dest_directory=solr_war_dest_dir)
l.info("Creating 'installed' file...")
installed_file_path = __solr_installed_file_path(dist_directory=dist_directory, solr_version=solr_version)
lock_file(installed_file_path)
l.info("Removing lock file...")
unlock_file(installing_file_path)
if not __solr_is_installed(dist_directory=dist_directory, solr_version=solr_version):
raise Exception("I've done everything but Solr is still not installed.")
def __remove_run_alone_lock_file(signum: int = None,
frame: int = None) -> None:
global __run_alone_function_lock_file
if __run_alone_function_lock_file is not None:
l.info("Caught SIGTERM, unlocking '%s'..." %
__run_alone_function_lock_file)
try:
unlock_file(__run_alone_function_lock_file)
except McUnlockFileException as exception:
# Not critical, the lock file might have been removed by some other process
l.warning("Unlocking file failed: %s" % str(exception))
else:
l.debug("Nothing to unlock.")
sys.exit(signum)
def __install_zookeeper(dist_directory: str = MC_DIST_DIR, zookeeper_version: str = MC_ZOOKEEPER_VERSION) -> None:
"""Install ZooKeeper to distribution directory; lock directory before installing and unlock afterwards."""
if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version):
raise McZooKeeperRunException("ZooKeeper %s is already installed in distribution directory '%s'." % (
zookeeper_version, dist_directory
))
zookeeper_path = __zookeeper_path(dist_directory=dist_directory, zookeeper_version=zookeeper_version)
log.info("Creating ZooKeeper directory...")
mkdir_p(zookeeper_path)
installing_file_path = __zookeeper_installing_file_path(dist_directory=dist_directory,
zookeeper_version=zookeeper_version)
log.info("Locking ZooKeeper directory for installation...")
lock_file(installing_file_path, timeout=MC_INSTALL_TIMEOUT)
# Waited for concurrent installation to finish?
if __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version):
log.info("While waiting for ZooKeeper directory to unlock, ZooKeeper got installed to said directory.")
return
zookeeper_dist_url = __zookeeper_dist_url(zookeeper_version=zookeeper_version)
log.info("Downloading ZooKeeper %s from %s..." % (zookeeper_version, zookeeper_dist_url))
zookeeper_tarball_dest_path = download_file_to_temp_path(source_url=zookeeper_dist_url)
log.info("Extracting %s to %s..." % (zookeeper_tarball_dest_path, zookeeper_path))
extract_tarball_to_directory(archive_file=zookeeper_tarball_dest_path,
dest_directory=zookeeper_path,
strip_root=True)
log.info("Creating 'installed' file...")
installed_file_path = __zookeeper_installed_file_path(dist_directory=dist_directory,
zookeeper_version=zookeeper_version)
lock_file(installed_file_path)
log.info("Removing lock file...")
unlock_file(installing_file_path)
if not __zookeeper_is_installed(dist_directory=dist_directory, zookeeper_version=zookeeper_version):
raise McZooKeeperRunException("I've done everything but ZooKeeper is still not installed.")
def __remove_run_alone_lock_file(signum: int = None,
frame: int = None,
no_exception: bool = False) -> None:
global __run_alone_function_lock_file
if __run_alone_function_lock_file is not None:
log.info("Caught SIGTERM, unlocking '%s'..." % __run_alone_function_lock_file)
try:
unlock_file(__run_alone_function_lock_file)
except McUnlockFileException as exception:
# Not critical, the lock file might have been removed by some other process
log.warning("Unlocking file failed: %s" % str(exception))
else:
log.debug("Nothing to unlock.")
if no_exception:
# noinspection PyProtectedMember
os._exit(signum)
else:
sys.exit(signum)
def __remove_run_alone_lock_file(signum: int = None,
frame: int = None,
no_exception: bool = False) -> None:
global __run_alone_function_lock_file
if __run_alone_function_lock_file is not None:
log.info("Caught SIGTERM, unlocking '%s'..." %
__run_alone_function_lock_file)
try:
unlock_file(__run_alone_function_lock_file)
except McUnlockFileException as exception:
# Not critical, the lock file might have been removed by some other process
log.warning("Unlocking file failed: %s" % str(exception))
else:
log.debug("Nothing to unlock.")
if no_exception:
# noinspection PyProtectedMember
os._exit(signum)
else:
sys.exit(signum)
def test_lock_unlock_file():
temp_dir = tempfile.mkdtemp()
lock_file_path = os.path.join(temp_dir, 'test.lock')
assert os.path.isfile(lock_file_path) is False
mc_paths.lock_file(lock_file_path)
assert os.path.isfile(lock_file_path) is True
mc_paths.unlock_file(lock_file_path)
assert os.path.isfile(lock_file_path) is False
# Try locking twice, with timeout
assert os.path.isfile(lock_file_path) is False
mc_paths.lock_file(lock_file_path)
assert os.path.isfile(lock_file_path) is True
with pytest.raises(mc_paths.McLockFileException):
mc_paths.lock_file(lock_file_path, 2)
assert os.path.isfile(lock_file_path) is True
mc_paths.unlock_file(lock_file_path)
assert os.path.isfile(lock_file_path) is False
# Try unlocking nonexistent file
assert os.path.isfile(lock_file_path) is False
with pytest.raises(mc_paths.McUnlockFileException):
mc_paths.unlock_file(lock_file_path)
assert os.path.isfile(lock_file_path) is False
def run_alone(isolated_function: Callable, *args, **kwargs) -> Any:
"""Run function while making sure that only a single instance of it is running."""
global __run_alone_function_lock_file
try:
function_unique_id = __function_unique_id(isolated_function)
except Exception as ex:
raise McUnableToDetermineCaller(
"Unable to determine caller script: %s" % str(ex))
timeout = 5
log.debug("Function unique ID: %s" % function_unique_id)
function_unique_id_hash = hashlib.sha256(bytes(function_unique_id,
'utf-8')).hexdigest()
log.debug("Function unique ID hash: %s" % function_unique_id_hash)
# Catch SIGINTs and SIGTERMs while running the function to be able to remove lock file afterwards
original_sigint_handler = signal.getsignal(signal.SIGINT)
original_sigterm_handler = signal.getsignal(signal.SIGTERM)
signal.signal(signal.SIGINT, __remove_run_alone_lock_file)
signal.signal(signal.SIGTERM, __remove_run_alone_lock_file)
atexit.register(__remove_run_alone_lock_file, signum=0, no_exception=True)
try:
if sys.platform.lower() == 'darwin':
# OS X -- /var/run is not world-writable by default
lock_file_path = '/var/tmp'
else:
# Linux -- keep lock files in '/var/run/lock' as they will be removed after reboot
lock_file_path = '/var/run/lock'
if not os.path.exists(lock_file_path):
raise McRunAloneException(
'Lock file location "%s" does not exist.' % lock_file_path)
if not os.access(lock_file_path, os.W_OK):
raise McRunAloneException(
'Lock file location "%s" exists but is not writable.' %
lock_file_path)
function_lock_file = os.path.join(lock_file_path,
function_unique_id_hash)
try:
lock_file(path=function_lock_file, timeout=timeout)
__run_alone_function_lock_file = function_lock_file
except McLockFileException as ex:
raise McScriptInstanceIsAlreadyRunning(
"Instance of %s is already running: %s" %
(str(isolated_function), str(ex)))
# noinspection PyCallingNonCallable
return_value = isolated_function(*args, **kwargs)
try:
unlock_file(__run_alone_function_lock_file)
except McUnlockFileException as exc:
# Not critical, the lock file might have been removed by some other process
log.warning("Unlocking file failed: %s" % str(exc))
except Exception as ex:
raise ex
# We want to reset signal handlers no matter what because if they remain set, weird things happen (e.g. pytest
# doesn't exit(1) on failures)
finally:
atexit.unregister(__remove_run_alone_lock_file)
signal.signal(signal.SIGINT, original_sigint_handler)
signal.signal(signal.SIGTERM, original_sigterm_handler)
__run_alone_function_lock_file = None
return return_value
def run_alone(isolated_function: Callable, *args, **kwargs) -> Any:
"""Run function while making sure that only a single instance of it is running."""
global __run_alone_function_lock_file
try:
function_unique_id = __function_unique_id(isolated_function)
except Exception as ex:
raise McUnableToDetermineCaller("Unable to determine caller script: %s" % str(ex))
timeout = 5
log.debug("Function unique ID: %s" % function_unique_id)
function_unique_id_hash = hashlib.sha256(bytes(function_unique_id, 'utf-8')).hexdigest()
log.debug("Function unique ID hash: %s" % function_unique_id_hash)
# Catch SIGINTs and SIGTERMs while running the function to be able to remove lock file afterwards
original_sigint_handler = signal.getsignal(signal.SIGINT)
original_sigterm_handler = signal.getsignal(signal.SIGTERM)
signal.signal(signal.SIGINT, __remove_run_alone_lock_file)
signal.signal(signal.SIGTERM, __remove_run_alone_lock_file)
atexit.register(__remove_run_alone_lock_file, signum=0, no_exception=True)
try:
if sys.platform.lower() == 'darwin':
# OS X -- /var/run is not world-writable by default
lock_file_path = '/var/tmp'
else:
# Linux -- keep lock files in '/var/run/lock' as they will be removed after reboot
lock_file_path = '/var/run/lock'
if not os.path.exists(lock_file_path):
raise McRunAloneException(
'Lock file location "%s" does not exist.' % lock_file_path
)
if not os.access(lock_file_path, os.W_OK):
raise McRunAloneException(
'Lock file location "%s" exists but is not writable.' % lock_file_path
)
function_lock_file = os.path.join(lock_file_path, function_unique_id_hash)
try:
lock_file(path=function_lock_file, timeout=timeout)
__run_alone_function_lock_file = function_lock_file
except McLockFileException as ex:
raise McScriptInstanceIsAlreadyRunning(
"Instance of %s is already running: %s" % (str(isolated_function), str(ex))
)
# noinspection PyCallingNonCallable
return_value = isolated_function(*args, **kwargs)
try:
unlock_file(__run_alone_function_lock_file)
except McUnlockFileException as exc:
# Not critical, the lock file might have been removed by some other process
log.warning("Unlocking file failed: %s" % str(exc))
except Exception as ex:
raise ex
# We want to reset signal handlers no matter what because if they remain set, weird things happen (e.g. pytest
# doesn't exit(1) on failures)
finally:
atexit.unregister(__remove_run_alone_lock_file)
signal.signal(signal.SIGINT, original_sigint_handler)
signal.signal(signal.SIGTERM, original_sigterm_handler)
__run_alone_function_lock_file = None
return return_value
def run_alone(function: Callable, *args, **kwargs) -> Any:
"""Run function while making sure that only a single instance of it is running."""
global __run_alone_function_lock_file
def __function_unique_id(func: Callable) -> str:
"""Return unique signature of the function, consisting of its absolute path and a name.
Retry locking for 5 seconds before giving up."""
module = inspect.getmodule(func)
signature = inspect.signature(func)
module_path = module.__file__
if not os.path.isfile(module_path):
raise Exception("Module '%s' for function '%s' does not exist." %
(module_path, str(func)))
function_name = func.__qualname__
if function_name is None or len(function_name) == 0:
raise Exception("Unable to determine function name: %s" %
str(func))
parameters_type = str(signature.parameters)
return_value_type = str(signature.return_annotation)
unique_id = '%(module_path)s:%(function_name)s:%(parameters)s:%(return_value)s' % {
'module_path': module_path,
'function_name': function_name,
'parameters': parameters_type,
'return_value': return_value_type,
}
return unique_id
# noinspection PyUnusedLocal
def __remove_run_alone_lock_file(signum: int = None,
frame: int = None) -> None:
global __run_alone_function_lock_file
if __run_alone_function_lock_file is not None:
l.info("Caught SIGTERM, unlocking '%s'..." %
__run_alone_function_lock_file)
try:
unlock_file(__run_alone_function_lock_file)
except McUnlockFileException as exception:
# Not critical, the lock file might have been removed by some other process
l.warning("Unlocking file failed: %s" % str(exception))
else:
l.debug("Nothing to unlock.")
sys.exit(signum)
try:
function_unique_id = __function_unique_id(function)
except Exception as ex:
raise McUnableToDetermineCaller(
"Unable to determine caller script: %s" % str(ex))
timeout = 5
l.debug("Function unique ID: %s" % function_unique_id)
function_unique_id_hash = hashlib.sha256(bytes(function_unique_id,
'utf-8')).hexdigest()
l.debug("Function unique ID hash: %s" % function_unique_id_hash)
# Catch SIGINTs and SIGTERMs while running the function to be able to remove lock file afterwards
original_sigint_handler = signal.getsignal(signal.SIGINT)
original_sigterm_handler = signal.getsignal(signal.SIGTERM)
signal.signal(signal.SIGINT, __remove_run_alone_lock_file)
signal.signal(signal.SIGTERM, __remove_run_alone_lock_file)
atexit.register(__remove_run_alone_lock_file)
function_lock_file = os.path.join("/var", "tmp", function_unique_id_hash)
try:
lock_file(path=function_lock_file, timeout=timeout)
__run_alone_function_lock_file = function_lock_file
except McLockFileException as ex:
raise McScriptInstanceIsAlreadyRunning(
"Instance of %s is already running: %s" % (str(function), str(ex)))
# noinspection PyCallingNonCallable
return_value = function(*args, **kwargs)
try:
unlock_file(__run_alone_function_lock_file)
except McUnlockFileException as exc:
# Not critical, the lock file might have been removed by some other process
l.warning("Unlocking file failed: %s" % str(exc))
# Reset signal handlers
atexit.unregister(__remove_run_alone_lock_file)
signal.signal(signal.SIGINT, original_sigint_handler)
signal.signal(signal.SIGTERM, original_sigterm_handler)
__run_alone_function_lock_file = None
return return_value
def run_alone(isolated_function: Callable, *args, **kwargs) -> Any:
"""Run function while making sure that only a single instance of it is running."""
global __run_alone_function_lock_file
def __function_unique_id(func: Callable) -> str:
"""Return unique signature of the function, consisting of its absolute path and a name.
Retry locking for 5 seconds before giving up."""
function_module = inspect.getmodule(func)
function_signature = inspect.signature(func)
module_path = function_module.__file__
if not os.path.isfile(module_path):
raise Exception("Module '%s' for function '%s' does not exist." %
(module_path, str(func)))
function_name = func.__qualname__
if function_name is None or len(function_name) == 0:
raise Exception("Unable to determine function name: %s" %
str(func))
parameters_type = str(function_signature.parameters)
return_value_type = str(function_signature.return_annotation)
unique_id = '%(module_path)s:%(function_name)s:%(parameters)s:%(return_value)s' % {
'module_path': module_path,
'function_name': function_name,
'parameters': parameters_type,
'return_value': return_value_type,
}
return unique_id
# noinspection PyUnusedLocal
def __remove_run_alone_lock_file(signum: int = None,
frame: int = None,
no_exception: bool = False) -> None:
global __run_alone_function_lock_file
if __run_alone_function_lock_file is not None:
log.info("Caught SIGTERM, unlocking '%s'..." %
__run_alone_function_lock_file)
try:
unlock_file(__run_alone_function_lock_file)
except McUnlockFileException as exception:
# Not critical, the lock file might have been removed by some other process
log.warning("Unlocking file failed: %s" % str(exception))
else:
log.debug("Nothing to unlock.")
if no_exception:
# noinspection PyProtectedMember
os._exit(signum)
else:
sys.exit(signum)
try:
function_unique_id = __function_unique_id(isolated_function)
except Exception as ex:
raise McUnableToDetermineCaller(
"Unable to determine caller script: %s" % str(ex))
timeout = 5
log.debug("Function unique ID: %s" % function_unique_id)
function_unique_id_hash = hashlib.sha256(bytes(function_unique_id,
'utf-8')).hexdigest()
log.debug("Function unique ID hash: %s" % function_unique_id_hash)
# Catch SIGINTs and SIGTERMs while running the function to be able to remove lock file afterwards
original_sigint_handler = signal.getsignal(signal.SIGINT)
original_sigterm_handler = signal.getsignal(signal.SIGTERM)
signal.signal(signal.SIGINT, __remove_run_alone_lock_file)
signal.signal(signal.SIGTERM, __remove_run_alone_lock_file)
atexit.register(__remove_run_alone_lock_file, signum=0, no_exception=True)
try:
if sys.platform.lower() == 'darwin':
# OS X -- /var/run is not world-writable by default
lock_file_path = '/var/tmp'
else:
# Linux -- keep lock files in '/var/run/lock' as they will be removed after reboot
lock_file_path = '/var/run/lock'
if not os.path.exists(lock_file_path):
raise McRunAloneException(
'Lock file location "%s" does not exist.' % lock_file_path)
if not os.access(lock_file_path, os.W_OK):
raise McRunAloneException(
'Lock file location "%s" exists but is not writable.' %
lock_file_path)
function_lock_file = os.path.join(lock_file_path,
function_unique_id_hash)
try:
lock_file(path=function_lock_file, timeout=timeout)
__run_alone_function_lock_file = function_lock_file
except McLockFileException as ex:
raise McScriptInstanceIsAlreadyRunning(
"Instance of %s is already running: %s" %
(str(isolated_function), str(ex)))
# noinspection PyCallingNonCallable
return_value = isolated_function(*args, **kwargs)
try:
unlock_file(__run_alone_function_lock_file)
except McUnlockFileException as exc:
# Not critical, the lock file might have been removed by some other process
log.warning("Unlocking file failed: %s" % str(exc))
except Exception as ex:
raise ex
# We want to reset signal handlers no matter what because if they remain set, weird things happen (e.g. pytest
# doesn't exit(1) on failures)
finally:
atexit.unregister(__remove_run_alone_lock_file)
signal.signal(signal.SIGINT, original_sigint_handler)
signal.signal(signal.SIGTERM, original_sigterm_handler)
__run_alone_function_lock_file = None
return return_value