def get_cache_manager(cls):
if not cls._cache_manager:
cls._cache_manager = RedisCacheManager(
state_class=cls.GetManyScrollState,
redis=redman.connection("apiserver"),
expiration_interval=config.get(
"services._mongo.scroll_state_expiration_seconds", 600),
)
return cls._cache_manager
def __init__(self, events_es=None, redis=None):
self.es = events_es or es_factory.connect("events")
self._metrics = EventMetrics(self.es)
self._skip_iteration_for_metric = set(
config.get("services.events.ignore_iteration.metrics", [])
)
self.redis = redis or redman.connection("apiserver")
self.debug_images_iterator = DebugImagesIterator(es=self.es, redis=self.redis)
self.debug_sample_history = DebugSampleHistory(es=self.es, redis=self.redis)
self.events_iterator = EventsIterator(es=self.es)
def __init__(self, redis=None):
self.redis: StrictRedis = redis or redman.connection("apiserver")
import os
import time
from contextlib import contextmanager
from time import sleep
from apiserver.redis_manager import redman
_redis = redman.connection("apiserver")
@contextmanager
def distributed_lock(name: str, timeout: int, max_wait: int = 0):
"""
Context manager that acquires a distributed lock on enter
and releases it on exit. The has a ttl equal to timeout seconds
If the lock can not be acquired for wait seconds (defaults to timeout * 2)
then the exception is thrown
"""
lock_name = f"dist_lock_{name}"
start = time.time()
max_wait = max_wait or timeout * 2
pid = os.getpid()
while _redis.set(lock_name, value=pid, ex=timeout, nx=True) is None:
sleep(1)
if time.time() - start > max_wait:
holder = _redis.get(lock_name)
raise Exception(
f"Could not acquire {name} lock for {max_wait} seconds. The lock is hold by {holder}"
)
try:
yield
def __init__(self, es=None, redis=None):
self.es_client = es or es_factory.connect("workers")
self.redis = redis or redman.connection("workers")
self._stats = WorkerStats(self.es_client)
def __init__(self, events_es=None, redis=None):
self.events_es = events_es or es_factory.connect("events")
self.redis: StrictRedis = redis or redman.connection("apiserver")
def __init__(self, redis=None):
self.redis = redis or redman.connection("apiserver")
self._task_tags = _TagsCache(Task, self.redis)
self._model_tags = _TagsCache(Model, self.redis)
class CheckUpdatesThread(Thread):
_enabled = bool(config.get("apiserver.check_for_updates.enabled", True))
_lock_name = "check_updates"
_redis = redman.connection("apiserver")
@attr.s(auto_attribs=True)
class _VersionResponse:
version: str
patch_upgrade: bool
description: str = None
def __init__(self):
super(CheckUpdatesThread, self).__init__(target=self._check_updates,
daemon=True)
@property
def update_interval(self):
return timedelta(seconds=max(
float(
config.get(
"apiserver.check_for_updates.check_interval_sec",
60 * 60 * 24,
)),
60 * 5,
))
def start(self) -> None:
if not self._enabled:
log.info("Checking for updates is disabled")
return
super(CheckUpdatesThread, self).start()
@property
def component_name(self) -> str:
return config.get("apiserver.check_for_updates.component_name",
"clearml-server")
def _check_new_version_available(self) -> Optional[_VersionResponse]:
url = config.get(
"apiserver.check_for_updates.url",
"https://updates.clear.ml/updates",
)
uid = Settings.get_by_key("server.uuid")
response = requests.get(
url,
json={
"versions": {
self.component_name: str(get_version())
},
"uid": uid
},
timeout=float(
config.get("apiserver.check_for_updates.request_timeout_sec",
3.0)),
)
if not response.ok:
return
response = response.json().get(self.component_name)
if not response:
return
latest_version = response.get("version")
if not latest_version:
return
cur_version = Version(get_version())
latest_version = Version(latest_version)
if cur_version >= latest_version:
return
return self._VersionResponse(
version=str(latest_version),
patch_upgrade=(latest_version.major == cur_version.major
and latest_version.minor == cur_version.minor),
description=response.get("description").split("\r\n"),
)
def _check_updates(self):
while not ThreadsManager.terminating:
# noinspection PyBroadException
try:
if self._redis.set(
self._lock_name,
value=datetime.utcnow().isoformat(),
ex=self.update_interval - timedelta(seconds=60),
nx=True,
):
response = self._check_new_version_available()
if response:
if response.patch_upgrade:
log.info(
f"{self.component_name.upper()} new package available: upgrade to v{response.version} "
f"is recommended!\nRelease Notes:\n{os.linesep.join(response.description)}"
)
else:
log.info(
f"{self.component_name.upper()} new version available: upgrade to v{response.version}"
f" is recommended!")
except Exception as ex:
log.exception("Failed obtaining updates: " + str(ex))
sleep(self.update_interval.total_seconds())