def _load_data(self):
"""This function will load data once and ignore it if the status is loading."""
with self._status_mutex:
if self.status == DataManagerStatus.LOADING.value:
logger.debug(
"Current status is %s , will ignore to load data.",
self.status)
return
self.status = DataManagerStatus.LOADING.value
with ComputingResourceManager.get_instance().get_executor(
max_processes_cnt=settings.MAX_PROCESSES_COUNT) as executor:
self._brief_cache.update_cache(executor)
brief_cache_update = time.time()
for _ in self._detail_cache.update_cache(executor):
update_interval = time.time() - brief_cache_update
logger.debug('Loading one round of detail cache taking %ss.',
update_interval)
if update_interval > 3: # Use 3 seconds as threshold to avoid updating too often
self._brief_cache.update_cache(executor)
brief_cache_update += update_interval
with self._status_mutex:
if not self._brief_cache.has_content(
) and not self._detail_cache.has_content():
self.status = DataManagerStatus.INVALID.value
else:
self.status = DataManagerStatus.DONE.value
logger.info("Load brief data end, and loader pool size is %r.",
self._detail_cache.loader_pool_size())
def load(self, executor=None):
"""
Load all log valid files.
When the file is reloaded, it will continue to load from where it left off.
Args:
executor (Optional[executor]): The Executor instance.
Returns:
bool, True if the train job is finished loading.
"""
logger.debug("Start to load data in ms data loader.")
if isinstance(executor, Executor):
return self._load(executor)
if executor is not None:
raise TypeError(
"'executor' should be an Executor instance or None.")
with ComputingResourceManager.get_instance().get_executor(
) as new_executor:
while not self._load(new_executor):
pass
return True