def _load(self, executor): """ Load all log valid files. When the file is reloaded, it will continue to load from where it left off. Args: executor (executor): The Executor instance. Returns: bool, True if the train job is finished loading. """ filenames = self.filter_valid_files() if not filenames: logger.warning("No valid files can be loaded, summary_dir: %s.", self._summary_dir) raise exceptions.SummaryLogPathInvalid() old_filenames = list(self._valid_filenames) self._valid_filenames = filenames self._check_files_deleted(filenames, old_filenames) finished = True for parser in self._parser_list: finished = parser.parse_files( executor, filenames, events_data=self._events_data) and finished return finished
def load(self, executor=None): """Load the data when loader is exist. Args: executor (Optional[Executor]): The executor instance. Returns: bool, True if the loader is finished loading. """ if self._loader is None: ms_dataloader = MSDataLoader(self._summary_dir) loaders = [ms_dataloader] for loader in loaders: if loader.filter_valid_files(): self._loader = loader break if self._loader is None: logger.warning( "No valid files can be loaded, summary_dir: %s.", self._summary_dir) raise exceptions.SummaryLogPathInvalid() return self._loader.load(executor)
def load(self): """Load the data when loader is exist.""" if self._loader is None: ms_dataloader = MSDataLoader(self._summary_dir) loaders = [ms_dataloader] for loader in loaders: if loader.filter_valid_files(): self._loader = loader break if self._loader is None: logger.warning( "No valid files can be loaded, summary_dir: %s.", self._summary_dir) raise exceptions.SummaryLogPathInvalid() self._loader.load()
def __init__(self, train_id, data_manager, tag=None): Validation.check_param_empty(train_id=train_id) super(GraphProcessor, self).__init__(data_manager) train_job = self._data_manager.get_train_job_by_plugin( train_id, PluginNameEnum.GRAPH.value) if train_job is None: raise exceptions.SummaryLogPathInvalid() if not train_job['tags']: raise ParamValueError( "Can not find any graph data in the train job.") if tag is None: tag = train_job['tags'][0] tensors = self._data_manager.list_tensors(train_id, tag=tag) self._graph = tensors[0].value
def load(self): """ Load all log valid files. When the file is reloaded, it will continue to load from where it left off. """ logger.debug("Start to load data in ms data loader.") filenames = self.filter_valid_files() if not filenames: logger.warning("No valid files can be loaded, summary_dir: %s.", self._summary_dir) raise exceptions.SummaryLogPathInvalid() old_filenames = list(self._valid_filenames) self._valid_filenames = filenames self._check_files_deleted(filenames, old_filenames) for parser in self._parser_list: parser.parse_files(filenames, events_data=self._events_data)
def get_single_train_task(self, plugin_name, train_id): """ get single train task. Args: plugin_name (str): Plugin name, refer `PluginNameEnum`. train_id (str): Specify a training job to query. Returns: {'train_jobs': list[TrainJob]}, refer to restful api. """ Validation.check_param_empty(plugin_name=plugin_name, train_id=train_id) Validation.check_plugin_name(plugin_name=plugin_name) train_job = self._data_manager.get_train_job_by_plugin( train_id=train_id, plugin_name=plugin_name) if train_job is None: raise exceptions.SummaryLogPathInvalid() return dict(train_jobs=[train_job])
def load(self, computing_resource_mgr): """Load the data when loader is exist. Args: computing_resource_mgr (ComputingResourceManager): The ComputingResourceManager instance. """ if self._loader is None: ms_dataloader = MSDataLoader(self._summary_dir) loaders = [ms_dataloader] for loader in loaders: if loader.filter_valid_files(): self._loader = loader break if self._loader is None: logger.warning( "No valid files can be loaded, summary_dir: %s.", self._summary_dir) raise exceptions.SummaryLogPathInvalid() self._loader.load(computing_resource_mgr)
def load(self, computing_resource_mgr): """ Load all log valid files. When the file is reloaded, it will continue to load from where it left off. Args: computing_resource_mgr (ComputingResourceManager): The ComputingResourceManager instance. """ logger.debug("Start to load data in ms data loader.") filenames = self.filter_valid_files() if not filenames: logger.warning("No valid files can be loaded, summary_dir: %s.", self._summary_dir) raise exceptions.SummaryLogPathInvalid() old_filenames = list(self._valid_filenames) self._valid_filenames = filenames self._check_files_deleted(filenames, old_filenames) with computing_resource_mgr.get_executor() as executor: for parser in self._parser_list: parser.parse_files(executor, filenames, events_data=self._events_data)