def load(self): """Start loading data from the latest summary file to the loader.""" self.status = _LoaderStatus.LOADING.value filenames = [] for filename in FileHandler.list_dir(self._loader_info['summary_dir']): if FileHandler.is_file( FileHandler.join(self._loader_info['summary_dir'], filename)): filenames.append(filename) filenames = ExplainLoader._filter_files(filenames) if not filenames: raise TrainJobNotExistError( 'No summary file found in %s, explain job will be delete.' % self._loader_info['summary_dir']) is_end = False while not is_end and self.status != _LoaderStatus.STOP.value: try: file_changed, is_end, event_dict = self._parser.list_events( filenames) except UnknownError: break if file_changed: logger.info( 'Summary file in %s update, reload the data in the summary.', self._loader_info['summary_dir']) self._clear_job() if event_dict: self._import_data_from_event(event_dict)
def load(self): """Start loading data from the latest summary file to the loader.""" filenames = [] for filename in FileHandler.list_dir(self._loader_info['summary_dir']): if FileHandler.is_file( FileHandler.join(self._loader_info['summary_dir'], filename)): filenames.append(filename) filenames = ExplainLoader._filter_files(filenames) if not filenames: raise TrainJobNotExistError( 'No summary file found in %s, explain job will be delete.' % self._loader_info['summary_dir']) is_end = False while not is_end: is_clean, is_end, event_dict = self._parser.parse_explain( filenames) if is_clean: logger.info( 'Summary file in %s update, reload the data in the summary.', self._loader_info['summary_dir']) self._clear_job() if event_dict: self._import_data_from_event(event_dict)
def filter_valid_files(self): """ Gets a list of valid files from the given file path. Returns: list[str], file name list. """ filenames = [] for filename in FileHandler.list_dir(self._summary_dir): if FileHandler.is_file(FileHandler.join(self._summary_dir, filename)): filenames.append(filename) valid_filenames = [] for parser in self._parser_list: valid_filenames.extend(parser.filter_files(filenames)) return list(set(valid_filenames))
def abc(): FileHandler.is_file('aaa') print('after')