def _check_integrity(self): root = self.root for fentry in (self.train_list + self.test_list): filename, md5 = fentry[0], fentry[1] fpath = os.path.join(root, self.base_folder, filename) if not check_integrity(fpath, md5): return False return True
def _check_integrity(self): for fentry in ['train', 'test', 'extra']: filename, md5 = self.split_list[fentry][1], self.split_list[ fentry][2] fpath = os.path.join(self.raw_folder, filename) if not check_integrity(fpath, md5): return False return True
def _load_meta(self): path = os.path.join(self.root, self.base_folder, self.meta['filename']) if not check_integrity(path, self.meta['md5']): raise RuntimeError('Dataset metadata file not found or corrupted.' + ' You can use download=True to download it') with open(path, 'rb') as infile: if sys.version_info[0] == 2: data = pickle.load(infile) else: data = pickle.load(infile, encoding='latin1') self.classes = data[self.meta['key']] self.class_to_idx = {_class: i for i, _class in enumerate(self.classes)}
def preprocessing(filepath: str) -> Tuple[List[str], List[str], List[str]]: """ This method is used to read the preprocessed training sentences :param filepath: the path from read :return: sentences = train_x labels = train_y pos = pos sentences """ sentences, labels, pos = utils.read_sentences_and_labels(filepath) # check whether the length are the same print("\t\tCheck Integrity", utils.check_integrity(sentences, labels)) return sentences, labels, pos
def _check_integrity(self): root = self.root md5 = self.split_list[self.split][2] fpath = os.path.join(root, self.filename) return check_integrity(fpath, md5)
def _check_integrity(self): zip_filename = self._get_target_folder() if not check_integrity(join(self.root, zip_filename + '.zip'), self.zips_md5[zip_filename]): return False return True