def _check_integrity(self):
root = self.root
for fentry in (self.train_list + self.test_list):
filename, md5 = fentry[0], fentry[1]
fpath = os.path.join(root, self.base_folder, filename)
if not check_integrity(fpath, md5):
return False
return True
def _check_integrity(self):
for fentry in ['train', 'test', 'extra']:
filename, md5 = self.split_list[fentry][1], self.split_list[
fentry][2]
fpath = os.path.join(self.raw_folder, filename)
if not check_integrity(fpath, md5):
return False
return True
def _load_meta(self):
path = os.path.join(self.root, self.base_folder, self.meta['filename'])
if not check_integrity(path, self.meta['md5']):
raise RuntimeError('Dataset metadata file not found or corrupted.' +
' You can use download=True to download it')
with open(path, 'rb') as infile:
if sys.version_info[0] == 2:
data = pickle.load(infile)
else:
data = pickle.load(infile, encoding='latin1')
self.classes = data[self.meta['key']]
self.class_to_idx = {_class: i for i, _class in enumerate(self.classes)}
def preprocessing(filepath: str) -> Tuple[List[str], List[str], List[str]]:
"""
This method is used to read the preprocessed training sentences
:param filepath: the path from read
:return:
sentences = train_x
labels = train_y
pos = pos sentences
"""
sentences, labels, pos = utils.read_sentences_and_labels(filepath)
# check whether the length are the same
print("\t\tCheck Integrity", utils.check_integrity(sentences, labels))
return sentences, labels, pos
def _check_integrity(self):
root = self.root
md5 = self.split_list[self.split][2]
fpath = os.path.join(root, self.filename)
return check_integrity(fpath, md5)
def _check_integrity(self):
zip_filename = self._get_target_folder()
if not check_integrity(join(self.root, zip_filename + '.zip'), self.zips_md5[zip_filename]):
return False
return True
