def _get_checksum(self): """ Get checksum based on a dict to ensure validity of cached DataSilo """ # keys in the dict identifies uniqueness for a given DataSilo. payload_dict = { "train_filename": str(Path(self.processor.train_filename).absolute()) } checksum = get_dict_checksum(payload_dict) return checksum
def _get_checksum(self): """ Get checksum based on a dict to ensure validity of cached DataSilo """ # keys in the dict identifies uniqueness for a given DataSilo. payload_dict = { "train_filename": str(Path(self.processor.train_filename).absolute()), "data_dir": str(self.processor.data_dir.absolute()), "max_seq_len": self.processor.max_seq_len, "dev_split": self.processor.dev_split, "tasks": self.processor.tasks } checksum = get_dict_checksum(payload_dict) return checksum