def __getattr__(self, item): # Not tested. Don't use !! if item == "field_arrays": raise AttributeError if isinstance(item, str) and item in self.field_arrays: return self.field_arrays[item] try: reader = DataLoaderRegister.get_reader(item) return reader except AttributeError: raise
class NativeDataSetLoader(DataSetLoader): """A simple example of DataSetLoader """ def __init__(self): super(NativeDataSetLoader, self).__init__() def load(self, path): ds = DataSet.read_csv(path, headers=("raw_sentence", "label"), sep="\t") ds.set_input("raw_sentence") ds.set_target("label") return ds DataLoaderRegister.set_reader(NativeDataSetLoader, 'read_naive') class RawDataSetLoader(DataSetLoader): """A simple example of raw data reader """ def __init__(self): super(RawDataSetLoader, self).__init__() def load(self, data_path, split=None): with open(data_path, "r", encoding="utf-8") as f: lines = f.readlines() lines = lines if split is None else [l.split(split) for l in lines] lines = list(filter(lambda x: len(x) > 0, lines)) return self.convert(lines)