def __init__(self, params): super(LocalFilelistLoader, self).__init__() check_required_params(['src'], params) self._source = params['src'] self._encoding = 'utf8' if 'encoding' not in params else params[ 'encoding'] self._fd = None
def __init__(self, params): super(TextLoader, self).__init__() check_required_params(['txt'], params) self._text = params['txt'] self._encoding = 'utf8' if 'encoding' not in params else params[ 'encoding'] self._content = None
def __init__(self, params): super(WebLoader, self).__init__() check_required_params(['url'], params) self._url = params['url'] self._encoding = 'utf8' if 'encoding' not in params else params[ 'encoding'] self._doc = None
def __init__(self, params): super(NGramTokenizer, self).__init__(params) check_required_params(['n'], params) try: self.n = int(params['n']) assert(self.n > 0) except ValueError: self.n = 1 self._imp_tokenizer = ChunkTokenizer()
def __init__(self, params): super(LocalKVFileLoader, self).__init__() check_required_params(['src'], params) self._source = params['src'] self._encoding = 'utf8' if 'encoding' not in params else params['encoding'] self._fd = None
def __init__(self, params): super(TextLoader, self).__init__() check_required_params(['txt'], params) self._text = params['txt'] self._encoding = 'utf8' if 'encoding' not in params else params['encoding'] self._content = None
def __init__(self, params): super(WebLoader, self).__init__() check_required_params(['url'], params) self._url = params['url'] self._encoding = 'utf8' if 'encoding' not in params else params['encoding'] self._doc = None