def __init__(self, params):
super(LocalFilelistLoader, self).__init__()
check_required_params(['src'], params)
self._source = params['src']
self._encoding = 'utf8' if 'encoding' not in params else params[
'encoding']
self._fd = None
def __init__(self, params):
super(TextLoader, self).__init__()
check_required_params(['txt'], params)
self._text = params['txt']
self._encoding = 'utf8' if 'encoding' not in params else params[
'encoding']
self._content = None
def __init__(self, params):
super(WebLoader, self).__init__()
check_required_params(['url'], params)
self._url = params['url']
self._encoding = 'utf8' if 'encoding' not in params else params[
'encoding']
self._doc = None
def __init__(self, params):
super(NGramTokenizer, self).__init__(params)
check_required_params(['n'], params)
try:
self.n = int(params['n'])
assert(self.n > 0)
except ValueError:
self.n = 1
self._imp_tokenizer = ChunkTokenizer()
def __init__(self, params):
super(LocalKVFileLoader, self).__init__()
check_required_params(['src'], params)
self._source = params['src']
self._encoding = 'utf8' if 'encoding' not in params else params['encoding']
self._fd = None
def __init__(self, params):
super(TextLoader, self).__init__()
check_required_params(['txt'], params)
self._text = params['txt']
self._encoding = 'utf8' if 'encoding' not in params else params['encoding']
self._content = None
def __init__(self, params):
super(WebLoader, self).__init__()
check_required_params(['url'], params)
self._url = params['url']
self._encoding = 'utf8' if 'encoding' not in params else params['encoding']
self._doc = None
