def zbl_generator_q(zbl_file_path, fields, filtered_by=['mc'], uni=False):
"""Faster version of zbl_generator (data is not filtered)."""
logging.info("[zbl_generator_q] fields=" + str(fields) + " filtered_by=" +
str(filtered_by) + " infile=" + str(zbl_file_path))
src_stream = zbl_record_generators.gen_text(zbl_file_path, fields,
filtered_by, uni)
wordslist_generator = stream_to_wordsstream_generator_simple(src_stream)
return wordslist_generator
def zbl_generator(zbl_file_path, fields, filtered_by = ['mc'], uni = False, line_filter=text_filter_lower, word_predicate=default_word_predicate):
"""Wrapper that simplifies reading from ZBL files.
Based on zbl_record_generators.gen_text.
Returns generator that generates lists of words (single line=single list)."""
src_stream = zbl_record_generators.gen_text(zbl_file_path, fields, filtered_by, uni)
wordslist_generator = stream_to_wordsstream_generator(src_stream, line_filter, word_predicate)
return wordslist_generator
def zbl_generator(zbl_file_path,
fields,
filtered_by=['mc'],
uni=False,
line_filter=text_filter_lower,
word_predicate=default_word_predicate):
"""Wrapper that simplifies reading from ZBL files.
Based on zbl_record_generators.gen_text.
Returns generator that generates lists of words (single line=single list)."""
src_stream = zbl_record_generators.gen_text(zbl_file_path, fields,
filtered_by, uni)
wordslist_generator = stream_to_wordsstream_generator(
src_stream, line_filter, word_predicate)
return wordslist_generator
def zbl_generator_q(zbl_file_path, fields, filtered_by = ['mc'], uni = False):
"""Faster version of zbl_generator (data is not filtered)."""
logging.info("[zbl_generator_q] fields="+str(fields)+" filtered_by="+str(filtered_by)+" infile="+str(zbl_file_path))
src_stream = zbl_record_generators.gen_text(zbl_file_path, fields, filtered_by, uni)
wordslist_generator = stream_to_wordsstream_generator_simple(src_stream)
return wordslist_generator
