def tokenize(
outfile, paths, base_path, no_shuffle, tokenizer_type, tokenizer_pickle,
doc_id_level, n_jobs, chunksize):
"""
Write later if module interface is needed. See _cli for the documentation.
"""
assert (paths == []) or (base_path is None)
if base_path:
paths = filefilter.get_paths(base_path, file_type='*', get_iter=True)
if no_shuffle is False:
paths = list(paths)
shuffle(paths)
if tokenizer_pickle is not None:
tokenizer = SaveLoad.load(tokenizer_pickle)
else:
tokenizer_dict = {'basic': text_processors.TokenizerBasic}
tokenizer = tokenizer_dict[tokenizer_type]()
formatter = text_processors.VWFormatter()
func = partial(_tokenize_one, tokenizer, formatter, doc_id_level)
results_iterator = imap_easy(func, paths, n_jobs, chunksize)
for result in results_iterator:
outfile.write(result + '\n')
def to_vw(self, outfile, n_jobs=1, chunksize=1000):
"""
Write our filestream to a VW (Vowpal Wabbit) formatted file.
Parameters
----------
outfile : filepath or buffer
n_jobs : Integer
Use n_jobs different jobs to do the processing. Set = 4 for 4
jobs. Set = -1 to use all available, -2 for all except 1,...
chunksize : Integer
Workers process this many jobs at once before pickling and sending
results to master. If this is too low, communication overhead
will dominate. If this is too high, jobs will not be distributed
evenly.
"""
# Note: This is similar to declass/cmd/files_to_vw.py
# This implementation is more complicated, due to the fact that a
# streamer specifies the method to extract doc_id from a stream.
# To be faithful to the streamer, we must therefore use the streamer
# to stream the files. This requires a combination of imap_easy and
# a chunker.
#
# Create an iterator over chunks of paths
path_group_iter = common.grouper(self.paths, chunksize)
formatter = text_processors.VWFormatter()
func = partial(_group_to_sstr, self, formatter)
# Process one group at a time...set imap_easy chunksize arg to 1
# since each group contains many paths.
results_iterator = imap_easy(func, path_group_iter, n_jobs, 1)
with smart_open(outfile, 'w') as open_outfile:
for group_results in results_iterator:
for sstr in group_results:
open_outfile.write(sstr + '\n')
def download_files(paths, n_jobs=7, chunksize=100):
"""
Wgets tjhe files, but uses the parallel_easy imap function to make things
go a lot faster. See documentation therein.
"""
imap_easy(_downld_func, paths, n_jobs, chunksize)
def test_imap_easy_3job(self):
result_iterator = base.imap_easy(abfunc, self.numbers, 3, 1)
result = []
for number in result_iterator:
result.append(number)
self.assertEqual(result, self.benchmark)
