def tokenize(outfile, paths, base_path, no_shuffle, tokenizer_type,
tokenizer_pickle, doc_id_level, n_jobs, chunksize):
"""
Write later if module interface is needed. See _cli for the documentation.
"""
assert (paths == []) or (base_path is None)
if base_path:
paths = filefilter.get_paths(base_path, file_type='*', get_iter=True)
if no_shuffle is False:
paths = list(paths)
shuffle(paths)
if tokenizer_pickle is not None:
tokenizer = SaveLoad.load(tokenizer_pickle)
else:
tokenizer_dict = {'basic': text_processors.TokenizerBasic}
tokenizer = tokenizer_dict[tokenizer_type]()
formatter = text_processors.VWFormatter()
func = partial(_tokenize_one, tokenizer, formatter, doc_id_level)
results_iterator = imap_easy(func, paths, n_jobs, chunksize)
for result in results_iterator:
outfile.write(result + '\n')
def countReadsPerFragment(fragmentCount, lookup_structure, options, args, triangular=True):
'''
slurps in all input fils in parallel
counts the reads per fragment and generates appropriate output files
'''
if (options.verbose):
print >> sys.stdout, "- %s STARTED : reading input files : %s" % (timeStamp(), str(args))
print >> sys.stdout, " FragmentCount: %d" % (fragmentCount)
fragmentList = np.zeros((fragmentCount,), dtype=np.uint16)
fragmentPairs = None
func = partial(countReadsPerFragmentParallel, fragmentCount=fragmentCount, lookup_structure=lookup_structure, triangular=triangular, options=options)
results_iterator = imap_easy(func, args, n_jobs=min(8, len(args)), chunksize=1)
if (options.verbose):
print >> sys.stdout, "- %s FINISHED: reading input files " % (timeStamp())
print >> sys.stdout, "- %s STARTED : combining input files " % (timeStamp())
# combine
for (fl,fp) in results_iterator:
fragmentList += fl
if fragmentPairs is not None:
fragmentPairs += fp
else:
fragmentPairs = fp
if (options.verbose):
print >> sys.stdout, "- %s FINISHED: combining input files " % (timeStamp())
return [ fragmentList, fragmentPairs ]
def tokenize(
outfile, paths, base_path, no_shuffle, tokenizer_type, tokenizer_pickle,
doc_id_level, n_jobs, chunksize):
"""
Write later if module interface is needed. See _cli for the documentation.
"""
assert (paths == []) or (base_path is None)
if base_path:
paths = filefilter.get_paths(base_path, file_type='*', get_iter=True)
if no_shuffle is False:
paths = list(paths)
shuffle(paths)
if tokenizer_pickle is not None:
tokenizer = SaveLoad.load(tokenizer_pickle)
else:
tokenizer_dict = {'basic': text_processors.TokenizerBasic}
tokenizer = tokenizer_dict[tokenizer_type]()
formatter = text_processors.VWFormatter()
func = partial(_tokenize_one, tokenizer, formatter, doc_id_level)
results_iterator = imap_easy(func, paths, n_jobs, chunksize)
for result in results_iterator:
outfile.write(result + '\n')
def to_vw(self, outfile, n_jobs=-1, chunksize=1000, raise_on_bad_id=True,
cache_list=None, cache_list_file=None):
"""
Write our filestream to a VW (Vowpal Wabbit) formatted file.
Parameters
----------
outfile : filepath or buffer
n_jobs : Integer
Use n_jobs different jobs to do the processing. Set = 4 for 4
jobs. Set = -1 to use all available, -2 for all except 1,...
chunksize : Integer
Workers process this many jobs at once before pickling and sending
results to master. If this is too low, communication overhead
will dominate. If this is too high, jobs will not be distributed
evenly.
cache_list : List of strings
Write these info_stream items to file on every iteration.
cache_list_file : filepath or buffer
"""
formatter = text_processors.VWFormatter()
func = partial(_to_sstr, formatter=formatter,
raise_on_bad_id=raise_on_bad_id, cache_list=cache_list)
results_iterator = imap_easy(func, self.info_stream(), n_jobs, chunksize)
if cache_list_file:
with smart_open(outfile, 'w') as open_outfile, \
smart_open(cache_list_file, 'w') as open_cache_file:
for result, cache_list in results_iterator:
open_outfile.write(result + '\n')
open_cache_file.write(str(cache_list) + '\n')
else:
with smart_open(outfile, 'w') as open_outfile:
for result, cache_list in results_iterator:
open_outfile.write(result + '\n')
def countReadsPerFragment(fragmentCount,
lookup_structure,
options,
args,
triangular=True):
'''
slurps in all input fils in parallel
counts the reads per fragment and generates appropriate output files
'''
if (options.verbose):
print >> sys.stdout, "- %s STARTED : reading input files : %s" % (
timeStamp(), str(args))
print >> sys.stdout, " FragmentCount: %d" % (fragmentCount)
fragmentList = np.zeros((fragmentCount, ), dtype=np.uint16)
fragmentPairs = None
func = partial(countReadsPerFragmentParallel,
fragmentCount=fragmentCount,
lookup_structure=lookup_structure,
triangular=triangular,
options=options)
results_iterator = imap_easy(func,
args,
n_jobs=min(8, len(args)),
chunksize=1)
if (options.verbose):
print >> sys.stdout, "- %s FINISHED: reading input files " % (
timeStamp())
print >> sys.stdout, "- %s STARTED : combining input files " % (
timeStamp())
# combine
for (fl, fp) in results_iterator:
fragmentList += fl
if fragmentPairs is not None:
fragmentPairs += fp
else:
fragmentPairs = fp
if (options.verbose):
print >> sys.stdout, "- %s FINISHED: combining input files " % (
timeStamp())
return [fragmentList, fragmentPairs]
def to_vw(self, outfile, n_jobs=1, chunksize=1000, raise_on_bad_id=True):
"""
Write our filestream to a VW (Vowpal Wabbit) formatted file.
Parameters
----------
outfile : filepath or buffer
n_jobs : Integer
Use n_jobs different jobs to do the processing. Set = 4 for 4
jobs. Set = -1 to use all available, -2 for all except 1,...
chunksize : Integer
Workers process this many jobs at once before pickling and sending
results to master. If this is too low, communication overhead
will dominate. If this is too high, jobs will not be distributed
evenly.
raise_on_bad_id : Boolean
If True, raise DocIDError when the doc_id (formed by self) is not
a valid VW "Tag". I.e. contains :, |, ', or whitespace.
If False, print warning.
"""
# Note: This is similar to rosetta/cmd/files_to_vw.py
# This implementation is more complicated, due to the fact that a
# streamer specifies the method to extract doc_id from a stream.
# To be faithful to the streamer, we must therefore use the streamer
# to stream the files. This requires a combination of imap_easy and
# a chunker.
#
# Create an iterator over chunks of paths
path_group_iter = common.grouper(self.paths, chunksize)
formatter = text_processors.VWFormatter()
func = partial(_group_to_sstr, self, formatter, raise_on_bad_id)
# Process one group at a time...set imap_easy chunksize arg to 1
# since each group contains many paths.
results_iterator = imap_easy(func, path_group_iter, n_jobs, 1)
with smart_open(outfile, 'w') as open_outfile:
for group_results in results_iterator:
for sstr in group_results:
open_outfile.write(sstr + '\n')
def to_vw(self, outfile, n_jobs=1, chunksize=1000, raise_on_bad_id=True):
"""
Write our filestream to a VW (Vowpal Wabbit) formatted file.
Parameters
----------
outfile : filepath or buffer
n_jobs : Integer
Use n_jobs different jobs to do the processing. Set = 4 for 4
jobs. Set = -1 to use all available, -2 for all except 1,...
chunksize : Integer
Workers process this many jobs at once before pickling and sending
results to master. If this is too low, communication overhead
will dominate. If this is too high, jobs will not be distributed
evenly.
raise_on_bad_id : Boolean
If True, raise DocIDError when the doc_id (formed by self) is not
a valid VW "Tag". I.e. contains :, |, ', or whitespace.
If False, print warning.
"""
# Note: This is similar to rosetta/cmd/files_to_vw.py
# This implementation is more complicated, due to the fact that a
# streamer specifies the method to extract doc_id from a stream.
# To be faithful to the streamer, we must therefore use the streamer
# to stream the files. This requires a combination of imap_easy and
# a chunker.
#
# Create an iterator over chunks of paths
path_group_iter = common.grouper(self.paths, chunksize)
formatter = text_processors.VWFormatter()
func = partial(_group_to_sstr, self, formatter, raise_on_bad_id)
# Process one group at a time...set imap_easy chunksize arg to 1
# since each group contains many paths.
results_iterator = imap_easy(func, path_group_iter, n_jobs, 1)
with smart_open(outfile, 'w') as open_outfile:
for group_results in results_iterator:
for sstr in group_results:
open_outfile.write(sstr + '\n')
def test_imap_easy_3job(self):
result_iterator = parallel_easy.imap_easy(abfunc, self.numbers, 3, 1)
result = []
for number in result_iterator:
result.append(number)
self.assertEqual(result, self.benchmark)
def test_imap_easy_3job_lambda(self):
result_iterator = parallel_easy.imap_easy(sqlambda, self.numbers, 3, 1, use_pathos=True)
result = []
for number in result_iterator:
result.append(number)
self.assertEqual(result, self.sqbenchmark)
