def preProcessFilesMP(dIn, dOut, jobSpec, glob, recursive, jobs):
"""Multiprocessing code to preprocess directories. Returns a count of ITUs
processed."""
if jobs < 0:
raise ValueError(
'preProcessFilesMP(): can not run with negative number of jobs: %d'
% jobs)
if jobs == 0:
jobs = multiprocessing.cpu_count()
assert jobs > 1, 'preProcessFilesMP(): number of jobs: %d???' % jobs
logging.info('plotLogPassesMP(): Setting multi-processing jobs to %d' %
jobs)
myTaskS = [
(t.filePathIn, t.filePathOut, jobSpec) \
for t in DirWalk.dirWalk(dIn, dOut, glob, recursive, bigFirst=True)
]
with multiprocessing.Pool(processes=jobs) as myPool:
if jobSpec.keepGoing:
fn = preprocessFileToOutputNoExcept
else:
fn = preprocessFileToOutput
myResults = [
r.get() for r in [myPool.apply_async(fn, t) for t in myTaskS]
]
return myResults
def preprocessDirToOutput(inDir, outDir, jobSpec, globMatch, recursive,
numJobs):
"""Pre-process all the files in a directory. Returns a count of the TUs.
This uses multiprocessing where possible."""
assert os.path.isdir(inDir)
if numJobs != 1:
results = preProcessFilesMP(inDir, outDir, jobSpec, globMatch,
recursive, numJobs)
else:
results = []
for t in DirWalk.dirWalk(inDir,
outDir,
globMatch,
recursive,
bigFirst=False):
if jobSpec.keepGoing:
fn = preprocessFileToOutputNoExcept
else:
fn = preprocessFileToOutput
results.append(fn(t.filePathIn, t.filePathOut, jobSpec))
# Write the linking HTML from the title and file paths.
# print('results', results)
_writeDirectoryIndexHTML(inDir, outDir, results, jobSpec)
def test_10(self):
"""TestDirWalk.test_10(): Fails if input does not exist."""
try:
for v in DirWalk.dirWalk('non_existent'):
print(v)
self.fail('DirWalk.ExceptionDirWalk not raised.')
except DirWalk.ExceptionDirWalk:
pass
def test_02(self):
"""TestDirWalk.test_02(): Input and output, no globbing or recursion."""
print()
for v in DirWalk.dirWalk('.',
theOut='spam',
theFnMatch=None,
recursive=False):
print(v)
def test_04(self):
"""TestDirWalk.test_04(): Input and output, *.py and recursion."""
print()
for v in DirWalk.dirWalk('.',
theOut='spam',
theFnMatch='*.py',
recursive=True):
print(v)
def test_12(self):
"""TestDirWalk.test_12(): raises."""
# self.assertRaises(ValueError, DirWalk.dirWalk, '.', None, 1)
try:
for v in DirWalk.dirWalk('.', None, 94):
print(v)
self.fail('ValueError not raised.')
except ValueError:
pass
def test_06(self):
"""TestDirWalk.test_06(): Input only, *.py, recursion and biggest first."""
print()
for v in DirWalk.dirWalk('.',
theOut=None,
theFnMatch='*.py',
recursive=True,
bigFirst=True):
print('{:8d}: {:s}'.format(os.path.getsize(v), v))
def test_05(self):
"""TestDirWalk.test_05(): Input and output, *.py, recursion and biggest first."""
print()
for v in DirWalk.dirWalk('.',
theOut='spam',
theFnMatch='*.py',
recursive=True,
bigFirst=True):
print('{:8d}: {:s}'.format(os.path.getsize(v.filePathIn), v))
def test_03(self):
"""TestDirWalk.test_03(): Input only, *.py and recursion."""
print()
for v in DirWalk.dirWalk('.', theFnMatch='*.py', recursive=True):
print(v)
def test_01(self):
"""TestDirWalk.test_01(): Input only, defaults."""
print()
for v in DirWalk.dirWalk('.'):
print(v)
def test_01(self):
"""TestGenBigFirst.test_01(): Input only, defaults."""
print()
for v in DirWalk.genBigFirst('.'):
print('{:8d}: {:s}'.format(os.path.getsize(v), v))