def test_calc_workers(self):
# for 100 folder:
# -- 50 workers
# -- 2 threads_per_worker
# -- 0 spare
res = stools.calcWorkers(threads=100, nfolders=100)
self.assertEqual(res, (50, 2, 0))
# for 10 folder:
# -- 10 workers
# -- 10 threads_per_worker
# -- 0 spare
res = stools.calcWorkers(threads=100, nfolders=10)
self.assertEqual(res, (10, 10, 0))
# for 9 folder:
# -- 9 workers
# -- 11 threads_per_worker
# -- 1 spare
res = stools.calcWorkers(threads=100, nfolders=9)
self.assertEqual(res, (9, 11, 1))
def test_calc_workers(self):
# for 100 folder:
# -- 50 workers
# -- 2 threads_per_worker
# -- 0 spare
res = stools.calcWorkers(threads=100, nfolders=100)
self.assertEqual(res, (50, 2, 0))
# for 10 folder:
# -- 10 workers
# -- 10 threads_per_worker
# -- 0 spare
res = stools.calcWorkers(threads=100, nfolders=10)
self.assertEqual(res, (10, 10, 0))
# for 9 folder:
# -- 9 workers
# -- 11 threads_per_worker
# -- 1 spare
res = stools.calcWorkers(threads=100, nfolders=9)
self.assertEqual(res, (9, 11, 1))
def main(restart, retry, email, threads, verbose, debug, stages, base_logger):
'''Run pG-lt'''
# setup folder to hold pickled args
temp_dir = os.path.join(os.getcwd(), 'tempfiles')
if not os.path.isdir(temp_dir):
os.mkdir(temp_dir)
argspath = os.path.join(temp_dir, 'arguments.p')
if restart:
if not os.path.isfile(argspath):
sys.exit('Cannot restart, are you sure you have already run \
pG-lt?')
stools.logMessage('program-restart', logger=base_logger, retry=retry)
with open(argspath, 'r') as file:
nworkers, threads_per_worker, folders, email, threads, verbose,\
debug, stages = pickle.load(file)
runner = Runner(folders=folders, nworkers=nworkers, stages=stages,
threads_per_worker=threads_per_worker, wd=os.getcwd(),
email=email, verbose=verbose, debug=debug,
logger=base_logger, retry=retry)
else:
if verbose:
stools.printHeader()
# search cwd for folders that contain names and parameter files
folders = stools.getFolders()
# calculate nworkers
nworkers, threads_per_worker, spare_threads =\
stools.calcWorkers(threads=threads, nfolders=len(folders))
# start message
stools.logMessage('program-start', logger=base_logger, folders=folders,
threads=threads_per_worker*nworkers, stages=stages,
spare_threads=spare_threads, email=email)
# setup runner
base_logger.info('Setting up files and folders ....')
runner = Runner(folders=folders, nworkers=nworkers, stages=stages,
threads_per_worker=threads_per_worker, wd=os.getcwd(),
email=email, verbose=verbose, debug=debug,
logger=base_logger, retry=False)
runner.setup()
base_logger.info('Done.')
# pickle runner for restart
with open(argspath, 'w') as file:
pickle.dump((nworkers, threads_per_worker, folders, email,
threads, verbose, debug, stages), file)
# run stages
runner.run()
