def createBatchSystem(config, jobStore=None, userScript=None):
"""
Creates an instance of the batch system specified in the given config. If a job store and
a user script are given then the user script can be hot deployed into the workflow.
:param toil.common.Config config: the current configuration
:param toil.jobStores.abstractJobStore.AbstractJobStore jobStore: an instance of a jobStore
:param ModuleDescriptor userScript: a handle to the Python module defining the root job
:return: an instance of a concrete subclass of AbstractBatchSystem
:rtype: batchSystems.abstractBatchSystem.AbstractBatchSystem
"""
kwargs = dict(config=config,
maxCores=config.maxCores,
maxMemory=config.maxMemory,
maxDisk=config.maxDisk)
if config.batchSystem == 'parasol':
from toil.batchSystems.parasol import ParasolBatchSystem
batchSystemClass = ParasolBatchSystem
elif config.batchSystem == 'single_machine' or config.batchSystem == 'singleMachine':
from toil.batchSystems.singleMachine import SingleMachineBatchSystem
batchSystemClass = SingleMachineBatchSystem
elif config.batchSystem == 'gridengine' or config.batchSystem == 'gridEngine':
from toil.batchSystems.gridengine import GridengineBatchSystem
batchSystemClass = GridengineBatchSystem
elif config.batchSystem == 'lsf' or config.batchSystem == 'LSF':
from toil.batchSystems.lsf import LSFBatchSystem
batchSystemClass = LSFBatchSystem
elif config.batchSystem == 'mesos' or config.batchSystem == 'Mesos':
from toil.batchSystems.mesos.batchSystem import MesosBatchSystem
batchSystemClass = MesosBatchSystem
kwargs['masterAddress'] = config.mesosMasterAddress
elif config.batchSystem == 'slurm' or config.batchSystem == 'Slurm':
from toil.batchSystems.slurm import SlurmBatchSystem
batchSystemClass = SlurmBatchSystem
else:
raise RuntimeError('Unrecognised batch system: %s' % config.batchSystem)
if not config.disableCaching and not batchSystemClass.supportsWorkerCleanup():
raise RuntimeError('%s currently does not support shared caching. Set the '
'--disableCaching flag if you want to '
'use this batch system.' % config.batchSystem)
logger.info('Using the %s' %
re.sub("([a-z])([A-Z])", "\g<1> \g<2>", batchSystemClass.__name__).lower())
if jobStore is not None:
if userScript is not None:
if not userScript.belongsToToil and batchSystemClass.supportsHotDeployment():
userScriptResource = userScript.saveAsResourceTo(jobStore)
with jobStore.writeSharedFileStream('userScript') as f:
f.write(userScriptResource.pickle())
kwargs['userScript'] = userScriptResource
else:
from toil.jobStores.abstractJobStore import NoSuchFileException
try:
with jobStore.readSharedFileStream('userScript') as f:
userScriptResource = Resource.unpickle(f.read())
kwargs['userScript'] = userScriptResource
except NoSuchFileException:
pass
return batchSystemClass(**kwargs)
