def __init__(self, name="myBatchTask", task_dir=None, cfg=None):
"""Create process from """
self.name = name
if os.path.isabs(task_dir):
# Since relative path differs on local and cluster.
# Ensure absolute or there will be no output.
self.task_dir = task_dir
else:
raise ValueError("ERROR: task_dir must be an absolute path")
if not os.path.exists(self.task_dir + "/log"):
os.makedirs(self.task_dir + "/log")
self.cfg = cfg if cfg is not None else self.templateCfg()
self.logger = Logger("task.log")
self.logger.setAbsLogfileDir(self.task_dir)
def __init__(self, name="myProcess", work_dir="testProcess", cfg=None):
self.name = name
self.work_dir = work_dir
self.cwd = os.getcwd()
self.cfg = cfg if cfg is not None else Process.templateConfig()
# Register services
self._services = OrderedDict()
self.addService('logger', Logger("runtime.log"))
self.addService('filemanager', FileManager())
self.addService('sourcemanager', SourceManager())
def beginSeq_registerServices(self):
"""Initialize all services."""
if not 'dbplayer' in self._services.keys():
dbplayer = FitDBPlayer(
absInputDir=os.path.join(modulePath, self.work_dir))
self.addService("dbplayer", dbplayer)
self.addService('logger', Logger("runtime.log", 999))
self.addService('filemanager', FileManager())
self.addService('sourcemanager', SourceManager())
for seq_obj in self._sequence:
if seq_obj.logger is None: setattr(seq_obj, "logger", self.logger)
for key, s in self._services.items():
setattr(s, "process", self)
if s.logger is None:
setattr(s, "logger", self.logger)
s._beginSeq()
class AbsBatchTaskWrapper:
""""""
def __init__(self, name="myBatchTask", task_dir=None, cfg=None):
"""Create process from """
self.name = name
if os.path.isabs(task_dir):
# Since relative path differs on local and cluster.
# Ensure absolute or there will be no output.
self.task_dir = task_dir
else:
raise ValueError("ERROR: task_dir must be an absolute path")
if not os.path.exists(self.task_dir + "/log"):
os.makedirs(self.task_dir + "/log")
self.cfg = cfg if cfg is not None else self.templateCfg()
self.logger = Logger("task.log")
self.logger.setAbsLogfileDir(self.task_dir)
@classmethod
def templateCfg(cls):
cfg = {
'nJobs': 1,
'queue': batchConfig.BATCH_QUEUE,
'work_dir': None,
}
return cfg
def createJdlBase(self):
""" Base of jdl script to be futher decorated in createJdl """
""""""
#transfer_output_files = job$(Process),job$(Process).tar.gz
templateJdl = """
getenv = True
log = log/{time}_{seqKey}_{binKey}_$(Process).log
output = log/{time}_{seqKey}_{binKey}_$(Process).out
error = log/{time}_{seqKey}_{binKey}_$(Process).err
+JobFlavour = "{JobFlavour}"
initialdir = {initialdir}
executable = {executable}
Notify_user = [email protected]
should_transfer_files = YES
when_to_transfer_output = ON_EXIT
""".format(
time=str(time.localtime().tm_yday)+'_'+str(time.localtime().tm_hour)+str(time.localtime().tm_min)+str(time.localtime().tm_sec),
seqKey="{seqKey}",
initialdir=self.task_dir,
JobFlavour=self.cfg['queue'],
executable="{executable}",
binKey="{binKey}")
return templateJdl
@abc.abstractmethod
def createJdl(self, parser_args):
""" To be customized by users. Start from createJdlBase. """
raise NotImplementedError
def getWrappedProcess(self, process, jobId, **kwargs):
""" To be customized by users. """
return process
def runWrappedProcess(self, process, jobId, wrapper_kwargs=None):
""" Serve as 'main' to be called on computing nodes """
if wrapper_kwargs is None:
wrapper_kwargs = {}
p = self.getWrappedProcess(process, jobId, **wrapper_kwargs)
if self.cfg['work_dir'] is None:
label = "SimFit" if p.cfg['args'].SimFit else p.cfg['args'].Year
p.work_dir = os.path.join(self.task_dir, "{0}_{1}_{jobId:04d}".format(label, p.cfg['args'].binKey, jobId=jobId))
elif isinstance(self.cfg['work_dir'], str):
p.work_dir = os.path.join(self.task_dir, self.cfg['work_dir'])
else:
p.work_dir = os.path.join(self.task_dir, self.cfg['work_dir'][jobId])
p.setSequence(p._sequence) ## ALERT
try:
p.beginSeq()
p.runSeq()
finally:
p.endSeq()
p.reset()
for obj in p._sequence: obj.reset()
# HTCondor does not transfer output directory but only file
os.chdir(self.task_dir)
class AbsBatchTaskWrapper:
""""""
def __init__(self, name="myBatchTask", task_dir=None, cfg=None):
"""Create process from """
self.name = name
if os.path.isabs(task_dir):
# Since relative path differs on local and cluster.
# Ensure absolute or there will be no output.
self.task_dir = task_dir
else:
raise ValueError("ERROR: task_dir must be an absolute path")
if not os.path.exists(self.task_dir + "/log"):
os.makedirs(self.task_dir + "/log")
self.cfg = cfg if cfg is not None else self.templateCfg()
self.logger = Logger("task.log")
self.logger.setAbsLogfileDir(self.task_dir)
@classmethod
def templateCfg(cls):
cfg = {
'nJobs': 1,
'queue': batchConfig.BATCH_QUEUE,
'work_dir': None
}
return cfg
def createJdlBase(self):
""" Base of jdl script to be futher decorated in createJdl """
""""""
templateJdl = """
getenv = True
log = log/log.$(Process)
output = log/out.$(Process)
error = log/err.$(Process)
+JobFlavour = "{JobFlavour}"
initialdir = {initialdir}
executable = {executable}
# when_to_transfer_output = ON_EXIT
# transfer_output_files = job$(Process),job$(Process).tar.gz
""".format(initialdir=self.task_dir,
JobFlavour=self.cfg['queue'],
executable=os.path.abspath(__main__.__file__),)
return templateJdl
@abc.abstractmethod
def createJdl(self, parser_args):
""" To be customized by users. Start from createJdlBase. """
raise NotImplementedError
def getWrappedProcess(self, process, jobId, **kwargs):
""" To be customized by users. """
return process
def runWrappedProcess(self, process, jobId, wrapper_kwargs=None):
""" Serve as 'main' to be called on computing nodes """
if wrapper_kwargs is None:
wrapper_kwargs = {}
p = self.getWrappedProcess(process, jobId, **wrapper_kwargs)
if self.cfg['work_dir'] is None:
p.work_dir = os.path.join(self.task_dir, "job{jobId:04d}".format(jobId=jobId))
elif isinstance(self.cfg['work_dir'], str):
p.work_dir = os.path.join(self.task_dir, self.cfg['work_dir'])
else:
p.work_dir = os.path.join(self.task_dir, self.cfg['work_dir'][jobId])
try:
p.beginSeq()
p.runSeq()
finally:
p.endSeq()
# HTCondor does not transfer output directory but only file
os.chdir(self.task_dir)