def execute(self):
environ["DWF_SIXDIGIT"] = self.job.getSixDigits()
print 'EXEC_DIR_ROOT: %s' % EXEC_DIR_ROOT
print 'instanceId : %s' % str(self.job.getSixDigits())
my_exec_dir = oPjoin(EXEC_DIR_ROOT, self.job.getSixDigits(), "local" if self.batchId == "-1" else str(self.batchId))
mkdir(my_exec_dir)
chdir(my_exec_dir)
self.logThis("execution directory %s", my_exec_dir)
# DMPSWSYS = getenv("DMPSWSYS")
# DAMPE_SW_DIR = getenv("DAMPE_SW_DIR",None)
# if DAMPE_SW_DIR is None:
# raise Exception("must define $DAMPE_SW_DIR")
# next, run the executable
# if not DAMPE_SW_DIR in DMPSWSYS:
# log.info("trying to re-source setup script.")
# job.sourceSetupScript()
rc = 0
rc += self.__prepare()
if rc: self.exit_app(rc,msg="Exiting after prepare step")
rc += self.__runPayload()
if rc: self.exit_app(rc,msg="Exiting after payload")
rc += self.__postRun()
if rc: self.exit_app(rc,msg="Exiting after post-run")
# finally, compile output file.
self.logThis("job complete, cleaning up working directory")
chdir(self.pwd)
rm(my_exec_dir)
try:
self.job.updateStatus("Done", "ApplicationComplete")
except Exception as err:
self.logThis("EXCEPTION: %s", err)
def main(args=None):
parser = ArgumentParser(usage="Usage: %(prog)s [options]", description="initialize dampe workflow")
parser.add_argument("-f", "--file", dest="file", type=str, default=None,
help='use this flag if you plan to provide a file')
opts = parser.parse_args(args)
if opts.file is not None:
src = opts.file
dmpROOT = dirname(abspath(DmpFile))
tg = oPjoin(dmpROOT, "config/settings.cfg")
safe_copy(src, tg, sleep='3s', attempts=3, debug=True)
return
def main(args=None):
parser = ArgumentParser(usage="Usage: %(prog)s [options]",
description="initialize dampe workflow")
parser.add_argument("-f",
"--file",
dest="file",
type=str,
default=None,
help='use this flag if you plan to provide a file')
opts = parser.parse_args(args)
if opts.file is not None:
src = opts.file
dmpROOT = dirname(abspath(DmpFile))
tg = oPjoin(dmpROOT, "config/settings.cfg")
safe_copy(src, tg, sleep='3s', attempts=3, debug=True)
return
def __postRun(self):
self.job.updateStatus("Running", "PreparingOutputData")
for fi in self.job.OutputFiles:
src = expandvars(fi['source'])
tg = expandvars(fi['target'])
self.logThis("Staging %s --> %s",oPjoin(abspath(self.pwd),src),tg)
_dir = dirname(tg)
try:
self.logThis("creating output directory %s", _dir)
try:
mkdir(_dir)
except IOError as err:
self.logThis("error creating output directory, trying to recover, error follows: ",err)
safe_copy(src, tg, attempts=4, sleep='4s', checksum=True)
self.job.registerDS(filename=tg, overwrite=True)
except Exception, e:
try:
self.job.updateStatus("Running" if self.debug else "Failed", camelize(e))
except Exception as err:
self.job.logError(err)
self.logThis("EXCEPTION: %s", err)
if not self.debug: return 6
def __prepare(self):
try:
self.job.updateStatus("Running", "PreparingInputData", hostname=gethostname(), batchId=self.batchId)
except Exception as err:
self.logThis("EXCEPTION: %s", err)
# first, set all variables
self.logThis("current environment settings")
for key, value in sorted(environ.iteritems()):
print "%s = %s"%(key,value)
self.logThis("end of environment dump")
# log.info("\n".join(["%s: %s"%(key,value) for key, value in sorted(environ.iteritems())]))
for fi in self.job.InputFiles:
src = expandvars(fi['source'])
tg = expandvars(fi['target'])
self.logThis("Staging %s --> %s",src,oPjoin(abspath(self.pwd),tg))
try:
safe_copy(src, tg, attempts=4, sleep='4s', checksum=True)
except IOError, e:
try:
self.job.updateStatus("Running" if self.debug else "Failed", camelize(e))
except Exception as err:
self.logThis("EXCEPTION: %s", err)
self.job.logError(err)
if not self.debug: return 4
"None",
"EXEC_DIR_ROOT":
"/tmp",
"ratio_mem":
"1.0",
"ratio_cpu":
"1.0",
"logfile":
"/tmp/flask.log",
"loglevel":
"INFO"
}
cfg = SafeConfigParser(defaults=__myDefaults)
cfg.read(oPjoin(DAMPE_WORKFLOW_ROOT, "config/settings.cfg"))
assert cfg.get("global",
"installation") in ['server', 'client'
], "installation must be server or client!"
environ["DAMPE_SW_DIR"] = cfg.get("site", "DAMPE_SW_DIR")
environ["DAMPE_WORKFLOW_ROOT"] = DAMPE_WORKFLOW_ROOT
# environ["DAMPE_URL"] = cfg.get("server","url")
# print "setting up externals"
# source_bash(cfg.get("site", "ExternalsScript"))
dbg = cfg.getboolean("global", "traceback")
sys.excepthook = exceptionHandler
# Set the path
from os.path import abspath, join as oPjoin, dirname
from sys import path as sys_path
sys_path.append(abspath(oPjoin(dirname(__file__), '..')))
from flask.ext.script import Manager, Server, Shell
import DmpWorkflow.core.models as DmpWorkflowModels
from DmpWorkflow.core import app, db
from DmpWorkflow.config.defaults import cfg
def _make_context():
return dict(app=app, db=db, models=DmpWorkflowModels)
manager = Manager(app)
# Turn on debugger by default and reloader
manager.add_command(
"runserver",
Server(use_debugger=cfg.getboolean("server", "use_debugger"),
use_reloader=cfg.getboolean("server", "use_reloader"),
host=cfg.get("server", "host")))
manager.add_command("shell", Shell(make_context=_make_context))
def main():
manager.run()
if __name__ == "__main__":
# Set the path
from os.path import abspath, join as oPjoin, dirname
from sys import path as sys_path
sys_path.append(abspath(oPjoin(dirname(__file__), '..')))
from flask.ext.script import Manager, Server, Shell
import DmpWorkflow.core.models as DmpWorkflowModels
from DmpWorkflow.core import app, db
from DmpWorkflow.config.defaults import cfg
def _make_context():
return dict(app=app, db=db, models=DmpWorkflowModels)
manager = Manager(app)
# Turn on debugger by default and reloader
manager.add_command("runserver", Server(use_debugger=cfg.getboolean("server", "use_debugger"),
use_reloader=cfg.getboolean("server", "use_reloader"),
host=cfg.get("server", "host"))
)
manager.add_command("shell", Shell(make_context=_make_context))
def main():
manager.run()
if __name__ == "__main__":
main()
"HPCextra": "",
"HPCqueue": "",
"HPCcputime": "24:00",
"HPCmemory": "1000.",
"HPCusername": "******",
"HPCclustername" : "None",
"EXEC_DIR_ROOT": "/tmp",
"ratio_mem": "1.0",
"ratio_cpu": "1.0",
"logfile": "/tmp/flask.log",
"loglevel": "INFO"
}
cfg = SafeConfigParser(defaults=__myDefaults)
cfg.read(oPjoin(DAMPE_WORKFLOW_ROOT, "config/settings.cfg"))
assert cfg.get("global", "installation") in ['server', 'client'], "installation must be server or client!"
environ["DAMPE_SW_DIR"] = cfg.get("site", "DAMPE_SW_DIR")
environ["DAMPE_WORKFLOW_ROOT"] = DAMPE_WORKFLOW_ROOT
# environ["DAMPE_URL"] = cfg.get("server","url")
# print "setting up externals"
# source_bash(cfg.get("site", "ExternalsScript"))
dbg = cfg.getboolean("global", "traceback")
sys.excepthook = exceptionHandler
DAMPE_WORKFLOW_URL = cfg.get("server", "url")
DAMPE_WORKFLOW_DIR = cfg.get("site", "workdir")