def execute(self): environ["DWF_SIXDIGIT"] = self.job.getSixDigits() print 'EXEC_DIR_ROOT: %s' % EXEC_DIR_ROOT print 'instanceId : %s' % str(self.job.getSixDigits()) my_exec_dir = oPjoin(EXEC_DIR_ROOT, self.job.getSixDigits(), "local" if self.batchId == "-1" else str(self.batchId)) mkdir(my_exec_dir) chdir(my_exec_dir) self.logThis("execution directory %s", my_exec_dir) # DMPSWSYS = getenv("DMPSWSYS") # DAMPE_SW_DIR = getenv("DAMPE_SW_DIR",None) # if DAMPE_SW_DIR is None: # raise Exception("must define $DAMPE_SW_DIR") # next, run the executable # if not DAMPE_SW_DIR in DMPSWSYS: # log.info("trying to re-source setup script.") # job.sourceSetupScript() rc = 0 rc += self.__prepare() if rc: self.exit_app(rc,msg="Exiting after prepare step") rc += self.__runPayload() if rc: self.exit_app(rc,msg="Exiting after payload") rc += self.__postRun() if rc: self.exit_app(rc,msg="Exiting after post-run") # finally, compile output file. self.logThis("job complete, cleaning up working directory") chdir(self.pwd) rm(my_exec_dir) try: self.job.updateStatus("Done", "ApplicationComplete") except Exception as err: self.logThis("EXCEPTION: %s", err)
def main(args=None): parser = ArgumentParser(usage="Usage: %(prog)s [options]", description="initialize dampe workflow") parser.add_argument("-f", "--file", dest="file", type=str, default=None, help='use this flag if you plan to provide a file') opts = parser.parse_args(args) if opts.file is not None: src = opts.file dmpROOT = dirname(abspath(DmpFile)) tg = oPjoin(dmpROOT, "config/settings.cfg") safe_copy(src, tg, sleep='3s', attempts=3, debug=True) return
def __postRun(self): self.job.updateStatus("Running", "PreparingOutputData") for fi in self.job.OutputFiles: src = expandvars(fi['source']) tg = expandvars(fi['target']) self.logThis("Staging %s --> %s",oPjoin(abspath(self.pwd),src),tg) _dir = dirname(tg) try: self.logThis("creating output directory %s", _dir) try: mkdir(_dir) except IOError as err: self.logThis("error creating output directory, trying to recover, error follows: ",err) safe_copy(src, tg, attempts=4, sleep='4s', checksum=True) self.job.registerDS(filename=tg, overwrite=True) except Exception, e: try: self.job.updateStatus("Running" if self.debug else "Failed", camelize(e)) except Exception as err: self.job.logError(err) self.logThis("EXCEPTION: %s", err) if not self.debug: return 6
def __prepare(self): try: self.job.updateStatus("Running", "PreparingInputData", hostname=gethostname(), batchId=self.batchId) except Exception as err: self.logThis("EXCEPTION: %s", err) # first, set all variables self.logThis("current environment settings") for key, value in sorted(environ.iteritems()): print "%s = %s"%(key,value) self.logThis("end of environment dump") # log.info("\n".join(["%s: %s"%(key,value) for key, value in sorted(environ.iteritems())])) for fi in self.job.InputFiles: src = expandvars(fi['source']) tg = expandvars(fi['target']) self.logThis("Staging %s --> %s",src,oPjoin(abspath(self.pwd),tg)) try: safe_copy(src, tg, attempts=4, sleep='4s', checksum=True) except IOError, e: try: self.job.updateStatus("Running" if self.debug else "Failed", camelize(e)) except Exception as err: self.logThis("EXCEPTION: %s", err) self.job.logError(err) if not self.debug: return 4
"None", "EXEC_DIR_ROOT": "/tmp", "ratio_mem": "1.0", "ratio_cpu": "1.0", "logfile": "/tmp/flask.log", "loglevel": "INFO" } cfg = SafeConfigParser(defaults=__myDefaults) cfg.read(oPjoin(DAMPE_WORKFLOW_ROOT, "config/settings.cfg")) assert cfg.get("global", "installation") in ['server', 'client' ], "installation must be server or client!" environ["DAMPE_SW_DIR"] = cfg.get("site", "DAMPE_SW_DIR") environ["DAMPE_WORKFLOW_ROOT"] = DAMPE_WORKFLOW_ROOT # environ["DAMPE_URL"] = cfg.get("server","url") # print "setting up externals" # source_bash(cfg.get("site", "ExternalsScript")) dbg = cfg.getboolean("global", "traceback") sys.excepthook = exceptionHandler
# Set the path from os.path import abspath, join as oPjoin, dirname from sys import path as sys_path sys_path.append(abspath(oPjoin(dirname(__file__), '..'))) from flask.ext.script import Manager, Server, Shell import DmpWorkflow.core.models as DmpWorkflowModels from DmpWorkflow.core import app, db from DmpWorkflow.config.defaults import cfg def _make_context(): return dict(app=app, db=db, models=DmpWorkflowModels) manager = Manager(app) # Turn on debugger by default and reloader manager.add_command( "runserver", Server(use_debugger=cfg.getboolean("server", "use_debugger"), use_reloader=cfg.getboolean("server", "use_reloader"), host=cfg.get("server", "host"))) manager.add_command("shell", Shell(make_context=_make_context)) def main(): manager.run() if __name__ == "__main__":
# Set the path from os.path import abspath, join as oPjoin, dirname from sys import path as sys_path sys_path.append(abspath(oPjoin(dirname(__file__), '..'))) from flask.ext.script import Manager, Server, Shell import DmpWorkflow.core.models as DmpWorkflowModels from DmpWorkflow.core import app, db from DmpWorkflow.config.defaults import cfg def _make_context(): return dict(app=app, db=db, models=DmpWorkflowModels) manager = Manager(app) # Turn on debugger by default and reloader manager.add_command("runserver", Server(use_debugger=cfg.getboolean("server", "use_debugger"), use_reloader=cfg.getboolean("server", "use_reloader"), host=cfg.get("server", "host")) ) manager.add_command("shell", Shell(make_context=_make_context)) def main(): manager.run() if __name__ == "__main__": main()
"HPCextra": "", "HPCqueue": "", "HPCcputime": "24:00", "HPCmemory": "1000.", "HPCusername": "******", "HPCclustername" : "None", "EXEC_DIR_ROOT": "/tmp", "ratio_mem": "1.0", "ratio_cpu": "1.0", "logfile": "/tmp/flask.log", "loglevel": "INFO" } cfg = SafeConfigParser(defaults=__myDefaults) cfg.read(oPjoin(DAMPE_WORKFLOW_ROOT, "config/settings.cfg")) assert cfg.get("global", "installation") in ['server', 'client'], "installation must be server or client!" environ["DAMPE_SW_DIR"] = cfg.get("site", "DAMPE_SW_DIR") environ["DAMPE_WORKFLOW_ROOT"] = DAMPE_WORKFLOW_ROOT # environ["DAMPE_URL"] = cfg.get("server","url") # print "setting up externals" # source_bash(cfg.get("site", "ExternalsScript")) dbg = cfg.getboolean("global", "traceback") sys.excepthook = exceptionHandler DAMPE_WORKFLOW_URL = cfg.get("server", "url") DAMPE_WORKFLOW_DIR = cfg.get("site", "workdir")