def report_crash(node, traceback=None, hostname=None): """Writes crash related information to a file """ name = node._id if node.result and hasattr(node.result, 'runtime') and \ node.result.runtime: if isinstance(node.result.runtime, list): host = node.result.runtime[0].hostname else: host = node.result.runtime.hostname else: if hostname: host = hostname else: host = gethostname() message = ['Node %s failed to run on host %s.' % (name, host)] logger.error(message) if not traceback: exc_type, exc_value, exc_traceback = sys.exc_info() traceback = format_exception(exc_type, exc_value, exc_traceback) timeofcrash = strftime('%Y%m%d-%H%M%S') login_name = pwd.getpwuid(os.geteuid())[0] crashfile = 'crash-%s-%s-%s.pklz' % (timeofcrash, login_name, name) crashdir = node.config['execution']['crashdump_dir'] if crashdir is None: crashdir = os.getcwd() if not os.path.exists(crashdir): os.makedirs(crashdir) crashfile = os.path.join(crashdir, crashfile) logger.info('Saving crash info to %s' % crashfile) logger.info(''.join(traceback)) savepkl(crashfile, dict(node=node, traceback=traceback)) #np.savez(crashfile, node=node, traceback=traceback) return crashfile
def report_crash(node, traceback=None, hostname=None): """Writes crash related information to a file """ name = node._id if node.result and hasattr(node.result, "runtime") and node.result.runtime: if isinstance(node.result.runtime, list): host = node.result.runtime[0].hostname else: host = node.result.runtime.hostname else: if hostname: host = hostname else: host = gethostname() message = ["Node %s failed to run on host %s." % (name, host)] logger.error(message) if not traceback: exc_type, exc_value, exc_traceback = sys.exc_info() traceback = format_exception(exc_type, exc_value, exc_traceback) timeofcrash = strftime("%Y%m%d-%H%M%S") login_name = getpass.getuser() crashfile = "crash-%s-%s-%s.pklz" % (timeofcrash, login_name, name) crashdir = node.config["execution"]["crashdump_dir"] if crashdir is None: crashdir = os.getcwd() if not os.path.exists(crashdir): os.makedirs(crashdir) crashfile = os.path.join(crashdir, crashfile) logger.info("Saving crash info to %s" % crashfile) logger.info("".join(traceback)) savepkl(crashfile, dict(node=node, traceback=traceback)) # np.savez(crashfile, node=node, traceback=traceback) return crashfile
def corr_to_zscore(in_file, distrib): import os import nipype.utils.filemanip as fmanip from nipype.utils.filemanip import fname_presuffix corrs = fmanip.loadpkl(in_file) dist = fmanip.loadpkl(distrib) zscore = (corrs["corr"] - dist[0]["mean"]) / dist[0]["std"] zfile = os.path.abspath(fname_presuffix(in_file, suffix="_zscore", newpath=os.getcwd())) corrs["zscore"] = zscore fmanip.savepkl(zfile, corrs) return zfile
def corr_to_zscore(in_file, distrib): import os import nipype.utils.filemanip as fmanip from nipype.utils.filemanip import fname_presuffix corrs = fmanip.loadpkl(in_file) dist = fmanip.loadpkl(distrib) zscore = (corrs['corr'] - dist[0]['mean']) / dist[0]['std'] zfile = os.path.abspath( fname_presuffix(in_file, suffix='_zscore', newpath=os.getcwd())) corrs['zscore'] = zscore fmanip.savepkl(zfile, corrs) return zfile
def _submit_job(self, node, updatehash=False): # use qsub to submit job and return sgeid # pickle node timestamp = strftime("%Y%m%d_%H%M%S") suffix = "%s_%s" % (timestamp, node._id) sge_dir = os.path.join(node.base_dir, "sge") if not os.path.exists(sge_dir): os.makedirs(sge_dir) pkl_file = os.path.join(sge_dir, "node_%s.pklz" % suffix) savepkl(pkl_file, dict(node=node, updatehash=updatehash)) # create python script to load and trap exception cmdstr = ( """import os import sys from traceback import format_exception from nipype.utils.filemanip import loadpkl, savepkl traceback=None print os.getcwd() try: info = loadpkl('%s') result = info['node'].run(updatehash=info['updatehash']) except: etype, eval, etr = sys.exc_info() traceback = format_exception(etype,eval,etr) result = info['node'].result resultsfile = os.path.join(node.output_dir(), 'result_%%s.pklz'%%info['node'].name) savepkl(resultsfile,dict(result=result, traceback=traceback)) """ % pkl_file ) pyscript = os.path.join(sge_dir, "pyscript_%s.py" % suffix) fp = open(pyscript, "wt") fp.writelines(cmdstr) fp.close() sgescript = "\n".join((self._template, "python %s" % pyscript)) sgescriptfile = os.path.join(sge_dir, "sgescript_%s.sh" % suffix) fp = open(sgescriptfile, "wt") fp.writelines(sgescript) fp.close() cmd = CommandLine("qsub", environ=os.environ.data) qsubargs = "" if self._qsub_args: qsubargs = self._qsub_args cmd.inputs.args = "%s %s" % (qsubargs, sgescriptfile) result = cmd.run() # retrieve sge taskid if not result.runtime.returncode: taskid = int(result.runtime.stdout.split(" ")[2]) self._pending[taskid] = node.output_dir() logger.debug("submitted sge task: %d for node %s" % (taskid, node._id)) else: raise RuntimeError("\n".join(("Could not submit sge task for node %s" % node._id, result.runtime.stderr))) return taskid
def _submit_job(self, node, updatehash=False): # use qsub to submit job and return sgeid # pickle node timestamp = strftime('%Y%m%d_%H%M%S') suffix = '%s_%s'%(timestamp, node._id) sge_dir = os.path.join(node.base_dir, 'sge') if not os.path.exists(sge_dir): os.makedirs(sge_dir) pkl_file = os.path.join(sge_dir,'node_%s.pklz'%suffix) savepkl(pkl_file, dict(node=node, updatehash=updatehash)) # create python script to load and trap exception cmdstr = """import os import sys from traceback import format_exception from nipype.utils.filemanip import loadpkl, savepkl traceback=None print os.getcwd() try: info = loadpkl('%s') result = info['node'].run(updatehash=info['updatehash']) except: etype, eval, etr = sys.exc_info() traceback = format_exception(etype,eval,etr) result = info['node'].result resultsfile = os.path.join(node.output_dir(), 'result_%%s.pklz'%%info['node'].name) savepkl(resultsfile,dict(result=result, traceback=traceback)) """%pkl_file pyscript = os.path.join(sge_dir, 'pyscript_%s.py'%suffix) fp = open(pyscript, 'wt') fp.writelines(cmdstr) fp.close() sgescript = """#!/usr/bin/env bash python %s """%(pyscript) sgescriptfile = os.path.join(sge_dir, 'sgescript_%s.sh'%suffix) fp = open(sgescriptfile, 'wt') fp.writelines(sgescript) fp.close() cmd = CommandLine('qsub', environ=os.environ.data) qsubargs = '' if node.config.has_key('sgeargs'): qsubargs = node.config['sgeargs'] cmd.inputs.args = '%s %s'%(qsubargs, sgescriptfile) result = cmd.run() # retrieve sge taskid if not result.runtime.returncode: taskid = int(result.runtime.stdout.split(' ')[2]) self._pending[taskid] = node.output_dir() logger.debug('submitted sge task: %d for node %s'%(taskid, node._id)) return taskid
def _submit_job(self, node, updatehash=False): """submit job and return taskid """ # pickle node timestamp = strftime('%Y%m%d_%H%M%S') if node._hierarchy: suffix = '%s_%s_%s'%(timestamp, node._hierarchy, node._id) batch_dir = os.path.join(node.base_dir, node._hierarchy.split('.')[0], 'batch') else: suffix = '%s_%s'%(timestamp, node._id) batch_dir = os.path.join(node.base_dir, 'batch') if not os.path.exists(batch_dir): os.makedirs(batch_dir) pkl_file = os.path.join(batch_dir, 'node_%s.pklz' % suffix) savepkl(pkl_file, dict(node=node, updatehash=updatehash)) # create python script to load and trap exception cmdstr = """import os import sys from socket import gethostname from traceback import format_exception from nipype.utils.filemanip import loadpkl, savepkl traceback=None print os.getcwd() try: info = loadpkl('%s') result = info['node'].run(updatehash=info['updatehash']) except: etype, eval, etr = sys.exc_info() traceback = format_exception(etype,eval,etr) result = info['node'].result resultsfile = os.path.join(info['node'].output_dir(), 'result_%%s.pklz'%%info['node'].name) savepkl(resultsfile,dict(result=result, hostname=gethostname(), traceback=traceback)) """ % pkl_file pyscript = os.path.join(batch_dir, 'pyscript_%s.py' % suffix) fp = open(pyscript, 'wt') fp.writelines(cmdstr) fp.close() batchscript = '\n'.join((self._template, '%s %s' % (sys.executable, pyscript))) batchscriptfile = os.path.join(batch_dir, 'batchscript_%s.sh' % suffix) fp = open(batchscriptfile, 'wt') fp.writelines(batchscript) fp.close() return self._submit_batchtask(batchscriptfile, node)
def create_pyscript(node, updatehash=False, store_exception=True): # pickle node timestamp = strftime('%Y%m%d_%H%M%S') if node._hierarchy: suffix = '%s_%s_%s' % (timestamp, node._hierarchy, node._id) batch_dir = os.path.join(node.base_dir, node._hierarchy.split('.')[0], 'batch') else: suffix = '%s_%s' % (timestamp, node._id) batch_dir = os.path.join(node.base_dir, 'batch') if not os.path.exists(batch_dir): os.makedirs(batch_dir) pkl_file = os.path.join(batch_dir, 'node_%s.pklz' % suffix) savepkl(pkl_file, dict(node=node, updatehash=updatehash)) mpl_backend = node.config["execution"]["matplotlib_backend"] # create python script to load and trap exception cmdstr = """import os import sys try: import matplotlib matplotlib.use('%s') except ImportError: pass from nipype import config, logging from nipype.utils.filemanip import loadpkl, savepkl from socket import gethostname from traceback import format_exception info = None pklfile = '%s' batchdir = '%s' from nipype.utils.filemanip import loadpkl, savepkl try: if not sys.version_info < (2, 7): from collections import OrderedDict config_dict=%s config.update_config(config_dict) config.update_matplotlib() logging.update_logging(config) traceback=None cwd = os.getcwd() info = loadpkl(pklfile) result = info['node'].run(updatehash=info['updatehash']) except Exception, e: etype, eval, etr = sys.exc_info() traceback = format_exception(etype,eval,etr) if info is None or not os.path.exists(info['node'].output_dir()): result = None resultsfile = os.path.join(batchdir, 'crashdump_%s.pklz') else: result = info['node'].result resultsfile = os.path.join(info['node'].output_dir(), 'result_%%s.pklz'%%info['node'].name) """ if store_exception: cmdstr += """ savepkl(resultsfile, dict(result=result, hostname=gethostname(), traceback=traceback)) """ else: cmdstr += """ if info is None: savepkl(resultsfile, dict(result=result, hostname=gethostname(), traceback=traceback)) else: from nipype.pipeline.plugins.base import report_crash report_crash(info['node'], traceback, gethostname()) raise Exception(e) """ cmdstr = cmdstr % (mpl_backend, pkl_file, batch_dir, node.config, suffix) pyscript = os.path.join(batch_dir, 'pyscript_%s.py' % suffix) fp = open(pyscript, 'wt') fp.writelines(cmdstr) fp.close() return pyscript
def create_pyscript(node, updatehash=False, store_exception=True): # pickle node timestamp = strftime('%Y%m%d_%H%M%S') if node._hierarchy: suffix = '%s_%s_%s' % (timestamp, node._hierarchy, node._id) batch_dir = os.path.join(node.base_dir, node._hierarchy.split('.')[0], 'batch') else: suffix = '%s_%s' % (timestamp, node._id) batch_dir = os.path.join(node.base_dir, 'batch') if not os.path.exists(batch_dir): os.makedirs(batch_dir) pkl_file = os.path.join(batch_dir, 'node_%s.pklz' % suffix) savepkl(pkl_file, dict(node=node, updatehash=updatehash)) # create python script to load and trap exception cmdstr = """import os import sys from socket import gethostname from traceback import format_exception info = None pklfile = '%s' batchdir = '%s' try: from nipype import config, logging import sys if not sys.version_info < (2, 7): from ordereddict import OrderedDict config_dict=%s config.update_config(config_dict) config.update_matplotlib() logging.update_logging(config) from nipype.utils.filemanip import loadpkl, savepkl traceback=None cwd = os.getcwd() info = loadpkl(pklfile) result = info['node'].run(updatehash=info['updatehash']) except Exception, e: etype, eval, etr = sys.exc_info() traceback = format_exception(etype,eval,etr) if info is None: result = None resultsfile = os.path.join(batchdir, 'crashdump_%s.pklz') else: result = info['node'].result resultsfile = os.path.join(info['node'].output_dir(), 'result_%%s.pklz'%%info['node'].name) """ if store_exception: cmdstr += """ savepkl(resultsfile, dict(result=result, hostname=gethostname(), traceback=traceback)) """ else: cmdstr += """ if info is None: savepkl(resultsfile, dict(result=result, hostname=gethostname(), traceback=traceback)) else: from nipype.pipeline.plugins.base import report_crash report_crash(info['node'], traceback, gethostname()) raise Exception(e) """ cmdstr = cmdstr % (pkl_file, batch_dir, node.config, suffix) pyscript = os.path.join(batch_dir, 'pyscript_%s.py' % suffix) fp = open(pyscript, 'wt') fp.writelines(cmdstr) fp.close() return pyscript
def _submit_job(self, node, updatehash=False): """submit job and return taskid """ # pickle node timestamp = strftime('%Y%m%d_%H%M%S') if node._hierarchy: suffix = '%s_%s_%s' % (timestamp, node._hierarchy, node._id) batch_dir = os.path.join(node.base_dir, node._hierarchy.split('.')[0], 'batch') else: suffix = '%s_%s' % (timestamp, node._id) batch_dir = os.path.join(node.base_dir, 'batch') if not os.path.exists(batch_dir): os.makedirs(batch_dir) pkl_file = os.path.join(batch_dir, 'node_%s.pklz' % suffix) savepkl(pkl_file, dict(node=node, updatehash=updatehash)) # create python script to load and trap exception cmdstr = """import os import sys from socket import gethostname from traceback import format_exception from nipype import config, logging config_dict=%s config.update_config(config_dict) logging.update_logging(config) from nipype.utils.filemanip import loadpkl, savepkl """ does_plot = (isinstance(node, Function) and "matplotlib" in node.inputs.function_str) if does_plot: cmdstr += "import matplotlib\n" cmdstr += "matplotlib.use('Agg')\n" cmdstr += """ traceback=None cwd = os.getcwd() print cwd pklfile = '%s' batchdir = '%s' info = None try: info = loadpkl(pklfile) result = info['node'].run(updatehash=info['updatehash']) except: etype, eval, etr = sys.exc_info() traceback = format_exception(etype,eval,etr) if info is None: result = None resultsfile = os.path.join(batchdir, 'crashdump_%s.pklz') else: result = info['node'].result resultsfile = os.path.join(info['node'].output_dir(), 'result_%%s.pklz'%%info['node'].name) savepkl(resultsfile, dict(result=result, hostname=gethostname(), traceback=traceback)) """ cmdstr = cmdstr % (node.config, pkl_file, batch_dir, suffix) pyscript = os.path.join(batch_dir, 'pyscript_%s.py' % suffix) fp = open(pyscript, 'wt') fp.writelines(cmdstr) fp.close() batchscript = '\n'.join((self._template, '%s %s' % (sys.executable, pyscript))) batchscriptfile = os.path.join(batch_dir, 'batchscript_%s.sh' % suffix) fp = open(batchscriptfile, 'wt') fp.writelines(batchscript) fp.close() return self._submit_batchtask(batchscriptfile, node)
def create_pyscript(node, updatehash=False, store_exception=True): # pickle node timestamp = strftime('%Y%m%d_%H%M%S') if node._hierarchy: suffix = '%s_%s_%s' % (timestamp, node._hierarchy, node._id) batch_dir = os.path.join(node.base_dir, node._hierarchy.split('.')[0], 'batch') else: suffix = '%s_%s' % (timestamp, node._id) batch_dir = os.path.join(node.base_dir, 'batch') if not os.path.exists(batch_dir): os.makedirs(batch_dir) pkl_file = os.path.join(batch_dir, 'node_%s.pklz' % suffix) savepkl(pkl_file, dict(node=node, updatehash=updatehash)) mpl_backend = node.config["execution"]["matplotlib_backend"] # create python script to load and trap exception cmdstr = """import os import sys can_import_matplotlib = True #Silently allow matplotlib to be ignored try: import matplotlib matplotlib.use('%s') except ImportError: can_import_matplotlib = False pass from nipype import config, logging from nipype.utils.filemanip import loadpkl, savepkl from socket import gethostname from traceback import format_exception info = None pklfile = '%s' batchdir = '%s' from nipype.utils.filemanip import loadpkl, savepkl try: if not sys.version_info < (2, 7): from collections import OrderedDict config_dict=%s config.update_config(config_dict) ## Only configure matplotlib if it was successfully imported, matplotlib is an optional component to nipype if can_import_matplotlib: config.update_matplotlib() logging.update_logging(config) traceback=None cwd = os.getcwd() info = loadpkl(pklfile) result = info['node'].run(updatehash=info['updatehash']) except Exception, e: etype, eval, etr = sys.exc_info() traceback = format_exception(etype,eval,etr) if info is None or not os.path.exists(info['node'].output_dir()): result = None resultsfile = os.path.join(batchdir, 'crashdump_%s.pklz') else: result = info['node'].result resultsfile = os.path.join(info['node'].output_dir(), 'result_%%s.pklz'%%info['node'].name) """ if store_exception: cmdstr += """ savepkl(resultsfile, dict(result=result, hostname=gethostname(), traceback=traceback)) """ else: cmdstr += """ if info is None: savepkl(resultsfile, dict(result=result, hostname=gethostname(), traceback=traceback)) else: from nipype.pipeline.plugins.base import report_crash report_crash(info['node'], traceback, gethostname()) raise Exception(e) """ cmdstr = cmdstr % (mpl_backend, pkl_file, batch_dir, node.config, suffix) pyscript = os.path.join(batch_dir, 'pyscript_%s.py' % suffix) fp = open(pyscript, 'wt') fp.writelines(cmdstr) fp.close() return pyscript