def report_crash(node, traceback=None, hostname=None):
"""Writes crash related information to a file
"""
name = node._id
if node.result and hasattr(node.result, 'runtime') and \
node.result.runtime:
if isinstance(node.result.runtime, list):
host = node.result.runtime[0].hostname
else:
host = node.result.runtime.hostname
else:
if hostname:
host = hostname
else:
host = gethostname()
message = ['Node %s failed to run on host %s.' % (name, host)]
logger.error(message)
if not traceback:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback = format_exception(exc_type, exc_value, exc_traceback)
timeofcrash = strftime('%Y%m%d-%H%M%S')
login_name = pwd.getpwuid(os.geteuid())[0]
crashfile = 'crash-%s-%s-%s.pklz' % (timeofcrash, login_name, name)
crashdir = node.config['execution']['crashdump_dir']
if crashdir is None:
crashdir = os.getcwd()
if not os.path.exists(crashdir):
os.makedirs(crashdir)
crashfile = os.path.join(crashdir, crashfile)
logger.info('Saving crash info to %s' % crashfile)
logger.info(''.join(traceback))
savepkl(crashfile, dict(node=node, traceback=traceback))
#np.savez(crashfile, node=node, traceback=traceback)
return crashfile
def report_crash(node, traceback=None, hostname=None):
"""Writes crash related information to a file
"""
name = node._id
if node.result and hasattr(node.result, "runtime") and node.result.runtime:
if isinstance(node.result.runtime, list):
host = node.result.runtime[0].hostname
else:
host = node.result.runtime.hostname
else:
if hostname:
host = hostname
else:
host = gethostname()
message = ["Node %s failed to run on host %s." % (name, host)]
logger.error(message)
if not traceback:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback = format_exception(exc_type, exc_value, exc_traceback)
timeofcrash = strftime("%Y%m%d-%H%M%S")
login_name = getpass.getuser()
crashfile = "crash-%s-%s-%s.pklz" % (timeofcrash, login_name, name)
crashdir = node.config["execution"]["crashdump_dir"]
if crashdir is None:
crashdir = os.getcwd()
if not os.path.exists(crashdir):
os.makedirs(crashdir)
crashfile = os.path.join(crashdir, crashfile)
logger.info("Saving crash info to %s" % crashfile)
logger.info("".join(traceback))
savepkl(crashfile, dict(node=node, traceback=traceback))
# np.savez(crashfile, node=node, traceback=traceback)
return crashfile
def corr_to_zscore(in_file, distrib):
import os
import nipype.utils.filemanip as fmanip
from nipype.utils.filemanip import fname_presuffix
corrs = fmanip.loadpkl(in_file)
dist = fmanip.loadpkl(distrib)
zscore = (corrs["corr"] - dist[0]["mean"]) / dist[0]["std"]
zfile = os.path.abspath(fname_presuffix(in_file, suffix="_zscore", newpath=os.getcwd()))
corrs["zscore"] = zscore
fmanip.savepkl(zfile, corrs)
return zfile
def corr_to_zscore(in_file, distrib):
import os
import nipype.utils.filemanip as fmanip
from nipype.utils.filemanip import fname_presuffix
corrs = fmanip.loadpkl(in_file)
dist = fmanip.loadpkl(distrib)
zscore = (corrs['corr'] - dist[0]['mean']) / dist[0]['std']
zfile = os.path.abspath(
fname_presuffix(in_file, suffix='_zscore', newpath=os.getcwd()))
corrs['zscore'] = zscore
fmanip.savepkl(zfile, corrs)
return zfile
def _submit_job(self, node, updatehash=False):
# use qsub to submit job and return sgeid
# pickle node
timestamp = strftime("%Y%m%d_%H%M%S")
suffix = "%s_%s" % (timestamp, node._id)
sge_dir = os.path.join(node.base_dir, "sge")
if not os.path.exists(sge_dir):
os.makedirs(sge_dir)
pkl_file = os.path.join(sge_dir, "node_%s.pklz" % suffix)
savepkl(pkl_file, dict(node=node, updatehash=updatehash))
# create python script to load and trap exception
cmdstr = (
"""import os
import sys
from traceback import format_exception
from nipype.utils.filemanip import loadpkl, savepkl
traceback=None
print os.getcwd()
try:
info = loadpkl('%s')
result = info['node'].run(updatehash=info['updatehash'])
except:
etype, eval, etr = sys.exc_info()
traceback = format_exception(etype,eval,etr)
result = info['node'].result
resultsfile = os.path.join(node.output_dir(), 'result_%%s.pklz'%%info['node'].name)
savepkl(resultsfile,dict(result=result, traceback=traceback))
"""
% pkl_file
)
pyscript = os.path.join(sge_dir, "pyscript_%s.py" % suffix)
fp = open(pyscript, "wt")
fp.writelines(cmdstr)
fp.close()
sgescript = "\n".join((self._template, "python %s" % pyscript))
sgescriptfile = os.path.join(sge_dir, "sgescript_%s.sh" % suffix)
fp = open(sgescriptfile, "wt")
fp.writelines(sgescript)
fp.close()
cmd = CommandLine("qsub", environ=os.environ.data)
qsubargs = ""
if self._qsub_args:
qsubargs = self._qsub_args
cmd.inputs.args = "%s %s" % (qsubargs, sgescriptfile)
result = cmd.run()
# retrieve sge taskid
if not result.runtime.returncode:
taskid = int(result.runtime.stdout.split(" ")[2])
self._pending[taskid] = node.output_dir()
logger.debug("submitted sge task: %d for node %s" % (taskid, node._id))
else:
raise RuntimeError("\n".join(("Could not submit sge task for node %s" % node._id, result.runtime.stderr)))
return taskid
def _submit_job(self, node, updatehash=False):
# use qsub to submit job and return sgeid
# pickle node
timestamp = strftime('%Y%m%d_%H%M%S')
suffix = '%s_%s'%(timestamp, node._id)
sge_dir = os.path.join(node.base_dir, 'sge')
if not os.path.exists(sge_dir):
os.makedirs(sge_dir)
pkl_file = os.path.join(sge_dir,'node_%s.pklz'%suffix)
savepkl(pkl_file, dict(node=node, updatehash=updatehash))
# create python script to load and trap exception
cmdstr = """import os
import sys
from traceback import format_exception
from nipype.utils.filemanip import loadpkl, savepkl
traceback=None
print os.getcwd()
try:
info = loadpkl('%s')
result = info['node'].run(updatehash=info['updatehash'])
except:
etype, eval, etr = sys.exc_info()
traceback = format_exception(etype,eval,etr)
result = info['node'].result
resultsfile = os.path.join(node.output_dir(), 'result_%%s.pklz'%%info['node'].name)
savepkl(resultsfile,dict(result=result, traceback=traceback))
"""%pkl_file
pyscript = os.path.join(sge_dir, 'pyscript_%s.py'%suffix)
fp = open(pyscript, 'wt')
fp.writelines(cmdstr)
fp.close()
sgescript = """#!/usr/bin/env bash
python %s
"""%(pyscript)
sgescriptfile = os.path.join(sge_dir, 'sgescript_%s.sh'%suffix)
fp = open(sgescriptfile, 'wt')
fp.writelines(sgescript)
fp.close()
cmd = CommandLine('qsub', environ=os.environ.data)
qsubargs = ''
if node.config.has_key('sgeargs'):
qsubargs = node.config['sgeargs']
cmd.inputs.args = '%s %s'%(qsubargs, sgescriptfile)
result = cmd.run()
# retrieve sge taskid
if not result.runtime.returncode:
taskid = int(result.runtime.stdout.split(' ')[2])
self._pending[taskid] = node.output_dir()
logger.debug('submitted sge task: %d for node %s'%(taskid, node._id))
return taskid
def _submit_job(self, node, updatehash=False):
"""submit job and return taskid
"""
# pickle node
timestamp = strftime('%Y%m%d_%H%M%S')
if node._hierarchy:
suffix = '%s_%s_%s'%(timestamp, node._hierarchy, node._id)
batch_dir = os.path.join(node.base_dir, node._hierarchy.split('.')[0], 'batch')
else:
suffix = '%s_%s'%(timestamp, node._id)
batch_dir = os.path.join(node.base_dir, 'batch')
if not os.path.exists(batch_dir):
os.makedirs(batch_dir)
pkl_file = os.path.join(batch_dir, 'node_%s.pklz' % suffix)
savepkl(pkl_file, dict(node=node, updatehash=updatehash))
# create python script to load and trap exception
cmdstr = """import os
import sys
from socket import gethostname
from traceback import format_exception
from nipype.utils.filemanip import loadpkl, savepkl
traceback=None
print os.getcwd()
try:
info = loadpkl('%s')
result = info['node'].run(updatehash=info['updatehash'])
except:
etype, eval, etr = sys.exc_info()
traceback = format_exception(etype,eval,etr)
result = info['node'].result
resultsfile = os.path.join(info['node'].output_dir(),
'result_%%s.pklz'%%info['node'].name)
savepkl(resultsfile,dict(result=result, hostname=gethostname(),
traceback=traceback))
""" % pkl_file
pyscript = os.path.join(batch_dir, 'pyscript_%s.py' % suffix)
fp = open(pyscript, 'wt')
fp.writelines(cmdstr)
fp.close()
batchscript = '\n'.join((self._template,
'%s %s' % (sys.executable, pyscript)))
batchscriptfile = os.path.join(batch_dir, 'batchscript_%s.sh' % suffix)
fp = open(batchscriptfile, 'wt')
fp.writelines(batchscript)
fp.close()
return self._submit_batchtask(batchscriptfile, node)
def report_crash(node, traceback=None, hostname=None):
"""Writes crash related information to a file
"""
name = node._id
if node.result and hasattr(node.result, 'runtime') and \
node.result.runtime:
if isinstance(node.result.runtime, list):
host = node.result.runtime[0].hostname
else:
host = node.result.runtime.hostname
else:
if hostname:
host = hostname
else:
host = gethostname()
message = ['Node %s failed to run on host %s.' % (name,
host)]
logger.error(message)
if not traceback:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback = format_exception(exc_type,
exc_value,
exc_traceback)
timeofcrash = strftime('%Y%m%d-%H%M%S')
login_name = pwd.getpwuid(os.geteuid())[0]
crashfile = 'crash-%s-%s-%s.pklz' % (timeofcrash,
login_name,
name)
crashdir = node.config['execution']['crashdump_dir']
if crashdir is None:
crashdir = os.getcwd()
if not os.path.exists(crashdir):
os.makedirs(crashdir)
crashfile = os.path.join(crashdir, crashfile)
logger.info('Saving crash info to %s' % crashfile)
logger.info(''.join(traceback))
savepkl(crashfile, dict(node=node, traceback=traceback))
#np.savez(crashfile, node=node, traceback=traceback)
return crashfile
def create_pyscript(node, updatehash=False, store_exception=True):
# pickle node
timestamp = strftime('%Y%m%d_%H%M%S')
if node._hierarchy:
suffix = '%s_%s_%s' % (timestamp, node._hierarchy, node._id)
batch_dir = os.path.join(node.base_dir,
node._hierarchy.split('.')[0],
'batch')
else:
suffix = '%s_%s' % (timestamp, node._id)
batch_dir = os.path.join(node.base_dir, 'batch')
if not os.path.exists(batch_dir):
os.makedirs(batch_dir)
pkl_file = os.path.join(batch_dir, 'node_%s.pklz' % suffix)
savepkl(pkl_file, dict(node=node, updatehash=updatehash))
mpl_backend = node.config["execution"]["matplotlib_backend"]
# create python script to load and trap exception
cmdstr = """import os
import sys
try:
import matplotlib
matplotlib.use('%s')
except ImportError:
pass
from nipype import config, logging
from nipype.utils.filemanip import loadpkl, savepkl
from socket import gethostname
from traceback import format_exception
info = None
pklfile = '%s'
batchdir = '%s'
from nipype.utils.filemanip import loadpkl, savepkl
try:
if not sys.version_info < (2, 7):
from collections import OrderedDict
config_dict=%s
config.update_config(config_dict)
config.update_matplotlib()
logging.update_logging(config)
traceback=None
cwd = os.getcwd()
info = loadpkl(pklfile)
result = info['node'].run(updatehash=info['updatehash'])
except Exception, e:
etype, eval, etr = sys.exc_info()
traceback = format_exception(etype,eval,etr)
if info is None or not os.path.exists(info['node'].output_dir()):
result = None
resultsfile = os.path.join(batchdir, 'crashdump_%s.pklz')
else:
result = info['node'].result
resultsfile = os.path.join(info['node'].output_dir(),
'result_%%s.pklz'%%info['node'].name)
"""
if store_exception:
cmdstr += """
savepkl(resultsfile, dict(result=result, hostname=gethostname(),
traceback=traceback))
"""
else:
cmdstr += """
if info is None:
savepkl(resultsfile, dict(result=result, hostname=gethostname(),
traceback=traceback))
else:
from nipype.pipeline.plugins.base import report_crash
report_crash(info['node'], traceback, gethostname())
raise Exception(e)
"""
cmdstr = cmdstr % (mpl_backend, pkl_file, batch_dir, node.config, suffix)
pyscript = os.path.join(batch_dir, 'pyscript_%s.py' % suffix)
fp = open(pyscript, 'wt')
fp.writelines(cmdstr)
fp.close()
return pyscript
def create_pyscript(node, updatehash=False, store_exception=True):
# pickle node
timestamp = strftime('%Y%m%d_%H%M%S')
if node._hierarchy:
suffix = '%s_%s_%s' % (timestamp, node._hierarchy, node._id)
batch_dir = os.path.join(node.base_dir,
node._hierarchy.split('.')[0],
'batch')
else:
suffix = '%s_%s' % (timestamp, node._id)
batch_dir = os.path.join(node.base_dir, 'batch')
if not os.path.exists(batch_dir):
os.makedirs(batch_dir)
pkl_file = os.path.join(batch_dir, 'node_%s.pklz' % suffix)
savepkl(pkl_file, dict(node=node, updatehash=updatehash))
# create python script to load and trap exception
cmdstr = """import os
import sys
from socket import gethostname
from traceback import format_exception
info = None
pklfile = '%s'
batchdir = '%s'
try:
from nipype import config, logging
import sys
if not sys.version_info < (2, 7):
from ordereddict import OrderedDict
config_dict=%s
config.update_config(config_dict)
config.update_matplotlib()
logging.update_logging(config)
from nipype.utils.filemanip import loadpkl, savepkl
traceback=None
cwd = os.getcwd()
info = loadpkl(pklfile)
result = info['node'].run(updatehash=info['updatehash'])
except Exception, e:
etype, eval, etr = sys.exc_info()
traceback = format_exception(etype,eval,etr)
if info is None:
result = None
resultsfile = os.path.join(batchdir, 'crashdump_%s.pklz')
else:
result = info['node'].result
resultsfile = os.path.join(info['node'].output_dir(),
'result_%%s.pklz'%%info['node'].name)
"""
if store_exception:
cmdstr += """
savepkl(resultsfile, dict(result=result, hostname=gethostname(),
traceback=traceback))
"""
else:
cmdstr += """
if info is None:
savepkl(resultsfile, dict(result=result, hostname=gethostname(),
traceback=traceback))
else:
from nipype.pipeline.plugins.base import report_crash
report_crash(info['node'], traceback, gethostname())
raise Exception(e)
"""
cmdstr = cmdstr % (pkl_file, batch_dir, node.config, suffix)
pyscript = os.path.join(batch_dir, 'pyscript_%s.py' % suffix)
fp = open(pyscript, 'wt')
fp.writelines(cmdstr)
fp.close()
return pyscript
def _submit_job(self, node, updatehash=False):
"""submit job and return taskid
"""
# pickle node
timestamp = strftime('%Y%m%d_%H%M%S')
if node._hierarchy:
suffix = '%s_%s_%s' % (timestamp, node._hierarchy, node._id)
batch_dir = os.path.join(node.base_dir,
node._hierarchy.split('.')[0],
'batch')
else:
suffix = '%s_%s' % (timestamp, node._id)
batch_dir = os.path.join(node.base_dir, 'batch')
if not os.path.exists(batch_dir):
os.makedirs(batch_dir)
pkl_file = os.path.join(batch_dir, 'node_%s.pklz' % suffix)
savepkl(pkl_file, dict(node=node, updatehash=updatehash))
# create python script to load and trap exception
cmdstr = """import os
import sys
from socket import gethostname
from traceback import format_exception
from nipype import config, logging
config_dict=%s
config.update_config(config_dict)
logging.update_logging(config)
from nipype.utils.filemanip import loadpkl, savepkl
"""
does_plot = (isinstance(node, Function) and
"matplotlib" in node.inputs.function_str)
if does_plot:
cmdstr += "import matplotlib\n"
cmdstr += "matplotlib.use('Agg')\n"
cmdstr += """
traceback=None
cwd = os.getcwd()
print cwd
pklfile = '%s'
batchdir = '%s'
info = None
try:
info = loadpkl(pklfile)
result = info['node'].run(updatehash=info['updatehash'])
except:
etype, eval, etr = sys.exc_info()
traceback = format_exception(etype,eval,etr)
if info is None:
result = None
resultsfile = os.path.join(batchdir, 'crashdump_%s.pklz')
else:
result = info['node'].result
resultsfile = os.path.join(info['node'].output_dir(),
'result_%%s.pklz'%%info['node'].name)
savepkl(resultsfile, dict(result=result, hostname=gethostname(),
traceback=traceback))
"""
cmdstr = cmdstr % (node.config, pkl_file, batch_dir, suffix)
pyscript = os.path.join(batch_dir, 'pyscript_%s.py' % suffix)
fp = open(pyscript, 'wt')
fp.writelines(cmdstr)
fp.close()
batchscript = '\n'.join((self._template,
'%s %s' % (sys.executable, pyscript)))
batchscriptfile = os.path.join(batch_dir, 'batchscript_%s.sh' % suffix)
fp = open(batchscriptfile, 'wt')
fp.writelines(batchscript)
fp.close()
return self._submit_batchtask(batchscriptfile, node)
def create_pyscript(node, updatehash=False, store_exception=True):
# pickle node
timestamp = strftime('%Y%m%d_%H%M%S')
if node._hierarchy:
suffix = '%s_%s_%s' % (timestamp, node._hierarchy, node._id)
batch_dir = os.path.join(node.base_dir,
node._hierarchy.split('.')[0], 'batch')
else:
suffix = '%s_%s' % (timestamp, node._id)
batch_dir = os.path.join(node.base_dir, 'batch')
if not os.path.exists(batch_dir):
os.makedirs(batch_dir)
pkl_file = os.path.join(batch_dir, 'node_%s.pklz' % suffix)
savepkl(pkl_file, dict(node=node, updatehash=updatehash))
mpl_backend = node.config["execution"]["matplotlib_backend"]
# create python script to load and trap exception
cmdstr = """import os
import sys
can_import_matplotlib = True #Silently allow matplotlib to be ignored
try:
import matplotlib
matplotlib.use('%s')
except ImportError:
can_import_matplotlib = False
pass
from nipype import config, logging
from nipype.utils.filemanip import loadpkl, savepkl
from socket import gethostname
from traceback import format_exception
info = None
pklfile = '%s'
batchdir = '%s'
from nipype.utils.filemanip import loadpkl, savepkl
try:
if not sys.version_info < (2, 7):
from collections import OrderedDict
config_dict=%s
config.update_config(config_dict)
## Only configure matplotlib if it was successfully imported, matplotlib is an optional component to nipype
if can_import_matplotlib:
config.update_matplotlib()
logging.update_logging(config)
traceback=None
cwd = os.getcwd()
info = loadpkl(pklfile)
result = info['node'].run(updatehash=info['updatehash'])
except Exception, e:
etype, eval, etr = sys.exc_info()
traceback = format_exception(etype,eval,etr)
if info is None or not os.path.exists(info['node'].output_dir()):
result = None
resultsfile = os.path.join(batchdir, 'crashdump_%s.pklz')
else:
result = info['node'].result
resultsfile = os.path.join(info['node'].output_dir(),
'result_%%s.pklz'%%info['node'].name)
"""
if store_exception:
cmdstr += """
savepkl(resultsfile, dict(result=result, hostname=gethostname(),
traceback=traceback))
"""
else:
cmdstr += """
if info is None:
savepkl(resultsfile, dict(result=result, hostname=gethostname(),
traceback=traceback))
else:
from nipype.pipeline.plugins.base import report_crash
report_crash(info['node'], traceback, gethostname())
raise Exception(e)
"""
cmdstr = cmdstr % (mpl_backend, pkl_file, batch_dir, node.config, suffix)
pyscript = os.path.join(batch_dir, 'pyscript_%s.py' % suffix)
fp = open(pyscript, 'wt')
fp.writelines(cmdstr)
fp.close()
return pyscript
