def corr_to_zscore(in_file, distrib):
import os
import nipype.utils.filemanip as fmanip
from nipype.utils.filemanip import fname_presuffix
corrs = fmanip.loadpkl(in_file)
dist = fmanip.loadpkl(distrib)
zscore = (corrs["corr"] - dist[0]["mean"]) / dist[0]["std"]
zfile = os.path.abspath(fname_presuffix(in_file, suffix="_zscore", newpath=os.getcwd()))
corrs["zscore"] = zscore
fmanip.savepkl(zfile, corrs)
return zfile
def _get_result(self, taskid):
if taskid not in self._pending:
raise Exception('Task %d not found' % taskid)
if self._is_pending(taskid):
return None
node_dir = self._pending[taskid]
logger.debug(os.listdir(os.path.realpath(os.path.join(node_dir,
'..'))))
logger.debug(os.listdir(node_dir))
glob(os.path.join(node_dir, 'result_*.pklz')).pop()
results_file = glob(os.path.join(node_dir, 'result_*.pklz'))[0]
result_data = loadpkl(results_file)
result_out = dict(result=None, traceback=None)
if isinstance(result_data, dict):
result_out['result'] = result_data['result']
result_out['traceback'] = result_data['traceback']
result_out['hostname'] = result_data['hostname']
if results_file:
crash_file = os.path.join(node_dir, 'crashstore.pklz')
os.rename(results_file, crash_file)
else:
result_out['result'] = result_data
return result_out
def _get_result(self, taskid):
if taskid not in self._pending:
raise Exception('Task %d not found' % taskid)
if self._is_pending(taskid):
return None
node_dir = self._pending[taskid]
# MIT HACK
# on the pbs system at mit the parent node directory needs to be
# accessed before internal directories become available. there
# is a disconnect when the queueing engine knows a job is
# finished to when the directories become statable.
t = time()
timeout = float(self._config['execution']['job_finished_timeout'])
timed_out = True
while (time() - t) < timeout:
try:
logger.debug(os.listdir(os.path.realpath(os.path.join(node_dir,
'..'))))
logger.debug(os.listdir(node_dir))
glob(os.path.join(node_dir, 'result_*.pklz')).pop()
timed_out = False
break
except Exception as e:
logger.debug(e)
sleep(2)
if timed_out:
result_data = {'hostname': 'unknown',
'result': None,
'traceback': None}
results_file = None
try:
error_message = ('Job id ({0}) finished or terminated, but '
'results file does not exist after ({1}) '
'seconds. Batch dir contains crashdump file '
'if node raised an exception.\n'
'Node working directory: ({2}) '.format(
taskid, timeout, node_dir))
raise IOError(error_message)
except IOError as e:
result_data['traceback'] = format_exc()
else:
results_file = glob(os.path.join(node_dir, 'result_*.pklz'))[0]
result_data = loadpkl(results_file)
result_out = dict(result=None, traceback=None)
if isinstance(result_data, dict):
result_out['result'] = result_data['result']
result_out['traceback'] = result_data['traceback']
result_out['hostname'] = result_data['hostname']
if results_file:
crash_file = os.path.join(node_dir, 'crashstore.pklz')
os.rename(results_file, crash_file)
else:
result_out['result'] = result_data
return result_out
def _get_result(self, taskid):
if taskid not in self._pending:
raise Exception('Task %d not found' % taskid)
if self._is_pending(taskid):
return None
node_dir = self._pending[taskid]
# MIT HACK
# on the pbs system at mit the parent node directory needs to be
# accessed before internal directories become available. there
# is a disconnect when the queueing engine knows a job is
# finished to when the directories become statable.
t = time()
timeout = float(self._config['execution']['job_finished_timeout'])
timed_out = True
while (time() - t) < timeout:
try:
logger.debug(os.listdir(os.path.realpath(os.path.join(node_dir,
'..'))))
logger.debug(os.listdir(node_dir))
glob(os.path.join(node_dir, 'result_*.pklz')).pop()
timed_out = False
break
except Exception as e:
logger.debug(e)
sleep(2)
if timed_out:
result_data = {'hostname': 'unknown',
'result': None,
'traceback': None}
results_file = None
try:
error_message = ('Job id ({0}) finished or terminated, but '
'results file does not exist after ({1}) '
'seconds. Batch dir contains crashdump file '
'if node raised an exception.\n'
'Node working directory: ({2}) '.format(
taskid, timeout, node_dir))
raise IOError(error_message)
except IOError as e:
result_data['traceback'] = format_exc()
else:
results_file = glob(os.path.join(node_dir, 'result_*.pklz'))[0]
result_data = loadpkl(results_file)
result_out = dict(result=None, traceback=None)
if isinstance(result_data, dict):
result_out['result'] = result_data['result']
result_out['traceback'] = result_data['traceback']
result_out['hostname'] = result_data['hostname']
if results_file:
crash_file = os.path.join(node_dir, 'crashstore.pklz')
os.rename(results_file, crash_file)
else:
result_out['result'] = result_data
return result_out
def _get_result(self, taskid):
if taskid not in self._pending:
raise Exception("Task %d not found" % taskid)
if self._is_pending(taskid):
return None
node_dir = self._pending[taskid]
# MIT HACK
# on the pbs system at mit the parent node directory needs to be
# accessed before internal directories become available. there
# is a disconnect when the queueing engine knows a job is
# finished to when the directories become statable.
t = time()
timeout = float(self._config["execution"]["job_finished_timeout"])
timed_out = True
while (time() - t) < timeout:
try:
glob(os.path.join(node_dir, "result_*.pklz")).pop()
timed_out = False
break
except Exception as e:
logger.debug(e)
sleep(2)
if timed_out:
result_data = {"hostname": "unknown", "result": None, "traceback": None}
results_file = None
try:
error_message = (
"Job id ({0}) finished or terminated, but "
"results file does not exist after ({1}) "
"seconds. Batch dir contains crashdump file "
"if node raised an exception.\n"
"Node working directory: ({2}) ".format(taskid, timeout, node_dir)
)
raise IOError(error_message)
except IOError as e:
result_data["traceback"] = format_exc()
else:
results_file = glob(os.path.join(node_dir, "result_*.pklz"))[0]
result_data = loadpkl(results_file)
result_out = dict(result=None, traceback=None)
if isinstance(result_data, dict):
result_out["result"] = result_data["result"]
result_out["traceback"] = result_data["traceback"]
result_out["hostname"] = result_data["hostname"]
if results_file:
crash_file = os.path.join(node_dir, "crashstore.pklz")
os.rename(results_file, crash_file)
else:
result_out["result"] = result_data
return result_out
def _reading_ci_output(self, dir_nm_el, out_nm, node=None):
"""used for current interfaces: checking if the output exists and returns the path if it does"""
if not node:
node = self
result_pklfile = os.path.join(
os.getcwd(), node.workingdir, dir_nm_el, node.interface.nn.name,
"result_{}.pklz".format(node.interface.nn.name))
if os.path.exists(result_pklfile):
out_file = getattr(loadpkl(result_pklfile).outputs, out_nm)
if os.path.exists(out_file):
return out_file
else:
return False
else:
return False
def _get_result(self, taskid):
if taskid not in self._pending:
raise Exception("SGE task %d not found" % taskid)
cmd = CommandLine("qstat")
cmd.inputs.args = "-j %d" % (taskid)
# retrieve sge taskid
result = cmd.run()
if result.runtime.stdout.startswith("="):
return None
node_dir = self._pending[taskid]
results_file = glob(os.path.join(node_dir, "result_*.pklz"))[0]
result_data = loadpkl(results_file)
result_out = dict(result=None, traceback=None)
if isinstance(result_data, dict):
result_out["result"] = result_data["result"]
result_out["traceback"] = result_data["traceback"]
os.remove(results_file)
else:
result_out["result"] = result_data
return result_out
def _get_result(self, taskid):
if taskid not in self._pending:
raise Exception('SGE task %d not found'%taskid)
cmd = CommandLine('qstat')
cmd.inputs.args = '-j %d'%(taskid)
# retrieve sge taskid
result = cmd.run()
if result.runtime.stdout.startswith('='):
return None
node_dir = self._pending[taskid]
results_file = glob(os.path.join(node_dir,'result_*.pklz'))[0]
result_data = loadpkl(results_file)
result_out = dict(result=None, traceback=None)
if isinstance(result_data, dict):
result_out['result'] = result_data['result']
result_out['traceback'] = result_data['traceback']
os.remove(results_file)
else:
result_out['result'] = result_data
return result_out
def _get_result(self, taskid):
if taskid not in self._pending:
raise Exception('Task %d not found'%taskid)
if self._is_pending(taskid):
return None
node_dir = self._pending[taskid]
# on the pbs system at mit the parent node directory needs to be
# accessed before internal directories become available.
logger.debug(os.listdir(os.path.realpath(os.path.join(node_dir,'..'))))
logger.debug(os.listdir(node_dir))
logger.debug(node_dir)
results_file = glob(os.path.join(node_dir,'result_*.pklz'))[0]
result_data = loadpkl(results_file)
result_out = dict(result=None, traceback=None)
if isinstance(result_data, dict):
result_out['result'] = result_data['result']
result_out['traceback'] = result_data['traceback']
result_out['hostname'] = result_data['hostname']
crash_file = os.path.join(node_dir,'crashstore.pklz')
os.rename(results_file, crash_file)
else:
result_out['result'] = result_data
return result_out
def _get_result(self, taskid):
if taskid not in self._pending:
raise Exception("Task %d not found" % taskid)
if self._is_pending(taskid):
return None
node_dir = self._pending[taskid]
glob(os.path.join(node_dir, "result_*.pklz")).pop()
results_file = glob(os.path.join(node_dir, "result_*.pklz"))[0]
result_data = loadpkl(results_file)
result_out = dict(result=None, traceback=None)
if isinstance(result_data, dict):
result_out["result"] = result_data["result"]
result_out["traceback"] = result_data["traceback"]
result_out["hostname"] = result_data["hostname"]
if results_file:
crash_file = os.path.join(node_dir, "crashstore.pklz")
os.rename(results_file, crash_file)
else:
result_out["result"] = result_data
return result_out
logger.debug(e)
sleep(2)
if timed_out:
result_data = {'hostname': 'unknown',
'result': None,
'traceback': None}
results_file = None
try:
raise IOError(('Job (%s) finished or terminated, but results file '
'does not exist. Batch dir contains crashdump '
'file if node raised an exception' % node_dir))
except IOError, e:
result_data['traceback'] = format_exc()
else:
results_file = glob(os.path.join(node_dir, 'result_*.pklz'))[0]
result_data = loadpkl(results_file)
result_out = dict(result=None, traceback=None)
if isinstance(result_data, dict):
result_out['result'] = result_data['result']
result_out['traceback'] = result_data['traceback']
result_out['hostname'] = result_data['hostname']
if results_file:
crash_file = os.path.join(node_dir, 'crashstore.pklz')
os.rename(results_file, crash_file)
else:
result_out['result'] = result_data
return result_out
def _submit_job(self, node, updatehash=False):
"""submit job and return taskid
"""
#!/usr/bin/env python
__author__ = 'akeshavan'
import sys
from nipype.utils.filemanip import loadpkl
if __name__ == "__main__":
if len(sys.argv) > 1:
crashes = sys.argv[1:]
for c in crashes:
pk = loadpkl(c)["node"]
print(pk, pk.inputs.mseID)
