def corr_to_zscore(in_file, distrib): import os import nipype.utils.filemanip as fmanip from nipype.utils.filemanip import fname_presuffix corrs = fmanip.loadpkl(in_file) dist = fmanip.loadpkl(distrib) zscore = (corrs["corr"] - dist[0]["mean"]) / dist[0]["std"] zfile = os.path.abspath(fname_presuffix(in_file, suffix="_zscore", newpath=os.getcwd())) corrs["zscore"] = zscore fmanip.savepkl(zfile, corrs) return zfile
def _get_result(self, taskid): if taskid not in self._pending: raise Exception('Task %d not found' % taskid) if self._is_pending(taskid): return None node_dir = self._pending[taskid] logger.debug(os.listdir(os.path.realpath(os.path.join(node_dir, '..')))) logger.debug(os.listdir(node_dir)) glob(os.path.join(node_dir, 'result_*.pklz')).pop() results_file = glob(os.path.join(node_dir, 'result_*.pklz'))[0] result_data = loadpkl(results_file) result_out = dict(result=None, traceback=None) if isinstance(result_data, dict): result_out['result'] = result_data['result'] result_out['traceback'] = result_data['traceback'] result_out['hostname'] = result_data['hostname'] if results_file: crash_file = os.path.join(node_dir, 'crashstore.pklz') os.rename(results_file, crash_file) else: result_out['result'] = result_data return result_out
def _get_result(self, taskid): if taskid not in self._pending: raise Exception('Task %d not found' % taskid) if self._is_pending(taskid): return None node_dir = self._pending[taskid] # MIT HACK # on the pbs system at mit the parent node directory needs to be # accessed before internal directories become available. there # is a disconnect when the queueing engine knows a job is # finished to when the directories become statable. t = time() timeout = float(self._config['execution']['job_finished_timeout']) timed_out = True while (time() - t) < timeout: try: logger.debug(os.listdir(os.path.realpath(os.path.join(node_dir, '..')))) logger.debug(os.listdir(node_dir)) glob(os.path.join(node_dir, 'result_*.pklz')).pop() timed_out = False break except Exception as e: logger.debug(e) sleep(2) if timed_out: result_data = {'hostname': 'unknown', 'result': None, 'traceback': None} results_file = None try: error_message = ('Job id ({0}) finished or terminated, but ' 'results file does not exist after ({1}) ' 'seconds. Batch dir contains crashdump file ' 'if node raised an exception.\n' 'Node working directory: ({2}) '.format( taskid, timeout, node_dir)) raise IOError(error_message) except IOError as e: result_data['traceback'] = format_exc() else: results_file = glob(os.path.join(node_dir, 'result_*.pklz'))[0] result_data = loadpkl(results_file) result_out = dict(result=None, traceback=None) if isinstance(result_data, dict): result_out['result'] = result_data['result'] result_out['traceback'] = result_data['traceback'] result_out['hostname'] = result_data['hostname'] if results_file: crash_file = os.path.join(node_dir, 'crashstore.pklz') os.rename(results_file, crash_file) else: result_out['result'] = result_data return result_out
def _get_result(self, taskid): if taskid not in self._pending: raise Exception("Task %d not found" % taskid) if self._is_pending(taskid): return None node_dir = self._pending[taskid] # MIT HACK # on the pbs system at mit the parent node directory needs to be # accessed before internal directories become available. there # is a disconnect when the queueing engine knows a job is # finished to when the directories become statable. t = time() timeout = float(self._config["execution"]["job_finished_timeout"]) timed_out = True while (time() - t) < timeout: try: glob(os.path.join(node_dir, "result_*.pklz")).pop() timed_out = False break except Exception as e: logger.debug(e) sleep(2) if timed_out: result_data = {"hostname": "unknown", "result": None, "traceback": None} results_file = None try: error_message = ( "Job id ({0}) finished or terminated, but " "results file does not exist after ({1}) " "seconds. Batch dir contains crashdump file " "if node raised an exception.\n" "Node working directory: ({2}) ".format(taskid, timeout, node_dir) ) raise IOError(error_message) except IOError as e: result_data["traceback"] = format_exc() else: results_file = glob(os.path.join(node_dir, "result_*.pklz"))[0] result_data = loadpkl(results_file) result_out = dict(result=None, traceback=None) if isinstance(result_data, dict): result_out["result"] = result_data["result"] result_out["traceback"] = result_data["traceback"] result_out["hostname"] = result_data["hostname"] if results_file: crash_file = os.path.join(node_dir, "crashstore.pklz") os.rename(results_file, crash_file) else: result_out["result"] = result_data return result_out
def _reading_ci_output(self, dir_nm_el, out_nm, node=None): """used for current interfaces: checking if the output exists and returns the path if it does""" if not node: node = self result_pklfile = os.path.join( os.getcwd(), node.workingdir, dir_nm_el, node.interface.nn.name, "result_{}.pklz".format(node.interface.nn.name)) if os.path.exists(result_pklfile): out_file = getattr(loadpkl(result_pklfile).outputs, out_nm) if os.path.exists(out_file): return out_file else: return False else: return False
def _get_result(self, taskid): if taskid not in self._pending: raise Exception("SGE task %d not found" % taskid) cmd = CommandLine("qstat") cmd.inputs.args = "-j %d" % (taskid) # retrieve sge taskid result = cmd.run() if result.runtime.stdout.startswith("="): return None node_dir = self._pending[taskid] results_file = glob(os.path.join(node_dir, "result_*.pklz"))[0] result_data = loadpkl(results_file) result_out = dict(result=None, traceback=None) if isinstance(result_data, dict): result_out["result"] = result_data["result"] result_out["traceback"] = result_data["traceback"] os.remove(results_file) else: result_out["result"] = result_data return result_out
def _get_result(self, taskid): if taskid not in self._pending: raise Exception('SGE task %d not found'%taskid) cmd = CommandLine('qstat') cmd.inputs.args = '-j %d'%(taskid) # retrieve sge taskid result = cmd.run() if result.runtime.stdout.startswith('='): return None node_dir = self._pending[taskid] results_file = glob(os.path.join(node_dir,'result_*.pklz'))[0] result_data = loadpkl(results_file) result_out = dict(result=None, traceback=None) if isinstance(result_data, dict): result_out['result'] = result_data['result'] result_out['traceback'] = result_data['traceback'] os.remove(results_file) else: result_out['result'] = result_data return result_out
def _get_result(self, taskid): if taskid not in self._pending: raise Exception('Task %d not found'%taskid) if self._is_pending(taskid): return None node_dir = self._pending[taskid] # on the pbs system at mit the parent node directory needs to be # accessed before internal directories become available. logger.debug(os.listdir(os.path.realpath(os.path.join(node_dir,'..')))) logger.debug(os.listdir(node_dir)) logger.debug(node_dir) results_file = glob(os.path.join(node_dir,'result_*.pklz'))[0] result_data = loadpkl(results_file) result_out = dict(result=None, traceback=None) if isinstance(result_data, dict): result_out['result'] = result_data['result'] result_out['traceback'] = result_data['traceback'] result_out['hostname'] = result_data['hostname'] crash_file = os.path.join(node_dir,'crashstore.pklz') os.rename(results_file, crash_file) else: result_out['result'] = result_data return result_out
def _get_result(self, taskid): if taskid not in self._pending: raise Exception("Task %d not found" % taskid) if self._is_pending(taskid): return None node_dir = self._pending[taskid] glob(os.path.join(node_dir, "result_*.pklz")).pop() results_file = glob(os.path.join(node_dir, "result_*.pklz"))[0] result_data = loadpkl(results_file) result_out = dict(result=None, traceback=None) if isinstance(result_data, dict): result_out["result"] = result_data["result"] result_out["traceback"] = result_data["traceback"] result_out["hostname"] = result_data["hostname"] if results_file: crash_file = os.path.join(node_dir, "crashstore.pklz") os.rename(results_file, crash_file) else: result_out["result"] = result_data return result_out
logger.debug(e) sleep(2) if timed_out: result_data = {'hostname': 'unknown', 'result': None, 'traceback': None} results_file = None try: raise IOError(('Job (%s) finished or terminated, but results file ' 'does not exist. Batch dir contains crashdump ' 'file if node raised an exception' % node_dir)) except IOError, e: result_data['traceback'] = format_exc() else: results_file = glob(os.path.join(node_dir, 'result_*.pklz'))[0] result_data = loadpkl(results_file) result_out = dict(result=None, traceback=None) if isinstance(result_data, dict): result_out['result'] = result_data['result'] result_out['traceback'] = result_data['traceback'] result_out['hostname'] = result_data['hostname'] if results_file: crash_file = os.path.join(node_dir, 'crashstore.pklz') os.rename(results_file, crash_file) else: result_out['result'] = result_data return result_out def _submit_job(self, node, updatehash=False): """submit job and return taskid """
#!/usr/bin/env python __author__ = 'akeshavan' import sys from nipype.utils.filemanip import loadpkl if __name__ == "__main__": if len(sys.argv) > 1: crashes = sys.argv[1:] for c in crashes: pk = loadpkl(c)["node"] print(pk, pk.inputs.mseID)