def check_csv(file, force): with open(file) as f: content = f.read() for line, row in enumerate(content.split()): dwi_mask = [element for element in row.split(',') if element] # handling w/space if len(dwi_mask) != 2: raise FileNotFoundError( f'Columns don\'t have same number of entries: check line {line} in {file}' ) dirCheckFlag = 1 for img in dwi_mask: if not exists(img): raise FileNotFoundError( f'{img} does not exist: check line {line} in {file}') elif dirCheckFlag: # create DTI and harmonization directory dtiPath = pjoin(dirname(img), 'dti') check_dir(dtiPath, force) harmPath = pjoin(dirname(img), 'harm') check_dir(harmPath, force) dirCheckFlag = 0
def get_log_handler(logpath=None, cfg=None): """ Create a handler object to go in the logger """ if logpath == '': logpath = None try: dcfg = get_config() except: dcfg = None envname = os.getenv('CRAWL_LOG') if logpath is not None: final_logpath = logpath elif envname: final_logpath = envname elif cfg: final_logpath = cfg.get_d('crawler', 'logpath', U.default_logpath()) elif dcfg: final_logpath = dcfg.get_d('crawler', 'logpath', U.default_logpath()) else: final_logpath = U.default_logpath() if cfg: maxBytes = cfg.get_size('crawler', 'logsize', 10*1024*1024) backupCount = cfg.get_size('crawler', 'logmax', 5) archdir = cfg.get_d('crawler', 'archive_dir', U.pathjoin(U.dirname(final_logpath), 'hpss_log_archive')) elif dcfg: maxBytes = dcfg.get_size('crawler', 'logsize', 10*1024*1024) backupCount = dcfg.get_size('crawler', 'logmax', 5) archdir = dcfg.get_d('crawler', 'archive_dir', U.pathjoin(U.dirname(final_logpath), 'hpss_log_archive')) else: maxBytes = 10*1024*1024 backupCount = 5 archdir = U.pathjoin(U.dirname(final_logpath), 'hpss_log_archive') fh = util.ArchiveLogfileHandler(final_logpath, maxBytes=maxBytes, backupCount=backupCount, archdir=archdir) strfmt = "%" + "(asctime)s [%s] " % U.hostname + '%' + "(message)s" fmt = logging.Formatter(strfmt, datefmt="%Y.%m%d %H:%M:%S") fh.setFormatter(fmt) fh.handleError = raiseError return fh
def organize_directory(self): if (self.scenes): # Iterate Over Scenes List for curr, scene in enumerate(self.scenes, 1): # Create Scene Folder util.mkdir( util.form_path([self.image_path, SCENE.format(curr)])) # Move Images To Scene Folder for image in scene: try: # Generate Source and Destination Paths src = util.absolute(image) dst = util.normalize( util.form_path([ util.dirname(image), SCENE.format(curr), util.basename(image) ])) # Move Images To Scene Folder util.move(src, dst) except FileNotFoundError: pass # Update Prompt print("Organized All Images ") else: util.perror("spectra: No scenes found to analyze")
def verifyNshmForAll(csvFile, N_shm): for imgPath in read_imgs_masks(csvFile)[0]: directory = dirname(imgPath) prefix = basename(imgPath).split('.nii')[0] bvalFile = pjoin(directory, prefix + '.bval') verifyNshm(N_shm, bvalFile)
def attach_to_max(contents): """ Defines commands to send to Max, establishes a connection to its commandPort, then sends the code to inject debugpy """ global run_code config = contents['arguments'] # Format the simulated attach response to send it back to the debugger # while we set up the debugpy in the background attach_code = ATTACH_TEMPLATE.format( debugpy_path=debugpy_path, hostname=config['debugpy']['host'], port=int(config['debugpy']['port']) ) # Format RUN_TEMPLATE to point to the temporary # file containing the code to run run_code = RUN_TEMPLATE.format( dir=dirname(config['program']), file_name=split(config['program'])[1][:-3] or basename(split(config['program'])[0])[:-3] ) # then send attach code log('Sending attach code to Max') send_py_code_to_max(attach_code) log('Successfully attached to Max') # Then start the max debugging threads run(start_debugging, ((config['debugpy']['host'], int(config['debugpy']['port'])),))
def select_windows(parameters, paths, merge_flag, obs_tag, syn_tag): """ Selects windows by comparing observed and synthetic traces; writes results to a json file :param parameters: dictionary passed directly to pyflex.Config :param paths.obs: ASDF observed data filename :param paths.syn: ASDF synthetic data filename :param paths.output: windows will be written to a JSON file with this name :param paths.log: information about the quantity and quality of windows will be written to a JSON file with this name :param obs_tag: observed data are read using this ASDF tag :param syn_tag: synthetic data are read using this ASDF tag """ from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.rank cwd = dirname(__file__) # read data fullpath = join(cwd, paths.obs) obs = pyasdf.ASDFDataSet(fullpath, compression=None, mode="a") event = obs.events[0] # read synthetics fullpath = join(cwd, paths.syn) syn = pyasdf.ASDFDataSet(fullpath, compression=None, mode="a") # generate pyflex.Config objects config = {} for channel, param in parameters.items(): config[channel] = pyflex.Config(**param) # wrapper is required for ASDF processing def wrapped_function(obs, syn): return pytomo3d.window.window_on_stream(obs[obs_tag], syn[syn_tag], config, station=obs.StationXML, event=event, user_modules=None, figure_mode=False, figure_dir=None, _verbose=False) # run window selection windows = obs.process_two_files_without_parallel_output( syn, wrapped_function) # save results if rank == 0: if merge_flag: windows = merge(windows) write_windows_json(paths.output, windows)
def joinBshells(imgPath, ref_bvals_file=None, ref_bvals=None, sep_prefix=None): if ref_bvals_file: print('Reading reference b-shell file ...') ref_bvals = read_bvals(ref_bvals_file) print('Joining b-shells for', imgPath) imgPath = local.path(imgPath) img = load(imgPath._path) dim = img.header['dim'][1:5] inPrefix = abspath(imgPath).split('.nii')[0] directory = dirname(inPrefix) prefix = basename(inPrefix) bvalFile = inPrefix + '.bval' bvecFile = inPrefix + '.bvec' if sep_prefix: harmPrefix = pjoin(directory, sep_prefix + prefix) else: harmPrefix = inPrefix if not isfile(harmPrefix + '.bval'): copyfile(bvalFile, harmPrefix + '.bval') if not isfile(harmPrefix + '.bvec'): copyfile(bvecFile, harmPrefix + '.bvec') bvals = np.array(read_bvals(inPrefix + '.bval')) joinedDwi = np.zeros((dim[0], dim[1], dim[2], dim[3]), dtype='float32') for bval in ref_bvals: # ind= np.where(bval==bvals)[0] ind = np.where(abs(bval - bvals) <= BSHELL_MIN_DIST)[0] if bval == 0.: b0Img = load(inPrefix + '_b0.nii.gz') b0 = b0Img.get_data() for i in ind: joinedDwi[:, :, :, i] = b0 else: b0_bshell = load(harmPrefix + f'_b{int(bval)}.nii.gz').get_data() joinedDwi[:, :, :, ind] = b0_bshell[:, :, :, 1:] if not isfile(harmPrefix + '.nii.gz'): save_nifti(harmPrefix + '.nii.gz', joinedDwi, b0Img.affine, b0Img.header) else: print(harmPrefix + '.nii.gz', 'already exists, not overwritten.')
def bscr_test(args): """test - run tests usage: bscr test [-t py.test|green|nosetests|unittest] [-n] [-d] Without -t, we use the first available of py.test, green, nosetests, or unittest. With -t, we attempt to run the tests with the specified test runner. The tests are optimized for py.test. They may not work well under green or nose. """ c = U.cmdline([{'opts': ['--dry-run', '-n'], 'action': 'store_true', 'help': 'see what would happen'}, {'name': 'tester', 'help': 'select a test runner'}]) (o, a) = c.parse(args) with util.Chdir(util.dirname(__file__)): target = util.pj(os.path.dirname(__file__), 'test') print("Running tests in %s" % target) if o.tester == '': if which('py.test'): util.run('py.test %s' % target, not o.dryrun) elif which('green'): util.run('green -v %s' % target, not o.dryrun) elif which('nosetests') and importable('nose_ignoredoc'): tl = glob.glob(util.pj(target, 'test_*.py')) util.run('nosetests -v -c nose.cfg %s' % " ".join(tl), not o.dryrun) else: tl = glob.glob(util.pj(target, 'test_*.py')) for t in tl: util.run("%s -v" % t, not o.dryrun) # p = subp.Popen([t, '-v']) # p.wait() elif o.tester == 'py.test': util.run('py.test %s' % target, not o.dryrun) elif o.tester == 'green': util.run('green -v %s' % target, not o.dryrun) elif o.tester == 'nose': tl = glob.glob(util.pj(target, 'test_*.py')) util.run('nosetests -v -c nose.cfg %s' % " ".join(tl), not o.dryrun) elif o.tester == 'unittest': tl = glob.glob(util.pj(target, 'test_*.py')) for t in tl: util.run("%s -v" % t, not o.dryrun) else: raise SystemExit("unrecognized tester: '%s'" % o.tester)
def combine_adjoint_sources(paths, tag, rotate=True, auxiliary_data=False): """ Sums adjoint sources from different ASDF files in a linear combination with user-supplied weights. Can be useful, for example, if previous data processing, window selection, and misfit measurements steps were carried out separately for different pass bands (e.g. long period, short period) or different phases (e.g. body waves, surface waves). param paths.input: ASDF files containing adjoint sources type paths.input: list param paths.weight: corresponding list of JSON files containing weights type paths.weight: list param paths.output: summed adjoint sources are written to this ASDF file type paths.output: str """ cwd = dirname(__file__) adjoint_sources_all = [] weights_all = [] for filenames in zip_catch(paths.input, paths.weights): # read adjoint sources fullname = join(cwd, filenames[0]) adjoint_sources_all += [ pyasdf.ASDFDataSet(fullname, mpi=False, mode='r') ] # read user-supplied weights fullname = join(cwd, filenames[1]) weights_all += [read_json(fullname)] # create output file shutil.copy(paths.input[0], paths.output) adjoint_sources_sum = pyasdf.ASDFDataSet(paths.output, mpi=False, mode="a") # overwite output file data with zeros for waveform in adjoint_sources_sum.waveforms: for trace in waveform[tag]: trace.data[:] = 0. # weighted sum of adjoint sources for weights, adjoint_sources in zip(weights_all, adjoint_sources_all): for station in weights: for trace1, trace2 in \ zip(adjoint_sources_sum.waveforms[station][tag], adjoint_sources.waveforms[station][tag]): trace1.data += weights[station][trace1.id] * trace2.data # write misfit pass
def on_receive_from_debugger(message): """ Intercept the initialize and attach requests from the debugger while debugpy is being set up """ # Load message contents into a dictionary contents = json.loads(message) log('Received from Debugger:', message) # Get the type of command the debugger sent cmd = contents['command'] if cmd == 'initialize': # Run init request once Maya connection is established and send success response to the debugger interface.send(json.dumps(json.loads( INITIALIZE_RESPONSE))) # load and dump to remove indents processed_seqs.append(contents['seq']) elif cmd == 'attach': # time to attach to Maya run(attach_to_maya, (contents, )) # Change arguments to valid ones for debugpy config = contents['arguments'] new_args = ATTACH_ARGS.format( dir=dirname(config['program']).replace('\\', '\\\\'), hostname=config['debugpy']['host'], port=int(config['debugpy']['port']), # filepath=config['program'].replace('\\', '\\\\') ) # Update the message with the new arguments to then be sent to debugpy contents = contents.copy() contents['arguments'] = json.loads(new_args) message = json.dumps(contents) # update contents to reflect new args log("New attach arguments loaded:", new_args) # Then just put the message in the debugpy queue debugpy_send_queue.put(message)
def attach_to_maya(contents): """ Defines commands to send to Maya, and sends the attach code to it. """ global attach_code, Maya_path config = contents['arguments'] # Format the simulated attach response to send it back to the debugger # while we set up the debugpy in the background attach_code = ATTACH_TEMPLATE.format( debugpy_path=debugpy_path, hostname=config['debugpy']['host'], port=int(config['debugpy']['port']), interpreter=config['interpreter'], ) # Copy code to temporary file and start a Maya console with it try: send_code_to_maya(attach_code) except Exception as e: # Raising exceptions shows the text in the Debugger's output. # Raise an error to show a potential solution to this problem. log("Exception occurred: \n\n" + str(e)) import platform module_path = join(dirname(__file__), 'resources', 'module') separator = ';' if platform.system() == 'Windows' else ':' raise Exception(""" Could not connect to Maya. Please ensure Maya is running. If this is your first time using the debug adapter, ensure the MAYA_MODULE_PATH environment variable is set correctly (ie contains {0}), then restart Maya and try again. """.format(module_path + separator)) # Then start the Maya debugging threads run(start_debugging, ((config['debugpy']['host'], int(config['debugpy']['port'])), ))
def load_priority_list(cls): """ If one or more priority list files are configured, read them and put their contents first in the list of Checkables to be processed """ rval = [] cfg = CrawlConfig.get_config() priglob = cfg.get_d('cv', 'priority', '') if priglob == '': return rval pricomp = cfg.get_d('cv', 'completed', U.pathjoin(U.dirname(priglob), 'completed')) for pripath in U.foldsort(glob.glob(priglob)): with open(pripath, 'r') as f: for line in f.readlines(): path = line.strip() rval.append(Checkable(path=path, type='f')) os.rename(pripath, U.pathjoin(pricomp, U.basename(pripath))) return rval
def process_traces(parameters, paths, tag1, tag2): """ Performs bandpass filtering and other data processing operations on ASDF data; writes processed waveforms in a new ASDF file or under a new ASDF tag :param parameters: dictionary passed directly to pytomo3d :param paths.input: input ASDF filename :param paths.output: output ASDF filename :param tag1: input ASDF waveforms are read using this tag :param tag2: output ASDF waveforms are written using this tag """ from mpi4py import MPI cwd = dirname(__file__) # read data fullpath = join(cwd, paths.input) ds = pyasdf.ASDFDataSet(fullpath, compression=None, mode="a") event = ds.events[0] # add event information latitude, longitude, origin_time = event_stats(ds) parameters['event_longitude'] = longitude parameters['event_latitude'] = latitude parameters['starttime'] = origin_time + parameters['starttime'] parameters['endtime'] = origin_time + parameters['endtime'] # wrapper is required for ASDF processing def wrapped_function(stream, inventory): parameters.update({"inventory": inventory}) return pytomo3d.signal.process_stream(stream, **parameters) # process data ds.process(wrapped_function, paths.output, {tag1: tag2}) del ds
def load_priority_list(cls): """ If one or more priority list files are configured, read them and put their contents first in the list of Checkables to be processed """ rval = [] cfg = CrawlConfig.get_config() priglob = cfg.get_d('cv', 'priority', '') if priglob == '': return rval pricomp = cfg.get_d('cv', 'completed', U.pathjoin(U.dirname(priglob), 'completed')) for pripath in U.foldsort(glob.glob(priglob)): with open(pripath, 'r') as f: for line in f.readlines(): path = line.strip() rval.append(Checkable(path=path, type='f')) os.rename(pripath, U.pathjoin(pricomp, U.basename(pripath))) return rval
def start_server(): app = Application([ (r'\/*', IndexHandler), (r'/ping\/*', PingHandler), (r'/cluster\/*', ClusterInfoHandler), (r'/appliance\/*', AppliancesHandler), (r'/appliance/(%s)\/*' % Appliance.ID_PATTERN, ApplianceHandler), (r'/appliance/(%s)/container\/*' % Appliance.ID_PATTERN, ContainersHandler), (r'/appliance/(%s)/volume\/*' % Appliance.ID_PATTERN, ApplianceVolumesHandler), (r'/appliance/(%s)/service\/*' % Appliance.ID_PATTERN, ServicesHandler), (r'/appliance/(%s)/job\/*' % Container.ID_PATTERN, JobsHandler), (r'/appliance/(%s)/ui\/*' % Appliance.ID_PATTERN, ApplianceUIHandler), (r'/appliance/(%s)/container/(%s)\/*' % (Appliance.ID_PATTERN, Container.ID_PATTERN), ContainerHandler), (r'/appliance/(%s)/volume/(%s)\/*' % (Appliance.ID_PATTERN, PersistentVolume.ID_PATTERN), ApplianceVolumeHandler), (r'/volume/(%s)\/*' % PersistentVolume.ID_PATTERN, GlobalVolumeHandler), (r'/static/(.*)', StaticFileHandler, dict(path='%s/static' % dirname(__file__))), (r'/api', SwaggerAPIHandler), (r'/api/ui', SwaggerUIHandler), ]) ssl_options = None if config.pivot.https: ssl_options = dict(certfile='/etc/pivot/server.pem', keyfile='/etc/pivot/server.key') server = tornado.httpserver.HTTPServer(app, ssl_options=ssl_options) server.bind(config.pivot.port) server.start(config.pivot.n_parallel) start_cluster_monitor() start_global_scheduler() tornado.ioloop.IOLoop.instance().start()
def write_adjoint_traces(misfit_type, misfit_parameters, filter_parameters, paths, obs_tag, syn_tag): """ Makes misfit measurements using observed and synthetic data; writes out misfit values and corresponding "adjoint sources" :param misfit_type: type of data misfit function, e.g. waveform_difference, multitaper_misfit :param misfit_parameters: dictionary passed directly to pyadjoint.Config :param paths.obs: ASDF observed data filename :param paths.syn: ASDF synthetic data filename :param paths.windows: JSON windows filename :param paths.adjoint_sources: adjoint_sources will be written to an ASDF file with this name :param paths.misfit: misfit values will be written a JSON file with this name :param obs_tag: observed data are read using this ASDF tag :param syn_tag: synthetic data are read using this ASDF tag """ from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.rank cwd = dirname(__file__) # read data fullpath = join(cwd, paths.obs) obs = pyasdf.ASDFDataSet(fullpath, compression=None, mode="a") event = obs.events[0] # read synthetics fullpath = join(cwd, paths.syn) syn = pyasdf.ASDFDataSet(fullpath, compression=None, mode="a") # read windows fullpath = join(cwd, paths.windows) windows = read_json_mpi(paths.windows, comm) # generate pyadjoint.Config objects config = pyadjoint.Config(**misfit_parameters) # wrapper is required for ASDF processing def wrapped_function(obs, syn): # TODO: modify pytomo3d to make the following # function call more readable? return pytomo3d.adjoint.calculate_and_process_adjsrc_on_stream( obs[obs_tag], syn[syn_tag], windows[obs._station_name], obs.StationXML, config, event, misfit_type, filter_parameters, figure_mode=False, figure_dir=None) adjoint_sources = obs.process_two_files_without_parallel_output( syn, wrapped_function) if rank == 0: # save as ASDF waveforms ds = pyasdf.ASDFDataSet(paths.adjoint_sources, mpi=False, mode="a") tag = 'processed_adjoint' add_adjoint_source_waveforms(ds, adjoint_sources, tag) del ds # write misfit pass
def process_images(self): if (self.image_list): # Initialize Process Counter curr = 0 # Initialize Hash List self.hashes = [] # Initalize Blurred Array self.blurred = [] # Load Image Data Map image_data = load_image_data(self.image_path) # Error Check Image Data Map if (image_data is None): image_data = {} # Calculate Hash Values for image in self.image_list: # Create Data Object if (not (image in image_data)): image_data[image] = data(lmod=util.get_lmod(image)) # Calculate Imagehash self.hashes.append(imagehash.average_hash(Image.open(image))) # End Imagehash Calculation---------------------------------------------------------------------------------------------------------------------------- # Store Image Name input_image = image # Store Recent Modification Time curr_lmod = util.get_lmod(image) # Calculate Blur Coefficient if ((image_data[image].vari is None) or (image_data[image].nmax is None) or (image_data[image].rmsv is None) or (image_data[image].lmod < curr_lmod)): # Compute RMS Value loaded_image = Image.open(image).convert('L') image_stats = ImageStat.Stat(loaded_image) image_rms = image_stats.rms[0] # Determine RMS Deficiency if (image_rms < self.rms_threshold): # Create Cache Folder try: util.mkdir( util.form_path([self.image_path, TEMP_FOLD])) except FileExistsError: pass # Create Cache File input_image = util.form_path([ util.dirname(util.absolute(image)), TEMP_FOLD, EQ_IMAGE.format(util.basename(image)) ]) # Equalize Image Histogram image_file = cv2.imread(image, cv2.IMREAD_GRAYSCALE) clahe = cv2.createCLAHE(clipLimit=1.125, tileGridSize=(4, 4)) eq_image = clahe.apply(image_file) cv2.imwrite(input_image, eq_image) # Ignore Future Warnings with warnings.catch_warnings(): warnings.filterwarnings("ignore") # Compute Laplace Matrix loaded_image = rgb2gray(io.imread(input_image)) laplace_data = laplace(loaded_image, ksize=10) # Store Image Data image_data[image].vari = variance(laplace_data) image_data[image].nmax = np.amax(laplace_data) image_data[image].rmsv = image_rms image_data[image].lmod = curr_lmod # Group Blurry Images if ((image_data[image].vari < self.var_threshold) and (image_data[image].nmax < self.max_threshold)): self.blurred.append(image) # Update Prompt print("\rProcessing Images - {}% ".format( int(curr * 100 / len(self.image_list))), end="") curr += 1 # End Variance Computation--------------------------------------------------------------------------------------------------------------------------------- # Write Computed Data To Data File with open(util.form_path([self.image_path, DATA_FILE]), 'w') as data_file: for image in image_data: if (image in self.image_list): data_file.write("{},{},{},{},{}\n".format( image, image_data[image].vari, image_data[image].nmax, image_data[image].rmsv, image_data[image].lmod)) # Close File data_file.close() # End Write Operation-------------------------------------------------------------------------------------------------------------------------------------- # Initialize Diff List self.hash_diffs = [] # Calculate Hash Differences for i in range(len(self.hashes) - 1): self.hash_diffs.append( (self.hashes[i + 1] - self.hashes[i]) * self.precision) # End Hash Difference Computation-------------------------------------------------------------------------------------------------------------------------- # Update Prompt print("\rProcessed All Images ") else: util.perror("spectra: Found no images to process")
def scan_cases(path): result = [] # 项目种类 project_kinds = util.safely_list_dir(path) for project_kind in project_kinds: projects = util.safely_list_dir(path, project_kind) # 遍历项目 for project in projects: versions = util.safely_list_dir(path, project_kind, project) # 遍历版本 for version in versions: pycharm_projects = util.safely_list_dir( path, project_kind, project, version) # 遍历pycharm工程 for pycharm_project in pycharm_projects: case_dir = util.join(path, project_kind, project, version, pycharm_project, 'src', 'cases') modules = util.safely_list_dir(case_dir) for m in modules: # 文件作为模块 if util.isfile(util.join(case_dir, m)): if util.suffix(m) == '.py': result.extend( parsePy2Dict(util.join(case_dir, m), util.join(path, project), 'cases', '', util.filename(m), [], None, version, pycharm_project, project_kind)) # 目录作为一级模块 else: # 在目录下的__init__.py里获取模块描述 module_desc = '' if util.isfile( util.join(case_dir, m, '__init__.py')): f = open(util.join(case_dir, m, '__init__.py')) p = ast.parse(f.read()).body module_desc = p[0].value.s if p else '' f.close() for root, dirs, files in util.walk( util.join(case_dir, m)): for file in files: if util.suffix(file) == '.py': modules = util.relative( util.join(root, file), case_dir)[0:-1] package = util.relative( util.join(root, file), util.dirname(case_dir))[0:-1] package = '.'.join(package) result.extend( parsePy2Dict( util.join(root, file), util.join(path, project), package, m, util.filename(file), modules, module_desc, version, pycharm_project, project_kind)) return result
def main(self): self.out = local.path(self.out) if self.out.exists(): if self.overwrite: self.out.delete() else: logging.error( "{} exists, use '--force' to overwrite it".format( self.out)) sys.exit(1) outxfms = self.out.dirname / self.out.stem + '-xfms.tgz' with TemporaryDirectory() as tmpdir, local.cwd(tmpdir): tmpdir = local.path(tmpdir) # fileinput() caused trouble reading data file in python 3, so switching to nrrd # if the hdr has 'nan' in space origin, the following will take care of that img = nrrd.read(self.dwi) dwi = img[0] hdr = img[1] hdr_out = hdr.copy() hdr_out['space origin'] = hdr_out['space origin'][0:3] nrrd.write('dwijoined.nhdr', dwi, header=hdr_out, compression_level=1) # we want to use this hdr to write a new .nhdr file with corresponding data file # so delete old data file from the hdr if 'data file' in hdr_out.keys(): del hdr_out['data file'] elif 'datafile' in hdr_out.keys(): del hdr_out['datafile'] if 'content' in hdr_out.keys(): del hdr_out['content'] logging.info('Dice the DWI') # Since fslmerge works along the 3rd axis only, dicing also has to be along that axis # So, use `unu permute` to reorient the volumes to be stacked along 3rd axis only # Include this issue in the tutorial (unu['convert', '-t', 'int16', '-i', 'dwijoined.nhdr'] | unu['dice', '-a', '3', '-o', 'Diffusion-G'])() vols = tmpdir.glob('Diffusion-G*.nrrd') vols.sort() logging.info('Extract the B0') bse_py('-i', 'dwijoined.nhdr', '-o', 'b0.nrrd') ConvertBetweenFileFormats('b0.nrrd', 'b0.nii.gz', 'short') logging.info('Register each volume to the B0') # use the following multi-processed loop pool = Pool(int(self.nproc)) res = pool.map_async(_Register_vol, vols) volsRegistered = res.get() pool.close() pool.join() # or use the following for loop # volsRegistered = [] # for vol in vols: # volnii = vol.with_suffix('.nii.gz') # ConvertBetweenFileFormats(vol, volnii, 'short') # logging.info('Run FSL flirt affine registration') # flirt('-interp' ,'sinc' # ,'-sincwidth' ,'7' # ,'-sincwindow' ,'blackman' # ,'-in', volnii # ,'-ref', 'b0.nii.gz' # ,'-nosearch' # ,'-o', volnii # ,'-omat', volnii.with_suffix('.txt', depth=2) # ,'-paddingsize', '1') # volsRegistered.append(volnii) fslmerge('-t', 'EddyCorrect-DWI', volsRegistered) transforms = tmpdir.glob('Diffusion-G*.txt') transforms.sort() # nibabel loading can be avoided by setting 'data file' = EddyCorrect-DWI.nii.gz # and 'byteskip' = -1 # Tashrif updated Pynrrd package to properly handle that new_dwi = nib.load('EddyCorrect-DWI.nii.gz').get_data() logging.info('Extract the rotations and realign the gradients') space = hdr_out['space'].lower() if (space == 'left'): spctoras = np.matrix([[-1, 0, 0], [0, -1, 0], [0, 0, 1]]) else: spctoras = np.matrix([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) mf = np.matrix(hdr['measurement frame']) # Transforms are in RAS so need to do inv(MF)*inv(SPC2RAS)*ROTATION*SPC2RAS*MF*GRADIENT mfras = mf.I * spctoras.I rasmf = spctoras * mf for (i, t) in enumerate(transforms): gDir = [ float(num) for num in hdr_out['DWMRI_gradient_' + '{:04}'.format(i)].split(' ') if num ] logging.info('Apply ' + t) tra = np.loadtxt(t) # removes the translation aff = np.matrix(tra[0:3, 0:3]) # computes the finite strain of aff to get the rotation rot = aff * aff.T # compute the square root of rot [el, ev] = np.linalg.eig(rot) eL = np.identity(3) * np.sqrt(el) sq = ev * eL * ev.I # finally the rotation is defined as rot = sq.I * aff newdir = np.dot(mfras * rot * rasmf, gDir) hdr_out['DWMRI_gradient_' + '{:04}'.format(i)] = (' ').join( str(x) for x in newdir.tolist()[0]) tar('cvzf', outxfms, transforms) nrrd.write(self.out, new_dwi, header=hdr_out, compression_level=1) if self.debug: tmpdir.copy( join(dirname(self.out), "eddy-debug-" + str(getpid())))
def main(self): self.out = local.path(self.out) if self.out.exists(): if self.overwrite: self.out.delete() else: logging.error("{} exists, use '--force' to overwrite it".format(self.out)) sys.exit(1) outxfms = self.out.dirname / self.out.stem+'_xfms.tgz' with TemporaryDirectory() as tmpdir, local.cwd(tmpdir): tmpdir = local.path(tmpdir) dicePrefix = 'vol' logging.info('Dice the DWI') fslsplit[self.dwi] & FG logging.info('Extract the B0') check_call((' ').join([pjoin(FILEDIR,'bse.py'), '-i', self.dwi._path, '-o', 'b0.nii.gz']), shell= True) logging.info('Register each volume to the B0') vols = sorted(tmpdir // (dicePrefix + '*.nii.gz')) # use the following multi-processed loop pool= Pool(int(self.nproc)) res= pool.map_async(_Register_vol, vols) volsRegistered= res.get() pool.close() pool.join() # or use the following for loop # volsRegistered = [] # for vol in vols: # volnii = vol.with_suffix('.nii.gz') # logging.info('Run FSL flirt affine registration') # flirt('-interp' ,'sinc' # ,'-sincwidth' ,'7' # ,'-sincwindow' ,'blackman' # ,'-in', volnii # ,'-ref', 'b0.nii.gz' # ,'-nosearch' # ,'-o', volnii # ,'-omat', volnii.with_suffix('.txt', depth=2) # ,'-paddingsize', '1') # volsRegistered.append(volnii) fslmerge('-t', 'EddyCorrect-DWI.nii.gz', volsRegistered) transforms = tmpdir.glob(dicePrefix+'*.txt') transforms.sort() logging.info('Extract the rotations and realign the gradients') bvecs= read_bvecs(self.bvecFile._path) bvecs_new= bvecs.copy() for (i,t) in enumerate(transforms): logging.info('Apply ' + t) tra = np.loadtxt(t) # removes the translation aff = np.matrix(tra[0:3,0:3]) # computes the finite strain of aff to get the rotation rot = aff*aff.T # compute the square root of rot [el, ev] = np.linalg.eig(rot) eL = np.identity(3)*np.sqrt(el) sq = ev*eL*ev.I # finally the rotation is defined as rot = sq.I*aff bvecs_new[i] = np.dot(rot,bvecs[i]).tolist()[0] tar('cvzf', outxfms, transforms) # save modified bvecs write_bvecs(self.out._path+'.bvec', bvecs_new) # save EddyCorrect-DWI local.path('EddyCorrect-DWI.nii.gz').copy(self.out._path+'.nii.gz') # copy bvals self.bvalFile.copy(self.out._path+'.bval') if self.debug: tmpdir.copy(pjoin(dirname(self.out),"eddy-debug-"+str(getpid())))
def makeAtlases(target, trainingTable, outPrefix, fusion, threads, debug): with TemporaryDirectory() as tmpdir: tmpdir = local.path(tmpdir) L= len(trainingTable) multiDataFrame= pd.concat([trainingTable, pd.DataFrame({'tmpdir': [tmpdir]*L, 'target': [str(target)]*L})], axis= 1) logging.info('Create {} atlases: compute transforms from images to target and apply over images'.format(L)) pool = multiprocessing.Pool(threads) # Use all available cores, otherwise specify the number you want as an argument pool.map_async(train2target, multiDataFrame.iterrows()) pool.close() pool.join() logging.info('Fuse warped labelmaps to compute output labelmaps') atlasimages = tmpdir // 'atlas*.nii.gz' # sorting is required for applying weight to corresponding labelmap atlasimages.sort() if fusion.lower() == 'wavg': ALPHA_DEFAULT= 0.45 logging.info('Compute MI between warped images and target') pool = multiprocessing.Pool(threads) for img in atlasimages: print('MI between {} and target'.format(img)) miFile= img+'.txt' pool.apply_async(func= computeMI, args= (target, img, miFile, )) pool.close() pool.join() mis= [] with open(tmpdir+'/MI.txt','w') as fw: for img in atlasimages: with open(img+'.txt') as f: mi= f.read().strip() fw.write(img+','+mi+'\n') mis.append(float(mi)) weights = weightsFromMIExp(mis, ALPHA_DEFAULT) target_header= load_nifti(target._path).header pool = multiprocessing.Pool(threads) # Use all available cores, otherwise specify the number you want as an argument for labelname in list(trainingTable)[1:]: # list(d) gets column names out = os.path.abspath(outPrefix+ f'_{labelname}.nii.gz') if os.path.exists(out): os.remove(out) labelmaps = tmpdir // (labelname + '*') labelmaps.sort() if fusion.lower() == 'avg': print(' ') # parellelize # fuseAvg(labelmaps, out, target_header) pool.apply_async(func= fuseAvg, args= (labelmaps, out, target_header, )) elif fusion.lower() == 'antsjointfusion': print(' ') # atlasimages are the warped images # labelmaps are the warped labels # parellelize # fuseAntsJointFusion(target, atlasimages, labelmaps, out) pool.apply_async(func= fuseAntsJointFusion, args= (target, atlasimages, labelmaps, out, )) elif fusion.lower() == 'wavg': print(' ') # parellelize # fuseWeightedAvg(labelmaps, weights, out, target_header) pool.apply_async(func= fuseWeightedAvg, args= (labelmaps, weights, out, target_header, )) else: print('Unrecognized fusion option: {}. Skipping.'.format(fusion)) pool.close() pool.join() if debug: tmpdir.copy(pjoin(dirname(outPrefix), 'atlas-debug-' + str(os.getpid())))
def new_logger(logpath='', cfg=None): """ Return a new logging object for this process. The log file path is derived from (in order): - logpath if set - environment ($CRAWL_LOG) - cfg - default (/var/log/hpssic.log if writable, else /tmp/hpssic.log) """ # ------------------------------------------------------------------------- def cfg_get(func, section, option, defval): if cfg: rval = func(section, option, defval) else: rval = defval return rval # ------------------------------------------------------------------------- def raiseError(record=None): raise envname = os.getenv('CRAWL_LOG') try: dcfg = get_config() except: dcfg = None if logpath != '': final_logpath = logpath elif envname: final_logpath = envname elif cfg: try: final_logpath = cfg.get('crawler', 'logpath') except NoOptionError: final_logpath = U.default_logpath() except NoSectionError: final_logpath = U.default_logpath() elif dcfg: try: final_logpath = dcfg.get('crawler', 'logpath') except NoOptionError: final_logpath = U.default_logpath() except NoSectionError: final_logpath = U.default_logpath() else: final_logpath = U.default_logpath() rval = logging.getLogger('hpssic') rval.setLevel(logging.INFO) host = U.hostname() for h in rval.handlers: h.close() del h if cfg: maxBytes = cfg.get_size('crawler', 'logsize', 10 * 1024 * 1024) backupCount = cfg.get_size('crawler', 'logmax', 5) archdir = cfg.get_d( 'crawler', 'archive_dir', U.pathjoin(U.dirname(final_logpath), 'hpss_log_archive')) else: maxBytes = 10 * 1024 * 1024 backupCount = 5 archdir = U.pathjoin(U.dirname(final_logpath), 'hpss_log_archive') fh = U.ArchiveLogfileHandler(final_logpath, maxBytes=maxBytes, backupCount=backupCount, archdir=archdir) strfmt = "%" + "(asctime)s [%s] " % host + '%' + "(message)s" fmt = logging.Formatter(strfmt, datefmt="%Y.%m%d %H:%M:%S") fh.setFormatter(fmt) fh.handleError = raiseError while 0 < len(rval.handlers): z = U.pop0(rval.handlers) del z rval.addHandler(fh) rval.info('-' * (55 - len(host))) return rval
@property def marathon(self): return self.__marathon @property def chronos(self): return self.__chronos @property def exhibitor(self): return self.__exhibitor @property def ceph(self): return self.__ceph config = Configuration.read_config('%s/config.yml'%dirname(__file__)) def get_global_scheduler(): try: sched_mod = '.'.join(config.pivot.scheduler.split('.')[:-1]) sched_class = config.pivot.scheduler.split('.')[-1] return getattr(importlib.import_module(sched_mod), sched_class)() except Exception as e: sys.stderr.write(str(e) + '\n') from schedule.universal import DefaultGlobalScheduler return DefaultGlobalScheduler()