def compute_custom_transform(self): """ Compute transform based on added control points. """ curr_points = [] for i, c in enumerate(self.curr_gscene.anchor_circle_items): pos = c.scenePos() curr_points.append((pos.x(), pos.y())) prev_points = [] for i, c in enumerate(self.prev_gscene.anchor_circle_items): pos = c.scenePos() prev_points.append((pos.x(), pos.y())) print(self.curr_gscene.active_section, np.array(curr_points)) print(self.prev_gscene.active_section, np.array(prev_points)) curr_points = np.array(curr_points) prev_points = np.array(prev_points) curr_centroid = curr_points.mean(axis=0) prev_centroid = prev_points.mean(axis=0) curr_points0 = curr_points - curr_centroid prev_points0 = prev_points - prev_centroid H = np.dot(curr_points0.T, prev_points0) U, S, VT = np.linalg.svd(H) R = np.dot(VT.T, U.T) t = -np.dot(R, curr_centroid) + prev_centroid print(R, t) # Write to custom transform file curr_section_fn = self.curr_gscene.active_section prev_section_fn = self.prev_gscene.active_section custom_tf_dir = os.path.join(self.stack_data_dir, self.stack + '_custom_transforms', curr_section_fn + '_to_' + prev_section_fn) execute_command("rm -rf %(out_dir)s; mkdir -p %(out_dir)s" % dict(out_dir=custom_tf_dir)) custom_tf_fp = os.path.join(custom_tf_dir, '%(curr_fn)s_to_%(prev_fn)s_customTransform.txt' % \ dict(curr_fn=curr_section_fn, prev_fn=prev_section_fn)) with open(custom_tf_fp, 'w') as f: f.write('%f %f %f %f %f %f\n' % (R[0,0], R[0,1], t[0], R[1,0], R[1,1], t[1])) self.apply_custom_transform() self.update_transformed_images_feeder()
def align_using_elastix(self): selected_elastix_parameter_name = str(self.alignment_ui.comboBox_parameters.currentText()) param_fn = os.path.join(UTILITY_DIR, 'preprocess', 'parameters', 'Parameters_' + selected_elastix_parameter_name + '.txt') curr_fn = self.curr_gscene.active_section prev_fn = self.prev_gscene.active_section out_dir = os.path.join(self.stack_data_dir, self.stack + '_custom_transforms', curr_fn + '_to_' + prev_fn) curr_fp = DataManager.get_image_filepath_v2(stack=self.stack, prep_id=None, fn=curr_fn, resol=self.tb_res, version=self.tb_version) prev_fp = DataManager.get_image_filepath_v2(stack=self.stack, prep_id=None, fn=prev_fn, resol=self.tb_res, version=self.tb_version ) # curr_fp = os.path.join(RAW_DATA_DIR, self.stack, curr_fn + '.' + self.tb_fmt) # prev_fp = os.path.join(RAW_DATA_DIR, self.stack, prev_fn + '.' + self.tb_fmt) execute_command('rm -rf %(out_dir)s; mkdir -p %(out_dir)s; elastix -f %(fixed_fn)s -m %(moving_fn)s -out %(out_dir)s -p %(param_fn)s' % \ dict(param_fn=param_fn, out_dir=out_dir, fixed_fn=prev_fp, moving_fn=curr_fp)) # section_filenames = self.get_sorted_filenames(valid_only=self.show_valid_only) self.update_transformed_images_feeder()
def save_everything(self): # Dump preprocessing info placeholder_indices = [idx+1 for idx, fn in enumerate(self.sorted_filenames) if fn == 'Placeholder'] placeholder_slide_positions = [(slide_name, pos) for slide_name, x in self.slide_position_to_fn.iteritems() for pos, fn in x.iteritems() if fn == 'Placeholder'] rescan_indices = [idx+1 for idx, fn in enumerate(self.sorted_filenames) if fn == 'Rescan'] rescan_slide_positions = [(slide_name, pos) for slide_name, x in self.slide_position_to_fn.iteritems() for pos, fn in x.iteritems() if fn == 'Rescan'] ul_pos = self.sorted_sections_gscene.corners['ul'].scenePos() lr_pos = self.sorted_sections_gscene.corners['lr'].scenePos() ul_x = int(ul_pos.x()) ul_y = int(ul_pos.y()) lr_x = int(lr_pos.x()) lr_y = int(lr_pos.y()) info = {'placeholder_indices': placeholder_indices, 'placeholder_slide_positions': placeholder_slide_positions, 'rescan_indices': rescan_indices, 'rescan_slide_positions': rescan_slide_positions, 'sorted_filenames': self.sorted_filenames, 'slide_position_to_fn': self.slide_position_to_fn, 'first_section': self.first_section, 'last_section': self.last_section, 'anchor_fn': self.anchor_fn, # 'bbox': (ul_x, lr_x, ul_y, lr_y) #xmin,xmax,ymin,ymax 'bbox': (ul_x, ul_y, lr_x+1-ul_x, lr_y+1-ul_y) #xmin,ymin,w,h } from datetime import datetime timestamp = datetime.now().strftime("%m%d%Y%H%M%S") pickle.dump(info, open(self.stack_data_dir + '/%(stack)s_preprocessInfo_%(timestamp)s.pkl' % {'stack': self.stack, 'timestamp':timestamp}, 'w')) execute_command('cd %(stack_data_dir)s && rm -f %(stack)s_preprocessInfo.pkl && ln -s %(stack)s_preprocessInfo_%(timestamp)s.pkl %(stack)s_preprocessInfo.pkl' % {'stack': self.stack, 'timestamp':timestamp, 'stack_data_dir':self.stack_data_dir}) self.save_crop() self.save_sorted_filenames() self.save()
def apply_custom_transform(self): # section_filenames = self.get_sorted_filenames(valid_only=self.show_valid_only) # curr_section_fn = section_filenames[self.valid_section_indices[self.curr_gscene.active_i]-1] # prev_section_fn = section_filenames[self.valid_section_indices[self.prev_gscene.active_i]-1] curr_section_fn = self.curr_gscene.active_section prev_section_fn = self.prev_gscene.active_section custom_tf_fn = os.path.join(self.stack_data_dir, self.stack+'_custom_transforms', curr_section_fn + '_to_' + prev_section_fn, curr_section_fn + '_to_' + prev_section_fn + '_customTransform.txt') with open(custom_tf_fn, 'r') as f: t11, t12, t13, t21, t22, t23 = map(float, f.readline().split()) prev_fp = DataManager.get_image_filepath_v2(stack=self.stack, prep_id=None, fn=prev_section_fn, resol=self.tb_res, version=self.tb_version ) curr_fp = DataManager.get_image_filepath_v2(stack=self.stack, prep_id=None, fn=curr_section_fn, resol=self.tb_res, version=self.tb_version ) prev_img_w, prev_img_h = identify_shape(prev_fp) output_image_fp = os.path.join(self.stack_data_dir, '%(stack)s_custom_transforms/%(curr_fn)s_to_%(prev_fn)s/%(curr_fn)s_alignedTo_%(prev_fn)s.tif' % \ dict(stack=self.stack, curr_fn=curr_section_fn, prev_fn=prev_section_fn) ) execute_command("convert %(curr_fp)s -virtual-pixel background +distort AffineProjection '%(sx)f,%(rx)f,%(ry)f,%(sy)f,%(tx)f,%(ty)f' -crop %(w)sx%(h)s%(x)s%(y)s\! -flatten -compress lzw %(output_fp)s" %\ dict(curr_fp=curr_fp, output_fp=output_image_fp, tb_fmt=self.tb_fmt, sx=t11, sy=t22, rx=t21, ry=t12, tx=t13, ty=t23, w=prev_img_w, h=prev_img_h, x='+0', y='+0', raw_data_dir=RAW_DATA_DIR))
parser.add_argument("out_version", type=str, help="Output image version") args = parser.parse_args() out_version = args.out_version input_spec = load_ini(args.input_spec) stack = input_spec['stack'] image_name_list = input_spec['image_name_list'] if image_name_list == 'all': image_name_list = DataManager.load_sorted_filenames(stack=stack)[0].keys() prep_id = input_spec['prep_id'] if prep_id == 'None': prep_id = None resol = input_spec['resol'] version = input_spec['version'] if version == 'None': version = None for img_name in image_name_list: t = time.time() in_fp = DataManager.get_image_filepath(stack=stack, resol=resol, version=version, fn=img_name) out_fp = DataManager.get_image_filepath(stack=stack, resol=resol, version=out_version, fn=img_name) create_parent_dir_if_not_exists(out_fp) cmd = """convert "%(in_fp)s" -normalize -depth 8 "%(out_fp)s" """ % {'in_fp': in_fp, 'out_fp': out_fp} execute_command(cmd) sys.stderr.write("Intensity normalize: %.2f seconds.\n" % (time.time() - t))
def run_distributed(stack, command, argument_type='single', kwargs_list=None, jobs_per_node=1, node_list=None, local_only=False, use_aws=False): """ Distributed executing a command. Args: local_only: run on local computer instead of AWS cluster jobs_per_node: kwargs_list: either dict of lists {kA: [vA1, vA2, ...], kB: [vB1, vB2, ...]} or list of dicts [{kA:vA1, kB:vB1}, {kA:vA2, kB:vB2}, ...]. argument_type: one of list, list2, single. If command takes one input item as argument, use "single". If command takes a list of input items as argument, use "list2". If command takes an argument called "kwargs_str", use "list". """ fileLocationManager = FileLocationManager(stack) print('run ddddddddddddddddistrib', kwargs_list) if use_aws: execute_command( 'rm -f /home/ubuntu/stderr_*; rm -f /home/ubuntu/stdout_*') else: execute_command('rm -f %s; rm -f %s' % (os.path.join(fileLocationManager.mouseatlas_tmp, 'stderr_*'), os.path.join('stdout_*'))) if local_only: sys.stderr.write("Run locally.\n") n_hosts = 1 else: # Use a fixed node list rather than letting SGE automatically determine the node list. # This allows for control over which input items go to which node. if node_list is None: node_list = get_node_list() n_hosts = len(node_list) sys.stderr.write('%d nodes available.\n' % (n_hosts)) if n_hosts == 0: print('NODE LIST LENGTH IS 0. NO HOSTS AVAILABLE') return if kwargs_list is None: kwargs_list = {'dummy': [None] * n_hosts} if isinstance(kwargs_list, dict): keys, vals = zip(*kwargs_list.items()) kwargs_list_as_list = [dict(zip(keys, t)) for t in zip(*vals)] kwargs_list_as_dict = kwargs_list else: kwargs_list_as_list = kwargs_list keys = kwargs_list[0].keys() vals = [t.values() for t in kwargs_list] kwargs_list_as_dict = dict(zip(keys, zip(*vals))) assert argument_type in [ 'single', 'list', 'list2' ], 'argument_type must be one of single, list, list2.' create_if_not_exists(fileLocationManager.mouseatlas_tmp) print('point 1') for node_i, (fi, li) in enumerate( first_last_tuples_distribute_over(0, len(kwargs_list_as_list) - 1, n_hosts)): temp_script = os.path.join(fileLocationManager.mouseatlas_tmp, 'runall.sh') temp_f = open(temp_script, 'w') for j, (fj, lj) in enumerate( first_last_tuples_distribute_over(fi, li, jobs_per_node)): print('node i j fj, lj', j, fj, lj) if argument_type == 'list': line = command % { 'kwargs_str': json.dumps(kwargs_list_as_list[fj:lj + 1]) } elif argument_type == 'list2': line = command % { key: json.dumps(vals[fj:lj + 1]) for key, vals in kwargs_list_as_dict.items() } elif argument_type == 'single': # It is important to wrap command_templates and kwargs_list_str in apostrphes. # That lets bash treat them as single strings. # Reference: http://stackoverflow.com/questions/15783701/which-characters-need-to-be-escaped-in-bash-how-do-we-know-it line = "%(generic_launcher_path)s %(command_template)s %(kwargs_list_str)s" % \ {'generic_launcher_path': os.path.join(UTILITY_DIR, 'sequential_dispatcher.py'), 'command_template': shell_escape(command), 'kwargs_list_str': shell_escape(json.dumps(kwargs_list_as_list[fj:lj+1])) } temp_f.write(line + ' &\n') temp_f.write('wait') temp_f.close() os.chmod(temp_script, 0o777) # Explicitly specify the node to submit jobs. # By doing so, we can control which files are available in the local scratch space of which node. # One can then assign downstream programs to specific nodes so they can read corresponding files from local scratch. if use_aws: stdout_template = '/home/ubuntu/stdout_%d.log' stderr_template = '/home/ubuntu/stderr_%d.log' else: stdout_template = os.path.join(fileLocationManager.mouseatlas_tmp, 'stdout_%d.log') stderr_template = os.path.join(fileLocationManager.mouseatlas_tmp, 'stderr_%d.log') if local_only: stdout_f = open(stdout_template % node_i, "w") stderr_f = open(stderr_template % node_i, "w") print('running temp_script', temp_script) subprocess.run(temp_script, shell=True, stdout=stdout_f, stderr=stderr_f) else: print('qsub -V -q all.q@%(node)s -o %(stdout_log)s -e %(stderr_log)s %(script)s' % \ dict(node=node_list[node_i], script=temp_script, stdout_log=stdout_template % node_i, stderr_log=stderr_template % node_i)) subprocess.call('qsub -V -q all.q@%(node)s -o %(stdout_log)s -e %(stderr_log)s %(script)s' % \ dict(node=node_list[node_i], script=temp_script, stdout_log=stdout_template % node_i, stderr_log=stderr_template % node_i), shell=True) sys.stderr.write( 'Jobs submitted. Use wait_qsub_complete() to wait for all execution to finish.\n' )
def transfer_data(from_fp, to_fp, from_hostname, to_hostname, is_dir, include_only=None, exclude_only=None, includes=None): assert from_hostname in [ 'localhost', 'workstation', 'oasis', 's3', 'ec2', 's3raw', 'ec2scratch' ], 'from_hostname must be one of localhost, workstation, oasis, s3, s3raw, ec2 or ec2scratch.' assert to_hostname in [ 'localhost', 'workstation', 'oasis', 's3', 'ec2', 's3raw', 'ec2scratch' ], 'to_hostname must be one of localhost, workstation, oasis, s3, s3raw, ec2 or ec2scratch.' to_parent = os.path.dirname(to_fp) if from_hostname in ['localhost', 'ec2', 'workstation', 'ec2scratch']: # upload if to_hostname in ['s3', 's3raw']: if is_dir: if includes is not None: execute_command( 'aws s3 cp --recursive \"%(from_fp)s\" \"s3://%(to_fp)s\" --exclude \"*\" %(includes_str)s' % dict(from_fp=from_fp, to_fp=to_fp, includes_str=" ".join( ['--include ' + incl for incl in includes]))) elif include_only is not None: execute_command( 'aws s3 cp --recursive \"%(from_fp)s\" \"s3://%(to_fp)s\" --exclude \"*\" --include \"%(include)s\"' % dict(from_fp=from_fp, to_fp=to_fp, include=include_only)) elif exclude_only is not None: execute_command( 'aws s3 cp --recursive \"%(from_fp)s\" \"s3://%(to_fp)s\" --include \"*\" --exclude \"%(exclude)s\"' % dict(from_fp=from_fp, to_fp=to_fp, exclude=exclude_only)) else: execute_command('aws s3 cp --recursive \"%(from_fp)s\" \"s3://%(to_fp)s\"' % \ dict(from_fp=from_fp, to_fp=to_fp)) else: execute_command('aws s3 cp \"%(from_fp)s\" \"s3://%(to_fp)s\"' % \ dict(from_fp=from_fp, to_fp=to_fp)) else: execute_command( "ssh %(to_hostname)s 'rm -rf \"%(to_fp)s\" && mkdir -p \"%(to_parent)s\"' && scp -r \"%(from_fp)s\" %(to_hostname)s:\"%(to_fp)s\"" % \ dict(from_fp=from_fp, to_fp=to_fp, to_hostname=to_hostname, to_parent=to_parent)) elif to_hostname in ['localhost', 'ec2', 'workstation', 'ec2scratch']: # download if from_hostname in ['s3', 's3raw']: # Clear existing folder/file if not include_only and not includes and not exclude_only: execute_command( 'rm -rf \"%(to_fp)s\" && mkdir -p \"%(to_parent)s\"' % dict(to_parent=to_parent, to_fp=to_fp)) # Download from S3 using aws commandline interface. if is_dir: if includes is not None: execute_command( 'aws s3 cp --recursive \"s3://%(from_fp)s\" \"%(to_fp)s\" --exclude \"*\" %(includes_str)s' % dict(from_fp=from_fp, to_fp=to_fp, includes_str=" ".join( ['--include ' + incl for incl in includes]))) elif include_only is not None: execute_command( 'aws s3 cp --recursive \"s3://%(from_fp)s\" \"%(to_fp)s\" --exclude \"*\" --include \"%(include)s\"' % dict(from_fp=from_fp, to_fp=to_fp, include=include_only)) elif exclude_only is not None: execute_command( 'aws s3 cp --recursive \"s3://%(from_fp)s\" \"%(to_fp)s\" --include \"*\" --exclude \"%(exclude)s\"' % dict(from_fp=from_fp, to_fp=to_fp, exclude=exclude_only)) else: execute_command( 'aws s3 cp --recursive \"s3://%(from_fp)s\" \"%(to_fp)s\"' % dict(from_fp=from_fp, to_fp=to_fp)) else: execute_command( 'aws s3 cp \"s3://%(from_fp)s\" \"%(to_fp)s\"' % dict(from_fp=from_fp, to_fp=to_fp)) else: execute_command( "scp -r %(from_hostname)s:\"%(from_fp)s\" \"%(to_fp)s\"" % dict(from_fp=from_fp, to_fp=to_fp, from_hostname=from_hostname)) else: # log onto another machine and perform upload from there. execute_command( "ssh %(from_hostname)s \"ssh %(to_hostname)s \'rm -rf \"%(to_fp)s\" && mkdir -p %(to_parent)s && scp -r \"%(from_fp)s\" %(to_hostname)s:\"%(to_fp)s\"\'\"" % \ dict(from_fp=from_fp, to_fp=to_fp, from_hostname=from_hostname, to_hostname=to_hostname, to_parent=to_parent))
"""Align consecutive images. Possible bad alignment pairs are written into a separate file. Usage 1: align_compose.py in.ini --op from_none_to_aligned """) parser.add_argument("input_spec", type=str, help="input specifier. ini") parser.add_argument("--op", type=str, help="operation id") args = parser.parse_args() print('args', args) input_spec = load_ini(args.input_spec) stack = input_spec['stack'] sqlController = SqlController() fileLocationManager = FileLocationManager(stack) execute_command('python align_v3.py %s --op %s' % (args.input_spec, args.op)) # execute_command('python compose_v3.py %s --op %s' % (args.input_spec, args.op)) #image_name_list = input_spec['image_name_list'] image_name_list = sqlController.get_image_list(stack, 'destination') #if image_name_list == 'all': # #image_name_list = DataManager.load_sorted_filenames(stack=stack)[0].keys() # image_name_list = map(lambda x: x[0], sorted(DataManager.load_sorted_filenames(stack=input_spec['stack'])[0].items(), key=lambda x: x[1])) #op = load_ini(os.path.join(DATA_ROOTDIR, 'CSHL_data_processed', input_spec['stack'], 'operation_configs', args.op + '.ini')) op = load_ini( os.path.join(fileLocationManager.operation_configs, args.op + '.ini')) assert op['type'] == 'warp', "Op type must be warp." assert op['base_prep_id'] == input_spec[ 'prep_id'], "Op requires %s, but input has prep %s." % ( op['base_prep_id'], input_spec['prep_id'])
print('do we ever get here?????????????') prev_img_name = kwarg['prev_img_name'] curr_img_name = kwarg['curr_img_name'] prev_fp = kwarg['prev_fp'] curr_fp = kwarg['curr_fp'] output_subdir = os.path.join(output_dir, curr_img_name + '_to_' + prev_img_name) if os.path.exists(output_subdir) and 'TransformParameters.0.txt' in os.listdir(output_subdir): sys.stderr.write('Result for aligning %s to %s already exists.\n' % (curr_img_name, prev_img_name)) if not regenerate: sys.stderr.write('Skip.\n' % (curr_img_name, prev_img_name)) continue execute_command('rm -rf \"%s\"' % output_subdir) create_if_not_exists(output_subdir) """ ret = execute_command('%(elastix_bin)s -f \"%(fixed_fp)s\" -m \"%(moving_fp)s\" -out \"%(output_subdir)s\" -p \"%(param_fp)s\"' % \ {'elastix_bin': ELASTIX_BIN, 'param_fp': param_fp, 'output_subdir': output_subdir, 'fixed_fp': prev_fp, 'moving_fp': curr_fp }) """ command = [ELASTIX_BIN, '-f', prev_fp, '-m', curr_fp, '-p', param_fp, '-out', output_subdir] print(" ".join(command)) ret = subprocess.run(command)
'w': str(w), 'h': str(h) } elif op_type == 'rotate': #print op_params #print orientation_argparse_str_to_imagemagick_str.keys() #print orientation_argparse_str_to_imagemagick_str[op_params] #print op_str op_str += ' ' + orientation_argparse_str_to_imagemagick_str[ op_params] else: raise Exception("Op_id must be either warp or crop.") assert args.input_fp is not None and args.output_fp is not None input_fp = args.input_fp output_fp = args.output_fp create_parent_dir_if_not_exists(output_fp) try: execute_command("convert \"%(input_fp)s\" +repage -virtual-pixel background -background %(bg_color)s %(op_str)s -flatten -compress lzw \"%(output_fp)s\"" % \ {'op_str': op_str, 'input_fp': input_fp, 'output_fp': output_fp, 'bg_color': pad_color}) except Exception as e: sys.stderr.write("ImageMagick convert failed for input_fp %s: %s\n" % (input_fp, e.message))