def quick_hdu_list(i): #READONLY assert isinstance(i, int) assert i in ALL_CHANNELS assert i in GOOD_CHANNELS isolate.SILENT = True xyt_filename = prefix + str(i) + insuffix assert isolate.is_xyt_filename(xyt_filename) filaments_filename = isolate.filament_filename_from_xyt_filename(xyt_filename) assert isolate.is_filament_filename(filaments_filename) isolate.SILENT = False return config.default_open(filaments_filename, mode='readonly', memmap=True, checksum=False)
def update_all_keys(filaments_filename, force=False): # Updates the properties corresponding to all known sources def do_update_all(hdu_list, force): if not SILENT: print "Updating all keys in", hdu_list.filename() if not force: print "update_all_keys() is a long operation that involves everything in config.sources..." if "y" not in raw_input("Continue? ([no]/yes): "): return "Aborted: update_all_keys()" exceptions = "" for key in config.sources.iterkeys(): # TODO NOT NECESSARILY VALID! try: update_key(hdu_list, key, force=force) # TODO Don't need to do anything with the output? except Exception as e: print e if len(exceptions) == 0: exceptions += "except " + key else: exceptions += ", " + key if len(exceptions) > 0 and string.count(exceptions, ",") > 0: exceptions = string.join(string.rsplit(exceptions, ",", 1), " and") return "All properties " + exceptions + "are now up to date in " + hdu_list.filename() if is_filament_filename(filaments_filename): # with config.default_open(filaments_filename, mode='update') as hdu_list: hdu_list = config.default_open(filaments_filename, mode="update") print do_update_all(hdu_list, force) return filaments_filename elif is_filament_hdu_list(filaments_filename): hdu_list = filaments_filename assert hdu_list.fileinfo(0)["filemode"] == "update" print do_update_all(hdu_list, force) return hdu_list, key else: raise ValueError("Unknown input encountered in update_all_key()...")
def update_all_keys(filaments_filename, force=False): #Updates the properties corresponding to all known sources def do_update_all(hdu_list, force): if not SILENT: print 'Updating all keys in', hdu_list.filename() if not force: print 'update_all_keys() is a long operation that involves everything in config.sources...' if 'y' not in raw_input('Continue? ([no]/yes): '): return 'Aborted: update_all_keys()' exceptions = '' for key in config.sources.iterkeys(): #TODO NOT NECESSARILY VALID! try: update_key(hdu_list, key, force=force) #TODO Don't need to do anything with the output? except Exception as e: print e if len(exceptions) == 0: exceptions += 'except '+key else: exceptions += ', '+key if len(exceptions) > 0 and string.count(exceptions, ',') > 0: exceptions = string.join(string.rsplit(exceptions, ',', 1), ' and') return 'All properties '+exceptions+'are now up to date in '+hdu_list.filename() if is_filament_filename(filaments_filename): #with config.default_open(filaments_filename, mode='update') as hdu_list: hdu_list = config.default_open(filaments_filename, mode='update') print do_update_all(hdu_list, force) return filaments_filename elif is_filament_hdu_list(filaments_filename): hdu_list = filaments_filename assert hdu_list.fileinfo(0)['filemode'] == 'update' print do_update_all(hdu_list, force) return hdu_list, key else: raise ValueError('Unknown input encountered in update_all_key()...')
def update_onoff_key(filaments_filename, key, force=False): # Computes the value of the dataset indicated by key (or correlation_data) for each filament # AVERAGES THE ON AND OFF REGIONS, THEN SUBTRACTS THEM # Saves the value to the associated header entry of each filament assert isinstance(key, str) def do_update(hdu_list, key, force): suffix = "_ONOFF" if not SILENT: print "Updating key:", key + suffix, "in", hdu_list.filename() if not force and key + suffix in hdu_list[skip].header: print key + suffix + " keyword found in filament header..." if "y" not in raw_input("Overwrite? ([no]/yes): "): return hdu_list, key if key in config.source_data: correlation_data = config.source_data[key] else: raise KeyError("No source_data for key: " + key + " in config.source_data") # print key, key+suffix, correlation_data.shape N = len(hdu_list) for i, hdu in enumerate(hdu_list[skip:]): rht.update_progress(float(i / N), message=key + suffix + ": " + str(i)) hdr = hdu.header if ( not force and (key + suffix + "_AVG" in hdr) and (key + suffix + "_MED" in hdr) and (key + suffix + "_TOT" in hdr) and (key + suffix in hdr) ): continue ONMASK, OFFMASK, LL, UR = config.Cloud.on_and_off_masks_from_HDU( hdu, transpose=True, shape=correlation_data.shape ) # mask_slice = np.s_[hdr['MIN_Y']:hdr['MAX_Y']+1, hdr['MIN_X']:hdr['MAX_X']+1] mask_slice = np.s_[LL[1] : UR[1] + 1, LL[0] : UR[0] + 1] inset = correlation_data[mask_slice] on_nonzero = np.nonzero(ONMASK) off_nonzero = np.nonzero(OFFMASK) on_avg = np.nanmean(inset[on_nonzero]) off_avg = np.nanmean(inset[off_nonzero]) on_med = scipy.stats.nanmedian(inset[on_nonzero]) off_med = scipy.stats.nanmedian(inset[off_nonzero]) hdr[key + suffix + "_AVG"] = float(on_avg - off_avg) hdr[key + suffix + "_MED"] = float(on_med - off_med) hdr[key + suffix + "_TOT"] = hdr[key + suffix + "_AVG"] * hdr["LITPIX"] hdr[key + suffix] = config.timestamp() hdu_list.flush() rht.update_progress(1.0) return if is_filament_filename(filaments_filename): # with config.default_open(filaments_filename, mode='update') as hdu_list: hdu_list = config.default_open(filaments_filename, mode="update") do_update(hdu_list, key, force) hdu_list.close() return filaments_filename elif is_filament_hdu_list(filaments_filename): hdu_list = filaments_filename assert hdu_list.fileinfo(0)["filemode"] == "update" do_update(hdu_list, key, force) return hdu_list, key else: raise ValueError("Unknown input encountered in update_onoff_key()...")
def isolate_all(xyt_filename, BINS=6, force=False, sparse=False): filaments_filename = filament_filename_from_xyt_filename(xyt_filename) # Assertions inside function if not SILENT: print "Isolating filaments from:", xyt_filename if not force and os.path.isfile(filaments_filename): if SILENT: return filaments_filename else: print "Filaments already saved as", filaments_filename if "y" not in raw_input("Run isolate_all() anyway? ([no]/yes): "): print "Aborted: isolate_all()" return filaments_filename hdu_list = config.default_open(xyt_filename) ntheta = hdu_list[0].header["NTHETA"] wlen = hdu_list[0].header["WLEN"] frac = hdu_list[0].header["FRAC"] naxis1 = hdu_list[0].header["NAXIS1"] naxis2 = hdu_list[0].header["NAXIS2"] original = hdu_list[0].header["ORIGINAL"] Hi = hdu_list[1].data["hi"] Hj = hdu_list[1].data["hj"] # Compute TheteRHT for all pixels given, then bin by theta B = map(rht.theta_rht, hdu_list[1].data["hthets"]) # List of theta_rht values C = np.multiply(np.asarray(B), BINS / np.pi).astype(np.int_) del B # Ready the output HDUList and close the input HDUList output_hdulist = fits.HDUList(hdu_list[0].copy()) # , open(filaments_filename, 'w')) #Overwrites hdu_list.close() # Set Assignment # unprocessed = list() list_of_HDUs = list() search_pattern = [ (-1, -1), (-1, 0), (-1, 1), (0, -1), ] # [(-1, 1), (-1,-1), (-1, 0), (0, -1), (-2, -2), (-2, -1), (-2, 0), (-2, 1), (-2, 2), (-1, -2), (-1, 2), (0,-2)] for bin in range(BINS): delimiter = np.nonzero(C == bin)[0] raw_points = zip(Hi[delimiter], Hj[delimiter]) del delimiter problem_size = len(raw_points) # message='Step '+str(bin+1)+'/'+str(BINS)+': (N='+str(problem_size)+')' # progress_bar = Progress(problem_size, message=message, incrementing=True) point_dict = dict([x[::-1] for x in enumerate(raw_points)]) set_dict = collections.defaultdict(list) # theta_dict = dict() for coord in raw_points: # rht.update_progress(0.3*(i/problem_size), message=message) # progress_bar.update() # theta_dict[coord] = B[point_dict[coord]] for rel_coord in search_pattern: try: j = point_dict[config.rel_add(coord, rel_coord)] set_dict[point_dict[coord]].append(j) except Exception: continue G = nx.from_dict_of_lists(set_dict) # Undirected graph made using set_dict as an adjacency list del set_dict # progress_bar = Progress(problem_size, message=message, incrementing=False) sources = range(problem_size) flags = np.ones((problem_size), dtype=np.int_) while len(sources) > 0: source = sources.pop() if not flags[source]: continue else: # rht.update_progress(0.3+0.3*(1.0-len(sources)/problem_size), message=message) # progress_bar.update(len(sources)) try: for member in nx.descendants(G, source): flags[member] = False point_dict[raw_points[member]] = source G.remove_node(member) # TODO Remove members from G if that would speed up subsequent calls? except nx.NetworkXError: # Assume we hit an isolated pixel (never made it into G) and move on pass del sources, flags, G histogram = np.bincount(map(point_dict.get, raw_points)) mask = np.nonzero(histogram >= int(frac * wlen))[0] del histogram # progress_bar = Progress(problem_size, message=message, incrementing=False) mask_dict = dict([x[::-1] for x in enumerate(mask)]) out_clouds = collections.defaultdict(list) while len(point_dict) > 0: temp = point_dict.popitem() try: # Keying into mask_dict is the only operation that ought to throw an exception out_clouds[mask_dict[temp[1]]].append(temp[0]) # progress_bar.update(len(point_dict)) # rht.update_progress(0.6+0.399*(1.0-len(point_dict)/problem_size), message=message) except Exception: continue while len(out_clouds) > 0: cloud = out_clouds.popitem()[1] # unprocessed.append(cloud) list_of_HDUs.append(config.Cloud(cloud).as_HDU(sparse=sparse)) # TODO Incorporate theta_dict # rht.update_progress(1.0, final_message='Finished joining '+str(problem_size)+' points! Time Elapsed:') # Convert lists of two-integer tuples into ImageHDUs # unprocessed.sort(key=len, reverse=True) # output_hdulist = fits.HDUList(map(config.Cloud.as_ImageHDU, map(config.Cloud, unprocessed))) # del unprocessed list_of_HDUs.sort(key=lambda h: h.header["DIAG"], reverse=False) while len(list_of_HDUs) > 0: output_hdulist.append(list_of_HDUs.pop()) # Output HDUList to File output_hdulist.writeto(filaments_filename, output_verify="silentfix", clobber=True, checksum=True) try: output_hdulist.flush() except Exception: pass try: output_hdulist.close() except Exception: pass if not SILENT: print "Results successfully output to " + filaments_filename return filaments_filename
def plot(filaments_filename, key=None, out_name=None, show=True, cut=config.passive_constant(True)): show &= not SILENT if not show and out_name is None: if not SILENT: raise ValueError("Unable to plot data without showing or saving it in isolate.plot()") else: return def do_plot(hdu_list, key, out_name, show, cut): filaments_filename = hdu_list.filename() if not SILENT: print "Plotting key:", key, "in", filaments_filename backproj = hdu_list.pop(0) # TODO make sure to pop skip hdus!s hdu_list = filter(cut, hdu_list) # TODO TURNS AN HDULIST INTO A LIST ~_~ """ try: """ if key is None: figure_args = {"figsize": (8, 6), "facecolor": "white", "dpi": 250} fig = plt.figure(**figure_args) # display = np.zeros((backproj.header['NAXIS1'], backproj.header['NAXIS2'])) display = np.zeros_like(backproj.data) display.fill(np.nan) N = len(hdu_list) for i, hdu in enumerate(hdu_list): hdr = hdu.header display[hdr["MIN_X"] : hdr["MAX_X"] + 1, hdr["MIN_Y"] : hdr["MAX_Y"] + 1][ config.Cloud.nonzero_data_from_HDU(hdu, transpose=True) ] = (N - i) ax1 = fig.add_axes([0.05, 0.05, 0.90, 0.90]) ax1.imshow(display) # .T) key = "Filaments" elif isinstance(key, str) and (key == "" or key in hdu_list[0].header): # skip].header: displays = dict() datasets = dict() titles = [key + suffix for suffix in config.applicable_methods[key]] # for suffix in suffixes: for title in titles: displays[title] = np.zeros_like(backproj.data) displays[title].fill(np.nan) datasets[title] = config.post_processing[key]( [hdu.header[title] for hdu in hdu_list] ) # TODO watch out for python list handling # for hdu in hdu_list: for i, hdu in enumerate(hdu_list): hdr = hdu.header mask_nonzero = config.Cloud.nonzero_data_from_HDU(hdu, transpose=True) mask_slice = np.s_[hdr["MIN_Y"] : hdr["MAX_Y"] + 1, hdr["MIN_X"] : hdr["MAX_X"] + 1] # for suffix in suffixes: for title in titles: # try: ##mask = displays[key+suffix][hdr['MIN_X']:hdr['MAX_X']+1, hdr['MIN_Y']:hdr['MAX_Y']+1] ##displays[key+suffix][hdr['MIN_Y']:hdr['MAX_Y']+1, hdr['MIN_X']:hdr['MAX_X']+1][config.Cloud.nonzero_data_from_HDU(hdu, transpose=True)].fill(datasets[key+suffix][i]) # thing = displays[title][mask_slice] # other_thing = datasets[title][i] # thing[mask_nonzero] = other_thing #datasets[title][i] #.fill(datasets[title][i]) displays[title][mask_slice][mask_nonzero] = datasets[title][i] # except Exception as e: # print e #TODO # print 'Failed to plot data from (1-indexed) HDU:', i+1+skip, title Nplots = len(titles) figure_args = {"figsize": (8, 2.5 * Nplots - 0.5), "facecolor": "white", "dpi": 250} fig = plt.figure(**figure_args) r = 4 gs = gridspec.GridSpec(Nplots * r, 5 * r) for i, title in enumerate(titles): ax1 = plt.subplot(gs[1 + r * i : r * (i + 1), 0:-4]) ax2 = plt.subplot(gs[1 + r * i : r * (i + 1) - 1, -3:-1]) RANGE = (np.nanmin(datasets[title]), np.nanmax(datasets[title])) ax2.hist(datasets[title], bins=50, range=RANGE, orientation="horizontal", histtype="stepfilled") plt.colorbar(ax1.imshow(displays[title], cmap="YlOrRd"), ax=ax2, fraction=0.10) plt.title(title, fontsize=8) else: raise KeyError("Unable to plot data using the given key: " + str(key)) plt.suptitle(key + " from " + filaments_filename, fontsize=12) if show: plt.show() if out_name is not None and isinstance(out_name, str): plt.savefig(out_name, dpi=500, format="png") """ except Exception as e: print e finally: """ plt.cla() plt.clf() plt.close() if is_filament_filename(filaments_filename): # with config.default_open(filaments_filename, mode='readonly') as hdu_list: hdu_list = config.default_open(filaments_filename, mode="readonly") do_plot(hdu_list, key=key, out_name=out_name, show=show, cut=cut) hdu_list.close() return filaments_filename elif is_filament_hdu_list(filaments_filename): hdu_list = filaments_filename do_plot(hdu_list, key=key, out_name=out_name, show=show, cut=cut) return hdu_list, key else: raise ValueError("Unknown input encountered in plot()...")
def update_key(filaments_filename, key, correlation_data=None, force=False): # Computes the value of the dataset indicated by key (or correlation_data) for each filament # Uses the functions indicated by key from config # Saves the value to the associated header entry of each filament assert isinstance(key, str) def do_update(hdu_list, key, correlation_data, force): if not SILENT: print "Updating key:", key, "in", hdu_list.filename() if not force and (key == "" or key in hdu_list[skip].header): print key + " keyword found in filament header..." if "y" not in raw_input("Overwrite? ([no]/yes): "): return hdu_list, key # correlation_data can be None if and only if the functions you will call expect Cloud objects # else, it must correspond to a source_data and applicable_methods entry if correlation_data is None: if key in config.source_data: correlation_data = config.source_data[key] else: raise KeyError("No source_data for key: " + key + " in config.source_data") if key not in config.applicable_methods: raise KeyError("No applicable_methods for key: " + key + " in config.applicable_methods") # for i, hdu in enumerate(hdu_list[skip:]): for hdu in hdu_list[skip:]: hdr = hdu.header if correlation_data is not None: # Assumes correlation_data can be indexed into using ndarray notation [LowerLeft to UpperRight+1] # Assumes config.Cloud.nonzero_data_from_HDU will return the pixel coords offset properly for the above masked region # Assumes func can take this weird ndarray view as input and return a scalar value for suffix in config.applicable_methods[key]: func = config.methods[suffix] hdr[key + suffix] = func( correlation_data[hdr["MIN_Y"] : hdr["MAX_Y"] + 1, hdr["MIN_X"] : hdr["MAX_X"] + 1][ config.Cloud.nonzero_data_from_HDU(hdu, transpose=True) ] ) hdr[key] = config.timestamp() else: # Assumes all func require a config.Cloud object to work tempCloud = config.Cloud(hdu) for suffix in config.applicable_methods[key]: func = config.Cloud.functions[suffix][0] hdr[key + suffix] = func(tempCloud) hdu_list.flush() if is_filament_filename(filaments_filename): # with config.default_open(filaments_filename, mode='update') as hdu_list: hdu_list = config.default_open(filaments_filename, mode="update") do_update(hdu_list, key, correlation_data, force) hdu_list.close() return filaments_filename elif is_filament_hdu_list(filaments_filename): hdu_list = filaments_filename assert hdu_list.fileinfo(0)["filemode"] == "update" do_update(hdu_list, key, correlation_data, force) return hdu_list, key else: raise ValueError("Unknown input encountered in update_key()...")
print '' print 'Starting onoff.py' print '{' for i in available_integers: print 'Started:', str(i) isolate.SILENT = True xyt_filename = prefix + str(i) + insuffix assert isolate.is_xyt_filename(xyt_filename) filaments_filename = isolate.filament_filename_from_xyt_filename(xyt_filename) #prefix + str(i) + outsuffix assert isolate.is_filament_filename(filaments_filename) #Open the file once, but allow for multiple uses hdu_list = config.default_open(filaments_filename, mode='update') assert isolate.is_filament_hdu_list(hdu_list) _key = 'GALFA'+str(i) config.include(config.GALFAx(i)) isolate.SILENT = False isolate.update_onoff_key(hdu_list, key=_key, force=True) #_____________________________________________________________________________BAD PROGRAMMING NEWKEY = _key+'_ONOFF' #config.applicable_methods[NEWKEY] = ['_AVG', '_MED', '_TOT'] #Jenky config.include({NEWKEY: ( None, config.identity, config.abs_log10,