def fromfits(cls, f): mbsc = cls() sc = np.squeeze(f['ARRAYDATA-MBFITS'].data.DATA) ts_mjd = f['DATAPAR-MBFITS'].data['MJD'] ts_az = f['DATAPAR-MBFITS'].data['AZIMUTH'] ts_el = f['DATAPAR-MBFITS'].data['ELEVATIO'] ts_daz = f['DATAPAR-MBFITS'].data['LONGOFF'] * 3600.0 # arcsec ts_del = f['DATAPAR-MBFITS'].data['LATOFF'] * 3600.0 # arcsec alist = [ts_mjd, ts_az, ts_el, ts_daz, ts_del] names = ['MJD', 'AZ', 'EL', 'DAZ', 'DEL'] dtypes = ['f8', 'f8', 'f8', 'f8', 'f8'] record = np.rec.fromarrays(alist, zip(names, dtypes)) hdu_data = fits.PrimaryHDU(sc) hdu_record = fits.BinTableHDU(record) hdu_data.header['ORIGIN'] = os.path.basename(f.filename()) hdu_data.header['EXTNAME'] = 'DATA' hdu_record.header['EXTNAME'] = 'RECORD' mbsc.append(hdu_data) mbsc.append(hdu_record) return mbsc
def write(self, outfile): """ Write the current object catalog to fits file. """ hdu = pyfits.BinTableHDU(self.data) hdu.writeto(outfile, clobber=True)
def __init__(self, hdu=None): """ An LDAC table can be instantiated either as am empty table or with a pyfits BinaryTable HDU (existing table). """ if hdu == None: self.hdu = pyfits.BinTableHDU() self.hdu.data = None # We make sure that the table has 'some' proper name: self.hdu.name = "DEFAULT" else: self.hdu = hdu
def saveLightCurvesRadial(ra, dec, filename="", radius=10, overwrite=False, skip=False): """ Given one set of coordinates, download all light curves around x arcminutes from the cluster center Parameters ----------- filename : string The name of the output file ra : float, apwlib.geometry.RA A Right Ascension in decimal degrees dec : float, apwlib.geometry.Dec A Declination in decimal degrees radius : float, optional Radius in arcminutes to search for light curves around the given ra, dec overwrite : bool, optional If a pickle exists, do you want to overwrite it? """ ra = c.parseDegrees(ra) dec = c.parseDegrees(dec) radiusDegrees = radius / 60.0 logging.debug("{0},{1} with radius={2} deg".format(ra, dec, radiusDegrees)) if filename.strip() == "": outputFilename = os.path.join( "data", "lightcurves", "{0}_{1}.fits".format(ra.degrees, dec.degrees)) else: outputFilename = filename logging.debug("Output file: {0}".format(outputFilename)) if os.path.exists(outputFilename) and not overwrite: raise IOError("{0} already exists!".format(outputFilename)) elif os.path.exists(outputFilename) and overwrite: logging.debug("You've chosen to overwrite the file!") os.remove(outputFilename) logging.debug("File deleted.") lightCurves = getLightCurvesRadial(ra, dec, radiusDegrees) hdu = pf.BinTableHDU(lightCurves) hdu.writeto(outputFilename) return
def mwrfits(mydict, filename, clobber=False, ascii=False): """ Write an dictionary as a binary table in a FITS file. Shape order is reverted to satisfy the FITS conventions (quick axis is first in FITS and second in python) Arguments --------- mydict: dict of ndarrays filename: string The name of the written fits file. clobber: bool Overwrites the output file if True. Returns -------- Returns nothing. Exceptions --------- ValueError if mydict is not a dict of arrays """ # check that dict contains only arrays for k, v in mydict.iteritems(): if not isinstance(v, np.ndarray): raise ValueError("Expected a dict of arrays.") # convert dict of ndarray as an array of size 1 mydtype = list() for k, v in mydict.iteritems(): mydtype.append((k, str(v.shape) + v.dtype.str)) arr = np.zeros(1, dtype=mydtype) for k, v in mydict.iteritems(): arr[k] = v if ascii: raise NotImplemented() # hdu = pyfits.TableHDU(arr) # not working ! else: hdu = pyfits.BinTableHDU(arr) # shape order is reverted to satisfy the FITS conventions # (quick axis is first in FITS and second in python) for i, v in enumerate(mydict.values()): hdu.header.update('TDIM' + str(i + 1), str(v.shape[::-1])) hdu.writeto(filename, clobber=clobber)
def _get(self, key): '''Attempts to load the right index and populate a new value''' tmp = Cat(self.filename, setup=False) # Convert a single integer key into a slice if isinstance(key,int): if key < 0: key += len(self) key = slice(key,key+1) # tmp.fits = pyfits.new_table(self.data[key].columns) tmp.fits = pyfits.BinTableHDU(self.data[key], header=self.header, name=self.name) tmp.name = self.name tmp.data = tmp.fits.data tmp.columns = tmp.fits.data.columns tmp.header = self.header tmp.names = self.names return tmp
def write_cube(spectra): """Create a FITS file with all spectra written.""" recarr = convert_spectra_to_recarray(spectra) img, img2 = convert_spectra_to_img(spectra, CRVAL1, CRPIX1) f1 = pf.PrimaryHDU(img) f2 = pf.ImageHDU(np.zeros((10, 10))) t3 = pf.BinTableHDU(recarr) f4 = pf.ImageHDU(img2) f5 = pf.ImageHDU(np.zeros((10, 10))) f4.header['CRVAL1'] = CRVAL1 f4.header['CRPIX1'] = -CRPIX1 f4.header['CTYPE1'] = 'WAVE-LOG' f4.header['CUNIT1'] = 'NM' towrite = pf.HDUList([f1, f2, t3, f4, f5]) towrite.writeto('test.fits', clobber=True)
def precomputeCoordinates(infile, outfile): import numpy.lib.recfunctions as rec hdu = pyfits.open(infile) data = hdu[1].data columns = hdu[1].columns names = [n.lower() for n in hdu[1].data.names] if 'glon' not in names and 'glat' not in names: logger.info("Writing 'GLON' and 'GLAT' columns") glon, glat = ugali.utils.projector.celToGal(data['RA'], data['DEC']) out = rec.append_fields(data,['GLON','GLAT'],[glon,glat], usemask=False,asrecarray=True) elif 'ra' not in names and 'dec' not in names: logger.info("Writing 'RA' and 'DEC' columns") ra, dec = ugali.utils.projector.galToCel(data['GLAT'], data['GLON']) out = rec.append_fields(data,['RA','DEC'],[ra,dec], usemask=False,asrecarray=True) hdu_out = pyfits.BinTableHDU(out) hdu_out.writeto(outfile, clobber=True)
def select_rows(tbhdu, indices): """ Read rows(indexes) from given HDU (catalog) A new table with only the asked table indexes, 'indices', is output. Input: - tbhdu [BinTableHDU] : FITS table HDU - indices [int,] : List of indexes to read from tbhdu Output: -> (new) BinTableHDU : sub-selection (rows) of tbhdu --- """ data = tbhdu.data.take(indices); return pyfits.BinTableHDU(data);
def _parsercp(rcpfile): with open(rcpfile) as f: d = json.load(f, object_pairs_hook=OrderedDict) pixel = np.array(d.keys(), int) g_pnt = np.array([d[key]['Gain_point'] for key in d], float) g_ext = np.array([d[key]['Gain_extended'] for key in d], float) p_daz = -np.array([d[key]['dAz'] for key in d], float) p_del = -np.array([d[key]['dEl'] for key in d], float) alist = [pixel, p_daz, p_del, g_pnt, g_ext] names = ['PIXEL', 'DAZ', 'DEL', 'GAIN_PNT', 'GAIN_EXT'] dtypes = ['i8', 'f8', 'f8', 'f8', 'f8'] rcp = np.rec.fromarrays(alist, zip(names, dtypes)) hdu_rcp = fits.BinTableHDU(rcp) hdu_rcp.header['EXTNAME'] = 'RCP' hdu_rcp.header['ORIGIN'] = os.path.basename(rcpfile) return hdu_rcp
def bootstrap_realization(realization, filename, photonList, photonData, counts): """Create random samples of the photon list and write them to fits files""" np.random.seed() indices = np.random.randint(0, counts, size=counts) photons = np.sort(photonList[indices]) bootstrapHDU = pyfits.BinTableHDU(photonData[1].data[photons], photonData[1].header) bootstrapHDU.header[NAXIS2] = counts bootstrapList = pyfits.HDUList( [photonData[0], bootstrapHDU, photonData[2]]) # Here we assume filename ends in .fits fn, ext = os.path.splitext(filename) outfile = '%s_bs_%d%s' % (fn, realization, ext) if os.path.isfile(outfile): os.remove(outfile) bootstrapList.writeto(outfile)
def writein(self, fitsname): f = pyfits.open(fitsname) newcol = pyfits.Column(name='EFFICIENCY_PARS', format='6E', array=self.final_results) newdata = pyfits.FITS_rec.from_columns([newcol]) table_hdu = pyfits.BinTableHDU(newdata, name='EFFICIENCY_PARAMS') headkeys = [] for key in [ 'EXTNAME', 'TELESCOP', 'INSTRUME', 'FILTER', 'HDUCLASS', 'HDUCLAS1', 'HDUCLAS2', 'HDUVERS', 'CCLS0001', 'CDTP0001', 'CCNM0001', 'CBD10001', 'CBD20001', 'CBD30001', 'CBD40001', 'CBD50001', 'CBD60001', 'CBD70001', 'CBD80001', 'CBD90001', 'CVSD0001', 'CVST0001', 'CDES0001' ]: headkeys.append([key, f[3].header[key]]) f[3] = table_hdu for pars in headkeys: f[3].header[pars[0]] = pars[1] f.writeto(fitsname, clobber=True, checksum=True) f.close()
def write_fits(self): """ Write the FITS log """ import time import getpass formats = {} formats['bool'] = 'L' formats['int16'] = 'I' formats['int32'] = 'J' formats['int64'] = 'K' formats['float32'] = 'E' formats['float64'] = 'D' formats['>i8'] = 'K' formats['>f8'] = 'D' #### Make the table columns, translating numpy data types to "TFORM" coldefs = [] TFORM = 'A' + str(np.array(self.images).dtype).split('S')[1] coldefs.append( pyfits.Column(name='images', array=np.array(self.images), format=TFORM)) for column in self.params.keys(): if column == 'comment': coldata = np.array(self.params['comment']) else: coldata = self.params[column] # dtype = str(coldata.dtype) #print column, dtype if dtype in formats.keys(): TFORM = formats[dtype] else: if 'S' not in dtype: print 'Unrecognized data type in: %s' % (dtype) return False # TFORM = 'A' + dtype.split('S')[1] # #data = self.params[column] if '>' in dtype: cast_types = {'>i8': np.int64, '>f8': np.float64} coldata = np.cast[cast_types[dtype]](coldata) # coldefs.append( pyfits.Column(name=column, array=coldata, format=TFORM)) #### Done, now make the binary table if pyfits.__version__ < '3.3': tbhdu = pyfits.new_table(coldefs) else: tbhdu = pyfits.BinTableHDU().from_columns(coldefs) linehdu = pyfits.ImageHDU(data=self.marked_lines, name='LINELIST') #### Primary HDU hdu = pyfits.PrimaryHDU() thdulist = pyfits.HDUList([hdu, tbhdu, linehdu]) #### Add modification time of "infile" to FITS header infile_mod_time = time.strftime( "%m/%d/%Y %I:%M:%S %p", time.localtime()) # os.path.getmtime(self.filename))) if pyfits.__version__ < '3.3': thdulist[0].header.update('MODTIME', infile_mod_time) thdulist[0].header.update('USER', getpass.getuser()) else: thdulist[0].header['MODTIME'] = infile_mod_time thdulist[0].header['USER'] = getpass.getuser() thdulist.writeto(self.logfile, clobber=True) print 'Log to file %s' % (self.logfile)
new.write(DirIni + '/' + nameOutput) else: print 'Merged file already found.' #Generate tables for each tomographic bin #TomoBin=[0.101, 0.301, 0.501, 0.701, 0.901, 10] TomoBin = [0.101, 0.301, 0.501, 0.701, 0.901, 1.201, 2.001] print 'Splitting the merged catalogue into different tomographic bins' for kk in range(0, len(TomoBin) - 1): t = pyfits.open(DirIni + '/' + nameOutput) tbdata = t[1].data ## Old 4-band photometry mask = (tbdata['ZB4_in'] >= TomoBin[kk]) & (tbdata['ZB4_in'] < TomoBin[kk + 1]) newtbdata = tbdata[mask] hdu = pyfits.BinTableHDU(data=newtbdata) print DirIni, kk hdu.writeto(DirIni + '/' + 'MasterCat_Tomo4Bin_' + str(kk + 1) + '.fits', clobber=True) ## New 9-bandp photometry mask = (tbdata['ZB9_in'] >= TomoBin[kk]) & (tbdata['ZB9_in'] < TomoBin[kk + 1]) newtbdata = tbdata[mask] hdu = pyfits.BinTableHDU(data=newtbdata) print DirIni, kk hdu.writeto(DirIni + '/' + 'MasterCat_Tomo9Bin_' + str(kk + 1) + '.fits', clobber=True)
def main(rerun, dataIds, fakes, root='/lustre/Subaru/SSP', rad=10): doCoadd = 'tract' in dataIds[0].keys() butler = dafPer.Butler(os.path.join(root, "rerun", rerun)) #read in fits file, replace with txt file or anything else fits = pyfits.open(fakes) data = fits[1].data radecCat = loadRaDec(data) ndata = len(data) datamask = np.ones(ndata, dtype=bool) ids = data["ID"] if "ID" in data.names else range(len(data)) idDict = dict(zip(ids, xrange(ndata))) for dataId in dataIds: print dataId try: sources = butler.get('deepCoadd_src' if doCoadd else 'src', dataId, immediate=True, flags=afwTable.SOURCE_IO_NO_FOOTPRINTS) cal_md = butler.get('deepCoadd_md' if doCoadd else 'calexp_md', dataId, immediate=True) calexp = butler.get('deepCoadd' if doCoadd else 'calexp', dataId, immediate=True) except: print "skipping", dataId continue if False: matches = afwTable.matchRaDec(sources, radecCat, 3.3 * afwGeom.arcseconds) for (src, fake, d) in matches: datamask[idDict[fake.getId()]] = False msk = calexp.getMaskedImage().getMask() detected = msk.clone() detected &= msk.getPlaneBitMask("DETECTED") wcs = calexp.getWcs() count, good_count = 0, 0 for i_d, datum in enumerate(radecCat): pixCoord = afwGeom.Point2I(wcs.skyToPixel(datum.getCoord())) pixBox = afwGeom.BoxI(pixCoord, afwGeom.Extent2I(1, 1)) pixBox.grow(rad) pixBox.clip(calexp.getBBox(afwImage.PARENT)) if pixBox.isEmpty(): continue else: count += 1 subMask = afwImage.MaskU(detected, pixBox, afwImage.PARENT) if sum(subMask.getArray().ravel()) != 0: datamask[i_d] = False else: good_count += 1 print count, good_count newdata = data[datamask] print ndata, len(newdata) hdu = pyfits.BinTableHDU(newdata) hdu.writeto('blank_sources.fits', clobber=True)
def solve(args): """ Calculate posterior distributions for model parameters given the data. """ # Make some checks _check_analysis_args(args) # Load the model and the data model = sick.models.Model(args.model) all_spectra = _parse_and_load_spectra(args) # Display some information about the model logger.info("Model information: {0}".format(model)) logger.info("Configuration:") map(logger.info, yaml.dump(model.configuration).split("\n")) # Define headers that we want in the results filename default_headers = ( "RA", "DEC", "COMMENT", "ELAPSED", "FIBRE_NUM", "LAT_OBS", "LONG_OBS", "MAGNITUDE", "NAME", "OBJECT", "RO_GAIN", "RO_NOISE", "UTEND", "UTDATE", "UTSTART", ) default_metadata = { "model": model.hash, "input_filenames": ", ".join(args.spectrum_filenames), "sick_version": sick.__version__, } if args.read_from_filename: with open(args.spectrum_filenames[0], "r") as fp: all_filenames = map(str.strip, fp.readlines()) # For each source, solve for i, spectra in enumerate(all_spectra, start=1): # Force spectra as a list if not isinstance(spectra, (list, tuple)): spectra = [spectra] logger.info("Starting on object #{0} (RA {1}, DEC {2} -- {3})".format( i, spectra[0].headers.get("RA", "None"), spectra[0].headers.get("DEC", "None"), spectra[0].headers.get("OBJECT", "Unknown"))) # Create metadata and put header information in if args.skip > i - 1: logger.info("Skipping object #{0}".format(i)) continue if args.number_to_solve != "all" and i > (int(args.number_to_solve) + args.skip): logger.info("We have analysed {0} spectra. Exiting..".format( args.number_to_solve)) break # If there are many spectra to analyse, include the run ID in the output filenames. # Update filename prefix if we are reading from a file if args.read_from_filename: filename_prefix = sick.utils.default_output_prefix( all_filenames[i].split()) else: filename_prefix = args.filename_prefix if len(all_spectra) > 1 and not args.read_from_filename: output = lambda x: os.path.join( args.output_dir, "-".join([filename_prefix, str(i), x])) else: output = lambda x: os.path.join(args.output_dir, "-".join( [filename_prefix, x])) # Does a solution already exist for this star? If so are we authorised to clobber it? if os.path.exists(output("result.json")): if not args.clobber: logger.info("Skipping object #{0} as a results file already exists"\ " ({1}) and we have been asked not to clobber it".format(i, output("result.json"))) continue else: logger.warn("Overwriting existing file {0}".format( output("result.json"))) metadata = {} header_columns = [] for header in default_headers: if header not in spectra[0].headers: continue header_columns.append(header) metadata[header] = spectra[0].headers[header] metadata.update({"run_id": i}) metadata.update(default_metadata) # Determine an initial point try: initial_theta, initial_r_chi_sq = model.initial_theta(spectra) except: logger.exception("Failed to get initial point") if args.debug: raise continue logger.info("Initial theta point with reduced chi_sq = {0:.2f} is {1}"\ .format(initial_r_chi_sq, model._dictify_theta(initial_theta))) # Save metadata about the initial point metadata["initial_theta"] = model._dictify_theta(initial_theta) metadata["initial_r_chi_sq"] = initial_r_chi_sq # Produce a plot projecting the initial value if args.plotting: projected_filename = output("projected-initial-theta.{}".format( args.plot_format)) fig = sick.plot.projection(model, spectra, theta=initial_theta) fig.savefig(projected_filename) logger.info("Created figure {}".format(projected_filename)) # Optimise the point if model.configuration["settings"]["optimise"]: optimised_theta, optimised_r_chi_sq, info = model.optimise( spectra, initial_theta=initial_theta, fixed=["z"] + ["z.{}".format(c) for c in model.channels]) logger.info("Optimised theta is {0}".format( model._dictify_theta(optimised_theta))) walker_theta = optimised_theta # Save metadata about the optimised value metadata["optimised_theta"] = model._dictify_theta(optimised_theta) metadata["optimised_r_chi_sq"] = optimised_r_chi_sq if args.plotting: projected_filename = output( "projected-optimised-theta.{}".format(args.plot_format)) fig = sick.plot.projection(model, spectra, theta=optimised_theta) fig.savefig(projected_filename) logger.info("Created figure {}".format(projected_filename)) else: # MCMC initial point will be the initial point walker_theta = initial_theta try: posteriors, sampler, info = model.infer(spectra, theta=walker_theta) except: logger.exception("Failed to analyse source #{0}:".format(i)) if args.debug: raise else: # Update results with the posteriors logger.info("Posteriors:") max_parameter_len = max(map(len, model.parameters)) for parameter in model.parameters: posterior_value, pos_uncertainty, neg_uncertainty = posteriors[ parameter] logger.info(" {0}: {1:.2e} ({2:+.2e}, {3:+.2e})".format( parameter.rjust(max_parameter_len), posterior_value, pos_uncertainty, neg_uncertainty)) metadata.update({ parameter: posterior_value, "u_maxabs_{0}".format(parameter): np.abs([neg_uncertainty, pos_uncertainty]).max(), "u_pos_{0}".format(parameter): pos_uncertainty, "u_neg_{0}".format(parameter): neg_uncertainty, }) # Save information related to the analysis metadata.update( dict( zip(["mean_snr_{}".format(c) for c in model.channels], [ np.nanmean(s.flux / (s.variance**0.5)) for s in spectra ]))) chain_filename = output("chain.fits") metadata.update({ "reduced_chi_sq": info["reduced_chi_sq"], "maximum_log_probability": np.nanmax(info["lnprobability"]), "chain_filename": chain_filename if args.save_chain_files else "", "time_elapsed": info["time_elapsed"], "final_mean_acceptance_fraction": info["mean_acceptance_fractions"][-1], "model_configuration": model.configuration }) for channel, length in info["autocorrelation_times"].iteritems(): metadata["tau_{}".format(channel)] = length walkers = model.configuration["settings"]["walkers"] chain_length = info["chain"].shape[0] * info["chain"].shape[1] chain = np.core.records.fromarrays( np.vstack([ np.arange(1, 1 + chain_length), np.arange(1, 1 + chain_length) % walkers, info["chain"].reshape(-1, len(model.parameters)).T, info["lnprobability"].reshape(-1, 1).T ]), names=["Iteration", "Sample"] + model.parameters + ["ln_probability"], formats=["i4", "i4"] + ["f8"] * (1 + len(model.parameters))) # Save the chain if args.save_chain_files: logger.info("Saving chains to {0}".format(chain_filename)) primary_hdu = pyfits.PrimaryHDU() table_hdu = pyfits.BinTableHDU(chain) hdulist = pyfits.HDUList([primary_hdu, table_hdu]) hdulist.writeto(chain_filename, clobber=True) else: logger.warn("Chain not saved to disk.") # Write the result to disk logger.info("Saving results to {0}".format(output("result.json"))) with open(output("result.json"), "wb+") as fp: json.dump(metadata, fp, indent=2) # Close sampler pool if model.configuration["settings"].get("threads", 1) > 1: sampler.pool.close() sampler.pool.join() # Save sampler state with open(output("model.state"), "wb+") as fp: pickle.dump([ sampler.chain[:, -1, :], sampler.lnprobability[:, -1], sampler.random_state ], fp, -1) # Plot results if args.plotting: # Plot the mean acceptance fractions acceptance_plot_filename = output("acceptance.{0}".format( args.plot_format)) fig = sick.plot.acceptance_fractions( info["mean_acceptance_fractions"], burn_in=model.configuration["settings"]["burn"]) fig.savefig(acceptance_plot_filename) logger.info( "Created figure {0}".format(acceptance_plot_filename)) # Plot the chains chain_plot_filename = output("chain.{0}".format( args.plot_format)) fig = sick.plot.chains( info["chain"], labels=sick.utils.latexify(model.parameters), truth_color='r', burn_in=model.configuration["settings"]["burn"], truths=[posteriors[p][0] for p in model.parameters]) fig.savefig(chain_plot_filename) logger.info("Created figure {0}".format(chain_plot_filename)) # Make a corner plot with just the astrophysical parameters corner_plot_filename = output("corner.{0}".format( args.plot_format)) indices = np.arange(len(model.grid_points.dtype.names)) fig = sick.plot.corner( sampler.chain.reshape(-1, len(model.parameters))[:, indices], labels=sick.utils.latexify(model.grid_points.dtype.names), truth_color='r', quantiles=[.16, .50, .84], verbose=False, truths=[ posteriors[p][0] for p in model.grid_points.dtype.names ]) fig.savefig(corner_plot_filename) logger.info("Created figure {0}".format(corner_plot_filename)) # Plot the autocorrelation autocorrelation_filename = output( "auto-correlation.{0}".format(args.plot_format)) fig = sick.plot.autocorrelation(sampler.chain) fig.savefig(autocorrelation_filename) logger.info( "Created figure {0}".format(autocorrelation_filename)) # Plot some spectra pp_spectra_plot_filename = output( "projected-spectra.{0}".format(args.plot_format)) fig = sick.plot.projection(model, spectra, chain=sampler.chain) fig.savefig(pp_spectra_plot_filename) logger.info( "Created figure {0}".format(pp_spectra_plot_filename)) # Closing the figures isn't enough; matplotlib leaks memory plt.close("all") # Delete some things del sampler, chain if args.save_chain_files: del primary_hdu, table_hdu, hdulist logger.info("Fin.") return True
datadir = '/work/bow/data/' #file = ['lat_photon_weekly_w0%d_p302_v001.fits'%(i+10) for i in range(90)] file = ['lat_photon_weekly_w%d_p302_v001.fits'%(i) for i in range(100,400)] #file = pyfits.open(datadir+'lat_photon_weekly_w200_p302_v001.fits') #photon = file[1].data for f in range(len(file)): fileph = pyfits.open(datadir+file[f]) photon = fileph[1].data energy = photon.field('ENERGY') > 8000 energy_cut = photon[energy] print len(energy_cut) zenith = energy_cut.field('ZENITH_ANGLE') > 100 zenith_cut = energy_cut[zenith] print len(zenith_cut) hdu = pyfits.BinTableHDU(zenith_cut) #hdu.writeto('select_photon_w%d.fits'%(f+10)) hdu.writeto('select_photon_w%d.fits'%(f+100))
scaledata = [bzero3, bscale3, bzero4, bscale4] except: #scaledata = [32768,1,2147483648,1] scaledata = [0, 1, 0, 1] data1 = hitrandom(table.header['NHITS'], NS, FS, scaledata, table.header['BEAMPOL']) data2 = data1.data of[ind].header = hdr of[ind].data = data2 #pyfits.append(outfile,data2,hdr) #pyfits.update(outfile,data2,hdr,ind) else: print "Error1" elif any(elem in table.header['EXTNAME'] for elem in delim): hdr1 = pyfits.BinTableHDU() hdr1 = table #pyfits.append(outfile,hdr1.data,hdr1.header) #print table.header['EXTNAME'] nSteps += 1 else: print "Error2" except: print "Error3" pass of.verify() of.writeto(outfile, clobber=True) of.close() #To test
def aggregate(args): """ Aggregate JSON-formatted results into a single tabular FITS file. """ if os.path.exists(args.output_filename): if not args.clobber: raise IOError("output filename {0} already exists and we have been "\ "asked not to clobber it".format(args.output_filename)) else: logger.warn("Overwriting existing filename {0}".format( args.output_filename)) # Let's just assume it all aggregates from JSON to a FITS filename results = [] for filename in args.result_filenames: with open(filename, "r") as fp: try: results.append(json.load(fp)) except: logger.exception( "Could not read results filename {0}".format(filename)) if args.debug: raise else: logging.debug( "Successfully loaded results from {0}".format(filename)) # Get header order and sort them columns = results[0].keys() sorted_columns = [] # Logic: RA, DEC then all other uppercase fields in alphabetical order # Then any other fields that have associated u_* headers in alphabetical order, as # well as their u_* columns # Then all the others in alphabetical order if "RA" in columns: sorted_columns.append("RA") if "DEC" in columns: sorted_columns.append("DEC") uppercase_columns = [] parameteral_columns = [] for column in columns: if column.isupper() and column not in sorted_columns: uppercase_columns.append(column) elif "u_pos_{0}".format(column) in columns: parameteral_columns.append(column) uppercase_columns, parameteral_columns = map( sorted, [uppercase_columns, parameteral_columns]) all_parameteral_columns = [] variants = ("{0}", "u_pos_{0}", "u_neg_{0}", "u_maxabs_{0}") for column in parameteral_columns: all_parameteral_columns.extend( [variant.format(column) for variant in variants]) sorted_columns.extend(uppercase_columns) sorted_columns.extend(all_parameteral_columns) other_columns = sorted(set(columns).difference(sorted_columns)) ignore_columns = ("model_configuration", "optimised_theta", "initial_theta") sorted_columns.extend(list(set(other_columns).difference(ignore_columns))) # Create data types formats = [("f8", "|S256")[isinstance(results[-1][each], (str, unicode))] \ for each in sorted_columns] # Create table data = [[result.get(each, ["", np.nan][formats[i] == "f8"]) \ for i, each in enumerate(sorted_columns)] for result in results] results_table = np.core.records.fromrecords(data, names=sorted_columns, formats=formats) # Write results to filename primary_hdu = pyfits.PrimaryHDU() table_hdu = pyfits.BinTableHDU(results_table) hdulist = pyfits.HDUList([primary_hdu, table_hdu]) hdulist.writeto(args.output_filename, clobber=args.clobber) logger.info("Successfully written results from {0} sources with {1} fields"\ " to {2}".format(len(results), len(results[0]), args.output_filename))
def write_pcm(sfname, receiver_params, valid_chans, outname="pcm.fits", fake=False): #This method writes out the receiver parameters into a file that's readable/usable by PSRCHIVE. #Most of it is getting the header/data formatting correct, so it's not very enlightening, but it is useful. shdulist = pyfits.open(sfname) head = shdulist[0].header head['OBS_MODE'] = "PCM" nchan = head['OBSNCHAN'] prihdu = pyfits.PrimaryHDU(header=head) new_hdulist = pyfits.BinTableHDU(header=shdulist[1].header, data=shdulist[1].data, name=shdulist[1].name) chans = np.linspace(head['OBSFREQ'] - head['OBSBW'] / 2.0, head['OBSFREQ'] + head['OBSBW'] / 2.0, nchan, dtype='d') weights = np.zeros(nchan) weights[valid_chans] = 1 Gs = weights.reshape(nchan, 1) expanded_receivers = np.zeros(nchan * 6).reshape(nchan, 6) vchan = 0 for i in range(nchan): if i in valid_chans: expanded_receivers[i] = receiver_params[vchan, [5, 4, 2, 0, 3, 1]] vchan += 1 params = np.append(Gs, expanded_receivers, axis=1) if not fake: params[params == 0] = np.NaN ncovar = 28 ncov = ncovar * nchan covars = np.ones(ncov) * 0.0005 write_params = np.concatenate(params) chisqs = np.ones(nchan) nfree = np.copy(weights) chans = np.asarray([chans]) weights = np.asarray([weights]) write_params = np.asarray([write_params]) covars = np.asarray([covars]) chisqs = np.asarray([chisqs]) nfree = np.asarray([nfree]) freqcol = pyfits.Column(name="DAT_FREQ", format="512D", array=chans) wtcol = pyfits.Column(name="DAT_WTS", format="512E", array=weights) datcol = pyfits.Column(name="DATA", format="3584E", array=write_params) covcol = pyfits.Column(name="COVAR", format="14336E", array=covars) chicol = pyfits.Column(name="CHISQ", format="512E", array=chisqs) freecol = pyfits.Column(name="NFREE", format="512J", array=nfree) cols = pyfits.ColDefs([freqcol, wtcol, datcol, covcol, chicol, freecol]) feed_hdu = pyfits.BinTableHDU.from_columns(cols, name="FEEDPAR") feed_hdu.header.comments[ 'TTYPE1'] = '[MHz] Centre frequency for each channel' feed_hdu.header.comments['TFORM1'] = 'NCHAN doubles' feed_hdu.header.comments['TTYPE2'] = 'Weights for each channel' feed_hdu.header.comments['TFORM2'] = 'NCHAN floats' feed_hdu.header.comments['TTYPE3'] = 'Cross-coupling data' feed_hdu.header.comments['TFORM3'] = 'NCPAR*NCHAN floats' feed_hdu.header.comments['TTYPE4'] = 'Formal covariances of coupling data' feed_hdu.header.comments['TFORM4'] = 'NCOVAR*NCHAN floats' feed_hdu.header.comments[ 'TTYPE5'] = 'Total chi-squared (objective merit function)' feed_hdu.header.comments['TFORM5'] = 'NCHAN floats' feed_hdu.header.comments['TTYPE6'] = 'Number of degrees of freedom' feed_hdu.header.comments['TFORM6'] = 'NCHAN long (32-bit) integers' feed_hdu.header['CAL_MTHD'] = ('van04e18', 'Cross-coupling method') feed_hdu.header['NCPAR'] = ('7', 'Number of coupling parameters') feed_hdu.header['NCOVAR'] = ('28', 'Number of parameter covariances') feed_hdu.header['NCHAN'] = ('512', 'Nr of channels in Feed coupling data') feed_hdu.header['EPOCH'] = ('56038.3352', '[MJD] Epoch of calibration obs') feed_hdu.header['TUNIT1'] = ('MHz', 'Units of field') feed_hdu.header['TDIM3'] = ('(7,512)', 'Dimensions (NCPAR,NCHAN)') feed_hdu.header['TDIM4'] = ('(28,512)', 'Dimensions (NCOVAR,NCHAN)') feed_hdu.header['EXTVER'] = ('1', 'auto assigned by template parser') feed_hdu.header['PAR_0000'] = ('G', 'scalar gain') feed_hdu.header['PAR_0001'] = ('gamma', 'differential gain (hyperbolic radians)') feed_hdu.header['PAR_0002'] = ('phi', 'differential phase (radians)') feed_hdu.header['PAR_0003'] = ('el0', 'ellipticity of receptor 0 (radians)') feed_hdu.header['PAR_0004'] = ('or0', 'orientation of receptor 0 (radians)') feed_hdu.header['PAR_0005'] = ('el1', 'ellipticity of receptor 1 (radians)') feed_hdu.header['PAR_0006'] = ('or1', 'orientation of receptor 1 (radians)') hdus = pyfits.HDUList(hdus=[prihdu, new_hdulist, feed_hdu]) hdus.writeto(outname, clobber=True)
import pyfits import os catalog_f = os.path.join(os.environ['SLREALIZERDIR'], 'data', 'qso_mock.fits') catalog = pyfits.open(catalog_f)[1].data mask = catalog['LENSID'] == 6136045 test_catalog = catalog[mask] hdu = pyfits.BinTableHDU(data=test_catalog) hdu.writeto('test_catalog.fits')
wlcat_file=psfcat_file.replace('psfcat','psfcat_findstars') initfile=pyfits.open(psfcat_file) findstars_file=pyfits.open(star_file) mask=findstars_file[1].data['star_flag']==1 # create new sextractor file with only these entries data=initfile[2].data[mask] # Need to make different copy of these to not fail hdu1=copy.copy(initfile[0]) hdu2=copy.copy(initfile[1]) hdu = pyfits.BinTableHDU(data) hdu.name='LDAC_OBJECTS' list=pyfits.HDUList([hdu1,hdu2, hdu]) list.writeto(wlcat_file,clobber=True) if args.rm_files: os.system('rm %s'%psfcat_file) # assign the psfcat_file to the new file psfcat_file=wlcat_file # If we want to cut the brighest magnitudes if args.mag_cut>0: magcut_file=psfcat_file.replace('psfcat','psfcat_magcut_%0.1f'%args.mag_cut)
def load_fits(file, config): """ utils.load_fits( file, config ) Loads the users fits object catalog. Pass the filename, and the configuration (a dictionary) describing data columns. """ catalog = {} # open up the file fits = pyfits.open(file) # look for a binary hdu with the appropriate fields check = ['ra', 'dec', 'mag', 'point_mag', 're', 'n', 'pa', 'ba'] use = None for (i, ext) in enumerate(fits): # don't bother checking if it isn't a binary fits table if type(ext) != type(pyfits.BinTableHDU()): continue # does it have all the necessary fields? # copy fits column names but in lowercase so they can be searched in a case insensitive way names = [] for field in ext.data.names: names.append(field.lower()) missing = '' for field in check: if not config[field].lower() in names: missing = config[field] break if missing: continue # okay, found our field! use = i break # did we find it? if use == None: if missing: raise ValueError( 'Could not load data catalog - could not locate a binary fits table with field "%s"' % missing) else: raise ValueError( 'Could not load data catalog - could not locate a binary fits table in input catalog' ) # now we can load the catalog cat = fits[use].data valid = cat.size # and start loading the data! # pretend everything is a point source # make sure that ids are strings ids = [] for i in range(cat.size): ids.append(str(cat.field(config['id'])[i])) catalog['id'] = np.asarray(ids) catalog['ra'] = cat.field(config['ra']) catalog['dec'] = cat.field(config['dec']) # sersic fields to load sersics = ['re', 'n', 'pa', 'ba', 're_err', 'n_err', 'pa_err', 'ba_err'] for field in sersics: if config.has_key(field): catalog[field] = np.zeros(valid) catalog['model'] = np.asarray(['point'] * valid).astype( '|S6' ) # numpy string arrays carry a length - expand to make room for 'sersic' if config.has_key('point_mag'): catalog['mag'] = cat.field(config['point_mag']) if config.has_key('point_mag_err'): catalog['mag_err'] = cat.field(config['point_mag_err']) else: catalog['mag'] = cat.field(config['mag']) if config.has_key('mag_err'): catalog['mag_err'] = cat.field(config['mag_err']) # figure out what is a sersic and what is a point source m = cat.field(config['model_type']) == 'sersic' p = cat.field(config['model_type']) == 'point' # number of unrecognized model types unknown = valid - m.sum() - p.sum() # load sersic-specific values if m.sum() > 0: catalog['mag'][m] = cat.field(config['mag'])[m] catalog['model'][m] = ['sersic'] if config.has_key('mag_err') and catalog.has_key('mag_err'): catalog['mag_err'][m] = cat.field(config['mag_err'])[m] for field in sersics: if config.has_key(field): catalog[field][m] = cat.field(config[field])[m] # return catalog and counts return (catalog, valid, unknown)
format='D', array=out_zphot), pyfits.Column(name='sersicfit', format='8D', array=out_sersicfit), pyfits.Column(name='bulgefit', format='16D', array=out_bulgefit), pyfits.Column(name='fit_status', format='5J', array=out_fit_status), pyfits.Column(name='fit_mad_s', format='D', array=out_fit_mad_s), pyfits.Column(name='fit_mad_b', format='D', array=out_fit_mad_b), pyfits.Column(name='fit_dvc_btt', format='D', array=out_fit_dvc_btt)] )) # Write outputs. print "Writing to file ",out_fitfile tbhdu.writeto(out_fitfile, clobber=True) # Write new subset of catalog file. print "Re-writing to file ",out_catfile galsim_cat = pyfits.BinTableHDU(galsim_cat[use_ind]) galsim_cat.writeto(out_catfile, clobber=True)
def filter(self, mask): return LDACCat( pyfits.BinTableHDU(data=self.hdu.data[mask], header=self.hdu.header))
cat_file = cat_file.replace('psfcat', 'psfcat_findstars') if not os.path.exists(cat_file.replace( 'psfcat', 'psfcat_findstars')) or args.force: initfile = pyfits.open(old_cat) findstars_file = pyfits.open(star_file) mask = findstars_file[1].data['star_flag'] == 1 # create new sextractor file with only these entries data = initfile[2].data[mask] # Need to make different copy of these to not fail hdu1 = copy.copy(initfile[0]) hdu2 = copy.copy(initfile[1]) hdu = pyfits.BinTableHDU(data) hdu.name = 'LDAC_OBJECTS' list = pyfits.HDUList([hdu1, hdu2, hdu]) list.writeto(cat_file, clobber=True) # If we want to cut the brighest magnitudes if args.mag_cut > 0: cut_file = cat_file.replace('psfcat', 'psfcat_mag_%0.1f' % args.mag_cut) if (not os.path.exists(cut_file) or args.force): # get the brightest 10 mags that have flags=0 and take the median just in case some were # selected name = cat_file hdu = 2
fieldSize = 42.0/60.0 SMASHcat = sys.argv[1] FieldsDat = sys.argv[2] SMASHroot, SMASHext = path.splitext(SMASHcat) fields = np.loadtxt(FieldsDat) fnum = (fields[:,0]).astype(int) ra = fields[:,1] dec = fields[:,2] nFields = len(fnum) smash = pf.open(SMASHcat) smashData = smash[1].data smashRa = smashData['RA'] smashDec = smashData['DEC'] for i in range(nFields): SMASHoutName = SMASHroot + ('_%d' % fnum[i]) + SMASHext raLow = ra[i] - 0.5*fieldSize / np.cos(dec[i]*np.pi/180) raHi = ra[i] + 0.5*fieldSize / np.cos(dec[i]*np.pi/180) decLow = dec[i] - 0.5*fieldSize decHi = dec[i] + 0.5*fieldSize inField = np.where((smashRa>raLow) & (smashRa<raHi) & (smashDec>decLow) & (smashDec<decHi)) if len(inField[0])>0: fieldSmashData = smashData[inField] hdu = pf.BinTableHDU(data=fieldSmashData) hdu.writeto(SMASHoutName) print fnum[i], len(inField[0]), SMASHoutName
dec = ff.DEC l, b = es.coords.eq2gal(ra, dec) theta = (90 - b) / 180. * 3.14159265 #Galactic coordinate phi = l / 180. * 3.14159265 nside = 2**5 hid = hp.ang2pix(nside, theta, phi) unqid = np.unique(hid) nid = len(unqid) for j in range(nid): ok = hid == unqid[j] if np.any(ok): fname = '/media/jghao/data/sdssdr8/patches/hpixFile/' + str( unqid[j]) + '.fit' if os.path.isfile(fname): fname = fname[:-4] + '_' + str(i) + '.fit' hdu = pf.BinTableHDU(ff[ok]) hdu.writeto(fname) #---------merge those multiple ones ------------ f = gl.glob('/media/jghao/data/sdssdr8/patches/hpixFile/*_?.fit') idx = [] for fname in f: idx_start = fname.find('hpixFile/') + 9 idx_end = fname.find('_') idx.append(int(fname[idx_start:idx_end])) idx = np.array(idx) unqidx = np.unique(idx) for j in unqidx: