def __init__(self, lon_ref, lat_ref, proj_type='ait'): self.lon_ref = lon_ref self.lat_ref = lat_ref self.proj_type = proj_type if proj_type.lower() == 'ait': self.rotator = SphericalRotator(lon_ref, lat_ref, zenithal=False) self.sphere_to_image_func = aitoffSphereToImage self.image_to_sphere_func = aitoffImageToSphere elif proj_type.lower() == 'tan': self.rotator = SphericalRotator(lon_ref, lat_ref, zenithal=True) self.sphere_to_image_func = gnomonicSphereToImage self.image_to_sphere_func = gnomonicImageToSphere elif proj_type.lower() == 'car': def rotate(lon, lat, invert=False): if invert: return lon + np.array([lon_ref]), lat + np.array([lat_ref]) else: return lon - np.array([lon_ref]), lat - np.array([lat_ref]) self.rotator = SphericalRotator(lon_ref, lat_ref, zenithal=False) # Monkey patch the rotate function self.rotator.rotate = rotate self.sphere_to_image_func = cartesianSphereToImage self.image_to_sphere_func = cartesianImageToSphere else: logger.warn('%s not recognized' % (proj_type))
def load_infiles(infiles,columns=None,multiproc=False): if isinstance(infiles,str): infiles = [infiles] logger.debug("Loading %s files..."%len(infiles)) args = list(zip(infiles,len(infiles)*[columns])) if multiproc: from multiprocessing import Pool processes = multiproc if multiproc > 0 else None p = Pool(processes,maxtasksperchild=1) out = p.map(load,args) else: out = [load(arg) for arg in args] dtype = out[0].dtype for i,d in enumerate(out): if d.dtype != dtype: # ADW: Not really safe... logger.warn("Casting input data to same type.") out[i] = d.astype(dtype) logger.debug('Concatenating arrays...') return np.concatenate(out)
def load_files(filenames, multiproc=False, **kwargs): """ Load a set of FITS files with kwargs. """ filenames = np.atleast_1d(filenames) logger.debug("Loading %s files..." % len(filenames)) kwargs = [dict(filename=f, **kwargs) for f in filenames] if multiproc: from multiprocessing import Pool processes = multiproc if multiproc > 0 else None pool = Pool(processes, maxtasksperchild=1) out = pool.map(load_file, kwargs) del pool else: out = [load_file(kw) for kw in kwargs] dtype = out[0].dtype for i, d in enumerate(out): if d.dtype != dtype: # ADW: Not really safe... logger.warn("Casting input data to same type.") out[i] = d.astype(dtype, copy=False) logger.debug('Concatenating arrays...') return np.concatenate(out)
def __init__(self, lon_ref, lat_ref, proj_type = 'ait'): self.lon_ref = lon_ref self.lat_ref = lat_ref self.proj_type = proj_type if proj_type.lower() == 'ait': self.rotator = SphericalRotator(lon_ref, lat_ref, zenithal=False) self.sphere_to_image_func = aitoffSphereToImage self.image_to_sphere_func = aitoffImageToSphere elif proj_type.lower() == 'tan': self.rotator = SphericalRotator(lon_ref, lat_ref, zenithal=True) self.sphere_to_image_func = gnomonicSphereToImage self.image_to_sphere_func = gnomonicImageToSphere elif proj_type.lower() == 'car': def rotate(lon,lat,invert=False): if invert: return lon + np.array([lon_ref]), lat + np.array([lat_ref]) else: return lon - np.array([lon_ref]), lat - np.array([lat_ref]) self.rotator = SphericalRotator(lon_ref, lat_ref, zenithal=False) # Monkey patch the rotate function self.rotator.rotate = rotate self.sphere_to_image_func = cartesianSphereToImage self.image_to_sphere_func = cartesianImageToSphere else: logger.warn('%s not recognized'%(proj_type))
def load_infiles(infiles, columns=None, multiproc=False): if isstring(infiles): infiles = [infiles] logger.debug("Loading %s files..." % len(infiles)) args = list(zip(infiles, len(infiles) * [columns])) if multiproc: from multiprocessing import Pool processes = multiproc if multiproc > 0 else None p = Pool(processes, maxtasksperchild=1) out = p.map(load, args) else: out = [load(arg) for arg in args] dtype = out[0].dtype for i, d in enumerate(out): if d.dtype != dtype: # ADW: Not really safe... logger.warn("Casting input data to same type.") out[i] = d.astype(dtype) logger.debug('Concatenating arrays...') return np.concatenate(out)
def _createFilenames(self): """ Create a masked records array of all filenames for the given set of pixels and store the existence of those files in the mask values. Parameters: ----------- None Returns: -------- recarray : pixels and mask value """ nside_catalog = self['coords']['nside_catalog'] npix = hp.nside2npix(nside_catalog) pixels = np.arange(npix) catalog_dir = self['catalog']['dirname'] catalog_base = self['catalog']['basename'] catalog_path = os.path.join(catalog_dir, catalog_base) mask_dir = self['mask']['dirname'] mask_base_1 = self['mask']['basename_1'] mask_base_2 = self['mask']['basename_2'] mask_path_1 = os.path.join(mask_dir, mask_base_1) mask_path_2 = os.path.join(mask_dir, mask_base_2) data = np.ma.empty(npix, dtype=[('pix', int), ('catalog', object), ('mask_1', object), ('mask_2', object)]) mask = np.ma.empty(npix, dtype=[('pix', bool), ('catalog', bool), ('mask_1', bool), ('mask_2', bool)]) # Build the filenames data['pix'] = pixels data['catalog'] = np.char.mod(catalog_path, pixels) data['mask_1'] = np.char.mod(mask_path_1, pixels) data['mask_2'] = np.char.mod(mask_path_2, pixels) # Build the mask of existing files using glob mask['catalog'] = ~np.in1d(data['catalog'], glob.glob(catalog_dir + '/*')) mask['mask_1'] = ~np.in1d(data['mask_1'], glob.glob(mask_dir + '/*')) mask['mask_2'] = ~np.in1d(data['mask_2'], glob.glob(mask_dir + '/*')) for name in ['catalog', 'mask_1', 'mask_2']: if np.all(mask[name]): logger.warn("All '%s' files masked" % name) # mask 'pix' if all files not present mask['pix'] = mask['catalog'] | mask['mask_1'] | mask['mask_2'] if np.all(mask['pix']): logger.warn("All pixels masked") return np.ma.MaskedArray(data, mask, fill_value=[-1, '', '', ''])
def __init__(self, type='chabrier'): """ Initialize an instance of an initial mass function. """ self.type = type if self.type == 'chabrier': self.pdf = chabrierIMF else: logger.warn('initial mass function type %s not recognized' % (self.type))
def _createFilenames(self): """ Create a masked records array of all filenames for the given set of pixels and store the existence of those files in the mask values. Parameters: ----------- None Returns: -------- recarray : pixels and mask value """ nside_catalog = self['coords']['nside_catalog'] npix = hp.nside2npix(nside_catalog) pixels = np.arange(npix) catalog_dir = self['catalog']['dirname'] catalog_base = self['catalog']['basename'] catalog_path = os.path.join(catalog_dir,catalog_base) mask_dir = self['mask']['dirname'] mask_base_1 = self['mask']['basename_1'] mask_base_2 = self['mask']['basename_2'] mask_path_1 = os.path.join(mask_dir,mask_base_1) mask_path_2 = os.path.join(mask_dir,mask_base_2) data = np.ma.empty(npix,dtype=[('pix',int), ('catalog',object), ('mask_1',object), ('mask_2',object)]) mask = np.ma.empty(npix,dtype=[('pix',bool), ('catalog',bool), ('mask_1',bool), ('mask_2',bool)]) # Build the filenames data['pix'] = pixels data['catalog'] = np.char.mod(catalog_path,pixels) data['mask_1'] = np.char.mod(mask_path_1,pixels) data['mask_2'] = np.char.mod(mask_path_2,pixels) # Build the mask of existing files using glob mask['catalog'] = ~np.in1d(data['catalog'],glob.glob(catalog_dir+'/*')) mask['mask_1'] = ~np.in1d(data['mask_1'],glob.glob(mask_dir+'/*')) mask['mask_2'] = ~np.in1d(data['mask_2'],glob.glob(mask_dir+'/*')) for name in ['catalog','mask_1','mask_2']: if np.all(mask[name]): logger.warn("All '%s' files masked"%name) # mask 'pix' if all files not present mask['pix'] = mask['catalog'] | mask['mask_1'] | mask['mask_2'] if np.all(mask['pix']): logger.warn("All pixels masked") return np.ma.MaskedArray(data, mask, fill_value=[-1,'','',''])
def _setup(self): self.nside_catalog = self.config['coords']['nside_catalog'] self.nside_mask = self.config['coords']['nside_mask'] self.nside_pixel = self.config['coords']['nside_pixel'] self.filenames = self.config.getFilenames() self.footfile = self.config['data']['footprint'] try: self.footprint = fitsio.read(self.footfile)['I'].ravel() except: logger.warn("Couldn't open %s; will pass through."%self.footfile) self.footprint = self.footfile
def __init__(self, infiles, roi): """ Parameters: ----------- infiles : list of sparse healpix mask files roi : roi object Returns: -------- mask : MaskBand object """ self.roi = roi self.config = self.roi.config # ADW: It's overkill to make the full map just to slim it # down, but we don't have a great way to go from map pixels to # roi pixels. nside, pixel, maglim = healpix.read_partial_map(infiles, column='MAGLIM') self.nside = nside # Sparse maps of pixels in various ROI regions self.mask_roi_sparse = maglim[self.roi.pixels] # Try to get the detection fraction self.frac_roi_sparse = (self.mask_roi_sparse > 0) try: logger.debug("Reading FRACDET...") nside, pixel, frac = healpix.read_partial_map(infiles, column='FRACDET') # This clipping might gloss over bugs... fractype = self.config['mask'].get('fractype', 'binary') fracmin = self.config['mask'].get('fracmin', 0.5) if fractype == 'binary': frac = np.where(frac < fracmin, 0.0, 1.0) elif fractype == 'full': frac = np.where(frac < fracmin, 0.0, frac) elif not fractype: pass else: msg = "Unrecognized fractype: %s" % fractype logger.warn(msg) self.frac_roi_sparse = np.clip(frac[self.roi.pixels], 0.0, 1.0) except ValueError as e: # No detection fraction present msg = "No 'FRACDET' column found in masks; assuming FRACDET = 1.0" logger.info(msg) # Explicitly zero the maglim of pixels with fracdet < fracmin self.mask_roi_sparse[self.frac_roi_sparse == 0] = 0.0
def __init__(self, infiles, roi): """ Parameters: ----------- infiles : list of sparse healpix mask files roi : roi object Returns: -------- mask : MaskBand object """ self.roi = roi self.config = self.roi.config # ADW: It's overkill to make the full map just to slim it # down, but we don't have a great way to go from map pixels to # roi pixels. nside,pixel,maglim = healpix.read_partial_map(infiles,column='MAGLIM') self.nside = nside # Sparse maps of pixels in various ROI regions self.mask_roi_sparse = maglim[self.roi.pixels] # Try to get the detection fraction self.frac_roi_sparse = (self.mask_roi_sparse > 0) try: logger.debug("Reading FRACDET...") nside,pixel,frac=healpix.read_partial_map(infiles,column='FRACDET') # This clipping might gloss over bugs... fractype = self.config['mask'].get('fractype','binary') fracmin = self.config['mask'].get('fracmin',0.5) if fractype == 'binary': frac = np.where(frac < fracmin, 0.0, 1.0) elif fractype == 'full': frac = np.where(frac < fracmin, 0.0, frac) elif not fractype: pass else: msg = "Unrecognized fractype: %s"%fractype logger.warn(msg) self.frac_roi_sparse = np.clip(frac[self.roi.pixels],0.0,1.0) except ValueError as e: # No detection fraction present msg = "No 'FRACDET' column found in masks; assuming FRACDET = 1.0" logger.info(msg) # Explicitly zero the maglim of pixels with fracdet < fracmin self.mask_roi_sparse[self.frac_roi_sparse == 0] = 0.0
def download(self,age=None,metallicity=None,outdir=None,force=False): """ Check valid parameter range and download isochrones from: http://stev.oapd.inaf.it/cgi-bin/cmd """ try: from urllib.error import URLError except ImportError: from urllib2 import URLError if age is None: age = float(self.age) if metallicity is None: metallicity = float(self.metallicity) if outdir is None: outdir = './' basename = self.params2filename(age,metallicity) outfile = os.path.join(outdir,basename) if os.path.exists(outfile) and not force: try: self.verify(outfile,self.survey,age,metallicity) logger.info("Found %s; skipping..."%(outfile)) return except Exception as e: msg = "Overwriting corrupted %s..."%(outfile) logger.warn(msg) os.remove(outfile) mkdir(outdir) self.print_info(age,metallicity) self.query_server(outfile,age,metallicity) if not os.path.exists(outfile): raise RuntimeError('Download failed') try: self.verify(outfile,self.survey,age,metallicity) except Exception as e: msg = "Output file is corrupted." logger.error(msg) msg = "Removing %s."%outfile logger.info(msg) os.remove(outfile) raise(e) return outfile
def download(self, age, metallicity, outdir=None, force=False): """ Check valid parameter range and download isochrones from: http://stev.oapd.inaf.it/cgi-bin/cmd """ if outdir is None: outdir = './' basename = self.isochrone.params2filename(age, metallicity) outfile = os.path.join(outdir, basename) if os.path.exists(outfile) and not force: try: self.verify(outfile, self.survey, age, metallicity) logger.info("Found %s; skipping..." % (outfile)) return except Exception as e: msg = "Overwriting corrupted %s..." % (outfile) logger.warn(msg) #os.remove(outfile) mkdir(outdir) self.print_info(age, metallicity) try: self.query_server(outfile, age, metallicity) except Exception as e: logger.debug(str(e)) raise RuntimeError('Bad server response') if not os.path.exists(outfile): raise RuntimeError('Download failed') try: self.verify(outfile, self.survey, age, metallicity) except Exception as e: msg = "Output file is corrupted." logger.error(msg) #os.remove(outfile) raise (e) return outfile
def download(self,age,metallicity,outdir=None,force=False): """ Check valid parameter range and download isochrones from: http://stev.oapd.inaf.it/cgi-bin/cmd """ if outdir is None: outdir = './' basename = self.isochrone.params2filename(age,metallicity) outfile = os.path.join(outdir,basename) if os.path.exists(outfile) and not force: try: self.verify(outfile,self.survey,age,metallicity) logger.info("Found %s; skipping..."%(outfile)) return except Exception as e: msg = "Overwriting corrupted %s..."%(outfile) logger.warn(msg) #os.remove(outfile) mkdir(outdir) self.print_info(age,metallicity) try: self.query_server(outfile,age,metallicity) except Exception as e: logger.debug(str(e)) raise RuntimeError('Bad server response') if not os.path.exists(outfile): raise RuntimeError('Download failed') try: self.verify(outfile,self.survey,age,metallicity) except Exception as e: msg = "Output file is corrupted." logger.error(msg) #os.remove(outfile) raise(e) return outfile
def run(self): if 'scan' in self.opts.run: logger.info("Running 'scan'...") farm = Farm(self.config,verbose=self.opts.verbose) farm.submit_all(coords=self.opts.coords,queue=self.opts.queue,debug=self.opts.debug) if 'merge' in self.opts.run: logger.info("Running 'merge'...") mergefile = self.config.mergefile roifile = self.config.roifile filenames = self.config.likefile.split('_%')[0]+'_*.fits' infiles = sorted(glob.glob(filenames)) if exists(mergefile) and not self.opts.force: logger.warn(" Found %s; skipping..."%mergefile) else: logger.info(" Merging likelihood files...") ugali.utils.healpix.merge_partial_maps(infiles,mergefile) if exists(roifile) and not self.opts.force: logger.warn(" Found %s; skipping..."%roifile) else: logger.info(" Merging likelihood headers...") ugali.utils.healpix.merge_likelihood_headers(infiles,roifile) #ugali.utils.skymap.mergeLikelihoodFiles(infiles,mergefile,roifile) if 'tar' in self.opts.run: logger.info("Running 'tar'...") outdir = mkdir(self.config['output']['likedir']) logdir = mkdir(join(outdir,'log')) scanfile = self.config.likefile.split('_%')[0]+'_[0-9]*.fits' tarfile = join(self.config.likefile.split('_%')[0]+'_pixels.tar.gz') jobname = 'tar' logfile = os.path.join(logdir,'scan_tar.log') cmd = 'tar --remove-files -cvzf %s %s'%(tarfile,scanfile) if exists(tarfile) and not self.opts.force: logger.warn(" Found %s; skipping..."%tarfile) else: logger.info(" Tarring likelihood files...") logger.info(cmd) self.batch.submit(cmd,jobname,logfile) if 'plot' in self.opts.run: # WARNING: Loading the full 3D healpix map is memory intensive. logger.info("Running 'plot'...") # Should do this in environment variable import matplotlib matplotlib.use('Agg') import pylab as plt import ugali.utils.plotting as plotting skymap = ugali.utils.skymap.readSparseHealpixMap(self.config.mergefile,'LOG_LIKELIHOOD')[1] plotting.plotSkymap(skymap) outdir = mkdir(self.config['output']['plotdir']) basename = os.path.basename(self.config.mergefile.replace('.fits','.png')) outfile = os.path.join(outdir,basename) plt.savefig(outfile)
def load_files(filenames,multiproc=False,**kwargs): """ Load a set of FITS files with kwargs. """ filenames = np.atleast_1d(filenames) logger.debug("Loading %s files..."%len(filenames)) kwargs = [dict(filename=f,**kwargs) for f in filenames] if multiproc: from multiprocessing import Pool processes = multiproc if multiproc > 0 else None p = Pool(processes,maxtasksperchild=1) out = p.map(load_file,kwargs) else: out = [load_file(kw) for kw in kwargs] dtype = out[0].dtype for i,d in enumerate(out): if d.dtype != dtype: # ADW: Not really safe... logger.warn("Casting input data to same type.") out[i] = d.astype(dtype,copy=False) logger.debug('Concatenating arrays...') return np.concatenate(out)
def run(args): try: p.download(*args) except Exception as e: logger.warn(str(e)) logger.error("Download failed.")
def run(self): if self.opts.coords is not None: coords = self.opts.coords names = vars(self.opts).get('names',len(coords)*['']) else: names,coords = self.parser.parse_targets(self.config.candfile) labels=[n.lower().replace(' ','_').replace('(','').replace(')','') for n in names] self.outdir=mkdir(self.config['output']['mcmcdir']) self.logdir=mkdir(join(self.outdir,'log')) args = list(zip(len(names)*[self.opts.config],names,labels,coords)) if 'mcmc' in self.opts.run: logger.info("Running 'mcmc'...") try: shutil.copy(self.opts.config,self.outdir) except Exception as e: logger.warn(e.message) for config,name,label,coord in args: glon,glat,radius = coord outfile = make_filenames(self.config,label)['samfile'] base = splitext(basename(outfile))[0] logfile=join(self.logdir,base+'.log') jobname=base script = self.config['mcmc']['script'] nthreads = self.config['mcmc']['nthreads'] srcmdl = self.config['mcmc'].get('srcmdl') if srcmdl is not None: try: shutil.copy(srcmdl,self.outdir) except Exception as e: logger.warn(e.message) logger.info('%s (%s)'%(name,srcmdl)) cmd='%s %s --name %s --srcmdl %s %s' % ( script,self.opts.config,name,srcmdl,outfile) else: logger.info('%s (%.4f,%.4f)'%(name,glon,glat)) cmd='%s %s --name %s --gal %.4f %.4f --grid %s'% ( script,self.opts.config,name,glon,glat,outfile) logger.info(cmd) self.batch.submit(cmd,jobname,logfile,n=nthreads,a='mpirun') if 'results' in self.opts.run: logger.info("Running 'results'...") if len(args) > 1: pool = Pool(maxtasksperchild=1) pool.map(do_results,args) else: do_results(*args) if 'membership' in self.opts.run: logger.info("Running 'membership'...") if len(args) > 1: pool = Pool(maxtasksperchild=1) pool.map(do_membership,args) else: do_membership(*args) if 'plot' in self.opts.run: logger.info("Running 'plot'...") if len(args) > 1: pool = Pool(maxtasksperchild=1) pool.map(do_plot,args) #map(do_plot,args) else: do_plot(*args) if 'collect' in self.opts.run: logger.info("Running 'collect'...") results = odict() srcmdl = odict() params = odict() for config,name,label,coord in args: srcfile = make_filenames(self.config,name)['srcfile'] results[name] = yaml.load(open(srcfile))['results'] srcmdl[name] = yaml.load(open(srcfile))['source'] params[name] = yaml.load(open(srcfile))['params'] for base,output in [('results.yaml',results),('srcmdl.yaml',srcmdl),('params.yaml',params)]: outfile = join(self.outdir,base) out = open(outfile,'w') out.write(yaml.dump(output)) out.close() if 'scan' in self.opts.run: logger.info("Running 'scan'...") for config,name,label,coord in args: logdir = mkdir('plots/log') logfile=join(logdir,'%s_lnlscan.log') cmd = 'python lnlscan.py %s --name %s --xpar %s --xbins 45 --ypar %s --ybins 45'%(self.opts.config,name,'age','metallicity') self.batch.submit(cmd,logfile=logfile) cmd = 'python lnlscan.py %s --name %s --xpar %s --xbins 45 --ypar %s --ybins 45'%(self.opts.config,name,'metallicity','distance_modulus') self.batch.submit(cmd,logfile=logfile) cmd = 'python lnlscan.py %s --name %s --xpar %s --xbins 45 --ypar %s --ybins 45'%(self.opts.config,name,'age','distance_modulus') self.batch.submit(cmd,logfile=logfile)
def search(self, coords=None, distance_modulus=None, extension=None, tolerance=1.e-2): """ Organize a grid search over ROI target pixels, distance moduli, and extensions. If coords, distance_modulus, or extension is specified, then the nearest value in the predefined scan grid is used. ***This may be different than the input value.** Parameters ---------- coords : (float,float) coordinate to search (matched to nearest scan value) distance_modulus : float distance modulus to search (matched to nearest scan value) extension : float extension to search (matched to nearest scan value) tolerance : float tolerance on richness maximization Returns ------- None """ nmoduli = len(self.distance_modulus_array) npixels = len(self.roi.pixels_target) self.loglike_array = np.zeros([nmoduli, npixels], dtype='f4') self.richness_array = np.zeros([nmoduli, npixels], dtype='f4') self.stellar_mass_array = np.zeros([nmoduli, npixels], dtype='f4') self.fraction_observable_array = np.zeros([nmoduli, npixels], dtype='f4') self.extension_fit_array = np.zeros([nmoduli, npixels], dtype='f4') # DEPRECATED: ADW 2019-04-27 self.richness_lower_array = np.zeros([nmoduli, npixels], dtype='f4') self.richness_upper_array = np.zeros([nmoduli, npixels], dtype='f4') self.richness_ulimit_array = np.zeros([nmoduli, npixels], dtype='f4') # Specific pixel/distance_modulus coord_idx, distance_modulus_idx, extension_idx = None, None, None if coords is not None: # Match to nearest grid coordinate index coord_idx = self.roi.indexTarget(coords[0], coords[1]) if distance_modulus is not None: # Match to nearest distance modulus index distance_modulus_idx = np.fabs(self.distance_modulus_array - distance_modulus).argmin() if extension is not None: # Match to nearest extension extension_idx = np.fabs(self.extension_array - extension).argmin() lon, lat = self.roi.pixels_target.lon, self.roi.pixels_target.lat logger.info('Looping over distance moduli in grid search ...') for ii, distance_modulus in enumerate(self.distance_modulus_array): # Specific distance if distance_modulus_idx is not None: if ii != distance_modulus_idx: continue logger.info(' (%-2i/%i) Distance Modulus=%.1f ...' % (ii + 1, nmoduli, distance_modulus)) # No objects, continue if len(self.loglike.catalog) == 0: logger.warn("No catalog objects") continue # Set distance_modulus once to save time self.loglike.set_params(distance_modulus=distance_modulus) # Loop over pixels for jj in range(0, npixels): # Specific pixel if coord_idx is not None: if jj != coord_idx: continue # Set kernel location self.loglike.set_params(lon=lon[jj], lat=lat[jj]) loglike = 0 # Loop over extensions for kk, ext in enumerate(self.extension_array): # Specific extension if extension_idx is not None: if kk != extension_idx: continue # Set extension self.loglike.set_params(extension=ext) # Doesn't re-sync distance_modulus each time self.loglike.sync_params() # Maximize the likelihood with respect to richness loglike, rich, p = self.loglike.fit_richness() if loglike < self.loglike_array[ii][jj]: # No loglike increase, continue continue self.loglike_array[ii][jj], self.richness_array[ii][ jj], parabola = loglike, rich, p self.stellar_mass_array[ii][ jj] = self.stellar_mass_conversion * self.richness_array[ ii][jj] self.fraction_observable_array[ii][jj] = self.loglike.f self.extension_fit_array[ii][jj] = self.source.extension # ADW: Careful, we are leaving the extension at the # last value in the array, not at the maximum... # Debug output args = (jj + 1, npixels, lon[jj], lat[jj], 2. * self.loglike_array[ii][jj], self.stellar_mass_array[ii][jj], self.fraction_observable_array[ii][jj], self.extension_fit_array[ii][jj]) msg = ' (%-3i/%i) Candidate at (%.2f, %.2f) ... ' msg += 'TS=%.1f, Mstar=%.2g, ObsFrac=%.2g, Ext=%.2g' logger.debug(msg % args) """ # This is debugging output if self.config['scan']['full_pdf']: DeprecationWarning("'full_pdf' is deprecated.") self.richness_lower_array[ii][jj], self.richness_upper_array[ii][jj] = self.loglike.richness_interval(0.6827) self.richness_ulimit_array[ii][jj] = parabola.bayesianUpperLimit(0.95) args = ( 2. * self.loglike_array[ii][jj], self.stellar_mass_conversion*self.richness_array[ii][jj], self.stellar_mass_conversion*self.richness_lower_array[ii][jj], self.stellar_mass_conversion*self.richness_upper_array[ii][jj], self.stellar_mass_conversion*self.richness_ulimit_array[ii][jj] ) msg = 'TS=%.1f, Stellar Mass=%.1f (%.1f -- %.1f @ 0.68 CL, < %.1f @ 0.95 CL)'%(args) logger.debug(msg) """ jj_max = self.loglike_array[ii].argmax() args = (jj_max + 1, npixels, lon[jj_max], lat[jj_max], 2. * self.loglike_array[ii][jj_max], self.stellar_mass_conversion * self.richness_array[ii][jj_max], self.extension_fit_array[ii][jj_max]) msg = ' (%-3i/%i) Max at (%.2f, %.2f) : TS=%.1f, Mstar=%.2g, Ext=%.2f' % ( args) logger.info(msg)
def getFilenames(self,pixels=None): """ Create a masked records array of all filenames for the given set of pixels and store the existence of those files in the mask values. If pixels is None, default behavior is to try to join grab dirname + basename with no pixel insertion. If pixels == -1, grab all pixels of 'nside_catalog'. Examples: f = getFilenames([1,2,3]) # All possible catalog files f['catalog'].data # All existing catalog files f['catalog'][~f.mask['catalog']] # or f['catalog'].compressed() # All missing mask_1 files f['mask_1'][f.mask['mask_1']] # Pixels where all files exist f['pix'][~f.mask['pix']] """ nside_catalog = self['coords']['nside_catalog'] if nside_catalog is None: pixels = [None] elif pixels is not None: pixels = [pixels] if np.isscalar(pixels) else pixels else: pixels = np.arange(healpy.nside2npix(nside_catalog)) npix = len(pixels) catalog_dir = self['catalog']['dirname'] catalog_base = self['catalog']['basename'] mask_dir = self['mask']['dirname'] mask_base_1 = self['mask']['basename_1'] mask_base_2 = self['mask']['basename_2'] data = np.ma.empty(npix,dtype=[('pix',int), ('catalog',object), ('mask_1',object), ('mask_2',object)]) mask = np.ma.empty(npix,dtype=[('pix',bool), ('catalog',bool), ('mask_1',bool), ('mask_2',bool)]) for ii,pix in enumerate(pixels): if pix is None: catalog = os.path.join(catalog_dir,catalog_base) mask_1 = os.path.join(mask_dir,mask_base_1) mask_2 = os.path.join(mask_dir,mask_base_2) else: catalog = os.path.join(catalog_dir,catalog_base%pix) mask_1 = os.path.join(mask_dir,mask_base_1%pix) mask_2 = os.path.join(mask_dir,mask_base_2%pix) data[ii]['pix'] = pix if pix is not None else -1 data[ii]['catalog'] = catalog data[ii]['mask_1'] = mask_1 data[ii]['mask_2'] = mask_2 mask[ii]['catalog'] = not os.path.exists(catalog) mask[ii]['mask_1'] = not os.path.exists(mask_1) mask[ii]['mask_2'] = not os.path.exists(mask_2) for name in ['catalog','mask_1','mask_2']: if np.all(mask[name]): logger.warn("All '%s' files masked"%name) # 'pix' is masked if all files not present mask['pix'] = mask['catalog'] | mask['mask_1'] | mask['mask_2'] if np.all(mask['pix']): logger.warn("All pixels masked") #return np.ma.mrecords.MaskedArray(data, mask, fill_value=[-1,None,None,None]) return np.ma.mrecords.MaskedArray(data, mask, fill_value=[-1,'','',''])
def run(self): if 'scan' in self.opts.run: logger.info("Running 'scan'...") farm = Farm(self.config, verbose=self.opts.verbose) farm.submit_all(coords=self.opts.coords, queue=self.opts.queue, debug=self.opts.debug) if 'merge' in self.opts.run: logger.info("Running 'merge'...") mergefile = self.config.mergefile roifile = self.config.roifile filenames = self.config.likefile.split('_%')[0] + '_*.fits' infiles = sorted(glob.glob(filenames)) if exists(mergefile) and not self.opts.force: logger.warn(" Found %s; skipping..." % mergefile) else: logger.info(" Merging likelihood files...") ugali.utils.healpix.merge_partial_maps(infiles, mergefile) if exists(roifile) and not self.opts.force: logger.warn(" Found %s; skipping..." % roifile) else: logger.info(" Merging likelihood headers...") ugali.utils.healpix.merge_likelihood_headers(infiles, roifile) #ugali.utils.skymap.mergeLikelihoodFiles(infiles,mergefile,roifile) if 'tar' in self.opts.run: logger.info("Running 'tar'...") outdir = mkdir(self.config['output']['likedir']) logdir = mkdir(join(outdir, 'log')) scanfile = self.config.likefile.split('_%')[0] + '_[0-9]*.fits' tarfile = join(self.config.likefile.split('_%')[0] + '_pixels.tar.gz') jobname = 'tar' logfile = os.path.join(logdir, 'scan_tar.log') cmd = 'tar --remove-files -cvzf %s %s' % (tarfile, scanfile) if exists(tarfile) and not self.opts.force: logger.warn(" Found %s; skipping..." % tarfile) else: logger.info(" Tarring likelihood files...") logger.info(cmd) self.batch.submit(cmd, jobname, logfile) if 'plot' in self.opts.run: # WARNING: Loading the full 3D healpix map is memory intensive. logger.info("Running 'plot'...") # Should do this in environment variable import matplotlib matplotlib.use('Agg') import pylab as plt import ugali.utils.plotting as plotting skymap = ugali.utils.skymap.readSparseHealpixMap( self.config.mergefile, 'LOG_LIKELIHOOD')[1] plotting.plotSkymap(skymap) outdir = mkdir(self.config['output']['plotdir']) basename = os.path.basename( self.config.mergefile.replace('.fits', '.png')) outfile = os.path.join(outdir, basename) plt.savefig(outfile)
def pixelizeCatalog(infiles, config, force=False): """ Break catalog into chunks by healpix pixel. Parameters: ----------- infiles : List of input files config : Configuration file force : Overwrite existing files (depricated) Returns: -------- None """ nside_catalog = config['coords']['nside_catalog'] nside_pixel = config['coords']['nside_pixel'] coordsys = config['coords']['coordsys'].upper() outdir = mkdir(config['catalog']['dirname']) filenames = config.getFilenames() lon_field = config['catalog']['lon_field'].upper() lat_field = config['catalog']['lat_field'].upper() # ADW: It would probably be better (and more efficient) to do the # pixelizing and the new column insertion separately. for i,filename in enumerate(infiles): logger.info('(%i/%i) %s'%(i+1, len(infiles), filename)) data = fitsio.read(filename) logger.info("%i objects found"%len(data)) if not len(data): continue columns = map(str.upper,data.dtype.names) names,arrs = [],[] if (lon_field in columns) and (lat_field in columns): lon,lat = data[lon_field],data[lat_field] elif coordsys == 'GAL': msg = "Columns '%s' and '%s' not found."%(lon_field,lat_field) msg += "\nConverting from RA,DEC" logger.warning(msg) lon,lat = cel2gal(data['RA'],data['DEC']) names += [lon_field,lat_field] arrs += [lon,lat] elif coordsys == 'CEL': msg = "Columns '%s' and '%s' not found."%(lon_field,lat_field) msg += "\nConverting from GLON,GLAT" lon,lat = gal2cel(data['GLON'],data['GLAT']) names += [lon_field,lat_field] arrs += [lon,lat] cat_pix = ang2pix(nside_catalog,lon,lat) pix_pix = ang2pix(nside_pixel,lon,lat) cat_pix_name = 'PIX%i'%nside_catalog pix_pix_name = 'PIX%i'%nside_pixel try: names += [cat_pix_name,pix_pix_name] arrs += [cat_pix,pix_pix] data=mlab.rec_append_fields(data,names=names,arrs=arrs) except ValueError as e: logger.warn(str(e)+'; not adding column.') #data[cat_pix_name] = cat_pix #data[pix_pix_name] = pix_pix for pix in np.unique(cat_pix): logger.debug("Processing pixel %s"%pix) arr = data[cat_pix == pix] outfile = filenames.data['catalog'][pix] if not os.path.exists(outfile): logger.debug("Creating %s"%outfile) out=fitsio.FITS(outfile,mode='rw') out.write(arr) hdr=healpix.header_odict(nside=nside_catalog, coord=coordsys[0]) for key in ['PIXTYPE','ORDERING','NSIDE','COORDSYS']: out[1].write_key(*list(hdr[key].values())) out[1].write_key('PIX',pix,comment='HEALPIX pixel for this file') else: out=fitsio.FITS(outfile,mode='rw') out[1].append(arr) logger.debug("Writing %s"%outfile) out.close()
def get_results(self,**kwargs): kwargs.setdefault('alpha',self.alpha) kwargs.setdefault('burn',self.nburn*self.nwalkers) # Calculate best-fit parameters from MCMC chain logger.debug('Estimating parameters...') estimate = self.estimate_params(**kwargs) params = {k:v[0] for k,v in estimate.items()} results = dict(estimate) # Extra parameters from the MCMC chain logger.debug('Estimating auxiliary parameters...') try: results['ra'] = self.estimate('ra',**kwargs) results['dec'] = self.estimate('dec',**kwargs) except KeyError: logger.warn("Didn't find 'ra' or 'dec'") ra,dec = gal2cel(results['lon'][0],results['lat'][0]) results['ra'] = ugali.utils.stats.interval(ra) results['dec'] = ugali.utils.stats.interval(dec) ra,dec = results['ra'][0],results['dec'][0] glon,glat = lon,lat = results['lon'][0],results['lat'][0] results.update(gal=[float(glon),float(glat)]) results.update(cel=[float(ra),float(dec)]) try: results['position_angle_cel'] = self.estimate('position_angle_cel',**kwargs) except KeyError: results['position_angle_cel'] = ugali.utils.stats.interval(np.nan) # Update the loglike to the best-fit parameters from the chain logger.debug('Calculating TS...') ts = 2*self.loglike.value(**params) results['ts'] = ugali.utils.stats.interval(ts,np.nan,np.nan) #lon,lat = estimate['lon'][0],estimate['lat'][0] # #results.update(gal=[float(lon),float(lat)]) #ra,dec = gal2cel(lon,lat) #results.update(cel=[float(ra),float(dec)]) #results['ra'] = ugali.utils.stats.interval(ra,np.nan,np.nan) #results['dec'] = ugali.utils.stats.interval(dec,np.nan,np.nan) # Celestial position angle # Break ambiguity in direction with '% 180.' pa,pa_err = results['position_angle'] pa_cel = gal2cel_angle(lon,lat,pa) % 180. pa_cel_err = np.array(pa_err) - pa + pa_cel results['position_angle_cel'] = ugali.utils.stats.interval(pa_cel,pa_cel_err[0],pa_cel_err[1]) mod,mod_err = estimate['distance_modulus'] dist = mod2dist(mod) dist_lo,dist_hi = [mod2dist(mod_err[0]),mod2dist(mod_err[1])] results['distance'] = ugali.utils.stats.interval(dist,dist_lo,dist_hi) dist,dist_err = results['distance'] ext,ext_err = estimate['extension'] ext_sigma = np.nan_to_num(np.array(ext_err) - ext) results['extension_arcmin'] = ugali.utils.stats.interval(60*ext,60*ext_err[0],60*ext_err[1]) # Radially symmetric extension (correct for ellipticity). ell,ell_err = estimate['ellipticity'] rext,rext_err = ext*np.sqrt(1-ell),np.array(ext_err)*np.sqrt(1-ell) rext_sigma = np.nan_to_num(np.array(rext_err) - rext) results['extension_radial'] = ugali.utils.stats.interval(rext,rext_err[0],rext_err[1]) results['extension_radial_arcmin'] = ugali.utils.stats.interval(60*rext,60*rext_err[0],60*rext_err[1]) # Bayes factor for ellipticity results['ellipticity_bayes_factor'] = self.bayes_factor('ellipticity',burn=kwargs['burn']) # Physical Size (should do this with the posteriors) # Radially symmetric dist_sigma = np.nan_to_num(np.array(dist_err) - dist) size = np.arctan(np.radians(ext)) * dist size_sigma = size * np.sqrt((ext_sigma/ext)**2 + (dist_sigma/dist)**2) size_err = [size-size_sigma[0],size+size_sigma[1]] results['physical_size'] = ugali.utils.stats.interval(size,size_err[0],size_err[1]) rsize = np.arctan(np.radians(rext)) * dist rsize_sigma = rsize * np.sqrt((rext_sigma/rext)**2 + (dist_sigma/dist)**2) rsize_err = [rsize-rsize_sigma[0],rsize+rsize_sigma[1]] results['physical_size_radial'] = ugali.utils.stats.interval(rsize,rsize_err[0],rsize_err[1]) # Richness rich,rich_err = estimate['richness'] # Number of observed stars (sum of p-values) nobs = self.loglike.p.sum() nobs_lo,nobs_hi = nobs + np.sqrt(nobs)*np.array([-1,1]) results['nobs'] = ugali.utils.stats.interval(nobs,nobs_lo,nobs_hi) # Number of predicted stars (pixelization effects?) npred = self.loglike.f*rich npred_lo,npred_hi = rich_err[0]*self.loglike.f,rich_err[1]*self.loglike.f results['npred'] = ugali.utils.stats.interval(npred,npred_lo,npred_hi) # Careful, depends on the isochrone... stellar_mass = self.source.stellar_mass() mass = rich*stellar_mass mass_lo,mass_hi = rich_err[0]*stellar_mass,rich_err[1]*stellar_mass results['mass'] = ugali.utils.stats.interval(mass,mass_lo,mass_hi) stellar_luminosity = self.source.stellar_luminosity() lum = rich*stellar_luminosity lum_lo,lum_hi = rich_err[0]*stellar_luminosity,rich_err[1]*stellar_luminosity results['luminosity'] = ugali.utils.stats.interval(lum,lum_lo,lum_hi) # Absolute magnitude only calculated for DES isochrones with g,r try: Mv = self.source.absolute_magnitude(rich) Mv_lo = self.source.absolute_magnitude(rich_err[0]) Mv_hi = self.source.absolute_magnitude(rich_err[1]) results['Mv'] = ugali.utils.stats.interval(Mv,Mv_lo,Mv_hi) except ValueError as e: logger.warning("Skipping absolute magnitude") logger.warn(str(e)) results['Mv'] = np.nan # ADW: WARNING this is very fragile. # Also, this is not quite right, should cut on the CMD available space kwargs = dict(richness=rich,mag_bright=16., mag_faint=23., n_trials=5000,alpha=self.alpha, seed=0) martin = self.config['results'].get('martin') if martin: logger.info("Calculating Martin magnitude...") if martin > 1: kwargs['n_trials'] = martin Mv_martin = self.source.isochrone.absolute_magnitude_martin(**kwargs) results['Mv_martin'] = Mv_martin else: logger.warning("Skipping Martin magnitude") results['Mv_martin'] = np.nan mu = surfaceBrightness(Mv, size, dist) results['surface_brightness'] = ugali.utils.stats.interval(mu,np.nan,np.nan) try: results['constellation'] = ang2const(lon,lat,self.coordsys)[1] except: pass results['iau'] = ugali.utils.projector.ang2iau(lon,lat) coord = SkyCoord(ra*u.deg,dec*u.deg,distance=dist*u.kpc) results['ra_sex'] = str(coord.ra.to_string()) results['dec_sex'] = str(coord.dec.to_string()) # Calculate some separations from GC, LMC, SMC #NED coordinates with de Grisj distance LMC = SkyCoord(80.8939*u.deg,-69.7561*u.deg,distance=49.89*u.kpc) #NED coordinates with de Grisj distance SMC = SkyCoord(13.1866*u.deg,-72.8286*u.deg,distance=61.94*u.kpc) # GC from astropy? GC = SkyCoord(266.4168262*u.deg,-29.0077969*u.deg,distance=8.0*u.kpc) results['d_gc'] = coord.separation_3d(GC).value results['d_lmc'] = coord.separation_3d(LMC).value results['d_smc'] = coord.separation_3d(SMC).value try: results['feh'] = float(self.source.isochrone.feh) except: results['feh'] = np.nan output = dict() output['params'] = params output['results'] = results return output
def runone(self, i): """ Run one simulation. Parameters: ----------- i : index of the simulation to run Returns: -------- results : result array """ results = self.results results[i]['FLAG'] = FLAG_PROC params = dict(list(zip(results[i].dtype.names, results[i]))) size = len(results) lon, lat = params['RA'], params['DEC'] distance_modulus = params['DISTANCE_MODULUS'] mc_source_id = params['MC_SOURCE_ID'] extension = np.degrees( np.arctan(params['R_PHYSICAL'] / params['DISTANCE'])) logger.info( '\n(%i/%i); (id, lon, lat, mod, ext) = (%i, %.2f, %.2f, %.1f, %.3f)' % (i + 1, size, mc_source_id, lon, lat, distance_modulus, extension)) if params['EBV'] > 0.2: msg = "High reddening region; skipping..." logger.warn(msg) results[i]['FLAG'] |= FLAG_EBV #raise Exception(msg) #results[i]['TS'] = np.nan #return # This links to the parameters in the data scan section = 'scan' # This links to the parameters in the simulate section #section = 'simulate' source = ugali.analysis.loglike.createSource(self.config, section=section, lon=lon, lat=lat) logger.info("Reading data catalog...") obs = ugali.analysis.loglike.createObservation(self.config, lon=lon, lat=lat) # Select just the simulated target of interest logger.info("Merging simulated catalog...") data = self.catalog.data[self.catalog.mc_source_id == mc_source_id].copy() data = np.array(data[list(obs.catalog.data.dtype.names)], dtype=obs.catalog.data.dtype) obs.catalog = ugali.observation.catalog.Catalog( self.config, data=np.concatenate([obs.catalog.data, data])) loglike = ugali.analysis.loglike.LogLikelihood(self.config, obs, source) # Mitigate memory overflow issues by cutting objects with # too many catalog stars if len(loglike.catalog) > 5e5: # 1e5 msg = "Large catalog (N_CATALOG = %i)." % len(loglike.catalog) logger.warn(msg) results[i]['FLAG'] |= FLAG_NOBJ grid = ugali.analysis.scan.GridSearch(self.config, loglike) self.grid = grid self.loglike = self.grid.loglike pix = self.loglike.roi.indexTarget(lon, lat) # ADW: Should fit_distance be free in order to model search procedure? if self.config['simulate'].get('fit_distance', False): fit_distance = None else: idx = np.fabs(grid.distance_modulus_array - distance_modulus).argmin() fit_distance = grid.distance_modulus_array[idx] try: grid.search(coords=(lon, lat), distance_modulus=fit_distance) results[i]['FLAG'] &= ~FLAG_FIT except ValueError as e: logger.error(str(e)) results[i]['FLAG'] |= FLAG_FIT mle = grid.mle() distance_idx = np.fabs(grid.distance_modulus_array - mle['distance_modulus']).argmin() ts = 2 * grid.loglike_array[distance_idx][pix] results[i]['FIT_KERNEL'] = grid.loglike.kernel.name results[i]['TS'] = ts results[i]['FIT_MASS'] = grid.stellar_mass_conversion * mle['richness'] results[i]['FIT_DISTANCE'] = mle['distance_modulus'] results[i]['FIT_EXTENSION'] = grid.loglike.kernel.extension err = grid.err() richness_err = (err['richness'][1] - err['richness'][0]) / 2. results[i][ 'FIT_MASS_ERR'] = grid.stellar_mass_conversion * richness_err distance_modulus_err = (err['distance_modulus'][1] - err['distance_modulus'][0]) / 2. results[i]['FIT_DISTANCE_ERR'] = distance_modulus_err """ fit_extension = [0.02, 0.07, 0.15] # half light radii (deg) # ADW: This won't work since we add extension inside the search loop for ext in fit_extension: grid.loglike.set_params(extension = ext) # Fit failures are often due to fracdet = 0 try: grid.search(coords=(lon,lat),distance_modulus=fit_distance) results[i]['FLAG'] &= ~FLAG_FIT except ValueError as e: logger.error(str(e)) results[i]['FLAG'] |= FLAG_FIT continue mle = grid.mle() ts = 2*grid.loglike_array[distance_idx][pix] if ts <= results[i]['TS']: logger.info("No TS increase; continuing...") continue results[i]['FIT_KERNEL'] = grid.loglike.kernel.name results[i]['TS'] = ts results[i]['FIT_MASS'] = grid.stellar_mass_conversion*mle['richness'] results[i]['FIT_DISTANCE'] = fit_distance #mle['distance_modulus'] results[i]['FIT_EXTENSION'] = ext err = grid.err() richness_err = (err['richness'][1]-err['richness'][0])/2. results[i]['FIT_MASS_ERR'] = grid.stellar_mass_conversion*richness_err distance_modulus_err = (err['distance_modulus'][1]-err['distance_modulus'][0])/2. results[i]['FIT_DISTANCE_ERR'] = distance_modulus_err """ return results
def run(self, population=None, catalog=None, outfile=None, mc_source_id=None): if not population: population = self.population if not catalog: catalog = self.catalog if mc_source_id is not None: sel = np.in1d(population['MC_SOURCE_ID'],mc_source_id) if not sel.sum(): msg = "Requested MC_SOURCE_ID not found: %i"%mc_source_id logger.warn(msg) return else: # Select only systems that are in the catalog sel = np.in1d(population['MC_SOURCE_ID'],catalog.mc_source_id) population = population[sel] size = len(population) dtype=[('KERNEL','S18'),('TS','>f4'),('FIT_KERNEL','S18'), ('FIT_EXTENSION','>f4'),('FIT_MASS','>f4'),('FIT_MASS_ERR','>f4'), ('FIT_DISTANCE','>f4'),('FIT_DISTANCE_ERR','>f4'),('FLAG','>i8')] results = np.array(np.nan*np.ones(size),dtype=dtype) results = recfuncs.merge_arrays([population,results],flatten=True,asrecarray=False,usemask=False) results['TS'] = -np.inf results['FLAG'] = 0 self.results = results if outfile: ugali.utils.fileio.write(outfile,results,clobber=True) for i,d in enumerate(results): params = dict(list(zip(results.dtype.names,d))) lon,lat = params['RA'],params['DEC'] distance_modulus = params['DISTANCE_MODULUS'] mc_source_id = params['MC_SOURCE_ID'] extension=np.degrees(np.arctan(params['R_PHYSICAL']/params['DISTANCE'])) logger.info('\n(%i/%i); (id, lon, lat, mod, ext) = (%i, %.2f, %.2f, %.1f, %.3f)'%(i+1,size,mc_source_id,lon,lat,distance_modulus,extension)) if params['EBV'] > 0.2: msg = "High reddening region E(B-V) > 0.2; skipping..." logger.info(msg) results[i]['FLAG'] |= FLAG_EBV results[i]['TS'] = np.nan continue source = ugali.analysis.loglike.createSource(self.config,section='scan',lon=lon,lat=lat) logger.info("Reading data catalog...") obs = ugali.analysis.loglike.createObservation(self.config,lon=lon,lat=lat) # Select just the simulated target of interest logger.info("Merging simulated catalog...") data = catalog.data[catalog.mc_source_id == mc_source_id].copy() data = np.array(data[list(obs.catalog.data.dtype.names)],dtype=obs.catalog.data.dtype) obs.catalog = ugali.observation.catalog.Catalog(self.config, data=np.concatenate([obs.catalog.data,data])) # Index of closest distance modulus loglike = ugali.analysis.loglike.LogLikelihood(self.config,obs,source) grid = ugali.analysis.scan.GridSearch(self.config,loglike) #if len(loglike.catalog) > 1.0e5: if len(loglike.catalog) > 5.0e5: msg = "High stellar density (N_CATALOG = %i); skipping..."%len(loglike.catalog) logger.warn(msg) results[i]['FLAG'] |= FLAG_DENSITY results[i]['TS'] = np.nan continue self.grid = grid self.loglike = self.grid.loglike pix = self.loglike.roi.indexTarget(lon,lat) # ADW: Should allow fit_distance to float in order to model search procedure? # Currently we are evaluating at the true distance. distance_idx = np.fabs(grid.distance_modulus_array-distance_modulus).argmin() fit_distance = grid.distance_modulus_array[distance_idx] fit_extension = [0.02, 0.07, 0.15] # half light radii (deg) for ext in fit_extension: grid.loglike.set_params(extension = ext) try: grid.search(coords=(lon,lat),distance_modulus=fit_distance) results[i]['FLAG'] &= ~FLAG_FRACDET except ValueError as e: logger.error(str(e)) results[i]['FLAG'] |= FLAG_FRACDET continue mle = grid.mle() ts = 2*grid.log_likelihood_sparse_array[distance_idx][pix] if ts <= results[i]['TS']: logger.info("No TS increase; continuing...") continue results[i]['FIT_KERNEL'] = grid.loglike.kernel.name results[i]['TS'] = ts results[i]['FIT_MASS'] = grid.stellar_mass_conversion*mle['richness'] results[i]['FIT_DISTANCE'] = fit_distance #mle['distance_modulus'] results[i]['FIT_EXTENSION'] = ext err = grid.err() richness_err = (err['richness'][1]-err['richness'][0])/2. results[i]['FIT_MASS_ERR'] = grid.stellar_mass_conversion*richness_err distance_modulus_err = (err['distance_modulus'][1]-err['distance_modulus'][0])/2. results[i]['FIT_DISTANCE_ERR'] = distance_modulus_err logger.info("Fit parameter values:") for d in dtype: logger.info('\t%s: %s'%(d[0], results[i][d[0]])) if (i%self.config['simulate']['save'])==0 and outfile: ugali.utils.fileio.write(outfile,results) if outfile: ugali.utils.fileio.write(outfile,results,clobber=True) return results
def get_results(self, **kwargs): kwargs.setdefault('alpha', self.alpha) kwargs.setdefault('burn', self.nburn * self.nwalkers) # Calculate best-fit parameters from MCMC chain logger.debug('Estimating parameters...') estimate = self.estimate_params(**kwargs) params = {k: v[0] for k, v in estimate.items()} results = dict(estimate) # Extra parameters from the MCMC chain logger.debug('Estimating auxiliary parameters...') try: results['ra'] = self.estimate('ra', **kwargs) results['dec'] = self.estimate('dec', **kwargs) except KeyError: logger.warn("Didn't find 'ra' or 'dec'") ra, dec = gal2cel(results['lon'][0], results['lat'][0]) results['ra'] = ugali.utils.stats.interval(ra) results['dec'] = ugali.utils.stats.interval(dec) ra, dec = results['ra'][0], results['dec'][0] glon, glat = lon, lat = results['lon'][0], results['lat'][0] results.update(gal=[float(glon), float(glat)]) results.update(cel=[float(ra), float(dec)]) try: results['position_angle_cel'] = self.estimate( 'position_angle_cel', **kwargs) except KeyError: results['position_angle_cel'] = ugali.utils.stats.interval(np.nan) # Update the loglike to the best-fit parameters from the chain logger.debug('Calculating TS...') ts = 2 * self.loglike.value(**params) results['ts'] = ugali.utils.stats.interval(ts, np.nan, np.nan) #lon,lat = estimate['lon'][0],estimate['lat'][0] # #results.update(gal=[float(lon),float(lat)]) #ra,dec = gal2cel(lon,lat) #results.update(cel=[float(ra),float(dec)]) #results['ra'] = ugali.utils.stats.interval(ra,np.nan,np.nan) #results['dec'] = ugali.utils.stats.interval(dec,np.nan,np.nan) # Celestial position angle # Break ambiguity in direction with '% 180.' pa, pa_err = results['position_angle'] pa_cel = gal2cel_angle(lon, lat, pa) % 180. pa_cel_err = np.array(pa_err) - pa + pa_cel results['position_angle_cel'] = ugali.utils.stats.interval( pa_cel, pa_cel_err[0], pa_cel_err[1]) mod, mod_err = estimate['distance_modulus'] dist = mod2dist(mod) dist_lo, dist_hi = [mod2dist(mod_err[0]), mod2dist(mod_err[1])] results['distance'] = ugali.utils.stats.interval( dist, dist_lo, dist_hi) dist, dist_err = results['distance'] ext, ext_err = estimate['extension'] ext_sigma = np.nan_to_num(np.array(ext_err) - ext) results['extension_arcmin'] = ugali.utils.stats.interval( 60 * ext, 60 * ext_err[0], 60 * ext_err[1]) # Radially symmetric extension (correct for ellipticity). ell, ell_err = estimate['ellipticity'] rext, rext_err = ext * np.sqrt(1 - ell), np.array(ext_err) * np.sqrt(1 - ell) rext_sigma = np.nan_to_num(np.array(rext_err) - rext) results['extension_radial'] = ugali.utils.stats.interval( rext, rext_err[0], rext_err[1]) results['extension_radial_arcmin'] = ugali.utils.stats.interval( 60 * rext, 60 * rext_err[0], 60 * rext_err[1]) # Bayes factor for ellipticity results['ellipticity_bayes_factor'] = self.bayes_factor( 'ellipticity', burn=kwargs['burn']) # Physical Size (should do this with the posteriors) # Radially symmetric dist_sigma = np.nan_to_num(np.array(dist_err) - dist) size = np.arctan(np.radians(ext)) * dist size_sigma = size * np.sqrt((ext_sigma / ext)**2 + (dist_sigma / dist)**2) size_err = [size - size_sigma[0], size + size_sigma[1]] results['physical_size'] = ugali.utils.stats.interval( size, size_err[0], size_err[1]) rsize = np.arctan(np.radians(rext)) * dist rsize_sigma = rsize * np.sqrt((rext_sigma / rext)**2 + (dist_sigma / dist)**2) rsize_err = [rsize - rsize_sigma[0], rsize + rsize_sigma[1]] results['physical_size_radial'] = ugali.utils.stats.interval( rsize, rsize_err[0], rsize_err[1]) # Richness rich, rich_err = estimate['richness'] # Number of observed stars (sum of p-values) nobs = self.loglike.p.sum() nobs_lo, nobs_hi = nobs + np.sqrt(nobs) * np.array([-1, 1]) results['nobs'] = ugali.utils.stats.interval(nobs, nobs_lo, nobs_hi) # Number of predicted stars (pixelization effects?) npred = self.loglike.f * rich npred_lo, npred_hi = rich_err[0] * self.loglike.f, rich_err[ 1] * self.loglike.f results['npred'] = ugali.utils.stats.interval(npred, npred_lo, npred_hi) # Careful, depends on the isochrone... stellar_mass = self.source.stellar_mass() mass = rich * stellar_mass mass_lo, mass_hi = rich_err[0] * stellar_mass, rich_err[ 1] * stellar_mass results['mass'] = ugali.utils.stats.interval(mass, mass_lo, mass_hi) stellar_luminosity = self.source.stellar_luminosity() lum = rich * stellar_luminosity lum_lo, lum_hi = rich_err[0] * stellar_luminosity, rich_err[ 1] * stellar_luminosity results['luminosity'] = ugali.utils.stats.interval(lum, lum_lo, lum_hi) Mv = self.source.absolute_magnitude(rich) Mv_lo = self.source.absolute_magnitude(rich_err[0]) Mv_hi = self.source.absolute_magnitude(rich_err[1]) results['Mv'] = ugali.utils.stats.interval(Mv, Mv_lo, Mv_hi) # ADW: WARNING this is very fragile. # Also, this is not quite right, should cut on the CMD available space kwargs = dict(richness=rich, mag_bright=16., mag_faint=23., n_trials=5000, alpha=self.alpha, seed=0) martin = self.config['results'].get('martin') if martin: logger.info("Calculating Martin magnitude...") if martin > 1: kwargs['n_trials'] = martin Mv_martin = self.source.isochrone.absolute_magnitude_martin( **kwargs) results['Mv_martin'] = Mv_martin else: logger.warning("Skipping Martin magnitude") results['Mv_martin'] = np.nan mu = surfaceBrightness(Mv, size, dist) results['surface_brightness'] = ugali.utils.stats.interval( mu, np.nan, np.nan) try: results['constellation'] = ugali.utils.projector.ang2const( lon, lat)[1] except: pass results['iau'] = ugali.utils.projector.ang2iau(lon, lat) coord = SkyCoord(ra * u.deg, dec * u.deg, distance=dist * u.kpc) results['ra_sex'] = str(coord.ra.to_string()) results['dec_sex'] = str(coord.dec.to_string()) # Calculate some separations from GC, LMC, SMC #NED coordinates with de Grisj distance LMC = SkyCoord(80.8939 * u.deg, -69.7561 * u.deg, distance=49.89 * u.kpc) #NED coordinates with de Grisj distance SMC = SkyCoord(13.1866 * u.deg, -72.8286 * u.deg, distance=61.94 * u.kpc) # GC from astropy? GC = SkyCoord(266.4168262 * u.deg, -29.0077969 * u.deg, distance=8.0 * u.kpc) results['d_gc'] = coord.separation_3d(GC).value results['d_lmc'] = coord.separation_3d(LMC).value results['d_smc'] = coord.separation_3d(SMC).value try: results['feh'] = float(self.source.isochrone.feh) except: results['feh'] = np.nan output = dict() output['params'] = params output['results'] = results return output
def runall(self, outfile=None, mc_source_id=None, rerun=False): """Run all sources in population. Parameters: ----------- outfile : file to write output to mc_source_id : list of sources to process (None is all sources) Returns: -------- results : processing results """ if mc_source_id is None: mc_source_id = np.unique(self.catalog.mc_source_id) # Select only systems that are in the catalog sel = np.in1d(self.population['MC_SOURCE_ID'], mc_source_id) if not sel.sum(): msg = "Requested MC_SOURCE_IDs not found in population." raise ValueError(msg) if os.path.exists(outfile) and rerun: # read the results from the existing outfile self.results = self.read_results(outfile) else: # create the results self.results = self.create_results(population=self.population[sel]) if not np.in1d(mc_source_id, self.results['MC_SOURCE_ID']).all(): msg = "Requested MC_SOURCE_IDs not found in results." raise ValueError(msg) if outfile: logger.info("Writing %s..." % outfile) self.write_results(outfile, clobber=True) for i, r in enumerate(self.results): # Skip if not in mc_source_id list if self.results[i]['MC_SOURCE_ID'] not in mc_source_id: msg = "%i skipped." % self.results[i]['MC_SOURCE_ID'] logger.info(msg) continue # Rerun in NOPROC or MEM flagged if (self.results[i]['FLAG'] & (FLAG_NOPROC | FLAG_MEM)) == 0: msg = "%i already processed." % self.results[i]['MC_SOURCE_ID'] logger.info(msg) continue start_time = time.time() try: self.runone(i) except MemoryError as e: msg = "Memory usage exceeded %.3f GB" % (self.mlimit / GB) logger.warn(msg) self.results[i]['FLAG'] |= FLAG_MEM except Exception as e: logger.error(str(e)) self.results[i]['FLAG'] |= FLAG_FIT runtime = time.time() - start_time self.results[i]['MEMORY'] = self.get_memory_usage() self.results[i]['RUNTIME'] = runtime logger.info("Fit parameter values:") for d in DTYPES: logger.info('\t%s: %s' % (d[0], self.results[i][d[0]])) logger.info("Memory usage: %.3f GB" % (self.get_memory_usage() / GB)) if (i % self.config['simulate']['save']) == 0 and outfile: logger.info("Writing %s..." % outfile) self.write_results(outfile, clobber=True) if outfile: logger.info("Writing %s..." % outfile) self.write_results(outfile, clobber=True) return self.results
def run(self, population=None, catalog=None, outfile=None, mc_source_id=None): if not population: population = self.population if not catalog: catalog = self.catalog if mc_source_id is not None: sel = np.in1d(population['MC_SOURCE_ID'], mc_source_id) if not sel.sum(): msg = "Requested MC_SOURCE_ID not found: %i" % mc_source_id logger.warn(msg) return else: # Select only systems that are in the catalog sel = np.in1d(population['MC_SOURCE_ID'], catalog.mc_source_id) population = population[sel] size = len(population) dtype = [('KERNEL', 'S18'), ('TS', '>f4'), ('FIT_KERNEL', 'S18'), ('FIT_EXTENSION', '>f4'), ('FIT_MASS', '>f4'), ('FIT_MASS_ERR', '>f4'), ('FIT_DISTANCE', '>f4'), ('FIT_DISTANCE_ERR', '>f4'), ('FLAG', '>i8')] results = np.array(np.nan * np.ones(size), dtype=dtype) results = recfuncs.merge_arrays([population, results], flatten=True, asrecarray=False, usemask=False) results['TS'] = -np.inf results['FLAG'] = 0 self.results = results if outfile: ugali.utils.fileio.write(outfile, results, clobber=True) for i, d in enumerate(results): params = dict(list(zip(results.dtype.names, d))) lon, lat = params['RA'], params['DEC'] distance_modulus = params['DISTANCE_MODULUS'] mc_source_id = params['MC_SOURCE_ID'] extension = np.degrees( np.arctan(params['R_PHYSICAL'] / params['DISTANCE'])) logger.info( '\n(%i/%i); (id, lon, lat, mod, ext) = (%i, %.2f, %.2f, %.1f, %.3f)' % (i + 1, size, mc_source_id, lon, lat, distance_modulus, extension)) if params['EBV'] > 0.2: msg = "High reddening region E(B-V) > 0.2; skipping..." logger.info(msg) results[i]['FLAG'] |= FLAG_EBV results[i]['TS'] = np.nan continue source = ugali.analysis.loglike.createSource(self.config, section='scan', lon=lon, lat=lat) logger.info("Reading data catalog...") obs = ugali.analysis.loglike.createObservation(self.config, lon=lon, lat=lat) # Select just the simulated target of interest logger.info("Merging simulated catalog...") data = catalog.data[catalog.mc_source_id == mc_source_id].copy() data = np.array(data[list(obs.catalog.data.dtype.names)], dtype=obs.catalog.data.dtype) obs.catalog = ugali.observation.catalog.Catalog( self.config, data=np.concatenate([obs.catalog.data, data])) # Index of closest distance modulus loglike = ugali.analysis.loglike.LogLikelihood( self.config, obs, source) grid = ugali.analysis.scan.GridSearch(self.config, loglike) #if len(loglike.catalog) > 1.0e5: if len(loglike.catalog) > 5.0e5: msg = "High stellar density (N_CATALOG = %i); skipping..." % len( loglike.catalog) logger.warn(msg) results[i]['FLAG'] |= FLAG_DENSITY results[i]['TS'] = np.nan continue self.grid = grid self.loglike = self.grid.loglike pix = self.loglike.roi.indexTarget(lon, lat) # ADW: Should allow fit_distance to float in order to model search procedure? # Currently we are evaluating at the true distance. distance_idx = np.fabs(grid.distance_modulus_array - distance_modulus).argmin() fit_distance = grid.distance_modulus_array[distance_idx] fit_extension = [0.02, 0.07, 0.15] # half light radii (deg) for ext in fit_extension: grid.loglike.set_params(extension=ext) try: grid.search(coords=(lon, lat), distance_modulus=fit_distance) results[i]['FLAG'] &= ~FLAG_FRACDET except ValueError as e: logger.error(str(e)) results[i]['FLAG'] |= FLAG_FRACDET continue mle = grid.mle() ts = 2 * grid.log_likelihood_sparse_array[distance_idx][pix] if ts <= results[i]['TS']: logger.info("No TS increase; continuing...") continue results[i]['FIT_KERNEL'] = grid.loglike.kernel.name results[i]['TS'] = ts results[i]['FIT_MASS'] = grid.stellar_mass_conversion * mle[ 'richness'] results[i][ 'FIT_DISTANCE'] = fit_distance #mle['distance_modulus'] results[i]['FIT_EXTENSION'] = ext err = grid.err() richness_err = (err['richness'][1] - err['richness'][0]) / 2. results[i][ 'FIT_MASS_ERR'] = grid.stellar_mass_conversion * richness_err distance_modulus_err = (err['distance_modulus'][1] - err['distance_modulus'][0]) / 2. results[i]['FIT_DISTANCE_ERR'] = distance_modulus_err logger.info("Fit parameter values:") for d in dtype: logger.info('\t%s: %s' % (d[0], results[i][d[0]])) if (i % self.config['simulate']['save']) == 0 and outfile: ugali.utils.fileio.write(outfile, results) if outfile: ugali.utils.fileio.write(outfile, results, clobber=True) return results
def run(self): if 'scan' in self.opts.run: logger.info("Running 'scan'...") farm = Farm(self.config, verbose=self.opts.verbose) farm.submit_all(coords=self.opts.coords, queue=self.opts.queue, debug=self.opts.debug) if 'merge' in self.opts.run: logger.info("Running 'merge'...") mergefile = self.config.mergefile roifile = self.config.roifile filenames = self.config.likefile.split('_%')[0] + '_*.fits' infiles = np.array(sorted(glob.glob(filenames))) if 'mergedir' in self.config['output']: mkdir(self.config['output']['mergedir']) pixels = np.char.rpartition( np.char.rpartition(infiles, '_')[:, 0], '_')[:, -1] pixels = pixels.astype(int) superpixel = healpix.superpixel( pixels, self.config['coords']['nside_likelihood'], self.config['coords']['nside_merge']) for pix in np.unique(superpixel): outfile = mergefile % pix if exists(outfile) and not self.opts.force: logger.warn(" Found %s; skipping..." % outfile) else: healpix.merge_partial_maps(infiles[superpixel == pix], outfile, multiproc=8) if exists(roifile) and not self.opts.force: logger.warn(" Found %s; skipping..." % roifile) else: logger.info(" Merging likelihood headers...") healpix.merge_likelihood_headers(infiles, roifile) if 'tar' in self.opts.run: logger.info("Running 'tar'...") outdir = mkdir(self.config['output']['likedir']) logdir = mkdir(join(outdir, 'log')) scanfile = self.config.likefile.split('_%')[0] + '_[0-9]*.fits' tarfile = join(self.config.likefile.split('_%')[0] + '_pixels.tar.gz') jobname = 'tar' logfile = os.path.join(logdir, 'scan_tar.log') cmd = 'tar --remove-files -cvzf %s %s' % (tarfile, scanfile) if exists(tarfile) and not self.opts.force: logger.warn(" Found %s; skipping..." % tarfile) else: logger.info(" Tarring likelihood files...") logger.info(cmd) self.batch.submit(cmd, jobname, logfile) if 'plot' in self.opts.run: # WARNING: Loading the full 3D healpix map is memory intensive. logger.info("Running 'plot'...") # Should do this in environment variable import matplotlib matplotlib.use('Agg') import pylab as plt import ugali.utils.plotting as plotting skymap = ugali.utils.skymap.readSparseHealpixMap( self.config.mergefile, 'LOG_LIKELIHOOD')[1] plotting.plotSkymap(skymap) outdir = mkdir(self.config['output']['plotdir']) basename = os.path.basename( self.config.mergefile.replace('.fits', '.png')) outfile = os.path.join(outdir, basename) plt.savefig(outfile) if 'check' in self.opts.run: # Check the completion fraction logger.info("Running 'check'...") import fitsio import numpy as np import healpy as hp from ugali.utils.skymap import inFootprint # Load the ROI file roi = fitsio.read(self.config.roifile) done = roi['PIXEL'] # Get all target pixels nside = self.config['coords']['nside_likelihood'] pixels = np.arange(hp.nside2npix(nside)) pixarea = hp.nside2pixarea(nside, degrees=True) foot = pixels[inFootprint(self.config, pixels)] # And find the pixels that haven't been processed undone = ~np.in1d(foot, done) hpxmap = np.zeros(len(pixels)) hpxmap[foot[undone]] = True logger.info("Found %i incomplete pixels with an area of %.1f deg^2." % (hpxmap.sum(), hpxmap.sum() * pixarea)) hp.write_map('check.fits.gz', hpxmap)
def _createFilenames(self,pixels=None): """ Create a masked records array of all filenames for the given set of pixels and store the existence of those files in the mask values. Examples: f = getFilenames([1,2,3]) # All possible catalog files f['catalog'].data # All existing catalog files f['catalog'][~f.mask['catalog']] # or f['catalog'].compressed() # All missing mask_1 files f['mask_1'][f.mask['mask_1']] # Pixels where all files exist f['pix'][~f.mask['pix']] Parameters: ----------- pixels : If pixels is None, grab all pixels of 'nside_catalog'. Returns: -------- recarray : pixels and mask value """ nside_catalog = self['coords']['nside_catalog'] # Deprecated: ADW 2018-06-17 #if nside_catalog is None: # pixels = [None] if pixels is not None: pixels = [pixels] if np.isscalar(pixels) else pixels else: pixels = np.arange(hp.nside2npix(nside_catalog)) npix = len(pixels) catalog_dir = self['catalog']['dirname'] catalog_base = self['catalog']['basename'] mask_dir = self['mask']['dirname'] mask_base_1 = self['mask']['basename_1'] mask_base_2 = self['mask']['basename_2'] data = np.ma.empty(npix,dtype=[('pix',int), ('catalog',object), ('mask_1',object), ('mask_2',object)]) mask = np.ma.empty(npix,dtype=[('pix',bool), ('catalog',bool), ('mask_1',bool), ('mask_2',bool)]) for ii,pix in enumerate(pixels): if pix is None: # DEPRECTATED: ADW 2018-06-17 # This is not really being used anymore catalog = os.path.join(catalog_dir,catalog_base) mask_1 = os.path.join(mask_dir,mask_base_1) mask_2 = os.path.join(mask_dir,mask_base_2) else: catalog = os.path.join(catalog_dir,catalog_base%pix) mask_1 = os.path.join(mask_dir,mask_base_1%pix) mask_2 = os.path.join(mask_dir,mask_base_2%pix) data[ii]['pix'] = pix if pix is not None else -1 data[ii]['catalog'] = catalog data[ii]['mask_1'] = mask_1 data[ii]['mask_2'] = mask_2 mask[ii]['catalog'] = not os.path.exists(catalog) mask[ii]['mask_1'] = not os.path.exists(mask_1) mask[ii]['mask_2'] = not os.path.exists(mask_2) for name in ['catalog','mask_1','mask_2']: if np.all(mask[name]): logger.warn("All '%s' files masked"%name) # mask 'pix' if all files not present mask['pix'] = mask['catalog'] | mask['mask_1'] | mask['mask_2'] if np.all(mask['pix']): logger.warn("All pixels masked") #return np.ma.mrecords.MaskedArray(data, mask, fill_value=[-1,None,None,None]) #return np.ma.mrecords.MaskedArray(data, mask, fill_value=[-1,'','','']) return np.ma.MaskedArray(data, mask, fill_value=[-1,'','',''])
def run(self): if self.opts.coords is not None: coords = self.opts.coords names = vars(self.opts).get('names', len(coords) * ['']) else: names, coords = self.parser.parse_targets(self.config.candfile) labels = [ n.lower().replace(' ', '_').replace('(', '').replace(')', '') for n in names ] self.outdir = mkdir(self.config['output']['mcmcdir']) self.logdir = mkdir(join(self.outdir, 'log')) args = list(zip(len(names) * [self.opts.config], names, labels, coords)) if 'mcmc' in self.opts.run: logger.info("Running 'mcmc'...") try: shutil.copy(self.opts.config, self.outdir) except Exception as e: logger.warn(e.message) for config, name, label, coord in args: glon, glat, radius = coord outfile = make_filenames(self.config, label)['samfile'] base = splitext(basename(outfile))[0] logfile = join(self.logdir, base + '.log') jobname = base script = self.config['mcmc']['script'] nthreads = self.config['mcmc']['nthreads'] srcmdl = self.config['mcmc'].get('srcmdl') if srcmdl is not None: try: shutil.copy(srcmdl, self.outdir) except Exception as e: logger.warn(e.message) logger.info('%s (%s)' % (name, srcmdl)) cmd = '%s %s --name %s --srcmdl %s %s' % ( script, self.opts.config, name, srcmdl, outfile) else: logger.info('%s (%.4f,%.4f)' % (name, glon, glat)) cmd = '%s %s --name %s --gal %.4f %.4f --grid %s' % ( script, self.opts.config, name, glon, glat, outfile) logger.info(cmd) self.batch.submit(cmd, jobname, logfile, n=nthreads, a='mpirun') if 'results' in self.opts.run: logger.info("Running 'results'...") if len(args) > 1: pool = Pool(maxtasksperchild=1) pool.map(do_results, args) else: do_results(*args) if 'membership' in self.opts.run: logger.info("Running 'membership'...") if len(args) > 1: pool = Pool(maxtasksperchild=1) pool.map(do_membership, args) else: do_membership(*args) if 'plot' in self.opts.run: logger.info("Running 'plot'...") if len(args) > 1: pool = Pool(maxtasksperchild=1) pool.map(do_plot, args) #map(do_plot,args) else: do_plot(*args) if 'collect' in self.opts.run: logger.info("Running 'collect'...") results = odict() srcmdl = odict() params = odict() for config, name, label, coord in args: srcfile = make_filenames(self.config, name)['srcfile'] results[name] = yaml.load(open(srcfile))['results'] srcmdl[name] = yaml.load(open(srcfile))['source'] params[name] = yaml.load(open(srcfile))['params'] for base, output in [('results.yaml', results), ('srcmdl.yaml', srcmdl), ('params.yaml', params)]: outfile = join(self.outdir, base) out = open(outfile, 'w') out.write(yaml.dump(output)) out.close() if 'scan' in self.opts.run: logger.info("Running 'scan'...") for config, name, label, coord in args: logdir = mkdir('plots/log') logfile = join(logdir, '%s_lnlscan.log') cmd = 'python lnlscan.py %s --name %s --xpar %s --xbins 45 --ypar %s --ybins 45' % ( self.opts.config, name, 'age', 'metallicity') self.batch.submit(cmd, logfile=logfile) cmd = 'python lnlscan.py %s --name %s --xpar %s --xbins 45 --ypar %s --ybins 45' % ( self.opts.config, name, 'metallicity', 'distance_modulus') self.batch.submit(cmd, logfile=logfile) cmd = 'python lnlscan.py %s --name %s --xpar %s --xbins 45 --ypar %s --ybins 45' % ( self.opts.config, name, 'age', 'distance_modulus') self.batch.submit(cmd, logfile=logfile)
def pixelizeCatalog(infiles, config, force=False): """ Break catalog into chunks by healpix pixel. Parameters: ----------- infiles : List of input files config : Configuration file force : Overwrite existing files (depricated) Returns: -------- None """ nside_catalog = config['coords']['nside_catalog'] nside_pixel = config['coords']['nside_pixel'] coordsys = config['coords']['coordsys'].upper() outdir = mkdir(config['catalog']['dirname']) filenames = config.getFilenames() lon_field = config['catalog']['lon_field'].upper() lat_field = config['catalog']['lat_field'].upper() # ADW: It would probably be better (and more efficient) to do the # pixelizing and the new column insertion separately. for i, filename in enumerate(infiles): logger.info('(%i/%i) %s' % (i + 1, len(infiles), filename)) data = fitsio.read(filename) logger.info("%i objects found" % len(data)) if not len(data): continue columns = map(str.upper, data.dtype.names) names, arrs = [], [] if (lon_field in columns) and (lat_field in columns): lon, lat = data[lon_field], data[lat_field] elif coordsys == 'GAL': msg = "Columns '%s' and '%s' not found." % (lon_field, lat_field) msg += "\nConverting from RA,DEC" logger.warning(msg) lon, lat = cel2gal(data['RA'], data['DEC']) names += [lon_field, lat_field] arrs += [lon, lat] elif coordsys == 'CEL': msg = "Columns '%s' and '%s' not found." % (lon_field, lat_field) msg += "\nConverting from GLON,GLAT" lon, lat = gal2cel(data['GLON'], data['GLAT']) names += [lon_field, lat_field] arrs += [lon, lat] cat_pix = ang2pix(nside_catalog, lon, lat) pix_pix = ang2pix(nside_pixel, lon, lat) cat_pix_name = 'PIX%i' % nside_catalog pix_pix_name = 'PIX%i' % nside_pixel try: names += [cat_pix_name, pix_pix_name] arrs += [cat_pix, pix_pix] data = mlab.rec_append_fields(data, names=names, arrs=arrs) except ValueError as e: logger.warn(str(e) + '; not adding column.') #data[cat_pix_name] = cat_pix #data[pix_pix_name] = pix_pix for pix in np.unique(cat_pix): logger.debug("Processing pixel %s" % pix) arr = data[cat_pix == pix] outfile = filenames.data['catalog'][pix] if not os.path.exists(outfile): logger.debug("Creating %s" % outfile) out = fitsio.FITS(outfile, mode='rw') out.write(arr) hdr = healpix.header_odict(nside=nside_catalog, coord=coordsys[0]) for key in ['PIXTYPE', 'ORDERING', 'NSIDE', 'COORDSYS']: out[1].write_key(*list(hdr[key].values())) out[1].write_key('PIX', pix, comment='HEALPIX pixel for this file') else: out = fitsio.FITS(outfile, mode='rw') out[1].append(arr) logger.debug("Writing %s" % outfile) out.close()