def __call__(self, cat): profileCol = getattr(cat, self.profileCol) sorted_cat = cat.filter(np.argsort(profileCol)) sorted_cat = sorted_cat.filter( np.logical_and(sorted_cat[self.profileCol] > self.minradii, sorted_cat[self.profileCol] < self.maxradii)) radii = [] shear = [] shearerr = [] avebeta = [] avebeta2 = [] ngals = [] for i in range(0, len(sorted_cat), self.ngals): maxtake = min(i + self.ngals, len(cat)) radii.append(np.mean(sorted_cat[self.profileCol][i:maxtake])) curmean, curerr = bootstrapmean(sorted_cat['ghat'][i:maxtake]) shear.append(curmean) shearerr.append(curerr) avebeta.append(np.mean(sorted_cat['beta_s'][i:maxtake])) avebeta2.append(np.mean(sorted_cat['beta_s'][i:maxtake]**2)) ngals.append(len(sorted_cat['ghat'][i:maxtake])) profile = catalog.Catalog() setattr(profile, self.profileCol, np.array(radii)) profile.ghat = np.array(shear) profile.sigma_ghat = np.array(shearerr) profile.beta_s = np.array(avebeta) profile.beta_s2 = np.array(avebeta2) profile.ngals = np.array(ngals) return profile
def __init__(self, mycatalog): self.__version = "0.2.3" self.__doTag = False self.__disk_position = 0 self.__dir_position = 0 self.cat = catalog.Catalog() self.parent_item = None self.last_item = catalog.Disk("NO NAME") self.__load(mycatalog)
def doBinning(self, galaxies): radii = [] shear = [] shearerr = [] avebeta = [] avebeta2 = [] ngals = [] profileCol = getattr(galaxies,self.profileCol) for i in range(self.nbins): if self.bincenters[i] < self.minradii or self.bincenters[i] > self.maxradii: continue mintake = self.bincenters[i] - self.binwidth/2. maxtake = self.bincenters[i] + self.binwidth/2. selected = galaxies.filter(np.logical_and(profileCol >= mintake, profileCol < maxtake)) if len(selected) < 2: radii.append(-1) shear.append(-1) shearerr.append(-1) avebeta.append(-1) avebeta2.append(-1) ngals.append(-1) continue radii.append(np.mean(getattr(selected,self.profileCol))) curmean, curerr = basicBinning.bootstrapmean(selected.ghat) shear.append(curmean) shearerr.append(curerr) avebeta.append(np.mean(selected.beta_s)) avebeta2.append(np.mean(selected.beta_s**2)) ngals.append(len(selected)) profile = catalog.Catalog() setattr(profile, self.profileCol, np.array(radii)) profile.ghat = np.array(shear) profile.sigma_ghat = np.array(shearerr) profile.beta_s = np.array(avebeta) profile.beta_s2 = np.array(avebeta2) profile.ngals = np.array(ngals) return profile
def __call__(self, cat): profileCol = getattr(cat, self.profileCol) if self.binspacing == 'linear': binedges = np.linspace(self.minradii, self.maxradii, self.nbins + 1) else: binedges = np.logspace(np.log10(self.minradii), np.log10(self.maxradii), self.nbins + 1) radii = [] shear = [] shearerr = [] avebeta = [] avebeta2 = [] ngals = [] for i in range(self.nbins): mintake = binedges[i] maxtake = binedges[i + 1] selected = cat.filter( np.logical_and(profileCol >= mintake, profileCol < maxtake)) if len(selected) < 2: radii.append(-1) shear.append(-1) shearerr.append(-1) avebeta.append(-1) avebeta2.append(-1) ngals.append(-1) continue radii.append(np.mean(getattr(selected, self.profileCol))) curmean, curerr = bootstrapmean(selected.ghat) shear.append(curmean) shearerr.append(curerr) avebeta.append(np.mean(selected.beta_s)) avebeta2.append(np.mean(selected.beta_s**2)) ngals.append(len(selected)) profile = catalog.Catalog() setattr(profile, self.profileCol, np.array(radii)) profile.ghat = np.array(shear) profile.sigma_ghat = np.array(shearerr) profile.beta_s = np.array(avebeta) profile.beta_s2 = np.array(avebeta2) profile.ngals = np.array(ngals) return profile
def loadCatalog(self): # if user passed a catalog name, try to open it if self.options.catalogToOpen != '': self.catalog = catalog.Catalog() try: self.catalog.openCatalog(self.options.catalogToOpen) except catalog.Error, e: print "\nError while trying to open new catalog." return False
def process_catalog_cmd(options): """Process the CATALOG command.""" cat = catalog.Catalog(options.catalog[0]) if options.version: revision = cat.revision print("Catalog revision: %s" % revision) print("Expected f/π version: %s" % Version.db_version(revision)) print(Version.version_string()) elif options.new: print("Creating catalog.") cat.create() elif options.upgrade: print("Upgrading catalog.") cat.upgrade() else: raise errors.InvalidCommand("Invalid command option.")
def loadCatalog(self, path): self.catalog = catalog.Catalog() try: # open catalog at given path self.catalog.openCatalog(path) except catalog.Error, e: self.log.error(e.msg) msg = wx.MessageDialog( self, "Error while trying to load catalog.\n Look at log file for more infos.", "", wx.OK | wx.ICON_ERROR) msg.ShowModal() return
def loadCatalog(self, path): # create new instance of Catalog class self.catalog = catalog.Catalog(self.notifyStatusChanged) try: # open catalog at given path self.catalog.openCatalog(path) except catalog.Error, e: self.log.error(e.msg) msg = wx.MessageDialog( self, "Error while trying to load catalog.\n Look at log file for more infos.", "", wx.OK | wx.ICON_ERROR) msg.ShowModal() self.catalog = None return
def generateStarCatalog(self, nstars, nside2=65536): """ Generates star catalog by sampling from healpix map. nstars : number of stars to generate nside2 : healpix grid use to subsample pixels note that since nside2 is never allocated it can be arbitrarily large. """ nside = hp.get_nside(self.smap.data) ra = [] th = [] filled_pixels = np.where(self.smap > 0)[0] densities = self.smap[filled_pixels] kpix = np.random.choice(filled_pixels, size=nstars, p=densities / np.sum(densities)) bincounts = np.bincount(kpix) kpix2 = np.unique(kpix) counts = bincounts[bincounts > 0] hh = nside2**2 / nside**2 i = 0 for i, c in enumerate(counts): rpix = np.random.randint(0, high=hh, size=c) nestpix = hp.ring2nest(nside, kpix2[i]) theta, phi = hp.pix2ang(nside2, hh * nestpix + rpix, nest=True) theta = 90. - theta * 180. / np.pi phi = phi * 180. / np.pi for j in range(0, len(theta)): ra.append(phi[j]) th.append(theta[j]) ra = np.array(ra) dec = np.array(th) z = np.random.normal(self.zmean, self.zsigma, nstars) catalog = cat.Catalog(nstars) catalog['ra'] = ra catalog['dec'] = dec catalog['z'] = z return catalog
def _open_catalog(options): cat = catalog.Catalog(options.catalog[0]) cat.open() return cat
def __init__(self, cdcat_file): self.__user="******" self.cat = catalog.Catalog() self.__load(cdcat_file)
#------------------------------------------------------------------------------ # Written by Anthony L. Leotta 9/6/2017 #------------------------------------------------------------------------------ import json import catalog # open config to get the API token and other parameters with open('config.json', 'r') as f: config = json.load(f) my_cat = catalog.Catalog(config) my_cat.load_pd_collections() searchs = ['Aegypten'] for keyword in searchs: results = my_cat.search(keyword) for result in results: #my_cat.download_collection(result, {'tif', 'g', 'b', 'f', 'r', 't', 'v', 'w'}) #my_cat.download_collection(result, {'tif', 'g', 'r', 'w'}) #my_cat.download_collection(result, {'tif', 'g', 'r', 'w'}) my_cat.download_collection(result, {'g', 'r', 'w'}) print('Run Stats:') print('Remote API Calls:',my_cat.remote_calls) print('Image Downloads:',my_cat.downloads)
def main(args): global evaluation global perf_total_predictions global perf_total_time if not os.path.exists(output_dir): os.makedirs(output_dir) if not os.path.exists(checkpoint_dir): print ("[classify] \033[91m ERROR!!\033[0m Missing directory "+checkpoint_dir+". Run step 4 first.") sys.exit(0) ckpt = tf.train.get_checkpoint_state(checkpoint_dir) #Load model just once samples = { 'data': tf.placeholder(tf.float32, shape=(1, cfg.win_size, cfg.n_traces), name='input_data'), 'cluster_id': tf.placeholder(tf.int64, shape=(1,), name='input_label') } model = models.get(cfg.model, samples, cfg, checkpoint_dir, is_training=False) sess = tf.Session() #memory process = psutil.Process(os.getpid()) mi = process.memory_info() rss_before, vms_before = mi.rss, mi.vms ### model.load(sess) mi = process.memory_info() rss_after, vms_after = mi.rss, mi.vms print("Size of model (megas) = "+str((rss_after-rss_before)/1000000.0)) print("Size of model (megas) = "+str((vms_after-vms_before)/1000000.0)) #print("Size of model (megas) = "+str((shared_after-shared_before)/1000000)) print '[classify] Evaluating using model at step {}'.format( sess.run(model.global_step)) stream_files = [file for file in os.listdir(stream_path) if fnmatch.fnmatch(file, args.pattern)] if len(stream_files)==0: print ("[classify] \033[91m ERROR!!\033[0m No files match the file pattern "+args.pattern+".") sys.exit(0) for stream_file in stream_files: stream_file_without_extension = os.path.split(stream_file)[-1].split(".mseed")[0] if args.catalog_path is None: metadata_path = os.path.join(stream_path, stream_file_without_extension+".csv") if os.path.isfile(metadata_path): print("[classify] Found groundtruth metadata in "+metadata_path+".") cat = pd.read_csv(metadata_path) evaluation = True else: print("[classify] Not found groundtruth metadata in "+metadata_path+".") cat = None else: #cat = pd.read_csv(args.catalog_path) #stations = pd.read_csv(args.stations_path) #Load metadata cat = catalog.Catalog() cat.import_json(args.catalog_path) evaluation = True predictions = predict(stream_path, stream_file, sess, model, samples, cat) #stream_file_without_extension = os.path.split(stream_file)[-1].split(".mseed")[0] #metadata_path = os.path.join(args.stream_path, stream_file_without_extension+".csv") #if os.path.isfile(metadata_path): #print("Found groundtruth metadata in "+metadata_path+".") #cat = pd.read_csv(metadata_path) #for idx, start_time in enumerate(predictions["start_time"]): # filtered_catalog = cat[ # ((cat.start_time >= predictions["start_time"][idx]) # & (cat.end_time <= predictions["end_time"][idx]))] # print(str(len(filtered_catalog["start_time"]))) #else: # print("Not found groundtruth metadata in "+metadata_path+".") #cat = load_catalog(metadata_path) #cat = filter_catalog(cat) sess.close() print("Average time per prediction in secs = "+str(perf_total_time/perf_total_predictions)) if evaluation: print("[classify] true positives = "+str(truePositives)) print("[classify] false positives = "+str(falsePositives)) print("[classify] true negatives = "+str(trueNegatives)) print("[classify] false negatives = "+str(falseNegatives)) if truePositives+falsePositives>0: print("[classify] precission = "+str(100*float(truePositives)/(truePositives+falsePositives))+"%") else: print("[classify] cannot compute precission as truePositives+falsePositives == 0") if truePositives+falseNegatives>0: print("[classify] recall = "+str(100*float(truePositives)/(truePositives+falseNegatives))+"%") else: print("[classify] cannot compute recall as truePositives+falseNegatives == 0") if truePositives+falsePositives+trueNegatives+falseNegatives>0: print("[classify] accuracy = "+str(100*float(truePositives+trueNegatives)/(truePositives+falsePositives+trueNegatives+falseNegatives))+"%") else: print("[classify] cannot compute accuracy as truePositives+falsePositives+trueNegatives+falseNegatives == 0")
NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)])) dec_deg = NP.concatenate((dec_deg, dec_deg_NVSS[NP.logical_and( NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)])) spindex = NP.concatenate((spindex, spindex_NVSS[NP.logical_and( NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)])) majax = NP.concatenate((majax, nvss_majax[NP.logical_and( NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)])) minax = NP.concatenate((minax, nvss_minax[NP.logical_and( NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)])) fluxes = NP.concatenate((fluxes, nvss_fpeak)) ctlgobj = CTLG.Catalog(catlabel, freq_catalog, NP.hstack((ra_deg.reshape(-1, 1), dec_deg.reshape(-1, 1))), fluxes, spectral_index=spindex, src_shape=NP.hstack( (majax.reshape(-1, 1), minax.reshape(-1, 1), NP.zeros(fluxes.size).reshape(-1, 1))), src_shape_units=['degree', 'degree', 'degree']) if backdrop_coords == 'radec': backdrop = griddata(NP.hstack( (ra_deg.reshape(-1, 1), dec_deg.reshape(-1, 1))), fluxes, NP.hstack( (xvect.reshape(-1, 1), yvect.reshape(-1, 1))), method='cubic') backdrop = backdrop.reshape(backdrop_xsize / 2, backdrop_xsize) elif backdrop_coords == 'dircos': if (telescope == 'mwa_dipole') or (obs_mode == 'drift'):
dec_deg = NP.concatenate((dec_deg, dec_deg_NVSS[NP.logical_and( NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)])) spindex = NP.concatenate((spindex, spindex_NVSS[NP.logical_and( NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)])) majax = NP.concatenate((majax, nvss_majax[NP.logical_and( NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)])) minax = NP.concatenate((minax, nvss_minax[NP.logical_and( NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)])) fluxes = NP.concatenate((fluxes, nvss_fpeak)) ctlgobj = CTLG.Catalog(catlabel, freq_catalog, NP.hstack( (ra_deg.reshape(-1, 1), dec_deg.reshape(-1, 1))), fluxes, spectral_index=spindex, src_shape=NP.hstack( (majax.reshape(-1, 1), minax.reshape(-1, 1), NP.zeros(fluxes.size).reshape(-1, 1))), src_shape_units=['degree', 'degree', 'degree']) # ctlgobj = CTLG.Catalog(catlabel, freq_catalog, NP.hstack((ra_deg.reshape(-1,1), dec_deg.reshape(-1,1))), fluxes, spectral_index=spindex) elif use_DSM: dsm_file = args['DSM_file_prefix'] + '{0:0d}.fits'.format(nside) hdulist = fits.open(dsm_file) pixres = hdulist[0].header['PIXAREA'] dsm_table = hdulist[1].data ra_deg = dsm_table['RA'] dec_deg = dsm_table['DEC'] temperatures = dsm_table['T_{0:.0f}'.format(freq / 1e6)] fluxes = temperatures * (2.0 * FCNST.k * freq**2 /
bpass = 1.0 * NP.ones(nchan) # Do the following next few lines only for MWA notch_interval = NP.round(1.28e6 / (freq_resolution * 1e3)) # bpass[::notch_interval] = 0.0 # bpass[1::notch_interval] = 0.0 # bpass[2::notch_interval] = 0.0 oversampling_factor = 1.0 # window = DSP.shaping(nchan, 1/oversampling_factor*CNST.rect_bnw_ratio, shape='bnw', peak=1.0) window = DSP.shaping(nchan, 1 / oversampling_factor, shape='rect', peak=1.0) bpass *= window ctlgobj = CTLG.Catalog( freq_catalog, NP.hstack((ra_deg.reshape(-1, 1), dec_deg.reshape(-1, 1))), fpeak) skymod = CTLG.SkyModel(ctlgobj) A_eff = 16.0 * (0.5 * FCNST.c / (freq_catalog * 1e9))**2 intrfrmtr = RI.Interferometer('B1', [1000.0, 0.0, 0.0], chans, telescope='mwa', latitude=-26.701, A_eff=A_eff, freq_scale='GHz') Tsys = 440.0 # in Kelvin t_snap = 40 * 60.0 # in seconds
os.remove(path) except Exception, e: self.log.error( "Error while trying to remove pre-existent file") msg = wx.MessageDialog( self, "Cannot overwrite selected file. Please choose another one.", "", wx.OK | wx.ICON_ERROR) msg.ShowModal() return # create a new catalog at the specified path self.catalog = catalog.Catalog(self.notifyStatusChanged) try: self.catalog.createNew(path) except catalog.Error, e: msg = wx.MessageDialog( self, "Cannot create catalog at given path. Please check log file for more details.", "", wx.OK | wx.ICON_ERROR) msg.ShowModal() self.catalog = None return # now load newly created catalog
def __set_context_catalog_name(context, catalog_name): context.catalog_name = catalog_name catalog_file = get_catalog_file(context) context.catalog_directory = os.path.dirname(catalog_file) context.catalog_file = catalog_file context.catalog = catalog.Catalog(context.catalog_name)
class MainFrame(wx.Frame): def __init__(self, parent, id, title): # parse command line arguments self.parseArgs() # set up log self.logSetup() # set up log globally # set up local log self.log = logging.getLogger('pyCatalog.main') # notify DEBUG turned ON self.log.debug("DEBUG messages turned ON.") # we call the init method of the inherited class wx.Frame.__init__(self, parent, wx.ID_ANY, title, size=(600, 400)) # GUI stuff self.setup_GUI() ## TODO: delete while subsituting with logging # create console window to log messages self.logViewer = cW.LogViewer(self, -1) # Initialize catalog variable self.catalog = None # even if we haven't yet loaded a catalog self.catalogSaved = True # starts a timer of 10 second for refreshing log viewer content self.timer = threading.Timer(10.0, self.logViewer.refreshLog) self.timer.start() self.buttonPanel.setLabel("no catalog loaded") def logSetup(self): #~ Level Numeric value #~ -------------------- ----------------------- #~ CRITICAL 50 #~ ERROR 40 #~ WARNING 30 #~ INFO 20 #~ DEBUG 10 #~ NOTSET 0 # set up logging to file logging.basicConfig( format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', datefmt='%m-%d %H:%M', filename=LOGFILENAME, filemode='w') # set logging level if self.options.debug == True: logging.getLogger('pyCatalog').setLevel(logging.DEBUG) else: logging.getLogger('pyCatalog').setLevel(logging.WARNING) ## CHECK logging level ## print logging.getLogger('pyCatalog').getEffectiveLevel() # define a Handler which writes INFO messages or higher to the sys.stderr console = logging.StreamHandler() console.setLevel(logging.WARNING) # set a format which is simpler for console use formatter = logging.Formatter( '%(name)-12s: %(levelname)-8s %(message)s') # tell the handler to use this format console.setFormatter(formatter) # add the handler to the root logger logging.getLogger('pyCatalog').addHandler(console) def parseArgs(self): # create new parser object parser = optparse.OptionParser() # set help message parser.usage = "usage: %prog [options] [catalog to open]" parser.add_option("-d", "--debug", action="store_true", dest="debug", help="Set debug messages ON.", default=False) ##parser.add_option("-c", "--create", dest="catalog", help="Create new catalog.", default="") (self.options, args) = parser.parse_args() # initialize variable self.options.catalog = '' # recuperiamo l'eventuale nome del catalogo da aprire if (len(args) > 0): self.options.catalog = args[0] def setup_GUI(self): # main container (toolbar and splitter) self.mainBox = wx.BoxSizer(wx.VERTICAL) # add the toolbar ##self.toolbar = cW.CustomToolBar(self) # add the button panel self.buttonPanel = cW.CustomButtonPanel(self) self.mainSplitter = wx.SplitterWindow(self, 1) # left panel setUp self.leftPanel = wx.Panel(self.mainSplitter, -1) self.leftSizer = wx.BoxSizer(wx.HORIZONTAL) self.deviceTree = cW.DeviceTree(self.leftPanel, -1) self.leftSizer.Add(self.deviceTree, 1, wx.EXPAND) self.leftPanel.SetSizer(self.leftSizer) # right panel setUp self.rightPanel = wx.Panel(self.mainSplitter, -1) self.rightSizer = wx.BoxSizer(wx.HORIZONTAL) self.fileTree = cW.FileTree(self.rightPanel, -1) self.rightSizer.Add(self.fileTree, 1, wx.EXPAND) self.rightPanel.SetSizer(self.rightSizer) # set up main splitter self.mainSplitter.SplitVertically(self.leftPanel, self.rightPanel) self.mainSplitter.SetMinimumPaneSize(20) # to prevent unsplitting self.mainSplitter.SetSashPosition(self.GetSize().x / 3) # set initial sash position # setUp a StatusBar at the bottom of the window with 2 text field self.statusBar = self.CreateStatusBar() # setup 'Catalog' menu self.catalogMenu = wx.Menu() self.catalogMenu.Append(wx.ID_NEW, help=" Create new catalog") wx.EVT_MENU(self, wx.ID_NEW, self.OnNewCatalog) self.catalogMenu.Append(wx.ID_OPEN, help=" Open existing catalog") wx.EVT_MENU(self, wx.ID_OPEN, self.OnOpen) self.catalogMenu.Append(wx.ID_SAVE, help=" Save current catalog") wx.EVT_MENU(self, wx.ID_SAVE, self.OnSaveCatalog) self.catalogMenu.Append(wx.ID_SAVEAS, help=" Save current catalog as ...") wx.EVT_MENU(self, wx.ID_SAVEAS, self.OnSaveCatalogAs) self.catalogMenu.AppendSeparator() self.catalogMenu.Append(wx.ID_EXIT, help=" Terminate the program") wx.EVT_MENU(self, wx.ID_EXIT, self.OnQuit) # setup 'View' menu self.viewMenu = wx.Menu() self.viewMenu.Append(wx.ID_PROPERTIES, "Show log viewer", " Show content of log file") wx.EVT_MENU(self, wx.ID_PROPERTIES, self.showLogViewer) # setup 'Tools' menu self.toolsMenu = wx.Menu() item = self.toolsMenu.Append(wx.NewId(), "Show category editor", " Show category editor") wx.EVT_MENU(self, item.GetId(), self.OnShowCategoryEditor) # setup 'Help' menu self.helpMenu = wx.Menu() self.helpMenu.Append(wx.ID_ABOUT, "&About", " Information about this program") wx.EVT_MENU(self, wx.ID_ABOUT, self.OnAbout) # setUp menubar self.menuBar = wx.MenuBar() self.menuBar.Append(self.catalogMenu, "&Catalog") self.menuBar.Append(self.viewMenu, "&View") self.menuBar.Append(self.toolsMenu, "&Tools") self.menuBar.Append(self.helpMenu, "?") self.SetMenuBar(self.menuBar) # lay out widgets in main sizer ##self.mainBox.Add(self.toolbar, 0, wx.EXPAND) self.mainBox.Add(self.buttonPanel, 0, wx.EXPAND) self.mainBox.Add(self.mainSplitter, 1, wx.EXPAND) self.SetSizer(self.mainBox) # main frame events self.Bind(wx.EVT_SIZE, self.OnSize) self.Bind(wx.EVT_CLOSE, self.OnQuit) # show main frame self.Centre() # show window in the center of the screen self.Show(True) def newCatalog(self): if (self.catalog != None and self.catalogSaved == False): # request user confirmation msg = wx.MessageDialog( self, "Current catalog has not been saved.\nDo you want to continue anyway ?", "", wx.YES_NO | wx.ICON_EXCLAMATION) response = msg.ShowModal() if response == wx.ID_NO: return dlg = wx.FileDialog(self, style=wx.FD_OVERWRITE_PROMPT | wx.FD_SAVE, wildcard="Catalog files (*.db)|*.db") retValue = dlg.ShowModal() # exit if no file is choosen if retValue == wx.ID_CANCEL: return path = os.path.join(dlg.GetDirectory(), dlg.GetFilename()) # CHECK if selected file already exists and try to delete it if os.path.exists(path): try: # to remove selected file os.remove(path) except Exception, e: self.log.error( "Error while trying to remove pre-existent file") msg = wx.MessageDialog( self, "Cannot overwrite selected file. Please choose another one.", "", wx.OK | wx.ICON_ERROR) msg.ShowModal() return # create a new catalog at the specified path self.catalog = catalog.Catalog() try: self.catalog.createNew(path) except catalog.Error, e: msg = wx.MessageDialog( self, "Cannot create catalog at given path. Please check log file for more details.", "", wx.OK | wx.ICON_ERROR) msg.ShowModal() self.catalog = None return
def setUp(self): self.catalog = catalog.Catalog()
class pyCatalog: def __init__(self): # initialize variables self.catalogSaved = True self.catalog = None # parse command line arguments self.parseArgs() # set up log self.logSetup() # set up log globally # set up local log self.log = logging.getLogger('pyCatalog.main') # notify DEBUG turned ON self.log.debug("DEBUG messages turned ON.") # load catalog if self.loadCatalog(): self.interact() else: print "\nCannot load catalog, quitting." print '\nbye\n' sys.exit() def loadCatalog(self): # if user passed a catalog name, try to open it if self.options.catalogToOpen != '': self.catalog = catalog.Catalog() try: self.catalog.openCatalog(self.options.catalogToOpen) except catalog.Error, e: print "\nError while trying to open new catalog." return False # if user choose to open new catalog if self.options.catalog != '': self.catalog = catalog.Catalog() try: self.catalog.createNew(self.options.catalog) except catalog.Error, e: print "\nError while creating new catalog." return False try: self.catalog.openCatalog(self.options.catalog) except catalog.Error, e: print "\nError while trying to open newly created catalog." return False
def Main(argv): try: output_directory = "./simulation_output_tpl/".format() if os.path.exists(output_directory): shutil.rmtree(output_directory) os.mkdir(output_directory) dlambda = 0.6 lambda_obs_min = 3800.0 lambda_obs_max = 12600.0 templates_path = "./templates" refPath = "/home/aschmitt/data/simu_linemodel/simulm_20160513/simulation_pfswlinemodel_20160513_10spcperbin/refz_bad12.txt" refData = loadRef(refPath) TFDir = "/home/aschmitt/data/simu_linemodel/simulm_20160513/simulation_pfswlinemodel_20160513_10spcperbin" catalogPath = "/home/aschmitt/data/simu_linemodel/simulm_20160513/amazed/linecatalogs/linecatalogamazedvacuum_B10E.txt" catLines = ctlg.Catalog(catalogPath) filenameList = [a[0] for a in refData] magList = [a[2] for a in refData] tplList = [a[3] for a in refData] zList = [a[1] for a in refData] print("size of refData = {}".format(len(filenameList))) print("first elt: file = {}, mag = {}, tpl = {}".format( filenameList[0], magList[0], tplList[0])) for k in range(len(filenameList)): tpl_name = tplList[k] z = zList[k] magRoundedRef = magList[k] #loop on the mag uncertainty, yes the mag ref value has been rounded +-0.1 during the creation of the magRoundedCheckList = [ magRoundedRef - 0.05, magRoundedRef - 0.025, magRoundedRef, magRoundedRef + 0.025, magRoundedRef + 0.05 ] filename = filenameList[k] #prepare the template spc with the given mag tplPath = os.path.join(templates_path, tpl_name) tpl = spectrum.Spectrum(tplPath, 'template', snorm=False) tfPath = os.path.join(TFDir, "{}_TF.fits".format(filenameList[k])) spcTF = spectrum.Spectrum(tfPath, 'pfs', snorm=False) print("tfpath = {}".format(tfPath)) maskOutLines = catLines.getMaskOutsideLines(spcTF.xvect, z) tplCheckList = [] lstSqList = [] for mag in magRoundedCheckList: tplTmp = tpl.copy() tplTmp.extendWavelengthRangeRed(lambda_obs_max) tplTmp.extendWavelengthRangeBlue(lambda_obs_min) tplTmp.applyLyaExtinction(redshift=z) tplTmp.applyRedshift(z) # #Check magFound and mag to be the same magFound = tplTmp.setMagIAB(mag) # # # if np.floor(magFound*10.0)!=np.floor(magRoundedRef*10.0): # print("magRoundedRef = {}, and magFound={}".format(magRoundedRef, magFound)) # else: # break tplTmp.applyLambdaCrop(lambda_obs_min, lambda_obs_max) tplTmp.interpolate( dx=0.1) #high sampling for the synthesis process tplTmp.correctZeros() tplTmp.interpolate(dx=dlambda) tplCheckList.append(tplTmp) lstErr = 0.0 for kk in range(spcTF.n - 1): if maskOutLines[kk] == 1.0: lstErr += (spcTF.yvect[kk] - tplTmp.yvect[kk])**2 print("for k={}, mag={}, lstSq = {}".format(k, mag, lstErr)) lstSqList.append(lstErr) kmin = np.argmin(lstSqList) print("found argmin = {}".format(kmin)) tpl = tplCheckList[kmin] exported_fits_path = tpl.exportFits(output_directory, "{}_B".format(filename), addNoise=False, exportNoiseSpectrum=False, noiseSigma=1.5e-19, addNameSuffix=False) exported_fits_name = os.path.split(exported_fits_path)[1] print("Exported: {}".format(exported_fits_name)) #stop print("END of continuum creation script.") except (KeyboardInterrupt): exit()
"--debug", type=int, default=argparse.SUPPRESS ) #Optional, we will use the value from the config file parser.add_argument("--catalog_path", type=str, default=None) #necessary if clustering parser.add_argument("--clusters_file_path", type=str, default=None) #necessary if clustering #parser.add_argument("--redirect_stdout_stderr",type=bool, default=False) args = parser.parse_args() cfg = config.Config(args) #Load metadata (ONLY WHEN LOCATION DETECTION) cat = None if args.catalog_path is not None: cat = catalog.Catalog() cat.import_json(args.catalog_path) clusters = clusters.Clusters() clusters.import_json(args.clusters_file_path) dataset_dir = args.prep_data_dir output_dir = args.tfrecords_dir #if args.redirect_stdout_stderr: # stdout_stderr_file = open(os.path.join(checkpoint_dir, 'stdout_stderr_file.txt'), 'w') # sys.stdout = stderr = stdout_stderr_file tf.app.run() #if args.redirect_stdout_stderr: # stdout_stderr_file.close()
chans = freq_catalog + (NP.arange(nchan) - 0.5 * nchan) * freq_resolution * 1e3 / 1e9 # in GHz bpass = 1.0*NP.ones(nchan) # Do the following next few lines only for MWA notch_interval = NP.round(1.28e6 / (freq_resolution * 1e3)) # bpass[::notch_interval] = 0.0 # bpass[1::notch_interval] = 0.0 # bpass[2::notch_interval] = 0.0 oversampling_factor = 1.0 # window = DSP.shaping(nchan, 1/oversampling_factor*CNST.rect_bnw_ratio, shape='bnw', peak=1.0) window = DSP.shaping(nchan, 1/oversampling_factor, shape='rect', peak=1.0) bpass *= window ctlgobj = CTLG.Catalog(freq_catalog*1e9, NP.hstack((ra_deg.reshape(-1,1), dec_deg.reshape(-1,1))), fpeak, spectral_index=spindex) # ctlgobj = CTLG.Catalog(freq_catalog, NP.hstack((ra_deg.reshape(-1,1), dec_deg.reshape(-1,1))), fpeak) skymod = CTLG.SkyModel(ctlgobj) A_eff = 16.0 * (0.5 * FCNST.c / (freq_catalog * 1e9))**2 intrfrmtr = RI.Interferometer('B1', [1000.0, 0.0, 0.0], chans, telescope='mwa', latitude=-26.701, A_eff=A_eff, freq_scale='GHz') obs_mode = 'drift' Tsys = 440.0 # in Kelvin t_snap = 60.0 # in seconds t_obs = 0.5 * 3600.0 # in seconds pointing_init = [0.0, intrfrmtr.latitude] lst_init = 0.0 * 15.0 # in degrees
flux_unit = 'Jy' freq_catalog = freq/1e9 # in GHz spindex = 0.0 if use_GSM: gsm_file = '/data3/t_nithyanandan/project_MWA/foregrounds/gsmdata.fits' hdulist = fits.open(gsm_file) pixres = hdulist[0].header['PIXAREA'] gsm_table = hdulist[1].data ra_deg = gsm_table['RA'] dec_deg = gsm_table['DEC'] temperatures = gsm_table['T_{0:.0f}'.format(freq/1e6)] fluxes = temperatures * (2.0* FCNST.k * freq**2 / FCNST.c**2) * pixres / CNST.Jy spindex = gsm_table['spindex'] + 2.0 ctlgobj = CTLG.Catalog(freq_catalog*1e9, NP.hstack((ra_deg.reshape(-1,1), dec_deg.reshape(-1,1))), fluxes, spectral_index=spindex, src_shape=NP.hstack((NP.sqrt(pixres*4/NP.pi)*NP.ones(fluxes.size).reshape(-1,1),NP.sqrt(pixres*4/NP.pi)*NP.ones(fluxes.size).reshape(-1,1),NP.zeros(fluxes.size).reshape(-1,1))), src_shape_units=['radian','radian','degree']) fg_str = 'gsm' hdulist.close() elif use_SUMSS: SUMSS_file = '/data3/t_nithyanandan/project_MWA/foregrounds/sumsscat.Mar-11-2008.txt' catalog = NP.loadtxt(SUMSS_file, usecols=(0,1,2,3,4,5,10,12,13,14,15,16)) ra_deg = 15.0 * (catalog[:,0] + catalog[:,1]/60.0 + catalog[:,2]/3.6e3) dec_dd = NP.loadtxt(SUMSS_file, usecols=(3,), dtype="|S3") sgn_dec_str = NP.asarray([dec_dd[i][0] for i in range(dec_dd.size)]) sgn_dec = 1.0*NP.ones(dec_dd.size) sgn_dec[sgn_dec_str == '-'] = -1.0 dec_deg = sgn_dec * (NP.abs(catalog[:,3]) + catalog[:,4]/60.0 + catalog[:,5]/3.6e3) fmajax = catalog[:,7] fminax = catalog[:,8] fpa = catalog[:,9] dmajax = catalog[:,10]
def genSimple(self, N=10000, bias=2.0, zdist=ZDist(), rmagdist=MagDist(), edist=EllipticityDist(), algorithm="peaks", extravar=0.5): """ Generate a catalog of fake objects Parameters ---------- N : int Number of objects to generata bias : float The bias at reference redshift (as specified by zdist.refz()) The algorithm actually operates with z=0 delta field so we convert this bias factor to bias factor today. zdist: object Object specifying redshift window function (Gaussian for the time being) rmagdist: object Object specifying rmag distribution (Gaussian for the time being) edist: object Object specifying intrinsic ellipticity distribution algorithm: string How to distribute objects. Valid options are i) "peaks" (put galaxy if delta value + extra white noise > limit determined so that bias is right). ii) "lognormal" probabilistically sample from lognormal transformation (with the right bias), iii) "random" for random catalogs (just window function) extravar: float Extra variance if algorithm=="peaks" """ random = False peaks = True if algorithm == "peaks": pass elif algorithm == "lognormal": peaks = False elif algorithm == "random": random = True else: print("Bad algorithm") stop() ## we have deltafield at z=0, so first get bias right # now convert bias at effective z to bias now gf = ct.get_growth_function(self.cosmology, np.array([0, zdist.refz()])) biasnow = bias * gf[1] / gf[0] if (peaks): cg0 = (1. + extravar) #Now need to calculate clipping at which I will produce sufficient number of objects # from maple fun = lambda l: (-(math.exp(-l**2 / cg0**2 / 0.2e1) * math.sqrt( 0.2e1) * math.pi**(-0.1e1 / 0.2e1) / cg0 / (math.erf( math.sqrt(0.2e1) / cg0 * l / 0.2e1) - 0.1e1)) - biasnow) ## first healtcheck if ((fun(-3) > 0) or (fun(3) < 0)): print("You are asking for a bias which is a bit extreme.") print(fun(-3) + biasnow, fun(3) + biasnow) stop() lim = brentq(fun, -3.0, 3.0) ## lim is now cut in sources of deltafield + extra*var. Let us just go through all of them lim *= sqrt(self.var) esg = sqrt(self.var * extravar) print("Picking up above ", lim, " with scatter ", esg, " sig=", sqrt(self.var)) else: lognmax = exp(1 + self.delta.max() * biasnow) print("lognmax=", lognmax) tx = self.Nx / 2 ty = self.Ny / 2 zlist = [] dlist = [] ralist = [] rlist = [] klist = [] thetao2sq = self.thetao2**2 tcc = 0 for cc in xrange(N): while (True): tcc += 1 z = zdist.sampleZ() kre = self.z2k(z) ## radial distance in indices ## now sample radially r = sqrt(np.random.uniform() * thetao2sq) phi = np.random.uniform(0, 2 * math.pi) ra = r * np.sin(phi) dec = r * np.cos(phi) k = int(kre) i = int(ra * kre + tx) j = int(dec * kre + ty) accept = random if (not accept): delta = self.delta[i, j, k] if (peaks): if esg > 0: accept = (delta + np.random.normal(0, esg)) > lim else: accept = delta > lim else: accept = (exp(1 + delta * biasnow) / lognmax) > np.random.uniform() ## now try to sample if accept: ## ok, now we really have it. zlist.append(z) dlist.append(dec) ralist.append(ra) rlist.append(kre * self.grid_spacing_h_Mpc) break print("Proposal accept rate:", float(N) / tcc) toret = cat.Catalog(N, max_sep=self.max_sep) toret["ra"] = np.array(ralist) toret["dec"] = np.array(dlist) toret["z"] = np.array(zlist) toret["r"] = np.array(rlist) toret["rmag"] = np.array( [rmagdist.sampleRMag(bias) for i in xrange(N)]) toret["e1"] = np.array([edist.sampleEOne() for i in xrange(N)]) toret["e2"] = np.array([edist.sampleEOne() for i in xrange(N)]) toret["g1"] = np.zeros(N) toret["g2"] = np.zeros(N) return toret