def __call__(self, cat):
profileCol = getattr(cat, self.profileCol)
sorted_cat = cat.filter(np.argsort(profileCol))
sorted_cat = sorted_cat.filter(
np.logical_and(sorted_cat[self.profileCol] > self.minradii,
sorted_cat[self.profileCol] < self.maxradii))
radii = []
shear = []
shearerr = []
avebeta = []
avebeta2 = []
ngals = []
for i in range(0, len(sorted_cat), self.ngals):
maxtake = min(i + self.ngals, len(cat))
radii.append(np.mean(sorted_cat[self.profileCol][i:maxtake]))
curmean, curerr = bootstrapmean(sorted_cat['ghat'][i:maxtake])
shear.append(curmean)
shearerr.append(curerr)
avebeta.append(np.mean(sorted_cat['beta_s'][i:maxtake]))
avebeta2.append(np.mean(sorted_cat['beta_s'][i:maxtake]**2))
ngals.append(len(sorted_cat['ghat'][i:maxtake]))
profile = catalog.Catalog()
setattr(profile, self.profileCol, np.array(radii))
profile.ghat = np.array(shear)
profile.sigma_ghat = np.array(shearerr)
profile.beta_s = np.array(avebeta)
profile.beta_s2 = np.array(avebeta2)
profile.ngals = np.array(ngals)
return profile
def __init__(self, mycatalog):
self.__version = "0.2.3"
self.__doTag = False
self.__disk_position = 0
self.__dir_position = 0
self.cat = catalog.Catalog()
self.parent_item = None
self.last_item = catalog.Disk("NO NAME")
self.__load(mycatalog)
def doBinning(self, galaxies):
radii = []
shear = []
shearerr = []
avebeta = []
avebeta2 = []
ngals = []
profileCol = getattr(galaxies,self.profileCol)
for i in range(self.nbins):
if self.bincenters[i] < self.minradii or self.bincenters[i] > self.maxradii:
continue
mintake = self.bincenters[i] - self.binwidth/2.
maxtake = self.bincenters[i] + self.binwidth/2.
selected = galaxies.filter(np.logical_and(profileCol >= mintake,
profileCol < maxtake))
if len(selected) < 2:
radii.append(-1)
shear.append(-1)
shearerr.append(-1)
avebeta.append(-1)
avebeta2.append(-1)
ngals.append(-1)
continue
radii.append(np.mean(getattr(selected,self.profileCol)))
curmean, curerr = basicBinning.bootstrapmean(selected.ghat)
shear.append(curmean)
shearerr.append(curerr)
avebeta.append(np.mean(selected.beta_s))
avebeta2.append(np.mean(selected.beta_s**2))
ngals.append(len(selected))
profile = catalog.Catalog()
setattr(profile, self.profileCol, np.array(radii))
profile.ghat = np.array(shear)
profile.sigma_ghat = np.array(shearerr)
profile.beta_s = np.array(avebeta)
profile.beta_s2 = np.array(avebeta2)
profile.ngals = np.array(ngals)
return profile
def __call__(self, cat):
profileCol = getattr(cat, self.profileCol)
if self.binspacing == 'linear':
binedges = np.linspace(self.minradii, self.maxradii,
self.nbins + 1)
else:
binedges = np.logspace(np.log10(self.minradii),
np.log10(self.maxradii), self.nbins + 1)
radii = []
shear = []
shearerr = []
avebeta = []
avebeta2 = []
ngals = []
for i in range(self.nbins):
mintake = binedges[i]
maxtake = binedges[i + 1]
selected = cat.filter(
np.logical_and(profileCol >= mintake, profileCol < maxtake))
if len(selected) < 2:
radii.append(-1)
shear.append(-1)
shearerr.append(-1)
avebeta.append(-1)
avebeta2.append(-1)
ngals.append(-1)
continue
radii.append(np.mean(getattr(selected, self.profileCol)))
curmean, curerr = bootstrapmean(selected.ghat)
shear.append(curmean)
shearerr.append(curerr)
avebeta.append(np.mean(selected.beta_s))
avebeta2.append(np.mean(selected.beta_s**2))
ngals.append(len(selected))
profile = catalog.Catalog()
setattr(profile, self.profileCol, np.array(radii))
profile.ghat = np.array(shear)
profile.sigma_ghat = np.array(shearerr)
profile.beta_s = np.array(avebeta)
profile.beta_s2 = np.array(avebeta2)
profile.ngals = np.array(ngals)
return profile
def loadCatalog(self):
# if user passed a catalog name, try to open it
if self.options.catalogToOpen != '':
self.catalog = catalog.Catalog()
try:
self.catalog.openCatalog(self.options.catalogToOpen)
except catalog.Error, e:
print "\nError while trying to open new catalog."
return False
def process_catalog_cmd(options):
"""Process the CATALOG command."""
cat = catalog.Catalog(options.catalog[0])
if options.version:
revision = cat.revision
print("Catalog revision: %s" % revision)
print("Expected f/π version: %s" % Version.db_version(revision))
print(Version.version_string())
elif options.new:
print("Creating catalog.")
cat.create()
elif options.upgrade:
print("Upgrading catalog.")
cat.upgrade()
else:
raise errors.InvalidCommand("Invalid command option.")
def loadCatalog(self, path):
self.catalog = catalog.Catalog()
try: # open catalog at given path
self.catalog.openCatalog(path)
except catalog.Error, e:
self.log.error(e.msg)
msg = wx.MessageDialog(
self,
"Error while trying to load catalog.\n Look at log file for more infos.",
"", wx.OK | wx.ICON_ERROR)
msg.ShowModal()
return
def loadCatalog(self, path):
# create new instance of Catalog class
self.catalog = catalog.Catalog(self.notifyStatusChanged)
try: # open catalog at given path
self.catalog.openCatalog(path)
except catalog.Error, e:
self.log.error(e.msg)
msg = wx.MessageDialog(
self,
"Error while trying to load catalog.\n Look at log file for more infos.",
"", wx.OK | wx.ICON_ERROR)
msg.ShowModal()
self.catalog = None
return
def generateStarCatalog(self, nstars, nside2=65536):
"""
Generates star catalog by sampling from healpix map.
nstars : number of stars to generate
nside2 : healpix grid use to subsample pixels
note that since nside2 is never allocated it can be arbitrarily large.
"""
nside = hp.get_nside(self.smap.data)
ra = []
th = []
filled_pixels = np.where(self.smap > 0)[0]
densities = self.smap[filled_pixels]
kpix = np.random.choice(filled_pixels,
size=nstars,
p=densities / np.sum(densities))
bincounts = np.bincount(kpix)
kpix2 = np.unique(kpix)
counts = bincounts[bincounts > 0]
hh = nside2**2 / nside**2
i = 0
for i, c in enumerate(counts):
rpix = np.random.randint(0, high=hh, size=c)
nestpix = hp.ring2nest(nside, kpix2[i])
theta, phi = hp.pix2ang(nside2, hh * nestpix + rpix, nest=True)
theta = 90. - theta * 180. / np.pi
phi = phi * 180. / np.pi
for j in range(0, len(theta)):
ra.append(phi[j])
th.append(theta[j])
ra = np.array(ra)
dec = np.array(th)
z = np.random.normal(self.zmean, self.zsigma, nstars)
catalog = cat.Catalog(nstars)
catalog['ra'] = ra
catalog['dec'] = dec
catalog['z'] = z
return catalog
def _open_catalog(options):
cat = catalog.Catalog(options.catalog[0])
cat.open()
return cat
def __init__(self, cdcat_file):
self.__user="******"
self.cat = catalog.Catalog()
self.__load(cdcat_file)
#------------------------------------------------------------------------------
# Written by Anthony L. Leotta 9/6/2017
#------------------------------------------------------------------------------
import json
import catalog
# open config to get the API token and other parameters
with open('config.json', 'r') as f:
config = json.load(f)
my_cat = catalog.Catalog(config)
my_cat.load_pd_collections()
searchs = ['Aegypten']
for keyword in searchs:
results = my_cat.search(keyword)
for result in results:
#my_cat.download_collection(result, {'tif', 'g', 'b', 'f', 'r', 't', 'v', 'w'})
#my_cat.download_collection(result, {'tif', 'g', 'r', 'w'})
#my_cat.download_collection(result, {'tif', 'g', 'r', 'w'})
my_cat.download_collection(result, {'g', 'r', 'w'})
print('Run Stats:')
print('Remote API Calls:',my_cat.remote_calls)
print('Image Downloads:',my_cat.downloads)
def main(args):
global evaluation
global perf_total_predictions
global perf_total_time
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if not os.path.exists(checkpoint_dir):
print ("[classify] \033[91m ERROR!!\033[0m Missing directory "+checkpoint_dir+". Run step 4 first.")
sys.exit(0)
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
#Load model just once
samples = {
'data': tf.placeholder(tf.float32,
shape=(1, cfg.win_size, cfg.n_traces),
name='input_data'),
'cluster_id': tf.placeholder(tf.int64,
shape=(1,),
name='input_label')
}
model = models.get(cfg.model, samples, cfg,
checkpoint_dir,
is_training=False)
sess = tf.Session()
#memory
process = psutil.Process(os.getpid())
mi = process.memory_info()
rss_before, vms_before = mi.rss, mi.vms
###
model.load(sess)
mi = process.memory_info()
rss_after, vms_after = mi.rss, mi.vms
print("Size of model (megas) = "+str((rss_after-rss_before)/1000000.0))
print("Size of model (megas) = "+str((vms_after-vms_before)/1000000.0))
#print("Size of model (megas) = "+str((shared_after-shared_before)/1000000))
print '[classify] Evaluating using model at step {}'.format(
sess.run(model.global_step))
stream_files = [file for file in os.listdir(stream_path) if
fnmatch.fnmatch(file, args.pattern)]
if len(stream_files)==0:
print ("[classify] \033[91m ERROR!!\033[0m No files match the file pattern "+args.pattern+".")
sys.exit(0)
for stream_file in stream_files:
stream_file_without_extension = os.path.split(stream_file)[-1].split(".mseed")[0]
if args.catalog_path is None:
metadata_path = os.path.join(stream_path, stream_file_without_extension+".csv")
if os.path.isfile(metadata_path):
print("[classify] Found groundtruth metadata in "+metadata_path+".")
cat = pd.read_csv(metadata_path)
evaluation = True
else:
print("[classify] Not found groundtruth metadata in "+metadata_path+".")
cat = None
else:
#cat = pd.read_csv(args.catalog_path)
#stations = pd.read_csv(args.stations_path)
#Load metadata
cat = catalog.Catalog()
cat.import_json(args.catalog_path)
evaluation = True
predictions = predict(stream_path, stream_file, sess, model, samples, cat)
#stream_file_without_extension = os.path.split(stream_file)[-1].split(".mseed")[0]
#metadata_path = os.path.join(args.stream_path, stream_file_without_extension+".csv")
#if os.path.isfile(metadata_path):
#print("Found groundtruth metadata in "+metadata_path+".")
#cat = pd.read_csv(metadata_path)
#for idx, start_time in enumerate(predictions["start_time"]):
# filtered_catalog = cat[
# ((cat.start_time >= predictions["start_time"][idx])
# & (cat.end_time <= predictions["end_time"][idx]))]
# print(str(len(filtered_catalog["start_time"])))
#else:
# print("Not found groundtruth metadata in "+metadata_path+".")
#cat = load_catalog(metadata_path)
#cat = filter_catalog(cat)
sess.close()
print("Average time per prediction in secs = "+str(perf_total_time/perf_total_predictions))
if evaluation:
print("[classify] true positives = "+str(truePositives))
print("[classify] false positives = "+str(falsePositives))
print("[classify] true negatives = "+str(trueNegatives))
print("[classify] false negatives = "+str(falseNegatives))
if truePositives+falsePositives>0:
print("[classify] precission = "+str(100*float(truePositives)/(truePositives+falsePositives))+"%")
else:
print("[classify] cannot compute precission as truePositives+falsePositives == 0")
if truePositives+falseNegatives>0:
print("[classify] recall = "+str(100*float(truePositives)/(truePositives+falseNegatives))+"%")
else:
print("[classify] cannot compute recall as truePositives+falseNegatives == 0")
if truePositives+falsePositives+trueNegatives+falseNegatives>0:
print("[classify] accuracy = "+str(100*float(truePositives+trueNegatives)/(truePositives+falsePositives+trueNegatives+falseNegatives))+"%")
else:
print("[classify] cannot compute accuracy as truePositives+falsePositives+trueNegatives+falseNegatives == 0")
NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)]))
dec_deg = NP.concatenate((dec_deg, dec_deg_NVSS[NP.logical_and(
NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)]))
spindex = NP.concatenate((spindex, spindex_NVSS[NP.logical_and(
NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)]))
majax = NP.concatenate((majax, nvss_majax[NP.logical_and(
NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)]))
minax = NP.concatenate((minax, nvss_minax[NP.logical_and(
NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)]))
fluxes = NP.concatenate((fluxes, nvss_fpeak))
ctlgobj = CTLG.Catalog(catlabel,
freq_catalog,
NP.hstack((ra_deg.reshape(-1, 1),
dec_deg.reshape(-1, 1))),
fluxes,
spectral_index=spindex,
src_shape=NP.hstack(
(majax.reshape(-1, 1), minax.reshape(-1, 1),
NP.zeros(fluxes.size).reshape(-1, 1))),
src_shape_units=['degree', 'degree', 'degree'])
if backdrop_coords == 'radec':
backdrop = griddata(NP.hstack(
(ra_deg.reshape(-1, 1), dec_deg.reshape(-1, 1))),
fluxes,
NP.hstack(
(xvect.reshape(-1, 1), yvect.reshape(-1, 1))),
method='cubic')
backdrop = backdrop.reshape(backdrop_xsize / 2, backdrop_xsize)
elif backdrop_coords == 'dircos':
if (telescope == 'mwa_dipole') or (obs_mode == 'drift'):
dec_deg = NP.concatenate((dec_deg, dec_deg_NVSS[NP.logical_and(
NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)]))
spindex = NP.concatenate((spindex, spindex_NVSS[NP.logical_and(
NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)]))
majax = NP.concatenate((majax, nvss_majax[NP.logical_and(
NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)]))
minax = NP.concatenate((minax, nvss_minax[NP.logical_and(
NP.logical_and(not_in_SUMSS_ind, bright_source_ind), PS_ind)]))
fluxes = NP.concatenate((fluxes, nvss_fpeak))
ctlgobj = CTLG.Catalog(catlabel,
freq_catalog,
NP.hstack(
(ra_deg.reshape(-1, 1), dec_deg.reshape(-1,
1))),
fluxes,
spectral_index=spindex,
src_shape=NP.hstack(
(majax.reshape(-1, 1), minax.reshape(-1, 1),
NP.zeros(fluxes.size).reshape(-1, 1))),
src_shape_units=['degree', 'degree', 'degree'])
# ctlgobj = CTLG.Catalog(catlabel, freq_catalog, NP.hstack((ra_deg.reshape(-1,1), dec_deg.reshape(-1,1))), fluxes, spectral_index=spindex)
elif use_DSM:
dsm_file = args['DSM_file_prefix'] + '{0:0d}.fits'.format(nside)
hdulist = fits.open(dsm_file)
pixres = hdulist[0].header['PIXAREA']
dsm_table = hdulist[1].data
ra_deg = dsm_table['RA']
dec_deg = dsm_table['DEC']
temperatures = dsm_table['T_{0:.0f}'.format(freq / 1e6)]
fluxes = temperatures * (2.0 * FCNST.k * freq**2 /
bpass = 1.0 * NP.ones(nchan)
# Do the following next few lines only for MWA
notch_interval = NP.round(1.28e6 / (freq_resolution * 1e3))
# bpass[::notch_interval] = 0.0
# bpass[1::notch_interval] = 0.0
# bpass[2::notch_interval] = 0.0
oversampling_factor = 1.0
# window = DSP.shaping(nchan, 1/oversampling_factor*CNST.rect_bnw_ratio, shape='bnw', peak=1.0)
window = DSP.shaping(nchan, 1 / oversampling_factor, shape='rect', peak=1.0)
bpass *= window
ctlgobj = CTLG.Catalog(
freq_catalog, NP.hstack((ra_deg.reshape(-1, 1), dec_deg.reshape(-1, 1))),
fpeak)
skymod = CTLG.SkyModel(ctlgobj)
A_eff = 16.0 * (0.5 * FCNST.c / (freq_catalog * 1e9))**2
intrfrmtr = RI.Interferometer('B1', [1000.0, 0.0, 0.0],
chans,
telescope='mwa',
latitude=-26.701,
A_eff=A_eff,
freq_scale='GHz')
Tsys = 440.0 # in Kelvin
t_snap = 40 * 60.0 # in seconds
os.remove(path)
except Exception, e:
self.log.error(
"Error while trying to remove pre-existent file")
msg = wx.MessageDialog(
self,
"Cannot overwrite selected file. Please choose another one.",
"", wx.OK | wx.ICON_ERROR)
msg.ShowModal()
return
# create a new catalog at the specified path
self.catalog = catalog.Catalog(self.notifyStatusChanged)
try:
self.catalog.createNew(path)
except catalog.Error, e:
msg = wx.MessageDialog(
self,
"Cannot create catalog at given path. Please check log file for more details.",
"", wx.OK | wx.ICON_ERROR)
msg.ShowModal()
self.catalog = None
return
# now load newly created catalog
def __set_context_catalog_name(context, catalog_name):
context.catalog_name = catalog_name
catalog_file = get_catalog_file(context)
context.catalog_directory = os.path.dirname(catalog_file)
context.catalog_file = catalog_file
context.catalog = catalog.Catalog(context.catalog_name)
class MainFrame(wx.Frame):
def __init__(self, parent, id, title):
# parse command line arguments
self.parseArgs()
# set up log
self.logSetup() # set up log globally
# set up local log
self.log = logging.getLogger('pyCatalog.main')
# notify DEBUG turned ON
self.log.debug("DEBUG messages turned ON.")
# we call the init method of the inherited class
wx.Frame.__init__(self, parent, wx.ID_ANY, title, size=(600, 400))
# GUI stuff
self.setup_GUI()
## TODO: delete while subsituting with logging
# create console window to log messages
self.logViewer = cW.LogViewer(self, -1)
# Initialize catalog variable
self.catalog = None
# even if we haven't yet loaded a catalog
self.catalogSaved = True
# starts a timer of 10 second for refreshing log viewer content
self.timer = threading.Timer(10.0, self.logViewer.refreshLog)
self.timer.start()
self.buttonPanel.setLabel("no catalog loaded")
def logSetup(self):
#~ Level Numeric value
#~ -------------------- -----------------------
#~ CRITICAL 50
#~ ERROR 40
#~ WARNING 30
#~ INFO 20
#~ DEBUG 10
#~ NOTSET 0
# set up logging to file
logging.basicConfig(
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=LOGFILENAME,
filemode='w')
# set logging level
if self.options.debug == True:
logging.getLogger('pyCatalog').setLevel(logging.DEBUG)
else:
logging.getLogger('pyCatalog').setLevel(logging.WARNING)
## CHECK logging level
## print logging.getLogger('pyCatalog').getEffectiveLevel()
# define a Handler which writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.WARNING)
# set a format which is simpler for console use
formatter = logging.Formatter(
'%(name)-12s: %(levelname)-8s %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger('pyCatalog').addHandler(console)
def parseArgs(self):
# create new parser object
parser = optparse.OptionParser()
# set help message
parser.usage = "usage: %prog [options] [catalog to open]"
parser.add_option("-d",
"--debug",
action="store_true",
dest="debug",
help="Set debug messages ON.",
default=False)
##parser.add_option("-c", "--create", dest="catalog", help="Create new catalog.", default="")
(self.options, args) = parser.parse_args()
# initialize variable
self.options.catalog = ''
# recuperiamo l'eventuale nome del catalogo da aprire
if (len(args) > 0):
self.options.catalog = args[0]
def setup_GUI(self):
# main container (toolbar and splitter)
self.mainBox = wx.BoxSizer(wx.VERTICAL)
# add the toolbar
##self.toolbar = cW.CustomToolBar(self)
# add the button panel
self.buttonPanel = cW.CustomButtonPanel(self)
self.mainSplitter = wx.SplitterWindow(self, 1)
# left panel setUp
self.leftPanel = wx.Panel(self.mainSplitter, -1)
self.leftSizer = wx.BoxSizer(wx.HORIZONTAL)
self.deviceTree = cW.DeviceTree(self.leftPanel, -1)
self.leftSizer.Add(self.deviceTree, 1, wx.EXPAND)
self.leftPanel.SetSizer(self.leftSizer)
# right panel setUp
self.rightPanel = wx.Panel(self.mainSplitter, -1)
self.rightSizer = wx.BoxSizer(wx.HORIZONTAL)
self.fileTree = cW.FileTree(self.rightPanel, -1)
self.rightSizer.Add(self.fileTree, 1, wx.EXPAND)
self.rightPanel.SetSizer(self.rightSizer)
# set up main splitter
self.mainSplitter.SplitVertically(self.leftPanel, self.rightPanel)
self.mainSplitter.SetMinimumPaneSize(20) # to prevent unsplitting
self.mainSplitter.SetSashPosition(self.GetSize().x /
3) # set initial sash position
# setUp a StatusBar at the bottom of the window with 2 text field
self.statusBar = self.CreateStatusBar()
# setup 'Catalog' menu
self.catalogMenu = wx.Menu()
self.catalogMenu.Append(wx.ID_NEW, help=" Create new catalog")
wx.EVT_MENU(self, wx.ID_NEW, self.OnNewCatalog)
self.catalogMenu.Append(wx.ID_OPEN, help=" Open existing catalog")
wx.EVT_MENU(self, wx.ID_OPEN, self.OnOpen)
self.catalogMenu.Append(wx.ID_SAVE, help=" Save current catalog")
wx.EVT_MENU(self, wx.ID_SAVE, self.OnSaveCatalog)
self.catalogMenu.Append(wx.ID_SAVEAS,
help=" Save current catalog as ...")
wx.EVT_MENU(self, wx.ID_SAVEAS, self.OnSaveCatalogAs)
self.catalogMenu.AppendSeparator()
self.catalogMenu.Append(wx.ID_EXIT, help=" Terminate the program")
wx.EVT_MENU(self, wx.ID_EXIT, self.OnQuit)
# setup 'View' menu
self.viewMenu = wx.Menu()
self.viewMenu.Append(wx.ID_PROPERTIES, "Show log viewer",
" Show content of log file")
wx.EVT_MENU(self, wx.ID_PROPERTIES, self.showLogViewer)
# setup 'Tools' menu
self.toolsMenu = wx.Menu()
item = self.toolsMenu.Append(wx.NewId(), "Show category editor",
" Show category editor")
wx.EVT_MENU(self, item.GetId(), self.OnShowCategoryEditor)
# setup 'Help' menu
self.helpMenu = wx.Menu()
self.helpMenu.Append(wx.ID_ABOUT, "&About",
" Information about this program")
wx.EVT_MENU(self, wx.ID_ABOUT, self.OnAbout)
# setUp menubar
self.menuBar = wx.MenuBar()
self.menuBar.Append(self.catalogMenu, "&Catalog")
self.menuBar.Append(self.viewMenu, "&View")
self.menuBar.Append(self.toolsMenu, "&Tools")
self.menuBar.Append(self.helpMenu, "?")
self.SetMenuBar(self.menuBar)
# lay out widgets in main sizer
##self.mainBox.Add(self.toolbar, 0, wx.EXPAND)
self.mainBox.Add(self.buttonPanel, 0, wx.EXPAND)
self.mainBox.Add(self.mainSplitter, 1, wx.EXPAND)
self.SetSizer(self.mainBox)
# main frame events
self.Bind(wx.EVT_SIZE, self.OnSize)
self.Bind(wx.EVT_CLOSE, self.OnQuit)
# show main frame
self.Centre() # show window in the center of the screen
self.Show(True)
def newCatalog(self):
if (self.catalog != None and self.catalogSaved == False):
# request user confirmation
msg = wx.MessageDialog(
self,
"Current catalog has not been saved.\nDo you want to continue anyway ?",
"", wx.YES_NO | wx.ICON_EXCLAMATION)
response = msg.ShowModal()
if response == wx.ID_NO:
return
dlg = wx.FileDialog(self,
style=wx.FD_OVERWRITE_PROMPT | wx.FD_SAVE,
wildcard="Catalog files (*.db)|*.db")
retValue = dlg.ShowModal()
# exit if no file is choosen
if retValue == wx.ID_CANCEL:
return
path = os.path.join(dlg.GetDirectory(), dlg.GetFilename())
# CHECK if selected file already exists and try to delete it
if os.path.exists(path):
try: # to remove selected file
os.remove(path)
except Exception, e:
self.log.error(
"Error while trying to remove pre-existent file")
msg = wx.MessageDialog(
self,
"Cannot overwrite selected file. Please choose another one.",
"", wx.OK | wx.ICON_ERROR)
msg.ShowModal()
return
# create a new catalog at the specified path
self.catalog = catalog.Catalog()
try:
self.catalog.createNew(path)
except catalog.Error, e:
msg = wx.MessageDialog(
self,
"Cannot create catalog at given path. Please check log file for more details.",
"", wx.OK | wx.ICON_ERROR)
msg.ShowModal()
self.catalog = None
return
def setUp(self):
self.catalog = catalog.Catalog()
class pyCatalog:
def __init__(self):
# initialize variables
self.catalogSaved = True
self.catalog = None
# parse command line arguments
self.parseArgs()
# set up log
self.logSetup() # set up log globally
# set up local log
self.log = logging.getLogger('pyCatalog.main')
# notify DEBUG turned ON
self.log.debug("DEBUG messages turned ON.")
# load catalog
if self.loadCatalog():
self.interact()
else:
print "\nCannot load catalog, quitting."
print '\nbye\n'
sys.exit()
def loadCatalog(self):
# if user passed a catalog name, try to open it
if self.options.catalogToOpen != '':
self.catalog = catalog.Catalog()
try:
self.catalog.openCatalog(self.options.catalogToOpen)
except catalog.Error, e:
print "\nError while trying to open new catalog."
return False
# if user choose to open new catalog
if self.options.catalog != '':
self.catalog = catalog.Catalog()
try:
self.catalog.createNew(self.options.catalog)
except catalog.Error, e:
print "\nError while creating new catalog."
return False
try:
self.catalog.openCatalog(self.options.catalog)
except catalog.Error, e:
print "\nError while trying to open newly created catalog."
return False
def Main(argv):
try:
output_directory = "./simulation_output_tpl/".format()
if os.path.exists(output_directory):
shutil.rmtree(output_directory)
os.mkdir(output_directory)
dlambda = 0.6
lambda_obs_min = 3800.0
lambda_obs_max = 12600.0
templates_path = "./templates"
refPath = "/home/aschmitt/data/simu_linemodel/simulm_20160513/simulation_pfswlinemodel_20160513_10spcperbin/refz_bad12.txt"
refData = loadRef(refPath)
TFDir = "/home/aschmitt/data/simu_linemodel/simulm_20160513/simulation_pfswlinemodel_20160513_10spcperbin"
catalogPath = "/home/aschmitt/data/simu_linemodel/simulm_20160513/amazed/linecatalogs/linecatalogamazedvacuum_B10E.txt"
catLines = ctlg.Catalog(catalogPath)
filenameList = [a[0] for a in refData]
magList = [a[2] for a in refData]
tplList = [a[3] for a in refData]
zList = [a[1] for a in refData]
print("size of refData = {}".format(len(filenameList)))
print("first elt: file = {}, mag = {}, tpl = {}".format(
filenameList[0], magList[0], tplList[0]))
for k in range(len(filenameList)):
tpl_name = tplList[k]
z = zList[k]
magRoundedRef = magList[k]
#loop on the mag uncertainty, yes the mag ref value has been rounded +-0.1 during the creation of the
magRoundedCheckList = [
magRoundedRef - 0.05, magRoundedRef - 0.025, magRoundedRef,
magRoundedRef + 0.025, magRoundedRef + 0.05
]
filename = filenameList[k]
#prepare the template spc with the given mag
tplPath = os.path.join(templates_path, tpl_name)
tpl = spectrum.Spectrum(tplPath, 'template', snorm=False)
tfPath = os.path.join(TFDir, "{}_TF.fits".format(filenameList[k]))
spcTF = spectrum.Spectrum(tfPath, 'pfs', snorm=False)
print("tfpath = {}".format(tfPath))
maskOutLines = catLines.getMaskOutsideLines(spcTF.xvect, z)
tplCheckList = []
lstSqList = []
for mag in magRoundedCheckList:
tplTmp = tpl.copy()
tplTmp.extendWavelengthRangeRed(lambda_obs_max)
tplTmp.extendWavelengthRangeBlue(lambda_obs_min)
tplTmp.applyLyaExtinction(redshift=z)
tplTmp.applyRedshift(z)
# #Check magFound and mag to be the same
magFound = tplTmp.setMagIAB(mag)
# #
# if np.floor(magFound*10.0)!=np.floor(magRoundedRef*10.0):
# print("magRoundedRef = {}, and magFound={}".format(magRoundedRef, magFound))
# else:
# break
tplTmp.applyLambdaCrop(lambda_obs_min, lambda_obs_max)
tplTmp.interpolate(
dx=0.1) #high sampling for the synthesis process
tplTmp.correctZeros()
tplTmp.interpolate(dx=dlambda)
tplCheckList.append(tplTmp)
lstErr = 0.0
for kk in range(spcTF.n - 1):
if maskOutLines[kk] == 1.0:
lstErr += (spcTF.yvect[kk] - tplTmp.yvect[kk])**2
print("for k={}, mag={}, lstSq = {}".format(k, mag, lstErr))
lstSqList.append(lstErr)
kmin = np.argmin(lstSqList)
print("found argmin = {}".format(kmin))
tpl = tplCheckList[kmin]
exported_fits_path = tpl.exportFits(output_directory,
"{}_B".format(filename),
addNoise=False,
exportNoiseSpectrum=False,
noiseSigma=1.5e-19,
addNameSuffix=False)
exported_fits_name = os.path.split(exported_fits_path)[1]
print("Exported: {}".format(exported_fits_name))
#stop
print("END of continuum creation script.")
except (KeyboardInterrupt):
exit()
"--debug", type=int, default=argparse.SUPPRESS
) #Optional, we will use the value from the config file
parser.add_argument("--catalog_path", type=str,
default=None) #necessary if clustering
parser.add_argument("--clusters_file_path", type=str,
default=None) #necessary if clustering
#parser.add_argument("--redirect_stdout_stderr",type=bool, default=False)
args = parser.parse_args()
cfg = config.Config(args)
#Load metadata (ONLY WHEN LOCATION DETECTION)
cat = None
if args.catalog_path is not None:
cat = catalog.Catalog()
cat.import_json(args.catalog_path)
clusters = clusters.Clusters()
clusters.import_json(args.clusters_file_path)
dataset_dir = args.prep_data_dir
output_dir = args.tfrecords_dir
#if args.redirect_stdout_stderr:
# stdout_stderr_file = open(os.path.join(checkpoint_dir, 'stdout_stderr_file.txt'), 'w')
# sys.stdout = stderr = stdout_stderr_file
tf.app.run()
#if args.redirect_stdout_stderr:
# stdout_stderr_file.close()
chans = freq_catalog + (NP.arange(nchan) - 0.5 * nchan) * freq_resolution * 1e3 / 1e9 # in GHz
bpass = 1.0*NP.ones(nchan)
# Do the following next few lines only for MWA
notch_interval = NP.round(1.28e6 / (freq_resolution * 1e3))
# bpass[::notch_interval] = 0.0
# bpass[1::notch_interval] = 0.0
# bpass[2::notch_interval] = 0.0
oversampling_factor = 1.0
# window = DSP.shaping(nchan, 1/oversampling_factor*CNST.rect_bnw_ratio, shape='bnw', peak=1.0)
window = DSP.shaping(nchan, 1/oversampling_factor, shape='rect', peak=1.0)
bpass *= window
ctlgobj = CTLG.Catalog(freq_catalog*1e9, NP.hstack((ra_deg.reshape(-1,1), dec_deg.reshape(-1,1))), fpeak, spectral_index=spindex)
# ctlgobj = CTLG.Catalog(freq_catalog, NP.hstack((ra_deg.reshape(-1,1), dec_deg.reshape(-1,1))), fpeak)
skymod = CTLG.SkyModel(ctlgobj)
A_eff = 16.0 * (0.5 * FCNST.c / (freq_catalog * 1e9))**2
intrfrmtr = RI.Interferometer('B1', [1000.0, 0.0, 0.0], chans, telescope='mwa',
latitude=-26.701, A_eff=A_eff, freq_scale='GHz')
obs_mode = 'drift'
Tsys = 440.0 # in Kelvin
t_snap = 60.0 # in seconds
t_obs = 0.5 * 3600.0 # in seconds
pointing_init = [0.0, intrfrmtr.latitude]
lst_init = 0.0 * 15.0 # in degrees
flux_unit = 'Jy'
freq_catalog = freq/1e9 # in GHz
spindex = 0.0
if use_GSM:
gsm_file = '/data3/t_nithyanandan/project_MWA/foregrounds/gsmdata.fits'
hdulist = fits.open(gsm_file)
pixres = hdulist[0].header['PIXAREA']
gsm_table = hdulist[1].data
ra_deg = gsm_table['RA']
dec_deg = gsm_table['DEC']
temperatures = gsm_table['T_{0:.0f}'.format(freq/1e6)]
fluxes = temperatures * (2.0* FCNST.k * freq**2 / FCNST.c**2) * pixres / CNST.Jy
spindex = gsm_table['spindex'] + 2.0
ctlgobj = CTLG.Catalog(freq_catalog*1e9, NP.hstack((ra_deg.reshape(-1,1), dec_deg.reshape(-1,1))), fluxes, spectral_index=spindex, src_shape=NP.hstack((NP.sqrt(pixres*4/NP.pi)*NP.ones(fluxes.size).reshape(-1,1),NP.sqrt(pixres*4/NP.pi)*NP.ones(fluxes.size).reshape(-1,1),NP.zeros(fluxes.size).reshape(-1,1))), src_shape_units=['radian','radian','degree'])
fg_str = 'gsm'
hdulist.close()
elif use_SUMSS:
SUMSS_file = '/data3/t_nithyanandan/project_MWA/foregrounds/sumsscat.Mar-11-2008.txt'
catalog = NP.loadtxt(SUMSS_file, usecols=(0,1,2,3,4,5,10,12,13,14,15,16))
ra_deg = 15.0 * (catalog[:,0] + catalog[:,1]/60.0 + catalog[:,2]/3.6e3)
dec_dd = NP.loadtxt(SUMSS_file, usecols=(3,), dtype="|S3")
sgn_dec_str = NP.asarray([dec_dd[i][0] for i in range(dec_dd.size)])
sgn_dec = 1.0*NP.ones(dec_dd.size)
sgn_dec[sgn_dec_str == '-'] = -1.0
dec_deg = sgn_dec * (NP.abs(catalog[:,3]) + catalog[:,4]/60.0 + catalog[:,5]/3.6e3)
fmajax = catalog[:,7]
fminax = catalog[:,8]
fpa = catalog[:,9]
dmajax = catalog[:,10]
def genSimple(self,
N=10000,
bias=2.0,
zdist=ZDist(),
rmagdist=MagDist(),
edist=EllipticityDist(),
algorithm="peaks",
extravar=0.5):
"""
Generate a catalog of fake objects
Parameters
----------
N : int
Number of objects to generata
bias : float
The bias at reference redshift (as specified by zdist.refz())
The algorithm actually operates with z=0 delta field so we convert
this bias factor to bias factor today.
zdist: object
Object specifying redshift window function (Gaussian for the time being)
rmagdist: object
Object specifying rmag distribution (Gaussian for the time being)
edist: object
Object specifying intrinsic ellipticity distribution
algorithm: string
How to distribute objects. Valid options are i) "peaks" (put galaxy if
delta value + extra white noise > limit determined so that bias is right).
ii) "lognormal" probabilistically sample from lognormal transformation
(with the right bias), iii) "random" for random catalogs (just window function)
extravar: float
Extra variance if algorithm=="peaks"
"""
random = False
peaks = True
if algorithm == "peaks":
pass
elif algorithm == "lognormal":
peaks = False
elif algorithm == "random":
random = True
else:
print("Bad algorithm")
stop()
## we have deltafield at z=0, so first get bias right
# now convert bias at effective z to bias now
gf = ct.get_growth_function(self.cosmology, np.array([0,
zdist.refz()]))
biasnow = bias * gf[1] / gf[0]
if (peaks):
cg0 = (1. + extravar)
#Now need to calculate clipping at which I will produce sufficient number of objects
# from maple
fun = lambda l: (-(math.exp(-l**2 / cg0**2 / 0.2e1) * math.sqrt(
0.2e1) * math.pi**(-0.1e1 / 0.2e1) / cg0 / (math.erf(
math.sqrt(0.2e1) / cg0 * l / 0.2e1) - 0.1e1)) - biasnow)
## first healtcheck
if ((fun(-3) > 0) or (fun(3) < 0)):
print("You are asking for a bias which is a bit extreme.")
print(fun(-3) + biasnow, fun(3) + biasnow)
stop()
lim = brentq(fun, -3.0, 3.0)
## lim is now cut in sources of deltafield + extra*var. Let us just go through all of them
lim *= sqrt(self.var)
esg = sqrt(self.var * extravar)
print("Picking up above ", lim, " with scatter ", esg, " sig=",
sqrt(self.var))
else:
lognmax = exp(1 + self.delta.max() * biasnow)
print("lognmax=", lognmax)
tx = self.Nx / 2
ty = self.Ny / 2
zlist = []
dlist = []
ralist = []
rlist = []
klist = []
thetao2sq = self.thetao2**2
tcc = 0
for cc in xrange(N):
while (True):
tcc += 1
z = zdist.sampleZ()
kre = self.z2k(z) ## radial distance in indices
## now sample radially
r = sqrt(np.random.uniform() * thetao2sq)
phi = np.random.uniform(0, 2 * math.pi)
ra = r * np.sin(phi)
dec = r * np.cos(phi)
k = int(kre)
i = int(ra * kre + tx)
j = int(dec * kre + ty)
accept = random
if (not accept):
delta = self.delta[i, j, k]
if (peaks):
if esg > 0:
accept = (delta + np.random.normal(0, esg)) > lim
else:
accept = delta > lim
else:
accept = (exp(1 + delta * biasnow) /
lognmax) > np.random.uniform()
## now try to sample
if accept:
## ok, now we really have it.
zlist.append(z)
dlist.append(dec)
ralist.append(ra)
rlist.append(kre * self.grid_spacing_h_Mpc)
break
print("Proposal accept rate:", float(N) / tcc)
toret = cat.Catalog(N, max_sep=self.max_sep)
toret["ra"] = np.array(ralist)
toret["dec"] = np.array(dlist)
toret["z"] = np.array(zlist)
toret["r"] = np.array(rlist)
toret["rmag"] = np.array(
[rmagdist.sampleRMag(bias) for i in xrange(N)])
toret["e1"] = np.array([edist.sampleEOne() for i in xrange(N)])
toret["e2"] = np.array([edist.sampleEOne() for i in xrange(N)])
toret["g1"] = np.zeros(N)
toret["g2"] = np.zeros(N)
return toret
