def finish(self, runUntil, runId): if runUntil != len(self.modules) and self.commandline.noCopy != True: print '... copying data bank snapshot, disable with --noCopy' os.chdir(self.cwd) if runId != None: dest = self.config.get("options", "data_dir") + 'data_bank' + str(runUntil-1) + '_run' + str(runId) + '.h5'; else: dest = self.config.get("options", "data_dir") + 'data_bank' + str(runUntil-1) + '.h5'; if self.config.has_option("options", "ptrepack"): os.system(self.config.get("options", "ptrepack") + ' -o ' + self.config.get("options", "data_dir") + 'data_bank.h5:/ ' + dest + ':/') else: os.system('ptrepack -o ' + self.config.get("options", "data_dir") + 'data_bank.h5:/ ' + dest + ':/') #saving run data for plotting if runUntil == len(self.modules): print '... saving data for runId ' + str(runId) os.chdir(self.cwd) with archive.archive('../../data/data_bank.h5') as src: with archive.archive('../../data/run' + str(runId) + '.h5', 'w') as dest: for f in src['/']: #Only Copy the config (first character uppercase) if (f[0].isupper()): print '/' , f copyHDF5Path(src, '/' + f, dest) #TODO: save more interesting data (nselectedgal, etc) dest['/Glue/run/runId'] = runId dest['/Glue/run/ngal'] = len(src['/gal/galaxy_index'])
def revert(s, *_): """ Revert back to old version! """ # check if file is in its old location source = s.index["source"] if not os.path.isfile(source): if confirm("Cannot find original file: {}\nPlesae provide a folder to recover the file in.".format(source)): path = cmds.fileDialog2(fileMode=3) if not path: return source = os.path.join(path[0], os.path.basename(source)) else: return if confirm("You will lose all unsaved changes if you continue.\nAre you sure?"): note = "AUTOSAVE: Recovering version '{}'".format(s.index["note"]) sup = popup.Startup(note) with sup: # we lied... we will save unsaved changes anyway! cmds.file(rename=source) cmds.file(save=True, force=True) archive.archive(note, source) # Recover file with zipfile.ZipFile(s.archive, "r") as z: with open(source, "w") as f: f.write(z.read(s.index["scene"])) # Load the file cmds.file(source, open=True, force=True)
def DetectPhotons(): # ----------------------------------------------------- # Import data and params # ----------------------------------------------------- print print "........................." print " Entering Generate Image " print "........................." databank_file = '../../../data/data_bank.h5' with archive.archive(databank_file, 'r') as ar: PhotonEnergies = ar['/Photon/Energy'] PhotonArrivalTimes = ar['/Photon/ArrivalTime'] PhotonPositionsX = ar['/Photon/PositionsX'] PhotonPositionsY = ar['/Photon/PositionsY'] position_resolution = ar['/Instrument/Detector/position_resolution'] time_resolution = ar['/Instrument/Detector/time_resolution'] energy_resolution = ar['/Instrument/Detector/energy_resolution'] seed_position_x = 9999 seed_position_y = 9999 seed_time = 9999 seed_energy = 9999 # Detect Photon Positions PhotonPositionsX_Detect = DetectQuantity(PhotonPositionsX, position_resolution, seed_quantity=seed_position_x) PhotonPositionsY_Detect = DetectQuantity(PhotonPositionsY, position_resolution, seed_quantity=seed_position_y) # Detect Photon Times PhotonArrivalTimes_Detect = DetectQuantity(PhotonArrivalTimes, time_resolution, seed_quantity=seed_time) # Detect Photon Energies PhotonEnergies_Detect = DetectQuantity(PhotonEnergies, position_resolution, seed_quantity=seed_energy) # ----------------------------------------------------- # Export data and params # ----------------------------------------------------- with archive.archive(databank_file, 'r') as ar: ar['/Photon/Energy_Detected'] = PhotonEnergies ar['/Photon/ArrivalTime_Detected'] = PhotonArrivalTimes ar['/Photon/PositionsX_Detected'] = PhotonPositionsX ar['/Photon/PositionsY_Detected'] = PhotonPositionsY ar['/Photon/Number_of_Photons'] = nphoton print print "........................." print " Exiting Generate Image " print "........................."
def HourlyBackup(self,*args): print 'Performing hourly save' if self.running: self.StopAutoSave() archive(join(self.runningfolder,self.foldername),self.archivefolder, folder='hourly',update_currentref=False,overwrite=True) if self.running: self.ResumeAutoSave() print 'Hourly save complete'
def mergeDatafile(self, datafile): #merge datafiles into the databank with archive.archive( self.config.get("options", "data_dir") + self.config.get("options", "data_bank"), 'a') as dest: with archive.archive(datafile, 'r') as src: print "... copying " + self.config.get( "data_bank", datafile) + " into " + self.config.get( "options", "data_bank") copyHDF5Path(src, "/", dest)
def DailyBackup(self,*args): print 'Performing daily backup' if self.running: self.StopAutoSave() archive(join(self.runningfolder,self.foldername),self.archivefolder) if os.path.exists(join(self.archivefolder,'hourly')): shutil.rmtree(join(self.archivefolder,'hourly')) if self.running: self.ResumeAutoSave() print 'Daily backup complete'
def GeneratePhotons(): # ----------------------------------------------------- # Import data and params # ----------------------------------------------------- print print "........................." print " Entering Generate Image " print "........................." databank_file = '../../../data/data_bank.h5' with archive.archive(databank_file, 'r') as ar: exptime = ar['/SurveyDesign/exposure_time'] magnitude_galaxy = ar['/Galaxy/magnitude/'] flux_spectrum_galaxy = ar['/Galaxy/spectrum'] image = ar['/Galaxy/image'] # Calculate number of photons nphoton, = magphoton(magnitude_galaxy, exptime=exptime) # Sample Photon Positions PhotonPositionsX, PhotonPositionsY = SamplePositions(image, nphoton) # Sample Photon Times PhotonArrivalTimes = SampleTimeArrival(nphoton, exptime, start_time=0.0, seed_time=9999) # Sample Photon Energies PhotonEnergies = SampleEnergySpectrum(nphoton) # Generate Photon Identification Numbers photon_id = numpy.arange(nphoton) # ----------------------------------------------------- # Export data and params # ----------------------------------------------------- with archive.archive(databank_file, 'r') as ar: ar['/Photon/Energy'] = PhotonEnergies ar['/Photon/ArrivalTime'] = PhotonArrivalTimes ar['/Photon/PositionsX'] = PhotonPositionsX ar['/Photon/PositionsY'] = PhotonPositionsY ar['/Photon/Number_of_Photons'] = nphoton print print "........................." print " Exiting Generate Image " print "........................."
def main(): args = parse_args() try: if args.action == "extract": if args.verbose: print "Extracting archive" extract.extract(infile=args.input, outfile=args.output, verbose=args.verbose) elif args.action == "archive": if args.verbose: print "Creating archive" archive.archive(infile=args.input, compression=args.compression, outfile=args.output, verbose=args.verbose) except (extract.ExtractException, archive.ArchiveException) as ex: print >> sys.stderr, ex.msg return ex.code return 0
def timings(self, runId): sumTime = 0 for i in range(len(self.modules)): try: sumTime += self.modules[i]['endtime'] - self.modules[i]['starttime'] except KeyError: pass timelist=[] modulelist=[] os.chdir(self.cwd) timingdata = [] for i in range(len(self.modules)): try: print "[" + str(i) + "] %s\t%.2fs\t%.1f%%\t%.0fMB" % (self.modules[i]['name'].ljust(25, " "), self.modules[i]['endtime'] - self.modules[i]['starttime'], (self.modules[i]['endtime'] - self.modules[i]['starttime']) / sumTime *100, self.modules[i]['memory_rss'] / (1024*1024)) modulelist.append(self.modules[i]['name']) timelist.append(self.modules[i]['endtime'] - self.modules[i]['starttime']) timingdata.append([self.modules[i]['name'].encode("ascii","ignore"), str(self.modules[i]['endtime'] - self.modules[i]['starttime']), str(self.modules[i]['starttime']), str(self.modules[i]['endtime']), str(self.modules[i]['memory_rss'])]) except KeyError: print "[" + str(i) + "] " + (self.modules[i]['name'].ljust(25, " ")) timingdata.append(['', '', '', '', '']) print "%s\t%.2fs\t%.1f%%" % ("Total".ljust(25, " "), sumTime, 100) os.chdir(self.cwd) with archive.archive('../../data/data_bank.h5', 'a') as ar: ar['/Glue/run/timings'] = timingdata ar['/Timing/timelist'] = timelist ar['/Timing/modulelist'] = modulelist if (runId == None): ar['/Glue/run/runId'] = '' else: ar['/Glue/run/runId'] = runId
def __init__(self): QtGui.QMainWindow.__init__(self) # member variables definition self.M1 = 0.0 self.M2 = 0.0 self.L1 = 0.0 self.L2 = 0.0 self.th1 = 0.0 self.th2 = 0.0 self.w1 = 0.0 self.w2 = 0.0 self.ts = 0.0 self.te = 0.0 self.dt = 0.0 # setup timer self.timer = QtCore.QTimer(self) self.timer.timeout.connect(self.timerCallback) self.timer_count = 0 self.current_time = 0.0 # dimension of the vtk view self.len_convert_factor = 100.0 # length convert factor 1m = 100 pixels self.X_lim = 500.0 self.Y_lim = 300.0 # archive (history) of the results self.archive = archive() # Setup UI widgets self.setupUi(self)
def mergeParamfile(self, file): param = ConfigParser.RawConfigParser() param.read(file) with archive.archive('../../data/data_bank.h5', 'a') as ar: print "Importing sections:", param.sections(), "from parameter file:", file for section in param.sections(): for option in param.options(section): ar["/" + section + "/" + option] = eval(param.get(section, option))
def loop_over_gal(self): #======================================================================================= # LOOP over galaxies # default add both ... 0 for both # 1: don't do it #======================================================================================= check_fiber_sel = numpy.where(self.fiber_selection_flag == True) self.coeffs = self.coeffs[check_fiber_sel, :] self.coeffs = numpy.reshape(self.coeffs, self.coeffs.shape[1:]) self.z_true = self.z_true[check_fiber_sel] flux_new = [] lam_new = [] galax_index_new = [] Ngal_new = len(check_fiber_sel[0]) print "... Loop over galaxies to create spectral templatesi [this number has been cut]", Ngal_new for i in range(Ngal_new): # calculate spectrum from coefficients flux_temporary, lam_temp = self.clean_spectrum(i) # Add temporary to output/new flux_new.append(numpy.array(flux_temporary)) lam_new.append(numpy.array(lam_temp)) #--------------------------------------------------------------------------------------- # END LOOP over galaxies #--------------------------------------------------------------------------------------- # make output into numpy arrays flux_new = numpy.array(flux_new) lam_new = numpy.array(lam_new) # DIAGNOSTIC print "... plotting diagnostics" Plotters.plot_spectrum(lam_new[0, :], flux_new[0, :], xmin=5000, xmax=10000, fig_number=0, plot_number=0, title='Galaxy Spectrum', base_directory='.', filename_addendum="", show_plot=show_plot) print "... Loop finished, ngal with spectra = ", len(flux_new) print "... shape of output arrays:" print "... ... wavelength", lam_new.shape print "... ... flux ", flux_new.shape # Output print "... output to databank" with archive.archive(databank_file, 'a') as ar: ar['/gal/wavelength'] = lam_new ar['/gal/flux'] = flux_new
def cosmos_insert_ra_dec(inputfile,outputfile): dataArray = importArrayFromCSVFile(inputfile) right_ascension = dataArray[:,1] declination = dataArray[:,2] # append to HDF file with archive.archive(outputfile,'a') as ar: ar['/gal/ra_true'] = right_ascension ar['/gal/dec_true'] = declination
def mergeParamfile(self, file): param = ConfigParser.RawConfigParser() param.read(file) with archive.archive('../../data/data_bank.h5', 'a') as ar: print "Importing sections:", param.sections( ), "from parameter file:", file for section in param.sections(): for option in param.options(section): ar["/" + section + "/" + option] = eval( param.get(section, option))
def cosmos_hdf_converter(inputfile,outputfile): dataArray = importArrayFromCSVFile(inputfile) # careful: index here is shifted by 1 wrt the index in the COSMOS header galaxy_index = dataArray[:,0] photometric_redshift = dataArray[:,1] spectroscopic_redshift = dataArray[:,31] magnitude_g = dataArray[:,14] magnitude_r = dataArray[:,15] magnitude_i = dataArray[:,16] magnitude_z = dataArray[:,17] magnitude_Y = dataArray[:,18] magnitude_J = dataArray[:,19] magnitude_H = dataArray[:,20] magnitude_K = dataArray[:,21] magnitude_g_error = dataArray[:,22] magnitude_r_error = dataArray[:,23] magnitude_i_error = dataArray[:,24] magnitude_z_error = dataArray[:,25] magnitude_Y_error = dataArray[:,26] magnitude_J_error = dataArray[:,27] magnitude_H_error = dataArray[:,28] magnitude_K_error = dataArray[:,29] #???ra = dataArray[:,XXX] # Stephanie gave us a new file we just need to import #???dec = dataArray[:,XXX] # write to HDF5 file with archive.archive(outputfile,'a') as ar: ar['/gal/galaxy_index'] = galaxy_index ar['/gal/redshift_photometric'] = photometric_redshift ar['/gal/redshift_spectroscopic'] = spectroscopic_redshift ar['/gal/magnitude_g'] = magnitude_g ar['/gal/magnitude_r'] = magnitude_r ar['/gal/magnitude_i'] = magnitude_i ar['/gal/magnitude_z'] = magnitude_z ar['/gal/magnitude_y'] = magnitude_Y ar['/gal/magnitude_j'] = magnitude_J ar['/gal/magnitude_h'] = magnitude_H ar['/gal/magnitude_k'] = magnitude_K ar['/gal/magnitude_error_g'] = magnitude_g_error ar['/gal/magnitude_error_r'] = magnitude_r_error ar['/gal/magnitude_error_i'] = magnitude_i_error ar['/gal/magnitude_error_z'] = magnitude_z_error ar['/gal/magnitude_error_y'] = magnitude_Y_error ar['/gal/magnitude_error_j'] = magnitude_J_error ar['/gal/magnitude_error_h'] = magnitude_H_error ar['/gal/magnitude_error_k'] = magnitude_K_error
def convert_sky_bkg(inputfile,outputfile): # from skybg_50_10 table = asciitable.read(inputfile) wave = [] power = [] for i in range(len(table)): wave.append(table[i][0]) power.append(table[i][1]) with archive.archive(outputfile,'a') as ar: ar['/sky/background/wave'] = numpy.array(wave) ar['/sky/background/power'] = numpy.array(power)
def convert_extinction(inputfile,outputfile): # from palomar table = asciitable.read(inputfile) abswave = [] abstemp = [] for i in range(len(table)): abswave.append(table[i][0]) abstemp.append(table[i][1]) with archive.archive(outputfile,'a') as ar: ar['/sky/extinction/wave'] = numpy.array(abswave) ar['/sky/extinction/power'] = numpy.array(abstemp)
def update(args): archive_cfg = read_subcmd_config('archive') rename_cfg = read_subcmd_config('rename') args.archive_dir = args.archive_dir or archive_cfg['archive_dir'] if not os.path.isdir(args.archive_dir): raise RuntimeError("archive-dir {} is not a directory".format( args.archive_dir)) files = filter(tarfile.is_tarfile, collect_files(args.archive_dir)) latest = natsorted(files, reverse=True)[0] args.archive = [latest] args.extract_dir = None archive.extract(args) rename.rename(args) args.path = [rename_cfg['out_dir']] archive.archive(args)
def importContinuumSensitivityFromCSVFile(inputfile,outputfile): dataArray = importArrayFromCSVFile(inputfile) wavelength = dataArray[:,0] * 1e-10 # conversion Angstroem to meter signal_to_noise_continuum = dataArray[:,1] signal_to_noise_emission_line = dataArray[:,2] # write to HDF5 file with archive.archive(outputfile,'a') as ar: ar['/instrument/sensitivity/wavelength_grid'] = wavelength ar['/instrument/sensitivity/signal_to_noise_continuum'] = signal_to_noise_continuum ar['/instrument/sensitivity/signal_to_noise_emission_line'] = signal_to_noise_emission_line
def GenerateImage(): # ----------------------------------------------------- # Import data and params # ----------------------------------------------------- print print "........................." print " Entering Generate Image " print "........................." databank_file = '../../../data/data_bank.h5' with archive.archive(databank_file, 'r') as ar: npix = ar['/Instrument/Detector/Number_of_Pixels'] profile_type = ar['/Instrument/Detector/Profile_Type'] # ----------------------------------------------------- # Perform Calculations # ----------------------------------------------------- # Create 2-D Array of points image = numpy.zeros( (npix,npix), dtype=float) # create profile and add to image image = GenerateProfile( image, npix, profile_type , sigma_x=1., sigma_y=1., correlation=0.) # ----------------------------------------------------- # Export data and params # ----------------------------------------------------- with archive.archive(databank_file, 'r') as ar: ar['/Galaxy/image'] print print "........................." print " Exiting Generate Image " print "........................."
def timings(self, runId): sumTime = 0 for i in range(len(self.modules)): try: sumTime += self.modules[i]['endtime'] - self.modules[i][ 'starttime'] except KeyError: pass timelist = [] modulelist = [] os.chdir(self.cwd) timingdata = [] for i in range(len(self.modules)): try: print "[" + str(i) + "] %s\t%.2fs\t%.1f%%\t%.0fMB" % ( self.modules[i]['name'].ljust(25, " "), self.modules[i]['endtime'] - self.modules[i]['starttime'], (self.modules[i]['endtime'] - self.modules[i]['starttime']) / sumTime * 100, self.modules[i]['memory_rss'] / (1024 * 1024)) modulelist.append(self.modules[i]['name']) timelist.append(self.modules[i]['endtime'] - self.modules[i]['starttime']) timingdata.append([ self.modules[i]['name'].encode("ascii", "ignore"), str(self.modules[i]['endtime'] - self.modules[i]['starttime']), str(self.modules[i]['starttime']), str(self.modules[i]['endtime']), str(self.modules[i]['memory_rss']) ]) except KeyError: print "[" + str(i) + "] " + (self.modules[i]['name'].ljust( 25, " ")) timingdata.append(['', '', '', '', '']) print "%s\t%.2fs\t%.1f%%" % ("Total".ljust(25, " "), sumTime, 100) os.chdir(self.cwd) with archive.archive('../../data/data_bank.h5', 'a') as ar: ar['/Glue/run/timings'] = timingdata ar['/Timing/timelist'] = timelist ar['/Timing/modulelist'] = modulelist if (runId == None): ar['/Glue/run/runId'] = '' else: ar['/Glue/run/runId'] = runId
def finish(self, runUntil, runId): if runUntil != len(self.modules) and self.commandline.noCopy != True: print '... copying data bank snapshot, disable with --noCopy' os.chdir(self.cwd) if runId != None: dest = self.config.get( "options", "data_dir") + 'data_bank' + str( runUntil - 1) + '_run' + str(runId) + '.h5' else: dest = self.config.get( "options", "data_dir") + 'data_bank' + str(runUntil - 1) + '.h5' if self.config.has_option("options", "ptrepack"): os.system( self.config.get("options", "ptrepack") + ' -o ' + self.config.get("options", "data_dir") + 'data_bank.h5:/ ' + dest + ':/') else: os.system('ptrepack -o ' + self.config.get("options", "data_dir") + 'data_bank.h5:/ ' + dest + ':/') #saving run data for plotting if runUntil == len(self.modules): print '... saving data for runId ' + str(runId) os.chdir(self.cwd) with archive.archive('../../data/data_bank.h5') as src: with archive.archive('../../data/run' + str(runId) + '.h5', 'w') as dest: for f in src['/']: #Only Copy the config (first character uppercase) if (f[0].isupper()): print '/', f copyHDF5Path(src, '/' + f, dest) #TODO: save more interesting data (nselectedgal, etc) dest['/Glue/run/runId'] = runId dest['/Glue/run/ngal'] = len(src['/gal/galaxy_index'])
def import_convert_spectra_templates(inputfile,outputfile): #input files #inputfile = 'k_nmf_derived.default.fits' # make this file name a parameter in one of the ini files #inputfile = 'k_nmf_derived.newdefault.fits' # open file hdunum = 0 hdu = pyfits.open(inputfile)[hdunum] # header ## read in header header = hdu.header ## read in header element nt = header['nt'] #print '... number of templates', nt # read in table elements loglam = numpy.array(numpy.log10(pyfits.open(inputfile)[11].data)) tspec_v0 = numpy.array(pyfits.open(inputfile)[1].data) tspec_v0_nl = numpy.array(pyfits.open(inputfile)[2].data) tspec_v0_nd = numpy.array(pyfits.open(inputfile)[3].data) tspec_v0_nd_nl = numpy.array(pyfits.open(inputfile)[4].data) tspec_v300 = numpy.array(pyfits.open(inputfile)[5].data) tspec_v300_nl = numpy.array(pyfits.open(inputfile)[6].data) tspec_v300_nd = numpy.array(pyfits.open(inputfile)[7].data) tspec_v300_nd_nl= numpy.array(pyfits.open(inputfile)[8].data) lspec_v300 = numpy.array(pyfits.open(inputfile)[9].data) # tmass = pyfits.open(inputfile)[17].data # tmetallicity = pyfits.open(inputfile)[18].data # tmass300 = pyfits.open(inputfile)[19].data # tmass1000 = pyfits.open(inputfile)[20].data # tmremain = pyfits.open(inputfile)[24].data # write to databank with archive.archive(outputfile,'a') as ar: ar['/spectral_templates/num_spectral_templates'] = nt # number of spectral spectral_templates; nt ar['/spectral_templates/log10_wavelength'] = loglam ar['/spectral_templates/template_spectrum_v0'] = tspec_v0 # not smoothed ar['/spectral_templates/template_spectrum_v0_nolines'] = tspec_v0_nl ar['/spectral_templates/template_spectrum_v0_nodust'] = tspec_v0_nd ar['/spectral_templates/template_spectrum_v0_nodust_nolines'] = tspec_v0_nd_nl ar['/spectral_templates/template_spectrum_v300'] = tspec_v300 # smoothed to 300kms(res) ar['/spectral_templates/template_spectrum_v300_nolines'] = tspec_v300_nl ar['/spectral_templates/template_spectrum_v300_nodust'] = tspec_v300_nd ar['/spectral_templates/template_spectrum_v300_nodust_nolines'] = tspec_v300_nd_nl ar['/spectral_templates/lspec_v300'] = lspec_v300
def importEmissionLineSensitivityFromCSVFile(inputfile,outputfile): dataArray = importArrayFromCSVFile(inputfile) wavelength = dataArray[:,0] * 1e-10 # conversion Angstroem to meter signal_to_noise_at_3sigma = dataArray[:,1] signal_to_noise_at_5sigma = dataArray[:,2] # write to HDF5 file with archive.archive(outputfile,'a') as ar: ar['/instrument/sensitivity/emission_line_wavelengths'] = wavelength # ar['/gal/wavelength/unit'] = 'meter' ar['/instrument/sensitivity/signal_to_noise_sigma3'] = signal_to_noise_at_3sigma ar['/instrument/sensitivity/signal_to_noise_sigma5'] = signal_to_noise_at_5sigma ar['/instrument/sensitivity/exposure_time'] = 3000
def test(self): print("Running Full Test Sequence") #the ingest function sorts and moves files by date into the working/media directory ingest.ingest(ingestdir, workingdir) #the crawl function performs a hash index of all files in the target directories workingdirsum = crawl.crawl(True, workingdir, jsondatadir) archivedirsum = crawl.crawl(False, archivedir, jsondatadir) #the dedupe function combines all hash indexes and analyzes the dataset for duplicates data_files = glob.glob(jsondatadir + '/*.json') #run the dedupe function dedupe.dedupe(data_files, duplicatedir) #after the dedupe function has moved duplicaes out, reindex workingdirsum = crawl.crawl(True, workingdir, jsondatadir) #the archive function pulls from the working/media directory and pools into sized volumes archive.archive(archivedir, jsondatadir, workingdir, mediasize) #validate that all files in duplicates exist elsewhere before moving to validated validate.validate(duplicatedir, workingdir, archivedir, validateddir) print("Daily Job Completed Successfully")
def __init__(self,parent,name,*args,**kwargs): self.name=name self.manga=Manga(self.name) self.archive=archive(self.name) if not "relief" in kwargs: kwargs["relief"]=RAISED if not "borderwidth" in kwargs: kwargs["borderwidth"]=3 Frame.__init__(self,parent,*args,**kwargs) self.__PICTURE=Label(self,image=self.getImage()) self.__PICTURE.grid(row=0,column=0,sticky=N+S+E+W) self.grid_rowconfigure(0,weight=1) self.grid_columnconfigure(0,weight=1) self.__NAME=Label(self,text=self.name) self.__NAME.grid(row=1,column=0,sticky=S) self.__CHAPTERS=Label(self,text="{} Chapters".format(len(self.manga.chapterList_have))) self.__CHAPTERS.grid(row=2,column=0,sticky=S) self.inProgress=False #to tell if the button should say update or cancel in the lower menu self.downloadInProgress=False self.archivingInProgress=False
def run_async(self): startProcessTime = time.time() if not self.params['skipArchive']: # retrieve images from lt-archive self.logger.info("(process.run_async) retrieving images from archive") ltarchive = archive(self.params['path_pw_list'], self.params['archive_credentials_id'], self.params['skycam_lup_db_credentials_id'], self.err, self.logger) ## search for images matching criteria MySQLLogFile = self.params['resPath'] + "skycamfiles" ltarchive.getMySQLLog(self.params['resPath'], "skycam", "skycam", self.params['dateFrom'], self.params['dateTo'], self.params['instrument'], MySQLLogFile) ## get the data ltarchive.getData(MySQLLogFile, self.params['resPath']) else: for f in os.listdir(self.params['tmpMockPath']): shutil.copyfile(self.params['tmpMockPath'] + str(f), self.params['resPath'] + str(f)) # decompress and sort images by ascending datetime decompress_files(self.params['resPath'], self.err, self.logger) images = sort_image_directory_UTC(self.params['resPath'], self.err, self.logger) if not images: self.err.setError(-12) self.err.handleError() # START THE PIPELINE pipe = pipeline(self.params, self.err, self.logger) # spawn pipeline instance pipe.run(images) # and run for these images # log error code with open(self.params['resPath'] + 'res.exitcode', 'w') as f: f.write(str(self.err.getError())) # zip files in directory and purge res dir archive_name = self.params['dateFrom'].replace(" ", "T").replace("-", "").replace(":", "") + ".tar" zip_output_files_in_directory(self.params['resPath'], archive_name, self.err, self.logger) self._purge_output_dir(skipTarFiles=True) elapsed = (time.time() - startProcessTime) self.logger.info("(process.run_async) child process finished in " + str(round(elapsed)) + "s")
def importTrainingCatalog(inputfile,outputfile): dataArray = importArrayFromCSVFile(inputfile) redshift_photometric = dataArray[:,1] galaxy_type = dataArray[:,2] magnitude_g = dataArray[:,5] magnitude_r = dataArray[:,8] magnitude_i = dataArray[:,11] magnitude_z = dataArray[:,14] magnitude_y = dataArray[:,17] magnitude_j = dataArray[:,20] magnitude_h = dataArray[:,23] magnitude_k = dataArray[:,26] # wavelength_Ly = dataArray[:,29] * 1e-10 # conversion Angstroem to meter # wavelength_OII = dataArray[:,32] * 1e-10 # wavelength_Hb = dataArray[:,35] * 1e-10 # wavelength_OIIIa = dataArray[:,38] * 1e-10 # wavelength_OIIIb = dataArray[:,41] * 1e-10 # wavelength_Ha = dataArray[:,44] * 1e-10 # write to HDF5 file with archive.archive(outputfile,'a') as ar: ar['/target_selection_training/gal/redshift_photometric'] = redshift_photometric ar['/target_selection_training/gal/galaxy_type'] = galaxy_type ar['/target_selection_training/gal/magnitude_g'] = magnitude_g ar['/target_selection_training/gal/magnitude_r'] = magnitude_r ar['/target_selection_training/gal/magnitude_i'] = magnitude_i ar['/target_selection_training/gal/magnitude_z'] = magnitude_z ar['/target_selection_training/gal/magnitude_y'] = magnitude_y ar['/target_selection_training/gal/magnitude_j'] = magnitude_j ar['/target_selection_training/gal/magnitude_h'] = magnitude_h ar['/target_selection_training/gal/magnitude_k'] = magnitude_k
def loop_over_gal(self): with archive.archive(data_bank, 'r') as ar: # galaxy information self.flux_all = ar['/gal/flux'] self.lam_all = ar[ '/gal/wavelength'] # one array only for every galaxy id_fiber = ar['/gal/fiber_id'] fiber_selection_flag = ar['/gal/fiber_selection_flag'] #======================================================================================= # cut data vectors #======================================================================================= where_id_fiber = numpy.where(fiber_selection_flag == True)[0] id_fiber_temp = id_fiber[where_id_fiber] self.air_mass = self.air_mass[where_id_fiber] self.z = self.z[where_id_fiber] nwhere_id_fiber = len(where_id_fiber) #======================================================================================= # LOOP over galaxies #======================================================================================= Ngal = len(id_fiber_temp) Vfluxobs_all = [] Vfluxerr_all = [] Vfluxobs_noisefree_all = [] print "... Looping over", Ngal, "galaxies..." for i in range(Ngal): #print "... processing galaxy", i+1, "out of", Ngal flux = numpy.array(self.flux_all[i], dtype='float32') lam = numpy.array(self.lam_all[i], dtype='float32') Vfluxobs, Vfluxobs_noise_free = self.observed_spectrum( lam, flux, i) # add to list Vfluxobs_all.append(Vfluxobs) Vfluxobs_noisefree_all.append(Vfluxobs_noise_free) print "... Looping over galaxies complete" #======================================================================================= # Output #======================================================================================= with archive.archive(data_bank, 'a') as ar: ar['/gal/flux_spectrum_observed'] = Vfluxobs_all ar['/gal/flux_spectrum_observed_noise_free'] = Vfluxobs_noisefree_all ar['/gal/wavelength_survey'] = self.lambda_survey #!!! is this the right wavelength to save #======================================================================================= # DIAGNOSTIC # plot the last galaxy spectrum as an example #======================================================================================= print "... plotting diagnostics" try: Plotters.plot_spectrum(self.lambda_survey, Vfluxobs_noise_free, xmin=5000, xmax=10000, fig_number=0, plot_number=0, title='Galaxy Spectrum Noise Free', base_directory='.', filename_addendum="", show_plot=show_plot) except (RuntimeError, TypeError, NameError): print 'bad plot' pass try: Plotters.plot_spectrum(self.lambda_survey, Vfluxobs, xmin=5000, xmax=10000, fig_number=1, plot_number=0, title='Galaxy Spectrum With Noise', base_directory='.', filename_addendum="", show_plot=show_plot) except (RuntimeError, TypeError, NameError): print 'bad plot' pass
import subprocess import numpy import scipy.interpolate as interpolate import scipy.integrate as integrate import random from math import exp, sqrt #from kcorrect import k_binspec import pylab import time # set databank file data_bank = '../../../data/data_bank.h5' show_plot = False # tell MSR to use method Cunha with archive.archive(data_bank, 'a') as ar: ar['/MeasureSpectroscopicRedshift/method'] = 'Cunha' class SimulateObservedGalaxySpectra: def __init__(self): #======================================================================================= # Input #======================================================================================= print print print '=========================' print 'Entering Noise Generator' print '========================='
#!/usr/bin/python # coding:utf-8 import metaxml import conf import os import sys import upload import conf import lookup from archive import Item import archive import getpass import assembly if __name__ == '__main__': item = Item(conf.iTMSTransporter, conf.distribute_account, conf.distribute_pwd, conf.bundle_short_version_string, conf.bundle_version, conf.project_path, conf.scheme, conf.configuration, conf.provisioning_profile_name, conf.vendor_id) # 开始打包 archive.archive(item) # 获取itmsp lookup.lookup(item) # 准备上传 assembly.assembly(item) # 开始上传 upload.upload(item)
int(columns))) elif args.cmd == 'interactive': interactive.run_interactive() # Start running archival elif args.cmd == 'archive': print('...starting archive loop') firstit = True while True: if not firstit: print('Sleeping 60s until next iteration...') time.sleep(60) jobs = Job.get_running_jobs(dir_cfg['log']) firstit = False archive.archive(dir_cfg, jobs) # Debugging: show the destination drive usage schedule elif args.cmd == 'dsched': dstdirs = dir_cfg['dst'] for (d, ph) in manager.dstdirs_to_furthest_phase(jobs).items(): print(' %s : %s' % (d, str(ph))) # # Job control commands # elif args.cmd in ['details', 'files', 'kill', 'suspend', 'resume']: print(args) selected = []
def DetectPhotons(): # ----------------------------------------------------- # Import data and params # ----------------------------------------------------- print print "........................." print " Entering Apply Filters " print "........................." databank_file = '../../../data/data_bank.h5' with archive.archive(databank_file, 'r') as ar: # filter information wavelength_fiducial = ar['/Instrument/Filter/wavelength_fiducial'] resolution_fiducial = ar['/Instrument/Filter/resolution_fiducial'] wavelength_min = ar['/Instrument/Filter/wavelength_minimum'] wavelength_max = ar['/Instrument/Filter/wavelength_maximum'] # photons spectrum = ar[] # ----------------------------------------------------- # Generate Filters # ----------------------------------------------------- Filters = GenerateFilters(wavelengths, wavelength_fiducial, resolution_fiducial) # ----------------------------------------------------- # Apply Filters # ... later: apply filter to each spectrum # ----------------------------------------------------- # loop over galaxies for gal in # ... apply filter to photon list how are photons used in EAZY photo-z calculator # ----------------------------------------------------- # Export data and params # ----------------------------------------------------- with archive.archive(databank_file, 'r') as ar: print print "........................." print " Exiting Generate Image " print "........................." # plot diagnostics # do the following for all new photon information # overlay photon energies on initial galaxy spectrum # plot photon times # positions vs. time # positions vs. energy # energy vs. time # compare all old vs. new photon information # Quality Assurance # ================================================ # Main # ================================================ def main(): ApplyDetectorMeasurement() #------------------------------- if __name__ == '__main__': main()
import metaxml import conf import os import sys import upload import conf import lookup from archive import Item import archive import getpass import assembly if __name__ == '__main__': item = Item(conf.iTMSTransporter, conf.distribute_account, conf.distribute_pwd, conf.bundle_short_version_string, conf.bundle_version, conf.project_path, conf.scheme, conf.configuration, conf.provisioning_profile_name, conf.vendor_id) # 开始打包 archive.archive(item) # 获取itmsp lookup.lookup(item) # 准备上传 assembly.assembly(item) # 开始上传 upload.upload(item)
def mergeDatafile(self, datafile): #merge datafiles into the databank with archive.archive(self.config.get("options", "data_dir") + self.config.get("options", "data_bank"), 'a') as dest: with archive.archive(datafile, 'r') as src: print "... copying " + self.config.get("data_bank", datafile) + " into " + self.config.get("options", "data_bank") copyHDF5Path(src, "/", dest)
"""List archive index """ from __future__ import print_function import sys import time import archive this = archive.archive() for i in this.idx: if i[:5] != "INDEX": continue i,m,o = this.get_entry(i) assert m == "mtree" o = o.splitlines() print(i, time.ctime(int(o[1])), o[0])
def throughput(): # ===================================================== # INPUT # ===================================================== databank_file = '../../../data/data_bank.h5' print print print "=========================" print "entering Throughput" print "=========================" with archive.archive(databank_file, 'r') as ar: # -------------------------------------------------------- # read hardcoded through puts for large elements wavelength_temp = ar['/Throughput/parameters/wavelength'] aperture_losses_gal = ar['/Throughput/parameters/aperture_losses_gal'] aperture_losses_star = ar['/Throughput/parameters/aperture_losses_star'] mohawk_frd = ar['/Throughput/parameters/mohawk_frd'] collimator = ar['/Throughput/parameters/collimator'] vph_gsolver = ar['/Throughput/parameters/vph_gsolver'] camera = ar['/Throughput/parameters/camera'] ccd = ar['/Throughput/parameters/ccd'] # -------------------------------------------------------- # read hardcoded throuhg puts from glass/coatings Al = ar['/Throughput/parameters/Al'] B270_air_glass = ar['/Throughput/parameters/B270_air_glass'] air_Bk7 = ar['/Throughput/parameters/air_Bk7'] B270_6mm = ar['/Throughput/parameters/B270_6mm'] ctio = ar['/Throughput/parameters/ctio'] air_silica = ar['/Throughput/parameters/air_silica'] blue_MgF = ar['/Throughput/parameters/blue_MgF'] fiber_material = ar['/Throughput/parameters/fiber_material'] red_MgF = ar['/Throughput/parameters/red_MgF'] LLF1 = ar['/Throughput/parameters/LLF1'] seso_a = ar['/Throughput/parameters/seso_a'] seso_b = ar['/Throughput/parameters/seso_b'] sf5 = ar['/Throughput/parameters/sf5'] bk7_25mm = ar['/Throughput/parameters/bk7_25mm'] lf5 = ar['/Throughput/parameters/lf5'] fk5 = ar['/Throughput/parameters/fk5'] lak33 = ar['/Throughput/parameters/lak33'] laf21 = ar['/Throughput/parameters/laf21'] prot_ag = ar['/Throughput/parameters/prot_ag'] ag_a = ar['/Throughput/parameters/ag_a'] ag_b = ar['/Throughput/parameters/ag_b'] blue_broad = ar['/Throughput/parameters/blue_broad'] red_broad = ar['/Throughput/parameters/red_broad'] edmond = ar['/Throughput/parameters/edmond'] solgel_b = ar['/Throughput/parameters/solgel_b'] solgel_r = ar['/Throughput/parameters/solgel_r'] solgel_plus = ar['/Throughput/parameters/solgel_plus'] print "... imported data" # -------------------------------------------------------- # defining each optical element in terms of material # throughput (label with column in original worksheet) ... can replace the some of the above #atmos = 10**(-0.4*ctio*1.15) primary = 0.98 * Al top_end = 1.-(2.91/(numpy.pi/4.* 4.**2)) # wfc = air_silica**2* seso_b**6* red_broad**4* 10**(-fiber_material* 0.000376 /10.) #adc = red_broad**4 *LLF1**(28/25)*bk7_25mm**(40.5/25) # fiber = 10**(-fiber_material*0.05/10) # print "... Combined minor optical elements" # -------------------------------------------------------- # combining optical elements telescope = primary * top_end * wfc mohawk_full = fiber * mohawk_frd # vph_mounted = vph_gsolver * red_broad**2 * bk7_25mm**(20./25.) spectrograph = collimator * vph_mounted * camera * ccd print "... Combined major optical elements with galaxy, including aperture losses" # -------------------------------------------------------- # combine elements with aperture losses for gal and star # this will have to be in a loop when applying the aperture losses for each gal # for a star, total_star = telescope * aperture_losses_star * mohawk_full * spectrograph # will have to apply galaxy aperture in SIMGALSPECOBSERVED total_gal = telescope * aperture_losses_gal * mohawk_full * spectrograph throughput_final = total_gal print "... computed final throughput" # ===================================================== # Output # ===================================================== with archive.archive(databank_file, 'a') as ar: ar['/Throughput/throughput'] = throughput_final print "... exported data to databank" # ===================================================== # Diagnostic # ===================================================== print "... basic diagnostic plots" Plotters.plot_throughput_spectrum(wavelength_temp, throughput_final, power1=telescope, power2=spectrograph, xmin=500, xmax=1000, fig_number=0, plot_number=0, label0 = '', label1 = '', label2='', title='ThroughputSpectrum', base_directory='.', filename_addendum="", show_plot=False) return
def __init__(self): #======================================================================================= # Input #======================================================================================= print print print '=========================' print 'Entering Noise Generator' print '=========================' with archive.archive(data_bank, 'r') as ar: # instrument information self.exposure = 20. * 60. #exposure = ar['/instrument/sensitivity/exposure_time'] # exposure = exposure time in seconds self.readnoise = 5. #readnoise = ar['/instrument/sensitivity/read_noise'] # add to param files # resnum = .25 #resnum = ar['/instrument/sensitivity/resolution'] self.Nwave = ar['/SimulateObservedGalaxySpectra/nb_pixels'] wavelength_range = ar[ '/SimulateObservedGalaxySpectra/wavelength_range'] wavelength_min = wavelength_range[0] wavelength_max = wavelength_range[1] fiber_aperture_radius = numpy.sqrt( 2.5 / numpy.pi ) #fiber_aperture_radius = ar['/Fiber_Allocation/fiber_radius'] # arcsec self.altitude = 2635. #altitude = ar['/instrument/sensitivity/altitude'] # meters ...add to param files self.worsening = 1 #worsening = ar['/instrument/sensitivity/worsening'] # add to param files self.z = ar['/gal/z_true'] tile_id_gal = ar['/gal/tile_ID'] airmass_tile = ar['/tiling/airmass'] tile_id_tile = ar['/tiling/tile_ID'] self.air_mass = ar['/gal/airmass'] # general constants self.hp = 6.62607e-27 #ergs.secs self.c = 2.9979e18 #Ang/secs self.seed_constant = 99999 # Throughput self.wavetrans = numpy.array( ar['/Throughput/parameters/wavelength']) self.trans = numpy.array(ar['/Throughput/throughput'] ) #!!! what should be the units here? # sky data self.wave = ar['/sky/background/wave'] self.atmtemp = ar['/sky/background/power'] self.abswave = ar['/sky/extinction/wave'] self.abstemp = ar['/sky/extinction/power'] # basic modifications to input data self.atmtemp = numpy.array(self.atmtemp, dtype='float64') self.wave = numpy.array(self.wave, dtype='float64') self.abstemp = numpy.array(self.abstemp) self.abswave = numpy.array(self.abswave) #======================================================================================= # obtain airmasses for galaxies #======================================================================================= #for i in range(Number_Tiles): # match_tile = numpy.where(tile_id_gal == tile_id_tile[i])[0] # self.airmass[match_tile] = airmass_tile[i] #======================================================================================= # Set constants #======================================================================================= # !!! to modify to access tile information to link to where galaxy is observed self.num_readout = 1 #Nwave = 557*resnum # 557 = number of pixels # self.Nwave = 1000*resnum # 557 = number of pixels resnum = float(self.Nwave) / 1000 dispersion = 2.5 # def: dispersion = (pixsize or width ) / resolution; 2.5 is when most of the information comes # in a limit # dellam = 7.14/resnum #R is not being used. Only dellam matters now. = resolution/dispersion dellam = float(wavelength_max - wavelength_min) / self.Nwave print "Resolution:", int(self.Nwave), "pixels" #======================================================================================= # Aperture #======================================================================================= self.aperture_area = numpy.pi * fiber_aperture_radius**2 self.Area = numpy.pi * 4**2 self.Areacm = self.Area * 10000. #converting from m^2 to cm^2 #======================================================================================= # SKY BACKGROUND # - Convert sky photons to photons/Ang. # - in skybg_50_10.dat the units are photons/s/nm/arcsec^2/m^2 #======================================================================================= self.atm = self.atmtemp * self.worsening * self.exposure * self.aperture_area * self.Area / 10. #because 10ang=1 nm self.wave *= 10. # converting angstroms #==================================== # smooth spectra to SURVEY resolution # I think I shouldn't smooth over atmospheric absorption since # smoothing only happens after the photons have been removed from atmosphere. #==================================== self.atm = Utilities.k_smooth_py(numpy.log10(self.wave), self.atm, 50) self.atm = numpy.array(self.atm, dtype='float64') #======================================================================================= # TRANSMISSION information #======================================================================================= self.wavetrans *= 10. # converting angstroms #trans *= 0.01 #======================================================================================= # Interpolate wavelengths onto grid with resolution # 1) make grid # 2) interpolate counts/fluxes onto grid # 3) Note that delta_lam will affect number of photons counted. # ***remember to multiply by delta_lam.*** #======================================================================================= self.lambda_survey = numpy.zeros(self.Nwave, dtype='float64') binwidth = numpy.zeros(self.Nwave) self.lambda_survey[0] = wavelength_min for i in numpy.arange(1, self.Nwave): self.lambda_survey[i] = self.lambda_survey[i - 1] + dellam for i in numpy.arange(1, self.Nwave - 1): self.lambda_survey[i] = self.lambda_survey[i] + ( self.lambda_survey[i + 1] - self.lambda_survey[i]) / 2. binwidth[i] = self.lambda_survey[i + 1] - self.lambda_survey[i] atmtemp_rebinned = numpy.zeros(self.lambda_survey.shape) k_binspec.k_binspec(self.wave, self.atmtemp, self.lambda_survey, atmtemp_rebinned) self.trans_interp = numpy.interp(self.lambda_survey, self.wavetrans, self.trans) self.A = self.hp * self.c / (self.lambda_survey * self.trans_interp) self.A[numpy.isnan(self.A)] = 0. self.B = self.worsening * self.exposure * self.aperture_area * self.Area / 10 * atmtemp_rebinned * self.trans_interp self.C = self.exposure * self.Areacm / ( self.hp * self.c) * self.lambda_survey * self.trans_interp abstemp_interp = numpy.interp(self.lambda_survey, self.abswave, self.abstemp) self.D = abstemp_interp * numpy.exp( (1700. - self.altitude) / 7000.) self.E = self.num_readout * self.readnoise**2
#======================================================================================= ##Calculate S/N #======================================================================================= Vatmerr = sqrt(Vatm+Vsed+readnoise**2) signoise = Vsed/Vatmerr Vfluxerr = Vflux/signoise #======================================================================================= ##Generate observed spectra #======================================================================================= Vfluxobs=np.zeros(Nwav,Ngal) # loop over galaxies for i in range(ngal): random.seed() gasdev = np.array([random.random() for x in range(Nwav)]) Vfluxobs[*,i] = Vflux[*,i]+ gasdev* Vfluxerr[*,i] # Make noise free Vfluxobs_noise_free=Vflux # Write out with archive.archive(data_bank,'a') as ar: ar['/gal/flux_spectrum_observed'] = Vfluxobs ar['/gal/flux_spectrum_observed_noise_free'] = Vfluxobs_noise_free ar['/gal/wavelength_survey'] = lambda_survey #!!! is this the right wavelength to save
def archiveManga(ignore=''): manga=int(mangaList.curselection()[0]) #know what the manga number in our list is tool=archive(mangas[manga].name) #get ready to archive using the tool tool.update_zipFiles() #make the archives. this is a good function in that it will remove archives such as 20-22 and make 20-25 print "Done Archiving"
def observe_galaxy(): data_bank = 'data_bank.h5' with archive.archive(data_bank,'r') as ar: # galaxy information gal_air_mass = ar['/gal/airmass'] # instrument information exposure = ar['/instrument/sensitivity/exposure_time'] # exposure = exposure time in seconds readnoise = ar['/instrument/sensitivity/read_noise'] # add to param files resnum = ar['/instrument/sensitivity/resolution'] resolution_kms = ar['/instrument/sensitivity/resolution_kms'] # res_instrument = 300 # res of instrument in km/s wavelength_min = ar['/instrument/wavelength_minimum'] # Angstrom fiber_aperture_radius = ar['fiber_allocation/fiber_radius'] worsening = ar['/instrument/sensitivity/worsening'] # add to param files altitude = ar['/instrument/sensitivity/altitude'] #meters ??? add to param files # general constants plancks_constant = ar['/general/physical_constants/plancks_constant'] speed_of_light = ar['/general/physical_constants/speed_of_light'] # sky data wave = ar['sky/background/wave'] temp = ar['sky/background/power'] abswave = ar['sky/extinction/wave'] abstemp = ar['sky/extinction/power'] # Throughtput wavetrans = ar['/gal/throughput/wavelength'] trans = ar['/gal/throughput/throughput'] #!!! what should be the units here? # !!! to modify to access tile information to link to where galaxy is observed #airmass=1.3 Ngal = len(gal) Nwav=557*resnum # 557 = number of pixels dispersion = 2.5 # def: dispersion = (pixsize or width ) / resolution; 2.5 is when most of the information comes # in a limit dellam=7.14/resnum #R is not being used. Only dellam matters now. = resolution/dispersion #======================================================================================= # Aperture #======================================================================================= aperture_area = np.pi*fiber_aperture_radius**2 Area = np.pi*4^2 Areacm = Area*10000. #converting from m^2 to cm^2 #======================================================================================= # SKY BACKGROUND # - Convert sky photons to photons/Ang. # - in skybg_50_10.dat the units are photons/s/nm/arcsec^2/m^2 #======================================================================================= atm = worsening* exposure* temp* aperture_area* Area/10. #because 10ang=1 nm wave = 10.* wave #======================================================================================= # TRANSMISSION information #======================================================================================= wavetrans = 10.* wavetrans # WHAT are the translation units? trans = 0.01* trans #============================================ # Modify ATMOSPHERIC ABSORPTION #============================================ absatm = abstemp*airmass*exp((1700.-altitude)/7000.) absatm(np.where(absatm > 1)) = 1 # file is bad ,...just make sure file is not bad #==================================== # smooth spectra to SURVEY resolution # I think I shouldn't smooth over atmospheric absorption since # smoothing only happens after the photons have been removed from atmosphere. #==================================== logwave = np.log10(wave) atm = my_smooth(logwave, atm, resolution) #atm = k_smooth(logwave,atm,300) #outatm=k_smooth(logwave,atm,1320) #======================================================================================= # Convert sed flux to photon counts. # - sed flux is in ergs/cm^2/s/Ang. # - photon counts is in photons/Ang. #======================================================================================= flux = flux/ (1.+zspectemp) #add redshift dimming #flux[*,i]=flux[*,i]/(1.+zspectemp[i])^2 #add redshift dimming sedphot = exposure* flux* lam* Areacm/ (hp*c) #approximate - correct is to integrate sedflux = exposure* flux* Areacm # #======================================================================================= # Redshift galaxy sed's. # - rest-frame flux assumes galaxy is 10pc away. # - doesn't matter when using coeffs calculated from # - kcorrect, since the code already applies the 1/r^2 dimming. #======================================================================================= lamz = lam* (1. + zspectemp) #======================================================================================= # Interpolate wavelengths onto grid with resolution # 1) make grid # 2) interpolate counts/fluxes onto grid # 3) Note that delta_lam will affect number of photons counted. # ***remember to multiply by delta_lam.*** #======================================================================================= lambda_survey = np.zeros(Nwav) binwidth = np.zeros(Nwav) lambda_survey[0] = wavelength_minimum for i in np.arange(1,Nwave): lambda_survey[i] = lambda_survey[i-1]+dellam for i in np.arange(1,Nwave-1): lambda_survey[i] = lambda_survey[i] + (lambda_survey[i+1]-lambda_survey[i])/2. binwidth[i] = lambda_survey[i+1] - lambda_survey[i] #======================================================================================= # Bin spectra to survey's resolutions by integrating over pixels # to convert flux density to flux. #======================================================================================= Vsed = np.zeros(Nwav,Ngal) Vflux = np.zeros(Nwav,Ngal) Vatm = k_binspec(wave,atm,lambda_survey) for i=0L,Nshort-1 do begin Vsed[*,i]=k_binspec(lamz[*,i],sedphot[*,i],lambda_survey) Vflux[*,i]=k_binspec(lamz[*,i],sedflux[*,i],lambda_survey)
def run_atm(self): """ Program sequence """ #==================================================== # Initialization Process #==================================================== print '===================' print ' Initializing ATM' print '===================' read_control.read_control(self) initialize.initialize(self) read_layers.read_layers(self) model_domain.model_domain(self) create_attm_cohort_arrays.create_attm_cohort_arrays(self) #========================================= # Initializing Site Specific Information #========================================= if self.Simulation_area.lower() == 'barrow': run_barrow.initialize_barrow(self) elif self.Simulation_area.lower() == 'tanana': run_tanana.initialize_tanana(self) #======================================= # READ MET Data, Calculate Degree Days, # and Calculate Climatic Data needed # for ecotype changes. #======================================= initialize.Met(self) #++++++++++++++++++++++++++++++++++++++++++++++ # ======================================== # INITIALIZE COHORT PROPERTIES # ======================================== #++++++++++++++++++++++++++++++++++++++++++++++ print '======================================' print ' Initializing Terrestrial Properties ' print '======================================' if self.Simulation_area.lower() == 'barrow': run_barrow.initialize_barrow_cohorts(self) elif self.Simulation_area.lower() == 'tanana': run_tanana.Terrestrial_Tanana(self) print '==================================================' print ' Starting the MAIN LOOP ' print '==================================================' initialize.run(self) if self.Simulation_area.lower() == 'barrow': run_barrow.run_barrow(self, time) elif self.Simulation_area.lower() == 'tanana': run_tanana.run_tanana(self, time) print '==================================================' print ' Finished the MAIN LOOP ' print '==================================================' # ------------------- # Simulation End Time # ------------------- clock.finish(self) #=========================== # Output Simulation Results #=========================== if self.results_onscreen.lower() == 'yes': results.on_screen(self) if self.archive_simulation.lower() == 'yes': results.on_file(self) # ================ # Archive Results # ================ if self.archive_simulation.lower() == 'yes': #---------------------------------------------------------------------------------------------------------- # Create the tarfile #---------------------------------------------------------------------------------------------------------- self.archive_file =tarfile.open(self.control['Run_dir']+self.Output_directory+str('/Archive/')+ \ self.archive_time+str('_')+self.simulation_name+".tar.gz", mode='w:gz') #---------------------------------------------------------------------------------------------------------- archive.read_archive(self) archive.archive(self) print '----------------------------------------' print ' Simulation Complete ' print '----------------------------------------'
usage() options = dict() options['vault'] = "NotSpecified" options['region'] = "us-east-1" options['access_key'] = None options['secret_key'] = None for option, value in opts: if option in ('--access_key'): options['access_key'] = value elif option in ('--secret_key'): options['secret_key'] = value elif option in ('--region'): options['region'] = value elif option in ('--vault'): options['vault'] = value elif option in ('--help'): usage() if args[0] == "create": a = archive.archive(args[1], options['vault']) a.create() elif args[0] == "validate": a = archive.archive(args[1], options['vault']) a.validate() elif args[0] == "upload": a = archive.archive(args[1], options['vault']) a.upload(options) else: sys.exit("Did not understand {0}".format(command))
import csvimport from matplotlib import pyplot import os print print print "=============================" print "entered Target Selection" print "=============================" inputfile = '../../../data/data_bank.h5' outputfile = inputfile # import parameters with archive.archive(inputfile,'r') as ar: training_catalog_location = ar['/Target_Selection/training_catalog_location'] mag_g_min = ar['/Target_Selection/mag_g_min'] mag_g_max = ar['/Target_Selection/mag_g_max'] mag_r_min = ar['/Target_Selection/mag_r_min'] mag_r_max = ar['/Target_Selection/mag_r_max'] mag_i_min = ar['/Target_Selection/mag_i_min'] mag_i_max = ar['/Target_Selection/mag_i_max'] mag_z_min = ar['/Target_Selection/mag_z_min'] mag_z_max = ar['/Target_Selection/mag_z_max'] mag_y_min = ar['/Target_Selection/mag_y_min'] mag_y_max = ar['/Target_Selection/mag_y_max'] photo_z_min = ar['/Target_Selection/photo_z_min'] photo_z_max = ar['/Target_Selection/photo_z_max'] type_cutoff = ar['/Target_Selection/type_cutoff'] max_training_set_size = ar['/Target_Selection/max_training_set_size']
return a tx("HELLO STOW 1.0 SERVER") cl = rx() assert cl == "HELLO STOW 1.0 CLIENT" tx("WELCOME PYSTOW") nm = rx() assert nm[:5] == "NAME " nm = nm.split()[1] tx("OK") this = archive.archive() mt = "" while True: i = rx() j = i.split() assert j[0] == "MTREE" l = int(j[1]) if l == 0: break; x = 0 while x < l: y = sys.stdin.read(l - x) mt += y x += len(y) a = rx()
def run_atm(self): """ Program sequence """ #==================================================== # Initialization Process #==================================================== print '===================' print ' Initializing ATM' print '===================' read_control.read_control(self) initialize.initialize(self) read_layers.read_layers(self) model_domain.model_domain(self) create_attm_cohort_arrays.create_attm_cohort_arrays(self) #========================================= # Initializing Site Specific Information #========================================= if self.Simulation_area.lower() == 'barrow': run_barrow.initialize_barrow(self) elif self.Simulation_area.lower() == 'tanana': run_tanana.initialize_tanana(self) elif self.Simulation_area.lower() == 'yukon': run_yukon.initialize_yukon(self) #======================================= # READ MET Data, Calculate Degree Days, # and Calculate Climatic Data needed # for ecotype changes. #======================================= initialize.Met(self) #++++++++++++++++++++++++++++++++++++++++++++++ # ======================================== # INITIALIZE COHORT PROPERTIES # ======================================== #++++++++++++++++++++++++++++++++++++++++++++++ print '======================================' print ' Initializing Terrestrial Properties ' print '======================================' if self.Simulation_area.lower() == 'barrow': run_barrow.initialize_barrow_cohorts(self) elif self.Simulation_area.lower() == 'tanana': run_tanana.Terrestrial_Tanana(self) print '==================================================' print ' Starting the MAIN LOOP ' print '==================================================' initialize.run(self) if self.Simulation_area.lower() == 'barrow': run_barrow.run_barrow(self, time) elif self.Simulation_area.lower() == 'tanana': run_tanana.run_tanana(self, time) print '==================================================' print ' Finished the MAIN LOOP ' print '==================================================' # ------------------- # Simulation End Time # ------------------- clock.finish(self) #=========================== # Output Simulation Results #=========================== if self.results_onscreen.lower() == 'yes': results.on_screen(self) if self.archive_simulation.lower() == 'yes': results.on_file(self) # ================ # Archive Results # ================ if self.archive_simulation.lower() == 'yes': archive.read_archive(self) archive.archive(self) #---------------------------------------------------------------------------------------------------------- # Create the tarfile #---------------------------------------------------------------------------------------------------------- self.archive_file =tarfile.open(self.control['Run_dir']+self.Output_directory+str('/Archive/')+ \ self.archive_time+str('_')+self.simulation_name+".tar.gz", mode='w:gz') #---------------------------------------------------------------------------------------------------------- if self.Simulation_area.lower() == 'barrow': os.chdir(self.control['Run_dir'] + self.Input_directory + '/Barrow/') print '----------------------------------------' print ' Simulation Complete ' print '----------------------------------------'
def run_attm(self): """ Program sequence """ #==================================================== # Initialization Process #==================================================== print '===================' print ' Initializing ATTM' print '===================' read_control.read_control(self) initialize.initialize(self) read_layers.read_layers(self) model_domain.model_domain(self) create_attm_cohort_arrays.create_attm_cohort_arrays(self) if self.Simulation_area.lower() == 'barrow': initial_cohort_population.barrow_initial_cohort_population(self) initial_cohort_check.barrow_initial_cohort_check(self) cohort_present.barrow_cohort_present(self) elif self.Simulation_area.lower() == 'tanana': initial_cohort_population.tanana_initial_cohort_population(self) initial_cohort_check.tanana_initial_cohort_check(self) cohort_present.tanana_cohort_present(self) #======================================= # READ MET Data & Calculate Degree Days #======================================= initialize.Met(self) #++++++++++++++++++++++++++++++++++++++++++++++ # ======================================== # INITIALIZE BARROW COHORT PROPERTIES # ======================================== #++++++++++++++++++++++++++++++++++++++++++++++ if self.Simulation_area.lower() == 'barrow': print '=================================== ' print ' Initializing Lake & Pond Properties' print '====================================' initialize.LakePond(self) set_lake_pond_depth.set_lake_pond_depth(self) set_lake_ice_depth_constant.set_lake_ice_depth_constant(self) set_ice_thickness_array.set_ice_thickness_array(self) climate_expansion_arrays.set_climate_expansion_arrays(self) set_pond_growth_array.set_pond_growth_array(self) print '=====================================' print ' Initializing Terrestrial Properties' print '=====================================' initialize.Terrestrial_Barrow(self) read_ice_content.read_ice_content(self) read_drainage_efficiency.read_drainage_efficiency(self) read_initial_ALD.read_initial_ALD(self) set_ALD_constant.set_ALD_constant(self) set_ALD_array.set_ALD_array(self) set_protective_layer.set_protective_layer(self) set_initial_cumulative_probability.set_initial_cumulative_probability(self) # Initializing Terrestrial Cohort Properties initialize.Wet_NPG(self) initialize.Wet_LCP(self) initialize.Wet_CLC(self) initialize.Wet_FCP(self) initialize.Wet_HCP(self) # Other needed information [in the future] initial_cohort_age.initial_cohort_age(self) elif self.Simulation_area.lower() == 'tanana': print '======================================' print ' Initializing Terrestrial Properties ' print '======================================' initialize.Terrestrial_Tanana(self) print '==================================================' print ' Starting the MAIN LOOP ' print '==================================================' initialize.run(self) for time in range(0, self.stop): if time == 0: if self.Simulation_area.lower() == 'barrow': cohorts.initial_barrow(self) elif self.Simulation_area.lower() == 'tanana': cohorts.initial_tanana(self) print ' at time step: ', time # ++++++++++++++++++++++++++++++++++++++ # Check for significant climatic event # ++++++++++++++++++++++++++++++++++++++ check_climate_event.check_climate_event(self) # ---------------------------------------------------------- # Looping over elements # ---------------------------------------------------------- for element in range(0, self.ATTM_nrows * self.ATTM_ncols): # ---------------------------------------------------- # Define the total fractional area of cohorts for # each element # ---------------------------------------------------- cohort_start = cohort_check.cohort_start(self, element, time) # ---------------------------------------------------- # Expand/Infill lake & ponds by prescribed rates # ---------------------------------------------------- lake_pond_expansion.lake_pond_expansion(self, element) lake_pond_expansion.pond_infill(self, element, time) # ---------------------------------------------------------- # Set active layer depth # --------------------------------------------------------- active_layer_depth.active_layer_depth(self, time, element) # ---------------------------------- # Cycle through terrestrial cohorts # ---------------------------------- check_Wet_NPG.check_Wet_NPG(self, element, time) check_Wet_LCP.check_Wet_LCP(self, element, time) check_Wet_CLC.check_Wet_CLC(self, element, time) check_Wet_FCP.check_Wet_FCP(self, element, time) check_Wet_HCP.check_Wet_HCP(self, element, time) # ---------------------------------- # Set pond/lake ice thickness depth # ---------------------------------- ice_thickness.ice_thickness(self, time, element) # ------------------------------ # Cycle through ponds and lakes # ------------------------------ check_Ponds.check_Ponds(self, element, time) check_Lakes.check_Lakes(self, element, time) # ------------------------------------------------- # Cohort Fraction Check (mass balance of cohorts) # ------------------------------------------------- cohort_check.cohort_check(self, element, time, cohort_start) if time == self.stop-1: if self.Simulation_area.lower() == 'barrow': cohorts.final_barrow(self) elif self.Simulation_area.lower() == 'tanana': cohorts.final_tanana(self) # ======================================================================== # END MAIN LOOP # ======================================================================== # ======================================================================== # OUTPUT RESULTS (if requested) # ======================================================================== # - - - - - - - - - # Fractional Areas # - - - - - - - - - Output_cohorts_by_year.Output_cohorts_by_year(self, time) # - - - - - - - - - - - - - # Dominant Fractional Area # - - - - - - - - - - - - - Output_cohorts_by_year.dominant_cohort(self) # Terrestrial_Control Output_cohorts_by_year.dominant_fractional_plot(self, time) # Terrestrial_Control # ================================= # OUTPUT ANIMATIONS (if requested) # ================================= # - - - - - - - - - - - - - - - # Fractional Area of Cohorts # - - - - - - - - - - - - - - - - Output_cohorts_by_year.write_Fractions_avi(self) Output_cohorts_by_year.write_Dominant_Cohort_avi(self) # Terrestrial_Control # ------------------- # Simulation End Time # ------------------- clock.finish(self) #=========================== # Output Simulation Results #=========================== if self.results_onscreen.lower() == 'yes': results.on_screen(self) if self.archive_simulation.lower() == 'yes': results.on_file(self) # ================ # Archive Results # ================ if self.archive_simulation.lower() == 'yes': #---------------------------------------------------------------------------------------------------------- # Create the tarfile #---------------------------------------------------------------------------------------------------------- self.archive_file =tarfile.open(self.control['Run_dir']+self.Output_directory+str('/Archive/')+ \ self.archive_time+str('_')+self.simulation_name+".tar.gz", mode='w:gz') #---------------------------------------------------------------------------------------------------------- archive.read_archive(self) archive.archive(self) print '----------------------------------------' print ' Simulation Complete ' print '----------------------------------------'
def __init__(self): print print print "========================================" print " Reconstruct Pure Galaxy Spectrum " print "========================================" #======================================================================================= # Input #======================================================================================= print "... Import data and params" with archive.archive(databank_file, 'r') as ar: # observation choices self.nolines = ar['/GenerateSpectrum/nolines'] # user inputs self.noextinct = ar['/GenerateSpectrum/noextinct'] self.vdisp = ar['/GenerateSpectrum/vdisp'] # templates self.loglam = numpy.array( ar['/spectral_templates/log10_wavelength']) self.tspec_v0 = numpy.array( ar['/spectral_templates/template_spectrum_v0']) # not smoothed self.tspec_v0_nl = numpy.array( ar['/spectral_templates/template_spectrum_v0_nolines']) self.tspec_v0_nd = numpy.array( ar['/spectral_templates/template_spectrum_v0_nodust']) self.tspec_v0_nd_nl = numpy.array( ar['/spectral_templates/template_spectrum_v0_nodust_nolines']) self.tspec_v300 = numpy.array( ar['/spectral_templates/template_spectrum_v300'] ) # smoothed to 300kms(res) self.tspec_v300_nl = numpy.array( ar['/spectral_templates/template_spectrum_v300_nolines']) self.tspec_v300_nd = numpy.array( ar['/spectral_templates/template_spectrum_v300_nodust']) self.tspec_v300_nd_nl = numpy.array( ar['/spectral_templates/template_spectrum_v300_nodust_nolines'] ) self.lspec_v300 = numpy.array(ar['/spectral_templates/lspec_v300']) # galaxy information self.coeffs = ar['/gal/coeffs'] self.z_true = ar['/gal/z_true'] self.fiber_selection_flag = ar['/gal/fiber_selection_flag'] print "... finished importing data" # ===================================================== # QA Test -- Incoming Ngal = len(self.fiber_selection_flag) # count galaxies Nwave = len(self.tspec_v0[0, :]) # count wavelengths fiber_selection_flag_shape_expected = (Ngal, ) check = (self.fiber_selection_flag.shape == fiber_selection_flag_shape_expected) if check: print "*** Incoming QA checksum PASSED ***" else: print "XX CHECKSUM FAILED XX" print "fiber selection flag shape not compliant withe expectation" sys.exit(1)
def curses_main(stdscr): # TODO: figure out how to pass the configs in from plotman.py instead of # duplicating the code here. with open('config.yaml', 'r') as ymlfile: cfg = yaml.load(ymlfile, Loader=yaml.FullLoader) dir_cfg = cfg['directories'] sched_cfg = cfg['scheduling'] plotting_cfg = cfg['plotting'] log = Log() plotting_active = True archiving_configured = 'archive' in dir_cfg archiving_active = archiving_configured (n_rows, n_cols) = map(int, stdscr.getmaxyx()) # Page layout. Currently requires at least ~40 rows. # TODO: make everything dynamically figure to best use available space header_height = 3 jobs_height = 10 dirs_height = 14 logscreen_height = n_rows - (header_height + jobs_height + dirs_height) header_pos = 0 jobs_pos = header_pos + header_height dirs_pos = jobs_pos + jobs_height logscreen_pos = dirs_pos + dirs_height plotting_status = '<startup>' # todo rename these msg? archiving_status = '<startup>' refresh_period = int(sched_cfg['polling_time_s']) stdscr.nodelay(True) # make getch() non-blocking stdscr.timeout(2000) header_win = curses.newwin(header_height, n_cols, header_pos, 0) log_win = curses.newwin(logscreen_height, n_cols, logscreen_pos, 0) jobs_win = curses.newwin(jobs_height, n_cols, jobs_pos, 0) dirs_win = curses.newwin(dirs_height, n_cols, dirs_pos, 0) jobs = Job.get_running_jobs(dir_cfg['log']) last_refresh = datetime.datetime.now() pressed_key = '' # For debugging while True: # TODO: handle resizing. Need to (1) figure out how to reliably get # the terminal size -- the recommended method doesn't seem to work: # (n_rows, n_cols) = [int(v) for v in stdscr.getmaxyx()] # Consider instead: # ...[int(v) for v in os.popen('stty size', 'r').read().split()] # and then (2) implement the logic to resize all the subwindows as above # stdscr.clear() linecap = n_cols - 1 logscreen_height = n_rows - (header_height + jobs_height + dirs_height) elapsed = (datetime.datetime.now() - last_refresh).total_seconds() # A full refresh scans for and reads info for running jobs from # scratch (i.e., reread their logfiles). Otherwise we'll only # initialize new jobs, and mostly rely on cached info. do_full_refresh = elapsed >= refresh_period if not do_full_refresh: jobs = Job.get_running_jobs(dir_cfg['log'], cached_jobs=jobs) else: last_refresh = datetime.datetime.now() jobs = Job.get_running_jobs(dir_cfg['log']) if plotting_active: (started, msg) = manager.maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg) if (started): log.log(msg) plotting_status = '<just started job>' jobs = Job.get_running_jobs(dir_cfg['log'], cached_jobs=jobs) else: plotting_status = msg if archiving_configured and archiving_active: # Look for running archive jobs. Be robust to finding more than one # even though the scheduler should only run one at a time. arch_jobs = archive.get_running_archive_jobs( dir_cfg['archive']) if arch_jobs: archiving_status = 'pid: ' + ', '.join(map(str, arch_jobs)) else: (should_start, status_or_cmd) = archive.archive(dir_cfg, jobs) if not should_start: archiving_status = status_or_cmd else: cmd = status_or_cmd log.log('Starting archive: ' + cmd) # TODO: do something useful with output instead of DEVNULL p = subprocess.Popen(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, start_new_session=True) # Directory prefixes, for abbreviation tmp_prefix = '' #os.path.commonpath(dir_cfg['tmp']) dst_prefix = '' #os.path.commonpath(dir_cfg['dst']) if archiving_configured: arch_prefix = dir_cfg['archive']['rsyncd_path'] # Header header_win.addnstr(0, 0, 'Plotman', linecap, curses.A_BOLD) timestamp = datetime.datetime.now().strftime("%H:%M:%S") refresh_msg = "now" if do_full_refresh else f"{int(elapsed)}s/{refresh_period}" header_win.addnstr(f" {timestamp} (refresh {refresh_msg})", linecap) header_win.addnstr(' | <P>lotting: ', linecap, curses.A_BOLD) header_win.addnstr( plotting_status_msg(plotting_active, plotting_status), linecap) header_win.addnstr(' <A>rchival: ', linecap, curses.A_BOLD) header_win.addnstr( archiving_status_msg(archiving_configured, archiving_active, archiving_status), linecap) # Oneliner progress display header_win.addnstr(1, 0, 'Jobs (%d): ' % len(jobs), linecap) header_win.addnstr('[' + reporting.job_viz(jobs) + ']', linecap) # These are useful for debugging. # header_win.addnstr(' term size: (%d, %d)' % (n_rows, n_cols), linecap) # Debuggin # if pressed_key: # header_win.addnstr(' (keypress %s)' % str(pressed_key), linecap) header_win.addnstr(2, 0, 'Prefixes:', linecap, curses.A_BOLD) header_win.addnstr(' tmp=', linecap, curses.A_BOLD) header_win.addnstr(tmp_prefix, linecap) header_win.addnstr(' dst=', linecap, curses.A_BOLD) header_win.addnstr(dst_prefix, linecap) if archiving_configured: header_win.addnstr(' archive=', linecap, curses.A_BOLD) header_win.addnstr(arch_prefix, linecap) header_win.addnstr(' (remote)', linecap) # Jobs jobs_win.addstr( 0, 0, reporting.status_report(jobs, n_cols, jobs_height, tmp_prefix, dst_prefix)) jobs_win.chgat(0, 0, curses.A_REVERSE) # Dirs. Collect reports as strings, then lay out. n_tmpdirs = len(dir_cfg['tmp']) n_tmpdirs_half = int(n_tmpdirs / 2) tmp_report_1 = reporting.tmp_dir_report(jobs, dir_cfg['tmp'], sched_cfg, n_cols, 0, n_tmpdirs_half, tmp_prefix) tmp_report_2 = reporting.tmp_dir_report(jobs, dir_cfg['tmp'], sched_cfg, n_cols, n_tmpdirs_half, n_tmpdirs, tmp_prefix) dst_report = reporting.dst_dir_report(jobs, dir_cfg['dst'], n_cols, dst_prefix) if archiving_configured: arch_report = reporting.arch_dir_report( archive.get_archdir_freebytes(dir_cfg['archive']), n_cols, arch_prefix) if not arch_report: arch_report = '<no archive dir info>' else: arch_report = '<archiving not configured>' tmp_h = max(len(tmp_report_1.splitlines()), len(tmp_report_2.splitlines())) tmp_w = len( max(tmp_report_1.splitlines() + tmp_report_2.splitlines(), key=len)) + 1 dst_h = len(dst_report.splitlines()) dst_w = len(max(dst_report.splitlines(), key=len)) + 1 arch_h = len(arch_report.splitlines()) + 1 arch_w = n_cols tmpwin_12_gutter = 3 tmpwin_dstwin_gutter = 6 maxtd_h = max([tmp_h, dst_h]) tmpwin_1 = curses.newwin(tmp_h, tmp_w, dirs_pos + int( (maxtd_h - tmp_h) / 2), 0) tmpwin_1.addstr(tmp_report_1) tmpwin_2 = curses.newwin(tmp_h, tmp_w, dirs_pos + int( (maxtd_h - tmp_h) / 2), tmp_w + tmpwin_12_gutter) tmpwin_2.addstr(tmp_report_2) tmpwin_1.chgat(0, 0, curses.A_REVERSE) tmpwin_2.chgat(0, 0, curses.A_REVERSE) dstwin = curses.newwin( dst_h, dst_w, dirs_pos + int((maxtd_h - dst_h) / 2), 2 * tmp_w + tmpwin_12_gutter + tmpwin_dstwin_gutter) dstwin.addstr(dst_report) dstwin.chgat(0, 0, curses.A_REVERSE) #archwin = curses.newwin(arch_h, arch_w, dirs_pos + maxtd_h, 0) #archwin.addstr(0, 0, 'Archive dirs free space', curses.A_REVERSE) #archwin.addstr(1, 0, arch_report) # Log. Could use a pad here instead of managing scrolling ourselves, but # this seems easier. log_win.addnstr( 0, 0, ('Log: %d (<up>/<down>/<end> to scroll)\n' % log.get_cur_pos()), linecap, curses.A_REVERSE) for i, logline in enumerate(log.cur_slice(logscreen_height - 1)): log_win.addnstr(i + 1, 0, logline, linecap) stdscr.noutrefresh() header_win.noutrefresh() jobs_win.noutrefresh() tmpwin_1.noutrefresh() tmpwin_2.noutrefresh() dstwin.noutrefresh() #archwin.noutrefresh() log_win.noutrefresh() curses.doupdate() key = stdscr.getch() if key == curses.KEY_UP: log.shift_slice(-1) pressed_key = 'up' elif key == curses.KEY_DOWN: log.shift_slice(1) pressed_key = 'dwn' elif key == curses.KEY_END: log.shift_slice_to_end() pressed_key = 'end' elif key == ord('p'): plotting_active = not plotting_active pressed_key = 'p' elif key == ord('a'): archiving_active = not archiving_active pressed_key = 'a' elif key == ord('q'): break else: pressed_key = key
def main(): print "Started fiber allocation. Importing from data bank..." # load all the inputs data_bank = "../../../data/data_bank.h5" with archive.archive(data_bank, "r") as ar: # numbers coming from param.ini fiber_pitch = ar["/Fiber_Allocation/fiber_pitch"] fiber_size = ar["/Fiber_Allocation/fiber_size"] Ndiameter = ar["/Fiber_Allocation/num_fibers_on_diameter"] n_pass_per_tile = ar["/Fiber_Allocation/n_pass_per_tile"] patrol_radius = ar["/Fiber_Allocation/patrol_radius"] allocation_method = ar["/Fiber_Allocation/allocation_method"] # numbers coming from the previous step in the pipeline gals_x_in = ar["/gal/ra_true"] gals_y_in = ar["/gal/dec_true"] gals_id_in = ar["/gal/galaxy_index"] number_of_tiles = ar["/tiling/number_of_tiles"] tiles_centers_ra = ar["/tiling/tile_centers_ra"] tiles_centers_dec = ar["/tiling/tile_centers_dec"] galaxy_tile_id_list = ar["/gal/tile_ID"] tile_id_list = ar["/tiling/tile_ID"] survey_selection_flag = ar["/gal/survey_selection_flag"] survey_selection_flag = np.array(survey_selection_flag) gals_x_in = gals_x_in[survey_selection_flag] gals_y_in = gals_y_in[survey_selection_flag] gals_id_in = gals_id_in[survey_selection_flag] galaxy_tile_id_list = galaxy_tile_id_list[survey_selection_flag] print "...done." selected = np.where(galaxy_tile_id_list > 0) n_in_tiles = np.size(selected) print "There are %d galaxies in tiles" % (n_in_tiles) print "Min-Max RA Tiles: %f %f" % (np.amin(tiles_centers_ra), np.amax(tiles_centers_ra)) print "Min-Max DEC Tiles: %f %f" % (np.amin(tiles_centers_dec), np.amax(tiles_centers_dec)) print "Min-Max RA Galaxies: %f %f" % (np.amin(gals_x_in), np.amax(gals_x_in)) print "Min-Max DEC Galaxies: %f %f" % (np.amin(gals_y_in), np.amax(gals_y_in)) # plot_name = "all_galaxies" # diagplot.plot_positions(gals_x_in[selected], gals_y_in[selected], plot_name) # redefine the positions of the galaxies and tile centers to a 'planar' frame gals_x_in = gals_x_in * np.cos(gals_y_in * math.pi / 180.0) tiles_centers_ra = tiles_centers_ra * np.cos(tiles_centers_dec * math.pi / 180.0) # initialize full galaxy structure all_gals = AM.MockGalaxyCatalog(x_in=gals_x_in, y_in=gals_y_in, id_in=gals_id_in) fraction_allocated = np.empty((0)) # loop over the tiles for ra_center, dec_center, tile_id in zip(tiles_centers_ra, tiles_centers_dec, tile_id_list): # create the unperturbed set of fibers for this pass in this tile fibers = AM.FiberSet( Ndiameter=Ndiameter, fiber_pitch=fiber_pitch, fiber_size=fiber_size, center_x=ra_center, center_y=dec_center ) # set to -1 the fiber_ID of the galaxies that should fall inside a tile index_inside = np.where((galaxy_tile_id_list == tile_id) & (all_gals.tile_ID < 0)) all_gals.tile_ID[index_inside] = -1 if np.size(index_inside) > 0: print "Galaxies to allocate", np.size(index_inside) # Finally. Make the allocation! AM.make_fiber_allocation( fibers, all_gals, tile_ID=tile_id, visit_ID=1, rank_criterion=allocation_method, patrol_radius=fibers.fiber_pitch, exclusion_radius=fibers.fiber_size * 0.5, ) n_alloc = np.where(all_gals.fiber_ID > 0) # update the number of allocated galaxies for a control plot gal_alloc = np.where(all_gals.fiber_ID > 0) n_total = np.size(all_gals.fiber_ID) n_alloc = np.size(gal_alloc) fraction_allocated = np.append(fraction_allocated, (1.0 * n_alloc / (1.0 * n_total))) # final stats gal_alloc = np.where(all_gals.fiber_ID > 0) n_total = np.size(all_gals.fiber_ID) n_alloc = np.size(gal_alloc) print "Allocation efficiency", (1.0 * n_alloc) / (1.0 * n_total) print "Initial number of galaxies (after target selection)", n_total print "Initial number of galaxies (in tiles)", n_in_tiles print "Galaxies that were allocated:", n_alloc inner_fiber_selection_flag = np.array((all_gals.fiber_ID > 0)) print "survey selection:", survey_selection_flag.shape, np.sum(survey_selection_flag) print "inner fiber selection:", inner_fiber_selection_flag.shape, np.sum(inner_fiber_selection_flag) fiber_selection_flag = survey_selection_flag fiber_selection_flag[survey_selection_flag] = inner_fiber_selection_flag print "fiber selection:", fiber_selection_flag.shape, np.sum(fiber_selection_flag) # Write result to data bank with archive.archive(data_bank, "a") as ar: ar["/gal/tile_id"] = all_gals.tile_ID ar["/gal/visit_id"] = all_gals.visit_ID ar["/gal/fiber_id"] = all_gals.fiber_ID ar["/gal/fiber_selection_flag"] = fiber_selection_flag # ===================================================== # Diagnostics # ===================================================== print "... diagnostic plots" # fiberid distribution try: check_fiber_sel = numpy.where(fiber_selection_flag == True)[0] fiber_id_sub = all_gals.fiber_ID[check_fiber_sel] width, center, bin_edges, hist = Utilities.make_histogram(fiber_id_sub, int(float(Ngal) / 10.0)) Plotters.plot_fa_hist_fiberid( center, width, hist, fig_number=0, plot_number=0, title="n(fiberid)", base_directory=".", show_plot=True ) except (RuntimeError, TypeError, NameError): print "bad plot" pass # galaxy positions for those that are allocated (ra/dec position plot) try: Plotters.plot_fa_gal_position( all_gals.x[gal_alloc], all_gals.y[gal_alloc], fig_number=1, plot_number=0, title="galaxy position", base_directory=".", show_plot=True, ) except (RuntimeError, TypeError, NameError): print "bad plot" pass # fraction allocated vs. tile number try: tile_list = np.arange(np.size(fraction_allocated)) Plotters.plot_fa_completeness( tile_list, fraction_allocated, fig_number=2, plot_number=0, title="fiber completeness", base_directory=".", show_plot=False, ) except (RuntimeError, TypeError, NameError): print "bad plot" pass