def main(args=None): from astropy.utils.compat import argparse parser = argparse.ArgumentParser( description=('Print the header(s) of a FITS file. ' 'All HDU extensions are shown by default. ' 'In the case of a compressed image, ' 'the decompressed header is shown.')) parser.add_argument('-e', '--ext', metavar='hdu', help='specify the HDU extension number or name') parser.add_argument('-c', '--compressed', action='store_true', help='for compressed image data, ' 'show the true header which describes ' 'the compression rather than the data') parser.add_argument('filename', nargs='+', help='path to one or more FITS files to display') args = parser.parse_args(args) try: for filename in args.filename: print(HeaderFormatter(filename, args.compressed).parse(args.ext)) except FormattingException as e: log.error(e) except IOError as e: # A 'Broken pipe' IOError may occur when stdout is closed prematurely, # eg when using `fitsheader file.fits | head`. We let this pass. pass
def hogg_iau_name_main(): # pragma: no cover from astropy.utils.compat import argparse parser = argparse.ArgumentParser( description=('Properly format astronomical source names ' + 'to the IAU convention.')) parser.add_argument('-P', '--precision', dest='precision', action='store', metavar='N', default=1, type=int, help='Digits of precision to add to the declination.') parser.add_argument('-p', '--prefix', dest='prefix', action='store', metavar='STR', default='SDSS', help='Add this prefix to the name.') parser.add_argument('ra', metavar='RA', type=float, help='Right Ascension.') parser.add_argument('dec', metavar='Dec', type=float, help='Declination.') options = parser.parse_args() print( hogg_iau_name(options.ra, options.dec, prefix=options.prefix, precision=options.precision)) return 0
def main(args=None): from astropy.utils.compat import argparse from time import time parser = argparse.ArgumentParser(description='Process some integers.') parser.add_argument('-c', '--use-cython', dest='cy', action='store_true', help='Use the Cython-based Prime number generator.') parser.add_argument('-t', '--timing', dest='time', action='store_true', help='Time the Fibonacci generator.') parser.add_argument('-p', '--print', dest='prnt', action='store_true', help='Print all of the Prime numbers.') parser.add_argument('n', metavar='N', type=int, help='Get Prime numbers up to this number.') res = parser.parse_args(args) pre = time() primes = do_primes(res.n, res.cy) post = time() print('Found {0} prime numbers'.format(len(primes))) print('Largest prime: {0}'.format(primes[-1])) if res.time: print('Running time: {0} s'.format(post - pre)) if res.prnt: print('Primes: {0}'.format(primes))
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='Scan stacked sky fibers for negative fluxes... which should not happen, but in fact does.') parser.add_argument( '--pattern', type=str, default='stacked*exp??????.*', metavar='PATTERN', help='File pattern for stacked sky fibers.' ) parser.add_argument( '--path', type=str, default='.', metavar='PATH', help='Path to work from, if not ''.''' ) parser.add_argument( '--rewrite', action='store_true', help='Flag to control whether or not negative values are replaced with 0' ) parser.add_argument( '--ltzero', type=float, default=0.5, metavar='LTZERO', help='Value below zero to consider negative' ) args = parser.parse_args() flux_list = [] exp_list = [] mask_list = [] wavelengths = None for file in os.listdir(args.path): if fnmatch.fnmatch(file, args.pattern): if file.endswith('.fits'): data = Table.read(os.path.join(args.path, file), format="fits") elif file.endswith('.csv'): data = Table.read(os.path.join(args.path, file), format="ascii.csv") #mask = data['ivar'] == 0 #Get rid of shit like this, if going to not just be my hacky util exp = int(file.split("-")[2][3:9]) neg_mask = data['flux'] < -(args.ltzero) set_neg_mask = neg_mask #& ~mask if np.any(set_neg_mask): print file, exp, data['wavelength'][set_neg_mask][0], data['flux'][set_neg_mask][0], if not args.rewrite: print "FOUND" else: print "REPAIRING..." data['ivar'][set_neg_mask] = 0 data['flux'][set_neg_mask] = 0 if file.endswith('.fits'): data.write(os.path.join(args.path, file), format="fits", overwrite=True) elif file.endswith('.csv'): data.write(os.path.join(args.path, file), format="ascii.csv", overwrite=True)
def main(args=None): from . import wcs from astropy.utils.compat import argparse parser = argparse.ArgumentParser( description=("Check the WCS keywords in a FITS file for " "compliance against the standards")) parser.add_argument('filename', nargs=1, help='Path to FITS file to check') args = parser.parse_args(args) print(wcs.validate(args.filename[0]))
def main(args=None): from . import table from astropy.utils.compat import argparse parser = argparse.ArgumentParser( description=("Check a VOTable file for compliance to the " "VOTable specification")) parser.add_argument('filename', nargs=1, help='Path to VOTable file to check') args = parser.parse_args(args) table.validate(args.filename[0])
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='Stack sky fibers from a PLATE/MJD into a single spectra.') parser.add_argument( '--fibers', type=str, default=None, metavar='FIBERS', required=True, help= 'File that contains list of PLATE, MJD, FIBER which are to be stacked (by PLATE/MJD).' ) parser.add_argument( '--output', type=str, default='FITS', metavar='OUTPUT', help='Output format, either of FITS or CSV, defaults to FITS.') parser.add_argument( '--pins', action='store_true', help= 'Whether or not to output "pins": file with wavelengths of prominent peaks between ~4400 and 5600' ) args = parser.parse_args() sky_fibers_table = Table.read(args.fibers, format='ascii') sky_fibers_table = sky_fibers_table.group_by(["PLATE", "MJD"]) progress_bar = ProgressBar(widgets=[Percentage(), Bar()], maxval=len(sky_fibers_table)).start() counter = 0 for group in sky_fibers_table.groups: exposures, stacks, pin_peaks = stack_exposures(group, use_cframe=True, find_pins=args.pins) save_stacks(stacks, group, exposures, args.output) if args.pins: save_pins(pin_peaks, group) counter += len(group) progress_bar.update(counter) progress_bar.finish()
def main(args=None): from astropy.utils.compat import argparse from time import time parser = argparse.ArgumentParser(description="Process some integers.") parser.add_argument( "-c", "--use-cython", dest="cy", action="store_true", help="Use the Cython-based Prime number generator.", ) parser.add_argument( "-t", "--timing", dest="time", action="store_true", help="Time the Fibonacci generator.", ) parser.add_argument( "-p", "--print", dest="prnt", action="store_true", help="Print all of the Prime numbers.", ) parser.add_argument( "n", metavar="N", type=int, help="Get Prime numbers up to this number." ) res = parser.parse_args(args) pre = time() primes = do_primes(res.n, res.cy) post = time() print("Found {0} prime numbers".format(len(primes))) print("Largest prime: {0}".format(primes[-1])) if res.time: print("Running time: {0} s".format(post - pre)) if res.prnt: print("Primes: {0}".format(primes))
def main(args=None): from astropy.utils.compat import argparse parser = argparse.ArgumentParser(description='Process some integers.') parser.add_argument('-c', '--use-cython', dest='cy', action='store_true', help='Use the Cython-based Fibonacci generator.') parser.add_argument('n', metavar='N', type=int, help='Run Fibonacci series up to this number.') res = parser.parse_args(args) fibs = do_fib(res.n, res.cy) print fibs
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description= 'Build and test models based on dim reductions and provided spectra') parser.add_argument('--spectra_path', type=str, default='.', metavar='PATH', help='Spectra path to work from, if not ' '.' '') parser.add_argument('--method', type=str, default='ICA', metavar='METHOD', help='Dim reduction method to load data for') parser.add_argument( '--file_path', type=str, default=None, metavar='FILE_PATH', help='COMPLETE path from which to load a dim reduction') args = parser.parse_args() data_model = None scaler = None if args.file_path is not None: data_model, scaler = ize.unpickle_model(filename=args.file_path) else: data_model, scaler = ize.unpickle_model(path=args.spectra_path, method=args.method) components = ize.get_components(args.method, data_model) offset = 0 for i, comp_i in enumerate(components): if i > 0: offset += np.max(np.abs(comp_i[comp_i < 0])) * 1.2 plt.plot(stack.skyexp_wlen_out, comp_i + offset) offset += np.max(comp_i[comp_i > 0]) * 1.2 plt.show() plt.close()
def main(argv=None): # parse command-line arguments parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-n', '--num-repeats', type=int, default=1000, help='number of times to repeat timing loops') parser.add_argument('--all', action='store_true', help='run all benchmark suites') parser.add_argument('--magnitude', action='store_true', help='benchmark magnitude calculations') parser.add_argument( '--save', type=str, default=None, help='Name of file to save results to (or print if not set)') parser.add_argument('--format', type=str, default='ascii.fixed_width_two_line', help='format to use for results') args = parser.parse_args(argv) results = astropy.table.Table(names=('Suite', 'Description', 'Time [us]'), dtype=('S8', 'S40', float)) if args.magnitude or args.all: results = magnitude_calculation(results, args.num_repeats) results.write(args.save, format=args.format, delimiter_pad=' ', position_char='=', formats={'Time [us]': '%.1f'}) return 0
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description= 'Build and test models based on dim reductions and provided spectra') parser.add_argument('--metadata_path', type=str, default='.', metavar='PATH', help='Metadata path to work from, if not ' '.' '') parser.add_argument('--lunar_metadata', type=str, default=None, metavar='LUNAR_METADATA', required=True, help='File containing lunar ephemeris metadata.') parser.add_argument('--start_dt', type=ani.valid_date, help='DateTime to plot sky for') parser.add_argument('--end_dt', type=ani.valid_date, help='DateTime to plot sky for') parser.add_argument('--ra', type=str) parser.add_argument('--dec', type=str) args = parser.parse_args() m = np.zeros(hp.nside2npix(NSIDE)) lunar_row, solar_row, ss_count, ss_area = ani.get_metadata_for_dt( args.datetime, args.lunar_metadata, args.solar_metadata, args.sunspot_metadata) fig = plt.figure(1, figsize=(10, 7.5)) hp.mollview(m, coord=['C'], title="Mollview image RING", fig=1) plt.show()
def main(args=None): """ This is the main function called by the `runvpfit` script. """ from astropy.utils.compat import argparse parser = argparse.ArgumentParser( description='Run VPFIT using a specified f26 file as input.') parser.add_argument('f26', help='f26 input filename') parser.add_argument('--include', help='path to the f26 file to include') parser.add_argument('--fwhm', type=float, default=10, help='FWHM of the LSF of the spectrograph in km/s') parser.add_argument('--cos-fuv', help='option to use the HST/COS FUV LSF', action='store_true') parser.add_argument('--cos-nuv', help='option to use the HST/COS NUV LSF', action='store_true') args = parser.parse_args(args) run_vpfit(args.f26, inc=args.include, fwhm=args.fwhm, cos_fuv=args.cos_fuv, cos_nuv=args.cos_nuv)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='Pull metadata from FITS for PLATE/MJD combos, output.') parser.add_argument('--obs_metadata', type=str, default=None, metavar='OBS_METADATA', required=True, help='File containing observation metadata.') parser.add_argument('--lunar_metadata', type=str, default=None, metavar='LUNAR_METADATA', required=True, help='File containing lunar ephemeris metadata.') parser.add_argument('--solar_metadata', type=str, default=None, metavar='SOLAR_METADATA', required=True, help='File containing solar ephemeris metadata.') parser.add_argument('--sunspot_metadata', type=str, default=None, metavar='SUNSPOT_METADATA', required=True, help='File containing sunspot metadata.') parser.add_argument( '--output', type=str, default='FITS', metavar='OUTPUT', help='Output format, either of FITS or CSV, defaults to FITS.') args = parser.parse_args() if args.output.upper() == 'CSV': obs_md_table = Table.read(args.obs_metadata, format="ascii.csv") elif args.output.upper == 'FITS': obs_md_table = Table.read(args.obs_metadata, format="fits") else: obs_md_table = Table.read(args.obs_metadata) lunar_md_table = Table.read(args.lunar_metadata, format="ascii.csv") lunar_md_table.rename_column('UTC', 'EPHEM_DATE') solar_md_table = Table.read(args.solar_metadata, format="ascii.csv") solar_md_table.rename_column('UTC', 'EPHEM_DATE') sunspot_md_table = Table.read(args.sunspot_metadata, format="ascii.csv") print "Table has {} entries".format(len(obs_md_table)) lookup_date, obs_md_table = find_ephemeris_lookup_date( obs_md_table['TAI-BEG'], obs_md_table['TAI-END'], obs_md_table) print "Successfully got {} ephemeris date entries".format(len(lookup_date)) ephem_date_col = Column(lookup_date, name="EPHEM_DATE") obs_md_table.add_column(ephem_date_col) sunspot_count, sunspot_area = find_sunspot_data(ephem_date_col, sunspot_md_table) sunspot_count_col = Column(sunspot_count, name="SS_COUNT") sunspot_area_col = Column(sunspot_area, name="SS_AREA") obs_md_table.add_column(sunspot_count_col) obs_md_table.add_column(sunspot_area_col) galactic_core = ascoord.SkyCoord(l=0.0, b=0.0, unit='deg', frame='galactic') #Join lunar data to the table obs_md_table = join( obs_md_table, lunar_md_table['EPHEM_DATE', 'RA_APP', 'DEC_APP', 'MG_APP', 'ELV_APP']) lunar_ra_dec = ascoord.SkyCoord(ra=obs_md_table['RA_APP'], dec=obs_md_table['DEC_APP'], unit='deg', frame='icrs') boresight_ra_dec = ascoord.SkyCoord(ra=obs_md_table['RA'], dec=obs_md_table['DEC'], unit='deg', frame='fk5') lunar_seps = boresight_ra_dec.separation(lunar_ra_dec).degree obs_md_table.add_column(Column(lunar_seps, dtype=float, name="LUNAR_SEP")) obs_md_table.rename_column("MG_APP", "LUNAR_MAGNITUDE") obs_md_table.rename_column("ELV_APP", "LUNAR_ELV") obs_md_table.remove_columns(['RA_APP', 'DEC_APP']) #Join solar data to the table obs_md_table = join( obs_md_table, solar_md_table['EPHEM_DATE', 'RA_APP', 'DEC_APP', 'ELV_APP']) solar_ra_dec = ascoord.SkyCoord(ra=obs_md_table['RA_APP'], dec=obs_md_table['DEC_APP'], unit='deg', frame='icrs') boresight_ra_dec = ascoord.SkyCoord(ra=obs_md_table['RA'], dec=obs_md_table['DEC'], unit='deg', frame='fk5') solar_seps = boresight_ra_dec.separation(solar_ra_dec).degree obs_md_table.add_column(Column(solar_seps, dtype=float, name="SOLAR_SEP")) obs_md_table.rename_column("ELV_APP", "SOLAR_ELV") obs_md_table.remove_columns(['RA_APP', 'DEC_APP']) #Add in galactic data boresight_ra_dec = ascoord.SkyCoord(ra=obs_md_table['RA'], dec=obs_md_table['DEC'], unit='deg', frame='fk5') obs_md_table.add_column( Column(boresight_ra_dec.separation(galactic_core).degree, dtype=float, name="GALACTIC_CORE_SEP")) boresight_ra_dec = ascoord.SkyCoord(ra=obs_md_table['RA'], dec=obs_md_table['DEC'], unit='deg', frame='fk5') obs_md_table.add_column( Column(boresight_ra_dec.transform_to('galactic').b.degree, dtype=float, name="GALACTIC_PLANE_SEP")) #print obs_md_table if args.output == 'CSV': obs_md_table.write("annotated_metadata.csv", format="ascii.csv") elif args.output == 'FITS': obs_md_table.write("annotated_metadata.fits", format="fits")
def prepare(args=None): # parse command-line arguments parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-v', '--verbose', action='store_true', help='provide verbose output on progress') parser.add_argument('--classname', choices=['qso', 'lrg', 'elg', 'elgem', 'star'], default=None, help='Spectral class to prepare.') parser.add_argument( '-k', '--num-kilo-spectra', type=int, default=1, metavar='K', help='Number of thousands of spectra to sample for the prior.') parser.add_argument('--downsampling', type=int, default=4, metavar='DS', help='Downsampling of 1A simulated pixels to use.') parser.add_argument('--include-emission', action='store_true', help='Add emission spectrum to ELG class.') parser.add_argument('--seed', type=int, default=None, metavar='S', help='Random seed to use for sampling templates.') parser.add_argument('--print-interval', type=int, default=500, metavar='P', help='Print messages for every P sampled spectra.') args = parser.parse_args(args) if args.classname is None: print('You must specify a spectral class.') return -1 sampler = bayez.sampler.Samplers[args.classname]() if args.verbose: sampler.print_summary() if args.seed is None: print('You must specify a seed to use.') return -1 simulator = bayez.simulation.Simulator( analysis_downsampling=args.downsampling, verbose=args.verbose) prior = bayez.prior.build_prior(args.classname, sampler, simulator, 1000 * args.num_kilo_spectra, seed=args.seed, print_interval=args.print_interval) # Save the prior. path = os.environ.get('BAYEZ_DATA', '.') name = os.path.join( path, '{}_{}_{}k.fits'.format(args.classname, args.downsampling, args.num_kilo_spectra)) if args.verbose: print('Saving prior to {}.'.format(name)) prior.save(name, clobber=True)
def main(args=None): parser = argparse.ArgumentParser( description='Perform basic arithmetic on two FITS files.') parser.add_argument('fits_filename', nargs=2, help='FITS filename (or scalar value)') parser.add_argument('-e1', '--exten1', metavar='exten1', type=int, default=0, help='') parser.add_argument('-e2', '--exten2', metavar='exten2', type=int, default=0, help='') parser.add_argument('operator', metavar='operator', help="Arithmetic operator. Must be one " "of '+', '-', '*', '/', '//', 'min', or 'max'") parser.add_argument('-f', '--fill_value', metavar='fill_value', type=float, default=0., help='') parser.add_argument('-k', '--keywords', metavar='keywords', type=str, default=None, help='') parser.add_argument('-o', '--outfilename', metavar='outfilename', type=str, default='imarith.fits', help='') parser.add_argument('-c', '--clobber', default=False, action='store_true', help='') args = parser.parse_args(args) # TODO: better FITS to NDData and NDData to FITS adapters try: nddata1 = np.float(args.fits_filename[0]) except ValueError: nddata1 = basic_fits_to_nddata(args.fits_filename[0], exten=args.exten1) try: nddata2 = np.float(args.fits_filename[1]) except ValueError: nddata2 = basic_fits_to_nddata(args.fits_filename[1], exten=args.exten2) if not isinstance(nddata1, NDData) and not isinstance(nddata2, NDData): raise ValueError('Both "fits_filenames" cannot be scalars.') keywords = None if args.keywords is not None: keywords = args.keywords.replace(' ', '').split(',') nddata = imarith(nddata1, nddata2, args.operator, fill_value=args.fill_value, keywords=keywords) basic_nddata_to_fits(nddata, args.outfilename, clobber=args.clobber)
def main(args=None): """ This is the main function called by the `velplot` script. """ from astropy.utils.compat import argparse from astropy.extern.configobj import configobj, validate from pkg_resources import resource_stream parser = argparse.ArgumentParser( description='Creates a stacked velocity plot.\nTo dump a default ' 'configuration file: velplot -d\nTo dump an extended ' 'default configuration file: velplot -dd', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('config', help='path to the configuration file') config = resource_stream(__name__, '/config/velplot.cfg') config_extended = resource_stream(__name__, '/config/velplot_extended.cfg') spec = resource_stream(__name__, '/config/velplot_specification.cfg') if len(sys.argv) > 1: if sys.argv[1] == '-d': cfg = ConfigObj(config) cfg.filename = '{0}/velplot.cfg'.format(os.getcwd()) cfg.write() return elif sys.argv[1] == '-dd': cfg = ConfigObj(config_extended) cfg.filename = '{0}/velplot.cfg'.format(os.getcwd()) cfg.write() return args = parser.parse_args(args) try: cfg = configobj.ConfigObj(args.config, configspec=spec) validator = validate.Validator() cfg.validate(validator) except: raise IOError('Configuration file could not be read') figname = cfg['FIGURE'].pop('filename') # Create list of transitions: fname = cfg['FIGURE'].pop('transitions') print('Reading transitions from ', fname) fh = open(fname) transitions = list(fh) fh.close() # Don't include transitions that are commented out: transitions = [transition for transition in transitions if not transition.startswith('#')] # Initialise figure: velplot = VelocityPlot(transitions, **cfg['FIGURE']) fname = cfg['DATA'].pop('filename') if not fname: raise IOError('no data to plot!') # Get spectrum information and plot: spectrum = (Table.read(fname) if fname.endswith('fits') else ascii.read(fname)) wavelength = spectrum[cfg['DATA'].pop('wavelength_column')] flux = spectrum[cfg['DATA'].pop('flux_column')] error = spectrum[cfg['DATA'].pop('error_column')] continuum = spectrum[cfg['DATA'].pop('continuum_column')] velplot.plot_data(wavelength, flux, error, continuum, **cfg['DATA']) # Get model information and plot if specified: fname = cfg['MODEL'].pop('filename') if fname: ion = cfg['MODEL'].pop('ion_column') redshift = cfg['MODEL'].pop('redshift_column') logn = cfg['MODEL'].pop('logn_column') b = cfg['MODEL'].pop('b_column') if fname.endswith('f26'): model = read_f26(fname) absorbers = model.absorbers else: table = (Table.read(fname) if fname.endswith('fits') else ascii.read(fname)) absorbers = [Absorber(row[ion], row[redshift], row[logn], row[b]) for row in table] velplot.plot_models(absorbers, **cfg['MODEL']) # Save: if figname: print('Saving to {0}'.format(figname)) velplot.savefig(figname) # Display: velplot.display()
def main(args=None): parser = argparse.ArgumentParser(description='Calculate image statistics.') parser.add_argument('fits_filename', metavar='fits_filename', nargs='*', help='FITS filename(s)') parser.add_argument('-e', '--exten', metavar='exten', type=int, default=0, help='') parser.add_argument('-s', '--sigma', metavar='sigma', type=float, default=3., help='The number of standard ' 'deviations to use as the clipping limit') parser.add_argument('-i', '--iters', metavar='iters', type=int, default=1, help='') parser.add_argument('-c', '--columns', metavar='columns', type=str, default='npixels, mean, std, min, max', help='') parser.add_argument('-l', '--lower', metavar='lower', type=float, default=None, help='') parser.add_argument('-u', '--upper', metavar='upper', type=float, default=None, help='') parser.add_argument('-m', '--mask_value', metavar='mask_value', type=float, default=None, help='') args = parser.parse_args(args) # TODO: better FITS to NDData object adapters! nddata = [] for fits_fn in args.fits_filename: nddata.append(basic_fits_to_nddata(fits_fn, exten=args.exten)) columns = args.columns.replace(' ', '').split(',') tbl = imstats(nddata, sigma=args.sigma, iters=args.iters, cenfunc=np.ma.median, varfunc=np.var, columns=columns, lower_bound=args.lower, upper_bound=args.upper, mask_value=args.mask_value) filenames = Column(args.fits_filename, name='filename') tbl.add_column(filenames, 0) tbl.pprint(max_lines=-1, max_width=-1)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description= 'Build and test models based on dim reductions and provided spectra') subparsers = parser.add_subparsers(dest='subparser_name') parser.add_argument('--metadata_path', type=str, default='.', metavar='PATH', help='Metadata path to work from, if not ' '.' '') parser.add_argument('--spectra_path', type=str, default='.', metavar='PATH', help='Spectra path to work from, if not ' '.' '') parser.add_argument('--method', type=str, default='ICA', metavar='METHOD', help='Dim reduction method to load data for') parser.add_argument('--n_jobs', type=int, default=1, metavar='N_JOBS', help='N_JOBS') parser.add_argument( '--model', type=str, choices=['ET', 'RF', 'GP', 'KNN', 'SVR'], default='ET', help= 'Which model type to use: ET (Extra Trees), RF (Random Forest), GP (Gaussian Process), KNN, or SVR (Support Vector Regression)' ) parser.add_argument( '--load_model', action='store_true', help='Whether or not to load the model from --model_path') parser.add_argument('--model_path', type=str, default='model.pkl', metavar='MODEL_PATH', help='COMPLETE path from which to load a model') parser.add_argument( '--metadata_flags', type=str, default='', metavar='METADATA_FLAGS', help='Flags specifying observational metadata pre-processing, e.g. LUNAR_MAG which takes the '\ 'magnitude and linearizes it (ignoring that it is an area magnitude)' ) parser.add_argument( '--compacted_path', type=str, default=None, metavar='COMPATED_PATH', help= 'Path to find compacted/arrayized data; setting this will cause --path, --pattern to be ignored' ) parser_compare = subparsers.add_parser('compare') parser_compare.add_argument( '--folds', type=int, default=3, metavar='TEST_FOLDS', help= 'Do k-fold cross validation with specified number of folds. Defaults to 3.' ) parser_compare.add_argument( '--iters', type=int, default=50, metavar='HYPER_FIT_ITERS', help='Number of iterations when fitting hyper-params') parser_compare.add_argument( '--outputfbk', action='store_true', help='If set, outputs \'grid_scores_\' data from RandomizedSearchCV') parser_compare.add_argument( '--save_best', action='store_true', help= 'Whether or not to save the (last/best) model built for e.g. --hyper_fit' ) parser_compare.add_argument( '--scorer', type=str, choices=['R2', 'MAE', 'MSE', 'LL', 'EXP_VAR', 'MAPED', 'MSEMV'], default='R2', help= 'Which scoring method to use to determine ranking of model instances.') parser_compare.add_argument( '--use_spectra', action='store_true', help= 'Whether scoring is done against the DM components or the predicted spectra' ) parser_compare.add_argument( '--ivar_cutoff', type=float, default=0.001, metavar='IVAR_CUTOFF', help='data with inverse variace below cutoff is masked as if ivar==0') parser_compare.add_argument( '--plot_final_errors', action='store_true', help='If set, will plot the errors from the final/best model, for the whole dataset, from ' + \ 'the best model re-trained on CV folds used for testing.' + \ 'Plots all errors on top of each other with low-ish alpha, to give a kind of visual ' + \ 'density map of errors.' ) args = parser.parse_args() obs_metadata = trim_observation_metadata( load_observation_metadata(args.metadata_path, flags=args.metadata_flags)) sources, components, exposures, wavelengths = ICAize.deserialize_data( args.spectra_path, args.method) source_model, ss, model_args = ICAize.unpickle_model( args.spectra_path, args.method) comb_flux_arr, comb_exposure_arr, comb_wavelengths = None, None, None if args.use_spectra: comb_flux_arr, comb_exposure_arr, comb_ivar_arr, comb_masks, comb_wavelengths = ICAize.load_data( args) filter_arr = np.in1d(comb_exposure_arr, exposures) comb_flux_arr = comb_flux_arr[filter_arr] comb_exposure_arr = comb_exposure_arr[filter_arr] sorted_inds = np.argsort(comb_exposure_arr) comb_flux_arr = comb_flux_arr[sorted_inds] comb_exposure_arr = comb_exposure_arr[sorted_inds] del comb_ivar_arr del comb_masks reduced_obs_metadata = obs_metadata[np.in1d(obs_metadata['EXP_ID'], exposures)] reduced_obs_metadata.sort('EXP_ID') sorted_inds = np.argsort(exposures) reduced_obs_metadata.remove_column('EXP_ID') md_len = len(reduced_obs_metadata) var_count = len(reduced_obs_metadata.columns) X_arr = np.array(reduced_obs_metadata).view('f8').reshape((md_len, -1)) Y_arr = sources[sorted_inds] if args.load_model: predictive_model = load_model(args.model_path) else: predictive_model = get_model(args.model) if args.subparser_name == 'compare': pdist = get_param_distribution_for_model(args.model, args.iters) scorer = None if args.scorer == 'R2': scorer = make_scorer(R2) elif args.scorer == 'MAE': if args.use_spectra: p_MAE_ = partial(MAE, Y_full=Y_arr, flux_arr=comb_flux_arr, source_model=source_model, ss=ss, source_model_args=model_args, method=args.method) scorer = make_scorer(p_MAE_, greater_is_better=False) else: scorer = make_scorer(MAE, greater_is_better=False) elif args.scorer == 'MSE': if args.use_spectra: p_MSE_ = partial(MSE, Y_full=Y_arr, flux_arr=comb_flux_arr, source_model=source_model, ss=ss, source_model_args=model_args, method=args.method) scorer = make_scorer(p_MSE_, greater_is_better=False) else: scorer = make_scorer(MSE, greater_is_better=False) elif args.scorer == 'MSEMV': if args.use_spectra: p_MSEMV_ = partial(MSEMV, Y_full=Y_arr, flux_arr=comb_flux_arr, source_model=source_model, ss=ss, source_model_args=model_args, method=args.method) scorer = make_scorer(p_MSEMV_, greater_is_better=False) else: scorer = make_scorer(MSEMV, greater_is_better=False) elif args.scorer == 'EXP_VAR': if args.use_spectra: p_EXP_VAR_ = partial(EXP_VAR, Y_full=Y_arr, flux_arr=comb_flux_arr, source_model=source_model, ss=ss, source_model_args=model_args, method=args.method) scorer = make_scorer(p_EXP_VAR_) else: scorer = make_scorer(EXP_VAR) elif args.scorer == 'MAPED': if args.use_spectra: p_MAPED_ = partial(MAPED, Y_full=Y_arr, flux_arr=comb_flux_arr, source_model=source_model, ss=ss, source_model_args=model_args, method=args.method) scorer = make_scorer(p_MAPED_, greater_is_better=False) else: scorer = make_scorer(MAPED, greater_is_better=False) elif args.scorer == 'LL': scorer = None folder = ShuffleSplit(exposures.shape[0], n_iter=args.folds, test_size=1.0 / args.folds, random_state=12345) if args.model == 'GP': predictive_model.random_start = args.folds rcv = GridSearchCV(predictive_model, param_grid=pdist, error_score=0, cv=3, n_jobs=args.n_jobs, scoring=scorer) #random_state=RANDOM_STATE, #n_iter=args.iters, else: rcv = RandomizedSearchCV(predictive_model, param_distributions=pdist, n_iter=args.iters, cv=folder, n_jobs=args.n_jobs, scoring=scorer) # This is going to fit X (metdata) to Y (DM'ed sources). But there are # really two tests here: how well hyperparams fit/predict the sources # and how well they fit/predict the actual source spectra. Until I know # better, I 'm going to need to build a way to test both. rcv.fit(X_arr, Y_arr) print(rcv.best_score_) print(rcv.best_params_) print(rcv.best_estimator_) if args.outputfbk: print("=+" * 10 + "=") for val in rcv.grid_scores_: print(val) print("=+" * 10 + "=") if args.save_best: save_model(rcv.best_estimator_, args.model_path) if args.plot_final_errors: for train_inds, test_inds in folder: rcv.best_estimator_.fit(X_arr[train_inds], Y_arr[train_inds]) predicted = rcv.best_estimator_.predict(X_arr[test_inds]) back_trans_flux = ICAize.inverse_transform( predicted, source_model, ss, args.method, model_args) diffs = np.abs(comb_flux_arr[test_inds] - back_trans_flux) #Is there not 'trick' to getting matplotlib to do this without a loop? for i in range(diffs.shape[0]): plt.plot(comb_wavelengths, diffs[i, :], 'b-', alpha=0.01) plt.show()
def convert_fits_to_hdf(args=None): """ Convert a FITS file to HDF5 in HDFITS format An input and output directory must be specified, and all files with a matching extension will be converted. Command line options set the compression algorithm and other run-time settings. """ # Parse options and arguments parser = argparse.ArgumentParser(description='Convert FITS files to HDF5 files in HDFITS format.') parser.add_argument('-c', '--compression', dest='comp', type=str, help='Data compression. Defaults to None, also lzf, bitshuffle, gzip') parser.add_argument('-x', '--extension', dest='ext', type=str, default='fits', help='File extension of FITS files. Defaults to .fits') parser.add_argument('-v', '--verbosity', dest='verbosity', type=int, default=4, help='verbosity level (default 0, up to 5)') parser.add_argument('-s', '--scaleoffset', dest='scale_offset', default=None, help='Add scale offset') parser.add_argument('-S', '--shuffle', dest='shuffle', action='store_true', default=None, help='Apply byte shuffle filter') parser.add_argument('-t', '--pytables', dest='table_type', action='store_true', default=None, help='Set output tables to be PyTables TABLE class, instead of HDFITES DATA_GROUP') parser.add_argument('-C', '--checksum', dest='checksum', action='store_true', default=None, help='Compute fletcher32 checksum on datasets.') parser.add_argument('dir_in', help='input directory') parser.add_argument('dir_out', help='output_directory') args = parser.parse_args() dir_in = args.dir_in dir_out = args.dir_out if not os.path.exists(dir_out): print("Creating directory %s" % dir_out) os.mkdir(dir_out) # Form a list of keyword arguments to pass to HDF5 export kwargs = {} if args.comp is not None: kwargs['compression'] = args.comp if args.scale_offset is not None: kwargs['scaleoffset'] = int(args.scale_offset) if args.shuffle is not None: kwargs['shuffle'] = args.shuffle if args.checksum is not None: kwargs['fletcher32'] = args.checksum if args.table_type is not None: kwargs['table_type'] = 'TABLE' else: kwargs['table_type'] = 'DATA_GROUP' pp = PrintLog(verbosity=args.verbosity) if args.verbosity == 0: warnings.simplefilter("ignore") pp.h1("FITS2HDF") pp.pa("Input directory: %s" % dir_in) pp.pa("Output directory: %s" % dir_out) pp.pa("Dataset creation arguments:") for key, val in kwargs.items(): pp.pa("%16s: %s" % (key, val)) # Create list of files to process filelist = os.listdir(dir_in) filelist = [fn for fn in filelist if fn.endswith(args.ext)] t_start = time.time() file_count = 0 for filename in filelist: file_in = os.path.join(dir_in, filename) file_out = os.path.join(dir_out, filename.split('.' + args.ext)[0] + '.h5') a = IdiHdulist() try: pp.pp("\nReading %s" % file_in) a = read_fits(file_in) pp.pp("Creating %s" % file_out) t1 = time.time() export_hdf(a, file_out, **kwargs) t2 = time.time() pp.pp("Input filesize: %sB" % os.path.getsize(file_in)) pp.pp("Output filesize: %sB" % os.path.getsize(file_out)) compfact = float(os.path.getsize(file_in)) / float(os.path.getsize(file_out)) pp.pp("Compression: %2.2fx" % compfact) pp.pp("Comp/write time: %2.2fs" % (t2 - t1)) file_count += 1 except IOError: pp.err("ERROR: Cannot load %s" % file_in) pp.h1("\nSUMMARY") pp.pa("Files created: %i" % file_count) pp.pa("Time taken: %2.2fs" % (time.time() - t_start))
def convert_hdf_to_fits(args=None): """ Convert a HDF5 (in HDFITS format) to a FITS file An input and output directory must be specified, and all files with a matching extension will be converted. Command line options set the run-time settings. """ # Parse options and arguments parser = argparse.ArgumentParser(description='Convert HDF5 in HDFITS format FITS files.') parser.add_argument('-x', '--extension', dest='ext', type=str, default='h5', help='File extension of HDFITS files. Defaults to .h5') parser.add_argument('-v', '--verbosity', dest='verbosity', type=int, default=4, help='verbosity level (default 0, up to 5)') parser.add_argument('dir_in', help='input directory') parser.add_argument('dir_out', help='output_directory') args = parser.parse_args() dir_in = args.dir_in dir_out = args.dir_out if not os.path.exists(dir_out): print("Creating directory %s" % dir_out) os.mkdir(dir_out) # Form a list of keyword arguments to pass to HDF5 export kwargs = {} pp = PrintLog(verbosity=args.verbosity) if args.verbosity == 0: warnings.simplefilter("ignore") pp.h1("HDF2FITS") pp.pa("Input directory: %s" % dir_in) pp.pa("Output directory: %s" % dir_out) pp.pa("Dataset creation arguments:") for key, val in kwargs.items(): pp.pa("%16s: %s" % (key, val)) # Create list of files to process filelist = os.listdir(dir_in) filelist = [fn for fn in filelist if fn.endswith(args.ext)] t_start = time.time() file_count = 0 for filename in filelist: file_in = os.path.join(dir_in, filename) file_out = os.path.join(dir_out, filename.split('.' + args.ext)[0] + '.fits') a = IdiHdulist() try: pp.pp("\nReading %s" % file_in) a = read_hdf(file_in) pp.pp("Creating %s" % file_out) t1 = time.time() export_fits(a, file_out, **kwargs) t2 = time.time() pp.pp("Input filesize: %sB" % os.path.getsize(file_in)) pp.pp("Output filesize: %sB" % os.path.getsize(file_out)) compfact = float(os.path.getsize(file_in)) / float(os.path.getsize(file_out)) pp.pp("Compression: %2.2fx" % compfact) pp.pp("Comp/write time: %2.2fs" % (t2 - t1)) file_count += 1 except IOError: pp.err("ERROR: Cannot load %s" % file_in) pp.h1("\nSUMMARY") pp.pa("Files created: %i" % file_count) pp.pa("Time taken: %2.2fs" % (time.time() - t_start))
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description= 'Compute PCA/ICA/NMF/etc. components over set of stacked spectra, save those out, and pickle model' ) parser.add_argument('--pattern', type=str, default='stacked*exp??????.*', metavar='PATTERN', help='File pattern for stacked sky fibers.') parser.add_argument('--path', type=str, default='.', metavar='PATH', help='Path to work from, if not ' '.' '') parser.add_argument( '--compacted_path', type=str, default=None, metavar='COMPATED_PATH', help= 'Path to find compacted/arrayized data; setting this will cause --path, --pattern to be ignored' ) parser.add_argument('--n_components', type=int, default=40, metavar='N_COMPONENTS', help='Number of ICA/PCA/etc. components') parser.add_argument( '--method', type=str, default='ICA', metavar='METHOD', choices=['ICA', 'PCA', 'SPCA', 'NMF', 'ISO', 'KPCA', 'FA', 'DL'], help='Which dim. reduction method to use') parser.add_argument( '--scale', action='store_true', help= 'Should inputs variance be scaled? Defaults to mean subtract and value scale, but w/out this does not scale variance.' ) parser.add_argument('--no_scale', action='store_true', help='Suppresses all scaling') parser.add_argument( '--ivar_cutoff', type=float, default=0.001, metavar='IVAR_CUTOFF', help='data with inverse variace below cutoff is masked as if ivar==0') parser.add_argument( '--n_iter', type=int, default=1200, metavar='MAX_ITER', help= 'Maximum number of iterations to allow for convergence. For SDSS data 1000 is a safe number of ICA, while SPCA requires larger values e.g. ~2000 to ~2500' ) parser.add_argument('--n_jobs', type=int, default=None, metavar='N_JOBS', help='N_JOBS') args = parser.parse_args() comb_flux_arr, comb_exposure_arr, comb_ivar_arr, comb_masks, comb_wavelengths = iz.load_data( args) model = iz.get_model(args.method, n=args.n_components, n_neighbors=None, max_iter=args.n_iter, random_state=iz.random_state, n_jobs=args.n_jobs) ss = None if args.no_scale: scaled_flux_arr = comb_flux_arr else: ss = skpp.StandardScaler(with_std=False) if args.scale: ss = skpp.StandardScaler(with_std=True) scaled_flux_arr = ss.fit_transform(comb_flux_arr) #Heavily copied from J. Vanderplas/astroML bayesian_blocks.py N = comb_wavelengths.size step = args.n_components * 4 edges = np.concatenate([ comb_wavelengths[:1:step], 0.5 * (comb_wavelengths[1::step] + comb_wavelengths[:-1:step]), comb_wavelengths[-1::step] ]) block_length = comb_wavelengths[-1::step] - edges # arrays to store the best configuration nn_vec = np.ones(N / step) * step best = np.zeros(N, dtype=float) last = np.zeros(N, dtype=int) for R in range(N / step): print("R: " + str(R)) width = block_length[:R + 1] - block_length[R + 1] count_vec = np.cumsum(nn_vec[:R + 1][::-1])[::-1] #width = nn_vec[:R + 1] - nn_vec[R + 1] #count_vec = np.cumsum(nn_vec[:R + 1][::-1])[::-1] #print(width) #print(count_vec) #raw_input("Pausing... ") fit_vec = map( lambda n: iz.score_via_CV(['LL'], scaled_flux_arr[:, :n], model, ss, args.method, folds=3, n_jobs=args.n_jobs), count_vec) fit_vec = [d["mle"] for d in fit_vec] #print(fit_vec) fit_vec[1:] += best[:R] #print(fit_vec) i_max = np.argmax(fit_vec) last[R] = i_max best[R] = fit_vec[i_max] #print(best) change_points = np.zeros(N / step, dtype=int) i_cp = N / step ind = N / step while True: i_cp -= 1 change_points[i_cp] = ind if ind == 0: break ind = last[ind - 1] change_points = change_points[i_cp:] print(edges[change_points]) '''
def main(args=None): """ This is the main function called by the `ivelplot` script. """ from astropy.utils.compat import argparse from astropy.extern.configobj import configobj, validate from pkg_resources import resource_stream parser = argparse.ArgumentParser( description='An interactive environment for absorption line ' 'identification and Voigt profile \nfitting with VPFIT.\n' '\nTo dump a default configuration file: ivelplot -d' '\nTo dump an extended default configuration file: ' 'ivelplot -dd', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('config', help='path to the configuration file') parser.add_argument('-z', '--redshift', help='redshift') parser.add_argument('--search', action='store_true', help='display a general search list of ions') parser.add_argument('--lyman', action='store_true', help='display the Lyman series transitions') parser.add_argument('--galactic', action='store_true', help='display the common Galactic transitions') parser.add_argument('--agn', action='store_true', help='display the common AGN associated transitions') config = resource_stream(__name__, '/config/ivelplot.cfg') config_extended = resource_stream( __name__, '/config/ivelplot_extended.cfg') spec = resource_stream(__name__, '/config/ivelplot_specification.cfg') if len(sys.argv) > 1: if sys.argv[1] == '-d': cfg = configobj.ConfigObj(config) cfg.filename = '{0}/ivelplot.cfg'.format(os.getcwd()) cfg.write() return elif sys.argv[1] == '-dd': cfg = configobj.ConfigObj(config_extended) cfg.filename = '{0}/ivelplot.cfg'.format(os.getcwd()) cfg.write() return args = parser.parse_args(args) try: cfg = configobj.ConfigObj(args.config, configspec=spec) validator = validate.Validator() cfg.validate(validator) except: raise IOError('Configuration file could not be read') fname = cfg['WINDOW'].pop('transitions') if args.search: fh = resource_stream(__name__, '/data/search.dat') transitions = list(fh) fh.close() elif args.lyman: fh = resource_stream(__name__, '/data/lyman.dat') transitions = list(fh) fh.close() elif args.galactic: fh = resource_stream(__name__, '/data/galactic.dat') transitions = list(fh) fh.close() elif args.agn: fh = resource_stream(__name__, '/data/agn.dat') transitions = list(fh) fh.close() else: print('Reading transitions from ', fname) fh = open(fname) transitions = list(fh) fh.close() transitions = [transition for transition in transitions if not transition.startswith('#')] fname = cfg['DATA'].pop('filename') if not fname: raise IOError('no data to plot!') spectrum = Table.read(fname) if fname[-4:] == 'fits' else ascii.read(fname) wavelength = spectrum[cfg['DATA'].pop('wavelength_column')] flux = spectrum[cfg['DATA'].pop('flux_column')] error = spectrum[cfg['DATA'].pop('error_column')] continuum = spectrum[cfg['DATA'].pop('continuum_column')] redshift = float(args.redshift) if args.redshift is not None else 0 cfg['MODEL']['system_width'] = (cfg['WINDOW']['vmax'] - cfg['WINDOW']['vmin']) cfg['MODEL']['absorbers'] = None print(info) app = QApplication(sys.argv) app.aboutToQuit.connect(app.deleteLater) desktop = app.desktop() screen = desktop.screenGeometry() width = screen.width() / desktop.physicalDpiX() * 0.88 fontsize = 0.7 * width label_fontsize = 0.6 * width cfg['WINDOW']['width'] = width cfg['WINDOW']['fontsize'] = fontsize cfg['WINDOW']['label_fontsize'] = label_fontsize velocity_plot = InteractiveVelocityPlot( fname, transitions, wavelength, flux, error, continuum, redshift, **cfg) velocity_plot.window.show() output_stream = OutputStream() output_stream.text_written.connect(velocity_plot.on_output) sys.stdout = output_stream sys.exit(app.exec_())
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='Add in solar and ecliptic data to observational metadata') parser.add_argument('--metadata_file_path', type=str, default=None, metavar='PATH', help='Metadata file path to work from, if not ' '.' '') parser.add_argument('--metadata_path', type=str, default='.', metavar='PATH', help='Metadata path to work from, if not ' '.' '') parser.add_argument( '--input', type=str, default='FITS', metavar='OUTPUT', help='Output format, either of FITS or CSV, defaults to FITS.') parser.add_argument('--solar_metadata', type=str, default='solar_ephemeris.csv', help='Solar metadata file (from parse_ephemeris.py)') args = parser.parse_args() if args.metadata_file_path is None: if args.input == 'CSV': obs_md_table = astab.Table.read(os.path.join( args.metadata_path, "annotated_metadata.csv"), format="ascii.csv") elif args.input == 'FITS': obs_md_table = astab.Table.read(os.path.join( args.metadata_path, "annotated_metadata.fits"), format="fits") else: obs_md_table = Table.read(args.metadata_file_path) solar_md_table = astab.Table.read(args.solar_metadata, format="ascii.csv") solar_md_table.rename_column('UTC', 'EPHEM_DATE') obs_md_table = astab.join( obs_md_table, solar_md_table['EPHEM_DATE', 'RA_ABS', 'DEC_ABS']) boresight_ra_dec = ascoord.SkyCoord( ra=obs_md_table['RA'], dec=obs_md_table['DEC'], distance=1.0, unit=('deg', 'deg', 'AU'), frame='fk5') #Change distance to... e.g. 1 Mpc boresight_ecliptic = boresight_ra_dec.transform_to( 'heliocentrictrueecliptic') solar_ra_dec = ascoord.SkyCoord(ra=obs_md_table['RA_ABS'], dec=obs_md_table['DEC_ABS'], distance=1.0, unit=('deg', 'deg', 'AU'), frame='icrs') solar_ecliptic = solar_ra_dec.transform_to('heliocentrictrueecliptic') obs_md_table.add_column( astab.Column(boresight_ecliptic.lat, dtype=float, name="ECLIPTIC_PLANE_SEP")) belp = np.mod(boresight_ecliptic.lon.value + 360.0, 360.0) selp = np.mod(solar_ecliptic.lon.value + 360.0, 360.0) lon_diff = np.abs(belp - selp) lon_diff[lon_diff > 180] -= 360 lon_diff = np.abs(lon_diff) obs_md_table.add_column( astab.Column(lon_diff, dtype=float, name="ECLIPTIC_PLANE_SOLAR_SEP")) obs_md_table.remove_columns(['RA_ABS', 'DEC_ABS']) if args.metadata_file_path is None: if args.input == 'CSV': obs_md_table.write(os.path.join(args.metadata_path, "annotated_metadata.csv"), format="ascii.csv") elif args.input == 'FITS': obs_md_table.write(os.path.join(args.metadata_path, "annotated_metadata.fits"), format="fits", overwrite=True) else: obs_md_table.write(args.metadata_file_path, overwrite=True)
def main(args=None): """This is the main function called by the `fitsheader` script.""" from astropy.utils.compat import argparse parser = argparse.ArgumentParser( description=('Print the header(s) of a FITS file. ' 'Optional arguments allow the desired extension(s), ' 'keyword(s), and output format to be specified. ' 'Note that in the case of a compressed image, ' 'the decompressed header is shown by default.')) parser.add_argument('-e', '--extension', metavar='HDU', action='append', dest='extensions', help='specify the extension by name or number; ' 'this argument can be repeated ' 'to select multiple extensions') parser.add_argument('-k', '--keyword', metavar='KEYWORD', action='append', dest='keywords', help='specify a keyword; this argument can be ' 'repeated to select multiple keywords; ' 'also supports wildcards') parser.add_argument('-t', '--table', nargs='?', default=False, metavar='FORMAT', help='print the header(s) in machine-readable table ' 'format; the default format is ' '"ascii.fixed_width" (can be "ascii.csv", ' '"ascii.html", "ascii.latex", "fits", etc)') parser.add_argument('-c', '--compressed', action='store_true', help='for compressed image data, ' 'show the true header which describes ' 'the compression rather than the data') parser.add_argument('filename', nargs='+', help='path to one or more files; ' 'wildcards are supported') args = parser.parse_args(args) # If `--table` was used but no format specified, # then use ascii.fixed_width by default if args.table is None: args.table = 'ascii.fixed_width' # Now print the desired headers try: if args.table: print_headers_as_table(args) else: print_headers_traditional(args) except IOError as e: # A 'Broken pipe' IOError may occur when stdout is closed prematurely, # eg. when calling `fitsheader file.fits | head`. We let this pass. pass
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description= 'Build and test models based on dim reductions and provided spectra') parser.add_argument('--metadata_path', type=str, default='.', metavar='PATH', help='Metadata path to work from, if not ' '.' '') parser.add_argument('--model_path', type=str, default='model.pkl', metavar='MODEL_PATH', help='COMPLETE path from which to load a model') parser.add_argument('--start_dt', type=ani.valid_date, help='DateTime to plot sky for') parser.add_argument('--end_dt', type=ani.valid_date, help='DateTime to plot sky for') parser.add_argument('--lunar_metadata', type=str, default=None, metavar='LUNAR_METADATA', required=True, help='File containing lunar ephemeris metadata.') parser.add_argument('--solar_metadata', type=str, default=None, metavar='SOLAR_METADATA', required=True, help='File containing solar ephemeris metadata.') parser.add_argument('--sunspot_metadata', type=str, default=None, metavar='SUNSPOT_METADATA', required=True, help='File containing sunspot metadata.') parser.add_argument('--method', type=str) parser.add_argument('--ra', type=str) parser.add_argument('--dec', type=str) parser.add_argument('--dm_path', type=str) args = parser.parse_args() obs_coord = coord.SkyCoord(args.ra, args.dec, frame='icrs') metadata_tups = ani.get_sky_for_coord(args.start_dt, args.end_dt, obs_coord, args.lunar_metadata, args.solar_metadata, args.sunspot_metadata) spectra, labels = ani.animate_sky_spectra_for_coord( args.start_dt, args.end_dt, obs_coord, args.lunar_metadata, args.solar_metadata, args.sunspot_metadata, args.model_path, args.dm_path, args.method) #print(spectra) xscale = np.arange(len(spectra[0])) #print(xscale) fig = plt.figure() ax = plt.axes(xlim=(0, 7200), ylim=(0, 500)) line, = ax.plot([], []) def init_func(): line.set_data([], []) return line, def animate_func(i, data, xscale): line.set_data(xscale, data[i]) plt.title(labels[i]) return line, p_animate_func = partial(animate_func, data=spectra, xscale=xscale) anim = animation.FuncAnimation(fig, p_animate_func, init_func=init_func, frames=len(spectra), interval=1000) #, blit=True) anim.save('polaris_sky_animation.mp4', fps=1, extra_args=['-vcodec', 'libx264']) plt.show()
def convert_fits_to_fits(args=None): """ Read a FITS file into the in-memory IDI format then back out into a FITS file An input and output directory must be specified, and all files with a matching extension will be converted. Command line options set the compression algorithm and other run-time settings. """ # Parse options and arguments parser = argparse.ArgumentParser(description='Convert FITS files to HDF5 files in HDFITS format.') parser.add_argument('-x', '--extension', dest='ext', type=str, default='fits', help='File extension of FITS files. Defaults to .fits') parser.add_argument('-v', '--verbosity', dest='vb', type=int, default=0, help='verbosity level (default 0, up to 5)') parser.add_argument('-w', '--nowarn', dest='warn', action='store_false', default=True, help='Turn off warnings created by FITS parsing') parser.add_argument('-o', '--overwrite', dest='overwrite', action='store_true', default=False, help='Automatically overwrite output files if already exist') parser.add_argument('dir_in', help='input directory') parser.add_argument('dir_out', help='output_directory') args = parser.parse_args() dir_in = args.dir_in dir_out = args.dir_out if not os.path.exists(dir_out): print("Creating directory %s" % dir_out) os.mkdir(dir_out) if not args.warn: warnings.simplefilter("ignore") try: assert dir_in != dir_out except AssertionError: print("Input directory cannot be same as output directory.") exit() # Create list of files to process filelist = os.listdir(dir_in) filelist = [fn for fn in filelist if fn.endswith(args.ext)] t1 = time.time() file_count = 0 for filename in filelist: file_in = os.path.join(dir_in, filename) file_out = os.path.join(dir_out, filename) a = IdiHdulist() try: a = read_fits(file_in) if os.path.exists(file_out): if args.overwrite: os.remove(file_out) else: qn = raw_input("%s exists. Overwrite (y/n)?" % file_out) if qn in ["y", "Y", "yes"]: os.remove(file_out) print("\nCreating %s" % file_out) export_fits(a, file_out) print("Input filesize: %sB" % os.path.getsize(file_in)) print("Output filesize: %sB" % os.path.getsize(file_out)) compfact = float(os.path.getsize(file_in)) / float(os.path.getsize(file_out)) print("Compression: %2.2fx" % compfact) file_count += 1 except IOError: print("ERROR: Cannot read/write %s" % file_in) print("\nSUMMARY") print("-------") print("Files created: %i" % file_count) print("Time taken: %2.2fs" % (time.time() - t1))
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='Compute PCA/ICA/NMF/etc. components over set of stacked spectra, save those out, and pickle model' ) subparsers = parser.add_subparsers(dest='subparser_name') parser.add_argument( '--pattern', type=str, default='stacked*exp??????.*', metavar='PATTERN', help='File pattern for stacked sky fibers.' ) parser.add_argument( '--path', type=str, default='.', metavar='PATH', help='Path to work from, if not ''.''' ) parser.add_argument( '--compacted_path', type=str, default=None, metavar='COMPATED_PATH', help='Path to find compacted/arrayized data; setting this will cause --path, --pattern to be ignored' ) parser.add_argument( '--method', type=str, default=['ICA'], metavar='METHOD', choices=['ICA', 'PCA', 'SPCA', 'NMF', 'ISO', 'KPCA', 'FA', 'DL'], nargs='+', help='Which dim. reduction method to use' ) parser.add_argument( '--scale', action='store_true', help='Should inputs be scaled? Will mean subtract and value scale, but does not scale variace.' ) parser.add_argument( '--ivar_cutoff', type=float, default=0.001, metavar='IVAR_CUTOFF', help='data with inverse variace below cutoff is masked as if ivar==0' ) parser.add_argument( '--n_iter', type=int, default=1200, metavar='MAX_ITER', help='Maximum number of iterations to allow for convergence. For SDSS data 1000 is a safe number of ICA, while SPCA requires larger values e.g. ~2000 to ~2500' ) parser.add_argument( '--n_jobs', type=int, default=None, metavar='N_JOBS', help='N_JOBS' ) parser_compare = subparsers.add_parser('compare') parser_compare.add_argument( '--max_components', type=int, default=50, metavar='COMP_MAX', help='Max number of components to use/test' ) parser_compare.add_argument( '--min_components', type=int, default=0, metavar='COMP_MIN', help='Min number of compoenents to use/test' ) parser_compare.add_argument( '--step_size', type=int, default=5, metavar='COMP_STEP', help='Step size from comp_min to comp_max' ) parser_compare.add_argument( '--comparison', choices=['EXP_VAR', 'R2', 'MSE', 'MAE'], nargs='*', default=['EXP_VAR'], help='Comparison methods: Explained variance (score), R2 (score), mean sq. error (loss), MEDIAN absolute error (loss)' ) parser_compare.add_argument( '--mle_if_avail', action='store_true', help='In additon to --comparison, include MLE if PCA or FA methods specified' ) parser_compare.add_argument( '--plot_example_reconstruction', action='store_true', help='Pick a random spectrum, plot its actual and reconstructed versions' ) parser_build = subparsers.add_parser('build') parser_build.add_argument( '--n_components', type=int, default=40, metavar='N_COMPONENTS', help='Number of ICA/PCA/etc. components' ) parser_build.add_argument( '--n_neighbors', type=int, default=10, metavar='N_NEIGHBORS', help='Number of neighbots for e.g. IsoMap' ) args = parser.parse_args() comb_flux_arr, comb_exposure_arr, comb_ivar_arr, comb_masks, comb_wavelengths = iz.load_data(args) if 'DL' in args.method: flux_arr = comb_flux_arr.astype(dtype=np.float64) else: flux_arr = comb_flux_arr scaled_flux_arr = None ss = None if args.scale: ss = skpp.StandardScaler(with_std=False) scaled_flux_arr = ss.fit_transform(flux_arr) else: scaled_flux_arr = flux_arr if args.subparser_name == 'compare': fig, ax1 = plt.subplots() ax2 = ax1.twinx() for method in args.method: model = iz.get_model(method, max_iter=args.n_iter, random_state=iz.random_state, n_jobs=args.n_jobs) scores = {} mles_and_covs = args.mle_if_avail and (method == 'FA' or method == 'PCA') n_components = np.arange(args.min_components, args.max_components+1, args.step_size) for n in n_components: print("Cross validating for n=" + str(n) + " on method " + method) model.n_components = n comparisons = iz.score_via_CV(args.comparison, flux_arr if method == 'NMF' else scaled_flux_arr, model, method, n_jobs=args.n_jobs, include_mle=mles_and_covs, modeler=_iter_modeler, scorer=_iter_scorer) for key, val in comparisons.items(): if key in scores: scores[key].append(val) else: scores[key] = [val] if mles_and_covs: #ax2.axhline(cov_mcd_score(scaled_flux_arr, args.scale), color='violet', label='MCD Cov', linestyle='--') ax2.axhline(cov_lw_score(scaled_flux_arr, args.scale), color='orange', label='LW Cov', linestyle='--') for key, score_list in scores.items(): if key != 'mle': ax1.plot(n_components, score_list, label=method + ':' + key + ' scores') else: ax2.plot(n_components, score_list, '-.', label=method + ' mle scores') ax1.set_xlabel('nb of components') ax1.set_ylabel('CV scores', figure=fig) ax1.legend(loc='lower left') ax2.legend(loc='lower right') plt.show()
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='Pull metadata from FITS for PLATE/MJD combos, output.') parser.add_argument('--plate', type=int, default=None, metavar='PLATE', help='Plate number to pull metadata for.') parser.add_argument( '--mjd', type=int, default=None, metavar='MJD', help= 'MJD of plate observation to use (can be omitted if only one value is possible)' ) parser.add_argument( '--fiber', type=int, default=1, metavar='FIBER', help= 'Fiber number identifying the spectrum of the requested PLATE-MJD to plot.' ) parser.add_argument( '--list_file', type=str, default=None, metavar='LIST_FILE', help= 'File that contains list of PLATE, MJD, FIBER records to output metadata for; e.g. the output from bossquery' ) parser.add_argument('--output', type=str, default='FITS', metavar='OUTPUT', help='Output format, either of ' 'FITS' ' or ' 'CSV' ', defaults to FITS.') parser.add_argument( '--dont_gather', action='store_true', help= 'Flag to prevent storing output in file; just dumps metadata to stdout.' ) args = parser.parse_args() if args.plate is not None and args.mjd is not None: file_deets(plate=args.plate, mjd=args.mjd, fiber=args.fiber) elif args.list_file is not None: plates_table = Table.read(args.list_file, format='ascii') exposure_table_list = [] exposure_table = None if not args.dont_gather: progress_bar = ProgressBar(widgets=[Percentage(), Bar()], maxval=len(plates_table)).start() counter = 0 for row in plates_table: try: exposure_data = file_deets(row['PLATE'], row['MJD'], gather=not args.dont_gather) if exposure_data is not None: exposure_table_list.append(Table(exposure_data)) except RuntimeError as re: print "Caught runtime:" print re if not args.dont_gather: counter += 1 progress_bar.update(counter) if not args.dont_gather: progress_bar.finish() if len(exposure_table_list): if len(exposure_table_list) > 1: exposure_table = vstack(exposure_table_list) else: exposure_table = exposure_table_list[0] if args.output.upper() == 'CSV': exposure_table.write("exposure_metadata.csv", format="ascii.csv") elif args.output.upper() == 'FITS': exposure_table.write("exposure_metadata.fits", format="fits")
def evaluate(args=None): # parse command-line arguments parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-v', '--verbose', action='store_true', help='provide verbose output on progress') parser.add_argument('--prior', default=None, help='Name of the prior to use, e.g., qso-4-10k.fits') parser.add_argument('-n', '--num-spectra', type=int, default=1, metavar='N', help='Number of spectra to use for evaluation.') parser.add_argument('--mag-err', type=float, default=0.1, metavar='dM', help='RMS error on targeting magnitudes to simulate.') parser.add_argument( '--quad-order', type=int, default=16, metavar='N', help='Quadrature order to use for magnitude marginalization.') parser.add_argument('--seed', type=int, default=None, metavar='S', help='Random seed to use for sampling templates.') args = parser.parse_args(args) if args.prior is None: print('You must specify a prior to use.') return -1 if args.seed is None: print('You must specify a seed to use.') return -1 # Parse the prior name, which is expected to have format xxx-nn-nnnk.fits basename, ext = os.path.splitext(args.prior) if ext != '.fits': print('Unexpected extension for prior filename: {}'.format(ext)) return -1 try: classname, downsampling, _ = basename.split('_') downsampling = int(downsampling) except ValueError: print('Badly formatted prior filename: {}.'.format(basename)) return -1 if classname not in ('qso', 'lrg', 'elg', 'elgem', 'star'): print('Invalid prior class name: {}.'.format(classname)) return -1 if args.verbose: print('Prior uses downsampling {} for class {}.'.format( downsampling, classname)) # Load the prior now. Prepend $BAYEZ_DATA unless we already have # an absolute path. path = os.environ.get('BAYEZ_DATA', '.') if not os.path.isabs(args.prior): args.prior = os.path.join(path, args.prior) if args.verbose: print('Reading prior from {}'.format(args.prior)) prior = bayez.prior.load_prior(args.prior) # Prepare to simulate spectra for evaluation. sampler = bayez.sampler.Samplers[classname]() simulator = bayez.simulation.Simulator(analysis_downsampling=downsampling, verbose=args.verbose) # Run the evaluation. estimator = bayez.estimator.RedshiftEstimator( prior, dz=0.001, quadrature_order=args.quad_order) results = bayez.estimator.estimate_batch( estimator, args.num_spectra, sampler, simulator, mag_err=args.mag_err, seed=args.seed, print_interval=500 if args.verbose else 0) # Save the results. name = os.path.join( path, '{}_q{:+d}_{}.fits'.format(basename, args.quad_order, args.seed)) if args.verbose: print('Saving results to {}'.format(name)) results.write(name, overwrite=True)