def update_info(zp=None):
"""
Update information in zeropoint file, e.g. after calibration.
Call first L{ivs.sed.model.calibrate} without arguments, and pass the output
to this function.
@param zp: updated contents from C{zeropoints.dat}
@type zp: recarray
"""
zp_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'zeropoints.dat')
zp_, comms = ascii.read2recarray(zp_file, return_comments=True)
existing = [str(i.strip()) for i in zp_['photband']]
resp_files = sorted(
glob.glob(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'filters/*')))
resp_files = [
os.path.basename(ff) for ff in resp_files
if not os.path.basename(ff) in existing
]
resp_files.remove('HUMAN.EYE')
resp_files.remove('HUMAN.CONES')
resp_files.remove('CONES.EYE')
if zp is None:
zp = zp_
logger.info(
'No new calibrations; previous information on existing response curves is copied'
)
else:
logger.info(
'Received new calibrations contents of zeropoints.dat will be updated'
)
#-- update info on previously non existing response curves
new_zp = np.zeros(len(resp_files), dtype=zp.dtype)
logger.info(
'Found {} new response curves, adding them with default information'.
format(len(resp_files)))
for i, respfile in enumerate(resp_files):
new_zp[i]['photband'] = respfile
new_zp[i]['eff_wave'] = float(eff_wave(respfile))
new_zp[i]['type'] = 'CCD'
new_zp[i]['vegamag'] = np.nan
new_zp[i]['ABmag'] = np.nan
new_zp[i]['STmag'] = np.nan
new_zp[i]['Flam0_units'] = 'erg/s/cm2/AA'
new_zp[i]['Fnu0_units'] = 'erg/s/cm2/AA'
new_zp[i]['source'] = 'nan'
zp = np.hstack([zp, new_zp])
sa = np.argsort(zp['photband'])
ascii.write_array(zp[sa],
'zeropoints.dat',
header=True,
auto_width=True,
comments=['#' + line for line in comms[:-2]],
use_float='%g')
def systematics(units='muHz'):
"""
Return a list of known systematic effects from the Kepler satellite.
"""
ffile = config.get_datafile('catalogs/kepler','systematics.dat')
systems = ascii.read2recarray(ffile)
systems['frequency'] = conversions.nconvert(systems['unit'],units,systems['frequency'])
systems['e_frequency'] = conversions.nconvert(systems['unit'],units,systems['e_frequency'])
systems['w_frequency'] = conversions.nconvert(systems['unit'],units,systems['w_frequency'])
return systems
def systematics(units='muHz'):
"""
Return a list of known systematic effects from the Kepler satellite.
"""
ffile = config.get_datafile('catalogs/kepler', 'systematics.dat')
systems = ascii.read2recarray(ffile)
systems['frequency'] = conversions.nconvert(systems['unit'], units,
systems['frequency'])
systems['e_frequency'] = conversions.nconvert(systems['unit'], units,
systems['e_frequency'])
systems['w_frequency'] = conversions.nconvert(systems['unit'], units,
systems['w_frequency'])
return systems
def search(ID, radius=1., filename=None):
"""
Retrieve datafiles from the Coralie catalogue.
We search on coordinates, pulled from SIMBAD. If the star ID is not
recognised, a string search is performed to match the 'targ name' field in the
FITS headers.
Only the s1d_A data are searched.
@param ID: ID of the star, understandable by SIMBAD
@type ID: str
@param radius: search radius around the coordinates
@type radius: 1
@param filename: write summary to outputfile if not None
@type filename: str
@return: record array with summary information on the observations, as well
as their location (column 'filename')
@rtype: numpy rec array
"""
data = ascii.read2recarray(config.get_datafile(
os.path.join('catalogs', 'coralie'), 'CoralieFullDataOverview.tsv'),
splitchar='\t')
info = sesame.search(ID)
if info:
ra, dec = info['jradeg'], info['jdedeg']
keep = np.sqrt((data['ra'] - ra)**2 +
(data['dec'] - dec)**2) < radius / 60.
else:
keep = [((re.compile(ID).search(objectn) is not None) and True
or False) for objectn in data['object']]
keep = np.array(keep)
data = data[keep]
logger.info('Found %d spectra' % (len(data)))
if filename is not None:
ascii.write_array(data, filename, auto_width=True, header=True)
else:
return data
def update_info(zp=None):
"""
Update information in zeropoint file, e.g. after calibration.
Call first L{ivs.sed.model.calibrate} without arguments, and pass the output
to this function.
@param zp: updated contents from C{zeropoints.dat}
@type zp: recarray
"""
zp_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),'zeropoints.dat')
zp_,comms = ascii.read2recarray(zp_file,return_comments=True)
existing = [str(i.strip()) for i in zp_['photband']]
resp_files = sorted(glob.glob(os.path.join(os.path.dirname(os.path.abspath(__file__)),'filters/*')))
resp_files = [os.path.basename(ff) for ff in resp_files if not os.path.basename(ff) in existing]
resp_files.remove('HUMAN.EYE')
resp_files.remove('HUMAN.CONES')
resp_files.remove('CONES.EYE')
if zp is None:
zp = zp_
logger.info('No new calibrations; previous information on existing response curves is copied')
else:
logger.info('Received new calibrations contents of zeropoints.dat will be updated')
#-- update info on previously non existing response curves
new_zp = np.zeros(len(resp_files),dtype=zp.dtype)
logger.info('Found {} new response curves, adding them with default information'.format(len(resp_files)))
for i,respfile in enumerate(resp_files):
new_zp[i]['photband'] = respfile
new_zp[i]['eff_wave'] = float(eff_wave(respfile))
new_zp[i]['type'] = 'CCD'
new_zp[i]['vegamag'] = np.nan
new_zp[i]['ABmag'] = np.nan
new_zp[i]['STmag'] = np.nan
new_zp[i]['Flam0_units'] = 'erg/s/cm2/AA'
new_zp[i]['Fnu0_units'] = 'erg/s/cm2/AA'
new_zp[i]['source'] = 'nan'
zp = np.hstack([zp,new_zp])
sa = np.argsort(zp['photband'])
ascii.write_array(zp[sa],'zeropoints.dat',header=True,auto_width=True,comments=['#'+line for line in comms[:-2]],use_float='%g')
def get_info(photbands=None):
"""
Return a record array containing all filter information.
The record arrays contains following columns:
- photband
- eff_wave
- type
- vegamag, vegamag_lit
- ABmag, ABmag_lit
- STmag, STmag_lit
- Flam0, Flam0_units, Flam0_lit
- Fnu0, Fnu0_units, Fnu0_lit,
- source
@param photbands: list of photbands to get the information from. The input
order is equal to the output order. If C{None}, all filters are returned.
@type photbands: iterable container (list, tuple, 1Darray)
@return: record array containing all information on the requested photbands.
@rtype: record array
"""
zp_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'zeropoints.dat')
zp = ascii.read2recarray(zp_file)
for iph in custom_filters:
if iph == '_prefer_file': continue
if 'zp' in custom_filters[iph]:
zp = np.hstack([zp, custom_filters[iph]['zp']])
zp = zp[np.argsort(zp['photband'])]
#-- list photbands in order given, and remove those that do not have
# zeropoints etc.
if photbands is not None:
order = np.searchsorted(zp['photband'], photbands)
zp = zp[order]
keep = (zp['photband'] == photbands)
zp = zp[keep]
return zp
def get_info(photbands=None):
"""
Return a record array containing all filter information.
The record arrays contains following columns:
- photband
- eff_wave
- type
- vegamag, vegamag_lit
- ABmag, ABmag_lit
- STmag, STmag_lit
- Flam0, Flam0_units, Flam0_lit
- Fnu0, Fnu0_units, Fnu0_lit,
- source
@param photbands: list of photbands to get the information from. The input
order is equal to the output order. If C{None}, all filters are returned.
@type photbands: iterable container (list, tuple, 1Darray)
@return: record array containing all information on the requested photbands.
@rtype: record array
"""
zp_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),'zeropoints.dat')
zp = ascii.read2recarray(zp_file)
for iph in custom_filters:
if iph=='_prefer_file': continue
if 'zp' in custom_filters[iph]:
zp = np.hstack([zp,custom_filters[iph]['zp']])
zp = zp[np.argsort(zp['photband'])]
#-- list photbands in order given, and remove those that do not have
# zeropoints etc.
if photbands is not None:
order = np.searchsorted(zp['photband'],photbands)
zp = zp[order]
keep = (zp['photband']==photbands)
zp = zp[keep]
return zp
def search(ID,radius=1.,filename=None):
"""
Retrieve datafiles from the Coralie catalogue.
We search on coordinates, pulled from SIMBAD. If the star ID is not
recognised, a string search is performed to match the 'targ name' field in the
FITS headers.
Only the s1d_A data are searched.
@param ID: ID of the star, understandable by SIMBAD
@type ID: str
@param radius: search radius around the coordinates
@type radius: 1
@param filename: write summary to outputfile if not None
@type filename: str
@return: record array with summary information on the observations, as well
as their location (column 'filename')
@rtype: numpy rec array
"""
data = ascii.read2recarray(config.get_datafile(os.path.join('catalogs','coralie'),'CoralieFullDataOverview.tsv'),splitchar='\t')
info = sesame.search(ID)
if info:
ra,dec = info['jradeg'],info['jdedeg']
keep = np.sqrt((data['ra']-ra)**2 + (data['dec']-dec)**2) < radius/60.
else:
keep = [((re.compile(ID).search(objectn) is not None) and True or False) for objectn in data['object']]
keep = np.array(keep)
data = data[keep]
logger.info('Found %d spectra'%(len(data)))
if filename is not None:
ascii.write_array(data,filename,auto_width=True,header=True)
else:
return data
def get_lines(teff,logg,z=0,atoms=None,ions=None,wrange=(-inf,inf),\
blend=0.0):
"""
Retrieve line transitions and strengths for a specific stellar type
Selection wavelength range in angstrom.
Ions should be a list of ions to include. This can either be a string or
a number
A lines is considerd a blend if the closest line is closer than C{blend} angstrom.
Returns record array with fields C{wavelength}, C{ion} and C{depth}.
Example usage:
Retrieve all Silicon lines between 4500 and 4600 for a B1V star.
>>> data = get_lines(20000,4.0,atoms=['Si'],wrange=(4500,4600))
>>> p = pl.figure()
>>> p = pl.vlines(data['wavelength'],1,1-data['depth'])
See how the depth of the Halpha line varies wrt temperature:
>>> teffs = range(5000,21000,1000) + range(22000,32000,2000) + range(30000,50000,50000)
>>> depths = np.zeros((len(teffs),7))
>>> for i,teff in enumerate(teffs):
... data = get_lines(teff,5.0,ions=['HI'],wrange=(3800,7000))
... depths[i] = data['depth']
>>> p = pl.figure();p = pl.title('Depth of Balmer lines (Halpha-Heta)')
>>> p = pl.plot(teffs,1-depths,'o-')
>>> p = pl.xlabel('Effective temperature');p = pl.grid()
"""
#-- get filepath
filename = 'mask.%d.%02d.p%02d'%(int(teff),int(logg*10),int(z))
filename = config.get_datafile(stellar,filename)
#-- read in the data and extract relevant columns
data = ascii.read2recarray(filename,skip_lines=1,dtype=[('wavelength','f8'),('ion','f8'),('depth','f8'),('c3','f8'),('c4','f8'),('c5','f8')])
data = pl.mlab.rec_drop_fields(data,['c3','c4','c5'])
data['wavelength'] *= 10.
#-- remove blends
if blend>0:
blends_left = np.hstack([0,np.diff(data['wavelength'])])
blends_right= np.hstack([np.diff(data['wavelength']),1e10])
keep = (blends_left>blend) & (blends_right>blend)
data = data[keep]
#-- only keep those transitions within a certain wavelength range
keep = (wrange[0]<=data['wavelength']) & (data['wavelength']<=wrange[1])
data = data[keep]
#-- only keep those transitions that belong to certain ions or atoms
if atoms is not None or ions is not None:
keep = np.array(np.zeros(len(data)),bool)
else:
keep = np.array(np.ones(len(data)),bool)
if atoms is not None:
#-- convert all atoms to their number and select the appropriate ones
atoms = [(isinstance(atom,str) and atomcode.index(atom.title()) or atom) for atom in atoms]
for atom in atoms:
keep = keep | (np.abs(data['ion']-atom)<0.5)
if ions is not None:
#-- convert all ions to their number and select the appropriate ones
ions = [(isinstance(ion,str) and name2ioncode(ion) or ion) for ion in ions]
for ion in ions:
keep = keep | (np.abs(data['ion']-ion)<0.005)
return data[keep]
def search(ID=None,time_range=None,prog_ID=None,data_type='cosmicsremoved_log',
radius=1.,filename=None):
"""
Retrieve datafiles from the Hermes catalogue.
B{If C{ID} is given}: A string search is performed to match the 'object'
field in the FITS headers. The coordinates are pulled from SIMBAD. If the
star ID is recognised by SIMBAD, an additional search is done based only on
the coordinates. The union of both searches is the final result.
B{If C{time_range} is given}: The search is confined within the defined
range. If you only give one day, the search is confined to the observations
made during the night starting at that day. If C{ID} is not given, all
observations will be returned of the given datatype.
B{If C{prog_ID} is given}: The search is performed to match the number of
the program. Individual stars are not queried in SIMBAD, so any information
that is missing in the header will not be corrected.
If you don't give either ID or time_range, the info on all data will be
returned. This is a huge amount of data, so it can take a while before it
is returned. Remember that the header of each spectrum is read in and checked.
Data type can be any of:
1. cosmicsremoved_log: return log merged without cosmics
2. cosmicsremoved_wavelength: return wavelength merged without cosmics
3. ext_log: return log merged with cosmics
4. ext_wavelength: return wavelength merged with cosmics
5. raw: raw files (also TECH..., i.e. any file in the raw directory)
This functions needs a C{HermesFullDataOverview.tsv} file located in one
of the datadirectories from C{config.py}, and subdirectory C{catalogs/hermes}.
If this file does not exist, you can create it with L{make_data_overview}.
If you want a summary file with the data you search for, you can give
C{filename} as an extra keyword argument. The results will be saved to that
file.
The columns in the returned record array are listed in L{make_data_overview},
but are repeated here (capital letters are directly retrieved from the
fits header, small letters are calculated values. The real header strings
are all small capitals):
1. UNSEQ
2. PROG_ID
3. OBSMODE
4. BVCOR
5. OBSERVER
6. OBJECT
7. RA
8. DEC
9. BJD
10. EXPTIME
11. PMTOTAL
12. DATE-AVG
13. OBJECT
14. airmass
15. filename
The column C{filename} contains a string with the absolute location of the
file. If you need any extra information from the header, you can easily
retrieve it.
If BVCOR or BJD are not available from the FITS header, this function will
attempt to calculate it. It will not succeed if the object's name is not
recognised by SIMBAD.
Example usage: retrieve all data on HD50230
>>> mydata = search('HD50230')
Keep only those with a long enough exposure time:
>>> myselection = mydata[mydata['exptime']>500]
Look up the 'telalt' value in the FITS headers of all these files via a fast
list comprehension:
>>> telalts = [pf.getheader(fname)['telalt'] for fname in myselection['filename']]
Search for all data of HD50230 taken in the night of 22 September 2009:
>>> data = search('HD50230',time_range='2009-9-22')
Or within an interval of a few days:
>>> data = search('HD50230',time_range=('2009-9-22','2009-9-30'))
Search for all data observed in a given night:
>>> data = search(time_range='2009-9-22')
B{Warning:} the heliocentric correction is not calculated when no ID is given,
so make sure it is present in the header if you need it, or calculate it yourself.
@param ID: ID of the star, understandable by SIMBAD
@type ID: str
@param time_range: range of dates to confine the search to
@type time_range: tuple strings of type '2009-09-23T04:24:35.712556' or '2009-09-23'
@param data_type: if None, all data will be returned. Otherwise, subset
'cosmicsremoved', 'merged' or 'raw'
@type data_type: str
@param radius: search radius around the coordinates (arcminutes)
@type radius: float
@param filename: write summary to outputfile if not None
@type filename: str
@return: record array with summary information on the observations, as well
as their location (column 'filename')
@rtype: numpy rec array
"""
#-- read in the data from the overview file, and get SIMBAD information
# of the star
ctlFile = '/STER/mercator/hermes/HermesFullDataOverview.tsv'
data = ascii.read2recarray(ctlFile, splitchar='\t')
#data = ascii.read2recarray(config.get_datafile(os.path.join('catalogs','hermes'),'HermesFullDataOverview.tsv'),splitchar='\t')
keep = np.array(np.ones(len(data)),bool)
#-- confined search within given time range
if time_range is not None:
if isinstance(time_range,str):
time_range = _timestamp2datetime(time_range)
time_range = (time_range,time_range+datetime.timedelta(days=1))
else:
time_range = (_timestamp2datetime(time_range[0]),_timestamp2datetime(time_range[1]))
keep = keep & np.array([(time_range[0]<=_timestamp2datetime(i)<=time_range[1]) for i in data['date-avg']],bool)
info = None
#-- search on ID
if ID is not None:
info = sesame.search(ID)
#-- first search on object name only
ID = ID.replace(' ','').replace('.','').replace('+','').replace('-','').replace('*','')
match_names = np.array([objectn.replace(' ','').replace('.','').replace('+','').replace('-','').replace('*','') for objectn in data['object']],str)
keep_id = [((((ID in objectn) or (objectn in ID)) and len(objectn)) and True or False) for objectn in match_names]
keep_id = np.array(keep_id)
# if we found the star on SIMBAD, we use its RA and DEC to match the star
if info:
ra,dec = info['jradeg'],info['jdedeg']
keep_id = keep_id | (np.sqrt((data['ra']-ra)**2 + (data['dec']-dec)**2) < radius/60.)
keep = keep & keep_id
if prog_ID is not None:
keep = keep & (data['prog_id']==prog_ID)
#-- if some data is found, we check if the C{data_type} string is contained
# with the file's name. If not, we remove it.
if np.any(keep):
data = data[keep]
if data_type is not None:
data_type == data_type.lower()
#-- now derive the location of the 'data_type' types from the raw
# files
if not data_type=='raw':
data['filename'] = [_derive_filelocation_from_raw(ff,data_type) for ff in data['filename']]
existing_files = np.array([ff!='naf' for ff in data['filename']],bool)
data = data[existing_files]
seqs = sorted(set(data['unseq']))
logger.info('ID={}/prog_ID={}: Found {:d} spectra (data type={} with unique unseqs)'.format(ID,prog_ID,len(seqs),data_type))
else:
data = data[:0]
logger.info('%s: Found no spectra'%(ID))
#-- we now check if the barycentric correction was calculated properly.
# If not, we calculate it here, but only if the object was found in
# SIMBAD. Else, we have no information on the ra and dec (if bvcorr was
# not calculated, ra and dec are not in the header).
for obs in data:
if ID is not None and info:
try:
jd = _timestamp2jd(obs['date-avg'])
except ValueError:
logger.info('Header probably corrupted for unseq {}: no info on time or barycentric correction'.format(obs['unseq']))
jd = np.nan
# the previous line is equivalent to:
# day = dateutil.parser.parse(header['DATE-AVG'])
# BJD = ephem.julian_date(day)
bvcorr, hjd = helcorr(ra/360.*24, dec, jd)
else:
break
if np.isnan(obs['bvcor']):
logger.info("Corrected 'bvcor' for unseq {} (missing in header)".format(obs['unseq']))
obs['bvcor'] = float(bvcorr)
if np.isnan(obs['bjd']):
logger.info("Corrected 'bjd' for unseq {} (missing in header)".format(obs['unseq']))
obs['bjd'] = float(hjd)
#-- do we need the information as a file, or as a numpy array?
if filename is not None:
ascii.write_array(data,filename,auto_width=True,header=True)
else:
return data
def make_data_overview():
"""
Summarize all Hermes data in a file for easy data retrieval.
The file is located in one of date data directories (see C{config.py}), in
subdirectories C{catalogs/hermes/HermesFullDataOverview.tsv}. If it doesn't
exist, it will be created. It contains the following columns, which are
extracted from the Hermes FITS headers (except C{filename}:
1. UNSEQ
2. PROG_ID
3. OBSMODE
4. BVCOR
5. OBSERVER
6. OBJECT
7. RA
8. DEC
9. BJD
10. EXPTIME
11. PMTOTAL
12. DATE-AVG
13. OBJECT
14. airmass
15. filename
This file can most easily be read with the L{ivs.inout.ascii} module and the
command:
>>> hermes_file = config.get_datafile(os.path.join('catalogs','hermes'),'HermesFullDataOverview.tsv')
>>> data = ascii.read2recarray(hermes_file,splitchar='\\t')
"""
logger.info('Collecting files...')
#-- all hermes data directories
dirs = sorted(glob.glob(os.path.join(config.ivs_dirs['hermes'],'20??????')))
dirs = [idir for idir in dirs if os.path.isdir(idir)]
obj_files = []
#-- collect in those directories the raw and relevant reduced files
for idir in dirs:
obj_files += sorted(glob.glob(os.path.join(idir,'raw','*.fits')))
#obj_files += sorted(glob.glob(os.path.join(idir,'reduced','*OBJ*wavelength_merged.fits')))
#obj_files += sorted(glob.glob(os.path.join(idir,'reduced','*OBJ*wavelength_merged_c.fits')))
#obj_files += sorted(glob.glob(os.path.join(idir,'reduced','*OBJ*log_merged.fits')))
#obj_files += sorted(glob.glob(os.path.join(idir,'reduced','*OBJ*log_merged_c.fits')))
#-- keep track of what is already in the file, if it exists:
try:
overview_file = config.get_datafile('catalogs/hermes','HermesFullDataOverview.tsv')
#overview_file = config.get_datafile(os.path.join('catalogs','hermes'),'HermesFullDataOverview.tsv')
overview_data = ascii.read2recarray(overview_file,splitchar='\t')
outfile = open(overview_file,'a')
logger.info('Found %d FITS files: appending to overview file %s'%(len(obj_files),overview_file))
# if not, begin a new file
except IOError:
overview_file = 'HermesFullDataOverview.tsv'
outfile = open(overview_file,'w')
outfile.write('#unseq prog_id obsmode bvcor observer object ra dec bjd exptime pmtotal date-avg airmass filename\n')
outfile.write('#i i a20 >f8 a50 a50 >f8 >f8 >f8 >f8 >f8 a30 >f8 a200\n')
overview_data = {'filename':[]}
logger.info('Found %d FITS files: starting new overview file %s'%(len(obj_files),overview_file))
#-- and summarize the contents in a tab separated file (some columns contain spaces)
existing_files = np.sort(overview_data['filename'])
for i,obj_file in enumerate(obj_files):
sys.stdout.write(chr(27)+'[s') # save cursor
sys.stdout.write(chr(27)+'[2K') # remove line
sys.stdout.write('Scanning %5d / %5d FITS files'%(i+1,len(obj_files)))
sys.stdout.flush() # flush to screen
#-- maybe this file is already processed: forget about it then
index = existing_files.searchsorted(obj_file)
if index<len(existing_files) and existing_files[index]==obj_file:
sys.stdout.write(chr(27)+'[u') # reset cursor
continue
#-- keep track of: UNSEQ, PROG_ID, OBSMODE, BVCOR, OBSERVER,
# OBJECT, RA, DEC, BJD, EXPTIME, DATE-AVG, PMTOTAL,
# airmass and filename (not part of fitsheader)
contents = dict(unseq=-1,prog_id=-1,obsmode='nan',bvcor=np.nan,observer='nan',
object='nan',ra=np.nan,dec=np.nan,
bjd=np.nan,exptime=np.nan,pmtotal=np.nan,airmass=np.nan,
filename=os.path.realpath(obj_file))
contents['date-avg'] = 'nan'
header = pf.getheader(obj_file)
for key in contents:
if key in header and key in ['unseq','prog_id']:
try: contents[key] = int(header[key])
except: pass
elif key in header and key in ['obsmode','observer','object','date-avg']:
contents[key] = str(header[key])
elif key in header and key in ['ra','dec','exptime','pmtotal','bjd','bvcor']:
contents[key] = float(header[key])
elif key=='airmass' and 'telalt' in header:
if float(header['telalt'])<90:
try:
contents[key] = airmass.airmass(90-float(header['telalt']))
except ValueError:
pass
outfile.write('%(unseq)d\t%(prog_id)d\t%(obsmode)s\t%(bvcor)f\t%(observer)s\t%(object)s\t%(ra)f\t%(dec)f\t%(bjd)f\t%(exptime)f\t%(pmtotal)f\t%(date-avg)s\t%(airmass)f\t%(filename)s\n'%contents)
outfile.flush()
sys.stdout.write(chr(27)+'[u') # reset cursor
outfile.close()
return overview_file
def search(ID=None,time_range=None,prog_ID=None,data_type='cosmicsremoved_log',
radius=1.,filename=None):
"""
Retrieve datafiles from the Hermes catalogue.
B{If C{ID} is given}: A string search is performed to match the 'object'
field in the FITS headers. The coordinates are pulled from SIMBAD. If the
star ID is recognised by SIMBAD, an additional search is done based only on
the coordinates. The union of both searches is the final result.
B{If C{time_range} is given}: The search is confined within the defined
range. If you only give one day, the search is confined to the observations
made during the night starting at that day. If C{ID} is not given, all
observations will be returned of the given datatype.
B{If C{prog_ID} is given}: The search is performed to match the number of
the program. Individual stars are not queried in SIMBAD, so any information
that is missing in the header will not be corrected.
If you don't give either ID or time_range, the info on all data will be
returned. This is a huge amount of data, so it can take a while before it
is returned. Remember that the header of each spectrum is read in and checked.
Data type can be any of:
1. cosmicsremoved_log: return log merged without cosmics
2. cosmicsremoved_wavelength: return wavelength merged without cosmics
3. ext_log: return log merged with cosmics
4. ext_wavelength: return wavelength merged with cosmics
5. raw: raw files (also TECH..., i.e. any file in the raw directory)
This functions needs a C{HermesFullDataOverview.tsv} file located in one
of the datadirectories from C{config.py}, and subdirectory C{catalogs/hermes}.
If this file does not exist, you can create it with L{make_data_overview}.
If you want a summary file with the data you search for, you can give
C{filename} as an extra keyword argument. The results will be saved to that
file.
The columns in the returned record array are listed in L{make_data_overview},
but are repeated here (capital letters are directly retrieved from the
fits header, small letters are calculated values. The real header strings
are all small capitals):
1. UNSEQ
2. PROG_ID
3. OBSMODE
4. BVCOR
5. OBSERVER
6. OBJECT
7. RA
8. DEC
9. BJD
10. EXPTIME
11. PMTOTAL
12. DATE-AVG
13. OBJECT
14. airmass
15. filename
The column C{filename} contains a string with the absolute location of the
file. If you need any extra information from the header, you can easily
retrieve it.
If BVCOR or BJD are not available from the FITS header, this function will
attempt to calculate it. It will not succeed if the object's name is not
recognised by SIMBAD.
Example usage: retrieve all data on HD50230
>>> mydata = search('HD50230')
Keep only those with a long enough exposure time:
>>> myselection = mydata[mydata['exptime']>500]
Look up the 'telalt' value in the FITS headers of all these files via a fast
list comprehension:
>>> telalts = [pf.getheader(fname)['telalt'] for fname in myselection['filename']]
Search for all data of HD50230 taken in the night of 22 September 2009:
>>> data = hermes.search('HD50230',time_range='2009-9-22')
Or within an interval of a few days:
>>> data = hermes.search('HD50230',time_range=('2009-9-23','2009-9-30'))
Search for all data observed in a given night:
>>> data = hermes.search(time_range='2009-9-22')
B{Warning:} the heliocentric correction is not calculated when no ID is given,
so make sure it is present in the header if you need it, or calculate it yourself.
@param ID: ID of the star, understandable by SIMBAD
@type ID: str
@param time_range: range of dates to confine the search to
@type time_range: tuple strings of type '2009-09-23T04:24:35.712556' or '2009-09-23'
@param data_type: if None, all data will be returned. Otherwise, subset
'cosmicsremoved', 'merged' or 'raw'
@type data_type: str
@param radius: search radius around the coordinates (arcminutes)
@type radius: float
@param filename: write summary to outputfile if not None
@type filename: str
@return: record array with summary information on the observations, as well
as their location (column 'filename')
@rtype: numpy rec array
"""
#-- read in the data from the overview file, and get SIMBAD information
# of the star
ctlFile = '/STER/mercator/hermes/HermesFullDataOverview.tsv'
data = ascii.read2recarray(ctlFile, splitchar='\t')
#data = ascii.read2recarray(config.get_datafile(os.path.join('catalogs','hermes'),'HermesFullDataOverview.tsv'),splitchar='\t')
keep = np.array(np.ones(len(data)),bool)
#-- confined search within given time range
if time_range is not None:
if isinstance(time_range,str):
time_range = _timestamp2datetime(time_range)
time_range = (time_range,time_range+datetime.timedelta(days=1))
else:
time_range = (_timestamp2datetime(time_range[0]),_timestamp2datetime(time_range[1]))
keep = keep & np.array([(time_range[0]<=_timestamp2datetime(i)<=time_range[1]) for i in data['date-avg']],bool)
info = None
#-- search on ID
if ID is not None:
info = sesame.search(ID)
#-- first search on object name only
ID = ID.replace(' ','').replace('.','').replace('+','').replace('-','').replace('*','')
match_names = np.array([objectn.replace(' ','').replace('.','').replace('+','').replace('-','').replace('*','') for objectn in data['object']],str)
keep_id = [((((ID in objectn) or (objectn in ID)) and len(objectn)) and True or False) for objectn in match_names]
keep_id = np.array(keep_id)
# if we found the star on SIMBAD, we use its RA and DEC to match the star
if info:
ra,dec = info['jradeg'],info['jdedeg']
keep_id = keep_id | (np.sqrt((data['ra']-ra)**2 + (data['dec']-dec)**2) < radius/60.)
keep = keep & keep_id
if prog_ID is not None:
keep = keep & (data['prog_id']==prog_ID)
#-- if some data is found, we check if the C{data_type} string is contained
# with the file's name. If not, we remove it.
if np.any(keep):
data = data[keep]
if data_type is not None:
data_type == data_type.lower()
#-- now derive the location of the 'data_type' types from the raw
# files
if not data_type=='raw':
data['filename'] = [_derive_filelocation_from_raw(ff,data_type) for ff in data['filename']]
existing_files = np.array([ff!='naf' for ff in data['filename']],bool)
data = data[existing_files]
seqs = sorted(set(data['unseq']))
logger.info('ID={}/prog_ID={}: Found {:d} spectra (data type={} with unique unseqs)'.format(ID,prog_ID,len(seqs),data_type))
else:
data = data[:0]
logger.info('%s: Found no spectra'%(ID))
#-- we now check if the barycentric correction was calculated properly.
# If not, we calculate it here, but only if the object was found in
# SIMBAD. Else, we have no information on the ra and dec (if bvcorr was
# not calculated, ra and dec are not in the header).
for obs in data:
if ID is not None and info:
try:
jd = _timestamp2jd(obs['date-avg'])
except ValueError:
logger.info('Header probably corrupted for unseq {}: no info on time or barycentric correction'.format(obs['unseq']))
jd = np.nan
# the previous line is equivalent to:
# day = dateutil.parser.parse(header['DATE-AVG'])
# BJD = ephem.julian_date(day)
bvcorr, hjd = helcorr(ra/360.*24, dec, jd)
else:
break
if np.isnan(obs['bvcor']):
logger.info("Corrected 'bvcor' for unseq {} (missing in header)".format(obs['unseq']))
obs['bvcor'] = float(bvcorr)
if np.isnan(obs['bjd']):
logger.info("Corrected 'bjd' for unseq {} (missing in header)".format(obs['unseq']))
obs['bjd'] = float(hjd)
#-- do we need the information as a file, or as a numpy array?
if filename is not None:
ascii.write_array(data,filename,auto_width=True,header=True)
else:
return data
def make_data_overview():
"""
Summarize all Hermes data in a file for easy data retrieval.
The file is located in one of date data directories (see C{config.py}), in
subdirectories C{catalogs/hermes/HermesFullDataOverview.tsv}. If it doesn't
exist, it will be created. It contains the following columns, which are
extracted from the Hermes FITS headers (except C{filename}:
1. UNSEQ
2. PROG_ID
3. OBSMODE
4. BVCOR
5. OBSERVER
6. OBJECT
7. RA
8. DEC
9. BJD
10. EXPTIME
11. PMTOTAL
12. DATE-AVG
13. OBJECT
14. airmass
15. filename
This file can most easily be read with the L{ivs.inout.ascii} module and the
command:
>>> hermes_file = config.get_datafile(os.path.join('catalogs','hermes'),'HermesFullDataOverview.tsv')
>>> data = ascii.read2recarray(hermes_file,splitchar='\\t')
"""
logger.info('Collecting files...')
#-- all hermes data directories
dirs = sorted(glob.glob(os.path.join(config.ivs_dirs['hermes'],'20??????')))
dirs = [idir for idir in dirs if os.path.isdir(idir)]
obj_files = []
#-- collect in those directories the raw and relevant reduced files
for idir in dirs:
obj_files += sorted(glob.glob(os.path.join(idir,'raw','*.fits')))
#obj_files += sorted(glob.glob(os.path.join(idir,'reduced','*OBJ*wavelength_merged.fits')))
#obj_files += sorted(glob.glob(os.path.join(idir,'reduced','*OBJ*wavelength_merged_c.fits')))
#obj_files += sorted(glob.glob(os.path.join(idir,'reduced','*OBJ*log_merged.fits')))
#obj_files += sorted(glob.glob(os.path.join(idir,'reduced','*OBJ*log_merged_c.fits')))
#-- keep track of what is already in the file, if it exists:
try:
overview_file = config.get_datafile('catalogs/hermes','HermesFullDataOverview.tsv')
#overview_file = config.get_datafile(os.path.join('catalogs','hermes'),'HermesFullDataOverview.tsv')
overview_data = ascii.read2recarray(overview_file,splitchar='\t')
outfile = open(overview_file,'a')
logger.info('Found %d FITS files: appending to overview file %s'%(len(obj_files),overview_file))
# if not, begin a new file
except IOError:
overview_file = 'HermesFullDataOverview.tsv'
outfile = open(overview_file,'w')
outfile.write('#unseq prog_id obsmode bvcor observer object ra dec bjd exptime pmtotal date-avg airmass filename\n')
outfile.write('#i i a20 >f8 a50 a50 >f8 >f8 >f8 >f8 >f8 a30 >f8 a200\n')
overview_data = {'filename':[]}
logger.info('Found %d FITS files: starting new overview file %s'%(len(obj_files),overview_file))
#-- and summarize the contents in a tab separated file (some columns contain spaces)
existing_files = np.sort(overview_data['filename'])
for i,obj_file in enumerate(obj_files):
sys.stdout.write(chr(27)+'[s') # save cursor
sys.stdout.write(chr(27)+'[2K') # remove line
sys.stdout.write('Scanning %5d / %5d FITS files'%(i+1,len(obj_files)))
sys.stdout.flush() # flush to screen
#-- maybe this file is already processed: forget about it then
index = existing_files.searchsorted(obj_file)
if index<len(existing_files) and existing_files[index]==obj_file:
sys.stdout.write(chr(27)+'[u') # reset cursor
continue
#-- keep track of: UNSEQ, PROG_ID, OBSMODE, BVCOR, OBSERVER,
# OBJECT, RA, DEC, BJD, EXPTIME, DATE-AVG, PMTOTAL,
# airmass and filename (not part of fitsheader)
contents = dict(unseq=-1,prog_id=-1,obsmode='nan',bvcor=np.nan,observer='nan',
object='nan',ra=np.nan,dec=np.nan,
bjd=np.nan,exptime=np.nan,pmtotal=np.nan,airmass=np.nan,
filename=os.path.realpath(obj_file))
contents['date-avg'] = 'nan'
header = pf.getheader(obj_file)
for key in contents:
if key in header and key in ['unseq','prog_id']:
try: contents[key] = int(header[key])
except: pass
elif key in header and key in ['obsmode','observer','object','date-avg']:
contents[key] = str(header[key])
elif key in header and key in ['ra','dec','exptime','pmtotal','bjd','bvcor']:
contents[key] = float(header[key])
elif key=='airmass' and 'telalt' in header:
if float(header['telalt'])<90:
try:
contents[key] = airmass.airmass(90-float(header['telalt']))
except ValueError:
pass
outfile.write('%(unseq)d\t%(prog_id)d\t%(obsmode)s\t%(bvcor)f\t%(observer)s\t%(object)s\t%(ra)f\t%(dec)f\t%(bjd)f\t%(exptime)f\t%(pmtotal)f\t%(date-avg)s\t%(airmass)f\t%(filename)s\n'%contents)
outfile.flush()
sys.stdout.write(chr(27)+'[u') # reset cursor
outfile.close()
return overview_file
def get_lines(teff,logg,z=0,atoms=None,ions=None,wrange=(-inf,inf),\
blend=0.0):
"""
Retrieve line transitions and strengths for a specific stellar type
Selection wavelength range in angstrom.
Ions should be a list of ions to include. This can either be a string or
a number
A lines is considerd a blend if the closest line is closer than C{blend} angstrom.
Returns record array with fields C{wavelength}, C{ion} and C{depth}.
Example usage:
Retrieve all Silicon lines between 4500 and 4600 for a B1V star.
>>> data = get_lines(20000,4.0,atoms=['Si'],ions=['CI','CII'],wrange=(4200,6800))
>>> p = pl.figure()
>>> p = pl.vlines(data['wavelength'],1,1-data['depth'])
See how the depth of the Halpha line varies wrt temperature:
>>> teffs = list(range(5000,21000,1000)) + list(range(22000,32000,2000)) + list(range(30000,50000,50000))
>>> depths = np.zeros((len(teffs),7))
>>> for i,teff in enumerate(teffs):
... data = get_lines(teff,5.0,ions=['HI'],wrange=(3800,7000))
... depths[i] = data['depth']
>>> p = pl.figure();p = pl.title('Depth of Balmer lines (Halpha-Heta)')
>>> p = pl.plot(teffs,1-depths,'o-')
>>> p = pl.xlabel('Effective temperature');p = pl.grid()
"""
#-- get filepath
filename = 'mask.%d.%02d.p%02d' % (int(teff), int(logg * 10), int(z))
filename = config.get_datafile(stellar, filename)
#-- read in the data and extract relevant columns
data = ascii.read2recarray(filename,
skip_lines=1,
dtype=[('wavelength', 'f8'), ('ion', 'f8'),
('depth', 'f8'), ('c3', 'f8'),
('c4', 'f8'), ('c5', 'f8')])
data = pl.mlab.rec_drop_fields(data, ['c3', 'c4', 'c5'])
data['wavelength'] *= 10.
#-- remove blends
if blend > 0:
blends_left = np.hstack([0, np.diff(data['wavelength'])])
blends_right = np.hstack([np.diff(data['wavelength']), 1e10])
keep = (blends_left > blend) & (blends_right > blend)
data = data[keep]
#-- only keep those transitions within a certain wavelength range
keep = (wrange[0] <= data['wavelength']) & (data['wavelength'] <=
wrange[1])
data = data[keep]
#-- only keep those transitions that belong to certain ions or atoms
if atoms is not None or ions is not None:
keep = np.array(np.zeros(len(data)), bool)
else:
keep = np.array(np.ones(len(data)), bool)
if atoms is not None:
#-- convert all atoms to their number and select the appropriate ones
atoms = [(isinstance(atom, str) and atomcode.index(atom.title())
or atom) for atom in atoms]
for atom in atoms:
keep = keep | (np.abs(data['ion'] - atom) < 0.5)
if ions is not None:
#-- convert all ions to their number and select the appropriate ones
ions = [(isinstance(ion, str) and name2ioncode(ion) or ion)
for ion in ions]
for ion in ions:
keep = keep | (np.abs(data['ion'] - ion) < 0.005)
return data[keep]
