def fetch(self,
config=GWSummConfigParser(),
segdb_error='raise',
datafind_error='raise',
**kwargs):
"""Finalise this state by fetching its defining segments,
either from global memory, or from the segment database
"""
# check we haven't done this before
if self.ready:
return self
# fetch data
if self.definition:
match = re.search('(%s)' % '|'.join(MATHOPS.keys()),
self.definition)
else:
match = None
if self.filename:
self._read_segments(self.filename)
elif match:
channel, thresh = self.definition.split(match.groups()[0])
channel = channel.rstrip()
thresh = float(thresh.strip())
self._fetch_data(channel,
thresh,
match.groups()[0],
config=config,
datafind_error=datafind_error,
**kwargs)
# fetch segments
elif self.definition:
self._fetch_segments(config=config,
segdb_error=segdb_error,
**kwargs)
# fetch null
else:
start = config.getfloat(DEFAULTSECT, 'gps-start-time')
end = config.getfloat(DEFAULTSECT, 'gps-end-time')
self.known = [(start, end)]
self.active = self.known
# restrict to given hours
if self.hours:
segs_ = SegmentList()
# get start day
d = Time(float(self.start), format='gps', scale='utc').datetime
d.replace(hour=0, minute=0, second=0, microsecond=0)
end_ = Time(float(self.end), format='gps', scale='utc').datetime
while d < end_:
# get GPS of day
t = to_gps(d)
# for each [start, end) hour pair, build a segment
for h0, h1 in self.hours:
segs_.append(Segment(t + h0 * 3600, t + h1 * 3600))
# increment and return
d += datetime.timedelta(1)
self.known &= segs_
self.active &= segs_
# FIXME
self.ready = True
return self
def add_pixels(self, tpf):
tpfdata = tpf[1].read()
aperture_shape = tpfdata['FLUX'][0].shape
# Get the pixel coordinates of the corner of the aperture
col, row = (self.template_tpf_header1['1CRV5P'],
self.template_tpf_header1['2CRV5P'])
height, width = aperture_shape[0], aperture_shape[1]
# Fill the data
mask = tpf[2].read() > 0
idx = self.cadenceno - tpfdata["CADENCENO"][0]
# When quality flag 65536 is raised, there is no data and the times are NaN.
if (tpfdata['QUALITY'][idx] & int(65536) > 0):
raise Exception(
'Error: Cadence {} does not appear to contain data!'.format(
self.cadenceno))
if self.add_background:
self.data[row:row+height, col:col+width][mask] = \
tpfdata['FLUX'][idx][mask] \
+ tpfdata['FLUX_BKG'][idx][mask]
self.uncert[row:row+height, col:col+width][mask] = \
np.sqrt(
(tpfdata['FLUX_ERR'][idx][mask])**2 +
(tpfdata['FLUX_BKG_ERR'][idx][mask])**2
)
else:
self.data[row:row+height, col:col+width][mask] = \
tpfdata['FLUX'][idx][mask]
self.uncert[row:row+height, col:col+width][mask] = \
tpfdata['FLUX_ERR'][idx][mask]
# If this is the first TPF being added, record the time and calculate DATE-OBS/END
if self.time is None:
self.time = tpfdata['TIME'][idx]
self.quality = tpfdata['QUALITY'][idx]
frametim = np.float(self.template_tpf_header1['FRAMETIM'])
num_frm = np.float(self.template_tpf_header1['NUM_FRM'])
# Calculate DATE-OBS from BJD time:
mjd_start = self.time \
+ np.float(self.template_tpf_header1['BJDREFI']) \
- frametim/3600./24./2. * num_frm \
- 2400000.5
self.mjdbeg = mjd_start
starttime = Time(mjd_start, format='mjd')
starttime = str(starttime.datetime)
self.dateobs = starttime.replace(' ', 'T') + 'Z'
# Calculate DATE-END:
mjd_end = self.time \
+ np.float(self.template_tpf_header1['BJDREFI']) \
+ frametim/3600./24./2. * num_frm \
- 2400000.5
self.mjdend = mjd_end
endtime = Time(mjd_end, format='mjd')
endtime = str(endtime.datetime)
self.dateend = endtime.replace(' ', 'T') + 'Z'
def fetch(self, config=GWSummConfigParser(), segdb_error='raise',
datafind_error='raise', **kwargs):
"""Finalise this state by fetching its defining segments,
either from global memory, or from the segment database
"""
# check we haven't done this before
if self.ready:
return self
# fetch data
if self.definition:
match = re.search('(%s)' % '|'.join(MATHOPS.keys()),
self.definition)
else:
match = None
if self.filename:
self._read_segments(self.filename)
elif match:
channel, thresh = self.definition.split(match.groups()[0])
channel = channel.rstrip()
thresh = float(thresh.strip())
self._fetch_data(channel, thresh, match.groups()[0], config=config,
datafind_error=datafind_error, **kwargs)
# fetch segments
elif self.definition:
self._fetch_segments(config=config, segdb_error=segdb_error,
**kwargs)
# fetch null
else:
start = config.getfloat(DEFAULTSECT, 'gps-start-time')
end = config.getfloat(DEFAULTSECT, 'gps-end-time')
self.known = [(start, end)]
self.active = self.known
# restrict to given hours
if self.hours:
segs_ = SegmentList()
# get start day
d = Time(float(self.start), format='gps', scale='utc').datetime
d.replace(hour=0, minute=0, second=0, microsecond=0)
end_ = Time(float(self.end), format='gps', scale='utc').datetime
while d < end_:
# get GPS of day
t = to_gps(d)
# for each [start, end) hour pair, build a segment
for h0, h1 in self.hours:
segs_.append(Segment(t + h0 * 3600, t + h1*3600))
# increment and return
d += datetime.timedelta(1)
self.known &= segs_
self.active &= segs_
# FIXME
self.ready = True
return self
def test_timezone_convenience_methods():
location = EarthLocation(-74.0 * u.deg, 40.7 * u.deg, 0 * u.m)
obs = Observer(location=location, timezone=pytz.timezone('US/Eastern'))
t = Time(57100.3, format='mjd')
assert (obs.astropy_time_to_datetime(t).hour == 3)
dt = datetime.datetime(2015, 3, 19, 3, 12)
assert (obs.datetime_to_astropy_time(dt).datetime == datetime.datetime(
2015, 3, 19, 7, 12))
assert (obs.astropy_time_to_datetime(
obs.datetime_to_astropy_time(dt)).replace(tzinfo=None) == dt)
# Test ndarray of times:
times = t + np.linspace(0, 24, 10) * u.hour
times_dt_ndarray = times.datetime
assert all(
(obs.datetime_to_astropy_time(times_dt_ndarray)).jd == (times +
4 * u.hour).jd)
# Test list of times:
times_dt_list = list(times.datetime)
assert all(
(obs.datetime_to_astropy_time(times_dt_list)).jd == (times +
4 * u.hour).jd)
dts = obs.astropy_time_to_datetime(times)
naive_dts = list(map(lambda t: t.replace(tzinfo=None), dts))
assert all(naive_dts == times_dt_ndarray - datetime.timedelta(hours=4))
def test_timezone_convenience_methods():
location = EarthLocation(-74.0*u.deg, 40.7*u.deg, 0*u.m)
obs = Observer(location=location,timezone=pytz.timezone('US/Eastern'))
t = Time(57100.3, format='mjd')
assert (obs.astropy_time_to_datetime(t).hour == 3)
dt = datetime.datetime(2015, 3, 19, 3, 12)
assert (obs.datetime_to_astropy_time(dt).datetime ==
datetime.datetime(2015, 3, 19, 7, 12))
assert (obs.astropy_time_to_datetime(obs.datetime_to_astropy_time(dt)).replace(
tzinfo=None) == dt)
# Test ndarray of times:
times = t + np.linspace(0, 24, 10)*u.hour
times_dt_ndarray = times.datetime
assert all((obs.datetime_to_astropy_time(times_dt_ndarray)).jd ==
(times + 4*u.hour).jd)
# Test list of times:
times_dt_list = list(times.datetime)
assert all((obs.datetime_to_astropy_time(times_dt_list)).jd ==
(times + 4*u.hour).jd)
dts = obs.astropy_time_to_datetime(times)
naive_dts = list(map(lambda t: t.replace(tzinfo=None), dts))
assert all(naive_dts == times_dt_ndarray - datetime.timedelta(hours=4))
def ant_trange(vis):
''' Figure out nominal times for tracking of old EOVSA antennas, and return time
range in CASA format
'''
import eovsa_array as ea
from astropy.time import Time
from taskinit import ms
# Get timerange from the visibility file
# msinfo = dict.fromkeys(['vis', 'scans', 'fieldids', 'btimes', 'btimestr', 'inttimes', 'ras', 'decs', 'observatory'])
ms.open(vis)
# metadata = ms.metadata()
scans = ms.getscansummary()
sk = np.sort(scans.keys())
vistrange = np.array([scans[sk[0]]['0']['BeginTime'],scans[sk[-1]]['0']['EndTime']])
# Get the Sun transit time, based on the date in the vis file name (must have UDByyyymmdd in the name)
aa = ea.eovsa_array()
date = vis.split('UDB')[-1][:8]
slashdate = date[:4] + '/' + date[4:6] + '/' + date[6:8]
aa.date = slashdate
sun = aa.cat['Sun']
mjd_transit = Time(aa.next_transit(sun).datetime(), format='datetime').mjd
# Construct timerange limits based on +/- 3h55m from transit time (when all dishes are nominally tracking)
# and clip the visibility range not to exceed those limits
mjdrange = np.clip(vistrange, mjd_transit - 0.1632, mjd_transit + 0.1632)
trange = Time(mjdrange[0], format='mjd').iso[:19] + '~' + Time(mjdrange[1], format='mjd').iso[:19]
trange = trange.replace('-', '/').replace(' ', '/')
return trange
def read_osc_input(filename, template=False):
""" self.filename in the form osc-name-ageIndex. E.g. osc-sn2002er-10"""
osc, objName, ageIdx = filename.split('-')
def read_json(url):
response = urlopen(url)
return json.loads(response.read(), object_pairs_hook=OrderedDict)
# Redshift
urlRedshift = "https://api.sne.space/" + objName + "/redshift/value"
redshift = read_json(urlRedshift)
redshift = float(redshift[next(iter(redshift))]['redshift'][0][0])
if template is False:
# Spectrum
urlSpectrum = "https://api.sne.space/" + objName + "/spectra/time+data?item={0}".format(
ageIdx)
data = read_json(urlSpectrum)
data = data[next(iter(data))]['spectra'][0][1]
wave, flux = np.array(list(map(list, zip(*data)))).astype(np.float)
return wave, flux, redshift
elif template is True:
# Age Max
urlAgeMax = "https://api.sne.space/" + objName + "/maxdate/value"
ageMax = read_json(urlAgeMax)
ageMax = ageMax[next(iter(ageMax))]['maxdate'][0][0]
ageMax = Time(ageMax.replace('/', '-')).mjd
# Type
urlTType = "https://api.sne.space/" + objName + "/claimedtype/value"
tType = read_json(urlTType)
tType = tType[next(
iter(tType))]['claimedtype'] # List of types... choose one [0][0]
# Spectrum
urlSpectrum = "https://api.sne.space/" + objName + "/spectra/time+data"
data = read_json(urlSpectrum)
data = data[next(iter(data))]['spectra']
nCols = len(data) # number of ages
waves = []
fluxes = []
ages = []
for datum in data:
age, spectrum = datum
age = age - ageMax
ages.append(age)
wave, flux = np.array(list(map(list, zip(*data)))).astype(np.float)
wave = wave / (redshift + 1) # De-redshift spectrum
waves.append(wave)
fluxes.append(flux)
return waves, fluxes, nCols, ages, tType
def standardizeMetadata(self):
run = self.header["RUN"].strip()
field = self.header["FIELD"].strip()
filter = self.header["COLOUR"].strip()
chip = self.header["CHIP"]
sciProg = f"{run}-{field}-{filter}-{chip}"
# TODO: Fix datetimes
# There is a timesys key but I have no idea how to generically instantiate
# timezone aware datetime and astropy Time seems not to work well with
# Django - the astrometadata is also broken!
if "UTC" in self.header["TIMESYS"].upper():
tzinfo = timezone.utc
else:
raise ValueError("Can not recognize time scale system that is used?")
jdstart = Time(self.header["JDSTART"], format="jd", scale="utc")
jdstart = jdstart.utc.datetime
jdstart = jdstart.replace(tzinfo=tzinfo)
jdend = Time(self.header["JDSTART"], format="jd", scale="utc")
jdend = jdend.utc.datetime
jdend = jdend.replace(tzinfo=tzinfo)
# TODO: filter out what is the filter standardization here?
meta = Metadata(
obs_lon=self.header["LOGITUD"],
obs_lat=self.header["LATITUD"],
obs_height=self.header["HEIGHT"],
datetime_begin=jdstart.isoformat(),
datetime_end=jdend.isoformat(),
telescope=self.header["OBSTEL"].strip(),
instrument=self.header["CAMERA"].strip(),
science_program=sciProg,
exposure_duration=self.header["EXPTIME"],
filter_name=self.header["COLOUR"].strip()
)
return meta
def set_info(longitude, latitude, date, ut):
observer = get_observer(longitude, latitude)
date = dt.strptime(re.split('T| ', date)[0], '%Y-%m-%d')
date = Time(date.replace(hour=int(ut)))
sunset = observer.sun_set_time(date).strftime('%d-%m-%Y %H:%M:%S')
sunrise = observer.sun_rise_time(date).strftime('%d-%m-%Y %H:%M:%S')
moon_phase = observer.moon_phase(date + 1 * u.day)
moon_altaz = observer.moon_altaz(date + 1 * u.day)
moon_alt = str(round(moon_altaz.alt.deg, 0))
moon_az = str(round(moon_altaz.az.deg, 0))
lst = observer.local_sidereal_time(date)
return sunset, sunrise, 100 - int(
moon_phase.value / np.pi * 100), ", ".join([moon_alt,
moon_az]), str(lst)
def ant_trange(vis):
''' Figure out nominal times for tracking of old EOVSA antennas, and return time
range in CASA format
'''
import eovsa_array as ea
from astropy.time import Time
# Get the Sun transit time, based on the date in the vis file name (must have UDByyyymmdd in the name)
aa = ea.eovsa_array()
date = vis.split('UDB')[-1][:8]
slashdate = date[:4] + '/' + date[4:6] + '/' + date[6:8]
aa.date = slashdate
sun = aa.cat['Sun']
mjd_transit = Time(aa.next_transit(sun).datetime(), format='datetime').mjd
# Construct timerange based on +/- 3h55m from transit time (when all dishes are nominally tracking)
trange = Time(mjd_transit - 0.1632, format='mjd').iso[:19] + '~' + Time(mjd_transit + 0.1632, format='mjd').iso[:19]
trange = trange.replace('-', '/').replace(' ', '/')
return trange
def observing_plan(self, date=None, remove_twilight=False, bright_time=False):
"""
Make a list of best tiles observable through the night.
Output: observing_plan.txt
# observing_ideal observing_start observing_stop fieldID path_to_config_file
1030 1000 1100 fieldID /observers_files/taipan/YYYYMMDD/tile_pk_HHMMSS.obs_config.json
observing_ideal = meridian transit time
observing_start, observing_stop: observing_ideal +/- dt, dt is determined using weights
Should times be given in UT?
What is the json file (e.g. list of stars with coordinates etc.?)
TODO: What is ObsConfig file?
"""
if date is None:
datenow=datetime.datetime.now().date()
date='%d-%02d-%02d'%(datenow.year, datenow.month, datenow.day)
datestring=date.replace('-', '')
date = Time('%s'%date)
else:
datestring=date.replace('-', '')
sun_set = self.observatory.sun_set_time(Time(date)).datetime
sunset=self.observatory.datetime_to_astropy_time(sun_set) # converts into different format (datetime to astropy)
sun_rise = self.observatory.sun_rise_time(Time(date) + TimeDelta(1.0, format='jd')).datetime
# Remove twilight time. Approx 1.5 hour.
# Remove for simulator. But not for the real observing plans.
if remove_twilight:
print 'No observations during twilight.'
sun_set += datetime.timedelta(minutes=90.0)
sun_rise -+ datetime.timedelta(minutes=90.0)
sunset=self.observatory.datetime_to_astropy_time(sun_set) # converts into different format (datetime to astropy)
f=open(params.params['observing_plan_filename']+'_%s.dat'%datestring, 'wb')
# UTC times when we want to observe each tile
times=[]
for i in range(100):
dt = datetime.timedelta(minutes=i*params.params['TIME_PER_TILE'])
t = sun_set + dt
if t < sun_rise:
times.append(t)
else:
break
print 'Selecting tiles for', date
timezone_correction=TimeDelta(3600.0*11.0, format='sec') # TODO
sunset_lt=self.observatory.datetime_to_astropy_time(sun_set) + timezone_correction
sunrise_lt=self.observatory.datetime_to_astropy_time(sun_rise) + timezone_correction
dark_time = sun_rise-sun_set
dark_time = dark_time.seconds
dark_time_hours = int(dark_time/3600.0)
dark_time_minutes = (dark_time - float(dark_time_hours)*3600.0)/60.0
print 'Sunset LT %d-%02d-%02d %02d:%02d'%(sunset_lt.value.year, sunset_lt.value.month, sunset_lt.value.day, sunset_lt.value.hour, sunset_lt.value.minute)
print 'Sunrise LT %d-%02d-%02d %02d:%02d'%(sunrise_lt.value.year, sunrise_lt.value.month, sunrise_lt.value.day, sunrise_lt.value.hour, sunrise_lt.value.minute)
print 'Nighttime duration %02d:%02d'%(dark_time_hours, dark_time_minutes)
print 'Number of tiles this night:', len(times)
print
# Convert UTC times to LST
times_lst = [Time(t).utc.sidereal_time('mean', longitude=params.params['LON']).value for t in times]
time_efficiency=[]
telescope_positions=[] # for testing purposes
selected_tiles=[]
count=1
new_tile_ids_to_be_added_to_list_of_all_observed_stars=set()
for t, lst in zip(times, times_lst):
t_start=datetime.datetime.now()
utc = self.observatory.datetime_to_astropy_time(t)
self.moon = get_moon(utc)
# ONLY BRIGHT TIME
if bright_time:
is_bright_time = self.check_if_moon_is_above_horizon(moon=self.moon, time=utc)
if is_bright_time:
pass
else:
continue
try:
self.ra_current=best_tile.TaipanTile.ra
self.dec_current=best_tile.TaipanTile.dec
except:
self.ra_current=lst
self.dec_current=-70.0 # TODO
self.local_sidereal_time=lst # TODO: check if this violates any other things. Why cant I insert LST to init_best_tiles...??
best_tile = self.find_best_tile()
if best_tile is None:
continue
# Update list of observed tiles
self.observed_tiles.add(best_tile.TaipanTile.field_id)
new_tile_ids_to_be_added_to_list_of_all_observed_stars.add(best_tile.TaipanTile.field_id)
"""
Code from this point on is only output formatting
"""
json_filename = create_obs_config_json.create_ObsConfig_json(tile=best_tile, utc=utc)
local_time=t+datetime.timedelta(hours=11.0)
print '%02d/%02d'%(count, len(times)), 'LT=%d-%02d-%02d %02d:%02d:%02d'%(local_time.year, local_time.month, local_time.day, local_time.hour, local_time.minute, local_time.second), 'UT=%d-%02d-%02d %02d:%02d:%02d'%(times[count-1].year, times[count-1].month, times[count-1].day, times[count-1].hour, times[count-1].minute, times[count-1].second), 'LST=%s'%(('%02.2f'%lst).rjust(5)), best_tile #, best_tile.meridian_transit_time.datetime
# Print out the data
H_amp = best_tile.estimate_best_time_interval_to_observe_tile()
if H_amp is None:
todo=True
# TODO: check if this times are during the night, not e.g. just before sunset or just after sunrise, otherwise limit them within sunset and sunrise time
observing_start = t - datetime.timedelta(hours=H_amp)
observing_stop = t + datetime.timedelta(hours=H_amp)
observing_ideal = Time(t - datetime.timedelta(hours=best_tile.hour_angle)).utc.sidereal_time('mean', longitude=params.params['LON']).value # Do I insert time NOW or time of meridian crossing?
"""
Print output
"""
line='%02d%02d %02d%02d %02d%02d %05d %s'%(t.hour, t.minute, observing_start.hour, observing_start.minute, observing_stop.hour, observing_stop.minute, best_tile.TaipanTile.field_id, json_filename)
#~ self.print_selected_tile_to_json(line)
# TODO: what happens with observing_ideal for tiles at ALT=90? Because there is a limit at 85 degrees.
f.write(line+'\n')
count+=1
t_end=datetime.datetime.now()
time_efficiency.append((t_end-t_start).seconds)
# PLOT
#~ telescope_positions.append([best_tile.TaipanTile.ra, best_tile.TaipanTile.dec, self.moon.ra.value, self.moon.dec.value, best_tile.angular_moon_distance])
#~ visualization.plot_selected_tile_with_neighbourhood(moon=self.moon, lst=lst, best_tiles=self.best_tiles_to_observe_now, tiles=self.tiles, best_tile=best_tile, i=count-1, ra_current=self.ra_current, dec_current=self.dec_current, telescope_positions=telescope_positions, observed_tile_ids=self.observed_tiles)
f.close()
manage_list_of_observed_tiles.add_tile_id_internal_to_the_list(new_tile_ids_to_be_added_to_list_of_all_observed_stars)
print 'Average time to find the next tile: [seconds]', np.mean(time_efficiency)
def main():
prefix = 'M8'
# Keep the two halves of the campsign separate
# Comment or uncomment to do each half:
filenames = glob.glob(
'/Volumes/Work/Field_9/TPFs_part1_superstamp/ktwo200*targ.fits')
# filenames = glob.glob('/Volumes/Work/Field_9/TPFs_part2_superstamp/ktwo200*targ.fits')
midfile = fits.open(filenames[0], mode='readonly', memmap=True)
cards0 = midfile[0].header.cards
cards1 = midfile[1].header.cards
cards2 = midfile[2].header.cards
time = midfile[1].data.field('TIME')[:] + 2454833.0
timecorr = midfile[1].data.field('TIMECORR')[:]
cadenceno = midfile[1].data.field('CADENCENO')[:]
quality = midfile[1].data.field('QUALITY')[:]
cosmic_rays = midfile[1].data.field('COSMIC_RAYS')[:]
pos_corr1 = midfile[1].data.field('POS_CORR1')[:]
pos_corr2 = midfile[1].data.field('POS_CORR2')[:]
bigarr = np.zeros([135, 220])
bigarr[:] = np.nan
bigarr = bigarr.astype('float32')
# Part 1 of Campaign 9
for i in range(0, 669) + range(670, 765) + range(766, 1290):
# Part 2 of Campaign 9
## for i in range(0,2022):
# only continue if there is valid data at this timestamp
if (~np.isnan(time[i])
and len(np.where((midfile[1].data['FLUX'])[i] != 0.0)[0]) != 0
and len(np.where(
(~np.isnan(midfile[1].data['FLUX'])[i]))[0]) != 0):
# construct output primary extension
outfile = prefix + '_BJD%.4f' % time[i] + '.fits'
print outfile
for fn in filenames:
with fits.open(fn) as f:
x = f[1].header['1CRV4P']
y = f[1].header['2CRV4P']
ch = f[0].header['CHANNEL']
mod = f[0].header['MODULE']
out = f[0].header['OUTPUT']
alldata = f[1].data['FLUX']
dim = alldata.shape
if (out == 1):
bigarr[y - 868 + 10:y - 868 + 10 + dim[1],
x - 1002:x - 1002 + dim[2]] = alldata[i]
if (out == 2):
bigarr[y - 858:y - 858 + dim[1], 1112 - (x + dim[2]) +
110:1112 - x + 110] = np.fliplr(alldata[i])
hdu0 = fits.PrimaryHDU(bigarr)
# add in primary keywords
for n in range(len(cards0)):
try:
if cards0[n].keyword not in hdu0.header.keys():
hdu0.header[cards0[n].keyword] = (cards0[n].value,
cards0[n].comment)
else:
hdu0.header.cards[
cards0[n].keyword].comment = cards0[n].comment
except:
pass
# add additional keywords
for k in range(len(cards1)):
if (cards1[k].keyword not in hdu0.header.keys()
and cards1[k].keyword[:4] not in [
'TTYP', 'TFOR', 'TUNI', 'TDIS', 'TDIM', 'WCAX',
'1CTY', '2CTY', '1CRP', '2CRP', '1CRV', '2CRV',
'1CUN', '2CUN', '1CDE', '2CDE', '1CTY', '2CTY',
'1CDL', '2CDL', '11PC', '12PC', '21PC', '22PC',
'WCSN', 'TFIE', 'XTEN', 'EXTN', 'PCOU', 'GCOU',
'TNUL', 'INHE'
]):
hdu0.header.set(cards1[k].keyword, cards1[k].value,
cards1[k].comment)
# and a few more keywords:
for j in range(len(cards2)):
try:
if cards2[j].keyword not in hdu0.header.keys():
hdu0.header.set(cards2[j].keyword, cards2[j].value,
cards2[j].comment)
except:
pass
# pull some additional information out of the TPF headers
try:
int_time = cards1['INT_TIME'].value
except:
print 'WARNING -- cannot find INT_TIME keyword'
try:
frametim = cards1['FRAMETIM'].value
except:
print 'WARNING -- cannot find FRAMETIM keyword'
try:
num_frm = cards1['NUM_FRM'].value
except:
print 'WARNING -- cannot find NUM_FRM keyword'
try:
hdu0.header.set('TELAPSE', frametim * num_frm,
'[s] elapsed time for exposure')
except:
hdu0.header.set('TELAPSE', -999,
'[s] elapsed time for exposure')
try:
hdu0.header.set('LIVETIME', int_time * num_frm,
'[s] TELASPE multiplied by DEADC')
except:
hdu0.header.set('LIVETIME', -999,
'[s] TELASPE multiplied by DEADC')
try:
hdu0.header.set('EXPOSURE', int_time * num_frm,
'[s] time on source')
except:
hdu0.header.set('EXPOSURE', -999, '[s] time on source')
try:
hdu0.header.set('MIDTIME', time[i],
'[BJD] mid-time of exposure')
except:
hdu0.header.set('MIDTIME', -999, '[BJD] mid-time of exposure')
try:
hdu0.header.set('TIMECORR', timecorr[i],
'[d] barycenter - timeslice correction')
except:
hd01.header.set('TIMECORR', -999,
'[d] barycenter - timeslice correction')
try:
hdu0.header.set('CADENCEN', cadenceno[i],
'unique cadence number')
except:
hdu0.header.set('CADENCEN', -999, 'unique cadence number')
try:
hdu0.header.set('QUALITY', quality[i], 'pixel quality flag')
except:
hdu0.header.set('QUALITY', -999, 'pixel quality flag')
try:
pc1 = str(pos_corr1[i])
pc2 = str(pos_corr2[i])
hdu0.header.set('POSCORR1', pc1,
'[pix] column position correction')
hdu0.header.set('POSCORR2', pc2,
'[pix] row position correction')
except:
hdu0.header.set('POSCORR1', -999,
'[pix] column position correction')
hdu0.header.set('POSCORR2', -999,
'[pix] row position correction')
# Edit and delete some keywords
hdu0.header['DATE'] = str(datetime.date.today())
hdu0.header['CREATOR'] = 'Ann Marie Cody'
hdu0.header.cards['CREATOR'].comment = 'file creator'
hdu0.header.set('OBJECT', 'M8', 'target')
hdu0.header['CHANNEL'] = UNDEFINED
hdu0.header.cards[
'CHANNEL'].comment = 'double-valued for this superstamp'
hdu0.header['OUTPUT'] = UNDEFINED
hdu0.header.cards[
'OUTPUT'].comment = 'double-valued for this superstamp'
hdu0.header.remove('PCOUNT')
hdu0.header.remove('GCOUNT')
hdu0.header.remove('PROCVER')
hdu0.header.remove('FILEVER')
hdu0.header.remove('TIMVERSN')
hdu0.header.remove('KEPLERID')
hdu0.header.remove('TTABLEID')
hdu0.header.remove('DATA_REL')
hdu0.header.remove('PMRA')
hdu0.header.remove('PMDEC')
hdu0.header.remove('PMTOTAL')
hdu0.header.remove('PARALLAX')
hdu0.header.remove('GLON')
hdu0.header.remove('GLAT')
hdu0.header.remove('GMAG')
hdu0.header.remove('RMAG')
hdu0.header.remove('IMAG')
hdu0.header.remove('ZMAG')
hdu0.header.remove('JMAG')
hdu0.header.remove('HMAG')
hdu0.header.remove('KMAG')
hdu0.header.remove('KEPMAG')
hdu0.header.remove('GRCOLOR')
hdu0.header.remove('JKCOLOR')
hdu0.header.remove('GKCOLOR')
hdu0.header.remove('TEFF')
hdu0.header.remove('LOGG')
hdu0.header.remove('FEH')
hdu0.header.remove('EBMINUSV')
hdu0.header.remove('AV')
hdu0.header.remove('RADIUS')
hdu0.header.remove('TMINDEX')
hdu0.header.remove('EXTEND')
hdu0.header.remove('NEXTEND')
hdu0.header.remove('EXTVER')
hdu0.header.remove('XTENSION')
hdu0.header.remove('INHERIT')
hdu0.header.remove('NPIXSAP')
hdu0.header.remove('NPIXMISS')
hdu0.header.remove('TIERABSO')
hdu0.header.remove('LC_START')
hdu0.header.remove('LC_END')
hdu0.header.remove('CDPP3_0')
hdu0.header.remove('CDPP6_0')
hdu0.header.remove('CDPP12_0')
hdu0.header.remove('CROWDSAP')
hdu0.header.remove('FLFRCSAP')
# hdu0.header.set('NEXTEND', 1, 'number of extensions')
hdu0.header.set('OBJECT', 'M8', 'target')
hdu0.header[
'TSTART'] = time[i] - frametim / 3600. / 24. / 2. * num_frm
hdu0.header[
'TSTOP'] = time[i] + frametim / 3600. / 24. / 2. * num_frm
hdu0.header.cards[
'TSTART'].comment = 'observation start time in BJD'
hdu0.header.cards[
'TSTOP'].comment = 'observation start time in BJD'
# Calculate TIME-OBS from BJD time:
temptime = Time(time[i] - frametim / 3600. / 24. / 2. * num_frm -
2400000.5 - timecorr[i] +
(0.25 + 0.62 *
(5 - cards1['TIMSLICE'].value)) / 86400.,
format='mjd')
temptime = str(temptime.datetime)
hdu0.header['DATE-OBS'] = temptime.replace(' ', 'T') + 'Z'
# Calculate DATE-END:
temptime = Time(time[i] + frametim / 3600. / 24. / 2. * num_frm -
2400000.5 - timecorr[i] +
(0.25 + 0.62 *
(5 - cards1['TIMSLICE'].value)) / 86400.,
format='mjd')
temptime = str(temptime.datetime)
hdu0.header['DATE-END'] = temptime.replace(' ', 'T') + 'Z'
# write output file
hdu0.writeto(outfile, checksum=True)
def tab2_BUT_tCLN_param_save():
with open(CleanID_dir + 'CASA_CLN_args.json', 'w') as fp:
json.dump(tab2_tCLN_Param_dict, fp)
tab2_Div_tCLN2.text = '<p>CASA script and arguments config file saved to <b>{}</b>.</p>'.format(
CleanID_dir)
timestrs = []
fits_local = []
fits_global = []
if 'twidth' in tab2_tCLN_Param_dict.keys():
val = tab2_tCLN_Param_dict['twidth']
exec('twidth = int({})'.format(val))
else:
twidth = 1
if 'workdir' in tab2_tCLN_Param_dict.keys():
val = tab2_tCLN_Param_dict['workdir']
exec('workdir = {}'.format(val))
else:
workdir = './'
# os.system('cp {} {}'.format(CleanID_dir + 'CASA_CLN_args.json', workdir))
os.system('cp {}/DataBrowser/ToClean/script_clean.py {}'.format(
suncasa_dir, CleanID_dir))
for ii in range(tab2_ntim):
iit = int(ii) / twidth * twidth
t0 = xx[iit] - tab2_dt / 2
datestr = Time(t0 / 3600. / 24.,
format='jd',
scale='utc',
precision=3,
out_subfmt='date').iso
timestr0 = Time(t0 / 3600. / 24.,
format='jd',
scale='utc',
precision=3,
out_subfmt='date_hms').iso
timestr0 = timestr0.split(' ')[1]
timestr = datestr.replace("-", "") + 'T' + timestr0.replace(":", "")
timestrs.append(timestr0)
fits_local.append(timestr + '.fits')
fits_global.append(timestr + '.fits')
timestrs = timestrs * int(tab2_nfreq)
fits_local = fits_local * int(tab2_nfreq)
fits_global = fits_global * int(tab2_nfreq)
freqstrs = ['{:.3f}'.format(ll) for ll in yy]
dspecDFout = pd.DataFrame({
'time': xx - xx[0],
'freq': yy,
'timestr': timestrs,
'freqstr': freqstrs,
'dspec': tab2_spec_plt.flatten(),
'fits_local': fits_local,
'fits_global': fits_global
})
with open(FS_dspecDF, 'wb') as fp:
pickle.dump(dspecDFout, fp)
tab2_Div_tCLN2.text = '<p>CASA script, arguments config file and dspecDF-base saved to <b>{}</b>. '.format(
CleanID_dir
) + 'Click the <b>clean</b> button to clean. When finished, \
go back to <b>QLook</b> window, select StrID <b>{}</b> and \
click <b>FSview</b> button again.</p>'.format(CleanID_dir,
struct_id[0:-1])
def download_cutouts(self, desg=None, clean_failed=True,
retry_failed=True):
import os
from tempfile import mktemp
import numpy as np
import astropy.units as u
from astropy.io import fits
from astropy.time import Time
from astropy.wcs import WCS
from .ztf import IRSA
path = self.config['cutout path'] + os.path.sep
if not os.path.exists(path):
os.system('mkdir ' + path)
fntemplate = os.path.join(
'{desg}', '{desg}-{datetime}-{prepost}{rh:.3f}-ztf.fits.gz')
if desg is None:
desg_constraint = ''
parameters = []
else:
desg_constraint = ' AND desg=? '
parameters = [desg]
if retry_failed:
sync_constraint = ''
else:
sync_constraint = 'AND sci_sync_date IS NULL '
count = self.db.execute('''
SELECT count() FROM found
WHERE sciimg=0
''' + sync_constraint + desg_constraint, parameters
).fetchone()[0]
if count == 0:
self.logger.info('No cutouts to download.')
return
self.logger.info('Downloading {} cutouts.'.format(count))
rows = self.fetch_iter('''
SELECT * FROM foundobs
WHERE sciimg=0
''' + sync_constraint + '''
''' + desg_constraint, parameters)
with IRSA(path, self.config.auth) as irsa:
for row in rows:
# check if target cutout directory exists
d = desg2file(row['desg'])
if not os.path.exists(path + d):
os.system('mkdir ' + path + d)
prepost = 'pre' if row['rdot'] < 0 else 'post'
sync_date = Time(float(row['obsjd']), format='jd').iso
t = sync_date.replace('-', '').replace(
':', '').replace(' ', '_')[:15]
fn = fntemplate.format(
desg=d, prepost=prepost, rh=row['rh'],
datetime=t)
if os.path.exists(path + fn):
self.logger.error(
path + fn +
' exists, but was not expected. Removing.'
)
os.unlink(path + fn)
sciurl = row['url'] + '&size=5arcmin'
sci_downloaded = self._download_file(
irsa, sciurl, path + fn, clean_failed=clean_failed)
if not sci_downloaded:
self.db.execute('''
UPDATE found SET
sci_sync_date=?,
sciimg=0,
mskimg=0,
scipsf=0,
diffimg=0,
diffpsf=0
WHERE foundid=?
''', (sync_date, row['foundid']))
self.db.commit()
continue
updates = {
'desg': (row['desg'], 'Target designation'),
'obsjd': (row['obsjd'], 'Shutter start time'),
'rh': (row['rh'], 'Heliocentric distance, au'),
'delta': (row['delta'], 'Observer-target distance, au'),
'phase': (row['phase'], 'Sun-target-observer angle, deg'),
'rdot': (row['rdot'], 'Heliocentric radial velocity, km/s'),
'selong': (row['selong'], 'Solar elongation, deg'),
'sangle': (row['sangle'], 'Projected target->Sun position angle, deg'),
'vangle': (row['vangle'], 'Projected velocity position angle, deg'),
'trueanom': (row['trueanomaly'], 'True anomaly (osculating), deg'),
'tmtp': (row['tmtp'], 'T-Tp (osculating), days'),
'tgtra': (row['ra'], 'Target RA, deg'),
'tgtdec': (row['dec'], 'Target Dec, deg'),
'tgtdra': (row['dra'], 'Target RA*cos(dec) rate of change, arcsec/s'),
'tgtddec': (row['ddec'], 'Target Dec rate of change, arcsec/s'),
'tgtrasig': (row['ra3sig'], 'Target RA 3-sigma uncertainty, arcsec'),
'tgtdesig': (row['dec3sig'], 'Target Dec 3-sigma uncertainty, arcsec'),
'foundid': (row['foundid'], 'ZChecker DB foundid'),
}
maskfn = mktemp(dir='/tmp')
_url = sciurl.replace('sciimg', 'mskimg')
mask_downloaded = self._download_file(
irsa, _url, maskfn, clean_failed=clean_failed)
psffn = mktemp(dir='/tmp')
_url = sciurl.replace('sciimg', 'sciimgdaopsfcent')
_url = _url[:_url.rfind('?')]
psf_downloaded = self._download_file(
irsa, _url, psffn, clean_failed=True)
difffn = mktemp(dir='/tmp')
#_url = sciurl.replace('sciimg.fits', 'scimrefdiffimg.fits.fz')
# diff_downloaded = self._download_file(
# irsa, _url, difffn, clean_failed=True)
diff_downloaded = False
diffpsffn = mktemp(dir='/tmp')
#_url = sciurl.replace('sciimg', 'diffimgpsf')
# if diff_downloaded: # no need to DL PSF if diff not DL'ed
# diffpsf_downloaded = self._download_file(
# irsa, _url, diffpsffn, clean_failed=True)
# else:
# diffpsf_downloaded = False
diffpsf_downloaded = False
# update header and add mask and PSF
with fits.open(path + fn, 'update') as hdu:
hdu[0].name = 'sci'
wcs = WCS(hdu[0].header)
x, y = wcs.all_world2pix(
row['ra'] * u.deg, row['dec'] * u.deg, 0)
updates['tgtx'] = int(
x), 'Target x coordinate, 0-based'
updates['tgty'] = int(
y), 'Target y coordinate, 0-based'
try:
hdu[0].header.update(updates)
except ValueError as e:
self.logger.error('Error creating FITS header for foundid {}: {}'.format(row['foundid'], str(e)))
hdu.close()
continue
if mask_downloaded:
with fits.open(maskfn) as mask:
mask[0].name = 'mask'
hdu.append(mask[0])
if psf_downloaded:
with fits.open(psffn) as psf:
psf[0].name = 'psf'
hdu.append(psf[0])
if diff_downloaded:
with fits.open(difffn) as diff:
diff[0].name = 'diff'
hdu.append(psf[0])
if diffpsf_downloaded:
with fits.open(diffpsffn) as diffpsf:
diffpsf[0].name = 'diff_psf'
hdu.append(psf[0])
for f in (maskfn, psffn, difffn, diffpsffn):
if os.path.exists(f):
os.unlink(f)
self.db.execute('''
UPDATE found SET
archivefile=?,
sci_sync_date=?,
sciimg=?,
mskimg=?,
scipsf=?,
diffimg=?,
diffpsf=?
WHERE foundid=?
''', (fn, sync_date, sci_downloaded, mask_downloaded,
psf_downloaded, diff_downloaded, diffpsf_downloaded,
row['foundid']))
self.db.commit()
self.logger.info(' [{}] {}'.format(
count, os.path.basename(fn)))
count -= 1
