def read(self, max_events=None):
"""
Read the file using the appropriate method depending on the file origin
Parameters
----------
max_events : int
Maximum number of events to read
Returns
-------
source : generator
A generator that can be iterated over to obtain events
"""
# Obtain relevent source
log.debug("[file] Reading file...")
if max_events:
log.info("[file] Max events being read = {}".format(max_events))
switch = {
'hessio':
lambda: hessio_event_source(get_path(self.input_path),
max_events=max_events),
'targetio':
lambda: targetio_source(self.input_path,
max_events=max_events),
}
try:
source = switch[self.origin]()
except KeyError:
log.exception("unknown file origin '{}'".format(self.origin))
raise
log.debug("[file] Reading complete")
return source
def to_region(self):
"""
Converts to region, ``regions.Region`` object
"""
coords = self.convert_coords()
log.debug(coords)
viz_keywords = ['color', 'dash', 'dashlist', 'width', 'font', 'symsize',
'symbol', 'symsize', 'fontsize', 'fontstyle', 'usetex',
'labelpos', 'labeloff', 'linewidth', 'linestyle',
'point', 'textangle', 'fontweight']
if isinstance(coords[0], SkyCoord):
reg = self.shape_to_sky_region[self.region_type](*coords)
elif isinstance(coords[0], PixCoord):
reg = self.shape_to_pixel_region[self.region_type](*coords)
else:
self._raise_error("No central coordinate")
reg.visual = RegionVisual()
reg.meta = RegionMeta()
# both 'text' and 'label' should be set to the same value, where we
# default to the 'text' value since that is the one used by ds9 regions
label = self.meta.get('text',
self.meta.get('label', ""))
if label != '':
reg.meta['label'] = label
for key in self.meta:
if key in viz_keywords:
reg.visual[key] = self.meta[key]
else:
reg.meta[key] = self.meta[key]
reg.meta['include'] = self.include
return reg
def __call__(self, *args, **kwargs):
log.debug("Fitter called with args={0} and kwargs={1}".format(args, kwargs))
use_lmfit = kwargs.pop('use_lmfit') if 'use_lmfit' in kwargs else self.use_lmfit
if use_lmfit:
return self.lmfitter(*args,**kwargs)
return self.fitter(*args,**kwargs)
def to_region(self):
"""
Converts to region, ``regions.Region`` object
"""
coords = self.convert_coords()
log.debug(coords)
viz_keywords = ['color', 'dash', 'dashlist', 'width', 'font', 'symsize',
'symsize', 'fontsize', 'fontstyle', 'usetex',
'labelpos', 'labeloff', 'linewidth', 'linestyle']
if isinstance(coords[0], BaseCoordinateFrame):
reg = self.shape_to_sky_region[self.region_type](*coords)
elif isinstance(coords[0], PixCoord):
reg = self.shape_to_pixel_region[self.region_type](*coords)
else:
self._raise_error("No central coordinate")
reg.visual = RegionVisual()
reg.meta = RegionMeta()
for key in self.meta.keys():
if key in viz_keywords:
reg.visual[key] = self.meta[key]
else:
reg.meta[key] = self.meta[key]
reg.meta['include'] = self.include
return reg
def status(XCov_filename):
ll_name = "xd_log_likelihood"
with h5py.File(XCov_filename, mode='r') as f:
if ll_name not in f['search']:
logger.info("0 done")
return
ll = f['search'][ll_name]
ndone = np.isfinite(ll).sum()
nnot = np.isnan(ll).sum()
logger.info("{} done, {} not done".format(ndone, nnot))
# check what blocks are unfinished
if nnot != 0:
idx, = np.where(np.isnan(ll))
diff = idx[1:]-idx[:-1]
derp, = np.where(diff > 1)
if 0 not in derp:
derp = np.concatenate(([0], derp, [len(idx)-1]))
logger.debug("Unfinished blocks:")
blocks = []
for d1,d2 in zip(derp[:-1],derp[1:]):
if d1 == 0:
blocks.append("{}-{}".format(idx[d1], idx[d2]))
else:
blocks.append("{}-{}".format(idx[d1+1], idx[d2]))
logger.debug(", ".join(blocks))
def _fast_reader(index_map, data):
"""
Use scipy.ndimage.find_objects to quickly identify subsets of the data
to increase speed of dendrogram loading
"""
flux_by_structure, indices_by_structure = {},{}
from scipy import ndimage
idxs = np.unique(index_map[index_map > -1])
# ndimage ignores 0 and -1, but we want index 0
object_slices = ndimage.find_objects(index_map+1)
index_cube = np.indices(index_map.shape)
# Need to have same length, otherwise assumptions above are wrong
assert len(idxs) == len(object_slices)
log.debug('Creating index maps for {0} indices...'.format(len(idxs)))
for idx,sl in ProgressBar(zip(idxs, object_slices)):
match = index_map[sl] == idx
sl2 = (slice(None),) + sl
match_inds = index_cube[sl2][:, match]
coords = list(zip(*match_inds))
dd = data[sl][match].tolist()
flux_by_structure[idx] = dd
indices_by_structure[idx] = coords
return flux_by_structure, indices_by_structure
def get_event(self, event_req, id_flag=False):
"""
Loop through events until the requested event is found
Parameters
----------
event_req : int
Event index requested
id_flag : bool
'event_req' refers to event_id instead of event_index
Returns
-------
event : `ctapipe` event-container
"""
if not id_flag:
log.info("[file][read] Finding event index {}...".format(event_req))
else:
log.info("[file][read] Finding event id {}...".format(event_req))
source = self.read()
for event in source:
event_id = event.dl0.event_id
index = event.count if not id_flag else event_id
if not index == event_req:
log.debug("[event_id] skipping event: {}".format(event_id))
continue
log.info("[file] Event {} found".format(event_req))
return event
log.info("[file][read] Event does not exist!")
return None
def button2action(self, event=None, debug=False, subtract=True,
powerlaw=None, fit_original=False,
spline=False,
spline_sampling=None,
spline_downsampler=np.median,
baseline_fit_color='orange', **kwargs):
"""
Do the baseline fitting and save and plot the results.
"""
if debug: print "Button 2/3 Baseline. Subtract=",subtract
if self.subtracted:
self.unsubtract()
# interactive guesspeakwidth passes this on; it should be excised
if 'nwidths' in kwargs:
kwargs.pop('nwidths')
if powerlaw is None:
powerlaw = self.powerlaw
self.clear_highlights()
self._xfit_units = self.Spectrum.xarr.unit
log.debug("Fitting baseline: powerlaw={0} "
"spline={1}".format(self.powerlaw, spline))
self.fit(powerlaw=powerlaw, includemask=self.includemask,
order=self.order, spline=spline,
spline_sampling=spline_sampling,
spline_downsampler=spline_downsampler)
if subtract:
if self.subtracted and fit_original:
# use the spectrum with the old baseline added in (that's what we fit to)
self.Spectrum.data = self.spectofit - self.basespec
else:
self.Spectrum.data -= self.basespec
self.subtracted = True
else:
if self.subtracted:
self.unsubtract()
self.subtracted = False
if self.Spectrum.plotter.axis is not None:
if debug: print "Plotting baseline"
if event is not None:
# preserve frame if fitting interactively
kwargs.update({'use_window_limits':True})
self.plot_baseline(baseline_fit_color=baseline_fit_color, **kwargs)
# disconnect interactive window (and more importantly, reconnect to
# original interactive cmds)
self.clear_all_connections()
if hasattr(self.Spectrum,'header'):
history.write_history(self.Spectrum.header,
"BASELINE order=%i pars=%s" % (self.order,
",".join([str(s) for s in self.baselinepars])) +
"(powerlaw)" if self.powerlaw else "")
def get_guesses(index, catalog, max_comp=3):
"""
put the guesses into flat format and limit their numbers
"""
amp,vc,vsig,r = catalog['Smean303', 'v_cen', 'v_rms',
'r303321'][index].columns.values()
keep_velo = (vsig < 10) & (vsig > 1)
if np.count_nonzero(keep_velo) > max_comp:
# Also exclude small objects when there are many overlaps
big_objs = catalog[index]['npix'] > 100
# but only if excluding them isn't overly restrictive
if np.count_nonzero(big_objs & keep_velo) >= max_comp:
keep_velo &= big_objs
keep_inds = np.argsort(amp[keep_velo])[-max_comp:]
log.debug('Kept {0}, amps {1}'.format(np.array(index)[keep_velo][keep_inds],
np.array(amp)[keep_velo][keep_inds]))
amp,vc,vsig,r = [a[keep_velo][keep_inds] for a in (amp,vc,vsig,r)]
glon = [xc - 360 if xc > 180 else xc
for xc in catalog['x_cen']]
# TODO: make sure the velocity units remain consistent
# The logic here is to filter out things at vlsr<-50 km/s that are not in Sgr C;
# these are mostly other lines in Sgr B2
result = [pars
for pars,glon in zip(zip(*[x.data for x in [amp,vc/1e3,vsig,r,amp]]),
glon)
]
return result
def annotations(self, shortvarnames=None, debug=False):
"""
Return a list of TeX-formatted labels
The values and errors are formatted so that only the significant digits
are displayed. Rounding is performed using the decimal package.
Parameters
----------
shortvarnames : list
A list of variable names (tex is allowed) to include in the
annotations. Defaults to self.shortvarnames
Examples
--------
>>> # Annotate a Gaussian
>>> sp.specfit.annotate(shortvarnames=['A','\\Delta x','\\sigma'])
"""
from decimal import Decimal # for formatting
svn = self.shortvarnames if shortvarnames is None else shortvarnames
# if pars need to be replicated....
if len(svn) < self.npeaks * self.npars:
svn = svn * self.npeaks
parvals = self.parinfo.values
parerrs = self.parinfo.errors
loop_list = [
(
parvals[ii + jj * self.npars + self.vheight],
parerrs[ii + jj * self.npars + self.vheight],
svn[ii + jj * self.npars],
self.parinfo.fixed[ii + jj * self.npars + self.vheight],
jj,
)
for jj in range(self.npeaks)
for ii in range(self.npars)
]
label_list = []
for (value, error, varname, fixed, varnumber) in loop_list:
log.debug(", ".join([str(x) for x in (value, error, varname, fixed, varnumber)]))
if fixed or error == 0:
label = "$%s(%i)$=%8s" % (
varname,
varnumber,
Decimal("%g" % value).quantize(Decimal("%0.6g" % (value))),
)
else:
label = "$%s(%i)$=%8s $\\pm$ %8s" % (
varname,
varnumber,
Decimal("%g" % value).quantize(Decimal("%0.2g" % (min(np.abs([value, error]))))),
Decimal("%g" % error).quantize(Decimal("%0.2g" % (error))),
)
label_list.append(label)
labels = tuple(mpcb.flatten(label_list))
return labels
def solar_system_shapiro_delay(self, toas, acc_delay=None):
"""
Returns total shapiro delay to due solar system objects.
If the PLANET_SHAPIRO model param is set to True then
planets are included, otherwise only the value for the
Sun is calculated.
Requires Astrometry or similar model that provides the
ssb_to_psb_xyz method for direction to pulsar.
If planets are to be included, TOAs.compute_posvels() must
have been called with the planets=True argument.
"""
# Start out with 0 delay with units of seconds
tbl = toas.table
delay = numpy.zeros(len(tbl))
for ii, key in enumerate(tbl.groups.keys):
grp = tbl.groups[ii]
obs = tbl.groups.keys[ii]['obs']
loind, hiind = tbl.groups.indices[ii:ii+2]
if key['obs'].lower() == 'barycenter':
log.debug("Skipping Shapiro delay for Barycentric TOAs")
continue
psr_dir = self.ssb_to_psb_xyz_ICRS(epoch=grp['tdbld'].astype(numpy.float64))
delay[loind:hiind] += self.ss_obj_shapiro_delay(grp['obs_sun_pos'],
psr_dir, self._ss_mass_sec['sun'])
if self.PLANET_SHAPIRO.value:
for pl in ('jupiter', 'saturn', 'venus', 'uranus'):
delay[loind:hiind] += self.ss_obj_shapiro_delay(grp['obs_'+pl+'_pos'],
psr_dir, self._ss_mass_sec[pl])
return delay * u.second
def containment_radius_image(self, fraction):
"""Compute containment radius image.
Parameters
----------
fraction : float
Containment fraction
Returns
-------
image : `numpy.ndarray`
Containment radius image
"""
out = np.zeros(self.shape, dtype=float)
npix = self.size
for ii in range(npix):
if (100 * ii) % npix == 0:
percent = 100. * ii / npix
log.debug('Processing pixel {ii:5d} of {npix:5d} ({percent:5.2f}%)'
''.format(**locals()))
try:
psf = self._get_psf(ii)
out.flat[ii] = psf.containment_radius(fraction)
except ValueError:
# This is what happens to pixels in the map without PSF info
out.flat[ii] = np.nan
return out
def test(self):
for n in range(self.N):
logger.debug("Orbit {}".format(n))
# x,v = self.w[:3,:,n],self.w[3:,:,n]
x = self.w.pos.value[...,n]
v = self.w.vel.value[...,n]
actions,angles,freqs = isochrone_xv_to_aa(x, v, self.potential)
for i in range(3):
assert np.allclose(actions[i,1:], actions[i,0], rtol=1E-5)
# Compare to genfunc
s_v = (v*u.kpc/u.Myr).to(u.km/u.s).value
s_w = np.vstack((x,s_v))
m = self.potential.parameters['m'] / 1E11
b = self.potential.parameters['b']
aa = np.array([toy_potentials.angact_iso(s_w[:,i].T, params=(m,b)) for i in range(s_w.shape[1])])
s_actions = (aa[:,:3]*u.km/u.s*u.kpc).decompose(galactic).value
s_angles = aa[:,3:]
assert np.allclose(actions, s_actions.T, rtol=1E-8)
assert_angles_allclose(angles, s_angles.T, rtol=1E-8)
# test roundtrip
x2,v2 = isochrone_aa_to_xv(actions, angles, self.potential)
assert np.allclose(x, x2, rtol=1E-8)
assert np.allclose(v, v2, rtol=1E-8)
def get_charge_resolution(self):
"""
Calculate and obtain the charge resolution graph arrays.
Returns
-------
true_charge : ndarray
The X axis true charges.
chargeres : ndarray
The Y axis charge resolution values.
chargeres_error : ndarray
The error on the charge resolution.
scaled_chargeres : ndarray
The Y axis charge resolution divided by the Goal.
scaled_chargeres_error : ndarray
The error on the charge resolution divided by the Goal.
"""
log.debug('[chargeres] Calculating charge resolution')
true_charge = np.fromiter(iter(self.sum_dict.keys()), dtype=int)
summed_charge = np.fromiter(iter(self.sum_dict.values()), dtype=float)
num = np.fromiter(iter(self.n_dict.values()), dtype=int)
chargeres = np.sqrt((summed_charge / num) + true_charge) / true_charge
chargeres_error = chargeres * (1 / np.sqrt(2 * num))
scale = self.goal(true_charge)
scaled_chargeres = chargeres/scale
scaled_chargeres_error = chargeres_error/scale
return true_charge, chargeres, chargeres_error, \
scaled_chargeres, scaled_chargeres_error
def test(self):
"""
!!!!! NOTE !!!!!
For Harmonic Oscillator, Sanders' code works for the units I use...
"""
for n in range(self.N):
logger.debug("Orbit {}".format(n))
# x,v = self.w[:3,:,n],self.w[3:,:,n]
x = self.w.pos.value[...,n]
v = self.w.vel.value[...,n]
actions,angles = harmonic_oscillator_xv_to_aa(x, v, self.potential)
for i in range(3):
assert np.allclose(actions[i,1:], actions[i,0], rtol=1E-5)
# Compare to genfunc
s_w = np.vstack((x,v))
omega = self.potential.parameters['omega']
aa = np.array([toy_potentials.angact_ho(s_w[:,i].T, omega=omega) for i in range(s_w.shape[1])])
s_actions = aa[:,:3]
s_angles = aa[:,3:]
assert np.allclose(actions, s_actions.T, rtol=1E-8)
assert_angles_allclose(angles, s_angles.T, rtol=1E-8)
def to_table(container):
"""
Convert a `ctapipe.core.Container` to an `astropy.Table` with one row
Parameters
----------
container: ctapipe.core.Container
Returns
-------
Table: astropy.Table
"""
names = list()
columns = list()
for k, v in writeable_items(container).items():
v_arr = np.array(v)
v_arr = v_arr.reshape((1,) + v_arr.shape)
log.debug("Creating column for item '{0}' of shape {1}".
format(k, v_arr.shape))
names.append(k)
columns.append(Column(v_arr))
return Table(data=columns, # dtypes are inferred by columns
names=names,
meta=container.meta)
def test(self):
"""
!!!!! NOTE !!!!!
For Harmonic Oscillator, Sanders' code works for the units I use...
"""
for n in range(self.N):
logger.debug("Orbit {}".format(n))
x,v = self.w[:,n,:3],self.w[:,n,3:]
ww = self.w[:,n]
actions,angles = harmonic_oscillator_xv_to_aa(x, v, self.potential)
for i in range(3):
assert np.allclose(actions[1:,i], actions[0,i], rtol=1E-5)
# Compare to genfunc
omega = self.potential.parameters['omega']
aa = np.array([toy_potentials.angact_ho(w, omega=omega) for w in ww])
s_actions = aa[:,:3]
s_angles = aa[:,3:]
assert np.allclose(actions, s_actions, rtol=1E-8)
assert np.allclose(angles, s_angles, rtol=1E-8)
# test roundtrip
# x2,v2 = harmonic_oscillator_aa_to_xv(actions, angles, self.potential)
# TODO: figure out transform back
continue
rel_err_x = np.abs((x2 - x) / x)
rel_err_v = np.abs((v2 - v) / v)
assert rel_err_x.max() < (1E-8)
assert rel_err_v.max() < (1E-8)
def _run_wrapper(self, index):
logger.info("Orbit {0}".format(index))
# unpack input argument dictionary
import gary.potential as gp
potential = gp.load(os.path.join(self.cache_path, self.config.potential_filename))
# read out just this initial condition
norbits = len(self.w0)
allfreqs = np.memmap(self.cache_file, mode='r',
shape=(norbits,), dtype=self.cache_dtype)
# short-circuit if this orbit is already done
if allfreqs['success'][index]:
logger.debug("Orbit {0} already successfully completed.".format(index))
return None
# Only pass in things specified in _run_kwargs (w0 and potential required)
kwargs = dict([(k,self.config[k]) for k in self.config.keys() if k in self._run_kwargs])
res = self.run(w0=self.w0[index], potential=potential, **kwargs)
res['index'] = index
# cache res into a tempfile, return name of tempfile
tmpfile = os.path.join(self._tmpdir, "{0}-{1}.pickle".format(self.__class__.__name__, index))
with open(tmpfile, 'w') as f:
pickle.dump(res, f)
return tmpfile
def test_actions(self):
# t = self.t[::10]
t = self.t
N_max = 6
for n in range(self.N):
print("\n\n")
logger.info("======================= Orbit {} =======================".format(n))
# w = self.w[:,::10,n]
w = self.w[...,n]
orb = self.orbit[:,n]
circ = orb.circulation()
# get values from Sanders' code
logger.debug("Computing actions from genfunc...")
s_actions,s_angles,s_freqs,toy_potential = sanders_act_ang_freq(t, w, circ, N_max=N_max)
logger.debug("Computing actions...")
ret = find_actions(orb, N_max=N_max, toy_potential=toy_potential)
actions = ret['actions']
angles = ret['angles']
freqs = ret['freqs']
logger.info("Action ratio: {}".format(actions / s_actions))
logger.info("Angle ratio: {}".format(angles / s_angles))
logger.info("Freq ratio: {}".format(freqs / s_freqs))
assert np.allclose(actions.value, s_actions, rtol=1E-5)
assert np.allclose(angles.value, s_angles, rtol=1E-5)
assert np.allclose(freqs.value, s_freqs, rtol=1E-5)
def make_action_files(t, w, potential, suffix="", overwrite=False,
force_harmonic_oscillator=False, N_max=6):
action_filename = os.path.join(plot_path, "actions{}.npy".format(suffix))
if overwrite and os.path.exists(action_filename):
os.remove(action_filename)
if not os.path.exists(action_filename):
# compute the actions and angles for the orbit
actions,angles,freqs = sd.cross_validate_actions(t, w[:,0], N_max=N_max, nbins=100,
force_harmonic_oscillator=force_harmonic_oscillator,
units=potential.units, skip_failures=True,
overlap=0) #w.shape[0]//100)
# now compute for the full time series
r = sd.find_actions(t, w[:,0], N_max=N_max, units=potential.units, return_Sn=True,
force_harmonic_oscillator=force_harmonic_oscillator)
full_actions,full_angles,full_freqs = r[:3]
Sn,dSn_dJ,nvecs = r[3:]
np.save(action_filename, (actions,angles,freqs) + r)
logger.debug("Actions computed and saved to file: {}".format(action_filename))
else:
r = np.load(action_filename)
actions,angles,freqs = r[:3]
full_actions,full_angles,full_freqs = r[3:6]
Sn,dSn_dJ,nvecs = r[6:]
logger.debug("Actions read from file: {}".format(action_filename))
return actions,angles,freqs,full_actions,full_angles,full_freqs
def test(self):
for n in range(self.N):
logger.debug("Orbit {}".format(n))
x,v = self.w[:,n,:3],self.w[:,n,3:]
s_v = (v*u.kpc/u.Myr).to(u.km/u.s).value
s_w = np.hstack((x,s_v))
actions,angles = isochrone_xv_to_aa(x, v, self.potential)
for i in range(3):
assert np.allclose(actions[1:,i], actions[0,i], rtol=1E-5)
# Compare to genfunc
m = self.potential.parameters['m'] / 1E11
b = self.potential.parameters['b']
aa = np.array([toy_potentials.angact_iso(w, params=(m,b)) for w in s_w])
s_actions = (aa[:,:3]*u.km/u.s*u.kpc).decompose(galactic).value
s_angles = aa[:,3:]
assert np.allclose(actions, s_actions, rtol=1E-8)
assert np.allclose(angles, s_angles, rtol=1E-8)
# test roundtrip
x2,v2 = isochrone_aa_to_xv(actions, angles, self.potential)
rel_err_x = np.abs((x2 - x) / x)
rel_err_v = np.abs((v2 - v) / v)
assert rel_err_x.max() < (1E-8)
assert rel_err_v.max() < (1E-8)
def _matplotlib_pil_bug_present():
"""
Determine whether PIL images should be pre-flipped due to a bug in Matplotlib.
Prior to Matplotlib 1.2.0, RGB images provided as PIL objects were
oriented wrongly. This function tests whether the bug is present.
"""
from matplotlib.image import pil_to_array
try:
from PIL import Image
except:
import Image
from astropy import log
array1 = np.array([[1, 2], [3, 4]], dtype=np.uint8)
image = Image.fromarray(array1)
array2 = pil_to_array(image)
if np.all(array1 == array2):
log.debug("PIL Image flipping bug not present in Matplotlib")
return False
elif np.all(array1 == array2[::-1, :]):
log.debug("PIL Image flipping bug detected in Matplotlib")
return True
else:
log.warn("Could not properly determine Matplotlib behavior for RGB images - image may be flipped incorrectly")
return False
def main(iso_filename, XCov_filename, interpolate=True, overwrite=False):
# FOR PARSEC ISOCHRONE (reversing it for interpolation)
iso = ascii.read(iso_filename, header_start=13)[:114][::-1]
iso = nprf.stack_arrays((iso[:25], iso[27:]),usemask=False) # because of stupid red clump turnaround
# FOR DARTMOUTH ISOCHRONE (reversing it for interpolation)
# iso = ascii.read(iso_filename, header_start=8)[::-1]
# output hdf5 file
with h5py.File(XCov_filename, mode='r+') as f:
# feature and covariance matrices for all stars
X = ps1_isoc_to_XCov(iso, W=mixing_matrix, interpolate=interpolate)
if 'isochrone' in f and overwrite:
f.__delitem__('isochrone')
logger.debug("Overwriting isochrone data")
if 'isochrone' not in f:
g = f.create_group('isochrone')
else:
g = f['isochrone']
if 'X' not in f['isochrone']:
g.create_dataset('X', X.shape, dtype='f', data=X)
f.flush()
logger.debug("Saved isochrone to {}".format(XCov_filename))
def _load_kernel_link(ephem, link=''):
load_kernel = False # a flag for checking if the kernel has been loaded
# search_list = [link,jpl_kernel_http, jpl_kernel_ftp]
# NOTE the JPL ftp site is disabled. Instead, we duplicated the JPL ftp
# site on nanograv server.
search_list = [link, nanograv_http, jpl_kernel_http, jpl_kernel_ftp]
if link != '':
search_list.append('')
for l in search_list:
if l == '':
ephem_link = ephem # Astropy default ephem does not like .bsp in the end.
else:
ephem_link = l+"%s.bsp" % ephem
if load_kernel:
break
try:
log.debug('Trying to set astropy ephemeris to {0}'.format(ephem_link))
coor.solar_system_ephemeris.set(ephem_link)
load_kernel = True
except Exception as ex:
#log.info('Exception! {0} {1} {2}'.format(type(ex), ex.args, ex))
try:
log.debug('Trying to download and set astropy ephemeris to {0}'.format(ephem_link))
aut.data.download_file(ephem_link, timeout=300, cache=True)
coor.solar_system_ephemeris.set(ephem_link)
load_kernel = True
except Exception as ex2:
#log.info('Exception2! {0} {1} {2}'.format(type(ex2), ex2.args, ex2))
load_kernel = False
return load_kernel
def cleanup_git_directory(directory='database/', delete_untracked=False,
allow_fail=True):
uncommitted_files = subprocess.call(['git', 'diff-files', '--quiet'],
cwd=directory)
if uncommitted_files:
log.debug("Found uncommitted file changes in {0}.".format(directory))
reset = subprocess.call(['git', 'reset', '--hard', 'HEAD'],
cwd=directory)
if allow_fail:
assert reset == 0
if delete_untracked:
untracked_deleted = subprocess.call(['git', 'clean', '-f'],
cwd=directory)
if allow_fail:
assert untracked_deleted == 0
uncommited_staged_changes = \
subprocess.call(['git', 'diff-index', '--quiet', '--cached', 'HEAD'],
cwd=directory)
if uncommited_staged_changes:
log.debug("Found uncommitted, staged changes in {0}."
.format(directory))
reset = subprocess.call(['git', 'reset', '--hard', 'HEAD'],
cwd=directory)
if allow_fail:
assert reset == 0
def _slow_reader(index_map, data):
"""
Loop over each valid pixel in the index_map and add its coordinates and
data to the flux_by_structure and indices_by_structure dicts
This is slower than _fast_reader but faster than that implementation would
be without find_objects. The bottleneck is doing `index_map == idx` N
times.
"""
flux_by_structure, indices_by_structure = {}, {}
# Do a fast iteration through d.data, adding the indices and data values
# to the two dictionaries declared above:
indices = np.array(np.where(index_map > -1)).transpose()
log.debug("Creating index maps for {0} coordinates...".format(len(indices)))
for coord in ProgressBar(indices):
coord = tuple(coord)
idx = index_map[coord]
if idx in flux_by_structure:
flux_by_structure[idx].append(data[coord])
indices_by_structure[idx].append(coord)
else:
flux_by_structure[idx] = [data[coord]]
indices_by_structure[idx] = [coord]
return flux_by_structure, indices_by_structure
def __init__(self, name, FPorbname, tt2tdb_mode = 'pint'):
if FPorbname.startswith('@'):
# Read multiple orbit files names
FPlist = []
fnames = [ll.strip() for ll in open(FPorbname[1:]).readlines()]
for fn in fnames:
FPlist.append(load_FPorbit(fn))
self.FPorb = vstack(FPlist)
# Make sure full table is sorted
self.FPorb.sort('MJD_TT')
else:
self.FPorb = load_FPorbit(FPorbname)
# Now build the interpolator here:
self.X = InterpolatedUnivariateSpline(self.FPorb['MJD_TT'],self.FPorb['X'])
self.Y = InterpolatedUnivariateSpline(self.FPorb['MJD_TT'],self.FPorb['Y'])
self.Z = InterpolatedUnivariateSpline(self.FPorb['MJD_TT'],self.FPorb['Z'])
self.Vx = InterpolatedUnivariateSpline(self.FPorb['MJD_TT'],self.FPorb['Vx'])
self.Vy = InterpolatedUnivariateSpline(self.FPorb['MJD_TT'],self.FPorb['Vy'])
self.Vz = InterpolatedUnivariateSpline(self.FPorb['MJD_TT'],self.FPorb['Vz'])
super(NICERObs, self).__init__(name=name, tt2tdb_mode=tt2tdb_mode)
# Print this warning once, mainly for @paulray
if self.tt2tdb_mode.lower().startswith('pint'):
log.debug('Using location=None for TT to TDB conversion (pint mode)')
elif self.tt2tdb_mode.lower().startswith('geo'):
log.warning('Using location geocenter for TT to TDB conversion')
def main(argv=None):
parser = argparse.ArgumentParser(description="PINT tool for command-line barycentering calculations.")
parser.add_argument("time",help="MJD (UTC, by default)")
parser.add_argument("--timescale",default="utc",
help="Time scale for MJD argument ('utc', 'tt', 'tdb'), default=utc")
parser.add_argument("--format",
help="Format for time argument ('mjd' or any astropy.Time format (e.g. 'isot'), see <http://docs.astropy.org/en/stable/time/#time-format>)",
default="mjd")
parser.add_argument("--freq",type=float,default=np.inf,
help="Frequency to use, MHz")
parser.add_argument("--obs",default="Geocenter",
help="Observatory code (default = Geocenter)")
parser.add_argument("--parfile",help="par file to read model from",default=None)
parser.add_argument("--ra",
help="RA to use (e.g. '12h22m33.2s' if not read from par file)")
parser.add_argument("--dec",
help="Decl. to use (e.g. '19d21m44.2s' if not read from par file)")
parser.add_argument("--dm",
help="DM to use (if not read from par file)",type=float,default=0.0)
parser.add_argument("--ephem",default="DE421",help="Ephemeris to use")
parser.add_argument("--use_gps",default=False,action='store_true',help="Apply GPS to UTC clock corrections")
parser.add_argument("--use_bipm",default=False,action='store_true',help="Use TT(BIPM) instead of TT(TAI)")
args = parser.parse_args(argv)
if args.format in ("mjd","jd", "unix"):
# These formats require conversion from string to longdouble first
fmt = args.format
# Never allow format == 'mjd' because it fails when scale is 'utc'
# Change 'mjd' to 'pulsar_mjd' to deal with this.
if fmt == "mjd":
fmt = "pulsar_mjd"
t = Time(np.longdouble(args.time),scale=args.timescale,format=fmt,
precision=9)
print(t)
else:
t = Time(args.time,scale=args.timescale,format=args.format, precision=9)
log.debug(t.iso)
t = toa.TOA(t,freq=args.freq,obs=args.obs)
# Build TOAs and compute TDBs and positions from ephemeris
ts = toa.get_TOAs_list([t],ephem=args.ephem, include_bipm=args.use_bipm,
include_gps=args.use_gps, planets=False)
if args.parfile is not None:
m=pint.models.get_model(args.parfile)
else:
# Construct model by hand
m=pint.models.StandardTimingModel
# Should check if 12:13:14.2 syntax is used and support that as well!
m.RAJ.quantity = Angle(args.ra)
m.DECJ.quantity = Angle(args.dec)
m.DM.quantity = args.dm*u.parsec/u.cm**3
tdbtimes = m.get_barycentric_toas(ts)
print("{0:.16f}".format(tdbtimes[0].value))
return
def get_fitvals(p, plot=False, order=order, second_ratio=second_ratio,
outfile=outfile, lock=lock):
if tuple(p) in fitted_positions:
return
log.debug("Fitting position {0}".format(p))
result = fit_position(p, dendrogram=dendrogram, catalog=catalog,
pcube=pcube,
plot=False, order=order,
second_ratio=second_ratio)
fitted_positions.append(tuple(p))
if result is None:
parvalues.append(None)
parerrors.append(None)
if outfile is not None:
with lock:
with open(outfilename, 'a') as outfile:
outfile.write("{0}, {1}, {2}, {3}\n".format(p[0], p[1], None, None))
outfile.flush()
return
else:
parvalues.append(result.specfit.parinfo.values)
parerrors.append(result.specfit.parinfo.errors)
if outfile is not None:
with lock:
with open(outfilename, 'a') as outfile:
outfile.write("{0}, {1}, {2}, {3}\n".format(p[0], p[1],
result.specfit.parinfo.values,
result.specfit.parinfo.errors))
outfile.flush()
return result.specfit.parinfo.values, result.specfit.parinfo.errors
def merge_light_catalogue():
"""Merge the light tiled catalogues into one big file."""
output_filename = os.path.join(constants.DESTINATION,
'concatenated',
'iphas-dr2-light.fits')
instring = ''
for lon in np.arange(25, 215+1, constants.STRIPWIDTH):
for part in ['a', 'b']:
path = os.path.join(constants.DESTINATION,
'concatenated',
'light',
'iphas-dr2-{0:03d}{1}-light.fits'.format(
lon, part))
instring += 'in={0} '.format(path)
# Warning: a bug in stilts causes long fieldIDs to be truncated if -utype S15 is not set
param = {'stilts': constants.STILTS,
'in': instring,
'out': output_filename}
cmd = '{stilts} tcat {in} countrows=true lazy=true ofmt=colfits-basic out={out}'
mycmd = cmd.format(**param)
log.debug(mycmd)
status = os.system(mycmd)
log.info('concat: '+str(status))
return status
def to_cache(response, cache_file):
log.debug("Caching data to {0}".format(cache_file))
with open(cache_file, "wb") as f:
pickle.dump(response, f)
def _nowcs_header(self):
"""
Return a copy of the header with no WCS information attached
"""
log.debug("Stripping WCS from header")
return wcs_utils.strip_wcs_from_header(self._header)
def _activate_form(self, response, form_index=0, inputs={}, cache=True,
method=None):
"""
Parameters
----------
method: None or str
Can be used to override the form-specified method
"""
# Extract form from response
root = BeautifulSoup(response.content, 'html5lib')
form = root.find_all('form')[form_index]
# Construct base url
form_action = form.get('action')
if "://" in form_action:
url = form_action
elif form_action.startswith('/'):
url = '/'.join(response.url.split('/', 3)[:3]) + form_action
else:
url = response.url.rsplit('/', 1)[0] + '/' + form_action
# Identify payload format
fmt = None
form_method = form.get('method').lower()
if form_method == 'get':
fmt = 'get' # get(url, params=payload)
elif form_method == 'post':
if 'enctype' in form.attrs:
if form.attrs['enctype'] == 'multipart/form-data':
fmt = 'multipart/form-data' # post(url, files=payload)
elif form.attrs['enctype'] == 'application/x-www-form-urlencoded':
fmt = 'application/x-www-form-urlencoded' # post(url, data=payload)
else:
fmt = 'post' # post(url, params=payload)
# Extract payload from form
payload = []
for form_elem in form.find_all(['input', 'select', 'textarea']):
value = None
is_file = False
tag_name = form_elem.name
key = form_elem.get('name')
if tag_name == 'input':
is_file = (form_elem.get('type') == 'file')
value = form_elem.get('value')
if form_elem.get('type') in ['checkbox','radio']:
if form_elem.has_attr('checked'):
if not value:
value = 'on'
else:
value = None
elif tag_name == 'select':
if form_elem.get('multiple') is not None:
value = []
for option in form_elem.select('option[value]'):
if option.get('selected') is not None:
value.append(option.get('value'))
else:
for option in form_elem.select('option[value]'):
if option.get('selected') is not None:
value = option.get('value')
# select the first option field if none is selected
if value is None:
value = form_elem.select('option[value]')[0].get('value')
if key in inputs:
value = str(inputs[key])
if (key is not None) and (value is not None):
if fmt == 'multipart/form-data':
if is_file:
payload.append(
(key, ('', '', 'application/octet-stream')))
else:
if type(value) is list:
for v in value:
payload.append((key, ('', v)))
else:
payload.append((key, ('', value)))
else:
if type(value) is list:
for v in value:
payload.append((key, v))
else:
payload.append((key, value))
# for future debugging
self._payload = payload
log.debug("Form: payload={0}".format(payload))
if method is not None:
fmt = method
# Send payload
if fmt == 'get':
response = self._request("GET", url, params=payload, cache=cache)
elif fmt == 'post':
response = self._request("POST", url, params=payload, cache=cache)
elif fmt == 'multipart/form-data':
response = self._request("POST", url, files=payload, cache=cache)
elif fmt == 'application/x-www-form-urlencoded':
response = self._request("POST", url, data=payload, cache=cache)
return response
def query_instrument(self, instrument, column_filters={}, columns=[],
open_form=False, help=False, cache=True, **kwargs):
"""
Query instrument specific raw data contained in the ESO archive.
Parameters
----------
instrument : string
Name of the instrument to query, one of the names returned by
`list_instruments()`.
column_filters : dict
Constraints applied to the query.
columns : list of strings
Columns returned by the query.
open_form : bool
If `True`, opens in your default browser the query form
for the requested instrument.
help : bool
If `True`, prints all the parameters accepted in
`column_filters` and `columns` for the requested `instrument`.
cache : bool
Cache the response for faster subsequent retrieval
Returns
-------
table : `~astropy.table.Table`
A table representing the data available in the archive for the
specified instrument, matching the constraints specified in
``kwargs``. The number of rows returned is capped by the
ROW_LIMIT configuration item.
"""
if instrument in ('feros','harps'):
url = 'http://archive.eso.org/wdb/wdb/eso/repro/form'
else:
url = "http://archive.eso.org/wdb/wdb/eso/{0}/form".format(instrument)
table = None
if open_form:
webbrowser.open(url)
elif help:
self._print_help(url, instrument)
else:
instrument_form = self._request("GET", url, cache=cache)
query_dict = {}
query_dict.update(column_filters)
# TODO: replace this with individually parsed kwargs
query_dict.update(kwargs)
query_dict["wdbo"] = "csv/download"
# Default to returning the DP.ID since it is needed for header acquisition
query_dict['tab_dp_id'] = (kwargs.pop('tab_dp_id')
if 'tab_db_id' in kwargs
else 'on')
for k in columns:
query_dict["tab_" + k] = True
if self.ROW_LIMIT >= 0:
query_dict["max_rows_returned"] = self.ROW_LIMIT
else:
query_dict["max_rows_returned"] = 10000
instrument_response = self._activate_form(instrument_form,
form_index=0,
inputs=query_dict, cache=cache)
content = instrument_response.content
#First line is always garbage
content = content.split(b'\n', 1)[1]
log.debug("Response content:\n{0}".format(content))
if _check_response(content):
try:
table = Table.read(BytesIO(content), format="ascii.csv", comment='^#')
except Exception as ex:
# astropy 0.3.2 raises an anonymous exception; this is
# intended to prevent that from causing real problems
if 'No reader defined' in ex.args[0]:
table = Table.read(BytesIO(content), format="ascii",
delimiter=',')
else:
raise ex
return table
else:
warnings.warn("Query returned no results", NoResultsWarning)
def _get_timeref(hdu):
event_hdr = hdu.header
timeref = event_hdr['TIMEREF']
log.debug("TIMEREF {0}".format(timeref))
return timeref
def _get_timesys(hdu):
event_hdr = hdu.header
timesys = event_hdr['TIMESYS']
log.debug("TIMESYS {0}".format(timesys))
return timesys
def ammonia(xarr,
trot=20,
tex=None,
ntot=14,
width=1,
xoff_v=0.0,
fortho=0.0,
tau=None,
fillingfraction=None,
return_tau=False,
background_tb=TCMB,
verbose=False,
return_components=False,
debug=False,
line_names=line_names):
"""
Generate a model Ammonia spectrum based on input temperatures, column, and
gaussian parameters
Parameters
----------
xarr: `pyspeckit.spectrum.units.SpectroscopicAxis`
Array of wavelength/frequency values
trot: float
The rotational temperature of the lines. This is the excitation
temperature that governs the relative populations of the rotational
states.
tex: float or None
Excitation temperature. Assumed LTE if unspecified (``None``) or if
tex>trot. This is the excitation temperature for *all* of the modeled
lines, which means we are explicitly assuming T_ex is the same for all
lines.
ntot: float
Total log column density of NH3. Can be specified as a float in the
range 5-25
width: float
Line width in km/s
xoff_v: float
Line offset in km/s
fortho: float
Fraction of NH3 molecules in ortho state. Default assumes all para
(fortho=0).
tau: None or float
If tau (optical depth in the 1-1 line) is specified, ntot is NOT fit
but is set to a fixed value. The optical depths of the other lines are
fixed relative to tau_oneone
fillingfraction: None or float
fillingfraction is an arbitrary scaling factor to apply to the model
return_tau: bool
Return a dictionary of the optical depths in each line instead of a
synthetic spectrum
return_components: bool
Return a list of arrays, one for each hyperfine component, instead of
just one array
background_tb : float
The background brightness temperature. Defaults to TCMB.
verbose: bool
More messages
debug: bool
For debugging.
Returns
-------
spectrum: `numpy.ndarray`
Synthetic spectrum with same shape as ``xarr``
component_list: list
List of `numpy.ndarray`'s, one for each hyperfine component
tau_dict: dict
Dictionary of optical depth values for the various lines
(if ``return_tau`` is set)
"""
from .ammonia_constants import (ckms, ccms, h, kb, Jortho, Jpara, Brot,
Crot)
# Convert X-units to frequency in GHz
if xarr.unit.to_string() != 'GHz':
xarr = xarr.as_unit('GHz')
if tex is None:
log.warning("Assuming tex=trot")
tex = trot
elif isinstance(tex, dict):
for k in tex:
assert k in line_names, "{0} not in line list".format(k)
line_names = tex.keys()
from .ammonia_constants import line_name_indices, line_names as original_line_names
# recreate line_names keeping only lines with a specified tex
# using this loop instead of tex.keys() preserves the order & data type
line_names = [k for k in original_line_names if k in line_names]
if 5 <= ntot <= 25:
# allow ntot to be specified as a logarithm. This is
# safe because ntot < 1e10 gives a spectrum of all zeros, and the
# plausible range of columns is not outside the specified range
lin_ntot = 10**ntot
else:
raise ValueError("ntot, the logarithmic total column density,"
" must be in the range 5 - 25")
tau_dict = {}
"""
Column density is the free parameter. It is used in conjunction with
the full partition function to compute the optical depth in each band
"""
Zpara = (2 * Jpara + 1) * np.exp(-h * (Brot * Jpara * (Jpara + 1) +
(Crot - Brot) * Jpara**2) /
(kb * trot))
Zortho = 2 * (2 * Jortho + 1) * np.exp(-h * (Brot * Jortho * (Jortho + 1) +
(Crot - Brot) * Jortho**2) /
(kb * trot))
Qpara = Zpara.sum()
Qortho = Zortho.sum()
log.debug("Partition Function: Q_ortho={0}, Q_para={1}".format(
Qortho, Qpara))
for linename in line_names:
if ortho_dict[linename]:
# define variable "ortho_or_para_frac" that will be the ortho
# fraction in the case of an ortho transition or the para
# fraction for a para transition
ortho_or_parafrac = fortho
Z = Zortho
Qtot = Qortho
else:
ortho_or_parafrac = 1.0 - fortho
Z = Zpara
Qtot = Qpara
# for a complete discussion of these equations, please see
# https://github.com/keflavich/pyspeckit/blob/ammonia_equations/examples/AmmoniaLevelPopulation.ipynb
# and
# http://low-sky.github.io/ammoniacolumn/
# and
# https://github.com/pyspeckit/pyspeckit/pull/136
# short variable names for readability
frq = freq_dict[linename]
partition = Z[line_name_indices[linename]]
aval = aval_dict[linename]
# Total population of the higher energy inversion transition
population_rotstate = lin_ntot * ortho_or_parafrac * partition / Qtot
if isinstance(tex, dict):
expterm = ((1 - np.exp(-h * frq / (kb * tex[linename]))) /
(1 + np.exp(-h * frq / (kb * tex[linename]))))
else:
expterm = ((1 - np.exp(-h * frq / (kb * tex))) /
(1 + np.exp(-h * frq / (kb * tex))))
fracterm = (ccms**2 * aval / (8 * np.pi * frq**2))
widthterm = (ckms / (width * frq * (2 * np.pi)**0.5))
tau_i = population_rotstate * fracterm * expterm * widthterm
tau_dict[linename] = tau_i
log.debug("Line {0}: tau={1}, expterm={2}, pop={3},"
" partition={4}".format(linename, tau_i, expterm,
population_rotstate, partition))
# allow tau(11) to be specified instead of ntot
# in the thin case, this is not needed: ntot plays no role
# this process allows you to specify tau without using the approximate equations specified
# above. It should remove ntot from the calculations anyway...
if tau is not None:
tau11_temp = tau_dict['oneone']
# re-scale all optical depths so that tau is as specified, but the relative taus
# are sest by the kinetic temperature and partition functions
for linename, t in iteritems(tau_dict):
tau_dict[linename] = t * tau / tau11_temp
if return_tau:
return tau_dict
model_spectrum = _ammonia_spectrum(xarr,
tex,
tau_dict,
width,
xoff_v,
fortho,
line_names,
background_tb=background_tb,
fillingfraction=fillingfraction,
return_components=return_components)
if model_spectrum.min() < 0 and background_tb == TCMB:
raise ValueError("Model dropped below zero. That is not possible "
" normally. Here are the input values: " +
("tex: {0} ".format(tex)) + ("trot: %f " % trot) +
("ntot: %f " % ntot) + ("width: %f " % width) +
("xoff_v: %f " % xoff_v) + ("fortho: %f " % fortho))
if verbose or debug:
log.info("trot: %g tex: %s ntot: %g width: %g xoff_v: %g "
"fortho: %g fillingfraction: %g" %
(trot, tex, ntot, width, xoff_v, fortho, fillingfraction))
return model_spectrum
def retrieve_data(self,
datasets,
continuation=False,
destination=None,
with_calib='none'):
"""
Retrieve a list of datasets form the ESO archive.
Parameters
----------
datasets : list of strings or string
List of datasets strings to retrieve from the archive.
destination: string
Directory where the files are copied.
Files already found in the destination directory are skipped,
unless continuation=True.
Default to astropy cache.
continuation : bool
Force the retrieval of data that are present in the destination
directory.
with_calib : string
Retrieve associated calibration files: 'none' (default), 'raw' for
raw calibrations, or 'processed' for processed calibrations.
Returns
-------
files : list of strings or string
List of files that have been locally downloaded from the archive.
Examples
--------
>>> dptbl = Eso.query_instrument('apex', pi_coi='ginsburg')
>>> dpids = [row['DP.ID'] for row in dptbl if 'Map' in row['Object']]
>>> files = Eso.retrieve_data(dpids)
"""
calib_options = {
'none': '',
'raw': 'CalSelectorRaw2Raw',
'processed': 'CalSelectorRaw2Master'
}
if with_calib not in calib_options:
raise ValueError("invalid value for 'with_calib', "
"it must be 'none', 'raw' or 'processed'")
if isinstance(datasets, six.string_types):
return_list = False
datasets = [datasets]
else:
return_list = True
if not isinstance(datasets, (list, tuple, np.ndarray)):
raise TypeError("Datasets must be given as a list of strings.")
# First: Detect datasets already downloaded
log.info("Detecting already downloaded datasets...")
datasets_to_download, files = self._check_existing_files(
datasets, continuation=continuation, destination=destination)
# Second: Check that the datasets to download are in the archive
log.info("Checking availability of datasets to download...")
valid_datasets = [
self.verify_data_exists(ds) for ds in datasets_to_download
]
if not all(valid_datasets):
invalid_datasets = [
ds for ds, v in zip(datasets_to_download, valid_datasets)
if not v
]
raise ValueError("The following data sets were not found on the "
"ESO servers: {0}".format(invalid_datasets))
# Third: Download the other datasets
log.info("Downloading datasets...")
if datasets_to_download:
if not self.authenticated():
self.login()
url = "http://archive.eso.org/cms/eso-data/eso-data-direct-retrieval.html"
with suspend_cache(self): # Never cache staging operations
log.info("Contacting retrieval server...")
retrieve_data_form = self._request("GET", url, cache=False)
retrieve_data_form.raise_for_status()
log.info("Staging request...")
inputs = {"list_of_datasets": "\n".join(datasets_to_download)}
data_confirmation_form = self._activate_form(
retrieve_data_form,
form_index=-1,
inputs=inputs,
cache=False)
data_confirmation_form.raise_for_status()
root = BeautifulSoup(data_confirmation_form.content,
'html5lib')
login_button = root.select('input[value=LOGIN]')
if login_button:
raise LoginError("Not logged in. "
"You must be logged in to download data.")
inputs = {}
if with_calib != 'none':
inputs['requestCommand'] = calib_options[with_calib]
# TODO: There may be another screen for Not Authorized; that
# should be included too
# form name is "retrieve"; no id
data_download_form = self._activate_form(
data_confirmation_form,
form_index=-1,
inputs=inputs,
cache=False)
log.info("Staging form is at {0}".format(
data_download_form.url))
root = BeautifulSoup(data_download_form.content, 'html5lib')
state = root.select('span[id=requestState]')[0].text
t0 = time.time()
while state not in ('COMPLETE', 'ERROR'):
time.sleep(2.0)
data_download_form = self._request("GET",
data_download_form.url,
cache=False)
root = BeautifulSoup(data_download_form.content,
'html5lib')
state = root.select('span[id=requestState]')[0].text
print("{0:20.0f}s elapsed".format(time.time() - t0),
end='\r')
sys.stdout.flush()
if state == 'ERROR':
raise RemoteServiceError("There was a remote service "
"error; perhaps the requested "
"file could not be found?")
if with_calib != 'none':
# when requested files with calibrations, some javascript is
# used to display the files, which prevent retrieving the files
# directly. So instead we retrieve the download script provided
# in the web page, and use it to extract the list of files.
# The benefit of this is also that in the download script the
# list of files is de-duplicated, whereas on the web page the
# calibration files would be duplicated for each exposure.
link = root.select('a[href$=/script]')[0]
if 'downloadRequest' not in link.text:
# Make sure that we found the correct link
raise RemoteServiceError(
"A link was found in the download file for the "
"calibrations that is not a downloadRequest link "
"and therefore appears invalid.")
href = link.attrs['href']
script = self._request("GET", href, cache=False)
fileLinks = re.findall(
r'"(https://dataportal.eso.org/dataPortal/api/requests/.*)"',
script.text)
# urls with api/ require using Basic Authentication, though
# it's easier for us to reuse the existing requests session (to
# avoid asking agin for a username/password if it is not
# stored). So we remove api/ from the urls:
fileLinks = [
f.replace(
'https://dataportal.eso.org/dataPortal/api/requests',
'https://dataportal.eso.org/dataPortal/requests')
for f in fileLinks
]
log.info("Detecting already downloaded datasets, "
"including calibrations...")
fileIds = [f.rsplit('/', maxsplit=1)[1] for f in fileLinks]
filteredIds, files = self._check_existing_files(
fileIds,
continuation=continuation,
destination=destination)
fileLinks = [
f for f, fileId in zip(fileLinks, fileIds)
if fileId in filteredIds
]
else:
fileIds = root.select('input[name=fileId]')
fileLinks = [
"http://dataportal.eso.org/dataPortal" +
fileId.attrs['value'].split()[1] for fileId in fileIds
]
log.debug("Files:\n{}".format('\n'.join(fileLinks)))
for fileLink in fileLinks:
fileId = fileLink.rsplit('/', maxsplit=1)[1]
log.info("Downloading file {0}...".format(fileId))
filename = self._request("GET",
fileLink,
save=True,
continuation=True)
if filename.endswith(('.gz', '.7z', '.bz2', '.xz')):
log.info("Unzipping file {0}...".format(fileId))
filename = system_tools.gunzip(filename)
if destination is not None:
log.info("Copying file {0} to {1}...".format(
fileId, destination))
destfile = os.path.join(destination,
os.path.basename(filename))
shutil.move(filename, destfile)
files.append(destfile)
else:
files.append(filename)
# Empty the redirect cache of this request session
# Only available and needed for requests versions < 2.17
try:
self._session.redirect_cache.clear()
except AttributeError:
pass
log.info("Done!")
if (not return_list) and (len(files) == 1):
files = files[0]
return files
def query_surveys(self,
surveys='',
cache=True,
help=False,
open_form=False,
**kwargs):
"""
Query survey Phase 3 data contained in the ESO archive.
Parameters
----------
survey : string or list
Name of the survey(s) to query. Should beone or more of the
names returned by `~astroquery.eso.EsoClass.list_surveys`. If
specified as a string, should be a comma-separated list of
survey names.
cache : bool
Cache the response for faster subsequent retrieval
Returns
-------
table : `~astropy.table.Table` or `None`
A table representing the data available in the archive for the
specified survey, matching the constraints specified in ``kwargs``.
The number of rows returned is capped by the ROW_LIMIT
configuration item. `None` is returned when the query has no
results.
"""
url = "http://archive.eso.org/wdb/wdb/adp/phase3_main/form"
if open_form:
webbrowser.open(url)
elif help:
self._print_surveys_help(url, cache=cache)
else:
survey_form = self._request("GET", url, cache=cache)
query_dict = kwargs
query_dict["wdbo"] = "csv/download"
if isinstance(surveys, six.string_types):
surveys = surveys.split(",")
query_dict['collection_name'] = surveys
if self.ROW_LIMIT >= 0:
query_dict["max_rows_returned"] = int(self.ROW_LIMIT)
else:
query_dict["max_rows_returned"] = 10000
survey_response = self._activate_form(survey_form,
form_index=0,
form_id='queryform',
inputs=query_dict,
cache=cache)
content = survey_response.content
# First line is always garbage
content = content.split(b'\n', 1)[1]
log.debug("Response content:\n{0}".format(content))
if _check_response(content):
table = Table.read(BytesIO(content),
format="ascii.csv",
comment="^#")
return table
else:
warnings.warn("Query returned no results", NoResultsWarning)
def query_instrument(self,
instrument,
column_filters={},
columns=[],
open_form=False,
help=False,
cache=True,
**kwargs):
"""
Query instrument specific raw data contained in the ESO archive.
Parameters
----------
instrument : string
Name of the instrument to query, one of the names returned by
`~astroquery.eso.EsoClass.list_instruments`.
column_filters : dict
Constraints applied to the query.
columns : list of strings
Columns returned by the query.
open_form : bool
If `True`, opens in your default browser the query form
for the requested instrument.
help : bool
If `True`, prints all the parameters accepted in
``column_filters`` and ``columns`` for the requested
``instrument``.
cache : bool
Cache the response for faster subsequent retrieval.
Returns
-------
table : `~astropy.table.Table`
A table representing the data available in the archive for the
specified instrument, matching the constraints specified in
``kwargs``. The number of rows returned is capped by the
ROW_LIMIT configuration item.
"""
if instrument in ('feros', 'harps', 'grond'):
url = 'http://archive.eso.org/wdb/wdb/eso/eso_archive_main/form'
else:
url = ("http://archive.eso.org/wdb/wdb/eso/{0}/form".format(
instrument))
table = None
if open_form:
webbrowser.open(url)
elif help:
self._print_instrument_help(url, instrument)
else:
instrument_form = self._request("GET", url, cache=cache)
query_dict = {}
query_dict.update(column_filters)
# TODO: replace this with individually parsed kwargs
query_dict.update(kwargs)
query_dict["wdbo"] = "csv/download"
# Default to returning the DP.ID since it is needed for header
# acquisition
query_dict['tab_dp_id'] = kwargs.pop('tab_dp_id', 'on')
if instrument in ('feros', 'harps', 'ground'):
query_dict['instrument'] = instrument.upper()
for k in columns:
query_dict["tab_" + k] = True
if self.ROW_LIMIT >= 0:
query_dict["max_rows_returned"] = self.ROW_LIMIT
else:
query_dict["max_rows_returned"] = 10000
# used to be form 0, but now there's a new 'logout' form at the top
# (form_index = -1 and 0 both work now that form_id is included)
instrument_response = self._activate_form(instrument_form,
form_index=-1,
form_id='queryform',
inputs=query_dict,
cache=cache)
content = instrument_response.content
# First line is always garbage
content = content.split(b'\n', 1)[1]
log.debug("Response content:\n{0}".format(content))
if _check_response(content):
table = Table.read(BytesIO(content),
format="ascii.csv",
comment='^#')
return table
else:
warnings.warn("Query returned no results", NoResultsWarning)
def fits_key_arithmetic(meta,
operand1,
operation,
operand2,
keylist=None,
handle_image=None):
"""Apply arithmetic to FITS keywords
meta : ordered_dict
FITS header of operand1 *after* processing by other arithmetic
operations. Sensible use of this feature requires
``handle_meta`` to be set to 'first_found' or callable that
returns a FITS header
operand1 : `NDData`-like instance
Generally the self of the calling object
operation : callable
The operation that is performed on the `NDData`. Supported are
`numpy.add`, `numpy.subtract`, `numpy.multiply` and
`numpy.true_divide`.
operand2 : `NDData`-like instance
Generally the self of the calling object
keylist : list
List of FITS keywords to apply ``operation`` to. Each keyword
value stands in the place of ``operand1`` and a new keyword
value is calculated using the ``operation`` and ``operand2.``
If ``operand2`` is an image, ``handle_image`` will be called
to convert it to a scalar or ``None`` (see ``handle_image``)
handle_image : callable
Called with arguments of fits_key_arithmetic (minus
``handle_image``) when ``operand2`` is an image. Return value
of ``None`` signifies application of ``operation`` would
nullify keywords in ``keylist,`` which are then removed. If
transformation of ``operand2`` into a scalar is possible
"""
if meta is None or keylist is None:
return meta
# Get a list of non-None values for our keylist
kvlist = [
kv for kv in [(k, meta.get(k)) for k in keylist] if kv[1] is not None
]
if kvlist is None:
return meta
dimso2 = sum(list(operand2.shape))
if dimso2 == 0:
# Scalar
o2 = operand2.data
else:
if handle_image is None:
o2 = None
else:
o2 = handle_image(meta,
operand1,
operation,
operand2,
keylist=keylist)
for k, v in kvlist:
if o2 is None:
del meta[k]
log.debug(
f'Cannot express operand2 as single number, deleting {k}')
else:
try:
unit = meta.cards(k).unit
except:
unit = operand1.unit
# Do the calculation with or without units
if operand1.unit is None and operand2.unit is None:
v = operation(v, o2)
elif operand1.unit is None:
v = operation(v * u.dimensionless_unscaled, o2 * operand2.unit)
elif operand2.unit is None:
v = operation(v * operand1.unit, o2 * u.dimensionless_unscaled)
else:
v = operation(v * operand1.unit, o2 * operand2.unit)
try:
meta[k] = v
except:
meta[k] = v.value
return meta
def make_spw_cube(spw='spw{0}', spwnum=0, fntemplate='OrionSourceI',
overwrite_existing=False, bmaj_limits=None,
fnsuffix="", filesuffix='image.pbcor.fits',
first_endchannel='*',
cropends=False,
minimize=True,
debug_mode=False,
check_last_plane=False,
add_beam_info=True):
"""
Parameters
----------
spw : str
String template for the input/output name
spwnum : int
The spectral window number
fntemplate : str
Filename template (goes into the glob)
overwrite_existing : bool
Overwrite data in the output cube?
cropends: bool or int
Number of pixels to crop off the ends of an image
minimize: bool
Compute the spatial minimal subcube before building the cube? Slices
for all subsequent cubes will be computed from the first cube.
"""
if debug_mode:
lvl = log.getEffectiveLevel()
log.setLevel('DEBUG')
spw = spw.format(spwnum)
big_filename = '{1}_{0}{2}_lines.fits'.format(spw, fntemplate, fnsuffix)
header_fn = glob.glob('OrionSourceI.B7.{0}.lines0-{4}.maskedclarkclean1000.{3}'
.format(spw, fntemplate, fnsuffix, filesuffix,
first_endchannel))
if len(header_fn) != 1:
raise ValueError("Found too many or too few matches: {0}".format(header_fn))
else:
header_fn = header_fn[0]
# First set up an empty file
if not os.path.exists(big_filename):
log.info("Creating large cube based on header {0}".format(header_fn))
if minimize:
cube0 = SpectralCube.read(header_fn)
slices = cube0.subcube_slices_from_mask(cube0.mask,
spatial_only=True)
# use the calculated 3rd dimension, plus the difference of the
# x and y slices
#header['NAXIS2'] = slices[1].stop-slices[1].start
#header['NAXIS1'] = slices[2].stop-slices[2].start
header = cube0[slices].header
else:
header = fits.getheader(header_fn)
# Make an arbitrary, small data before prepping the header
data = np.zeros((100, 100), dtype=np.float32)
hdu = fits.PrimaryHDU(data=data, header=header)
cdelt_sign = np.sign(hdu.header['CDELT3'])
# Set the appropriate output size (this can be extracted from the LISTOBS)
naxis3_in = header['NAXIS3']
header['NAXIS3'] = nchans_total[spwnum]
header_wcs = wcs.WCS(fits.getheader(header_fn))
header_specwcs = header_wcs.sub([wcs.WCSSUB_SPECTRAL])
if cdelt_sign == -1:
ind0, ind1 = getinds(header_fn)
#5/20/2017: redoing some of this, and the text below is frightening but no longer relevant
# a +1 was on the next line before an edit on 4/10/2017
# it may have been rendered irrelevant when I included +1
# channel in each cube? Not clear - the arithmetic no longer
# makes sense but is empirically necessary.
assert ind0 == 0
# these reindex the cube so that it has an increasing cdelt.
header['CRPIX3'] = 1 #nchans_total[spwnum]
header['CRVAL3'] = header_specwcs.wcs_pix2world([nchans_total[spwnum]],1)[0][0]
header['CDELT3'] = np.abs(header_specwcs.wcs.cdelt[0])
# ensure that the new CRVAL evaluated at its own position matches
# the CRVAL3. This should be impossible to fail unless WCS itself
# fails
newheaderspecwcs = wcs.WCS(header).sub([wcs.WCSSUB_SPECTRAL])
crval3 = newheaderspecwcs.wcs_pix2world([header['CRPIX3']], 1)[0][0]
np.testing.assert_array_almost_equal_nulp(crval3, header['CRVAL3'])
shape = (header['NAXIS3'], header['NAXIS2'], header['NAXIS1'])
# Write to disk
header.tofile(big_filename)
# Using the 'append' io method, update the *header*
with open(big_filename, 'rb+') as fobj:
# Seek past the length of the header, plus the length of the
# data we want to write.
# The -1 is to account for the final byte that we are about to
# write:
# 'seek' works on bytes, so divide #bits / (bytes/bit)
fobj.seek(len(header.tostring()) + (shape[0] *
shape[1] *
shape[2] *
int(np.abs(header['BITPIX'])/8)) -
1)
fobj.write(b'\0')
big_cube = SpectralCube.read(big_filename)
header_cube = SpectralCube.read(header_fn)
# in both cases, SpectralCube sorts the extrema
if cdelt_sign == 1:
np.testing.assert_array_almost_equal_nulp(big_cube.spectral_extrema[0].value,
header_cube.spectral_extrema[0].value)
np.testing.assert_array_almost_equal_nulp(big_cube.wcs.wcs.cdelt,
header_cube.wcs.wcs.cdelt)
elif cdelt_sign == -1:
np.testing.assert_array_almost_equal_nulp(big_cube.spectral_extrema[1].value,
header_cube.spectral_extrema[1].value)
np.testing.assert_array_almost_equal_nulp(big_cube.wcs.wcs.cdelt[-1]*-1,
header_cube.wcs.wcs.cdelt[-1])
log.info("Cube creation completed. Now moving on to populating it.")
# Find the appropriate files (this is NOT a good way to do this! Better to
# provide a list. But wildcards are quick & easy...
fileglob = "OrionSourceI.B7.{0}.lines*{3}".format(spw, fntemplate, fnsuffix,
filesuffix)
files = glob.glob(fileglob)
log.info("Files to be merged with glob {0}: ".format(fileglob))
log.info(str(files))
# open the file in update mode (it should have the right dims now)
hdul = fits.open(big_filename, mode='update')
main_wcs = wcs.WCS(hdul[0].header).sub([wcs.WCSSUB_SPECTRAL])
if add_beam_info:
shape = hdul[0].data.shape[0]
if len(hdul) > 1 and isinstance(hdul[1], fits.BinTableHDU):
pass
else:
hdul.append(fits.BinTableHDU(np.recarray(shape,
names=['BMAJ','BMIN','BPA','CHAN','POL'],
formats=['f4','f4','f4','i4','i4'])))
# sorted so that we deal with zero first, since it has potential to be a problem.
for fn in ProgressBar(sorted(files)):
log.info("inds={0} fn={1}".format(getinds(fn), fn))
ind0,ind1 = getinds(fn)
# this is not correct...?
# or maybe it only applies if cropends is set....
# if ind0 == 0:
# ind1 = ind1 + 1
cdelt = fits.getheader(fn)['CDELT3']
if 'cdelt_sign' not in locals():
cdelt_sign = np.sign(cdelt)
log.warn("cdelt_sign was not defined: overwriting a"
" previously-existing file. "
"This may not be what you want; the data could be going "
"opposite the parent cube. Check that the original "
"header is OK. sign(CDELT) is now {0}, "
"while for the big header it is {1}"
.format(cdelt_sign,
np.sign(fits.getheader(big_filename)['CDELT3'])))
if cropends:
# don't crop 1st or last pixel in full cube
if ind0 > 0:
log.debug("ind0 going from {0} to {1}".format(ind0,ind0+cropends))
ind0 = ind0 + cropends
if cdelt_sign == 1:
dataind0 = cropends
log.debug("dataind0 going to {0}".format(cropends))
else:
dataind1 = -cropends
log.debug("dataind1 going to {0}".format(-cropends))
else:
if cdelt_sign == 1:
dataind0 = 0
log.debug("dataind0 going to {0}".format(0))
elif cdelt_sign == -1:
log.debug("dataind1 going to {0}".format(None))
dataind1 = None
if (ind1 < nchans_total[spwnum] - 1):
log.debug("ind1 going from {0} to {1}".format(ind1,ind1-cropends))
ind1 = ind1 - cropends
if cdelt_sign == 1:
dataind1 = - cropends
log.debug("dataind1 going to {0}".format(-cropends))
elif cdelt_sign == -1:
dataind0 = cropends
log.debug("dataind0 going to {0}".format(cropends))
else:
if cdelt_sign == 1:
dataind1 = None
else:
log.debug("dataind0 going to {0}".format(0))
dataind0 = 0
else:
dataind0 = 0
dataind1 = None
if cdelt_sign == -1:
log.debug("Reversing indices from {0} {1} to ".format(ind0,ind1))
ind1, ind0 = (nchans_total[spwnum] - ind0,
nchans_total[spwnum] - ind1)
log.debug("{0} {1}".format(ind0, ind1))
if ind0 < 0:
ind0 = 0
log.info("inds have been remapped to {0}, {1}".format(ind0, ind1))
plane = hdul[0].data[ind0]
lastplane = hdul[0].data[ind1-1]
if np.all(plane == 0) or overwrite_existing or (check_last_plane and np.all(lastplane==0)):
log.info("Replacing indices {0}->{2} {1}"
.format(getinds(fn), fn, (ind0,ind1)))
data = fits.getdata(fn)
dwcs = wcs.WCS(fits.getheader(fn)).sub([wcs.WCSSUB_SPECTRAL])
dataind1 = data.shape[0]+(dataind1 or 0)
# handle the case where I made the indices NOT match the cube...
# this is really stupid and should be removed because I should have
# made the input cubes correct. Oh well.
if np.abs(ind1 - ind0) < np.abs(dataind1 - dataind0):
dataind1 = dataind0 + np.abs(ind1-ind0)
if cdelt_sign == -1:
dataind0, dataind1 = dataind1, dataind0
dwcs0 = dwcs.wcs_pix2world([dataind0-1], 0)[0][0]
dwcs1 = dwcs.wcs_pix2world([dataind1], 0)[0][0]
else:
dwcs0 = dwcs.wcs_pix2world([dataind0], 0)[0][0]
dwcs1 = dwcs.wcs_pix2world([dataind1-1], 0)[0][0]
hwcs0 = main_wcs.wcs_pix2world([ind0], 0)[0][0]
hwcs1 = main_wcs.wcs_pix2world([ind1-1], 0)[0][0]
if not np.isclose(hwcs0, dwcs0, atol=0.5*np.abs(cdelt), rtol=0):
log.error("current data, big cube indices: {0},{1} and {2},{3}"
.format(dataind0,dataind1,ind0,ind1))
raise ValueError("World coordinates of first pixels do not match: {0} - {1} = {2} ({3} cdelt)"
.format(dwcs0,hwcs0,dwcs0-hwcs0,(dwcs0-hwcs0)/cdelt))
if not np.isclose(hwcs1, dwcs1, atol=0.5*np.abs(cdelt), rtol=0):
log.error("current data, big cube indices: {0},{1} and {2},{3}"
.format(dataind0,dataind1,ind0,ind1))
raise ValueError("World coordinates of last pixels do not match: {0} - {1} = {2} ({3} cdelt)"
.format(dwcs1,hwcs1,dwcs1-hwcs1,(dwcs1-hwcs1)/cdelt))
if 'slices' not in locals():
if minimize:
log.info("Determining slices")
cube0 = SpectralCube.read(header_fn)
slices = cube0.subcube_slices_from_mask(cube0.mask,
spatial_only=True)
log.info("Slices are {0}".format(slices))
else:
slices = (slice(None),)*3
if bmaj_limits is not None:
log.info("Identifying acceptable beams")
beamtable = fits.open(fn)[1]
ok_beam = ((beamtable.data['BMAJ'] > bmaj_limits[0]) &
(beamtable.data['BMAJ'] < bmaj_limits[1]))
data[~ok_beam] = np.nan
log.info("Found {0} bad beams of {1}".format((~ok_beam).sum(),
ok_beam.size))
if cdelt_sign == -1:
if dataind1 == 0:
dataslice = slice(dataind0-1, None, -1)
elif dataind1 >= 1:
dataslice = slice(dataind0-1, dataind1-1, -1)
else:
raise ValueError("Something is wrong with dataind0")
else:
dataslice = slice(dataind0, dataind1, 1)
log.info("Dataslice is {0}".format(dataslice))
assert hdul[0].data[ind0:ind1].shape == data[dataslice, slices[1], slices[2]].shape
if not debug_mode:
if add_beam_info:
log.info("Adding beam information")
beamtable = fits.open(fn)[1]
hdul[1].data[ind0:ind1] = beamtable.data[dataslice]
log.info("Inserting data")
hdul[0].data[ind0:ind1,:,:] = data[dataslice, slices[1], slices[2]]
log.info("Flushing")
hdul.flush()
log.info("Done with iteration for {0}".format(fn))
if debug_mode:
log.setLevel(lvl)
def stage_data(self, uids, expand_tarfiles=False, return_json=False):
"""
Obtain table of ALMA files
Parameters
----------
uids : list or str
A list of valid UIDs or a single UID.
UIDs should have the form: 'uid://A002/X391d0b/X7b'
expand_tarfiles : bool
Expand the tarfiles to obtain lists of all contained files. If
this is specified, the parent tarfile will *not* be included
return_json : bool
Return a list of the JSON data sets returned from the query. This
is primarily intended as a debug routine, but may be useful if there
are unusual scheduling block layouts.
Returns
-------
data_file_table : Table
A table containing 3 columns: the UID, the file URL (for future
downloading), and the file size
"""
dataarchive_url = self._get_dataarchive_url()
# allow for the uid to be specified as single entry
if isinstance(uids, str):
uids = [uids]
tables = []
for uu in uids:
log.debug("Retrieving metadata for {0}".format(uu))
uid = clean_uid(uu)
req = self._request(
'GET',
'{dataarchive_url}/rh/data/expand/{uid}'.format(
dataarchive_url=dataarchive_url, uid=uid),
cache=False)
req.raise_for_status()
try:
jdata = req.json()
# Note this exception does not work in Python 2.7
except json.JSONDecodeError:
if 'Central Authentication Service' in req.text or 'recentRequests' in req.url:
# this indicates a wrong server is being used;
# the "pre-feb2020" stager will be phased out
# when the new services are deployed
raise RemoteServiceError(
"Failed query! This shouldn't happen - please "
"report the issue as it may indicate a change in "
"the ALMA servers.")
else:
raise
if return_json:
tables.append(jdata)
else:
if jdata['type'] != 'PROJECT':
log.error(
"Skipped uid {uu} because it is not a project and"
"lacks the appropriate metadata; it is a "
"{jdata}".format(uu=uu, jdata=jdata['type']))
continue
if expand_tarfiles:
table = uid_json_to_table(
jdata, productlist=['ASDM', 'PIPELINE_PRODUCT'])
else:
table = uid_json_to_table(jdata,
productlist=[
'ASDM', 'PIPELINE_PRODUCT',
'PIPELINE_PRODUCT_TARFILE',
'PIPELINE_AUXILIARY_TARFILE'
])
table['sizeInBytes'].unit = u.B
table.rename_column('sizeInBytes', 'size')
table.add_column(
Column(data=[
'{dataarchive_url}/dataPortal/{name}'.format(
dataarchive_url=dataarchive_url, name=name)
for name in table['name']
],
name='URL'))
isp = self.is_proprietary(uid)
table.add_column(
Column(data=[isp for row in table], name='isProprietary'))
tables.append(table)
log.debug("Completed metadata retrieval for {0}".format(uu))
if len(tables) == 0:
raise ValueError("No valid UIDs supplied.")
if return_json:
return tables
table = table_vstack(tables)
return table
def main(data_file, hdf5_key, cache_filename):
# Read the orbital parameters!
samples_filename = "{}.h5".format(os.path.splitext(cache_filename)[0])
_path,_basename = os.path.split(samples_filename)
if not _path:
samples_filename = os.path.join(paths.root, "cache", samples_filename)
logger.debug("Reading orbital parameter samples from '{}'".format(samples_filename))
name = os.path.splitext(_basename)[0]
plot_path = os.path.join(paths.plots, name)
logger.info("Saving plots to: {}".format(plot_path))
os.makedirs(plot_path, exist_ok=True)
with h5py.File(samples_filename, 'r') as g:
jitter = g.attrs['jitter_m/s']*u.m/u.s
P_min = g.attrs['P_min_day']*u.day
P_max = g.attrs['P_max_day']*u.day
# read the orbital parameters
opars = OrbitalParams.from_hdf5(g)
# ------------------------------------------------------------------------
# Read the data
full_path = os.path.abspath(data_file)
logger.debug("Reading data from input file at '{}'".format(full_path))
with h5py.File(full_path, 'r') as f:
if hdf5_key is not None:
g = f[hdf5_key]
else:
g = f
bmjd = g['mjd'][:]
rv = quantity_from_hdf5(g, 'rv')
rv_err = quantity_from_hdf5(g, 'rv_err')
data = RVData(bmjd, rv, stddev=rv_err)
data_jitter = RVData(bmjd, rv, stddev=np.sqrt(rv_err**2 + jitter**2))
# ------------------------------------------------------------------------
# plot RV curves
dmjd = bmjd.max() - bmjd.min()
t_grid = np.linspace(bmjd.min()-0.1*dmjd, bmjd.max()+0.1*dmjd, 1024)
fig,ax = plt.subplots(1,1,figsize=(15,5))
# HACK: where to set the units to plot in?
rv_unit = u.km/u.s
# UNDER-plot the data with jitter error-bars
data_jitter.plot(ax=ax, rv_unit=rv_unit,
marker=None, ecolor='#de2d26', alpha=0.5)
# HACK: where to set the number of lines to plot?
plot_rv_curves(opars[:512], t_grid, rv_unit=rv_unit,
data=data, ax=ax)
ax.set_title(name)
fig.tight_layout()
fig.savefig(os.path.join(plot_path, 'rv-curves.png'), dpi=300)
# ------------------------------------------------------------------------
# Corner plot!
fig = plot_corner(opars, alpha=0.1) # TODO: kwargs??
fig.suptitle(name, fontsize=26)
fig.savefig(os.path.join(plot_path, 'corner.png'), dpi=300)
def __init__(self, tbackground=2.73, gridsize=[250., 101., 100.]):
t0 = time.time()
self.texgrid303 = texgrid303 = fits.getdata(
gpath('fjdu_pH2CO_303_tex_5kms.fits'))
self.taugrid303 = taugrid303 = fits.getdata(
gpath('fjdu_pH2CO_303_tau_5kms.fits'))
self.texgrid321 = texgrid321 = fits.getdata(
gpath('fjdu_pH2CO_321_tex_5kms.fits'))
self.taugrid321 = taugrid321 = fits.getdata(
gpath('fjdu_pH2CO_321_tau_5kms.fits'))
self.texgrid322 = texgrid322 = fits.getdata(
gpath('fjdu_pH2CO_322_tex_5kms.fits'))
self.taugrid322 = taugrid322 = fits.getdata(
gpath('fjdu_pH2CO_322_tau_5kms.fits'))
self.texgrid404 = texgrid404 = fits.getdata(
gpath('fjdu_pH2CO_404_tex_5kms.fits'))
self.taugrid404 = taugrid404 = fits.getdata(
gpath('fjdu_pH2CO_404_tau_5kms.fits'))
self.texgrid422 = texgrid422 = fits.getdata(
gpath('fjdu_pH2CO_422_tex_5kms.fits'))
self.taugrid422 = taugrid422 = fits.getdata(
gpath('fjdu_pH2CO_422_tau_5kms.fits'))
self.texgrid423 = texgrid423 = fits.getdata(
gpath('fjdu_pH2CO_423_tex_5kms.fits'))
self.taugrid423 = taugrid423 = fits.getdata(
gpath('fjdu_pH2CO_423_tau_5kms.fits'))
self.hdr = hdr = hdrb = fits.getheader(
gpath('fjdu_pH2CO_303_tex_5kms.fits'))
t1 = time.time()
log.debug("Loading grids took {0:0.1f} seconds".format(t1 - t0))
self.Tbackground = tbackground
self.tline303a = ((1.0 - np.exp(-np.array(self.taugrid303))) *
(self.texgrid303 - self.Tbackground))
self.tline321a = ((1.0 - np.exp(-np.array(self.taugrid321))) *
(self.texgrid321 - self.Tbackground))
self.tline322a = ((1.0 - np.exp(-np.array(self.taugrid322))) *
(self.texgrid322 - self.Tbackground))
self.tline404a = ((1.0 - np.exp(-np.array(self.taugrid404))) *
(self.texgrid404 - self.Tbackground))
self.tline423a = ((1.0 - np.exp(-np.array(self.taugrid423))) *
(self.texgrid423 - self.Tbackground))
self.tline422a = ((1.0 - np.exp(-np.array(self.taugrid422))) *
(self.texgrid422 - self.Tbackground))
zinds, yinds, xinds = np.indices(self.tline303a.shape)
upsample_factor = np.array(
[
gridsize[0] / self.tline303a.shape[0], # temperature
gridsize[1] / self.tline303a.shape[1], # density
gridsize[2] / self.tline303a.shape[2]
], # column
dtype='float')
uzinds, uyinds, uxinds = upsinds = np.indices(
[x * us for x, us in zip(self.tline303a.shape, upsample_factor)],
dtype='float')
self.tline303 = map_coordinates(self.tline303a,
upsinds /
upsample_factor[:, None, None, None],
mode='nearest')
self.tline321 = map_coordinates(self.tline321a,
upsinds /
upsample_factor[:, None, None, None],
mode='nearest')
self.tline322 = map_coordinates(self.tline322a,
upsinds /
upsample_factor[:, None, None, None],
mode='nearest')
self.tline404 = map_coordinates(self.tline404a,
upsinds /
upsample_factor[:, None, None, None],
mode='nearest')
self.tline422 = map_coordinates(self.tline422a,
upsinds /
upsample_factor[:, None, None, None],
mode='nearest')
self.tline423 = map_coordinates(self.tline423a,
upsinds /
upsample_factor[:, None, None, None],
mode='nearest')
self.tline = {
303: self.tline303,
321: self.tline321,
322: self.tline322,
422: self.tline422,
423: self.tline423,
404: self.tline404,
}
assert self.hdr['CTYPE2'].strip() == 'LOG-DENS'
assert self.hdr['CTYPE1'].strip() == 'LOG-COLU'
self.columnarr = (
(uxinds + self.hdr['CRPIX1'] - 1) * self.hdr['CDELT1'] /
float(upsample_factor[2]) + self.hdr['CRVAL1']) # log column
self.densityarr = (
(uyinds + self.hdr['CRPIX2'] - 1) * self.hdr['CDELT2'] /
float(upsample_factor[1]) + self.hdr['CRVAL2']) # log density
self.temparr = (
(uzinds + self.hdr['CRPIX3'] - 1) * self.hdr['CDELT3'] /
float(upsample_factor[0]) + self.hdr['CRVAL3']) # lin temperature
self.drange = [self.densityarr.min(), self.densityarr.max()]
self.crange = [self.columnarr.min(), self.columnarr.max()]
self.trange = [self.temparr.min(), self.temparr.max()]
self.darr = self.densityarr[0, :, 0]
self.carr = self.columnarr[0, 0, :]
self.tarr = self.temparr[:, 0, 0]
self.axes = {'dens': self.darr, 'col': self.carr, 'tem': self.tarr}
self.labels = {
'dens': 'Density $n(\mathrm{H}_2)$ [log cm$^{-3}$]',
'col': 'p-H$_2$CO [log cm$^{-2}$/(km s$^{-1}$ pc)]',
'tem': 'Temperature (K)'
}
# While the individual lines are subject to filling factor uncertainties, the
# ratio is not.
self.modelratio1 = self.tline321 / self.tline303
self.modelratio2 = self.tline322 / self.tline321
self.modelratio_423_404 = self.tline423 / self.tline404
self.modelratio_422_404 = self.tline422 / self.tline404
self.modelratio_404_303 = self.tline404 / self.tline303
self.model_logabundance = np.log10(10**self.columnarr / u.pc.to(u.cm) /
10**self.densityarr)
t2 = time.time()
log.debug("Grid initialization took {0:0.1f} seconds total,"
" {1:0.1f} since loading grids.".format(t2 - t0, t2 - t1))
def download_files(self,
files,
savedir=None,
cache=True,
continuation=True,
skip_unauthorized=True):
"""
Given a list of file URLs, download them
Note: Given a list with repeated URLs, each will only be downloaded
once, so the return may have a different length than the input list
Parameters
----------
files : list
List of URLs to download
savedir : None or str
The directory to save to. Default is the cache location.
cache : bool
Cache the download?
continuation : bool
Attempt to continue where the download left off (if it was broken)
skip_unauthorized : bool
If you receive "unauthorized" responses for some of the download
requests, skip over them. If this is False, an exception will be
raised.
"""
if self.USERNAME:
auth = self._get_auth_info(self.USERNAME)
else:
auth = None
downloaded_files = []
if savedir is None:
savedir = self.cache_location
for fileLink in unique(files):
try:
log.debug("Downloading {0} to {1}".format(fileLink, savedir))
check_filename = self._request('HEAD',
fileLink,
auth=auth,
stream=True)
check_filename.raise_for_status()
if 'text/html' in check_filename.headers['Content-Type']:
raise ValueError("Bad query. This can happen if you "
"attempt to download proprietary "
"data when not logged in")
filename = self._request("GET",
fileLink,
save=True,
savedir=savedir,
timeout=self.TIMEOUT,
cache=cache,
auth=auth,
continuation=continuation)
downloaded_files.append(filename)
except requests.HTTPError as ex:
if ex.response.status_code == 401:
if skip_unauthorized:
log.info("Access denied to {url}. Skipping to"
" next file".format(url=fileLink))
continue
else:
raise (ex)
elif ex.response.status_code == 403:
log.error("Access denied to {url}".format(url=fileLink))
if 'dataPortal' in fileLink and 'sso' not in fileLink:
log.error("The URL may be incorrect. Try using "
"{0} instead of {1}".format(
fileLink.replace('dataPortal/',
'dataPortal/sso/'),
fileLink))
raise ex
elif ex.response.status_code == 500:
# empirically, this works the second time most of the time...
filename = self._request("GET",
fileLink,
save=True,
savedir=savedir,
timeout=self.TIMEOUT,
cache=cache,
auth=auth,
continuation=continuation)
downloaded_files.append(filename)
else:
raise ex
return downloaded_files
raise ValueError("IERS B data requested for future MJD {}".format(mjd))
might_be_old = is_url_in_cache(IERS_B_URL)
iers_b = IERS_B.open(download_file(IERS_B_URL, cache=True))
if might_be_old and iers_b[-1]["MJD"].to_value(u.d) < mjd:
# Try wiping the download and re-downloading
clear_download_cache(IERS_B_URL)
iers_b = IERS_B.open(download_file(IERS_B_URL, cache=True))
if iers_b[-1]["MJD"].to_value(u.d) < mjd:
raise ValueError(
"IERS B data not yet available for MJD {}".format(mjd))
return iers_b
# On import, make sure the IERS and leap seconds tables are updated.
log.debug(
"Running get_iers_up_to_date() to update IERS B table, and checking for updated leap seconds."
)
get_iers_up_to_date()
astropy.time.update_leap_seconds()
SECS_PER_DAY = erfa.DAYSEC
# Earth rotation rate in radians per UT1 second
#
# This is from Capitaine, Guinot, McCarthy, 2000 and is
# in IAU Resolution B1.8 on the Earth Rotation Angle (ERA)
# and the relation of it to UT1. The number 1.00273781191135448
# below is a defining constant. See here:
# http://iau-c31.nict.go.jp/pdf/2009_IAUGA_JD6/JD06_capitaine_wallace.pdf
OM = 1.00273781191135448 * 2.0 * np.pi / SECS_PER_DAY
# arcsec to radians
asec2rad = 4.84813681109536e-06
while i<len(gti_t0):
span = (gti_t1[i]-t0)*86400
gap = (gti_t0[i]-t1)*86400
#log.info(" {0} {1:.6f} {2:.6f} span {3} gap {4}".format(i,gti_t0[i],gti_t1[i],span,gap))
if span<args.maxspan and gap<args.maxgap:
# If still within MAX, just move end time to end of this GTI
t1 = gti_t1[i]
exp += gti_dt[i]
i+=1
else:
span = (t1-t0)*86400
#log.info("DEBUG: {0} {1} {2} {3}".format(exp,args.minexp,span,args.minspan))
if (exp>args.minexp) and (span>args.minspan):
# This GTI pushes us beyond MAX, so write chunk and start new
log.info("Writing chunk {0} - {1} (span {2}, exp {3})".format(t0,t1,(t1-t0)*86400,exp))
log.debug("METs {} - {}, MJDREF {}".format(mjd2met(t0),mjd2met(t1),MJDREF))
write_chunk(args.eventname,mjd2met(t0),mjd2met(t1),args.outbase,fileidx)
fileidx += 1
else:
log.info("DISCARDING chunk {0} - {1} (span {2}, exp {3})".format(t0,t1,(t1-t0)*86400,exp))
t0 = gti_t0[i]
t1 = gti_t0[i]
exp = 0.0
# If there is one segment left, do it.
if (exp>args.minexp) and (span>args.minspan):
log.info("Writing final chunk {0} - {1} (span {2}, exp {3})".format(t0,t1,(t1-t0)*86400,exp))
write_chunk(args.eventname,mjd2met(t0),mjd2met(t1),args.outbase,fileidx)
fileidx += 1
else:
log.info("DISCARDING final chunk {0} - {1} (span {2}, exp {3})".format(t0,t1,(t1-t0)*86400,exp))
def stage_data(self, uids):
"""
Stage ALMA data
Parameters
----------
uids : list or str
A list of valid UIDs or a single UID.
UIDs should have the form: 'uid://A002/X391d0b/X7b'
Returns
-------
data_file_table : Table
A table containing 3 columns: the UID, the file URL (for future
downloading), and the file size
"""
"""
With log.set_level(10)
INFO: Staging files... [astroquery.alma.core]
DEBUG: First request URL: https://almascience.eso.org/rh/submission [astroquery.alma.core]
DEBUG: First request payload: {'dataset': [u'ALMA+uid___A002_X3b3400_X90f']} [astroquery.alma.core]
DEBUG: First response URL: https://almascience.eso.org/rh/checkAuthenticationStatus/3f98de33-197e-4692-9afa-496842032ea9/submission [astroquery.alma.core]
DEBUG: Request ID: 3f98de33-197e-4692-9afa-496842032ea9 [astroquery.alma.core]
DEBUG: Submission URL: https://almascience.eso.org/rh/submission/3f98de33-197e-4692-9afa-496842032ea9 [astroquery.alma.core]
.DEBUG: Data list URL: https://almascience.eso.org/rh/requests/anonymous/786823226 [astroquery.alma.core]
"""
if isinstance(uids, six.string_types):
uids = [uids]
if not isinstance(uids, (list, tuple, np.ndarray)):
raise TypeError("Datasets must be given as a list of strings.")
log.info("Staging files...")
self._get_dataarchive_url()
url = urljoin(self.dataarchive_url, 'rh/submission')
log.debug("First request URL: {0}".format(url))
#'ALMA+uid___A002_X391d0b_X7b'
#payload = [('dataset','ALMA+'+clean_uid(uid)) for uid in uids]
payload = {'dataset': ['ALMA+' + clean_uid(uid) for uid in uids]}
log.debug("First request payload: {0}".format(payload))
self._staging_log = {'first_post_url': url}
# Request staging for the UIDs
# This component cannot be cached, since the returned data can change
# if new data are uploaded
response = self._request('POST',
url,
data=payload,
timeout=self.TIMEOUT,
cache=False)
self._staging_log['initial_response'] = response
log.debug("First response URL: {0}".format(response.url))
response.raise_for_status()
if 'j_spring_cas_security_check' in response.url:
time.sleep(1)
# CANNOT cache this stage: it not a real data page! results in
# infinite loops
response = self._request('POST',
url,
data=payload,
timeout=self.TIMEOUT,
cache=False)
self._staging_log['initial_response'] = response
if 'j_spring_cas_security_check' in response.url:
log.warn("Staging request was not successful. Try again?")
response.raise_for_status()
if 'j_spring_cas_security_check' in response.url:
raise RemoteServiceError("Could not access data. This error "
"can arise if the data are private and "
"you do not have access rights or are "
"not logged in.")
request_id = response.url.split("/")[-2]
assert len(request_id) == 36
self._staging_log['request_id'] = request_id
log.debug("Request ID: {0}".format(request_id))
# Submit a request for the specific request ID identified above
submission_url = urljoin(self.dataarchive_url,
os.path.join('rh/submission', request_id))
log.debug("Submission URL: {0}".format(submission_url))
self._staging_log['submission_url'] = submission_url
staging_submission = self._request('GET', submission_url, cache=True)
self._staging_log['staging_submission'] = staging_submission
staging_submission.raise_for_status()
data_page_url = staging_submission.url
self._staging_log['data_page_url'] = data_page_url
dpid = data_page_url.split("/")[-1]
assert len(dpid) == 9
self._staging_log['staging_page_id'] = dpid
# CANNOT cache this step: please_wait will happen infinitely
data_page = self._request('GET', data_page_url, cache=False)
self._staging_log['data_page'] = data_page
data_page.raise_for_status()
has_completed = False
while not has_completed:
time.sleep(1)
summary = self._request('GET',
os.path.join(data_page_url, 'summary'),
cache=False)
summary.raise_for_status()
print(".", end='')
sys.stdout.flush()
has_completed = summary.json()['complete']
self._staging_log['summary'] = summary
summary.raise_for_status()
self._staging_log['json_data'] = json_data = summary.json()
username = self._username if hasattr(self,
'_username') else 'anonymous'
# templates:
# https://almascience.eso.org/dataPortal/requests/keflavich/946895898/ALMA/
# 2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar/2013.1.00308.S_uid___A001_X196_X93_001_of_001.tar
# uid___A002_X9ee74a_X26f0/2013.1.00308.S_uid___A002_X9ee74a_X26f0.asdm.sdm.tar
url_decomposed = urlparse(data_page_url)
base_url = ('{uri.scheme}://{uri.netloc}/'
'dataPortal/requests/{username}/'
'{staging_page_id}/ALMA'.format(
uri=url_decomposed,
staging_page_id=dpid,
username=username,
))
tbl = self._json_summary_to_table(json_data, base_url=base_url)
# staging_root = BeautifulSoup(data_page.content)
# downloadFileURL = staging_root.find('form').attrs['action']
# data_list_url = os.path.split(downloadFileURL)[0]
# # Old version, unreliable: data_list_url = staging_submission.url
# log.debug("Data list URL: {0}".format(data_list_url))
# self._staging_log['data_list_url'] = data_list_url
# time.sleep(1)
# data_list_page = self._request('GET', data_list_url, cache=True)
# self._staging_log['data_list_page'] = data_list_page
# data_list_page.raise_for_status()
# if 'Error' in data_list_page.text:
# errormessage = staging_root.find('div', id='errorContent').string.strip()
# raise RemoteServiceError(errormessage)
# tbl = self._parse_staging_request_page(data_list_page)
return tbl
def join_catalog(self, catalog, r_tol, bandpass, no_new=False):
"""Join a specific observational catalog to the Star table.
Parameters
----------
catalog :
The :class:`starplex.database.models.Catalog` to join to the
`Star` table.
r_tol : float
Join search radius in arcseconds.
bandpass : float
The :class:`starplex.database.models.Bandpass` to use to sort
the stars being added by brightness.
no_new : bool
If ``True`` then stars from this catalog will be matched to stars
in the ``Star`` table, but unmatched stars will not create new
entries. This option can be useful for matching observed star
catalogs to a reference catalog. Default is ``False``.
"""
r_tol_m = degree_to_meter(r_tol / 3600.)
matched_count = 0
new_count = 0
# Query catalog stars for this catalog, ordering brightnest to
# faintest; that are not in the Star table already
cstar_query = self._s.query(CatalogStar).\
join(Observation, Observation.catalogstar_id == CatalogStar.id).\
filter(CatalogStar.catalog == catalog).\
filter(CatalogStar.star == None).\
order_by(Observation.mag.desc())
log.debug("cstar_query.count {0:d}".format(cstar_query.count()))
for i, cstar in enumerate(cstar_query):
# FIXME returns too many stars
q = self._s.query(Star).\
filter(cstar.coord.ST_DWithin(
Star.coord, r_tol_m, False)).\
order_by(cstar.coord.ST_Distance(Star.coord))
_ingested = False
if i % 100 == 0:
log.debug("{0:d}, {1:d}".format(i, q.count()))
if q.count() > 0:
for matched_star in q:
# Check this star is not already included in this catalog
# FIXME this query seems wrong?
star_catalogs_q = self._s.query(Catalog.id).\
join(CatalogStar,
CatalogStar.catalog_id == Catalog.id).\
join(Star,
Star.id == CatalogStar.star_id).\
filter(matched_star.id == Star.id)
# unpack to a list of Catalog ids
catalog_ids = [x[0] for x in star_catalogs_q.all()]
if i % 100 == 0:
log.debug("\t{}".format(str(matched_star)))
log.debug("\tcount of member catalogs: {0:d}".
format(star_catalogs_q.count()))
log.debug("\t{}".format(str(catalog_ids)))
if cstar.catalog_id not in catalog_ids:
# This star can be joined
self._add_to_star(cstar, matched_star)
matched_count += 1
_ingested = True
break
if _ingested == False and not no_new:
# No match; add this star directly
self._add_new_star(cstar)
new_count += 1
if i % 100 == 0:
log.debug("\tmatched %i new %i" % (matched_count, new_count))
def from_config(cls, config):
""" Construct a StreamModel from a configuration dictionary.
Typically comes from a YAML file via `streams.io.read_config`.
Parameters
----------
config : dict
"""
if not isinstance(config, dict):
from ..io import read_config
config = read_config(config)
# Set the log level based on config file - default is debug
log_level = config.get('log_level', "DEBUG")
logger.setLevel(getattr(logging, log_level.upper()))
# Use a seed for random number generators
seed = config.get('seed', np.random.randint(100000))
logger.debug("Using seed: {}".format(seed))
np.random.seed(seed)
random.seed(seed)
# Read star data from specified file
star_file = config.get('star_data')
if not os.path.exists(star_file):
star_file = os.path.join(config['streams_path'], star_file)
star_data = np.genfromtxt(star_file, names=True)
prog_file = config.get('progenitor_data')
if not os.path.exists(prog_file):
prog_file = os.path.join(config['streams_path'], prog_file)
prog_data = np.genfromtxt(prog_file, names=True)
# If limiting the number of stars
# TODO: allow selecting on some expression
try:
nstars = int(config.get('nstars'))
except TypeError:
nstars = None
if nstars is not None:
star_data = star_data[:nstars]
logger.info("Using {} stars".format(len(star_data)))
# Turn star and progenitor tables into 6D arrays with proper structure
stars_obs = np.vstack([star_data[name] for name in heliocentric_names]).T.copy()
try:
stars_err = np.vstack([star_data["err_{}".format(name)] for name in heliocentric_names]).T.copy()
except ValueError:
logger.warning("Star data uncertainty columns misnamed or don't exist.")
stars_err = None
stars = StreamComponent(stars_obs, err=stars_err,
parameters=OrderedDict([('tail',star_data['tail'])]))
# -------------------------------------------------------------------------------
prog_cfg = config.get('progenitor')
prog_obs = np.vstack([prog_data[name] for name in heliocentric_names]).T.copy()
try:
prog_err = np.vstack([prog_data["err_{}".format(name)] for name in heliocentric_names]).T.copy()
except ValueError:
logger.warning("Progenitor data uncertainty columns misnamed or don't exist.")
prog_err = None
# Progenitor mass:
try:
m0 = float(prog_cfg['mass'])
except:
m0 = eval(prog_cfg['mass'])
if isinstance(m0, BasePrior):
prior = m0
m0 = ModelParameter(name="m0", shape=(1,), prior=prior)
else:
frozen = m0
m0 = ModelParameter(name="m0", shape=(1,), prior=BasePrior())
m0.frozen = frozen
# Progenitor mass-loss:
try:
mdot = float(prog_cfg['mass_loss_rate'])
except:
mdot = eval(prog_cfg['mass_loss_rate'])
if isinstance(mdot, BasePrior):
prior = mdot
mdot = ModelParameter(name="mdot", shape=(1,), prior=prior)
else:
frozen = mdot
mdot = ModelParameter(name="mdot", shape=(1,), prior=BasePrior())
mdot.frozen = frozen
prog = StreamComponent(prog_obs, err=prog_err,
parameters=OrderedDict([('m0',m0), ('mdot',mdot)]))
# -------------------------------------------------------------------------------
# Read integration stuff
dt = float(config['integration'].get('dt'))
nintegrate = int(config['integration'].get('nsteps'))
logger.debug("Will integrate for {} steps, with a timestep of {} Myr"
.format(nintegrate, dt))
logger.info("Integration time: {} Myr".format(nintegrate*dt))
# -------------------------------------------------------------------------------
# Potential
try:
Potential = getattr(sp, config["potential"]["class"])
except AttributeError:
Potential = eval(config["potential"]["class"])
logger.info("Using potential '{}'...".format(Potential))
# potential parameters to vary
vary_pars = config["potential"].get("parameters", dict())
for k,v in vary_pars.items():
vary_pars[k] = eval(v)
# get fixed parameters
fixed_pars = config["potential"].get("fixed", dict())
potential = RewinderPotential(Potential, priors=vary_pars, fixed_pars=fixed_pars)
# -------------------------------------------------------------------------------
# Hyper-parameters
hyperpars = config.get('hyperparameters', dict())
for name,v in hyperpars.items():
logger.debug("Adding hyper-parameter: {}".format(name))
try:
prior = float(v)
logger.debug("--- {} = {}".format(name,prior))
except ValueError:
prior = eval(v)
logger.debug("--- prior passed in for {}: {}".format(name,prior))
if isinstance(prior, BasePrior):
hyperpars[name] = ModelParameter(name=name, prior=prior)
else:
hyperpars[name] = ModelParameter(name=name, prior=BasePrior())
hyperpars[name].frozen = prior
# -------------------------------------------------------------------------------
# Initialize the model
model = cls(rewinder_potential=potential,
progenitor=prog, stars=stars,
dt=dt, nsteps=nintegrate,
extra_parameters=hyperpars)
model.config = config
return model
def __new__(self, xarr, unit=None, refX=None, redshift=None,
refX_unit=None, velocity_convention=None, use128bits=False,
bad_unit_response='raise', equivalencies=u.spectral(),
center_frequency=None, center_frequency_unit=None):
"""
Make a new spectroscopic axis instance
Default units Hz
Parameter
----------
xarr : np.ndarray
An array of X-axis values in whatever unit specified
unit : str
Any valid spectroscopic X-axis unit (km/s, Hz, angstroms, etc.).
Spaces will be removed.
refX : float
Reference frequency/wavelength
refX_unit : str | astropy.units.Unit
Units of the reference frequency/wavelength
center_frequency: float
The reference frequency for determining a velocity.
Required for conversions between frequency/wavelength/energy and velocity.
(redundant with refX)
center_frequency_unit: string
If converting between velocity and any other spectroscopic type,
need to specify the central frequency around which that velocity is
calculated.
(redundant with refX_unit)
equivalencies : list
astropy equivalencies list containing tuples of the form:
(from_unit, to_unit, forward, backward)
forward and backward are functions that convert values between those units
bad_unit_response : 'raise' | 'pixel'
What should pyspeckit do if the units are not recognized? Default
is to raise an exception. Can make pixel units instead
"""
if use128bits:
dtype='float128'
else:
dtype='float64'
# Only need to convert xarr to array if it's not already one (e.g., if
# it's a list)
if not isinstance(xarr, np.ndarray):
subarr = np.array(xarr, dtype=dtype)
log.debug("Created subarr from a non-ndarray {0}".format(type(xarr)))
else:
if not xarr.flags['OWNDATA']:
# nothing owns its data. We nearly always have to copy this. =(
#log.warning("The X array does not 'own' its data."
# " It will therefore be copied.")
#warnings.warn("The X array does not 'own' its data."
# " It will therefore be copied.")
subarr = xarr.copy()
else:
log.debug("xarr owns its own data. Continuing as normal.")
subarr = xarr
subarr = subarr.view(self)
# Only need to set xarr's unit if it's not a quantity
# or if it is unitless
if not isinstance(xarr, u.Quantity) or xarr.unit == u.dimensionless_unscaled:
subarr._unit = self.validate_unit(unit, bad_unit_response)
subarr.refX = refX
if refX_unit is None:
if hasattr(refX, 'unit'):
refX_unit = refX.unit
elif subarr._unit in frequency_dict:
refX_unit = subarr.unit
else:
refX_unit = 'Hz'
subarr.refX_unit = refX_unit
subarr.redshift = redshift
subarr.wcshead = {}
subarr.velocity_convention = velocity_convention
if center_frequency is None:
if refX is not None:
center_frequency = refX
if center_frequency_unit is None:
if refX_unit is not None:
center_frequency_unit = refX_unit
else:
center_frequency_unit = unit
if center_frequency is not None:
subarr.center_frequency = u.Quantity(center_frequency, center_frequency_unit)
else:
subarr.center_frequency = None
temp1, temp2 = self.find_equivalencies(subarr.velocity_convention,
subarr.center_frequency,
equivalencies)
subarr.center_frequency, subarr._equivalencies = temp1,temp2
return subarr
def slice(self,
start=None,
stop=None,
unit='pixel',
copy=True,
xcopy=True,
preserve_fits=False):
"""
Slicing the spectrum
.. WARNING:: this is the same as cropping right now, but it returns a
copy instead of cropping inplace
Parameters
----------
start : numpy.float or int or astropy quantity
start of slice
stop : numpy.float or int or astropy quantity
stop of slice
unit : str
allowed values are any supported physical unit, 'pixel'
copy : bool
Return a 'view' of the data or a copy?
preserve_fits : bool
Save the fitted parameters from self.fitter?
"""
if hasattr(start, 'unit'):
start_ind = self.xarr.x_to_pix(start)
elif unit in ('pixel', 'pixels'):
start_ind = start
else:
start_ind = self.xarr.x_to_pix(start, xval_unit=unit)
if hasattr(stop, 'unit'):
stop_ind = self.xarr.x_to_pix(stop)
elif unit in ('pixel', 'pixels'):
stop_ind = stop
else:
stop_ind = self.xarr.x_to_pix(stop, xval_unit=unit)
if start_ind > stop_ind:
start_ind, stop_ind = stop_ind, start_ind
spectrum_slice = slice(start_ind, stop_ind)
log.debug("Slicing from {start} to {stop} with unit {unit} and copy="
"{copy}, xcopy={xcopy}, preserve_fits={preserve_fits}."
"start_ind = {start_ind}, stop_ind= {stop_ind}".format(
start=start,
stop=stop,
unit=unit,
copy=copy,
xcopy=xcopy,
preserve_fits=preserve_fits,
start_ind=start_ind,
stop_ind=stop_ind))
if copy:
sp = self.copy()
else:
sp = self
sp.data = sp.data[spectrum_slice]
if sp.error is not None:
sp.error = sp.error[spectrum_slice]
if copy or xcopy:
sp.xarr = sp.xarr[spectrum_slice].copy()
else:
sp.xarr = sp.xarr[spectrum_slice]
if copy:
# create new specfit / baseline instances (otherwise they'll be the wrong length)
sp._register_fitters(registry=self.Registry)
sp.baseline = baseline.Baseline(sp)
sp.specfit = fitters.Specfit(sp, Registry=sp.Registry)
else:
# inplace modification
sp.baseline.crop(start_ind, stop_ind)
sp.specfit.crop(start_ind, stop_ind)
if preserve_fits:
sp.specfit.modelpars = self.specfit.modelpars
sp.specfit.parinfo = self.specfit.parinfo
sp.baseline.baselinepars = self.baseline.baselinepars
sp.baseline.order = self.baseline.order
return sp
def compute_posvels(self, ephem="DE421", planets=False):
"""Compute positions and velocities of the observatories and Earth.
Compute the positions and velocities of the observatory (wrt
the Geocenter) and the center of the Earth (referenced to the
SSB) for each TOA. The JPL solar system ephemeris can be set
using the 'ephem' parameter. The positions and velocities are
set with PosVel class instances which have astropy units.
"""
# Record the planets choice for this instance
self.planets = planets
log.info(
'Compute positions and velocities of observatories and Earth (planets = {0}), using {1} ephemeris'
.format(planets, ephem))
# Remove any existing columns
cols_to_remove = ['ssb_obs_pos', 'ssb_obs_vel', 'obs_sun_pos']
for c in cols_to_remove:
if c in self.table.colnames:
log.info('Column {0} already exists. Removing...'.format(c))
self.table.remove_column(c)
for p in ('jupiter', 'saturn', 'venus', 'uranus'):
name = 'obs_' + p + '_pos'
if name in self.table.colnames:
log.info('Column {0} already exists. Removing...'.format(name))
self.table.remove_column(name)
self.table.meta['ephem'] = ephem
ssb_obs_pos = table.Column(name='ssb_obs_pos',
data=numpy.zeros((self.ntoas, 3),
dtype=numpy.float64),
unit=u.km,
meta={
'origin': 'SSB',
'obj': 'OBS'
})
ssb_obs_vel = table.Column(name='ssb_obs_vel',
data=numpy.zeros((self.ntoas, 3),
dtype=numpy.float64),
unit=u.km / u.s,
meta={
'origin': 'SSB',
'obj': 'OBS'
})
obs_sun_pos = table.Column(name='obs_sun_pos',
data=numpy.zeros((self.ntoas, 3),
dtype=numpy.float64),
unit=u.km,
meta={
'origin': 'OBS',
'obj': 'SUN'
})
if planets:
plan_poss = {}
for p in ('jupiter', 'saturn', 'venus', 'uranus'):
name = 'obs_' + p + '_pos'
plan_poss[name] = table.Column(name=name,
data=numpy.zeros(
(self.ntoas, 3),
dtype=numpy.float64),
unit=u.km,
meta={
'origin': 'OBS',
'obj': p
})
# Now step through in observatory groups
for ii, key in enumerate(self.table.groups.keys):
grp = self.table.groups[ii]
obs = self.table.groups.keys[ii]['obs']
loind, hiind = self.table.groups.indices[ii:ii + 2]
site = get_observatory(obs)
tdb = time.Time(grp['tdb'], precision=9)
ssb_obs = site.posvel(tdb, ephem)
log.debug("SSB obs pos {0}".format(ssb_obs.pos[:, 0]))
ssb_obs_pos[loind:hiind, :] = ssb_obs.pos.T.to(u.km)
ssb_obs_vel[loind:hiind, :] = ssb_obs.vel.T.to(u.km / u.s)
sun_obs = objPosVel_wrt_SSB('sun', tdb, ephem) - ssb_obs
obs_sun_pos[loind:hiind, :] = sun_obs.pos.T.to(u.km)
if planets:
for p in ('jupiter', 'saturn', 'venus', 'uranus'):
name = 'obs_' + p + '_pos'
dest = p
pv = objPosVel_wrt_SSB(dest, tdb, ephem) - ssb_obs
plan_poss[name][loind:hiind, :] = pv.pos.T.to(u.km)
cols_to_add = [ssb_obs_pos, ssb_obs_vel, obs_sun_pos]
if planets:
cols_to_add += plan_poss.values()
log.info('Adding columns ' + ' '.join([cc.name for cc in cols_to_add]))
self.table.add_columns(cols_to_add)
"""
This is obsolete, even though it was used. tp_7m_12m_combine.py is an improved
and more efficient version of this.
"""
import numpy as np
from spectral_cube import SpectralCube
from astropy.convolution import convolve, convolve_fft, Gaussian1DKernel
from uvcombine import spectral_regrid, feather_simple, fourier_combine_cubes
from astropy import units as u
from astropy import log
log.setLevel("DEBUG")
log.debug('Reading...')
cube = SpectralCube.read('SgrB2_b3_7M_12M.HC3N.image.pbcor_medsub.fits').with_spectral_unit(u.GHz)
cube.allow_huge_operations=True
log.debug("Read file")
cube_k = cube.to(u.K, cube.beam.jtok_equiv(cube.spectral_axis))
log.debug("Converted first file to K")
tpcube = SpectralCube.read('../tp/tp_concat.spw17.image.fits').with_spectral_unit(u.GHz)
crop_freqs = cube_k.spectral_axis[0], cube_k.spectral_axis[-1]
crop_channels = sorted((tpcube.closest_spectral_channel(crop_freqs[0]), tpcube.closest_spectral_channel(crop_freqs[1])))
tpcube = tpcube[crop_channels[0]-1:crop_channels[1]+1]
log.debug("Read tp freq")
tpcube_k = tpcube.to(u.K, tpcube.beam.jtok_equiv(tpcube.spectral_axis))
log.debug("Converted TP to K")
# determine smooth factor
kw = cube_k.spectral_axis.diff().mean() / tpcube_k.spectral_axis.diff().mean()
log.debug("determined kernel")
tpcube_k_smooth = tpcube_k.spectral_smooth(Gaussian1DKernel(kw/(8*np.log(2))**0.5))
def __init__(self, rewinder_potential, progenitor, stars, dt, nsteps, extra_parameters,
selfgravity=True, nsamples=128):
""" Model for tidal streams that uses backwards integration to Rewind
the positions of stars.
Parameters
----------
rewinder_potential : streams.RewinderPotential
progenitor : streams.Progenitor
stars : streams.Stars
dt, nsteps : float
Integration parameters.
Other Parameters
----------------
kwargs
Any other keyword arguments passed in are assumed to be additional
parameters for the model.
"""
# integration
self.dt = dt
self.nsteps = nsteps
self.args = (self.dt, self.nsteps)
self.parameters = OrderedDict()
self.rewinder_potential = rewinder_potential
self.progenitor = progenitor
self.stars = stars
self.nstars = len(stars.data)
self.selfgravity = selfgravity
for name,p in self.rewinder_potential.parameters.items():
logger.debug("Adding parameter {}".format(name))
self.add_parameter(p, group='potential')
for name,p in self.progenitor.parameters.items():
logger.debug("Adding parameter {}".format(name))
self.add_parameter(p, group='progenitor')
for name,p in extra_parameters.items():
logger.debug("Adding parameter {}".format(name))
self.add_parameter(p, group='hyper')
self.perfect_stars = False
if self.stars.err is None:
self.perfect_stars = True
logger.warning("No uncertainties on stars")
self.perfect_prog = False
if self.progenitor.err is None:
self.perfect_prog = True
logger.warning("No uncertainties on progenitor")
self.perfect_data = self.perfect_stars and self.perfect_prog
if self.perfect_data:
logger.warning("Perfect data!")
self._ln_likelihood_tmp = np.empty((self.nsteps, self.nstars))
if not self.perfect_prog:
# add progenitor position as parameters
for i,name in enumerate(heliocentric_names):
logger.debug("Adding progenitor parameter {}".format(name))
p = ModelParameter(name=name, prior=BasePrior())
self.add_parameter(p, group='progenitor')
self.nsamples = nsamples
if not self.perfect_stars:
tot_samples = 10000
# draw samples for each star
impo_samples_hel = np.zeros((self.nstars,tot_samples,6))
impo_samples_hel[...,:2] = self.stars.data[:,np.newaxis,:2] # copy over l,b
impo_samples_hel[...,2:] = np.random.normal(self.stars.data[:,None,2:],
self.stars.err[:,None,2:],
size=(self.nstars,tot_samples,4)) # TODO: missing data!!??
impo_samples = hel_to_gal(impo_samples_hel.reshape(self.nstars*tot_samples,6))
# compute prior probabilities for the samples
ldp = np.array([self.stars.ln_data_prob(impo_samples_hel[:,i]) for i in range(tot_samples)])
self.impo_samples_lnprob = ldp.T
# transform to galactocentric
self.impo_samples_gal = impo_samples.reshape(self.nstars,tot_samples,6)
# TODO: pre-allocate? set the tail assignments
# tail = np.array(self.stars.parameters['tail'])
self._ln_likelihood_tmp = np.zeros((self.nsteps, self.nsamples))
def _parse_kwargs(self,
min_frequency=None,
max_frequency=None,
band='any',
top20=None,
chemical_name=None,
chem_re_flags=0,
energy_min=None,
energy_max=None,
energy_type=None,
intensity_lower_limit=None,
intensity_type=None,
transition=None,
version=None,
exclude=None,
only_NRAO_recommended=None,
line_lists=None,
line_strengths=None,
energy_levels=None,
export=None,
export_limit=None,
noHFS=None,
displayHFS=None,
show_unres_qn=None,
show_upper_degeneracy=None,
show_molecule_tag=None,
show_qn_code=None,
show_lovas_labref=None,
show_lovas_obsref=None,
show_orderedfreq_only=None,
show_nrao_recommended=None,
parse_chemistry_locally=True):
"""
The Splatalogue service returns lines with rest frequencies in the
range [min_frequency, max_frequency].
Parameters
----------
min_frequency : `astropy.units`
Minimum frequency (or any spectral() equivalent)
max_frequency : `astropy.units`
Maximum frequency (or any spectral() equivalent)
band : str
The observing band. If it is not 'any', it overrides
minfreq/maxfreq.
top20: str
One of ``'comet'``, ``'planet'``, ``'top20'``, ``'ism_hotcore'``,
``'ism_darkcloud'``, ``'ism_diffusecloud'``.
Overrides chemical_name
chemical_name : str
Name of the chemical to search for. Treated as a regular
expression. An empty set ('', (), [], {}) will match *any*
species. Examples:
``'H2CO'`` - 13 species have H2CO somewhere in their formula.
``'Formaldehyde'`` - There are 8 isotopologues of Formaldehyde
(e.g., H213CO).
``'formaldehyde'`` - Thioformaldehyde,Cyanoformaldehyde.
``'formaldehyde',chem_re_flags=re.I`` - Formaldehyde,thioformaldehyde,
and Cyanoformaldehyde.
``' H2CO '`` - Just 1 species, H2CO. The spaces prevent including
others.
parse_chemistry_locally : bool
Attempt to determine the species ID #'s locally before sending the
query? This will prevent queries that have no matching species.
It also performs a more flexible regular expression match to the
species IDs. See the examples in `get_species_ids`
chem_re_flags : int
See the `re` module
energy_min : `None` or float
Energy range to include. See energy_type
energy_max : `None` or float
Energy range to include. See energy_type
energy_type : ``'el_cm1'``, ``'eu_cm1'``, ``'eu_k'``, ``'el_k'``
Type of energy to restrict. L/U for lower/upper state energy,
cm/K for *inverse* cm, i.e. wavenumber, or K for Kelvin
intensity_lower_limit : `None` or float
Lower limit on the intensity. See intensity_type
intensity_type : `None` or ``'sij'``, ``'cdms_jpl'``, ``'aij'``
The type of intensity on which to place a lower limit
transition : str
e.g. 1-0
version : ``'v1.0'``, ``'v2.0'``, ``'v3.0'`` or ``'vall'``
Data version
exclude : list
Types of lines to exclude. Default is:
(``'potential'``, ``'atmospheric'``, ``'probable'``)
Can also exclude ``'known'``.
To exclude nothing, use 'none', not the python object None, since
the latter is meant to indicate 'leave as default'
only_NRAO_recommended : bool
Show only NRAO recommended species?
line_lists : list
Options:
Lovas, SLAIM, JPL, CDMS, ToyoMA, OSU, Recomb, Lisa, RFI
line_strengths : list
* CDMS/JPL Intensity : ls1
* Sij : ls3
* Aij : ls4
* Lovas/AST : ls5
energy_levels : list
* E_lower (cm^-1) : el1
* E_lower (K) : el2
* E_upper (cm^-1) : el3
* E_upper (K) : el4
export : bool
Set up arguments for the export server (as opposed to the HTML
server)?
export_limit : int
Maximum number of lines in output file
noHFS : bool
No HFS Display
displayHFS : bool
Display HFS Intensity
show_unres_qn : bool
Display Unresolved Quantum Numbers
show_upper_degeneracy : bool
Display Upper State Degeneracy
show_molecule_tag : bool
Display Molecule Tag
show_qn_code : bool
Display Quantum Number Code
show_lovas_labref : bool
Display Lab Ref
show_lovas_obsref : bool
Display Obs Ref
show_orderedfreq_only : bool
Display Ordered Frequency ONLY
show_nrao_recommended : bool
Display NRAO Recommended Frequencies
Returns
-------
payload : dict
Dictionary of the parameters to send to the SPLAT page
"""
payload = {
'submit': 'Search',
'frequency_units': 'GHz',
}
if band != 'any':
if band not in self.FREQUENCY_BANDS:
raise ValueError("Invalid frequency band.")
if min_frequency is not None or max_frequency is not None:
warnings.warn("Band was specified, so the frequency "
"specification is overridden")
payload['band'] = band
elif min_frequency is not None and max_frequency is not None:
# allow setting payload without having *ANY* valid frequencies set
min_frequency = min_frequency.to(u.GHz, u.spectral())
max_frequency = max_frequency.to(u.GHz, u.spectral())
if min_frequency > max_frequency:
min_frequency, max_frequency = max_frequency, min_frequency
payload['from'] = min_frequency.value
payload['to'] = max_frequency.value
if top20 is not None:
if top20 in self.TOP20_LIST:
payload['top20'] = top20
else:
raise ValueError("Top20 is not one of the allowed values")
elif chemical_name in ('', {}, (), [], set()):
# include all
payload['sid[]'] = []
elif chemical_name is not None:
if parse_chemistry_locally:
species_ids = self.get_species_ids(chemical_name,
chem_re_flags)
if len(species_ids) == 0:
raise ValueError("No matching chemical species found.")
payload['sid[]'] = list(species_ids.values())
else:
payload['chemical_name'] = chemical_name
if energy_min is not None:
payload['energy_range_from'] = float(energy_min)
if energy_max is not None:
payload['energy_range_to'] = float(energy_max)
if energy_type is not None:
validate_energy_type(energy_type)
payload['energy_range_type'] = energy_type
if intensity_type is not None:
payload['lill'] = 'lill_' + intensity_type
if intensity_lower_limit is not None:
payload[payload['lill']] = intensity_lower_limit
if transition is not None:
payload['tran'] = transition
if version in self.versions:
payload['data_version'] = version
elif version is not None:
raise ValueError("Invalid version specified. Allowed versions "
"are {vers}".format(vers=str(self.versions)))
if exclude == 'none':
for e in ('potential', 'atmospheric', 'probable', 'known'):
# Setting a keyword value to 'None' removes it (see query_lines_async)
log.debug("Setting no_{0} to None".format(e))
payload['no_' + e] = None
elif exclude is not None:
for e in exclude:
payload['no_' + e] = 'no_' + e
if only_NRAO_recommended:
payload['include_only_nrao'] = 'include_only_nrao'
if line_lists is not None:
if type(line_lists) not in (tuple, list):
raise TypeError("Line lists should be a list of linelist "
"names. See Splatalogue.ALL_LINE_LISTS")
for L in self.ALL_LINE_LISTS:
kwd = 'display' + L
if L in line_lists:
payload[kwd] = kwd
else:
payload[kwd] = ''
if line_strengths is not None:
for LS in line_strengths:
payload[LS] = LS
if energy_levels is not None:
for EL in energy_levels:
payload[EL] = EL
for b in ("noHFS", "displayHFS", "show_unres_qn",
"show_upper_degeneracy", "show_molecule_tag", "show_qn_code",
"show_lovas_labref", "show_orderedfreq_only",
"show_lovas_obsref", "show_nrao_recommended"):
if locals()[b]:
payload[b] = b
# default arg, unmodifiable...
payload['jsMath'] = 'font:symbol,warn:0'
payload['__utma'] = ''
payload['__utmc'] = ''
if export:
payload['submit'] = 'Export'
payload['export_delimiter'] = 'colon' # or tab or comma
payload['export_type'] = 'current'
payload['offset'] = 0
payload['range'] = 'on'
if export_limit is not None:
payload['limit'] = export_limit
else:
payload['limit'] = self.LINES_LIMIT
return payload
def make_finder_chart_from_image_and_catalog(
image,
catalog,
save_prefix,
alma_kwargs={
'public': False,
'science': False
},
bands=(3, 4, 5, 6, 7, 8, 9, 10),
private_band_colors=(
'maroon',
'red',
'orange',
'coral',
'brown',
'yellow',
'mediumorchid',
'palegoldenrod',
),
public_band_colors=(
'blue',
'cyan',
'green',
'turquoise',
'teal',
'darkslategrey',
'chartreuse',
'lime',
),
integration_time_contour_levels=np.logspace(0, 5, base=2, num=6),
save_masks=False,
use_saved_masks=False,
linewidth=1,
):
"""
Create a "finder chart" showing where ALMA has pointed in various bands,
including different color coding for public/private data and each band.
Contours are set at various integration times.
Parameters
----------
image : fits.PrimaryHDU or fits.ImageHDU object
The image to overlay onto
catalog : astropy.Table object
The catalog of ALMA observations
save_prefix : str
The prefix for the output files. Both .reg and .png files will be
written. The .reg files will have the band numbers and
public/private appended, while the .png file will be named
prefix_almafinderchart.png
alma_kwargs : dict
Keywords to pass to the ALMA archive when querying.
private_band_colors / public_band_colors : tuple
A tuple or list of colors to be associated with private/public
observations in the various bands
integration_time_contour_levels : list or np.array
The levels at which to draw contours in units of seconds. Default is
log-spaced (2^n) seconds: [ 1., 2., 4., 8., 16., 32.])
"""
import aplpy
import pyregion
all_bands = bands
bands = used_bands = [int(band) for band in np.unique(catalog['Band'])]
log.info("The bands used include: {0}".format(used_bands))
band_colors_priv = dict(zip(all_bands, private_band_colors))
band_colors_pub = dict(zip(all_bands, public_band_colors))
log.info("Color map private: {0}".format(band_colors_priv))
log.info("Color map public: {0}".format(band_colors_pub))
if use_saved_masks:
hit_mask_public = {}
hit_mask_private = {}
for band in bands:
pubfile = '{0}_band{1}_public.fits'.format(save_prefix, band)
if os.path.exists(pubfile):
hit_mask_public[band] = fits.getdata(pubfile)
privfile = '{0}_band{1}_private.fits'.format(save_prefix, band)
if os.path.exists(privfile):
hit_mask_private[band] = fits.getdata(privfile)
else:
today = np.datetime64('today')
# At least temporarily obsolete
# private_circle_parameters = {
# band: [(row['RA'], row['Dec'], np.mean(rad).to(u.deg).value)
# for row, rad in zip(catalog, primary_beam_radii)
# if not row['Release date'] or
# (np.datetime64(row['Release date']) > today and row['Band'] == band)]
# for band in bands}
# public_circle_parameters = {
# band: [(row['RA'], row['Dec'], np.mean(rad).to(u.deg).value)
# for row, rad in zip(catalog, primary_beam_radii)
# if row['Release date'] and
# (np.datetime64(row['Release date']) <= today and row['Band'] == band)]
# for band in bands}
# unique_private_circle_parameters = {
# band: np.array(list(set(private_circle_parameters[band])))
# for band in bands}
# unique_public_circle_parameters = {
# band: np.array(list(set(public_circle_parameters[band])))
# for band in bands}
release_dates = np.array(catalog['Release date'], dtype=np.datetime64)
for band in bands:
log.info("BAND {0}".format(band))
privrows = sum((catalog['Band'] == band) & (release_dates > today))
pubrows = sum((catalog['Band'] == band) & (release_dates <= today))
log.info("PUBLIC: Number of rows: {0}".format(pubrows, ))
log.info("PRIVATE: Number of rows: {0}.".format(privrows))
log.debug('Creating regions')
prv_regions = {
band: pyregion.ShapeList([
add_meta_to_reg(fp, {'integration': row['Integration']})
for row in catalog for fp in footprint_to_reg(row['Footprint'])
if (not row['Release date']) or (np.datetime64(
row['Release date']) > today and row['Band'] == band)
])
for band in bands
}
pub_regions = {
band: pyregion.ShapeList([
add_meta_to_reg(fp, {'integration': row['Integration']})
for row in catalog for fp in footprint_to_reg(row['Footprint'])
if row['Release date'] and (np.datetime64(row['Release date'])
<= today and row['Band'] == band)
])
for band in bands
}
log.debug('Creating masks')
prv_mask = {
band:
fits.PrimaryHDU(prv_regions[band].get_mask(image).astype('int'),
header=image.header)
for band in bands if prv_regions[band]
}
pub_mask = {
band:
fits.PrimaryHDU(pub_regions[band].get_mask(image).astype('int'),
header=image.header)
for band in bands if pub_regions[band]
}
hit_mask_public = {
band: np.zeros_like(image.data)
for band in pub_mask
}
hit_mask_private = {
band: np.zeros_like(image.data)
for band in prv_mask
}
mywcs = wcs.WCS(image.header)
for band in bands:
log.debug(
'Adding integration-scaled masks for Band: {0}'.format(band))
shapes = prv_regions[band]
for shape in shapes:
# private: release_date = 'sometime' says when it will be released
(xlo, xhi, ylo,
yhi), mask = pyregion_subset(shape, hit_mask_private[band],
mywcs)
log.debug("{0},{1},{2},{3}: {4}".format(
xlo, xhi, ylo, yhi, mask.sum()))
hit_mask_private[band][
ylo:yhi, xlo:xhi] += shape.meta['integration'] * mask
if save_masks:
shapes.write('{0}_band{1}_private.reg'.format(
save_prefix, band))
shapes = pub_regions[band]
for shape in shapes:
# public: release_date = '' should mean already released
(xlo, xhi, ylo,
yhi), mask = pyregion_subset(shape, hit_mask_public[band],
mywcs)
log.debug("{0},{1},{2},{3}: {4}".format(
xlo, xhi, ylo, yhi, mask.sum()))
hit_mask_public[band][
ylo:yhi, xlo:xhi] += shape.meta['integration'] * mask
if save_masks:
shapes.write('{0}_band{1}_public.reg'.format(
save_prefix, band))
if save_masks:
for band in bands:
if band in hit_mask_public:
if hit_mask_public[band].any():
hdu = fits.PrimaryHDU(data=hit_mask_public[band],
header=image.header)
hdu.writeto('{0}_band{1}_public.fits'.format(
save_prefix, band),
clobber=True)
if band in hit_mask_private:
if hit_mask_private[band].any():
hdu = fits.PrimaryHDU(data=hit_mask_private[band],
header=image.header)
hdu.writeto('{0}_band{1}_private.fits'.format(
save_prefix, band),
clobber=True)
fig = aplpy.FITSFigure(fits.HDUList(image), convention='calabretta')
fig.show_grayscale(stretch='arcsinh', vmid=np.nanmedian(image.data))
for band in bands:
if band in hit_mask_public:
if hit_mask_public[band].any():
fig.show_contour(fits.PrimaryHDU(data=hit_mask_public[band],
header=image.header),
levels=integration_time_contour_levels,
colors=[band_colors_pub[int(band)]] *
len(integration_time_contour_levels),
linewidth=linewidth,
convention='calabretta')
if band in hit_mask_private:
if hit_mask_private[band].any():
fig.show_contour(fits.PrimaryHDU(data=hit_mask_private[band],
header=image.header),
levels=integration_time_contour_levels,
colors=[band_colors_priv[int(band)]] *
len(integration_time_contour_levels),
linewidth=linewidth,
convention='calabretta')
fig.save('{0}_almafinderchart.png'.format(save_prefix))
return image, catalog, hit_mask_public, hit_mask_private
def _activate_form(self,
response,
form_index=0,
form_id=None,
inputs={},
cache=True,
method=None):
"""
Parameters
----------
method: None or str
Can be used to override the form-specified method
"""
# Extract form from response
root = BeautifulSoup(response.content, 'html5lib')
if form_id is None:
form = root.find_all('form')[form_index]
else:
form = root.find_all('form', id=form_id)[form_index]
# Construct base url
form_action = form.get('action')
if "://" in form_action:
url = form_action
elif form_action.startswith('/'):
url = '/'.join(response.url.split('/', 3)[:3]) + form_action
else:
url = response.url.rsplit('/', 1)[0] + '/' + form_action
# Identify payload format
fmt = None
form_method = form.get('method').lower()
if form_method == 'get':
fmt = 'get' # get(url, params=payload)
elif form_method == 'post':
if 'enctype' in form.attrs:
if form.attrs['enctype'] == 'multipart/form-data':
fmt = 'multipart/form-data' # post(url, files=payload)
elif form.attrs[
'enctype'] == 'application/x-www-form-urlencoded':
fmt = 'application/x-www-form-urlencoded' # post(url, data=payload)
else:
raise Exception("enctype={0} is not supported!".format(
form.attrs['enctype']))
else:
fmt = 'application/x-www-form-urlencoded' # post(url, data=payload)
# Extract payload from form
payload = []
for form_elem in form.find_all(['input', 'select', 'textarea']):
value = None
is_file = False
tag_name = form_elem.name
key = form_elem.get('name')
if tag_name == 'input':
is_file = (form_elem.get('type') == 'file')
value = form_elem.get('value')
if form_elem.get('type') in ['checkbox', 'radio']:
if form_elem.has_attr('checked'):
if not value:
value = 'on'
else:
value = None
elif tag_name == 'select':
if form_elem.get('multiple') is not None:
value = []
if form_elem.select('option[value]'):
for option in form_elem.select('option[value]'):
if option.get('selected') is not None:
value.append(option.get('value'))
else:
for option in form_elem.select('option'):
if option.get('selected') is not None:
# bs4 NavigableString types have bad,
# undesirable properties that result
# in recursion errors when caching
value.append(str(option.string))
else:
if form_elem.select('option[value]'):
for option in form_elem.select('option[value]'):
if option.get('selected') is not None:
value = option.get('value')
# select the first option field if none is selected
if value is None:
value = form_elem.select('option[value]')[0].get(
'value')
else:
# survey form just uses text, not value
for option in form_elem.select('option'):
if option.get('selected') is not None:
value = str(option.string)
# select the first option field if none is selected
if value is None:
value = str(form_elem.select('option')[0].string)
if key in inputs:
if isinstance(inputs[key], list):
# list input is accepted (for array uploads)
value = inputs[key]
else:
value = str(inputs[key])
if (key is not None): # and (value is not None):
if fmt == 'multipart/form-data':
if is_file:
payload.append(
(key, ('', '', 'application/octet-stream')))
else:
if type(value) is list:
for v in value:
entry = (key, ('', v))
# Prevent redundant key, value pairs
# (can happen if the form repeats them)
if entry not in payload:
payload.append(entry)
elif value is None:
entry = (key, ('', ''))
if entry not in payload:
payload.append(entry)
else:
entry = (key, ('', value))
if entry not in payload:
payload.append(entry)
else:
if type(value) is list:
for v in value:
entry = (key, v)
if entry not in payload:
payload.append(entry)
else:
entry = (key, value)
if entry not in payload:
payload.append(entry)
# for future debugging
self._payload = payload
log.debug("Form: payload={0}".format(payload))
if method is not None:
fmt = method
log.debug("Method/format = {0}".format(fmt))
# Send payload
if fmt == 'get':
response = self._request("GET", url, params=payload, cache=cache)
elif fmt == 'multipart/form-data':
response = self._request("POST", url, files=payload, cache=cache)
elif fmt == 'application/x-www-form-urlencoded':
response = self._request("POST", url, data=payload, cache=cache)
return response
def _single_orbit_find_actions(orbit, N_max, toy_potential=None,
force_harmonic_oscillator=False):
"""
Find approximate actions and angles for samples of a phase-space orbit,
`w`, at times `t`. Uses toy potentials with known, analytic action-angle
transformations to approximate the true coordinates as a Fourier sum.
This code is adapted from Jason Sanders'
`genfunc <https://github.com/jlsanders/genfunc>`_
.. todo::
Wrong shape for w -- should be (6,n) as usual...
Parameters
----------
orbit : `~gala.dynamics.Orbit`
N_max : int
Maximum integer Fourier mode vector length, |n|.
toy_potential : Potential (optional)
Fix the toy potential class.
force_harmonic_oscillator : bool (optional)
Force using the harmonic oscillator potential as the toy potential.
"""
if orbit.norbits > 1:
raise ValueError("must be a single orbit")
if toy_potential is None:
toy_potential = fit_toy_potential(orbit, force_harmonic_oscillator=force_harmonic_oscillator)
else:
logger.debug("Using *fixed* toy potential: {}".format(toy_potential.parameters))
if isinstance(toy_potential, IsochronePotential):
orbit_align = orbit.align_circulation_with_z()
w = orbit_align.w()
dxyz = (1,2,2)
circ = np.sign(w[0,0]*w[4,0]-w[1,0]*w[3,0])
sign = np.array([1.,circ,1.])
orbit = orbit_align
elif isinstance(toy_potential, HarmonicOscillatorPotential):
dxyz = (2,2,2)
sign = 1.
w = orbit.w()
else:
raise ValueError("Invalid toy potential.")
t = orbit.t.value
# Now find toy actions and angles
aaf = toy_potential.action_angle(orbit)
if aaf[0].ndim > 2:
aa = np.vstack((aaf[0].value[...,0], aaf[1].value[...,0]))
else:
aa = np.vstack((aaf[0].value, aaf[1].value))
if np.any(np.isnan(aa)):
ix = ~np.any(np.isnan(aa),axis=0)
aa = aa[:,ix]
t = t[ix]
warnings.warn("NaN value in toy actions or angles!")
if sum(ix) > 1:
raise ValueError("Too many NaN value in toy actions or angles!")
t1 = time.time()
A,b,nvecs = _action_prepare(aa, N_max, dx=dxyz[0], dy=dxyz[1], dz=dxyz[2])
actions = np.array(solve(A,b))
logger.debug("Action solution found for N_max={}, size {} symmetric"
" matrix in {} seconds"
.format(N_max,len(actions),time.time()-t1))
t1 = time.time()
A,b,nvecs = _angle_prepare(aa, t, N_max, dx=dxyz[0], dy=dxyz[1], dz=dxyz[2], sign=sign)
angles = np.array(solve(A,b))
logger.debug("Angle solution found for N_max={}, size {} symmetric"
" matrix in {} seconds"
.format(N_max,len(angles),time.time()-t1))
# Just some checks
if len(angles) > len(aa):
warnings.warn("More unknowns than equations!")
J = actions[:3] # * sign
theta = angles[:3]
freqs = angles[3:6] # * sign
return dict(actions=J*aaf[0].unit, angles=theta*aaf[1].unit, freqs=freqs*aaf[2].unit,
Sn=actions[3:], dSn_dJ=angles[6:], nvecs=nvecs)
