def download_hitran(m, i, numin, numax):
"""
Download HITRAN data for a particular molecule. Based on fetch function from
hapi.py.
Parameters
----------
m : int
HITRAN molecule number
i : int
HITRAN isotopologue number
numin : real
lower wavenumber bound
numax : real
upper wavenumber bound
"""
iso_id = str(ISO[(m, i)][ISO_INDEX["id"]])
mol_name = ISO[(m, i)][ISO_INDEX["mol_name"]]
filename = os.path.join(cache_location, "{0}.data".format(mol_name))
CHUNK = 64 * 1024
data = dict(iso_ids_list=iso_id, numin=numin, numax=numax)
with open(filename, "w") as fp:
response = commons.send_request(HITRAN_URL, data, 10, request_type="GET")
if "Content-Length" in response.headers:
total_length = response.headers.get("Content-Length")
pb = ProgressBar(int(total_length))
for chunk in response.iter_content(chunk_size=CHUNK):
fp.write(chunk.decode("utf-8"))
try:
pb.update(CHUNK)
except NameError:
pass
def init_progressbar(self):
"""
Initialise the progress bar.
This only happens if run command is called with ``progressbar=True``.
"""
self.progressbar = ProgressBar(self.command_count())
def _download_file(self, url, local_filepath, timeout=None, auth=None):
"""
Download a file. Resembles `astropy.utils.data.download_file` but uses
the local ``_session``
"""
response = self._session.get(url, timeout=timeout, stream=True,
auth=auth)
if 'content-length' in response.headers:
length = int(response.headers['content-length'])
else:
length = 1
pb = ProgressBar(length)
blocksize = astropy.utils.data.conf.download_block_size
bytes_read = 0
with open(local_filepath, 'wb') as f:
for block in response.iter_content(blocksize):
f.write(block)
bytes_read += blocksize
pb.update(bytes_read if bytes_read <= length else length)
response.close()
def ratio_to_dens_slow(ratio, c11, c22):
"""
Shape:
ratio [z,y,x]
c11 [y,x]
c22 [y,x]
"""
assert c11.size == c22.size == ratio[0,:,:].size
fshape = [ratio.shape[0], ratio.shape[1]*ratio.shape[2]]
rrs = ratio.reshape(fshape).T
outc = (ratio*0).reshape(fshape) + np.nan
# set up a grid...
pb = ProgressBar((c11.size))
for ii,(r,c1,c2) in enumerate(zip(rrs, c11.flat, c22.flat)):
#print r.shape,c1,c2
if np.isfinite(c1) and np.isfinite(c2) and np.any(np.isfinite(r)):
tauratio, ok = get_tau_ratio(c1,c2)
inds = np.argsort(tauratio[ok])
outc[:,ii] = np.interp(r, tauratio[ok][inds], dens[ok][inds], np.nan, np.nan)
pb.update()
#pb.finish()
return outc.reshape(ratio.shape)
def compute_transform(self, show_progress=True, scale_normalization=True,
use_pyfftw=False, threads=1, pyfftw_kwargs={}):
'''
Compute the wavelet transform at each scale.
Parameters
----------
show_progress : bool, optional
Show a progress bar during the creation of the covariance matrix.
scale_normalization: bool, optional
Compute the transform with the correct scale-invariant
normalization.
use_pyfftw : bool, optional
Enable to use pyfftw, if it is installed.
threads : int, optional
Number of threads to use in FFT when using pyfftw.
pyfftw_kwargs : Passed to
See `here <http://hgomersall.github.io/pyFFTW/pyfftw/builders/builders.html>`_
for a list of accepted kwargs.
'''
if use_pyfftw:
if PYFFTW_FLAG:
use_fftn = fftn
use_ifftn = ifftn
else:
warn("pyfftw not installed. Using numpy.fft functions.")
use_fftn = np.fft.fftn
use_ifftn = np.fft.ifftn
else:
use_fftn = np.fft.fftn
use_ifftn = np.fft.ifftn
n0, m0 = self.data.shape
A = len(self.scales)
self._Wf = np.zeros((A, n0, m0), dtype=np.float)
factor = 2
if not scale_normalization:
factor = 4
Warning("Transform values are only reliable with the proper scale"
" normalization. When disabled, the slope of the transform"
" CANNOT be used for physical interpretation.")
pix_scales = self._to_pixel(self.scales).value
if show_progress:
bar = ProgressBar(len(pix_scales))
for i, an in enumerate(pix_scales):
psi = MexicanHat2DKernel(an)
self._Wf[i] = \
convolve_fft(self.data, psi, normalize_kernel=False,
fftn=use_fftn, ifftn=use_ifftn).real * \
an**factor
if show_progress:
bar.update(i + 1)
def simulate(self, max_its=np.inf, max_time=np.inf * u.s):
"""Simulates the plasma as set up, either for the given number of
iterations or until the simulation reaches the given time.
Parameters
----------
max_its : int
Tells the simulation to run for a set number of iterations.
max_time : astropy.units.Quantity
Maximum total (in-simulation) time to allow the simulation to run.
Must have units of time.
Examples
--------
# >>> # Run a simulation for exactly one thousand iterations.
# >>> myplasma.simulate(max_time=1000)
# >>> # Run a simulation for up to half an hour of simulation time.
# >>> myplasma.simulate(max_time=30*u.minute)
"""
if np.isinf(max_its) and np.isinf(max_time.value):
raise ValueError("Either max_time or max_its must be set.")
physics = self.simulation_physics
dt = physics.dt
if np.isinf(max_time):
pb = ProgressBar(max_its)
else:
pb = ProgressBar(int(max_time / dt))
with pb as bar:
while (physics.current_iteration < max_its
and physics.current_time < max_time):
physics.time_stepper()
bar.update()
def extract_poly_slice(cube, polygons, width=1.0):
nx = len(polygons)
nz = cube.shape[0]
slice = np.zeros((nz, nx))
p = ProgressBar(len(polygons))
for i, polygon in enumerate(polygons):
p.update()
# Find bounding box
bbxmin = int(round(np.min(polygon.x))-1)
bbxmax = int(round(np.max(polygon.x))+2)
bbymin = int(round(np.min(polygon.y))-1)
bbymax = int(round(np.max(polygon.y))+2)
# Loop through pixels that might overlap
for xmin in np.arange(bbxmin, bbxmax):
for ymin in np.arange(bbymin, bbymax):
area = square_polygon_overlap_area(xmin-0.5, xmin+0.5,
ymin-0.5, ymin+0.5,
polygon.x, polygon.y)
if area > 0:
slice[:, i] += cube[:, ymin, xmin] * area
print("")
return slice
def flat_cubes(date, lbda_min=7000, lbda_max=9000, ref="dome"):
""" """
baseroot = io.CUBE_PROD_ROOTS["cube"]["root"]
newroot = io.CUBE_PROD_ROOTS["flat"]["root"]
# -------------- #
# The Reference #
# -------------- #
reffile = io.get_night_cubes(date, kind="cube", target=ref)
if len(reffile) == 0:
raise ValueError("No cube reference for target %s in night %s" %
(ref, date))
refcube = get_sedmcube(reffile[0])
flatfied = refcube.get_slice(lbda_min, lbda_max, usemean=True)
print(flatfied.mean())
# ----------------- #
# Build flat cubes #
# ----------------- #
def build_flat_cube(cubefile):
cube_ = get_sedmcube(cubefile)
print(cubefile)
cube_.scale_by(flatfied)
cube_.writeto(cube_.filename.replace(baseroot, newroot))
from astropy.utils.console import ProgressBar
cubefiles = io.get_night_cubes(date, kind="cube")
print(cubefiles)
ProgressBar.map(build_flat_cube, cubefiles)
def create_modelcube(self, njobs=1, verbose=True):
"""
Generates a "clean" datacube from the scousepy decomposition. Returns a
clean cube
Parameters
----------
self : instance of the scousepy class
njobs : Number
number of cpus
verbose: bool
verbose output
"""
# Time it
starttime = time.time()
cube = self.cube
x = np.array(cube.world[:, 0, 0][0])
if (self.ppv_vol[0] is not None) & (self.ppv_vol[1] is not None):
trimids = np.where((x > self.ppv_vol[0]) & (x < self.ppv_vol[1]))[0]
_cube = cube[min(trimids):max(trimids) + 1, :, :]
_modelcube = np.full_like(_cube, np.nan)
if verbose:
print("")
print("Generating models:")
print("")
args = [self]
inputs = [[key] + args for key in self.indiv_dict.keys()]
if njobs == 1:
mods = ProgressBar.map(genmodel, inputs)
else:
mods = parallel_map(genmodel, inputs, numcores=njobs)
mergedmods = [mod for mod in mods]
mergedmods = np.asarray(mergedmods)
if verbose:
print("")
print("Creating model cube:")
print("")
progress_bar = ProgressBar(self.indiv_dict.keys())
for i, key in enumerate(self.indiv_dict.keys()):
_modelcube[:, self.indiv_dict[key].coordinates[0],
self.indiv_dict[key].coordinates[1]] = mergedmods[i]
if verbose:
progress_bar.update()
endtime = time.time()
if verbose:
print("")
print('Process completed in: {0} minutes'.format(
(endtime - starttime) / 60.))
print("")
return SpectralCube(data=_modelcube, wcs=_cube.wcs)
def download_hitran(m, i, numin, numax):
"""
Download HITRAN data for a particular molecule. Based on fetch function from
hapi.py.
Parameters
----------
m : int
HITRAN molecule number
i : int
HITRAN isotopologue number
numin : real
lower wavenumber bound
numax : real
upper wavenumber bound
"""
iso_id = str(ISO[(m, i)][ISO_INDEX['id']])
mol_name = ISO[(m, i)][ISO_INDEX['mol_name']]
filename = os.path.join(cache_location, '{0}.data'.format(mol_name))
CHUNK = 64 * 1024
data = dict(iso_ids_list=iso_id, numin=numin, numax=numax)
with open(filename, 'w') as fp:
response = commons.send_request(HITRAN_URL,
data,
10,
request_type='GET')
if 'Content-Length' in response.headers:
total_length = response.headers.get('Content-Length')
pb = ProgressBar(int(total_length))
for chunk in response.iter_content(chunk_size=CHUNK):
fp.write(chunk.decode('utf-8'))
try:
pb.update(CHUNK)
except NameError:
pass
def generate_2d_parametermap(scouseobject, spectrum_parameter, verbose=False):
"""
Create a 2D map of a given spectral parameter
Parameters
----------
scouseobject : Instance of the scousepy class
"""
blankmap = np.zeros(scouseobject.cube.shape[1:])
blankmap[:] = np.nan
if verbose:
print("")
progress_bar = ProgressBar(len(scouseobject.indiv_dict.items()))
for ind, spec in scouseobject.indiv_dict.items():
cy, cx = spec.coordinates
if getattr(spec.model, 'ncomps') != 0:
blankmap[cy, cx] = getattr(spec.model, spectrum_parameter)
else:
blankmap[cy, cx] = np.nan
if verbose:
progress_bar.update()
return blankmap
def surface_solution_numerical(): # inward extension of J=-J_p at r=R.
global n, Jp, kx, L, delta, radius, s, sigmai, phix_full
global Js, Jscheck, Hs
Js = -Jp
amps = np.zeros(n, dtype=np.cdouble)
Jprefac = np.sqrt(6.0) / (16.0 * np.pi**3) * kx**2 * L / delta
amps = -np.pi * Jprefac / (kx * radius) * np.exp(-kx * radius * np.abs(s) -
1j * s * sigmai)
Jscheck = np.zeros(n, dtype=np.cdouble)
Hs = np.zeros(n, dtype=np.cdouble)
print('Solving surface solution...\n')
pb = ProgressBar(n * n)
for i in range(n):
for j in range(n):
z = np.abs(kx * radius * s[j])
i0 = spherical_in(0, z, derivative=False)
di0 = spherical_in(0, z, derivative=True)
rat = di0 / i0
Hs[i] = Hs[i] + (ds / 2.0 / np.pi) * np.exp(
1j * s[j] * sigma[i]) * amps[j] * (-np.abs(s[j]) * rat / 3.0 /
phix_full[i])
Jscheck[i] = Jscheck[i] + (ds / 2.0 / np.pi) * np.exp(
1j * s[j] * sigma[i]) * amps[j]
pb.update()
def fit_ch3cn_lines(spectra, save_prefix, velo=56*u.km/u.s, ampguess=-0.01):
all_ch3cn = table.vstack([ch3cn, ch3cn_v])
all_ch3cn.add_column(table.Column(name='FittedAmplitude', data=np.zeros(len(all_ch3cn))))
all_ch3cn.add_column(table.Column(name='FittedCenter', data=np.zeros(len(all_ch3cn))))
all_ch3cn.add_column(table.Column(name='FittedWidth', data=np.zeros(len(all_ch3cn))))
all_ch3cn.add_column(table.Column(name='FittedAmplitudeError', data=np.zeros(len(all_ch3cn))))
all_ch3cn.add_column(table.Column(name='FittedCenterError', data=np.zeros(len(all_ch3cn))))
all_ch3cn.add_column(table.Column(name='FittedWidthError', data=np.zeros(len(all_ch3cn))))
vkms = velo.to(u.km/u.s).value
pl.figure(1).clf()
ax = pl.gca()
pb = ProgressBar(len(spectra) * len(all_ch3cn))
ii = 0
for sp in spectra:
sp.xarr.convert_to_unit(u.GHz)
mid = np.median(sp.data)
for line in all_ch3cn:
frq = line['Freq-GHz']*u.GHz
if sp.xarr.in_range(frq*(1-velo/constants.c)):
offset = ii*0.000 + mid
ii += 1
sp.xarr.convert_to_unit(u.km/u.s, refX=frq)
sp.plotter(axis=ax, clear=False, offset=offset)
sp.specfit(fittype='vheightgaussian',
guesses=[mid, ampguess, vkms, 2],)
line['FittedAmplitude'] = sp.specfit.parinfo['AMPLITUDE0'].value
line['FittedCenter'] = sp.specfit.parinfo['SHIFT0'].value
line['FittedWidth'] = sp.specfit.parinfo['WIDTH0'].value
line['FittedAmplitudeError'] = sp.specfit.parinfo['AMPLITUDE0'].error
line['FittedCenterError'] = sp.specfit.parinfo['SHIFT0'].error
line['FittedWidthError'] = sp.specfit.parinfo['WIDTH0'].error
sp.xarr.convert_to_unit(u.GHz)
pb.update()
pl.xlim(vkms-14, vkms+14)
#pl.ylim(0, offset)
pl.draw()
pl.show()
pl.savefig(save_prefix+"_spectra_overlay.png")
pl.figure(2).clf()
pl.plot(all_ch3cn['E_U (K)'], all_ch3cn['FittedWidth'], 'o')
pl.xlabel("E$_U$ (K)")
pl.ylabel("$\sigma$ (km/s)")
pl.ylim(0,3.5)
pl.savefig(save_prefix+"_sigma_vs_eupper.png")
pl.figure(3).clf()
pl.plot(all_ch3cn['E_U (K)'], all_ch3cn['FittedCenter'], 'o')
pl.xlabel("E$_U$ (K)")
pl.ylabel("$v_{lsr}$ (km/s)")
pl.ylim(vkms-3, vkms+3)
pl.savefig(save_prefix+"_vcen_vs_eupper.png")
def _map(self, func, items):
# FIXME: ProgressBar.map(..., multiprocess=True) uses imap_unordered,
# but we want the result to come back in order. This should be fixed,
# or at least correctly documented, in Astropy.
if self.multiprocess:
_, result = zip(*sorted(ProgressBar.map(_mapfunc(func),
list(enumerate(items)),
multiprocess=True)))
return list(result)
else:
return ProgressBar.map(func, items, multiprocess=False)
def spectral_regrid(cube, outgrid):
"""
Spectrally regrid a cube onto a new spectral output grid
(this is apparently redundant with regrid_cube_hdu)
"""
assert isinstance(cube, SpectralCube)
inaxis = cube.spectral_axis.to(outgrid.unit)
indiff = np.mean(np.diff(inaxis))
outdiff = np.mean(np.diff(outgrid))
if outdiff < 0:
outgrid = outgrid[::-1]
outdiff = np.mean(np.diff(outgrid))
if indiff < 0:
cubedata = cube.filled_data[::-1]
inaxis = cube.spectral_axis.to(outgrid.unit)[::-1]
indiff = np.mean(np.diff(inaxis))
else:
cubedata = cube.filled_data[:]
if indiff < 0 or outdiff < 0:
raise ValueError("impossible.")
assert np.all(np.diff(outgrid) > 0)
assert np.all(np.diff(inaxis) > 0)
np.testing.assert_allclose(np.diff(outgrid),
outdiff,
err_msg="Output grid must be linear")
if outdiff > 2 * indiff:
raise ValueError("Input grid has too small a spacing. It needs to be "
"smoothed prior to resampling.")
newcube = np.empty([outgrid.size, cube.shape[1], cube.shape[2]])
yy, xx = np.indices(cube.shape[1:])
pb = ProgressBar(xx.size)
for ix, iy in (zip(xx.flat, yy.flat)):
newcube[:, iy, ix] = np.interp(outgrid.value, inaxis.value,
cubedata[:, iy, ix].value)
pb.update()
newheader = cube.header
newheader['CRPIX3'] = 1
newheader['CRVAL3'] = outgrid[0].value
newheader['CDELT3'] = outdiff.value
newheader['CUNIT3'] = outgrid.unit.to_string('FITS')
return fits.PrimaryHDU(data=newcube, header=newheader)
def spectral_regrid(cube, outgrid):
"""
Spectrally regrid a cube onto a new spectral output grid
(this is apparently redundant with regrid_cube_hdu)
"""
assert isinstance(cube, SpectralCube)
inaxis = cube.spectral_axis.to(outgrid.unit)
indiff = np.mean(np.diff(inaxis))
outdiff = np.mean(np.diff(outgrid))
if outdiff < 0:
outgrid=outgrid[::-1]
outdiff = np.mean(np.diff(outgrid))
if indiff < 0:
cubedata = cube.filled_data[::-1]
inaxis = cube.spectral_axis.to(outgrid.unit)[::-1]
indiff = np.mean(np.diff(inaxis))
else:
cubedata = cube.filled_data[:]
if indiff < 0 or outdiff < 0:
raise ValueError("impossible.")
assert np.all(np.diff(outgrid) > 0)
assert np.all(np.diff(inaxis) > 0)
np.testing.assert_allclose(np.diff(outgrid), outdiff,
err_msg="Output grid must be linear")
if outdiff > 2 * indiff:
raise ValueError("Input grid has too small a spacing. It needs to be "
"smoothed prior to resampling.")
newcube = np.empty([outgrid.size, cube.shape[1], cube.shape[2]])
yy,xx = np.indices(cube.shape[1:])
pb = ProgressBar(xx.size)
for ix, iy in (zip(xx.flat, yy.flat)):
newcube[:,iy,ix] = np.interp(outgrid.value, inaxis.value,
cubedata[:,iy,ix].value)
pb.update()
newheader = cube.header
newheader['CRPIX3'] = 1
newheader['CRVAL3'] = outgrid[0].value
newheader['CDELT3'] = outdiff.value
newheader['CUNIT3'] = outgrid.unit.to_string('FITS')
return fits.PrimaryHDU(data=newcube, header=newheader)
def mass_photometry(fname, outfile):
filenames = glob('/lustre/aoc/students/bmcclell/w51/' + fname)
objs = []
print('Loading files')
pb = ProgressBar(len(filenames))
for f in filenames:
rs = dendrocat.RadioSource(fits.open(f))
objs.append(rs)
pb.update()
#n = np.shape(objs[0].data)[0]
#center = regions.PixCoord(n/2, n/2)
#reg = regions.CirclePixelRegion(center, 3200)
#mask = reg.to_mask()
#img = mask.to_image((n, n)).astype(bool)
#objs[0].data = np.where(img==True, objs[0].data, np.nan)
# Debugging
#plt.figure()
#plt.imshow(objs[0].data)
#plt.show()
#objs[0].threshold = 4.5
#objs[0].min_value = 1.1e-4
#objs[0].min_delta = 1.2*objs[0].min_value
#objs[0].to_catalog()
#objs[0].autoreject()
#objs[0].reject([44001, 44032])
#dendrocat.utils.save_regions(objs[0].catalog, '/users/bmcclell/nrao/reg/test_mass_photometry.reg')
#print('Autorejection complete')
t = Table.read('/users/bmcclell/nrao/cat/w51IRS2_photometered.dat',
format='ascii')
#for col in t.colnames:
# if 'GHz' in col:
# t.remove_column(col)
mc = dendrocat.MasterCatalog(*objs, catalog=t)
print('\nMaster Catalog made')
start = time.time()
mc.photometer(dendrocat.ellipse)
stop = time.time()
print('Ellipse apertures photometered. Time: {} s'.format(stop - start))
start = time.time()
mc.photometer(dendrocat.annulus)
stop = time.time()
print('Annulus apertures photometered. Time: {} s'.format(stop - start))
start = time.time()
mc.catalog.write(outfile, format='ascii', overwrite=True)
stop = time.time()
print('Catalog written. Time: {} s'.format(stop - start))
def download_file(url, outdir=rawpath):
r = requests.get(url, verify=False, stream=True)
_, cdisp = cgi.parse_header(r.headers['content-disposition'])
outfilename = cdisp['filename']
fullname = os.path.join(outdir, outfilename)
pb = ProgressBar(int(r.headers['content-length']))
with open(fullname, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
f.write(chunk)
f.flush()
pb.update(pb._current_value + 1024)
return fullname
def download_file(url, outdir=rawpath):
r = requests.get(url, verify=False, stream=True)
_, cdisp = cgi.parse_header(r.headers['content-disposition'])
outfilename = cdisp['filename']
fullname = os.path.join(outdir, outfilename)
pb = ProgressBar(int(r.headers['content-length']))
with open(fullname, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
f.write(chunk)
f.flush()
pb.update(pb._current_value + 1024)
return fullname
def get_progressbar(gen, notebook=False):
"""
"""
from astropy.utils.console import ProgressBar
if not notebook:
gen = ProgressBar(gen)
else:
try:
gen = ProgressBar(gen, ipython_widget=True)
except ImportError as e:
warnings.warn('ProgressBar in notebook not working. Is ipywidgets installed?')
raise e
return gen
def fit_all_tex(xaxis, cube, cubefrequencies, indices, degeneracies,
ecube=None,
replace_bad=False):
"""
Parameters
----------
replace_bad : bool
Attempt to replace bad (negative) values with their upper limits?
"""
tmap = np.empty(cube.shape[1:])
Nmap = np.empty(cube.shape[1:])
yy,xx = np.indices(cube.shape[1:])
pb = ProgressBar(xx.size)
count=0
for ii,jj in (zip(yy.flat, xx.flat)):
if any(np.isnan(cube[:,ii,jj])):
tmap[ii,jj] = np.nan
else:
if replace_bad:
uplims = nupper_of_kkms(replace_bad, cubefrequencies,
einsteinAij[indices], degeneracies,).value
else:
uplims = None
nuppers = nupper_of_kkms(cube[:,ii,jj], cubefrequencies,
einsteinAij[indices], degeneracies,
)
if ecube is not None:
nupper_error = nupper_of_kkms(ecube[:,ii,jj], cubefrequencies,
einsteinAij[indices], degeneracies,).value
uplims = 3 * nupper_error
if replace_bad:
raise ValueError("replace_bad is ignored now...")
else:
nupper_error = None
fit_result = fit_tex(xaxis, nuppers.value,
errors=nupper_error,
uplims=uplims)
tmap[ii,jj] = fit_result[1].value
Nmap[ii,jj] = fit_result[0].value
pb.update(count)
count+=1
return tmap,Nmap
def extract_poly_slice(cube, polygons):
nx = len(polygons)
nz = cube.shape[0]
total_slice = np.zeros((nz, nx))
total_area = np.zeros((nz, nx))
p = ProgressBar(len(polygons))
for i, polygon in enumerate(polygons):
p.update()
# Find bounding box
bbxmin = int(round(np.min(polygon.x)) - 1)
bbxmax = int(round(np.max(polygon.x)) + 2)
bbymin = int(round(np.min(polygon.y)) - 1)
bbymax = int(round(np.max(polygon.y)) + 2)
# Clip to cube box
bbxmin = max(bbxmin, 0)
bbxmax = min(bbxmax, cube.shape[2])
bbymin = max(bbymin, 0)
bbymax = min(bbymax, cube.shape[1])
# Loop through pixels that might overlap
for xmin in np.arange(bbxmin, bbxmax):
for ymin in np.arange(bbymin, bbymax):
area = square_polygon_overlap_area(xmin - 0.5, xmin + 0.5,
ymin - 0.5, ymin + 0.5,
polygon.x, polygon.y)
if area > 0:
dataslice = cube[:, ymin, xmin]
good_values = np.isfinite(dataslice)
if np.any(good_values):
total_slice[good_values,
i] += dataslice[good_values] * area
total_area[good_values, i] += area
total_slice[total_area == 0.] = np.nan
total_slice[total_area > 0.] /= total_area[total_area > 0.]
print("")
return total_slice
def fit_all_tex(xaxis, cube, cubefrequencies, indices, degeneracies,
ecube=None,
replace_bad=False):
"""
Parameters
----------
replace_bad : bool
Attempt to replace bad (negative) values with their upper limits?
"""
tmap = np.empty(cube.shape[1:])
Nmap = np.empty(cube.shape[1:])
yy,xx = np.indices(cube.shape[1:])
pb = ProgressBar(xx.size)
count=0
for ii,jj in (zip(yy.flat, xx.flat)):
if any(np.isnan(cube[:,ii,jj])):
tmap[ii,jj] = np.nan
else:
if replace_bad:
uplims = nupper_of_kkms(replace_bad, cubefrequencies,
einsteinAij[indices], degeneracies,).value
else:
uplims = None
nuppers = nupper_of_kkms(cube[:,ii,jj], cubefrequencies,
einsteinAij[indices], degeneracies,
)
if ecube is not None:
nupper_error = nupper_of_kkms(ecube[:,ii,jj], cubefrequencies,
einsteinAij[indices], degeneracies,).value
uplims = 3 * nupper_error
if replace_bad:
raise ValueError("replace_bad is ignored now...")
else:
nupper_error = None
fit_result = fit_tex(xaxis, nuppers.value,
errors=nupper_error,
uplims=uplims)
tmap[ii,jj] = fit_result[1].value
Nmap[ii,jj] = fit_result[0].value
pb.update(count)
count+=1
return tmap,Nmap
def _convolve_model_dir_2(model_dir, filters, overwrite=False):
for f in filters:
if f.name is None:
raise Exception("filter name needs to be set")
if f.central_wavelength is None:
raise Exception("filter central wavelength needs to be set")
# Create 'convolved' sub-directory if needed
if not os.path.exists(model_dir + '/convolved'):
os.mkdir(model_dir + '/convolved')
# Find all SED files to convolve
sed_cube = SEDCube.read(os.path.join(model_dir, 'flux.fits'), order='nu')
par_table = load_parameter_table(model_dir)
if not np.all(par_table['MODEL_NAME'] == sed_cube.names):
raise ValueError(
"Model names in SED cube and parameter file do not match")
log.info("{0} SEDs found in {1}".format(sed_cube.n_models, model_dir))
# Set up convolved fluxes
fluxes = [
ConvolvedFluxes(model_names=sed_cube.names,
apertures=sed_cube.apertures,
initialize_arrays=True) for i in range(len(filters))
]
# Set up list of binned filters
binned_filters = [f.rebin(sed_cube.nu) for f in filters]
# We do the unit conversion - if needed - at the last minute
val_factor = sed_cube.val.unit.to(u.mJy)
unc_factor = sed_cube.unc.unit.to(u.mJy)
# Loop over apertures
for i_ap in ProgressBar(range(sed_cube.n_ap)):
sed_val = sed_cube.val[i_ap].transpose()
sed_unc = sed_cube.val[i_ap].transpose()
for i, f in enumerate(binned_filters):
response = f.response.astype(sed_val.dtype)
fluxes[i].flux[:, i_ap] = np.sum(sed_val * response,
axis=1) * val_factor
fluxes[i].error[:, i_ap] = np.sqrt(
np.sum((sed_unc * response)**2, axis=1)) * unc_factor
for i, f in enumerate(binned_filters):
fluxes[i].central_wavelength = f.central_wavelength
fluxes[i].apertures = sed_cube.apertures
fluxes[i].model_names = sed_cube.names
fluxes[i].write(model_dir + '/convolved/' + f.name + '.fits',
overwrite=overwrite)
def calculate_emissivity(self, savefile, **kwargs):
"""
Calculate and store emissivity for every ion in the model
"""
notebook = kwargs.get('notebook', True)
self.emissivity_savefile = savefile
with h5py.File(savefile, 'w') as hf:
with ProgressBar(len(self._ion_list),
ipython_widget=notebook) as progress:
for ion in self:
pop = ion.level_populations(self.density)
# NOTE: populations not available for every ion
if pop is None:
warnings.warn(
f'Cannot compute level populations for {ion.ion_name}'
)
continue
upper_level = ion.transitions.upper_level[~ion.transitions.
is_twophoton]
wavelength = ion.transitions.wavelength[~ion.transitions.
is_twophoton]
A = ion.transitions.A[~ion.transitions.is_twophoton]
i_upper = fiasco.util.vectorize_where(
ion._elvlc['level'], upper_level)
emissivity = pop[:, :, i_upper] * A * u.photon
emissivity = emissivity[:, :, np.argsort(wavelength)]
wavelength = np.sort(wavelength)
grp = hf.create_group(ion.ion_name)
ds = grp.create_dataset('wavelength',
data=wavelength.value)
ds.attrs['units'] = wavelength.unit.to_string()
ds = grp.create_dataset('emissivity', data=emissivity.data)
ds.attrs['units'] = emissivity.unit.to_string()
progress.update()
def run(self):
SERVER = os.environ["PY4SCI_SERVER"]
USER = os.environ["PY4SCI_USER"]
import getpass
from ftplib import FTP
from astropy.utils.console import ProgressBar
ftp = FTP(SERVER)
ftp.login(user=USER, passwd=getpass.getpass())
ftp.cwd('/public_html/PY4SCI_WS_2013_14')
for slides in ProgressBar.iterate(
glob.glob('lectures/*.html') + ['lectures/custom.css'] +
glob.glob('problems/data/*') + glob.glob('problems/*.html') +
glob.glob('practice/data/*') + glob.glob('practice/*.html')):
try:
remote_size = ftp.size(slides)
except:
remote_size = None
local_size = os.path.getsize(slides)
if local_size != remote_size:
ftp.storbinary('STOR ' + slides, open(slides, 'rb'))
ftp.storbinary('STOR index.html', open('index.html', 'rb'))
ftp.quit()
def run(self):
SERVER = os.environ["PY4SCI_SERVER"]
USER = os.environ["PY4SCI_USER"]
import getpass
from ftplib import FTP
from astropy.utils.console import ProgressBar
ftp = FTP(SERVER)
ftp.login(user=USER, passwd=getpass.getpass())
ftp.cwd('/public_html/PY4SCI_WS_2013_14')
for slides in ProgressBar.iterate(glob.glob('lectures/data/*')
+ glob.glob('lectures/*.html')
+ ['lectures/custom.css']
+ glob.glob('problems/data/*')
+ glob.glob('problems/*.html')
+ glob.glob('practice/data/*')
+ glob.glob('practice/*.html')):
try:
remote_size = ftp.size(slides)
except:
remote_size = None
local_size = os.path.getsize(slides)
if local_size != remote_size:
ftp.storbinary('STOR ' + slides, open(slides, 'rb'))
ftp.storbinary('STOR index.html', open('index.html', 'rb'))
ftp.quit()
def run(self):
SERVER = os.environ["PY4SCI_SERVER"]
USER = os.environ["PY4SCI_USER"]
import getpass
from ftplib import FTP
from astropy.utils.console import ProgressBar
ftp = FTP(SERVER)
ftp.login(user=USER, passwd=getpass.getpass())
ftp.cwd('/public_html/astropy4herts')
for slides in ProgressBar.iterate(glob.glob('notebooks/data/*')
+ glob.glob('notebooks/*.html')):
try:
remote_size = ftp.size(slides)
except:
remote_size = None
local_size = os.path.getsize(slides)
if local_size != remote_size:
ftp.storbinary('STOR ' + slides, open(slides, 'rb'))
ftp.storbinary('STOR notebooks.html', open('notebooks.html', 'rb'))
ftp.quit()
def run(self, maxiter = 4, verbose = False):
"""
Full artillery :-)
- Find saturated stars
- Run maxiter L.A.Cosmic iterations (stops if no more cosmics are found)
Stops if no cosmics are found or if maxiter is reached.
"""
if self.satlevel > 0 and self.satstars is None:
self.findsatstars(verbose=verbose)
if verbose:
print("Starting %i L.A.Cosmic iterations ..." % maxiter)
for i in ProgressBar(range(1, maxiter+1)):
if verbose:
print("Iteration %i" % i)
iterres = self.lacosmiciteration(verbose=verbose)
if verbose:
print("%i cosmic pixels (%i new)" % (iterres["niter"], iterres["nnew"]))
#self.clean(mask = iterres["mask"]) # No, we want clean to operate on really clean pixels only !
# Thus we always apply it on the full mask, as lacosmic does :
self.clean(verbose=verbose)
# But note that for huge cosmics, one might want to revise this.
# Thats why I added a feature to skip saturated stars !
if iterres["niter"] == 0:
break
def _slow_reader(index_map, data):
"""
Loop over each valid pixel in the index_map and add its coordinates and
data to the flux_by_structure and indices_by_structure dicts
This is slower than _fast_reader but faster than that implementation would
be without find_objects. The bottleneck is doing `index_map == idx` N
times.
"""
flux_by_structure, indices_by_structure = {}, {}
# Do a fast iteration through d.data, adding the indices and data values
# to the two dictionaries declared above:
indices = np.array(np.where(index_map > -1)).transpose()
log.debug('Creating index maps for {0} coordinates...'.format(
len(indices)))
for coord in ProgressBar(indices):
coord = tuple(coord)
idx = index_map[coord]
if idx in flux_by_structure:
flux_by_structure[idx].append(data[coord])
indices_by_structure[idx].append(coord)
else:
flux_by_structure[idx] = [data[coord]]
indices_by_structure[idx] = [coord]
return flux_by_structure, indices_by_structure
def write_data_to_csv(data, filename, mode="new"):
"""
Write a dataset to a CSV-formatted file with a data header.
Parameters
----------
data : `dict`
The data to be written as a dictionary of NumPy arrays
filename : `str`
The filename to write the data to.
mode : {'new', 'append'}, optional
Write a "new" file or "append" to an existing file?
"""
if mode == "new":
fmode = "w"
elif mode == "append":
fmode = "a+"
if sys.version_info >= (3, 0, 0):
f = open(filename, fmode, newline='')
else:
f = open(filename, fmode + 'b')
w = csv.DictWriter(f, list(data.keys()))
if mode == "new":
w.writeheader()
num_points = len(list(data.values())[0])
for i in ProgressBar(list(range(num_points))):
row = dict([(k, v[i]) for k, v in list(data.items())])
w.writerow(row)
f.close()
def init_progressbar(self):
"""
Initialise the progress bar.
This only happens if run command is called with ``progressbar=True``.
"""
self.progressbar = ProgressBar(self.command_count())
def get_molecules(self, cache=True):
"""
Scrape the list of valid molecules
"""
if cache and hasattr(self, '_molecule_dict'):
return self._molecule_dict
elif cache and os.path.isfile(self.moldict_path):
with open(self.moldict_path, 'r') as f:
md = json.load(f)
return md
main_url = 'http://home.strw.leidenuniv.nl/~moldata/'
response = self._request('GET', main_url, cache=cache)
response.raise_for_status()
soup = BeautifulSoup(response.content)
links = soup.find_all('a', href=True)
datfile_urls = [
url for link in ProgressBar(links)
for url in self._find_datfiles(link['href'], base_url=main_url)
]
molecule_re = re.compile(
r'http://[a-zA-Z0-9.]*/~moldata/datafiles/([A-Z0-9a-z_+@-]*).dat')
molecule_dict = {
molecule_re.search(url).groups()[0]: url
for url in datfile_urls
}
with open(self.moldict_path, 'w') as f:
s = json.dumps(molecule_dict)
f.write(s)
return molecule_dict
def kpub_import(args=None):
"""Import publications from a csv file.
The csv file must contain entries of the form "bibcode,mission,science".
The actual metadata of each publication will be grabbed using the ADS API,
hence this routine may take 10-20 minutes to complete.
"""
parser = argparse.ArgumentParser(
description="Batch-import papers into the Kepler/K2 publication list "
"from a CSV file. The CSV file must have three columns "
"(bibcode,mission,science) separated by commas. "
"For example: '2004ApJ...610.1199G,kepler,astrophysics'.")
parser.add_argument('-f',
metavar='dbfile',
type=str,
default=DEFAULT_DB,
help="Location of the Kepler/K2 publication list db. "
"Defaults to ~/.kpub.db.")
parser.add_argument('csvfile', help="Filename of the csv file to ingest.")
args = parser.parse_args(args)
db = PublicationDB(args.f)
for line in ProgressBar(open(args.csvfile, 'r').readlines()):
col = line.split(',') # Naive csv parsing
db.add_by_bibcode(col[0], mission=col[1], science=col[2].strip())
def pymedian(self, header=None):
try:
import fitsio
except ImportError:
self._logger.error('fitsio is required !')
raise
data = [fitsio.FITS(f)[1] for f in self.files]
# shape = data[0].get_dims()
cube = np.empty(self.shape, dtype=np.float64)
expmap = np.empty(self.shape, dtype=np.int32)
valid_pix = np.zeros(self.nfiles, dtype=np.int32)
nl = self.shape[0]
self._logger.info('Looping on the %d planes of the cube', nl)
for l in ProgressBar(range(nl)):
arr = np.array([c[l, :, :][0] for c in data])
cube[l, :, :] = np.nanmedian(arr, axis=0)
expmap[l, :, :] = (~np.isnan(arr)).astype(int).sum(axis=0)
valid_pix += (~np.isnan(arr)).astype(int).sum(axis=1).sum(axis=1)
# no valid pixels
npixels = np.prod(self.shape)
no_valid_pix = npixels - valid_pix
stat_pix = Table([self.files, no_valid_pix],
names=['FILENAME', 'NPIX_NAN'])
kwargs = dict(expnb=_compute_expnb(expmap),
header=header,
method='obj.cubelist.pymedian')
expmap = self.save_combined_cube(expmap,
unit=u.dimensionless_unscaled,
**kwargs)
cube = self.save_combined_cube(cube, **kwargs)
return cube, expmap, stat_pix
def _get_barycorr_bvcs_withvels(coos, loc, injupyter=False):
"""
Gets the barycentric correction of the test data from the
http://astroutils.astronomy.ohio-state.edu/exofast/barycorr.html web site.
Requires the https://github.com/tronsgaard/barycorr python interface to that
site.
Provided to reproduce the test data above, but not required to actually run
the tests.
"""
import barycorr
from astropy.utils.console import ProgressBar
bvcs = []
for coo in ProgressBar(coos, ipython_widget=injupyter):
res = barycorr.bvc(test_input_time.utc.jd,
coo.ra.deg,
coo.dec.deg,
lat=loc.geodetic[1].deg,
lon=loc.geodetic[0].deg,
pmra=coo.pm_ra_cosdec.to_value(u.mas / u.yr),
pmdec=coo.pm_dec.to_value(u.mas / u.yr),
parallax=coo.distance.to_value(
u.mas, equivalencies=u.parallax()),
rv=coo.radial_velocity.to_value(u.m / u.s),
epoch=test_input_time.utc.jd,
elevation=loc.geodetic[2].to(u.m).value)
bvcs.append(res)
return bvcs * u.m / u.s
def run(self, obs_list, selection=None):
"""
Run MapMaker for a list of observations to create
stacked counts, exposure and background maps
Parameters
--------------
obs_list : `~gammapy.data.ObservationList`
List of observations
selection : list
List of str, selecting which maps to make.
Available: 'counts', 'exposure', 'background'
By default, all maps are made.
Returns
-----------
maps: dict of stacked counts, background and exposure maps.
"""
selection = _check_selection(selection)
# Initialise zero-filled maps
for name in selection:
unit = 'm2 s' if name == 'exposure' else ''
self.maps[name] = Map.from_geom(self.geom, unit=unit)
for obs in ProgressBar(obs_list):
try:
self._process_obs(obs, selection)
except NoOverlapError:
log.info(
'Skipping observation {}, not contained in map.'.format(
obs.obs_id))
continue
return self.maps
def _find_and_fit_peaks(self):
"""Find and fit peaks in each PSD. This can be done in parallell if requested.
Configuration Items:
- ``FMTS.fitting`` The dictionary of parameters used for :func`find_and_fit_peaks`.
- ``FMTS.multiprocess`` `(bool)` whether to parallelize.
Peaks are stored in an object array, and the number of peaks at each mode is stored in a separate, parallel array.
"""
from astropy.utils.console import ProgressBar
kwargs = dict(self.config["FMTS.fitting"])
psd = self.psd
template = self.template_ft
omega = self.omega
args = [ ((k,l),psd[:,k,l],template,omega,kwargs) for k,l in itertools.product(range(self.psd.shape[1]),range(self.psd.shape[2])) ]
peaks = ProgressBar.map(pool_find_and_fit_peaks_in_modes,args,multiprocess=self.config["FMTS.multiprocess"])
for peak_mode,ident in peaks:
k,l = ident
self.peaks[k,l] = peak_mode
self.npeaks[k,l] = len(peak_mode)
self.log.info("Found %d peaks",np.sum(self.npeaks))
def generate_calibrated_images(self,
output_dir,
output_suffix='corrected'):
"""
Dark subtract and flat field all data images, save them to `output_dir`.
This is useful for producing images that can be passed to astrometry.net
"""
n_images = len(self.data_image_paths)
print("Generating calibrated images...")
with ProgressBar(n_images) as bar:
for i, image_path in enumerate(self.data_image_paths):
bar.update(i)
image_header = fits.getheader(image_path)
image_data = ((fits.getdata(image_path) - self.master_dark) /
self.master_flat)
# TODO: Should this line be in our out?
#image_data[image_data < 0] = np.median(image_data)
file_name = image_path.split(os.sep)[-1].split('.fits')[0]
fits.writeto(os.path.join(output_dir,
file_name + output_suffix + '.fits'),
image_data,
clobber=True,
header=image_header)
def get_lightcurves(self, progress_bar=False, notebook=False):
"""
"""
if not self.is_set():
raise AttributeError("plan, generator or instrument not set")
lcs = LightcurveCollection(empty=True)
gen = izip(self.generator.get_lightcurve_full_param(),
self._get_observations_())
if progress_bar:
self._assign_obs_fields_(progress_bar=True, notebook=notebook)
self._assign_non_field_obs_(progress_bar=True, notebook=notebook)
print 'Generating lightcurves'
with ProgressBar(self.generator.ntransient,
ipython_widget=notebook) as bar:
for k, (p, obs) in enumerate(gen):
if obs is not None:
lcs.add(self._get_lightcurve_(p, obs, k))
bar.update()
else:
for k, (p, obs) in enumerate(gen):
if obs is not None:
lcs.add(self._get_lightcurve_(p, obs, k))
return lcs
def get_all_fluxes(weblog_list, mapping=None):
data_dict = {}
for weblog in ProgressBar(weblog_list):
try:
data = get_calibrator_fluxes(weblog)
name, _ = get_human_readable_name(weblog, mapping=mapping)
data_dict[name] = data
except ValueError:
continue
flux_data = {
name: {
ii: {
'date': key[4],
'ms': key[1],
'calibrator': key[0],
'spw': key[2],
'freq': key[3],
'measurement': value
}
for ii, (key, value) in enumerate(data_.items())
}
for name, data_ in data_dict.items()
}
return flux_data
def run(self, obs_list):
"""
Run MapMaker for a list of observations to create
stacked counts, exposure and background maps
Parameters
--------------
obs_list: `~gammapy.data.ObservationList`
List of observations
Returns
-----------
maps: dict of stacked counts, background and exposure maps.
"""
from astropy.utils.console import ProgressBar
for obs in ProgressBar(obs_list):
self.process_obs(obs)
self.maps = {
'counts_map': self.counts_map,
'background_map': self.background_map,
'exposure_map': self.exposure_map
}
return self.maps
def extract_poly_slice(cube, polygons):
nx = len(polygons)
nz = cube.shape[0]
total_slice = np.zeros((nz, nx))
total_area = np.zeros((nz, nx))
p = ProgressBar(len(polygons))
for i, polygon in enumerate(polygons):
p.update()
# Find bounding box
bbxmin = int(round(np.min(polygon.x)) - 1)
bbxmax = int(round(np.max(polygon.x)) + 2)
bbymin = int(round(np.min(polygon.y)) - 1)
bbymax = int(round(np.max(polygon.y)) + 2)
# Clip to cube box
bbxmin = max(bbxmin, 0)
bbxmax = min(bbxmax, cube.shape[2])
bbymin = max(bbymin, 0)
bbymax = min(bbymax, cube.shape[1])
# Loop through pixels that might overlap
for xmin in np.arange(bbxmin, bbxmax):
for ymin in np.arange(bbymin, bbymax):
area = square_polygon_overlap_area(xmin - 0.5, xmin + 0.5, ymin - 0.5, ymin + 0.5, polygon.x, polygon.y)
if area > 0:
total_slice[:, i] += cube[:, ymin, xmin] * area
total_area[:, i] += area
total_slice[total_area == 0.0] = np.nan
total_slice[total_area > 0.0] /= total_area[total_area > 0.0]
print("")
return total_slice
def _HEADER_data_size(self, files):
"""
Given a list of file URLs, return the data size. This is useful for
assessing how much data you might be downloading!
(This is discouraged by the ALMA archive, as it puts unnecessary load
on their system)
"""
totalsize = 0 * u.B
data_sizes = {}
pb = ProgressBar(len(files))
for ii, fileLink in enumerate(files):
response = self._request('HEAD', fileLink, stream=False,
cache=False, timeout=self.TIMEOUT)
filesize = (int(response.headers['content-length']) * u.B).to(u.GB)
totalsize += filesize
data_sizes[fileLink] = filesize
log.debug("File {0}: size {1}".format(fileLink, filesize))
pb.update(ii + 1)
response.raise_for_status()
return data_sizes, totalsize.to(u.GB)
def _fast_reader(index_map, data):
"""
Use scipy.ndimage.find_objects to quickly identify subsets of the data
to increase speed of dendrogram loading
"""
flux_by_structure, indices_by_structure = {}, {}
from scipy import ndimage
idxs = np.unique(index_map[index_map > -1])
# ndimage ignores 0 and -1, but we want index 0
object_slices = ndimage.find_objects(index_map + 1)
# find_objects returns a tuple that includes many None values that we
# need to get rid of.
object_slices = [x for x in object_slices if x is not None]
index_cube = np.indices(index_map.shape)
# Need to have same length, otherwise assumptions above are wrong
assert len(idxs) == len(object_slices)
log.debug("Creating index maps for {0} indices...".format(len(idxs)))
p = ProgressBar(len(object_slices))
for idx, sl in zip(idxs, object_slices):
match = index_map[sl] == idx
sl2 = (slice(None),) + sl
match_inds = index_cube[sl2][:, match]
coords = list(zip(*match_inds))
dd = data[sl][match].tolist()
flux_by_structure[idx] = dd
indices_by_structure[idx] = coords
p.update()
return flux_by_structure, indices_by_structure
def fit_all_tex(xaxis, cube, cubefrequencies, degeneracies,
einsteinAij,
errorcube=None,
replace_bad=False):
"""
Parameters
----------
replace_bad : bool
Attempt to replace bad (negative) values with their upper limits?
"""
tmap = np.empty(cube.shape[1:])
Nmap = np.empty(cube.shape[1:])
yy,xx = np.indices(cube.shape[1:])
pb = ProgressBar(xx.size)
count=0
for ii,jj in (zip(yy.flat, xx.flat)):
if any(np.isnan(cube[:,ii,jj])):
tmap[ii,jj] = np.nan
else:
if replace_bad:
neg = cube[:,ii,jj] <= 0
cube[neg,ii,jj] = replace_bad
nuppers = nupper_of_kkms(cube[:,ii,jj], cubefrequencies,
einsteinAij, degeneracies)
if errorcube is not None:
enuppers = nupper_of_kkms(errorcube[:,ii,jj], cubefrequencies,
einsteinAij, degeneracies)
fit_result = fit_tex(xaxis, nuppers.value, errors=enuppers.value)
tmap[ii,jj] = fit_result[1].value
Nmap[ii,jj] = fit_result[0].value
pb.update(count)
count+=1
return tmap,Nmap
def update_mass_table(drpall, mass_table_old=None, limit=None, mlband='i'):
'''
'''
# what galaxies are available to aggregate?
res_fnames = glob(os.path.join(basedir, 'results/*-*/*-*_res.fits'))[:limit]
# filter out whose that have not been done
if mass_table_old is None:
already_aggregated = [False for _ in range(len(res_fnames))]
else:
already_aggregated = [os.path.split(fn)[1].split('_')[0] in old_mass_table['plateifu']
for fn in res_fnames]
res_fnames = [fn for done, fn in zip(already_aggregated, res_fnames)]
# aggregate individual galaxies, and stack them
mass_tables_new = list(ProgressBar.map(
partial(mass_agg_onegal, mlband=mlband), res_fnames, multiprocess=False, step=5))
mass_table_new = t.vstack(mass_tables_new)
# if there was an old mass table, stack it with the new one
if mass_table_old is None:
mass_table = mass_table_new
else:
mass_table = t.vstack([mass_table_old, mass_table_new], join_type='inner')
cmlr = totalmass.cmlr_kwargs
missing_flux = (mass_table['nsa_absmag'].to(m.Mgy) - \
mass_table['ifu_absmag'].to(m.Mgy)).clip(
a_min=0.*m.Mgy, a_max=np.inf*m.Mgy)
mag_missing_flux = missing_flux.to(u.ABmag)
cb1, cb2 = cmlr['cb1'], cmlr['cb2']
color_missing_flux = mag_missing_flux[:, totalmass.StellarMass.bands_ixs[cb1]] - \
mag_missing_flux[:, totalmass.StellarMass.bands_ixs[cb2]]
color_missing_flux[~np.isfinite(color_missing_flux)] = np.inf
mass_table['outer_ml_cmlr'] = np.polyval(cmlr['cmlr_poly'], color_missing_flux.value) * \
u.dex(m.m_to_l_unit)
mass_table['outer_lum'] = mag_missing_flux.to(
u.dex(m.bandpass_sol_l_unit),
totalmass.bandpass_flux_to_solarunits(totalmass.StellarMass.absmag_sun))
mass_table['outer_mass_ring'] = \
(mass_table['outer_lum'][:, totalmass.StellarMass.bands_ixs['i']] + \
mass_table['outer_ml_ring']).to(u.Msun)
mass_table['outer_mass_cmlr'] = \
(mass_table['outer_lum'][:, totalmass.StellarMass.bands_ixs['i']] + \
mass_table['outer_ml_cmlr']).to(u.Msun)
return mass_table
# Clean up
outfile.close()
infile.close()
def convert_to_dmc(input_fn, output_fn):
"""Convert from csv to pipe-delimited.
"""
syscall("""sed "s/,/|/g" {} > {}""".format(input_fn, output_fn))
if __name__ == "__main__":
input_filenames = glob.glob(os.path.join(CROPPED_CATS_DIR, "*-cropped"))
log.info("Converting cropped catalogues into csv format")
ProgressBar.map(convert_to_csv,
input_filenames,
multiprocess=True, step=1)
log.info("Removing objects appearing twice in the cropped tiles")
ProgressBar.map(remove_duplicates_from_tiles,
input_filenames,
multiprocess=True, step=1)
log.info("Removing objects already in EPIC with stilts "
"using a {} arcsec matching radius".format(MATCHING_RADIUS))
ProgressBar.map(remove_duplicates,
input_filenames,
multiprocess=True, step=1)
merge_fn = os.path.join(FINAL_CAT_DIR, "merged.dmc.csv")
log.info("Merging the tiles into {}".format(merge_fn))
def make_validation_report(
urls=None, destdir='astropy.io.votable.validator.results',
multiprocess=True, stilts=None):
"""
Validates a large collection of web-accessible VOTable files.
Generates a report as a directory tree of HTML files.
Parameters
----------
urls : list of strings, optional
If provided, is a list of HTTP urls to download VOTable files
from. If not provided, a built-in set of ~22,000 urls
compiled by HEASARC will be used.
destdir : path, optional
The directory to write the report to. By default, this is a
directory called ``'results'`` in the current directory. If the
directory does not exist, it will be created.
multiprocess : bool, optional
If `True` (default), perform validations in parallel using all
of the cores on this machine.
stilts : path, optional
To perform validation with ``votlint`` from the the Java-based
`STILTS <http://www.star.bris.ac.uk/~mbt/stilts/>`_ VOTable
parser, in addition to `astropy.io.votable`, set this to the
path of the ``'stilts.jar'`` file. ``java`` on the system shell
path will be used to run it.
Notes
-----
Downloads of each given URL will be performed only once and cached
locally in *destdir*. To refresh the cache, remove *destdir*
first.
"""
from astropy.utils.console import (color_print, ProgressBar, Spinner)
if stilts is not None:
if not os.path.exists(stilts):
raise ValueError(
'{0} does not exist.'.format(stilts))
destdir = os.path.abspath(destdir)
if urls is None:
with Spinner('Loading URLs', 'green') as s:
urls = get_urls(destdir, s)
else:
color_print('Marking URLs', 'green')
for url in ProgressBar.iterate(urls):
with result.Result(url, root=destdir) as r:
r['expected'] = type
args = [(url, destdir) for url in urls]
color_print('Downloading VO files', 'green')
ProgressBar.map(
download, args, multiprocess=multiprocess)
color_print('Validating VO files', 'green')
ProgressBar.map(
validate_vo, args, multiprocess=multiprocess)
if stilts is not None:
color_print('Validating with votlint', 'green')
votlint_args = [(stilts, x, destdir) for x in urls]
ProgressBar.map(
votlint_validate, votlint_args, multiprocess=multiprocess)
color_print('Generating HTML files', 'green')
ProgressBar.map(
write_html_result, args, multiprocess=multiprocess)
with Spinner('Grouping results', 'green') as s:
subsets = result.get_result_subsets(urls, destdir, s)
color_print('Generating index', 'green')
html.write_index(subsets, urls, destdir)
color_print('Generating subindices', 'green')
subindex_args = [(subset, destdir, len(urls)) for subset in subsets]
ProgressBar.map(
write_subindex, subindex_args, multiprocess=multiprocess)
def measure_dendrogram_properties(dend=None, cube303=cube303,
cube321=cube321, cube13co=cube13co,
cube18co=cube18co, noise_cube=noise_cube,
sncube=sncube,
suffix="",
last_index=None,
plot_some=True,
line='303',
write=True):
assert (cube321.shape == cube303.shape == noise_cube.shape ==
cube13co.shape == cube18co.shape == sncube.shape)
assert sncube.wcs is cube303.wcs is sncube.mask._wcs
metadata = {}
metadata['data_unit'] = u.K
metadata['spatial_scale'] = 7.2 * u.arcsec
metadata['beam_major'] = 30 * u.arcsec
metadata['beam_minor'] = 30 * u.arcsec
metadata['wavelength'] = 218.22219*u.GHz
metadata['velocity_scale'] = u.km/u.s
metadata['wcs'] = cube303.wcs
keys = [
'density_chi2',
'expected_density',
'dmin1sig_chi2',
'dmax1sig_chi2',
'column_chi2',
'expected_column',
'cmin1sig_chi2',
'cmax1sig_chi2',
'temperature_chi2',
'expected_temperature',
'tmin1sig_chi2',
'tmax1sig_chi2',
'eratio321303',
'ratio321303',
'logh2column',
'elogh2column',
'logabundance',
'elogabundance',
]
obs_keys = [
'Stot303',
'Smin303',
'Smax303',
'Stot321',
'Smean303',
'Smean321',
'npix',
'e303',
'e321',
'r321303',
'er321303',
'13cosum',
'c18osum',
'13comean',
'c18omean',
's_ntotal',
'index',
'is_leaf',
'parent',
'root',
'lon',
'lat',
'vcen',
'higaldusttem',
'reff',
'dustmass',
'dustmindens',
'bad',
#'tkin_turb',
]
columns = {k:[] for k in (keys+obs_keys)}
log.debug("Initializing dendrogram temperature fitting loop")
# FORCE wcs to match
# (technically should reproject here)
cube13co._wcs = cube18co._wcs = cube303.wcs
cube13co.mask._wcs = cube18co.mask._wcs = cube303.wcs
if line == '303':
maincube = cube303
elif line == '321':
maincube = cube321
else:
raise ValueError("Unrecognized line: {0}".format(line))
# Prepare an array to hold the fitted temperatures
tcubedata = np.empty(maincube.shape, dtype='float32')
tcubedata[:] = np.nan
tcubeleafdata = np.empty(maincube.shape, dtype='float32')
tcubeleafdata[:] = np.nan
nbad = 0
catalog = ppv_catalog(dend, metadata)
pb = ProgressBar(len(catalog))
for ii,row in enumerate(catalog):
structure = dend[row['_idx']]
assert structure.idx == row['_idx'] == ii
dend_obj_mask = BooleanArrayMask(structure.get_mask(), wcs=cube303.wcs)
dend_inds = structure.indices()
view = (slice(dend_inds[0].min(), dend_inds[0].max()+1),
slice(dend_inds[1].min(), dend_inds[1].max()+1),
slice(dend_inds[2].min(), dend_inds[2].max()+1),)
#view2 = cube303.subcube_slices_from_mask(dend_obj_mask)
submask = dend_obj_mask[view]
#assert np.count_nonzero(submask.include()) == np.count_nonzero(dend_obj_mask.include())
sn = sncube[view].with_mask(submask)
sntot = sn.sum().value
#np.testing.assert_almost_equal(sntot, structure.values().sum(), decimal=0)
c303 = cube303[view].with_mask(submask)
c321 = cube321[view].with_mask(submask)
co13sum = cube13co[view].with_mask(submask).sum().value
co18sum = cube18co[view].with_mask(submask).sum().value
if hasattr(co13sum,'__len__'):
raise TypeError(".sum() applied to an array has yielded a non scalar.")
npix = submask.include().sum()
assert npix == structure.get_npix()
Stot303 = c303.sum().value
if np.isnan(Stot303):
raise ValueError("NaN in cube. This can't happen: the data from "
"which the dendrogram was derived can't have "
"NaN pixels.")
Smax303 = c303.max().value
Smin303 = c303.min().value
Stot321 = c321.sum().value
if npix == 0:
raise ValueError("npix=0. This is impossible.")
Smean303 = Stot303/npix
if Stot303 <= 0 and line=='303':
raise ValueError("The 303 flux is <=0. This isn't possible because "
"the dendrogram was derived from the 303 data with a "
"non-zero threshold.")
elif Stot303 <= 0 and line=='321':
Stot303 = 0
Smean303 = 0
elif Stot321 <= 0 and line=='321':
raise ValueError("The 321 flux is <=0. This isn't possible because "
"the dendrogram was derived from the 321 data with a "
"non-zero threshold.")
if np.isnan(Stot321):
raise ValueError("NaN in 321 line")
Smean321 = Stot321/npix
#error = (noise_cube[view][submask.include()]).sum() / submask.include().sum()**0.5
var = ((noise_cube[dend_obj_mask.include()]**2).sum() / npix**2)
error = var**0.5
if np.isnan(error):
raise ValueError("error is nan: this is impossible by definition.")
if line == '321' and Stot303 == 0:
r321303 = np.nan
er321303 = np.nan
elif Stot321 < 0:
r321303 = error / Smean303
er321303 = (r321303**2 * (var/Smean303**2 + 1))**0.5
else:
r321303 = Stot321 / Stot303
er321303 = (r321303**2 * (var/Smean303**2 + var/Smean321**2))**0.5
for c in columns:
assert len(columns[c]) == ii
columns['index'].append(row['_idx'])
columns['s_ntotal'].append(sntot)
columns['Stot303'].append(Stot303)
columns['Smax303'].append(Smax303)
columns['Smin303'].append(Smin303)
columns['Stot321'].append(Stot321)
columns['Smean303'].append(Smean303)
columns['Smean321'].append(Smean321)
columns['npix'].append(npix)
columns['e303'].append(error)
columns['e321'].append(error)
columns['r321303'].append(r321303)
columns['er321303'].append(er321303)
columns['13cosum'].append(co13sum)
columns['c18osum'].append(co18sum)
columns['13comean'].append(co13sum/npix)
columns['c18omean'].append(co18sum/npix)
columns['is_leaf'].append(structure.is_leaf)
columns['parent'].append(structure.parent.idx if structure.parent else -1)
columns['root'].append(get_root(structure).idx)
s_main = maincube._data[dend_inds]
x,y,z = maincube.world[dend_inds]
lon = ((z.value-(360*(z.value>180)))*s_main).sum()/s_main.sum()
lat = (y*s_main).sum()/s_main.sum()
vel = (x*s_main).sum()/s_main.sum()
columns['lon'].append(lon)
columns['lat'].append(lat.value)
columns['vcen'].append(vel.value)
mask2d = dend_obj_mask.include().max(axis=0)[view[1:]]
logh2column = np.log10(np.nanmean(column_regridded.data[view[1:]][mask2d]) * 1e22)
if np.isnan(logh2column):
log.info("Source #{0} has NaNs".format(ii))
logh2column = 24
elogh2column = elogabundance
columns['higaldusttem'].append(np.nanmean(dusttem_regridded.data[view[1:]][mask2d]))
r_arcsec = row['radius']*u.arcsec
reff = (r_arcsec*(8.5*u.kpc)).to(u.pc, u.dimensionless_angles())
mass = ((10**logh2column*u.cm**-2)*np.pi*reff**2*2.8*constants.m_p).to(u.M_sun)
density = (mass/(4/3.*np.pi*reff**3)/constants.m_p/2.8).to(u.cm**-3)
columns['reff'].append(reff.value)
columns['dustmass'].append(mass.value)
columns['dustmindens'].append(density.value)
mindens = np.log10(density.value)
if mindens < 3:
mindens = 3
if (r321303 < 0 or np.isnan(r321303)) and line != '321':
raise ValueError("Ratio <0: This can't happen any more because "
"if either num/denom is <0, an exception is "
"raised earlier")
#for k in columns:
# if k not in obs_keys:
# columns[k].append(np.nan)
elif (r321303 < 0 or np.isnan(r321303)) and line == '321':
for k in keys:
columns[k].append(np.nan)
else:
# Replace negatives for fitting
if Smean321 <= 0:
Smean321 = error
mf.set_constraints(ratio321303=r321303, eratio321303=er321303,
#ratio321322=ratio2, eratio321322=eratio2,
logh2column=logh2column, elogh2column=elogh2column,
logabundance=logabundance, elogabundance=elogabundance,
taline303=Smean303, etaline303=error,
taline321=Smean321, etaline321=error,
mindens=mindens,
linewidth=10)
row_data = mf.get_parconstraints()
row_data['ratio321303'] = r321303
row_data['eratio321303'] = er321303
for k in row_data:
columns[k].append(row_data[k])
# Exclude bad velocities from cubes
if row['v_cen'] < -80e3 or row['v_cen'] > 180e3:
# Skip: there is no real structure down here
nbad += 1
is_bad = True
else:
is_bad = False
tcubedata[dend_obj_mask.include()] = row_data['expected_temperature']
if structure.is_leaf:
tcubeleafdata[dend_obj_mask.include()] = row_data['expected_temperature']
columns['bad'].append(is_bad)
width = row['v_rms']*u.km/u.s
lengthscale = reff
#REMOVED in favor of despotic version done in dendrograms.py
# we use the analytic version here; the despotic version is
# computed elsewhere (with appropriate gcor factors)
#columns['tkin_turb'].append(heating.tkin_all(10**row_data['density_chi2']*u.cm**-3,
# width,
# lengthscale,
# width/lengthscale,
# columns['higaldusttem'][-1]*u.K,
# crir=0./u.s))
if len(set(len(c) for k,c in columns.items())) != 1:
print("Columns are different lengths. This is not allowed.")
import ipdb; ipdb.set_trace()
for c in columns:
assert len(columns[c]) == ii+1
if plot_some and not is_bad and (ii-nbad % 100 == 0 or ii-nbad < 50):
try:
log.info("T: [{tmin1sig_chi2:7.2f},{expected_temperature:7.2f},{tmax1sig_chi2:7.2f}]"
" R={ratio321303:8.4f}+/-{eratio321303:8.4f}"
" Smean303={Smean303:8.4f} +/- {e303:8.4f}"
" Stot303={Stot303:8.2e} npix={npix:6d}"
.format(Smean303=Smean303, Stot303=Stot303,
npix=npix, e303=error, **row_data))
pl.figure(1)
pl.clf()
mf.denstemplot()
pl.savefig(fpath("dendrotem/diagnostics/{0}_{1}.png".format(suffix,ii)))
pl.figure(2).clf()
mf.parplot1d_all(levels=[0.68268949213708585])
pl.savefig(fpath("dendrotem/diagnostics/1dplot{0}_{1}.png".format(suffix,ii)))
pl.draw()
pl.show()
except Exception as ex:
print ex
pass
else:
pb.update(ii+1)
if last_index is not None and ii >= last_index:
break
if last_index is not None:
catalog = catalog[:last_index+1]
for k in columns:
if k not in catalog.keys():
catalog.add_column(table.Column(name=k, data=columns[k]))
for mid,lo,hi,letter in (('expected_temperature','tmin1sig_chi2','tmax1sig_chi2','t'),
('expected_density','dmin1sig_chi2','dmax1sig_chi2','d'),
('expected_column','cmin1sig_chi2','cmax1sig_chi2','c')):
catalog.add_column(table.Column(name='elo_'+letter,
data=catalog[mid]-catalog[lo]))
catalog.add_column(table.Column(name='ehi_'+letter,
data=catalog[hi]-catalog[mid]))
if write:
catalog.write(tpath('PPV_H2CO_Temperature{0}.ipac'.format(suffix)), format='ascii.ipac')
# Note that there are overlaps in the catalog, which means that ORDER MATTERS
# in the above loop. I haven't yet checked whether large scale overwrites
# small or vice-versa; it may be that both views of the data are interesting.
tcube = SpectralCube(data=tcubedata, wcs=cube303.wcs,
mask=cube303.mask, meta={'unit':'K'},
header=cube303.header,
)
tcubeleaf = SpectralCube(data=tcubeleafdata, wcs=cube303.wcs,
mask=cube303.mask, meta={'unit':'K'},
header=cube303.header,
)
if write:
log.info("Writing TemperatureCube")
outpath = 'TemperatureCube_DendrogramObjects{0}.fits'
tcube.write(hpath(outpath.format(suffix)),
overwrite=True)
outpath_leaf = 'TemperatureCube_DendrogramObjects{0}_leaves.fits'
tcubeleaf.write(hpath(outpath_leaf.format(suffix)),
overwrite=True)
return catalog, tcube
def add_data_to_cube(cubefilename, data=None, filename=None, fileheader=None,
flatheader='header.txt',
cubeheader='cubeheader.txt', nhits=None,
smoothto=1, baselineorder=5, velocityrange=None,
excludefitrange=None, noisecut=np.inf, do_runscript=False,
linefreq=None, allow_smooth=True,
data_iterator=data_iterator,
coord_iterator=coord_iterator,
velo_iterator=velo_iterator,
progressbar=False, coordsys='galactic',
datalength=None,
velocity_offset=0.0, negative_mean_cut=None,
add_with_kernel=False, kernel_fwhm=None, fsw=False,
kernel_function=Gaussian2DKernel,
diagnostic_plot_name=None, chmod=False,
continuum_prefix=None,
debug_breakpoint=False,
default_unit=u.km/u.s,
make_continuum=True,
weightspec=None,
varweight=False):
"""
Given a .fits file that contains a binary table of spectra (e.g., as
you would get from the GBT mapping "pipeline" or the reduce_map.pro aoidl
file provided by Adam Ginsburg), adds each spectrum into the cubefile.
velocity_offset : 0.0
Amount to add to the velocity vector before adding it to the cube
(useful for FSW observations)
weightspec : np.ndarray
A spectrum with the same size as the input arrays but containing the relative
weights of the data
"""
#if not default_unit.is_equivalent(u.km/u.s):
# raise TypeError("Default unit is not a velocity equivalent.")
if type(nhits) is str:
log.debug("Loading nhits from %s" % nhits)
nhits = pyfits.getdata(nhits)
elif type(nhits) is not np.ndarray:
raise TypeError("nhits must be a .fits file or an ndarray, but it is ",type(nhits))
naxis2,naxis1 = nhits.shape
if velocity_offset and not fsw:
raise ValueError("Using a velocity offset, but obs type is not "
"frequency switched; this is almost certainly wrong, "
"but if there's a case for it I'll remove this.")
if not hasattr(velocity_offset,'unit'):
velocity_offset = velocity_offset*default_unit
contimage = np.zeros_like(nhits)
nhits_once = np.zeros_like(nhits)
log.debug("Loading data cube {0}".format(cubefilename))
t0 = time.time()
# rescale image to weight by number of observations
image = pyfits.getdata(cubefilename)*nhits
log.debug(" ".join(("nhits statistics: mean, std, nzeros, size",str(nhits.mean()),str(nhits.std()),str(np.sum(nhits==0)), str(nhits.size))))
log.debug(" ".join(("Image statistics: mean, std, nzeros, size",str(image.mean()),str(image.std()),str(np.sum(image==0)), str(image.size), str(np.sum(np.isnan(image))))))
log.debug(" ".join(("nhits shape: ",str(nhits.shape))))
# default is to set empty pixels to NAN; have to set them
# back to zero
image[image!=image] = 0.0
header = pyfits.getheader(cubefilename)
# debug print "Cube shape: ",image.shape," naxis3: ",header.get('NAXIS3')," nhits shape: ",nhits.shape
log.debug("".join(("Image statistics: mean, std, nzeros, size",str(image.mean()),str(image.std()),str(np.sum(image==0)), str(image.size))))
flathead = get_header(flatheader)
naxis3 = image.shape[0]
wcs = pywcs.WCS(flathead)
cwcs = pywcs.WCS(header)
vwcs = cwcs.sub([pywcs.WCSSUB_SPECTRAL])
vunit = u.Unit(vwcs.wcs.cunit[vwcs.wcs.spec])
cubevelo = vwcs.wcs_pix2world(np.arange(naxis3),0)[0] * vunit
cd3 = vwcs.wcs.cdelt[vwcs.wcs.spec] * vunit
if not vunit.is_equivalent(default_unit):
raise ValueError("The units of the cube and the velocity axis are "
"possibly not equivalent. Change default_unit to "
"the appropriate unit (probably {0})".format(vunit))
if add_with_kernel:
if wcs.wcs.has_cd():
cd = np.abs(wcs.wcs.cd[1,1])
else:
cd = np.abs(wcs.wcs.cdelt[1])
# Alternative implementation; may not work for .cd?
#cd = np.abs(np.prod((wcs.wcs.get_cdelt() * wcs.wcs.get_pc().diagonal())))**0.5
if velocityrange is not None:
if hasattr(velocityrange, 'unit'):
v1,v4 = velocityrange
else:
v1,v4 = velocityrange * default_unit
ind1 = np.argmin(np.abs(np.floor(v1-cubevelo)))
ind2 = np.argmin(np.abs(np.ceil(v4-cubevelo)))+1
# stupid hack. REALLY stupid hack. Don't crop.
if np.abs(ind2-image.shape[0]) < 5:
ind2 = image.shape[0]
if np.abs(ind1) < 5:
ind1 = 0
#print "Velo match for v1,v4 = %f,%f: %f,%f" % (v1,v4,cubevelo[ind1],cubevelo[ind2])
# print "Updating CRPIX3 from %i to %i. Cropping to indices %i,%i" % (header.get('CRPIX3'),header.get('CRPIX3')-ind1,ind1,ind2)
# I think this could be disastrous: cubevelo is already set, but now we're changing how it's set in the header!
# I don't think there's any reason to have this in the first place
# header.set('CRPIX3',header.get('CRPIX3')-ind1)
# reset v1,v4 to the points we just selected
v1 = cubevelo[ind1]
v4 = cubevelo[ind2-1]
else:
ind1=0
ind2 = image.shape[0]
v1,v4 = min(cubevelo),max(cubevelo)
# debug print "Cube has %i v-axis pixels from %f to %f. Crop range is %f to %f" % (naxis3,cubevelo.min(),cubevelo.max(),v1,v4)
#if abs(cdelt) < abs(cd3):
# print "Spectra have CD=%0.2f, cube has CD=%0.2f. Will smooth & interpolate." % (cdelt,cd3)
# Disable progressbar if debug-logging is enabled (they clash)
if progressbar and 'ProgressBar' in globals() and log.level > 10:
if datalength is None:
pb = ProgressBar(len(data))
else:
pb = ProgressBar(datalength)
else:
progressbar = False
skipped = []
for spectrum,pos,velo in zip(data_iterator(data,fsw=fsw),
coord_iterator(data,coordsys_out=coordsys),
velo_iterator(data,linefreq=linefreq)):
if log.level <= 10:
t1 = time.time()
if not hasattr(velo,'unit'):
velo = velo * default_unit
glon,glat = pos
cdelt = velo[1]-velo[0]
if cdelt < 0:
# for interpolation, require increasing X axis
spectrum = spectrum[::-1]
velo = velo[::-1]
if log.level < 5:
log.debug("Reversed spectral axis... ")
if (velo.max() < cubevelo.min() or velo.min() > cubevelo.max()):
raise ValueError("Data out of range.")
if progressbar and log.level > 10:
pb.update()
velo += velocity_offset
if glon != 0 and glat != 0:
x,y = wcs.wcs_world2pix(glon,glat,0)
if np.isnan(x) or np.isnan(y):
log.warn("".join(("Skipping NaN point {0}, {1} ...".format(glon,glat))))
continue
if log.level < 10:
log.debug("".join(("At point {0},{1} ...".format(glon,glat),)))
if abs(cdelt) < abs(cd3) and allow_smooth:
# need to smooth before interpolating to preserve signal
kernwidth = abs(cd3/cdelt/2.35).decompose().value
if kernwidth > 2 and kernwidth < 10:
xr = kernwidth*5
npx = np.ceil(xr*2 + 1)
elif kernwidth > 10:
raise ValueError('Too much smoothing')
else:
xr = 5
npx = 11
#kernel = np.exp(-(np.linspace(-xr,xr,npx)**2)/(2.0*kernwidth**2))
#kernel /= kernel.sum()
kernel = Gaussian1DKernel(stddev=kernwidth, x_size=npx)
smspec = np.convolve(spectrum,kernel,mode='same')
datavect = np.interp(cubevelo.to(default_unit).value,
velo.to(default_unit).value,
smspec)
else:
datavect = np.interp(cubevelo.to(default_unit).value,
velo.to(default_unit).value,
spectrum)
OK = (datavect[ind1:ind2] == datavect[ind1:ind2])
if excludefitrange is None:
include = OK
else:
# Exclude certain regions (e.g., the spectral lines) when computing the noise
include = OK.copy()
if not hasattr(excludefitrange,'unit'):
excludefitrange = excludefitrange * default_unit
# Convert velocities to indices
exclude_inds = [np.argmin(np.abs(np.floor(v-cubevelo))) for v in excludefitrange]
# Loop through exclude_inds pairwise
for (i1,i2) in zip(exclude_inds[:-1:2],exclude_inds[1::2]):
# Do not include the excluded regions
include[i1:i2] = False
if include.sum() == 0:
raise ValueError("All data excluded.")
noiseestimate = datavect[ind1:ind2][include].std()
contestimate = datavect[ind1:ind2][include].mean()
if noiseestimate > noisecut:
log.info("Skipped a data point at %f,%f in file %s because it had excessive noise %f" % (x,y,filename,noiseestimate))
skipped.append(True)
continue
elif negative_mean_cut is not None and contestimate < negative_mean_cut:
log.info("Skipped a data point at %f,%f in file %s because it had negative continuum %f" % (x,y,filename,contestimate))
skipped.append(True)
continue
elif OK.sum() == 0:
log.info("Skipped a data point at %f,%f in file %s because it had NANs" % (x,y,filename))
skipped.append(True)
continue
elif OK.sum()/float(abs(ind2-ind1)) < 0.5:
log.info("Skipped a data point at %f,%f in file %s because it had %i NANs" % (x,y,filename,np.isnan(datavect[ind1:ind2]).sum()))
skipped.append(True)
continue
if log.level < 10:
log.debug("did not skip...")
if varweight:
weight = 1./noiseestimate**2
else:
weight = 1.
if weightspec is None:
wspec = weight
else:
wspec = weight * weightspec
if 0 < int(np.round(x)) < naxis1 and 0 < int(np.round(y)) < naxis2:
if add_with_kernel:
fwhm = np.sqrt(8*np.log(2))
kernel_size = kd = int(np.ceil(kernel_fwhm/fwhm/cd * 5))
if kernel_size < 5:
kernel_size = kd = 5
if kernel_size % 2 == 0:
kernel_size = kd = kernel_size+1
if kernel_size > 100:
raise ValueError("Huge kernel - are you sure?")
kernel_middle = mid = (kd-1)/2.
xinds,yinds = (np.mgrid[:kd,:kd]-mid+np.array([np.round(x),np.round(y)])[:,None,None]).astype('int')
# This kernel is NOT centered, and that's the bloody point.
# (I made a very stupid error and used Gaussian2DKernel,
# which is strictly centered, in a previous version)
kernel2d = np.exp(-((xinds-x)**2+(yinds-y)**2)/(2*(kernel_fwhm/fwhm/cd)**2))
dim1 = ind2-ind1
vect_to_add = np.outer(datavect[ind1:ind2],kernel2d).reshape([dim1,kd,kd])
vect_to_add[True-OK] = 0
# need to slice out edges
if yinds.max() >= naxis2 or yinds.min() < 0:
yok = (yinds[0,:] < naxis2) & (yinds[0,:] >= 0)
xinds,yinds = xinds[:,yok],yinds[:,yok]
vect_to_add = vect_to_add[:,:,yok]
kernel2d = kernel2d[:,yok]
if xinds.max() >= naxis1 or xinds.min() < 0:
xok = (xinds[:,0] < naxis1) & (xinds[:,0] >= 0)
xinds,yinds = xinds[xok,:],yinds[xok,:]
vect_to_add = vect_to_add[:,xok,:]
kernel2d = kernel2d[xok,:]
image[ind1:ind2,yinds,xinds] += vect_to_add*wspec
# NaN spectral bins are not appropriately downweighted... but they shouldn't exist anyway...
nhits[yinds,xinds] += kernel2d*weight
contimage[yinds,xinds] += kernel2d * contestimate*weight
nhits_once[yinds,xinds] += kernel2d*weight
else:
image[ind1:ind2,int(np.round(y)),int(np.round(x))][OK] += datavect[ind1:ind2][OK]*weight
nhits[int(np.round(y)),int(np.round(x))] += weight
contimage[int(np.round(y)),int(np.round(x))] += contestimate*weight
nhits_once[int(np.round(y)),int(np.round(x))] += weight
if log.level < 10:
log.debug("Z-axis indices are %i,%i..." % (ind1,ind2,))
log.debug("Added a data point at %i,%i" % (int(np.round(x)),int(np.round(y))))
skipped.append(False)
else:
skipped.append(True)
log.info("Skipped a data point at x,y=%f,%f "
"lon,lat=%f,%f in file %s because "
"it's out of the grid" % (x,y,glon,glat,filename))
if debug_breakpoint:
import ipdb
ipdb.set_trace()
if log.level <= 10:
dt = time.time() - t1
log.debug("Completed x,y={x:4.0f},{y:4.0f}"
" ({x:6.2f},{y:6.2f}) in {dt:6.2g}s".format(x=float(x),
y=float(y),
dt=dt))
log.info("Completed 'add_data' loop for"
" {0} in {1}s".format(cubefilename, time.time()-t0))
if excludefitrange is not None:
# this block redefining "include" is used for diagnostics (optional)
ind1a = np.argmin(np.abs(np.floor(v1-velo)))
ind2a = np.argmin(np.abs(np.ceil(v4-velo)))+1
dname = 'DATA' if 'DATA' in data.dtype.names else 'SPECTRA'
OK = (data[dname][0,:]==data[dname][0,:])
OK[:ind1a] = False
OK[ind2a:] = False
include = OK
# Convert velocities to indices
exclude_inds = [np.argmin(np.abs(np.floor(v-velo))) for v in excludefitrange]
# Loop through exclude_inds pairwise
for (i1,i2) in zip(exclude_inds[:-1:2],exclude_inds[1::2]):
# Do not include the excluded regions
include[i1:i2] = False
if include.sum() == 0:
raise ValueError("All data excluded.")
else:
dname = 'DATA' if 'DATA' in data.dtype.names else 'SPECTRA'
include = slice(None)
if diagnostic_plot_name:
from mpl_plot_templates import imdiagnostics
pylab.clf()
dd = data[dname][:,include]
imdiagnostics(dd,axis=pylab.gca())
pylab.savefig(diagnostic_plot_name, bbox_inches='tight')
# Save a copy with the bad stuff flagged out; this should tell whether flagging worked
skipped = np.array(skipped,dtype='bool')
dd[skipped,:] = -999
maskdata = np.ma.masked_equal(dd,-999)
pylab.clf()
imdiagnostics(maskdata, axis=pylab.gca())
dpn_pre,dpn_suf = os.path.splitext(diagnostic_plot_name)
dpn_flagged = dpn_pre+"_flagged"+dpn_suf
pylab.savefig(dpn_flagged, bbox_inches='tight')
log.info("Saved diagnostic plot %s and %s" % (diagnostic_plot_name,dpn_flagged))
log.debug("nhits statistics: mean, std, nzeros, size {0} {1} {2} {3}".format(nhits.mean(),nhits.std(),np.sum(nhits==0), nhits.size))
log.debug("Image statistics: mean, std, nzeros, size {0} {1} {2} {3}".format(image.mean(),image.std(),np.sum(image==0), image.size))
imav = image/nhits
if log.level <= 10:
nnan = np.count_nonzero(np.isnan(imav))
log.debug("imav statistics: mean, std, nzeros, size, nnan, ngood: {0} {1} {2} {3} {4} {5}".format(imav.mean(),imav.std(),np.sum(imav==0), imav.size, nnan, imav.size-nnan))
log.debug("imav shape: {0}".format(imav.shape))
subcube = imav[ind1:ind2,:,:]
if log.level <= 10:
nnan = np.sum(np.isnan(subcube))
print("subcube statistics: mean, std, nzeros, size, nnan, ngood:",np.nansum(subcube)/subcube.size,np.std(subcube[subcube==subcube]),np.sum(subcube==0), subcube.size, nnan, subcube.size-nnan)
print("subcube shape: ",subcube.shape)
H = header.copy()
if fileheader is not None:
for k,v in fileheader.items():
if 'RESTFRQ' in k or 'RESTFREQ' in k:
header.set(k,v)
#if k[0] == 'C' and '1' in k and k[-1] != '1':
# header.set(k.replace('1','3'), v)
moreH = get_header(cubeheader)
for k,v in H.items():
header.set(k,v)
for k,v in moreH.items():
header.set(k,v)
HDU = pyfits.PrimaryHDU(data=subcube,header=header)
HDU.writeto(cubefilename,clobber=True,output_verify='fix')
outpre = cubefilename.replace(".fits","")
include = np.ones(imav.shape[0],dtype='bool')
if excludefitrange is not None:
# this block redifining "include" is used for continuum
ind1a = np.argmin(np.abs(np.floor(v1-cubevelo)))
ind2a = np.argmin(np.abs(np.ceil(v4-cubevelo)))+1
# Convert velocities to indices
exclude_inds = [np.argmin(np.abs(np.floor(v-cubevelo))) for v in excludefitrange]
# Loop through exclude_inds pairwise
for (i1,i2) in zip(exclude_inds[:-1:2],exclude_inds[1::2]):
# Do not include the excluded regions
include[i1:i2] = False
if include.sum() == 0:
raise ValueError("All data excluded.")
HDU2 = pyfits.PrimaryHDU(data=nhits,header=flathead)
HDU2.writeto(outpre+"_nhits.fits",clobber=True,output_verify='fix')
#OKCube = (imav==imav)
#contmap = np.nansum(imav[naxis3*0.1:naxis3*0.9,:,:],axis=0) / OKCube.sum(axis=0)
if make_continuum:
contmap = np.nansum(imav[include,:,:],axis=0) / include.sum()
HDU2 = pyfits.PrimaryHDU(data=contmap,header=flathead)
HDU2.writeto(outpre+"_continuum.fits",clobber=True,output_verify='fix')
if continuum_prefix is not None:
# Solo continuum image (just this obs set)
HDU2.data = contimage / nhits_once
HDU2.writeto(continuum_prefix+"_continuum.fits",clobber=True,output_verify='fix')
HDU2.data = nhits_once
HDU2.writeto(continuum_prefix+"_nhits.fits",clobber=True,output_verify='fix')
log.info("Writing script file {0}".format(outpre+"_starlink.sh"))
scriptfile = open(outpre+"_starlink.sh",'w')
outpath,outfn = os.path.split(cubefilename)
outpath,pre = os.path.split(outpre)
print(("#!/bin/bash"), file=scriptfile)
if outpath != '':
print(('cd %s' % outpath), file=scriptfile)
print(('. /star/etc/profile'), file=scriptfile)
print(('kappa > /dev/null'), file=scriptfile)
print(('convert > /dev/null'), file=scriptfile)
print(('fits2ndf %s %s' % (outfn,outfn.replace(".fits",".sdf"))), file=scriptfile)
if excludefitrange is not None:
v2v3 = ""
for v2,v3 in zip(excludefitrange[::2],excludefitrange[1::2]):
v2v3 += "%0.2f %0.2f " % (v2.to(default_unit).value,v3.to(default_unit).value)
print(('mfittrend %s ranges=\\\"%0.2f %s %0.2f\\\" order=%i axis=3 out=%s' % (outfn.replace(".fits",".sdf"),v1.to(default_unit).value,v2v3,v4.to(default_unit).value,baselineorder,outfn.replace(".fits","_baseline.sdf"))), file=scriptfile)
else:
print(('mfittrend %s ranges=\\\"%0.2f %0.2f\\\" order=%i axis=3 out=%s' % (outfn.replace(".fits",".sdf"),v1.to(default_unit).value,v4.to(default_unit).value,baselineorder,outfn.replace(".fits","_baseline.sdf"))), file=scriptfile)
print(('sub %s %s %s' % (outfn.replace(".fits",".sdf"),outfn.replace(".fits","_baseline.sdf"),outfn.replace(".fits","_sub.sdf"))), file=scriptfile)
print(('sqorst %s_sub mode=pixelscale axis=3 pixscale=%i out=%s_vrebin' % (pre,smoothto,pre)), file=scriptfile)
print(('gausmooth %s_vrebin fwhm=1.0 axes=[1,2] out=%s_smooth' % (pre,pre)), file=scriptfile)
print(('#collapse %s estimator=mean axis="VRAD" low=-400 high=500 out=%s_continuum' % (pre,pre)), file=scriptfile)
print(('rm %s_sub.fits' % (pre)), file=scriptfile)
print(('ndf2fits %s_sub %s_sub.fits' % (pre,pre)), file=scriptfile)
print(('rm %s_smooth.fits' % (pre)), file=scriptfile)
print(('ndf2fits %s_smooth %s_smooth.fits' % (pre,pre)), file=scriptfile)
print(("# Fix STARLINK's failure to respect header keywords."), file=scriptfile)
print(('sethead %s_smooth.fits RESTFRQ=`gethead RESTFRQ %s.fits`' % (pre,pre)), file=scriptfile)
print(('rm %s_baseline.sdf' % (pre)), file=scriptfile)
print(('rm %s_smooth.sdf' % (pre)), file=scriptfile)
print(('rm %s_sub.sdf' % (pre)), file=scriptfile)
print(('rm %s_vrebin.sdf' % (pre)), file=scriptfile)
print(('rm %s.sdf' % (pre)), file=scriptfile)
scriptfile.close()
if chmod:
scriptfilename = (outpre+"_starlink.sh").replace(" ","")
#subprocess.call("chmod +x {0}".format(scriptfilename), shell=True)
st = os.stat(scriptfilename)
os.chmod(scriptfilename, st.st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH | stat.S_IXUSR)
if do_runscript:
runscript(outpre)
_fix_ms_kms_file(outpre+"_sub.fits")
_fix_ms_kms_file(outpre+"_smooth.fits")
if log.level <= 20:
log.info("Completed {0} in {1}s".format(pre, time.time()-t0))
def fourier_combine_cubes(cube1, cube2, highresextnum=0,
highresscalefactor=1.0,
lowresscalefactor=1.0, lowresfwhm=1*u.arcmin,
return_regridded_cube2=False,
return_hdu=False,
):
"""
Fourier combine two data cubes
Parameters
----------
cube1 : SpectralCube
highresfitsfile : str
The high-resolution FITS file
cube2 : SpectralCube
lowresfitsfile : str
The low-resolution (single-dish) FITS file
highresextnum : int
The extension number to use from the high-res FITS file
highresscalefactor : float
lowresscalefactor : float
A factor to multiply the high- or low-resolution data by to match the
low- or high-resolution data
lowresfwhm : `astropy.units.Quantity`
The full-width-half-max of the single-dish (low-resolution) beam;
or the scale at which you want to try to match the low/high resolution
data
return_hdu : bool
Return an HDU instead of just a cube. It will contain two image
planes, one for the real and one for the imaginary data.
return_regridded_cube2 : bool
Return the 2nd cube regridded into the pixel space of the first?
"""
if isinstance(cube1, str):
cube1 = SpectralCube.read(cube1)
if isinstance(cube2, str):
cube2 = SpectralCube.read(cube2)
#cube1 = spectral_cube.io.fits.load_fits_cube(highresfitsfile,
# hdu=highresextnum)
im1 = cube1._data # want the raw data for this
hd1 = cube1.header
assert hd1['NAXIS'] == im1.ndim == 3
w1 = cube1.wcs
pixscale = np.abs(w1.wcs.get_cdelt()[0]) # REPLACE EVENTUALLY...
cube2 = cube2.to(cube1.unit)
assert cube1.unit == cube2.unit, 'Cubes must have same or equivalent unit'
assert cube1.unit.is_equivalent(u.Jy/u.beam) or cube1.unit.is_equivalent(u.K), "Cubes must have brightness units."
#f2 = regrid_fits_cube(lowresfitsfile, hd1)
f2 = regrid_cube_hdu(cube2.hdu, hd1)
w2 = wcs.WCS(f2.header)
nax1,nax2,nax3 = (hd1['NAXIS1'],
hd1['NAXIS2'],
hd1['NAXIS3'])
dcube1 = im1 * highresscalefactor
dcube2 = f2.data * lowresscalefactor
outcube = np.empty_like(dcube1)
xgrid,ygrid = (np.indices([nax2,nax1])-np.array([(nax2-1.)/2,(nax1-1.)/2.])[:,None,None])
fwhm = np.sqrt(8*np.log(2))
# sigma in pixels
sigma = ((lowresfwhm/fwhm/(pixscale*u.deg)).decompose().value)
#sigma_fftspace = (1/(4*np.pi**2*sigma**2))**0.5
sigma_fftspace = (2*np.pi*sigma)**-1
log.debug('sigma = {0}, sigma_fftspace={1}'.format(sigma, sigma_fftspace))
kernel = np.fft.fftshift(np.exp(-(xgrid**2+ygrid**2)/(2*sigma**2)))
# convert the kernel, which is just a gaussian in image space,
# to its corresponding kernel in fourier space
kfft = np.abs(np.fft.fft2(kernel)) # should be mostly real
# normalize the kernel
kfft/=kfft.max()
ikfft = 1-kfft
pb = ProgressBar(dcube1.shape[0])
for ii,(im1,im2) in enumerate(zip(dcube1, dcube2)):
fft1 = np.fft.fft2(np.nan_to_num(im1))
fft2 = np.fft.fft2(np.nan_to_num(im2))
fftsum = kfft*fft2 + ikfft*fft1
combo = np.fft.ifft2(fftsum)
outcube[ii,:,:] = combo.real
pb.update(ii+1)
if return_regridded_cube2:
return outcube, f2
elif return_hdu:
return fits.PrimaryHDU(data=outcube, header=w1.to_header())
else:
return outcube
def convolve_model_dir(model_dir, filters, overwrite=False):
"""
Convolve all the model SEDs in a model directory
Parameters
----------
model_dir : str
The path to the model directory
filters : list
A list of :class:`~sedfitter.filter.Filter` objects to use for the
convolution
overwrite : bool, optional
Whether to overwrite the output files
"""
for f in filters:
if f.name is None:
raise Exception("filter name needs to be set")
if f.central_wavelength is None:
raise Exception("filter central wavelength needs to be set")
# Create 'convolved' sub-directory if needed
if not os.path.exists(model_dir + '/convolved'):
os.mkdir(model_dir + '/convolved')
# Find all SED files to convolve
sed_files = (glob.glob(model_dir + '/seds/*.fits.gz') +
glob.glob(model_dir + '/seds/*/*.fits.gz') +
glob.glob(model_dir + '/seds/*.fits') +
glob.glob(model_dir + '/seds/*/*.fits'))
par_table = load_parameter_table(model_dir)
if len(sed_files) == 0:
raise Exception("No SEDs found in %s" % model_dir)
else:
log.info("{0} SEDs found in {1}".format(len(sed_files), model_dir))
# Find out apertures
first_sed = SED.read(sed_files[0])
n_ap = first_sed.n_ap
apertures = first_sed.apertures
# Set up convolved fluxes
fluxes = [ConvolvedFluxes(model_names=np.zeros(len(sed_files), dtype='U30' if six.PY3 else 'S30'), apertures=apertures, initialize_arrays=True) for i in range(len(filters))]
# Set up list of binned filters
binned_filters = []
binned_nu = None
# Loop over SEDs
b = ProgressBar(len(sed_files))
for im, sed_file in enumerate(sed_files):
log.debug('Convolving {0}'.format(os.path.basename(sed_file)))
# Read in SED
s = SED.read(sed_file, unit_freq=u.Hz, unit_flux=u.mJy, order='nu')
# Check if filters need to be re-binned
try:
assert binned_nu is not None
np.testing.assert_array_almost_equal_nulp(s.nu.value, binned_nu.value, 100)
except AssertionError:
log.info('Rebinning filters')
binned_filters = [f.rebin(s.nu) for f in filters]
binned_nu = s.nu
b.update()
# Convolve
for i, f in enumerate(binned_filters):
fluxes[i].central_wavelength = f.central_wavelength
fluxes[i].apertures = apertures
fluxes[i].model_names[im] = s.name
if n_ap == 1:
fluxes[i].flux[im] = np.sum(s.flux * f.response)
fluxes[i].error[im] = np.sqrt(np.sum((s.error * f.response) ** 2))
else:
fluxes[i].flux[im, :] = np.sum(s.flux * f.response, axis=1)
fluxes[i].error[im] = np.sqrt(np.sum((s.error * f.response) ** 2, axis=1))
for i, f in enumerate(binned_filters):
fluxes[i].sort_to_match(par_table['MODEL_NAME'])
fluxes[i].write(model_dir + '/convolved/' + f.name + '.fits',
overwrite=overwrite)
def compute_bispectrum(self, show_progress=True, use_pyfftw=False,
threads=1, nsamples=100, seed=1000,
mean_subtract=False, **pyfftw_kwargs):
'''
Do the computation.
Parameters
----------
show_progress : optional, bool
Show progress bar while sampling the bispectrum.
use_pyfftw : bool, optional
Enable to use pyfftw, if it is installed.
threads : int, optional
Number of threads to use in FFT when using pyfftw.
nsamples : int, optional
Sets the number of samples to take at each vector
magnitude.
seed : int, optional
Sets the seed for the distribution draws.
mean_subtract : bool, optional
Subtract the mean from the data before computing. This removes the
"zero frequency" (i.e., constant) portion of the power, resulting
in a loss of phase coherence along the k_1=k_2 line.
pyfft_kwargs : Passed to
`~turbustat.statistics.rfft_to_fft.rfft_to_fft`. See
`here <https://hgomersall.github.io/pyFFTW/pyfftw/interfaces/interfaces.html#interfaces-additional-args>`_
for a list of accepted kwargs.
'''
if mean_subtract:
norm_data = self.data - self.data.mean()
else:
norm_data = self.data
if use_pyfftw:
if PYFFTW_FLAG:
if pyfftw_kwargs.get('threads') is not None:
pyfftw_kwargs.pop('threads')
fftarr = fft2(norm_data,
threads=threads,
**pyfftw_kwargs)
else:
warn("pyfftw not installed. Reverting to using numpy.")
use_pyfftw = False
if not use_pyfftw:
fftarr = np.fft.fft2(norm_data)
conjfft = np.conj(fftarr)
bispec_shape = (int(self.shape[0] / 2.), int(self.shape[1] / 2.))
self._bispectrum = np.zeros(bispec_shape, dtype=np.complex)
self._bicoherence = np.zeros(bispec_shape, dtype=np.float)
self._tracker = np.zeros(self.shape, dtype=np.int16)
biconorm = np.ones_like(self.bispectrum, dtype=float)
if show_progress:
bar = ProgressBar(np.prod(fftarr.shape) / 4.)
prod = product(range(int(fftarr.shape[0] / 2.)),
range(int(fftarr.shape[1] / 2.)))
with NumpyRNGContext(seed):
for n, (k1mag, k2mag) in enumerate(prod):
phi1 = ra.uniform(0, 2 * np.pi, nsamples)
phi2 = ra.uniform(0, 2 * np.pi, nsamples)
k1x = np.asarray([int(k1mag * np.cos(angle))
for angle in phi1])
k2x = np.asarray([int(k2mag * np.cos(angle))
for angle in phi2])
k1y = np.asarray([int(k1mag * np.sin(angle))
for angle in phi1])
k2y = np.asarray([int(k2mag * np.sin(angle))
for angle in phi2])
k3x = np.asarray([int(k1mag * np.cos(ang1) +
k2mag * np.cos(ang2))
for ang1, ang2 in zip(phi1, phi2)])
k3y = np.asarray([int(k1mag * np.sin(ang1) +
k2mag * np.sin(ang2))
for ang1, ang2 in zip(phi1, phi2)])
samps = fftarr[k1x, k1y] * fftarr[k2x, k2y] * conjfft[k3x, k3y]
self._bispectrum[k1mag, k2mag] = np.sum(samps)
biconorm[k1mag, k2mag] = np.sum(np.abs(samps))
# Track where we're sampling from in fourier space
self._tracker[k1x, k1y] += 1
self._tracker[k2x, k2y] += 1
self._tracker[k3x, k3y] += 1
if show_progress:
bar.update(n + 1)
self._bicoherence = (np.abs(self.bispectrum) / biconorm)
self._bispectrum_amp = np.log10(np.abs(self.bispectrum))
catalog_k = np.sort([f for f in catalog if "k" in f])
catalog_j = np.sort([f for f in catalog if "j" in f])
#Obtiene el seeing
color_print('-Obteniendo menor seeing...','cyan')
ks,ksi = lowest_seeing(fits_k)
js,jsi = lowest_seeing(fits_j)
cmd_fits = fits_j[jsi], fits_k[ksi]
cmd_cata = catalog_j[jsi], catalog_k[ksi]
print '\tMenor seeing en K: %s (%f)' % (cmd_fits[1],ks)
print '\tMenor seeing en J: %s (%f)' % (cmd_fits[0],js)
#Obtiene los RA y DEC con WCS
color_print('-Obteniendo RADEC...','cyan')
if get_RADEC:
ProgressBar.map(XYtoRADEC,np.transpose([fits_k,catalog_k]),multiprocess=True)
ProgressBar.map(XYtoRADEC,np.transpose([fits_j,catalog_j]),multiprocess=True)
cmd_dat = cmd_fits[0].replace('fits','dat'), cmd_fits[1].replace('fits','dat')
#CMD (match y creacion)
color_print('-Match CMD...','cyan')
execute = 'sh stilts tmatch2 ifmt1=ascii ifmt2=ascii matcher=sky ofmt=ascii values1="RA DEC" values2="RA DEC" '
if match_CMD:
exec_CMD = 'in1=%s in2=%s out=%s params=%.1f progress=none join=all1' % (folder+cmd_dat[1],folder+cmd_dat[0],results+'CMD.dat',match_tolerance)
os.system(execute+exec_CMD)
#Match de todas las epocas con la de referencia (menor seeing)
color_print('-Match epocas...','cyan')
execute = execute.replace('values1="RA DEC"','values1="RA_1 DEC_1"')
if match_epo:
def quick_render_movie(self, outdir, size=256, nframes=30,
camera_angle=(0,0,1), north_vector=(0,0,1),
rot_vector=(1,0,0),
colormap='doom',
cmap_range='auto',
transfer_function='auto',
start_index=0,
image_prefix="",
output_filename='out.mp4',
log_scale=False,
rescale=True):
"""
Create a movie rotating the cube 360 degrees from
PP -> PV -> PP -> PV -> PP
Parameters
----------
outdir: str
The output directory in which the individual image frames and the
resulting output mp4 file should be stored
size: int
The size of the individual output frame in pixels (i.e., size=256
will result in a 256x256 image)
nframes: int
The number of frames in the resulting movie
camera_angle: 3-tuple
The initial angle of the camera
north_vector: 3-tuple
The vector of 'north' in the data cube. Default is coincident with
the spectral axis
rot_vector: 3-tuple
The vector around which the camera will be rotated
colormap: str
A valid colormap. See `yt.show_colormaps`
transfer_function: 'auto' or `yt.visualization.volume_rendering.TransferFunction`
Either 'auto' to use the colormap specified, or a valid
TransferFunction instance
log_scale: bool
Should the colormap be log scaled?
rescale: bool
If True, the images will be rescaled to have a common 95th
percentile brightness, which can help reduce flickering from having
a single bright pixel in some projections
start_index : int
The number of the first image to save
image_prefix : str
A string to prepend to the image name for each image that is output
output_filename : str
The movie file name to output. The suffix may affect the file type
created. Defaults to 'out.mp4'. Will be placed in ``outdir``
Returns
-------
"""
if not ytOK:
raise IOError("yt could not be imported. Cube renderings are not possible.")
scale = np.max(self.cube.shape)
if not os.path.exists(outdir):
os.makedirs(outdir)
elif not os.path.isdir(outdir):
raise OSError("Output directory {0} exists and is not a directory.".format(outdir))
if cmap_range == 'auto':
upper = self.cube.max().value
lower = self.cube.std().value * 3
cmap_range = [lower,upper]
if transfer_function == 'auto':
tfh = self.auto_transfer_function(cmap_range, log=log_scale)
tfh.tf.map_to_colormap(cmap_range[0], cmap_range[1], colormap=colormap)
tf = tfh.tf
else:
tf = transfer_function
center = self.dataset.domain_center
cam = self.dataset.h.camera(center, camera_angle, scale, size, tf,
north_vector=north_vector, fields='flux')
im = cam.snapshot()
images = [im]
pb = ProgressBar(nframes)
for ii,im in enumerate(cam.rotation(2 * np.pi, nframes,
rot_vector=rot_vector)):
images.append(im)
im.write_png(os.path.join(outdir,"%s%04i.png" % (image_prefix,
ii+start_index)),
rescale=False)
pb.update(ii+1)
log.info("Rendering complete in {0}s".format(time.time() - pb._start_time))
if rescale:
_rescale_images(images, os.path.join(outdir, image_prefix))
pipe = _make_movie(outdir, prefix=image_prefix,
filename=output_filename)
return images
def convolve_model_dir_monochromatic(model_dir, overwrite=False, max_ram=8,
wav_min=-np.inf * u.micron, wav_max=np.inf * u.micron):
"""
Convolve all the model SEDs in a model directory
Parameters
----------
model_dir : str
The path to the model directory
overwrite : bool, optional
Whether to overwrite the output files
max_ram : float, optional
The maximum amount of RAM that can be used (in Gb)
wav_min : float, optional
The minimum wavelength to consider. Only wavelengths above this value
will be output.
wav_max : float, optional
The maximum wavelength to consider. Only wavelengths below this value
will be output.
"""
modpar = parfile.read(os.path.join(model_dir, 'models.conf'), 'conf')
if modpar.get('version', 1) > 1:
raise ValueError("monochromatic filters are no longer used for new-style model directories")
# Create 'convolved' sub-directory if needed
if not os.path.exists(model_dir + '/convolved'):
os.mkdir(model_dir + '/convolved')
# Find all SED files to convolve
sed_files = (glob.glob(model_dir + '/seds/*.fits.gz') +
glob.glob(model_dir + '/seds/*/*.fits.gz') +
glob.glob(model_dir + '/seds/*.fits') +
glob.glob(model_dir + '/seds/*/*.fits'))
par_table = load_parameter_table(model_dir)
# Find number of models
n_models = len(sed_files)
if n_models == 0:
raise Exception("No SEDs found in %s" % model_dir)
else:
log.info("{0} SEDs found in {1}".format(n_models, model_dir))
# Find out apertures and wavelengths
first_sed = SED.read(sed_files[0])
n_ap = first_sed.n_ap
apertures = first_sed.apertures
n_wav = first_sed.n_wav
wavelengths = first_sed.wav
# For model grids that are very large, it is not possible to compute all
# fluxes in one go, so we need to process in chunks in wavelength space.
chunk_size = min(n_wav, int(np.floor(max_ram * 1024. ** 3 / (4. * 2. * n_models * n_ap))))
if chunk_size == n_wav:
log.info("Producing all monochromatic files in one go")
else:
log.info("Producing monochromatic files in chunks of {0}".format(chunk_size))
filters = Table()
filters['wav'] = wavelengths
filters['filter'] = np.zeros(wavelengths.shape, dtype='S10')
# Figure out range of wavelength indices to use
# (wavelengths array is sorted in reverse order)
jlo = n_wav - 1 - (wavelengths[::-1].searchsorted(wav_max) - 1)
jhi = n_wav - 1 - wavelengths[::-1].searchsorted(wav_min)
chunk_size = min(chunk_size, jhi - jlo + 1)
# Loop over wavelength chunks
for jmin in range(jlo, jhi, chunk_size):
# Find upper wavelength to compute
jmax = min(jmin + chunk_size - 1, jhi)
log.info('Processing wavelengths {0} to {1}'.format(jmin, jmax))
# Set up convolved fluxes
fluxes = [ConvolvedFluxes(model_names=np.zeros(n_models, dtype='U30' if six.PY3 else 'S30'), apertures=apertures, initialize_arrays=True) for i in range(chunk_size)]
b = ProgressBar(len(sed_files))
# Loop over SEDs
for im, sed_file in enumerate(sed_files):
b.update()
log.debug('Processing {0}'.format(os.path.basename(sed_file)))
# Read in SED
s = SED.read(sed_file, unit_freq=u.Hz, unit_flux=u.mJy, order='nu')
# Convolve
for j in range(chunk_size):
fluxes[j].central_wavelength = wavelengths[j + jmin]
fluxes[j].apertures = apertures
fluxes[j].model_names[im] = s.name
if n_ap == 1:
fluxes[j].flux[im] = s.flux[0, j + jmin]
fluxes[j].error[im] = s.error[0, j + jmin]
else:
fluxes[j].flux[im, :] = s.flux[:, j + jmin]
fluxes[j].error[im, :] = s.error[:, j + jmin]
for j in range(chunk_size):
fluxes[j].sort_to_match(par_table['MODEL_NAME'])
fluxes[j].write('{0:s}/convolved/MO{1:03d}.fits'.format(model_dir, j + jmin + 1),
overwrite=overwrite)
filters['filter'][j + jmin] = "MO{0:03d}".format(j + jmin + 1)
return filters
def compute_surface(self, boundary='continuous', show_progress=True):
'''
Computes the SCF up to the given lag value. This is an
expensive operation and could take a long time to calculate.
Parameters
----------
boundary : {"continuous", "cut"}
Treat the boundary as continuous (wrap-around) or cut values
beyond the edge (i.e., for most observational data).
show_progress : bool, optional
Show a progress bar when computing the surface. =
'''
if boundary not in ["continuous", "cut"]:
raise ValueError("boundary must be 'continuous' or 'cut'.")
self._scf_surface = np.zeros((self.size, self.size))
# Convert the lags into pixel units.
pix_lags = self._to_pixel(self.roll_lags).value
dx = pix_lags.copy()
dy = pix_lags.copy()
if show_progress:
bar = ProgressBar(len(dx) * len(dy))
for n, (x_shift, y_shift) in enumerate(product(dx, dy)):
i, j = np.unravel_index(n, (len(dx), len(dy)))
if x_shift == 0 and y_shift == 0:
self._scf_surface[j, i] = 1.
if x_shift == 0:
tmp = self.data
else:
if float(x_shift).is_integer():
shift_func = pixel_shift
else:
shift_func = fourier_shift
tmp = shift_func(self.data, x_shift, axis=1)
if y_shift != 0:
if float(y_shift).is_integer():
shift_func = pixel_shift
else:
shift_func = fourier_shift
tmp = shift_func(tmp, y_shift, axis=2)
if boundary is "cut":
# Always round up to the nearest integer.
x_shift = np.ceil(x_shift).astype(int)
y_shift = np.ceil(y_shift).astype(int)
if x_shift < 0:
x_slice_data = slice(None, tmp.shape[1] + x_shift)
x_slice_tmp = slice(-x_shift, None)
else:
x_slice_data = slice(x_shift, None)
x_slice_tmp = slice(None, tmp.shape[1] - x_shift)
if y_shift < 0:
y_slice_data = slice(None, tmp.shape[2] + y_shift)
y_slice_tmp = slice(-y_shift, None)
else:
y_slice_data = slice(y_shift, None)
y_slice_tmp = slice(None, tmp.shape[2] - y_shift)
data_slice = (slice(None), x_slice_data, y_slice_data)
tmp_slice = (slice(None), x_slice_tmp, y_slice_tmp)
elif boundary is "continuous":
data_slice = (slice(None),) * 3
tmp_slice = (slice(None),) * 3
values = \
np.nansum(((self.data[data_slice] - tmp[tmp_slice]) ** 2),
axis=0) / \
(np.nansum(self.data[data_slice] ** 2, axis=0) +
np.nansum(tmp[tmp_slice] ** 2, axis=0))
scf_value = 1. - \
np.sqrt(np.nansum(values) / np.sum(np.isfinite(values)))
if scf_value > 1:
raise ValueError("Cannot have a correlation above 1. Check "
"your input data. Contact the TurbuStat "
"authors if the problem persists.")
self._scf_surface[j, i] = scf_value
if show_progress:
bar.update(n + 1)
name = np.genfromtxt(folder+'zinfo_img',unpack=True,usecols=(0,),dtype='string')
k_mask = np.array(['k' in f for f in name])
sek,elk,yrk = np.transpose([se,el,yr])[k_mask].T
yrk = (yrk-yrk[0])/365.242199
color_print('Recopilando archivos de epocas...','cyan')
epochs = glob.glob('./%s/*.*' % match_folder)
color_print('Realizando match de la MF con las epocas','cyan')
ejecuta = 'java -jar %s/stilts.jar tmatch2 in1=./%s values1="ID" ifmt1=ascii ' % (stilts_folder, master)
def mf_match(ep):
ej2 = 'in2=%s values2="ID_1" ifmt2=ascii icmd2=\'keepcols "ID_1 X Y"\' matcher=exact find=best join=1and2 out=./%s/%s ofmt=ascii progress=none ocmd="delcols ID_1"' % (ep, match_master, ep.split('/')[-1].replace('.match','.mfma'))
os.system(ejecuta + ej2)
ProgressBar.map(mf_match,epochs,multiprocess=True)
color_print('Realizando transformaciones lineales','cyan')
matches = glob.glob('./%s/*.*' % match_master)
bid = np.genfromtxt(locales,unpack=True,usecols=(0,))
def shift(ep):
ids,x1,y1,mk,mj,x2,y2 = np.genfromtxt(ep,unpack=True)
local_mask = np.in1d(ids,bid)
lid,lx1,ly1,lx2,ly2 = np.transpose([ids,x1,y1,x2,y2])[local_mask].T
loc_xy = np.transpose([lx2,ly2])
nbrs = NN(n_neighbors=vecinos, algorithm='auto').fit(loc_xy)
coo_xy = np.transpose([x2,y2])
