def emissive_radiance_old(emissivity, T, wl):
"""Radiance of a surface due to emission"""
h = 6.62607004e-34 # m2 kg s-1
c = 299792458 # m s-1
numerator = 2.0 * h * (c**2) # m4 kg s-1
wl_m = wl * 1e-9
numerator_per_lam5 = numerator * pow(wl_m, -5) # kg s-1 m-1
k = 1.380648520 - 23 # Boltzmann constant, m2 kg s-2 K-1
denom = s.exp(h * c / (k * wl_m * T)) - 1.0 # dimensionless
L = numerator_per_lam5 / denom # Watts per m3
cm2_per_m2, nm_per_m, uW_per_W = 10000, 1e9, 1e6
conversion = cm2_per_m2 * nm_per_m * uW_per_W / s.pi # -> uW nm-1 cm-2 sr-1
L = L * conversion
ddenom_dT = s.exp(h * c / (k * wl_m * T)) * h * c * (-1.0) / (pow(
k * wl_m * T, 2)) * k * wl_m
dL_dT = -numerator_per_lam5 / pow(denom, 2.0) * ddenom_dT * conversion
L = L * emissivity
dL_dT = dL_dT * emissivity
L[s.logical_not(s.isfinite(L))] = 0
dL_dT[s.logical_not(s.isfinite(dL_dT))] = 0
return L, dL_dT
def filter_nana(ts_data, ts_ticks):
if ts_data.ndim > 1:
raise ValueError("filtering-NaN is only defined for" +
"one dimensional time series data.")
ts_ticks = compress(logical_not(isnan(ts_data)), ts_ticks)
ts_data = compress(logical_not(isnan(ts_data)), ts_data)
return ts_data, ts_ticks
def _check_bounds(self, x_new, y_new):
"""Check the inputs for being in the bounds of the interpolated data.
Args:
x_new (float array):
y_new (float array):
Returns:
out_of_bounds (Boolean array): The mask on x_new and y_new of
values that are NOT of bounds.
"""
below_bounds_x = x_new < self._xlim[0]
above_bounds_x = x_new > self._xlim[1]
below_bounds_y = y_new < self._ylim[0]
above_bounds_y = y_new > self._ylim[1]
# !! Could provide more information about which values are out of bounds
if self.bounds_error and below_bounds_x.any():
raise ValueError("A value in x is below the interpolation "
"range.")
if self.bounds_error and above_bounds_x.any():
raise ValueError("A value in x is above the interpolation "
"range.")
if self.bounds_error and below_bounds_y.any():
raise ValueError("A value in y is below the interpolation "
"range.")
if self.bounds_error and above_bounds_y.any():
raise ValueError("A value in y is above the interpolation "
"range.")
out_of_bounds = scipy.logical_not(scipy.logical_or(scipy.logical_or(below_bounds_x, above_bounds_x),
scipy.logical_or(below_bounds_y, above_bounds_y)))
return out_of_bounds
def add_data_2_map(data, ra_inds, dec_inds, map, noise_i=None, weight=1):
"""Add a data masked array to a map.
This function also adds the weight to the noise matrix for diagonal noise.
"""
ntime = len(ra_inds)
shape = sp.shape(map)
if len(dec_inds) != ntime or len(data[:, 0]) != ntime:
raise ValueError('Time axis of data, ra_inds and dec_inds must be'
' same length.')
if not noise_i is None and map.shape != noise_i.shape:
raise ValueError('Inverse noise array must be the same size as the map'
' or None.')
for time_ind in range(ntime):
if (ra_inds[time_ind] >= 0 and ra_inds[time_ind] < shape[0]
and dec_inds[time_ind] >= 0 and dec_inds[time_ind] < shape[1]):
# Get unmasked
unmasked_inds = sp.logical_not(ma.getmaskarray(data[time_ind, :]))
ind_map = (ra_inds[time_ind], dec_inds[time_ind], unmasked_inds)
map[ind_map] += (weight * data)[time_ind, unmasked_inds]
if not noise_i is None:
if not hasattr(weight, '__iter__'):
noise_i[ind_map] += weight
else:
noise_i[ind_map] += weight[unmasked_inds]
def _check_bounds(self, x_new, y_new):
"""Check the inputs for being in the bounds of the interpolated data.
Args:
x_new (float array):
y_new (float array):
Returns:
out_of_bounds (Boolean array): The mask on x_new and y_new of
values that are NOT of bounds.
"""
below_bounds_x = x_new < self._xlim[0]
above_bounds_x = x_new > self._xlim[1]
below_bounds_y = y_new < self._ylim[0]
above_bounds_y = y_new > self._ylim[1]
# !! Could provide more information about which values are out of bounds
if self.bounds_error and below_bounds_x.any():
raise ValueError("A value in x is below the interpolation "
"range.")
if self.bounds_error and above_bounds_x.any():
raise ValueError("A value in x is above the interpolation "
"range.")
if self.bounds_error and below_bounds_y.any():
raise ValueError("A value in y is below the interpolation "
"range.")
if self.bounds_error and above_bounds_y.any():
raise ValueError("A value in y is above the interpolation "
"range.")
out_of_bounds = scipy.logical_not(scipy.logical_or(scipy.logical_or(below_bounds_x, above_bounds_x),
scipy.logical_or(below_bounds_y, above_bounds_y)))
return out_of_bounds
def add_data_2_map(data, ra_inds, dec_inds, map, noise_i=None, weight=1):
"""Add a data masked array to a map.
This function also adds the weight to the noise matrix for diagonal noise.
"""
ntime = len(ra_inds)
shape = sp.shape(map)
if len(dec_inds) != ntime or len(data[:, 0]) != ntime:
raise ValueError("Time axis of data, ra_inds and dec_inds must be" " same length.")
if not noise_i is None and map.shape != noise_i.shape:
raise ValueError("Inverse noise array must be the same size as the map" " or None.")
for time_ind in range(ntime):
if (
ra_inds[time_ind] >= 0
and ra_inds[time_ind] < shape[0]
and dec_inds[time_ind] >= 0
and dec_inds[time_ind] < shape[1]
):
# Get unmasked
unmasked_inds = sp.logical_not(ma.getmaskarray(data[time_ind, :]))
ind_map = (ra_inds[time_ind], dec_inds[time_ind], unmasked_inds)
map[ind_map] += (weight * data)[time_ind, unmasked_inds]
if not noise_i is None:
if not hasattr(weight, "__iter__"):
noise_i[ind_map] += weight
else:
noise_i[ind_map] += weight[unmasked_inds]
def cross(series, cross=0, direction='cross'):
"""
From http://stackoverflow.com/questions/10475488/calculating-crossing-intercept-points-of-a-series-or-dataframe
Given a Series returns all the index values where the data values equal
the 'cross' value.
Direction can be 'rising' (for rising edge), 'falling' (for only falling
edge), or 'cross' for both edges
"""
# Find if values are above or bellow yvalue crossing:
above=series.values > cross
below=scipy.logical_not(above)
left_shifted_above = above[1:]
left_shifted_below = below[1:]
x_crossings = []
# Find indexes on left side of crossing point
if direction == 'rising':
idxs = (left_shifted_above & below[0:-1]).nonzero()[0]
elif direction == 'falling':
idxs = (left_shifted_below & above[0:-1]).nonzero()[0]
else:
rising = left_shifted_above & below[0:-1]
falling = left_shifted_below & above[0:-1]
idxs = (rising | falling).nonzero()[0]
# Calculate x crossings with interpolation using formula for a line:
x1 = series.index.values[idxs]
x2 = series.index.values[idxs+1]
y1 = series.values[idxs]
y2 = series.values[idxs+1]
x_crossings = (cross-y1)*(x2-x1)/(y2-y1) + x1
return x_crossings
def pixel_counts(data, ra_inds, dec_inds, pixel_hits, map_shape=(-1, -1)):
"""Counts the hits on each unique pixel.
Returns pix_list, a list of tuples, each tuple is a (ra,dec) index on a
map pixel hit on this scan. The list only contains unique entries. The
array pixel_hits (preallocated for performance), is
filled with the number of hits on each of these pixels as a function of
frequency index. Only the entries pixel_hits[:len(pix_list),:]
are meaningful.
"""
if ra_inds.shape != dec_inds.shape or ra_inds.ndim != 1:
raise ValueError('Ra and Dec arrays not properly shaped.')
if (pixel_hits.shape[-1] != data.shape[-1]
or pixel_hits.shape[0] < len(ra_inds)):
raise ValueError('counts not allowcated to right shape.')
pix_list = []
for ii in range(len(ra_inds)):
pix = (ra_inds[ii], dec_inds[ii])
if ((map_shape[0] > -1 and pix[0] >= map_shape[0])
or (map_shape[1] > -1 and pix[1] >= map_shape[1]) or pix[0] < 0
or pix[1] < 0):
continue
elif not pix in pix_list:
pix_list.append(pix)
unmasked_freqs = sp.logical_not(ma.getmaskarray(data)[ii, :])
pixel_hits[pix_list.index(pix), unmasked_freqs] += 1
return pix_list
def pixel_counts(data, ra_inds, dec_inds, pixel_hits, map_shape=(-1, -1)):
"""Counts the hits on each unique pixel.
Returns pix_list, a list of tuples, each tuple is a (ra,dec) index on a
map pixel hit on this scan. The list only contains unique entries. The
array pixel_hits (preallowcated for performance), is
filled with the number of hits on each of these pixels as a function of
frequency index. Only the entries pixel_hits[:len(pix_list), :]
are meaningful.
"""
if ra_inds.shape != dec_inds.shape or ra_inds.ndim != 1:
raise ValueError("Ra and Dec arrays not properly shaped.")
if pixel_hits.shape[-1] != data.shape[-1] or pixel_hits.shape[0] < len(ra_inds):
raise ValueError("counts not allowcated to right shape.")
pix_list = []
for ii in range(len(ra_inds)):
pix = (ra_inds[ii], dec_inds[ii])
if (
(map_shape[0] > -1 and pix[0] >= map_shape[0])
or (map_shape[1] > -1 and pix[1] >= map_shape[1])
or pix[0] < 0
or pix[1] < 0
):
continue
elif not pix in pix_list:
pix_list.append(pix)
unmasked_freqs = sp.logical_not(ma.getmaskarray(data)[ii, :])
pixel_hits[pix_list.index(pix), unmasked_freqs] += 1
return pix_list
def assess_calibration(self):
"""Assess if PredPol is calibrated by conditioning on predicted intensity
and checking the correlation between number of crimes and demographics.
Returns: a 2D array where the first dimension is the number of days in
the test set and the second dimension is the number of bins for the
range of predicted intensities, as computed by `sp.histogram_bin_edges`.
The entry in the ith row and jth column is the Pearson correlation
coefficient between race and actual number of crimes in the jth bin of
predicted intensity for the ith day.
"""
black = self.pred_obj.grid_cells.black
not_nan = sp.logical_not(sp.isnan(black.values))
bins = sp.histogram_bin_edges(self.get_predicted_intensities(), bins='auto')
correlations = sp.empty((len(self.lambda_columns), len(bins)))
correlations[:] = sp.nan
for i, (lambda_col, actual_col) in self._iterator():
idx_bins = sp.digitize(self.results[lambda_col], bins)
for j in range(len(bins)):
idx_selected = sp.logical_and(idx_bins == j, not_nan)
if sp.sum(idx_selected) > 2:
actual = self.results.loc[idx_selected, actual_col]
demographics = black.loc[idx_selected]
correlations[i, j] = sp.stats.pearsonr(actual, demographics)[0]
return correlations
def plot_mat(self):
###
da = sp.copy(self._mat["DA"])
if sp.trace(da) != 0.:
da = utils.getCorrelationMatrix(da)
w = (self._mat["WE"] > 0.) & (self._mat["NB"] > 10)
da[sp.logical_not(w)] = sp.nan
fig = plt.figure()
ax = fig.add_subplot(111)
plt.imshow(da, origin="lower", interpolation='nearest')
cbar = plt.colorbar()
plt.grid(True)
cbar.formatter.set_powerlimits((0, 0))
cbar.update_ticks()
plt.show()
###
for k in range(5):
x = sp.arange(sp.diag(self._mat["DA"], k=k).size)
y = sp.diag(self._mat["DA"], k=k)
w = (sp.diag(self._mat["WE"], k=k) > 0.) & (sp.diag(
self._mat["NB"], k=k) > 10.)
x = x[w]
y = y[w]
plt.plot(x, y, linewidth=4, alpha=0.7)
plt.grid()
plt.show()
return
def plot_2d(self, log=False):
crt = 1. / scipy.constants.degree
if ((self._we > 0.).sum() == 0):
print("no data")
return
origin = 'lower'
extent = [
crt * self._rt_min, crt * self._rt_max, self._rp_min, self._rp_max
]
if (self._correlation == 'o_f' or self._correlation == 'f_f2'):
origin = 'upper'
extent = [
crt * self._rt_min, crt * self._rt_max, self._rp_max,
self._rp_min
]
yyy = sp.copy(self._da)
w = (self._we > 0.) & (self._nb > 10.)
yyy[sp.logical_not(w)] = float('nan')
if log:
yyy[w] = sp.log10(sp.absolute(yyy[w]))
yyy = utils.convert1DTo2D(yyy, self._np, self._nt)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_xticks([
i for i in sp.arange(crt * self._rt_min, crt * self._rt_max, crt *
self._binSizeT * 10)
])
ax.set_yticks([
i
for i in sp.arange(self._rp_min, self._rp_max, self._binSizeP * 10)
])
plt.imshow(yyy,
origin=origin,
extent=extent,
interpolation='nearest',
aspect='auto')
cbar = plt.colorbar()
if not log:
cbar.set_label(r'$\xi(\lambda_{1}/\lambda_{2},\theta)$', size=40)
else:
cbar.set_label(
r'$ \log10 \, |\xi(\lambda_{1}/\lambda_{2},\theta)| $',
size=40)
plt.xlabel(r'$\theta \, [\mathrm{deg}]$', fontsize=40)
plt.ylabel(r'$\lambda_{1}/\lambda_{2}$', fontsize=40)
plt.grid(True)
cbar.formatter.set_powerlimits((0, 0))
cbar.update_ticks()
plt.show()
return
def readSRI_h5(fn,params,timelims = None):
assert isinstance(params,(tuple,list))
h5fn = Path(fn).expanduser()
'''This will read the SRI formated h5 files for RISR and PFISR.'''
coordnames = 'Spherical'
# Set up the dictionary to find the data
pathdict = {'Ne':('/FittedParams/Ne', None),
'dNe':('/FittedParams/Ne',None),
'Vi':('/FittedParams/Fits', (0,3)),
'dVi':('/FittedParams/Errors',(0,3)),
'Ti':('/FittedParams/Fits', (0,1)),
'dTi':('/FittedParams/Errors',(0,1)),
'Te':('/FittedParams/Fits', (-1,1)),
'Ti':('/FittedParams/Errors',(-1,1))}
with h5py.File(str(h5fn),'r',libver='latest') as f:
# Get the times and time lims
times = f['/Time/UnixTime'].value
# get the sensor location
sensorloc = np.array([f['/Site/Latitude'].value,
f['/Site/Longitude'].value,
f['/Site/Altitude'].value])
# Get the locations of the data points
rng = f['/FittedParams/Range'].value / 1e3
angles = f['/BeamCodes'][:,1:3]
nt = times.shape[0]
if timelims is not None:
times = times[(times[:,0]>= timelims[0]) & (times[:,1]<timelims[1]) ,:]
nt = times.shape[0]
# allaz, allel corresponds to rng.ravel()
allaz = np.tile(angles[:,0],rng.shape[1])
allel = np.tile(angles[:,1],rng.shape[1])
dataloc =np.vstack((rng.ravel(),allaz,allel)).T
# Read in the data
data = {}
with h5py.File(str(h5fn),'r',libver='latest') as f:
for istr in params:
if not istr in pathdict.keys(): #list() NOT needed
logging.error('{} is not a valid parameter name.'.format(istr))
continue
curpath = pathdict[istr][0]
curint = pathdict[istr][-1]
if curint is None: #3-D data
tempdata = f[curpath]
else: #5-D data -> 3-D data
tempdata = f[curpath][:,:,:,curint[0],curint[1]]
data[istr] = np.array([tempdata[iT,:,:].ravel() for iT in range(nt)]).T
# remove nans from SRI file
nanlog = sp.any(sp.isnan(dataloc),1)
keeplog = sp.logical_not(nanlog)
dataloc = dataloc[keeplog]
for ikey in data.keys():
data[ikey]= data[ikey][keeplog]
return (data,coordnames,dataloc,sensorloc,times)
def readSRI_h5(fn,params,timelims = None):
assert isinstance(params,(tuple,list))
h5fn = Path(fn).expanduser()
'''This will read the SRI formated h5 files for RISR and PFISR.'''
coordnames = 'Spherical'
# Set up the dictionary to find the data
pathdict = {'Ne':('/FittedParams/Ne', None),
'dNe':('/FittedParams/Ne',None),
'Vi':('/FittedParams/Fits', (0,3)),
'dVi':('/FittedParams/Errors',(0,3)),
'Ti':('/FittedParams/Fits', (0,1)),
'dTi':('/FittedParams/Errors',(0,1)),
'Te':('/FittedParams/Fits', (-1,1)),
'Ti':('/FittedParams/Errors',(-1,1))}
with h5py.File(str(h5fn),'r',libver='latest') as f:
# Get the times and time lims
times = f['/Time/UnixTime'].value
# get the sensor location
sensorloc = np.array([f['/Site/Latitude'].value,
f['/Site/Longitude'].value,
f['/Site/Altitude'].value])
# Get the locations of the data points
rng = f['/FittedParams/Range'].value / 1e3
angles = f['/BeamCodes'][:,1:3]
nt = times.shape[0]
if timelims is not None:
times = times[(times[:,0]>= timelims[0]) & (times[:,1]<timelims[1]) ,:]
nt = times.shape[0]
# allaz, allel corresponds to rng.ravel()
allaz = np.tile(angles[:,0],rng.shape[1])
allel = np.tile(angles[:,1],rng.shape[1])
dataloc =np.vstack((rng.ravel(),allaz,allel)).T
# Read in the data
data = {}
with h5py.File(str(h5fn),'r',libver='latest') as f:
for istr in params:
if not istr in pathdict.keys(): #list() NOT needed
logging.error('{} is not a valid parameter name.'.format(istr))
continue
curpath = pathdict[istr][0]
curint = pathdict[istr][-1]
if curint is None: #3-D data
tempdata = f[curpath]
else: #5-D data -> 3-D data
tempdata = f[curpath][:,:,:,curint[0],curint[1]]
data[istr] = np.array([tempdata[iT,:,:].ravel() for iT in range(nt)]).T
# remove nans from SRI file
nanlog = sp.any(sp.isnan(dataloc),1)
keeplog = sp.logical_not(nanlog)
dataloc = dataloc[keeplog]
for ikey in data.keys():
data[ikey]= data[ikey][keeplog]
return (data,coordnames,dataloc,sensorloc,times)
def rebin(Data, n_bins_combined) :
"""The function that acctually does the rebinning on a Data Block."""
nt = Data.data.shape[0]
new_nt = nt // n_bins_combined
new_shape = (new_nt,) + Data.data.shape[1:]
unmask = sp.logical_not(ma.getmaskarray(Data.data))
data = Data.data.filled(0)
# Allowcate memeory for the rebinned data.
new_data = ma.zeros(new_shape, dtype=data.dtype)
counts = sp.zeros(new_shape, dtype=int)
# Add up the bins to be combined.
for ii in range(n_bins_combined):
new_data += data[ii:new_nt * n_bins_combined:n_bins_combined,...]
counts += unmask[ii:new_nt * n_bins_combined:n_bins_combined,...]
new_data[counts == 0] = ma.masked
counts[counts == 0] = 1
new_data /= counts
Data.set_data(new_data)
# Now deal with all the other records that aren't the main data.
for field_name in Data.field.iterkeys():
# DATE-OBS is a string field so we have to write special code for it.
if field_name == "DATE-OBS":
time_field = Data.field[field_name]
new_field = sp.empty(new_nt, dtype=Data.field[field_name].dtype)
# Convert to float, average, then convert back to a string.
time_float = utils.time2float(time_field)
for ii in range(new_nt):
tmp_time = sp.mean(time_float[n_bins_combined * ii
: n_bins_combined * (ii + 1)])
new_field[ii] = utils.float2time(tmp_time)
Data.set_field(field_name, new_field,
axis_names=Data.field_axes[field_name],
format=Data.field_formats[field_name])
continue
# Only change fields that have a 'time' axis.
try:
time_axis = list(Data.field_axes[field_name]).index('time')
except ValueError:
continue
# For now, the time axis has to be the first axis.
if time_axis != 0:
msg = "Expected time to be the first axis for all fields."
raise NotImplementedError(msg)
field_data = Data.field[field_name]
if not field_data.dtype.name == "float64":
msg = "Field data type is not float. Handle explicitly."
raise NotImplementedError(msg)
new_field = sp.empty(field_data.shape[:time_axis] + (new_nt,)
+ field_data.shape[time_axis + 1:],
dtype=field_data.dtype)
for ii in range(new_nt):
tmp_data = sp.sum(field_data[n_bins_combined * ii
:n_bins_combined * (ii + 1),...], 0)
tmp_data /= n_bins_combined
new_field[ii,...] = tmp_data
Data.set_field(field_name, new_field,
axis_names=Data.field_axes[field_name],
format=Data.field_formats[field_name])
def _p(self, s):
"""Transition probability function
Parameters
-----------
s : int
Integer representing the province
Returns
-----------
sprime : list
Each element is a tuple with the transition state
and the probability of transitioning to that state.
"""
# I enumerate states such that
# 0 = [0, ...., 0]
# 1 = [1, 0, 0, 0, ... 0 ]
# 2 = [0, 1, 0, 0, ... 0 ]
# 2^k = [0, 0, ..., 0, 1]
# 2^(k+1) - 1 = [1, 1, ..., 1, 1]
sbinary = int2binary(s, width=self.k)[::-1]
# indices of Revolting provinces
R = sbinary.nonzero()[0].tolist()
# indices of Non-revolting provinces
C = sp.logical_not(sbinary).nonzero()[0]
P = [1]
newstate = [s]
for i, si in enumerate(sbinary):
## Calculate
Ci = list(set(C) - set([i]))
if Ci:
dminc = self.D[sp.ix_([i], Ci)].min()
else:
dminc = self.dmax
Ri = list(set(R) - set([i]))
if Ri:
dminr = sp.c_[self.D[sp.ix_([i], Ri)]].min()
else:
dminr = self.dmax
# if i in revolt
if si:
Si = s - 2**i
Pi = dminr / dminc
# if i not in revolt
else:
Si = s + 2**i
Pi = dminc / dminr
P.append(Pi)
newstate.append(Si)
newstate = sp.array(newstate)
P = sp.array(P)
P /= P.sum()
return (P.tolist(), newstate.tolist())
def goodGIW(time, shot, name="c_W"):
"""extract values which are only valid, otherwise place in zeros"""
temp = GIWData(shot, data=name)
interp = scipy.interpolate.interp1d(temp.time,
temp.data,
bounds_error=False)
output = interp(time)
output[ scipy.logical_not(scipy.isfinite(output))] = 0. #force all bad values to zero
return output
def get_time_trapped(self, trap_num=None, straight_shots=False):
#adjusted this function to isolate flies that went straight to traps
mask_trapped = self.mode == self.Mode_Trapped
if straight_shots:
mask_trapped = mask_trapped & scipy.logical_not(self.ever_tracked)
if trap_num is None:
return self.t_in_trap[mask_trapped]
else:
mask_trapped_in_num = mask_trapped & (self.trap_num == trap_num)
return self.t_in_trap[mask_trapped_in_num]
def msd_of_linker_end_points(self, fregions, flabels, rregions, rlabels):
"""
takes definitions of regions and labels for forward and reverse
strands assuming each region in the forward direction coorosponds to a region
in the reverse direction and returns the mean squaared distance
of the start and end points of each region.
"""
if len(fregions) != len(rregions):
return (float('Inf'))
flinks = fregions[scipy.logical_not(flabels)]
rlinks = rregions[scipy.logical_not(rlabels)]
s = sum([
self._d[t1, t2] for t1, t2 in zip(flinks[:, 0], rlinks[:, 1] - 1)
if self._d[t1, t2]
]) / len(flinks)
f = sum([
self._d[t1, t2] for t1, t2 in zip(flinks[:, 1] - 1, rlinks[:, 0])
if self._d[t1, t2]
]) / len(flinks)
return ((s + f) / 2)
def calcaverage_sigmacutloop(self,data,mask=None,noise=None,Nsigma=3.0,Nitmax=10,fixmean=None,verbose=0,saveused=False,median_firstiteration=True):
"""
mask must have same dimensions than data. If mask[x]=True, then data[x] is not used.
noise must have same dimensions than data. If noise != None, then the error weighted mean is calculated.
if saveused, then self.use contains array of datapoints used, and self.clipped the array of datapoints clipped
median_firstiteration: in the first iteration, use the median instead the mean. This is more robust if there is a population of bad measurements
"""
self.reset()
#self.i=0
#self.converged=False
while ((self.i<Nitmax) or (Nitmax==0)) and (not self.converged):
medianflag = median_firstiteration and (self.i==0) and (Nsigma!=None)
if noise is None:
self.calcaverage_sigmacut(data,mask=mask,Nsigma=Nsigma,fixmean=fixmean,medianflag=medianflag,verbose=verbose)
else:
self.calcaverage_errorcut(data,mask=mask,noise=noise,Nsigma=Nsigma,medianflag=medianflag,verbose=verbose)
#if verbose>=2:
# print(self.__str__())
# Not converged???
if self.stdev==None or self.stdev==0.0 or self.mean==None:
self.converged=False
break
# Only do a sigma cut if wanted
if Nsigma == None or Nsigma == 0.0:
self.converged=True
break
# No changes anymore? If yes converged!!!
if (self.i>0) and (self.Nchanged==0):
self.converged=True
break
self.i+=1
if saveused:
if mask is None:
self.clipped = scipy.logical_not(self.use)
else:
self.clipped = scipy.logical_not(self.use) & scipy.logical_not(mask)
else:
del(self.use)
return(0)
def velocity_dof(domain, ax):
# Calculate velocity dof numbers forr each cell
rm = roll( domain, 1, axis=ax )
type_3 = logical_and( domain, rm )
type_2 = logical_or( domain, rm )
dof = cumsum( logical_not( logical_or( type_3, type_2 ) ) ).reshape( domain.shape ) - 1
# Do logic to figure out type 2 and 3
dof[type_2 == 1] = -2
dof[type_3 == 1] = -3
return dof.astype(int64)
def calcArealDens(l, te, halfte, rho, tped, tcore, nped, ncore):
minrho = scipy.argmin(rho)
spline1 = scipy.interpolate.interp1d(rho[:minrho + 1],
l[:minrho + 1],
bounds_error=False,
kind='linear')
spline2 = scipy.interpolate.interp1d(rho[minrho:],
l[minrho:],
bounds_error=False,
kind='linear')
# step 3, find te rho locations
ne = GIWprofiles.ne(GIWprofiles.te2rho2(te, tcore, tped), ncore, nped)
rhohalfte = GIWprofiles.te2rho2(halfte, tcore, tped)
bounds = scipy.array([rho[minrho], 1.])
boundte = GIWprofiles.te(bounds, tcore, tped)
bndidx = scipy.searchsorted(
halfte, boundte) #add proper endpoints to the temperature array
rhohalfte = scipy.insert(rhohalfte, bndidx,
bounds) #assumes that Te is positively increasing
#step 4, find l location for those 1/2 te locations AND rho=1 for endpoints
l1 = spline1(rhohalfte)
deltal1 = abs(l1[:-1] - l1[1:])
deltal1[scipy.logical_not(scipy.isfinite(deltal1))] = 0.
l2 = spline2(rhohalfte)
deltal2 = abs(l2[:-1] - l2[1:])
deltal2[scipy.logical_not(scipy.isfinite(deltal2))] = 0.
#plt.semilogx(te,ne*(deltal1*deltal2)/1e19, '.')
#plt.xlabel('deltal2')
#plt.show()
return pow(ne, 2) * (deltal1 + deltal2)
def weights2(inp, te, shot, time):
""" pull out data vs time necessary to calculate the weights"""
#condition initialized values
output = scipy.zeros((len(time), 2))
r = inp[0]
z = inp[1]
l = inp[2]
#load GIW ne, Te data
tped = goodGIW(time, shot, name="t_e_ped")
tcore = goodGIW(time, shot, name="t_e_core")
nped = goodGIW(time, shot, name="n_e_ped")
ncore = goodGIW(time, shot, name="n_e_core")
good = scipy.arange(len(time))[scipy.logical_and(
ncore != 0, tcore != 0)] #take only good data
#use spline of the GIW data to solve for the proper Te, otherwise dont evaluate
logte = scipy.log(te)
halfte = scipy.exp(logte[1:] / 2. +
logte[:-1] / 2.) #not going to worry about endpoints
# because I trust te is big enough to be larger than the range of the profile
#step 1, use array of r,z values to solve for rho
eq = eqtools.AUGDDData(shot)
rho = eq.rz2rho('psinorm', r, z, time[good], each_t=True,
sqrt=True) #solve at each time
rhomin = scipy.nanmin(rho, axis=1)
temax = GIWprofiles.te(rhomin, tcore[good], tped[good])
idx = 0
#step 2, construct 2 splines of l(rho)
for i in good:
temp = calcArealDens(l, te, halfte, rho[idx], tped[i], tcore[i],
nped[i], ncore[i])
temp[scipy.logical_not(scipy.isfinite(temp))] = 0.
temp = scipy.sum(temp)
output[i, 0] = temp
output[i, 1] = temax[idx]
idx += 1
#print(idx),
#step 8, return values for storage
return output
def flush_buffers(self):
"""Write to file, and refresh the memory map object"""
if self.format == 'ENVI':
if self.write:
for row, frame in self.frames.items():
valid = s.logical_not(s.isnan(frame[:, 0]))
if self.file.metadata['interleave'] == 'bil':
self.memmap[row, :, valid] = frame[valid, :].T
else:
self.memmap[row, valid, :] = frame[valid, :]
self.frames = OrderedDict()
del self.file
self.file = envi.open(self.fname + '.hdr', self.fname)
self.open_map_with_retries()
def plot_2d(self, x_power=0):
if ((self._we > 0.).sum() == 0):
print("no data")
return
origin = 'lower'
extent = [self._rt_min, self._rt_max, self._rp_min, self._rp_max]
if (self._correlation == 'o_f' or self._correlation == 'f_f2'):
origin = 'upper'
extent = [self._rt_min, self._rt_max, self._rp_max, self._rp_min]
yyy = sp.copy(self._da)
w = (self._we > 0.) & (self._nb > 10.)
yyy[sp.logical_not(w)] = float('nan')
xxx = utils.convert1DTo2D(self._r, self._np, self._nt)
yyy = utils.convert1DTo2D(yyy, self._np, self._nt)
coef = sp.power(xxx, x_power)
fig = plt.figure()
ax = fig.add_subplot(111)
#ax.set_xticks([ i for i in sp.arange(self._minX2D-50., self._maxX2D+50., 50.) ])
#ax.set_yticks([ i for i in sp.arange(self._minY2D-50., self._maxY2D+50., 50.) ])
plt.imshow(coef * yyy,
origin=origin,
extent=extent,
interpolation='nearest')
cbar = plt.colorbar()
if (x_power == 0):
cbar.set_label(r'$\xi(\, r_{\parallel},r_{\perp} \,)$', size=40)
if (x_power == 1):
cbar.set_label(r'$|r|.\xi(\, r_{\parallel},r_{\perp} \,)$',
size=40)
if (x_power == 2):
cbar.set_label(r'$|r|^{2}.\xi(\, r_{\parallel},r_{\perp} \,)$',
size=40)
plt.xlabel(r'$r_{\perp} \, [h^{-1} \, \rm{Mpc}]$', fontsize=40)
plt.ylabel(r'$r_{\parallel} \, [h^{-1} \, \rm{Mpc}]$', fontsize=40)
plt.grid(True)
cbar.formatter.set_powerlimits((0, 0))
cbar.update_ticks()
plt.show()
return
def compare_attributes(atts1,atts2):
self.assertEqual(len(atts1.keys()),len(atts2.keys()),"{}".format(nameRun))
self.assertListEqual(sorted(atts1.keys()),sorted(atts2.keys()),"{}".format(nameRun))
for item in atts1:
nequal = True
if isinstance(atts1[item],numpy.ndarray):
nequal = sp.logical_not(sp.array_equal(atts1[item],atts2[item]))
else:
nequal = atts1[item]!=atts2[item]
if nequal:
print("WARNING: {}: not exactly equal, using allclose for {}".format(nameRun,item))
print(atts1[item],atts2[item])
allclose = sp.allclose(atts1[item],atts2[item])
self.assertTrue(allclose,"{}".format(nameRun))
return
def masked_subtract_mean(data, mask, axis):
"""Subtracts the mean from data along axis given a mask."""
# Slice for broadcasting to the origional shape.
up_broad = [slice(None)] * data.ndim
up_broad[axis] = None
up_broad = tuple(up_broad)
# Calculate the mean.
un_mask = sp.logical_not(mask)
counts = sp.sum(un_mask, axis)
counts[counts == 0] = 1
mean = sp.sum(data * un_mask, axis) / counts
# Subtract the mean.
data = data - mean[up_broad] * un_mask
return data
def _van_rossum_multiunit_dist_for_trial_pair(a, b, weighting, tau, kernel):
if kernel is None:
spike_counts = sp.atleast_2d([st.size for st in a + b])
k_dist = spike_counts.T * (spike_counts - spike_counts.T)
else:
k_dist = kernel.summed_dist_matrix(a + b)
non_diagonal = sp.logical_not(sp.eye(len(a)))
summed_population = (sp.trace(k_dist) - sp.trace(k_dist, len(a)) -
sp.trace(k_dist, -len(a)))
labeled_line = (sp.sum(k_dist[:len(a), :len(a)][non_diagonal]) +
sp.sum(k_dist[len(a):, len(a):][non_diagonal]) -
sp.sum(k_dist[:len(a), len(a):][non_diagonal]) -
sp.sum(k_dist[len(a):, :len(a)][non_diagonal]))
return sp.sqrt(summed_population + weighting * labeled_line)
def masked_subtract_mean(data, mask, axis):
"""Subtracts the mean from data along axis given a mask."""
# Slice for broadcasting to the origional shape.
up_broad = [slice(None)] * data.ndim
up_broad[axis] = None
up_broad = tuple(up_broad)
# Calculate the mean.
un_mask = sp.logical_not(mask)
counts = sp.sum(un_mask, axis)
counts[counts == 0] = 1
mean = sp.sum(data * un_mask, axis) / counts
# Subtract the mean.
data = data - mean[up_broad] * un_mask
return data
def nothing(noth):
# If requested, remove the time gradient from all channels.
if remove_slope:
un_mask = sp.logical_not(ma.getmaskarray(NoiseData.data))
NoiseData.calc_time()
time = NoiseData.time
n_time = len(time)
# Test if the mask is the same for all slices. If it is, that greatly
# reduces the work as we only have to generate one set of polynomials.
all_masks_same = True
for jj in range(n_time):
if sp.all(un_mask[jj,...] == un_mask[jj,0,0,0]):
continue
else:
all_masks_same = False
break
if all_masks_same:
polys = misc.ortho_poly(time, 2, un_mask[:,0,0,0], 0)
polys.shape = (2, len(time), 1, 1, 1)
else:
polys = misc.ortho_poly(time[:,None,None,None], 2, un_mask, 0)
# Subtract the slope mode (1th mode) out of the NoiseData.
slope_amps = sp.sum(polys[1,...] * un_mask * NoiseData.data.filled(0),
0)
NoiseData.data -= polys[1,...] * slope_amps
# Iteratively flag on sliding scale to get closer and closer to desired
# threshold.
n_time = Data.data.shape[0]
max_thres = sp.sqrt(n_time)/2.
n_iter = 3
thresholds = (max_thres ** (n_iter - 1 - sp.arange(n_iter))
* thres ** sp.arange(n_iter)) ** (1./(n_iter - 1))
for threshold in thresholds:
# Get the deviation from the mean.
residuals = ma.anom(NoiseData.data, 0).filled(0)
# Get indices above the threshold.
mask = abs(residuals) > threshold * ma.std(NoiseData.data, 0)
# Mask the data.
Data.data[mask] = ma.masked
NoiseData.data[mask] = ma.masked
# Now flag for very noisey channels.
if max_noise_factor > 0:
vars = ma.var(NoiseData.data, 0)
mean_vars = ma.mean(vars, -1).filled(0)
bad_chans = vars.filled(0) > max_noise_factor * mean_vars[:,:,None]
Data.data[:,bad_chans] = ma.masked
NoiseData.data[:,bad_chans] = ma.masked
def nothing(noth):
# If requested, remove the time gradient from all channels.
if remove_slope:
un_mask = sp.logical_not(ma.getmaskarray(NoiseData.data))
NoiseData.calc_time()
time = NoiseData.time
n_time = len(time)
# Test if the mask is the same for all slices. If it is, that greatly
# reduces the work as we only have to generate one set of polynomials.
all_masks_same = True
for jj in range(n_time):
if sp.all(un_mask[jj, ...] == un_mask[jj, 0, 0, 0]):
continue
else:
all_masks_same = False
break
if all_masks_same:
polys = misc.ortho_poly(time, 2, un_mask[:, 0, 0, 0], 0)
polys.shape = (2, len(time), 1, 1, 1)
else:
polys = misc.ortho_poly(time[:, None, None, None], 2, un_mask, 0)
# Subtract the slope mode (1th mode) out of the NoiseData.
slope_amps = sp.sum(polys[1, ...] * un_mask * NoiseData.data.filled(0),
0)
NoiseData.data -= polys[1, ...] * slope_amps
# Iteratively flag on sliding scale to get closer and closer to desired
# threshold.
n_time = Data.data.shape[0]
max_thres = sp.sqrt(n_time) / 2.
n_iter = 3
thresholds = (max_thres**(n_iter - 1 - sp.arange(n_iter)) *
thres**sp.arange(n_iter))**(1. / (n_iter - 1))
for threshold in thresholds:
# Get the deviation from the mean.
residuals = ma.anom(NoiseData.data, 0).filled(0)
# Get indices above the threshold.
mask = abs(residuals) > threshold * ma.std(NoiseData.data, 0)
# Mask the data.
Data.data[mask] = ma.masked
NoiseData.data[mask] = ma.masked
# Now flag for very noisey channels.
if max_noise_factor > 0:
vars = ma.var(NoiseData.data, 0)
mean_vars = ma.mean(vars, -1).filled(0)
bad_chans = vars.filled(0) > max_noise_factor * mean_vars[:, :, None]
Data.data[:, bad_chans] = ma.masked
NoiseData.data[:, bad_chans] = ma.masked
def test_off_map(self) :
Data = self.blocks[0]
Data.calc_freq()
map = self.map
map[:,:,:] = 0.0
Data.data[:,:,:,:] = 0.0
# Rig the pointing but put one off the map.
def rigged_pointing() :
Data.ra = map.get_axis('ra')[range(10)]
Data.dec = map.get_axis('dec')[range(10)]
Data.ra[3] = Data.ra[3] - 8.0
Data.calc_pointing = rigged_pointing
smd.sub_map(Data, map)
self.assertTrue(sp.alltrue(ma.getmaskarray(Data.data[3,:,:,:])))
self.assertTrue(sp.alltrue(sp.logical_not(
ma.getmaskarray((Data.data[[0,1,2,4,5,6,7,8,9],:,:,:])))))
def _van_rossum_multiunit_dist_for_trial_pair(a, b, weighting, tau, kernel):
if kernel is None:
spike_counts = sp.atleast_2d([st.size for st in a + b])
k_dist = spike_counts.T * (spike_counts - spike_counts.T)
else:
k_dist = kernel.summed_dist_matrix(a + b)
non_diagonal = sp.logical_not(sp.eye(len(a)))
summed_population = (
sp.trace(k_dist) - sp.trace(k_dist, len(a)) - sp.trace(k_dist, -len(a)))
labeled_line = (
sp.sum(k_dist[:len(a), :len(a)][non_diagonal]) +
sp.sum(k_dist[len(a):, len(a):][non_diagonal]) -
sp.sum(k_dist[:len(a), len(a):][non_diagonal]) -
sp.sum(k_dist[len(a):, :len(a)][non_diagonal]))
return sp.sqrt(summed_population + weighting * labeled_line)
def load_rt(self, fn):
"""Load the results of a LibRadTran run."""
wl, rdn0, irr = s.loadtxt(self.lut_dir+'/LUT_'+fn+'_alb0.out').T
wl, rdn025, irr = s.loadtxt(self.lut_dir+'/LUT_'+fn+'_alb025.out').T
wl, rdn05, irr = s.loadtxt(self.lut_dir+'/LUT_'+fn+'_alb05.out').T
# Replace a few zeros in the irradiance spectrum via interpolation
good = irr > 1e-15
bad = s.logical_not(good)
irr[bad] = interp1d(wl[good], irr[good])(wl[bad])
# Translate to Top of Atmosphere (TOA) reflectance
rhoatm = rdn0 / 10.0 / irr * s.pi # Translate to uW nm-1 cm-2 sr-1
rho025 = rdn025 / 10.0 / irr * s.pi
rho05 = rdn05 / 10.0 / irr * s.pi
# Resample TOA reflectances to simulate the instrument observation
rhoatm = resample_spectrum(rhoatm, wl, self.wl, self.fwhm)
rho025 = resample_spectrum(rho025, wl, self.wl, self.fwhm)
rho05 = resample_spectrum(rho05, wl, self.wl, self.fwhm)
irr = resample_spectrum(irr, wl, self.wl, self.fwhm)
# Calculate some atmospheric optical constants
sphalb = 2.8*(2.0*rho025-rhoatm-rho05)/(rho025-rho05)
transm = (rho05-rhoatm)*(2.0-sphalb)
# For now, don't estimate this term!!
# TODO: Have LibRadTran calculate it directly
transup = s.zeros(self.wl.shape)
# Get solar zenith, translate to irradiance at zenith = 0
with open(self.lut_dir+'/LUT_'+fn+'.zen', 'r') as fin:
output = fin.read().split()
solzen, solaz = [float(q) for q in output[1:]]
self.coszen = s.cos(solzen/360.0*2.0*s.pi)
irr = irr / self.coszen
self.solar_irr = irr.copy()
results = {"wl": self.wl, 'solzen': solzen, 'irr': irr,
"solzen": solzen, "rhoatm": rhoatm, "transm": transm,
"sphalb": sphalb, "transup": transup}
return results
def change_times2state_vec(change_times,total_time,initial_state='low'):
"""Converts change times to a state vector. A state vector is a np.array of
bool or int type, 1 or True denotes 'high' and 0 or False denotes 'low'. Each
index corresponds to one step of the RIO timebase (which is 1 ms at the
moment)
Parameters
----------
change_times : np.array
total_time: int
the total lenght
initial_state: 'low' or 'high'
denotes whether the first state is low or high.
Returns
-------
state_vec: np.array
the state vector
"""
state_vec = sp.zeros(total_time)
# cast if not array
if type(change_times) != sp.ndarray:
change_times = sp.array(change_times)
# deconstruct, always pairs of two if even, thats all, if uneven, add one at end
if len(change_times) % 2 == 1:
# append the last timepoint if last pulse doesn't end
change_times = sp.concatenate((change_times,[total_time]))
# deconstruct into pulses
for i in range(0,len(change_times),2):
state_vec[change_times[i]+1:change_times[i+1]+1] = 1
state_vec = state_vec.astype('bool')
if initial_state == 'high':
state_vec = sp.logical_not(state_vec)
return state_vec
def read(file_name, map_inds=None, feedback=2):
"""Read a map from an image fits file.
"""
fname_abbr = ku.abbreviate_file_path(file_name)
if feedback > 0:
print 'Opening file: ' + fname_abbr
# Open the fits file.
hdulist = pyfits.open(file_name)
history = bf.get_history_header(hdulist[0].header)
history.add('Read from file.', 'File name: ' + fname_abbr)
map_list = []
if map_inds is None:
map_inds = range(1, len(hdulist))
elif not hasattr(map_inds, '__iter__'):
map_inds = (map_inds, )
elif len(scans) == 0:
map_inds = range(1, len(hdulist))
for ii in map_inds:
data = hdulist[ii].data
# Set the data attriute
Map = data_map.DataMap()
Map.set_data(sp.swapaxes(data, 0, 2))
# Masked data is stored in FITS files as float('nan')
Map.data[sp.logical_not(sp.isfinite(Map.data))] = ma.masked
Map.history = history
# Set the other fields.
for field_name in fields:
if not field_name in hdulist[1].header.keys():
continue
value = hdulist[1].header[field_name]
Map.set_field(field_name, value)
map_list.append(Map)
if len(map_list) == 1:
map_list = map_list[0]
return map_list
def read(file_name, map_inds=None, feedback=2):
"""Read a map from an image fits file.
"""
fname_abbr = ku.abbreviate_file_path(file_name)
if feedback > 0:
print "Opening file: " + fname_abbr
# Open the fits file.
hdulist = pyfits.open(file_name)
history = bf.get_history_header(hdulist[0].header)
history.add("Read from file.", "File name: " + fname_abbr)
map_list = []
if map_inds is None:
map_inds = range(1, len(hdulist))
elif not hasattr(map_inds, "__iter__"):
map_inds = (map_inds,)
elif len(scans) == 0:
map_inds = range(1, len(hdulist))
for ii in map_inds:
data = hdulist[ii].data
# Set the data attriute
Map = data_map.DataMap()
Map.set_data(sp.swapaxes(data, 0, 2))
# Masked data is stored in FITS files as float('nan')
Map.data[sp.logical_not(sp.isfinite(Map.data))] = ma.masked
Map.history = history
# Set the other fields.
for field_name in fields:
if not field_name in hdulist[1].header.keys():
continue
value = hdulist[1].header[field_name]
Map.set_field(field_name, value)
map_list.append(Map)
if len(map_list) == 1:
map_list = map_list[0]
return map_list
def test_off_map(self):
Data = self.blocks[0]
Data.calc_freq()
map = self.map
map[:, :, :] = 0.0
Data.data[:, :, :, :] = 0.0
# Rig the pointing but put one off the map.
def rigged_pointing():
Data.ra = map.get_axis('ra')[range(10)]
Data.dec = map.get_axis('dec')[range(10)]
Data.ra[3] = Data.ra[3] - 8.0
Data.calc_pointing = rigged_pointing
smd.sub_map(Data, map)
self.assertTrue(sp.alltrue(ma.getmaskarray(Data.data[3, :, :, :])))
self.assertTrue(
sp.alltrue(
sp.logical_not(
ma.getmaskarray(
(Data.data[[0, 1, 2, 4, 5, 6, 7, 8, 9], :, :, :])))))
def gen(conn, tedata, data, idx, time, shot, name='shots2016'):
# GET THAT SIIIIICK DATA YEAH
tped = goodGIW(time, shot, name="t_e_ped")
tcore = goodGIW(time, shot, name="t_e_core")
nped = goodGIW(time, shot, name="n_e_ped")
ncore = goodGIW(time, shot, name="n_e_core")
output = scipy.zeros(data.shape)
for i in xrange(len(time)):
print(i),
ne = GIWprofiles.ne(GIWprofiles.te2rho2(tedata, tcore[i], tped[i]),
ncore[i], nped[i])
ne[scipy.logical_not(scipy.isfinite(ne))] = 0.
output[i] = data[i] * ne
#multiply for new data
writeData(idx, output, conn, name)
'NT': nt,
'RTMAX': rt_max,
'RPMIN': rp_min,
'RPMAX': rp_max
}
h.close()
### same header
for i in range(nbData):
for k in ['NSIDE', 'HLPXSCHM', 'NP', 'NT', 'RTMAX', 'RPMIN', 'RPMAX']:
assert data[i][k] == data[0][k]
### Add unshared healpix as empty data
for i in range(nbData):
for j in range(nbData):
w = sp.logical_not(sp.in1d(data[j]['HEALPID'], data[i]['HEALPID']))
if w.sum() > 0:
new_healpix = data[j]['HEALPID'][w]
nb_new_healpix = new_healpix.size
nb_bins = data[i]['DA'].shape[1]
print("Some healpix are unshared in data {} vs. {}: {}".format(
i, j, new_healpix))
data[i]['DA'] = sp.append(data[i]['DA'],
sp.zeros((nb_new_healpix, nb_bins)),
axis=0)
data[i]['WE'] = sp.append(data[i]['WE'],
sp.zeros((nb_new_healpix, nb_bins)),
axis=0)
data[i]['HEALPID'] = sp.append(data[i]['HEALPID'], new_healpix)
### Sort the data by the healpix values
data['CO'] = sp.diag(co)
else:
print('INFO: Compute covariance from sub-sampling')
### To have same number of HEALPix
for d1 in list(lst_file.keys()):
for d2 in list(lst_file.keys()):
if data[d1]['NSIDE']!=data[d2]['NSIDE']:
print('ERROR: NSIDE are different: {} != {}'.format(data[d1]['NSIDE'],data[d2]['NSIDE']))
sys.exit()
if data[d1]['HLPXSCHM']!=data[d2]['HLPXSCHM']:
print('ERROR: HLPXSCHM are different: {} != {}'.format(data[d1]['HLPXSCHM'],data[d2]['HLPXSCHM']))
sys.exit()
w = sp.logical_not( sp.in1d(data[d1]['HEALPID'],data[d2]['HEALPID']) )
if w.sum()!=0:
print('WARNING: HEALPID are different by {} for {}:{} and {}:{}'.format(w.sum(),d1,data[d1]['HEALPID'].size,d2,data[d2]['HEALPID'].size))
new_healpix = data[d1]['HEALPID'][w]
nb_new_healpix = new_healpix.size
nb_bins = data[d2]['WE'].shape[1]
data[d2]['HEALPID'] = sp.append(data[d2]['HEALPID'],new_healpix)
data[d2]['WE'] = sp.append(data[d2]['WE'],sp.zeros((nb_new_healpix,nb_bins)),axis=0)
### Sort the data by the healpix values
for d1 in list(lst_file.keys()):
sort = sp.array(data[d1]['HEALPID']).argsort()
data[d1]['WE'] = data[d1]['WE'][sort]
data[d1]['HEALPID'] = data[d1]['HEALPID'][sort]
if corr=='AUTO':
data = data.fillna(data.median(axis=0), axis=0)
data_describe = data.describe(include=[object])
for c in categorical_columns:
data[c] = data[c].fillna(data_describe[c]['top'])
'''
data_describe = data.describe(include=[object])
binary_columns = [c for c in categorical_columns if data_describe[c]['unique'] == 2]
nonbinary_columns = [c for c in categorical_columns if data_describe[c]['unique'] > 2]
print('Binary columns:', binary_columns, 'Nonbinary columns: ', nonbinary_columns)
for c in binary_columns:
top = data_describe[c]['top']
top_items = data[c] == top
data.loc[top_items, c] = 0
data.loc[np.logical_not(top_items), c] = 1
'''
Заменим признак A4 тремя признаками: A4_u, A4_y, A4_l.
Если признак A4 принимает значение u, то признак A4_u равен 1, A4_y равен 0, A4_l равен 0.
Если признак A4 принимает значение y, то признак A4_y равен 0, A4_y равен 1, A4_l равен 0.
Если признак A4 принимает значение l, то признак A4_l равен 0, A4_y равен 0, A4_l равен 1.
'''
data_nonbinary = pd.get_dummies(data[nonbinary_columns])
#scaler = StandardScaler()
#data_numerical = data[numerical_columns]
#scaler.fit_transform(data_numerical)
data_numerical = data[numerical_columns]
def sub_map(Data, Maps, correlate=False, pols=(), make_plots=False,
interpolation='nearest') :
"""Subtracts a Map out of Data."""
# Import locally since many machines don't have matplotlib.
if make_plots :
import matplotlib.pyplot as plt
# Convert pols to an interable.
if pols is None :
pols = range(Data.dims[1])
elif not hasattr(pols, '__iter__') :
pols = (pols, )
elif len(pols) == 0 :
pols = range(Data.dims[1])
# If solving for gains, need a place to store them.
if correlate :
out_gains = sp.empty((len(pols),) + Data.dims[2:4])
for pol_ind in pols :
# Check if there one map was passed or multiple.
if isinstance(Maps, list) or isinstance(Maps, tuple) :
if len(Maps) != len(pols) :
raise ValueError("Must provide one map, or one map per "
"polarization.")
Map = Maps[pol_ind]
else :
Map = Maps
if not Map.axes == ('freq', 'ra', 'dec') :
raise ValueError("Expected map axes to be ('freq', 'ra', 'dec').")
Data.calc_pointing()
Data.calc_freq()
# Map Parameters.
centre = (Map.info['freq_centre'], Map.info['ra_centre'],
Map.info['dec_centre'])
shape = Map.shape
spacing = (Map.info['freq_delta'], Map.info['ra_delta'],
Map.info['dec_delta'])
# Nearest code is depricated. We could just use the general code.
if interpolation == 'nearest' :
# These indices are the length of the time axis. Integer indicies.
ra_ind = map.tools.calc_inds(Data.ra, centre[1], shape[1],
spacing[1])
dec_ind = map.tools.calc_inds(Data.dec, centre[2], shape[2],
spacing[2])
# Exclude indices that are off map or out of band. Boolian indices.
on_map_inds = sp.logical_and(
sp.logical_and(ra_ind>=0, ra_ind<shape[1]),
sp.logical_and(dec_ind>=0, dec_ind<shape[2]))
# Make an array of map data the size of the time stream data.
submap = Map[:, ra_ind[on_map_inds], dec_ind[on_map_inds]]
else :
map_ra = Map.get_axis('ra')
map_dec = Map.get_axis('dec')
on_map_inds = sp.logical_and(
sp.logical_and(Data.ra > min(map_ra), Data.ra < max(map_ra)),
sp.logical_and(Data.dec > min(map_dec), Data.dec<max(map_dec)))
submap = sp.empty((Map.shape[0], sp.sum(on_map_inds)), dtype=float)
jj = 0
for ii in range(len(on_map_inds)) :
if on_map_inds[ii] :
submap[:, jj] = Map.slice_interpolate([1, 2],
[Data.ra[ii], Data.dec[ii]], kind=interpolation)
jj += 1
# Length of the data frequency axis.
freq_ind = map.tools.calc_inds(Data.freq, centre[0], shape[0],
spacing[0])
in_band_inds = sp.logical_and(freq_ind >= 0, freq_ind < shape[0])
submap = submap[freq_ind[in_band_inds], ...]
# Broadcast to the same shape and combine.
covered_inds = sp.logical_and(on_map_inds[:, sp.newaxis],
in_band_inds[sp.newaxis, :])
# submap is the size of the data that is on the map. Expand to full
# size of data.
subdata = sp.zeros(sp.shape(covered_inds))
subdata[covered_inds] = sp.rollaxis(submap, 1, 0).flatten()
subdata[sp.logical_not(covered_inds)] = 0.0
# Now start using the actual data. Loop over cal and pol indicies.
for cal_ind in range(Data.dims[2]) :
data = Data.data[:,pol_ind, cal_ind, :]
data[sp.logical_not(covered_inds)] = ma.masked
# Find the common good indicies.
un_mask = sp.logical_not(data.mask)
# Find the number of good indicies at each frequency.
counts = sp.sum(un_mask, 0)
counts[counts == 0] = -1
# Subtract out the mean from the map.
tmp_subdata = (subdata - sp.sum(un_mask*subdata, 0)/counts)
# Correlate to solve for an unknown gain.
if correlate :
tmp_data = data.filled(0.0)
tmp_data = (tmp_data - sp.sum(un_mask*data, 0)
/ counts)
gain = (sp.sum(un_mask*tmp_subdata*tmp_data, 0) /
sp.sum(un_mask*tmp_subdata*tmp_subdata, 0))
gain[counts == -1] = 0.0
out_gains[pol_ind,cal_ind,:] = gain
else :
gain = 1.0
# Now do the subtraction and mask the off map data. We use the
# mean subtracted map, to preserve data mean.
if make_plots :
plt.figure()
plt.plot(ma.mean((gain*tmp_subdata), -1), '.b')
plt.plot(ma.mean((tmp_subdata), -1), '.r')
plt.plot(ma.mean((data - ma.mean(data, 0)), -1), '.g')
#plt.plot(ma.mean((data), -1), '.g')
#plt.plot((gain*tmp_subdata)[:, 45], '.b')
#plt.plot((data - ma.mean(data, 0))[:, 45], '.g')
data[...] -= gain*tmp_subdata
if correlate :
return out_gains
def multiply_by_cal(Data, CalData) :
"""Function scales data by the noise cal temperature.
"""
# For now we just assume that the cal and polarizations are arranged in a
# certain way and then check to make sure we are right.
calibrate_to_I = False
if tuple(Data.field['CRVAL4']) == (-5, -7, -8, -6) :
xx_ind = 0
yy_ind = 3
xy_inds = [1,2]
elif tuple(Data.field['CRVAL4']) == (1, 2, 3, 4) :
# This is a hack. Completly temporairy.
calibrate_to_I = True
else :
raise ce.DataError('Polarization types not as expected in data.')
cal_xx_ind = 0
cal_yy_ind = 1
if (CalData.field['CRVAL4'][cal_xx_ind] != -5 or
CalData.field['CRVAL4'][cal_yy_ind] != -6) :
raise ce.DataError('Polarization types not as expected in cal.')
# Cal should only have 1 time, 1 cal state and 2 polarizations.
if CalData.dims[:3] != (1,2,1) :
raise ce.DataError('Cal temperature data has wrong dimensions.')
# Cal state should be special state 'R'.
if CalData.field['CAL'][0] != 'R' :
raise ce.DataError("Cal state in cal temperture data should be "
"'R'.")
# Bring the Cal data to the same frequencies as the other data.
Data.calc_freq()
CalData.calc_freq()
if sp.allclose(Data.freq, CalData.freq) :
cdata = CalData.data
elif abs(Data.field['CDELT1']) <= abs(CalData.field['CDELT1']) :
calfunc = interpolate.interp1d(CalData.freq, CalData.data,
fill_value=sp.nan, bounds_error=False)
cdata = ma.array(calfunc(Data.freq))
cdata[sp.logical_not(sp.isfinite(cdata))] = ma.masked
else :
nf = len(Data.freq)
width = abs(Data.field['CDELT1'])
cdata = ma.empty((1,2,1,nf))
for find in range(nf) :
f = Data.freq[find]
inds, = sp.where(sp.logical_and(CalData.freq >= f - width/2.0,
CalData.freq < f + width/2.0))
cdata[:,:,:,find] = ma.mean(CalData.data[:,:,:,inds], 3)
if calibrate_to_I :
Data.data *= (cdata[0,cal_xx_ind,0,:] + cdata[0,cal_yy_ind,0,:])/2.0
else :
# Loop over times and cal and scale each polarization appropriately.
for tind in range(Data.dims[0]) :
for cind in range(Data.dims[2]) :
Data.data[tind,xx_ind,cind,:] *= cdata[0,cal_xx_ind,0,:]
Data.data[tind,yy_ind,cind,:] *= cdata[0,cal_yy_ind,0,:]
Data.data[tind,xy_inds,cind,:] *= ma.sqrt(
cdata[0,cal_yy_ind,0,:] * cdata[0,cal_xx_ind,0,:])
def doTestEdtWithHalo(self, haloSz=0):
if (isinstance(haloSz, int) or ((sys.version_info.major < 3) and isinstance(haloSz, long))):
if (haloSz < 0):
haloSz = 0
haloSz = sp.array((haloSz,)*3)
subDirName = "doTestEdtWithHalo_%s" % ("x".join(map(str, haloSz)), )
outDir = self.createTmpDir(subDirName)
#outDir = subDirName
if (mpi.world != None):
if ((mpi.world.Get_rank() == 0) and (not os.path.exists(outDir))):
os.makedirs(subDirName)
mpi.world.barrier()
segDds = mango.zeros(self.imgShape, mtype="segmented", halo=haloSz)
segDds.setAllToValue(segDds.mtype.maskValue())
mango.data.fill_ellipsoid(segDds, centre=self.centre, radius=(self.radius*1.05,)*3, fill=0)
mango.data.fill_ellipsoid(segDds, centre=self.centre, radius=(self.radius,)*3, fill=1)
mango.io.writeDds(os.path.join(outDir,"segmentedSphere.nc"), segDds)
dtDds = mango.image.distance_transform_edt(segDds, 1)
mango.io.writeDds(os.path.join(outDir,"distance_mapSphereEdt.nc"), dtDds)
segDds.updateHaloRegions()
dtDds.updateHaloRegions()
segDds.setBorderToValue(segDds.mtype.maskValue())
dtDds.setBorderToValue(dtDds.mtype.maskValue())
slc = []
for d in range(len(haloSz)):
slc.append(slice(haloSz[d], segDds.asarray().shape[d]-haloSz[d]))
slc = tuple(slc)
arr = dtDds.subd_h.asarray()
sbeg = dtDds.subd_h.origin
send = sbeg + dtDds.subd_h.shape
coords = np.ogrid[sbeg[0]:send[0],sbeg[1]:send[1],sbeg[2]:send[2]]
distDds = mango.copy(dtDds)
distArr = distDds.subd_h.asarray()
distArr[...] = \
sp.where(
sp.logical_and(dtDds.subd_h.asarray() != dtDds.mtype.maskValue(), dtDds.subd_h.asarray() >= 0),
self.radius
-
sp.sqrt(
((coords[0]-self.centre[0])**2)
+
((coords[1]-self.centre[1])**2)
+
((coords[2]-self.centre[2])**2)
),
dtDds.subd_h.asarray()
)
mango.io.writeDds(os.path.join(outDir,"distance_mapSphereDistRef.nc"), distDds)
rootLogger.info("max diff = %s" % (np.max(sp.absolute(distArr - dtDds.subd_h.asarray())),))
self.assertTrue(
sp.all(
sp.absolute(distArr - dtDds.subd_h.asarray())
<=
1.0
)
)
self.assertTrue(
sp.all(
sp.logical_not(sp.logical_xor(
segDds.asarray() == 1,
dtDds.asarray() >= 0
))
)
)
self.assertTrue(
sp.all(
sp.logical_not(sp.logical_xor(
segDds.asarray() == segDds.mtype.maskValue(),
dtDds.asarray() == dtDds.mtype.maskValue()
))
)
)
self.assertTrue(
sp.all(
sp.logical_not(sp.logical_xor(
segDds.asarray() == 0,
dtDds.asarray() == -1
))
)
)
self.assertTrue(sp.all(segDds.halo == dtDds.halo))
self.assertTrue(sp.all(segDds.shape == dtDds.shape))
self.assertTrue(sp.all(segDds.origin == dtDds.origin), "%s != %s" % (segDds.origin, dtDds.origin))
self.assertTrue(sp.all(segDds.mpi.shape == dtDds.mpi.shape))
def invertRSTO(RSTO,Iono,alpha_list=1e-2,invtype='tik',rbounds=[100,200],Nlin=0):
""" This will run the inversion program given an ionocontainer, an alpha and """
nlout,ntout,nl=Iono.Param_List.shape
if Nlin !=0:
nl=Nlin
nlin=len(RSTO.Cart_Coords_In)
time_out=RSTO.Time_Out
time_in=RSTO.Time_In
overlaps = RSTO.overlaps
xin,yin,zin=RSTO.Cart_Coords_In.transpose()
z_u=sp.unique(zin)
rplane=sp.sqrt(xin**2+yin**2)*sp.sign(xin)
r_u=sp.unique(rplane)
n_z=z_u.size
n_r=r_u.size
dims= [n_r,n_z]
rin,azin,elin=RSTO.Sphere_Coords_In.transpose()
anglist=RSTO.simparams['angles']
ang_vec=sp.array([[i[0],i[1]] for i in anglist])
# trim out cruft
zmin,zmax=[150,500]
rpmin,rpmax=rbounds#[-50,100]#[100,200]
altlog= sp.logical_and(zin>zmin,zin<zmax)
rplog=sp.logical_and(rplane>rpmin,rplane<rpmax)
allrng= RSTO.simparams['Rangegatesfinal']
dR=allrng[1]-allrng[0]
nldir=sp.ceil(int(nl)/2.)
posang_log1= sp.logical_and(ang_vec[:,0]<=180.,ang_vec[:,0]>=0)
negang_log1 = sp.logical_or(ang_vec[:,0]>180.,ang_vec[:,0]<0)
azin_pos = sp.logical_and(azin<=180.,azin>=0)
azin_neg = sp.logical_or(azin>180.,azin<0)
minangpos=0
minangneg=0
if sp.any(posang_log1):
minangpos=ang_vec[posang_log1,1].min()
if sp.any(negang_log1):
minangneg=ang_vec[negang_log1,1].min()
rngbounds=[allrng[0]-nldir*dR,allrng[-1]+nldir*dR]
rng_log=sp.logical_and(rin>rngbounds[0],rin<rngbounds[1])
elbounds_pos=sp.logical_and(azin_pos,elin>minangpos)
elbounds_neg=sp.logical_and(azin_neg,elin>minangneg)
elbounds=sp.logical_or(elbounds_pos,elbounds_neg)
keeplog=sp.logical_and(sp.logical_and(rng_log,elbounds),sp.logical_and(altlog,rplog))
keeplist=sp.where(keeplog)[0]
nlin_red=len(keeplist)
# set up derivative matrix
dx,dy=diffmat(dims)
dx_red=dx[keeplist][:,keeplist]
dy_red=dy[keeplist][:,keeplist]
# need the sparse vstack to make srue things stay sparse
D=sp.sparse.vstack((dx_red,dy_red))
# New parameter matrix
new_params=sp.zeros((nlin,len(time_out),nl),dtype=Iono.Param_List.dtype)
if isinstance(alpha_list,numbers.Number):
alpha_list=[alpha_list]*nl
ave_datadif=sp.zeros((len(time_out),nl))
ave_data_const = sp.zeros_like(ave_datadif)
q=1e10
for itimen, itime in enumerate(time_out):
print('Making Outtime {0:d} of {1:d}'.format(itimen+1,len(time_out)))
#allovers=overlaps[itimen]
#curintimes=[i[0] for i in allovers]
#for it_in_n,it in enumerate(curintimes):
#print('\t Making Intime {0:d} of {1:d}'.format(it_in_n+1,len(curintimes)))
#A=RSTO.RSTMat[itimen*nlout:(itimen+1)*nlout,it*nlin:(it+1)*nlin]
A=RSTO.RSTMat[itimen*nlout:(itimen+1)*nlout,itimen*nlin:(itimen+1)*nlin]
Acvx=cvx.Constant(A[:,keeplist])
for ip in range(nl):
alpha=alpha_list[ip]*2
print('\t\t Making Lag {0:d} of {1:d}'.format(ip+1,nl))
datain=Iono.Param_List[:,itimen,ip]
xr=cvx.Variable(nlin_red)
xi=cvx.Variable(nlin_red)
if invtype.lower()=='tik':
constr=alpha*cvx.norm(xr,2)
consti=alpha*cvx.norm(xi,2)
elif invtype.lower()=='tikd':
constr=alpha*cvx.norm(D*xr,2)
consti=alpha*cvx.norm(D*xi,2)
elif invtype.lower()=='tv':
constr=alpha*cvx.norm(D*xr,1)
consti=alpha*cvx.norm(D*xi,1)
br=datain.real/q
bi=datain.imag/q
if ip==0:
objective=cvx.Minimize(cvx.norm(Acvx*xr-br,2)+constr)
constraints= [xr>=0]
prob=cvx.Problem(objective)
result=prob.solve(verbose=True,solver=cvx.SCS,use_indirect=True,max_iters=4000)
# new_params[keeplog,it,ip]=xr.value.flatten()
xcomp=sp.array(xr.value).flatten()*q
else:
objective=cvx.Minimize(cvx.norm(Acvx*xr-br,2)+constr)
prob=cvx.Problem(objective)
result=prob.solve(verbose=True,solver=cvx.SCS,use_indirect=True,max_iters=4000)
objective=cvx.Minimize(cvx.norm(Acvx*xi-bi,2)+consti)
prob=cvx.Problem(objective)
result=prob.solve(verbose=True,solver=cvx.SCS,use_indirect=True,max_iters=4000)
xcomp=sp.array(xr.value + 1j*xi.value).flatten()*q
# new_params[keeplog,it,ip]=xcomp
new_params[keeplog,itimen,ip]=xcomp
ave_datadif[itimen,ip]=sp.sqrt(sp.nansum(sp.absolute(A[:,keeplist].dot(xcomp)-datain)**2))
if invtype.lower()=='tik':
sumconst=sp.sqrt(sp.nansum(sp.power(sp.absolute(xcomp),2)))
elif invtype.lower()=='tikd':
dx=D.dot(xcomp)
sumconst=sp.sqrt(sp.nansum(sp.power(sp.absolute(dx),2)))
elif invtype.lower()=='tv':
dx=D.dot(xcomp)
sumconst=sp.nansum(sp.absolute(dx))
ave_data_const[itimen,ip]=sumconst
# set up nans
new_params[sp.logical_not(keeplog),itimen]=sp.nan
datadif=sp.nanmean(ave_datadif,axis=0)
constval=sp.nanmean(ave_data_const,axis=0)
ionoout=IonoContainer(coordlist=RSTO.Cart_Coords_In,paramlist=new_params,times = time_out,sensor_loc = sp.zeros(3),ver =0,coordvecs =
['x','y','z'],paramnames=Iono.Param_Names[:Nlin])
return (ionoout,datadif,constval)
def read(self, scans=None, IFs=None, force_tuple=False) :
"""Read in data from the fits file.
This method reads data from the fits file including the files history
and basically every peice of data that could be needed. It is done,
one scan and one IF at a time. Each scan and IF is returned in an
instance of the DataBlock class (defined in another module of this
package).
Arguments:
scans: Which scans in the file to be processed. A list of
integers, with 0 corresponding to the lowest numbered scan.
Default is all of them.
IFs: Which intermediate frequencies (also called frequency windows)
to process. A list of integers with 0 coorsponding to the
lowest frequency present. Default is all of them.
TODO : Overlapping frequency windows stiched together somehow.
force_tuple: By default, if there is only a single output Data
Block, it is returned not wraped in a tuple, but if we want to
loop over the output we can force the output to be a tuple,
even if it only has one element.
Returns: Instance of the DataBlock class, or a tuple of these instances
(if asked for multiple scans and IFs).
"""
# We want scans and IFs to be a sequence of indicies.
if scans is None :
scans = range(len(self.scan_set))
elif not hasattr(scans, '__iter__') :
scans = (scans, )
elif len(scans) == 0 :
scans = range(len(self.scan_set))
if IFs is None :
IFs = range(len(self.IF_set))
elif not hasattr(IFs, '__iter__') :
IFs = (IFs, )
elif len(IFs) == 0 :
IFs = range(len(self.IF_set))
# Sequence of output DataBlock objects.
output = ()
for scan_ind in scans :
for IF_ind in IFs :
# Choose the appropriate records from the file, get that data.
inds_sif = self.get_scan_IF_inds(scan_ind, IF_ind)
Data_sif = db.DataBlock(self.fitsdata.field('DATA')[inds_sif])
# Masked data is stored in FITS files as float('nan')
Data_sif.data[sp.logical_not(sp.isfinite(
Data_sif.data))] = ma.masked
# Now iterate over the fields and add them
# to the data block.
for field, axis in fields_and_axes.iteritems() :
# See if this fits file has the key we are looking for.
if not field in self.fitsdata._names :
continue
# First get the 'FITS' format string.
field_format = self.hdulist[1].columns.formats[
self.hdulist[1].columns.names.index(field)]
if axis :
# From the indices in inds_sif, we only need a
# subset: which_data will subscript inds_sif.
temp_data = self.fitsdata.field(field)[inds_sif]
# For reshaping at the end.
field_shape = []
for ii, single_axis in enumerate(Data_sif.axes[0:-1]) :
# For each axis, slice out all the data except the
# stuff we need.
which_data = [slice(None)] * 3
if single_axis in axis :
field_shape.append(Data_sif.dims[ii])
else :
which_data[ii] = [0]
temp_data = temp_data[tuple(which_data)]
temp_data.shape = tuple(field_shape)
Data_sif.set_field(field, temp_data, axis, field_format)
else :
Data_sif.set_field(field, self.fitsdata.field(field)
[inds_sif[0,0,0]], axis, field_format)
if hasattr(self, 'history') :
Data_sif.history = db.History(self.history)
else :
self.history =bf.get_history_header(self.hdulist[0].header)
#self.set_history(Data_sif)
fname_abbr = ku.abbreviate_file_path(self.fname)
self.history.add('Read from file.', ('File name: ' +
fname_abbr, ))
Data_sif.history = db.History(self.history)
Data_sif.verify()
output = output + (Data_sif, )
if self.feedback > 2 :
print 'Read finished.'
if len(output) == 1 and not force_tuple :
return output[0]
else :
return output
def removeQuadOff(input_unw_path, width, length):
ramp_removed_unw_path = "ramp_removed_" + input_unw_path[input_unw_path.rfind("/") + 1 : input_unw_path.rfind(".")] + ".unw";
assert not os.path.exists(ramp_removed_unw_path), "\n***** ERROR: " + ramp_removed_unw_path + " already exists, exiting...\n";
import subprocess;
mag = input_unw_path[input_unw_path.rfind("/") + 1 : input_unw_path.rfind(".")] + ".mag";
phs = input_unw_path[input_unw_path.rfind("/") + 1 : input_unw_path.rfind(".")] + ".phs";
cmd = "\nrmg2mag_phs " + input_unw_path + " " + mag + " " + phs + " " + width + "\n";
cmd += "\nrm " + mag + "\n";
subprocess.call(cmd,shell=True);
infile = open(phs, "rb");
import pylab;
indat = pylab.fromfile(infile,pylab.float32,-1).reshape(int(length), int(width));
#indat = pylab.fromfile(infile,pylab.float32,-1).reshape(int(width) * int(length), -1);
infile.close();
import scipy;
x = scipy.arange(0, int(length));
y = scipy.arange(0, int(width));
import numpy;
x_grid, y_grid = numpy.meshgrid(x, y);
indices = numpy.arange(0, int(width) * int(length));
mx = scipy.asarray(x_grid).reshape(-1);
my = scipy.asarray(y_grid).reshape(-1);
d = scipy.asarray(indat).reshape(-1);
nonan_ids = indices[scipy.logical_not(numpy.isnan(d))];
mx = mx[nonan_ids];
my = my[nonan_ids];
d = d[nonan_ids];
init_mx = scipy.asarray(x_grid).reshape(-1)[nonan_ids];
init_my = scipy.asarray(y_grid).reshape(-1)[nonan_ids];
#ramp_removed = scipy.asarray(indat).reshape(-1)[nonan_ids];
ramp_removed = scipy.zeros(int(length) * int(width))[nonan_ids];
init_m_ones = scipy.ones(int(length) * int(width))[nonan_ids];
# init_xs = [init_m_ones, init_mx, init_my, scipy.multiply(init_mx,init_my), scipy.power(init_mx,2), scipy.power(init_my,2)];
init_xs = [init_mx];
print(len(init_xs));
p0 = scipy.zeros(len(init_xs));
p = scipy.zeros(len(init_xs));
import scipy.optimize;
for i in scipy.arange(0,10):
m_ones = scipy.ones(scipy.size(mx));
# xs = [m_ones, mx, my, scipy.multiply(mx,my), scipy.power(mx,2), scipy.power(my,2)];
xs = [mx];
G = scipy.vstack(xs).T;
print(mx);
plsq = scipy.optimize.leastsq(residuals, p0, args = (d, xs));
res = d - peval(xs, plsq[0]);
mod = plsq[0];
p = p + mod;
print(plsq[0]);
# synth = G * scipy.matrix(mod).T;
# cutoff = res.std(axis=0,ddof=1);
#print(cutoff);
# indices = numpy.arange(0, numpy.size(mx));
# good_ids = indices[abs(res) <= cutoff];
# plt.figure(i + 2);
# plt.plot(mx,d,'b.',label='alloff');
# plt.plot(mx[good_ids],synth[good_ids],'.',label='fit',color='lightgreen');
# plt.plot(mx[bad_ids],d[bad_ids],'r.',label='cull #' + str(i + 1));
# plt.legend();
# mx = mx[good_ids];
# my = my[good_ids];
# d = res[good_ids];
d = res;
print(sum(res));
# ramp_removed = scipy.asarray(ramp_removed - peval(init_xs, plsq[0]));
ramp_removed = scipy.asarray(ramp_removed + peval(init_xs, plsq[0]));
d = scipy.asarray(indat).reshape(-1);
for i in range(0, scipy.size(nonan_ids)):
d[nonan_ids[i]] = ramp_removed[i];
ramp_removed = d.reshape(int(length), int(width));
# import matplotlib;
# matplotlib.pyplot.imshow(scipy.array(indat),interpolation='nearest',origin='lower');
# matplotlib.pyplot.show();
outfile = open(ramp_removed_unw_path, "wb");
outoff = scipy.matrix(scipy.hstack((ramp_removed, ramp_removed)), scipy.float32);
outoff.tofile(outfile);
outfile.close();
def make_masked_time_stream(Blocks, ntime=None) :
"""Converts Data Blocks into a single uniformly sampled time stream.
Also produces the mask giving whether elements are valid entries or came
from a zero pad. This produes the required inputs for calculating a
windowed power spectrum.
Parameters
----------
Blocks : tuple of DataBlock objects.
ntime : int
Total number of time bins in output arrays. If shorter than required
extra data is truncated. If longer, extra data is masked. Default is
to use exactly the number that fits all the data.
Returns
-------
time_stream : array
All the data in `Blocks` but concatenated along the time axis and
padded with zeros such that the time axis is uniformly sampled and
uninterupted.
mask : array same shape as `time_stream`
1.0 if data in the correspoonding `time_stream` element is filled
and 0 if the data was missing. This is like a window where
time_stream = mask*real_data.
dt : float
The time step of the returned time stream.
"""
# Shape of all axes except the time axis.
back_shape = Blocks[0].dims[1:]
# Get the time sample spacing.
Blocks[0].calc_time()
dt = abs(sp.mean(sp.diff(Blocks[0].time)))
# Find the beginning and the end of the time axis by looping through
# blocks.
min_time = float('inf')
max_time = 0.0
for Data in Blocks :
Data.calc_time()
min_time = min(min_time, min(Data.time))
max_time = max(min_time, max(Data.time))
# Ensure that the time sampling is uniform.
if not (sp.allclose(abs(sp.diff(Data.time)), dt, rtol=0.1)
and sp.allclose(abs(sp.mean(sp.diff(Data.time))), dt,
rtol=0.001)) :
msg = ("Time sampling not uniformly spaced or Data Blocks don't "
"agree on sampling.")
raise ce.DataError(msg)
# Ensure the shapes are right.
if Data.dims[1:] != back_shape :
msg = ("All data blocks must have the same shape except the time "
"axis.")
raise ce.DataError(msg)
# Calculate the time axis.
if not ntime :
time = sp.arange(min_time, max_time + dt, dt)
ntime = len(time)
else :
time = sp.arange(ntime)*dt + min_time
# Allowcate memory for the outputs.
time_stream = sp.zeros((ntime,) + back_shape, dtype=float)
mask = sp.zeros((ntime,) + back_shape, dtype=sp.float32)
# Loop over all times and fill in the arrays.
for Data in Blocks :
# Apply an offset to the time in case the start of the Data Block
# doesn't line up with the time array perfectly.
offset = time[sp.argmin(abs(time - Data.time[0]))] - Data.time[0]
for ii in range(Data.dims[0]) :
ind = sp.argmin(abs(time - (Data.time[ii] + offset)))
if abs(time[ind] - (Data.time[ii])) < 0.5*dt :
if sp.any(mask[ind, ...]) :
msg = "Overlapping times in Data Blocks."
raise ce.DataError(msg)
time_stream[ind, ...] = Data.data[ii, ...].filled(0.0)
mask[ind, ...] = sp.logical_not(Data.data.mask[ii, ...])
return time_stream, mask, dt
def make_masked_time_stream(Blocks, ntime=None, window=None,
return_means=False, subtract_slope=False) :
"""Converts Data Blocks into a single uniformly sampled time stream.
Also produces the mask giving whether elements are valid entries or came
from a zero pad. This produes the required inputs for calculating a
windowed power spectrum.
Parameters
----------
Blocks : tuple of DataBlock objects.
ntime : int
Total number of time bins in output arrays. If shorter than required
extra data is truncated. If longer, extra data is masked. Default is
to use exactly the number that fits all the data. Set to a negitive
factor to zero pad to a power of 2 and by at least at least the factor.
window : string or tuple
Type of window to apply to each DataBlock. Valid options are the valid
arguments to scipy.signal.get_window(). By default, don't window.
return_means : bool
Whether to return an array of the channed means.
subtract_slope : bool
Whether to subtract a linear function of time from each channel.
Returns
-------
time_stream : array
All the data in `Blocks` but concatenated along the time axis and
padded with zeros such that the time axis is uniformly sampled and
uninterupted.
mask : array same shape as `time_stream`
1.0 if data in the correspoonding `time_stream` element is filled
and 0 if the data was missing. This is like a window where
time_stream = mask*real_data.
dt : float
The time step of the returned time stream.
means : array (optional)
The mean from each channel.
"""
# Shape of all axes except the time axis.
back_shape = Blocks[0].dims[1:]
# Get the time sample spacing.
Blocks[0].calc_time()
dt = abs(sp.mean(sp.diff(Blocks[0].time)))
# Find the beginning and the end of the time axis by looping through
# blocks.
# Also get the time axis and the mask
# for calculating basis polynomials.
unmask = sp.zeros((0,) + back_shape, dtype=bool)
time = sp.zeros((0,), dtype=float)
start_ind = []
min_time = float('inf')
max_time = 0.0
#mean_time = 0.0
#n_data_times = 0
for Data in Blocks :
Data.calc_time()
start_ind.append(len(time))
time = sp.concatenate((time, Data.time))
this_unmask = sp.logical_not(ma.getmaskarray(Data.data))
unmask = sp.concatenate((unmask, this_unmask), 0)
# Often the start or the end of a scan is completly masked. Make sure
# we don't start till the first unmasked time and end at the last
# unmasked time.
time_unmask = sp.alltrue(ma.getmaskarray(Data.data), -1)
time_unmask = sp.alltrue(time_unmask, -1)
time_unmask = sp.alltrue(time_unmask, -1)
if sp.alltrue(time_unmask):
continue
time_unmask = sp.logical_not(time_unmask)
min_time = min(min_time, min(Data.time[time_unmask]))
max_time = max(min_time, max(Data.time[time_unmask]))
#mean_time += sp.sum(Data.time[time_unmask])
#n_data_times += len(Data.time[time_unmask])
# Ensure that the time sampling is uniform.
if not (sp.allclose(abs(sp.diff(Data.time)), dt, rtol=0.1)
and sp.allclose(abs(sp.mean(sp.diff(Data.time))), dt,
rtol=0.001)) :
msg = ("Time sampling not uniformly spaced or Data Blocks don't "
"agree on sampling.")
raise ce.DataError(msg)
# Ensure the shapes are right.
if Data.dims[1:] != back_shape :
msg = ("All data blocks must have the same shape except the time "
"axis.")
raise ce.DataError(msg)
# Now calculate basis polynomials for the mean mode and the slope mode.
polys = misc.ortho_poly(time[:,None,None,None], 2, unmask, 0)
#mean_time /= n_data_times
#if n_data_times == 0:
# n_data_times = 1
# Very important to subtract the mean out of the signal, otherwise the
# window coupling to the mean (0) mode will dominate everything. Can also
# optionally take out a slope.
# Old algorithm.
#total_sum = 0.0
#total_counts = 0
#total_slope = 0.0
#time_norm = 0.0
#for Data in Blocks:
# total_sum += sp.sum(Data.data.filled(0), 0)
# total_counts += ma.count(Data.data, 0)
# total_slope += sp.sum(Data.data.filled(0)
# * (Data.time[:,None,None,None] - mean_time), 0)
# time_norm += sp.sum(sp.logical_not(ma.getmaskarray(Data.data))
# * (Data.time[:,None,None,None] - mean_time)**2, 0)
#total_counts[total_counts == 0] = 1
#time_norm[time_norm == 0.0] = 1
#total_mean = total_sum / total_counts
#total_slope /= time_norm
# New algorithm.
mean_amp = 0
slope_amp = 0
for ii, Data in enumerate(Blocks):
si = start_ind[ii]
this_nt = Data.dims[0]
data = Data.data.filled(0)
mean_amp += sp.sum(data * unmask[si:si + this_nt,...]
* polys[0,si:si + this_nt,...], 0)
slope_amp += sp.sum(data * unmask[si:si + this_nt,...]
* polys[1,si:si + this_nt,...], 0)
polys[0,...] *= mean_amp
polys[1,...] *= slope_amp
# Calculate the time axis.
if min_time > max_time:
min_time = 0
max_time = 6 * dt
if not ntime :
ntime = (max_time - min_time) // dt + 1
elif ntime < 0:
# 0 pad by a factor of at least -ntime, but at most 10% more than this.
time_min = -ntime * (max_time - min_time) / dt
n_block = 1
while n_block < time_min/20.0:
n_block *= 2
ntime = (time_min//n_block + 1) * n_block
time = sp.arange(ntime)*dt + min_time
# Allowcate memory for the outputs.
time_stream = sp.zeros((ntime,) + back_shape, dtype=float)
mask = sp.zeros((ntime,) + back_shape, dtype=sp.float32)
# Loop over all times and fill in the arrays.
for ii, Data in enumerate(Blocks):
this_nt = Data.dims[0]
si = start_ind[ii]
# Subtract the mean calculated above.
this_data = Data.data.copy()
this_data -= polys[0,si:si + this_nt,...]
# If desired, subtract of the linear function of time.
if subtract_slope:
#this_data -= (total_slope
# * (Data.time[:,None,None,None] - mean_time))
this_data -= polys[1,si:si + this_nt,...]
# Find the first and last unmasked times.
time_unmask = sp.alltrue(ma.getmaskarray(this_data), -1)
time_unmask = sp.alltrue(time_unmask, -1)
time_unmask = sp.alltrue(time_unmask, -1)
if sp.alltrue(time_unmask):
continue
time_unmask = sp.logical_not(time_unmask)
unmasked_ind, = sp.where(time_unmask)
first_ind = min(unmasked_ind)
last_ind = max(unmasked_ind)
# Ensure that the time sampling is uniform.
if not (sp.allclose(abs(sp.diff(Data.time)), dt, rtol=0.1)
and sp.allclose(abs(sp.mean(sp.diff(Data.time))), dt,
rtol=0.001)) :
msg = ("Time sampling not uniformly spaced or Data Blocks don't "
"agree on sampling.")
raise ce.DataError(msg)
# Ensure the shapes are right.
if Data.dims[1:] != back_shape :
msg = ("All data blocks must have the same shape except the time "
"axis.")
# Apply an offset to the time in case the start of the Data Block
# doesn't line up with the time array perfectly.
offset = (time[sp.argmin(abs(time - Data.time[first_ind]))]
- Data.time[first_ind])
# Generate window function.
if window:
window_function = sig.get_window(window, last_ind - first_ind + 1)
for ii in range(first_ind, last_ind + 1) :
ind = sp.argmin(abs(time - (Data.time[ii] + offset)))
if abs(time[ind] - (Data.time[ii])) < 0.5*dt :
if sp.any(mask[ind, ...]) :
msg = "Overlapping times in Data Blocks."
raise ce.DataError(msg)
if window:
window_value = window_function[ii - first_ind]
else :
window_value = 1.0
time_stream[ind, ...] = (window_value
* this_data[ii, ...].filled(0.0))
mask[ind, ...] = window_value * sp.logical_not(ma.getmaskarray(
this_data)[ii, ...])
if return_means:
return time_stream, mask, dt, polys[0,0,...]
else :
return time_stream, mask, dt
def flag(Data, NoiseData, thres=3.0, max_noise_factor=-1, modes_subtract=1,
filter_type='edge'):
"""Flags data for outliers using a signal subtracted data set.
Flags outliers of in a time stream data by looking at a version of the data
that has had the signal subtracted out of it. Each frequency channel,
polarization and cal state are treated separately.
Parameters
----------
Data : DataBlock Object
Data to be flaged. Upon exit, this object will have new flags.
NoiseData : DataBlock Object
Version of `Data` with the signal subtracted.
thres : float
Threshold for flagging in units of sigma (default is 3.0).
modes_subtract : int
How many modes to remove for high pass filtering.
filter_type : {'edge', 'gaussian', 'gaussian/edge'}
Type of high pass filtering to use.
"""
# Get the mask and the data as normal arrays.
# Copy seems to be nessisary if the mask is None.
data = NoiseData.data.filled(0).copy()
mask = ma.getmaskarray(NoiseData.data)
## High pass filter the data to make outliers stand out.
un_mask = sp.logical_not(mask)
NoiseData.calc_time()
time = NoiseData.time
n_time = len(time)
# How many basis polynomials we need and with what fraction of each mode
# gets subtracted out..
if filter_type == 'edge':
n_polys = modes_subtract
subtract_weights = sp.ones(n_polys)
elif filter_type == 'gaussian' or filter_type == 'gaussian/edge':
n_polys = 4 * modes_subtract
subtract_weights = sp.exp(-(sp.arange(n_polys, dtype=float)
/ modes_subtract)**2 / 2.)
if filter_type == 'gaussian/edge':
subtract_weights[0:2] = 1.
# Test if the mask is the same for all slices. If it is, that greatly
# reduces the work as we only have to generate one set of polynomials.
all_masks_same = True
for jj in range(n_time):
if sp.all(un_mask[jj,...] == un_mask[jj,0,0,0]):
continue
else:
all_masks_same = False
break
if all_masks_same:
polys = misc.ortho_poly(time, n_polys, un_mask[:,0,0,0], 0)
polys.shape = (n_polys, len(time), 1, 1, 1)
else:
polys = misc.ortho_poly(time[:,None,None,None], n_polys, un_mask, 0)
# Subtract the slope mode (1th mode) out of the NoiseData.
amps = sp.sum(data * un_mask * polys, 1)
amps *= subtract_weights[:,None,None,None]
data -= sp.sum(amps[:,None,:,:,:] * un_mask[None,:,:,:,:] * polys, 0)
## Do the main outlier flagging.
# Iteratively flag on sliding scale to get closer and closer to desired
# threshold.
max_thres = sp.sqrt(n_time)/2.
n_iter = 3
thresholds = (max_thres ** (n_iter - 1 - sp.arange(n_iter))
* thres ** sp.arange(n_iter)) ** (1./(n_iter - 1))
for threshold in thresholds:
# Subtract the mean from every channel.
this_data = masked_subtract_mean(data, mask, 0)
# Calculate the variance.
un_mask = sp.logical_not(mask)
counts = sp.sum(un_mask, 0)
counts[counts == 0] = 1
std = sp.sqrt(sp.sum(this_data**2 * un_mask, 0) / counts)
bad_inds = abs(this_data) > threshold * std
# If any polarization or cal state is masked, they all should be.
bad_inds = sp.any(sp.any(bad_inds, 1), 1)
mask[bad_inds[:,None,None,:]] = True
## Now look for times with excusion frequency average
# (achromatic out-liers).
# Compute the frequency mean.
un_mask = sp.logical_not(mask)
counts = sp.sum(un_mask, -1)
fmean_un_mask = counts >= 1
counts[counts == 0] = 1
fmean = sp.sum(data * un_mask, -1) / counts
# Subtract the time mean.
fmean = masked_subtract_mean(fmean, sp.logical_not(fmean_un_mask), 0)
# Get the variance.
counts = sp.sum(fmean_un_mask, 0)
counts[counts == 0] = 1
fmean_std = sp.sqrt(sp.sum(fmean**2 * fmean_un_mask, 0) / counts)
# Flag any time that is an outlier (for any polarization or cal state).
bad_times = sp.any(sp.any(abs(fmean) > thres * fmean_std, 1), 1)
mask[bad_times,:,:,:] = True
## Flag for very noisy channels.
if max_noise_factor > 0:
# Do this a few times to make sure we get everything.
for ii in range(3):
this_data = masked_subtract_mean(data, mask, 0)
# Compute varience accounting for the mask.
un_mask = sp.logical_not(mask)
counts = sp.sum(un_mask, 0)
vars_un_mask = counts >= 1
counts[counts == 0] = 1
vars = sp.sum(this_data**2 * un_mask, 0) / counts
# Find the mean of the variences.
counts = sp.sum(vars_un_mask, -1)
counts[counts == 0] = 1
mean_vars = sp.sum(vars * vars_un_mask, -1) / counts
# Find channels that stand out (for any polarization or cal state).
bad_chans = sp.any(sp.any(vars > max_noise_factor *
mean_vars[:,:,None], 0), 0)
mask[:,:,:,bad_chans] = True
## Transfer the mask to the DataBlock objects.
Data.data[mask] = ma.masked
NoiseData.data[mask] = ma.masked
def flag(Data,
NoiseData,
thres=3.0,
max_noise_factor=-1,
modes_subtract=1,
filter_type='edge'):
"""Flags data for outliers using a signal subtracted data set.
Flags outliers of in a time stream data by looking at a version of the data
that has had the signal subtracted out of it. Each frequency channel,
polarization and cal state are treated separately.
Parameters
----------
Data : DataBlock Object
Data to be flaged. Upon exit, this object will have new flags.
NoiseData : DataBlock Object
Version of `Data` with the signal subtracted.
thres : float
Threshold for flagging in units of sigma (default is 3.0).
modes_subtract : int
How many modes to remove for high pass filtering.
filter_type : {'edge', 'gaussian', 'gaussian/edge'}
Type of high pass filtering to use.
"""
# Get the mask and the data as normal arrays.
# Copy seems to be nessisary if the mask is None.
data = NoiseData.data.filled(0).copy()
mask = ma.getmaskarray(NoiseData.data)
## High pass filter the data to make outliers stand out.
un_mask = sp.logical_not(mask)
NoiseData.calc_time()
time = NoiseData.time
n_time = len(time)
# How many basis polynomials we need and with what fraction of each mode
# gets subtracted out..
if filter_type == 'edge':
n_polys = modes_subtract
subtract_weights = sp.ones(n_polys)
elif filter_type == 'gaussian' or filter_type == 'gaussian/edge':
n_polys = 4 * modes_subtract
subtract_weights = sp.exp(
-(sp.arange(n_polys, dtype=float) / modes_subtract)**2 / 2.)
if filter_type == 'gaussian/edge':
subtract_weights[0:2] = 1.
# Test if the mask is the same for all slices. If it is, that greatly
# reduces the work as we only have to generate one set of polynomials.
all_masks_same = True
for jj in range(n_time):
if sp.all(un_mask[jj, ...] == un_mask[jj, 0, 0, 0]):
continue
else:
all_masks_same = False
break
if all_masks_same:
polys = misc.ortho_poly(time, n_polys, un_mask[:, 0, 0, 0], 0)
polys.shape = (n_polys, len(time), 1, 1, 1)
else:
polys = misc.ortho_poly(time[:, None, None, None], n_polys, un_mask, 0)
# Subtract the slope mode (1th mode) out of the NoiseData.
amps = sp.sum(data * un_mask * polys, 1)
amps *= subtract_weights[:, None, None, None]
data -= sp.sum(amps[:, None, :, :, :] * un_mask[None, :, :, :, :] * polys,
0)
## Do the main outlier flagging.
# Iteratively flag on sliding scale to get closer and closer to desired
# threshold.
max_thres = sp.sqrt(n_time) / 2.
n_iter = 3
thresholds = (max_thres**(n_iter - 1 - sp.arange(n_iter)) *
thres**sp.arange(n_iter))**(1. / (n_iter - 1))
for threshold in thresholds:
# Subtract the mean from every channel.
this_data = masked_subtract_mean(data, mask, 0)
# Calculate the variance.
un_mask = sp.logical_not(mask)
counts = sp.sum(un_mask, 0)
counts[counts == 0] = 1
std = sp.sqrt(sp.sum(this_data**2 * un_mask, 0) / counts)
bad_inds = abs(this_data) > threshold * std
# If any polarization or cal state is masked, they all should be.
bad_inds = sp.any(sp.any(bad_inds, 1), 1)
mask[bad_inds[:, None, None, :]] = True
## Now look for times with excusion frequency average
# (achromatic out-liers).
# Compute the frequency mean.
un_mask = sp.logical_not(mask)
counts = sp.sum(un_mask, -1)
fmean_un_mask = counts >= 1
counts[counts == 0] = 1
fmean = sp.sum(data * un_mask, -1) / counts
# Subtract the time mean.
fmean = masked_subtract_mean(fmean, sp.logical_not(fmean_un_mask), 0)
# Get the variance.
counts = sp.sum(fmean_un_mask, 0)
counts[counts == 0] = 1
fmean_std = sp.sqrt(sp.sum(fmean**2 * fmean_un_mask, 0) / counts)
# Flag any time that is an outlier (for any polarization or cal state).
bad_times = sp.any(sp.any(abs(fmean) > thres * fmean_std, 1), 1)
mask[bad_times, :, :, :] = True
## Flag for very noisy channels.
if max_noise_factor > 0:
# Do this a few times to make sure we get everything.
for ii in range(3):
this_data = masked_subtract_mean(data, mask, 0)
# Compute varience accounting for the mask.
un_mask = sp.logical_not(mask)
counts = sp.sum(un_mask, 0)
vars_un_mask = counts >= 1
counts[counts == 0] = 1
vars = sp.sum(this_data**2 * un_mask, 0) / counts
# Find the mean of the variences.
counts = sp.sum(vars_un_mask, -1)
counts[counts == 0] = 1
mean_vars = sp.sum(vars * vars_un_mask, -1) / counts
# Find channels that stand out (for any polarization or cal state).
bad_chans = sp.any(
sp.any(vars > max_noise_factor * mean_vars[:, :, None], 0), 0)
mask[:, :, :, bad_chans] = True
## Transfer the mask to the DataBlock objects.
Data.data[mask] = ma.masked
NoiseData.data[mask] = ma.masked
def flowisentropic(**flow):
"""
Evaluate the isentropic relations with any flow variable.
This function accepts a given set of specific heat ratios and
a single input of isentropic flow variables. Inputs can be a single
scalar or an array_like data structure.
Parameters
----------
gamma : array_like, optional
Specific heat ratio. Values must be greater than 1.
M : array_like
Mach number. Values must be greater than or equal to 0.
T : array_like
Temperature ratio T/T0. Values must be 0 <= T <= 1.
P : array_like
Pressure ratio P/P0. Values must be 0 <= P <= 1.
rho : array_like
Density ratio rho/rho0. Values must be 0 <= rho <= 1.
sub : array_like
Subsonic area ratio A/A*. Values must be greater than or equal
to 1.
sup : array_like
Supersonic area ratio A/A*. Values must be greater than or
equal to 1.
Returns
-------
out : (M, T, P, rho, area)
Tuple of Mach number, temperature ratio, pressure ratio, density
ratio and area ratio.
Notes
-----
This function accepts one and only one of the isentropic flow
variables. It will raise an Exception when more than one input
is given.
Examples
--------
>>> flowisentropic(M=3)
(3.0, 0.35714285714285715, 0.027223683703862824, 0.076226314370815895,
4.2345679012345689)
>>> flowisentropic(gamma=1.4, sup=1.6)
(1.9352576078182122, 0.57174077399894296, 0.14131786852470815,
0.24717122680666009, 1.6000000000000001)
>>> flowisentropic(T=sp.linspace(0, 1, 100))
(array, array, array, array, array)
"""
#parse the input
gamma, flow, mtype, itype = _flowinput(flow)
#calculate gamma-ratios for use in the equations
a = (gamma+1) / 2
b = (gamma-1) / 2
c = a / (gamma-1)
#preshape mach array
M = sp.empty(flow.shape, sp.float64)
#use the isentropic relations to solve for the mach number
if mtype in ["mach", "m"]:
if (flow < 0).any() or not sp.isreal(flow).all():
raise Exception("Mach number inputs must be real numbers" \
" greater than or equal to 0.")
M = flow
elif mtype in ["temp", "t"]:
if (flow < 0).any() or (flow > 1).any():
raise Exception("Temperature ratio inputs must be real numbers" \
" 0 <= T <= 1.")
M[flow == 0] = sp.inf
M[flow != 0] = sp.sqrt((1/b[flow != 0])*(flow[flow != 0]**(-1) - 1))
elif mtype in ["pres", "p"]:
if (flow < 0).any() or (flow > 1).any():
raise Exception("Pressure ratio inputs must be real numbers" \
" 0 <= P <= 1.")
M[flow == 0] = sp.inf
M[flow != 0] = sp.sqrt((1/b[flow != 0]) * \
(flow[flow != 0]**((gamma[flow != 0]-1)/-gamma[flow != 0]) - 1))
elif mtype in ["dens", "d", "rho"]:
if (flow < 0).any() or (flow > 1).any():
raise Exception("Density ratio inputs must be real numbers" \
" 0 <= rho <= 1.")
M[flow == 0] = sp.inf
M[flow != 0] = sp.sqrt((1/b[flow != 0]) * \
(flow[flow != 0]**((gamma[flow != 0]-1)/-1) - 1))
elif mtype in ["sub", "sup"]:
if (flow < 1).any():
raise Exception("Area ratio inputs must be real numbers greater" \
" than or equal to 1.")
M[:] = 0.2 if mtype == "sub" else 1.8
for _ in xrange(_AETB_iternum):
K = M ** 2
f = -flow + a**(-c) * ((1+b*K)**c) / M #mach-area relation
g = a**(-c) * ((1+b*K)**(c-1)) * (b*(2*c - 1)*K - 1) / K #deriv
M = M - (f / g) #Newton-Raphson
M[flow == 1] = 1
M[sp.isinf(flow)] = sp.inf
else:
raise Exception("Keyword input must be an acceptable string to" \
" select input parameter.")
d = 1 + b*M**2
T = d**(-1)
P = d**(-gamma/(gamma-1))
rho = d**(-1/(gamma-1))
area = sp.empty(M.shape, sp.float64)
r = sp.logical_and(M != 0, sp.isfinite(M))
area[r] = a[r]**(-c[r]) * d[r]**c[r] / M[r]
area[sp.logical_not(r)] = sp.inf
return from_ndarray(itype, M, T, P, rho, area)
def default_costmatrix(numstates, dtype=numpy.int):
"a square array with zeroes along the diagonal, ones elsewhere"
return scipy.logical_not(scipy.identity(numstates)).astype(float)
def execute(self, nprocesses=1) :
"""Worker funciton."""
params = self.params
# Make parent directory and write parameter file.
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params, params['output_root'] + 'params.ini',
prefix=prefix)
save_noise_diag = params['save_noise_diag']
in_root = params['input_root']
all_out_fname_list = []
all_in_fname_list = []
# Figure out what the band names are.
bands = params['bands']
if not bands:
map_files = glob.glob(in_root + 'dirty_map_' + pol_str + "_*.npy")
bands = []
root_len = len(in_root + 'dirty_map_')
for file_name in map_files:
bands.append(file_name[root_len:-4])
# Loop over files to process.
for pol_str in params['polarizations']:
for band in bands:
if band == -1:
band_str = ''
else:
band_str = "_" + repr(band)
dmap_fname = (in_root + 'dirty_map_' + pol_str +
band_str + '.npy')
all_in_fname_list.append(
kiyopy.utils.abbreviate_file_path(dmap_fname))
# Load the dirty map and the noise matrix.
dirty_map = algebra.load(dmap_fname)
dirty_map = algebra.make_vect(dirty_map)
if dirty_map.axes != ('freq', 'ra', 'dec') :
msg = ("Expeced dirty map to have axes ('freq',"
"'ra', 'dec'), but it has axes: "
+ str(dirty_map.axes))
raise ce.DataError(msg)
shape = dirty_map.shape
# Initialize the clean map.
clean_map = algebra.info_array(sp.zeros(dirty_map.shape))
clean_map.info = dict(dirty_map.info)
clean_map = algebra.make_vect(clean_map)
# If needed, initialize a map for the noise diagonal.
if save_noise_diag :
noise_diag = algebra.zeros_like(clean_map)
if params["from_eig"]:
# Solving from eigen decomposition of the noise instead of
# the noise itself.
# Load in the decomposition.
evects_fname = (in_root + 'noise_evects_' + pol_str +
+ band_str + '.npy')
if self.feedback > 1:
print "Using dirty map: " + dmap_fname
print "Using eigenvectors: " + evects_fname
evects = algebra.open_memmap(evects_fname, 'r')
evects = algebra.make_mat(evects)
evals_inv_fname = (in_root + 'noise_evalsinv_' + pol_str
+ "_" + repr(band) + '.npy')
evals_inv = algebra.load(evals_inv_fname)
evals_inv = algebra.make_mat(evals_inv)
# Solve for the map.
if params["save_noise_diag"]:
clean_map, noise_diag = solve_from_eig(evals_inv,
evects, dirty_map, True, self.feedback)
else:
clean_map = solve_from_eig(evals_inv,
evects, dirty_map, False, self.feedback)
# Delete the eigen vectors to recover memory.
del evects
else:
# Solving from the noise.
noise_fname = (in_root + 'noise_inv_' + pol_str +
band_str + '.npy')
if self.feedback > 1:
print "Using dirty map: " + dmap_fname
print "Using noise inverse: " + noise_fname
all_in_fname_list.append(
kiyopy.utils.abbreviate_file_path(noise_fname))
noise_inv = algebra.open_memmap(noise_fname, 'r')
noise_inv = algebra.make_mat(noise_inv)
# Two cases for the noise. If its the same shape as the map
# then the noise is diagonal. Otherwise, it should be
# block diagonal in frequency.
if noise_inv.ndim == 3 :
if noise_inv.axes != ('freq', 'ra', 'dec') :
msg = ("Expeced noise matrix to have axes "
"('freq', 'ra', 'dec'), but it has: "
+ str(noise_inv.axes))
raise ce.DataError(msg)
# Noise inverse can fit in memory, so copy it.
noise_inv_memory = sp.array(noise_inv, copy=True)
# Find the non-singular (covered) pixels.
max_information = noise_inv_memory.max()
good_data = noise_inv_memory < 1.0e-10*max_information
# Make the clean map.
clean_map[good_data] = (dirty_map[good_data]
/ noise_inv_memory[good_data])
if save_noise_diag :
noise_diag[good_data] = \
1/noise_inv_memory[good_data]
elif noise_inv.ndim == 5 :
if noise_inv.axes != ('freq', 'ra', 'dec', 'ra',
'dec'):
msg = ("Expeced noise matrix to have axes "
"('freq', 'ra', 'dec', 'ra', 'dec'), "
"but it has: " + str(noise_inv.axes))
raise ce.DataError(msg)
# Arrange the dirty map as a vector.
dirty_map_vect = sp.array(dirty_map) # A view.
dirty_map_vect.shape = (shape[0], shape[1]*shape[2])
frequencies = dirty_map.get_axis('freq')/1.0e6
# Allowcate memory only once.
noise_inv_freq = sp.empty((shape[1], shape[2],
shape[1], shape[2]), dtype=float)
if self.feedback > 1 :
print "Inverting noise matrix."
# Block diagonal in frequency so loop over frequencies.
for ii in xrange(dirty_map.shape[0]) :
if self.feedback > 1:
print "Frequency: ", "%5.1f"%(frequencies[ii]),
if self.feedback > 2:
print ", start mmap read:",
sys.stdout.flush()
noise_inv_freq[...] = noise_inv[ii, ...]
if self.feedback > 2:
print "done, start eig:",
sys.stdout.flush()
noise_inv_freq.shape = (shape[1]*shape[2],
shape[1]*shape[2])
# Solve the map making equation by diagonalization.
noise_inv_diag, Rot = sp.linalg.eigh(
noise_inv_freq, overwrite_a=True)
if self.feedback > 2:
print "done",
map_rotated = sp.dot(Rot.T, dirty_map_vect[ii])
# Zero out infinite noise modes.
bad_modes = (noise_inv_diag
< 1.0e-5 * noise_inv_diag.max())
if self.feedback > 1:
print ", discarded: ",
print "%4.1f" % (100.0 * sp.sum(bad_modes)
/ bad_modes.size),
print "% of modes",
if self.feedback > 2:
print ", start rotations:",
sys.stdout.flush()
map_rotated[bad_modes] = 0.
noise_inv_diag[bad_modes] = 1.0
# Solve for the clean map and rotate back.
map_rotated /= noise_inv_diag
map = sp.dot(Rot, map_rotated)
if self.feedback > 2:
print "done",
sys.stdout.flush()
# Fill the clean array.
map.shape = (shape[1], shape[2])
clean_map[ii, ...] = map
if save_noise_diag :
# Using C = R Lambda R^T
# where Lambda = diag(1/noise_inv_diag).
temp_noise_diag = 1/noise_inv_diag
temp_noise_diag[bad_modes] = 0
# Multiply R by the diagonal eigenvalue matrix.
# Broadcasting does equivalent of mult by diag
# matrix.
temp_mat = Rot*temp_noise_diag
# Multiply by R^T, but only calculate the
# diagonal elements.
for jj in range(shape[1]*shape[2]) :
temp_noise_diag[jj] = sp.dot(
temp_mat[jj,:], Rot[jj,:])
temp_noise_diag.shape = (shape[1], shape[2])
noise_diag[ii, ...] = temp_noise_diag
# Return workspace memory to origional shape.
noise_inv_freq.shape = (shape[1], shape[2],
shape[1], shape[2])
if self.feedback > 1:
print ""
sys.stdout.flush()
elif noise_inv.ndim == 6 :
if save_noise_diag:
# OLD WAY.
#clean_map, noise_diag, chol = solve(noise_inv,
# dirty_map, True, feedback=self.feedback)
# NEW WAY.
clean_map, noise_diag, noise_inv_diag, chol = \
solve(noise_fname, noise_inv, dirty_map,
True, feedback=self.feedback)
else:
# OLD WAY.
#clean_map, chol = solve(noise_inv, dirty_map,
# False, feedback=self.feedback)
# NEW WAY.
clean_map, noise_inv_diag, chol = \
solve(noise_fname, noise_inv, dirty_map,
False, feedback=self.feedback)
if params['save_cholesky']:
chol_fname = (params['output_root'] + 'chol_'
+ pol_str + band_str + '.npy')
sp.save(chol_fname, chol)
if params['save_noise_inv_diag']:
noise_inv_diag_fname = (params['output_root'] +
'noise_inv_diag_' + pol_str + band_str
+ '.npy')
algebra.save(noise_inv_diag_fname, noise_inv_diag)
# Delete the cholesky to recover memory.
del chol
else :
raise ce.DataError("Noise matrix has bad shape.")
# In all cases delete the noise object to recover memeory.
del noise_inv
# Write the clean map to file.
out_fname = (params['output_root'] + 'clean_map_'
+ pol_str + band_str + '.npy')
if self.feedback > 1:
print "Writing clean map to: " + out_fname
algebra.save(out_fname, clean_map)
all_out_fname_list.append(
kiyopy.utils.abbreviate_file_path(out_fname))
if save_noise_diag :
noise_diag_fname = (params['output_root'] + 'noise_diag_'
+ pol_str + band_str + '.npy')
algebra.save(noise_diag_fname, noise_diag)
all_out_fname_list.append(
kiyopy.utils.abbreviate_file_path(noise_diag_fname))
# Check the clean map for faileur.
if not sp.alltrue(sp.isfinite(clean_map)):
n_bad = sp.sum(sp.logical_not(sp.isfinite(clean_map)))
msg = ("Non finite entries found in clean map. Solve"
" failed. %d out of %d entries bad"
% (n_bad, clean_map.size))
raise RuntimeError(msg)
