def __init__(self, x_wc, K): self.x_wc = x_wc self.x_cw = coord_xfms.ssc.inverse(self.x_wc) # camera calibration matrix self.K = K # camera center self.C = self.x_wc[0:3] # transforms points in world frame to points in camera frame self.wHc = coord_xfms.xyzrph2matrix(self.x_cw) # transforms points in camera frame to points in world frame self.cHw = coord_xfms.xyzrph2matrix(self.x_wc) # the stupid [R t] notation that I absolutely hate self.Rt = self.wHc[0:3, :] # projects points in world frame to image plane of camera self.P = K.dot(self.Rt) # used for normalized image points self.invK = pl.inv(K[0:3, 0:3]) # used in back-projecting points to rays self.pinvP = pl.pinv(self.P)
def __init__ (self, x_wc, K): self.x_wc = x_wc self.x_cw = coord_xfms.ssc.inverse (self.x_wc) # camera calibration matrix self.K = K # camera center self.C = self.x_wc[0:3] # transforms points in world frame to points in camera frame self.wHc = coord_xfms.xyzrph2matrix (self.x_cw) # transforms points in camera frame to points in world frame self.cHw = coord_xfms.xyzrph2matrix (self.x_wc) # the stupid [R t] notation that I absolutely hate self.Rt = self.wHc[0:3,:] # projects points in world frame to image plane of camera self.P = K.dot (self.Rt) # used for normalized image points self.invK = pl.inv (K[0:3,0:3]) # used in back-projecting points to rays self.pinvP = pl.pinv (self.P)
def find_factors(idat, odat, k = None): """ A routine to compute the main predictors (linear combinations of coordinates) in idat to predict odat. *Parameters* idat: d x n data matrix, with n measurements, each of dimension d odat: q x n data matrix with n measurements, each of dimension q *Returns* **Depending on whether or not** *k* **is provided, the returned value is different** * if k is given, compute the first k regressors and return an orthogonal matrix that contains the regressors in its colums, i.e. reg[0,:] is the first regressor * if k is not given or None, return a d-dimensional vector v(k) explaining which fraction of the total predictable variance can be explained using only k regressors. **NOTE** #. idat and odat must have zero mean #. To interpret the regressors, it is advisable to have the for columns of idat having the same variance """ # transform into z-scores u, s, v = svd(idat, full_matrices = False) su = dot(diag(1./s), u.T) z = dot(su,idat) # ! Note that the covariance of z is *NOT* 1, but 1/n; z*z.T = 1 ! # least-squares regression: A = dot(odat, pinv(z)) uA, sigma_A, vA = svd(A, full_matrices = False) if k is None: vk = cumsum(sigma_A**2) / sum(sigma_A**2) return vk else: # choose k predictors sigma_A1 = sigma_A.copy() sigma_A1[k:] = 0 A1 = reduce(dot, [uA, diag(sigma_A1), vA]) B = dot(A1, su) uB, sigma_B, vB = svd(B, full_matrices = False) regs = vB[:k,:].T return regs
def _calculate_svd(pp, r_0, beta, N_piercepoints): """ Returns result (U) of svd for K-L vectors Parameters ---------- pp : array Array of piercepoint locations r_0: float Scale size of amp fluctuations (m) beta: float Power-law index for amp structure function (5/3 => pure Kolmogorov turbulence) N_piercepoints : int Number of piercepoints Returns ------- C : array C matrix pinvC : array Inv(C) matrix U : array Unitary matrix """ import numpy as np from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye D = np.resize(pp, (N_piercepoints, N_piercepoints, 3)) D = np.transpose(D, (1, 0, 2)) - D D2 = np.sum(D**2, axis=2) C = -(D2 / r_0**2)**(beta / 2.0) / 2.0 pinvC = pinv(C, rcond=1e-3) U, S, V = svd(C) return C, pinvC, U
def makeTECparmdb(H, solset, TECsolTab, timewidths, freq, freqwidth): """Returns TEC screen parmdb parameters H - H5parm object solset - solution set with TEC screen parameters TECsolTab = solution table with tecscreen values timewidths - time widths of output parmdb freq - frequency of output parmdb freqwidth - frequency width of output parmdb """ from pylab import pinv global ipbar, pbar station_dict = H.getAnt(solset) station_names = station_dict.keys() station_positions = station_dict.values() source_dict = H.getSou(solset) source_names = source_dict.keys() source_positions = source_dict.values() tec_sf = solFetcher(TECsolTab) tec_screen, axis_vals = tec_sf.getValues() times = axis_vals['time'] beta = TECsolTab._v_attrs['beta'] r_0 = TECsolTab._v_attrs['r_0'] height = TECsolTab._v_attrs['height'] order = TECsolTab._v_attrs['order'] pp = tec_sf.t.piercepoint N_sources = len(source_names) N_times = len(times) N_freqs = 1 N_stations = len(station_names) N_piercepoints = N_sources * N_stations freqs = freq freqwidths = freqwidth parms = {} v = {} v['times'] = times v['timewidths'] = timewidths v['freqs'] = freqs v['freqwidths'] = freqwidths for station_name in station_names: for source_name in source_names: v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'TECfit_white:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() for k in range(N_times): D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3)) D = np.transpose(D, (1, 0, 2)) - D D2 = np.sum(D**2, axis=2) C = -(D2 / (r_0**2))**(beta / 2.0) / 2.0 tec_fit_white = np.dot(pinv(C), tec_screen[:, k, :].reshape(N_piercepoints)) pp_idx = 0 for src, source_name in enumerate(source_names): for sta, station_name in enumerate(station_names): parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = pp[k, pp_idx, 0] parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = pp[k, pp_idx, 1] parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = pp[k, pp_idx, 2] parmname = 'TECfit_white:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx] parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx] parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx] pp_idx += 1 pbar.update(ipbar) ipbar += 1 time_start = times[0] - timewidths[0] / 2 time_end = times[-1] + timewidths[-1] / 2 v['times'] = np.array([(time_start + time_end) / 2]) v['timewidths'] = np.array([time_end - time_start]) v_r0 = v.copy() v_r0['values'] = np.array(r_0, dtype=np.double, ndmin=2) parms['r_0'] = v_r0 v_beta = v.copy() v_beta['values'] = np.array(beta, dtype=np.double, ndmin=2) parms['beta'] = v_beta v_height = v.copy() v_height['values'] = np.array(height, dtype=np.double, ndmin=2) parms['height'] = v_height return parms
def add_stations(station_selection, phases0, phases1, flags, mask, station_names, station_positions, source_names, source_selection, times, freqs, r, nband_min=2, soln_type='phase', nstations_max=None, excluded_stations=None, t_step=5, tec_step1=5, tec_step2=21, search_full_tec_range=False): """ Adds stations to TEC fitting using an iterative initial-guess search to ensure the global min is found Keyword arguments: station_selection -- indices of stations to use in fitting phases0 -- XX phase solutions phases1 -- YY phase solutions flags -- phase solution flags (0 = use, 1 = flagged) mask -- mask for sources and frequencies (0 = ignore, 1 = use) station_names -- array of station names source_names -- array of source names source_selection -- indices of sources to use in fitting times -- array of times freqs -- array of frequencies r -- array of TEC solutions returned by fit_tec_per_source_pair() nband_min -- min number of bands for a source to be used soln_type -- type of phase solution: 'phase' or 'scalarphase' nstations_max -- max number of stations to use excluded_stations -- stations to exclude t_step -- try full TEC range every t_step number of solution times tec_step1 -- number of steps in TEC subrange (+/- last TEC fit value) tec_step2 -- number of steps in full TEC range (-0.1 -- 0.1) search_full_tec_range -- always search the full TEC range (-0.1 -- 0.1) """ from pylab import pinv, newaxis, find, amin import numpy as np from lofar.expion import baselinefitting import progressbar N_sources_selected = len(source_selection) N_stations_selected = len(station_selection) N_piercepoints = N_sources_selected * N_stations_selected N_times = len(times) N_stations = len(station_names) N_sources = len(source_names) N_pairs = 0 for ii, i in enumerate(source_selection): for jj, j in enumerate(source_selection): if j == i: break subband_selection = find(mask[i, :] * mask[j, :]) if len(subband_selection) < nband_min: continue N_pairs += 1 D = np.resize(station_positions, (N_stations, N_stations, 3)) D = np.transpose(D, (1, 0, 2)) - D D = np.sqrt(np.sum(D**2, axis=2)) station_selection1 = station_selection stations_to_add = np.array([ i for i in range(len(station_names)) if i not in station_selection1 and station_names[i] not in excluded_stations ]) if len(stations_to_add) == 0: return station_selection1, r # Check if desired number of stations is already reached if nstations_max is not None: if len(station_selection1) >= nstations_max: return station_selection1, r logging.info("Using fitting with iterative search for remaining stations " "(up to {0} stations in total)".format(nstations_max)) q = r while len(stations_to_add) > 0: D1 = D[stations_to_add[:, newaxis], station_selection1[newaxis, :]] minimum_distance = amin(D1, axis=1) station_to_add = stations_to_add[np.argmin(minimum_distance)] station_selection1 = np.append(station_selection1, station_to_add) N_stations_selected1 = len(station_selection1) # Remove station from list stations_to_add = stations_to_add[stations_to_add != station_to_add] sols_list = [] eq_list = [] min_e_list = [] ipbar = 0 logging.info('Fitting TEC values with {0} included...'.format( station_names[station_to_add])) pbar = progressbar.ProgressBar(maxval=N_pairs * N_times).start() for ii, i in enumerate(source_selection): for jj, j in enumerate(source_selection): if j == i: break subband_selection = find(mask[i, :] * mask[j, :]) if len(subband_selection) < nband_min: continue logging.debug('Adding {0} for source pair: {1}-{2}'.format( station_names[station_to_add], i, j)) p0 = phases0[i, station_selection1[:, newaxis], subband_selection[newaxis, :], :] - phases0[ j, station_selection1[:, newaxis], subband_selection[newaxis, :], :] p0 = p0 - np.mean(p0, axis=0)[newaxis, :, :] if soln_type != 'scalarphase': p1 = phases1[i, station_selection1[:, newaxis], subband_selection[newaxis, :], :] - phases1[ j, station_selection1[:, newaxis], subband_selection[newaxis, :], :] p1 = p1 - np.mean(p1, axis=0)[newaxis, :, :] A = np.zeros((len(subband_selection), 1)) A[:, 0] = 8.44797245e9 / freqs[subband_selection] flags_source_pair = flags[ i, station_selection1[:, newaxis], subband_selection[newaxis, :], :] * flags[ j, station_selection1[:, newaxis], subband_selection[newaxis, :], :] constant_parms = np.zeros((1, N_stations_selected1), dtype=np.bool) sols = np.zeros((N_times, N_stations_selected1), dtype=np.float) p_0_best = None for t_idx in range(N_times): if np.mod(t_idx, t_step) == 0: min_e = np.Inf if p_0_best is not None and not search_full_tec_range: min_tec = p_0_best[0, -1] - 0.02 max_tec = p_0_best[0, -1] + 0.02 nsteps = tec_step1 else: min_tec = -0.1 max_tec = 0.1 nsteps = tec_step2 logging.debug( ' Trying initial guesses between {0} and ' '{1} TECU'.format(min_tec, max_tec)) for offset in np.linspace(min_tec, max_tec, nsteps): p_0 = np.zeros((1, N_stations_selected1), np.double) p_0[0, :N_stations_selected1 - 1] = (q[ii, t_idx, :] - q[jj, t_idx, :])[newaxis, :] p_0[0, -1] = offset x = p0[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol0 -= np.mean(sol0) residual = np.mod( np.dot(A, sol0) - x.T + np.pi, 2 * np.pi) - np.pi residual = residual[f.T == 0] e = np.var(residual) if soln_type != 'scalarphase': x = p1[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol1 = baselinefitting.fit( x.T, A, p_0, f, constant_parms) sol1 -= np.mean(sol1) residual = np.mod( np.dot(A, sol1) - x.T + np.pi, 2 * np.pi) - np.pi residual = residual[f.T == 0] e += np.var(residual) else: sol1 = sol0 if e < min_e: logging.debug( ' Found new min variance of {0} ' 'with initial guess of {1} TECU'.format( e, p_0[0, -1])) min_e = e p_0_best = p_0 sols[t_idx, :] = (sol0[0, :] + sol1[0, :]) / 2 else: # Use previous init x = p0[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0_best, f, constant_parms) sol0 -= np.mean(sol0) if soln_type != 'scalarphase': x = p1[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol1 = baselinefitting.fit(x.T, A, p_0_best, f, constant_parms) sol1 -= np.mean(sol1) else: sol1 = sol0 sols[t_idx, :] = (sol0[0, :] + sol1[0, :]) / 2 ipbar += 1 pbar.update(ipbar) ### Remove outliers logging.debug(' Searching for outliers...') for kk in range(10): s = sols[:, -1].copy() selection = np.zeros(len(s), np.bool) for t_idx in range(len(s)): start_idx = np.max([t_idx - 10, 0]) end_idx = np.min([t_idx + 10, len(s)]) selection[t_idx] = np.sum( abs(s[start_idx:end_idx] - s[t_idx]) < 0.02) > ( end_idx - start_idx - 8) outliers = find(np.logical_not(selection)) if len(outliers) == 0: break for t_idx in outliers: try: idx0 = find(selection[:t_idx])[-1] except IndexError: idx0 = -1 try: idx1 = find(selection[t_idx + 1:])[0] + t_idx + 1 except IndexError: idx1 = -1 if idx0 == -1: s[t_idx] = s[idx1] elif idx1 == -1: s[t_idx] = s[idx0] else: s[t_idx] = (s[idx0] * (idx1 - t_idx) + s[idx1] * (t_idx - idx0)) / (idx1 - idx0) p_0 = np.zeros((1, N_stations_selected1), np.double) p_0[0, :] = sols[t_idx, :] p_0[0, -1] = s[t_idx] x = p0[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol0 -= np.mean(sol0) if soln_type != 'scalarphase': x = p1[:, :, t_idx].copy() sol1 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol1 -= np.mean(sol1) else: sol1 = sol0 sols[t_idx, :] = (sol0[0, :] + sol1[0, :]) / 2 weight = 1.0 sols_list.append(weight * sols) min_e_list.append(min_e) eq = np.zeros(N_sources) eq[ii] = weight eq[jj] = -weight eq_list.append(eq) sols = np.array(sols_list) B = np.array(eq_list) pinvB = pinv(B) q = np.dot(pinvB, sols.transpose([1, 0, 2])) pbar.finish() if nstations_max is not None: if N_stations_selected1 == nstations_max: break return station_selection1, q
def fit_tec_per_source_pair(phases, flags, mask, freqs, init_sols=None, init_sols_per_pair=False, propagate=False, nband_min=2): """Fits TEC values to phase solutions per source pair Returns TEC solutions as array of shape (N_sources, N_times, N_stations) Keyword arguments: phases -- array of phase solutions flags -- phase solution flags (0 = use, 1 = flagged) mask -- mask for sources and frequencies (0 = ignore, 1 = use) freqs -- array of frequencies init_sols -- solutions to use to initialize the fits init_sols_per_pair -- init_sols are per source pair (not per source) propagate -- propagate solutions from previous solution nband_min -- min number of bands for a source to be used """ from pylab import pinv, newaxis, find import numpy as np from lofar.expion import baselinefitting import progressbar sols_list = [] eq_list = [] vars_list = [] var_eq_list = [] N_sources = phases.shape[0] N_stations = phases.shape[1] N_times = phases.shape[3] N_pairs = 0 for i in range(N_sources): for j in range(i): subband_selection = find(mask[i, :] * mask[j, :]) if len(subband_selection) < nband_min: continue N_pairs += 1 if init_sols is None and not init_sols_per_pair: init_sols = np.zeros((N_sources, N_times, N_stations), dtype=np.float) elif init_sols is None and init_sols_per_pair: init_sols = np.zeros((N_pairs, N_times, N_stations), dtype=np.float) source_pairs = [] k = 0 ipbar = 0 logging.info('Fitting TEC values...') pbar = progressbar.ProgressBar(maxval=N_pairs * N_times).start() for i in range(N_sources): for j in range(i): subband_selection = find(mask[i, :] * mask[j, :]) if len(subband_selection) < nband_min: continue source_pairs.append((i, j)) p = phases[i, :, subband_selection, :] - phases[j, :, subband_selection, :] A = np.zeros((len(subband_selection), 1)) A[:, 0] = 8.44797245e9 / freqs[subband_selection] flags_source_pair = flags[i, :, subband_selection, :] * flags[ j, :, subband_selection, :] constant_parms = np.zeros((1, N_stations), dtype=np.bool) sols = np.zeros((N_times, N_stations), dtype=np.float) if init_sols_per_pair: p_0 = init_sols[k, 0, :][newaxis, :] else: p_0 = (init_sols[i, 0, :] - init_sols[j, 0, :])[newaxis, :] for t_idx in range(N_times): x = p[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() if not propagate: if init_sols_per_pair: p_0 = init_sols[k, t_idx, :][newaxis, :] else: p_0 = (init_sols[i, t_idx, :] - init_sols[j, 0, :])[newaxis, :] sol = baselinefitting.fit(x, A, p_0, f, constant_parms) if propagate: p_0 = sol.copy() sols[t_idx, :] = sol[0, :] ipbar += 1 pbar.update(ipbar) sols = sols[:, :] - np.mean(sols[:, :], axis=1)[:, newaxis] weight = len(subband_selection) sols_list.append(sols * weight) eq = np.zeros(N_sources) eq[i] = weight eq[j] = -weight eq_list.append(eq) k += 1 pbar.finish() sols = np.array(sols_list) B = np.array(eq_list) source_selection = find(np.sum(abs(B), axis=0)) N_sources = len(source_selection) if N_sources == 0: logging.error( 'All sources have fewer than the required minimum number of bands.' ) return None, None pinvB = pinv(B) r = np.dot(pinvB, sols.transpose([1, 0, 2])) r = r[source_selection, :, :] return r, source_selection
def _fit_tec_screen(station_names, source_names, pp, airmass, rr, weights, times, height, order, r_0, beta, outQueue): """ Fits a screen to given TEC values using Karhunen-Lo`eve base vectors Parameters ---------- station_names: array Array of station names source_names: array Array of source names pp: array Array of piercepoint locations airmass: array Array of airmass values (note: not currently used) rr: array Array of TEC values to fit screen to weights: array Array of weights times: array Array of times height: float Height of screen (m) order: int Order of screen (i.e., number of KL base vectors to keep) r_0: float Scale size of phase fluctuations (m) beta: float Power-law index for phase structure function (5/3 => pure Kolmogorov turbulence) """ import numpy as np from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye logging.info('Fitting screens...') # Initialize arrays N_stations = len(station_names) N_sources = len(source_names) N_times = len(times) N_piercepoints = N_sources * N_stations tec_fit_white_all = np.zeros((N_times, N_sources, N_stations)) tec_residual_all = np.zeros((N_times, N_sources, N_stations)) for k in range(N_times): D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3)) D = np.transpose(D, (1, 0, 2)) - D D2 = np.sum(D**2, axis=2) C = -(D2 / r_0**2)**(beta / 2.0) / 2.0 pinvC = pinv(C, rcond=1e-3) U, S, V = svd(C) invU = pinv(np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], U[:, :order])), rcond=1e-3) # Calculate screen rr1 = np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], rr[:, k])) tec_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) tec_fit_white_all[k, :, :] = tec_fit.reshape((N_sources, N_stations)) residual = rr - np.dot(C, tec_fit)[:, newaxis] tec_residual_all[k, :, :] = residual.reshape((N_sources, N_stations)) outQueue.put([tec_fit_white_all, tec_residual_all, times])
def _fit_phase_screen(station_names, source_names, pp, airmass, rr, weights, times, height, order, r_0, beta, outQueue): """ Fits a screen to given phase values using Karhunen-Lo`eve base vectors Parameters ---------- station_names: array Array of station names source_names: array Array of source names pp: array Array of piercepoint locations airmass: array Array of airmass values (note: not currently used) rr: array Array of phase values to fit screen to weights: array Array of weights times: array Array of times height: float Height of screen (m) order: int Order of screen (i.e., number of KL base vectors to keep) r_0: float Scale size of phase fluctuations (m) beta: float Power-law index for phase structure function (5/3 => pure Kolmogorov turbulence) """ import numpy as np from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye logging.info('Fitting screens...') # Initialize arrays N_stations = len(station_names) N_sources = len(source_names) N_times = len(times) N_piercepoints = N_sources * N_stations real_fit_white_all = np.zeros((N_times, N_sources, N_stations)) imag_fit_white_all = np.zeros((N_times, N_sources, N_stations)) phase_fit_white_all = np.zeros((N_times, N_sources, N_stations)) real_residual_all = np.zeros((N_times, N_sources, N_stations)) imag_residual_all = np.zeros((N_times, N_sources, N_stations)) phase_residual_all = np.zeros((N_times, N_sources, N_stations)) # Change phase to real/imag rr_real = np.cos(rr) rr_imag = np.sin(rr) for k in range(N_times): try: D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3)) D = np.transpose(D, (1, 0, 2)) - D D2 = np.sum(D**2, axis=2) C = -(D2 / r_0**2)**(beta / 2.0) / 2.0 pinvC = pinv(C, rcond=1e-3) U, S, V = svd(C) invU = pinv(np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], U[:, :order])), rcond=1e-3) # Calculate real screen rr1 = np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], rr_real[:, k])) real_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) real_fit_white_all[k, :, :] = real_fit.reshape((N_sources, N_stations)) residual = rr_real - np.dot(C, real_fit)[:, newaxis] real_residual_all[k, :, :] = residual.reshape((N_sources, N_stations)) # Calculate imag screen rr1 = np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], rr_imag[:, k])) imag_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) imag_fit_white_all[k, :, :] = imag_fit.reshape((N_sources, N_stations)) residual = rr_imag - np.dot(C, imag_fit)[:, newaxis] imag_residual_all[k, :, :] = residual.reshape((N_sources, N_stations)) # Calculate phase screen phase_fit = np.dot(pinvC, np.arctan2(np.dot(C, imag_fit), np.dot(C, real_fit))) phase_fit_white_all[k, :, :] = phase_fit.reshape((N_sources, N_stations)) residual = rr - np.dot(C, phase_fit)[:, newaxis] phase_residual_all[k, :, :] = residual.reshape((N_sources, N_stations)) except: # Set screen to zero if fit did not work logging.debug('Screen fit failed for timeslot {}'.format(k)) real_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations)) real_residual_all[k, :, :] = np.ones((N_sources, N_stations)) imag_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations)) imag_residual_all[k, :, :] = np.ones((N_sources, N_stations)) phase_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations)) phase_residual_all[k, :, :] = np.ones((N_sources, N_stations)) outQueue.put([real_fit_white_all, real_residual_all, imag_fit_white_all, imag_residual_all, phase_fit_white_all, phase_residual_all, times])
def correction_light(I, method, show_light, mask=None): """Corrige la derive eclairement :I: array_like ou iplimage :method: 'polynomial' or 'frequency' :show_light: option affiche correction (true|false) :mask: array de zone non interet :returns: iplimage 32bit """ from progress import * import Tkinter if type(I) == cv.iplimage: if I.nChannels == 3: if method == 'None': I = RGB2L(I) I = cv2array(I)[:, :, 0] I = pymorph.hmin(I, 15, pymorph.sedisk(3)) I = array2cv(I) cv.EqualizeHist(I, I) return I I = RGB2L(I) I_32bit = cv.CreateImage(cv.GetSize(I), cv.IPL_DEPTH_32F, 1) cv.ConvertScale(I, I_32bit, 3000.0, 0.0) I = cv.CloneImage(I_32bit) I = cv2array(I)[:, :, 0] elif len(I.shape) == 3: I = (I[:, :, 0] + I[:, :, 0] + I[:, :, 0])\ / 3.0 # A modifier: non utiliser dans notre cas elif method == 'None': I = array2cv(I) cv.EqualizeHist(I, I) return I I = np.log(I + 10 ** (-6)) (H, W) = np.shape(I) I_out = I * 0 + 10 ** (-6) if method == 'polynomial': ## I = M.A avec A coeff. du polynome I_flat = I.flatten() degree = 3 print("modification degree 3") #degree du polynome nb_coeff = (degree + 1) * (degree + 2) / 2 # nombre coefficient [yy, xx] = np.meshgrid(np.arange(W, dtype=np.float64), np.arange(H, dtype=np.float64)) if mask is not None: xx[mask] = 0 yy[mask] = 0 # Creation de M try: M = np.zeros((H * W, nb_coeff), dtype=np.float64) except MemoryError: print MemoryError return MemoryError i, j = 0, 0 # i,j degree de x,y #Bar progression bar = Tkinter.Tk(className='Correcting Light...') m = Meter(bar, relief='ridge', bd=3) m.pack(fill='x') m.set(0.0, 'Starting correction...') for col in np.arange(nb_coeff): M[:, col] = (xx.flatten() ** i) * (yy.flatten() ** j) i += 1 m.set(0.5 * float(col) / (nb_coeff - 1)) if i + j == degree + 1: i = 0 j += 1 # Resolution au sens des moindres carree: pseudo-inverse try: M = pl.pinv(M) A = np.dot(M, I_flat) except ValueError: return ValueError # Calcul de la surface i, j = 0, 0 surface = np.zeros((H, W), dtype=np.float64) for cmpt in np.arange(nb_coeff): surface += A[cmpt] * (xx ** i) * (yy ** j) # forme quadratique i += 1 m.set(0.5 + 0.5 * float(cmpt) / (nb_coeff - 1)) if i + j == degree + 1: i = 0 j += 1 bar.destroy() I_out = np.exp(I / surface) light = surface elif method == 'frequency': Rx, Ry = 2, 2 # zero padding N = [H, W] filtre = np.zeros((N[1], N[0])) centre_x = round(N[0] / 2) centre_y = round(N[1] / 2) print("FFT2D...") I_fourier = pl.fftshift(pl.fft2(I, N)) # Gaussian filter [xx, yy] = np.meshgrid(np.arange(N[0], dtype=np.float), np.arange(N[1], dtype=np.float)) filtre = np.exp(-2 * ((xx - centre_x) ** 2 + (yy - centre_y) ** 2) / (Rx ** 2 + Ry ** 2)) filtre = pl.transpose(filtre) I_fourier = I_fourier * filtre print("IFFT2D...") I_out = (np.abs(pl.ifft2(pl.ifftshift(I_fourier), N)))[0:H, 0:W] light = I_out I_out = np.exp(I / I_out) else: light = I * 0 I_out = I # Display Light if show_light: light = ((light - light.min()) * 3000.0 / light.max()).astype('float32') light = array2cv(light) fig = pl.figure() pl.imshow(light) fig.show() I_out = (I_out - I_out.min()) * 3000.0 / I_out.max() I_out = I_out.astype('uint8') #chapeau haut de forme I_out = pymorph.hmin(I_out, 25, pymorph.sedisk(3)) #Conversion en iplimage et ajustement contraste gr = array2cv(I_out) cv.EqualizeHist(gr, gr) return gr
def make_tec_screen_plots(pp, rr, tec_fit_white, tec_fit, station_positions, source_names, times, height, order, beta_val, r_0, prefix = 'frame_', remove_gradient=True): """Makes plots of TEC screens""" import pylab import numpy as np import os from operations.tecscreen import calc_piercepoint import progressbar root_dir = os.path.dirname(prefix) if root_dir == '': root_dir = './' prestr = os.path.basename(prefix) try: os.makedirs(root_dir) except OSError: pass N_stations = station_positions.shape[0] N_sources = len(source_names) N_times = len(times) A = pylab.concatenate([pylab.kron(np.eye(N_sources), np.ones((N_stations,1))), pylab.kron(np.ones((N_sources,1)), np.eye(N_stations))], axis=1) N_piercepoints = N_sources * N_stations P = np.eye(N_piercepoints) - np.dot(np.dot(A, pylab.pinv(np.dot(A.T, A))), A.T) x,y,z = station_positions[0, :] east = np.array([-y, x, 0]) east = east / pylab.norm(east) north = np.array([ -x, -y, (x*x + y*y)/z]) north = north / pylab.norm(north) up = np.array([x,y,z]) up = up / pylab.norm(up) T = pylab.concatenate([east[:, pylab.newaxis], north[:, pylab.newaxis]], axis=1) pp1 = np.dot(pp[0, :, :], T) lower = np.amin(pp1, axis=0) upper = np.amax(pp1, axis=0) extent = upper - lower lower = lower - 0.05 * extent upper = upper + 0.05 * extent extent = upper - lower N = 50 xr = np.arange(lower[0], upper[0], extent[0]/N) yr = np.arange(lower[1], upper[1], extent[1]/N) screen = np.zeros((N, N, N_times)) fitted_tec1 = tec_fit.transpose([0, 2, 1]).reshape(N_piercepoints, N_times) + np.dot(P, rr-tec_fit.transpose([0, 2, 1]).reshape(N_piercepoints, N_times)) logging.info('Calculating TEC screen images...') pbar = progressbar.ProgressBar(maxval=N_times).start() ipbar = 0 for k in range(N_times): f = tec_fit_white[:, k, :] for i, x in enumerate(xr[0: N]): for j, y in enumerate(yr[0: N]): p = calc_piercepoint(np.dot(np.array([x, y]), np.array([east, north])), up, height) d2 = np.sum(np.square(pp[k, :, :] - p[0]), axis=1) c = -(d2 / ( r_0**2 ) )**( beta_val / 2.0 ) / 2.0 screen[j, i, k] = np.dot(c, f.reshape(N_piercepoints)) # Fit and remove a gradient if remove_gradient: xs, ys = np.indices(screen.shape[0:2]) zs = screen[:, :, k] XYZ = [] for xf, yf in zip(xs.flatten().tolist(), ys.flatten().tolist()): XYZ.append([xf, yf, zs[xf, yf]]) XYZ = np.array(XYZ) a, b, c = fitPLaneLTSQ(XYZ) grad_plane = a * xs + b * ys + c screen[:, :, k] = screen[:, :, k] - grad_plane for t in range(fitted_tec1.shape[0]): xs_pt = np.where(np.array(xr) > pp1[t, 0])[0][0] ys_pt = np.where(np.array(yr) > pp1[t, 1])[0][0] grad_plane_pt = a * ys_pt + b * xs_pt + c fitted_tec1[t, k] = fitted_tec1[t, k] - grad_plane_pt pbar.update(ipbar) ipbar += 1 pbar.finish() vmin = np.min([np.amin(screen), np.amin(fitted_tec1)]) vmax = np.max([np.amax(screen), np.amax(fitted_tec1)]) logging.info('Plotting TEC screens...') fig1 = pylab.figure(figsize = (7, 7)) pbar = progressbar.ProgressBar(maxval=N_times).start() ipbar = 0 for k in range(N_times): pylab.clf() im = pylab.imshow(screen[:, :, k], cmap = pylab.cm.jet, origin = 'lower', interpolation = 'nearest', extent = (xr[0], xr[-1], yr[0], yr[-1]), vmin = vmin, vmax = vmax) sm = pylab.cm.ScalarMappable(cmap = pylab.cm.jet, norm = pylab.normalize(vmin = vmin, vmax=vmax)) sm._A = [] pylab.title(str(i)) cbar = pylab.colorbar() cbar.set_label('TECU', rotation=270) x = [] y = [] s = [] c = [] for j in range(fitted_tec1.shape[0]): x.append(pp1[j,0]) y.append(pp1[j,1]) xs = np.where(np.array(xr) > pp1[j,0])[0][0] ys = np.where(np.array(yr) > pp1[j,1])[0][0] s.append(max(2400*abs(fitted_tec1[j, k] - screen[ys, xs, k]), 10)) c.append(sm.to_rgba(fitted_tec1[j, k])) pylab.scatter(x, y, s=s, c=c) labels = source_names for label, xl, yl in zip(labels, x[0::N_stations], y[0::N_stations]): pylab.annotate( label, xy = (xl, yl), xytext = (-20, 20), textcoords = 'offset points', ha = 'right', va = 'bottom', bbox = dict(boxstyle = 'round,pad=0.5', fc = 'gray', alpha = 0.5), arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0')) pylab.title('Time {0}'.format(k)) pylab.xlim(xr[-1], xr[0]) pylab.ylim(yr[0], yr[-1]) pylab.xlabel('Projected Distance (m)') pylab.ylabel('Projected Distance (m)') pylab.savefig(root_dir+'/'+prestr+'frame%0.3i.png' % k) pbar.update(ipbar) ipbar += 1 pbar.finish() pylab.close(fig1)
def fit_tec_per_source_pair(phases, flags, mask, freqs, init_sols=None, init_sols_per_pair=False, propagate=False, nband_min=2): """Fits TEC values to phase solutions per source pair Returns TEC solutions as array of shape (N_sources, N_times, N_stations) Keyword arguments: phases -- array of phase solutions flags -- phase solution flags (0 = use, 1 = flagged) mask -- mask for sources and frequencies (0 = ignore, 1 = use) freqs -- array of frequencies init_sols -- solutions to use to initialize the fits init_sols_per_pair -- init_sols are per source pair (not per source) propagate -- propagate solutions from previous solution nband_min -- min number of bands for a source to be used """ from pylab import pinv, newaxis, find import numpy as np from lofar.expion import baselinefitting try: import progressbar except ImportError: import losoto.progressbar as progressbar sols_list = [] eq_list = [] vars_list = [] var_eq_list = [] N_sources = phases.shape[0] N_stations = phases.shape[1] N_times = phases.shape[3] N_pairs = 0 for i in xrange(N_sources): for j in xrange(i): subband_selection = find(mask[i, :] * mask[j, :]) if len(subband_selection) < nband_min: continue N_pairs += 1 if init_sols is None and not init_sols_per_pair: init_sols = np.zeros((N_sources, N_times, N_stations), dtype = np.float) elif init_sols is None and init_sols_per_pair: init_sols = np.zeros((N_pairs, N_times, N_stations), dtype = np.float) source_pairs = [] k = 0 ipbar = 0 logging.info('Fitting TEC values...') pbar = progressbar.ProgressBar(maxval=N_pairs*N_times).start() for i in xrange(N_sources): for j in xrange(i): subband_selection = find(mask[i, :] * mask[j, :]) if len(subband_selection) < nband_min: continue source_pairs.append((i, j)) p = phases[i, :, subband_selection, :] - phases[j, :, subband_selection, :] A = np.zeros((len(subband_selection), 1)) A[:, 0] = 8.44797245e9/freqs[subband_selection] flags_source_pair = flags[i, :, subband_selection, :] * flags[j, :, subband_selection, :] constant_parms = np.zeros((1, N_stations), dtype = np.bool) sols = np.zeros((N_times, N_stations), dtype = np.float) if init_sols_per_pair: p_0 = init_sols[k, 0, :][newaxis, :] else: p_0 = (init_sols[i, 0, :] - init_sols[j, 0, :])[newaxis, :] for t_idx in xrange(N_times): x = p[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() if not propagate: if init_sols_per_pair: p_0 = init_sols[k, t_idx, :][newaxis, :] else: p_0 = (init_sols[i, t_idx, :] - init_sols[j, 0, :])[newaxis, :] sol = baselinefitting.fit(x, A, p_0, f, constant_parms) if propagate: p_0 = sol.copy() sols[t_idx, :] = sol[0, :] ipbar += 1 pbar.update(ipbar) sols = sols[:, :] - np.mean(sols[:, :], axis=1)[:, newaxis] weight = len(subband_selection) sols_list.append(sols*weight) eq = np.zeros(N_sources) eq[i] = weight eq[j] = -weight eq_list.append(eq) k += 1 pbar.finish() sols = np.array(sols_list) B = np.array(eq_list) source_selection = find(np.sum(abs(B), axis=0)) N_sources = len(source_selection) if N_sources == 0: logging.error('All sources have fewer than the required minimum number of bands.') return None, None pinvB = pinv(B) r = np.dot(pinvB, sols.transpose([1, 0, 2])) r = r[source_selection, :, :] return r, source_selection
def getControlMaps(state_r, state_l, dataset, conf=None, indices=None): """ This function creates the linear mappings that are required for creating a controlled SLIP: * parameter prediction schemes These are mappings from [CoM state, additional states] -> [SLIP parameters], which tell the system how to update the SLIP parameters each step * Non-SLIP state propagators These are mappings from [CoM state, additional states] -> [additional states], which tell the system how the additional states (not part of original SLIP) evolve from apex to apex (discrete dynamics). As these mappings are affine mappings, the corresponding lifts (constants) are also given. This implies that a periodic SLIP solution is computed. This function uses bootstrap. *NOTE* Do *not* detrend the input - use "original" data! *NOTE* It is assumed that the first 3 dimensions of the data contain the CoM state at apex! :args: state_r (n-by-d array): the full state of the system at right apices. First three dimensions are considered as state of the CoM [height, velocity in horizontal plane (2x)] state_l (n-by-d array): the full state of the system at left apices. *NOTE* it is assumed that a left apex follows a right apex, i.e. state_r[0, :] is before state_l[0, :] dataset: a dataset obtained from build_dataset(). Essentially, this is an object with .all_param_r [n-by-5 array] .all_param_l [n-by-5 array] .yminL [list, n elements] .yminR [list, n elements] .TR [list, n elements] .TL [list, n elements] .masses [list, n elements] conf: a mutils.io.saveable object, containing detrending info (int .dt_window and bool .dt_medfilter fields) indices (list or 1d array of int): indices to use for regression. If NONE, use bootstrap and average over matrices. :returns: (ctrl_r, ctrl_l), (prop_r, prop_l), (ref_state_r, ref_state_l, ref_param_r, ref_param_l, ref_addDict) Tuples containing the (1) parameter update maps, (2) state propagator maps, (3) reference states and parameters and additional SLIP """ if conf == None: conf = mio.saveable() conf.dt_window = 30 conf.dt_medfilter = False if indices is not None: indices = array(indices).squeeze() d = dataset # shortcut addDict = {'m': mean(d.masses), 'g': -9.81} #def getPeriodicOrbit2(ICr, Tr, yminr, ICl, Tl, yminl, m, startParams=[14000., #[ICp_r, Pp_r, dE_r], [ICp_l, Pp_l, dE_l] = ICp_r = mean(d.all_IC_r, axis=0) ICp_l = mean(d.all_IC_l, axis=0) TR = mean(vstack(d.TR)) TL = mean(vstack(d.TL)) yminr = mean(vstack(d.yminR)) yminl = mean(vstack(d.yminL)) Pp_r, Pp_l = getPeriodicOrbit2(ICp_r, TR, yminr, ICp_l, TL, yminl, mean(d.masses), startParams=mean(vstack(d.all_param_r), axis=0)[:5]) # change last element to be energy input - this is consistent with the format used in the rest of the code if False: # obsolete code Pp_r = array(Pp_r) Pp_l = array(Pp_l) Pp_r[4] = dE_r Pp_r = Pp_r[:5] Pp_l[4] = dE_l Pp_l = Pp_l[:5] # now: detrend data # non-SLIP state dt_nss_r = fda.dt_movingavg(state_r[:, 3:], conf.dt_window, conf.dt_medfilter) dt_nss_l = fda.dt_movingavg(state_l[:, 3:], conf.dt_window, conf.dt_medfilter) # full state dt_fs_r = fda.dt_movingavg(state_r, conf.dt_window, conf.dt_medfilter) dt_fs_l = fda.dt_movingavg(state_l, conf.dt_window, conf.dt_medfilter) # SLIP parameters dt_pl = fda.dt_movingavg(d.all_param_l, conf.dt_window, conf.dt_medfilter) dt_pr = fda.dt_movingavg(d.all_param_r, conf.dt_window, conf.dt_medfilter) # compute non-SLIP state prediction maps (propagators) if indices is None: _, all_prop_r, _ = fda.fitData(dt_fs_r, dt_nss_l, nps=1, nrep=100, sections=[ 0, ], rcond=1e-8) _, all_prop_l, _ = fda.fitData(dt_fs_l[:-1, :], dt_nss_r[1:, :], nps=1, nrep=100, sections=[ 0, ], rcond=1e-8) prop_r = fda.meanMat(all_prop_r) prop_l = fda.meanMat(all_prop_l) else: prop_r = dot(dt_nss_l[indices, :].T, pinv(dt_fs_r[indices, :].T, rcond=1e-8)) prop_l = dot(dt_nss_r[indices[:-1] + 1, :].T, pinv(dt_fs_l[indices[:-1], :].T, rcond=1e-8)) # compute parameter prediction maps if indices is None: _, all_ctrl_r, _ = fda.fitData(dt_fs_r, dt_pr, nps=1, nrep=100, sections=[ 0, ], rcond=1e-8) _, all_ctrl_l, _ = fda.fitData(dt_fs_l, dt_pl, nps=1, nrep=100, sections=[ 0, ], rcond=1e-8) ctrl_r = fda.meanMat(all_ctrl_r) ctrl_l = fda.meanMat(all_ctrl_l) else: ctrl_r = dot(dt_pr[indices, :].T, pinv(dt_fs_r[indices, :].T, rcond=1e-8)) ctrl_l = dot(dt_pl[indices, :].T, pinv(dt_fs_l[indices, :].T, rcond=1e-8)) return (ctrl_r, ctrl_l), (prop_r, prop_l), (ICp_r, ICp_l, Pp_r, Pp_l, addDict)
def getControlMaps(state_r, state_l, dataset, conf=None, indices=None): """ This function creates the linear mappings that are required for creating a controlled SLIP: * parameter prediction schemes These are mappings from [CoM state, additional states] -> [SLIP parameters], which tell the system how to update the SLIP parameters each step * Non-SLIP state propagators These are mappings from [CoM state, additional states] -> [additional states], which tell the system how the additional states (not part of original SLIP) evolve from apex to apex (discrete dynamics). As these mappings are affine mappings, the corresponding lifts (constants) are also given. This implies that a periodic SLIP solution is computed. This function uses bootstrap. *NOTE* Do *not* detrend the input - use "original" data! *NOTE* It is assumed that the first 3 dimensions of the data contain the CoM state at apex! :args: state_r (n-by-d array): the full state of the system at right apices. First three dimensions are considered as state of the CoM [height, velocity in horizontal plane (2x)] state_l (n-by-d array): the full state of the system at left apices. *NOTE* it is assumed that a left apex follows a right apex, i.e. state_r[0, :] is before state_l[0, :] dataset: a dataset obtained from build_dataset(). Essentially, this is an object with .all_param_r [n-by-5 array] .all_param_l [n-by-5 array] .yminL [list, n elements] .yminR [list, n elements] .TR [list, n elements] .TL [list, n elements] .masses [list, n elements] conf: a mutils.io.saveable object, containing detrending info (int .dt_window and bool .dt_medfilter fields) indices (list or 1d array of int): indices to use for regression. If NONE, use bootstrap and average over matrices. :returns: (ctrl_r, ctrl_l), (prop_r, prop_l), (ref_state_r, ref_state_l, ref_param_r, ref_param_l, ref_addDict) Tuples containing the (1) parameter update maps, (2) state propagator maps, (3) reference states and parameters and additional SLIP """ if conf == None: conf = mio.saveable() conf.dt_window=30 conf.dt_medfilter=False if indices is not None: indices = array(indices).squeeze() d = dataset # shortcut addDict = { 'm' : mean(d.masses), 'g' : -9.81 } #def getPeriodicOrbit2(ICr, Tr, yminr, ICl, Tl, yminl, m, startParams=[14000., #[ICp_r, Pp_r, dE_r], [ICp_l, Pp_l, dE_l] = ICp_r = mean(d.all_IC_r, axis=0) ICp_l = mean(d.all_IC_l, axis=0) TR = mean(vstack(d.TR)) TL = mean(vstack(d.TL)) yminr = mean(vstack(d.yminR)) yminl = mean(vstack(d.yminL)) Pp_r, Pp_l = getPeriodicOrbit2(ICp_r, TR, yminr, ICp_l, TL, yminl, mean(d.masses), startParams=mean(vstack(d.all_param_r), axis=0)[:5]) # change last element to be energy input - this is consistent with the format used in the rest of the code if False: # obsolete code Pp_r = array(Pp_r) Pp_l = array(Pp_l) Pp_r[4] = dE_r Pp_r = Pp_r[:5] Pp_l[4] = dE_l Pp_l = Pp_l[:5] # now: detrend data # non-SLIP state dt_nss_r = fda.dt_movingavg(state_r[:, 3:], conf.dt_window, conf.dt_medfilter) dt_nss_l = fda.dt_movingavg(state_l[:, 3:],conf.dt_window, conf.dt_medfilter) # full state dt_fs_r = fda.dt_movingavg(state_r, conf.dt_window, conf.dt_medfilter) dt_fs_l = fda.dt_movingavg(state_l, conf.dt_window, conf.dt_medfilter) # SLIP parameters dt_pl = fda.dt_movingavg(d.all_param_l, conf.dt_window, conf.dt_medfilter) dt_pr = fda.dt_movingavg(d.all_param_r, conf.dt_window, conf.dt_medfilter) # compute non-SLIP state prediction maps (propagators) if indices is None: _, all_prop_r, _ = fda.fitData(dt_fs_r, dt_nss_l, nps=1, nrep=100, sections=[0,], rcond=1e-8) _, all_prop_l, _ = fda.fitData(dt_fs_l[:-1, :], dt_nss_r[1:, :], nps=1, nrep=100, sections=[0,], rcond=1e-8) prop_r = fda.meanMat(all_prop_r) prop_l = fda.meanMat(all_prop_l) else: prop_r = dot(dt_nss_l[indices, :].T, pinv(dt_fs_r[indices, :].T, rcond=1e-8)) prop_l = dot(dt_nss_r[indices[:-1] + 1, :].T, pinv(dt_fs_l[indices[:-1], :].T, rcond=1e-8)) # compute parameter prediction maps if indices is None: _, all_ctrl_r, _ = fda.fitData(dt_fs_r, dt_pr, nps=1, nrep=100, sections=[0,], rcond=1e-8) _, all_ctrl_l, _ = fda.fitData(dt_fs_l, dt_pl, nps=1, nrep=100, sections=[0,], rcond=1e-8) ctrl_r = fda.meanMat(all_ctrl_r) ctrl_l = fda.meanMat(all_ctrl_l) else: ctrl_r = dot(dt_pr[indices, :].T, pinv(dt_fs_r[indices, :].T, rcond=1e-8)) ctrl_l = dot(dt_pl[indices, :].T, pinv(dt_fs_l[indices, :].T, rcond=1e-8)) return (ctrl_r, ctrl_l), (prop_r, prop_l), (ICp_r, ICp_l, Pp_r, Pp_l, addDict)
def fit_screen_to_tec(station_names, source_names, pp, airmass, rr, times, height, order, r_0, beta): """ Fits a screen to given TEC values using Karhunen-Lo`eve base vectors Keyword arguments: station_names -- array of station names source_names -- array of source names pp -- array of piercepoint locations airmass -- array of airmass values rr -- array of TEC solutions times -- array of times height -- height of screen (m) order -- order of screen (i.e., number of KL base vectors to keep) r_0 -- scale size of phase fluctuations (m) beta -- power-law index for phase structure function (5/3 => pure Kolmogorov turbulence) """ import numpy as np from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye try: import progressbar except ImportError: import losoto.progressbar as progressbar logging.info('Fitting screens to TEC values...') N_stations = len(station_names) N_sources = len(source_names) N_times = len(times) tec_fit_all = np.zeros((N_times, N_sources, N_stations)) residual_all = np.zeros((N_times, N_sources, N_stations)) A = concatenate([kron(eye(N_sources), np.ones((N_stations, 1))), kron(np.ones((N_sources, 1)), eye(N_stations))], axis=1) N_piercepoints = N_sources * N_stations P = eye(N_piercepoints) - np.dot(np.dot(A, pinv(np.dot(A.T, A))), A.T) pbar = progressbar.ProgressBar(maxval=N_times).start() ipbar = 0 for k in range(N_times): try: D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3)) D = np.transpose(D, (1, 0, 2)) - D D2 = np.sum(D**2, axis=2) C = -(D2 / r_0**2)**(beta / 2.0) / 2.0 P1 = eye(N_piercepoints) - np.ones((N_piercepoints, N_piercepoints)) / N_piercepoints C1 = np.dot(np.dot(P1, C), P1) U, S, V = svd(C1) B = np.dot(P, np.dot(np.diag(airmass[k, :]), U[:, :order])) pinvB = pinv(B, rcond=1e-3) rr1 = np.dot(P, rr[:, k]) tec_fit = np.dot(U[:, :order], np.dot(pinvB, rr1)) tec_fit_all[k, :, :] = tec_fit.reshape((N_sources, N_stations)) residual = rr1 - np.dot(P, tec_fit) residual_all[k, :, :] = residual.reshape((N_sources, N_stations)) except: # Set screen to zero if fit did not work logging.debug('Tecscreen fit failed for timeslot {0}'.format(k)) tec_fit_all[k, :, :] = np.zeros((N_sources, N_stations)) residual_all[k, :, :] = np.ones((N_sources, N_stations)) pbar.update(ipbar) ipbar += 1 pbar.finish() return tec_fit_all, residual_all
known_columns.append(c) dG0_f[c, 0] = temp except Exception: continue unknown_columns = sorted(list(set(range(len(all_cids))).difference(known_columns))) # find all the indices of columns not in known_columns unknown_rows = find(isnan(dG0_r)) known_rows = sorted(list(set(range(len(reactions))).difference(unknown_rows))) # find all the indices of rows with measured dG0_r S_measured = S[known_rows, :] b = dG0_r[known_rows] - dot(S_measured[:, known_columns], dG0_f[known_columns]) S_red = S_measured[:, unknown_columns] print "Formation energies from from GC and from linear regression: " # linear regression to solve the missing formation energies (S*x = b) inv_corr_mat = pinv(dot(S_red.T, S_red)) x = dot(dot(inv_corr_mat, S_red.T), b) for c in xrange(len(unknown_columns)): dG0_f[unknown_columns[c], 0] = x[c, 0] csv_out = csv.writer(open('../res/acetogens_compounds.csv', 'w')) for c in xrange(len(all_cids)): if (c in known_columns): csv_out.writerow(['C%05d' % all_cids[c], '%8.1f' % dG0_f[c, 0], 'Group Contribution']) else: csv_out.writerow(['C%05d' % all_cids[c], '%8.1f' % dG0_f[c, 0], 'Calculated']) #figure() #plot(b, dot(S_red, dG0_f[unknown_columns]), '.') figure() plot(dG0_r[known_rows], dot(S[known_rows, :], dG0_f), '.')
def make_tec_screen_plots(pp, tec_screen, residuals, station_positions, source_names, times, height, order, beta_val, r_0, prefix='frame_', remove_gradient=True, show_source_names=False, min_tec=None, max_tec=None): """Makes plots of TEC screens Keyword arguments: pp -- array of piercepoint locations tec_screen -- array of TEC screen values at the piercepoints residuals -- array of TEC screen residuals at the piercepoints source_names -- array of source names times -- array of times height -- height of screen (m) order -- order of screen (e.g., number of KL base vectors to keep) r_0 -- scale size of phase fluctuations (m) beta_val -- power-law index for phase structure function (5/3 => pure Kolmogorov turbulence) prefix -- prefix for output file names remove_gradient -- fit and remove a gradient from each screen show_source_names -- label sources on screen plots min_tec -- minimum TEC value for plot range max_tec -- maximum TEC value for plot range """ from pylab import kron, concatenate, pinv, norm, newaxis, normalize import matplotlib.pyplot as plt from mpl_toolkits.axes_grid1.inset_locator import inset_axes import numpy as np import os from operations.tecscreen import calc_piercepoint import progressbar root_dir = os.path.dirname(prefix) if root_dir == '': root_dir = './' prestr = os.path.basename(prefix) + 'screen_' try: os.makedirs(root_dir) except OSError: pass N_stations = station_positions.shape[0] N_sources = len(source_names) N_times = len(times) A = concatenate([ kron(np.eye(N_sources), np.ones((N_stations, 1))), kron(np.ones((N_sources, 1)), np.eye(N_stations)) ], axis=1) N_piercepoints = N_sources * N_stations P = np.eye(N_piercepoints) - np.dot(np.dot(A, pinv(np.dot(A.T, A))), A.T) x, y, z = station_positions[0, :] east = np.array([-y, x, 0]) east = east / norm(east) north = np.array([-x, -y, (x * x + y * y) / z]) north = north / norm(north) up = np.array([x, y, z]) up = up / norm(up) T = concatenate([east[:, newaxis], north[:, newaxis]], axis=1) pp1 = np.dot(pp[0, :, :], T) lower = np.amin(pp1, axis=0) upper = np.amax(pp1, axis=0) extent = upper - lower lower = lower - 0.05 * extent upper = upper + 0.05 * extent extent = upper - lower Nx = 25 Ny = int(extent[1] / extent[0] * np.float(Nx)) xr = np.arange(lower[0], upper[0], extent[0] / Nx) yr = np.arange(lower[1], upper[1], extent[1] / Ny) screen = np.zeros((Nx, Ny, N_times)) gradient = np.zeros((Nx, Ny, N_times)) residuals = residuals.transpose([0, 2, 1]).reshape(N_piercepoints, N_times) fitted_tec1 = tec_screen.transpose([0, 2, 1]).reshape( N_piercepoints, N_times) + residuals logging.info('Calculating TEC screen images...') pbar = progressbar.ProgressBar(maxval=N_times).start() ipbar = 0 for k in range(N_times): D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3)) D = np.transpose(D, (1, 0, 2)) - D D2 = np.sum(D**2, axis=2) C = -(D2 / r_0**2)**(beta_val / 2.0) / 2.0 f = np.dot(pinv(C), tec_screen[:, k, :].reshape(N_piercepoints)) for i, x in enumerate(xr[0:Nx]): for j, y in enumerate(yr[0:Ny]): p = calc_piercepoint( np.dot(np.array([x, y]), np.array([east, north])), up, height) d2 = np.sum(np.square(pp[k, :, :] - p[0]), axis=1) c = -(d2 / (r_0**2))**(beta_val / 2.0) / 2.0 screen[i, j, k] = np.dot(c, f) # Fit and remove a gradient. # Plot gradient in lower-left corner with its own color bar? if remove_gradient: xs, ys = np.indices(screen.shape[0:2]) zs = screen[:, :, k] XYZ = [] for xf, yf in zip(xs.flatten().tolist(), ys.flatten().tolist()): XYZ.append([xf, yf, zs[xf, yf]]) XYZ = np.array(XYZ) a, b, c = fitPLaneLTSQ(XYZ) grad_plane = a * xs + b * ys + c gradient[:, :, k] = grad_plane screen[:, :, k] = screen[:, :, k] - grad_plane screen[:, :, k] = screen[:, :, k] - np.mean(screen[:, :, k]) for t in range(fitted_tec1.shape[0]): xs_pt = np.where(np.array(xr) > pp1[t, 0])[0][0] ys_pt = np.where(np.array(yr) > pp1[t, 1])[0][0] grad_plane_pt = a * xs_pt + b * ys_pt + c fitted_tec1[t, k] = fitted_tec1[t, k] - grad_plane_pt fitted_tec1[:, k] = fitted_tec1[:, k] - np.mean(fitted_tec1[:, k]) pbar.update(ipbar) ipbar += 1 pbar.finish() if min_tec is None: vmin = np.min([np.amin(screen), np.amin(fitted_tec1)]) else: vmin = min_tec if max_tec is None: vmax = np.max([np.amax(screen), np.amax(fitted_tec1)]) else: vmax = max_tec logging.info('Plotting TEC screens...') fig, ax = plt.subplots(figsize=[7, 7]) pbar = progressbar.ProgressBar(maxval=N_times).start() ipbar = 0 for k in range(N_times): plt.clf() im = plt.imshow(screen.transpose([1, 0, 2])[:, :, k], cmap=plt.cm.jet, origin='lower', interpolation='nearest', extent=(xr[0] / 1000.0, xr[-1] / 1000.0, yr[0] / 1000.0, yr[-1] / 1000.0), vmin=vmin, vmax=vmax) sm = plt.cm.ScalarMappable(cmap=plt.cm.jet, norm=normalize(vmin=vmin, vmax=vmax)) sm._A = [] cbar = plt.colorbar(im) cbar.set_label('TECU', rotation=270) x = [] y = [] s = [] c = [] for j in range(fitted_tec1.shape[0]): x.append(pp1[j, 0] / 1000.0) y.append(pp1[j, 1] / 1000.0) xs = np.where(np.array(xr) > pp1[j, 0])[0][0] ys = np.where(np.array(yr) > pp1[j, 1])[0][0] fit_screen_diff = abs(fitted_tec1[j, k] - screen[xs, ys, k]) s.append(max(20 * fit_screen_diff / 0.01, 10)) c.append(sm.to_rgba(fitted_tec1[j, k])) plt.scatter(x, y, s=s, c=c) if show_source_names: labels = source_names for label, xl, yl in zip(labels, x[0::N_stations], y[0::N_stations]): plt.annotate(label, xy=(xl, yl), xytext=(-2, 2), textcoords='offset points', ha='right', va='bottom') plt.title('Screen {0}'.format(k)) plt.xlim(xr[-1] / 1000.0, xr[0] / 1000.0) plt.ylim(yr[0] / 1000.0, yr[-1] / 1000.0) plt.xlabel('Projected Distance along RA (km)') plt.ylabel('Projected Distance along Dec (km)') if remove_gradient: axins = inset_axes(ax, width="15%", height="10%", loc=2) axins.imshow(gradient.transpose([1, 0, 2])[:, :, k], cmap=plt.cm.jet, origin='lower', interpolation='nearest', extent=(xr[0] / 1000.0, xr[-1] / 1000.0, yr[0] / 1000.0, yr[-1] / 1000.0), vmin=vmin, vmax=vmax) plt.xticks(visible=False) plt.yticks(visible=False) axins.set_xlim(xr[-1] / 1000.0, xr[0] / 1000.0) axins.set_ylim(yr[0] / 1000.0, yr[-1] / 1000.0) plt.savefig(root_dir + '/' + prestr + 'frame%0.3i.png' % k) pbar.update(ipbar) ipbar += 1 pbar.finish() plt.close(fig)
def make_tec_screen_plots(pp, tec_screen, residuals, station_positions, source_names, times, height, order, beta_val, r_0, prefix = 'frame_', remove_gradient=True, show_source_names=False, min_tec=None, max_tec=None): """Makes plots of TEC screens Keyword arguments: pp -- array of piercepoint locations tec_screen -- array of TEC screen values at the piercepoints residuals -- array of TEC screen residuals at the piercepoints source_names -- array of source names times -- array of times height -- height of screen (m) order -- order of screen (e.g., number of KL base vectors to keep) r_0 -- scale size of phase fluctuations (m) beta_val -- power-law index for phase structure function (5/3 => pure Kolmogorov turbulence) prefix -- prefix for output file names remove_gradient -- fit and remove a gradient from each screen show_source_names -- label sources on screen plots min_tec -- minimum TEC value for plot range max_tec -- maximum TEC value for plot range """ from pylab import kron, concatenate, pinv, norm, newaxis, normalize import matplotlib.pyplot as plt from mpl_toolkits.axes_grid1.inset_locator import inset_axes import numpy as np import os from operations.tecscreen import calc_piercepoint import progressbar root_dir = os.path.dirname(prefix) if root_dir == '': root_dir = './' prestr = os.path.basename(prefix) + 'screen_' try: os.makedirs(root_dir) except OSError: pass N_stations = station_positions.shape[0] N_sources = len(source_names) N_times = len(times) A = concatenate([kron(np.eye(N_sources), np.ones((N_stations,1))), kron(np.ones((N_sources,1)), np.eye(N_stations))], axis=1) N_piercepoints = N_sources * N_stations P = np.eye(N_piercepoints) - np.dot(np.dot(A, pinv(np.dot(A.T, A))), A.T) x, y, z = station_positions[0, :] east = np.array([-y, x, 0]) east = east / norm(east) north = np.array([ -x, -y, (x*x + y*y)/z]) north = north / norm(north) up = np.array([x ,y, z]) up = up / norm(up) T = concatenate([east[:, newaxis], north[:, newaxis]], axis=1) pp1 = np.dot(pp[0, :, :], T) lower = np.amin(pp1, axis=0) upper = np.amax(pp1, axis=0) extent = upper - lower lower = lower - 0.05 * extent upper = upper + 0.05 * extent extent = upper - lower Nx = 25 Ny = int(extent[1] / extent[0] * np.float(Nx)) xr = np.arange(lower[0], upper[0], extent[0]/Nx) yr = np.arange(lower[1], upper[1], extent[1]/Ny) screen = np.zeros((Nx, Ny, N_times)) gradient = np.zeros((Nx, Ny, N_times)) residuals = residuals.transpose([0, 2, 1]).reshape(N_piercepoints, N_times) fitted_tec1 = tec_screen.transpose([0, 2, 1]).reshape(N_piercepoints, N_times) + residuals logging.info('Calculating TEC screen images...') pbar = progressbar.ProgressBar(maxval=N_times).start() ipbar = 0 for k in range(N_times): D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3)) D = np.transpose(D, (1, 0, 2)) - D D2 = np.sum(D**2, axis=2) C = -(D2 / r_0**2)**(beta_val / 2.0) / 2.0 f = np.dot(pinv(C), tec_screen[:, k, :].reshape(N_piercepoints)) for i, x in enumerate(xr[0: Nx]): for j, y in enumerate(yr[0: Ny]): p = calc_piercepoint(np.dot(np.array([x, y]), np.array([east, north])), up, height) d2 = np.sum(np.square(pp[k, :, :] - p[0]), axis=1) c = -(d2 / ( r_0**2 ))**(beta_val / 2.0) / 2.0 screen[i, j, k] = np.dot(c, f) # Fit and remove a gradient. # Plot gradient in lower-left corner with its own color bar? if remove_gradient: xs, ys = np.indices(screen.shape[0:2]) zs = screen[:, :, k] XYZ = [] for xf, yf in zip(xs.flatten().tolist(), ys.flatten().tolist()): XYZ.append([xf, yf, zs[xf, yf]]) XYZ = np.array(XYZ) a, b, c = fitPLaneLTSQ(XYZ) grad_plane = a * xs + b * ys + c gradient[:, :, k] = grad_plane screen[:, :, k] = screen[:, :, k] - grad_plane screen[:, :, k] = screen[:, :, k] - np.mean(screen[:, :, k]) for t in range(fitted_tec1.shape[0]): xs_pt = np.where(np.array(xr) > pp1[t, 0])[0][0] ys_pt = np.where(np.array(yr) > pp1[t, 1])[0][0] grad_plane_pt = a * xs_pt + b * ys_pt + c fitted_tec1[t, k] = fitted_tec1[t, k] - grad_plane_pt fitted_tec1[:, k] = fitted_tec1[:, k] - np.mean(fitted_tec1[:, k]) pbar.update(ipbar) ipbar += 1 pbar.finish() if min_tec is None: vmin = np.min([np.amin(screen), np.amin(fitted_tec1)]) else: vmin = min_tec if max_tec is None: vmax = np.max([np.amax(screen), np.amax(fitted_tec1)]) else: vmax = max_tec logging.info('Plotting TEC screens...') fig, ax = plt.subplots(figsize=[7, 7]) pbar = progressbar.ProgressBar(maxval=N_times).start() ipbar = 0 for k in range(N_times): plt.clf() im = plt.imshow(screen.transpose([1, 0, 2])[:, :, k], cmap = plt.cm.jet, origin = 'lower', interpolation = 'nearest', extent = (xr[0]/1000.0, xr[-1]/1000.0, yr[0]/1000.0, yr[-1]/1000.0), vmin=vmin, vmax=vmax) sm = plt.cm.ScalarMappable(cmap=plt.cm.jet, norm=normalize(vmin=vmin, vmax=vmax)) sm._A = [] cbar = plt.colorbar(im) cbar.set_label('TECU', rotation=270) x = [] y = [] s = [] c = [] for j in range(fitted_tec1.shape[0]): x.append(pp1[j, 0] / 1000.0) y.append(pp1[j, 1] / 1000.0) xs = np.where(np.array(xr) > pp1[j, 0])[0][0] ys = np.where(np.array(yr) > pp1[j, 1])[0][0] fit_screen_diff = abs(fitted_tec1[j, k] - screen[xs, ys, k]) s.append(max(20*fit_screen_diff/0.01, 10)) c.append(sm.to_rgba(fitted_tec1[j, k])) plt.scatter(x, y, s=s, c=c) if show_source_names: labels = source_names for label, xl, yl in zip(labels, x[0::N_stations], y[0::N_stations]): plt.annotate( label, xy = (xl, yl), xytext = (-2, 2), textcoords = 'offset points', ha = 'right', va = 'bottom') plt.title('Screen {0}'.format(k)) plt.xlim(xr[-1]/1000.0, xr[0]/1000.0) plt.ylim(yr[0]/1000.0, yr[-1]/1000.0) plt.xlabel('Projected Distance along RA (km)') plt.ylabel('Projected Distance along Dec (km)') if remove_gradient: axins = inset_axes(ax, width="15%", height="10%", loc=2) axins.imshow(gradient.transpose([1, 0, 2])[:, : ,k], cmap = plt.cm.jet, origin = 'lower', interpolation = 'nearest', extent = (xr[0]/1000.0, xr[-1]/1000.0, yr[0]/1000.0, yr[-1]/1000.0), vmin=vmin, vmax=vmax) plt.xticks(visible=False) plt.yticks(visible=False) axins.set_xlim(xr[-1]/1000.0, xr[0]/1000.0) axins.set_ylim(yr[0]/1000.0, yr[-1]/1000.0) plt.savefig(root_dir+'/'+prestr+'frame%0.3i.png' % k) pbar.update(ipbar) ipbar += 1 pbar.finish() plt.close(fig)
def _fit_phase_screen(station_names, source_names, pp, airmass, rr, weights, times, height, order, r_0, beta, outQueue): """ Fits a screen to given phase values using Karhunen-Lo`eve base vectors Parameters ---------- station_names: array Array of station names source_names: array Array of source names pp: array Array of piercepoint locations airmass: array Array of airmass values (note: not currently used) rr: array Array of phase values to fit screen to weights: array Array of weights times: array Array of times height: float Height of screen (m) order: int Order of screen (i.e., number of KL base vectors to keep) r_0: float Scale size of phase fluctuations (m) beta: float Power-law index for phase structure function (5/3 => pure Kolmogorov turbulence) """ import numpy as np from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye logging.info('Fitting screens...') # Initialize arrays N_stations = len(station_names) N_sources = len(source_names) N_times = len(times) N_piercepoints = N_sources * N_stations real_fit_white_all = np.zeros((N_times, N_sources, N_stations)) imag_fit_white_all = np.zeros((N_times, N_sources, N_stations)) phase_fit_white_all = np.zeros((N_times, N_sources, N_stations)) real_residual_all = np.zeros((N_times, N_sources, N_stations)) imag_residual_all = np.zeros((N_times, N_sources, N_stations)) phase_residual_all = np.zeros((N_times, N_sources, N_stations)) # Change phase to real/imag rr_real = np.cos(rr) rr_imag = np.sin(rr) for k in range(N_times): try: D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3)) D = np.transpose(D, (1, 0, 2)) - D D2 = np.sum(D**2, axis=2) C = -(D2 / r_0**2)**(beta / 2.0) / 2.0 pinvC = pinv(C, rcond=1e-3) U, S, V = svd(C) invU = pinv(np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], U[:, :order])), rcond=1e-3) # Calculate real screen rr1 = np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], rr_real[:, k])) real_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) real_fit_white_all[k, :, :] = real_fit.reshape( (N_sources, N_stations)) residual = rr_real - np.dot(C, real_fit)[:, newaxis] real_residual_all[k, :, :] = residual.reshape( (N_sources, N_stations)) # Calculate imag screen rr1 = np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], rr_imag[:, k])) imag_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) imag_fit_white_all[k, :, :] = imag_fit.reshape( (N_sources, N_stations)) residual = rr_imag - np.dot(C, imag_fit)[:, newaxis] imag_residual_all[k, :, :] = residual.reshape( (N_sources, N_stations)) # Calculate phase screen phase_fit = np.dot( pinvC, np.arctan2(np.dot(C, imag_fit), np.dot(C, real_fit))) phase_fit_white_all[k, :, :] = phase_fit.reshape( (N_sources, N_stations)) residual = rr - np.dot(C, phase_fit)[:, newaxis] phase_residual_all[k, :, :] = residual.reshape( (N_sources, N_stations)) except: # Set screen to zero if fit did not work logging.debug('Screen fit failed for timeslot {}'.format(k)) real_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations)) real_residual_all[k, :, :] = np.ones((N_sources, N_stations)) imag_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations)) imag_residual_all[k, :, :] = np.ones((N_sources, N_stations)) phase_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations)) phase_residual_all[k, :, :] = np.ones((N_sources, N_stations)) outQueue.put([ real_fit_white_all, real_residual_all, imag_fit_white_all, imag_residual_all, phase_fit_white_all, phase_residual_all, times ])
def makeTECparmdb(H, solset, TECsolTab, timewidths, freq, freqwidth): """Returns TEC screen parmdb parameters H - H5parm object solset - solution set with TEC screen parameters TECsolTab = solution table with tecscreen values timewidths - time widths of output parmdb freq - frequency of output parmdb freqwidth - frequency width of output parmdb """ from pylab import pinv global ipbar, pbar station_dict = H.getAnt(solset) station_names = station_dict.keys() station_positions = station_dict.values() source_dict = H.getSou(solset) source_names = source_dict.keys() source_positions = source_dict.values() tec_sf = solFetcher(TECsolTab) tec_screen, axis_vals = tec_sf.getValues() times = axis_vals['time'] beta = TECsolTab._v_attrs['beta'] r_0 = TECsolTab._v_attrs['r_0'] height = TECsolTab._v_attrs['height'] order = TECsolTab._v_attrs['order'] pp = tec_sf.t.piercepoint N_sources = len(source_names) N_times = len(times) N_freqs = 1 N_stations = len(station_names) N_piercepoints = N_sources * N_stations freqs = freq freqwidths = freqwidth parms = {} v = {} v['times'] = times v['timewidths'] = timewidths v['freqs'] = freqs v['freqwidths'] = freqwidths for station_name in station_names: for source_name in source_names: v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'TECfit_white:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() for k in range(N_times): D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3)) D = np.transpose(D, ( 1, 0, 2 )) - D D2 = np.sum(D**2, axis=2) C = -(D2 / (r_0**2))**(beta / 2.0) / 2.0 tec_fit_white = np.dot(pinv(C), tec_screen[:, k, :].reshape(N_piercepoints)) pp_idx = 0 for src, source_name in enumerate(source_names): for sta, station_name in enumerate(station_names): parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = pp[k, pp_idx, 0] parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = pp[k, pp_idx, 1] parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = pp[k, pp_idx, 2] parmname = 'TECfit_white:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx] parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx] parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx] pp_idx += 1 pbar.update(ipbar) ipbar += 1 time_start = times[0] - timewidths[0]/2 time_end = times[-1] + timewidths[-1]/2 v['times'] = np.array([(time_start + time_end) / 2]) v['timewidths'] = np.array([time_end - time_start]) v_r0 = v.copy() v_r0['values'] = np.array(r_0, dtype=np.double, ndmin=2) parms['r_0'] = v_r0 v_beta = v.copy() v_beta['values'] = np.array(beta, dtype=np.double, ndmin=2) parms['beta'] = v_beta v_height = v.copy() v_height['values'] = np.array(height, dtype=np.double, ndmin=2) parms['height'] = v_height return parms
def _fit_screen(station_names, source_names, full_matrices, pp, rr, weights, order, r_0, beta, screen_type): """ Fits a screen to amplitudes or phases using Karhunen-Lo`eve base vectors Parameters ---------- station_names: array Array of station names source_names: array Array of source names full_matrices : list of arrays List of [C, pivC, U] matrices for all piercepoints pp: array Array of piercepoint locations airmass: array Array of airmass values (note: not currently used) rr: array Array of amp values to fit screen to weights: array Array of weights order: int Order of screen (i.e., number of KL base vectors to keep) r_0: float Scale size of amp fluctuations (m) beta: float Power-law index for amp structure function (5/3 => pure Kolmogorov turbulence) screen_type : str Type of screen: 'phase' or 'amplitude' Returns ------- screen_fit_white_all, screen_residual_all : array, array Arrays of screen and residual (actual - screen) values """ import numpy as np from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye # Identify flagged directions N_sources_all = len(source_names) unflagged = np.where(weights > 0.0) N_sources = len(source_names[unflagged]) # Initialize arrays N_stations = len(station_names) N_piercepoints = N_sources * N_stations N_piercepoints_all = N_sources_all * N_stations screen_fit_all = np.zeros((N_sources_all, N_stations)) pp_all = pp.copy() rr_all = rr.copy() pp = pp_all[unflagged[0], :] w = np.diag(weights[unflagged]) # Calculate matrices if N_sources == N_sources_all: C, pinvC, U = full_matrices else: # Recalculate for unflagged directions C, pinvC, U = _calculate_svd(pp, r_0, beta, N_piercepoints) invU = pinv(np.dot(np.transpose(U[:, :order]), np.dot(w, U)[:, :order]), rcond=1e-3) # Fit screen to unflagged directions if screen_type == 'phase': # Change phase to real/imag rr_real = np.cos(rr[unflagged]) rr_imag = np.sin(rr[unflagged]) # Calculate real screen rr1 = np.dot(np.transpose(U[:, :order]), np.dot(w, rr_real)) real_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) # Calculate imag screen rr1 = np.dot(np.transpose(U[:, :order]), np.dot(w, rr_imag)) imag_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) # Calculate phase screen screen_fit = np.arctan2(np.dot(C, imag_fit), np.dot(C, real_fit)) screen_fit_white = np.dot(pinvC, screen_fit) elif screen_type == 'amplitude': # Calculate log(amp) screen rr = rr[unflagged] rr1 = np.dot(np.transpose(U[:, :order]), np.dot(w, np.log10(rr))) amp_fit_log = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) # Calculate amp screen screen_fit = 10**(np.dot(C, amp_fit_log)) screen_fit_white = np.dot(pinvC, screen_fit) elif screen_type == 'tec': # Calculate tec screen rr = rr[unflagged] rr1 = np.dot(np.transpose(U[:, :order]), np.dot(w, rr)) tec_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) # Calculate amp screen screen_fit = np.dot(C, tec_fit) screen_fit_white = np.dot(pinvC, screen_fit) # Calculate screen in all directions if N_sources != N_sources_all: screen_fit_all[unflagged[0], :] = screen_fit[:, newaxis] flagged = np.where(weights <= 0.0) for findx in flagged[0]: p = pp_all[findx, :] d2 = np.sum(np.square(pp - p), axis=1) c = -(d2 / ( r_0**2 ))**(beta / 2.0) / 2.0 screen_fit_all[findx, :] = np.dot(c, screen_fit_white) C, pinvC, U = full_matrices screen_fit_white_all = np.dot(pinvC, screen_fit_all) screen_residual_all = rr_all - screen_fit_all.reshape(N_piercepoints_all) else: screen_fit_white_all = screen_fit_white screen_residual_all = rr_all - np.dot(C, screen_fit_white) screen_fit_white_all = screen_fit_white_all.reshape((N_sources_all, N_stations)) screen_residual_all = screen_residual_all.reshape((N_sources_all, N_stations)) return (screen_fit_white_all, screen_residual_all)
continue unknown_columns = sorted( list(set(range(len(all_cids))).difference(known_columns)) ) # find all the indices of columns not in known_columns unknown_rows = find(isnan(dG0_r)) known_rows = sorted(list(set(range(len(reactions))).difference( unknown_rows))) # find all the indices of rows with measured dG0_r S_measured = S[known_rows, :] b = dG0_r[known_rows] - dot(S_measured[:, known_columns], dG0_f[known_columns]) S_red = S_measured[:, unknown_columns] print "Formation energies from from GC and from linear regression: " # linear regression to solve the missing formation energies (S*x = b) inv_corr_mat = pinv(dot(S_red.T, S_red)) x = dot(dot(inv_corr_mat, S_red.T), b) for c in xrange(len(unknown_columns)): dG0_f[unknown_columns[c], 0] = x[c, 0] csv_out = csv.writer(open('../res/acetogens_compounds.csv', 'w')) for c in xrange(len(all_cids)): if (c in known_columns): csv_out.writerow([ 'C%05d' % all_cids[c], '%8.1f' % dG0_f[c, 0], 'Group Contribution' ]) else: csv_out.writerow( ['C%05d' % all_cids[c], '%8.1f' % dG0_f[c, 0], 'Calculated'])
def add_stations(station_selection, phases0, phases1, flags, mask, station_names, station_positions, source_names, source_selection, times, freqs, r, nband_min=2, soln_type='phase', nstations_max=None, excluded_stations=None, t_step=5, tec_step1=5, tec_step2=21, search_full_tec_range=True): """ Adds stations to TEC fitting using an iterative initial-guess search to ensure the global min is found Keyword arguments: station_selection -- indices of stations to use in fitting phases0 -- XX phase solutions phases1 -- YY phase solutions flags -- phase solution flags (0 = use, 1 = flagged) mask -- mask for sources and frequencies (0 = ignore, 1 = use) station_names -- array of station names source_names -- array of source names source_selection -- indices of sources to use in fitting times -- array of times freqs -- array of frequencies r -- array of TEC solutions returned by fit_tec_per_source_pair() nband_min -- min number of bands for a source to be used soln_type -- type of phase solution: 'phase' or 'scalarphase' nstations_max -- max number of stations to use excluded_stations -- stations to exclude t_step -- try full TEC range every t_step number of solution times tec_step1 -- number of steps in TEC subrange (+/- last TEC fit value) tec_step2 -- number of steps in full TEC range (-0.1 -- 0.1) search_full_tec_range -- always search the full TEC range (-0.1 -- 0.1) """ from pylab import pinv, newaxis, find, amin import numpy as np from lofar.expion import baselinefitting try: import progressbar except ImportError: import losoto.progressbar as progressbar N_sources_selected = len(source_selection) N_stations_selected = len(station_selection) N_piercepoints = N_sources_selected * N_stations_selected N_times = len(times) N_stations = len(station_names) N_sources = len(source_names) N_pairs = 0 for ii, i in enumerate(source_selection): for jj, j in enumerate(source_selection): if j == i: break subband_selection = find(mask[i,:] * mask[j,:]) if len(subband_selection) < nband_min: continue N_pairs += 1 D = np.resize(station_positions, (N_stations, N_stations, 3)) D = np.transpose(D, (1, 0, 2)) - D D = np.sqrt(np.sum(D**2, axis=2)) station_selection1 = station_selection stations_to_add = np.array([i for i in xrange(len(station_names)) if i not in station_selection1 and station_names[i] not in excluded_stations]) if len(stations_to_add) == 0: return station_selection1, r # Check if desired number of stations is already reached if nstations_max is not None: if len(station_selection1) >= nstations_max: return station_selection1, r logging.info("Using fitting with iterative search for remaining stations " "(up to {0} stations in total)".format(nstations_max)) q = r while len(stations_to_add)>0: D1 = D[stations_to_add[:,newaxis], station_selection1[newaxis,:]] minimum_distance = amin(D1, axis=1) station_to_add = stations_to_add[np.argmin(minimum_distance)] station_selection1 = np.append(station_selection1, station_to_add) N_stations_selected1 = len(station_selection1) # Remove station from list stations_to_add = stations_to_add[stations_to_add != station_to_add] sols_list = [] eq_list = [] min_e_list = [] ipbar = 0 logging.info('Fitting TEC values with {0} included...'.format( station_names[station_to_add])) pbar = progressbar.ProgressBar(maxval=N_pairs*N_times).start() for ii, i in enumerate(source_selection): for jj, j in enumerate(source_selection): if j == i: break subband_selection = find(mask[i,:] * mask[j,:]) if len(subband_selection) < nband_min: continue logging.debug('Adding {0} for source pair: {1}-{2}'.format( station_names[station_to_add], i, j)) p0 = phases0[i, station_selection1[:,newaxis], subband_selection[newaxis,:], :] - phases0[j, station_selection1[:,newaxis], subband_selection[newaxis,:], :] p0 = p0 - np.mean(p0, axis=0)[newaxis,:,:] if soln_type != 'scalarphase': p1 = phases1[i, station_selection1[:,newaxis], subband_selection[newaxis,:], :] - phases1[j, station_selection1[:,newaxis], subband_selection[newaxis,:], :] p1 = p1 - np.mean(p1, axis=0)[newaxis, :, :] A = np.zeros((len(subband_selection), 1)) A[:, 0] = 8.44797245e9 / freqs[subband_selection] sd = (0.1 * 30e6) / freqs[subband_selection] # standard deviation of phase solutions as function of frequency (rad) flags_source_pair = flags[i, station_selection1[:,newaxis], subband_selection[newaxis,:], :] * flags[j, station_selection1[:,newaxis], subband_selection[newaxis,:], :] constant_parms = np.zeros((1, N_stations_selected1), dtype = np.bool) sols = np.zeros((N_times, N_stations_selected1), dtype = np.float) p_0_best = None for t_idx in xrange(N_times): if np.mod(t_idx, t_step) == 0: min_e = np.Inf if p_0_best is not None and not search_full_tec_range: min_tec = p_0_best[0, -1] - 0.02 max_tec = p_0_best[0, -1] + 0.02 nsteps = tec_step1 else: min_tec = -0.1 max_tec = 0.1 nsteps = tec_step2 logging.debug(' Trying initial guesses between {0} and ' '{1} TECU'.format(min_tec, max_tec)) for offset in np.linspace(min_tec, max_tec, nsteps): p_0 = np.zeros((1, N_stations_selected1), np.double) p_0[0, :N_stations_selected1-1] = (q[ii, t_idx, :] - q[jj, t_idx, :])[newaxis,:] p_0[0, -1] = offset x = p0[:,:,t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol0 -= np.mean(sol0) residual = np.mod(np.dot(A, sol0) - x.T + np.pi, 2 * np.pi) - np.pi e = np.var(residual[f.T==0]) if soln_type != 'scalarphase': x = p1[:,:,t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol1 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol1 -= np.mean(sol1) residual = np.mod(np.dot(A, sol1) - x.T + np.pi, 2 * np.pi) - np.pi residual = residual[f.T==0] e += np.var(residual) else: sol1 = sol0 if e < min_e: logging.debug(' Found new min variance of {0} ' 'with initial guess of {1} TECU'.format(e, p_0[0, -1])) min_e = e p_0_best = p_0 sols[t_idx, :] = (sol0[0, :] + sol1[0, :])/2 else: # Use previous init x = p0[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0_best, f, constant_parms) sol0 -= np.mean(sol0) if soln_type != 'scalarphase': x = p1[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol1 = baselinefitting.fit(x.T, A, p_0_best, f, constant_parms) sol1 -= np.mean(sol1) else: sol1 = sol0 sols[t_idx, :] = (sol0[0, :] + sol1[0, :])/2 ipbar += 1 pbar.update(ipbar) ### Remove outliers logging.debug(' Searching for outliers...') for kk in xrange(10): s = sols[:, -1].copy() selection = np.zeros(len(s), np.bool) for t_idx in xrange(len(s)): start_idx = np.max([t_idx-10, 0]) end_idx = np.min([t_idx+10, len(s)]) selection[t_idx] = np.sum(abs(s[start_idx:end_idx] - s[t_idx]) < 0.02) > (end_idx - start_idx - 8) outliers = find(np.logical_not(selection)) if len(outliers) == 0: break for t_idx in outliers: try: idx0 = find(selection[:t_idx])[-1] except IndexError: idx0 = -1 try: idx1 = find(selection[t_idx+1:])[0] + t_idx + 1 except IndexError: idx1 = -1 if idx0 == -1: s[t_idx] = s[idx1] elif idx1 == -1: s[t_idx] = s[idx0] else: s[t_idx] = (s[idx0] * (idx1-t_idx) + s[idx1] * (t_idx-idx0)) / (idx1-idx0) p_0 = np.zeros((1, N_stations_selected1), np.double) p_0[0,:] = sols[t_idx,:] p_0[0,-1] = s[t_idx] x = p0[:,:,t_idx].copy() f = flags_source_pair[:,:,t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol0 -= np.mean(sol0) if soln_type != 'scalarphase': x = p1[:,:,t_idx].copy() sol1 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol1 -= np.mean(sol1) else: sol1 = sol0 sols[t_idx, :] = (sol0[0,:] + sol1[0,:])/2 weight = 1.0 sols_list.append(weight*sols) min_e_list.append(min_e) eq = np.zeros(N_sources) eq[ii] = weight eq[jj] = -weight eq_list.append(eq) sols = np.array(sols_list) B = np.array(eq_list) pinvB = pinv(B) q = np.dot(pinvB, sols.transpose([1,0,2])) pbar.finish() if nstations_max is not None: if N_stations_selected1 == nstations_max: break return station_selection1, q
def fit_screen_to_tec(station_names, source_names, pp, airmass, rr, times, height, order, r_0, beta): """ Fits a screen to given TEC values using Karhunen-Lo`eve base vectors Keyword arguments: station_names -- array of station names source_names -- array of source names pp -- array of piercepoint locations airmass -- array of airmass values rr -- array of TEC solutions times -- array of times height -- height of screen (m) order -- order of screen (i.e., number of KL base vectors to keep) r_0 -- scale size of phase fluctuations (m) beta -- power-law index for phase structure function (5/3 => pure Kolmogorov turbulence) """ import numpy as np from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye try: import progressbar except ImportError: import losoto.progressbar as progressbar logging.info('Fitting screens to TEC values...') N_stations = len(station_names) N_sources = len(source_names) N_times = len(times) tec_fit_all = np.zeros((N_times, N_sources, N_stations)) residual_all = np.zeros((N_times, N_sources, N_stations)) A = concatenate([ kron(eye(N_sources), np.ones((N_stations, 1))), kron(np.ones((N_sources, 1)), eye(N_stations)) ], axis=1) N_piercepoints = N_sources * N_stations P = eye(N_piercepoints) - np.dot(np.dot(A, pinv(np.dot(A.T, A))), A.T) pbar = progressbar.ProgressBar(maxval=N_times).start() ipbar = 0 for k in range(N_times): try: D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3)) D = np.transpose(D, (1, 0, 2)) - D D2 = np.sum(D**2, axis=2) C = -(D2 / r_0**2)**(beta / 2.0) / 2.0 P1 = eye(N_piercepoints) - np.ones( (N_piercepoints, N_piercepoints)) / N_piercepoints C1 = np.dot(np.dot(P1, C), P1) U, S, V = svd(C1) B = np.dot(P, np.dot(np.diag(airmass[k, :]), U[:, :order])) pinvB = pinv(B, rcond=1e-3) rr1 = np.dot(P, rr[:, k]) tec_fit = np.dot(U[:, :order], np.dot(pinvB, rr1)) tec_fit_all[k, :, :] = tec_fit.reshape((N_sources, N_stations)) residual = rr1 - np.dot(P, tec_fit) residual_all[k, :, :] = residual.reshape((N_sources, N_stations)) except: # Set screen to zero if fit did not work logging.debug('Tecscreen fit failed for timeslot {0}'.format(k)) tec_fit_all[k, :, :] = np.zeros((N_sources, N_stations)) residual_all[k, :, :] = np.ones((N_sources, N_stations)) pbar.update(ipbar) ipbar += 1 pbar.finish() return tec_fit_all, residual_all
def _fit_screen(station_names, source_names, full_matrices, pp, rr, weights, order, r_0, beta, screen_type): """ Fits a screen to amplitudes or phases using Karhunen-Lo`eve base vectors Parameters ---------- station_names: array Array of station names source_names: array Array of source names full_matrices : list of arrays List of [C, pivC, U] matrices for all piercepoints pp: array Array of piercepoint locations airmass: array Array of airmass values (note: not currently used) rr: array Array of amp values to fit screen to weights: array Array of weights order: int Order of screen (i.e., number of KL base vectors to keep) r_0: float Scale size of amp fluctuations (m) beta: float Power-law index for amp structure function (5/3 => pure Kolmogorov turbulence) screen_type : str Type of screen: 'phase' or 'amplitude' Returns ------- screen_fit_white_all, screen_residual_all : array, array Arrays of screen and residual (actual - screen) values """ import numpy as np from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye # Identify flagged directions N_sources_all = len(source_names) unflagged = np.where(weights > 0.0) N_sources = len(source_names[unflagged]) # Initialize arrays N_stations = len(station_names) N_piercepoints = N_sources * N_stations N_piercepoints_all = N_sources_all * N_stations screen_fit_all = np.zeros((N_sources_all, N_stations)) pp_all = pp.copy() rr_all = rr.copy() pp = pp_all[unflagged[0], :] w = np.diag(weights[unflagged]) # Calculate matrices if N_sources == N_sources_all: C, pinvC, U = full_matrices else: # Recalculate for unflagged directions C, pinvC, U = _calculate_svd(pp, r_0, beta, N_piercepoints) invU = pinv(np.dot(np.transpose(U[:, :order]), np.dot(w, U)[:, :order]), rcond=1e-3) # Fit screen to unflagged directions if screen_type == 'phase': # Change phase to real/imag rr_real = np.cos(rr[unflagged]) rr_imag = np.sin(rr[unflagged]) # Calculate real screen rr1 = np.dot(np.transpose(U[:, :order]), np.dot(w, rr_real)) real_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) # Calculate imag screen rr1 = np.dot(np.transpose(U[:, :order]), np.dot(w, rr_imag)) imag_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) # Calculate phase screen screen_fit = np.arctan2(np.dot(C, imag_fit), np.dot(C, real_fit)) screen_fit_white = np.dot(pinvC, screen_fit) else: # Calculate log(amp) screen rr = rr[unflagged] rr1 = np.dot(np.transpose(U[:, :order]), np.dot(w, np.log10(rr))) amp_fit_log = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) # Calculate amp screen screen_fit = 10**(np.dot(C, amp_fit_log)) screen_fit_white = np.dot(pinvC, screen_fit) # Calculate screen in all directions if N_sources != N_sources_all: screen_fit_all[unflagged[0], :] = screen_fit[:, newaxis] flagged = np.where(weights <= 0.0) for findx in flagged[0]: p = pp_all[findx, :] d2 = np.sum(np.square(pp - p), axis=1) c = -(d2 / (r_0**2))**(beta / 2.0) / 2.0 screen_fit_all[findx, :] = np.dot(c, screen_fit_white) C, pinvC, U = full_matrices screen_fit_white_all = np.dot(pinvC, screen_fit_all) screen_residual_all = rr_all - screen_fit_all.reshape( N_piercepoints_all) else: screen_fit_white_all = screen_fit_white screen_residual_all = rr_all - np.dot(C, screen_fit_white) screen_fit_white_all = screen_fit_white_all.reshape( (N_sources_all, N_stations)) screen_residual_all = screen_residual_all.reshape( (N_sources_all, N_stations)) return (screen_fit_white_all, screen_residual_all)
def calcSlipParams3D2(IC, m, FS, ymin, T, P0 = [14000., 1.16, 1., 0., 0.]): """ calculates a set of SLIP parameters that result in the desired motion. :args: IC (3x float) : initial condition y, vx, vz FS (3x float) : final state y, vx, vz ymin : minimal vertical excursion T : total step duration P0 (5x float) : initial guess for parameters [k, alpha, L0, beta, dE] (dE is ignored. It is calculated from IC and FS) :returns: [k, alpha, L0, beta, dE] parameter vector that results in the desired state """ dE = (FS[0]-IC[0])*m*9.81 + .5*m*(FS[1]**2 + FS[2]**2 - IC[1]**2 - IC[2]**2) k, alpha, L0, beta, _ = P0 def getDiff(t, y): """ returns the difference in desired params """ delta = [T - t[-1], ymin - min(y[:,1]), #FS - y[-1,[1,3,5]], FS[[0,2]] - y[-1,[1,5]] ] return hstack(delta) rcond = 1e-3 # for pinverting the jacobian. this will be adapted during the process init_success = False while not init_success: try: pars = [k, L0, m, alpha, beta, dE] t, y = sl.qSLIP_step3D(IC, pars) init_success = True except ValueError: L0 -= .02 d0 = getDiff(t, y) nd0 = norm(d0) #print \"difference: \", nd0, d0 cancel = False niter = 0 while nd0 > 1e-6 and not cancel: niter += 1 if niter > 20: print "too many iterations" cancel = True break # calculate jacobian dDelta / dP #print \"rep:\", niter hs = array([10, .0001, .0001, .0001]) pdims = [0, 1, 3, 4] J = [] for dim in range(4): parsp = [k, L0, m, alpha, beta, dE][:] parsm = [k, L0, m, alpha, beta, dE][:] parsp[pdims[dim]] += hs[dim] parsm[pdims[dim]] -= hs[dim] t, y = sl.qSLIP_step3D(IC, parsm) dm = getDiff(t, y) # circumvent "too low starting condition": try: t, y = sl.qSLIP_step3D(IC, parsp) dp = getDiff(t, y) J.append((dp - dm)/(2*hs[dim])) except ValueError: # run unilateral derivative instead parsp = [k, L0, m, alpha, beta, dE] t, y = sl.qSLIP_step3D(IC, parsm) dp = getDiff(t, y) J.append((dp - dm)/(hs[dim])) print "|1>" J = vstack(J).T pred = [k, L0, alpha, beta] update = dot(pinv(J, rcond=rcond), d0) nrm = norm(update * array([.001, 10, 10, 10])) success = False cancel = False rep = 0 while not (success or cancel): rep += 1 #print \"update:\", update if rep > 3: # reset if rcond < 1e-10: #print \"rcond too small!\" cancel = True else: rcond /= 100 pars = [k, L0, m, alpha, beta, dE] break dk, dL0, dalpha, dbeta = update pars = [k - dk, L0 - dL0, m, alpha - dalpha, beta - dbeta, dE] try: t, y = sl.qSLIP_step3D(IC, pars) d0 = getDiff(t, y) #print "d0 = ", norm(d0) except (ValueError, sl.SimFailError, TypeError): #print "escaping ..." update /= 2 continue if norm(d0) < nd0: success = True nd0 = norm(d0) else: update /= 2 #print \"difference: \", norm(d0), d0 k, L0, alpha, beta = array(pars)[[0,1,3,4]] #if not cancel: # print \"converged!\" return array([k, alpha, L0, beta, dE])