def extract_max_values(stack_grid, search_info, f_stack, n_max=5e7): # get basic info nx = search_info['nx'] ny = search_info['ny'] nz = search_info['nz'] dx = search_info['dx'] dy = search_info['dy'] dz = search_info['dz'] x_orig = search_info['x_orig'] y_orig = search_info['y_orig'] z_orig = search_info['z_orig'] nb, nt = stack_grid.shape max_val = f_stack.create_dataset('max_val', (nt, ), 'f') max_val_smooth = f_stack.create_dataset('max_val_smooth', (nt, ), 'f') max_x = f_stack.create_dataset('max_x', (nt, ), 'f') max_y = f_stack.create_dataset('max_y', (nt, ), 'f') max_z = f_stack.create_dataset('max_z', (nt, ), 'f') # create temporary datasets max_ib = f_stack.create_dataset('max_ib', (nt, ), 'i') max_ix = f_stack.create_dataset('max_ix', (nt, ), 'i') max_iy = f_stack.create_dataset('max_iy', (nt, ), 'i') max_iz = f_stack.create_dataset('max_iz', (nt, ), 'i') # extract values dt = int(n_max / nb) if nt <= dt: # do the extraction in one step max_ib[:] = np.argmax(stack_grid, 0) max_val[:] = np.max(stack_grid, 0) else: # do the extraction in steps n = nt / dt logging.debug( 'Number of values exceeds %d. Doing extraction in %d steps' % (n_max, n)) for i in islice(count(0), n): max_ib[i * dt:(i + 1) * dt] = np.argmax( stack_grid[:, i * dt:(i + 1) * dt], 0) max_val[i * dt:(i + 1) * dt] = np.max( stack_grid[:, i * dt:(i + 1) * dt], 0) max_ib[n * dt:nt] = np.argmax(stack_grid[:, n * dt:nt], 0) max_val[n * dt:nt] = np.max(stack_grid[:, n * dt:nt], 0) # find the corresponding x,y,z values max_ix, max_iy, max_iz = np.unravel_index(max_ib, (nx, ny, nz)) max_x[:] = max_ix[:] * dx + x_orig max_y[:] = max_iy[:] * dy + y_orig max_z[:] = max_iz[:] * dz + z_orig max_val_smooth[:] = smooth(np.array(max_val), 51) # clean up temporary datasets del f_stack['max_ib'] del f_stack['max_ix'] del f_stack['max_iy'] del f_stack['max_iz']
def extract_max_values(stack_grid,search_info,f_stack,n_max=5e7): # get basic info nx=search_info['nx'] ny=search_info['ny'] nz=search_info['nz'] dx=search_info['dx'] dy=search_info['dy'] dz=search_info['dz'] x_orig=search_info['x_orig'] y_orig=search_info['y_orig'] z_orig=search_info['z_orig'] nb,nt=stack_grid.shape max_val=f_stack.create_dataset('max_val',(nt,),'f') max_val_smooth=f_stack.create_dataset('max_val_smooth',(nt,),'f') max_x=f_stack.create_dataset('max_x',(nt,),'f') max_y=f_stack.create_dataset('max_y',(nt,),'f') max_z=f_stack.create_dataset('max_z',(nt,),'f') # create temporary datasets max_ib=f_stack.create_dataset('max_ib',(nt,),'i') max_ix=f_stack.create_dataset('max_ix',(nt,),'i') max_iy=f_stack.create_dataset('max_iy',(nt,),'i') max_iz=f_stack.create_dataset('max_iz',(nt,),'i') # extract values dt=int(n_max/nb) if nt <= dt : # do the extraction in one step max_ib[:]=np.argmax(stack_grid,0) max_val[:]=np.max(stack_grid,0) else: # do the extraction in steps n=nt/dt logging.debug('Number of values exceeds %d. Doing extraction in %d steps'%(n_max,n)) for i in islice(count(0),n): max_ib[i*dt:(i+1)*dt]=np.argmax(stack_grid[:,i*dt:(i+1)*dt],0) max_val[i*dt:(i+1)*dt]=np.max(stack_grid[:,i*dt:(i+1)*dt],0) max_ib[n*dt:nt]=np.argmax(stack_grid[:,n*dt:nt],0) max_val[n*dt:nt]=np.max(stack_grid[:,n*dt:nt],0) # find the corresponding x,y,z values max_ix,max_iy,max_iz=np.unravel_index(max_ib,(nx,ny,nz)) max_x[:]=max_ix[:]*dx+x_orig max_y[:]=max_iy[:]*dy+y_orig max_z[:]=max_iz[:]*dz+z_orig max_val_smooth[:] = smooth(np.array(max_val),51) # clean up temporary datasets del f_stack['max_ib'] del f_stack['max_ix'] del f_stack['max_iy'] del f_stack['max_iz']
def test_smoothing(self): from filters import smooth x=np.arange(100) max_val=100.*np.exp(-(x-50.)*(x-50.)/(10.*10.))+np.random.rand(100) max_x=np.random.rand(100) max_y=np.random.rand(100) max_z=np.random.rand(100) max_val_smooth=smooth(max_val) left_trig=right_trig=3 locs_smooth=trigger_locations_inner(max_val_smooth,max_x,max_y,max_z,left_trig,right_trig,0.0,1.0) self.assertAlmostEqual(locs_smooth[0]['o_time'],50.,2)
def filter_max_stack(st_max, corner): # maximum of stack is too rough - filter it logging.debug("Doing filtering") st_filt = st_max.copy() st_filt.clear() for tr in st_max.traces: tr_filt = tr.copy() #x_filt=lowpass(tr_filt.data,corner,1/tr.stats.delta,zerophase=True) x_filt = smooth(tr_filt.data) tr_filt.data = x_filt st_filt.append(tr_filt) logging.debug("Done!") return st_filt
def filter_max_stack(st_max,corner): # maximum of stack is too rough - filter it logging.debug("Doing filtering") st_filt = st_max.copy() st_filt.clear() for tr in st_max.traces: tr_filt=tr.copy() #x_filt=lowpass(tr_filt.data,corner,1/tr.stats.delta,zerophase=True) x_filt=smooth(tr_filt.data) tr_filt.data=x_filt st_filt.append(tr_filt) logging.debug("Done!") return st_filt
def test_smoothing(self): from filters import smooth x = np.arange(100) max_val = 100. * np.exp(-(x - 50.) * (x - 50.) / (10. * 10.)) + np.random.rand(100) max_x = np.random.rand(100) max_y = np.random.rand(100) max_z = np.random.rand(100) max_val_smooth = smooth(max_val) left_trig = right_trig = 3 locs_smooth = trigger_locations_inner(max_val_smooth, max_x, max_y, max_z, left_trig, right_trig, 0.0, 1.0) self.assertAlmostEqual(locs_smooth[0]['o_time'], 50., 2)
def csv2wav(fname, smoothing_data): ''' Converts a simulation file in .csv format to .wav files -------- Arguments: fname : str Name of simulation file smoothing_data : tuple 2-tuple containing (1) smoothing window to use for time series and (2) length of smoothing window ''' assert is_simulation_data(fname), \ "Error: file does not seem to have extension .csv!" angles, num_samples, time_stamps = load_simulation_data_from_csv(fname) USE_TIME_STAMPS = True t, angles, num_samples = make_time_series( \ angles, \ num_samples, \ time_stamps, \ USE_TIME_STAMPS \ ) # Smoothing, if requested smoothing_window, window_len = smoothing_data if smoothing_window != None: logString( \ "Smoothing time series with " + smoothing_window + \ " window of length " + str(window_len) \ ) for i in range(angles.shape[0]): angles[i, :] = smooth(angles[i, :], window_len=window_len, window=smoothing_window) # Make .wav files fname_base = os.path.splitext(fname)[0] fname = os.path.join(get_data_dir(), fname_base) write_wave(angles[OUTER, :], fname + "_outer") write_wave(angles[INNER, :], fname + "_inner") return
def analyze(fname, imu_to_plot, estimate, use_calibration, \ use_legacy_sign_convention, use_time_stamps, plot_slice, \ make_wav, anim_data, smoothing_data): ''' Visualizes logged data -------- Arguments: fname : str Name of log file whose data is to be analyzed imu_to_plot : str Indicates whether the data for the base IMU or the lamp IMU is to be analzed, or both estimate : bool Estimates angles if True, otherwise plots raw data use_calibration : bool If True, applies rotations and offsets to the raw data, based on the contents of the .ini files. This can be used to account for the fact that the IMUs are mounted at angles relative to the lamp and base use_legacy_sign_convention : bool If True, transforms the data set from the old acceleration sign convention to the new one. Meant for data sets recorded on MCU firmware older than July 2019 use_time_stamps : bool If True, uses the time stamps in the data log to construct the time series. Assuming the clock on the computer doing the recording is trustworthy, this should be very robust. Missing entries are accounted for. Otherwise, the time series is constructed based on sampling rate * number of samples, and has no connection to "true time" plot_slice : string String containing start time and end time to plot between make_wav : bool Indicates whether to make .wav file from angle data anim_data : tuple 3-tuple containing (1) bool indicating whether or not to animate, (2) animation type, and (3) animation arguments smoothing_data : tuple 2-tuple containing (1) smoothing window to use for time series and (2) length of smoothing window ''' make_data_dir() if fname == "latest": glob_str = os.path.join(get_data_dir(), '*.dat') files = glob.glob(glob_str) fname = max(files, key=os.path.getctime) imu_data, num_samples, time_stamps = load_data_from_file( \ fname, \ use_calibration=use_calibration, \ use_legacy_sign_convention=use_legacy_sign_convention \ ) t, imu_data, num_samples = make_time_series( \ imu_data, \ num_samples, \ time_stamps, \ use_time_stamps \ ) angles = get_angles(imu_data, num_samples) # Smoothing, if requested smoothing_window, window_len = smoothing_data if smoothing_window != None: logString( \ "Smoothing time series with " + smoothing_window + \ " window of length " + str(window_len) \ ) for i in range(angles.shape[0]): angles[i, :] = smooth(angles[i, :], window_len=window_len, window=smoothing_window) if make_wav: # Combine the pitch and roll from each IMU into a single value for each angles[OUTER:INNER+1,:] = angles[BASE_OUTER:BASE_INNER+1,:] + \ angles[LAMP_OUTER:LAMP_INNER+1,:] fname_base = os.path.splitext(fname)[0] fname = os.path.join(get_data_dir(), fname_base) write_wave(angles[OUTER, :], fname + "_outer") write_wave(angles[INNER, :], fname + "_inner") return if plot_slice: t_start, t_end = plot_slice.split(',') t_start = np.round(float(t_start), 2) t_end = np.round(float(t_end), 2) # Bounds check! if t_end > t[-1]: logString( \ "--plot slice end time exceeds end time of data ({0})".format(t[-1]) \ ) quit() start_idx = int(np.round(t_start * (num_samples / (t[-1] - t[0])))) end_idx = int(np.round(t_end * (num_samples / (t[-1] - t[0])))) # Slice! t = t[start_idx:end_idx + 1] imu_data = imu_data[:, start_idx:end_idx + 1] angles = angles[:, start_idx:end_idx + 1] num_samples = end_idx - start_idx + 1 do_animate, anim_type, anim_args = anim_data if do_animate: aa = anim.Animate(t, angles, fname) if anim_type == 'phase': aa.do_phase_space_animation(imu_to_plot) elif anim_type == 'top_down': do_decomp = anim_args == 'decomp' aa.do_birds_eye_view_animation(imu_to_plot, do_decomp) elif anim_type == 'pendulum': if imu_to_plot == 'lamp' or imu_to_plot == 'both': if anim_args == 'outer' or anim_args == 'both': aa.do_pendulum_animation(LAMP_OUTER) if anim_args == 'inner' or anim_args == 'both': aa.do_pendulum_animation(LAMP_INNER) if imu_to_plot == 'base' or imu_to_plot == 'both': if anim_args == 'outer' or anim_args == 'both': aa.do_pendulum_animation(BASE_OUTER) if anim_args == 'inner' or anim_args == 'both': aa.do_pendulum_animation(BASE_INNER) else: assert (False), "Animation processing messed up!" return fig, ax = plt.subplots() size = 2 if estimate == "none": if imu_to_plot == "base" or imu_to_plot == "both": ax.plot(t, imu_data[IMU_BASE_IDX + ACC_IDX + X_IDX], c="blue", label="Base Ax") ax.plot(t, imu_data[IMU_BASE_IDX + ACC_IDX + Y_IDX], c="red", label="Base Ay") ax.plot(t, imu_data[IMU_BASE_IDX + ACC_IDX + Z_IDX], c="green", label="Base Az") ax.plot(t, imu_data[IMU_BASE_IDX + GYRO_IDX + X_IDX], c="gold", label="Base Vx") ax.plot(t, imu_data[IMU_BASE_IDX + GYRO_IDX + Y_IDX], c="black", label="Base Vy") ax.plot(t, imu_data[IMU_BASE_IDX + GYRO_IDX + Z_IDX], c="magenta", label="Base Vz") if imu_to_plot == "lamp" or imu_to_plot == "both": ax.plot(t, imu_data[IMU_LAMP_IDX + ACC_IDX + X_IDX], c="blue", label="Lamp Ax") ax.plot(t, imu_data[IMU_LAMP_IDX + ACC_IDX + Y_IDX], c="red", label="Lamp Ay") ax.plot(t, imu_data[IMU_LAMP_IDX + ACC_IDX + Z_IDX], c="green", label="Lamp Az") ax.plot(t, imu_data[IMU_LAMP_IDX + GYRO_IDX + X_IDX], c="gold", label="Lamp Vx") ax.plot(t, imu_data[IMU_LAMP_IDX + GYRO_IDX + Y_IDX], c="black", label="Lamp Vy") ax.plot(t, imu_data[IMU_LAMP_IDX + GYRO_IDX + Z_IDX], c="magenta", label="Lamp Vz") ax.legend() plt.title('Raw data vs time') plt.xlabel('Time (s)') plt.ylabel('Raw data ($m/s^2$ and $^\circ$)') fig_name = "raw_" + imu_to_plot + "_" fig_name += os.path.splitext(os.path.basename(fname))[0] if plot_slice: fig_name += "_from%.2fto%.2f" % (t_start, t_end) fig_name += '.png' fig_name = os.path.join(get_data_dir(), os.path.dirname(fname), fig_name) plt.savefig(fig_name) logString("Saved fig to {0}".format(fig_name)) plt.close() else: if estimate == "ind_angles": # Plot pitch and roll separately for each IMU if imu_to_plot == "base" or imu_to_plot == "both": ax.plot(t, angles[BASE_OUTER], c="blue", label="Base (outer)") ax.plot(t, angles[BASE_INNER], c="red", label="Base (inner)") if imu_to_plot == "lamp" or imu_to_plot == "both": ax.plot(t, angles[LAMP_OUTER], c="green", label="Lamp (outer)") ax.plot(t, angles[LAMP_INNER], c="gold", label="Lamp (inner)") fig_name = estimate + "_" + imu_to_plot + "_imu_" else: # Combine the pitch and roll from each IMU into a single value for each angles[OUTER:INNER+1,:] = angles[BASE_OUTER:BASE_INNER+1,:] + \ angles[LAMP_OUTER:LAMP_INNER+1,:] ax.plot(t, angles[OUTER], c="blue", label="Outer gimbal") ax.plot(t, angles[INNER], c="red", label="Inner gimbal") fig_name = estimate + "_" ax.legend() plt.title('Angles vs time') plt.xlabel('Time (s)') plt.ylabel('Angle ($^\circ$)') fig_name += os.path.splitext(os.path.basename(fname))[0] if plot_slice: fig_name += "_from%.2fto%.2f" % (t_start, t_end) if use_calibration: fig_name += "_calibrated" fig_name += '.png' fig_name = os.path.join(get_data_dir(), os.path.dirname(fname), fig_name) plt.savefig(fig_name) logString("Saved fig to {0}".format(fig_name)) plt.close()
def extract_max_values(stack_grid, search_info, f_stack, use_ram=False, n_max=5e8): """ Extracts maximum stack value from a 4D migrated grid. Also extracts x_max, y_max, z_max. TODO : flesh out this doc-string. :param stack_grid: :param search_info: :param f_stack: :param use_fam: :param n_max: """ # get basic info nx = search_info['nx'] ny = search_info['ny'] nz = search_info['nz'] dx = search_info['dx'] dy = search_info['dy'] dz = search_info['dz'] x_orig = search_info['x_orig'] y_orig = search_info['y_orig'] z_orig = search_info['z_orig'] nb, nt = stack_grid.shape if use_ram: max_val = np.empty(nt, dtype='float32') max_val_smooth = np.empty(nt, dtype='float32') max_x = np.empty(nt, dtype='float32') max_y = np.empty(nt, dtype='float32') max_z = np.empty(nt, dtype='float32') # create temporary datasets max_ib = np.empty(nt, dtype=int) max_ix = np.empty(nt, dtype=int) max_iy = np.empty(nt, dtype=int) max_iz = np.empty(nt, dtype=int) else: max_val = f_stack.create_dataset('max_val', (nt, ), 'f') max_val_smooth = f_stack.create_dataset('max_val_smooth', (nt, ), 'f') max_x = f_stack.create_dataset('max_x', (nt, ), 'f') max_y = f_stack.create_dataset('max_y', (nt, ), 'f') max_z = f_stack.create_dataset('max_z', (nt, ), 'f') # create temporary datasets max_ib = f_stack.create_dataset('max_ib', (nt, ), 'i') max_ix = f_stack.create_dataset('max_ix', (nt, ), 'i') max_iy = f_stack.create_dataset('max_iy', (nt, ), 'i') max_iz = f_stack.create_dataset('max_iz', (nt, ), 'i') # extract values dt = int(n_max / nb) if nt <= dt: # do the extraction in one step max_ib[:] = np.argmax(stack_grid, 0) max_val[:] = np.max(stack_grid, 0) else: # do the extraction in steps n = nt / dt logging.info('Number of values exceeds %d. Doing extraction in %d\ steps' % (n_max, n)) for i in islice(count(0), n): max_ib[i * dt:(i + 1) * dt] = np.argmax( stack_grid[:, i * dt:(i + 1) * dt], 0) max_val[i * dt:(i + 1) * dt] = np.max( stack_grid[:, i * dt:(i + 1) * dt], 0) max_ib[n * dt:nt] = np.argmax(stack_grid[:, n * dt:nt], 0) max_val[n * dt:nt] = np.max(stack_grid[:, n * dt:nt], 0) # find the corresponding x,y,z values max_ix, max_iy, max_iz = np.unravel_index(max_ib, (nx, ny, nz)) max_x[:] = max_ix[:] * dx + x_orig max_y[:] = max_iy[:] * dy + y_orig max_z[:] = max_iz[:] * dz + z_orig max_val_smooth[:] = smooth(np.array(max_val), 51) if use_ram: # add datasets to hdf5file max_val = f_stack.create_dataset('max_val', data=max_val) max_val_smooth = f_stack.create_dataset('max_val_smooth', data=max_val_smooth) max_x = f_stack.create_dataset('max_x', data=max_x) max_y = f_stack.create_dataset('max_y', data=max_y) max_z = f_stack.create_dataset('max_z', data=max_z) else: # clean up temporary datasets del f_stack['max_ib'] del f_stack['max_ix'] del f_stack['max_iy'] del f_stack['max_iz']
def do_locations_trigger_setup_and_run(opdict): base_path=opdict['base_path'] # parse command line data_dir=os.path.join(base_path,'data',opdict['datadir']) kurt_files=glob.glob(os.path.join(data_dir,opdict['kurtglob'])) data_files=glob.glob(os.path.join(data_dir,opdict['dataglob'])) kurt_files.sort() data_files.sort() dataglob=opdict['dataglob'] time_grids=get_interpolated_time_grids(opdict) logging.info("Starting log for combine_stacks.") out_path=os.path.join(base_path,'out',opdict['outdir']) stack_path=os.path.join(out_path,'stack') reloc=opdict['reloc'] if reloc: loc_path=os.path.join(out_path,'reloc') stack_files=glob.glob(os.path.join(stack_path,'reloc_stack_all*.hdf5')) stack_files.sort() else: loc_path=os.path.join(out_path,'loc') stack_files=glob.glob(os.path.join(stack_path,'stack_all*.hdf5')) stack_files.sort() n_stacks=len(stack_files) if n_stacks == 0 : raise UserWarning('Empty list of stacks in %s'%(stack_path)) loc_filename=os.path.join(loc_path,"locations.dat") logging.info("Path for stack files : %s"%stack_path) logging.info("Path for loc files : %s"%loc_path) logging.info("Location file : %s"%loc_filename) # DO DATA PREP ACCORDING TO RELOC OR NOT logging.info("\nDealing with continuous location, so merging stack files directly ...\n") # get basic info from first file f_stack = h5py.File(stack_files[0],'r') max_val = f_stack['max_val'] dt = max_val.attrs['dt'] f_stack.close() # get start times (get first and last times) start_times=[] end_times=[] for fname in stack_files: f_stack = h5py.File(fname,'r') max_val = f_stack['max_val'] start_times.append(utcdatetime.UTCDateTime(max_val.attrs['start_time'])) end_times.append( utcdatetime.UTCDateTime(max_val.attrs['start_time'])+dt*len(max_val)) f_stack.close() first_start_time = min(start_times) last_end_time = max(end_times) nt_full=int((last_end_time-first_start_time)/dt)+1 # create - assume all stacks are of the same length and will be concatenated end to end # (this will give more than enough space) f = h5py.File(os.path.join(stack_path,'combined_stack_all.hdf5'),'w') cmax_val = f.create_dataset('max_val',(nt_full,), 'f', chunks=(nt_full,)) cmax_x = f.create_dataset('max_x',(nt_full,), 'f', chunks=(nt_full,)) cmax_y = f.create_dataset('max_y',(nt_full,), 'f', chunks=(nt_full,)) cmax_z = f.create_dataset('max_z',(nt_full,), 'f', chunks=(nt_full,)) # concatenate unsmoothed versions of max_val to avoid # problems at file starts and ends for i in range(n_stacks): f_stack = h5py.File(stack_files[i],'r') max_val = f_stack['max_val'] max_x = f_stack['max_x'] max_y = f_stack['max_y'] max_z = f_stack['max_z'] # get time info for this stack nt = len(max_val) start_time = utcdatetime.UTCDateTime(max_val.attrs['start_time']) ibegin=np.int((start_time-first_start_time)/dt) # copy data over into the right place cmax_val[ibegin:ibegin+nt] = max_val[:] cmax_x[ibegin:ibegin+nt] = max_x[:] cmax_y[ibegin:ibegin+nt] = max_y[:] cmax_z[ibegin:ibegin+nt] = max_z[:] # close the stack f_stack.close() # create the smoothed version of the max stack cmax_val_smooth = f.create_dataset('max_val_smooth',(nt_full,), 'f', chunks=(nt_full,)) cmax_val_smooth[:] = smooth(np.array(cmax_val),51) for name in f: dset=f[name] dset.attrs['dt']=dt dset.attrs['start_time']=first_start_time.isoformat() # DO TRIGGERING AND LOCATION if opdict['auto_loclevel']: loclevel=opdict['snr_loclevel']*np.median(cmax_val_smooth) opdict['loclevel']=loclevel else: loclevel=opdict['loclevel'] left_trig=loclevel right_trig=loclevel loc_list=trigger_locations_inner(cmax_val_smooth[:],cmax_x,cmax_y,cmax_z,left_trig,right_trig,first_start_time,dt) logging.info('Found %d initial.'%(len(loc_list))) # close the stack file f.close() loc_file=open(loc_filename,'w') write_header_options(loc_file,opdict) snr_limit=opdict['snr_limit'] snr_tr_limit=opdict['snr_tr_limit'] sn_time=opdict['sn_time'] n_kurt_min=opdict['n_kurt_min'] n_ok=0 locs=[] for loc in loc_list: if number_good_kurtosis_for_location(kurt_files,data_files,loc,time_grids,snr_limit,snr_tr_limit,sn_time) > n_kurt_min: logging.info("Max = %.2f, %s - %.2fs + %.2f s, x=%.4f pm %.4f km, y=%.4f pm %.4f km, z=%.4f pm %.4f km"%(loc['max_trig'],loc['o_time'].isoformat(),loc['o_err_left'], loc['o_err_right'],loc['x_mean'],loc['x_sigma'],loc['y_mean'],loc['y_sigma'],loc['z_mean'],loc['z_sigma'])) loc_file.write("Max = %.2f, %s - %.2f s + %.2f s, x= %.4f pm %.4f km, y= %.4f pm %.4f km, z= %.4f pm %.4f km\n"%(loc['max_trig'],loc['o_time'].isoformat(),loc['o_err_left'], loc['o_err_right'],loc['x_mean'],loc['x_sigma'],loc['y_mean'],loc['y_sigma'],loc['z_mean'],loc['z_sigma'])) n_ok=n_ok+1 locs.append(loc) else: logging.info("Not enough kurtosis picks for : Max = %.2f, %s - %.2fs + %.2fs, x=%.4f pm %.4f, y=%.4f pm %.4f, z=%.4f pm %.4f"%(loc['max_trig'],loc['o_time'].isoformat(),loc['o_err_left'], loc['o_err_right'],loc['x_mean'],loc['x_sigma'],loc['y_mean'],loc['y_sigma'],loc['z_mean'],loc['z_sigma'])) loc_file.close() logging.info('Wrote %d locations to file %s.'%(n_ok,loc_filename)) return locs
def do_locations_trigger_setup_and_run(opdict): base_path = opdict['base_path'] # parse command line data_dir = os.path.join(base_path, 'data', opdict['datadir']) kurt_files = glob.glob(os.path.join(data_dir, opdict['kurtglob'])) data_files = glob.glob(os.path.join(data_dir, opdict['dataglob'])) kurt_files.sort() data_files.sort() dataglob = opdict['dataglob'] time_grids = get_interpolated_time_grids(opdict) logging.info("Starting log for combine_stacks.") out_path = os.path.join(base_path, 'out', opdict['outdir']) stack_path = os.path.join(out_path, 'stack') reloc = opdict['reloc'] if reloc: loc_path = os.path.join(out_path, 'reloc') stack_files = glob.glob( os.path.join(stack_path, 'reloc_stack_all*.hdf5')) stack_files.sort() else: loc_path = os.path.join(out_path, 'loc') stack_files = glob.glob(os.path.join(stack_path, 'stack_all*.hdf5')) stack_files.sort() n_stacks = len(stack_files) if n_stacks == 0: raise UserWarning('Empty list of stacks in %s' % (stack_path)) loc_filename = os.path.join(loc_path, "locations.dat") logging.info("Path for stack files : %s" % stack_path) logging.info("Path for loc files : %s" % loc_path) logging.info("Location file : %s" % loc_filename) # DO DATA PREP ACCORDING TO RELOC OR NOT logging.info( "\nDealing with continuous location, so merging stack files directly ...\n" ) # get basic info from first file f_stack = h5py.File(stack_files[0], 'r') max_val = f_stack['max_val'] dt = max_val.attrs['dt'] f_stack.close() # get start times (get first and last times) start_times = [] end_times = [] for fname in stack_files: f_stack = h5py.File(fname, 'r') max_val = f_stack['max_val'] start_times.append(utcdatetime.UTCDateTime( max_val.attrs['start_time'])) end_times.append( utcdatetime.UTCDateTime(max_val.attrs['start_time']) + dt * len(max_val)) f_stack.close() first_start_time = min(start_times) last_end_time = max(end_times) nt_full = int((last_end_time - first_start_time) / dt) + 1 # create - assume all stacks are of the same length and will be concatenated end to end # (this will give more than enough space) f = h5py.File(os.path.join(stack_path, 'combined_stack_all.hdf5'), 'w') cmax_val = f.create_dataset('max_val', (nt_full, ), 'f', chunks=(nt_full, )) cmax_x = f.create_dataset('max_x', (nt_full, ), 'f', chunks=(nt_full, )) cmax_y = f.create_dataset('max_y', (nt_full, ), 'f', chunks=(nt_full, )) cmax_z = f.create_dataset('max_z', (nt_full, ), 'f', chunks=(nt_full, )) # concatenate unsmoothed versions of max_val to avoid # problems at file starts and ends for i in range(n_stacks): f_stack = h5py.File(stack_files[i], 'r') max_val = f_stack['max_val'] max_x = f_stack['max_x'] max_y = f_stack['max_y'] max_z = f_stack['max_z'] # get time info for this stack nt = len(max_val) start_time = utcdatetime.UTCDateTime(max_val.attrs['start_time']) ibegin = np.int((start_time - first_start_time) / dt) # copy data over into the right place cmax_val[ibegin:ibegin + nt] = max_val[:] cmax_x[ibegin:ibegin + nt] = max_x[:] cmax_y[ibegin:ibegin + nt] = max_y[:] cmax_z[ibegin:ibegin + nt] = max_z[:] # close the stack f_stack.close() # create the smoothed version of the max stack cmax_val_smooth = f.create_dataset('max_val_smooth', (nt_full, ), 'f', chunks=(nt_full, )) cmax_val_smooth[:] = smooth(np.array(cmax_val), 51) for name in f: dset = f[name] dset.attrs['dt'] = dt dset.attrs['start_time'] = first_start_time.isoformat() # DO TRIGGERING AND LOCATION if opdict['auto_loclevel']: loclevel = opdict['snr_loclevel'] * np.median(cmax_val_smooth) opdict['loclevel'] = loclevel else: loclevel = opdict['loclevel'] left_trig = loclevel right_trig = loclevel loc_list = trigger_locations_inner(cmax_val_smooth[:], cmax_x, cmax_y, cmax_z, left_trig, right_trig, first_start_time, dt) logging.info('Found %d initial.' % (len(loc_list))) # close the stack file f.close() loc_file = open(loc_filename, 'w') write_header_options(loc_file, opdict) snr_limit = opdict['snr_limit'] snr_tr_limit = opdict['snr_tr_limit'] sn_time = opdict['sn_time'] n_kurt_min = opdict['n_kurt_min'] n_ok = 0 locs = [] for loc in loc_list: if number_good_kurtosis_for_location( kurt_files, data_files, loc, time_grids, snr_limit, snr_tr_limit, sn_time) > n_kurt_min: logging.info( "Max = %.2f, %s - %.2fs + %.2f s, x=%.4f pm %.4f km, y=%.4f pm %.4f km, z=%.4f pm %.4f km" % (loc['max_trig'], loc['o_time'].isoformat(), loc['o_err_left'], loc['o_err_right'], loc['x_mean'], loc['x_sigma'], loc['y_mean'], loc['y_sigma'], loc['z_mean'], loc['z_sigma'])) loc_file.write( "Max = %.2f, %s - %.2f s + %.2f s, x= %.4f pm %.4f km, y= %.4f pm %.4f km, z= %.4f pm %.4f km\n" % (loc['max_trig'], loc['o_time'].isoformat(), loc['o_err_left'], loc['o_err_right'], loc['x_mean'], loc['x_sigma'], loc['y_mean'], loc['y_sigma'], loc['z_mean'], loc['z_sigma'])) n_ok = n_ok + 1 locs.append(loc) else: logging.info( "Not enough kurtosis picks for : Max = %.2f, %s - %.2fs + %.2fs, x=%.4f pm %.4f, y=%.4f pm %.4f, z=%.4f pm %.4f" % (loc['max_trig'], loc['o_time'].isoformat(), loc['o_err_left'], loc['o_err_right'], loc['x_mean'], loc['x_sigma'], loc['y_mean'], loc['y_sigma'], loc['z_mean'], loc['z_sigma'])) loc_file.close() logging.info('Wrote %d locations to file %s.' % (n_ok, loc_filename)) return locs
def process_kurtosis(self, win, recursive=False, pre_rmean=False, pre_taper=False, post_taper=True): """ Processing waveform using kurtosis (from statlib package). Calls filters.sw_kurtosis1(), and overwrites the waveform. Sets self.prof to 'Kurtosis' :param win: length of the window (in seconds) on which to calculate the kurtosis :param recursive: If ``True`` applies recursive kurtosis calculation :param pre_rmean: If ``True`` removes mean of signal before processing. :param pre_taper: If ``True`` applies taper to signal before processing. :param post_taper: If ``True`` applies taper to signal after processing. :type recursive: boolean, optional :type pre_rmean: boolean, optional :type pre_taper: boolean, optional :type post_taper: boolean, optional """ logging.info("Applying kurtosis to single traces, window = %.2f s\n" % win) dt = self.dt if pre_rmean: self.rmean() if pre_taper: self.taper() # process each trace independently for itr in range(self.stream.count()): tr = self.stream.traces[itr] starttime = tr.stats.starttime x = tr.data npts = len(tr.data) xs = np.zeros(npts) if recursive: C = 1 - dt / float(win) xs = rec_kurtosis_old(x, C) # Chassande-Mottin style kurtosis #C1=dt/float(win) #xs=rec_kurtosis(x,C1) # smooth xs try: xs_filt = smooth(xs) except ValueError: xs_filt = xs else: # run the sliding window kurtosis nwin = int(win / dt) if len(x) > 3 * nwin: xs = sw_kurtosis1(x, nwin) xs_filt = smooth(xs) # fix up the starttime of the trace tr.stats.starttime = starttime + (nwin - 1) * dt else: xs_filt = xs # Save xs values as waveform tr.data = xs_filt # put trace back into stream self.stream.traces[itr] = tr # apply taper after kurtosis calculation if required if post_taper: self.taper() # set the process flag self.proc = 'Kurtosis'
def extract_max_values(stack_grid, search_info, f_stack, use_ram=False, n_max=5e8): """ Extracts maximum stack value from a 4D migrated grid. Also extracts x_max, y_max, z_max. TODO : flesh out this doc-string. :param stack_grid: :param search_info: :param f_stack: :param use_fam: :param n_max: """ # get basic info nx = search_info['nx'] ny = search_info['ny'] nz = search_info['nz'] dx = search_info['dx'] dy = search_info['dy'] dz = search_info['dz'] x_orig = search_info['x_orig'] y_orig = search_info['y_orig'] z_orig = search_info['z_orig'] nb, nt = stack_grid.shape if use_ram: max_val = np.empty(nt, dtype='float32') max_val_smooth = np.empty(nt, dtype='float32') max_x = np.empty(nt, dtype='float32') max_y = np.empty(nt, dtype='float32') max_z = np.empty(nt, dtype='float32') # create temporary datasets max_ib = np.empty(nt, dtype=int) max_ix = np.empty(nt, dtype=int) max_iy = np.empty(nt, dtype=int) max_iz = np.empty(nt, dtype=int) else: max_val = f_stack.create_dataset('max_val', (nt, ), 'f') max_val_smooth = f_stack.create_dataset('max_val_smooth', (nt, ), 'f') max_x = f_stack.create_dataset('max_x', (nt, ), 'f') max_y = f_stack.create_dataset('max_y', (nt, ), 'f') max_z = f_stack.create_dataset('max_z', (nt, ), 'f') # create temporary datasets max_ib = f_stack.create_dataset('max_ib', (nt, ), 'i') max_ix = f_stack.create_dataset('max_ix', (nt, ), 'i') max_iy = f_stack.create_dataset('max_iy', (nt, ), 'i') max_iz = f_stack.create_dataset('max_iz', (nt, ), 'i') # extract values dt = int(n_max/nb) if nt <= dt: # do the extraction in one step max_ib[:] = np.argmax(stack_grid, 0) max_val[:] = np.max(stack_grid, 0) else: # do the extraction in steps n = nt/dt logging.info('Number of values exceeds %d. Doing extraction in %d\ steps' % (n_max, n)) for i in islice(count(0), n): max_ib[i*dt:(i+1)*dt] = np.argmax(stack_grid[:, i*dt:(i+1)*dt], 0) max_val[i*dt:(i+1)*dt] = np.max(stack_grid[:, i*dt:(i+1)*dt], 0) max_ib[n*dt:nt] = np.argmax(stack_grid[:, n*dt:nt], 0) max_val[n*dt:nt] = np.max(stack_grid[:, n*dt:nt], 0) # find the corresponding x,y,z values max_ix, max_iy, max_iz = np.unravel_index(max_ib, (nx, ny, nz)) max_x[:] = max_ix[:]*dx+x_orig max_y[:] = max_iy[:]*dy+y_orig max_z[:] = max_iz[:]*dz+z_orig max_val_smooth[:] = smooth(np.array(max_val), 51) if use_ram: # add datasets to hdf5file max_val = f_stack.create_dataset('max_val', data=max_val) max_val_smooth = f_stack.create_dataset('max_val_smooth', data=max_val_smooth) max_x = f_stack.create_dataset('max_x', data=max_x) max_y = f_stack.create_dataset('max_y', data=max_y) max_z = f_stack.create_dataset('max_z', data=max_z) else: # clean up temporary datasets del f_stack['max_ib'] del f_stack['max_ix'] del f_stack['max_iy'] del f_stack['max_iz']
def process_kurtosis(self, win, recursive=False, pre_rmean=False, pre_taper=False, post_taper=True): """ Processing waveform using kurtosis (from statlib package). Calls filters.sw_kurtosis1(), and overwrites the waveform. Sets self.prof to 'Kurtosis' :param win: length of the window (in seconds) on which to calculate the kurtosis :param recursive: If ``True`` applies recursive kurtosis calculation :param pre_rmean: If ``True`` removes mean of signal before processing. :param pre_taper: If ``True`` applies taper to signal before processing. :param post_taper: If ``True`` applies taper to signal after processing. :type recursive: boolean, optional :type pre_rmean: boolean, optional :type pre_taper: boolean, optional :type post_taper: boolean, optional """ logging.info("Applying kurtosis to single traces, window = %.2f s\n" % win) dt = self.dt if pre_rmean: self.rmean() if pre_taper: self.taper() # process each trace independently for itr in range(self.stream.count()): tr = self.stream.traces[itr] starttime = tr.stats.starttime x = tr.data npts = len(tr.data) xs = np.zeros(npts) if recursive: C = 1-dt/float(win) xs = rec_kurtosis_old(x, C) # Chassande-Mottin style kurtosis #C1=dt/float(win) #xs=rec_kurtosis(x,C1) # smooth xs try: xs_filt = smooth(xs) except ValueError: xs_filt = xs else: # run the sliding window kurtosis nwin = int(win/dt) if len(x) > 3*nwin: xs = sw_kurtosis1(x, nwin) xs_filt = smooth(xs) # fix up the starttime of the trace tr.stats.starttime = starttime + (nwin-1)*dt else: xs_filt = xs # Save xs values as waveform tr.data = xs_filt # put trace back into stream self.stream.traces[itr] = tr # apply taper after kurtosis calculation if required if post_taper: self.taper() # set the process flag self.proc = 'Kurtosis'
def kurto(origin_time, info, opdict): """ Finds for each Waveloc event and for each station the best filtering parameters for kurtosis computation. Writes them into the dictionary info. :param origin_time: origin time of the signal :param info: dictionary of parameters :param opdict: dictionary of the Waveloc parameters and options :type origin_time: utcdatetime :type info: dictionary :type opdict: dictionary :rtype: dictionary :returns: info """ verbose = opdict['verbose'] kwin = opdict['kwin'] start_time = origin_time-5.0 end_time = origin_time+20.0 dt = info['dt'] # Trace x = waveval(info['data_ini'], start_time, end_time, dt, info['tdeb_data']) if not x.any() and x.all(): return info # Initial kurtosis (trace filtered between 4-10Hz) kurtx = waveval(info['kurt_ini'], start_time, end_time, dt, info['tdeb_kurt']) kurtx = smooth(kurtx) N = len(x) N2 = np.log2(N)-7 nlevel = int(np.fix(N2)) snr_ref = np.max(np.abs(x))/np.mean(np.abs(x)) snr_kurt_ref = np.max(np.abs(kurtx))/np.mean(np.abs(kurtx)) kmax_ref = np.max(kurtx) # maximum of the kurtosis # Compute the kurtogram and keep best frequencies if verbose: import matplotlib.gridspec as gridspec G = gridspec.GridSpec(3, 2) fig = plt.figure(figsize=(15, 6)) fig.set_facecolor('white') fig.add_subplot(G[:, 0]) Kwav, Level_w, freq_w, c, f_lower, f_upper = \ Fast_Kurtogram(np.array(x, dtype=float), nlevel, verbose, Fs=1/dt, opt2=1) # Comparison of the kurtosis computed in the new frequency band and the old # one (criterion : snr, kmax) # 1. Read the initial data wf = Waveform() wf.read_from_file(info['data_file'], starttime=start_time-kwin, endtime=end_time+kwin) nbpts = int(kwin*1./dt) # 2. Filter the trace with kurtogram frequencies wf.bp_filter(f_lower, f_upper) x_filt = wf.values x_filt = x_filt[nbpts:-nbpts] # 3. Compute the kurtosis wf.process_kurtosis(kwin, recursive=opdict['krec']) new_kurtx = wf.values if opdict['krec']: new_kurtx = new_kurtx[nbpts+1:-nbpts-1] else: new_kurtx = new_kurtx[:-nbpts-1] snr = np.max(np.abs(x_filt))/np.mean(np.abs(x_filt)) snr_kurt = np.max(np.abs(new_kurtx))/np.mean(np.abs(new_kurtx)) kmax = np.max(new_kurtx) if snr > snr_ref and kmax >= kmax_ref: info['filter'].append((round(f_lower*100)/100, round(f_upper*100)/100)) if 'new_kurt_file' in info: info = write_file(info, start_time, end_time, new_kurtx) else: info['filter'].append((0, 50)) if verbose and snr > 3: print "snr:", snr, " ; snr_ref:", snr_ref print "snr new kurtosis:", snr_kurt, " ; snr kurtosis reference:",\ snr_kurt_ref print "kurtosis max, kurt_ref :", kmax, kmax_ref plot_trace(fig, G, x, x_filt, kurtx, new_kurtx, info, f_lower, f_upper, snr, snr_ref, snr_kurt, kmax, kmax_ref, origin_time) plt.show() return info
def kurto(origin_time, info, opdict): """ Finds for each Waveloc event and for each station the best filtering parameters for kurtosis computation. Writes them into the dictionary info. :param origin_time: origin time of the signal :param info: dictionary of parameters :param opdict: dictionary of the Waveloc parameters and options :type origin_time: utcdatetime :type info: dictionary :type opdict: dictionary :rtype: dictionary :returns: info """ verbose = opdict['verbose'] kwin = opdict['kwin'] start_time = origin_time - 5.0 end_time = origin_time + 20.0 dt = info['dt'] # Trace x = waveval(info['data_ini'], start_time, end_time, dt, info['tdeb_data']) if not x.any() and x.all(): return info # Initial kurtosis (trace filtered between 4-10Hz) kurtx = waveval(info['kurt_ini'], start_time, end_time, dt, info['tdeb_kurt']) kurtx = smooth(kurtx) N = len(x) N2 = np.log2(N) - 7 nlevel = int(np.fix(N2)) snr_ref = np.max(np.abs(x)) / np.mean(np.abs(x)) snr_kurt_ref = np.max(np.abs(kurtx)) / np.mean(np.abs(kurtx)) kmax_ref = np.max(kurtx) # maximum of the kurtosis # Compute the kurtogram and keep best frequencies if verbose: import matplotlib.gridspec as gridspec G = gridspec.GridSpec(3, 2) fig = plt.figure(figsize=(15, 6)) fig.set_facecolor('white') fig.add_subplot(G[:, 0]) Kwav, Level_w, freq_w, c, f_lower, f_upper = \ Fast_Kurtogram(np.array(x, dtype=float), nlevel, verbose, Fs=1/dt, opt2=1) # Comparison of the kurtosis computed in the new frequency band and the old # one (criterion : snr, kmax) # 1. Read the initial data wf = Waveform() wf.read_from_file(info['data_file'], starttime=start_time - kwin, endtime=end_time + kwin) nbpts = int(kwin * 1. / dt) # 2. Filter the trace with kurtogram frequencies wf.bp_filter(f_lower, f_upper) x_filt = wf.values x_filt = x_filt[nbpts:-nbpts] # 3. Compute the kurtosis wf.process_kurtosis(kwin, recursive=opdict['krec']) new_kurtx = wf.values if opdict['krec']: new_kurtx = new_kurtx[nbpts + 1:-nbpts - 1] else: new_kurtx = new_kurtx[:-nbpts - 1] snr = np.max(np.abs(x_filt)) / np.mean(np.abs(x_filt)) snr_kurt = np.max(np.abs(new_kurtx)) / np.mean(np.abs(new_kurtx)) kmax = np.max(new_kurtx) if snr > snr_ref and kmax >= kmax_ref: info['filter'].append( (round(f_lower * 100) / 100, round(f_upper * 100) / 100)) if 'new_kurt_file' in info: info = write_file(info, start_time, end_time, new_kurtx) else: info['filter'].append((0, 50)) if verbose and snr > 3: print "snr:", snr, " ; snr_ref:", snr_ref print "snr new kurtosis:", snr_kurt, " ; snr kurtosis reference:",\ snr_kurt_ref print "kurtosis max, kurt_ref :", kmax, kmax_ref plot_trace(fig, G, x, x_filt, kurtx, new_kurtx, info, f_lower, f_upper, snr, snr_ref, snr_kurt, kmax, kmax_ref, origin_time) plt.show() return info