def deriv_data(names, new_names=None, suffix=None, overwrite=None): old_names = pyspedas.tnames(names) if len(old_names) < 1: print('avg_data error: No pytplot names were provided.') return if suffix is None: suffix = '-der' if overwrite is not None: n_names = old_names elif new_names is None: n_names = [s + suffix for s in old_names] else: n_names = new_names if isinstance(n_names, str): n_names = [n_names] if len(n_names) != len(old_names): n_names = [s + suffix for s in old_names] for i, old in enumerate(old_names): new = n_names[i] if new != old: pyspedas.tcopy(old, new) data = pytplot.data_quants[new] data_new = data.differentiate('time').copy() pytplot.data_quants[new].values = data_new.values print('deriv_data was applied to: ' + new)
def yclip(names, ymin, ymax, flag=None, new_names=None, suffix=None, overwrite=None): old_names = pyspedas.tnames(names) if len(old_names) < 1: print('yclip error: No pytplot names were provided.') return if suffix is None: suffix = '-clip' if flag is None: flag = np.nan if overwrite is not None: n_names = old_names elif new_names is None: n_names = [s + suffix for s in old_names] else: n_names = new_names if isinstance(n_names, str): n_names = [n_names] if len(n_names) != len(old_names): n_names = [s + suffix for s in old_names] for i, old in enumerate(old_names): new = n_names[i] if new != old: pyspedas.tcopy(old, new) data = pytplot.data_quants[new].values try: for j, v in enumerate(data): try: iterator = enumerate(v) except TypeError: if not np.isnan(v) and (v <= ymin or v >= ymax): data[j] = flag else: for k, s in iterator: if not np.isnan(s) and (s <= ymin or s >= ymax): data[j][k] = flag except TypeError: # data Not itterable print("Cannot clip data.") print('yclip was applied to: ' + new)
def subtract_average(names, new_names=None, suffix=None, overwrite=None, median=None): old_names = pyspedas.tnames(names) if len(old_names) < 1: print('Subtract Average error: No pytplot names were provided.') return if suffix is None: if median: suffix = '-m' else: suffix = '-d' if overwrite is not None: n_names = old_names elif new_names is None: n_names = [s + suffix for s in old_names] else: n_names = new_names if isinstance(n_names, str): n_names = [n_names] if len(n_names) != len(old_names): n_names = [s + suffix for s in old_names] old_names = pyspedas.tnames(names) for i, old in enumerate(old_names): new = n_names[i] if new != old: pyspedas.tcopy(old, new) data = pytplot.data_quants[new].values dim = data.shape if median: if len(dim) == 1: data -= numpy.median(data, axis=0) else: for i in range(dim[1]): data[:, i] -= numpy.median(data[:, i], axis=0) ptype = 'Median' else: if len(dim) == 1: data -= numpy.mean(data, axis=0) else: for i in range(dim[1]): data[:, i] -= numpy.mean(data[:, i], axis=0) ptype = 'Mean' pytplot.data_quants[new].values = data print('Subtract ' + ptype + ' was applied to: ' + new)
def deriv_data(names, new_names=None, suffix=None, overwrite=None): """ Compute the derivative. Parameters ---------- names: str/list of str List of pytplot names. new_names: str/list of str, optional List of new_names for pytplot variables. If not given, then a suffix is applied. suffix: str, optional A suffix to apply. Default is '-avg'. overwrite: bool, optional Replace the existing tplot name. Returns ------- None. """ old_names = pyspedas.tnames(names) if len(old_names) < 1: print('deriv_data error: No pytplot names were provided.') return if suffix is None: suffix = '-der' if overwrite is not None: n_names = old_names elif new_names is None: n_names = [s + suffix for s in old_names] else: n_names = new_names if isinstance(n_names, str): n_names = [n_names] if len(n_names) != len(old_names): n_names = [s + suffix for s in old_names] for i, old in enumerate(old_names): new = n_names[i] if new != old: pyspedas.tcopy(old, new) data = pytplot.data_quants[new] data_new = data.differentiate('time').copy() pytplot.data_quants[new].values = data_new.values print('deriv_data was applied to: ' + new)
def tsmooth(names, width=10, median=None, preserve_nans=None, new_names=None, suffix=None, overwrite=None): old_names = pyspedas.tnames(names) if len(old_names) < 1: print('tsmooth error: No pytplot names were provided.') return if suffix is None: suffix = '-s' if overwrite is not None: n_names = old_names elif new_names is None: n_names = [s + suffix for s in old_names] else: n_names = new_names if isinstance(n_names, str): n_names = [n_names] if len(n_names) != len(old_names): n_names = [s + suffix for s in old_names] for i, old in enumerate(old_names): new = n_names[i] if new != old: pyspedas.tcopy(old, new) data = pytplot.data_quants[new].values dim = data.shape if len(dim) == 1: data = smooth(data, width=width, preserve_nans=preserve_nans) else: for k in range(dim[1]): data[:, k] = smooth(data[:, k], width=width, preserve_nans=preserve_nans) pytplot.data_quants[new].values = data print('tsmooth was applied to: ' + new)
def test_tcopy(self): store_data('test', data={'x': [1, 2, 3], 'y': [5, 5, 5]}) tcopy('test') tcopy('test', 'another-copy') t, d = get_data('test-copy') self.assertTrue(t.tolist() == [1, 2, 3]) self.assertTrue(d.tolist() == [5, 5, 5]) t, d = get_data('another-copy') self.assertTrue(t.tolist() == [1, 2, 3]) self.assertTrue(d.tolist() == [5, 5, 5]) # the following should gracefully error tcopy('doesnt exist', 'another-copy') tcopy(['another-copy', 'test'], 'another-copy')
def time_clip(names, time_start, time_end, new_names=None, suffix=None, overwrite=None): """ Clip data from time_start to time_end. Parameters: names: str/list of str List of pytplot names. time_start : float Start time. time_end : float End time. new_names: str/list of str, optional List of new_names for pytplot variables. If '', then pytplot variables are replaced. If not given, then a suffix is applied. suffix: str, optional A suffix to apply. Default is '-m'. overwrite: bool, optional Replace the existing tplot name. Returns ------- None. """ old_names = pyspedas.tnames(names) if len(old_names) < 1: print('Time clip error: No pytplot names were provided.') return if suffix is None: suffix = '-tclip' if overwrite is not None: n_names = old_names elif new_names is None: n_names = [s + suffix for s in old_names] else: n_names = new_names if isinstance(n_names, str): n_names = [n_names] if len(n_names) != len(old_names): n_names = [s + suffix for s in old_names] for j in range(len(old_names)): if old_names[j] != n_names[j]: pyspedas.tcopy(old_names[j], n_names[j]) alldata = pytplot.get_data(n_names[j]) time = alldata[0] data = alldata[1] index_start = 0 index_end = len(time) if index_end < 1: print('Time clip found empty list.') continue new_time = pyspedas.time_float(time) new_time_start = pyspedas.time_float(time_start) new_time_end = pyspedas.time_float(time_end) if new_time_start > new_time_end: print('Error: Start time is larger than end time.') continue if (new_time_start > new_time[-1]) or (new_time_end < new_time[0]): print('Time clip returns empty data.') continue if (new_time_start <= new_time[0]) and (new_time_end >= new_time[-1]): print('Time clip returns full data set.') continue for i in range(index_end): if new_time[i] >= new_time_start: index_start = i break found_end = index_end for i in range(index_start, index_end): if new_time[i] > new_time_end: found_end = i break index_end = found_end tmp_q = data_quants[n_names[j]] if 'v1' in tmp_q.coords.keys(): if len(tmp_q.coords['v1'].values.shape) == 2: v1_data = tmp_q.coords['v1'].values[index_start:index_end, :] else: v1_data = tmp_q.coords['v1'].values if 'v2' in tmp_q.coords.keys(): if len(tmp_q.coords['v2'].values.shape) == 2: v2_data = tmp_q.coords['v2'].values[index_start:index_end, :] else: v2_data = tmp_q.coords['v2'].values if 'v3' in tmp_q.coords.keys(): if len(tmp_q.coords['v3'].values.shape) == 2: v3_data = tmp_q.coords['v3'].values[index_start:index_end, :] else: v3_data = tmp_q.coords['v3'].values if 'v' in tmp_q.coords.keys(): if len(tmp_q.coords['v'].values.shape) == 2: v_data = tmp_q.coords['v'].values[index_start:index_end, :] else: v_data = tmp_q.coords['v'].values if 'spec_bins' in tmp_q.coords.keys(): if len(tmp_q.coords['spec_bins'].values.shape) == 2: v_data = tmp_q.coords['spec_bins']\ .values[index_start:index_end, :] else: v_data = tmp_q.coords['spec_bins'].values try: if 'v1' in tmp_q.coords.keys() and\ 'v2' in tmp_q.coords.keys() and\ 'v3' in tmp_q.coords.keys(): pytplot.store_data(n_names[j], data={ 'x': time[index_start:index_end], 'y': data[index_start:index_end, :, :, :], 'v1': v1_data, 'v2': v2_data, 'v3': v3_data }) elif 'v1' in tmp_q.coords.keys() and\ 'v2' in tmp_q.coords.keys(): pytplot.store_data(n_names[j], data={ 'x': time[index_start:index_end], 'y': data[index_start:index_end, :, :], 'v1': v1_data, 'v2': v2_data }) elif 'v1' in tmp_q.coords.keys(): pytplot.store_data(n_names[j], data={ 'x': time[index_start:index_end], 'y': data[index_start:index_end, :], 'v1': v1_data }) elif 'spec_bins' in tmp_q.coords.keys(): pytplot.store_data(n_names[j], data={ 'x': time[index_start:index_end], 'y': data[index_start:index_end, :], 'v': v_data }) elif 'v' in tmp_q.coords.keys(): pytplot.store_data(n_names[j], data={ 'x': time[index_start:index_end], 'y': data[index_start:index_end, :], 'v': v_data }) elif data.ndim == 1: pytplot.store_data(n_names[j], data={ 'x': time[index_start:index_end], 'y': data[index_start:index_end] }) else: pytplot.store_data(n_names[j], data={ 'x': time[index_start:index_end], 'y': data[index_start:index_end] }) except: print('Problem time clipping: ' + n_names[j]) continue print('Time clip was applied to: ' + n_names[j])
def subtract_average(names, new_names=None, suffix=None, overwrite=None, median=None): """ Subtracts the average or the median from data. Parameters ---------- names: str/list of str List of pytplot names. new_names: str/list of str, optional List of new_names for pytplot variables. If not given, then a suffix is applied. suffix: str, optional A suffix to apply. Default is '-d'. overwrite: bool, optional If set, then pytplot variables are replaced. median: float, optional If it is 0 or not set, then it computes the mean. Otherwise, it computes the median. Returns ------- None. """ old_names = pyspedas.tnames(names) if len(old_names) < 1: print('Subtract Average error: No pytplot names were provided.') return if suffix is None: if median: suffix = '-m' else: suffix = '-d' if overwrite is not None: n_names = old_names elif new_names is None: n_names = [s + suffix for s in old_names] else: n_names = new_names if isinstance(n_names, str): n_names = [n_names] if len(n_names) != len(old_names): n_names = [s + suffix for s in old_names] old_names = pyspedas.tnames(names) for i, old in enumerate(old_names): new = n_names[i] if new != old: pyspedas.tcopy(old, new) data = pytplot.data_quants[new].values dim = data.shape if median: if len(dim) == 1: data -= numpy.median(data, axis=0) else: for i in range(dim[1]): data[:, i] -= numpy.median(data[:, i], axis=0) ptype = 'Median' else: if len(dim) == 1: data -= numpy.mean(data, axis=0) else: for i in range(dim[1]): data[:, i] -= numpy.mean(data[:, i], axis=0) ptype = 'Mean' pytplot.data_quants[new].values = data print('Subtract ' + ptype + ' was applied to: ' + new)
def clean_spikes(names, nsmooth=10, thresh=0.3, sub_avg=False, new_names=None, suffix=None, overwrite=None): """ Clean spikes from data. Parameters ---------- names: str/list of str List of pytplot names. new_names: str/list of str, optional List of new_names for pytplot variables. If not given, then a suffix is applied. suffix: str, optional A suffix to apply. Default is '-avg'. overwrite: bool, optional Replace the existing tplot name. nsmooth: int, optional the number of data points for smoothing thresh: float, optional threshold value sub_avg: bool, optional if set, subtract the average value of the data prior to checking for spikes Returns ------- None. """ old_names = pyspedas.tnames(names) if len(old_names) < 1: print('clean_spikes error: No pytplot names were provided.') return if suffix is None: suffix = '-despike' if overwrite is not None: n_names = old_names elif new_names is None: n_names = [s + suffix for s in old_names] else: n_names = new_names if isinstance(n_names, str): n_names = [n_names] if len(n_names) != len(old_names): n_names = [s + suffix for s in old_names] for old_idx, old in enumerate(old_names): new = n_names[old_idx] tmp = new + '_tmp_data' # Create new if old != new: pyspedas.tcopy(old, new) # Perform subtract_average or just copy the values if sub_avg: subtract_average(new, new_names=tmp) else: pyspedas.tcopy(new, tmp) # Find spikes tmps = tmp + '-s' tsmooth(tmp, new_names=tmps, width=nsmooth) ds0 = pytplot.get_data(tmps) # smoothed out values ds = ds0[1] dor0 = pytplot.get_data(tmp) # original values d0 = dor0[1] dn = d0.copy() # final values dim = dn.shape if len(dim) == 1: # One dim data. for i in range(dim[0]): # compare smoothed out values to original values if abs(d0[i] - ds[i]) > thresh * abs(ds[i]): dn[i] = np.NaN # for spikes, set to NaN else: # More than one dim data. for j in range(dim[1]): # print("j = ", j) for i in range(dim[0]): # compare smoothed out values to original values if abs(d0[i, j] - ds[i, j]) > thresh * abs(ds[i, j]): dn[i, j] = np.NaN # for spikes, set to NaN # pytplot.data_quants[new] = d pytplot.replace_data(new, dn) # remove temp data del pytplot.data_quants[tmp] del pytplot.data_quants[tmps] print('clean_spikes was applied to: ' + new)
def yclip(names, ymin, ymax, flag=None, new_names=None, suffix=None, overwrite=None): """ Clip data values. Parameters ---------- names: str/list of str List of pytplot names. ymin: float Minimum value to clip. ymax: float Maximum value to clip. flag: float, optional Values outside (ymin, ymax) are replaced with flag. Default is float('nan'). new_names: str/list of str, optional List of new_names for pytplot variables. If not given, then a suffix is applied. suffix: str, optional A suffix to apply. Default is '-clip'. overwrite: bool, optional Replace the existing tplot name. Returns ------- None. """ old_names = pyspedas.tnames(names) if len(old_names) < 1: print('yclip error: No pytplot names were provided.') return if suffix is None: suffix = '-clip' if flag is None: flag = np.nan if overwrite is not None: n_names = old_names elif new_names is None: n_names = [s + suffix for s in old_names] else: n_names = new_names if isinstance(n_names, str): n_names = [n_names] if len(n_names) != len(old_names): n_names = [s + suffix for s in old_names] for i, old in enumerate(old_names): new = n_names[i] if new != old: pyspedas.tcopy(old, new) data = pytplot.data_quants[new].values try: for j, v in enumerate(data): try: iterator = enumerate(v) except TypeError: if not np.isnan(v) and (v <= ymin or v >= ymax): data[j] = flag else: for k, s in iterator: if not np.isnan(s) and (s <= ymin or s >= ymax): data[j][k] = flag except TypeError: # data Not itterable print("Cannot clip data.") print('yclip was applied to: ' + new)
def time_clip(names, time_start, time_end, new_names=None, suffix=None): old_names = pyspedas.tnames(names) if len(old_names) < 1: print('Time clip error: No pytplot names were provided.') return if suffix is None: suffix = '-tclip' if new_names is None: n_names = [s + suffix for s in old_names] elif new_names == '': n_names = old_names else: n_names = new_names if len(n_names) != len(old_names): n_names = [s + suffix for s in old_names] for j in range(len(old_names)): if old_names[j] != n_names[j]: pyspedas.tcopy(old_names[j], n_names[j]) time, data = pytplot.get_data(n_names[j]) index_start = 0 index_end = len(time) if index_end < 1: print('Time clip found empty list.') continue new_time = pyspedas.time_float(time) new_time_start = pyspedas.time_float(time_start) new_time_end = pyspedas.time_float(time_end) if new_time_start > new_time_end: print('Error: Start time is larger than end time.') continue if (new_time_start > new_time[-1]) or (new_time_end < new_time[0]): print('Time clip returns empty data.') continue if (new_time_start <= new_time[0]) and (new_time_end >= new_time[-1]): print('Time clip returns full data set.') continue for i in range(index_end): if new_time[i] >= new_time_start: index_start = i break found_end = index_end for i in range(index_start, index_end): if new_time[i] > new_time_end: found_end = i break index_end = found_end pytplot.store_data(n_names[j], data={'x': time[index_start:index_end], 'y': data[index_start:index_end, :]}) print('Time clip was applied to: ' + n_names[j])
def time_clip(names, time_start, time_end, new_names=None, suffix=None): old_names = pyspedas.tnames(names) if len(old_names) < 1: print('Time clip error: No pytplot names were provided.') return if suffix is None: suffix = '-tclip' if new_names is None: n_names = [s + suffix for s in old_names] elif new_names == '': n_names = old_names else: n_names = new_names if len(n_names) != len(old_names): n_names = [s + suffix for s in old_names] for j in range(len(old_names)): if old_names[j] != n_names[j]: pyspedas.tcopy(old_names[j], n_names[j]) alldata = pytplot.get_data(n_names[j]) time = alldata[0] data = alldata[1] index_start = 0 index_end = len(time) if index_end < 1: print('Time clip found empty list.') continue new_time = pyspedas.time_float(time) new_time_start = pyspedas.time_float(time_start) new_time_end = pyspedas.time_float(time_end) if new_time_start > new_time_end: print('Error: Start time is larger than end time.') continue if (new_time_start > new_time[-1]) or (new_time_end < new_time[0]): print('Time clip returns empty data.') continue if (new_time_start <= new_time[0]) and (new_time_end >= new_time[-1]): print('Time clip returns full data set.') continue for i in range(index_end): if new_time[i] >= new_time_start: index_start = i break found_end = index_end for i in range(index_start, index_end): if new_time[i] > new_time_end: found_end = i break index_end = found_end tmp_dquant = data_quants[n_names[j]] if 'v1' in tmp_dquant.coords.keys(): if len(tmp_dquant.coords['v1'].values.shape) is 2: v1_data = tmp_dquant.coords['v1'].values[ index_start:index_end, :] else: v1_data = tmp_dquant.coords['v1'].values if 'v2' in tmp_dquant.coords.keys(): if len(tmp_dquant.coords['v2'].values.shape) is 2: v2_data = tmp_dquant.coords['v2'].values[ index_start:index_end, :] else: v2_data = tmp_dquant.coords['v2'].values if 'v3' in tmp_dquant.coords.keys(): if len(tmp_dquant.coords['v3'].values.shape) is 2: v3_data = tmp_dquant.coords['v3'].values[ index_start:index_end, :] else: v3_data = tmp_dquant.coords['v3'].values if 'v' in tmp_dquant.coords.keys(): if len(tmp_dquant.coords['v'].values.shape) is 2: v_data = tmp_dquant.coords['v'].values[ index_start:index_end, :] else: v_data = tmp_dquant.coords['v'].values if 'spec_bins' in tmp_dquant.coords.keys(): if len(tmp_dquant.coords['spec_bins'].values.shape) is 2: v_data = tmp_dquant.coords['spec_bins'].values[ index_start:index_end, :] else: v_data = tmp_dquant.coords['spec_bins'].values try: if 'v1' in tmp_dquant.coords.keys( ) and 'v2' in tmp_dquant.coords.keys( ) and 'v3' in tmp_dquant.coords.keys(): pytplot.store_data(n_names[j], data={ 'x': time[index_start:index_end], 'y': data[index_start:index_end, :, :, :], 'v1': v1_data, 'v2': v2_data, 'v3': v3_data }) elif 'v1' in tmp_dquant.coords.keys( ) and 'v2' in tmp_dquant.coords.keys(): pytplot.store_data(n_names[j], data={ 'x': time[index_start:index_end], 'y': data[index_start:index_end, :, :], 'v1': v1_data, 'v2': v2_data }) elif 'v1' in tmp_dquant.coords.keys(): pytplot.store_data(n_names[j], data={ 'x': time[index_start:index_end], 'y': data[index_start:index_end, :], 'v1': v1_data }) elif 'spec_bins' in tmp_dquant.coords.keys(): pytplot.store_data(n_names[j], data={ 'x': time[index_start:index_end], 'y': data[index_start:index_end, :], 'v': v_data }) elif 'v' in tmp_dquant.coords.keys(): pytplot.store_data(n_names[j], data={ 'x': time[index_start:index_end], 'y': data[index_start:index_end, :], 'v': v_data }) elif data.ndim == 1: pytplot.store_data(n_names[j], data={ 'x': time[index_start:index_end], 'y': data[index_start:index_end] }) else: pytplot.store_data(n_names[j], data={ 'x': time[index_start:index_end], 'y': data[index_start:index_end, ...] }) except IndexError: print('Problem time clipping: ' + n_names[j]) continue print('Time clip was applied to: ' + n_names[j])
def clean_spikes(names, nsmooth=10, thresh=0.3, sub_avg=False, new_names=None, suffix=None, overwrite=None): old_names = pyspedas.tnames(names) if len(old_names) < 1: print('clean_spikes error: No pytplot names were provided.') return if suffix is None: suffix = '-despike' if overwrite is not None: n_names = old_names elif new_names is None: n_names = [s + suffix for s in old_names] else: n_names = new_names if isinstance(n_names, str): n_names = [n_names] if len(n_names) != len(old_names): n_names = [s + suffix for s in old_names] for i, old in enumerate(old_names): new = n_names[i] tmp = new + '_tmp_data' # Create new if old != new: pyspedas.tcopy(old, new) # Perform subtract_average or just copy the values if sub_avg: subtract_average(new, new_names=tmp) else: pyspedas.tcopy(new, tmp) # Find spikes tmps = tmp + '-s' tsmooth(tmp, new_names=tmps, width=nsmooth) ds = pytplot.data_quants[tmps].values # smoothed out values d0 = pytplot.data_quants[tmp].values # original values d = pytplot.data_quants[new] # final values dim = d.shape for j in range(dim[1]): # print("j = ", j) for i in range(dim[0]): # compare smoothed out values to original values if abs(d0[i, j] - ds[i, j]) > thresh * abs(ds[i, j]): d[i, j] = np.NaN # for spikes, set to NaN pytplot.data_quants[new] = d # remove temp data del pytplot.data_quants[tmp] del pytplot.data_quants[tmps] print('clean_spikes was applied to: ' + new)
def tsmooth(names, width=10, median=None, preserve_nans=None, new_names=None, suffix=None, overwrite=None): """ Smooths data. Parameters ---------- names: str/list of str List of pytplot names. width: int, optional Data window to use for smoothing. The default is 10. median: bool, optional Apply the median as well. preserve_nans: bool, optional If None, then replace NaNs. new_names: str/list of str, optional List of new_names for pytplot variables. If not given, then a suffix is applied. suffix: str, optional A suffix to apply. Default is '-s'. overwrite: bool, optional Replace the existing tplot name. Returns ------- None. """ old_names = pyspedas.tnames(names) if len(old_names) < 1: print('tsmooth error: No pytplot names were provided.') return if suffix is None: suffix = '-s' if overwrite is not None: n_names = old_names elif new_names is None: n_names = [s + suffix for s in old_names] else: n_names = new_names if isinstance(n_names, str): n_names = [n_names] if len(n_names) != len(old_names): n_names = [s + suffix for s in old_names] for i, old in enumerate(old_names): new = n_names[i] if new != old: pyspedas.tcopy(old, new) data = pytplot.data_quants[new].values dim = data.shape if len(dim) == 1: data = smooth(data, width=width, preserve_nans=preserve_nans) else: for k in range(dim[1]): data[:, k] = smooth(data[:, k], width=width, preserve_nans=preserve_nans) pytplot.data_quants[new].values = data print('tsmooth was applied to: ' + new)