def deserialize_dispatch_seq(channel_ser): """Returns a list of list of channel pairs, as created by serialize dispatch_seq Parameters: =========== channel_ser: JSON serialization of the channel pairs Returns: ======== List of list of channel pairs """ dispatch_seq = [] for pair_list in channel_ser: new_list = [] for pair in pair_list: ch1 = channel(pair["ch1"]["dev"], pair["ch1"]["ch_v"], pair["ch1"]["ch_h"]) ch2 = channel(pair["ch2"]["dev"], pair["ch2"]["ch_v"], pair["ch2"]["ch_h"]) new_list.append(channel_pair(ch1, ch2)) dispatch_seq.append(new_list) return dispatch_seq # End of file backend.py
def test_cross_phase(self): """compares output of cross_phase fluctana routine to output from cross_phase diagnostic kernel""" def cross_phase_rmc(): #import sys, os #sys.path.append(os.pardir) sys.path.append("/global/homes/r/rkube/repos/fluctana_rmc") from fluctana import FluctAna, KstarEcei # HOW TO RUN # ./python3 check_correlation.py 10186 [15.9,16] ECEI_L1303 ECEI_L1403 shot = 18431 trange = [-0.1, -0.08] clist = [['ECEI_L1102'], ['ECEI_L0906']] # call fluctana A = FluctAna() # add data A.add_data(KstarEcei( shot=shot, clist=clist[0], data_path='/global/cscratch1/sd/rkube/KSTAR/kstar_streaming/'), trange=trange, norm=1) A.add_data(KstarEcei( shot=shot, clist=clist[1], data_path='/global/cscratch1/sd/rkube/KSTAR/kstar_streaming/'), trange=trange, norm=1) # do fft; full = 1 A.fftbins(nfft=512, window='hann', overlap=0.5, detrend=1, full=1) # calculate correlation using data sets done and dtwo. results are saved in A.Dlist[dtwo].val A.cross_phase(done=0, dtwo=1) return (A) A = cross_phase_rmc() # Call the diagnostic kernel ch_it = [channel_pair(self._ch1, self._ch2)] res, _ = cross_phase(self._fft_data, ch_it, self._config_fft["fft_params"], None) res = np.squeeze(res) self.assertTrue( np.linalg.norm(A.Dlist[1].val[0][1:] - res) / np.linalg.norm(res) < 512)
def __init__(self, task_config, fft_config, ecei_config): """Initialize the object with a fixed channel list, a fixed name of the analysis to be performed and a fixed set of parameters for the analysis routine. Inputs: ======= channel_range: list of strings, defines the name of the channels. This should probably match the name of the channels in the BP file. task_config: dict, defines parameters of the analysis to be performed fft_config dict, gives parameters of the fourier-transformed data """ # Stores the description of the task. This can be arbitrary self.description = task_config["description"] # Stores the name of the analysis we are going to execute self.analysis = task_config["analysis"] # Parse the reference and cross channels. self.ref_channels = channel_range.from_str(task_config["ref_channels"]) # These channels serve as the cross-data for the spectral diagnostics self.cmp_channels = channel_range.from_str(task_config["cmp_channels"]) self.task_config = task_config self.fft_config = fft_config self.ecei_config = ecei_config self.storage_scheme = { "ref_channels": self.ref_channels.to_str(), "cmp_channels": self.cmp_channels.to_str() } # Construct a list of unique channels # F.ex. we have ref_channels [(1,1), (1,2), (1,3)] and cmp_channels = [(1,1), (1,2)] # The unique list of channels is then # (1,1) x (1,1), (1,1) x (1,2) # (1,2) x (1,2) !!! Omit (1,2) x (1,1) # (1,3) x (1,1) # (1,3) x (1,2) channel_pairs = [ channel_pair(cr, cx) for cr in self.ref_channels for cx in self.cmp_channels ] # Make a list, so that we don't exhause the iterator after the first call. self.unique_channels = list( more_itertools.distinct_combinations(channel_pairs, 1)) self.channel_chunk_size = task_config["channel_chunk_size"]
def __init__(self, task_config, fft_config, ecei_config, storage_config): """Initialize the object with a fixed channel list, a fixed name of the analysis to be performed and a fixed set of parameters for the analysis routine. Inputs: ======= task_config: dict, defines parameters of the analysis to be performed fft_config: dict, gives parameters of the fourier-transformed data ecei_config: dict, information on ecei diagnostic """ self.task_config = task_config self.ecei_config = ecei_config self.storage_config = storage_config self.logger = logging.getLogger("simple") # Stores the description of the task. This can be arbitrary self.description = task_config["task_description"] # Stores the name of the analysis we are going to execute self.analysis = task_config["analysis"] if self.analysis == "cross_phase": self.kernel = kernel_crossphase_64_cy elif self.analysis == "cross_power": self.kernel = kernel_crosspower_64_cy elif self.analysis == "cross_correlation": self.kernel = kernel_crosscorr elif self.analysis == "coherence": self.kernel = kernel_coherence_64_cy elif self.analysis == "skw": self.kernel = kernel_skw elif self.analysis == "bicoherence": self.kernel = kernel_bicoherence elif self.analysis == "null": self.kernel = kernel_null else: raise NameError(f"Unknown analysis task {self.analysis}") # Parse the reference and cross channels. self.ref_channels = channel_range.from_str(task_config["ref_channels"]) # These channels serve as the cross-data for the spectral diagnostics self.cmp_channels = channel_range.from_str(task_config["cmp_channels"]) # Construct a list of unique channels # F.ex. we have ref_channels [(1,1), (1,2), (1,3)] and cmp_channels = [(1,1), (1,2)] # The unique list of channels is then # (1,1) x (1,1), (1,1) x (1,2) # (1,2) x (1,2) !!! Omits (1,2) x (1,1) # (1,3) x (1,1) # (1,3) x (1,2) channel_pairs = [channel_pair(cr, cx) for cr in self.ref_channels for cx in self.cmp_channels] # Make a list, so that we don't exhaust the iterator after the first call. self.unique_channels = [i[0] for i in more_itertools.distinct_combinations(channel_pairs, 1)] # Number of channel pairs per future self.channel_chunk_size = task_config["channel_chunk_size"] # Total number of chunks, i.e. the number of futures appended to the list per call to calculate self.num_chunks = (len(self.unique_channels) + self.channel_chunk_size - 1) // self.channel_chunk_size # Get the configuration from task_fft_scipy, but don't store the object. fft_config["fsample"] = ecei_config["SampleRate"] * 1e3 self.my_fft = task_fft_scipy(self.channel_chunk_size, fft_config, normalize=True, detrend=True) self.fft_params = self.my_fft.get_fft_params() self.storage_backend = None if self.storage_config["backend"] == "numpy": self.storage_backend = backends.backend_numpy(self.storage_config) elif self.storage_config["backend"] == "mongo": self.storage_backend = backends.backend_mongodb(self.storage_config) elif self.storage_config["backend"] == "null": self.storage_backend = backends.backend_null(self.storage_config) else: raise NameError(f"Unknown storage backend requested: {self.storage_config}") self.storage_backend.store_metadata(self.task_config, self.get_dispatch_sequence())
def test_results(self): def fftbins(x, dt, nfft, window, overlap, do_detrend, full): # IN : 1 x tnum data # OUT : bins x faxis fftdata tnum = len(x) bins = int(np.fix((int(tnum / nfft) - overlap) / (1.0 - overlap))) win = np.hanning(nfft) #bins, win = fft_window(tnum, nfft, window, overlap) win_factor = np.mean(win**2) # window factors print("***fftbins: win_factor = {0:f}".format(win_factor)) # make an x-axis # ax = np.fft.fftfreq(nfft, d=dt) # full 0~fN -fN~-f1 if np.mod(nfft, 2) == 0: # even nfft ax = np.hstack([ ax[0:int(nfft / 2)], -(ax[int(nfft / 2)]), ax[int(nfft / 2):nfft] ]) if full == 1: # full shift to -fN ~ 0 ~ fN ax = np.fft.fftshift(ax) else: # half 0~fN ax = ax[0:int(nfft / 2 + 1)] # make fftdata if full == 1: # full shift to -fN ~ 0 ~ fN if np.mod(nfft, 2) == 0: # even nfft fftdata = np.zeros((bins, nfft + 1), dtype=np.complex_) else: # odd nfft fftdata = np.zeros((bins, nfft), dtype=np.complex_) else: # half 0 ~ fN fftdata = np.zeros((bins, int(nfft / 2 + 1)), dtype=np.complex_) for b in range(bins): idx1 = int(b * np.fix(nfft * (1 - overlap))) idx2 = idx1 + nfft #print("***bin {0:d}, idx1 = {1:d}, idx2 = {2:d}".format(b, idx1, idx2)) sx = x[idx1:idx2] if do_detrend == 1: sx = detrend(sx, type='linear') sx = detrend(sx, type='constant') # subtract mean sx = sx * win # apply window function # get fft SX = np.fft.fft(sx, n=nfft) / nfft # divide by the length if np.mod(nfft, 2) == 0: # even nfft SX = np.hstack([ SX[0:int(nfft / 2)], np.conj(SX[int(nfft / 2)]), SX[int(nfft / 2):nfft] ]) if full == 1: # shift to -fN ~ 0 ~ fN SX = np.fft.fftshift(SX) else: # half 0 ~ fN SX = SX[0:int(nfft / 2 + 1)] fftdata[b, :] = SX return ax, fftdata, win_factor def test_rmc(): sys.path.append("/global/homes/r/rkube/repos/fluctana_rmc") from fluctana import FluctAna, KstarEcei # HOW TO RUN # ./python3 check_correlation.py 10186 [15.9,16] ECEI_L1303 ECEI_L1403 shot = 18431 trange = [-0.1, -0.08] clist = [['ECEI_L1102'], ['ECEI_L0906']] # call fluctana A = FluctAna() # add data A.add_data(KstarEcei( shot=shot, clist=clist[0], data_path='/global/cscratch1/sd/rkube/KSTAR/kstar_streaming/'), trange=trange, norm=1) A.add_data(KstarEcei( shot=shot, clist=clist[1], data_path='/global/cscratch1/sd/rkube/KSTAR/kstar_streaming/'), trange=trange, norm=1) # list data A.list_data() # do fft; full = 1 A.fftbins(nfft=512, window='hann', overlap=0.5, detrend=0, full=1) # calculate correlation using data sets done and dtwo. results are saved in A.Dlist[dtwo].val A.correlation(done=0, dtwo=1) # plot the results; dnum = data set number, cnl = channel number list to plot #A.mplot(dnum=1,cnl=range(len(A.Dlist[1].clist)),type='val') return (A) tic = timeit.default_timer() A = test_rmc() toc = timeit.default_timer() print(f"Fluctana takes {(toc - tic):6.4f}s") L1102_fa = np.squeeze(A.Dlist[0].data) L1102_ax, L1102_ft, win_factor = fftbins(L1102_fa, dt=2e-6, nfft=512, window="hann", overlap=0.5, do_detrend=1, full=1) print(L1102_ft.shape) L0906_fa = np.squeeze(A.Dlist[1].data) L0906ax, L0906_ft, win_factor = fftbins(L0906_fa, dt=2e-6, nfft=512, window="hann", overlap=0.5, do_detrend=1, full=1) ################################################################################################################ # # Set up for distributed analysis # ################################################################################################################ c1 = channel('L', 11, 2) c2 = channel('L', 9, 6) print("Channel 1: ", c1, ", idx = ", c1.idx()) print("Channel 2: ", c2, ", idx = ", c2.idx()) with np.load("test_data/io_array_tr_s0001.npz") as df: # Load transformed data, as generated by datareader io_array_tr = df["io_array"] print("io_array_tr.shape = ", io_array_tr.shape) print("io_array_tr.shape = ", io_array_tr.shape) print(f"Channel idx for {c1}: {c1.idx()}, {c2}: {c2.idx()}") with open("tests_analysis/config_fft.json", "r") as df: config_fft = json.load(df) config_fft["fft_params"][ "fsample"] = config_fft["ECEI_cfg"]["SampleRate"] * 1e3 config_fft["fft_params"]["nfft"] = 512 win = get_window(config_fft["fft_params"]["window"], config_fft["fft_params"]["nfft"]) win_factor = (win**2).mean() print(f"win_Factor = {win_factor}") config_fft["fft_params"]["win_factor"] = win_factor my_fft = task_fft_scipy(10_000, config_fft["fft_params"], normalize=True, detrend=True) fft_data = my_fft.do_fft_local(io_array_tr) assert (np.linalg.norm(io_array_tr[c1.idx(), :] - L1102_fa) / np.linalg.norm(L1102_fa) < 1e-6) assert (np.linalg.norm(io_array_tr[c2.idx(), :] - L0906_fa) / np.linalg.norm(L0906_fa) < 1e-6) # Call the ch_it = [channel_pair(c1, c2)] res, _ = cross_corr(fft_data, ch_it, config_fft["fft_params"], None) res = np.squeeze(res) assert (np.linalg.norm(A.Dlist[1].val[0][1:] - res * 128) / np.linalg.norm(res) < 512)
def test_answer(): print( "Check that the mapping from v and h to a linear channel number is correct" ) ch_idx = 1 for vv in range(1, 25): for hh in range(1, 9): assert (channels.ch_num_to_vh(ch_idx) == (vv, hh)) assert (channels.ch_vh_to_num(vv, hh) == ch_idx) ch_idx += 1 assert (ch_idx == 193) print("Testing channel.from_str()") for _ in range(50): dd_list = ['L', 'H', 'G', 'HT', 'GR', 'HR'] dd = dd_list[np.random.randint(0, len(dd_list))] vv = np.random.randint(1, 25) hh = np.random.randint(1, 9) ch_str = f"{dd:s}{vv:02d}{hh:02d}" assert (channels.channel.from_str(ch_str) == channels.channel( dd, vv, hh)) print("Testing serialization") for _ in range(50): dd_list = ['L', 'H', 'G', 'HT', 'GR', 'HR'] dd = dd_list[np.random.randint(0, len(dd_list))] vv = np.random.randint(1, 25) hh = np.random.randint(1, 9) ch = channels.channel(dd, vv, hh) assert (ch == channels.channel.from_json(ch.to_json())) print("Testing equality functions of channel_pair") ch1 = channels.channel("G", 2, 3) ch2 = channels.channel("G", 4, 6) assert (channels.channel_pair(ch1, ch2) == channels.channel_pair(ch2, ch1)) assert (hash(channels.channel_pair(ch1, ch2)) == hash( channels.channel_pair(ch2, ch1))) print("Testing channel pair serialization") for _ in range(50): dd_list = ['L', 'H', 'G', 'HT', 'GR', 'HR'] dd = dd_list[np.random.randint(0, len(dd_list))] vv = np.random.randint(1, 25) hh = np.random.randint(1, 9) ch1 = channels.channel(dd, vv, hh) vv = np.random.randint(1, 25) hh = np.random.randint(1, 9) ch2 = channels.channel(dd, vv, hh) pair = channels.channel_pair(ch1, ch2) assert (channels.channel_pair.from_json(pair.to_json()) == pair) print("Testing channel_range") print("Linear selection:") vv, hh = channels.ch_num_to_vh(1) ch1 = channels.channel("L", vv, hh) vv, hh = channels.ch_num_to_vh(192) ch2 = channels.channel("L", vv, hh) crg = channels.channel_range(ch1, ch2, "linear") i = 1 for c in crg: assert (c.idx() == i) i -= -1 print("Testing sets of channel pairs") print( "Note that this test can fail if duplicate pairs are inserted in chpair_list" ) print("upon generation") num_unq_pairs = 20 # Generate random channel pairs ch1_list = [ channels.channel("L", vv, hh) for vv, hh in zip(np.random.randint(1, 25, num_unq_pairs), np.random.randint(1, 9, num_unq_pairs)) ] ch2_list = [ channels.channel("L", vv, hh) for vv, hh in zip(np.random.randint(1, 25, num_unq_pairs), np.random.randint(1, 9, num_unq_pairs)) ] chpair_list = [ channels.channel_pair(ch1, ch2) for ch1, ch2 in zip(ch1_list, ch2_list) ] assert (len(chpair_list) == num_unq_pairs) # Now we add random duplicates num_duplicates = 0 for _ in range(num_duplicates): # Select a random channel pair to duplicate. Note that len(chpair_list) increases in each # iteration as we append chidx = np.random.randint(0, len(chpair_list)) # Append this channel pair to the list. Take a 50/50 chance to switch the channels too. ch2, ch1 = chpair_list[chidx] if np.random.randint(0, 2) == True: chpair_list.append(channels.channel_pair(ch2, ch1)) else: chpair_list.append(channels.channel_pair(ch1, ch2)) assert (len(chpair_list) == num_unq_pairs + num_duplicates) # Convert chpair_list, which includes 19 duplicates, to a set. chpair_set = set(chpair_list) assert (len(chpair_set) == num_unq_pairs) # # Test iteration over channel ranges using unqiue channel pairs # res = [channel_pair(c1, c2) for c1 in crg1 for c2 in crg2] # # res contains duplicate paris, f.ex. (L0202, L0203) and (L0203, L0202) # unq = unique_everseen(res) # unq_list = list(unq) # print("Non-unique: {0:d}, unique channel pairs: {1:d}".format(len(res), len(unq_list))) # End of file test_channels.py
with np.load("/global/homes/r/rkube/repos/delta/test_data/fft_array_s0001.npz" ) as df: fft_data = df["fft_data"] fft_data_64 = np.ascontiguousarray(fft_data) fft_data_32 = np.require(fft_data_64, dtype=np.complex64, requirements=['A', 'C']) ################################################### # Generate channels to iterate over ref_chrg = channel_range.from_str("L0101-2408") cmp_chrg = channel_range.from_str("L0101-2408") channel_pairs = [channel_pair(cr, cx) for cr in ref_chrg for cx in cmp_chrg] unique_channels = [ i[0] for i in more_itertools.distinct_combinations(channel_pairs, 1) ] #ch_it = [channel_pair(channel("L", i, 1), channel("L", i, 2)) for i in range(1, 25)] #ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 3)) for i in range(1, 25)] #h_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 4)) for i in range(1, 25)] #ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 4)) for i in range(1, 25)] #ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 5)) for i in range(1, 25)] #ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 6)) for i in range(1, 25)] #ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 7)) for i in range(1, 25)] #ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 8)) for i in range(1, 25)] ch1_idx_arr = np.array([int(ch_pair.ch1.idx()) for ch_pair in unique_channels], dtype=np.uint64)
# Encoding: UTF-8 """ Find out how to get a channel list from tget_dispatch_sequence and pickle it """ import sys sys.path.append("/global/homes/r/rkube/repos/delta") import more_itertools from itertools import chain from analysis.channels import channel, channel_range, channel_pair ref_channels = channel_range.from_str("L0101-2408") cmp_channels = channel_range.from_str("L0101-2408") channel_pairs = [ channel_pair(ref, cmp) for ref in ref_channels for cmp in cmp_channels ] unique_channels = list(more_itertools.distinct_combinations(channel_pairs, 1)) print("Number of channel pairs: {0:d}".format(len(channel_pairs))) print("Unique channel paits: {0:d}".format(len(unique_channels))) ch_list = [u[0] for u in unqiue_channels] np.savez("dispatch_seq.npz", ch_list=ch_list) # End of file test_channel_it.py