def deserialize_dispatch_seq(channel_ser):
"""Returns a list of list of channel pairs, as created by serialize dispatch_seq
Parameters:
===========
channel_ser: JSON serialization of the channel pairs
Returns:
========
List of list of channel pairs
"""
dispatch_seq = []
for pair_list in channel_ser:
new_list = []
for pair in pair_list:
ch1 = channel(pair["ch1"]["dev"], pair["ch1"]["ch_v"],
pair["ch1"]["ch_h"])
ch2 = channel(pair["ch2"]["dev"], pair["ch2"]["ch_v"],
pair["ch2"]["ch_h"])
new_list.append(channel_pair(ch1, ch2))
dispatch_seq.append(new_list)
return dispatch_seq
# End of file backend.py
def test_cross_phase(self):
"""compares output of cross_phase fluctana routine to output from
cross_phase diagnostic kernel"""
def cross_phase_rmc():
#import sys, os
#sys.path.append(os.pardir)
sys.path.append("/global/homes/r/rkube/repos/fluctana_rmc")
from fluctana import FluctAna, KstarEcei
# HOW TO RUN
# ./python3 check_correlation.py 10186 [15.9,16] ECEI_L1303 ECEI_L1403
shot = 18431
trange = [-0.1, -0.08]
clist = [['ECEI_L1102'], ['ECEI_L0906']]
# call fluctana
A = FluctAna()
# add data
A.add_data(KstarEcei(
shot=shot,
clist=clist[0],
data_path='/global/cscratch1/sd/rkube/KSTAR/kstar_streaming/'),
trange=trange,
norm=1)
A.add_data(KstarEcei(
shot=shot,
clist=clist[1],
data_path='/global/cscratch1/sd/rkube/KSTAR/kstar_streaming/'),
trange=trange,
norm=1)
# do fft; full = 1
A.fftbins(nfft=512, window='hann', overlap=0.5, detrend=1, full=1)
# calculate correlation using data sets done and dtwo. results are saved in A.Dlist[dtwo].val
A.cross_phase(done=0, dtwo=1)
return (A)
A = cross_phase_rmc()
# Call the diagnostic kernel
ch_it = [channel_pair(self._ch1, self._ch2)]
res, _ = cross_phase(self._fft_data, ch_it,
self._config_fft["fft_params"], None)
res = np.squeeze(res)
self.assertTrue(
np.linalg.norm(A.Dlist[1].val[0][1:] - res) /
np.linalg.norm(res) < 512)
def __init__(self, task_config, fft_config, ecei_config):
"""Initialize the object with a fixed channel list, a fixed name of the analysis to be performed
and a fixed set of parameters for the analysis routine.
Inputs:
=======
channel_range: list of strings, defines the name of the channels. This should probably match the
name of the channels in the BP file.
task_config: dict, defines parameters of the analysis to be performed
fft_config dict, gives parameters of the fourier-transformed data
"""
# Stores the description of the task. This can be arbitrary
self.description = task_config["description"]
# Stores the name of the analysis we are going to execute
self.analysis = task_config["analysis"]
# Parse the reference and cross channels.
self.ref_channels = channel_range.from_str(task_config["ref_channels"])
# These channels serve as the cross-data for the spectral diagnostics
self.cmp_channels = channel_range.from_str(task_config["cmp_channels"])
self.task_config = task_config
self.fft_config = fft_config
self.ecei_config = ecei_config
self.storage_scheme = {
"ref_channels": self.ref_channels.to_str(),
"cmp_channels": self.cmp_channels.to_str()
}
# Construct a list of unique channels
# F.ex. we have ref_channels [(1,1), (1,2), (1,3)] and cmp_channels = [(1,1), (1,2)]
# The unique list of channels is then
# (1,1) x (1,1), (1,1) x (1,2)
# (1,2) x (1,2) !!! Omit (1,2) x (1,1)
# (1,3) x (1,1)
# (1,3) x (1,2)
channel_pairs = [
channel_pair(cr, cx) for cr in self.ref_channels
for cx in self.cmp_channels
]
# Make a list, so that we don't exhause the iterator after the first call.
self.unique_channels = list(
more_itertools.distinct_combinations(channel_pairs, 1))
self.channel_chunk_size = task_config["channel_chunk_size"]
def __init__(self, task_config, fft_config, ecei_config, storage_config):
"""Initialize the object with a fixed channel list, a fixed name of the analysis to be performed
and a fixed set of parameters for the analysis routine.
Inputs:
=======
task_config: dict, defines parameters of the analysis to be performed
fft_config: dict, gives parameters of the fourier-transformed data
ecei_config: dict, information on ecei diagnostic
"""
self.task_config = task_config
self.ecei_config = ecei_config
self.storage_config = storage_config
self.logger = logging.getLogger("simple")
# Stores the description of the task. This can be arbitrary
self.description = task_config["task_description"]
# Stores the name of the analysis we are going to execute
self.analysis = task_config["analysis"]
if self.analysis == "cross_phase":
self.kernel = kernel_crossphase_64_cy
elif self.analysis == "cross_power":
self.kernel = kernel_crosspower_64_cy
elif self.analysis == "cross_correlation":
self.kernel = kernel_crosscorr
elif self.analysis == "coherence":
self.kernel = kernel_coherence_64_cy
elif self.analysis == "skw":
self.kernel = kernel_skw
elif self.analysis == "bicoherence":
self.kernel = kernel_bicoherence
elif self.analysis == "null":
self.kernel = kernel_null
else:
raise NameError(f"Unknown analysis task {self.analysis}")
# Parse the reference and cross channels.
self.ref_channels = channel_range.from_str(task_config["ref_channels"])
# These channels serve as the cross-data for the spectral diagnostics
self.cmp_channels = channel_range.from_str(task_config["cmp_channels"])
# Construct a list of unique channels
# F.ex. we have ref_channels [(1,1), (1,2), (1,3)] and cmp_channels = [(1,1), (1,2)]
# The unique list of channels is then
# (1,1) x (1,1), (1,1) x (1,2)
# (1,2) x (1,2) !!! Omits (1,2) x (1,1)
# (1,3) x (1,1)
# (1,3) x (1,2)
channel_pairs = [channel_pair(cr, cx) for cr in self.ref_channels for cx in self.cmp_channels]
# Make a list, so that we don't exhaust the iterator after the first call.
self.unique_channels = [i[0] for i in more_itertools.distinct_combinations(channel_pairs, 1)]
# Number of channel pairs per future
self.channel_chunk_size = task_config["channel_chunk_size"]
# Total number of chunks, i.e. the number of futures appended to the list per call to calculate
self.num_chunks = (len(self.unique_channels) + self.channel_chunk_size - 1) // self.channel_chunk_size
# Get the configuration from task_fft_scipy, but don't store the object.
fft_config["fsample"] = ecei_config["SampleRate"] * 1e3
self.my_fft = task_fft_scipy(self.channel_chunk_size, fft_config, normalize=True, detrend=True)
self.fft_params = self.my_fft.get_fft_params()
self.storage_backend = None
if self.storage_config["backend"] == "numpy":
self.storage_backend = backends.backend_numpy(self.storage_config)
elif self.storage_config["backend"] == "mongo":
self.storage_backend = backends.backend_mongodb(self.storage_config)
elif self.storage_config["backend"] == "null":
self.storage_backend = backends.backend_null(self.storage_config)
else:
raise NameError(f"Unknown storage backend requested: {self.storage_config}")
self.storage_backend.store_metadata(self.task_config, self.get_dispatch_sequence())
def test_results(self):
def fftbins(x, dt, nfft, window, overlap, do_detrend, full):
# IN : 1 x tnum data
# OUT : bins x faxis fftdata
tnum = len(x)
bins = int(np.fix((int(tnum / nfft) - overlap) / (1.0 - overlap)))
win = np.hanning(nfft)
#bins, win = fft_window(tnum, nfft, window, overlap)
win_factor = np.mean(win**2) # window factors
print("***fftbins: win_factor = {0:f}".format(win_factor))
# make an x-axis #
ax = np.fft.fftfreq(nfft, d=dt) # full 0~fN -fN~-f1
if np.mod(nfft, 2) == 0: # even nfft
ax = np.hstack([
ax[0:int(nfft / 2)], -(ax[int(nfft / 2)]),
ax[int(nfft / 2):nfft]
])
if full == 1: # full shift to -fN ~ 0 ~ fN
ax = np.fft.fftshift(ax)
else: # half 0~fN
ax = ax[0:int(nfft / 2 + 1)]
# make fftdata
if full == 1: # full shift to -fN ~ 0 ~ fN
if np.mod(nfft, 2) == 0: # even nfft
fftdata = np.zeros((bins, nfft + 1), dtype=np.complex_)
else: # odd nfft
fftdata = np.zeros((bins, nfft), dtype=np.complex_)
else: # half 0 ~ fN
fftdata = np.zeros((bins, int(nfft / 2 + 1)),
dtype=np.complex_)
for b in range(bins):
idx1 = int(b * np.fix(nfft * (1 - overlap)))
idx2 = idx1 + nfft
#print("***bin {0:d}, idx1 = {1:d}, idx2 = {2:d}".format(b, idx1, idx2))
sx = x[idx1:idx2]
if do_detrend == 1:
sx = detrend(sx, type='linear')
sx = detrend(sx, type='constant') # subtract mean
sx = sx * win # apply window function
# get fft
SX = np.fft.fft(sx, n=nfft) / nfft # divide by the length
if np.mod(nfft, 2) == 0: # even nfft
SX = np.hstack([
SX[0:int(nfft / 2)],
np.conj(SX[int(nfft / 2)]), SX[int(nfft / 2):nfft]
])
if full == 1: # shift to -fN ~ 0 ~ fN
SX = np.fft.fftshift(SX)
else: # half 0 ~ fN
SX = SX[0:int(nfft / 2 + 1)]
fftdata[b, :] = SX
return ax, fftdata, win_factor
def test_rmc():
sys.path.append("/global/homes/r/rkube/repos/fluctana_rmc")
from fluctana import FluctAna, KstarEcei
# HOW TO RUN
# ./python3 check_correlation.py 10186 [15.9,16] ECEI_L1303 ECEI_L1403
shot = 18431
trange = [-0.1, -0.08]
clist = [['ECEI_L1102'], ['ECEI_L0906']]
# call fluctana
A = FluctAna()
# add data
A.add_data(KstarEcei(
shot=shot,
clist=clist[0],
data_path='/global/cscratch1/sd/rkube/KSTAR/kstar_streaming/'),
trange=trange,
norm=1)
A.add_data(KstarEcei(
shot=shot,
clist=clist[1],
data_path='/global/cscratch1/sd/rkube/KSTAR/kstar_streaming/'),
trange=trange,
norm=1)
# list data
A.list_data()
# do fft; full = 1
A.fftbins(nfft=512, window='hann', overlap=0.5, detrend=0, full=1)
# calculate correlation using data sets done and dtwo. results are saved in A.Dlist[dtwo].val
A.correlation(done=0, dtwo=1)
# plot the results; dnum = data set number, cnl = channel number list to plot
#A.mplot(dnum=1,cnl=range(len(A.Dlist[1].clist)),type='val')
return (A)
tic = timeit.default_timer()
A = test_rmc()
toc = timeit.default_timer()
print(f"Fluctana takes {(toc - tic):6.4f}s")
L1102_fa = np.squeeze(A.Dlist[0].data)
L1102_ax, L1102_ft, win_factor = fftbins(L1102_fa,
dt=2e-6,
nfft=512,
window="hann",
overlap=0.5,
do_detrend=1,
full=1)
print(L1102_ft.shape)
L0906_fa = np.squeeze(A.Dlist[1].data)
L0906ax, L0906_ft, win_factor = fftbins(L0906_fa,
dt=2e-6,
nfft=512,
window="hann",
overlap=0.5,
do_detrend=1,
full=1)
################################################################################################################
#
# Set up for distributed analysis
#
################################################################################################################
c1 = channel('L', 11, 2)
c2 = channel('L', 9, 6)
print("Channel 1: ", c1, ", idx = ", c1.idx())
print("Channel 2: ", c2, ", idx = ", c2.idx())
with np.load("test_data/io_array_tr_s0001.npz") as df:
# Load transformed data, as generated by datareader
io_array_tr = df["io_array"]
print("io_array_tr.shape = ", io_array_tr.shape)
print("io_array_tr.shape = ", io_array_tr.shape)
print(f"Channel idx for {c1}: {c1.idx()}, {c2}: {c2.idx()}")
with open("tests_analysis/config_fft.json", "r") as df:
config_fft = json.load(df)
config_fft["fft_params"][
"fsample"] = config_fft["ECEI_cfg"]["SampleRate"] * 1e3
config_fft["fft_params"]["nfft"] = 512
win = get_window(config_fft["fft_params"]["window"],
config_fft["fft_params"]["nfft"])
win_factor = (win**2).mean()
print(f"win_Factor = {win_factor}")
config_fft["fft_params"]["win_factor"] = win_factor
my_fft = task_fft_scipy(10_000,
config_fft["fft_params"],
normalize=True,
detrend=True)
fft_data = my_fft.do_fft_local(io_array_tr)
assert (np.linalg.norm(io_array_tr[c1.idx(), :] - L1102_fa) /
np.linalg.norm(L1102_fa) < 1e-6)
assert (np.linalg.norm(io_array_tr[c2.idx(), :] - L0906_fa) /
np.linalg.norm(L0906_fa) < 1e-6)
# Call the
ch_it = [channel_pair(c1, c2)]
res, _ = cross_corr(fft_data, ch_it, config_fft["fft_params"], None)
res = np.squeeze(res)
assert (np.linalg.norm(A.Dlist[1].val[0][1:] - res * 128) /
np.linalg.norm(res) < 512)
def test_answer():
print(
"Check that the mapping from v and h to a linear channel number is correct"
)
ch_idx = 1
for vv in range(1, 25):
for hh in range(1, 9):
assert (channels.ch_num_to_vh(ch_idx) == (vv, hh))
assert (channels.ch_vh_to_num(vv, hh) == ch_idx)
ch_idx += 1
assert (ch_idx == 193)
print("Testing channel.from_str()")
for _ in range(50):
dd_list = ['L', 'H', 'G', 'HT', 'GR', 'HR']
dd = dd_list[np.random.randint(0, len(dd_list))]
vv = np.random.randint(1, 25)
hh = np.random.randint(1, 9)
ch_str = f"{dd:s}{vv:02d}{hh:02d}"
assert (channels.channel.from_str(ch_str) == channels.channel(
dd, vv, hh))
print("Testing serialization")
for _ in range(50):
dd_list = ['L', 'H', 'G', 'HT', 'GR', 'HR']
dd = dd_list[np.random.randint(0, len(dd_list))]
vv = np.random.randint(1, 25)
hh = np.random.randint(1, 9)
ch = channels.channel(dd, vv, hh)
assert (ch == channels.channel.from_json(ch.to_json()))
print("Testing equality functions of channel_pair")
ch1 = channels.channel("G", 2, 3)
ch2 = channels.channel("G", 4, 6)
assert (channels.channel_pair(ch1, ch2) == channels.channel_pair(ch2, ch1))
assert (hash(channels.channel_pair(ch1, ch2)) == hash(
channels.channel_pair(ch2, ch1)))
print("Testing channel pair serialization")
for _ in range(50):
dd_list = ['L', 'H', 'G', 'HT', 'GR', 'HR']
dd = dd_list[np.random.randint(0, len(dd_list))]
vv = np.random.randint(1, 25)
hh = np.random.randint(1, 9)
ch1 = channels.channel(dd, vv, hh)
vv = np.random.randint(1, 25)
hh = np.random.randint(1, 9)
ch2 = channels.channel(dd, vv, hh)
pair = channels.channel_pair(ch1, ch2)
assert (channels.channel_pair.from_json(pair.to_json()) == pair)
print("Testing channel_range")
print("Linear selection:")
vv, hh = channels.ch_num_to_vh(1)
ch1 = channels.channel("L", vv, hh)
vv, hh = channels.ch_num_to_vh(192)
ch2 = channels.channel("L", vv, hh)
crg = channels.channel_range(ch1, ch2, "linear")
i = 1
for c in crg:
assert (c.idx() == i)
i -= -1
print("Testing sets of channel pairs")
print(
"Note that this test can fail if duplicate pairs are inserted in chpair_list"
)
print("upon generation")
num_unq_pairs = 20
# Generate random channel pairs
ch1_list = [
channels.channel("L", vv, hh)
for vv, hh in zip(np.random.randint(1, 25, num_unq_pairs),
np.random.randint(1, 9, num_unq_pairs))
]
ch2_list = [
channels.channel("L", vv, hh)
for vv, hh in zip(np.random.randint(1, 25, num_unq_pairs),
np.random.randint(1, 9, num_unq_pairs))
]
chpair_list = [
channels.channel_pair(ch1, ch2)
for ch1, ch2 in zip(ch1_list, ch2_list)
]
assert (len(chpair_list) == num_unq_pairs)
# Now we add random duplicates
num_duplicates = 0
for _ in range(num_duplicates):
# Select a random channel pair to duplicate. Note that len(chpair_list) increases in each
# iteration as we append
chidx = np.random.randint(0, len(chpair_list))
# Append this channel pair to the list. Take a 50/50 chance to switch the channels too.
ch2, ch1 = chpair_list[chidx]
if np.random.randint(0, 2) == True:
chpair_list.append(channels.channel_pair(ch2, ch1))
else:
chpair_list.append(channels.channel_pair(ch1, ch2))
assert (len(chpair_list) == num_unq_pairs + num_duplicates)
# Convert chpair_list, which includes 19 duplicates, to a set.
chpair_set = set(chpair_list)
assert (len(chpair_set) == num_unq_pairs)
# # Test iteration over channel ranges using unqiue channel pairs
# res = [channel_pair(c1, c2) for c1 in crg1 for c2 in crg2]
# # res contains duplicate paris, f.ex. (L0202, L0203) and (L0203, L0202)
# unq = unique_everseen(res)
# unq_list = list(unq)
# print("Non-unique: {0:d}, unique channel pairs: {1:d}".format(len(res), len(unq_list)))
# End of file test_channels.py
with np.load("/global/homes/r/rkube/repos/delta/test_data/fft_array_s0001.npz"
) as df:
fft_data = df["fft_data"]
fft_data_64 = np.ascontiguousarray(fft_data)
fft_data_32 = np.require(fft_data_64,
dtype=np.complex64,
requirements=['A', 'C'])
###################################################
# Generate channels to iterate over
ref_chrg = channel_range.from_str("L0101-2408")
cmp_chrg = channel_range.from_str("L0101-2408")
channel_pairs = [channel_pair(cr, cx) for cr in ref_chrg for cx in cmp_chrg]
unique_channels = [
i[0] for i in more_itertools.distinct_combinations(channel_pairs, 1)
]
#ch_it = [channel_pair(channel("L", i, 1), channel("L", i, 2)) for i in range(1, 25)]
#ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 3)) for i in range(1, 25)]
#h_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 4)) for i in range(1, 25)]
#ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 4)) for i in range(1, 25)]
#ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 5)) for i in range(1, 25)]
#ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 6)) for i in range(1, 25)]
#ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 7)) for i in range(1, 25)]
#ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 8)) for i in range(1, 25)]
ch1_idx_arr = np.array([int(ch_pair.ch1.idx()) for ch_pair in unique_channels],
dtype=np.uint64)
# Encoding: UTF-8
"""
Find out how to get a channel list from tget_dispatch_sequence
and pickle it
"""
import sys
sys.path.append("/global/homes/r/rkube/repos/delta")
import more_itertools
from itertools import chain
from analysis.channels import channel, channel_range, channel_pair
ref_channels = channel_range.from_str("L0101-2408")
cmp_channels = channel_range.from_str("L0101-2408")
channel_pairs = [
channel_pair(ref, cmp) for ref in ref_channels for cmp in cmp_channels
]
unique_channels = list(more_itertools.distinct_combinations(channel_pairs, 1))
print("Number of channel pairs: {0:d}".format(len(channel_pairs)))
print("Unique channel paits: {0:d}".format(len(unique_channels)))
ch_list = [u[0] for u in unqiue_channels]
np.savez("dispatch_seq.npz", ch_list=ch_list)
# End of file test_channel_it.py