def perform_data_datamining(mirnov_angles, misc_data_dict, n_clusters = 16, n_iterations = 60):
feat_obj = clust.feature_object(instance_array = mirnov_angles, misc_data_dict = misc_data_dict)
z = feat_obj.cluster(method="EM_VMM",n_clusters=n_clusters,n_iterations = n_iterations,start='k_means',verbose=0)
#z.plot_VM_distributions()
#z = feat_obj.cluster(method="k_means",n_clusters=n_clusters,n_iterations = n_iterations)
#z.fit_vonMises()
return z
def multi_extract(shot_selection,array_name, other_arrays = None, other_array_labels = None, meta_data = None,
n_cpus=8, NFFT = 2048, overlap = 4, extraction_settings = None, method = 'svd'):
'''Runs through all the shots in shot_selection other_arrays is a
list of the other arrays you want to get information from '''
if extraction_settings == None:
if method == 'svd':
extraction_settings = {'power_cutoff' : 0.05, 'min_svs': 2}
elif method == 'stft':
extraction_settings = {'n_pts': 20, 'lower_freq': 1500, 'cutoff_by' : 'sigma_eq', 'filter_cutoff': 20}
#Get the scan details
if method == 'svd':
wrapper = single_shot_svd_wrapper
elif method == 'stft':
wrapper = single_shot_stft_wrapper
else:
raise ValueError('method is not a valid choice : choose svd, stft')
shot_list, start_times, end_times = H1_scan_list.return_scan_details(shot_selection)
rep = itertools.repeat
if other_arrays == None: other_arrays = ['ElectronDensity','H1ToroidalNakedCoil']
if other_array_labels == None: other_array_labels = [['ne_static','ne_mode'],[None,'naked_coil']]
if meta_data == None : meta_data = ['kh','heating_freq','main_current','sec_current', 'shot']
input_data_iter = itertools.izip(shot_list, rep(array_name),
rep(other_arrays),
rep(other_array_labels),
start_times, end_times,
rep(NFFT), rep(overlap),rep(meta_data), rep(extraction_settings))
#generate the shot list for each worker
if n_cpus>1:
pool_size = n_cpus
pool = Pool(processes=pool_size, maxtasksperchild=3)
print 'creating pool map'
results = pool.map(wrapper, input_data_iter)
print 'waiting for pool to close '
pool.close()
print 'joining pool'
pool.join()
print 'pool finished'
else:
results = map(wrapper, input_data_iter)
start=1
for i,tmp in enumerate(results):
print i
if tmp[0]!=None:
if start==1:
instance_array = copy.deepcopy(tmp[0])
misc_data_dict = copy.deepcopy(tmp[1])
start = 0
else:
instance_array = np.append(instance_array, tmp[0],axis=0)
for i in misc_data_dict.keys():
misc_data_dict[i] = np.append(misc_data_dict[i], tmp[1][i],axis=0)
else:
print 'One shot may have failed....'
return clust.feature_object(instance_array = instance_array, misc_data_dict = misc_data_dict)
def multi_stft(shot_selection,
array_names,
n_cpus=1,
NFFT=2048,
perform_datamining=1,
overlap=4,
n_pts=20,
lower_freq=1500,
filter_cutoff=20,
cutoff_by='sigma_eq'):
#Get the scan details
shot_list, start_times, end_times = H1_scan_list.return_scan_details(
shot_selection)
rep = itertools.repeat
hop = NFFT / overlap
input_data = itertools.izip(shot_list, rep(array_names), rep(NFFT),
rep(hop), rep(n_pts), rep(lower_freq),
rep(None), start_times, end_times,
rep(perform_datamining), rep(filter_cutoff),
rep(cutoff_by))
if n_cpus > 1:
pool_size = n_cpus
pool = Pool(processes=pool_size, maxtasksperchild=3)
print 'creating pool map'
results = pool.map(single_shot_wrapper, input_data)
print 'waiting for pool to close '
pool.close()
pool.join()
print 'pool finished'
else:
results = map(single_shot_wrapper, input_data)
start = 1
#Put everything back together
kappa_list = []
for i, tmp in enumerate(results):
print i
kappa_list.append(copy.deepcopy(tmp[2]))
if tmp[0] != None:
if start == 1:
instance_array = copy.deepcopy(tmp[0])
misc_data_dict = copy.deepcopy(tmp[1])
start = 0
else:
instance_array = np.append(instance_array, tmp[0], axis=0)
for i in misc_data_dict.keys():
misc_data_dict[i] = np.append(misc_data_dict[i],
tmp[1][i],
axis=0)
else:
print 'something has failed....'
return clust.feature_object(instance_array=instance_array,
misc_data_dict=misc_data_dict), kappa_list
def perform_data_datamining(
mirnov_angles, misc_data_dict,
datamining_settings): # n_clusters = 16, n_iterations = 60):
feat_obj = clust.feature_object(
instance_array=mirnov_angles,
instance_array_amps=misc_data_dict['mirnov_data'],
misc_data_dict=misc_data_dict)
#z = feat_obj.cluster(method="EM_VMM",n_clusters=n_clusters,n_iterations = n_iterations,start='k_means',verbose=0)
#print 'perform datamining', datamining_settings
z = feat_obj.cluster(**datamining_settings)
#n_clusters=n_clusters,n_iterations = n_iterations,start='k_means',verbose=0)
#z.plot_VM_distributions()
#z = feat_obj.cluster(method="k_means",n_clusters=n_clusters,n_iterations = n_iterations)
#z.fit_vonMises()
return z
def multi_stft(shot_selection, array_names, n_cpus=1, NFFT=2048, perform_datamining = 1, overlap=4, n_pts=20, lower_freq = 1500, filter_cutoff = 20, cutoff_by = 'sigma_eq'):
#Get the scan details
shot_list, start_times, end_times = H1_scan_list.return_scan_details(shot_selection)
rep = itertools.repeat
hop = NFFT/overlap
input_data = itertools.izip(shot_list, rep(array_names), rep(NFFT),
rep(hop), rep(n_pts), rep(lower_freq), rep(None),
start_times, end_times, rep(perform_datamining),
rep(filter_cutoff), rep(cutoff_by))
if n_cpus>1:
pool_size = n_cpus
pool = Pool(processes=pool_size, maxtasksperchild=3)
print 'creating pool map'
results = pool.map(single_shot_wrapper, input_data)
print 'waiting for pool to close '
pool.close();pool.join()
print 'pool finished'
else:
results = map(single_shot_wrapper, input_data)
start=1
#Put everything back together
kappa_list = []
for i,tmp in enumerate(results):
print i
kappa_list.append(copy.deepcopy(tmp[2]))
if tmp[0]!=None:
if start==1:
instance_array = copy.deepcopy(tmp[0])
misc_data_dict = copy.deepcopy(tmp[1])
start = 0
else:
instance_array = np.append(instance_array, tmp[0],axis=0)
for i in misc_data_dict.keys():
misc_data_dict[i] = np.append(misc_data_dict[i], tmp[1][i],axis=0)
else:
print 'something has failed....'
return clust.feature_object(instance_array = instance_array, misc_data_dict = misc_data_dict), kappa_list
def multi_svd(
shot_selection,
array_name,
other_arrays=None,
other_array_labels=None,
meta_data=None,
n_cpus=8,
NFFT=2048,
power_cutoff=0.05,
min_svs=2,
overlap=4,
):
'''Runs through all the shots in shot_selection other_arrays is a
list of the other arrays you want to get information from '''
#Get the scan details
shot_list, start_times, end_times = H1_scan_list.return_scan_details(
shot_selection)
rep = itertools.repeat
if other_arrays == None:
other_arrays = ['ElectronDensity', 'H1ToroidalNakedCoil']
if other_array_labels == None:
other_array_labels = [['ne_static', 'ne_mode'], [None, 'naked_coil']]
if meta_data == None:
meta_data = [
'kh', 'heating_freq', 'main_current', 'sec_current', 'shot'
]
input_data_iter = itertools.izip(shot_list, rep(array_name),
rep(other_arrays),
rep(other_array_labels),
start_times, end_times, rep(NFFT),
rep(power_cutoff), rep(min_svs),
rep(overlap), rep(meta_data))
#generate the shot list for each worker
if n_cpus > 1:
pool_size = n_cpus
pool = Pool(processes=pool_size, maxtasksperchild=3)
print 'creating pool map'
results = pool.map(single_shot_svd_wrapper, input_data_iter)
print 'waiting for pool to close '
pool.close()
print 'joining pool'
pool.join()
print 'pool finished'
else:
results = map(single_shot_svd_wrapper, input_data_iter)
start = 1
for i, tmp in enumerate(results):
print i
if tmp[0] != None:
if start == 1:
instance_array = copy.deepcopy(tmp[0])
misc_data_dict = copy.deepcopy(tmp[1])
start = 0
else:
instance_array = np.append(instance_array, tmp[0], axis=0)
for i in misc_data_dict.keys():
misc_data_dict[i] = np.append(misc_data_dict[i],
tmp[1][i],
axis=0)
else:
print 'One shot may have failed....'
return clust.feature_object(instance_array=instance_array,
misc_data_dict=misc_data_dict)
def multi_extract(shot_selection,
array_name,
other_arrays=None,
other_array_labels=None,
meta_data=None,
n_cpus=8,
NFFT=2048,
overlap=4,
extraction_settings=None,
method='svd',
start_times=None,
end_times=None):
'''Runs through all the shots in shot_selection other_arrays is a
list of the other arrays you want to get information from '''
if extraction_settings == None:
if method == 'svd':
extraction_settings = {'power_cutoff': 0.05, 'min_svs': 2}
elif method == 'stft':
extraction_settings = {
'n_pts': 20,
'lower_freq': 1500,
'cutoff_by': 'sigma_eq',
'filter_cutoff': 20
}
#Get the scan details
if method == 'svd':
wrapper = single_shot_svd_wrapper
elif method == 'stft':
wrapper = single_shot_stft_wrapper
else:
raise ValueError('method is not a valid choice : choose svd, stft')
#Check to see if a shot list was provided
shot_list_nums = False
try:
for i in shot_selection:
tmp = int(i)
shot_list_nums = True
shot_list = shot_selection
except:
print('List of shots not provided, looking up data in H1_scan_list')
if not shot_list_nums:
shot_list, start_times, end_times = H1_scan_list.return_scan_details(
shot_selection)
rep = itertools.repeat
if other_arrays == None:
other_arrays = ['ElectronDensity', 'H1ToroidalNakedCoil']
if other_array_labels == None:
other_array_labels = [['ne_static', 'ne_mode'], [None, 'naked_coil']]
if meta_data == None:
meta_data = [
'kh', 'heating_freq', 'main_current', 'sec_current', 'shot'
]
input_data_iter = itertools.izip(shot_list, rep(array_name),
rep(other_arrays),
rep(other_array_labels), start_times,
end_times, rep(NFFT), rep(overlap),
rep(meta_data), rep(extraction_settings))
#generate the shot list for each worker
if n_cpus > 1:
pool_size = n_cpus
pool = Pool(processes=pool_size, maxtasksperchild=3)
print 'creating pool map'
results = pool.map(wrapper, input_data_iter)
print 'waiting for pool to close '
pool.close()
print 'joining pool'
pool.join()
print 'pool finished'
else:
results = map(wrapper, input_data_iter)
start = 1
for i, tmp in enumerate(results):
print i
if tmp[0] != None:
if start == 1:
instance_array = copy.deepcopy(tmp[0])
misc_data_dict = copy.deepcopy(tmp[1])
start = 0
else:
instance_array = np.append(instance_array, tmp[0], axis=0)
for i in misc_data_dict.keys():
misc_data_dict[i] = np.append(misc_data_dict[i],
tmp[1][i],
axis=0)
else:
print 'One shot may have failed....'
return clust.feature_object(
instance_array=instance_array,
instance_array_amps=+misc_data_dict['mirnov_data'],
misc_data_dict=misc_data_dict)
def multi_extract_DIIID(shot_selection,
array_name,
other_arrays=None,
other_array_labels=None,
meta_data=None,
n_cpus=8,
NFFT=2048,
overlap=4,
extraction_settings=None,
method="svd",
start_times=None,
end_times=None):
# Runs through all of the shots in shot_selection. other_arrays is a list of other arrays to
# pull information from. Adapted from multi_extract from below
# ( will delete multi_extract(...) once it becomes obsolete )
# John Gresl 10/07/2016
if extraction_settings is None:
if method.lower() == "svd":
extraction_settings = {"power_cutoff": 0.05, "min_svs": 2}
elif method.lower() == "stft":
extraction_settings = {
"n_pts": 20,
"lower_freq": 1500,
"cutoff_by": "sigma_eq",
"filter_cutoff": 20
}
# Fetch the scan details
if method.lower() == "svd":
wrapper = single_shot_svd_wrapper
elif method.lower() == "stft":
wrapper = single_shot_stft_wrapper
else:
raise ValueError(
"'method' is not a valid choice! Choose either svd or stft")
# Gather shot numbers
if type(shot_selection[0]) == str:
print(
"List of shots not provided. Looking up shots in DIIID_Scan_List.py: {}"
.format(shot_selection))
(shot_list, start_times,
end_times) = DIIID_scan_list.return_scan_details(shot_selection)
rep = itertools.repeat
if other_arrays is None:
# other_arrays = ["ElectronDensity", "H1ToroidalNakedCoil"] # TODO: Find DIIID names for these devices
other_arrays = []
if other_array_labels is None:
# other_array_labels = [["ne_static", "ne_mode"], [None, "naked_coil"]] # TODO: Same as ^
other_array_labels = []
if meta_data is None:
meta_data = [
"kh", "heating_freq", "main_current", "sec_current", "shot"
]
# Construct the input data iterable for our single_shot_wrapper mapper.
input_data_iter = itertools.izip(shot_list, rep(array_name),
rep(other_arrays),
rep(other_array_labels), start_times,
end_times, rep(NFFT), rep(overlap),
rep(meta_data), rep(extraction_settings))
# Generate the shot list for each worker
if n_cpus > 1:
pool = Pool(processes=n_cpus, maxtasksperchild=3)
results = pool.map(wrapper, input_data_iter)
pool.close()
pool.join()
else:
results = map(wrapper, input_data_iter)
start = True
for i, tmp in enumerate(results):
if tmp[0] is not None:
if start:
instance_array = copy.deepcopy(tmp[0])
misc_data_dict = copy.deepcopy(tmp[1])
start = False
else:
instance_array = np.append(instance_array, tmp[0], axis=0)
for i in misc_data_dict.keys():
misc_data_dict[i] = np.append(misc_data_dict[i],
tmp[1][i],
axis=0)
else:
print("One shot may have failed.")
return clust.feature_object(
instance_array=instance_array,
instance_array_amps=+misc_data_dict["mirnov_data"],
misc_data_dict=misc_data_dict)
