def multi_extract(shot_selection,array_name, other_arrays = None, other_array_labels = None, meta_data = None,
n_cpus=8, NFFT = 2048, overlap = 4, extraction_settings = None, method = 'svd'):
'''Runs through all the shots in shot_selection other_arrays is a
list of the other arrays you want to get information from '''
if extraction_settings == None:
if method == 'svd':
extraction_settings = {'power_cutoff' : 0.05, 'min_svs': 2}
elif method == 'stft':
extraction_settings = {'n_pts': 20, 'lower_freq': 1500, 'cutoff_by' : 'sigma_eq', 'filter_cutoff': 20}
#Get the scan details
if method == 'svd':
wrapper = single_shot_svd_wrapper
elif method == 'stft':
wrapper = single_shot_stft_wrapper
else:
raise ValueError('method is not a valid choice : choose svd, stft')
shot_list, start_times, end_times = H1_scan_list.return_scan_details(shot_selection)
rep = itertools.repeat
if other_arrays == None: other_arrays = ['ElectronDensity','H1ToroidalNakedCoil']
if other_array_labels == None: other_array_labels = [['ne_static','ne_mode'],[None,'naked_coil']]
if meta_data == None : meta_data = ['kh','heating_freq','main_current','sec_current', 'shot']
input_data_iter = itertools.izip(shot_list, rep(array_name),
rep(other_arrays),
rep(other_array_labels),
start_times, end_times,
rep(NFFT), rep(overlap),rep(meta_data), rep(extraction_settings))
#generate the shot list for each worker
if n_cpus>1:
pool_size = n_cpus
pool = Pool(processes=pool_size, maxtasksperchild=3)
print 'creating pool map'
results = pool.map(wrapper, input_data_iter)
print 'waiting for pool to close '
pool.close()
print 'joining pool'
pool.join()
print 'pool finished'
else:
results = map(wrapper, input_data_iter)
start=1
for i,tmp in enumerate(results):
print i
if tmp[0]!=None:
if start==1:
instance_array = copy.deepcopy(tmp[0])
misc_data_dict = copy.deepcopy(tmp[1])
start = 0
else:
instance_array = np.append(instance_array, tmp[0],axis=0)
for i in misc_data_dict.keys():
misc_data_dict[i] = np.append(misc_data_dict[i], tmp[1][i],axis=0)
else:
print 'One shot may have failed....'
return clust.feature_object(instance_array = instance_array, misc_data_dict = misc_data_dict)
def multi_stft(shot_selection,
array_names,
n_cpus=1,
NFFT=2048,
perform_datamining=1,
overlap=4,
n_pts=20,
lower_freq=1500,
filter_cutoff=20,
cutoff_by='sigma_eq'):
#Get the scan details
shot_list, start_times, end_times = H1_scan_list.return_scan_details(
shot_selection)
rep = itertools.repeat
hop = NFFT / overlap
input_data = itertools.izip(shot_list, rep(array_names), rep(NFFT),
rep(hop), rep(n_pts), rep(lower_freq),
rep(None), start_times, end_times,
rep(perform_datamining), rep(filter_cutoff),
rep(cutoff_by))
if n_cpus > 1:
pool_size = n_cpus
pool = Pool(processes=pool_size, maxtasksperchild=3)
print 'creating pool map'
results = pool.map(single_shot_wrapper, input_data)
print 'waiting for pool to close '
pool.close()
pool.join()
print 'pool finished'
else:
results = map(single_shot_wrapper, input_data)
start = 1
#Put everything back together
kappa_list = []
for i, tmp in enumerate(results):
print i
kappa_list.append(copy.deepcopy(tmp[2]))
if tmp[0] != None:
if start == 1:
instance_array = copy.deepcopy(tmp[0])
misc_data_dict = copy.deepcopy(tmp[1])
start = 0
else:
instance_array = np.append(instance_array, tmp[0], axis=0)
for i in misc_data_dict.keys():
misc_data_dict[i] = np.append(misc_data_dict[i],
tmp[1][i],
axis=0)
else:
print 'something has failed....'
return clust.feature_object(instance_array=instance_array,
misc_data_dict=misc_data_dict), kappa_list
def multi_stft(shot_selection, array_names, n_cpus=1, NFFT=2048, perform_datamining = 1, overlap=4, n_pts=20, lower_freq = 1500, filter_cutoff = 20, cutoff_by = 'sigma_eq'):
#Get the scan details
shot_list, start_times, end_times = H1_scan_list.return_scan_details(shot_selection)
rep = itertools.repeat
hop = NFFT/overlap
input_data = itertools.izip(shot_list, rep(array_names), rep(NFFT),
rep(hop), rep(n_pts), rep(lower_freq), rep(None),
start_times, end_times, rep(perform_datamining),
rep(filter_cutoff), rep(cutoff_by))
if n_cpus>1:
pool_size = n_cpus
pool = Pool(processes=pool_size, maxtasksperchild=3)
print 'creating pool map'
results = pool.map(single_shot_wrapper, input_data)
print 'waiting for pool to close '
pool.close();pool.join()
print 'pool finished'
else:
results = map(single_shot_wrapper, input_data)
start=1
#Put everything back together
kappa_list = []
for i,tmp in enumerate(results):
print i
kappa_list.append(copy.deepcopy(tmp[2]))
if tmp[0]!=None:
if start==1:
instance_array = copy.deepcopy(tmp[0])
misc_data_dict = copy.deepcopy(tmp[1])
start = 0
else:
instance_array = np.append(instance_array, tmp[0],axis=0)
for i in misc_data_dict.keys():
misc_data_dict[i] = np.append(misc_data_dict[i], tmp[1][i],axis=0)
else:
print 'something has failed....'
return clust.feature_object(instance_array = instance_array, misc_data_dict = misc_data_dict), kappa_list
def multi_svd(
shot_selection,
array_name,
other_arrays=None,
other_array_labels=None,
meta_data=None,
n_cpus=8,
NFFT=2048,
power_cutoff=0.05,
min_svs=2,
overlap=4,
):
'''Runs through all the shots in shot_selection other_arrays is a
list of the other arrays you want to get information from '''
#Get the scan details
shot_list, start_times, end_times = H1_scan_list.return_scan_details(
shot_selection)
rep = itertools.repeat
if other_arrays == None:
other_arrays = ['ElectronDensity', 'H1ToroidalNakedCoil']
if other_array_labels == None:
other_array_labels = [['ne_static', 'ne_mode'], [None, 'naked_coil']]
if meta_data == None:
meta_data = [
'kh', 'heating_freq', 'main_current', 'sec_current', 'shot'
]
input_data_iter = itertools.izip(shot_list, rep(array_name),
rep(other_arrays),
rep(other_array_labels),
start_times, end_times, rep(NFFT),
rep(power_cutoff), rep(min_svs),
rep(overlap), rep(meta_data))
#generate the shot list for each worker
if n_cpus > 1:
pool_size = n_cpus
pool = Pool(processes=pool_size, maxtasksperchild=3)
print 'creating pool map'
results = pool.map(single_shot_svd_wrapper, input_data_iter)
print 'waiting for pool to close '
pool.close()
print 'joining pool'
pool.join()
print 'pool finished'
else:
results = map(single_shot_svd_wrapper, input_data_iter)
start = 1
for i, tmp in enumerate(results):
print i
if tmp[0] != None:
if start == 1:
instance_array = copy.deepcopy(tmp[0])
misc_data_dict = copy.deepcopy(tmp[1])
start = 0
else:
instance_array = np.append(instance_array, tmp[0], axis=0)
for i in misc_data_dict.keys():
misc_data_dict[i] = np.append(misc_data_dict[i],
tmp[1][i],
axis=0)
else:
print 'One shot may have failed....'
return clust.feature_object(instance_array=instance_array,
misc_data_dict=misc_data_dict)
def multi_extract(shot_selection,
array_name,
other_arrays=None,
other_array_labels=None,
meta_data=None,
n_cpus=8,
NFFT=2048,
overlap=4,
extraction_settings=None,
method='svd',
start_times=None,
end_times=None):
'''Runs through all the shots in shot_selection other_arrays is a
list of the other arrays you want to get information from '''
if extraction_settings == None:
if method == 'svd':
extraction_settings = {'power_cutoff': 0.05, 'min_svs': 2}
elif method == 'stft':
extraction_settings = {
'n_pts': 20,
'lower_freq': 1500,
'cutoff_by': 'sigma_eq',
'filter_cutoff': 20
}
#Get the scan details
if method == 'svd':
wrapper = single_shot_svd_wrapper
elif method == 'stft':
wrapper = single_shot_stft_wrapper
else:
raise ValueError('method is not a valid choice : choose svd, stft')
#Check to see if a shot list was provided
shot_list_nums = False
try:
for i in shot_selection:
tmp = int(i)
shot_list_nums = True
shot_list = shot_selection
except:
print('List of shots not provided, looking up data in H1_scan_list')
if not shot_list_nums:
shot_list, start_times, end_times = H1_scan_list.return_scan_details(
shot_selection)
rep = itertools.repeat
if other_arrays == None:
other_arrays = ['ElectronDensity', 'H1ToroidalNakedCoil']
if other_array_labels == None:
other_array_labels = [['ne_static', 'ne_mode'], [None, 'naked_coil']]
if meta_data == None:
meta_data = [
'kh', 'heating_freq', 'main_current', 'sec_current', 'shot'
]
input_data_iter = itertools.izip(shot_list, rep(array_name),
rep(other_arrays),
rep(other_array_labels), start_times,
end_times, rep(NFFT), rep(overlap),
rep(meta_data), rep(extraction_settings))
#generate the shot list for each worker
if n_cpus > 1:
pool_size = n_cpus
pool = Pool(processes=pool_size, maxtasksperchild=3)
print 'creating pool map'
results = pool.map(wrapper, input_data_iter)
print 'waiting for pool to close '
pool.close()
print 'joining pool'
pool.join()
print 'pool finished'
else:
results = map(wrapper, input_data_iter)
start = 1
for i, tmp in enumerate(results):
print i
if tmp[0] != None:
if start == 1:
instance_array = copy.deepcopy(tmp[0])
misc_data_dict = copy.deepcopy(tmp[1])
start = 0
else:
instance_array = np.append(instance_array, tmp[0], axis=0)
for i in misc_data_dict.keys():
misc_data_dict[i] = np.append(misc_data_dict[i],
tmp[1][i],
axis=0)
else:
print 'One shot may have failed....'
return clust.feature_object(
instance_array=instance_array,
instance_array_amps=+misc_data_dict['mirnov_data'],
misc_data_dict=misc_data_dict)