def multi_extract(shot_selection,array_name, other_arrays = None, other_array_labels = None, meta_data = None, n_cpus=8, NFFT = 2048, overlap = 4, extraction_settings = None, method = 'svd'): '''Runs through all the shots in shot_selection other_arrays is a list of the other arrays you want to get information from ''' if extraction_settings == None: if method == 'svd': extraction_settings = {'power_cutoff' : 0.05, 'min_svs': 2} elif method == 'stft': extraction_settings = {'n_pts': 20, 'lower_freq': 1500, 'cutoff_by' : 'sigma_eq', 'filter_cutoff': 20} #Get the scan details if method == 'svd': wrapper = single_shot_svd_wrapper elif method == 'stft': wrapper = single_shot_stft_wrapper else: raise ValueError('method is not a valid choice : choose svd, stft') shot_list, start_times, end_times = H1_scan_list.return_scan_details(shot_selection) rep = itertools.repeat if other_arrays == None: other_arrays = ['ElectronDensity','H1ToroidalNakedCoil'] if other_array_labels == None: other_array_labels = [['ne_static','ne_mode'],[None,'naked_coil']] if meta_data == None : meta_data = ['kh','heating_freq','main_current','sec_current', 'shot'] input_data_iter = itertools.izip(shot_list, rep(array_name), rep(other_arrays), rep(other_array_labels), start_times, end_times, rep(NFFT), rep(overlap),rep(meta_data), rep(extraction_settings)) #generate the shot list for each worker if n_cpus>1: pool_size = n_cpus pool = Pool(processes=pool_size, maxtasksperchild=3) print 'creating pool map' results = pool.map(wrapper, input_data_iter) print 'waiting for pool to close ' pool.close() print 'joining pool' pool.join() print 'pool finished' else: results = map(wrapper, input_data_iter) start=1 for i,tmp in enumerate(results): print i if tmp[0]!=None: if start==1: instance_array = copy.deepcopy(tmp[0]) misc_data_dict = copy.deepcopy(tmp[1]) start = 0 else: instance_array = np.append(instance_array, tmp[0],axis=0) for i in misc_data_dict.keys(): misc_data_dict[i] = np.append(misc_data_dict[i], tmp[1][i],axis=0) else: print 'One shot may have failed....' return clust.feature_object(instance_array = instance_array, misc_data_dict = misc_data_dict)
def multi_stft(shot_selection, array_names, n_cpus=1, NFFT=2048, perform_datamining=1, overlap=4, n_pts=20, lower_freq=1500, filter_cutoff=20, cutoff_by='sigma_eq'): #Get the scan details shot_list, start_times, end_times = H1_scan_list.return_scan_details( shot_selection) rep = itertools.repeat hop = NFFT / overlap input_data = itertools.izip(shot_list, rep(array_names), rep(NFFT), rep(hop), rep(n_pts), rep(lower_freq), rep(None), start_times, end_times, rep(perform_datamining), rep(filter_cutoff), rep(cutoff_by)) if n_cpus > 1: pool_size = n_cpus pool = Pool(processes=pool_size, maxtasksperchild=3) print 'creating pool map' results = pool.map(single_shot_wrapper, input_data) print 'waiting for pool to close ' pool.close() pool.join() print 'pool finished' else: results = map(single_shot_wrapper, input_data) start = 1 #Put everything back together kappa_list = [] for i, tmp in enumerate(results): print i kappa_list.append(copy.deepcopy(tmp[2])) if tmp[0] != None: if start == 1: instance_array = copy.deepcopy(tmp[0]) misc_data_dict = copy.deepcopy(tmp[1]) start = 0 else: instance_array = np.append(instance_array, tmp[0], axis=0) for i in misc_data_dict.keys(): misc_data_dict[i] = np.append(misc_data_dict[i], tmp[1][i], axis=0) else: print 'something has failed....' return clust.feature_object(instance_array=instance_array, misc_data_dict=misc_data_dict), kappa_list
def multi_stft(shot_selection, array_names, n_cpus=1, NFFT=2048, perform_datamining = 1, overlap=4, n_pts=20, lower_freq = 1500, filter_cutoff = 20, cutoff_by = 'sigma_eq'): #Get the scan details shot_list, start_times, end_times = H1_scan_list.return_scan_details(shot_selection) rep = itertools.repeat hop = NFFT/overlap input_data = itertools.izip(shot_list, rep(array_names), rep(NFFT), rep(hop), rep(n_pts), rep(lower_freq), rep(None), start_times, end_times, rep(perform_datamining), rep(filter_cutoff), rep(cutoff_by)) if n_cpus>1: pool_size = n_cpus pool = Pool(processes=pool_size, maxtasksperchild=3) print 'creating pool map' results = pool.map(single_shot_wrapper, input_data) print 'waiting for pool to close ' pool.close();pool.join() print 'pool finished' else: results = map(single_shot_wrapper, input_data) start=1 #Put everything back together kappa_list = [] for i,tmp in enumerate(results): print i kappa_list.append(copy.deepcopy(tmp[2])) if tmp[0]!=None: if start==1: instance_array = copy.deepcopy(tmp[0]) misc_data_dict = copy.deepcopy(tmp[1]) start = 0 else: instance_array = np.append(instance_array, tmp[0],axis=0) for i in misc_data_dict.keys(): misc_data_dict[i] = np.append(misc_data_dict[i], tmp[1][i],axis=0) else: print 'something has failed....' return clust.feature_object(instance_array = instance_array, misc_data_dict = misc_data_dict), kappa_list
def multi_svd( shot_selection, array_name, other_arrays=None, other_array_labels=None, meta_data=None, n_cpus=8, NFFT=2048, power_cutoff=0.05, min_svs=2, overlap=4, ): '''Runs through all the shots in shot_selection other_arrays is a list of the other arrays you want to get information from ''' #Get the scan details shot_list, start_times, end_times = H1_scan_list.return_scan_details( shot_selection) rep = itertools.repeat if other_arrays == None: other_arrays = ['ElectronDensity', 'H1ToroidalNakedCoil'] if other_array_labels == None: other_array_labels = [['ne_static', 'ne_mode'], [None, 'naked_coil']] if meta_data == None: meta_data = [ 'kh', 'heating_freq', 'main_current', 'sec_current', 'shot' ] input_data_iter = itertools.izip(shot_list, rep(array_name), rep(other_arrays), rep(other_array_labels), start_times, end_times, rep(NFFT), rep(power_cutoff), rep(min_svs), rep(overlap), rep(meta_data)) #generate the shot list for each worker if n_cpus > 1: pool_size = n_cpus pool = Pool(processes=pool_size, maxtasksperchild=3) print 'creating pool map' results = pool.map(single_shot_svd_wrapper, input_data_iter) print 'waiting for pool to close ' pool.close() print 'joining pool' pool.join() print 'pool finished' else: results = map(single_shot_svd_wrapper, input_data_iter) start = 1 for i, tmp in enumerate(results): print i if tmp[0] != None: if start == 1: instance_array = copy.deepcopy(tmp[0]) misc_data_dict = copy.deepcopy(tmp[1]) start = 0 else: instance_array = np.append(instance_array, tmp[0], axis=0) for i in misc_data_dict.keys(): misc_data_dict[i] = np.append(misc_data_dict[i], tmp[1][i], axis=0) else: print 'One shot may have failed....' return clust.feature_object(instance_array=instance_array, misc_data_dict=misc_data_dict)
def multi_extract(shot_selection, array_name, other_arrays=None, other_array_labels=None, meta_data=None, n_cpus=8, NFFT=2048, overlap=4, extraction_settings=None, method='svd', start_times=None, end_times=None): '''Runs through all the shots in shot_selection other_arrays is a list of the other arrays you want to get information from ''' if extraction_settings == None: if method == 'svd': extraction_settings = {'power_cutoff': 0.05, 'min_svs': 2} elif method == 'stft': extraction_settings = { 'n_pts': 20, 'lower_freq': 1500, 'cutoff_by': 'sigma_eq', 'filter_cutoff': 20 } #Get the scan details if method == 'svd': wrapper = single_shot_svd_wrapper elif method == 'stft': wrapper = single_shot_stft_wrapper else: raise ValueError('method is not a valid choice : choose svd, stft') #Check to see if a shot list was provided shot_list_nums = False try: for i in shot_selection: tmp = int(i) shot_list_nums = True shot_list = shot_selection except: print('List of shots not provided, looking up data in H1_scan_list') if not shot_list_nums: shot_list, start_times, end_times = H1_scan_list.return_scan_details( shot_selection) rep = itertools.repeat if other_arrays == None: other_arrays = ['ElectronDensity', 'H1ToroidalNakedCoil'] if other_array_labels == None: other_array_labels = [['ne_static', 'ne_mode'], [None, 'naked_coil']] if meta_data == None: meta_data = [ 'kh', 'heating_freq', 'main_current', 'sec_current', 'shot' ] input_data_iter = itertools.izip(shot_list, rep(array_name), rep(other_arrays), rep(other_array_labels), start_times, end_times, rep(NFFT), rep(overlap), rep(meta_data), rep(extraction_settings)) #generate the shot list for each worker if n_cpus > 1: pool_size = n_cpus pool = Pool(processes=pool_size, maxtasksperchild=3) print 'creating pool map' results = pool.map(wrapper, input_data_iter) print 'waiting for pool to close ' pool.close() print 'joining pool' pool.join() print 'pool finished' else: results = map(wrapper, input_data_iter) start = 1 for i, tmp in enumerate(results): print i if tmp[0] != None: if start == 1: instance_array = copy.deepcopy(tmp[0]) misc_data_dict = copy.deepcopy(tmp[1]) start = 0 else: instance_array = np.append(instance_array, tmp[0], axis=0) for i in misc_data_dict.keys(): misc_data_dict[i] = np.append(misc_data_dict[i], tmp[1][i], axis=0) else: print 'One shot may have failed....' return clust.feature_object( instance_array=instance_array, instance_array_amps=+misc_data_dict['mirnov_data'], misc_data_dict=misc_data_dict)