def get_data(self,): self.data = pf.getDevice('H1').acq.getdata(self.shot, self.array).reduce_time([self.start_time, self.end_time]) self.data = self.data.subtract_mean(copy=False).normalise(method='v',separate=True,copy=False) self.timebase = self.data.timebase self.data_fft = self.data.generate_frequency_series(self.samples,self.samples/self.overlap) print self.other_arrays, self.other_array_labels if self.other_arrays == None: self.other_array_labels = [] if self.other_arrays == None: self.other_arrays = []; if self.meta_data == None : self.meta_data = [] self.other_arrays_fft = [] for i in self.other_arrays: tmp = pf.getDevice('H1').acq.getdata(self.shot, i).change_time_base(self.data.timebase) self.other_arrays_fft.append(tmp.generate_frequency_series(self.samples,self.samples/self.overlap)) self.instance_array_list = [] self.misc_data_dict = {} #How to deal with the static case? for i in self.other_array_labels: if i[0]!=None: self.misc_data_dict[i[0]] = [] if i[1]!=None: self.misc_data_dict[i[1]] = [] #self.fs_values = ['p','a12','H','freq','E'] for i in self.meta_data: self.misc_data_dict[i]=[]
def extract_polarisation_data(current_shot): try: #MDSTree=MDS.Tree('mirnov',current_shot) coil_1x=pf.getDevice('H1').acq.getdata(current_shot,'H1ToroidalMirnov_1x') coil_1y=pf.getDevice('H1').acq.getdata(current_shot,'H1ToroidalMirnov_1y') coil_1z=pf.getDevice('H1').acq.getdata(current_shot,'H1ToroidalMirnov_1z') #Need to narrow the time down.... maybe extract the above data elsewhere #print 'successful extraction of pyrex coil' return coil_1x, coil_1y, coil_1z except: print 'Error getting polarisation data' return 0,0,0
def get_primary_array(self, ): data_list = [] new_signal_length = 0 for i, arr in enumerate(self.array): #data_list.append(pf.getDevice('H1').acq.getdata(self.shot, arr).reduce_time([self.start_time, self.end_time])) data_list.append( pf.getDevice('DIIID').acq.getdata(self.shot, arr).reduce_time( [self.start_time, self.end_time])) if i == 0: self.timebase = data_list[-1].timebase else: data_list[-1] = data_list[-1].change_time_base(self.timebase) new_signal_length += data_list[-1].signal.shape[0] new_sig = np.zeros((new_signal_length, self.timebase.shape[0]), dtype=float) start_ind = 0 self.data = data_list[0] for i in data_list: end_ind = start_ind + i.signal.shape[0] new_sig[start_ind:end_ind, :] = +i.signal start_ind = end_ind self.data.signal = +new_sig self.data.channels.extend(i.channels) self.data_fft = self.data.generate_frequency_series( self.samples, self.samples / self.overlap)
def test_single_mirnov_channel_kappah_from_metadata(self): h1test = pyfusion.getDevice('H1') #shot_kh = (58073, 0.74) shot_kh = (58123, 0.74) # TODO: why doesn't this work with thick client?? data = h1test.acq.getdata(shot_kh[0], 'H1_mirnov_array_1_coil_1') print(shot_kh)
def test_single_mirnov_channel_kappah_from_metadata(self): h1test = pyfusion.getDevice("H1") # shot_kh = (58073, 0.74) shot_kh = (58123, 0.74) # TODO: why doesn't this work with thick client?? data = h1test.acq.getdata(shot_kh[0], "H1_mirnov_array_1_coil_1") print(shot_kh)
def test_single_mirnov_channel_kappah_as_argument(self): d = pyfusion.getDevice('H1') data = d.acq.getdata(58123, 'H1_mirnov_array_1_coil_1') self.assertTrue(isinstance(data, TimeseriesData)) from pyfusion.data.base import PfMetaData self.assertTrue(isinstance(data.meta, PfMetaData)) """
def get_data(self, ): self.get_primary_array() #self.data = pf.getDevice('H1').acq.getdata(self.shot, self.array).reduce_time([self.start_time, self.end_time]) self.data = self.data.subtract_mean(copy=False).normalise( method='v', separate=True, copy=False) self.data_segmented = self.data.segment(self.samples, overlap=self.overlap, datalist=1) #print self.other_arrays, self.other_array_labels self.other_arrays_segmented = [] for i in self.other_arrays: #tmp = pf.getDevice('H1').acq.getdata(self.shot, i).change_time_base(self.data.timebase) tmp = pf.getDevice('DIIID').acq.getdata( self.shot, i).change_time_base(self.data.timebase) self.other_arrays_segmented.append( tmp.segment(self.samples, overlap=self.overlap, datalist=1)) self.instance_array_list = [] self.misc_data_dict = {'RMS': [], 'time': [], 'svs': []} #How to deal with the static case? for i in self.other_array_labels: if i[0] != None: self.misc_data_dict[i[0]] = [] if i[1] != None: self.misc_data_dict[i[1]] = [] self.fs_values = ['p', 'a12', 'H', 'freq', 'E'] for i in self.meta_data: self.misc_data_dict[i] = [] for i in self.fs_values: self.misc_data_dict[i] = [] self.misc_data_dict['mirnov_data'] = []
def __init__(self, shot, i_diag, v_diag, dev_name="W7X", debug=debug, plot=1, verbose=0, params=None): self.dev = pyfusion.getDevice(dev_name) self.shot = shot self.verbose = verbose self.i_diag = i_diag self.v_diag = v_diag self.debug = debug self.plot = plot self.select = None self.t_comp = (0.1,0.2) self.params = params self.figs = [] self.suffix = '' # this gets put at the end of the fig name (title bar) self.imeasfull = self.dev.acq.getdata(shot, i_diag) self.vmeasfull = self.dev.acq.getdata(shot, v_diag) comlen = min(len(self.vmeasfull.timebase), len(self.imeasfull.timebase)) FFT_size = nice_FFT_size(comlen-2, -1) # the minus 2 is a fudge to hide small inconsistencies in reduce_time # e.g. 20160310 9 W7X_L5_LPALLI self.imeasfull = self.imeasfull.reduce_time([self.imeasfull.timebase[0], self.imeasfull.timebase[FFT_size]]) self.vmeasfull = self.vmeasfull.reduce_time([self.vmeasfull.timebase[0], self.vmeasfull.timebase[FFT_size]]) if self.params is not None: self.process_swept_Langmuir(**self.params)
def test_single_mirnov_channel_kappah_as_argument(self): d = pyfusion.getDevice("H1") data = d.acq.getdata(58123, "H1_mirnov_array_1_coil_1") self.assertTrue(isinstance(data, TimeseriesData)) from pyfusion.data.base import PfMetaData self.assertTrue(isinstance(data.meta, PfMetaData)) """
def test_device_getdata_single_shot(self): dev = pyfusion.getDevice("TestDevice") # what we want... expected_data = dev.acq.getdata(12345, "test_timeseries_shot_unique") # what we get.... data = dev.getdata(12345, "test_timeseries_shot_unique") self.assertEqual(expected_data, data)
def test_get_data(self): """Check that we end up with the correct data class starting from Device""" from pyfusion import getDevice test_device = getDevice(self.listed_device) test_data = test_device.acquisition.getdata(self.shot_number, timeseries_test_channel_1) from pyfusion.data.timeseries import TimeseriesData self.assertTrue(isinstance(test_data, TimeseriesData))
def test_plot_signals(self): # in this position, the local data/test.cfg is used, so to run separately, # you need to point your PYTHON_CONFIG_FILE at that file. # - doesn't work as of Mar 7 - dev = pyfusion.getDevice('H1') print('\n'.join(pyfusion.conf.utils.dump(eol=''))) dat = dev.acq.getdata(58123,'Test_H1_multi_small') dat.plot_signals()
def get_array_data(current_shot, array_name, time_window=None,new_timebase=None): array_cutoff_locs = [0] data = pf.getDevice('H1').acq.getdata(current_shot, array_name) if new_timebase!=None: print('interpolating onto a new timebase....') data = data.change_time_base(new_timebase) if time_window!=None: data = data.reduce_time(time_window) return data
def extract_data(current_shot,array): tries,success=(0,0) while tries<10 and success==0: try: data=pf.getDevice('H1').acq.getdata(current_shot,array) success=1 #print 'Data extracted on Shot : %d'%(current_shot) except (MDS.TdiException, MDS.TreeException) as e: print current_shot, e tries=tries+1 time.sleep(0.5) data=None return data
def get_probe_angles(input_data, closed=False): """ return a list of thetas for a given signal (timeseries) or a string that specifies it. get_probe_angles('W7X:W7X_MIRNOV_41_BEST_LOOP:(20180912,43)') This is a kludgey way to read coordinates. Should be through acquisition.base or acquisition.'device' rather than looking up config directly """ import pyfusion if isinstance(input_data, str): pieces = input_data.split(':') if len(pieces) == 3: dev_name, diag_name, shotstr = pieces shot_number = eval(shotstr) dev = pyfusion.getDevice(dev_name) data = dev.acq.getdata(shot_number, diag_name, time_range=[0, 0.1]) else: from pyfusion.data.timeseries import TimeseriesData, Timebase, Signal from pyfusion.data.base import Channel, ChannelList, Coords input_data = TimeseriesData(Timebase([0, 1]), Signal([0, 1])) dev_name, diag_name = pieces # channels are amongst options opts = pyfusion.config.pf_options('Diagnostic', diag_name) chans = [ pyfusion.config.pf_get('Diagnostic', diag_name, opt) for opt in opts if 'channel_' in opt ] # for now, assume config_name is some as name input_data.channels = ChannelList( *[Channel(ch, Coords('?', [0, 0, 0])) for ch in chans]) Phi = np.array([ 2 * np.pi / 360 * float( pyfusion.config.get( 'Diagnostic:{cn}'.format( cn=c.config_name if c.config_name != '' else c.name), 'Coords_reduced').split(',')[0]) for c in input_data.channels ]) Theta = np.array([ 2 * np.pi / 360 * float( pyfusion.config.get( 'Diagnostic:{cn}'.format( cn=c.config_name if c.config_name != '' else c.name), 'Coords_reduced').split(',')[1]) for c in input_data.channels ]) if closed: Phi = np.append(Phi, Phi[0]) Theta = np.append(Theta, Theta[0]) return (dict(Theta=Theta, Phi=Phi))
def get_data(self, ): # data_list = [] # new_signal_length = 0 # for i, arr in enumerate(self.array): # data_list.append(pf.getDevice('H1').acq.getdata(self.shot, arr).reduce_time([self.start_time, self.end_time])) # if i==0: # self.timebase = data_list[-1].timebase # else: # data_list[-1] = data_list[-1].change_time_base(self.timebase) # new_signal_length += data_list[-1].signal.shape[0] # new_sig = np.zeros((new_signal_length,self.timebase.shape[0]), dtype = float) # start_ind = 0 # self.data = data_list[0] # for i in data_list: # end_ind = start_ind + i.signal.shape[0] # new_sig[start_ind:end_ind, :] = +i.signal # start_ind = end_ind # self.data.signal = +new_sig # print self.data.signal.shape self.get_primary_array() #self.data = pf.getDevice('H1').acq.getdata(self.shot, self.array).reduce_time([self.start_time, self.end_time]) #self.data = self.data.subtract_mean(copy=False)#.normalise(method='v',separate=True,copy=False) #self.timebase = self.data.timebase print self.other_arrays, self.other_array_labels #if self.other_arrays == None: self.other_array_labels = [] #if self.other_arrays == None: self.other_arrays = []; #if self.meta_data == None : self.meta_data = [] self.other_arrays_fft = [] for i in self.other_arrays: #tmp = pf.getDevice('H1').acq.getdata(self.shot, i).change_time_base(self.data.timebase) tmp = pf.getDevice('DIIID').acq.getdata( self.shot, i).change_time_base(self.data.timebase) self.other_arrays_fft.append( tmp.generate_frequency_series(self.samples, self.samples / self.overlap)) self.instance_array_list = [] self.misc_data_dict = {} #How to deal with the static case? for i in self.other_array_labels: if i[0] != None: self.misc_data_dict[i[0]] = [] if i[1] != None: self.misc_data_dict[i[1]] = [] #self.fs_values = ['p','a12','H','freq','E'] for i in self.meta_data: self.misc_data_dict[i] = []
def get_data(self,): self.data = pf.getDevice('H1').acq.getdata(self.shot, self.array).reduce_time([self.start_time, self.end_time]) self.data = self.data.subtract_mean(copy=False).normalise(method='v',separate=True,copy=False) self.data_segmented = self.data.segment(self.samples, overlap = self.overlap, datalist = 1) print self.other_arrays, self.other_array_labels if self.other_arrays == None: self.other_array_labels = [] if self.other_arrays == None: self.other_arrays = []; if self.meta_data == None : self.meta_data = [] self.other_arrays_segmented = [] for i in self.other_arrays: tmp = pf.getDevice('H1').acq.getdata(self.shot, i).change_time_base(self.data.timebase) self.other_arrays_segmented.append(tmp.segment(self.samples, overlap = self.overlap, datalist = 1)) self.instance_array_list = [] self.misc_data_dict = {'RMS':[],'time':[], 'svs':[]} #How to deal with the static case? for i in self.other_array_labels: if i[0]!=None: self.misc_data_dict[i[0]] = [] if i[1]!=None: self.misc_data_dict[i[1]] = [] self.fs_values = ['p','a12','H','freq','E'] for i in self.meta_data: self.misc_data_dict[i]=[] for i in self.fs_values: self.misc_data_dict[i]=[]
def test_shot_flucstrucs(self): """Just check that the number of flucstrucs is the same whether we use flucstruc directly on the shot data with the segment kwarg or whether we explicitly call the segment method. """ n_samples = 90 dev = pyfusion.getDevice("TestDevice") # version with explicit calls to segment: explicit_data = dev.getdata(12345,"test_multichannel_timeseries_large") explicit_dataset = pyfusion.data.base.DataSet() for seg in explicit_data.segment(n_samples): explicit_dataset.update(seg.flucstruc()) # version using flucstruc segment shortcut shortcut_flucstrucs = dev.getdata(12345,"test_multichannel_timeseries_large").flucstruc(segment=n_samples) self.assertEqual(len(explicit_dataset), len(shortcut_flucstrucs))
def get_MPM_data(shot_number=[20160309,32]): """ retrieve MPM x,y,z from hardware channel 15. Note that the sample rate is 5MS/s so we should use downsampling of some sort """ dev_name = 'W7M' dev = pyfusion.getDevice(dev_name) data = dev.acq.getdata(shot_number, 'W7M_MPM_R') MPM_r = data.signal x0, y0, z0 = -5930.11862, -2254.21244, -168 x1, y1, z1 = -5604.82757, -2125.03934, -168 veclen = np.sqrt((x1-x0)**2+(y1-y0)**2+(z1-z0)**2); MPM_r = -1 * MPM_r xpos = x0/1e3 - MPM_r * (x1-x0)/veclen ypos = y0/1e3 - MPM_r * (y1-y0)/veclen zpos = z0/1e3 - MPM_r * (z1-z0)/veclen return(np.array([xpos, ypos, zpos])/1e0)
def get_mag_config(shot, from_db=True, ratios=True): dev = pyfusion.getDevice('W7X') currents = [] try: if from_db: coil_list = ',' + ','.join('ITrim_{i}'.format(i=i) for i in range(1, 6)) coil_list += ',' + ','.join('IPlanar_{a}'.format(a=a) for a in ['A', 'B']) coil_list += ',' + ','.join('INonPlanar_{i}'.format(i=i) for i in range(1, 6)) coil = coil_list # convenience for error message result = conn.execute('select shot' + coil_list + ' from summ where shot={shot}'.format( shot=shot[0] * 1000 + shot[1])) data = result.fetchone() ddat = dict(data) if ddat['INonPlanar_1'] is None: pyfusion.logger.warning('Guessing NonPlanar currents') ddat['INonPlanar_1'] = 12400. ddat['IPlanar_B'] = 4836. ddat.update(dict(ratio=ddat['IPlanar_B'] / ddat['INonPlanar_1'])) ref = ddat['INonPlanar_1'] return ([ data.INonPlanar_1, ddat['IPlanar_B'] / ref, ddat['IPlanar_B'] / ref ], ddat) else: for i, coil in enumerate(['NonPlanar_1', 'Planar_A', 'Planar_B']): data = dev.acq.getdata(shot, 'W7X_I' + coil) currents.append(np.average(data.signal)) if ratios and coil is not 'NonPlanar_1': currents[i] = currents[i] / currents[ 0] # no point rounding - still get 0.390000001 return currents, None # None instead of the dictionary except Exception as reason: print('data not found for coil {coil}, shot {shot} \n{r}'.format( coil=coil, shot=shot, r=reason)) return None, None
def test_device_getdatat_multishot(self): dev = pyfusion.getDevice("TestDevice") diag = "test_timeseries_shot_unique" shot_list = [100, 200, 300] # expected to get a dataset with the data from each shot expected_dataset = pyfusion.data.base.DataSet() for shot in shot_list: expected_dataset.add(dev.acq.getdata(shot, diag)) dataset = dev.getdata(shot_list, diag) # TODO: better checking if two datasets are same. ## using sets checks if the object is same, not using __eq__ ? for i in dataset: is_in_other = False for j in expected_dataset: if i == j: is_in_other = True if not is_in_other: assert False
ne_set = 'ElectronDensity' shot_range = range(86507, 86517+1) # the +1 ensures 86517 is the last shot time_range = [0.02, 0.04] n_samples = 1024 # define the time interval for averaging n_e overlap=1.0 # take an extra n_samples for overlap - 1/2 before and 1/2 after exception = Exception import pyfusion.utils exec(pf.utils.process_cmd_line_args()) # prepare an empty lists for data - lists are easy to append to ne_profile = [] shot = [] t_mid = [] dev = pf.getDevice(device_name) # open the device (choose the experiment) for shot_number in shot_range: try: d = dev.acq.getdata(shot_number, ne_set) if time_range != None: d.reduce_time(time_range) sections = d.segment(n_samples, overlap) print(d.history, len(sections)) for ss,t_seg in enumerate(sections): ne_profile.append(np.average(t_seg.signal,axis=1)) t_mid.append(np.average(t_seg.timebase)) shot.append(shot_number) except exception as reason:
_var_defaults = """ verbose=1 dev_name = "W7X" sweepV_name = "W7X_L57_LP01_U" shot_number = [20160310,9] t_range=[1.2,1.4] method = 2 sweep_freq = 500 # sinusoidal sweep freq in Hz Vpp = 90*2 # pp value of signal before it was clipped clip_level_minus = -88 # value to ensure even soft clipping is excluded """ locs_before = locals().copy() exec(_var_defaults) locs_after = locals().copy() from pyfusion.utils import process_cmd_line_args exec(process_cmd_line_args()) vars = {} for var in locs_after: if not var in locs_before: vars.update({var: locals()[var]}) for k in 'sweepV_name,dev_name,shot_number,locs_before'.split(','): vars.pop(k) dev = pyfusion.getDevice(dev_name) data = dev.acq.getdata(shot_number,sweepV_name) x_restored = restore_sin(data, **vars)
def test_getdevice(self): h1test = pyfusion.getDevice("H1") from pyfusion.devices.H1.device import H1 self.assertTrue(isinstance(h1test, H1))
import pyfusion as pf dev = pf.getDevice("W7X") data = dev.acq.getdata([20181009, 24], 'W7X_MIRNOV_41_3') data.plot_signals()
def test_getDevice_from_pf(self): device = pyfusion.getDevice("H1")
Needs full data files, ** so won't run with download package alone.** Note: PDF versions of the figure need rasterization to properly render the fine lines. Beware memory problems if you use rasterization on anything but the figure as a whole. See example2b.py for more info. """ import pyfusion as pf # (we will assume these three import lines in all future examples) import numpy as np import matplotlib.pyplot as plt import os plt.figure('Example 2 extra , Figure 6') mono = False # monochrome figures required by JSPF if not os.path.isfile('ne_profile2.npz'): # if not there, make it print('generating npz file') dev = pf.getDevice( 'H1Local' ) # open the device (choose the experiment: e.g H-1, LHD, Heliotron-J) # prepare empty lists for ne_profile, shot and time of measurement ne_profile, t_mid, shot = [], [], [] # for shot_number in range(86507, 86517+1): # the +1 ensures 86517 is the last shot lst = list(range(83130, 83212 + 1, 1)) # 133,162,163 weird, 166 172 missing, 171 and 205 channel8 starts late. for sh in [83133, 83162, 83163, 83166, 83171, 83172, 83205]: lst.remove(sh) try: d = dev.acq.getdata(lst[0], 'ElectronDensity15') except LookupError as reason: raise LookupError( '{r}\n\n** Only works with access to full H1 data set **'.format( r=reason))
""" Example 1, JSPF tutorial: simple density profile scan In this file, data was downsampled to save space in the download package. # The following line was used to downsample the data: run pyfusion/examples/save_to_local.py shot_list=range(86507,86517+1) overwrite_local=1 dev_name='H1Local' diag_name='ElectronDensity15' downsample=100 local_dir='pyfusion/examples/JSPF_tutorial/local_data' """ import pyfusion as pf # (we will assume these three import lines in all future examples) import numpy as np import matplotlib.pyplot as plt import os plt.figure('Example 1') dev = pf.getDevice('H1Local') # open the device (choose the experiment - e.g H-1, LHD, Heliotron-J) ne_profile, t_mid, shot = [ ], [ ], [ ] # prepare empty lists for ne_profile, shot and time of measurement # next line redirects pyfusion to find downsampled local data in ./local_data pf.config.set('global','localdatapath','local_data') for shot_number in range(86507, 86517+1): # the +1 ensures 86517 is the last shot d = dev.acq.getdata(shot_number, 'ElectronDensity15') # a multichannel diagnostic sections = d.segment(n_samples=.001) # break into time segments # work through each time segment, extracting the average density during that time for seg in sections: ne_profile.append(np.average(seg.signal,axis=1)) # axis=1 -> average over time, not channel t_mid.append(np.average(seg.timebase)) shot.append(shot_number) # store the data in a DA (Dictionary of Arrays) object, which is like a DataFrame in R or python panda myDA = pf.data.DA_datamining.DA(dict(shot=shot, ne_profile=ne_profile, t_mid=t_mid))
import pyfusion import numpy as np import pylab as pl from pyfusion.utils import fix2pi_skips, modtwopi diag_name = 'VSL_SMALL' diag_name = 'MP' dev=pyfusion.getDevice("LHD") data=dev.acq.getdata(27233,diag_name) #data.plot_signals() # extract the phi DIrectly from the cfg file Phi = np.array([2*np.pi/360*float(pyfusion.config.get ('Diagnostic:{cn}'. format(cn=c.name), 'Coords_reduced') .split(',')[0]) for c in data.channels]) pl.subplot(121) pl.plot(Phi, linestyle='steps') Phi_circ = fix2pi_skips(np.append(Phi, Phi[0]), around=0) dp = np.diff(Phi_circ) pl.plot(dp, linestyle='steps') pl.figure() pl.subplot(122) phases = [] for N in range(-5, 5): ph = modtwopi(N*dp, offset=0) phases.append(ph) pl.plot(phases[-1], label='N={N}'.format(N=N)) pl.legend()
def test_acq(self): test_device = pyfusion.getDevice("TestWebDevice") test_data = test_device.acq.getdata(58123, "TestMirnovOne") self.assertAlmostEqual(test_data.signal[0], 0.00244184375)
from numpy import sqrt, mean import pyfusion as pf device_name = 'H1LocalSmall' # 'LHD' n_samples = 512 overlap = 1.0 exception = Exception time_range = None min_svs = 1 max_H = 0.97 import pyfusion.utils exec(pf.utils.process_cmd_line_args()) dev = pf.getDevice(device_name) #min_shot = 84000 #max_shot = 94000 #every_nth = 10 #shot_range = range(int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3])) if device_name == 'LHD': shot_range = [27233] diag_name = 'MP' #shot_range = range(90090, 90110) elif device_name.find('H1') >= 0: shot_range = [58123] diag_name = 'H1TestPoloidal' #'H1Poloidal1' #letter l number 1
def testH1Config(self): import pyfusion # print(pyfusion.conf.utils.diff()) h1 = pyfusion.getDevice('H1') test_mirnov = h1.acq.getdata(58123, 'H1_mirnov_array_1_coil_1') self.assertEqual(round(test_mirnov.signal[0], 8), -0.00732422)
""" LHD version of example2.py Takes several minutes. Need to be on site and have access to LHD data. Should be able to paste in the next two parts from example2.py after extracting this. """ import pyfusion as pf # (we will assume these three import lines in all future examples) import numpy as np import matplotlib.pyplot as plt plt.figure('Example 1 - LHD') from pyfusion.utils import get_local_shot_numbers shots = np.sort( get_local_shot_numbers('fircall', local_path='/data/datamining/cache/fircall/', number_posn=[-13, -8])) dev = pf.getDevice('LHD') # open the device (choose the experiment) ne_profile, t_mid, shot = [], [], [ ] # prepare empty lists for ne_profile, shot and time of measurement for shot_number in shots: # the +1 ensures 86517 is the last shot d = dev.acq.getdata(shot_number, 'LHD_n_e_array') # a multichannel diagnostic sections = d.segment(n_samples=128) # break into time segments # work through each time segment, extracting the average density during that time for seg in sections: ne_profile.append(np.average( seg.signal, axis=1)) # axis=1 -> avg over time, not channel t_mid.append(np.average(seg.timebase)) shot.append(shot_number) # store the data in a DA (Dictionary of Arrays) object, which is like a DataFrame in R or panda myDA = pf.data.DA_datamining.DA( dict(shot=shot, ne_profile=ne_profile, t_mid=t_mid))
maxlen = 20000 # limit detailed plots unless plots > 0 order = dict(Isat=0, Te=1, Vf=2) double=1 delay = None # only affects the error mean value iters = 10 # number of charlie iters first_save = iters - 1 # default is to save just the final point exec(_var_defaults) from pyfusion.utils import process_cmd_line_args exec(process_cmd_line_args()) if time_range is not None: ROI = ' '.join([str('{t:.6f}'.format(t=t)) for t in time_range]) pyfusion.config.set('Acquisition:W7M', 'ROI', ROI) dev = pyfusion.getDevice("W7M") ip_data = dev.acq.getdata(shot_number, 'W7M_MLP_I') vp_data = dev.acq.getdata(shot_number, 'W7M_MLP_U') isat_data = dev.acq.getdata(shot_number, 'W7M_MLP_IS') # Not much use to run without CK, but at least get it so that slopes can be calculated ck_data = dev.acq.getdata(shot_number, 'W7M_MLP_CK', contin=True) err_data = dev.acq.getdata(shot_number, 'W7M_MLP_ERR') # if time range has a 3rd element, it is an ROI, and the bounds have been set # assuming we are not using an npz. if time_range is not None and len(time_range) == 2: for dat in [ip_data, vp_data, isat_data, ck_data, err_data]: if dat is not None: dat.reduce_time(time_range, copy=False) if (len(ip_data.timebase) > maxlen) and plots > 1: wait_for_confirmation('Detailed plots of {n} points will be slow!'
bads = [] verbose = 0 exclude = ['NE_14', 'NE_5'] # these are masked dev_name = 'H1Local' diag = 'ElectronDensity15' shot_range=range(92763,92810+1) shot_range=range(92902,92905+1) shot_range=range(92919,92966+1) dt=0.005 """ exec(_var_defaults) from pyfusion.utils import process_cmd_line_args exec(process_cmd_line_args()) dev = pf.getDevice(dev_name) # open the device (choose the experiment - e.g H-1, LHD, Heliotron-J) ne_profile, t_mid, shot = [ ], [ ], [ ] # prepare empty lists for ne_profile, shot and time of measurement for shot_number in shot_range: # the +1 ensures 86517 is the last shot try: d = dev.acq.getdata(shot_number, diag) # a multichannel diagnostic sections = d.segment(n_samples=dt) # break into time segments # work through each time segment, extracting the average density during that time for seg in sections: ne_profile.append(np.average(seg.signal,axis=1)) # axis=1 -> average over time, not channel t_mid.append(np.average(seg.timebase)) shot.append(shot_number) except Exception as reason: bads.append([shot_number, reason]) msg = 'skipping shot {s}' if verbose>0:
def testH1Config(self): import pyfusion # print(pyfusion.conf.utils.diff()) h1 = pyfusion.getDevice('H1') test_mirnov = h1.acq.getdata(58123, 'H1_mirnov_array_1_coil_1') self.assertEqual(round(test_mirnov.signal[0],8), -0.00732422)
# set these both to () to stop on errors shot_exception = () # Exception # () to see message - catches and displays all errors node_exception = () if debug > 0 else Exception # () ditto for nodes. errs = dict(shot=[]) # error list for the shot overall (i.e. if tree is not found) for diag in diags.keys(): errs.update({diag:[]}) # error list for each diagnostic print("""If off-line, set pyfusion.TIMEOUT=0 to prevent long delays The search for valid shot numbers is now much quicker """) # will take a while (~ 0.7 sec/day?), about 2 mins as of 10/2017 start = seconds() dev = pyfusion.getDevice(devname) # 'H1Local') if 'W7' in devname: if (len(np.shape(srange)) == 3) or (np.shape(srange) == (2,2)): ansexp = input('About to expand shot range: Continue? (Y/n/q)') if len(ans)==0 or ans.lower()[0] != 'y': sys.exit() pyfusion.logging.info(str('Starting with {st}, shape is {shp}' .format(st=srangestr, shp=np.shape(srange)))) if len(np.shape(srange)) == 3: # a bunch of begin/ends srange = [sh for sr in srange for sh in expand_shot_range(*sr)] elif np.shape(srange) == (2,2): print('assume a simple range') srange = expand_shot_range(*srange) else: # already a list pass
exec(pf.utils.process_cmd_line_args()) if help==1: print(__doc__) exit() #dev_name='LHD' if dev_name == 'LHD': if diag_name == '': diag_name= 'MP2010' if shot_number is None: shot_number = 27233 #shot_range = range(90090, 90110) elif dev_name.find('H1')>=0: if diag_name == '': diag_name = "H1DTacqAxial" if shot_number is None: shot_number = 69270 device = pf.getDevice(dev_name) try: old_shot except: old_shot=0 if old_shot>0: # we can expect the variables to be still around, run with -i if (old_diag != diag_name) or (old_shot != shot_number): old_shot=0 if old_shot == 0: d = device.acq.getdata(shot_number, diag_name) # ~ 50MB for 6ch 1MS. (27233MP) old_shot = shot_number old_diag = diag_name
import pyfusion as pf pf.config.get('global','database') # 'sqlite:///sqlite.txt' pf.read_config('shaun_feb_2010.cfg') from pyfusion.conf.utils import get_config_as_dict get_config_as_dict('Device','H1') get_config_as_dict('Diagnostic','H1PoloidalAll') import pyfusion as pf h1=pf.getDevice("H1") data=h1.acq.getdata(70071,'H1ToroidalAxial') data.meta.keys() data.plot_signals() # overlay fs on spectrum run examples/plot_specgram.py dev_name='LHD' shot_number=27233 hold=0 time_range=[.35,.5] NFFT=256 noverlap=220 run examples/plot_text_pyfusion.py filename='PF2_120229_MP_27233_27233_1_256.dat' hold=1 min_e=0.8 freq_scale=1e3 colorbar();xlim(0.35,.5);ylim(0,150000) # mode identification overlaid on spectrum - used in LHD report Feb 2012 run examples/plot_specgram.py dev_name='LHD' shot_number=27233 hold=0 time_range=[.35,1.5] NFFT=256 noverlap=220 clim(-210,-40) run examples/mode_identify_example_2012.py hold=1 fsfile='PF2_120229_MP_27233_27233_1_256.dat' xlim(0.35,.5);ylim(0,150000) xlabel('Time (s)'); ylabel('Frequency (kHz)') # chirp following example run examples/plot_text_pyfusion.py filename='PF2_120229_MP_27233_27233_1_256.dat' hold=1 min_e=0.8 freq_scale=1e3 plot=1 time_range=[0.35,.4]
maxrows = 6 maxcols = 8 t_range = [] sharex='all' sharey='all' update = 1 # if 1, show plot panels as they are produced - if more, every updateth # Note that this doesn't help on anaconda windows 2.5, although it should be # useful in environments where we have better control of windows plot_fun='plot_signals' # or 'plot_specgram' # To control the number ofx ticks # gca().locator_params(nbins=4, axis='x') """ exec(_var_defaults) exec(process_cmd_line_args()) dev = pyfusion.getDevice(dev_name) i = 0 axs = None # try to keep grouped shots in columns row_widths = [len(r) for r in shot_list if np.shape(r) is not ()] if len(row_widths ) > 0 and shot_list[0][0] < 20160000: # only for int shots so far print('Detected a grouped shot list - matching ncols and nrows to it.') ncols = np.max(row_widths) nrows = len(shot_list) padded_list = [] for r in range(nrows): padded_row = ncols * [None] for c, sh in enumerate(shot_list[r]): padded_row[c] = sh
scl=500 clim=None boydsdata=1 # this can also be used to check for random matches - should give 0 hits of flipped ylim=(0,100) xlim=(0,0.06) remerge=0 time_range=[0.0,0.06] channel_number=0 """ exec(_var_defaults) exec(process_cmd_line_args()) (tf, tt) = time_range dev_name = "H1Local" device = pf.getDevice(dev_name) diag_name = 'H1ToroidalAxial' NFFT = 2048 noverlap = NFFT * 7 / 8 hold = 1 shot_number = sht cmap = cm.jet # see also cm.gray_r etc tm = localtime() hdr = str('PF2_{yy:02d}{mm:02d}{dd:02d}_'.format(yy=tm.tm_year - 2000, mm=tm.tm_mon, dd=tm.tm_mday, hh=tm.tm_hour)) # a local da wins, if not there then try DA_file try:
def single_shot_fluc_strucs(shot=None, array=None, other_arrays=None, other_array_labels=None, start_time=0.001, end_time = 0.08, samples=1024, power_cutoff = 0.1, n_svs = 2, overlap = 4, meta_data=None): '''This function will extract all the important information from a flucstruc and put it into the form that is useful for clustering using hte clustering module. SH: 8June2013 ''' print(os.getpid(), shot) time_bounds = [start_time, end_time] #extract data for array, naked_coil and ne_array, then reduce_time, interpolate etc... data = pf.getDevice('H1').acq.getdata(shot, array).reduce_time([start_time, end_time]) data = data.subtract_mean(copy=False).normalise(method='v',separate=True,copy=False) data_segmented = data.segment(samples,overlap=overlap, datalist = 1) print(other_arrays, other_array_labels) if other_arrays is None: other_array_labels = [] if other_arrays is None: other_arrays = []; if meta_data is None : meta_data = [] #Get the naked coil and interferometer array if required #Need to include the standard interferometer channels somehow. other_arrays_segmented = [] for i in other_arrays: tmp = pf.getDevice('H1').acq.getdata(shot, i).change_time_base(data.timebase) other_arrays_segmented.append(tmp.segment(samples, overlap = overlap, datalist = 1)) instance_array_list = [] misc_data_dict = {'RMS':[],'time':[], 'svs':[]} #How to deal with the static case? for i in other_array_labels: if i[0]!=None: misc_data_dict[i[0]] = [] if i[1]!=None: misc_data_dict[i[1]] = [] #This should probably be hard coded in... fs_values = ['p','a12','H','freq','E'] for i in meta_data: misc_data_dict[i]=[] for i in fs_values: misc_data_dict[i]=[] #Cycle through the time segments looking for flucstrucs for seg_loc in range(len(data_segmented)): data_seg = data_segmented[seg_loc] time_seg_average_time = np.mean([data_seg.timebase[0],data_seg.timebase[-1]]) fs_set = data_seg.flucstruc() #Need to check my usage of rfft.... seems different to scipy.fftpack.rfft approach other_arrays_data_fft = [] for i in other_arrays_segmented: other_arrays_data_fft.append(np.fft.rfft(i[seg_loc].signal)/samples) if not np.allclose(i[seg_loc].timebase,data_seg.timebase): print("WARNING possible timebase mismatch between other array data and data!!!") d = (data_seg.timebase[1] - data_seg.timebase[0]) val = 1.0/(samples*d) N = samples//2 + 1 frequency_base = np.round((np.arange(0, N, dtype=int)) * val,4) #get the valid flucstrucs valid_fs = [] for fs in fs_set: if (fs.p > power_cutoff) and (len(fs.svs()) >= n_svs): valid_fs.append(fs) #extract the useful information from the valid flucstrucs for fs in valid_fs: for i in fs_values: misc_data_dict[i].append(getattr(fs,i)) misc_data_dict['svs'].append(len(fs.svs())) #for i in meta_values: misc_data_dict[i].append(eval(i)) for i in meta_data: try: misc_data_dict[i].append(copy.deepcopy(data.meta[i])) except KeyError: misc_data_dict[i].append(None) misc_data_dict['RMS'].append((np.mean(data.scales**2))**0.5) misc_data_dict['time'].append(time_seg_average_time) #other array data tmp_loc = np.argmin(np.abs(misc_data_dict['freq'][-1]-frequency_base)) for i,tmp_label in zip(other_arrays_data_fft, other_array_labels): if tmp_label[0]!=None: misc_data_dict[tmp_label[0]].append(np.abs(i[:,0])) if tmp_label[1]!=None: misc_data_dict[tmp_label[1]].append(np.abs(i[:,tmp_loc])) phases = np.array([tmp_phase.delta for tmp_phase in fs.dphase]) phases[np.abs(phases)<0.001]=0 instance_array_list.append(phases) #convert lists to arrays.... for i in misc_data_dict.keys():misc_data_dict[i]=np.array(misc_data_dict[i]) return np.array(instance_array_list), misc_data_dict
import pyfusion as pf h1=pf.getDevice("LHD") data=h1.acq.getdata(27233,'MP') data.plot_signals()
def test_multichannel_mirnov_bean_kappah_as_argument(self): d = pyfusion.getDevice("H1") data = d.acq.getdata(58123, "H1_mirnov_array_1")
""" LHD version of example2.py Takes several minutes. Need to be on site and have access to LHD data. Should be able to paste in the next two parts from example2.py after extracting this. """ import pyfusion as pf # (we will assume these three import lines in all future examples) import numpy as np import matplotlib.pyplot as plt plt.figure("Example 1 - LHD") from pyfusion.utils import get_local_shot_numbers shots = np.sort(get_local_shot_numbers("fircall", local_path="/data/datamining/cache/fircall/", number_posn=[-13, -8])) dev = pf.getDevice("LHD") # open the device (choose the experiment) ne_profile, t_mid, shot = [], [], [] # prepare empty lists for ne_profile, shot and time of measurement for shot_number in shots: # the +1 ensures 86517 is the last shot d = dev.acq.getdata(shot_number, "LHD_n_e_array") # a multichannel diagnostic sections = d.segment(n_samples=128) # break into time segments # work through each time segment, extracting the average density during that time for seg in sections: ne_profile.append(np.average(seg.signal, axis=1)) # axis=1 -> avg over time, not channel t_mid.append(np.average(seg.timebase)) shot.append(shot_number) # store the data in a DA (Dictionary of Arrays) object, which is like a DataFrame in R or panda myDA = pf.data.DA_datamining.DA(dict(shot=shot, ne_profile=ne_profile, t_mid=t_mid)) myDA.save("LHD_ne_profile.npz") myDA.write_arff("ne_profile.arff", ["ne_profile"]) myDA.extract(locals())
def test_kh(self): h1test = pyfusion.getDevice("H1") # shot_kh = (58073, 0.74) shot_kh = (58123, 0.74) # bdb - need to check - what is kappa here? data = h1test.acq.getdata(shot_kh[0], "H1_mirnov_array_1_coil_1")
import pyfusion as pf h1 = pf.getDevice("LHD") data = h1.acq.getdata(27233, 'MP') data.plot_signals()
def test_getDevice_from_pf(self): device = pyfusion.getDevice(self.listed_device)
def single_shot_fluc_strucs_new(shot, array, time_bounds, samples, fs_dictionary = None, serial_number=0, power_cutoff = 0.1, n_svs = 2, ignore_ne_fail = 0, ne_array = 0, naked_coil = 0, overlap = 4): #This is supposed to replace the old single_shot_fluc struc with the new #instance_array, misc_details_dict approach.... data = pf.getDevice('H1').acq.getdata(shot, array).reduce_time(time_bounds) data = data.subtract_mean(copy=False).normalise(method='v',separate=True,copy=False) data_segmented = data.segment(samples,overlap=overlap,datalist = 1) #Get the naked coil and interferometer array if required if ne_array: ne_data = pf.getDevice('H1').acq.getdata(shot, "ElectronDensity").change_time_base(data.timebase) ne_data_segmented = ne_data.segment(samples,overlap=overlap,datalist=1) else: ne_data_segmented = [None for i in data_segmented] if naked_coil: naked_coil = pf.getDevice('H1').acq.getdata(shot, "H1ToroidalNakedCoil").change_time_base(data.timebase) naked_coil_segmented = naked_coil.segment(samples,overlap=overlap,datalist=1) else: naked_coil_segmented = [None for i in data_segmented] meta_data = ['kh','heating_freq','main_current','sec_current', 'shot'] #kh, heating_freq, main_current, sec_current = data.meta['kh'], data.meta['heating_freq'],data.meta['main_current'], data.meta['sec_current'] #data.meta['shot'] = shot #remove the mean, normalise and segment the data.... #Segment the data.... instance_array_list = [] misc_data_dict = {'RMS':[],'time':[], 'svs':[]} if naked_coil: misc_data_dict['naked_coil'] = [] if ne_array: misc_data_dict['ne_mode'] = []; misc_data_dict['ne_static'] = [] meta_values = ['kh','main_current','heating_freq','shot','sec_current'] fs_values = ['p','a12','H','freq','E'] for i in meta_values: misc_data_dict[i]=[] for i in fs_values: misc_data_dict[i]=[] for data_seg, ne_seg, naked_coil_seg in zip(data_segmented, ne_data_segmented, naked_coil_segmented): time_seg_average_time=np.mean([data_seg.timebase[0],data_seg.timebase[-1]]) fs_set = data_seg.flucstruc() if ne_seg!=None: ne_fft = np.fft.rfft(ne_seg.signal)/samples if not np.allclose(ne_seg.timebase,data_seg.timebase): print "WARNING possible timebase mismatch between ne_data and data!!!" if naked_coil_seg!=None: naked_fft = np.fft.rfft(naked_coil_seg.signal)/samples if not np.allclose(naked_coil_seg.timebase,data_seg.timebase): print "WARNING possible timebase mismatch between ne_data and naked coil!!!" d = (data_seg.timebase[1] - data_seg.timebase[0]) val = 1.0/(samples*d) N = samples//2 + 1 frequency_base = np.round((np.arange(0, N, dtype=int)) * val,4) #get the valid flucstrucs valid_fs = [] #make a list of the valid flucstrucs for fs in fs_set: if (fs.p>power_cutoff) and (len(fs.svs())>=n_svs): valid_fs.append(fs) #extract the useful information from the valid flucstrucs for fs in valid_fs: for i in fs_values: misc_data_dict[i].append(getattr(fs,i)) misc_data_dict['svs'].append(fs.svs()) #for i in meta_values: misc_data_dict[i].append(eval(i)) for i in meta_values: #misc_data_dict[i].append(eval(i)) try: misc_data_dict[i].append(copy.deepcopy(data.meta[i])) except KeyError: misc_data_dict[i].append(None) phases = np.array([tmp_phase.delta for tmp_phase in fs.dphase]) phases[np.abs(phases)<0.001]=0 instance_array_list.append(phases) misc_data_dict['RMS'].append((np.mean(data.scales**2))**0.5) misc_data_dict['time'].append(time_seg_average_time) #ne_data and naked_coil_data tmp_loc = np.argmin(np.abs(misc_data_dict['freq'][-1]-frequency_base)) if ne_seg!=None: misc_data_dict['ne_static'].append(np.abs(ne_fft[:,0])) misc_data_dict['ne_mode'].append(ne_fft[:,tmp_loc]) if naked_coil_seg!=None: misc_data_dict['naked_coil'].append(naked_fft[:,tmp_loc]) #convert lists to arrays.... for i in misc_data_dict.keys():misc_data_dict[i]=np.array(misc_data_dict[i]) return np.array(instance_array_list), misc_data_dict
def single_shot_fluc_strucs(shot=None, array=None, other_arrays=None, other_array_labels=None, start_time=0.001, end_time=0.08, samples=1024, power_cutoff=0.1, n_svs=2, overlap=4, meta_data=None): '''This function will extract all the important information from a flucstruc and put it into the form that is useful for clustering using hte clustering module. SH: 8June2013 ''' print(os.getpid(), shot) time_bounds = [start_time, end_time] #extract data for array, naked_coil and ne_array, then reduce_time, interpolate etc... data = pf.getDevice('DIIID').acq.getdata(shot, array).reduce_time( [start_time, end_time]) data = data.subtract_mean(copy=False).normalise(method='v', separate=True, copy=False) data_segmented = data.segment(samples, overlap=overlap, datalist=1) print other_arrays, other_array_labels if other_arrays == None: other_array_labels = [] if other_arrays == None: other_arrays = [] if meta_data == None: meta_data = [] #Get the naked coil and interferometer array if required #Need to include the standard interferometer channels somehow. other_arrays_segmented = [] for i in other_arrays: tmp = pf.getDevice('DIIID').acq.getdata(shot, i).change_time_base( data.timebase) other_arrays_segmented.append( tmp.segment(samples, overlap=overlap, datalist=1)) instance_array_list = [] misc_data_dict = {'RMS': [], 'time': [], 'svs': []} #How to deal with the static case? for i in other_array_labels: if i[0] != None: misc_data_dict[i[0]] = [] if i[1] != None: misc_data_dict[i[1]] = [] #This should probably be hard coded in... fs_values = ['p', 'a12', 'H', 'freq', 'E'] for i in meta_data: misc_data_dict[i] = [] for i in fs_values: misc_data_dict[i] = [] #Cycle through the time segments looking for flucstrucs for seg_loc in range(len(data_segmented)): data_seg = data_segmented[seg_loc] time_seg_average_time = np.mean( [data_seg.timebase[0], data_seg.timebase[-1]]) fs_set = data_seg.flucstruc() #Need to check my usage of rfft.... seems different to scipy.fftpack.rfft approach other_arrays_data_fft = [] for i in other_arrays_segmented: other_arrays_data_fft.append( np.fft.rfft(i[seg_loc].signal) / samples) if not np.allclose(i[seg_loc].timebase, data_seg.timebase): print "WARNING possible timebase mismatch between other array data and data!!!" d = (data_seg.timebase[1] - data_seg.timebase[0]) val = 1.0 / (samples * d) N = samples // 2 + 1 frequency_base = np.round((np.arange(0, N, dtype=int)) * val, 4) #get the valid flucstrucs valid_fs = [] for fs in fs_set: if (fs.p > power_cutoff) and (len(fs.svs()) >= n_svs): valid_fs.append(fs) #extract the useful information from the valid flucstrucs for fs in valid_fs: for i in fs_values: misc_data_dict[i].append(getattr(fs, i)) misc_data_dict['svs'].append(len(fs.svs())) #for i in meta_values: misc_data_dict[i].append(eval(i)) for i in meta_data: try: misc_data_dict[i].append(copy.deepcopy(data.meta[i])) except KeyError: misc_data_dict[i].append(None) misc_data_dict['RMS'].append((np.mean(data.scales**2))**0.5) misc_data_dict['time'].append(time_seg_average_time) #other array data tmp_loc = np.argmin( np.abs(misc_data_dict['freq'][-1] - frequency_base)) for i, tmp_label in zip(other_arrays_data_fft, other_array_labels): if tmp_label[0] != None: misc_data_dict[tmp_label[0]].append((i[:, 0])) if tmp_label[1] != None: misc_data_dict[tmp_label[1]].append((i[:, tmp_loc])) phases = np.array([tmp_phase.delta for tmp_phase in fs.dphase]) phases[np.abs(phases) < 0.001] = 0 instance_array_list.append(phases) #convert lists to arrays.... for i in misc_data_dict.keys(): misc_data_dict[i] = np.array(misc_data_dict[i]) return np.array(instance_array_list), misc_data_dict
from numpy import sqrt, mean import pyfusion as pf device_name = 'H1LocalSmall' # 'LHD' n_samples = 512 overlap=1.0 exception=Exception time_range = None min_svs=1 max_H=0.97 import pyfusion.utils exec(pf.utils.process_cmd_line_args()) dev = pf.getDevice(device_name) #min_shot = 84000 #max_shot = 94000 #every_nth = 10 #shot_range = range(int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3])) if device_name == 'LHD': shot_range = [27233] diag_name= 'MP' #shot_range = range(90090, 90110) elif device_name.find('H1')>=0: shot_range = [58123] diag_name = 'H1TestPoloidal' #'H1Poloidal1' #letter l number 1