for seg in sections: ne_profile.append(np.average(seg.signal,axis=1)) # axis=1 -> average over time, not channel t_mid.append(np.average(seg.timebase)) shot.append(shot_number) except Exception as reason: bads.append([shot_number, reason]) msg = 'skipping shot {s}' if verbose>0: msg += ' because\n {r}' print(msg.format(s=shot_number, r=reason)) # store the data in a DA (Dictionary of Arrays) object, which is like a DataFrame in R or python panda #myDA = pf.data.DA_datamining.Masked_DA( # DAordict = dict(shot=shot, ne_profile=np.array(ne_profile), t_mid=t_mid), valid_keys=['ne_profile'], mask=0) myDA = DA(dict(shot=shot, ne_profile=np.array(ne_profile), t_mid=t_mid)) myDA.masked = Masked_DA(valid_keys=['ne_profile'], baseDA=myDA) myDA.da['mask'] = np.ones(shape=np.shape(myDA[myDA.masked.valid_keys[0]])).astype(np.uint8) channels = [ch.name.split(':')[-1] for ch in seg.channels] myDA.infodict.update(dict(channels = channels)) for (c, ch) in enumerate(channels): if np.any([ex in ch for ex in exclude]): # if any of exclude in that channel myDA.da['mask'][:,c] = 0 myDA.save('/tmp/density_scan') # the next step - write to arff # myDA.write_arff('ne_profile.arff',['ne_profile']) import matplotlib.pyplot as plt def pl(array, comment=None,**kwargs): plt.figure(num=comment) # coment written to window title plt.plot(array, **kwargs)
def write_DA(self, filename): from pyfusion.data.DA_datamining import DA, Masked_DA dd = {} res = np.array(self.fitdata, dtype=np.float32) nt = len(res) nc = len(res[0]) for key in ['date', 'progId', 'shot']: dd[key] = np.zeros(nt, dtype=np.int64) dd['date'][:] = self.shot[0] dd['progId'][:] = self.shot[1] dd['shot'][:] = self.shot[1] + 1000*self.shot[0] for key in ['nits','maxits']: dd[key] = np.zeros([nt,nc], dtype=np.uint16) # make all the f32 arrays - note - ne is just I0 for now - fixed below lookup = [(0, 't_mid'), (1, 'Te'), (2, 'Vf'), (3, 'I0'), (4, 'resid'), (5, 'nits'), (6, 'maxits'), (7, 'Ie_Ii'), (3, 'ne18')] if self.fitter.fit_params.get('esterr',False): lookup.extend([(8, 'eTe'), (9, 'eVf'), (10, 'eI0') ]) for (ind, key) in lookup: if key not in dd: dd[key] = np.zeros([nt, nc], dtype=np.float32) dd[key][:] = res[:, :, ind] # fudge t_mid is not a vector...should fix properly dd['t_mid'] = dd['t_mid'][:, 0] dd['info'] = dict(params=self.actual_params, coords=[self.coords[ic] for ic in self.select], #area=[self.area[ic] for ic in self.select], # needs to be in npz file etc first shotdata=dict(shot=[self.shot], utc_ns=[self.imeas.utc[0]]), channels=[chn.replace(self.dev.name+'_', '') .replace('_I', '') for chn in [self.i_chans[ic] for ic in self.select]], orig_name = os.path.split(filename)[-1], username = os.getlogin()) da = DA(dd) da.masked = Masked_DA(['Te', 'I0', 'Vf', 'ne18', 'Ie_Ii'], baseDA=da) # da.da['mask']=(da['resid']/abs(da['I0']) < .7) & (da['nits']<100) # da.da['mask'] = ((da['resid']/abs(da['I0']) < .7) & (da['nits'] < da['maxits']) # from version 0.7.0 onwards, resid is already normed to I0 lpf = self.fitter.actual_fparams['lpf'] # Note: these multilines ('down to here') can be applied to a DA by # pasting to reset mask AFTER uncommenting the following # line # lpf = da['info']['params']['actual_fit_params']['lpf'] rthr = 0.7 # LP20160309_29_L53__amoebaNone1.2N_2k.npz is < .12 others # None 0310_9 up to 0.7-0.8 if lpf is not None: rthr = rthr * np.sqrt(lpf/100.0) da.da['mask'] = ((da['resid'] < rthr) & (da['nits'] < da['maxits']) & (np.abs(da['Vf']) < 200) & (np.abs(da['Te']) < 200) & (da['I0']>0.0004)) # additional restriction applied if the error estimate is available if 'eTe' in da.da: # want error not too big and smaller than temp da.da['mask'] &= ((np.abs(da['eTe']) < 100) & (np.abs(da['eTe']) < np.abs(da['Te']))) # down to here qe = 1.602e-19 mp = 1.67e-27 fact = 1/(0.6*qe)*np.sqrt(self.amu*mp/(qe))/1e18 # units of 1e18 # check if each channel has an area for (c, chn) in enumerate([self.i_chans[ic] for ic in self.select]): cd = get_config_as_dict('Diagnostic', chn) A = cd.get('area', None) if A is None: A = 1.0e-6 pyfusion.logging.warn('Defaulting area for {chn} to {A}'.format(chn=chn, A=A)) A = float(A) da.da['ne18'][:, c] = fact/A * da['I0'][:, c]/np.sqrt(da['Te'][:, c]) da.save(filename)