def do_fetch(self): chan_name = (self.diag_name.split('-'))[-1] # remove - filename_dict = {'diag_name': chan_name, 'shot': self.shot} self.basename = path.join(pf.config.get('global', 'localdatapath'), data_filename % filename_dict) files_exist = path.exists(self.basename) if not files_exist: raise Exception, "file " + self.basename + " not found." else: signal_dict = newload(self.basename) if ((chan_name == array(['MP5', 'HMP13', 'HMP05'])).any()): flip = -1. print('flip') else: flip = 1. if self.diag_name[0] == '-': flip = -flip # coords = get_coords_for_channel(**self.__dict__) ch = Channel(self.diag_name, Coords('dummy', (0, 0, 0))) output_data = TimeseriesData( timebase=Timebase(signal_dict['timebase']), signal=Signal(flip * signal_dict['signal']), channels=ch) # bdb - used "fetcher" instead of "self" in the "direct from LHD data" version output_data.config_name = self.config_name # when using saved files, same as name output_data.meta.update({'shot': self.shot}) return output_data
def do_fetch(self): channel_length = int(self.length) outdata=np.zeros(1024*2*256+1) ## !! really should put a wrapper around gethjdata to do common stuff # outfile is only needed if the direct passing of binary won't work # with tempfile.NamedTemporaryFile(prefix="pyfusion_") as outfile: ierror, getrets = gethjdata.gethjdata(self.shot,channel_length,self.path, verbose=VERBOSE, opt=1, ierror=2, outdata=outdata, outname='') if ierror != 0: raise LookupError('hj Okada style data not found for {s}:{c}'.format(s=self.shot, c=self.path)) ch = Channel(self.path, Coords('dummy', (0,0,0))) # the intent statement causes the out var to be returned in the result lsit # looks like the time,data is interleaved in a 1x256 array # it is fed in as real*64, but returns as real*32! (as per fortran decl) debug_(pyfusion.DEBUG, 4, key='Heliotron_fetch', msg='after call to getdata') # timebase in secs (ms in raw data) - could add a preferred unit? # this is partly allowed for in savez_compressed, newload, and # for plotting, in the config file. # important that the 1e-3 be inside the Timebase() output_data = TimeseriesData(timebase=Timebase(1e-3 * getrets[1::2]), signal=Signal(getrets[2::2]), channels=ch) output_data.meta.update({'shot':self.shot}) if pyfusion.VERBOSE>0: print('HJ config name',self.config_name) output_data.config_name = self.config_name stprms = get_static_params(shot=self.shot,signal=self.path) if len(list(stprms)) == 0: # maybe this should be ignored - how do we use it? raise LookupError(' failure to get params for {shot}:{path}' .format(shot=self.shot, path=self.path)) output_data.params = stprms return output_data
def do_fetch(self): chan_name = (self.diag_name.split('-'))[-1] # remove - filename_dict = {'diag_name':chan_name, 'shot':self.shot} self.basename = path.join(pf.config.get('global', 'localdatapath'), data_filename %filename_dict) files_exist = path.exists(self.basename) if not files_exist: raise Exception, "file " + self.basename + " not found." else: signal_dict = newload(self.basename) if ((chan_name == array(['MP5','HMP13','HMP05'])).any()): flip = -1. print('flip') else: flip = 1. if self.diag_name[0]=='-': flip = -flip # coords = get_coords_for_channel(**self.__dict__) ch = Channel(self.diag_name, Coords('dummy', (0,0,0))) output_data = TimeseriesData(timebase=Timebase(signal_dict['timebase']), signal=Signal(flip*signal_dict['signal']), channels=ch) # bdb - used "fetcher" instead of "self" in the "direct from LHD data" version output_data.config_name = self.config_name # when using saved files, same as name output_data.meta.update({'shot':self.shot}) return output_data
def do_fetch(self): channel_length = int(self.length) outdata = np.zeros(1024 * 2 * 256 + 1) ## !! really should put a wrapper around gethjdata to do common stuff # outfile is only needed if the direct passing of binary won't work # with tempfile.NamedTemporaryFile(prefix="pyfusion_") as outfile: # get in two steps to make debugging easier allrets = gethjdata.gethjdata(self.shot, channel_length, self.path, verbose=VERBOSE, opt=1, ierror=2, isample=-1, outdata=outdata, outname='') ierror, isample, getrets = allrets if ierror != 0: raise LookupError( 'hj Okada style data not found for {s}:{c}'.format( s=self.shot, c=self.path)) ch = Channel(self.path, Coords('dummy', (0, 0, 0))) # the intent statement causes the out var to be returned in the result lsit # looks like the time,data is interleaved in a 1x256 array # it is fed in as real*64, but returns as real*32! (as per fortran decl) debug_(pyfusion.DEBUG, 4, key='Heliotron_fetch', msg='after call to getdata') # timebase in secs (ms in raw data) - could add a preferred unit? # this is partly allowed for in savez_compressed, newload, and # for plotting, in the config file. # important that the 1e-3 be inside the Timebase() output_data = TimeseriesData(timebase=Timebase( 1e-3 * getrets[1::2][0:isample]), signal=Signal(getrets[2::2][0:isample]), channels=ch) output_data.meta.update({'shot': self.shot}) if pyfusion.VERBOSE > 0: print('HJ config name', self.config_name) output_data.config_name = self.config_name stprms = get_static_params(shot=self.shot, signal=self.path) if len(list(stprms) ) == 0: # maybe this should be ignored - how do we use it? raise LookupError( ' failure to get params for {shot}:{path}'.format( shot=self.shot, path=self.path)) output_data.params = stprms return output_data
def do_fetch(self): if pyfusion.DBG() > 2: print('in fetch',self.diag_name, self) debug_(pyfusion.DEBUG, level=3, key='igetfile_fetch') diag = self.config_name infodict = eval(eval(self.info)) vardict = get_basic_diagnostics(diags=[diag], times=None, shot=self.shot, file_info={diag:infodict}, debug=1, exception=None) debug_(pyfusion.DEBUG, level=2, key='after get_basic') output_data = TimeseriesData(timebase=Timebase(vardict['check_tm']), signal=Signal(vardict[self.config_name]), channels=Channel(self.config_name,Coords('dummy',(0,0,0)))) output_data.config_name = self.config_name # ??? bdb - my fault? return output_data
def do_fetch(self): print(self.pointname) print(self.shot) if self.NC!=None: print(self.NC) t_name = '{}_time'.format(self.pointname) NC_vars = self.NC.variables.keys() if self.pointname in NC_vars: print('Reading cache!!!!') t_axis = self.NC.variables[t_name].data[:].copy() data = self.NC.variables[self.pointname].data[:].copy() else: tmp = self.acq.connection.get('ptdata2("{}",{})'.format(self.pointname, self.shot)) data = tmp.data() tmp = self.acq.connection.get('dim_of(ptdata2("{}",{}))'.format(self.pointname, self.shot)) t_axis = tmp.data() self.write_cache = True print(t_axis) print(data) coords = get_coords_for_channel(**self.__dict__) ch = Channel(self.pointname, coords) # con=MDS.Connection('atlas.gat.com::') # pointname = 'MPI66M067D' # shot = 164950 # tmp = con.get('ptdata2("{}",{})'.format(pointname, shot)) # dat = tmp.data() # tmp = con.get('dim_of(ptdata2("{}",{}))'.format(pointname, shot)) # t = tmp.data() if self.NC!=None and self.write_cache: print self.pointname self.NC.createDimension(t_name, len(t_axis)) f_time = self.NC.createVariable(t_name,'d',(t_name,)) f_time[:] = +t_axis print('Wrote time') sig = self.NC.createVariable(self.pointname,'f',(t_name,)) sig[:] = +data print('Wrote signal') output_data = TimeseriesData(timebase=Timebase(t_axis), signal=Signal(data), channels=ch) # output_data = super(DIIIDDataFetcherPTdata, self).do_fetch() # coords = get_coords_for_channel(**self.__dict__) # ch = Channel(self.mds_path, coords) # output_data.channels = ch # output_data.meta.update({'shot':self.shot, 'kh':self.get_kh()}) # print(ch) output_data.config_name = ch self.fetch_mode = 'ptdata' return output_data
def get_tsd_from_node(fetcher, node): """Return pyfusion TimeSeriesData corresponding to an MDSplus signal node.""" # TODO: load actual coordinates ch = Channel(fetcher.mds_path_components['nodepath'], Coords('dummy', (0, 0, 0))) signal = Signal(node.data()) dim = node.dim_of().data() # TODO: stupid hack, the test signal has dim of [[...]], real data # has [...]. Figure out why. (...probably because original signal # uses a build_signal function) if len(dim) == 1: dim = dim[0] timebase = Timebase(dim) output_data = TimeseriesData(timebase=timebase, signal=signal, channels=ch) output_data.config_name = fetcher.config_name #bdb config_name fix output_data.meta.update({'shot': fetcher.shot}) return output_data
def get_tsd_from_node(fetcher, node): """Return pyfusion TimeSeriesData corresponding to an MDSplus signal node.""" # TODO: load actual coordinates ch = Channel(fetcher.mds_path_components['nodepath'], Coords('dummy', (0,0,0))) signal = Signal(node.data()) dim = node.dim_of().data() # TODO: stupid hack, the test signal has dim of [[...]], real data # has [...]. Figure out why. (...probably because original signal # uses a build_signal function) if len(dim) == 1: dim = dim[0] timebase = Timebase(dim) output_data = TimeseriesData(timebase=timebase, signal=signal, channels=ch) output_data.config_name = fetcher.config_name #bdb config_name fix output_data.meta.update({'shot':fetcher.shot}) return output_data
def do_fetch(self): print("Shot: {}\nPoint Name: {}".format(self.shot, self.pointname)) if not hasattr(self, 'NC'): self.NC = None if self.NC is not None: #print(self.NC) t_name = '{}_time'.format(self.pointname) NC_vars = self.NC.variables.keys() else: NC_vars = [] if self.pointname in NC_vars: print(' Pointname in NC cache, Reading...\n') t_axis = self.NC.variables[t_name].data[:].copy() data = self.NC.variables[self.pointname].data[:].copy() self.write_cache = False else: print(' Fetching from ptdata') tmp = self.acq.connection.get('ptdata2("{}",{})'.format( self.pointname, self.shot)) data = tmp.data() tmp = self.acq.connection.get('dim_of(ptdata2("{}",{}))'.format( self.pointname, self.shot)) t_axis = tmp.data() self.write_cache = True coords = get_coords_for_channel(**self.__dict__) ch = Channel(self.pointname, coords) if self.NC is not None and self.write_cache: print("\t Writing to NC file disabled temporarily.") #print(' Writing pointname to NC file\n') #self.NC.createDimension(t_name, len(t_axis)) #f_time = self.NC.createVariable(t_name,'d',(t_name,)) #f_time[:] = +t_axis # sig = self.NC.createVariable(self.pointname,'f',(t_name,)) #sig[:] = +data output_data = TimeseriesData(timebase=Timebase(t_axis), signal=Signal(data), channels=ch) output_data.config_name = ch self.fetch_mode = 'ptdata' return output_data
def do_fetch(self): sig = self.conn.get(self.mds_path) dim = self.conn.get('DIM_OF(' + self.mds_path + ')') scl = 1.0 coords = get_coords_for_channel(**self.__dict__) ch = Channel(self.config_name, coords) timedata = dim.data() output_data = TimeseriesData(timebase=Timebase(1e-9 * timedata), signal=scl * Signal(sig), channels=ch) output_data.meta.update({'shot': self.shot}) if hasattr(self, 'mdsshot'): # intended for checks - not yet used. output_data.mdsshot = self.mdsshot output_data.config_name = self.config_name output_data.utc = [timedata[0], timedata[-1]] #output_data.units = dat['units'] if 'units' in dat else '' debug_(pyfusion.DEBUG, level=1, key='W7M_do_fetch', msg='entering W7X MDS do_fetch') return (output_data)
def do_fetch(self): chan_name = (self.diag_name.split('-'))[-1] # remove - filename_dict = {'shot':self.shot, # goes with Boyd's local stg 'config_name':self.config_name} #filename_dict = {'diag_name':self.diag_name, # goes with retrieve names # 'channel_number':self.channel_number, # 'shot':self.shot} debug_(pf.DEBUG, 4, key='local_fetch') for each_path in pf.config.get('global', 'localdatapath').split('+'): self.basename = path.join(each_path, data_filename %filename_dict) files_exist = path.exists(self.basename) if files_exist: break if not files_exist: raise Exception("file {fn} not found. (localdatapath was {p})" .format(fn=self.basename, p=pf.config.get('global', 'localdatapath').split('+'))) else: signal_dict = newload(self.basename) if ((chan_name == array(['MP5','HMP13','HMP05'])).any()): flip = -1. print('flip') else: flip = 1. if self.diag_name[0]=='-': flip = -flip # coords = get_coords_for_channel(**self.__dict__) ch = Channel(self.diag_name, Coords('dummy', (0,0,0))) output_data = TimeseriesData(timebase=Timebase(signal_dict['timebase']), signal=Signal(flip*signal_dict['signal']), channels=ch) # bdb - used "fetcher" instead of "self" in the "direct from LHD data" version output_data.config_name = self.config_name # when using saved files, same as name output_data.meta.update({'shot':self.shot}) return output_data
def do_fetch(self): # Definitions: # data_utc: is meant to be the first and last timestamps of the saved # or the retrieved data if not saved. (at least from 3 march 2016) # shot_f, shot_t: are beginning and end of shot (from programs) at least from 3 march 2016 # utc: seems to be same as data_utc # seg_f_u: up til 20 march 2020, seg_f_u appears to be the *requested* segment, (now fixed - renamed to req_f_u) # utc0 - only 9.9 or so. Should be in file or set in base.py # In this revision, the only changes are - allow for self.time_range, # variable names f, t changed to f_u, t_u (the desired data utc) # and comment fixes, # in preparation for including time_range and other cleanups. # My W7X shots are either of the form from_utc, to_utc, # or date (8dig) and shot (progId) # the format is in the acquisition properties, to avoid # repetition in each individual diagnostic t_start = seconds() if self.shot[1] > 1e9 or hasattr( self, 'time_range') and self.time_range is not None: # we have start and end in UTC instead of shot no # need the shot and utc to get t1 to set zero t if hasattr(self, 'time_range') and self.time_range is not None: if self.shot[1] > 1e9: raise ValueError( "Can't supply shot as a utc pair and specify a time_range" ) actual_shot = self.shot f_u = None # set to None to make sure we don't use it else: f_u, t_u = self.shot # Initialize to the requested utc range actual_shot = get_shot_number([f_u, t_u]) progs = get_programs(actual_shot) # need shot to look up progs # need prog to get trigger - not tested for OP1.1 if len(progs) > 1: raise LookupError( 'fetch: too many programs found - covers > 1 shot?') #this_prog = [prog for prog in progs if (f_u >= progs[prog]['from'] and # t_u <= progs[prog]['upto'])] if len(progs) == 1: this_prog = progs.values()[0] # on shot 20180724,10, this trigger[1] is an empty list trigger = this_prog['trigger']['1'] # This fallback to trigger[0] mean that more rubbish shots are saved than # if we only look at the proper trigger (1) - here is an example # run pyfusion/examples/plot_signals shot_number=[20180724,10] diag_name="W7X_UTDU_LP15_I" dev_name='W7X' if len(trigger) == 0: # example above print('** No Trigger 1 on shot {s}'.format(s=actual_shot)) debug_(pyfusion.DEBUG, 0, key="noTrig1", msg="No Trigger 1 found") # take any that exist, at 60 trigger = [ trr[0] + int(60 * 1e9) for trr in this_prog['trigger'].values() if len(trr) > 0 ] if len(trigger) == 0: raise LookupError( 'No Triggers at all on shot {s}'.format( s=actual_shot)) utc_0 = trigger[ 0] # utc_0 is the first trigger (usu 61 sec ahead of data) else: print( 'Unable to look up programs - assuming this is a test shot' ) utc_0 = f_u # better than nothing - probably a 'private' test/calibration shot if f_u is None: # shorthand for have time range f_u = utc_0 + int(1e9 * (self.time_range[0])) # + 61)) t_u = utc_0 + int( 1e9 * (self.time_range[1])) # + 61)) was 61 rel to prog['from'] else: # self.shot is an 8,3 digit shot and time range not specified actual_shot = self.shot f_u, t_u = get_shot_utc( actual_shot ) # f_u is the start of the overall shot - i.e about plasma time -61 sec. # at present, ECH usually starts 61 secs after t0 # so it is usually sufficient to request a later start than t0 pre_trig_secs = self.pre_trig_secs if hasattr( self, 'pre_trig_secs') else 0.3 # should get this from programs really - code is already above. We need to just move it. pyfusion.utils.warn('fetch: still using hard-coded 61 secs') utc_0 = f_u + int(1e9 * (61)) # utc_0 is plasma initiation in utc f_u = utc_0 - int( 1e9 * pre_trig_secs) # f_u is the first time wanted # make sure we have the following defined regardless of how we got here shot_f_u, shot_t_u = get_shot_utc(actual_shot) req_f_u = f_u # req_f_u is the start of the desired data segment - sim. for req_t_u req_t_u = t_u # A URL STYLE diagnostic - used for a one-off rather than an array # this could be moved to setup so that the error info is more complete if hasattr(self, 'url'): fmt = self.url # add from= further down: +'_signal.json?from={req_f_u}&upto={req_t_u}' params = {} # check consistency - # url should be literal - no params (only for fmt) - gain, units are OK as they are not params if hasattr(self, 'params'): pyfusion.utils.warn( 'URL diagnostic {n} should not have params <{p}>'.format( n=self.config_name, p=self.params)) else: # a pattern-based one - used for arrays of probes if hasattr(self, 'fmt'): # does the diagnostic have one? fmt = self.fmt elif hasattr(self.acq, 'fmt'): # else use the acq.fmt fmt = self.acq.fmt else: # so far we have no quick way to check the server is online raise LookupError('no fmt - perhaps pyfusion.cfg has been ' 'edited because the url is not available') params = eval('dict(' + self.params + ')') # Originally added to fix up erroneous ECH alias mapping if ECH - only # 6 sources work if I don't. But it seems to help with many others # This implementation is kludgey but proves the principle, and # means we don't have to refer to any raw.. signals # would be nice if they made a formal way to do this. if 'upto' not in fmt: fmt += '_signal.json?from={req_f_u}&upto={req_t_u}' assert req_f_u == f_u, 'req_f_u error' assert req_t_u == t_u, 'req_t_u error' # params.update(req_f_u=req_f_u, req_t_u=req_t_u, shot_f_u=shot_f_u) url = fmt.format(**params) # substitute the channel params debug_(pyfusion.DEBUG, 2, key="url", msg="middle of work on urls") if np.any([ nm in url for nm in 'Rf,Tower5,MainCoils,ControlCoils,TrimCoils,Mirnov,Interfer,_NBI_' .split(',') ]): from pyfusion.acquisition.W7X.get_url_parms import get_signal_url # replace the main middle bit with the expanded one from the GUI tgt = url.split('KKS/')[1].split('/scaled')[0].split('_signal')[0] # construct a time filter for the shot self.tgt = tgt # for the sake of error_info filt = '?filterstart={req_f_u}&filterstop={req_t_u}'.format( **params) # get the url with the filter url = url.replace(tgt, get_signal_url(tgt, filt)).split('KKS/')[-1] # take the filter back out - we will add the exact one later url = url.replace(filt, '/') # nSamples now needs a reduction mechanism http://archive-webapi.ipp-hgw.mpg.de/ # minmax is increasingly slow for nSamples>10k, 100k hopeless # Should ignore the test comparing the first two elements of the tb # prevent reduction (NSAMPLES=...) to avoid the bug presently in codac if (('nSamples' not in url) and (pyfusion.NSAMPLES != 0) and not (hasattr(self, 'allow_reduction') and int(self.allow_reduction) == 0)): url += '&reduction=minmax&nSamples={ns}'.format( ns=pyfusion.NSAMPLES) debug_(pyfusion.DEBUG, 2, key="url", msg="work on urls") # we need %% in pyfusion.cfg to keep py3 happy # however with the new get_signal_url, this will all disappear if sys.version < '3.0.0' and '%%' in url: url = url.replace('%%', '%') if 'StationDesc.82' in url: # fix spike bug in scaled QRP data url = url.replace('/scaled/', '/unscaled/') if pyfusion.CACHE: # Needed for improperly configured cygwin systems: e.g.IPP Virual PC # Perhaps this should be executed at startup of pyfusion? cygbin = "c:\\cygwin\\bin" if os.path.exists(cygbin) and not cygbin in os.environ['path']: os.environ['path'] += ";" + cygbin print('using wget on {url}'.format(url=url)) retcode = os.system('wget -x "{url}"'.format(url=url)) # retcode = os.system('c:\\cygwin\\bin\\bash.exe -c "/bin/wget {url}"'.format(url=url)) debug_(retcode != 0, level=1, key='wget', msg="wget error or DEBUG='wget'") # now read from the local copy - seems like urls need full paths # appears to be a feature! http://stackoverflow.com/questions/7857416/file-uri-scheme-and-relative-files # /home/bdb112/pyfusion/working/pyfusion/archive-webapi.ipp-hgw.mpg.de/ArchiveDB/codac/W7X/CoDaStationDesc.82/DataModuleDesc.181_DATASTREAM/7/Channel_7/scaled/_signal.json?from=1457626020000000000&upto=1457626080000000000&nSamples=10000 # url = url.replace('http://','file:///home/bdb112/pyfusion/working/pyfusion/') url = url.replace('http://', 'file:/' + os.getcwd() + '/') if 'win' in os.sys.platform: # weven thoug it seems odd, want 'file:/c:\\cygwin\\home\\bobl\\pyfusion\\working\\pyfusion/archive-webapi.ipp-hgw.mpg.de/ArchiveDB/codac/W7X/CoDaStationDesc.82/DataModuleDesc.192_DATASTREAM/4/Channel_4/scaled/_signal.json@from=147516863807215960&upto=1457516863809815961' url = url.replace('?', '@') # nicer replace - readback still fails in Win, untested on unix systems print('now trying the cached copy we just grabbed: {url}'.format( url=url)) if (req_f_u > shot_t_u) or (req_t_u < shot_f_u): pyfusion.utils.warn( 'segment requested is outside the shot times for ' + str(actual_shot)) if pyfusion.VERBOSE > 0: print( '======== fetching url over {dt:.1f} secs from {fr:.1f} to {tt:.1f} =========\n[{u}]' .format(u=url, dt=(params['req_t_u'] - params['req_f_u']) / 1e9, fr=(params['req_f_u'] - shot_f_u) / 1e9, tt=(params['req_t_u'] - shot_f_u) / 1e9)) # seems to take twice as long as timeout requested. # haven't thought about python3 for the json stuff yet # This is not clean - should loop for timeout in [pyfusion.TIMEOUT, 3*pyfusion.TIMEOUT] try: # dat = json.load(urlopen(url,timeout=pyfusion.TIMEOUT)) works # but follow example in # http://webservices.ipp-hgw.mpg.de/docs/howtoREST.html#python # Some extracts in examples/howtoREST.py # dat = json.loads(urlopen(url,timeout=pyfusion.TIMEOUT).read().decode('utf-8')) t_pre = seconds() # for long shots, adjust strategy and timeout to reduce memory consumption ONE = 4 # memory conservation tricks only apply for DEBUG<1 # Thin allows the cutoff value to be increased in come cases # uncomment the following two for testing the exception handler ## timeout = pyfusion.TIMEOUT ## raise httplib.IncompleteRead('test') if (req_t_u - req_f_u) / 1e9 > pyfusion.VERY_LONG: size_MS = 2 * ( req_t_u - req_f_u ) / 1e9 # approximate - later on calc from dt i.e. MSamples if pyfusion.NSAMPLES != 0: # allow for subsampled data size_MS = pyfusion.NSAMPLES / 1e6 timeout = 8 * size_MS + pyfusion.TIMEOUT # don't make timeout too small! print('On-the-fly conversion: Setting timeout to {tmo}'.format( tmo=timeout)) dat = json.load(urlopen(url, timeout=timeout)) t_text = seconds() else: timeout = pyfusion.TIMEOUT txtform = urlopen(url, timeout=timeout).read() t_text = seconds() print('{tm} {tp:.2f} prep, {tt:.2f} fetch without decode, '. format(tm=time.strftime('%H:%M:%S'), tp=t_pre - t_start, tt=t_text - t_pre)), sys.stdout.flush() dat = json.loads(txtform.decode('utf-8')) if pyfusion.DEBUG < ONE: txtform = None # release memory t_conv = seconds() # for 10MS of mirnov 0.06 prep, 9.61 fetch 19.03 conv #print('{tp:.2f} prep, {tt:.2f} fetch {tc:.2f} conv'. print('{tc:.2f} conv'.format(tp=t_pre - t_start, tt=t_text - t_pre, tc=t_conv - t_text)) except socket.timeout as reason: # the following url access is a slightly different form? # should check if this is better tested by the URL module print('{tm} {tp:.2f} prep, {tt:.2f} timeout. '.format( tp=t_pre - t_start, tt=seconds() - t_pre, tm=time.strftime('%H:%M:%S'))), print( '****** first timeout error, try again with longer timeout *****' ) timeout *= 3 dat = json.load(urlopen(url, timeout=timeout)) except MemoryError as reason: raise # too dangerous to do anything else except to reraise except httplib.IncompleteRead as reason: msg = str( '** IncompleteRead after {tinc:.0f}/{timeout:.0f}s ** on {c}: {u} \n{r}' .format(tinc=seconds() - t_start, c=self.config_name, u=url, r=reason, timeout=timeout)) pyfusion.logging.error( msg ) # don't want to disturb the original exception, so raise <nothing> i.e. reraise raise # possibly a memory error really? - not the case for 4114 20180912.48 except Exception as reason: if pyfusion.VERBOSE >= 0: print( '**** Exception (Memory? out of disk space?) OR timeout of {timeout} **** on {c}: {u} \n{r}' .format(c=self.config_name, u=url, r=reason, timeout=timeout)) raise # re raises the last exception # this form will default to repair = 2 for all LP probes. #default_repair = -1 if 'Desc.82/' in url else 0 # Override acq.repair with the probe value default_repair = int(self.repair) if hasattr( self, 'repair') else 2 if 'Desc.82/' in url else 0 # this form follows the config file settings self.repair = int(self.repair) if hasattr(self, 'repair') else default_repair dimraw = np.array(dat['dimensions']) if ('nSamples' not in url): # skip this check if we are decimating if np.abs(req_f_u - dimraw[0]) > 2000: print( '** Start is delayed by >2 us {dtdel:,} relative to the request' .format(dtdel=dimraw[0] - req_f_u)) if (req_t_u - dimraw[-1]) > 2000: print( '** End is earlier by >2 us {dtdel:,} relative to the request' .format(dtdel=req_t_u - dimraw[-1])) output_data_utc = [dat['dimensions'][0], dat['dimensions'][-1]] if pyfusion.DEBUG < ONE: dat['dimensions'] = None # release memory # adjust dim only (not dim_raw so that zero time remains at t1 dim = dimraw - utc_0 # decimation with NSAMPLES will make the timebase look wrong - so disable repair if pyfusion.NSAMPLES != 0 or self.repair == 0 or self.repair == -1: pass # leave as is # need at least this clipping for Langmuir probes in Op1.1 elif self.repair == 1: dim = np.clip(dim, 0, 1e99) elif self.repair == 2: dim, msg = regenerate_dim(dim) if msg is not None: print('shot {s}, {c}: {m}'.format(s=self.shot, c=self.config_name, m=msg)) else: raise ValueError( 'repair value of {r} not understood'.format(r=self.repair)) if pyfusion.VERBOSE > 2: print('repair', self.repair) #ch = Channel(self.config_name, Coords('dummy', (0,0,0))) # this probably should be in base.py coords = get_coords_for_channel(**self.__dict__) # used to be bare_chan? should we include - signs? ch = Channel(self.config_name, coords) scl = 1 / 3277.24 if dat['datatype'].lower() == 'short' else 1 if self.repair == -1: output_data = TimeseriesData(timebase=Timebase(dimraw), signal=scl * Signal(dat['values']), channels=ch) else: output_data = TimeseriesData(timebase=Timebase(1e-9 * dim), signal=scl * Signal(dat['values']), channels=ch) output_data.meta.update({'shot': self.shot}) output_data.utc = output_data_utc # this copy was saved earlier so we could delete the large array to save space output_data.units = dat['units'] if 'units' in dat else '' # this is a minor duplication - at least it gets saved via params params['data_utc'] = output_data.utc params['utc_0'] = utc_0 # hopefully t0 -- useful in npz files # Warning - this could slow things down! - but allows # corrupted time to be re-calculated as algorithms improve. # and storage as differences takes very little space. params['diff_dimraw'] = dimraw params['diff_dimraw'][1:] = np.diff(dimraw) if pyfusion.DEBUG < ONE: dimraw = None # NOTE!!! need float128 to process dimraw, and cumsum won't return ints # or automatically promote to 128bit (neither do simple ops like *, +) params['pyfusion_version'] = pyfusion.version.get_version() if pyfusion.VERBOSE > 0: print('shot {s}, config name {c}'.format(c=self.config_name, s=self.shot)) output_data.config_name = self.config_name debug_(pyfusion.DEBUG, 2, key='W7XDataFetcher') output_data.params = params ### the total shot utc. output_data.utc = [f_u, t_u] return output_data
def fetch_data_from_file(fetcher): prm_dict = read_prm_file(fetcher.basename + ".prm") bytes = int(prm_dict["DataLength(byte)"][0]) bits = int(prm_dict["Resolution(bit)"][0]) if not (prm_dict.has_key("ImageType")): # if so assume unsigned bytes_per_sample = 2 dat_arr = Array.array("H") offset = 2 ** (bits - 1) dtype = np.dtype("uint16") else: if prm_dict["ImageType"][0] == "INT16": bytes_per_sample = 2 if prm_dict["BinaryCoding"][0] == "offset_binary": dat_arr = Array.array("H") offset = 2 ** (bits - 1) dtype = np.dtype("uint16") elif prm_dict["BinaryCoding"][0] == "shifted_2's_complementary": dat_arr = Array.array("h") offset = 0 dtype = np.dtype("int16") else: raise NotImplementedError, " binary coding " + prm_dict["BinaryCoding"] fp = open(fetcher.basename + ".dat", "rb") dat_arr.fromfile(fp, bytes / bytes_per_sample) fp.close() # print(fetcher.config_name) clockHz = None if prm_dict.has_key("SamplingClock"): clockHz = double(prm_dict["SamplingClock"][0]) if prm_dict.has_key("SamplingInterval"): clockHz = clockHz / double(prm_dict["SamplingInterval"][0]) if prm_dict.has_key("ClockSpeed"): if clockHz != None: pyfusion.utils.warn("Apparent duplication of clock speed information") clockHz = double(prm_dict["ClockSpeed"][0]) clockHz = LHD_A14_clk(fetcher.shot) # see above if clockHz != None: timebase = arange(len(dat_arr)) / clockHz else: raise NotImplementedError, "timebase not recognised" ch = Channel("%s-%s" % (fetcher.diag_name, fetcher.channel_number), Coords("dummy", (0, 0, 0))) # if fetcher.gain != None: # this may have worked once...not now! # gain = fetcher.gain # else: # was - crude!! if channel == 20: arr = -arr # (MP5 and HMP13 need flipping) try: gain = float(fetcher.gain) except: gain = 1 # dodgy - should only apply to a diag in a list - don't want to define -MP5 separately - see other comment on "-" # if fetcher.diag_name[0]=='-': flip = -1 # else: flip = 1 # not sure if this needs a factor of two for RangePolarity,Bipolar (A14) scale_factor = flip * double(prm_dict["Range"][0]) / (2 ** bits) # not sure how this worked before I added array() - but has using # array slowed things? I clearly went to trouble using tailored ints above? output_data = TimeseriesData( timebase=Timebase(timebase), signal=Signal(scale_factor * gain * (array(dat_arr) - offset)), channels=ch ) output_data.meta.update({"shot": fetcher.shot}) output_data.config_name = fetcher.config_name return output_data
def read_data_from_file(fetcher): prm_dict = read_prm_file(fetcher.basename+".prm") bytes = int(prm_dict['DataLength(byte)'][0]) bits = int(prm_dict['Resolution(bit)'][0]) if 'ImageType' not in prm_dict: #if so assume unsigned bytes_per_sample = 2 dat_arr = Array.array('H') offset = 2**(bits-1) dtyp = np.dtype('uint16') else: if prm_dict['ImageType'][0] == 'INT16': bytes_per_sample = 2 if prm_dict['BinaryCoding'][0] == 'offset_binary': dat_arr = Array.array('H') offset = 2**(bits-1) dtyp = np.dtype('uint16') elif prm_dict['BinaryCoding'][0] == "shifted_2's_complementary": dat_arr = Array.array('h') offset = 0 dtyp = np.dtype('int16') # this was added for the VSL digitisers elif prm_dict['BinaryCoding'][0] == "2's_complementary": # not sure about this dat_arr = Array.array('h') offset = 0 dtyp = np.dtype('int16') else: raise NotImplementedError(' binary coding {pd}'.format(pd=prm_dict['BinaryCoding'])) """ fp = open(fetcher.basename + '.dat', 'rb') dat_arr.fromfile(fp, bytes/bytes_per_sample) fp.close() """ dat_arr = np.fromfile(fetcher.basename + '.dat',dtyp) #print(dat_arr[0:10]) #print(fetcher.config_name) if fetcher.timeOK: # we have retrieve_t data! # check for ArrayDataType: float is float32 # skip is 0 as there is no initial digitiser type token tprm_dict = read_prm_file(fetcher.basename+".tprm",skip=0) if pyfusion.VERBOSE>1: print(tprm_dict) ftype = tprm_dict['ArrayDataType'][0] floats = dict(float = 'float32', double='float64') timebase = np.fromfile(fetcher.basename + '.time', np.dtype(floats[ftype])) else: # use the info from the .prm file clockHz = None if 'SamplingClock' in prm_dict: clockHz = double(prm_dict['SamplingClock'][0]) if 'SamplingInterval' in prm_dict: clockHz = clockHz/double(prm_dict['SamplingInterval'][0]) if 'ClockInterval(uSec)' in prm_dict: # VSL dig clockHz = 1e6/double(prm_dict['ClockInterval(uSec)'][0]) if 'ClockSpeed' in prm_dict: if clockHz != None: pyfusion.utils.warn('Apparent duplication of clock speed information') clockHz = double(prm_dict['ClockSpeed'][0]) clockHz = LHD_A14_clk(fetcher.shot) # see above if clockHz != None: if 'PreSamples/Ch' in prm_dict: # needed for "WE" e.g. VSL pretrig = float(prm_dict['PreSamples/Ch'][0])/clockHz else: pretrig = 0. timebase = arange(len(dat_arr))/clockHz - pretrig else: debug_(pyfusion.DEBUG, level=4, key='LHD read debug') raise NotImplementedError("timebase not recognised") debug_(pyfusion.DEBUG, level=4, key='LHD read debug') ch = Channel("{dn}-{dc}" .format(dn=fetcher.diag_name, dc=fetcher.channel_number), Coords('dummy', (0,0,0))) # if fetcher.gain != None: # this may have worked once...not now! # gain = fetcher.gain # else: # was - crude!! if channel == 20: arr = -arr # (MP5 and HMP13 need flipping) try: gain = float(fetcher.gain) except: gain = 1 # dodgy - should only apply to a diag in a list - don't want to define -MP5 separately - see other comment on "-" #if fetcher.diag_name[0]=='-': flip = -1 #else: flip = 1 # not sure if this needs a factor of two for RangePolarity,Bipolar (A14) rng=None for key in 'Range,Range(V)'.split(','): # equivalent alteratives rng=prm_dict.get(key) if rng is not None: break scale_factor = flip*double(rng[0])/(2**bits) # not sure how this worked before I added array() - but has using # array slowed things? I clearly went to trouble using tailored ints above? # - yes array(dat_arr) takes 1.5 sec for 4MS!! # Answer - using numpy fromfile('file',dtype=numpy.int16) - 16ms instead! # NOTE! ctype=int32 required if the array is already an np array - can be fixed once Array code is removed (next version) output_data = TimeseriesData(timebase=Timebase(timebase), signal=Signal(scale_factor*gain*(array(dat_arr,dtype=np.int32)-offset)), channels=ch) #print(output_data.signal[0:5],offset,(array(dat_arr)-offset)[0:5]) output_data.meta.update({'shot':fetcher.shot}) output_data.config_name = fetcher.config_name output_data.params = prm_dict return output_data
def fetch_data_from_file(fetcher): prm_dict = read_prm_file(fetcher.basename + ".prm") bytes = int(prm_dict['DataLength(byte)'][0]) bits = int(prm_dict['Resolution(bit)'][0]) if not (prm_dict.has_key('ImageType')): #if so assume unsigned bytes_per_sample = 2 dat_arr = Array.array('H') offset = 2**(bits - 1) dtype = np.dtype('uint16') else: if prm_dict['ImageType'][0] == 'INT16': bytes_per_sample = 2 if prm_dict['BinaryCoding'][0] == 'offset_binary': dat_arr = Array.array('H') offset = 2**(bits - 1) dtype = np.dtype('uint16') elif prm_dict['BinaryCoding'][0] == "shifted_2's_complementary": dat_arr = Array.array('h') offset = 0 dtype = np.dtype('int16') else: raise NotImplementedError, ' binary coding ' + prm_dict[ 'BinaryCoding'] fp = open(fetcher.basename + '.dat', 'rb') dat_arr.fromfile(fp, bytes / bytes_per_sample) fp.close() #print(fetcher.config_name) clockHz = None if prm_dict.has_key('SamplingClock'): clockHz = double(prm_dict['SamplingClock'][0]) if prm_dict.has_key('SamplingInterval'): clockHz = clockHz / double(prm_dict['SamplingInterval'][0]) if prm_dict.has_key('ClockSpeed'): if clockHz != None: pyfusion.utils.warn( 'Apparent duplication of clock speed information') clockHz = double(prm_dict['ClockSpeed'][0]) clockHz = LHD_A14_clk(fetcher.shot) # see above if clockHz != None: timebase = arange(len(dat_arr)) / clockHz else: raise NotImplementedError, "timebase not recognised" ch = Channel("%s-%s" % (fetcher.diag_name, fetcher.channel_number), Coords('dummy', (0, 0, 0))) # if fetcher.gain != None: # this may have worked once...not now! # gain = fetcher.gain # else: # was - crude!! if channel == 20: arr = -arr # (MP5 and HMP13 need flipping) try: gain = float(fetcher.gain) except: gain = 1 # dodgy - should only apply to a diag in a list - don't want to define -MP5 separately - see other comment on "-" #if fetcher.diag_name[0]=='-': flip = -1 #else: flip = 1 # not sure if this needs a factor of two for RangePolarity,Bipolar (A14) scale_factor = flip * double(prm_dict['Range'][0]) / (2**bits) # not sure how this worked before I added array() - but has using # array slowed things? I clearly went to trouble using tailored ints above? output_data = TimeseriesData(timebase=Timebase(timebase), signal=Signal(scale_factor * gain * (array(dat_arr) - offset)), channels=ch) output_data.meta.update({'shot': fetcher.shot}) output_data.config_name = fetcher.config_name return output_data
def do_fetch(self): # my W7X shots are of the form from_utc, to_utc # or date (8dig) and shot (progId) # the format is in the acquisition properties, to avoid # repetition in each individual diagnostic if self.shot[1]>1e9: # we have start and end in UTC f,t = self.shot else: f,t = get_shot_utc(*self.shot) # A URL STYLE diagnostic - used for a one-off # this could be moved to setup so that the error info is more complete if hasattr(self,'url'): fmt = self.url+'_signal.json?from={shot_f}&upto={shot_t}' fmt = self.url+'_signal.json?from={shot_f}&upto={shot_t}&nSamples=200000' params = {} else: # a pattern-based one - used for arrays of probes if hasattr(self,'fmt'): # does the diagnostic have one? fmt = self.fmt elif hasattr(self.acq,'fmt'): # else use the acq.fmt fmt = self.acq.fmt else: # so far we have no quick way to check the server is online raise LookupError('no fmt - perhaps pyfusion.cfg has been ' 'edited because the url is not available') params = eval('dict('+self.params+')') if 'upto' not in fmt: fmt += '_signal.json?from={shot_f}&upto={shot_t}' if ('nSamples' not in fmt) and (pyfusion.NSAMPLES != 0): fmt += '&nSamples={ns}'.format(ns=pyfusion.NSAMPLES) params.update(shot_f=f, shot_t=t) url = fmt.format(**params) if pyfusion.CACHE: print('using wget on {url}'.format(url=url)) os.system('wget -x "{url}"'.format(url=url)) # now read from the local copy - it is in the wd, so only // # but it seems we need the full path for now url = url.replace('http://','file:///home/bdb112/pyfusion/working/pyfusion/') print('now trying the cached copy we just grabbed: {url}'.format(url=url)) if pyfusion.VERBOSE > 0: print('===> fetching url {u}'.format(u=url)) # seems to take twice as long as timeout requested. # haven't thought about python3 for the json stuff yet try: # dat = json.load(urlopen(url,timeout=pyfusion.TIMEOUT)) works # but follow example in # http://webservices.ipp-hgw.mpg.de/docs/howtoREST.html#python, dat = json.loads(urlopen(url,timeout=pyfusion.TIMEOUT).read().decode()) except socket.timeout: # should check if this is better tested by the URL module print('****** first timeout error *****') dat = json.load(urlopen(url,timeout=3*pyfusion.TIMEOUT)) except Exception as reason: if pyfusion.VERBOSE: print('********Exception***** on {c}: {u} \n{r}' .format(c=self.config_name, u=url, r=reason)) raise # this form will default to repair = 2 for all LP probes. default_repair = 2 if 'Desc.82/' in url else 0 # this form follows the config file settings self.repair = int(self.repair) if hasattr(self, 'repair') else default_repair dimraw = np.array(dat['dimensions']) dim = dimraw - dimraw[0] if self.repair == 0: pass # leave as is # need at least this clipping for Langmuir probes in Op1.1 elif self.repair == 1: dim = np.clip(dim, 0, 1e99) elif self.repair == 2: dim, msg = regenerate_dim(dim) if msg is not None: print('shot {s}, {c}: {m}' .format(s=self.shot, c=self.config_name, m=msg)) else: raise ValueError('repair value of {r} not understood'.format(r=self.repair)) if pyfusion.VERBOSE>2: print('repair',self.repair) #ch = Channel(self.config_name, Coords('dummy', (0,0,0))) # this probably should be in base.py coords = get_coords_for_channel(**self.__dict__) # used to be bare_chan? should we include - signs? ch = Channel(self.config_name, coords) output_data = TimeseriesData(timebase=Timebase(1e-9*dim), signal=Signal(dat['values']), channels=ch) output_data.meta.update({'shot': self.shot}) output_data.utc = [dat['dimensions'][0], dat['dimensions'][-1]] output_data.units = dat['units'] if 'units' in dat else '' # this is a minor duplication - at least it gets saved via params params['data_utc'] = output_data.utc # Warning - this could slow things down! - but allows corrupted time to be re-calculated as algorithms improve. params['diff_dimraw'] = dimraw params['diff_dimraw'][1:] = np.diff(dimraw) params['pyfusion_version'] = pyfusion.version.get_version() if pyfusion.VERBOSE > 0: print('shot {s}, config name {c}' .format(c=self.config_name, s=self.shot)) output_data.config_name = self.config_name debug_(pyfusion.DEBUG, 2, key='W7XDataFetcher') output_data.params = params ### the total shot utc. output_data.utc = [f, t] return output_data
def try_fetch_local(input_data, bare_chan): """ return data if in the local cache, otherwise None doesn't work for single channel HJ data. sgn (not gain) be only be used at the single channel base/fetch level """ for each_path in pyfusion.config.get('global', 'localdatapath').split('+'): # check for multi value shot number, e.g. utc bounds for W7-X data shot = input_data.shot # MDSplus style path to access sorted files into folders by shot path, patt = os.path.split(each_path) # print(patt) if len(patt) == 2*len(patt.replace('~','')): # a subdir code based on date subdir = '' # reverse the order of both the pattern and the shot so a posn is 0th char strshot = str(shot[0]) if len(np.shape(shot))>0 else str(shot) revshot = strshot[::-1] for i,ch in enumerate(patt): if (i%2) == 0: if ch != '~': raise LookupError("Can't parse {d} as a MDS style subdir" .format(d=patt)) continue subdir += revshot[ord(ch) - ord('a')] else: subdir = patt debug_(pyfusion.DEBUG, 3, key='MDS style subdir', msg=each_path) each_path = os.path.join(path, subdir) if isinstance(shot, (tuple, list, ndarray)): shot_str = '{s0}_{s1}'.format(s0=shot[0], s1=shot[1]) else: shot_str = str(shot) input_data.localname = os.path.join(each_path, '{shot}_{bc}.npz' .format(shot=shot_str, bc=bare_chan)) # original - data_filename %filename_dict) if pyfusion.VERBOSE>2: print(each_path,input_data.localname) files_exist = os.path.exists(input_data.localname) debug_(pyfusion.DEBUG, 3, key='try_local_fetch') if files_exist: intmp = np.any([st in input_data.localname.lower() for st in ['tmp', 'temp']]) # add anything you wish to warn about if pyfusion.VERBOSE>0 or intmp: if intmp: pyfusion.logging.warning('Using {f} in temporary directory!' .format(f=input_data.localname)) print('found local data in {f}'. format(f=input_data.localname)) break if not files_exist: return None signal_dict = newload(input_data.localname) if 'params' in signal_dict and 'name' in signal_dict['params'] and 'W7X_L5' in signal_dict['params']['name']: if signal_dict['params']['pyfusion_version'] < '0.6.8b': raise ValueError('probe assignments in error LP11-22 in {fn}' .format(fn=input_data.localname)) if np.nanmax(signal_dict['timebase']) == 0: pyfusion.logging.warning('making a fake timebase for {fn}' .format(fn=input_data.localname)) signal_dict['timebase'] = 2e-6*np.cumsum(1.0 + 0*signal_dict['signal']) coords = get_coords_for_channel(**input_data.__dict__) #ch = Channel(bare_chan, Coords('dummy', (0,0,0))) ch = Channel(bare_chan, coords) output_data = TimeseriesData(timebase=Timebase(signal_dict['timebase']), signal=Signal(signal_dict['signal']), channels=ch) # bdb - used "fetcher" instead of "self" in the "direct from LHD data" version # when using saved files, should use the name - not input_data.config_name # it WAS the config_name coming from the raw format. output_data.config_name = bare_chan # would be nice to get to the gain here - but how - maybe setup will get it output_data.meta.update({'shot':input_data.shot}) if 'params' in signal_dict: output_data.params = signal_dict['params'] if 'utc' in signal_dict['params']: output_data.utc = signal_dict['params'].get('utc',None) else: # yes, it seems like duplication, but no output_data.utc = None output_data.params = dict(comment = 'old npz file has no params') oldsrc = ', originally from ' + output_data.params['source'] if 'source' in output_data.params else '' output_data.params.update(dict(source='from npz cache' + oldsrc)) return(output_data)