def make_title(formatstr, input_data, channum=None, at_dict = {}, min_length=3, raw_names=False): """ Return a string describing the shot number, channel name etc using a formatstr which refers to items in a dictionary (at_dict), assembled in this routine, based on input_data and an optional dictionary which contains anything not otherwise available in input_data """ ## at_dict.update({'shot': input_data.meta['shot']}) exception = () if pyfusion.DBG() > 3 else Exception try: at_dict.update(input_data.meta) # this gets all of it! if channum is None: name = '' else: if isinstance(input_data.channels, list): chan = input_data.channels[channum] else: chan = input_data.channels if raw_names: name = chan.name else: name = chan.config_name at_dict.update({'units':chan.units}) at_dict.update({'name': name}) # replace internal strings of non-numbers with a single . a14_input03 -> 14.03 short_name='' last_was_number = False discarded='' for c in name: # start from the first char if c>='0' and c<='9': short_name += c last_was_number=True else: if last_was_number: short_name += '.' else: discarded += c last_was_number=False if len(short_name) <= min_length: # if it fits, have the lot if len(name)<8: short_name=name # else allow 4 more chars - makes about 6-8 chars else: short_name = discarded[-4:] + short_name at_dict.update({'short_name': short_name}) return(formatstr.format(**at_dict)) except exception as ex: warn('in make_title for format="%s", at_dict=%s' % (formatstr, at_dict), exception=ex) return('')
def set_axis_if_OK(ax, xlims, ylims): """ set axes if they containg at least a bit of the plot originally meant to be used when forcing specific axes, to allow for attempt to force axes suitable for LHD on HJ data for example """ from pyfusion.utils.utils import warn if overlap(ax.get_xlim(), xlims) and overlap(ax.get_ylim(), ylims): ax.axis(xlims[0], xlims[1], ylims[0], ylims[1]) else: warn('suppressed attempt to set axis outside ranges\n{att:s} cf {act:s}' .format(att=(xlims,ylims), act=(ax.get_xlim(),ax.get_ylim())))
def set_axis_if_OK(ax, xlims, ylims): """ set axes if they containg at least a bit of the plot originally meant to be used when forcing specific axes, to allow for attempt to force axes suitable for LHD on HJ data for example """ from pyfusion.utils.utils import warn if overlap(ax.get_xlim(), xlims) and overlap(ax.get_ylim(), ylims): ax.axis(xlims[0], xlims[1], ylims[0], ylims[1]) else: warn( 'suppressed attempt to set axis outside ranges\n{att:s} cf {act:s}' .format(att=(xlims, ylims), act=(ax.get_xlim(), ax.get_ylim())))
def subdivide_interval(pts, overlap=None, debug=0): """ return several intervals which straddle pts with overlap of ov The lowest x value is special - the point of division is much closer to that x then zero. overlap is a tuple (minoverlap, max), and describes the total overlap """ if overlap == None: overlap = [np.max(pts) / 50, np.max(pts) / 20] if len(overlap) == 1: warn("overlap should have a min and a max") overlap = [overlap / 3.0, overlap] if (np.diff(pts) < 0).any(): warn("points out of order - reordering") pts = np.sort(pts) begins = [] ends = [] for (i, x) in enumerate(pts): if i == 0: divider = x * 0.8 - overlap[1] / 2.0 else: divider = (x + pts[i - 1]) / 2.0 if i == 0: begins.append(0) ends.append(divider + overlap[1] / 2.0) else: this_overlap = min(max((divider - last_divider) / 20, overlap[0]), overlap[1]) / 2.0 begins.append(last_divider - this_overlap) ends.append(divider + this_overlap) last_divider = divider if debug > 1: print(begins, pts, ends) if debug > 2: import pylab as pl pl.figure() for i in range(len(pts)): pl.plot([begins[i], ends[i]], [i, i]) if i > 0: pl.plot([pts[i - 1], pts[i - 1]], [i, i], "o") pl.ylim(-1, 20) pl.show() return (begins, ends)
def subdivide_interval(pts, overlap=None, debug=0): """ return several intervals which straddle pts with overlap of ov The lowest x value is special - the point of division is much closer to that x then zero. overlap is a tuple (minoverlap, max), and describes the total overlap """ if overlap is None: overlap = [np.max(pts) / 50, np.max(pts) / 20] if len(overlap) == 1: warn('overlap should have a min and a max') overlap = [overlap / 3.0, overlap] if (np.diff(pts) < 0).any(): warn('points out of order - reordering') pts = np.sort(pts) begins = [] ends = [] for (i, x) in enumerate(pts): if i == 0: divider = x * 0.8 - overlap[1] / 2. else: divider = (x + pts[i - 1]) / 2. if i == 0: begins.append(0) ends.append(divider + overlap[1] / 2.) else: this_overlap = min(max( (divider - last_divider) / 20, overlap[0]), overlap[1]) / 2. begins.append(last_divider - this_overlap) ends.append(divider + this_overlap) last_divider = divider if debug > 1: print(begins, pts, ends) if debug > 2: import pylab as pl pl.figure() for i in range(len(pts)): pl.plot([begins[i], ends[i]], [i, i]) if i > 0: pl.plot([pts[i - 1], pts[i - 1]], [i, i], 'o') pl.ylim(-1, 20) pl.show() return (begins, ends)
def make_title(formatstr, input_data, channum=None, dict={}, min_length=3): """ return a string describing the shot number, channel name etc using a formatstr which referes to items in a dictionary, assembled in this routine, based on input_data and an optional dictionary which contains anything not otherwise available in input_data """ ## dict.update({'shot': input_data.meta['shot']}) try: dict.update(input_data.meta) # this gets all of it! if channum == None: name = "" else: try: name = input_data.channels[channum].name except: name = input_data.channels.name dict.update({"name": name}) # replace internal strings of non-numbers with a single . a14_input03 -> 14.03 short_name = "" last_was_number = False for c in name: if c >= "0" and c <= "9": short_name += c last_was_number = True else: if last_was_number: short_name += "." last_was_number = False if len(short_name) <= min_length: short_name = name dict.update({"short_name": short_name}) return formatstr % dict except Exception as ex: warn('in make_title for format="%s", dict=%s' % (formatstr, dict), exception=ex) return ""
def make_title(formatstr, input_data, channum=None, dict={}, min_length=3): """ return a string describing the shot number, channel name etc using a formatstr which referes to items in a dictionary, assembled in this routine, based on input_data and an optional dictionary which contains anything not otherwise available in input_data """ ## dict.update({'shot': input_data.meta['shot']}) try: dict.update(input_data.meta) # this gets all of it! if channum == None: name = '' else: try: name = input_data.channels[channum].name except: name = input_data.channels.name dict.update({'name': name}) # replace internal strings of non-numbers with a single . a14_input03 -> 14.03 short_name = '' last_was_number = False for c in name: if c >= '0' and c <= '9': short_name += c last_was_number = True else: if last_was_number: short_name += '.' last_was_number = False if len(short_name) <= min_length: short_name = name dict.update({'short_name': short_name}) return (formatstr % dict) except Exception as ex: warn('in make_title for format="%s", dict=%s' % (formatstr, dict), exception=ex) return ('')
def get_basic_diagnostics(diags=None, shot=54196, times=None, delay=None, exception=False, debug=0): """ return a list of np.arrays of normally numeric values for the times given, for the given shot. Will access server if env('IGETFILE') points to an exe, else accesses cache """ global HJ_summary # if no exception given and we are not debugging # note - exception=None is a valid entry, meaning tolerate no exceptions # so the "default" we use is False if exception == False and debug == 0: exception = Exception if diags is None: diags = "<n_e19>,b_0,i_p,w_p,dw_pdt,dw_pdt2".split(',') if len(np.shape(diags)) == 0: diags = [diags] # LHD only if delay is None: delay = get_delay(shot) if times is None: times = np.linspace(0, 4, 4000) times = np.array(times) vals = {} # create an extra time array to allow a cross check vals.update({'check_tm': times}) vals.update({'check_shot': np.zeros(len(times), dtype=np.int) + shot}) debug_(pyfusion.DEBUG, 2, key='get_basic') for diag in diags: if not (diag in file_info): warn('diagnostic {0} not found in shot {1}'.format(diag, shot), stacklevel=2) vals.update({diag: np.nan + times}) debug_(pyfusion.DEBUG, 2, key='get_basic') else: info = file_info[diag] varname = info['name'] infofmt = info['format'] subfolder = infofmt.split('@')[0] filepath = os.path.sep.join( [localigetfilepath, subfolder, infofmt]) if ':' in varname: (oper, varname) = varname.split(':') else: oper = None if '(' in varname: try: left, right = varname.split('(') varname, rest = right.split(')') except: raise ValueError( 'in expression {v} - parens?'.format(varname)) if infofmt.find('.npz') > 0: try: test = HJ_summary.keys() except: csvfilename = acq_HJ + '/' + infofmt if pyfusion.DBG() > 1: print('looking for HeliotronJ summary in' + csvfilename) print('reloading {0}'.format(csvfilename)) HJ_summary = np.load(csvfilename) val = HJ_summary[varname][shot] valarr = np.double(val) + (times * 0) elif 'get_static_params' in infofmt: pdicts = eval(infofmt.format(shot=shot)) if len(pdicts) == 0: print('empty dictionary returned') val = pdicts[varname] valarr = np.double(val) + (times * 0) else: # read signal from data system debug_(max(pyfusion.DEBUG, debug), level=4, key='find_data') try: #get HJparams channel = info['name'] outdata = np.zeros(1024 * 2 * 256 + 1) channel_length = (len(outdata) - 1) / 2 # outdfile only needed for opt=1 (get data via temp file) # with tempfile.NamedTemporaryFile(prefix="pyfusion_") as outdfile: ierror, getrets = gethjdata.gethjdata(shot, channel_length, info['name'], verbose=VERBOSE, opt=1, ierror=2, outdata=outdata, outname='') if ierror != 0: raise LookupError('data not found for {s}:{c}'.format( s=shot, c=channel)) ch = Channel(info['name'], Coords('dummy', (0, 0, 0))) # timebase in secs (was ms in raw data) dg = TimeseriesData(timebase=Timebase(1e-3 * getrets[1::2]), signal=Signal(getrets[2::2]), channels=ch) except exception as reason: if debug > 0: print('exception running gethjdata {r} {a}', format(r=reason, a=reason.args)) dg = None #break # give up and try next diagnostic if dg is None: # messy - break doesn't do what I want? valarr = None else: nd = 1 # initially only deal with single channels (HJ) # get the column(s) of the array corresponding to the name w = [0] if (oper in 'sum,average,rms,max,min'.split(',')): if oper == 'sum': op = np.sum elif oper == 'average': op = np.average elif oper == 'min': op = np.min elif oper == 'std': op = np.std else: raise ValueError( 'operator {o} in {n} not known to get_basic_diagnostics' .format(o=oper, n=info['name'])) # valarr = op(dg.data[:,nd+w],1) valarr = op(dg.data[:, nd + w], 1) else: if len(w) != 1: raise LookupError( 'Need just one instance of variable {0} in {1}' .format(varname, dg.filename)) dg.data = dg.signal # fudge compatibility if len(np.shape(dg.data)) != 1: # 2 for igetfile raise LookupError( 'insufficient data for {0} in {1}'.format( varname, dg.filename)) #valarr = dg.data[:,nd+w[0]] #tim = dg.data[:,0] - delay valarr = dg.signal tim = dg.timebase # fudge until we can gete the number of points valarr = valarr[:np.argmax(tim)] tim = tim[:np.argmax(tim)] if oper == 'ddt': # derivative operator valarr = np.diff(valarr) / (np.average(np.diff(tim))) tim = (tim[0:-1] + tim[1:]) / 2.0 if oper == 'ddt2': # abd(ddw)*derivative operator dw = np.diff(valarr) / (np.average(np.diff(tim))) ddw = np.diff(dw) / (np.average(np.diff(tim))) tim = tim[2:] valarr = 4e-6 * dw[1:] * np.abs(ddw) if (len(tim) < 10) or (np.std(tim) < 0.1): raise ValueError('Insufficient points or degenerate' 'timebase data in {0}, {1}'.format( varname, dg.filename)) valarr = (stineman_interp(times, tim, valarr)) w = np.where(times > max(tim)) valarr[w] = np.nan if valarr is not None: vals.update({diag: valarr}) debug_(max(pyfusion.DEBUG, debug), level=5, key='interp') return (vals)
def get_basic_diagnostics(diags=None, shot=54196, times=None, delay=None, exception=False, debug=0): """ return a list of np.arrays of normally numeric values for the times given, for the given shot. Will access server if env('IGETFILE') points to an exe, else accesses cache """ global lhd_summary # if no exception given and we are not debugging # note - exception=None is a valid entry, meaning tolerate no exceptions # so the "default" we use is False if exception == False and debug == 0: exception = Exception if diags is None: diags = "<n_e19>,b_0,i_p,w_p,dw_pdt,dw_pdt2".split(',') if len(np.shape(diags)) == 0: diags = [diags] if delay is None: delay = get_delay(shot) if times is None: times = np.linspace(0, 4, 4000) times = np.array(times) vals = {} # create an extra time array to allow a cross check vals.update({'check_tm': times}) vals.update({'check_shot': np.zeros(len(times), dtype=np.int) + shot}) for diag in diags: if diag not in file_info: warn('diagnostic {0} not found in shot {1}'.format(diag, shot), stacklevel=2) vals.update({diag: np.nan + times}) else: info = file_info[diag] varname = info['name'] subfolder = info['format'].split('@')[0] filepath = os.path.sep.join( [localigetfilepath, subfolder, info['format']]) if ':' in varname: (oper, varname) = varname.split(':') else: oper = None if info['format'].find('.csv') > 0: try: test = lhd_summary.keys() except: csvfilename = acq_LHD + '/' + info['format'] if pyfusion.DBG() > 1: print('looking for lhd summary in' + csvfilename) if not os.path.exists(csvfilename): csvfilename += ".bz2" print('reloading {0}'.format(csvfilename)) lhd_summary = read_csv_data(csvfilename, header=3) # should make this more formal - last shots # from an 'extra' file, and finally, from shot info if shot > 117000: # fudge to get latest data lhd_summary = np.load(acq_LHD + '/LHD_summary.npz')['LHD'].tolist() print('loading newer shots from a separate file - fix-me') # val = lhd_summary[varname][shot-70000] # not needed # else: val = lhd_summary[varname][shot] valarr = np.double(val) + (times * 0) else: debug_(max(pyfusion.DBG(), debug), level=4, key='find_data') try: dg = igetfile(filepath, shot=shot, debug=debug - 1) except IOError: try: dg = igetfile(filepath + '.bz2', shot=shot, debug=debug - 1) except IOError: try: dg = igetfile(filepath + '.gz', shot=shot, debug=debug - 1) except exception as details: if debug > 0: print('diag at {fp} not found'.format( fp=filepath)) print(details, details.args) dg = None #break # give up and try next diagnostic if dg is None: # messy - break doesn't do what I want? valarr = None else: nd = dg.vardict['DimNo'] if nd != 1: raise ValueError( 'Expecting a 1 D array in {0}, got {1}!'.format( dg.filename, nd)) # pre re. w = np.where(np.array(dg.vardict['ValName'])==varname)[0] matches = [ re.match(varname, nam) != None for nam in dg.vardict['ValName'] ] w = np.where(np.array(matches) != False)[0] # get the column(s) of the array corresponding to the name if (oper in 'sum,average,rms,max,min'.split(',')): if oper == 'sum': op = np.sum elif oper == 'average': op = np.average elif oper == 'min': op = np.min elif oper == 'std': op = np.std else: raise ValueError( 'operator {o} in {n} not known to get_basic_diagnostics' .format(o=oper, n=info['name'])) valarr = op(dg.data[:, nd + w], 1) else: if len(w) != 1: raise LookupError( 'Need just one instance of variable {0} in {1}' .format(varname, dg.filename)) if len(np.shape(dg.data)) != 2: raise LookupError( 'insufficient data for {0} in {1}'.format( varname, dg.filename)) valarr = dg.data[:, nd + w[0]] tim = dg.data[:, 0] - delay if oper == 'ddt': # derivative operator valarr = np.diff(valarr) / (np.average(np.diff(tim))) tim = (tim[0:-1] + tim[1:]) / 2.0 if oper == 'ddt2': # abd(ddw)*derivative operator dw = np.diff(valarr) / (np.average(np.diff(tim))) ddw = np.diff(dw) / (np.average(np.diff(tim))) tim = tim[2:] valarr = 4e-6 * dw[1:] * np.abs(ddw) if (len(tim) < 10) or (np.std(tim) < 0.1): raise ValueError('Insufficient points or degenerate' 'timebase data in {0}, {1}'.format( varname, dg.filename)) valarr = (stineman_interp(times, tim, valarr)) w = np.where(times > max(tim)) valarr[w] = np.nan if valarr != None: vals.update({diag: valarr}) debug_(max(pyfusion.DBG(), debug), level=5, key='interp') return (vals)
def get_basic_diagnostics(diags=None, shot=54196, times=None, delay=None, exception=False, debug=0): """ return a list of np.arrays of normally numeric values for the times given, for the given shot. """ global lhd_summary # if no exception given and we are not debugging # note - exception=None is a valid entry, meaning tolerate no exceptions # so the "default" we use is False if exception==False and debug==0: exception=Exception if diags == None: diags = "<n_e19>,b_0,i_p,w_p,dw_pdt,dw_pdt2".split(',') if len(np.shape(diags)) == 0: diags = [diags] if delay == None: delay = get_delay(shot) if times == None: times = np.linspace(0,4,4000) times = np.array(times) vals = {} # create an extra time array to allow a cross check vals.update({'check_tm':times}) vals.update({'check_shot':np.zeros(len(times),dtype=np.int)+shot}) for diag in diags: if not(file_info.has_key(diag)): warn('diagnostic {0} not found in shot {1}'.format(diag, shot),stacklevel=2) vals.update({diag: np.nan + times}) else: info = file_info[diag] varname = info['name'] subfolder = info['format'].split('@')[0] filepath = os.path.sep.join([localigetfilepath,subfolder,info['format']]) if ':' in varname: (oper,varname) = varname.split(':') else: oper = None if info['format'].find('.csv') > 0: try: test=lhd_summary.keys() except: print('reloading {0}'.format(info['format'])) lhd_summary = read_csv_data(acq_LHD+'/'+info['format'], header=3) val = lhd_summary[varname][shot] valarr = np.double(val)+(times*0) else: debug_(max(pyfusion.DEBUG, debug), level=4, key='find_data') try: dg = igetfile(filepath, shot=shot, debug=debug-1) except IOError: try: dg = igetfile(filepath+'.bz2', shot=shot, debug=debug-1) except IOError: try: dg = igetfile(filepath + '.gz', shot=shot, debug=debug-1) except exception: if debug>0: print('diag at {fp} not found' .format(fp=filepath)) dg=None #break # give up and try next diagnostic if dg==None: # messy - break doesn't do what I want? valarr=None else: nd=dg.vardict['DimNo'] if nd != 1: raise ValueError( 'Expecting a 1 D array in {0}, got {1}!' .format(dg.filename, nd)) # pre re. w = np.where(np.array(dg.vardict['ValName'])==varname)[0] matches = [re.match(varname,nam) != None for nam in dg.vardict['ValName']] w = np.where(np.array(matches) != False)[0] # get the column(s) of the array corresponding to the name if (oper in 'sum,average,rms,max,min'.split(',')): if oper=='sum': op = np.sum elif oper=='average': op = np.average elif oper=='min': op = np.min elif oper=='std': op = np.std else: raise ValueError('operator {o} in {n} not known to get_basic_diagnostics' .format(o=oper, n=info['name'])) valarr = op(dg.data[:,nd+w],1) else: if len(w) != 1: raise LookupError( 'Need just one instance of variable {0} in {1}' .format(varname, dg.filename)) if len(np.shape(dg.data))!=2: raise LookupError( 'insufficient data for {0} in {1}' .format(varname, dg.filename)) valarr = dg.data[:,nd+w[0]] tim = dg.data[:,0] - delay if oper == 'ddt': # derivative operator valarr = np.diff(valarr)/(np.average(np.diff(tim))) tim = (tim[0:-1] + tim[1:])/2.0 if oper == 'ddt2': # abd(ddw)*derivative operator dw = np.diff(valarr)/(np.average(np.diff(tim))) ddw = np.diff(dw)/(np.average(np.diff(tim))) tim = tim[2:] valarr = 4e-6 * dw[1:] * np.abs(ddw) if (len(tim) < 10) or (np.std(tim)<0.1): raise ValueError('Insufficient points or degenerate' 'timebase data in {0}, {1}' .format(varname, dg.filename)) valarr = (stineman_interp(times, tim, valarr)) w = np.where(times > max(tim)) valarr[w] = np.nan if valarr != None: vals.update({diag: valarr}) debug_(max(pyfusion.DEBUG, debug), level=5, key='interp') return(vals)
def get_basic_diagnostics(diags=None, file_info=file_info, shot=54196, times=None, delay=None, exception=False, debug=0): """ return a list of np.arrays of normally numeric values for the times given, for the given shot. Will access server if env('IGETFILE') points to an exe, else accesses cache This is the first version to specifically allow for access through pyfusion.cfg There are two types of access: I/ single diag on its own timebase II/ the original multi diag on a given timebase (i.e. that from flucstrcs) Stage 1 puts the file_info into .cfg file just for I/ single diag access. Ideally the file_info for II/ sho;d be in .cfg also. For stage I/, we call it with a file_info dict constructed on the spot as a dictionary with one just entry (for diags[0]). """ global lhd_summary # if no exception given and we are not debugging # note - exception=None is a valid entry, meaning tolerate no exceptions # so the "default" we use is False if exception==False and debug==0: exception=Exception if diags is None: diags = "<n_e19>,b_0,i_p,w_p,dw_pdt,dw_pdt2".split(',') if len(np.shape(diags)) == 0: diags = [diags] if delay is None: delay = get_delay(shot) if times is None: if len(diags)>1: times = np.linspace(0,4,4000) # this is a crude guess. # else leave it None else: # make sure it is an array times = np.array(times) vals = {} for diag in diags: if not(diag in file_info): warn('diagnostic {0} not found in shot {1}'.format(diag, shot),stacklevel=2) vals.update({diag: np.nan + times}) else: info = file_info[diag] varname = info['name'] # refers to name for igetfile - can contain ':' subfolder = info['format'].split('@')[0] filepath = os.path.sep.join([localigetfilepath,subfolder,info['format']]) if ':' in varname: (oper,varname) = varname.split(':') else: oper = None if info['format'].find('.csv') > 0: try: test=list(lhd_summary.keys()) except: csvfilename = acq_LHD+'/'+info['format'] if pyfusion.DBG() > 1: print('looking for lhd summary in' + csvfilename) if not os.path.exists(csvfilename): csvfilename += ".bz2" print('reloading {0}'.format(csvfilename)) lhd_summary = read_csv_data(csvfilename, header=3) # should make this more formal - last shots # from an 'extra' file, and finally, from shot info if shot>117000: # fudge to get latest data lhd_summary = np.load(acq_LHD+'/LHD_summary.npz')['LHD'].tolist() print('loading newer shots from a separate file - fix-me') # val = lhd_summary[varname][shot-70000] # not needed # else: val = lhd_summary[varname][shot] valarr = np.double(val)+(times*0) else: try: # now igetfile checks for .gz etc dg = igetfile(filepath, shot=shot, debug=debug-1) except exception as details: if debug>0: print('diag at {fp} not found' .format(fp=filepath)) print(details,details.args) dg=None #break # give up and try next diagnostic if dg is None: # messy - break doesn't do what I want? valarr=None else: nd=dg.vardict['DimNo'] if nd != 1: raise ValueError( 'Expecting a 1 D array in {0}, got {1}!' .format(dg.filename, nd)) # pre re. w = np.where(np.array(dg.vardict['ValName'])==varname)[0] matches = [re.match(varname,nam) != None for nam in dg.vardict['ValName']] w = np.where(np.array(matches) != False)[0] # get the column(s) of the array corresponding to the name if (oper in 'sum,average,rms,max,min'.split(',')): if oper=='sum': op = np.sum elif oper=='average': op = np.average elif oper=='min': op = np.min elif oper=='std': op = np.std else: raise ValueError('operator {o} in {n} not known to get_basic_diagnostics' .format(o=oper, n=info['name'])) valarr = op(dg.data[:,nd+w],1) else: if len(w) != 1: raise LookupError( 'Need just one instance of variable {0} in {1}' .format(varname, dg.filename)) if len(np.shape(dg.data))!=2: raise LookupError( 'insufficient data for {0} in {1}' .format(varname, dg.filename)) valarr = dg.data[:,nd+w[0]] tim = dg.data[:,0] - delay if oper == 'ddt': # derivative operator valarr = np.diff(valarr)/(np.average(np.diff(tim))) tim = (tim[0:-1] + tim[1:])/2.0 if oper == 'ddt2': # abd(ddw)*derivative operator dw = np.diff(valarr)/(np.average(np.diff(tim))) ddw = np.diff(dw)/(np.average(np.diff(tim))) tim = tim[2:] valarr = 4e-6 * dw[1:] * np.abs(ddw) if (len(tim) < 10) or (np.std(tim)<0.1): raise ValueError('Insufficient points or degenerate' 'timebase data in {0}, {1}' .format(varname, dg.filename)) if times is not None: debug_(max(pyfusion.DEBUG, debug), level=5, key='interp') valarr = (stineman_interp(times, tim, valarr)) w = np.where(times > max(tim)) valarr[w] = np.nan else: times = tim if valarr is not None: vals.update({diag: valarr}) # create an extra time array to allow a cross check vals.update({'check_tm':times}) vals.update({'check_shot':np.zeros(len(times),dtype=np.int)+shot}) return(vals)
def get_basic_diagnostics(diags=None, shot=54196, times=None, delay=None, exception=False, debug=0): """ return a list of np.arrays of normally numeric values for the times given, for the given shot. Will access server if env('IGETFILE') points to an exe, else accesses cache """ global HJ_summary # if no exception given and we are not debugging # note - exception=None is a valid entry, meaning tolerate no exceptions # so the "default" we use is False if exception==False and debug==0: exception=Exception if diags is None: diags = "<n_e19>,b_0,i_p,w_p,dw_pdt,dw_pdt2".split(',') if len(np.shape(diags)) == 0: diags = [diags] # LHD only if delay is None: delay = get_delay(shot) if times is None: times = np.linspace(0,4,4000) times = np.array(times) vals = {} # create an extra time array to allow a cross check vals.update({'check_tm':times}) vals.update({'check_shot':np.zeros(len(times),dtype=np.int)+shot}) debug_(pyfusion.DEBUG,2,key='get_basic') for diag in diags: if not(diag in file_info): warn('diagnostic {0} not found in shot {1}'.format(diag, shot),stacklevel=2) vals.update({diag: np.nan + times}) debug_(pyfusion.DEBUG,2,key='get_basic') else: info = file_info[diag] varname = info['name'] infofmt = info['format'] subfolder = infofmt.split('@')[0] filepath = os.path.sep.join([localigetfilepath,subfolder,infofmt]) if ':' in varname: (oper,varname) = varname.split(':') else: oper = None if '(' in varname: try: left,right = varname.split('(') varname,rest=right.split(')') except: raise ValueError('in expression {v} - parens?'.format(varname)) if infofmt.find('.npz') > 0: try: test=HJ_summary.keys() except: csvfilename = acq_HJ+'/'+infofmt if pyfusion.DBG() > 1: print('looking for HeliotronJ summary in' + csvfilename) print('reloading {0}'.format(csvfilename)) HJ_summary = np.load(csvfilename) val = HJ_summary[varname][shot] valarr = np.double(val)+(times*0) elif 'get_static_params' in infofmt: pdicts = eval(infofmt.format(shot=shot)) if len(pdicts)==0: print('empty dictionary returned') val = pdicts[varname] valarr = np.double(val)+(times*0) else: # read signal from data system debug_(max(pyfusion.DEBUG, debug), level=4, key='find_data') try: #get HJparams channel = info['name'] outdata=np.zeros(1024*2*256+1) channel_length =(len(outdata)-1)/2 # outdfile only needed for opt=1 (get data via temp file) # with tempfile.NamedTemporaryFile(prefix="pyfusion_") as outdfile: ierror, getrets=gethjdata.gethjdata(shot,channel_length, info['name'], verbose=VERBOSE, opt=1, ierror=2, outdata=outdata, outname='') if ierror != 0: raise LookupError('data not found for {s}:{c}'.format(s=shot, c=channel)) ch = Channel(info['name'], Coords('dummy', (0,0,0))) # timebase in secs (was ms in raw data) dg = TimeseriesData(timebase=Timebase(1e-3 * getrets[1::2]), signal=Signal(getrets[2::2]), channels=ch) except exception as reason: if debug>0: print('exception running gethjdata {r} {a}', format(r=reason, a=reason.args)) dg=None #break # give up and try next diagnostic if dg is None: # messy - break doesn't do what I want? valarr=None else: nd = 1 # initially only deal with single channels (HJ) # get the column(s) of the array corresponding to the name w = [0] if (oper in 'sum,average,rms,max,min'.split(',')): if oper=='sum': op = np.sum elif oper=='average': op = np.average elif oper=='min': op = np.min elif oper=='std': op = np.std else: raise ValueError('operator {o} in {n} not known to get_basic_diagnostics' .format(o=oper, n=info['name'])) # valarr = op(dg.data[:,nd+w],1) valarr = op(dg.data[:,nd+w],1) else: if len(w) != 1: raise LookupError( 'Need just one instance of variable {0} in {1}' .format(varname, dg.filename)) dg.data = dg.signal # fudge compatibility if len(np.shape(dg.data))!=1: # 2 for igetfile raise LookupError( 'insufficient data for {0} in {1}' .format(varname, dg.filename)) #valarr = dg.data[:,nd+w[0]] #tim = dg.data[:,0] - delay valarr = dg.signal tim = dg.timebase # fudge until we can gete the number of points valarr = valarr[:np.argmax(tim)] tim = tim[:np.argmax(tim)] if oper == 'ddt': # derivative operator valarr = np.diff(valarr)/(np.average(np.diff(tim))) tim = (tim[0:-1] + tim[1:])/2.0 if oper == 'ddt2': # abd(ddw)*derivative operator dw = np.diff(valarr)/(np.average(np.diff(tim))) ddw = np.diff(dw)/(np.average(np.diff(tim))) tim = tim[2:] valarr = 4e-6 * dw[1:] * np.abs(ddw) if (len(tim) < 10) or (np.std(tim)<0.1): raise ValueError('Insufficient points or degenerate' 'timebase data in {0}, {1}' .format(varname, dg.filename)) valarr = (stineman_interp(times, tim, valarr)) w = np.where(times > max(tim)) valarr[w] = np.nan if valarr is not None: vals.update({diag: valarr}) debug_(max(pyfusion.DEBUG, debug), level=5, key='interp') return(vals)
def make_mask(NA, norm_passband, norm_stopband, input_data, taper): """ works well now, except that the stopband is adjusted to be symmetric about the passband (take the average of the differences The problem with crashes (zero mask) was solved by shifting the mask before and after integrating, also a test for aliasing (on the mask before integration). """ mask = np.zeros(NA) # define the 4 key points # /npblow-------------npbhi\ # ___nsbl/ \nsbhi____ n_sb_low = int(norm_stopband[0] * NA / 2) n_pb_low = int(norm_passband[0] * NA / 2) n_pb_hi = int(norm_passband[1] * NA / 2) n_sb_hi = int(norm_stopband[1] * NA / 2) dt = float(np.average(np.diff(input_data.timebase))) if n_sb_hi >= len(mask): raise ValueError('Filter frequency too high for data - units ' 'problem? - sample spacing is {dt:.2g}'.format(dt=dt)) # twid is the transition width, and should default so that the sloped part is the same width as the flat? # !!! twid is not an input - !!!! doesn't do that yet. # make the transition width an even number, and the larger of the two # need to pull this code out and be sure it works. twid = 2 * (1 + max(n_pb_low - n_sb_low, n_sb_hi - n_pb_hi) // 2) if (twid > (n_pb_low - n_sb_low) * 3) or (twid > (n_sb_hi - n_pb_hi) * 3): print( '*********** Warning - unbalanced cutoff rate between high and low end' ' will cause the cutoff rates to be equalised widening one and reducing the other' ' difference between stop and pass bands should be similar ar both ends.' ) if (twid < 4): # or (n_sb_low < 0): #< not requ since fixed if taper == 2: raise ValueError( 'taper 2 requires a bigger margin between stop and pass') elif taper is None: warn('defaulting taper to 1 as band edges are sharp: twid={twid}'. format(twid=twid)) taper = 1 else: if taper is None: taper = 2 if taper == 1: # _____ # / \ # / \ # ______/ \___ # want 0 at sb low and sb high, 1 at pb low and pb high # present code does not quite do this. # try to prevent zero width or very narrow (DC only) filters. if n_sb_low < 0: n_sb_low = 0 if n_pb_low < 0: n_pb_low = 0 if n_pb_hi < 1: n_pb_hi = 1 if n_sb_hi <= n_pb_hi: n_sb_hi = n_pb_hi + 1 for n in range(n_sb_low, n_pb_low + 1): if n_sb_low == n_pb_low: # allow for pass=stop on low side mask[n] = 1. else: mask[n] = float(n - n_sb_low) / (n_pb_low - n_sb_low ) # trapezoid for n in range(n_pb_hi, n_sb_hi + 1): mask[n] = float(n_sb_hi - n) / (n_sb_hi - n_pb_hi) # trapezoid for n in range(n_pb_low, n_pb_hi + 1): mask[n] = 1 elif taper == 2: # Note - must symmetrise (so that cumsum works) # _ # / \ # | | # ______/ \___ # want 0 at sb low and sb high, 1 at pb low and pb high # this means that the peak of the mask before integration is halfway between sb_low and pb_low # and pb_low - sb_low is an even number # present code does not quite do this. n_sb_low = n_pb_low - twid # sacrifice the stop band, not the pass n_sb_hi = n_pb_hi + twid low_mid = n_pb_low - twid // 2 high_mid = n_pb_hi + twid // 2 for n in range(n_sb_low, low_mid): mask[n] = float(n - n_sb_low) / (low_mid - 1 - n_sb_low ) # trapezoid mask[2 * low_mid - n - 1] = mask[n] #down ramp - repeat max #wid_up = n_sb_hi - n_pb_hi for n in range(n_pb_hi, high_mid): # negative tri mask[n] = float(n_pb_hi - n) / (high_mid - n_pb_hi - 1 ) # trapezoid mask[2 * high_mid - n - 1] = mask[n] before_integration = mask # after running filters.py, this should be OK # make_mask(512, [0.8,.93], [0.9,.98],dat,2) # but changing 0.98 to 0.99 will give aliasing error. if np.max(np.abs(mask[NA // 2 - 4:NA // 2 + 4])) > 0: raise ValueError('mask aliasing error') # note: ifftshift is only different for an odd data length # the fftshifts were necessary to avoid weirdness if the # stopband went below zero freq. mask = np.fft.ifftshift(np.cumsum(np.fft.fftshift(mask))) # integrate if pyfusion.DBG() > 1: nonr = 0.5 / dt fig = plt.figure() ax1 = fig.add_subplot(111) ax1.plot(np.arange(len(mask)) / dt / float(NA), mask, '.-') ax1.plot(np.arange(len(mask)) / dt / float(NA), before_integration) ax1.set_xlabel( 'real freq. units, (norm on top scale), npoints={NA}, norm/real = {nonr}' .format(NA=NA, nonr=nonr)) ax2 = ax1.twiny() # this is my hack - it should be OK, but may be a little out ax2.set_xlim(np.array(ax1.get_xlim()) / nonr) fig.suptitle( 'mask before normalisation - twid={twid}'.format(twid=twid)) plt.show(0) if np.max(mask) == 0: raise ValueError( 'zero mask, ' 'norm_passband = {pb}, norm_stopband={sb}, taper {t}'.format( pb=norm_passband, sb=norm_stopband, t=taper)) mask = mask / np.max(mask) # reflection only required for complex data # this even and odd is not totally thought through...but it seems OK if np.mod(NA, 2) == 0: mask[:NA / 2:-1] = mask[1:(NA / 2)] # even else: mask[:1 + NA / 2:-1] = mask[1:(NA / 2)] # odd return (mask)
def filter_fourier_bandpass(input_data, passband, stopband, taper=None, debug=None): """ Note: Is MUCH (2.2x faster) more efficient to use real ffts, (implemented April) Use a Fourier space taper/tophat or pseudo gaussian filter to perform narrowband filtering (much narrower than butterworth). Problem is that bursts may generate ringing. (should be better with taper=2) >>> tb = dummytb(np.linspace(0,20,512)) >>> w = 2*np.pi* 1 # 1 Hertz >>> dat = dummysig(tb,np.sin(w*tb.timebase)*(tb.timebase<np.max(tb.timebase)/3)) >>> fop = filter_fourier_bandpass(dat,[0.9,1.1],[0.8,1.2],debug=2).signal[0] """ if debug == None: debug = pyfusion.DEBUG # normalising makes it easier to think about - also for But'w'h norm_passband = input_data.timebase.normalise_freq(np.array(passband)) norm_stopband = input_data.timebase.normalise_freq(np.array(stopband)) NS = len(input_data.signal[0]) NA = next_nice_number(NS) # take a little more to speed up FFT mask = np.zeros(NA) # define the 4 key points # /npblow-------------npbhi\ # ___nsbl/ \nsbhi____ n_sb_low = int(norm_stopband[0]*NA/2) n_pb_low = int(norm_passband[0]*NA/2) n_pb_hi = int(norm_passband[1]*NA/2) n_sb_hi = int(norm_stopband[1]*NA/2) wid = max(n_pb_low - n_sb_low,n_sb_hi - n_pb_hi) if wid < 4: if taper == 2: raise ValueError( 'taper 2 requres a bigger margin between stop and pass') elif taper == None: warn('defaulting taper to 1 as band edges are sharp') taper = 1 else: if taper == None: taper = 2 if taper==1: for n in range(n_sb_low,n_pb_low+1): if n_pb_low == n_sb_low: # allow for pass=stop on low side mask[n]=1. else: mask[n] = float(n - n_sb_low)/(n_pb_low - n_sb_low) # trapezoid for n in range(n_pb_hi,n_sb_hi+1): mask[n] = float(n_sb_hi - n)/(n_sb_hi - n_pb_hi) # trapezoid for n in range(n_pb_low,n_pb_hi+1): mask[n] = 1 elif taper == 2: # symmetrise (so that cumsum works) n_pb_low = n_sb_low+wid n_sb_hi = n_pb_hi+wid for n in range(n_sb_low,n_pb_low+1): mask[n] = float(n - n_sb_low)/(n_pb_low - n_sb_low) # trapezoid mask[2*n_pb_low-n+1] = mask[n] #down ramp wid_up = n_sb_hi - n_pb_hi for n in range(n_pb_hi,n_sb_hi+1): mask[n] = -float(n_sb_hi - n)/(n_sb_hi - n_pb_hi) # trapezoid mask[2*n_pb_hi - n - 1] = mask[n] mask = np.cumsum(mask) # integrate mask = mask/np.max(mask) # reflection only required for complex data # this even and odd is not totally thought through...but it seems OK if np.mod(NA,2)==0: mask[:NA/2:-1] = mask[1:(NA/2)] # even else: mask[:1+NA/2:-1] = mask[1:(NA/2)] # odd output_data = copy.deepcopy(input_data) # was output_data = input_data if (pyfusion.fft_type == 'fftw3'): # should migrate elsewhere import pyfftw tdtype = np.float32 fdtype = np.complex64 # this could be useful to cache. simd_align = pyfftw.simd_alignment # 16 at the moment. tdom = pyfftw.n_byte_align(np.zeros(NA,dtype=tdtype), simd_align) FT = pyfftw.n_byte_align_empty(NA/2+1, simd_align, fdtype) ids = [[id(tdom),id(FT)]] # check to see if it moves out of alignment fwd = pyfftw.FFTW(tdom, FT, direction='FFTW_FORWARD', **pyfusion.fftw3_args) rev = pyfftw.FFTW(FT, tdom, direction='FFTW_BACKWARD', **pyfusion.fftw3_args) else: tdtype = np.float32 tdom = np.zeros(NA,dtype=tdtype) # example of tuning #pyfusion.fftw3_args= {'planning_timelimit': 50.0, 'threads':1, 'flags':['FFTW_MEASURE']} for i,s in enumerate(output_data.signal): #if len(output_data.signal) == 1: print('bug for a single signal') #time run -i pyfusion/examples/plot_svd.py "dev_name='LHD'" start_time=.497 "normalise='r'" shot_number=90091 numpts=512 diag_name=MP2010HMPno612 "filter=dict(centre=8e3,bw=5e3,taper=2)" plot_mag=1 plot_phase=1 separate=1 closed=0 time_range=[0.0000,4.] # 4.5 cf 15.8diag_name=MP2010HMPno612, time_range=[0.0000,2.80000] # 0, 4.194304 2**21 samples, 21.8 cf 6.8 1thr # (0,2)secs 90091 =2000000 samples 17 np, 5.73 2thread, nosimd, 6.1 1thread (mem bw?) 3.2 sec no filt # note - the above are on an intermeittently loaded E4300 2 processor, below on 4 core 5/760 # 0, 4.194304 2**21 samples, 10.9 cf 3.16 thr2 3.47 1thr and 2.0 secs no filter # for 17 fft/ifft takes about 1.16 sec 2 threads - should be (27.5ms+28.6)*17 = 952ms (14.2 2thr) OK # duplicate the fft execute lines 4.3(3.47) 2thr 3.7(3.16) extra 810ms (expect 14.2ms * 2 * 17) =482 # the difference between 2 and 1thr should be 14*2*17 ms 500ms. # orignall - 90ms/channel extra in reverse trasnform - maybe the 50 sec limit stopped optimization # next _nice: 5.74 for 10 sec lenny # E4300: 1thr 9.3 (39np) for 10 sec 90091; 5.5 for 4 sec (19.6 np) if (pyfusion.fft_type == 'fftw3'): # fftw3 nosim, no thread 2.8s cf 10s tdom[0:NS]=s # indexed to make sure tdom stays put if NS != NA: tdom[NS:]=0. fwd.execute() FT[:] = FT * mask[0:NA/2+1] # 12ms rev.execute() output_data.signal[i] = tdom[0:NS] ids.append([id(tdom),id(FT)]) else: # default to numpy tdom[0:NS] = s FT = np.fft.fft(tdom) IFT = np.fft.ifft(mask*FT) if np.max(np.abs(IFT.imag)) > 1e-6*np.max(np.abs(IFT.real)): pyfusion.logger.warning("inverse fft imag part > 1e-6") output_data.signal[i] = IFT.real[0:NS] if debug>2: print('ids of fftw3 input and output: {t}'.format(t=ids)) if debug>0: import pylab as pl pl.figure() pl.plot(mask,'r.-',label='mask') pl.plot(np.abs(FT)/len(mask),label='FT') pl.plot(input_data.signal[0],label='input') pl.plot(output_data.signal[0],label='output') pl.legend() pl.show() debug_(debug, 2, key='filter_fourier') if np.max(mask) == 0: raise ValueError('Filter blocks all signals') return output_data
# has [...]. Figure out why. (...probably because original signal # uses a build_signal function) if len(dim) == 1: dim = dim[0] timebase = Timebase(dim) output_data = TimeseriesData(timebase=timebase, signal=signal, channels=ch) output_data.config_name = fetcher.config_name #bdb config_name fix output_data.meta.update({'shot':fetcher.shot}) return output_data # Don't import in the header, so that .npz files can be used - e.g. JSPF_tutorial example1 try: import MDSplus except ImportError: warn(' No MIT MDSplus software found - will only work on local .npz data' ' Try easy_install mdsplus, or see the ubuntu/redhat... mdsplus distros ' 'http://www.mdsplus.org ' 'if you wish to access native MDSplus data') class MDSPlusDataFetcher(BaseDataFetcher): """Determine which access mode should be used, and fetch the MDSplus data.""" def setup(self): self.mds_path_components = get_tree_path(self.mds_path) if hasattr(self.acq, '%s_path' %self.mds_path_components['tree']): self.tree = MDSplus.Tree(self.mds_path_components['tree'], self.shot) self.fetch_mode = 'local_path_mode' # this refers to access by _path e.g. h1data_path # bdb wants to call it local_path_mode, but maybe # TestNoSQLTestDeviceGetdata fails elif self.acq.server_mode == 'mds':
def filter_fourier_bandpass(input_data, passband, stopband, taper=None, debug=None): """ Note: Is MUCH (2.2x faster) more efficient to use real ffts, (implemented April) Use a Fourier space taper/tophat or pseudo gaussian filter to perform narrowband filtering (much narrower than butterworth). Problem is that bursts may generate ringing. (should be better with taper=2) >>> tb = dummytb(np.linspace(0,20,512)) >>> w = 2*np.pi* 1 # 1 Hertz >>> dat = dummysig(tb,np.sin(w*tb.timebase)*(tb.timebase<np.max(tb.timebase)/3)) >>> fop = filter_fourier_bandpass(dat,[0.9,1.1],[0.8,1.2],debug=2).signal[0] """ if debug == None: debug = pyfusion.DEBUG # normalising makes it easier to think about - also for But'w'h norm_passband = input_data.timebase.normalise_freq(np.array(passband)) norm_stopband = input_data.timebase.normalise_freq(np.array(stopband)) NS = len(input_data.signal[0]) NA = next_nice_number(NS) # take a little more to speed up FFT mask = np.zeros(NA) # define the 4 key points # /npblow-------------npbhi\ # ___nsbl/ \nsbhi____ n_sb_low = int(norm_stopband[0] * NA / 2) n_pb_low = int(norm_passband[0] * NA / 2) n_pb_hi = int(norm_passband[1] * NA / 2) n_sb_hi = int(norm_stopband[1] * NA / 2) wid = max(n_pb_low - n_sb_low, n_sb_hi - n_pb_hi) if wid < 4: if taper == 2: raise ValueError( 'taper 2 requres a bigger margin between stop and pass') elif taper == None: warn('defaulting taper to 1 as band edges are sharp') taper = 1 else: if taper == None: taper = 2 if taper == 1: for n in range(n_sb_low, n_pb_low + 1): if n_pb_low == n_sb_low: # allow for pass=stop on low side mask[n] = 1. else: mask[n] = float(n - n_sb_low) / (n_pb_low - n_sb_low ) # trapezoid for n in range(n_pb_hi, n_sb_hi + 1): mask[n] = float(n_sb_hi - n) / (n_sb_hi - n_pb_hi) # trapezoid for n in range(n_pb_low, n_pb_hi + 1): mask[n] = 1 elif taper == 2: # symmetrise (so that cumsum works) n_pb_low = n_sb_low + wid n_sb_hi = n_pb_hi + wid for n in range(n_sb_low, n_pb_low + 1): mask[n] = float(n - n_sb_low) / (n_pb_low - n_sb_low) # trapezoid mask[2 * n_pb_low - n + 1] = mask[n] #down ramp wid_up = n_sb_hi - n_pb_hi for n in range(n_pb_hi, n_sb_hi + 1): mask[n] = -float(n_sb_hi - n) / (n_sb_hi - n_pb_hi) # trapezoid mask[2 * n_pb_hi - n - 1] = mask[n] mask = np.cumsum(mask) # integrate mask = mask / np.max(mask) # reflection only required for complex data # this even and odd is not totally thought through...but it seems OK if np.mod(NA, 2) == 0: mask[:NA / 2:-1] = mask[1:(NA / 2)] # even else: mask[:1 + NA / 2:-1] = mask[1:(NA / 2)] # odd output_data = copy.deepcopy(input_data) # was output_data = input_data if (pyfusion.fft_type == 'fftw3'): # should migrate elsewhere import pyfftw tdtype = np.float32 fdtype = np.complex64 # this could be useful to cache. simd_align = pyfftw.simd_alignment # 16 at the moment. tdom = pyfftw.n_byte_align(np.zeros(NA, dtype=tdtype), simd_align) FT = pyfftw.n_byte_align_empty(NA / 2 + 1, simd_align, fdtype) ids = [[id(tdom), id(FT)]] # check to see if it moves out of alignment fwd = pyfftw.FFTW(tdom, FT, direction='FFTW_FORWARD', **pyfusion.fftw3_args) rev = pyfftw.FFTW(FT, tdom, direction='FFTW_BACKWARD', **pyfusion.fftw3_args) else: tdtype = np.float32 tdom = np.zeros(NA, dtype=tdtype) # example of tuning #pyfusion.fftw3_args= {'planning_timelimit': 50.0, 'threads':1, 'flags':['FFTW_MEASURE']} for i, s in enumerate(output_data.signal): #if len(output_data.signal) == 1: print('bug for a single signal') #time run -i pyfusion/examples/plot_svd.py "dev_name='LHD'" start_time=.497 "normalise='r'" shot_number=90091 numpts=512 diag_name=MP2010HMPno612 "filter=dict(centre=8e3,bw=5e3,taper=2)" plot_mag=1 plot_phase=1 separate=1 closed=0 time_range=[0.0000,4.] # 4.5 cf 15.8diag_name=MP2010HMPno612, time_range=[0.0000,2.80000] # 0, 4.194304 2**21 samples, 21.8 cf 6.8 1thr # (0,2)secs 90091 =2000000 samples 17 np, 5.73 2thread, nosimd, 6.1 1thread (mem bw?) 3.2 sec no filt # note - the above are on an intermeittently loaded E4300 2 processor, below on 4 core 5/760 # 0, 4.194304 2**21 samples, 10.9 cf 3.16 thr2 3.47 1thr and 2.0 secs no filter # for 17 fft/ifft takes about 1.16 sec 2 threads - should be (27.5ms+28.6)*17 = 952ms (14.2 2thr) OK # duplicate the fft execute lines 4.3(3.47) 2thr 3.7(3.16) extra 810ms (expect 14.2ms * 2 * 17) =482 # the difference between 2 and 1thr should be 14*2*17 ms 500ms. # orignall - 90ms/channel extra in reverse trasnform - maybe the 50 sec limit stopped optimization # next _nice: 5.74 for 10 sec lenny # E4300: 1thr 9.3 (39np) for 10 sec 90091; 5.5 for 4 sec (19.6 np) if (pyfusion.fft_type == 'fftw3' ): # fftw3 nosim, no thread 2.8s cf 10s tdom[0:NS] = s # indexed to make sure tdom stays put if NS != NA: tdom[NS:] = 0. fwd.execute() FT[:] = FT * mask[0:NA / 2 + 1] # 12ms rev.execute() output_data.signal[i] = tdom[0:NS] ids.append([id(tdom), id(FT)]) else: # default to numpy tdom[0:NS] = s FT = np.fft.fft(tdom) IFT = np.fft.ifft(mask * FT) if np.max(np.abs(IFT.imag)) > 1e-6 * np.max(np.abs(IFT.real)): pyfusion.logger.warning("inverse fft imag part > 1e-6") output_data.signal[i] = IFT.real[0:NS] if debug > 2: print('ids of fftw3 input and output: {t}'.format(t=ids)) if debug > 0: import pylab as pl pl.figure() pl.plot(mask, 'r.-', label='mask') pl.plot(np.abs(FT) / len(mask), label='FT') pl.plot(input_data.signal[0], label='input') pl.plot(output_data.signal[0], label='output') pl.legend() pl.show() debug_(debug, 2, key='filter_fourier') if np.max(mask) == 0: raise ValueError('Filter blocks all signals') return output_data
raise ValueError('sel and csel must have the same length') try: cldata if oldclusterfile != clusterfile: 1/0 print('using previous clusterfile data' ' - set oldclusterfile=None to prevent') except: print('loading cluster data from {cf}'.format(cf=clusterfile)) cldata = np.load(clusterfile) oldclusterfile = clusterfile oldsel = sel for k in cldata.keys(): exec("{v}=cldata['{k}']".format(v=k,k=k)) if np.shape(sel) != np.shape(oldsel) or (sel != oldsel).any(): warn('sel value tried to change from {o} to {n} - i.e. cldata has different sel, ' .format(o=oldsel, n=sel)) sel = oldsel if csel is not None: subset=subset[:,csel] # this contrivance allows us to test on uniformly distributed phases if uniform_random is not None: print('evaluating {ur}'.format(ur=uniform_random)) phases = eval(uniform_random) shot = np.array(np.shape(phases)[0]*[shot[0]]) freq = np.array(np.shape(phases)[0]*[freq[0]]) frlow = np.array(np.shape(phases)[0]*[0]) frhigh = np.array(np.shape(phases)[0]*[9e9]) t_mid = np.array(np.shape(phases)[0]*[t_mid[-1]]) cc5 = [] # cluster centres for cl in cls:
def make_title(formatstr, input_data, channum=None, at_dict={}, min_length=3, raw_names=False): """ Return a string describing the shot number, channel name etc using a formatstr which refers to items in a dictionary (at_dict), assembled in this routine, based on input_data and an optional dictionary which contains anything not otherwise available in input_data """ ## at_dict.update({'shot': input_data.meta['shot']}) exception = () if pyfusion.DBG() > 3 else Exception try: at_dict.update(input_data.meta) # this gets all of it! if channum is None: name = '' else: if isinstance(input_data.channels, list): chan = input_data.channels[channum] else: chan = input_data.channels if raw_names: name = chan.name else: name = chan.config_name try: # remove leading W7X_ LHD_ etc from channel labels. if 'device' in input_data.params.values()[0]['params']: device = input_data.params.values()[0]['params']['device'] name = name.replace(device + '_', '') except: pass at_dict.update({'units': chan.units}) at_dict.update({'name': name}) # replace internal strings of non-numbers with a single . a14_input03 -> 14.03 short_name = '' last_was_number = False discarded = '' debug_(pyfusion.DEBUG, 4, key='make_title') for c in name: # start from the first char if c >= '0' and c <= '9': short_name += c last_was_number = True else: if last_was_number: short_name += '.' else: discarded += c last_was_number = False if len(short_name) <= min_length: # if it fits, have the lot if len(name) < 8: short_name = name # else allow 4 more chars - makes about 6-8 chars else: short_name = discarded[-4:] + short_name at_dict.update({'short_name': short_name}) return (formatstr.format(**at_dict)) except exception as ex: warn('in make_title for format="%s", at_dict=%s' % (formatstr, at_dict), exception=ex) return ('')
if len(dim) == 1: dim = dim[0] timebase = Timebase(dim) output_data = TimeseriesData(timebase=timebase, signal=signal, channels=ch) output_data.config_name = fetcher.config_name #bdb config_name fix output_data.meta.update({'shot': fetcher.shot}) return output_data # Don't import in the header, so that .npz files can be used - e.g. JSPF_tutorial example1 try: import MDSplus except ImportError: warn( ' No MIT MDSplus software found - will only work on local .npz data' ' Try easy_install mdsplus, or see the ubuntu/redhat... mdsplus distros ' 'http://www.mdsplus.org ' 'if you wish to access native MDSplus data') class MDSPlusDataFetcher(BaseDataFetcher): """Determine which access mode should be used, and fetch the MDSplus data.""" def setup(self): self.mds_path_components = get_tree_path(self.mds_path) if hasattr(self.acq, '%s_path' % self.mds_path_components['tree']): self.tree = MDSplus.Tree(self.mds_path_components['tree'], self.shot) self.fetch_mode = 'local_path_mode' # this refers to access by _path e.g. h1data_path # bdb wants to call it local_path_mode, but maybe # TestNoSQLTestDeviceGetdata fails
for idx in ord: t_seg = ord_segs[idx] if np.isnan(np.sum(t_seg.signal)): # if signal is missing, skip continue if max_bands > 1: # keep it simple if one band (ipk, fpk, apk) = find_signal_spectral_peaks(t_seg.timebase, t_seg.signal[0], minratio=0.1) w_in_range = np.where(fpk < fmax)[0] (ipk, fpk, apk) = (ipk[w_in_range], fpk[w_in_range], apk[w_in_range]) if len(ipk) > max_bands: if pyfusion.DBG() > 0: # in python3 at least, the traceback info is # out one level. warn('too many peaks - choosing largest {mb}'.format( mb=max_bands)) fpk = np.sort(fpk[np.argsort(apk)[-(max_bands + 1):]]) (lfs, hfs) = subdivide_interval(np.append(fpk, fmax), debug=0, overlap=(df / 2, df * 2)) elif max_bands == 1: # these two compare with and without fourier filter (lfs, hfs) = ([0], [fmax * 0.99]) else: (lfs, hfs) = ([0], [fmax]) for i in range(len(lfs)): (frlow, frhigh) = (lfs[i], hfs[i]) if max_bands > 0: f_seg = t_seg.filter_fourier_bandpass(
if fmax is None: fmax = 0.5/np.average(np.diff(ord_segs[0].timebase)) - df timeinfo('beginning flucstruc loop') for idx in ord: t_seg = ord_segs[idx] if max_bands>1: # keep it simple if one band (ipk, fpk, apk) = find_signal_spectral_peaks(t_seg.timebase, t_seg.signal[0],minratio=0.1) w_in_range = np.where(fpk < fmax)[0] (ipk, fpk, apk) = (ipk[w_in_range], fpk[w_in_range], apk[w_in_range]) if len(ipk)>max_bands: if pyfusion.DBG() > 0: # in python3 at least, the traceback info is # out one level. warn('too many peaks - choosing largest {mb}'.format(mb=max_bands)) fpk = np.sort(fpk[np.argsort(apk)[-(max_bands+1):]]) (lfs, hfs) = subdivide_interval(np.append(fpk, fmax), debug=0, overlap=(df/2,df*2)) elif max_bands == 1: # these two compare with and without fourier filter (lfs,hfs) = ([0],[fmax*0.99]) else: (lfs,hfs) = ([0],[fmax]) for i in range(len(lfs)): (frlow,frhigh)= (lfs[i],hfs[i]) if max_bands > 0: f_seg = t_seg.filter_fourier_bandpass(
pass ord_segs = [] for ii, t_seg in enumerate(sections): ord_segs.append(t_seg) ord = argsort([average(t_seg.timebase) for t_seg in ord_segs]) if fmax == None: fmax = 0.5 / np.average(np.diff(ord_segs[0].timebase)) - df for idx in ord: t_seg = ord_segs[idx] (ipk, fpk, apk) = find_signal_spectral_peaks(t_seg.timebase, t_seg.signal[0], minratio=0.1) w_in_range = np.where(fpk < fmax)[0] (ipk, fpk, apk) = (ipk[w_in_range], fpk[w_in_range], apk[w_in_range]) if len(ipk) > max_bands: warn("too many peaks - reducing to {mb}".format(mb=max_bands)) fpk = np.sort(fpk[np.argsort(apk)[-(max_bands + 1) :]]) (lfs, hfs) = subdivide_interval(np.append(fpk, fmax), debug=0, overlap=(df / 2, df * 2)) for i in range(len(lfs)): (frlow, frhigh) = (lfs[i], hfs[i]) f_seg = t_seg.filter_fourier_bandpass([lfs[i], hfs[i]], [lfs[i] - df, hfs[i] + df]) fs_set = f_seg.flucstruc(method=method, separate=separate) for fs in fs_set: if count == 0: # show history if info says to, and avoid starting line with a digit if info > 0: print("< " + fs.history.replace("\n201", "\n< 201")) print( "Shot time SVS freq Amp a12 p H frlow frhigh {np:2d} Phases".format( np=len(fs.dphase)
except: pass ord_segs = [] for ii,t_seg in enumerate(sections): ord_segs.append(t_seg) ord = argsort([average(t_seg.timebase) for t_seg in ord_segs]) if fmax == None: fmax = 0.5/np.average(np.diff(ord_segs[0].timebase)) - df for idx in ord: t_seg = ord_segs[idx] (ipk, fpk, apk) = find_signal_spectral_peaks(t_seg.timebase, t_seg.signal[0],minratio=0.1) w_in_range = np.where(fpk < fmax)[0] (ipk, fpk, apk) = (ipk[w_in_range], fpk[w_in_range], apk[w_in_range]) if len(ipk)>max_bands: warn('too many peaks - reducing to {mb}'.format(mb=max_bands)) fpk = np.sort(fpk[np.argsort(apk)[-(max_bands+1):]]) (lfs, hfs) = subdivide_interval(np.append(fpk, fmax), debug=0, overlap=(df/2,df*2)) for i in range(len(lfs)): (frlow,frhigh)= (lfs[i],hfs[i]) f_seg = t_seg.filter_fourier_bandpass( [lfs[i],hfs[i]], [lfs[i]-df,hfs[i]+df]) fs_set = f_seg.flucstruc(method=method, separate=separate) for fs in fs_set: if count==0: # show history if info says to, and avoid starting line with a digit if info > 0: print('< '+fs.history.replace('\n201','\n< 201')) print('Shot time SVS freq Amp a12 p H frlow frhigh {np:2d} Phases'.format(np=len(fs.dphase))) count += 1 if fs.H < max_H and fs.p>0.01 and len(fs.svs())>=min_svs: