def make_title(formatstr, input_data, channum=None, at_dict = {}, min_length=3, raw_names=False):
""" Return a string describing the shot number, channel name etc using
a formatstr which refers to items in a dictionary (at_dict), assembled in
this routine, based on input_data and an optional dictionary which
contains anything not otherwise available in input_data
"""
## at_dict.update({'shot': input_data.meta['shot']})
exception = () if pyfusion.DBG() > 3 else Exception
try:
at_dict.update(input_data.meta) # this gets all of it!
if channum is None:
name = ''
else:
if isinstance(input_data.channels, list):
chan = input_data.channels[channum]
else:
chan = input_data.channels
if raw_names:
name = chan.name
else:
name = chan.config_name
at_dict.update({'units':chan.units})
at_dict.update({'name': name})
# replace internal strings of non-numbers with a single . a14_input03 -> 14.03
short_name=''
last_was_number = False
discarded=''
for c in name: # start from the first char
if c>='0' and c<='9':
short_name += c
last_was_number=True
else:
if last_was_number: short_name += '.'
else: discarded += c
last_was_number=False
if len(short_name) <= min_length:
# if it fits, have the lot
if len(name)<8: short_name=name
# else allow 4 more chars - makes about 6-8 chars
else: short_name = discarded[-4:] + short_name
at_dict.update({'short_name': short_name})
return(formatstr.format(**at_dict))
except exception as ex:
warn('in make_title for format="%s", at_dict=%s' % (formatstr, at_dict),
exception=ex)
return('')
def set_axis_if_OK(ax, xlims, ylims):
"""
set axes if they containg at least a bit of the plot
originally meant to be used when forcing specific axes, to allow
for attempt to force axes suitable for LHD on HJ data for example
"""
from pyfusion.utils.utils import warn
if overlap(ax.get_xlim(), xlims) and overlap(ax.get_ylim(), ylims):
ax.axis(xlims[0], xlims[1], ylims[0], ylims[1])
else:
warn('suppressed attempt to set axis outside ranges\n{att:s} cf {act:s}'
.format(att=(xlims,ylims), act=(ax.get_xlim(),ax.get_ylim())))
def set_axis_if_OK(ax, xlims, ylims):
"""
set axes if they containg at least a bit of the plot
originally meant to be used when forcing specific axes, to allow
for attempt to force axes suitable for LHD on HJ data for example
"""
from pyfusion.utils.utils import warn
if overlap(ax.get_xlim(), xlims) and overlap(ax.get_ylim(), ylims):
ax.axis(xlims[0], xlims[1], ylims[0], ylims[1])
else:
warn(
'suppressed attempt to set axis outside ranges\n{att:s} cf {act:s}'
.format(att=(xlims, ylims), act=(ax.get_xlim(), ax.get_ylim())))
def subdivide_interval(pts, overlap=None, debug=0):
""" return several intervals which straddle pts
with overlap of ov
The lowest x value is special - the point of division is much closer
to that x then zero.
overlap is a tuple (minoverlap, max), and describes the total overlap
"""
if overlap == None:
overlap = [np.max(pts) / 50, np.max(pts) / 20]
if len(overlap) == 1:
warn("overlap should have a min and a max")
overlap = [overlap / 3.0, overlap]
if (np.diff(pts) < 0).any():
warn("points out of order - reordering")
pts = np.sort(pts)
begins = []
ends = []
for (i, x) in enumerate(pts):
if i == 0:
divider = x * 0.8 - overlap[1] / 2.0
else:
divider = (x + pts[i - 1]) / 2.0
if i == 0:
begins.append(0)
ends.append(divider + overlap[1] / 2.0)
else:
this_overlap = min(max((divider - last_divider) / 20, overlap[0]), overlap[1]) / 2.0
begins.append(last_divider - this_overlap)
ends.append(divider + this_overlap)
last_divider = divider
if debug > 1:
print(begins, pts, ends)
if debug > 2:
import pylab as pl
pl.figure()
for i in range(len(pts)):
pl.plot([begins[i], ends[i]], [i, i])
if i > 0:
pl.plot([pts[i - 1], pts[i - 1]], [i, i], "o")
pl.ylim(-1, 20)
pl.show()
return (begins, ends)
def subdivide_interval(pts, overlap=None, debug=0):
""" return several intervals which straddle pts
with overlap of ov
The lowest x value is special - the point of division is much closer
to that x then zero.
overlap is a tuple (minoverlap, max), and describes the total overlap
"""
if overlap is None: overlap = [np.max(pts) / 50, np.max(pts) / 20]
if len(overlap) == 1:
warn('overlap should have a min and a max')
overlap = [overlap / 3.0, overlap]
if (np.diff(pts) < 0).any():
warn('points out of order - reordering')
pts = np.sort(pts)
begins = []
ends = []
for (i, x) in enumerate(pts):
if i == 0:
divider = x * 0.8 - overlap[1] / 2.
else:
divider = (x + pts[i - 1]) / 2.
if i == 0:
begins.append(0)
ends.append(divider + overlap[1] / 2.)
else:
this_overlap = min(max(
(divider - last_divider) / 20, overlap[0]), overlap[1]) / 2.
begins.append(last_divider - this_overlap)
ends.append(divider + this_overlap)
last_divider = divider
if debug > 1: print(begins, pts, ends)
if debug > 2:
import pylab as pl
pl.figure()
for i in range(len(pts)):
pl.plot([begins[i], ends[i]], [i, i])
if i > 0: pl.plot([pts[i - 1], pts[i - 1]], [i, i], 'o')
pl.ylim(-1, 20)
pl.show()
return (begins, ends)
def make_title(formatstr, input_data, channum=None, dict={}, min_length=3):
""" return a string describing the shot number, channel name etc using a formatstr
which referes to items in a dictionary, assembled in this routine, based on input_data
and an optional dictionary which contains anything not otherwise available in input_data
"""
## dict.update({'shot': input_data.meta['shot']})
try:
dict.update(input_data.meta) # this gets all of it!
if channum == None:
name = ""
else:
try:
name = input_data.channels[channum].name
except:
name = input_data.channels.name
dict.update({"name": name})
# replace internal strings of non-numbers with a single . a14_input03 -> 14.03
short_name = ""
last_was_number = False
for c in name:
if c >= "0" and c <= "9":
short_name += c
last_was_number = True
else:
if last_was_number:
short_name += "."
last_was_number = False
if len(short_name) <= min_length:
short_name = name
dict.update({"short_name": short_name})
return formatstr % dict
except Exception as ex:
warn('in make_title for format="%s", dict=%s' % (formatstr, dict), exception=ex)
return ""
def make_title(formatstr, input_data, channum=None, dict={}, min_length=3):
""" return a string describing the shot number, channel name etc using a formatstr
which referes to items in a dictionary, assembled in this routine, based on input_data
and an optional dictionary which contains anything not otherwise available in input_data
"""
## dict.update({'shot': input_data.meta['shot']})
try:
dict.update(input_data.meta) # this gets all of it!
if channum == None:
name = ''
else:
try:
name = input_data.channels[channum].name
except:
name = input_data.channels.name
dict.update({'name': name})
# replace internal strings of non-numbers with a single . a14_input03 -> 14.03
short_name = ''
last_was_number = False
for c in name:
if c >= '0' and c <= '9':
short_name += c
last_was_number = True
else:
if last_was_number: short_name += '.'
last_was_number = False
if len(short_name) <= min_length: short_name = name
dict.update({'short_name': short_name})
return (formatstr % dict)
except Exception as ex:
warn('in make_title for format="%s", dict=%s' % (formatstr, dict),
exception=ex)
return ('')
def get_basic_diagnostics(diags=None,
shot=54196,
times=None,
delay=None,
exception=False,
debug=0):
""" return a list of np.arrays of normally numeric values for the
times given, for the given shot.
Will access server if env('IGETFILE') points to an exe, else accesses cache
"""
global HJ_summary
# if no exception given and we are not debugging
# note - exception=None is a valid entry, meaning tolerate no exceptions
# so the "default" we use is False
if exception == False and debug == 0: exception = Exception
if diags is None: diags = "<n_e19>,b_0,i_p,w_p,dw_pdt,dw_pdt2".split(',')
if len(np.shape(diags)) == 0: diags = [diags]
# LHD only if delay is None: delay = get_delay(shot)
if times is None:
times = np.linspace(0, 4, 4000)
times = np.array(times)
vals = {}
# create an extra time array to allow a cross check
vals.update({'check_tm': times})
vals.update({'check_shot': np.zeros(len(times), dtype=np.int) + shot})
debug_(pyfusion.DEBUG, 2, key='get_basic')
for diag in diags:
if not (diag in file_info):
warn('diagnostic {0} not found in shot {1}'.format(diag, shot),
stacklevel=2)
vals.update({diag: np.nan + times})
debug_(pyfusion.DEBUG, 2, key='get_basic')
else:
info = file_info[diag]
varname = info['name']
infofmt = info['format']
subfolder = infofmt.split('@')[0]
filepath = os.path.sep.join(
[localigetfilepath, subfolder, infofmt])
if ':' in varname: (oper, varname) = varname.split(':')
else: oper = None
if '(' in varname:
try:
left, right = varname.split('(')
varname, rest = right.split(')')
except:
raise ValueError(
'in expression {v} - parens?'.format(varname))
if infofmt.find('.npz') > 0:
try:
test = HJ_summary.keys()
except:
csvfilename = acq_HJ + '/' + infofmt
if pyfusion.DBG() > 1:
print('looking for HeliotronJ summary in' +
csvfilename)
print('reloading {0}'.format(csvfilename))
HJ_summary = np.load(csvfilename)
val = HJ_summary[varname][shot]
valarr = np.double(val) + (times * 0)
elif 'get_static_params' in infofmt:
pdicts = eval(infofmt.format(shot=shot))
if len(pdicts) == 0:
print('empty dictionary returned')
val = pdicts[varname]
valarr = np.double(val) + (times * 0)
else: # read signal from data system
debug_(max(pyfusion.DEBUG, debug), level=4, key='find_data')
try:
#get HJparams
channel = info['name']
outdata = np.zeros(1024 * 2 * 256 + 1)
channel_length = (len(outdata) - 1) / 2
# outdfile only needed for opt=1 (get data via temp file)
# with tempfile.NamedTemporaryFile(prefix="pyfusion_") as outdfile:
ierror, getrets = gethjdata.gethjdata(shot,
channel_length,
info['name'],
verbose=VERBOSE,
opt=1,
ierror=2,
outdata=outdata,
outname='')
if ierror != 0:
raise LookupError('data not found for {s}:{c}'.format(
s=shot, c=channel))
ch = Channel(info['name'], Coords('dummy', (0, 0, 0)))
# timebase in secs (was ms in raw data)
dg = TimeseriesData(timebase=Timebase(1e-3 *
getrets[1::2]),
signal=Signal(getrets[2::2]),
channels=ch)
except exception as reason:
if debug > 0:
print('exception running gethjdata {r} {a}',
format(r=reason, a=reason.args))
dg = None
#break # give up and try next diagnostic
if dg is None: # messy - break doesn't do what I want?
valarr = None
else:
nd = 1 # initially only deal with single channels (HJ)
# get the column(s) of the array corresponding to the name
w = [0]
if (oper in 'sum,average,rms,max,min'.split(',')):
if oper == 'sum': op = np.sum
elif oper == 'average': op = np.average
elif oper == 'min': op = np.min
elif oper == 'std': op = np.std
else:
raise ValueError(
'operator {o} in {n} not known to get_basic_diagnostics'
.format(o=oper, n=info['name']))
# valarr = op(dg.data[:,nd+w],1)
valarr = op(dg.data[:, nd + w], 1)
else:
if len(w) != 1:
raise LookupError(
'Need just one instance of variable {0} in {1}'
.format(varname, dg.filename))
dg.data = dg.signal # fudge compatibility
if len(np.shape(dg.data)) != 1: # 2 for igetfile
raise LookupError(
'insufficient data for {0} in {1}'.format(
varname, dg.filename))
#valarr = dg.data[:,nd+w[0]]
#tim = dg.data[:,0] - delay
valarr = dg.signal
tim = dg.timebase
# fudge until we can gete the number of points
valarr = valarr[:np.argmax(tim)]
tim = tim[:np.argmax(tim)]
if oper == 'ddt': # derivative operator
valarr = np.diff(valarr) / (np.average(np.diff(tim)))
tim = (tim[0:-1] + tim[1:]) / 2.0
if oper == 'ddt2': # abd(ddw)*derivative operator
dw = np.diff(valarr) / (np.average(np.diff(tim)))
ddw = np.diff(dw) / (np.average(np.diff(tim)))
tim = tim[2:]
valarr = 4e-6 * dw[1:] * np.abs(ddw)
if (len(tim) < 10) or (np.std(tim) < 0.1):
raise ValueError('Insufficient points or degenerate'
'timebase data in {0}, {1}'.format(
varname, dg.filename))
valarr = (stineman_interp(times, tim, valarr))
w = np.where(times > max(tim))
valarr[w] = np.nan
if valarr is not None: vals.update({diag: valarr})
debug_(max(pyfusion.DEBUG, debug), level=5, key='interp')
return (vals)
def get_basic_diagnostics(diags=None,
shot=54196,
times=None,
delay=None,
exception=False,
debug=0):
""" return a list of np.arrays of normally numeric values for the
times given, for the given shot.
Will access server if env('IGETFILE') points to an exe, else accesses cache
"""
global lhd_summary
# if no exception given and we are not debugging
# note - exception=None is a valid entry, meaning tolerate no exceptions
# so the "default" we use is False
if exception == False and debug == 0: exception = Exception
if diags is None: diags = "<n_e19>,b_0,i_p,w_p,dw_pdt,dw_pdt2".split(',')
if len(np.shape(diags)) == 0: diags = [diags]
if delay is None: delay = get_delay(shot)
if times is None:
times = np.linspace(0, 4, 4000)
times = np.array(times)
vals = {}
# create an extra time array to allow a cross check
vals.update({'check_tm': times})
vals.update({'check_shot': np.zeros(len(times), dtype=np.int) + shot})
for diag in diags:
if diag not in file_info:
warn('diagnostic {0} not found in shot {1}'.format(diag, shot),
stacklevel=2)
vals.update({diag: np.nan + times})
else:
info = file_info[diag]
varname = info['name']
subfolder = info['format'].split('@')[0]
filepath = os.path.sep.join(
[localigetfilepath, subfolder, info['format']])
if ':' in varname: (oper, varname) = varname.split(':')
else: oper = None
if info['format'].find('.csv') > 0:
try:
test = lhd_summary.keys()
except:
csvfilename = acq_LHD + '/' + info['format']
if pyfusion.DBG() > 1:
print('looking for lhd summary in' + csvfilename)
if not os.path.exists(csvfilename):
csvfilename += ".bz2"
print('reloading {0}'.format(csvfilename))
lhd_summary = read_csv_data(csvfilename, header=3)
# should make this more formal - last shots
# from an 'extra' file, and finally, from shot info
if shot > 117000: # fudge to get latest data
lhd_summary = np.load(acq_LHD +
'/LHD_summary.npz')['LHD'].tolist()
print('loading newer shots from a separate file - fix-me')
# val = lhd_summary[varname][shot-70000] # not needed
# else:
val = lhd_summary[varname][shot]
valarr = np.double(val) + (times * 0)
else:
debug_(max(pyfusion.DBG(), debug), level=4, key='find_data')
try:
dg = igetfile(filepath, shot=shot, debug=debug - 1)
except IOError:
try:
dg = igetfile(filepath + '.bz2',
shot=shot,
debug=debug - 1)
except IOError:
try:
dg = igetfile(filepath + '.gz',
shot=shot,
debug=debug - 1)
except exception as details:
if debug > 0:
print('diag at {fp} not found'.format(
fp=filepath))
print(details, details.args)
dg = None
#break # give up and try next diagnostic
if dg is None: # messy - break doesn't do what I want?
valarr = None
else:
nd = dg.vardict['DimNo']
if nd != 1:
raise ValueError(
'Expecting a 1 D array in {0}, got {1}!'.format(
dg.filename, nd))
# pre re. w = np.where(np.array(dg.vardict['ValName'])==varname)[0]
matches = [
re.match(varname, nam) != None
for nam in dg.vardict['ValName']
]
w = np.where(np.array(matches) != False)[0]
# get the column(s) of the array corresponding to the name
if (oper in 'sum,average,rms,max,min'.split(',')):
if oper == 'sum': op = np.sum
elif oper == 'average': op = np.average
elif oper == 'min': op = np.min
elif oper == 'std': op = np.std
else:
raise ValueError(
'operator {o} in {n} not known to get_basic_diagnostics'
.format(o=oper, n=info['name']))
valarr = op(dg.data[:, nd + w], 1)
else:
if len(w) != 1:
raise LookupError(
'Need just one instance of variable {0} in {1}'
.format(varname, dg.filename))
if len(np.shape(dg.data)) != 2:
raise LookupError(
'insufficient data for {0} in {1}'.format(
varname, dg.filename))
valarr = dg.data[:, nd + w[0]]
tim = dg.data[:, 0] - delay
if oper == 'ddt': # derivative operator
valarr = np.diff(valarr) / (np.average(np.diff(tim)))
tim = (tim[0:-1] + tim[1:]) / 2.0
if oper == 'ddt2': # abd(ddw)*derivative operator
dw = np.diff(valarr) / (np.average(np.diff(tim)))
ddw = np.diff(dw) / (np.average(np.diff(tim)))
tim = tim[2:]
valarr = 4e-6 * dw[1:] * np.abs(ddw)
if (len(tim) < 10) or (np.std(tim) < 0.1):
raise ValueError('Insufficient points or degenerate'
'timebase data in {0}, {1}'.format(
varname, dg.filename))
valarr = (stineman_interp(times, tim, valarr))
w = np.where(times > max(tim))
valarr[w] = np.nan
if valarr != None: vals.update({diag: valarr})
debug_(max(pyfusion.DBG(), debug), level=5, key='interp')
return (vals)
def get_basic_diagnostics(diags=None, shot=54196, times=None, delay=None, exception=False, debug=0):
""" return a list of np.arrays of normally numeric values for the
times given, for the given shot.
"""
global lhd_summary
# if no exception given and we are not debugging
# note - exception=None is a valid entry, meaning tolerate no exceptions
# so the "default" we use is False
if exception==False and debug==0: exception=Exception
if diags == None: diags = "<n_e19>,b_0,i_p,w_p,dw_pdt,dw_pdt2".split(',')
if len(np.shape(diags)) == 0: diags = [diags]
if delay == None: delay = get_delay(shot)
if times == None:
times = np.linspace(0,4,4000)
times = np.array(times)
vals = {}
# create an extra time array to allow a cross check
vals.update({'check_tm':times})
vals.update({'check_shot':np.zeros(len(times),dtype=np.int)+shot})
for diag in diags:
if not(file_info.has_key(diag)):
warn('diagnostic {0} not found in shot {1}'.format(diag, shot),stacklevel=2)
vals.update({diag: np.nan + times})
else:
info = file_info[diag]
varname = info['name']
subfolder = info['format'].split('@')[0]
filepath = os.path.sep.join([localigetfilepath,subfolder,info['format']])
if ':' in varname: (oper,varname) = varname.split(':')
else: oper = None
if info['format'].find('.csv') > 0:
try:
test=lhd_summary.keys()
except:
print('reloading {0}'.format(info['format']))
lhd_summary = read_csv_data(acq_LHD+'/'+info['format'], header=3)
val = lhd_summary[varname][shot]
valarr = np.double(val)+(times*0)
else:
debug_(max(pyfusion.DEBUG, debug), level=4, key='find_data')
try:
dg = igetfile(filepath, shot=shot, debug=debug-1)
except IOError:
try:
dg = igetfile(filepath+'.bz2', shot=shot, debug=debug-1)
except IOError:
try:
dg = igetfile(filepath + '.gz', shot=shot, debug=debug-1)
except exception:
if debug>0: print('diag at {fp} not found'
.format(fp=filepath))
dg=None
#break # give up and try next diagnostic
if dg==None: # messy - break doesn't do what I want?
valarr=None
else:
nd=dg.vardict['DimNo']
if nd != 1:
raise ValueError(
'Expecting a 1 D array in {0}, got {1}!'
.format(dg.filename, nd))
# pre re. w = np.where(np.array(dg.vardict['ValName'])==varname)[0]
matches = [re.match(varname,nam)
!= None for nam in dg.vardict['ValName']]
w = np.where(np.array(matches) != False)[0]
# get the column(s) of the array corresponding to the name
if (oper in 'sum,average,rms,max,min'.split(',')):
if oper=='sum': op = np.sum
elif oper=='average': op = np.average
elif oper=='min': op = np.min
elif oper=='std': op = np.std
else: raise ValueError('operator {o} in {n} not known to get_basic_diagnostics'
.format(o=oper, n=info['name']))
valarr = op(dg.data[:,nd+w],1)
else:
if len(w) != 1:
raise LookupError(
'Need just one instance of variable {0} in {1}'
.format(varname, dg.filename))
if len(np.shape(dg.data))!=2:
raise LookupError(
'insufficient data for {0} in {1}'
.format(varname, dg.filename))
valarr = dg.data[:,nd+w[0]]
tim = dg.data[:,0] - delay
if oper == 'ddt': # derivative operator
valarr = np.diff(valarr)/(np.average(np.diff(tim)))
tim = (tim[0:-1] + tim[1:])/2.0
if oper == 'ddt2': # abd(ddw)*derivative operator
dw = np.diff(valarr)/(np.average(np.diff(tim)))
ddw = np.diff(dw)/(np.average(np.diff(tim)))
tim = tim[2:]
valarr = 4e-6 * dw[1:] * np.abs(ddw)
if (len(tim) < 10) or (np.std(tim)<0.1):
raise ValueError('Insufficient points or degenerate'
'timebase data in {0}, {1}'
.format(varname, dg.filename))
valarr = (stineman_interp(times, tim, valarr))
w = np.where(times > max(tim))
valarr[w] = np.nan
if valarr != None: vals.update({diag: valarr})
debug_(max(pyfusion.DEBUG, debug), level=5, key='interp')
return(vals)
def get_basic_diagnostics(diags=None, file_info=file_info, shot=54196, times=None, delay=None, exception=False, debug=0):
""" return a list of np.arrays of normally numeric values for the
times given, for the given shot.
Will access server if env('IGETFILE') points to an exe, else accesses cache
This is the first version to specifically allow for access through pyfusion.cfg
There are two types of access:
I/ single diag on its own timebase
II/ the original multi diag on a given timebase (i.e. that from flucstrcs)
Stage 1 puts the file_info into .cfg file just for I/ single diag access.
Ideally the file_info for II/ sho;d be in .cfg also.
For stage I/, we call it with a file_info dict constructed on the spot
as a dictionary with one just entry (for diags[0]).
"""
global lhd_summary
# if no exception given and we are not debugging
# note - exception=None is a valid entry, meaning tolerate no exceptions
# so the "default" we use is False
if exception==False and debug==0: exception=Exception
if diags is None: diags = "<n_e19>,b_0,i_p,w_p,dw_pdt,dw_pdt2".split(',')
if len(np.shape(diags)) == 0: diags = [diags]
if delay is None: delay = get_delay(shot)
if times is None:
if len(diags)>1:
times = np.linspace(0,4,4000) # this is a crude guess.
# else leave it None
else:
# make sure it is an array
times = np.array(times)
vals = {}
for diag in diags:
if not(diag in file_info):
warn('diagnostic {0} not found in shot {1}'.format(diag, shot),stacklevel=2)
vals.update({diag: np.nan + times})
else:
info = file_info[diag]
varname = info['name'] # refers to name for igetfile - can contain ':'
subfolder = info['format'].split('@')[0]
filepath = os.path.sep.join([localigetfilepath,subfolder,info['format']])
if ':' in varname: (oper,varname) = varname.split(':')
else: oper = None
if info['format'].find('.csv') > 0:
try:
test=list(lhd_summary.keys())
except:
csvfilename = acq_LHD+'/'+info['format']
if pyfusion.DBG() > 1: print('looking for lhd summary in' + csvfilename)
if not os.path.exists(csvfilename):
csvfilename += ".bz2"
print('reloading {0}'.format(csvfilename))
lhd_summary = read_csv_data(csvfilename, header=3)
# should make this more formal - last shots
# from an 'extra' file, and finally, from shot info
if shot>117000: # fudge to get latest data
lhd_summary = np.load(acq_LHD+'/LHD_summary.npz')['LHD'].tolist()
print('loading newer shots from a separate file - fix-me')
# val = lhd_summary[varname][shot-70000] # not needed
# else:
val = lhd_summary[varname][shot]
valarr = np.double(val)+(times*0)
else:
try: # now igetfile checks for .gz etc
dg = igetfile(filepath, shot=shot, debug=debug-1)
except exception as details:
if debug>0: print('diag at {fp} not found'
.format(fp=filepath))
print(details,details.args)
dg=None
#break # give up and try next diagnostic
if dg is None: # messy - break doesn't do what I want?
valarr=None
else:
nd=dg.vardict['DimNo']
if nd != 1:
raise ValueError(
'Expecting a 1 D array in {0}, got {1}!'
.format(dg.filename, nd))
# pre re. w = np.where(np.array(dg.vardict['ValName'])==varname)[0]
matches = [re.match(varname,nam)
!= None for nam in dg.vardict['ValName']]
w = np.where(np.array(matches) != False)[0]
# get the column(s) of the array corresponding to the name
if (oper in 'sum,average,rms,max,min'.split(',')):
if oper=='sum': op = np.sum
elif oper=='average': op = np.average
elif oper=='min': op = np.min
elif oper=='std': op = np.std
else: raise ValueError('operator {o} in {n} not known to get_basic_diagnostics'
.format(o=oper, n=info['name']))
valarr = op(dg.data[:,nd+w],1)
else:
if len(w) != 1:
raise LookupError(
'Need just one instance of variable {0} in {1}'
.format(varname, dg.filename))
if len(np.shape(dg.data))!=2:
raise LookupError(
'insufficient data for {0} in {1}'
.format(varname, dg.filename))
valarr = dg.data[:,nd+w[0]]
tim = dg.data[:,0] - delay
if oper == 'ddt': # derivative operator
valarr = np.diff(valarr)/(np.average(np.diff(tim)))
tim = (tim[0:-1] + tim[1:])/2.0
if oper == 'ddt2': # abd(ddw)*derivative operator
dw = np.diff(valarr)/(np.average(np.diff(tim)))
ddw = np.diff(dw)/(np.average(np.diff(tim)))
tim = tim[2:]
valarr = 4e-6 * dw[1:] * np.abs(ddw)
if (len(tim) < 10) or (np.std(tim)<0.1):
raise ValueError('Insufficient points or degenerate'
'timebase data in {0}, {1}'
.format(varname, dg.filename))
if times is not None:
debug_(max(pyfusion.DEBUG, debug), level=5, key='interp')
valarr = (stineman_interp(times, tim, valarr))
w = np.where(times > max(tim))
valarr[w] = np.nan
else:
times = tim
if valarr is not None: vals.update({diag: valarr})
# create an extra time array to allow a cross check
vals.update({'check_tm':times})
vals.update({'check_shot':np.zeros(len(times),dtype=np.int)+shot})
return(vals)
def get_basic_diagnostics(diags=None, shot=54196, times=None, delay=None, exception=False, debug=0):
""" return a list of np.arrays of normally numeric values for the
times given, for the given shot.
Will access server if env('IGETFILE') points to an exe, else accesses cache
"""
global HJ_summary
# if no exception given and we are not debugging
# note - exception=None is a valid entry, meaning tolerate no exceptions
# so the "default" we use is False
if exception==False and debug==0: exception=Exception
if diags is None: diags = "<n_e19>,b_0,i_p,w_p,dw_pdt,dw_pdt2".split(',')
if len(np.shape(diags)) == 0: diags = [diags]
# LHD only if delay is None: delay = get_delay(shot)
if times is None:
times = np.linspace(0,4,4000)
times = np.array(times)
vals = {}
# create an extra time array to allow a cross check
vals.update({'check_tm':times})
vals.update({'check_shot':np.zeros(len(times),dtype=np.int)+shot})
debug_(pyfusion.DEBUG,2,key='get_basic')
for diag in diags:
if not(diag in file_info):
warn('diagnostic {0} not found in shot {1}'.format(diag, shot),stacklevel=2)
vals.update({diag: np.nan + times})
debug_(pyfusion.DEBUG,2,key='get_basic')
else:
info = file_info[diag]
varname = info['name']
infofmt = info['format']
subfolder = infofmt.split('@')[0]
filepath = os.path.sep.join([localigetfilepath,subfolder,infofmt])
if ':' in varname: (oper,varname) = varname.split(':')
else: oper = None
if '(' in varname:
try:
left,right = varname.split('(')
varname,rest=right.split(')')
except:
raise ValueError('in expression {v} - parens?'.format(varname))
if infofmt.find('.npz') > 0:
try:
test=HJ_summary.keys()
except:
csvfilename = acq_HJ+'/'+infofmt
if pyfusion.DBG() > 1: print('looking for HeliotronJ summary in' + csvfilename)
print('reloading {0}'.format(csvfilename))
HJ_summary = np.load(csvfilename)
val = HJ_summary[varname][shot]
valarr = np.double(val)+(times*0)
elif 'get_static_params' in infofmt:
pdicts = eval(infofmt.format(shot=shot))
if len(pdicts)==0:
print('empty dictionary returned')
val = pdicts[varname]
valarr = np.double(val)+(times*0)
else: # read signal from data system
debug_(max(pyfusion.DEBUG, debug), level=4, key='find_data')
try:
#get HJparams
channel = info['name']
outdata=np.zeros(1024*2*256+1)
channel_length =(len(outdata)-1)/2
# outdfile only needed for opt=1 (get data via temp file)
# with tempfile.NamedTemporaryFile(prefix="pyfusion_") as outdfile:
ierror, getrets=gethjdata.gethjdata(shot,channel_length,
info['name'],
verbose=VERBOSE, opt=1,
ierror=2,
outdata=outdata, outname='')
if ierror != 0:
raise LookupError('data not found for {s}:{c}'.format(s=shot, c=channel))
ch = Channel(info['name'], Coords('dummy', (0,0,0)))
# timebase in secs (was ms in raw data)
dg = TimeseriesData(timebase=Timebase(1e-3 * getrets[1::2]),
signal=Signal(getrets[2::2]), channels=ch)
except exception as reason:
if debug>0:
print('exception running gethjdata {r} {a}', format(r=reason, a=reason.args))
dg=None
#break # give up and try next diagnostic
if dg is None: # messy - break doesn't do what I want?
valarr=None
else:
nd = 1 # initially only deal with single channels (HJ)
# get the column(s) of the array corresponding to the name
w = [0]
if (oper in 'sum,average,rms,max,min'.split(',')):
if oper=='sum': op = np.sum
elif oper=='average': op = np.average
elif oper=='min': op = np.min
elif oper=='std': op = np.std
else: raise ValueError('operator {o} in {n} not known to get_basic_diagnostics'
.format(o=oper, n=info['name']))
# valarr = op(dg.data[:,nd+w],1)
valarr = op(dg.data[:,nd+w],1)
else:
if len(w) != 1:
raise LookupError(
'Need just one instance of variable {0} in {1}'
.format(varname, dg.filename))
dg.data = dg.signal # fudge compatibility
if len(np.shape(dg.data))!=1: # 2 for igetfile
raise LookupError(
'insufficient data for {0} in {1}'
.format(varname, dg.filename))
#valarr = dg.data[:,nd+w[0]]
#tim = dg.data[:,0] - delay
valarr = dg.signal
tim = dg.timebase
# fudge until we can gete the number of points
valarr = valarr[:np.argmax(tim)]
tim = tim[:np.argmax(tim)]
if oper == 'ddt': # derivative operator
valarr = np.diff(valarr)/(np.average(np.diff(tim)))
tim = (tim[0:-1] + tim[1:])/2.0
if oper == 'ddt2': # abd(ddw)*derivative operator
dw = np.diff(valarr)/(np.average(np.diff(tim)))
ddw = np.diff(dw)/(np.average(np.diff(tim)))
tim = tim[2:]
valarr = 4e-6 * dw[1:] * np.abs(ddw)
if (len(tim) < 10) or (np.std(tim)<0.1):
raise ValueError('Insufficient points or degenerate'
'timebase data in {0}, {1}'
.format(varname, dg.filename))
valarr = (stineman_interp(times, tim, valarr))
w = np.where(times > max(tim))
valarr[w] = np.nan
if valarr is not None: vals.update({diag: valarr})
debug_(max(pyfusion.DEBUG, debug), level=5, key='interp')
return(vals)
def make_mask(NA, norm_passband, norm_stopband, input_data, taper):
""" works well now, except that the stopband is adjusted to be
symmetric about the passband (take the average of the differences
The problem with crashes (zero mask) was solved by shifting the
mask before and after integrating, also a test for aliasing (on the
mask before integration).
"""
mask = np.zeros(NA)
# define the 4 key points
# /npblow-------------npbhi\
# ___nsbl/ \nsbhi____
n_sb_low = int(norm_stopband[0] * NA / 2)
n_pb_low = int(norm_passband[0] * NA / 2)
n_pb_hi = int(norm_passband[1] * NA / 2)
n_sb_hi = int(norm_stopband[1] * NA / 2)
dt = float(np.average(np.diff(input_data.timebase)))
if n_sb_hi >= len(mask):
raise ValueError('Filter frequency too high for data - units '
'problem? - sample spacing is {dt:.2g}'.format(dt=dt))
# twid is the transition width, and should default so that the sloped part is the same width as the flat?
# !!! twid is not an input - !!!! doesn't do that yet.
# make the transition width an even number, and the larger of the two
# need to pull this code out and be sure it works.
twid = 2 * (1 + max(n_pb_low - n_sb_low, n_sb_hi - n_pb_hi) // 2)
if (twid > (n_pb_low - n_sb_low) * 3) or (twid > (n_sb_hi - n_pb_hi) * 3):
print(
'*********** Warning - unbalanced cutoff rate between high and low end'
' will cause the cutoff rates to be equalised widening one and reducing the other'
' difference between stop and pass bands should be similar ar both ends.'
)
if (twid < 4): # or (n_sb_low < 0): #< not requ since fixed
if taper == 2:
raise ValueError(
'taper 2 requires a bigger margin between stop and pass')
elif taper is None:
warn('defaulting taper to 1 as band edges are sharp: twid={twid}'.
format(twid=twid))
taper = 1
else:
if taper is None:
taper = 2
if taper == 1:
# _____
# / \
# / \
# ______/ \___
# want 0 at sb low and sb high, 1 at pb low and pb high
# present code does not quite do this.
# try to prevent zero width or very narrow (DC only) filters.
if n_sb_low < 0: n_sb_low = 0
if n_pb_low < 0: n_pb_low = 0
if n_pb_hi < 1: n_pb_hi = 1
if n_sb_hi <= n_pb_hi: n_sb_hi = n_pb_hi + 1
for n in range(n_sb_low, n_pb_low + 1):
if n_sb_low == n_pb_low: # allow for pass=stop on low side
mask[n] = 1.
else:
mask[n] = float(n - n_sb_low) / (n_pb_low - n_sb_low
) # trapezoid
for n in range(n_pb_hi, n_sb_hi + 1):
mask[n] = float(n_sb_hi - n) / (n_sb_hi - n_pb_hi) # trapezoid
for n in range(n_pb_low, n_pb_hi + 1):
mask[n] = 1
elif taper == 2:
# Note - must symmetrise (so that cumsum works)
# _
# / \
# | |
# ______/ \___
# want 0 at sb low and sb high, 1 at pb low and pb high
# this means that the peak of the mask before integration is halfway between sb_low and pb_low
# and pb_low - sb_low is an even number
# present code does not quite do this.
n_sb_low = n_pb_low - twid # sacrifice the stop band, not the pass
n_sb_hi = n_pb_hi + twid
low_mid = n_pb_low - twid // 2
high_mid = n_pb_hi + twid // 2
for n in range(n_sb_low, low_mid):
mask[n] = float(n - n_sb_low) / (low_mid - 1 - n_sb_low
) # trapezoid
mask[2 * low_mid - n - 1] = mask[n] #down ramp - repeat max
#wid_up = n_sb_hi - n_pb_hi
for n in range(n_pb_hi, high_mid): # negative tri
mask[n] = float(n_pb_hi - n) / (high_mid - n_pb_hi - 1
) # trapezoid
mask[2 * high_mid - n - 1] = mask[n]
before_integration = mask
# after running filters.py, this should be OK
# make_mask(512, [0.8,.93], [0.9,.98],dat,2)
# but changing 0.98 to 0.99 will give aliasing error.
if np.max(np.abs(mask[NA // 2 - 4:NA // 2 + 4])) > 0:
raise ValueError('mask aliasing error')
# note: ifftshift is only different for an odd data length
# the fftshifts were necessary to avoid weirdness if the
# stopband went below zero freq.
mask = np.fft.ifftshift(np.cumsum(np.fft.fftshift(mask))) # integrate
if pyfusion.DBG() > 1:
nonr = 0.5 / dt
fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.plot(np.arange(len(mask)) / dt / float(NA), mask, '.-')
ax1.plot(np.arange(len(mask)) / dt / float(NA), before_integration)
ax1.set_xlabel(
'real freq. units, (norm on top scale), npoints={NA}, norm/real = {nonr}'
.format(NA=NA, nonr=nonr))
ax2 = ax1.twiny()
# this is my hack - it should be OK, but may be a little out
ax2.set_xlim(np.array(ax1.get_xlim()) / nonr)
fig.suptitle(
'mask before normalisation - twid={twid}'.format(twid=twid))
plt.show(0)
if np.max(mask) == 0:
raise ValueError(
'zero mask, '
'norm_passband = {pb}, norm_stopband={sb}, taper {t}'.format(
pb=norm_passband, sb=norm_stopband, t=taper))
mask = mask / np.max(mask)
# reflection only required for complex data
# this even and odd is not totally thought through...but it seems OK
if np.mod(NA, 2) == 0: mask[:NA / 2:-1] = mask[1:(NA / 2)] # even
else: mask[:1 + NA / 2:-1] = mask[1:(NA / 2)] # odd
return (mask)
def filter_fourier_bandpass(input_data, passband, stopband, taper=None, debug=None):
"""
Note: Is MUCH (2.2x faster) more efficient to use real ffts, (implemented April)
Use a Fourier space taper/tophat or pseudo gaussian filter to perform
narrowband filtering (much narrower than butterworth).
Problem is that bursts may generate ringing. (should be better with taper=2)
>>> tb = dummytb(np.linspace(0,20,512))
>>> w = 2*np.pi* 1 # 1 Hertz
>>> dat = dummysig(tb,np.sin(w*tb.timebase)*(tb.timebase<np.max(tb.timebase)/3))
>>> fop = filter_fourier_bandpass(dat,[0.9,1.1],[0.8,1.2],debug=2).signal[0]
"""
if debug == None: debug = pyfusion.DEBUG
# normalising makes it easier to think about - also for But'w'h
norm_passband = input_data.timebase.normalise_freq(np.array(passband))
norm_stopband = input_data.timebase.normalise_freq(np.array(stopband))
NS = len(input_data.signal[0])
NA = next_nice_number(NS)
# take a little more to speed up FFT
mask = np.zeros(NA)
# define the 4 key points
# /npblow-------------npbhi\
# ___nsbl/ \nsbhi____
n_sb_low = int(norm_stopband[0]*NA/2)
n_pb_low = int(norm_passband[0]*NA/2)
n_pb_hi = int(norm_passband[1]*NA/2)
n_sb_hi = int(norm_stopband[1]*NA/2)
wid = max(n_pb_low - n_sb_low,n_sb_hi - n_pb_hi)
if wid < 4:
if taper == 2:
raise ValueError(
'taper 2 requres a bigger margin between stop and pass')
elif taper == None:
warn('defaulting taper to 1 as band edges are sharp')
taper = 1
else:
if taper == None:
taper = 2
if taper==1:
for n in range(n_sb_low,n_pb_low+1):
if n_pb_low == n_sb_low: # allow for pass=stop on low side
mask[n]=1.
else:
mask[n] = float(n - n_sb_low)/(n_pb_low - n_sb_low) # trapezoid
for n in range(n_pb_hi,n_sb_hi+1):
mask[n] = float(n_sb_hi - n)/(n_sb_hi - n_pb_hi) # trapezoid
for n in range(n_pb_low,n_pb_hi+1):
mask[n] = 1
elif taper == 2:
# symmetrise (so that cumsum works)
n_pb_low = n_sb_low+wid
n_sb_hi = n_pb_hi+wid
for n in range(n_sb_low,n_pb_low+1):
mask[n] = float(n - n_sb_low)/(n_pb_low - n_sb_low) # trapezoid
mask[2*n_pb_low-n+1] = mask[n] #down ramp
wid_up = n_sb_hi - n_pb_hi
for n in range(n_pb_hi,n_sb_hi+1):
mask[n] = -float(n_sb_hi - n)/(n_sb_hi - n_pb_hi) # trapezoid
mask[2*n_pb_hi - n - 1] = mask[n]
mask = np.cumsum(mask) # integrate
mask = mask/np.max(mask)
# reflection only required for complex data
# this even and odd is not totally thought through...but it seems OK
if np.mod(NA,2)==0: mask[:NA/2:-1] = mask[1:(NA/2)] # even
else: mask[:1+NA/2:-1] = mask[1:(NA/2)] # odd
output_data = copy.deepcopy(input_data) # was output_data = input_data
if (pyfusion.fft_type == 'fftw3'):
# should migrate elsewhere
import pyfftw
tdtype = np.float32
fdtype = np.complex64
# this could be useful to cache.
simd_align = pyfftw.simd_alignment # 16 at the moment.
tdom = pyfftw.n_byte_align(np.zeros(NA,dtype=tdtype), simd_align)
FT = pyfftw.n_byte_align_empty(NA/2+1, simd_align, fdtype)
ids = [[id(tdom),id(FT)]] # check to see if it moves out of alignment
fwd = pyfftw.FFTW(tdom, FT, direction='FFTW_FORWARD',
**pyfusion.fftw3_args)
rev = pyfftw.FFTW(FT, tdom, direction='FFTW_BACKWARD',
**pyfusion.fftw3_args)
else:
tdtype = np.float32
tdom = np.zeros(NA,dtype=tdtype)
# example of tuning
#pyfusion.fftw3_args= {'planning_timelimit': 50.0, 'threads':1, 'flags':['FFTW_MEASURE']}
for i,s in enumerate(output_data.signal):
#if len(output_data.signal) == 1: print('bug for a single signal')
#time run -i pyfusion/examples/plot_svd.py "dev_name='LHD'" start_time=.497 "normalise='r'" shot_number=90091 numpts=512 diag_name=MP2010HMPno612 "filter=dict(centre=8e3,bw=5e3,taper=2)" plot_mag=1 plot_phase=1 separate=1 closed=0 time_range=[0.0000,4.]
# 4.5 cf 15.8diag_name=MP2010HMPno612, time_range=[0.0000,2.80000]
# 0, 4.194304 2**21 samples, 21.8 cf 6.8 1thr
# (0,2)secs 90091 =2000000 samples 17 np, 5.73 2thread, nosimd, 6.1 1thread (mem bw?) 3.2 sec no filt
# note - the above are on an intermeittently loaded E4300 2 processor, below on 4 core 5/760
# 0, 4.194304 2**21 samples, 10.9 cf 3.16 thr2 3.47 1thr and 2.0 secs no filter
# for 17 fft/ifft takes about 1.16 sec 2 threads - should be (27.5ms+28.6)*17 = 952ms (14.2 2thr) OK
# duplicate the fft execute lines 4.3(3.47) 2thr 3.7(3.16) extra 810ms (expect 14.2ms * 2 * 17) =482
# the difference between 2 and 1thr should be 14*2*17 ms 500ms.
# orignall - 90ms/channel extra in reverse trasnform - maybe the 50 sec limit stopped optimization
# next _nice: 5.74 for 10 sec lenny
# E4300: 1thr 9.3 (39np) for 10 sec 90091; 5.5 for 4 sec (19.6 np)
if (pyfusion.fft_type == 'fftw3'): # fftw3 nosim, no thread 2.8s cf 10s
tdom[0:NS]=s # indexed to make sure tdom stays put
if NS != NA: tdom[NS:]=0.
fwd.execute()
FT[:] = FT * mask[0:NA/2+1] # 12ms
rev.execute()
output_data.signal[i] = tdom[0:NS]
ids.append([id(tdom),id(FT)])
else: # default to numpy
tdom[0:NS] = s
FT = np.fft.fft(tdom)
IFT = np.fft.ifft(mask*FT)
if np.max(np.abs(IFT.imag)) > 1e-6*np.max(np.abs(IFT.real)):
pyfusion.logger.warning("inverse fft imag part > 1e-6")
output_data.signal[i] = IFT.real[0:NS]
if debug>2: print('ids of fftw3 input and output: {t}'.format(t=ids))
if debug>0:
import pylab as pl
pl.figure()
pl.plot(mask,'r.-',label='mask')
pl.plot(np.abs(FT)/len(mask),label='FT')
pl.plot(input_data.signal[0],label='input')
pl.plot(output_data.signal[0],label='output')
pl.legend()
pl.show()
debug_(debug, 2, key='filter_fourier')
if np.max(mask) == 0: raise ValueError('Filter blocks all signals')
return output_data
# has [...]. Figure out why. (...probably because original signal
# uses a build_signal function)
if len(dim) == 1:
dim = dim[0]
timebase = Timebase(dim)
output_data = TimeseriesData(timebase=timebase, signal=signal, channels=ch)
output_data.config_name = fetcher.config_name #bdb config_name fix
output_data.meta.update({'shot':fetcher.shot})
return output_data
# Don't import in the header, so that .npz files can be used - e.g. JSPF_tutorial example1
try:
import MDSplus
except ImportError:
warn(' No MIT MDSplus software found - will only work on local .npz data'
' Try easy_install mdsplus, or see the ubuntu/redhat... mdsplus distros '
'http://www.mdsplus.org '
'if you wish to access native MDSplus data')
class MDSPlusDataFetcher(BaseDataFetcher):
"""Determine which access mode should be used, and fetch the MDSplus data."""
def setup(self):
self.mds_path_components = get_tree_path(self.mds_path)
if hasattr(self.acq, '%s_path' %self.mds_path_components['tree']):
self.tree = MDSplus.Tree(self.mds_path_components['tree'],
self.shot)
self.fetch_mode = 'local_path_mode' # this refers to access by _path e.g. h1data_path
# bdb wants to call it local_path_mode, but maybe
# TestNoSQLTestDeviceGetdata fails
elif self.acq.server_mode == 'mds':
def filter_fourier_bandpass(input_data,
passband,
stopband,
taper=None,
debug=None):
"""
Note: Is MUCH (2.2x faster) more efficient to use real ffts, (implemented April)
Use a Fourier space taper/tophat or pseudo gaussian filter to perform
narrowband filtering (much narrower than butterworth).
Problem is that bursts may generate ringing. (should be better with taper=2)
>>> tb = dummytb(np.linspace(0,20,512))
>>> w = 2*np.pi* 1 # 1 Hertz
>>> dat = dummysig(tb,np.sin(w*tb.timebase)*(tb.timebase<np.max(tb.timebase)/3))
>>> fop = filter_fourier_bandpass(dat,[0.9,1.1],[0.8,1.2],debug=2).signal[0]
"""
if debug == None: debug = pyfusion.DEBUG
# normalising makes it easier to think about - also for But'w'h
norm_passband = input_data.timebase.normalise_freq(np.array(passband))
norm_stopband = input_data.timebase.normalise_freq(np.array(stopband))
NS = len(input_data.signal[0])
NA = next_nice_number(NS)
# take a little more to speed up FFT
mask = np.zeros(NA)
# define the 4 key points
# /npblow-------------npbhi\
# ___nsbl/ \nsbhi____
n_sb_low = int(norm_stopband[0] * NA / 2)
n_pb_low = int(norm_passband[0] * NA / 2)
n_pb_hi = int(norm_passband[1] * NA / 2)
n_sb_hi = int(norm_stopband[1] * NA / 2)
wid = max(n_pb_low - n_sb_low, n_sb_hi - n_pb_hi)
if wid < 4:
if taper == 2:
raise ValueError(
'taper 2 requres a bigger margin between stop and pass')
elif taper == None:
warn('defaulting taper to 1 as band edges are sharp')
taper = 1
else:
if taper == None:
taper = 2
if taper == 1:
for n in range(n_sb_low, n_pb_low + 1):
if n_pb_low == n_sb_low: # allow for pass=stop on low side
mask[n] = 1.
else:
mask[n] = float(n - n_sb_low) / (n_pb_low - n_sb_low
) # trapezoid
for n in range(n_pb_hi, n_sb_hi + 1):
mask[n] = float(n_sb_hi - n) / (n_sb_hi - n_pb_hi) # trapezoid
for n in range(n_pb_low, n_pb_hi + 1):
mask[n] = 1
elif taper == 2:
# symmetrise (so that cumsum works)
n_pb_low = n_sb_low + wid
n_sb_hi = n_pb_hi + wid
for n in range(n_sb_low, n_pb_low + 1):
mask[n] = float(n - n_sb_low) / (n_pb_low - n_sb_low) # trapezoid
mask[2 * n_pb_low - n + 1] = mask[n] #down ramp
wid_up = n_sb_hi - n_pb_hi
for n in range(n_pb_hi, n_sb_hi + 1):
mask[n] = -float(n_sb_hi - n) / (n_sb_hi - n_pb_hi) # trapezoid
mask[2 * n_pb_hi - n - 1] = mask[n]
mask = np.cumsum(mask) # integrate
mask = mask / np.max(mask)
# reflection only required for complex data
# this even and odd is not totally thought through...but it seems OK
if np.mod(NA, 2) == 0: mask[:NA / 2:-1] = mask[1:(NA / 2)] # even
else: mask[:1 + NA / 2:-1] = mask[1:(NA / 2)] # odd
output_data = copy.deepcopy(input_data) # was output_data = input_data
if (pyfusion.fft_type == 'fftw3'):
# should migrate elsewhere
import pyfftw
tdtype = np.float32
fdtype = np.complex64
# this could be useful to cache.
simd_align = pyfftw.simd_alignment # 16 at the moment.
tdom = pyfftw.n_byte_align(np.zeros(NA, dtype=tdtype), simd_align)
FT = pyfftw.n_byte_align_empty(NA / 2 + 1, simd_align, fdtype)
ids = [[id(tdom), id(FT)]] # check to see if it moves out of alignment
fwd = pyfftw.FFTW(tdom,
FT,
direction='FFTW_FORWARD',
**pyfusion.fftw3_args)
rev = pyfftw.FFTW(FT,
tdom,
direction='FFTW_BACKWARD',
**pyfusion.fftw3_args)
else:
tdtype = np.float32
tdom = np.zeros(NA, dtype=tdtype)
# example of tuning
#pyfusion.fftw3_args= {'planning_timelimit': 50.0, 'threads':1, 'flags':['FFTW_MEASURE']}
for i, s in enumerate(output_data.signal):
#if len(output_data.signal) == 1: print('bug for a single signal')
#time run -i pyfusion/examples/plot_svd.py "dev_name='LHD'" start_time=.497 "normalise='r'" shot_number=90091 numpts=512 diag_name=MP2010HMPno612 "filter=dict(centre=8e3,bw=5e3,taper=2)" plot_mag=1 plot_phase=1 separate=1 closed=0 time_range=[0.0000,4.]
# 4.5 cf 15.8diag_name=MP2010HMPno612, time_range=[0.0000,2.80000]
# 0, 4.194304 2**21 samples, 21.8 cf 6.8 1thr
# (0,2)secs 90091 =2000000 samples 17 np, 5.73 2thread, nosimd, 6.1 1thread (mem bw?) 3.2 sec no filt
# note - the above are on an intermeittently loaded E4300 2 processor, below on 4 core 5/760
# 0, 4.194304 2**21 samples, 10.9 cf 3.16 thr2 3.47 1thr and 2.0 secs no filter
# for 17 fft/ifft takes about 1.16 sec 2 threads - should be (27.5ms+28.6)*17 = 952ms (14.2 2thr) OK
# duplicate the fft execute lines 4.3(3.47) 2thr 3.7(3.16) extra 810ms (expect 14.2ms * 2 * 17) =482
# the difference between 2 and 1thr should be 14*2*17 ms 500ms.
# orignall - 90ms/channel extra in reverse trasnform - maybe the 50 sec limit stopped optimization
# next _nice: 5.74 for 10 sec lenny
# E4300: 1thr 9.3 (39np) for 10 sec 90091; 5.5 for 4 sec (19.6 np)
if (pyfusion.fft_type == 'fftw3'
): # fftw3 nosim, no thread 2.8s cf 10s
tdom[0:NS] = s # indexed to make sure tdom stays put
if NS != NA: tdom[NS:] = 0.
fwd.execute()
FT[:] = FT * mask[0:NA / 2 + 1] # 12ms
rev.execute()
output_data.signal[i] = tdom[0:NS]
ids.append([id(tdom), id(FT)])
else: # default to numpy
tdom[0:NS] = s
FT = np.fft.fft(tdom)
IFT = np.fft.ifft(mask * FT)
if np.max(np.abs(IFT.imag)) > 1e-6 * np.max(np.abs(IFT.real)):
pyfusion.logger.warning("inverse fft imag part > 1e-6")
output_data.signal[i] = IFT.real[0:NS]
if debug > 2: print('ids of fftw3 input and output: {t}'.format(t=ids))
if debug > 0:
import pylab as pl
pl.figure()
pl.plot(mask, 'r.-', label='mask')
pl.plot(np.abs(FT) / len(mask), label='FT')
pl.plot(input_data.signal[0], label='input')
pl.plot(output_data.signal[0], label='output')
pl.legend()
pl.show()
debug_(debug, 2, key='filter_fourier')
if np.max(mask) == 0: raise ValueError('Filter blocks all signals')
return output_data
raise ValueError('sel and csel must have the same length')
try:
cldata
if oldclusterfile != clusterfile:
1/0
print('using previous clusterfile data'
' - set oldclusterfile=None to prevent')
except:
print('loading cluster data from {cf}'.format(cf=clusterfile))
cldata = np.load(clusterfile)
oldclusterfile = clusterfile
oldsel = sel
for k in cldata.keys(): exec("{v}=cldata['{k}']".format(v=k,k=k))
if np.shape(sel) != np.shape(oldsel) or (sel != oldsel).any():
warn('sel value tried to change from {o} to {n} - i.e. cldata has different sel, '
.format(o=oldsel, n=sel))
sel = oldsel
if csel is not None:
subset=subset[:,csel]
# this contrivance allows us to test on uniformly distributed phases
if uniform_random is not None:
print('evaluating {ur}'.format(ur=uniform_random))
phases = eval(uniform_random)
shot = np.array(np.shape(phases)[0]*[shot[0]])
freq = np.array(np.shape(phases)[0]*[freq[0]])
frlow = np.array(np.shape(phases)[0]*[0])
frhigh = np.array(np.shape(phases)[0]*[9e9])
t_mid = np.array(np.shape(phases)[0]*[t_mid[-1]])
cc5 = [] # cluster centres
for cl in cls:
raise ValueError('sel and csel must have the same length')
try:
cldata
if oldclusterfile != clusterfile:
1/0
print('using previous clusterfile data'
' - set oldclusterfile=None to prevent')
except:
print('loading cluster data from {cf}'.format(cf=clusterfile))
cldata = np.load(clusterfile)
oldclusterfile = clusterfile
oldsel = sel
for k in cldata.keys(): exec("{v}=cldata['{k}']".format(v=k,k=k))
if np.shape(sel) != np.shape(oldsel) or (sel != oldsel).any():
warn('sel value tried to change from {o} to {n} - i.e. cldata has different sel, '
.format(o=oldsel, n=sel))
sel = oldsel
if csel is not None:
subset=subset[:,csel]
# this contrivance allows us to test on uniformly distributed phases
if uniform_random is not None:
print('evaluating {ur}'.format(ur=uniform_random))
phases = eval(uniform_random)
shot = np.array(np.shape(phases)[0]*[shot[0]])
freq = np.array(np.shape(phases)[0]*[freq[0]])
frlow = np.array(np.shape(phases)[0]*[0])
frhigh = np.array(np.shape(phases)[0]*[9e9])
t_mid = np.array(np.shape(phases)[0]*[t_mid[-1]])
cc5 = [] # cluster centres
for cl in cls:
def make_title(formatstr,
input_data,
channum=None,
at_dict={},
min_length=3,
raw_names=False):
""" Return a string describing the shot number, channel name etc using
a formatstr which refers to items in a dictionary (at_dict), assembled in
this routine, based on input_data and an optional dictionary which
contains anything not otherwise available in input_data
"""
## at_dict.update({'shot': input_data.meta['shot']})
exception = () if pyfusion.DBG() > 3 else Exception
try:
at_dict.update(input_data.meta) # this gets all of it!
if channum is None:
name = ''
else:
if isinstance(input_data.channels, list):
chan = input_data.channels[channum]
else:
chan = input_data.channels
if raw_names:
name = chan.name
else:
name = chan.config_name
try: # remove leading W7X_ LHD_ etc from channel labels.
if 'device' in input_data.params.values()[0]['params']:
device = input_data.params.values()[0]['params']['device']
name = name.replace(device + '_', '')
except:
pass
at_dict.update({'units': chan.units})
at_dict.update({'name': name})
# replace internal strings of non-numbers with a single . a14_input03 -> 14.03
short_name = ''
last_was_number = False
discarded = ''
debug_(pyfusion.DEBUG, 4, key='make_title')
for c in name: # start from the first char
if c >= '0' and c <= '9':
short_name += c
last_was_number = True
else:
if last_was_number: short_name += '.'
else: discarded += c
last_was_number = False
if len(short_name) <= min_length:
# if it fits, have the lot
if len(name) < 8:
short_name = name
# else allow 4 more chars - makes about 6-8 chars
else:
short_name = discarded[-4:] + short_name
at_dict.update({'short_name': short_name})
return (formatstr.format(**at_dict))
except exception as ex:
warn('in make_title for format="%s", at_dict=%s' %
(formatstr, at_dict),
exception=ex)
return ('')
if len(dim) == 1:
dim = dim[0]
timebase = Timebase(dim)
output_data = TimeseriesData(timebase=timebase, signal=signal, channels=ch)
output_data.config_name = fetcher.config_name #bdb config_name fix
output_data.meta.update({'shot': fetcher.shot})
return output_data
# Don't import in the header, so that .npz files can be used - e.g. JSPF_tutorial example1
try:
import MDSplus
except ImportError:
warn(
' No MIT MDSplus software found - will only work on local .npz data'
' Try easy_install mdsplus, or see the ubuntu/redhat... mdsplus distros '
'http://www.mdsplus.org '
'if you wish to access native MDSplus data')
class MDSPlusDataFetcher(BaseDataFetcher):
"""Determine which access mode should be used, and fetch the MDSplus data."""
def setup(self):
self.mds_path_components = get_tree_path(self.mds_path)
if hasattr(self.acq, '%s_path' % self.mds_path_components['tree']):
self.tree = MDSplus.Tree(self.mds_path_components['tree'],
self.shot)
self.fetch_mode = 'local_path_mode' # this refers to access by _path e.g. h1data_path
# bdb wants to call it local_path_mode, but maybe
# TestNoSQLTestDeviceGetdata fails
for idx in ord:
t_seg = ord_segs[idx]
if np.isnan(np.sum(t_seg.signal)): # if signal is missing, skip
continue
if max_bands > 1: # keep it simple if one band
(ipk, fpk, apk) = find_signal_spectral_peaks(t_seg.timebase,
t_seg.signal[0],
minratio=0.1)
w_in_range = np.where(fpk < fmax)[0]
(ipk, fpk, apk) = (ipk[w_in_range], fpk[w_in_range],
apk[w_in_range])
if len(ipk) > max_bands:
if pyfusion.DBG() > 0:
# in python3 at least, the traceback info is
# out one level.
warn('too many peaks - choosing largest {mb}'.format(
mb=max_bands))
fpk = np.sort(fpk[np.argsort(apk)[-(max_bands + 1):]])
(lfs, hfs) = subdivide_interval(np.append(fpk, fmax),
debug=0,
overlap=(df / 2, df * 2))
elif max_bands == 1: # these two compare with and without fourier filter
(lfs, hfs) = ([0], [fmax * 0.99])
else:
(lfs, hfs) = ([0], [fmax])
for i in range(len(lfs)):
(frlow, frhigh) = (lfs[i], hfs[i])
if max_bands > 0:
f_seg = t_seg.filter_fourier_bandpass(
if fmax is None:
fmax = 0.5/np.average(np.diff(ord_segs[0].timebase)) - df
timeinfo('beginning flucstruc loop')
for idx in ord:
t_seg = ord_segs[idx]
if max_bands>1: # keep it simple if one band
(ipk, fpk, apk) = find_signal_spectral_peaks(t_seg.timebase, t_seg.signal[0],minratio=0.1)
w_in_range = np.where(fpk < fmax)[0]
(ipk, fpk, apk) = (ipk[w_in_range], fpk[w_in_range], apk[w_in_range])
if len(ipk)>max_bands:
if pyfusion.DBG() > 0:
# in python3 at least, the traceback info is
# out one level.
warn('too many peaks - choosing largest {mb}'.format(mb=max_bands))
fpk = np.sort(fpk[np.argsort(apk)[-(max_bands+1):]])
(lfs, hfs) = subdivide_interval(np.append(fpk, fmax), debug=0, overlap=(df/2,df*2))
elif max_bands == 1: # these two compare with and without fourier filter
(lfs,hfs) = ([0],[fmax*0.99])
else:
(lfs,hfs) = ([0],[fmax])
for i in range(len(lfs)):
(frlow,frhigh)= (lfs[i],hfs[i])
if max_bands > 0:
f_seg = t_seg.filter_fourier_bandpass(
pass
ord_segs = []
for ii, t_seg in enumerate(sections):
ord_segs.append(t_seg)
ord = argsort([average(t_seg.timebase) for t_seg in ord_segs])
if fmax == None:
fmax = 0.5 / np.average(np.diff(ord_segs[0].timebase)) - df
for idx in ord:
t_seg = ord_segs[idx]
(ipk, fpk, apk) = find_signal_spectral_peaks(t_seg.timebase, t_seg.signal[0], minratio=0.1)
w_in_range = np.where(fpk < fmax)[0]
(ipk, fpk, apk) = (ipk[w_in_range], fpk[w_in_range], apk[w_in_range])
if len(ipk) > max_bands:
warn("too many peaks - reducing to {mb}".format(mb=max_bands))
fpk = np.sort(fpk[np.argsort(apk)[-(max_bands + 1) :]])
(lfs, hfs) = subdivide_interval(np.append(fpk, fmax), debug=0, overlap=(df / 2, df * 2))
for i in range(len(lfs)):
(frlow, frhigh) = (lfs[i], hfs[i])
f_seg = t_seg.filter_fourier_bandpass([lfs[i], hfs[i]], [lfs[i] - df, hfs[i] + df])
fs_set = f_seg.flucstruc(method=method, separate=separate)
for fs in fs_set:
if count == 0:
# show history if info says to, and avoid starting line with a digit
if info > 0:
print("< " + fs.history.replace("\n201", "\n< 201"))
print(
"Shot time SVS freq Amp a12 p H frlow frhigh {np:2d} Phases".format(
np=len(fs.dphase)
except: pass
ord_segs = []
for ii,t_seg in enumerate(sections):
ord_segs.append(t_seg)
ord = argsort([average(t_seg.timebase) for t_seg in ord_segs])
if fmax == None:
fmax = 0.5/np.average(np.diff(ord_segs[0].timebase)) - df
for idx in ord:
t_seg = ord_segs[idx]
(ipk, fpk, apk) = find_signal_spectral_peaks(t_seg.timebase, t_seg.signal[0],minratio=0.1)
w_in_range = np.where(fpk < fmax)[0]
(ipk, fpk, apk) = (ipk[w_in_range], fpk[w_in_range], apk[w_in_range])
if len(ipk)>max_bands:
warn('too many peaks - reducing to {mb}'.format(mb=max_bands))
fpk = np.sort(fpk[np.argsort(apk)[-(max_bands+1):]])
(lfs, hfs) = subdivide_interval(np.append(fpk, fmax), debug=0, overlap=(df/2,df*2))
for i in range(len(lfs)):
(frlow,frhigh)= (lfs[i],hfs[i])
f_seg = t_seg.filter_fourier_bandpass(
[lfs[i],hfs[i]], [lfs[i]-df,hfs[i]+df])
fs_set = f_seg.flucstruc(method=method, separate=separate)
for fs in fs_set:
if count==0:
# show history if info says to, and avoid starting line with a digit
if info > 0: print('< '+fs.history.replace('\n201','\n< 201'))
print('Shot time SVS freq Amp a12 p H frlow frhigh {np:2d} Phases'.format(np=len(fs.dphase)))
count += 1
if fs.H < max_H and fs.p>0.01 and len(fs.svs())>=min_svs:
