def test_stft():
print(
'>>>>>>>>>>>>>>>>>>> Test Short Time Fourier Transform (STFT) <<<<<<<<<<<<<<<<<<<<<<<<'
)
print(
"**** Generating 0.1 s long test data signal with linearly changing frequency: 10-100kHz"
)
f = np.linspace(1e4, 1e5, num=11)
coord = flap.Coordinate(name='Time',
start=0.0,
step=0.01,
mode=flap.CoordinateMode(equidistant=True),
dimension_list=[0])
f_obj = flap.DataObject(data_array=f,
coordinates=[coord],
data_unit=flap.Unit(name='Frequency', unit='Hz'))
d = flap.get_data('TESTDATA',
name='TEST-1-1',
options={
'Scaling': 'Volt',
'Frequency': f_obj,
'Length': 0.1,
'Row number': 1,
'Column number': 1
},
object_name='TESTDATA')
flap.stft('TESTDATA', output_name='TEST_STFT')
flap.abs_value('TEST_STFT', output_name='TEST_STFT')
flap.list_data_objects()
plt.figure()
flap.plot('TEST_STFT', axes=['Time', 'Frequency'], plot_type='image')
def test_testdata():
print()
print(
"\n>>>>>>>>>>>>>>>>>>> Test TESTDATA data source <<<<<<<<<<<<<<<<<<<<<<<<"
)
flap.delete_data_object('*', exp_id='*')
print(
"**** Generating 0.01 s long test data signals on [4,5] matrix with fixed frequency changing from channel to channel"
)
d = flap.get_data('TESTDATA',
name='TEST-*-*',
options={
'Scaling': 'Volt',
'Frequency': [1e3, 1e4],
'Length': 1e-2,
'Row number': 4,
'Column number': 5
},
object_name='TESTDATA')
plt.figure()
print("**** Plotting row 2")
d.plot(slicing={'Row': 2}, axes=['Time'])
print(
"**** Generating 0.01 s long test data signal with linearly changing frequency: 10-100kHz"
)
f = np.linspace(1e4, 1e5, num=11)
coord = flap.Coordinate(name='Time',
start=0.0,
step=0.001,
mode=flap.CoordinateMode(equidistant=True),
dimension_list=[0])
f_obj = flap.DataObject(data_array=f,
coordinates=[coord],
data_unit=flap.Unit(name='Frequency', unit='Hz'))
flap.list_data_objects(f_obj)
d = flap.get_data('TESTDATA',
name='TEST-1-1',
options={
'Scaling': 'Volt',
'Frequency': f_obj,
'Length': 0.01,
'Row number': 1,
'Column number': 1
},
object_name='TESTDATA')
plt.figure()
d.plot(axes='Time', options={'All': True})
def test_filter():
plt.close('all')
print()
print('>>>>>>>>>>>>>>>>>>> Test filter <<<<<<<<<<<<<<<<<<<<<<<<')
flap.delete_data_object('*')
print(
"**** Generating 10 square wave signals and filtering with integrating filter, 10 microsec"
)
t = np.arange(1000) * 1e-6
d = np.ndarray((len(t), 10), dtype=float)
for i in range(10):
d[:, i] = np.sign(np.sin(math.pi * 2 * (1e4 + i * 1e3) * t)) + 1
c = flap.Coordinate(name='Time',
unit='Second',
mode=flap.CoordinateMode(equidistant=True),
start=0.0,
step=1e-6,
dimension_list=[0])
d = flap.DataObject(data_array=d, coordinates=[c])
flap.add_data_object(d, "Signal")
plt.figure()
d.plot(options={'Y sep': 3})
di = d.filter_data(coordinate='Time',
intervals=flap.Intervals(np.array([1e-4, 6e-4]),
np.array([2e-4, 8e-4])),
options={
'Type': 'Int',
'Tau': 10e-6
}).plot(options={'Y sep': 3})
print("**** Filtering with differential filter, 10 microsec")
plt.figure()
d.plot(options={'Y sep': 3})
flap.filter_data('Signal',
output_name='Signal_filt',
coordinate='Time',
intervals=flap.Intervals(np.array([1e-4, 6e-4]),
np.array([2e-4, 8e-4])),
options={
'Type': 'Diff',
'Tau': 10e-6
})
flap.plot('Signal_filt', options={'Y sep': 3})
print(
"**** Generating random data, 1 million points and overplotting spectra with various filters."
)
d = flap.get_data('TESTDATA',
name='TEST-1-1',
options={
'Signal': 'Random',
'Scaling': 'Digit',
'Length': 1
},
object_name='Signal')
plt.figure()
flap.filter_data('Signal',
output_name='Signal_filt',
coordinate='Time',
options={
'Type': 'Int',
'Tau': 16e-6
})
flap.apsd('Signal',options={'Log':True,'Res':20,'Range':[100,5e5]},output_name='Signal_APSD')\
.plot(options={'Log x':True, 'Log y': True})
plotid = flap.apsd('Signal_filt',options={'Log':True,'Res':20,'Range':[100,5e5]},output_name='Signal_APSD')\
.plot(options={'Log x':True, 'Log y': True})
plotid.plt_axis_list[-1].set_title("{'Type':'Int','Tau':16e-6}")
plt.figure()
flap.filter_data('Signal',
output_name='Signal_filt',
coordinate='Time',
options={
'Type': 'Diff',
'Tau': 16e-6
})
flap.apsd('Signal',options={'Log':True,'Res':20,'Range':[100,5e5]},output_name='Signal_APSD')\
.plot(options={'Log x':True, 'Log y': True})
plotid = flap.apsd('Signal_filt',options={'Log':True,'Res':20,'Range':[100,5e5]},output_name='Signal_APSD')\
.plot(options={'Log x':True, 'Log y': True})
plotid.plt_axis_list[-1].set_title("{'Type':'Diff','Tau':16e-6}")
plt.figure()
flap.filter_data('Signal',
output_name='Signal_filt',
coordinate='Time',
options={
'Type': 'Lowpass',
'f_high': 5e4
})
flap.apsd('Signal',options={'Log':True,'Res':20,'Range':[100,5e5]},output_name='Signal_APSD')\
.plot(options={'Log x':True, 'Log y': True})
plotid = flap.apsd('Signal_filt',options={'Log':True,'Res':20,'Range':[100,5e5]},output_name='Signal_APSD')\
.plot(options={'Log x':True, 'Log y': True})
plotid.plt_axis_list[-1].set_title("{'Type':'Lowpass','f_high':5e4}")
plt.figure()
flap.filter_data('Signal',
output_name='Signal_filt',
coordinate='Time',
options={
'Type': 'Highpass',
'f_low': 1e4,
'f_high': 5e4
})
flap.apsd('Signal',options={'Log':True,'Res':20,'Range':[100,5e5]},output_name='Signal_APSD')\
.plot(options={'Log x':True, 'Log y': True})
plotid = flap.apsd('Signal_filt',options={'Log':True,'Res':20,'Range':[100,5e5]},output_name='Signal_APSD')\
.plot(options={'Log x':True, 'Log y': True})
plotid.plt_axis_list[-1].set_title(
"{'Type':'Highpass','f_low':1e4,'f_high':5e4}")
plt.figure()
flap.filter_data('Signal',
output_name='Signal_filt',
coordinate='Time',
options={
'Type': 'Bandpass',
'f_low': 5e3,
'f_high': 5e4
})
flap.apsd('Signal',options={'Log':True,'Res':20,'Range':[100,5e5]},output_name='Signal_APSD')\
.plot(options={'Log x':True, 'Log y': True})
plotid = flap.apsd('Signal_filt',options={'Log':True,'Res':20,'Range':[100,5e5]},output_name='Signal_APSD')\
.plot(options={'Log x':True, 'Log y': True})
plotid.plt_axis_list[-1].set_title(
"{'Type':'Bandpass','f_low':5e3,'f_high':5e4}")
plt.figure()
print("**** Bandpower signal [5e4-2e5] Hz, inttime 20 microsec")
flap.filter_data('Signal',
output_name='Signal_filt',
coordinate='Time',
options={
'Type': 'Bandpass',
'f_low': 5e4,
'f_high': 2e5,
'Power': True,
'Inttime': 20e-6
})
plotid = flap.plot('Signal_filt')
plotid.plt_axis_list[-1].set_title(
"'Type':'Bandpass','f_low':5e4,'f_high':2e5, 'Power':True, 'Inttime':20e-6}"
)
def get_fit_nstx_thomson_profiles(
exp_id=None, #Shot number
pressure=False, #Return the pressure profile paramenters
temperature=False, #Return the temperature profile parameters
density=False, #Return the density profile parameters
spline_data=False, #Calculate the results from the spline data (no error is going to be taken into account)
device_coordinates=False, #Calculate the results as a function of device coordinates
radial_range=None, #Radial range of the pedestal (only works when the device coorinates is set)
flux_coordinates=False, #Calculate the results in flux coordinates
flux_range=None, #The normalaized flux coordinates range for returning the results
test=False,
output_name=None,
return_parameters=False,
plot_time=None,
pdf_object=None,
):
"""
Returns a dataobject which has the largest corresponding gradient based on the tanh fit.
Fitting is based on publication https://aip.scitation.org/doi/pdf/10.1063/1.4961554
The linear background is not usitlized, instead of the mtanh, only tanh is used.
"""
if ((device_coordinates and flux_range is not None)
or (flux_coordinates and radial_range is not None)):
raise ValueError(
'When flux or device coordinates are set, only flux or radial range can be set! Returning...'
)
d = flap_nstx_thomson_data(exp_id=exp_id,
force_mdsplus=False,
pressure=pressure,
temperature=temperature,
density=density,
spline_data=False,
add_flux_coordinates=True,
output_name='THOMSON_DATA')
if flux_coordinates:
r_coord_name = 'Flux r'
if device_coordinates:
r_coord_name = 'Device R'
time = d.coordinate('Time')[0][0, :]
thomson_profile = {
'Time': time,
'Data': d.data,
'Error': d.error,
'Device R': d.coordinate('Device R')[0],
'Flux r': d.coordinate('Flux r')[0],
'Height': np.zeros(time.shape),
'Width': np.zeros(time.shape),
'Slope': np.zeros(time.shape),
'Position': np.zeros(time.shape),
'SOL offset': np.zeros(time.shape),
'Max gradient': np.zeros(time.shape),
'Error': {
'Height': np.zeros(time.shape),
'SOL offset': np.zeros(time.shape),
'Position': np.zeros(time.shape),
'Width': np.zeros(time.shape),
'Max gradient': np.zeros(time.shape)
}
}
if test:
plt.figure()
if flux_range is not None:
x_range = flux_range
if radial_range is not None:
x_range = radial_range
# def mtanh_fit_function(r, b_height, b_sol, b_pos, b_width, b_slope): #This version of the code is not working due to the b_slope linear dependence
# def mtanh(x,b_slope):
# return ((1+b_slope*x)*np.exp(x)-np.exp(-x))/(np.exp(x)+np.exp(-x))
# return (b_height-b_sol)/2*(mtanh((b_pos-r)/(2*b_width),b_slope)+1)+b_sol
def tanh_fit_function(r, b_height, b_sol, b_pos, b_width):
def tanh(x):
return (np.exp(x) - np.exp(-x)) / (np.exp(x) + np.exp(-x))
return (b_height - b_sol) / 2 * (tanh(
(b_pos - r) / (2 * b_width)) + 1) + b_sol
for i_time in range(len(time)):
x_data = d.coordinate(r_coord_name)[0][:, i_time]
y_data = d.data[:, i_time]
y_data_error = d.error[:, i_time]
if r_coord_name == 'Flux r':
x_data = x_data[np.argmin(x_data):]
if np.sum(np.isinf(x_data)) != 0:
continue
try:
ind_coord = np.where(
np.logical_and(x_data > x_range[0], x_data <= x_range[1]))
x_data = x_data[ind_coord]
y_data = y_data[ind_coord]
y_data_error = y_data_error[ind_coord]
# print(x_data)
# print(ind_coord)
p0 = [
y_data[0], #b_height
y_data[-1], #b_sol
x_data[0], #b_pos
(x_data[-1] - x_data[0]) / 2., #b_width
#(y_data[0]-y_data[-1])/(x_data[0]-x_data[-1]), #b_slope this is supposed to be some kind of liear modification to the
#tanh function called mtanh. It messes up the fitting quite a bit and it's not useful at all.
]
popt, pcov = curve_fit(tanh_fit_function,
x_data,
y_data,
sigma=y_data_error,
p0=p0)
perr = np.sqrt(np.diag(pcov))
if test or (plot_time is not None
and np.abs(plot_time - time[i_time]) < 1e-3):
plt.cla()
plt.scatter(x_data, y_data, color='tab:blue')
plt.errorbar(x_data,
y_data,
yerr=y_data_error,
marker='o',
color='tab:blue',
ls='')
plt.plot(x_data, tanh_fit_function(x_data, *popt))
if flux_coordinates:
xlabel = 'PSI_norm'
else:
xlabel = 'Device R [m]'
if temperature:
profile_string = 'temperature'
ylabel = 'Temperature [keV]'
elif density:
profile_string = 'density'
ylabel = 'Density [1/m3]'
elif pressure:
profile_string = 'pressure'
ylabel = 'Pressure [kPa]'
time_string = ' @ ' + str(time[i_time])
plt.title('Fit ' + profile_string + ' profile of ' +
str(exp_id) + time_string)
plt.xlabel(xlabel)
plt.ylabel(ylabel)
plt.pause(0.001)
if pdf_object is not None:
pdf_object.savefig()
else:
pass
# plt.plot(x_data,mtanh_fit_function(x_data,*p0))
thomson_profile['Height'][i_time] = popt[0]
thomson_profile['SOL offset'][i_time] = popt[1]
thomson_profile['Position'][i_time] = popt[2]
thomson_profile['Width'][i_time] = popt[3]
thomson_profile['Max gradient'][i_time] = (popt[1] - popt[0]) / (
popt[3]) #from paper and pen calculation
thomson_profile['Error']['Height'][i_time] = perr[0]
thomson_profile['Error']['SOL offset'][i_time] = perr[1]
thomson_profile['Error']['Position'][i_time] = perr[2]
thomson_profile['Error']['Width'][i_time] = perr[3]
thomson_profile['Error']['Max gradient'][i_time] = 1 / (np.abs(
popt[3])) * (np.abs(perr[1]) + np.abs(perr[0])) + np.abs(
(popt[1] - popt[0]) / (popt[3]**2)) * np.abs(perr[3])
#thomson_profile_parameters['Slope'][i_time]=popt[4]
except:
popt = [np.nan, np.nan, np.nan, np.nan]
coord = []
coord.append(
copy.deepcopy(
flap.Coordinate(
name='Time',
unit='s',
mode=flap.CoordinateMode(equidistant=True),
start=time[0],
step=time[1] - time[0],
#shape=time_arr.shape,
dimension_list=[0])))
coord.append(
copy.deepcopy(
flap.Coordinate(name='Sample',
unit='n.a.',
mode=flap.CoordinateMode(equidistant=True),
start=0,
step=1,
dimension_list=[0])))
if device_coordinates:
grad_unit = '/m'
if flux_coordinates:
grad_unit = '/psi'
if pressure:
data_unit = flap.Unit(name='Pressure gradient', unit='kPa' + grad_unit)
elif temperature:
data_unit = flap.Unit(name='Temperature gradient',
unit='keV' + grad_unit)
elif density:
data_unit = flap.Unit(name='Density gradient', unit='m-3' + grad_unit)
if spline_data:
data_title = 'NSTX Thomson gradient'
else:
data_title = 'NSTX Thomson gradient spline'
d = flap.DataObject(exp_id=exp_id,
data_array=thomson_profile['Max gradient'],
data_unit=data_unit,
coordinates=coord,
data_title=data_title)
if output_name is not None:
flap.add_data_object(d, output_name)
if not return_parameters:
return d
else:
return thomson_profile
def get_nstx_thomson_gradient(exp_id=None,
pressure=False,
temperature=False,
density=False,
r_pos=None,
spline_data=False,
output_name=None,
device_coordinates=True,
flux_coordinates=False):
#Data is RADIUS x TIME
if pressure + density + temperature != 1:
raise ValueError(
'Only one of the inputs should be set (pressure, temperature, density)!'
)
if device_coordinates + flux_coordinates != 1:
raise ValueError(
'Either device_coordinates or flux_coordinates can be set, not both.'
)
thomson = flap_nstx_thomson_data(exp_id,
pressure=pressure,
temperature=temperature,
density=density,
output_name='THOMSON_FOR_GRADIENT')
thomson_spline = flap_nstx_thomson_data(exp_id,
pressure=pressure,
temperature=temperature,
density=density,
spline_data=True,
output_name=None)
if device_coordinates:
radial_coordinate = thomson.coordinate('Device R')[0][:, 0]
spline_radial_coordinate = thomson_spline.coordinate('Device R')[0][:,
0]
if flux_coordinates:
radial_coordinate = thomson.coordinate('Flux r')[0][:, 0]
spline_radial_coordinate = thomson_spline.coordinate('Flux r')[0][:, 0]
time_vector = thomson.coordinate('Time')[0][0, :]
data = thomson.data
error = thomson.error
interp_data = thomson_spline.data
#Calculation of the numerical gradient and interpolating the values for the given r_pos
data_gradient = np.asarray([
(data[2:, i] - data[:-2, i]) /
(2 * (radial_coordinate[2:] - radial_coordinate[:-2]))
for i in range(len(time_vector))
]).T
data_gradient_error = np.asarray([
(np.abs(error[2:, i]) + np.abs(error[:-2, i])) /
(2 * (radial_coordinate[2:] - radial_coordinate[:-2]))
for i in range(len(time_vector))
]).T
interp_data_gradient = np.asarray([
(interp_data[2:, i] - interp_data[:-2, i]) /
(2 * (spline_radial_coordinate[2:] - spline_radial_coordinate[:-2]))
for i in range(len(time_vector))
]).T
#Interpolation for the r_pos
if r_pos is not None:
r_pos_gradient = np.asarray([
np.interp(r_pos, radial_coordinate[1:-1], data_gradient[:, i])
for i in range(len(time_vector))
])
r_pos_gradient_spline = np.asarray([
np.interp(r_pos, spline_radial_coordinate[1:-1],
interp_data_gradient[:, i])
for i in range(len(time_vector))
])
ind_r = np.argmin(np.abs(radial_coordinate[1:-1] - r_pos))
if radial_coordinate[ind_r] < r_pos:
R1 = radial_coordinate[1:-1][ind_r]
R2 = radial_coordinate[1:-1][ind_r + 1]
ind_R1 = ind_r
ind_R2 = ind_r + 1
else:
R1 = radial_coordinate[1:-1][ind_r - 1]
R2 = radial_coordinate[1:-1][ind_r]
ind_R1 = ind_r - 1
ind_R2 = ind_r
#Result of error propagation (basically average biased error between the two neighboring radii)
r_pos_gradient_error=np.abs((r_pos-R1)/(R2-R1))*data_gradient_error[ind_R2,:]+\
np.abs((r_pos-R2)/(R2-R1))*data_gradient_error[ind_R1,:]
coord = []
coord.append(
copy.deepcopy(
flap.Coordinate(
name='Time',
unit='s',
mode=flap.CoordinateMode(equidistant=True),
start=time_vector[0],
step=time_vector[1] - time_vector[0],
#shape=time_arr.shape,
dimension_list=[0])))
coord.append(
copy.deepcopy(
flap.Coordinate(name='Sample',
unit='n.a.',
mode=flap.CoordinateMode(equidistant=True),
start=0,
step=1,
dimension_list=[0])))
if device_coordinates:
grad_unit = '/m'
if flux_coordinates:
grad_unit = '/psi'
if pressure:
data_unit = flap.Unit(name='Pressure gradient',
unit='kPa' + grad_unit)
elif temperature:
data_unit = flap.Unit(name='Temperature gradient',
unit='keV' + grad_unit)
elif density:
data_unit = flap.Unit(name='Density gradient',
unit='m-3' + grad_unit)
if not spline_data:
d = flap.DataObject(exp_id=exp_id,
data_array=r_pos_gradient,
error=r_pos_gradient_error,
data_unit=data_unit,
coordinates=coord,
data_title='NSTX Thomson gradient')
else:
d = flap.DataObject(exp_id=exp_id,
data_array=r_pos_gradient_spline,
data_unit=data_unit,
coordinates=coord,
data_title='NSTX Thomson gradient spline')
else:
coord = []
coord.append(
copy.deepcopy(
flap.Coordinate(
name='Time',
unit='s',
mode=flap.CoordinateMode(equidistant=True),
start=time_vector[0],
step=time_vector[1] - time_vector[0],
#shape=time_arr.shape,
dimension_list=[1])))
coord.append(
copy.deepcopy(
flap.Coordinate(name='Sample',
unit='n.a.',
mode=flap.CoordinateMode(equidistant=True),
start=0,
step=1,
dimension_list=[1])))
if pressure:
data_unit = flap.Unit(name='Pressure gradient', unit='kPa/m')
elif temperature:
data_unit = flap.Unit(name='Temperature gradient', unit='keV/m')
elif density:
data_unit = flap.Unit(name='Density gradient', unit='m-3/m')
if device_coordinates:
radial_coordinate_name = 'Device R'
radial_unit = 'm'
if flux_coordinates:
radial_coordinate_name = 'Flux r'
radial_unit = ''
if not spline_data:
coord.append(
copy.deepcopy(
flap.Coordinate(
name=radial_coordinate_name,
unit=radial_unit,
mode=flap.CoordinateMode(equidistant=False),
values=radial_coordinate[1:-1],
shape=radial_coordinate[1:-1].shape,
dimension_list=[0])))
d = flap.DataObject(exp_id=exp_id,
data_array=data_gradient,
error=data_gradient_error,
data_unit=data_unit,
coordinates=coord,
data_title='NSTX Thomson gradient')
else:
coord.append(
copy.deepcopy(
flap.Coordinate(
name=radial_coordinate_name,
unit=radial_unit,
mode=flap.CoordinateMode(equidistant=False),
values=spline_radial_coordinate[1:-1],
shape=spline_radial_coordinate[1:-1].shape,
dimension_list=[0])))
d = flap.DataObject(exp_id=exp_id,
data_array=interp_data_gradient,
data_unit=data_unit,
coordinates=coord,
data_title='NSTX Thomson gradient spline')
if output_name is not None:
flap.add_data_object(d, output_name)
return d
def flap_nstx_thomson_data(exp_id=None,
force_mdsplus=False,
pressure=False,
temperature=False,
density=False,
spline_data=False,
add_flux_coordinates=True,
output_name=None,
test=False):
"""
Returns the Thomson scattering processed data from the MDSplus tree as
a dictionary containing all the necessary parameters. The description of
the dictionary can be seen below.
"""
if pressure + temperature + density != 1:
raise ValueError(
'Either pressure or temperature or density can be set, neither none, nor more than one.'
)
if exp_id is None:
raise TypeError('exp_id must be set.')
wd = flap.config.get_all_section('Module NSTX_GPI')['Local datapath']
filename = wd + '/' + str(exp_id) + '/nstx_mdsplus_thomson_' + str(
exp_id) + '.pickle'
if not os.path.exists(filename) or force_mdsplus:
conn = mds.Connection('skylark.pppl.gov:8501')
conn.openTree('activespec', exp_id)
mdsnames = [
'ts_times', #The time vector of the measurement (60Hz measurement with the Thomson)
'FIT_RADII', #Radius of the measurement
'FIT_R_WIDTH', #N/A (proably error of the radious)
'FIT_TE', #Electron temperature profile numpy array([radius,time])
'FIT_TE_ERR', #The error for Te (symmetric)
'FIT_NE', #Electron density profile numpy array([radius,time])
'FIT_NE_ERR', #The error for ne (symmetric)
'FIT_PE', #Electron pressure profile numpy array([radius,time])
'FIT_PE_ERR', #The error for pe (symmetric)
'SPLINE_RADII', #Spline fit of the previous results (4times interpolation compared to the previous ones)
'SPLINE_NE', #Spline fit ne without error
'SPLINE_PE', #Spline fit pe without error
'SPLINE_TE', #Spline fit Te without error
'TS_LD', #N/A
'LASER_ID', #ID of the Thomson laser
'VALID', #Validity of the measurement
'DATEANALYZED', #The date when the analysis was done for the data
'COMMENT'
] #Comment for the analysis
thomson = {}
for name in mdsnames:
thomson[name] = conn.get('\TS_BEST:' + name).data()
if name == 'ts_times' and type(thomson[name]) is str:
raise ValueError('No Thomson data available.')
thomson['FIT_R_WIDTH'] /= 100.
thomson['FIT_RADII'] /= 100.
thomson['SPLINE_RADII'] /= 100.
thomson['FIT_NE'] *= 10e6
thomson['FIT_NE_ERR'] *= 10e6
thomson['SPLINE_NE'] *= 10e6
conn.closeAllTrees()
conn.disconnect()
try:
pickle.dump(thomson, open(filename, 'wb'))
except:
raise IOError(
'The path ' + filename +
' cannot be accessed. Pickle file cannot be created.')
else:
thomson = pickle.load(open(filename, 'rb'))
thomson_time = thomson['ts_times']
coord = []
coord.append(
copy.deepcopy(
flap.Coordinate(
name='Time',
unit='s',
mode=flap.CoordinateMode(equidistant=True),
start=thomson_time[0],
step=thomson_time[1] - thomson_time[0],
#shape=time_arr.shape,
dimension_list=[1])))
coord.append(
copy.deepcopy(
flap.Coordinate(name='Sample',
unit='n.a.',
mode=flap.CoordinateMode(equidistant=True),
start=0,
step=1,
dimension_list=[1])))
if spline_data:
thomson_r_coord = thomson['SPLINE_RADII']
if pressure:
data_arr = thomson['SPLINE_PE']
data_arr_err = None
data_unit = flap.Unit(name='Pressure', unit='kPa')
elif temperature:
data_arr = thomson['SPLINE_TE']
data_arr_err = None
data_unit = flap.Unit(name='Temperature', unit='keV')
elif density:
data_arr = thomson['SPLINE_NE']
data_arr_err = None
data_unit = flap.Unit(name='Density', unit='m-3')
else:
thomson_r_coord = thomson['FIT_RADII']
if pressure:
data_arr = thomson['FIT_PE']
data_arr_err = thomson['FIT_PE_ERR']
data_unit = flap.Unit(name='Pressure', unit='kPa')
elif temperature:
data_arr = thomson['FIT_TE']
data_arr_err = thomson['FIT_TE_ERR']
data_unit = flap.Unit(name='Temperature', unit='keV')
elif density:
data_arr = thomson['FIT_NE']
data_arr_err = thomson['FIT_NE_ERR']
data_unit = flap.Unit(name='Density', unit='m-3')
coord.append(
copy.deepcopy(
flap.Coordinate(name='Device R',
unit='m',
mode=flap.CoordinateMode(equidistant=False),
values=thomson_r_coord,
shape=thomson_r_coord.shape,
dimension_list=[0])))
if test:
plt.figure()
if add_flux_coordinates:
try:
psi_rz_obj = flap.get_data('NSTX_MDSPlus',
name='\EFIT02::\PSIRZ',
exp_id=exp_id,
object_name='PSIRZ_FOR_COORD')
psi_mag = flap.get_data('NSTX_MDSPlus',
name='\EFIT02::\SSIMAG',
exp_id=exp_id,
object_name='SSIMAG_FOR_COORD')
psi_bdry = flap.get_data('NSTX_MDSPlus',
name='\EFIT02::\SSIBRY',
exp_id=exp_id,
object_name='SSIBRY_FOR_COORD')
except:
raise ValueError("The PSIRZ MDSPlus node cannot be reached.")
psi_values = psi_rz_obj.data[:, :, 32]
psi_t_coord = psi_rz_obj.coordinate('Time')[0][:, 0, 0]
psi_r_coord = psi_rz_obj.coordinate(
'Device R')[0][:, :,
32] #midplane is the middle coordinate in the array
#Do the interpolation
psi_values_spat_interpol = np.zeros(
[thomson_r_coord.shape[0], psi_t_coord.shape[0]])
for index_t in range(psi_t_coord.shape[0]):
norm_psi_values = (psi_values[index_t, :] - psi_mag.data[index_t]
) / (psi_bdry.data[index_t] -
psi_mag.data[index_t])
norm_psi_values[np.isnan(norm_psi_values)] = 0.
psi_values_spat_interpol[:, index_t] = np.interp(
thomson_r_coord, psi_r_coord[index_t, :], norm_psi_values)
psi_values_total_interpol = np.zeros(data_arr.shape)
for index_r in range(data_arr.shape[0]):
psi_values_total_interpol[index_r, :] = np.interp(
thomson_time, psi_t_coord,
psi_values_spat_interpol[index_r, :])
if test:
for index_t in range(len(thomson_time)):
plt.cla()
plt.plot(thomson_r_coord, psi_values_total_interpol[:,
index_t])
plt.pause(0.5)
psi_values_total_interpol[np.isnan(psi_values_total_interpol)] = 0.
coord.append(
copy.deepcopy(
flap.Coordinate(name='Flux r',
unit='',
mode=flap.CoordinateMode(equidistant=False),
values=psi_values_total_interpol,
shape=psi_values_total_interpol.shape,
dimension_list=[0, 1])))
if test:
plt.plot(psi_values_total_interpol, data_arr)
d = flap.DataObject(data_array=data_arr,
error=data_arr_err,
data_unit=data_unit,
coordinates=coord,
exp_id=exp_id,
data_title='NSTX Thomson data')
if output_name is not None:
flap.add_data_object(d, output_name)
return d
def testdata_get_data(exp_id=None,
data_name='*',
no_data=False,
options=None,
coordinates=None,
data_source=None):
""" Data read function for flap test data source
Channel names: TEST-col-row: Signals on a 15x5 spatial matrix
VIDEO: A test image with timescale as for the signals.
options:
'Scaling': 'Volt', 'Digit'
'Signal' :'Sin' Sine signals
'Const.' : Constant values with row*COLUMN_NUMBER+column
'Complex-Sin': Same as Sine but an imaginary cosine is added
'Random': Random (normal dist.) signal in all channels
'Row number': Number of rows for signal matrix
'Column number': Number of columns for signal matrix
'Matrix angle': The angle [deg] of the signal matrix
'Image' : 'Gauss' Gaussian spot
'Random' Random int16 values between 0 and 4095
'Spotsize': Full width at half maximum of spot in pixels
'Width' : Image width in pixels (x size)
'Height' : Image height in pixels (y size)
'Frequency' : <number> Fixed frequency of signals [Hz]
For "Gauss" video data the rotation frequency of the spot
: [f2,f2]: Changes from channel-to-channel between these frequencies [Hz]
: data object: should be data object with Time coordinate and Frequency as data
If has one channel describes frequency vs time for all channels
If multiple signals (coordinate Signal name describes signals) describes frequency
vs time for channels. Should have the same number of channels as for the generation.
'Length': Length in second. The sample rate is 1 MHz
'Samplerate': Sample rate [Hz]
"""
if (data_source is None):
data_source = 'TESTDATA'
default_options = {
'Row number': 10,
'Column number': 15,
'Matrix angle': 0.0,
'Scaling': 'Volt',
'Signal': 'Sin',
'Image': 'Gauss',
'Spotsize': 100,
'Width': 640,
'Height': 480,
'Frequency': 1e3,
'Length': 0.1,
'Samplerate': 1e6
}
_options = flap.config.merge_options(default_options,
options,
data_source=data_source)
ROW_NUMBER = _options['Row number']
COLUMN_NUMBER = _options['Column number']
alpha = _options['Matrix angle']
# creating a list of signal names
signal_list = []
for row in range(ROW_NUMBER):
for column in range(COLUMN_NUMBER):
signal_list.append('TEST-' + str(column + 1) + '-' + str(row + 1))
signal_list.append('VIDEO')
# Selecting the desired channels
try:
signal_select, signal_index = flap.select_signals(
signal_list, data_name)
except ValueError as e:
raise e
try:
signal_select.index('VIDEO')
test_image = True
except:
test_image = False
if (test_image and (len(signal_select) != 1)):
raise ValueError(
"VIDEO data cannot be read together with test signals.")
meas_timerange = [0, _options['Length']]
meas_sampletime = 1. / _options['Samplerate']
meas_sample = int(
np.rint(
float((meas_timerange[1] - meas_timerange[0]) / meas_sampletime)) +
1)
read_range = None
read_samplerange = None
# Checking whether time or sample range selection == present
if (coordinates is not None):
if (type(coordinates) != list):
_coordinates = [coordinates]
else:
_coordinates = coordinates
for coord in _coordinates:
if (type(coord) != flap.Coordinate):
raise TypeError(
"Coordinate description should be flap.Coordinate.")
if (coord.unit.name == 'Time'):
if (coord.mode.equidistant):
read_range = [
float(coord.c_range[0]),
float(coord.c_range[1])
]
else:
raise NotImplementedError(
"Non-equidistant Time axis is not implemented yet.")
break
if coord.unit.name == 'Sample':
if (coord.mode.equidistant):
read_samplerange = coord.c_range
else:
raise NotImplementedError(
"Non-equidistant Sample axis is not implemented yet.")
break
# If no sample or time range is given read all data
if ((read_range is None) and (read_samplerange is None)):
read_samplerange = [0, meas_sample]
# Determining the read_samplerange and read_range
if (read_range is not None):
read_range = np.array(read_range)
if (read_samplerange is None):
read_samplerange = np.rint(read_range /
float(meas_sampletime)).astype(int)
else:
read_samplerange = np.array(read_samplerange).astype(int)
if ((read_samplerange[1] < 0) or (read_samplerange[0] >= meas_sample)):
raise ValueError("No data in time range.")
if (read_samplerange[0] < 0):
read_samplerange[0] = 0
if (read_samplerange[1] >= meas_sample):
read_samplerange[1] = meas_sample - 1
if (read_range is None):
read_range = read_samplerange * meas_sampletime + meas_timerange[0]
ndata = int(read_samplerange[1] - read_samplerange[0] + 1)
# Checking whether scaling to Volt is requested
if (_options['Scaling'] == 'Volt'):
scale_to_volts = True
#dtype = float #UNUSED
data_unit = flap.Unit(name='Signal', unit='Volt')
else:
scale_to_volts = False
#dtype = np.int16 #UNUSED
data_unit = flap.Unit(name='Signal', unit='Digit')
if (not test_image):
# Arranging the signals in 2D
# These will collect the row and column coordinatees for all signals
row_list = []
column_list = []
# In these we collect how many channels are active in each row and
# column
rows_act = np.array([0] * ROW_NUMBER)
columns_act = np.array([0] * COLUMN_NUMBER)
for i in range(len(signal_select)):
s_split = signal_select[i].split('-')
if (len(s_split) != 3):
continue
c = int(s_split[1])
r = int(s_split[2])
row_list.append(r)
column_list.append(c)
rows_act[r - 1] += 1
columns_act[c - 1] += 1
if (len(signal_select) == 1):
twodim = False
else:
# Determining whether the channel matrix fills a rectangular area
# indices where rows and columns have channels
ind_rows_act_n0 = np.where(rows_act != 0)[0]
ind_columns_act_n0 = np.where(columns_act != 0)[0]
twodim = ((np.min(rows_act[ind_rows_act_n0]) == np.max(
rows_act[ind_rows_act_n0]))
and (np.min(columns_act[ind_columns_act_n0]) == np.max(
columns_act[ind_columns_act_n0])))
if ((len(ind_rows_act_n0)*len(ind_columns_act_n0) != len(signal_select)) \
or (len(ind_rows_act_n0) == 1) or (len(ind_columns_act_n0) == 1)):
twodim = False
data_arr = None
if (len(signal_select) != 1):
if (twodim):
row_coords = np.arange(ROW_NUMBER,
dtype=int)[ind_rows_act_n0] + 1
column_coords = np.arange(COLUMN_NUMBER,
dtype=int)[ind_columns_act_n0] + 1
data_shape = (len(column_coords), len(row_coords), ndata)
# if (no_data == False):
# data_arr = np.empty(data_shape, dtype=dtype)
signal_matrix = np.empty((len(column_coords), len(row_coords)),
dtype=np.dtype(object))
else:
data_shape = (len(signal_select), ndata)
# if (no_data == False):
# data_arr = np.empty(data_shape, dtype=dtype)
else:
data_shape = [ndata]
# Reading the signals
for i in range(len(row_list)):
# Determining row and column numbers
r = row_list[i]
c = column_list[i]
signal_type = _options['Signal']
if ((signal_type == 'Sin') or (signal_type == 'Complex-Sin')):
amp = (r-float(ROW_NUMBER)/2)**2 / (2*(float(ROW_NUMBER)/4)**2) \
* (c-float(COLUMN_NUMBER)/2)**2 / (2*(float(COLUMN_NUMBER)/4)**2)
amp = math.exp(-amp)
n_sample = round(_options['Samplerate'] * _options['Length'])
# Sample times for all signal
t = np.arange(meas_sample) * meas_sampletime
if (type(_options['Frequency']) == list):
flist = _options['Frequency']
f = float(flist[1] - flist[0]) / (len(signal_select) -
1) * i + flist[0]
ph = t * 2 * math.pi * f
elif (type(_options['Frequency']) == flap.DataObject):
if (len(_options['Frequency'].shape) == 1):
tf, t_low, t_high = _options['Frequency'].coordinate(
'Time', options={'Change only': True})
f = _options['Frequency'].data
if ((np.amin(tf) > 0)
or (np.amax(tf) < _options['Length'])):
raise ValueError(
'Frequency time evolution is not available for sample range [0,{:f}]s.'
.format(_options['Length']))
fi = np.interp(t, tf, f)
ph = np.cumsum(2 * math.pi * fi /
_options['Samplerate'])
else:
raise NotImplementedError(
"Variable frequency for multiple channels not supported yet."
)
else:
f = float(_options['Frequency'])
ph = t * 2 * math.pi * f
if (signal_type == 'Sin'):
signal = np.sin(ph) * amp
signal = np.rint(signal / meas_ADC_step)
if (signal_type == 'Complex-Sin'):
signal_re = np.sin(ph) * amp
signal_re = np.rint(signal_re / meas_ADC_step)
signal_im = np.cos(ph) * amp
signal_im = np.rint(signal_im / meas_ADC_step)
signal = np.empty(signal_re.shape, dtype=complex)
signal.real = signal_re
signal.imag = signal_im
elif (signal_type == 'Const.'):
signal = np.full(int(meas_sample),
float(r - 1) * COLUMN_NUMBER + float(c - 1))
elif (signal_type == 'Random'):
signal = np.random.randn(int(meas_sample)) + 10.
else:
raise ValueError("Signal type '" + signal_type +
"' is not understood.")
if (len(signal_select) == 1):
if (no_data == False):
data_arr = signal[int(read_samplerange[0]
):int(read_samplerange[1]) + 1]
else:
if (no_data == False):
d = signal[int(read_samplerange[0]
):int(read_samplerange[1]) + 1]
if (twodim):
if (no_data == False):
if (data_arr is None):
data_arr = np.empty(data_shape, dtype=d.dtype)
data_arr[list(column_coords).index(c),
list(row_coords).index(r), :] = d
signal_matrix[list(column_coords).index(c),
list(row_coords).index(r)] = signal_select[i]
else:
if (no_data == False):
if (data_arr is None):
data_arr = np.empty(data_shape, dtype=d.dtype)
data_arr[i, :] = d
if (scale_to_volts):
if (no_data == False):
data_arr = data_arr * meas_ADC_step
# Adding coordinates: Sample, Time, Signal name, Column, Row
coord = [None] * 5
# Sample and Time
c_mode = flap.CoordinateMode(equidistant=True)
if (twodim):
dimension_list = [2]
else:
if (len(signal_select) == 1):
dimension_list = [0]
else:
dimension_list = [1]
coord[0] = copy.deepcopy(
flap.Coordinate(name='Time',
unit='Second',
mode=c_mode,
shape=[],
start=read_range[0],
step=meas_sampletime,
dimension_list=dimension_list))
coord[1] = copy.deepcopy(
flap.Coordinate(name='Sample',
unit='',
mode=c_mode,
shape=[],
start=read_samplerange[0],
step=1,
dimension_list=dimension_list))
# Signal name
c_mode = flap.CoordinateMode(equidistant=False)
if (twodim):
dimension_list = [0, 1]
value = signal_matrix
shape = signal_matrix.shape
else:
value = signal_select
if (len(signal_select) == 1):
dimension_list = []
shape = []
else:
dimension_list = [0]
shape = len(signal_select)
coord[2] = copy.deepcopy(
flap.Coordinate(name='Signal name',
unit='',
mode=c_mode,
shape=shape,
values=np.array(value),
dimension_list=dimension_list))
# Column
c_mode = flap.CoordinateMode(equidistant=False)
if (twodim):
dimension_list = [0]
values = column_coords
shape = len(values)
else:
values = column_list
if (len(values) == 1):
dimension_list = []
shape = []
else:
dimension_list = [0]
shape = len(values)
coord[3] = copy.deepcopy(
flap.Coordinate(name='Column',
unit='',
mode=c_mode,
shape=shape,
values=values,
dimension_list=dimension_list))
# Row
c_mode = flap.CoordinateMode(equidistant=False)
if (twodim):
dimension_list = [1]
values = row_coords
shape = len(values)
else:
values = row_list
if (len(values) == 1):
dimension_list = []
shape = shape
else:
dimension_list = [0]
shape = len(values)
coord[4] = copy.deepcopy(
flap.Coordinate(name='Row',
unit='',
mode=c_mode,
shape=shape,
values=values,
dimension_list=dimension_list))
data_title = "Test data"
if (len(signal_select) == 1):
data_title += " (" + signal_select[0] + ")"
d = flap.DataObject(data_array=data_arr,
data_unit=data_unit,
coordinates=coord,
exp_id=exp_id,
data_title=data_title,
data_shape=data_shape)
else:
# VIDEO
image_xsize = _options['Width']
image_ysize = _options['Height']
spotwidth = _options['Spotsize']
if (no_data == False):
if (_options['Image'] == 'Gauss'):
f = float(_options['Frequency'])
t = np.arange(ndata, dtype=float) * meas_sampletime
amp = np.sin(t * 4.5 * math.pi * f)**2 * 3000 + 1000
center_x = image_xsize / 2 + (np.sin(
t * 2 * math.pi * f) * image_xsize / 4) * (t / t[-1] + 0.2)
center_y = image_ysize / 2 + (np.cos(
t * 2 * math.pi * f) * image_ysize / 4) * (t / t[-1] + 0.2)
x, y = np.meshgrid(range(0, image_xsize),
range(0, image_ysize))
data_arr = np.empty((x.shape[0], x.shape[1], t.size),
dtype=np.int16)
for it in range(len(t)):
data_arr[:, :,
it] = (np.exp(-((x - center_x[it])**2 +
(y - center_y[it])**2) / 2 /
(spotwidth**2 + spotwidth**2)) *
amp[it]).astype(np.int16)
elif (_options['Image'] == 'Random'):
data_arr = np.random.randint(0,
high=4095,
size=(image_xsize, image_ysize,
ndata),
dtype=np.int16)
coord = [None] * 4
coord[0] = copy.deepcopy(
flap.Coordinate(name='Time',
unit='Second',
mode=flap.CoordinateMode(equidistant=True),
shape=[],
start=read_range[0],
step=meas_sampletime,
dimension_list=[2]))
coord[1] = copy.deepcopy(
flap.Coordinate(name='Sample',
unit='',
mode=flap.CoordinateMode(equidistant=True),
shape=[],
start=read_samplerange[0],
step=1,
dimension_list=[2]))
coord[2] = copy.deepcopy(
flap.Coordinate(name='Image x',
unit='Pixel',
mode=flap.CoordinateMode(equidistant=True),
shape=[],
start=0,
step=1,
dimension_list=[1]))
coord[3] = copy.deepcopy(
flap.Coordinate(name='Image y',
unit='Pixel',
mode=flap.CoordinateMode(equidistant=True),
shape=[],
start=0,
step=1,
dimension_list=[0]))
data_title = "Test images"
d = flap.DataObject(data_array=data_arr,
data_unit=flap.Unit(name='Image', unit='Digit'),
coordinates=coord,
exp_id=exp_id,
data_title=data_title)
return d
def getsignal_ppf(exp_id, source, no_data=False, options={}):
""" Signal reading function for the JET API PPF
Does not require using the FLAP storage
Input: same as get_data()
Output: DataObject for the given source
"""
import ppf
import getdat as jpf
# Function used for getting either ppf or jpf data from the server
#
options_default = {
"Sequence": 0,
"UID": "jetppf",
"fix0": 0,
"reshape": 0,
"no_x": 0,
"no_t": 0,
"Only Info": 0,
"nwds": 0,
"Check Time Equidistant": False,
"Cache Data": False
}
options = {**options_default, **options}
#checking if the data is already in the cache
curr_path = os.path.dirname(os.path.abspath(__file__))
location = os.path.sep.join(curr_path.split(os.path.sep))
split_source = source.split("/")
filename = os.path.join(os.path.sep.join([location,"cached"]),
("_".join(split_source)+"-"+options["UID"]+\
"-"+str(exp_id)+".hdf5").lower())
if os.path.exists(filename):
return flap.load(filename)
# obtaining the data from the server
# if the UID ends with -team, then it looks in the config file, whether
# there is a team defined under the name given in UID in the config file
# if so, it will loop over the name of the team members
split_uid = options["UID"].split("-")
if split_uid[-1] == "team":
uid_list = flap.config.get("Module JET_API",
options["UID"],
evaluate=True)
else:
if options["UID"] == "curr_user":
options["UID"] = pwd.getpwuid(os.getuid()).pw_name
uid_list = [options["UID"]]
if split_source[0].upper() == "PPF":
for uid in uid_list:
raw_ppf = \
ppf.ppfdata(exp_id,split_source[1],split_source[2],
seq=options["Sequence"], uid=uid,
fix0=options["fix0"], reshape=options["reshape"],
no_x=options["no_x"], no_t=options["no_t"],
no_data=options["Only Info"])
[
data, x, t, nd, nx, nt, dunits, xunits, tunits, desc, comm,
seq, ier
] = raw_ppf
if ier == 0:
break
if ier != 0:
raise ValueError("Error reading " + source + " for shot " +
str(exp_id) + ", UID " + options["UID"] +
":\nIER " + str(ier) + ": " + desc)
elif split_source[0].upper() == "JPF":
for uid in uid_list:
raw_jpf = jpf.getdat("/".join(split_source[1:]),
exp_id,
nwds=options["nwds"])
[data, t, nwds, desc, dunits, ier] = raw_jpf
if ier == 0:
break
options["no_x"] = 0
x = []
tunits = ""
if ier != 0:
raise ValueError("Error reading " + source + " for shot " +
str(exp_id) + ":\nIER " + str(ier))
else:
raise ValueError("Source should be either PPF or JPF")
# converting data to flap DataObject
coordinates = []
info = "Obtained at " + str(date.today()) + ", uid " + uid + "\n"
if options["no_t"] == 0 and (len(t) != 0):
tunit = ["secs", "s", "seconds"]
if tunits.lower() in tunit:
tunits = "Second"
equidist = False
if options['Check Time Equidistant'] is True:
timesteps = np.array(t[1:]) - np.array(t[0:-1])
equidistance = np.linalg.norm(
timesteps - timesteps[0]) / np.linalg.norm(timesteps)
if equidistance < 1e-6:
info = info + "Time variable is taken as equidistant to an accuracy of "+\
str(equidistance)+"\n"
time_coord = flap.Coordinate(
name='Time',
unit=tunits,
start=t[0],
shape=t.shape,
step=np.mean(timesteps),
mode=flap.CoordinateMode(equidistant=True),
dimension_list=[0])
equidist = True
if equidist is False:
if options['Check Time Equidistant'] is True:
info = info + "Time variable is not equidistant, deviation: " + str(
equidistance) + "\n"
time_coord = flap.Coordinate(
name='Time',
unit=tunits,
values=t,
shape=t.shape,
mode=flap.CoordinateMode(equidistant=False),
dimension_list=[0])
coordinates.append(time_coord)
if options["no_x"] == 0 and (len(x) != 0):
if len(x) > 1:
x_coord = flap.Coordinate(
name='X',
unit=xunits,
values=x,
shape=x.shape,
mode=flap.CoordinateMode(equidistant=False),
dimension_list=[1])
coordinates.append(x_coord)
if options["Only Info"] == 0 and (len(data) != 0):
unit = flap.Unit(name=source, unit=dunits)
if "x_coord" in locals():
data = data.reshape((len(t), len(x)))
signal = flap.DataObject(data_array=data,
data_unit=unit,
coordinates=coordinates,
exp_id=str(exp_id),
data_title=desc,
data_shape=data.shape,
info=info)
if "comm" in locals():
signal.info = signal.info + comm + "\n"
if no_data is True:
signal.data = None
if options["Cache Data"] is True:
flap.save(signal, filename)
return signal
else:
# if one is only ineterested in the metadata
return raw_ppf
def nstx_gpi_get_data(exp_id=None,
data_name=None,
no_data=False,
options=None,
coordinates=None,
data_source=None):
#translate the input variables to the actual directory name on portal
#copy the file from the portal to PC if it is not already there
#interpret the .cin file
# read the header
# read the data
if (exp_id is None):
raise ValueError('exp_id should be set for NSTX GPI.')
if (type(exp_id) is not int):
raise TypeError("exp_id should be an integer and not %s" %
(type(exp_id)))
default_options = {
'Local datapath': 'data',
'Datapath': None,
'Scaling': 'Digit',
'Offset timerange': None,
'Calibration': False,
'Calib. path': 'cal',
'Calib. file': None,
'Phase': None,
'State': None,
'Start delay': 0,
'End delay': 0,
'Download only': False
}
_options = flap.config.merge_options(default_options,
options,
data_source='NSTX_GPI')
#folder decoder
folder = {
'_0_': 'Phantom71-5040',
'_1_': 'Phantom710-9206',
'_2_': 'Phantom73-6747',
'_3_': 'Phantom73-6663',
'_4_': 'Phantom73-8032',
'_5_': 'Phantom710-9205',
'_6_': 'Miro4-9373'
}
data_title = 'NSTX GPI data'
if (exp_id < 118929):
year = 2005
if (exp_id >= 118929) and (exp_id < 122270):
year = 2006
if (exp_id >= 122270) and (exp_id < 126511):
year = 2007
if (exp_id >= 126511) and (exp_id < 131565):
year = 2008
if (exp_id >= 131565) and (exp_id < 137110):
year = 2009
if (exp_id >= 137110):
year = 2010
if (year < 2006):
cam = '_0_'
if (year == 2007 or year == 2008):
cam = '_1_'
if (year == 2009):
cam = '_2_'
if (year == 2010):
cam = '_5_'
if (year < 2006):
file_name = 'nstx' + str(exp_id) + '.cin'
else:
file_name = 'nstx' + cam + str(exp_id) + '.cin'
file_folder=_options['Datapath']+'/'+folder[cam]+\
'/'+str(year)+'/'
remote_file_name = file_folder + file_name
local_file_folder = _options['Local datapath'] + '/' + str(exp_id) + '/'
if not os.path.exists(_options['Local datapath']):
raise SystemError("The local datapath cannot be found.")
return
if not (os.path.exists(local_file_folder + file_name)):
if not (os.path.exists(local_file_folder)):
try:
os.mkdir(local_file_folder)
except:
raise SystemError("The folder cannot be created." +
"Dumping the file to scratch")
local_file_folder = _options['Local datapath'] + '/scratch'
p = subprocess.Popen([
"scp", _options['User'] + "@" + _options['Server'] + ':' +
remote_file_name, local_file_folder
])
os.waitpid(p.pid, 0)
if not (os.path.exists(local_file_folder + file_name)):
raise SystemError(
"The file couldn't be transferred to the local directory.")
if (_options['Download only']):
d = flap.DataObject(data_array=np.asarray([0, 1]),
data_unit=None,
coordinates=None,
exp_id=exp_id,
data_title=data_title,
info={'Options': _options},
data_source="NSTX_GPI")
return d
images = pims.Cine(local_file_folder + file_name)
data_arr = np.flip(
np.asarray(images[:], dtype=np.int16),
2) #The original data is 80x64, this line converts it to 64x80
data_unit = flap.Unit(name='Signal', unit='Digit')
#The header dict contains the capture information along with the entire image number and the first_image_no (when the recording started)
#The frame_rate corresponds with the one from IDL.
trigger_time = images.header_dict['first_image_no'] / images.frame_rate
coord = [None] * 6
coord[0] = (
copy.deepcopy(
flap.Coordinate(
name='Time',
unit='s',
mode=flap.CoordinateMode(equidistant=True),
start=trigger_time,
step=1 / float(images.frame_rate),
#shape=time_arr.shape,
dimension_list=[0])))
coord[1] = (copy.deepcopy(
flap.Coordinate(name='Sample',
unit='n.a.',
mode=flap.CoordinateMode(equidistant=True),
start=0,
step=1,
dimension_list=[0])))
coord[2] = (copy.deepcopy(
flap.Coordinate(name='Image x',
unit='Pixel',
mode=flap.CoordinateMode(equidistant=True),
start=0,
step=1,
shape=[],
dimension_list=[1])))
coord[3] = (copy.deepcopy(
flap.Coordinate(name='Image y',
unit='Pixel',
mode=flap.CoordinateMode(equidistant=True),
start=0,
step=1,
shape=[],
dimension_list=[2])))
#Get the spatial calibration for the GPI data
#This spatial calibration is based on the rz_map.dat which used a linear
#approximation for the transformation between pixel and spatial coordinates
#This needs to be updated as soon as more information is available on the
#calibration coordinates.
coeff_r = np.asarray([3.7183594, -0.77821046, 1402.8097
]) / 1000. #The coordinates are in meters
coeff_z = np.asarray([0.18090118, 3.0657776, 70.544312
]) / 1000. #The coordinates are in meters
# This part is not producing appropriate results due to the equidistant spacing and double
# coefficients. Slicing is only possible for single steps.
# coord[4]=(copy.deepcopy(flap.Coordinate(name='Device R',
# unit='m',
# mode=flap.CoordinateMode(equidistant=True),
# start=coeff_r[2],
# step=[coeff_r[0],coeff_r[1]],
# dimension_list=[1,2]
# )))
#
# coord[5]=(copy.deepcopy(flap.Coordinate(name='Device z',
# unit='m',
# mode=flap.CoordinateMode(equidistant=True),
# start=coeff_z[2],
# step=[coeff_z[0],coeff_z[1]],
# dimension_list=[1,2]
# )))
r_coordinates = np.zeros([64, 80])
z_coordinates = np.zeros([64, 80])
for i_x in range(64):
for i_y in range(80):
r_coordinates[
i_x, i_y] = coeff_r[0] * i_x + coeff_r[1] * i_y + coeff_r[2]
z_coordinates[
i_x, i_y] = coeff_z[0] * i_x + coeff_z[1] * i_y + coeff_z[2]
coord[4] = (copy.deepcopy(
flap.Coordinate(name='Device R',
unit='m',
mode=flap.CoordinateMode(equidistant=False),
values=r_coordinates,
shape=r_coordinates.shape,
dimension_list=[1, 2])))
coord[5] = (copy.deepcopy(
flap.Coordinate(name='Device z',
unit='m',
mode=flap.CoordinateMode(equidistant=False),
values=z_coordinates,
shape=z_coordinates.shape,
dimension_list=[1, 2])))
_options["Trigger time [s]"] = trigger_time
_options["FPS"] = images.frame_rate
_options["Sample time [s]"] = 1 / float(images.frame_rate)
_options["Exposure time [s]"] = images.tagged_blocks['exposure_only'][0]
_options["X size"] = images.frame_shape[0]
_options["Y size"] = images.frame_shape[1]
_options["Bits"] = images.bitmapinfo_dict["bi_bit_count"]
d = flap.DataObject(data_array=data_arr,
data_unit=data_unit,
coordinates=coord,
exp_id=exp_id,
data_title=data_title,
info={'Options': _options},
data_source="NSTX_GPI")
return d
def calculate_magnetics_spectrogram(exp_id=None,
time_range=None,
channel=1,
time_res=1e-3,
freq_res=None,
frange=None,
recalc=False,
plot=True,
pdf=False,
pdfobject=None,
):
wd=flap.config.get_all_section('Module NSTX_GPI')['Working directory']
filename=flap_nstx.analysis.filename(exp_id=exp_id,
working_directory=wd+'/processed_data',
time_range=time_range,
comment='magnetic_spectrogram_hf_ch'+str(channel)+'_tr_'+str(time_res)+'_frange_'+str(frange[0])+'_'+str(frange[1]),
extension='pickle')
if not recalc and not os.path.exists(filename):
print('File doesn\'t exist, needs to be calculated!')
recalc=True
if recalc or not os.path.exists(filename):
if freq_res is None:
freq_res=2/time_res
magnetics=flap.get_data('NSTX_MDSPlus',
name='\OPS_PC::\\BDOT_L1DMIVVHF'+str(channel)+'_RAW',
exp_id=139901,
object_name='MIRNOV')
magnetics.coordinates.append(copy.deepcopy(flap.Coordinate(name='Time equi',
unit='s',
mode=flap.CoordinateMode(equidistant=True),
shape = [],
start=magnetics.coordinate('Time')[0][0],
step=magnetics.coordinate('Time')[0][1]-magnetics.coordinate('Time')[0][0],
dimension_list=[0])))
n_time=int((time_range[1]-time_range[0])/time_res)
spectrum=[]
for i in range(n_time-1):
spectrum.append(flap.apsd('MIRNOV',
coordinate='Time equi',
intervals={'Time equi':flap.Intervals(time_range[0]+(i-0.5)*time_res,
time_range[0]+(i+1.5)*time_res)},
options={'Res':freq_res,
'Range':frange,
'Interval':1,
'Trend':None,
'Logarithmic':False,
'Hanning':True},
output_name='MIRNOV_TWIN_APSD').data)
time=np.arange(n_time-1)*time_res+time_range[0]
freq=flap.get_data_object_ref('MIRNOV_TWIN_APSD').coordinate('Frequency')[0]
data=np.asarray(spectrum).T
pickle.dump((time,freq,data), open(filename, 'wb'))
else:
time, freq, data = pickle.load(open(filename, 'rb'))
if plot:
import matplotlib
matplotlib.use('QT5Agg')
import matplotlib.pyplot as plt
else:
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
if pdf:
filename=flap_nstx.analysis.filename(exp_id=exp_id,
working_directory=wd+'/plots',
time_range=time_range,
comment='magnetic_spectrogram_hf_ch'+str(channel)+'_tr_'+str(time_res)+'_frange_'+str(frange[0])+'_'+str(frange[1]),
extension='pdf')
spectrogram_pdf=PdfPages(filename)
plt.figure()
plt.contourf(time,
freq/1000.,
data,
locator=ticker.LogLocator(),
cmap='jet',
levels=101)
plt.title('BDOT_L1DMIVVHF'+str(channel)+' spectrogram for '+str(exp_id)+' with fres '+str(1/time_res/1000.)+'kHz')
plt.xlabel('Time [s]')
plt.ylabel('Frequency [kHz]')
plt.pause(0.001)
if pdf:
spectrogram_pdf.savefig()
spectrogram_pdf.close()
def add_coordinate(data_object, coordinates, exp_id=None, options=None):
#This part of the code provides normalized flux coordinates for the GPI data
if ('Flux r' in coordinates):
try:
gpi_time = data_object.coordinate('Time')[0][:, 0, 0]
gpi_r_coord = data_object.coordinate('Device R')[0]
gpi_z_coord = data_object.coordinate('Device z')[0]
except:
raise ValueError('R,z or t coordinates are missing.')
try:
psi_rz_obj = flap.get_data('NSTX_MDSPlus',
name='\EFIT02::\PSIRZ',
exp_id=data_object.exp_id,
object_name='PSIRZ_FOR_COORD')
psi_mag = flap.get_data('NSTX_MDSPlus',
name='\EFIT02::\SSIMAG',
exp_id=data_object.exp_id,
object_name='SSIMAG_FOR_COORD')
psi_bdry = flap.get_data('NSTX_MDSPlus',
name='\EFIT02::\SSIBRY',
exp_id=data_object.exp_id,
object_name='SSIBRY_FOR_COORD')
except:
raise ValueError("The PSIRZ MDSPlus node cannot be reached.")
psi_values = psi_rz_obj.data
psi_t_coord = psi_rz_obj.coordinate('Time')[0][:, 0, 0]
psi_r_coord = psi_rz_obj.coordinate('Device R')[0]
psi_z_coord = psi_rz_obj.coordinate('Device z')[0]
#Do the interpolation
psi_values_spat_interpol = np.zeros(
[psi_t_coord.shape[0], gpi_r_coord.shape[1], gpi_r_coord.shape[2]])
for index_t in range(psi_t_coord.shape[0]):
points = np.asarray([
psi_r_coord[index_t, :, :].flatten(),
psi_z_coord[index_t, :, :].flatten()
]).transpose()
psi_values_spat_interpol[index_t, :, :] = interpolate.griddata(
points,
psi_values[index_t, :, :].transpose().flatten(),
np.asarray([
gpi_r_coord[0, :, :].flatten(),
gpi_z_coord[0, :, :].flatten()
]).transpose(),
method='cubic').reshape(
psi_values_spat_interpol[index_t, :, :].shape)
psi_values_spat_interpol[index_t, :, :] = (
psi_values_spat_interpol[index_t, :, :] - psi_mag.data[index_t]
) / (psi_bdry.data[index_t] - psi_mag.data[index_t])
psi_values_spat_interpol[np.isnan(psi_values_spat_interpol)] = 0.
psi_values_total_interpol = np.zeros(data_object.data.shape)
for index_r in range(gpi_r_coord.shape[1]):
for index_z in range(gpi_r_coord.shape[2]):
psi_values_total_interpol[:, index_r, index_z] = np.interp(
gpi_time, psi_t_coord, psi_values_spat_interpol[:, index_r,
index_z])
psi_values_total_interpol[np.isnan(psi_values_total_interpol)] = 0.
new_coordinates = (copy.deepcopy(
flap.Coordinate(name='Flux r',
unit='',
mode=flap.CoordinateMode(equidistant=False),
values=psi_values_total_interpol,
shape=psi_values_total_interpol.shape,
dimension_list=[0, 1, 2])))
data_object.coordinates.append(new_coordinates)
if ('Flux theta' in coordinates):
print("ADDING FLUX THETA IS DEPRECATED AND MIGHT FAIL.")
try:
gpi_time = data_object.coordinate('Time')[0][:, 0, 0]
gpi_r_coord = data_object.coordinate('Device R')[0]
gpi_z_coord = data_object.coordinate('Device z')[0]
except:
raise ValueError('R,z or t coordinates are missing.')
try:
psi_rz_obj = flap.get_data('NSTX_MDSPlus',
name='\EFIT01::\PSIRZ',
exp_id=data_object.exp_id,
object_name='PSIRZ_FOR_COORD')
r_mag_axis = flap.get_data('NSTX_MDSPlus',
name='\EFIT01::\RMAXIS',
exp_id=data_object.exp_id,
object_name='RMAXIS_FOR_COORD')
z_mag_axis = flap.get_data('NSTX_MDSPlus',
name='\EFIT01::\ZMAXIS',
exp_id=data_object.exp_id,
object_name='ZMAXIS_FOR_COORD')
r_bdry_obj = flap.get_data('NSTX_MDSPlus',
name='\EFIT01::\RBDRY',
exp_id=data_object.exp_id,
object_name='SEP X OBJ')
z_bdry_obj = flap.get_data('NSTX_MDSPlus',
name='\EFIT01::\ZBDRY',
exp_id=data_object.exp_id,
object_name='SEP Y OBJ')
except:
raise ValueError("The PSIRZ MDSPlus node cannot be reached.")
try:
#if True:
psi_values = psi_rz_obj.data
psi_t_coord = psi_rz_obj.coordinate('Time')[0][:, 0, 0]
psi_r_coord = psi_rz_obj.coordinate('Device R')[0]
psi_z_coord = psi_rz_obj.coordinate('Device z')[0]
r_bdry = r_bdry_obj.data
z_bdry = z_bdry_obj.data
r_maxis = r_mag_axis.data
z_maxis = z_mag_axis.data
except:
raise ValueError("The flux data cannot be found.")
nlevel = 101
angle_values_spat_interpol = np.zeros(
[psi_t_coord.shape[0], gpi_r_coord.shape[1], gpi_r_coord.shape[2]])
for index_t in range(psi_t_coord.shape[0]):
poloidal_coord = np.zeros([0, 3])
#Get the contour plot paths of the constant psi surfaces
psi_contour = plt.contour(psi_r_coord[index_t, :, :].transpose(),
psi_z_coord[index_t, :, :].transpose(),
psi_values[index_t, :, :],
levels=nlevel)
for index_collections in range(
len(psi_contour.collections
)): #Iterate through all the constant surfaces
n_paths = len(
psi_contour.collections[index_collections].get_paths()
) #Get the actual paths, there might be more for one constant outside the separatrix.
if n_paths > 0:
for index_paths in range(n_paths):
path = psi_contour.collections[
index_collections].get_paths(
)[index_paths].vertices # Get the actual coordinates of the curve.
#The following code calculates the angles. The full arclength is calculated along with the partial arclengths.
# The angle is then difined as the fraction of the arclength fraction and the entire arclength.
arclength = 0.
current_path_angles = []
arclength_0 = np.sqrt((path[0, 0] - path[-1, 0])**2 +
(path[0, 1] - path[-1, 1])**2)
for index_path_points in range(-1,
len(path[:, 0]) - 1):
arclength += np.sqrt(
(path[index_path_points + 1, 0] -
path[index_path_points, 0])**2 +
(path[index_path_points + 1, 1] -
path[index_path_points, 1])**2)
current_path_angles.append([
path[index_path_points + 1,
0], path[index_path_points + 1, 1],
arclength - arclength_0
])
current_path_angles = np.asarray(current_path_angles)
min_ind = 0
#The angles needs to be measured from the midplane. Hence a correction rotation subtracted from the angle.
for index_path_points in range(len(path[:, 0])):
if ((np.abs(path[index_path_points, 1] -
z_maxis[index_t]) <
np.abs(path[min_ind, 1] - z_maxis[index_t]))
and (path[index_path_points, 0] >
r_maxis[index_t])):
min_ind = index_path_points
rotation = (current_path_angles[min_ind, 2] /
(arclength - arclength_0)) * 2. * np.pi
current_path_angles[:, 2] = (
current_path_angles[:, 2] /
(arclength - arclength_0)) * 2 * np.pi
current_path_angles[:, 2] = -1 * (
current_path_angles[:, 2] - rotation)
#The angles are corrected to be between -Pi and +Pi
for i_angle in range(len(current_path_angles[:, 2])):
if current_path_angles[i_angle, 2] > np.pi:
current_path_angles[i_angle, 2] -= 2 * np.pi
if current_path_angles[i_angle, 2] < -np.pi:
current_path_angles[i_angle, 2] += 2 * np.pi
poloidal_coord = np.append(poloidal_coord,
current_path_angles,
axis=0)
points = poloidal_coord[:, (1, 0)]
values = poloidal_coord[:, 2]
#No point of having coordinates outside the separatrix
#The following needs to be done:
#1. Calculate the angles at 90% of the separatrix
boundary_fraction = 0.90
boundary_data = np.asarray([r_bdry[index_t], z_bdry[index_t]])
points_at_fraction = np.asarray([
(boundary_data[0, :] - r_maxis[index_t]) * boundary_fraction +
r_maxis[index_t],
(boundary_data[1, :] - z_maxis[index_t]) * boundary_fraction +
z_maxis[index_t]
])
values_at_fraction = interpolate.griddata(
points,
values, (points_at_fraction[1, :], points_at_fraction[0, :]),
method='cubic')
values_at_fraction[np.isnan(values_at_fraction)] = 0.
#2. Redefine the poloidal coord vector to only have points inside the 95% of the separatrix
bfrac_path = []
for index_bdry in range(len(points_at_fraction[0, :])):
bfrac_path.append((points_at_fraction[0, index_bdry],
points_at_fraction[1, index_bdry]))
bfrac_path = pltPath.Path(bfrac_path)
poloidal_coord_new = np.zeros([0, 3])
poloidal_coord_new = [[0, 0, 0]]
for index_poloidal_coord in range(len(poloidal_coord[:, 0])):
if (bfrac_path.contains_point(
poloidal_coord[index_poloidal_coord, (0, 1)])):
poloidal_coord_new.append(
poloidal_coord[index_poloidal_coord, :])
poloidal_coord_new = np.asarray(poloidal_coord_new)[1:-1]
if (len(poloidal_coord_new[:, 0]) < 4):
points = poloidal_coord[:, (1, 0)]
values = poloidal_coord[:, 2]
else:
#3. Add points to poloidal coord vector with some resolution (close to the EFIT resolution)
for index_expansion in range(21):
zoom = (index_expansion * 0.05) + 1.
expanded_points = np.asarray([
(boundary_data[0, :] - r_maxis[index_t]) * zoom +
r_maxis[index_t],
(boundary_data[1, :] - z_maxis[index_t]) * zoom +
z_maxis[index_t]
])
new_points = np.asarray([
expanded_points[0, :], expanded_points[1, :],
values_at_fraction
])
poloidal_coord_new = np.append(poloidal_coord_new,
new_points.transpose(),
axis=0)
points = poloidal_coord_new[:, (1, 0)]
values = poloidal_coord_new[:, 2]
#4. Do the interpolation with the redefined poloidal coordinates.
#The angles are calculated for the GPI's coordinate frame.
angle_values_spat_interpol[index_t, :, :] = interpolate.griddata(
points,
values, (gpi_r_coord[0, :, :].transpose(),
gpi_z_coord[0, :, :].transpose()),
method='cubic').transpose()
if (options is not None and options['Debug']):
#if True:
plt.cla()
plt.tricontourf(points[:, 1], points[:, 0], values, levels=51)
#plt.colorbar()
plt.scatter(gpi_r_coord[0, :, :], gpi_z_coord[0, :, :])
plt.scatter(r_bdry[index_t, :], z_bdry[index_t, :])
plt.axis('equal')
plt.show()
plt.pause(1)
#Temporal interpolation for the angle values
angle_values_total_interpol = np.zeros(data_object.data.shape)
for index_r in range(gpi_r_coord.shape[1]):
for index_z in range(gpi_r_coord.shape[2]):
angle_values_total_interpol[:, index_r, index_z] = np.interp(
gpi_time, psi_t_coord,
angle_values_spat_interpol[:, index_r, index_z])
plt.cla()
new_coordinates = (copy.deepcopy(
flap.Coordinate(name='Flux theta',
unit='rad',
mode=flap.CoordinateMode(equidistant=False),
values=angle_values_total_interpol,
shape=angle_values_total_interpol.shape,
dimension_list=[0, 1, 2])))
data_object.coordinates.append(new_coordinates)
if ('Device theta' in coordinates):
try:
gpi_time = data_object.coordinate('Time')[0][:, 0, 0]
gpi_r_coord = data_object.coordinate('Device R')[0]
gpi_z_coord = data_object.coordinate('Device z')[0]
except:
raise ValueError('R,z or t coordinates are missing.')
try:
r_mag_axis = flap.get_data('NSTX_MDSPlus',
name='\EFIT01::\RMAXIS',
exp_id=data_object.exp_id,
object_name='RMAXIS_FOR_COORD')
z_mag_axis = flap.get_data('NSTX_MDSPlus',
name='\EFIT01::\ZMAXIS',
exp_id=data_object.exp_id,
object_name='ZMAXIS_FOR_COORD')
except:
raise ValueError("The PSIRZ MDSPlus node cannot be reached.")
try:
#if True:
t_maxis = r_mag_axis.coordinate('Time')[0]
r_maxis = r_mag_axis.data
z_maxis = z_mag_axis.data
except:
raise ValueError("The flux data cannot be found.")
r_maxis_at_gpi_range = np.interp(gpi_time, t_maxis, r_maxis)
z_maxis_at_gpi_range = np.interp(gpi_time, t_maxis, z_maxis)
poloidal_angles = np.zeros(gpi_r_coord.shape)
for index_t in range(gpi_time.shape[0]):
y = gpi_z_coord[index_t, :, :] - z_maxis_at_gpi_range[index_t]
x = gpi_r_coord[index_t, :, :] - r_maxis_at_gpi_range[index_t]
poloidal_angles[index_t, :, :] = np.arctan(y / x)
new_coordinates = (copy.deepcopy(
flap.Coordinate(name='Device theta',
unit='rad',
mode=flap.CoordinateMode(equidistant=False),
values=poloidal_angles,
shape=poloidal_angles.shape,
dimension_list=[0, 1, 2])))
data_object.coordinates.append(new_coordinates)
return data_object
def mdsplus_get_data(exp_id=None,
data_name=None,
no_data=False,
options=None,
coordinates=None,
data_source=None):
""" Data read function for the MDSplus database
exp_id: exp_id number, integer or YYYYMMDD.xxx
data_name: Channel names [\]tree::node
or virtual names:
CR-x: Correlation reflectometry, x is antenna A,B,C,D
coordinates: List of flap.Coordinate() or a single flap.Coordinate
Defines read ranges. The following coordinates are interpreted:
'Sample': The read samples
'Time': The read times
Only a single range is interpreted.
options: Dictionary. Defaults will be read from <data_source> section in configuration file.
'Protocol': For ssh connection use 'ssh://'.
'Server': Server name (default: mds-trm-1.ipp-hgw.mpg.de)
'User': User name for access. Password-free access should be set up for this user.
'Virtual name file': A file name to translate virtual names to MDS+ entries. For
format see mds_virtual_names()
'Verbose': (bool) Write progress information during data read.
'Cache data': (bool) Cache data to options['Cache directory'] and read it from there
'Cache directory': (str) Name of the cache directory
'Time unit': The time unit of the returned data. Default: s
The saved data is in the original units.
'MDS time unit': The time unit used in reading data from the MDS tree. This is
useful when the MDS tree does not have a proper time unit.
"""
if (exp_id is None):
raise ValueError('exp_id should be set for reading mdsplus data.')
default_options = {'Server': None,
'Protocol': None,
'User': None,
'Virtual name file': None,
'Verbose': True,
'Cache data': False,
'Cache directory': None,
'Time unit':'s',
'MDS time unit' : None
}
_options = flap.config.merge_options(default_options,options,data_source=data_source)
if (data_source is None):
data_source = 'MDSPlus'
if (exp_id is None):
raise ValueError("exp_id must be set for reading data from MDSPlus.")
if (type(exp_id) is not int):
raise TypeError("exp_is must be an integer for MDSPlus.")
exp_id_mds = exp_id
if (_options['Server'] is None):
raise ValueError("Option 'Server' should be set for using MDSPlus.")
#if no username and protocol then open a server (e.g. NSTX or KSTAR)
if ((_options['Protocol'] is None) and (_options['User'] is None)):
connection_name = _options['Server']
#if protocol and username exists then use the following syntax (e.g. W7-X)
if ((_options['Protocol'] is not None) and (_options['User'] is not None)):
connection_name = _options['Protocol'] + _options['User'] + '@' + _options['Server']
#Error handing if one of the parameters is not set.
if (((_options['Protocol'] is not None) and (_options['User'] is None)) or
((_options['Protocol'] is None) and (_options['User'] is not None))):
raise ValueError("If Protocol is set then Username must be set, as well.")
if ((type(data_name) is not str) and (type(data_name) is not list)):
raise ValueError("data_name should be a string or list of strings.")
if (_options['Virtual name file'] is not None):
try:
if _options['Virtual name file'] == 'self':
_options['Virtual name file'] = flap.config.__flap_config.file_name
virt_names, virt_mds_txt, virt_mds = mds_virtual_names(data_name, exp_id, _options['Virtual name file'])
except Exception as e:
raise e
else:
print(_options)
raise ValueError('No virtual names available.')
read_range = None
read_samplerange = None
if (coordinates is not None):
if (type(coordinates) is not list):
_coordinates = [coordinates]
else:
_coordinates = coordinates
for coord in _coordinates:
if (type(coord) is not flap.Coordinate):
raise TypeError("Coordinate description should be flap.Coordinate.")
if (coord.unit.name == 'Time'):
if (coord.mode.equidistant):
read_range = [float(coord.c_range[0]),float(coord.c_range[1])]
read_range_unit = coord.unit.unit
if ((read_range_unit == '') or (read_range_unit is None)):
read_range_unit = _options['Time unit']
else:
raise NotImplementedError("Non-equidistant Time axis is not implemented yet.")
break
if (coord.unit.unit == 'Millisecond'):
read_range = [read_range[0]*1e-3, read_range[1]*1e-3]
elif (coord.unit.unit == 'Microsecond'):
read_range = [read_range[0]*1e-6, read_range[1]*1e-6]
elif (coord.unit.unit == 'Nanosecond'):
read_range = [read_range[0]*1e-9, read_range[1]*1e-9]
elif (coord.uni.unit == 'Second'):
pass
else:
raise ValueError("Unknown time unit '"+coord.unit.unit+"'. Valid: Second, Millisecond, Microsecond, Nanosecond.")
signal_list = []
data_list = []
common_time = None
for name, mds_descr in zip(virt_names,virt_mds):
# Assembling a list of MDS nodes needed for this data
mds_request_list = []
if (name is None):
# This was not recognized as virtual name
mds_request_list = [mds_descr]
signal_list.append(mds_descr)
readtype = 0
else:
# This was recongnized as a virtual signal
signal_list.append(name)
if (type(mds_descr) is not list):
# This was recognized as a single MDS node
mds_request_list = [mds_descr]
readtype = 0
else:
# This is a composite virtual signal
mds_request_list = mds_descr[1:]
readtype = 1
# Reading the required nodes
this_data_list = []
for mds_name in mds_request_list:
mds_name_split = mds_name.split('::')
if (len(mds_name_split) != 2):
raise ValueError("Invalid mds name '{:s}', missing tree name? Data name is tree::node".format(mds_name))
tree_name = mds_name_split[0].strip()
if (tree_name[0] == '\\'):
tree_name = tree_name[1:]
node_name = mds_name_split[1]
if ((_options['Cache data']) and (_options['Cache directory'] != None)):
filename = str(exp_id)+'_'+mds_name
for c in ['\\',':']:
filename = filename.replace(c,'_')
#filename = os.path.join(_options['Cache directory'],filename+'.pickle')
directory=os.path.join(_options['Cache directory'],str(exp_id))
if not (os.path.exists(directory)):
try:
os.mkdir(directory)
except:
raise SystemError("The shot folder cannot be created. Cache directory might not be present.")
filename = os.path.join(directory,filename+'.pickle')
try:
f = io.open(filename,'rb')
mdsdata_pickle = pickle.load(f)
f.close()
try:
if (mdsdata_pickle['MDSdata cache']):
mdsdata = mdsdata_pickle['Data']
mdsdata_unit = mdsdata_pickle['Data unit']
mdsdata_spat = mdsdata_pickle['Data dimension']
mdsdata_spat_unit = mdsdata_pickle['Data dimension unit']
mdsdata_time = mdsdata_pickle['Time']
mdsdata_time_unit = mdsdata_pickle['Time unit']
mdsdata_time_step = mdsdata_pickle['Time step']
del mdsdata_pickle
data_cached = True
except:
data_cached = False
except:
data_cached = False
else:
data_cached = False
if (not data_cached):
try:
conn
except NameError:
try:
if (_options['Verbose']):
print("Connecting to "+connection_name)
conn = mds.Connection(connection_name)
except Exception as e:
raise e
try:
conn.openTree(tree_name,exp_id_mds)
except mds.MdsException as e:
raise RuntimeError("Error connecting to tree {:s}, experiment {:s}".format(tree_name,str(exp_id)))
if (_options['Verbose']):
print("Reading "+node_name)
try:
mdsdata = conn.get(node_name).data()
mdsdata_unit = conn.get('units('+node_name+')').data()
mdsdata_spat=[]
mdsdata_spat_unit=[]
for dim_ind in range(1,len(mdsdata.shape)):
mdsdata_spat.append(conn.get('dim_of('+node_name+','+str(dim_ind)+')').data())
mdsdata_spat_unit.append(conn.get('units(dim_of('+node_name+','+str(dim_ind)+'))').data())
mdsdata_time = conn.get('dim_of('+node_name+',0)').data()
mdsdata_time_unit = conn.get('units(dim_of('+node_name+'))').data()
if (_options['MDS time unit'] is not None):
mdsdata_time_unit = _options['MDS time unit']
if (mdsdata_time_unit ==' ') or (mdsdata_time_unit is None):
raise ValueError("Unknown time unit.")
if not (len(mdsdata_time) < 2):
mdsdata_time_step=mdsdata_time[1]-mdsdata_time[0]
else:
mdsdata_time_step=0.
except:
raise RuntimeError("Cannot read MDS node: {:s}".format(node_name))
if (not data_cached and (_options['Cache data']) and (_options['Cache directory'] is not None)):
while True:
try:
f = io.open(filename,"wb")
except:
print("Warning: Cannot open cache file: "+filename)
break
mdsdata_pickle = {}
mdsdata_pickle['MDSdata cache'] = True
mdsdata_pickle['Data'] = copy.deepcopy(mdsdata)
mdsdata_pickle['Data unit'] = mdsdata_unit
mdsdata_pickle['Data dimension'] = mdsdata_spat
mdsdata_pickle['Data dimension unit'] = mdsdata_spat_unit
mdsdata_pickle['Time'] = mdsdata_time
mdsdata_pickle['Time unit'] = mdsdata_time_unit
mdsdata_pickle['Time step'] = mdsdata_time_step
try:
pickle.dump(mdsdata_pickle,f)
del mdsdata_pickle
except Exception as e:
print("Warning: Cannot write cache file: "+filename)
break
try:
f.close()
except Exception as e:
print("Warning: Cannot write cache file: "+filename)
break
mdsdata_time_unit_int = flap.tools.time_unit_translation(mdsdata_time_unit, max_value=mdsdata_time.max())
if (mdsdata_time_unit ==' ') or (mdsdata_time_unit is None):
mdsdata_time_unit = 's'
if (read_range is not None):
output_time_unit_int = flap.tools.time_unit_translation(read_range_unit)
read_ind = np.nonzero(np.logical_and(mdsdata_time * mdsdata_time_unit_int >= read_range[0] * output_time_unit_int,
mdsdata_time * mdsdata_time_unit_int <= read_range[1] * output_time_unit_int
)
)[0]
if (len(read_ind) == 0):
raise ValueError("No data in time interval.")
mdsdata_time = mdsdata_time[read_ind]
mdsdata = mdsdata[read_ind]
else:
read_ind = [0, len(mdsdata)]
if (common_time is not None):
if ((len(common_time) != len(mdsdata_time)) or \
(math.fabs(common_time_step - mdsdata_time_step) / common_time_step > 1e-3) or \
(math.fabs(mdsdata_time_unit_int != common_time_unit_int) / common_time_unit_int > 1e-3) or \
(np.nonzero(np.abs(common_time - mdsdata_time) \
> math.fabs(common_time[1] - common_time[0]) * 0.1)[0].size != 0)
):
raise ValueError("Different timescales for signals. Not possible to return in one flap.DataObject.")
else:
common_time = mdsdata_time
common_time_unit_int = mdsdata_time_unit_int
common_time_step = mdsdata_time_step
this_data_list.append(mdsdata)
del mdsdata
if (readtype == 0):
data_list.append(this_data_list[0])
elif (readtype == 1):
data_list.append(this_data_list[0] + 1j * this_data_list[1])
# Determining data type
dtype = int
for i in range(len(data_list)):
if (dtype is not complex) and (data_list[i].dtype.kind == 'f'):
dtype = float
if (data_list[i].dtype.kind == 'c'):
dtype = complex
if (len(data_list) == 1):
data = data_list[0]
signal_dim = []
else:
data = np.empty((len(data_list[0]),len(data_list)),dtype=dtype)
for i in range(len(data_list)):
data[:,i] = data_list[i].astype(dtype)
signal_dim = [1]
dt = (common_time[1:] - common_time[:-1])
if (np.nonzero(np.abs(dt - dt[0]) / dt[0] > 0.01)[0].size != 0):
coord_type = flap.CoordinateMode(equidistant=False)
else:
coord_type = flap.CoordinateMode(equidistant=True)
if flap.tools.time_unit_translation(_options['Time unit']) != flap.tools.time_unit_translation(mdsdata_time_unit):
new_time_unit_int = flap.tools.time_unit_translation(_options['Time unit'])
output_time_unit_scaling = common_time_unit_int / new_time_unit_int
time_unit=_options['Time unit']
else:
time_unit=mdsdata_time_unit
output_time_scaling = 1
coord = []
if (coord_type.equidistant):
coord.append(copy.deepcopy(flap.Coordinate(name='Time',
unit=time_unit,
mode=coord_type,
shape = [],
start=common_time[0] * output_time_unit_scaling,
step= common_time_step * output_time_unit_scaling ,
dimension_list=[0])
))
else:
coord.append(copy.deepcopy(flap.Coordinate(name='Time',
unit=time_unit,
mode=coord_type,
values=common_time * output_time_unit_scaling,
shape = common_time.shape,
dimension_list=[0])
))
if ((len(data.shape) > 1) and (len(np.asarray(mdsdata_spat).shape) > 2)):
for dim_ind in range(0,len(data.shape)-1):
#now only the zeroth time is gotten from the vector
spatial_data=np.asarray(mdsdata_spat[dim_ind][0][:])
if (((mdsdata_spat_unit[dim_ind] == 'm') or
(mdsdata_spat_unit[dim_ind] == 'mm')) and
(dim_ind < 3)):
names=['Device R','Device z']
coord.append(copy.deepcopy(flap.Coordinate(name=names[dim_ind],
unit=mdsdata_spat_unit[dim_ind],
mode=flap.CoordinateMode(equidistant=False),
values=spatial_data,
shape=spatial_data.shape,
dimension_list=[dim_ind+1])))
else:
coord.append(copy.deepcopy(flap.Coordinate(name='Dimension '+str(dim_ind+1),
unit=mdsdata_spat_unit[dim_ind],
mode=flap.CoordinateMode(equidistant=False),
values=spatial_data,
shape=spatial_data.shape,
dimension_list=[dim_ind+1])))
coord.append(copy.deepcopy(flap.Coordinate(name='Sample',
unit=mdsdata_unit,
mode=flap.CoordinateMode(equidistant=True),
start=read_ind[0],
step=1,
dimension_list=[0])
#dimension_list=[0])
))
coord.append(copy.deepcopy(flap.Coordinate(name='Signal name',
shape=tuple([len(signal_list)]),
unit='',
mode=flap.CoordinateMode(equidistant=False),
values=signal_list,
dimension_list=signal_dim)
))
coord.append(copy.deepcopy(flap.Coordinate(name='Channel',
shape=tuple([len(signal_list)]),
unit='',
mode=flap.CoordinateMode(equidistant=False),
values=np.arange(len(signal_list)) + 1,
dimension_list=signal_dim)))
coord.append(copy.deepcopy(flap.Coordinate(name='mds description',
shape=tuple([len(virt_mds_txt)]),
unit='',
mode=flap.CoordinateMode(equidistant=False),
values=virt_mds_txt,
dimension_list=signal_dim)))
data_title = data_source +' data'
d = flap.DataObject(data_array=data,
data_unit=flap.Unit(name=''.join(signal_list),unit=mdsdata_unit),
coordinates=coord,
exp_id=exp_id,
data_title=data_title,
data_source=data_source)
return d
def w7x_camera_get_data(exp_id=None,
data_name=None,
no_data=False,
options=None,
coordinates=None,
data_source=None):
""" Data read function for the W7-X EDICAM and Photron cameras (HDF5 format)
data_name: Usually AEQ21_PHOTRON_ROIPx, ... (string) depending on configuration file
exp_id: Experiment ID, YYYYMMDD.xxx, e.g. 20181018.016
Options:
Datapath: the base path at which the camera files can be found (e.g. /data/W7X)
Time: the date and time the recording was made: 123436 (12:34:36)
Camera name: either EDICAM or PHOTRON
Port: the port number the camera was used, e.g. AEQ20
"""
default_options = {
'Datapath': 'data',
'Timing path': 'data',
'Time': None,
'Max_size': 4 # in GB!
}
_options = flap.config.merge_options(default_options,
options,
data_source='W7X_CAMERA')
name_split = data_name.split("_")
port = name_split[0]
cam_name = name_split[1].upper()
roi_num = name_split[2]
datapath = _options['Datapath']
time = _options['Time']
max_size = _options['Max_size']
timing_path = _options['Timing path']
if (coordinates is None):
_coordinates = []
else:
if (type(coordinates) is not list):
_coordinates = [coordinates]
else:
_coordinates = coordinates
if port is None:
raise ValueError(
"Port name and number should be set for W7X camera! (E.g. AEQ20)")
elif 'aeq' not in port.lower():
raise ValueError("Port name should contain AEQ!")
if cam_name is None:
raise ValueError("Camera name should be set for W7X camera!")
elif cam_name.lower() == "edicam":
cam_str = "edi"
elif cam_name.lower() == "photron":
cam_str = "phot"
else:
raise ValueError(
"Camera name should be either EDICAM or PHOTRON, not {}.".format(
cam_name))
if (exp_id is None):
raise ValueError('Both exp_id should be set for W7X camera.')
exp_id_split = exp_id.split('.')
date = exp_id_split[0]
exp_num = exp_id_split[1]
dp = os.path.join(datapath, cam_name.upper(), port.upper(), date)
dp_timing = os.path.join(timing_path, date)
flist = os.listdir(dp)
if (time is None):
filename_mask = "_".join(
[port.lower(),
cam_str.lower(), date, exp_num, ("*.h5")])
else:
filename_mask = "_".join(
[port.lower(),
cam_str.lower(), date, exp_num, (time + ".h5")])
fnames = fnmatch.filter(flist, filename_mask)
if (len(fnames) > 1):
if (time is not None):
raise ValueError(
"Multiple files found, 'Time' option should be set?")
else:
raise ValueError("Multiple files found:{:s}.".format(
os.path.join(dp, filename_mask)))
elif (len(fnames) == 0):
if (time is not None):
filename_mask = "_".join(
[port.lower(),
cam_str.lower(), date, (time + ".h5")])
fnames = fnmatch.filter(flist, filename_mask)
if (len(fnames) == 0):
raise ValueError("Cannot find any file for this measurement.")
else:
time = fnames[0].split('_')[3]
time = time.split('.')[0]
else:
raise ValueError(
"Cannot find file without time parameter. Filename mask:" +
filename_mask + " dp:" + dp)
else:
time = fnames[0].split('_')[4]
time = time.split('.')[0]
path = os.path.join(dp, fnames[0])
if (cam_name == 'EDICAM'):
# Getting the file info
with h5py.File(path, 'r') as h5_obj:
try:
info = get_camera_config_h5(h5_obj, roi_num)
except Exception as e:
print("Camera config is not found:")
print(e)
try:
info = get_camera_config_ascii(path)
except Exception as e:
print("Cannot read the info file!")
print(e)
finally:
if info is None:
info = dict()
print(info)
# Read the time vectors
with h5py.File(path, 'r') as h5_obj:
try:
time_vec_etu = np.array(h5_obj['ROIP']['{}'.format(
roi_num.upper())]['{}ETU'.format(roi_num.upper())])
#print("ETU time vector found!")
except Exception as e:
print("Cannot read ETU! Error message:")
print(e)
time_vec_etu = None
try:
time_vec_w7x = np.array(h5_obj['ROIP']['{}'.format(
roi_num.upper())]['{}W7XTime'.format(roi_num.upper())])
#print("W7-X time vector found!")
except Exception as e:
print("Cannot read W7-X time units (ns)! Error message:")
print(e)
time_vec_w7x = None
if time_vec_w7x is not None:
print(
"Using W7-X time vector [ns] for time vector [s] calculation!"
)
time_vec_sec = (time_vec_w7x - time_vec_w7x[0]) / 1.e9
elif time_vec_etu is not None:
print(
"Using ETU time vector [100 ns] for time vector [s] calculation!"
)
time_vec_sec = (time_vec_etu - time_vec_etu[0]) / 1.e7
else:
print("Cannot find any meaningful time vector!")
print("Exiting...")
raise IOError("No time vector found!")
# Open the file and check the data path and read the data
try:
h5_obj = h5py.h5f.open(path.encode('utf-8'))
# print(roi_num)
h5_path = '/ROIP/{}/{}Data'.format(roi_num.upper(),
roi_num.upper())
# print("Opening {} with path {}.".format(roi_num.upper(), h5_path))
h5_data = h5py.h5d.open(h5_obj, h5_path.encode('utf-8'))
# print("Data size: {}.".format(h5_data.shape))
except Exception as e:
print("Something bad happened:")
print(e)
# Read the data
data_space = h5_data.get_space()
dims = data_space.shape
if coordinates == None:
# Indices contain everything!
frame_vec = np.arange(0, dims[2])
else:
# Take indices from the coordinates!
# Only time coordinates are working as of now (2019. June 11.)
for coord in _coordinates:
if (type(coord) is not flap.Coordinate):
raise TypeError(
"Coordinate description should be flap.Coordinate.")
if (coord.unit.name is
'Time'): # assuming the unit to be Second
# if (coord.unit.unit is not 'Second'):
# raise NotImplementedError("Your time coordinate unit is not in Seconds! Cannot use it (yet).")
if (coord.c_range is None):
raise NotImplementedError(
"At present only simple tie range selection is supported."
)
read_range = [
float(coord.c_range[0]),
float(coord.c_range[1])
]
# Since np.where gives back indices, it is the same as the frame_vec
n_frames = len(time_vec_sec)
frame_vec = np.where((time_vec_sec >= read_range[0])
& (time_vec_sec <= read_range[1]))[0]
time_vec_sec = time_vec_sec[frame_vec]
time_vec_etu = time_vec_etu[frame_vec]
time_vec_w7x = time_vec_w7x[frame_vec]
else:
raise NotImplementedError(
"Coordinate selection for image coordinates is not supported yet."
)
dt = time_vec_sec[1:] - time_vec_sec[0:-1]
if (np.nonzero((np.abs(dt[0] - dt) / dt[0]) > 0.001)[0].size == 0):
time_equidistant = True
time_step = dt[0]
time_start = time_vec_sec[0]
else:
time_equidistant = False
# TODO: make this for the spatial coordinates as well! (Binning etc.)
x = (0, dims[0])
y = (0, dims[1])
# We will set data_shape in flap.DataObject to show what the shape would be if data was read
if (no_data):
data_arr = None
data_shape = (dims[0], dims[1], len(frame_vec))
else:
file_size = os.path.getsize(path) # in bytes!
file_size = file_size / 1024**3 # in GB
fraction = len(frame_vec) / n_frames
if file_size * fraction > max_size:
print(
"The expected read size from {} is too large. (size: {} GB, limit: {} GB.)"
.format(path, file_size * fraction, max_size))
raise IOError("File size is too large!")
data_arr = read_hdf5_arr(h5_data, x, y, frame_vec)
data_shape = data_arr.shape
h5_obj.close()
elif (cam_name == 'PHOTRON'):
time_fn = os.path.join(
dp_timing, "_".join(
[port.lower(), cam_str, date, time, 'integ', ('v1' + ".sav")]))
time_fn = time_fn.replace(
'\\',
'/',
)
try:
idldat = io.readsav(time_fn, python_dict=True, verbose=False)
except IOError as e:
raise IOError("Error reading file {:s}.".format(time_fn))
time_vec_sec = idldat['resa'][0][4]
time_vec_etu = None
time_vec_w7x = None
try:
frame_per_trig = idldat['resa'][0][15]['frame_per_trig'][0]
except:
dt = time_vec_sec[1:] - time_vec_sec[:-1]
ind = np.nonzero(dt > dt[0] * 2)[0]
if (len(ind) != 0):
frame_per_trig = ind[0] + 1
else:
frame_per_trig = len(time_vec_sec)
rec_rate = idldat['resa'][0][15]['rec_rate'][0]
trig_times = []
for i in range(0, len(time_vec_sec), frame_per_trig):
trig_times.append(time_vec_sec[i])
trig_times = np.array(trig_times)
meas_end_times = trig_times + 1. / rec_rate * (frame_per_trig - 1)
# Open the file and check the data path and read the data
try:
h5_obj = h5py.h5f.open(path.encode('utf-8'))
# print(roi_num)
h5_path = '/ROIP/{}/{}Data'.format(roi_num.upper(),
roi_num.upper())
# print("Opening {} with path {}.".format(roi_num.upper(), h5_path))
h5_data = h5py.h5d.open(h5_obj, h5_path.encode('utf-8'))
# print("Data size: {}.".format(h5_data.shape))
except Exception as e:
print("Something bad happened:")
print(e)
data_space = h5_data.get_space()
dims = data_space.shape
if (dims[2] != len(time_vec_sec)):
RuntimeError(
"Frame number in HDF5 file and time file are different.")
n_frames = dims[2]
for coord in _coordinates:
if (type(coord) is not flap.Coordinate):
raise TypeError(
"Coordinate description should be flap.Coordinate.")
if (coord.unit.name is 'Time'): # assuming the unit to be Second
# if (coord.unit.unit is not 'Second'):
# raise NotImplementedError("Your time coordinate unit is not in Seconds! Cannot use it (yet).")
if (coord.c_range is None):
raise NotImplementedError(
"At present only simple tie range selection is supported."
)
read_range = [float(coord.c_range[0]), float(coord.c_range[1])]
# Since np.where gives back indices, it is the same as the frame_vec
frame_vec = np.nonzero(
np.logical_and((time_vec_sec >= read_range[0]),
(time_vec_sec < read_range[1])))[0]
if (len(frame_vec) == 0):
raise ValueError("No data in time range.")
start_block = int(frame_vec[0] // frame_per_trig)
end_block = int(frame_vec[-1] // frame_per_trig)
if (end_block == start_block):
time_equidistant = True
time_step = 1. / rec_rate
time_start = time_vec_sec[frame_vec[0]]
else:
time_equidistant = False
else:
raise NotImplementedError(
"Coordinate selection for image coordinates is not supported yet."
)
try:
frame_vec
except NameError:
frame_vec = np.arange(len(time_vec_sec), dtype=np.int32)
if (len(trig_times) != 1):
time_equidistant = False
else:
time_equidistant = True
x = (0, dims[0])
y = (0, dims[1])
info = {}
with h5py.File(path, 'r') as h5_obj_config:
try:
info['X Start'] = h5_obj_config['Settings']['X pos'][0]
info['Y Start'] = h5_obj_config['Settings']['Y pos'][0]
except Exception as e:
raise IOError(
"Could not find ROI x and y position in HDF5 file.")
# We will set data_shape in flap.DataObject to show what the shape would be if data was read
if (no_data):
data_arr = None
data_shape = (dims[0], dims[1], len(frame_vec))
else:
file_size = os.path.getsize(path) # in bytes!
file_size = file_size / 1024**3 # in GB
fraction = len(frame_vec) / n_frames
if file_size * fraction > max_size:
print(
"The expected read size from {} is too large. (size: {} GB, limit: {} GB.)"
.format(path, file_size * fraction, max_size))
raise IOError("File size is too large!")
data_arr = read_hdf5_arr(h5_data, x, y, frame_vec)
data_arr = np.flip(data_arr, axis=0)
data_shape = data_arr.shape
h5_obj.close()
else:
raise ValueError("Invalid camera name.")
coord = []
# TODO: check for equidistant time coordinates!
if (time_equidistant):
coord.append(
copy.deepcopy(
flap.Coordinate(name='Time',
unit='Second',
mode=flap.CoordinateMode(equidistant=True),
start=time_start,
step=time_step,
shape=[],
dimension_list=[2])))
else:
coord.append(
copy.deepcopy(
flap.Coordinate(name='Time',
unit='Second',
mode=flap.CoordinateMode(equidistant=False),
values=time_vec_sec,
shape=time_vec_sec.shape,
dimension_list=[2])))
if (time_vec_etu is not None):
coord.append(
copy.deepcopy(
flap.Coordinate(name='ETUTime',
unit='ETU',
mode=flap.CoordinateMode(equidistant=False),
values=time_vec_etu,
shape=time_vec_etu.shape,
dimension_list=[2])))
if (time_vec_w7x is not None):
coord.append(
copy.deepcopy(
flap.Coordinate(name='W7XTime',
unit='Nanosecond',
mode=flap.CoordinateMode(equidistant=False),
values=time_vec_w7x,
shape=time_vec_w7x.shape,
dimension_list=[2])))
coord.append(
copy.deepcopy(
flap.Coordinate(name='Sample',
unit='',
mode=flap.CoordinateMode(equidistant=False),
values=frame_vec,
shape=frame_vec.shape,
dimension_list=[2])))
coord.append(
copy.deepcopy(
flap.Coordinate(name='Image x',
unit='Pixel',
mode=flap.CoordinateMode(equidistant=True),
start=int(info['X Start']),
step=int(1),
shape=[],
dimension_list=[0])))
coord.append(
copy.deepcopy(
flap.Coordinate(name='Image y',
unit='Pixel',
mode=flap.CoordinateMode(equidistant=True),
start=int(info['Y Start']),
step=int(1),
shape=[],
dimension_list=[1])))
data_title = "W7-X CAMERA data: {}".format(data_name)
d = flap.DataObject(data_array=data_arr,
data_shape=data_shape,
data_unit=flap.Unit(name='Frame', unit='Digit'),
coordinates=coord,
exp_id=exp_id,
data_title=data_title,
info={'Options': _options},
data_source="W7X_CAMERA")
return d
def nstx_gpi_generate_synthetic_data(exp_id=None, #Artificial exp_id, should be starting from zero and not the one which is used by e.g. background_shot
time=None, #Time to be simulated in seconds
sampling_time=2.5e-6, #The sampling time of the diagnostic
#General parameters
n_structures=3,
amplitude=0.5, #Amplitude of the structure relative to the background.
add_background=True,
background_shot=139901, #The original of the background of the simulated signal.
background_time_range=[0.31,0.32], #The time range of the background in the background_shot.
poloidal_velocity=1e3, #The poloidal velocity of the structures, can be a list [n_structure]
start_position=[1.41,0.195],
radial_size=0.3, #The radial size of the structures.
poloidal_size=0.1, #The poloidal size of the structures.
#Parameters for a gaussian object
gaussian=False,
radial_velocity=1e2, #The radial velocity of the structures, can be a list [n_structure]
poloidal_size_velocity=0., #The velocity of the size change in mm/ms
radial_size_velocity=0.,
rotation=False, #Set rotation for the structure.
rotation_frequency=None, #Set the frequency of the rotation of the structure.
#Parameters for sinusoidal object
sinusoidal=False,
waveform_divider=1,
y_lambda=0.05, #Wavelength in the y direction
output_name=None, #Output name of the generated flap.data_object
test=False, #Testing/debugging switch (mainly plotting and printing error messages)
):
if rotation_frequency is None:
rotation_frequency=0
n_time=int(time/sampling_time)
data_arr=np.zeros([n_time,64,80])
background=np.zeros([64,80])
if add_background:
background=flap.get_data('NSTX_GPI',
exp_id=139901,
name='',
object_name='GPI_RAW')
background=background.slice_data(slicing={'Time':flap.Intervals(background_time_range[0],
background_time_range[1])},
summing={'Time':'Mean'})
amplitude=amplitude*background.data.max()
#Spatial positions
coeff_r=np.asarray([3.7183594,-0.77821046,1402.8097])/1000. #The coordinates are in meters
coeff_z=np.asarray([0.18090118,3.0657776,70.544312])/1000. #The coordinates are in meters
r_coordinates=np.zeros([64,80])
z_coordinates=np.zeros([64,80])
for i_x in range(64):
for i_y in range(80):
r_coordinates[i_x,i_y]=coeff_r[0]*i_x+coeff_r[1]*i_y+coeff_r[2]
z_coordinates[i_x,i_y]=coeff_z[0]*i_x+coeff_z[1]*i_y+coeff_z[2]
if gaussian:
r0=start_position
for i_frames in range(n_time):
for i_structures in range(n_structures):
cur_time=i_frames * sampling_time
rot_arg=2*np.pi*rotation_frequency*cur_time
a=(np.cos(rot_arg)/(radial_size+radial_size_velocity*cur_time))**2+\
(np.sin(rot_arg)/(poloidal_size+poloidal_size_velocity*cur_time))**2
b=-0.5*np.sin(2*rot_arg)/(radial_size+radial_size_velocity*cur_time)**2+\
0.5*np.sin(2*rot_arg)/(poloidal_size+poloidal_size_velocity*cur_time)**2
c=(np.sin(rot_arg)/(radial_size+radial_size_velocity*cur_time))**2+\
(np.cos(rot_arg)/(poloidal_size+poloidal_size_velocity*cur_time))**2
x0=r0[i_structures,0]+radial_velocity[i_structures]*cur_time
y0=r0[i_structures,1]+poloidal_velocity[i_structures]*cur_time
frame=np.zeros([64,80])
for j_vertical in range(80):
for k_radial in range(64):
x=r_coordinates[k_radial,j_vertical]
y=z_coordinates[k_radial,j_vertical]
if (x > x0+radial_size*2 or
x < x0-radial_size*2 or
y > y0+radial_size*2 or
y < y0-radial_size*2):
frame[k_radial,j_vertical]=0.
else:
frame[k_radial,j_vertical]=(amplitude[i_structures]*np.exp(-0.5*(a*(x-x0)**2 +
2*b*(x-x0)*(y-y0) +
c*(y-y0)**2))
+background.data[k_radial,j_vertical])
data_arr[i_frames,:,:]+=frame
if sinusoidal:
x0=start_position[0]
y0=start_position[1]
ky=np.pi/poloidal_size
omega=poloidal_velocity*ky
phi0=0
for i_frames in range(n_time):
cur_time=i_frames * sampling_time
for j_vertical in range(80):
for k_radial in range(64):
x=r_coordinates[k_radial,j_vertical]
y=z_coordinates[k_radial,j_vertical]
A=1/np.sqrt(2*np.pi*radial_size)*np.exp(-0.5*(np.abs(x-(x0+radial_velocity*cur_time))/(radial_size/2.355))**2)
arg=ky*y-omega*cur_time+phi0
division=(scipy.signal.square(arg/waveform_divider, duty=0.5/waveform_divider)+1)/2.
data_arr[i_frames,k_radial,j_vertical]=amplitude*A*np.sin(arg)*division
data_arr[i_frames,k_radial,j_vertical]+=background.data[k_radial,j_vertical]
#Adding the coordinates to the data object:
coord = [None]*6
coord[0]=(copy.deepcopy(flap.Coordinate(name='Time',
unit='s',
mode=flap.CoordinateMode(equidistant=True),
start=0.,
step=sampling_time,
#shape=time_arr.shape,
dimension_list=[0]
)))
coord[1]=(copy.deepcopy(flap.Coordinate(name='Sample',
unit='n.a.',
mode=flap.CoordinateMode(equidistant=True),
start=0,
step=1,
dimension_list=[0]
)))
coord[2]=(copy.deepcopy(flap.Coordinate(name='Image x',
unit='Pixel',
mode=flap.CoordinateMode(equidistant=True),
start=0,
step=1,
shape=[],
dimension_list=[1]
)))
coord[3]=(copy.deepcopy(flap.Coordinate(name='Image y',
unit='Pixel',
mode=flap.CoordinateMode(equidistant=True),
start=0,
step=1,
shape=[],
dimension_list=[2]
)))
coord[4]=(copy.deepcopy(flap.Coordinate(name='Device R',
unit='m',
mode=flap.CoordinateMode(equidistant=False),
values=r_coordinates,
shape=r_coordinates.shape,
dimension_list=[1,2]
)))
coord[5]=(copy.deepcopy(flap.Coordinate(name='Device z',
unit='m',
mode=flap.CoordinateMode(equidistant=False),
values=z_coordinates,
shape=z_coordinates.shape,
dimension_list=[1,2]
)))
_options={}
_options["Trigger time [s]"]=0.
_options["FPS"]=1/sampling_time
_options["Sample time [s]"]=sampling_time
_options["Exposure time [s]"]=2.1e-6
_options["X size"]=64
_options["Y size"]=80
_options["Bits"]=32
d = flap.DataObject(data_array=data_arr,
data_unit=flap.Unit(name='Signal',unit='Digit'),
coordinates=coord,
exp_id=exp_id,
data_title='Simulated signal',
info={'Options':_options},
data_source="NSTX_GPI")
if output_name is not None:
flap.add_data_object(d,output_name)
return d
def add_coordinate(data_object, coordinates=None, exp_id=None, options=None):
""" This is the registered function for adding coordinates to the
data object.
handled coordinates:
Device x, Device z, Device r
"""
for new_coord in coordinates:
if (new_coord == 'Device y'):
# Adding a constant 0 as the z coordinate is always 0
data_object.add_coordinate_object(
copy.deepcopy(
flap.Coordinate(
name='Device y',
unit='cm',
mode=flap.CoordinateMode(equidistant=False),
shape=[],
values=float(0),
dimension_list=[])))
continue
if ((new_coord == 'Device x') or (new_coord == 'Device z')):
# Checking whether the new coordinates have already been calculated
try:
Device_x
Device_z
except NameError:
# If not calculating both
# We need to find on which dimensions the row and column coordinate changes
# x and y will change on both
# Getting the flap.Coordinate instances of the Row and Column coordinates
try:
c_row = data_object.get_coordinate_object('Row')
c_column = data_object.get_coordinate_object('Column')
except Exception as e:
raise e
# Creating the dimension list of the new coordinate which will be the combined
# list of Row and Column
dimension_list = copy.copy(c_column.dimension_list)
if (type(dimension_list) != list):
dimension_list = [dimension_list]
dimlist_row = copy.copy(c_row.dimension_list)
if (type(dimlist_row) != list):
dimlist_row = [dimlist_row]
for d in dimlist_row:
try:
dimension_list.index(d)
except (ValueError, IndexError):
dimension_list.append(d)
# Creating an index list where these dimensions are fully indexed, the rest is 0
index = [0] * len(data_object.shape)
if (len(dimension_list) != 0):
# If at least one of the coordinates != scalar
dimension_list.sort()
for i in dimension_list:
index[i] = ...
# Getting data for this data points
try:
row, row_l, row_h = data_object.coordinate('Row',
index=index)
column, column_l, column_h = data_object.coordinate(
'Column', index=index)
except Exception as e:
raise e
if (len(dimension_list) != 0):
sh = []
for i in dimension_list:
sh.append(data_object.shape[i])
row = np.reshape(row, tuple(sh))
column = np.reshape(column, tuple(sh))
# Calculating the new coordinates
xr = (column - 1) * 0.5
zr = (row - 1) * 0.4
alpha_rad = alpha / 180. * math.pi
Device_x = xr * math.cos(alpha_rad) - zr * math.sin(alpha_rad)
Device_z = xr * math.sin(alpha_rad) + zr * math.cos(alpha_rad)
if (Device_x.size == 1):
shape = []
else:
shape = Device_x.shape
# Adding the new coordinate
if (new_coord == 'Device x'):
data = Device_x
else:
data = Device_z
cc = flap.Coordinate(name=new_coord,
unit='cm',
mode=flap.CoordinateMode(equidistant=False),
shape=shape,
values=data,
dimension_list=dimension_list)
data_object.add_coordinate_object(copy.deepcopy(cc))
def plot_results_for_paper():
pearson=False
wd=flap.config.get_all_section('Module NSTX_GPI')['Working directory']
#Figure 1
'''NO CODE IS NEEDED'''
#Figure 2
'''NO CODE IS NEEDED'''
#Figure 3
from flap_nstx.analysis import show_nstx_gpi_video_frames
#fig, ax = plt.subplots(figsize=(6.5,5))
if plot[3]:
gs=GridSpec(5,2)
ax,fig=plt.subplots(figsize=(8.5/2.54,6))
pdf=PdfPages(wd+'/plots/figure_3_139901_basic_plots.pdf')
plt.subplot(gs[0,0])
flap.get_data('NSTX_MDSPlus',
name='\WF::\DALPHA',
exp_id=139901,
object_name='DALPHA').plot(options={'Axes visibility':[False,True]})
plt.xlim([0,1.2])
plt.subplot(gs[1,0])
flap.get_data('NSTX_GPI',
name='',
exp_id=139901,
object_name='GPI').slice_data(summing={'Image x':'Mean', 'Image y':'Mean'}).plot(options={'Axes visibility':[False,True]})
plt.xlim([0,1.2])
plt.xlim([0,1.2])
plt.subplot(gs[2,0])
flap.get_data('NSTX_MDSPlus',
name='IP',
exp_id=139901,
object_name='IP').plot(options={'Axes visibility':[False,True]})
plt.xlim([0,1.2])
plt.subplot(gs[3,0])
d=flap_nstx_thomson_data(exp_id=139901, density=True, output_name='DENSITY')
dR = d.coordinate('Device R')[0][:,:]-np.insert(d.coordinate('Device R')[0][0:-1,:],0,0,axis=0)
LID=np.sum(d.data*dR,axis=0)
plt.plot(d.coordinate('Time')[0][0,:],LID)
plt.title('Line integrated density')
plt.xlabel('Time [s]')
plt.ylabel('n_e [m^-2]')
plt.xlim([0,1.2])
ax=plt.gca()
ax.get_xaxis().set_visible(False)
plt.subplot(gs[4,0])
magnetics=flap.get_data('NSTX_MDSPlus',
name='\OPS_PC::\\BDOT_L1DMIVVHF5_RAW',
exp_id=139901,
object_name='MIRNOV')
magnetics.coordinates.append(copy.deepcopy(flap.Coordinate(name='Time equi',
unit='s',
mode=flap.CoordinateMode(equidistant=True),
shape = [],
start=magnetics.coordinate('Time')[0][0],
step=magnetics.coordinate('Time')[0][1]-magnetics.coordinate('Time')[0][0],
dimension_list=[0])))
magnetics.filter_data(coordinate='Time equi',
options={'Type':'Bandpass',
'f_low':100e3,
'f_high':500e3,
'Design':'Elliptic'}).plot()
plt.xlim([0,1.2])
plt.subplot(gs[0,1])
flap.get_data('NSTX_MDSPlus',
name='\WF::\DALPHA',
exp_id=139901,
object_name='DALPHA').plot(options={'Axes visibility':[False,False]})
plt.xlim([0.25,0.4])
plt.subplot(gs[1,1])
flap.get_data('NSTX_GPI',
name='',
exp_id=139901,
object_name='GPI').slice_data(summing={'Image x':'Mean', 'Image y':'Mean'}).plot(options={'Axes visibility':[False,False]})
plt.xlim([0.25,0.4])
plt.subplot(gs[2,1])
flap.get_data('NSTX_MDSPlus',
name='IP',
exp_id=139901,
object_name='IP').plot(options={'Axes visibility':[False,False]})
plt.xlim([0.25,0.4])
plt.subplot(gs[3,1])
d=flap_nstx_thomson_data(exp_id=139901, density=True, output_name='DENSITY')
dR = d.coordinate('Device R')[0][:,:]-np.insert(d.coordinate('Device R')[0][0:-1,:],0,0,axis=0)
LID=np.sum(d.data*dR,axis=0)
plt.plot(d.coordinate('Time')[0][0,:],LID)
plt.title('Line integrated density')
plt.xlabel('Time [s]')
plt.ylabel('n_e [m^-2]')
plt.xlim([0.25,0.4])
ax=plt.gca()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.subplot(gs[4,1])
magnetics=flap.get_data('NSTX_MDSPlus',
name='\OPS_PC::\\BDOT_L1DMIVVHF5_RAW',
exp_id=139901,
object_name='MIRNOV')
magnetics.coordinates.append(copy.deepcopy(flap.Coordinate(name='Time equi',
unit='s',
mode=flap.CoordinateMode(equidistant=True),
shape = [],
start=magnetics.coordinate('Time')[0][0],
step=magnetics.coordinate('Time')[0][1]-magnetics.coordinate('Time')[0][0],
dimension_list=[0])))
magnetics.filter_data(coordinate='Time equi',
options={'Type':'Bandpass',
'f_low':100e3,
'f_high':500e3,
'Design':'Elliptic'}).plot(slicing={'Time':flap.Intervals(0.25,0.4)})
plt.xlim([0.25,0.4])
ax=plt.gca()
ax.get_yaxis().set_visible(False)
pdf.savefig()
pdf.close()
if plot[4]:
plt.figure()
ax,fig=plt.subplots(figsize=(3.35*2,5.5))
pdf=PdfPages(wd+'/plots/figure_5_139901_0.3249158_30_frame.pdf')
show_nstx_gpi_video_frames(exp_id=139901,
start_time=0.3249158,
n_frame=30,
logz=False,
z_range=[0,3900],
plot_filtered=False,
normalize=False,
cache_data=False,
plot_flux=False,
plot_separatrix=True,
flux_coordinates=False,
device_coordinates=True,
new_plot=False,
save_pdf=True,
colormap='gist_ncar',
save_for_paraview=False,
colorbar_visibility=True
)
pdf.savefig()
pdf.close()
#Figure 5
if plot[5] or plot[6] or plot[7]:
try:
d1,d2,d3,d4=pickle.load(open(wd+'/processed_data/fig_6_8_flap_object.pickle','rb'))
flap.add_data_object(d1, 'GPI_SLICED_FULL')
flap.add_data_object(d2, 'GPI_GAS_CLOUD')
flap.add_data_object(d3, 'GPI_SLICED_DENORM_CCF_VEL')
flap.add_data_object(d4, 'GPI_CCF_F_BY_F')
except:
calculate_nstx_gpi_avg_frame_velocity(exp_id=139901,
time_range=[0.325-1e-3,0.325+1e-3],
plot=False,
subtraction_order_for_velocity=1,
skip_structure_calculation=False,
correlation_threshold=0.,
pdf=False,
nlevel=51,
nocalc=False,
filter_level=3,
normalize_for_size=True,
normalize_for_velocity=True,
threshold_coeff=1.,
normalize_f_high=1e3,
normalize='roundtrip',
velocity_base='cog',
return_results=False,
plot_gas=True)
pickle.dump((flap.get_data_object('GPI_SLICED_FULL'),
flap.get_data_object('GPI_GAS_CLOUD'),
flap.get_data_object('GPI_SLICED_DENORM_CCF_VEL'),
flap.get_data_object('GPI_CCF_F_BY_F')), open(wd+'/processed_data/fig_6_8_flap_object.pickle','wb'))
if plot[5]:
pdf=PdfPages(wd+'/plots/figure_6_normalization.pdf')
times=[0.3245,0.3249560,0.3255]
signals=['GPI_SLICED_FULL',
'GPI_GAS_CLOUD',
'GPI_SLICED_DENORM_CCF_VEL']
gs=GridSpec(3,3)
plt.figure()
ax,fig=plt.subplots(figsize=(3.35,4))
titles=['Raw frame', 'Gas cloud', 'Normalized']
for index_grid_x in range(3):
for index_grid_y in range(3):
plt.subplot(gs[index_grid_x,index_grid_y])
visibility=[True,True]
if index_grid_x != 3-1:
visibility[0]=False
if index_grid_y != 0:
visibility[1]=False
# if index_grid_x == 0:
# z_range=[0,4096]
# elif index_grid_x == 1:
# z_range=[0,400]
# elif index_grid_x == 2:
# z_range=[0,40]
z_range=None
flap.plot(signals[index_grid_x],
plot_type='contour',
slicing={'Time':times[index_grid_y]},
axes=['Image x', 'Image y'],
options={'Z range':z_range,
'Interpolation': 'Closest value',
'Clear':False,
'Equal axes':True,
'Axes visibility':visibility,
#'Colormap':'gist_ncar',
'Colorbar':True,
#'Overplot options':oplot_options,
},
plot_options={'levels':51},
)
if index_grid_x == 0:
#ax=plt.gca()
plt.title(f"{times[index_grid_y]*1e3:.3f}"+' '+titles[index_grid_x])
else:
plt.title(titles[index_grid_x])
pdf.savefig()
pdf.close()
#Figure 6
if plot[6]:
flap.get_data('NSTX_GPI',exp_id=139901,
name='',
object_name='GPI')
flap.slice_data('GPI', slicing={'Time':flap.Intervals(0.3245,0.3255)}, output_name='GPI_SLICED_FULL')
data_object_name='GPI_SLICED_DENORM_CCF_VEL'
detrended=flap_nstx.analysis.detrend_multidim(data_object_name,
exp_id=139901,
order=4,
coordinates=['Image x', 'Image y'],
output_name='GPI_DETREND_VEL')
d=copy.deepcopy(flap.get_data_object(data_object_name))
d.data=d.data-detrended.data
flap.add_data_object(d,'GPI_TREND')
signals=[data_object_name,
'GPI_TREND',
'GPI_DETREND_VEL']
pdf=PdfPages(wd+'/plots/figure_7_trend_subtraction.pdf')
gs=GridSpec(1,3)
plt.figure()
ax,fig=plt.subplots(figsize=(8.5/2.54,2))
for index_grid_x in range(3):
plt.subplot(gs[index_grid_x])
visibility=[True,True]
if index_grid_x != 0:
visibility[1]=False
z_range=[0,10]
colorbar=False
flap.plot(signals[index_grid_x],
plot_type='contour',
slicing={'Time':0.3249560},
#slicing={'Sample':29808},
axes=['Image x', 'Image y'],
options={'Interpolation': 'Closest value',
'Clear':False,
'Equal axes':True,
'Axes visibility':visibility,
#'Colormap':colormap,
'Colorbar':True,
#'Overplot options':oplot_options,
},
plot_options={'levels':51},
)
#fig.tight_layout()
pdf.savefig()
pdf.close()
#Figure 7
if plot[7]:
pdf=PdfPages(wd+'/plots/figure_8_CCF_frame_by_frame.pdf')
gs=GridSpec(1,3)
plt.figure()
ax,fig=plt.subplots(figsize=(8.5/2.54,2))
plt.subplot(gs[0])
flap.plot('GPI_SLICED_FULL',
plot_type='contour',
slicing={'Sample':29806},
axes=['Image x', 'Image y'],
options={
'Z range':[0,4096],
'Interpolation': 'Closest value',
'Clear':False,
'Equal axes':True,
'Axes visibility':[True,True],
'Colormap':'gist_ncar',
'Colorbar':False,
#'Overplot options':oplot_options,
},
plot_options={'levels':51},
)
plt.title("324.959ms")
plt.subplot(gs[1])
flap.plot('GPI_SLICED_FULL',
plot_type='contour',
slicing={'Sample':29807},
axes=['Image x', 'Image y'],
options={'Z range':[0,4096],
'Interpolation': 'Closest value',
'Clear':False,
'Equal axes':True,
'Axes visibility':[True,False],
'Colorbar':False,
'Colormap':'gist_ncar',
},
plot_options={'levels':51},
)
plt.title("324.961ms")
plt.subplot(gs[2])
flap.plot('GPI_CCF_F_BY_F',
plot_type='contour',
slicing={'Sample':29807, 'Image x':flap.Intervals(-10,10),'Image y':flap.Intervals(-10,10)},
axes=['Image x', 'Image y'],
options={
#'Z range':[0,2048],
'Interpolation': 'Closest value',
'Clear':False,
'Equal axes':True,
'Axes visibility':[True,True],
#'Colormap':colormap,
'Colorbar':True,
#'Overplot options':oplot_options,
},
plot_options={'levels':51},
)
plt.title("CCF")
pdf.savefig()
pdf.close()
#Figure 8
if plot[8]:
#2x2 frames with the found structures during an ELM burst
calculate_nstx_gpi_avg_frame_velocity(exp_id=139901,
time_range=[0.32495,0.325],
plot=False,
subtraction_order_for_velocity=4,
skip_structure_calculation=False,
correlation_threshold=0.5,
pdf=True,
nlevel=51,
nocalc=False,
filter_level=5,
normalize_for_size=True,
normalize_for_velocity=True,
threshold_coeff=1.,
normalize_f_high=1e3,
normalize='roundtrip',
velocity_base='cog',
return_results=False,
plot_gas=True,
structure_pixel_calc=True,
structure_pdf_save=True,
test_structures=True
)
#Post processing done with illustrator
#Figure 9
if plot[9]:
#2x3
#Synthetic GPI signal
#Postprocessing done with illustrator
nstx_gpi_generate_synthetic_data(exp_id=1,
time=0.0001,
amplitude=1.0,
output_name='test',
poloidal_velocity=3e3,
radial_velocity=0.,
poloidal_size=0.10,
radial_size=0.05,
waveform_divider=1,
sinusoidal=True)
d=flap.get_data_object('test', exp_id=1)
d.data=d.data-np.mean(d.data,axis=0)
calculate_nstx_gpi_avg_frame_velocity(data_object='test',
exp_id=1,
time_range=[0.000000,0.00005],
plot=False,
subtraction_order_for_velocity=1,
skip_structure_calculation=False,
correlation_threshold=0.5,
pdf=True,
nlevel=51,
nocalc=False,
filter_level=5,
normalize_for_size=False,
normalize_for_velocity=False,
threshold_coeff=1.,
normalize_f_high=1e3,
normalize=None,
velocity_base='cog',
return_results=False,
plot_gas=False,
structure_pixel_calc=True,
structure_pdf_save=True,
test_structures=True
)
#Figure 10
if plot[10]:
#Single shot results
calculate_nstx_gpi_avg_frame_velocity(exp_id=139901,
time_range=[0.325-2e-3,0.325+2e-3],
plot_time_range=[0.325-0.5e-3,0.325+0.5e-3],
plot=True,
subtraction_order_for_velocity=4,
skip_structure_calculation=False,
correlation_threshold=0.6,
pdf=True,
nlevel=51,
nocalc=True,
gpi_plane_calculation=True,
filter_level=5,
normalize_for_size=True,
normalize_for_velocity=True,
threshold_coeff=1.,
normalize_f_high=1e3,
normalize='roundtrip',
velocity_base='cog',
return_results=False,
plot_gas=True,
plot_for_publication=True,
plot_scatter=False,
overplot_average=False,
overplot_str_vel=False)
#2x3
#Done with Illustrator
#Figure 12
if plot[11]:
#Conditional averaged results
calculate_avg_velocity_results(pdf=True,
plot=True,
plot_max_only=True,
plot_for_publication=True,
normalized_velocity=True,
subtraction_order=4,
normalized_structure=True,
opacity=0.5,
correlation_threshold=0.6,
gpi_plane_calculation=True,
plot_scatter=False)
#Post processing done with Illustrator
#Figure 11
if plot[12]:
if pearson:
pdf=PdfPages(wd+'/plots/figure_13_pearson_matrix.pdf')
pearson=calculate_nstx_gpi_correlation_matrix(calculate_average=False,
gpi_plane_calculation=True,
window_average=0.050e-3,
elm_burst_window=True)
data=pearson[:,:,0]
variance=pearson[:,:,1]
data[10,10]=-1
plt.figure()
plt.subplots(figsize=(8.5/2.54,8.5/2.54/1.618))
plt.matshow(data, cmap='seismic')
plt.xticks(ticks=np.arange(11), labels=['Velocity ccf R', #0,1
'Velocity ccf z', #0,1
'Velocity str max R', #2,3
'Velocity str max z', #2,3
'Size max R', #4,5
'Size max z', #4,5
'Position max R', #6,7
'Position max z', #6,7
'Area max', #8
'Elongation max', #9
'Angle max'], rotation='vertical')
plt.yticks(ticks=np.arange(11), labels=['Velocity ccf R', #0,1
'Velocity ccf z', #0,1
'Velocity str max R', #2,3
'Velocity str max z', #2,3
'Size max R', #4,5
'Size max z', #4,5
'Position max R', #6,7
'Position max z', #6,7
'Area max', #8
'Elongation max', #9
'Angle max'])
plt.colorbar()
plt.show()
pdf.savefig()
plt.figure()
plt.subplots(figsize=(8.5/2.54,8.5/2.54/1.618))
variance[10,10]=-1
variance[9,9]=1
plt.matshow(variance, cmap='seismic')
#plt.matshow(data, cmap='gist_ncar')
plt.xticks(ticks=np.arange(11), labels=['Velocity ccf R', #0,1
'Velocity ccf z', #0,1
'Velocity str max R', #2,3
'Velocity str max z', #2,3
'Size max R', #4,5
'Size max z', #4,5
'Position max R', #6,7
'Position max z', #6,7
'Area max', #8
'Elongation max', #9
'Angle max'], rotation='vertical')
plt.yticks(ticks=np.arange(11), labels=['Velocity ccf R', #0,1
'Velocity ccf z', #0,1
'Velocity str max R', #2,3
'Velocity str max z', #2,3
'Size max R', #4,5
'Size max z', #4,5
'Position max R', #6,7
'Position max z', #6,7
'Area max', #8
'Elongation max', #9
'Angle max'])
plt.colorbar()
plt.show()
pdf.savefig()
pdf.close()
else:
pdf=PdfPages(wd+'/plots/figure_13_dependence.pdf')
plt.figure()
plt.subplots(figsize=(17/2.54,17/2.54/1.618))
plot_all_parameters_vs_all_other_average(window_average=0.2e-3, symbol_size=0.3, plot_error=True)
pdf.savefig()
pdf.close()
def getsignal_sal(exp_id, source, no_data=False, options={}):
""" Signal reading function for the JET API SAL
Does not require using the FLAP storage
Input: same as get_data()
Output: DataObject for the given source
"""
# Function used for getting either ppf or jpf data from the server
#
from jet.data import sal
options_default = {
"Sequence": 0,
"UID": "jetppf",
"Check Time Equidistant": True,
"Cache Data": True
}
options = {**options_default, **options}
#checking if the data is already in the cache
curr_path = os.path.dirname(os.path.abspath(__file__))
location = os.path.sep.join(curr_path.split(os.path.sep))
split_source = source.split("/")
filename = os.path.join(os.path.sep.join([location,"cached"]),
("_".join(split_source)+"-"+options["UID"]+\
"-"+str(exp_id)+".hdf5").lower())
if os.path.exists(filename):
return flap.load(filename)
# obtaining the data from the server
# if the UID ends with -team, then it looks in the config file, whether
# there is a team defined under the name given in UID in the config file
# if so, it will loop over the name of the team members
split_uid = options["UID"].split("-")
if split_uid[-1] == "team":
uid_list = flap.config.get("Module JET_API",
options["UID"],
evaluate=True)
else:
if options["UID"] == "curr_user":
options["UID"] = pwd.getpwuid(os.getuid()).pw_name
uid_list = [options["UID"]]
# has to do some character conversion in the names as SAL uses different
# naming as the standard ppf names
character_dict = {
'>': '_out_',
'<': '_in_',
':': '_sq_',
'$': '_do_',
':': '_sq_',
'&': '_et_',
' ': '_sp_'
}
node = split_source[2].lower()
for char in character_dict.keys():
node = character_dict[char].join(node.split(char))
signal_error = None
error_string = str()
for uid in uid_list:
try:
if split_source[0].lower() == "ppf":
source_string = "/pulse/"+str(exp_id)+"/"+split_source[0].lower()+"/signal/"+uid.lower()+"/"+split_source[1].lower()\
+"/"+node
elif split_source[0].lower() == "jpf":
source_string = "/pulse/"+str(exp_id)+"/"+split_source[0].lower()+"/"+split_source[1].lower()\
+"/"+node+"/data"
else:
raise ValueError("Source should be either jpf or ppf")
if options["Sequence"] != 0:
source_string = source_string + ":" + str(options["Sequence"])
raw_signal = sal.get(source_string)
data = raw_signal.data
signal_error = raw_signal.error
except sal.SALException as e:
error_string = error_string + source_string + " " + str(e) + "\n"
if not (signal_error is None):
raise ValueError("Error reading " + source + " for shot " +
str(exp_id) + ", UID " + options["UID"] + ":\nIER " +
str(ier) + ": " + desc)
elif not ("data" in locals()):
raise ValueError("No data found with errors: \n" + error_string)
# converting data to flap DataObject
# possible names for the time coordinate
time_names = [
'time', 't', 'jpf time vector', 'ppf time vector', 'time vector',
'the ppf t-vector.'
]
coordinates = []
data = data.reshape(raw_signal.shape)
info = "Obtained at " + str(date.today()) + ", uid " + uid + "\n"
coord_dimension = 0
for coord in raw_signal.dimensions:
name = coord.description
values = np.array(coord.data)
unit = coord.units
equidist = False
if name.lower() in time_names or coord.temporal is True:
tunit = ["secs", "s", "seconds", "second"]
if unit.lower() in tunit:
unit = "Second"
name = 'Time'
equidist = False
if options['Check Time Equidistant'] is True and (unit == "Second" or coord.temporal is True)\
and (len(values)==1 and values[0]==-1):
timesteps = np.array(values[1:]) - np.array(values[0:-1])
equidistance = np.linalg.norm(
timesteps - timesteps[0]) / np.linalg.norm(timesteps)
if equidistance < 1e-6:
info = info + "Time variable is taken as equidistant to an accuracy of "+\
str(equidistance)+"\n"
coord_object = flap.Coordinate(
name=name,
unit=values,
start=values[0],
shape=values.shape,
step=np.mean(timesteps),
mode=flap.CoordinateMode(equidistant=True),
dimension_list=[0])
equidist = True
if equidist is False:
coord_object = flap.Coordinate(
name=name,
unit=unit,
values=values,
shape=values.shape,
mode=flap.CoordinateMode(equidistant=False),
dimension_list=[coord_dimension])
coordinates.append(coord_object)
coord_dimension = coord_dimension + 1
unit = flap.Unit(name=source, unit=raw_signal.units)
signal = flap.DataObject(data_array=data,
data_unit=unit,
coordinates=coordinates,
exp_id=str(exp_id),
data_title=raw_signal.description,
data_shape=data.shape,
info=info)
if "summary" in dir(raw_signal):
signal.info = signal.info + raw_signal.summary().description + "\n"
if options["Cache Data"] is True:
flap.save(signal, filename)
return signal
