def scopeRawToFull(src,
dest,
port=14,
tdiode_hdf=None,
verbose=False,
debug=False,
vdist=False):
"""
Parameters
----------
src: hdfPath object
Path string to a raw hdf5 file containing bdot data
dest: hdfPath object
Path string to location processed bdot data should be written out
tdiode_hdf: hdfPath object
Path to a raw hdf5 file containing tdiode data. If no HDF file is
provided, no timing correction will be applied.
port: float
port at which the probe is located
Returns
-------
True (if executes to the end)
"""
# ******
# Load data from the raw HDF file
# ******
with h5py.File(src.file, 'r') as sf:
#Get the datagroup
srcgrp = sf[src.group]
#Create dictionary of attributes
attrs = hdftools.readAttrs(srcgrp)
#Check for keys always required by this function
req_keys = []
#Process the required keys, throwing an error if any cannot be found
csvtools.missingKeys(attrs, req_keys, fatal_error=True)
#Extract the shape of the source data
nshots, nti, nchan = srcgrp['data'].shape
if verbose:
print("Opening destination HDF file")
#Create the destination file directory if necessary
hdftools.requireDirs(dest.file)
#Open the destination file
#This exists WITHIN the open statement for the source file, so the
#source file is open at the same time.
with h5py.File(dest.file, 'a') as df:
#Throw an error if this group already exists
if dest.group is not '/' and dest.group in df.keys():
raise hdftools.hdfGroupExists(dest)
destgrp = df.require_group(dest.group)
#Copy over attributes
hdftools.copyAttrs(srcgrp, destgrp)
#Load the time vector
t = srcgrp['time']
#If tdiode_hdf is set, load the pre-processed tdiode data
if tdiode_hdf is not None:
if verbose:
print("Loading tdiode array from file.")
with h5py.File(tdiode_hdf.file, 'r') as sf:
grp = sf[tdiode_hdf.group]
t0indarr = grp['t0indarr'][:]
goodshots = grp['goodshots'][:]
tdiode_attrs = hdftools.readAttrs(grp)
#If tdiode was digitized with a different dt, this correction
#will be necessary
dt_ratio = float(attrs['dt'][0]) / float(tdiode_attrs['dt'][0])
t0indarr = (t0indarr / dt_ratio).astype(np.int32)
#We will remove up to max_t0shift indices from each array such that
#the t0 indices all line up.
min_t0ind = np.min(t0indarr[goodshots])
max_t0shift = np.max(t0indarr[goodshots]) - min_t0ind
#Compute new nti
nti = nti - max_t0shift
t = t[0:nti] - t[min_t0ind]
#Throw an error if this dataset already exists
if 'data' in destgrp.keys():
raise hdftools.hdfDatasetExists(str(dest) + ' -> ' + "'data'")
#Create the dataset 'data' appropriate to whether or not output
#data will be gridded
if verbose:
print("Creating 'data' group in destination file")
destgrp.require_dataset('data', (nshots, nti),
np.float32,
chunks=(1, np.min([nti, 20000])),
compression='gzip')
#Initialize time-remaining printout
tr = util.timeRemaining(nshots)
if verbose:
print("Beginning processing data shot-by-shot.")
#Chunking data processing loop limits memory usage
for i in range(nshots):
#Update time remaining
if verbose:
tr.updateTimeRemaining(i)
#If a tdiode hdf was supplied, calculate the index correction
#here
if tdiode_hdf is not None:
#Calculate the starting and ending arrays for the data
ta = t0indarr[i] - min_t0ind
tb = ta + nti
else:
#By default, read in the entire dataset
ta = None
tb = None
if debug:
print("Data range: [" + str(ta) + "," + str(tb) + "]")
#Read in the data from the source file
signal = np.squeeze(srcgrp['data'][i, ta:tb])
destgrp['data'][i, :] = signal
destgrp['data'].attrs['unit'] = ''
if 'pos' in srcgrp:
destgrp.copy(srcgrp['pos'], 'pos')
destgrp.require_dataset('shots', (nshots, ), np.int32,
chunks=True)[:] = srcgrp['shots'][:]
destgrp['shots'].attrs['unit'] = srcgrp['shots'].attrs['unit']
dimlabels = ['shots', 'time']
destgrp.require_dataset('time', (nti, ), np.float32,
chunks=True)[:] = t
destgrp['time'].attrs['unit'] = 's'
destgrp['data'].attrs['dimensions'] = [
s.encode('utf-8') for s in dimlabels
]
if verbose:
print("End of Monochromator routine!")
return True
def chunked_array_op(src,
dest,
ax,
op,
newshape,
delsrc=False,
verbose=False,
**args):
"""
Apply one of the array functions to an entire dataset, breaking the
dataset up into chunks to keep memory load low.
src -> Source dataset (hdfpath object)
dest -> Destination dataset path (hdfpath object)
ax -> Axis (0 indexed) to average
op -> Function to be applied. This function must be one of the op functions
defined in this file, and must be included in the elif tree in this function
newshape -> Shape the new dataset will be after op has been applied
delsrc -> Boolean, if true src file will be deleted after operation
verbose -> Boolean, if true activates printouts
"""
with h5py.File(src.file, 'r') as sf:
srcgrp = sf[src.group]
#Check source is valid dataset
validDataset(srcgrp)
#Load information about source dataset
oldshape = list(srcgrp['data'].shape)
ndim = len(oldshape)
dimlabels = hdftools.arrToStrList(
srcgrp['data'].attrs['dimensions'][:])
#Get ax index
axind = getAxInd(ax, dimlabels)
#Decide on a chunking axis
#Get a list of the axes indices ordered by chunk size, largest to smallest
chunks = np.flip(np.argsort(srcgrp['data'].chunks))
#Chose the largest one that ISN'T the chosen axis
chunkax = chunks[0]
if chunkax == axind:
chunkax = chunks[1]
print("Chunking axis: " + str(dimlabels[chunkax]))
if srcgrp['data'].chunks[chunkax] < 2:
print("WARNING: POSSIBLE INEFFICENT CHUNKING DETECTED!")
#Determine optimal chunksize (along chunkax)
ideal_chunk_elms = 1e7 #1e7*4 bytes (per float32) ~ 40mb, which is good
nper = np.product(oldshape) / oldshape[
chunkax] #number of values per chunk ax value
chunksize = int(np.round(ideal_chunk_elms / nper))
if chunksize < 1:
chunksize = 1
#Determine nchunks
nchunks = int(np.ceil(oldshape[chunkax] / chunksize))
#Create the destination dataset
with h5py.File(dest.file, 'w') as df:
destgrp = df[dest.group]
#Copy all the dataset attributes
hdftools.copyAttrs(srcgrp, destgrp)
#Create new data array
destgrp.require_dataset('data',
newshape,
np.float32,
chunks=True,
compression='gzip')
destgrp['data'].attrs['unit'] = srcgrp['data'].attrs['unit']
if verbose:
print(srcgrp['data'].shape)
print(destgrp['data'].shape)
#Copy the axes over, except the one being operated on
#That axis will be copied over later, with changes
for axis in dimlabels:
if axis != ax:
srcgrp.copy(axis, destgrp)
#Create the axis being operated on...unless it is now trivial
#Newshape was determined above, and is specific to the op
if newshape[axind] > 1:
destgrp.require_dataset(ax, (newshape[axind], ),
np.float32,
chunks=True)
destgrp[ax].attrs['unit'] = srcgrp[ax].attrs['unit']
new_dimlabels = dimlabels #No changes
else:
new_dimlabels = dimlabels.pop(axind) #Trivial: remove this dim
destgrp['data'].attrs['dimensions'] = hdftools.strListToArr(
new_dimlabels)
#Initialize time-remaining printout
#Chunks are big, so report more often than usual
tr = util.timeRemaining(nchunks, reportevery=1)
for i in range(nchunks):
#Update time remaining
if verbose:
tr.updateTimeRemaining(i)
sl = [slice(None)] * ndim
#Assemble the chunk slices
if i != nchunks - 1:
sl[chunkax] = slice(i * chunksize, (i + 1) * chunksize,
None)
else:
sl[chunkax] = slice(i * chunksize, None, None)
#Apply op to the chunk
op(srcgrp['data'], destgrp['data'], sl, axind, args)
#Make the new axis by applying op to the old axis
op(srcgrp[ax], destgrp[ax], [slice(None)], 0, args)
#If requested, delete the source file
if delsrc:
os.remove(src.file)
def bdotRawToFull(src, dest,
tdiode_hdf=None, grid=False, integrate=True,
calibrate =True, highfreq_calibrate=True,
angle_correction = True, remove_offset = True,
replace_badshots = True,
verbose=False, debug = False,
offset_range=(0,100), offset_rel_t0 = (False, False),
grid_precision=0.1, strict_grid=False, strict_axes = False):
""" Integrates bdot data, calibrates output using information about the probe.
Corrects for probe angle based on which drive is being used.
Parameters
----------
src: hdfPath object
Path string to a raw hdf5 file containing bdot data
dest: hdfPath object
Path string to location processed bdot data should be written out
tdiode_hdf: hdfPath object
Path to a raw hdf5 file containing tdiode data. If no HDF file is
provided, no timing correction will be applied.
grid: Boolean
If grid is true, output will be written in cartesian grid array
format, eg. [nti, nx, ny, nz, nreps, nchan]. Otherwise, output will
be in [nshots, nti, nchan] format
integrate: Boolean
If True, integrate the bdot data (usually you want to do this).
Default is True
calibrate: Boolean
If True, calculate and apply ANY calibration factors
to the data. Default is True.
highfreq_calibrate: Boolean
If True, calculate and apply the high frequency calibration
factors to the data. Default is True. If the 'tau' variables are
not specified in the probe metadata, the HF calibration won't be
applied regardless of this keyword.
angle_correction: Boolean
If True, apply any angular correction between axes that is
required based on the motion_format keyword in the metadata. If
false, no correction is applied regardless of the metadata.
Default is True.
remove_offset: Boolean
If True, remove an offset from the data based on the offset_range
specified in those keywords. If False, data will remain as-is.
Default is True.
replace_badshots: Boolean
If True, semi-intelligently replace bad shots with neighboring
good shots. If False, data remains as-is.
Default is True.
offset_range: tuple
Tuple of indices between which the average of the signal will be
computed and subtracted from the entire signal to correct for
offset. This should be a segment with just noise, ideally at the
very beginning of the dataset. Longer is better.
Default is (0,100)
offset_rel_t0: Tuple of booleans
If either of these values is set to True, the coorresponding
offset_range value will be taken to be relative to the t0 index
for that each shot. For example, if t0=2000 for a shot,
offset_range=(10, -100), and offset_rel_t0 = (False, True), then
the offset will be computed over the range (10, 1900)
grid_precision: float
This is the precision to which position values will be rounded
before being fit onto the grid. Only applies to fuzzy axis and grid
creation.
strict_axes: boolean
If true, attempt to calculate axes from saved grid parameters.
Default is false, which attempts to calculate axes by looking at
position values.
strict_grid: boolean
If true, strictly unravel data onto the axes, assuming the probe
moved in order reps->X->Y->Z. This will NOT correctly handle
points where the probe was not at the requested position. Default
is false, which applys "fuzzy gridding", which tries to find the
best grid position for each shot individually.
Returns
-------
True (if executes to the end)
"""
# ******
# Load data from the raw HDF file
# ******
with h5py.File(src.file, 'r') as sf:
#Get the datagroup
srcgrp = sf[src.group]
#Create dictionary of attributes
attrs = hdftools.readAttrs(srcgrp)
#Check for keys always required by this function
req_keys = ['xarea', 'yarea', 'zarea',
'xatten', 'yatten', 'zatten', 'gain',
'xpol', 'ypol', 'zpol', 'roll',
'probe_origin_x', 'probe_origin_y', 'probe_origin_z',
'dt', 'nturns']
if 'pos' in srcgrp:
pos = srcgrp['pos'][:] #Read the entire array in
#If pos array exists, there are keywords required for that too.
motion_format = attrs['motion_format'][0]
if motion_format == 'fixed_pivot' and angle_correction:
req_keys = req_keys + ['rot_center_x', 'rot_center_y', 'rot_center_z']
elif motion_format == 'cartesian' and angle_correction:
pass
elif not angle_correction:
pass
else:
raise ValueError("Motion format unrecognized: " + str(attrs['motion_format'][0]) )
else:
#If no position information is given, a single explicit position
#is required.
req_keys = req_keys + ['xpos', 'ypos', 'zpos']
grid = False #Can't grid data if there's no pos array!
motion_format = None
#Process the required keys, throwing an error if any cannot be found
csvtools.missingKeys(attrs, req_keys, fatal_error=True)
#Extract the shape of the source data
nshots, nti, nchan = srcgrp['data'].shape
#If requested by keyword, apply gridding
if grid:
shotgridind, xaxis, yaxis, zaxis, nx, ny, nz, nreps, nshots = postools.grid(
pos, attrs, strict_axes=strict_axes,
strict_grid=strict_grid, grid_precision=grid_precision,
invert=False)
if verbose:
print("Opening destination HDF file")
#Create the destination file directory if necessary
hdftools.requireDirs(dest.file)
#Open the destination file
#This exists WITHIN the open statement for the source file, so the
#source file is open at the same time.
with h5py.File(dest.file, 'a') as df:
#Throw an error if this group already exists
if dest.group is not '/' and dest.group in df.keys():
raise hdftools.hdfGroupExists(dest)
destgrp = df.require_group(dest.group)
#Copy over attributes
hdftools.copyAttrs(srcgrp, destgrp)
#Load the time vector
t = srcgrp['time']
#If a timing diode is being applied, correct the time vector here.
if tdiode_hdf is not None:
if verbose:
print("Loading tdiode array from file.")
with h5py.File(tdiode_hdf.file, 'r') as sf:
grp = sf[tdiode_hdf.group]
t0indarr = grp['t0indarr'][:]
goodshots = grp['goodshots'][:]
badshots = grp['badshots'][:]
tdiode_attrs = hdftools.readAttrs(grp)
#If tdiode was digitized with a different dt, this correction
#will be necessary
dt_ratio = float(attrs['dt'][0])/float(tdiode_attrs['dt'][0])
t0indarr = (t0indarr/dt_ratio).astype(np.int32)
#We will remove up to max_t0shift indices from each array such that
#the t0 indices all line up.
min_t0ind = np.min(t0indarr[goodshots])
max_t0shift = np.max(t0indarr[goodshots]) - min_t0ind
#Compute new nti
nti = nti - max_t0shift
t = t[0:nti] - t[min_t0ind]
#Throw an error if this dataset already exists
if 'data' in destgrp.keys():
raise hdftools.hdfDatasetExists(str(dest) + ' -> ' + "'data'")
#Create the dataset 'data' appropriate to whether or not output
#data will be gridded
if verbose:
print("Creating 'data' group in destination file")
if grid:
destgrp.require_dataset('data', (nti, nx, ny, nz, nreps, nchan), np.float32, chunks=(np.min([nti, 20000]),1,1,1,1,1), compression='gzip')
else:
destgrp.require_dataset('data', (nshots, nti, nchan), np.float32, chunks=(1, np.min([nti, 20000]), 1), compression='gzip')
# dt -> s
dt = ( attrs['dt'][0]*u.Unit(attrs['dt'][1])).to(u.s).value
if calibrate:
#First calculate the low frequency calibration factors
calAx, calAy, calAz = calibrationFactorsLF(attrs)
#If HF calibration factors are provided, calculate those
#calibraton constants too
if 'xtau' in attrs.keys() and highfreq_calibrate:
calBx, calBy, calBz = calibrationFactorsHF(attrs)
else:
calBx, calBy, calBz = None,None,None
#This segment of code checks for bad shots and replaces them with
#Neighboring good shots
shotlist = np.arange(nshots)
if replace_badshots and tdiode_hdf is not None:
for i in shotlist:
if i in badshots:
#If the shot is bad, determine the best neighbor shot
#to replace it with
before_shot = i
after_shot = i
#Identify nearest good shot before and after
while before_shot in badshots:
before_shot = before_shot - 1
while after_shot in badshots:
after_shot = after_shot + 1
#If position data is provided, use that to determine
#the best match
if 'pos' in srcgrp:
before_dist = (np.power(pos[i,0] - pos[before_shot,0],2) +
np.power(pos[i,1] - pos[before_shot,1],2) +
np.power(pos[i,2] - pos[before_shot,2],2) )
after_dist = (np.power(pos[i,0] - pos[after_shot,0],2) +
np.power(pos[i,1] - pos[after_shot,1],2) +
np.power(pos[i,2] - pos[after_shot,2],2) )
if before_dist > after_dist:
best_match = after_shot
else:
best_match = before_shot
#Otherwise just chose the earlier shot as the default
else:
best_match = before_shot
if verbose:
print("Replaced bad shot " + str(i) + " with " + str(best_match))
#Actually make the substitution
shotlist[i] = best_match
#Initialize time-remaining printout
tr = util.timeRemaining(nshots)
if verbose:
print("Beginning processing data shot-by-shot.")
#Chunking data processing loop limits memory usage
for ind in range(nshots):
#i == ind unless this is a bad shot
i = shotlist[ind]
#Update time remaining
if verbose:
tr.updateTimeRemaining(i)
#If a tdiode hdf was supplied, calculate the index correction
#here
if tdiode_hdf is not None and remove_offset:
#Calculate the starting and ending arrays for the data
ta = t0indarr[ind] - min_t0ind
tb = ta + nti
#Calculate the range over which to calculate the offset
#for each shot
#If offset_rel_t0 is set for either point, add the t0 array
if offset_rel_t0[0]:
offset_a = offset_range[0] + t0indarr[i] - ta
else:
offset_a = offset_range[0]
if offset_rel_t0[1]:
offset_b = offset_range[1] + t0indarr[i] - ta
else:
offset_b = offset_range[1]
#added this to deal with cases where you have a timing diode but don't want to remove voltage offset
elif tdiode_hdf is not None and remove_offset == False:
#Calculate the starting and ending arrays for the data
ta = t0indarr[ind] - min_t0ind
tb = ta + nti
offset_a = offset_range[0]
offset_b = offset_range[1]
else:
#By default, read in the entire dataset
ta = None
tb = None
offset_a = offset_range[0]
offset_b = offset_range[1]
if debug:
print("Data range: [" + str(ta) + "," + str(tb) + "]")
print("Offset range: [" + str(offset_a) + "," +
str(offset_b) + "]")
#Read in the data from the source file
dbx = srcgrp['data'][i,ta:tb, 0]
dby = srcgrp['data'][i,ta:tb, 1]
dbz = srcgrp['data'][i,ta:tb, 2]
if remove_offset:
#Remove offset from each channel
dbx = dbx - np.mean(dbx[offset_a:offset_b])
dby = dby - np.mean(dby[offset_a:offset_b])
dbz = dbz - np.mean(dbz[offset_a:offset_b])
if integrate:
#Intgrate
bx = np.cumsum(dbx)*dt
by = np.cumsum(dby)*dt
bz = np.cumsum(dbz)*dt
else:
bx,by,bz = dbx, dby, dbz
if calibrate:
#Apply the high-frequency calibration if one was
#provided
if calBx is not None and highfreq_calibrate:
bx = bx + calBx*dbx
by = by + calBy*dby
bz = bz + calBz*dbz
#Apply the low-frequency calibration factors
#Probe pol dir is included in these
bx = bx*calAx
by = by*calAy
bz = bz*calAz
#If a motion_format is set, apply the appropriate probe angle correction
if motion_format == 'cartesian' and angle_correction:
#Don't need to make any correction
pass
elif motion_format == 'fixed_pivot' and angle_correction:
#x,y,z is the probe's current position
x,y,z = srcgrp['pos'][i, :]
#rx, ry, rz is the location of the probe rotation point
#i.e. the center of the ball valve.
rx, ry, rz = attrs['rot_center_x'][0],attrs['rot_center_y'][0],attrs['rot_center_z'][0]
#x-rx, y-ry, z-rz is a vector pointing along the probe
#shaft towards the probe tip
#pitch is the angle of the probe shaft to the xz plane
pitch = np.arctan( (y-ry) / (x-rx) )
#yaw is the angle of the probe shaft to the xy plane
yaw = np.arctan( (z-rz) / (x-rx) )
if debug:
print("****Fixed Pivot Debug*******")
print("(x,y,z) = ({:5.2f},{:5.2f},{:5.2f})".format(x,y,z))
print("(rx,ry,rz) = ({:5.2f},{:5.2f},{:5.2f})".format(rx,ry,rz))
print("Pitch: " + str(np.degrees(pitch)))
print("Yaw: " + str(np.degrees(yaw)))
#If the probe is coming from the -X direction, its calibrated Z axis is already off by 180 degrees.
#This is because the probes are calibrated to match the East side of LAPD
if ((x-rx) > 0.0):
yaw = yaw + np.pi
#Roll is rotation of the probe about its axis, with
#y+ oriented up as roll=0
#This should be zero, unless a probe was later discovered
#to be incorrectly calibrated, so that the +Y mark was
#wrong
roll, unit = attrs['roll']
if unit != 'rad':
np.radians(roll)
#Matrix is the first Tait-Bryan matrix XZY from https://en.wikipedia.org/wiki/Euler_angles
#1 -> roll
#2 -> pitch
#3 -> yaw
bx = (np.cos(pitch)*np.cos(yaw)*bx -
np.sin(pitch)*by +
np.cos(pitch)*np.sin(yaw)*bz)
by = ((np.sin(roll)*np.sin(yaw) + np.cos(roll)*np.cos(yaw)*np.sin(pitch))*bx +
np.cos(roll)*np.cos(pitch)*by +
(np.cos(roll)*np.sin(pitch)*np.sin(yaw) - np.cos(yaw)*np.sin(roll))*bz)
bz = ((np.cos(yaw)*np.sin(roll)*np.sin(pitch) - np.cos(roll)*np.sin(yaw))*bx +
np.cos(pitch)*np.sin(roll)*by +
(np.cos(roll)*np.cos(yaw) + np.sin(roll)*np.sin(pitch)*np.sin(yaw))*bz)
if grid:
#Get location to write this datapoint from the shotgridind
xi = shotgridind[ind, 0]
yi = shotgridind[ind, 1]
zi = shotgridind[ind, 2]
repi = shotgridind[ind, 3]
#Write data
try:
#print(f"length destgrp selected {len(destgrp['data'][:, xi, yi, zi, repi, 0])}")
destgrp['data'][:, xi, yi, zi, repi, 0] = bx
destgrp['data'][:, xi, yi, zi, repi, 1] = by
destgrp['data'][:, xi, yi, zi, repi, 2] = bz
except ValueError as e:
print("ERROR!")
print(destgrp['data'].shape)
print(bx.shape)
print([xi, yi, zi, repi])
raise(e)
else:
#Write data
destgrp['data'][ind,:, 0] = bx
destgrp['data'][ind,:, 1] = by
destgrp['data'][ind,:, 2] = bz
if verbose:
print("Writing axes to destination file")
#Write the axes as required by the format of the data written
if motion_format is not None:
#Add the other axes and things we'd like in this file
destgrp.require_dataset('pos', (nshots, 3), np.float32, chunks=True)[:] = srcgrp['pos'][0:nshots]
for k in srcgrp['pos'].attrs.keys():
destgrp['pos'].attrs[k] = srcgrp['pos'].attrs[k]
if grid:
dimlabels = ['time', 'xaxis', 'yaxis', 'zaxis', 'reps', 'chan']
destgrp.require_dataset('xaxis', (nx,), np.float32, chunks=True)[:] = xaxis
destgrp['xaxis'].attrs['unit'] = attrs['motion_unit'][0]
destgrp.require_dataset('yaxis', (ny,), np.float32, chunks=True)[:] = yaxis
destgrp['yaxis'].attrs['unit'] = attrs['motion_unit'][0]
destgrp.require_dataset('zaxis', (nz,), np.float32, chunks=True)[:] = zaxis
destgrp['zaxis'].attrs['unit'] = attrs['motion_unit'][0]
destgrp.require_dataset('reps', (nreps,), np.int32, chunks=True)[:] = np.arange(nreps)
destgrp['reps'].attrs['unit'] = ''
else:
dimlabels = ['shots', 'time', 'chan']
destgrp.require_dataset('shots', (nshots,), np.int32, chunks=True)[:] = srcgrp['shots'][:]
destgrp['shots'].attrs['unit'] = srcgrp['shots'].attrs['unit']
destgrp.require_dataset('chan', (nchan,), np.int32, chunks=True)[:] = srcgrp['chan'][:]
destgrp['chan'].attrs['unit'] = srcgrp['chan'].attrs['unit']
destgrp.require_dataset('time', (nti,), np.float32, chunks=True)
destgrp['time'][:] = t
destgrp['time'].attrs['unit'] = srcgrp['time'].attrs['unit']
if calibrate:
destgrp['data'].attrs['unit'] = 'G'
else:
destgrp['data'].attrs['unit'] = 'V'
destgrp['data'].attrs['dimensions'] = [s.encode('utf-8') for s in dimlabels]
del(bx,by,bz)
if verbose:
print("End of BDOT routine!")
return True
def tdiodeRawToFull(src,
dest,
verbose=False,
badshotratio=None,
fatal_badshot_percentage=None):
with h5py.File(src.file, 'r') as sf:
srcgrp = sf[src.group]
#Get an array of all the t0 indices
t0indarr = calcT0ind(srcgrp, verbose=verbose)
#Get an array of all the good shots and badshots (indices)
badshots, goodshots = findBadShots(
srcgrp,
t0indarr,
verbose=verbose,
badshotratio=badshotratio,
fatal_badshot_percentage=fatal_badshot_percentage)
#Replace any bad shots with a standin avg value
#(Later should overwrite bad shots with good neighboring shots)
t0indarr[badshots] = int(np.median(t0indarr[goodshots]))
t = srcgrp['time'][:]
nshots = srcgrp['shots'].shape[0]
nti = srcgrp['time'].shape[0]
nchan = srcgrp['chan'].shape[0]
#Find the t0 for each t0ind
t0arr = t[t0indarr.astype(int)]
#Create the destination file directory if necessary
hdftools.requireDirs(dest.file)
with h5py.File(dest.file) as df:
destgrp = df[dest.group]
destgrp['t0arr'] = t0arr
destgrp['t0indarr'] = t0indarr
destgrp['badshots'] = badshots
destgrp['goodshots'] = goodshots
hdftools.copyAttrs(srcgrp, destgrp)
#Apply the tdiode correction to the data, as a check
#If this is working correctly, the full tdiode files will show
#the tdiode's all lined up...
min_t0ind = np.min(t0indarr[goodshots])
max_t0shift = np.max(t0indarr[goodshots]) - min_t0ind
#Compute new nti
nti = nti - max_t0shift
destgrp.require_dataset('data', (nshots, nti, nchan),
np.float32,
chunks=(1, np.min([nti, 20000]), 1),
compression='gzip')
destgrp['data'].attrs['dimensions'] = srcgrp['data'].attrs[
'dimensions']
destgrp['data'].attrs['unit'] = srcgrp['data'].attrs['unit']
for i in range(0, nshots):
ta = t0indarr[i] - min_t0ind
tb = ta + nti
destgrp['data'][i, :, :] = srcgrp['data'][i, ta:tb, :]
destgrp.require_dataset('time', (nti, ), np.float32,
chunks=True)[:] = t[0:nti] - t[min_t0ind]
destgrp['time'].attrs['unit'] = srcgrp['time'].attrs['unit']
destgrp.require_dataset('chan', (nchan, ), np.int32,
chunks=True)[:] = srcgrp['chan'][:]
destgrp['chan'].attrs['unit'] = srcgrp['chan'].attrs['unit']
destgrp.require_dataset('shots', (nshots, ), np.int32,
chunks=True)[:] = srcgrp['shots'][:]
destgrp['shots'].attrs['unit'] = srcgrp['shots'].attrs['unit']
return dest
def isatRawToFull(src,
dest,
ti=1.0,
mu=4.0,
tdiode_hdf=None,
grid=False,
offset_range=(0, 100),
offset_rel_t0=(False, False),
verbose=False,
debug=False,
grid_precision=0.1,
strict_grid=False,
strict_axes=False):
""" Integrates isat Langmuir probe data, calibrates output using information about the probe.
Parameters
----------
src: hdfPath object
Path string to a raw hdf5 file containing data
dest: hdfPath object
Path string to location processed data should be written out
ti: Ion temperature (eV). Default assumption is 1 eV, which is typical
of the LAPD LaB6 plasma. Scaling is as 1/sqrt(Ti).
mu: Ion mass number (m_i/m_p = mu). Default is 4.0, for Helium.
tdiode_hdf: hdfPath object
Path to a raw hdf5 file containing tdiode data. If no HDF file is
provided, no timing correction will be applied.
grid: Boolean
If grid is true, output will be written in cartesian grid array
format, eg. [nti, nx, ny, nz, nreps, nchan]. Otherwise, output will
be in [nshots, nti, nchan] format
offset_range: tuple
Tuple of indices between which the average of the signal will be
computed and subtracted from the entire signal to correct for
offset. This should be a segment with just noise, ideally at the
very beginning of the dataset. Longer is better.
Default is (0,100)
offset_rel_t0: Tuple of booleans
If either of these values is set to True, the coorresponding
offset_range value will be taken to be relative to the t0 index
for that each shot. For example, if t0=2000 for a shot,
offset_range=(10, -100), and offset_rel_t0 = (False, True), then
the offset will be computed over the range (10, 1900)
grid_precision: float
This is the precision to which position values will be rounded
before being fit onto the grid. Only applies to fuzzy axis and grid
creation.
strict_axes: boolean
If true, attempt to calculate axes from saved grid parameters.
Default is false, which attempts to calculate axes by looking at
position values.
strict_grid: boolean
If true, strictly unravel data onto the axes, assuming the probe
moved in order reps->X->Y->Z. This will NOT correctly handle
points where the probe was not at the requested position. Default
is false, which applys "fuzzy gridding", which tries to find the
best grid position for each shot individually.
Returns
-------
True (if executes to the end)
"""
# ******
# Load data from the raw HDF file
# ******
with h5py.File(src.file, 'r') as sf:
#Get the datagroup
srcgrp = sf[src.group]
#Create dictionary of attributes
attrs = hdftools.readAttrs(srcgrp)
#Check for keys always required by this function
req_keys = [
'area', 'atten', 'gain', 'resistor', 'dir', 'pol',
'probe_origin_x', 'probe_origin_y', 'probe_origin_z', 'dt'
]
if 'pos' in srcgrp:
pos = srcgrp['pos'][:] #Read the entire array in
else:
#If no position information is given, a single explicit position
#is required.
req_keys = req_keys + ['probe_xpos', 'probe_ypos', 'probe_zpos']
grid = False #Can't grid data if there's no pos array!
#Process the required keys, throwing an error if any cannot be found
csvtools.missingKeys(attrs, req_keys, fatal_error=True)
#Extract the shape of the source data
nshots, nti, nchan = srcgrp['data'].shape
#If requested by keyword, apply gridding
if grid:
shotgridind, xaxis, yaxis, zaxis, nx, ny, nz, nreps, nshots = postools.grid(
pos,
attrs,
strict_axes=strict_axes,
strict_grid=strict_grid,
grid_precision=grid_precision,
invert=False)
if verbose:
print("Opening destination HDF file")
#Create the destination file directory if necessary
hdftools.requireDirs(dest.file)
#Open the destination file
#This exists WITHIN the open statement for the source file, so the
#source file is open at the same time.
#remove files if they already exist
if os.path.exists(dest.file):
os.remove(dest.file)
with h5py.File(dest.file, 'a') as df:
#Throw an error if this group already exists
if dest.group is not '/' and dest.group in df.keys():
raise hdftools.hdfGroupExists(dest)
destgrp = df.require_group(dest.group)
#Copy over attributes
hdftools.copyAttrs(srcgrp, destgrp)
#Load the time vector
t = srcgrp['time']
#If tdiode_hdf is set, load the pre-processed tdiode data
if tdiode_hdf is not None:
if verbose:
print("Loading tdiode array from file.")
with h5py.File(tdiode_hdf.file, 'r') as sf:
grp = sf[tdiode_hdf.group]
t0indarr = grp['t0indarr'][:]
goodshots = grp['goodshots'][:]
tdiode_attrs = hdftools.readAttrs(grp)
#If tdiode was digitized with a different dt, this correction
#will be necessary
dt_ratio = float(attrs['dt'][0]) / float(tdiode_attrs['dt'][0])
t0indarr = (t0indarr / dt_ratio).astype(np.int32)
#We will remove up to max_t0shift indices from each array such that
#the t0 indices all line up.
min_t0ind = np.min(t0indarr[goodshots])
max_t0shift = np.max(t0indarr[goodshots]) - min_t0ind
#Compute new nti
nti = nti - max_t0shift
t = t[0:nti] - t[min_t0ind]
#Throw an error if this dataset already exists
if 'data' in destgrp.keys():
raise hdftools.hdfDatasetExists(str(dest) + ' -> ' + "'data'")
#Create the dataset 'data' appropriate to whether or not output
#data will be gridded
if verbose:
print("Creating 'data' group in destination file")
if grid:
destgrp.require_dataset('data', (nti, nx, ny, nz, nreps),
np.float32,
chunks=(np.min([nti,
20000]), 1, 1, 1, 1),
compression='gzip')
else:
destgrp.require_dataset('data', (nshots, nti),
np.float32,
chunks=(1, np.min([nti, 20000])),
compression='gzip')
dt = (attrs['dt'][0] * u.Unit(attrs['dt'][1])).to(u.s).value
resistor = float(attrs['resistor'][0]) #Ohms
area = (attrs['area'][0] * u.Unit(attrs['area'][1])).to(
u.m**2).value
#Initialize time-remaining printout
tr = util.timeRemaining(nshots)
if verbose:
print("Beginning processing data shot-by-shot.")
#Chunking data processing loop limits memory usage
for i in range(nshots):
#Update time remaining
if verbose:
tr.updateTimeRemaining(i)
#If a tdiode hdf was supplied, calculate the index correction
#here
if tdiode_hdf is not None:
#Calculate the starting and ending arrays for the data
ta = t0indarr[i] - min_t0ind
tb = ta + nti
else:
#By default, read in the entire dataset
ta = None
tb = None
if debug:
print("Data range: [" + str(ta) + "," + str(tb) + "]")
#Read in the data from the source file
voltage = srcgrp['data'][i, ta:tb, 0]
#Calculate density
#Equation is 2 from this paper: 10.1119/1.2772282
#This is valid for the regime Te~Ti, which is approx true in
#LAPD
density = 1.6e9 * np.sqrt(mu) * voltage / (resistor * area
) #cm^-3
if grid:
#Get location to write this datapoint from the shotgridind
xi = shotgridind[i, 0]
yi = shotgridind[i, 1]
zi = shotgridind[i, 2]
repi = shotgridind[i, 3]
#Write data
try:
destgrp['data'][:, xi, yi, zi, repi] = density
except ValueError as e:
print("ERROR!")
print(destgrp['data'].shape)
print(voltage.shape)
print([xi, yi, zi, repi])
raise (e)
else:
#Write data
destgrp['data'][i, :] = density
if verbose:
print("Writing axes to destination file")
if grid:
#Add the other axes and things we'd like in this file
destgrp.require_dataset(
'pos', (nshots, 3), np.float32,
chunks=True)[:] = srcgrp['pos'][0:nshots]
for k in srcgrp['pos'].attrs.keys():
destgrp['pos'].attrs[k] = srcgrp['pos'].attrs[k]
dimlabels = ['time', 'xaxis', 'yaxis', 'zaxis', 'reps']
destgrp.require_dataset('xaxis', (nx, ),
np.float32,
chunks=True)[:] = xaxis
destgrp['xaxis'].attrs['unit'] = attrs['motion_unit'][0]
destgrp.require_dataset('yaxis', (ny, ),
np.float32,
chunks=True)[:] = yaxis
destgrp['yaxis'].attrs['unit'] = attrs['motion_unit'][0]
destgrp.require_dataset('zaxis', (nz, ),
np.float32,
chunks=True)[:] = zaxis
destgrp['zaxis'].attrs['unit'] = attrs['motion_unit'][0]
destgrp.require_dataset('reps', (nreps, ),
np.int32,
chunks=True)[:] = np.arange(nreps)
destgrp['reps'].attrs['unit'] = ''
else:
dimlabels = ['shots', 'time']
destgrp.require_dataset('shots', (nshots, ),
np.int32,
chunks=True)[:] = srcgrp['shots'][:]
destgrp['shots'].attrs['unit'] = srcgrp['shots'].attrs['unit']
destgrp.require_dataset('time', (nti, ), np.float32, chunks=True)
destgrp['time'][:] = t
destgrp['time'].attrs['unit'] = srcgrp['time'].attrs['unit']
destgrp['data'].attrs['unit'] = 'cm^{-3}'
destgrp['data'].attrs['dimensions'] = [
s.encode('utf-8') for s in dimlabels
]
if verbose:
print("End of isat Langmuir routine!")
return True
def vsweepLangmuirRawToFull(src,
ndest,
tdest,
grid=True,
verbose=False,
plots=False,
debug=False,
grid_precision=0.1,
strict_grid=False,
strict_axes=False):
""" Fits sweept Langmuir probe data and creates two full save files
containing the calculated density and temperature
Parameters
----------
src: hdfPath object
Path string to a raw hdf5 file containing swept Langmuir probe data
There should be two channels: the first being the Langmuir current
and the second being the ramp voltage.
ndest: hdfPath object
Path string to location processed density data is written out
tdest: hdfPath object
Path string to location processed temperature data is written out
grid: Boolean
If grid is true, output will be written in cartesian grid array
format, eg. [nti, nx, ny, nz, nreps, nchan]. Otherwise, output will
be in [nshots, nti, nchan] format
grid_precision: float
This is the precision to which position values will be rounded
before being fit onto the grid. Only applies to fuzzy axis and grid
creation.
strict_axes: boolean
If true, attempt to calculate axes from saved grid parameters.
Default is false, which attempts to calculate axes by looking at
position values.
strict_grid: boolean
If true, strictly unravel data onto the axes, assuming the probe
moved in order reps->X->Y->Z. This will NOT correctly handle
points where the probe was not at the requested position. Default
is false, which applys "fuzzy gridding", which tries to find the
best grid position for each shot individually.
Returns
-------
True (if executes to the end)
"""
# ******
# Load data from the raw HDF file
# ******
with h5py.File(src.file, 'r') as sf:
#Get the datagroup
srcgrp = sf[src.group]
#Create dictionary of attributes
attrs = hdftools.readAttrs(srcgrp)
#Check for keys always required by this function
req_keys = [
'area', 'resistor', 'gain', 'atten', 'ramp_gain', 'ramp_atten',
'probe_origin_x', 'probe_origin_y', 'probe_origin_z'
]
if 'pos' in srcgrp:
pos = srcgrp['pos'][:] #Read the entire array in
else:
#If no position information is given, a single explicit position
#is required.
req_keys = req_keys + ['xpos', 'ypos', 'zpos']
grid = False #Can't grid data if there's no pos array!
#Process the required keys, throwing an error if any cannot be found
csvtools.missingKeys(attrs, req_keys, fatal_error=True)
#Extract the shape of the source data
nshots, nti, nchan = srcgrp['data'].shape
#If requested by keyword, apply gridding
if grid:
shotlist, xaxis, yaxis, zaxis, nx, ny, nz, nreps, nshots = postools.grid(
pos,
attrs,
strict_axes=strict_axes,
strict_grid=strict_grid,
grid_precision=grid_precision,
invert=True)
if verbose:
print("Opening destination HDF files")
#Create the destination file directory if necessary
#hdftools.requireDirs(ndest.file)
#hdftools.requireDirs(tdest.file)
#Open the destination file
#This exists WITHIN the open statement for the source file, so the
#source file is open at the same time.
#Check if the output files exist already: if so, delete them
if os.path.exists(ndest.file):
os.remove(ndest.file)
if os.path.exists(tdest.file):
os.remove(tdest.file)
with h5py.File(ndest.file, 'a') as ndf:
with h5py.File(tdest.file, 'a') as tdf:
#Throw an error if this group already exists
if ndest.group != '/' and ndest.group in ndf.keys():
raise hdftools.hdfGroupExists(ndest)
if tdest.group != '/' and tdest.group in tdf.keys():
raise hdftools.hdfGroupExists(tdest)
ndestgrp = ndf.require_group(ndest.group)
tdestgrp = tdf.require_group(tdest.group)
grps = [ndestgrp, tdestgrp]
for grp in grps:
hdftools.copyAttrs(srcgrp, grp)
#Throw an error if this dataset already exists
if 'data' in grp.keys():
raise hdftools.hdfDatasetExists(
str(grp) + ' -> ' + "'data'")
#Determine the time vector and nti
#Assume first shot is representative of the vramp
vramp = srcgrp['data'][0, :, 1]
time = srcgrp['time'][:]
peaktimes, start, end = find_sweeps(time, vramp, plots=plots)
nti = len(peaktimes)
time = peaktimes
#Create the dataset 'data' appropriate to whether or not output
#data will be gridded
if verbose:
print("Creating 'data' group in destination file")
if grid:
ndestgrp.require_dataset('data', (nti, nx, ny, nz),
np.float32,
chunks=True,
compression='gzip')
ndestgrp.require_dataset('error', (nti, nx, ny, nz, 5),
np.float32,
chunks=True,
compression='gzip')
tdestgrp.require_dataset('data', (nti, nx, ny, nz),
np.float32,
chunks=True,
compression='gzip')
tdestgrp.require_dataset('error', (nti, nx, ny, nz, 5),
np.float32,
chunks=True,
compression='gzip')
else:
ndestgrp.require_dataset('data', (nti, ),
np.float32,
chunks=True,
compression='gzip')
ndestgrp.require_dataset('error', (nti, 5),
np.float32,
chunks=True,
compression='gzip')
tdestgrp.require_dataset('data', (nti, ),
np.float32,
chunks=True,
compression='gzip')
tdestgrp.require_dataset('error', (nti, 5),
np.float32,
chunks=True,
compression='gzip')
probe_gain = float(attrs['gain'][0])
probe_atten = float(attrs['atten'][0])
ramp_gain = float(attrs['ramp_gain'][0])
ramp_atten = float(attrs['ramp_atten'][0])
resistor = float(attrs['resistor'][0]) #Ohms
area = (attrs['area'][0] * u.Unit(attrs['area'][1])).to(
u.cm**2).value
probe_calib = np.power(10, probe_atten / 20.0) / probe_gain
ramp_calib = np.power(10, ramp_atten / 20.0) / ramp_gain
if grid:
#Initialize time-remaining printout
tr = util.timeRemaining(nx * ny * nz, reportevery=20)
for xi in range(nx):
for yi in range(ny):
for zi in range(nz):
i = zi + yi * nz + xi * nz * ny
if verbose:
tr.updateTimeRemaining(i)
s = shotlist[xi, yi, zi, :]
current = srcgrp['data'][
s, :, 0] * probe_calib / resistor
vramp = srcgrp['data'][s, :, 1] * ramp_calib
#Average over shots
current = np.mean(current, axis=0)
vramp = np.mean(vramp, axis=0)
for ti in range(nti):
a = int(start[ti])
b = int(end[ti])
vpp, kTe, esat, vthe, density, error = vsweep_fit(
vramp[a:b],
current[a:b],
esat_range=None,
exp_range=None,
plots=False,
area=area)
ndestgrp['data'][ti, xi, yi, zi] = density
ndestgrp['error'][ti, xi, yi,
zi, :] = error
tdestgrp['data'][ti, xi, yi, zi] = kTe
tdestgrp['error'][ti, xi, yi,
zi, :] = error
else: #Not gridded
current = srcgrp['data'][:, :, 0]
current = np.mean(current, axis=0) * probe_calib / resistor
vramp = srcgrp['data'][:, :, 1]
vramp = np.mean(vramp, axis=0) * ramp_calib
for ti in range(nti):
a = start[ti]
b = end[ti]
vpp, kTe, esat, vthe, density, error = vsweep_fit(
vramp[a:b],
current[a:b],
esat_range=None,
exp_range=None,
plots=False,
area=area)
ndestgrp['data'][ti] = density
ndestgrp['error'][ti, :] = error
tdestgrp['data'][ti] = kTe
tdestgrp['error'][ti, :] = error
for grp in grps:
#Write the axes as required by the format of the data written
if grid:
grp.require_dataset(
'pos', (nshots, 3), np.float32,
chunks=True)[:] = srcgrp['pos'][0:nshots]
for k in srcgrp['pos'].attrs.keys():
grp['pos'].attrs[k] = srcgrp['pos'].attrs[k]
dimlabels = ['time', 'xaxis', 'yaxis', 'zaxis']
grp.require_dataset('xaxis', (nx, ),
np.float32,
chunks=True)[:] = xaxis
grp['xaxis'].attrs['unit'] = attrs['motion_unit'][0]
grp.require_dataset('yaxis', (ny, ),
np.float32,
chunks=True)[:] = yaxis
grp['yaxis'].attrs['unit'] = attrs['motion_unit'][0]
grp.require_dataset('zaxis', (nz, ),
np.float32,
chunks=True)[:] = zaxis
grp['zaxis'].attrs['unit'] = attrs['motion_unit'][0]
else:
dimlabels = ['time']
grp.require_dataset('time', (nti, ),
np.float32,
chunks=True)
grp['time'][:] = time
grp['time'].attrs['unit'] = srcgrp['time'].attrs['unit']
grp['data'].attrs['unit'] = 'G'
ndestgrp['data'].attrs['unit'] = 'cm^{-3}'
tdestgrp['data'].attrs['unit'] = 'eV'
ndestgrp['data'].attrs['dimensions'] = [
s.encode('utf-8') for s in dimlabels
]
tdestgrp['data'].attrs['dimensions'] = [
s.encode('utf-8') for s in dimlabels
]
if verbose:
print("End of Sweept Langmuir routine!")
return True