def readgssi(infile,
outfile=None,
verbose=False,
antfreq=None,
frmt='python',
plotting=False,
figsize=7,
dpi=150,
stack=1,
x='seconds',
z='nanoseconds',
histogram=False,
colormap='gray',
colorbar=False,
zero=[None, None, None, None],
gain=1,
freqmin=None,
freqmax=None,
reverse=False,
bgr=False,
win=0,
dewow=False,
normalize=False,
specgram=False,
noshow=False,
spm=None,
start_scan=0,
num_scans=-1,
epsr=None,
title=True,
zoom=[0, 0, 0, 0]):
"""
This is the primary directive function. It coordinates calls to reading, filtering, translation, and plotting functions, and should be used as the overarching processing function in most cases.
:param str infile: Input DZT data file
:param str outfile: Base output file name for plots, CSVs, and other products. Defaults to :py:data:`None`, which will cause the output filename to take a form similar to the input. The default will let the file be named via the descriptive naming function :py:data:`readgssi.functions.naming()`.
:param bool verbose: Whether or not to display (a lot of) information about the workings of the program. Defaults to :py:data:`False`. Can be helpful for debugging but also to see various header values and processes taking place.
:param int antfreq: User setting for antenna frequency. Defaults to :py:data:`None`, which will cause the program to try to determine the frequency from the antenna name in the header of the input file. If the antenna name is not in the dictionary :py:data:`readgssi.constants.ANT`, the function will try to determine the frequency by decoding integers in the antenna name string.
:param str frmt: The output format to be passed to :py:mod:`readgssi.translate`. Defaults to :py:data:`None`. Presently, this can be set to :py:data:`frmt='csv'`, :py:data:`'numpy'`, :py:data:`'gprpy'`, or :py:data:`'object'` (which will return the header dictionary, the image arrays, and the gps coordinates as objects). Plotting will not interfere with output (i.e. you can output to CSV and plot a PNG in the same command).
:param bool plotting: Whether to plot the radargram using :py:func:`readgssi.plot.radargram`. Defaults to :py:data:`False`.
:param int figsize: Plot size in inches to be passed to :py:func:`readgssi.plot.radargram`.
:param int dpi: Dots per inch (DPI) for figure creation.
:param int stack: Number of consecutive traces to stack (horizontally) using :py:func:`readgssi.arrayops.stack`. Defaults to 1 (no stacking). Especially good for handling long radar lines. Algorithm combines consecutive traces together using addition, which reduces noise and enhances signal. The more stacking is done, generally the clearer signal will become. The tradeoff is that you will reduce the length of the X-axis. Sometimes this is desirable (i.e. for long survey lines).
:param str x: The units to display on the x-axis during plotting. Defaults to :py:data:`x='seconds'`. Acceptable values are :py:data:`x='distance'` (which sets to meters), :py:data:`'km'`, :py:data:`'m'`, :py:data:`'cm'`, :py:data:`'mm'`, :py:data:`'kilometers'`, :py:data:`'meters'`, etc., for distance; :py:data:`'seconds'`, :py:data:`'s'`, :py:data:`'temporal'` or :py:data:`'time'` for seconds, and :py:data:`'traces'`, :py:data:`'samples'`, :py:data:`'pulses'`, or :py:data:`'columns'` for traces.
:param str z: The units to display on the z-axis during plotting. Defaults to :py:data:`z='nanoseconds'`. Acceptable values are :py:data:`z='depth'` (which sets to meters), :py:data:`'m'`, :py:data:`'cm'`, :py:data:`'mm'`, :py:data:`'meters'`, etc., for depth; :py:data:`'nanoseconds'`, :py:data:`'ns'`, :py:data:`'temporal'` or :py:data:`'time'` for seconds, and :py:data:`'samples'` or :py:data:`'rows'` for samples.
:param bool histogram: Whether to plot a histogram of array values at plot time.
:type colormap: :py:class:`str` or :class:`matplotlib.colors.Colormap`
:param colormap: Plot using a Matplotlib colormap. Defaults to :py:data:`gray` which is colorblind-friendly and behaves similarly to the RADAN default, but :py:data:`seismic` is a favorite of many due to its diverging nature.
:param bool colorbar: Whether to display a graded color bar at plot time.
:param list[int,int,int,int] zero: A list of values representing the amount of samples to slice off each channel. Defaults to :py:data:`None` for all channels, which will end up being set as :py:data:`[2,2,2,2]` for a four-channel file (2 is the number of rows down that GSSI stores mark information in).
:param int gain: The amount of gain applied to plots. Defaults to 1. Gain is applied as a ratio of the standard deviation of radargram values to the value set here.
:param int freqmin: Minimum frequency value to feed to the vertical triangular FIR bandpass filter :py:func:`readgssi.filtering.triangular`. Defaults to :py:data:`None` (no filter).
:param int freqmax: Maximum frequency value to feed to the vertical triangular FIR bandpass filter :py:func:`readgssi.filtering.triangular`. Defaults to :py:data:`None` (no filter).
:param bool reverse: Whether to read the array backwards (i.e. flip horizontally; :py:func:`readgssi.arrayops.flip`). Defaults to :py:data:`False`. Useful for lining up travel directions of files run opposite each other.
:param int bgr: Background removal filter applied after stacking (:py:func:`readgssi.filtering.bgr`). Defaults to :py:data:`False` (off). :py:data:`bgr=True` must be accompanied by a valid value for :py:data:`win`.
:param int win: Window size for background removal filter (:py:func:`readgssi.filtering.bgr`). If :py:data:`bgr=True` and :py:data:`win=0`, the full-width row average will be subtracted from each row. If :py:data:`bgr=True` and :py:data:`win=50`, a moving window will calculate the average of 25 cells on either side of the current cell, and subtract that average from the cell value, using :py:func:`scipy.ndimage.uniform_filter1d` with :py:data:`mode='constant'` and :py:data:`cval=0`. This is useful for removing non-uniform horizontal average, but the tradeoff is that it creates ghost data half the window size away from vertical figures, and that a window size set too low will obscure any horizontal layering longer than the window size.
:param bool dewow: Whether to apply a vertical dewow filter (experimental). See :py:func:`readgssi.filtering.dewow`.
:param bool normalize: Distance normalization (:py:func:`readgssi.arrayops.distance_normalize`). Defaults to :py:data:`False`.
:param bool specgram: Produce a spectrogram of a trace in the array using :py:func:`readgssi.plot.spectrogram`. Defaults to :py:data:`False` (if :py:data:`True`, defaults to a trace roughly halfway across the profile). This is mostly for debugging and is not currently accessible from the command line.
:param bool noshow: If :py:data:`True`, this will suppress the matplotlib interactive window and simply save a file. This is useful for processing many files in a folder without user input.
:param float spm: User-set samples per meter. This overrides the value read from the header, and typically doesn't need to be set if the samples per meter value was set correctly at survey time. This value does not need to be set if GPS input (DZG file) is present and the user sets :py:data:`normalize=True`.
:param int start_scan: zero based start scan to read data from. Defaults to zero.
:param int num_scans: number of scans to read from the file, Defaults to -1, which reads from start_scan to end of file.
:param float epsr: Epsilon_r, otherwise known as relative permittivity, or dielectric constant. This determines the speed at which waves travel through the first medium they encounter. It is used to calculate the profile depth if depth units are specified on the Z-axis of plots.
:param bool title: Whether to display descriptive titles on plots. Defaults to :py:data:`True`.
:param list[int,int,int,int] zoom: Zoom extents to set programmatically for matplotlib plots. Must pass a list of four integers: :py:data:`[left, right, up, down]`. Since the z-axis begins at the top, the "up" value is actually the one that displays lower on the page. All four values are axis units, so if you are working in nanoseconds, 10 will set a limit 10 nanoseconds down. If your x-axis is in seconds, 6 will set a limit 6 seconds from the start of the survey. It may be helpful to display the matplotlib interactive window at full extents first, to determine appropriate extents to set for this parameter. If extents are set outside the boundaries of the image, they will be set back to the boundaries. If two extents on the same axis are the same, the program will default to plotting full extents for that axis.
:rtype: header (:py:class:`dict`), radar array (:py:class:`numpy.ndarray`), gps (False or :py:class:`pandas.DataFrame`)
"""
if infile:
# read the file
try:
if verbose:
fx.printmsg('reading...')
fx.printmsg('input file: %s' % (infile))
r = readdzt(infile,
gps=normalize,
spm=spm,
start_scan=start_scan,
num_scans=num_scans,
epsr=epsr,
verbose=verbose)
# time zero per channel
r[0]['timezero'] = [None, None, None, None]
for i in range(r[0]['rh_nchan']):
try:
r[0]['timezero'][i] = int(list(zero)[i])
except (TypeError, IndexError):
fx.printmsg(
'WARNING: no time zero specified for channel %s, defaulting to 2'
% i)
r[0]['timezero'][i] = 2
# print a bunch of header info
if verbose:
fx.printmsg('success. header values:')
header_info(r[0], r[1])
except IOError as e: # the user has selected an inaccessible or nonexistent file
fx.printmsg(
"ERROR: DZT file is inaccessable or does not exist at %s" %
(os.path.abspath(infile)))
raise IOError(e)
infile_ext = os.path.splitext(infile)[1]
infile_basename = os.path.splitext(infile)[0]
else:
raise IOError('ERROR: no input file specified')
rhf_sps = r[0]['rhf_sps']
rhf_spm = r[0]['rhf_spm']
line_dur = r[0]['sec']
for chan in list(range(r[0]['rh_nchan'])):
try:
ANT[r[0]['rh_antname'][chan]]
except KeyError as e:
print(
'--------------------WARNING - PLEASE READ---------------------'
)
fx.printmsg(
'WARNING: could not read frequency for antenna name %s' % e)
if antfreq:
fx.printmsg('using user-specified antenna frequency.')
r[0]['antfreq'] = antfreq
else:
fx.printmsg(
'WARNING: trying to use frequency of %s MHz (estimated)...'
% (r[0]['antfreq'][chan]))
fx.printmsg('more info: rh_ant=%s' % (r[0]['rh_ant']))
fx.printmsg(' known_ant=%s' % (r[0]['known_ant']))
fx.printmsg(
"please submit a bug report with this warning, the antenna name and frequency"
)
fx.printmsg('at https://github.com/iannesbitt/readgssi/issues/new')
fx.printmsg(
'or send via email to ian (dot) nesbitt (at) gmail (dot) com.')
fx.printmsg(
'if possible, please attach a ZIP file with the offending DZT inside.'
)
print(
'--------------------------------------------------------------'
)
# create a list of n arrays, where n is the number of channels
arr = r[1].astype(np.int32)
chans = list(range(r[0]['rh_nchan']))
# set up list of arrays
img_arr = arr[:r[0]['rh_nchan'] * r[0][
'rh_nsamp']] # test if we understand data structure. arrays should be stacked nchan*nsamp high
new_arr = {}
for ar in chans:
a = []
a = img_arr[(ar) * r[0]['rh_nsamp']:(ar + 1) *
r[0]['rh_nsamp']] # break apart
new_arr[ar] = a[r[0]['timezero'][ar]:, :int(
img_arr.shape[1])] # put into dict form
img_arr = new_arr # overwrite
del arr, new_arr
for ar in img_arr:
"""
filter and construct an output file or plot from the current channel's array
"""
if verbose:
fx.printmsg('beginning processing for channel %s (antenna %s)' %
(ar, r[0]['rh_antname'][ar]))
# execute filtering functions if necessary
if normalize:
r[0], img_arr[ar], r[2] = arrayops.distance_normalize(
header=r[0], ar=img_arr[ar], gps=r[2], verbose=verbose)
if dewow:
# dewow
img_arr[ar] = filtering.dewow(ar=img_arr[ar], verbose=verbose)
if freqmin and freqmax:
# vertical triangular bandpass
img_arr[ar] = filtering.triangular(ar=img_arr[ar],
header=r[0],
freqmin=freqmin,
freqmax=freqmax,
zerophase=True,
verbose=verbose)
if stack != 1:
# horizontal stacking
img_arr[ar], stack = arrayops.stack(ar=img_arr[ar],
stack=stack,
verbose=verbose)
else:
stack = 1
if bgr:
# background removal
img_arr[ar] = filtering.bgr(ar=img_arr[ar],
header=r[0],
win=win,
verbose=verbose)
else:
win = None
if reverse:
# read array backwards
img_arr[ar] = arrayops.flip(img_arr[ar], verbose=verbose)
## file naming
# name the output file
if ar == 0: # first channel?
orig_outfile = outfile # preserve the original
else:
outfile = orig_outfile # recover the original
if outfile:
outfile_ext = os.path.splitext(outfile)[1]
outfile = '%s' % (os.path.join(os.path.splitext(outfile)[0]))
if len(chans) > 1:
outfile = '%sc%s' % (outfile, ar) # avoid naming conflicts
else:
outfile = fx.naming(infile_basename=infile_basename,
chans=chans,
chan=ar,
normalize=normalize,
zero=r[0]['timezero'][ar],
stack=stack,
reverse=reverse,
bgr=bgr,
win=win,
dewow=dewow,
freqmin=freqmin,
freqmax=freqmax,
plotting=plotting,
gain=gain)
if plotting:
plot.radargram(ar=img_arr[ar],
header=r[0],
freq=r[0]['antfreq'][ar],
verbose=verbose,
figsize=figsize,
dpi=dpi,
stack=stack,
x=x,
z=z,
gain=gain,
colormap=colormap,
colorbar=colorbar,
noshow=noshow,
outfile=outfile,
win=win,
title=title,
zero=r[0]['timezero'][ar],
zoom=zoom)
if histogram:
plot.histogram(ar=img_arr[ar], verbose=verbose)
if specgram:
plot.spectrogram(ar=img_arr[ar],
header=header,
freq=r[0]['antfreq'][ar],
verbose=verbose)
if frmt != None:
if verbose:
fx.printmsg('outputting to %s...' % frmt)
for ar in img_arr:
# is there an output filepath given?
outfile_abspath = os.path.abspath(
outfile) # set output to given location
# what is the output format
if frmt in 'csv':
translate.csv(ar=img_arr[ar],
outfile_abspath=outfile_abspath,
header=r[0],
verbose=verbose)
elif frmt in 'h5':
translate.h5(ar=img_arr[ar],
infile_basename=infile_basename,
outfile_abspath=outfile_abspath,
verbose=verbose)
elif frmt in 'segy':
translate.segy(ar=img_arr[ar],
outfile_abspath=outfile_abspath,
verbose=verbose)
elif frmt in 'numpy':
translate.numpy(ar=img_arr[ar],
outfile_abspath=outfile_abspath,
verbose=verbose)
elif frmt in 'gprpy':
translate.gprpy(ar=img_arr[ar],
outfile_abspath=outfile_abspath,
header=r[0],
verbose=verbose)
if frmt in ('object', 'python'):
return r[0], img_arr, r[2]
def readgssi(infile,
outfile=None,
antfreq=None,
frmt=None,
plotting=False,
figsize=10,
stack=1,
x='seconds',
z='nanoseconds',
verbose=False,
histogram=False,
colormap='Greys',
colorbar=False,
zero=0,
gain=1,
freqmin=None,
freqmax=None,
reverse=False,
bgr=False,
dewow=False,
normalize=False,
specgram=False,
noshow=False,
spm=None,
epsr=None):
"""
primary radar processing function
includes calls to reading, filtering, translation, and plotting functions
"""
if infile:
# read the file
try:
if verbose:
fx.printmsg('reading...')
fx.printmsg('input file: %s' % (infile))
r = readdzt(infile,
gps=normalize,
spm=spm,
epsr=epsr,
verbose=verbose)
if verbose:
fx.printmsg('success. header values:')
header_info(r[0], r[1])
except IOError as e: # the user has selected an inaccessible or nonexistent file
fx.printmsg(
"ERROR: DZT file is inaccessable or does not exist at %s" %
(os.path.abspath(infile)))
raise IOError(e)
infile_ext = os.path.splitext(infile)[1]
infile_basename = os.path.splitext(infile)[0]
else:
raise IOError('ERROR: no input file specified')
rhf_sps = r[0]['rhf_sps']
rhf_spm = r[0]['rhf_spm']
line_dur = r[0]['sec']
for chan in list(range(r[0]['rh_nchan'])):
try:
ANT[r[0]['rh_antname'][chan]]
except KeyError as e:
print(
'--------------------WARNING - PLEASE READ---------------------'
)
fx.printmsg(
'WARNING: could not read frequency for antenna name %s' % e)
if antfreq:
fx.printmsg('using user-specified antenna frequency.')
r[0]['antfreq'] = antfreq
else:
fx.printmsg(
'WARNING: trying to use frequency of %s MHz (estimated)...'
% (r[0]['antfreq'][chan]))
fx.printmsg('more info: rh_ant=%s' % (r[0]['rh_ant']))
fx.printmsg(' known_ant=%s' % (r[0]['known_ant']))
fx.printmsg(
"please submit a bug report with this warning, the antenna name and frequency"
)
fx.printmsg('at https://github.com/iannesbitt/readgssi/issues/new')
fx.printmsg(
'or send via email to ian (dot) nesbitt (at) gmail (dot) com.')
fx.printmsg(
'if possible, please attach a ZIP file with the offending DZT inside.'
)
print(
'--------------------------------------------------------------'
)
# create a list of n arrays, where n is the number of channels
arr = r[1].astype(np.int32)
chans = list(range(r[0]['rh_nchan']))
timezero = 0
#timezero = abs(round(float(r[0]['rh_nsamp'])/float(r[0]['rhf_range'])*float(r[0]['rhf_position'])))
img_arr = arr[timezero:r[0]['rh_nchan'] * r[0]['rh_nsamp']]
new_arr = {}
for ar in chans:
a = []
a = img_arr[(ar) * r[0]['rh_nsamp']:(ar + 1) * r[0]['rh_nsamp']]
new_arr[ar] = a[zero:, :int(img_arr.shape[1])]
img_arr = new_arr
del arr, new_arr
for ar in img_arr:
"""
filter and construct an output file or plot from the current channel's array
"""
if verbose:
fx.printmsg('beginning processing for channel %s (antenna %s)' %
(ar, r[0]['rh_antname'][ar]))
# execute filtering functions if necessary
if normalize:
r[0], img_arr[ar], r[2] = arrayops.distance_normalize(
header=r[0], ar=img_arr[ar], gps=r[2], verbose=verbose)
if stack != 1:
# horizontal stacking
img_arr[ar], stack = arrayops.stack(ar=img_arr[ar],
stack=stack,
verbose=verbose)
else:
stack = 1
if reverse:
img_arr[ar] = arrayops.flip(img_arr[ar], verbose=verbose)
if bgr:
# background removal
img_arr[ar] = filtering.bgr(ar=img_arr[ar], verbose=verbose)
if dewow:
# dewow
img_arr[ar] = filtering.dewow(ar=img_arr[ar], verbose=verbose)
if freqmin and freqmax:
# vertical bandpass
img_arr[ar] = filtering.bp(ar=img_arr[ar],
header=r[0],
freqmin=freqmin,
freqmax=freqmax,
verbose=verbose)
# name the output file
if ar == 0: # first channel?
orig_outfile = outfile # preserve the original
else:
outfile = orig_outfile # recover the original
if outfile:
outfile_ext = os.path.splitext(outfile)[1]
outfile = '%s' % (os.path.join(os.path.splitext(outfile)[0]))
if len(chans) > 1:
outfile = '%sc%s' % (outfile, ar) # avoid naming conflicts
else:
"""
~~~ The Seth Campbell Honorary Naming Scheme ~~~
"""
outfile = '%sc%s' % (os.path.join(infile_basename), ar)
if normalize:
outfile = '%sDn' % (outfile)
if zero and (zero > 0):
outfile = '%sTz%s' % (outfile, zero)
if stack > 1:
outfile = '%sS%s' % (outfile, stack)
if reverse:
outfile = '%sRv' % (outfile)
if bgr:
outfile = '%sBgr' % (outfile)
if dewow:
outfile = '%sDw' % (outfile)
if freqmin and freqmax:
outfile = '%sB%s-%s' % (outfile, freqmin, freqmax)
if plotting:
outfile = '%sG%s' % (outfile, int(gain))
if frmt != None:
if verbose:
fx.printmsg('outputting to %s...' % frmt)
for ar in img_arr:
# is there an output filepath given?
outfile_abspath = os.path.abspath(
outfile) # set output to given location
# what is the output format
if frmt in 'csv':
translate.csv(ar=img_arr[ar],
outfile_abspath=outfile_abspath,
header=r[0],
verbose=verbose)
elif frmt in 'h5':
translate.h5(ar=img_arr[ar],
infile_basename=infile_basename,
outfile_abspath=outfile_abspath,
verbose=verbose)
elif frmt in 'segy':
translate.segy(ar=img_arr[ar],
outfile_abspath=outfile_abspath,
verbose=verbose)
elif frmt in 'numpy':
translate.numpy(ar=img_arr[ar],
outfile_abspath=outfile_abspath,
verbose=verbose)
elif frmt in 'gprpy':
translate.gprpy(ar=img_arr[ar],
outfile_abspath=outfile_abspath,
header=r[0],
verbose=verbose)
if plotting:
plot.radargram(ar=img_arr[ar],
header=r[0],
freq=r[0]['antfreq'][ar],
verbose=verbose,
figsize=figsize,
stack=stack,
x=x,
z=z,
gain=gain,
colormap=colormap,
colorbar=colorbar,
noshow=noshow,
outfile=outfile)
if histogram:
plot.histogram(ar=img_arr[ar], verbose=verbose)
if specgram:
plot.spectrogram(ar=img_arr[ar],
header=header,
freq=r[0]['antfreq'][ar],
verbose=verbose)