def processTrackCoverage(filename, step, trackCoverageFileName, geo):
shp = SSDM.createCoverageSHP(trackCoverageFileName)
reader = pyall.ALLReader(filename)
# create the coverage polygon
createTrackCoverage(reader, shp, step, geo)
print("Trackcoverage created for: %s" % (filename))
reader.close()
return shp
def processTrackLine(filename, step, trackLineFileName, geo):
shp = SSDM.createSurveyTracklineSHP(trackLineFileName)
reader = pyall.ALLReader(filename)
# create the track polyline
totalDistanceRun = createTrackLine(reader, shp, step, geo)
print("Trackplot created for: %s, Length: %.3f" %
(filename, totalDistanceRun))
reader.close()
return shp
def processTrackPoint(filename, step, trackPointFileName, geo):
# dirname, basename = os.path.split(filename)
# trackPointFileName = tempfile.NamedTemporaryFile(prefix=basename, dir=dirname)
shp = SSDM.createPointShapeFile(trackPointFileName)
reader = pyall.ALLReader(filename)
# create the track point with a point recoard and metadata per ping
createTrackPoint(reader, shp, step, geo)
print("Trackpoint created for: %s" % (filename))
reader.close()
return shp
def loadNavigation(fileName):
'''loads all the navigation into lists'''
navigation = []
r = pyall.ALLReader(fileName)
while r.moreData():
TypeOfDatagram, datagram = r.readDatagram()
if (TypeOfDatagram == 'P'):
datagram.read()
navigation.append(
[datagram.Time, datagram.Latitude, datagram.Longitude])
r.close()
return navigation
def computeXYResolution(fileName):
'''compute the approximate across and alongtrack resolution so we can make a nearly isometric Image'''
'''we compute the across track by taking the average Dx value between beams'''
'''we compute the alongtracks by computing the linear length between all nav updates and dividing this by the number of pings'''
xResolution = 1
YResolution = 1
prevLong = 0
prevLat = 0
r = pyall.ALLReader(fileName)
recCount = 0
acrossMeans = np.array([])
alongIntervals = np.array([])
leftExtents = np.array([])
rightExtents = np.array([])
beamCount = 0
distanceTravelled = 0.0
navigation = []
selectedPositioningSystem = None
while r.moreData():
TypeOfDatagram, datagram = r.readDatagram()
if (TypeOfDatagram == 'P'):
datagram.read()
if (selectedPositioningSystem == None):
selectedPositioningSystem = datagram.Descriptor
if (selectedPositioningSystem == datagram.Descriptor):
if prevLat == 0:
prevLat = datagram.Latitude
prevLong = datagram.Longitude
range, bearing1, bearing2 = geodetic.calculateRangeBearingFromGeographicals(
prevLong, prevLat, datagram.Longitude, datagram.Latitude)
# print (range,bearing1)
distanceTravelled += range
navigation.append([
recCount,
r.currentRecordDateTime(), datagram.Latitude,
datagram.Longitude
])
prevLat = datagram.Latitude
prevLong = datagram.Longitude
if (TypeOfDatagram == 'X') or (TypeOfDatagram == 'D'):
datagram.read()
if datagram.NBeams > 1:
datagram.AcrossTrackDistance = [
x for x in datagram.AcrossTrackDistance if x != 0.0
]
if (len(datagram.AcrossTrackDistance) > 0):
acrossMeans = np.append(
acrossMeans,
np.average(
abs(
np.diff(
np.asarray(
datagram.AcrossTrackDistance)))))
leftExtents = np.append(leftExtents,
min(datagram.AcrossTrackDistance))
rightExtents = np.append(rightExtents,
max(datagram.AcrossTrackDistance))
recCount = recCount + 1
beamCount = max(beamCount, len(datagram.Depth))
r.close()
if recCount == 0:
return 0, 0, 0, 0, 0, []
xResolution = np.average(acrossMeans)
# distanceTravelled = 235
yResolution = distanceTravelled / recCount
return xResolution, yResolution, beamCount, np.min(leftExtents), np.max(
rightExtents), distanceTravelled, navigation
def createWaterfall(filename,
colors,
beamCount,
shadeScale=1,
zoom=1.0,
annotate=True,
xResolution=1,
yResolution=1,
rotate=False,
gray=False,
leftExtent=-100,
rightExtent=100,
distanceTravelled=0,
navigation=[]):
print("Processing file: ", filename)
r = pyall.ALLReader(filename)
totalrecords = r.getRecordCount()
start_time = time.time() # time the process
recCount = 0
waterfall = []
minDepth = 9999.0
maxDepth = -minDepth
outputResolution = beamCount * zoom
isoStretchFactor = (yResolution / xResolution) * zoom
print("xRes %.2f yRes %.2f isoStretchFactor %.2f outputResolution %.2f" %
(xResolution, yResolution, isoStretchFactor, outputResolution))
while r.moreData():
TypeOfDatagram, datagram = r.readDatagram()
if (TypeOfDatagram == 0):
continue
if (TypeOfDatagram == 'X') or (TypeOfDatagram == 'D'):
datagram.read()
if datagram.NBeams == 0:
continue
# if datagram.SerialNumber == 275:
for d in range(len(datagram.Depth)):
datagram.Depth[
d] = datagram.Depth[d] + datagram.TransducerDepth
# we need to remember the actual data extents so we can set the color palette mappings to the same limits.
minDepth = min(minDepth, min(datagram.Depth))
maxDepth = max(maxDepth, max(datagram.Depth))
waterfall.insert(0, np.asarray(datagram.Depth))
# we need to stretch the data to make it isometric, so lets use numpy interp routing to do that for Us
# datagram.AcrossTrackDistance.reverse()
xp = np.array(
datagram.AcrossTrackDistance
) #the x distance for the beams of a ping. we could possibly use the real values here instead todo
# datagram.Depth.reverse()
fp = np.array(datagram.Depth) #the depth list as a numpy array
# fp = geodetic.medfilt(fp,31)
x = np.linspace(
leftExtent, rightExtent, outputResolution
) #the required samples needs to be about the same as the original number of samples, spread across the across track range
# newDepths = np.interp(x, xp, fp, left=0.0, right=0.0)
# run a median filter to remove crazy noise
# newDepths = geodetic.medfilt(newDepths,7)
# waterfall.insert(0, np.asarray(newDepths))
recCount += 1
if r.currentRecordDateTime().timestamp() % 30 == 0:
percentageRead = (recCount / totalrecords)
update_progress("Decoding .all file", percentageRead)
update_progress("Decoding .all file", 1)
r.close()
# we have all data loaded, so now lets make a waterfall image...
#---------------------------------------------------------------
print("Correcting for vessel speed...")
# we now need to interpolate in the along track direction so we have apprximate isometry
npGrid = np.array(waterfall)
stretchedGrid = np.empty((0, int(len(npGrid) * isoStretchFactor)))
for column in npGrid.T:
y = np.linspace(0, len(column),
len(column) * isoStretchFactor) #the required samples
yp = np.arange(len(column))
w2 = np.interp(y, yp, column, left=0.0, right=0.0)
# w2 = geodetic.medfilt(w2,7)
stretchedGrid = np.append(stretchedGrid, [w2], axis=0)
npGrid = stretchedGrid
npGrid = np.ma.masked_values(npGrid, 0.0)
if gray:
print("Hillshading...")
#Create hillshade a little brighter and invert so hills look like hills
colorMap = None
npGrid = npGrid.T * shadeScale * -1.0
hs = sr.calcHillshade(npGrid, 1, 45, 30)
img = Image.fromarray(hs).convert('RGBA')
else:
print("Color mapping...")
npGrid = npGrid.T
# calculate color height map
cmrgb = cm.colors.ListedColormap(colors, name='from_list', N=None)
colorMap = cm.ScalarMappable(cmap=cmrgb)
colorMap.set_clim(vmin=minDepth, vmax=maxDepth)
colorArray = colorMap.to_rgba(npGrid, alpha=None, bytes=True)
colorImage = Image.frombuffer(
'RGBA', (colorArray.shape[1], colorArray.shape[0]), colorArray,
'raw', 'RGBA', 0, 1)
#Create hillshade a little darker as we are blending it. we do not need to invert as we are subtracting the shade from the color image
npGrid = npGrid * shadeScale
hs = sr.calcHillshade(npGrid, 1, 45, 5)
img = Image.fromarray(hs).convert('RGBA')
# now blend the two images
img = ImageChops.subtract(colorImage, img).convert('RGB')
if annotate:
#rotate the image if the user requests this. It is a little better for viewing in a browser
annotateWaterfall(img, navigation, isoStretchFactor)
meanDepth = np.average(waterfall)
waterfallPixelSize = (abs(rightExtent) + abs(rightExtent)) / img.width
# print ("Mean Depth %.2f" % meanDepth)
imgLegend = createLegend(filename, img.width,
(abs(leftExtent) + abs(rightExtent)),
distanceTravelled, waterfallPixelSize,
minDepth, maxDepth, meanDepth, colorMap)
img = spliceImages(img, imgLegend)
if rotate:
img = img.rotate(-90, expand=True)
img.save(os.path.splitext(filename)[0] + '.png')
print("Saved to: ", os.path.splitext(filename)[0] + '.png')
def createWaterfall(filename,
colorScale,
beamCount,
zoom=1.0,
clip=0,
invert=True,
annotate=True,
xResolution=1,
yResolution=1,
rotate=False,
leftExtent=-100,
rightExtent=100,
distanceTravelled=0,
navigation=[]):
print("Processing file: ", filename)
r = pyall.ALLReader(filename)
totalrecords = r.getRecordCount()
start_time = time.time() # time the process
recCount = 0
waterfall = []
currentBathyDatagram = None
minBS = 9999.0
maxBS = -minBS
outputResolution = beamCount * zoom
isoStretchFactor = (yResolution / xResolution) * zoom
print("xRes %.2f yRes %.2f isoStretchFactor %.2f" %
(xResolution, yResolution, isoStretchFactor))
while r.moreData():
TypeOfDatagram, datagram = r.readDatagram()
# if (TypeOfDatagram == 0):
# continue
if (TypeOfDatagram == 'X') or (TypeOfDatagram == 'D'):
datagram.read()
currentBathyDatagram = datagram
if (TypeOfDatagram == 'Y'):
datagram.read()
if currentBathyDatagram is None:
continue
if currentBathyDatagram.NBeams == 0:
continue
for i, b in enumerate(datagram.beams):
# currentBathyDatagram.Reflectivity[i] = statistics.mean(b.samples)
currentBathyDatagram.Reflectivity[i] = max(b.samples)
# currentBathyDatagram.Reflectivity[i] = b.samples[b.centreSampleNumber-1]
# we need to remember the actual data extents so we can set the color palette mappings to the same limits.
minBS = min(minBS, min(currentBathyDatagram.Reflectivity))
maxBS = max(maxBS, max(currentBathyDatagram.Reflectivity))
# print ("MinBS %.3f MaxBS %.3f" % (minBS, maxBS))
# waterfall.insert(0, np.abs( np.asarray(datagram.Reflectivity)))
# we need to stretch the data to make it isometric, so lets use numpy interp routing to do that for Us
# datagram.AcrossTrackDistance.reverse()
xp = np.array(
currentBathyDatagram.AcrossTrackDistance
) #the x distance for the beams of a ping. we could possibly use the real values here instead todo
# datagram.Backscatter.reverse()
fp = np.abs(np.array(currentBathyDatagram.Reflectivity)
) #the Backscatter list as a numpy array
# fp = geodetic.medfilt(fp,31)
x = np.linspace(
leftExtent, rightExtent, outputResolution
) #the required samples needs to be about the same as the original number of samples, spread across the across track range
newBackscatters = np.interp(x, xp, fp, left=0.0, right=0.0)
# run a median filter to remove crazy noise
# newBackscatters = geodetic.medfilt(newBackscatters,3)
waterfall.insert(0, np.asarray(newBackscatters))
recCount += 1
if r.currentRecordDateTime().timestamp() % 30 == 0:
percentageRead = (recCount / totalrecords)
update_progress("Decoding .all file", percentageRead)
update_progress("Decoding .all file", 1)
r.close()
# we have all data loaded, so now lets make a waterfall image...
#---------------------------------------------------------------
print("Correcting for vessel speed...")
# we now need to interpolate in the along track direction so we have apprximate isometry
npGrid = np.array(waterfall)
stretchedGrid = np.empty((0, int(len(npGrid) * isoStretchFactor)))
for column in npGrid.T:
y = np.linspace(0, len(column),
len(column) * isoStretchFactor) #the required samples
yp = np.arange(len(column))
w2 = np.interp(y, yp, column, left=0.0, right=0.0)
# w2 = geodetic.medfilt(w2,7)
stretchedGrid = np.append(stretchedGrid, [w2], axis=0)
npGrid = stretchedGrid
# npGrid = np.ma.masked_values(npGrid, 0.0)
if colorScale.lower() == "graylog":
print("Converting to Image with graylog scale...")
img = samplesToGrayImageLogarithmic(npGrid, invert, clip)
elif colorScale.lower() == "gray":
print("Converting to Image with gray scale...")
img = samplesToGrayImage(npGrid, invert, clip)
if annotate:
#rotate the image if the user requests this. It is a little better for viewing in a browser
annotateWaterfall(img, navigation, isoStretchFactor)
meanBackscatter = np.average(waterfall)
waterfallPixelSize = (abs(rightExtent) + abs(rightExtent)) / img.width
# print ("Mean Backscatter %.2f" % meanBackscatter)
imgLegend = createLegend(filename, img.width,
(abs(leftExtent) + abs(rightExtent)),
distanceTravelled, waterfallPixelSize, minBS,
maxBS, meanBackscatter, colorMap)
img = spliceImages(img, imgLegend)
if rotate:
img = img.rotate(-90, expand=True)
img.save(os.path.splitext(filename)[0] + '.png')
print("Saved to: ", os.path.splitext(filename)[0] + '.png')
def main():
parser = argparse.ArgumentParser(
description=
'Read Kongsberg ALL file and create a caris hvf vessel config file.')
parser.add_argument(
'-i',
dest='inputFile',
action='store',
help=
'-i <ALLfilename> : input ALL filename to image. It can also be a wildcard, e.g. *.all'
)
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
# we need to remember the previous record so we only create uniq values, not duplicates
prevNav1Params = {}
prevPitchParams = ""
prevRollParams = ""
prevHeaveParams = ""
prevGyroParams = ""
prevWaterlineParams = ""
prevDepthParams = ""
fileCounter = 0
print("processing with settings: ", args)
root = createHVFRoot(datetime.now())
for filename in glob(args.inputFile):
r = pyall.ALLReader(filename)
start_time = time.time() # time the process
InstallationRecordCount = 0
while r.moreData():
# read a datagram. If we support it, return the datagram type and aclass for that datagram
# The user then needs to call the read() method for the class to undertake a fileread and binary decode. This keeps the read super quick.
TypeOfDatagram, datagram = r.readDatagram()
if TypeOfDatagram == 'I':
datagram.read()
# print (datagram.installationParameters)
root, prevNav1Params = createNavSensor(
root, r.currentRecordDateTime(),
datagram.installationParameters, prevNav1Params)
root, prevGyroParams = createGyroSensor(
root, r.currentRecordDateTime(),
datagram.installationParameters, prevGyroParams)
root, prevHeaveParams = createHeaveSensor(
root, r.currentRecordDateTime(),
datagram.installationParameters, prevHeaveParams)
root, prevPitchParams = createPitchSensor(
root, r.currentRecordDateTime(),
datagram.installationParameters, prevPitchParams)
root, prevRollParams = createRollSensor(
root, r.currentRecordDateTime(),
datagram.installationParameters, prevRollParams)
root, prevWaterlineParams = createWaterlineSensor(
root, r.currentRecordDateTime(),
datagram.installationParameters, prevWaterlineParams)
root, prevDepthParams = createDepthSensor(
root, r.currentRecordDateTime(),
datagram.installationParameters, prevDepthParams,
datagram.EMModel, datagram.SerialNumber)
InstallationRecordCount = InstallationRecordCount + 1
update_progress(
"Processed file: %s InstallationRecords: %d" %
(filename, InstallationRecordCount),
(fileCounter / len(args.inputFile)))
fileCounter += 1
f = open('file.hvf', 'w')
f.write(prettify(root))
f.close()
update_progress(
"Processed all files. InstallationRecords: %d" %
(InstallationRecordCount), 1)
def convert(fileName):
recCount = 0
r = pyall.ALLReader(fileName)
eprint("loading navigation...")
navigation = r.loadNavigation()
# eprint("done.")
arr = np.array(navigation)
times = arr[:, 0]
latitudes = arr[:, 1]
longitudes = arr[:, 2]
start_time = time.time() # time the process
while r.moreData():
TypeOfDatagram, datagram = r.readDatagram()
if (TypeOfDatagram == 'X') or (TypeOfDatagram == 'D'):
datagram.read()
recDate = r.currentRecordDateTime()
if datagram.NBeams > 1:
# interpolate so we know where the ping is located
lat = np.interp(pyall.to_timestamp(recDate),
times,
latitudes,
left=None,
right=None)
lon = np.interp(pyall.to_timestamp(recDate),
times,
longitudes,
left=None,
right=None)
latRad = math.radians(lat)
lonRad = math.radians(lon)
# needed for an optimised algorithm
localradius = calculateradiusFromLatitude(lat)
# for each beam in the ping, compute the real world position
for i in range(len(datagram.Depth)):
#native python version are faster than numpy
# given the Dx,Dy soundings, compute a range, bearing so we can correccttly map out the soundings
brg = 90 - ((180 / math.pi) *
math.atan2(datagram.AlongTrackDistance[i],
datagram.AcrossTrackDistance[i]))
rng = math.sqrt((datagram.AcrossTrackDistance[i]**2) +
(datagram.AlongTrackDistance[i]**2))
# x,y = positionFromRngBrg4(lat, lon, rng, brg + datagram.Heading)
x, y = destinationPoint(lat, lon, rng,
brg + datagram.Heading,
localradius)
# a faster algorithm
# x, y = positionFromRngBrg2(localradius, latRad, lonRad, rng, brg + datagram.Heading)
# based on the transducer position, range and bearing to the sounding, compute the sounding position.
# x,y,h = geodetic.calculateGeographicalPositionFromRangeBearing(lat, lon, brg + datagram.Heading, rng)
# print ("%.10f, %.10f" % (x1 - x, y1 - y))
print("%.10f, %.10f, %.3f" %
(x, y, datagram.Depth[i] + datagram.TransducerDepth))
recCount = recCount + 1
r.close()
eprint("Duration %.3fs" % (time.time() - start_time)) # time the process
return navigation
def process(args):
# if its a file, handle it nicely.
if os.path.isfile(args.inputFolder):
matches = [args.inputFolder]
missionname = os.path.basename(
os.path.dirname(os.path.normpath(args.inputFolder)))
args.opath = os.path.join(
os.path.dirname(os.path.normpath(args.inputFolder)), "GIS")
else:
matches = fileutils.findFiles(True, args.inputFolder, "*.all")
missionname = os.path.basename(
args.inputFolder) #this folder should be the MISSION NAME
args.opath = os.path.join(args.inputFolder, "GIS")
# trackPointFileName = os.path.join(args.opath, args.odir, missionname + "_DGPSHiPAPData_TrackPoint.shp")
# trackPointFileName = fileutils.addFileNameAppendage(trackPointFileName, args.odix)
# trackPointFileName = fileutils.createOutputFileName(trackPointFileName)
# matches = fileutils.findFiles(True, args.inputFolder, "*.all")
if len(args.outputFile) == 0:
fname, ext = os.path.splitext(os.path.expanduser(matches[0]))
args.outputFile = fname
# args.outputFile = "Track"
if len(args.opath) == 0:
args.opath = os.path.dirname(os.path.abspath(args.outputFile))
# trackLineFileName = os.path.join(os.path.dirname(os.path.abspath(args.outputFile)), fname + "_MBESLine.shp")
trackLineFileName = os.path.join(args.opath, args.odir,
missionname + "Survey_TrackLines.shp")
trackLineFileName = addFileNameAppendage(trackLineFileName, args.odix)
trackLineFileName = fileutils.createOutputFileName(trackLineFileName)
# trackPointFileName = os.path.join(os.path.dirname(os.path.abspath(args.outputFile)), fname + "_MBESPoint.shp")
trackPointFileName = os.path.join(args.opath, args.odir,
missionname + "Survey_TrackPoint.shp")
trackPointFileName = addFileNameAppendage(trackPointFileName, args.odix)
trackPointFileName = fileutils.createOutputFileName(trackPointFileName)
# trackCoverageFileName = os.path.join(os.path.dirname(os.path.abspath(args.outputFile)), fname + "_trackCoverage.shp")
trackCoverageFileName = os.path.join(
args.opath, args.odir, missionname + "Survey_TrackCoverage.shp")
trackCoverageFileName = addFileNameAppendage(trackCoverageFileName,
args.odix)
trackCoverageFileName = fileutils.createOutputFileName(
trackCoverageFileName)
#load the python proj projection object library if the user has requested it
geo = geodetic.geodesy(args.epsg)
# if int(args.epsg) == 4326:
# args.epsg = "0"
# if len(args.epsg) > 0:
# projection = geodetic.loadProj(args.epsg)
# else:
# projection = None
# if projection == None:
# args.epsg = 4326
# open the output files once only.
# create the destination shape files
TPshp = None
TLshp = None
TCshp = None
if args.trackall:
args.trackpoint = True
args.trackline = True
args.trackcoverage = True
if args.trackpoint:
TPshp = SSDM.createPointShapeFile(trackPointFileName)
if args.trackline:
TLshp = SSDM.createSurveyTracklineSHP(trackLineFileName)
if args.trackcoverage:
TCshp = SSDM.createCoverageSHP(trackCoverageFileName)
for filename in matches:
reader = pyall.ALLReader(filename)
if args.trackpoint:
print("Processing Track point:", filename)
# TPshp = processTrackPoint(filename, float(args.step),trackPointFileName, geo)
createTrackPoint(reader, TPshp, float(args.step), geo)
if args.trackline:
print("Processing Track line:", filename)
# TLshp = processTrackLine(filename, float(args.step), trackLineFileName, geo)
totalDistanceRun = createTrackLine(reader, TLshp, float(args.step),
geo)
# print ("%s Trackplot Length: %.3f" % (filename, totalDistanceRun))
if args.trackcoverage:
print("Processing Track coverage:", filename)
# TCshp = processTrackCoverage(filename, float(args.step), trackCoverageFileName, geo)
createTrackCoverage(reader, TCshp, float(args.step), geo)
# update_progress("Processed: %s (%d/%d)" % (filename, fileCounter, len(matches)), (fileCounter/len(matches)))
# fileCounter +=1
#now we can write out the results to a shape file...
# update_progress("Process Complete: ", (fileCounter/len(matches)))
if args.trackpoint:
print("Saving track point: %s" % trackPointFileName)
TPshp.save(trackPointFileName)
# now write out a prj file so the data has a spatial Reference
filename = trackPointFileName.replace('.shp', '.prj')
geodetic.writePRJ(filename, args.epsg)
if args.dgn:
dgnwrite.convert2DGN(trackPointFileName)
if args.trackline:
print("Saving track line: %s" % trackLineFileName)
TLshp.save(trackLineFileName)
# now write out a prj file so the data has a spatial Reference
filename = trackLineFileName.replace('.shp', '.prj')
geodetic.writePRJ(filename, args.epsg)
if args.dgn:
dgnwrite.convert2DGN(trackLineFileName)
if args.trackcoverage:
print("Saving coverage polygon: %s" % trackCoverageFileName)
TCshp.save(trackCoverageFileName)
# now write out a prj file so the data has a spatial Reference
filename = trackCoverageFileName.replace('.shp', '.prj')
geodetic.writePRJ(filename, args.epsg)
if args.dgn:
dgnwrite.convert2DGN(trackCoverageFileName)
def __init__(self, file_path):
Scan.__init__(self, file_path)
self.reader = open(self.file_path, 'rb')
self.all_reader = pyall.ALLReader(self.file_path)