def _change_granularity(self, input_df):
"""This function looks at the Date columns of the df and modifies the df according to the granularity (in minutes).
This function expects self.granularity to be an positive int <= 60*24"""
df = input_df.copy()
gran = self.granularity
if gran == 1:
print("Granularity is set to 1 minute.")
return df
new_df = pd.DataFrame(columns=df.columns)
start = df.index[0]
# Get the starting minute as a multiple
m = start.minute
start += timedelta(minutes=(gran - m + 1))
oldest = max(df.index)
#Loop over the entire dataframe. assumption is that candle df is in 1 min intervals
length = df.shape[0]
i = 0
while True:
if i > 100 and i % 20 == 0:
printProgressBar(i,
length,
prefix='Progress:',
suffix='Complete')
end = start + timedelta(minutes=gran - 1)
data = df.loc[(df.index >= start) & (df.index <= end)]
try:
# Note that timestamps are the close time
candle = pd.DataFrame(
{
'BTCOpen': data.iloc[0]['BTCOpen'],
'BTCHigh': max(data['BTCHigh']),
'BTCLow': min(data['BTCLow']),
'BTCClose': data.iloc[-1]['BTCClose'],
'BTCVolume': sum(data['BTCVolume'])
},
index=[end])
new_df = new_df.append(candle)
# Handle empty slices (ignore)
except IndexError:
pass
if end >= oldest: break
start += timedelta(minutes=gran)
# This is for printing the progress bar
try:
i = df.index.get_loc(start)
except KeyError:
pass
# print('')
# print('Dataframe with updated granularity:')
# print(new_df.head())
return new_df
def FastGetComplete(directory2Data,CompleteListOfFiles_vertical):
'''fast way to get teh complete list of files'''
try:
Com = scp.loadmat(directory2Data.dir_verticalwork+'/'+'Com.mat')['Com']
except:
Com = []
for ik in range(len(CompleteListOfFiles_vertical)):
tools.printProgressBar(ik,len(CompleteListOfFiles_vertical),prefix = 'Com: ', suffix = 'Completed', length = 50)
Com = np.hstack((Com,int(CompleteListOfFiles_vertical['ping_time'][ik])))
scp.savemat(directory2Data.dir_verticalwork+'/'+'Com.mat',mdict = {'Com':Com})
Com = scp.loadmat(directory2Data.dir_verticalwork+'/'+'Com.mat')['Com']
return Com
def CopyFileFromTapeServer(OS, liste, directory2Data):
#Get list of files that has not been copied to the correct structure
print(' -Copy files from tape-server ', end='\r')
#Check if file exist, this is a bugfix when working localy
if os.path.isdir(OS + liste['SonarDataPath']) == True:
print(' -Search files on tape-server ', end='\r')
#Get list of files on server and list that has not been copied
ListFromServer = os.listdir(OS + liste['SonarDataPath'])
print(' -Get files not copied ', end='\r')
ListOfFilesNotcopied = list(
set(ListFromServer) -
set(os.listdir(directory2Data.dir_originalrawdata)))
print(' -Start copy files from tape-server ', end='\r')
#Go through each file that has not been copyed
for i in np.arange(len(ListOfFilesNotcopied)):
#Print a progressbar for the user
tools.printProgressBar(i,
len(ListOfFilesNotcopied),
prefix='Copy files',
suffix='Files left: ' +
str(len(ListOfFilesNotcopied) - i),
decimals=1,
length=50)
if '.raw' in ListOfFilesNotcopied[i]:
#Copy the files
try:
copyfile(
OS + liste['SonarDataPath'] + '/' +
ListOfFilesNotcopied[i],
directory2Data.dir_originalrawdata + '/' +
ListOfFilesNotcopied[i])
except:
print(' * Bad file', end='\r')
else:
dummy = 1
def GetShortListOfFiles(CompleteListOfFiles,startTime,endTime):
ShortListOfFiles = []
for i in range(len(CompleteListOfFiles[:][:])):
tools.printProgressBar(i + 1, len(CompleteListOfFiles[:,0]), prefix = 'Make short list:', suffix = 'Completed ', length = 50)
# print(CompleteListOfFiles[:][i])
start = SecondsBetweenTransect(startTime,int(CompleteListOfFiles[:][i][0]))
end = SecondsBetweenTransect(endTime,int(CompleteListOfFiles[:][i][0]))
if start<0:
if end >0:
if ShortListOfFiles == []:
ShortListOfFiles = CompleteListOfFiles[i,:]
else:
ShortListOfFiles = np.vstack((ShortListOfFiles,CompleteListOfFiles[i,:]))
return ShortListOfFiles
def MakeWork(makeNewWork,data_directory,SearchMatrixName,
prefix,compensate,clusterTH,Threshold):
try:
import matplotlib.pyplot as plt
except:
dummy = 1
#Sjekk om dett er cluster TH eller Threshold
CatThr = 3
#Get all mat files
os.chdir(data_directory.dir_search)
FileNames = glob('*.mat')
random.shuffle(FileNames)
#Loop through each search matrix (transect)
for SearchMatrixName in FileNames:
ticken = 0
if not os.path.isfile(data_directory.dir_work+'/Horizontal_'+str(ticken)+'_'+SearchMatrixName) == True:
#Load search matrix
SearchMatrix = sc.loadmat(data_directory.dir_search+'/'+SearchMatrixName)
#Get the data from the files
SVres_port = SearchMatrix['SVres_port']
SVres_stb = SearchMatrix['SVres_stb']
R_s = SearchMatrix['R_s']
res = SearchMatrix['res']
# SVres_portGhost = SearchMatrix['SVres_portGhost']
# SVres_stbGhost = SearchMatrix['SVres_stbGhost']
DistanceTraveled = SearchMatrix['DistanceTraveled']
ListOfFilesWithinTimeInterval = SearchMatrix['ListOfFilesWithinTimeInterval']
try:
plt.figure(1)
plt.clf()
plt.subplot(2,1,1)
plt.imshow(10*np.log10(SVres_port),aspect = 'auto')
plt.subplot(2,1,2)
plt.imshow(10*np.log10(SVres_stb),aspect = 'auto')
plt.savefig(data_directory.dir_search+'/'+SearchMatrixName.replace('.mat','search.jpg'))
except:
dummy = 1
# Finding the location of the schools in the search matrix
# ADD may be replaced with an other function
Cat_port, X_port = GetSpikesFromEchogram(SVres_port,CatThr,
np.hstack((np.reshape(SVres_port,(-1,)),
np.reshape(SVres_stb,(-1,)))))
Cat_stb, X_stb = GetSpikesFromEchogram(SVres_stb,CatThr,
np.hstack((np.reshape(SVres_port,(-1,)),
np.reshape(SVres_stb,(-1,)))))
#Get the location of each school in xy coordinates
Bananatool = [R_s,0,0,res]
x__school_traj_port,y__school_traj_port = FindLocationOfSchool(Cat_port, Bananatool, DistanceTraveled)
x__school_traj_stb,y__school_traj_stb = FindLocationOfSchool(Cat_stb, Bananatool, DistanceTraveled)
y__school_traj_stb = -y__school_traj_stb
try:
plt.figure(1)
plt.clf()
plt.plot(x__school_traj_port,y__school_traj_port,'k.')
plt.plot(x__school_traj_stb,y__school_traj_stb,'k.')
plt.savefig(data_directory.dir_search+'/'+SearchMatrixName.replace('.mat','filt.jpg'))
except:
dummy = 1
#Start writing work files.
#Go through each ping
WorkPhi = np.array([])
WorkBeam = np.array([])
WorkFile = np.array([])
WorkTime = np.array([])
WorkIDX = np.array([])
for ii in range(len(DistanceTraveled.T)):
#Progress for user
tools.printProgressBar(ii + 1, len(ListOfFilesWithinTimeInterval[:,0]), prefix = 'Scanning files:', suffix = len(DistanceTraveled.T)-ii, length = 50)
#If first ping
if ii == 0:
#Open the file and get variables
fileID = Dataset(data_directory.dir_rawdata+'/'+ ListOfFilesWithinTimeInterval[ii][1],'r',format = 'NETCDF4')
variables = tools.GetVariablesFromNC(fileID,'Beam_group1',ListOfFilesWithinTimeInterval,ii)
fileID.close()
#Compute the range
rangeCorr = 3;
samplespace =variables.soundvelocity*variables.sampleinterval/2
Range = np.arange(0,len(variables.BeamAmplitudeData))*samplespace-rangeCorr
#Compute the pixel location with direction and range
X =np.reshape(np.dot(Range[:,np.newaxis], np.cos(np.deg2rad(variables.diry))[:,np.newaxis].T),(-1))
Y =np.reshape(np.dot(Range[:,np.newaxis], np.sin(np.deg2rad(variables.diry))[:,np.newaxis].T),(-1))
Phi = np.reshape(np.dot(np.ones(Range[:,np.newaxis].shape),(variables.diry)[:,np.newaxis].T),(-1))
Range = np.reshape(np.dot(Range[:,np.newaxis],np.ones((variables.diry)[:,np.newaxis].T.shape)),(-1))
idx = np.where(abs(x__school_traj_port-DistanceTraveled[0,ii]) <=700)[0]
for iik in range(len(idx)):
r_port = np.sqrt((x__school_traj_port[idx[iik]]-(X+DistanceTraveled[0,ii]))**2+(y__school_traj_port[idx[iik]]-Y)**2)
WorkPhi = np.hstack((WorkPhi,Phi[np.where(r_port<=R_s[0])[0]]))
WorkBeam = np.hstack((WorkBeam,Range[np.where(r_port<=R_s[0])[0]]))
WorkTime = np.hstack((WorkTime,np.repeat(int(ListOfFilesWithinTimeInterval[ii][0]),len(Phi[np.where(r_port<=R_s[0])[0]]))))
WorkFile = np.hstack((WorkFile,np.repeat(ListOfFilesWithinTimeInterval[ii][1],len(Phi[np.where(r_port<=R_s[0])[0]]))))
WorkIDX = np.hstack((WorkIDX,np.repeat(int(ListOfFilesWithinTimeInterval[ii][2]),len(Phi[np.where(r_port<=R_s[0])[0]]))))
# plt.draw()
# plt.pause(0.001)
#
# for iip in np.where(r_port<=R_s[0])[0]:
## if np.min(r_port)<=R_s[0]:
# if len(WorkFileOutput) == 0:
# WorkFileOutput=np.hstack((int(ListOfFilesWithinTimeInterval[ii][0]),
# ListOfFilesWithinTimeInterval[ii][1],
# int(ListOfFilesWithinTimeInterval[ii][2]),
# Phi[iip],Range[iip]))
#
#
# else:
# WorkFileOutput = np.vstack((WorkFileOutput
# ,np.hstack((int(ListOfFilesWithinTimeInterval[ii][0]),
# ListOfFilesWithinTimeInterval[ii][1],
# int(ListOfFilesWithinTimeInterval[ii][2])
# ,Phi[iip],Range[iip]))))
idx = np.where(abs(x__school_traj_stb-DistanceTraveled[0,ii]) <=700)[0]
for iik in range(len(idx)):
r_stb = np.sqrt((x__school_traj_stb[idx[iik]]-(X+DistanceTraveled[0,ii]))**2+(y__school_traj_stb[idx[iik]]-Y)**2)
WorkPhi = np.hstack((WorkPhi,Phi[np.where(r_stb<=R_s[0])[0]]))
WorkBeam = np.hstack((WorkBeam,Range[np.where(r_stb<=R_s[0])[0]]))
WorkPhi = np.hstack((WorkPhi,Phi[np.where(r_stb<=R_s[0])[0]]))
WorkBeam = np.hstack((WorkBeam,Range[np.where(r_stb<=R_s[0])[0]]))
WorkTime = np.hstack((WorkTime,np.repeat(int(ListOfFilesWithinTimeInterval[ii][0]),len(Phi[np.where(r_stb<=R_s[0])[0]]))))
WorkFile = np.hstack((WorkFile,np.repeat(ListOfFilesWithinTimeInterval[ii][1],len(Phi[np.where(r_stb<=R_s[0])[0]]))))
WorkIDX = np.hstack((WorkIDX,np.repeat(int(ListOfFilesWithinTimeInterval[ii][2]),len(Phi[np.where(r_stb<=R_s[0])[0]]))))
# print(WorkFileOutput.shape[0])
#Make a new file so it does not become too large
if len(WorkPhi)>2000000:
sc.savemat(data_directory.dir_work+'/Horizontal_'+str(ticken)+'_'+SearchMatrixName,mdict={'WorkPhi':WorkPhi,'WorkBeam':WorkBeam,'WorkFile':WorkFile,'WorkIDX':WorkIDX,'WorkTime':WorkTime})
ticken = ticken+1
WorkPhi = np.array([])
WorkBeam = np.array([])
WorkFile = np.array([])
WorkTime = np.array([])
WorkIDX = np.array([])
sc.savemat(data_directory.dir_work+'/Horizontal_'+str(ticken)+'_'+SearchMatrixName,mdict={'WorkPhi':WorkPhi,'WorkBeam':WorkBeam,'WorkFile':WorkFile,'WorkIDX':WorkIDX,'WorkTime':WorkTime})
# #Skip those pings that don't have relevant data
# if np.nansum([(x__school_traj_port<=np.nanmax(X)+DistanceTraveled[0,ii])&(x__school_traj_port>=np.nanmin(X)+DistanceTraveled[0,ii])]) >0:
#
#
#
#
# #go through each pixel and find if it is relevant data
# for iik in range(len(X)):
#
# print(len(X)-iik,end = '\r')
#
#
# #Find the distance between pixels and the search matrix output
# r_stb = np.sqrt((x__school_traj_stb-(X[iik]+DistanceTraveled[0,ii]))**2+(y__school_traj_stb-Y[iik])**2)
# r_port = np.sqrt((x__school_traj_port-(X[iik]+DistanceTraveled[0,ii]))**2+(y__school_traj_port-Y[iik])**2)
#
#
#
# if np.min(r_stb)<=R_s[0]:
# if len(WorkFileOutput) == 0:
# WorkFileOutput=np.hstack((int(ListOfFilesWithinTimeInterval[ii][0]),
# ListOfFilesWithinTimeInterval[ii][1],
# int(ListOfFilesWithinTimeInterval[ii][2]),
# Phi[iik],Range[iik]))
#
#
# else:
# WorkFileOutput = np.vstack((WorkFileOutput
# ,np.hstack((int(ListOfFilesWithinTimeInterval[ii][0]),
# ListOfFilesWithinTimeInterval[ii][1],
# int(ListOfFilesWithinTimeInterval[ii][2])
# ,Phi[iik],Range[iik]))))
#
#
# if np.min(r_port)<=R_s[0]:
# if len(WorkFileOutput) == 0:
# WorkFileOutput=np.hstack((int(ListOfFilesWithinTimeInterval[ii][0]),
# ListOfFilesWithinTimeInterval[ii][1],
# int(ListOfFilesWithinTimeInterval[ii][2]),
# Phi[iik],Range[iik]))
#
#
# else:
# WorkFileOutput = np.vstack((WorkFileOutput
# ,np.hstack((int(ListOfFilesWithinTimeInterval[ii][0]),
# ListOfFilesWithinTimeInterval[ii][1],
# int(ListOfFilesWithinTimeInterval[ii][2])
# ,Phi[iik],Range[iik]))))
#
#
#
try:
import matplotlib.pyplot as plt
plt.figure(1)
plt.clf()
plt.imshow((Cat_port),aspect = 'auto')
plt.colorbar()
plt.draw()
plt.savefig(data_directory.dir_search+'/'+SearchMatrixName.replace('.mat','2.jpg'))
plt.figure(2)
plt.clf()
plt.plot(x__school_traj_port,y__school_traj_port,'r.')
plt.plot(x__school_traj_stb,-y__school_traj_stb,'b.')
plt.draw()
plt.savefig(data_directory.dir_search+'/'+SearchMatrixName.replace('.mat','3.jpg'))
except:
dummy=1
def makeIDX(directory2Data, beam_mode):
import numpy as np
import os
from tools import tools
from netCDF4 import Dataset
#Sort all the .nc files
print('Get list of files')
ListOfFiles = np.sort(os.listdir(directory2Data.dir_rawdata))
print('Got list of files')
tot_index = 0
f = Dataset(directory2Data.dir_src + '/IDX_' + beam_mode + '.nc',
'w',
format='NETCDF4')
tempgrp = f.createGroup('IDXinfo')
tempgrp.createDimension('dim', None)
nchar_t = tempgrp.createVLType(np.str, 'nchar')
ping = tempgrp.createVariable('ping_time',
np.int64, ('dim', ),
chunksizes=(1024, ))
FileL = tempgrp.createVariable('FileList',
nchar_t, ('dim', ),
chunksizes=(512, ))
beamidx = tempgrp.createVariable('BeamIDX',
nchar_t, ('dim'),
chunksizes=(512, ))
idx = tempgrp.createVariable('IDX', int, 'dim', chunksizes=(512, ))
#This helps to prevent opening and closing the same file multiple times
oldFileName = ''
#Loop through each file
for i in range(len(ListOfFiles)):
#Print progress
tools.printProgressBar(i + 1,
len(ListOfFiles),
prefix='Stacking: ',
suffix='Completed',
length=50)
#If this is a new file, close the last and open this
if ListOfFiles[i] != oldFileName:
try:
fid_nc.close()
except:
dummy = 1
fid_nc = Dataset(
os.path.join(directory2Data.dir_rawdata, ListOfFiles[i]), 'r')
oldFileName = ListOfFiles[i]
#Get the correct beam group
if fid_nc.groups['Sonar'].groups['Beam_group1'].beam_mode == beam_mode:
beamgrp = 'Beam_group1'
elif fid_nc.groups['Sonar'].groups[
'Beam_group2'].beam_mode == beam_mode:
beamgrp = 'Beam_group2'
#Get the beam data
beam_data = fid_nc.groups['Sonar'].groups[beamgrp]
#Loop through each ping
for ii in range(len(beam_data.variables['ping_time'][:])):
FileL[tot_index] = str(ListOfFiles[i]) #FileList
idx[tot_index] = ii
beamidx[tot_index] = str(beamgrp)
ping_out = int(beam_data.variables['ping_time'][ii])
ping[tot_index] = ping_out #ping_time
tot_index = tot_index + 1
f.close()
def MakeVerticalIndex(ListOfFilesWithinTimeInterval, RemoveToCloseValues, R_s,
res, directory2Data, dirnc, beamgrp, start_time,
log_start, stop_time, lat_start, lat_stop, lon_start,
lon_stop):
#For bookkeeping
transectID = log_start
DataMatrix = []
time0 = 0
ping_counter = 0
r_mask = 0
b_mask = 0
ping_mask = 0
#Loop through all files within the time interval
for filename_index in range(0, len(ListOfFilesWithinTimeInterval[:, 0])):
#Print the progression
tools.printProgressBar(filename_index + 1,
len(ListOfFilesWithinTimeInterval[:, 0]),
prefix='Make Vertical:',
suffix='Completed',
length=50)
#Get the full path name
filename = os.path.join(
dirnc, ListOfFilesWithinTimeInterval[filename_index, 1])
#Load the nc file
fileID = Dataset(filename, 'r', format='NETCDF4')
#Get the group with the vertical fan data
if fileID.groups['Sonar'].groups[
'Beam_group1'].beam_mode == 'Vertical':
beamgrp = 'Beam_group1'
elif fileID.groups['Sonar'].groups[
'Beam_group2'].beam_mode == 'Vertical':
beamgrp = 'Beam_group2'
#Get the vertical beam data
#FIKS slik at den henter vertikal data
variables = tools.GetVariablesFromNC(fileID, beamgrp,
ListOfFilesWithinTimeInterval,
filename_index)
fileID.close()
try:
#The sonar data often includes corrputed value of
#the transmit power that destroys the analysis.
#This will fix this problum, but the sv values are not
#correct.
#ADD this data should be labeled when making the work files
#so the user can now that it is corrupted.
if variables.transmitpower == 0:
variables.transmitpower = 4633
#Compute the sv and TS
#ADD TS are not used here !!!
sv, RangeOut = tools.ApplyTVG(
10 * np.log10(variables.BeamAmplitudeData),
variables.soundvelocity[0], variables.sampleinterval,
variables.transmitpower, variables.absorptioncoefficient[0],
variables.frequency, variables.pulslength, variables.gaintx,
variables.equivalentbeamangle, variables.sacorrection,
variables.dirx)
#Remove data too close to the vessel
sv[np.where(RangeOut <= RemoveToCloseValues)] = np.nan
sv[:, np.where(variables.dirx >= 60)] = np.nan
# sv[np.where(sv<-65)] = np.nan
#Stack the vertical beam data
if DataMatrix == []:
sv_x, sv_y = sv.shape
DataMatrix = 10**(sv / 10)[:, :, np.newaxis]
time0 = variables.time
ping_counter = ping_counter + 1
else:
if sv_x > sv.shape[0]:
sv = np.append(sv,
np.nan * np.ones(
(sv_x - sv.shape[0], sv_y)),
axis=0)
elif sv_x < sv.shape[0]:
DataMatrix = np.append(
DataMatrix,
np.nan * np.ones(
(sv.shape[0] - sv_x, sv_y, DataMatrix.shape[2])),
axis=0)
sv_x, sv_y = sv.shape
DataMatrix = np.dstack((DataMatrix, 10**(sv / 10)[:, :,
np.newaxis]))
time0 = np.hstack((time0, variables.time))
ping_counter = ping_counter + 1
except AttributeError:
print('* bad file')
#Find the median
Medianen = np.nanmedian(DataMatrix, axis=2)
#Get index of everything above the median
[r_mask0, b_mask0, ping_mask0] = np.where(
DataMatrix >= 4 *
np.repeat(Medianen[:, :, np.newaxis], len(DataMatrix[0,
0, :]), axis=2))
#
if ping_mask0 != []:
ping_mask0 = time0[ping_mask0]
#Stack the data
r_mask = np.hstack((r_mask, r_mask0))
b_mask = np.hstack((b_mask, b_mask0))
ping_mask = np.hstack((ping_mask, ping_mask0))
DataMatrix = []
ping_counter = 0
#Try saving the data
try:
scp.savemat(directory2Data.dir_work + '/' + 'Vertical_T' +
str(transectID) + '.mat',
mdict={
'r_mask': r_mask,
'b_mask': b_mask,
'ping_mask': ping_mask,
'start_time': start_time,
'log_start': log_start,
'stop_time': stop_time,
'lat_start': lat_start,
'lat_stop': lat_stop,
'lon_start': lon_start,
'lon_stop': lon_stop
})
except TypeError:
print(start_time, log_start, stop_time, lat_start, lat_stop, lon_start,
lon_stop)
def MakeVerticalLuf20(CompleteListOfFiles_vertical, directory2Data, nation,
cruice_id, platform):
outloc = '/'
#Some user input data
# integration_dist = 1
pel_ch_thickness = 10
NumChannels = 35
ChannelSize = 350
#Something for bookkeeping
NASC_out = []
Depth_bottom = 0
#Convert stuff to integer,
#This is to speed up the program
Com = FastGetComplete(directory2Data, CompleteListOfFiles_vertical)
#Something about the channels
#This should be made more general
Depth = np.linspace(0, ChannelSize,
NumChannels) + ChannelSize / NumChannels / 2
Delta_Z = ChannelSize / NumChannels / 2
NASC = np.nan * np.linspace(0, ChannelSize, NumChannels).T[:, np.newaxis]
Work_list = os.listdir(directory2Data.dir_work)
Work_I = np.arange(len(Work_list))
random.shuffle(Work_I)
#Loop through each work file
for work_i in Work_I:
work = Work_list[work_i]
NASC_out = []
print_sa = True
if work != 'Com.mat':
if not os.path.isfile(directory2Data.dir_result + outloc +
'/Vertical/' + work.replace('mat', 'xml')):
#Load mask of the filtered data
try:
workfile = scp.loadmat(directory2Data.dir_work + '/' +
work)
#Start information in the text
root = ET.Element("echosounder_dataset")
ET.SubElement(root, 'report_time').text = str(
datetime.datetime.fromtimestamp(
(time.time())).strftime('%Y-%m-%d %H:%M:%S'))
ET.SubElement(
root,
'lsss_version').text = "PNMDformats v 0.1 - vertical"
ET.SubElement(root, 'nation').text = nation
ET.SubElement(root, 'platform').text = platform
ET.SubElement(root, 'cruise').text = cruice_id
#Make the distance list hirarchy
distance_list = ET.SubElement(root, 'distance_list')
try:
pings = np.unique(workfile['ping_mask'])
except:
pings = []
if not pings == []:
try:
tiime = workfile['start_time'][0]
stime = tiime[:4] + '-' + tiime[4:6] + '-' + tiime[
6:8] + ' ' + tiime[9:11] + ':' + tiime[
11:13] + ':' + tiime[13:]
except:
stime = ''
try:
tiime = workfile['stop_time'][0]
etime = tiime[:4] + '-' + tiime[4:6] + '-' + tiime[
6:8] + ' ' + tiime[9:11] + ':' + tiime[
11:13] + ':' + tiime[13:]
except:
etime = ''
try:
latstart = workfile['lat_start'][0]
except:
latstart = ''
try:
latstop = workfile['lat_stop'][0]
except:
latstop = ''
try:
lonstart = workfile['lon_start'][0]
except:
lonstart = ''
try:
lonstop = workfile['lon_stop'][0]
except:
lonstop = ''
try:
logstart = workfile['log_start'][0]
except:
logstart = ''
distance = tools.addLogDistanceInfo(
distance_list, latstart, lonstart, stime, latstop,
lonstop, etime, 1, 10, logstart)
#Loop through each unique ping
for ping_idx in range(len(pings[1:])):
tools.printProgressBar(ping_idx,
len(pings),
prefix='LUF20 of ' + work +
':',
suffix='Completed',
length=50)
#Idx stuff
idx = np.where(
workfile['ping_mask'] == pings[ping_idx + 1])
idx_file = np.where(Com == (pings[ping_idx + 1]))
#Get the file name of the idx file
filname = directory2Data.dir_rawdata + '/' + CompleteListOfFiles_vertical[
idx_file[1][0], 1]
#Load file
fileID = Dataset(filname, 'r', format='NETCDF4')
#Get the group with the vertical fan data
if fileID.groups['Sonar'].groups[
'Beam_group1'].beam_mode == 'Vertical':
beamgrp = 'Beam_group1'
elif fileID.groups['Sonar'].groups[
'Beam_group2'].beam_mode == 'Vertical':
beamgrp = 'Beam_group2'
#Get the complete list of files
Comp = CompleteListOfFiles_vertical[idx_file[1][0],
2]
#Get variables from .nc file
variables = tools.GetVariablesFromNC(
fileID, beamgrp, False, Comp)
#Close file
fileID.close()
if print_sa == True:
sa_by_acocat = tools.addFrequencyLevelInfo(
distance, variables.frequency, 0,
NumChannels, 0)
print_sa = False
if Depth_bottom == 0:
delta_lat = 600 / 111111
try:
delta_lon = 600 / (111111 * np.cos(
np.deg2rad(
float(workfile['lat_start'][0]))))
except:
delta_lon = 600 / 111111
# try:
Depth_bottom = GetBottomDepth(
float(workfile['lat_start'][0]),
float(workfile['lat_stop'][0]),
float(workfile['lon_start'][0]),
float(workfile['lon_stop'][0]), delta_lat,
delta_lon)
# except:
# Depth_bottom = 1000
# print('bad bottom ')
#Get file name
try:
sv, RangeOut = tools.ApplyTVG(
10 * np.log10(variables.BeamAmplitudeData),
variables.soundvelocity[0],
variables.sampleinterval,
variables.transmitpower,
variables.absorptioncoefficient[0],
variables.frequency, variables.pulslength,
variables.gaintx,
variables.equivalentbeamangle,
variables.sacorrection, variables.dirx)
# sv[np.where(RangeOut<=RemoveToCloseValues)] = np.nan
sv[np.where(RangeOut <= 30)] = np.nan
sv[:, np.where(variables.dirx >= 60)] = np.nan
sv[np.where(sv < -70)] = np.nan
#Go linear
# sv = 10**(sv/10)
#Make a temporarly amtrix
sv2 = sv #*np.nan
sv2 = sv * np.nan
sv2[workfile['r_mask'][idx],
workfile['b_mask'][idx]] = sv[
workfile['r_mask'][idx],
workfile['b_mask'][idx]]
# sv2[np.where(sv2<-60)] = np.nan
sv2 = 10**(sv2 / 10)
sv2[np.isnan(sv2)] = 0
#Remove surface beam
# sv2[:,np.where(variables.dirx < 3)] = np.nan
# sv[:,np.where(variables.dirx < 3)] = np.nan
#Select only at a specific distance from vessel
X = np.cos(np.deg2rad(
variables.dirx)) * np.sin(
np.deg2rad(variables.diry))
X = (X[:, np.newaxis] *
RangeOut[:, np.newaxis].T).T
sv2[np.where(abs(X) > 300)] = np.nan
sv2[np.where(abs(X) < 250)] = np.nan
# sv[np.where(abs(X)>300)] = np.nan
# sv[np.where(abs(X)<250)] = np.nan
#Get the depth of each pixel
Y = np.sin(np.deg2rad(
variables.dirx)) * np.sin(
np.deg2rad(variables.diry))
Y = (abs(Y[:, np.newaxis] *
RangeOut[:, np.newaxis].T)).T
#Integrate in depth channels
for i in range(len(Depth)):
temp = sv2[np.where(
abs(
abs(Y) -
i * ChannelSize / NumChannels +
Delta_Z) <= Delta_Z)]
temp2 = np.count_nonzero(~np.isnan(temp))
NASC[i, -1] = np.nansum(temp) / temp2
except IndexError:
print('bad ping')
A = np.nansum(NASC, axis=1)[:, np.newaxis] / len(
NASC[0, :])
A = A * (1852**2) * pel_ch_thickness * 4 * np.pi
# print(A)
if NASC_out == []:
NASC_out = A
else:
NASC_out = np.hstack((NASC_out, A))
for ikk in range(len(A)):
if Depth[ikk] > (abs(Depth_bottom) - 150):
A[ikk] = 0
NASC_out[ikk, -1] = 0
if A[ikk] > 0:
sa_value = ET.SubElement(sa_by_acocat, 'sa')
sa_value.set('ch', str(ikk + 1))
sa_value.text = str(A[ikk][0])
except:
print('bad file')
tools.indent(root)
tree = ET.ElementTree(root)
tree.write(directory2Data.dir_result + outloc + '/Vertical/' +
work.replace('mat', 'xml'),
xml_declaration=True,
encoding='utf-8',
method="xml")
def MakeSearch(ListOfFilesWithinTimeInterval, RemoveToCloseValues, R_s, res,
directory2Data, dirnc, beamgrp):
#Something for bookkeeping and progress
#This should be cleaned
PingCount = True
MakeWdistStuff = True
MaximumDetectionRange = 10000
NominalCalibraitonGain = []
lat = np.array([])
lon = np.array([])
travelDist = np.array([])
TimeStamp = np.array([])
svMatrix = np.array([])
DistanceMatrix = np.array([])
BeamDirectionMatrix = np.array([])
RangeMatrix = np.array([])
TiltVec = np.array([])
#Loop through all files within the time interval
for filename_index in range(0, len(ListOfFilesWithinTimeInterval[:, 0])):
#Print the progression
tools.printProgressBar(filename_index + 1,
len(ListOfFilesWithinTimeInterval[:, 0]),
prefix='Make SearchMatrix:',
suffix='Completed ',
length=50)
#Get the full path name
filename = os.path.join(
dirnc, ListOfFilesWithinTimeInterval[filename_index, 1])
#Load the nc file
fileID = Dataset(filename, 'r', format='NETCDF4')
#Get data from the nc file
variables = tools.GetVariablesFromNC(fileID, beamgrp,
ListOfFilesWithinTimeInterval,
filename_index)
#Stack the nmea position data
NMEA_idx = np.where(
abs(variables.NMEA_time[:] -
float(ListOfFilesWithinTimeInterval[filename_index, 0])) ==
np.min(
abs(variables.NMEA_time[:] -
float(ListOfFilesWithinTimeInterval[filename_index, 0]))))
lat = np.hstack((lat, variables.Latitude[NMEA_idx]))
lon = np.hstack((lon, variables.Longitude[NMEA_idx]))
#close the nc file
fileID.close()
#Find the distance the vessel has traveled
#ADD check if the function can be simplified, or if
#a similar function is avaliable to be downloaded
DistanceTraveled, travelDist = tools.ComputeDistance(
travelDist, lat, lon)
#Get the calibration gain and add it to the data
#ADD calibration cain is not jet avaliable.
#the function may be chainged once tis is ready#
if not NominalCalibraitonGain:
gain, FrequencyGain, PulslengthGain = tools.GainAdjustment(
variables.pulslength * 1E3, variables.frequency / 1E3,
variables.gaintx + variables.gainrx)
gain = variables.gaintx + PulslengthGain
else:
gain, FrequencyGain, PulslengthGain = tools.GainAdjustment(
variables.pulslength * 1E3, variables.frequency / 1E3,
NominalCalibraitonGain)
gain = variables.gaintx + PulslengthGain
#The sonar data often includes corrputed value of
#the transmit power that destroys the analysis.
#This will fix this problum, but the sv values are not
#correct.
#ADD this data should be labeled when making the work files
#so the user can now that it is corrupted.
if variables.transmitpower == 0:
variables.transmitpower = 4633
#Compute the sv and TS
#ADD TS are not used here !!!
sv, RangeOut = tools.ApplyTVG(
10 * np.log10(variables.BeamAmplitudeData),
variables.soundvelocity, variables.sampleinterval,
variables.transmitpower, variables.absorptioncoefficient,
variables.frequency, variables.pulslength, gain,
variables.equivalentbeamangle, variables.sacorrection,
variables.dirx)
#Remove data too close to the vessel
sv[np.where(RangeOut <= RemoveToCloseValues)] = np.nan
#ADD Temporary filter that will be deleted once checked !!!
if np.nanmean(sv) > 0:
sv = np.nan * np.ones(sv.shape)
#Protocoll to identify if the batch of data is
#finished loaded
if PingCount == True:
NumberOfPingsInBatch = 5000
if DistanceTraveled.max() >= MaximumDetectionRange:
NumberOfPingsInBatch = len(DistanceTraveled)
PingCount = False
#Get the timestamp of the ping
#Needed when generating work file
TimeStamp = np.hstack((TimeStamp, filename))
TiltVec = np.hstack((TiltVec, variables.dirx[0]))
if variables.dirx[0] < np.nanmedian(TiltVec):
print('Tilt ble endret', end='\r')
sv = np.nan * np.ones(sv.shape)
#Remove vessel wake
sv[:, np.where(abs(variables.diry) > 165)] = np.nan
#Implement the inteferance removal filter
#ADD it is still under development and must be
#properly tested !!!
#Uncommet to remove spikes from the data
#ADD a more proper spike detection and removal must be made in the future !!!
# if SpikeIDX.shape[0] == 0:
# SpikeIDX = np.nanmean(sv,axis=1)[:,np.newaxis]
# elif SpikeIDX.shape[1] <=5:
# SpikeIDX = np.hstack((SpikeIDX,np.nanmean(sv,axis=1)[:,np.newaxis]))
#
# else:
# SpikeIDX = np.hstack((SpikeIDX[:,1:],np.nanmean(sv,axis=1)[:,np.newaxis]))
#
# spike = np.where(( np.nanmean(sv,axis=1))>( np.nanmean(SpikeIDX,axis=1)+6))
# sv[spike,:] = np.nan
#
#If the first file of the transect
if len(svMatrix) == 0:
BananaTool = [R_s, 0, len(RangeOut), res]
MaximumDetectionRange = 2 * np.nanmax(RangeOut)
#Start making the buffer matrix
svMatrix = sv
DistanceMatrix = np.ones((len(sv), 64)) * DistanceTraveled[-1]
BeamDirectionMatrix = np.repeat(variables.diry[:, np.newaxis].T,
len(sv),
axis=0)
RangeMatrix = np.repeat(RangeOut[:, np.newaxis], 64, axis=1)
#If the buffer size is large enough
elif filename_index >= NumberOfPingsInBatch:
#if first ping after buffersize is large enough.
#ADD make the if sentence more simpler by
#Only have the makeWdiststuff !!!
# if ((filename_index == NumberOfPingsInBatch) or (filename_index == (NumberOfPingsInBatch+1)))and MakeWdistStuff == True:
if MakeWdistStuff == True:
#Start making the distance matrix for
#the transect
#ADD needs a way to correct this
#if the vessel speed or direction has
#changed !!!
print('Generating Distance Matrix: ', end='\r')
Wdist_port, Wdist_stb = tools.GetDistanceMatrix(
DistanceMatrix, RangeMatrix, BeamDirectionMatrix, svMatrix,
int(variables.dirx[0] + 90), BananaTool)
#Start making the ghost school matrix
print('Generating Distance Matrix for ghost: ', end='\r')
Wdist_portGhost, Wdist_stbGhost = tools.GetDistanceMatrix(
DistanceMatrix, RangeMatrix, BeamDirectionMatrix + 90,
svMatrix, int(variables.dirx[0] + 90), BananaTool)
#Correct the size of the sv and range
#ADD check if it is necesarry or if it is
#already fixed above !!!
AddNaN2Matrix = len(svMatrix[:, 0]) - len(RangeOut)
# sv = np.vstack((sv,np.nan*np.ones((AddNaN2Matrix,64))))
if AddNaN2Matrix > 0:
sv = np.vstack((sv, np.nan * np.ones((AddNaN2Matrix, 64))))
RangeOut = np.vstack((RangeOut[:,
np.newaxis], np.nan * np.ones(
(AddNaN2Matrix, 1))))
else:
RangeOut = RangeOut[:, np.newaxis]
sv = sv[:len(svMatrix[:, 0])]
#Add new data and delete the redundent ping
#to the buffer matrix
svMatrix = np.dstack((svMatrix, sv))[:, :, 1:]
#ADD sjekk om dette er nødvendig !!!
# DistanceMatrix = np.dstack((DistanceMatrix, np.ones((len(sv),64))*DistanceTraveled[-1]))[:,:,1:]
# BeamDirectionMatrix = np.dstack((BeamDirectionMatrix, np.repeat(diry.T,len(sv),axis=0)))[:,:,1:]
# RangeMatrix = np.dstack((RangeMatrix, np.repeat(RangeOut,64,axis=1)))[:,:,1:]
#Transform into a 1D array, and make it linear
sv_mat = 10**(np.reshape(svMatrix, (-1, 1)) / 10)
if platform.system() == 'Linux1':
sV_port = np.asarray(
Parallel(n_jobs=multiprocessing.cpu_count())(
delayed(tools.ConvertToechogram)(Wdist_port[i], sv_mat)
for i in range(
len(
range(int(BananaTool[1]), int(BananaTool[2]),
int(BananaTool[3]))))))
sV_stb = np.asarray(
Parallel(n_jobs=multiprocessing.cpu_count())(
delayed(tools.ConvertToechogram)(Wdist_stb[i], sv_mat)
for i in range(
len(
range(int(BananaTool[1]), int(BananaTool[2]),
int(BananaTool[3]))))))
sV_portGhost = np.asarray(
Parallel(n_jobs=multiprocessing.cpu_count())(
delayed(tools.ConvertToechogram)(Wdist_portGhost[i],
sv_mat)
for i in range(
len(
range(int(BananaTool[1]), int(BananaTool[2]),
int(BananaTool[3]))))))
sV_stbGhost = np.asarray(
Parallel(n_jobs=multiprocessing.cpu_count())(
delayed(tools.ConvertToechogram)(Wdist_stbGhost[i],
sv_mat)
for i in range(
len(
range(int(BananaTool[1]), int(BananaTool[2]),
int(BananaTool[3]))))))
else:
#
# sV_stb = []
# sV_port = []
# sV_portGhost = []
# sV_stbGhost = []
#
#
sV_port, sV_stb, sV_portGhost, sV_stbGhost = tools.ConvertToechogram2(
Wdist_port, Wdist_stb, Wdist_portGhost, Wdist_stbGhost,
sv_mat,
range(
len(
range(int(BananaTool[1]), int(BananaTool[2]),
int(BananaTool[3])))))
# for indeks in range(len(range(int(BananaTool[1]),int(BananaTool[2]),int(BananaTool[3])))):
# sV_port = np.hstack((sV_port,tools.ConvertToechogram(Wdist_port[indeks],sv_mat)))
# sV_stb = np.hstack((sV_stb,tools.ConvertToechogram(Wdist_stb[indeks],sv_mat)))
# sV_portGhost = np.hstack((sV_portGhost,tools.ConvertToechogram(Wdist_portGhost[indeks],sv_mat)))
# sV_stbGhost = np.hstack((sV_stbGhost,tools.ConvertToechogram(Wdist_stbGhost[indeks],sv_mat)))
#Add dimension on the data, a bug fix
#ADD also add the ghost school !!!
sV_port = sV_port[:, np.newaxis]
sV_stb = sV_stb[:, np.newaxis]
sV_portGhost = sV_portGhost[:, np.newaxis]
sV_stbGhost = sV_stbGhost[:, np.newaxis]
#Start making the the search matrix
if MakeWdistStuff == True:
#Make the first value in search matrix
#ADD in future make it possible to
#analyze incomplete buffer. !!!
#ADD also add the chost school !!!
SVres_port = sV_port
SVres_stb = sV_stb
SVres_portGhost = sV_portGhost
SVres_stbGhost = sV_stbGhost
MakeWdistStuff = False
else:
#add new value to the search matrix
#ADD also add the chost school !!
sV_port = np.vstack(
(sV_port, np.nan *
np.ones(len(SVres_port[:, 0]) -
len(sV_port[:, 0]))[:, np.newaxis]))
sV_stb = np.vstack(
(sV_stb, np.nan *
np.ones(len(SVres_stb[:, 0]) -
len(sV_stb[:, 0]))[:, np.newaxis]))
sV_portGhost = np.vstack((sV_portGhost, np.nan * np.ones(
len(SVres_portGhost[:, 0]) -
len(sV_portGhost[:, 0]))[:, np.newaxis]))
sV_stbGhost = np.vstack((sV_stbGhost, np.nan * np.ones(
len(SVres_stbGhost[:, 0]) -
len(sV_stbGhost[:, 0]))[:, np.newaxis]))
SVres_port = np.hstack((SVres_port, sV_port))
SVres_stb = np.hstack((SVres_stb, sV_stb))
SVres_portGhost = np.hstack((SVres_portGhost, sV_portGhost))
SVres_stbGhost = np.hstack((SVres_stbGhost, sV_stbGhost))
# try:
# import matplotlib.pyplot as plt
# plt.figure(1)
# plt.clf()
# plt.imshow(10*np.log10(SVres_port),aspect = 'auto')
# plt.colorbar()
# plt.draw()
# plt.savefig(directory2Data.replace('mat','jpg'))
# except:
# dummy=1
# print(DirectoryToRESULT+'/SearchMatrix'+str(transectCode)+'.mat')
# scipy.io.savemat(DirectoryToRESULT+'/SearchMatrix'+str(transectCode)+'.mat',
# mdict={'SVres_port': (SVres_port),
# 'SVres_stb':(SVres_stb),
# 'SVres_portGhost':(SVres_portGhost),
# 'SVres_stbGhost':(SVres_stbGhost),
# 'DistanceTraveled':DistanceTraveled,
# 'ListOfFilesWithinTimeInterval':ListOfFilesWithinTimeInterval})
#If the buffer size is insufficient, keep stacking
else:
AddNaN = len(svMatrix[:, 0]) - len(RangeOut)
if DistanceTraveled[-1] - DistanceTraveled[-2] > 2:
if AddNaN > 0:
sv = np.vstack((sv, np.nan * np.ones((AddNaN, 64))))
RangeOut = np.vstack(
(RangeOut[:, np.newaxis], np.nan * np.ones(
(AddNaN, 1))))
else:
sv = sv[:svMatrix.shape[0], :]
RangeOut = RangeOut[:svMatrix.shape[0], np.newaxis]
svMatrix = np.dstack((svMatrix, sv))
DistanceMatrix = np.dstack(
(DistanceMatrix, np.ones(
(len(sv), 64)) * DistanceTraveled[-1]))
BeamDirectionMatrix = np.dstack(
(BeamDirectionMatrix,
np.repeat(variables.diry[:, np.newaxis].T,
len(sv),
axis=0)))
RangeMatrix = np.dstack(
(RangeMatrix, np.repeat(RangeOut[:, np.newaxis],
64,
axis=1)))
else:
svMatrix = sv
DistanceMatrix = np.ones((len(sv), 64)) * DistanceTraveled[-1]
DistanceTraveled = DistanceTraveled[1:]
BeamDirectionMatrix = np.repeat(variables.diry[:,
np.newaxis].T,
len(sv),
axis=0)
RangeMatrix = np.repeat(RangeOut[:, np.newaxis], 64, axis=1)
#Save the data
try:
scipy.io.savemat(directory2Data,
mdict={
'SVres_port': (SVres_port),
'SVres_stb': (SVres_stb),
'R_s': R_s,
'res': res,
'SVres_portGhost': (SVres_portGhost),
'SVres_stbGhost': (SVres_stbGhost),
'DistanceTraveled': DistanceTraveled,
'ListOfFilesWithinTimeInterval':
ListOfFilesWithinTimeInterval,
'NumberOfPingsInBatch': NumberOfPingsInBatch
})
except UnboundLocalError:
print('empty files', end='\r')
def MakeReport(CompleteListOfFiles,directory2Data, nation,cruice_id,platform):
import random
import datetime, time
from tools import tools
outloc = '/'
#Some user input data
# integration_dist = 1
pel_ch_thickness = 10
NumChannels = 35
PhantomEchoDist = 275
PhantomEchoWidth = 50
TH = -70
ChannelSize = NumChannels*pel_ch_thickness
Depth_bottom = 0
#Something for bookkeeping
NASC_out = []
#Convert stuff to integer,
#This is to speed up the program
Com = FastGetComplete(directory2Data,CompleteListOfFiles)
#Something about the channels
#This should be made more general
Depth = np.linspace(0,ChannelSize,NumChannels)+ChannelSize/NumChannels/2
Delta_Z = ChannelSize/NumChannels/2
NASC = np.nan*np.linspace(0,ChannelSize,NumChannels).T[:,np.newaxis]
Work_list = os.listdir(directory2Data.dir_verticalwork)
Work_I = np.arange(len(Work_list))
random.shuffle(Work_I)
#Loop through each work file
counter = 1
for work_i in Work_I:
tools.printProgressBar(counter, len(Work_I), prefix = 'Copy files', suffix = 'Files left: '+
str(len(Work_I)-counter) , decimals = 1, length = 50)
# print('files left: ' +str(len(Work_I)-counter),end='\r')
counter = counter+1
work = Work_list[work_i]
if work != 'Com.mat':
try:
if not os.path.isfile(directory2Data.dir_resultvertical+'/Report_'+work):
# directory2Data.dir_work+'/'+'Vertical_T'+str(transectID)+'.mat'
#Load mask of the filtered data
workfile = scp.loadmat(directory2Data.dir_work+'/'+work)
try:
pings = np.unique(workfile['ping_mask'])
except:
pings = []
if pings == []:
print('check: '+ directory2Data.dir_work+'/'+work)
else:
#Loop through each unique ping
for ping_idx in range(len(pings[1:])):
tools.printProgressBar(ping_idx,len(pings),prefix = 'LUF20 of '+work+':', suffix = 'Completed', length = 50)
#Idx stuff
idx = np.where(workfile['ping_mask']==pings[ping_idx+1])
idx_file = np.where(Com == (pings[ping_idx+1]))
File = CompleteListOfFiles['FileList'][np.where(CompleteListOfFiles['ping_time']==pings[ping_idx+1])][0]
IDX = CompleteListOfFiles['IDX'][np.where(CompleteListOfFiles['ping_time']==pings[ping_idx+1])][0]
#Get the file name of the idx file
filname = directory2Data.dir_rawdata+'/'+CompleteListOfFiles['FileList'][idx_file[0]]
filname = directory2Data.dir_rawdata+'/'+File
#Load file
fileID = Dataset(filname,'r',format = 'NETCDF4')
#Get the group with the vertical fan data
if fileID.groups['Sonar'].groups['Beam_group1'].beam_mode == 'Vertical':
beamgrp = 'Beam_group1'
elif fileID.groups['Sonar'].groups['Beam_group2'].beam_mode == 'Vertical':
beamgrp = 'Beam_group2'
#Get variables from .nc file
variables = tools.GetVariablesFromNC(fileID,beamgrp,False,IDX)
#Close file
fileID.close()
# if print_sa == True:
# sa_by_acocat = tools.addFrequencyLevelInfo(distance,variables.frequency,0,NumChannels,0)
# print_sa = False
if Depth_bottom == 0:
delta_lat = 2000/111111
try:
delta_lon = 2000/(111111*np.cos(np.deg2rad(float(workfile['lat_start'][0]))))
except:
delta_lon = 2000/111111
Depth_bottom = -1000
try:
Depth_bottom = GetBottomDepth(float(workfile['lat_start'][0]),float(workfile['lat_stop'][0]),
float(workfile['lon_start'][0]),float(workfile['lon_stop'][0]),delta_lat,delta_lon)
except:
Depth_bottom = -1000
#Get file name
# try:
sv, RangeOut= tools.ApplyTVG(10*np.log10(variables.BeamAmplitudeData),
variables.soundvelocity[0],
variables.sampleinterval,
variables.transmitpower,
variables.absorptioncoefficient[0],
variables.frequency,
variables.pulslength,
variables.gaintx,
variables.equivalentbeamangle,
variables.sacorrection,
variables.dirx)
# sv[np.where(RangeOut<=RemoveToCloseValues)] = np.nan
#
sv[np.where(RangeOut<=30)] = np.nan
sv[:,np.where(variables.dirx>=60)] = np.nan
sv[np.where(sv<TH)] = np.nan
#Get the depth of each pixel
Y= np.sin(np.deg2rad(variables.dirx))*np.sin(np.deg2rad(variables.diry))
Y = (abs(Y[:,np.newaxis]*RangeOut[:,np.newaxis].T)).T
sv[np.where(Y>=abs(Depth_bottom+150))] = np.nan
X= np.cos(np.deg2rad(variables.dirx))*np.sin(np.deg2rad(variables.diry))
X = (X[:,np.newaxis]*RangeOut[:,np.newaxis].T).T
sv[np.where(abs(X)>(PhantomEchoDist+PhantomEchoWidth/2))] = np.nan
sv[np.where(abs(X)<(PhantomEchoDist-PhantomEchoWidth/2))] = np.nan
try:
sv2 = sv*np.nan
sv2[workfile['r_mask'][idx],workfile['b_mask'][idx]]=sv[workfile['r_mask'][idx],workfile['b_mask'][idx]]
sv2 = 10**(sv2/10)
sv2[np.isnan(sv2)] = 0
#Remove surface beam
# sv2[:,np.where(variables.dirx < 3)] = np.nan
# sv[:,np.where(variables.dirx < 3)] = np.nan
#
#Select only at a specific distance from vessel
# sv[np.where(abs(X)>300)] = np.nan
# sv[np.where(abs(X)<250)] = np.nan
#
#
#
#
#Integrate in depth channels
for i in range(len(Depth)):
temp = sv2[np.where(abs(abs(Y)-i*ChannelSize/NumChannels +Delta_Z)<=Delta_Z)]
temp2 = np.count_nonzero(~np.isnan(temp))
NASC[i,-1]= np.nansum(temp)/temp2
#
## except IndexError:
## print('bad ping')
#
#
A = np.nansum(NASC,axis=1)[:,np.newaxis]/len(NASC[0,:])
A = A*(1852**2)*pel_ch_thickness*4*np.pi
Liste = {}
Liste['Report_time'] = str(datetime.datetime.fromtimestamp((time.time())).strftime('%Y-%m-%d %H:%M:%S'))
#General overview
Liste['Instrument'] = {}
Liste['Calibration'] = {}
Liste['DataAcquisition'] = {}
Liste['DataProcessingMethod'] = {}
Liste['Cruice'] = {}
#Instrument section
#Må gå inn i xml fil
Liste['Instrument']['Frequency'] = variables.frequency
Liste['Instrument']['TransducerLocation'] = 'AA'
Liste['Instrument']['TransducerManufacturer'] = 'Simrad'
Liste['Instrument']['TransducerModel'] = 'SU90'
Liste['Instrument']['TransducerSerial'] = 'Unknown'
Liste['Instrument']['TransducerBeamType'] = 'M2'
Liste['Instrument']['TransducerDepth'] = ''
Liste['Instrument']['TransducerOrientation'] = ''
Liste['Instrument']['TransducerPSI'] = ''
Liste['Instrument']['TransducerBeamAngleMajor'] = ''
Liste['Instrument']['TransducerBeamAngleMinor'] = ''
Liste['Instrument']['TransceiverManufacturer'] = ''
Liste['Instrument']['TransceiverModel'] = ''
Liste['Instrument']['TransceiverSerial'] = ''
Liste['Instrument']['TransducerOrientation'] = ''
#Calibration Section
Liste['Calibration']['Date'] = ''
Liste['Calibration']['AcquisitionMethod'] = ''
Liste['Calibration']['ProcessingMethod'] = ''
Liste['Calibration']['AccuracyEstimate'] = ''
#Calibration Section
Liste['DataAcquisition']['SoftwareName'] = ''
Liste['DataAcquisition']['SoftwareVersion'] = ''
Liste['DataAcquisition']['StoredDataFormat'] = ''
Liste['DataAcquisition']['StoredDataFormatVersion'] = ''
Liste['DataAcquisition']['ConvertedDataFormat'] = ''
Liste['DataAcquisition']['ConvertedDataFormatVersion'] = ''
Liste['DataAcquisition']['PingDutyCycle'] = ''
#Data Processing
Liste['DataProcessingMethod']['SoftwareName'] = 'pysonar'
Liste['DataProcessingMethod']['SoftwareVersion'] = '0.1'
Liste['DataProcessingMethod']['TriwaveCorrection '] = 'NA'
Liste['DataProcessingMethod']['ChannelID'] = ''
Liste['DataProcessingMethod']['Bandwidth'] = ''
Liste['DataProcessingMethod']['Frequency'] = variables.frequency
Liste['DataProcessingMethod']['TransceiverPower'] = variables.transmitpower
Liste['DataProcessingMethod']['TransmitPulseLength'] = variables.pulslength
Liste['DataProcessingMethod']['OnAxisGain'] = variables.gaintx
Liste['DataProcessingMethod']['OnAxisGainUnit'] = 'dB'
Liste['DataProcessingMethod']['SaCorrection'] = variables.sacorrection
Liste['DataProcessingMethod']['Absorption'] = variables.absorptioncoefficient[0]
Liste['DataProcessingMethod']['AbsorptionDescription'] = 'Nominal'
Liste['DataProcessingMethod']['SoundSpeed'] = variables.soundvelocity[0]
Liste['DataProcessingMethod']['SoundSpeedDescription'] = 'Nominal'
Liste['DataProcessingMethod']['TransducerPSI'] = ''
#Cruice
Liste['Cruice']['Survey'] = ''
Liste['Cruice']['Country'] = 'NO'
Liste['Cruice']['Nation'] = nation
Liste['Cruice']['Platform'] = platform
Liste['Cruice']['StartDate'] = ''
Liste['Cruice']['StopDate'] = ''
Liste['Cruice']['Organisation'] = ''
Liste['Cruice']['LocalID'] = cruice_id
# try:
tiime = workfile['start_time'][0]
stime = tiime[:4]+'-'+tiime[4:6]+'-'+tiime[6:8]+' '+tiime[9:11]+':'+tiime[11:13]+':'+tiime[13:]
# except:
# stime = ''
try:
tiime = workfile['stop_time'][0]
etime = tiime[:4]+'-'+tiime[4:6]+'-'+tiime[6:8]+' '+tiime[9:11]+':'+tiime[11:13]+':'+tiime[13:]
except:
etime = ''
Liste['Cruice']['Log'] = {}
Liste['Cruice']['Log']['Distance'] = workfile['log_start'][0]
Liste['Cruice']['Log']['TimeStart'] = stime
Liste['Cruice']['Log']['TimeStop'] = etime
Liste['Cruice']['Log']['Latitude_start'] = workfile['lat_start'][0]
Liste['Cruice']['Log']['Longitude_start'] = workfile['lon_start'][0]
Liste['Cruice']['Log']['Latitude_stop'] = workfile['lat_stop'][0]
Liste['Cruice']['Log']['Longitude_stop'] = workfile['lon_stop'][0]
Liste['Cruice']['Log']['Bottom_depth'] = Depth_bottom
Liste['Cruice']['Log']['Origin'] = ''
Liste['Cruice']['Log']['Validity'] = ''
Liste['Cruice']['Log']['Sample'] = {}
Liste['Cruice']['Log']['Sample']['ChannelThickness'] = pel_ch_thickness
Liste['Cruice']['Log']['Sample']['Acocat'] = 'Unknown'
Liste['Cruice']['Log']['Sample']['SvThreshold'] = str(TH)
Liste['Cruice']['Log']['Sample']['PingAxisInterval'] = 1 #samme som integration thickness
Liste['Cruice']['Log']['Sample']['PingAxisIntervalType'] = 'distance'
Liste['Cruice']['Log']['Sample']['PingAxisIntervalUnit'] = 'nmi'
Liste['Cruice']['Log']['Sample']['DataValue'] = A
Liste['Cruice']['Log']['Sample']['DataType'] = 'C'
Liste['Cruice']['Log']['Sample']['DataUnit'] = 'm2nm-2'
Liste['Cruice']['Log']['Sample']['PhantomEchoDistance'] = PhantomEchoDist
Liste['Cruice']['Log']['Sample']['PhantomEchoWidth'] = PhantomEchoWidth
except:
print('Badping')
try:
import scipy.io as sc
sc.savemat(directory2Data.dir_resultvertical+'/Report_'+work,mdict=Liste)
Depth_bottom = 0
except:
dummy = 1
except:
print('bad')
#
# from SendMail import send_email
# send_email('bad stuff for Report_'+work)
#Må generaliseres
#
## print(A)
# if NASC_out == []:
# NASC_out = A
# else:
# NASC_out = np.hstack((NASC_out, A))
#
#
# for ikk in range(len(A)):
# if Depth[ikk]>(abs(Depth_bottom)-150):
# A[ikk] = 0
# NASC_out[ikk,-1] = 0
#
#
# if A[ikk]>0:
## sa_value = ET.SubElement(sa_by_acocat,'sa')
# sa_value.set('ch',str(ikk+1))
# sa_value.text = str(A[ikk][0])
#
#
#
# except:
# print('bad file')
# tools.indent(root)
# tree = ET.ElementTree(root)
# tree.write(directory2Data.dir_result+outloc+'/Vertical/'+work.replace('mat','xml'), xml_declaration=True, encoding='utf-8', method="xml")
def MakeHorizontalLuf20(CompleteListOfFiles, directory2Data, start_time,
log_start, stop_time, lat_start, lat_stop, lon_start,
lon_stop, TimeIDX, nation, cruice_id, platform):
sonar_draft = 8
Threshold = -75
#Fiksed stuff
integrator_dist = 1
pel_ch_thickness = 1
NumChannels = 300
# ChannelSize = 350
#Start information in the text
root = ET.Element("echosounder_dataset")
ET.SubElement(root, 'report_time').text = str(
datetime.datetime.fromtimestamp(
(time.time())).strftime('%Y-%m-%d %H:%M:%S'))
ET.SubElement(root, 'lsss_version').text = "PNMDformats v 0.1 - vertical"
ET.SubElement(root, 'nation').text = nation
ET.SubElement(root, 'platform').text = platform
ET.SubElement(root, 'cruise').text = cruice_id
#Make the distance list hirarchy
distance_list = ET.SubElement(root, 'distance_list')
#Get list of work files
Listoffiles = os.listdir(directory2Data.dir_work)
#Go through each log distance
for i in range(len(start_time)):
#Make empty vector to start preparing for stacking
SA = np.zeros((NumChannels, 1)) * np.nan
#Convert the start and stop time to correct format
stime = stop_time[i][:4] + '-' + start_time[i][4:6] + '-' + start_time[
i][6:8] + ' ' + start_time[i][9:11] + ':' + start_time[i][
11:13] + ':' + start_time[i][13:]
etime = stop_time[i][:4] + '-' + stop_time[i][4:6] + '-' + stop_time[
i][6:8] + ' ' + stop_time[i][9:11] + ':' + stop_time[i][
11:13] + ':' + stop_time[i][13:]
#Print the distance list information to xml
distance = tools.addLogDistanceInfo(distance_list, lat_start[i],
lon_start[i], stime, lat_stop[i],
lon_stop[i], etime,
integrator_dist, pel_ch_thickness,
log_start[i])
#Print the frequecy level information to xlm
sa_by_acocat = tools.addFrequencyLevelInfo(distance, 26000, 0,
NumChannels, Threshold)
for iii in range(CompleteListOfFiles.shape[0]):
ein = SecondsBetweenTransect(start_time[i].replace('T', ''),
int(CompleteListOfFiles[iii][0]))
to = SecondsBetweenTransect(stop_time[i].replace('T', ''),
int(CompleteListOfFiles[iii][0]))
if ein <= 0 and to >= 0:
tools.printProgressBar(
iii,
CompleteListOfFiles.shape[0],
prefix=str(CompleteListOfFiles.shape[0] - iii) +
' Status for HLuf20',
suffix='Completed',
length=50)
for file in Listoffiles:
if 'Horizontal' in file:
Mat = scp.loadmat(directory2Data.dir_work + '\\' +
file)
WorkFile = Mat['WorkFile']
WorkIDX = Mat['WorkIDX'][0]
WorkTime = Mat['WorkTime'][0]
WorkPhi = Mat['WorkPhi'][0]
WorkBeam = Mat['WorkBeam'][0]
pingIDX = np.where(
WorkTime == CompleteListOfFiles[iii][0])[0]
if len(pingIDX) > 0:
#Get the file name of the idx file
filname = directory2Data.dir_rawdata + '/' + np.unique(
WorkFile[pingIDX])[0]
#Load file
fileID = Dataset(filname, 'r', format='NETCDF4')
#Get the group with the vertical fan data
if fileID.groups['Sonar'].groups[
'Beam_group1'].beam_mode == 'Horizontal':
beamgrp = 'Beam_group1'
elif fileID.groups['Sonar'].groups[
'Beam_group2'].beam_mode == 'Horizontal':
beamgrp = 'Beam_group2'
#Get variables from .nc file
variables = tools.GetVariablesFromNC(
fileID, beamgrp, False,
int(np.unique(WorkIDX[pingIDX])[0]))
fileID.close()
sv, RangeOut = tools.ApplyTVG(
10 * np.log10(variables.BeamAmplitudeData),
variables.soundvelocity[0],
variables.sampleinterval,
variables.transmitpower,
variables.absorptioncoefficient[0],
variables.frequency, variables.pulslength,
variables.gaintx,
variables.equivalentbeamangle,
variables.sacorrection, variables.dirx)
sv_ind = np.nan * sv
for ipp in range(len(variables.diry)):
WorkPhi[np.where(
WorkPhi == variables.diry[ipp])] = int(ipp)
for ipp in range(len(np.unique(RangeOut))):
WorkBeam[np.where(WorkBeam == np.unique(
RangeOut)[ipp])] = int(ipp)
Depth = np.dot(
RangeOut[:, np.newaxis],
np.sin(
np.deg2rad(variables.dirx[:, np.newaxis])).
T) + sonar_draft
sv_ind[WorkBeam.astype(int),
WorkPhi.astype(int)] = sv[
WorkBeam.astype(int),
WorkPhi.astype(int)]
#Threshold
sv_ind[np.where(sv_ind < Threshold)] = np.nan
#Go linear
sv_ind = 10**(sv_ind / 10)
for depth_i in range(NumChannels):
a = np.where(np.round(Depth) == depth_i)
if len(a[0]) > 0:
SA[depth_i,
-1] = SA[depth_i, -1] + np.nansum(
sv_ind[a]) / len(a[0])
SA = np.hstack((SA, np.nan * np.zeros(
(NumChannels, 1))))
elif ein <= 0 and to < 0:
SA2 = SA
lenSA = SA2.shape[1]
if lenSA > 3:
A = np.nansum(SA2,
axis=1) / lenSA * 4 * np.pi * 1852 * 1852
for ikk in range(len(A)):
if A[ikk] > 0:
sa_value = ET.SubElement(sa_by_acocat, 'sa')
sa_value.set('ch', str(ikk + 1))
sa_value.text = str(A[ikk])
tools.indent(root)
tree = ET.ElementTree(root)
tree.write(directory2Data.dir_result +
'\HLUF20.xml',
xml_declaration=True,
encoding='utf-8',
method="xml")
break
tools.indent(root)
tree = ET.ElementTree(root)
tree.write(directory2Data.dir_result + '\HLUF20.xml',
xml_declaration=True,
encoding='utf-8',
method="xml")
def ComListOfFiles(directory2Data,beam_mode):
#This function makes an idx of all files
#Open the IDX files
try:
fd = open(directory2Data.dir_src+'/'+beam_mode+'pingtime.csv','r')
fd2 = open(directory2Data.dir_src+'/'+beam_mode+'FileList.csv','r')
fd3 = open(directory2Data.dir_src+'/'+beam_mode+'IDX.csv','r')
fd4 = open(directory2Data.dir_src+'/'+beam_mode+'BeamIDX.csv','r')
# fd5 = open(directory2Data.dir_src+'/'+beam_mode+'LogDist.csv','r')
A = np.asarray(fd.read().split(','))[:-1]
B = np.asarray(fd2.read().split(','))[:-1]
C = np.asarray(fd3.read().split(','))[:-1]
D = np.asarray(fd4.read().split(','))[:-1]
CompleteListOfFiles = np.array((A.T,B.T,C.T)).T
# If the files don't exist, create them
except FileNotFoundError:
#Sort all the .nc files
ListOfFiles = np.sort(os.listdir(directory2Data.dir_rawdata))
#open and rewrite IDX files
fd = open(directory2Data.dir_src+'/'+beam_mode+'pingtime.csv','w')
fd2 = open(directory2Data.dir_src+'/'+beam_mode+'FileList.csv','w')
fd3 = open(directory2Data.dir_src+'/'+beam_mode+'IDX.csv','w')
fd4 = open(directory2Data.dir_src+'/'+beam_mode+'BeamIDX.csv','w')
# fd5 = open(directory2Data.dir_src+'/'+beam_mode+'LogDist.csv','w')
#This helps to prevent opening and closing the same file multiple times
oldFileName = ''
#Loop through each file
CompleteListOfFiles = []
for i in range(len(ListOfFiles)):
#Print progress
tools.printProgressBar(i+1,len(ListOfFiles),prefix = 'Stacking: ', suffix = 'Completed', length = 50)
try:
#If this is a new file, close the last and open this
if ListOfFiles[i]!=oldFileName:
try:
fid_nc.close()
except:
k=1
fid_nc = Dataset(os.path.join(directory2Data.dir_rawdata, ListOfFiles[i]),'r')
oldFileName = ListOfFiles[i]
#Get the correct beam group
if fid_nc.groups['Sonar'].groups['Beam_group1'].beam_mode == beam_mode:
beamgrp = 'Beam_group1'
elif fid_nc.groups['Sonar'].groups['Beam_group2'].beam_mode == beam_mode:
beamgrp = 'Beam_group2'
#Get the beam data
beam_data = fid_nc.groups['Sonar'].groups[beamgrp]
#Loop through each ping
for ii in range(len(beam_data.variables['ping_time'])):
timeID = np.where(abs(beam_data.variables['ping_time'][ii]-fid_nc.groups['Platform'].variables['time1'][:]/100)==
np.nanmin(abs(beam_data.variables['ping_time'][ii]-fid_nc.groups['Platform'].variables['time1'][:]/100)))
fd.write(str(beam_data.variables['ping_time'][ii])+',')
fd2.write(ListOfFiles[i]+',')
fd3.write(str(ii)+',')
fd4.write(beamgrp+',')
except:
print(' *bad ping*')
#Close files
fd.close()
fd2.close()
fd3.close()
fd4.close()
#Reopen the files
fd=open(directory2Data.dir_src+'/'+beam_mode+'pingtime.csv','r')
fd2=open(directory2Data.dir_src+'/'+beam_mode+'FileList.csv','r')
fd3=open(directory2Data.dir_src+'/'+beam_mode+'IDX.csv','r')
fd4 = open(directory2Data.dir_src+'/'+beam_mode+'BeamIDX.csv','r')
# fd5 = open(directory2Data.dir_src+'/'+beam_mode+'LogDist.csv','r')
#Get the IDX information
A = np.asarray(fd.read().split(','))[:-1]
B = np.asarray(fd2.read().split(','))[:-1]
C = np.asarray(fd3.read().split(','))[:-1]
D = np.asarray(fd4.read().split(','))[:-1]
# logdist = np.asarray(fd5.read().split(','))[:-1]
CompleteListOfFiles = np.array((A.T,B.T,C.T)).T
#Close the files
fd.close()
fd2.close()
fd3.close()
fd4.close()
fd5.close()
return CompleteListOfFiles, D