def __init__(self,
configdir,
filterdir,
folderTrain1=None,
folderTrain2=None,
recogniser=None,
imgWidth=0,
CLI=False):
self.filterdir = filterdir
self.configdir = configdir
cl = SupportClasses.ConfigLoader()
self.FilterDict = cl.filters(filterdir, bats=False)
self.LearningDict = cl.learningParams(
os.path.join(configdir, "LearningParams.txt"))
self.sp = SignalProc.SignalProc(self.LearningDict['sgramWindowWidth'],
self.LearningDict['sgramHop'])
self.imgsize = [self.LearningDict['imgX'], self.LearningDict['imgY']]
self.tmpdir1 = False
self.tmpdir2 = False
self.CLI = CLI
if CLI:
self.filterName = recogniser
self.folderTrain1 = folderTrain1
self.folderTrain2 = folderTrain2
self.imgWidth = imgWidth
self.autoThr = True
self.correction = True
self.annotatedAll = True
else:
self.autoThr = False
self.correction = False
self.imgWidth = imgWidth
def __init__(self, configdir, species, calltypes, fs, length, windowwidth,
inc, imageheight, imagewidth):
self.species = species
self.length = length
self.windowwidth = windowwidth
self.inc = inc
self.imageheight = imageheight
self.imagewidth = imagewidth
self.calltypes = calltypes
self.fs = fs
cl = SupportClasses.ConfigLoader()
self.LearningDict = cl.learningParams(
os.path.join(configdir, "LearningParams.txt"))
def __init__(self,
parent,
mode="GUI",
configdir='',
sdir='',
recogniser=None,
wind=False):
# read config and filters from user location
self.configdir = configdir
self.configfile = os.path.join(configdir, "AviaNZconfig.txt")
self.ConfigLoader = SupportClasses.ConfigLoader()
self.config = self.ConfigLoader.config(self.configfile)
self.saveConfig = True
self.filtersDir = os.path.join(configdir, self.config['FiltersDir'])
self.FilterDicts = self.ConfigLoader.filters(self.filtersDir)
if mode == "GUI":
self.CLI = False
self.testmode = False
if parent is None:
print(
"ERROR: must provide a parent UI or specify CLI/testmode")
return
self.ui = parent
elif mode == "CLI":
self.CLI = True
self.testmode = False
elif mode == "test":
self.CLI = False
self.testmode = True
else:
print("ERROR: unrecognized mode ", mode)
return
self.dirName = []
# In CLI/test modes, immediately run detection on init
if self.CLI:
self.dirName = sdir
self.species = [recogniser]
self.wind = wind
self.detect()
elif self.testmode:
self.dirName = sdir
self.species = [recogniser]
self.wind = wind
self.filesDone = []
self.detect()
def __init__(self,
configdir,
filterdir,
folderTrain1=None,
folderTrain2=None,
recogniser=None,
imgWidth=0,
CLI=False):
# Two important things:
# 1. LearningParams.txt, which a dictionary of parameters *** including spectrogram parameters
# 2. CLI: whether it runs off the command line, which makes picking the ROC curve parameters hard
# Qn: what is imgWidth? Why not a learning param?
self.filterdir = filterdir
self.configdir = configdir
cl = SupportClasses.ConfigLoader()
self.FilterDict = cl.filters(filterdir, bats=False)
self.LearningDict = cl.learningParams(
os.path.join(configdir, "LearningParams.txt"))
self.sp = SignalProc.SignalProc(self.LearningDict['sgramWindowWidth'],
self.LearningDict['sgramHop'])
self.imgsize = [self.LearningDict['imgX'], self.LearningDict['imgY']]
self.tmpdir1 = False
self.tmpdir2 = False
self.ROCdata = {}
self.CLI = CLI
if CLI:
self.filterName = recogniser
self.folderTrain1 = folderTrain1
self.folderTrain2 = folderTrain2
self.imgWidth = imgWidth
self.autoThr = True
self.correction = True
self.annotatedAll = True
else:
self.autoThr = False
self.correction = False
self.imgWidth = imgWidth
def __init__(self,
testDir,
currfilt,
filtname,
configdir,
filterdir,
CLI=False):
""" currfilt: the recognizer to be used (dict) """
self.testDir = testDir
self.outfile = open(os.path.join(self.testDir, "test-results.txt"),
"w")
self.currfilt = currfilt
self.filtname = filtname
self.configdir = configdir
self.filterdir = filterdir
# Note: this is just the species name, unlike the self.species in Batch mode
species = self.currfilt['species']
self.sampleRate = self.currfilt['SampleRate']
self.calltypes = []
for fi in self.currfilt['Filters']:
self.calltypes.append(fi['calltype'])
self.outfile.write("Recogniser name: %s\n" % (filtname))
self.outfile.write("Species name: %s\n" % (species))
self.outfile.write("Using data: %s\n" % (self.testDir))
# 0. Generate GT files from annotations in test folder
self.manSegNum = 0
self.window = 1
inc = None
print('Generating GT...')
for root, dirs, files in os.walk(self.testDir):
for file in files:
wavFile = os.path.join(root, file)
if file.lower().endswith('.wav') and os.stat(
wavFile).st_size != 0 and file + '.data' in files:
segments = Segment.SegmentList()
segments.parseJSON(wavFile + '.data')
self.manSegNum += len(segments.getSpecies(species))
# Currently, we ignore call types here and just
# look for all calls for the target species.
segments.exportGT(wavFile, species, resolution=self.window)
if self.manSegNum == 0:
print("ERROR: no segments for species %s found" % species)
self.text = 0
return
# 1. Run Batch Processing upto WF and generate .tempdata files (no post-proc)
avianz_batch = AviaNZ_batch.AviaNZ_batchProcess(
parent=None,
configdir=self.configdir,
mode="test",
sdir=self.testDir,
recogniser=filtname,
wind=1)
# NOTE: will use wind-robust detection
# 2. Report statistics of WF followed by general post-proc steps (no CNN but wind-merge neighbours-delete short)
self.text = self.getSummary(CNN=False)
# 3. Report statistics of WF followed by post-proc steps (wind-CNN-merge neighbours-delete short)
if "CNN" in self.currfilt:
cl = SupportClasses.ConfigLoader()
filterlist = cl.filters(self.filterdir, bats=False)
CNNDicts = cl.CNNmodels(filterlist, self.filterdir, [filtname])
# Providing one filter, so only one CNN should be returned:
if len(CNNDicts) != 1:
print("ERROR: Couldn't find a unique matching CNN!")
self.outfile.write("No matching CNN found!\n")
self.outfile.write("-- End of testing --\n")
self.outfile.close()
return
CNNmodel = list(CNNDicts)[0]
self.text = self.getSummary(CNN=True)
self.outfile.write("-- End of testing --\n")
self.outfile.close()
print("Testing output written to " +
os.path.join(self.testDir, "test-results.txt"))
def mainlauncher(cli, cheatsheet, zooniverse, infile, imagefile, batchmode,
training, testing, sdir1, sdir2, recogniser, wind, width,
command):
# adapt path to allow this to be launched from wherever
import sys, os
if getattr(sys, 'frozen', False):
appdir = sys._MEIPASS
else:
appdir = os.path.dirname(os.path.abspath(__file__))
os.chdir(appdir)
# print("Using python at", sys.path)
# print(os.environ)
# print("Version", sys.version)
try:
import platform, json, shutil
from jsonschema import validate
import SupportClasses
except Exception as e:
print("ERROR: could not import packages")
raise
# determine location of config file and bird lists
if platform.system() == 'Windows':
# Win
configdir = os.path.expandvars(os.path.join("%APPDATA%", "AviaNZ"))
elif platform.system() == 'Linux' or platform.system() == 'Darwin':
# Unix
configdir = os.path.expanduser("~/.avianz/")
else:
print("ERROR: what OS is this? %s" % platform.system())
raise
# if config and bird files not found, copy from distributed backups.
# so these files will always exist on load (although they could be corrupt)
# (exceptions here not handled and should always result in crashes)
if not os.path.isdir(configdir):
print("Creating config dir %s" % configdir)
try:
os.makedirs(configdir)
except Exception as e:
print("ERROR: failed to make config dir")
print(e)
raise
# pre-run check of config file validity
confloader = SupportClasses.ConfigLoader()
configschema = json.load(open("Config/config.schema"))
learnparschema = json.load(open("Config/learnpar.schema"))
try:
config = confloader.config(os.path.join(configdir, "AviaNZconfig.txt"))
validate(instance=config, schema=configschema)
learnpar = confloader.learningParams(
os.path.join(configdir, "LearningParams.txt"))
validate(instance=learnpar, schema=learnparschema)
print("successfully validated config file")
except Exception as e:
print("Warning: config file failed validation with:")
print(e)
try:
shutil.copy2("Config/AviaNZconfig.txt", configdir)
shutil.copy2("Config/LearningParams.txt", configdir)
except Exception as e:
print("ERROR: failed to copy essential config files")
print(e)
raise
# check and if needed copy any other necessary files
necessaryFiles = [
"ListCommonBirds.txt", "ListDOCBirds.txt", "ListBats.txt",
"LearningParams.txt"
]
for f in necessaryFiles:
if not os.path.isfile(os.path.join(configdir, f)):
print("File %s not found in config dir, providing default" % f)
try:
shutil.copy2(os.path.join("Config", f), configdir)
except Exception as e:
print("ERROR: failed to copy essential config files")
print(e)
raise
# copy over filters to ~/.avianz/Filters/:
filterdir = os.path.join(configdir, "Filters/")
if not os.path.isdir(filterdir):
print("Creating filter dir %s" % filterdir)
os.makedirs(filterdir)
for f in os.listdir("Filters"):
ff = os.path.join("Filters", f) # Kiwi.txt
if not os.path.isfile(os.path.join(filterdir,
f)): # ~/.avianz/Filters/Kiwi.txt
print("Recogniser %s not found, providing default" % f)
try:
shutil.copy2(
ff, filterdir) # cp Filters/Kiwi.txt ~/.avianz/Filters/
except Exception as e:
print("Warning: failed to copy recogniser %s to %s" %
(ff, filterdir))
print(e)
# run splash screen:
if cli:
print("Starting AviaNZ in CLI mode")
if batchmode:
import AviaNZ_batch
if os.path.isdir(sdir1) and recogniser in confloader.filters(
filterdir).keys():
avianzbatch = AviaNZ_batch.AviaNZ_batchProcess(
parent=None,
mode="CLI",
configdir=configdir,
sdir=sdir1,
recogniser=recogniser,
wind=wind)
print("Analysis complete, closing AviaNZ")
else:
print(
"ERROR: valid input dir (-d) and recogniser name (-r) are essential for batch processing"
)
raise
elif training:
import Training
if os.path.isdir(sdir1) and os.path.isdir(
sdir2) and recogniser in confloader.filters(
filterdir).keys() and width > 0:
training = Training.CNNtrain(configdir,
filterdir,
sdir1,
sdir2,
recogniser,
width,
CLI=True)
training.cliTrain()
print("Training complete, closing AviaNZ")
else:
print(
"ERROR: valid input dirs (-d and -e) and recogniser name (-r) are essential for training"
)
raise
elif testing:
import Training
filts = confloader.filters(filterdir)
if os.path.isdir(sdir1) and recogniser in filts:
testing = Training.CNNtest(sdir1,
filts[recogniser],
recogniser,
configdir,
filterdir,
CLI=True)
print("Testing complete, closing AviaNZ")
else:
print(
"ERROR: valid input dir (-d) and recogniser name (-r) are essential for training"
)
raise
else:
if (cheatsheet or zooniverse) and isinstance(infile, str):
import AviaNZ
avianz = AviaNZ(configdir=configdir,
CLI=True,
cheatsheet=cheatsheet,
zooniverse=zooniverse,
firstFile=infile,
imageFile=imagefile,
command=command)
print("Analysis complete, closing AviaNZ")
else:
print("ERROR: valid input file (-f) is needed")
raise
else:
task = None
print("Starting AviaNZ in GUI mode")
from PyQt5.QtWidgets import QApplication
app = QApplication(sys.argv)
# a hack to fix default font size (Win 10 suggests 7 pt for QLabels for some reason)
QApplication.setFont(QApplication.font("QMenu"))
while True:
# splash screen?
if task is None:
# This screen asks what you want to do, then processes the response
import Dialogs
first = Dialogs.StartScreen()
first.show()
app.exec_()
task = first.getValues()
avianz = None
if task == 1:
import AviaNZ_manual
avianz = AviaNZ_manual.AviaNZ(configdir=configdir)
elif task == 2:
import AviaNZ_batch_GUI
avianz = AviaNZ_batch_GUI.AviaNZ_batchWindow(
configdir=configdir)
elif task == 3:
import AviaNZ_batch_GUI
avianz = AviaNZ_batch_GUI.AviaNZ_reviewAll(configdir=configdir)
elif task == 4:
import SplitAnnotations
avianz = SplitAnnotations.SplitData()
# catch bad initialiation
if avianz:
avianz.activateWindow()
else:
return
out = app.exec_()
QApplication.closeAllWindows()
QApplication.processEvents()
# catch exit code to see if restart requested:
# (note: do not use this for more complicated cleanup,
# no guarantees that it is returned before program closes)
if out == 0:
# default quit
break
elif out == 1:
# restart to splash screen
task = None
elif out == 2:
# request switch to Splitter
task = 4
def __init__(self,testDir,currfilt,configdir,filterdir,CLI=False):
self.testDir = testDir
self.outfile = open(os.path.join(self.testDir, "test-results.txt"),"w")
if CLI:
cl = SupportClasses.ConfigLoader()
self.FilterDict = cl.filters(filterdir, bats=False)
if currfilt.lower().endswith('.txt'):
self.currfilt = self.FilterDict[currfilt[:-4]]
else:
self.currfilt = self.FilterDict[currfilt]
else:
self.currfilt = currfilt
self.configdir = configdir
self.filterdir = filterdir
self.species = self.currfilt['species']
self.sampleRate = self.currfilt['SampleRate']
self.calltypes = []
for fi in self.currfilt['Filters']:
self.calltypes.append(fi['calltype'])
self.outfile.write("Recogniser name: %s\n" %(self.currfilt))
self.outfile.write("Species name: %s\n" % (self.species))
self.outfile.write("Using data: %s\n" % (self.testDir))
# 0. Generate GT files from annotations in test folder
self.manSegNum = 0
self.window = 1
inc = None
print('Generating GT...')
for root, dirs, files in os.walk(self.testDir):
for file in files:
wavFile = os.path.join(root, file)
if file.lower().endswith('.wav') and os.stat(wavFile).st_size != 0 and file + '.data' in files:
segments = Segment.SegmentList()
segments.parseJSON(wavFile + '.data')
self.manSegNum += len(segments.getSpecies(self.species))
# Currently, we ignore call types here and just
# look for all calls for the target species.
segments.exportGT(wavFile, self.species, window=self.window, inc=inc)
if self.manSegNum == 0:
print("ERROR: no segments for species %s found" % self.species)
self.flag = False
self.text = 0
return
# 1. Run Batch Processing upto WF and generate .tempdata files (no post-proc)
avianz_batch = AviaNZ_batch.AviaNZ_batchProcess(parent=None, configdir=self.configdir, mode="test",
sdir=self.testDir, recogniser=self.species, wind=True)
# 2. Report statistics of WF followed by general post-proc steps (no CNN but wind-merge neighbours-delete short)
self.flag, self.text = self.getSummary(avianz_batch, CNN=False)
# 3. Report statistics of WF followed by post-proc steps (wind-CNN-merge neighbours-delete short)
if "CNN" in self.currfilt:
cl = SupportClasses.ConfigLoader()
filterlist = cl.filters(self.filterdir, bats=False)
CNNDicts = cl.CNNmodels(filterlist, self.filterdir, [self.species])
if self.species in CNNDicts.keys():
CNNmodel = CNNDicts[self.species]
flag, text = self.getSummary(avianz_batch, CNN=True, CNNmodel=CNNmodel)
else:
print("Couldn't find a matching CNN!")
self.outfile.write("-- End of testing --\n")
self.outfile.close()
return
self.outfile.write("-- End of testing --\n")
self.outfile.close()
# Tidy up
for root, dirs, files in os.walk(self.testDir):
for file in files:
if file.endswith('.tmpdata'):
os.remove(os.path.join(root, file))
if CLI:
print("Output written to " + os.path.join(self.testDir, "test-results.txt"))
def mainlauncher(cli, cheatsheet, zooniverse, infile, imagefile, batchmode, training, testing, sdir1, sdir2, recogniser, wind, width, command):
# determine location of config file and bird lists
if platform.system() == 'Windows':
# Win
configdir = os.path.expandvars(os.path.join("%APPDATA%", "AviaNZ"))
elif platform.system() == 'Linux' or platform.system() == 'Darwin':
# Unix
configdir = os.path.expanduser("~/.avianz/")
else:
print("ERROR: what OS is this? %s" % platform.system())
sys.exit()
# if config and bird files not found, copy from distributed backups.
# so these files will always exist on load (although they could be corrupt)
# (exceptions here not handled and should always result in crashes)
if not os.path.isdir(configdir):
print("Creating config dir %s" % configdir)
try:
os.makedirs(configdir)
except Exception as e:
print("ERROR: failed to make config dir")
print(e)
sys.exit()
# pre-run check of config file validity
confloader = SupportClasses.ConfigLoader()
configschema = json.load(open("Config/config.schema"))
try:
config = confloader.config(os.path.join(configdir, "AviaNZconfig.txt"))
validate(instance=config, schema=configschema)
print("successfully validated config file")
except Exception as e:
print("Warning: config file failed validation with:")
print(e)
try:
shutil.copy2("Config/AviaNZconfig.txt", configdir)
except Exception as e:
print("ERROR: failed to copy essential config files")
print(e)
sys.exit()
# check and if needed copy any other necessary files
necessaryFiles = ["ListCommonBirds.txt", "ListDOCBirds.txt", "ListBats.txt", "LearningParams.txt"]
for f in necessaryFiles:
if not os.path.isfile(os.path.join(configdir, f)):
print("File %s not found in config dir, providing default" % f)
try:
shutil.copy2(os.path.join("Config", f), configdir)
except Exception as e:
print("ERROR: failed to copy essential config files")
print(e)
sys.exit()
# copy over filters to ~/.avianz/Filters/:
filterdir = os.path.join(configdir, "Filters/")
if not os.path.isdir(filterdir):
print("Creating filter dir %s" % filterdir)
os.makedirs(filterdir)
for f in os.listdir("Filters"):
ff = os.path.join("Filters", f) # Kiwi.txt
if not os.path.isfile(os.path.join(filterdir, f)): # ~/.avianz/Filters/Kiwi.txt
print("Recogniser %s not found, providing default" % f)
try:
shutil.copy2(ff, filterdir) # cp Filters/Kiwi.txt ~/.avianz/Filters/
except Exception as e:
print("Warning: failed to copy recogniser %s to %s" % (ff, filterdir))
print(e)
# run splash screen:
if cli:
print("Starting AviaNZ in CLI mode")
if batchmode:
import AviaNZ_batch
if os.path.isdir(sdir1) and recogniser in confloader.filters(filterdir).keys():
avianzbatch = AviaNZ_batch.AviaNZ_batchProcess(parent=None, mode="CLI", configdir=configdir, sdir=sdir1, recogniser=recogniser, wind=wind)
print("Analysis complete, closing AviaNZ")
else:
print("ERROR: valid input dir (-d) and recogniser name (-r) are essential for batch processing")
sys.exit()
elif training:
import Training
if os.path.isdir(sdir1) and os.path.isdir(sdir2) and recogniser in confloader.filters(filterdir).keys() and width>0:
training = Training.CNNtrain(configdir,filterdir,sdir1,sdir2,recogniser,width,CLI=True)
training.cliTrain()
print("Training complete, closing AviaNZ")
else:
print("ERROR: valid input dirs (-d and -e) and recogniser name (-r) are essential for training")
sys.exit()
elif testing:
import Training
if os.path.isdir(sdir1) and recogniser in confloader.filters(filterdir).keys():
testing = Training.CNNtest(sdir1,recogniser,configdir,filterdir,CLI=True)
print("Testing complete, closing AviaNZ")
else:
print("ERROR: valid input dir (-d) and recogniser name (-r) are essential for training")
sys.exit()
else:
if (cheatsheet or zooniverse) and isinstance(infile, str):
import AviaNZ
avianz = AviaNZ(configdir=configdir, CLI=True, cheatsheet=cheatsheet, zooniverse=zooniverse,
firstFile=infile, imageFile=imagefile, command=command)
print("Analysis complete, closing AviaNZ")
else:
print("ERROR: valid input file (-f) is needed")
sys.exit()
else:
print("Starting AviaNZ in GUI mode")
# This screen asks what you want to do, then processes the response
import Dialogs
from PyQt5.QtWidgets import QApplication
app = QApplication(sys.argv)
first = Dialogs.StartScreen()
first.show()
app.exec_()
task = first.getValues()
avianz = None
if task == 1:
import AviaNZ_manual
avianz = AviaNZ_manual.AviaNZ(configdir=configdir)
elif task==2:
import AviaNZ_batch_GUI
avianz = AviaNZ_batch_GUI.AviaNZ_batchWindow(configdir=configdir)
elif task==3:
import AviaNZ_batch_GUI
avianz = AviaNZ_batch_GUI.AviaNZ_reviewAll(configdir=configdir)
if avianz:
avianz.show()
else:
return
out = app.exec_()
QApplication.closeAllWindows()
# restart requested:
if out == 1:
mainlauncher()
elif out == 2:
import SplitAnnotations
avianz = SplitAnnotations.SplitData()
avianz.show()
app.exec_()
print("Processing complete, returning to AviaNZ")
QApplication.closeAllWindows()