def getDurationOfEventWithList( connection, file, eventName, idAnimalA , animalList, minFrame=None, maxFrame=None ):
durationOfEvent = 0
for animalCandidate in animalList:
timeLine = EventTimeLineCached( connection , file, eventName , idAnimalA , animalCandidate.baseId, minFrame=minFrame, maxFrame=maxFrame )
durationOfEvent += timeLine.getTotalLength()
return durationOfEvent
genoA = pool.animalDictionnary[animal].genotype
except:
pass
for behavEvent in behaviouralEventOneMouse:
print("computing individual event: {}".format(behavEvent))
behavEventTimeLine = EventTimeLineCached(connection,
file,
behavEvent,
animal,
minFrame=tmin,
maxFrame=tmax)
totalEventDuration = behavEventTimeLine.getTotalLength()
nbEvent = behavEventTimeLine.getNumberOfEvent(minFrame=tmin,
maxFrame=tmax)
print("total event duration: ", totalEventDuration)
animalDic[rfid][behavEventTimeLine.eventName +
" TotalLen"] = totalEventDuration
animalDic[rfid][behavEventTimeLine.eventName + " Nb"] = nbEvent
print(behavEventTimeLine.eventName, genoA,
behavEventTimeLine.idA, totalEventDuration, nbEvent)
print("writing...")
'''
file strain sex group day exp idA idB minTime maxTime tot_dist
'''
header = [
def process( file ):
print(file)
chronoFullFile = Chronometer("File " + file )
connection = sqlite3.connect( file )
BuildDataBaseIndex.buildDataBaseIndex( connection, force=False )
# TODO: flush events,
# TODO: recompute per segment of windowT.
currentT = minT
try:
flushEvents( connection )
while currentT < maxT:
currentMinT = currentT
currentMaxT = currentT+ windowT
if ( currentMaxT > maxT ):
currentMaxT = maxT
chronoTimeWindowFile = Chronometer("File "+ file+ " currentMinT: "+ str(currentMinT)+ " currentMaxT: " + str(currentMaxT) );
processTimeWindow( connection, currentMinT, currentMaxT )
chronoTimeWindowFile.printTimeInS()
currentT += windowT
print("Full file process time: ")
chronoFullFile.printTimeInS()
TEST_WINDOWING_COMPUTATION = False
if ( TEST_WINDOWING_COMPUTATION ):
print("*************")
print("************* TEST START SECTION")
print("************* Test if results are the same with or without the windowing.")
# display and record to a file all events found, checking with rolling idA from None to 4. Save nbEvent and total len
eventTimeLineList = []
eventList = getAllEvents( connection )
file = open("outEvent"+str(windowT)+".txt","w")
file.write( "Event name\nnb event\ntotal duration" )
for eventName in eventList:
for idAnimalA in range( 0,5 ):
idA = idAnimalA
if idA == 0:
idA = None
timeLine = EventTimeLineCached( connection, file, eventName, idA, minFrame=minT, maxFrame=maxT )
eventTimeLineList.append( timeLine )
file.write( timeLine.eventNameWithId+"\t"+str(len(timeLine.eventList))+"\t"+str(timeLine.getTotalLength())+"\n" )
file.close()
#plotMultipleTimeLine( eventTimeLineList )
print("************* END TEST")
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
error = ''.join('!! ' + line for line in lines)
t = TaskLogger( connection )
t.addLog( error )
print( error, file=sys.stderr )
raise FileProcessException()
def process( file ):
print(file)
mem = virtual_memory()
availableMemoryGB = mem.total / 1000000000
print( "Total memory on computer: (GB)", availableMemoryGB )
if availableMemoryGB < 10:
print( "Not enough memory to use cache load of events.")
disableEventTimeLineCache()
chronoFullFile = Chronometer("File " + file )
connection = sqlite3.connect( file )
# update missing fields
try:
connection = sqlite3.connect( file )
c = connection.cursor()
query = "ALTER TABLE EVENT ADD METADATA TEXT";
c.execute( query )
connection.commit()
except:
print( "METADATA field already exists" , file )
BuildDataBaseIndex.buildDataBaseIndex( connection, force=False )
# build sensor data
animalPool = AnimalPool( )
animalPool.loadAnimals( connection )
#animalPool.buildSensorData(file)
currentT = minT
try:
flushEvents( connection )
while currentT < maxT:
currentMinT = currentT
currentMaxT = currentT+ windowT
if ( currentMaxT > maxT ):
currentMaxT = maxT
chronoTimeWindowFile = Chronometer("File "+ file+ " currentMinT: "+ str(currentMinT)+ " currentMaxT: " + str(currentMaxT) );
processTimeWindow( connection, file, currentMinT, currentMaxT )
chronoTimeWindowFile.printTimeInS()
currentT += windowT
print("Full file process time: ")
chronoFullFile.printTimeInS()
TEST_WINDOWING_COMPUTATION = False
if ( TEST_WINDOWING_COMPUTATION ):
print("*************")
print("************* TEST START SECTION")
print("************* Test if results are the same with or without the windowing.")
# display and record to a file all events found, checking with rolling idA from None to 4. Save nbEvent and total len
eventTimeLineList = []
eventList = getAllEvents( connection )
file = open("outEvent"+str(windowT)+".txt","w")
file.write( "Event name\nnb event\ntotal duration" )
for eventName in eventList:
for animal in range( 0,5 ):
idA = animal
if idA == 0:
idA = None
timeLine = EventTimeLineCached( connection, file, eventName, idA, minFrame=minT, maxFrame=maxT )
eventTimeLineList.append( timeLine )
file.write( timeLine.eventNameWithId+"\t"+str(len(timeLine.eventList))+"\t"+str(timeLine.getTotalLength())+"\n" )
file.close()
#plotMultipleTimeLine( eventTimeLineList )
print("************* END TEST")
flushEventTimeLineCache()
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
error = ''.join('!! ' + line for line in lines)
t = TaskLogger( connection )
t.addLog( error )
flushEventTimeLineCache()
print( error, file=sys.stderr )
raise FileProcessException()
def computeProfilePairs(files, tmin, tmax, text_file, animalDic):
for file in files:
print(file)
connection = sqlite3.connect(file)
pool = AnimalPool()
pool.loadAnimals(connection)
genoList = []
rfidList = []
animalDic[file] = {}
for animal in pool.animalDictionnary.keys():
print("computing individual animal: {}".format(animal))
rfid = pool.animalDictionnary[animal].RFID
print("RFID: ".format(rfid))
animalDic[file][rfid] = {}
''' store the animal '''
# animalDic[rfid]["animal"] = pool.animalDictionnary[animal]
genoA = None
try:
genoA = pool.animalDictionnary[animal].genotype
except:
pass
animalDic[file][rfid][genoA] = {}
genoList.append(genoA)
rfidList.append(rfid)
COMPUTE_TOTAL_DISTANCE = True
if (COMPUTE_TOTAL_DISTANCE == True):
pool.animalDictionnary[animal].loadDetection(lightLoad=True)
animalDic[file][rfid][genoA][
'distance'] = pool.animalDictionnary[animal].getDistance(
tmin=tmin, tmax=tmax)
for behavEvent in behaviouralEvents:
print("computing individual event: {}".format(behavEvent))
behavEventTimeLine = EventTimeLineCached(connection,
file,
behavEvent,
animal,
minFrame=tmin,
maxFrame=tmax)
totalEventDuration = behavEventTimeLine.getTotalLength()
nbEvent = behavEventTimeLine.getNumberOfEvent(minFrame=tmin,
maxFrame=tmax)
print("total event duration: ", totalEventDuration)
animalDic[file][rfid][genoA][behavEventTimeLine.eventName +
" TotalLen"] = totalEventDuration
animalDic[file][rfid][genoA][behavEventTimeLine.eventName +
" Nb"] = nbEvent
print(behavEventTimeLine.eventName, genoA,
behavEventTimeLine.idA, totalEventDuration, nbEvent)
rfidPair = '{}-{}'.format(rfidList[0], rfidList[1])
animalDic[file][rfidPair] = {}
pairType = '{}-{}'.format(genoList[0], genoList[1])
animalDic[file][rfidPair][pairType] = {}
animalDic[file][rfidPair][pairType]['distance'] = 'NA'
for behavEvent in behaviouralEvents:
print("computing individual event: {}".format(behavEvent))
behavEventTimeLine = EventTimeLineCached(connection,
file,
behavEvent,
minFrame=tmin,
maxFrame=tmax)
totalEventDuration = behavEventTimeLine.getTotalLength()
nbEvent = behavEventTimeLine.getNumberOfEvent(minFrame=tmin,
maxFrame=tmax)
print("total event duration: ", totalEventDuration)
animalDic[file][rfidPair][pairType][
behavEventTimeLine.eventName +
" TotalLen"] = totalEventDuration
animalDic[file][rfidPair][pairType][behavEventTimeLine.eventName +
" Nb"] = nbEvent
print(behavEventTimeLine.eventName, pairType, totalEventDuration,
nbEvent)
print(animalDic)
print("writing...")
'''
file strain sex genotype group day exp idA idB minTime maxTime
'''
header = [
"file", "strain", "sex", "genotype", "group", "day", "exp", "RFID",
"minTime", "maxTime"
]
for name in header:
text_file.write("{}\t".format(name))
''' write event keys '''
firstAnimalKey = next(iter(animalDic[file]))
print('firstAnimal: ', firstAnimalKey)
genoFirstAnimal = list(animalDic[file][firstAnimalKey].keys())[0]
eventDic = animalDic[file][firstAnimalKey][genoFirstAnimal]
for k in eventDic.keys():
print('k: ', k)
text_file.write("{}\t".format(k.replace(" ", "")))
text_file.write("\n")
for rfidAnimal in animalDic[file].keys():
print(rfidAnimal)
animalId = rfidAnimal
animalGeno = list(animalDic[file][rfidAnimal].keys())[0]
text_file.write("{}\t".format(file))
text_file.write("{}\t".format("strain"))
text_file.write("{}\t".format("sex"))
text_file.write("{}\t".format(animalGeno))
text_file.write("{}\t".format("group"))
text_file.write("{}\t".format("day"))
text_file.write("{}\t".format("exp"))
text_file.write("{}\t".format(animalId))
text_file.write("{}\t".format(tmin))
text_file.write("{}\t".format(tmax))
for kEvent in animalDic[file][animalId][animalGeno].keys():
text_file.write("{}\t".format(
animalDic[file][animalId][animalGeno][kEvent]))
text_file.write("\n")
text_file.write("\n")
text_file.close()
# Create a json file to store the computation
jsonFileName = 'profile_unidentified_pairs.json'
with open(jsonFileName, 'w') as fp:
json.dump(animalDic, fp, indent=4)
print("json file with acoustic measurements created for ", jsonFileName)
print("done.")
return animalDic
print("computing {} ".format(behavEvent))
for animal in pool.animalDictionnary.keys():
for idAnimalB in pool.animalDictionnary.keys():
if (animal == idAnimalB):
continue
event = EventTimeLineCached(connection,
file,
behavEvent,
animal,
idAnimalB,
minFrame=minT,
maxFrame=maxT)
totalEventDuration = event.getTotalLength()
nbEvent = event.getNumberOfEvent(minFrame=minT,
maxFrame=maxT)
print(behavEvent, pool.animalDictionnary[animal].RFID,
pool.animalDictionnary[idAnimalB].RFID)
resSame = [
file, behavEvent,
pool.animalDictionnary[animal].RFID,
pool.animalDictionnary[idAnimalB].RFID, minT, maxT,
totalEventDuration, nbEvent
]
text_file.write(
"{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n".format(
def computeProfile(file, minT, maxT, night):
connection = sqlite3.connect( file )
pool = AnimalPool( )
pool.loadAnimals( connection )
animalData = {}
for animal in pool.animalDictionnary.keys():
print( "computing individual animal: {}".format( animal ))
rfid = pool.animalDictionnary[animal].RFID
print( "RFID: {}".format( rfid ) )
animalData[rfid]= {}
animalData[rfid]["animal"] = pool.animalDictionnary[animal]
animalData[rfid]["file"] = file
genoA = None
try:
genoA=pool.animalDictionnary[animal].genotype
except:
pass
for behavEvent in behaviouralEventOneMouse[:-2]:
print( "computing individual event: {}".format(behavEvent))
behavEventTimeLine = EventTimeLineCached( connection, file, behavEvent, animal, minFrame=minT, maxFrame=maxT )
totalEventDuration = behavEventTimeLine.getTotalLength()
nbEvent = behavEventTimeLine.getNumberOfEvent(minFrame = minT, maxFrame = maxT )
print( "total event duration: " , totalEventDuration )
animalData[rfid][behavEventTimeLine.eventName+" TotalLen"] = totalEventDuration
animalData[rfid][behavEventTimeLine.eventName+" Nb"] = nbEvent
print(behavEventTimeLine.eventName, genoA, behavEventTimeLine.idA, totalEventDuration, nbEvent)
header = ["file","strain","sex","group","day","exp","RFID","genotype", "user1", "minTime","maxTime"]
for name in header:
text_file.write( "{}\t".format ( name ) )
for kAnimal in animalData:
animalData[kAnimal]["experiment"] = file
COMPUTE_TOTAL_DISTANCE = True
if ( COMPUTE_TOTAL_DISTANCE == True ):
animalData[kAnimal]["animal"].loadDetection( start=minT, end=maxT, lightLoad = True )
animalData[kAnimal]["totalDistance"] = animalData[kAnimal]["animal"].getDistance( tmin=minT,tmax=maxT)/100
else:
animalData[kAnimal]["totalDistance"] = "totalDistance"
#write event keys
firstAnimalKey = next(iter(animalData))
firstAnimal = animalData[firstAnimalKey]
for k in firstAnimal.keys():
text_file.write( "{}\t".format( k.replace(" ", "") ) )
text_file.write("\n")
for kAnimal in animalData:
text_file.write( "{}\t".format( file ) )
text_file.write( "{}\t".format( "strain" ) )
text_file.write( "{}\t".format( "sex" ) )
text_file.write( "{}\t".format( night ) )
text_file.write( "{}\t".format( "exp" ) )
text_file.write( "{}\t".format( animalData[kAnimal]["animal"].RFID ) )
text_file.write( "{}\t".format( animalData[kAnimal]["animal"].genotype ) )
text_file.write( "{}\t".format( minT ) )
text_file.write( "{}\t".format( maxT ) )
for kEvent in firstAnimal.keys():
text_file.write( "{}\t".format( animalData[kAnimal][kEvent] ) )
text_file.write( "\n" )
return animalData
def computeProfilePerIndividual(file, minT, maxT, genoList, categoryList,
behaviouralEventListTwoMice):
connection = sqlite3.connect(file)
pool = AnimalPool()
pool.loadAnimals(connection)
indList = []
for animal in pool.animalDictionnary.keys():
print("computing individual animal: {}".format(animal))
rfid = pool.animalDictionnary[animal].RFID
indList.append(rfid)
sortedIndList = sorted(indList)
print('sorted list: ', sortedIndList)
groupName = sortedIndList[0]
for ind in sortedIndList[1:]:
print('ind: ', ind)
groupName + ind
animalData = {}
for animal in pool.animalDictionnary.keys():
print("computing individual animal: {}".format(animal))
rfid = pool.animalDictionnary[animal].RFID
print("RFID: {}".format(rfid))
animalData[rfid] = {}
#store the animal
animalData[rfid]["animal"] = pool.animalDictionnary[animal].name
animalObject = pool.animalDictionnary[animal]
animalData[rfid]["file"] = file
animalData[rfid]['genotype'] = pool.animalDictionnary[animal].genotype
animalData[rfid]['sex'] = pool.animalDictionnary[animal].sex
animalData[rfid]['group'] = groupName
animalData[rfid]['strain'] = pool.animalDictionnary[animal].strain
animalData[rfid]['age'] = pool.animalDictionnary[animal].age
for cat in categoryList:
for behavEvent in behaviouralEventListTwoMice:
animalData[rfid][behavEvent + cat] = {}
for geno in genoList:
animalData[rfid][behavEvent + cat][geno] = {}
genoA = None
try:
genoA = pool.animalDictionnary[animal].genotype
except:
pass
for behavEvent in behaviouralEventListTwoMice:
print("computing individual event: {}".format(behavEvent))
for idAnimalB in pool.animalDictionnary.keys():
if animal == idAnimalB:
continue
genoB = pool.animalDictionnary[idAnimalB].genotype
behavEventTimeLine = EventTimeLineCached(connection,
file,
behavEvent,
animal,
idAnimalB,
minFrame=minT,
maxFrame=maxT)
#clean the behavioural event timeline:
behavEventTimeLine.mergeCloseEvents(
numberOfFrameBetweenEvent=1)
behavEventTimeLine.removeEventsBelowLength(maxLen=3)
totalEventDuration = behavEventTimeLine.getTotalLength()
nbEvent = behavEventTimeLine.getNumberOfEvent(minFrame=minT,
maxFrame=maxT)
print("total event duration: ", totalEventDuration)
animalData[rfid][behavEventTimeLine.eventName +
" TotalLen"][genoB][pool.animalDictionnary[
idAnimalB].RFID] = totalEventDuration
animalData[rfid][behavEventTimeLine.eventName + " Nb"][genoB][
pool.animalDictionnary[idAnimalB].RFID] = nbEvent
if nbEvent == 0:
meanDur = 0
else:
meanDur = totalEventDuration / nbEvent
animalData[rfid][behavEventTimeLine.eventName + " MeanDur"][
genoB][pool.animalDictionnary[idAnimalB].RFID] = meanDur
print(behavEventTimeLine.eventName, genoA,
behavEventTimeLine.idA, genoB, behavEventTimeLine.idB,
totalEventDuration, nbEvent, meanDur)
connection.close()
return animalData