def __init__(self, logfile):
self.lp = LP(logfile)
self.fileManager = FM(projectID = self.lp.projectID)
self.node = self.lp.uname.split("node='")[1].split("'")[0]
self.lastFrameTime = self.lp.frames[-1].time
self.masterDirectory = self.fileManager.localMasterDir
self.projectDirectory = self.fileManager.localProjectDir
self.credentialDrive = self.fileManager.localCredentialDrive
self.credentialSpreadsheet = self.fileManager.localCredentialSpreadsheet
self._createImage()
f = self.uploadImage(self.projectDirectory + self.lp.tankID + '.jpg', self.lp.tankID)
self.insertImage(f)
def runCommand(self, command, projectID):
# This function is used to run a specific command found int he master Controller Google Spreadsheet
self.projectID = projectID
# Rename files to make code more readable
self.projectDirectory = self.fileManager.localProjectDir
self.loggerFile = self.fileManager.localLogfile
self.frameDirectory = self.fileManager.localFrameDir
self.videoDirectory = self.fileManager.localVideoDir
self.backupDirectory = self.fileManager.localBackupDir
if command not in self.commands:
self._reinstructError(command +
' is not a valid command. Options are ' +
str(self.commands))
if command == 'Stop':
if self.piCamera:
if self.camera.recording:
self.camera.stop_recording()
self._print('PiCameraStopped: Time: ' +
str(datetime.datetime.now()) +
',,File: Videos/' +
str(self.videoCounter).zfill(4) + "_vid.h264")
command = [
'python3', 'unit_scripts/process_video.py',
self.videoDirectory + str(self.videoCounter).zfill(4) +
'_vid.h264'
]
command += [str(self.camera.framerate[0]), self.projectID]
self._print(command)
self.processes.append(subprocess.Popen(command))
try:
if self.device == 'kinect2':
self.K2device.stop()
if self.device == 'kinect':
freenect.sync_stop()
freenect.shutdown(self.a)
except:
self._print('ErrorStopping kinect')
self._closeFiles()
self._modifyPiGS(command='None', status='AwaitingCommand')
return
if command == 'UploadData':
self._modifyPiGS(command='None')
self._uploadFiles()
return
if command == 'LocalDelete':
if os.path.exists(self.projectDirectory):
shutil.rmtree(self.projectDirectory)
self._modifyPiGS(command='None', status='AwaitingCommand')
return
self._modifyPiGS(command='None', status='Running', error='')
if command == 'New':
# Project Directory should not exist. If it does, report error
if os.path.exists(self.projectDirectory):
self._reinstructError(
'New command cannot be run if ouput directory already exists. Use Rewrite or Restart'
)
if command == 'Rewrite':
if os.path.exists(self.projectDirectory):
shutil.rmtree(self.projectDirectory)
os.makedirs(self.projectDirectory)
if command in ['New', 'Rewrite']:
self.masterStart = datetime.datetime.now()
if command == 'New':
os.makedirs(self.projectDirectory)
os.makedirs(self.frameDirectory)
os.makedirs(self.videoDirectory)
os.makedirs(self.backupDirectory)
#self._createDropboxFolders()
self.frameCounter = 1
self.videoCounter = 1
if command == 'Restart':
logObj = LP(self.loggerFile)
self.masterStart = logObj.master_start
#self.r = logObj.bounding_shape
self.frameCounter = logObj.lastFrameCounter + 1
self.videoCounter = logObj.lastVideoCounter + 1
if self.system != logObj.system or self.device != logObj.device or self.piCamera != logObj.camera:
self._reinstructError(
'Restart error. System, device, or camera does not match what is in logfile'
)
self.lf = open(self.loggerFile, 'a')
self._modifyPiGS(start=str(self.masterStart))
if command in ['New', 'Rewrite']:
self._print('MasterStart: System: ' + self.system + ',,Device: ' +
self.device + ',,Camera: ' + str(self.piCamera) +
',,Uname: ' + str(platform.uname()) + ',,TankID: ' +
self.tankID + ',,ProjectID: ' + self.projectID)
self._print('MasterRecordInitialStart: Time: ' +
str(self.masterStart))
self._print(
'PrepFiles: FirstDepth: PrepFiles/FirstDepth.npy,,LastDepth: PrepFiles/LastDepth.npy,,PiCameraRGB: PiCameraRGB.jpg,,DepthRGB: DepthRGB.jpg'
)
picamera_settings = {
'AnalogGain': str(self.camera.analog_gain),
'AWB_Gains': str(self.camera.awb_gains),
'AWB_Mode': str(self.camera.awb_mode),
'Brightness': str(self.camera.brightness),
'ClockMode': str(self.camera.clock_mode),
'Contrast': str(self.camera.contrast),
'Crop': str(self.camera.crop),
'DigitalGain': str(self.camera.digital_gain),
'ExposureCompensation': str(self.camera.exposure_compensation),
'ExposureMode': str(self.camera.exposure_mode),
'ExposureSpeed': str(self.camera.exposure_speed),
'FrameRate': str(self.camera.framerate),
'ImageDenoise': str(self.camera.image_denoise),
'MeterMode': str(self.camera.meter_mode),
'RawFormat': str(self.camera.raw_format),
'Resolution': str(self.camera.resolution),
'Saturation': str(self.camera.saturation),
'SensorMode': str(self.camera.sensor_mode),
'Sharpness': str(self.camera.sharpness),
'ShutterSpeed': str(self.camera.shutter_speed),
'VideoDenoise': str(self.camera.video_denoise),
'VideoStabilization': str(self.camera.video_stabilization)
}
self._print('PiCameraSettings: ' + ',,'.join([
x + ': ' + picamera_settings[x]
for x in sorted(picamera_settings.keys())
]))
#self._createROI(useROI = False)
else:
self._print('MasterRecordRestart: Time: ' +
str(datetime.datetime.now()))
# Start kinect
self._start_kinect()
# Diagnose speed
self._diagnose_speed()
# Capture data
self.captureFrames()
def _uploadFiles(self):
self._modifyPiGS(status='Finishing converting and uploading of videos')
for p in self.processes:
p.communicate()
for movieFile in os.listdir(self.videoDirectory):
if '.h264' in movieFile:
command = [
'python3', 'unit_scripts/process_video.py',
self.videoDirectory + movieFile
]
command += [str(self.camera.framerate[0]), self.projectID]
self._print(command)
self.processes.append(subprocess.Popen(command))
for p in self.processes:
p.communicate()
self._modifyPiGS(status='Creating prep files')
# Move files around as appropriate
prepDirectory = self.projectDirectory + 'PrepFiles/'
shutil.rmtree(prepDirectory) if os.path.exists(prepDirectory) else None
os.makedirs(prepDirectory)
lp = LP(self.loggerFile)
self.frameCounter = lp.lastFrameCounter + 1
videoObj = [
x for x in lp.movies
if x.startTime.hour >= 8 and x.startTime.hour <= 20
][0]
subprocess.call([
'cp', self.projectDirectory + videoObj.pic_file,
prepDirectory + 'PiCameraRGB.jpg'
])
subprocess.call([
'cp', self.projectDirectory + lp.movies[-1].pic_file,
prepDirectory + 'LastPiCameraRGB.jpg'
])
# Find depthfile that is closest to the video file time
depthObj = [x for x in lp.frames if x.time > videoObj.startTime][0]
subprocess.call([
'cp', self.projectDirectory + depthObj.pic_file,
prepDirectory + 'DepthRGB.jpg'
])
if not os.path.isdir(self.frameDirectory):
self._modifyPiGS(status='Error: ' + self.frameDirectory +
' does not exist.')
return
subprocess.call([
'cp', self.frameDirectory + 'Frame_000001.npy',
prepDirectory + 'FirstDepth.npy'
])
subprocess.call([
'cp', self.frameDirectory + 'Frame_' +
str(self.frameCounter - 1).zfill(6) + '.npy',
prepDirectory + 'LastDepth.npy'
])
try:
self._modifyPiGS(status='Uploading data to cloud')
self.fileManager.uploadData(self.frameDirectory, tarred=True)
#print(prepDirectory)
self.fileManager.uploadData(prepDirectory)
#print(self.videoDirectory)
self.fileManager.uploadData(self.videoDirectory)
#print(self.loggerFile)
self.fileManager.uploadData(self.loggerFile)
self._modifyPiGS(error='UploadSuccessful, ready for delete')
except Exception as e:
print('UploadError: ' + str(e))
self._modifyPiGS(error='UploadFailed, Need to rerun')
raise Exception
def getProjectStates(self):
# Dictionary to hold row of data
row_data = {
'projectID': self.projectID,
'tankID': '',
'StartingFiles': False,
'Prep': False,
'Depth': False,
'Cluster': False,
'ClusterClassification': False,
'Summary': False
}
# List the files needed for each analysis
necessaryFiles = {}
necessaryFiles['StartingFiles'] = [
self.localLogfile, self.localPrepDir, self.localFrameTarredDir,
self.localVideoDir, self.localFirstFrame, self.localLastFrame,
self.localPiRGB, self.localDepthRGB
]
necessaryFiles['Prep'] = [
self.localTrayFile, self.localTransMFile, self.localVideoCropFile
]
necessaryFiles['Depth'] = [self.localSmoothDepthFile]
necessaryFiles['Cluster'] = [
self.localAllClipsDir, self.localManualLabelClipsDir,
self.localManualLabelFramesDir
]
necessaryFiles['ClusterClassification'] = [
self.localAllLabeledClustersFile
]
necessaryFiles['Summary'] = [
self.localSummaryDir, self.localAllLabeledClustersFile,
self.localSmoothDepthFile, self.localTrayFile,
self.localTransMFile, self.localLogfile
]
print('Starting and downloading logfile for project ' + self.projectID)
# Try to download and read logfile
try:
self.downloadData(self.localLogfile)
except FileNotFoundError:
print(' Cantfind Log File')
row_data['StartingFiles'] = False
pdb.set_trace()
return row_data
self.lp = LP(self.localLogfile)
if self.lp.malformed_file:
row_data['StartingFiles'] = False
print(' Malformed Log File')
pdb.set_trace()
return row_data
# Get additional files necessary for analysis based on videos
for index, vid_obj in enumerate(self.lp.movies):
vid_obj = self.returnVideoObject(index)
necessaryFiles['StartingFiles'].append(vid_obj.localVideoFile)
necessaryFiles['Cluster'].append(vid_obj.localLabeledClustersFile)
necessaryFiles['Cluster'].append(vid_obj.localAllClipsDir[:-1] +
'.tar')
necessaryFiles['Cluster'].append(
vid_obj.localManualLabelClipsDir[:-1] + '.tar')
necessaryFiles['Cluster'].append(
vid_obj.localManualLabelFramesDir[:-1] + '.tar')
row_data['tankID'] = self.lp.tankID
# Check if files exists
print('Checking if individual files exist')
directories = {}
for analysis_type, analysis_files in necessaryFiles.items():
for analysis_file in analysis_files:
directories[os.path.dirname(
os.path.realpath(analysis_file))] = []
for local_path in directories:
cloud_path = local_path.replace(self.localMasterDir,
self.cloudMasterDir)
output = subprocess.run(['rclone', 'lsf', cloud_path],
capture_output=True,
encoding='utf-8')
directories[local_path] = [
x.rstrip('/') for x in output.stdout.split('\n')
]
for analysis_type, local_files in necessaryFiles.items():
row_data[analysis_type] = all([
os.path.basename(x)
in directories[os.path.dirname(os.path.realpath(x))]
for x in local_files
])
return row_data
def __init__(self,
projectID=None,
modelID=None,
summaryFile=None,
rcloneRemote='cichlidVideo:',
masterDir='McGrath/Apps/CichlidPiData/'):
# Identify directory for temporary local files
if platform.node() == 'raspberrypi' or 'Pi' in platform.node(
) or 'bt-' in platform.node() or 'sv-' in platform.node():
self._identifyPiDirectory()
elif platform.node() == 'ebb-utaka.biosci.gatech.edu':
self.localMasterDir = '/mnt/Storage/' + os.getenv(
'USER') + '/Temp/CichlidAnalyzer/'
else:
self.localMasterDir = os.getenv('HOME').rstrip(
'/') + '/' + 'Temp/CichlidAnalyzer/'
# Identify cloud directory for rclone
self.rcloneRemote = rcloneRemote
# On some computers, the first directory is McGrath, on others it's BioSci-McGrath. Use rclone to figure out which
output = subprocess.run(
['rclone', 'lsf', self.rcloneRemote + masterDir],
capture_output=True,
encoding='utf-8')
if output.stderr == '':
self.cloudMasterDir = self.rcloneRemote + masterDir
else:
output = subprocess.run(
['rclone', 'lsf', self.rcloneRemote + 'BioSci-' + masterDir],
capture_output=True,
encoding='utf-8')
if output.stderr == '':
self.cloudMasterDir = self.rcloneRemote + 'BioSci-' + masterDir
else:
raise Exception('Cant find master directory (' + masterDir +
') in rclone remote (' + rcloneRemote + '')
if projectID is not None:
self.createProjectData(projectID)
try:
self.downloadData(self.localLogfile)
self.lp = LP(self.localLogfile)
except FileNotFoundError:
pass
self.modelID = modelID
self.localMLDir = self.localMasterDir + '__MachineLearningModels/'
if modelID is not None:
print(modelID)
self.createMLData(modelID)
if summaryFile is not None:
self.localAnalysisStatesDir = self.localMasterDir + '__AnalysisStates/' + summaryFile.split(
'.')[0] + '/'
self.localSummaryFile = self.localAnalysisStatesDir + summaryFile
self.localEuthData = self.localAnalysisStatesDir + 'euthanization_data.csv'
else:
self.localEuthData = None
# Create file names and parameters
self.createPiData()
self.createAnnotationData()
self._createParameters()