def __init__(self, logfile): self.lp = LP(logfile) self.fileManager = FM(projectID = self.lp.projectID) self.node = self.lp.uname.split("node='")[1].split("'")[0] self.lastFrameTime = self.lp.frames[-1].time self.masterDirectory = self.fileManager.localMasterDir self.projectDirectory = self.fileManager.localProjectDir self.credentialDrive = self.fileManager.localCredentialDrive self.credentialSpreadsheet = self.fileManager.localCredentialSpreadsheet self._createImage() f = self.uploadImage(self.projectDirectory + self.lp.tankID + '.jpg', self.lp.tankID) self.insertImage(f)
def runCommand(self, command, projectID): # This function is used to run a specific command found int he master Controller Google Spreadsheet self.projectID = projectID # Rename files to make code more readable self.projectDirectory = self.fileManager.localProjectDir self.loggerFile = self.fileManager.localLogfile self.frameDirectory = self.fileManager.localFrameDir self.videoDirectory = self.fileManager.localVideoDir self.backupDirectory = self.fileManager.localBackupDir if command not in self.commands: self._reinstructError(command + ' is not a valid command. Options are ' + str(self.commands)) if command == 'Stop': if self.piCamera: if self.camera.recording: self.camera.stop_recording() self._print('PiCameraStopped: Time: ' + str(datetime.datetime.now()) + ',,File: Videos/' + str(self.videoCounter).zfill(4) + "_vid.h264") command = [ 'python3', 'unit_scripts/process_video.py', self.videoDirectory + str(self.videoCounter).zfill(4) + '_vid.h264' ] command += [str(self.camera.framerate[0]), self.projectID] self._print(command) self.processes.append(subprocess.Popen(command)) try: if self.device == 'kinect2': self.K2device.stop() if self.device == 'kinect': freenect.sync_stop() freenect.shutdown(self.a) except: self._print('ErrorStopping kinect') self._closeFiles() self._modifyPiGS(command='None', status='AwaitingCommand') return if command == 'UploadData': self._modifyPiGS(command='None') self._uploadFiles() return if command == 'LocalDelete': if os.path.exists(self.projectDirectory): shutil.rmtree(self.projectDirectory) self._modifyPiGS(command='None', status='AwaitingCommand') return self._modifyPiGS(command='None', status='Running', error='') if command == 'New': # Project Directory should not exist. If it does, report error if os.path.exists(self.projectDirectory): self._reinstructError( 'New command cannot be run if ouput directory already exists. Use Rewrite or Restart' ) if command == 'Rewrite': if os.path.exists(self.projectDirectory): shutil.rmtree(self.projectDirectory) os.makedirs(self.projectDirectory) if command in ['New', 'Rewrite']: self.masterStart = datetime.datetime.now() if command == 'New': os.makedirs(self.projectDirectory) os.makedirs(self.frameDirectory) os.makedirs(self.videoDirectory) os.makedirs(self.backupDirectory) #self._createDropboxFolders() self.frameCounter = 1 self.videoCounter = 1 if command == 'Restart': logObj = LP(self.loggerFile) self.masterStart = logObj.master_start #self.r = logObj.bounding_shape self.frameCounter = logObj.lastFrameCounter + 1 self.videoCounter = logObj.lastVideoCounter + 1 if self.system != logObj.system or self.device != logObj.device or self.piCamera != logObj.camera: self._reinstructError( 'Restart error. System, device, or camera does not match what is in logfile' ) self.lf = open(self.loggerFile, 'a') self._modifyPiGS(start=str(self.masterStart)) if command in ['New', 'Rewrite']: self._print('MasterStart: System: ' + self.system + ',,Device: ' + self.device + ',,Camera: ' + str(self.piCamera) + ',,Uname: ' + str(platform.uname()) + ',,TankID: ' + self.tankID + ',,ProjectID: ' + self.projectID) self._print('MasterRecordInitialStart: Time: ' + str(self.masterStart)) self._print( 'PrepFiles: FirstDepth: PrepFiles/FirstDepth.npy,,LastDepth: PrepFiles/LastDepth.npy,,PiCameraRGB: PiCameraRGB.jpg,,DepthRGB: DepthRGB.jpg' ) picamera_settings = { 'AnalogGain': str(self.camera.analog_gain), 'AWB_Gains': str(self.camera.awb_gains), 'AWB_Mode': str(self.camera.awb_mode), 'Brightness': str(self.camera.brightness), 'ClockMode': str(self.camera.clock_mode), 'Contrast': str(self.camera.contrast), 'Crop': str(self.camera.crop), 'DigitalGain': str(self.camera.digital_gain), 'ExposureCompensation': str(self.camera.exposure_compensation), 'ExposureMode': str(self.camera.exposure_mode), 'ExposureSpeed': str(self.camera.exposure_speed), 'FrameRate': str(self.camera.framerate), 'ImageDenoise': str(self.camera.image_denoise), 'MeterMode': str(self.camera.meter_mode), 'RawFormat': str(self.camera.raw_format), 'Resolution': str(self.camera.resolution), 'Saturation': str(self.camera.saturation), 'SensorMode': str(self.camera.sensor_mode), 'Sharpness': str(self.camera.sharpness), 'ShutterSpeed': str(self.camera.shutter_speed), 'VideoDenoise': str(self.camera.video_denoise), 'VideoStabilization': str(self.camera.video_stabilization) } self._print('PiCameraSettings: ' + ',,'.join([ x + ': ' + picamera_settings[x] for x in sorted(picamera_settings.keys()) ])) #self._createROI(useROI = False) else: self._print('MasterRecordRestart: Time: ' + str(datetime.datetime.now())) # Start kinect self._start_kinect() # Diagnose speed self._diagnose_speed() # Capture data self.captureFrames()
def _uploadFiles(self): self._modifyPiGS(status='Finishing converting and uploading of videos') for p in self.processes: p.communicate() for movieFile in os.listdir(self.videoDirectory): if '.h264' in movieFile: command = [ 'python3', 'unit_scripts/process_video.py', self.videoDirectory + movieFile ] command += [str(self.camera.framerate[0]), self.projectID] self._print(command) self.processes.append(subprocess.Popen(command)) for p in self.processes: p.communicate() self._modifyPiGS(status='Creating prep files') # Move files around as appropriate prepDirectory = self.projectDirectory + 'PrepFiles/' shutil.rmtree(prepDirectory) if os.path.exists(prepDirectory) else None os.makedirs(prepDirectory) lp = LP(self.loggerFile) self.frameCounter = lp.lastFrameCounter + 1 videoObj = [ x for x in lp.movies if x.startTime.hour >= 8 and x.startTime.hour <= 20 ][0] subprocess.call([ 'cp', self.projectDirectory + videoObj.pic_file, prepDirectory + 'PiCameraRGB.jpg' ]) subprocess.call([ 'cp', self.projectDirectory + lp.movies[-1].pic_file, prepDirectory + 'LastPiCameraRGB.jpg' ]) # Find depthfile that is closest to the video file time depthObj = [x for x in lp.frames if x.time > videoObj.startTime][0] subprocess.call([ 'cp', self.projectDirectory + depthObj.pic_file, prepDirectory + 'DepthRGB.jpg' ]) if not os.path.isdir(self.frameDirectory): self._modifyPiGS(status='Error: ' + self.frameDirectory + ' does not exist.') return subprocess.call([ 'cp', self.frameDirectory + 'Frame_000001.npy', prepDirectory + 'FirstDepth.npy' ]) subprocess.call([ 'cp', self.frameDirectory + 'Frame_' + str(self.frameCounter - 1).zfill(6) + '.npy', prepDirectory + 'LastDepth.npy' ]) try: self._modifyPiGS(status='Uploading data to cloud') self.fileManager.uploadData(self.frameDirectory, tarred=True) #print(prepDirectory) self.fileManager.uploadData(prepDirectory) #print(self.videoDirectory) self.fileManager.uploadData(self.videoDirectory) #print(self.loggerFile) self.fileManager.uploadData(self.loggerFile) self._modifyPiGS(error='UploadSuccessful, ready for delete') except Exception as e: print('UploadError: ' + str(e)) self._modifyPiGS(error='UploadFailed, Need to rerun') raise Exception
def getProjectStates(self): # Dictionary to hold row of data row_data = { 'projectID': self.projectID, 'tankID': '', 'StartingFiles': False, 'Prep': False, 'Depth': False, 'Cluster': False, 'ClusterClassification': False, 'Summary': False } # List the files needed for each analysis necessaryFiles = {} necessaryFiles['StartingFiles'] = [ self.localLogfile, self.localPrepDir, self.localFrameTarredDir, self.localVideoDir, self.localFirstFrame, self.localLastFrame, self.localPiRGB, self.localDepthRGB ] necessaryFiles['Prep'] = [ self.localTrayFile, self.localTransMFile, self.localVideoCropFile ] necessaryFiles['Depth'] = [self.localSmoothDepthFile] necessaryFiles['Cluster'] = [ self.localAllClipsDir, self.localManualLabelClipsDir, self.localManualLabelFramesDir ] necessaryFiles['ClusterClassification'] = [ self.localAllLabeledClustersFile ] necessaryFiles['Summary'] = [ self.localSummaryDir, self.localAllLabeledClustersFile, self.localSmoothDepthFile, self.localTrayFile, self.localTransMFile, self.localLogfile ] print('Starting and downloading logfile for project ' + self.projectID) # Try to download and read logfile try: self.downloadData(self.localLogfile) except FileNotFoundError: print(' Cantfind Log File') row_data['StartingFiles'] = False pdb.set_trace() return row_data self.lp = LP(self.localLogfile) if self.lp.malformed_file: row_data['StartingFiles'] = False print(' Malformed Log File') pdb.set_trace() return row_data # Get additional files necessary for analysis based on videos for index, vid_obj in enumerate(self.lp.movies): vid_obj = self.returnVideoObject(index) necessaryFiles['StartingFiles'].append(vid_obj.localVideoFile) necessaryFiles['Cluster'].append(vid_obj.localLabeledClustersFile) necessaryFiles['Cluster'].append(vid_obj.localAllClipsDir[:-1] + '.tar') necessaryFiles['Cluster'].append( vid_obj.localManualLabelClipsDir[:-1] + '.tar') necessaryFiles['Cluster'].append( vid_obj.localManualLabelFramesDir[:-1] + '.tar') row_data['tankID'] = self.lp.tankID # Check if files exists print('Checking if individual files exist') directories = {} for analysis_type, analysis_files in necessaryFiles.items(): for analysis_file in analysis_files: directories[os.path.dirname( os.path.realpath(analysis_file))] = [] for local_path in directories: cloud_path = local_path.replace(self.localMasterDir, self.cloudMasterDir) output = subprocess.run(['rclone', 'lsf', cloud_path], capture_output=True, encoding='utf-8') directories[local_path] = [ x.rstrip('/') for x in output.stdout.split('\n') ] for analysis_type, local_files in necessaryFiles.items(): row_data[analysis_type] = all([ os.path.basename(x) in directories[os.path.dirname(os.path.realpath(x))] for x in local_files ]) return row_data
def __init__(self, projectID=None, modelID=None, summaryFile=None, rcloneRemote='cichlidVideo:', masterDir='McGrath/Apps/CichlidPiData/'): # Identify directory for temporary local files if platform.node() == 'raspberrypi' or 'Pi' in platform.node( ) or 'bt-' in platform.node() or 'sv-' in platform.node(): self._identifyPiDirectory() elif platform.node() == 'ebb-utaka.biosci.gatech.edu': self.localMasterDir = '/mnt/Storage/' + os.getenv( 'USER') + '/Temp/CichlidAnalyzer/' else: self.localMasterDir = os.getenv('HOME').rstrip( '/') + '/' + 'Temp/CichlidAnalyzer/' # Identify cloud directory for rclone self.rcloneRemote = rcloneRemote # On some computers, the first directory is McGrath, on others it's BioSci-McGrath. Use rclone to figure out which output = subprocess.run( ['rclone', 'lsf', self.rcloneRemote + masterDir], capture_output=True, encoding='utf-8') if output.stderr == '': self.cloudMasterDir = self.rcloneRemote + masterDir else: output = subprocess.run( ['rclone', 'lsf', self.rcloneRemote + 'BioSci-' + masterDir], capture_output=True, encoding='utf-8') if output.stderr == '': self.cloudMasterDir = self.rcloneRemote + 'BioSci-' + masterDir else: raise Exception('Cant find master directory (' + masterDir + ') in rclone remote (' + rcloneRemote + '') if projectID is not None: self.createProjectData(projectID) try: self.downloadData(self.localLogfile) self.lp = LP(self.localLogfile) except FileNotFoundError: pass self.modelID = modelID self.localMLDir = self.localMasterDir + '__MachineLearningModels/' if modelID is not None: print(modelID) self.createMLData(modelID) if summaryFile is not None: self.localAnalysisStatesDir = self.localMasterDir + '__AnalysisStates/' + summaryFile.split( '.')[0] + '/' self.localSummaryFile = self.localAnalysisStatesDir + summaryFile self.localEuthData = self.localAnalysisStatesDir + 'euthanization_data.csv' else: self.localEuthData = None # Create file names and parameters self.createPiData() self.createAnnotationData() self._createParameters()