def _collectPositves(self, imgPath, segments): """Collect all positive scoring segments Copy the images for all segments that score highter than > .5 to google drive folder settings.positivePictures. These will be used to train future models. Also, copy the full image for reference. Args: imgPath (str): path name for main image segments (list): List of dictionary containing information on each segment """ positiveSegments = 0 ppath = pathlib.PurePath(imgPath) googleDrive = self.google_services['drive'] for segmentInfo in segments: if segmentInfo['score'] > .5: if hasattr(settings, 'positivePicturesDir'): pp = pathlib.PurePath(segmentInfo['imgPath']) destPath = os.path.join(settings.positivePicturesDir, pp.name) shutil.copy(segmentInfo['imgPath'], destPath) else: goog_helper.uploadFile(googleDrive, settings.positivePictures, segmentInfo['imgPath']) positiveSegments += 1 if positiveSegments > 0: # Commenting out saving full images for now to reduce data # goog_helper.uploadFile(googleDrive, settings.positivePictures, imgPath) logging.warning('Found %d positives in image %s', positiveSegments, ppath.name)
def uploadToDrive(service, imgPath, cameraID, imgClass): parent = settings.IMG_CLASSES[imgClass] dirName = '' dirID = parent if cameraID != None: (dirID, dirName) = goog_helper.getDirForClassCamera(service, settings.IMG_CLASSES, imgClass, cameraID) goog_helper.uploadFile(service, dirID, imgPath) print('Uploaded file ', imgPath, ' to ', imgClass, dirName)
def collectPositves(service, imgPath, origImgPath, segments): """Collect all positive scoring segments Copy the images for all segments that score highter than > .5 to google drive folder settings.positivePictures. These will be used to train future models. Also, copy the full image for reference. Args: imgPath (str): path name for main image segments (list): List of dictionary containing information on each segment """ positiveSegments = 0 ppath = pathlib.PurePath(origImgPath) imgNameNoExt = str(os.path.splitext(ppath.name)[0]) origImg = None for segmentInfo in segments: if segmentInfo['score'] > .5: if imgPath != origImgPath: if not origImg: origImg = Image.open(origImgPath) cropCoords = (segmentInfo['MinX'], segmentInfo['MinY'], segmentInfo['MaxX'], segmentInfo['MaxY']) croppedOrigImg = origImg.crop(cropCoords) cropImgName = imgNameNoExt + '_Crop_' + 'x'.join( list(map(lambda x: str(x), cropCoords))) + '.jpg' cropImgPath = os.path.join(str(ppath.parent), cropImgName) croppedOrigImg.save(cropImgPath, format='JPEG') croppedOrigImg.close() if hasattr(settings, 'positivePicturesDir'): destPath = os.path.join(settings.positivePicturesDir, cropImgName) shutil.copy(cropImgPath, destPath) else: goog_helper.uploadFile(service, settings.positivePictures, cropImgPath) os.remove(cropImgPath) if hasattr(settings, 'positivePicturesDir'): pp = pathlib.PurePath(segmentInfo['imgPath']) destPath = os.path.join(settings.positivePicturesDir, pp.name) shutil.copy(segmentInfo['imgPath'], destPath) else: goog_helper.uploadFile(service, settings.positivePictures, segmentInfo['imgPath']) positiveSegments += 1 if positiveSegments > 0: # Commenting out saving full images for now to reduce data # goog_helper.uploadFile(service, settings.positivePictures, imgPath) logging.warning('Found %d positives in image %s', positiveSegments, ppath.name)
def _recordDetection(self, camera, timestamp, imgPath, annotatedFile, fireSegment): """Record that a smoke/fire has been detected Record the detection with useful metrics in 'detections' table in SQL DB. Also, upload image file to google drive Args: camera (str): camera name timestamp (int): imgPath: filepath of the image annotatedFile: filepath of the image with annotated box and score fireSegment (dictionary): dictionary with information for the segment with fire/smoke Returns: List of Google drive IDs for the uploaded image files """ logging.warning('Fire detected by camera %s, image %s, segment %s', camera, imgPath, str(fireSegment)) # upload file to google drive detection dir driveFileIDs = [] googleDrive = self.google_services['drive'] driveFile = goog_helper.uploadFile(googleDrive, settings.detectionPictures, imgPath) if driveFile: driveFileIDs.append(driveFile['id']) driveFile = goog_helper.uploadFile(googleDrive, settings.detectionPictures, annotatedFile) if driveFile: driveFileIDs.append(driveFile['id']) logging.warning('Uploaded to google drive detections folder %s', str(driveFileIDs)) dbRow = { 'CameraName': camera, 'Timestamp': timestamp, 'MinX': fireSegment['MinX'], 'MinY': fireSegment['MinY'], 'MaxX': fireSegment['MaxX'], 'MaxY': fireSegment['MaxY'], 'Score': fireSegment['score'], 'HistAvg': fireSegment['HistAvg'], 'HistMax': fireSegment['HistMax'], 'HistNumSamples': fireSegment['HistNumSamples'], 'ImageID': driveFileIDs[0] if driveFileIDs else '' } self.dbManager.add_data('detections', dbRow) return driveFileIDs
def main(): reqArgs = [ ["d", "dirID", "ID of google drive directory"], ["f", "fileName", "fileName of google drive file"], ] optArgs = [ ["u", "upload", "(optional) performs upload vs. download"], [ "s", "startTime", "(optional) performs search with modifiedTime > startTime" ], [ "e", "endTime", "(optional) performs search with modifiedTime < endTime" ], ["l", "listOnly", "(optional) list vs. download"], [ "r", "remove", "(optional) performs remove/delete vs. download (value must be 'delete')" ], [ "m", "maxFiles", "override default of 100 for max number of files to operate on" ], ] args = collect_args.collectArgs( reqArgs, optionalArgs=optArgs, parentParsers=[goog_helper.getParentParser()]) maxFiles = int(args.maxFiles) if args.maxFiles else 100 googleServices = goog_helper.getGoogleServices(settings, args) # default mode is to download a single file operation = 'download' multipleFiles = False batchMode = True MAX_BATCH_SIZE = 25 # increasing size beyond 60 tends to generate http 500 errors if args.upload: operation = 'upload' elif args.remove: if args.remove != 'delete': logging.error( "value for remove must be 'delete', but instead is %s", args.remove) exit(1) operation = 'delete' elif args.listOnly: operation = 'list' if args.startTime or args.endTime: multipleFiles = True if not multipleFiles: if operation == 'upload': goog_helper.uploadFile(googleServices['drive'], args.dirID, args.fileName) else: assert operation == 'download' goog_helper.downloadFile(googleServices['drive'], args.dirID, args.fileName, args.fileName) else: nextPageToken = 'init' processedFiles = 0 while True: batch = None batchCount = 0 (items, nextPageToken) = goog_helper.searchFiles(googleServices['drive'], args.dirID, args.startTime, args.endTime, args.fileName, npt=nextPageToken) firstLast = '' if len(items) > 0: firstLast = str(items[0]) + ' to ' + str(items[-1]) logging.warning('Found %d files: %s', len(items), firstLast) if operation == 'list': logging.warning('All files: %s', items) for item in items: if operation == 'delete': if batchMode: if not batch: batch = googleServices[ 'drive'].new_batch_http_request( callback=delete_file) batch.add(googleServices['drive'].files().delete( fileId=item["id"], supportsTeamDrives=True)) batchCount += 1 if batchCount == MAX_BATCH_SIZE: logging.warning('Execute batch with %d items', batchCount) batch.execute() batch = None batchCount = 0 else: googleServices['drive'].files().delete( fileId=item["id"], supportsTeamDrives=True).execute() elif operation == 'download': goog_helper.downloadFileByID(googleServices['drive'], item['id'], item['name']) if batch and batchCount: batch.execute() batch = None processedFiles += len(items) logging.warning('Processed %d of max %d. NextToken: %s', processedFiles, maxFiles, bool(nextPageToken)) if (processedFiles >= maxFiles) or not nextPageToken: break # exit if we processed enough files or no files left logging.warning('Done')