def _collectPositves(self, imgPath, segments):
"""Collect all positive scoring segments
Copy the images for all segments that score highter than > .5 to google drive folder
settings.positivePictures. These will be used to train future models.
Also, copy the full image for reference.
Args:
imgPath (str): path name for main image
segments (list): List of dictionary containing information on each segment
"""
positiveSegments = 0
ppath = pathlib.PurePath(imgPath)
googleDrive = self.google_services['drive']
for segmentInfo in segments:
if segmentInfo['score'] > .5:
if hasattr(settings, 'positivePicturesDir'):
pp = pathlib.PurePath(segmentInfo['imgPath'])
destPath = os.path.join(settings.positivePicturesDir,
pp.name)
shutil.copy(segmentInfo['imgPath'], destPath)
else:
goog_helper.uploadFile(googleDrive,
settings.positivePictures,
segmentInfo['imgPath'])
positiveSegments += 1
if positiveSegments > 0:
# Commenting out saving full images for now to reduce data
# goog_helper.uploadFile(googleDrive, settings.positivePictures, imgPath)
logging.warning('Found %d positives in image %s', positiveSegments,
ppath.name)
def uploadToDrive(service, imgPath, cameraID, imgClass):
parent = settings.IMG_CLASSES[imgClass]
dirName = ''
dirID = parent
if cameraID != None:
(dirID, dirName) = goog_helper.getDirForClassCamera(service, settings.IMG_CLASSES, imgClass, cameraID)
goog_helper.uploadFile(service, dirID, imgPath)
print('Uploaded file ', imgPath, ' to ', imgClass, dirName)
def collectPositves(service, imgPath, origImgPath, segments):
"""Collect all positive scoring segments
Copy the images for all segments that score highter than > .5 to google drive folder
settings.positivePictures. These will be used to train future models.
Also, copy the full image for reference.
Args:
imgPath (str): path name for main image
segments (list): List of dictionary containing information on each segment
"""
positiveSegments = 0
ppath = pathlib.PurePath(origImgPath)
imgNameNoExt = str(os.path.splitext(ppath.name)[0])
origImg = None
for segmentInfo in segments:
if segmentInfo['score'] > .5:
if imgPath != origImgPath:
if not origImg:
origImg = Image.open(origImgPath)
cropCoords = (segmentInfo['MinX'], segmentInfo['MinY'],
segmentInfo['MaxX'], segmentInfo['MaxY'])
croppedOrigImg = origImg.crop(cropCoords)
cropImgName = imgNameNoExt + '_Crop_' + 'x'.join(
list(map(lambda x: str(x), cropCoords))) + '.jpg'
cropImgPath = os.path.join(str(ppath.parent), cropImgName)
croppedOrigImg.save(cropImgPath, format='JPEG')
croppedOrigImg.close()
if hasattr(settings, 'positivePicturesDir'):
destPath = os.path.join(settings.positivePicturesDir,
cropImgName)
shutil.copy(cropImgPath, destPath)
else:
goog_helper.uploadFile(service, settings.positivePictures,
cropImgPath)
os.remove(cropImgPath)
if hasattr(settings, 'positivePicturesDir'):
pp = pathlib.PurePath(segmentInfo['imgPath'])
destPath = os.path.join(settings.positivePicturesDir, pp.name)
shutil.copy(segmentInfo['imgPath'], destPath)
else:
goog_helper.uploadFile(service, settings.positivePictures,
segmentInfo['imgPath'])
positiveSegments += 1
if positiveSegments > 0:
# Commenting out saving full images for now to reduce data
# goog_helper.uploadFile(service, settings.positivePictures, imgPath)
logging.warning('Found %d positives in image %s', positiveSegments,
ppath.name)
def _recordDetection(self, camera, timestamp, imgPath, annotatedFile,
fireSegment):
"""Record that a smoke/fire has been detected
Record the detection with useful metrics in 'detections' table in SQL DB.
Also, upload image file to google drive
Args:
camera (str): camera name
timestamp (int):
imgPath: filepath of the image
annotatedFile: filepath of the image with annotated box and score
fireSegment (dictionary): dictionary with information for the segment with fire/smoke
Returns:
List of Google drive IDs for the uploaded image files
"""
logging.warning('Fire detected by camera %s, image %s, segment %s',
camera, imgPath, str(fireSegment))
# upload file to google drive detection dir
driveFileIDs = []
googleDrive = self.google_services['drive']
driveFile = goog_helper.uploadFile(googleDrive,
settings.detectionPictures, imgPath)
if driveFile:
driveFileIDs.append(driveFile['id'])
driveFile = goog_helper.uploadFile(googleDrive,
settings.detectionPictures,
annotatedFile)
if driveFile:
driveFileIDs.append(driveFile['id'])
logging.warning('Uploaded to google drive detections folder %s',
str(driveFileIDs))
dbRow = {
'CameraName': camera,
'Timestamp': timestamp,
'MinX': fireSegment['MinX'],
'MinY': fireSegment['MinY'],
'MaxX': fireSegment['MaxX'],
'MaxY': fireSegment['MaxY'],
'Score': fireSegment['score'],
'HistAvg': fireSegment['HistAvg'],
'HistMax': fireSegment['HistMax'],
'HistNumSamples': fireSegment['HistNumSamples'],
'ImageID': driveFileIDs[0] if driveFileIDs else ''
}
self.dbManager.add_data('detections', dbRow)
return driveFileIDs
def main():
reqArgs = [
["d", "dirID", "ID of google drive directory"],
["f", "fileName", "fileName of google drive file"],
]
optArgs = [
["u", "upload", "(optional) performs upload vs. download"],
[
"s", "startTime",
"(optional) performs search with modifiedTime > startTime"
],
[
"e", "endTime",
"(optional) performs search with modifiedTime < endTime"
],
["l", "listOnly", "(optional) list vs. download"],
[
"r", "remove",
"(optional) performs remove/delete vs. download (value must be 'delete')"
],
[
"m", "maxFiles",
"override default of 100 for max number of files to operate on"
],
]
args = collect_args.collectArgs(
reqArgs,
optionalArgs=optArgs,
parentParsers=[goog_helper.getParentParser()])
maxFiles = int(args.maxFiles) if args.maxFiles else 100
googleServices = goog_helper.getGoogleServices(settings, args)
# default mode is to download a single file
operation = 'download'
multipleFiles = False
batchMode = True
MAX_BATCH_SIZE = 25 # increasing size beyond 60 tends to generate http 500 errors
if args.upload:
operation = 'upload'
elif args.remove:
if args.remove != 'delete':
logging.error(
"value for remove must be 'delete', but instead is %s",
args.remove)
exit(1)
operation = 'delete'
elif args.listOnly:
operation = 'list'
if args.startTime or args.endTime:
multipleFiles = True
if not multipleFiles:
if operation == 'upload':
goog_helper.uploadFile(googleServices['drive'], args.dirID,
args.fileName)
else:
assert operation == 'download'
goog_helper.downloadFile(googleServices['drive'], args.dirID,
args.fileName, args.fileName)
else:
nextPageToken = 'init'
processedFiles = 0
while True:
batch = None
batchCount = 0
(items,
nextPageToken) = goog_helper.searchFiles(googleServices['drive'],
args.dirID,
args.startTime,
args.endTime,
args.fileName,
npt=nextPageToken)
firstLast = ''
if len(items) > 0:
firstLast = str(items[0]) + ' to ' + str(items[-1])
logging.warning('Found %d files: %s', len(items), firstLast)
if operation == 'list':
logging.warning('All files: %s', items)
for item in items:
if operation == 'delete':
if batchMode:
if not batch:
batch = googleServices[
'drive'].new_batch_http_request(
callback=delete_file)
batch.add(googleServices['drive'].files().delete(
fileId=item["id"], supportsTeamDrives=True))
batchCount += 1
if batchCount == MAX_BATCH_SIZE:
logging.warning('Execute batch with %d items',
batchCount)
batch.execute()
batch = None
batchCount = 0
else:
googleServices['drive'].files().delete(
fileId=item["id"],
supportsTeamDrives=True).execute()
elif operation == 'download':
goog_helper.downloadFileByID(googleServices['drive'],
item['id'], item['name'])
if batch and batchCount:
batch.execute()
batch = None
processedFiles += len(items)
logging.warning('Processed %d of max %d. NextToken: %s',
processedFiles, maxFiles, bool(nextPageToken))
if (processedFiles >= maxFiles) or not nextPageToken:
break # exit if we processed enough files or no files left
logging.warning('Done')
