def main():
reqArgs = [
["o", "outputDir", "local directory to save images and segments"],
["i", "inputCsv", "csvfile with contents of Fuego Cropped Images"],
]
optArgs = [["s", "startRow", "starting row"],
["e", "endRow", "ending row"],
[
"d", "display",
"(optional) specify any value to display image and boxes"
]]
args = collect_args.collectArgs(
reqArgs,
optionalArgs=optArgs,
parentParsers=[goog_helper.getParentParser()])
startRow = int(args.startRow) if args.startRow else 0
endRow = int(args.endRow) if args.endRow else 1e9
googleServices = goog_helper.getGoogleServices(settings, args)
cameraCache = {}
with open(args.inputCsv) as csvFile:
csvreader = csv.reader(csvFile)
for (rowIndex, csvRow) in enumerate(csvreader):
if rowIndex < startRow:
continue
if rowIndex > endRow:
print('Reached end row', rowIndex, endRow)
exit(0)
[cropName, minX, minY, maxX, maxY, fileName] = csvRow[:6]
minX = int(minX)
minY = int(minY)
maxX = int(maxX)
maxY = int(maxY)
dirID = getCameraDir(googleServices['drive'], cameraCache,
fileName)
localFilePath = os.path.join(args.outputDir, fileName)
if not os.path.isfile(localFilePath):
goog_helper.downloadFile(googleServices['drive'], dirID,
fileName, localFilePath)
imgOrig = Image.open(localFilePath)
squareCoords = rect_to_squares.rect_to_squares(
minX, minY, maxX, maxY, imgOrig.size[0], imgOrig.size[1],
MIN_SIZE)
# print(squareCoords)
imgNameNoExt = str(os.path.splitext(fileName)[0])
for coords in squareCoords:
cropImgName = imgNameNoExt + '_Crop_' + 'x'.join(
list(map(lambda x: str(x), coords))) + '.jpg'
cropImgPath = os.path.join(args.outputDir, 'cropped',
cropImgName)
cropped_img = imgOrig.crop(coords)
cropped_img.save(cropImgPath, format='JPEG')
print('Processed row: %s, file: %s, num squares: %d' %
(rowIndex, fileName, len(squareCoords)))
if args.display:
squareCoords.append((minX, minY, maxX, maxY))
displayImageWithScores(imgOrig, squareCoords)
imageDisplay(imgOrig)
def main():
reqArgs = [
["o", "outputDir", "local directory to save images and segments"],
["i", "inputCsv", "csvfile with contents of Fuego Cropped Images"],
]
optArgs = [
["s", "startRow", "starting row"],
["e", "endRow", "ending row"],
[
"d", "display",
"(optional) specify any value to display image and boxes"
],
]
args = collect_args.collectArgs(
reqArgs,
optionalArgs=optArgs,
parentParsers=[goog_helper.getParentParser()])
startRow = int(args.startRow) if args.startRow else 0
endRow = int(args.endRow) if args.endRow else 1e9
googleServices = goog_helper.getGoogleServices(settings, args)
cameraCache = {}
with open(args.inputCsv) as csvFile:
csvreader = csv.reader(csvFile)
for (rowIndex, csvRow) in enumerate(csvreader):
if rowIndex < startRow:
continue
if rowIndex > endRow:
logging.warning('Reached end row: %d, %d', rowIndex, endRow)
exit(0)
logging.warning('row %d: %s', rowIndex, csvRow[:2])
[cameraName, cropName] = csvRow[:2]
if not cameraName:
continue
fileName = re.sub('_Crop[^.]+', '',
cropName) # get back filename for whole image
dirID = getCameraDir(googleServices['drive'], cameraCache,
fileName)
localFilePath = os.path.join(args.outputDir, fileName)
if not os.path.isfile(localFilePath):
goog_helper.downloadFile(googleServices['drive'], dirID,
fileName, localFilePath)
logging.warning('local %s', fileName)
cropInfo = re.findall('_Crop_(\d+)x(\d+)x(\d+)x(\d+)', cropName)
if len(cropInfo) != 1:
logging.error('Failed to parse crop info %s, %s', cropName,
cropInfo)
exit(1)
cropInfo = list(map(lambda x: int(x), cropInfo[0]))
logging.warning('Dims: %s', cropInfo)
imgOrig = Image.open(localFilePath)
rect_to_squares.cutBoxesFixed(imgOrig, args.outputDir, fileName,
lambda x: checkCoords(x, cropInfo))
def main():
reqArgs = [
["d", "dirID", "ID of google drive directory"],
["f", "fileName", "fileName of google drive file"],
]
optArgs = [
["u", "upload", "(optional) performs upload vs. download"],
[
"s", "startTime",
"(optional) performs search with modifiedTime > startTime"
],
[
"e", "endTime",
"(optional) performs search with modifiedTime < endTime"
],
["l", "listOnly", "(optional) list vs. download"],
[
"r", "remove",
"(optional) performs remove/delete vs. download (value must be 'delete')"
],
[
"m", "maxFiles",
"override default of 100 for max number of files to operate on"
],
]
args = collect_args.collectArgs(
reqArgs,
optionalArgs=optArgs,
parentParsers=[goog_helper.getParentParser()])
maxFiles = int(args.maxFiles) if args.maxFiles else 100
googleServices = goog_helper.getGoogleServices(settings, args)
# default mode is to download a single file
operation = 'download'
multipleFiles = False
batchMode = True
MAX_BATCH_SIZE = 25 # increasing size beyond 60 tends to generate http 500 errors
if args.upload:
operation = 'upload'
elif args.remove:
if args.remove != 'delete':
logging.error(
"value for remove must be 'delete', but instead is %s",
args.remove)
exit(1)
operation = 'delete'
elif args.listOnly:
operation = 'list'
if args.startTime or args.endTime:
multipleFiles = True
if not multipleFiles:
if operation == 'upload':
goog_helper.uploadFile(googleServices['drive'], args.dirID,
args.fileName)
else:
assert operation == 'download'
goog_helper.downloadFile(googleServices['drive'], args.dirID,
args.fileName, args.fileName)
else:
nextPageToken = 'init'
processedFiles = 0
while True:
batch = None
batchCount = 0
(items,
nextPageToken) = goog_helper.searchFiles(googleServices['drive'],
args.dirID,
args.startTime,
args.endTime,
args.fileName,
npt=nextPageToken)
firstLast = ''
if len(items) > 0:
firstLast = str(items[0]) + ' to ' + str(items[-1])
logging.warning('Found %d files: %s', len(items), firstLast)
if operation == 'list':
logging.warning('All files: %s', items)
for item in items:
if operation == 'delete':
if batchMode:
if not batch:
batch = googleServices[
'drive'].new_batch_http_request(
callback=delete_file)
batch.add(googleServices['drive'].files().delete(
fileId=item["id"], supportsTeamDrives=True))
batchCount += 1
if batchCount == MAX_BATCH_SIZE:
logging.warning('Execute batch with %d items',
batchCount)
batch.execute()
batch = None
batchCount = 0
else:
googleServices['drive'].files().delete(
fileId=item["id"],
supportsTeamDrives=True).execute()
elif operation == 'download':
goog_helper.downloadFileByID(googleServices['drive'],
item['id'], item['name'])
if batch and batchCount:
batch.execute()
batch = None
processedFiles += len(items)
logging.warning('Processed %d of max %d. NextToken: %s',
processedFiles, maxFiles, bool(nextPageToken))
if (processedFiles >= maxFiles) or not nextPageToken:
break # exit if we processed enough files or no files left
logging.warning('Done')
def main():
reqArgs = [
["o", "outputDir", "local directory to save images segments"],
["i", "inputCsv", "csvfile with contents of Fuego Cropped Images"],
]
optArgs = [
["s", "startRow", "starting row"],
["e", "endRow", "ending row"],
["d", "display", "(optional) specify any value to display image and boxes"],
["x", "minDiffX", "(optional) override default minDiffX of 299"],
["y", "minDiffY", "(optional) override default minDiffY of 299"],
["a", "minArea", "(optional) override default throw away areas < 1/100 of 299x299"],
["t", "throwSize", "(optional) override default throw away size of 1000x1000"],
["g", "growRatio", "(optional) override default grow ratio of 1.2"],
["m", "minusMinutes", "(optional) subtract images from given number of minutes ago"],
]
args = collect_args.collectArgs(reqArgs, optionalArgs=optArgs, parentParsers=[goog_helper.getParentParser()])
startRow = int(args.startRow) if args.startRow else 0
endRow = int(args.endRow) if args.endRow else 1e9
minDiffX = int(args.minDiffX) if args.minDiffX else 299
minDiffY = int(args.minDiffY) if args.minDiffY else 299
throwSize = int(args.throwSize) if args.throwSize else 1000
growRatio = float(args.growRatio) if args.growRatio else 1.2
minArea = int(args.minArea) if args.minArea else int(299*2.99)
minusMinutes = int(args.minusMinutes) if args.minusMinutes else 0
googleServices = goog_helper.getGoogleServices(settings, args)
camArchives = img_archive.getHpwrenCameraArchives(googleServices['sheet'], settings)
if minusMinutes:
timeGapDelta = datetime.timedelta(seconds = 60*minusMinutes)
cameraCache = {}
skippedTiny = []
skippedHuge = []
skippedArchive = []
with open(args.inputCsv) as csvFile:
csvreader = csv.reader(csvFile)
for (rowIndex, csvRow) in enumerate(csvreader):
if rowIndex < startRow:
continue
if rowIndex > endRow:
print('Reached end row', rowIndex, endRow)
break
[cropName, minX, minY, maxX, maxY, fileName] = csvRow[:6]
minX = int(minX)
minY = int(minY)
maxX = int(maxX)
maxY = int(maxY)
oldCoords = (minX, minY, maxX, maxY)
if ((maxX - minX) > throwSize) and ((maxY - minY) > throwSize):
logging.warning('Skip large image: dx=%d, dy=%d, name=%s', maxX - minX, maxY - minY, fileName)
skippedHuge.append((rowIndex, fileName, maxX - minX, maxY - minY))
continue
if ((maxX - minX) * (maxY - minY)) < minArea:
logging.warning('Skipping tiny image with area: %d, name=%s', (maxX - minX) * (maxY - minY), fileName)
skippedTiny.append((rowIndex, fileName, (maxX - minX) * (maxY - minY)))
continue
# get base image from google drive that was uploaded by sort_images.py
dirID = getCameraDir(googleServices['drive'], cameraCache, fileName)#-##REPLACE DEP. GDRIVE W HPREWN#
localFilePath = os.path.join(settings.downloadDir, fileName)#sets a path for that image() not yet downloadedby this iteration
print('local', localFilePath)
if not os.path.isfile(localFilePath):# if file has not been downloaded by a previous iteration
print('download', fileName)
#+##REPLACE DEP. GDRIVE W HPREWN#nameParsed = img_archive.parseFilename(fileName)#parses file name into dictionary of parts name,unixtime,etc.
#+##REPLACE DEP. GDRIVE W HPREWN#matchingCams = list(filter(lambda x: nameParsed['cameraID'] == x['id'], camArchives))#filter through camArchives for ids matching cameraid
#+##REPLACE DEP. GDRIVE W HPREWN#if len(matchingCams) != 1:#if we cannot determine where the image will come from we cannot use the image
#+##REPLACE DEP. GDRIVE W HPREWN# logging.warning('Skipping camera without archive: %d, %s', len(matchingCams), str(matchingCams))
#+##REPLACE DEP. GDRIVE W HPREWN# skippedArchive.append((rowIndex, fileName, matchingCams))
#+##REPLACE DEP. GDRIVE W HPREWN# continue
#+##REPLACE DEP. GDRIVE W HPREWN#archiveDirs = matchingCams[0]['dirs']
#+##REPLACE DEP. GDRIVE W HPREWN#logging.warning('Found %s directories', archiveDirs)
#+##REPLACE DEP. GDRIVE W HPREWN#time = datetime.datetime.fromtimestamp(nameParsed['unixTime'])
#+##REPLACE DEP. GDRIVE W HPREWN#for dirName in archiveDirs:#search directories of camera for a time near
#+##REPLACE DEP. GDRIVE W HPREWN# logging.warning('Searching for files in dir %s', dirName)
#+##REPLACE DEP. GDRIVE W HPREWN# imgPaths = img_archive.downloadFilesHpwren(settings.downloadDir, nameParsed['cameraID'], dirName, time, time, 1, 0)
#+##REPLACE DEP. GDRIVE W HPREWN# if imgPaths:
#+##REPLACE DEP. GDRIVE W HPREWN# localFilePath = imgPaths[0]
#+##REPLACE DEP. GDRIVE W HPREWN# break
#+##REPLACE DEP. GDRIVE W HPREWN#if not imgPaths:
#+##REPLACE DEP. GDRIVE W HPREWN# logging.warning('Skipping image not found: %s', fileName)
#+##REPLACE DEP. GDRIVE W HPREWN# skippedArchive.append((rowIndex, fileName, time))#archive that images were skipped
#+##REPLACE DEP. GDRIVE W HPREWN# continue
goog_helper.downloadFile(googleServices['drive'], dirID, fileName, localFilePath)#-##REPLACE DEP. GDRIVE W HPREWN#
imgOrig = Image.open(localFilePath)#opens image
# if in subracted images mode, download an earlier image and subtract
if minusMinutes:
nameParsed = img_archive.parseFilename(fileName)#parses file name into dictionary of parts name,unixtime,etc.
dt = datetime.datetime.fromtimestamp(nameParsed['unixTime'])
dt -= timeGapDelta
earlierImgPath = None
files = img_archive.getHpwrenImages(googleServices, settings, settings.downloadDir, camArchives, nameParsed['cameraID'], dt, dt, 1)
if files:
earlierImgPath = files[0]
else:
logging.warning('Skipping image without prior image: %s, %s', str(dt), fileName)
skippedArchive.append((rowIndex, fileName, dt))
continue
logging.warning('Subtracting old image %s', earlierImgPath)
earlierImg = Image.open(earlierImgPath)
diffImg = img_archive.diffImages(imgOrig, earlierImg)
# realImgOrig = imgOrig # is this useful?
imgOrig = diffImg
fileNameParts = os.path.splitext(fileName)
fileName = str(fileNameParts[0]) + ('_Diff%d' % minusMinutes) + fileNameParts[1]
# crop the full sized image to show just the smoke, but shifted and flipped
# shifts and flips increase number of segments for training and also prevent overfitting by perturbing data
cropCoords = getCropCoords((minX, minY, maxX, maxY), minDiffX, minDiffY, growRatio, (imgOrig.size[0], imgOrig.size[1]))
for newCoords in cropCoords:
# XXXX - save work if old=new?
print('coords old,new', oldCoords, newCoords)
imgNameNoExt = str(os.path.splitext(fileName)[0])
cropImgName = imgNameNoExt + '_Crop_' + 'x'.join(list(map(lambda x: str(x), newCoords))) + '.jpg'
cropImgPath = os.path.join(args.outputDir, cropImgName)
cropped_img = imgOrig.crop(newCoords)
cropped_img.save(cropImgPath, format='JPEG')
flipped_img = cropped_img.transpose(Image.FLIP_LEFT_RIGHT)
flipImgName = imgNameNoExt + '_Crop_' + 'x'.join(list(map(lambda x: str(x), newCoords))) + '_Flip.jpg'
flipImgPath = os.path.join(args.outputDir, flipImgName)
flipped_img.save(flipImgPath, format='JPEG')
print('Processed row: %s, file: %s' % (rowIndex, fileName))
if args.display:
displayCoords = [oldCoords] + cropCoords
displayImageWithScores(imgOrig, displayCoords)
imageDisplay(imgOrig)
logging.warning('Skipped tiny images %d, %s', len(skippedTiny), str(skippedTiny))
logging.warning('Skipped huge images %d, %s', len(skippedHuge), str(skippedHuge))
logging.warning('Skipped images without archives %d, %s', len(skippedArchive), str(skippedArchive))