def load_and_run_detector(options, detector=None): imageFileNames = options_to_images(options) if options.forceCpu: os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ['CUDA_VISIBLE_DEVICES'] = '-1' print('Running detector on {} images'.format(len(imageFileNames))) if len(imageFileNames) == 0: print('Warning: no files available') return # Load detector if necessary if detector is None: startTime = time.time() print('Loading model...') detector = TFDetector(options.detectorFile) elapsed = time.time() - startTime print("Loaded model in {}".format( humanfriendly.format_timespan(elapsed))) # Run detector on target images boxes, scores, classes, imageFileNames = generate_detections( detector, imageFileNames, options) assert len(boxes) == len(imageFileNames) print('Writing output...') df = detector_output_to_api_output(imageFileNames, options, boxes, scores, classes) # PERF: iterrows is the wrong way to do this for large files if options.outputPathReplacements is not None: for iRow, row in df.iterrows(): for query in options.outputPathReplacements: replacement = options.outputPathReplacements[query] row['image_path'] = row['image_path'].replace( query, replacement) if options.outputRelativeFilenames and os.path.isdir(options.imageFile): for iRow, row in df.iterrows(): row['image_path'] = os.path.relpath(row['image_path'], options.imageFile) # While we're in transition between formats, write out the old format and # convert to the new format if options.outputFile.endswith('.csv'): write_api_results(df, options.outputFile) else: tempfilename = next(tempfile._get_candidate_names()) + '.csv' write_api_results(df, tempfilename) convert_output_format.convert_csv_to_json(tempfilename, options.outputFile) os.remove(tempfilename) return boxes, scores, classes, imageFileNames
def update_detection_table(RepeatDetectionResults, options, outputFilename=None): detectionResults = RepeatDetectionResults.detectionResults # An array of length nDirs, where each element is a list of DetectionLocation # objects for that directory that have been flagged as suspicious suspiciousDetectionsByDirectory = RepeatDetectionResults.suspiciousDetections nBboxChanges = 0 print('Updating output table') # For each suspicious detection (two loops) for iDir, directoryEvents in enumerate(suspiciousDetectionsByDirectory): for iDetectionEvent, detectionEvent in enumerate(directoryEvents): locationBbox = detectionEvent.bbox for iInstance, instance in enumerate(detectionEvent.instances): instanceBbox = instance.bbox # This should match the bbox for the detection event iou = ct_utils.get_iou(instanceBbox, locationBbox) # There are instances where iou is very close to the threshold so cannot use > assert iou >= options.iouThreshold assert instance.filename in RepeatDetectionResults.filenameToRow iRow = RepeatDetectionResults.filenameToRow[instance.filename] row = detectionResults.iloc[iRow] rowDetections = row['detections'] detectionToModify = rowDetections[instance.iDetection] # Make sure the bounding box matches assert (instanceBbox[0:3] == detectionToModify['bbox'][0:3]) # Make the probability negative, if it hasn't been switched by # another bounding box if detectionToModify['conf'] >= 0: detectionToModify['conf'] = -1 * detectionToModify['conf'] nBboxChanges += 1 # ...for each instance # ...for each detection # ...for each directory # Update maximum probabilities # For each row... nProbChanges = 0 nProbChangesToNegative = 0 nProbChangesAcrossThreshold = 0 for iRow, row in detectionResults.iterrows(): detections = row['detections'] if len(detections) == 0: continue maxPOriginal = float(row['max_detection_conf']) assert maxPOriginal >= 0 maxP = None nNegative = 0 for iDetection, detection in enumerate(detections): p = detection['conf'] if p < 0: nNegative += 1 if (maxP is None) or (p > maxP): maxP = p if abs(maxP - maxPOriginal) > 0.00000001: # We should only be making detections *less* likely assert maxP < maxPOriginal # row['max_confidence'] = str(maxP) detectionResults.at[iRow, 'max_detection_conf'] = maxP nProbChanges += 1 if maxP < 0: nProbChangesToNegative += 1 if maxPOriginal >= options.confidenceMin and maxP < options.confidenceMin: nProbChangesAcrossThreshold += 1 # Negative probabilities should be the only reason maxP changed, so # we should have found at least one negative value assert nNegative > 0 # ...if there was a change to the max probability for this row # ...for each row if outputFilename is not None: write_api_results(detectionResults, RepeatDetectionResults.otherFields, outputFilename) print( 'Finished updating detection table\nChanged {} detections that impacted {} maxPs ({} to negative) ({} across confidence threshold)' .format(nBboxChanges, nProbChanges, nProbChangesToNegative, nProbChangesAcrossThreshold)) return detectionResults
def update_detection_table(RepeatDetectionResults, options, outputFilename=None): detectionResults = RepeatDetectionResults.detectionResults # An array of length nDirs, where each element is a list of DetectionLocation # objects for that directory that have been flagged as suspicious suspiciousDetectionsByDirectory = RepeatDetectionResults.suspiciousDetections nBboxChanges = 0 print('Updating output table') # For each directory for iDir, directoryEvents in enumerate(suspiciousDetectionsByDirectory): # For each suspicious detection group in this directory for iDetectionEvent, detectionEvent in enumerate(directoryEvents): locationBbox = detectionEvent.bbox # For each instance of this suspicious detection for iInstance, instance in enumerate(detectionEvent.instances): instanceBbox = instance.bbox # This should match the bbox for the detection event iou = ct_utils.get_iou(instanceBbox, locationBbox) # The bbox for this instance should be almost the same as the bbox # for this detection group, where "almost" is defined by the IOU # threshold. assert iou >= options.iouThreshold # if iou < options.iouThreshold: # print('IOU warning: {},{}'.format(iou,options.iouThreshold)) assert instance.filename in RepeatDetectionResults.filenameToRow iRow = RepeatDetectionResults.filenameToRow[instance.filename] row = detectionResults.iloc[iRow] rowDetections = row['detections'] detectionToModify = rowDetections[instance.iDetection] # Make sure the bounding box matches assert (instanceBbox[0:3] == detectionToModify['bbox'][0:3]) # Make the probability negative, if it hasn't been switched by # another bounding box if detectionToModify['conf'] >= 0: detectionToModify['conf'] = -1 * detectionToModify['conf'] nBboxChanges += 1 # ...for each instance # ...for each detection # ...for each directory # Update maximum probabilities # For each row... nProbChanges = 0 nProbChangesToNegative = 0 nProbChangesAcrossThreshold = 0 for iRow, row in detectionResults.iterrows(): detections = row['detections'] if isinstance(detections, float): assert isinstance(row['failure'], str) continue if len(detections) == 0: continue maxPOriginal = float(row['max_detection_conf']) # No longer strictly true; sometimes I run RDE on RDE output # assert maxPOriginal >= 0 assert maxPOriginal >= -1.0 maxP = None nNegative = 0 for iDetection, detection in enumerate(detections): p = detection['conf'] if p < 0: nNegative += 1 if (maxP is None) or (p > maxP): maxP = p if abs(maxP - maxPOriginal) > 1e-3: # We should only be making detections *less* likely assert maxP < maxPOriginal # row['max_confidence'] = str(maxP) detectionResults.at[iRow, 'max_detection_conf'] = maxP nProbChanges += 1 if (maxP < 0) and (maxPOriginal >= 0): nProbChangesToNegative += 1 if (maxPOriginal >= options.confidenceMin) and (maxP < options.confidenceMin): nProbChangesAcrossThreshold += 1 # Negative probabilities should be the only reason maxP changed, so # we should have found at least one negative value assert nNegative > 0 # ...if there was a meaningful change to the max probability for this row # ...for each row # If we're also writing output... if outputFilename is not None and len(outputFilename) > 0: write_api_results(detectionResults, RepeatDetectionResults.otherFields, outputFilename) print( 'Finished updating detection table\nChanged {} detections that impacted {} maxPs ({} to negative) ({} across confidence threshold)' .format(nBboxChanges, nProbChanges, nProbChangesToNegative, nProbChangesAcrossThreshold)) return detectionResults