def store_data(self): rowtitle = [ "Product Name", "Price", "Location", "Add post date", "Buy link" ] w = DataWriter('OLX.csv', rowtitle) w.writer(self.details)
def __init__(self, name="datawriterhdf5"): """ Constructor. """ DataWriter.__init__(self, name) ModuleDataWriterHDF5.__init__(self) return
def __init__(self, name="datawritervtk"): """ Constructor. """ DataWriter.__init__(self, name) ModuleDataWriterVTK.__init__(self) return
def _configure(self): """ Configure object. """ try: DataWriter._configure(self) except ValueError, err: aliases = ", ".join(self.aliases) raise ValueError("Error while configuring VTK output " "(%s):\n%s" % (aliases, err.message))
def initialize(self, normalizer): """ Initialize writer. """ DataWriter.initialize(self, normalizer, self.filename) timeScale = normalizer.timeScale() ModuleDataWriterHDF5.filename(self, self.filename) ModuleDataWriterHDF5.timeScale(self, timeScale.value) return
def initialize(self, normalizer): """ Initialize writer. """ DataWriter.initialize(self, normalizer, self.filename) timeScale = normalizer.timeScale() timeConstantN = normalizer.nondimensionalize(self.timeConstant, timeScale) ModuleDataWriterVTK.filename(self, self.filename) ModuleDataWriterVTK.timeScale(self, timeScale.value) ModuleDataWriterVTK.timeFormat(self, self.timeFormat) ModuleDataWriterVTK.timeConstant(self, timeConstantN) ModuleDataWriterVTK.precision(self, self.precision) return
def main(): if len(sys.argv) != 4: print 'Missing file operands!' print 'PerfDataViewer.py [program] [input_data_file] [output_data_file]' return program = str(sys.argv[1]) inFileName = str(sys.argv[2]) outFileName = str(sys.argv[3]) #TODO: move these to configuration files eventsList = ['0x149','0x151','0x2a2','0x126','0x227','0x224','0x8a2','0x1b0','0x20f0','0x2f1','0x1f2','0x1b8','0x2b8','0x4b8','0x40cb'] ppc_eventsList = ['0x3c046','0x2c048','0x2f080','0x26080','0x30881','0x26182','0x26880','0xd0a2','0xd0a0'] arffHeader = ['@relation function_level_badfs_badma_good\n',\ '@attribute r0149 numeric\n','@attribute r0151 numeric\n','@attribute r02a2 numeric\n','@attribute r0126 numeric\n',\ '@attribute r0227 numeric\n','@attribute r0224 numeric\n','@attribute r08a2 numeric\n','@attribute r01b0 numeric\n',\ '@attribute r20f0 numeric\n','@attribute r02f1 numeric\n','@attribute r01f2 numeric\n','@attribute r01b8 numeric\n',\ '@attribute r02b8 numeric\n','@attribute r04b8 numeric\n','@attribute r40cb numeric\n','@attribute status {good, badfs, badma}\n',\ '@data\n'] ppc_arffHeader = ['@relation function_level_badfs_badma_good\n',\ '@attribute r3c046 numeric\n', '@attribute r2c048 numeric\n', '@attribute r2f080 numeric\n',\ '@attribute r26080 numeric\n', '@attribute r30881 numeric\n', '@attribute r26182 numeric\n',\ '@attribute r26880 numeric\n', '@attribute rd0a2 numeric\n', '@attribute rd0a0 numeric\n',\ '@attribute status {good, badfs, badma}\n', '@data\n'] perfData = PerfData() perfFileReader = FileReader(inFileName) dataParser = DataParser(perfFileReader, perfData, program) eventsHolder = EventsHolder(eventsList) eventsHolder.setInstructionCountRawEvent('0xc0') arffWriter = ArffWriter(outFileName,arffHeader) dataWriter = DataWriter(arffWriter, perfData, eventsHolder) dataParser.parse() print(perfData.getDataStore()) dataWriter.writeToArffFile() print outFileName + ' file was created successfully.'
def test_write_to_json(): """Tests the write_to_json function of the DataWriter, which should be able to take in an dictionary and output file path, and write that dictionary to the specified output file. Returns ------- None """ dr = DataReader("test_data1.csv") hrm = HRM_Processor(dr) dw = DataWriter(hrm) metrics = {"test": 5, "another word": 18.5} output_file = "test.json" dw.write_to_json(metrics, output_file) with open(output_file) as infile: written_data = json.load(infile) assert written_data == metrics
def test_DataWriter_init_write_to_dict(): """Tests that the construction of a DataWriter object creates a .json file with the base file name that is the same as the file name of the .csv file passed into the original DataReader. Returns ------- None """ dr = DataReader("test_data1.csv") hrm = HRM_Processor(dr) dw = DataWriter(hrm) assert os.path.isfile("test_data1.json") os.remove("test_data1.json")
def test_DataWriter_init_valid_case(): """Tests the initialization of the DataWriter object, and that it successfully gets the output_dict from the input_dict and stores it as metrics. Returns ------- None """ dr = DataReader("test_data1.csv") hrm = HRM_Processor(dr) dw = DataWriter(hrm) assert dw.metrics == hrm.output_dict
def test_DataWriter_init_invalid_case(hrm): """Tests the initialization of the DataWriter object for a case where the data from the HRM_Processor object is not valid. Parameters ---------- hrm: HRM_Processor Generic HRM_Processor made from a DataReader with test_file.csv capsys: Pytest fixture A decorator for getting outputs printed to the console Returns ------- None """ hrm.isValid = False # Force invalidity for testing case with pytest.raises(ValueError): dw = DataWriter(hrm)
def main(): logging.basicConfig(filename="HRM_logs.txt", format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') file_name = get_file_name() wants_duration = get_wants_duration() try: if wants_duration: duration = get_duration() dr = DataReader(file_name, duration) else: dr = DataReader(file_name) hrm = HRM_Processor(dr) dw = DataWriter(hrm) except (FileNotFoundError, ValueError, TypeError): logging.info("Driver script terminated unsuccessfully.") logging.info("Successful termination of HRM_Driver")
def simulate_generations(self, num_generations, print_best): file_path = "csv/ESN_Results.csv" dw = DataWriter() dr = DataReader() dw.init_table(file_path) p = Predictor() mapping = dr.get_mapping() images = dr.get_images(112800, 28, 28) # 112800 images in data set scale_factor = 10 for i in range(num_generations): sum = 0 best_score = -100 best_accuracy = -100 best_net = [] engines = [] for net in self.networks: engine = [net, 0, 0] engines.append(engine) p.make_predictions(engines, mapping, images, scale_factor) for j in range(len(engines)): self.networks[j].fitness = engines[j][1] if engines[j][2] > best_score: best_score = engines[j][2] best_net = self.networks[j] if engines[j][1] > best_accuracy: best_accuracy = engines[j][1] avg_accuracy = self.avg_fitness(self.networks) # avg accuracy for j in range(len(engines)): self.networks[j].fitness = engines[j][2] # change fitness to score avg_score = self.avg_fitness(self.networks) # avg accuracy avg_size = self.avg_network_size() if print_best: best_net.show_net() print("-----------------------------------\t\t\t\t\t\t\n Generation " + str(i+1) + " results\n-----------------------------------\n", end='\n') print("Highest accuracy: " + str(best_accuracy*100) + "%\nHighest score: " + str(best_score**(1.0/scale_factor)) + "\nAverage accuracy: " + str(avg_accuracy*100) + "%\nAverage score: " + str(avg_score**(1.0/scale_factor)) + "\nNum species: " + str(len(self.species)) + "\nInnovs tried: " + str(self.networks[0].master_innov[0]) + "\nAverage connections per network: " + str(avg_size) + "\n") non_jit = self.construct_non_jit(best_net) pickle.dump(non_jit, open("neural_net.txt", "wb")) dw.write_row(file_path, [i+1, best_accuracy*100, avg_accuracy*100, best_score**(1.0/scale_factor), avg_score**(1.0/scale_factor), avg_size]) if i != num_generations-1: self.prepare_next_gen(math.ceil(self.pop_size/10)) print("\nStarting Generation " + str(i+2) + ": Species = " + str(len(self.species)) + ", Innovs = " + str(self.networks[0].master_innov[0]), end='\n') print("Finished simulation!")
def initialize(self, normalizer): """ Initialize writer. """ DataWriter.initialize(self, normalizer) return
def store_data(self): rowtitle = ["Product Name", "Price", "Rating", "No. of reviews", "Delivery Date", "Delivery Type", "Buy link"] w = DataWriter('Amazon.csv', rowtitle) w.writer(self.details)
class Controller(): def __init__(self, aConfigFilePath, aDebugFlag=False): # Config parameters default values # Timelapse interval hours, minutes and seconds used to calculate the # interval in seconds self.timelapseIntervalH = 1.0 self.timelapseIntervalM = 0.0 self.timelapseIntervalS = 0.0 # Number of images to take when motion is detected by the PIR self.motionSequenceCount = 5 # Number of images that are used by a motion detection algorithm before # up-to-date results are indicated by the produced difference image self.motionSequencePreload = 2 # Delay between each image taken when triggered by the PIR self.motionSequenceDelay = 2.0 # Percentage difference image area that needs to be covered by a # minimum number of blobs in order to evaluate that motion is happening self.motionAreaPercent = 60 self.motionMaxBlobs = 10 # Where timelapse images are saved self.timelapseFilePath = "./timelapse/" # Where motion images/diffs are saved self.motionFilePath = "./motion/" # Where temporary files are put, for example to get the current # brightness in the scene self.tempFilePath = "./temp/" # Path where hourly and motion logs are placed self.logFilePath = "./logs/" # TM: path to the queue folder self.queuePath = "./queue/" # The name if the most recently captured timelaps image self.timelapseLatestImage = "timelapse.jpg" #IR: Folder to save the true sequences self.miniSequencePath = "./miniSequence/" # Threshold (0-255) for the scene brightness when the IR led is # switched on self.brightnessThreshold = 25 # Timeout (seconds) for when the brightness in the scene needs to be # re-evaluated self.brightnessTimeout = 60.0 # The resolution of the camera self.resolutionX = 640 self.resolutionY = 480 # Flag to produce debug output to the terminal while running self.debug = aDebugFlag self.FOVMaxDistance = 20 self.FOVMinAreaDetected = 1 self.FOVVertical = 54 self.FOVHorizontal = 41 # IR: Defines downsampling level self.downLevel = 0 # TM: ROI self.ROI = None #TM: enable/disable SMS feature self.sendSMS = False # TM: SMS recipient self.textMessageRecipient = None # TM: queue implementation self.processingImage = False self.imgSetsInQ = 0 self.queueIndex = 0 self.currentImageSetToProcess = 0 self.numberOfImgSetsProcessed = 0 self.lastBackgroundRefrestTime = 0 self.backgroundRefreshWaitTime = 60.0 #60.0 * 5 self.previousMotionImgTime = 0 self.timeToWaitForNextMotionImg = 10.0 self._savingImageSequence = False self.sequenceTimeStampList = list() self.imageBrightnessList = list() self.PIRLastCountList = list() # Load config file parameters to override the default values self.parser = SafeConfigParser() self.setConfig(aConfigFilePath) # Paths setup, create the paths if they do not exist if not self._directoryExists(self.tempFilePath): self._createDirectory(self.tempFilePath) if not self._directoryExists(self.motionFilePath): self._createDirectory(self.motionFilePath) if not self._directoryExists(self.logFilePath): self._createDirectory(self.logFilePath) if not self._directoryExists(self.timelapseFilePath): self._createDirectory(self.timelapseFilePath) # TM; queue folder if not self._directoryExists(self.queuePath): self._createDirectory(self.queuePath) # IR: true sequence folder if not self._directoryExists(self.miniSequencePath): self._createDirectory(self.miniSequencePath) # Timelapse setup self.timelapseInterval = (self.timelapseIntervalH * 60.0 * 60.0 + self.timelapseIntervalM * 60.0 + self.timelapseIntervalS) self.timelapseLastTime = 0.0 # Brightness measurements setup self.brightnessLastTime = 0.0 self.brightness = 0 # Inputs/outputs sensor.initGPIO() self.PIR = sensor.PIR(sensor.PIR_PIN, sensor.GPIO.PUD_DOWN) self.PIRLastCount = 0 self.DHTType = Adafruit_DHT.DHT22 self.DHTPin = 23 #TM: Queue implementation self.PIR.enableInterrupt(self._takeImageCallbackPIR) self.PIRTriggered = False self.enableCallbackPIR = False # Camera setup self.resolution = (self.resolutionX, self.resolutionY) self.cam = picamera.PiCamera() self.cam.resolution = self.resolution # TODO(geir): Create a function to switch off camera LED in sensor.py sensor.GPIO.output(sensor.CAM_LED, False) # Difference image area that is considered too small to evaluate the # movement in the scene as true self.motionMinArea = \ motion.getCutoffObjectArea(self.resolutionX * self.resolutionY, self.FOVMinAreaDetected, self.FOVVertical, self.FOVHorizontal, self.FOVMaxDistance) # IR: downsampling self.originalResolutionX = self.resolutionX self.originalResolutionY = self.resolutionY self.defaultMotionMinArea = self.motionMinArea # TODO(geir): Consider wrapping the below in a helper function to # reduce init clutter # Data logging setup hourlyDataDescription = [ "AliveCount", "Timestamp", "Brightness", "ExternalTemp", "Humidity" ] hourlyDataFilePath = os.path.join(self.logFilePath, "hourlydata.csv") self.hourlyDataRecorder = DataWriter(hourlyDataFilePath, hourlyDataDescription) self.hourlyLastTime = 0.0 self.hourlyAliveCount = 1 # TMorton # Added processing time data self.processingEndTime = 0 self.processingStartTime = 0 motionDataDescription = [ "AliveCount", "Timestamp", "Brightness", "Threshold", # Do we need this? "BlobCount", "MaxBlobSize", "Verdict", "ProcessingTime" ] motionDataFilePath = os.path.join(self.logFilePath, "motiondata.csv") self.motionDataRecorder = DataWriter(motionDataFilePath, motionDataDescription) self.motionAliveCount = 1 # Image processor(s) self.processor = ThreeBlur() self.strProcessor = "ThreeBlur" #self.processor = backSub() #self.strProcessor = "backSub" self.processor.setConfig(aConfigFilePath) # do this if backSub self._doBackgroundRefresh() #Overwrites the ROI image from the processor object #if the ROI is downsampled (if the ROI is used) self.downSampleROI('ROI.jpg', self.downLevel) #Send starting SMS if self.textMessageRecipient is not None: self.sms = TextMessage() self.trueMotionCount = 0 self.lastTextMessageSendTime = time.time() self.sms.connectPhone() self.sms.setRecipient(str(self.textMessageRecipient)) self.sms.setContent("FoxBox On") self.sms.sendMessage() self.sms.disconnectPhone() def run(self): self.enableCallbackPIR = True while True: if self.textMessageRecipient is not None: self._sendTextMessage() if self._timeForTimelapse(): self.enableCallbackPIR = False while self._savingImageSequence == True: pass self._doTimelapse() self.enableCallbackPIR = True if self._timeForHourlyData(): self.enableCallbackPIR = False while self._savingImageSequence == True: pass self._doHourlyData() self.enableCallbackPIR = True if self.PIRTriggered == False: self.enableCallbackPIR = False while self._savingImageSequence == True: pass self._doBackgroundRefresh() self.enableCallbackPIR = True self._processImagesFromQ() time.sleep(1) def _timeForTimelapse(self): currentTime = time.time() if (currentTime - self.timelapseLastTime) > self.timelapseInterval: self.timelapseLastTime = currentTime return True else: return False def _doTimelapse(self): if self.debug: print("Performing timelapse") fileName = time.strftime("%Y%m%d-%H%M%S.jpg") savePath = os.path.join(self.timelapseFilePath, fileName) # tempFilePath = os.path.join(self.tempFilePath, # self.timelapseLatestImage) if self._getCurrentBrightness() < self.brightnessThreshold: sensor.activateIRLED() self.cam.capture(savePath) # TODO(geir): Evaluate if we should apply a visual indication of the # current ROI in the image before saving it? # Ensure LED is off sensor.deactivateIRLED() # T Morton def _doBackgroundRefresh(self): if self.strProcessor == "backSub": tempFilePath = os.path.join(self.tempFilePath, self.timelapseLatestImage) backgroundCurrentTime = time.time() if (backgroundCurrentTime - self.lastBackgroundRefrestTime ) > self.backgroundRefreshWaitTime: self.lastBackgroundRefrestTime = backgroundCurrentTime if self.debug: print("Refreshing background") if self._getCurrentBrightness() < self.brightnessThreshold: sensor.activateIRLED() self.cam.capture(tempFilePath) # IR: Downsample background image self.backgroundDownSample(self.downLevel) sensor.deactivateIRLED() # IR: Reset variables to defaults self.resolutionX = self.originalResolutionX self.resolutionY = self.originalResolutionY self.motionMinArea = self.defaultMotionMinArea def _timeForHourlyData(self): currentTime = time.time() if (currentTime - self.hourlyLastTime) > 3600.0: self.hourlyLastTime = currentTime return True else: return False def _doHourlyData(self): if self.debug: print("Performing hourly data") timestamp = time.strftime("%Y%m%d-%H%M%S") brightness = self._getCurrentBrightness() DHTData = Adafruit_DHT.read_retry(self.DHTType, self.DHTPin) #DHTData = Adafruit_DHT.read(self.DHTType, self.DHTPin) if DHTData[0] is None: print "Humidity is None" humidity = 0 else: humidity = round(DHTData[0], 2) if DHTData[1] is None: print "temperature is None" temperature = 0 else: temperature = round(DHTData[1], 2) dataList = [ self.hourlyAliveCount, timestamp, brightness, temperature, humidity ] self.hourlyDataRecorder.writeData(dataList) self.hourlyAliveCount += 1 if self.debug: print("Got hourly data: ") print(dataList) # T Morton: SMS def _sendTextMessage(self): currentTime = time.time() if (currentTime - self.lastTextMessageSendTime) > (60 * 30): self.lastTextMessageSendTime = currentTime self.sms.connectPhone() self.sms.setRecipient(str(self.textMessageRecipient)) self.sms.setContent( "FoxBox Hourly Motion Report. Number of True Detections: " + str(self.trueMotionCount)) self.sms.sendMessage() self.sms.disconnectPhone() print "message sent successfully" self.trueMotionCount = 0 def _takeImageCallbackPIR(self, channel): #print("Interrupt Triggered") self.PIRTriggered = True if self.enableCallbackPIR == True: currentTime = time.time() if (currentTime - self.previousMotionImgTime ) > self.timeToWaitForNextMotionImg: self.previousMotionImgTime = currentTime print("Saving image sequence to queue") self._saveImageSequenceToQ(self.motionSequenceCount, self.motionSequenceDelay) self.PIRTriggered = False def _getQueueFilename(self, aQueueIndex): if aQueueIndex < 10: rQueueIndex = '000' + str(aQueueIndex) elif aQueueIndex > 9 & aQueueIndex < 100: rQueueIndex = '00' + str(aQueueIndex) elif aQueueIndex > 99 & aQueueIndex < 1000: rQueueIndex = '0' + strre(aQueueIndex) return rQueueIndex def _saveImageSequenceToQ(self, aCount, aDeltaTime): #TM: Boolean indicates that images are being saved (used by run loop) self._savingImageSequence = True if self._getCurrentBrightness() < self.brightnessThreshold: sensor.activateIRLED() queueFilename = self._getQueueFilename(self.queueIndex) captureCount = 0 sequenceTimeStamp = time.strftime("%Y%m%d-%H%M%S") while (captureCount < aCount): imgPath = self.queuePath + queueFilename + \ "_" + str(captureCount) + ".jpg" pictureCurrentTime = time.time() self.cam.capture(imgPath) afterPictureCurrentTime = time.time() PictureDeltaTime = afterPictureCurrentTime - pictureCurrentTime #print PictureDeltaTime if (PictureDeltaTime < aDeltaTime): time.sleep(aDeltaTime - PictureDeltaTime) captureCount += 1 #TM: records time image taken self.sequenceTimeStampList.append(sequenceTimeStamp) #TM: records brightness at time image taken imageBrightness = self._getCurrentBrightness() self.imageBrightnessList.append(imageBrightness) # Increments the index used for filename convention ##CircularQ if self.queueIndex < 999: self.queueIndex += 1 else: self.queueIndex = 0 # Increments the actual number of image sets in the queue self.imgSetsInQ += 1 sensor.deactivateIRLED() self._savingImageSequence = False def _loadImageSequenceFromQ(self, aImgSequence): imgList = list() #IR: Saving the full size images fullSizeImgList = list() queueFilename = self._getQueueFilename(aImgSequence) sequence = queueFilename + '_?' for filename in sorted(glob.glob(self.queuePath + sequence + "*.jpg")): PathAndFile = os.path.splitext(filename)[0] latestFilename = ntpath.basename(PathAndFile) # IR: Downsample images and append to list self.resolutionX = self.originalResolutionX self.resolutionY = self.originalResolutionY img = cv2.imread(self.queuePath + latestFilename + ".jpg", cv2.CV_LOAD_IMAGE_COLOR) #IR: Saving the full size images fullSizeImgList.append(img) if (self.downLevel > 0): img = self.imageDownsampling(img, self.downLevel) imgList.append(img) #IR: Return both the full size and downsampled lists return imgList, fullSizeImgList def _processImagesFromQ(self): if self.imgSetsInQ > 0: self.processingImage = True self._doMotionDetectionFromQ(self.currentImageSetToProcess) self.processingImage = False self._deleteProcessedImgsFromQ(self.currentImageSetToProcess) ##CircularQ if self.currentImageSetToProcess < 999: self.currentImageSetToProcess += 1 else: self.currentImageSetToProcess = 0 ## elif os.listdir(self.queuePath) == []: self.queueIndex = 0 self.currentImageSetToProcess = 0 self.sequenceTimeStampList = list() self.imageBrightnessList = list() self.PIRLastCountList = list() def _deleteProcessedImgsFromQ(self, aImgSequence): queueFilename = self._getQueueFilename(aImgSequence) filenameSearch = queueFilename + '_?' for filename in glob.glob(self.queuePath + filenameSearch + ".jpg"): # Calling os.system breaks the parallel thread. #cmd = 'sudo rm ' + filename # os.system(cmd) os.remove(filename) self.imgSetsInQ -= 1 def _doMotionDetectionFromQ(self, aImageSet): if self.debug: print("Performing motion detection") self.processingStartTime = time.time() #IR: self.changeDownsamplingLevel() imgList, fullSizeImgList = self._loadImageSequenceFromQ(aImageSet) diffList = self._getDiffSequence(imgList) if self.motionSequencePreload <= 0: startIndex = 0 else: startIndex = self.motionSequencePreload - 1 largestAreaIndex = None largestAreaBlobs = None largestAreaBlobCount = None largestArea = -1.0 for diffIndex in xrange(startIndex, self.motionSequenceCount): blobData = motion.getBlobs(diffList[diffIndex]) blobs = blobData[0] num_blobs = blobData[1] blobAreaList = motion.getBlobAreaList(blobs, num_blobs) largestBlobArea = motion.getMaxAreaFromBlobs(blobAreaList) if largestBlobArea > largestArea: largestArea = largestBlobArea largestAreaIndex = diffIndex largestAreaBlobs = blobs largestAreaBlobCount = num_blobs self.processingEndTime = time.time() aliveCount = self.motionAliveCount #TM : get timestamp from list for when image was taken timestamp = self.sequenceTimeStampList[aImageSet] #timestamp = time.strftime("%Y%m%d-%H%M%S") #TM : get brightness from list for when image was taken brightness = self.imageBrightnessList[aImageSet] #brightness = self._getCurrentBrightness() threshold = 0 # TODO(geir): Need it? blobCount = largestAreaBlobCount maxBlobSize = largestBlobArea verdict = motion.evalMotionPropotionalBlobArea(largestAreaBlobs, largestAreaBlobCount, self.motionAreaPercent, self.motionMaxBlobs, self.motionMinArea) # TMorton: SMS #if self.sendSMS == True: if self.textMessageRecipient is not None: if verdict == True: self.trueMotionCount += 1 # TM: records the time it takes to process an image processingTime = round(self.processingEndTime - self.processingStartTime) # Save statistics in CSV dataList = [ aliveCount, timestamp, brightness, threshold, blobCount, maxBlobSize, verdict, processingTime ] self.motionDataRecorder.writeData(dataList) imageName = timestamp + ".jpg" diffName = timestamp + "-" + str(verdict) + "-diff.jpg" imagePath = os.path.join(self.motionFilePath, imageName) diffPath = os.path.join(self.motionFilePath, diffName) cv2.imwrite(imagePath, imgList[largestAreaIndex]) cv2.imwrite(diffPath, diffList[largestAreaIndex]) #IR: Save true image sequence to miniSequence Folder # at the original size of the images if verdict == True: for listIndex in range(len(fullSizeImgList)): imgName = timestamp + "-" + str(listIndex) + ".jpg" imgFullPath = os.path.join(self.miniSequencePath, imgName) cv2.imwrite(imgFullPath, fullSizeImgList[listIndex]) self.motionAliveCount += 1 if self.debug: print("Got motion data: ") print(dataList) # TM: Keeps track of the number of image sets that have been processed self.numberOfImgSetsProcessed += 1 # IR: Reset variables to defaults self.resolutionX = self.originalResolutionX self.resolutionY = self.originalResolutionY self.motionMinArea = self.defaultMotionMinArea # IR: downsampling function def imageDownsampling(self, aImage, aDownLevel): # aImage= the input array image # downLevel = how many levels of sampling # perform the downsampling tempImg = aImage for n in range(aDownLevel): imgDownsampled = cv2.pyrDown(tempImg) #print 'downsampling' tempImg = imgDownsampled # change the global resolution parameters of the image self.resolutionX = self.resolutionX / (2**aDownLevel) self.resolutionY = self.resolutionY / (2**aDownLevel) # recalculate the motion min area with the new resolution self.motionMinArea = \ motion.getCutoffObjectArea(self.resolutionX * self.resolutionY, self.FOVMinAreaDetected, self.FOVVertical, self.FOVHorizontal, self.FOVMaxDistance) return tempImg def _getCurrentBrightness(self): currentTime = time.time() if (currentTime - self.brightnessLastTime) > self.brightnessTimeout: self.brightnessLastTime = currentTime imagePath = os.path.join(self.tempFilePath, "brightness.jpg") self.cam.capture(imagePath) image = cv2.imread(imagePath, 0) brightness = motion.getBrightness(image) self.brightness = round(brightness[0], 2) #if self.debug: # print("Got brightness: " + str(self.brightness)) return self.brightness def _getDiffSequence(self, aImgList): diffList = list() for image in aImgList: diff = self.processor.getDiff(image) diffList.append(diff) return diffList def _directoryExists(self, aDirectory): return os.path.exists(aDirectory) def _createDirectory(self, aDirectory): os.makedirs(aDirectory) def setConfig(self, aConfig): if self.parser.read(aConfig): # General variables self.timelapseFilePath = self.setParam(self.timelapseFilePath, 'str', 'General', 'timelapseFilePath') self.motionFilePath = self.setParam(self.motionFilePath, 'str', 'General', 'motionFilePath') self.tempFilePath = self.setParam(self.tempFilePath, 'str', 'General', 'tempFilePath') self.logFilePath = self.setParam(self.logFilePath, 'str', 'General', 'logFilePath') self.timelapseLatestImage = self.setParam(self.logFilePath, 'str', 'General', 'timelapseLatestImage') # TM: loads queue path self.queuePath = self.setParam(self.queuePath, 'str', 'General', 'queuePath') # IR: loads miniSequence path (where true images are saved) self.miniSequencePath = self.setParam(self.miniSequencePath, 'str', 'General', 'miniSequencePath') # Controller variables self.timelapseIntervalH = self.setParam(self.timelapseIntervalH, 'float', 'Controller', 'timelapseIntervalH') self.timelapseIntervalM = self.setParam(self.timelapseIntervalM, 'float', 'Controller', 'timelapseIntervalM') self.timelapseIntervalS = self.setParam(self.timelapseIntervalS, 'float', 'Controller', 'timelapseIntervalS') self.motionSequenceCount = self.setParam(self.motionSequenceCount, 'int', 'Controller', 'motionSequenceCount') self.motionSequencePreload = \ self.setParam(self.motionSequencePreload, 'int', 'Controller', 'motionSequencePreload') self.motionSequenceDelay = self.setParam(self.motionSequenceDelay, 'float', 'Controller', 'motionSequenceDelay') self.motionAreaPercent = self.setParam(self.motionAreaPercent, 'int', 'Controller', 'motionAreaPercent') self.motionMaxBlobs = self.setParam(self.motionMaxBlobs, 'int', 'Controller', 'motionMaxBlobs') self.brightnessThreshold = self.setParam(self.brightnessThreshold, 'int', 'Controller', 'brightnessThreshold') self.brightnessTimeout = self.setParam(self.brightnessTimeout, 'float', 'Controller', 'brightnessTimeout') self.resolutionX = self.setParam(self.resolutionX, 'int', 'Controller', 'resolutionX') self.resolutionY = self.setParam(self.resolutionY, 'int', 'Controller', 'resolutionY') self.FOVMaxDistance = self.setParam(self.FOVMaxDistance, 'int', 'Controller', 'FOVMaxDistance') self.FOVMinAreaDetected = self.setParam(self.FOVMinAreaDetected, 'int', 'Controller', 'FOVMinAreaDetected') self.FOVVertical = self.setParam(self.FOVVertical, 'int', 'Controller', 'FOVVertical') self.FOVHorizontal = self.setParam(self.FOVHorizontal, 'int', 'Controller', 'FOVHorizontal') # TM: added for down-sampling self.downLevel = self.setParam(self.downLevel, 'int', 'Controller', 'downLevel') # TM: added for queue self.backgroundRefreshWaitTime = self.setParam( self.backgroundRefreshWaitTime, 'float', 'Controller', 'backgroundRefreshWaitTime') # TM: added for queue self.timeToWaitForNextMotionImg = self.setParam( self.timeToWaitForNextMotionImg, 'float', 'Controller', 'timeToWaitForNextMotionImg') #TM: Added for ROI down-sampling self.ROI = self.setParam(self.ROI, 'str', 'General', 'ROI') #TM: Added for SMS feature #self.sendSMS = self.setParam(self.sendSMS, 'bool', 'General', 'sendSMS') #TM: SMS recipient self.textMessageRecipient = self.setParam( self.textMessageRecipient, 'str', 'General', 'textMessageRecipient') self.debug = self.setParam(self.debug, 'bool', 'Controller', 'debug') def setParam(self, aParam, aParamType, aSection, aOption): param = None if self.parser.has_option(aSection, aOption): if aParamType == 'int': param = self.parser.getint(aSection, aOption) elif aParamType == 'float': param = self.parser.getfloat(aSection, aOption) elif aParamType == 'bool': param = self.parser.getboolean(aSection, aOption) elif aParamType == 'str': param = self.parser.get(aSection, aOption) if param is None: print(aOption + " in " + aSection + " was not found. Using" + " default value.") param = aParam return param # IR: ROI downsampling function def downSampleROI(self, aROIimageName, aDownLevel): #read the ROI image to an array #perform downsampling on the image #send the downsampled ROI to the processor object #this way the original ROI can be downsampled as many times #at different levels if self.ROI is not None: imgROI = cv2.imread(aROIimageName, 0) #TM: Saving a copy of original ROI might be needed in the future for n in range(aDownLevel): downsampledROI = cv2.pyrDown(imgROI) imgROI = downsampledROI print 'ROI downsampling' #remove the blur regions between from the ROI downsampled image retval, imgROI = cv2.threshold(imgROI, 127, 255, cv2.THRESH_BINARY) self.processor.loadROIfromControl(imgROI) def backgroundDownSample(self, aDownLevel): if self.strProcessor == "backSub": backgroundPath = os.path.join(self.tempFilePath, self.timelapseLatestImage) backgroundImg = cv2.imread(backgroundPath, cv2.CV_LOAD_IMAGE_COLOR) for n in range(aDownLevel): downsampledBackground = cv2.pyrDown(backgroundImg) backgroundImg = downsampledBackground print 'backgound downsampling' self.processor.loadBackgroundFromControl(backgroundImg) #IR : two levels of downsampling including ROI def changeDownsamplingLevel(self): #checks the number of image sets in the Q #if the number is above the threshold #increases the downsampling level to reduce the queue length #maximum time to wait in the cue cueThreshold = 5 # assumed max number of images in Q upperCueThreshold = 8 lowerCueThreshold = 1 #if self.downLevel == 0 and self.imgSetsInQ > cueThreshold: if self.downLevel == 0 and self.imgSetsInQ > cueThreshold and self.imgSetsInQ < upperCueThreshold: self.downLevel += 1 self.downSampleROI("ROI.jpg", self.downLevel) self.backgroundDownSample(self.downLevel) elif self.downLevel == 1 and self.imgSetsInQ <= lowerCueThreshold: self.downLevel -= 1 self.downSampleROI("ROI.jpg", self.downLevel) self.backgroundDownSample(self.downLevel) elif self.downLevel < 2 and self.imgSetsInQ >= upperCueThreshold: #self.downLevel +=1 self.downLevel = 2 self.downSampleROI("ROI.jpg", self.downLevel) self.backgroundDownSample(self.downLevel) elif self.downLevel == 2 and self.imgSetsInQ < cueThreshold: self.downLevel -= 1 self.downSampleROI("ROI.jpg", self.downLevel) self.backgroundDownSample(self.downLevel) else: pass
def __init__(self, aConfigFilePath, aDebugFlag=False): # Config parameters default values # Timelapse interval hours, minutes and seconds used to calculate the # interval in seconds self.timelapseIntervalH = 1.0 self.timelapseIntervalM = 0.0 self.timelapseIntervalS = 0.0 # Number of images to take when motion is detected by the PIR self.motionSequenceCount = 5 # Number of images that are used by a motion detection algorithm before # up-to-date results are indicated by the produced difference image self.motionSequencePreload = 2 # Delay between each image taken when triggered by the PIR self.motionSequenceDelay = 2.0 # Percentage difference image area that needs to be covered by a # minimum number of blobs in order to evaluate that motion is happening self.motionAreaPercent = 60 self.motionMaxBlobs = 10 # Where timelapse images are saved self.timelapseFilePath = "./timelapse/" # Where motion images/diffs are saved self.motionFilePath = "./motion/" # Where temporary files are put, for example to get the current # brightness in the scene self.tempFilePath = "./temp/" # Path where hourly and motion logs are placed self.logFilePath = "./logs/" # TM: path to the queue folder self.queuePath = "./queue/" # The name if the most recently captured timelaps image self.timelapseLatestImage = "timelapse.jpg" #IR: Folder to save the true sequences self.miniSequencePath = "./miniSequence/" # Threshold (0-255) for the scene brightness when the IR led is # switched on self.brightnessThreshold = 25 # Timeout (seconds) for when the brightness in the scene needs to be # re-evaluated self.brightnessTimeout = 60.0 # The resolution of the camera self.resolutionX = 640 self.resolutionY = 480 # Flag to produce debug output to the terminal while running self.debug = aDebugFlag self.FOVMaxDistance = 20 self.FOVMinAreaDetected = 1 self.FOVVertical = 54 self.FOVHorizontal = 41 # IR: Defines downsampling level self.downLevel = 0 # TM: ROI self.ROI = None #TM: enable/disable SMS feature self.sendSMS = False # TM: SMS recipient self.textMessageRecipient = None # TM: queue implementation self.processingImage = False self.imgSetsInQ = 0 self.queueIndex = 0 self.currentImageSetToProcess = 0 self.numberOfImgSetsProcessed = 0 self.lastBackgroundRefrestTime = 0 self.backgroundRefreshWaitTime = 60.0 #60.0 * 5 self.previousMotionImgTime = 0 self.timeToWaitForNextMotionImg = 10.0 self._savingImageSequence = False self.sequenceTimeStampList = list() self.imageBrightnessList = list() self.PIRLastCountList = list() # Load config file parameters to override the default values self.parser = SafeConfigParser() self.setConfig(aConfigFilePath) # Paths setup, create the paths if they do not exist if not self._directoryExists(self.tempFilePath): self._createDirectory(self.tempFilePath) if not self._directoryExists(self.motionFilePath): self._createDirectory(self.motionFilePath) if not self._directoryExists(self.logFilePath): self._createDirectory(self.logFilePath) if not self._directoryExists(self.timelapseFilePath): self._createDirectory(self.timelapseFilePath) # TM; queue folder if not self._directoryExists(self.queuePath): self._createDirectory(self.queuePath) # IR: true sequence folder if not self._directoryExists(self.miniSequencePath): self._createDirectory(self.miniSequencePath) # Timelapse setup self.timelapseInterval = (self.timelapseIntervalH * 60.0 * 60.0 + self.timelapseIntervalM * 60.0 + self.timelapseIntervalS) self.timelapseLastTime = 0.0 # Brightness measurements setup self.brightnessLastTime = 0.0 self.brightness = 0 # Inputs/outputs sensor.initGPIO() self.PIR = sensor.PIR(sensor.PIR_PIN, sensor.GPIO.PUD_DOWN) self.PIRLastCount = 0 self.DHTType = Adafruit_DHT.DHT22 self.DHTPin = 23 #TM: Queue implementation self.PIR.enableInterrupt(self._takeImageCallbackPIR) self.PIRTriggered = False self.enableCallbackPIR = False # Camera setup self.resolution = (self.resolutionX, self.resolutionY) self.cam = picamera.PiCamera() self.cam.resolution = self.resolution # TODO(geir): Create a function to switch off camera LED in sensor.py sensor.GPIO.output(sensor.CAM_LED, False) # Difference image area that is considered too small to evaluate the # movement in the scene as true self.motionMinArea = \ motion.getCutoffObjectArea(self.resolutionX * self.resolutionY, self.FOVMinAreaDetected, self.FOVVertical, self.FOVHorizontal, self.FOVMaxDistance) # IR: downsampling self.originalResolutionX = self.resolutionX self.originalResolutionY = self.resolutionY self.defaultMotionMinArea = self.motionMinArea # TODO(geir): Consider wrapping the below in a helper function to # reduce init clutter # Data logging setup hourlyDataDescription = [ "AliveCount", "Timestamp", "Brightness", "ExternalTemp", "Humidity" ] hourlyDataFilePath = os.path.join(self.logFilePath, "hourlydata.csv") self.hourlyDataRecorder = DataWriter(hourlyDataFilePath, hourlyDataDescription) self.hourlyLastTime = 0.0 self.hourlyAliveCount = 1 # TMorton # Added processing time data self.processingEndTime = 0 self.processingStartTime = 0 motionDataDescription = [ "AliveCount", "Timestamp", "Brightness", "Threshold", # Do we need this? "BlobCount", "MaxBlobSize", "Verdict", "ProcessingTime" ] motionDataFilePath = os.path.join(self.logFilePath, "motiondata.csv") self.motionDataRecorder = DataWriter(motionDataFilePath, motionDataDescription) self.motionAliveCount = 1 # Image processor(s) self.processor = ThreeBlur() self.strProcessor = "ThreeBlur" #self.processor = backSub() #self.strProcessor = "backSub" self.processor.setConfig(aConfigFilePath) # do this if backSub self._doBackgroundRefresh() #Overwrites the ROI image from the processor object #if the ROI is downsampled (if the ROI is used) self.downSampleROI('ROI.jpg', self.downLevel) #Send starting SMS if self.textMessageRecipient is not None: self.sms = TextMessage() self.trueMotionCount = 0 self.lastTextMessageSendTime = time.time() self.sms.connectPhone() self.sms.setRecipient(str(self.textMessageRecipient)) self.sms.setContent("FoxBox On") self.sms.sendMessage() self.sms.disconnectPhone()
import os import logging from DataReader import DataReader from DataWriter import DataWriter from HRM_Processor import HRM_Processor os.chdir("test_data") for i in range(1, 33): base_file_name = "test_data" + str(i) file_name = base_file_name + ".csv" print(file_name) logging.basicConfig(filename=base_file_name + "_logs.txt", format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') try: dr = DataReader(file_name) hrm = HRM_Processor(dr) dw = DataWriter(hrm) except (FileNotFoundError, ValueError, TypeError): logging.info("Attempt to make process file was terminated.") # Remove all handlers associated with the root logger object. # this allows for the creation of a new log file for each test file for handler in logging.root.handlers[:]: logging.root.removeHandler(handler)
def __init__(self, aConfigFilePath, aDebugFlag=False): # Config parameters default values # Timelapse interval hours, minutes and seconds used to calculate the # interval in seconds self.timelapseIntervalH = 1.0 self.timelapseIntervalM = 0.0 self.timelapseIntervalS = 0.0 # Number of images to take when motion is detected by the PIR self.motionSequenceCount = 5 # Number of images that are used by a motion detection algorithm before # up-to-date results are indicated by the produced difference image self.motionSequencePreload = 2 # Delay between each image taken when triggered by the PIR self.motionSequenceDelay = 1.0 # Percentage difference image area that needs to be covered by a # minimum number of blobs in order to evaluate that motion is happening self.motionAreaPercent = 60 self.motionMaxBlobs = 10 # Where timelapse images are saved self.timelapseFilePath = "./timelapse/" # Where motion images/diffs are saved self.motionFilePath = "./motion/" # Where temporary files are put, for example to get the current # brightness in the scene self.tempFilePath = "./temp/" # Path where hourly and motion logs are placed self.logFilePath = "./logs/" # The name if the most recently captured timelaps image self.timelapseLatestImage = "timelapse.jpg" # Threshold (0-255) for the scene brightness when the IR led is # switched on self.brightnessThreshold = 25 # Timeout (seconds) for when the brightness in the scene needs to be # re-evaluated self.brightnessTimeout = 60.0 # The resolution of the camera self.originalResolutionX = 640 self.originalResolutionY = 480 self.resolutionX = self.originalResolutionX self.resolutionY = self.originalResolutionY # Flag to produce debug output to the terminal while running self.debug = aDebugFlag self.FOVMaxDistance = 20 self.FOVMinAreaDetected = 1 self.FOVVertical = 54 self.FOVHorizontal = 41 # Radu: define the downsampling level self.downLevel = 0 # T Morton: adds a path to a folder for the image processing queue self.queuePath = "./queue/" if not self._directoryExists(self.queuePath): self._createDirectory(self.queuePath) # Load config file parameters to override the default values self.parser = SafeConfigParser() self.setConfig(aConfigFilePath) # Paths setup, create the paths if they do not exist if not self._directoryExists(self.tempFilePath): self._createDirectory(self.tempFilePath) if not self._directoryExists(self.motionFilePath): self._createDirectory(self.motionFilePath) if not self._directoryExists(self.logFilePath): self._createDirectory(self.logFilePath) if not self._directoryExists(self.timelapseFilePath): self._createDirectory(self.timelapseFilePath) # Timelapse setup self.timelapseInterval = (self.timelapseIntervalH * 60.0 * 60.0 + self.timelapseIntervalM * 60.0 + self.timelapseIntervalS) self.timelapseLastTime = 0.0 # background refresh setup self.firstImage = True self.backgroundRefreshDelay = False self.backgroundRefreshDelayTime = 60 * 10 self.previousTime = 0.0 self.delayCount = 0 self.delayCountMax = 2 # Brightness measurements setup self.brightnessLastTime = 0.0 self.brightness = 0 # Inputs/outputs sensor.initGPIO() self.PIR = sensor.PIR(sensor.PIR_PIN, sensor.GPIO.PUD_DOWN) # T Morton: queue implementation self.processingImage = False self.imgSetsInQ = 0 self.queueIndex = 0 self.currentImageSetToProcess = 0 self.numberOfImgSetsProcessed = 0 self.PIR.enableInterrupt(self._takeImageCallbackPIR) self.PIRCount = 0 self.previousTimeForImg = 0 self.timeToWait = 10 self.PIRLastCount = 0 self.DHTType = Adafruit_DHT.DHT22 self.DHTPin = 23 # Camera setup self.resolution = (self.resolutionX, self.resolutionY) self.cam = picamera.PiCamera() self.cam.resolution = self.resolution # TODO(geir): Create a function to switch off camera LED in sensor.py sensor.GPIO.output(sensor.CAM_LED, False) #if (self.downLevel > 0): #self.resolutionX = self.resolutionX / (2 ** self.downLevel ) #self.resolutionY = self.resolutionY / (2 ** self.downLevel) # Difference image area that is considered too small to evaluate the # movement in the scene as true self.motionMinArea = \ motion.getCutoffObjectArea(self.resolutionX * self.resolutionY, self.FOVMinAreaDetected, self.FOVVertical, self.FOVHorizontal, self.FOVMaxDistance) self.defaultMotionMinArea = self.motionMinArea # TODO(geir): Consider wrapping the below in a helper function to # reduce init clutter # Data logging setup hourlyDataDescription = ["AliveCount", "Timestamp", "Brightness", "ExternalTemp", "Humidity"] hourlyDataFilePath = os.path.join(self.logFilePath, "hourlydata.csv") self.hourlyDataRecorder = DataWriter(hourlyDataFilePath, hourlyDataDescription) self.hourlyLastTime = 0.0 self.hourlyAliveCount = 1 motionDataDescription = ["AliveCount", "Timestamp", "ImageName", "Brightness", "Threshold", # Do we need this? "PIRCount", "BlobCount", "MaxBlobSize", "Verdict"] motionDataFilePath = os.path.join(self.logFilePath, "motiondata.csv") self.motionDataRecorder = DataWriter(motionDataFilePath, motionDataDescription) self.motionAliveCount = 1 #ROI image downsampling if self.downLevel > 0: self.ROIdownsampling('ROI.jpg', self.downLevel) # Image processor(s) self.processor = ThreeBlur() #self.processor = backSub() self.processor.setConfig(aConfigFilePath) # TODO(geir): Arrange for a system that can use multiple algorithms """
def dw(): """Create a basic DataWriter object""" dr_for_DW = DataReader('test_data1.csv') hrm_for_DW = HRM_Processor(dr_for_DW) dw = DataWriter(hrm_for_DW) return dw
def store_data(self): rowtitle = ["Product Name", "Price", "Rating", "No. of reviews", "Buy link"] w = DataWriter("Flipkart.csv", rowtitle) w.writer(self.details)