def store_data(self):
rowtitle = [
"Product Name", "Price", "Location", "Add post date", "Buy link"
]
w = DataWriter('OLX.csv', rowtitle)
w.writer(self.details)
def test_DataWriter_init_valid_case():
"""Tests the initialization of the DataWriter object, and that it
successfully gets the output_dict from the input_dict and stores it as
metrics.
Returns
-------
None
"""
dr = DataReader("test_data1.csv")
hrm = HRM_Processor(dr)
dw = DataWriter(hrm)
assert dw.metrics == hrm.output_dict
def test_DataWriter_init_write_to_dict():
"""Tests that the construction of a DataWriter object creates a .json
file with the base file name that is the same as the file name of the
.csv file passed into the original DataReader.
Returns
-------
None
"""
dr = DataReader("test_data1.csv")
hrm = HRM_Processor(dr)
dw = DataWriter(hrm)
assert os.path.isfile("test_data1.json")
os.remove("test_data1.json")
def test_DataWriter_init_invalid_case(hrm):
"""Tests the initialization of the DataWriter object for a case where
the data from the HRM_Processor object is not valid.
Parameters
----------
hrm: HRM_Processor
Generic HRM_Processor made from a DataReader with test_file.csv
capsys: Pytest fixture
A decorator for getting outputs printed to the console
Returns
-------
None
"""
hrm.isValid = False # Force invalidity for testing case
with pytest.raises(ValueError):
dw = DataWriter(hrm)
def simulate_generations(self, num_generations, print_best):
file_path = "csv/ESN_Results.csv"
dw = DataWriter()
dr = DataReader()
dw.init_table(file_path)
p = Predictor()
mapping = dr.get_mapping()
images = dr.get_images(112800, 28, 28) # 112800 images in data set
scale_factor = 10
for i in range(num_generations):
sum = 0
best_score = -100
best_accuracy = -100
best_net = []
engines = []
for net in self.networks:
engine = [net, 0, 0]
engines.append(engine)
p.make_predictions(engines, mapping, images, scale_factor)
for j in range(len(engines)):
self.networks[j].fitness = engines[j][1]
if engines[j][2] > best_score:
best_score = engines[j][2]
best_net = self.networks[j]
if engines[j][1] > best_accuracy:
best_accuracy = engines[j][1]
avg_accuracy = self.avg_fitness(self.networks) # avg accuracy
for j in range(len(engines)):
self.networks[j].fitness = engines[j][2] # change fitness to score
avg_score = self.avg_fitness(self.networks) # avg accuracy
avg_size = self.avg_network_size()
if print_best:
best_net.show_net()
print("-----------------------------------\t\t\t\t\t\t\n Generation " + str(i+1) + " results\n-----------------------------------\n", end='\n')
print("Highest accuracy: " + str(best_accuracy*100) + "%\nHighest score: " + str(best_score**(1.0/scale_factor)) + "\nAverage accuracy: " + str(avg_accuracy*100) + "%\nAverage score: " + str(avg_score**(1.0/scale_factor)) + "\nNum species: " + str(len(self.species)) + "\nInnovs tried: " + str(self.networks[0].master_innov[0]) + "\nAverage connections per network: " + str(avg_size) + "\n")
non_jit = self.construct_non_jit(best_net)
pickle.dump(non_jit, open("neural_net.txt", "wb"))
dw.write_row(file_path, [i+1, best_accuracy*100, avg_accuracy*100, best_score**(1.0/scale_factor), avg_score**(1.0/scale_factor), avg_size])
if i != num_generations-1:
self.prepare_next_gen(math.ceil(self.pop_size/10))
print("\nStarting Generation " + str(i+2) + ": Species = " + str(len(self.species)) + ", Innovs = " + str(self.networks[0].master_innov[0]), end='\n')
print("Finished simulation!")
def main():
logging.basicConfig(filename="HRM_logs.txt",
format='%(asctime)s %(levelname)s:%(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p')
file_name = get_file_name()
wants_duration = get_wants_duration()
try:
if wants_duration:
duration = get_duration()
dr = DataReader(file_name, duration)
else:
dr = DataReader(file_name)
hrm = HRM_Processor(dr)
dw = DataWriter(hrm)
except (FileNotFoundError, ValueError, TypeError):
logging.info("Driver script terminated unsuccessfully.")
logging.info("Successful termination of HRM_Driver")
def test_write_to_json():
"""Tests the write_to_json function of the DataWriter, which should be
able to take in an dictionary and output file path, and write that
dictionary to the specified output file.
Returns
-------
None
"""
dr = DataReader("test_data1.csv")
hrm = HRM_Processor(dr)
dw = DataWriter(hrm)
metrics = {"test": 5, "another word": 18.5}
output_file = "test.json"
dw.write_to_json(metrics, output_file)
with open(output_file) as infile:
written_data = json.load(infile)
assert written_data == metrics
def dw():
"""Create a basic DataWriter object"""
dr_for_DW = DataReader('test_data1.csv')
hrm_for_DW = HRM_Processor(dr_for_DW)
dw = DataWriter(hrm_for_DW)
return dw
def __init__(self, aConfigFilePath, aDebugFlag=False):
# Config parameters default values
# Timelapse interval hours, minutes and seconds used to calculate the
# interval in seconds
self.timelapseIntervalH = 1.0
self.timelapseIntervalM = 0.0
self.timelapseIntervalS = 0.0
# Number of images to take when motion is detected by the PIR
self.motionSequenceCount = 5
# Number of images that are used by a motion detection algorithm before
# up-to-date results are indicated by the produced difference image
self.motionSequencePreload = 2
# Delay between each image taken when triggered by the PIR
self.motionSequenceDelay = 1.0
# Percentage difference image area that needs to be covered by a
# minimum number of blobs in order to evaluate that motion is happening
self.motionAreaPercent = 60
self.motionMaxBlobs = 10
# Where timelapse images are saved
self.timelapseFilePath = "./timelapse/"
# Where motion images/diffs are saved
self.motionFilePath = "./motion/"
# Where temporary files are put, for example to get the current
# brightness in the scene
self.tempFilePath = "./temp/"
# Path where hourly and motion logs are placed
self.logFilePath = "./logs/"
# The name if the most recently captured timelaps image
self.timelapseLatestImage = "timelapse.jpg"
# Threshold (0-255) for the scene brightness when the IR led is
# switched on
self.brightnessThreshold = 25
# Timeout (seconds) for when the brightness in the scene needs to be
# re-evaluated
self.brightnessTimeout = 60.0
# The resolution of the camera
self.originalResolutionX = 640
self.originalResolutionY = 480
self.resolutionX = self.originalResolutionX
self.resolutionY = self.originalResolutionY
# Flag to produce debug output to the terminal while running
self.debug = aDebugFlag
self.FOVMaxDistance = 20
self.FOVMinAreaDetected = 1
self.FOVVertical = 54
self.FOVHorizontal = 41
# Radu: define the downsampling level
self.downLevel = 0
# T Morton: adds a path to a folder for the image processing queue
self.queuePath = "./queue/"
if not self._directoryExists(self.queuePath):
self._createDirectory(self.queuePath)
# Load config file parameters to override the default values
self.parser = SafeConfigParser()
self.setConfig(aConfigFilePath)
# Paths setup, create the paths if they do not exist
if not self._directoryExists(self.tempFilePath):
self._createDirectory(self.tempFilePath)
if not self._directoryExists(self.motionFilePath):
self._createDirectory(self.motionFilePath)
if not self._directoryExists(self.logFilePath):
self._createDirectory(self.logFilePath)
if not self._directoryExists(self.timelapseFilePath):
self._createDirectory(self.timelapseFilePath)
# Timelapse setup
self.timelapseInterval = (self.timelapseIntervalH * 60.0 * 60.0 +
self.timelapseIntervalM * 60.0 +
self.timelapseIntervalS)
self.timelapseLastTime = 0.0
# background refresh setup
self.firstImage = True
self.backgroundRefreshDelay = False
self.backgroundRefreshDelayTime = 60 * 10
self.previousTime = 0.0
self.delayCount = 0
self.delayCountMax = 2
# Brightness measurements setup
self.brightnessLastTime = 0.0
self.brightness = 0
# Inputs/outputs
sensor.initGPIO()
self.PIR = sensor.PIR(sensor.PIR_PIN, sensor.GPIO.PUD_DOWN)
# T Morton: queue implementation
self.processingImage = False
self.imgSetsInQ = 0
self.queueIndex = 0
self.currentImageSetToProcess = 0
self.numberOfImgSetsProcessed = 0
self.PIR.enableInterrupt(self._takeImageCallbackPIR)
self.PIRCount = 0
self.previousTimeForImg = 0
self.timeToWait = 10
self.PIRLastCount = 0
self.DHTType = Adafruit_DHT.DHT22
self.DHTPin = 23
# Camera setup
self.resolution = (self.resolutionX, self.resolutionY)
self.cam = picamera.PiCamera()
self.cam.resolution = self.resolution
# TODO(geir): Create a function to switch off camera LED in sensor.py
sensor.GPIO.output(sensor.CAM_LED, False)
#if (self.downLevel > 0):
#self.resolutionX = self.resolutionX / (2 ** self.downLevel )
#self.resolutionY = self.resolutionY / (2 ** self.downLevel)
# Difference image area that is considered too small to evaluate the
# movement in the scene as true
self.motionMinArea = \
motion.getCutoffObjectArea(self.resolutionX * self.resolutionY,
self.FOVMinAreaDetected,
self.FOVVertical,
self.FOVHorizontal,
self.FOVMaxDistance)
self.defaultMotionMinArea = self.motionMinArea
# TODO(geir): Consider wrapping the below in a helper function to
# reduce init clutter
# Data logging setup
hourlyDataDescription = ["AliveCount",
"Timestamp",
"Brightness",
"ExternalTemp",
"Humidity"]
hourlyDataFilePath = os.path.join(self.logFilePath, "hourlydata.csv")
self.hourlyDataRecorder = DataWriter(hourlyDataFilePath,
hourlyDataDescription)
self.hourlyLastTime = 0.0
self.hourlyAliveCount = 1
motionDataDescription = ["AliveCount",
"Timestamp",
"ImageName",
"Brightness",
"Threshold", # Do we need this?
"PIRCount",
"BlobCount",
"MaxBlobSize",
"Verdict"]
motionDataFilePath = os.path.join(self.logFilePath, "motiondata.csv")
self.motionDataRecorder = DataWriter(motionDataFilePath,
motionDataDescription)
self.motionAliveCount = 1
#ROI image downsampling
if self.downLevel > 0:
self.ROIdownsampling('ROI.jpg', self.downLevel)
# Image processor(s)
self.processor = ThreeBlur()
#self.processor = backSub()
self.processor.setConfig(aConfigFilePath)
# TODO(geir): Arrange for a system that can use multiple algorithms
"""
import os
import logging
from DataReader import DataReader
from DataWriter import DataWriter
from HRM_Processor import HRM_Processor
os.chdir("test_data")
for i in range(1, 33):
base_file_name = "test_data" + str(i)
file_name = base_file_name + ".csv"
print(file_name)
logging.basicConfig(filename=base_file_name + "_logs.txt",
format='%(asctime)s %(levelname)s:%(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p')
try:
dr = DataReader(file_name)
hrm = HRM_Processor(dr)
dw = DataWriter(hrm)
except (FileNotFoundError, ValueError, TypeError):
logging.info("Attempt to make process file was terminated.")
# Remove all handlers associated with the root logger object.
# this allows for the creation of a new log file for each test file
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
def __init__(self, aConfigFilePath, aDebugFlag=False):
# Config parameters default values
# Timelapse interval hours, minutes and seconds used to calculate the
# interval in seconds
self.timelapseIntervalH = 1.0
self.timelapseIntervalM = 0.0
self.timelapseIntervalS = 0.0
# Number of images to take when motion is detected by the PIR
self.motionSequenceCount = 5
# Number of images that are used by a motion detection algorithm before
# up-to-date results are indicated by the produced difference image
self.motionSequencePreload = 2
# Delay between each image taken when triggered by the PIR
self.motionSequenceDelay = 2.0
# Percentage difference image area that needs to be covered by a
# minimum number of blobs in order to evaluate that motion is happening
self.motionAreaPercent = 60
self.motionMaxBlobs = 10
# Where timelapse images are saved
self.timelapseFilePath = "./timelapse/"
# Where motion images/diffs are saved
self.motionFilePath = "./motion/"
# Where temporary files are put, for example to get the current
# brightness in the scene
self.tempFilePath = "./temp/"
# Path where hourly and motion logs are placed
self.logFilePath = "./logs/"
# TM: path to the queue folder
self.queuePath = "./queue/"
# The name if the most recently captured timelaps image
self.timelapseLatestImage = "timelapse.jpg"
#IR: Folder to save the true sequences
self.miniSequencePath = "./miniSequence/"
# Threshold (0-255) for the scene brightness when the IR led is
# switched on
self.brightnessThreshold = 25
# Timeout (seconds) for when the brightness in the scene needs to be
# re-evaluated
self.brightnessTimeout = 60.0
# The resolution of the camera
self.resolutionX = 640
self.resolutionY = 480
# Flag to produce debug output to the terminal while running
self.debug = aDebugFlag
self.FOVMaxDistance = 20
self.FOVMinAreaDetected = 1
self.FOVVertical = 54
self.FOVHorizontal = 41
# IR: Defines downsampling level
self.downLevel = 0
# TM: ROI
self.ROI = None
#TM: enable/disable SMS feature
self.sendSMS = False
# TM: SMS recipient
self.textMessageRecipient = None
# TM: queue implementation
self.processingImage = False
self.imgSetsInQ = 0
self.queueIndex = 0
self.currentImageSetToProcess = 0
self.numberOfImgSetsProcessed = 0
self.lastBackgroundRefrestTime = 0
self.backgroundRefreshWaitTime = 60.0 #60.0 * 5
self.previousMotionImgTime = 0
self.timeToWaitForNextMotionImg = 10.0
self._savingImageSequence = False
self.sequenceTimeStampList = list()
self.imageBrightnessList = list()
self.PIRLastCountList = list()
# Load config file parameters to override the default values
self.parser = SafeConfigParser()
self.setConfig(aConfigFilePath)
# Paths setup, create the paths if they do not exist
if not self._directoryExists(self.tempFilePath):
self._createDirectory(self.tempFilePath)
if not self._directoryExists(self.motionFilePath):
self._createDirectory(self.motionFilePath)
if not self._directoryExists(self.logFilePath):
self._createDirectory(self.logFilePath)
if not self._directoryExists(self.timelapseFilePath):
self._createDirectory(self.timelapseFilePath)
# TM; queue folder
if not self._directoryExists(self.queuePath):
self._createDirectory(self.queuePath)
# IR: true sequence folder
if not self._directoryExists(self.miniSequencePath):
self._createDirectory(self.miniSequencePath)
# Timelapse setup
self.timelapseInterval = (self.timelapseIntervalH * 60.0 * 60.0 +
self.timelapseIntervalM * 60.0 +
self.timelapseIntervalS)
self.timelapseLastTime = 0.0
# Brightness measurements setup
self.brightnessLastTime = 0.0
self.brightness = 0
# Inputs/outputs
sensor.initGPIO()
self.PIR = sensor.PIR(sensor.PIR_PIN, sensor.GPIO.PUD_DOWN)
self.PIRLastCount = 0
self.DHTType = Adafruit_DHT.DHT22
self.DHTPin = 23
#TM: Queue implementation
self.PIR.enableInterrupt(self._takeImageCallbackPIR)
self.PIRTriggered = False
self.enableCallbackPIR = False
# Camera setup
self.resolution = (self.resolutionX, self.resolutionY)
self.cam = picamera.PiCamera()
self.cam.resolution = self.resolution
# TODO(geir): Create a function to switch off camera LED in sensor.py
sensor.GPIO.output(sensor.CAM_LED, False)
# Difference image area that is considered too small to evaluate the
# movement in the scene as true
self.motionMinArea = \
motion.getCutoffObjectArea(self.resolutionX * self.resolutionY,
self.FOVMinAreaDetected,
self.FOVVertical,
self.FOVHorizontal,
self.FOVMaxDistance)
# IR: downsampling
self.originalResolutionX = self.resolutionX
self.originalResolutionY = self.resolutionY
self.defaultMotionMinArea = self.motionMinArea
# TODO(geir): Consider wrapping the below in a helper function to
# reduce init clutter
# Data logging setup
hourlyDataDescription = [
"AliveCount", "Timestamp", "Brightness", "ExternalTemp", "Humidity"
]
hourlyDataFilePath = os.path.join(self.logFilePath, "hourlydata.csv")
self.hourlyDataRecorder = DataWriter(hourlyDataFilePath,
hourlyDataDescription)
self.hourlyLastTime = 0.0
self.hourlyAliveCount = 1
# TMorton
# Added processing time data
self.processingEndTime = 0
self.processingStartTime = 0
motionDataDescription = [
"AliveCount",
"Timestamp",
"Brightness",
"Threshold", # Do we need this?
"BlobCount",
"MaxBlobSize",
"Verdict",
"ProcessingTime"
]
motionDataFilePath = os.path.join(self.logFilePath, "motiondata.csv")
self.motionDataRecorder = DataWriter(motionDataFilePath,
motionDataDescription)
self.motionAliveCount = 1
# Image processor(s)
self.processor = ThreeBlur()
self.strProcessor = "ThreeBlur"
#self.processor = backSub()
#self.strProcessor = "backSub"
self.processor.setConfig(aConfigFilePath)
# do this if backSub
self._doBackgroundRefresh()
#Overwrites the ROI image from the processor object
#if the ROI is downsampled (if the ROI is used)
self.downSampleROI('ROI.jpg', self.downLevel)
#Send starting SMS
if self.textMessageRecipient is not None:
self.sms = TextMessage()
self.trueMotionCount = 0
self.lastTextMessageSendTime = time.time()
self.sms.connectPhone()
self.sms.setRecipient(str(self.textMessageRecipient))
self.sms.setContent("FoxBox On")
self.sms.sendMessage()
self.sms.disconnectPhone()
def store_data(self):
rowtitle = ["Product Name", "Price", "Rating", "No. of reviews", "Delivery Date", "Delivery Type", "Buy link"]
w = DataWriter('Amazon.csv', rowtitle)
w.writer(self.details)
def store_data(self):
rowtitle = ["Product Name", "Price", "Rating", "No. of reviews", "Buy link"]
w = DataWriter("Flipkart.csv", rowtitle)
w.writer(self.details)
