def generate_data(household, start, end):
raw_data = generate_data.generate_data_range(
iResidents=household["no_residents"],
Dwell=household["appliance_status"],
iIrradianceThreshold = household["lights_irradiance"],
iRandomHouse = household["lights_house"],
from_date=start,
to_date=end)
header = raw_data[0][3:]
return [{'sensor_name': header[i], 'usage': v*1000, 'n_measurements': 1, 'timestamp': start} for i, v in enumerate(raw_data[1][3:])]
def generate_averages(household, appliance_status, from_date, to_date):
raw_data = generate_data.generate_data_range(
iResidents=household["no_residents"],
Dwell=appliance_status,
iIrradianceThreshold = household["lights_irradiance"],
iRandomHouse = household["lights_house"],
from_date=from_date,
to_date=to_date)
matrix = numpy.array(raw_data[1:])
matrix = matrix[:, 3:]
header = raw_data[0][3:]
return [{'sensor_name': header[i], 'average': v*1000, 'count': (len(raw_data)-1)} for i, v in enumerate(numpy.mean(matrix, axis=0))]
def generate_sensor_data(config_file: str, from_date: datetime,
to_date: datetime):
print("generate_sensor_date(config_file=%s, from=%s, to=%s" %
(config_file, from_date, to_date))
random.seed(datetime.now()) # different result each time
json_file = open(config_file, "r")
config_data = json.load(json_file)
json_file.close()
config = None
for config in config_data:
appliance_status = []
# Edits by Evert
print("Loading household {}".format(config["id_household"]))
sensors = []
for name in config["appliances"]:
sensor = Sensor(name)
sensors.append(sensor)
print(
" - loaded sensor with ID {s.ID} with appliance name `{s.appname}`"
.format(s=sensor))
lightSensor = LightSensor(config["lights_id"])
#for row in appliance.appliances:
#status = "NO"
#name = row[0]
#if name in sensors:
#status = "YES"
#appliance_status.append(status)
data = generate_data.generate_data_range(
iResidents=config["no_residents"],
Sensors=sensors,
lightSensor=lightSensor,
iIrradianceThreshold=config["lights_irradiance"],
iRandomHouse=config["lights_house"],
from_date=from_date,
to_date=to_date)
outputfile = "data_house_{}.csv".format(config["id_household"])
save_file = open(outputfile, 'w')
for row in data:
for col in row:
save_file.write(str(col))
save_file.write(";")
save_file.write("\n")
save_file.close()
print("saved %s" % outputfile)
def generate_sensor_data(config_file: str, from_date: datetime, to_date: datetime):
print("generate_sensor_date(config_file=%s, from=%s, to=%s" % (
config_file, from_date, to_date))
random.seed(datetime.now()) # different result each time
json_file = open(config_file, "r")
config_data = json.load(json_file)
json_file.close()
config = None
for config in config_data:
appliance_status = []
# Edits by Evert
print("Loading household {}".format(config["id_household"]))
sensors = []
for name in config["appliances"]:
sensor = Sensor(name)
sensors.append(sensor)
print(" - loaded sensor with ID {s.ID} with appliance name `{s.appname}`".format(s=sensor))
lightSensor = LightSensor(config["lights_id"])
#for row in appliance.appliances:
#status = "NO"
#name = row[0]
#if name in sensors:
#status = "YES"
#appliance_status.append(status)
data = generate_data.generate_data_range(
iResidents=config["no_residents"],
Sensors=sensors,
lightSensor=lightSensor,
iIrradianceThreshold=config["lights_irradiance"],
iRandomHouse=config["lights_house"],
from_date=from_date,
to_date=to_date)
outputfile = "data_house_{}.csv".format(config["id_household"])
save_file = open(outputfile, 'w')
for row in data:
for col in row:
save_file.write(str(col))
save_file.write(";")
save_file.write("\n")
save_file.close()
print("saved %s" % outputfile)
def generate_averages(household, appliance_status, from_date, to_date):
raw_data = generate_data.generate_data_range(
iResidents=household["no_residents"],
Dwell=appliance_status,
iIrradianceThreshold=household["lights_irradiance"],
iRandomHouse=household["lights_house"],
from_date=from_date,
to_date=to_date)
matrix = numpy.array(raw_data[1:])
matrix = matrix[:, 3:]
header = raw_data[0][3:]
return [{
'sensor_name': header[i],
'average': v * 1000
} for i, v in enumerate(numpy.mean(matrix, axis=0))]
def populate_data(config, from_date, to_date):
data = {}
for household in config:
household_id = household['id_household']
print("Generating household " + str(household_id))
appliance_status = []
for row in appliance.appliances:
status = "NO"
if row[0] in household["appliances"]:
status = "YES"
appliance_status.append(status)
data[household_id] = { 'yearly':{}, 'monthly':{}, 'daily':{}, 'recent':{} }
for year in range(from_date.year, to_date.year):
start_date = datetime(year, 1, 1)
data[household_id]['yearly'][start_date] = generate_averages( household, appliance_status, from_date, datetime(year+1, 1, 1) )
for month in range(1, to_date.month):
start_date = datetime(to_date.year, month, 1)
if start_date < from_date:
continue
data[household_id]['monthly'][start_date] = generate_averages( household, appliance_status, start_date, datetime(to_date.year, month+1, 1) )
for day in range(1, to_date.day):
start_date = datetime(to_date.year, to_date.month, day)
if start_date < from_date:
continue
data[household_id]['daily'][start_date] = generate_averages( household, appliance_status, start_date, datetime(to_date.year, to_date.month, day+1) )
start_date = max(to_date.replace(hour=0, minute=0, second=0, microsecond=0), from_date)
raw_data = generate_data.generate_data_range(
iResidents=household["no_residents"],
Dwell=appliance_status,
iIrradianceThreshold = household["lights_irradiance"],
iRandomHouse = household["lights_house"],
from_date=start_date,
to_date=to_date)
header = raw_data[0][3:]
data[household_id]['recent'] = [ {v[0]:[{'sensor_name':header[i], 'average': v[i]*1000} for i in range(3, len(header))]} for v in raw_data[1:] ]
return data
def populate_data(config, from_date, to_date):
data = {}
for household in config:
household_id = household['id_household']
print("Generating household " + str(household_id))
appliance_status = []
for row in appliance.appliances:
status = "NO"
if row[0] in household["appliances"]:
status = "YES"
appliance_status.append(status)
data[household_id] = {
'yearly': {},
'monthly': {},
'daily': {},
'recent': {}
}
for year in range(from_date.year, to_date.year):
start_date = datetime(year, 1, 1)
data[household_id]['yearly'][start_date] = generate_averages(
household, appliance_status, from_date,
datetime(year + 1, 1, 1))
for month in range(1, to_date.month):
start_date = datetime(to_date.year, month, 1)
if start_date < from_date:
continue
data[household_id]['monthly'][start_date] = generate_averages(
household, appliance_status, start_date,
datetime(to_date.year, month + 1, 1))
for day in range(1, to_date.day):
start_date = datetime(to_date.year, to_date.month, day)
if start_date < from_date:
continue
data[household_id]['daily'][start_date] = generate_averages(
household, appliance_status, start_date,
datetime(to_date.year, to_date.month, day + 1))
start_date = max(
to_date.replace(hour=0, minute=0, second=0, microsecond=0),
from_date)
raw_data = generate_data.generate_data_range(
iResidents=household["no_residents"],
Dwell=appliance_status,
iIrradianceThreshold=household["lights_irradiance"],
iRandomHouse=household["lights_house"],
from_date=start_date,
to_date=to_date)
header = raw_data[0][3:]
data[household_id]['recent'] = [{
v[0]: [{
'sensor_name': header[i],
'average': v[i] * 1000
} for i in range(3, len(header))]
} for v in raw_data[1:]]
return data
def populate_data(config, from_date, to_date):
data = {'yearly':[], 'monthly':[], 'daily':[], 'recent':[]}
for household in config:
household_id = household['id_household']
print("Generating household " + str(household_id))
appliance_status = []
for row in appliance.appliances:
status = "NO"
if row[0] in household["appliances"]:
status = "YES"
appliance_status.append(status)
sensor_list = Sensor.objects.filter(home=household_id).values('name', 'id')
sensor_map = {v['name']:v['id'] for v in sensor_list}
for year in range(from_date.year, to_date.year):
start_date = datetime(year, 1, 1)
averages = generate_averages( household, appliance_status, start_date, datetime(year+1, 1, 1) )
data['yearly'] += [YearlyData(sensor_id=sensor_map[ v['sensor_name'] ], usage=v['average'], n_measurements=v['count'], timestamp=start_date) for v in averages]
monthly_start = to_date.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - relativedelta(years=1)
for month in range(12):
start_date = monthly_start + relativedelta(months=month)
stop_date = monthly_start + relativedelta(months=month+1)
if start_date < from_date:
continue
averages = generate_averages( household, appliance_status, start_date, stop_date )
data['monthly'] += [MonthlyData(sensor_id=sensor_map[ v['sensor_name'] ], usage=v['average'], n_measurements=v['count'], timestamp=start_date) for v in averages]
daily_start = to_date.replace(hour=0, minute=0, second=0, microsecond=0) - relativedelta(months=1)
for day in range((to_date-daily_start).days):
start_date = daily_start + timedelta(days=day)
stop_date = daily_start + timedelta(days=day+1)
if start_date < from_date:
continue
averages = generate_averages( household, appliance_status, start_date, stop_date )
data['daily'] += [DailyData(sensor_id=sensor_map[ v['sensor_name'] ], usage=v['average'], n_measurements=v['count'], timestamp=start_date) for v in averages]
start_date = max(to_date.replace(hour=0, minute=0, second=0, microsecond=0), from_date)
raw_data = generate_data.generate_data_range(
iResidents=household["no_residents"],
Dwell=appliance_status,
iIrradianceThreshold = household["lights_irradiance"],
iRandomHouse = household["lights_house"],
from_date=start_date,
to_date=to_date)
header = raw_data[0][3:]
for v in raw_data[1:]:
data['recent'] += [ RecentData(sensor_id=sensor_map[ header[i] ], usage=v[i]*1000, n_measurements=1, timestamp=v[0]) for i in range(3, len(header)) ]
#data_count = len(data)
#print("Saving " + str(data_count) + " objects...")
#data_1000 = data_count/1000
#for i, datum in enumerate(data):
# if ((i % data_1000) == 0):
# print(str(datetime.now()) + " - progress: " + str(float(i*100)/float(data_count)) + "%")
# super(SensorData, datum).save()
print("Saving " + str(len(data['yearly'])) + " yearly data entries...")
YearlyData.objects.bulk_create(data['yearly'])
print("Saving " + str(len(data['monthly'])) + " monthly data entries...")
MonthlyData.objects.bulk_create(data['monthly'])
print("Saving " + str(len(data['daily'])) + " daily data entries...")
DailyData.objects.bulk_create(data['daily'])
print("Saving " + str(len(data['recent'])) + " recent data entries...")
RecentData.objects.bulk_create(data['recent'])
return data
def populate_data(config, from_date, to_date):
data = {'yearly': [], 'monthly': [], 'daily': [], 'recent': []}
for household in config:
household_id = household['id_household']
print("Generating household " + str(household_id))
appliance_status = []
for row in appliance.appliances:
status = "NO"
if row[0] in household["appliances"]:
status = "YES"
appliance_status.append(status)
sensor_list = Sensor.objects.filter(home=household_id).values(
'name', 'id')
sensor_map = {v['name']: v['id'] for v in sensor_list}
for year in range(from_date.year, to_date.year):
start_date = datetime(year, 1, 1)
averages = generate_averages(household, appliance_status,
start_date, datetime(year + 1, 1, 1))
data['yearly'] += [
YearlyData(sensor_id=sensor_map[v['sensor_name']],
usage=v['average'],
n_measurements=v['count'],
timestamp=start_date) for v in averages
]
monthly_start = to_date.replace(
day=1, hour=0, minute=0, second=0,
microsecond=0) - relativedelta(years=1)
for month in range(12):
start_date = monthly_start + relativedelta(months=month)
stop_date = monthly_start + relativedelta(months=month + 1)
if start_date < from_date:
continue
averages = generate_averages(household, appliance_status,
start_date, stop_date)
data['monthly'] += [
MonthlyData(sensor_id=sensor_map[v['sensor_name']],
usage=v['average'],
n_measurements=v['count'],
timestamp=start_date) for v in averages
]
daily_start = to_date.replace(
hour=0, minute=0, second=0,
microsecond=0) - relativedelta(months=1)
for day in range((to_date - daily_start).days):
start_date = daily_start + timedelta(days=day)
stop_date = daily_start + timedelta(days=day + 1)
if start_date < from_date:
continue
averages = generate_averages(household, appliance_status,
start_date, stop_date)
data['daily'] += [
DailyData(sensor_id=sensor_map[v['sensor_name']],
usage=v['average'],
n_measurements=v['count'],
timestamp=start_date) for v in averages
]
start_date = max(
to_date.replace(hour=0, minute=0, second=0, microsecond=0),
from_date)
raw_data = generate_data.generate_data_range(
iResidents=household["no_residents"],
Dwell=appliance_status,
iIrradianceThreshold=household["lights_irradiance"],
iRandomHouse=household["lights_house"],
from_date=start_date,
to_date=to_date)
header = raw_data[0][3:]
for v in raw_data[1:]:
data['recent'] += [
RecentData(sensor_id=sensor_map[header[i]],
usage=v[i] * 1000,
n_measurements=1,
timestamp=v[0]) for i in range(3, len(header))
]
#data_count = len(data)
#print("Saving " + str(data_count) + " objects...")
#data_1000 = data_count/1000
#for i, datum in enumerate(data):
# if ((i % data_1000) == 0):
# print(str(datetime.now()) + " - progress: " + str(float(i*100)/float(data_count)) + "%")
# super(SensorData, datum).save()
print("Saving " + str(len(data['yearly'])) + " yearly data entries...")
YearlyData.objects.bulk_create(data['yearly'])
print("Saving " + str(len(data['monthly'])) + " monthly data entries...")
MonthlyData.objects.bulk_create(data['monthly'])
print("Saving " + str(len(data['daily'])) + " daily data entries...")
DailyData.objects.bulk_create(data['daily'])
print("Saving " + str(len(data['recent'])) + " recent data entries...")
RecentData.objects.bulk_create(data['recent'])
return data
