def test_15_minutes():
"""should be able to write 15 minute files, and read them back correctly."""
m = NEM12(to_participant="123")
readings = [[
datetime(2004, 4, 18, 0, 0) + timedelta(minutes=15 * (i + 1)), i, "A"
] for i in range(24 * 4)]
m.add_readings(
nmi="123",
nmi_configuration="E1B1B2",
nmi_suffix="E1",
uom="kWh",
readings=readings,
)
output_file = f"tests/{m.nem_filename()}.zip"
m.output_zip(output_file)
readback = nr.read_nem_file(output_file)
readback_readings = readback.readings["123"]["E1"]
assert len(readback_readings) == len(readings)
for i in range(len(readback_readings)):
assert readback_readings[
i].t_end == readback_readings[i].t_start + timedelta(minutes=15)
assert readback_readings[i].t_end == readings[i][0]
assert readback_readings[i].read_value == readings[i][1]
assert readback_readings[i].quality_method == readings[i][2]
def test_Date12_parse():
""" test that Date12 is parsed correctly """
# 200402070911 = 2004-02-07, 09:11
meter_data = nr.read_nem_file(
os.path.abspath("examples/unzipped/Example_NEM12_multiple_meters.csv"))
assert meter_data.header.creation_date == datetime.datetime(
2004, 2, 7, 9, 11)
def import_export_nem13(input_file):
"""Create export from import"""
# Read in example
ex = nr.read_nem_file(input_file)
# Create output file
m = NEM13(to_participant=ex.header.to_participant)
# Add in interval readings
for nmi in ex.readings:
nmi_configuration = "".join(list(ex.transactions[nmi].keys()))
for channel in ex.readings[nmi]:
# Build list of readings
for read in ex.readings[nmi][channel]:
m.add_reading(
nmi=nmi,
nmi_configuration=nmi_configuration,
register_id=None,
nmi_suffix=channel,
previous_read=read.val_start,
previous_read_date=read.t_start,
current_read=read.val_end,
current_read_date=read.t_end,
current_quality_method=read.quality_method,
quantity=read.read_value,
uom=read.uom,
)
# Export to file
output_file = f"tests/{m.nem_filename()}.csv"
output = m.output_csv(file_path=output_file)
return output
def test_correct_records():
meter_data = nr.read_nem_file(
'examples/unzipped/Example_NEM12_multiple_quality.csv')
readings = meter_data.readings['CCCC123456']['E1']
assert len(readings) == 48
assert readings[0].read_value == pytest.approx(18.023, 0.1)
assert readings[-1].read_value == pytest.approx(14.733, 0.1)
def test_15_minutes():
""" should be able to write 15 minute files, and read them back correctly.
"""
m = NEM12(to_participant='123')
readings = [[
datetime.datetime(2004, 4, 18, 0, 0) +
datetime.timedelta(minutes=15 * (i + 1)), i, 'A'
] for i in range(24 * 4)]
m.add_readings(nmi='123',
nmi_configuration='E1B1B2',
nmi_suffix='E1',
uom='kWh',
interval_length=15,
readings=readings)
output_filename = "tests/test_output_15_minute.csv"
m.nem_output(file_name=output_filename)
readback = nr.read_nem_file(output_filename)
readback_readings = readback.readings["123"]["E1"]
assert len(readback_readings) == len(readings)
for i in range(len(readback_readings)):
assert readback_readings[i].t_end == readback_readings[
i].t_start + datetime.timedelta(minutes=15)
assert readback_readings[i].t_end == readings[i][0]
assert readback_readings[i].read_value == readings[i][1]
assert readback_readings[i].quality_method == readings[i][2]
def test_correct_channels():
meter_data = nr.read_nem_file(
'examples/unzipped/Example_NEM12_actual_interval.csv')
readings = meter_data.readings['VABD000163']
assert len(readings) == 2
assert 'E1' in readings
assert 'Q1' in readings
def test_unzipped_examples():
""" Open and parse unzipped example files """
test_path = os.path.abspath('examples/unzipped')
for file_name in os.listdir(test_path):
test_file = os.path.join(test_path, file_name)
meter_data = nr.read_nem_file(test_file)
assert meter_data.header.version_header in ['NEM12', 'NEM13']
def test_correct_channels():
meter_data = nr.read_nem_file(
"examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True
)
readings = meter_data.readings["VABD000163"]
assert len(readings) == 2
assert "E1" in readings
assert "Q1" in readings
def test_nem13_examples():
""" Open and parse zipped NEM13 example files """
test_path = os.path.abspath("examples/nem13")
for file_name in os.listdir(test_path):
test_file = os.path.join(test_path, file_name)
meter_data = nr.read_nem_file(test_file)
assert meter_data.header.version_header in ["NEM12", "NEM13"]
def export_nem12(example):
""" Read in a NEM12 file and check if it is exported the same """
# Read in example
example_file = 'tests/{}.csv'.format(example)
ex = nr.read_nem_file(example_file)
# Create output file
output_file = 'tests/test_output_{}.csv'.format(example)
m = NEM12(to_participant=ex.header.to_participant)
# Add in interval readings
to_load = []
for nmi in ex.readings:
nmi_configuration = ''.join(list(ex.transactions[nmi].keys()))
for channel in ex.readings[nmi]:
# Build list of readings
for read in ex.readings[nmi][channel]:
to_load.append([
read.t_end, read.read_value, read.quality_method,
read.event_code, read.event_desc
])
# Get common atributes from last reading
last = ex.readings[nmi][channel][-1:][0]
uom = last.uom
interval_length = int((last.t_end - last.t_start).seconds / 60)
ch = m.add_readings(nmi=nmi,
nmi_configuration=nmi_configuration,
nmi_suffix=channel,
uom=uom,
interval_length=interval_length,
readings=to_load)
# Export to file
output = m.nem_output(file_name=output_file)
# Compare files
original = []
with open(example_file, newline='') as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar='"')
for row in reader:
original.append(row)
output = []
with open(output_file, newline='') as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar='"')
for row in reader:
output.append(row)
# Compare rows
for i, row in enumerate(original):
record_indicator = row[0]
if record_indicator not in ['100', '200']:
for j, col in enumerate(row):
if j not in [4, 5, 53, 54]:
assert col.rstrip('0') == output[i][j].rstrip(
'0'), f'[{i},{j}] did not match'
def test_nem13_readings():
""" Test the NEM13 consumption data """
meter_data = nr.read_nem_file(
"examples/unzipped/Example_NEM13_consumption_data.csv")
readings = meter_data.readings["VABC005890"]["11"]
unit_count = 0
for record in readings:
unit_count += record[2] # Read value should be third record in tuple
assert unit_count == pytest.approx(1312.1, 0.1)
def print_meter_record(file_path, rows=5):
""" Output readings for specified number of rows to console """
m = nr.read_nem_file(file_path)
print('Header:', m.header)
print('Transactions:', m.transactions)
for nmi in m.readings:
for channel in m.readings[nmi]:
print(nmi, 'Channel', channel)
for reading in m.readings[nmi][channel][-rows:]:
print('', reading)
def test_nem12_examples():
""" Open and parse zipped NEM12 example files """
skips = ["NEM12#Scenario10#ETSAMDP#NEMMCO.zip"] # 300 Row has new line
test_path = os.path.abspath("examples/nem12")
for file_name in os.listdir(test_path):
if file_name in skips:
continue
test_file = os.path.join(test_path, file_name)
meter_data = nr.read_nem_file(test_file)
assert meter_data.header.version_header in ["NEM12", "NEM13"]
def test_importexport_zippednem12():
"""Check that a zipped NEM12 is unchanged"""
example_file = "examples/year_example.csv"
output_file = import_export_nem12(example_file, zip_output=True)
f1 = nr.read_nem_file(example_file)
f2 = nr.read_nem_file(output_file)
nmi = "NMIxxxxxxx"
ch = "E1"
for i, read1 in enumerate(f1.readings[nmi][ch]):
read2 = f2.readings[nmi][ch][i]
# Cannot compare all attributes as meter serial number is not included
assert read1.t_start == read2.t_start
assert read1.read_value == read2.read_value
assert read1.quality_method == read2.quality_method
assert read1.event_code == read2.event_code
assert read1.event_desc == read2.event_desc
def export_nem13(example):
""" Read in a NEM13 file and check if it is exported the same """
# Read in example
example_file = 'tests/{}.csv'.format(example)
ex = nr.read_nem_file(example_file)
# Create output file
output_file = 'tests/test_output_{}.csv'.format(example)
m = NEM13(to_participant=ex.header.to_participant)
# Add in interval readings
to_load = []
for nmi in ex.readings:
nmi_configuration = ''.join(list(ex.transactions[nmi].keys()))
for channel in ex.readings[nmi]:
# Build list of readings
for read in ex.readings[nmi][channel]:
ch = m.add_reading(nmi=nmi,
nmi_configuration=nmi_configuration,
register_id=None,
nmi_suffix=channel,
previous_read=read.read_start,
previous_read_date=read.t_start,
current_read=read.read_end,
current_read_date=read.t_end,
current_quality_method=read.quality_method,
quantity=read.read_value,
uom=read.uom
)
# Export to file
output = m.nem_output(file_name=output_file)
# Compare files
original = []
with open(example_file, newline='') as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar='"')
for row in reader:
original.append(row)
output = []
with open(output_file, newline='') as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar='"')
for row in reader:
output.append(row)
# Compare rows
for i, row in enumerate(original):
record_indicator = row[0]
if record_indicator not in ['100', '200']:
for j, col in enumerate(row):
if j not in [3, 5, 6, 10, 21, 22]:
assert cleanse_val(col) == cleanse_val(output[i][j]), '[{i},{j}] did not match'.format(i=i, j=j)
def test_unzipped_examples():
""" Open and parse unzipped example files """
skips = ["Example_NEM12_powercor.csv", "Example_NEM12_powercor_missing_fields.csv"]
test_path = os.path.abspath("examples/unzipped")
for file_name in os.listdir(test_path):
print(file_name)
if file_name in skips:
continue
test_file = os.path.join(test_path, file_name)
meter_data = nr.read_nem_file(test_file)
assert meter_data.header.version_header in ["NEM12", "NEM13"]
def test_correct_records():
meter_data = nr.read_nem_file("examples/unzipped/Example_NEM12_actual_interval.csv")
readings = meter_data.readings["VABD000163"]
assert len(readings["E1"]) == 48
assert readings["E1"][10].read_value == pytest.approx(1.11, 0.1)
assert readings["E1"][-1].read_value == pytest.approx(1.11, 0.1)
assert len(readings["Q1"]) == 48
assert readings["Q1"][10].read_value == pytest.approx(2.22, 0.1)
assert readings["Q1"][-1].read_value == pytest.approx(2.22, 0.1)
assert readings["Q1"][-1].quality_method == "A"
def test_correct_records():
meter_data = nr.read_nem_file(
'examples/unzipped/Example_NEM12_actual_interval.csv')
readings = meter_data.readings['VABD000163']
assert len(readings['E1']) == 48
assert readings['E1'][10].read_value == pytest.approx(1.11, 0.1)
assert readings['E1'][-1].read_value == pytest.approx(1.11, 0.1)
assert len(readings['Q1']) == 48
assert readings['Q1'][10].read_value == pytest.approx(2.22, 0.1)
assert readings['Q1'][-1].read_value == pytest.approx(2.22, 0.1)
assert readings['Q1'][-1].quality_method == 'A'
def test_correct_quality():
meter_data = nr.read_nem_file(
'examples/unzipped/Example_NEM12_multiple_quality.csv')
readings = meter_data.readings['CCCC123456']['E1']
assert readings[0].quality_method == 'F14'
assert readings[10].quality_method == 'F14'
assert readings[19].quality_method == 'F14'
assert readings[20].quality_method == 'A'
assert readings[22].quality_method == 'A'
assert readings[23].quality_method == 'A'
assert readings[25].quality_method == 'S14'
assert readings[30].quality_method == 'S14'
assert readings[47].quality_method == 'S14'
def test_correct_quality():
meter_data = nr.read_nem_file(
"examples/unzipped/Example_NEM12_multiple_quality.csv")
readings = meter_data.readings["CCCC123456"]["E1"]
assert readings[0].quality_method == "F14"
assert readings[10].quality_method == "F14"
assert readings[19].quality_method == "F14"
assert readings[20].quality_method == "A"
assert readings[22].quality_method == "A"
assert readings[23].quality_method == "A"
assert readings[25].quality_method == "S14"
assert readings[30].quality_method == "S14"
assert readings[47].quality_method == "S14"
def test_correct_records():
meter_data = nr.read_nem_file(
"examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True
)
readings = meter_data.readings["VABD000163"]
assert len(readings["E1"]) == 96
assert readings["E1"][10].read_value == pytest.approx(1.11, 0.1)
assert readings["E1"][-1].read_value == pytest.approx(3.33, 0.1)
assert len(readings["Q1"]) == 96
assert readings["Q1"][10].read_value == pytest.approx(2.22, 0.1)
assert readings["Q1"][-1].read_value == pytest.approx(4.44, 0.1)
assert readings["Q1"][-1].quality_method == "A"
def import_export_nem12(input_file, zip_output=False):
"""Create export from import"""
# Read in example
ex = nr.read_nem_file(input_file)
# Create output file
m = NEM12(to_participant=ex.header.to_participant)
# Add in interval readings
for nmi in ex.readings:
nmi_configuration = "".join(list(ex.transactions[nmi].keys()))
for nmi_suffix in ex.readings[nmi]:
to_load = []
# Build list of readings
for read in ex.readings[nmi][nmi_suffix]:
to_load.append([
read.t_end,
read.read_value,
read.quality_method,
read.event_code,
read.event_desc,
])
# Get common atributes from last reading
last = ex.readings[nmi][nmi_suffix][-1:][0]
uom = last.uom
m.add_readings(
nmi=nmi,
nmi_configuration=nmi_configuration,
nmi_suffix=nmi_suffix,
uom=uom,
readings=to_load,
)
# Export to file
if zip_output:
output_file = f"tests/{m.nem_filename()}.zip"
output = m.output_zip(file_path=output_file)
else:
output_file = f"tests/{m.nem_filename()}.csv"
output = m.output_csv(file_path=output_file)
return output
def output_as_csv(file_name, nmi=None, output_file=None):
"""
Transpose all channels and output a csv that is easier
to read and do charting on
:param file_name: The NEM file to process
:param nmi: Which NMI to output if more than one
:param output_file: Specify different output location
:returns: The file that was created
"""
m = read_nem_file(file_name)
if nmi is None:
nmi = list(m.readings.keys())[0] # Use first NMI
channels = list(m.transactions[nmi].keys())
num_records = len(m.readings[nmi][channels[0]])
last_date = m.readings[nmi][channels[0]][-1].t_end
if output_file is None:
output_file = '{}_{}_transposed.csv'.format(
nmi, last_date.strftime('%Y%m%d'))
with open(output_file, 'w', newline='') as csvfile:
cwriter = csv.writer(csvfile,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
heading_list = ['period_start', 'period_end']
for channel in channels:
heading_list.append(channel)
heading_list.append('quality_method')
cwriter.writerow(heading_list)
for i in range(0, num_records):
t_start = m.readings[nmi][channels[0]][i].t_start
t_end = m.readings[nmi][channels[0]][i].t_end
quality_method = m.readings[nmi][channels[0]][i].quality_method
row_list = [t_start, t_end]
for ch in channels:
val = m.readings[nmi][ch][i].read_value
row_list.append(val)
row_list.append(quality_method)
cwriter.writerow(row_list)
return output_file
def test_optional_scheduled_read():
meter_data = nr.read_nem_file(
"examples/unzipped/Example_NEM12_no_scheduled_read.csv")
readings = meter_data.readings["NMI111"]["E1"]
assert len(readings) == 96
def test_correct_channels():
meter_data = nr.read_nem_file("examples/unzipped/Example_NEM12_actual_interval.csv")
readings = meter_data.readings["VABD000163"]
assert len(readings) == 2
assert "E1" in readings
assert "Q1" in readings
def test_correct_NMIs():
meter_data = nr.read_nem_file("examples/unzipped/Example_NEM12_actual_interval.csv")
assert len(meter_data.readings) == 1
assert "VABD000163" in meter_data.readings
def test_missing_fields():
meter_data = nr.read_nem_file(
"examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True
)
def test_zipped_load():
meter_data = nr.read_nem_file(
"examples/nem12/Example_NEM12_powercor.csv.zip", ignore_missing_header=True
)
def test_correct_NMIs():
meter_data = nr.read_nem_file(
"examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True
)
assert len(meter_data.readings) == 1
assert "VABD000163" in meter_data.readings
""" Test Suite
"""
import pytest
import datetime
from nemreader import read_nem_file
import os
import sys
sys.path.insert(0,
os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from qldtariffs import get_daily_charges, get_monthly_charges
INTERVAL_READINGS = read_nem_file(
'examples/example_NEM12.csv').readings['3044076134']['E1']
MANUAL_READINGS = read_nem_file(
'examples/example_NEM13.csv').readings['3044076134']['11']
def test_nem12_data_daily():
""" Test example NEM12 daily summary """
daily_summaries = get_daily_charges(INTERVAL_READINGS, 'ergon', 't14')
dec1 = daily_summaries[datetime.date(2016, 12, 1)]
assert dec1.total == pytest.approx(6.29)
daily_summaries = get_daily_charges(INTERVAL_READINGS, 'ergon', 't14')
dec1 = daily_summaries[datetime.date(2016, 9, 6)]
assert dec1.total == pytest.approx(7.17)
def test_nem12_data_monthly():
