def test_15_minutes(): """should be able to write 15 minute files, and read them back correctly.""" m = NEM12(to_participant="123") readings = [[ datetime(2004, 4, 18, 0, 0) + timedelta(minutes=15 * (i + 1)), i, "A" ] for i in range(24 * 4)] m.add_readings( nmi="123", nmi_configuration="E1B1B2", nmi_suffix="E1", uom="kWh", readings=readings, ) output_file = f"tests/{m.nem_filename()}.zip" m.output_zip(output_file) readback = nr.read_nem_file(output_file) readback_readings = readback.readings["123"]["E1"] assert len(readback_readings) == len(readings) for i in range(len(readback_readings)): assert readback_readings[ i].t_end == readback_readings[i].t_start + timedelta(minutes=15) assert readback_readings[i].t_end == readings[i][0] assert readback_readings[i].read_value == readings[i][1] assert readback_readings[i].quality_method == readings[i][2]
def test_Date12_parse(): """ test that Date12 is parsed correctly """ # 200402070911 = 2004-02-07, 09:11 meter_data = nr.read_nem_file( os.path.abspath("examples/unzipped/Example_NEM12_multiple_meters.csv")) assert meter_data.header.creation_date == datetime.datetime( 2004, 2, 7, 9, 11)
def import_export_nem13(input_file): """Create export from import""" # Read in example ex = nr.read_nem_file(input_file) # Create output file m = NEM13(to_participant=ex.header.to_participant) # Add in interval readings for nmi in ex.readings: nmi_configuration = "".join(list(ex.transactions[nmi].keys())) for channel in ex.readings[nmi]: # Build list of readings for read in ex.readings[nmi][channel]: m.add_reading( nmi=nmi, nmi_configuration=nmi_configuration, register_id=None, nmi_suffix=channel, previous_read=read.val_start, previous_read_date=read.t_start, current_read=read.val_end, current_read_date=read.t_end, current_quality_method=read.quality_method, quantity=read.read_value, uom=read.uom, ) # Export to file output_file = f"tests/{m.nem_filename()}.csv" output = m.output_csv(file_path=output_file) return output
def test_correct_records(): meter_data = nr.read_nem_file( 'examples/unzipped/Example_NEM12_multiple_quality.csv') readings = meter_data.readings['CCCC123456']['E1'] assert len(readings) == 48 assert readings[0].read_value == pytest.approx(18.023, 0.1) assert readings[-1].read_value == pytest.approx(14.733, 0.1)
def test_15_minutes(): """ should be able to write 15 minute files, and read them back correctly. """ m = NEM12(to_participant='123') readings = [[ datetime.datetime(2004, 4, 18, 0, 0) + datetime.timedelta(minutes=15 * (i + 1)), i, 'A' ] for i in range(24 * 4)] m.add_readings(nmi='123', nmi_configuration='E1B1B2', nmi_suffix='E1', uom='kWh', interval_length=15, readings=readings) output_filename = "tests/test_output_15_minute.csv" m.nem_output(file_name=output_filename) readback = nr.read_nem_file(output_filename) readback_readings = readback.readings["123"]["E1"] assert len(readback_readings) == len(readings) for i in range(len(readback_readings)): assert readback_readings[i].t_end == readback_readings[ i].t_start + datetime.timedelta(minutes=15) assert readback_readings[i].t_end == readings[i][0] assert readback_readings[i].read_value == readings[i][1] assert readback_readings[i].quality_method == readings[i][2]
def test_correct_channels(): meter_data = nr.read_nem_file( 'examples/unzipped/Example_NEM12_actual_interval.csv') readings = meter_data.readings['VABD000163'] assert len(readings) == 2 assert 'E1' in readings assert 'Q1' in readings
def test_unzipped_examples(): """ Open and parse unzipped example files """ test_path = os.path.abspath('examples/unzipped') for file_name in os.listdir(test_path): test_file = os.path.join(test_path, file_name) meter_data = nr.read_nem_file(test_file) assert meter_data.header.version_header in ['NEM12', 'NEM13']
def test_correct_channels(): meter_data = nr.read_nem_file( "examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True ) readings = meter_data.readings["VABD000163"] assert len(readings) == 2 assert "E1" in readings assert "Q1" in readings
def test_nem13_examples(): """ Open and parse zipped NEM13 example files """ test_path = os.path.abspath("examples/nem13") for file_name in os.listdir(test_path): test_file = os.path.join(test_path, file_name) meter_data = nr.read_nem_file(test_file) assert meter_data.header.version_header in ["NEM12", "NEM13"]
def export_nem12(example): """ Read in a NEM12 file and check if it is exported the same """ # Read in example example_file = 'tests/{}.csv'.format(example) ex = nr.read_nem_file(example_file) # Create output file output_file = 'tests/test_output_{}.csv'.format(example) m = NEM12(to_participant=ex.header.to_participant) # Add in interval readings to_load = [] for nmi in ex.readings: nmi_configuration = ''.join(list(ex.transactions[nmi].keys())) for channel in ex.readings[nmi]: # Build list of readings for read in ex.readings[nmi][channel]: to_load.append([ read.t_end, read.read_value, read.quality_method, read.event_code, read.event_desc ]) # Get common atributes from last reading last = ex.readings[nmi][channel][-1:][0] uom = last.uom interval_length = int((last.t_end - last.t_start).seconds / 60) ch = m.add_readings(nmi=nmi, nmi_configuration=nmi_configuration, nmi_suffix=channel, uom=uom, interval_length=interval_length, readings=to_load) # Export to file output = m.nem_output(file_name=output_file) # Compare files original = [] with open(example_file, newline='') as csvfile: reader = csv.reader(csvfile, delimiter=',', quotechar='"') for row in reader: original.append(row) output = [] with open(output_file, newline='') as csvfile: reader = csv.reader(csvfile, delimiter=',', quotechar='"') for row in reader: output.append(row) # Compare rows for i, row in enumerate(original): record_indicator = row[0] if record_indicator not in ['100', '200']: for j, col in enumerate(row): if j not in [4, 5, 53, 54]: assert col.rstrip('0') == output[i][j].rstrip( '0'), f'[{i},{j}] did not match'
def test_nem13_readings(): """ Test the NEM13 consumption data """ meter_data = nr.read_nem_file( "examples/unzipped/Example_NEM13_consumption_data.csv") readings = meter_data.readings["VABC005890"]["11"] unit_count = 0 for record in readings: unit_count += record[2] # Read value should be third record in tuple assert unit_count == pytest.approx(1312.1, 0.1)
def print_meter_record(file_path, rows=5): """ Output readings for specified number of rows to console """ m = nr.read_nem_file(file_path) print('Header:', m.header) print('Transactions:', m.transactions) for nmi in m.readings: for channel in m.readings[nmi]: print(nmi, 'Channel', channel) for reading in m.readings[nmi][channel][-rows:]: print('', reading)
def test_nem12_examples(): """ Open and parse zipped NEM12 example files """ skips = ["NEM12#Scenario10#ETSAMDP#NEMMCO.zip"] # 300 Row has new line test_path = os.path.abspath("examples/nem12") for file_name in os.listdir(test_path): if file_name in skips: continue test_file = os.path.join(test_path, file_name) meter_data = nr.read_nem_file(test_file) assert meter_data.header.version_header in ["NEM12", "NEM13"]
def test_importexport_zippednem12(): """Check that a zipped NEM12 is unchanged""" example_file = "examples/year_example.csv" output_file = import_export_nem12(example_file, zip_output=True) f1 = nr.read_nem_file(example_file) f2 = nr.read_nem_file(output_file) nmi = "NMIxxxxxxx" ch = "E1" for i, read1 in enumerate(f1.readings[nmi][ch]): read2 = f2.readings[nmi][ch][i] # Cannot compare all attributes as meter serial number is not included assert read1.t_start == read2.t_start assert read1.read_value == read2.read_value assert read1.quality_method == read2.quality_method assert read1.event_code == read2.event_code assert read1.event_desc == read2.event_desc
def export_nem13(example): """ Read in a NEM13 file and check if it is exported the same """ # Read in example example_file = 'tests/{}.csv'.format(example) ex = nr.read_nem_file(example_file) # Create output file output_file = 'tests/test_output_{}.csv'.format(example) m = NEM13(to_participant=ex.header.to_participant) # Add in interval readings to_load = [] for nmi in ex.readings: nmi_configuration = ''.join(list(ex.transactions[nmi].keys())) for channel in ex.readings[nmi]: # Build list of readings for read in ex.readings[nmi][channel]: ch = m.add_reading(nmi=nmi, nmi_configuration=nmi_configuration, register_id=None, nmi_suffix=channel, previous_read=read.read_start, previous_read_date=read.t_start, current_read=read.read_end, current_read_date=read.t_end, current_quality_method=read.quality_method, quantity=read.read_value, uom=read.uom ) # Export to file output = m.nem_output(file_name=output_file) # Compare files original = [] with open(example_file, newline='') as csvfile: reader = csv.reader(csvfile, delimiter=',', quotechar='"') for row in reader: original.append(row) output = [] with open(output_file, newline='') as csvfile: reader = csv.reader(csvfile, delimiter=',', quotechar='"') for row in reader: output.append(row) # Compare rows for i, row in enumerate(original): record_indicator = row[0] if record_indicator not in ['100', '200']: for j, col in enumerate(row): if j not in [3, 5, 6, 10, 21, 22]: assert cleanse_val(col) == cleanse_val(output[i][j]), '[{i},{j}] did not match'.format(i=i, j=j)
def test_unzipped_examples(): """ Open and parse unzipped example files """ skips = ["Example_NEM12_powercor.csv", "Example_NEM12_powercor_missing_fields.csv"] test_path = os.path.abspath("examples/unzipped") for file_name in os.listdir(test_path): print(file_name) if file_name in skips: continue test_file = os.path.join(test_path, file_name) meter_data = nr.read_nem_file(test_file) assert meter_data.header.version_header in ["NEM12", "NEM13"]
def test_correct_records(): meter_data = nr.read_nem_file("examples/unzipped/Example_NEM12_actual_interval.csv") readings = meter_data.readings["VABD000163"] assert len(readings["E1"]) == 48 assert readings["E1"][10].read_value == pytest.approx(1.11, 0.1) assert readings["E1"][-1].read_value == pytest.approx(1.11, 0.1) assert len(readings["Q1"]) == 48 assert readings["Q1"][10].read_value == pytest.approx(2.22, 0.1) assert readings["Q1"][-1].read_value == pytest.approx(2.22, 0.1) assert readings["Q1"][-1].quality_method == "A"
def test_correct_records(): meter_data = nr.read_nem_file( 'examples/unzipped/Example_NEM12_actual_interval.csv') readings = meter_data.readings['VABD000163'] assert len(readings['E1']) == 48 assert readings['E1'][10].read_value == pytest.approx(1.11, 0.1) assert readings['E1'][-1].read_value == pytest.approx(1.11, 0.1) assert len(readings['Q1']) == 48 assert readings['Q1'][10].read_value == pytest.approx(2.22, 0.1) assert readings['Q1'][-1].read_value == pytest.approx(2.22, 0.1) assert readings['Q1'][-1].quality_method == 'A'
def test_correct_quality(): meter_data = nr.read_nem_file( 'examples/unzipped/Example_NEM12_multiple_quality.csv') readings = meter_data.readings['CCCC123456']['E1'] assert readings[0].quality_method == 'F14' assert readings[10].quality_method == 'F14' assert readings[19].quality_method == 'F14' assert readings[20].quality_method == 'A' assert readings[22].quality_method == 'A' assert readings[23].quality_method == 'A' assert readings[25].quality_method == 'S14' assert readings[30].quality_method == 'S14' assert readings[47].quality_method == 'S14'
def test_correct_quality(): meter_data = nr.read_nem_file( "examples/unzipped/Example_NEM12_multiple_quality.csv") readings = meter_data.readings["CCCC123456"]["E1"] assert readings[0].quality_method == "F14" assert readings[10].quality_method == "F14" assert readings[19].quality_method == "F14" assert readings[20].quality_method == "A" assert readings[22].quality_method == "A" assert readings[23].quality_method == "A" assert readings[25].quality_method == "S14" assert readings[30].quality_method == "S14" assert readings[47].quality_method == "S14"
def test_correct_records(): meter_data = nr.read_nem_file( "examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True ) readings = meter_data.readings["VABD000163"] assert len(readings["E1"]) == 96 assert readings["E1"][10].read_value == pytest.approx(1.11, 0.1) assert readings["E1"][-1].read_value == pytest.approx(3.33, 0.1) assert len(readings["Q1"]) == 96 assert readings["Q1"][10].read_value == pytest.approx(2.22, 0.1) assert readings["Q1"][-1].read_value == pytest.approx(4.44, 0.1) assert readings["Q1"][-1].quality_method == "A"
def import_export_nem12(input_file, zip_output=False): """Create export from import""" # Read in example ex = nr.read_nem_file(input_file) # Create output file m = NEM12(to_participant=ex.header.to_participant) # Add in interval readings for nmi in ex.readings: nmi_configuration = "".join(list(ex.transactions[nmi].keys())) for nmi_suffix in ex.readings[nmi]: to_load = [] # Build list of readings for read in ex.readings[nmi][nmi_suffix]: to_load.append([ read.t_end, read.read_value, read.quality_method, read.event_code, read.event_desc, ]) # Get common atributes from last reading last = ex.readings[nmi][nmi_suffix][-1:][0] uom = last.uom m.add_readings( nmi=nmi, nmi_configuration=nmi_configuration, nmi_suffix=nmi_suffix, uom=uom, readings=to_load, ) # Export to file if zip_output: output_file = f"tests/{m.nem_filename()}.zip" output = m.output_zip(file_path=output_file) else: output_file = f"tests/{m.nem_filename()}.csv" output = m.output_csv(file_path=output_file) return output
def output_as_csv(file_name, nmi=None, output_file=None): """ Transpose all channels and output a csv that is easier to read and do charting on :param file_name: The NEM file to process :param nmi: Which NMI to output if more than one :param output_file: Specify different output location :returns: The file that was created """ m = read_nem_file(file_name) if nmi is None: nmi = list(m.readings.keys())[0] # Use first NMI channels = list(m.transactions[nmi].keys()) num_records = len(m.readings[nmi][channels[0]]) last_date = m.readings[nmi][channels[0]][-1].t_end if output_file is None: output_file = '{}_{}_transposed.csv'.format( nmi, last_date.strftime('%Y%m%d')) with open(output_file, 'w', newline='') as csvfile: cwriter = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) heading_list = ['period_start', 'period_end'] for channel in channels: heading_list.append(channel) heading_list.append('quality_method') cwriter.writerow(heading_list) for i in range(0, num_records): t_start = m.readings[nmi][channels[0]][i].t_start t_end = m.readings[nmi][channels[0]][i].t_end quality_method = m.readings[nmi][channels[0]][i].quality_method row_list = [t_start, t_end] for ch in channels: val = m.readings[nmi][ch][i].read_value row_list.append(val) row_list.append(quality_method) cwriter.writerow(row_list) return output_file
def test_optional_scheduled_read(): meter_data = nr.read_nem_file( "examples/unzipped/Example_NEM12_no_scheduled_read.csv") readings = meter_data.readings["NMI111"]["E1"] assert len(readings) == 96
def test_correct_channels(): meter_data = nr.read_nem_file("examples/unzipped/Example_NEM12_actual_interval.csv") readings = meter_data.readings["VABD000163"] assert len(readings) == 2 assert "E1" in readings assert "Q1" in readings
def test_correct_NMIs(): meter_data = nr.read_nem_file("examples/unzipped/Example_NEM12_actual_interval.csv") assert len(meter_data.readings) == 1 assert "VABD000163" in meter_data.readings
def test_missing_fields(): meter_data = nr.read_nem_file( "examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True )
def test_zipped_load(): meter_data = nr.read_nem_file( "examples/nem12/Example_NEM12_powercor.csv.zip", ignore_missing_header=True )
def test_correct_NMIs(): meter_data = nr.read_nem_file( "examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True ) assert len(meter_data.readings) == 1 assert "VABD000163" in meter_data.readings
""" Test Suite """ import pytest import datetime from nemreader import read_nem_file import os import sys sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from qldtariffs import get_daily_charges, get_monthly_charges INTERVAL_READINGS = read_nem_file( 'examples/example_NEM12.csv').readings['3044076134']['E1'] MANUAL_READINGS = read_nem_file( 'examples/example_NEM13.csv').readings['3044076134']['11'] def test_nem12_data_daily(): """ Test example NEM12 daily summary """ daily_summaries = get_daily_charges(INTERVAL_READINGS, 'ergon', 't14') dec1 = daily_summaries[datetime.date(2016, 12, 1)] assert dec1.total == pytest.approx(6.29) daily_summaries = get_daily_charges(INTERVAL_READINGS, 'ergon', 't14') dec1 = daily_summaries[datetime.date(2016, 9, 6)] assert dec1.total == pytest.approx(7.17) def test_nem12_data_monthly():