def setUp(self):
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
os.environ['PGTZ'] = 'UTC'
database_info = envinfo.DB_CONF
self.pony = PonyManager(database_info)
self.pony.define_solar_models()
self.pony.binddb(create_tables=True)
def setUpORM(self):
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
orm.rollback()
self.pony = PonyManager(envinfo.DB_CONF)
self.pony.define_all_models()
self.pony.binddb(create_tables=True)
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.create_tables()
self.maxDiff = None
def setUpClass(cls):
from conf import envinfo
assert envinfo.SETTINGS_MODULE == 'conf.settings.testing'
cls.pony = PonyManager(envinfo.DB_CONF)
cls.pony.define_all_models()
cls.pony.binddb(create_tables=True)
from api_server.plantmonitor_api import api
cls.api = api
def setUp(self):
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
orm.rollback()
self.pony = PonyManager(envinfo.DB_CONF)
self.pony.define_all_models()
self.pony.binddb(create_tables=True)
self.pony.db.drop_all_tables(with_all_data=True)
self.storage = PonyMetricStorage(self.pony.db)
self.maxDiff = None
# orm.set_sql_debug(True)
self.pony.db.create_tables()
# database.generate_mapping(create_tables=True)
orm.db_session.__enter__()
class CheckDBConnection_Test(unittest.TestCase):
def setUp(self):
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
orm.rollback()
self.pony = PonyManager(envinfo.DB_CONF)
self.pony.define_all_models()
self.pony.binddb(create_tables=True)
self.pony.db.drop_all_tables(with_all_data=True)
#orm.set_sql_debug(True)
self.pony.db.create_tables()
self.pony.db.disconnect()
# database.generate_mapping(create_tables=True)
# orm.db_session.__enter__()
def tearDown(self):
orm.rollback()
# orm.db_session.__exit__()
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.disconnect()
def test__check_db_connection(self):
check_db_connection()
def test__check_db_connectionCLI(self):
runner = CliRunner()
result = runner.invoke(check_db_connection_CLI, [])
self.assertEqual(0, result.exit_code)
def task_daily_download_from_api_meteologica(test_env=True):
pony = PonyManager(envinfo.DB_CONF)
pony.define_all_models()
pony.binddb(create_tables=False)
configdb = ns.load('conf/config_meteologica.yaml')
with orm.db_session:
downloadMeterForecasts(pony.db, configdb, test_env=test_env)
def importPlantCLI(yaml):
yamlFilename = click.format_filename(yaml)
logger.debug(yamlFilename)
from conf import envinfo
pony = PonyManager(envinfo.DB_CONF)
pony.define_all_models()
pony.binddb(create_tables=False)
importPlantsFromFile(pony.db, yamlFilename)
def computeIntegralMetrics():
from conf import envinfo
pony = PonyManager(envinfo.DB_CONF)
pony.define_all_models()
pony.binddb()
irradiance = integrateIrradiance(pony.db)
insertHourlySensorIrradiationMetrics(pony.db, irradiance, 'integratedIrradiation_wh_m2')
orm.flush()
expectedEnergy = integrateExpectedPower(pony.db)
insertHourlySensorIrradiationMetrics(pony.db, expectedEnergy, 'expected_energy_wh')
def check_db_connection():
from conf import envinfo
pony = PonyManager(envinfo.DB_CONF)
pony.define_all_models()
pony.binddb()
pony.db.check_tables()
with orm.db_session:
plants = pony.db.Plant.select()[:].to_list()
print(plants)
def list_plants(self):
from conf import envinfo
database_info = envinfo.DB_CONF
solar_db = PonyManager(database_info)
solar_db.define_solar_models()
solar_db.binddb(create_tables=False)
with orm.db_session:
plant_info = orm.select(
(p.name, p.location.latitude, p.location.longitude)
for p in solar_db.db.Plant)[:]
return plant_info
def task_meters_erp_to_orm():
pony = PonyManager(envinfo.DB_CONF)
pony.define_all_models()
pony.binddb(create_tables=False)
r = ReadingsFacade(pony.db)
try:
# TODO mock measures or fake meters
r.transfer_ERP_readings_to_model()
except Exception as err:
logger.error("[ERROR] %s" % err)
raise
def main():
args = parseArguments()
configfile = args.get('--config', 'conf/config.yaml')
configdb = ns.load(configfile)
configdb.update(args)
from conf import envinfo
pony = PonyManager(envinfo.DB_CONF)
pony.define_all_models()
pony.binddb()
downloadMeterForecasts(pony.db, configdb)
def task():
try:
plantname = envinfo.ACTIVEPLANT_CONF['activeplant']
# TODO remove legacy
if plantname == 'Alcolea':
legacyInfluxUpload()
plant = ProductionPlant()
apiconfig = envinfo.API_CONFIG
if not plant.load('conf/modmap.yaml', plantname):
logger.error('Error loading yaml definition file...')
sys.exit(-1)
plant_data = get_plant_reading(plant)
if not plant_data:
logger.error(
"Getting reading from {} failed. Aborting.".format(plantname))
return
pony_manager = PonyManager(envinfo.DB_CONF)
pony_manager.define_all_models()
pony_manager.binddb()
ponyStorage = PonyMetricStorage(pony_manager.db)
apiStorage = ApiMetricStorage(apiconfig)
logger.info("**** Saving data in database ****")
logger.debug("plant_data: {}".format(plant_data))
ponyStorage.insertPlantData(plant_data)
logger.info("**** Saving data in Api ****")
result = apiStorage.insertPlantData(plant_data)
logger.debug("api response: {}".format(result))
except Exception as err:
logger.exception("[ERROR] %s" % err)
def sun_events_update(self, start, end, plants=None):
from conf import envinfo
database_info = envinfo.DB_CONF
solar_db = PonyManager(database_info)
solar_db.define_solar_models()
solar_db.binddb(create_tables=True)
with orm.db_session:
if not plants:
target_plants = orm.select(p for p in solar_db.db.Plant)[:]
else:
target_plants = solar_db.db.Plant.select(
lambda p: p.name in plants)[:]
if not target_plants:
print("No plants in database. Returning.")
return
for plant in target_plants:
if not plant.location:
print("Plant {} doesn't have a location".format(
plant.name))
continue
lat = plant.location.latitude
lon = plant.location.longitude
sun_events_gen = SunEventsGenerator(lat, lon)
sun_events = sun_events_gen.generate_sunevents(start=start,
end=end)
print(sun_events)
solar_db.db.SolarEvent.insertPlantSolarEvents(
plant, sun_events)
class DailyDownload_Test(unittest.TestCase):
def createApi(self, **kwds):
config = self.createConfig()
params = dict(
wsdl=config['meteo_test_url'],
username=config['meteo_user'],
password=config['meteo_password'],
lastDateFile=self.getLastDateFile(),
lastDateDownloadFile=self.getLastDateDownloadFile(),
showResponses=False,
)
params.update(kwds)
return MeteologicaApi(**params)
def createConfig(self):
return ns.load('conf/configdb_test.yaml')
# TODO refactor the need of a lastDate download in MeteologicaApi, db should suffice
def getLastDateFile(self):
return 'lastDateFile-test.yaml'
def getLastDateDownloadFile(self):
return 'lastDateDownloadFile-test.yaml'
def cleanLastDateFile(self):
f = Path(self.getLastDateFile())
if f.exists(): f.unlink()
def setUp(self):
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
orm.rollback()
self.pony = PonyManager(envinfo.DB_CONF)
self.pony.define_all_models()
self.pony.binddb(create_tables=True)
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.create_tables()
orm.db_session.__enter__()
def tearDown(self):
orm.rollback()
orm.db_session.__exit__()
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.disconnect()
# comment this if you want to check the date format
self.cleanLastDateFile()
def mainFacility(self):
return 'Alcolea'
def otherFacility(self):
return 'Arigato'
def sampleForecastMeta(self, plant_name, forecastdate):
return {
'facilityId': self.mainFacility(),
'variableId': 'prod',
'predictorId': 'aggregated',
'granularity': '60',
'forecastDate': forecastdate,
'errorcode': None,
}
def samplePlantData(self, plant_name, forecastdate, forecasts,
packet_time):
return {
'plant':
plant_name,
'version':
'1.0',
'time':
packet_time,
'devices': [{
'id': 'ForecastMetadata:',
'errorcode': None,
'variable': 'prod',
'predictor': 'aggregated',
'forecastdate': forecastdate,
'granularity': 60,
'forecasts': forecasts,
}]
}
def createPlantmonitorDB(self):
configdb = ns.load('conf/configdb_test.yaml')
return PlantmonitorDB(configdb)
def test__createForecast__OnePlant(self):
alcolea = self.pony.db.Plant(
name=self.mainFacility(),
codename='SomEnergia_{}'.format(self.mainFacility()),
)
time = datetime.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=datetime.timezone.utc)
forecastMeta = {
'variable': 'prod',
'predictor': 'aggregated',
'granularity': '60',
'forecastdate': time,
'errorcode': '',
}
forecastData = {
self.mainFacility(): [
(todtaware("2040-01-02 00:00:00"), 10),
(todtaware("2040-01-02 01:00:00"), 20),
],
}
forecastMetadata = self.pony.db.ForecastMetadata.create(plant=alcolea,
**forecastMeta)
self.assertIsNotNone(forecastMetadata)
for f in forecastData[self.mainFacility()]:
forecastMetadata.insertForecast(
time=f[0],
percentil10=None,
percentil50=f[1],
percentil90=None,
)
orm.flush()
expected = [{
'forecastMetadata':
1,
'percentil10':
None,
'percentil50':
10,
'percentil90':
None,
'time':
datetime.datetime(2040, 1, 2, 0, 0, tzinfo=datetime.timezone.utc)
}, {
'forecastMetadata':
1,
'percentil10':
None,
'percentil50':
20,
'percentil90':
None,
'time':
datetime.datetime(2040, 1, 2, 1, 0, tzinfo=datetime.timezone.utc)
}]
forecasts = [f.to_dict() for f in self.pony.db.Forecast.select()]
self.assertListEqual(forecasts, expected)
def test__lastForecastDownloadDate__NoData(self):
alibaba = self.pony.db.Plant(
name=self.mainFacility(),
codename='SomEnergia_{}'.format(self.mainFacility()),
)
forecastDate = alibaba.lastForecastDownloaded()
self.assertIsNone(forecastDate)
def test__lastForecastDownloadDate__SomeData(self):
alcolea = self.pony.db.Plant(
name=self.mainFacility(),
codename='SomEnergia_{}'.format(self.mainFacility()),
)
time = datetime.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=datetime.timezone.utc)
forecastMeta = self.sampleForecastMeta(self.mainFacility(), time)
data = {
self.mainFacility(): [
(todt("2040-01-02 00:00:00"), 10),
(todt("2040-01-02 01:00:00"), 20),
],
}
status = 'OK'
forecastDate = time
forecastMetadata = self.pony.db.ForecastMetadata.create(
plant=alcolea, forecastdate=forecastDate, errorcode=status)
forecastMetadata.addForecasts(data[self.mainFacility()])
orm.flush()
forecastDate = alcolea.lastForecastDownloaded()
expectedForecastDate = time
self.assertEqual(forecastDate, expectedForecastDate)
def test_getForecast(self):
data = {
"SomEnergia_{}".format(self.mainFacility()): [
(todt("2040-01-02 00:00:00"), 10),
(todt("2040-01-02 01:00:00"), 20),
],
}
fromDate = todt("{}-01-01 00:00:00".format(
datetime.datetime.now().year))
toDate = todt("{}-01-03 00:00:00".format(datetime.datetime.now().year))
with self.createApi() as api:
for facility, values in data.items():
resultForecast = api.getForecast(facility, fromDate, toDate)
self.assertEqual(
resultForecast[0][0],
todtaware("{}-01-01 00:00:00".format(
datetime.datetime.now().year)))
def test__downloadMeterForecasts__uptodate(self):
alcolea = self.pony.db.Plant(
name=self.mainFacility(),
codename='SomEnergia_{}'.format(self.mainFacility()),
)
forecastDate = datetime.datetime.now(datetime.timezone.utc)
oldForecastMetadata = self.pony.db.ForecastMetadata.create(
plant=alcolea, forecastdate=forecastDate, errorcode='OK')
p50 = 1000
time = datetime.datetime.now(datetime.timezone.utc)
oldForecastMetadata.insertForecast(percentil10=None,
percentil50=p50,
percentil90=None,
time=time)
orm.flush()
statuses = downloadMeterForecasts(self.pony.db, self.createConfig())
forecastMetadata = self.pony.db.ForecastMetadata.select().order_by(
orm.desc(self.pony.db.ForecastMetadata.forecastdate)).first()
self.assertEqual(forecastMetadata, oldForecastMetadata)
forecasts = forecastMetadata.registries
self.assertEqual(statuses['SomEnergia_Alcolea'],
ForecastStatus.UPTODATE)
def test__downloadMeterForecasts__newForecasts(self):
alcolea = self.pony.db.Plant(
name=self.mainFacility(),
codename='SomEnergia_{}'.format(self.mainFacility()),
)
time = datetime.datetime.now(datetime.timezone.utc)
forecastDate = time - datetime.timedelta(hours=1)
oldForecastMetadata = self.pony.db.ForecastMetadata.create(
plant=alcolea, forecastdate=forecastDate, errorcode='OK')
p50 = 1000
oldForecastMetadata.insertForecast(percentil10=None,
percentil50=p50,
percentil90=None,
time=time)
orm.flush()
statuses = downloadMeterForecasts(self.pony.db, self.createConfig())
forecastMetadata = self.pony.db.ForecastMetadata.select().order_by(
orm.desc(self.pony.db.ForecastMetadata.forecastdate)).first()
self.assertNotEqual(forecastMetadata, oldForecastMetadata)
self.assertEqual(statuses['SomEnergia_Alcolea'], ForecastStatus.OK)
forecasts = orm.select(
f for f in forecastMetadata.registries).order_by(lambda: f.time)[:]
print(forecasts)
expectedTime = time.replace(
minute=0, second=0, microsecond=0) + datetime.timedelta(days=14,
hours=1)
expectedNumForecasts = 14 * 24 + 1
self.assertEqual(len(forecasts), expectedNumForecasts)
self.assertEqual(forecasts[-1].time, expectedTime)
def test__getMeterReadings__None(self):
plant = self.pony.db.Plant(
name=self.mainFacility(),
codename='SomEnergia_{}'.format(self.mainFacility()),
)
self.pony.db.Meter(plant=plant, name="1234")
orm.flush()
with orm.db_session:
data = getMeterReadings(self.pony.db, plant.codename)
self.assertEqual(data, [])
def test__getMeterReadings__UnexistingFacility(self):
with orm.db_session:
data = getMeterReadings(self.pony.db, facility="fake")
# TODO return None or [] ?
self.assertEqual(data, None)
def test__getMeterReadings(self):
plant = self.pony.db.Plant(
name=self.mainFacility(),
codename='SomEnergia_{}'.format(self.mainFacility()),
)
time = todtaware('2020-01-01 00:00:00')
meter = self.pony.db.Meter(plant=plant, name='1234')
export_energy_wh = 210
meter.insertRegistry(time=time,
export_energy_wh=export_energy_wh,
import_energy_wh=0,
r1_VArh=0,
r2_VArh=0,
r3_VArh=0,
r4_VArh=0)
orm.flush()
expected = [
(todtaware('2020-01-01 00:00:00'), 210),
]
with orm.db_session:
result = getMeterReadings(self.pony.db, plant.codename)
self.assertEqual(expected, result)
def test__getMeterReadingsFromLastUpload__noMeter(self):
plant = self.pony.db.Plant(
name=self.mainFacility(),
codename='SomEnergia_{}'.format(self.mainFacility()),
)
time = todtaware('2020-01-01 00:00:00')
orm.flush()
with orm.db_session:
result = getMeterReadings(self.pony.db, plant.codename)
self.assertIsNone(result)
def test__getMeterReadingsFromLastUpload__None(self):
plant = self.pony.db.Plant(
name=self.mainFacility(),
codename='SomEnergia_{}'.format(self.mainFacility()),
)
time = todtaware('2020-01-01 00:00:00')
meter = self.pony.db.Meter(plant=plant, name='1234')
orm.flush()
with orm.db_session:
result = getMeterReadings(self.pony.db, plant.codename)
self.assertEqual([], result)
def test__getMeterReadingsFromLastUpload__OneReading(self):
plant = self.pony.db.Plant(
name=self.mainFacility(),
codename='SomEnergia_{}'.format(self.mainFacility()),
)
time = todtaware('2020-01-01 00:00:00')
meter = self.pony.db.Meter(plant=plant, name='1234')
export_energy_wh = 210
meter.insertRegistry(time=time,
export_energy_wh=export_energy_wh,
import_energy_wh=0,
r1_VArh=0,
r2_VArh=0,
r3_VArh=0,
r4_VArh=0)
orm.flush()
expected = [
(todtaware('2020-01-01 00:00:00'), 210),
]
with orm.db_session:
result = getMeterReadings(self.pony.db, plant.codename)
self.assertEqual(expected, result)
def test__uploadFacilityMeterReadings__checkResponses(self):
plant = self.pony.db.Plant(
name=self.mainFacility(),
codename='SomEnergia_{}'.format(self.mainFacility()),
)
time = todtaware('2020-01-01 00:00:00')
meter = self.pony.db.Meter(plant=plant, name='1234')
export_energy_wh = 210
meter.insertRegistry(time=time,
export_energy_wh=export_energy_wh,
import_energy_wh=0,
r1_VArh=0,
r2_VArh=0,
r3_VArh=0,
r4_VArh=0)
orm.flush()
with self.createApi() as api:
response = uploadFacilityMeterReadings(self.pony.db, api,
plant.codename)
expectedResponse = 'OK'
self.assertEqual(response, expectedResponse)
def test__uploadMeterReadings__checkResponses(self):
plant = self.pony.db.Plant(
name=self.mainFacility(),
codename='SomEnergia_{}'.format(self.mainFacility()),
)
time = todtaware('2020-01-01 00:00:00')
meter = self.pony.db.Meter(plant=plant, name='1234')
export_energy_wh = 210
meter.insertRegistry(time=time,
export_energy_wh=export_energy_wh,
import_energy_wh=0,
r1_VArh=0,
r2_VArh=0,
r3_VArh=0,
r4_VArh=0)
orm.flush()
with self.createApi() as api:
responses = uploadMeterReadings(self.pony.db, self.createConfig())
expectedResponse = 'OK'
self.assertEqual(responses[plant.codename], expectedResponse)
def test__lastForecastDownloaded__checkLastDateAdded(self):
alcolea = self.pony.db.Plant(
name=self.mainFacility(),
codename='SomEnergia_{}'.format(self.mainFacility()),
)
oldForecastDate = datetime.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=datetime.timezone.utc)
newForecastDate = oldForecastDate + datetime.timedelta(hours=1)
status = 'OK'
oldForecastMetadata = self.pony.db.ForecastMetadata.create(
plant=alcolea, forecastdate=oldForecastDate, errorcode=status)
newForecastMetadata = self.pony.db.ForecastMetadata.create(
plant=alcolea, forecastdate=newForecastDate, errorcode=status)
oldForecast = [
(todt("2040-01-02 00:00:00"), 10),
(todt("2040-01-02 01:00:00"), 20),
]
newForecast = [
(todt("2040-01-02 02:00:00"), 30),
(todt("2040-01-02 03:00:00"), 40),
]
oldForecastMetadata.addForecasts(oldForecast)
newForecastMetadata.addForecasts(newForecast)
orm.flush()
lastForecastDate = alcolea.lastForecastDownloaded()
expectedForecastDate = newForecastDate
self.assertEqual(lastForecastDate, expectedForecastDate)
def __test_downloadForecast_setup(self):
forecastMeta = self.sampleForecastMeta(self.mainFacility(), time)
status = 'OK'
forecastDate = time
forecastMetadata = self.pony.db.ForecastMetadata.create(
plant=alcolea, forecastdate=forecastDate, errorcode=status)
forecastMetadata.addForecasts(data[self.mainFacility()])
orm.flush()
expected = [
(todtaware('2020-01-01 00:00:00'), 210),
(todtaware('2020-01-01 01:00:00'), 320),
(todtaware('2020-01-01 02:00:00'), 230),
(todtaware('2020-01-01 03:00:00'), 340),
]
# TODO old tests, are they still useful to refactor?
def _test_downloadForecastToEmptyDB(self):
configdb = self.createConfig()
download_meter_data(configdb, test_env=True)
forecastDB = {}
with self.createPlantmonitorDB() as db:
forecastDB = db.getForecast()
now = datetime.datetime.now(datetime.timezone.utc)
fromDate = now - datetime.timedelta(days=14)
toDate = now
forecast = []
with self.createApi() as api:
forecast = api.getForecast(self.mainFacility(), fromDate, toDate)
self.assertListEqual(forecastDB[self.mainFacility()], forecast)
def __test_downloadForecastToEmptyDB_manyfacilities(self):
configdb = self.createConfig()
data = {
self.mainFacility(): [
(todt("2040-01-02 00:00:00"), 10),
(todt("2040-01-02 01:00:00"), 20),
],
}
fromDate = todt("2020-01-01 00:00:00")
toDate = todt("2020-01-01 00:00:00")
forecasts = {}
with self.createApi() as api:
for facility in data.keys():
forecast = api.getForecast(facility, fromDate, toDate)
forecasts[facility] = forecast
downloadResponses = download_meter_data(configdb, test_env=True)
self.assertTrue((self.mainFacility(),
"OK") in downloadResponses.items())
forecastDBData = {}
with self.createPlantmonitorDB() as db:
forecastDBData = db.getForecast(self.mainFacility())
logger.info(forecastDBData)
logger.info(forecasts)
self.assertDictEqual(forecastDBData, forecasts[self.mainFacility()])
def __test_downloadForecastToDB(self):
configdb = self.createConfig()
data = {
self.mainFacility(): [
(todt("2040-01-02 00:00:00"), 10),
(todt("2040-01-02 01:00:00"), 20),
],
}
fromDate = todt("2020-01-01 00:00:00")
toDate = todt("2020-01-01 00:00:00")
with self.createApi() as api:
for facility in data.keys():
response = api.getForecast(facility, fromDate, toDate)
with self.createPlantmonitorDB() as db:
db.addFacilityMeterRelation(self.mainFacility(), '123401234')
tworeadings = {
self.mainFacility(): [
(todt("2040-01-02 00:00:00"), 10),
(todt("2040-01-02 01:00:00"), 20),
],
}
db.addMeterData(tworeadings)
responses = download_meter_data(configdb, test_env=True)
self.assertDictEqual({self.mainFacility(): "OK"}, responses)
# TODO check uploaded content from api if they ever add GetObservations
with self.createApi() as api:
self.assertEqual(api.lastDateUploaded(self.mainFacility()),
todt("2040-01-02 01:00:00"))
def __test_downloadForecastToDB_ManyFacilities(self):
config = self.createConfig()
with self.createPlantmonitorDB() as db:
db.addFacilityMeterRelation(self.otherFacility(), '432104321')
db.addFacilityMeterRelation(self.mainFacility(), '123401234')
data = {
self.mainFacility(): [
(todt("2040-01-02 00:00:00"), 10),
(todt("2040-01-02 01:00:00"), 20),
],
self.otherFacility(): [
(todt("2040-01-02 00:00:00"), 210),
(todt("2040-01-02 01:00:00"), 340),
],
}
db.addMeterData(data)
responses = download_meter_data(config, test_env=True)
self.assertDictEqual(
{
self.mainFacility(): "OK",
self.otherFacility(): "OK"
}, responses)
def __test_downloadForecastToDB_mismatchFacilities(self):
config = self.createConfig()
with self.createPlantmonitorDB() as db:
db.addFacilityMeterRelation(self.unexistantFacility(), '432104321')
data = {
self.unexistantFacility(): [
(todt("2040-01-02 00:00:00"), 10),
(todt("2040-01-02 01:00:00"), 20),
],
}
db.addMeterData(data)
responses = download_meter_data(configdb=config, test_env=True)
self.assertDictEqual(
{
self.unexistantFacility():
"INVALID_FACILITY_ID: {}".format(self.unexistantFacility())
}, responses)
with self.createApi() as api:
self.assertEqual(api.lastDateUploaded(self.unexistantFacility()),
None)
def __test_downloadForecastToDB_mixedFacilities(self):
config = self.createConfig()
with self.createPlantmonitorDB() as db:
db.addFacilityMeterRelation(self.mainFacility(), '123401234')
db.addFacilityMeterRelation(self.unexistantFacility(), '432104321')
data = {
self.mainFacility(): [
(todt("2040-01-02 00:00:00"), 50),
(todt("2040-01-02 01:00:00"), 70),
],
self.unexistantFacility(): [
(todt("2040-01-02 00:00:00"), 10),
(todt("2040-01-02 01:00:00"), 20),
],
}
db.addMeterData(data)
responses = download_meter_data(configdb=config, test_env=True)
self.assertDictEqual(
{
self.mainFacility():
"OK",
self.unexistantFacility():
"INVALID_FACILITY_ID: {}".format(self.unexistantFacility())
}, responses)
with self.createApi() as api:
self.assertEqual(api.lastDateUploaded(self.mainFacility()),
todt("2040-01-02 01:00:00"))
self.assertEqual(api.lastDateUploaded(self.unexistantFacility()),
None)
class ImportPlant_Test(unittest.TestCase):
from yamlns.testutils import assertNsEqual
def setUpORM(self):
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
orm.rollback()
self.pony = PonyManager(envinfo.DB_CONF)
self.pony.define_all_models()
self.pony.binddb(create_tables=True)
self.pony.db.drop_all_tables(with_all_data=True)
#orm.set_sql_debug(True)
self.pony.db.create_tables()
# database.generate_mapping(create_tables=True)
# orm.db_session.__enter__()
def tearDownORM(self):
orm.rollback()
# orm.db_session.__exit__()
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.disconnect()
def setUp(self):
self.setUpORM()
def tearDown(self):
self.tearDownORM()
def test_importExportPlant(self):
with orm.db_session:
nsplants = ns.loads("""\
plants:
- plant:
name: alcolea
codename: SCSOM04
description: la bonica planta
meters:
- meter:
name: '1234578'
inverters:
- inverter:
name: '5555'
- inverter:
name: '6666'
irradiationSensors:
- irradiationSensor:
name: alberto
temperatureAmbientSensors:
- temperatureAmbientSensor:
name: joana
temperatureModuleSensors:
- temperatureModuleSensor:
name: pol""")
importPlants(self.pony.db, nsplants)
#TODO test the whole fixture, not just the plant data
expectedPlantsNS = exportPlants(self.pony.db)
self.assertNsEqual(expectedPlantsNS, nsplants)
def test__importPlantCLI_NoFile(self):
runner = CliRunner()
result = runner.invoke(importPlantCLI, [])
self.assertEqual(2, result.exit_code)
def test__importPlantCLI_File(self):
fakePlantYaml = 'fakeplant.yaml'
content = """\
plants:
- plant:
name: alcolea
codename: SCSOM04
description: la bonica planta
meters:
- meter:
name: '1234578'
inverters:
- inverter:
name: '5555'
- inverter:
name: '6666'
irradiationSensors:
- irradiationSensor:
name: alberto
temperatureAmbientSensors:
- temperatureAmbientSensor:
name: joana
temperatureModuleSensors:
- temperatureModuleSensor:
name: pol"""
p = Path(fakePlantYaml)
with p.open("w", encoding="utf-8") as f:
f.write(content)
runner = CliRunner()
result = runner.invoke(importPlantCLI, [fakePlantYaml])
print(result)
self.assertEqual(0, result.exit_code)
p.unlink()
with orm.db_session:
plantns = exportPlants(self.pony.db)
self.assertNsEqual(plantns, content)
def test__importPlant_File(self):
fakePlantsYaml = 'fakeplant.yaml'
content = """\
plants:
- plant:
name: alcolea
codename: SCSOM04
description: la bonica planta
meters:
- meter:
name: '1234578'
inverters:
- inverter:
name: '5555'
- inverter:
name: '6666'
irradiationSensors:
- irradiationSensor:
name: alberto
temperatureAmbientSensors:
- temperatureAmbientSensor:
name: joana
temperatureModuleSensors:
- temperatureModuleSensor:
name: pol"""
p = Path(fakePlantsYaml)
with p.open("w", encoding="utf-8") as f:
f.write(content)
importPlantsFromFile(self.pony.db, fakePlantsYaml)
p.unlink()
with orm.db_session:
plantns = exportPlants(self.pony.db)
self.assertNsEqual(plantns, content)
def test__importPlant_File__withMunicipalities(self):
fakePlantsYaml = 'fakeplant.yaml'
content = """\
municipalities:
- municipality:
name: Figueres
ineCode: '17066'
countryCode: ES
country: Spain
regionCode: '08'
region: Catalonia
provinceCode: '17'
province: Girona
plants:
- plant:
name: alcolea
codename: SCSOM04
description: la bonica planta
meters:
- meter:
name: '1234578'
inverters:
- inverter:
name: '5555'
- inverter:
name: '6666'
irradiationSensors:
- irradiationSensor:
name: alberto
temperatureAmbientSensors:
- temperatureAmbientSensor:
name: joana
temperatureModuleSensors:
- temperatureModuleSensor:
name: pol"""
p = Path(fakePlantsYaml)
with p.open("w", encoding="utf-8") as f:
f.write(content)
importPlantsFromFile(self.pony.db, fakePlantsYaml)
p.unlink()
with orm.db_session:
plantns = exportPlants(self.pony.db)
self.assertNsEqual(plantns, content)
class Storage_Test(unittest.TestCase):
from yamlns.testutils import assertNsEqual
def setUp(self):
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
orm.rollback()
self.pony = PonyManager(envinfo.DB_CONF)
self.pony.define_all_models()
self.pony.binddb(create_tables=True)
self.pony.db.drop_all_tables(with_all_data=True)
self.storage = PonyMetricStorage(self.pony.db)
self.maxDiff = None
# orm.set_sql_debug(True)
self.pony.db.create_tables()
# database.generate_mapping(create_tables=True)
orm.db_session.__enter__()
def tearDown(self):
orm.rollback()
orm.db_session.__exit__()
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.disconnect()
def samplePlantData(self, time):
plantData = {
"plant":
"alibaba",
"devices":
sorted([{
"id":
"Meter:1234578",
"readings": [{
"time": time,
"export_energy_wh": 1,
"import_energy_wh": 123,
"r1_VArh": 1234,
"r2_VArh": 124,
"r3_VArh": 1234,
"r4_VArh": 124,
}],
}, {
"id": "SensorTemperatureAmbient:5555",
"readings": [{
"time": time,
"temperature_dc": 170,
}]
}],
key=lambda d: d['id']),
}
return plantData
def test_Environment(self):
#TODO will it be too late if the config is misconfigured?
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
def test_connection(self):
with orm.db_session:
self.assertEqual(self.pony.db.get_connection().status, 1)
def test__datetimeToStr(self):
time = datetime.datetime.now(datetime.timezone.utc)
plant_data = self.samplePlantData(time)
ApiMetricStorage.datetimeToStr(plant_data)
expected_plant_data = self.samplePlantData(time.isoformat())
self.assertDictEqual(plant_data, expected_plant_data)
#TODO deprecated as soon as we switch plant_data
# test instead registries_to_plant_data in task.py
def __test_PublishOrmOneInverterRegistry(self):
plant_name = 'SomEnergia_Alibaba'
inverter_name = 'Mary'
with orm.db_session:
alcolea = self.pony.db.Plant(name=plant_name,
codename='SOMSC01',
description='descripción de planta')
inverter = self.pony.db.Inverter(name=inverter_name, plant=alcolea)
metrics = ns([
('daily_energy_h_wh', 0),
('daily_energy_l_wh', 17556),
('e_total_h_wh', 566),
('e_total_l_wh', 49213),
('h_total_h_h', 0),
('h_total_l_h', 18827),
('pac_r_w', 0),
('pac_s_w', 0),
('pac_t_w', 0),
('powerreactive_t_v', 0),
('powerreactive_r_v', 0),
('powerreactive_s_v', 0),
('temp_inv_dc', 320),
('time', datetime.datetime.now(datetime.timezone.utc)),
# Sensors registers obtained from inverters
('probe1value', 443),
('probe2value', 220),
('probe3value', 0),
('probe4value', 0),
])
self.storage.storeInverterMeasures(plant_name, inverter_name,
metrics)
metrics.pop('probe1value')
metrics.pop('probe2value')
metrics.pop('probe3value')
metrics.pop('probe4value')
expectedRegistry = dict(metrics)
expectedRegistry['inverter'] = inverter.id
inverterReadings = self.storage.inverterReadings()
self.assertEqual(inverterReadings, [expectedRegistry])
def __test_PublishOrmIfInverterNotExist(self):
inverter_name = 'Alice'
plant_name = 'SomEnergia_Alibaba'
with orm.db_session:
alcolea = self.pony.db.Plant(name=plant_name,
codename='SOMSC01',
description='descripción de planta')
inverter = self.pony.db.Inverter(name=inverter_name, plant=alcolea)
metrics = ns([
('daily_energy_h_wh', 0),
('daily_energy_l_wh', 17556),
('e_total_h_wh', 566),
('e_total_l_wh', 49213),
('h_total_h_h', 0),
('h_total_l_h', 18827),
('pac_r_w', 0),
('pac_s_w', 0),
('pac_t_w', 0),
('powerreactive_t_v', 0),
('powerreactive_r_v', 0),
('powerreactive_s_v', 0),
('temp_inv_dc', 320),
('time', datetime.datetime.now(datetime.timezone.utc)),
# Sensors registers obtained from inverters
('probe1value', 443),
('probe2value', 220),
('probe3value', 0),
('probe4value', 0),
])
self.storage.storeInverterMeasures(plant_name, "UnknownInverter",
metrics)
self.assertListEqual(self.storage.inverterReadings(), [])
def __test_PublishOrmIfPlantNotExist(self):
inverter_name = 'Alice'
plant_name = 'SomEnergia_Alibaba'
with orm.db_session:
alcolea = self.pony.db.Plant(name=plant_name,
codename='SOMSC01',
description='descripción de planta')
inverter = self.pony.db.Inverter(name=inverter_name, plant=alcolea)
metrics = ns([
('daily_energy_h_wh', 0),
('daily_energy_l_wh', 17556),
('e_total_h_wh', 566),
('e_total_l_wh', 49213),
('h_total_h_h', 0),
('h_total_l_h', 18827),
('pac_r_w', 0),
('pac_s_w', 0),
('pac_t_w', 0),
('powerreactive_t_v', 0),
('powerreactive_r_v', 0),
('powerreactive_s_v', 0),
('temp_inv_dc', 320),
('time', datetime.datetime.now(datetime.timezone.utc)),
# Sensors registers obtained from inverters
('probe1value', 443),
('probe2value', 220),
('probe3value', 0),
('probe4value', 0),
])
self.storage.storeInverterMeasures('UnknownPlant', inverter_name,
metrics)
self.assertListEqual(self.storage.inverterReadings(), [])
def test__PonyMetricStorage_insertPlantData__storeTemperatureSensor(self):
sensor_name = 'Alice'
plant_name = 'SomEnergia_Alibaba'
time = datetime.datetime.now(datetime.timezone.utc)
with orm.db_session:
alcolea = self.pony.db.Plant(name=plant_name,
codename='SOMSC01',
description='descripción de planta')
sensor = self.pony.db.SensorTemperatureAmbient(name=sensor_name,
plant=alcolea)
orm.flush()
plant_data = {
"plant":
plant_name,
"version":
"1.0",
"time":
time.isoformat(), #consider using fastapi.jsonable_encoder
"devices": [{
"id":
"SensorTemperatureAmbient:Alice",
"readings": [{
"temperature_dc": 120,
"time": time,
}]
}]
}
self.storage.insertPlantData(plant_data)
orm.flush()
expected_plant_data = {
'plant':
plant_name,
'devices': [{
'id':
'SensorTemperatureAmbient:Alice',
'readings': [{
"temperature_dc": 120,
"time": time,
}]
}],
}
self.assertDictEqual(self.storage.plantData(plant_name),
expected_plant_data)
def test__PonyMetricStorage_insertPlantData__storeTemperatureSensorWithoutTemperature(
self):
sensor_name = 'Alice'
plant_name = 'SomEnergia_Alibaba'
time = datetime.datetime.now(datetime.timezone.utc)
with orm.db_session:
alcolea = self.pony.db.Plant(name=plant_name,
codename='SOMSC01',
description='descripción de planta')
sensor = self.pony.db.SensorTemperatureAmbient(name=sensor_name,
plant=alcolea)
orm.flush()
plant_data = {
"plant":
plant_name,
"version":
"1.0",
"time":
time.isoformat(), #consider using fastapi.jsonable_encoder
"devices": [{
"id":
"SensorTemperatureAmbient:Alice",
"readings": [{
"temperature_dc": None,
"time": time,
}]
}]
}
self.storage.insertPlantData(plant_data)
orm.flush()
expected_plant_data = {
'plant':
plant_name,
'devices': [{
'id':
'SensorTemperatureAmbient:Alice',
'readings': [{
"temperature_dc": None,
"time": time,
}]
}],
}
self.assertDictEqual(self.storage.plantData(plant_name),
expected_plant_data)
class Queries_Test(TestCase):
from b2btest.b2btest import assertB2BEqual
@classmethod
def setUpClass(cls):
''
def setUp(self):
self.maxDiff = None
self.b2bdatapath = 'b2bdata'
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
orm.rollback()
self.pony = PonyManager(envinfo.DB_CONF)
self.pony.define_all_models()
self.pony.binddb(create_tables=True)
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.create_tables()
orm.db_session.__enter__()
def tearDown(self):
orm.rollback()
orm.db_session.__exit__()
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.disconnect()
def samplePlantNS(self): # Copied from test_models.py:samplePlantNS
alcoleaPlantNS = ns.loads("""\
name: myplant
codename: PLANTCODE
description: Plant Description
meters:
- meter:
name: '1234578'
irradiationSensors:
- irradiationSensor:
name: irradiationSensor1
""")
return alcoleaPlantNS
# irradiance exported with
# `\copy (select to_char(time at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS'), irradiation_w_m2
# from sensorirradiationregistry order by time desc) to '/tmp/irradiance-2021-07-21-Alcolea.csv' with csv;`
def readTimeseriesCSV(self, csv_filepath):
with open(csv_filepath) as csv_file:
content = csv_file.read()
timeseries = [(datetime.datetime.fromisoformat(l[0]).astimezone(
datetime.timezone.utc), int(l[1]))
for l in (r.split(',') for r in content.split('\n')
if r)]
return timeseries
def setupPlant(self):
plantDefinition = self.samplePlantNS()
plant = self.pony.db.Plant(
name=plantDefinition.name,
codename=plantDefinition.codename,
)
plant.importPlant(plantDefinition)
plant.flush()
self.plant = plant.id
self.sensor = self.pony.db.SensorIrradiation.select().first().id
def importData(self, sensor, filename):
irradianceTimeSeries = self.readTimeseriesCSV(filename)
for time, irradiation_w_m2 in irradianceTimeSeries:
self.pony.db.SensorIrradiationRegistry(
sensor=sensor,
time=time,
irradiation_w_m2=int(round(irradiation_w_m2)),
temperature_dc=0,
)
self.pony.db.flush()
return [(time, self.plant, sensor, irradiation_w_m2)
for time, irradiation_w_m2 in irradianceTimeSeries]
def assertOutputB2B(self, result):
result = "\n".join(
("{}, {}, {:.9f}".format(time.isoformat() if time else None, sensor
or int('nan'), irradiation
or float('nan'))
for time, sensor, irradiation in result))
self.assertB2BEqual(result)
def test_irradiation(self):
self.setupPlant()
self.importData(self.sensor,
'b2bdata/irradiance-2021-07-21-Alcolea.csv')
query = Path('queries/view_irradiation.sql').read_text(encoding='utf8')
result = self.pony.db.select(query)
self.assertOutputB2B(result)
def test_irradiation_oneDay(self):
self.setupPlant()
self.importData(self.sensor,
'b2bdata/irradiance-2021-07-21-Alcolea.csv')
query = Path('queries/view_irradiation.sql').read_text(encoding='utf8')
result = [
r for r in self.pony.db.select(query)
if todtaware('2021-06-01 00:00:00') <= r.time
and r.time < todtaware('2021-06-02 00:00:00') and r.sensor == 1
]
self.assertOutputB2B(result)
def test_irradiation_incompleteLastDay(self):
self.setupPlant()
self.importData(
self.sensor,
'b2bdata/irradiance-2021-07-21-Alcolea_test_cases.csv')
query = Path('queries/view_irradiation.sql').read_text(encoding='utf8')
result = [
r.irradiation_w_m2_h for r in self.pony.db.select(query)
if todtaware('2021-07-14 9:00:00') <= r.time
and r.time < todtaware('2021-07-14 10:00:00') and r.sensor == 1
][0]
self.assertAlmostEqual(result, 551.254630277778, places=5)
def test_irradiation_halfHourWithReadings(self):
self.setupPlant()
self.importData(
self.sensor,
'b2bdata/irradiance-2021-07-21-Alcolea_test_cases.csv')
query = Path('queries/view_irradiation.sql').read_text(encoding='utf8')
result = [
r.irradiation_w_m2_h for r in self.pony.db.select(query)
if todtaware('2021-07-14 8:00:00') <= r.time
and r.time < todtaware('2021-07-14 9:00:00')
][0]
self.assertAlmostEqual(result, 415.2791252777778, places=5)
def test_irradiation_second_sensors(self):
self.setupPlant()
plant = self.plant
sensor_florida = self.pony.db.SensorIrradiation(
plant=plant, name="irradiationSensor2")
self.importData(sensor_florida,
'b2bdata/irradiance-2021-07-21-Florida.csv')
query = Path('queries/view_irradiation.sql').read_text(encoding='utf8')
result = self.pony.db.select(query)
self.assertOutputB2B(result)
def test_irradiation_multiple_sensors(self):
self.setupPlant()
self.importData(self.sensor,
'b2bdata/irradiance-2021-07-21-Alcolea.csv')
plant = self.plant
sensor_florida = self.pony.db.SensorIrradiation(
plant=plant, name="irradiationSensor2")
self.importData(sensor_florida,
'b2bdata/irradiance-2021-07-21-Florida.csv')
query = Path('queries/view_irradiation.sql').read_text(encoding='utf8')
result = self.pony.db.select(query)
self.assertOutputB2B(result)
# TODO test output of one sensor equal to two sensors for the one sensor
def _test_irradiation_multiple_sensors_same_as_one(self):
self.setupPlant()
self.importData(self.sensor,
'b2bdata/irradiance-2021-07-21-Alcolea.csv')
plant = self.plant
sensor_florida = self.pony.db.SensorIrradiation(
plant=plant, name="irradiationSensor2")
self.importData(sensor_florida,
'b2bdata/irradiance-2021-07-21-Florida.csv')
query = Path('queries/view_irradiation.sql').read_text(encoding='utf8')
result = self.pony.db.select(query)
self.assertOutputB2B(result)
def test_irradiation_oneHour_againstTrapezoidal(self):
# remember that you have an trapezoidal approximation script for csv at scripts/integrate_csv
from scripts.integrate_csv import integrate
trapzIrradiationTS = integrate(
'b2bdata/irradiance-2021-07-21-Alcolea_test_cases.csv')
self.assertAlmostEqual(trapzIrradiationTS[0][1],
448.17932250650483,
places=3)
class SunEvents_Test(unittest.TestCase):
def setUp(self):
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
os.environ['PGTZ'] = 'UTC'
database_info = envinfo.DB_CONF
self.pony = PonyManager(database_info)
self.pony.define_solar_models()
self.pony.binddb(create_tables=True)
def tearDown(self):
'''
binddb calls gneerate_mapping which creates the tables outside the transaction
drop them
'''
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.disconnect()
def createPlant(self):
with orm.db_session:
p = self.pony.db.Plant(name='roger', codename='Som_roger')
self.pony.db.PlantLocation(plant=p, latitude=41.967599, longitude=2.837782)
def createPlantWithoutLocation(self):
with orm.db_session:
p = self.pony.db.Plant(name='roger', codename='Som_roger')
def test__checkEnvironment(self):
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
def test__sunevents_click(self):
self.createPlant()
runner = CliRunner()
result = runner.invoke(
plant_sun_events_update, '--start 2021-11-29 --end 2021-11-29 --plant roger'.split()
)
self.assertEqual(result.exit_code, 0)
def test__sun_events_update__no_plant(self):
# run sun_events
start = datetime.datetime(2021, 11, 29, 3, 0, 0, 0, tzinfo=datetime.timezone.utc)
end = datetime.datetime(2021, 11, 30, 3, 0, 0, 0, tzinfo=datetime.timezone.utc)
se = SunEvents()
se.sun_events_update(start=start, end=end)
with orm.db_session:
result = orm.select((s.plant.name, s.sunrise, s.sunset) for s in self.pony.db.SolarEvent)[:].to_list()
expected = []
self.assertListEqual(result, expected)
def test__sun_events_update__no_plant_no_location(self):
self.createPlantWithoutLocation()
# run sun_events
start = datetime.datetime(2021, 11, 29, 3, 0, 0, 0, tzinfo=datetime.timezone.utc)
end = datetime.datetime(2021, 11, 30, 3, 0, 0, 0, tzinfo=datetime.timezone.utc)
se = SunEvents()
se.sun_events_update(start=start, end=end)
with orm.db_session:
result = orm.select((s.plant.name, s.sunrise, s.sunset) for s in self.pony.db.SolarEvent)[:].to_list()
expected = []
self.assertListEqual(result, expected)
def test__sun_events_update__one_plant(self):
self.createPlant()
# run sun_events
start = datetime.datetime(2021, 11, 29, 3, 0, 0, 0, tzinfo=datetime.timezone.utc)
end = datetime.datetime(2021, 11, 30, 3, 0, 0, 0, tzinfo=datetime.timezone.utc)
se = SunEvents()
se.sun_events_update(start=start, end=end)
with orm.db_session:
result = orm.select((s.plant.name, s.sunrise, s.sunset) for s in self.pony.db.SolarEvent)[:].to_list()
expected_sunrise = datetime.datetime(2021, 11, 29, 7, 23, 36, 140813, tzinfo=datetime.timezone.utc)
expected_sunset = datetime.datetime(2021, 11, 29, 15, 50, 5, 924495, tzinfo=datetime.timezone.utc)
expected = [('roger', expected_sunrise, expected_sunset)]
self.assertListEqual(result, expected)
class Models_Test(unittest.TestCase):
from yamlns.testutils import assertNsEqual
def setUp(self):
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
orm.rollback()
self.pony = PonyManager(envinfo.DB_CONF)
self.pony.define_all_models()
self.pony.binddb(create_tables=True)
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.create_tables()
orm.db_session.__enter__()
def tearDown(self):
orm.rollback()
orm.db_session.__exit__()
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.disconnect()
def samplePlantsNS(self):
alcoleaPlantsNS = ns.loads("""\
municipalities:
- municipality:
name: Figueres
ineCode: '17066'
countryCode: ES
country: Spain
regionCode: '09'
region: Catalonia
provinceCode: '17'
province: Girona
- municipality:
name: Girona
ineCode: '17079'
plants:
- plant:
name: alcolea
codename: SCSOM04
description: la bonica planta
municipality: '17066'
meters:
- meter:
name: '1234578'
inverters:
- inverter:
name: '5555'
- plant:
name: figuerea
codename: Som_figuerea
description: la bonica planta
municipality: '17079'
meters:
- meter:
name: '9876'
- meter:
name: '5432'
inverters:
- inverter:
name: '4444'
- inverter:
name: '2222'
irradiationSensors:
- irradiationSensor:
name: oriol
temperatureModuleSensors:
- temperatureModuleSensor:
name: joan
temperatureAmbientSensors:
- temperatureAmbientSensor:
name: benjami
""")
return alcoleaPlantsNS
def samplePlantNS(self):
alcoleaPlantNS = ns.loads("""\
name: alcolea
codename: SCSOM04
description: la bonica planta
meters:
- meter:
name: '1234578'
inverters:
- inverter:
name: '5555'
- inverter:
name: '6666'
irradiationSensors:
- irradiationSensor:
name: alberto
temperatureModuleSensors:
- temperatureModuleSensor:
name: pol
temperatureAmbientSensors:
- temperatureAmbientSensor:
name: joana
""")
return alcoleaPlantNS
def samplePlantNSWithStrings(self):
alcoleaPlantNS = ns.loads("""\
name: alcolea
codename: SCSOM04
description: la bonica planta
inverters:
- inverter:
name: '5555'
strings:
- string1
- string2
- inverter:
name: '6666'
""")
return alcoleaPlantNS
def samplePlantNSWithModuleParameters(self):
alcoleaPlantNS = ns.loads("""\
name: alcolea
codename: SCSOM04
description: la bonica planta
moduleParameters:
nominalPowerMWp: 2.16
efficiency: 15.5
nModules: 4878
Imp: 9.07
Vmp: 37.5
temperatureCoefficientI: 0.05
temperatureCoefficientV: -0.31
temperatureCoefficientPmax: -0.442
irradiationSTC: 1000.0
temperatureSTC: 25
degradation: 97.5
Voc: 46.1
Isc: 9.5
meters:
- meter:
name: '1234578'
inverters:
- inverter:
name: '5555'
- inverter:
name: '6666'
irradiationSensors:
- irradiationSensor:
name: alberto
temperatureModuleSensors:
- temperatureModuleSensor:
name: pol
temperatureAmbientSensors:
- temperatureAmbientSensor:
name: joana
""")
return alcoleaPlantNS
def samplePlantsData(self, time, dt):
plantsData = [{
"plant":
"alcolea",
"devices":
sorted([{
"id":
"Meter:1234578",
"readings": [{
"time": time,
"export_energy_wh": 1,
"import_energy_wh": 123,
"r1_VArh": 1234,
"r2_VArh": 124,
"r3_VArh": 1234,
"r4_VArh": 124,
}],
}, {
"id":
"Inverter:5555",
"readings": [{
"time": time,
"power_w": 156,
"energy_wh": 154,
"intensity_cc_mA": 222,
"intensity_ca_mA": 500,
"voltage_cc_mV": 200,
"voltage_ca_mV": 100,
"uptime_h": 15,
"temperature_dc": 170,
}]
}],
key=lambda d: d['id']),
}, {
"plant":
"figuerea",
"devices":
sorted([{
"id":
"Meter:9876",
"readings": [{
"time": time,
"export_energy_wh": 1,
"import_energy_wh": 123,
"r1_VArh": 1234,
"r2_VArh": 124,
"r3_VArh": 1234,
"r4_VArh": 124,
}],
}, {
"id":
"Meter:5432",
"readings": [{
"time": time,
"export_energy_wh": 1,
"import_energy_wh": 123,
"r1_VArh": 1234,
"r2_VArh": 124,
"r3_VArh": 1234,
"r4_VArh": 124,
}],
}],
key=lambda d: d['id']),
}]
return plantsData
def test_Plant_importExportPlant(self):
alcoleaPlantNS = self.samplePlantNS()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
orm.flush()
#TODO test the whole fixture, not just the plant data
plantns = alcolea.exportPlant()
self.assertNsEqual(plantns, alcoleaPlantNS)
def test_Plant_importExport_ExtraFields(self):
alcoleaPlantNS = self.samplePlantNS()
extraSensor = ns()
extraSensor['extraSensor'] = ns([('name', 'foosensor')])
extraSensor2 = ns()
extraSensor2['extraSensor'] = ns([('name', 'boosensor')])
alcoleaPlantNS['extraSensors'] = [extraSensor, extraSensor2]
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
orm.flush()
#TODO test the whole fixture, not just the plant data
plantns = alcolea.exportPlant()
self.assertNsEqual(plantns, self.samplePlantNS())
def test_Plant_importExport_MissingFields(self):
alcoleaPlantNS = self.samplePlantNS()
del alcoleaPlantNS['temperatureAmbientSensors']
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
orm.flush()
expectedPlantNS = alcoleaPlantNS
expectedPlantNS['temperatureAmbientSensors'] = []
#TODO test the whole fixture, not just the plant data
plantns = alcolea.exportPlant()
self.assertNsEqual(plantns, expectedPlantNS)
def test__Plant_importExport__ModuleParameters(self):
alcoleaPlantNS = self.samplePlantNSWithModuleParameters()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
orm.flush()
expectedPlantNS = alcoleaPlantNS
expectedPlantNS['moduleParameters'] = {
'nModules': 4878,
'Imp': 9070,
'Vmp': 37500,
'temperatureCoefficientI': 50,
'temperatureCoefficientV': -310,
'irradiationSTC': 1000,
'temperatureSTC': 250,
'degradation': 9750,
'Voc': 46100,
'Isc': 9500,
}
#TODO test the whole fixture, not just the plant data
plantns = alcolea.exportPlant()
self.assertNsEqual(plantns, expectedPlantNS)
def test__Plant_importExport__Strings(self):
alcoleaPlantNS = self.samplePlantNSWithStrings()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
orm.flush()
expectedPlantNS = alcoleaPlantNS
# expectedPlantNS.inverters[1].inverter['strings'] = []
#TODO test the whole fixture, not just the plant data
plantns = alcolea.exportPlant(skipEmpty=True)
self.assertNsEqual(plantns, expectedPlantNS)
def test_Plant_importExport_EmptyPlant(self):
emptyPlantNS = ns.loads("""\
name: alcolea
codename: SCSOM04
description: la bonica planta
""")
expectedPlantNS = ns.loads("""\
codename: SCSOM04
description: la bonica planta
inverters: []
irradiationSensors: []
meters: []
name: alcolea
temperatureAmbientSensors: []
temperatureModuleSensors: []
""")
empty = self.pony.db.Plant(name=emptyPlantNS.name,
codename=emptyPlantNS.codename)
empty = empty.importPlant(emptyPlantNS)
#TODO test the whole fixture, not just the plant data
plantns = empty.exportPlant()
self.assertNsEqual(plantns, expectedPlantNS)
def test_Plant_importPlants__manyPlants(self):
plantsns = self.samplePlantsNS()
importPlants(self.pony.db, plantsns)
resultPlantns = exportPlants(self.pony.db, skipEmpty=True)
self.assertNsEqual(plantsns, resultPlantns)
def test__Plant__getMeter(self):
plantName = 'alibaba'
alibaba = self.pony.db.Plant(
name=plantName, codename='SomEnergia_{}'.format(plantName))
meterName = '1234'
meter = self.pony.db.Meter(plant=alibaba, name=meterName)
orm.flush()
resultMeter = alibaba.getMeter()
self.assertEqual(resultMeter, meter)
def test__Plant__getMeter__ManyGivesNewest(self):
plantName = 'alibaba'
alibaba = self.pony.db.Plant(
name=plantName, codename='SomEnergia_{}'.format(plantName))
oldmeter = self.pony.db.Meter(plant=alibaba, name='oldmeter')
newmeter = self.pony.db.Meter(plant=alibaba, name='newmeter')
orm.flush()
resultMeter = alibaba.getMeter()
self.assertEqual(resultMeter, newmeter)
def test_Municipality__importMunicipality__OneMunicipality(self):
plantsns = ns()
municipalitiesns = ns.loads("""\
- municipality:
name: Figueres
ineCode: '17066'
countryCode: ES
country: Spain
regionCode: '08'
region: Catalonia
provinceCode: '17'
province: Girona
""")
plantsns['municipalities'] = municipalitiesns
importPlants(self.pony.db, plantsns)
storedMunicipalitiesns = exportPlants(self.pony.db)
plantsns['plants'] = []
self.assertNsEqual(plantsns, storedMunicipalitiesns)
def test_Plant_importPlants__ManyMunicipalities(self):
plantsns = ns()
municipalitiesns = ns.loads("""\
- municipality:
name: Figueres
ineCode: '17066'
- municipality:
name: Girona
ineCode: '17003'
""")
plantsns['municipalities'] = municipalitiesns
importPlants(self.pony.db, plantsns)
storedMunicipalitiesns = exportPlants(self.pony.db)
plantsns['plants'] = []
self.assertNsEqual(plantsns, storedMunicipalitiesns)
def test_Plant_importPlants__manyPlants_ManyMunicipalities(self):
plantsns = self.samplePlantsNS()
# municipalitiesns = ns.loads("""\
# - municipality:
# name: Figueres
# ineCode: '17066'
# - municipality:
# name: Girona
# ineCode: '17003'
# """)
# plantsns['municipalities'] = municipalitiesns
importPlants(self.pony.db, plantsns)
# orm.flush()
resultPlantns = exportPlants(self.pony.db, skipEmpty=True)
self.assertNsEqual(plantsns, resultPlantns)
def test__Plant_importPlantsData__many(self):
plantsns = self.samplePlantsNS()
importPlants(self.pony.db, plantsns)
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
delta = dt.timedelta(minutes=30)
plantsData = self.samplePlantsData(time, delta)
self.pony.db.Plant.insertPlantsData(plantsData)
# orm.flush()
for plantData in plantsData:
self.assertDictEqual(
dict(plantData),
self.pony.db.Plant.get(name=plantData['plant']).plantData(
skipEmpty=True))
def test__Plant_str2model__sameNameDifferentPlant(self):
plantsns = self.samplePlantsNS()
plantsns.plants[0].plant.inverters[0].inverter.name = '4444'
importPlants(self.pony.db, plantsns)
orm.flush()
plant = self.pony.db.Plant.get(name='alcolea')
alcoleainverter = self.pony.db.Plant.str2device(
plant, 'Inverter', '4444')
self.assertNotEqual(alcoleainverter, None)
plant = self.pony.db.Plant.get(name='figuerea')
figuereainverter = self.pony.db.Plant.str2device(
plant, 'Inverter', '4444')
self.assertNotEqual(figuereainverter, None)
self.assertNotEqual(alcoleainverter.plant.name,
figuereainverter.plant.name)
def test__Meter_getRegistries__emptyRegistries(self):
alcoleaPlantNS = self.samplePlantNS()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
registries = self.pony.db.Meter[1].getRegistries()
expectedRegistries = []
self.assertListEqual(registries, expectedRegistries)
def test__Meter_getRegistries__OneRegistry(self):
alcoleaPlantNS = self.samplePlantNS()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
self.pony.db.Meter[1].insertRegistry(
time=time,
export_energy_wh=10,
import_energy_wh=5,
r1_VArh=3,
r2_VArh=2,
r3_VArh=4,
r4_VArh=1,
)
registries = self.pony.db.Meter[1].getRegistries()
expectedRegistries = [{
'time': time,
'export_energy_wh': 10,
'import_energy_wh': 5,
'r1_VArh': 3,
'r2_VArh': 2,
'r3_VArh': 4,
'r4_VArh': 1,
}]
self.assertListEqual(registries, expectedRegistries)
def test__Meter_getRegistries__OneRegistry_two_meters(self):
alcoleaPlantNS = self.samplePlantNS()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
self.pony.db.Meter(plant=alcolea, name="Albertinho")
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
self.pony.db.Meter[1].insertRegistry(
time=time,
export_energy_wh=10,
import_energy_wh=5,
r1_VArh=3,
r2_VArh=2,
r3_VArh=4,
r4_VArh=1,
)
self.pony.db.Meter[2].insertRegistry(
time=time,
export_energy_wh=110,
import_energy_wh=15,
r1_VArh=13,
r2_VArh=12,
r3_VArh=14,
r4_VArh=11,
)
registries = self.pony.db.Meter[1].getRegistries()
expectedRegistries = [{
'time': time,
'export_energy_wh': 10,
'import_energy_wh': 5,
'r1_VArh': 3,
'r2_VArh': 2,
'r3_VArh': 4,
'r4_VArh': 1,
}]
self.assertListEqual(registries, expectedRegistries)
def test__Meter_getRegistries__OneRegistry_filter_date(self):
with orm.db_session:
alcoleaPlantNS = self.samplePlantNS()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
delta = dt.timedelta(minutes=30)
self.pony.db.Meter[1].insertRegistry(
time=time,
export_energy_wh=10,
import_energy_wh=5,
r1_VArh=3,
r2_VArh=2,
r3_VArh=4,
r4_VArh=1,
)
fromdate = time - delta
todate = time + delta
registries = self.pony.db.Meter[1].getRegistries(fromdate=fromdate,
todate=todate)
expectedRegistries = [{
'time': time,
'export_energy_wh': 10,
'import_energy_wh': 5,
'r1_VArh': 3,
'r2_VArh': 2,
'r3_VArh': 4,
'r4_VArh': 1,
}]
self.assertListEqual(registries, expectedRegistries)
def test__Meter_getLastReadingsDate__oneMeterOnePlantEmptyReadings(self):
alcoleaPlantNS = self.samplePlantNS()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
result = self.pony.db.Meter[1].getLastReadingDate()
self.assertIsNone(result)
def test__Meter_getLastReadingsDate__oneMeterOnePlantOneReading(self):
alcoleaPlantNS = self.samplePlantNS()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
delta = dt.timedelta(minutes=30)
meter = self.pony.db.Meter[1]
meter.insertRegistry(
time=time,
export_energy_wh=10,
import_energy_wh=5,
r1_VArh=3,
r2_VArh=2,
r3_VArh=4,
r4_VArh=1,
)
result = meter.getLastReadingDate()
expectedResult = time
self.assertEqual(result, expectedResult)
def test__inverter_getRegistries__OneRegistry(self):
alcoleaPlantNS = self.samplePlantNS()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
self.pony.db.Inverter[1].insertRegistry(
power_w=1,
energy_wh=2,
intensity_cc_mA=3,
intensity_ca_mA=4,
voltage_cc_mV=5,
voltage_ca_mV=6,
uptime_h=7,
temperature_dc=8,
time=time,
)
registries = self.pony.db.Inverter[1].getRegistries()
expectedRegistries = [{
"power_w": 1,
"energy_wh": 2,
"intensity_cc_mA": 3,
"intensity_ca_mA": 4,
"voltage_cc_mV": 5,
"voltage_ca_mV": 6,
"uptime_h": 7,
"temperature_dc": 8,
"time": time,
}]
self.assertListEqual(registries, expectedRegistries)
def test__string_getRegistries__OneRegistry(self):
alcoleaPlantNS = self.samplePlantNSWithStrings()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
self.pony.db.String[1].insertRegistry(
intensity_mA=3,
time=time,
)
registries = self.pony.db.String[1].getRegistries()
expectedRegistries = [{
"intensity_mA": 3,
"time": time,
}]
self.assertListEqual(registries, expectedRegistries)
def __test__forecastMetadata(self):
pass
def test__createDevice(self):
oneplant = self.pony.db.Plant(name="Alice", codename="LaSuisse")
oneplant.createDevice("Inverter", "Bob")
plantData = oneplant.exportPlant()
expectedPlant = {
"name": "Alice",
"codename": "LaSuisse",
"description": '',
"meters": [],
"inverters": [{
"inverter": {
"name": "Bob"
},
}],
"irradiationSensors": [],
"temperatureAmbientSensors": [],
"temperatureModuleSensors": [],
}
self.assertDictEqual(plantData, expectedPlant)
def test__Plant__plantData__InverterWithStrings(self):
alcoleaPlantNS = self.samplePlantNSWithStrings()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
registry = {
'energy_wh': 100,
'intensity_ca_mA': 1,
'intensity_cc_mA': 1,
'power_w': 100,
'temperature_dc': 1,
'time': time,
'uptime_h': 1,
'voltage_ca_mV': 1,
'voltage_cc_mV': 1
}
self.pony.db.Inverter[1].insertRegistry(**registry)
plantdata = alcolea.plantData()
expectedPlantData = {
"plant":
"alcolea",
"devices": [{
'id': 'Inverter:5555',
'readings': [registry]
}, {
'id': 'Inverter:6666',
'readings': []
}]
}
expectedPlantData["devices"].sort(key=lambda x: x['id'])
self.assertDictEqual(plantdata, expectedPlantData)
# TODO fix plantData construction of an inverter with strings
def _test__Inverter__plantData__withStrings(self):
alcoleaPlantNS = self.samplePlantNSWithStrings()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
inverterRegistry = {
'energy_wh': 100,
'intensity_ca_mA': 1,
'intensity_cc_mA': 1,
'power_w': 100,
'temperature_dc': 1,
'time': time,
'uptime_h': 1,
'voltage_ca_mV': 1,
'voltage_cc_mV': 1
}
stringsRegistry = [100, 200]
self.pony.db.Inverter[1].insertRegistry(**inverterRegistry)
self.pony.db.String[1].insertRegistry(intensity_mA=stringsRegistry[0],
time=time)
self.pony.db.String[2].insertRegistry(intensity_mA=stringsRegistry[1],
time=time)
inverterPlantdata = self.pony.db.Inverter[1].plantData()
expected = {
'id':
'Inverter:5555',
'readings': [{
**inverterRegistry, 'String_intensity_mA:string1': 100,
'String_intensity_mA:string2': 200
}]
}
self.assertDictEqual(plantdata, expected)
def _test__String__plantData(self):
alcoleaPlantNS = self.samplePlantNSWithStrings()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
inverterRegistry = {
'energy_wh': 100,
'intensity_ca_mA': 1,
'intensity_cc_mA': 1,
'power_w': 100,
'temperature_dc': 1,
'time': time,
'uptime_h': 1,
'voltage_ca_mV': 1,
'voltage_cc_mV': 1
}
stringsRegistry = [100, 200]
self.pony.db.Inverter[1].insertRegistry(**inverterRegistry)
self.pony.db.String[1].insertRegistry(intensity_mA=stringsRegistry[0],
time=time)
self.pony.db.String[2].insertRegistry(intensity_mA=stringsRegistry[1],
time=time)
plantdata = alcolea.plantData()
# option 1b
expectedPlantData = {
"plant":
"alcolea",
"devices": [{
'id':
'Inverter:5555',
'readings': [{
**inverterRegistry, 'String_intensity_mA:string1':
100,
'String_intensity_mA:string2':
200
}]
}, {
'id': 'Inverter:6666',
'readings': []
}]
}
expectedPlantData["devices"].sort(key=lambda x: x['id'])
self.assertDictEqual(plantdata, expectedPlantData)
def test__Plant__plantData(self):
alcoleaPlantNS = self.samplePlantNS()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
mRegistry = {
"export_energy_wh": 10,
"import_energy_wh": 5,
"r1_VArh": 3,
"r2_VArh": 2,
"r3_VArh": 4,
"r4_VArh": 1,
"time": time,
}
self.pony.db.Meter[1].insertRegistry(**mRegistry)
self.pony.db.SensorIrradiation[1].insertRegistry(
time=time,
irradiation_w_m2=15,
temperature_dc=250,
)
plantdata = alcolea.plantData()
expectedPlantData = {
"plant":
"alcolea",
"devices": [{
'id': 'Inverter:5555',
'readings': []
}, {
'id': 'Inverter:6666',
'readings': []
}, {
"id": "Meter:1234578",
"readings": [mRegistry]
}, {
"id":
"SensorIrradiation:alberto",
"readings": [{
"irradiation_w_m2": 15,
'temperature_dc': 250,
"time": time,
}]
}, {
"id": "SensorTemperatureAmbient:joana",
"readings": []
}, {
"id": "SensorTemperatureModule:pol",
"readings": []
}]
}
expectedPlantData["devices"].sort(key=lambda x: x['id'])
self.assertDictEqual(plantdata, expectedPlantData)
def test__Plant_insertPlantData(self):
alcoleaPlantNS = self.samplePlantNS()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
orm.flush()
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
plantData = {
"plant":
"alcolea",
"devices": [{
'id': 'Inverter:5555',
'readings': []
}, {
'id': 'Inverter:6666',
'readings': []
}, {
"id":
"Meter:1234578",
"readings": [{
"export_energy_wh": 10,
"import_energy_wh": 5,
"r1_VArh": 3,
"r2_VArh": 2,
"r3_VArh": 4,
"r4_VArh": 1,
"time": time,
}]
}, {
"id":
"SensorIrradiation:alberto",
"readings": [{
"irradiation_w_m2": 15,
"temperature_dc": 250,
"time": time,
}]
}, {
"id": "SensorTemperatureAmbient:joana",
"readings": []
}, {
"id": "SensorTemperatureModule:pol",
"readings": []
}]
}
alcolea.insertPlantData(plantData)
plantDataResult = alcolea.plantData()
self.assertDictEqual(plantData, plantDataResult)
def _test__Plant_insertPlantData__Strings__Empty(self):
alcoleaPlantNS = self.samplePlantNSWithStrings()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
orm.flush()
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
plantData = {
"plant":
"alcolea",
"devices": [{
'id': 'Inverter:5555',
'readings': []
}, {
'id': 'Inverter:6666',
'readings': []
}]
}
alcolea.insertPlantData(plantData)
plantDataResult = alcolea.plantData()
self.assertDictEqual(plantData, plantDataResult)
def test__Plant_insertPlantData__NewStrings__OneInverter(self):
alcoleaPlantNS = self.samplePlantNSWithStrings()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
orm.flush()
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
plantData = {
"plant":
"alcolea",
"devices": [{
'id':
'Inverter:5555',
'readings': [{
'energy_wh': 100,
'intensity_ca_mA': 1,
'intensity_cc_mA': 1,
'power_w': 100,
'temperature_dc': 1,
'time': time,
'uptime_h': 1,
'voltage_ca_mV': 1,
'voltage_cc_mV': 1,
'String:string1:intensity_mA': 100,
'String:string2:intensity_mA': 200
}]
}, {
'id': 'Inverter:6666',
'readings': []
}]
}
alcolea.insertPlantData(plantData)
# TODO build a plantData with data with strings
plantdata_result = alcolea.plantData()
del plantData['devices'][0]['readings'][0][
'String:string1:intensity_mA']
del plantData['devices'][0]['readings'][0][
'String:string2:intensity_mA']
self.assertDictEqual(plantData, plantdata_result)
expected_strings = [{
'time': time,
'intensity_mA': 100
}, {
'time': time,
'intensity_mA': 200
}]
string_readings = [{
'time': r.time,
'intensity_mA': r.intensity_mA
} for r in self.pony.db.StringRegistry.select()]
self.assertListEqual(string_readings, expected_strings)
def test__municipality_insert(self):
figueres = self.pony.db.Municipality(
countryCode="ES",
country="Spain",
regionCode="09",
region="Catalonia",
provinceCode="17",
province="Girona",
ineCode="17066",
name="Figueres",
)
alcolea = self.pony.db.Plant(name="Alcolea",
codename="Som_Alcolea",
municipality=figueres)
municipality = self.pony.db.Plant.get(name="Alcolea").municipality
self.assertDictEqual(figueres.to_dict(), municipality.to_dict())
def test__location_insert(self):
alcolea = self.pony.db.Plant(name="Alcolea", codename="Som_Alcolea")
alcoleaLatLong = (42.26610810248693, 2.958334450785599)
alcoleaLocation = self.pony.db.PlantLocation(
plant=alcolea,
latitude=alcoleaLatLong[0],
longitude=alcoleaLatLong[1])
alcolea.location = alcoleaLocation
location = self.pony.db.Plant.get(name="Alcolea").location
self.assertEqual(alcoleaLatLong, location.getLatLong())
def test__importPlants__oneplant(self):
pass
def test__Meter_updateMeterProtocol_byDefault(self):
alcoleaPlantNS = self.samplePlantNSWithModuleParameters()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
orm.flush()
meter = self.pony.db.Meter.get(name='1234578')
self.assertEqual(meter.connection_protocol, 'ip')
def test__Meter_updateMeterProtocol_afterUpdate(self):
alcoleaPlantNS = self.samplePlantNSWithModuleParameters()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
orm.flush()
self.pony.db.Meter.updateMeterProtocol({
'1234578': 'moxa',
})
meter = self.pony.db.Meter.get(name='1234578')
self.assertEqual(meter.connection_protocol, 'moxa')
def test__Meter_updateMeterProtocol_unknownMetersIgnored(self):
# This adds many more meters such as 9876 which we are not updating
plantsns = self.samplePlantsNS()
importPlants(self.pony.db, plantsns)
self.pony.db.Meter.updateMeterProtocol({
'1234578': 'moxa',
})
meter = self.pony.db.Meter.get(name='1234578')
self.assertEqual(meter.connection_protocol, 'moxa')
meter = self.pony.db.Meter.get(name='9876')
self.assertEqual(meter.connection_protocol, 'ip')
def test__Meter_updateMeterProtocol_deprecatedMetersIgnored(self):
plantsns = self.samplePlantsNS()
importPlants(self.pony.db, plantsns)
# Should not fail
self.pony.db.Meter.updateMeterProtocol({
'1234578': 'moxa',
'OLDMETER': 'moxa',
})
meter = self.pony.db.Meter.get(name='OLDMETER')
self.assertEqual(meter, None)
@unittest.skipIf(
True,
"we don't support String creation directly, only as inverter registry")
def test__String_createDevice(self):
alcoleaPlantNS = self.samplePlantNS()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea.importPlant(alcoleaPlantNS)
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
dev = alcolea.createDevice(classname='String',
devicename='5555-string1')
expected = {'id': 1, 'inverter': 1, 'name': 'string1'}
self.assertIsNotNone(dev)
self.assertDictEqual(expected, dev.to_dict())
def test__String_insertRegistry__new(self):
alcoleaPlantNS = self.samplePlantNS()
alcolea = self.pony.db.Plant(name=alcoleaPlantNS.name,
codename=alcoleaPlantNS.codename)
alcolea = alcolea.importPlant(alcoleaPlantNS)
time = dt.datetime(2020,
12,
10,
15,
5,
10,
588861,
tzinfo=dt.timezone.utc)
stringIntensities = [400, 400, 300, 500, 2000, 300, 400, 1000, 300]
self.pony.db.Inverter[1].insertRegistry(
power_w=1,
energy_wh=2,
intensity_cc_mA=3,
intensity_ca_mA=4,
voltage_cc_mV=5,
voltage_ca_mV=6,
uptime_h=7,
temperature_dc=8,
time=time,
)
registries = self.pony.db.Inverter[1].getRegistries()
expectedRegistries = [{
"power_w": 1,
"energy_wh": 2,
"intensity_cc_mA": 3,
"intensity_ca_mA": 4,
"voltage_cc_mV": 5,
"voltage_ca_mV": 6,
"uptime_h": 7,
"temperature_dc": 8,
"time": time,
}]
self.assertListEqual(registries, expectedRegistries)
class Migrations_Test(unittest.TestCase):
from yamlns.testutils import assertNsEqual
def createConfig(self):
configdb = ns.load('conf/configdb_test.yaml')
#TODO add dump database setting (if we end up not replicating it)
configdb['psql_db'] = "testdump"
return configdb
def setUpORM(self):
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
orm.rollback()
self.pony = PonyManager(envinfo.DB_CONF)
self.pony.define_all_models()
self.pony.binddb(create_tables=True)
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.create_tables()
self.maxDiff = None
# database.generate_mapping(create_tables=True)
# orm.db_session.__enter__()
def tearDownORM(self):
orm.rollback()
# orm.db_session.__exit__()
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.disconnect()
def setUp(self):
# configdb = self.createConfig()
# dumpfile = configdb['dumpfile']
# print("dumpfile: {}".format(dumpfile))
# PlantmonitorDB(configdb).demoDBsetupFromDump(configdb, dumpfile)
self.setUpORM()
def tearDown(self):
# configdb = self.createConfig()
# PlantmonitorDB(configdb).dropDatabase(configdb)
self.tearDownORM()
def createPlantmonitorDB(self):
configdb = self.createConfig()
return PlantmonitorDB(configdb)
def mainFacility(self):
return 'SomEnergia_Amapola'
def __test_playground(self):
self.assertTrue(False)
def __test_loadDump(self):
# Get data to migrate
with self.createPlantmonitorDB() as db:
dumpdir = "./dumpPlantmonitor"
excluded = [
'facility_meter', 'teoric_energy', 'integrated_sensors'
]
loadDump(db, dumpdir, excluded)
# TODO check data
cur = db._client.cursor()
cur.execute("select count(*) from facility_meter;")
facilityMeterCountReg = cur.fetchall()
cur.execute("select count(*) from integrated_sensors;")
integratedSensorsCountReg = cur.fetchall()
self.assertEqual(facilityMeterCountReg, 123)
self.assertEqual(integratedSensorsCountReg, 123)
def test_insertInverterRegistrySignature(self):
#if the insertInvertRegistrySignature changes, the migration code has to
# change as well, given that it relies on column order
signature = self.pony.db.Inverter.insertRegistry.__code__.co_varnames
expectedSignature = (
"self",
"power_w",
"energy_wh",
"intensity_cc_mA",
"intensity_ca_mA",
"voltage_cc_mV",
"voltage_ca_mV",
"uptime_h",
"temperature_dc",
"time",
)
self.assertTupleEqual(expectedSignature, signature)
def test_tableInverterColumns(self):
# if the insertInvertRegistrySignature changes, the migration code has to
# change as well, given that it relies on column order
with self.createPlantmonitorDB() as db:
cur = db._client.cursor()
cur.execute('select * from sistema_inversor limit 1;')
colNames = [c[0] for c in cur.description]
expectedColnames = [
'time', 'inverter_name', 'location', '1HR', '1HR0', '2HR2',
'3HR3', '4HR4', 'daily_energy_h', 'daily_energy_l',
'e_total_h', 'e_total_l', 'h_total_h', 'h_total_l', 'pac_r',
'pac_s', 'pac_t', 'powereactive_t', 'powerreactive_r',
'powerreactive_s', 'probe1value', 'probe2value', 'probe3value',
'probe4value', 'temp_inv'
]
self.assertListEqual(expectedColnames, colNames)
def __test_orderbytime(self):
# pg_dump pg_restore fails to restore the timescale time index
with self.createPlantmonitorDB() as db:
cur = db._client.cursor()
cur.execute(
"select time from sistema_inversor order by time asc limit 1;")
records = cur.fetchall()
self.assertEqual(len(records), 1)
def test_Plant_importExportPlant(self):
with orm.db_session:
alcoleaPlantYAML = ns.loads("""\
plants:
- plant:
name: alcolea
codename: SCSOM04
description: la bonica planta
meters:
- meter:
name: '1234578'
inverters:
- inverter:
name: '5555'
- inverter:
name: '6666'
irradiationSensors:
- irradiationSensor:
name: alberto
temperatureAmbientSensors:
- temperatureAmbientSensor:
name: joana
temperatureModuleSensors:
- temperatureModuleSensor:
name: juanpe
integratedSensors:
- integratedSensor:
name: voki""")
alcoleaPlant = alcoleaPlantYAML.plants[0].plant
alcolea = self.pony.db.Plant(name=alcoleaPlant.name,
codename=alcoleaPlant.codename)
alcolea = alcolea.importPlant(alcoleaPlant)
orm.flush()
#TODO test the whole fixture, not just the plant data
plantns = alcolea.exportPlant()
self.assertNsEqual(plantns, alcoleaPlant)
def test_db_records(self):
# Get data to migrate
with self.createPlantmonitorDB() as db:
curr = db._client.cursor()
curr.execute("select current_database()")
db_name = curr.fetchone()[0]
self.assertEqual(db_name, "testdump")
curr.execute("select count(*) from sistema_inversor;")
numrecords = curr.fetchone()[0]
self.assertGreater(numrecords, 100)
curr.execute(
'select time, inverter_name, location, "1HR", daily_energy_h, daily_energy_l from sistema_inversor;'
)
records = curr.fetchmany(2)
# WARNING fetchall does not work on timescale tables / ultrabig tables
# we might need server-side cursors
# https://www.psycopg.org/docs/usage.html#server-side-cursors
#curr.fetchall()
self.assertEqual(records,
[(dt.datetime(2020, 2, 27, 0, 0, 2, 958223),
'inversor1', 'Florida', None, 0, 0),
(dt.datetime(2020, 2, 27, 0, 0, 3, 152901),
'inversor2', 'Florida', None, 0, 0)])
def __test_registryInsert_PhantomObjectDebug(self):
with orm.db_session:
plant = Plant(name='alcolea', codename='alcolea')
inverter = Inverter(name='inversor1', plant=plant)
time = dt.datetime(2020, 2, 27, 0, 45, 57, 520238)
# time = pytz.utc.localize(time)
time = time.replace(tzinfo=dt.timezone.utc)
temp_inv = 190
values = [0, 0, 492, 29998, 0, 16166, 0, 0, 0, 0, 0, 0]
inverter.insertRegistry(*values, temp_inv_dc=temp_inv, time=time)
orm.select(r for r in self.pony.db.InverterRegistry).show()
def __test_migrateLegacyTableToPony_meter(self):
tableName = 'sistema_contador'
plantColumnName = 'location'
deviceColumnName = 'contador'
deviceType = 'Meter'
with orm.db_session:
migrateLegacyInverterTableToPony(self.createConfig(),
tableName=tableName,
plantColumnName=plantColumnName,
deviceColumnName=deviceColumnName,
deviceType=deviceType,
excerpt=True)
def test_migrateLegacySensorIrradiationTableToPony_sensor(self):
tableName = 'sensors'
dataColumnName = 'irradiation_w_m2'
plantName = 'Alcolea'
deviceName = 'irradiation_alcolea'
deviceType = 'SensorIrradiation'
migrateLegacySensorIrradiationTableToPony(self.createConfig(),
plantName=plantName,
deviceName=deviceName,
excerpt=True)
with self.createPlantmonitorDB() as db:
# retrieve expected
curr = db._client.cursor()
curr.execute("select time, {} from {} limit 1;".format(
dataColumnName, tableName))
expectedFirstRegistry = list(curr.fetchone())
expectedTime = expectedFirstRegistry[0].replace(tzinfo=dt.timezone.utc)
expectedIrrValue = expectedFirstRegistry[1]
expectedTempValue = expectedFirstRegistry[1]
expectedMigrateRegistryList = [
expectedTime, plantName, deviceName, expectedIrrValue,
expectedTempValue
]
with orm.db_session:
query = self.pony.db.SensorIrradiationRegistry.select().order_by(
SensorIrradiationRegistry.time)
migratedRegistry = query.first()
id, time, irrValue, tempValue = list(
migratedRegistry.to_dict().values())
migratedRegistryList = [
time, migratedRegistry.sensor.plant.name,
migratedRegistry.sensor.name, irrValue, tempValue
]
self.assertListEqual(migratedRegistryList, expectedMigrateRegistryList)
# @TODO deprecated integrated sensors are deprecated
def _test_migrateLegacySensorTableToPony_sensor(self):
tableName = 'integrated_sensors'
dataColumnName = 'integral_irradiation_wh_m2'
plantName = 'Alcolea'
deviceName = 'irradiation_alcolea'
deviceType = 'SensorIntegratedIrradiation'
migrateLegacySensorTableToPony(self.createConfig(),
plantName=plantName,
tableName=tableName,
deviceType=deviceType,
dataColumnName=dataColumnName,
deviceName=deviceName,
excerpt=True)
with self.createPlantmonitorDB() as db:
# retrieve expected
curr = db._client.cursor()
curr.execute("select time, {} from {} limit 1;".format(
dataColumnName, tableName))
expectedFirstRegistry = list(curr.fetchone())
expectedTime = expectedFirstRegistry[0].replace(tzinfo=dt.timezone.utc)
expectedValue = expectedFirstRegistry[1]
expectedMigrateRegistryList = [
expectedTime, plantName, deviceName, expectedValue
]
with orm.db_session:
query = self.pony.db.HourlySensorIrradiationRegistry.select(
).order_by(HourlySensorIrradiationRegistry.time)
migratedRegistry = query.first()
id, time, migratedValue = list(migratedRegistry.to_dict().values())
migratedRegistryList = [
time, migratedRegistry.sensor.plant.name,
migratedRegistry.sensor.name, migratedValue
]
self.assertListEqual(migratedRegistryList, expectedMigrateRegistryList)
# TODO fix this test
def test_migrateLegacyMeterTableToPony_checkFacilityMeter(self):
plantName = 'Alcolea'
createPlants()
migrateLegacyMeterTableToPony(self.createConfig(), excerpt=True)
dbFacilityMeter = {}
with self.createPlantmonitorDB() as db:
# retrieve expected
dbFacilityMeter = db.getFacilityMeter()
# excluded plants
excludedFacilities = [
'SomEnergia_Exiom', 'SomEnergia_Fontivsolar',
'SomEnergia_La_Florida', 'SomEnergia_Alcolea'
]
dbFacilityMeter = set(
(f, m) for f, m in dbFacilityMeter if f not in excludedFacilities)
with orm.db_session:
q = orm.select((m.plant.codename, m.name) for m in Meter)
ormFacilityMeter = set(q[:])
# fetch facilitymeter data
self.assertSetEqual(ormFacilityMeter, dbFacilityMeter)
def test_migrateLegacyMeterTableToPony_meters(self):
plantName = 'Alcolea'
meterName = '501600324'
createPlants()
migrateLegacyMeterTableToPony(self.createConfig(), excerpt=True)
with self.createPlantmonitorDB() as db:
# retrieve expected
curr = db._client.cursor()
curr.execute(
"select * from sistema_contador where name = '{}' limit 1;".
format(meterName))
expectedFirstRegistry = list(curr.fetchone())
expectedTime = expectedFirstRegistry[0].replace(tzinfo=dt.timezone.utc)
expectedMeterName = expectedFirstRegistry[1]
expectedValuesList = expectedFirstRegistry[2:]
expectedMigrateRegistryList = [
expectedTime, plantName, expectedMeterName
] + expectedValuesList
with orm.db_session:
query = self.pony.db.MeterRegistry.select().order_by(
self.pony.db.MeterRegistry.time)
migratedRegistry = query.first()
id, time, *migratedRegistrylist = list(
migratedRegistry.to_dict().values())
migratedRegistryList = [
time, migratedRegistry.meter.plant.name,
migratedRegistry.meter.name
] + migratedRegistrylist
self.assertListEqual(migratedRegistryList, expectedMigrateRegistryList)
def __test_migrateLegacyTableToPony_sensor(self):
tableName = 'sensors'
plantColumnName = None
deviceColumnName = None
deviceType = 'SensorIrradiation'
migrateLegacyInverterTableToPony(self.createConfig(),
tableName=tableName,
plantColumnName=plantColumnName,
deviceColumnName=deviceColumnName,
deviceType=deviceType,
excerpt=True)
def test_migrateLegacyInverterTableToPony_10records(self):
# Get data to migrate
# Assumed there on db
numrecords = 10
# TODO fix order by time or extract first record plant and inverter_name from pony
plantName = 'Alcolea'
inversorName = 'inversor1'
migrateLegacyInverterTableToPony(self.createConfig(), excerpt=True)
with self.createPlantmonitorDB() as db:
# retrieve expected
curr = db._client.cursor()
#curr.execute("select * from sistema_inversor where order by location, inverter_name, time desc limit 10;")
curr.execute(
"select distinct on (time, location, inverter_name) * from sistema_inversor where location = '{}' and inverter_name = '{}' limit 1;"
.format(plantName, inversorName))
expectedFirstRegistry = list(curr.fetchone())
with orm.db_session:
numplants = orm.count(p for p in self.pony.db.Plant)
numinverters = orm.count(i for i in self.pony.db.Inverter)
numregistries = orm.count(r for r in self.pony.db.InverterRegistry)
self.assertEqual(numinverters * numrecords, numregistries)
expectedTime = expectedFirstRegistry[0].replace(tzinfo=dt.timezone.utc)
expectedPlant = expectedFirstRegistry[2]
expectedInverter = expectedFirstRegistry[1]
expectedEnergy = int(
round((expectedFirstRegistry[8] << 16) + expectedFirstRegistry[9]))
expectedTempInv = expectedFirstRegistry[-1] * 100
expectedMigrateRegistryList = [
expectedTime, expectedPlant, expectedInverter, expectedEnergy,
expectedTempInv
]
with orm.db_session:
query = orm.select(
r for r in self.pony.db.InverterRegistry
if r.inverter.name == expectedInverter
and r.inverter.plant.name == expectedPlant).order_by(
InverterRegistry.time)
migratedRegistry = query.first()
id, time, power, energy, *migratedRegistryList, temperature_dc = list(
migratedRegistry.to_dict().values())
migratedRegistryList = [
time, migratedRegistry.inverter.plant.name,
migratedRegistry.inverter.name, energy, temperature_dc
]
self.assertListEqual(migratedRegistryList, expectedMigrateRegistryList)
def _test_fullMigration_plants(self):
migrateLegacyToPony(self.createConfig(), excerpt=True)
expectedPlants = [
'Alcolea', 'Riudarenes_SM', 'Fontivsolar I', 'Riudarenes_BR',
'Riudarenes_ZE', 'Florida', 'Matallana'
]
with orm.db_session:
plants = list(orm.select(p.name for p in self.pony.db.Plant))
self.assertCountEqual(plants, expectedPlants)
def test_fullMigration_FloridaCheck(self):
migrateLegacyToPony(self.createConfig(), excerpt=True)
with orm.db_session:
plants = list(
orm.select(p.name for p in self.pony.db.Plant
if p.name == "La Florida"))
self.assertListEqual(plants, [])
def __test_migrateLegacyInverterTableToPony_DaylightSavingTime(self):
# TODO
migrateLegacyInverterTableToPony(self.createConfig(), excerpt=False)
# skipped per slow, but numbers don't match 1769262 vs 1769465
def __test_fullMigration_InverterCount_noexcerpt(self):
migrateLegacyInverterTableToPony(self.createConfig(), excerpt=False)
with self.createPlantmonitorDB() as db:
curr = db._client.cursor()
#curr.execute("select * from sistema_inversor where order by location, inverter_name, time desc limit 10;")
curr.execute(
"select count(distinct (time, location, inverter_name)) from sistema_inversor"
)
expectednumregistries = curr.fetchone()[0]
with orm.db_session:
numregistries = orm.count(r for r in self.pony.db.InverterRegistry)
self.assertEqual(numregistries, expectednumregistries)
# TODO there are repeated records 2020-02-26 15:00:00 with different energy values
# How does distinct choose one
def __test_fullMigration_MeterCount_noexcerpt(self):
migrateLegacyMeterTableToPony(self.createConfig(), excerpt=True)
with self.createPlantmonitorDB() as db:
curr = db._client.cursor()
#curr.execute("select * from sistema_inversor where order by location, inverter_name, time desc limit 10;")
curr.execute(
"select count(distinct (time, name)) from sistema_contador")
expectednumregistries = curr.fetchone()[0]
numregistries = orm.count(r for r in self.pony.db.MeterRegistry)
self.assertEqual(numregistries, expectednumregistries)
def __test_fullMigration_noexcerpt(self):
migrateLegacyToPony(self.createConfig(), excerpt=False)
with self.createPlantmonitorDB() as db:
curr = db._client.cursor()
#curr.execute("select * from sistema_inversor where order by location, inverter_name, time desc limit 10;")
curr.execute(
"select count(distinct (time, location, inverter_name)) from sistema_inversor"
)
expectednumInverterRegistries = curr.fetchone()[0]
curr.execute("select count(*) from sistema_contador")
expectednumMeterRegistries = curr.fetchone()[0]
with orm.db_session:
numInverterRegistries = orm.count(
r for r in self.pony.db.InverterRegistry)
self.assertEqual(numInverterRegistries,
expectednumInverterRegistries)
numMeterRegistries = orm.count(r
for r in self.pony.db.MeterRegistry)
self.assertEqual(numMeterRegistries, expectednumMeterRegistries)
def test_fullMigration_excerpt(self):
expectednumInverterRegistries = 60
expectednumMeterRegistries = 70
migrateLegacyToPony(self.createConfig(), excerpt=True)
with orm.db_session:
numInverterRegistries = orm.count(
r for r in self.pony.db.InverterRegistry)
self.assertEqual(numInverterRegistries,
expectednumInverterRegistries)
numMeterRegistries = orm.count(r
for r in self.pony.db.MeterRegistry)
self.assertEqual(numMeterRegistries, expectednumMeterRegistries)
class ReadingsFacade_Test(unittest.TestCase):
def setUp(self):
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
orm.rollback()
self.pony = PonyManager(envinfo.DB_CONF)
self.pony.define_all_models()
self.pony.binddb(create_tables=True)
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.create_tables()
orm.db_session.__enter__()
def tearDown(self):
orm.rollback()
orm.db_session.__exit__()
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.disconnect()
def samplePlantsNS(self, altMeter="12345678"):
alcoleaPlantsNS = ns.loads("""\
municipalities:
- municipality:
name: Figueres
ineCode: '17066'
countryCode: ES
country: Spain
regionCode: '09'
region: Catalonia
provinceCode: '17'
province: Girona
- municipality:
name: Girona
ineCode: '17079'
plants:
- plant:
name: alcolea
codename: SCSOM04
description: la bonica planta
municipality: '17066'
meters:
- meter:
name: '{}'
inverters:
- inverter:
name: '5555'
- plant:
name: figuerea
codename: Som_figuerea
description: la bonica planta
municipality: '17079'
meters:
- meter:
name: '5432'
- meter:
name: '9876'
inverters:
- inverter:
name: '4444'
- inverter:
name: '2222'
irradiationSensors:
- irradiationSensor:
name: oriol
temperatureModuleSensors:
- temperatureModuleSensor:
name: joan
temperatureAmbientSensors:
- temperatureAmbientSensor:
name: benjami
integratedSensors:
- integratedSensor:
name: david""".format(altMeter))
return alcoleaPlantsNS
def samplePlantsData(self, time, dt, altMeter="12345678"):
plantsData = [
{
"plant": "alcolea",
"devices":
sorted([{
"id": "Meter:{}".format(altMeter),
"readings": [{
"time": time,
"export_energy_wh": 1,
"import_energy_wh": 123,
"r1_VArh": 1234,
"r2_VArh": 124,
"r3_VArh": 1234,
"r4_VArh": 124,
}],
},
{
"id": "Inverter:5555",
"readings": [{
"time": time,
"power_w": 156,
"energy_wh": 154,
"intensity_cc_mA": 222,
"intensity_ca_mA": 500,
"voltage_cc_mV": 200,
"voltage_ca_mV": 100,
"uptime_h": 15,
"temperature_dc": 170,
}]
}],key=lambda d: d['id']),
},
{
"plant": "figuerea",
"devices":
sorted([{
"id": "Meter:9876",
"readings": [{
"time": time,
"export_energy_wh": 1,
"import_energy_wh": 123,
"r1_VArh": 1234,
"r2_VArh": 124,
"r3_VArh": 1234,
"r4_VArh": 124,
}],
},
{
"id": "Meter:5432",
"readings": [{
"time": time,
"export_energy_wh": 1,
"import_energy_wh": 123,
"r1_VArh": 1234,
"r2_VArh": 124,
"r3_VArh": 1234,
"r4_VArh": 124,
}],
}],key=lambda d: d['id']),
}]
return plantsData
def samplePlantsData_DifferentTimes(self, basetime, differenttime, altMeter='12345678'):
plantsData = [
{
"plant": "alcolea",
"devices":
sorted([{
"id": "Meter:{}".format(altMeter),
"readings": [{
"time": differenttime,
"export_energy_wh": 1,
"import_energy_wh": 123,
"r1_VArh": 1234,
"r2_VArh": 124,
"r3_VArh": 1234,
"r4_VArh": 124,
}],
},
{
"id": "Inverter:5555",
"readings": [{
"time": basetime,
"power_w": 156,
"energy_wh": 154,
"intensity_cc_mA": 222,
"intensity_ca_mA": 500,
"voltage_cc_mV": 200,
"voltage_ca_mV": 100,
"uptime_h": 15,
"temperature_dc": 170,
}]
}],key=lambda d: d['id']),
},
{
"plant": "figuerea",
"devices":
sorted([{
"id": "Meter:9876",
"readings": [{
"time": basetime,
"export_energy_wh": 1,
"import_energy_wh": 123,
"r1_VArh": 1234,
"r2_VArh": 124,
"r3_VArh": 1234,
"r4_VArh": 124,
}],
},
{
"id": "Meter:5432",
"readings": [{
"time": basetime,
"export_energy_wh": 1,
"import_energy_wh": 123,
"r1_VArh": 1234,
"r2_VArh": 124,
"r3_VArh": 1234,
"r4_VArh": 124,
}],
}],key=lambda d: d['id']),
}]
return plantsData
def setupPlants(self, time, alternativeMeter="12345678", plantsData = None):
delta = dt.timedelta(minutes=30)
plantsns = self.samplePlantsNS(altMeter=alternativeMeter)
importPlants(self.pony.db, plantsns)
plantsData = plantsData or self.samplePlantsData(time, delta, altMeter=alternativeMeter)
self.pony.db.Plant.insertPlantsData(plantsData)
orm.flush()
def test_getNewMetersReadings__noMeterInERP(self):
time = dt.datetime(2020, 12, 10, 15, 5, 10, 588861, tzinfo=dt.timezone.utc)
self.setupPlants(time)
mock = MagicMock(side_effect=self.mockMeterReadingsSideEffects)
with patch('plantmonitor.readings_facade.ReadingsFacade.measuresFromDate', mock):
r = ReadingsFacade(db = self.pony.db)
r.erpMeters = []
plantsData = r.getNewMetersReadings()
# orm returns unordered
for plantdata in plantsData:
plantdata["devices"].sort(key=lambda d: d['id'])
expectedPlantsData = [
{
"plant": "alcolea",
"devices": [],
},
{
"plant": "figuerea",
"devices": [],
}]
self.assertListEqual(expectedPlantsData, plantsData)
# TODO defragilize this tests via fake or mock
def test_getNewMetersReadings__meterInERP_OldReadings(self):
lastReadingTime = dt.datetime(2020, 12, 10, 15, 5, 10, 588861, tzinfo=dt.timezone.utc)
erpMeter = "88300864" # removed meter
self.setupPlants(lastReadingTime, erpMeter)
r = ReadingsFacade(self.pony.db)
plantsData = r.getNewMetersReadings()
# orm returns unordered
for plantdata in plantsData:
plantdata["devices"].sort(key=lambda d: d['id'])
expectedPlantsData = sorted([
{
"plant": "alcolea",
"devices":
[{
"id": "Meter:88300864",
"readings": [],
}],
},
{
"plant": "figuerea",
"devices": [],
}], key=lambda d: d['plant'])
self.assertListEqual(expectedPlantsData, plantsData)
def test_getNewMetersReadings__meterInERPmanyReadings(self):
lastReadingTime = dt.datetime(2019, 10, 1, 15, 5, 10, 588861, tzinfo=dt.timezone.utc)
erpMeter = "88300864"
self.setupPlants(lastReadingTime, erpMeter)
r = ReadingsFacade(self.pony.db)
upto = dt.datetime(2019, 10, 1, 17, 5, 10, 588861, tzinfo=dt.timezone.utc)
plantsData = r.getNewMetersReadings(upto)
# orm returns unordered
for plantdata in plantsData:
plantdata["devices"].sort(key=lambda d: d['id'])
expectedPlantsData = [
{
"plant": "alcolea",
"devices":
[{
"id": "Meter:{}".format(erpMeter),
"readings": [{
"time": dt.datetime(2019,10,1,16,00,00, tzinfo=dt.timezone.utc),
"export_energy_wh": 1139000,
"import_energy_wh": 0,
"r1_VArh": 0,
"r2_VArh": 1000,
"r3_VArh": 2000,
"r4_VArh": 0,
},
{
"time": dt.datetime(2019,10,1,17,00,00, tzinfo=dt.timezone.utc),
"export_energy_wh": 568000,
"import_energy_wh": 0,
"r1_VArh": 0,
"r2_VArh": 0,
"r3_VArh": 8000,
"r4_VArh": 0,
}]
}],
},
{
"plant": "figuerea",
"devices": [],
}]
self.assertListEqual(expectedPlantsData, plantsData)
def _test_transfer_ERP_readings_to_model(self):
r = ReadingsFacade(self.pony.db)
r.erpMeters = ['12345678']
# TODO mock measures or fake meters
r.transfer_ERP_readings_to_model(refreshERPmeters=False)
def test__warnNewMeters(self):
erpMeter = "88300864"
self.setupPlants(dt.datetime.now(tz=dt.timezone.utc), erpMeter)
r = ReadingsFacade(self.pony.db)
r.erpMeters = ['3141519']
newMeters = r.warnNewMeters()
expectedNewMeters = ['3141519']
self.assertListEqual(newMeters, expectedNewMeters)
def test_checkNewMeters(self):
erpMeter = "88300864"
self.setupPlants(dt.datetime.now(tz=dt.timezone.utc), erpMeter)
newMeter = "3141519"
meterNames = [erpMeter, newMeter]
r = ReadingsFacade(self.pony.db)
newMeters = r.checkNewMeters(meterNames)
self.assertListEqual(newMeters, [newMeter])
def test__refreshERPmeters__mockTelemeasure(self):
erpMeter = '88300864'
self.setupPlants(dt.datetime.now(tz=dt.timezone.utc), erpMeter)
erpMeters = ['12345678', '87654321']
# mock telemeasure_meters_name
r = ReadingsFacade(self.pony.db)
r.telemeasureMetersNames = MagicMock(return_value=['12345678', '87654321'])
r.refreshERPmeters()
newMeters = r.erpMeters
self.assertListEqual(newMeters, erpMeters)
# TODO this test is just to remind us that inputting timestamp instead of timestamptz
# in an entity causes the obscure pony.orm.core.UnrepeatableReadError: Phantom object.
# times should always have dt.datetime.now(tz=dt.timezone.utc)
def test__setupPlants__registries(self):
erpMeter = '88300864'
self.setupPlants(dt.datetime.now(), erpMeter)
with self.assertRaises(orm.core.UnrepeatableReadError):
self.pony.db.MeterRegistry.select().show()
def _test__getDeviceReadings__differentLastDates(self):
erpMeter = '88300864'
self.setupPlants(dt.datetime.now(tz=dt.timezone.utc), erpMeter)
plant = self.pony.db.Plant.get(name='alcolea')
meter = self.pony.db.Meter.get(plant=plant, name=erpMeter)
print(meter.to_dict())
self.assertIsNotNone(meter)
self.pony.db.MeterRegistry.select().show()
lastDate = meter.getLastReadingDate()
# mock telemeasure_meters_name
erp_meters = ['12345678', '87654321']
# mockreturn = [(
# measure['utc_timestamp'],
# int(measure['ae']),
# int(measure['ai']),
# int(measure['r1']),
# int(measure['r2']),
# int(measure['r3']),
# int(measure['r4']),
# )
# for measure in sorted(measures,
# key=lambda x: x['utc_timestamp'])
# ]
mock = MagicMock(return_value=['12345678', '87654321'])
with patch('plantmonitor.readings_facade.ReadingsFacade.measuresFromDate', mock):
r = ReadingsFacade(self.pony.db)
newMeters = r.getDeviceReadings(meterName=erpMeter, lastDate=lastDate, upto=None)
self.assertListEqual(newMeters, erp_meters)
def assertPlantsData(self, plants_data, meterReadings):
# print("meterreadings\n{}\n mock\n{}\n".format(plants_data, meterReadings))
plants_data_meters = sorted([device['id'].split(':')[1] for plant_data in plants_data for device in plant_data['devices']])
meters = sorted(meterReadings.keys())
self.assertListEqual(plants_data_meters, meters)
for plant_data in plants_data:
for device in plant_data['devices']:
meterName = device['id'].split(':')[1]
data = (meterName, meterReadings[meterName])
expected = (meterName, [
(
reading['time'].strftime("%Y-%m-%d %H:%M:%S"),
reading['export_energy_wh']/1000.0,
0/1000.0,0,0,0,0
) for reading in device['readings']
])
self.assertTupleEqual(data, expected)
# def assertMockReadings(self, plants_data, mock_readings):
# readings = [
# self.assertListEqual(
# mock_readings,
# [(
# reading['time'].strftime("%Y-%m-%d %H:%M:%S"),
# reading['export_energy_wh'],
# 0,0,0,0,0
# ) for reading in device['readings']]
# )
# for plant_data in plants_data for device in plant_data['devices']
# ]
def mock_meter_readings(self, time, random=True):
import random
def toUTCstr(dtaware):
return dtaware.astimezone(tz=dt.timezone.utc).replace(tzinfo=None).strftime("%Y-%m-%d %H:%M:%S")
time.replace(minute=0,second=0,microsecond=0)
delta = dt.timedelta(hours=1)
values = [0]*8 + [11, 80, 366, 716, 991, 1182, 1293, 1315, 1268, 1132, 894, 609, 323, 104, 1, 0]
i = 0
while True:
# dr = random.randint(30,70) if random else 0
dr = 0
yield (toUTCstr(time + i*delta), values[(time.hour+i)%24] + dr,0,0,0,0,0)
i = i + 1
def setupReadingsDB(self, time, readingsDict=None):
def toUTCstr(dtaware):
return dtaware.astimezone(tz=dt.timezone.utc).replace(tzinfo=None).strftime("%Y-%m-%d %H:%M:%S")
h = dt.timedelta(hours=1)
self.readingsDB = readingsDict or {
'12345678': [
(toUTCstr(time+0*h), 11,0,0,0,0,0),
(toUTCstr(time+1*h), 11,0,0,0,0,0),
(toUTCstr(time+2*h), 80,0,0,0,0,0),
(toUTCstr(time+3*h), 366,0,0,0,0,0),
],
'1234': [
(toUTCstr(time+0*h), 11,0,0,0,0,0),
(toUTCstr(time+1*h), 11,0,0,0,0,0),
(toUTCstr(time+2*h), 80,0,0,0,0,0),
(toUTCstr(time+3*h), 366,0,0,0,0,0),
],
'9876': [
(toUTCstr(time+0*h), 11,0,0,0,0,0),
(toUTCstr(time+1*h), 11,0,0,0,0,0),
(toUTCstr(time+2*h), 80,0,0,0,0,0),
(toUTCstr(time+3*h), 366,0,0,0,0,0),
],
'5432': [
(toUTCstr(time+0*h), 11,0,0,0,0,0),
(toUTCstr(time+1*h), 11,0,0,0,0,0),
(toUTCstr(time+2*h), 80,0,0,0,0,0),
(toUTCstr(time+3*h), 366,0,0,0,0,0),
]
}
def mockMeterReadingsSideEffects(self, meterName, beyond, upto):
if not beyond:
return [r for r in self.readingsDB[meterName] if todtaware(r[0]) < upto]
else:
return [r for r in self.readingsDB[meterName] if beyond < todtaware(r[0]) and todtaware(r[0]) < upto]
def test__mock_meter_readings(self):
time = dt.datetime(2019,10,1,16,00,00, tzinfo=dt.timezone.utc)
readingGenerator = self.mock_meter_readings(time, random=False)
readings = list(next(readingGenerator) for i in range(4))
self.assertTrue(readings)
def setupMockReadings(self, erpMeters, meterTimes):
readingGenerators = {
'Meter:{}'.format(name): self.mock_meter_readings(meterTime, random=False)
for name, meterTime in zip(erpMeters, meterTimes)
}
mockMeterReadings = {
id:list(next(readingGenerator) for i in range(4))
for id, readingGenerator in readingGenerators.items()
}
mockSideEffect = [
readings for id, readings in mockMeterReadings.items()
]
return mockMeterReadings, mockSideEffect
def test__getLastReadingDate(self):
erpMeter = '12345678'
time = dt.datetime.now(tz=dt.timezone.utc)
self.setupPlants(time, erpMeter)
lastDates = [meter.getLastReadingDate() for meter in self.pony.db.Meter.select()]
self.assertListEqual(lastDates, [time]*len(lastDates))
def test__transferERPreadingsToModel__manyMetersSameDates_ManyNewReadings(self):
erpMeter = '12345678'
ormtime = dt.datetime(2019,10,1,16,00,00, tzinfo=dt.timezone.utc)
erptime = dt.datetime(2020,10,1,16,00,00, tzinfo=dt.timezone.utc)
self.setupPlants(ormtime, erpMeter)
self.setupReadingsDB(erptime)
erpMeters = ['12345678', '1234', '9876', '5432']
mock = MagicMock(side_effect=self.mockMeterReadingsSideEffects)
with patch('plantmonitor.readings_facade.ReadingsFacade.measuresFromDate', mock):
r = ReadingsFacade(self.pony.db)
r.erpMeters = erpMeters
plantsData = r.getNewMetersReadings()
expectedErpMeters = ['12345678', '9876', '5432']
expectedMeterReadings = {k:v for k,v in self.readingsDB.items() if k in expectedErpMeters}
self.assertPlantsData(plantsData, expectedMeterReadings)
def test__transferERPreadingsToModel__manyMetersDifferentDates_OneWithoutNewReadings(self):
erpMeter = '12345678'
obsoleteTime = dt.datetime(2020,10,1,16,00,00, tzinfo=dt.timezone.utc)
updatedTime = dt.datetime(2021,4,1,16,00,00, tzinfo=dt.timezone.utc)
erpTime = dt.datetime(2021,3,1,16,00,00, tzinfo=dt.timezone.utc)
plantsData_different = self.samplePlantsData_DifferentTimes(basetime=obsoleteTime, differenttime=updatedTime)
self.setupPlants(alternativeMeter=erpMeter, time=None, plantsData=plantsData_different)
self.setupReadingsDB(erpTime)
erpMeters = [erpMeter, '1234', '9876', '5432']
mock = MagicMock(side_effect=self.mockMeterReadingsSideEffects)
with patch('plantmonitor.readings_facade.ReadingsFacade.measuresFromDate', mock):
r = ReadingsFacade(self.pony.db)
r.erpMeters = erpMeters
plants_data = r.getNewMetersReadings()
expectedMeters = [erpMeter, '9876', '5432']
expectedMeterReadings = {k:v for k,v in self.readingsDB.items() if k in expectedMeters}
expectedMeterReadings[erpMeter] = []
self.assertPlantsData(plants_data, expectedMeterReadings)
def test__transferERPreadingsToModel__ormMetersNotInERP(self):
erpMeter = '12345678'
ormtime = dt.datetime(2019,10,1,16,00,00, tzinfo=dt.timezone.utc)
self.setupPlants(ormtime, alternativeMeter=erpMeter)
erptime = dt.datetime(2018,10,1,16,00,00, tzinfo=dt.timezone.utc)
self.setupReadingsDB(erptime)
erpMeters = ['1234', '9876'] # missing erpMeter and 5432
mock = MagicMock(side_effect=self.mockMeterReadingsSideEffects)
with patch('plantmonitor.readings_facade.ReadingsFacade.measuresFromDate', mock):
r = ReadingsFacade(self.pony.db)
r.erpMeters = erpMeters
plants_data = r.getNewMetersReadings()
expectedPlantsData = {'9876':[]}
self.assertPlantsData(plants_data, expectedPlantsData)
def test__transferERPreadingsToModel__ormEmpty(self):
erpMeter = '12345678'
emptyPlantsData = [{
"plant": "alcolea",
"devices": []
}]
self.setupPlants(None, alternativeMeter=erpMeter, plantsData=emptyPlantsData)
erptime = dt.datetime(2020,10,1,16,00,00, tzinfo=dt.timezone.utc)
self.setupReadingsDB(erptime)
erpMeters = ['12345678', '1234', '9876', '5432']
mock = MagicMock(side_effect=self.mockMeterReadingsSideEffects)
with patch('plantmonitor.readings_facade.ReadingsFacade.measuresFromDate', mock):
r = ReadingsFacade(self.pony.db)
r.erpMeters = erpMeters
plants_data = r.getNewMetersReadings()
expectedErpMeters = ['12345678', '9876', '5432']
expectedPlantsData = {k:v for k,v in self.readingsDB.items() if k in expectedErpMeters}
self.assertPlantsData(plants_data, expectedPlantsData)
def _test__transferERPreadingsToModel__differentLastDates(self):
erpMeter = '12345678'
self.setupPlants(dt.datetime.now(tz=dt.timezone.utc), erpMeter)
erpMeters = ['12345678', '87654321']
lastDate = dt.datetime(2019,10,1,16,00,00, tzinfo=dt.timezone.utc)
mockreturn = [(
measure['utc_timestamp'],
int(measure['ae']),
int(measure['ai']),
int(measure['r1']),
int(measure['r2']),
int(measure['r3']),
int(measure['r4']),
)
for measure in sorted(measures,
key=lambda x: x['utc_timestamp'])
]
mock = MagicMock(return_value=['12345678', '87654321'])
with patch('plantmonitor.readings_facade.ReadingsFacade.measuresFromDate', mock):
r = ReadingsFacade(self.pony.db)
r.erpMeters = erpMeters
newMeters = r.transfer_ERP_readings_to_model(refreshERPmeters=False)
self.assertListEqual(newMeters, erpMeters)
class ORMSetup_Test(unittest.TestCase):
from yamlns.testutils import assertNsEqual
def setUp(self):
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
orm.rollback()
self.pony = PonyManager(envinfo.DB_CONF)
self.pony.define_all_models()
self.pony.binddb(create_tables=True)
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.create_tables()
self.maxDiff = None
orm.db_session.__enter__()
def tearDown(self):
orm.rollback()
orm.db_session.__exit__()
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.disconnect()
def fillPlantRegistries(self, plant):
timeStart = datetime.datetime(2020,
7,
28,
9,
21,
7,
881064,
tzinfo=datetime.timezone.utc)
sensorsIrr = list(
self.pony.db.SensorIrradiation.select(lambda p: p.plant == plant))
#TODO assert not empty
sensorIrr = sensorsIrr[0]
sensorTemp = list(
self.pony.db.SensorTemperatureAmbient.select(
lambda p: p.plant == plant))[0]
dt = datetime.timedelta(minutes=5)
value = 60
dv = 10
n = 3
for i in range(n):
sensorIrr.insertRegistry(
time=timeStart + i * dt,
irradiation_w_m2=value + i * dv,
temperature_dc=300 + value + i * (dv + 10),
)
sensorTemp.insertRegistry(
time=timeStart + i * dt,
temperature_dc=300 + value + i * (dv + 10),
)
sensorIrr.insertHourlySensorIrradiationMetric(
time=timeStart + i * dt,
integratedIrradiation_wh_m2=5000 + value + i * (dv + 50),
)
plantRegistries = {
'irradiation':
[(timeStart + i * dt, value + i * dv) for i in range(n)],
'temperature': [(timeStart + i * dt, 300 + value + i * (dv + 10))
for i in range(n)],
'integratedIrr':
[(timeStart + i * dt, 5000 + value + i * (dv + 50))
for i in range(n)],
}
return plantRegistries
def test_Environment(self):
#TODO will it be too late if the config is misconfigured?
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
def test_connection(self):
with orm.db_session:
self.assertEqual(self.pony.db.get_connection().status, 1)
def __test_timescaleTables(self):
with orm.db_session:
# orm.set_sql_debug(True)
tablesToTimescale = self.pony.getTablesToTimescale()
#no raises
self.pony.timescaleTables(tablesToTimescale)
numColumnsTimescaleMetadata = 11
timescaleStringColumn = 3
hypertableName = 2
cur = self.pony.db.execute(
"select * from _timescaledb_catalog.hypertable")
hypertables = cur.fetchall()
hypertablesNames = [
t[hypertableName] for t in hypertables
if len(t) == numColumnsTimescaleMetadata
and t[timescaleStringColumn] == '_timescaledb_internal'
]
tablesToTimescaleLowerCase = [
name.lower() for name in tablesToTimescale
]
self.assertListEqual(hypertablesNames, tablesToTimescaleLowerCase)
def test_meters_whenNone(self):
with orm.db_session:
alcolea = self.pony.db.Plant(name='SomEnergia_Alcolea',
codename='SOMSC01',
description='descripción de planta')
self.assertMeterListEqual("""\
data: []
""")
def test_InsertOnePlantOneMeter(self):
with orm.db_session:
alcolea = self.pony.db.Plant(name='SomEnergia_Alcolea',
codename='SOMSC01',
description='descripción de planta')
meter = self.pony.db.Meter(name='Mary', plant=alcolea)
self.assertMeterListEqual("""\
data:
- name: Mary
plant_name: SomEnergia_Alcolea
plant_code: SOMSC01
plant_description: descripción de planta
""")
def assertMeterListEqual(self, expected):
self.assertNsEqual(
ns(data=[
ns(
name=name,
plant_name=plantname,
plant_code=plantcode,
plant_description=description,
) for name, plantname, plantcode, description in orm.select((
meter.name,
meter.plant.name,
meter.plant.codename,
meter.plant.description,
) for meter in self.pony.db.Meter)
]), expected)
def assertMeterRegistryEqual(self, plantCode, meterName, expected):
plant = self.pony.db.Plant.get(codename=plantCode)
meter = self.pony.db.Meter.get(plant=plant, name=meterName)
registry = [
ns(
# yamlns reads datetimes just as date, compare the string
time=str(line.time),
export_energy_wh=line.export_energy_wh,
import_energy_wh=line.import_energy_wh,
r1_VArh=line.r1_VArh,
r2_VArh=line.r2_VArh,
r3_VArh=line.r3_VArh,
r4_VArh=line.r4_VArh,
) for line in orm.select(
l for l in self.pony.db.MeterRegistry if l.meter == meter)
]
self.assertNsEqual(ns(registry=registry), expected)
def test_registry_whenEmpty(self):
with orm.db_session:
alcolea = self.pony.db.Plant(name='SomEnergia_Alcolea',
codename='SOMSC01',
description='descripción de planta')
meter = self.pony.db.Meter(name='Mary', plant=alcolea)
self.assertMeterRegistryEqual(
'SOMSC01', 'Mary', """\
registry: []
""")
def test_registry_singleEntry(self):
with orm.db_session:
alcolea = self.pony.db.Plant(name='SomEnergia_Alcolea',
codename='SOMSC01',
description='descripción de planta')
meter = self.pony.db.Meter(name='Mary', plant=alcolea)
meter.insertRegistry(
time=datetime.datetime(2020,
10,
20,
0,
0,
0,
tzinfo=datetime.timezone.utc),
export_energy_wh=10,
import_energy_wh=77,
r1_VArh=0,
r2_VArh=0,
r3_VArh=0,
r4_VArh=0,
)
self.assertMeterRegistryEqual(
'SOMSC01', 'Mary', """\
registry:
- time: '2020-10-20 00:00:00+00:00'
export_energy_wh: 10
import_energy_wh: 77
r1_VArh: 0
r2_VArh: 0
r3_VArh: 0
r4_VArh: 0
""")
def test_InverterRegistry_singleEntry(self):
with orm.db_session:
alcolea = self.pony.db.Plant(name='SomEnergia_Alcolea',
codename='SOMSC01',
description='descripción de planta')
inverter = self.pony.db.Inverter(name='Mary', plant=alcolea)
inverterDataDict = {
'time':
datetime.datetime(2020,
10,
20,
0,
0,
0,
tzinfo=datetime.timezone.utc),
'power_w':
10,
'energy_wh':
10,
'intensity_cc_mA':
10,
'intensity_ca_mA':
10,
'voltage_cc_mV':
10,
'voltage_ca_mV':
10,
'uptime_h':
10,
'temperature_dc':
10,
}
inverter.insertRegistry(**inverterDataDict)
expectedRegistry = inverterDataDict
expectedRegistry['inverter'] = 1
oneRegistry = orm.select(
r for r in self.pony.db.InverterRegistry).first()
oneRegistryList = oneRegistry.to_dict()
self.assertDictEqual(expectedRegistry, oneRegistryList)
def test_InvertersRegistries_multipleRegistries(self):
pass
def test_InsertOnePlantOneMeterOneRegistry(self):
with orm.db_session:
alcolea = self.pony.db.Plant(name='SomEnergia_Alcolea',
codename='SOMSC01',
description='descripción de planta')
meter = self.pony.db.Meter(name='Mary', plant=alcolea)
meterRegistry = self.pony.db.MeterRegistry(
meter=meter,
time=datetime.datetime.now(datetime.timezone.utc),
export_energy_wh=10,
import_energy_wh=77,
r1_VArh=0,
r2_VArh=0,
r3_VArh=0,
r4_VArh=0,
)
def test_ReadOnePlantOneMeterOneRegistry(self):
with orm.db_session:
alcolea = self.pony.db.Plant(name='SomEnergia_Alcolea',
codename='SOMSC01',
description='A fotovoltaic plant')
meter = self.pony.db.Meter(name='Mary', plant=alcolea)
meterRegistry = self.pony.db.MeterRegistry(
meter=meter,
time=datetime.datetime.now(datetime.timezone.utc),
export_energy_wh=10,
import_energy_wh=77,
r1_VArh=0,
r2_VArh=0,
r3_VArh=0,
r4_VArh=0,
)
alcolea_read = self.pony.db.Plant[1]
self.assertEqual(alcolea_read, alcolea)
self.assertEqual(alcolea_read.name, alcolea.name)
def test_InsertTwoPlantTwoMeterOneRegistry(self):
with orm.db_session:
alcolea = self.pony.db.Plant(name='SomEnergia_Alcolea',
codename='SOMSC01',
description='descripción de planta')
alcometer = self.pony.db.Meter(name='meter1', plant=alcolea)
fonti = self.pony.db.Plant(name='SomEnergia_Fontisolar',
codename='SOMSC02',
description='descripción de planta')
fontimeter = self.pony.db.Meter(name='meter1', plant=fonti)
def test_InsertOnePlantOneSensor(self):
with orm.db_session:
alcolea = self.pony.db.Plant(name='SomEnergia_Alcolea',
codename='SOMSC01',
description='descripción de planta')
sensor = self.pony.db.SensorTemperatureAmbient(name='TempAlcolea',
plant=alcolea)
sensor_read = self.pony.db.Sensor[1]
self.assertEqual(sensor_read, sensor)
def test_InsertOnePlantOneSensorOneRegistry(self):
with orm.db_session:
alcolea = self.pony.db.Plant(name='SomEnergia_Alcolea',
codename='SOMSC01',
description='descripción de planta')
sensor = self.pony.db.SensorIrradiation(name='IrradAlcolea',
plant=alcolea)
sensorRegistry = sensor.insertRegistry(time=datetime.datetime.now(
datetime.timezone.utc),
irradiation_w_m2=68,
temperature_dc=250)
sensor_registry_read = list(
self.pony.db.SensorIrradiationRegistry.select())[0]
self.assertEqual(sensor_registry_read, sensorRegistry)
def test_InsertOneForecast(self):
with orm.db_session:
alcolea = self.pony.db.Plant(name='SomEnergia_Alcolea',
codename='SOMSC01',
description='descripción de planta')
forecastVariable = self.pony.db.ForecastVariable(name='prod')
forecastPredictor = self.pony.db.ForecastPredictor(
name='aggregated')
forecastMetadata = self.pony.db.ForecastMetadata(
errorcode='',
plant=alcolea,
variable=forecastVariable,
predictor=forecastPredictor,
forecastdate=datetime.datetime.now(datetime.timezone.utc),
granularity=60,
)
forecast = forecastMetadata.insertForecast(
time=datetime.datetime.now(datetime.timezone.utc),
percentil10=10,
percentil50=50,
percentil90=90,
)
forecast_read = list(self.pony.db.Forecast.select())[0]
self.assertEqual(forecast_read, forecast)
def test_GetRegistriesFromOneSensorInDateRange(self):
with orm.db_session:
alcolea = self.pony.db.Plant(name='SomEnergia_Alcolea',
codename='SOMSC01',
description='descripción de planta')
sensor = self.pony.db.SensorIrradiation(name='IrradAlcolea',
plant=alcolea)
timetz = datetime.datetime(2020,
7,
28,
9,
21,
7,
881064,
tzinfo=datetime.timezone.utc)
dt = datetime.timedelta(minutes=5)
value = 60
dv = 10
for i in range(5):
self.pony.db.SensorIrradiationRegistry(
sensor=sensor,
time=timetz + i * dt,
irradiation_w_m2=value + i * dv,
)
oneday = datetime.datetime.strptime('2020-07-28', '%Y-%m-%d')
query = orm.select(r.time
for r in self.pony.db.SensorIrradiationRegistry
if oneday.date() == r.time.date())
expected = [timetz + i * dt for i in range(5)]
self.assertListEqual(list(query), expected)
def test_fixtureCreation(self):
with orm.db_session:
plantsNS = ns.loads("""\
plants:
- plant:
name: alcolea
codename: SCSOM04
description: la bonica planta
meters:
- meter:
name: '1234578'
inverters:
- inverter:
name: '5555'
- inverter:
name: '6666'
irradiationSensors:
- irradiationSensor:
name: alberto
temperatureModuleSensors:
- temperatureModuleSensor:
name: pol
temperatureAmbientSensors:
- temperatureAmbientSensor:
name: joana
""")
importPlants(self.pony.db, plantsNS)
#TODO test the whole fixture, not just the plant data
expectedPlantns = exportPlants(self.pony.db)
self.assertNsEqual(expectedPlantns, plantsNS)
def test_fillPlantRegistries(self):
with orm.db_session:
alcolea = self.pony.db.Plant(name='SomEnergia_Alcolea',
codename='SOMSC01',
description='descripción de planta')
alcolea.createPlantFixture()
plantRegistries = self.fillPlantRegistries(alcolea)
n = len(plantRegistries['irradiation'])
self.assertEqual(n, 3)
self.assertListEqual(
list(plantRegistries.keys()),
['irradiation', 'temperature', 'integratedIrr'])
def test_GetRegistriesFromManySensorsInDateRange(self):
with orm.db_session:
alcolea = self.pony.db.Plant(name='SomEnergia_Alcolea',
codename='SOMSC01',
description='descripción de planta')
alcolea.createPlantFixture()
plantRegistries = self.fillPlantRegistries(alcolea)
n = len(plantRegistries['irradiation'])
# TODO: First select returns empty results unless we commit
orm.commit()
expectedPlantRegistries = [(
plantRegistries['irradiation'][i][0],
plantRegistries['irradiation'][i][1],
plantRegistries['temperature'][i][1],
plantRegistries['integratedIrr'][i][1],
) for i in range(n)]
q1 = orm.select(r for r in self.pony.db.SensorIrradiationRegistry
if r.sensor.plant == alcolea)
q2 = orm.select(
r for r in self.pony.db.SensorTemperatureAmbientRegistry
if r.sensor.plant == alcolea)
q3 = orm.select(
r for r in self.pony.db.HourlySensorIrradiationRegistry
if r.sensor.plant == alcolea)
qresult = orm.select(
(r1_w.time, r1_w.irradiation_w_m2, r2_w.temperature_dc,
r3_w.integratedIrradiation_wh_m2, r1_w.sensor, r2_w.sensor,
r3_w.sensor) for r1_w in q1 for r2_w in q2 for r3_w in q3
if r1_w.time == r2_w.time and r2_w.time == r3_w.time)
plantRegistries = [(t.astimezone(datetime.timezone.utc), irr, temp,
integ)
for t, irr, temp, integ, s1, s2, s3 in qresult]
self.assertListEqual(plantRegistries, expectedPlantRegistries)
class ExpectedPower_Test(TestCase):
from b2btest.b2btest import assertB2BEqual
@classmethod
def setUpClass(cls):
''
def setUp(self):
self.maxDiff = None
self.b2bdatapath = 'b2bdata'
from conf import envinfo
self.assertEqual(envinfo.SETTINGS_MODULE, 'conf.settings.testing')
orm.rollback()
self.pony = PonyManager(envinfo.DB_CONF)
self.pony.define_all_models()
self.pony.binddb(create_tables=True)
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.create_tables()
orm.db_session.__enter__()
def tearDown(self):
orm.rollback()
orm.db_session.__exit__()
self.pony.db.drop_all_tables(with_all_data=True)
self.pony.db.disconnect()
def samplePlantNS(self): # Copied from test_models.py:samplePlantNS
alcoleaPlantNS = ns.loads("""\
name: myplant
codename: PLANTCODE
description: Plant Description
meters:
- meter:
name: '1234578'
irradiationSensors:
- irradiationSensor:
name: irradiationSensor1
""")
return alcoleaPlantNS
def setupPlant(self):
plantDefinition = self.samplePlantNS()
plant = self.pony.db.Plant(
name=plantDefinition.name,
codename=plantDefinition.codename,
)
plant.importPlant(plantDefinition)
plant.flush()
self.plant = plant.id
self.sensor = self.pony.db.SensorIrradiation.select().first().id
def importData(self, sensor, filename, outputColumn):
tsvContent = readTimedDataTsv(filename, [
'Temperatura modul',
'Irradiación (W/m2)',
outputColumn or
'Potencia parque calculada con temperatura kW con degradación placas',
])
for time, temperature_c, irradiation_w_m2, outputValue in tsvContent:
self.pony.db.SensorIrradiationRegistry(
sensor=self.sensor,
time=datetime.datetime.fromisoformat(time),
irradiation_w_m2=int(round(irradiation_w_m2)),
temperature_dc=int(round(temperature_c * 10)),
)
orm.flush()
return [(time, self.plant, self.sensor, outputValue) for time,
temperature_c, irradiation_w_m2, outputValue in tsvContent]
parametersAlcolea = dict(
nominalPowerMWp=2.16,
efficiency=15.5,
nModules=8640, # plant parameter
Imp=8.27, # A, module model param
Vmp=30.2, # V, module model param
temperatureCoefficientI=0.088, # %/ºC, module model param
temperatureCoefficientV=-0.352, # %/ºC, module model param
temperatureCoefficientPmax=-0.442, # %/ºC, module model param
irradiationSTC=1000.0, # W/m2, module model param
temperatureSTC=25, # ºC, module model param
Isc=8.75, # A, module model param
Voc=37.8, # V, module model param
#degradation=97.0, # %, module model param
degradation=95 *
.9, # %, module model param # According the excel formula
correctionFactorPercent=90., # %, plant parameter
)
parametersFlorida = dict(
nominalPowerMWp=2.16, # TODO use Florida values
efficiency=15.5, # TODO use Florida values
nModules=4878, # plant parameter
Imp=9.07, # A, module model param
Vmp=37.5, # V, module model param
temperatureCoefficientI=0.05, # %/ºC, module model param
temperatureCoefficientV=-0.31, # %/ºC, module model param
temperatureCoefficientPmax=-0.442, # %/ºC, module model param
irradiationSTC=1000.0, # W/m2, module model param
temperatureSTC=25, # ºC, module model param
degradation=97.5, # %, module model param
Isc=9.5, # A, module model param
Voc=46.1, # V, module model param
correctionFactorPercent=90., # %, plant parameter
)
def setPlantParameters(self, **data):
data = ns(data)
plant = self.pony.db.PlantModuleParameters(
plant=self.plant,
nominal_power_wp=int(data.nominalPowerMWp * 1000000),
efficency_cpercent=int(data.efficiency * 100),
n_modules=data.nModules,
max_power_current_ma=int(data.Imp * 1000),
max_power_voltage_mv=int(data.Vmp * 1000),
current_temperature_coefficient_mpercent_c=int(
data.temperatureCoefficientI * 1000),
voltage_temperature_coefficient_mpercent_c=int(
data.temperatureCoefficientV * 1000),
max_power_temperature_coefficient_mpercent_c=int(
data.temperatureCoefficientPmax * 1000),
standard_conditions_irradiation_w_m2=int(data.irradiationSTC),
standard_conditions_temperature_dc=int(data.temperatureSTC * 10),
degradation_cpercent=int(data.degradation * 100),
opencircuit_voltage_mv=int(data.Voc * 1000),
shortcircuit_current_ma=int(data.Isc * 1000),
expected_power_correction_factor_cpercent=int(
data.get('correctionFactorPercent', 100) * 100),
)
plant.flush()
def assertOutputB2B(self, result):
def x2utcisoformat(date):
if not hasattr(date, 'isoformat'):
date = datetime.datetime.fromisoformat(date)
date = date.astimezone(datetime.timezone.utc)
return date.isoformat()
result = "\n".join((
"{}\n{:.6f}".format(x2utcisoformat(time), value)
for time, plant, sensor, value, temperature, irradiance in result))
self.assertB2BEqual(result)
# TODO fancy_replace calls make B2B too slow when changing timezone
# see https://bugs.python.org/issue6931 I guess
# def assertOutputB2BNoPlant(self, result):
# tz = pytz.timezone("Europe/Madrid")
# result = "\n".join((
# "{},{},{:.9f}".format(t[0].astimezone(tz).isoformat(),sensor.name,t[1])
# for sensor, l in result.items() for t in l
# ))
# self.assertB2BEqual(result)
def assertOutputB2BNoPlant(self, result):
result = "\n".join(("{},{},{:.9f}".format(t[0].isoformat(),
sensor.name, t[1])
for sensor, l in result.items() for t in l))
self.assertB2BEqual(result)
def test_expectedPower_Florida_2020_09(self):
self.setupPlant()
self.setPlantParameters(**dict(
self.parametersFlorida,
correctionFactorPercent=100,
))
expected = self.importData(
self.sensor,
'b2bdata/expectedPower-2020-09-Florida.csv',
'Potencia parque calculada con temperatura kW con degradación placas',
)
query = Path('queries/view_expected_power.sql').read_text(
encoding='utf8')
result = self.pony.db.select(query)
self.assertOutputB2B(result) # first run with expected instead result
def test_expectedPower_Florida_2020_09_withCorrection(self):
self.setupPlant()
self.setPlantParameters(**dict(
self.parametersFlorida,
correctionFactorPercent=90,
))
expected = self.importData(
self.sensor,
'b2bdata/expectedPower-2020-09-Florida.csv',
'Potencia parque calculada con temperatura kW con degradación placas',
)
query = Path('queries/view_expected_power.sql').read_text(
encoding='utf8')
result = self.pony.db.select(query)
self.assertOutputB2B(result) # first run with expected instead result
def test_expectedPower_Alcolea_2021_04_noCorrections(self):
self.setupPlant()
self.setPlantParameters(**dict(
self.parametersAlcolea,
correctionFactorPercent=100,
))
expected = self.importData(
self.sensor,
'b2bdata/expectedPower-2021-04-Alcolea.csv',
'Potencia parque calculada con temperatura kW con degradación placas',
)
query = Path('queries/view_expected_power.sql').read_text(
encoding='utf8')
result = self.pony.db.select(query)
self.assertOutputB2B(result) # first run with expected instead result
def test_integrateExpectedPower_Florida_2020_09(self):
self.setupPlant()
self.setPlantParameters(**dict(
self.parametersFlorida,
correctionFactorPercent=100,
))
expected = None
# expected = self.importData(self.sensor,
# 'b2bdata/expectedEnergy-2020-09-Florida.csv',
# 'Potencia parque calculada con temperatura kW con degradación placas',
# )
self.importData(
self.sensor,
'b2bdata/expectedPower-2020-09-Florida.csv',
'Potencia parque calculada con temperatura kW con degradación placas',
)
time = datetime.datetime(2020,
9,
1,
2,
5,
10,
588861,
tzinfo=datetime.timezone.utc)
fromDate = time.replace(hour=14, minute=0, second=0, microsecond=0)
toDate = fromDate + datetime.timedelta(days=30)
query = Path('queries/view_expected_power.sql').read_text(
encoding='utf8')
expectedPowerQuery = self.pony.db.select(query)
# end of setup
result = integrateExpectedPower(self.pony.db, fromDate, toDate)
self.assertOutputB2BNoPlant(result)
# TODO add expectedEnergy as a view instead of a redash query
def _test_expectedEnergy_Alcolea_2021_04(self):
self.setupPlant()
self.setPlantParameters(**dict(
self.parametersFlorida,
correctionFactorPercent=90,
))
self.importData(
self.sensor,
'b2bdata/expectedPower-2021-04-Alcolea.csv',
'Potencia parque calculada con temperatura kW con degradación placas',
)
# TODO how to add a view on a test?
query = Path('queries/view_expected_power.sql').read_text(
encoding='utf8')
result = database.select(query)
query = Path('queries/view_expected_energy.sql').read_text(
encoding='utf8')
result = database.select(query)
self.assertOutputB2B(result) # first run with expected instead result
#TODO find a way to not use Union because of time
class Device(BaseModel):
id: str
readings: List[Dict[str, Union[int, datetime.datetime, None]]]
class PlantReading(BaseModel):
plant: str
version: str
time: datetime.datetime
devices: List[Device]
api = FastAPI()
from conf import envinfo
pony_manager = PonyManager(envinfo.DB_CONF)
pony_manager.define_all_models()
pony_manager.binddb()
storage = PonyMetricStorage(pony_manager.db)
@api.get('/version')
def api_version():
return dict(
version='1.0',
)
@api.put('/plant/{plant_id}/readings')
async def api_putPlantReadings(plant_id: str, plant_reading: PlantReading):
logger.info("Putting plant data into plant {} : {}".format(plant_id, plant_reading))
with orm.db_session: