def _sync_orm(self):
if self._sync_running:
logger.info('ORM sync (PulseCounter): Already running')
return False
self._sync_running = True
start = time.time()
logger.info('ORM sync (PulseCounter)')
try:
for pulse_counter_dto in self._master_controller.load_pulse_counters():
pulse_counter_id = pulse_counter_dto.id
pulse_counter = PulseCounter.get_or_none(number=pulse_counter_id)
if pulse_counter is None:
pulse_counter = PulseCounter(number=pulse_counter_id,
name='PulseCounter {0}'.format(pulse_counter_id),
source='master',
persistent=False)
pulse_counter.save()
duration = time.time() - start
logger.info('ORM sync (PulseCounter): completed after {0:.1f}s'.format(duration))
except CommunicationTimedOutException as ex:
logger.error('ORM sync (PulseCounter): Failed: {0}'.format(ex))
except Exception:
logger.exception('ORM sync (PulseCounter): Failed')
finally:
self._sync_running = False
return True
def dto_to_orm(
pulse_counter_dto): # type: (PulseCounterDTO) -> PulseCounter
pulse_counter = PulseCounter.get_or_none(number=pulse_counter_dto.id)
if pulse_counter is None:
pulse_counter = PulseCounter(number=pulse_counter_dto.id,
name='',
source='gateway',
persistent=False)
if 'name' in pulse_counter_dto.loaded_fields:
pulse_counter.name = pulse_counter_dto.name
if 'persistent' in pulse_counter_dto.loaded_fields:
pulse_counter.persistent = pulse_counter_dto.persistent
return pulse_counter
def set_amount_of_pulse_counters(self, amount): # type: (int) -> int
_ = self
# This does not make a lot of sense in an ORM driven implementation, but is for legacy purposes.
# The legacy implementation heavily depends on the number (legacy id) and the fact that there should be no
# gaps between them. If there are gaps, legacy upstream code will most likely break.
# TODO: Fix legacy implementation once the upstream can manage this better
amount_of_master_pulse_counters = PulseCounter.select().where(PulseCounter.source == 'master').count()
if amount < amount_of_master_pulse_counters:
raise ValueError('Amount should be >= {0}'.format(amount_of_master_pulse_counters))
# Assume amount is 27:
# - This means n master driven PulseCounters
# - This means 27-n gateway driven PulseCounters
# The `number` field will contain 0-(n-1) (zero-based counting), this means that any
# number above or equal to the amount can be removed (>= n)
PulseCounter.delete().where(PulseCounter.number >= amount).execute()
for number in range(amount_of_master_pulse_counters, amount):
pulse_counter = PulseCounter.get_or_none(number=number)
if pulse_counter is None:
pulse_counter = PulseCounter(number=number,
name='PulseCounter {0}'.format(number),
source='gateway',
persistent=False)
pulse_counter.save()
return amount
def test_pulse_counter_up_down(self):
""" Test adding and removing pulse counters. """
SetUpTestInjections(master_controller=Mock(),
maintenance_controller=Mock())
controller = PulseCounterController()
for i in range(24):
PulseCounter(number=i,
name='PulseCounter {0}'.format(i),
source='master',
persistent=False).save()
# Only master pulse counters
controller.set_amount_of_pulse_counters(24)
self.assertEqual(24, controller.get_amount_of_pulse_counters())
# Add virtual pulse counters
controller.set_amount_of_pulse_counters(28)
self.assertEqual(28, controller.get_amount_of_pulse_counters())
# Add virtual pulse counter
controller.set_amount_of_pulse_counters(29)
self.assertEqual(29, controller.get_amount_of_pulse_counters())
# Remove virtual pulse counter
controller.set_amount_of_pulse_counters(28)
self.assertEqual(28, controller.get_amount_of_pulse_counters())
# Set virtual pulse counters to 0
controller.set_amount_of_pulse_counters(24)
self.assertEqual(24, controller.get_amount_of_pulse_counters())
# Set the number of pulse counters to low
with self.assertRaises(ValueError):
controller.set_amount_of_pulse_counters(23)
def load_pulse_counter(self, pulse_counter_id): # type: (int) -> PulseCounterDTO
pulse_counter = PulseCounter.select(PulseCounter, Room) \
.join_from(PulseCounter, Room, join_type=JOIN.LEFT_OUTER) \
.where(PulseCounter.number == pulse_counter_id) \
.get() # type: PulseCounter # TODO: Load dict
if pulse_counter.source == 'master':
pulse_counter_dto = self._master_controller.load_pulse_counter(pulse_counter_id=pulse_counter_id)
pulse_counter_dto.room = pulse_counter.room.number if pulse_counter.room is not None else None
else:
pulse_counter_dto = PulseCounterMapper.orm_to_dto(pulse_counter)
return pulse_counter_dto
def load_pulse_counters(self): # type: () -> List[PulseCounterDTO]
pulse_counter_dtos = []
for pulse_counter in list(PulseCounter.select(PulseCounter, Room)
.join_from(PulseCounter, Room, join_type=JOIN.LEFT_OUTER)): # TODO: Load dicts
if pulse_counter.source == 'master':
pulse_counter_dto = self._master_controller.load_pulse_counter(pulse_counter_id=pulse_counter.number)
pulse_counter_dto.room = pulse_counter.room.number if pulse_counter.room is not None else None
pulse_counter_dto.name = pulse_counter.name # Use longer ORM name
else:
pulse_counter_dto = PulseCounterMapper.orm_to_dto(pulse_counter)
pulse_counter_dtos.append(pulse_counter_dto)
return pulse_counter_dtos
def test_config(self):
master_pulse_counters = {}
def _save_pulse_counters(data):
for dto in data:
master_pulse_counters[dto.id] = dto
master_controller_mock = Mock()
master_controller_mock.load_pulse_counter = lambda pulse_counter_id: master_pulse_counters[pulse_counter_id]
master_controller_mock.save_pulse_counters = _save_pulse_counters
SetUpTestInjections(master_controller=master_controller_mock,
maintenance_controller=Mock())
controller = PulseCounterController()
# Simulate master contents & initial sync
for i in range(24):
master_pulse_counters[i] = PulseCounterDTO(id=i, name=u'PulseCounter {0}'.format(i), persistent=False)
PulseCounter(number=i, name='PulseCounter {0}'.format(i), source='master', persistent=False).save()
controller.set_amount_of_pulse_counters(26)
controller.save_pulse_counters([
PulseCounterDTO(id=1, name='Water', input_id=10, room=1),
PulseCounterDTO(id=4, name='Gas', input_id=11, room=2),
PulseCounterDTO(id=25, name='Electricity', input_id=None, room=3, persistent=True)
])
received_dtos = controller.load_pulse_counters()
expected_dtos = [PulseCounterDTO(id=i, name=u'PulseCounter {0}'.format(i))
for i in range(26)]
expected_dtos[1] = PulseCounterDTO(id=1, name='Water', input_id=10, room=1)
expected_dtos[4] = PulseCounterDTO(id=4, name='Gas', input_id=11, room=2)
expected_dtos[25] = PulseCounterDTO(id=25, name='Electricity', input_id=None, room=3, persistent=True)
self.assertEqual(expected_dtos, received_dtos)
# Try to set input on virtual pulse counter
controller.save_pulse_counters([PulseCounterDTO(id=25, name='Electricity', input_id=22, room=3)])
self.assertEqual(PulseCounterDTO(id=25, name='Electricity', room=3, persistent=True), controller.load_pulse_counter(25))
# Get configuration for existing master pulse counter
self.assertEqual(PulseCounterDTO(id=1, name='Water', input_id=10, room=1, persistent=False), controller.load_pulse_counter(1))
# Get configuration for unexisting pulse counter
with self.assertRaises(DoesNotExist):
controller.save_pulse_counters([PulseCounterDTO(id=26, name='Electricity')])
# Set configuration for unexisting pulse counter
with self.assertRaises(DoesNotExist):
controller.load_pulse_counter(26)
def save_pulse_counters(self, pulse_counters): # type: (List[PulseCounterDTO]) -> None
pulse_counters_to_save = []
for pulse_counter_dto in pulse_counters:
pulse_counter = PulseCounter.get_or_none(number=pulse_counter_dto.id) # type: PulseCounter
if pulse_counter is None:
raise DoesNotExist('A PulseCounter with id {0} could not be found'.format(pulse_counter_dto.id))
if pulse_counter.source == 'master':
# Only master pulse counters will be passed to the MasterController batch save
pulse_counters_to_save.append(pulse_counter_dto)
if 'name' in pulse_counter_dto.loaded_fields:
pulse_counter.name = pulse_counter_dto.name
elif pulse_counter.source == 'gateway':
pulse_counter = PulseCounterMapper.dto_to_orm(pulse_counter_dto)
else:
logger.warning('Trying to save a PulseCounter with unknown source {0}'.format(pulse_counter.source))
continue
if 'room' in pulse_counter_dto.loaded_fields:
if pulse_counter_dto.room is None:
pulse_counter.room = None
elif 0 <= pulse_counter_dto.room <= 100:
pulse_counter.room, _ = Room.get_or_create(number=pulse_counter_dto.room)
pulse_counter.save()
self._master_controller.save_pulse_counters(pulse_counters_to_save)
def test_pulse_counter_status(self):
data = {'pv0': 0, 'pv1': 1, 'pv2': 2, 'pv3': 3, 'pv4': 4, 'pv5': 5, 'pv6': 6, 'pv7': 7,
'pv8': 8, 'pv9': 9, 'pv10': 10, 'pv11': 11, 'pv12': 12, 'pv13': 13, 'pv14': 14,
'pv15': 15, 'pv16': 16, 'pv17': 17, 'pv18': 18, 'pv19': 19, 'pv20': 20, 'pv21': 21,
'pv22': 22, 'pv23': 23}
def _do_command(api):
return data
master_communicator = Mock()
master_communicator.do_command = _do_command
SetUpTestInjections(master_communicator=master_communicator,
configuration_controller=Mock(),
eeprom_controller=Mock())
SetUpTestInjections(master_controller=MasterClassicController(),
maintenance_controller=Mock())
for i in range(24):
PulseCounter(number=i, name='PulseCounter {0}'.format(i), source='master', persistent=False).save()
controller = PulseCounterController()
controller.set_amount_of_pulse_counters(26)
controller.set_value(24, 123)
controller.set_value(25, 456)
values_dict = controller.get_values()
values = [values_dict[i] for i in sorted(values_dict.keys())]
self.assertEqual(list(range(0, 24)) + [123, 456], values)
# Set pulse counter for unexisting pulse counter
with self.assertRaises(DoesNotExist):
controller.set_value(26, 789)
# Set pulse counter for physical pulse counter
with self.assertRaises(ValueError):
controller.set_value(23, 789)
def get_persistence(self): # type: () -> Dict[int, bool]
_ = self
return {pulse_counter.number: pulse_counter.persistent for pulse_counter
in PulseCounter.select()}
def get_values(self): # type: () -> Dict[int, Optional[int]]
pulse_counter_values = {} # type: Dict[int, Optional[int]]
pulse_counter_values.update(self._master_controller.get_pulse_counter_values())
for pulse_counter in PulseCounter.select().where(PulseCounter.source == 'gateway'):
pulse_counter_values[pulse_counter.number] = self._counts.get(pulse_counter.number)
return pulse_counter_values
def set_value(self, pulse_counter_id, value): # type: (int, int) -> int
pulse_counter = PulseCounter.get(number=pulse_counter_id)
if pulse_counter.source == 'master':
raise ValueError('Cannot set pulse counter value for a Master controlled PulseCounter')
self._counts[pulse_counter_id] = value
return value
def get_amount_of_pulse_counters(self): # type: () -> int
_ = self
return PulseCounter.select(fn.Max(PulseCounter.number)).scalar() + 1
def _migrate(cls,
master_controller=INJECTED
): # type: (MasterClassicController) -> None
# Core(+) platforms never had non-ORM rooms
if Platform.get_platform() in Platform.CoreTypes:
return
# Import legacy code
@Inject
def _load_eeprom_extension(eeprom_extension=INJECTED):
# type: (EepromExtension) -> EepromExtension
return eeprom_extension
eext_controller = _load_eeprom_extension()
from master.classic.eeprom_models import (OutputConfiguration,
InputConfiguration,
SensorConfiguration,
ShutterConfiguration,
ShutterGroupConfiguration,
PulseCounterConfiguration)
rooms = {} # type: Dict[int, Room]
floors = {} # type: Dict[int, Floor]
# Rooms and floors
logger.info('* Rooms & floors')
for room_id in range(100):
try:
RoomsMigrator._get_or_create_room(eext_controller,
room_id,
rooms,
floors,
skip_empty=True)
except Exception:
logger.exception('Could not migrate single RoomConfiguration')
# Main objects
for eeprom_model, orm_model, filter_ in [
(OutputConfiguration, Output, lambda o: True),
(InputConfiguration, Input, lambda i: i.module_type in ['i', 'I']),
(SensorConfiguration, Sensor, lambda s: True),
(ShutterConfiguration, Shutter, lambda s: True),
(ShutterGroupConfiguration, ShutterGroup, lambda s: True)
]:
logger.info('* {0}s'.format(eeprom_model.__name__))
try:
for classic_orm in master_controller._eeprom_controller.read_all(
eeprom_model):
try:
object_id = classic_orm.id
if object_id is None:
continue
if not filter_(classic_orm):
RoomsMigrator._delete_eext_fields(
eext_controller, eeprom_model.__name__,
object_id, ['room'])
continue
try:
room_id = int(
RoomsMigrator._read_eext_fields(
eext_controller, eeprom_model.__name__,
object_id, ['room']).get('room', 255))
except ValueError:
room_id = 255
object_orm, _ = orm_model.get_or_create(
number=object_id) # type: ignore
if room_id == 255:
object_orm.room = None
else:
object_orm.room = RoomsMigrator._get_or_create_room(
eext_controller, room_id, rooms, floors)
object_orm.save()
RoomsMigrator._delete_eext_fields(
eext_controller, eeprom_model.__name__, object_id,
['room'])
except Exception:
logger.exception('Could not migrate single {0}'.format(
eeprom_model.__name__))
except Exception:
logger.exception('Could not migrate {0}s'.format(
eeprom_model.__name__))
# PulseCounters
pulse_counter = None # type: Optional[PulseCounter]
# - Master
try:
logger.info('* PulseCounters (master)')
for pulse_counter_classic_orm in master_controller._eeprom_controller.read_all(
PulseCounterConfiguration):
try:
pulse_counter_id = pulse_counter_classic_orm.id
try:
room_id = int(
RoomsMigrator._read_eext_fields(
eext_controller, 'PulseCounterConfiguration',
pulse_counter_id, ['room']).get('room', 255))
except ValueError:
room_id = 255
pulse_counter = PulseCounter.get_or_none(
number=pulse_counter_id)
if pulse_counter is None:
pulse_counter = PulseCounter(
number=pulse_counter_id,
name=pulse_counter_classic_orm.name,
persistent=False,
source=u'master')
else:
pulse_counter.name = pulse_counter_classic_orm.name
pulse_counter.persistent = False
pulse_counter.source = u'master'
if room_id == 255:
pulse_counter.room = None
else:
pulse_counter.room = RoomsMigrator._get_or_create_room(
eext_controller, room_id, rooms, floors)
pulse_counter.save()
RoomsMigrator._delete_eext_fields(
eext_controller, 'PulseCounterConfiguration',
pulse_counter_id, ['room'])
except Exception:
logger.exception(
'Could not migrate classic master PulseCounter')
except Exception:
logger.exception('Could not migrate classic master PulseCounters')
# - Old SQLite3
old_sqlite_db = constants.get_pulse_counter_database_file()
if os.path.exists(old_sqlite_db):
try:
logger.info('* PulseCounters (gateway)')
import sqlite3
connection = sqlite3.connect(
old_sqlite_db,
detect_types=sqlite3.PARSE_DECLTYPES,
check_same_thread=False,
isolation_level=None)
cursor = connection.cursor()
for row in cursor.execute(
'SELECT id, name, room, persistent FROM pulse_counters ORDER BY id ASC;'
):
try:
pulse_counter_id = int(row[0])
room_id = int(row[2])
pulse_counter = PulseCounter.get_or_none(
number=pulse_counter_id)
if pulse_counter is None:
pulse_counter = PulseCounter(
number=pulse_counter_id,
name=str(row[1]),
persistent=row[3] >= 1,
source=u'gateway')
else:
pulse_counter.name = str(row[1])
pulse_counter.persistent = row[3] >= 1
pulse_counter.source = u'gateway'
if room_id == 255:
pulse_counter.room = None
else:
pulse_counter.room = RoomsMigrator._get_or_create_room(
eext_controller, room_id, rooms, floors)
pulse_counter.save()
except Exception:
logger.exception(
'Could not migratie gateway PulseCounter')
os.rename(old_sqlite_db, '{0}.bak'.format(old_sqlite_db))
except Exception:
logger.exception('Could not migrate gateway PulseCounters')
