def create(cls, fromchannel: Channel, tochannel: Channel,
address: Address) -> DimeClient:
dimec = Dime(fromchannel, address)
ok = dimec.start()
if not ok:
raise ValueError('Could not start dimec')
return cls(fromchannel, tochannel, address, dimec)
def main(bind, port, dhost, dport):
print(f'Connecting to dime on tcp://{dhost}:{dport}')
dimec = Dime('geovis', f'tcp://{dhost}:{dport}')
ok = dimec.start()
if not ok:
raise ValueError('Could not start dime client')
return
global _g_dimec
_g_dimec = dimec
print(f'Listening on {bind}:{port}')
start_server = serve(handler, bind, port)
loop = get_event_loop()
loop.run_until_complete(start_server)
loop.run_forever()
class MiniPMU(object):
def __init__(self, name: str='', dime_address: str='ipc:///tmp/dime',
pmu_idx: list=list(), max_store: int=1000, pmu_ip: str='0.0.0.0', pmu_port: int=1410,
**kwargs):
"""
Create a MiniPMU instance for PMU data streaming over Mininet.
Assumptions made for
Parameters
----------
name
dime_address
pmu_idx
max_store
pmu_ip
pmu_port
kwargs
"""
assert name, 'PMU Receiver name is empty'
assert pmu_idx, 'PMU idx is empty'
self.name = name
self.dime_address = dime_address
self.pmu_idx = pmu_idx
self.max_store = max_store
# for recording
self.max_store_record = 30 * 600 # 600 seconds
self.reset = True
self.pmu_configured = False
self.pmu_streaming = False
self.reset_var()
self.dimec = Dime(self.name, self.dime_address)
self.pmu = Pmu(ip=pmu_ip, port=pmu_port)
def reset_var(self, retain_data=False):
"""
Reset flags and memory
:return: None
"""
if not self.reset:
return
self.bus_name = []
self.var_idx = {'am': [],
'vm': [],
'w': [],
}
self.fn = 60
self.Vn = []
self.Varheader = list()
self.Idxvgs = dict()
self.SysParam = dict()
self.SysName = dict()
self.Varvgs = ndarray([])
self.t = ndarray([])
self.data = ndarray([])
self.count = 0
# recording storage
if not retain_data:
self.t_record = ndarray([])
self.data_record = ndarray([])
self.count_record = 0
self.counter_replay = 0 # replay index into `data_record` and `t_record`
self.record_state = RecordState.IDLE
self.last_data = None
self.last_t = None
def start_dime(self):
"""
Starts the dime client stored in `self.dimec`
"""
# logger.info('Connecting to server at {}'.format(self.dime_address))
assert self.dimec.start()
# logger.info('DiME client connected')
def respond_to_sim(self):
"""
DEPRECIATED: Respond with data streaming configuration to the simulator
:return: None
"""
pass
def get_bus_name(self):
"""
Return bus names based on ``self.pmu_idx`` and store bus names to
``self.bus_name``
:return: list of bus names
"""
# assign generic bus names
self.bus_name = list(self.pmu_idx)
for i in range(len(self.bus_name)):
self.bus_name[i] = 'Bus_' + str(self.bus_name[i])
# assign names from SysName if present
if len(self.SysName) > 0:
for i in range(len(self.bus_name)):
self.bus_name[i] = self.SysName['Bus'][self.pmu_idx[i] - 1]
# logger.debug('PMU names changed to: {}'.format(self.bus_name))
return self.bus_name
def get_bus_Vn(self):
"""
Retrieve Bus.Vn
Returns
-------
"""
self.Vn = [1] * len(self.pmu_idx)
for i, idx in enumerate(self.pmu_idx):
self.Vn[i] = self.SysParam['Bus'][idx][1] * 1000 # get Vn
# logger.info('Retrieved bus Vn {}'.format(self.Vn))
def config_pmu(self):
"""
Sets the ConfigFrame2 of the PMU
:return: None
"""
self.cfg = ConfigFrame2(pmu_id_code=self.pmu_idx[0], # PMU_ID
time_base=1000000, # TIME_BASE
num_pmu=1, # Number of PMUs included in data frame
station_name=self.bus_name[0], # Station name
id_code=self.pmu_idx[0], # Data-stream ID(s)
data_format=(True, True, True, True), # Data format - POLAR; PH - REAL; AN - REAL; FREQ - REAL;
phasor_num=1, # Number of phasors
analog_num=1, # Number of analog values
digital_num=1, # Number of digital status words
channel_names=["V_PHASOR", "ANALOG1", "BREAKER 1 STATUS",
"BREAKER 2 STATUS", "BREAKER 3 STATUS", "BREAKER 4 STATUS", "BREAKER 5 STATUS",
"BREAKER 6 STATUS", "BREAKER 7 STATUS", "BREAKER 8 STATUS", "BREAKER 9 STATUS",
"BREAKER A STATUS", "BREAKER B STATUS", "BREAKER C STATUS", "BREAKER D STATUS",
"BREAKER E STATUS", "BREAKER F STATUS", "BREAKER G STATUS"], # Channel Names
ph_units=[(0, 'v')], # Conversion factor for phasor channels - (float representation, not important)
an_units=[(1, 'pow')], # Conversion factor for analog channels
dig_units=[(0x0000, 0xffff)], # Mask words for digital status words
f_nom=60.0, # Nominal frequency
cfg_count=1, # Configuration change count
data_rate=30) # Rate of phasor data transmission)
self.hf = HeaderFrame(self.pmu_idx[0], # PMU_ID
"MiniPMU <{name}> {pmu_idx}".format(name=self.name, pmu_idx = self.pmu_idx)) # Header Message
self.pmu.set_configuration(self.cfg)
self.pmu.set_header(self.hf)
# self.pmu.run()
def find_var_idx(self):
"""
Returns a dictionary of the indices into Varheader based on
`self.pmu_idx`. Items in `self.pmu_idx` uses 1-indexing.
For example, if `self.pmu_idx` == [1, 2], this function will return
the indices of
- Idxvgs.Pmu.vm[0] and Idxvgs.Pmu.vm[1] as vm
- Idxvgs.Pmu.am[0] and Idxvgs.Pmu.am[1] as am
- Idxvgs.Bus.w_Busfreq[0] and Idxvgs.Bus.w_Busfreq[1] as w
in the dictionary `self. var_idx` with the above fields.
:return: ``var_idx`` in ``pmudata``
"""
npmu = len(self.Idxvgs['Pmu']['vm'][0])
self.var_idx['vm'] = [int(i) - 1 for i in self.pmu_idx]
self.var_idx['am'] = [npmu + int(i) - 1 for i in self.pmu_idx]
self.var_idx['w'] = [2 * npmu + int(i) - 1 for i in self.pmu_idx]
# TODO: make it static
@property
def vgsvaridx(self):
return array(self.var_idx['vm'] +
self.var_idx['am'] +
self.var_idx['w'], dtype=int)
def init_storage(self, flush=False):
"""
Initialize data storage `self.t` and `self.data`
:return: if the storage has been reset
"""
# TODO: make it more efficient??
ret = False
if self.count % self.max_store == 0:
self.t = zeros(shape=(self.max_store, 1), dtype=float)
self.data = zeros(shape=(self.max_store, len(self.pmu_idx * 3)),
dtype=float)
self.count = 0
ret = True
else:
ret = False
if (self.count_record % self.max_store_record == 0) or (flush is True):
self.t_record = zeros(shape=(self.max_store_record, 1),
dtype=float)
self.data_record = zeros(shape=(self.max_store_record,
len(self.pmu_idx * 3)), dtype=float)
self.count_record = 0
self.counter_replay = 0
ret = ret and True
else:
ret = False
return ret
def sync_and_handle(self):
"""
Sync and call data processing functins
:return:
"""
ret = False
var = self.dimec.sync()
if var is False or None:
return ret
# if self.reset is True:
# logger.info('[{name}] variable <{var}> synced.'
# .format(name=self.name, var=var))
data = self.dimec.workspace[var]
if var in ('SysParam', 'Idxvgs', 'Varheader'):
# only handle these three variables during reset cycle
if self.reset is True:
self.__dict__[var] = data
# else:
# logger.info('{} not handled outside reset cycle'.format(var))
elif var == 'pmudata':
# only handle pmudata during normal cycle
if self.reset is False:
# logger.info('In, t={:.4f}'.format(data['t']))
self.handle_measurement_data(data)
# else:
# logger.info('{} not handled during reset cycle'.format(var))
# handle SysName any time
elif var == 'SysName':
self.__dict__[var] = data
self.get_bus_name()
elif var == 'DONE' and data == 1:
self.reset = True
self.reset_var(retain_data=True)
elif var == 'pmucmd' and isinstance(data, dict):
cmd = ''
if data.get('record', 0) == 1:
# start recording
if self.record_state == RecordState.IDLE \
or self.record_state == RecordState.RECORDED:
self.record_state = RecordState.RECORDING
cmd = 'start recording'
# else:
# logger.warning('cannot start recording in state {}'
# .format(self.record_state))
elif data.get('record', 0) == 2:
# stop recording if started
if self.record_state == RecordState.RECORDING:
cmd = 'stop recording'
self.record_state = RecordState.RECORDED
# else:
# logger.warning('cannot stop recording in state {}'
# .format(self.record_state))
if data.get('replay', 0) == 1:
# start replay
if self.record_state == RecordState.RECORDED:
cmd = 'start replay'
self.record_state = RecordState.REPLAYING
# else:
# logger.warning('cannot start replaying in state {}'
# .format(self.record_state))
if data.get('replay', 0) == 2:
# stop replay but retain the saved data
if self.record_state == RecordState.REPLAYING:
cmd = 'stop replay'
self.record_state = RecordState.RECORDED
# else:
# logger.warning('cannot stop replaying in state {}'
# .format(self.record_state))
if data.get('flush', 0) == 1:
# flush storage
cmd = 'flush storage'
self.init_storage(flush=True)
self.record_state = RecordState.IDLE
# if cmd:
# logger.info('[{name}] <{cmd}>'.format(name=self.name, cmd=cmd))
# else:
# logger.info('[{name}] {cmd} not handled during normal ops'
# .format(name=self.name, cmd=var))
return var
def handle_measurement_data(self, data):
"""
Store synced data into self.data and return in a tuple of (t, values)
:return: (t, vars)
"""
self.init_storage()
self.data[self.count, :] = data['vars'][0, self.vgsvaridx].reshape(-1)
self.t[self.count, :] = data['t']
self.count += 1
# record
if self.record_state == RecordState.RECORDING:
self.data_record[self.count_record, :] = \
data['vars'][0, self.vgsvaridx].reshape(-1)
self.t_record[self.count_record, :] = data['t']
self.count_record += 1
self.last_data = data['vars']
self.last_t = data['t']
return data['t'], data['vars']
def run(self):
"""
Process control function
:return None
"""
self.start_dime()
self.pmu.run()
while True:
if self.reset is True:
# receive init and respond
# logger.info('[{name}] Entering reset mode..'
# .format(name=self.name))
while True:
var = self.sync_and_handle()
if var is False:
time.sleep(0.01)
if len(self.Varheader) > 0\
and len(self.Idxvgs) > 0\
and len(self.SysParam) > 0 \
and len(self.SysName) > 0:
self.find_var_idx()
self.get_bus_Vn()
break
self.respond_to_sim()
if self.pmu_configured is False:
self.config_pmu()
self.pmu_configured = True
self.reset = False
# logger.debug('Entering sync and short sleep...')
var = self.sync_and_handle()
time.sleep(0.001)
if var is False:
continue
elif var == 'pmudata':
if self.pmu.clients and not self.reset:
if self.record_state == RecordState.REPLAYING:
# prepare recorded data
npmu = len(self.pmu_idx)
v_mag = self.data_record[self.counter_replay, :npmu] * self.Vn[0]
v_ang = wrap_angle(self.data_record[self.counter_replay, npmu:2*npmu])
v_freq = self.data_record[self.counter_replay, 2*npmu:3*npmu] * self.fn
self.counter_replay += 1
# at the end of replay, reset
if self.counter_replay == self.count_record:
self.counter_replay = 0
self.record_state = RecordState.RECORDED
else:
# use fresh data
v_mag = self.last_data[0, self.var_idx['vm']] * self.Vn[0]
v_ang = wrap_angle(self.last_data[0, self.var_idx['am']])
v_freq = self.last_data[0, self.var_idx['w']] * self.fn
# TODO: add noise to data
try:
# TODO: fix multiple measurement (multi-bus -> one PMU case)
self.pmu.send_data(phasors=[(v_mag, v_ang)],
analog=[9.99],
digital=[0x0001],
#freq=(v_freq-60)*1000
freq = v_freq
)
# logger.info('Out, f={f:.5f}, vm={vm:.1f}, am={am:.2f}'.format(f=v_freq[0], vm=v_mag[0], am=v_ang[0]))
except Exception as e:
logger.exception(e)
from andes_addon.dime import Dime
dimec = Dime('ISLANDING', 'tcp://192.168.1.200:5000')
dimec.start()
event = {'id': [143, 146, 135],
'name': ['Line', 'Line', 'Line'],
'time': [-1, -1, -1],
'duration': [0, 0, 0],
'action': [0, 0, 0]
}
dimec.send_var('sim', 'Event', event)
dimec.exit()
def broadcastCommFiles(rootFolder="./", port=8819):
root = Path(rootFolder)
flow = root / 'stats' / 'net_stats_flow.csv'
port = root / 'stats' / 'net_stats_port.csv'
wecc = root / 'topology' / 'config_wecc_full.csv'
node = root / 'topology' / 'sw_port_node.csv'
switches: Dict[Idx, Tuple[Longitude, Latitude]] = {}
pmus: Dict[Idx, Tuple[Longitude, Latitude]] = {}
pdcs: Dict[Idx, Tuple[Longitude, Latitude]] = {}
macs: Dict[Idx, MAC] = {}
links: List[Tuple[Idx, Idx]] = []
with open(wecc, 'r') as f:
reader = DictReader(f)
for row in reader:
Idx = row['Idx']
Type = row['Type']
Longitude = row['Latitude']
Latitude = row['Longitude']
MAC = row['MAC'].replace(':', '')
From = row['From']
To = row['To']
if Type == 'Switch':
switches[Idx] = (float(Longitude), float(Latitude))
macs[Idx] = MAC
elif Type == 'PMU':
pmus[Idx] = (float(Longitude), float(Latitude))
elif Type == 'PDC':
pdcs[Idx] = (float(Longitude), float(Latitude))
elif Type == 'Link':
links.append((From, To))
elif Type in ('Region', 'HwIntf'):
pass
else:
print('Unknown: ' + Type)
LTBNET_params = {}
Switch = np.zeros((len(switches), 2))
LTBNET_params['Switch'] = Switch
SwitchIndex: Dict[Idx, int] = {}
SwitchIndexInv: Dict[int, Idx] = {}
for i, (Idx, (Longitude, Latitude)) in enumerate(switches.items()):
SwitchIndex[Idx] = i
SwitchIndexInv[i] = Idx
Switch[i] = (Latitude, Longitude)
Pmu = np.zeros((len(pmus), 2))
LTBNET_params['Pmu'] = Pmu
PmuIndex: Dict[Idx, int] = {}
PmuIndexInv: Dict[int, Idx] = {}
for i, (Idx, (Longitude, Latitude)) in enumerate(pmus.items()):
PmuIndex[Idx] = i
PmuIndexInv[i] = Idx
Pmu[i] = (Latitude, Longitude)
Pdc = np.zeros((len(pdcs), 2))
LTBNET_params['Pdc'] = Pdc
PdcIndex: Dict[Idx, int] = {}
PdcIndexInv: Dict[int, Idx] = {}
for i, (Idx, (Longitude, Latitude)) in enumerate(pdcs.items()):
PdcIndex[Idx] = i
PdcIndexInv[i] = Idx
Pdc[i] = (Latitude, Longitude)
Link = np.zeros((len(links), 4))
LTBNET_params['Link'] = Link
LinkIndex: Dict[Tuple[Idx, Idx], int] = {}
LinkIndexInv: Dict[int, Tuple[Idx, Idx]] = {}
for i, (From, To) in enumerate(links):
LinkIndex[From, To] = i
LinkIndexInv[i] = (From, To)
FromType = None
FromIndex = None
if From in switches:
FromType = 0
FromIndex = SwitchIndex[From]
elif From in pmus:
FromType = 1
FromIndex = PmuIndex[From]
elif From in pdcs:
FromType = 2
FromIndex = PdcIndex[From]
else:
print(From)
raise NotImplementedError
ToType = None
ToIndex = None
if To in switches:
ToType = 0
ToIndex = SwitchIndex[To]
elif To in pmus:
ToType = 1
ToIndex = PmuIndex[To]
elif To in pdcs:
ToType = 2
ToIndex = PdcIndex[To]
else:
raise NotImplementedError
Link[i] = (FromType, FromIndex, ToType, ToIndex)
LTBNET_header = []
LTBNET_idx = {}
LTBNET_vars = {}
dimec = Dime('LTBNET', 'tcp://127.0.0.1:8519')
ok = dimec.start()
if not ok:
print('bad!')
exit()
dimec.broadcast('LTBNET_params', LTBNET_params)
'''
Here we garner the port information from the sw_port_node file,
since the information does not have to be sent across dime, we use
this data to simplify what we shall stream through dime vars later
'''
macandporttoidx: Dict[Tuple[MAC, int], Idx] = {}
idtoswitch: Dict[SID, Idx] = {}
mactoidx: Dict[MAC, Idx] = {}
with open(node, 'r') as f:
reader = DictReader(f)
for row in reader:
MAC = row['MAC']
To = row['Node_Name']
SID = row['Switch_ID']
Port = int(row['Port'])
From = row['Idx']
macandporttoidx[MAC, Port] = To
idtoswitch[SID] = From
mactoidx[MAC] = From
# Mac and Port mapped to an internal name (i.e. "s1"),
# change that internal name from "s1" to actual switch S_BCTC
for key, val in macandporttoidx.items():
if val in idtoswitch:
macandporttoidx[key] = idtoswitch[val]
print(macandporttoidx)
# mac maps to an internal name (i.e "s1"),
# change that intenal name from "s1" to actual switch S_BCTC
for key, val in mactoidx.items():
if val in idtoswitch:
mactoidx[key] = idtoswitch[val]
# change (MAC, Port) -> IDX to (IDX, Port) -> Idx
# this process could be skipped; however, it makes debugging easier
fullportmap: Dict[Tuple[Idx, int], Idx] = {}
for key, theidx in macandporttoidx.items():
themac = key[0]
theport = int(key[1])
fullportmap[(mactoidx[themac], theport)] = theidx
print(fullportmap)
# FlowDict = Dictionary{TimeAtSecond[TimeAtMilliSecond, fromType, fromIndex, toType, toIndex, packets, bytes]}
flowDict: Dict[int, List[Tuple[float, int, int, int, int, int, int]]] = {}
with open(flow, 'r') as f:
reader = DictReader(f)
gotTime = False
startTime = 0
for row in reader:
flowMac = row['datapath']
flowTime = float(row['epoch-time'])
flowOutPort = int(row['out-port'], 16)
flowPackets = int(row['packets'])
flowBytes = int(row['bytes'])
fromNode = mactoidx[flowMac]
toNode = fullportmap[(fromNode, flowOutPort)]
# Why subtracr a set time?
# A: Because our data set starts at a UNIX TimeStamp.
# Subtract the initial time to set the staring value's time stamp to 0
if not gotTime:
startTime = int(flowTime)
gotTime = True
aggregateTime = int(flowTime - startTime)
FromType = None
FromIndex = None
if fromNode in switches:
FromType = 0
FromIndex = SwitchIndex[fromNode]
elif fromNode in pmus:
FromType = 1
FromIndex = PmuIndex[fromNode]
elif fromNode in pdcs:
FromType = 2
FromIndex = PdcIndex[fromNode]
else:
print(fromNode)
raise NotImplementedError
ToType = None
ToIndex = None
if toNode in switches:
ToType = 0
ToIndex = SwitchIndex[toNode]
elif toNode in pmus:
ToType = 1
ToIndex = PmuIndex[toNode]
elif toNode in pdcs:
ToType = 2
ToIndex = PdcIndex[toNode]
else:
print(fromNode, toNode)
raise NotImplementedError
if aggregateTime not in flowDict:
flowDict[aggregateTime] = []
flowTime -= 1559160380
flowDict[aggregateTime].append(
(flowTime, FromType, FromIndex, ToType, ToIndex, flowPackets,
flowBytes))
# Create a timer, every second - broadcast new data
currentTime = timer()
oldTime = -1
maxTime = -1
for key in flowDict.keys():
if key > maxTime:
maxTime = key
while (True):
# For every second, try to get the new data
newTime = int(timer() - currentTime)
if newTime != oldTime:
if newTime in flowDict:
Transfer = np.zeros((len(flowDict[newTime]), 7))
LTBNET_vars['Transfer'] = Transfer
for i, (flowTime, FromType, FromIndex, ToType, ToIndex,
flowPackets,
flowBytes) in enumerate(flowDict[newTime]):
Transfer[i] = (flowTime, FromType, FromIndex, ToType,
ToIndex, flowPackets, flowBytes)
print(len(flowDict[newTime]))
dimec.broadcast('LTBNET_vars', LTBNET_vars)
oldTime = newTime
# If the file is coming to a close, exit
if newTime >= maxTime:
break
class MiniPDC(object):
"""A MiniPDC connecting to multiple PMUs and a DiME server
"""
def __init__(self,
name,
dime_address,
ip_list,
port_list=None,
loglevel=logging.INFO):
self._name = name
self._dime_address = dime_address
self._loglevel = loglevel
self.dimec = Dime(name, dime_address)
self.ip_list = ip_list
self.port_list = port_list # not being used now
# check if the lengths of `ip_list` and `port_list` match
self.pdc = {}
self.header = {}
self.config = {}
self.last_var = None
# state flags
self.andes_online = False
# self.pdc_started = False
@property
def npmu(self):
return len(self.ip_list)
def initialize(self):
"""
Reset or initialize, it is the same thing
Returns
-------
"""
pass
def sync_and_handle(self):
""" Sync from DiME and handle the received data
"""
self.last_var = self.dimec.sync()
val = None
if self.last_var not in (None, False):
val = self.dimec.workspace[self.last_var]
else:
return
if self.last_var == 'DONE' and int(val) == 1:
self.andes_online = False
self.initialize()
pass
return self.last_var
def start_dime(self):
logger.info('Connecting to DiME at {}'.format(self._dime_address))
self.dimec.start()
try:
self.dimec.exit()
except:
pass
self.dimec.start()
logger.info('DiME connected')
def init_pdc(self):
for idx, item in enumerate(self.ip_list):
pmu_idx = int(item.split('.')[3])
self.pdc[idx] = Pdc(pdc_id=pmu_idx,
pmu_ip=self.ip_list[idx],
pmu_port=1410)
self.pdc[idx].logger.setLevel("INFO")
logger.info('PDC initialized')
def get_header_config(self):
for idx, item in self.pdc.items(): # each item is a PDC
item.run() # Connect to PMU
self.header[idx] = item.get_header()
self.config[idx] = item.get_config()
for idx, item in self.pdc.items(): # each item is a PDC
item.start() # Request to start sending measurements
self.pdc_started = True
logger.info('PMU Header and ConfigFrame received')
def collect_data(self):
pass
def process_data(self):
pass
def run(self):
pass