def test_alarm_output(sim_config_file, sim_result_file):
simulation_id = None
sim_config_file = os.path.join(
os.path.dirname(__file__),
f"simulation_config_files/{sim_config_file}")
sim_result_file = os.path.join(
os.path.dirname(__file__),
f"simulation_baseline_files/{sim_result_file}")
assert os.path.exists(
sim_config_file
), f"File {sim_config_file} must exist to run simulation test"
with startup_containers():
with gappsd() as gapps:
os.makedirs("/tmp/output", exist_ok=True)
with open("/tmp/output/simulation.output", 'w') as outfile:
sim_complete = False
rcvd_measurement = False
def onmeasurement(sim, timestep, measurements):
nonlocal rcvd_measurement
# if not rcvd_measurement:
rcvd_measurement = True
LOGGER.info('A measurement happened at %s', timestep)
def onfinishsimulation(sim):
nonlocal sim_complete
sim_complete = True
print("Completed simulator")
with open(sim_config_file) as fp:
run_config = json.load(fp)
print(run_config["simulation_config"]["start_time"])
sim = Simulation(gapps, run_config)
LOGGER.info('Starting the simulation')
sim.add_onmesurement_callback(onmeasurement)
sim.add_oncomplete_callback(onfinishsimulation)
LOGGER.info('sim.add_onmesurement_callback')
LOGGER.info("Querying Alarm topic for alarms")
sim.start_simulation()
print(sim.simulation_id)
alarms_topic = t.service_output_topic('gridappsd-alarms',
sim.simulation_id)
log_topic = t.simulation_output_topic(sim.simulation_id)
input_topic = t.simulation_input_topic(sim.simulation_id)
gapps.subscribe(alarms_topic, on_message)
gapps.subscribe(log_topic, on_message)
# gapps.subscribe(input_topic, on_message)
while not sim_complete:
LOGGER.info('Sleeping')
sleep(30)
def test_alarm_output(gridappsd_client, sim_config_file, sim_result_file):
sim_config_file = os.path.join(os.path.dirname(__file__), f"simulation_config_files/{sim_config_file}")
sim_result_file = os.path.join(os.path.dirname(__file__), f"simulation_baseline_files/{sim_result_file}")
assert os.path.exists(sim_config_file), f"File {sim_config_file} must exist to run simulation test"
gapps = gridappsd_client
# Allow proven to come up
sleep(30)
sim_complete = False
rcvd_measurement = False
#def onmeasurement(sim, timestep, measurements):
# nonlocal rcvd_measurement
# if not rcvd_measurement:
# rcvd_measurement = True
# LOGGER.info('A measurement happened at %s', timestep)
def onfinishsimulation(sim):
nonlocal sim_complete
sim_complete = True
LOGGER.info('Simulation Complete')
with open(sim_config_file) as fp:
LOGGER.info('Loading config')
run_config = json.load(fp)
LOGGER.info(f'Simulation start time {run_config["simulation_config"]["start_time"]}')
sim = Simulation(gapps, run_config)
LOGGER.info('Starting the simulation')
sim.start_simulation()
#LOGGER.info('sim.add_onmesurement_callback')
#sim.add_onmesurement_callback(onmeasurement)
LOGGER.info('sim.add_oncomplete_callback')
sim.add_oncomplete_callback(onfinishsimulation)
LOGGER.info("Querying for alarm topic")
alarms_topic = t.service_output_topic('gridappsd-alarms', sim.simulation_id)
log_topic = t.simulation_output_topic(sim.simulation_id)
gapps.subscribe(alarms_topic, on_message)
gapps.subscribe(log_topic, on_message)
while not sim_complete:
LOGGER.info('Sleeping')
sleep(30)
def _main():
global appName, sim_id, feeder_mrid, gapps
if len(sys.argv) < 2 or '-help' in sys.argv:
usestr = '\nUsage: ' + sys.argv[0] + ' #nodes\n'
usestr += '''
Optional command line arguments:
-help: show this usage message
'''
print(usestr, file=sys.stderr, flush=True)
exit()
appName = sys.argv[0]
sim_config_file = './sim_starter/' + sys.argv[1] + '-config.json'
gapps = GridAPPSD()
with open(sim_config_file) as config_fp:
sim_config = json.load(config_fp)
# if there is a sys.argv[2] command line argument, that indicates not to start a simulation
if len(sys.argv) < 3:
print('SIM_STARTER initializing simulation from: ' + sim_config_file,
file=sys.stderr,
flush=True)
sim = Simulation(gapps, sim_config)
print('SIM_STARTER about to start simulation...',
file=sys.stderr,
flush=True)
sim.start_simulation(timeout=90)
sim_id = sim.simulation_id
print('SIM_STARTER simulation started with id: ' + sim_id,
file=sys.stderr,
flush=True)
print(sim_id, flush=True)
# need to dump sim_config before passing it on to downstream modules
# in order to convert single quotes to double quotes
sim_req = json.dumps(sim_config)
print(sim_req, flush=True)
gapps.disconnect()
def test_simulation_no_duplicate_measurement_timestamps(
gridappsd_client: GridAPPSD):
num_measurements = 0
timestamps = set()
def measurement(sim, timestamp, measurement):
nonlocal num_measurements
num_measurements += 1
assert timestamp not in timestamps
timestamps.add(timestamp)
gapps = gridappsd_client
sim = Simulation(gapps, base_config())
sim.add_onmeasurement_callback(measurement)
sim.start_simulation()
sim.run_loop()
# did we get a measurement?
assert num_measurements > 0
# if empty then we know the simulation did not work.
assert timestamps
"id": "gridappsd-sensor-simulator",
"user_options": {
"sensors-config": {
"_99db0dc7-ccda-4ed5-a772-a7db362e9818": {
"nominal-value": 100,
"perunit-confidence-band": 0.02,
"aggregation-interval": 5,
"perunit-drop-rate": 0.01
},
"_ee65ee31-a900-4f98-bf57-e752be924c4d": {},
"_f2673c22-654b-452a-8297-45dae11b1e14": {}
},
"random-seed": 0,
"default-aggregation-interval": 30,
"passthrough-if-not-specified": False,
"default-perunit-confidence-band": 0.01,
"default-perunit-drop-rate": 0.05
}
}]
}
gapps = GridAPPSD()
request = {
"configurationType": "CIM Dictionary",
"parameters": {
"model_id": "_C1C3E687-6FFD-C753-582B-632A27E28507"
}
}
simulation = Simulation(gapps, run_config_123)
simulation.start_simulation()
def test_simulation_output(sim_config_file, sim_result_file):
sim_config_file = os.path.join(
os.path.dirname(__file__),
f"simulation_config_files/{sim_config_file}")
sim_result_file = os.path.join(
os.path.dirname(__file__),
f"simulation_baseline_files/{sim_result_file}")
assert os.path.exists(
sim_config_file
), f"File {sim_config_file} must exist to run simulation test"
# assert os.path.exists(sim_result_file), f"File {sim_result_file} must exist to run simulation test"
with startup_containers():
# Allow proven to come up
sleep(30)
starttime = int(time())
with gappsd() as gapps:
os.makedirs("/tmp/output", exist_ok=True)
with open("/tmp/output/simulation.output", 'w') as outfile:
LOGGER.info('Configuring simulation')
sim_complete = False
rcvd_measurement = False
rcvd_first_measurement = 0
are_we_paused = False
with open(sim_config_file) as fp:
LOGGER.info('Reading config')
run_config = json.load(fp)
run_config["simulation_config"]["start_time"] = str(
starttime)
sim = Simulation(gapps, run_config)
def onmeasurement(sim, timestep, measurements):
LOGGER.info('Measurement received at %s', timestep)
nonlocal rcvd_measurement
nonlocal rcvd_first_measurement
nonlocal are_we_paused
if not are_we_paused and not rcvd_first_measurement:
LOGGER.debug("Pausing sim now")
sim.pause()
are_we_paused = True
LOGGER.debug(f"ARWEPAUSED {are_we_paused}")
# Setting up so if we get another measurement wheil we
# are paused we know it
rcvd_measurement = False
if not rcvd_measurement:
print(f"A measurement happened at {timestep}")
# outfile.write(f"{timestep}|{json.dumps(measurements)}\n")
data = {"data": measurements}
outfile.write(json.dumps(data))
rcvd_measurement = True
else:
rcvd_measurement = True
rcvd_first_measurement = True
# sleep here until rcvd_measuremnt = True again
def ontimestep(sim, timestep):
print("Timestamp: {}".format(timestep))
def onfinishsimulation(sim):
nonlocal sim_complete
sim_complete = True
LOGGER.info('Simulation Complete')
LOGGER.info(f"Start time is {starttime}")
LOGGER.info('Loading config')
# tm: typo in add_onmesurement
LOGGER.info('sim.add_onmesurement_callback')
LOGGER.info('Starting sim')
sim.start_simulation()
print(sim.simulation_id)
sim.add_onmesurement_callback(onmeasurement)
sim.add_ontimestep_callback(ontimestep)
gapps.subscribe(t.simulation_log_topic(sim.simulation_id),
on_message)
sim.add_oncomplete_callback(onfinishsimulation)
LOGGER.info('sim.add_oncomplete_callback')
secs = 0
while secs < 30:
LOGGER.info(f"Sleep {secs}")
# we have to wait until the first measurement is called before
# the are_we_paused could have a chance of being set
secs += 1
sleep(1)
paused_seconds = 0
while are_we_paused:
LOGGER.info(f"PAUSED {paused_seconds}")
paused_seconds += 1
# s
if paused_seconds > 30:
LOGGER.info('Resuming simulation')
sim.resume()
LOGGER.info('Resumed simulation')
are_we_paused = False
break
sleep(1)
assert not are_we_paused, "We should have came out of the paused_seconds > 30"
while not sim_complete:
LOGGER.info('Sleeping')
sleep(5)
class GridAPPSDSimIntegration(BaseSimIntegration):
"""
The class is responsible for integration with GridAPPSD co-simulation platform.
It provides integration support to register configuration, start, stop, publish,
receive messages, pause and resume simulation
"""
def __init__(self, config, pubsub):
super(GridAPPSDSimIntegration, self).__init__(config)
self._work_callback = None
self.config = config
self.gridappsd = None
self.sim = None
self.event_callbacks = {}
self.topic_callbacks = {}
self.sim_id = None
self.username = None
self.password = None
def register_inputs(self, config=None, callback=None, **kwargs):
"""
Register configuration parameters with GridAppsD.
The config parameters may include but not limited to:
- power_system_config
- application_config
- simulation_config
- test_config
- service_configs
: Register agent callback method
:return:
"""
self.config = config
self.username = self.config.pop('username', 'system')
self.password = self.config.pop('password', 'manager')
self._work_callback = callback
def register_event_callbacks(self, callbacks={}):
"""
Register for event callbacks for event notifications such as
- on measurement change
- on timestep change
- on finish
"""
_log.debug("Registering for event callbacks")
self.event_callbacks = callbacks
def register_topic_callbacks(self, callbacks={}):
"""
Register for any simulation topic callbacks
"""
_log.debug("Registering for topic callbacks")
self.topic_callbacks = callbacks
def start_simulation(self, *args, **kwargs):
"""
Simulation start activities involve:
- Creating GridAppsD connection gevent thread
- Registering for event callbacks (if specified)
- Registering for topic callbacks if specified
- Starting simulation based on the input config
:return:
"""
try:
self.gridappsd = GridAPPSD(override_threading=self.receiver_thread,
username=self.username,
password=self.password)
_log.debug('Gridappsd connected')
_log.debug(f"connection config is: {self.config}")
self.sim = Simulation(self.gridappsd, self.config)
_log.debug('Gridappsd adding onstart callback')
# Register for onstart callback to know if simulation has started
self.sim.add_onstart_callback(self.sim_on_start)
# Register event callbacks - on measurement, on timestep, on finish
for name, cb in self.event_callbacks.items():
if name == 'MEASUREMENT':
_log.debug('Gridappsd adding measurement callback')
self.sim.add_onmesurement_callback(cb)
elif name == 'TIMESTEP':
_log.debug('Gridappsd adding timestep callback')
self.sim.add_ontimestep_callback(cb)
elif name == 'FINISH':
_log.debug('Gridappsd adding finish callback')
self.sim.add_oncomplete_callback(cb)
# Register/Subscribe for simulation topics
for topic, cb in self.topic_callbacks:
_log.debug('Gridappsd subscribing to topics callback')
self.gridappsd.subscribe(topic, cb)
# Starting GridAppsD simulation
self.sim.start_simulation()
_log.debug(f"Gridappsd simulation id: {self.sim.simulation_id}")
except stomp.exception.NotConnectedException as ex:
_log.error("Unable to connect to GridAPPSD: {}".format(ex))
raise ex
def sim_on_start(self, sim):
"""
Simulation on start callback to get notified when simulation starts
"""
_log.debug(
f"GridAppsD simulation id inside sim_on_start(): {sim.simulation_id}"
)
self.sim_id = sim.simulation_id
def receiver_thread(self, arg):
"""
GridAPPSD connection thread
"""
self._receiver_thread = gevent.threading.Thread(group=None, target=arg)
self._receiver_thread.daemon = True # Don't let thread prevent termination
self._receiver_thread.start()
_log.debug('Gridappsd receiver_thread started!')
return self._receiver_thread
def publish_to_simulation(self, topic, message, **kwargs):
"""
Publish message to GridAppsD
:param topic: GridAppsD publication topic
:param message: message
:return:
"""
self.gridappsd.send(topic, message)
def pause_simulation(self, timeout=None, **kwargs):
"""
Pause the GridAppsD simulation
"""
if timeout is None:
self.sim.pause()
else:
self.sim.pause(timeout)
def resume_simulation(self, *args, **kwargs):
"""
Resume the GridAppsD simulation
"""
self.sim.resume()
def is_sim_installed(self, **kwargs):
"""
Flag to indicate if GridAppsD is installed
"""
return HAS_GAPPSD
def stop_simulation(self, *args, **kwargs):
"""
Stop the simulation if running and disconnect from GridAppsD server
:return:
"""
_log.debug('Stopping the simulation')
try:
if self.sim_id is not None:
self.sim.stop()
_log.debug('Disconnect GridAppsd')
if self.gridappsd is not None:
self.gridappsd.disconnect()
except Exception:
_log.error("Error stop GridAPPSD simulation")
def test_timeseries_output(sim_config_file, sim_result_file):
simulation_id = None
sim_config_file = os.path.join(os.path.dirname(__file__), f"simulation_config_files/{sim_config_file}")
sim_result_file = os.path.join(os.path.dirname(__file__), f"simulation_baseline_files/{sim_result_file}")
# sim_test_config = os.path.join(os.path.dirname(__file__), f"simulation_baseline_files/{sim_test_file}")
assert os.path.exists(sim_config_file), f"File {sim_config_file} must exist to run simulation test"
# assert os.path.exists(sim_result_file), f"File {sim_result_file} must exist to run simulation test"
with startup_containers():
with gappsd() as gapps:
os.makedirs("/tmp/output", exist_ok=True)
with open("/tmp/output/simulation.json", 'w') as outfile:
sim_complete = False
rcvd_measurement = False
def onmeasurement(sim, timestep, measurements):
nonlocal rcvd_measurement
# print(rcvd_measurement)
if not rcvd_measurement:
# print(f"A measurement happened at {timestep}")
LOGGER.info('Measurement received at %s', timestep)
outfile.write(f"{timestep}|{json.dumps(measurements)}\n")
with open(sim_config_file) as fp:
run_config = json.load(fp)
starttime = run_config["simulation_config"]["start_time"]
# run_config["simulation_config"]["start_time"] = str(starttime)
print(run_config["simulation_config"]["start_time"])
def onfinishsimulation(sim):
nonlocal sim_complete
sim_complete = True
LOGGER.info('Simulation Complete')
sim = Simulation(gapps, run_config)
sim.add_oncomplete_callback(onfinishsimulation)
sim.add_onmesurement_callback(onmeasurement)
sim.start_simulation()
LOGGER.info("Starting simulation")
print(sim.simulation_id)
with open("./simulation_config_files/weather_data.json", 'r') as g:
LOGGER.info('Querying weather data from timeseries')
query1 = json.load(g)
a = gapps.get_response(t.TIMESERIES, query1, timeout=60)
LOGGER.info('Weather data received ')
assert "Diffuse" in a["data"][0], "Weather data query does not have expected output"
LOGGER.info('Weather data query has expected output')
while not sim_complete:
sleep(5)
with open("./simulation_config_files/timeseries_query.json", 'r') as f:
query2 = json.load(f)
query2["queryFilter"]["simulation_id"] = sim.simulation_id
LOGGER.info('Querying simulation data from timeseries')
q = gapps.get_response(t.TIMESERIES, query2, timeout=300)
LOGGER.info('Simulation data received for Timeseries API')
file2 = open("./out-input.txt", 'w')
file2.write(json.dumps(q))
assert "hasSimulationMessageType" in q["data"][0], "Simulation data query does not have expected output"
LOGGER.info('Simulation data query has expected output')
with open("./simulation_config_files/sensor_query.json", 'r') as file:
sensor_query = json.load(file)
sensor_query["queryFilter"]["simulation_id"] = sim.simulation_id
LOGGER.info('Querying GridAPPS-D sensor simulator data from timeseries')
result = gapps.get_response(t.TIMESERIES, sensor_query, timeout=300)
assert "hasSimulationMessageType" in result["data"][0], "Sensor simulator data does not have expected output"
LOGGER.info('Query response received for GridAPPS-D sensor simulator data from timeseries')
def test_simulation_output(gridappsd_client, sim_config_file, sim_result_file):
global resume_msg
global pause_msg
sim_config_file = os.path.join(os.path.dirname(__file__), f"simulation_config_files/{sim_config_file}")
sim_expected_results_file = os.path.join(os.path.dirname(__file__), f"simulation_baseline_files/{sim_result_file}")
sim_actual_result_file = f"/tmp/output/{sim_result_file}"
os.makedirs(Path(sim_actual_result_file).parent, exist_ok=True)
assert os.path.exists(sim_config_file), f"File {sim_config_file} must exist to run simulation test"
# assert os.path.exists(sim_result_file), f"File {sim_result_file} must exist to run simulation test"
gapps = gridappsd_client
sleep(30)
os.makedirs("/tmp/output", exist_ok=True)
with open(sim_actual_result_file, 'w') as outfile:
LOGGER.info('Configuring simulation')
sim_complete = False
rcvd_measurement = False
rcvd_first_measurement = 0
are_we_paused = False
received_measurment_count = 0
LOGGER.info('Loading config')
with open(sim_config_file) as fp:
LOGGER.info('Reading config')
run_config = json.load(fp)
# run_config["simulation_config"]["start_time"] = str(starttime)
sim = Simulation(gapps, run_config)
def onmeasurement(sim, timestep, measurements):
LOGGER.info('Measurement received at %s', timestep)
nonlocal rcvd_measurement
nonlocal received_measurment_count
received_measurment_count += 1
try:
if not rcvd_measurement:
print(f"A measurement happened at {timestep}")
# outfile.write(f"{timestep}|{json.dumps(measurements)}\n")
data = {"data": measurements}
outfile.write(json.dumps(data))
rcvd_measurement = True
else:
rcvd_measurement = True
# rcvd_first_measurement = True
except Exception as ex:
LOGGER.error(f"An exception handled in callback {ex}")
# sleep here until rcvd_measuremnt = True again
# def ontimestep(sim, timestep):
# # print("Timestamp: {}".format(timestep))
def onfinishsimulation(sim):
nonlocal sim_complete
sim_complete = True
LOGGER.info('Simulation Complete')
LOGGER.info('sim.add_onmesurement_callback')
LOGGER.info('Starting sim')
sim.start_simulation()
print(sim.simulation_id)
sim.add_onmesurement_callback(onmeasurement)
# sim.add_ontimestep_callback(ontimestep)
gapps.subscribe(t.simulation_log_topic(sim.simulation_id), on_message)
sim.add_oncomplete_callback(onfinishsimulation)
LOGGER.info('sim.add_oncomplete_callback')
secs = 0
num_measurements_before_pause = 3
while received_measurment_count < num_measurements_before_pause:
LOGGER.info(f"Waiting for at least {num_measurements_before_pause} measurements"
f"but have {received_measurment_count} time take {secs}s ")
secs += 1
sleep(1)
LOGGER.debug("Pausing sim now")
sim.pause()
are_we_paused = True
paused_seconds = 0
while are_we_paused:
LOGGER.info(f"PAUSED {paused_seconds}")
paused_seconds += 1
if paused_seconds > 30:
LOGGER.info('Resuming simulation')
sim.resume()
LOGGER.info('Resumed simulation')
are_we_paused = False
break
sleep(1)
assert not are_we_paused, "We should have came out of the paused_seconds > 30"
assert "paused" in pause_msg, 'Pause command not called'
sleep(10)
assert "resumed" in resume_msg, 'Resume command not called'
while not sim_complete:
LOGGER.info('Sleeping')
sleep(5)
import os
os.environ["GRIDAPPSD_USER"] = "******"
os.environ["GRIDAPPSD_PASSWORD"] = "******"
os.environ["GRIDAPPSD_ADDRESS"] = "gridappsd"
os.environ["GRIDAPPSD_PORT"] = "61613"
output_queue = Queue()
gapps = GridAPPSD()
with open(
"/repos/gridappsd-python/examples/default_run_simulation_ieee9500_final_config.yaml"
) as fp:
config = yaml.safe_load(fp)
sim = Simulation(gapps=gapps, run_config=config)
sim_id = sim.start_simulation()
def data_output(topic, message):
print(f"data_output {json.dumps(message)[:50]}")
output_queue.put(message)
gapps.subscribe(simulation_output_topic(sim.simulation_id), data_output)
def write_thread(queue, filename):
with open(filename, "w") as f:
while True:
content = queue.get()
if content == "DONE":