def test_alarm_output(sim_config_file, sim_result_file):
simulation_id = None
sim_config_file = os.path.join(
os.path.dirname(__file__),
f"simulation_config_files/{sim_config_file}")
sim_result_file = os.path.join(
os.path.dirname(__file__),
f"simulation_baseline_files/{sim_result_file}")
assert os.path.exists(
sim_config_file
), f"File {sim_config_file} must exist to run simulation test"
with startup_containers():
with gappsd() as gapps:
os.makedirs("/tmp/output", exist_ok=True)
with open("/tmp/output/simulation.output", 'w') as outfile:
sim_complete = False
rcvd_measurement = False
def onmeasurement(sim, timestep, measurements):
nonlocal rcvd_measurement
# if not rcvd_measurement:
rcvd_measurement = True
LOGGER.info('A measurement happened at %s', timestep)
def onfinishsimulation(sim):
nonlocal sim_complete
sim_complete = True
print("Completed simulator")
with open(sim_config_file) as fp:
run_config = json.load(fp)
print(run_config["simulation_config"]["start_time"])
sim = Simulation(gapps, run_config)
LOGGER.info('Starting the simulation')
sim.add_onmesurement_callback(onmeasurement)
sim.add_oncomplete_callback(onfinishsimulation)
LOGGER.info('sim.add_onmesurement_callback')
LOGGER.info("Querying Alarm topic for alarms")
sim.start_simulation()
print(sim.simulation_id)
alarms_topic = t.service_output_topic('gridappsd-alarms',
sim.simulation_id)
log_topic = t.simulation_output_topic(sim.simulation_id)
input_topic = t.simulation_input_topic(sim.simulation_id)
gapps.subscribe(alarms_topic, on_message)
gapps.subscribe(log_topic, on_message)
# gapps.subscribe(input_topic, on_message)
while not sim_complete:
LOGGER.info('Sleeping')
sleep(30)
def test_simulation_output(sim_config_file, sim_result_file):
sim_config_file = os.path.join(
os.path.dirname(__file__),
f"simulation_config_files/{sim_config_file}")
sim_result_file = os.path.join(
os.path.dirname(__file__),
f"simulation_baseline_files/{sim_result_file}")
assert os.path.exists(
sim_config_file
), f"File {sim_config_file} must exist to run simulation test"
# assert os.path.exists(sim_result_file), f"File {sim_result_file} must exist to run simulation test"
with startup_containers():
# Allow proven to come up
sleep(30)
starttime = int(time())
with gappsd() as gapps:
os.makedirs("/tmp/output", exist_ok=True)
with open("/tmp/output/simulation.output", 'w') as outfile:
LOGGER.info('Configuring simulation')
sim_complete = False
rcvd_measurement = False
rcvd_first_measurement = 0
are_we_paused = False
with open(sim_config_file) as fp:
LOGGER.info('Reading config')
run_config = json.load(fp)
run_config["simulation_config"]["start_time"] = str(
starttime)
sim = Simulation(gapps, run_config)
def onmeasurement(sim, timestep, measurements):
LOGGER.info('Measurement received at %s', timestep)
nonlocal rcvd_measurement
nonlocal rcvd_first_measurement
nonlocal are_we_paused
if not are_we_paused and not rcvd_first_measurement:
LOGGER.debug("Pausing sim now")
sim.pause()
are_we_paused = True
LOGGER.debug(f"ARWEPAUSED {are_we_paused}")
# Setting up so if we get another measurement wheil we
# are paused we know it
rcvd_measurement = False
if not rcvd_measurement:
print(f"A measurement happened at {timestep}")
# outfile.write(f"{timestep}|{json.dumps(measurements)}\n")
data = {"data": measurements}
outfile.write(json.dumps(data))
rcvd_measurement = True
else:
rcvd_measurement = True
rcvd_first_measurement = True
# sleep here until rcvd_measuremnt = True again
def ontimestep(sim, timestep):
print("Timestamp: {}".format(timestep))
def onfinishsimulation(sim):
nonlocal sim_complete
sim_complete = True
LOGGER.info('Simulation Complete')
LOGGER.info(f"Start time is {starttime}")
LOGGER.info('Loading config')
# tm: typo in add_onmesurement
LOGGER.info('sim.add_onmesurement_callback')
LOGGER.info('Starting sim')
sim.start_simulation()
print(sim.simulation_id)
sim.add_onmesurement_callback(onmeasurement)
sim.add_ontimestep_callback(ontimestep)
gapps.subscribe(t.simulation_log_topic(sim.simulation_id),
on_message)
sim.add_oncomplete_callback(onfinishsimulation)
LOGGER.info('sim.add_oncomplete_callback')
secs = 0
while secs < 30:
LOGGER.info(f"Sleep {secs}")
# we have to wait until the first measurement is called before
# the are_we_paused could have a chance of being set
secs += 1
sleep(1)
paused_seconds = 0
while are_we_paused:
LOGGER.info(f"PAUSED {paused_seconds}")
paused_seconds += 1
# s
if paused_seconds > 30:
LOGGER.info('Resuming simulation')
sim.resume()
LOGGER.info('Resumed simulation')
are_we_paused = False
break
sleep(1)
assert not are_we_paused, "We should have came out of the paused_seconds > 30"
while not sim_complete:
LOGGER.info('Sleeping')
sleep(5)
class GridAPPSDSimIntegration(BaseSimIntegration):
"""
The class is responsible for integration with GridAPPSD co-simulation platform.
It provides integration support to register configuration, start, stop, publish,
receive messages, pause and resume simulation
"""
def __init__(self, config, pubsub):
super(GridAPPSDSimIntegration, self).__init__(config)
self._work_callback = None
self.config = config
self.gridappsd = None
self.sim = None
self.event_callbacks = {}
self.topic_callbacks = {}
self.sim_id = None
self.username = None
self.password = None
def register_inputs(self, config=None, callback=None, **kwargs):
"""
Register configuration parameters with GridAppsD.
The config parameters may include but not limited to:
- power_system_config
- application_config
- simulation_config
- test_config
- service_configs
: Register agent callback method
:return:
"""
self.config = config
self.username = self.config.pop('username', 'system')
self.password = self.config.pop('password', 'manager')
self._work_callback = callback
def register_event_callbacks(self, callbacks={}):
"""
Register for event callbacks for event notifications such as
- on measurement change
- on timestep change
- on finish
"""
_log.debug("Registering for event callbacks")
self.event_callbacks = callbacks
def register_topic_callbacks(self, callbacks={}):
"""
Register for any simulation topic callbacks
"""
_log.debug("Registering for topic callbacks")
self.topic_callbacks = callbacks
def start_simulation(self, *args, **kwargs):
"""
Simulation start activities involve:
- Creating GridAppsD connection gevent thread
- Registering for event callbacks (if specified)
- Registering for topic callbacks if specified
- Starting simulation based on the input config
:return:
"""
try:
self.gridappsd = GridAPPSD(override_threading=self.receiver_thread,
username=self.username,
password=self.password)
_log.debug('Gridappsd connected')
_log.debug(f"connection config is: {self.config}")
self.sim = Simulation(self.gridappsd, self.config)
_log.debug('Gridappsd adding onstart callback')
# Register for onstart callback to know if simulation has started
self.sim.add_onstart_callback(self.sim_on_start)
# Register event callbacks - on measurement, on timestep, on finish
for name, cb in self.event_callbacks.items():
if name == 'MEASUREMENT':
_log.debug('Gridappsd adding measurement callback')
self.sim.add_onmesurement_callback(cb)
elif name == 'TIMESTEP':
_log.debug('Gridappsd adding timestep callback')
self.sim.add_ontimestep_callback(cb)
elif name == 'FINISH':
_log.debug('Gridappsd adding finish callback')
self.sim.add_oncomplete_callback(cb)
# Register/Subscribe for simulation topics
for topic, cb in self.topic_callbacks:
_log.debug('Gridappsd subscribing to topics callback')
self.gridappsd.subscribe(topic, cb)
# Starting GridAppsD simulation
self.sim.start_simulation()
_log.debug(f"Gridappsd simulation id: {self.sim.simulation_id}")
except stomp.exception.NotConnectedException as ex:
_log.error("Unable to connect to GridAPPSD: {}".format(ex))
raise ex
def sim_on_start(self, sim):
"""
Simulation on start callback to get notified when simulation starts
"""
_log.debug(
f"GridAppsD simulation id inside sim_on_start(): {sim.simulation_id}"
)
self.sim_id = sim.simulation_id
def receiver_thread(self, arg):
"""
GridAPPSD connection thread
"""
self._receiver_thread = gevent.threading.Thread(group=None, target=arg)
self._receiver_thread.daemon = True # Don't let thread prevent termination
self._receiver_thread.start()
_log.debug('Gridappsd receiver_thread started!')
return self._receiver_thread
def publish_to_simulation(self, topic, message, **kwargs):
"""
Publish message to GridAppsD
:param topic: GridAppsD publication topic
:param message: message
:return:
"""
self.gridappsd.send(topic, message)
def pause_simulation(self, timeout=None, **kwargs):
"""
Pause the GridAppsD simulation
"""
if timeout is None:
self.sim.pause()
else:
self.sim.pause(timeout)
def resume_simulation(self, *args, **kwargs):
"""
Resume the GridAppsD simulation
"""
self.sim.resume()
def is_sim_installed(self, **kwargs):
"""
Flag to indicate if GridAppsD is installed
"""
return HAS_GAPPSD
def stop_simulation(self, *args, **kwargs):
"""
Stop the simulation if running and disconnect from GridAppsD server
:return:
"""
_log.debug('Stopping the simulation')
try:
if self.sim_id is not None:
self.sim.stop()
_log.debug('Disconnect GridAppsd')
if self.gridappsd is not None:
self.gridappsd.disconnect()
except Exception:
_log.error("Error stop GridAPPSD simulation")
# from pprint import pprint
# pprint(config['gridappsd'])
with run_containers(config, stop_after=True) as containers:
# Watches the log on the container for the MYSQL data.
containers.wait_for_log_pattern("gridappsd", "MYSQL")
gappsd = GridAPPSD()
gappsd.connect()
assert gappsd.connected
time.sleep(10)
sim = Simulation(gappsd, run_config=run_config)
sim.add_onstart_callback(onstart)
sim.add_oncomplete_callback(onfinish)
sim.add_ontimestep_callback(ontimestep)
sim.add_onmesurement_callback(onmeasurement)
sim.start_simulation()
sim.pause()
gappsd.subscribe(t.simulation_output_topic(sim.simulation_id), onsimulationoutput)
gappsd.subscribe(t.application_output_topic(GRIDAPPSD_SERVICE_ID, sim.simulation_id), onsensoroutput)
time.sleep(10)
sim.resume()
sim.run_loop()
print("Shutting down")
fh_sim_measurement.close()
fh_sensor_measurement.close()
fncs_measurements.close()
def test_timeseries_output(sim_config_file, sim_result_file):
simulation_id = None
sim_config_file = os.path.join(os.path.dirname(__file__), f"simulation_config_files/{sim_config_file}")
sim_result_file = os.path.join(os.path.dirname(__file__), f"simulation_baseline_files/{sim_result_file}")
# sim_test_config = os.path.join(os.path.dirname(__file__), f"simulation_baseline_files/{sim_test_file}")
assert os.path.exists(sim_config_file), f"File {sim_config_file} must exist to run simulation test"
# assert os.path.exists(sim_result_file), f"File {sim_result_file} must exist to run simulation test"
with startup_containers():
with gappsd() as gapps:
os.makedirs("/tmp/output", exist_ok=True)
with open("/tmp/output/simulation.json", 'w') as outfile:
sim_complete = False
rcvd_measurement = False
def onmeasurement(sim, timestep, measurements):
nonlocal rcvd_measurement
# print(rcvd_measurement)
if not rcvd_measurement:
# print(f"A measurement happened at {timestep}")
LOGGER.info('Measurement received at %s', timestep)
outfile.write(f"{timestep}|{json.dumps(measurements)}\n")
with open(sim_config_file) as fp:
run_config = json.load(fp)
starttime = run_config["simulation_config"]["start_time"]
# run_config["simulation_config"]["start_time"] = str(starttime)
print(run_config["simulation_config"]["start_time"])
def onfinishsimulation(sim):
nonlocal sim_complete
sim_complete = True
LOGGER.info('Simulation Complete')
sim = Simulation(gapps, run_config)
sim.add_oncomplete_callback(onfinishsimulation)
sim.add_onmesurement_callback(onmeasurement)
sim.start_simulation()
LOGGER.info("Starting simulation")
print(sim.simulation_id)
with open("./simulation_config_files/weather_data.json", 'r') as g:
LOGGER.info('Querying weather data from timeseries')
query1 = json.load(g)
a = gapps.get_response(t.TIMESERIES, query1, timeout=60)
LOGGER.info('Weather data received ')
assert "Diffuse" in a["data"][0], "Weather data query does not have expected output"
LOGGER.info('Weather data query has expected output')
while not sim_complete:
sleep(5)
with open("./simulation_config_files/timeseries_query.json", 'r') as f:
query2 = json.load(f)
query2["queryFilter"]["simulation_id"] = sim.simulation_id
LOGGER.info('Querying simulation data from timeseries')
q = gapps.get_response(t.TIMESERIES, query2, timeout=300)
LOGGER.info('Simulation data received for Timeseries API')
file2 = open("./out-input.txt", 'w')
file2.write(json.dumps(q))
assert "hasSimulationMessageType" in q["data"][0], "Simulation data query does not have expected output"
LOGGER.info('Simulation data query has expected output')
with open("./simulation_config_files/sensor_query.json", 'r') as file:
sensor_query = json.load(file)
sensor_query["queryFilter"]["simulation_id"] = sim.simulation_id
LOGGER.info('Querying GridAPPS-D sensor simulator data from timeseries')
result = gapps.get_response(t.TIMESERIES, sensor_query, timeout=300)
assert "hasSimulationMessageType" in result["data"][0], "Sensor simulator data does not have expected output"
LOGGER.info('Query response received for GridAPPS-D sensor simulator data from timeseries')
def test_simulation_output(gridappsd_client, sim_config_file, sim_result_file):
global resume_msg
global pause_msg
sim_config_file = os.path.join(os.path.dirname(__file__), f"simulation_config_files/{sim_config_file}")
sim_expected_results_file = os.path.join(os.path.dirname(__file__), f"simulation_baseline_files/{sim_result_file}")
sim_actual_result_file = f"/tmp/output/{sim_result_file}"
os.makedirs(Path(sim_actual_result_file).parent, exist_ok=True)
assert os.path.exists(sim_config_file), f"File {sim_config_file} must exist to run simulation test"
# assert os.path.exists(sim_result_file), f"File {sim_result_file} must exist to run simulation test"
gapps = gridappsd_client
sleep(30)
os.makedirs("/tmp/output", exist_ok=True)
with open(sim_actual_result_file, 'w') as outfile:
LOGGER.info('Configuring simulation')
sim_complete = False
rcvd_measurement = False
rcvd_first_measurement = 0
are_we_paused = False
received_measurment_count = 0
LOGGER.info('Loading config')
with open(sim_config_file) as fp:
LOGGER.info('Reading config')
run_config = json.load(fp)
# run_config["simulation_config"]["start_time"] = str(starttime)
sim = Simulation(gapps, run_config)
def onmeasurement(sim, timestep, measurements):
LOGGER.info('Measurement received at %s', timestep)
nonlocal rcvd_measurement
nonlocal received_measurment_count
received_measurment_count += 1
try:
if not rcvd_measurement:
print(f"A measurement happened at {timestep}")
# outfile.write(f"{timestep}|{json.dumps(measurements)}\n")
data = {"data": measurements}
outfile.write(json.dumps(data))
rcvd_measurement = True
else:
rcvd_measurement = True
# rcvd_first_measurement = True
except Exception as ex:
LOGGER.error(f"An exception handled in callback {ex}")
# sleep here until rcvd_measuremnt = True again
# def ontimestep(sim, timestep):
# # print("Timestamp: {}".format(timestep))
def onfinishsimulation(sim):
nonlocal sim_complete
sim_complete = True
LOGGER.info('Simulation Complete')
LOGGER.info('sim.add_onmesurement_callback')
LOGGER.info('Starting sim')
sim.start_simulation()
print(sim.simulation_id)
sim.add_onmesurement_callback(onmeasurement)
# sim.add_ontimestep_callback(ontimestep)
gapps.subscribe(t.simulation_log_topic(sim.simulation_id), on_message)
sim.add_oncomplete_callback(onfinishsimulation)
LOGGER.info('sim.add_oncomplete_callback')
secs = 0
num_measurements_before_pause = 3
while received_measurment_count < num_measurements_before_pause:
LOGGER.info(f"Waiting for at least {num_measurements_before_pause} measurements"
f"but have {received_measurment_count} time take {secs}s ")
secs += 1
sleep(1)
LOGGER.debug("Pausing sim now")
sim.pause()
are_we_paused = True
paused_seconds = 0
while are_we_paused:
LOGGER.info(f"PAUSED {paused_seconds}")
paused_seconds += 1
if paused_seconds > 30:
LOGGER.info('Resuming simulation')
sim.resume()
LOGGER.info('Resumed simulation')
are_we_paused = False
break
sleep(1)
assert not are_we_paused, "We should have came out of the paused_seconds > 30"
assert "paused" in pause_msg, 'Pause command not called'
sleep(10)
assert "resumed" in resume_msg, 'Resume command not called'
while not sim_complete:
LOGGER.info('Sleeping')
sleep(5)