def test_alarm_output(sim_config_file, sim_result_file):
simulation_id = None
sim_config_file = os.path.join(
os.path.dirname(__file__),
f"simulation_config_files/{sim_config_file}")
sim_result_file = os.path.join(
os.path.dirname(__file__),
f"simulation_baseline_files/{sim_result_file}")
assert os.path.exists(
sim_config_file
), f"File {sim_config_file} must exist to run simulation test"
with startup_containers():
with gappsd() as gapps:
os.makedirs("/tmp/output", exist_ok=True)
with open("/tmp/output/simulation.output", 'w') as outfile:
sim_complete = False
rcvd_measurement = False
def onmeasurement(sim, timestep, measurements):
nonlocal rcvd_measurement
# if not rcvd_measurement:
rcvd_measurement = True
LOGGER.info('A measurement happened at %s', timestep)
def onfinishsimulation(sim):
nonlocal sim_complete
sim_complete = True
print("Completed simulator")
with open(sim_config_file) as fp:
run_config = json.load(fp)
print(run_config["simulation_config"]["start_time"])
sim = Simulation(gapps, run_config)
LOGGER.info('Starting the simulation')
sim.add_onmesurement_callback(onmeasurement)
sim.add_oncomplete_callback(onfinishsimulation)
LOGGER.info('sim.add_onmesurement_callback')
LOGGER.info("Querying Alarm topic for alarms")
sim.start_simulation()
print(sim.simulation_id)
alarms_topic = t.service_output_topic('gridappsd-alarms',
sim.simulation_id)
log_topic = t.simulation_output_topic(sim.simulation_id)
input_topic = t.simulation_input_topic(sim.simulation_id)
gapps.subscribe(alarms_topic, on_message)
gapps.subscribe(log_topic, on_message)
# gapps.subscribe(input_topic, on_message)
while not sim_complete:
LOGGER.info('Sleeping')
sleep(30)
def test_alarm_output(gridappsd_client, sim_config_file, sim_result_file):
sim_config_file = os.path.join(os.path.dirname(__file__), f"simulation_config_files/{sim_config_file}")
sim_result_file = os.path.join(os.path.dirname(__file__), f"simulation_baseline_files/{sim_result_file}")
assert os.path.exists(sim_config_file), f"File {sim_config_file} must exist to run simulation test"
gapps = gridappsd_client
# Allow proven to come up
sleep(30)
sim_complete = False
rcvd_measurement = False
#def onmeasurement(sim, timestep, measurements):
# nonlocal rcvd_measurement
# if not rcvd_measurement:
# rcvd_measurement = True
# LOGGER.info('A measurement happened at %s', timestep)
def onfinishsimulation(sim):
nonlocal sim_complete
sim_complete = True
LOGGER.info('Simulation Complete')
with open(sim_config_file) as fp:
LOGGER.info('Loading config')
run_config = json.load(fp)
LOGGER.info(f'Simulation start time {run_config["simulation_config"]["start_time"]}')
sim = Simulation(gapps, run_config)
LOGGER.info('Starting the simulation')
sim.start_simulation()
#LOGGER.info('sim.add_onmesurement_callback')
#sim.add_onmesurement_callback(onmeasurement)
LOGGER.info('sim.add_oncomplete_callback')
sim.add_oncomplete_callback(onfinishsimulation)
LOGGER.info("Querying for alarm topic")
alarms_topic = t.service_output_topic('gridappsd-alarms', sim.simulation_id)
log_topic = t.simulation_output_topic(sim.simulation_id)
gapps.subscribe(alarms_topic, on_message)
gapps.subscribe(log_topic, on_message)
while not sim_complete:
LOGGER.info('Sleeping')
sleep(30)
def start_simulation(self, *args, **kwargs):
"""
Simulation start activities involve:
- Creating GridAppsD connection gevent thread
- Registering for event callbacks (if specified)
- Registering for topic callbacks if specified
- Starting simulation based on the input config
:return:
"""
try:
self.gridappsd = GridAPPSD(override_threading=self.receiver_thread,
username=self.username,
password=self.password)
_log.debug('Gridappsd connected')
_log.debug(f"connection config is: {self.config}")
self.sim = Simulation(self.gridappsd, self.config)
_log.debug('Gridappsd adding onstart callback')
# Register for onstart callback to know if simulation has started
self.sim.add_onstart_callback(self.sim_on_start)
# Register event callbacks - on measurement, on timestep, on finish
for name, cb in self.event_callbacks.items():
if name == 'MEASUREMENT':
_log.debug('Gridappsd adding measurement callback')
self.sim.add_onmesurement_callback(cb)
elif name == 'TIMESTEP':
_log.debug('Gridappsd adding timestep callback')
self.sim.add_ontimestep_callback(cb)
elif name == 'FINISH':
_log.debug('Gridappsd adding finish callback')
self.sim.add_oncomplete_callback(cb)
# Register/Subscribe for simulation topics
for topic, cb in self.topic_callbacks:
_log.debug('Gridappsd subscribing to topics callback')
self.gridappsd.subscribe(topic, cb)
# Starting GridAppsD simulation
self.sim.start_simulation()
_log.debug(f"Gridappsd simulation id: {self.sim.simulation_id}")
except stomp.exception.NotConnectedException as ex:
_log.error("Unable to connect to GridAPPSD: {}".format(ex))
raise ex
def _main():
global appName, sim_id, feeder_mrid, gapps
if len(sys.argv) < 2 or '-help' in sys.argv:
usestr = '\nUsage: ' + sys.argv[0] + ' #nodes\n'
usestr += '''
Optional command line arguments:
-help: show this usage message
'''
print(usestr, file=sys.stderr, flush=True)
exit()
appName = sys.argv[0]
sim_config_file = './sim_starter/' + sys.argv[1] + '-config.json'
gapps = GridAPPSD()
with open(sim_config_file) as config_fp:
sim_config = json.load(config_fp)
# if there is a sys.argv[2] command line argument, that indicates not to start a simulation
if len(sys.argv) < 3:
print('SIM_STARTER initializing simulation from: ' + sim_config_file,
file=sys.stderr,
flush=True)
sim = Simulation(gapps, sim_config)
print('SIM_STARTER about to start simulation...',
file=sys.stderr,
flush=True)
sim.start_simulation(timeout=90)
sim_id = sim.simulation_id
print('SIM_STARTER simulation started with id: ' + sim_id,
file=sys.stderr,
flush=True)
print(sim_id, flush=True)
# need to dump sim_config before passing it on to downstream modules
# in order to convert single quotes to double quotes
sim_req = json.dumps(sim_config)
print(sim_req, flush=True)
gapps.disconnect()
def test_simulation_no_duplicate_measurement_timestamps(
gridappsd_client: GridAPPSD):
num_measurements = 0
timestamps = set()
def measurement(sim, timestamp, measurement):
nonlocal num_measurements
num_measurements += 1
assert timestamp not in timestamps
timestamps.add(timestamp)
gapps = gridappsd_client
sim = Simulation(gapps, base_config())
sim.add_onmeasurement_callback(measurement)
sim.start_simulation()
sim.run_loop()
# did we get a measurement?
assert num_measurements > 0
# if empty then we know the simulation did not work.
assert timestamps
"id": "gridappsd-sensor-simulator",
"user_options": {
"sensors-config": {
"_99db0dc7-ccda-4ed5-a772-a7db362e9818": {
"nominal-value": 100,
"perunit-confidence-band": 0.02,
"aggregation-interval": 5,
"perunit-drop-rate": 0.01
},
"_ee65ee31-a900-4f98-bf57-e752be924c4d": {},
"_f2673c22-654b-452a-8297-45dae11b1e14": {}
},
"random-seed": 0,
"default-aggregation-interval": 30,
"passthrough-if-not-specified": False,
"default-perunit-confidence-band": 0.01,
"default-perunit-drop-rate": 0.05
}
}]
}
gapps = GridAPPSD()
request = {
"configurationType": "CIM Dictionary",
"parameters": {
"model_id": "_C1C3E687-6FFD-C753-582B-632A27E28507"
}
}
simulation = Simulation(gapps, run_config_123)
simulation.start_simulation()
# "attribute": "Switch.open",
# "value": 0
# }
# ]
# },
# "event_type": "ScheduledCommandEvent",
# "occuredDateTime": 1570041140,
# "stopDateTime": 1570041200
# }]
# },
}
#Start the simulation....
gapps_sim = GridAPPSD()
simulation = Simulation(gapps_sim, run_config_13)
simulation.start_simulation()
simulation_id = simulation.simulation_id
print(simulation_id)
#Test the callback function
sim_output_topic = simulation_output_topic(simulation_id)
f = open('csvwritertest.csv', 'w')
gapps.subscribe(sim_output_topic, callback)
sim_log_topic = simulation_log_topic(simulation_id)
gapps.subscribe(sim_log_topic, callback2)
def _main():
def test_simulation_output(sim_config_file, sim_result_file):
sim_config_file = os.path.join(
os.path.dirname(__file__),
f"simulation_config_files/{sim_config_file}")
sim_result_file = os.path.join(
os.path.dirname(__file__),
f"simulation_baseline_files/{sim_result_file}")
assert os.path.exists(
sim_config_file
), f"File {sim_config_file} must exist to run simulation test"
# assert os.path.exists(sim_result_file), f"File {sim_result_file} must exist to run simulation test"
with startup_containers():
# Allow proven to come up
sleep(30)
starttime = int(time())
with gappsd() as gapps:
os.makedirs("/tmp/output", exist_ok=True)
with open("/tmp/output/simulation.output", 'w') as outfile:
LOGGER.info('Configuring simulation')
sim_complete = False
rcvd_measurement = False
rcvd_first_measurement = 0
are_we_paused = False
with open(sim_config_file) as fp:
LOGGER.info('Reading config')
run_config = json.load(fp)
run_config["simulation_config"]["start_time"] = str(
starttime)
sim = Simulation(gapps, run_config)
def onmeasurement(sim, timestep, measurements):
LOGGER.info('Measurement received at %s', timestep)
nonlocal rcvd_measurement
nonlocal rcvd_first_measurement
nonlocal are_we_paused
if not are_we_paused and not rcvd_first_measurement:
LOGGER.debug("Pausing sim now")
sim.pause()
are_we_paused = True
LOGGER.debug(f"ARWEPAUSED {are_we_paused}")
# Setting up so if we get another measurement wheil we
# are paused we know it
rcvd_measurement = False
if not rcvd_measurement:
print(f"A measurement happened at {timestep}")
# outfile.write(f"{timestep}|{json.dumps(measurements)}\n")
data = {"data": measurements}
outfile.write(json.dumps(data))
rcvd_measurement = True
else:
rcvd_measurement = True
rcvd_first_measurement = True
# sleep here until rcvd_measuremnt = True again
def ontimestep(sim, timestep):
print("Timestamp: {}".format(timestep))
def onfinishsimulation(sim):
nonlocal sim_complete
sim_complete = True
LOGGER.info('Simulation Complete')
LOGGER.info(f"Start time is {starttime}")
LOGGER.info('Loading config')
# tm: typo in add_onmesurement
LOGGER.info('sim.add_onmesurement_callback')
LOGGER.info('Starting sim')
sim.start_simulation()
print(sim.simulation_id)
sim.add_onmesurement_callback(onmeasurement)
sim.add_ontimestep_callback(ontimestep)
gapps.subscribe(t.simulation_log_topic(sim.simulation_id),
on_message)
sim.add_oncomplete_callback(onfinishsimulation)
LOGGER.info('sim.add_oncomplete_callback')
secs = 0
while secs < 30:
LOGGER.info(f"Sleep {secs}")
# we have to wait until the first measurement is called before
# the are_we_paused could have a chance of being set
secs += 1
sleep(1)
paused_seconds = 0
while are_we_paused:
LOGGER.info(f"PAUSED {paused_seconds}")
paused_seconds += 1
# s
if paused_seconds > 30:
LOGGER.info('Resuming simulation')
sim.resume()
LOGGER.info('Resumed simulation')
are_we_paused = False
break
sleep(1)
assert not are_we_paused, "We should have came out of the paused_seconds > 30"
while not sim_complete:
LOGGER.info('Sleeping')
sleep(5)
"simulation_name":"test9500new",
"power_flow_solver_method":"NR",
"model_creation_config":{
"load_scaling_factor":"1",
"schedule_name":"ieeezipload",
"z_fraction":"0","i_fraction":"1","p_fraction":"0","randomize_zipload_fractions":false,"use_houses":false}},
"test_config":{
"events":[{
"message":{
"forward_differences":[{
"object":"_792127B0-9B3E-43EC-9D23-FD46F5A2F20D",
"attribute":"Switch.open","value":1}],
"reverse_differences":[{"object":"_792127B0-9B3E-43EC-9D23-FD46F5A2F20D",
"attribute":"Switch.open",
"value":0
}]
},
"event_type":"ScheduledCommandEvent",
"occuredDateTime": 1373814600,
"stopDateTime": 1373817600}],
"appId":""
},
"service_configs":[]
}
"""
gapps = GridAPPSD()
simulation = Simulation(gapps, run_config_9500)
simulation.start_simulation()
for cfile in data_tests:
config_file = str(TEST_DATA_DIR.joinpath(cfile))
run_config = merge_with_simulation_config(config_file)
# from pprint import pprint
# pprint(config['gridappsd'])
with run_containers(config, stop_after=True) as containers:
# Watches the log on the container for the MYSQL data.
containers.wait_for_log_pattern("gridappsd", "MYSQL")
gappsd = GridAPPSD()
gappsd.connect()
assert gappsd.connected
time.sleep(10)
sim = Simulation(gappsd, run_config=run_config)
sim.add_onstart_callback(onstart)
sim.add_oncomplete_callback(onfinish)
sim.add_ontimestep_callback(ontimestep)
sim.add_onmesurement_callback(onmeasurement)
sim.start_simulation()
sim.pause()
gappsd.subscribe(t.simulation_output_topic(sim.simulation_id), onsimulationoutput)
gappsd.subscribe(t.application_output_topic(GRIDAPPSD_SERVICE_ID, sim.simulation_id), onsensoroutput)
time.sleep(10)
sim.resume()
sim.run_loop()
print("Shutting down")
def test_timeseries_output(sim_config_file, sim_result_file):
simulation_id = None
sim_config_file = os.path.join(os.path.dirname(__file__), f"simulation_config_files/{sim_config_file}")
sim_result_file = os.path.join(os.path.dirname(__file__), f"simulation_baseline_files/{sim_result_file}")
# sim_test_config = os.path.join(os.path.dirname(__file__), f"simulation_baseline_files/{sim_test_file}")
assert os.path.exists(sim_config_file), f"File {sim_config_file} must exist to run simulation test"
# assert os.path.exists(sim_result_file), f"File {sim_result_file} must exist to run simulation test"
with startup_containers():
with gappsd() as gapps:
os.makedirs("/tmp/output", exist_ok=True)
with open("/tmp/output/simulation.json", 'w') as outfile:
sim_complete = False
rcvd_measurement = False
def onmeasurement(sim, timestep, measurements):
nonlocal rcvd_measurement
# print(rcvd_measurement)
if not rcvd_measurement:
# print(f"A measurement happened at {timestep}")
LOGGER.info('Measurement received at %s', timestep)
outfile.write(f"{timestep}|{json.dumps(measurements)}\n")
with open(sim_config_file) as fp:
run_config = json.load(fp)
starttime = run_config["simulation_config"]["start_time"]
# run_config["simulation_config"]["start_time"] = str(starttime)
print(run_config["simulation_config"]["start_time"])
def onfinishsimulation(sim):
nonlocal sim_complete
sim_complete = True
LOGGER.info('Simulation Complete')
sim = Simulation(gapps, run_config)
sim.add_oncomplete_callback(onfinishsimulation)
sim.add_onmesurement_callback(onmeasurement)
sim.start_simulation()
LOGGER.info("Starting simulation")
print(sim.simulation_id)
with open("./simulation_config_files/weather_data.json", 'r') as g:
LOGGER.info('Querying weather data from timeseries')
query1 = json.load(g)
a = gapps.get_response(t.TIMESERIES, query1, timeout=60)
LOGGER.info('Weather data received ')
assert "Diffuse" in a["data"][0], "Weather data query does not have expected output"
LOGGER.info('Weather data query has expected output')
while not sim_complete:
sleep(5)
with open("./simulation_config_files/timeseries_query.json", 'r') as f:
query2 = json.load(f)
query2["queryFilter"]["simulation_id"] = sim.simulation_id
LOGGER.info('Querying simulation data from timeseries')
q = gapps.get_response(t.TIMESERIES, query2, timeout=300)
LOGGER.info('Simulation data received for Timeseries API')
file2 = open("./out-input.txt", 'w')
file2.write(json.dumps(q))
assert "hasSimulationMessageType" in q["data"][0], "Simulation data query does not have expected output"
LOGGER.info('Simulation data query has expected output')
with open("./simulation_config_files/sensor_query.json", 'r') as file:
sensor_query = json.load(file)
sensor_query["queryFilter"]["simulation_id"] = sim.simulation_id
LOGGER.info('Querying GridAPPS-D sensor simulator data from timeseries')
result = gapps.get_response(t.TIMESERIES, sensor_query, timeout=300)
assert "hasSimulationMessageType" in result["data"][0], "Sensor simulator data does not have expected output"
LOGGER.info('Query response received for GridAPPS-D sensor simulator data from timeseries')
def test_simulation_output(gridappsd_client, sim_config_file, sim_result_file):
global resume_msg
global pause_msg
sim_config_file = os.path.join(os.path.dirname(__file__), f"simulation_config_files/{sim_config_file}")
sim_expected_results_file = os.path.join(os.path.dirname(__file__), f"simulation_baseline_files/{sim_result_file}")
sim_actual_result_file = f"/tmp/output/{sim_result_file}"
os.makedirs(Path(sim_actual_result_file).parent, exist_ok=True)
assert os.path.exists(sim_config_file), f"File {sim_config_file} must exist to run simulation test"
# assert os.path.exists(sim_result_file), f"File {sim_result_file} must exist to run simulation test"
gapps = gridappsd_client
sleep(30)
os.makedirs("/tmp/output", exist_ok=True)
with open(sim_actual_result_file, 'w') as outfile:
LOGGER.info('Configuring simulation')
sim_complete = False
rcvd_measurement = False
rcvd_first_measurement = 0
are_we_paused = False
received_measurment_count = 0
LOGGER.info('Loading config')
with open(sim_config_file) as fp:
LOGGER.info('Reading config')
run_config = json.load(fp)
# run_config["simulation_config"]["start_time"] = str(starttime)
sim = Simulation(gapps, run_config)
def onmeasurement(sim, timestep, measurements):
LOGGER.info('Measurement received at %s', timestep)
nonlocal rcvd_measurement
nonlocal received_measurment_count
received_measurment_count += 1
try:
if not rcvd_measurement:
print(f"A measurement happened at {timestep}")
# outfile.write(f"{timestep}|{json.dumps(measurements)}\n")
data = {"data": measurements}
outfile.write(json.dumps(data))
rcvd_measurement = True
else:
rcvd_measurement = True
# rcvd_first_measurement = True
except Exception as ex:
LOGGER.error(f"An exception handled in callback {ex}")
# sleep here until rcvd_measuremnt = True again
# def ontimestep(sim, timestep):
# # print("Timestamp: {}".format(timestep))
def onfinishsimulation(sim):
nonlocal sim_complete
sim_complete = True
LOGGER.info('Simulation Complete')
LOGGER.info('sim.add_onmesurement_callback')
LOGGER.info('Starting sim')
sim.start_simulation()
print(sim.simulation_id)
sim.add_onmesurement_callback(onmeasurement)
# sim.add_ontimestep_callback(ontimestep)
gapps.subscribe(t.simulation_log_topic(sim.simulation_id), on_message)
sim.add_oncomplete_callback(onfinishsimulation)
LOGGER.info('sim.add_oncomplete_callback')
secs = 0
num_measurements_before_pause = 3
while received_measurment_count < num_measurements_before_pause:
LOGGER.info(f"Waiting for at least {num_measurements_before_pause} measurements"
f"but have {received_measurment_count} time take {secs}s ")
secs += 1
sleep(1)
LOGGER.debug("Pausing sim now")
sim.pause()
are_we_paused = True
paused_seconds = 0
while are_we_paused:
LOGGER.info(f"PAUSED {paused_seconds}")
paused_seconds += 1
if paused_seconds > 30:
LOGGER.info('Resuming simulation')
sim.resume()
LOGGER.info('Resumed simulation')
are_we_paused = False
break
sleep(1)
assert not are_we_paused, "We should have came out of the paused_seconds > 30"
assert "paused" in pause_msg, 'Pause command not called'
sleep(10)
assert "resumed" in resume_msg, 'Resume command not called'
while not sim_complete:
LOGGER.info('Sleeping')
sleep(5)
from gridappsd.topics import simulation_output_topic
from queue import Queue
import os
os.environ["GRIDAPPSD_USER"] = "******"
os.environ["GRIDAPPSD_PASSWORD"] = "******"
os.environ["GRIDAPPSD_ADDRESS"] = "gridappsd"
os.environ["GRIDAPPSD_PORT"] = "61613"
output_queue = Queue()
gapps = GridAPPSD()
with open(
"/repos/gridappsd-python/examples/default_run_simulation_ieee9500_final_config.yaml"
) as fp:
config = yaml.safe_load(fp)
sim = Simulation(gapps=gapps, run_config=config)
sim_id = sim.start_simulation()
def data_output(topic, message):
print(f"data_output {json.dumps(message)[:50]}")
output_queue.put(message)
gapps.subscribe(simulation_output_topic(sim.simulation_id), data_output)
def write_thread(queue, filename):
with open(filename, "w") as f:
while True: