def publish_record(self, topic_suffix, message):
headers = {
headers_mod.DATE: utils.format_timestamp(utils.get_aware_utc_now())
}
message["TimeStamp"] = utils.format_timestamp(self.current_datetime)
topic = "/".join([self.record_topic, topic_suffix])
self.vip.pubsub.publish("pubsub", topic, headers, message).get()
def schedule_run(self,
cur_exp_time,
cur_analysis_time,
start_of_cycle=False):
# 191218DJH: The logic in this section should be REPLACED by the new market state machine. See method go().
"""
Run when first start or run at beginning of hour
:return:
"""
# Balance
market = self.markets[0] # Assume only 1 TNS market per node
# market.balance(self)
# market.events(self)
# self.campus.prep_transactive_signal(market, self)
# self.campus.send_transactive_signal(self, self.city_supply_topic,
# start_of_cycle=start_of_cycle)
# Schedule to run next hour with start_of_cycle = True
cur_exp_time = parser.parse(cur_exp_time)
cur_analysis_time = parser.parse(cur_analysis_time)
next_exp_time, next_analysis_time = self.get_next_exp_time(
cur_exp_time, cur_analysis_time)
self.core.schedule(next_exp_time,
self.schedule_run,
format_timestamp(next_exp_time),
format_timestamp(next_analysis_time),
start_of_cycle=True)
def schedule_run(self,
cur_exp_time,
cur_analysis_time,
start_of_cycle=False):
"""
Run when first start or run at beginning of hour
:return:
"""
# Balance
market = self.markets[0] # Assume only 1 TNS market per node
market.balance(self)
prices = market.marginalPrices
prices = prices[-25:]
prices = [x.value for x in prices]
_time = format_timestamp(Timer.get_cur_time())
self.vip.pubsub.publish(peer='pubsub',
topic=self.price_topic,
message={
'prices': prices,
'current_time': _time
})
self.campus.model.prep_transactive_signal(market, self)
self.campus.model.send_transactive_signal(
self, self.city_supply_topic, start_of_cycle=start_of_cycle)
# Schedule to run next hour with start_of_cycle = True
cur_exp_time = parser.parse(cur_exp_time)
cur_analysis_time = parser.parse(cur_analysis_time)
next_exp_time, next_analysis_time = self.get_next_exp_time(
cur_exp_time, cur_analysis_time)
self.core.schedule(next_exp_time,
self.schedule_run,
format_timestamp(next_exp_time),
format_timestamp(next_analysis_time),
start_of_cycle=True)
def schedule_for_actuator(self, groups):
# create start and end timestamps
_now = get_aware_utc_now()
str_start = format_timestamp(_now)
_end = _now + td(seconds=10)
str_end = format_timestamp(_end)
schedule_request = []
# wrap the topic and timestamps up in a list and add it to the schedules list
_log.debug(
f'[Conninuous Roller Agent INFO] - ACTUATOR DEBUG GROUP IS {groups}'
)
for group in groups:
for key, value in self.nested_group_map[group].items():
if key not in ('score', 'shed_count'):
topic_sched_group_l1n = '/'.join(
[self.building_topic, str(value)])
schedule_request.append(
[topic_sched_group_l1n, str_start, str_end])
# send the request to the actuator
result = self.vip.rpc.call('platform.actuator', 'request_new_schedule',
self.core.identity, 'my_schedule', 'HIGH',
schedule_request).get(timeout=90)
_log.debug(
f'[Conninuous Roller Agent INFO] - ACTUATOR SCHEDULE EVENT SUCESS {result}'
)
def publish_data(self, topic, value, time_stamp):
headers = {headers_mod.DATE: format_timestamp(get_aware_utc_now())}
message = {"Value": value}
message["TimeStamp"] = format_timestamp(time_stamp)
topic = "/".join([self.logging_topic, topic])
_log.debug("LOGGING {} - {} - {}".format(topic, value, time_stamp))
self.parent.vip.pubsub.publish("pubsub", topic, headers, message).get()
def schedule_run(self,
cur_exp_time,
cur_analysis_time,
start_of_cycle=False):
"""
Run when first start or run at beginning of hour
:return:
"""
# Balance
market = self.markets[0] # Assume only 1 TNS market per node
market.balance(self)
self.campus.model.prep_transactive_signal(market, self)
self.campus.model.send_transactive_signal(
self, self.city_supply_topic, start_of_cycle=start_of_cycle)
# Schedule to run next hour with start_of_cycle = True
cur_exp_time = parser.parse(cur_exp_time)
cur_analysis_time = parser.parse(cur_analysis_time)
next_exp_time, next_analysis_time = self.get_next_exp_time(
cur_exp_time, cur_analysis_time)
self.core.schedule(next_exp_time,
self.schedule_run,
format_timestamp(next_exp_time),
format_timestamp(next_analysis_time),
start_of_cycle=True)
def publish_target_info_pgne(self, cur_analysis_time_utc):
cur_analysis_time_utc = parser.parse(cur_analysis_time_utc)
target_messages = self.get_target_info_pgne(format_timestamp(cur_analysis_time_utc), 'UTC')
if len(target_messages) > 0:
target_topic = '/'.join(['analysis', 'target_agent', self.site, self.building, 'goal'])
for target_message in target_messages:
headers = {'Date': format_timestamp(get_aware_utc_now())}
self.vip.pubsub.publish(
'pubsub', target_topic, headers, target_message).get(timeout=15)
_log.debug("TargetAgent {topic}: {value}".format(
topic=target_topic,
value=target_message))
gevent.sleep(2)
# Schedule next run at min 30 of next hour only if current min >= 30
one_hour = timedelta(hours=1)
cur_min = cur_analysis_time_utc.minute
next_analysis_time = cur_analysis_time_utc.replace(minute=30,
second=0,
microsecond=0)
if cur_min >= 30:
next_analysis_time += one_hour
next_run_time = next_analysis_time
if self.dr_mode == 'dev':
next_run_time = get_aware_utc_now() + timedelta(seconds=15)
if self.dr_mode != 'manual':
self.core.schedule(next_run_time, self.publish_target_info,
format_timestamp(next_analysis_time))
def on_ilc_start(self, peer, sender, bus, topic, headers, message):
cur_time = self.local_tz.localize(datetime.now())
cur_time_utc = cur_time.astimezone(pytz.utc)
one_hour = timedelta(hours=1)
prev_time_utc = cur_time_utc - one_hour
self.publish_target_info(format_timestamp(prev_time_utc))
self.publish_target_info(format_timestamp(cur_time_utc))
def get_forecast_weatherservice(self, mkt):
"""
Uses VOLTTRON DarkSky weather agent running on local or remote platform to
get 24 hour forecast for weather data.
:param mkt:
:return:
"""
weather_results = None
weather_data = None
try:
result = self.parent.vip.rpc.call(self.weather_vip,
"get_hourly_forecast",
self.location,
external_platform=self.remote_platform).get(timeout=15)
weather_results = result[0]["weather_results"]
except (gevent.Timeout, RemoteError) as ex:
_log.warning("RPC call to {} failed for weather forecast: {}".format(self.weather_vip, ex))
if weather_results is not None:
try:
weather_data = [[parser.parse(oat[0]).astimezone(self.localtz), oat[1][self.oat_point_name]] for oat in weather_results]
weather_data = [[oat[0].replace(tzinfo=None), oat[1]] for oat in weather_data]
except KeyError:
if not self.predictedValues:
raise Exception("Measurement Point Name is not correct")
# How do we deal with never getting weather information? Exit?
except Exception as ex:
if not self.predictedValues:
raise Exception("Exception {} processing weather data.".format(ex))
# Copy weather data to predictedValues
if weather_data is not None:
self.predictedValues = []
for ti in mkt.timeIntervals:
# Find item which has the same timestamp as ti.timeStamp
start_time = ti.startTime.replace(minute=0)
items = [x[1] for x in weather_data if x[0] == start_time]
# Create interval value and add it to predicted values
if items:
temp = items[0]
interval_value = IntervalValue(self, ti, mkt, MeasurementType.PredictedValue, temp)
self.predictedValues.append(interval_value)
elif self.predictedValues:
hour_gap = mkt.timeIntervals[0].startTime - self.predictedValues[0].timeInterval.startTime
max_hour_gap = timedelta(hours=4)
if hour_gap > max_hour_gap:
self.predictedValues = []
raise Exception('No weather data for time: {}'.format(utils.format_timestamp(mkt.timeIntervals[0].startTime)))
else:
predictedValues = []
for i in range(1, len(mkt.timeIntervals)):
interval_value = IntervalValue(self, mkt.timeIntervals[i], mkt, MeasurementType.PredictedValue, self.predictedValues[i-1].value)
predictedValues.append(interval_value)
self.predictedValues = predictedValues
else:
raise Exception(
'No weather data for time: {}'.format(utils.format_timestamp(mkt.timeIntervals[0].startTime)))
def test_record_topic(publish_agent, query_agent):
"""
Test if record topic message is getting forwarded to historian running on
another instance.
:param publish_agent: Fake agent used to publish messages to bus in
volttron_instance1. Calling this fixture makes sure all the dependant
fixtures are called to setup and start volttron_instance1 and forwareder
agent and returns the instance of fake agent to publish
:param query_agent: Fake agent used to query sqlhistorian in
volttron_instance2. Calling this fixture makes sure all the dependant
fixtures are called to setup and start volttron_instance2 and sqlhistorian
agent and returns the instance of a fake agent to query the historian
"""
# Create timestamp
print("\n** test_record_topic **")
now = utils.format_timestamp(datetime.utcnow())
print("now is ", now)
headers = {
headers_mod.DATE: now,
headers_mod.TIMESTAMP: now
}
# Publish messages
publish(publish_agent, topics.RECORD, headers, 1)
# sleep so that records gets inserted with unique timestamp
gevent.sleep(0.5)
time2 = utils.format_timestamp(datetime.utcnow())
headers = {
headers_mod.DATE: time2,
headers_mod.TIMESTAMP: time2
}
publish(publish_agent, topics.RECORD, headers, 'value0')
# sleep so that records gets inserted with unique timestamp
gevent.sleep(0.5)
time3 = utils.format_timestamp(datetime.utcnow())
headers = {
headers_mod.DATE: time3,
headers_mod.TIMESTAMP: time3
}
publish(publish_agent, topics.RECORD, headers, {'key': 'value'})
gevent.sleep(0.5)
result = query_agent.vip.rpc.call('platform.historian',
'query',
topic=topics.RECORD,
start=now,
order="FIRST_TO_LAST").get(timeout=10)
print('Query Result', result)
assert (len(result['values']) == 3)
assert (result['values'][0][1] == 1)
assert (result['values'][1][1] == 'value0')
assert (result['values'][2][1] == {'key': 'value'})
assert result['values'][2][0] == time3 + '+00:00'
def validate_cache_result_forecast(locations, api_result, cache_result):
for result in api_result:
time_in_results = False
for cr in cache_result:
if utils.format_timestamp(cr[2]) == result["generation_time"]:
for record in result["weather_results"]:
if utils.format_timestamp(cr[3]).startswith(record[0]):
time_in_results = True
assert jsonapi.loads(cr[1]) in locations
assert record[1] == jsonapi.loads(cr[4])
break
assert time_in_results
def validate_cache_result_forecast(locations, api_result, cache_result):
for result in api_result:
time_in_results = False
for cr in cache_result:
if utils.format_timestamp(cr[2]) == result["generation_time"]:
for record in result["weather_results"]:
if utils.format_timestamp(cr[3]).startswith(record[0]):
time_in_results = True
assert ujson.loads(cr[1]) in locations
assert record[1] == ujson.loads(cr[4])
break
assert time_in_results
def train_components(self, now=None):
"""Gather training parameters, query historian for training data,
then pass data to model for further processing
:param now: timestamp for simulation mode
"""
if self.next_training is None:
return
if now is None:
# We are being driven by a greenlet in realtime
# Always run when we are told
now = utils.get_aware_utc_now()
else:
# We are being driven by simulation
# Return if we are not ready to run again
if self.next_training > now:
return
# Train both forecast and component models
for forecast_models in (False, True):
results = {}
all_parameters = self.model.get_training_parameters(
forecast_models)
for name, parameters in all_parameters.items():
window, sources = parameters
end = now
start = end - timedelta(days=window)
training_data = {}
for topic in sources:
value = self.vip.rpc.call(
self.historian_vip_id,
"query",
topic,
utils.format_timestamp(start),
utils.format_timestamp(end),
).get(timeout=4)
training_data[topic] = value
results[name] = training_data
self.model.apply_all_training_data(results, forecast_models)
try:
self.next_training = next(self.training_schedule)
LOG.info("Next training scheduled for {}"
"".format(self.next_training))
except StopIteration:
self.next_training = None
self.historian_training = False
LOG.info("No more trainings scheduled")
def actuator_request(self, site, command_equip):
"""
Calls the actuator"s request_new_schedule method to get
device schedule
:param command_equip: contains the names of the devices
that will be scheduled with the ActuatorAgent.
:type: dict or list
:returns: Return result from request_new_schedule method
and True or False for error in scheduling device.
:rtype: boolean
:Return Values:
request_error = True/False
warning:: Calling without previously scheduling a device and not within
the time allotted will raise a LockError"""
_now = get_aware_utc_now()
str_now = format_timestamp(_now)
_end = _now + td(minutes=self.device_lock_duration)
str_end = format_timestamp(_end)
for device in command_equip:
actuation_device = site['base_actuator_path'](unit=device,
point="")
schedule_request = [[actuation_device, str_now, str_end]]
try:
_log.info("Make Request {} for start {} and end {}".format(
actuation_device, str_now, str_end))
result = self.actuation_vip.call(
"platform.actuator", "request_new_schedule", "rcx",
actuation_device, "HIGH",
schedule_request).get(timeout=15)
except RemoteError as ex:
_log.warning(
"Failed to schedule device {} (RemoteError): {}".
format(device, str(ex)))
request_error = True
if result["result"] == "FAILURE":
if result["info"] == "TASK_ID_ALREADY_EXISTS":
_log.info("Task to schedule device already exists " +
device)
request_error = False
else:
_log.warning(
"Failed to schedule device (unavailable) " +
device)
request_error = True
else:
request_error = False
return request_error
def report(self):
""" Report result of diagnostic analysis and publish
to the VOLTTRON message bus.
:return: None
"""
# Multiple control steps and analysis can occur for each diagnostic
# if self.fault_condition == all then all steps must have a fault
# condition for a fault to be reported.
self.headers = {
"Date": format_timestamp(get_aware_utc_now()),
"Timestamp": format_timestamp(get_aware_utc_now())
}
analysis = {}
if -1 in self.evaluations:
LOG.debug("Diagnostic %s resulted in inconclusive result",
self.name)
analysis = {"result": -1}
for publish_topic in self.analysis_topic:
self.vip.pubsub.publish("pubsub",
self.analysis_topic,
headers=self.headers,
message=analysis)
self.evaluations = []
return
if self.fault_condition == "any":
if False in self.evaluations:
LOG.debug("%s - no fault detected", self.name)
analysis = {"result": self.non_fault_code}
else:
LOG.debug("%s - fault detected", self.name)
analysis = {"result": self.fault_code}
# Multiple control steps and analysis can occur for each diagnostic
# if self.fault_condition == "any"" then any step where a
# fault condition is detected will lead to reporting a fault.
else:
if True in self.evaluations:
LOG.debug("%s - fault detected", self.name)
analysis = {"result": self.fault_code}
else:
LOG.debug("%s - no fault detected", self.name)
analysis = {"result": self.non_fault_code}
# Reinitialize evaluations list for use in next diagnostic run.
for publish_topic in self.analysis_topic:
self.vip.pubsub.publish("pubsub",
publish_topic,
headers=self.headers,
message=analysis)
self.evaluations = []
def actuator_request(self, score_order):
'''request access to devices.'''
_now = get_aware_utc_now()
str_now = format_timestamp(_now)
_end = _now + longest_possible_curtail + actuator_schedule_buffer
str_end = format_timestamp(_end)
ctrl_dev = []
already_handled = dict(
(device, True) for device in self.scheduled_devices)
for item in score_order:
device, point = item
_log.debug('Reserving device: ' + device)
if device in already_handled:
if already_handled[device]:
_log.debug(
'Skipping reserve device (previously reserved): ' +
device)
ctrl_dev.append(item)
continue
curtailed_device = base_rpc_path(unit=device, point='')
schedule_request = [[curtailed_device, str_now, str_end]]
try:
if self.kill_signal_recieved:
break
result = self.vip.rpc.call('platform.actuator',
'request_new_schedule',
agent_id, device, 'HIGH',
schedule_request).get(timeout=5)
except RemoteError as ex:
_log.warning(
'Failed to schedule device {} (RemoteError): {}'.
format(device, str(ex)))
continue
if result['result'] == 'FAILURE':
_log.warn('Failed to schedule device (unavailable) ' +
device)
already_handled[device] = False
else:
already_handled[device] = True
self.scheduled_devices.add(device)
ctrl_dev.append(item)
return ctrl_dev
def update_status(self, status, context=None):
"""
Updates the internal state of the `Status` object.
This method will throw errors if the context is not serializable or
if the status parameter is not within the ACCEPTABLE_STATUS tuple.
:param status:
:param context:
:return:
"""
if status not in ACCEPTABLE_STATUS:
raise ValueError('Invalid status value {}'.format(status))
try:
jsonapi.dumps(context)
except TypeError:
raise ValueError('Context must be JSON serializable.')
status_changed = status != self._status
self._status = status
self._context = context
self._last_updated = format_timestamp(get_aware_utc_now())
if status_changed and self._status_changed_callback:
print(self._status_changed_callback())
def periodic_read(self, now):
# we not use self.core.schedule to prevent drift.
next_scrape_time = now + datetime.timedelta(seconds=self.interval)
# Sanity check now.
# This is specifically for when this is running in a VM that gets
# suspended and then resumed.
# If we don't make this check a resumed VM will publish one event
# per minute of
# time the VM was suspended for.
test_now = utils.get_aware_utc_now()
if test_now - next_scrape_time > datetime.timedelta(seconds=self.interval):
next_scrape_time = self.find_starting_datetime(test_now)
self.periodic_read_event = self.core.schedule(next_scrape_time, self.periodic_read, next_scrape_time)
_log.debug("scraping device: " + self.device_name)
try:
results = self.interface.scrape_all()
except Exception as ex:
_log.error('Failed to scrape ' + self.device_name + ': ' + str(ex))
return
if results:
utcnow_string = utils.format_timestamp(utils.get_aware_utc_now())
headers = {headers_mod.DATE: utcnow_string,
headers_mod.TIMESTAMP: utcnow_string, }
for point, value in results.iteritems():
depth_first_topic, breadth_first_topic = self.get_paths_for_point(point)
message = [value, self.meta_data[point]]
self._publish_wrapper(depth_first_topic, headers=headers, message=message)
self._publish_wrapper(breadth_first_topic, headers=headers, message=message)
message = [results, self.meta_data]
self._publish_wrapper(self.all_path_depth, headers=headers, message=message)
self._publish_wrapper(self.all_path_breadth, headers=headers, message=message)
def new_data(self, peer, sender, bus, topic, headers, message):
"""
Call back method for curtailable device data subscription.
:param peer:
:param sender:
:param bus:
:param topic:
:param headers:
:param message:
:return:
"""
if self.kill_signal_received:
return
_log.info("Data Received for {}".format(topic))
# topic of form: devices/campus/building/device
device_name = self.device_topic_map[topic]
data = message[0]
meta = message[1]
now = parser.parse(headers["Date"])
current_time_str = format_timestamp(now)
parsed_data = parse_sympy(data)
subdevices = self.curtailment.get_device(device_name).command_status.keys()
for subdevice in subdevices:
status = self.curtailment.get_device(device_name).currently_curtailed[subdevice]
_log.debug("Device: {} -- subdevice: {} -- status: {}".format(device_name, subdevice, status))
self.criteria.get_device(device_name[0]).criteria_status(subdevice, status)
self.criteria.get_device(device_name[0]).ingest_data(now, parsed_data)
self.curtailment.get_device(device_name).ingest_data(parsed_data)
self.create_device_status_publish(current_time_str, device_name, data, topic, meta)
def check_load(self, bldg_power, current_time):
"""
Check whole building power and if the value is above the
the demand limit (demand_limit) then initiate the ILC (AHP)
sequence.
:param bldg_power:
:param current_time:
:return:
"""
_log.debug("Checking building load.")
if self.demand_limit is None:
result = "Demand goal has not been set. Current load: ({load}) kW.".format(load=bldg_power)
else:
result = "Current load: ({load}) kW is below demand limit of {limit} kW.".format(load=bldg_power,
limit=self.demand_limit)
if self.demand_limit is not None and bldg_power > self.demand_limit:
result = "Current load of {} kW exceeds demand limit of {} kW.".format(bldg_power, self.demand_limit)
scored_devices = self.criteria.get_score_order()
on_devices = self.curtailment.get_on_devices()
score_order = [device for scored in scored_devices for device in on_devices if scored in [(device[0], device[1])]]
_log.debug("Scored devices: {}".format(scored_devices))
_log.debug("On devices: {}".format(on_devices))
_log.debug("Scored and on devices: {}".format(score_order))
if not score_order:
_log.info("All devices are off, nothing to curtail.")
return
self.device_group_size = None
scored_devices = self.actuator_request(score_order)
self.curtail(scored_devices, bldg_power, current_time)
self.create_application_status(format_timestamp(current_time), result)
def on_polling(self):
if self.zip is None and (self.region is None or self.city is None):
return
kwargs = {}
if self.zip is not None:
kwargs['zip'] = self.zip
topic = 'weather2/polling/current/ZIP/{zip}/all'.format(zip=self.zip)
else:
kwargs['region'] = self.region
kwargs['city'] = self.city
topic = 'weather2/polling/current/{region}/{city}/all'.format(
region=self.region,
city=self.city
)
wu_resp = self.wu_service.current(**kwargs)
publish_items = self.build_resp_current(wu_resp)
if len(publish_items) > 0:
headers = {
HEADER_NAME_DATE: format_timestamp(utils.get_aware_utc_now()),
HEADER_NAME_CONTENT_TYPE: headers_mod.CONTENT_TYPE.JSON
}
self.vip.pubsub.publish(peer='pubsub',
topic=topic,
message=publish_items,
headers=headers)
_log.debug(publish_items)
def _on_platform_log_message(self, peer, sender, bus, topic, headers,
message):
""" Receive message from a registered platform
This method is called with stats from the registered platform agents.
"""
_log.debug('Got topic: {}'.format(topic))
_log.debug('Got message: {}'.format(message))
topicsplit = topic.split('/')
platform_uuid = topicsplit[2]
# For devices we use everything between devices/../all as a unique
# key for determining the last time it was seen.
key = '/'.join(topicsplit[:])
_log.debug("key is: {}".format(key))
uuid = topicsplit[2]
point_list = []
for point, item in message.iteritems():
point_list.append(point)
stats = {
'topic': key,
'points': point_list,
'last_published_utc': format_timestamp(get_aware_utc_now())
}
self._registry.update_performance(platform_uuid=platform_uuid,
performance=stats)
def publish_all(self, observation, topic_prefix="weather", headers={}):
utcnow = utils.get_aware_utc_now()
utcnow_string = utils.format_timestamp(utcnow)
headers.update({HEADER_NAME_DATE: utcnow_string,
headers_mod.TIMESTAMP: utcnow_string})
self.publish_subtopic(self.build_dictionary(observation),
topic_prefix, headers)
def read_callback(self, results):
# XXX: Does a warning need to be printed?
if results is None:
return
now = utils.format_timestamp(datetime.utcnow())
headers = {
headers_mod.CONTENT_TYPE: headers_mod.CONTENT_TYPE.JSON,
headers_mod.DATE: now,
headers_mod.TIMESTAMP: now
}
for point, value in results.iteritems():
if isinstance(value, bool):
value = int(value)
self.add('/'+point, value)
try:
for point, value in results.iteritems():
if isinstance(value, bool):
value = int(value)
depth, breadth = self.get_paths_for_point('/'+point)
self.publish_json(depth, headers, value, self.meta_data[point], flags=zmq.NOBLOCK)
self.publish_json(breadth, headers, value, self.meta_data[point], flags=zmq.NOBLOCK)
self.publish_json(self.all_path_depth, headers, results, self.meta_data, flags=zmq.NOBLOCK)
self.publish_json(self.all_path_breadth, headers, results, self.meta_data, flags=zmq.NOBLOCK)
except zmq.error.Again:
print ("Warning: platform not running, topics not published. (Data to smap historian is unaffected by this warning)")
def clear_market(self):
price = None
quantity = None
error_code = None
error_message = None
aux = {}
if (self.state in [ACCEPT_ALL_OFFERS, ACCEPT_BUY_OFFERS, ACCEPT_SELL_OFFERS]):
error_code = SHORT_OFFERS
error_message = 'The market {} failed to recieve all the expected offers. The state is {}.'.format(self.market_name, self.state)
elif (self.state != MARKET_DONE):
error_code = BAD_STATE
error_message = 'Programming error in Market class. State of {} and clear market signal arrived. This represents a logic error.'.format(self.state)
else:
if not self.has_market_formed():
error_code = NOT_FORMED
error_message = 'The market {} has not received a buy and a sell reservation.'.format(self.market_name)
else:
quantity, price, aux = self.offers.settle()
if price is None:
error_code = NO_INTERSECT
error_message = "Error: The supply and demand curves do not intersect. The market {} failed to clear.".format(self.market_name)
_log.info("Clearing price for Market: {} Price: {} Qty: {}".format(self.market_name, price, quantity))
timestamp = self._get_time()
timestamp_string = utils.format_timestamp(timestamp)
self.publish(peer='pubsub',
topic=MARKET_CLEAR,
message=[timestamp_string, self.market_name, quantity, price])
self.publish(peer='pubsub',
topic=MARKET_RECORD,
message=[timestamp_string, self.market_name, quantity, price])
if error_message is not None:
self.publish(peer='pubsub',
topic=MARKET_ERROR,
message=[timestamp_string, self.market_name, error_code, error_message, aux])
def _on_device_message(self, peer, sender, bus, topic, headers, message):
# only deal with agents that have not been forwarded.
if headers.get('X-Forwarded', None):
return
# only listen to the ending all message.
if not re.match('.*/all$', topic):
return
topicsplit = topic.split('/')
# For devices we use everything between devices/../all as a unique
# key for determining the last time it was seen.
key = '/'.join(topicsplit[1: -1])
anon_topic = self._topic_replace_map[key]
if not anon_topic:
anon_topic = key
for sr in self._topic_replace_list:
_log.debug(
'anon replacing {}->{}'.format(sr['from'], sr['to']))
anon_topic = anon_topic.replace(sr['from'],
sr['to'])
_log.debug('anon after replacing {}'.format(anon_topic))
_log.debug('Anon topic is: {}'.format(anon_topic))
self._topic_replace_map[key] = anon_topic
_log.debug('DEVICES ON PLATFORM ARE: {}'.format(self._devices))
self._devices[anon_topic] = {
'points': message[0].keys(),
'last_published_utc': format_timestamp(get_aware_utc_now())
}
def new_data(self, peer, sender, bus, topic, headers, message):
"""
Call back method for curtailable device data subscription.
:param peer:
:param sender:
:param bus:
:param topic:
:param headers:
:param message:
:return:
"""
if self.kill_signal_received:
return
_log.info("Data Received for {}".format(topic))
# topic of form: devices/campus/building/device
device_name = self.device_topic_map[topic]
data = message[0]
meta = message[1]
now = parser.parse(headers["Date"])
current_time_str = format_timestamp(now)
parsed_data = parse_sympy(data)
subdevices = self.curtailment.get_device(device_name).command_status.keys()
for subdevice in subdevices:
status = self.curtailment.get_device(device_name).currently_curtailed[subdevice]
_log.debug("Device: {} -- subdevice: {} -- status: {}".format(device_name, subdevice, status))
self.criteria.get_device(device_name[0]).criteria_status(subdevice, status)
self.criteria.get_device(device_name[0]).ingest_data(now, parsed_data)
self.curtailment.get_device(device_name).ingest_data(parsed_data)
self.create_device_status_publish(current_time_str, device_name, data, topic, meta)
def check_load(self, bldg_power, current_time):
"""
Check whole building power and if the value is above the
the demand limit (demand_limit) then initiate the ILC (AHP)
sequence.
:param bldg_power:
:param current_time:
:return:
"""
_log.debug("Checking building load.")
if self.demand_limit is None:
result = "Demand goal has not been set. Current load: ({load}) kW.".format(load=bldg_power)
else:
result = "Current load: ({load}) kW is below demand limit of {limit} kW.".format(load=bldg_power,
limit=self.demand_limit)
if self.demand_limit is not None and bldg_power > self.demand_limit:
result = "Current load of {} kW exceeds demand limit of {} kW.".format(bldg_power, self.demand_limit)
scored_devices = self.criteria.get_score_order()
on_devices = self.curtailment.get_on_devices()
score_order = [device for scored in scored_devices for device in on_devices if scored in [(device[0], device[1])]]
_log.debug("Scored devices: {}".format(scored_devices))
_log.debug("On devices: {}".format(on_devices))
_log.debug("Scored and on devices: {}".format(score_order))
if not score_order:
_log.info("All devices are off, nothing to curtail.")
return
self.device_group_size = None
scored_devices = self.actuator_request(score_order)
self.curtail(scored_devices, bldg_power, current_time)
self.create_application_status(format_timestamp(current_time), result)
def update_values(self, data, topic_name, start, end, values):
if start.tzinfo:
data[0] = data[0].replace(tzinfo=tzutc())
if data[0] >= start and data[0] < end:
result_value = self.json_string_to_dict(data[1])
values[topic_name].append(
(utils.format_timestamp(data[0]), result_value))
def on_heartbeat_topic(self, peer, sender, bus, topic, headers, message):
global counter
# Test various RPC calls to the Chargepoint driver.
counter += 1
if counter > 1:
# result = self.set_chargepoint_point('shedState', 0)
# result = self.get_chargepoint_point('stationMacAddr')
# result = self.get_chargepoint_point('Lat')
# result = self.get_chargepoint_point('Long')
# result = self.set_chargepoint_point('allowedLoad', 10)
# result = self.set_chargepoint_point('percentShed', 50)
# result = self.set_chargepoint_point('clearAlarms', True)
# result = self.get_chargepoint_point('alarmType')
# result = self.get_chargepoint_point('sessionID')
result = self.get_chargepoint_point('Status')
# result = self.get_chargepoint_point('stationRightsProfile')
now = utils.format_timestamp(datetime.datetime.now())
# Also publish a test pub/sub message just for kicks.
result = self.publish_message('test_topic/test_subtopic',
{
headers_mod.DATE: now,
headers_mod.TIMESTAMP: now
},
[{'property_1': 1, 'property_2': 2}, {'property_3': 3, 'property_4': 4}])
counter = 0
def publish_baseline(self, df, cur_time):
"""
This method is obsolete. Keep here for reference only.
"""
topic_tmpl = "analysis/PGnE/{campus}/{building}/"
topic_prefix = topic_tmpl.format(campus=self.site,
building=self.building)
headers = {'Date': format_timestamp(cur_time)}
last_idx = len(df.index)-1
sec_last_idx = last_idx - 1
#avg 10 day
topic1 = topic_prefix + "avg10"
value1 = df['pow_avg'][last_idx]
#adj avg 10 day
topic2 = topic_prefix + "adj_avg10"
value_hr1 = df['pow_adj_avg'][sec_last_idx]
value_hr2 = df['pow_adj_avg'][last_idx]
#avg 5 hottest in 10 day
topic3 = topic_prefix + "hot5_avg10"
value3 = df['hot5_pow_avg'][last_idx]
#adj avg 5 hottest in 10 day
topic4 = topic_prefix + "hot5_adj_avg10"
value4 = df['hot5_pow_adj_avg'][last_idx]
#publish to message bus: only 10 day adjustment
meta = {'type': 'float', 'tz': self.tz, 'units': 'kW'}
msg = [{
"value_hr1": value_hr1,
"value_hr2": value_hr2
}, {
"value_hr1": meta,
"value_hr2": meta
}]
self.vip.pubsub.publish(
'pubsub', topic2, headers, msg).get(timeout=10)
def _set_override_off(self, pattern):
pattern = pattern.lower()
# If pattern exactly matches
if pattern in self._override_patterns:
self._override_patterns.discard(pattern)
# Cancel any pending override events
self._cancel_override_events(pattern)
self._override_devices.clear()
patterns = dict()
# Build override devices list again
for pat in self._override_patterns:
for device in self.instances:
device = device.lower()
if fnmatch.fnmatch(device, pat):
self._override_devices.add(device)
if self._override_interval_events[pat] is None:
patterns[pat] = str(0.0)
else:
evt, end_time = self._override_interval_events[pat]
patterns[pat] = utils.format_timestamp(end_time)
self.vip.config.set("override_patterns", jsonapi.dumps(patterns))
else:
_log.error("Override Pattern did not match!")
raise OverrideError(
"Pattern {} does not exist in list of override patterns".
format(pattern))
def update_status(self, status, context=None):
"""
Updates the internal state of the `Status` object.
This method will throw errors if the context is not serializable or
if the status parameter is not within the ACCEPTABLE_STATUS tuple.
:param status:
:param context:
:return:
"""
if status not in ACCEPTABLE_STATUS:
raise ValueError('Invalid status value {}'.format(status))
try:
jsonapi.dumps(context)
except TypeError:
raise ValueError('Context must be JSON serializable.')
status_changed = status != self._status
self._status = status
self._context = context
self._last_updated = format_timestamp(get_aware_utc_now())
if status_changed and self._status_changed_callback:
self._status_changed_callback()
def on_heartbeat_topic(self, peer, sender, bus, topic, headers, message):
global counter
# Test various RPC calls to the Chargepoint driver.
counter += 1
if counter > 1:
# result = self.set_chargepoint_point('shedState', 0)
# result = self.get_chargepoint_point('stationMacAddr')
# result = self.get_chargepoint_point('Lat')
# result = self.get_chargepoint_point('Long')
# result = self.set_chargepoint_point('allowedLoad', 10)
# result = self.set_chargepoint_point('percentShed', 50)
# result = self.set_chargepoint_point('clearAlarms', True)
# result = self.get_chargepoint_point('alarmType')
# result = self.get_chargepoint_point('sessionID')
result = self.get_chargepoint_point('Status')
# result = self.get_chargepoint_point('stationRightsProfile')
now = utils.format_timestamp(datetime.datetime.now())
# Also publish a test pub/sub message just for kicks.
result = self.publish_message('test_topic/test_subtopic', {
headers_mod.DATE: now,
headers_mod.TIMESTAMP: now
}, [{
'property_1': 1,
'property_2': 2
}, {
'property_3': 3,
'property_4': 4
}])
counter = 0
def process_point(self, now, topic, min_value=None, max_value=None, output_topic=None,
aggregate_method=None):
"""
This is where the magic happens.
Introducing new or changing methods to clean/massage/introduce new data go here.
Currently this function republishes the most recent data as is if it
is not too old and falls within the min and max settings.
If the most recent value is stale it uses an average of the last 30 days.
New keyword arguments to this function can be added as needed and will be passed
straight from the arguments to a topic in the configuration file.
:param now:
:param topic:
:param min_value:
:param max_value:
:param output_topic:
:param aggregate_method:
"""
_log.debug("Processing topic: {}".format(topic))
if output_topic is None:
_log.error("No output topic for {}".format(topic))
return
# Query the data from the historian
results = self.vip.rpc.call("platform.historian", "query", topic, "now -1d").get(timeout=5.0)
values = results["values"]
if not values:
_log.error("No values for {}".format(topic))
return
last_timestamp, value = values[-1]
last_timestamp = utils.parse_timestamp_string(last_timestamp)
if now - last_timestamp > timedelta(seconds=self.period):
_log.warning("Data used for {} is stale".format(topic))
if aggregate_method == "avg":
results = self.vip.rpc.call("platform.historian", "query", topic, "now -30d").get(timeout=5.0)
values = results["values"]
average = sum(x[1] for x in values)
average /= len(values)
value = average
# Do something here to fake a better value.
# Make sure the value is within bounds.
if min_value is not None:
value = max(min_value, value)
if max_value is not None:
value = min(max_value, value)
#Publish the result.
self.vip.pubsub.publish("pubsub", output_topic,
headers={headers.TIMESTAMP: utils.format_timestamp(now), "source": topic},
message=value)
def publish(self, device_topic):
headers = {'Date': utils.format_timestamp(datetime.utcnow()\
.astimezone(dateutil.tz.gettz(self.timezone)))}
message = [{
'excess_operation': bool(self.excess_operation),
'device_status': bool(self.device_status),
'device_true_time': int(self.device_true_time)
}, {
'excess_operation': {
'units': 'None',
'tz': 'UTC',
'data_type': 'bool'
},
'device_status': {
'units': 'None',
'tz': 'UTC',
'data_type': 'bool'
},
'device_true_time': {
'units': 'seconds',
'tz': 'UTC',
'data_type': 'integer'
}
}]
device_topic = device_topic.replace("all", "report/all")
try:
self.vip.pubsub.publish(peer='pubsub',
topic=device_topic,
message=message,
headers=headers)
except Exception as e:
_log.error("In Publish: {}".format(str(e)))
def _on_device_message(self, peer, sender, bus, topic, headers, message):
# only deal with agents that have not been forwarded.
if headers.get('X-Forwarded', None):
return
# only listen to the ending all message.
if not re.match('.*/all$', topic):
return
topicsplit = topic.split('/')
# For devices we use everything between devices/../all as a unique
# key for determining the last time it was seen.
key = '/'.join(topicsplit[1:-1])
anon_topic = self._topic_replace_map[key]
if not anon_topic:
anon_topic = key
for sr in self._topic_replace_list:
_log.debug('anon replacing {}->{}'.format(
sr['from'], sr['to']))
anon_topic = anon_topic.replace(sr['from'], sr['to'])
_log.debug('anon after replacing {}'.format(anon_topic))
_log.debug('Anon topic is: {}'.format(anon_topic))
self._topic_replace_map[key] = anon_topic
_log.debug('DEVICES ON PLATFORM ARE: {}'.format(self._devices))
self._devices[anon_topic] = {
'points': message[0].keys(),
'last_published_utc': format_timestamp(get_aware_utc_now())
}
def publish_test_data(publish_agent, start_time, start_reading, count):
reading = start_reading
time = start_time
print("publishing test data starttime is {} utcnow is {}".format(
start_time, datetime.utcnow()))
print("publishing test data value string {} at {}".format(
reading, datetime.now()))
float_meta = {'units': 'F', 'tz': 'UTC', 'type': 'float'}
for i in range(0, count):
#print("publishing reading {} at time {}".format(reading,
# time.isoformat()))
# Create a message for all points.
all_message = [{
'in_temp': reading,
'out_temp': reading
}, {
'in_temp': float_meta,
'out_temp': float_meta
}]
time_str = utils.format_timestamp(time)
headers = {headers_mod.DATE: time_str, headers_mod.TIMESTAMP: time_str}
publish_agent.vip.pubsub.publish('pubsub',
"devices/device1/all",
headers=headers,
message=all_message).get(timeout=10)
reading += 1
time = time + timedelta(seconds=30)
def query_current_weather(self, location):
"""
Retrieve data from the Darksky API, return formatted current data and
store forecast data in cache
:param location: location dictionary requested by the user
:return: Timestamp and data for current data from the Darksky API
"""
darksky_response = self.get_darksky_data('get_current_weather',
location)
if 'currently' not in darksky_response:
_log.error(
"Current data not found in Dark Sky response: {}".format(
darksky_response))
current_response = darksky_response.pop('currently')
# Darksky required attribution
current_response["attribution"] = "Powered by Dark Sky"
current_time = datetime.datetime.fromtimestamp(
current_response['time'],
pytz.timezone(darksky_response['timezone']))
current_time = current_time.astimezone(pytz.utc)
if not self.performance_mode:
# if performance mode isn't running we'll be receiving extra data
# that we can store to help with conserving daily api calls
for service in SERVICES_MAPPING:
if service is not 'get_current_weather' and \
SERVICES_MAPPING[service]['json_name'] in \
darksky_response:
service_response = darksky_response.pop(
SERVICES_MAPPING[service]['json_name'])
service_data = self.format_multientry_response(
location, service_response, service,
darksky_response['timezone'])
self.store_weather_records(service, service_data)
return format_timestamp(current_time), current_response
def _update_device_state_and_schedule(self, now):
_log.debug("_update_device_state_and_schedule")
# Sanity check now.
# This is specifically for when this is running in a VM that gets suspeded and then resumed.
# If we don't make this check a resumed VM will publish one event per minute of
# time the VM was suspended for.
test_now = datetime.datetime.now()
if test_now - now > datetime.timedelta(minutes=3):
now = test_now
self._device_states = self._schedule_manager.get_schedule_state(now)
schedule_next_event_time = self._schedule_manager.get_next_event_time(now)
new_update_event_time = self._get_ajusted_next_event_time(now, schedule_next_event_time)
for device, state in self._device_states.iteritems():
header = self._get_headers(state.agent_id, time=utils.format_timestamp(now), task_id=state.task_id)
header['window'] = state.time_remaining
topic = topics.ACTUATOR_SCHEDULE_ANNOUNCE_RAW.replace('{device}', device)
self.vip.pubsub.publish('pubsub', topic, headers=header)
if self._update_event is not None:
# This won't hurt anything if we are canceling ourselves.
self._update_event.cancel()
self._update_event = self.core.schedule(new_update_event_time,
self._update_schedule_state,
new_update_event_time)
def new_supply_signal(self, peer, sender, bus, topic, headers, message):
_log.debug("At {}, {} receives new supply records: {}".format(Timer.get_cur_time(),
self.name, message))
supply_curves = message['curves']
start_of_cycle = message['start_of_cycle']
self.campus.model.receive_transactive_signal(self, supply_curves)
_log.debug("At {}, mixmarket state is {}, start_of_cycle {}".format(Timer.get_cur_time(),
self.mix_market_running,
start_of_cycle))
db_topic = "/".join([self.db_topic, self.name, "CampusSupply"])
message = supply_curves
headers = {headers_mod.DATE: format_timestamp(Timer.get_cur_time())}
self.vip.pubsub.publish("pubsub", db_topic, headers, message).get()
if start_of_cycle:
_log.debug("At {}, start of cycle. "
"Mixmarket state before overriding is {}".format(Timer.get_cur_time(),
self.mix_market_running))
# if self.simulation:
# self.run_ep_sim(start_of_cycle)
# else:
self.start_mixmarket(start_of_cycle)
def _set_override_off(self, pattern):
pattern = pattern.lower()
# If pattern exactly matches
if pattern in self._override_patterns:
self._override_patterns.discard(pattern)
# Cancel any pending override events
self._cancel_override_events(pattern)
self._override_devices.clear()
patterns = dict()
# Build override devices list again
for pat in self._override_patterns:
for device in self.instances:
device = device.lower()
if fnmatch.fnmatch(device, pat):
self._override_devices.add(device)
if self._override_interval_events[pat] is None:
patterns[pat] = str(0.0)
else:
evt, end_time = self._override_interval_events[pat]
patterns[pat] = utils.format_timestamp(end_time)
self.vip.config.set("override_patterns", jsonapi.dumps(patterns))
else:
_log.error("Override Pattern did not match!")
raise OverrideError(
"Pattern {} does not exist in list of override patterns".format(pattern))
def publish_devices_fake_data(publish_agent, time=None):
# Publish fake data. The format mimics the format used by VOLTTRON drivers.
# Make some random readings
global DEVICES_ALL_TOPIC
reading = random_uniform(30, 100)
meta = {'units': 'F', 'tz': 'UTC', 'type': 'float'}
# Create a message for all points.
all_message = [{
'OutsideAirTemperature': reading,
'MixedAirTemperature': reading,
'DamperSignal': reading
}, {
'OutsideAirTemperature': meta,
'MixedAirTemperature': meta,
'DamperSignal': meta
}]
# Create timestamp
if not time:
time = utils.format_timestamp(datetime.utcnow())
# now = '2015-12-02T00:00:00'
headers = {headers_mod.DATE: time, headers_mod.TIMESTAMP: time}
print("Published time in header: " + time)
# Publish messages
publish(publish_agent, DEVICES_ALL_TOPIC, headers, all_message)
return time, reading, meta
def test_calls_exceeded(volttron_instance, cleanup_cache, query_agent, weather):
weather_uuid = weather[0]
identity = weather[1]
version = query_agent.vip.rpc.call(identity, 'get_version').get(timeout=3)
cwd = volttron_instance.volttron_home
database_file = "/".join([cwd, "agents", weather_uuid, "darkskyagent-" + version, "darkskyagent-" + version +
".agent-data", "weather.sqlite"])
sqlite_connection = sqlite3.connect(database_file)
cursor = sqlite_connection.cursor()
for i in range(0, 100):
time = format_timestamp(get_aware_utc_now() + timedelta(seconds=i))
insert_query = """INSERT INTO API_CALLS (CALL_TIME) VALUES (?);"""
cursor.execute(insert_query, (time,))
sqlite_connection.commit()
locations = [{"lat": 39.7555, "long": -105.2211}]
query_data = query_agent.vip.rpc.call(identity, 'get_current_weather', locations).get(timeout=30)
assert query_data[0]['weather_error'] == 'No calls currently available for the configured API key'
assert not query_data[0].get('weather_results')
query_data = query_agent.vip.rpc.call(identity, 'get_hourly_forecast', locations).get(timeout=30)
assert query_data[0]['weather_error'] == 'No calls currently available for the configured API key'
assert not query_data[0].get('weather_results')
delete_query = "DROP TABLE IF EXISTS API_CALLS;"
cursor.execute(delete_query)
create_query = """CREATE TABLE API_CALLS (CALL_TIME TIMESTAMP NOT NULL);"""
cursor.execute(create_query)
sqlite_connection.commit()
def publish_analysis_results(self, results):
"""
Publish table_data in analysis results to the message bus for
capture by the data historian.
:param results: Results object containing commands for devices,
log messages and table data.
:type results: Results object \\volttron.platform.agent.driven
:returns: Same as results param.
:rtype: Results object \\volttron.platform.agent.driven
"""
to_publish = defaultdict(dict)
for app, analysis_table in results.table_output.items():
try:
name_timestamp = app.split("&")
timestamp = name_timestamp[1]
except:
timestamp = self.received_input_datetime
timestamp = format_timestamp(timestamp)
headers = {headers_mod.CONTENT_TYPE: headers_mod.CONTENT_TYPE.JSON, headers_mod.DATE: timestamp, }
for entry in analysis_table:
for point, result in entry.items():
for device in command_devices:
publish_topic = "/".join([publish_base, device, point])
analysis_topic = topics.RECORD(subtopic=publish_topic)
datatype = str(type(value))
to_publish[analysis_topic] = result
for result_topic, result in to_publish.items():
self.vip.pubsub.publish("pubsub", result_topic, headers, result)
to_publish.clear()
return results
def publish_report(self):
for device, _cls in self.device_dict.items():
for sensor_topic in _cls.sensors:
evaluation = _cls.evaluate(sensor_topic)
if evaluation is None:
continue
report_topic = _cls.report_topic[sensor_topic]
for index, row in evaluation.iterrows():
ts = utils.format_timestamp(row["timestamp"])
fault_condition = round(row["fault"])
headers = {'Date': ts}
message = [{
'fault_condition': fault_condition
}, {
'fault_condition': {
'tz': 'UTC',
'data_type': 'integer'
}
}]
try:
self.vip.pubsub.publish(peer='pubsub',
topic=report_topic,
message=message,
headers=headers)
except Exception as e:
_log.error("In Publish: {}".format(str(e)))
def test_multi_messagebus_custom_topic_forwarder(multi_messagebus_forwarder):
"""
Forward Historian test for custom topics with multi message bus combinations
:return:
"""
from_instance, to_instance = multi_messagebus_forwarder
publish_agent = from_instance.dynamic_agent
subscriber_agent = to_instance.dynamic_agent
subscriber_agent.callback = MagicMock(name="callback")
subscriber_agent.callback.reset_mock()
subscriber_agent.vip.pubsub.subscribe(
peer='pubsub', prefix='foo', callback=subscriber_agent.callback).get()
#subscriber_agent.vip.pubsub.list(subscriber_agent.core.identity)
# Create timestamp
now = utils.format_timestamp(datetime.utcnow())
print("now is ", now)
headers = {headers_mod.DATE: now, headers_mod.TIMESTAMP: now}
gevent.sleep(5)
for i in range(0, 5):
topic = "foo/grid_signal"
value = 78.5 + i
publish(publish_agent, topic, headers, value)
gevent.sleep(0.1)
gevent.sleep(1)
assert subscriber_agent.callback.call_count == 5
def main(database_name):
db = sqlite3.connect(database_name)
c = db.cursor()
c.execute("select max(rowid) from data;")
count = c.fetchone()[0]
#Batches of 1000
#We do this because of a bug in the sqlite implementation in python
#which causes problems with nested cursors.
for i in range(0, count, 1000):
c.execute(
"select rowid, ts from data where rowid > ? order by rowid asc limit 1000;",
(i, ))
rows = c.fetchall()
print("processing rowid:", i + 1, "to", i + len(rows))
for rowid, ts in rows:
#Skip already converted rows.
if "T" in ts:
continue
new_ts = format_timestamp(parse_timestamp_string(ts))
c.execute("update data set ts = ? where rowid = ?;",
(new_ts, rowid))
db.commit()
def update_values(self, data, topic_name, start, end, values):
if start.tzinfo:
data[0] = data[0].replace(tzinfo=tzutc())
if data[0] >= start and data[0] < end:
result_value = self.json_string_to_dict(data[1])
values[topic_name].append(
(utils.format_timestamp(data[0]), result_value))
def _set_override_off(self, pattern):
"""Turn off override condition on all devices matching the pattern. It removes the pattern from the override
patterns set, clears the list of overriden devices and reevaluates the state of devices. It then cancels the
pending override event and removes pattern from the config store.
:param pattern: Override pattern to be removed.
:type pattern: str
"""
pattern = pattern.lower()
# If pattern exactly matches
if pattern in self._override_patterns:
self._override_patterns.discard(pattern)
# Cancel any pending override events
self._cancel_override_events(pattern)
self._override_devices.clear()
patterns = dict()
# Build override devices list again
for pat in self._override_patterns:
for device in self.instances:
device = device.lower()
if fnmatch.fnmatch(device, pat):
self._override_devices.add(device)
if self._override_interval_events[pat] is None:
patterns[pat] = str(0.0)
else:
evt, end_time = self._override_interval_events[pat]
patterns[pat] = utils.format_timestamp(end_time)
self.vip.config.set("override_patterns", jsonapi.dumps(patterns))
else:
_log.error("Override Pattern did not match!")
raise OverrideError(
"Pattern {} does not exist in list of override patterns".format(pattern))
def begin_collect_offers(self, timestamp):
_log.debug("send_collect_offers_request at {}".format(timestamp))
self.start_offers_has_markets()
self.market_list.collect_offers()
unformed_markets = self.market_list.unformed_market_list()
self.vip.pubsub.publish(peer='pubsub',
topic=MARKET_BID,
message=[utils.format_timestamp(timestamp), unformed_markets])
def publish_weather_report(self, topic_name, parsed_weather_data):
now = utils.format_timestamp(datetime.utcnow())
headers = {
headers_mod.CONTENT_TYPE: headers_mod.CONTENT_TYPE.PLAIN_TEXT,
headers_mod.DATE: now,
headers_mod.TIMESTAMP: now
}
self.vip.pubsub.publish('pubsub', topic_name, headers, json.dumps(parsed_weather_data))
def send_collect_reservations_request(self, timestamp):
_log.debug("send_collect_reservations_request at {}".format(timestamp))
self.start_reservations()
self.market_list.send_market_failure_errors()
self.market_list.clear_reservations()
self.vip.pubsub.publish(peer='pubsub',
topic=MARKET_RESERVE,
message=utils.format_timestamp(timestamp))
def actuator_request(self, score_order):
"""
Request schedule to interact with devices via rpc call to actuator agent.
:param score_order: ahp priority for devices (curtailment priority).
:return:
"""
current_time = get_aware_utc_now()
start_time_str = format_timestamp(current_time)
end_curtail_time = current_time + self.longest_possible_curtail + self.actuator_schedule_buffer
end_time_str = format_timestamp(end_curtail_time)
curtailable_device = []
already_handled = dict((device[0], True) for device in self.scheduled_devices)
for item in score_order:
device, token, device_actuator = item
_log.debug("Reserving device: {}".format(device))
if device in already_handled:
if already_handled[device]:
_log.debug("Skipping reserve device (previously reserved): " + device)
curtailable_device.append(item)
continue
curtailed_device = self.base_rpc_path(unit=device, point="")
schedule_request = [[curtailed_device, start_time_str, end_time_str]]
try:
if self.kill_signal_received:
break
result = self.vip.rpc.call(device_actuator, "request_new_schedule",
self.agent_id, device, "HIGH", schedule_request).get(timeout=5)
except RemoteError as ex:
_log.warning("Failed to schedule device {} (RemoteError): {}".format(device, str(ex)))
continue
if result["result"] == "FAILURE":
_log.warn("Failed to schedule device (unavailable) " + device)
already_handled[device] = False
else:
already_handled[device] = True
self.scheduled_devices.add((device, device_actuator))
curtailable_device.append(item)
return curtailable_device
def publish_heartbeat(self):
now = utils.format_timestamp(datetime.utcnow())
headers = {
headers_mod.CONTENT_TYPE: headers_mod.CONTENT_TYPE.PLAIN_TEXT,
headers_mod.DATE: now,
headers_mod.TIMESTAMP: now
}
result = self.vip.pubsub.publish('pubsub', 'heartbeat/NodeRedSubscriber', headers, now)
result.get(timeout=10)
def publish_loop(self):
"""Publish data from file to message bus."""
# We cannot reset the value in the config until we are in a separate greenlet.
# We cannot call set in a config handler.
if self._reset_playback:
self._line_marker = 0
if self._remember_playback:
self.vip.config.set(LINE_MARKER_CONFIG, str(0), send_update=False)
while True:
current_line = -1
for row in self._data:
current_line += 1
if current_line < self._line_marker:
continue
self._line_marker += 1
if self._use_timestamp and "Timestamp" in row:
now = row['Timestamp']
if not self.check_frequency(now):
continue
else:
now = utils.format_timestamp(datetime.datetime.now())
headers = {HEADER_NAME_DATE: now, HEADER_NAME_TIMESTAMP: now}
row.pop('Timestamp', None)
publish_dict = self.build_publishes(row)
_log.debug("Publishing data for timestamp: {}".format(now))
for topic, message in publish_dict.iteritems():
self._publish_point_all(topic, message, self._meta_data, headers)
if self._remember_playback:
self.vip.config.set(LINE_MARKER_CONFIG, str(self._line_marker), send_update=False)
gevent.sleep(self._publish_interval)
# Reset line marker.
self._line_marker = 0
if self._remember_playback:
self.vip.config.set(LINE_MARKER_CONFIG, str(self._line_marker), send_update=False)
if not self._replay_data:
sys.exit(0)
break
# Reset the csv reader if we are reading from a file.
_log.debug("Restarting playback.")
# Reset data frequency counter.
self._next_allowed_publish = None
if not isinstance(self._input_data, list):
handle = open(self._input_data, 'rb')
self._data = csv.DictReader(handle)
def publish_to_historian(self, to_publish_list):
_log.debug("publish_to_historian number of items: {}".format(
len(to_publish_list)))
try:
for stored_index, row in enumerate(to_publish_list):
ts = utils.format_timestamp(row['timestamp'])
source = row['source']
topic = row['topic']
meta = row['meta']
value = row['value']
value_string = str(value)
# Check type of value from metadata if it exists,
# then cast value to that type
try:
value_type = meta["type"]
value = influxdbutils.value_type_matching(value_type, value)
except KeyError:
_log.info("Metadata doesn't include \'type\' keyword")
except ValueError:
_log.warning("Metadata specifies \'type\' of value is {} while "
"value={} is type {}".format(value_type, value, type(value)))
topic_id = topic.lower()
# If the topic is not in the list
if topic_id not in self._topic_id_map:
self._topic_id_map[topic_id] = topic
self._meta_dicts[topic_id] = {}
# If topic's metadata changes, update its metadata.
if topic_id in self._topic_id_map and meta != self._meta_dicts[topic_id]:
_log.info("Updating meta for topic {} at {}".format(topic_id, ts))
self._meta_dicts[topic_id] = meta
# Insert the meta into the database
influxdbutils.insert_meta(self._client, topic_id, topic, meta, ts)
# Else if topic name in database changes, update.
elif topic_id in self._topic_id_map and self._topic_id_map[topic_id] != topic:
_log.info("Updating actual topic name {} in database for topic id {}".format(topic, topic_id))
self._topic_id_map[topic_id] = topic
# Update topic name in the database
influxdbutils.insert_meta(self._client, topic_id, topic, meta, ts)
# Insert data point
influxdbutils.insert_data_point(self._client, ts, topic_id, source, value, value_string)
# After all data points are published
self.report_all_handled()
_log.info("Store ALL data in to_publish_list to InfluxDB client")
except ConnectionError, err:
raise err
def publish_minute_data_for_two_hours(agent):
now = get_aware_utc_now()
# expection[datetime]={oat:b,mixed:c,damper:d}
expectation = {}
for h in xrange(2):
data_by_time = {}
for m in xrange(60):
# Because timestamps in mongo are only concerned with the first
# three digits after the decimal we do this to give some
# randomness here.
myint = random.randint(0, 1000)
mymicro = str(myint)+'000'
now = datetime(now.year, now.month, now.day, h, m,
random.randint(0, 59), int(mymicro))
# Make some random readings
oat_reading = random.uniform(30, 100)
mixed_reading = oat_reading + random.uniform(-5, 5)
damper_reading = random.uniform(0, 100)
# Create a message for all points.
all_message = [{
'OutsideAirTemperature': oat_reading,
'MixedAirTemperature': mixed_reading,
'DamperSignal': damper_reading},
{'OutsideAirTemperature':
{'units': 'F', 'tz': 'UTC', 'type': 'float'},
'MixedAirTemperature': {'units': 'F', 'tz': 'UTC', 'type': 'float'},
'DamperSignal': {'units': '%', 'tz': 'UTC', 'type': 'float'}
}]
now_iso_string = format_timestamp(now)
data_by_time[now_iso_string] = {
"oat_point": oat_reading,
"mixed_point": mixed_reading,
"damper_point": damper_reading
}
# now = '2015-12-02T00:00:00'
headers = {
headers_mod.DATE: now_iso_string
}
# Publish messages
agent.vip.pubsub.publish(
'pubsub', ALL_TOPIC, headers, all_message).get(timeout=10)
expectation[now] = {
query_points['oat_point']: oat_reading,
query_points['mixed_point']: mixed_reading,
query_points['damper_point']: damper_reading
}
gevent.sleep(0.1)
return expectation
