def on_alert_message(self, peer, sender, bus, topic, headers, message): """ Callback for alert messages that come into the platform. :param peer: :param sender: :param bus: :param topic: :param headers: :param message: """ _log.info('Alert message captured for topic: {}'.format(topic)) if not self.current_config.get('send_alerts_enabled'): _log.warn('Alert message found but not sent enable alerts ' 'enable by setting send_alerts_enabled to True') return mailkey = headers.get(ALERT_KEY, None) if not mailkey: _log.error( "alert_key not found in header " + "for message topic: {} message: {}".format(topic, message)) return last_sent_key = tuple([mailkey, topic]) last_sent_time = self.sent_alert_emails[last_sent_key] should_send = False # python sets this to 0 if it hasn't ever been sent. if not last_sent_time: should_send = True else: current_time = get_utc_seconds_from_epoch() allow_frequency_seconds = self.current_config[ 'allow_frequency_seconds'] if last_sent_time + allow_frequency_seconds < current_time: should_send = True if not should_send: _log.debug('Waiting for time to pass for email.') return from_address = self.current_config['alert_from_address'] recipients = self.current_config['alert_to_addresses'] if isinstance(recipients, basestring): recipients = [recipients] # After here we are going to attempt to send the email out subject = "Alert for {} {}".format(topic, mailkey) # Use unicode to protect against encod error # http://stackoverflow.com/questions/25891541/attributeerror-encode msg = MIMEText(unicode(message)) msg['To'] = ', '.join(recipients) msg['FROM'] = from_address msg['Subject'] = subject self.send_email(from_address, recipients, subject, msg) # we assume the email will go through. self.sent_alert_emails[last_sent_key] = get_utc_seconds_from_epoch()
def on_alert_message(self, peer, sender, bus, topic, headers, message): """ Callback for alert messages that come into the platform. :param peer: :param sender: :param bus: :param topic: :param headers: :param message: """ _log.info('Alert message captured for topic: {}'.format(topic)) if not self.current_config.get('send_alerts_enabled'): _log.warn('Alert message found but not sent enable alerts ' 'enable by setting send_alerts_enabled to True') return mailkey = headers.get(ALERT_KEY, None) if not mailkey: _log.error("alert_key not found in header " + "for message topic: {} message: {}" .format(topic, message)) return last_sent_key = tuple([mailkey, topic]) last_sent_time = self.sent_alert_emails[last_sent_key] should_send = False # python sets this to 0 if it hasn't ever been sent. if not last_sent_time: should_send = True else: current_time = get_utc_seconds_from_epoch() allow_frequency_seconds = self.current_config['allow_frequency_seconds'] if last_sent_time + allow_frequency_seconds < current_time: should_send=True if not should_send: _log.debug('Waiting for time to pass for email.') return from_address = self.current_config['alert_from_address'] recipients = self.current_config['alert_to_addresses'] if isinstance(recipients, basestring): recipients = [recipients] # After here we are going to attempt to send the email out subject = "Alert for {} {}".format(topic, mailkey) # Use unicode to protect against encod error # http://stackoverflow.com/questions/25891541/attributeerror-encode msg = MIMEText(unicode(message)) msg['To'] = ', '.join(recipients) msg['FROM'] = from_address msg['Subject'] = subject self.send_email(from_address, recipients, subject, msg) # we assume the email will go through. self.sent_alert_emails[last_sent_key] = get_utc_seconds_from_epoch()
def add_agent_user(self, agent_name, agent_dir): """ Invokes sudo to create a unique unix user for the agent. :param agent_name: :param agent_dir: :return: """ # Ensure the agent users unix group exists self.add_agent_user_group() # Create a USER_ID file, truncating existing USER_ID files which # should at this point be considered unsafe user_id_path = os.path.join(agent_dir, "USER_ID") with open(user_id_path, "w+") as user_id_file: volttron_agent_user = "******".format( str(get_utc_seconds_from_epoch()).replace(".", "")) _log.info("Creating volttron user {}".format(volttron_agent_user)) group = "volttron_{}".format(self.instance_name) useradd = ['sudo', 'useradd', volttron_agent_user, '-r', '-G', group] useradd_process = subprocess.Popen( useradd, stdout=PIPE, stderr=PIPE) stdout, stderr = useradd_process.communicate() if useradd_process.returncode != 0: # TODO alert? raise RuntimeError("Creating {} user failed: {}".format( volttron_agent_user, stderr)) user_id_file.write(volttron_agent_user) return volttron_agent_user
def _add_to_cache(self, device, topic, value): cached_time = get_utc_seconds_from_epoch() try: self._cache[device][topic] = (float(value), cached_time) except: _log.error( "Topic \"{}\" on device \"{}\" contained value that was not " "castable as float".format(topic, device))
def stop_iam(): _log.debug('Done publishing i am responses.') stop_timestamp = get_utc_seconds_from_epoch() self._pub_to_vc(iam_topic, message=dict(status="FINISHED IAM", timestamp=stop_timestamp)) agent_to_use.vip.pubsub.unsubscribe('pubsub', topics.BACNET_I_AM, handle_iam)
def schedule(self, deadline, func, *args, **kwargs): if hasattr(deadline, 'timetuple'): #deadline = time.mktime(deadline.timetuple()) deadline = utils.get_utc_seconds_from_epoch(deadline) event = ScheduledEvent(func, args, kwargs) heapq.heappush(self._schedule, (deadline, event)) self._schedule_event.set() return event
def schedule(cls, deadline, *args, **kwargs): # pylint: disable=no-self-argument if hasattr(deadline, 'timetuple'): #deadline = time.mktime(deadline.timetuple()) deadline = utils.get_utc_seconds_from_epoch(deadline) def decorate(method): annotate(method, list, 'core.schedule', (deadline, args, kwargs)) return method return decorate
def schedule(cls, deadline, *args, **kwargs): # pylint: disable=no-self-argument if hasattr(deadline, 'timetuple'): # deadline = time.mktime(deadline.timetuple()) deadline = utils.get_utc_seconds_from_epoch(deadline) def decorate(method): annotate(method, list, 'core.schedule', (deadline, args, kwargs)) return method return decorate
def publish_next(self): if self._publish_event is not None: self._publish_event.cancel() if self._gapps.connected: self._gapps.send("/queue/test", "foo bar") else: print("Not connected") now = get_aware_utc_now() print(utils.get_utc_seconds_from_epoch(now)) next_update_time = now + datetime.timedelta(seconds=5) self._publish_event = self.core.schedule(next_update_time, self.publish_next) gevent.sleep(0.1)
def health(self): """ Returns a Status object as a dictionary. This will be populated by the heartbeat from the external instance that this object is monitoring, unless it has been over 10 seconds since the instance has been reached. In that case the health will be BAD. :return: """ now = get_utc_seconds_from_epoch() self._health = Status.build(GOOD_STATUS, "Platform here!") # if now > self._last_time_verified_connection + 10: # self._health = Status.build( # BAD_STATUS, # "Platform hasn't been reached in over 10 seconds.") return self._health.as_dict()
def publish_next(self): _log.debug("Publishing next") if self._publish_event is not None: self._publish_event.cancel() self._gapps.send(self._test_topic, "foo bar") if self._gapps.connected: self._gapps.send("/queue/test", "foo bar") else: print("Not connected") now = get_aware_utc_now() print(utils.get_utc_seconds_from_epoch(now)) next_update_time = now + datetime.timedelta(seconds=5) _log.debug(f'Scheduling nex time {next_update_time}') self._publish_event = self.core.schedule(next_update_time, self.publish_next) _log.debug(f'After scheduling next time {next_update_time}') gevent.sleep(0.1)
def health(self): """ Returns a Status object as a dictionary. This will be populated by the heartbeat from the external instance that this object is monitoring, unless it has been over 10 seconds since the instance has been reached. In that case the health will be BAD. :return: """ now = get_utc_seconds_from_epoch() self._health = Status.build( GOOD_STATUS, "Platform here!" ) # if now > self._last_time_verified_connection + 10: # self._health = Status.build( # BAD_STATUS, # "Platform hasn't been reached in over 10 seconds.") return self._health.as_dict()
def scrape(self, env, data): scrape_time = get_utc_seconds_from_epoch() keys_to_delete = defaultdict(list) if len(self._cache) > 0: result = "# TYPE volttron_data gauge\n" for device, device_topics in self._cache.iteritems(): device_tags = device.replace("-", "_").split('/') for topic, value in device_topics.iteritems(): if value[1] + self._cache_time > scrape_time: metric_props = re.split(self._tag_delimiter_re, topic.lower()) metric_tag_str = ( "campus=\"{}\",building=\"{}\"," "device=\"{}\",").format(*device_tags) for i, prop in enumerate(metric_props): metric_tag_str += "tag{}=\"{}\",".format(i, prop) result += "{}{{{}topic=\"{}\"}} {}\n".format( re.sub(" |/|-", "_", device), metric_tag_str, topic.replace(" ", "_"), value[0]) else: try: keys_to_delete[device].append(topic) except Exception as e: _log.error("Could not delete stale topic") _log.exception(e) continue else: result = "#No Data to Scrape" for device, delete_topics in keys_to_delete.iteritems(): for topic in delete_topics: del self._cache[device][topic] gzip_compress = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16) data = gzip_compress.compress(result) + gzip_compress.flush() return "200 OK", base64.b64encode(data), [ ('Content-Type', 'text/plain'), ('Content-Encoding', 'gzip')]
def scrape(self, env, data): scrape_time = get_utc_seconds_from_epoch() keys_to_delete = defaultdict(list) if len(self._cache) > 0: result = "# TYPE volttron_data gauge\n" for device, device_topics in self._cache.iteritems(): device_tags = device.replace("-", "_").split('/') for topic, value in device_topics.iteritems(): if value[1] + self._cache_time > scrape_time: metric_props = re.split(self._tag_delimiter_re, topic.lower()) metric_tag_str = ("campus=\"{}\",building=\"{}\"," "device=\"{}\",").format( *device_tags) for i, prop in enumerate(metric_props): metric_tag_str += "tag{}=\"{}\",".format(i, prop) result += "{}{{{}topic=\"{}\"}} {}\n".format( re.sub(" |/|-", "_", device), metric_tag_str, topic.replace(" ", "_"), value[0]) else: try: keys_to_delete[device].append(topic) except Exception as e: _log.error("Could not delete stale topic") _log.exception(e) continue else: result = "#No Data to Scrape" for device, delete_topics in keys_to_delete.iteritems(): for topic in delete_topics: del self._cache[device][topic] gzip_compress = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16) data = gzip_compress.compress(result) + gzip_compress.flush() return "200 OK", base64.b64encode(data), [('Content-Type', 'text/plain'), ('Content-Encoding', 'gzip')]
def publish_to_historian(self, to_publish_list): _log.debug("publish_to_historian number of items: {}".format( len(to_publish_list))) # Verify that we have actually gone through the historian_setup code # before we attempt to do anything else. if not self._initialized: self.historian_setup() if not self._initialized: return if self._wait_until is not None: ct = get_utc_seconds_from_epoch() if ct > self._wait_until: self._wait_until = None else: _log.debug('Waiting to attempt to write to database.') return try: if self._connection is None: self._connection = self.get_connection() cursor = self._connection.cursor() batch_data = [] for row in to_publish_list: ts = utils.format_timestamp(row['timestamp']) source = row['source'] topic = row['topic'] value = row['value'] meta = row['meta'] # Handle the serialization of data here because we can't pass # an array as a string so we create a string from the value. if isinstance(value, list) or isinstance(value, dict): value = dumps(value) if topic not in self._topic_set: try: cursor.execute(insert_topic_query(self._schema), (topic, )) except ProgrammingError as ex: if ex.args[0].startswith( 'DocumentAlreadyExistsException'): self._topic_set.add(topic) else: _log.error( "Unknown error during topic insert {} {}". format(type(ex), ex.args)) else: self._topic_set.add(topic) batch_data.append((ts, topic, source, value, meta)) try: query = insert_data_query(self._schema) _log.debug("Inserting batch data: {}".format(batch_data)) cursor.executemany(query, batch_data) except ProgrammingError as ex: _log.error( "Invalid data detected during batch insert: {}".format( ex.args)) _log.debug("Attempting singleton insert.") insert = insert_data_query(self._schema) for id in range(len(batch_data)): try: batch = batch_data[id] cursor.execute(insert, batch) except ProgrammingError: _log.debug('Invalid data not saved {}'.format( to_publish_list[id])) self.report_handled(to_publish_list[id]) except Exception as ex: _log.error("Exception Type: {} ARGS: {}".format( type(ex), ex.args)) else: self.report_handled(to_publish_list[id]) except Exception as ex: _log.error("Exception Type: {} ARGS: {}".format( type(ex), ex.args)) else: self.report_all_handled() except TypeError as ex: _log.error("AFTER EXCEPTION: {} ARGS: {}".format( type(ex), ex.args)) except Exception as ex: _log.error("Unknown Exception {} {}".format(type(ex), ex.args)) finally: if cursor is not None: cursor.close() cursor = None
def _schedule_callback(self, deadline, callback): deadline = utils.get_utc_seconds_from_epoch(deadline) heapq.heappush(self._schedule, (deadline, self.get_tie_breaker(), callback)) if self._schedule_event: self._schedule_event.set()
def publish_to_historian(self, to_publish_list): _log.debug("publish_to_historian number of items: {}".format( len(to_publish_list))) # Verify that we have actually gone through the historian_setup code # before we attempt to do anything else. if not self._initialized: self.historian_setup() if not self._initialized: return if self._wait_until is not None: ct = get_utc_seconds_from_epoch() if ct > self._wait_until: self._wait_until = None else: _log.debug('Waiting to attempt to write to database.') return try: if self._connection is None: self._connection = self.get_connection() cursor = self._connection.cursor() batch_data = [] for row in to_publish_list: ts = utils.format_timestamp(row['timestamp']) source = row['source'] topic = row['topic'] value = row['value'] meta = row['meta'] # Handle the serialization of data here because we can't pass # an array as a string so we create a string from the value. if isinstance(value, list) or isinstance(value, dict): value = dumps(value) if topic not in self._topic_set: try: cursor.execute(insert_topic_query(self._schema), (topic,)) except ProgrammingError as ex: if ex.args[0].startswith( 'DocumentAlreadyExistsException'): self._topic_set.add(topic) else: _log.error( "Unknown error during topic insert {} {}".format( type(ex), ex.args )) else: self._topic_set.add(topic) batch_data.append( (ts, topic, source, value, meta) ) try: query = insert_data_query(self._schema) _log.debug("Inserting batch data: {}".format(batch_data)) cursor.executemany(query, batch_data) except ProgrammingError as ex: _log.error( "Invalid data detected during batch insert: {}".format( ex.args)) _log.debug("Attempting singleton insert.") insert = insert_data_query(self._schema) for id in range(len(batch_data)): try: batch = batch_data[id] cursor.execute(insert, batch) except ProgrammingError: _log.debug('Invalid data not saved {}'.format( to_publish_list[id] )) self.report_handled(to_publish_list[id]) except Exception as ex: _log.error( "Exception Type: {} ARGS: {}".format(type(ex), ex.args)) else: self.report_handled(to_publish_list[id]) except Exception as ex: _log.error( "Exception Type: {} ARGS: {}".format(type(ex), ex.args)) else: self.report_all_handled() except TypeError as ex: _log.error( "AFTER EXCEPTION: {} ARGS: {}".format(type(ex), ex.args)) except Exception as ex: _log.error( "Unknown Exception {} {}".format(type(ex), ex.args) ) finally: if cursor is not None: cursor.close() cursor = None
def start_bacnet_scan(self, iam_topic, proxy_identity, low_device_id=None, high_device_id=None, target_address=None, scan_length=5, instance_address=None, instance_serverkey=None): """This function is a wrapper around the bacnet proxy scan. """ agent_to_use = self if instance_address is not None: agent_to_use = build_agent(address=instance_address, identity="proxy_bacnetplatform", publickey=self.core.publickey, secretkey=self.core.secretkey, serverkey=instance_serverkey) if proxy_identity not in agent_to_use.vip.peerlist().get(timeout=5): raise Unreachable( "Can't reach agent identity {}".format(proxy_identity)) _log.info('Starting bacnet_scan with who_is request to {}'.format( proxy_identity)) def handle_iam(peer, sender, bus, topic, headers, message): proxy_identity = sender address = message['address'] device_id = message['device_id'] bn = BACnetReader(agent_to_use.vip.rpc, proxy_identity) message['device_name'] = bn.read_device_name(address, device_id) message['device_description'] = bn.read_device_description( address, device_id) self._pub_to_vc(iam_topic, message=message) def stop_iam(): _log.debug('Done publishing i am responses.') stop_timestamp = get_utc_seconds_from_epoch() self._pub_to_vc(iam_topic, message=dict(status="FINISHED IAM", timestamp=stop_timestamp)) agent_to_use.vip.pubsub.unsubscribe('pubsub', topics.BACNET_I_AM, handle_iam) agent_to_use.vip.pubsub.subscribe('pubsub', topics.BACNET_I_AM, handle_iam) timestamp = get_utc_seconds_from_epoch() self._pub_to_vc(iam_topic, message=dict(status="STARTED IAM", timestamp=timestamp)) agent_to_use.vip.rpc.call( proxy_identity, "who_is", low_device_id=low_device_id, high_device_id=high_device_id, target_address=target_address).get(timeout=5.0) gevent.spawn_later(float(scan_length), stop_iam)
def publish_to_historian(self, to_publish_list): _log.debug("publish_to_historian number of items: {}".format( len(to_publish_list))) start_time = get_utc_seconds_from_epoch() if self._client is None: success = self._establish_client_connection() if not success: return try: cursor = self._client.cursor() batch_data = [] for row in to_publish_list: ts = utils.format_timestamp(row['timestamp']) source = row['source'] topic = row['topic'] value = row['value'] meta = row['meta'] topic_lower = topic.lower() # Handle the serialization of data here because we can't pass # an array as a string so we create a string from the value. if isinstance(value, list) or isinstance(value, dict): value = dumps(value) if topic_lower not in self._topic_meta: try: cursor.execute(insert_topic_query(self._schema, self._topic_table), (topic, meta)) except ProgrammingError as ex: if ex.args[0].startswith( 'SQLActionException[DuplicateKeyException'): self._topic_meta[topic_lower] = meta else: _log.error(repr(ex)) _log.error( "Unknown error during topic insert {} {}".format( type(ex), ex.args )) else: self._topic_meta[topic_lower] = meta else: # check if metadata matches old_meta = self._topic_meta.get(topic_lower) if not old_meta: old_meta = {} if set(old_meta.items()) != set(meta.items()): _log.debug( 'Updating meta for topic: {} {}'.format(topic, meta)) self._topic_meta[topic_lower] = meta cursor.execute(update_topic_query(self._schema, self._topic_table), (meta, topic)) batch_data.append( (ts, topic, source, value, meta) ) try: query = insert_data_query(self._schema, self._data_table) # _log.debug("Inserting batch data: {}".format(batch_data)) results = cursor.executemany(query, batch_data) index = 0 failures = [] for r in results: if r['rowcount'] != 1: failures.append(index) index += 1 if failures: for findex in failures: data = batch_data[findex] _log.error("Failed to insert data {}".format(data)) self.report_handled(to_publish_list[findex]) except ProgrammingError as ex: _log.error( "Invalid data detected during batch insert: {}".format( ex.args)) _log.debug("Attempting singleton insert.") insert = insert_data_query(self._schema, self._data_table) for id in range(len(batch_data)): try: batch = batch_data[id] cursor.execute(insert, batch) except ProgrammingError: _log.debug('Invalid data not saved {}'.format( batch )) except Exception as ex: _log.error(repr(ex)) else: self.report_handled(to_publish_list[id]) except Exception as ex: _log.error( "Exception Type: {} ARGS: {}".format(type(ex), ex.args)) else: self.report_all_handled() except TypeError as ex: _log.error(repr(ex)) _log.error( "AFTER EXCEPTION: {} ARGS: {}".format(type(ex), ex.args)) except Exception as ex: _log.error(repr(ex)) _log.error( "Unknown Exception {} {}".format(type(ex), ex.args) ) finally: if cursor is not None: cursor.close() cursor = None
def schedule(self, deadline, func, *args, **kwargs): deadline = utils.get_utc_seconds_from_epoch(deadline) event = ScheduledEvent(func, args, kwargs) heapq.heappush(self._schedule, (deadline, event)) self._schedule_event.set() return event
def _schedule_callback(self, deadline, callback): deadline = utils.get_utc_seconds_from_epoch(deadline) heapq.heappush(self._schedule, (deadline, callback)) self._schedule_event.set()
def publish_to_historian(self, to_publish_list): # _log.debug("publish_to_historian number of items: {}".format( # len(to_publish_list))) start_time = get_utc_seconds_from_epoch() if self._client is None: success = self._establish_client_connection() if not success: return try: cursor = self._client.cursor() batch_data = [] for row in to_publish_list: ts = utils.format_timestamp(row['timestamp']) source = row['source'] topic = self.get_renamed_topic(row['topic']) value = row['value'] meta = row['meta'] # Handle the serialization of data here because we can't pass # an array as a string so we create a string from the value. if isinstance(value, list) or isinstance(value, dict): value = dumps(value) if topic not in self._topic_set: try: cursor.execute(insert_topic_query(self._schema), (topic,)) except ProgrammingError as ex: if ex.args[0].startswith( 'SQLActionException[DuplicateKeyException'): self._topic_set.add(topic) else: _log.error(repr(ex)) _log.error( "Unknown error during topic insert {} {}".format( type(ex), ex.args )) else: self._topic_set.add(topic) batch_data.append( (ts, topic, source, value, meta) ) try: query = insert_data_query(self._schema) # _log.debug("Inserting batch data: {}".format(batch_data)) results = cursor.executemany(query, batch_data) index = 0 failures = [] for r in results: if r['rowcount'] == -1: failures.append(index) index += 1 if failures: for findex in failures: data = batch_data[findex] _log.error("Failed to insert data {}".format(data)) self.report_handled(to_publish_list[findex]) except ProgrammingError as ex: _log.error( "Invalid data detected during batch insert: {}".format( ex.args)) _log.debug("Attempting singleton insert.") insert = insert_data_query(self._schema) for id in range(len(batch_data)): try: batch = batch_data[id] cursor.execute(insert, batch) except ProgrammingError: _log.debug('Invalid data not saved {}'.format( to_publish_list[id] )) self.report_handled(to_publish_list[id]) except Exception as ex: _log.error(repr(ex)) else: self.report_handled(to_publish_list[id]) except Exception as ex: _log.error( "Exception Type: {} ARGS: {}".format(type(ex), ex.args)) else: self.report_all_handled() except TypeError as ex: _log.error(repr(ex)) _log.error( "AFTER EXCEPTION: {} ARGS: {}".format(type(ex), ex.args)) except Exception as ex: _log.error(repr(ex)) _log.error( "Unknown Exception {} {}".format(type(ex), ex.args) ) finally: if cursor is not None: cursor.close() cursor = None