Ejemplo n.º 1
0
    def on_alert_message(self, peer, sender, bus, topic, headers, message):
        """
        Callback for alert messages that come into the platform.

        :param peer:
        :param sender:
        :param bus:
        :param topic:
        :param headers:
        :param message:
        """
        _log.info('Alert message captured for topic: {}'.format(topic))
        if not self.current_config.get('send_alerts_enabled'):
            _log.warn('Alert message found but not sent enable alerts '
                      'enable by setting send_alerts_enabled to True')
            return
        mailkey = headers.get(ALERT_KEY, None)

        if not mailkey:
            _log.error(
                "alert_key not found in header " +
                "for message topic: {} message: {}".format(topic, message))
            return

        last_sent_key = tuple([mailkey, topic])
        last_sent_time = self.sent_alert_emails[last_sent_key]

        should_send = False
        # python sets this to 0 if it hasn't ever been sent.
        if not last_sent_time:
            should_send = True
        else:
            current_time = get_utc_seconds_from_epoch()
            allow_frequency_seconds = self.current_config[
                'allow_frequency_seconds']
            if last_sent_time + allow_frequency_seconds < current_time:
                should_send = True

        if not should_send:
            _log.debug('Waiting for time to pass for email.')
            return

        from_address = self.current_config['alert_from_address']
        recipients = self.current_config['alert_to_addresses']
        if isinstance(recipients, basestring):
            recipients = [recipients]

        # After here we are going to attempt to send the email out
        subject = "Alert for {} {}".format(topic, mailkey)

        # Use unicode to protect against encod error
        # http://stackoverflow.com/questions/25891541/attributeerror-encode
        msg = MIMEText(unicode(message))
        msg['To'] = ', '.join(recipients)
        msg['FROM'] = from_address
        msg['Subject'] = subject
        self.send_email(from_address, recipients, subject, msg)

        # we assume the email will go through.
        self.sent_alert_emails[last_sent_key] = get_utc_seconds_from_epoch()
Ejemplo n.º 2
0
    def on_alert_message(self, peer, sender, bus, topic, headers, message):
        """
        Callback for alert messages that come into the platform.

        :param peer:
        :param sender:
        :param bus:
        :param topic:
        :param headers:
        :param message:
        """
        _log.info('Alert message captured for topic: {}'.format(topic))
        if not self.current_config.get('send_alerts_enabled'):
            _log.warn('Alert message found but not sent enable alerts '
                      'enable by setting send_alerts_enabled to True')
            return
        mailkey = headers.get(ALERT_KEY, None)

        if not mailkey:
            _log.error("alert_key not found in header "
                       + "for message topic: {} message: {}"
                       .format(topic, message))
            return

        last_sent_key = tuple([mailkey, topic])
        last_sent_time = self.sent_alert_emails[last_sent_key]

        should_send = False
        # python sets this to 0 if it hasn't ever been sent.
        if not last_sent_time:
            should_send = True
        else:
            current_time = get_utc_seconds_from_epoch()
            allow_frequency_seconds = self.current_config['allow_frequency_seconds']
            if last_sent_time + allow_frequency_seconds < current_time:
                should_send=True

        if not should_send:
            _log.debug('Waiting for time to pass for email.')
            return

        from_address = self.current_config['alert_from_address']
        recipients = self.current_config['alert_to_addresses']
        if isinstance(recipients, basestring):
            recipients = [recipients]

        # After here we are going to attempt to send the email out
        subject = "Alert for {} {}".format(topic, mailkey)

        # Use unicode to protect against encod error
        # http://stackoverflow.com/questions/25891541/attributeerror-encode
        msg = MIMEText(unicode(message))
        msg['To'] = ', '.join(recipients)
        msg['FROM'] = from_address
        msg['Subject'] = subject
        self.send_email(from_address, recipients, subject, msg)

        # we assume the email will go through.
        self.sent_alert_emails[last_sent_key] = get_utc_seconds_from_epoch()
Ejemplo n.º 3
0
    def add_agent_user(self, agent_name, agent_dir):
        """
        Invokes sudo to create a unique unix user for the agent.
        :param agent_name:
        :param agent_dir:
        :return:
        """

        # Ensure the agent users unix group exists
        self.add_agent_user_group()

        # Create a USER_ID file, truncating existing USER_ID files which
        # should at this point be considered unsafe
        user_id_path = os.path.join(agent_dir, "USER_ID")

        with open(user_id_path, "w+") as user_id_file:
            volttron_agent_user = "******".format(
                str(get_utc_seconds_from_epoch()).replace(".", ""))
            _log.info("Creating volttron user {}".format(volttron_agent_user))
            group = "volttron_{}".format(self.instance_name)
            useradd = ['sudo', 'useradd', volttron_agent_user, '-r', '-G', group]
            useradd_process = subprocess.Popen(
                useradd, stdout=PIPE, stderr=PIPE)
            stdout, stderr = useradd_process.communicate()
            if useradd_process.returncode != 0:
                # TODO alert?
                raise RuntimeError("Creating {} user failed: {}".format(
                    volttron_agent_user, stderr))
            user_id_file.write(volttron_agent_user)
        return volttron_agent_user
Ejemplo n.º 4
0
 def _add_to_cache(self, device, topic, value):
     cached_time = get_utc_seconds_from_epoch()
     try:
         self._cache[device][topic] = (float(value), cached_time)
     except:
         _log.error(
             "Topic \"{}\" on device \"{}\" contained value that was not "
             "castable as float".format(topic, device))
Ejemplo n.º 5
0
 def _add_to_cache(self, device, topic, value):
     cached_time = get_utc_seconds_from_epoch()
     try:
         self._cache[device][topic] = (float(value), cached_time)
     except:
         _log.error(
             "Topic \"{}\" on device \"{}\" contained value that was not "
             "castable as float".format(topic, device))
Ejemplo n.º 6
0
 def stop_iam():
     _log.debug('Done publishing i am responses.')
     stop_timestamp = get_utc_seconds_from_epoch()
     self._pub_to_vc(iam_topic,
                     message=dict(status="FINISHED IAM",
                                  timestamp=stop_timestamp))
     agent_to_use.vip.pubsub.unsubscribe('pubsub', topics.BACNET_I_AM,
                                         handle_iam)
Ejemplo n.º 7
0
 def schedule(self, deadline, func, *args, **kwargs):
     if hasattr(deadline, 'timetuple'):
         #deadline = time.mktime(deadline.timetuple())
         deadline = utils.get_utc_seconds_from_epoch(deadline)
     event = ScheduledEvent(func, args, kwargs)
     heapq.heappush(self._schedule, (deadline, event))
     self._schedule_event.set()
     return event
Ejemplo n.º 8
0
    def schedule(cls, deadline, *args, **kwargs):   # pylint: disable=no-self-argument
        if hasattr(deadline, 'timetuple'):
            #deadline = time.mktime(deadline.timetuple())
            deadline = utils.get_utc_seconds_from_epoch(deadline)

        def decorate(method):
            annotate(method, list, 'core.schedule', (deadline, args, kwargs))
            return method
        return decorate
Ejemplo n.º 9
0
    def schedule(cls, deadline, *args, **kwargs):  # pylint: disable=no-self-argument
        if hasattr(deadline, 'timetuple'):
            # deadline = time.mktime(deadline.timetuple())
            deadline = utils.get_utc_seconds_from_epoch(deadline)

        def decorate(method):
            annotate(method, list, 'core.schedule', (deadline, args, kwargs))
            return method

        return decorate
Ejemplo n.º 10
0
 def publish_next(self):
     if self._publish_event is not None:
         self._publish_event.cancel()
     if self._gapps.connected:
         self._gapps.send("/queue/test", "foo bar")
     else:
         print("Not connected")
     now = get_aware_utc_now()
     print(utils.get_utc_seconds_from_epoch(now))
     next_update_time = now + datetime.timedelta(seconds=5)
     self._publish_event = self.core.schedule(next_update_time,
                                              self.publish_next)
     gevent.sleep(0.1)
Ejemplo n.º 11
0
    def health(self):
        """
        Returns a Status object as a dictionary.  This will be populated
        by the heartbeat from the external instance that this object is
        monitoring, unless it has been over 10 seconds since the instance
        has been reached.  In that case the health will be BAD.

        :return:
        """
        now = get_utc_seconds_from_epoch()
        self._health = Status.build(GOOD_STATUS, "Platform here!")
        # if now > self._last_time_verified_connection + 10:
        #     self._health = Status.build(
        #         BAD_STATUS,
        #         "Platform hasn't been reached in over 10 seconds.")
        return self._health.as_dict()
Ejemplo n.º 12
0
 def publish_next(self):
     _log.debug("Publishing next")
     if self._publish_event is not None:
         self._publish_event.cancel()
     self._gapps.send(self._test_topic, "foo bar")
     if self._gapps.connected:
         self._gapps.send("/queue/test", "foo bar")
     else:
         print("Not connected")
     now = get_aware_utc_now()
     print(utils.get_utc_seconds_from_epoch(now))
     next_update_time = now + datetime.timedelta(seconds=5)
     _log.debug(f'Scheduling nex time {next_update_time}')
     self._publish_event = self.core.schedule(next_update_time,
                                              self.publish_next)
     _log.debug(f'After scheduling next time {next_update_time}')
     gevent.sleep(0.1)
Ejemplo n.º 13
0
    def health(self):
        """
        Returns a Status object as a dictionary.  This will be populated
        by the heartbeat from the external instance that this object is
        monitoring, unless it has been over 10 seconds since the instance
        has been reached.  In that case the health will be BAD.

        :return:
        """
        now = get_utc_seconds_from_epoch()
        self._health = Status.build(
            GOOD_STATUS,
            "Platform here!"
        )
        # if now > self._last_time_verified_connection + 10:
        #     self._health = Status.build(
        #         BAD_STATUS,
        #         "Platform hasn't been reached in over 10 seconds.")
        return self._health.as_dict()
Ejemplo n.º 14
0
    def scrape(self, env, data):
        scrape_time = get_utc_seconds_from_epoch()
        keys_to_delete = defaultdict(list)
        if len(self._cache) > 0:
            result = "# TYPE volttron_data gauge\n"
            for device, device_topics in self._cache.iteritems():
                device_tags = device.replace("-", "_").split('/')
                for topic, value in device_topics.iteritems():
                    if value[1] + self._cache_time > scrape_time:
                        metric_props = re.split(self._tag_delimiter_re,
                                                topic.lower())
                        metric_tag_str = (
                            "campus=\"{}\",building=\"{}\","
                            "device=\"{}\",").format(*device_tags)
                        for i, prop in enumerate(metric_props):
                            metric_tag_str += "tag{}=\"{}\",".format(i, prop)
                        result += "{}{{{}topic=\"{}\"}} {}\n".format(
                            re.sub(" |/|-", "_", device), metric_tag_str,
                            topic.replace(" ", "_"), value[0])
                    else:
                        try:
                            keys_to_delete[device].append(topic)
                        except Exception as e:
                            _log.error("Could not delete stale topic")
                            _log.exception(e)
                        continue
        else:
            result = "#No Data to Scrape"
        for device, delete_topics in keys_to_delete.iteritems():
            for topic in delete_topics:
                del self._cache[device][topic]
        gzip_compress = zlib.compressobj(9, zlib.DEFLATED,
                                         zlib.MAX_WBITS | 16)
        data = gzip_compress.compress(result) + gzip_compress.flush()

        return "200 OK", base64.b64encode(data), [
            ('Content-Type', 'text/plain'),
            ('Content-Encoding', 'gzip')]
Ejemplo n.º 15
0
    def scrape(self, env, data):
        scrape_time = get_utc_seconds_from_epoch()
        keys_to_delete = defaultdict(list)
        if len(self._cache) > 0:
            result = "# TYPE volttron_data gauge\n"
            for device, device_topics in self._cache.iteritems():
                device_tags = device.replace("-", "_").split('/')
                for topic, value in device_topics.iteritems():
                    if value[1] + self._cache_time > scrape_time:
                        metric_props = re.split(self._tag_delimiter_re,
                                                topic.lower())
                        metric_tag_str = ("campus=\"{}\",building=\"{}\","
                                          "device=\"{}\",").format(
                                              *device_tags)
                        for i, prop in enumerate(metric_props):
                            metric_tag_str += "tag{}=\"{}\",".format(i, prop)
                        result += "{}{{{}topic=\"{}\"}} {}\n".format(
                            re.sub(" |/|-", "_", device), metric_tag_str,
                            topic.replace(" ", "_"), value[0])
                    else:
                        try:
                            keys_to_delete[device].append(topic)
                        except Exception as e:
                            _log.error("Could not delete stale topic")
                            _log.exception(e)
                        continue
        else:
            result = "#No Data to Scrape"
        for device, delete_topics in keys_to_delete.iteritems():
            for topic in delete_topics:
                del self._cache[device][topic]
        gzip_compress = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16)
        data = gzip_compress.compress(result) + gzip_compress.flush()

        return "200 OK", base64.b64encode(data), [('Content-Type',
                                                   'text/plain'),
                                                  ('Content-Encoding', 'gzip')]
Ejemplo n.º 16
0
    def publish_to_historian(self, to_publish_list):
        _log.debug("publish_to_historian number of items: {}".format(
            len(to_publish_list)))
        # Verify that we have actually gone through the historian_setup code
        # before we attempt to do anything else.
        if not self._initialized:
            self.historian_setup()
            if not self._initialized:
                return

        if self._wait_until is not None:
            ct = get_utc_seconds_from_epoch()
            if ct > self._wait_until:
                self._wait_until = None
            else:
                _log.debug('Waiting to attempt to write to database.')
                return
        try:
            if self._connection is None:
                self._connection = self.get_connection()

            cursor = self._connection.cursor()

            batch_data = []

            for row in to_publish_list:
                ts = utils.format_timestamp(row['timestamp'])
                source = row['source']
                topic = row['topic']
                value = row['value']
                meta = row['meta']

                # Handle the serialization of data here because we can't pass
                # an array as a string so we create a string from the value.
                if isinstance(value, list) or isinstance(value, dict):
                    value = dumps(value)

                if topic not in self._topic_set:
                    try:
                        cursor.execute(insert_topic_query(self._schema),
                                       (topic, ))
                    except ProgrammingError as ex:
                        if ex.args[0].startswith(
                                'DocumentAlreadyExistsException'):
                            self._topic_set.add(topic)
                        else:
                            _log.error(
                                "Unknown error during topic insert {} {}".
                                format(type(ex), ex.args))
                    else:
                        self._topic_set.add(topic)

                batch_data.append((ts, topic, source, value, meta))

            try:
                query = insert_data_query(self._schema)
                _log.debug("Inserting batch data: {}".format(batch_data))
                cursor.executemany(query, batch_data)
            except ProgrammingError as ex:
                _log.error(
                    "Invalid data detected during batch insert: {}".format(
                        ex.args))
                _log.debug("Attempting singleton insert.")
                insert = insert_data_query(self._schema)
                for id in range(len(batch_data)):
                    try:
                        batch = batch_data[id]
                        cursor.execute(insert, batch)
                    except ProgrammingError:
                        _log.debug('Invalid data not saved {}'.format(
                            to_publish_list[id]))
                        self.report_handled(to_publish_list[id])
                    except Exception as ex:
                        _log.error("Exception Type: {} ARGS: {}".format(
                            type(ex), ex.args))
                    else:
                        self.report_handled(to_publish_list[id])

            except Exception as ex:
                _log.error("Exception Type: {} ARGS: {}".format(
                    type(ex), ex.args))

            else:
                self.report_all_handled()
        except TypeError as ex:
            _log.error("AFTER EXCEPTION: {} ARGS: {}".format(
                type(ex), ex.args))
        except Exception as ex:
            _log.error("Unknown Exception {} {}".format(type(ex), ex.args))

        finally:
            if cursor is not None:
                cursor.close()
                cursor = None
Ejemplo n.º 17
0
 def _schedule_callback(self, deadline, callback):
     deadline = utils.get_utc_seconds_from_epoch(deadline)
     heapq.heappush(self._schedule,
                    (deadline, self.get_tie_breaker(), callback))
     if self._schedule_event:
         self._schedule_event.set()
Ejemplo n.º 18
0
    def publish_to_historian(self, to_publish_list):
        _log.debug("publish_to_historian number of items: {}".format(
            len(to_publish_list)))
        # Verify that we have actually gone through the historian_setup code
        # before we attempt to do anything else.
        if not self._initialized:
            self.historian_setup()
            if not self._initialized:
                return

        if self._wait_until is not None:
            ct = get_utc_seconds_from_epoch()
            if ct > self._wait_until:
                self._wait_until = None
            else:
                _log.debug('Waiting to attempt to write to database.')
                return
        try:
            if self._connection is None:
                self._connection = self.get_connection()

            cursor = self._connection.cursor()

            batch_data = []

            for row in to_publish_list:
                ts = utils.format_timestamp(row['timestamp'])
                source = row['source']
                topic = row['topic']
                value = row['value']
                meta = row['meta']

                # Handle the serialization of data here because we can't pass
                # an array as a string so we create a string from the value.
                if isinstance(value, list) or isinstance(value, dict):
                    value = dumps(value)

                if topic not in self._topic_set:
                    try:
                        cursor.execute(insert_topic_query(self._schema),
                                       (topic,))
                    except ProgrammingError as ex:
                        if ex.args[0].startswith(
                                'DocumentAlreadyExistsException'):
                            self._topic_set.add(topic)
                        else:
                            _log.error(
                                "Unknown error during topic insert {} {}".format(
                                    type(ex), ex.args
                                ))
                    else:
                        self._topic_set.add(topic)

                batch_data.append(
                    (ts, topic, source, value, meta)
                )

            try:
                query = insert_data_query(self._schema)
                _log.debug("Inserting batch data: {}".format(batch_data))
                cursor.executemany(query, batch_data)
            except ProgrammingError as ex:
                _log.error(
                    "Invalid data detected during batch insert: {}".format(
                        ex.args))
                _log.debug("Attempting singleton insert.")
                insert = insert_data_query(self._schema)
                for id in range(len(batch_data)):
                    try:
                        batch = batch_data[id]
                        cursor.execute(insert, batch)
                    except ProgrammingError:
                        _log.debug('Invalid data not saved {}'.format(
                            to_publish_list[id]
                        ))
                        self.report_handled(to_publish_list[id])
                    except Exception as ex:
                        _log.error(
                            "Exception Type: {} ARGS: {}".format(type(ex),
                                                                 ex.args))
                    else:
                        self.report_handled(to_publish_list[id])

            except Exception as ex:
                _log.error(
                    "Exception Type: {} ARGS: {}".format(type(ex), ex.args))

            else:
                self.report_all_handled()
        except TypeError as ex:
            _log.error(
                "AFTER EXCEPTION: {} ARGS: {}".format(type(ex), ex.args))
        except Exception as ex:
            _log.error(
                "Unknown Exception {} {}".format(type(ex), ex.args)
            )

        finally:
            if cursor is not None:
                cursor.close()
                cursor = None
Ejemplo n.º 19
0
    def start_bacnet_scan(self,
                          iam_topic,
                          proxy_identity,
                          low_device_id=None,
                          high_device_id=None,
                          target_address=None,
                          scan_length=5,
                          instance_address=None,
                          instance_serverkey=None):
        """This function is a wrapper around the bacnet proxy scan.
        """
        agent_to_use = self
        if instance_address is not None:
            agent_to_use = build_agent(address=instance_address,
                                       identity="proxy_bacnetplatform",
                                       publickey=self.core.publickey,
                                       secretkey=self.core.secretkey,
                                       serverkey=instance_serverkey)

        if proxy_identity not in agent_to_use.vip.peerlist().get(timeout=5):
            raise Unreachable(
                "Can't reach agent identity {}".format(proxy_identity))
        _log.info('Starting bacnet_scan with who_is request to {}'.format(
            proxy_identity))

        def handle_iam(peer, sender, bus, topic, headers, message):
            proxy_identity = sender
            address = message['address']
            device_id = message['device_id']
            bn = BACnetReader(agent_to_use.vip.rpc, proxy_identity)
            message['device_name'] = bn.read_device_name(address, device_id)
            message['device_description'] = bn.read_device_description(
                address, device_id)

            self._pub_to_vc(iam_topic, message=message)

        def stop_iam():
            _log.debug('Done publishing i am responses.')
            stop_timestamp = get_utc_seconds_from_epoch()
            self._pub_to_vc(iam_topic,
                            message=dict(status="FINISHED IAM",
                                         timestamp=stop_timestamp))
            agent_to_use.vip.pubsub.unsubscribe('pubsub', topics.BACNET_I_AM,
                                                handle_iam)

        agent_to_use.vip.pubsub.subscribe('pubsub', topics.BACNET_I_AM,
                                          handle_iam)

        timestamp = get_utc_seconds_from_epoch()

        self._pub_to_vc(iam_topic,
                        message=dict(status="STARTED IAM",
                                     timestamp=timestamp))

        agent_to_use.vip.rpc.call(
            proxy_identity,
            "who_is",
            low_device_id=low_device_id,
            high_device_id=high_device_id,
            target_address=target_address).get(timeout=5.0)

        gevent.spawn_later(float(scan_length), stop_iam)
Ejemplo n.º 20
0
    def publish_to_historian(self, to_publish_list):
        _log.debug("publish_to_historian number of items: {}".format(
            len(to_publish_list)))
        start_time = get_utc_seconds_from_epoch()
        if self._client is None:
            success = self._establish_client_connection()
            if not success:
                return

        try:
            cursor = self._client.cursor()

            batch_data = []

            for row in to_publish_list:
                ts = utils.format_timestamp(row['timestamp'])
                source = row['source']
                topic = row['topic']
                value = row['value']
                meta = row['meta']
                topic_lower = topic.lower()

                # Handle the serialization of data here because we can't pass
                # an array as a string so we create a string from the value.
                if isinstance(value, list) or isinstance(value, dict):
                    value = dumps(value)

                if topic_lower not in self._topic_meta:
                    try:
                        cursor.execute(insert_topic_query(self._schema, self._topic_table),
                                       (topic, meta))
                    except ProgrammingError as ex:
                        if ex.args[0].startswith(
                                'SQLActionException[DuplicateKeyException'):
                            self._topic_meta[topic_lower] = meta
                        else:
                            _log.error(repr(ex))
                            _log.error(
                                "Unknown error during topic insert {} {}".format(
                                    type(ex), ex.args
                                ))
                    else:
                        self._topic_meta[topic_lower] = meta
                else:
                    # check if metadata matches
                    old_meta = self._topic_meta.get(topic_lower)
                    if not old_meta:
                        old_meta = {}
                    if set(old_meta.items()) != set(meta.items()):
                        _log.debug(
                            'Updating meta for topic: {} {}'.format(topic,
                                                                    meta))
                        self._topic_meta[topic_lower] = meta
                        cursor.execute(update_topic_query(self._schema, self._topic_table),
                                       (meta, topic))

                batch_data.append(
                    (ts, topic, source, value, meta)
                )

            try:
                query = insert_data_query(self._schema, self._data_table)
                # _log.debug("Inserting batch data: {}".format(batch_data))
                results = cursor.executemany(query, batch_data)

                index = 0
                failures = []
                for r in results:
                    if r['rowcount'] != 1:
                        failures.append(index)
                    index += 1

                if failures:
                    for findex in failures:
                        data = batch_data[findex]
                        _log.error("Failed to insert data {}".format(data))
                        self.report_handled(to_publish_list[findex])

            except ProgrammingError as ex:
                _log.error(
                    "Invalid data detected during batch insert: {}".format(
                        ex.args))
                _log.debug("Attempting singleton insert.")
                insert = insert_data_query(self._schema, self._data_table)
                for id in range(len(batch_data)):
                    try:
                        batch = batch_data[id]
                        cursor.execute(insert, batch)
                    except ProgrammingError:
                        _log.debug('Invalid data not saved {}'.format(
                            batch
                        ))
                    except Exception as ex:
                        _log.error(repr(ex))
                    else:
                        self.report_handled(to_publish_list[id])

            except Exception as ex:
                _log.error(
                    "Exception Type: {} ARGS: {}".format(type(ex), ex.args))

            else:
                self.report_all_handled()
        except TypeError as ex:
            _log.error(repr(ex))
            _log.error(
                "AFTER EXCEPTION: {} ARGS: {}".format(type(ex), ex.args))
        except Exception as ex:
            _log.error(repr(ex))
            _log.error(
                "Unknown Exception {} {}".format(type(ex), ex.args)
            )

        finally:
            if cursor is not None:
                cursor.close()
                cursor = None
Ejemplo n.º 21
0
 def schedule(self, deadline, func, *args, **kwargs):
     deadline = utils.get_utc_seconds_from_epoch(deadline)
     event = ScheduledEvent(func, args, kwargs)
     heapq.heappush(self._schedule, (deadline, event))
     self._schedule_event.set()
     return event
Ejemplo n.º 22
0
 def _schedule_callback(self, deadline, callback):
     deadline = utils.get_utc_seconds_from_epoch(deadline)
     heapq.heappush(self._schedule, (deadline, callback))
     self._schedule_event.set()
Ejemplo n.º 23
0
 def schedule(self, deadline, func, *args, **kwargs):
     deadline = utils.get_utc_seconds_from_epoch(deadline)
     event = ScheduledEvent(func, args, kwargs)
     heapq.heappush(self._schedule, (deadline, event))
     self._schedule_event.set()
     return event
Ejemplo n.º 24
0
    def publish_to_historian(self, to_publish_list):
        # _log.debug("publish_to_historian number of items: {}".format(
        #     len(to_publish_list)))
        start_time = get_utc_seconds_from_epoch()
        if self._client is None:
            success = self._establish_client_connection()
            if not success:
                return

        try:
            cursor = self._client.cursor()

            batch_data = []

            for row in to_publish_list:
                ts = utils.format_timestamp(row['timestamp'])
                source = row['source']
                topic = self.get_renamed_topic(row['topic'])
                value = row['value']
                meta = row['meta']

                # Handle the serialization of data here because we can't pass
                # an array as a string so we create a string from the value.
                if isinstance(value, list) or isinstance(value, dict):
                    value = dumps(value)

                if topic not in self._topic_set:
                    try:
                        cursor.execute(insert_topic_query(self._schema),
                                       (topic,))
                    except ProgrammingError as ex:
                        if ex.args[0].startswith(
                                'SQLActionException[DuplicateKeyException'):
                            self._topic_set.add(topic)
                        else:
                            _log.error(repr(ex))
                            _log.error(
                                "Unknown error during topic insert {} {}".format(
                                    type(ex), ex.args
                                ))
                    else:
                        self._topic_set.add(topic)

                batch_data.append(
                    (ts, topic, source, value, meta)
                )

            try:
                query = insert_data_query(self._schema)
                # _log.debug("Inserting batch data: {}".format(batch_data))
                results = cursor.executemany(query, batch_data)

                index = 0
                failures = []
                for r in results:
                    if r['rowcount'] == -1:
                        failures.append(index)
                    index += 1

                if failures:
                    for findex in failures:
                        data = batch_data[findex]
                        _log.error("Failed to insert data {}".format(data))
                        self.report_handled(to_publish_list[findex])

            except ProgrammingError as ex:
                _log.error(
                    "Invalid data detected during batch insert: {}".format(
                        ex.args))
                _log.debug("Attempting singleton insert.")
                insert = insert_data_query(self._schema)
                for id in range(len(batch_data)):
                    try:
                        batch = batch_data[id]
                        cursor.execute(insert, batch)
                    except ProgrammingError:
                        _log.debug('Invalid data not saved {}'.format(
                            to_publish_list[id]
                        ))
                        self.report_handled(to_publish_list[id])
                    except Exception as ex:
                        _log.error(repr(ex))
                    else:
                        self.report_handled(to_publish_list[id])

            except Exception as ex:
                _log.error(
                    "Exception Type: {} ARGS: {}".format(type(ex), ex.args))

            else:
                self.report_all_handled()
        except TypeError as ex:
            _log.error(repr(ex))
            _log.error(
                "AFTER EXCEPTION: {} ARGS: {}".format(type(ex), ex.args))
        except Exception as ex:
            _log.error(repr(ex))
            _log.error(
                "Unknown Exception {} {}".format(type(ex), ex.args)
            )

        finally:
            if cursor is not None:
                cursor.close()
                cursor = None