def publish_metrics(): try: # You will need to use Local Resource Access to map the hosts /proc to a directory accessible in the lambda ps.PROCFS_PATH = os.environ['PROCFS_PATH'] core_name = os.environ['AWS_IOT_THING_NAME'] topic = "$aws/things/" + core_name + "/defender/metrics/json" sample_interval_seconds = os.environ['SAMPLE_INTERVAL_SECONDS'] if sample_interval_seconds < MIN_INTERVAL_SECONDS: sample_interval_seconds = MIN_INTERVAL_SECONDS print "Collector running on device: " + core_name print "Metrics topic: " + topic print "Sampling interval: " + sample_interval_seconds + " seconds" metrics_collector = collector.Collector(short_metrics_names=False) while True: metric = metrics_collector.collect_metrics() client.publish(topic=topic, payload=metric.to_json_string()) sleep(float(sample_interval_seconds)) except Exception as e: print "Error: " + str(e) return
def test_collector_custom_metrics(mock_cpu_percent, cpu_percent): mock_cpu_percent.return_value = cpu_percent new_collector = collector.Collector(short_metrics_names=False, use_custom_metrics=True) metrics_output = new_collector.collect_metrics() assert metrics_output.cpu_metrics["number"] == 25.65
def main(): # Read in command-line parameters args = parse_args() io.init_logging(getattr(io.LogLevel, args.verbosity), 'stderr') if not args.dry_run: client_id = "" thing_name = "" if not args.client_id: client_id = gethostname() else: client_id = args.client_id if not args.thing_name: thing_name = client_id else: thing_name = args.thing_name iot_client = IoTClientWrapper(args.endpoint, args.root_ca_path, args.certificate_path, args.private_key_path, client_id, args.signing_region, args.proxy_host, args.proxy_port, args.use_websocket) iot_client.connect() # client_id must match a registered thing name in your account topic = "$aws/things/" + thing_name + "/defender/metrics/" + args.format # Subscribe to the accepted/rejected topics to indicate status of published metrics reports iot_client.subscribe(topic + "/accepted", custom_callback) iot_client.subscribe(topic + "/rejected", custom_callback) sample_rate = args.upload_interval # Collector samples metrics from the system, it can track the previous metric to generate deltas coll = collector.Collector(args.short_tags, args.custom_metrics) metric = None first_sample = True # don't publish first sample, so we can accurately report delta metrics while True: metric = coll.collect_metrics() if args.dry_run: print(metric.to_json_string(pretty_print=True)) if args.format == 'cbor': with open("cbor_metrics", "w+b") as outfile: outfile.write(bytearray(metric.to_cbor())) else: if first_sample: first_sample = False elif args.format == "cbor": iot_client.publish(topic, bytearray(metric.to_cbor())) else: iot_client.publish(topic, metric.to_json_string()) sleep(float(sample_rate))
def start_metrics_collection( region_name, cp_endpoint_url, client_id, iot_client, topic, sample_rate ): # Collector samples metrics from the system, it can track the previous metric to generate deltas coll = collector.Collector(False) metric = None first_sample = ( True # don't publish first sample, so we can accurately report delta metrics ) while True: logging.info("collecting metrics") metric = coll.collect_metrics() if first_sample: first_sample = False else: session = boto3.session.Session(region_name=region_name) # This is a cheap hack to ensure we reset the creds every so often, # since the temporary creds expire. SDK doesn't seem to have a way # to reset these creds other than periodically updating these creds # by calling iot_client.configureIAMCredentials or subclassing the # MQTT client for listening to the onOffline callback. Details in # this SIM: https://t.corp.amazon.com/issues/SDK-15249/communication credentials = session.get_credentials() iot_client.configureIAMCredentials( credentials.access_key, credentials.secret_key, credentials.token ) report_id = metric._v1_metrics().get("header").get("report_id") iot_client.publish(topic=topic, payload=metric.to_json_string(), QoS=0) logging.info("Published report with report_id: {}".format(report_id)) max_iterations = 5 while max_iterations > 0: # Sleep 10s to allow receiving a response for the latest publish. sleep(10) max_iterations = max_iterations - 1 if latest_accepted_report_id == report_id: logging.info( "Received successful ack for reportId: {}".format( latest_accepted_report_id ) ) break logging.info( "Republishing report with reportId: {}, last accepted reportId: {}".format( report_id, latest_accepted_report_id ) ) iot_client.publish(topic=topic, payload=metric.to_json_string(), QoS=0) sleep(float(sample_rate))
def test_collector_collect_network_stats( mock_net_connections, mock_io_counters, mock_if_addrs, net_connections, if_addrs, net_io_counters, ): mock_net_connections.return_value = net_connections mock_io_counters.return_value = net_io_counters mock_if_addrs.return_value = if_addrs new_collector = collector.Collector(short_metrics_names=False) metrics_output = new_collector.collect_metrics() assert len(metrics_output.network_stats) == 0
def publish_metrics(): # thing name must match a registered thing name in your account topic = "$aws/things/" + THING_NAME + "/defender/metrics/json" try: metrics_collector = collector.Collector(short_metrics_names=False) while True: metric = metrics_collector.collect_metrics() client.publish(topic=topic, payload=metric.to_json_string()) sleep(float(SAMPLE_RATE_SECONDS)) except Exception as e: print "Error: " + str(e) return
def test_collector_listening_ports(self): self.__use_mock_net_connections(self.__setup_net_connections()) self.__use_mock_net_if_addrs(self.__setup_if_addrs()) self.__use_mock_net_io_counters() new_collector = collector.Collector(short_metrics_names=False) metrics_output = new_collector.collect_metrics() assert len(metrics_output.listening_ports('UDP')) == 3 assert metrics_output.listening_ports('UDP')[0]['port'] == 66666 assert metrics_output.listening_ports('UDP')[1]['port'] == 77777 assert metrics_output.listening_ports('UDP')[2]['port'] == 88888 assert len(metrics_output.listening_ports('TCP')) == 1 assert metrics_output.listening_ports('TCP')[0]['port'] == 44444
def test_collector_collect_network_stats(self): self.__use_mock_net_connections(self.__setup_net_connections()) self.__use_mock_net_if_addrs(self.__setup_if_addrs()) self.__use_mock_net_io_counters() self.mock_net_io_counters_result.bytes_recv = 10000 self.mock_net_io_counters_result.bytes_sent = 20000 self.mock_net_io_counters_result.packets_recv = 30000 self.mock_net_io_counters_result.packets_sent = 40000 new_collector = collector.Collector(short_metrics_names=False) metrics_output = new_collector.collect_metrics() assert metrics_output.network_stats['bytes_in'] == 10000 assert metrics_output.network_stats['packets_in'] == 30000 assert metrics_output.network_stats['bytes_out'] == 20000 assert metrics_output.network_stats['packets_out'] == 40000
def test_collector_collect_network_stats( mock_net_connections, mock_io_counters, mock_if_addrs, net_connections, if_addrs, net_io_counters, ): mock_net_connections.return_value = net_connections mock_io_counters.return_value = net_io_counters mock_if_addrs.return_value = if_addrs new_collector = collector.Collector(short_metrics_names=False) metrics_output = new_collector.collect_metrics() assert metrics_output.network_stats["bytes_in"] == 10000 assert metrics_output.network_stats["packets_in"] == 30000 assert metrics_output.network_stats["bytes_out"] == 20000 assert metrics_output.network_stats["packets_out"] == 40000
def test_collector_net_connections( mock_net_connections, mock_io_counters, mock_if_addrs, net_connections, if_addrs, net_io_counters, ): mock_net_connections.return_value = net_connections mock_io_counters.return_value = net_io_counters mock_if_addrs.return_value = if_addrs new_collector = collector.Collector(short_metrics_names=False) metrics_output = new_collector.collect_metrics() assert metrics_output.network_connections[0][ "remote_addr"] == "11.0.0.1:123" assert metrics_output.network_connections[0]["local_interface"] == "em1" assert metrics_output.network_connections[0]["local_port"] == 11111 assert metrics_output.network_connections[1][ "remote_addr"] == "11.0.0.2:234" assert metrics_output.network_connections[1]["local_interface"] == "em1" assert metrics_output.network_connections[1]["local_port"] == 22222 assert metrics_output.network_connections[2][ "remote_addr"] == "11.0.0.3:345" assert metrics_output.network_connections[2]["local_interface"] == "em1" assert metrics_output.network_connections[2]["local_port"] == 33333 assert metrics_output.network_connections[3][ "remote_addr"] == "11.0.0.5:567" assert metrics_output.network_connections[3]["local_interface"] == "em1" assert metrics_output.network_connections[3]["local_port"] == 44444 assert metrics_output.network_connections[4][ "remote_addr"] == "11.0.0.6:678" assert metrics_output.network_connections[4]["local_interface"] == "em1" assert metrics_output.network_connections[4]["local_port"] == 66666 assert metrics_output.network_connections[5][ "remote_addr"] == "11.0.0.7:789" assert metrics_output.network_connections[5]["local_interface"] is None assert metrics_output.network_connections[5]["local_port"] == 77777
def test_collector_listening_ports( mock_net_connections, mock_io_counters, mock_if_addrs, net_connections, if_addrs, net_io_counters, ): mock_net_connections.return_value = net_connections mock_io_counters.return_value = net_io_counters mock_if_addrs.return_value = if_addrs new_collector = collector.Collector(short_metrics_names=False) metrics_output = new_collector.collect_metrics() assert len(metrics_output.listening_ports("UDP")) == 3 assert metrics_output.listening_ports("UDP")[0]["port"] == 66666 assert metrics_output.listening_ports("UDP")[1]["port"] == 77777 assert metrics_output.listening_ports("UDP")[2]["port"] == 88888 assert len(metrics_output.listening_ports("TCP")) == 1 assert metrics_output.listening_ports("TCP")[0]["port"] == 44444
def test_collector_net_connections(self): self.__use_mock_net_connections(self.__setup_net_connections()) self.__use_mock_net_if_addrs(self.__setup_if_addrs()) self.__use_mock_net_io_counters() new_collector = collector.Collector(short_metrics_names=False) metrics_output = new_collector.collect_metrics() assert metrics_output.network_connections[0][ 'remote_addr'] == '11.0.0.1:123' assert metrics_output.network_connections[0][ 'local_interface'] == 'em1' assert metrics_output.network_connections[0]['local_port'] == 11111 assert metrics_output.network_connections[1][ 'remote_addr'] == '11.0.0.2:234' assert metrics_output.network_connections[1][ 'local_interface'] == 'em1' assert metrics_output.network_connections[1]['local_port'] == 22222 assert metrics_output.network_connections[2][ 'remote_addr'] == '11.0.0.3:345' assert metrics_output.network_connections[2][ 'local_interface'] == 'em1' assert metrics_output.network_connections[2]['local_port'] == 33333 assert metrics_output.network_connections[3][ 'remote_addr'] == '11.0.0.5:567' assert metrics_output.network_connections[3][ 'local_interface'] == 'em1' assert metrics_output.network_connections[3]['local_port'] == 44444 assert metrics_output.network_connections[4][ 'remote_addr'] == '11.0.0.6:678' assert metrics_output.network_connections[4][ 'local_interface'] == 'em1' assert metrics_output.network_connections[4]['local_port'] == 66666 assert metrics_output.network_connections[5][ 'remote_addr'] == '11.0.0.7:789' assert metrics_output.network_connections[5]['local_interface'] == None assert metrics_output.network_connections[5]['local_port'] == 77777