def histogram(self, metric, value=1, timestamp=None, tags=None): timer(257, time.clock(), 's') metric_name = '%s.%s' % (self.metric_prefix, metric) timestamp = timestamp or int(time.time()) _tags = ','.join(sorted(tags)) self.histograms[metric_name][_tags][timestamp].append(value) timer(257, time.clock(), 'e')
def _initialize(self): timer(243, time.clock(), 's') self.counts = defaultdict( lambda: defaultdict(lambda: defaultdict(int))) self.histograms = defaultdict( lambda: defaultdict(lambda: defaultdict(list))) timer(243, time.clock(), 'e')
def increment(self, metric, value=1, timestamp=None, tags=None): timer(251, time.clock(), 's') metric_name = '%s.%s' % (self.metric_prefix, metric) timestamp = timestamp or int(time.time()) _tags = ','.join(sorted(tags)) self.counts[metric_name][_tags][timestamp] += value timer(251, time.clock(), 'e')
def process_bytes(_bytes, tags, timestamp): timer(236, 'stats.histogram', 'c') stats.histogram("bytes.per_request", int(_bytes), tags=tags, timestamp=timestamp) timer(237, 'stats.increment', 'c') stats.increment("bytes.total", int(_bytes), tags=tags, timestamp=timestamp)
def process_packets(packets, tags, timestamp): timer(231, 'stats.histogram', 'c') stats.histogram("packets.per_request", int(packets), tags=tags, timestamp=timestamp) timer(232, 'stats.increment', 'c') stats.increment("packets.total", int(packets), tags=tags, timestamp=timestamp)
def compute_node_ip(events): timer(58, time.clock(), 's') ip_count = Counter() timer(58, time.clock(), 'e') timer(59, time.clock(), 'fs') for event in events: timer(59, time.clock(), 'fex') timer(61, time.clock(), 'is') src_ip, dest_ip = event['message'].split(" ", 5)[3:5] if len(src_ip) > 1 and len(dest_ip) > 1: # account for '-' timer(61, time.clock(), 'iex') timer(62, time.clock(), 's') ip_count[src_ip] += 1 ip_count[dest_ip] += 1 timer(62, time.clock(), 'e') timer(61, time.clock(), 'ie') timer(59, time.clock(), 'fe') timer(65, time.clock(), 'is') most_comm = ip_count.most_common() if most_comm: timer(65, time.clock(), 'iex') timer(66, time.clock(), 'is') if most_comm[0][1] > 1: # we have several events timer(66, time.clock(), 'iex') try: return ip_count.most_common()[0][0] finally: timer(65, time.clock(), 'ie') timer(66, time.clock(), 'ie') timer(66, time.clock(), 'ie') timer(65, time.clock(), 'ie') timer(68, time.clock(), 's') try: return 'unknown' finally: timer(68, time.clock(), 'e')
def lambda_handler(event, context): timer(321, time.clock(), 's') # event is a dict containing a base64 string gzipped event = event['awslogs']['data'].decode('base64') event = json.loads(event) #event = json.loads(gzip.GzipFile(fileobj=StringIO(event['awslogs']['data'].decode('base64'))).read()) #function_arn = context.invoked_function_arn function_arn = 'arn:aws:lambda:us-east-1:1234123412:function:VPCFlowLogs' # 'arn:aws:lambda:us-east-1:1234123412:function:VPCFlowLogs' region, account = function_arn.split(':', 5)[3:5] tags = ["region:%s" % region, "aws_account:%s" % account] unsupported_messages = 0 timer(321, time.clock(), 'e') timer(334, 'compute_node_ip', 'c') node_ip = compute_node_ip(event['logEvents']) timer(336, time.clock(), 'fs') for event in event['logEvents']: timer(336, time.clock(), 'fex') timer(337, time.clock(), 'is') message = "2 123456789010 eni-abc123de 172.31.16.139 172.31.16.21 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK" #message = event['message'] if message[0] != "2": timer(337, time.clock(), 'iex') timer(340, time.clock(), 's') unsupported_messages += 1 timer(340, time.clock(), 'e') try: continue finally: timer(337, time.clock(), 'ie') timer(337, time.clock(), 'ie') timer(342, time.clock(), 's') timestamp = event['timestamp'] / 1000 timer(342, time.clock(), 'e') timer(343, 'process_message', 'c') process_message(message, tags, timestamp, node_ip) timer(336, time.clock(), 'fe') timer(345, time.clock(), 'is') if unsupported_messages: timer(345, time.clock(), 'iex') timer(346, time.clock(), 's') print("Unsupported vpc flowlog message type, please contact Datadog") timer(346, time.clock(), 'e') timer(347, 'stats.increment', 'c') stats.increment("unsupported_message", value=unsupported_messages, tags=tags) timer(345, time.clock(), 'ie') timer(351, 'stats.flush', 'c') stats.flush()
def process_message(message, tags, timestamp, node_ip): timer(35, time.clock(), 'is') version, account_id, interface_id, srcaddr, dstaddr, srcport, dstport, protocol, packets, _bytes, start, end, action, log_status = message.split( " ") detailed_tags = [ "interface_id:%s" % interface_id, "protocol:%s" % protocol_id_to_name(protocol), "ip:%s" % node_ip, ] + tags if srcaddr == node_ip: timer(35, time.clock(), 'iex') timer(43, time.clock(), 's') detailed_tags.append("direction:outbound") timer(43, time.clock(), 'e') timer(35, time.clock(), 'ie') timer(44, time.clock(), 'is') if dstaddr == node_ip: timer(44, time.clock(), 'iex') timer(45, time.clock(), 's') detailed_tags.append("direction:inbound") timer(45, time.clock(), 'e') timer(44, time.clock(), 'ie') timer(47, 'process_log_status', 'c') process_log_status(log_status, detailed_tags, timestamp) timer(48, time.clock(), 'is') if log_status == 'NODATA': timer(48, time.clock(), 'iex') return timer(48, time.clock(), 'ie') timer(51, 'process_action', 'c') process_action(action, detailed_tags, timestamp) timer(52, 'process_duration', 'c') process_duration(start, end, detailed_tags, timestamp) timer(53, 'process_packets', 'c') process_packets(packets, detailed_tags, timestamp) timer(54, 'process_bytes', 'c') process_bytes(_bytes, detailed_tags, timestamp)
def flush(self): timer(263, time.clock(), 's') percentiles_to_submit = [0, 50, 90, 95, 99, 100] series = [] timer(263, time.clock(), 'e') timer(265, time.clock(), 'fs') for metric_name, count_payload in self.counts.iteritems(): timer(265, time.clock(), 'fex') timer(266, time.clock(), 'fs') for tag_set, datapoints in count_payload.iteritems(): timer(266, time.clock(), 'fex') timer(267, time.clock(), 's') points = [(ts, val) for ts, val in datapoints.iteritems()] series.append({ 'metric': metric_name, 'points': points, 'type': 'count', 'tags': tag_set.split(','), }) timer(267, time.clock(), 'e') timer(266, time.clock(), 'fe') timer(265, time.clock(), 'fe') timer(277, time.clock(), 'fs') for metric_name, histogram_payload in self.histograms.iteritems(): timer(277, time.clock(), 'fex') timer(278, time.clock(), 'fs') for tag_set, datapoints in histogram_payload.iteritems(): timer(278, time.clock(), 'fex') timer(279, time.clock(), 's') percentiles = defaultdict(list) timer(279, time.clock(), 'e') timer(280, time.clock(), 'fs') for ts, values in datapoints.iteritems(): timer(280, time.clock(), 'fex') timer(281, time.clock(), 's') values.sort() total_points = len(values) timer(281, time.clock(), 'e') timer(283, time.clock(), 'fs') for pct in percentiles_to_submit: timer(283, time.clock(), 'fex') timer(284, time.clock(), 's') percentiles[pct].append( (ts, values[max(0, int( (pct - 1) * total_points / 100))])) timer(284, time.clock(), 'e') timer(283, time.clock(), 'fe') timer(280, time.clock(), 'fe') timer(286, time.clock(), 'fs') for pct, points in percentiles.iteritems(): timer(286, time.clock(), 'fex') timer(288, time.clock(), 'is') metric_suffix = 'p%s' % pct if pct == 0: timer(288, time.clock(), 'iex') timer(289, time.clock(), 's') metric_suffix = 'min' timer(289, time.clock(), 'e') timer(288, time.clock(), 'ie') timer(290, time.clock(), 'is') if pct == 50: timer(290, time.clock(), 'iex') timer(291, time.clock(), 's') metric_suffix = 'median' timer(291, time.clock(), 'e') timer(290, time.clock(), 'ie') timer(292, time.clock(), 'is') if pct == 100: timer(292, time.clock(), 'iex') timer(293, time.clock(), 's') metric_suffix = 'max' timer(293, time.clock(), 'e') timer(292, time.clock(), 'ie') timer(294, time.clock(), 's') series.append({ 'metric': '%s.%s' % (metric_name, metric_suffix), 'points': points, 'type': 'gauge', 'tags': tag_set.split(','), }) timer(286, time.clock(), 'fe') timer(278, time.clock(), 'fe') timer(277, time.clock(), 'fe') timer(303, 'stats._initialize', 'c') self._initialize() timer(305, time.clock(), 's') metrics_dict = { 'series': series, } creds = urllib.urlencode(datadog_keys) data = json.dumps(metrics_dict) url = '%s?%s' % (datadog_keys.get( 'api_host', 'https://app.datadoghq.com/api/v1/series'), creds) #req = urllib2.Request(url, data, {'Content-Type': 'application/json'}) #response = urllib2.urlopen(req) print('INFO Submitted data with status {}') #print('INFO Submitted data with status {}'.format(response.getcode())) timer(305, time.clock(), 'e')
def __init__(self): timer(247, time.clock(), 's') self._initialize() self.metric_prefix = "aws.vpc.flowlogs" timer(247, time.clock(), 'e')
def process_duration(start, end, tags, timestamp): timer(227, 'stats.histogram', 'c') stats.histogram("duration.per_request", int(int(end) - int(start)), tags=tags, timestamp=timestamp)
def process_action(action, tags, timestamp): timer(223, 'stats.increment', 'c') stats.increment("action", tags=["action:%s" % action] + tags, timestamp=timestamp)
def process_log_status(log_status, tags, timestamp): timer(219, 'stats.increment', 'c') stats.increment("log_status", tags=["status:%s" % log_status] + tags, timestamp=timestamp)
def protocol_id_to_name(protocol): timer(72, time.clock(), 'is') if protocol == '-': timer(72, time.clock(), 'iex') return protocol timer(72, time.clock(), 'ie') timer(74, time.clock(), 's') protocol_map = { 0: "HOPOPT", 1: "ICMP", 2: "IGMP", 3: "GGP", 4: "IPv4", 5: "ST", 6: "TCP", 7: "CBT", 8: "EGP", 9: "IGP", 10: "BBN-RCC-MON", 11: "NVP-II", 12: "PUP", 13: "ARGUS", 14: "EMCON", 15: "XNET", 16: "CHAOS", 17: "UDP", 18: "MUX", 19: "DCN-MEAS", 20: "HMP", 21: "PRM", 22: "XNS-IDP", 23: "TRUNK-1", 24: "TRUNK-2", 25: "LEAF-1", 26: "LEAF-2", 27: "RDP", 28: "IRTP", 29: "ISO-TP4", 30: "NETBLT", 31: "MFE-NSP", 32: "MERIT-INP", 33: "DCCP", 34: "3PC", 35: "IDPR", 36: "XTP", 37: "DDP", 38: "IDPR-CMTP", 39: "TP++", 40: "IL", 41: "IPv6", 42: "SDRP", 43: "IPv6-Route", 44: "IPv6-Frag", 45: "IDRP", 46: "RSVP", 47: "GRE", 48: "DSR", 49: "BNA", 50: "ESP", 51: "AH", 52: "I-NLSP", 53: "SWIPE", 54: "NARP", 55: "MOBILE", 56: "TLSP", 57: "SKIP", 58: "IPv6-ICMP", 59: "IPv6-NoNxt", 60: "IPv6-Opts", 62: "CFTP", 64: "SAT-EXPAK", 65: "KRYPTOLAN", 66: "RVD", 67: "IPPC", 69: "SAT-MON", 70: "VISA", 71: "IPCV", 72: "CPNX", 73: "CPHB", 74: "WSN", 75: "PVP", 76: "BR-SAT-MON", 77: "SUN-ND", 78: "WB-MON", 79: "WB-EXPAK", 80: "ISO-IP", 81: "VMTP", 82: "SECURE-VMTP", 83: "VINES", 84: "TTP", 84: "IPTM", 85: "NSFNET-IGP", 86: "DGP", 87: "TCF", 88: "EIGRP", 89: "OSPFIGP", 90: "Sprite-RPC", 91: "LARP", 92: "MTP", 93: "AX.25", 94: "IPIP", 95: "MICP", 96: "SCC-SP", 97: "ETHERIP", 98: "ENCAP", 100: "GMTP", 101: "IFMP", 102: "PNNI", 103: "PIM", 104: "ARIS", 105: "SCPS", 106: "QNX", 107: "A/N", 108: "IPComp", 109: "SNP", 110: "Compaq-Peer", 111: "IPX-in-IP", 112: "VRRP", 113: "PGM", 115: "L2TP", 116: "DDX", 117: "IATP", 118: "STP", 119: "SRP", 120: "UTI", 121: "SMP", 122: "SM", 123: "PTP", 124: "ISIS", 125: "FIRE", 126: "CRTP", 127: "CRUDP", 128: "SSCOPMCE", 129: "IPLT", 130: "SPS", 131: "PIPE", 132: "SCTP", 133: "FC", 134: "RSVP-E2E-IGNORE", 135: "Mobility", 136: "UDPLite", 137: "MPLS-in-IP", 138: "manet", 139: "HIP", 140: "Shim6", 141: "WESP", 142: "ROHC", } try: return protocol_map.get(int(protocol), protocol) finally: timer(74, time.clock(), 'e')