def post_query(self, table, start_time=None, end_time=None, select_fields=None, where_clause=None, sort_fields=None, sort=None, limit=None, filter=None, sync=True,dir=None): res = None try: flows_url = OpServerUtils.opserver_query_url( self._ip, str(self._port)) print flows_url query_dict = OpServerUtils.get_query_dict( table, start_time, end_time, select_fields, where_clause, sort_fields, sort, limit, filter, dir) print json.dumps(query_dict) res = [] resp = OpServerUtils.post_url_http( flows_url, json.dumps(query_dict), sync) if sync: if resp is not None: res = json.loads(resp) res = res['value'] else: if resp is not None: resp = json.loads(resp) qid = resp['href'].rsplit('/', 1)[1] result = OpServerUtils.get_query_result( self._ip, str(self._port), qid, 30) for item in result: res.append(item) except Exception as e: print str(e) finally: return res
def post_query_json(self, json_str, sync=True): ''' this module is to support raw query given in json format ''' res = None try: flows_url = OpServerUtils.opserver_query_url(self._ip, str(self._port)) print flows_url print "query is: ", json_str res = [] resp = OpServerUtils.post_url_http(flows_url, json_str, sync) if sync: if resp is not None: res = json.loads(resp) res = res['value'] else: if resp is not None: resp = json.loads(resp) qid = resp['href'].rsplit('/', 1)[1] result = OpServerUtils.get_query_result(self._ip, str(self._port), qid, 30) for item in result: res.append(item) except Exception as e: print str(e) finally: return res
def take_action(self, parsed_args): ip_addr = self.http_ip http_port = self.http_port cmd_name = self.cmd_name for command in self.cmd_list: if cmd_name in command[1].keys(): tab_url = "http://" + ip_addr + ":" +\ http_port + "/Snh_" + command[0].encode("utf-8") arg_count = 0 if len(command[1].values()[0].values()): tab_url = tab_url + "?" for params in command[1].values()[0].values(): for args in params: args[0] = args[0].encode("utf-8") if arg_count > 0: tab_url = tab_url + "&" tab_url = tab_url + args[0] + "=" if hasattr(parsed_args, args[0]) == True: if getattr(parsed_args, args[0]) != None: tab_url = tab_url + getattr( parsed_args, args[0])[0] arg_count = arg_count + 1 tables = self.web_invoke(tab_url) if tables: doc = xmltodict.parse(tables) OpServerUtils.messages_dict_scrub(doc) print json.dumps(doc, indent=4)
def post_query_json(self, json_str, sync=True): ''' this module is to support raw query given in json format ''' res = None try: flows_url = OpServerUtils.opserver_query_url(self._ip, str(self._port)) print(flows_url) print("query is: ", json_str) res = [] resp = OpServerUtils.post_url_http(flows_url, json_str, self._user, self._password, sync, headers=self._headers) if sync: if resp is not None: res = json.loads(resp) res = res['value'] else: if resp is not None: resp = json.loads(resp) qid = resp['href'].rsplit('/', 1)[1] result = OpServerUtils.get_query_result(self._ip, str(self._port), qid, self._user, self._password, 30, headers = self._headers) for item in result: res.append(item) except Exception as e: print(str(e)) finally: return res
def take_action(self, parsed_args): cmd_name = self.cmd_name for command in self.cmd_list: if cmd_name == command.cli_name: url = self._prepare_query_url(parsed_args, command) result = self.web_invoke(url) if result: output = xmltodict.parse(result) OpServerUtils.messages_dict_scrub(output) print json.dumps(output, indent=4)
def post_uve_request(self, table, json_body): url = 'http://%s:%s/analytics/uves/%s' % (self._ip, str(self._port), table) try: res = OpServerUtils.post_url_http(url, json_body, sync=True) res = json.loads(res) except Exception as e: print 'Error: POST uve request: %s' % str(e) return None else: return res
def post_purge_query_json(self, json_str, sync=True): ''' this module is to support raw purge query given in json format ''' res = None try: purge_request_url = \ OpServerUtils.opserver_database_purge_query_url( self._ip, str(self._port)) print purge_request_url print "query is: ", json_str resp = OpServerUtils.post_url_http( purge_request_url, json_str, sync) if resp is not None: res = json.loads(resp) except Exception as e: print str(e) finally: return res
def main(): querier = StatQuerier() if querier.parse_args() != 0: return tab_url = "http://" + querier._args.opserver_ip + ":" + querier._args.opserver_port +\ "/analytics/table/StatTable." + querier._args.table schematxt = OpServerUtils.get_url_http(tab_url + "/schema") schema = json.loads(schematxt.text)['columns'] if len(querier._args.select)==0: for pp in schema: if pp['index']: valuetxt = OpServerUtils.get_url_http(tab_url + "/column-values/" + pp['name']) print "%s : %s %s" % (pp['name'],pp['datatype'], valuetxt.text) else: print "%s : %s" % (pp['name'],pp['datatype']) else: result = querier.query(schema) querier.display(result)
def post_purge_query_json(self, json_str, sync=True): ''' this module is to support raw purge query given in json format ''' res = None try: purge_request_url = \ OpServerUtils.opserver_database_purge_query_url( self._ip, str(self._port)) print purge_request_url print "query is: ", json_str resp = OpServerUtils.post_url_http(purge_request_url, json_str, self._user, self._password, sync) if resp is not None: res = json.loads(resp) except Exception as e: print str(e) finally: return res
def post_uve_request(self, table, json_body): url = 'http://%s:%s/analytics/uves/%s' % (self._ip, str(self._port), table) try: res = OpServerUtils.post_url_http(url, json_body, user=self._user, password=self._password, sync=True, headers=self._headers) res = json.loads(res) except Exception as e: print('Error: POST uve request: %s' % str(e)) return None else: return res
def post_alarm_request(self, table, json_body): url = 'http://%s:%s/analytics/alarms/%s' % (self._ip, str( self._port), table) try: res = OpServerUtils.post_url_http(url, json_body, sync=True) res = json.loads(res) except Exception as e: print 'Error: POST alarm request: %s' % str(e) return None else: return res
def post_query(self, table, start_time=None, end_time=None, select_fields=None, where_clause='', sort_fields=None, sort=None, limit=None, filter=None, sync=True,dir=None, session_type=None, is_service_instance=None): res = None try: flows_url = OpServerUtils.opserver_query_url( self._ip, str(self._port)) print(flows_url) query_dict = OpServerUtils.get_query_dict( table, start_time=start_time, end_time=end_time, select_fields=select_fields, where_clause=where_clause, sort_fields=sort_fields, sort=sort, limit=limit, filter=filter, dir=dir, is_service_instance=is_service_instance, session_type=session_type) print(json.dumps(query_dict)) res = [] resp = OpServerUtils.post_url_http( flows_url, json.dumps(query_dict), self._user, self._password, sync, headers=self._headers) if sync: if resp is not None: res = json.loads(resp) res = res['value'] else: if resp is not None: resp = json.loads(resp) qid = resp['href'].rsplit('/', 1)[1] result = OpServerUtils.get_query_result( self._ip, str(self._port), qid, self._user, self._password, 30, headers=self._headers) for item in result: res.append(item) except Exception as e: print(str(e)) finally: return res
def query(self,schema): query_url = OpServerUtils.opserver_query_url( self._args.opserver_ip, self._args.opserver_port) query_dict = OpServerUtils.get_query_dict( "StatTable." + self._args.table, str(self._start_time), str(self._end_time), select_fields = self._args.select, where_clause = "AND".join(self._args.where), sort_fields = self._args.sort) print json.dumps(query_dict) resp = OpServerUtils.post_url_http( query_url, json.dumps(query_dict), sync = True) res = None if resp is not None: res = json.loads(resp) res = res['value'] return res
def main(): querier = StatQuerier() if querier.parse_args() != 0: return tab_url = "http://" + querier._args.opserver_ip + ":" + querier._args.opserver_port +\ "/analytics/table/StatTable." + querier._args.table schematxt = OpServerUtils.get_url_http(tab_url + "/schema") schema = json.loads(schematxt.text)['columns'] if len(querier._args.select) == 0: for pp in schema: if pp['index']: valuetxt = OpServerUtils.get_url_http(tab_url + "/column-values/" + pp['name']) print "%s : %s %s" % (pp['name'], pp['datatype'], valuetxt.text) else: print "%s : %s" % (pp['name'], pp['datatype']) else: result = querier.query(schema) querier.display(result)
def query(self, schema): query_url = OpServerUtils.opserver_query_url(self._args.opserver_ip, self._args.opserver_port) query_dict = OpServerUtils.get_query_dict( "StatTable." + self._args.table, str(self._start_time), str(self._end_time), select_fields=self._args.select, where_clause="AND".join(self._args.where), sort_fields=self._args.sort) print json.dumps(query_dict) resp = OpServerUtils.post_url_http(query_url, json.dumps(query_dict), sync=True) res = None if resp is not None: res = json.loads(resp) res = res['value'] return res
def test_href_agg(self): print "*** Running test_href_agg ***" uvevn = MakeUVEVirtualNetwork(None, "abc-corp:vn-00", "10.10.10.10", ifs=["host1:eth0"]) pa = ParallelAggregator(uvevn, {"ObjectIf": "if"}) res = pa.aggregate("abc-corp:vn-00", True, "127.0.0.1:8081") print json.dumps(res, indent=4, sort_keys=True) uvetest = MakeUVEVirtualNetwork( None, "abc-corp:vn-00", "10.10.10.10", ifs=[{"name": "host1:eth0", "href": "127.0.0.1:8081/analytics/uves/if/host1:eth0?cfilt=mystruct"}], ) cn = OpServerUtils.uve_attr_flatten(uvetest["abc-corp:vn-00"]["UVEVirtualNetwork"]["ifs"]["10.10.10.10"]) self.assertEqual(cn, res["UVEVirtualNetwork"]["ifs"])
def test_href_agg(self): logging.info("*** Running test_href_agg ***") uvevn = MakeUVEVirtualNetwork( None, "abc-corp:vn-00", "10.10.10.10", ifs=["host1:eth0"], ) pa = ParallelAggregator(uvevn, {"ObjectIf":"if"}) res = pa.aggregate("abc-corp:vn-00", True, "127.0.0.1:8081") logging.info(json.dumps(res, indent=4, sort_keys=True)) uvetest = MakeUVEVirtualNetwork( None, "abc-corp:vn-00", "10.10.10.10", ifs=[{"name":"host1:eth0", "href":"127.0.0.1:8081/analytics/uves/if/host1:eth0?cfilt=mystruct"}], ) cn = OpServerUtils.uve_attr_flatten( uvetest["abc-corp:vn-00"]['UVEVirtualNetwork'][ "ifs"]["10.10.10.10"]) self.assertEqual(cn, res['UVEVirtualNetwork']['ifs'])
def parse_args(self): """ Eg. python log.py --opserver-ip 127.0.0.1 --opserver-port 8081 --source 127.0.0.1 --module bgp | cfgm | vnswad --message-type UveVirtualMachineConfigTrace --category xmpp --level SYS_INFO | SYS_ERROR --object vn | vm --object-id name --object-select-field ObjectLog | SystemLog --reverse --verbose --raw --trace BgpPeerTraceBuf [--start-time now-10m --end-time now] | --last 10m """ defaults = { 'opserver_ip': '127.0.0.1', 'opserver_port': '8081', 'start_time': 'now-10m', 'end_time': 'now', } parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.set_defaults(**defaults) parser.add_argument("--opserver-ip", help="IP address of OpServer") parser.add_argument("--opserver-port", help="Port of OpServer") parser.add_argument( "--start-time", help="Logs start time (format now-10m, now-1h)") parser.add_argument("--end-time", help="Logs end time") parser.add_argument( "--last", help="Logs from last time period (format 10m, 1d)") parser.add_argument("--source", help="Logs from source address") parser.add_argument( "--module", help="Logs from module", choices=ModuleNames.values()) parser.add_argument("--category", help="Logs of category") parser.add_argument("--level", help="Logs of level") parser.add_argument("--message-type", help="Logs of message type") parser.add_argument("--reverse", action="store_true", help="Show logs in reverse chronological order") parser.add_argument( "--verbose", action="store_true", help="Show internal information") parser.add_argument( "--all", action="store_true", help="Show all logs") parser.add_argument( "--raw", action="store_true", help="Show raw XML messages") parser.add_argument( "--object", help="Logs of object type", choices=OBJECT_TABLE_LIST) parser.add_argument("--object-id", help="Logs of object name") parser.add_argument( "--object-select-field", help="Select field to filter the log", choices=[VizConstants.OBJECT_LOG, VizConstants.SYSTEM_LOG]) parser.add_argument("--trace", help="Dump trace buffer") parser.add_argument("--limit", help="Limit the number of messages") self._args = parser.parse_args() if self._args.last is not None: self._args.last = '-' + self._args.last self._start_time = OpServerUtils.convert_to_utc_timestamp_usec( self._args.last) self._end_time = OpServerUtils.convert_to_utc_timestamp_usec('now') else: try: if (self._args.start_time.isdigit() and self._args.end_time.isdigit()): self._start_time = int(self._args.start_time) self._end_time = int(self._args.end_time) else: self._start_time =\ OpServerUtils.convert_to_utc_timestamp_usec( self._args.start_time) self._end_time =\ OpServerUtils.convert_to_utc_timestamp_usec( self._args.end_time) except: print 'Incorrect start-time (%s) or end-time (%s) format' %\ (self._args.start_time, self._args.end_time) return -1 return 0
def display(self, result): if result == [] or result is None: return messages_dict_list = result for messages_dict in messages_dict_list: if VizConstants.TIMESTAMP in messages_dict: message_dt = datetime.datetime.fromtimestamp( int(messages_dict[VizConstants.TIMESTAMP]) / OpServerUtils.USECS_IN_SEC) message_dt += datetime.timedelta( microseconds=(int(messages_dict[VizConstants.TIMESTAMP]) % OpServerUtils.USECS_IN_SEC)) message_ts = message_dt.strftime(OpServerUtils.TIME_FORMAT_STR) else: message_ts = 'Time: NA' if VizConstants.SOURCE in messages_dict: source = messages_dict[VizConstants.SOURCE] else: source = 'Source: NA' if VizConstants.NODE_TYPE in messages_dict: node_type = messages_dict[VizConstants.NODE_TYPE] else: node_type = '' if VizConstants.MODULE in messages_dict: module = messages_dict[VizConstants.MODULE] else: module = 'Module: NA' if VizConstants.INSTANCE_ID in messages_dict: instance_id = messages_dict[VizConstants.INSTANCE_ID] else: instance_id = '' if VizConstants.MESSAGE_TYPE in messages_dict: message_type = messages_dict[VizConstants.MESSAGE_TYPE] else: message_type = 'Message Type: NA' if VizConstants.SANDESH_TYPE in messages_dict: sandesh_type = messages_dict[VizConstants.SANDESH_TYPE] else: sandesh_type = SandeshType.INVALID if self._args.object is None: if VizConstants.CATEGORY in messages_dict: category = messages_dict[VizConstants.CATEGORY] else: category = 'Category: NA' if VizConstants.LEVEL in messages_dict: level = SandeshLevel._VALUES_TO_NAMES[messages_dict[ VizConstants.LEVEL]] else: level = 'Level: NA' if VizConstants.SEQUENCE_NUM in messages_dict: seq_num = messages_dict[VizConstants.SEQUENCE_NUM] else: seq_num = 'Sequence Number: NA' if VizConstants.DATA in messages_dict: # Convert XML data to dict if self._args.raw: data_str = messages_dict[VizConstants.DATA] else: OpServerUtils.messages_xml_data_to_dict( messages_dict, VizConstants.DATA) if isinstance(messages_dict[VizConstants.DATA], dict): data_dict = messages_dict[VizConstants.DATA] data_str = OpServerUtils.messages_data_dict_to_str( data_dict, message_type, sandesh_type) else: data_str = messages_dict[VizConstants.DATA] else: data_str = 'Data not present' if self._args.trace is not None: print '{0} {1}:{2} {3}'.format(message_ts, message_type, seq_num, data_str) else: print '{0} {1} [{2}:{3}:{4}:{5}][{6}] : {7}:{8} {9}'.format( message_ts, source, node_type, module, instance_id, category, level, message_type, seq_num, data_str) else: for obj_sel_field in self._args.object_select_field: if obj_sel_field in messages_dict: if self._args.raw: data_str = messages_dict[obj_sel_field] else: # Convert XML data to dict OpServerUtils.messages_xml_data_to_dict( messages_dict, obj_sel_field) if isinstance(messages_dict[obj_sel_field], dict): data_dict = messages_dict[obj_sel_field] data_str =\ OpServerUtils.messages_data_dict_to_str( data_dict, message_type, sandesh_type) else: data_str = messages_dict[obj_sel_field] if data_str: print '{0} {1} [{2}:{3}:{4}] : {5}: {6}'.format( message_ts, source, node_type, module, instance_id, message_type, data_str)
def display(self, result): if result == [] or result is None: return flow_dict_list = result for flow_dict in flow_dict_list: # Setup time if self._SETUP_TIME in flow_dict: setup_time = int(flow_dict[self._SETUP_TIME]) if setup_time != 0: setup_dt = datetime.datetime.fromtimestamp( setup_time / OpServerUtils.USECS_IN_SEC) setup_dt += datetime.timedelta( microseconds= (setup_time % OpServerUtils.USECS_IN_SEC)) setup_ts = setup_dt.strftime( OpServerUtils.TIME_FORMAT_STR) else: setup_ts = 'Setup Time: NA' else: setup_ts = 'Setup Time: NA' # Teardown time if self._TEARDOWN_TIME in flow_dict: teardown_time = int(flow_dict[ self._TEARDOWN_TIME]) if teardown_time != 0: teardown_dt = datetime.datetime.fromtimestamp( teardown_time / OpServerUtils.USECS_IN_SEC) teardown_dt += datetime.timedelta( microseconds= (teardown_time % OpServerUtils.USECS_IN_SEC)) teardown_ts = teardown_dt.strftime( OpServerUtils.TIME_FORMAT_STR) else: teardown_ts = 'Active' else: teardown_ts = 'Active' # VRouter if self._VROUTER in flow_dict: vrouter = flow_dict[self._VROUTER] else: vrouter = 'VRouter: NA' # Direction if self._DIRECTION in flow_dict: direction = int(flow_dict[self._DIRECTION]) if direction == 1: direction = 'ingress' elif direction == 0: direction = 'egress' else: direction = 'Direction: Invalid' else: direction = 'Direction: NA' # Flow UUID if VizConstants.FLOW_TABLE_UUID in flow_dict: flow_uuid = flow_dict[VizConstants.FLOW_TABLE_UUID] else: flow_uuid = 'UUID: NA' # Source VN if self._SOURCE_VN in flow_dict: source_vn = flow_dict[self._SOURCE_VN] else: source_vn = 'Source VN: NA' # Destination VN if self._DESTINATION_VN in flow_dict: destination_vn = flow_dict[self._DESTINATION_VN] else: destination_vn = 'Destination VN: NA' # Source IP if self._SOURCE_IP in flow_dict: source_ip = flow_dict[self._SOURCE_IP] else: source_ip = 'Source IP: NA' # Destination IP if self._DESTINATION_IP in flow_dict: destination_ip = flow_dict[self._DESTINATION_IP] else: destination_ip = 'Destination IP: NA' # Source port if self._SOURCE_PORT in flow_dict: source_port = flow_dict[self._SOURCE_PORT] else: source_port = 'Source Port: NA' # Destination port if self._DESTINATION_PORT in flow_dict: destination_port = flow_dict[self._DESTINATION_PORT] else: destination_port = 'Destination Port: NA' # Protocol if self._PROTOCOL in flow_dict: protocol = OpServerUtils.ip_protocol_to_str( int(flow_dict[self._PROTOCOL])) else: protocol = 'Protocol: NA' # Action if self._ACTION in flow_dict: action = flow_dict[self._ACTION] else: action = '' # Agg packets and bytes if VizConstants.FLOW_TABLE_AGG_BYTES in flow_dict: agg_bytes = int(flow_dict[VizConstants.FLOW_TABLE_AGG_BYTES]) else: agg_bytes = 'Agg Bytes: NA' if VizConstants.FLOW_TABLE_AGG_PKTS in flow_dict: agg_pkts = int(flow_dict[VizConstants.FLOW_TABLE_AGG_PKTS]) else: agg_pkts = 'Agg Packets: NA' print '{0}({1}) {2} [{3} -- {4}] {5} '\ '{6}:{7}:{8} ---> {9}:{10}:{11} [{12} P ({13} B)]'\ ' : {14}'.format( vrouter, direction, action, setup_ts, teardown_ts, protocol, source_vn, source_ip, source_port, destination_vn, destination_ip, destination_port, agg_pkts, agg_bytes, flow_uuid)
def parse_args(self): """ Eg. python stats.py --opserver-ip 127.0.0.1 --opserver-port 8081 --table AnalyticsCpuState.cpu_info --where name=a6s40 cpu_info.module_id=Collector --select "T=60 SUM(cpu_info.cpu_share)" --sort "SUM(cpu_info.cpu_share)" [--start-time now-10m --end-time now] | --last 10m python stats.py --table AnalyticsCpuState.cpu_info """ defaults = { 'opserver_ip': '127.0.0.1', 'opserver_port': '8081', 'start_time': 'now-10m', 'end_time': 'now', 'select' : [], 'where' : [], 'sort': [] } parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.set_defaults(**defaults) parser.add_argument("--opserver-ip", help="IP address of OpServer") parser.add_argument("--opserver-port", help="Port of OpServer") parser.add_argument( "--start-time", help="Logs start time (format now-10m, now-1h)") parser.add_argument("--end-time", help="Logs end time") parser.add_argument( "--last", help="Logs from last time period (format 10m, 1d)") parser.add_argument( "--table", help="StatTable to query", choices=STAT_TABLE_LIST) parser.add_argument( "--select", help="List of Select Terms", nargs='+') parser.add_argument( "--where", help="List of Where Terms to be ANDed", nargs='+') parser.add_argument( "--sort", help="List of Sort Terms", nargs='+') self._args = parser.parse_args() if self._args.table is None: return -1 if self._args.last is not None: self._args.last = '-' + self._args.last self._start_time = OpServerUtils.convert_to_utc_timestamp_usec( self._args.last) self._end_time = OpServerUtils.convert_to_utc_timestamp_usec('now') else: try: if (self._args.start_time.isdigit() and self._args.end_time.isdigit()): self._start_time = int(self._args.start_time) self._end_time = int(self._args.end_time) else: self._start_time =\ OpServerUtils.convert_to_utc_timestamp_usec( self._args.start_time) self._end_time =\ OpServerUtils.convert_to_utc_timestamp_usec( self._args.end_time) except: print 'Incorrect start-time (%s) or end-time (%s) format' %\ (self._args.start_time, self._args.end_time) return -1 return 0
def parse_args(self): """ Eg. python stats.py --opserver-ip 127.0.0.1 --opserver-port 8081 --table AnalyticsCpuState.cpu_info --where name=a6s40 cpu_info.module_id=Collector --select "T=60 SUM(cpu_info.cpu_share)" --sort "SUM(cpu_info.cpu_share)" [--start-time now-10m --end-time now] | --last 10m python stats.py --table AnalyticsCpuState.cpu_info """ defaults = { 'opserver_ip': '127.0.0.1', 'opserver_port': '8081', 'start_time': 'now-10m', 'end_time': 'now', 'select': [], 'where': [], 'sort': [] } parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.set_defaults(**defaults) parser.add_argument("--opserver-ip", help="IP address of OpServer") parser.add_argument("--opserver-port", help="Port of OpServer") parser.add_argument("--start-time", help="Logs start time (format now-10m, now-1h)") parser.add_argument("--end-time", help="Logs end time") parser.add_argument("--last", help="Logs from last time period (format 10m, 1d)") parser.add_argument("--table", help="StatTable to query", choices=STAT_TABLE_LIST) parser.add_argument("--select", help="List of Select Terms", nargs='+') parser.add_argument("--where", help="List of Where Terms to be ANDed", nargs='+') parser.add_argument("--sort", help="List of Sort Terms", nargs='+') self._args = parser.parse_args() if self._args.table is None: return -1 if self._args.last is not None: self._args.last = '-' + self._args.last self._start_time = OpServerUtils.convert_to_utc_timestamp_usec( self._args.last) self._end_time = OpServerUtils.convert_to_utc_timestamp_usec('now') else: try: if (self._args.start_time.isdigit() and self._args.end_time.isdigit()): self._start_time = int(self._args.start_time) self._end_time = int(self._args.end_time) else: self._start_time =\ OpServerUtils.convert_to_utc_timestamp_usec( self._args.start_time) self._end_time =\ OpServerUtils.convert_to_utc_timestamp_usec( self._args.end_time) except: print 'Incorrect start-time (%s) or end-time (%s) format' %\ (self._args.start_time, self._args.end_time) return -1 return 0
def query(self): start_time, end_time = OpServerUtils.get_start_end_time( self._start_time, self._end_time) messages_url = OpServerUtils.opserver_query_url( self._args.opserver_ip, self._args.opserver_port) where_msg = [] where_obj = [] filter = [] if self._args.source is not None: source_match = OpServerUtils.Match(name=VizConstants.SOURCE, value=self._args.source, op=OpServerUtils.MatchOp.EQUAL) where_msg.append(source_match.__dict__) if self._args.module is not None: module_match = OpServerUtils.Match(name=VizConstants.MODULE, value=self._args.module, op=OpServerUtils.MatchOp.EQUAL) where_msg.append(module_match.__dict__) if self._args.category is not None: category_match = OpServerUtils.Match( name=VizConstants.CATEGORY, value=self._args.category, op=OpServerUtils.MatchOp.EQUAL) where_msg.append(category_match.__dict__) if self._args.message_type is not None: message_type_match = OpServerUtils.Match( name=VizConstants.MESSAGE_TYPE, value=self._args.message_type, op=OpServerUtils.MatchOp.EQUAL) where_msg.append(message_type_match.__dict__) if self._args.level is not None: level_match = OpServerUtils.Match( name=VizConstants.LEVEL, value=SandeshLevel._NAMES_TO_VALUES[self._args.level], op=OpServerUtils.MatchOp.GEQ) filter.append(level_match.__dict__) if self._args.node_type is not None: node_type_match = OpServerUtils.Match( name=VizConstants.NODE_TYPE, value=self._args.node_type, op=OpServerUtils.MatchOp.EQUAL) filter.append(node_type_match.__dict__) if self._args.instance_id is not None: instance_id_match = OpServerUtils.Match( name=VizConstants.INSTANCE_ID, value=self._args.instance_id, op=OpServerUtils.MatchOp.EQUAL) filter.append(instance_id_match.__dict__) if (self._args.object is not None or self._args.object_id is not None or self._args.object_select_field is not None): # Object Table Query where_obj = list(where_msg) if self._args.object is not None: if self._args.object in OBJECT_TABLE_LIST: table = self._args.object else: print 'Unknown object table [%s]' % (self._args.object) return None else: print 'Object required for query' return None if self._args.object_id is not None: id_match = OpServerUtils.Match(name=OpServerUtils.OBJECT_ID, value=self._args.object_id, op=OpServerUtils.MatchOp.EQUAL) where_obj.append(id_match.__dict__) else: print 'Object id required for table [%s]' % (self._args.object) return None if self._args.object_select_field is not None: if ((self._args.object_select_field != VizConstants.OBJECT_LOG) and (self._args.object_select_field != VizConstants.SYSTEM_LOG)): print 'Invalid object-select-field. '\ 'Valid values are "%s" or "%s"' \ % (VizConstants.OBJECT_LOG, VizConstants.SYSTEM_LOG) return None obj_sel_field = [self._args.object_select_field] self._args.object_select_field = obj_sel_field else: self._args.object_select_field = obj_sel_field = [ VizConstants.OBJECT_LOG, VizConstants.SYSTEM_LOG ] where = [where_obj] select_list = [ VizConstants.TIMESTAMP, VizConstants.SOURCE, VizConstants.MODULE, VizConstants.MESSAGE_TYPE, ] + obj_sel_field elif self._args.trace is not None: table = VizConstants.COLLECTOR_GLOBAL_TABLE if self._args.source is None: print 'Source is required for trace buffer dump' return None if self._args.module is None: print 'Module is required for trace buffer dump' return None trace_buf_match = OpServerUtils.Match( name=VizConstants.CATEGORY, value=self._args.trace, op=OpServerUtils.MatchOp.EQUAL) where_msg.append(trace_buf_match.__dict__) where = [where_msg] select_list = [ VizConstants.TIMESTAMP, VizConstants.MESSAGE_TYPE, VizConstants.SEQUENCE_NUM, VizConstants.DATA ] sandesh_type_filter = OpServerUtils.Match( name=VizConstants.SANDESH_TYPE, value=str(SandeshType.TRACE), op=OpServerUtils.MatchOp.EQUAL) filter.append(sandesh_type_filter.__dict__) else: # Message Table Query table = VizConstants.COLLECTOR_GLOBAL_TABLE # Message Table contains both systemlog and objectlog. # Add a filter to return only systemlogs if not self._args.all: sandesh_type_filter = OpServerUtils.Match( name=VizConstants.SANDESH_TYPE, value=str(SandeshType.SYSTEM), op=OpServerUtils.MatchOp.EQUAL) filter.append(sandesh_type_filter.__dict__) if len(where_msg): where = [where_msg] else: where = None select_list = [ VizConstants.TIMESTAMP, VizConstants.SOURCE, VizConstants.MODULE, VizConstants.CATEGORY, VizConstants.MESSAGE_TYPE, VizConstants.SEQUENCE_NUM, VizConstants.DATA, VizConstants.SANDESH_TYPE, VizConstants.LEVEL, VizConstants.NODE_TYPE, VizConstants.INSTANCE_ID, ] if len(filter) == 0: filter = None # Add sort by timestamp if self._args.reverse: sort_op = OpServerUtils.SortOp.DESCENDING else: sort_op = OpServerUtils.SortOp.ASCENDING sort_fields = [VizConstants.TIMESTAMP] if self._args.limit: limit = int(self._args.limit) else: limit = None messages_query = OpServerUtils.Query(table, start_time=start_time, end_time=end_time, select_fields=select_list, where=where, filter=filter, sort=sort_op, sort_fields=sort_fields, limit=limit) if self._args.verbose: print 'Performing query: {0}'.format( json.dumps(messages_query.__dict__)) print '' resp = OpServerUtils.post_url_http(messages_url, json.dumps(messages_query.__dict__)) result = {} if resp is not None: resp = json.loads(resp) qid = resp['href'].rsplit('/', 1)[1] result = OpServerUtils.get_query_result(self._args.opserver_ip, self._args.opserver_port, qid) return result
def query(self): start_time, end_time = OpServerUtils.get_start_end_time( self._start_time, self._end_time) flow_url = OpServerUtils.opserver_query_url( self._args.opserver_ip, self._args.opserver_port) where = [] filter = [] if self._args.vrouter is not None: vrouter_match = OpServerUtils.Match( name=self._VROUTER, value=self._args.vrouter, op=OpServerUtils.MatchOp.EQUAL) where.append(vrouter_match.__dict__) if self._args.source_vn is not None: source_vn_match = OpServerUtils.Match( name=self._SOURCE_VN, value=self._args.source_vn, op=OpServerUtils.MatchOp.EQUAL) where.append(source_vn_match.__dict__) if self._args.destination_vn is not None: dest_vn_match = OpServerUtils.Match( name=self._DESTINATION_VN, value=self._args.destination_vn, op=OpServerUtils.MatchOp.EQUAL) where.append(dest_vn_match.__dict__) if self._args.source_ip is not None: source_ip_match = OpServerUtils.Match( name=self._SOURCE_IP, value=self._args.source_ip, op=OpServerUtils.MatchOp.EQUAL) where.append(source_ip_match.__dict__) if self._args.destination_ip is not None: dest_ip_match = OpServerUtils.Match( name=self._DESTINATION_IP, value=self._args.destination_ip, op=OpServerUtils.MatchOp.EQUAL) where.append(dest_ip_match.__dict__) if self._args.protocol is not None: protocol_match = OpServerUtils.Match( name=self._PROTOCOL, value=self._args.protocol, op=OpServerUtils.MatchOp.EQUAL) where.append(protocol_match.__dict__) if self._args.source_port is not None: source_port_match = OpServerUtils.Match( name=self._SOURCE_PORT, value=self._args.source_port, op=OpServerUtils.MatchOp.EQUAL) where.append(source_port_match.__dict__) if self._args.destination_port is not None: dest_port_match = OpServerUtils.Match( name=self._DESTINATION_PORT, value=self._args.destination_port, op=OpServerUtils.MatchOp.EQUAL) where.append(dest_port_match.__dict__) if self._args.action is not None: action_match = OpServerUtils.Match( name=self._ACTION, value=self._args.action, op=OpServerUtils.MatchOp.EQUAL) filter.append(action_match.__dict__) # Flow Record Table Query table = VizConstants.FLOW_TABLE if len(where) == 0: where = None else: where = [where] select_list = [ VizConstants.FLOW_TABLE_UUID, self._VROUTER, self._SETUP_TIME, self._TEARDOWN_TIME, self._SOURCE_VN, self._DESTINATION_VN, self._SOURCE_IP, self._DESTINATION_IP, self._PROTOCOL, self._SOURCE_PORT, self._DESTINATION_PORT, self._ACTION, self._DIRECTION, VizConstants.FLOW_TABLE_AGG_BYTES, VizConstants.FLOW_TABLE_AGG_PKTS, ] if len(filter) == 0: filter = None flow_query = OpServerUtils.Query(table, start_time=start_time, end_time=end_time, select_fields=select_list, where=where, filter=filter, dir=self._args.direction) if self._args.verbose: print 'Performing query: {0}'.format( json.dumps(flow_query.__dict__)) print '' resp = OpServerUtils.post_url_http( flow_url, json.dumps(flow_query.__dict__)) result = {} if resp is not None: resp = json.loads(resp) qid = resp['href'].rsplit('/', 1)[1] result = OpServerUtils.get_query_result( self._args.opserver_ip, self._args.opserver_port, qid) return result
def display(self, result): if result == [] or result is None: return messages_dict_list = result for messages_dict in messages_dict_list: if VizConstants.TIMESTAMP in messages_dict: message_dt = datetime.datetime.fromtimestamp( int(messages_dict[VizConstants.TIMESTAMP]) / OpServerUtils.USECS_IN_SEC) message_dt += datetime.timedelta( microseconds= (int(messages_dict[VizConstants.TIMESTAMP]) % OpServerUtils.USECS_IN_SEC)) message_ts = message_dt.strftime(OpServerUtils.TIME_FORMAT_STR) else: message_ts = 'Time: NA' if VizConstants.SOURCE in messages_dict: source = messages_dict[VizConstants.SOURCE] else: source = 'Source: NA' if VizConstants.MODULE in messages_dict: module = messages_dict[VizConstants.MODULE] else: module = 'Module: NA' if VizConstants.MESSAGE_TYPE in messages_dict: message_type = messages_dict[VizConstants.MESSAGE_TYPE] else: message_type = 'Message Type: NA' if VizConstants.SANDESH_TYPE in messages_dict: sandesh_type = messages_dict[VizConstants.SANDESH_TYPE] else: sandesh_type = SandeshType.INVALID if self._args.object is None: if VizConstants.CATEGORY in messages_dict: category = messages_dict[VizConstants.CATEGORY] else: category = 'Category: NA' if VizConstants.LEVEL in messages_dict: level = SandeshLevel._VALUES_TO_NAMES[ messages_dict[VizConstants.LEVEL]] else: level = 'Level: NA' if VizConstants.SEQUENCE_NUM in messages_dict: seq_num = messages_dict[VizConstants.SEQUENCE_NUM] else: seq_num = 'Sequence Number: NA' if VizConstants.DATA in messages_dict: # Convert XML data to dict if self._args.raw: data_str = messages_dict[VizConstants.DATA] else: OpServerUtils.messages_xml_data_to_dict( messages_dict, VizConstants.DATA) if isinstance(messages_dict[VizConstants.DATA], dict): data_dict = messages_dict[VizConstants.DATA] data_str = OpServerUtils.messages_data_dict_to_str( data_dict, message_type, sandesh_type) else: data_str = messages_dict[VizConstants.DATA] else: data_str = 'Data not present' if self._args.trace is not None: print '{0} {1}:{2} {3}'.format( message_ts, message_type, seq_num, data_str) else: print '{0} {1} [{2}:{3}][{4}] : {5}:{6} {7}'.format( message_ts, source, module, category, level, message_type, seq_num, data_str) else: for obj_sel_field in self._args.object_select_field: if obj_sel_field in messages_dict: if self._args.raw: data_str = messages_dict[obj_sel_field] else: # Convert XML data to dict OpServerUtils.messages_xml_data_to_dict( messages_dict, obj_sel_field) if isinstance(messages_dict[obj_sel_field], dict): data_dict = messages_dict[obj_sel_field] data_str =\ OpServerUtils.messages_data_dict_to_str( data_dict, message_type, sandesh_type) else: data_str = messages_dict[obj_sel_field] if data_str: print '{0} [{1}:{2}] : {3}: {4}'.format( message_ts, source, module, message_type, data_str)
def parse_args(self): """ Eg. python flow.py --opserver-ip 127.0.0.1 --opserver-port 8081 --vrouter a6s23 --source-vn default-domain:default-project:vn1 --destination-vn default-domain:default-project:vn2 --source-ip 1.1.1.1 --destination-ip 2.2.2.2 --protocol TCP --source-port 32678 --destination-port 80 --action drop --direction ingress [--start-time now-10m --end-time now] | --last 10m """ defaults = { 'opserver_ip': '127.0.0.1', 'opserver_port': '8081', 'start_time': 'now-10m', 'end_time': 'now', 'direction' : 'ingress', } parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.set_defaults(**defaults) parser.add_argument("--opserver-ip", help="IP address of OpServer") parser.add_argument("--opserver-port", help="Port of OpServer") parser.add_argument( "--start-time", help="Flow record start time (format now-10m, now-1h)") parser.add_argument("--end-time", help="Flow record end time") parser.add_argument( "--last", help="Flow records from last time period (format 10m, 1d)") parser.add_argument("--vrouter", help="Flow records from vrouter") parser.add_argument("--source-vn", help="Flow records with source virtual network") parser.add_argument("--destination-vn", help="Flow records with destination virtual network") parser.add_argument("--source-ip", help="Flow records with source IP address") parser.add_argument("--destination-ip", help="Flow records with destination IP address") parser.add_argument("--protocol", help="Flow records with protocol") parser.add_argument("--source-port", help="Flow records with source port", type=int) parser.add_argument("--destination-port", help="Flow records with destination port", type=int) parser.add_argument("--action", help="Flow records with action") parser.add_argument("--direction", help="Flow direction", choices=['ingress', 'egress']) parser.add_argument( "--verbose", action="store_true", help="Show internal information") self._args = parser.parse_args() # Validate start-time and end-time if self._args.last is not None: self._args.last = '-' + self._args.last self._start_time = OpServerUtils.convert_to_utc_timestamp_usec( self._args.last) self._end_time = OpServerUtils.convert_to_utc_timestamp_usec('now') else: try: if (self._args.start_time.isdigit() and self._args.end_time.isdigit()): self._start_time = int(self._args.start_time) self._end_time = int(self._args.end_time) else: self._start_time =\ OpServerUtils.convert_to_utc_timestamp_usec( self._args.start_time) self._end_time =\ OpServerUtils.convert_to_utc_timestamp_usec( self._args.end_time) except: print 'Incorrect start-time (%s) or end-time (%s) format' %\ (self._args.start_time, self._args.end_time) return -1 # Validate flow arguments if self._args.source_ip is not None and self._args.source_vn is None: print 'Please provide source virtual network in addtion to '\ 'source IP address' return -1 if self._args.destination_ip is not None and \ self._args.destination_vn is None: print 'Please provide destination virtual network in addtion to '\ 'destination IP address' return -1 if self._args.source_port is not None and self._args.protocol is None: print 'Please provide protocol in addtion to source port' return -1 if self._args.destination_port is not None and \ self._args.protocol is None: print 'Please provide protocol in addtion to '\ 'destination port' return -1 # Convert direction if self._args.direction.lower() == "ingress": self._args.direction = 1 elif self._args.direction.lower() == "egress": self._args.direction = 0 else: print 'Direction should be ingress or egress' return -1 # Protocol if self._args.protocol is not None: if self._args.protocol.isalpha(): protocol = OpServerUtils.str_to_ip_protocol( self._args.protocol) if protocol == -1: print 'Please provide valid protocol' return -1 self._args.protocol = protocol return 0
def parse_args(self): """ Eg. python log.py --opserver-ip 127.0.0.1 --opserver-port 8081 --source 127.0.0.1 --node-type Control --module bgp | cfgm | vnswad --instance-id 0 --message-type UveVirtualMachineConfigTrace --category xmpp --level SYS_INFO | SYS_ERROR --object vn | vm --object-id name --object-select-field ObjectLog | SystemLog --reverse --verbose --raw --trace BgpPeerTraceBuf [--start-time now-10m --end-time now] | --last 10m """ defaults = { 'opserver_ip': '127.0.0.1', 'opserver_port': '8081', 'start_time': 'now-10m', 'end_time': 'now', } parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.set_defaults(**defaults) parser.add_argument("--opserver-ip", help="IP address of OpServer") parser.add_argument("--opserver-port", help="Port of OpServer") parser.add_argument("--start-time", help="Logs start time (format now-10m, now-1h)") parser.add_argument("--end-time", help="Logs end time") parser.add_argument("--last", help="Logs from last time period (format 10m, 1d)") parser.add_argument("--source", help="Logs from source address") parser.add_argument("--node-type", help="Logs from node type", choices=NodeTypeNames.values()) parser.add_argument("--module", help="Logs from module", choices=ModuleNames.values()) parser.add_argument("--instance-id", help="Logs from module instance") parser.add_argument("--category", help="Logs of category") parser.add_argument("--level", help="Logs of level") parser.add_argument("--message-type", help="Logs of message type") parser.add_argument("--reverse", action="store_true", help="Show logs in reverse chronological order") parser.add_argument("--verbose", action="store_true", help="Show internal information") parser.add_argument("--all", action="store_true", help="Show all logs") parser.add_argument("--raw", action="store_true", help="Show raw XML messages") parser.add_argument("--object", help="Logs of object type", choices=OBJECT_TABLE_LIST) parser.add_argument("--object-id", help="Logs of object name") parser.add_argument( "--object-select-field", help="Select field to filter the log", choices=[VizConstants.OBJECT_LOG, VizConstants.SYSTEM_LOG]) parser.add_argument("--trace", help="Dump trace buffer") parser.add_argument("--limit", help="Limit the number of messages") self._args = parser.parse_args() if self._args.last is not None: self._args.last = '-' + self._args.last self._start_time = OpServerUtils.convert_to_utc_timestamp_usec( self._args.last) self._end_time = OpServerUtils.convert_to_utc_timestamp_usec('now') else: try: if (self._args.start_time.isdigit() and self._args.end_time.isdigit()): self._start_time = int(self._args.start_time) self._end_time = int(self._args.end_time) else: self._start_time =\ OpServerUtils.convert_to_utc_timestamp_usec( self._args.start_time) self._end_time =\ OpServerUtils.convert_to_utc_timestamp_usec( self._args.end_time) except: print 'Incorrect start-time (%s) or end-time (%s) format' %\ (self._args.start_time, self._args.end_time) return -1 return 0
def display(self, result): if result == [] or result is None: return flow_dict_list = result for flow_dict in flow_dict_list: # Setup time if self._SETUP_TIME in flow_dict: setup_time = int(flow_dict[self._SETUP_TIME]) if setup_time != 0: setup_dt = datetime.datetime.fromtimestamp( setup_time / OpServerUtils.USECS_IN_SEC) setup_dt += datetime.timedelta( microseconds=(setup_time % OpServerUtils.USECS_IN_SEC)) setup_ts = setup_dt.strftime(OpServerUtils.TIME_FORMAT_STR) else: setup_ts = 'Setup Time: NA' else: setup_ts = 'Setup Time: NA' # Teardown time if self._TEARDOWN_TIME in flow_dict: teardown_time = int(flow_dict[self._TEARDOWN_TIME]) if teardown_time != 0: teardown_dt = datetime.datetime.fromtimestamp( teardown_time / OpServerUtils.USECS_IN_SEC) teardown_dt += datetime.timedelta( microseconds=(teardown_time % OpServerUtils.USECS_IN_SEC)) teardown_ts = teardown_dt.strftime( OpServerUtils.TIME_FORMAT_STR) else: teardown_ts = 'Active' else: teardown_ts = 'Active' # VRouter if self._VROUTER in flow_dict: vrouter = flow_dict[self._VROUTER] else: vrouter = 'VRouter: NA' # Direction if self._DIRECTION in flow_dict: direction = int(flow_dict[self._DIRECTION]) if direction == 1: direction = 'ingress' elif direction == 0: direction = 'egress' else: direction = 'Direction: Invalid' else: direction = 'Direction: NA' # Flow UUID if VizConstants.FLOW_TABLE_UUID in flow_dict: flow_uuid = flow_dict[VizConstants.FLOW_TABLE_UUID] else: flow_uuid = 'UUID: NA' # Source VN if self._SOURCE_VN in flow_dict: source_vn = flow_dict[self._SOURCE_VN] else: source_vn = 'Source VN: NA' # Destination VN if self._DESTINATION_VN in flow_dict: destination_vn = flow_dict[self._DESTINATION_VN] else: destination_vn = 'Destination VN: NA' # Source IP if self._SOURCE_IP in flow_dict: source_ip = flow_dict[self._SOURCE_IP] else: source_ip = 'Source IP: NA' # Destination IP if self._DESTINATION_IP in flow_dict: destination_ip = flow_dict[self._DESTINATION_IP] else: destination_ip = 'Destination IP: NA' # Source port if self._SOURCE_PORT in flow_dict: source_port = flow_dict[self._SOURCE_PORT] else: source_port = 'Source Port: NA' # Destination port if self._DESTINATION_PORT in flow_dict: destination_port = flow_dict[self._DESTINATION_PORT] else: destination_port = 'Destination Port: NA' # Protocol if self._PROTOCOL in flow_dict: protocol = OpServerUtils.ip_protocol_to_str( int(flow_dict[self._PROTOCOL])) else: protocol = 'Protocol: NA' # Action if self._ACTION in flow_dict: action = flow_dict[self._ACTION] else: action = '' # Agg packets and bytes if VizConstants.FLOW_TABLE_AGG_BYTES in flow_dict: agg_bytes = int(flow_dict[VizConstants.FLOW_TABLE_AGG_BYTES]) else: agg_bytes = 'Agg Bytes: NA' if VizConstants.FLOW_TABLE_AGG_PKTS in flow_dict: agg_pkts = int(flow_dict[VizConstants.FLOW_TABLE_AGG_PKTS]) else: agg_pkts = 'Agg Packets: NA' print '{0}({1}) {2} [{3} -- {4}] {5} '\ '{6}:{7}:{8} ---> {9}:{10}:{11} [{12} P ({13} B)]'\ ' : {14}'.format( vrouter, direction, action, setup_ts, teardown_ts, protocol, source_vn, source_ip, source_port, destination_vn, destination_ip, destination_port, agg_pkts, agg_bytes, flow_uuid)
def parse_args(self): """ Eg. python flow.py --opserver-ip 127.0.0.1 --opserver-port 8081 --vrouter a6s23 --source-vn default-domain:default-project:vn1 --destination-vn default-domain:default-project:vn2 --source-ip 1.1.1.1 --destination-ip 2.2.2.2 --protocol TCP --source-port 32678 --destination-port 80 --action drop --direction ingress [--start-time now-10m --end-time now] | --last 10m """ defaults = { 'opserver_ip': '127.0.0.1', 'opserver_port': '8081', 'start_time': 'now-10m', 'end_time': 'now', 'direction': 'ingress', } parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.set_defaults(**defaults) parser.add_argument("--opserver-ip", help="IP address of OpServer") parser.add_argument("--opserver-port", help="Port of OpServer") parser.add_argument( "--start-time", help="Flow record start time (format now-10m, now-1h)") parser.add_argument("--end-time", help="Flow record end time") parser.add_argument( "--last", help="Flow records from last time period (format 10m, 1d)") parser.add_argument("--vrouter", help="Flow records from vrouter") parser.add_argument("--source-vn", help="Flow records with source virtual network") parser.add_argument( "--destination-vn", help="Flow records with destination virtual network") parser.add_argument("--source-ip", help="Flow records with source IP address") parser.add_argument("--destination-ip", help="Flow records with destination IP address") parser.add_argument("--protocol", help="Flow records with protocol") parser.add_argument("--source-port", help="Flow records with source port", type=int) parser.add_argument("--destination-port", help="Flow records with destination port", type=int) parser.add_argument("--action", help="Flow records with action") parser.add_argument("--direction", help="Flow direction", choices=['ingress', 'egress']) parser.add_argument("--verbose", action="store_true", help="Show internal information") self._args = parser.parse_args() # Validate start-time and end-time if self._args.last is not None: self._args.last = '-' + self._args.last self._start_time = OpServerUtils.convert_to_utc_timestamp_usec( self._args.last) self._end_time = OpServerUtils.convert_to_utc_timestamp_usec('now') else: try: if (self._args.start_time.isdigit() and self._args.end_time.isdigit()): self._start_time = int(self._args.start_time) self._end_time = int(self._args.end_time) else: self._start_time =\ OpServerUtils.convert_to_utc_timestamp_usec( self._args.start_time) self._end_time =\ OpServerUtils.convert_to_utc_timestamp_usec( self._args.end_time) except: print 'Incorrect start-time (%s) or end-time (%s) format' %\ (self._args.start_time, self._args.end_time) return -1 # Validate flow arguments if self._args.source_ip is not None and self._args.source_vn is None: print 'Please provide source virtual network in addtion to '\ 'source IP address' return -1 if self._args.destination_ip is not None and \ self._args.destination_vn is None: print 'Please provide destination virtual network in addtion to '\ 'destination IP address' return -1 if self._args.source_port is not None and self._args.protocol is None: print 'Please provide protocol in addtion to source port' return -1 if self._args.destination_port is not None and \ self._args.protocol is None: print 'Please provide protocol in addtion to '\ 'destination port' return -1 # Convert direction if self._args.direction.lower() == "ingress": self._args.direction = 1 elif self._args.direction.lower() == "egress": self._args.direction = 0 else: print 'Direction should be ingress or egress' return -1 # Protocol if self._args.protocol is not None: if self._args.protocol.isalpha(): protocol = OpServerUtils.str_to_ip_protocol( self._args.protocol) if protocol == -1: print 'Please provide valid protocol' return -1 self._args.protocol = protocol return 0
def query(self): start_time, end_time = OpServerUtils.get_start_end_time( self._start_time, self._end_time) messages_url = OpServerUtils.opserver_query_url( self._args.opserver_ip, self._args.opserver_port) where_msg = [] where_obj = [] filter = [] if self._args.source is not None: source_match = OpServerUtils.Match(name=VizConstants.SOURCE, value=self._args.source, op=OpServerUtils.MatchOp.EQUAL) where_msg.append(source_match.__dict__) if self._args.module is not None: module_match = OpServerUtils.Match(name=VizConstants.MODULE, value=self._args.module, op=OpServerUtils.MatchOp.EQUAL) where_msg.append(module_match.__dict__) if self._args.category is not None: category_match = OpServerUtils.Match( name=VizConstants.CATEGORY, value=self._args.category, op=OpServerUtils.MatchOp.EQUAL) where_msg.append(category_match.__dict__) if self._args.message_type is not None: message_type_match = OpServerUtils.Match( name=VizConstants.MESSAGE_TYPE, value=self._args.message_type, op=OpServerUtils.MatchOp.EQUAL) where_msg.append(message_type_match.__dict__) if self._args.level is not None: level_match = OpServerUtils.Match( name=VizConstants.LEVEL, value=SandeshLevel._NAMES_TO_VALUES[self._args.level], op=OpServerUtils.MatchOp.GEQ) filter.append(level_match.__dict__) if (self._args.object is not None or self._args.object_id is not None or self._args.object_select_field is not None): # Object Table Query where_obj = list(where_msg) if self._args.object is not None: if self._args.object in OBJECT_TABLE_LIST: table = self._args.object else: print 'Unknown object table [%s]' % (self._args.object) return None else: print 'Object required for query' return None if self._args.object_id is not None: id_match = OpServerUtils.Match(name=OpServerUtils.OBJECT_ID, value=self._args.object_id, op=OpServerUtils.MatchOp.EQUAL) where_obj.append(id_match.__dict__) else: print 'Object id required for table [%s]' % (self._args.object) return None if self._args.object_select_field is not None: if ((self._args.object_select_field != VizConstants.OBJECT_LOG) and (self._args.object_select_field != VizConstants.SYSTEM_LOG)): print 'Invalid object-select-field. '\ 'Valid values are "%s" or "%s"' \ % (VizConstants.OBJECT_LOG, VizConstants.SYSTEM_LOG) return None obj_sel_field = [self._args.object_select_field] self._args.object_select_field = obj_sel_field else: self._args.object_select_field = obj_sel_field = [ VizConstants.OBJECT_LOG, VizConstants.SYSTEM_LOG] where = [where_obj] select_list = [ VizConstants.TIMESTAMP, VizConstants.SOURCE, VizConstants.MODULE, VizConstants.MESSAGE_TYPE, ] + obj_sel_field elif self._args.trace is not None: table = VizConstants.COLLECTOR_GLOBAL_TABLE if self._args.source is None: print 'Source is required for trace buffer dump' return None if self._args.module is None: print 'Module is required for trace buffer dump' return None trace_buf_match = OpServerUtils.Match( name=VizConstants.CATEGORY, value=self._args.trace, op=OpServerUtils.MatchOp.EQUAL) where_msg.append(trace_buf_match.__dict__) where = [where_msg] select_list = [ VizConstants.TIMESTAMP, VizConstants.MESSAGE_TYPE, VizConstants.SEQUENCE_NUM, VizConstants.DATA ] sandesh_type_filter = OpServerUtils.Match( name=VizConstants.SANDESH_TYPE, value=str( SandeshType.TRACE), op=OpServerUtils.MatchOp.EQUAL) filter.append(sandesh_type_filter.__dict__) else: # Message Table Query table = VizConstants.COLLECTOR_GLOBAL_TABLE # Message Table contains both systemlog and objectlog. # Add a filter to return only systemlogs if not self._args.all: sandesh_type_filter = OpServerUtils.Match( name=VizConstants.SANDESH_TYPE, value=str( SandeshType.SYSTEM), op=OpServerUtils.MatchOp.EQUAL) filter.append(sandesh_type_filter.__dict__) if len(where_msg): where = [where_msg] else: where = None select_list = [ VizConstants.TIMESTAMP, VizConstants.SOURCE, VizConstants.MODULE, VizConstants.CATEGORY, VizConstants.MESSAGE_TYPE, VizConstants.SEQUENCE_NUM, VizConstants.DATA, VizConstants.SANDESH_TYPE, VizConstants.LEVEL, ] if len(filter) == 0: filter = None # Add sort by timestamp if self._args.reverse: sort_op = OpServerUtils.SortOp.DESCENDING else: sort_op = OpServerUtils.SortOp.ASCENDING sort_fields = [VizConstants.TIMESTAMP] if self._args.limit: limit = int(self._args.limit) else: limit = None messages_query = OpServerUtils.Query(table, start_time=start_time, end_time=end_time, select_fields=select_list, where=where, filter=filter, sort=sort_op, sort_fields=sort_fields, limit=limit) if self._args.verbose: print 'Performing query: {0}'.format( json.dumps(messages_query.__dict__)) print '' resp = OpServerUtils.post_url_http( messages_url, json.dumps(messages_query.__dict__)) result = {} if resp is not None: resp = json.loads(resp) qid = resp['href'].rsplit('/', 1)[1] result = OpServerUtils.get_query_result( self._args.opserver_ip, self._args.opserver_port, qid) return result
def query(self): start_time, end_time = OpServerUtils.get_start_end_time( self._start_time, self._end_time) flow_url = OpServerUtils.opserver_query_url(self._args.opserver_ip, self._args.opserver_port) where = [] filter = [] if self._args.vrouter is not None: vrouter_match = OpServerUtils.Match(name=self._VROUTER, value=self._args.vrouter, op=OpServerUtils.MatchOp.EQUAL) where.append(vrouter_match.__dict__) if self._args.source_vn is not None: source_vn_match = OpServerUtils.Match( name=self._SOURCE_VN, value=self._args.source_vn, op=OpServerUtils.MatchOp.EQUAL) where.append(source_vn_match.__dict__) if self._args.destination_vn is not None: dest_vn_match = OpServerUtils.Match( name=self._DESTINATION_VN, value=self._args.destination_vn, op=OpServerUtils.MatchOp.EQUAL) where.append(dest_vn_match.__dict__) if self._args.source_ip is not None: source_ip_match = OpServerUtils.Match( name=self._SOURCE_IP, value=self._args.source_ip, op=OpServerUtils.MatchOp.EQUAL) where.append(source_ip_match.__dict__) if self._args.destination_ip is not None: dest_ip_match = OpServerUtils.Match( name=self._DESTINATION_IP, value=self._args.destination_ip, op=OpServerUtils.MatchOp.EQUAL) where.append(dest_ip_match.__dict__) if self._args.protocol is not None: protocol_match = OpServerUtils.Match( name=self._PROTOCOL, value=self._args.protocol, op=OpServerUtils.MatchOp.EQUAL) where.append(protocol_match.__dict__) if self._args.source_port is not None: source_port_match = OpServerUtils.Match( name=self._SOURCE_PORT, value=self._args.source_port, op=OpServerUtils.MatchOp.EQUAL) where.append(source_port_match.__dict__) if self._args.destination_port is not None: dest_port_match = OpServerUtils.Match( name=self._DESTINATION_PORT, value=self._args.destination_port, op=OpServerUtils.MatchOp.EQUAL) where.append(dest_port_match.__dict__) if self._args.action is not None: action_match = OpServerUtils.Match(name=self._ACTION, value=self._args.action, op=OpServerUtils.MatchOp.EQUAL) filter.append(action_match.__dict__) # Flow Record Table Query table = VizConstants.FLOW_TABLE if len(where) == 0: where = None else: where = [where] select_list = [ VizConstants.FLOW_TABLE_UUID, self._VROUTER, self._SETUP_TIME, self._TEARDOWN_TIME, self._SOURCE_VN, self._DESTINATION_VN, self._SOURCE_IP, self._DESTINATION_IP, self._PROTOCOL, self._SOURCE_PORT, self._DESTINATION_PORT, self._ACTION, self._DIRECTION, VizConstants.FLOW_TABLE_AGG_BYTES, VizConstants.FLOW_TABLE_AGG_PKTS, ] if len(filter) == 0: filter = None flow_query = OpServerUtils.Query(table, start_time=start_time, end_time=end_time, select_fields=select_list, where=where, filter=filter, dir=self._args.direction) if self._args.verbose: print 'Performing query: {0}'.format( json.dumps(flow_query.__dict__)) print '' resp = OpServerUtils.post_url_http(flow_url, json.dumps(flow_query.__dict__)) result = {} if resp is not None: resp = json.loads(resp) qid = resp['href'].rsplit('/', 1)[1] result = OpServerUtils.get_query_result(self._args.opserver_ip, self._args.opserver_port, qid) return result