def post_query(self, table, start_time=None, end_time=None,
select_fields=None, where_clause=None,
sort_fields=None, sort=None, limit=None,
filter=None, sync=True,dir=None):
res = None
try:
flows_url = OpServerUtils.opserver_query_url(
self._ip, str(self._port))
print flows_url
query_dict = OpServerUtils.get_query_dict(
table, start_time, end_time,
select_fields,
where_clause,
sort_fields, sort, limit, filter, dir)
print json.dumps(query_dict)
res = []
resp = OpServerUtils.post_url_http(
flows_url, json.dumps(query_dict), sync)
if sync:
if resp is not None:
res = json.loads(resp)
res = res['value']
else:
if resp is not None:
resp = json.loads(resp)
qid = resp['href'].rsplit('/', 1)[1]
result = OpServerUtils.get_query_result(
self._ip, str(self._port), qid, 30)
for item in result:
res.append(item)
except Exception as e:
print str(e)
finally:
return res
def post_query_json(self, json_str, sync=True):
'''
this module is to support raw query given in json format
'''
res = None
try:
flows_url = OpServerUtils.opserver_query_url(self._ip, str(self._port))
print flows_url
print "query is: ", json_str
res = []
resp = OpServerUtils.post_url_http(flows_url, json_str, sync)
if sync:
if resp is not None:
res = json.loads(resp)
res = res['value']
else:
if resp is not None:
resp = json.loads(resp)
qid = resp['href'].rsplit('/', 1)[1]
result = OpServerUtils.get_query_result(self._ip, str(self._port), qid, 30)
for item in result:
res.append(item)
except Exception as e:
print str(e)
finally:
return res
def take_action(self, parsed_args):
ip_addr = self.http_ip
http_port = self.http_port
cmd_name = self.cmd_name
for command in self.cmd_list:
if cmd_name in command[1].keys():
tab_url = "http://" + ip_addr + ":" +\
http_port + "/Snh_" + command[0].encode("utf-8")
arg_count = 0
if len(command[1].values()[0].values()):
tab_url = tab_url + "?"
for params in command[1].values()[0].values():
for args in params:
args[0] = args[0].encode("utf-8")
if arg_count > 0:
tab_url = tab_url + "&"
tab_url = tab_url + args[0] + "="
if hasattr(parsed_args, args[0]) == True:
if getattr(parsed_args, args[0]) != None:
tab_url = tab_url + getattr(
parsed_args, args[0])[0]
arg_count = arg_count + 1
tables = self.web_invoke(tab_url)
if tables:
doc = xmltodict.parse(tables)
OpServerUtils.messages_dict_scrub(doc)
print json.dumps(doc, indent=4)
def post_query(self, table, start_time=None, end_time=None,
select_fields=None, where_clause=None,
sort_fields=None, sort=None, limit=None,
filter=None, sync=True,dir=None):
res = None
try:
flows_url = OpServerUtils.opserver_query_url(
self._ip, str(self._port))
print flows_url
query_dict = OpServerUtils.get_query_dict(
table, start_time, end_time,
select_fields,
where_clause,
sort_fields, sort, limit, filter, dir)
print json.dumps(query_dict)
res = []
resp = OpServerUtils.post_url_http(
flows_url, json.dumps(query_dict), sync)
if sync:
if resp is not None:
res = json.loads(resp)
res = res['value']
else:
if resp is not None:
resp = json.loads(resp)
qid = resp['href'].rsplit('/', 1)[1]
result = OpServerUtils.get_query_result(
self._ip, str(self._port), qid, 30)
for item in result:
res.append(item)
except Exception as e:
print str(e)
finally:
return res
def post_query_json(self, json_str, sync=True):
'''
this module is to support raw query given in json format
'''
res = None
try:
flows_url = OpServerUtils.opserver_query_url(self._ip,
str(self._port))
print(flows_url)
print("query is: ", json_str)
res = []
resp = OpServerUtils.post_url_http(flows_url, json_str,
self._user, self._password, sync, headers=self._headers)
if sync:
if resp is not None:
res = json.loads(resp)
res = res['value']
else:
if resp is not None:
resp = json.loads(resp)
qid = resp['href'].rsplit('/', 1)[1]
result = OpServerUtils.get_query_result(self._ip,
str(self._port), qid, self._user, self._password, 30,
headers = self._headers)
for item in result:
res.append(item)
except Exception as e:
print(str(e))
finally:
return res
def take_action(self, parsed_args):
cmd_name = self.cmd_name
for command in self.cmd_list:
if cmd_name == command.cli_name:
url = self._prepare_query_url(parsed_args, command)
result = self.web_invoke(url)
if result:
output = xmltodict.parse(result)
OpServerUtils.messages_dict_scrub(output)
print json.dumps(output, indent=4)
def post_uve_request(self, table, json_body):
url = 'http://%s:%s/analytics/uves/%s' % (self._ip, str(self._port), table)
try:
res = OpServerUtils.post_url_http(url, json_body, sync=True)
res = json.loads(res)
except Exception as e:
print 'Error: POST uve request: %s' % str(e)
return None
else:
return res
def post_purge_query_json(self, json_str, sync=True):
'''
this module is to support raw purge query given in json format
'''
res = None
try:
purge_request_url = \
OpServerUtils.opserver_database_purge_query_url(
self._ip, str(self._port))
print purge_request_url
print "query is: ", json_str
resp = OpServerUtils.post_url_http(
purge_request_url, json_str, sync)
if resp is not None:
res = json.loads(resp)
except Exception as e:
print str(e)
finally:
return res
def main():
querier = StatQuerier()
if querier.parse_args() != 0:
return
tab_url = "http://" + querier._args.opserver_ip + ":" + querier._args.opserver_port +\
"/analytics/table/StatTable." + querier._args.table
schematxt = OpServerUtils.get_url_http(tab_url + "/schema")
schema = json.loads(schematxt.text)['columns']
if len(querier._args.select)==0:
for pp in schema:
if pp['index']:
valuetxt = OpServerUtils.get_url_http(tab_url + "/column-values/" + pp['name'])
print "%s : %s %s" % (pp['name'],pp['datatype'], valuetxt.text)
else:
print "%s : %s" % (pp['name'],pp['datatype'])
else:
result = querier.query(schema)
querier.display(result)
def post_purge_query_json(self, json_str, sync=True):
'''
this module is to support raw purge query given in json format
'''
res = None
try:
purge_request_url = \
OpServerUtils.opserver_database_purge_query_url(
self._ip, str(self._port))
print purge_request_url
print "query is: ", json_str
resp = OpServerUtils.post_url_http(purge_request_url, json_str,
self._user, self._password,
sync)
if resp is not None:
res = json.loads(resp)
except Exception as e:
print str(e)
finally:
return res
def post_uve_request(self, table, json_body):
url = 'http://%s:%s/analytics/uves/%s' % (self._ip, str(self._port), table)
try:
res = OpServerUtils.post_url_http(url, json_body, user=self._user,
password=self._password, sync=True, headers=self._headers)
res = json.loads(res)
except Exception as e:
print('Error: POST uve request: %s' % str(e))
return None
else:
return res
def post_alarm_request(self, table, json_body):
url = 'http://%s:%s/analytics/alarms/%s' % (self._ip, str(
self._port), table)
try:
res = OpServerUtils.post_url_http(url, json_body, sync=True)
res = json.loads(res)
except Exception as e:
print 'Error: POST alarm request: %s' % str(e)
return None
else:
return res
def post_query(self, table, start_time=None, end_time=None,
select_fields=None, where_clause='',
sort_fields=None, sort=None, limit=None,
filter=None, sync=True,dir=None, session_type=None,
is_service_instance=None):
res = None
try:
flows_url = OpServerUtils.opserver_query_url(
self._ip, str(self._port))
print(flows_url)
query_dict = OpServerUtils.get_query_dict(
table, start_time=start_time, end_time=end_time,
select_fields=select_fields,
where_clause=where_clause,
sort_fields=sort_fields, sort=sort, limit=limit, filter=filter,
dir=dir,
is_service_instance=is_service_instance,
session_type=session_type)
print(json.dumps(query_dict))
res = []
resp = OpServerUtils.post_url_http(
flows_url, json.dumps(query_dict), self._user, self._password,
sync, headers=self._headers)
if sync:
if resp is not None:
res = json.loads(resp)
res = res['value']
else:
if resp is not None:
resp = json.loads(resp)
qid = resp['href'].rsplit('/', 1)[1]
result = OpServerUtils.get_query_result(
self._ip, str(self._port), qid, self._user,
self._password, 30, headers=self._headers)
for item in result:
res.append(item)
except Exception as e:
print(str(e))
finally:
return res
def query(self,schema):
query_url = OpServerUtils.opserver_query_url(
self._args.opserver_ip,
self._args.opserver_port)
query_dict = OpServerUtils.get_query_dict(
"StatTable." + self._args.table, str(self._start_time), str(self._end_time),
select_fields = self._args.select,
where_clause = "AND".join(self._args.where),
sort_fields = self._args.sort)
print json.dumps(query_dict)
resp = OpServerUtils.post_url_http(
query_url, json.dumps(query_dict), sync = True)
res = None
if resp is not None:
res = json.loads(resp)
res = res['value']
return res
def main():
querier = StatQuerier()
if querier.parse_args() != 0:
return
tab_url = "http://" + querier._args.opserver_ip + ":" + querier._args.opserver_port +\
"/analytics/table/StatTable." + querier._args.table
schematxt = OpServerUtils.get_url_http(tab_url + "/schema")
schema = json.loads(schematxt.text)['columns']
if len(querier._args.select) == 0:
for pp in schema:
if pp['index']:
valuetxt = OpServerUtils.get_url_http(tab_url +
"/column-values/" +
pp['name'])
print "%s : %s %s" % (pp['name'], pp['datatype'],
valuetxt.text)
else:
print "%s : %s" % (pp['name'], pp['datatype'])
else:
result = querier.query(schema)
querier.display(result)
def query(self, schema):
query_url = OpServerUtils.opserver_query_url(self._args.opserver_ip,
self._args.opserver_port)
query_dict = OpServerUtils.get_query_dict(
"StatTable." + self._args.table,
str(self._start_time),
str(self._end_time),
select_fields=self._args.select,
where_clause="AND".join(self._args.where),
sort_fields=self._args.sort)
print json.dumps(query_dict)
resp = OpServerUtils.post_url_http(query_url,
json.dumps(query_dict),
sync=True)
res = None
if resp is not None:
res = json.loads(resp)
res = res['value']
return res
def test_href_agg(self):
print "*** Running test_href_agg ***"
uvevn = MakeUVEVirtualNetwork(None, "abc-corp:vn-00", "10.10.10.10", ifs=["host1:eth0"])
pa = ParallelAggregator(uvevn, {"ObjectIf": "if"})
res = pa.aggregate("abc-corp:vn-00", True, "127.0.0.1:8081")
print json.dumps(res, indent=4, sort_keys=True)
uvetest = MakeUVEVirtualNetwork(
None,
"abc-corp:vn-00",
"10.10.10.10",
ifs=[{"name": "host1:eth0", "href": "127.0.0.1:8081/analytics/uves/if/host1:eth0?cfilt=mystruct"}],
)
cn = OpServerUtils.uve_attr_flatten(uvetest["abc-corp:vn-00"]["UVEVirtualNetwork"]["ifs"]["10.10.10.10"])
self.assertEqual(cn, res["UVEVirtualNetwork"]["ifs"])
def test_href_agg(self):
logging.info("*** Running test_href_agg ***")
uvevn = MakeUVEVirtualNetwork(
None, "abc-corp:vn-00", "10.10.10.10",
ifs=["host1:eth0"],
)
pa = ParallelAggregator(uvevn, {"ObjectIf":"if"})
res = pa.aggregate("abc-corp:vn-00", True, "127.0.0.1:8081")
logging.info(json.dumps(res, indent=4, sort_keys=True))
uvetest = MakeUVEVirtualNetwork(
None, "abc-corp:vn-00", "10.10.10.10",
ifs=[{"name":"host1:eth0",
"href":"127.0.0.1:8081/analytics/uves/if/host1:eth0?cfilt=mystruct"}],
)
cn = OpServerUtils.uve_attr_flatten(
uvetest["abc-corp:vn-00"]['UVEVirtualNetwork'][
"ifs"]["10.10.10.10"])
self.assertEqual(cn, res['UVEVirtualNetwork']['ifs'])
def parse_args(self):
"""
Eg. python log.py --opserver-ip 127.0.0.1
--opserver-port 8081
--source 127.0.0.1
--module bgp | cfgm | vnswad
--message-type UveVirtualMachineConfigTrace
--category xmpp
--level SYS_INFO | SYS_ERROR
--object vn | vm
--object-id name
--object-select-field ObjectLog | SystemLog
--reverse
--verbose
--raw
--trace BgpPeerTraceBuf
[--start-time now-10m --end-time now] | --last 10m
"""
defaults = {
'opserver_ip': '127.0.0.1',
'opserver_port': '8081',
'start_time': 'now-10m',
'end_time': 'now',
}
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.set_defaults(**defaults)
parser.add_argument("--opserver-ip", help="IP address of OpServer")
parser.add_argument("--opserver-port", help="Port of OpServer")
parser.add_argument(
"--start-time", help="Logs start time (format now-10m, now-1h)")
parser.add_argument("--end-time", help="Logs end time")
parser.add_argument(
"--last", help="Logs from last time period (format 10m, 1d)")
parser.add_argument("--source", help="Logs from source address")
parser.add_argument(
"--module", help="Logs from module", choices=ModuleNames.values())
parser.add_argument("--category", help="Logs of category")
parser.add_argument("--level", help="Logs of level")
parser.add_argument("--message-type", help="Logs of message type")
parser.add_argument("--reverse", action="store_true",
help="Show logs in reverse chronological order")
parser.add_argument(
"--verbose", action="store_true", help="Show internal information")
parser.add_argument(
"--all", action="store_true", help="Show all logs")
parser.add_argument(
"--raw", action="store_true", help="Show raw XML messages")
parser.add_argument(
"--object", help="Logs of object type", choices=OBJECT_TABLE_LIST)
parser.add_argument("--object-id", help="Logs of object name")
parser.add_argument(
"--object-select-field", help="Select field to filter the log",
choices=[VizConstants.OBJECT_LOG, VizConstants.SYSTEM_LOG])
parser.add_argument("--trace", help="Dump trace buffer")
parser.add_argument("--limit", help="Limit the number of messages")
self._args = parser.parse_args()
if self._args.last is not None:
self._args.last = '-' + self._args.last
self._start_time = OpServerUtils.convert_to_utc_timestamp_usec(
self._args.last)
self._end_time = OpServerUtils.convert_to_utc_timestamp_usec('now')
else:
try:
if (self._args.start_time.isdigit() and
self._args.end_time.isdigit()):
self._start_time = int(self._args.start_time)
self._end_time = int(self._args.end_time)
else:
self._start_time =\
OpServerUtils.convert_to_utc_timestamp_usec(
self._args.start_time)
self._end_time =\
OpServerUtils.convert_to_utc_timestamp_usec(
self._args.end_time)
except:
print 'Incorrect start-time (%s) or end-time (%s) format' %\
(self._args.start_time, self._args.end_time)
return -1
return 0
def display(self, result):
if result == [] or result is None:
return
messages_dict_list = result
for messages_dict in messages_dict_list:
if VizConstants.TIMESTAMP in messages_dict:
message_dt = datetime.datetime.fromtimestamp(
int(messages_dict[VizConstants.TIMESTAMP]) /
OpServerUtils.USECS_IN_SEC)
message_dt += datetime.timedelta(
microseconds=(int(messages_dict[VizConstants.TIMESTAMP]) %
OpServerUtils.USECS_IN_SEC))
message_ts = message_dt.strftime(OpServerUtils.TIME_FORMAT_STR)
else:
message_ts = 'Time: NA'
if VizConstants.SOURCE in messages_dict:
source = messages_dict[VizConstants.SOURCE]
else:
source = 'Source: NA'
if VizConstants.NODE_TYPE in messages_dict:
node_type = messages_dict[VizConstants.NODE_TYPE]
else:
node_type = ''
if VizConstants.MODULE in messages_dict:
module = messages_dict[VizConstants.MODULE]
else:
module = 'Module: NA'
if VizConstants.INSTANCE_ID in messages_dict:
instance_id = messages_dict[VizConstants.INSTANCE_ID]
else:
instance_id = ''
if VizConstants.MESSAGE_TYPE in messages_dict:
message_type = messages_dict[VizConstants.MESSAGE_TYPE]
else:
message_type = 'Message Type: NA'
if VizConstants.SANDESH_TYPE in messages_dict:
sandesh_type = messages_dict[VizConstants.SANDESH_TYPE]
else:
sandesh_type = SandeshType.INVALID
if self._args.object is None:
if VizConstants.CATEGORY in messages_dict:
category = messages_dict[VizConstants.CATEGORY]
else:
category = 'Category: NA'
if VizConstants.LEVEL in messages_dict:
level = SandeshLevel._VALUES_TO_NAMES[messages_dict[
VizConstants.LEVEL]]
else:
level = 'Level: NA'
if VizConstants.SEQUENCE_NUM in messages_dict:
seq_num = messages_dict[VizConstants.SEQUENCE_NUM]
else:
seq_num = 'Sequence Number: NA'
if VizConstants.DATA in messages_dict:
# Convert XML data to dict
if self._args.raw:
data_str = messages_dict[VizConstants.DATA]
else:
OpServerUtils.messages_xml_data_to_dict(
messages_dict, VizConstants.DATA)
if isinstance(messages_dict[VizConstants.DATA], dict):
data_dict = messages_dict[VizConstants.DATA]
data_str = OpServerUtils.messages_data_dict_to_str(
data_dict, message_type, sandesh_type)
else:
data_str = messages_dict[VizConstants.DATA]
else:
data_str = 'Data not present'
if self._args.trace is not None:
print '{0} {1}:{2} {3}'.format(message_ts, message_type,
seq_num, data_str)
else:
print '{0} {1} [{2}:{3}:{4}:{5}][{6}] : {7}:{8} {9}'.format(
message_ts, source, node_type, module, instance_id,
category, level, message_type, seq_num, data_str)
else:
for obj_sel_field in self._args.object_select_field:
if obj_sel_field in messages_dict:
if self._args.raw:
data_str = messages_dict[obj_sel_field]
else:
# Convert XML data to dict
OpServerUtils.messages_xml_data_to_dict(
messages_dict, obj_sel_field)
if isinstance(messages_dict[obj_sel_field], dict):
data_dict = messages_dict[obj_sel_field]
data_str =\
OpServerUtils.messages_data_dict_to_str(
data_dict, message_type,
sandesh_type)
else:
data_str = messages_dict[obj_sel_field]
if data_str:
print '{0} {1} [{2}:{3}:{4}] : {5}: {6}'.format(
message_ts, source, node_type, module,
instance_id, message_type, data_str)
def display(self, result):
if result == [] or result is None:
return
flow_dict_list = result
for flow_dict in flow_dict_list:
# Setup time
if self._SETUP_TIME in flow_dict:
setup_time = int(flow_dict[self._SETUP_TIME])
if setup_time != 0:
setup_dt = datetime.datetime.fromtimestamp(
setup_time /
OpServerUtils.USECS_IN_SEC)
setup_dt += datetime.timedelta(
microseconds=
(setup_time %
OpServerUtils.USECS_IN_SEC))
setup_ts = setup_dt.strftime(
OpServerUtils.TIME_FORMAT_STR)
else:
setup_ts = 'Setup Time: NA'
else:
setup_ts = 'Setup Time: NA'
# Teardown time
if self._TEARDOWN_TIME in flow_dict:
teardown_time = int(flow_dict[
self._TEARDOWN_TIME])
if teardown_time != 0:
teardown_dt = datetime.datetime.fromtimestamp(
teardown_time /
OpServerUtils.USECS_IN_SEC)
teardown_dt += datetime.timedelta(
microseconds=
(teardown_time %
OpServerUtils.USECS_IN_SEC))
teardown_ts = teardown_dt.strftime(
OpServerUtils.TIME_FORMAT_STR)
else:
teardown_ts = 'Active'
else:
teardown_ts = 'Active'
# VRouter
if self._VROUTER in flow_dict:
vrouter = flow_dict[self._VROUTER]
else:
vrouter = 'VRouter: NA'
# Direction
if self._DIRECTION in flow_dict:
direction = int(flow_dict[self._DIRECTION])
if direction == 1:
direction = 'ingress'
elif direction == 0:
direction = 'egress'
else:
direction = 'Direction: Invalid'
else:
direction = 'Direction: NA'
# Flow UUID
if VizConstants.FLOW_TABLE_UUID in flow_dict:
flow_uuid = flow_dict[VizConstants.FLOW_TABLE_UUID]
else:
flow_uuid = 'UUID: NA'
# Source VN
if self._SOURCE_VN in flow_dict:
source_vn = flow_dict[self._SOURCE_VN]
else:
source_vn = 'Source VN: NA'
# Destination VN
if self._DESTINATION_VN in flow_dict:
destination_vn = flow_dict[self._DESTINATION_VN]
else:
destination_vn = 'Destination VN: NA'
# Source IP
if self._SOURCE_IP in flow_dict:
source_ip = flow_dict[self._SOURCE_IP]
else:
source_ip = 'Source IP: NA'
# Destination IP
if self._DESTINATION_IP in flow_dict:
destination_ip = flow_dict[self._DESTINATION_IP]
else:
destination_ip = 'Destination IP: NA'
# Source port
if self._SOURCE_PORT in flow_dict:
source_port = flow_dict[self._SOURCE_PORT]
else:
source_port = 'Source Port: NA'
# Destination port
if self._DESTINATION_PORT in flow_dict:
destination_port = flow_dict[self._DESTINATION_PORT]
else:
destination_port = 'Destination Port: NA'
# Protocol
if self._PROTOCOL in flow_dict:
protocol = OpServerUtils.ip_protocol_to_str(
int(flow_dict[self._PROTOCOL]))
else:
protocol = 'Protocol: NA'
# Action
if self._ACTION in flow_dict:
action = flow_dict[self._ACTION]
else:
action = ''
# Agg packets and bytes
if VizConstants.FLOW_TABLE_AGG_BYTES in flow_dict:
agg_bytes = int(flow_dict[VizConstants.FLOW_TABLE_AGG_BYTES])
else:
agg_bytes = 'Agg Bytes: NA'
if VizConstants.FLOW_TABLE_AGG_PKTS in flow_dict:
agg_pkts = int(flow_dict[VizConstants.FLOW_TABLE_AGG_PKTS])
else:
agg_pkts = 'Agg Packets: NA'
print '{0}({1}) {2} [{3} -- {4}] {5} '\
'{6}:{7}:{8} ---> {9}:{10}:{11} [{12} P ({13} B)]'\
' : {14}'.format(
vrouter, direction, action, setup_ts, teardown_ts,
protocol, source_vn, source_ip, source_port, destination_vn,
destination_ip, destination_port, agg_pkts, agg_bytes,
flow_uuid)
def parse_args(self):
"""
Eg. python stats.py --opserver-ip 127.0.0.1
--opserver-port 8081
--table AnalyticsCpuState.cpu_info
--where name=a6s40 cpu_info.module_id=Collector
--select "T=60 SUM(cpu_info.cpu_share)"
--sort "SUM(cpu_info.cpu_share)"
[--start-time now-10m --end-time now] | --last 10m
python stats.py --table AnalyticsCpuState.cpu_info
"""
defaults = {
'opserver_ip': '127.0.0.1',
'opserver_port': '8081',
'start_time': 'now-10m',
'end_time': 'now',
'select' : [],
'where' : [],
'sort': []
}
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.set_defaults(**defaults)
parser.add_argument("--opserver-ip", help="IP address of OpServer")
parser.add_argument("--opserver-port", help="Port of OpServer")
parser.add_argument(
"--start-time", help="Logs start time (format now-10m, now-1h)")
parser.add_argument("--end-time", help="Logs end time")
parser.add_argument(
"--last", help="Logs from last time period (format 10m, 1d)")
parser.add_argument(
"--table", help="StatTable to query", choices=STAT_TABLE_LIST)
parser.add_argument(
"--select", help="List of Select Terms", nargs='+')
parser.add_argument(
"--where", help="List of Where Terms to be ANDed", nargs='+')
parser.add_argument(
"--sort", help="List of Sort Terms", nargs='+')
self._args = parser.parse_args()
if self._args.table is None:
return -1
if self._args.last is not None:
self._args.last = '-' + self._args.last
self._start_time = OpServerUtils.convert_to_utc_timestamp_usec(
self._args.last)
self._end_time = OpServerUtils.convert_to_utc_timestamp_usec('now')
else:
try:
if (self._args.start_time.isdigit() and
self._args.end_time.isdigit()):
self._start_time = int(self._args.start_time)
self._end_time = int(self._args.end_time)
else:
self._start_time =\
OpServerUtils.convert_to_utc_timestamp_usec(
self._args.start_time)
self._end_time =\
OpServerUtils.convert_to_utc_timestamp_usec(
self._args.end_time)
except:
print 'Incorrect start-time (%s) or end-time (%s) format' %\
(self._args.start_time, self._args.end_time)
return -1
return 0
def parse_args(self):
"""
Eg. python stats.py --opserver-ip 127.0.0.1
--opserver-port 8081
--table AnalyticsCpuState.cpu_info
--where name=a6s40 cpu_info.module_id=Collector
--select "T=60 SUM(cpu_info.cpu_share)"
--sort "SUM(cpu_info.cpu_share)"
[--start-time now-10m --end-time now] | --last 10m
python stats.py --table AnalyticsCpuState.cpu_info
"""
defaults = {
'opserver_ip': '127.0.0.1',
'opserver_port': '8081',
'start_time': 'now-10m',
'end_time': 'now',
'select': [],
'where': [],
'sort': []
}
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.set_defaults(**defaults)
parser.add_argument("--opserver-ip", help="IP address of OpServer")
parser.add_argument("--opserver-port", help="Port of OpServer")
parser.add_argument("--start-time",
help="Logs start time (format now-10m, now-1h)")
parser.add_argument("--end-time", help="Logs end time")
parser.add_argument("--last",
help="Logs from last time period (format 10m, 1d)")
parser.add_argument("--table",
help="StatTable to query",
choices=STAT_TABLE_LIST)
parser.add_argument("--select", help="List of Select Terms", nargs='+')
parser.add_argument("--where",
help="List of Where Terms to be ANDed",
nargs='+')
parser.add_argument("--sort", help="List of Sort Terms", nargs='+')
self._args = parser.parse_args()
if self._args.table is None:
return -1
if self._args.last is not None:
self._args.last = '-' + self._args.last
self._start_time = OpServerUtils.convert_to_utc_timestamp_usec(
self._args.last)
self._end_time = OpServerUtils.convert_to_utc_timestamp_usec('now')
else:
try:
if (self._args.start_time.isdigit()
and self._args.end_time.isdigit()):
self._start_time = int(self._args.start_time)
self._end_time = int(self._args.end_time)
else:
self._start_time =\
OpServerUtils.convert_to_utc_timestamp_usec(
self._args.start_time)
self._end_time =\
OpServerUtils.convert_to_utc_timestamp_usec(
self._args.end_time)
except:
print 'Incorrect start-time (%s) or end-time (%s) format' %\
(self._args.start_time, self._args.end_time)
return -1
return 0
def query(self):
start_time, end_time = OpServerUtils.get_start_end_time(
self._start_time, self._end_time)
messages_url = OpServerUtils.opserver_query_url(
self._args.opserver_ip, self._args.opserver_port)
where_msg = []
where_obj = []
filter = []
if self._args.source is not None:
source_match = OpServerUtils.Match(name=VizConstants.SOURCE,
value=self._args.source,
op=OpServerUtils.MatchOp.EQUAL)
where_msg.append(source_match.__dict__)
if self._args.module is not None:
module_match = OpServerUtils.Match(name=VizConstants.MODULE,
value=self._args.module,
op=OpServerUtils.MatchOp.EQUAL)
where_msg.append(module_match.__dict__)
if self._args.category is not None:
category_match = OpServerUtils.Match(
name=VizConstants.CATEGORY,
value=self._args.category,
op=OpServerUtils.MatchOp.EQUAL)
where_msg.append(category_match.__dict__)
if self._args.message_type is not None:
message_type_match = OpServerUtils.Match(
name=VizConstants.MESSAGE_TYPE,
value=self._args.message_type,
op=OpServerUtils.MatchOp.EQUAL)
where_msg.append(message_type_match.__dict__)
if self._args.level is not None:
level_match = OpServerUtils.Match(
name=VizConstants.LEVEL,
value=SandeshLevel._NAMES_TO_VALUES[self._args.level],
op=OpServerUtils.MatchOp.GEQ)
filter.append(level_match.__dict__)
if self._args.node_type is not None:
node_type_match = OpServerUtils.Match(
name=VizConstants.NODE_TYPE,
value=self._args.node_type,
op=OpServerUtils.MatchOp.EQUAL)
filter.append(node_type_match.__dict__)
if self._args.instance_id is not None:
instance_id_match = OpServerUtils.Match(
name=VizConstants.INSTANCE_ID,
value=self._args.instance_id,
op=OpServerUtils.MatchOp.EQUAL)
filter.append(instance_id_match.__dict__)
if (self._args.object is not None or self._args.object_id is not None
or self._args.object_select_field is not None):
# Object Table Query
where_obj = list(where_msg)
if self._args.object is not None:
if self._args.object in OBJECT_TABLE_LIST:
table = self._args.object
else:
print 'Unknown object table [%s]' % (self._args.object)
return None
else:
print 'Object required for query'
return None
if self._args.object_id is not None:
id_match = OpServerUtils.Match(name=OpServerUtils.OBJECT_ID,
value=self._args.object_id,
op=OpServerUtils.MatchOp.EQUAL)
where_obj.append(id_match.__dict__)
else:
print 'Object id required for table [%s]' % (self._args.object)
return None
if self._args.object_select_field is not None:
if ((self._args.object_select_field != VizConstants.OBJECT_LOG)
and (self._args.object_select_field !=
VizConstants.SYSTEM_LOG)):
print 'Invalid object-select-field. '\
'Valid values are "%s" or "%s"' \
% (VizConstants.OBJECT_LOG,
VizConstants.SYSTEM_LOG)
return None
obj_sel_field = [self._args.object_select_field]
self._args.object_select_field = obj_sel_field
else:
self._args.object_select_field = obj_sel_field = [
VizConstants.OBJECT_LOG, VizConstants.SYSTEM_LOG
]
where = [where_obj]
select_list = [
VizConstants.TIMESTAMP,
VizConstants.SOURCE,
VizConstants.MODULE,
VizConstants.MESSAGE_TYPE,
] + obj_sel_field
elif self._args.trace is not None:
table = VizConstants.COLLECTOR_GLOBAL_TABLE
if self._args.source is None:
print 'Source is required for trace buffer dump'
return None
if self._args.module is None:
print 'Module is required for trace buffer dump'
return None
trace_buf_match = OpServerUtils.Match(
name=VizConstants.CATEGORY,
value=self._args.trace,
op=OpServerUtils.MatchOp.EQUAL)
where_msg.append(trace_buf_match.__dict__)
where = [where_msg]
select_list = [
VizConstants.TIMESTAMP, VizConstants.MESSAGE_TYPE,
VizConstants.SEQUENCE_NUM, VizConstants.DATA
]
sandesh_type_filter = OpServerUtils.Match(
name=VizConstants.SANDESH_TYPE,
value=str(SandeshType.TRACE),
op=OpServerUtils.MatchOp.EQUAL)
filter.append(sandesh_type_filter.__dict__)
else:
# Message Table Query
table = VizConstants.COLLECTOR_GLOBAL_TABLE
# Message Table contains both systemlog and objectlog.
# Add a filter to return only systemlogs
if not self._args.all:
sandesh_type_filter = OpServerUtils.Match(
name=VizConstants.SANDESH_TYPE,
value=str(SandeshType.SYSTEM),
op=OpServerUtils.MatchOp.EQUAL)
filter.append(sandesh_type_filter.__dict__)
if len(where_msg):
where = [where_msg]
else:
where = None
select_list = [
VizConstants.TIMESTAMP,
VizConstants.SOURCE,
VizConstants.MODULE,
VizConstants.CATEGORY,
VizConstants.MESSAGE_TYPE,
VizConstants.SEQUENCE_NUM,
VizConstants.DATA,
VizConstants.SANDESH_TYPE,
VizConstants.LEVEL,
VizConstants.NODE_TYPE,
VizConstants.INSTANCE_ID,
]
if len(filter) == 0:
filter = None
# Add sort by timestamp
if self._args.reverse:
sort_op = OpServerUtils.SortOp.DESCENDING
else:
sort_op = OpServerUtils.SortOp.ASCENDING
sort_fields = [VizConstants.TIMESTAMP]
if self._args.limit:
limit = int(self._args.limit)
else:
limit = None
messages_query = OpServerUtils.Query(table,
start_time=start_time,
end_time=end_time,
select_fields=select_list,
where=where,
filter=filter,
sort=sort_op,
sort_fields=sort_fields,
limit=limit)
if self._args.verbose:
print 'Performing query: {0}'.format(
json.dumps(messages_query.__dict__))
print ''
resp = OpServerUtils.post_url_http(messages_url,
json.dumps(messages_query.__dict__))
result = {}
if resp is not None:
resp = json.loads(resp)
qid = resp['href'].rsplit('/', 1)[1]
result = OpServerUtils.get_query_result(self._args.opserver_ip,
self._args.opserver_port,
qid)
return result
def query(self):
start_time, end_time = OpServerUtils.get_start_end_time(
self._start_time,
self._end_time)
flow_url = OpServerUtils.opserver_query_url(
self._args.opserver_ip,
self._args.opserver_port)
where = []
filter = []
if self._args.vrouter is not None:
vrouter_match = OpServerUtils.Match(
name=self._VROUTER,
value=self._args.vrouter,
op=OpServerUtils.MatchOp.EQUAL)
where.append(vrouter_match.__dict__)
if self._args.source_vn is not None:
source_vn_match = OpServerUtils.Match(
name=self._SOURCE_VN,
value=self._args.source_vn,
op=OpServerUtils.MatchOp.EQUAL)
where.append(source_vn_match.__dict__)
if self._args.destination_vn is not None:
dest_vn_match = OpServerUtils.Match(
name=self._DESTINATION_VN,
value=self._args.destination_vn,
op=OpServerUtils.MatchOp.EQUAL)
where.append(dest_vn_match.__dict__)
if self._args.source_ip is not None:
source_ip_match = OpServerUtils.Match(
name=self._SOURCE_IP,
value=self._args.source_ip,
op=OpServerUtils.MatchOp.EQUAL)
where.append(source_ip_match.__dict__)
if self._args.destination_ip is not None:
dest_ip_match = OpServerUtils.Match(
name=self._DESTINATION_IP,
value=self._args.destination_ip,
op=OpServerUtils.MatchOp.EQUAL)
where.append(dest_ip_match.__dict__)
if self._args.protocol is not None:
protocol_match = OpServerUtils.Match(
name=self._PROTOCOL,
value=self._args.protocol,
op=OpServerUtils.MatchOp.EQUAL)
where.append(protocol_match.__dict__)
if self._args.source_port is not None:
source_port_match = OpServerUtils.Match(
name=self._SOURCE_PORT,
value=self._args.source_port,
op=OpServerUtils.MatchOp.EQUAL)
where.append(source_port_match.__dict__)
if self._args.destination_port is not None:
dest_port_match = OpServerUtils.Match(
name=self._DESTINATION_PORT,
value=self._args.destination_port,
op=OpServerUtils.MatchOp.EQUAL)
where.append(dest_port_match.__dict__)
if self._args.action is not None:
action_match = OpServerUtils.Match(
name=self._ACTION,
value=self._args.action,
op=OpServerUtils.MatchOp.EQUAL)
filter.append(action_match.__dict__)
# Flow Record Table Query
table = VizConstants.FLOW_TABLE
if len(where) == 0:
where = None
else:
where = [where]
select_list = [
VizConstants.FLOW_TABLE_UUID,
self._VROUTER,
self._SETUP_TIME,
self._TEARDOWN_TIME,
self._SOURCE_VN,
self._DESTINATION_VN,
self._SOURCE_IP,
self._DESTINATION_IP,
self._PROTOCOL,
self._SOURCE_PORT,
self._DESTINATION_PORT,
self._ACTION,
self._DIRECTION,
VizConstants.FLOW_TABLE_AGG_BYTES,
VizConstants.FLOW_TABLE_AGG_PKTS,
]
if len(filter) == 0:
filter = None
flow_query = OpServerUtils.Query(table,
start_time=start_time,
end_time=end_time,
select_fields=select_list,
where=where,
filter=filter,
dir=self._args.direction)
if self._args.verbose:
print 'Performing query: {0}'.format(
json.dumps(flow_query.__dict__))
print ''
resp = OpServerUtils.post_url_http(
flow_url, json.dumps(flow_query.__dict__))
result = {}
if resp is not None:
resp = json.loads(resp)
qid = resp['href'].rsplit('/', 1)[1]
result = OpServerUtils.get_query_result(
self._args.opserver_ip, self._args.opserver_port, qid)
return result
def display(self, result):
if result == [] or result is None:
return
messages_dict_list = result
for messages_dict in messages_dict_list:
if VizConstants.TIMESTAMP in messages_dict:
message_dt = datetime.datetime.fromtimestamp(
int(messages_dict[VizConstants.TIMESTAMP]) /
OpServerUtils.USECS_IN_SEC)
message_dt += datetime.timedelta(
microseconds=
(int(messages_dict[VizConstants.TIMESTAMP]) %
OpServerUtils.USECS_IN_SEC))
message_ts = message_dt.strftime(OpServerUtils.TIME_FORMAT_STR)
else:
message_ts = 'Time: NA'
if VizConstants.SOURCE in messages_dict:
source = messages_dict[VizConstants.SOURCE]
else:
source = 'Source: NA'
if VizConstants.MODULE in messages_dict:
module = messages_dict[VizConstants.MODULE]
else:
module = 'Module: NA'
if VizConstants.MESSAGE_TYPE in messages_dict:
message_type = messages_dict[VizConstants.MESSAGE_TYPE]
else:
message_type = 'Message Type: NA'
if VizConstants.SANDESH_TYPE in messages_dict:
sandesh_type = messages_dict[VizConstants.SANDESH_TYPE]
else:
sandesh_type = SandeshType.INVALID
if self._args.object is None:
if VizConstants.CATEGORY in messages_dict:
category = messages_dict[VizConstants.CATEGORY]
else:
category = 'Category: NA'
if VizConstants.LEVEL in messages_dict:
level = SandeshLevel._VALUES_TO_NAMES[
messages_dict[VizConstants.LEVEL]]
else:
level = 'Level: NA'
if VizConstants.SEQUENCE_NUM in messages_dict:
seq_num = messages_dict[VizConstants.SEQUENCE_NUM]
else:
seq_num = 'Sequence Number: NA'
if VizConstants.DATA in messages_dict:
# Convert XML data to dict
if self._args.raw:
data_str = messages_dict[VizConstants.DATA]
else:
OpServerUtils.messages_xml_data_to_dict(
messages_dict, VizConstants.DATA)
if isinstance(messages_dict[VizConstants.DATA], dict):
data_dict = messages_dict[VizConstants.DATA]
data_str = OpServerUtils.messages_data_dict_to_str(
data_dict, message_type, sandesh_type)
else:
data_str = messages_dict[VizConstants.DATA]
else:
data_str = 'Data not present'
if self._args.trace is not None:
print '{0} {1}:{2} {3}'.format(
message_ts, message_type, seq_num, data_str)
else:
print '{0} {1} [{2}:{3}][{4}] : {5}:{6} {7}'.format(
message_ts, source, module,
category, level, message_type, seq_num, data_str)
else:
for obj_sel_field in self._args.object_select_field:
if obj_sel_field in messages_dict:
if self._args.raw:
data_str = messages_dict[obj_sel_field]
else:
# Convert XML data to dict
OpServerUtils.messages_xml_data_to_dict(
messages_dict, obj_sel_field)
if isinstance(messages_dict[obj_sel_field], dict):
data_dict = messages_dict[obj_sel_field]
data_str =\
OpServerUtils.messages_data_dict_to_str(
data_dict, message_type,
sandesh_type)
else:
data_str = messages_dict[obj_sel_field]
if data_str:
print '{0} [{1}:{2}] : {3}: {4}'.format(
message_ts, source, module,
message_type, data_str)
def parse_args(self):
"""
Eg. python flow.py --opserver-ip 127.0.0.1
--opserver-port 8081
--vrouter a6s23
--source-vn default-domain:default-project:vn1
--destination-vn default-domain:default-project:vn2
--source-ip 1.1.1.1
--destination-ip 2.2.2.2
--protocol TCP
--source-port 32678
--destination-port 80
--action drop
--direction ingress
[--start-time now-10m --end-time now] | --last 10m
"""
defaults = {
'opserver_ip': '127.0.0.1',
'opserver_port': '8081',
'start_time': 'now-10m',
'end_time': 'now',
'direction' : 'ingress',
}
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.set_defaults(**defaults)
parser.add_argument("--opserver-ip", help="IP address of OpServer")
parser.add_argument("--opserver-port", help="Port of OpServer")
parser.add_argument(
"--start-time", help="Flow record start time (format now-10m, now-1h)")
parser.add_argument("--end-time", help="Flow record end time")
parser.add_argument(
"--last", help="Flow records from last time period (format 10m, 1d)")
parser.add_argument("--vrouter", help="Flow records from vrouter")
parser.add_argument("--source-vn",
help="Flow records with source virtual network")
parser.add_argument("--destination-vn",
help="Flow records with destination virtual network")
parser.add_argument("--source-ip",
help="Flow records with source IP address")
parser.add_argument("--destination-ip",
help="Flow records with destination IP address")
parser.add_argument("--protocol", help="Flow records with protocol")
parser.add_argument("--source-port",
help="Flow records with source port", type=int)
parser.add_argument("--destination-port",
help="Flow records with destination port", type=int)
parser.add_argument("--action", help="Flow records with action")
parser.add_argument("--direction", help="Flow direction",
choices=['ingress', 'egress'])
parser.add_argument(
"--verbose", action="store_true", help="Show internal information")
self._args = parser.parse_args()
# Validate start-time and end-time
if self._args.last is not None:
self._args.last = '-' + self._args.last
self._start_time = OpServerUtils.convert_to_utc_timestamp_usec(
self._args.last)
self._end_time = OpServerUtils.convert_to_utc_timestamp_usec('now')
else:
try:
if (self._args.start_time.isdigit() and
self._args.end_time.isdigit()):
self._start_time = int(self._args.start_time)
self._end_time = int(self._args.end_time)
else:
self._start_time =\
OpServerUtils.convert_to_utc_timestamp_usec(
self._args.start_time)
self._end_time =\
OpServerUtils.convert_to_utc_timestamp_usec(
self._args.end_time)
except:
print 'Incorrect start-time (%s) or end-time (%s) format' %\
(self._args.start_time, self._args.end_time)
return -1
# Validate flow arguments
if self._args.source_ip is not None and self._args.source_vn is None:
print 'Please provide source virtual network in addtion to '\
'source IP address'
return -1
if self._args.destination_ip is not None and \
self._args.destination_vn is None:
print 'Please provide destination virtual network in addtion to '\
'destination IP address'
return -1
if self._args.source_port is not None and self._args.protocol is None:
print 'Please provide protocol in addtion to source port'
return -1
if self._args.destination_port is not None and \
self._args.protocol is None:
print 'Please provide protocol in addtion to '\
'destination port'
return -1
# Convert direction
if self._args.direction.lower() == "ingress":
self._args.direction = 1
elif self._args.direction.lower() == "egress":
self._args.direction = 0
else:
print 'Direction should be ingress or egress'
return -1
# Protocol
if self._args.protocol is not None:
if self._args.protocol.isalpha():
protocol = OpServerUtils.str_to_ip_protocol(
self._args.protocol)
if protocol == -1:
print 'Please provide valid protocol'
return -1
self._args.protocol = protocol
return 0
def parse_args(self):
"""
Eg. python log.py --opserver-ip 127.0.0.1
--opserver-port 8081
--source 127.0.0.1
--node-type Control
--module bgp | cfgm | vnswad
--instance-id 0
--message-type UveVirtualMachineConfigTrace
--category xmpp
--level SYS_INFO | SYS_ERROR
--object vn | vm
--object-id name
--object-select-field ObjectLog | SystemLog
--reverse
--verbose
--raw
--trace BgpPeerTraceBuf
[--start-time now-10m --end-time now] | --last 10m
"""
defaults = {
'opserver_ip': '127.0.0.1',
'opserver_port': '8081',
'start_time': 'now-10m',
'end_time': 'now',
}
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.set_defaults(**defaults)
parser.add_argument("--opserver-ip", help="IP address of OpServer")
parser.add_argument("--opserver-port", help="Port of OpServer")
parser.add_argument("--start-time",
help="Logs start time (format now-10m, now-1h)")
parser.add_argument("--end-time", help="Logs end time")
parser.add_argument("--last",
help="Logs from last time period (format 10m, 1d)")
parser.add_argument("--source", help="Logs from source address")
parser.add_argument("--node-type",
help="Logs from node type",
choices=NodeTypeNames.values())
parser.add_argument("--module",
help="Logs from module",
choices=ModuleNames.values())
parser.add_argument("--instance-id", help="Logs from module instance")
parser.add_argument("--category", help="Logs of category")
parser.add_argument("--level", help="Logs of level")
parser.add_argument("--message-type", help="Logs of message type")
parser.add_argument("--reverse",
action="store_true",
help="Show logs in reverse chronological order")
parser.add_argument("--verbose",
action="store_true",
help="Show internal information")
parser.add_argument("--all", action="store_true", help="Show all logs")
parser.add_argument("--raw",
action="store_true",
help="Show raw XML messages")
parser.add_argument("--object",
help="Logs of object type",
choices=OBJECT_TABLE_LIST)
parser.add_argument("--object-id", help="Logs of object name")
parser.add_argument(
"--object-select-field",
help="Select field to filter the log",
choices=[VizConstants.OBJECT_LOG, VizConstants.SYSTEM_LOG])
parser.add_argument("--trace", help="Dump trace buffer")
parser.add_argument("--limit", help="Limit the number of messages")
self._args = parser.parse_args()
if self._args.last is not None:
self._args.last = '-' + self._args.last
self._start_time = OpServerUtils.convert_to_utc_timestamp_usec(
self._args.last)
self._end_time = OpServerUtils.convert_to_utc_timestamp_usec('now')
else:
try:
if (self._args.start_time.isdigit()
and self._args.end_time.isdigit()):
self._start_time = int(self._args.start_time)
self._end_time = int(self._args.end_time)
else:
self._start_time =\
OpServerUtils.convert_to_utc_timestamp_usec(
self._args.start_time)
self._end_time =\
OpServerUtils.convert_to_utc_timestamp_usec(
self._args.end_time)
except:
print 'Incorrect start-time (%s) or end-time (%s) format' %\
(self._args.start_time, self._args.end_time)
return -1
return 0
def display(self, result):
if result == [] or result is None:
return
flow_dict_list = result
for flow_dict in flow_dict_list:
# Setup time
if self._SETUP_TIME in flow_dict:
setup_time = int(flow_dict[self._SETUP_TIME])
if setup_time != 0:
setup_dt = datetime.datetime.fromtimestamp(
setup_time / OpServerUtils.USECS_IN_SEC)
setup_dt += datetime.timedelta(
microseconds=(setup_time % OpServerUtils.USECS_IN_SEC))
setup_ts = setup_dt.strftime(OpServerUtils.TIME_FORMAT_STR)
else:
setup_ts = 'Setup Time: NA'
else:
setup_ts = 'Setup Time: NA'
# Teardown time
if self._TEARDOWN_TIME in flow_dict:
teardown_time = int(flow_dict[self._TEARDOWN_TIME])
if teardown_time != 0:
teardown_dt = datetime.datetime.fromtimestamp(
teardown_time / OpServerUtils.USECS_IN_SEC)
teardown_dt += datetime.timedelta(
microseconds=(teardown_time %
OpServerUtils.USECS_IN_SEC))
teardown_ts = teardown_dt.strftime(
OpServerUtils.TIME_FORMAT_STR)
else:
teardown_ts = 'Active'
else:
teardown_ts = 'Active'
# VRouter
if self._VROUTER in flow_dict:
vrouter = flow_dict[self._VROUTER]
else:
vrouter = 'VRouter: NA'
# Direction
if self._DIRECTION in flow_dict:
direction = int(flow_dict[self._DIRECTION])
if direction == 1:
direction = 'ingress'
elif direction == 0:
direction = 'egress'
else:
direction = 'Direction: Invalid'
else:
direction = 'Direction: NA'
# Flow UUID
if VizConstants.FLOW_TABLE_UUID in flow_dict:
flow_uuid = flow_dict[VizConstants.FLOW_TABLE_UUID]
else:
flow_uuid = 'UUID: NA'
# Source VN
if self._SOURCE_VN in flow_dict:
source_vn = flow_dict[self._SOURCE_VN]
else:
source_vn = 'Source VN: NA'
# Destination VN
if self._DESTINATION_VN in flow_dict:
destination_vn = flow_dict[self._DESTINATION_VN]
else:
destination_vn = 'Destination VN: NA'
# Source IP
if self._SOURCE_IP in flow_dict:
source_ip = flow_dict[self._SOURCE_IP]
else:
source_ip = 'Source IP: NA'
# Destination IP
if self._DESTINATION_IP in flow_dict:
destination_ip = flow_dict[self._DESTINATION_IP]
else:
destination_ip = 'Destination IP: NA'
# Source port
if self._SOURCE_PORT in flow_dict:
source_port = flow_dict[self._SOURCE_PORT]
else:
source_port = 'Source Port: NA'
# Destination port
if self._DESTINATION_PORT in flow_dict:
destination_port = flow_dict[self._DESTINATION_PORT]
else:
destination_port = 'Destination Port: NA'
# Protocol
if self._PROTOCOL in flow_dict:
protocol = OpServerUtils.ip_protocol_to_str(
int(flow_dict[self._PROTOCOL]))
else:
protocol = 'Protocol: NA'
# Action
if self._ACTION in flow_dict:
action = flow_dict[self._ACTION]
else:
action = ''
# Agg packets and bytes
if VizConstants.FLOW_TABLE_AGG_BYTES in flow_dict:
agg_bytes = int(flow_dict[VizConstants.FLOW_TABLE_AGG_BYTES])
else:
agg_bytes = 'Agg Bytes: NA'
if VizConstants.FLOW_TABLE_AGG_PKTS in flow_dict:
agg_pkts = int(flow_dict[VizConstants.FLOW_TABLE_AGG_PKTS])
else:
agg_pkts = 'Agg Packets: NA'
print '{0}({1}) {2} [{3} -- {4}] {5} '\
'{6}:{7}:{8} ---> {9}:{10}:{11} [{12} P ({13} B)]'\
' : {14}'.format(
vrouter, direction, action, setup_ts, teardown_ts,
protocol, source_vn, source_ip, source_port, destination_vn,
destination_ip, destination_port, agg_pkts, agg_bytes,
flow_uuid)
def parse_args(self):
"""
Eg. python flow.py --opserver-ip 127.0.0.1
--opserver-port 8081
--vrouter a6s23
--source-vn default-domain:default-project:vn1
--destination-vn default-domain:default-project:vn2
--source-ip 1.1.1.1
--destination-ip 2.2.2.2
--protocol TCP
--source-port 32678
--destination-port 80
--action drop
--direction ingress
[--start-time now-10m --end-time now] | --last 10m
"""
defaults = {
'opserver_ip': '127.0.0.1',
'opserver_port': '8081',
'start_time': 'now-10m',
'end_time': 'now',
'direction': 'ingress',
}
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.set_defaults(**defaults)
parser.add_argument("--opserver-ip", help="IP address of OpServer")
parser.add_argument("--opserver-port", help="Port of OpServer")
parser.add_argument(
"--start-time",
help="Flow record start time (format now-10m, now-1h)")
parser.add_argument("--end-time", help="Flow record end time")
parser.add_argument(
"--last",
help="Flow records from last time period (format 10m, 1d)")
parser.add_argument("--vrouter", help="Flow records from vrouter")
parser.add_argument("--source-vn",
help="Flow records with source virtual network")
parser.add_argument(
"--destination-vn",
help="Flow records with destination virtual network")
parser.add_argument("--source-ip",
help="Flow records with source IP address")
parser.add_argument("--destination-ip",
help="Flow records with destination IP address")
parser.add_argument("--protocol", help="Flow records with protocol")
parser.add_argument("--source-port",
help="Flow records with source port",
type=int)
parser.add_argument("--destination-port",
help="Flow records with destination port",
type=int)
parser.add_argument("--action", help="Flow records with action")
parser.add_argument("--direction",
help="Flow direction",
choices=['ingress', 'egress'])
parser.add_argument("--verbose",
action="store_true",
help="Show internal information")
self._args = parser.parse_args()
# Validate start-time and end-time
if self._args.last is not None:
self._args.last = '-' + self._args.last
self._start_time = OpServerUtils.convert_to_utc_timestamp_usec(
self._args.last)
self._end_time = OpServerUtils.convert_to_utc_timestamp_usec('now')
else:
try:
if (self._args.start_time.isdigit()
and self._args.end_time.isdigit()):
self._start_time = int(self._args.start_time)
self._end_time = int(self._args.end_time)
else:
self._start_time =\
OpServerUtils.convert_to_utc_timestamp_usec(
self._args.start_time)
self._end_time =\
OpServerUtils.convert_to_utc_timestamp_usec(
self._args.end_time)
except:
print 'Incorrect start-time (%s) or end-time (%s) format' %\
(self._args.start_time, self._args.end_time)
return -1
# Validate flow arguments
if self._args.source_ip is not None and self._args.source_vn is None:
print 'Please provide source virtual network in addtion to '\
'source IP address'
return -1
if self._args.destination_ip is not None and \
self._args.destination_vn is None:
print 'Please provide destination virtual network in addtion to '\
'destination IP address'
return -1
if self._args.source_port is not None and self._args.protocol is None:
print 'Please provide protocol in addtion to source port'
return -1
if self._args.destination_port is not None and \
self._args.protocol is None:
print 'Please provide protocol in addtion to '\
'destination port'
return -1
# Convert direction
if self._args.direction.lower() == "ingress":
self._args.direction = 1
elif self._args.direction.lower() == "egress":
self._args.direction = 0
else:
print 'Direction should be ingress or egress'
return -1
# Protocol
if self._args.protocol is not None:
if self._args.protocol.isalpha():
protocol = OpServerUtils.str_to_ip_protocol(
self._args.protocol)
if protocol == -1:
print 'Please provide valid protocol'
return -1
self._args.protocol = protocol
return 0
def query(self):
start_time, end_time = OpServerUtils.get_start_end_time(
self._start_time,
self._end_time)
messages_url = OpServerUtils.opserver_query_url(
self._args.opserver_ip,
self._args.opserver_port)
where_msg = []
where_obj = []
filter = []
if self._args.source is not None:
source_match = OpServerUtils.Match(name=VizConstants.SOURCE,
value=self._args.source,
op=OpServerUtils.MatchOp.EQUAL)
where_msg.append(source_match.__dict__)
if self._args.module is not None:
module_match = OpServerUtils.Match(name=VizConstants.MODULE,
value=self._args.module,
op=OpServerUtils.MatchOp.EQUAL)
where_msg.append(module_match.__dict__)
if self._args.category is not None:
category_match = OpServerUtils.Match(
name=VizConstants.CATEGORY,
value=self._args.category,
op=OpServerUtils.MatchOp.EQUAL)
where_msg.append(category_match.__dict__)
if self._args.message_type is not None:
message_type_match = OpServerUtils.Match(
name=VizConstants.MESSAGE_TYPE,
value=self._args.message_type,
op=OpServerUtils.MatchOp.EQUAL)
where_msg.append(message_type_match.__dict__)
if self._args.level is not None:
level_match = OpServerUtils.Match(
name=VizConstants.LEVEL,
value=SandeshLevel._NAMES_TO_VALUES[self._args.level],
op=OpServerUtils.MatchOp.GEQ)
filter.append(level_match.__dict__)
if (self._args.object is not None or
self._args.object_id is not None or
self._args.object_select_field is not None):
# Object Table Query
where_obj = list(where_msg)
if self._args.object is not None:
if self._args.object in OBJECT_TABLE_LIST:
table = self._args.object
else:
print 'Unknown object table [%s]' % (self._args.object)
return None
else:
print 'Object required for query'
return None
if self._args.object_id is not None:
id_match = OpServerUtils.Match(name=OpServerUtils.OBJECT_ID,
value=self._args.object_id,
op=OpServerUtils.MatchOp.EQUAL)
where_obj.append(id_match.__dict__)
else:
print 'Object id required for table [%s]' % (self._args.object)
return None
if self._args.object_select_field is not None:
if ((self._args.object_select_field !=
VizConstants.OBJECT_LOG) and
(self._args.object_select_field !=
VizConstants.SYSTEM_LOG)):
print 'Invalid object-select-field. '\
'Valid values are "%s" or "%s"' \
% (VizConstants.OBJECT_LOG,
VizConstants.SYSTEM_LOG)
return None
obj_sel_field = [self._args.object_select_field]
self._args.object_select_field = obj_sel_field
else:
self._args.object_select_field = obj_sel_field = [
VizConstants.OBJECT_LOG, VizConstants.SYSTEM_LOG]
where = [where_obj]
select_list = [
VizConstants.TIMESTAMP,
VizConstants.SOURCE,
VizConstants.MODULE,
VizConstants.MESSAGE_TYPE,
] + obj_sel_field
elif self._args.trace is not None:
table = VizConstants.COLLECTOR_GLOBAL_TABLE
if self._args.source is None:
print 'Source is required for trace buffer dump'
return None
if self._args.module is None:
print 'Module is required for trace buffer dump'
return None
trace_buf_match = OpServerUtils.Match(
name=VizConstants.CATEGORY,
value=self._args.trace,
op=OpServerUtils.MatchOp.EQUAL)
where_msg.append(trace_buf_match.__dict__)
where = [where_msg]
select_list = [
VizConstants.TIMESTAMP,
VizConstants.MESSAGE_TYPE,
VizConstants.SEQUENCE_NUM,
VizConstants.DATA
]
sandesh_type_filter = OpServerUtils.Match(
name=VizConstants.SANDESH_TYPE,
value=str(
SandeshType.TRACE),
op=OpServerUtils.MatchOp.EQUAL)
filter.append(sandesh_type_filter.__dict__)
else:
# Message Table Query
table = VizConstants.COLLECTOR_GLOBAL_TABLE
# Message Table contains both systemlog and objectlog.
# Add a filter to return only systemlogs
if not self._args.all:
sandesh_type_filter = OpServerUtils.Match(
name=VizConstants.SANDESH_TYPE,
value=str(
SandeshType.SYSTEM),
op=OpServerUtils.MatchOp.EQUAL)
filter.append(sandesh_type_filter.__dict__)
if len(where_msg):
where = [where_msg]
else:
where = None
select_list = [
VizConstants.TIMESTAMP,
VizConstants.SOURCE,
VizConstants.MODULE,
VizConstants.CATEGORY,
VizConstants.MESSAGE_TYPE,
VizConstants.SEQUENCE_NUM,
VizConstants.DATA,
VizConstants.SANDESH_TYPE,
VizConstants.LEVEL,
]
if len(filter) == 0:
filter = None
# Add sort by timestamp
if self._args.reverse:
sort_op = OpServerUtils.SortOp.DESCENDING
else:
sort_op = OpServerUtils.SortOp.ASCENDING
sort_fields = [VizConstants.TIMESTAMP]
if self._args.limit:
limit = int(self._args.limit)
else:
limit = None
messages_query = OpServerUtils.Query(table,
start_time=start_time,
end_time=end_time,
select_fields=select_list,
where=where,
filter=filter,
sort=sort_op,
sort_fields=sort_fields,
limit=limit)
if self._args.verbose:
print 'Performing query: {0}'.format(
json.dumps(messages_query.__dict__))
print ''
resp = OpServerUtils.post_url_http(
messages_url, json.dumps(messages_query.__dict__))
result = {}
if resp is not None:
resp = json.loads(resp)
qid = resp['href'].rsplit('/', 1)[1]
result = OpServerUtils.get_query_result(
self._args.opserver_ip, self._args.opserver_port, qid)
return result
def query(self):
start_time, end_time = OpServerUtils.get_start_end_time(
self._start_time, self._end_time)
flow_url = OpServerUtils.opserver_query_url(self._args.opserver_ip,
self._args.opserver_port)
where = []
filter = []
if self._args.vrouter is not None:
vrouter_match = OpServerUtils.Match(name=self._VROUTER,
value=self._args.vrouter,
op=OpServerUtils.MatchOp.EQUAL)
where.append(vrouter_match.__dict__)
if self._args.source_vn is not None:
source_vn_match = OpServerUtils.Match(
name=self._SOURCE_VN,
value=self._args.source_vn,
op=OpServerUtils.MatchOp.EQUAL)
where.append(source_vn_match.__dict__)
if self._args.destination_vn is not None:
dest_vn_match = OpServerUtils.Match(
name=self._DESTINATION_VN,
value=self._args.destination_vn,
op=OpServerUtils.MatchOp.EQUAL)
where.append(dest_vn_match.__dict__)
if self._args.source_ip is not None:
source_ip_match = OpServerUtils.Match(
name=self._SOURCE_IP,
value=self._args.source_ip,
op=OpServerUtils.MatchOp.EQUAL)
where.append(source_ip_match.__dict__)
if self._args.destination_ip is not None:
dest_ip_match = OpServerUtils.Match(
name=self._DESTINATION_IP,
value=self._args.destination_ip,
op=OpServerUtils.MatchOp.EQUAL)
where.append(dest_ip_match.__dict__)
if self._args.protocol is not None:
protocol_match = OpServerUtils.Match(
name=self._PROTOCOL,
value=self._args.protocol,
op=OpServerUtils.MatchOp.EQUAL)
where.append(protocol_match.__dict__)
if self._args.source_port is not None:
source_port_match = OpServerUtils.Match(
name=self._SOURCE_PORT,
value=self._args.source_port,
op=OpServerUtils.MatchOp.EQUAL)
where.append(source_port_match.__dict__)
if self._args.destination_port is not None:
dest_port_match = OpServerUtils.Match(
name=self._DESTINATION_PORT,
value=self._args.destination_port,
op=OpServerUtils.MatchOp.EQUAL)
where.append(dest_port_match.__dict__)
if self._args.action is not None:
action_match = OpServerUtils.Match(name=self._ACTION,
value=self._args.action,
op=OpServerUtils.MatchOp.EQUAL)
filter.append(action_match.__dict__)
# Flow Record Table Query
table = VizConstants.FLOW_TABLE
if len(where) == 0:
where = None
else:
where = [where]
select_list = [
VizConstants.FLOW_TABLE_UUID,
self._VROUTER,
self._SETUP_TIME,
self._TEARDOWN_TIME,
self._SOURCE_VN,
self._DESTINATION_VN,
self._SOURCE_IP,
self._DESTINATION_IP,
self._PROTOCOL,
self._SOURCE_PORT,
self._DESTINATION_PORT,
self._ACTION,
self._DIRECTION,
VizConstants.FLOW_TABLE_AGG_BYTES,
VizConstants.FLOW_TABLE_AGG_PKTS,
]
if len(filter) == 0:
filter = None
flow_query = OpServerUtils.Query(table,
start_time=start_time,
end_time=end_time,
select_fields=select_list,
where=where,
filter=filter,
dir=self._args.direction)
if self._args.verbose:
print 'Performing query: {0}'.format(
json.dumps(flow_query.__dict__))
print ''
resp = OpServerUtils.post_url_http(flow_url,
json.dumps(flow_query.__dict__))
result = {}
if resp is not None:
resp = json.loads(resp)
qid = resp['href'].rsplit('/', 1)[1]
result = OpServerUtils.get_query_result(self._args.opserver_ip,
self._args.opserver_port,
qid)
return result
