def run(self):
criteria = self.job.criteria
netshark = DeviceManager.get_device(criteria.netshark_device)
self.export_name = str(path_to_class(netshark,
criteria.netshark_source_name))
source = netshark.get_capture_job_by_name(self.export_name)
timefilter = TimeFilter(criteria.starttime, criteria.endtime)
handle = Job._compute_handle(self.table, criteria)
# check if pcaps directory exists, if not make the directory
if not os.path.exists(PCAP_DIR):
os.mkdir(PCAP_DIR)
while self.all_pcap_size > settings.PCAP_SIZE_LIMIT:
self.delete_oldest_pcap()
self.filename = add_pcap_dir('%s.pcap' % handle)
filters = ([BpfFilter(filt) for filt in self.table.options.filters]
or None)
with netshark.create_export(
source, timefilter, filters=filters,
wait_for_data=self.table.options.wait_for_data,
wait_duration=self.table.options.wait_duration) as e:
self.download(e)
return QueryComplete(pandas.DataFrame([dict(filename=self.filename)]))
def run(self):
criteria = self.job.criteria
netshark = DeviceManager.get_device(criteria.netshark_device)
self.export_name = str(
path_to_class(netshark, criteria.netshark_source_name))
source = netshark.get_capture_job_by_name(self.export_name)
timefilter = TimeFilter(criteria.starttime, criteria.endtime)
handle = Job._compute_handle(self.table, criteria)
# check if pcaps directory exists, if not make the directory
if not os.path.exists(PCAP_DIR):
os.mkdir(PCAP_DIR)
while self.all_pcap_size > settings.PCAP_SIZE_LIMIT:
self.delete_oldest_pcap()
self.filename = add_pcap_dir('%s.pcap' % handle)
filters = ([BpfFilter(filt) for filt in self.table.options.filters]
or None)
with netshark.create_export(
source,
timefilter,
filters=filters,
wait_for_data=self.table.options.wait_for_data,
wait_duration=self.table.options.wait_duration) as e:
self.download(e)
return QueryComplete(pandas.DataFrame([dict(filename=self.filename)]))
def __init__(self, shark, handle, config=None, source=None):
super(View4, self).__init__()
self.shark = shark
self.handle = handle
self.source = source
self._outputs = {}
self.timestamp_format = APITimestampFormat.NANOSECOND
if config is None:
self.config = shark.api.view.get_config(handle, timestamp_format=self.timestamp_format)
else:
self.config = config
if source is None:
path = self.config['input_source']['path']
self.source = path_to_class(shark, path)
def __init__(self, shark, handle, config=None, source=None):
super(View4, self).__init__()
self.shark = shark
self.handle = handle
self.source = source
self._outputs = {}
self.timestamp_format = APITimestampFormat.NANOSECOND
if config is None:
self.config = shark.api.view.get_config(
handle, timestamp_format=self.timestamp_format)
else:
self.config = config
if source is None:
path = self.config['input_source']['path']
self.source = path_to_class(shark, path)
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
self.timeseries = False # if key column called 'time' is created
self.column_names = []
# Resolution comes in as a time_delta
resolution = timedelta_total_seconds(criteria.resolution)
default_delta = 1000000000 # one second
self.delta = int(default_delta * resolution) # sample size interval
if criteria.netshark_device == '':
logger.debug('%s: No netshark device selected' % self.table)
self.job.mark_error("No NetShark Device Selected")
return False
#self.fake_run()
#return True
shark = DeviceManager.get_device(criteria.netshark_device)
logger.debug("Creating columns for NetShark table %d" % self.table.id)
# Create Key/Value Columns
columns = []
for tc in self.table.get_columns(synthetic=False):
tc_options = tc.options
if ( tc.iskey and tc.name == 'time' and
tc_options.extractor == 'sample_time'):
# don't create column, use the sample time for timeseries
self.timeseries = True
self.column_names.append('time')
continue
elif tc.iskey:
c = Key(tc_options.extractor,
description=tc.label,
default_value=tc_options.default_value)
else:
if tc_options.operation:
try:
operation = getattr(Operation, tc_options.operation)
except AttributeError:
operation = Operation.sum
print ('ERROR: Unknown operation attribute '
'%s for column %s.' %
(tc_options.operation, tc.name))
else:
operation = Operation.none
c = Value(tc_options.extractor,
operation,
description=tc.label,
default_value=tc_options.default_value)
self.column_names.append(tc.name)
columns.append(c)
# Identify Sort Column
sortidx = None
if self.table.sortcols is not None:
sortcol = Column.objects.get(table=self.table,
name=self.table.sortcols[0])
sort_name = sortcol.options.extractor
for i, c in enumerate(columns):
if c.field == sort_name:
sortidx = i
break
# Initialize filters
criteria = self.job.criteria
filters = []
filterexpr = self.job.combine_filterexprs(
exprs=criteria.netshark_filterexpr,
joinstr="&"
)
if filterexpr:
filters.append(NetSharkFilter(filterexpr))
tf = TimeFilter(start=criteria.starttime, end=criteria.endtime)
filters.append(tf)
logger.info("Setting netshark table %d timeframe to %s" % (self.table.id,
str(tf)))
# Get source type from options
try:
with lock:
source = path_to_class(shark,
self.job.criteria.netshark_source_name)
except RvbdHTTPException, e:
source = None
raise e
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
self.timeseries = False # if key column called 'time' is created
self.column_names = []
# Resolution comes in as a time_delta
resolution = timedelta_total_seconds(criteria.resolution)
default_delta = 1000000000 # one second
self.delta = int(default_delta * resolution) # sample size interval
if criteria.netshark_device == '':
logger.debug('%s: No netshark device selected' % self.table)
self.job.mark_error("No NetShark Device Selected")
return False
shark = DeviceManager.get_device(criteria.netshark_device)
logger.debug("Creating columns for NetShark table %d" % self.table.id)
# Create Key/Value Columns
columns = []
for tc in self.table.get_columns(synthetic=False):
tc_options = tc.options
if (tc.iskey and tc.name == 'time' and
tc_options.extractor == 'sample_time'):
# don't create column, use the sample time for timeseries
self.timeseries = True
self.column_names.append('time')
continue
elif tc.iskey:
c = Key(tc_options.extractor,
description=tc.label,
default_value=tc_options.default_value)
else:
if tc_options.operation:
try:
operation = getattr(Operation, tc_options.operation)
except AttributeError:
operation = Operation.sum
print ('ERROR: Unknown operation attribute '
'%s for column %s.' %
(tc_options.operation, tc.name))
else:
operation = Operation.none
c = Value(tc_options.extractor,
operation,
description=tc.label,
default_value=tc_options.default_value)
self.column_names.append(tc.name)
columns.append(c)
# Identify Sort Column
sortidx = None
if self.table.sortcols is not None:
sortcol = Column.objects.get(table=self.table,
name=self.table.sortcols[0])
sort_name = sortcol.options.extractor
for i, c in enumerate(columns):
if c.field == sort_name:
sortidx = i
break
# Initialize filters
criteria = self.job.criteria
filters = []
if hasattr(criteria, 'netshark_filterexpr'):
logger.debug('calculating netshark filter expression ...')
filterexpr = self.job.combine_filterexprs(
exprs=criteria.netshark_filterexpr,
joinstr="&"
)
if filterexpr:
logger.debug('applying netshark filter expression: %s'
% filterexpr)
filters.append(NetSharkFilter(filterexpr))
if hasattr(criteria, 'netshark_bpf_filterexpr'):
# TODO evaluate how to combine multiple BPF filters
# this will just apply one at a time
filterexpr = criteria.netshark_bpf_filterexpr
logger.debug('applying netshark BPF filter expression: %s'
% filterexpr)
filters.append(BpfFilter(filterexpr))
resolution = criteria.resolution
if resolution.seconds == 1:
sampling_time_msec = 1000
elif resolution.microseconds == 1000:
sampling_time_msec = 1
if criteria.duration > parse_timedelta('1s'):
msg = ("Cannot run a millisecond report with a duration "
"longer than 1 second")
raise ValueError(msg)
else:
sampling_time_msec = 1000
# Get source type from options
logger.debug("NetShark Source: %s" %
self.job.criteria.netshark_source_name)
source = path_to_class(
shark, self.job.criteria.netshark_source_name)
live = source.is_live()
persistent = criteria.get('netshark_persistent', False)
if live and not persistent:
raise ValueError("Live views must be run with persistent set")
view = None
if persistent:
# First, see a view by this title already exists
# Title is the table name plus a criteria hash including
# all criteria *except* the timeframe
h = hashlib.md5()
h.update('.'.join([c.name for c in
self.table.get_columns()]))
for k, v in criteria.iteritems():
if criteria.is_timeframe_key(k):
continue
h.update('%s:%s' % (k, v))
title = '/'.join(['steelscript-appfwk', str(self.table.id),
self.table.namespace, self.table.name,
h.hexdigest()])
view = NetSharkViews.find_by_name(shark, title)
logger.debug("Persistent view title: %s" % title)
else:
# Only assign a title for persistent views
title = None
if not view:
# Not persistent, or not yet created...
if not live:
# Cannot attach time filter to a live view,
# it will be added later at get_data() time
tf = TimeFilter(start=criteria.starttime,
end=criteria.endtime)
filters.append(tf)
logger.info("Setting netshark table %d timeframe to %s" %
(self.table.id, str(tf)))
# Create it
with lock:
logger.debug("%s: Creating view for table %s" %
(str(self), str(self.table)))
view = shark.create_view(
source, columns, filters=filters, sync=False,
name=title, sampling_time_msec=sampling_time_msec)
if not live:
done = False
logger.debug("Waiting for netshark table %d to complete" %
self.table.id)
while not done:
time.sleep(0.5)
with lock:
s = view.get_progress()
self.job.mark_progress(s)
self.job.save()
done = view.is_ready()
logger.debug("Retrieving data for timeframe: %s - %s" %
(datetime_to_nanoseconds(criteria.starttime),
datetime_to_nanoseconds(criteria.endtime)))
# Retrieve the data
with lock:
getdata_kwargs = {}
if sortidx:
getdata_kwargs['sortby'] = sortidx
if self.table.options.aggregated:
getdata_kwargs['aggregated'] = self.table.options.aggregated
else:
getdata_kwargs['delta'] = self.delta
if live:
# For live views, attach the time frame to the get_data()
getdata_kwargs['start'] = (
datetime_to_nanoseconds(criteria.starttime))
getdata_kwargs['end'] = (
datetime_to_nanoseconds(criteria.endtime))
self.data = view.get_data(**getdata_kwargs)
if not persistent:
view.close()
if self.table.rows > 0:
self.data = self.data[:self.table.rows]
self.parse_data()
logger.info("NetShark Report %s returned %s rows" %
(self.job, len(self.data)))
return QueryComplete(self.data)
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
self.timeseries = False # if key column called 'time' is created
self.column_names = []
# Resolution comes in as a time_delta
resolution = timedelta_total_seconds(criteria.resolution)
default_delta = 1000000000 # one second
self.delta = int(default_delta * resolution) # sample size interval
if criteria.netshark_device == '':
logger.debug('%s: No netshark device selected' % self.table)
self.job.mark_error("No NetShark Device Selected")
return False
shark = DeviceManager.get_device(criteria.netshark_device)
logger.debug("Creating columns for NetShark table %d" % self.table.id)
# Create Key/Value Columns
columns = []
for tc in self.table.get_columns(synthetic=False):
tc_options = tc.options
if (tc.iskey and tc.name == 'time'
and tc_options.extractor == 'sample_time'):
# don't create column, use the sample time for timeseries
self.timeseries = True
self.column_names.append('time')
continue
elif tc.iskey:
c = Key(tc_options.extractor,
description=tc.label,
default_value=tc_options.default_value)
else:
if tc_options.operation:
try:
operation = getattr(Operation, tc_options.operation)
except AttributeError:
operation = Operation.sum
print('ERROR: Unknown operation attribute '
'%s for column %s.' %
(tc_options.operation, tc.name))
else:
operation = Operation.none
c = Value(tc_options.extractor,
operation,
description=tc.label,
default_value=tc_options.default_value)
self.column_names.append(tc.name)
columns.append(c)
# Identify Sort Column
sortidx = None
if self.table.sortcols is not None:
sortcol = Column.objects.get(table=self.table,
name=self.table.sortcols[0])
sort_name = sortcol.options.extractor
for i, c in enumerate(columns):
if c.field == sort_name:
sortidx = i
break
# Initialize filters
criteria = self.job.criteria
filters = []
if hasattr(criteria, 'netshark_filterexpr'):
logger.debug('calculating netshark filter expression ...')
filterexpr = self.job.combine_filterexprs(
exprs=criteria.netshark_filterexpr, joinstr="&")
if filterexpr:
logger.debug('applying netshark filter expression: %s' %
filterexpr)
filters.append(NetSharkFilter(filterexpr))
if hasattr(criteria, 'netshark_bpf_filterexpr'):
# TODO evaluate how to combine multiple BPF filters
# this will just apply one at a time
filterexpr = criteria.netshark_bpf_filterexpr
logger.debug('applying netshark BPF filter expression: %s' %
filterexpr)
filters.append(BpfFilter(filterexpr))
resolution = criteria.resolution
if resolution.seconds == 1:
sampling_time_msec = 1000
elif resolution.microseconds == 1000:
sampling_time_msec = 1
if criteria.duration > parse_timedelta('1s'):
msg = ("Cannot run a millisecond report with a duration "
"longer than 1 second")
raise ValueError(msg)
else:
sampling_time_msec = 1000
# Get source type from options
logger.debug("NetShark Source: %s" %
self.job.criteria.netshark_source_name)
source = path_to_class(shark, self.job.criteria.netshark_source_name)
live = source.is_live()
persistent = criteria.get('netshark_persistent', False)
if live and not persistent:
raise ValueError("Live views must be run with persistent set")
view = None
if persistent:
# First, see a view by this title already exists
# Title is the table name plus a criteria hash including
# all criteria *except* the timeframe
h = hashlib.md5()
h.update('.'.join([c.name for c in self.table.get_columns()]))
for k, v in criteria.iteritems():
if criteria.is_timeframe_key(k):
continue
h.update('%s:%s' % (k, v))
title = '/'.join([
'steelscript-appfwk',
str(self.table.id), self.table.namespace, self.table.name,
h.hexdigest()
])
view = NetSharkViews.find_by_name(shark, title)
logger.debug("Persistent view title: %s" % title)
else:
# Only assign a title for persistent views
title = None
timefilter = TimeFilter(start=criteria.starttime, end=criteria.endtime)
if not view:
# Not persistent, or not yet created...
if not live:
# Cannot attach time filter to a live view,
# it will be added later at get_data() time
if criteria.starttime and criteria.endtime:
filters.append(timefilter)
logger.info("Setting netshark table %d timeframe to %s" %
(self.table.id, str(timefilter)))
else:
# if times are set to zero, don't add to filter
# this will process entire timeframe of source instead
logger.info("Not setting netshark table %d timeframe" %
self.table.id)
# Create it
with lock:
logger.debug("%s: Creating view for table %s" %
(str(self), str(self.table)))
view = shark.create_view(source,
columns,
filters=filters,
sync=False,
name=title,
sampling_time_msec=sampling_time_msec)
if not live:
done = False
logger.debug("Waiting for netshark table %d to complete" %
self.table.id)
while not done:
time.sleep(0.5)
with lock:
s = view.get_progress()
self.job.mark_progress(s)
self.job.save()
done = view.is_ready()
logger.debug("Retrieving data for timeframe: %s" % timefilter)
# Retrieve the data
with lock:
getdata_kwargs = {}
if sortidx:
getdata_kwargs['sortby'] = sortidx
if self.table.options.aggregated:
getdata_kwargs['aggregated'] = self.table.options.aggregated
else:
getdata_kwargs['delta'] = self.delta
if live:
# For live views, attach the time frame to the get_data()
getdata_kwargs['start'] = (datetime_to_nanoseconds(
criteria.starttime))
getdata_kwargs['end'] = (datetime_to_nanoseconds(
criteria.endtime))
self.data = view.get_data(**getdata_kwargs)
if not persistent:
view.close()
if self.table.rows > 0:
self.data = self.data[:self.table.rows]
self.parse_data()
logger.info("NetShark Report %s returned %s rows" %
(self.job, len(self.data)))
return QueryComplete(self.data)