def run(self): criteria = self.job.criteria netshark = DeviceManager.get_device(criteria.netshark_device) self.export_name = str(path_to_class(netshark, criteria.netshark_source_name)) source = netshark.get_capture_job_by_name(self.export_name) timefilter = TimeFilter(criteria.starttime, criteria.endtime) handle = Job._compute_handle(self.table, criteria) # check if pcaps directory exists, if not make the directory if not os.path.exists(PCAP_DIR): os.mkdir(PCAP_DIR) while self.all_pcap_size > settings.PCAP_SIZE_LIMIT: self.delete_oldest_pcap() self.filename = add_pcap_dir('%s.pcap' % handle) filters = ([BpfFilter(filt) for filt in self.table.options.filters] or None) with netshark.create_export( source, timefilter, filters=filters, wait_for_data=self.table.options.wait_for_data, wait_duration=self.table.options.wait_duration) as e: self.download(e) return QueryComplete(pandas.DataFrame([dict(filename=self.filename)]))
def run(self): criteria = self.job.criteria netshark = DeviceManager.get_device(criteria.netshark_device) self.export_name = str( path_to_class(netshark, criteria.netshark_source_name)) source = netshark.get_capture_job_by_name(self.export_name) timefilter = TimeFilter(criteria.starttime, criteria.endtime) handle = Job._compute_handle(self.table, criteria) # check if pcaps directory exists, if not make the directory if not os.path.exists(PCAP_DIR): os.mkdir(PCAP_DIR) while self.all_pcap_size > settings.PCAP_SIZE_LIMIT: self.delete_oldest_pcap() self.filename = add_pcap_dir('%s.pcap' % handle) filters = ([BpfFilter(filt) for filt in self.table.options.filters] or None) with netshark.create_export( source, timefilter, filters=filters, wait_for_data=self.table.options.wait_for_data, wait_duration=self.table.options.wait_duration) as e: self.download(e) return QueryComplete(pandas.DataFrame([dict(filename=self.filename)]))
def __init__(self, shark, handle, config=None, source=None): super(View4, self).__init__() self.shark = shark self.handle = handle self.source = source self._outputs = {} self.timestamp_format = APITimestampFormat.NANOSECOND if config is None: self.config = shark.api.view.get_config(handle, timestamp_format=self.timestamp_format) else: self.config = config if source is None: path = self.config['input_source']['path'] self.source = path_to_class(shark, path)
def __init__(self, shark, handle, config=None, source=None): super(View4, self).__init__() self.shark = shark self.handle = handle self.source = source self._outputs = {} self.timestamp_format = APITimestampFormat.NANOSECOND if config is None: self.config = shark.api.view.get_config( handle, timestamp_format=self.timestamp_format) else: self.config = config if source is None: path = self.config['input_source']['path'] self.source = path_to_class(shark, path)
def run(self): """ Main execution method """ criteria = self.job.criteria self.timeseries = False # if key column called 'time' is created self.column_names = [] # Resolution comes in as a time_delta resolution = timedelta_total_seconds(criteria.resolution) default_delta = 1000000000 # one second self.delta = int(default_delta * resolution) # sample size interval if criteria.netshark_device == '': logger.debug('%s: No netshark device selected' % self.table) self.job.mark_error("No NetShark Device Selected") return False #self.fake_run() #return True shark = DeviceManager.get_device(criteria.netshark_device) logger.debug("Creating columns for NetShark table %d" % self.table.id) # Create Key/Value Columns columns = [] for tc in self.table.get_columns(synthetic=False): tc_options = tc.options if ( tc.iskey and tc.name == 'time' and tc_options.extractor == 'sample_time'): # don't create column, use the sample time for timeseries self.timeseries = True self.column_names.append('time') continue elif tc.iskey: c = Key(tc_options.extractor, description=tc.label, default_value=tc_options.default_value) else: if tc_options.operation: try: operation = getattr(Operation, tc_options.operation) except AttributeError: operation = Operation.sum print ('ERROR: Unknown operation attribute ' '%s for column %s.' % (tc_options.operation, tc.name)) else: operation = Operation.none c = Value(tc_options.extractor, operation, description=tc.label, default_value=tc_options.default_value) self.column_names.append(tc.name) columns.append(c) # Identify Sort Column sortidx = None if self.table.sortcols is not None: sortcol = Column.objects.get(table=self.table, name=self.table.sortcols[0]) sort_name = sortcol.options.extractor for i, c in enumerate(columns): if c.field == sort_name: sortidx = i break # Initialize filters criteria = self.job.criteria filters = [] filterexpr = self.job.combine_filterexprs( exprs=criteria.netshark_filterexpr, joinstr="&" ) if filterexpr: filters.append(NetSharkFilter(filterexpr)) tf = TimeFilter(start=criteria.starttime, end=criteria.endtime) filters.append(tf) logger.info("Setting netshark table %d timeframe to %s" % (self.table.id, str(tf))) # Get source type from options try: with lock: source = path_to_class(shark, self.job.criteria.netshark_source_name) except RvbdHTTPException, e: source = None raise e
def run(self): """ Main execution method """ criteria = self.job.criteria self.timeseries = False # if key column called 'time' is created self.column_names = [] # Resolution comes in as a time_delta resolution = timedelta_total_seconds(criteria.resolution) default_delta = 1000000000 # one second self.delta = int(default_delta * resolution) # sample size interval if criteria.netshark_device == '': logger.debug('%s: No netshark device selected' % self.table) self.job.mark_error("No NetShark Device Selected") return False shark = DeviceManager.get_device(criteria.netshark_device) logger.debug("Creating columns for NetShark table %d" % self.table.id) # Create Key/Value Columns columns = [] for tc in self.table.get_columns(synthetic=False): tc_options = tc.options if (tc.iskey and tc.name == 'time' and tc_options.extractor == 'sample_time'): # don't create column, use the sample time for timeseries self.timeseries = True self.column_names.append('time') continue elif tc.iskey: c = Key(tc_options.extractor, description=tc.label, default_value=tc_options.default_value) else: if tc_options.operation: try: operation = getattr(Operation, tc_options.operation) except AttributeError: operation = Operation.sum print ('ERROR: Unknown operation attribute ' '%s for column %s.' % (tc_options.operation, tc.name)) else: operation = Operation.none c = Value(tc_options.extractor, operation, description=tc.label, default_value=tc_options.default_value) self.column_names.append(tc.name) columns.append(c) # Identify Sort Column sortidx = None if self.table.sortcols is not None: sortcol = Column.objects.get(table=self.table, name=self.table.sortcols[0]) sort_name = sortcol.options.extractor for i, c in enumerate(columns): if c.field == sort_name: sortidx = i break # Initialize filters criteria = self.job.criteria filters = [] if hasattr(criteria, 'netshark_filterexpr'): logger.debug('calculating netshark filter expression ...') filterexpr = self.job.combine_filterexprs( exprs=criteria.netshark_filterexpr, joinstr="&" ) if filterexpr: logger.debug('applying netshark filter expression: %s' % filterexpr) filters.append(NetSharkFilter(filterexpr)) if hasattr(criteria, 'netshark_bpf_filterexpr'): # TODO evaluate how to combine multiple BPF filters # this will just apply one at a time filterexpr = criteria.netshark_bpf_filterexpr logger.debug('applying netshark BPF filter expression: %s' % filterexpr) filters.append(BpfFilter(filterexpr)) resolution = criteria.resolution if resolution.seconds == 1: sampling_time_msec = 1000 elif resolution.microseconds == 1000: sampling_time_msec = 1 if criteria.duration > parse_timedelta('1s'): msg = ("Cannot run a millisecond report with a duration " "longer than 1 second") raise ValueError(msg) else: sampling_time_msec = 1000 # Get source type from options logger.debug("NetShark Source: %s" % self.job.criteria.netshark_source_name) source = path_to_class( shark, self.job.criteria.netshark_source_name) live = source.is_live() persistent = criteria.get('netshark_persistent', False) if live and not persistent: raise ValueError("Live views must be run with persistent set") view = None if persistent: # First, see a view by this title already exists # Title is the table name plus a criteria hash including # all criteria *except* the timeframe h = hashlib.md5() h.update('.'.join([c.name for c in self.table.get_columns()])) for k, v in criteria.iteritems(): if criteria.is_timeframe_key(k): continue h.update('%s:%s' % (k, v)) title = '/'.join(['steelscript-appfwk', str(self.table.id), self.table.namespace, self.table.name, h.hexdigest()]) view = NetSharkViews.find_by_name(shark, title) logger.debug("Persistent view title: %s" % title) else: # Only assign a title for persistent views title = None if not view: # Not persistent, or not yet created... if not live: # Cannot attach time filter to a live view, # it will be added later at get_data() time tf = TimeFilter(start=criteria.starttime, end=criteria.endtime) filters.append(tf) logger.info("Setting netshark table %d timeframe to %s" % (self.table.id, str(tf))) # Create it with lock: logger.debug("%s: Creating view for table %s" % (str(self), str(self.table))) view = shark.create_view( source, columns, filters=filters, sync=False, name=title, sampling_time_msec=sampling_time_msec) if not live: done = False logger.debug("Waiting for netshark table %d to complete" % self.table.id) while not done: time.sleep(0.5) with lock: s = view.get_progress() self.job.mark_progress(s) self.job.save() done = view.is_ready() logger.debug("Retrieving data for timeframe: %s - %s" % (datetime_to_nanoseconds(criteria.starttime), datetime_to_nanoseconds(criteria.endtime))) # Retrieve the data with lock: getdata_kwargs = {} if sortidx: getdata_kwargs['sortby'] = sortidx if self.table.options.aggregated: getdata_kwargs['aggregated'] = self.table.options.aggregated else: getdata_kwargs['delta'] = self.delta if live: # For live views, attach the time frame to the get_data() getdata_kwargs['start'] = ( datetime_to_nanoseconds(criteria.starttime)) getdata_kwargs['end'] = ( datetime_to_nanoseconds(criteria.endtime)) self.data = view.get_data(**getdata_kwargs) if not persistent: view.close() if self.table.rows > 0: self.data = self.data[:self.table.rows] self.parse_data() logger.info("NetShark Report %s returned %s rows" % (self.job, len(self.data))) return QueryComplete(self.data)
def run(self): """ Main execution method """ criteria = self.job.criteria self.timeseries = False # if key column called 'time' is created self.column_names = [] # Resolution comes in as a time_delta resolution = timedelta_total_seconds(criteria.resolution) default_delta = 1000000000 # one second self.delta = int(default_delta * resolution) # sample size interval if criteria.netshark_device == '': logger.debug('%s: No netshark device selected' % self.table) self.job.mark_error("No NetShark Device Selected") return False shark = DeviceManager.get_device(criteria.netshark_device) logger.debug("Creating columns for NetShark table %d" % self.table.id) # Create Key/Value Columns columns = [] for tc in self.table.get_columns(synthetic=False): tc_options = tc.options if (tc.iskey and tc.name == 'time' and tc_options.extractor == 'sample_time'): # don't create column, use the sample time for timeseries self.timeseries = True self.column_names.append('time') continue elif tc.iskey: c = Key(tc_options.extractor, description=tc.label, default_value=tc_options.default_value) else: if tc_options.operation: try: operation = getattr(Operation, tc_options.operation) except AttributeError: operation = Operation.sum print('ERROR: Unknown operation attribute ' '%s for column %s.' % (tc_options.operation, tc.name)) else: operation = Operation.none c = Value(tc_options.extractor, operation, description=tc.label, default_value=tc_options.default_value) self.column_names.append(tc.name) columns.append(c) # Identify Sort Column sortidx = None if self.table.sortcols is not None: sortcol = Column.objects.get(table=self.table, name=self.table.sortcols[0]) sort_name = sortcol.options.extractor for i, c in enumerate(columns): if c.field == sort_name: sortidx = i break # Initialize filters criteria = self.job.criteria filters = [] if hasattr(criteria, 'netshark_filterexpr'): logger.debug('calculating netshark filter expression ...') filterexpr = self.job.combine_filterexprs( exprs=criteria.netshark_filterexpr, joinstr="&") if filterexpr: logger.debug('applying netshark filter expression: %s' % filterexpr) filters.append(NetSharkFilter(filterexpr)) if hasattr(criteria, 'netshark_bpf_filterexpr'): # TODO evaluate how to combine multiple BPF filters # this will just apply one at a time filterexpr = criteria.netshark_bpf_filterexpr logger.debug('applying netshark BPF filter expression: %s' % filterexpr) filters.append(BpfFilter(filterexpr)) resolution = criteria.resolution if resolution.seconds == 1: sampling_time_msec = 1000 elif resolution.microseconds == 1000: sampling_time_msec = 1 if criteria.duration > parse_timedelta('1s'): msg = ("Cannot run a millisecond report with a duration " "longer than 1 second") raise ValueError(msg) else: sampling_time_msec = 1000 # Get source type from options logger.debug("NetShark Source: %s" % self.job.criteria.netshark_source_name) source = path_to_class(shark, self.job.criteria.netshark_source_name) live = source.is_live() persistent = criteria.get('netshark_persistent', False) if live and not persistent: raise ValueError("Live views must be run with persistent set") view = None if persistent: # First, see a view by this title already exists # Title is the table name plus a criteria hash including # all criteria *except* the timeframe h = hashlib.md5() h.update('.'.join([c.name for c in self.table.get_columns()])) for k, v in criteria.iteritems(): if criteria.is_timeframe_key(k): continue h.update('%s:%s' % (k, v)) title = '/'.join([ 'steelscript-appfwk', str(self.table.id), self.table.namespace, self.table.name, h.hexdigest() ]) view = NetSharkViews.find_by_name(shark, title) logger.debug("Persistent view title: %s" % title) else: # Only assign a title for persistent views title = None timefilter = TimeFilter(start=criteria.starttime, end=criteria.endtime) if not view: # Not persistent, or not yet created... if not live: # Cannot attach time filter to a live view, # it will be added later at get_data() time if criteria.starttime and criteria.endtime: filters.append(timefilter) logger.info("Setting netshark table %d timeframe to %s" % (self.table.id, str(timefilter))) else: # if times are set to zero, don't add to filter # this will process entire timeframe of source instead logger.info("Not setting netshark table %d timeframe" % self.table.id) # Create it with lock: logger.debug("%s: Creating view for table %s" % (str(self), str(self.table))) view = shark.create_view(source, columns, filters=filters, sync=False, name=title, sampling_time_msec=sampling_time_msec) if not live: done = False logger.debug("Waiting for netshark table %d to complete" % self.table.id) while not done: time.sleep(0.5) with lock: s = view.get_progress() self.job.mark_progress(s) self.job.save() done = view.is_ready() logger.debug("Retrieving data for timeframe: %s" % timefilter) # Retrieve the data with lock: getdata_kwargs = {} if sortidx: getdata_kwargs['sortby'] = sortidx if self.table.options.aggregated: getdata_kwargs['aggregated'] = self.table.options.aggregated else: getdata_kwargs['delta'] = self.delta if live: # For live views, attach the time frame to the get_data() getdata_kwargs['start'] = (datetime_to_nanoseconds( criteria.starttime)) getdata_kwargs['end'] = (datetime_to_nanoseconds( criteria.endtime)) self.data = view.get_data(**getdata_kwargs) if not persistent: view.close() if self.table.rows > 0: self.data = self.data[:self.table.rows] self.parse_data() logger.info("NetShark Report %s returned %s rows" % (self.job, len(self.data))) return QueryComplete(self.data)