def collect(self, jobs=None): logger.debug("%s: bizhours.collect: %s" % (self, jobs)) basetable = Table.from_ref( self.table.options.related_tables['basetable'] ) # collect all key names keynames = [] istime = False for key in basetable.get_columns(iskey=True): keynames.append(key.name) if key.istime(): istime = True # Now collect the data total_secs = 0 dfs = [] idx = 0 for jid, job in jobs.iteritems(): if job.status == Job.ERROR: raise AnalysisException("%s for %s-%s failed: %s" % (job, job.criteria.starttime, job.criteria.endtime, job.message)) subdf = job.data() logger.debug("%s: returned %d rows" % (job, len(subdf) if subdf is not None else 0)) if subdf is None: continue logger.debug("%s: actual_criteria %s" % (job, job.actual_criteria)) t0 = job.actual_criteria.starttime t1 = job.actual_criteria.endtime if not istime: subdf['__secs__'] = timedelta_total_seconds(t1 - t0) total_secs += timedelta_total_seconds(t1 - t0) idx += 1 dfs.append(subdf) if len(dfs) == 0: return QueryComplete(None) df = pandas.concat(dfs, ignore_index=True) if not istime: if 'aggregate' in self.table.options: ops = self.table.options['aggregate'] for col in basetable.get_columns(iskey=False): if col.name not in ops: ops[col.name] = 'sum' else: ops = 'sum' df = avg_groupby_aggregate(df, keynames, ops, '__secs__', total_secs) return QueryComplete(df)
def render(self, name, value, attrs): initial_time = attrs.get('initial_time', None) if initial_time: m = re.match("now *- *(.+)", initial_time) if m: secs = timedelta_total_seconds(parse_timedelta(m.group(1))) initial_time = ( "d = new Date(); d.setSeconds(d.getSeconds()-%d);" % secs) else: initial_time = "d = new Date('{}');".format(initial_time) else: initial_time = "d = new Date();" round_initial = attrs.get('round_initial', None) if round_initial: js = (' p = %d*1000; ' 'd = new Date(Math.floor(d.getTime() / p) * p);' % round_initial) initial_time += js # only pin manually entered times if we are rounding above a minute if round_initial >= 60: step = int(round_initial) / 60 force_round_time = 'true' else: step = 15 force_round_time = 'false' else: step = 15 force_round_time = 'false' msg = ''' <span class="input-group-addon"> <span id="timenow_{name}" class="glyphicon glyphicon-time" title="Set time/date to now"> </span> </span> {0} <script type="text/javascript"> $("#id_{name}").timepicker({{ step: {step}, scrollDefaultNow:true, forceRoundTime: {force_round_time}, timeFormat:"g:i:s a" }}); $("#timenow_{name}").click(function() {{ $("#id_{name}").timepicker("setTime", new Date()); }}); {initial_time} // align to timezone var offset = ($('#tz').html()/100)*60*60*1000; d = rvbd.timeutil.convertDateToUTC(d); d.setTime(d.getTime() + offset); $("#id_{name}").timepicker("setTime", d); </script> ''' return msg.format(super(TimeWidget, self).render(name, value, attrs), name=name, initial_time=initial_time, step=step, force_round_time=force_round_time)
def as_text(self): """Return certain field values as a dict for simple json parsing.""" result = {} for k, v in self.cleaned_data.iteritems(): if isinstance(v, datetime.datetime): result[k] = v.isoformat() elif isinstance(v, datetime.timedelta): result[k] = str(timedelta_total_seconds(v)) + " seconds" elif isinstance(v, UploadedFile): # look for uploaded files, save them off to another # temporary file and return the path for use in JSON # consumers of this file will need to clean them up # TODO this will be replaced by the File Storage App newtemp = tempfile.NamedTemporaryFile(delete=False) v.seek(0) shutil.copyfileobj(v, newtemp) v.close() newtemp.close() result[k] = newtemp.name else: result[k] = v return result
def schedule_job(self, name, job_config): job_options, interval = self.parse_config(job_config) if interval['offset'] > datetime.timedelta(0): delta = interval['delta'] offset = interval['offset'] now = datetime.datetime.now(pytz.UTC) # this gives the latest rounded time in the past # so we add interval to it to get a future run time delta_secs = timeutils.timedelta_total_seconds(delta) next_run_time = timeutils.round_time(now, round_to=delta_secs, trim=True) next_run_time += (delta + offset) logger.debug( 'Setting next run time to %s (delta: %s, offset: %s)' % (next_run_time, delta, offset)) job_options['next_run_time'] = next_run_time logger.debug('Scheduling job named %s with kwargs: %s' % (name, job_options)) self.scheduler.add_job(name=name, trigger='interval', **job_options)
def render(self, name, value, attrs): initial_time = attrs.get('initial_time', None) if initial_time: m = re.match("now *- *(.+)", initial_time) if m: secs = timedelta_total_seconds(parse_timedelta(m.group(1))) initial_time = ( "d = new Date(); d.setSeconds(d.getSeconds()-%d);" % secs) else: initial_time = "d = '%s';" % initial_time else: initial_time = "d = new Date();" msg = ''' {0} <span id="timenow_{name}" class="icon-time" title="Set time/date to now"> </span> <script type="text/javascript"> $("#id_{name}").timepicker({{ step: 15, scrollDefaultNow:true, timeFormat:"g:i:s a" }}); $("#timenow_{name}").click(function() {{ $("#id_{name}").timepicker("setTime", new Date()); }}); {initial_time} $("#id_{name}").timepicker("setTime", d); </script> ''' #'$("#id_{name}").timepicker("setTime", new Date());' return msg.format(super(TimeWidget, self).render(name, value, attrs), name=name, initial_time=initial_time)
def as_text(self): """Return certain field values as a dict for simple json parsing.""" result = {} for k, v in self.cleaned_data.iteritems(): if isinstance(v, datetime.datetime): result[k] = v.isoformat() elif isinstance(v, datetime.timedelta): result[k] = str(timedelta_total_seconds(v)) + " seconds" elif isinstance(v, UploadedFile): # look for uploaded files, save them off to another # temporary file and return the path for use in JSON # consumers of this file will need to clean them up # TODO this will be replaced by the File Storage App newtemp = tempfile.NamedTemporaryFile(delete=False) v.seek(0) shutil.copyfileobj(v, newtemp) v.close() newtemp.close() result[k] = newtemp.name else: result[k] = v return result
def post_process_table(self, field_options): resolution = field_options['resolution'] if resolution != 'auto': if isinstance(resolution, int): res = resolution else: res = int(timedelta_total_seconds(parse_timedelta(resolution))) resolution = Report.RESOLUTION_MAP[res] field_options['resolution'] = resolution fields_add_device_selection(self, keyword='netprofiler_device', label='NetProfiler', module='netprofiler', enabled=True) duration = field_options['duration'] if isinstance(duration, int): duration = "%d min" % duration fields_add_time_selection(self, initial_duration=duration, durations=field_options['durations']) fields_add_resolution(self, initial=field_options['resolution'], resolutions=field_options['resolutions'], special_values=['auto']) self.fields_add_filterexpr()
def _round_times(self): """Round the start/end time in criteria to reflect what data will be returned from data source and saved in db (based on investigation of NetProfiler/NetShark query results). Start/End time needs to round down to closest time in integer resolutions. However, if either starttime or endtime in criteria is already in integer resolutions, it should remain the same. :return: start time, end time, resolution """ self.job.criteria.compute_times() resolution = getattr(self.job.criteria, 'resolution', None) if not resolution: raise AttributeError("The data source table '%s' needs to have " "'resolution' field." % self.ds_table.name) resolution_sec = timedelta_total_seconds(self.job.criteria.resolution) starttime = round_time(self.job.criteria.starttime, round_to=resolution_sec, trim=True) endtime = round_time(self.job.criteria.endtime, round_to=resolution_sec, trim=True) return starttime, endtime, resolution
def render(self, name, value, attrs): initial_date = attrs.get('initial_date', None) if initial_date: m = re.match("now *- *(.+)", initial_date) if m: secs = timedelta_total_seconds(parse_timedelta(m.group(1))) initial_date = ( "d = new Date(); d.setSeconds(d.getSeconds()-%d);" \ % secs) else: initial_date = "d = '%s';" % initial_date else: initial_date = "d = new Date();" msg = ''' {0} <span id="datenow_{name}" class="icon-calendar" title="Set date to today"> </span> <script type="text/javascript"> $("#id_{name}").datepicker({{ format: "mm/dd/YY", defaultDate: +2, autoclose: true }}); {initial_date} $("#id_{name}").datepicker("setDate", d); $("#datenow_{name}").click(function() {{ $("#id_{name}").datepicker("setDate", new Date()); }}); </script> ''' return msg.format(super(DateWidget, self).render(name, value, attrs), name=name, initial_date=initial_date)
def post_process_table(self, field_options): resolution = field_options['resolution'] if resolution != 'auto': if isinstance(resolution, int): res = resolution else: res = int(timedelta_total_seconds(parse_timedelta(resolution))) resolution = Report.RESOLUTION_MAP[res] field_options['resolution'] = resolution fields_add_device_selection(self, keyword='netprofiler_device', label='NetProfiler', module='netprofiler', enabled=True) duration = field_options['duration'] if isinstance(duration, int): duration = "%d min" % duration fields_add_time_selection(self, initial_duration=duration, durations=field_options['durations']) fields_add_resolution(self, initial=field_options['resolution'], resolutions=field_options['resolutions'], special_values=['auto']) self.fields_add_filterexpr()
def _round_times(self): """Round the start/end time in criteria to reflect what data will be returned from data source and saved in db (based on investigation of NetProfiler/NetShark query results). Start/End time needs to round down to closest time in integer resolutions. However, if either starttime or endtime in criteria is already in integer resolutions, it should remain the same. :return: start time, end time, resolution """ self.job.criteria.compute_times() resolution = getattr(self.job.criteria, 'resolution', None) if not resolution: raise AttributeError("The data source table '%s' needs to have " "'resolution' field." % self.ds_table.name) resolution_sec = timedelta_total_seconds(self.job.criteria.resolution) starttime = round_time(self.job.criteria.starttime, round_to=resolution_sec, trim=True) endtime = round_time(self.job.criteria.endtime, round_to=resolution_sec, trim=True) return starttime, endtime, resolution
def _prepare_report_args(self): class Args(object): pass args = Args() criteria = self.job.criteria if criteria.netprofiler_device == '': logger.debug('%s: No netprofiler device selected' % self.table) self.job.mark_error("No NetProfiler Device Selected") return False args.profiler = DeviceManager.get_device(criteria.netprofiler_device) args.columns = [ col.name for col in self.table.get_columns(synthetic=False) ] args.sortcol = None if self.table.sortcols is not None: args.sortcol = self.table.sortcols[0] args.timefilter = TimeFilter(start=criteria.starttime, end=criteria.endtime) logger.info("Running NetProfiler table %d report for timeframe %s" % (self.table.id, str(args.timefilter))) if ('datafilter' in criteria) and (criteria.datafilter is not None): args.datafilter = criteria.datafilter.split(',') else: args.datafilter = None args.trafficexpr = TrafficFilter( self.job.combine_filterexprs( exprs=criteria.netprofiler_filterexpr)) # Incoming criteria.resolution is a timedelta logger.debug('NetProfiler report got criteria resolution %s (%s)' % (criteria.resolution, type(criteria.resolution))) if criteria.resolution != 'auto': rsecs = int(timedelta_total_seconds(criteria.resolution)) args.resolution = Report.RESOLUTION_MAP[rsecs] else: args.resolution = 'auto' logger.debug('NetProfiler report using resolution %s (%s)' % (args.resolution, type(args.resolution))) args.limit = (self.table.options.limit if hasattr( self.table.options, 'limit') else None) if getattr(self.table.options, 'interface', False): args.centricity = 'int' else: args.centricity = 'hos' return args
def _prepare_report_args(self): class Args(object): pass args = Args() criteria = self.job.criteria if criteria.netprofiler_device == '': logger.debug('%s: No netprofiler device selected' % self.table) self.job.mark_error("No NetProfiler Device Selected") return False args.profiler = DeviceManager.get_device(criteria.netprofiler_device) args.columns = [col.name for col in self.table.get_columns(synthetic=False)] args.sortcol = None if self.table.sortcols is not None: args.sortcol = self.table.sortcols[0] args.timefilter = TimeFilter(start=criteria.starttime, end=criteria.endtime) logger.info("Running NetProfiler table %d report for timeframe %s" % (self.table.id, str(args.timefilter))) if ('datafilter' in criteria) and (criteria.datafilter is not None): args.datafilter = criteria.datafilter.split(',') else: args.datafilter = None args.trafficexpr = TrafficFilter( self.job.combine_filterexprs(exprs=criteria.netprofiler_filterexpr) ) # Incoming criteria.resolution is a timedelta logger.debug('NetProfiler report got criteria resolution %s (%s)' % (criteria.resolution, type(criteria.resolution))) if criteria.resolution != 'auto': rsecs = int(timedelta_total_seconds(criteria.resolution)) args.resolution = Report.RESOLUTION_MAP[rsecs] else: args.resolution = 'auto' logger.debug('NetProfiler report using resolution %s (%s)' % (args.resolution, type(args.resolution))) args.limit = (self.table.options.limit if hasattr(self.table.options, 'limit') else None) if getattr(self.table.options, 'interface', False): args.centricity = 'int' else: args.centricity = 'hos' return args
def decompress(self, value): if isinstance(value, str) or isinstance(value, unicode): value = timedelta_total_seconds(parse_timedelta(value)) if value: m = [v for v in self.choices if v[0] == value] if len(m) == 1: return m[0] else: return [0, '%d min' % (value / 60)] return [None, None]
def decompress(self, value): if isinstance(value, str) or isinstance(value, unicode): value = timedelta_total_seconds(parse_timedelta(value)) if value: m = [v for v in self.choices if v[0] == value] if len(m) == 1: return m[0] else: return [0, '%d min' % (value / 60)] return [None, None]
def _get_url_fields(flds): for k, v in flds.iteritems(): if k in ['starttime', 'endtime']: yield (k, str(datetime_to_seconds(v))) elif k in ['duration', 'resolution']: try: yield (k, str(int(timedelta_total_seconds(v)))) except AttributeError: # v is of special value, not a string of some duration yield (k, v.replace(' ', '+')) else: # use + as encoded white space yield (k, str(v).replace(' ', '+')) yield ('auto_run', 'true')
def _get_url_fields(flds): for k, v in flds.iteritems(): if k in ['starttime', 'endtime']: yield (k, str(datetime_to_seconds(v))) elif k in ['duration', 'resolution']: try: yield (k, str(int(timedelta_total_seconds(v)))) except AttributeError: # v is of special value, not a string of some duration yield (k, v.replace(' ', '+')) else: # use + as encoded white space yield (k, str(v).replace(' ', '+')) yield ('auto_run', 'true')
def resample(df, timecol, interval, how): """Resample the input dataframe. :param str timecol: the name of the column containing the row time :param timedelta,str interval: the new interval :param how: method for down or resampling (see pandas.Dataframe.resample) """ df[timecol] = pandas.DatetimeIndex(df[timecol]) df.set_index(timecol, inplace=True) if isinstance(interval, timedelta): interval = '%ss' % (timedelta_total_seconds(parse_timedelta(interval))) df = df.resample(interval, how=how).reset_index() return df
def resample(df, timecol, interval, how): """Resample the input dataframe. :param str timecol: the name of the column containing the row time :param timedelta,str interval: the new interval :param how: method for down or resampling (see pandas.Dataframe.resample) """ df[timecol] = pandas.DatetimeIndex(df[timecol]) df.set_index(timecol, inplace=True) if isinstance(interval, timedelta): interval = '%ss' % (timedelta_total_seconds(parse_timedelta(interval))) df = df.resample(interval, how=how).reset_index() return df
def resample(df, timecol, interval, how='sum'): """Resample the input dataframe. :param str timecol: the name of the column containing the row time :param timedelta,str interval: the new interval :param how: method for down or resampling (see pandas.Dataframe.resample) """ df[timecol] = pandas.DatetimeIndex(df[timecol]) df.set_index(timecol, inplace=True) if isinstance(interval, timedelta): interval = '%ss' % (timedelta_total_seconds(parse_timedelta(interval))) # use new pandas reasmple API # http://pandas.pydata.org/pandas-docs/stable/whatsnew.html#resample-api r = df.resample(interval) df = getattr(r, how)() df.reset_index(inplace=True) return df
def resample(df, timecol, interval, how='sum'): """Resample the input dataframe. :param str timecol: the name of the column containing the row time :param timedelta,str interval: the new interval :param how: method for down or resampling (see pandas.Dataframe.resample) """ df[timecol] = pandas.DatetimeIndex(df[timecol]) df.set_index(timecol, inplace=True) if isinstance(interval, timedelta): interval = '%ss' % (timedelta_total_seconds(parse_timedelta(interval))) # use new pandas reasmple API # http://pandas.pydata.org/pandas-docs/stable/whatsnew.html#resample-api r = df.resample(interval) df = getattr(r, how)() df.reset_index(inplace=True) return df
def render(self, name, value, attrs): initial_date = attrs.get('initial_date', None) if initial_date: m = re.match("now *- *(.+)", initial_date) if m: secs = timedelta_total_seconds(parse_timedelta(m.group(1))) initial_date = ( "d = new Date(); d.setSeconds(d.getSeconds()-%d);" % secs ) else: initial_date = "d = '%s';" % initial_date else: initial_date = "d = new Date();" round_initial = attrs.get('round_initial', None) if round_initial: js = (' p = %d*1000; ' 'd = new Date(Math.floor(d.getTime() / p) * p);' % round_initial) initial_date += js msg = ''' <span class="input-group-addon"> <span id="datenow_{name}" class="glyphicon glyphicon-calendar" title="Set date to today"> </span> </span> {0} <script type="text/javascript"> $("#id_{name}").datepicker({{ format: "mm/dd/YY", defaultDate: +2, autoclose: true }}); {initial_date} $("#id_{name}").datepicker("setDate", d); $("#datenow_{name}").click(function() {{ $("#id_{name}").datepicker("setDate", new Date()); }}); </script> ''' return msg.format( super(DateWidget, self).render(name, value, attrs), name=name, initial_date=initial_date )
def schedule_job(self, name, job_config): job_options, interval = self.parse_config(job_config) if interval['offset'] > datetime.timedelta(0): delta = interval['delta'] offset = interval['offset'] now = datetime.datetime.now(pytz.UTC) # this gives the latest rounded time in the past # so we add interval to it to get a future run time total_offset = timeutils.timedelta_total_seconds(delta + offset) next_run_time = timeutils.round_time(now, round_to=total_offset, trim=True) next_run_time += (delta + offset) logger.debug('Setting next run time to %s (delta: %s, offset: %s)' % (next_run_time, delta, offset)) job_options['next_run_time'] = next_run_time logger.debug('Scheduling job named %s with kwargs: %s' % (name, job_options)) self.scheduler.add_job(name=name, trigger='interval', **job_options)
def run(self): """ Main execution method """ criteria = self.job.criteria if criteria.netprofiler_device == '': logger.debug('%s: No netprofiler device selected' % self.table) self.job.mark_error("No NetProfiler Device Selected") return False #self.fake_run() #return True profiler = DeviceManager.get_device(criteria.netprofiler_device) report = steelscript.netprofiler.core.report.SingleQueryReport(profiler) columns = [col.name for col in self.table.get_columns(synthetic=False)] sortcol = None if self.table.sortcols is not None: sortcol = self.table.sortcols[0] tf = TimeFilter(start=criteria.starttime, end=criteria.endtime) logger.info("Running NetProfiler table %d report for timeframe %s" % (self.table.id, str(tf))) if ('datafilter' in criteria) and (criteria.datafilter is not None): datafilter = criteria.datafilter.split(',') else: datafilter = None trafficexpr = TrafficFilter( self.job.combine_filterexprs(exprs=criteria.netprofiler_filterexpr) ) # Incoming criteria.resolution is a timedelta logger.debug('NetProfiler report got criteria resolution %s (%s)' % (criteria.resolution, type(criteria.resolution))) if criteria.resolution != 'auto': rsecs = int(timedelta_total_seconds(criteria.resolution)) resolution = steelscript.netprofiler.core.report.Report.RESOLUTION_MAP[rsecs] else: resolution = 'auto' logger.debug('NetProfiler report using resolution %s (%s)' % (resolution, type(resolution))) with lock: centricity = 'int' if self.table.options.interface else 'hos' report.run(realm=self.table.options.realm, groupby=profiler.groupbys[self.table.options.groupby], centricity=centricity, columns=columns, timefilter=tf, trafficexpr=trafficexpr, data_filter=datafilter, resolution=resolution, sort_col=sortcol, sync=False ) done = False logger.info("Waiting for report to complete") while not done: time.sleep(0.5) with lock: s = report.status() self.job.safe_update(progress=int(s['percent'])) done = (s['status'] == 'completed') # Retrieve the data with lock: query = report.get_query_by_index(0) self.data = query.get_data() tz = criteria.starttime.tzinfo # Update criteria criteria.starttime = (datetime.datetime .utcfromtimestamp(query.actual_t0) .replace(tzinfo=tz)) criteria.endtime = (datetime.datetime .utcfromtimestamp(query.actual_t1) .replace(tzinfo=tz)) self.job.safe_update(actual_criteria=criteria) if self.table.rows > 0: self.data = self.data[:self.table.rows] logger.info("Report %s returned %s rows" % (self.job, len(self.data))) return True
def run(self): """ Main execution method """ criteria = self.job.criteria self.timeseries = False # if key column called 'time' is created self.column_names = [] # Resolution comes in as a time_delta resolution = timedelta_total_seconds(criteria.resolution) default_delta = 1000000000 # one second self.delta = int(default_delta * resolution) # sample size interval if criteria.netshark_device == '': logger.debug('%s: No netshark device selected' % self.table) self.job.mark_error("No NetShark Device Selected") return False shark = DeviceManager.get_device(criteria.netshark_device) logger.debug("Creating columns for NetShark table %d" % self.table.id) # Create Key/Value Columns columns = [] for tc in self.table.get_columns(synthetic=False): tc_options = tc.options if (tc.iskey and tc.name == 'time' and tc_options.extractor == 'sample_time'): # don't create column, use the sample time for timeseries self.timeseries = True self.column_names.append('time') continue elif tc.iskey: c = Key(tc_options.extractor, description=tc.label, default_value=tc_options.default_value) else: if tc_options.operation: try: operation = getattr(Operation, tc_options.operation) except AttributeError: operation = Operation.sum print('ERROR: Unknown operation attribute ' '%s for column %s.' % (tc_options.operation, tc.name)) else: operation = Operation.none c = Value(tc_options.extractor, operation, description=tc.label, default_value=tc_options.default_value) self.column_names.append(tc.name) columns.append(c) # Identify Sort Column sortidx = None if self.table.sortcols is not None: sortcol = Column.objects.get(table=self.table, name=self.table.sortcols[0]) sort_name = sortcol.options.extractor for i, c in enumerate(columns): if c.field == sort_name: sortidx = i break # Initialize filters criteria = self.job.criteria filters = [] if hasattr(criteria, 'netshark_filterexpr'): logger.debug('calculating netshark filter expression ...') filterexpr = self.job.combine_filterexprs( exprs=criteria.netshark_filterexpr, joinstr="&") if filterexpr: logger.debug('applying netshark filter expression: %s' % filterexpr) filters.append(NetSharkFilter(filterexpr)) if hasattr(criteria, 'netshark_bpf_filterexpr'): # TODO evaluate how to combine multiple BPF filters # this will just apply one at a time filterexpr = criteria.netshark_bpf_filterexpr logger.debug('applying netshark BPF filter expression: %s' % filterexpr) filters.append(BpfFilter(filterexpr)) resolution = criteria.resolution if resolution.seconds == 1: sampling_time_msec = 1000 elif resolution.microseconds == 1000: sampling_time_msec = 1 if criteria.duration > parse_timedelta('1s'): msg = ("Cannot run a millisecond report with a duration " "longer than 1 second") raise ValueError(msg) else: sampling_time_msec = 1000 # Get source type from options logger.debug("NetShark Source: %s" % self.job.criteria.netshark_source_name) source = path_to_class(shark, self.job.criteria.netshark_source_name) live = source.is_live() persistent = criteria.get('netshark_persistent', False) if live and not persistent: raise ValueError("Live views must be run with persistent set") view = None if persistent: # First, see a view by this title already exists # Title is the table name plus a criteria hash including # all criteria *except* the timeframe h = hashlib.md5() h.update('.'.join([c.name for c in self.table.get_columns()])) for k, v in criteria.iteritems(): if criteria.is_timeframe_key(k): continue h.update('%s:%s' % (k, v)) title = '/'.join([ 'steelscript-appfwk', str(self.table.id), self.table.namespace, self.table.name, h.hexdigest() ]) view = NetSharkViews.find_by_name(shark, title) logger.debug("Persistent view title: %s" % title) else: # Only assign a title for persistent views title = None timefilter = TimeFilter(start=criteria.starttime, end=criteria.endtime) if not view: # Not persistent, or not yet created... if not live: # Cannot attach time filter to a live view, # it will be added later at get_data() time if criteria.starttime and criteria.endtime: filters.append(timefilter) logger.info("Setting netshark table %d timeframe to %s" % (self.table.id, str(timefilter))) else: # if times are set to zero, don't add to filter # this will process entire timeframe of source instead logger.info("Not setting netshark table %d timeframe" % self.table.id) # Create it with lock: logger.debug("%s: Creating view for table %s" % (str(self), str(self.table))) view = shark.create_view(source, columns, filters=filters, sync=False, name=title, sampling_time_msec=sampling_time_msec) if not live: done = False logger.debug("Waiting for netshark table %d to complete" % self.table.id) while not done: time.sleep(0.5) with lock: s = view.get_progress() self.job.mark_progress(s) self.job.save() done = view.is_ready() logger.debug("Retrieving data for timeframe: %s" % timefilter) # Retrieve the data with lock: getdata_kwargs = {} if sortidx: getdata_kwargs['sortby'] = sortidx if self.table.options.aggregated: getdata_kwargs['aggregated'] = self.table.options.aggregated else: getdata_kwargs['delta'] = self.delta if live: # For live views, attach the time frame to the get_data() getdata_kwargs['start'] = (datetime_to_nanoseconds( criteria.starttime)) getdata_kwargs['end'] = (datetime_to_nanoseconds( criteria.endtime)) self.data = view.get_data(**getdata_kwargs) if not persistent: view.close() if self.table.rows > 0: self.data = self.data[:self.table.rows] self.parse_data() logger.info("NetShark Report %s returned %s rows" % (self.job, len(self.data))) return QueryComplete(self.data)
def render(self, name, value, attrs): initial_time = attrs.get('initial_time', None) if initial_time: m = re.match("now *- *(.+)", initial_time) if m: secs = timedelta_total_seconds(parse_timedelta(m.group(1))) initial_time = ( "d = new Date(); d.setSeconds(d.getSeconds()-%d);" % secs ) else: initial_time = "d = new Date('{}');".format(initial_time) else: initial_time = "d = new Date();" round_initial = attrs.get('round_initial', None) if round_initial: js = (' p = %d*1000; ' 'd = new Date(Math.floor(d.getTime() / p) * p);' % round_initial) initial_time += js # only pin manually entered times if we are rounding above a minute if round_initial >= 60: step = int(round_initial) / 60 force_round_time = 'true' else: step = 15 force_round_time = 'false' else: step = 15 force_round_time = 'false' msg = ''' <span class="input-group-addon"> <span id="timenow_{name}" class="glyphicon glyphicon-time" title="Set time/date to now"> </span> </span> {0} <script type="text/javascript"> $("#id_{name}").timepicker({{ step: {step}, scrollDefaultNow:true, forceRoundTime: {force_round_time}, timeFormat:"g:i:s a" }}); $("#timenow_{name}").click(function() {{ $("#id_{name}").timepicker("setTime", new Date()); }}); {initial_time} // align to timezone var offset = ($('#tz').html()/100)*60*60*1000; d = rvbd.timeutil.convertDateToUTC(d); d.setTime(d.getTime() + offset); $("#id_{name}").timepicker("setTime", d); </script> ''' return msg.format( super(TimeWidget, self).render(name, value, attrs), name=name, initial_time=initial_time, step=step, force_round_time=force_round_time )
def compute_synthetic(self, job, df): """ Compute the synthetic columns from DF a two-dimensional array of the non-synthetic columns. Synthesis occurs as follows: 1. Compute all synthetic columns where compute_post_resample is False 2. If the table is a time-based table with a defined resolution, the result is resampled. 3. Any remaining columns are computed. """ if df is None: return None all_columns = job.get_columns() all_col_names = [c.name for c in all_columns] def compute(df, syncols): for syncol in syncols: expr = syncol.compute_expression g = tokenize.generate_tokens(StringIO(expr).readline) newexpr = "" getvalue = False getclose = False for ttype, tvalue, _, _, _ in g: if getvalue: if ttype != tokenize.NAME: msg = "Invalid syntax, expected {name}: %s" % tvalue raise ValueError(msg) elif tvalue in all_col_names: newexpr += "df['%s']" % tvalue elif tvalue in job.criteria: newexpr += '"%s"' % str(job.criteria.get(tvalue)) else: raise ValueError("Invalid variable name: %s" % tvalue) getclose = True getvalue = False elif getclose: if ttype != tokenize.OP and tvalue != "}": msg = "Invalid syntax, expected {name}: %s" % tvalue raise ValueError(msg) getclose = False elif ttype == tokenize.OP and tvalue == "{": getvalue = True else: newexpr += tvalue newexpr += ' ' try: df[syncol.name] = eval(newexpr) except NameError as e: m = (('%s: expression failed: %s, check ' 'APPFWK_SYNTHETIC_MODULES: %s') % (self, newexpr, str(e))) logger.exception(m) raise TableComputeSyntheticError(m) # 1. Compute synthetic columns where post_resample is False compute(df, [ col for col in all_columns if (col.synthetic and col.compute_post_resample is False) ]) # 2. Resample colmap = {} timecol = None for col in all_columns: colmap[col.name] = col if col.istime(): timecol = col.name if self.resample: if timecol is None: raise (TableComputeSyntheticError( "%s: 'resample' is set but no 'time' column'" % self)) if (('resolution' not in job.criteria) and ('resample_resolution' not in job.criteria)): raise (TableComputeSyntheticError( ("%s: 'resample' is set but criteria missing " + "'resolution' or 'resample_resolution'") % self)) how = {} for k in df.keys(): if k == timecol or k not in colmap: continue how[k] = colmap[k].resample_operation if 'resample_resolution' in job.criteria: resolution = job.criteria.resample_resolution else: resolution = job.criteria.resolution resolution = timedelta_total_seconds(resolution) if resolution < 1: raise (TableComputeSyntheticError( ("Table %s cannot resample at a resolution " + "less than 1 second") % self)) logger.debug('%s: resampling to %ss' % (self, int(resolution))) indexed = df.set_index(timecol) resampled = indexed.resample('%ss' % int(resolution), how, convention='end').reset_index() df = resampled # 3. Compute remaining synthetic columns (post_resample is True) compute(df, [ c for c in all_columns if (c.synthetic and c.compute_post_resample is True) ]) return df
def process(cls, widget, job, data): class ColInfo: def __init__(self, col, dataindex, axis, istime=False, isdate=False): self.col = col self.key = cleankey(col.name) self.dataindex = dataindex self.axis = axis self.istime = istime self.isdate = isdate t_cols = job.get_columns() colinfo = {} # map by widget key # columns of None is a special case, just use all # defined columns other than time if widget.options.columns is None: valuecolnames = [ col.name for col in t_cols if not col.istime() and not col.isdate() ] else: valuecolnames = widget.options.columns # Column keys are the 'cleaned' column names w_keys = [cleankey(n) for n in valuecolnames] # Retrieve the desired value columns # ...and the indices for the value values # (as the 'data' has *all* columns) time_colinfo = None for i, c in enumerate(t_cols): if c.istime(): ci = ColInfo(c, i, -1, istime=True) time_colinfo = ci elif c.isdate(): ci = ColInfo(c, i, -1, isdate=True) time_colinfo = ci elif c.name in valuecolnames: if c.isnumeric(): ci = ColInfo(c, i, -1, istime=False, isdate=False) else: raise KeyError( "Cannot graph non-numeric data in timeseries widget: " "column {0}".format(c.name)) colinfo[ci.key] = ci if widget.options.altaxis: altaxis = widget.options.altaxis axes_def = { '0': { 'position': 'left', 'columns': [col for col in valuecolnames if col not in altaxis] }, '1': { 'position': 'right', 'columns': [col for col in valuecolnames if col in altaxis] } } else: axes_def = {'0': {'position': 'left', 'columns': valuecolnames}} w_series = [] axes = Axes(axes_def) # Setup the time axis w_axes = { "time": { "keys": ["time"], "position": "bottom", "type": "time", "styles": { "label": { "fontSize": "8pt", "rotation": "-45" } } } } # Create a better time format depending on t0/t1 t_dataindex = time_colinfo.dataindex t0 = data[0][t_dataindex] t1 = data[-1][t_dataindex] if not hasattr(t0, 'utcfromtimestamp'): t0 = timeutils.sec_string_to_datetime(t0) t1 = timeutils.sec_string_to_datetime(t1) total_seconds = timeutils.timedelta_total_seconds(t1 - t0) if total_seconds < 2: w_axes['time']['formatter'] = 'formatTimeMs' elif total_seconds < 120: w_axes['time']['labelFormat'] = '%k:%M:%S' elif total_seconds < (24 * 60 * 60): w_axes['time']['labelFormat'] = '%k:%M' elif time_colinfo.isdate: w_axes['time']['formatter'] = 'formatDate' else: w_axes['time']['labelFormat'] = '%D %k:%M' # Setup the other axes, checking the axis for each column for w_key in w_keys: # Need to interate the valuecolnames array to preserve order ci = colinfo[w_key] w_series.append({ "xKey": "time", "xDisplayName": "Time", "yKey": ci.key, "yDisplayName": ci.col.label, "styles": { "line": { "weight": 1 }, "marker": { "height": 3, "width": 3 } } }) ci.axis = axes.getaxis(ci.col.name) axis_name = 'axis' + str(ci.axis) if axis_name not in w_axes: w_axes[axis_name] = { "type": "numeric", "position": axes.position(ci.axis), "keys": [] } w_axes[axis_name]['keys'].append(ci.key) # Output row data rows = [] # min/max values by axis 0/1 minval = {} maxval = {} stacked = widget.options.stacked # Iterate through all rows if input data for rawrow in data: t = rawrow[t_dataindex] try: t = timeutils.datetime_to_microseconds(t) / 1000 except AttributeError: t = t * 1000 row = {'time': t} rowmin = {} rowmax = {} for ci in colinfo.values(): if ci.istime or ci.isdate: continue a = ci.axis val = rawrow[ci.dataindex] row[ci.key] = val if val != '' else None # If stacked and there is only one value, use that # value as the rowmin. If stacked and there is more than # one value for the axis, use a floor of 0 to give proper # context. if a not in rowmin: rowmin[a] = val if val != '' else 0 rowmax[a] = val if val != '' else 0 else: rowmin[a] = (0 if stacked else min(rowmin[a], val)) rowmax[a] = ((rowmax[a] + val) if stacked else max(rowmax[a], val)) for a in rowmin.keys(): minval[a] = rowmin[a] if (a not in minval) else min( minval[a], rowmin[a]) maxval[a] = rowmax[a] if (a not in maxval) else max( maxval[a], rowmax[a]) rows.append(row) # Setup the scale values for the axes for ci in colinfo.values(): if ci.istime or ci.isdate: continue axis_name = 'axis' + str(ci.axis) if minval and maxval: n = NiceScale(minval[ci.axis], maxval[ci.axis]) w_axes[axis_name]['minimum'] = "%.10f" % n.nicemin w_axes[axis_name]['maximum'] = "%.10f" % n.nicemax w_axes[axis_name]['tickExponent'] = math.log10(n.tickspacing) w_axes[axis_name]['styles'] = { 'majorUnit': { 'count': n.numticks } } else: # empty data which would result in keyError above w_axes[axis_name]['minimum'] = "0" w_axes[axis_name]['maximum'] = "1" w_axes[axis_name]['tickExponent'] = 1 w_axes[axis_name]['styles'] = {'majorUnit': {'count': 1}} if ci.col.units == ci.col.UNITS_PCT: w_axes[axis_name]['formatter'] = 'formatPct' else: w_axes[axis_name]['formatter'] = 'formatMetric' if stacked: charttype = "area" elif widget.options.bar: charttype = "column" else: charttype = "combo" data = { "chartTitle": widget.title.format(**job.actual_criteria), "type": charttype, "stacked": stacked, "dataProvider": rows, "seriesCollection": w_series, "axes": w_axes, "legend": { "position": "bottom", "fontSize": "8pt", "styles": { "gap": 0 } }, "interactionType": "planar" } # logger.debug("data:\n\n%s\n" % data) return data
def run(self): """ Main execution method """ criteria = self.job.criteria self.timeseries = False # if key column called 'time' is created self.column_names = [] # Resolution comes in as a time_delta resolution = timedelta_total_seconds(criteria.resolution) default_delta = 1000000000 # one second self.delta = int(default_delta * resolution) # sample size interval if criteria.netshark_device == '': logger.debug('%s: No netshark device selected' % self.table) self.job.mark_error("No NetShark Device Selected") return False #self.fake_run() #return True shark = DeviceManager.get_device(criteria.netshark_device) logger.debug("Creating columns for NetShark table %d" % self.table.id) # Create Key/Value Columns columns = [] for tc in self.table.get_columns(synthetic=False): tc_options = tc.options if ( tc.iskey and tc.name == 'time' and tc_options.extractor == 'sample_time'): # don't create column, use the sample time for timeseries self.timeseries = True self.column_names.append('time') continue elif tc.iskey: c = Key(tc_options.extractor, description=tc.label, default_value=tc_options.default_value) else: if tc_options.operation: try: operation = getattr(Operation, tc_options.operation) except AttributeError: operation = Operation.sum print ('ERROR: Unknown operation attribute ' '%s for column %s.' % (tc_options.operation, tc.name)) else: operation = Operation.none c = Value(tc_options.extractor, operation, description=tc.label, default_value=tc_options.default_value) self.column_names.append(tc.name) columns.append(c) # Identify Sort Column sortidx = None if self.table.sortcols is not None: sortcol = Column.objects.get(table=self.table, name=self.table.sortcols[0]) sort_name = sortcol.options.extractor for i, c in enumerate(columns): if c.field == sort_name: sortidx = i break # Initialize filters criteria = self.job.criteria filters = [] filterexpr = self.job.combine_filterexprs( exprs=criteria.netshark_filterexpr, joinstr="&" ) if filterexpr: filters.append(NetSharkFilter(filterexpr)) tf = TimeFilter(start=criteria.starttime, end=criteria.endtime) filters.append(tf) logger.info("Setting netshark table %d timeframe to %s" % (self.table.id, str(tf))) # Get source type from options try: with lock: source = path_to_class(shark, self.job.criteria.netshark_source_name) except RvbdHTTPException, e: source = None raise e
def run(self): criteria = self.job.criteria tzname = criteria.business_hours_tzname logger.debug("%s: timezone: %s" % (self.job, tzname)) tz = pytz.timezone(tzname) # Convert to datetime objects in the requested timezone st = criteria.starttime.astimezone(tz) et = criteria.endtime.astimezone(tz) logger.debug("%s: times: %s - %s" % (self.job, st, et)) # Business hours start/end, as string "HH:MMam" like 8:00am sb = parse_time(criteria.business_hours_start) eb = parse_time(criteria.business_hours_end) weekends = criteria.business_hours_weekends # Iterate from st to et until times = [] t = st while t <= et: # Set t0/t1 to date of t but time of sb/eb t0 = replace_time(t, sb, tz) t1 = replace_time(t, eb, tz) # Advance t by 1 day t = t + datetime.timedelta(days=1) # Strip timezone, then re-add to account for DST t = tz.localize(t.replace(tzinfo=None)) # Skip weekends if not weekends and t0.weekday() >= 5: continue # Now see if we have any overlap of biz hours for today if et < t0: # Report end time is today before biz hours start, all done break if et < t1: # Report end time is today in the middle of biz hours, adjust t1 = et if t1 < st: # Report start time occurs today *after* biz end, nothing today continue if t0 < st: # Report start time occurs today in the middle of the biz hours # Adjust t0 t0 = st logger.debug("%s: start: %s, end: %s, duration: %s" % (self.job, str(t0), str(t1), str(timedelta_total_seconds(t1-t0)))) times.append((t0, t1, timedelta_total_seconds(t1-t0))) if len(times) == 0: data = None else: # manually assign data types to avoid Python 2.6 issues # when converting date times columns = ['starttime', 'endtime', 'totalsecs'] s1 = pandas.Series([x[0] for x in times], dtype='datetime64[ns]') s2 = pandas.Series([x[1] for x in times], dtype='datetime64[ns]') tt = pandas.Series([x[2] for x in times], dtype='int') # create dataframe then assign using correct column ordering df = pandas.DataFrame(dict(zip(columns, [s1, s2, tt]))) data = df[columns] return QueryComplete(data)
def run(self): """ Main execution method """ criteria = self.job.criteria self.timeseries = False # if key column called 'time' is created self.column_names = [] # Resolution comes in as a time_delta resolution = timedelta_total_seconds(criteria.resolution) default_delta = 1000000000 # one second self.delta = int(default_delta * resolution) # sample size interval if criteria.netshark_device == '': logger.debug('%s: No netshark device selected' % self.table) self.job.mark_error("No NetShark Device Selected") return False shark = DeviceManager.get_device(criteria.netshark_device) logger.debug("Creating columns for NetShark table %d" % self.table.id) # Create Key/Value Columns columns = [] for tc in self.table.get_columns(synthetic=False): tc_options = tc.options if (tc.iskey and tc.name == 'time' and tc_options.extractor == 'sample_time'): # don't create column, use the sample time for timeseries self.timeseries = True self.column_names.append('time') continue elif tc.iskey: c = Key(tc_options.extractor, description=tc.label, default_value=tc_options.default_value) else: if tc_options.operation: try: operation = getattr(Operation, tc_options.operation) except AttributeError: operation = Operation.sum print ('ERROR: Unknown operation attribute ' '%s for column %s.' % (tc_options.operation, tc.name)) else: operation = Operation.none c = Value(tc_options.extractor, operation, description=tc.label, default_value=tc_options.default_value) self.column_names.append(tc.name) columns.append(c) # Identify Sort Column sortidx = None if self.table.sortcols is not None: sortcol = Column.objects.get(table=self.table, name=self.table.sortcols[0]) sort_name = sortcol.options.extractor for i, c in enumerate(columns): if c.field == sort_name: sortidx = i break # Initialize filters criteria = self.job.criteria filters = [] if hasattr(criteria, 'netshark_filterexpr'): logger.debug('calculating netshark filter expression ...') filterexpr = self.job.combine_filterexprs( exprs=criteria.netshark_filterexpr, joinstr="&" ) if filterexpr: logger.debug('applying netshark filter expression: %s' % filterexpr) filters.append(NetSharkFilter(filterexpr)) if hasattr(criteria, 'netshark_bpf_filterexpr'): # TODO evaluate how to combine multiple BPF filters # this will just apply one at a time filterexpr = criteria.netshark_bpf_filterexpr logger.debug('applying netshark BPF filter expression: %s' % filterexpr) filters.append(BpfFilter(filterexpr)) resolution = criteria.resolution if resolution.seconds == 1: sampling_time_msec = 1000 elif resolution.microseconds == 1000: sampling_time_msec = 1 if criteria.duration > parse_timedelta('1s'): msg = ("Cannot run a millisecond report with a duration " "longer than 1 second") raise ValueError(msg) else: sampling_time_msec = 1000 # Get source type from options logger.debug("NetShark Source: %s" % self.job.criteria.netshark_source_name) source = path_to_class( shark, self.job.criteria.netshark_source_name) live = source.is_live() persistent = criteria.get('netshark_persistent', False) if live and not persistent: raise ValueError("Live views must be run with persistent set") view = None if persistent: # First, see a view by this title already exists # Title is the table name plus a criteria hash including # all criteria *except* the timeframe h = hashlib.md5() h.update('.'.join([c.name for c in self.table.get_columns()])) for k, v in criteria.iteritems(): if criteria.is_timeframe_key(k): continue h.update('%s:%s' % (k, v)) title = '/'.join(['steelscript-appfwk', str(self.table.id), self.table.namespace, self.table.name, h.hexdigest()]) view = NetSharkViews.find_by_name(shark, title) logger.debug("Persistent view title: %s" % title) else: # Only assign a title for persistent views title = None if not view: # Not persistent, or not yet created... if not live: # Cannot attach time filter to a live view, # it will be added later at get_data() time tf = TimeFilter(start=criteria.starttime, end=criteria.endtime) filters.append(tf) logger.info("Setting netshark table %d timeframe to %s" % (self.table.id, str(tf))) # Create it with lock: logger.debug("%s: Creating view for table %s" % (str(self), str(self.table))) view = shark.create_view( source, columns, filters=filters, sync=False, name=title, sampling_time_msec=sampling_time_msec) if not live: done = False logger.debug("Waiting for netshark table %d to complete" % self.table.id) while not done: time.sleep(0.5) with lock: s = view.get_progress() self.job.mark_progress(s) self.job.save() done = view.is_ready() logger.debug("Retrieving data for timeframe: %s - %s" % (datetime_to_nanoseconds(criteria.starttime), datetime_to_nanoseconds(criteria.endtime))) # Retrieve the data with lock: getdata_kwargs = {} if sortidx: getdata_kwargs['sortby'] = sortidx if self.table.options.aggregated: getdata_kwargs['aggregated'] = self.table.options.aggregated else: getdata_kwargs['delta'] = self.delta if live: # For live views, attach the time frame to the get_data() getdata_kwargs['start'] = ( datetime_to_nanoseconds(criteria.starttime)) getdata_kwargs['end'] = ( datetime_to_nanoseconds(criteria.endtime)) self.data = view.get_data(**getdata_kwargs) if not persistent: view.close() if self.table.rows > 0: self.data = self.data[:self.table.rows] self.parse_data() logger.info("NetShark Report %s returned %s rows" % (self.job, len(self.data))) return QueryComplete(self.data)
def run(self): """ Main execution method. """ criteria = self.job.criteria if criteria.netprofiler_device == '': logger.debug('%s: No netprofiler device selected' % self.table) self.job.mark_error("No NetProfiler Device Selected") return False profiler = DeviceManager.get_device(criteria.netprofiler_device) report = steelscript.netprofiler.core.report.MultiQueryReport(profiler) tf = TimeFilter(start=criteria.starttime, end=criteria.endtime) logger.info("Running NetProfilerTemplateTable table %d report " "for timeframe %s" % (self.table.id, str(tf))) trafficexpr = TrafficFilter( self.job.combine_filterexprs(exprs=criteria.profiler_filterexpr)) # Incoming criteria.resolution is a timedelta logger.debug('NetProfiler report got criteria resolution %s (%s)' % (criteria.resolution, type(criteria.resolution))) if criteria.resolution != 'auto': rsecs = int(timedelta_total_seconds(criteria.resolution)) resolution = steelscript.netprofiler.core.report.Report.RESOLUTION_MAP[ rsecs] else: resolution = 'auto' logger.debug('NetProfiler report using resolution %s (%s)' % (resolution, type(resolution))) with lock: res = report.run(template_id=self.table.options.template_id, timefilter=tf, trafficexpr=trafficexpr, resolution=resolution) if res is True: logger.info("Report template complete.") self.job.safe_update(progress=100) # Retrieve the data with lock: query = report.get_query_by_index(0) data = query.get_data() headers = report.get_legend() tz = criteria.starttime.tzinfo # Update criteria criteria.starttime = (datetime.datetime.utcfromtimestamp( query.actual_t0).replace(tzinfo=tz)) criteria.endtime = (datetime.datetime.utcfromtimestamp( query.actual_t1).replace(tzinfo=tz)) self.job.safe_update(actual_criteria=criteria) # create dataframe with all of the default headers df = pandas.DataFrame(data, columns=[h.key for h in headers]) # now filter down to the columns requested by the table columns = [col.name for col in self.table.get_columns(synthetic=False)] self.data = df[columns] logger.info("Report %s returned %s rows" % (self.job, len(self.data))) return True
def compute_synthetic(self, job, df): """ Compute the synthetic columns from DF a two-dimensional array of the non-synthetic columns. Synthesis occurs as follows: 1. Compute all synthetic columns where compute_post_resample is False 2. If the table is a time-based table with a defined resolution, the result is resampled. 3. Any remaining columns are computed. """ if df is None: return None all_columns = job.get_columns() all_col_names = [c.name for c in all_columns] def compute(df, syncols): for syncol in syncols: expr = syncol.compute_expression g = tokenize.generate_tokens(StringIO(expr).readline) newexpr = "" getvalue = False getclose = False for ttype, tvalue, _, _, _ in g: if getvalue: if ttype != tokenize.NAME: msg = "Invalid syntax, expected {name}: %s" % tvalue raise ValueError(msg) elif tvalue in all_col_names: newexpr += "df['%s']" % tvalue elif tvalue in job.criteria: newexpr += '"%s"' % str(job.criteria.get(tvalue)) else: raise ValueError("Invalid variable name: %s" % tvalue) getclose = True getvalue = False elif getclose: if ttype != tokenize.OP and tvalue != "}": msg = "Invalid syntax, expected {name}: %s" % tvalue raise ValueError(msg) getclose = False elif ttype == tokenize.OP and tvalue == "{": getvalue = True else: newexpr += tvalue newexpr += ' ' try: df[syncol.name] = eval(newexpr) except NameError as e: m = (('%s: expression failed: %s, check ' 'APPFWK_SYNTHETIC_MODULES: %s') % (self, newexpr, str(e))) logger.exception(m) raise TableComputeSyntheticError(m) # 1. Compute synthetic columns where post_resample is False compute(df, [col for col in all_columns if (col.synthetic and col.compute_post_resample is False)]) # 2. Resample colmap = {} timecol = None for col in all_columns: colmap[col.name] = col if col.istime(): timecol = col.name if self.resample: if timecol is None: raise (TableComputeSyntheticError ("%s: 'resample' is set but no 'time' column'" % self)) if (('resolution' not in job.criteria) and ('resample_resolution' not in job.criteria)): raise (TableComputeSyntheticError (("%s: 'resample' is set but criteria missing " + "'resolution' or 'resample_resolution'") % self)) how = {} for k in df.keys(): if k == timecol or k not in colmap: continue how[k] = colmap[k].resample_operation if 'resample_resolution' in job.criteria: resolution = job.criteria.resample_resolution else: resolution = job.criteria.resolution resolution = timedelta_total_seconds(resolution) if resolution < 1: raise (TableComputeSyntheticError (("Table %s cannot resample at a resolution " + "less than 1 second") % self)) logger.debug('%s: resampling to %ss' % (self, int(resolution))) indexed = df.set_index(timecol) resampled = indexed.resample('%ss' % int(resolution), how, convention='end').reset_index() df = resampled # 3. Compute remaining synthetic columns (post_resample is True) compute(df, [c for c in all_columns if (c.synthetic and c.compute_post_resample is True)]) return df
def create(cls, shark, interface, name, packet_retention_size_limit, packet_retention_packet_limit=None, packet_retention_time_limit=None, bpf_filter=None, snap_length=65525, indexing_size_limit=None, indexing_synced=False, indexing_time_limit=None, start_immediately=True, requested_start_time=None, requested_stop_time=None, stop_rule_size_limit=None, stop_rule_packet_limit=None, stop_rule_time_limit=None): """Create a new capture job""" def _calc_size(size, total): if isinstance(size, str) and size[-1] == '%': size = total * int(size[:-1]) / 100 elif not isinstance(size, (int, long)) and size is not None: size = utils.human2bytes(size) return size stats = shark.get_stats() packet_storage_total = stats['storage']['packet_storage']['total'] index_storage_total = stats['storage']['os_storage']['index_storage']['total'] packet_retention_size_limit = _calc_size(packet_retention_size_limit, packet_storage_total) stop_rule_size_limit = _calc_size(stop_rule_size_limit, packet_storage_total) if stop_rule_size_limit else None indexing_size_limit = _calc_size(indexing_size_limit, index_storage_total) if indexing_size_limit else None packet_retention_packet_limit = int(packet_retention_packet_limit) if packet_retention_packet_limit else None stop_rule_size_limit = int(stop_rule_size_limit) if stop_rule_size_limit else None stop_rule_packet_limit = int(stop_rule_packet_limit) if stop_rule_packet_limit else None snap_length = int(snap_length) if snap_length else 65535 indexing_size_limit = int(indexing_size_limit) if indexing_size_limit else None if indexing_time_limit: indexing_time_limit = int(timeutils.timedelta_total_seconds(indexing_time_limit)) if packet_retention_time_limit: packet_retention_time_limit = int(timeutils.timedelta_total_seconds(packet_retention_time_limit)) if stop_rule_time_limit: stop_rule_time_limit = int(timeutils.timedelta_total_seconds(stop_rule_time_limit)) if requested_start_time: requested_start_time = requested_start_time.strftime(cls._timefmt) if requested_stop_time: requested_stop_time = requested_stop_time.strftime(cls._timefmt) jobrequest = {'interface_name': interface.id} jobrequest['name'] = name if packet_retention_size_limit or packet_retention_packet_limit or packet_retention_time_limit: jobrequest['packet_retention'] = dict() if packet_retention_size_limit: jobrequest['packet_retention']['size_limit'] = packet_retention_size_limit if packet_retention_packet_limit: jobrequest['packet_retention']['packet_limit'] = packet_retention_packet_limit if packet_retention_time_limit: jobrequest['packet_retention']['time_limit'] = packet_retention_time_limit if bpf_filter: jobrequest['bpf_filter'] = bpf_filter if requested_start_time: jobrequest['requested_start_time'] = requested_start_time if requested_stop_time: jobrequest['requested_stop_time'] = requested_stop_time if stop_rule_size_limit or stop_rule_packet_limit or stop_rule_time_limit: jobrequest['stop_rule'] = dict() if stop_rule_size_limit: jobrequest['stop_rule']['size_limit'] = stop_rule_size_limit if stop_rule_packet_limit: jobrequest['stop_rule']['packet_limit'] = stop_rule_packet_limit if stop_rule_time_limit: jobrequest['stop_rule']['time_limit'] = stop_rule_time_limit if snap_length: jobrequest['snap_length'] = int(snap_length) # Indexing logic table # case size synced time valid? # 1 None False None valid - index is disabled # 2 None False int error # 3 None True None error # 4 None True int error # 5 int False None valid # 6 int False int valid # 7 int True None valid # 8 int True int valid if (indexing_synced or indexing_time_limit) and not indexing_size_limit: raise NetSharkException( 'indexing_size_limit must be specified ' 'with indexing_synced or indexing_time_limit') elif indexing_size_limit: jobrequest['indexing'] = dict() jobrequest['indexing']['size_limit'] = indexing_size_limit if indexing_synced: jobrequest['indexing']['synced'] = indexing_synced if indexing_time_limit: jobrequest['indexing']['time_limit'] = indexing_time_limit jobrequest['start_immediately'] = start_immediately job_id = shark.api.jobs.add(jobrequest) job = cls(shark, job_id) return job
def run(self, template_id, timefilter=None, resolution="auto", query=None, trafficexpr=None, data_filter=None, sync=True, custom_criteria=None): """Create the report and begin running the report on NetProfiler. If the `sync` option is True, periodically poll until the report is complete, otherwise return immediately. :param int template_id: numeric id of the template to use for the report :param timefilter: range of time to query, instance of :class:`TimeFilter` :param str resolution: data resolution, such as (1min, 15min, etc.), defaults to 'auto' :param str query: query object containing criteria :param trafficexpr: instance of :class:`TrafficFilter` :param str data_filter: deprecated filter to run against report data :param bool sync: if True, poll for status until the report is complete """ self.template_id = template_id if timefilter is None: self.timefilter = TimeFilter.parse_range("last 5 min") else: self.timefilter = timefilter self.query = query self.trafficexpr = trafficexpr self.data_filter = data_filter self.id = None self.queries = list() self.last_status = None if resolution not in ["auto", "1min", "15min", "hour", "6hour", "day", "week", "month"]: rd = parse_timedelta(resolution) resolution = self.RESOLUTION_MAP[int(timedelta_total_seconds(rd))] self.resolution = resolution start = datetime_to_seconds(self.timefilter.start) end = datetime_to_seconds(self.timefilter.end) criteria = RecursiveUpdateDict(**{"time_frame": {"start": int(start), "end": int(end)} }) if self.query is not None: criteria["query"] = self.query if self.resolution != "auto": criteria["time_frame"]["resolution"] = self.resolution if self.data_filter: criteria['deprecated'] = {self.data_filter[0]: self.data_filter[1]} if self.trafficexpr is not None: criteria["traffic_expression"] = self.trafficexpr.filter if custom_criteria: for k, v in custom_criteria.iteritems(): criteria[k] = v to_post = {"template_id": self.template_id, "criteria": criteria} logger.debug("Posting JSON: %s" % to_post) response = self.profiler.api.report.reports(data=to_post) try: self.id = int(response['id']) except KeyError: raise ValueError( "failed to retrieve report id from report creation response: %s" % response) logger.info("Created report %d" % self.id) if sync: self.wait_for_complete()
def run(self): """ Main execution method. """ criteria = self.job.criteria if criteria.netprofiler_device == '': logger.debug('%s: No netprofiler device selected' % self.table) self.job.mark_error("No NetProfiler Device Selected") return False profiler = DeviceManager.get_device(criteria.netprofiler_device) report = steelscript.netprofiler.core.report.MultiQueryReport(profiler) tf = TimeFilter(start=criteria.starttime, end=criteria.endtime) logger.info("Running NetProfilerTemplateTable table %d report " "for timeframe %s" % (self.table.id, str(tf))) trafficexpr = TrafficFilter( self.job.combine_filterexprs(exprs=criteria.profiler_filterexpr) ) # Incoming criteria.resolution is a timedelta logger.debug('NetProfiler report got criteria resolution %s (%s)' % (criteria.resolution, type(criteria.resolution))) if criteria.resolution != 'auto': rsecs = int(timedelta_total_seconds(criteria.resolution)) resolution = steelscript.netprofiler.core.report.Report.RESOLUTION_MAP[rsecs] else: resolution = 'auto' logger.debug('NetProfiler report using resolution %s (%s)' % (resolution, type(resolution))) with lock: res = report.run(template_id=self.table.options.template_id, timefilter=tf, trafficexpr=trafficexpr, resolution=resolution) if res is True: logger.info("Report template complete.") self.job.safe_update(progress=100) # Retrieve the data with lock: query = report.get_query_by_index(0) data = query.get_data() headers = report.get_legend() tz = criteria.starttime.tzinfo # Update criteria criteria.starttime = (datetime.datetime .utcfromtimestamp(query.actual_t0) .replace(tzinfo=tz)) criteria.endtime = (datetime.datetime .utcfromtimestamp(query.actual_t1) .replace(tzinfo=tz)) self.job.safe_update(actual_criteria=criteria) # create dataframe with all of the default headers df = pandas.DataFrame(data, columns=[h.key for h in headers]) # now filter down to the columns requested by the table columns = [col.name for col in self.table.get_columns(synthetic=False)] self.data = df[columns] logger.info("Report %s returned %s rows" % (self.job, len(self.data))) return True
def process(cls, widget, job, data): class ColInfo: def __init__(self, col, dataindex, axis, istime=False, isdate=False): self.col = col self.key = cleankey(col.name) self.dataindex = dataindex self.axis = axis self.istime = istime self.isdate = isdate t_cols = job.get_columns() colinfo = {} # map by widget key # columns of None is a special case, just use all # defined columns other than time if widget.options.columns is None: valuecolnames = [col.name for col in t_cols if not col.istime() and not col.isdate()] else: valuecolnames = widget.options.columns # Column keys are the 'cleaned' column names w_keys = [cleankey(n) for n in valuecolnames] # Retrieve the desired value columns # ...and the indices for the value values # (as the 'data' has *all* columns) time_colinfo = None for i, c in enumerate(t_cols): if c.istime(): ci = ColInfo(c, i, -1, istime=True) time_colinfo = ci elif c.isdate(): ci = ColInfo(c, i, -1, isdate=True) time_colinfo = ci elif c.name in valuecolnames: if c.isnumeric(): ci = ColInfo(c, i, -1, istime=False, isdate=False) else: raise KeyError( "Cannot graph non-numeric data in timeseries widget: " "column {0}".format(c.name)) colinfo[ci.key] = ci if widget.options.altaxis: altaxis = widget.options.altaxis axes_def = {'0': {'position': 'left', 'columns': [col for col in valuecolnames if col not in altaxis]}, '1': {'position': 'right', 'columns': [col for col in valuecolnames if col in altaxis]} } else: axes_def = {'0': {'position': 'left', 'columns': valuecolnames}} w_series = [] axes = Axes(axes_def) # Setup the time axis w_axes = {"time": {"keys": ["time"], "position": "bottom", "type": "time", "styles": {"label": {"fontSize": "8pt", "rotation": "-45"}}}} # Create a better time format depending on t0/t1 t_dataindex = time_colinfo.dataindex t0 = data[0][t_dataindex] t1 = data[-1][t_dataindex] if not hasattr(t0, 'utcfromtimestamp'): t0 = timeutils.sec_string_to_datetime(t0) t1 = timeutils.sec_string_to_datetime(t1) total_seconds = timeutils.timedelta_total_seconds(t1 - t0) if total_seconds < 2: w_axes['time']['formatter'] = 'formatTimeMs' elif total_seconds < 120: w_axes['time']['labelFormat'] = '%k:%M:%S' elif total_seconds < (24 * 60 * 60): w_axes['time']['labelFormat'] = '%k:%M' elif time_colinfo.isdate: w_axes['time']['formatter'] = 'formatDate' else: w_axes['time']['labelFormat'] = '%D %k:%M' # Setup the other axes, checking the axis for each column for w_key in w_keys: # Need to interate the valuecolnames array to preserve order ci = colinfo[w_key] w_series.append({"xKey": "time", "xDisplayName": "Time", "yKey": ci.key, "yDisplayName": ci.col.label, "styles": {"line": {"weight": 1}, "marker": {"height": 3, "width": 3}}}) ci.axis = axes.getaxis(ci.col.name) axis_name = 'axis' + str(ci.axis) if axis_name not in w_axes: w_axes[axis_name] = {"type": "numeric", "position": axes.position(ci.axis), "keys": [] } w_axes[axis_name]['keys'].append(ci.key) # Output row data rows = [] # min/max values by axis 0/1 minval = {} maxval = {} stacked = widget.options.stacked # Iterate through all rows if input data for rawrow in data: t = rawrow[t_dataindex] try: t = timeutils.datetime_to_microseconds(t) / 1000 except AttributeError: t = t * 1000 row = {'time': t} rowmin = {} rowmax = {} for ci in colinfo.values(): if ci.istime or ci.isdate: continue a = ci.axis val = rawrow[ci.dataindex] row[ci.key] = val if val != '' else None # If stacked and there is only one value, use that # value as the rowmin. If stacked and there is more than # one value for the axis, use a floor of 0 to give proper # context. if a not in rowmin: rowmin[a] = val if val != '' else 0 rowmax[a] = val if val != '' else 0 else: rowmin[a] = (0 if stacked else min(rowmin[a], val)) rowmax[a] = ((rowmax[a] + val) if stacked else max(rowmax[a], val)) for a in rowmin.keys(): minval[a] = rowmin[a] if (a not in minval) else min(minval[a], rowmin[a]) maxval[a] = rowmax[a] if (a not in maxval) else max(maxval[a], rowmax[a]) rows.append(row) # Setup the scale values for the axes for ci in colinfo.values(): if ci.istime or ci.isdate: continue axis_name = 'axis' + str(ci.axis) if minval and maxval: n = NiceScale(minval[ci.axis], maxval[ci.axis]) w_axes[axis_name]['minimum'] = "%.10f" % n.nicemin w_axes[axis_name]['maximum'] = "%.10f" % n.nicemax w_axes[axis_name]['tickExponent'] = math.log10(n.tickspacing) w_axes[axis_name]['styles'] = { 'majorUnit': {'count': n.numticks} } else: # empty data which would result in keyError above w_axes[axis_name]['minimum'] = "0" w_axes[axis_name]['maximum'] = "1" w_axes[axis_name]['tickExponent'] = 1 w_axes[axis_name]['styles'] = {'majorUnit': {'count': 1}} if ci.col.units == ci.col.UNITS_PCT: w_axes[axis_name]['formatter'] = 'formatPct' else: w_axes[axis_name]['formatter'] = 'formatMetric' if stacked: charttype = "area" elif widget.options.bar: charttype = "column" else: charttype = "combo" data = { "chartTitle": widget.title.format(**job.actual_criteria), "type": charttype, "stacked": stacked, "dataProvider": rows, "seriesCollection": w_series, "axes": w_axes, "legend": {"position": "bottom", "fontSize": "8pt", "styles": {"gap": 0}}, "interactionType": "planar" } # logger.debug("data:\n\n%s\n" % data) return data
def create(cls, shark, interface, name, packet_retention_size_limit, packet_retention_packet_limit=None, packet_retention_time_limit=None, bpf_filter=None, snap_length=65525, indexing_size_limit=None, indexing_synced=False, indexing_time_limit=None, start_immediately=True, requested_start_time=None, requested_stop_time=None, stop_rule_size_limit=None, stop_rule_packet_limit=None, stop_rule_time_limit=None): """Create a new capture job""" def _calc_size(size, total): if isinstance(size, str) and size[-1] == '%': size = total * int(size[:-1]) / 100 elif not isinstance(size, (int, long)) and size is not None: size = datautils.human2bytes(size) return size stats = shark.get_stats() packet_storage_total = stats['storage']['packet_storage']['total'] index_storage_total = stats['storage']['os_storage']['index_storage'][ 'total'] packet_retention_size_limit = _calc_size(packet_retention_size_limit, packet_storage_total) stop_rule_size_limit = _calc_size( stop_rule_size_limit, packet_storage_total) if stop_rule_size_limit else None indexing_size_limit = _calc_size( indexing_size_limit, index_storage_total) if indexing_size_limit else None packet_retention_packet_limit = int( packet_retention_packet_limit ) if packet_retention_packet_limit else None stop_rule_size_limit = int( stop_rule_size_limit) if stop_rule_size_limit else None stop_rule_packet_limit = int( stop_rule_packet_limit) if stop_rule_packet_limit else None snap_length = int(snap_length) if snap_length else 65535 indexing_size_limit = int( indexing_size_limit) if indexing_size_limit else None if indexing_time_limit: indexing_time_limit = int( timeutils.timedelta_total_seconds(indexing_time_limit)) if packet_retention_time_limit: packet_retention_time_limit = int( timeutils.timedelta_total_seconds(packet_retention_time_limit)) if stop_rule_time_limit: stop_rule_time_limit = int( timeutils.timedelta_total_seconds(stop_rule_time_limit)) if requested_start_time: requested_start_time = requested_start_time.strftime(cls._timefmt) if requested_stop_time: requested_stop_time = requested_stop_time.strftime(cls._timefmt) jobrequest = {'interface_name': interface.id} jobrequest['name'] = name if packet_retention_size_limit or packet_retention_packet_limit or packet_retention_time_limit: jobrequest['packet_retention'] = dict() if packet_retention_size_limit: jobrequest['packet_retention'][ 'size_limit'] = packet_retention_size_limit if packet_retention_packet_limit: jobrequest['packet_retention'][ 'packet_limit'] = packet_retention_packet_limit if packet_retention_time_limit: jobrequest['packet_retention'][ 'time_limit'] = packet_retention_time_limit if bpf_filter: jobrequest['bpf_filter'] = bpf_filter if requested_start_time: jobrequest['requested_start_time'] = requested_start_time if requested_stop_time: jobrequest['requested_stop_time'] = requested_stop_time if stop_rule_size_limit or stop_rule_packet_limit or stop_rule_time_limit: jobrequest['stop_rule'] = dict() if stop_rule_size_limit: jobrequest['stop_rule']['size_limit'] = stop_rule_size_limit if stop_rule_packet_limit: jobrequest['stop_rule'][ 'packet_limit'] = stop_rule_packet_limit if stop_rule_time_limit: jobrequest['stop_rule']['time_limit'] = stop_rule_time_limit if snap_length: jobrequest['snap_length'] = int(snap_length) # Indexing logic table # case size synced time valid? # 1 None False None valid - index is disabled # 2 None False int error # 3 None True None error # 4 None True int error # 5 int False None valid # 6 int False int valid # 7 int True None valid # 8 int True int valid if (indexing_synced or indexing_time_limit) and not indexing_size_limit: raise NetSharkException( 'indexing_size_limit must be specified ' 'with indexing_synced or indexing_time_limit') elif indexing_size_limit: jobrequest['indexing'] = dict() jobrequest['indexing']['size_limit'] = indexing_size_limit if indexing_synced: jobrequest['indexing']['synced'] = indexing_synced if indexing_time_limit: jobrequest['indexing']['time_limit'] = indexing_time_limit jobrequest['start_immediately'] = start_immediately job_id = shark.api.jobs.add(jobrequest) job = cls(shark, job_id) return job
def run(self, template_id, timefilter=None, resolution="auto", query=None, trafficexpr=None, data_filter=None, sync=True, custom_criteria=None): """Create the report and begin running the report on NetProfiler. If the `sync` option is True, periodically poll until the report is complete, otherwise return immediately. :param int template_id: numeric id of the template to use for the report :param timefilter: range of time to query, instance of :class:`TimeFilter` :param str resolution: data resolution, such as (1min, 15min, etc.), defaults to 'auto' :param str query: query object containing criteria :param trafficexpr: instance of :class:`TrafficFilter` :param str data_filter: deprecated filter to run against report data :param bool sync: if True, poll for status until the report is complete """ self.template_id = template_id if timefilter is None: self.timefilter = TimeFilter.parse_range("last 5 min") else: self.timefilter = timefilter self.query = query self.trafficexpr = trafficexpr self.data_filter = data_filter self.id = None self.queries = list() self.last_status = None if resolution not in [ "auto", "1min", "15min", "hour", "6hour", "day", "week", "month" ]: rd = parse_timedelta(resolution) resolution = self.RESOLUTION_MAP[int(timedelta_total_seconds(rd))] self.resolution = resolution start = datetime_to_seconds(self.timefilter.start) end = datetime_to_seconds(self.timefilter.end) criteria = RecursiveUpdateDict( **{"time_frame": { "start": int(start), "end": int(end) }}) if self.query is not None: criteria["query"] = self.query if self.resolution != "auto": criteria["time_frame"]["resolution"] = self.resolution if self.data_filter: criteria['deprecated'] = {self.data_filter[0]: self.data_filter[1]} if self.trafficexpr is not None: criteria["traffic_expression"] = self.trafficexpr.filter if custom_criteria: for k, v in custom_criteria.iteritems(): criteria[k] = v to_post = {"template_id": self.template_id, "criteria": criteria} logger.debug("Posting JSON: %s" % to_post) response = self.profiler.api.report.reports(data=to_post) try: self.id = int(response['id']) except KeyError: raise ValueError( "failed to retrieve report id from report creation response: %s" % response) logger.info("Created report %d" % self.id) if sync: self.wait_for_complete()