def compute_times(target, tables, criteria, params): tzname = criteria.business_hours_tzname logger.debug("timezone: %s" % tzname) tz = pytz.timezone(tzname) # Convert to datetime objects in the requested timezone st = criteria.starttime.astimezone(tz) et = criteria.endtime.astimezone(tz) logger.debug("times: %s - %s" % (st, et)) # Business hours start/end, as string "HH:MMam" like 8:00am sb = parse_time(criteria.business_hours_start) eb = parse_time(criteria.business_hours_end) weekends = criteria.business_hours_weekends # Iterate from st to et until times = [] t = st while t <= et: # Set t0/t1 to date of t but time of sb/eb t0_t = replace_time(t, sb) t1_t = replace_time(t, eb) # Advance t by 1 day t = t + datetime.timedelta(days=1) # Skip weekends if not weekends and t0_t.weekday() >= 5: continue # Now see if we have any overlap of busines hours for today if et < t0_t: # Report end time is today before busines hours start, all done break if et < t1_t: # Report end time is today in the middle of busines hours, adjust t1_t = et if t1_t < st: # Report start time occurs today *after* business end, nothing today continue if t0_t < st: # Report start time occurs today in the middle of the business hours # Adjust t0 t0_t = st t0 = datetime_to_seconds(t0_t) t1 = datetime_to_seconds(t1_t) #logger.debug(" START: %s END: %s" % (str(t0_t), str(t1_t))) times.append([t0 * 1000, t1 * 1000, t1 - t0]) if len(times) == 0: return None else: return pandas.DataFrame(times, columns=['starttime', 'endtime', 'totalsecs'])
def compute_times(target, tables, criteria, params): tzname = criteria.business_hours_tzname logger.debug("timezone: %s" % tzname) tz = pytz.timezone(tzname) # Convert to datetime objects in the requested timezone st = criteria.starttime.astimezone(tz) et = criteria.endtime.astimezone(tz) logger.debug("times: %s - %s" % (st, et)) # Business hours start/end, as string "HH:MMam" like 8:00am sb = parse_time(criteria.business_hours_start) eb = parse_time(criteria.business_hours_end) weekends = criteria.business_hours_weekends # Iterate from st to et until times = [] t = st while t <= et: # Set t0/t1 to date of t but time of sb/eb t0_t = replace_time(t, sb) t1_t = replace_time(t, eb) # Advance t by 1 day t = t + datetime.timedelta(days=1) # Skip weekends if not weekends and t0_t.weekday() >= 5: continue # Now see if we have any overlap of busines hours for today if et < t0_t: # Report end time is today before busines hours start, all done break if et < t1_t: # Report end time is today in the middle of busines hours, adjust t1_t = et if t1_t < st: # Report start time occurs today *after* business end, nothing today continue if t0_t < st: # Report start time occurs today in the middle of the business hours # Adjust t0 t0_t = st t0 = datetime_to_seconds(t0_t) t1 = datetime_to_seconds(t1_t) #logger.debug(" START: %s END: %s" % (str(t0_t), str(t1_t))) times.append([t0*1000, t1*1000, t1-t0]) if len(times) == 0: return None else: return pandas.DataFrame(times, columns=['starttime', 'endtime', 'totalsecs'])
def analysis_generate_data(query, tables, criteria, params): t0 = datetime_to_seconds(criteria.starttime) t1 = datetime_to_seconds(criteria.endtime) data = [] for t in range(t0, t1, params['source_resolution']): data.append([t, 1]) df = pandas.DataFrame(data, columns=['time', 'value']) df['time'] = pandas.DatetimeIndex(df['time'] * 1000000000) return df
def analysis_generate_data(query, tables, criteria, params): t0 = datetime_to_seconds(criteria.starttime) t1 = datetime_to_seconds(criteria.endtime) data = [] for t in range(t0, t1, params['source_resolution']): data.append([t, 1]) df = pandas.DataFrame(data, columns=['time', 'value']) df['time'] = pandas.DatetimeIndex(df['time']*1000000000) return df
def send_trap(self): """ Send a SNMP trap with id `trapid` to the IP address `manager` """ oid = self.options.eoid # cascade enterprise Object ID trapid = self.options.trapid # base string for trap indicators community = self.options.community manager_ip = self.options.manager_ip severity = self.options.severity description = self.trap_description url = self.options.trap_url alert_level = self.options.alert_level now = timeutils.datetime_to_seconds(datetime.datetime.now()) trapname = '.'.join([oid, trapid]) ntf = ntforg.NotificationOriginator() err = ntf.sendNotification(ntforg.CommunityData(community), ntforg.UdpTransportTarget((manager_ip, 162)), 'trap', trapname, ('1.3.6.1.2.1.1.3.0', rfc1902.Integer(0)), # Uptime ('1.3.6.1.4.1.7054.71.2.1.0', rfc1902.Integer(severity)), # Severity ('1.3.6.1.4.1.7054.71.2.3.0', rfc1902.OctetString(description)), ('1.3.6.1.4.1.7054.71.2.4.0', rfc1902.Integer(0)), # Event ID ('1.3.6.1.4.1.7054.71.2.5.0', rfc1902.OctetString(url)), ('1.3.6.1.4.1.7054.71.2.7.0', rfc1902.Integer(alert_level)), # Alert Level ('1.3.6.1.4.1.7054.71.2.8.0', rfc1902.Integer(now)), # Start Time ('1.3.6.1.4.1.7054.71.2.16.0', rfc1902.Integer(0)), # Source Count ('1.3.6.1.4.1.7054.71.2.18.0', rfc1902.Integer(0)), # Destination Count ('1.3.6.1.4.1.7054.71.2.20.0', rfc1902.Integer(0)), # Protocol Count ('1.3.6.1.4.1.7054.71.2.22.0', rfc1902.Integer(0)), # Port Count )
def process(cls, widget, job, data): newdata = [] for row in data: newrow = [] for col in row: if isinstance(col, datetime.datetime): col = datetime_to_seconds(col) newrow.append(col) newdata.append(newrow) return newdata
def create_debug_zipfile(no_summary=False): """ Collects logfiles and system info into a zipfile for download/email `no_summary` indicates whether to include system information from the helper script `flyscript_about.py` as part of the zipped package. Default is to include the file. """ # setup correct timezone based on admin settings admin = User.objects.filter(is_superuser=True)[0] tz = pytz.timezone(admin.userprofile.timezone) current_tz = os.environ['TZ'] try: # save TZ to environment for zip to use os.environ['TZ'] = str(tz) # if zlib is available, then let's compress the files # otherwise we will just append them like a tarball try: import zlib compression = zipfile.ZIP_DEFLATED except ImportError: compression = zipfile.ZIP_STORED # setup the name, correct timezone, and open the zipfile now = datetime_to_seconds(datetime.now(tz)) archive_name = os.path.join(settings.PROJECT_ROOT, 'debug-%d.zip' % now) myzip = zipfile.ZipFile(archive_name, 'w', compression=compression) try: # find all of the usual logfiles filelist = glob.glob(os.path.join(settings.PROJECT_ROOT, 'log*')) logging.debug('zipping log files ...') for fname in filelist: debug_fileinfo(fname) myzip.write(fname) if not no_summary: logging.debug('running about script') response = '\n'.join(system_info()) logging.debug('zipping about script') myzip.writestr('system_summary.txt', response) finally: myzip.close() finally: # return env to its prior state os.environ['TZ'] = current_tz return archive_name
def handle(self, *args, **options): """ Main command handler """ self.options = options if options['table_list']: # print out the id's instead of processing anything tables = Table.objects.all() for t in tables: self.console('%5d - %s' % (t.id, t)) elif options['table_list_by_report']: # or print them out organized by report/widget/table output = [] reports = Report.objects.all() for report in reports: for widget in report.widget_set.all(): for table in widget.tables.all(): line = [table.id, report.title, widget.title, table] output.append(line) Formatter.print_table(output, ['ID', 'Report', 'Widget', 'Table']) else: if 'table_id' in options and options['table_id'] is not None: table = Table.objects.get(id=options['table_id']) elif 'table_name' in options: table = Table.objects.get(name=options['table_name']) else: raise ValueError("Must specify either --table-id or --table-name to run a table") # Django gives us a nice error if we can't find the table self.console('Table %s found.' % table) # Look for a related report widgets = Widget.objects.filter(tables__in=[table]) if len(widgets) > 0: report = widgets[0].report form = create_report_criteria_form(report=report) else: form = None add_options = {} if 'criteria' in options and options['criteria'] is not None: for s in options['criteria']: (k,v) = s.split(':', 1) add_options[k] = v if 'endtime' in options and options['endtime'] is not None: try: endtime = form.fields['endtime'].clean(options['endtime']) except ValidationError: raise ValidationError("Could not parse endtime: %s, try MM/DD/YYYY HH:MM am" % options['endtime']) tz = pytz.timezone("US/Eastern") endtime = endtime.replace(tzinfo=tz) else: endtime = datetime.datetime.now() criteria = Criteria(endtime=datetime_to_seconds(endtime), duration=options['duration'], filterexpr=options['filterexpr'], table=table, ignore_cache=options['ignore_cache']) if form: for k,field in form.fields.iteritems(): if not k.startswith('criteria_'): continue tc = TableCriteria.objects.get(pk=k.split('_')[1]) if ( options['criteria'] is not None and tc.keyword in add_options): val = add_options[tc.keyword] else: val = field.initial # handle table criteria and generate children objects tc = TableCriteria.get_instance(k, val) criteria[k] = tc for child in tc.children.all(): child.value = val criteria['criteria_%d' % child.id] = child columns = [c.name for c in table.get_columns()] if options['only_columns']: print columns return job = Job.create(table=table, criteria=criteria) job.save() self.console('Job created: %s' % job) self.console('Criteria: %s' % criteria.print_details()) start_time = datetime.datetime.now() job.start() self.console('Job running . . ', ending='') # wait for results while not job.done(): #self.console('. ', ending='') #self.stdout.flush() time.sleep(1) end_time = datetime.datetime.now() delta = end_time - start_time seconds = float(delta.microseconds + (delta.seconds + delta.days * 24 * 3600) * 10**6) / 10**6 self.console('Done!! (elapsed time: %.2f seconds)' % seconds) self.console('') # Need to refresh the column list in case the job changed them (ephemeral cols) columns = [c.name for c in table.get_columns()] if job.status == job.COMPLETE: if options['as_csv']: Formatter.print_csv(job.values(), columns) else: Formatter.print_table(job.values(), columns) else: self.console("Job completed with an error:") self.console(job.message) sys.exit(1)
def run( self, template_id, timefilter=None, resolution="auto", query=None, trafficexpr=None, data_filter=None, sync=True ): """Create the report on Profiler and begin running the report. If the `sync` option is True, periodically poll until the report is complete, otherwise return immediately. `template_id` is the numeric id of the template to use for the report `timefilter` is the range of time to query, a TimeFilter object `resolution` is the data resolution (1min, 15min, etc.), defaults to 'auto' `query` is the query object containing criteria `trafficexpr` is a TrafficFilter object `data_filter` is a deprecated filter to run against report data `sync` if True, poll for status until the report is complete """ self.template_id = template_id self.custom_columns = False if self.template_id != 184: # the columns in this report won't match, use custom columns instead self.custom_columns = True if timefilter is None: self.timefilter = TimeFilter.parse_range("last 5 min") else: self.timefilter = timefilter self.query = query self.trafficexpr = trafficexpr self.data_filter = data_filter self.id = None self.queries = list() self.last_status = None if resolution not in ["auto", "1min", "15min", "hour", "6hour", "day", "week", "month"]: rd = parse_timedelta(resolution) resolution = self.RESOLUTION_MAP[int(timedelta_total_seconds(rd))] self.resolution = resolution start = datetime_to_seconds(self.timefilter.start) end = datetime_to_seconds(self.timefilter.end) # using a RecursiveUpdateDict criteria = RecursiveUpdateDict(**{"time_frame": {"start": int(start), "end": int(end)}}) if self.query is not None: criteria["query"] = self.query if self.resolution != "auto": criteria["time_frame"]["resolution"] = self.resolution if self.data_filter: criteria["deprecated"] = {self.data_filter[0]: self.data_filter[1]} if self.trafficexpr is not None: criteria["traffic_expression"] = self.trafficexpr.filter to_post = {"template_id": self.template_id, "criteria": criteria} logger.debug("Posting JSON: %s" % to_post) response = self.profiler.api.report.reports(data=to_post) try: self.id = int(response["id"]) except KeyError: raise ValueError("failed to retrieve report id from report creation response: %s" % response) logger.info("Created report %d" % self.id) if sync: self.wait_for_complete()