Esempio n. 1
0
    def print_columns(self, paginate=None):
        """ Print out data in a nice formatted table

        `paginate` option will insert a new header after that many
        rows have been printed.  Defaults to None (only single header
        output).
        """
        if not self.data:
            print "No data found."
            return

        if self.options.typelist:
            # two columns only
            headers = ['type_id', 'type']
            data = self.data
        elif self.options.ipaddr:
            # single dict
            headers = self.data.keys()
            data = [self.data.values()]
        else:
            # assume objects are uniform and take keys from first one
            headers = self.data[0].keys()
            data = [d.values() for d in self.data]

        Formatter.print_table(data, headers, paginate=30)
Esempio n. 2
0
    def main(self):
        if self.options.list_groupbys:
            header =  ["GroupBy", "Id"]
            data = [(k, v) for k,v in self.profiler.groupbys.iteritems()]
            data.sort()
            Formatter.print_table(data, header)
        else:
            if self.options.ids:
                columns = self.profiler.get_columns_by_ids(self.options.ids)
            else:
                o = self.options

                # find groupby looking in keys and values
                if o.groupby in self.profiler.groupbys:
                    groupby = self.profiler.groupbys[o.groupby]
                elif o.groupby in self.profiler.groupbys.values():
                    groupby = o.groupby
                else:
                    groupby = None

                args = {
                    'realms': [o.realm] if o.realm else None,
                    'centricities': [o.centricity] if o.centricity else None,
                    'groupbys': [groupby] if groupby else None,
                    }
                columns = self.profiler.search_columns(**args)

            columns.sort(key=lambda x: x.key)
            self.print_columns(columns)
Esempio n. 3
0
    def handle(self, *args, **options):
        """ Main command handler. """

        if options['job_list']:
            # print out the id's instead of processing anything
            columns = ['ID', 'Table', 'Created', 'Touched', 'Sts', 'Refs',
                       'Progress', 'Data file']
            data = []
            for j in Job.objects.all():
                datafile = j.datafile()
                if not os.path.exists(datafile):
                    datafile += " (missing)"
                data.append([j.id, j.table.name, j.created, j.touched,
                             j.status, j.refcount, j.progress, datafile])

            Formatter.print_table(data, columns)

        elif options['job_data']:
            job = Job.objects.get(id=options['job_data'])

            columns = [c.name for c in job.table.get_columns()]

            if job.status == job.COMPLETE:
                Formatter.print_table(job.values(), columns)

        elif options['job_age']:
            Job.age_jobs(force=True)
                
        elif options['job_flush']:
            Job.objects.all().delete()
Esempio n. 4
0
    def main(self):
        """ Setup query and run report with default column set
        """
        if self.options.timerange:
            timefilter = TimeFilter.parse_range(self.options.timerange)
        else:
            timefilter = TimeFilter(self.options.time0, self.options.time1)
        trafficexpr = TrafficFilter(self.options.trafficexpr)

        columns = [self.profiler.columns.key.srv_host_ip,
                   self.profiler.columns.key.app_info,
                   self.profiler.columns.key.start_time,
                   self.profiler.columns.key.end_time,
                   self.profiler.columns.value.s2c_total_bytes,
                   self.profiler.columns.value.s2c_total_pkts,
                   self.profiler.columns.value.response_time,
                   self.profiler.columns.value.server_delay]

        report = TrafficFlowListReport(self.profiler)
        report.run(columns, timefilter=timefilter, trafficexpr=trafficexpr)
        data = report.get_data()
        report.delete()

        headers = [c.key for c in columns]

        Formatter.print_table(data, headers)
Esempio n. 5
0
    def handle(self, *args, **options):
        """ Main command handler
        """
        self.options = options

        if options['job_list']:
            # print out the id's instead of processing anything
            columns = ['ID', 'Table', 'Created', 'Touched', 'Sts', 'Refs', 'Progress', 'Data file']
            data = []
            for j in Job.objects.all():
                datafile = j.datafile()
                if not os.path.exists(datafile):
                    datafile += " (missing)"
                data.append ([j.id, j.table.name, j.created, j.touched,
                              j.status, j.refcount, j.progress, datafile])

            Formatter.print_table(data, columns)

        elif options['job_data']:
            job = Job.objects.get(id=options['job_data'])

            columns = [c.name for c in job.table.get_columns()]

            if job.status == job.COMPLETE:
                Formatter.print_table(job.values(), columns)

        elif options['job_age']:
            Job.age_jobs(force=True)
                
        elif options['job_flush']:
            Job.objects.all().delete()
Esempio n. 6
0
 def main(self):
     headers = ['ID', 'Description', 'Type']
     data = [(f.id, f.description, f.type) for f in self.shark.get_extractor_fields()]
     if self.options.sort_id:
         data.sort()
     Formatter.print_table(data, 
                           headers,
                           padding=2,
                           max_width=int(self.options.table_width),
                           long_column=1,
                           wrap_columns=(not self.options.truncate))
Esempio n. 7
0
    def print_columns(self, columns):
        keys = []
        values = []
        for c in columns:
            if (self.options.filter and
                self.options.filter.lower() not in c.label.lower()):
                continue

            item = (c.key, c.label, c.id)
            if c.iskey:
                keys.append(item)
            else:
                values.append(item)

        Formatter.print_table(keys, ['Key Columns', 'Label', 'ID'])
        print ''
        Formatter.print_table(values, ['Value Columns', 'Label', 'ID'])
Esempio n. 8
0
    def print_columns(self, columns):
        keys = []
        values = []
        for c in columns:
            if (self.options.filter
                    and self.options.filter.lower() not in c.label.lower()):
                continue

            item = (c.key, c.label, c.id)
            if c.iskey:
                keys.append(item)
            else:
                values.append(item)

        Formatter.print_table(keys, ['Key Columns', 'Label', 'ID'])
        print ''
        Formatter.print_table(values, ['Value Columns', 'Label', 'ID'])
    def main(self):
        if self.options.list_groupbys:
            header =  ["GroupBy", "Id"]
            data = [(k, v) for k,v in self.profiler.groupbys.iteritems()]
            Formatter.print_table(data, header)
            
        if self.options.list_columns:
            if self.options.ids:
                columns = self.profiler.get_columns_by_ids(self.options.ids)
            else:
                o = self.options
                args = {
                    'realms': [o.realm] if o.realm else None,
                    'centricities': [o.centricity] if o.centricity else None,
                    'groupbys': [self.profiler.groupbys[o.groupby]] if o.groupby else None,
                    }
                columns = self.profiler.search_columns(**args)

            columns.sort(key=lambda x: x.key)
            self.print_columns(columns)
Esempio n. 10
0
    def main(self):
        if self.options.list_groupbys:
            header = ["GroupBy", "Id"]
            data = [(k, v) for k, v in self.profiler.groupbys.iteritems()]
            Formatter.print_table(data, header)

        if self.options.list_columns:
            if self.options.ids:
                columns = self.profiler.get_columns_by_ids(self.options.ids)
            else:
                o = self.options
                args = {
                    'realms': [o.realm] if o.realm else None,
                    'centricities': [o.centricity] if o.centricity else None,
                    'groupbys':
                    [self.profiler.groupbys[o.groupby]] if o.groupby else None,
                }
                columns = self.profiler.search_columns(**args)

            columns.sort(key=lambda x: x.key)
            self.print_columns(columns)
Esempio n. 11
0
    def main(self):
        """ Setup query and run report with default column set
        """
        if self.options.timerange:
            timefilter = TimeFilter.parse_range(self.options.timerange)
        else:
            timefilter = TimeFilter(self.options.time0, self.options.time1)

        if self.options.trafficexpr:
            trafficexpr = TrafficFilter(self.options.trafficexpr)
        else:
            trafficexpr = None

        legend_columns, all_data = self.identity_report(timefilter=timefilter,
                                                        trafficexpr=trafficexpr,
                                                        testfile=self.options.testfile)

        legend, activity = self.analyze_login_data(all_data, legend_columns)

        if self.options.timeseries_report:
            headers, tbl_data = self.generate_traffic(activity, legend, 'timeseries')
        elif self.options.summary_report:
            headers, tbl_data = self.generate_traffic(activity, legend, 'summary')
        else:
            headers = ('Host IP', 'Login Time', 'Logout Time', 'Duration')
            tbl_data = [(x[0], format_time(x[1]), format_time(x[2]), x[3]) 
                                                                for x in activity]

        if self.options.csv:
            Formatter.print_csv(tbl_data, headers)
        elif self.options.tsv:
            Formatter.print_csv(tbl_data, headers, delim='\t')
        else:
            Formatter.print_table(tbl_data, headers)
Esempio n. 12
0
    def handle(self, *args, **options):
        """ Main command handler. """

        if options['job_list']:
            # print out the id's instead of processing anything
            columns = ['ID', 'PID', 'Table', 'Created', 'Touched', 'Sts',
                       'Refs', 'Progress', 'Data file']
            data = []
            for j in Job.objects.all().order_by('id'):
                datafile = j.datafile()
                if not os.path.exists(datafile):
                    datafile += " (missing)"
                parent_id = j.parent.id if j.parent else '--'
                data.append([j.id, parent_id, j.table.name, j.created,
                             j.touched, j.status, j.refcount, j.progress,
                             datafile])

            Formatter.print_table(data, columns)

        elif options['job_data']:
            job = Job.objects.get(id=options['job_data'])

            columns = [c.name for c in job.table.get_columns()]

            if job.status == job.COMPLETE:
                Formatter.print_table(job.values(), columns)

        elif options['job_age']:
            logger.debug('Aging all jobs.')
            Job.age_jobs(force=True)

        elif options['job_flush']:
            logger.debug('Flushing all jobs.')
            while Job.objects.count():
                ids = Job.objects.values_list('pk', flat=True)[:100]
                Job.objects.filter(pk__in=ids).delete()
Esempio n. 13
0
    def handle(self, *args, **options):
        """ Main command handler
        """
        self.options = options

        if options['table_list']:
            # print out the id's instead of processing anything
            tables = Table.objects.all()
            for t in tables:
                self.console('%5d - %s' % (t.id, t))
        elif options['table_list_by_report']:
            # or print them out organized by report/widget/table
            output = []
            reports = Report.objects.all()
            for report in reports:
                for section in report.section_set.all():
                    for widget in section.widget_set.all():
                        for table in widget.tables.all():
                            line = [table.id, report.title, widget.title, table]
                            output.append(line)
            Formatter.print_table(output, ['ID', 'Report', 'Widget', 'Table'])
        elif options['criteria_list']:
            if 'table_id' in options and options['table_id'] is not None:
                table = Table.objects.get(id=options['table_id'])
            elif 'table_name' in options:
                table = Table.objects.get(name=options['table_name'])
            else:
                raise ValueError("Must specify either --table-id or "
                                 "--table-name to run a table")

            form = self.get_form(table)

            output = [[c.keyword, c.label] for c in form._tablefields.values()]
            Formatter.print_table(output, ['Keyword', 'Label'])
        else:
            if 'table_id' in options and options['table_id'] is not None:
                table = Table.objects.get(id=options['table_id'])
            elif 'table_name' in options:
                table = Table.objects.get(name=options['table_name'])
            else:
                raise ValueError("Must specify either --table-id or "
                                 "--table-name to run a table")
                
            # Django gives us a nice error if we can't find the table
            self.console('Table %s found.' % table)

            # Parse criteria options
            criteria_options = {}
            if 'criteria' in options and options['criteria'] is not None:
                for s in options['criteria']:
                    (k, v) = s.split(':', 1)
                    criteria_options[k] = v

            form = self.get_form(table, data=criteria_options)

            if not form.is_valid(check_unknown=True):
                self.console('Invalid criteria:')
                logger.error('Invalid criteria: %s' %
                             ','.join('%s:%s' % (k, v)
                                      for k, v in form.errors.iteritems()))
                for k, v in form.errors.iteritems():
                    self.console('  %s: %s' % (k, ','.join(v)))
                
                sys.exit(1)

            criteria = form.criteria()

            columns = [c.name for c in table.get_columns()]

            if options['only_columns']:
                print columns
                return

            job = Job.create(table=table, criteria=criteria)
            job.save()

            self.console('Job created: %s' % job)
            self.console('Criteria: %s' % criteria.print_details())

            start_time = datetime.datetime.now()
            job.start()
            self.console('Job running . . ', ending='')

            # wait for results
            while not job.done():
                #self.console('. ', ending='')
                #self.stdout.flush()
                time.sleep(1)

            end_time = datetime.datetime.now()
            delta = end_time - start_time
            seconds = float(delta.microseconds + 
                            (delta.seconds + delta.days * 24 * 3600) * 10**6) / 10**6

            self.console('Done!! (elapsed time: %.2f seconds)' % seconds)
            self.console('')

            # Need to refresh the column list in case the job changed them
            # (ephemeral cols)
            columns = [c.name for c in table.get_columns()]

            if job.status == job.COMPLETE:
                if options['as_csv']:
                    if options['output_file']:
                        with open(options['output_file'], 'w') as f:
                            for line in Formatter.get_csv(job.values(), columns):
                                f.write(line)
                                f.write('\n')
                    else:
                        Formatter.print_csv(job.values(), columns)
                else:
                    Formatter.print_table(job.values(), columns)
            else:
                self.console("Job completed with an error:")
                self.console(job.message)
                sys.exit(1)
Esempio n. 14
0
 def print_data(self, data, header):
     if self.options.as_csv:
         Formatter.print_csv(data, header)
     else:
         Formatter.print_table(data, header)
Esempio n. 15
0
    def handle(self, *args, **options):
        """ Main command handler
        """
        self.options = options

        if options['table_list']:
            # print out the id's instead of processing anything
            tables = Table.objects.all()
            for t in tables:
                self.console('%5d - %s' % (t.id, t))
        elif options['table_list_by_report']:
            # or print them out organized by report/widget/table
            output = []
            reports = Report.objects.all()
            for report in reports:
                for section in report.section_set.all():
                    for widget in section.widget_set.all():
                        for table in widget.tables.all():
                            line = [
                                table.id, report.title, widget.title, table
                            ]
                            output.append(line)
            Formatter.print_table(output, ['ID', 'Report', 'Widget', 'Table'])
        elif options['criteria_list']:
            if 'table_id' in options and options['table_id'] is not None:
                table = Table.objects.get(id=options['table_id'])
            elif 'table_name' in options:
                table = Table.objects.get(name=options['table_name'])
            else:
                raise ValueError("Must specify either --table-id or "
                                 "--table-name to run a table")

            form = self.get_form(table)

            output = [[c.keyword, c.label] for c in form._tablefields.values()]
            Formatter.print_table(output, ['Keyword', 'Label'])
        else:
            if 'table_id' in options and options['table_id'] is not None:
                table = Table.objects.get(id=options['table_id'])
            elif 'table_name' in options:
                table = Table.objects.get(name=options['table_name'])
            else:
                raise ValueError("Must specify either --table-id or "
                                 "--table-name to run a table")

            # Django gives us a nice error if we can't find the table
            self.console('Table %s found.' % table)

            # Parse criteria options
            criteria_options = {}
            if 'criteria' in options and options['criteria'] is not None:
                for s in options['criteria']:
                    (k, v) = s.split(':', 1)
                    criteria_options[k] = v

            form = self.get_form(table, data=criteria_options)

            if not form.is_valid(check_unknown=True):
                self.console('Invalid criteria:')
                logger.error('Invalid criteria: %s' %
                             ','.join('%s:%s' % (k, v)
                                      for k, v in form.errors.iteritems()))
                for k, v in form.errors.iteritems():
                    self.console('  %s: %s' % (k, ','.join(v)))

                sys.exit(1)

            criteria = form.criteria()

            columns = [c.name for c in table.get_columns()]

            if options['only_columns']:
                print columns
                return

            job = Job.create(table=table, criteria=criteria)
            job.save()

            self.console('Job created: %s' % job)
            self.console('Criteria: %s' % criteria.print_details())

            start_time = datetime.datetime.now()
            job.start()
            self.console('Job running . . ', ending='')

            # wait for results
            while not job.done():
                #self.console('. ', ending='')
                #self.stdout.flush()
                time.sleep(1)

            end_time = datetime.datetime.now()
            delta = end_time - start_time
            seconds = float(delta.microseconds +
                            (delta.seconds + delta.days * 24 * 3600) *
                            10**6) / 10**6

            self.console('Done!! (elapsed time: %.2f seconds)' % seconds)
            self.console('')

            # Need to refresh the column list in case the job changed them
            # (ephemeral cols)
            columns = [c.name for c in table.get_columns()]

            if job.status == job.COMPLETE:
                if options['as_csv']:
                    if options['output_file']:
                        with open(options['output_file'], 'w') as f:
                            for line in Formatter.get_csv(
                                    job.values(), columns):
                                f.write(line)
                                f.write('\n')
                    else:
                        Formatter.print_csv(job.values(), columns)
                else:
                    Formatter.print_table(job.values(), columns)
            else:
                self.console("Job completed with an error:")
                self.console(job.message)
                sys.exit(1)
Esempio n. 16
0
    def handle(self, *args, **options):
        """ Main command handler
        """
        self.options = options

        if options['table_list']:
            # print out the id's instead of processing anything
            tables = Table.objects.all()
            for t in tables:
                self.console('%5d - %s' % (t.id, t))
        elif options['table_list_by_report']:
            # or print them out organized by report/widget/table
            output = []
            reports = Report.objects.all()
            for report in reports:
                for widget in report.widget_set.all():
                    for table in widget.tables.all():
                        line = [table.id, report.title, widget.title, table]
                        output.append(line)
            Formatter.print_table(output, ['ID', 'Report', 'Widget', 'Table'])
        else:
            if 'table_id' in options and options['table_id'] is not None:
                table = Table.objects.get(id=options['table_id'])
            elif 'table_name' in options:
                table = Table.objects.get(name=options['table_name'])
            else:
                raise ValueError("Must specify either --table-id or --table-name to run a table")
                
            # Django gives us a nice error if we can't find the table
            self.console('Table %s found.' % table)

            # Look for a related report
            widgets = Widget.objects.filter(tables__in=[table])
            if len(widgets) > 0:
                report = widgets[0].report
                form = create_report_criteria_form(report=report)
            else:
                form = None

            add_options = {}
            if 'criteria' in options and options['criteria'] is not None:
                for s in options['criteria']:
                    (k,v) = s.split(':', 1)
                    add_options[k] = v

            if 'endtime' in options and options['endtime'] is not None:
                try:
                    endtime = form.fields['endtime'].clean(options['endtime'])
                except ValidationError:
                    raise ValidationError("Could not parse endtime: %s, try MM/DD/YYYY HH:MM am" % options['endtime'])
                tz = pytz.timezone("US/Eastern")
                endtime = endtime.replace(tzinfo=tz)
            else:
                endtime = datetime.datetime.now()

            criteria = Criteria(endtime=datetime_to_seconds(endtime),
                                duration=options['duration'],
                                filterexpr=options['filterexpr'],
                                table=table,
                                ignore_cache=options['ignore_cache'])


            if form:
                for k,field in form.fields.iteritems():
                    if not k.startswith('criteria_'): continue

                    tc = TableCriteria.objects.get(pk=k.split('_')[1])

                    if (  options['criteria'] is not None and
                          tc.keyword in add_options):
                        val = add_options[tc.keyword]
                    else:
                        val = field.initial

                    # handle table criteria and generate children objects
                    tc = TableCriteria.get_instance(k, val) 
                    criteria[k] = tc
                    for child in tc.children.all():
                        child.value = val
                        criteria['criteria_%d' % child.id] = child

            columns = [c.name for c in table.get_columns()]

            if options['only_columns']:
                print columns
                return

            job = Job.create(table=table, criteria=criteria)
            job.save()

            self.console('Job created: %s' % job)
            self.console('Criteria: %s' % criteria.print_details())

            start_time = datetime.datetime.now()
            job.start()
            self.console('Job running . . ', ending='')

            # wait for results
            while not job.done():
                #self.console('. ', ending='')
                #self.stdout.flush()
                time.sleep(1)

            end_time = datetime.datetime.now()
            delta = end_time - start_time
            seconds = float(delta.microseconds + 
                            (delta.seconds + delta.days * 24 * 3600) * 10**6) / 10**6

            self.console('Done!! (elapsed time: %.2f seconds)' % seconds)
            self.console('')

            # Need to refresh the column list in case the job changed them (ephemeral cols)
            columns = [c.name for c in table.get_columns()]

            if job.status == job.COMPLETE:
                if options['as_csv']:
                    Formatter.print_csv(job.values(), columns)
                else:
                    Formatter.print_table(job.values(), columns)
            else:
                self.console("Job completed with an error:")
                self.console(job.message)
                sys.exit(1)
Esempio n. 17
0
    def handle(self, *args, **options):
        """ Main command handler
        """
        self.options = options

        if options['table_list']:
            # print out the id's instead of processing anything
            tables = Table.objects.all()
            for t in tables:
                self.console('%5d - %s' % (t.id, t))
        elif options['table_list_by_report']:
            # or print them out organized by report/widget/table
            output = []
            reports = Report.objects.all()
            for report in reports:
                for widget in report.widget_set.all():
                    for table in widget.tables.all():
                        line = [table.id, report.title, widget.title, table]
                        output.append(line)
            Formatter.print_table(output, ['ID', 'Report', 'Widget', 'Table'])
        else:
            table = Table.objects.get(id=options['table_id'])
            # Django gives us a nice error if we can't find the table
            self.console('Table %s found.' % table)

            criteria = Criteria(endtime=options['endtime'],
                                duration=options['duration'],
                                filterexpr=options['filterexpr'],
                                table=table,
                                ignore_cache=options['ignore_cache'])

            job = Job(table=table, criteria=criteria)
            job.save()

            try:
                self.console('Job created: %s' % job)
                self.console('Criteria: %s' % criteria.print_details())

                start_time = datetime.datetime.now()
                job.start()
                self.console('Job running . . ', ending='')

                # wait for results
                while not job.done():
                    self.console('. ', ending='')
                    self.stdout.flush()
                    time.sleep(1)

                end_time = datetime.datetime.now()
                delta = end_time - start_time
                self.console('Done!! (took roughly %.2f seconds)' % delta.total_seconds())
                self.console('')

                columns = [c.name for c in table.get_columns()]
                if not options['csv']:
                    Formatter.print_table(job.data(), columns)
                else:
                    Formatter.print_csv(job.data(), columns)

            finally:
                job.delete()