def qps_parse_args():
usage = "%prog " \
"-r <datetime_regexp_mask> " \
"-F <timestamp_format_string> " \
"-W <sliding_window_interval_seconds>" \
"-i <ignore_missing_datefield_errors>"
parser = OptionParser(usage=usage)
parser.add_option("-r", "--re", dest="dt_re", default=None,
help="Regular expression to lookup datetime in logrow")
parser.add_option("-F", "--dateformat", dest="dateformat",
help="Format string for parsing date-time field (used with --datetime)")
parser.add_option("-W", '--window-size', dest="window_size", type=int, default=None,
help="Sliding window interval (in seconds)")
parser.add_option("-i", "--ignore", dest="ignore", default=None, action="store_true",
help="Ignore missing datefield errors (skip lines with missing/unparse-able datefield)")
parser.add_option("-P", "--profile", dest="profile", default='qps',
help="Configuration profile (section in configuration file)")
options, args = parser.parse_args()
# Interpolate from configuration and open filehandle
options.dt_re = interpolate_config(options.dt_re, options.profile, 're')
options.dateformat = interpolate_config(options.dateformat,
options.profile, 'dateformat', default=False)
options.window_size = interpolate_config(options.window_size,
options.profile, 'window_size', type=int)
options.ignore = interpolate_config(options.ignore, options.profile, 'ignore',
default=False, type=bool)
return AttrDict(options.__dict__), args
def sumstat_parse_args():
usage = "%prog -d <delimiter> [--reverse]"
parser = OptionParser(usage=usage)
parser.add_option(
"-r",
"--reverse",
dest="reverse",
action="store_true",
help=
"Reverse ordering of entries (toggle between increasing/decreasing sort order"
)
parser.add_option("-d",
"--delimiter",
dest="delimiter",
help="Delimiter character for field-separation")
parser.add_option(
"-P",
"--profile",
dest="profile",
default='qps',
help="Configuration profile (section in configuration file)")
options, args = parser.parse_args()
options.delimiter = interpolate_config(options.delimiter, options.profile,
'delimiter')
options.reverse = interpolate_config(options.reverse,
options.profile,
'reverse',
type=bool,
default=False)
return AttrDict(options.__dict__), args
def urlparse_parse_args():
usage = "%prog -p <url_part>"
parser = OptionParser(usage=usage)
parser.add_option(
"-p",
"--part",
dest="part",
default=None,
help=
"Part of URL to print out. Valid values: scheme, domain, netloc, path, query"
)
parser.add_option(
"-q",
"--query-params",
dest="query_params",
default=None,
help=
"Query parameters to print. Used in conjunction with '-p query'. Can specify multiple ones seperated by comma"
)
parser.add_option(
"-d",
"--decode",
dest="decode",
action="store_true",
help=
"Decode mode - Unquote input text, translating %xx characters and '+' into spaces"
)
parser.add_option(
"-P",
"--profile",
dest="profile",
default='qps',
help="Configuration profile (section in configuration file)")
options, args = parser.parse_args()
if not options.decode and not options.part:
parser.error(
"Must supply -p (part) when not working in decode (-d) mode. See --help for usage instructions."
)
# Interpolate from configuration and open filehandle
options.part = interpolate_config(options.part,
options.profile,
'part',
default=False)
options.query_params = interpolate_config(options.query_params,
options.profile,
'query_params',
default=False)
options.decode = interpolate_config(options.decode,
options.profile,
'decode',
default=False)
return AttrDict(options.__dict__), args
def logsample_parse_args():
parser = OptionParser()
parser.add_option("-n",
"--num-samples",
dest="num_samples",
type=int,
help="Number of samples to produce")
parser.add_option(
"-w",
"--weighted",
dest="weighted",
action="store_true",
help="Use Weighted Reservoir Sampling (needs -f and -d parameters)")
parser.add_option(
"-f",
"--field",
dest="field",
type=int,
help="Index of field to use as weight for weighted sampling (-w)")
parser.add_option(
"-d",
"--delimiter",
dest="delimiter",
help=
"Delimiter character for field-separation used by weighted sampling (-w)"
)
parser.add_option(
"-P",
"--profile",
dest="profile",
default='logsample',
help="Configuration profile (section in configuration file)")
options, args = parser.parse_args()
# Interpolate from configuration
options.num_samples = interpolate_config(options.num_samples,
options.profile,
'num_samples',
type=int)
options.weighted = interpolate_config(options.weighted,
options.profile,
'weighted',
type=bool,
default=False)
options.field = interpolate_config(options.field,
options.profile,
'field',
type=int,
default=False)
options.delimiter = interpolate_config(options.delimiter,
options.profile,
'delimiter',
default=' ')
return AttrDict(options.__dict__), args
def sumstat_parse_args():
usage = dedent("""
%prog -d <delimiter> [--reverse]
Generates summary statistics
for a given logfile of the form:
<count> <value>
logfile is expected to be pre-sorted by count.
""")
parser = OptionParser(usage=usage)
parser.add_option(
"-r",
"--reverse",
dest="reverse",
action="store_true",
help=
"Reverse ordering of entries (toggle between increasing/decreasing sort order"
) # noqa
parser.add_option("-d",
"--delimiter",
dest="delimiter",
help="Delimiter character for field-separation")
parser.add_option(
"-P",
"--profile",
dest="profile",
default='qps',
help="Configuration profile (section in configuration file)") # noqa
options, args = parser.parse_args()
options.delimiter = interpolate_config(options.delimiter,
options.profile,
'delimiter',
default=' ') # noqa
options.reverse = interpolate_config(options.reverse,
options.profile,
'reverse',
type=bool,
default=False) # noqa
return AttrDict(options.__dict__), args
def flattenjson_parse_args():
parser = OptionParser()
parser.add_option(
"-f",
"--field",
dest="field",
default=None,
help="JSON root field to extract objects from (should point to a list)"
) # noqa
parser.add_option(
"-P",
"--profile",
dest="profile",
default='flattenjson',
help="Configuration profile (section in configuration file)") # noqa
options, args = parser.parse_args()
options.field = interpolate_config(options.field, options.profile, 'field')
return AttrDict(options.__dict__), args
def geoip_parse_args():
parser = OptionParser()
parser.add_option("-r", "--re", dest="ip_re", default=None,
help="Regular expression to lookup IP in logrow")
parser.add_option("-f", "--filter", dest="filter", default=None,
help="Country/Area Code to filter to (e.g 'United States')")
parser.add_option("-p", "--print", dest="printline", default=None, action="store_true",
help="Print original log line with the geolocation. By default we only print <country, ip>")
parser.add_option("-P", "--profile", dest="profile", default='geoip',
help="Configuration profile (section in configuration file)")
options, args = parser.parse_args()
# Interpolate from configuration
options.ip_re = interpolate_config(options.ip_re, options.profile, 'ip_re')
options.filter = interpolate_config(options.filter, options.profile, 'filter',
default=False)
options.printline = interpolate_config(options.printline, options.profile, 'print',
type=bool, default=False)
return AttrDict(options.__dict__), args
def logjoin_parse_args():
usage = "%prog " \
"-f <field> " \
"-d <delimiter_character> " \
"-t <timestamp_format_string>"
parser = OptionParser(usage=usage)
parser.add_option("-f", "--field", dest="field", type=int,
help="Index of field to use as field to join on")
parser.add_option("-d", "--delimiter", dest="delimiter",
help="Delimiter character for field-separation")
parser.add_option("-b", "--backend", dest="backend",
help="Backend to use for joining. Currently available backends: 'sqlalchemy'")
parser.add_option("-C", "--join-connect-string", dest="join_connect_string",
help="Connection string (e.g sqlalchemy db URI)")
parser.add_option("-F", "--join-remote-fields", dest="join_remote_fields",
help="Fields to include from right join clause")
parser.add_option("-N", "--join-remote-name", dest="join_remote_name",
help="Name of resource to join to (e.g file name, table name)")
parser.add_option("-K", "--join-remote-key", dest="join_remote_key",
help="Name of remote key field to join on (e.g table field, file column index)")
parser.add_option("-P", "--profile", dest="profile", default='qps',
help="Configuration profile (section in configuration file)")
options, args = parser.parse_args()
# Interpolate from configuration
options.field = interpolate_config(options.field, options.profile, 'field', type=int)
options.delimiter = interpolate_config(options.delimiter, options.profile, 'delimiter', default=' ')
options.backend = interpolate_config(options.backend, options.profile, 'backend')
options.join_connect_string = interpolate_config(options.join_connect_string, options.profile, 'join_connect_string')
options.join_remote_fields = interpolate_config(options.join_remote_fields, options.profile, 'join_remote_fields')
options.join_remote_name = interpolate_config(options.join_remote_name, options.profile, 'join_remote_name')
options.join_remote_key = interpolate_config(options.join_remote_key, options.profile, 'join_remote_key')
return AttrDict(options.__dict__), args
def logplot_parse_args():
parser = OptionParser()
parser.add_option(
"-b",
"--backend",
dest="backend",
help=
"Backend to use for plotting. Currently available backends: 'gchart', 'matplotlib'"
)
parser.add_option("-T", "--type", dest="type",
help="Chart type. Available types: 'pie', 'histogram', 'line'." \
"Availability might differ due to backend.")
parser.add_option("-f",
"--field",
dest="field",
type=int,
help="Index of field to use as main input for plot")
parser.add_option("-d",
"--delimiter",
dest="delimiter",
help="Delimiter character for field-separation")
parser.add_option("-o", "--output", dest="output", help="Output filename")
parser.add_option("-W",
"--width",
dest="width",
type=int,
help="Plot Width")
parser.add_option("-H",
"--height",
dest="height",
type=int,
help="Plot Height")
parser.add_option(
"-L",
"--limit",
dest="limit",
type=int,
help="Only plot the top N rows, sorted decreasing by key")
parser.add_option("-l",
"--legend",
dest="legend",
action="store_true",
help="Render Plot Legend")
parser.add_option("-t", "--title", dest="title", help="Plot Title")
parser.add_option(
"--datefield",
dest="datefield",
type=int,
help="Index of field to use as date-time source (for timeseries plots)"
)
parser.add_option(
"--dateformat",
dest="dateformat",
help="Format string for parsing date-time field (for timeseries plots)"
)
parser.add_option(
"-P",
"--profile",
dest="profile",
default='logplot',
help="Configuration profile (section in configuration file)")
options, args = parser.parse_args()
# Interpolate from configuration
options.backend = interpolate_config(options.backend, options.profile,
'backend')
options.type = interpolate_config(options.type, options.profile, 'type')
options.field = interpolate_config(options.field,
options.profile,
'field',
type=int)
options.delimiter = interpolate_config(options.delimiter, options.profile,
'delimiter')
options.output = interpolate_config(options.output,
options.profile,
'output',
default=False)
options.width = interpolate_config(options.width,
options.profile,
'width',
type=int)
options.height = interpolate_config(options.height,
options.profile,
'height',
type=int)
options.limit = interpolate_config(options.limit,
options.profile,
'limit',
type=int,
default=False)
options.legend = interpolate_config(options.legend,
options.profile,
'legend',
type=bool,
default=False)
options.title = interpolate_config(options.title,
options.profile,
'title',
default=False)
options.datefield = interpolate_config(options.datefield,
options.profile,
'datefield',
type=int,
default=False)
options.dateformat = interpolate_config(options.dateformat,
options.profile,
'dateformat',
default=False)
return AttrDict(options.__dict__), args
def logfilter_parse_args():
usage = "%prog " \
"-b <blacklist_file> " \
"[--reverse]"
parser = OptionParser(usage=usage)
parser.add_option("-b",
"--blacklist",
dest="blacklist",
default=None,
help="Blacklist (whitelist when in --reverse mode) file")
parser.add_option("-I",
"--ignore-case",
dest="ignorecase",
action="store_true",
help="Ignore case when matching")
parser.add_option(
"-W",
"--word-boundaries",
dest="word_boundaries",
action="store_true",
help=
"Only match on word boundaries (e.g start/end of line and/or spaces)")
parser.add_option(
"-A",
'--with-acora',
dest='with_acora',
action="store_true",
help=
"Use Aho-Corasick multiple string pattern matching instead of regexps. Suitable for whole word matching"
)
parser.add_option("-r",
"--reverse",
dest="reverse",
action="store_true",
help="Reverse filtering")
parser.add_option("-p",
"--print",
dest="printlines",
action="store_true",
help="Print non-filtered lines")
parser.add_option("--parser", dest="parser",
help="Feed logs through a parser. Useful when reading encoded/escaped formats (e.g JSON) and when " \
"selecting parsed fields rather than matching via regular expression.")
parser.add_option(
"-d",
"--delimiter",
dest="delimiter",
help=
"Delimiter character for field-separation (when not using a --parser)")
parser.add_option("-f",
"--field",
dest="field",
help="Index of field to use for filtering against")
parser.add_option(
"-P",
"--profile",
dest="profile",
default='logfilter',
help="Configuration profile (section in configuration file)")
options, args = parser.parse_args()
# Interpolate from configuration and open filehandle
options.field = interpolate_config(options.field, options.profile, 'field')
options.delimiter = interpolate_config(options.delimiter,
options.profile,
'delimiter',
default=' ')
options.blacklist = open(
interpolate_config(options.blacklist, options.profile, 'blacklist'),
"r")
options.parser = interpolate_config(options.parser,
options.profile,
'parser',
default=False)
options.reverse = interpolate_config(options.reverse,
options.profile,
'reverse',
default=False,
type=bool)
options.ignorecase = interpolate_config(options.ignorecase,
options.profile,
'ignorecase',
default=False,
type=bool)
options.word_boundaries = interpolate_config(options.word_boundaries,
options.profile,
'word_boundaries',
default=False,
type=bool)
options.with_acora = interpolate_config(options.with_acora,
options.profile,
'with_acora',
default=False,
type=bool)
options.printlines = interpolate_config(options.printlines,
options.profile,
'print',
default=False,
type=bool)
if options.parser and not options.field:
parser.error(
"Must supply --field parameter when using parser-based matching.")
return AttrDict(options.__dict__), args
def filterbots_parse_args():
usage = "%prog " \
"-u <useragents_blacklist_file> " \
"-i <ips_blacklist_file> " \
"-r <ip_useragent_regexp>"
parser = OptionParser(usage=usage)
parser.add_option("-u",
"--bots-ua",
dest="bots_ua",
default=None,
help="Bots useragents blacklist file")
parser.add_option("-i",
"--bots-ips",
dest="bots_ips",
default=None,
help="Bots ips blacklist file")
parser.add_option("-r", "--ip-ua-re", dest="ip_ua_re", default=None,
help="Regular expression to match IP/useragent fields." \
"Should have an 'ip' and 'ua' named groups")
parser.add_option("-p",
"--print",
dest="printlines",
action="store_true",
help="Print non-filtered lines")
parser.add_option(
"-t",
"--pattern",
dest="pattern",
action="store_true",
help=
"Use pattern analysis to filter bots (See documentation for details)")
parser.add_option("-R",
"--reverse",
dest="reverse",
action="store_true",
help="Reverse filtering")
parser.add_option("--parser", dest="parser",
help="Feed logs through a parser. Useful when reading encoded/escaped formats (e.g JSON) and when " \
"selecting parsed fields rather than matching via regular expression.")
parser.add_option("-f", "--ip-ua-fields", dest="ip_ua_fields",
help="Field(s) Selector for filtering bots when using a parser (--parser). Format should be " \
" 'ua:<ua_field_name>,ip:<ip_field_name>'. If one of these is missing, it will not be used for filtering.")
parser.add_option(
"-P",
"--profile",
dest="profile",
default='filterbots',
help="Configuration profile (section in configuration file)")
options, args = parser.parse_args()
# Interpolate from configuration and open filehandle
options.bots_ua = open(
interpolate_config(options.bots_ua, options.profile, 'bots_ua'), "r")
options.bots_ips = open(
interpolate_config(options.bots_ips, options.profile, 'bots_ips'), "r")
options.ip_ua_re = interpolate_config(options.ip_ua_re,
options.profile,
'ip_ua_re',
default=False)
options.parser = interpolate_config(options.parser,
options.profile,
'parser',
default=False)
#options.format = interpolate_config(options.format, options.profile, 'format',
# default=False)
options.ip_ua_fields = interpolate_config(options.ip_ua_fields,
options.profile,
'ip_ua_fields',
default=False)
options.pattern = interpolate_config(options.pattern,
options.profile,
'pattern',
default=False,
type=bool)
options.reverse = interpolate_config(options.reverse,
options.profile,
'reverse',
default=False,
type=bool)
options.printlines = interpolate_config(options.printlines,
options.profile,
'print',
default=False,
type=bool)
if options.parser and not options.ip_ua_fields:
parser.error(
"Must supply --ip-ua-fields parameter when using parser-based matching."
)
return AttrDict(options.__dict__), args
def logtail_parse_args():
usage = "%prog " \
"--date-format <date_format>" \
"--start-date <start_date> "
parser = OptionParser(usage=usage)
parser.add_option(
"--date-format",
dest="date_format",
help="Date format (Using date utility notation, e.g '%Y-%m-%d')")
parser.add_option("--start-date",
dest="start_date",
help="Start date expression (e.g '120 minutes ago')")
parser.add_option("--parser", dest="parser",
help="Feed logs through a parser. Useful when reading encoded/escaped formats (e.g JSON) and when " \
"selecting parsed fields rather than matching via regular expression.")
parser.add_option(
"-d",
"--delimiter",
dest="delimiter",
help=
"Delimiter character for field-separation (when not using a --parser)")
parser.add_option("-f",
"--field",
dest="field",
help="Index of field to use for filtering against")
parser.add_option("-p",
"--print",
dest="printlines",
action="store_true",
help="Print non-filtered lines")
parser.add_option(
"-P",
"--profile",
dest="profile",
default='logtail',
help="Configuration profile (section in configuration file)")
options, args = parser.parse_args()
# Interpolate from configuration and open filehandle
options.date_format = interpolate_config(options.date_format,
options.profile, 'date_format')
options.start_date = interpolate_config(options.start_date,
options.profile, 'start_date')
options.field = interpolate_config(options.field, options.profile, 'field')
options.delimiter = interpolate_config(options.delimiter,
options.profile,
'delimiter',
default=' ')
options.parser = interpolate_config(options.parser,
options.profile,
'parser',
default=False)
options.printlines = interpolate_config(options.printlines,
options.profile,
'print',
default=False,
type=bool)
if options.parser and not options.field:
parser.error(
"Must supply --field parameter when using parser-based matching.")
return AttrDict(options.__dict__), args
def logmerge_parse_args():
usage = "%prog -f <field> -d <delimiter> filename1 filename2 ..."
parser = OptionParser(usage=usage)
parser.add_option(
"-f",
"--field",
dest="field",
default=None,
help="Field index to use as key for sorting by (1-based)")
parser.add_option("-d",
"--delimiter",
dest="delimiter",
default=None,
help="Delimiter character for fields in logfile")
parser.add_option(
"-n",
"--numeric",
dest="numeric",
default=None,
action="store_true",
help="Parse key field value as numeric and sort accordingly")
parser.add_option(
"-t",
"--datetime",
dest="datetime",
default=None,
action="store_true",
help=
"Parse key field value as a date/time timestamp and sort accordingly")
parser.add_option(
"-F",
"--dateformat",
dest="dateformat",
help="Format string for parsing date-time field (used with --datetime)"
)
parser.add_option(
"-p",
"--parser",
dest="parser",
default=None,
help=
"Log format parser (e.g 'CommonLogFormat'). See documentation for available parsers."
)
parser.add_option(
"-P",
"--profile",
dest="profile",
default='logmerge',
help="Configuration profile (section in configuration file)")
options, args = parser.parse_args()
# Interpolate from configuration
options.field = interpolate_config(options.field, options.profile, 'field')
options.delimiter = interpolate_config(options.delimiter,
options.profile,
'delimiter',
default=' ')
options.numeric = interpolate_config(options.numeric,
options.profile,
'numeric',
default=False,
type=bool)
options.datetime = interpolate_config(options.datetime,
options.profile,
'datetime',
default=False,
type=bool)
options.dateformat = interpolate_config(options.dateformat,
options.profile,
'dateformat',
default=False)
options.parser = interpolate_config(options.parser,
options.profile,
'parser',
default=False)
return AttrDict(options.__dict__), args
def logparse_parse_args():
parser = OptionParser()
parser.add_option(
"-p",
"--parser",
dest="parser",
default=None,
help=
"Log format parser (e.g 'CommonLogFormat'). See documentation for available parsers."
)
parser.add_option(
"-F",
"--format",
dest="format",
default=None,
help=
"Format string. Used by the parser (e.g AccessLog format specifier)")
parser.add_option("-f",
"--field",
dest="field",
default=None,
help="Parsed Field index to output")
parser.add_option(
"-i",
"--ignore",
dest="ignore",
default=None,
action="store_true",
help="Ignore missing fields errors (skip lines with missing fields)")
parser.add_option(
"-H",
"--header",
dest="header",
default=None,
action="store_true",
help="Prepend a header describing the selected fields to output.")
parser.add_option(
"-P",
"--profile",
dest="profile",
default='logparse',
help="Configuration profile (section in configuration file)")
options, args = parser.parse_args()
# Interpolate from configuration
options.parser = interpolate_config(options.parser, options.profile,
'parser')
options.format = interpolate_config(options.format,
options.profile,
'format',
default=False)
options.field = interpolate_config(options.field, options.profile, 'field')
options.ignore = interpolate_config(options.ignore,
options.profile,
'ignore',
default=False,
type=bool)
options.header = interpolate_config(options.header,
options.profile,
'header',
default=False,
type=bool)
return AttrDict(options.__dict__), args