def processArguments():
'''init process arguments '''
parser = OptionParser(usage="Usage: deepin-screenshot [options] [arg]", version="deepin-screenshot v2.1")
parser.add_option("-f", "--full", action="store_true", dest="fullscreen", help=_("Take a screenshot of full screen"))
parser.add_option("-w", "--window", action="store_true", dest="window", help=_("Take a screenshot of the window"))
parser.add_option("-d", "--delay", dest="delay", type="int", help=_("Take a screenshot after NUM seconds"), metavar="NUM")
parser.add_option("-s", "--save", dest="save_file", help=_("save screenshot to FILE"), metavar="FILE")
parser.add_option("--sub", action="store_true", dest="sub", help=_("run as a subprocess"))
parser.add_option("-n", "--new", action="store_true", dest="new", help=_("run a new process"))
parser.add_option("-I", "--icon", action="store_true", dest="icon")
#parser.add_option("-a", "--area", help="Grab an area of the screen instead of the entire screen", action="store_true")
#parser.add_option("-e", "--border-effect", action="store_true", dest="border_effect", help="Effect to add to the border")
#parser.add_option("-i", "--interactive", action="store_true", help="Interactively set options")
#parser.add_option("-b", "--include-border", action="store_true", help="Include the window border with the screenshot")
#parser.add_option("-B", "--remove-border", action="store_true", help="Remove the window border from the screenshot")
#parser.add_option("-c", "--clipboard", help="Send the grab directly to the clipboard", action="store_true")
parser.get_option('-h').help = _("show this help message and exit")
parser.get_option('--version').help = _("show program's version number and exit")
import sys
if '-h' in sys.argv or '--help' in sys.argv:
parser.remove_option("-I")
(options, args) = parser.parse_args()
if not options.new and IS_EXISTS:
print "deepint-screenshot has run"
exit(1)
if options.fullscreen and options.window:
parser.error("options -f and -w are mutually exclusive")
config.OPTION_ICON = options.icon
config.OPTION_FULLSCREEN = options.fullscreen
config.OPTION_WINDOWN = options.window
config.OPTION_NEW = options.new
config.OPTION_FILE = options.save_file
config.OPTION_SUB = options.sub
if options.delay:
notify("Deepin Screenshot", 0, summary=_("DScreenshot"),
body=_("Deepin Screenshot will start in %d seconds.") % options.delay, timeout=(options.delay-0.5)*1000)
loop = gobject.MainLoop()
gobject.timeout_add_seconds(options.delay, loop.quit)
loop.run()
if options.save_file:
parserFile = parser_path(str(options.save_file))
if options.fullscreen:
pixbuf = get_screenshot_pixbuf(True)
pixbuf.save(parserFile[0], parserFile[1])
elif options.window:
pixbuf = get_screenshot_pixbuf(False)
pixbuf.save(parserFile[0], parserFile[1])
else:
main()
elif options.fullscreen:
open_file_dialog()
elif options.window:
open_file_dialog(False)
else:
main()
class TestOptionParser(BaseTest):
def setUp(self):
self.parser = OptionParser()
self.parser.add_option("-v", "--verbose", "-n", "--noisy",
action="store_true", dest="verbose")
self.parser.add_option("-q", "--quiet", "--silent",
action="store_false", dest="verbose")
def test_add_option_no_Option(self):
self.assertTypeError(self.parser.add_option,
"not an Option instance: None", None)
def test_add_option_invalid_arguments(self):
self.assertTypeError(self.parser.add_option,
"invalid arguments", None, None)
def test_get_option(self):
opt1 = self.parser.get_option("-v")
self.assert_(isinstance(opt1, Option))
self.assertEqual(opt1._short_opts, ["-v", "-n"])
self.assertEqual(opt1._long_opts, ["--verbose", "--noisy"])
self.assertEqual(opt1.action, "store_true")
self.assertEqual(opt1.dest, "verbose")
def test_get_option_equals(self):
opt1 = self.parser.get_option("-v")
opt2 = self.parser.get_option("--verbose")
opt3 = self.parser.get_option("-n")
opt4 = self.parser.get_option("--noisy")
self.assert_(opt1 is opt2 is opt3 is opt4)
def test_has_option(self):
self.assert_(self.parser.has_option("-v"))
self.assert_(self.parser.has_option("--verbose"))
def assert_removed(self):
self.assert_(self.parser.get_option("-v") is None)
self.assert_(self.parser.get_option("--verbose") is None)
self.assert_(self.parser.get_option("-n") is None)
self.assert_(self.parser.get_option("--noisy") is None)
self.failIf(self.parser.has_option("-v"))
self.failIf(self.parser.has_option("--verbose"))
self.failIf(self.parser.has_option("-n"))
self.failIf(self.parser.has_option("--noisy"))
self.assert_(self.parser.has_option("-q"))
self.assert_(self.parser.has_option("--silent"))
def test_remove_short_opt(self):
self.parser.remove_option("-n")
self.assert_removed()
def test_remove_long_opt(self):
self.parser.remove_option("--verbose")
self.assert_removed()
def test_remove_nonexistent(self):
self.assertRaises(self.parser.remove_option, ValueError,
"no such option 'foo'", funcargs=['foo'])
def main():
kinds = ('Exercise',
'Video',
'Playlist',
'ExerciseVideo',
'ExercisePlaylist',
'VideoPlaylist')
parser = OptionParser(usage="%prog [options] upload|download",
description="Uploads the sample data to a server or downloads it from the server.")
parser.add_option("-U", "--url", default="http://*****:*****@example.com",
help="The username to use.")
parser.add_option("-k", "--kinds", default=','.join(kinds),
help="The comma separated list of kinds.")
parser.add_option("-p", "--python", default=(sys.executable if platform.system() == "Windows" else None), help="Path of python executable.")
parser.add_option("-a", "--appcfg", default='appcfg.py', help="Path of appcfg.py (Google App Engine).")
(options, args) = parser.parse_args()
if len(args) < 1:
parser.print_help()
return
for kind in options.kinds.split(','):
filename='%s.data' % kind
call_args = [options.appcfg,
'--url=%s' % options.url,
'--email=%s' % options.email,
'--application=khan-staging',
'--kind=%s' % kind,
'--filename=%s' % filename]
if options.email == parser.get_option('--email').default:
call_args.append('--passin')
if options.python is not None:
call_args.insert(0, options.python)
if args[0] == 'upload':
call_args.append('upload_data')
elif args[0] == 'download':
if os.path.exists(filename):
os.remove(filename)
call_args.append('download_data')
else:
parser.print_help()
return
print ' '.join(call_args)
if options.email == parser.get_option('--email').default:
process = subprocess.Popen(call_args, stdin=subprocess.PIPE)
# Send a newline for the password prompt
process.communicate("\n")
else:
process = subprocess.Popen(call_args)
process.wait()
def test_option(self):
parser = OptionParser()
self.pi.options( parser, env={} )
self.assertEqual( parser.has_option( '--xml' ), True )
self.assertEqual( parser.has_option( '--xml-formatter' ), True )
opt = parser.get_option( '--xml' )
self.assertEqual( opt.action, 'store_true' )
self.assertEqual( opt.default, False )
opt = parser.get_option( '--xml-formatter' )
self.assertEqual( opt.action, 'store' )
self.assertEqual( opt.default, 'nosexml.PrettyPrintFormatter' )
class TestTypeAliases(BaseTest):
def setUp(self):
self.parser = OptionParser()
def test_type_aliases(self):
self.parser.add_option("-x", type=int)
self.parser.add_option("-s", type=str)
self.parser.add_option("-t", type="str")
self.assertEquals(self.parser.get_option("-x").type, "int")
self.assertEquals(self.parser.get_option("-s").type, "string")
self.assertEquals(self.parser.get_option("-t").type, "string")
def main(argv):
pp = pprint.PrettyPrinter(indent=4)
arg_handler = OptionParser("**************** DPLL 3 SAT Solver ****************")
arg_handler.add_option('-f', '--file', type='string', help='Dimacs file in CNF, Format -->> in cwd: file.dimacs else: /path/to/file/dimacs_file.dimacs')
arg_handler.add_option("-v", "--verbose", action="store_true", dest="verbose")
options, args = arg_handler.parse_args()
option_dict = vars(options)
file_name = None
if arg_handler.get_option("-f"):
file_name = option_dict['file']
else:
raise FileInputError("Need to provide a Dimacs file")
verbose = False
if arg_handler.get_option("-v"):
verbose = True
logger = set_logger(file_name, verbose=verbose)
####################
# Parse Dimacs file
#
parse_file = DimacsParser(file_name,logger)
parse_file.parse()
(clauses, symbols, model) = parse_file.sentences()
logger.debug("parsed Dimacs file")
logger.debug("clauses: %s"%(clauses))
logger.debug("created Literals() and add symbols: %s"%(symbols))
logger.debug("model: %s"%(model))
####################
# SAT Solver
#
solver = SAT(logger=logger, clauses=clauses, symbols=symbols, model=model)
logger.info("calling SAT solver")
satisfiable = solver.dpll_satisfiable()
####################
# Result
#
if satisfiable:
logger.info("*******************************")
logger.info("CNF Sentence is Satisfiable")
logger.info("*******************************")
assignment = solver.get_assignment()
pp.pprint("Literal assignment: %s"%assignment)
else:
logger.info("*******************************")
logger.info("CNF Sentence is Unsatisfiable")
logger.info("*******************************")
class RestrictedPluginManager(DefaultPluginManager):
"""Plugin manager that restricts the plugin list to those not
excluded by a list of exclude methods. Any plugin that implements
an excluded method will be removed from the manager's plugin list
after plugins are loaded.
"""
def __init__(self, plugins=(), exclude=(), load=True):
DefaultPluginManager.__init__(self, plugins)
self.load = load
self.exclude = exclude
self.excluded = []
self._excludedOpts = None
def excludedOption(self, name):
if self._excludedOpts is None:
from optparse import OptionParser
self._excludedOpts = OptionParser(add_help_option=False)
for plugin in self.excluded:
plugin.options(self._excludedOpts, env={})
return self._excludedOpts.get_option('--' + name)
def loadPlugins(self):
if self.load:
DefaultPluginManager.loadPlugins(self)
allow = []
for plugin in self.plugins:
ok = True
for method in self.exclude:
if hasattr(plugin, method):
ok = False
self.excluded.append(plugin)
break
if ok:
allow.append(plugin)
self.plugins = allow
class TestConflictOverride(BaseTest):
def setUp(self):
self.parser = OptionParser(usage=SUPPRESS_USAGE)
self.parser.set_conflict_handler("resolve")
self.parser.add_option("-n", "--dry-run",
action="store_true", dest="dry_run",
help="don't do anything")
self.parser.add_option("--dry-run", "-n",
action="store_const", const=42, dest="dry_run",
help="dry run mode")
def test_conflict_override_opts(self):
opt = self.parser.get_option("--dry-run")
self.assertEqual(opt._short_opts, ["-n"])
self.assertEqual(opt._long_opts, ["--dry-run"])
def test_conflict_override_help(self):
self.assertStdoutEquals(["-h"], """\
options:
-h, --help show this help message and exit
-n, --dry-run dry run mode
""")
def test_conflict_override_args(self):
self.assertParseOK(["-n"],
{'dry_run': 42},
[])
class TestConflictOverride(BaseTest):
def setUp(self):
self.parser = OptionParser(usage=SUPPRESS_USAGE)
self.parser.set_conflict_handler("resolve")
self.parser.add_option("-n",
"--dry-run",
action="store_true",
dest="dry_run",
help="don't do anything")
self.parser.add_option("--dry-run",
"-n",
action="store_const",
const=42,
dest="dry_run",
help="dry run mode")
def test_conflict_override_opts(self):
opt = self.parser.get_option("--dry-run")
self.assertEqual(opt._short_opts, ["-n"])
self.assertEqual(opt._long_opts, ["--dry-run"])
def test_conflict_override_help(self):
self.assertStdoutEquals(["-h"], """\
options:
-h, --help show this help message and exit
-n, --dry-run dry run mode
""")
def test_conflict_override_args(self):
self.assertParseOK(["-n"], {'dry_run': 42}, [])
class RestrictedPluginManager(DefaultPluginManager):
"""Plugin manager that restricts the plugin list to those not
excluded by a list of exclude methods. Any plugin that implements
an excluded method will be removed from the manager's plugin list
after plugins are loaded.
"""
def __init__(self, plugins=(), exclude=(), load=True):
DefaultPluginManager.__init__(self, plugins)
self.load = load
self.exclude = exclude
self.excluded = []
self._excludedOpts = None
def excludedOption(self, name):
if self._excludedOpts is None:
from optparse import OptionParser
self._excludedOpts = OptionParser(add_help_option=False)
for plugin in self.excluded:
plugin.options(self._excludedOpts, env={})
return self._excludedOpts.get_option('--' + name)
def loadPlugins(self):
if self.load:
DefaultPluginManager.loadPlugins(self)
allow = []
for plugin in self.plugins:
ok = True
for method in self.exclude:
if hasattr(plugin, method):
ok = False
self.excluded.append(plugin)
break
if ok:
allow.append(plugin)
self.plugins = allow
def saveConfiguration(executable, options, filename):
optParser = OptionParser()
pullOptions(executable, optParser)
cmd = [executable, "--save-configuration", filename]
for option, value in options.__dict__.iteritems():
o = "--" + option.replace("_", "-")
opt = optParser.get_option(o)
if value and opt.default != value:
cmd.append(o)
if opt.action != "store_true":
cmd.append(str(value))
subprocess.call(cmd)
def call(executable, options):
optParser = OptionParser()
pullOptions(executable, optParser)
cmd = [executable]
for option, value in options.__dict__.iteritems():
o = "--" + option.replace("_", "-")
opt = optParser.get_option(o)
if opt is not None and value is not None and opt.default != value:
cmd.append(o)
if opt.action != "store_true":
cmd.append(str(value))
return subprocess.call(cmd)
def call(executable, options):
optParser = OptionParser()
pullOptions(executable, optParser)
cmd = [executable]
for option, value in options.__dict__.iteritems():
o = "--" + option.replace("_", "-")
opt = optParser.get_option(o)
if opt is not None and value is not None and opt.default != value:
cmd.append(o)
if opt.action != "store_true":
cmd.append(str(value))
return subprocess.call(cmd)
def main():
options = OptionParser(usage="%prog [options] LOGFILE")
options.add_option("-c", "--conf", action="store", type="string",
dest="conf", default="/etc/loganomaly/loganomaly.conf", metavar="FILE",
help="Configuration file")
options.add_option("-s", "--statedb", action="store", type="string",
dest="statedb", default="/var/loganomaly/state.db", metavar="FILE",
help="Name of the file to store state database in [default: %default]")
options.add_option("-i", "--ignore", action="store", type="string",
dest="ignore", default=None, metavar="FILE",
help="Path to file containing list of patterns to ignore")
options.add_option("-l", "--limit", action="store", type="int",
dest="limit", default=100, metavar="N",
help="Limit output to N lines [default: %default]")
options.add_option("-t", "--test", action="store_true",
dest="test", default=False,
help="Do not save state to state db")
options.get_option("-h").help = "Show this message and exit"
config = Config(options)
if not config.ignore:
options.error("Specify --ignore")
if not config.statedb:
options.error("Specify --statedb")
if len(config.args) == 0:
options.error("Specify LOGFILE")
if config.test:
statedb = NullStateDB()
else:
statedb = SQLiteStateDB(config.statedb)
filter = Filter(config.ignore)
for logfile in config.args:
log = FilteredLog(logfile, filter, statedb)
found = False
i = 0
for line in log:
if not found:
print "Anomalies found in log file: %s\n\n" % log
found = True
if i <= config.limit:
print line
i += 1
if found and i > config.limit:
print "...\n%s more lines found" % (i-config.limit)
class TestChoice(BaseTest):
def setUp(self):
self.parser = OptionParser(usage=SUPPRESS_USAGE)
self.parser.add_option("-c", action="store", type="choice",
dest="choice", choices=["one", "two", "three"])
def test_valid_choice(self):
self.assertParseOK(["-c", "one", "xyz"],
{'choice': 'one'},
["xyz"])
def test_invalid_choice(self):
self.assertParseFail(["-c", "four", "abc"],
"option -c: invalid choice: 'four' "
"(choose from 'one', 'two', 'three')")
def test_add_choice_option(self):
self.parser.add_option("-d", "--default",
choices=["four", "five", "six"])
opt = self.parser.get_option("-d")
self.assertEqual(opt.type, "choice")
self.assertEqual(opt.action, "store")
def getArgs(self):
"""
This function parses the command line parameters and arguments
"""
usage = "python %prog [options]"
parser = OptionParser(usage=usage)
try:
parser.add_option('--version',
dest='showversion',
action='store_true',
help="Show program's version number and exit")
target = OptionGroup(parser, "TARGET", "a domain, or a ip")
target.add_option('-d',
'--domain',
dest='domain',
type='str',
help='domain name')
target.add_option('-i',
'--ip',
dest='ipaddr',
help='target host ip')
parser.add_option_group(target)
option = parser.get_option('--version')
option._short_opts = ['-version']
option._long_opts = []
(args, _) = parser.parse_args()
except (OptionError, TypeError) as e:
parser.error(e)
else:
return args
def getArgs(self):
"""
This function parses the command line parameters and arguments
"""
usage = "python %prog [options]"
parser = OptionParser(usage=usage)
try:
parser.add_option('--version',
dest='showversion',
action='store_true',
help="Show program's version number and exit")
target = OptionGroup(parser, "TARGET", "a domain, or a ip")
target.add_option('-d', '--domain',
dest='domain',
type='str',
help='domain name')
target.add_option('-i', '--ip',
dest='ipaddr',
help='target host ip')
parser.add_option_group(target)
option = parser.get_option('--version')
option._short_opts = ['-version']
option._long_opts = []
(args, _) = parser.parse_args()
except (OptionError, TypeError) as e:
parser.error(e)
else:
return args
class TestChoice(BaseTest):
def setUp(self):
self.parser = OptionParser(usage=SUPPRESS_USAGE)
self.parser.add_option("-c",
action="store",
type="choice",
dest="choice",
choices=["one", "two", "three"])
def test_valid_choice(self):
self.assertParseOK(["-c", "one", "xyz"], {'choice': 'one'}, ["xyz"])
def test_invalid_choice(self):
self.assertParseFail(["-c", "four", "abc"],
"option -c: invalid choice: 'four' "
"(choose from 'one', 'two', 'three')")
def test_add_choice_option(self):
self.parser.add_option("-d",
"--default",
choices=["four", "five", "six"])
opt = self.parser.get_option("-d")
self.assertEqual(opt.type, "choice")
self.assertEqual(opt.action, "store")
"-u",
"--upper-bound",
action="store",
type="int",
dest="upperBound",
help="The upper bound on the range of the input space. [Default is %default].",
default=maxint,
metavar="<INT>",
)
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="Be verbose.", default=False)
(opts, args) = parser.parse_args(argv[1:])
if opts.numOfParameters is None:
print("Missing option " + str(parser.get_option("-n")))
exit(0)
elif opts.program is None:
print("Missing option " + str(parser.get_option("-p")))
exit(0)
elif opts.config is None:
print("Missing option " + str(parser.get_option("-c")))
exit(0)
elif opts.simplescalar is None:
print("Missing option " + str(parser.get_option("-s")))
exit(0)
if opts.debug:
print("SimpleScalar = " + opts.simplescalar)
print("Program = " + opts.program)
print("Lower bound = " + str(opts.lowerBound))
def sg_search_main(duration: float):
print('LAN search is starting...')
loop = asyncio.get_event_loop()
tasks = asyncio.gather(
sg_search_task(duration),
)
loop.run_until_complete(tasks)
if __name__ == "__main__":
try:
from optparse import OptionParser
parser = OptionParser(usage=('Usage: %prog [options]\n\n' + SG_SEARCH_VERINFO))
parser.get_option("-h").help = "Show this help message and exit."
parser.add_option("--version",
action="store_true", default=False, dest="show_version",
help="Show version.")
parser.add_option("-d", "--duration",
action="store", type="float", default=DEFAULT_DURATION, dest="duration",
help="The duration of the search, in seconds.")
(options, args) = parser.parse_args()
if options.show_version:
print(SG_SEARCH_VERINFO)
sys.exit(0)
sg_search_main(options.duration)
sys.exit(0)
except KeyboardInterrupt:
injection.add_option("--os-cmd",
action="store",
dest="os_cmd",
default = False,
help="Execute a single operating system command.")
parser.add_option_group(target)
parser.add_option_group(request)
parser.add_option_group(enumeration)
parser.add_option_group(file_access)
parser.add_option_group(injection)
# Dirty hack from SQLMAP, to display longer options without breaking into two lines.
def _(self, *args):
_ = parser.formatter._format_option_strings(*args)
if len(_) > settings.MAX_OPTION_LENGTH:
_ = ("%%.%ds.." % (settings.MAX_OPTION_LENGTH - parser.formatter.indent_increment)) % _
return _
parser.formatter._format_option_strings = parser.formatter.format_option_strings
parser.formatter.format_option_strings = type(parser.formatter.format_option_strings)(_, parser, type(parser))
option = parser.get_option("-h")
option.help = option.help.capitalize().replace("Show this help message and exit", "Show help and exit.")
(options, args) = parser.parse_args()
#eof
def get_option_parser(defaults):
"""Creates OptionParser and adds shell options (flags)
Default values are loaded in initially
"""
parser = OptionParser()
parser.set_defaults(**defaults)
parser.add_option("-i", "--impalad", dest="impalad",
help="<host:port> of impalad to connect to \t\t")
parser.add_option("-b", "--kerberos_host_fqdn", dest="kerberos_host_fqdn",
help="If set, overrides the expected hostname of the Impalad's "
"kerberos service principal. impala-shell will check that "
"the server's principal matches this hostname. This may be "
"used when impalad is configured to be accessed via a "
"load-balancer, but it is desired for impala-shell to talk "
"to a specific impalad directly.")
parser.add_option("-q", "--query", dest="query",
help="Execute a query without the shell")
parser.add_option("-f", "--query_file", dest="query_file",
help="Execute the queries in the query file, delimited by ;."
" If the argument to -f is \"-\", then queries are read from"
" stdin and terminated with ctrl-d.")
parser.add_option("-k", "--kerberos", dest="use_kerberos",
action="store_true", help="Connect to a kerberized impalad")
parser.add_option("-o", "--output_file", dest="output_file",
help=("If set, query results are written to the "
"given file. Results from multiple semicolon-terminated "
"queries will be appended to the same file"))
parser.add_option("-B", "--delimited", dest="write_delimited",
action="store_true",
help="Output rows in delimited mode")
parser.add_option("--print_header", dest="print_header",
action="store_true",
help="Print column names in delimited mode"
" when pretty-printed.")
parser.add_option("--output_delimiter", dest="output_delimiter",
help="Field delimiter to use for output in delimited mode")
parser.add_option("-s", "--kerberos_service_name",
dest="kerberos_service_name",
help="Service name of a kerberized impalad")
parser.add_option("-V", "--verbose", dest="verbose",
action="store_true",
help="Verbose output")
parser.add_option("-p", "--show_profiles", dest="show_profiles",
action="store_true",
help="Always display query profiles after execution")
parser.add_option("--quiet", dest="verbose",
action="store_false",
help="Disable verbose output")
parser.add_option("-v", "--version", dest="version",
action="store_true",
help="Print version information")
parser.add_option("-c", "--ignore_query_failure", dest="ignore_query_failure",
action="store_true", help="Continue on query failure")
parser.add_option("-d", "--database", dest="default_db",
help="Issues a use database command on startup \t")
parser.add_option("-l", "--ldap", dest="use_ldap",
action="store_true",
help="Use LDAP to authenticate with Impala. Impala must be configured"
" to allow LDAP authentication. \t\t")
parser.add_option("-u", "--user", dest="user",
help="User to authenticate with.")
parser.add_option("--ssl", dest="ssl",
action="store_true",
help="Connect to Impala via SSL-secured connection \t")
parser.add_option("--ca_cert", dest="ca_cert",
help=("Full path to "
"certificate file used to authenticate Impala's SSL certificate."
" May either be a copy of Impala's certificate (for self-signed "
"certs) or the certificate of a trusted third-party CA. If not set, "
"but SSL is enabled, the shell will NOT verify Impala's server "
"certificate"))
parser.add_option("--config_file", dest="config_file",
help=("Specify the configuration file to load options. "
"The following sections are used: [impala], "
"[impala.query_options]. Section names are case sensitive. "
"Specifying this option within a config file will have "
"no effect. Only specify this as an option in the commandline."
))
parser.add_option("--history_file", dest="history_file",
help=("The file in which to store shell history. This may also be "
"configured using the IMPALA_HISTFILE environment variable."))
parser.add_option("--live_summary", dest="print_summary", action="store_true",
help="Print a query summary every 1s while the query is running.")
parser.add_option("--live_progress", dest="print_progress", action="store_true",
help="Print a query progress every 1s while the query is running.")
parser.add_option("--auth_creds_ok_in_clear", dest="creds_ok_in_clear",
action="store_true", help="If set, LDAP authentication " +
"may be used with an insecure connection to Impala. " +
"WARNING: Authentication credentials will therefore be sent " +
"unencrypted, and may be vulnerable to attack.")
parser.add_option("--ldap_password_cmd",
help="Shell command to run to retrieve the LDAP password")
parser.add_option("--var", dest="keyval", action="append",
help="Defines a variable to be used within the Impala session."
" Can be used multiple times to set different variables."
" It must follow the pattern \"KEY=VALUE\","
" KEY starts with an alphabetic character and"
" contains alphanumeric characters or underscores.")
parser.add_option("-Q", "--query_option", dest="query_options", action="append",
help="Sets the default for a query option."
" Can be used multiple times to set different query options."
" It must follow the pattern \"KEY=VALUE\","
" KEY must be a valid query option. Valid query options "
" can be listed by command 'set'.")
# add default values to the help text
for option in parser.option_list:
# since the quiet flag is the same as the verbose flag
# we need to make sure to print the opposite value for it
# (print quiet is false since verbose is true)
if option == parser.get_option('--quiet'):
option.help += " [default: %s]" % (not defaults['verbose'])
elif option != parser.get_option('--help'):
# don't want to print default value for help
option.help += " [default: %default]"
return parser
def main(args=None):
"""The main function; parses options and plots"""
# ---------- build and read options ----------
from optparse import OptionParser
optParser = OptionParser()
optParser.add_option("-n", "--net", dest="net", metavar="FILE",
help="Defines the network to read")
optParser.add_option("-i", "--dump-inputs", dest="dumps", metavar="FILE",
help="Defines the dump-output files to use as input")
optParser.add_option("-m", "--measures", dest="measures",
default="speed,entered", help="Define which measure to plot")
optParser.add_option("--min-width", dest="minWidth",
type="float", default=.5, help="Defines the minimum edge width")
optParser.add_option("--max-width", dest="maxWidth",
type="float", default=3, help="Defines the maximum edge width")
optParser.add_option("--log-colors", dest="logColors", action="store_true",
default=False, help="If set, colors are log-scaled")
optParser.add_option("--log-widths", dest="logWidths", action="store_true",
default=False, help="If set, widths are log-scaled")
optParser.add_option("--min-color-value", dest="colorMin",
type="float", default=None,
help="If set, defines the minimum edge color value")
optParser.add_option("--max-color-value", dest="colorMax",
type="float", default=None,
help="If set, defines the maximum edge color value")
optParser.add_option("--min-width-value", dest="widthMin",
type="float", default=None,
help="If set, defines the minimum edge width value")
optParser.add_option("--max-width-value", dest="widthMax",
type="float", default=None,
help="If set, defines the maximum edge width value")
optParser.add_option("-v", "--verbose", dest="verbose", action="store_true",
default=False,
help="If set, the script says what it's doing")
# standard plot options
helpers.addInteractionOptions(optParser)
helpers.addPlotOptions(optParser)
helpers.addNetOptions(optParser)
# Override the help string for the output option
outputOpt = optParser.get_option("--output")
outputOpt.help = "Comma separated list of filename(s) the figure shall be written to; " +\
"for multiple time intervals use \'\%s\' in the filename as a " +\
"placeholder for the beginning of the time interval"
# parse
options, remaining_args = optParser.parse_args(args=args)
if options.net == None:
print("Error: a network to load must be given.")
return 1
if options.verbose:
print("Reading network from '%s'" % options.net)
net = sumolib.net.readNet(options.net)
if options.measures == None:
print("Error: a dump file must be given.")
return 1
times = []
hc = None
colorDump = options.dumps.split(",")[0]
colorMeasure = options.measures.split(",")[0]
if colorDump:
if options.verbose:
print("Reading colors from '%s'" % colorDump)
hc = WeightsReader(colorMeasure)
sumolib.output.parse_sax(colorDump, hc)
times = hc._edge2value
hw = None
widthDump = options.dumps.split(",")[1]
widthMeasure = options.measures.split(",")[1]
if widthDump != "":
if options.verbose:
print("Reading widths from '%s'" % widthDump)
hw = WeightsReader(widthMeasure)
sumolib.output.parse_sax(widthDump, hw)
times = hw._edge2value
# Should we also save the figure to a file / list of files (comma
# separated)? Then we need to check the output filename(s)
if options.output:
options.nolegend = True
optOutputNames = options.output
# If we have multiple intervals to be plotted, make sure we have
# proper output filenames (with a %s as a placeholder in it)
if len(times) > 1 and optOutputNames.find('%s') < 0:
print('Warning: multiple time intervals detected, but ' +
'the output filename(s) do not contain a \'%s\' placeholder. ' +
'Continuing by using a default placeholder.')
# Modify each filename by putting a '-%s' right before the
# extension
filenames = optOutputNames.split(',')
for i in range(0, len(filenames)):
filename, extension = os.path.splitext(filenames[i])
filenames[i] = filename + '-%s' + extension
optOutputNames = ','.join(filenames)
# Now go through each time interval and create the figures
for t in times:
if options.verbose:
print("Processing interval with a beginning of %s" % t)
colors = {}
maxColorValue = None
minColorValue = None
for e in net._id2edge:
if hc and t in hc._edge2value and e in hc._edge2value[t]:
if options.colorMax != None and hc._edge2value[t][e] > options.colorMax:
hc._edge2value[t][e] = options.colorMax
if options.colorMin != None and hc._edge2value[t][e] < options.colorMin:
hc._edge2value[t][e] = options.colorMin
if maxColorValue == None or maxColorValue < hc._edge2value[t][e]:
maxColorValue = hc._edge2value[t][e]
if minColorValue == None or minColorValue > hc._edge2value[t][e]:
minColorValue = hc._edge2value[t][e]
colors[e] = hc._edge2value[t][e]
if options.colorMax != None:
maxColorValue = options.colorMax
if options.colorMin != None:
minColorValue = options.colorMin
if options.logColors:
helpers.logNormalise(colors, maxColorValue)
else:
helpers.linNormalise(colors, minColorValue, maxColorValue)
for e in colors:
colors[e] = helpers.getColor(options, colors[e], 1.)
if options.verbose:
print("Color values are between %s and %s" %
(minColorValue, maxColorValue))
widths = {}
maxWidthValue = None
minWidthValue = None
for e in net._id2edge:
if hw and t in hw._edge2value and e in hw._edge2value[t]:
v = abs(hw._edge2value[t][e])
if options.widthMax != None and v > options.widthMax:
v = options.widthMax
if options.widthMin != None and v < options.widthMin:
v = options.widthMin
if not maxWidthValue or maxWidthValue < v:
maxWidthValue = v
if not minWidthValue or minWidthValue > v:
minWidthValue = v
widths[e] = v
if options.widthMax != None:
maxWidthValue = options.widthMax
if options.widthMin != None:
minWidthValue = options.widthMin
if options.logWidths:
helpers.logNormalise(widths, options.colorMax)
else:
helpers.linNormalise(widths, minWidthValue, maxWidthValue)
for e in widths:
widths[e] = options.minWidth + widths[e] * \
(options.maxWidth - options.minWidth)
if options.verbose:
print("Width values are between %s and %s" %
(minWidthValue, maxWidthValue))
fig, ax = helpers.openFigure(options)
ax.set_aspect("equal", None, 'C')
helpers.plotNet(net, colors, widths, options)
# drawing the legend, at least for the colors
sm = plt.cm.ScalarMappable(cmap=matplotlib.cm.get_cmap(options.colormap),
norm=matplotlib.colors.Normalize(vmin=minColorValue,
vmax=maxColorValue))
# "fake up the array of the scalar mappable. Urgh..."
# (pelson, http://stackoverflow.com/questions/8342549/matplotlib-add-colorbar-to-a-sequence-of-line-plots)
sm._A = []
plt.colorbar(sm)
# Should we also save the figure to a file / list of files (comma
# separated)?
if options.output:
# If we have a "%s" in the name of the output then replace it with the
# interval begin of the current interval
expandedOutputNames = optOutputNames
if expandedOutputNames.find('%s') >= 0:
expandedOutputNames = expandedOutputNames.replace("%s", str(t))
# Can be used to print additional text in the figure:
#
# m, s = divmod(int(t), 60)
# h, m = divmod(m, 60)
# timeStr = "%02d:%02d:%02d" % (h, m, s)
# ax.text(0.2, 0.2, timeStr, bbox={
# 'facecolor': 'white', 'pad': 10}, size=16)
helpers.closeFigure(fig, ax, options, False, expandedOutputNames)
return 0
if __name__ == "__main__":
parser = OptionParser()
parser.add_option("-l",
"--listen",
dest="listen",
help="Hostname/address to listen on")
parser.add_option("-p",
"--port",
dest="port",
type="int",
help="Port to listen on")
(options, args) = parser.parse_args()
#parser.check_required("-l")
#parser.check_required("-p")
if parser.get_option("-l") == None:
print "-l is required"
parser.print_help()
quit(-1)
if parser.get_option("-p") == None:
print "-p is required"
parser.print_help()
quit(-1)
root = InkLevelD()
print("starting out...")
# INADDR_ANY: listen on all interfaces
cherrypy.server.socket_host = options.listen
cherrypy.server.socket_port = options.port
cherrypy.quickstart(InkLevelD())
class VeryBasicScript(object):
#=====================
def __init__(self,optargs=sys.argv[1:],quiet=False):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.quiet = quiet
# self.timestamp = apParam.makeTimestamp()
# if not self.quiet:
# apDisplay.printMsg("Time stamp: "+self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
if not self.quiet:
apDisplay.printMsg("Function name: "+self.functionname)
apParam.setUmask()
self.parsePythonPath()
# loadavg = os.getloadavg()[0]
# if loadavg > 2.0:
# apDisplay.printMsg("Load average is %.2f, wait for %.1f second " % (round(loadavg,2),loadavg**2))
# time.sleep(loadavg**2)
# apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
### setup default parser: run directory, etc.
self.setParams(optargs)
self.checkConflicts()
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### any custom init functions go here
self.onInit()
def setParams(self,optargs=sys.argv[1:]):
self.parser = OptionParser()
self.setupParserOptions()
self.params = apParam.convertParserToParams(self.parser)
self.checkForDuplicateCommandLineInputs(optargs)
#=====================
def checkForDuplicateCommandLineInputs(self,optargs=sys.argv[1:]):
args = optargs
argmdict = {}
for arg in args:
elements=arg.split('=')
opt = elements[0].lower()
if opt[0] == "-":
## if action='append', then opt is allowed multiple times
option = self.parser.get_option(opt)
if option is not None and option.action == 'append':
multiple_ok = True
else:
multiple_ok = False
if opt in argmdict and not multiple_ok:
apDisplay.printError("Multiple arguments were supplied for argument: "+str(opt))
argmdict[opt] = True
#=====================
def createDefaultStats(self):
self.stats = {}
self.stats['starttime'] = time.time()
self.stats['count'] = 1
self.stats['lastcount'] = 0
self.stats['startmem'] = mem.active()
self.stats['memleak'] = 0
self.stats['peaksum'] = 0
self.stats['lastpeaks'] = None
self.stats['imagesleft'] = 1
self.stats['peaksumsq'] = 0
self.stats['timesum'] = 0
self.stats['timesumsq'] = 0
self.stats['skipcount'] = 0
self.stats['waittime'] = 0
self.stats['lastimageskipped'] = False
self.stats['notpair'] = 0
self.stats['memlist'] = [mem.active()]
#=====================
def close(self):
self.onClose()
# loadavg = os.getloadavg()[0]
# if loadavg > 2.0:
# apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
# time.sleep(loadavg**2)
apParam.closeFunctionLog(functionname=self.functionname,
logfile=self.logfile, msg=(not self.quiet))
if self.quiet is False:
apDisplay.printMsg("Ended at "+time.strftime("%a, %d %b %Y %H:%M:%S"))
if apDisplay.isDebugOn():
apDisplay.printDebug("Memory increase during run: %.3f MB"%((mem.active()-self.startmem)/1024.0))
apDisplay.printColor("Total run time:\t"+apDisplay.timeString(time.time()-self.t0),"green")
#=====================
def parsePythonPath(self):
pythonpath = os.environ.get("PYTHONPATH")
if pythonpath is None:
return
paths = pythonpath.split(":")
leginons = {}
appions = {}
for p in paths:
if "appion" in p:
appions[p] = None
if "leginon" in p:
leginons[p] = None
leginons = leginons.keys()
appions = appions.keys()
# if len(appions) > 1:
# apDisplay.printWarning("There is more than one appion directory in your PYTHONPATH")
# print appions
# if len(leginons) > 1:
# apDisplay.printWarning("There is more than one leginon directory in your PYTHONPATH")
# print leginons
#######################################################
#### ITEMS BELOW CAN BE SPECIFIED IN A NEW PROGRAM ####
#######################################################
#=====================
def setupParserOptions(self):
"""
set the input parameters
this function should be rewritten in each program
"""
apDisplay.printError("you did not create a 'setupParserOptions' function in your script")
self.parser.set_usage("Usage: %prog --commit --description='<text>' [options]")
self.parser.add_option("--stackid", dest="stackid", type="int",
help="ID for particle stack (optional)", metavar="INT")
#=====================
def checkConflicts(self):
"""
make sure the necessary parameters are set correctly
"""
apDisplay.printError("you did not create a 'checkConflicts' function in your script")
return
#=====================
def start(self):
"""
this is the main component of the script
where all the processing is done
"""
raise NotImplementedError()
#=====================
def onInit(self):
return
#=====================
def onClose(self):
return
def cmdLineParser():
"""
This function parses the command line parameters and arguments
"""
checkSystemEncoding()
_ = os.path.normpath(sys.argv[0])
usage = "%s%s [options]" % ("python " if not IS_WIN else "", \
"\"%s\"" % _ if " " in _ else _)
parser = OptionParser(usage=usage)
try:
parser.add_option("--hh", dest="advancedHelp",
action="store_true",
help="Show advanced help message and exit")
parser.add_option("--version", dest="showVersion",
action="store_true",
help="Show program's version number and exit")
parser.add_option("-v", dest="verbose", type="int",
help="Verbosity level: 0-6 (default %d)" % defaults.verbose)
# Target options
target = OptionGroup(parser, "Target", "At least one of these "
"options has to be provided to define the target(s)")
target.add_option("-d", dest="direct", help="Connection string "
"for direct database connection")
target.add_option("-u", "--url", dest="url", help="Target URL (e.g. \"http://www.site.com/vuln.php?id=1\")")
target.add_option("-l", dest="logFile", help="Parse target(s) from Burp "
"or WebScarab proxy log file")
target.add_option("-x", dest="sitemapUrl", help="Parse target(s) from remote sitemap(.xml) file")
target.add_option("-m", dest="bulkFile", help="Scan multiple targets given "
"in a textual file ")
target.add_option("-r", dest="requestFile",
help="Load HTTP request from a file")
target.add_option("-g", dest="googleDork",
help="Process Google dork results as target URLs")
target.add_option("-c", dest="configFile",
help="Load options from a configuration INI file")
# Request options
request = OptionGroup(parser, "Request", "These options can be used "
"to specify how to connect to the target URL")
request.add_option("--data", dest="data",
help="Data string to be sent through POST")
request.add_option("--param-del", dest="paramDel",
help="Character used for splitting parameter values")
request.add_option("--cookie", dest="cookie",
help="HTTP Cookie header value")
request.add_option("--cookie-del", dest="cookieDel",
help="Character used for splitting cookie values")
request.add_option("--load-cookies", dest="loadCookies",
help="File containing cookies in Netscape/wget format")
request.add_option("--drop-set-cookie", dest="dropSetCookie",
action="store_true",
help="Ignore Set-Cookie header from response")
request.add_option("--user-agent", dest="agent",
help="HTTP User-Agent header value")
request.add_option("--random-agent", dest="randomAgent",
action="store_true",
help="Use randomly selected HTTP User-Agent header value")
request.add_option("--host", dest="host",
help="HTTP Host header value")
request.add_option("--referer", dest="referer",
help="HTTP Referer header value")
request.add_option("--headers", dest="headers",
help="Extra headers (e.g. \"Accept-Language: fr\\nETag: 123\")")
request.add_option("--auth-type", dest="authType",
help="HTTP authentication type "
"(Basic, Digest, NTLM or PKI)")
request.add_option("--auth-cred", dest="authCred",
help="HTTP authentication credentials "
"(name:password)")
request.add_option("--auth-private", dest="authPrivate",
help="HTTP authentication PEM private key file")
request.add_option("--ignore-401", dest="ignore401", action="store_true",
help="Ignore HTTP Error 401 (Unauthorized)")
request.add_option("--proxy", dest="proxy",
help="Use a proxy to connect to the target URL")
request.add_option("--proxy-cred", dest="proxyCred",
help="Proxy authentication credentials "
"(name:password)")
request.add_option("--proxy-file", dest="proxyFile",
help="Load proxy list from a file")
request.add_option("--ignore-proxy", dest="ignoreProxy", action="store_true",
help="Ignore system default proxy settings")
request.add_option("--tor", dest="tor",
action="store_true",
help="Use Tor anonymity network")
request.add_option("--tor-port", dest="torPort",
help="Set Tor proxy port other than default")
request.add_option("--tor-type", dest="torType",
help="Set Tor proxy type (HTTP (default), SOCKS4 or SOCKS5)")
request.add_option("--check-tor", dest="checkTor",
action="store_true",
help="Check to see if Tor is used properly")
request.add_option("--delay", dest="delay", type="float",
help="Delay in seconds between each HTTP request")
request.add_option("--timeout", dest="timeout", type="float",
help="Seconds to wait before timeout connection "
"(default %d)" % defaults.timeout)
request.add_option("--retries", dest="retries", type="int",
help="Retries when the connection timeouts "
"(default %d)" % defaults.retries)
request.add_option("--randomize", dest="rParam",
help="Randomly change value for given parameter(s)")
request.add_option("--safe-url", dest="safUrl",
help="URL address to visit frequently during testing")
request.add_option("--safe-freq", dest="saFreq", type="int",
help="Test requests between two visits to a given safe URL")
request.add_option("--skip-urlencode", dest="skipUrlEncode",
action="store_true",
help="Skip URL encoding of payload data")
request.add_option("--force-ssl", dest="forceSSL",
action="store_true",
help="Force usage of SSL/HTTPS")
request.add_option("--hpp", dest="hpp",
action="store_true",
help="Use HTTP parameter pollution method")
request.add_option("--eval", dest="evalCode",
help="Evaluate provided Python code before the request (e.g. \"import hashlib;id2=hashlib.md5(id).hexdigest()\")")
# Optimization options
optimization = OptionGroup(parser, "Optimization", "These "
"options can be used to optimize the "
"performance of sqlmap")
optimization.add_option("-o", dest="optimize",
action="store_true",
help="Turn on all optimization switches")
optimization.add_option("--predict-output", dest="predictOutput", action="store_true",
help="Predict common queries output")
optimization.add_option("--keep-alive", dest="keepAlive", action="store_true",
help="Use persistent HTTP(s) connections")
optimization.add_option("--null-connection", dest="nullConnection", action="store_true",
help="Retrieve page length without actual HTTP response body")
optimization.add_option("--threads", dest="threads", type="int",
help="Max number of concurrent HTTP(s) "
"requests (default %d)" % defaults.threads)
# Injection options
injection = OptionGroup(parser, "Injection", "These options can be "
"used to specify which parameters to test "
"for, provide custom injection payloads and "
"optional tampering scripts")
injection.add_option("-p", dest="testParameter",
help="Testable parameter(s)")
injection.add_option("--skip", dest="skip",
help="Skip testing for given parameter(s)")
injection.add_option("--dbms", dest="dbms",
help="Force back-end DBMS to this value")
injection.add_option("--dbms-cred", dest="dbmsCred",
help="DBMS authentication credentials (user:password)")
injection.add_option("--os", dest="os",
help="Force back-end DBMS operating system "
"to this value")
injection.add_option("--invalid-bignum", dest="invalidBignum",
action="store_true",
help="Use big numbers for invalidating values")
injection.add_option("--invalid-logical", dest="invalidLogical",
action="store_true",
help="Use logical operations for invalidating values")
injection.add_option("--invalid-string", dest="invalidString",
action="store_true",
help="Use random strings for invalidating values")
injection.add_option("--no-cast", dest="noCast",
action="store_true",
help="Turn off payload casting mechanism")
injection.add_option("--no-escape", dest="noEscape",
action="store_true",
help="Turn off string escaping mechanism")
injection.add_option("--prefix", dest="prefix",
help="Injection payload prefix string")
injection.add_option("--suffix", dest="suffix",
help="Injection payload suffix string")
injection.add_option("--tamper", dest="tamper",
help="Use given script(s) for tampering injection data")
# Detection options
detection = OptionGroup(parser, "Detection", "These options can be "
"used to customize the detection phase")
detection.add_option("--level", dest="level", type="int",
help="Level of tests to perform (1-5, "
"default %d)" % defaults.level)
detection.add_option("--risk", dest="risk", type="int",
help="Risk of tests to perform (0-3, "
"default %d)" % defaults.level)
detection.add_option("--string", dest="string",
help="String to match when "
"query is evaluated to True")
detection.add_option("--not-string", dest="notString",
help="String to match when "
"query is evaluated to False")
detection.add_option("--regexp", dest="regexp",
help="Regexp to match when "
"query is evaluated to True")
detection.add_option("--code", dest="code", type="int",
help="HTTP code to match when "
"query is evaluated to True")
detection.add_option("--text-only", dest="textOnly",
action="store_true",
help="Compare pages based only on the textual content")
detection.add_option("--titles", dest="titles",
action="store_true",
help="Compare pages based only on their titles")
# Techniques options
techniques = OptionGroup(parser, "Techniques", "These options can be "
"used to tweak testing of specific SQL "
"injection techniques")
techniques.add_option("--technique", dest="tech",
help="SQL injection techniques to use "
"(default \"%s\")" % defaults.tech)
techniques.add_option("--time-sec", dest="timeSec",
type="int",
help="Seconds to delay the DBMS response "
"(default %d)" % defaults.timeSec)
techniques.add_option("--union-cols", dest="uCols",
help="Range of columns to test for UNION query SQL injection")
techniques.add_option("--union-char", dest="uChar",
help="Character to use for bruteforcing number of columns")
techniques.add_option("--union-from", dest="uFrom",
help="Table to use in FROM part of UNION query SQL injection")
techniques.add_option("--dns-domain", dest="dnsName",
help="Domain name used for DNS exfiltration attack")
techniques.add_option("--second-order", dest="secondOrder",
help="Resulting page URL searched for second-order "
"response")
# Fingerprint options
fingerprint = OptionGroup(parser, "Fingerprint")
fingerprint.add_option("-f", "--fingerprint", dest="extensiveFp",
action="store_true",
help="Perform an extensive DBMS version fingerprint")
# Enumeration options
enumeration = OptionGroup(parser, "Enumeration", "These options can "
"be used to enumerate the back-end database "
"management system information, structure "
"and data contained in the tables. Moreover "
"you can run your own SQL statements")
enumeration.add_option("-a", "--all", dest="getAll",
action="store_true", help="Retrieve everything")
enumeration.add_option("-b", "--banner", dest="getBanner",
action="store_true", help="Retrieve DBMS banner")
enumeration.add_option("--current-user", dest="getCurrentUser",
action="store_true",
help="Retrieve DBMS current user")
enumeration.add_option("--current-db", dest="getCurrentDb",
action="store_true",
help="Retrieve DBMS current database")
enumeration.add_option("--hostname", dest="getHostname",
action="store_true",
help="Retrieve DBMS server hostname")
enumeration.add_option("--is-dba", dest="isDba",
action="store_true",
help="Detect if the DBMS current user is DBA")
enumeration.add_option("--users", dest="getUsers", action="store_true",
help="Enumerate DBMS users")
enumeration.add_option("--passwords", dest="getPasswordHashes",
action="store_true",
help="Enumerate DBMS users password hashes")
enumeration.add_option("--privileges", dest="getPrivileges",
action="store_true",
help="Enumerate DBMS users privileges")
enumeration.add_option("--roles", dest="getRoles",
action="store_true",
help="Enumerate DBMS users roles")
enumeration.add_option("--dbs", dest="getDbs", action="store_true",
help="Enumerate DBMS databases")
enumeration.add_option("--tables", dest="getTables", action="store_true",
help="Enumerate DBMS database tables")
enumeration.add_option("--columns", dest="getColumns", action="store_true",
help="Enumerate DBMS database table columns")
enumeration.add_option("--schema", dest="getSchema", action="store_true",
help="Enumerate DBMS schema")
enumeration.add_option("--count", dest="getCount", action="store_true",
help="Retrieve number of entries for table(s)")
enumeration.add_option("--dump", dest="dumpTable", action="store_true",
help="Dump DBMS database table entries")
enumeration.add_option("--dump-all", dest="dumpAll", action="store_true",
help="Dump all DBMS databases tables entries")
enumeration.add_option("--search", dest="search", action="store_true",
help="Search column(s), table(s) and/or database name(s)")
enumeration.add_option("--comments", dest="getComments", action="store_true",
help="Retrieve DBMS comments")
enumeration.add_option("-D", dest="db",
help="DBMS database to enumerate")
enumeration.add_option("-T", dest="tbl",
help="DBMS database table(s) to enumerate")
enumeration.add_option("-C", dest="col",
help="DBMS database table column(s) to enumerate")
enumeration.add_option("-X", dest="excludeCol",
help="DBMS database table column(s) to not enumerate")
enumeration.add_option("-U", dest="user",
help="DBMS user to enumerate")
enumeration.add_option("--exclude-sysdbs", dest="excludeSysDbs",
action="store_true",
help="Exclude DBMS system databases when "
"enumerating tables")
enumeration.add_option("--where", dest="dumpWhere",
help="Use WHERE condition while table dumping")
enumeration.add_option("--start", dest="limitStart", type="int",
help="First query output entry to retrieve")
enumeration.add_option("--stop", dest="limitStop", type="int",
help="Last query output entry to retrieve")
enumeration.add_option("--first", dest="firstChar", type="int",
help="First query output word character to retrieve")
enumeration.add_option("--last", dest="lastChar", type="int",
help="Last query output word character to retrieve")
enumeration.add_option("--sql-query", dest="query",
help="SQL statement to be executed")
enumeration.add_option("--sql-shell", dest="sqlShell",
action="store_true",
help="Prompt for an interactive SQL shell")
enumeration.add_option("--sql-file", dest="sqlFile",
help="Execute SQL statements from given file(s)")
# User-defined function options
brute = OptionGroup(parser, "Brute force", "These "
"options can be used to run brute force "
"checks")
brute.add_option("--common-tables", dest="commonTables", action="store_true",
help="Check existence of common tables")
brute.add_option("--common-columns", dest="commonColumns", action="store_true",
help="Check existence of common columns")
# User-defined function options
udf = OptionGroup(parser, "User-defined function injection", "These "
"options can be used to create custom user-defined "
"functions")
udf.add_option("--udf-inject", dest="udfInject", action="store_true",
help="Inject custom user-defined functions")
udf.add_option("--shared-lib", dest="shLib",
help="Local path of the shared library")
# File system options
filesystem = OptionGroup(parser, "File system access", "These options "
"can be used to access the back-end database "
"management system underlying file system")
filesystem.add_option("--file-read", dest="rFile",
help="Read a file from the back-end DBMS "
"file system")
filesystem.add_option("--file-write", dest="wFile",
help="Write a local file on the back-end "
"DBMS file system")
filesystem.add_option("--file-dest", dest="dFile",
help="Back-end DBMS absolute filepath to "
"write to")
# Takeover options
takeover = OptionGroup(parser, "Operating system access", "These "
"options can be used to access the back-end "
"database management system underlying "
"operating system")
takeover.add_option("--os-cmd", dest="osCmd",
help="Execute an operating system command")
takeover.add_option("--os-shell", dest="osShell",
action="store_true",
help="Prompt for an interactive operating "
"system shell")
takeover.add_option("--os-pwn", dest="osPwn",
action="store_true",
help="Prompt for an OOB shell, "
"Meterpreter or VNC")
takeover.add_option("--os-smbrelay", dest="osSmb",
action="store_true",
help="One click prompt for an OOB shell, "
"Meterpreter or VNC")
takeover.add_option("--os-bof", dest="osBof",
action="store_true",
help="Stored procedure buffer overflow "
"exploitation")
takeover.add_option("--priv-esc", dest="privEsc",
action="store_true",
help="Database process user privilege escalation")
takeover.add_option("--msf-path", dest="msfPath",
help="Local path where Metasploit Framework "
"is installed")
takeover.add_option("--tmp-path", dest="tmpPath",
help="Remote absolute path of temporary files "
"directory")
# Windows registry options
windows = OptionGroup(parser, "Windows registry access", "These "
"options can be used to access the back-end "
"database management system Windows "
"registry")
windows.add_option("--reg-read", dest="regRead",
action="store_true",
help="Read a Windows registry key value")
windows.add_option("--reg-add", dest="regAdd",
action="store_true",
help="Write a Windows registry key value data")
windows.add_option("--reg-del", dest="regDel",
action="store_true",
help="Delete a Windows registry key value")
windows.add_option("--reg-key", dest="regKey",
help="Windows registry key")
windows.add_option("--reg-value", dest="regVal",
help="Windows registry key value")
windows.add_option("--reg-data", dest="regData",
help="Windows registry key value data")
windows.add_option("--reg-type", dest="regType",
help="Windows registry key value type")
# General options
general = OptionGroup(parser, "General", "These options can be used "
"to set some general working parameters")
#general.add_option("-x", dest="xmlFile",
# help="Dump the data into an XML file")
general.add_option("-s", dest="sessionFile",
help="Load session from a stored (.sqlite) file")
general.add_option("-t", dest="trafficFile",
help="Log all HTTP traffic into a "
"textual file")
general.add_option("--batch", dest="batch",
action="store_true",
help="Never ask for user input, use the default behaviour")
general.add_option("--charset", dest="charset",
help="Force character encoding used for data retrieval")
general.add_option("--crawl", dest="crawlDepth", type="int",
help="Crawl the website starting from the target URL")
general.add_option("--csv-del", dest="csvDel",
help="Delimiting character used in CSV output "
"(default \"%s\")" % defaults.csvDel)
general.add_option("--dump-format", dest="dumpFormat",
help="Format of dumped data (CSV (default), HTML or SQLITE)")
general.add_option("--eta", dest="eta",
action="store_true",
help="Display for each output the "
"estimated time of arrival")
general.add_option("--flush-session", dest="flushSession",
action="store_true",
help="Flush session files for current target")
general.add_option("--forms", dest="forms",
action="store_true",
help="Parse and test forms on target URL")
general.add_option("--fresh-queries", dest="freshQueries",
action="store_true",
help="Ignore query results stored in session file")
general.add_option("--hex", dest="hexConvert",
action="store_true",
help="Use DBMS hex function(s) for data retrieval")
general.add_option("--output-dir", dest="outputDir",
action="store",
help="Custom output directory path")
general.add_option("--parse-errors", dest="parseErrors",
action="store_true",
help="Parse and display DBMS error messages from responses")
general.add_option("--pivot-column", dest="pivotColumn",
help="Pivot column name")
general.add_option("--save", dest="saveCmdline",
action="store_true",
help="Save options to a configuration INI file")
general.add_option("--scope", dest="scope",
help="Regexp to filter targets from provided proxy log")
general.add_option("--test-filter", dest="testFilter",
help="Select tests by payloads and/or titles (e.g. ROW)")
general.add_option("--update", dest="updateAll",
action="store_true",
help="Update sqlmap")
# Miscellaneous options
miscellaneous = OptionGroup(parser, "Miscellaneous")
miscellaneous.add_option("-z", dest="mnemonics",
help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
miscellaneous.add_option("--alert", dest="alert",
help="Run host OS command(s) when SQL injection is found")
miscellaneous.add_option("--answers", dest="answers",
help="Set question answers (e.g. \"quit=N,follow=N\")")
miscellaneous.add_option("--beep", dest="beep", action="store_true",
help="Make a beep sound when SQL injection is found")
miscellaneous.add_option("--check-waf", dest="checkWaf",
action="store_true",
help="Heuristically check for WAF/IPS/IDS protection")
miscellaneous.add_option("--cleanup", dest="cleanup",
action="store_true",
help="Clean up the DBMS from sqlmap specific "
"UDF and tables")
miscellaneous.add_option("--dependencies", dest="dependencies",
action="store_true",
help="Check for missing (non-core) sqlmap dependencies")
miscellaneous.add_option("--disable-coloring", dest="disableColoring",
action="store_true",
help="Disable console output coloring")
miscellaneous.add_option("--gpage", dest="googlePage", type="int",
help="Use Google dork results from specified page number")
miscellaneous.add_option("--identify-waf", dest="identifyWaf",
action="store_true",
help="Make a through testing for a WAF/IPS/IDS protection")
miscellaneous.add_option("--mobile", dest="mobile",
action="store_true",
help="Imitate smartphone through HTTP User-Agent header")
miscellaneous.add_option("--page-rank", dest="pageRank",
action="store_true",
help="Display page rank (PR) for Google dork results")
miscellaneous.add_option("--purge-output", dest="purgeOutput",
action="store_true",
help="Safely remove all content from output directory")
miscellaneous.add_option("--smart", dest="smart",
action="store_true",
help="Conduct through tests only if positive heuristic(s)")
miscellaneous.add_option("--sqlmap-shell", dest="sqlmapShell", action="store_true",
help="Prompt for an interactive sqlmap shell")
miscellaneous.add_option("--wizard", dest="wizard",
action="store_true",
help="Simple wizard interface for beginner users")
# Hidden and/or experimental options
parser.add_option("--dummy", dest="dummy", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--pickled-options", dest="pickledOptions",
help=SUPPRESS_HELP)
parser.add_option("--profile", dest="profile", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--binary-fields", dest="binaryFields",
help=SUPPRESS_HELP)
parser.add_option("--cpu-throttle", dest="cpuThrottle", type="int",
help=SUPPRESS_HELP)
parser.add_option("--force-dns", dest="forceDns", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--smoke-test", dest="smokeTest", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--live-test", dest="liveTest", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--stop-fail", dest="stopFail", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--run-case", dest="runCase", help=SUPPRESS_HELP)
parser.add_option_group(target)
parser.add_option_group(request)
parser.add_option_group(optimization)
parser.add_option_group(injection)
parser.add_option_group(detection)
parser.add_option_group(techniques)
parser.add_option_group(fingerprint)
parser.add_option_group(enumeration)
parser.add_option_group(brute)
parser.add_option_group(udf)
parser.add_option_group(filesystem)
parser.add_option_group(takeover)
parser.add_option_group(windows)
parser.add_option_group(general)
parser.add_option_group(miscellaneous)
# Dirty hack to display longer options without breaking into two lines
def _(self, *args):
_ = parser.formatter._format_option_strings(*args)
if len(_) > MAX_HELP_OPTION_LENGTH:
_ = ("%%.%ds.." % (MAX_HELP_OPTION_LENGTH - parser.formatter.indent_increment)) % _
return _
parser.formatter._format_option_strings = parser.formatter.format_option_strings
parser.formatter.format_option_strings = type(parser.formatter.format_option_strings)(_, parser, type(parser))
# Dirty hack for making a short option -hh
option = parser.get_option("--hh")
option._short_opts = ["-hh"]
option._long_opts = []
# Dirty hack for inherent help message of switch -h
option = parser.get_option("-h")
option.help = option.help.capitalize().replace("this help", "basic help")
argv = []
prompt = False
advancedHelp = True
for arg in sys.argv:
argv.append(getUnicode(arg, system=True))
checkDeprecatedOptions(argv)
prompt = "--sqlmap-shell" in argv
if prompt:
parser.usage = ""
cmdLineOptions.sqlmapShell = True
_ = ["x", "q", "exit", "quit", "clear"]
for option in parser.option_list:
_.extend(option._long_opts)
_.extend(option._short_opts)
for group in parser.option_groups:
for option in group.option_list:
_.extend(option._long_opts)
_.extend(option._short_opts)
autoCompletion(AUTOCOMPLETE_TYPE.SQLMAP, commands=_)
while True:
command = None
try:
command = raw_input("sqlmap-shell> ").strip()
except (KeyboardInterrupt, EOFError):
print
raise SqlmapShellQuitException
if not command:
continue
elif command.lower() == "clear":
clearHistory()
print "[i] history cleared"
saveHistory(AUTOCOMPLETE_TYPE.SQLMAP)
elif command.lower() in ("x", "q", "exit", "quit"):
raise SqlmapShellQuitException
elif command[0] != '-':
print "[!] invalid option(s) provided"
print "[i] proper example: '-u http://www.site.com/vuln.php?id=1 --banner'"
else:
saveHistory(AUTOCOMPLETE_TYPE.SQLMAP)
loadHistory(AUTOCOMPLETE_TYPE.SQLMAP)
break
for arg in shlex.split(command):
argv.append(getUnicode(arg, system=True))
# Hide non-basic options in basic help case
for i in xrange(len(argv)):
if argv[i] == "-hh":
argv[i] = "-h"
elif argv[i] == "--version":
print VERSION_STRING.split('/')[-1]
raise SystemExit
elif argv[i] == "-h":
advancedHelp = False
for group in parser.option_groups[:]:
found = False
for option in group.option_list:
if option.dest not in BASIC_HELP_ITEMS:
option.help = SUPPRESS_HELP
else:
found = True
if not found:
parser.option_groups.remove(group)
try:
(args, _) = parser.parse_args(argv)
except SystemExit:
if "-h" in argv and not advancedHelp:
print "\n[!] to see full list of options run with '-hh'"
raise
# Expand given mnemonic options (e.g. -z "ign,flu,bat")
for i in xrange(len(argv) - 1):
if argv[i] == "-z":
expandMnemonics(argv[i + 1], parser, args)
if args.dummy:
args.url = args.url or DUMMY_URL
if not any((args.direct, args.url, args.logFile, args.bulkFile, args.googleDork, args.configFile, \
args.requestFile, args.updateAll, args.smokeTest, args.liveTest, args.wizard, args.dependencies, \
args.purgeOutput, args.pickledOptions, args.sitemapUrl)):
errMsg = "missing a mandatory option (-d, -u, -l, -m, -r, -g, -c, -x, --wizard, --update, --purge-output or --dependencies), "
errMsg += "use -h for basic or -hh for advanced help"
parser.error(errMsg)
return args
except (OptionError, TypeError), e:
parser.error(e)
def cmdLineParser(argv=None):
"""
This function parses the command line parameters and arguments
"""
if not argv:
argv = sys.argv
checkSystemEncoding()
# Reference: https://stackoverflow.com/a/4012683 (Note: previously used "...sys.getfilesystemencoding() or UNICODE_ENCODING")
_ = getUnicode(os.path.basename(argv[0]), encoding=sys.stdin.encoding)
usage = "%s%s [options]" % ("%s " % os.path.basename(sys.executable) if not IS_WIN else "", "\"%s\"" % _ if " " in _ else _)
parser = ArgumentParser(usage=usage)
try:
parser.add_argument("--hh", dest="advancedHelp", action="store_true",
help="Show advanced help message and exit")
parser.add_argument("--version", dest="showVersion", action="store_true",
help="Show program's version number and exit")
parser.add_argument("-v", dest="verbose", type=int,
help="Verbosity level: 0-6 (default %d)" % defaults.verbose)
# Target options
target = parser.add_argument_group("Target", "At least one of these options has to be provided to define the target(s)")
target.add_argument("-d", dest="direct",
help="Connection string for direct database connection")
target.add_argument("-u", "--url", dest="url",
help="Target URL (e.g. \"http://www.site.com/vuln.php?id=1\")")
target.add_argument("-l", dest="logFile",
help="Parse target(s) from Burp or WebScarab proxy log file")
target.add_argument("-m", dest="bulkFile",
help="Scan multiple targets given in a textual file ")
target.add_argument("-r", dest="requestFile",
help="Load HTTP request from a file")
target.add_argument("-g", dest="googleDork",
help="Process Google dork results as target URLs")
target.add_argument("-c", dest="configFile",
help="Load options from a configuration INI file")
# Request options
request = parser.add_argument_group("Request", "These options can be used to specify how to connect to the target URL")
request.add_argument("-A", "--user-agent", dest="agent",
help="HTTP User-Agent header value")
request.add_argument("-H", "--header", dest="header",
help="Extra header (e.g. \"X-Forwarded-For: 127.0.0.1\")")
request.add_argument("--method", dest="method",
help="Force usage of given HTTP method (e.g. PUT)")
request.add_argument("--data", dest="data",
help="Data string to be sent through POST (e.g. \"id=1\")")
request.add_argument("--param-del", dest="paramDel",
help="Character used for splitting parameter values (e.g. &)")
request.add_argument("--cookie", dest="cookie",
help="HTTP Cookie header value (e.g. \"PHPSESSID=a8d127e..\")")
request.add_argument("--cookie-del", dest="cookieDel",
help="Character used for splitting cookie values (e.g. ;)")
request.add_argument("--load-cookies", dest="loadCookies",
help="File containing cookies in Netscape/wget format")
request.add_argument("--drop-set-cookie", dest="dropSetCookie", action="store_true",
help="Ignore Set-Cookie header from response")
request.add_argument("--mobile", dest="mobile", action="store_true",
help="Imitate smartphone through HTTP User-Agent header")
request.add_argument("--random-agent", dest="randomAgent", action="store_true",
help="Use randomly selected HTTP User-Agent header value")
request.add_argument("--host", dest="host",
help="HTTP Host header value")
request.add_argument("--referer", dest="referer",
help="HTTP Referer header value")
request.add_argument("--headers", dest="headers",
help="Extra headers (e.g. \"Accept-Language: fr\\nETag: 123\")")
request.add_argument("--auth-type", dest="authType",
help="HTTP authentication type (Basic, Digest, NTLM or PKI)")
request.add_argument("--auth-cred", dest="authCred",
help="HTTP authentication credentials (name:password)")
request.add_argument("--auth-file", dest="authFile",
help="HTTP authentication PEM cert/private key file")
request.add_argument("--ignore-code", dest="ignoreCode",
help="Ignore (problematic) HTTP error code (e.g. 401)")
request.add_argument("--ignore-proxy", dest="ignoreProxy", action="store_true",
help="Ignore system default proxy settings")
request.add_argument("--ignore-redirects", dest="ignoreRedirects", action="store_true",
help="Ignore redirection attempts")
request.add_argument("--ignore-timeouts", dest="ignoreTimeouts", action="store_true",
help="Ignore connection timeouts")
request.add_argument("--proxy", dest="proxy",
help="Use a proxy to connect to the target URL")
request.add_argument("--proxy-cred", dest="proxyCred",
help="Proxy authentication credentials (name:password)")
request.add_argument("--proxy-file", dest="proxyFile",
help="Load proxy list from a file")
request.add_argument("--tor", dest="tor", action="store_true",
help="Use Tor anonymity network")
request.add_argument("--tor-port", dest="torPort",
help="Set Tor proxy port other than default")
request.add_argument("--tor-type", dest="torType",
help="Set Tor proxy type (HTTP, SOCKS4 or SOCKS5 (default))")
request.add_argument("--check-tor", dest="checkTor", action="store_true",
help="Check to see if Tor is used properly")
request.add_argument("--delay", dest="delay", type=float,
help="Delay in seconds between each HTTP request")
request.add_argument("--timeout", dest="timeout", type=float,
help="Seconds to wait before timeout connection (default %d)" % defaults.timeout)
request.add_argument("--retries", dest="retries", type=int,
help="Retries when the connection timeouts (default %d)" % defaults.retries)
request.add_argument("--randomize", dest="rParam",
help="Randomly change value for given parameter(s)")
request.add_argument("--safe-url", dest="safeUrl",
help="URL address to visit frequently during testing")
request.add_argument("--safe-post", dest="safePost",
help="POST data to send to a safe URL")
request.add_argument("--safe-req", dest="safeReqFile",
help="Load safe HTTP request from a file")
request.add_argument("--safe-freq", dest="safeFreq", type=int,
help="Test requests between two visits to a given safe URL")
request.add_argument("--skip-urlencode", dest="skipUrlEncode", action="store_true",
help="Skip URL encoding of payload data")
request.add_argument("--csrf-token", dest="csrfToken",
help="Parameter used to hold anti-CSRF token")
request.add_argument("--csrf-url", dest="csrfUrl",
help="URL address to visit for extraction of anti-CSRF token")
request.add_argument("--csrf-method", dest="csrfMethod",
help="HTTP method to use during anti-CSRF token page visit")
request.add_argument("--force-ssl", dest="forceSSL", action="store_true",
help="Force usage of SSL/HTTPS")
request.add_argument("--chunked", dest="chunked", action="store_true",
help="Use HTTP chunked transfer encoded (POST) requests")
request.add_argument("--hpp", dest="hpp", action="store_true",
help="Use HTTP parameter pollution method")
request.add_argument("--eval", dest="evalCode",
help="Evaluate provided Python code before the request (e.g. \"import hashlib;id2=hashlib.md5(id).hexdigest()\")")
# Optimization options
optimization = parser.add_argument_group("Optimization", "These options can be used to optimize the performance of sqlmap")
optimization.add_argument("-o", dest="optimize", action="store_true",
help="Turn on all optimization switches")
optimization.add_argument("--predict-output", dest="predictOutput", action="store_true",
help="Predict common queries output")
optimization.add_argument("--keep-alive", dest="keepAlive", action="store_true",
help="Use persistent HTTP(s) connections")
optimization.add_argument("--null-connection", dest="nullConnection", action="store_true",
help="Retrieve page length without actual HTTP response body")
optimization.add_argument("--threads", dest="threads", type=int,
help="Max number of concurrent HTTP(s) requests (default %d)" % defaults.threads)
# Injection options
injection = parser.add_argument_group("Injection", "These options can be used to specify which parameters to test for, provide custom injection payloads and optional tampering scripts")
injection.add_argument("-p", dest="testParameter",
help="Testable parameter(s)")
injection.add_argument("--skip", dest="skip",
help="Skip testing for given parameter(s)")
injection.add_argument("--skip-static", dest="skipStatic", action="store_true",
help="Skip testing parameters that not appear to be dynamic")
injection.add_argument("--param-exclude", dest="paramExclude",
help="Regexp to exclude parameters from testing (e.g. \"ses\")")
injection.add_argument("--param-filter", dest="paramFilter",
help="Select testable parameter(s) by place (e.g. \"POST\")")
injection.add_argument("--dbms", dest="dbms",
help="Force back-end DBMS to provided value")
injection.add_argument("--dbms-cred", dest="dbmsCred",
help="DBMS authentication credentials (user:password)")
injection.add_argument("--os", dest="os",
help="Force back-end DBMS operating system to provided value")
injection.add_argument("--invalid-bignum", dest="invalidBignum", action="store_true",
help="Use big numbers for invalidating values")
injection.add_argument("--invalid-logical", dest="invalidLogical", action="store_true",
help="Use logical operations for invalidating values")
injection.add_argument("--invalid-string", dest="invalidString", action="store_true",
help="Use random strings for invalidating values")
injection.add_argument("--no-cast", dest="noCast", action="store_true",
help="Turn off payload casting mechanism")
injection.add_argument("--no-escape", dest="noEscape", action="store_true",
help="Turn off string escaping mechanism")
injection.add_argument("--prefix", dest="prefix",
help="Injection payload prefix string")
injection.add_argument("--suffix", dest="suffix",
help="Injection payload suffix string")
injection.add_argument("--tamper", dest="tamper",
help="Use given script(s) for tampering injection data")
# Detection options
detection = parser.add_argument_group("Detection", "These options can be used to customize the detection phase")
detection.add_argument("--level", dest="level", type=int,
help="Level of tests to perform (1-5, default %d)" % defaults.level)
detection.add_argument("--risk", dest="risk", type=int,
help="Risk of tests to perform (1-3, default %d)" % defaults.risk)
detection.add_argument("--string", dest="string",
help="String to match when query is evaluated to True")
detection.add_argument("--not-string", dest="notString",
help="String to match when query is evaluated to False")
detection.add_argument("--regexp", dest="regexp",
help="Regexp to match when query is evaluated to True")
detection.add_argument("--code", dest="code", type=int,
help="HTTP code to match when query is evaluated to True")
detection.add_argument("--smart", dest="smart", action="store_true",
help="Perform thorough tests only if positive heuristic(s)")
detection.add_argument("--text-only", dest="textOnly", action="store_true",
help="Compare pages based only on the textual content")
detection.add_argument("--titles", dest="titles", action="store_true",
help="Compare pages based only on their titles")
# Techniques options
techniques = parser.add_argument_group("Techniques", "These options can be used to tweak testing of specific SQL injection techniques")
techniques.add_argument("--technique", dest="technique",
help="SQL injection techniques to use (default \"%s\")" % defaults.technique)
techniques.add_argument("--time-sec", dest="timeSec", type=int,
help="Seconds to delay the DBMS response (default %d)" % defaults.timeSec)
techniques.add_argument("--union-cols", dest="uCols",
help="Range of columns to test for UNION query SQL injection")
techniques.add_argument("--union-char", dest="uChar",
help="Character to use for bruteforcing number of columns")
techniques.add_argument("--union-from", dest="uFrom",
help="Table to use in FROM part of UNION query SQL injection")
techniques.add_argument("--dns-domain", dest="dnsDomain",
help="Domain name used for DNS exfiltration attack")
techniques.add_argument("--second-url", dest="secondUrl",
help="Resulting page URL searched for second-order response")
techniques.add_argument("--second-req", dest="secondReq",
help="Load second-order HTTP request from file")
# Fingerprint options
fingerprint = parser.add_argument_group("Fingerprint")
fingerprint.add_argument("-f", "--fingerprint", dest="extensiveFp", action="store_true",
help="Perform an extensive DBMS version fingerprint")
# Enumeration options
enumeration = parser.add_argument_group("Enumeration", "These options can be used to enumerate the back-end database management system information, structure and data contained in the tables")
enumeration.add_argument("-a", "--all", dest="getAll", action="store_true",
help="Retrieve everything")
enumeration.add_argument("-b", "--banner", dest="getBanner", action="store_true",
help="Retrieve DBMS banner")
enumeration.add_argument("--current-user", dest="getCurrentUser", action="store_true",
help="Retrieve DBMS current user")
enumeration.add_argument("--current-db", dest="getCurrentDb", action="store_true",
help="Retrieve DBMS current database")
enumeration.add_argument("--hostname", dest="getHostname", action="store_true",
help="Retrieve DBMS server hostname")
enumeration.add_argument("--is-dba", dest="isDba", action="store_true",
help="Detect if the DBMS current user is DBA")
enumeration.add_argument("--users", dest="getUsers", action="store_true",
help="Enumerate DBMS users")
enumeration.add_argument("--passwords", dest="getPasswordHashes", action="store_true",
help="Enumerate DBMS users password hashes")
enumeration.add_argument("--privileges", dest="getPrivileges", action="store_true",
help="Enumerate DBMS users privileges")
enumeration.add_argument("--roles", dest="getRoles", action="store_true",
help="Enumerate DBMS users roles")
enumeration.add_argument("--dbs", dest="getDbs", action="store_true",
help="Enumerate DBMS databases")
enumeration.add_argument("--tables", dest="getTables", action="store_true",
help="Enumerate DBMS database tables")
enumeration.add_argument("--columns", dest="getColumns", action="store_true",
help="Enumerate DBMS database table columns")
enumeration.add_argument("--schema", dest="getSchema", action="store_true",
help="Enumerate DBMS schema")
enumeration.add_argument("--count", dest="getCount", action="store_true",
help="Retrieve number of entries for table(s)")
enumeration.add_argument("--dump", dest="dumpTable", action="store_true",
help="Dump DBMS database table entries")
enumeration.add_argument("--dump-all", dest="dumpAll", action="store_true",
help="Dump all DBMS databases tables entries")
enumeration.add_argument("--search", dest="search", action="store_true",
help="Search column(s), table(s) and/or database name(s)")
enumeration.add_argument("--comments", dest="getComments", action="store_true",
help="Check for DBMS comments during enumeration")
enumeration.add_argument("--statements", dest="getStatements", action="store_true",
help="Retrieve SQL statements being run on DBMS")
enumeration.add_argument("-D", dest="db",
help="DBMS database to enumerate")
enumeration.add_argument("-T", dest="tbl",
help="DBMS database table(s) to enumerate")
enumeration.add_argument("-C", dest="col",
help="DBMS database table column(s) to enumerate")
enumeration.add_argument("-X", dest="exclude",
help="DBMS database identifier(s) to not enumerate")
enumeration.add_argument("-U", dest="user",
help="DBMS user to enumerate")
enumeration.add_argument("--exclude-sysdbs", dest="excludeSysDbs", action="store_true",
help="Exclude DBMS system databases when enumerating tables")
enumeration.add_argument("--pivot-column", dest="pivotColumn",
help="Pivot column name")
enumeration.add_argument("--where", dest="dumpWhere",
help="Use WHERE condition while table dumping")
enumeration.add_argument("--start", dest="limitStart", type=int,
help="First dump table entry to retrieve")
enumeration.add_argument("--stop", dest="limitStop", type=int,
help="Last dump table entry to retrieve")
enumeration.add_argument("--first", dest="firstChar", type=int,
help="First query output word character to retrieve")
enumeration.add_argument("--last", dest="lastChar", type=int,
help="Last query output word character to retrieve")
enumeration.add_argument("--sql-query", dest="sqlQuery",
help="SQL statement to be executed")
enumeration.add_argument("--sql-shell", dest="sqlShell", action="store_true",
help="Prompt for an interactive SQL shell")
enumeration.add_argument("--sql-file", dest="sqlFile",
help="Execute SQL statements from given file(s)")
# Brute force options
brute = parser.add_argument_group("Brute force", "These options can be used to run brute force checks")
brute.add_argument("--common-tables", dest="commonTables", action="store_true",
help="Check existence of common tables")
brute.add_argument("--common-columns", dest="commonColumns", action="store_true",
help="Check existence of common columns")
brute.add_argument("--common-files", dest="commonFiles", action="store_true",
help="Check existence of common files")
# User-defined function options
udf = parser.add_argument_group("User-defined function injection", "These options can be used to create custom user-defined functions")
udf.add_argument("--udf-inject", dest="udfInject", action="store_true",
help="Inject custom user-defined functions")
udf.add_argument("--shared-lib", dest="shLib",
help="Local path of the shared library")
# File system options
filesystem = parser.add_argument_group("File system access", "These options can be used to access the back-end database management system underlying file system")
filesystem.add_argument("--file-read", dest="fileRead",
help="Read a file from the back-end DBMS file system")
filesystem.add_argument("--file-write", dest="fileWrite",
help="Write a local file on the back-end DBMS file system")
filesystem.add_argument("--file-dest", dest="fileDest",
help="Back-end DBMS absolute filepath to write to")
# Takeover options
takeover = parser.add_argument_group("Operating system access", "These options can be used to access the back-end database management system underlying operating system")
takeover.add_argument("--os-cmd", dest="osCmd",
help="Execute an operating system command")
takeover.add_argument("--os-shell", dest="osShell", action="store_true",
help="Prompt for an interactive operating system shell")
takeover.add_argument("--os-pwn", dest="osPwn", action="store_true",
help="Prompt for an OOB shell, Meterpreter or VNC")
takeover.add_argument("--os-smbrelay", dest="osSmb", action="store_true",
help="One click prompt for an OOB shell, Meterpreter or VNC")
takeover.add_argument("--os-bof", dest="osBof", action="store_true",
help="Stored procedure buffer overflow "
"exploitation")
takeover.add_argument("--priv-esc", dest="privEsc", action="store_true",
help="Database process user privilege escalation")
takeover.add_argument("--msf-path", dest="msfPath",
help="Local path where Metasploit Framework is installed")
takeover.add_argument("--tmp-path", dest="tmpPath",
help="Remote absolute path of temporary files directory")
# Windows registry options
windows = parser.add_argument_group("Windows registry access", "These options can be used to access the back-end database management system Windows registry")
windows.add_argument("--reg-read", dest="regRead", action="store_true",
help="Read a Windows registry key value")
windows.add_argument("--reg-add", dest="regAdd", action="store_true",
help="Write a Windows registry key value data")
windows.add_argument("--reg-del", dest="regDel", action="store_true",
help="Delete a Windows registry key value")
windows.add_argument("--reg-key", dest="regKey",
help="Windows registry key")
windows.add_argument("--reg-value", dest="regVal",
help="Windows registry key value")
windows.add_argument("--reg-data", dest="regData",
help="Windows registry key value data")
windows.add_argument("--reg-type", dest="regType",
help="Windows registry key value type")
# General options
general = parser.add_argument_group("General", "These options can be used to set some general working parameters")
general.add_argument("-s", dest="sessionFile",
help="Load session from a stored (.sqlite) file")
general.add_argument("-t", dest="trafficFile",
help="Log all HTTP traffic into a textual file")
general.add_argument("--answers", dest="answers",
help="Set predefined answers (e.g. \"quit=N,follow=N\")")
general.add_argument("--batch", dest="batch", action="store_true",
help="Never ask for user input, use the default behavior")
general.add_argument("--binary-fields", dest="binaryFields",
help="Result fields having binary values (e.g. \"digest\")")
general.add_argument("--check-internet", dest="checkInternet", action="store_true",
help="Check Internet connection before assessing the target")
general.add_argument("--cleanup", dest="cleanup", action="store_true",
help="Clean up the DBMS from sqlmap specific UDF and tables")
general.add_argument("--crawl", dest="crawlDepth", type=int,
help="Crawl the website starting from the target URL")
general.add_argument("--crawl-exclude", dest="crawlExclude",
help="Regexp to exclude pages from crawling (e.g. \"logout\")")
general.add_argument("--csv-del", dest="csvDel",
help="Delimiting character used in CSV output (default \"%s\")" % defaults.csvDel)
general.add_argument("--charset", dest="charset",
help="Blind SQL injection charset (e.g. \"0123456789abcdef\")")
general.add_argument("--dump-format", dest="dumpFormat",
help="Format of dumped data (CSV (default), HTML or SQLITE)")
general.add_argument("--encoding", dest="encoding",
help="Character encoding used for data retrieval (e.g. GBK)")
general.add_argument("--eta", dest="eta", action="store_true",
help="Display for each output the estimated time of arrival")
general.add_argument("--flush-session", dest="flushSession", action="store_true",
help="Flush session files for current target")
general.add_argument("--forms", dest="forms", action="store_true",
help="Parse and test forms on target URL")
general.add_argument("--fresh-queries", dest="freshQueries", action="store_true",
help="Ignore query results stored in session file")
general.add_argument("--gpage", dest="googlePage", type=int,
help="Use Google dork results from specified page number")
general.add_argument("--har", dest="harFile",
help="Log all HTTP traffic into a HAR file")
general.add_argument("--hex", dest="hexConvert", action="store_true",
help="Use hex conversion during data retrieval")
general.add_argument("--output-dir", dest="outputDir", action="store",
help="Custom output directory path")
general.add_argument("--parse-errors", dest="parseErrors", action="store_true",
help="Parse and display DBMS error messages from responses")
general.add_argument("--preprocess", dest="preprocess",
help="Use given script(s) for preprocessing of response data")
general.add_argument("--repair", dest="repair", action="store_true",
help="Redump entries having unknown character marker (%s)" % INFERENCE_UNKNOWN_CHAR)
general.add_argument("--save", dest="saveConfig",
help="Save options to a configuration INI file")
general.add_argument("--scope", dest="scope",
help="Regexp to filter targets from provided proxy log")
general.add_argument("--skip-waf", dest="skipWaf", action="store_true",
help="Skip heuristic detection of WAF/IPS protection")
general.add_argument("--table-prefix", dest="tablePrefix",
help="Prefix used for temporary tables (default: \"%s\")" % defaults.tablePrefix)
general.add_argument("--test-filter", dest="testFilter",
help="Select tests by payloads and/or titles (e.g. ROW)")
general.add_argument("--test-skip", dest="testSkip",
help="Skip tests by payloads and/or titles (e.g. BENCHMARK)")
general.add_argument("--web-root", dest="webRoot",
help="Web server document root directory (e.g. \"/var/www\")")
# Miscellaneous options
miscellaneous = parser.add_argument_group("Miscellaneous", "These options do not fit into any other category")
miscellaneous.add_argument("-z", dest="mnemonics",
help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
miscellaneous.add_argument("--alert", dest="alert",
help="Run host OS command(s) when SQL injection is found")
miscellaneous.add_argument("--beep", dest="beep", action="store_true",
help="Beep on question and/or when SQL injection is found")
miscellaneous.add_argument("--dependencies", dest="dependencies", action="store_true",
help="Check for missing (optional) sqlmap dependencies")
miscellaneous.add_argument("--disable-coloring", dest="disableColoring", action="store_true",
help="Disable console output coloring")
miscellaneous.add_argument("--list-tampers", dest="listTampers", action="store_true",
help="Display list of available tamper scripts")
miscellaneous.add_argument("--offline", dest="offline", action="store_true",
help="Work in offline mode (only use session data)")
miscellaneous.add_argument("--purge", dest="purge", action="store_true",
help="Safely remove all content from sqlmap data directory")
miscellaneous.add_argument("--results-file", dest="resultsFile",
help="Location of CSV results file in multiple targets mode")
miscellaneous.add_argument("--sqlmap-shell", dest="sqlmapShell", action="store_true",
help="Prompt for an interactive sqlmap shell")
miscellaneous.add_argument("--tmp-dir", dest="tmpDir",
help="Local directory for storing temporary files")
miscellaneous.add_argument("--unstable", dest="unstable", action="store_true",
help="Adjust options for unstable connections")
miscellaneous.add_argument("--update", dest="updateAll", action="store_true",
help="Update sqlmap")
miscellaneous.add_argument("--wizard", dest="wizard", action="store_true",
help="Simple wizard interface for beginner users")
# Hidden and/or experimental options
parser.add_argument("--base64", dest="base64Parameter",
help=SUPPRESS) # "Parameter(s) containing Base64 encoded values"
parser.add_argument("--crack", dest="hashFile",
help=SUPPRESS) # "Load and crack hashes from a file (standalone)"
parser.add_argument("--dummy", dest="dummy", action="store_true",
help=SUPPRESS)
parser.add_argument("--murphy-rate", dest="murphyRate", type=int,
help=SUPPRESS)
parser.add_argument("--debug", dest="debug", action="store_true",
help=SUPPRESS)
parser.add_argument("--disable-precon", dest="disablePrecon", action="store_true",
help=SUPPRESS)
parser.add_argument("--disable-stats", dest="disableStats", action="store_true",
help=SUPPRESS)
parser.add_argument("--profile", dest="profile", action="store_true",
help=SUPPRESS)
parser.add_argument("--force-dbms", dest="forceDbms",
help=SUPPRESS)
parser.add_argument("--force-dns", dest="forceDns", action="store_true",
help=SUPPRESS)
parser.add_argument("--force-partial", dest="forcePartial", action="store_true",
help=SUPPRESS)
parser.add_argument("--force-pivoting", dest="forcePivoting", action="store_true",
help=SUPPRESS)
parser.add_argument("--gui", dest="gui", action="store_true",
help=SUPPRESS)
parser.add_argument("--smoke-test", dest="smokeTest", action="store_true",
help=SUPPRESS)
parser.add_argument("--live-test", dest="liveTest", action="store_true",
help=SUPPRESS)
parser.add_argument("--vuln-test", dest="vulnTest", action="store_true",
help=SUPPRESS)
parser.add_argument("--stop-fail", dest="stopFail", action="store_true",
help=SUPPRESS)
parser.add_argument("--run-case", dest="runCase",
help=SUPPRESS)
# API options
parser.add_argument("--api", dest="api", action="store_true",
help=SUPPRESS)
parser.add_argument("--taskid", dest="taskid",
help=SUPPRESS)
parser.add_argument("--database", dest="database",
help=SUPPRESS)
# Dirty hack to display longer options without breaking into two lines
if hasattr(parser, "formatter"):
def _(self, *args):
retVal = parser.formatter._format_option_strings(*args)
if len(retVal) > MAX_HELP_OPTION_LENGTH:
retVal = ("%%.%ds.." % (MAX_HELP_OPTION_LENGTH - parser.formatter.indent_increment)) % retVal
return retVal
parser.formatter._format_option_strings = parser.formatter.format_option_strings
parser.formatter.format_option_strings = type(parser.formatter.format_option_strings)(_, parser)
else:
def _format_action_invocation(self, action):
retVal = self.__format_action_invocation(action)
if len(retVal) > MAX_HELP_OPTION_LENGTH:
retVal = ("%%.%ds.." % (MAX_HELP_OPTION_LENGTH - self._indent_increment)) % retVal
return retVal
parser.formatter_class.__format_action_invocation = parser.formatter_class._format_action_invocation
parser.formatter_class._format_action_invocation = _format_action_invocation
# Dirty hack for making a short option '-hh'
if hasattr(parser, "get_option"):
option = parser.get_option("--hh")
option._short_opts = ["-hh"]
option._long_opts = []
else:
for action in get_actions(parser):
if action.option_strings == ["--hh"]:
action.option_strings = ["-hh"]
break
# Dirty hack for inherent help message of switch '-h'
if hasattr(parser, "get_option"):
option = parser.get_option("-h")
option.help = option.help.capitalize().replace("this help", "basic help")
else:
for action in get_actions(parser):
if action.option_strings == ["-h", "--help"]:
action.help = action.help.capitalize().replace("this help", "basic help")
break
_ = []
advancedHelp = True
extraHeaders = []
tamperIndex = None
# Reference: https://stackoverflow.com/a/4012683 (Note: previously used "...sys.getfilesystemencoding() or UNICODE_ENCODING")
for arg in argv:
_.append(getUnicode(arg, encoding=sys.stdin.encoding))
argv = _
checkOldOptions(argv)
if "--gui" in argv:
runGui(parser)
elif "--sqlmap-shell" in argv:
_createHomeDirectories()
parser.usage = ""
cmdLineOptions.sqlmapShell = True
commands = set(("x", "q", "exit", "quit", "clear"))
commands.update(get_all_options(parser))
autoCompletion(AUTOCOMPLETE_TYPE.SQLMAP, commands=commands)
while True:
command = None
try:
# Note: in Python2 command should not be converted to Unicode before passing to shlex (Reference: https://bugs.python.org/issue1170)
command = _input("sqlmap-shell> ").strip()
except (KeyboardInterrupt, EOFError):
print()
raise SqlmapShellQuitException
if not command:
continue
elif command.lower() == "clear":
clearHistory()
dataToStdout("[i] history cleared\n")
saveHistory(AUTOCOMPLETE_TYPE.SQLMAP)
elif command.lower() in ("x", "q", "exit", "quit"):
raise SqlmapShellQuitException
elif command[0] != '-':
dataToStdout("[!] invalid option(s) provided\n")
dataToStdout("[i] proper example: '-u http://www.site.com/vuln.php?id=1 --banner'\n")
else:
saveHistory(AUTOCOMPLETE_TYPE.SQLMAP)
loadHistory(AUTOCOMPLETE_TYPE.SQLMAP)
break
try:
for arg in shlex.split(command):
argv.append(getUnicode(arg, encoding=sys.stdin.encoding))
except ValueError as ex:
raise SqlmapSyntaxException("something went wrong during command line parsing ('%s')" % getSafeExString(ex))
for i in xrange(len(argv)):
longOptions = set(re.findall(r"\-\-([^= ]+?)=", parser.format_help()))
if argv[i] == "-hh":
argv[i] = "-h"
elif i == 1 and re.search(r"\A(http|www\.|\w[\w.-]+\.\w{2,})", argv[i]) is not None:
argv[i] = "--url=%s" % argv[i]
elif len(argv[i]) > 1 and all(ord(_) in xrange(0x2018, 0x2020) for _ in ((argv[i].split('=', 1)[-1].strip() or ' ')[0], argv[i][-1])):
dataToStdout("[!] copy-pasting illegal (non-console) quote characters from Internet is, well, illegal (%s)\n" % argv[i])
raise SystemExit
elif len(argv[i]) > 1 and u"\uff0c" in argv[i].split('=', 1)[-1]:
dataToStdout("[!] copy-pasting illegal (non-console) comma characters from Internet is, well, illegal (%s)\n" % argv[i])
raise SystemExit
elif re.search(r"\A-\w=.+", argv[i]):
dataToStdout("[!] potentially miswritten (illegal '=') short option detected ('%s')\n" % argv[i])
raise SystemExit
elif argv[i] in DEPRECATED_OPTIONS:
argv[i] = ""
elif argv[i].startswith("--tamper"):
if tamperIndex is None:
tamperIndex = i if '=' in argv[i] else (i + 1 if i + 1 < len(argv) and not argv[i + 1].startswith('-') else None)
else:
argv[tamperIndex] = "%s,%s" % (argv[tamperIndex], argv[i].split('=')[1] if '=' in argv[i] else (argv[i + 1] if i + 1 < len(argv) and not argv[i + 1].startswith('-') else ""))
argv[i] = ""
elif argv[i] == "-H":
if i + 1 < len(argv):
extraHeaders.append(argv[i + 1])
elif argv[i] == "-r":
for j in xrange(i + 2, len(argv)):
value = argv[j]
if os.path.isfile(value):
argv[i + 1] += ",%s" % value
argv[j] = ''
else:
break
elif re.match(r"\A\d+!\Z", argv[i]) and argv[max(0, i - 1)] == "--threads" or re.match(r"\A--threads.+\d+!\Z", argv[i]):
argv[i] = argv[i][:-1]
conf.skipThreadCheck = True
elif argv[i] == "--version":
print(VERSION_STRING.split('/')[-1])
raise SystemExit
elif argv[i] in ("-h", "--help"):
advancedHelp = False
for group in get_groups(parser)[:]:
found = False
for option in get_actions(group):
if option.dest not in BASIC_HELP_ITEMS:
option.help = SUPPRESS
else:
found = True
if not found:
get_groups(parser).remove(group)
elif '=' in argv[i] and not argv[i].startswith('-') and argv[i].split('=')[0] in longOptions and re.search(r"\A-\w\Z", argv[i - 1]) is None:
dataToStdout("[!] detected usage of long-option without a starting hyphen ('%s')\n" % argv[i])
raise SystemExit
for verbosity in (_ for _ in argv if re.search(r"\A\-v+\Z", _)):
try:
if argv.index(verbosity) == len(argv) - 1 or not argv[argv.index(verbosity) + 1].isdigit():
conf.verbose = verbosity.count('v') + 1
del argv[argv.index(verbosity)]
except (IndexError, ValueError):
pass
try:
(args, _) = parser.parse_known_args(argv) if hasattr(parser, "parse_known_args") else parser.parse_args(argv)
except UnicodeEncodeError as ex:
dataToStdout("\n[!] %s\n" % getUnicode(ex.object.encode("unicode-escape")))
raise SystemExit
except SystemExit:
if "-h" in argv and not advancedHelp:
dataToStdout("\n[!] to see full list of options run with '-hh'\n")
raise
if extraHeaders:
if not args.headers:
args.headers = ""
delimiter = "\\n" if "\\n" in args.headers else "\n"
args.headers += delimiter + delimiter.join(extraHeaders)
# Expand given mnemonic options (e.g. -z "ign,flu,bat")
for i in xrange(len(argv) - 1):
if argv[i] == "-z":
expandMnemonics(argv[i + 1], parser, args)
if args.dummy:
args.url = args.url or DUMMY_URL
if not any((args.direct, args.url, args.logFile, args.bulkFile, args.googleDork, args.configFile, args.requestFile, args.updateAll, args.smokeTest, args.vulnTest, args.liveTest, args.wizard, args.dependencies, args.purge, args.listTampers, args.hashFile)):
errMsg = "missing a mandatory option (-d, -u, -l, -m, -r, -g, -c, --list-tampers, --wizard, --update, --purge or --dependencies). "
errMsg += "Use -h for basic and -hh for advanced help\n"
parser.error(errMsg)
return args
except (ArgumentError, TypeError) as ex:
parser.error(ex)
except SystemExit:
# Protection against Windows dummy double clicking
if IS_WIN:
dataToStdout("\nPress Enter to continue...")
_input()
raise
debugMsg = "parsing command line"
logger.debug(debugMsg)
action="store_true",
dest="view",
help="Display the graph.",
default=False)
parser.add_option("-w",
"--wcet-all",
action="store_true",
dest="wcet_all",
help="Plot WCET estimate line derived from all capacity constraints.",
default=False)
(opts, args) = parser.parse_args(argv[1:])
if opts.report is None:
print("Missing option " + str(parser.get_option("-r")))
exit(0)
elif opts.program is None:
print("Missing option " + str(parser.get_option("-p")))
exit(0)
xAxis = []
metLine = []
wcet_All_Line = []
wcet_DFS_Line = []
actualWCETLine = []
lowestMET = maxint
lowestWCET = maxint
highestMET = 0
highestWCET = 0
xAxisUpperLimit = 1
class CmdBase(object):
doesLogging = True
"""
Class used for all Zenoss commands
"""
def __init__(self, noopts=0, args=None, should_log=None):
zope.component.provideAdapter(DefaultTraversable, (None, ))
# This explicitly loads all of the products - must happen first!
from OFS.Application import import_products
import_products()
#make sure we aren't in debug mode
import Globals
Globals.DevelopmentMode = False
# We must import ZenossStartup at this point so that all Zenoss daemons
# and tools will have any ZenPack monkey-patched methods available.
import Products.ZenossStartup
unused(Products.ZenossStartup)
zcml.load_site()
import Products.ZenWidgets
load_config_override('scriptmessaging.zcml', Products.ZenWidgets)
self.usage = "%prog [options]"
self.noopts = noopts
self.inputArgs = args
# inputArgs was created to allow unit tests to pass in command line
# arguments and get around whatever Zope was doing to sys.argv.
if self.inputArgs is None:
self.inputArgs = sys.argv[1:]
self.parser = None
self.args = []
self.buildParser()
self.buildOptions()
# Get defaults from global.conf. They will be overridden by
# daemon-specific config file or command line arguments.
applyGlobalConfToParser(self.parser)
self.parseOptions()
if self.options.configfile:
self.parser.defaults = self.getConfigFileDefaults(
self.options.configfile)
# We've updated the parser with defaults from configs, now we need
# to reparse our command-line to get the correct overrides from
# the command-line
self.parseOptions()
if should_log is not None:
self.doesLogging = should_log
if self.doesLogging:
self.setupLogging()
def buildParser(self):
"""
Create the options parser
"""
if not self.parser:
from Products.ZenModel.ZenossInfo import ZenossInfo
try:
zinfo = ZenossInfo('')
version = str(zinfo.getZenossVersion())
except Exception:
from Products.ZenModel.ZVersion import VERSION
version = VERSION
self.parser = OptionParser(usage=self.usage,
version="%prog " + version,
option_class=LogSeverityOption)
def buildOptions(self):
"""
Basic options setup. Other classes should call this before adding
more options
"""
self.buildParser()
if self.doesLogging:
group = OptionGroup(self.parser, "Logging Options")
group.add_option(
'-v',
'--logseverity',
dest='logseverity',
default='INFO',
type='loglevel',
help='Logging severity threshold',
)
group.add_option(
'--logpath',
dest='logpath',
default=zenPath('log'),
type='str',
help='Override the default logging path; default $ZENHOME/log')
group.add_option(
'--maxlogsize',
dest='maxLogKiloBytes',
default=10240,
type='int',
help='Max size of log file in KB; default 10240',
)
group.add_option(
'--maxbackuplogs',
dest='maxBackupLogs',
default=3,
type='int',
help='Max number of back up log files; default 3',
)
self.parser.add_option_group(group)
self.parser.add_option("-C",
"--configfile",
dest="configfile",
help="Use an alternate configuration file")
self.parser.add_option("--genconf",
action="store_true",
default=False,
help="Generate a template configuration file")
self.parser.add_option(
"--genxmltable",
action="store_true",
default=False,
help="Generate a Docbook table showing command-line switches.")
self.parser.add_option(
"--genxmlconfigs",
action="store_true",
default=False,
help="Generate an XML file containing command-line switches.")
def parseOptions(self):
"""
Uses the optparse parse previously populated and performs common options.
"""
if self.noopts:
args = []
else:
args = self.inputArgs
(self.options, self.args) = self.parser.parse_args(args=args)
if self.options.genconf:
self.generate_configs(self.parser, self.options)
if self.options.genxmltable:
self.generate_xml_table(self.parser, self.options)
if self.options.genxmlconfigs:
self.generate_xml_configs(self.parser, self.options)
def getConfigFileDefaults(self, filename, correctErrors=True):
# TODO: This should be refactored - duplicated code with CmdBase.
"""
Parse a config file which has key-value pairs delimited by white space,
and update the parser's option defaults with these values.
@parameter filename: name of configuration file
@type filename: string
"""
options = self.parser.get_default_values()
lines = self.loadConfigFile(filename)
if lines:
lines, errors = self.validateConfigFile(
filename, lines, correctErrors=correctErrors)
args = self.getParamatersFromConfig(lines)
try:
self.parser._process_args([], args, options)
except (BadOptionError, OptionValueError) as err:
print >> sys.stderr, 'WARN: %s in config file %s' % (err,
filename)
return options.__dict__
def getGlobalConfigFileDefaults(self):
# Deprecated: This method is going away - it is duplicated in GlobalConfig.py
"""
Parse a config file which has key-value pairs delimited by white space,
and update the parser's option defaults with these values.
"""
filename = zenPath('etc', 'global.conf')
options = self.parser.get_default_values()
lines = self.loadConfigFile(filename)
if lines:
args = self.getParamatersFromConfig(lines)
try:
self.parser._process_args([], args, options)
except (BadOptionError, OptionValueError):
# Ignore it, we only care about our own options as defined in the parser
pass
return options.__dict__
def loadConfigFile(self, filename):
# TODO: This should be refactored - duplicated code with CmdBase.
"""
Parse a config file which has key-value pairs delimited by white space.
@parameter filename: path to the configuration file
@type filename: string
"""
lines = []
if not os.path.exists(filename):
return lines
try:
with open(filename) as file:
for line in file:
if line.lstrip().startswith('#') or line.strip() == '':
lines.append(dict(type='comment', line=line))
else:
try:
# add default blank string for keys with no default value
# valid delimiters are space, ':' and/or '=' (see ZenUtils/config.py)
key, value = (
re.split(r'[\s:=]+', line.strip(), 1) + [
'',
])[:2]
except ValueError:
lines.append(
dict(type='option',
line=line,
key=line.strip(),
value=None,
option=None))
else:
option = self.parser.get_option('--%s' % key)
lines.append(
dict(type='option',
line=line,
key=key,
value=value,
option=option))
except IOError as e:
errorMessage = (
'WARN: unable to read config file {filename} '
'-- skipping. ({exceptionName}: {exception})').format(
filename=filename,
exceptionName=e.__class__.__name__,
exception=e)
print >> sys.stderr, errorMessage
return []
return lines
def validateConfigFile(self,
filename,
lines,
correctErrors=True,
warnErrors=True):
"""
Validate config file lines which has key-value pairs delimited by white space,
and validate that the keys exist for this command's option parser. If
the option does not exist or has an empty value it will comment it out
in the config file.
@parameter filename: path to the configuration file
@type filename: string
@parameter lines: lines from config parser
@type lines: list
@parameter correctErrors: Whether or not invalid conf values should be
commented out.
@type correctErrors: boolean
"""
output = []
errors = []
validLines = []
date = datetime.datetime.now().isoformat()
errorTemplate = '## Commenting out by config parser (%s) on %s: %%s\n' % (
sys.argv[0], date)
for lineno, line in enumerate(lines):
if line['type'] == 'comment':
output.append(line['line'])
elif line['type'] == 'option':
if line['value'] is None:
errors.append(
(lineno + 1, 'missing value for "%s"' % line['key']))
output.append(errorTemplate % 'missing value')
output.append('## %s' % line['line'])
elif line['option'] is None:
errors.append(
(lineno + 1, 'unknown option "%s"' % line['key']))
output.append(errorTemplate % 'unknown option')
output.append('## %s' % line['line'])
else:
validLines.append(line)
output.append(line['line'])
else:
errors.append((lineno + 1, 'unknown line "%s"' % line['line']))
output.append(errorTemplate % 'unknown line')
output.append('## %s' % line['line'])
if errors:
if correctErrors:
for lineno, message in errors:
print >> sys.stderr, 'INFO: Commenting out %s on line %d in %s' % (
message, lineno, filename)
with open(filename, 'w') as file:
file.writelines(output)
if warnErrors:
for lineno, message in errors:
print >> sys.stderr, 'WARN: %s on line %d in %s' % (
message, lineno, filename)
return validLines, errors
def getParamatersFromConfig(self, lines):
# Deprecated: This method is going away
return _convertConfigLinesToArguments(self.parser, lines)
def setupLogging(self):
"""
Set common logging options
"""
rlog = logging.getLogger()
rlog.setLevel(logging.WARN)
mname = self.__class__.__name__
self.log = logging.getLogger("zen." + mname)
zlog = logging.getLogger("zen")
try:
loglevel = int(self.options.logseverity)
except ValueError:
loglevel = getattr(logging, self.options.logseverity.upper(),
logging.INFO)
zlog.setLevel(loglevel)
logdir = self.checkLogpath()
if logdir:
logfile = os.path.join(logdir, mname.lower() + ".log")
maxBytes = self.options.maxLogKiloBytes * 1024
backupCount = self.options.maxBackupLogs
h = logging.handlers.RotatingFileHandler(logfile,
maxBytes=maxBytes,
backupCount=backupCount)
h.setFormatter(
logging.Formatter(
"%(asctime)s %(levelname)s %(name)s: %(message)s",
"%Y-%m-%d %H:%M:%S"))
rlog.addHandler(h)
else:
logging.basicConfig()
def checkLogpath(self):
"""
Validate the logpath is valid
"""
if not self.options.logpath:
return None
else:
logdir = self.options.logpath
if not os.path.exists(logdir):
# try creating the directory hierarchy if it doesn't exist...
try:
os.makedirs(logdir)
except OSError:
raise SystemExit(
"logpath:%s doesn't exist and cannot be created" %
logdir)
elif not os.path.isdir(logdir):
raise SystemExit("logpath:%s exists but is not a directory" %
logdir)
return logdir
def pretty_print_config_comment(self, comment):
"""
Quick and dirty pretty printer for comments that happen to be longer than can comfortably
be seen on the display.
"""
max_size = 40
#
# As a heuristic we'll accept strings that are +- text_window
# size in length.
#
text_window = 5
if len(comment) <= max_size + text_window:
return comment
#
# First, take care of embedded newlines and expand them out to array entries
#
new_comment = []
all_lines = comment.split('\n')
for line in all_lines:
if len(line) <= max_size + text_window:
new_comment.append(line)
continue
start_position = max_size - text_window
while len(line) > max_size + text_window:
index = line.find(' ', start_position)
if index > 0:
new_comment.append(line[0:index])
line = line[index:]
else:
if start_position == 0:
#
# If we get here it means that the line is just one big string with no spaces
# in it. There's nothing that we can do except print it out. Doh!
#
new_comment.append(line)
break
#
# Okay, haven't found anything to split on -- go back and try again
#
start_position = start_position - text_window
if start_position < 0:
start_position = 0
else:
new_comment.append(line)
return "\n# ".join(new_comment)
def generate_configs(self, parser, options):
"""
Create a configuration file based on the long-form of the option names
@parameter parser: an optparse parser object which contains defaults, help
@parameter options: parsed options list containing actual values
"""
#
# Header for the configuration file
#
unused(options)
daemon_name = os.path.basename(sys.argv[0])
daemon_name = daemon_name.replace('.py', '')
print """#
# Configuration file for %s
#
# To enable a particular option, uncomment the desired entry.
#
# Parameter Setting
# --------- -------""" % (daemon_name)
options_to_ignore = ('help', 'version', '', 'genconf', 'genxmltable')
#
# Create an entry for each of the command line flags
#
# NB: Ideally, this should print out only the option parser dest
# entries, rather than the command line options.
#
import re
for opt in getAllParserOptionsGen(parser):
if opt.help is SUPPRESS_HELP:
continue
#
# Get rid of the short version of the command
#
option_name = re.sub(r'.*/--', '', "%s" % opt)
#
# And what if there's no short version?
#
option_name = re.sub(r'^--', '', "%s" % option_name)
#
# Don't display anything we shouldn't be displaying
#
if option_name in options_to_ignore:
continue
#
# Find the actual value specified on the command line, if any,
# and display it
#
value = getattr(parser.values, opt.dest)
default_value = parser.defaults.get(opt.dest)
if default_value is NO_DEFAULT or default_value is None:
default_value = ""
default_string = ""
if default_value != "":
default_string = ", default: " + str(default_value)
comment = self.pretty_print_config_comment(opt.help +
default_string)
#
# NB: I would prefer to use tabs to separate the parameter name
# and value, but I don't know that this would work.
#
print """#
# %s
#%s %s""" % (comment, option_name, value)
#
# Pretty print and exit
#
print "#"
sys.exit(0)
def generate_xml_table(self, parser, options):
"""
Create a Docbook table based on the long-form of the option names
@parameter parser: an optparse parser object which contains defaults, help
@parameter options: parsed options list containing actual values
"""
#
# Header for the configuration file
#
unused(options)
daemon_name = os.path.basename(sys.argv[0])
daemon_name = daemon_name.replace('.py', '')
print """<?xml version="1.0" encoding="UTF-8"?>
<section version="4.0" xmlns="http://docbook.org/ns/docbook"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns:mml="http://www.w3.org/1998/Math/MathML"
xmlns:html="http://www.w3.org/1999/xhtml"
xmlns:db="http://docbook.org/ns/docbook"
xml:id="%s.options"
>
<title>%s Options</title>
<para />
<table frame="all">
<caption>%s <indexterm><primary>Daemons</primary><secondary>%s</secondary></indexterm> options</caption>
<tgroup cols="2">
<colspec colname="option" colwidth="1*" />
<colspec colname="description" colwidth="2*" />
<thead>
<row>
<entry> <para>Option</para> </entry>
<entry> <para>Description</para> </entry>
</row>
</thead>
<tbody>
""" % (daemon_name, daemon_name, daemon_name, daemon_name)
options_to_ignore = ('help', 'version', '', 'genconf', 'genxmltable')
#
# Create an entry for each of the command line flags
#
# NB: Ideally, this should print out only the option parser dest
# entries, rather than the command line options.
#
import re
for opt in getAllParserOptionsGen(parser):
if opt.help is SUPPRESS_HELP:
continue
#
# Create a Docbook-happy version of the option strings
# Yes, <arg></arg> would be better semantically, but the output
# just looks goofy in a table. Use literal instead.
#
all_options = '<literal>' + re.sub(
r'/', '</literal>,</para> <para><literal>',
"%s" % opt) + '</literal>'
#
# Don't display anything we shouldn't be displaying
#
option_name = re.sub(r'.*/--', '', "%s" % opt)
option_name = re.sub(r'^--', '', "%s" % option_name)
if option_name in options_to_ignore:
continue
default_value = parser.defaults.get(opt.dest)
if default_value is NO_DEFAULT or default_value is None:
default_value = ""
default_string = ""
if default_value != "":
default_string = "<para> Default: <literal>" + str(
default_value) + "</literal></para>\n"
comment = self.pretty_print_config_comment(opt.help)
#
# TODO: Determine the variable name used and display the --option_name=variable_name
#
if opt.action in ['store_true', 'store_false']:
print """<row>
<entry> <para>%s</para> </entry>
<entry>
<para>%s</para>
%s</entry>
</row>
""" % (all_options, comment, default_string)
else:
target = '=<replaceable>' + opt.dest.lower() + '</replaceable>'
all_options = all_options + target
all_options = re.sub(r',', target + ',', all_options)
print """<row>
<entry> <para>%s</para> </entry>
<entry>
<para>%s</para>
%s</entry>
</row>
""" % (all_options, comment, default_string)
#
# Close the table elements
#
print """</tbody></tgroup>
</table>
<para />
</section>
"""
sys.exit(0)
def generate_xml_configs(self, parser, options):
"""
Create an XML file that can be used to create Docbook files
as well as used as the basis for GUI-based daemon option
configuration.
"""
#
# Header for the configuration file
#
unused(options)
daemon_name = os.path.basename(sys.argv[0])
daemon_name = daemon_name.replace('.py', '')
export_date = datetime.datetime.now()
print """<?xml version="1.0" encoding="UTF-8"?>
<!-- Default daemon configuration generated on %s -->
<configuration id="%s" >
""" % (export_date, daemon_name)
options_to_ignore = (
'help',
'version',
'',
'genconf',
'genxmltable',
'genxmlconfigs',
)
#
# Create an entry for each of the command line flags
#
# NB: Ideally, this should print out only the option parser dest
# entries, rather than the command line options.
#
import re
for opt in getAllParserOptionsGen(parser):
if opt.help is SUPPRESS_HELP:
continue
#
# Don't display anything we shouldn't be displaying
#
option_name = re.sub(r'.*/--', '', "%s" % opt)
option_name = re.sub(r'^--', '', "%s" % option_name)
if option_name in options_to_ignore:
continue
default_value = parser.defaults.get(opt.dest)
if default_value is NO_DEFAULT or default_value is None:
default_string = ""
else:
default_string = str(default_value)
#
# TODO: Determine the variable name used and display the --option_name=variable_name
#
if opt.action in ['store_true', 'store_false']:
print """ <option id="%s" type="%s" default="%s" help="%s" />
""" % (
option_name,
"boolean",
default_string,
quote(opt.help),
)
else:
target = opt.dest.lower()
print """ <option id="%s" type="%s" default="%s" target="%s" help="%s" />
""" % (
option_name,
opt.type,
quote(default_string),
target,
quote(opt.help),
)
#
# Close the table elements
#
print """
</configuration>
"""
sys.exit(0)
class BasicScript(object):
#=====================
def __init__(self,optargs=sys.argv[1:],quiet=False):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.quiet = quiet
self.timestamp = apParam.makeTimestamp()
if not self.quiet:
apDisplay.printMsg("Time stamp: "+self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
if not self.quiet:
apDisplay.printMsg("Function name: "+self.functionname)
apParam.setUmask()
self.parsePythonPath()
loadavg = os.getloadavg()[0]
if loadavg > 2.0:
apDisplay.printMsg("Load average is %.2f, wait for %.1f second " % (round(loadavg,2),loadavg**2))
time.sleep(loadavg**2)
apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
### setup default parser: run directory, etc.
self.setParams(optargs)
self.checkConflicts()
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### any custom init functions go here
self.onInit()
def setParams(self,optargs=sys.argv[1:]):
self.parser = OptionParser()
self.setupParserOptions()
self.params = apParam.convertParserToParams(self.parser)
self.checkForDuplicateCommandLineInputs(optargs)
#=====================
def checkForDuplicateCommandLineInputs(self,optargs=sys.argv[1:]):
args = optargs
argmdict = {}
for arg in args:
elements=arg.split('=')
opt = elements[0].lower()
if opt[0] == "-":
## if action='append', then opt is allowed multiple times
option = self.parser.get_option(opt)
if option is not None and option.action == 'append':
multiple_ok = True
else:
multiple_ok = False
if opt in argmdict and not multiple_ok:
apDisplay.printError("Multiple arguments were supplied for argument: "+str(opt))
argmdict[opt] = True
#=====================
def createDefaultStats(self):
self.stats = {}
self.stats['starttime'] = time.time()
self.stats['count'] = 1
self.stats['lastcount'] = 0
self.stats['startmem'] = mem.active()
self.stats['memleak'] = 0
self.stats['peaksum'] = 0
self.stats['lastpeaks'] = None
self.stats['imagesleft'] = 1
self.stats['peaksumsq'] = 0
self.stats['timesum'] = 0
self.stats['timesumsq'] = 0
self.stats['skipcount'] = 0
self.stats['waittime'] = 0
self.stats['lastimageskipped'] = False
self.stats['notpair'] = 0
self.stats['memlist'] = [mem.active()]
#=====================
def close(self):
self.onClose()
loadavg = os.getloadavg()[0]
if loadavg > 2.0:
apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
time.sleep(loadavg**2)
apParam.closeFunctionLog(functionname=self.functionname,
logfile=self.logfile, msg=(not self.quiet))
if self.quiet is False:
apDisplay.printMsg("Ended at "+time.strftime("%a, %d %b %Y %H:%M:%S"))
apDisplay.printMsg("Memory increase during run: %.3f MB"%((mem.active()-self.startmem)/1024.0))
apDisplay.printColor("Total run time:\t"+apDisplay.timeString(time.time()-self.t0),"green")
#=====================
def parsePythonPath(self):
pythonpath = os.environ.get("PYTHONPATH")
if pythonpath is None:
return
paths = pythonpath.split(":")
leginons = {}
appions = {}
for p in paths:
if "appion" in p:
appions[p] = None
if "leginon" in p:
leginons[p] = None
leginons = leginons.keys()
appions = appions.keys()
if len(appions) > 1:
apDisplay.printWarning("There is more than one appion directory in your PYTHONPATH")
print appions
if len(leginons) > 1:
apDisplay.printWarning("There is more than one leginon directory in your PYTHONPATH")
print leginons
#######################################################
#### ITEMS BELOW CAN BE SPECIFIED IN A NEW PROGRAM ####
#######################################################
#=====================
def setupParserOptions(self):
"""
set the input parameters
this function should be rewritten in each program
"""
apDisplay.printError("you did not create a 'setupParserOptions' function in your script")
self.parser.set_usage("Usage: %prog --commit --description='<text>' [options]")
self.parser.add_option("--stackid", dest="stackid", type="int",
help="ID for particle stack (optional)", metavar="INT")
#=====================
def checkConflicts(self):
"""
make sure the necessary parameters are set correctly
"""
apDisplay.printError("you did not create a 'checkConflicts' function in your script")
if self.params['runname'] is None:
apDisplay.printError("enter a run name ID, e.g. --runname=run1")
if self.params['description'] is None:
apDisplay.printError("enter a description, e.g. --description='awesome data'")
#=====================
def start(self):
"""
this is the main component of the script
where all the processing is done
"""
raise NotImplementedError()
#=====================
def onInit(self):
return
#=====================
def onClose(self):
return
def parse_options():
parser = OptionParser(usage="Usage: %prog [options]")
parser.add_option('--config',
action='store',
type='string',
default='',
help="Bot config file")
parser.add_option('--kafka_config',
action='store',
type='string',
default='',
help="Bot kafka config file")
parser.add_option(
'--use_curl',
action='store_true',
default=False,
help=
"if set then use CURL multi connections, otherwise UserStream greenlets"
)
parser.add_option('--username',
action='store',
type='string',
default=get_var('TWITTER_BOT_USERNAME',
'*****@*****.**'),
help="[default: %%default]")
parser.add_option(
'--password',
action='store',
type='string',
help=
'Solariat user password (required when --post_creator=http_post_api)')
parser.add_option(
'--url',
action='store',
type='string',
help='Base url (required when --post_creator=http_post_api)')
parser.add_option(
'--post_creator',
action='store',
type='string',
default='factory_by_user',
help=
'Post creation strategy (factory_by_user | http_post_api) [default: %%default]'
)
parser.add_option('--post_creator_senders',
action='store',
type='int',
default=4,
help='Number of each post creator http senders')
parser.add_option('--bot_user_agent',
action='store',
type='string',
default=None)
parser.add_option('--pidfile', action='store', type='string')
parser.add_option('--lockfile',
action='store',
type='string',
default=get_default_lockfile(),
help="[default: %default]")
parser.add_option('--dumpfile',
action='store',
type='string',
help='file to dump incoming posts')
parser.add_option(
'--concurrency',
metavar='N',
action='store',
type='int',
default=2,
help="number of post-creating green threads [default: %default]")
parser.add_option(
'--multiprocess_concurrency',
metavar='N1',
action='store',
type='int',
default=0,
help=
"number of post-creating processes, 0 - do not fork, start greenlets in current process [default: %default]"
)
parser.add_option('--mode',
action='store',
type='string',
default='dev',
help="app mode: %s [default: %%default]" %
', '.join(DB_NAME_MAP))
parser.add_option('--only_accounts',
action='store',
type='string',
help='accounts to track only, comma separated ids')
parser.add_option('--exclude_accounts',
action='store',
type='string',
help='exclude accounts, comma separated ids')
options, args = parser.parse_args()
config_from_file = {}
if options.config:
with open(options.config) as conf:
try:
config_from_file = yaml.load(conf)
except:
print("Cannot load config from file %s" % options.config)
# merge options from config file,
# command-line passed arguments have priority
for opt, value in config_from_file.items():
opt_val = getattr(options, opt, None)
parser_opt = parser.get_option('--' + opt)
if not opt_val or opt_val is None or not parser_opt or parser_opt.default == opt_val:
setattr(options, opt, value)
if not options.username:
parser.print_help()
sys.exit(1)
if options.post_creator == 'http_post_api' and not (options.password
and options.url):
parser.print_help()
sys.exit(1)
if options.pidfile:
open(options.pidfile, "w+").write(str(os.getpid()))
if not hasattr(options, 'use_kafka'):
setattr(options, 'use_kafka', False)
return options
locale.setlocale(locale.LC_ALL, '')
import lazygal
from lazygal.generators import Album
import lazygal.config
import lazygal.eyecandy
import lazygal.log
usage = _("usage: %prog [options] albumdir")
parser = OptionParser(usage=usage)
# The help option must be changed to comply with i18n.
parser.get_option('-h').help = _("Show this help message and exit.")
parser.add_option("", "--quiet",
action="store_true",
dest="quiet",
help=_("Don't output anything except for errors."))
parser.add_option("", "--debug",
action="store_true",
dest="debug",
help=_("Output everything that lazygal is doing."))
parser.add_option("-o", "--output-directory",
action="store", type="string",
dest="dest_dir",
help=_("Directory where web pages, slides and thumbs will be written (default is current directory)."))
parser.add_option("-t", "--theme",
action="store", type="string",
def prepare_options(cgi_mode):
from optparse import OptionParser, OptionValueError, OptionGroup
parser = OptionParser(usage="""Usage: %prog [ -z <target-zone>
-x <zone-xfr-from> -t <ttl> --output-rrset -d <domain> -s <service>
-n <instance-name> --instname-sed PATTERN REPL]""",
description="""DNS-SD browser that converts
avahi-browse output to a DNS zone/rrset""",
epilog=None)
parser.add_option('-z',
'--target-zone',
default='example.com',
help='Target zone')
parser.add_option('-x',
'--zone-xfr-from',
default='localhost',
help='DNS server to transfer target zone from')
parser.add_option('-t',
'--ttl',
type="int",
default=1800,
help="""TTL for created DNS resource records (default:
%default)""")
parser.add_option('--output-rrset',
action="store_true",
default=False,
help="""Return only created DNS resource records rather
than a full zone (default: %default)""")
parser.add_option('-f',
'--output-format',
default='dns',
choices=['dns', 'json'],
help='Output format: dns, json (default: %default)')
parser.add_option('-d',
'--domain',
action="append",
default=['local'],
help="""DNS-SD domain (default: local).
This option should be used once for each domain you want to browse.""")
parser.add_option('-s',
'--service',
action="append",
default=[None],
help="""Service name (default: all services).
This option should be used once for each service you want to enumerate.
Well known subtypes (HTTP and IPP) are enumerated automatically, other
subtypes should be specified explicitly. Specifying a subtype automatically
enumerates the master type as well.""")
parser.add_option('-n',
'--instance-name',
default=None,
help="""Instance name to search for""")
parser.add_option('--instname-sed',
nargs=2,
default=None,
help="""The pattern and replacement string to use for
regex replace on each instance name.""")
parser.add_option('--instname-sed-service',
action="append",
default=[None],
help="""Regex replace on instance names
should be applied only to instances of these services. This option should be
used once for each service.""")
printgroup = OptionGroup(parser, "Options specific to Bonjour Printing")
printgroup.add_option('--location-map',
default='{}',
help="""A dictionary mapping instance names to
locations""")
printgroup.add_option('--priority-map',
default='{}',
help="""A dictionary mapping instance names, types or
tuples of both to priorities""")
parser.add_option_group(printgroup)
if cgi_mode:
import cgi
#import cgitb
#cgitb.enable()
sys.stderr = sys.stdout
form = cgi.FieldStorage()
options = []
if form.getfirst('target_zone'):
options.extend(['--target-zone', form.getfirst('target_zone')])
if form.getfirst('zone_xfr_from'):
options.extend(['--zone-xfr-from', form.getfirst('zone_xfr_from')])
if form.getfirst('ttl'):
options.extend(['--ttl', form.getfirst('ttl')])
if form.getfirst('output_rrset'):
options.append('--output-rrset')
if form.getfirst('output_format'):
options.extend(['--output-format', form.getfirst('output_format')])
[options.extend(['--domain', dom]) for dom in form.getlist('domain')]
[options.extend(['--service', svc]) for svc in form.getlist('service')]
options.extend(['--instance-name', form.getfirst('instance_name')])
if form.getfirst('instname_pattern') and \
form.getfirst('instname_repl'):
options.extend([
'--instname-sed',
form.getfirst('instname_pattern'),
form.getfirst('instname_repl')
])
[
options.extend(['--instname-sed-service', svc])
for svc in form.getlist('instname_sed_service')
]
if form.getfirst('location_map'):
options.extend(['--location-map', form.getfirst('location_map')])
if form.getfirst('priority_map'):
options.extend(['--priority-map', form.getfirst('priority_map')])
(options, args) = parser.parse_args(options)
# get rid of defaults because action=append doesnt
if form.getlist('domain'):
del options.domain[0]
if form.getlist('service'):
del options.service[0]
if form.getlist('instname_sed_service'):
del options.instname_sed_service[0]
else:
(options, args) = parser.parse_args(sys.argv[1:])
# get rid of defaults because action=append doesnt
if True in [
arg.find(opt_str) == 0 for arg in sys.argv[1:]
for opt_str in str(parser.get_option('--domain')).split('/')
]:
del options.domain[0]
if True in [
arg.find(opt_str) == 0 for arg in sys.argv[1:]
for opt_str in str(parser.get_option('--service')).split('/')
]:
del options.service[0]
if True in [
arg.find(opt_str) == 0
for arg in sys.argv[1:] for opt_str in str(
parser.get_option('--instname-sed-service')).split('/')
]:
del options.instname_sed_service[0]
# for opt in ['service', 'domain']:
# opt = parser.get_option('--%s' % opt)
# if True in [arg.find(opt_str) == 0 for arg in sys.argv[1:]
# for opt_str in str(opt).split('/')]:
# del getattr(options, opt.dest)[0]
#print options
return options
def get_option_parser(defaults):
"""Creates OptionParser and adds shell options (flags)
Default values are loaded in initially
"""
parser = OptionParser()
parser.set_defaults(**defaults)
parser.add_option("-i",
"--impalad",
dest="impalad",
help="<host:port> of impalad to connect to \t\t")
parser.add_option(
"-b",
"--kerberos_host_fqdn",
dest="kerberos_host_fqdn",
help="If set, overrides the expected hostname of the Impalad's "
"kerberos service principal. impala-shell will check that "
"the server's principal matches this hostname. This may be "
"used when impalad is configured to be accessed via a "
"load-balancer, but it is desired for impala-shell to talk "
"to a specific impalad directly.")
parser.add_option("-q",
"--query",
dest="query",
help="Execute a query without the shell")
parser.add_option(
"-f",
"--query_file",
dest="query_file",
help="Execute the queries in the query file, delimited by ;."
" If the argument to -f is \"-\", then queries are read from"
" stdin and terminated with ctrl-d.")
parser.add_option("-k",
"--kerberos",
dest="use_kerberos",
action="store_true",
help="Connect to a kerberized impalad")
parser.add_option(
"-o",
"--output_file",
dest="output_file",
help=("If set, query results are written to the "
"given file. Results from multiple semicolon-terminated "
"queries will be appended to the same file"))
parser.add_option("-B",
"--delimited",
dest="write_delimited",
action="store_true",
help="Output rows in delimited mode")
parser.add_option("--print_header",
dest="print_header",
action="store_true",
help="Print column names in delimited mode"
" when pretty-printed.")
parser.add_option(
"--output_delimiter",
dest="output_delimiter",
help="Field delimiter to use for output in delimited mode")
parser.add_option("-s",
"--kerberos_service_name",
dest="kerberos_service_name",
help="Service name of a kerberized impalad")
parser.add_option("-V",
"--verbose",
dest="verbose",
action="store_true",
help="Verbose output")
parser.add_option("-p",
"--show_profiles",
dest="show_profiles",
action="store_true",
help="Always display query profiles after execution")
parser.add_option("--quiet",
dest="verbose",
action="store_false",
help="Disable verbose output")
parser.add_option("-v",
"--version",
dest="version",
action="store_true",
help="Print version information")
parser.add_option("-c",
"--ignore_query_failure",
dest="ignore_query_failure",
action="store_true",
help="Continue on query failure")
parser.add_option("-d",
"--database",
dest="default_db",
help="Issues a use database command on startup \t")
parser.add_option(
"-l",
"--ldap",
dest="use_ldap",
action="store_true",
help="Use LDAP to authenticate with Impala. Impala must be configured"
" to allow LDAP authentication. \t\t")
parser.add_option("-u",
"--user",
dest="user",
help="User to authenticate with.")
parser.add_option("--ssl",
dest="ssl",
action="store_true",
help="Connect to Impala via SSL-secured connection \t")
parser.add_option(
"--ca_cert",
dest="ca_cert",
help=
("Full path to "
"certificate file used to authenticate Impala's SSL certificate."
" May either be a copy of Impala's certificate (for self-signed "
"certs) or the certificate of a trusted third-party CA. If not set, "
"but SSL is enabled, the shell will NOT verify Impala's server "
"certificate"))
parser.add_option(
"--config_file",
dest="config_file",
help=("Specify the configuration file to load options. "
"The following sections are used: [impala], "
"[impala.query_options]. Section names are case sensitive. "
"Specifying this option within a config file will have "
"no effect. Only specify this as an option in the commandline."))
parser.add_option(
"--history_file",
dest="history_file",
help=("The file in which to store shell history. This may also be "
"configured using the IMPALA_HISTFILE environment variable."))
parser.add_option(
"--live_summary",
dest="print_summary",
action="store_true",
help="Print a query summary every 1s while the query is running.")
parser.add_option(
"--live_progress",
dest="print_progress",
action="store_true",
help="Print a query progress every 1s while the query is running.")
parser.add_option(
"--auth_creds_ok_in_clear",
dest="creds_ok_in_clear",
action="store_true",
help="If set, LDAP authentication " +
"may be used with an insecure connection to Impala. " +
"WARNING: Authentication credentials will therefore be sent " +
"unencrypted, and may be vulnerable to attack.")
parser.add_option(
"--ldap_password_cmd",
help="Shell command to run to retrieve the LDAP password")
parser.add_option(
"--var",
dest="keyval",
action="append",
help="Defines a variable to be used within the Impala session."
" Can be used multiple times to set different variables."
" It must follow the pattern \"KEY=VALUE\","
" KEY starts with an alphabetic character and"
" contains alphanumeric characters or underscores.")
parser.add_option(
"-Q",
"--query_option",
dest="query_options",
action="append",
help="Sets the default for a query option."
" Can be used multiple times to set different query options."
" It must follow the pattern \"KEY=VALUE\","
" KEY must be a valid query option. Valid query options "
" can be listed by command 'set'.")
# add default values to the help text
for option in parser.option_list:
# since the quiet flag is the same as the verbose flag
# we need to make sure to print the opposite value for it
# (print quiet is false since verbose is true)
if option == parser.get_option('--quiet'):
option.help += " [default: %s]" % (not defaults['verbose'])
elif option != parser.get_option('--help'):
# don't want to print default value for help
option.help += " [default: %default]"
return parser
class TestOptionParser(BaseTest):
def setUp(self):
self.parser = OptionParser()
self.parser.add_option("-v",
"--verbose",
"-n",
"--noisy",
action="store_true",
dest="verbose")
self.parser.add_option("-q",
"--quiet",
"--silent",
action="store_false",
dest="verbose")
def test_add_option_no_Option(self):
self.assertTypeError(self.parser.add_option,
"not an Option instance: None", None)
def test_add_option_invalid_arguments(self):
self.assertTypeError(self.parser.add_option, "invalid arguments", None,
None)
def test_get_option(self):
opt1 = self.parser.get_option("-v")
self.assert_(isinstance(opt1, Option))
self.assertEqual(opt1._short_opts, ["-v", "-n"])
self.assertEqual(opt1._long_opts, ["--verbose", "--noisy"])
self.assertEqual(opt1.action, "store_true")
self.assertEqual(opt1.dest, "verbose")
def test_get_option_equals(self):
opt1 = self.parser.get_option("-v")
opt2 = self.parser.get_option("--verbose")
opt3 = self.parser.get_option("-n")
opt4 = self.parser.get_option("--noisy")
self.assert_(opt1 is opt2 is opt3 is opt4)
def test_has_option(self):
self.assert_(self.parser.has_option("-v"))
self.assert_(self.parser.has_option("--verbose"))
def assert_removed(self):
self.assert_(self.parser.get_option("-v") is None)
self.assert_(self.parser.get_option("--verbose") is None)
self.assert_(self.parser.get_option("-n") is None)
self.assert_(self.parser.get_option("--noisy") is None)
self.failIf(self.parser.has_option("-v"))
self.failIf(self.parser.has_option("--verbose"))
self.failIf(self.parser.has_option("-n"))
self.failIf(self.parser.has_option("--noisy"))
self.assert_(self.parser.has_option("-q"))
self.assert_(self.parser.has_option("--silent"))
def test_remove_short_opt(self):
self.parser.remove_option("-n")
self.assert_removed()
def test_remove_long_opt(self):
self.parser.remove_option("--verbose")
self.assert_removed()
def test_remove_nonexistent(self):
self.assertRaises(self.parser.remove_option,
ValueError,
"no such option 'foo'",
funcargs=['foo'])
# Parse
(options, args) = parser.parse_args()
# Target config or netcdf
if len(args) > 0:
ncfile = args[0]
else:
ncfile = options.ncfile
if ncfile is not None:
options.force = True
# Bounds
defaults = [('xmin', -720), ('xmax', 720), ('ymin', -90), ('ymax', 90)]
for bname, bdef in defaults:
if getattr(options, bname) is None:
desc = parser.get_option('--' + bname).help
if options.force:
value = bdef
else:
while True:
try:
value = float(
raw_input(
'Please set %s (or use -f or --%s options): ' %
(desc.lower(), bname)).strip())
break
except ValueError:
print 'Please enter a float'
if getattr(options, bname) is None:
setattr(options, bname, value)
lon = (options.xmin, options.xmax)
def get_option_parser(defaults):
"""Creates OptionParser and adds shell options (flags)
Default values are loaded in initially
"""
parser = OptionParser()
parser.set_defaults(**defaults)
parser.add_option("-i", "--impalad", dest="impalad",
help="<host:port> of impalad to connect to \t\t")
parser.add_option("-q", "--query", dest="query",
help="Execute a query without the shell")
parser.add_option("-f", "--query_file", dest="query_file",
help="Execute the queries in the query file, delimited by ;")
parser.add_option("-k", "--kerberos", dest="use_kerberos",
action="store_true", help="Connect to a kerberized impalad")
parser.add_option("-o", "--output_file", dest="output_file",
help=("If set, query results are written to the "
"given file. Results from multiple semicolon-terminated "
"queries will be appended to the same file"))
parser.add_option("-B", "--delimited", dest="write_delimited",
action="store_true",
help="Output rows in delimited mode")
parser.add_option("--print_header", dest="print_header",
action="store_true",
help="Print column names in delimited mode"
" when pretty-printed.")
parser.add_option("--output_delimiter", dest="output_delimiter",
help="Field delimiter to use for output in delimited mode")
parser.add_option("-s", "--kerberos_service_name",
dest="kerberos_service_name",
help="Service name of a kerberized impalad")
parser.add_option("-V", "--verbose", dest="verbose",
action="store_true",
help="Verbose output")
parser.add_option("-p", "--show_profiles", dest="show_profiles",
action="store_true",
help="Always display query profiles after execution")
parser.add_option("--quiet", dest="verbose",
action="store_false",
help="Disable verbose output")
parser.add_option("-v", "--version", dest="version",
action="store_true",
help="Print version information")
parser.add_option("-c", "--ignore_query_failure", dest="ignore_query_failure",
action="store_true", help="Continue on query failure")
parser.add_option("-r", "--refresh_after_connect", dest="refresh_after_connect",
action="store_true",
help="Refresh Impala catalog after connecting \t")
parser.add_option("-d", "--database", dest="default_db",
help="Issues a use database command on startup \t")
parser.add_option("-l", "--ldap", dest="use_ldap",
action="store_true",
help="Use LDAP to authenticate with Impala. Impala must be configured"
" to allow LDAP authentication. \t\t")
parser.add_option("-u", "--user", dest="user",
help="User to authenticate with.")
parser.add_option("--ssl", dest="ssl",
action="store_true",
help="Connect to Impala via SSL-secured connection \t")
parser.add_option("--ca_cert", dest="ca_cert",
help=("Full path to "
"certificate file used to authenticate Impala's SSL certificate."
" May either be a copy of Impala's certificate (for self-signed "
"certs) or the certificate of a trusted third-party CA. If not set, "
"but SSL is enabled, the shell will NOT verify Impala's server "
"certificate"))
parser.add_option("--config_file", dest="config_file",
help=("Specify the configuration file to load options. "
"File must have case-sensitive '[impala]' header. "
"Specifying this option within a config file will have "
"no effect. Only specify this as a option in the commandline."
))
parser.add_option("--live_summary", dest="print_summary", action="store_true",
help="Print a query summary every 1s while the query is running.")
parser.add_option("--live_progress", dest="print_progress", action="store_true",
help="Print a query progress every 1s while the query is running.")
parser.add_option("--auth_creds_ok_in_clear", dest="creds_ok_in_clear",
action="store_true", help="If set, LDAP authentication " +
"may be used with an insecure connection to Impala. " +
"WARNING: Authentication credentials will therefore be sent " +
"unencrypted, and may be vulnerable to attack.")
parser.add_option("--ldap_password_cmd",
help="Shell command to run to retrieve the LDAP password")
parser.add_option("--var", dest="keyval", action="append",
help="Define variable(s) to be used within the Impala session.")
# add default values to the help text
for option in parser.option_list:
# since the quiet flag is the same as the verbose flag
# we need to make sure to print the opposite value for it
# (print quiet is false since verbose is true)
if option == parser.get_option('--quiet'):
option.help += " [default: %s]" % (not defaults['verbose'])
elif option != parser.get_option('--help'):
# don't want to print default value for help
option.help += " [default: %default]"
return parser
def main():
# TODO : query the App Engine metadata to determine all kinds
kinds = ('Comment',
'DailyActivityLog',
'DiscussAnswer',
'DiscussQuestion',
'ExercisePlaylist',
'ExerciseVideo',
'Exercise',
'FeedbackNotification',
'Feedback',
'HourlyActivityLog',
'Playlist',
'ProblemLog',
'Question',
'Setting',
'StemmedIndex',
'UserBadge',
'UserData',
'UserExercise',
'UserPlaylist',
'UserVideo',
'VideoLog',
'VideoPlaylist',
'Video',
'YouTubeSyncStepLog'
)
parser = OptionParser(usage="%prog [options]",
description="Downloads a data and progress file for each entity kind in SQLite3 format.")
parser.add_option("-U", "--url", default="http://*****:*****@example.com",
help="The username to use.")
parser.add_option("-k", "--kinds", default=','.join(kinds),
help="The comma separated list of kinds.")
parser.add_option("-p", "--python", default=(sys.executable if platform.system() == "Windows" else None), help="Path of python executable.")
parser.add_option("-a", "--appcfg", default='appcfg.py', help="Path of appcfg.py (Google App Engine).")
parser.add_option("-A", "--application", default='khanexercises', help="GAE application name")
# Performance options
parser.add_option("-n", "--num_threads", default='55', help="num_threads, passed to bulkloader")
parser.add_option("-s", "--batch_size", default='500', help="batch_size, passed to bulkloader")
parser.add_option("-b", "--bandwidth_limit", default='10000000', help="bandwidth_limit, passed to bulkloader")
parser.add_option("-r", "--rps_limit", default='15000', help="rps_limit of threads, passed to bulkloader")
parser.add_option("-c", "--http_limit", default='40', help="http_limit, passed to bulkloader")
parser.add_option("-x", "--passin", default='', help="Path to file containing Google App Engine password")
(options, args) = parser.parse_args()
files = []
for kind in options.kinds.split(','):
filename='bulkloader-results.%s.data' % kind
db_filename='bulkloader-progress.%s.data' % kind
files.append(filename)
files.append(db_filename)
if os.path.exists(filename):
os.remove(filename)
if os.path.exists(db_filename):
os.remove(db_filename)
call_args = [options.appcfg,
'--url=%s' % options.url,
'--email=%s' % options.email,
'--kind=%s' % kind,
'--db_filename=%s' % db_filename,
'--filename=%s' % filename,
'--num_threads=%s' % options.num_threads,
'--batch_size=%s' % options.batch_size,
'--bandwidth_limit=%s' % options.bandwidth_limit,
'--rps_limit=%s' % options.rps_limit,
'--http_limit=%s' % options.http_limit
]
if options.application is not None:
call_args.append('--application=%s' % options.application)
if options.email == parser.get_option('--email').default or options.passin:
call_args.append('--passin')
if options.python is not None:
call_args.insert(0, options.python)
call_args.append('download_data')
print ' '.join(call_args)
if options.email == parser.get_option('--email').default:
process = subprocess.Popen(call_args, stdin=subprocess.PIPE)
# Send a newline for the password prompt
process.communicate("\n")
elif options.passin:
f = open(options.passin, "r")
password = f.read()
f.close()
# Send contents of password file for the password prompt
process = subprocess.Popen(call_args, stdin=subprocess.PIPE)
process.communicate(password)
else:
process = subprocess.Popen(call_args)
process.wait()
# zip up all created progress and results files
os.system("zip bulkloader.zip bulkloader-*.data")
os.system("rm bulkloader-*") # warning: this will blow away the logs and extra results db's, too!
def cmdLineParser(argv=None):
"""
This function parses the command line parameters and arguments
"""
if not argv:
argv = sys.argv
checkSystemEncoding()
# Reference: https://stackoverflow.com/a/4012683 (Note: previously used "...sys.getfilesystemencoding() or UNICODE_ENCODING")
_ = getUnicode(os.path.basename(argv[0]), encoding=sys.stdin.encoding)
usage = "%s%s [options]" % ("python " if not IS_WIN else "", "\"%s\"" % _ if " " in _ else _)
parser = OptionParser(usage=usage)
try:
parser.add_option("--hh", dest="advancedHelp",
action="store_true",
help="Show advanced help message and exit")
parser.add_option("--version", dest="showVersion",
action="store_true",
help="Show program's version number and exit")
parser.add_option("-v", dest="verbose", type="int",
help="Verbosity level: 0-6 (default %d)" % defaults.verbose)
# Target options
target = OptionGroup(parser, "Target", "At least one of these "
"options has to be provided to define the target(s)")
target.add_option("-d", dest="direct", help="Connection string "
"for direct database connection")
target.add_option("-u", "--url", dest="url", help="Target URL (e.g. \"http://www.site.com/vuln.php?id=1\")")
target.add_option("-l", dest="logFile", help="Parse target(s) from Burp "
"or WebScarab proxy log file")
target.add_option("-x", dest="sitemapUrl", help="Parse target(s) from remote sitemap(.xml) file")
target.add_option("-m", dest="bulkFile", help="Scan multiple targets given "
"in a textual file ")
target.add_option("-r", dest="requestFile",
help="Load HTTP request from a file")
target.add_option("-g", dest="googleDork",
help="Process Google dork results as target URLs")
target.add_option("-c", dest="configFile",
help="Load options from a configuration INI file")
# Request options
request = OptionGroup(parser, "Request", "These options can be used "
"to specify how to connect to the target URL")
request.add_option("--method", dest="method",
help="Force usage of given HTTP method (e.g. PUT)")
request.add_option("--data", dest="data",
help="Data string to be sent through POST")
request.add_option("--param-del", dest="paramDel",
help="Character used for splitting parameter values")
request.add_option("--cookie", dest="cookie",
help="HTTP Cookie header value")
request.add_option("--cookie-del", dest="cookieDel",
help="Character used for splitting cookie values")
request.add_option("--load-cookies", dest="loadCookies",
help="File containing cookies in Netscape/wget format")
request.add_option("--drop-set-cookie", dest="dropSetCookie", action="store_true",
help="Ignore Set-Cookie header from response")
request.add_option("--user-agent", dest="agent",
help="HTTP User-Agent header value")
request.add_option("--random-agent", dest="randomAgent", action="store_true",
help="Use randomly selected HTTP User-Agent header value")
request.add_option("--host", dest="host",
help="HTTP Host header value")
request.add_option("--referer", dest="referer",
help="HTTP Referer header value")
request.add_option("-H", "--header", dest="header",
help="Extra header (e.g. \"X-Forwarded-For: 127.0.0.1\")")
request.add_option("--headers", dest="headers",
help="Extra headers (e.g. \"Accept-Language: fr\\nETag: 123\")")
request.add_option("--auth-type", dest="authType",
help="HTTP authentication type (Basic, Digest, NTLM or PKI)")
request.add_option("--auth-cred", dest="authCred",
help="HTTP authentication credentials (name:password)")
request.add_option("--auth-file", dest="authFile",
help="HTTP authentication PEM cert/private key file")
request.add_option("--ignore-code", dest="ignoreCode", type="int",
help="Ignore HTTP error code (e.g. 401)")
request.add_option("--ignore-proxy", dest="ignoreProxy", action="store_true",
help="Ignore system default proxy settings")
request.add_option("--ignore-redirects", dest="ignoreRedirects", action="store_true",
help="Ignore redirection attempts")
request.add_option("--ignore-timeouts", dest="ignoreTimeouts", action="store_true",
help="Ignore connection timeouts")
request.add_option("--proxy", dest="proxy",
help="Use a proxy to connect to the target URL")
request.add_option("--proxy-cred", dest="proxyCred",
help="Proxy authentication credentials (name:password)")
request.add_option("--proxy-file", dest="proxyFile",
help="Load proxy list from a file")
request.add_option("--tor", dest="tor", action="store_true",
help="Use Tor anonymity network")
request.add_option("--tor-port", dest="torPort",
help="Set Tor proxy port other than default")
request.add_option("--tor-type", dest="torType",
help="Set Tor proxy type (HTTP, SOCKS4 or SOCKS5 (default))")
request.add_option("--check-tor", dest="checkTor", action="store_true",
help="Check to see if Tor is used properly")
request.add_option("--delay", dest="delay", type="float",
help="Delay in seconds between each HTTP request")
request.add_option("--timeout", dest="timeout", type="float",
help="Seconds to wait before timeout connection (default %d)" % defaults.timeout)
request.add_option("--retries", dest="retries", type="int",
help="Retries when the connection timeouts (default %d)" % defaults.retries)
request.add_option("--randomize", dest="rParam",
help="Randomly change value for given parameter(s)")
request.add_option("--safe-url", dest="safeUrl",
help="URL address to visit frequently during testing")
request.add_option("--safe-post", dest="safePost",
help="POST data to send to a safe URL")
request.add_option("--safe-req", dest="safeReqFile",
help="Load safe HTTP request from a file")
request.add_option("--safe-freq", dest="safeFreq", type="int",
help="Test requests between two visits to a given safe URL")
request.add_option("--skip-urlencode", dest="skipUrlEncode", action="store_true",
help="Skip URL encoding of payload data")
request.add_option("--csrf-token", dest="csrfToken",
help="Parameter used to hold anti-CSRF token")
request.add_option("--csrf-url", dest="csrfUrl",
help="URL address to visit to extract anti-CSRF token")
request.add_option("--force-ssl", dest="forceSSL", action="store_true",
help="Force usage of SSL/HTTPS")
request.add_option("--hpp", dest="hpp", action="store_true",
help="Use HTTP parameter pollution method")
request.add_option("--eval", dest="evalCode",
help="Evaluate provided Python code before the request (e.g. \"import hashlib;id2=hashlib.md5(id).hexdigest()\")")
# Optimization options
optimization = OptionGroup(parser, "Optimization", "These options can be used to optimize the performance of sqlmap")
optimization.add_option("-o", dest="optimize", action="store_true",
help="Turn on all optimization switches")
optimization.add_option("--predict-output", dest="predictOutput", action="store_true",
help="Predict common queries output")
optimization.add_option("--keep-alive", dest="keepAlive", action="store_true",
help="Use persistent HTTP(s) connections")
optimization.add_option("--null-connection", dest="nullConnection", action="store_true",
help="Retrieve page length without actual HTTP response body")
optimization.add_option("--threads", dest="threads", type="int",
help="Max number of concurrent HTTP(s) "
"requests (default %d)" % defaults.threads)
# Injection options
injection = OptionGroup(parser, "Injection", "These options can be used to specify which parameters to test for, provide custom injection payloads and optional tampering scripts")
injection.add_option("-p", dest="testParameter",
help="Testable parameter(s)")
injection.add_option("--skip", dest="skip",
help="Skip testing for given parameter(s)")
injection.add_option("--skip-static", dest="skipStatic", action="store_true",
help="Skip testing parameters that not appear to be dynamic")
injection.add_option("--param-exclude", dest="paramExclude",
help="Regexp to exclude parameters from testing (e.g. \"ses\")")
injection.add_option("--dbms", dest="dbms",
help="Force back-end DBMS to provided value")
injection.add_option("--dbms-cred", dest="dbmsCred",
help="DBMS authentication credentials (user:password)")
injection.add_option("--os", dest="os",
help="Force back-end DBMS operating system to provided value")
injection.add_option("--invalid-bignum", dest="invalidBignum", action="store_true",
help="Use big numbers for invalidating values")
injection.add_option("--invalid-logical", dest="invalidLogical", action="store_true",
help="Use logical operations for invalidating values")
injection.add_option("--invalid-string", dest="invalidString", action="store_true",
help="Use random strings for invalidating values")
injection.add_option("--no-cast", dest="noCast", action="store_true",
help="Turn off payload casting mechanism")
injection.add_option("--no-escape", dest="noEscape", action="store_true",
help="Turn off string escaping mechanism")
injection.add_option("--prefix", dest="prefix",
help="Injection payload prefix string")
injection.add_option("--suffix", dest="suffix",
help="Injection payload suffix string")
injection.add_option("--tamper", dest="tamper",
help="Use given script(s) for tampering injection data")
# Detection options
detection = OptionGroup(parser, "Detection", "These options can be used to customize the detection phase")
detection.add_option("--level", dest="level", type="int",
help="Level of tests to perform (1-5, default %d)" % defaults.level)
detection.add_option("--risk", dest="risk", type="int",
help="Risk of tests to perform (1-3, default %d)" % defaults.risk)
detection.add_option("--string", dest="string",
help="String to match when query is evaluated to True")
detection.add_option("--not-string", dest="notString",
help="String to match when query is evaluated to False")
detection.add_option("--regexp", dest="regexp",
help="Regexp to match when query is evaluated to True")
detection.add_option("--code", dest="code", type="int",
help="HTTP code to match when query is evaluated to True")
detection.add_option("--text-only", dest="textOnly", action="store_true",
help="Compare pages based only on the textual content")
detection.add_option("--titles", dest="titles", action="store_true",
help="Compare pages based only on their titles")
# Techniques options
techniques = OptionGroup(parser, "Techniques", "These options can be used to tweak testing of specific SQL injection techniques")
techniques.add_option("--technique", dest="tech",
help="SQL injection techniques to use (default \"%s\")" % defaults.tech)
techniques.add_option("--time-sec", dest="timeSec", type="int",
help="Seconds to delay the DBMS response (default %d)" % defaults.timeSec)
techniques.add_option("--union-cols", dest="uCols",
help="Range of columns to test for UNION query SQL injection")
techniques.add_option("--union-char", dest="uChar",
help="Character to use for bruteforcing number of columns")
techniques.add_option("--union-from", dest="uFrom",
help="Table to use in FROM part of UNION query SQL injection")
techniques.add_option("--dns-domain", dest="dnsDomain",
help="Domain name used for DNS exfiltration attack")
techniques.add_option("--second-url", dest="secondUrl",
help="Resulting page URL searched for second-order response")
techniques.add_option("--second-req", dest="secondReq",
help="Load second-order HTTP request from file")
# Fingerprint options
fingerprint = OptionGroup(parser, "Fingerprint")
fingerprint.add_option("-f", "--fingerprint", dest="extensiveFp", action="store_true",
help="Perform an extensive DBMS version fingerprint")
# Enumeration options
enumeration = OptionGroup(parser, "Enumeration", "These options can be used to enumerate the back-end database management system information, structure and data contained in the tables. Moreover you can run your own SQL statements")
enumeration.add_option("-a", "--all", dest="getAll", action="store_true",
help="Retrieve everything")
enumeration.add_option("-b", "--banner", dest="getBanner", action="store_true",
help="Retrieve DBMS banner")
enumeration.add_option("--current-user", dest="getCurrentUser", action="store_true",
help="Retrieve DBMS current user")
enumeration.add_option("--current-db", dest="getCurrentDb", action="store_true",
help="Retrieve DBMS current database")
enumeration.add_option("--hostname", dest="getHostname", action="store_true",
help="Retrieve DBMS server hostname")
enumeration.add_option("--is-dba", dest="isDba", action="store_true",
help="Detect if the DBMS current user is DBA")
enumeration.add_option("--users", dest="getUsers", action="store_true",
help="Enumerate DBMS users")
enumeration.add_option("--passwords", dest="getPasswordHashes", action="store_true",
help="Enumerate DBMS users password hashes")
enumeration.add_option("--privileges", dest="getPrivileges", action="store_true",
help="Enumerate DBMS users privileges")
enumeration.add_option("--roles", dest="getRoles", action="store_true",
help="Enumerate DBMS users roles")
enumeration.add_option("--dbs", dest="getDbs", action="store_true",
help="Enumerate DBMS databases")
enumeration.add_option("--tables", dest="getTables", action="store_true",
help="Enumerate DBMS database tables")
enumeration.add_option("--columns", dest="getColumns", action="store_true",
help="Enumerate DBMS database table columns")
enumeration.add_option("--schema", dest="getSchema", action="store_true",
help="Enumerate DBMS schema")
enumeration.add_option("--count", dest="getCount", action="store_true",
help="Retrieve number of entries for table(s)")
enumeration.add_option("--dump", dest="dumpTable", action="store_true",
help="Dump DBMS database table entries")
enumeration.add_option("--dump-all", dest="dumpAll", action="store_true",
help="Dump all DBMS databases tables entries")
enumeration.add_option("--search", dest="search", action="store_true",
help="Search column(s), table(s) and/or database name(s)")
enumeration.add_option("--comments", dest="getComments", action="store_true",
help="Check for DBMS comments during enumeration")
enumeration.add_option("-D", dest="db",
help="DBMS database to enumerate")
enumeration.add_option("-T", dest="tbl",
help="DBMS database table(s) to enumerate")
enumeration.add_option("-C", dest="col",
help="DBMS database table column(s) to enumerate")
enumeration.add_option("-X", dest="exclude",
help="DBMS database identifier(s) to not enumerate")
enumeration.add_option("-U", dest="user",
help="DBMS user to enumerate")
enumeration.add_option("--exclude-sysdbs", dest="excludeSysDbs", action="store_true",
help="Exclude DBMS system databases when enumerating tables")
enumeration.add_option("--pivot-column", dest="pivotColumn",
help="Pivot column name")
enumeration.add_option("--where", dest="dumpWhere",
help="Use WHERE condition while table dumping")
enumeration.add_option("--start", dest="limitStart", type="int",
help="First dump table entry to retrieve")
enumeration.add_option("--stop", dest="limitStop", type="int",
help="Last dump table entry to retrieve")
enumeration.add_option("--first", dest="firstChar", type="int",
help="First query output word character to retrieve")
enumeration.add_option("--last", dest="lastChar", type="int",
help="Last query output word character to retrieve")
enumeration.add_option("--sql-query", dest="query",
help="SQL statement to be executed")
enumeration.add_option("--sql-shell", dest="sqlShell", action="store_true",
help="Prompt for an interactive SQL shell")
enumeration.add_option("--sql-file", dest="sqlFile",
help="Execute SQL statements from given file(s)")
# Brute force options
brute = OptionGroup(parser, "Brute force", "These options can be used to run brute force checks")
brute.add_option("--common-tables", dest="commonTables", action="store_true",
help="Check existence of common tables")
brute.add_option("--common-columns", dest="commonColumns", action="store_true",
help="Check existence of common columns")
# User-defined function options
udf = OptionGroup(parser, "User-defined function injection", "These options can be used to create custom user-defined functions")
udf.add_option("--udf-inject", dest="udfInject", action="store_true",
help="Inject custom user-defined functions")
udf.add_option("--shared-lib", dest="shLib",
help="Local path of the shared library")
# File system options
filesystem = OptionGroup(parser, "File system access", "These options can be used to access the back-end database management system underlying file system")
filesystem.add_option("--file-read", dest="rFile",
help="Read a file from the back-end DBMS file system")
filesystem.add_option("--file-write", dest="wFile",
help="Write a local file on the back-end DBMS file system")
filesystem.add_option("--file-dest", dest="dFile",
help="Back-end DBMS absolute filepath to write to")
# Takeover options
takeover = OptionGroup(parser, "Operating system access", "These options can be used to access the back-end database management system underlying operating system")
takeover.add_option("--os-cmd", dest="osCmd",
help="Execute an operating system command")
takeover.add_option("--os-shell", dest="osShell", action="store_true",
help="Prompt for an interactive operating system shell")
takeover.add_option("--os-pwn", dest="osPwn", action="store_true",
help="Prompt for an OOB shell, Meterpreter or VNC")
takeover.add_option("--os-smbrelay", dest="osSmb", action="store_true",
help="One click prompt for an OOB shell, Meterpreter or VNC")
takeover.add_option("--os-bof", dest="osBof", action="store_true",
help="Stored procedure buffer overflow "
"exploitation")
takeover.add_option("--priv-esc", dest="privEsc", action="store_true",
help="Database process user privilege escalation")
takeover.add_option("--msf-path", dest="msfPath",
help="Local path where Metasploit Framework is installed")
takeover.add_option("--tmp-path", dest="tmpPath",
help="Remote absolute path of temporary files directory")
# Windows registry options
windows = OptionGroup(parser, "Windows registry access", "These options can be used to access the back-end database management system Windows registry")
windows.add_option("--reg-read", dest="regRead", action="store_true",
help="Read a Windows registry key value")
windows.add_option("--reg-add", dest="regAdd", action="store_true",
help="Write a Windows registry key value data")
windows.add_option("--reg-del", dest="regDel", action="store_true",
help="Delete a Windows registry key value")
windows.add_option("--reg-key", dest="regKey",
help="Windows registry key")
windows.add_option("--reg-value", dest="regVal",
help="Windows registry key value")
windows.add_option("--reg-data", dest="regData",
help="Windows registry key value data")
windows.add_option("--reg-type", dest="regType",
help="Windows registry key value type")
# General options
general = OptionGroup(parser, "General", "These options can be used to set some general working parameters")
general.add_option("-s", dest="sessionFile",
help="Load session from a stored (.sqlite) file")
general.add_option("-t", dest="trafficFile",
help="Log all HTTP traffic into a textual file")
general.add_option("--batch", dest="batch", action="store_true",
help="Never ask for user input, use the default behavior")
general.add_option("--binary-fields", dest="binaryFields",
help="Result fields having binary values (e.g. \"digest\")")
general.add_option("--check-internet", dest="checkInternet", action="store_true",
help="Check Internet connection before assessing the target")
general.add_option("--crawl", dest="crawlDepth", type="int",
help="Crawl the website starting from the target URL")
general.add_option("--crawl-exclude", dest="crawlExclude",
help="Regexp to exclude pages from crawling (e.g. \"logout\")")
general.add_option("--csv-del", dest="csvDel",
help="Delimiting character used in CSV output (default \"%s\")" % defaults.csvDel)
general.add_option("--charset", dest="charset",
help="Blind SQL injection charset (e.g. \"0123456789abcdef\")")
general.add_option("--dump-format", dest="dumpFormat",
help="Format of dumped data (CSV (default), HTML or SQLITE)")
general.add_option("--encoding", dest="encoding",
help="Character encoding used for data retrieval (e.g. GBK)")
general.add_option("--eta", dest="eta", action="store_true",
help="Display for each output the estimated time of arrival")
general.add_option("--flush-session", dest="flushSession", action="store_true",
help="Flush session files for current target")
general.add_option("--forms", dest="forms", action="store_true",
help="Parse and test forms on target URL")
general.add_option("--fresh-queries", dest="freshQueries", action="store_true",
help="Ignore query results stored in session file")
general.add_option("--har", dest="harFile",
help="Log all HTTP traffic into a HAR file")
general.add_option("--hex", dest="hexConvert", action="store_true",
help="Use hex conversion during data retrieval")
general.add_option("--output-dir", dest="outputDir", action="store",
help="Custom output directory path")
general.add_option("--parse-errors", dest="parseErrors", action="store_true",
help="Parse and display DBMS error messages from responses")
general.add_option("--save", dest="saveConfig",
help="Save options to a configuration INI file")
general.add_option("--scope", dest="scope",
help="Regexp to filter targets from provided proxy log")
general.add_option("--test-filter", dest="testFilter",
help="Select tests by payloads and/or titles (e.g. ROW)")
general.add_option("--test-skip", dest="testSkip",
help="Skip tests by payloads and/or titles (e.g. BENCHMARK)")
general.add_option("--update", dest="updateAll", action="store_true",
help="Update sqlmap")
# Miscellaneous options
miscellaneous = OptionGroup(parser, "Miscellaneous")
miscellaneous.add_option("-z", dest="mnemonics",
help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
miscellaneous.add_option("--alert", dest="alert",
help="Run host OS command(s) when SQL injection is found")
miscellaneous.add_option("--answers", dest="answers",
help="Set question answers (e.g. \"quit=N,follow=N\")")
miscellaneous.add_option("--beep", dest="beep", action="store_true",
help="Beep on question and/or when SQL injection is found")
miscellaneous.add_option("--cleanup", dest="cleanup", action="store_true",
help="Clean up the DBMS from sqlmap specific UDF and tables")
miscellaneous.add_option("--dependencies", dest="dependencies", action="store_true",
help="Check for missing (non-core) sqlmap dependencies")
miscellaneous.add_option("--disable-coloring", dest="disableColoring", action="store_true",
help="Disable console output coloring")
miscellaneous.add_option("--gpage", dest="googlePage", type="int",
help="Use Google dork results from specified page number")
miscellaneous.add_option("--identify-waf", dest="identifyWaf", action="store_true",
help="Make a thorough testing for a WAF/IPS/IDS protection")
miscellaneous.add_option("--list-tampers", dest="listTampers", action="store_true",
help="Display list of available tamper scripts")
miscellaneous.add_option("--mobile", dest="mobile", action="store_true",
help="Imitate smartphone through HTTP User-Agent header")
miscellaneous.add_option("--offline", dest="offline", action="store_true",
help="Work in offline mode (only use session data)")
miscellaneous.add_option("--purge", dest="purge", action="store_true",
help="Safely remove all content from sqlmap data directory")
miscellaneous.add_option("--skip-waf", dest="skipWaf", action="store_true",
help="Skip heuristic detection of WAF/IPS/IDS protection")
miscellaneous.add_option("--smart", dest="smart", action="store_true",
help="Conduct thorough tests only if positive heuristic(s)")
miscellaneous.add_option("--sqlmap-shell", dest="sqlmapShell", action="store_true",
help="Prompt for an interactive sqlmap shell")
miscellaneous.add_option("--tmp-dir", dest="tmpDir",
help="Local directory for storing temporary files")
miscellaneous.add_option("--web-root", dest="webRoot",
help="Web server document root directory (e.g. \"/var/www\")")
miscellaneous.add_option("--wizard", dest="wizard", action="store_true",
help="Simple wizard interface for beginner users")
# Hidden and/or experimental options
parser.add_option("--dummy", dest="dummy", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--murphy-rate", dest="murphyRate", type="int",
help=SUPPRESS_HELP)
parser.add_option("--disable-precon", dest="disablePrecon", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--disable-stats", dest="disableStats", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--profile", dest="profile", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--force-dbms", dest="forceDbms",
help=SUPPRESS_HELP)
parser.add_option("--force-dns", dest="forceDns", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--force-pivoting", dest="forcePivoting", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--force-threads", dest="forceThreads", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--smoke-test", dest="smokeTest", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--live-test", dest="liveTest", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--stop-fail", dest="stopFail", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--run-case", dest="runCase", help=SUPPRESS_HELP)
# API options
parser.add_option("--api", dest="api", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--taskid", dest="taskid", help=SUPPRESS_HELP)
parser.add_option("--database", dest="database", help=SUPPRESS_HELP)
parser.add_option_group(target)
parser.add_option_group(request)
parser.add_option_group(optimization)
parser.add_option_group(injection)
parser.add_option_group(detection)
parser.add_option_group(techniques)
parser.add_option_group(fingerprint)
parser.add_option_group(enumeration)
parser.add_option_group(brute)
parser.add_option_group(udf)
parser.add_option_group(filesystem)
parser.add_option_group(takeover)
parser.add_option_group(windows)
parser.add_option_group(general)
parser.add_option_group(miscellaneous)
# Dirty hack to display longer options without breaking into two lines
def _(self, *args):
retVal = parser.formatter._format_option_strings(*args)
if len(retVal) > MAX_HELP_OPTION_LENGTH:
retVal = ("%%.%ds.." % (MAX_HELP_OPTION_LENGTH - parser.formatter.indent_increment)) % retVal
return retVal
parser.formatter._format_option_strings = parser.formatter.format_option_strings
parser.formatter.format_option_strings = type(parser.formatter.format_option_strings)(_, parser, type(parser))
# Dirty hack for making a short option '-hh'
option = parser.get_option("--hh")
option._short_opts = ["-hh"]
option._long_opts = []
# Dirty hack for inherent help message of switch '-h'
option = parser.get_option("-h")
option.help = option.help.capitalize().replace("this help", "basic help")
_ = []
prompt = False
advancedHelp = True
extraHeaders = []
# Reference: https://stackoverflow.com/a/4012683 (Note: previously used "...sys.getfilesystemencoding() or UNICODE_ENCODING")
for arg in argv:
_.append(getUnicode(arg, encoding=sys.stdin.encoding))
argv = _
checkDeprecatedOptions(argv)
prompt = "--sqlmap-shell" in argv
if prompt:
parser.usage = ""
cmdLineOptions.sqlmapShell = True
_ = ["x", "q", "exit", "quit", "clear"]
for option in parser.option_list:
_.extend(option._long_opts)
_.extend(option._short_opts)
for group in parser.option_groups:
for option in group.option_list:
_.extend(option._long_opts)
_.extend(option._short_opts)
autoCompletion(AUTOCOMPLETE_TYPE.SQLMAP, commands=_)
while True:
command = None
try:
command = raw_input("sqlmap-shell> ").strip()
command = getUnicode(command, encoding=sys.stdin.encoding)
except (KeyboardInterrupt, EOFError):
print
raise SqlmapShellQuitException
if not command:
continue
elif command.lower() == "clear":
clearHistory()
dataToStdout("[i] history cleared\n")
saveHistory(AUTOCOMPLETE_TYPE.SQLMAP)
elif command.lower() in ("x", "q", "exit", "quit"):
raise SqlmapShellQuitException
elif command[0] != '-':
dataToStdout("[!] invalid option(s) provided\n")
dataToStdout("[i] proper example: '-u http://www.site.com/vuln.php?id=1 --banner'\n")
else:
saveHistory(AUTOCOMPLETE_TYPE.SQLMAP)
loadHistory(AUTOCOMPLETE_TYPE.SQLMAP)
break
try:
for arg in shlex.split(command):
argv.append(getUnicode(arg, encoding=sys.stdin.encoding))
except ValueError, ex:
raise SqlmapSyntaxException("something went wrong during command line parsing ('%s')" % ex.message)
for i in xrange(len(argv)):
if argv[i] == "-hh":
argv[i] = "-h"
elif len(argv[i]) > 1 and all(ord(_) in xrange(0x2018, 0x2020) for _ in ((argv[i].split('=', 1)[-1].strip() or ' ')[0], argv[i][-1])):
dataToStdout("[!] copy-pasting illegal (non-console) quote characters from Internet is, well, illegal (%s)\n" % argv[i])
raise SystemExit
elif len(argv[i]) > 1 and u"\uff0c" in argv[i].split('=', 1)[-1]:
dataToStdout("[!] copy-pasting illegal (non-console) comma characters from Internet is, well, illegal (%s)\n" % argv[i])
raise SystemExit
elif re.search(r"\A-\w=.+", argv[i]):
dataToStdout("[!] potentially miswritten (illegal '=') short option detected ('%s')\n" % argv[i])
raise SystemExit
elif argv[i] == "-H":
if i + 1 < len(argv):
extraHeaders.append(argv[i + 1])
elif re.match(r"\A\d+!\Z", argv[i]) and argv[max(0, i - 1)] == "--threads" or re.match(r"\A--threads.+\d+!\Z", argv[i]):
argv[i] = argv[i][:-1]
conf.skipThreadCheck = True
elif argv[i] == "--version":
print VERSION_STRING.split('/')[-1]
raise SystemExit
elif argv[i] in ("-h", "--help"):
advancedHelp = False
for group in parser.option_groups[:]:
found = False
for option in group.option_list:
if option.dest not in BASIC_HELP_ITEMS:
option.help = SUPPRESS_HELP
else:
found = True
if not found:
parser.option_groups.remove(group)
for verbosity in (_ for _ in argv if re.search(r"\A\-v+\Z", _)):
try:
if argv.index(verbosity) == len(argv) - 1 or not argv[argv.index(verbosity) + 1].isdigit():
conf.verbose = verbosity.count('v') + 1
del argv[argv.index(verbosity)]
except (IndexError, ValueError):
pass
try:
(args, _) = parser.parse_args(argv)
except UnicodeEncodeError, ex:
dataToStdout("\n[!] %s\n" % ex.object.encode("unicode-escape"))
raise SystemExit
def main(prefix):
if not prefix.startswith('/usr'):
florun.base_dir = prefix
florun.locale_dir = os.path.join(prefix, 'florun', 'locale')
florun.icons_dir = os.path.join(prefix, 'florun', 'icons')
else:
prefix = os.path.join(prefix, '..')
prefix = os.path.abspath(os.path.normpath(prefix))
florun.base_dir = os.path.expanduser("~")
florun.locale_dir = os.path.join(prefix, 'share', 'locale')
florun.icons_dir = os.path.join(prefix, 'share', 'florun', 'icons')
# Set up the path to translation files
gettext.install('florun', florun.locale_dir, unicode=True)
florun._ = _
# Parse command-line arguments
parser = OptionParser(usage='%s [options]' % florun.__title__)
parser.add_option("-v", "--version",
dest="version", default=False, action="store_true",
help=_("Show version"))
parser.add_option("-e", "--edit",
dest="edit", default=None,
help=_("Edit specified Florun file"))
parser.add_option("-x", "--execute",
dest="execute", default=None,
help=_("Execute specified Florun file"))
parser.add_option("-l", "--level",
dest="level", default=logging.DEBUG, type='int',
help=_("Logging level for messages (1:debug 2:info, 3:warning, 4:errors, 5:critical)"))
# We distinguish official args from args passed to flow
args = sys.argv[1:]
last = len(args)
for i, arg in enumerate(args):
if arg.startswith('-') and parser.get_option(arg) is None:
last = i
break
# Parse only first part of args
(options, useless) = parser.parse_args(args[:last])
logging.basicConfig()
florun.flow.logger.setLevel(options.level)
florun.gui.logger.setLevel(options.level)
if options.version:
showversion()
return 0
florun.gui.logger.debug("Prefix : '%s'" % prefix)
florun.gui.logger.debug("Icons : '%s'" % florun.icons_dir)
if options.edit is not None:
# Run editor on specified flow
return florun.gui.main(args, options.edit)
elif options.execute is not None:
try:
# Load flow definition
wf = florun.flow.Flow.load(options.execute)
# Check if flow uses cmd line args
cmdparamnodes = wf.CLIParameterNodes()
if len(cmdparamnodes) > 0:
# Parse the rest of args
parser = OptionParser()
for node in cmdparamnodes:
parser.add_option("-%s" % node.paramname[:1],
"--%s" % node.paramname,
dest=node.paramname)
(options, parsedargs) = parser.parse_args(args[last:])
for node in cmdparamnodes:
node.options = options
runner = florun.flow.Runner(wf)
runner.start()
except FlowError, e:
florun.flow.logger.exception(e)
return 1
except KeyboardInterrupt, e:
florun.flow.logger.error(_("Interrupted by user"))
return 2
def get_option_parser(defaults):
"""Creates OptionParser and adds shell options (flags)
Default values are loaded in initially
"""
parser = OptionParser()
parser.add_option("-i",
"--impalad",
dest="impalad",
help="<host:port> of impalad to connect to \t\t")
parser.add_option(
"-b",
"--kerberos_host_fqdn",
dest="kerberos_host_fqdn",
help="If set, overrides the expected hostname of the Impalad's "
"kerberos service principal. impala-shell will check that "
"the server's principal matches this hostname. This may be "
"used when impalad is configured to be accessed via a "
"load-balancer, but it is desired for impala-shell to talk "
"to a specific impalad directly.")
parser.add_option("-q",
"--query",
dest="query",
help="Execute a query without the shell")
parser.add_option(
"-f",
"--query_file",
dest="query_file",
help="Execute the queries in the query file, delimited by ;."
" If the argument to -f is \"-\", then queries are read from"
" stdin and terminated with ctrl-d.")
parser.add_option("-k",
"--kerberos",
dest="use_kerberos",
action="store_true",
help="Connect to a kerberized impalad")
parser.add_option(
"-o",
"--output_file",
dest="output_file",
help=("If set, query results are written to the "
"given file. Results from multiple semicolon-terminated "
"queries will be appended to the same file"))
parser.add_option("-B",
"--delimited",
dest="write_delimited",
action="store_true",
help="Output rows in delimited mode")
parser.add_option("--print_header",
dest="print_header",
action="store_true",
help="Print column names in delimited mode"
" when pretty-printed.")
parser.add_option(
"--output_delimiter",
dest="output_delimiter",
help="Field delimiter to use for output in delimited mode")
parser.add_option("-s",
"--kerberos_service_name",
dest="kerberos_service_name",
help="Service name of a kerberized impalad")
parser.add_option("-V",
"--verbose",
dest="verbose",
action="store_true",
help="Verbose output")
parser.add_option("-p",
"--show_profiles",
dest="show_profiles",
action="store_true",
help="Always display query profiles after execution")
parser.add_option("--quiet",
dest="verbose",
action="store_false",
help="Disable verbose output")
parser.add_option("-v",
"--version",
dest="version",
action="store_true",
help="Print version information")
parser.add_option("-c",
"--ignore_query_failure",
dest="ignore_query_failure",
action="store_true",
help="Continue on query failure")
parser.add_option("-d",
"--database",
dest="default_db",
help="Issues a use database command on startup \t")
parser.add_option(
"-l",
"--ldap",
dest="use_ldap",
action="store_true",
help="Use LDAP to authenticate with Impala. Impala must be configured"
" to allow LDAP authentication. \t\t")
parser.add_option("-u",
"--user",
dest="user",
help="User to authenticate with.")
parser.add_option("--ssl",
dest="ssl",
action="store_true",
help="Connect to Impala via SSL-secured connection \t")
parser.add_option(
"--ca_cert",
dest="ca_cert",
help=
("Full path to "
"certificate file used to authenticate Impala's SSL certificate."
" May either be a copy of Impala's certificate (for self-signed "
"certs) or the certificate of a trusted third-party CA. If not set, "
"but SSL is enabled, the shell will NOT verify Impala's server "
"certificate"))
parser.add_option(
"--config_file",
dest="config_file",
help=("Specify the configuration file to load options. "
"The following sections are used: [impala], "
"[impala.query_options]. Section names are case sensitive. "
"Specifying this option within a config file will have "
"no effect. Only specify this as an option in the commandline."))
parser.add_option(
"--history_file",
dest="history_file",
help=("The file in which to store shell history. This may also be "
"configured using the IMPALA_HISTFILE environment variable."))
parser.add_option(
"--live_summary",
dest="live_summary",
action="store_true",
help="Print a query summary every 1s while the query is running.")
parser.add_option(
"--live_progress",
dest="live_progress",
action="store_true",
help="Print a query progress every 1s while the query is running."
" The default value of the flag is True in the interactive mode."
" If live_progress is set to False in a config file, this flag"
" will override it")
parser.add_option(
"--disable_live_progress",
dest="live_progress",
action="store_false",
help="A command line flag allows users to disable live_progress in"
" the interactive mode.")
parser.add_option(
"--auth_creds_ok_in_clear",
dest="creds_ok_in_clear",
action="store_true",
help="If set, LDAP authentication " +
"may be used with an insecure connection to Impala. " +
"WARNING: Authentication credentials will therefore be sent " +
"unencrypted, and may be vulnerable to attack.")
parser.add_option(
"--ldap_password_cmd",
dest="ldap_password_cmd",
help="Shell command to run to retrieve the LDAP password")
parser.add_option(
"--var",
dest="keyval",
action="append",
help="Defines a variable to be used within the Impala session."
" Can be used multiple times to set different variables."
" It must follow the pattern \"KEY=VALUE\","
" KEY starts with an alphabetic character and"
" contains alphanumeric characters or underscores.")
parser.add_option(
"-Q",
"--query_option",
dest="query_options",
action="append",
help="Sets the default for a query option."
" Can be used multiple times to set different query options."
" It must follow the pattern \"KEY=VALUE\","
" KEY must be a valid query option. Valid query options "
" can be listed by command 'set'.")
parser.add_option(
"-t",
"--client_connect_timeout_ms",
help="Timeout in milliseconds after which impala-shell will time out"
" if it fails to connect to Impala server. Set to 0 to disable any"
" timeout.")
parser.add_option(
"--protocol",
dest="protocol",
default="beeswax",
help="Protocol to use for client/server connection. Valid inputs are "
"['hs2', 'hs2-http', 'beeswax']. 'hs2-http' uses HTTP transport "
"to speak to the coordinator while 'hs2' and 'beeswax' use the "
"binary TCP based transport. Beeswax support is deprecated "
"and will be removed in the future.")
parser.add_option(
"--http_path",
dest="http_path",
default="cliservice",
help="Default http path on the coordinator to connect to. The final "
"connection URL looks like <http(s)>://<coordinator-host>:<port>/"
"<http_path>. While the coordinator server implementation does not "
"enforce any http path for the incoming requests, deployments could "
"still put it behind a loadbalancer that can expect the traffic at a "
"certain path.")
# add default values to the help text
for option in parser.option_list:
if option.dest is not None:
# option._short_opts returns a list of short options, e.g. ["-Q"].
# option._long_opts returns a list of long options, e.g. ["--query_option"].
# The code below removes the - from the short option and -- from the long option.
short_opt = option._short_opts[0][1:] if len(
option._short_opts) > 0 else None
long_opt = option._long_opts[0][2:] if len(
option._long_opts) > 0 else None
# In order to set the default flag values, optparse requires the keys to be the
# dest names. The default flag values are set in impala_shell_config_defaults.py and
# the default flag values may contain default values that are not for flags.
if short_opt in defaults:
if option.dest not in defaults:
defaults[option.dest] = defaults[short_opt]
elif type(defaults[option.dest]) == list:
defaults[option.dest].extend(defaults[short_opt])
elif long_opt in defaults:
if option.dest not in defaults:
defaults[option.dest] = defaults[long_opt]
elif type(defaults[option.dest]) == list:
defaults[option.dest].extend(defaults[long_opt])
# since the quiet flag is the same as the verbose flag
# we need to make sure to print the opposite value for it
# (print quiet is false since verbose is true)
if option == parser.get_option('--quiet'):
option.help += " [default: %s]" % (not defaults['verbose'])
# print default value of disable_live_progress in the help messages as opposite
# value for default value of live_progress
# (print disable_live_progress is false since live_progress is true)
elif option == parser.get_option('--disable_live_progress'):
option.help += " [default: %s]" % (not defaults['live_progress'])
elif option != parser.get_option(
'--help') and option.help is not SUPPRESS_HELP:
# don't want to print default value for help or options without help text
option.help += " [default: %default]"
# mutually exclusive flags should not be used in the same time
if '--live_progress' in sys.argv and '--disable_live_progress' in sys.argv:
parser.error(
"options --live_progress and --disable_live_progress are mutually "
"exclusive")
if '--verbose' in sys.argv and '--quiet' in sys.argv:
parser.error("options --verbose and --quiet are mutually exclusive")
parser.set_defaults(**defaults)
return parser
def cmdLineParser(argv=None):
if not argv:
argv = sys.argv
try:
usage = "usage: python3 %prog [options] arg1 arg2"
parser = OptionParser(usage=usage)
parser.add_option("--hh",
dest="advancedHelp",
action="store_true",
help="Show advanced help message and exit")
parser.add_option("--version",
dest="showVersion",
action="store_true",
help="Show program's version number and exit")
parser.add_option("-t",
"--thread",
dest="threads",
default=1,
type=int,
help="Set the number of threads, default 1.")
parser.add_option("-v",
"--verbose",
dest="verbose",
default=False,
action="store_true",
help="Show the result as verbosely as possible.")
parser.add_option("-o",
"--output",
dest="out",
action="store",
help="Store the result to a file.")
cipher = OptionGroup(
parser, "Cipher",
"Cipher module, have both classic cipher and advanced cipher methods."
)
cipher.add_option(
"--cipher",
dest="cipher",
help=
"encrypt or decrypt text using the given method, they are: reverse, caesar, transposition, affine, sub, bacon, vigenere"
)
cipher.add_option("-E",
"--encrypt",
dest="encrypt",
default=False,
action="store_true",
help="enable encrypt mode.")
cipher.add_option("-D",
"--decrypt",
dest="decrypt",
default=False,
action="store_true",
help="enable decrypt mode.")
cipher.add_option(
"-B",
"--brute",
dest="brute",
default=False,
action="store_true",
help=
"use the brute force to decrypt text, use with the option --decrypt"
)
cipher.add_option(
"-T",
"--text",
dest="text",
help="input the text waiting to be ciphered, please use with \"\"")
cipher.add_option("--cipher-key",
dest="key",
help="specify the key used in encrypt or decrypt.")
cipher.add_option("-f",
"--file",
dest="file",
help="Use file content as the input text.")
cipher.add_option("--lower",
dest="lower",
default=False,
action="store_true",
help="Set the output as lower letters.")
parser.add_option_group(cipher)
dns = OptionGroup(parser, "DNS Proxy",
"A simple DNS proxy, can modify dns record.")
dns.add_option("--dns",
dest="dnsproxy",
default=False,
action="store_true",
help="Enable DNS proxy module.")
dns.add_option(
"--fakeip",
dest="fakeip",
metavar="192.0.2.1",
action="store",
help=
"IP address to use for matching DNS queries. If you use this parameter without specifying domain names, then all \'A\' queries will be spoofed. Consider using --file argument if you need to define more than one IP address."
)
dns.add_option("--fakeipv6",
dest="fakeipv6",
metavar="2001:db8::1",
action="store",
help="Use IPV6 IP address for fakeip.")
dns.add_option("--fakemail",
dest="fakemail",
metavar="mail.fake.com",
action="store",
help="Use fake MX record.")
dns.add_option("--fakealias",
dest="fakealias",
metavar="www.fake.com",
action="store",
help="Use fake CNAME record.")
dns.add_option("--fakens",
dest="fakens",
metavar="ns.fake.com",
action="store",
help="Use fake NS record.")
dns.add_option(
"--fakedomain",
dest="fakedomain",
metavar="baidu.com",
action="store",
help=
"A list separated by comma, specify the domains which will be resolved to fake values given above."
)
dns.add_option(
"--truedomain",
dest="truedomain",
metavar="baidu.com",
action="store",
help=
"A list separated by comma, specify the domains which will be resolved to true values."
)
dns.add_option(
"--nameserver",
dest="nameserver",
metavar="8.8.8.8#53 or 4.2.2.1#53#tcp or 2001:4860:4860::8888",
default="8.8.8.8",
action="store",
help=
"A list separated by comma of alternative DNS servers to use with proxied requests. Nameservers can have either IP or IP#PORT format. A randomly selected server from the list will be used for proxy requests when provided with multiple servers. By default, the tool uses Google\'s public DNS server 8.8.8.8 when running in IPv4 mode and 2001:4860:4860::8888 when running in IPv6 mode."
)
dns.add_option(
"--interface",
metavar="127.0.0.1 or ::1",
default="127.0.0.1",
action="store",
help=
"Define an interface to use for the DNS listener. By default, the tool uses 127.0.0.1 for IPv4 mode and ::1 for IPv6 mode."
)
dns.add_option("--tcp",
dest="tcp",
default=False,
action="store_true",
help="Enable TCP mode.")
dns.add_option("--ipv6",
dest="ipv6",
default=False,
action="store_true",
help="Run in IPV6 mode.")
parser.add_option_group(dns)
ssh = OptionGroup(
parser, "SSH",
"Module for ssh key generate, forwarding and so on.")
ssh.add_option("--ssh",
dest="ssh",
default=False,
action="store_true",
help="Enable ssh module.")
ssh.add_option(
"-d",
"--direct",
dest="sshdirect",
metavar="user@ssh_host:ssh_host_port",
action="store",
help=
"Connect to the remote ssh server directly, such as the openssh, can be used with the -P to input the password or --private-key to use key file."
)
ssh.add_option("--ssh-keygen",
dest="sshkeygen",
default=False,
action="store_true",
help="Generate ssh key pair.")
ssh.add_option("--bits",
dest="bits",
metavar="bits",
default=1024,
type="int",
action="store",
help="Number of bits in the key to create")
ssh.add_option("--ktype",
dest="ktype",
metavar="ktype",
default="rsa",
action="store",
help="Specify type of key to create (dsa or rsa).")
ssh.add_option(
"-L",
dest="sshlocal",
metavar="local_port:host:host_port",
action="store",
help=
"Set up a forward tunnel across an SSH server, and was followed by the parttern \033[4m[local_port]:[host]:[host_port]\033[0m, this is similar to the openssh -L option."
)
ssh.add_option(
"-R",
dest="sshremote",
metavar="remote_port:host:host_port",
action="store",
help=
"Set up a remote port forward tunnel across ana SSH server, and was followed by the parttern \033[4m[remote_port]:[host]:[host_port]\033[0m, this is similar to the openssh -R option."
)
ssh.add_option(
"--remote",
dest="remote",
metavar="user@ssh_ip:ssh_port",
action="store",
help=
"Remote user, host and port to forward to, like the openssh, followed by the pattern \033[4m[user@][ssh_ip]:[ssh_port]\033[0m."
)
ssh.add_option(
"-P",
dest="password",
default=False,
action="store_true",
help="Read password (for key or password auth) from stdin.")
ssh.add_option("--private-key",
dest="privfile",
action="store",
help="Private key file to use for SSH authentication.")
parser.add_option_group(ssh)
option = parser.get_option("--hh")
option._short_opts = ["-hh"]
option._long_opts = []
option = parser.get_option("-h")
option.help = option.help.capitalize().replace("this help",
"basic help")
_ = []
advancedHelp = True
for arg in argv:
_.append(arg)
argv = _
for i in range(len(argv)):
if argv[i] == "-hh":
argv[i] = "-h"
elif re.search(r"\A-\w=.+", argv[i]):
print(
"[!] potentially miswritten (illegal '=') short option detected ('%s')\n"
% argv[i])
raise SystemExit
elif argv[i] == "--version":
print(VERSION)
raise SystemExit
elif argv[i] == "-h":
advancedHelp = False
for group in parser.option_groups[:]:
found = False
for option in group.option_list:
if option.dest not in BASIC_HELP_ITEMS:
option.help = SUPPRESS_HELP
else:
found = True
if not found:
parser.option_groups.remove(group)
elif argv[i] == "--cipher=reverse":
argv.append("-E")
try:
(options, args) = parser.parse_args(argv)
except SystemExit:
if "-h" in sys.argv and not advancedHelp:
dataToStdout(
"\033[32m\n[!] to see full list of options run with '-hh'\n\033[0m"
)
raise
if len(argv) == 1:
errMsg = "missing a mandatory option, "
errMsg += "use -h for basic or -hh for advanced help"
parser.error(errMsg)
return options
except SystemExit:
raise
def main(args=None):
"""The main function; parses options and plots"""
# ---------- build and read options ----------
from optparse import OptionParser
optParser = OptionParser()
optParser.add_option("-n", "--net", dest="net", metavar="FILE",
help="Defines the network to read")
optParser.add_option("-i", "--dump-inputs", dest="dumps", metavar="FILE",
help="Defines the dump-output files to use as input")
optParser.add_option("-m", "--measures", dest="measures",
default="speed,entered", help="Define which measure to plot")
optParser.add_option("--min-width", dest="minWidth",
type="float", default=.5, help="Defines the minimum edge width")
optParser.add_option("--max-width", dest="maxWidth",
type="float", default=3, help="Defines the maximum edge width")
optParser.add_option("--log-colors", dest="logColors", action="store_true",
default=False, help="If set, colors are log-scaled")
optParser.add_option("--log-widths", dest="logWidths", action="store_true",
default=False, help="If set, widths are log-scaled")
optParser.add_option("--min-color-value", dest="colorMin",
type="float", default=None,
help="If set, defines the minimum edge color value")
optParser.add_option("--max-color-value", dest="colorMax",
type="float", default=None,
help="If set, defines the maximum edge color value")
optParser.add_option("--min-width-value", dest="widthMin",
type="float", default=None,
help="If set, defines the minimum edge width value")
optParser.add_option("--max-width-value", dest="widthMax",
type="float", default=None,
help="If set, defines the maximum edge width value")
optParser.add_option("-v", "--verbose", dest="verbose", action="store_true",
default=False,
help="If set, the script says what it's doing")
optParser.add_option("--color-bar-label", dest="colorBarLabel", default="",
help="The label to put on the color bar")
# standard plot options
helpers.addInteractionOptions(optParser)
helpers.addPlotOptions(optParser)
helpers.addNetOptions(optParser)
# Override the help string for the output option
outputOpt = optParser.get_option("--output")
outputOpt.help = "Comma separated list of filename(s) the figure shall be written to; " +\
"for multiple time intervals use \'\%s\' in the filename as a " +\
"placeholder for the beginning of the time interval"
# parse
options, remaining_args = optParser.parse_args(args=args)
if options.net is None:
print("Error: a network to load must be given.")
return 1
if options.verbose:
print("Reading network from '%s'" % options.net)
net = sumolib.net.readNet(options.net)
if options.measures is None:
print("Error: a dump file must be given.")
return 1
times = []
hc = None
colorDump = options.dumps.split(",")[0]
colorMeasure = options.measures.split(",")[0]
if colorDump:
if options.verbose:
print("Reading colors from '%s'" % colorDump)
hc = WeightsReader(colorMeasure)
sumolib.output.parse_sax(colorDump, hc)
times = hc._edge2value
hw = None
widthDump = options.dumps.split(",")[1]
widthMeasure = options.measures.split(",")[1]
if widthDump != "":
if options.verbose:
print("Reading widths from '%s'" % widthDump)
hw = WeightsReader(widthMeasure)
sumolib.output.parse_sax(widthDump, hw)
times = hw._edge2value
# Should we also save the figure to a file / list of files (comma
# separated)? Then we need to check the output filename(s)
if options.output:
options.nolegend = True
optOutputNames = options.output
# If we have multiple intervals to be plotted, make sure we have
# proper output filenames (with a %s as a placeholder in it)
if len(times) > 1 and optOutputNames.find('%s') < 0:
print('Warning: multiple time intervals detected, but ' +
'the output filename(s) do not contain a \'%s\' placeholder. ' +
'Continuing by using a default placeholder.')
# Modify each filename by putting a '-%s' right before the
# extension
filenames = optOutputNames.split(',')
for i in range(0, len(filenames)):
filename, extension = os.path.splitext(filenames[i])
filenames[i] = filename + '-%s' + extension
optOutputNames = ','.join(filenames)
# Now go through each time interval and create the figures
for t in times:
if options.verbose:
print("Processing interval with a beginning of %s" % t)
colors = {}
maxColorValue = None
minColorValue = None
for e in net._id2edge:
if hc and t in hc._edge2value and e in hc._edge2value[t]:
if options.colorMax is not None and hc._edge2value[t][e] > options.colorMax:
hc._edge2value[t][e] = options.colorMax
if options.colorMin is not None and hc._edge2value[t][e] < options.colorMin:
hc._edge2value[t][e] = options.colorMin
if maxColorValue is None or maxColorValue < hc._edge2value[t][e]:
maxColorValue = hc._edge2value[t][e]
if minColorValue is None or minColorValue > hc._edge2value[t][e]:
minColorValue = hc._edge2value[t][e]
colors[e] = hc._edge2value[t][e]
if options.colorMax is not None:
maxColorValue = options.colorMax
if options.colorMin is not None:
minColorValue = options.colorMin
if options.logColors:
helpers.logNormalise(colors, maxColorValue)
else:
helpers.linNormalise(colors, minColorValue, maxColorValue)
for e in colors:
colors[e] = helpers.getColor(options, colors[e], 1.)
if options.verbose:
print("Color values are between %s and %s" %
(minColorValue, maxColorValue))
widths = {}
maxWidthValue = None
minWidthValue = None
for e in net._id2edge:
if hw and t in hw._edge2value and e in hw._edge2value[t]:
v = abs(hw._edge2value[t][e])
if options.widthMax is not None and v > options.widthMax:
v = options.widthMax
if options.widthMin is not None and v < options.widthMin:
v = options.widthMin
if maxWidthValue is None or maxWidthValue < v:
maxWidthValue = v
if minWidthValue is None or minWidthValue > v:
minWidthValue = v
widths[e] = v
if options.widthMax is not None:
maxWidthValue = options.widthMax
if options.widthMin is not None:
minWidthValue = options.widthMin
if options.logWidths:
helpers.logNormalise(widths, options.colorMax)
else:
helpers.linNormalise(widths, minWidthValue, maxWidthValue)
for e in widths:
widths[e] = options.minWidth + widths[e] * \
(options.maxWidth - options.minWidth)
if options.verbose:
print("Width values are between %s and %s" %
(minWidthValue, maxWidthValue))
fig, ax = helpers.openFigure(options)
ax.set_aspect("equal", None, 'C')
helpers.plotNet(net, colors, widths, options)
# drawing the legend, at least for the colors
norm = matplotlib.colors.LogNorm if options.logColors else matplotlib.colors.Normalize
sm = plt.cm.ScalarMappable(cmap=matplotlib.cm.get_cmap(options.colormap),
norm=norm(vmin=minColorValue, vmax=maxColorValue))
# "fake up the array of the scalar mappable. Urgh..."
# (pelson, http://stackoverflow.com/questions/8342549/matplotlib-add-colorbar-to-a-sequence-of-line-plots)
sm._A = []
color_bar = plt.colorbar(sm)
color_bar.set_label(options.colorBarLabel)
# Should we also save the figure to a file / list of files (comma
# separated)?
if options.output:
# If we have a "%s" in the name of the output then replace it with the
# interval begin of the current interval
expandedOutputNames = optOutputNames
if expandedOutputNames.find('%s') >= 0:
expandedOutputNames = expandedOutputNames.replace("%s", str(t))
# Can be used to print additional text in the figure:
#
# m, s = divmod(int(t), 60)
# h, m = divmod(m, 60)
# timeStr = "%02d:%02d:%02d" % (h, m, s)
# ax.text(0.2, 0.2, timeStr, bbox={
# 'facecolor': 'white', 'pad': 10}, size=16)
helpers.closeFigure(fig, ax, options, False, expandedOutputNames)
return 0
def cmdLineParser():
"""
This function parses the command line parameters and arguments
"""
usage = "%s%s [options]" % ("python " if not IS_WIN else "", \
"\"%s\"" % sys.argv[0] if " " in sys.argv[0] else sys.argv[0])
parser = OptionParser(usage=usage)
try:
parser.add_option("--hh", dest="advancedHelp",
action="store_true",
help="Show advanced help message and exit")
parser.add_option("-v", dest="verbose", type="int",
help="Verbosity level: 0-6 (default %d)" % defaults.verbose)
# Target options
target = OptionGroup(parser, "Target", "At least one of these "
"options has to be specified to set the source "
"to get target urls from")
target.add_option("-d", dest="direct", help="Direct "
"connection to the database")
target.add_option("-u", "--url", dest="url", help="Target url")
target.add_option("-l", dest="logFile", help="Parse targets from Burp "
"or WebScarab proxy logs")
target.add_option("-m", dest="bulkFile", help="Scan multiple targets enlisted "
"in a given textual file ")
target.add_option("-r", dest="requestFile",
help="Load HTTP request from a file")
target.add_option("-g", dest="googleDork",
help="Process Google dork results as target urls")
target.add_option("-c", dest="configFile",
help="Load options from a configuration INI file")
# Request options
request = OptionGroup(parser, "Request", "These options can be used "
"to specify how to connect to the target url")
request.add_option("--data", dest="data",
help="Data string to be sent through POST")
request.add_option("--param-del", dest="pDel",
help="Character used for splitting parameter values")
request.add_option("--cookie", dest="cookie",
help="HTTP Cookie header")
request.add_option("--load-cookies", dest="loC",
help="File containing cookies in Netscape/wget format")
request.add_option("--cookie-urlencode", dest="cookieUrlencode",
action="store_true",
help="URL Encode generated cookie injections")
request.add_option("--drop-set-cookie", dest="dropSetCookie",
action="store_true",
help="Ignore Set-Cookie header from response")
request.add_option("--user-agent", dest="agent",
help="HTTP User-Agent header")
request.add_option("--random-agent", dest="randomAgent",
action="store_true",
help="Use randomly selected HTTP User-Agent header")
request.add_option("--randomize", dest="rParam",
help="Randomly change value for given parameter(s)")
request.add_option("--force-ssl", dest="forceSSL",
action="store_true",
help="Force usage of SSL/HTTPS requests")
request.add_option("--host", dest="host",
help="HTTP Host header")
request.add_option("--referer", dest="referer",
help="HTTP Referer header")
request.add_option("--headers", dest="headers",
help="Extra headers (e.g. \"Accept-Language: fr\\nETag: 123\")")
request.add_option("--auth-type", dest="aType",
help="HTTP authentication type "
"(Basic, Digest or NTLM)")
request.add_option("--auth-cred", dest="aCred",
help="HTTP authentication credentials "
"(name:password)")
request.add_option("--auth-cert", dest="aCert",
help="HTTP authentication certificate ("
"key_file,cert_file)")
request.add_option("--proxy", dest="proxy",
help="Use a HTTP proxy to connect to the target url")
request.add_option("--proxy-cred", dest="pCred",
help="HTTP proxy authentication credentials "
"(name:password)")
request.add_option("--ignore-proxy", dest="ignoreProxy", action="store_true",
help="Ignore system default HTTP proxy")
request.add_option("--delay", dest="delay", type="float",
help="Delay in seconds between each HTTP request")
request.add_option("--timeout", dest="timeout", type="float",
help="Seconds to wait before timeout connection "
"(default %d)" % defaults.timeout)
request.add_option("--retries", dest="retries", type="int",
help="Retries when the connection timeouts "
"(default %d)" % defaults.retries)
request.add_option("--scope", dest="scope",
help="Regexp to filter targets from provided proxy log")
request.add_option("--safe-url", dest="safUrl",
help="Url address to visit frequently during testing")
request.add_option("--safe-freq", dest="saFreq", type="int",
help="Test requests between two visits to a given safe url")
request.add_option("--skip-urlencode", dest="skipUrlEncode",
action="store_true",
help="Skip URL encoding of POST data")
request.add_option("--eval", dest="evalCode",
help="Evaluate provided Python code before the request (e.g. \"import hashlib;id2=hashlib.md5(id).hexdigest()\")")
# Optimization options
optimization = OptionGroup(parser, "Optimization", "These "
"options can be used to optimize the "
"performance of sqlmap")
optimization.add_option("-o", dest="optimize",
action="store_true",
help="Turn on all optimization switches")
optimization.add_option("--predict-output", dest="predictOutput", action="store_true",
help="Predict common queries output")
optimization.add_option("--keep-alive", dest="keepAlive", action="store_true",
help="Use persistent HTTP(s) connections")
optimization.add_option("--null-connection", dest="nullConnection", action="store_true",
help="Retrieve page length without actual HTTP response body")
optimization.add_option("--threads", dest="threads", type="int",
help="Max number of concurrent HTTP(s) "
"requests (default %d)" % defaults.threads)
# Injection options
injection = OptionGroup(parser, "Injection", "These options can be "
"used to specify which parameters to test "
"for, provide custom injection payloads and "
"optional tampering scripts")
injection.add_option("-p", dest="testParameter",
help="Testable parameter(s)")
injection.add_option("--dbms", dest="dbms",
help="Force back-end DBMS to this value")
injection.add_option("--os", dest="os",
help="Force back-end DBMS operating system "
"to this value")
injection.add_option("--invalid-bignum", dest="invalidBignum",
action="store_true",
help="Use big numbers for invalidating values")
injection.add_option("--invalid-logical", dest="invalidLogical",
action="store_true",
help="Use logical operations for invalidating values")
injection.add_option("--no-cast", dest="noCast",
action="store_true",
help="Turn off payload casting mechanism")
injection.add_option("--prefix", dest="prefix",
help="Injection payload prefix string")
injection.add_option("--suffix", dest="suffix",
help="Injection payload suffix string")
injection.add_option("--skip", dest="skip",
help="Skip testing for given parameter(s)")
injection.add_option("--tamper", dest="tamper",
help="Use given script(s) for tampering injection data")
# Detection options
detection = OptionGroup(parser, "Detection", "These options can be "
"used to specify how to parse "
"and compare page content from "
"HTTP responses when using blind SQL "
"injection technique")
detection.add_option("--level", dest="level", type="int",
help="Level of tests to perform (1-5, "
"default %d)" % defaults.level)
detection.add_option("--risk", dest="risk", type="int",
help="Risk of tests to perform (0-3, "
"default %d)" % defaults.level)
detection.add_option("--string", dest="string",
help="String to match when "
"query is evaluated to True")
detection.add_option("--regexp", dest="regexp",
help="Regexp to match when "
"query is evaluated to True")
detection.add_option("--code", dest="code", type="int",
help="HTTP code to match when "
"query is evaluated to True")
detection.add_option("--text-only", dest="textOnly",
action="store_true",
help="Compare pages based only on the textual content")
detection.add_option("--titles", dest="titles",
action="store_true",
help="Compare pages based only on their titles")
# Techniques options
techniques = OptionGroup(parser, "Techniques", "These options can be "
"used to tweak testing of specific SQL "
"injection techniques")
techniques.add_option("--technique", dest="tech",
help="SQL injection techniques to test for "
"(default \"%s\")" % defaults.tech)
techniques.add_option("--time-sec", dest="timeSec",
type="int",
help="Seconds to delay the DBMS response "
"(default %d)" % defaults.timeSec)
techniques.add_option("--union-cols", dest="uCols",
help="Range of columns to test for UNION query SQL injection")
techniques.add_option("--union-char", dest="uChar",
help="Character to use for bruteforcing number of columns")
techniques.add_option("--dns-domain", dest="dName",
help="Domain name used for DNS exfiltration attack")
# Fingerprint options
fingerprint = OptionGroup(parser, "Fingerprint")
fingerprint.add_option("-f", "--fingerprint", dest="extensiveFp",
action="store_true",
help="Perform an extensive DBMS version fingerprint")
# Enumeration options
enumeration = OptionGroup(parser, "Enumeration", "These options can "
"be used to enumerate the back-end database "
"management system information, structure "
"and data contained in the tables. Moreover "
"you can run your own SQL statements")
enumeration.add_option("-b", "--banner", dest="getBanner",
action="store_true", help="Retrieve DBMS banner")
enumeration.add_option("--current-user", dest="getCurrentUser",
action="store_true",
help="Retrieve DBMS current user")
enumeration.add_option("--current-db", dest="getCurrentDb",
action="store_true",
help="Retrieve DBMS current database")
enumeration.add_option("--is-dba", dest="isDba",
action="store_true",
help="Detect if the DBMS current user is DBA")
enumeration.add_option("--users", dest="getUsers", action="store_true",
help="Enumerate DBMS users")
enumeration.add_option("--passwords", dest="getPasswordHashes",
action="store_true",
help="Enumerate DBMS users password hashes")
enumeration.add_option("--privileges", dest="getPrivileges",
action="store_true",
help="Enumerate DBMS users privileges")
enumeration.add_option("--roles", dest="getRoles",
action="store_true",
help="Enumerate DBMS users roles")
enumeration.add_option("--dbs", dest="getDbs", action="store_true",
help="Enumerate DBMS databases")
enumeration.add_option("--tables", dest="getTables", action="store_true",
help="Enumerate DBMS database tables")
enumeration.add_option("--columns", dest="getColumns", action="store_true",
help="Enumerate DBMS database table columns")
enumeration.add_option("--schema", dest="getSchema", action="store_true",
help="Enumerate DBMS schema")
enumeration.add_option("--count", dest="getCount", action="store_true",
help="Retrieve number of entries for table(s)")
enumeration.add_option("--dump", dest="dumpTable", action="store_true",
help="Dump DBMS database table entries")
enumeration.add_option("--dump-all", dest="dumpAll", action="store_true",
help="Dump all DBMS databases tables entries")
enumeration.add_option("--search", dest="search", action="store_true",
help="Search column(s), table(s) and/or database name(s)")
enumeration.add_option("-D", dest="db",
help="DBMS database to enumerate")
enumeration.add_option("-T", dest="tbl",
help="DBMS database table to enumerate")
enumeration.add_option("-C", dest="col",
help="DBMS database table column to enumerate")
enumeration.add_option("-U", dest="user",
help="DBMS user to enumerate")
enumeration.add_option("--exclude-sysdbs", dest="excludeSysDbs",
action="store_true",
help="Exclude DBMS system databases when "
"enumerating tables")
enumeration.add_option("--start", dest="limitStart", type="int",
help="First query output entry to retrieve")
enumeration.add_option("--stop", dest="limitStop", type="int",
help="Last query output entry to retrieve")
enumeration.add_option("--first", dest="firstChar", type="int",
help="First query output word character to retrieve")
enumeration.add_option("--last", dest="lastChar", type="int",
help="Last query output word character to retrieve")
enumeration.add_option("--sql-query", dest="query",
help="SQL statement to be executed")
enumeration.add_option("--sql-shell", dest="sqlShell",
action="store_true",
help="Prompt for an interactive SQL shell")
# User-defined function options
brute = OptionGroup(parser, "Brute force", "These "
"options can be used to run brute force "
"checks")
brute.add_option("--common-tables", dest="commonTables", action="store_true",
help="Check existence of common tables")
brute.add_option("--common-columns", dest="commonColumns", action="store_true",
help="Check existence of common columns")
# User-defined function options
udf = OptionGroup(parser, "User-defined function injection", "These "
"options can be used to create custom user-defined "
"functions")
udf.add_option("--udf-inject", dest="udfInject", action="store_true",
help="Inject custom user-defined functions")
udf.add_option("--shared-lib", dest="shLib",
help="Local path of the shared library")
# File system options
filesystem = OptionGroup(parser, "File system access", "These options "
"can be used to access the back-end database "
"management system underlying file system")
filesystem.add_option("--file-read", dest="rFile",
help="Read a file from the back-end DBMS "
"file system")
filesystem.add_option("--file-write", dest="wFile",
help="Write a local file on the back-end "
"DBMS file system")
filesystem.add_option("--file-dest", dest="dFile",
help="Back-end DBMS absolute filepath to "
"write to")
# Takeover options
takeover = OptionGroup(parser, "Operating system access", "These "
"options can be used to access the back-end "
"database management system underlying "
"operating system")
takeover.add_option("--os-cmd", dest="osCmd",
help="Execute an operating system command")
takeover.add_option("--os-shell", dest="osShell",
action="store_true",
help="Prompt for an interactive operating "
"system shell")
takeover.add_option("--os-pwn", dest="osPwn",
action="store_true",
help="Prompt for an out-of-band shell, "
"meterpreter or VNC")
takeover.add_option("--os-smbrelay", dest="osSmb",
action="store_true",
help="One click prompt for an OOB shell, "
"meterpreter or VNC")
takeover.add_option("--os-bof", dest="osBof",
action="store_true",
help="Stored procedure buffer overflow "
"exploitation")
takeover.add_option("--priv-esc", dest="privEsc",
action="store_true",
help="Database process' user privilege escalation")
takeover.add_option("--msf-path", dest="msfPath",
help="Local path where Metasploit Framework "
"is installed")
takeover.add_option("--tmp-path", dest="tmpPath",
help="Remote absolute path of temporary files "
"directory")
# Windows registry options
windows = OptionGroup(parser, "Windows registry access", "These "
"options can be used to access the back-end "
"database management system Windows "
"registry")
windows.add_option("--reg-read", dest="regRead",
action="store_true",
help="Read a Windows registry key value")
windows.add_option("--reg-add", dest="regAdd",
action="store_true",
help="Write a Windows registry key value data")
windows.add_option("--reg-del", dest="regDel",
action="store_true",
help="Delete a Windows registry key value")
windows.add_option("--reg-key", dest="regKey",
help="Windows registry key")
windows.add_option("--reg-value", dest="regVal",
help="Windows registry key value")
windows.add_option("--reg-data", dest="regData",
help="Windows registry key value data")
windows.add_option("--reg-type", dest="regType",
help="Windows registry key value type")
# General options
general = OptionGroup(parser, "General", "These options can be used "
"to set some general working parameters" )
#general.add_option("-x", dest="xmlFile",
# help="Dump the data into an XML file")
general.add_option("-t", dest="trafficFile",
help="Log all HTTP traffic into a "
"textual file")
general.add_option("--batch", dest="batch",
action="store_true",
help="Never ask for user input, use the default behaviour")
general.add_option("--charset", dest="charset",
help="Force character encoding used for data retrieval")
general.add_option("--check-tor", dest="checkTor",
action="store_true",
help="Check to see if Tor is used properly")
general.add_option("--crawl", dest="crawlDepth", type="int",
help="Crawl the website starting from the target url")
general.add_option("--csv-del", dest="csvDel",
help="Delimiting character used in CSV output "
"(default \"%s\")" % defaults.csvDel)
general.add_option("--dbms-cred", dest="dCred",
help="DBMS authentication credentials (user:password)")
general.add_option("--eta", dest="eta",
action="store_true",
help="Display for each output the "
"estimated time of arrival")
general.add_option("--flush-session", dest="flushSession",
action="store_true",
help="Flush session files for current target")
general.add_option("--forms", dest="forms",
action="store_true",
help="Parse and test forms on target url")
general.add_option("--fresh-queries", dest="freshQueries",
action="store_true",
help="Ignores query results stored in session file")
general.add_option("--hex", dest="hexConvert",
action="store_true",
help="Uses DBMS hex function(s) for data retrieval")
general.add_option("--output-dir", dest="oDir",
action="store",
help="Custom output directory path")
general.add_option("--parse-errors", dest="parseErrors",
action="store_true",
help="Parse and display DBMS error messages from responses")
general.add_option("--replicate", dest="replicate",
action="store_true",
help="Replicate dumped data into a sqlite3 database")
general.add_option("--save", dest="saveCmdline",
action="store_true",
help="Save options to a configuration INI file")
general.add_option("--tor", dest="tor",
action="store_true",
help="Use Tor anonymity network")
general.add_option("--tor-port", dest="torPort",
help="Set Tor proxy port other than default")
general.add_option("--tor-type", dest="torType",
help="Set Tor proxy type (HTTP - default, SOCKS4 or SOCKS5)")
general.add_option("--update", dest="updateAll",
action="store_true",
help="Update sqlmap")
# Miscellaneous options
miscellaneous = OptionGroup(parser, "Miscellaneous")
miscellaneous.add_option("-z", dest="mnemonics",
help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
miscellaneous.add_option("--beep", dest="beep",
action="store_true",
help="Sound alert when SQL injection found")
miscellaneous.add_option("--check-payload", dest="checkPayload",
action="store_true",
help="Offline WAF/IPS/IDS payload detection testing")
miscellaneous.add_option("--check-waf", dest="checkWaf",
action="store_true",
help="Check for existence of WAF/IPS/IDS protection")
miscellaneous.add_option("--cleanup", dest="cleanup",
action="store_true",
help="Clean up the DBMS by sqlmap specific "
"UDF and tables")
miscellaneous.add_option("--dependencies", dest="dependencies",
action="store_true",
help="Check for missing sqlmap dependencies")
miscellaneous.add_option("--disable-hash", dest="disableHash",
action="store_true",
help="Disable password hash cracking mechanism")
miscellaneous.add_option("--disable-like", dest="disableLike",
action="store_true",
help="Disable LIKE search of identificator names")
miscellaneous.add_option("--gpage", dest="googlePage", type="int",
help="Use Google dork results from specified page number")
miscellaneous.add_option("--mobile", dest="mobile",
action="store_true",
help="Imitate smartphone through HTTP User-Agent header")
miscellaneous.add_option("--page-rank", dest="pageRank",
action="store_true",
help="Display page rank (PR) for Google dork results")
miscellaneous.add_option("--purge-output", dest="purgeOutput",
action="store_true",
help="Safely remove all content from output directory")
miscellaneous.add_option("--smart", dest="smart",
action="store_true",
help="Conduct through tests only if positive heuristic(s)")
miscellaneous.add_option("--test-filter", dest="tstF",
help="Select tests by payloads and/or titles (e.g. ROW)")
miscellaneous.add_option("--wizard", dest="wizard",
action="store_true",
help="Simple wizard interface for beginner users")
# Hidden and/or experimental options
parser.add_option("--profile", dest="profile", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--cpu-throttle", dest="cpuThrottle", type="int",
help=SUPPRESS_HELP)
parser.add_option("--smoke-test", dest="smokeTest", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--live-test", dest="liveTest", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--real-test", dest="realTest", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--run-case", dest="runCase", type="int",
help=SUPPRESS_HELP)
parser.add_option_group(target)
parser.add_option_group(request)
parser.add_option_group(optimization)
parser.add_option_group(injection)
parser.add_option_group(detection)
parser.add_option_group(techniques)
parser.add_option_group(fingerprint)
parser.add_option_group(enumeration)
parser.add_option_group(brute)
parser.add_option_group(udf)
parser.add_option_group(filesystem)
parser.add_option_group(takeover)
parser.add_option_group(windows)
parser.add_option_group(general)
parser.add_option_group(miscellaneous)
# Dirty hack for making a short option -hh
option = parser.get_option("--hh")
option._short_opts = ["-hh"]
option._long_opts = []
# Dirty hack for inherent help message of switch -h
option = parser.get_option("-h")
option.help = option.help.capitalize().replace("this help", "basic help")
args = []
advancedHelp = True
for arg in sys.argv:
args.append(getUnicode(arg, system=True))
# Hide non-basic options in basic help case
for i in xrange(len(sys.argv)):
if sys.argv[i] == '-hh':
sys.argv[i] = '-h'
elif sys.argv[i] == '-h':
advancedHelp = False
for group in parser.option_groups[:]:
found = False
for option in group.option_list:
if option.dest not in BASIC_HELP_ITEMS:
option.help = SUPPRESS_HELP
else:
found = True
if not found:
parser.option_groups.remove(group)
try:
(args, _) = parser.parse_args(args)
except SystemExit:
if '-h' in sys.argv and not advancedHelp:
print "\n[!] to see full list of options run with '-hh'"
raise
# Expand given mnemonic options (e.g. -z "ign,flu,bat")
for i in xrange(len(sys.argv) - 1):
if sys.argv[i] == '-z':
expandMnemonics(sys.argv[i+1], parser, args)
if not any((args.direct, args.url, args.logFile, args.bulkFile, args.googleDork, args.configFile, \
args.requestFile, args.updateAll, args.smokeTest, args.liveTest, args.realTest, args.wizard, args.dependencies, args.purgeOutput)):
errMsg = "missing a mandatory option (-d, -u, -l, -m, -r, -g, -c, --wizard, --update, --purge-output or --dependencies), "
errMsg += "use -h for basic or -hh for advanced help"
parser.error(errMsg)
return args
except (OptionError, TypeError), e:
parser.error(e)
def cmdLineParser():
"""
This function parses the command line parameters and arguments
"""
usage = "%s%s [options]" % ("python " if not IS_WIN else "", \
"\"%s\"" % sys.argv[0] if " " in sys.argv[0] else sys.argv[0])
parser = OptionParser(usage=usage)
try:
parser.add_option("--hh",
dest="advancedHelp",
action="store_true",
help="Show advanced help message and exit")
parser.add_option("-v",
dest="verbose",
type="int",
help="Verbosity level: 0-6 (default %d)" %
defaults.verbose)
# Target options
target = OptionGroup(
parser, "Target", "At least one of these "
"options has to be specified to set the source "
"to get target urls from")
target.add_option("-d",
dest="direct",
help="Direct "
"connection to the database")
target.add_option("-u", "--url", dest="url", help="Target url")
target.add_option("-l",
dest="logFile",
help="Parse targets from Burp "
"or WebScarab proxy logs")
target.add_option("-m",
dest="bulkFile",
help="Scan multiple targets enlisted "
"in a given textual file ")
target.add_option("-r",
dest="requestFile",
help="Load HTTP request from a file")
target.add_option("-g",
dest="googleDork",
help="Process Google dork results as target urls")
target.add_option("-c",
dest="configFile",
help="Load options from a configuration INI file")
# Request options
request = OptionGroup(
parser, "Request", "These options can be used "
"to specify how to connect to the target url")
request.add_option("--data",
dest="data",
help="Data string to be sent through POST")
request.add_option(
"--param-del",
dest="pDel",
help="Character used for splitting parameter values")
request.add_option("--cookie",
dest="cookie",
help="HTTP Cookie header")
request.add_option(
"--load-cookies",
dest="loadCookies",
help="File containing cookies in Netscape/wget format")
request.add_option("--drop-set-cookie",
dest="dropSetCookie",
action="store_true",
help="Ignore Set-Cookie header from response")
request.add_option("--user-agent",
dest="agent",
help="HTTP User-Agent header")
request.add_option("--random-agent",
dest="randomAgent",
action="store_true",
help="Use randomly selected HTTP User-Agent header")
request.add_option("--randomize",
dest="rParam",
help="Randomly change value for given parameter(s)")
request.add_option("--force-ssl",
dest="forceSSL",
action="store_true",
help="Force usage of SSL/HTTPS requests")
request.add_option("--host", dest="host", help="HTTP Host header")
request.add_option("--referer",
dest="referer",
help="HTTP Referer header")
request.add_option(
"--headers",
dest="headers",
help="Extra headers (e.g. \"Accept-Language: fr\\nETag: 123\")")
request.add_option("--auth-type",
dest="aType",
help="HTTP authentication type "
"(Basic, Digest or NTLM)")
request.add_option("--auth-cred",
dest="aCred",
help="HTTP authentication credentials "
"(name:password)")
request.add_option("--auth-cert",
dest="aCert",
help="HTTP authentication certificate ("
"key_file,cert_file)")
request.add_option(
"--proxy",
dest="proxy",
help="Use a HTTP proxy to connect to the target url")
request.add_option("--proxy-cred",
dest="pCred",
help="HTTP proxy authentication credentials "
"(name:password)")
request.add_option("--ignore-proxy",
dest="ignoreProxy",
action="store_true",
help="Ignore system default HTTP proxy")
request.add_option("--delay",
dest="delay",
type="float",
help="Delay in seconds between each HTTP request")
request.add_option("--timeout",
dest="timeout",
type="float",
help="Seconds to wait before timeout connection "
"(default %d)" % defaults.timeout)
request.add_option("--retries",
dest="retries",
type="int",
help="Retries when the connection timeouts "
"(default %d)" % defaults.retries)
request.add_option(
"--scope",
dest="scope",
help="Regexp to filter targets from provided proxy log")
request.add_option(
"--safe-url",
dest="safUrl",
help="Url address to visit frequently during testing")
request.add_option(
"--safe-freq",
dest="saFreq",
type="int",
help="Test requests between two visits to a given safe url")
request.add_option("--skip-urlencode",
dest="skipUrlEncode",
action="store_true",
help="Skip URL encoding of payload data")
request.add_option(
"--eval",
dest="evalCode",
help=
"Evaluate provided Python code before the request (e.g. \"import hashlib;id2=hashlib.md5(id).hexdigest()\")"
)
# Optimization options
optimization = OptionGroup(
parser, "Optimization", "These "
"options can be used to optimize the "
"performance of sqlmap")
optimization.add_option("-o",
dest="optimize",
action="store_true",
help="Turn on all optimization switches")
optimization.add_option("--predict-output",
dest="predictOutput",
action="store_true",
help="Predict common queries output")
optimization.add_option("--keep-alive",
dest="keepAlive",
action="store_true",
help="Use persistent HTTP(s) connections")
optimization.add_option(
"--null-connection",
dest="nullConnection",
action="store_true",
help="Retrieve page length without actual HTTP response body")
optimization.add_option("--threads",
dest="threads",
type="int",
help="Max number of concurrent HTTP(s) "
"requests (default %d)" % defaults.threads)
# Injection options
injection = OptionGroup(
parser, "Injection", "These options can be "
"used to specify which parameters to test "
"for, provide custom injection payloads and "
"optional tampering scripts")
injection.add_option("-p",
dest="testParameter",
help="Testable parameter(s)")
injection.add_option("--dbms",
dest="dbms",
help="Force back-end DBMS to this value")
injection.add_option("--os",
dest="os",
help="Force back-end DBMS operating system "
"to this value")
injection.add_option("--invalid-bignum",
dest="invalidBignum",
action="store_true",
help="Use big numbers for invalidating values")
injection.add_option(
"--invalid-logical",
dest="invalidLogical",
action="store_true",
help="Use logical operations for invalidating values")
injection.add_option("--no-cast",
dest="noCast",
action="store_true",
help="Turn off payload casting mechanism")
injection.add_option("--no-unescape",
dest="noUnescape",
action="store_true",
help="Turn off string unescaping mechanism")
injection.add_option("--prefix",
dest="prefix",
help="Injection payload prefix string")
injection.add_option("--suffix",
dest="suffix",
help="Injection payload suffix string")
injection.add_option("--skip",
dest="skip",
help="Skip testing for given parameter(s)")
injection.add_option(
"--tamper",
dest="tamper",
help="Use given script(s) for tampering injection data")
# Detection options
detection = OptionGroup(
parser, "Detection", "These options can be "
"used to specify how to parse "
"and compare page content from "
"HTTP responses when using blind SQL "
"injection technique")
detection.add_option("--level",
dest="level",
type="int",
help="Level of tests to perform (1-5, "
"default %d)" % defaults.level)
detection.add_option("--risk",
dest="risk",
type="int",
help="Risk of tests to perform (0-3, "
"default %d)" % defaults.level)
detection.add_option("--string",
dest="string",
help="String to match when "
"query is evaluated to True")
detection.add_option("--not-string",
dest="notString",
help="String to match when "
"query is evaluated to False")
detection.add_option("--regexp",
dest="regexp",
help="Regexp to match when "
"query is evaluated to True")
detection.add_option("--code",
dest="code",
type="int",
help="HTTP code to match when "
"query is evaluated to True")
detection.add_option(
"--text-only",
dest="textOnly",
action="store_true",
help="Compare pages based only on the textual content")
detection.add_option("--titles",
dest="titles",
action="store_true",
help="Compare pages based only on their titles")
# Techniques options
techniques = OptionGroup(
parser, "Techniques", "These options can be "
"used to tweak testing of specific SQL "
"injection techniques")
techniques.add_option("--technique",
dest="tech",
help="SQL injection techniques to test for "
"(default \"%s\")" % defaults.tech)
techniques.add_option("--time-sec",
dest="timeSec",
type="int",
help="Seconds to delay the DBMS response "
"(default %d)" % defaults.timeSec)
techniques.add_option(
"--union-cols",
dest="uCols",
help="Range of columns to test for UNION query SQL injection")
techniques.add_option(
"--union-char",
dest="uChar",
help="Character to use for bruteforcing number of columns")
techniques.add_option(
"--dns-domain",
dest="dnsName",
help="Domain name used for DNS exfiltration attack")
techniques.add_option(
"--second-order",
dest="secondOrder",
help="Resulting page url searched for second-order "
"response")
# Fingerprint options
fingerprint = OptionGroup(parser, "Fingerprint")
fingerprint.add_option(
"-f",
"--fingerprint",
dest="extensiveFp",
action="store_true",
help="Perform an extensive DBMS version fingerprint")
# Enumeration options
enumeration = OptionGroup(
parser, "Enumeration", "These options can "
"be used to enumerate the back-end database "
"management system information, structure "
"and data contained in the tables. Moreover "
"you can run your own SQL statements")
enumeration.add_option("-b",
"--banner",
dest="getBanner",
action="store_true",
help="Retrieve DBMS banner")
enumeration.add_option("--current-user",
dest="getCurrentUser",
action="store_true",
help="Retrieve DBMS current user")
enumeration.add_option("--current-db",
dest="getCurrentDb",
action="store_true",
help="Retrieve DBMS current database")
enumeration.add_option("--hostname",
dest="getHostname",
action="store_true",
help="Retrieve DBMS server hostname")
enumeration.add_option("--is-dba",
dest="isDba",
action="store_true",
help="Detect if the DBMS current user is DBA")
enumeration.add_option("--users",
dest="getUsers",
action="store_true",
help="Enumerate DBMS users")
enumeration.add_option("--passwords",
dest="getPasswordHashes",
action="store_true",
help="Enumerate DBMS users password hashes")
enumeration.add_option("--privileges",
dest="getPrivileges",
action="store_true",
help="Enumerate DBMS users privileges")
enumeration.add_option("--roles",
dest="getRoles",
action="store_true",
help="Enumerate DBMS users roles")
enumeration.add_option("--dbs",
dest="getDbs",
action="store_true",
help="Enumerate DBMS databases")
enumeration.add_option("--tables",
dest="getTables",
action="store_true",
help="Enumerate DBMS database tables")
enumeration.add_option("--columns",
dest="getColumns",
action="store_true",
help="Enumerate DBMS database table columns")
enumeration.add_option("--schema",
dest="getSchema",
action="store_true",
help="Enumerate DBMS schema")
enumeration.add_option("--count",
dest="getCount",
action="store_true",
help="Retrieve number of entries for table(s)")
enumeration.add_option("--dump",
dest="dumpTable",
action="store_true",
help="Dump DBMS database table entries")
enumeration.add_option("--dump-all",
dest="dumpAll",
action="store_true",
help="Dump all DBMS databases tables entries")
enumeration.add_option(
"--search",
dest="search",
action="store_true",
help="Search column(s), table(s) and/or database name(s)")
enumeration.add_option("-D",
dest="db",
help="DBMS database to enumerate")
enumeration.add_option("-T",
dest="tbl",
help="DBMS database table to enumerate")
enumeration.add_option("-C",
dest="col",
help="DBMS database table column to enumerate")
enumeration.add_option("-U",
dest="user",
help="DBMS user to enumerate")
enumeration.add_option("--exclude-sysdbs",
dest="excludeSysDbs",
action="store_true",
help="Exclude DBMS system databases when "
"enumerating tables")
enumeration.add_option("--start",
dest="limitStart",
type="int",
help="First query output entry to retrieve")
enumeration.add_option("--stop",
dest="limitStop",
type="int",
help="Last query output entry to retrieve")
enumeration.add_option(
"--first",
dest="firstChar",
type="int",
help="First query output word character to retrieve")
enumeration.add_option(
"--last",
dest="lastChar",
type="int",
help="Last query output word character to retrieve")
enumeration.add_option("--sql-query",
dest="query",
help="SQL statement to be executed")
enumeration.add_option("--sql-shell",
dest="sqlShell",
action="store_true",
help="Prompt for an interactive SQL shell")
enumeration.add_option(
"--sql-file",
dest="sqlFile",
help="Execute SQL statements from given file(s)")
# User-defined function options
brute = OptionGroup(
parser, "Brute force", "These "
"options can be used to run brute force "
"checks")
brute.add_option("--common-tables",
dest="commonTables",
action="store_true",
help="Check existence of common tables")
brute.add_option("--common-columns",
dest="commonColumns",
action="store_true",
help="Check existence of common columns")
# User-defined function options
udf = OptionGroup(
parser, "User-defined function injection", "These "
"options can be used to create custom user-defined "
"functions")
udf.add_option("--udf-inject",
dest="udfInject",
action="store_true",
help="Inject custom user-defined functions")
udf.add_option("--shared-lib",
dest="shLib",
help="Local path of the shared library")
# File system options
filesystem = OptionGroup(
parser, "File system access", "These options "
"can be used to access the back-end database "
"management system underlying file system")
filesystem.add_option("--file-read",
dest="rFile",
help="Read a file from the back-end DBMS "
"file system")
filesystem.add_option("--file-write",
dest="wFile",
help="Write a local file on the back-end "
"DBMS file system")
filesystem.add_option("--file-dest",
dest="dFile",
help="Back-end DBMS absolute filepath to "
"write to")
# Takeover options
takeover = OptionGroup(
parser, "Operating system access", "These "
"options can be used to access the back-end "
"database management system underlying "
"operating system")
takeover.add_option("--os-cmd",
dest="osCmd",
help="Execute an operating system command")
takeover.add_option("--os-shell",
dest="osShell",
action="store_true",
help="Prompt for an interactive operating "
"system shell")
takeover.add_option("--os-pwn",
dest="osPwn",
action="store_true",
help="Prompt for an out-of-band shell, "
"meterpreter or VNC")
takeover.add_option("--os-smbrelay",
dest="osSmb",
action="store_true",
help="One click prompt for an OOB shell, "
"meterpreter or VNC")
takeover.add_option("--os-bof",
dest="osBof",
action="store_true",
help="Stored procedure buffer overflow "
"exploitation")
takeover.add_option("--priv-esc",
dest="privEsc",
action="store_true",
help="Database process' user privilege escalation")
takeover.add_option("--msf-path",
dest="msfPath",
help="Local path where Metasploit Framework "
"is installed")
takeover.add_option("--tmp-path",
dest="tmpPath",
help="Remote absolute path of temporary files "
"directory")
# Windows registry options
windows = OptionGroup(
parser, "Windows registry access", "These "
"options can be used to access the back-end "
"database management system Windows "
"registry")
windows.add_option("--reg-read",
dest="regRead",
action="store_true",
help="Read a Windows registry key value")
windows.add_option("--reg-add",
dest="regAdd",
action="store_true",
help="Write a Windows registry key value data")
windows.add_option("--reg-del",
dest="regDel",
action="store_true",
help="Delete a Windows registry key value")
windows.add_option("--reg-key",
dest="regKey",
help="Windows registry key")
windows.add_option("--reg-value",
dest="regVal",
help="Windows registry key value")
windows.add_option("--reg-data",
dest="regData",
help="Windows registry key value data")
windows.add_option("--reg-type",
dest="regType",
help="Windows registry key value type")
# General options
general = OptionGroup(
parser, "General", "These options can be used "
"to set some general working parameters")
#general.add_option("-x", dest="xmlFile",
# help="Dump the data into an XML file")
general.add_option("-t",
dest="trafficFile",
help="Log all HTTP traffic into a "
"textual file")
general.add_option(
"--batch",
dest="batch",
action="store_true",
help="Never ask for user input, use the default behaviour")
general.add_option(
"--charset",
dest="charset",
help="Force character encoding used for data retrieval")
general.add_option("--check-tor",
dest="checkTor",
action="store_true",
help="Check to see if Tor is used properly")
general.add_option(
"--crawl",
dest="crawlDepth",
type="int",
help="Crawl the website starting from the target url")
general.add_option("--csv-del",
dest="csvDel",
help="Delimiting character used in CSV output "
"(default \"%s\")" % defaults.csvDel)
general.add_option(
"--dbms-cred",
dest="dbmsCred",
help="DBMS authentication credentials (user:password)")
general.add_option("--eta",
dest="eta",
action="store_true",
help="Display for each output the "
"estimated time of arrival")
general.add_option("--flush-session",
dest="flushSession",
action="store_true",
help="Flush session files for current target")
general.add_option("--forms",
dest="forms",
action="store_true",
help="Parse and test forms on target url")
general.add_option("--fresh-queries",
dest="freshQueries",
action="store_true",
help="Ignores query results stored in session file")
general.add_option("--hex",
dest="hexConvert",
action="store_true",
help="Uses DBMS hex function(s) for data retrieval")
general.add_option("--output-dir",
dest="oDir",
action="store",
help="Custom output directory path")
general.add_option(
"--parse-errors",
dest="parseErrors",
action="store_true",
help="Parse and display DBMS error messages from responses")
general.add_option(
"--replicate",
dest="replicate",
action="store_true",
help="Replicate dumped data into a sqlite3 database")
general.add_option("--save",
dest="saveCmdline",
action="store_true",
help="Save options to a configuration INI file")
general.add_option("--tor",
dest="tor",
action="store_true",
help="Use Tor anonymity network")
general.add_option("--tor-port",
dest="torPort",
help="Set Tor proxy port other than default")
general.add_option(
"--tor-type",
dest="torType",
help="Set Tor proxy type (HTTP - default, SOCKS4 or SOCKS5)")
general.add_option("--update",
dest="updateAll",
action="store_true",
help="Update sqlmap")
# Miscellaneous options
miscellaneous = OptionGroup(parser, "Miscellaneous")
miscellaneous.add_option(
"-z",
dest="mnemonics",
help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
miscellaneous.add_option(
"--check-payload",
dest="checkPayload",
action="store_true",
help="Offline WAF/IPS/IDS payload detection testing")
miscellaneous.add_option(
"--check-waf",
dest="checkWaf",
action="store_true",
help="Check for existence of WAF/IPS/IDS protection")
miscellaneous.add_option("--cleanup",
dest="cleanup",
action="store_true",
help="Clean up the DBMS by sqlmap specific "
"UDF and tables")
miscellaneous.add_option("--dependencies",
dest="dependencies",
action="store_true",
help="Check for missing sqlmap dependencies")
miscellaneous.add_option("--disable-coloring",
dest="disableColoring",
action="store_true",
help="Disable console output coloring")
miscellaneous.add_option(
"--gpage",
dest="googlePage",
type="int",
help="Use Google dork results from specified page number")
miscellaneous.add_option(
"--mobile",
dest="mobile",
action="store_true",
help="Imitate smartphone through HTTP User-Agent header")
miscellaneous.add_option(
"--page-rank",
dest="pageRank",
action="store_true",
help="Display page rank (PR) for Google dork results")
miscellaneous.add_option(
"--purge-output",
dest="purgeOutput",
action="store_true",
help="Safely remove all content from output directory")
miscellaneous.add_option(
"--smart",
dest="smart",
action="store_true",
help="Conduct through tests only if positive heuristic(s)")
miscellaneous.add_option(
"--test-filter",
dest="testFilter",
help="Select tests by payloads and/or titles (e.g. ROW)")
miscellaneous.add_option(
"--wizard",
dest="wizard",
action="store_true",
help="Simple wizard interface for beginner users")
# Hidden and/or experimental options
parser.add_option("--beep",
dest="beep",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--profile",
dest="profile",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--cpu-throttle",
dest="cpuThrottle",
type="int",
help=SUPPRESS_HELP)
parser.add_option("--force-dns",
dest="forceDns",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--smoke-test",
dest="smokeTest",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--live-test",
dest="liveTest",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--run-case",
dest="runCase",
type="int",
help=SUPPRESS_HELP)
parser.add_option_group(target)
parser.add_option_group(request)
parser.add_option_group(optimization)
parser.add_option_group(injection)
parser.add_option_group(detection)
parser.add_option_group(techniques)
parser.add_option_group(fingerprint)
parser.add_option_group(enumeration)
parser.add_option_group(brute)
parser.add_option_group(udf)
parser.add_option_group(filesystem)
parser.add_option_group(takeover)
parser.add_option_group(windows)
parser.add_option_group(general)
parser.add_option_group(miscellaneous)
# Dirty hack to display longer options without breaking into two lines
def _(self, *args):
_ = parser.formatter._format_option_strings(*args)
if len(_) > MAX_HELP_OPTION_LENGTH:
_ = ("%%.%ds.." % (MAX_HELP_OPTION_LENGTH -
parser.formatter.indent_increment)) % _
return _
parser.formatter._format_option_strings = parser.formatter.format_option_strings
parser.formatter.format_option_strings = type(
parser.formatter.format_option_strings)(_, parser, type(parser))
# Dirty hack for making a short option -hh
option = parser.get_option("--hh")
option._short_opts = ["-hh"]
option._long_opts = []
# Dirty hack for inherent help message of switch -h
option = parser.get_option("-h")
option.help = option.help.capitalize().replace("this help",
"basic help")
args = []
advancedHelp = True
for arg in sys.argv:
args.append(getUnicode(arg, system=True))
# Hide non-basic options in basic help case
for i in xrange(len(sys.argv)):
if sys.argv[i] == '-hh':
sys.argv[i] = '-h'
elif sys.argv[i] == '-h':
advancedHelp = False
for group in parser.option_groups[:]:
found = False
for option in group.option_list:
if option.dest not in BASIC_HELP_ITEMS:
option.help = SUPPRESS_HELP
else:
found = True
if not found:
parser.option_groups.remove(group)
try:
(args, _) = parser.parse_args(args)
except SystemExit:
if '-h' in sys.argv and not advancedHelp:
print "\n[!] to see full list of options run with '-hh'"
raise
# Expand given mnemonic options (e.g. -z "ign,flu,bat")
for i in xrange(len(sys.argv) - 1):
if sys.argv[i] == '-z':
expandMnemonics(sys.argv[i + 1], parser, args)
if not any((args.direct, args.url, args.logFile, args.bulkFile, args.googleDork, args.configFile, \
args.requestFile, args.updateAll, args.smokeTest, args.liveTest, args.wizard, args.dependencies, args.purgeOutput)):
errMsg = "missing a mandatory option (-d, -u, -l, -m, -r, -g, -c, --wizard, --update, --purge-output or --dependencies), "
errMsg += "use -h for basic or -hh for advanced help"
parser.error(errMsg)
return args
except (OptionError, TypeError), e:
parser.error(e)
parser.add_option('-i', '--interval', dest='interval', type='int', default=1,
help='interval to print data', metavar='update_interval')
parser.add_option('-w', '--window', dest='window', type='string', default='64k',
help='iperf window size', metavar='tcp_window')
parser.add_option('-d', '--description', dest='description', type='string',
help='test comment (location, antenna name, distance, etc.)',
metavar='description')
parser.add_option('', '--csv-file', dest='csvfile', type='string',
default=None, help='csv filename', metavar='csvfile.csv')
parser.add_option('', '--summary-file', dest='summaryfile', type='string',
default=None, help='summary filename', metavar='summaryfile.txt')
(options, args) = parser.parse_args()
required_options = ('--client', '--description')
error_message = None
for opt in required_options:
opt_object = parser.get_option(opt)
if eval('options.' + opt_object.dest) == None:
if error_message == None:
error_message = 'Error: %s is a required option.\n' % \
"/".join(opt_object._short_opts)
else:
error_message = '%s Error: %s is a required option.\n' % \
(error_message, "/".join(opt_object._short_opts))
if not error_message == None:
error_message = '\n %s \n %s' % (error_message, parser.format_help())
parser.exit(status=2, msg=error_message)
test = None
try:
csvfile = summaryfile = None
csvfile_option = parser.get_option('--csv-file')
class MRJobLauncher(object):
"""Handle running a MapReduce job on an executable from the command line.
This class will eventually support running arbitrary executables; for now
it only supports :py:class:`~mrjob.job.MRJob` subclasses. Up to v0.5 it is
effectively part of the :py:class:`~mrjob.job.MRJob` class itself and
should not be used externally in any way.
"""
#: :py:class:`optparse.Option` subclass to use with the
#: :py:class:`optparse.OptionParser` instance.
OPTION_CLASS = Option
def __init__(self, script_path=None, args=None, from_cl=False):
"""
:param script_path: Path to script unless it's the first item of *args*
:param args: Command line arguments
:param from_cl: If not using sys.argv but still comming from the
command line (as opposed to a script, e.g. from
mrjob.cmd), don't override the option parser error
function (exit instead of throwing ValueError).
"""
if script_path is not None:
script_path = os.path.abspath(script_path)
self._script_path = script_path
# make sure we respect the $TZ (time zone) environment variable
if hasattr(time, 'tzset'):
time.tzset()
self._passthrough_options = []
self._file_options = []
self.option_parser = OptionParser(usage=self._usage(),
option_class=self.OPTION_CLASS,
add_help_option=False)
self.configure_options()
# don't pass None to parse_args unless we're actually running
# the MRJob script
if args is _READ_ARGS_FROM_SYS_ARGV:
self._cl_args = sys.argv[1:]
else:
# don't pass sys.argv to self.option_parser, and have it
# raise an exception on error rather than printing to stderr
# and exiting.
self._cl_args = args or []
def error(msg):
raise ValueError(msg)
if not from_cl:
self.option_parser.error = error
self.load_options(self._cl_args)
# Make it possible to redirect stdin, stdout, and stderr, for testing
# See sandbox(), below.
#
# These should always read/write bytes, not unicode. Generally,
# on Python 2, sys.std* can read and write bytes, whereas on Python 3,
# you need to use sys.std*.buffer (which doesn't exist on Python 2).
#
# However, certain Python 3 environments, such as Jupyter notebook,
# act more like Python 2. See #1441.
self.stdin = getattr(sys.stdin, 'buffer', sys.stdin)
self.stdout = getattr(sys.stdout, 'buffer', sys.stdout)
self.stderr = getattr(sys.stderr, 'buffer', sys.stderr)
@classmethod
def _usage(cls):
"""Command line usage string for this class"""
return ("usage: mrjob run [script path|executable path|--help]"
" [options] [input files]")
def _print_help(self, options):
"""Print help for this job. This will either print runner
or basic help. Override to allow other kinds of help."""
if options.runner:
_print_help_for_runner(options.runner, options.deprecated)
else:
_print_basic_help(self.option_parser,
self._usage(),
options.deprecated)
@classmethod
def run(cls, args=_READ_ARGS_FROM_SYS_ARGV):
"""Entry point for running job from the command-line.
This is also the entry point when a mapper or reducer is run
by Hadoop Streaming.
Does one of:
* Print step information (:option:`--steps`). See :py:meth:`show_steps`
* Run a mapper (:option:`--mapper`). See :py:meth:`run_mapper`
* Run a combiner (:option:`--combiner`). See :py:meth:`run_combiner`
* Run a reducer (:option:`--reducer`). See :py:meth:`run_reducer`
* Run the entire job. See :py:meth:`run_job`
"""
# load options from the command line
launcher = cls(args=args)
launcher.run_job()
def execute(self):
# Launcher only runs jobs, doesn't do any Hadoop Streaming stuff
self.run_job()
def make_runner(self):
"""Make a runner based on command-line arguments, so we can
launch this job on EMR, on Hadoop, or locally.
:rtype: :py:class:`mrjob.runner.MRJobRunner`
"""
if self.options.runner == 'emr':
# avoid requiring dependencies (such as boto3) for other runners
from mrjob.emr import EMRJobRunner
return EMRJobRunner(**self.emr_job_runner_kwargs())
elif self.options.runner == 'dataproc':
from mrjob.dataproc import DataprocJobRunner
return DataprocJobRunner(**self.dataproc_job_runner_kwargs())
elif self.options.runner == 'hadoop':
from mrjob.hadoop import HadoopJobRunner
return HadoopJobRunner(**self.hadoop_job_runner_kwargs())
elif self.options.runner == 'inline':
raise ValueError("inline is not supported in the multi-lingual"
" launcher.")
else:
# run locally by default
from mrjob.local import LocalMRJobRunner
return LocalMRJobRunner(**self.local_job_runner_kwargs())
@classmethod
def set_up_logging(cls, quiet=False, verbose=False, stream=None):
"""Set up logging when running from the command line. This is also
used by the various command-line utilities.
:param bool quiet: If true, don't log. Overrides *verbose*.
:param bool verbose: If true, set log level to ``DEBUG`` (default is
``INFO``)
:param bool stream: Stream to log to (default is ``sys.stderr``)
This will also set up a null log handler for boto3, so we don't get
warnings if boto3 tries to log about throttling and whatnot.
"""
if quiet:
log_to_null(name='mrjob')
log_to_null(name='__main__')
else:
log_to_stream(name='mrjob', debug=verbose, stream=stream)
log_to_stream(name='__main__', debug=verbose, stream=stream)
log_to_null(name='boto3')
def run_job(self):
"""Run the all steps of the job, logging errors (and debugging output
if :option:`--verbose` is specified) to STDERR and streaming the
output to STDOUT.
Called from :py:meth:`run`. You'd probably only want to call this
directly from automated tests.
"""
# self.stderr is strictly binary, need to wrap it so it's possible
# to log to it in Python 3
log_stream = codecs.getwriter('utf_8')(self.stderr)
self.set_up_logging(quiet=self.options.quiet,
verbose=self.options.verbose,
stream=log_stream)
with self.make_runner() as runner:
try:
runner.run()
except StepFailedException as e:
# no need for a runner stacktrace if step failed; runners will
# log more useful information anyway
log.error(str(e))
sys.exit(1)
if not self.options.no_output:
for chunk in runner.cat_output():
self.stdout.write(chunk)
self.stdout.flush()
### Command-line arguments ###
def configure_options(self):
"""Define arguments for this script. Called from :py:meth:`__init__()`.
Re-define to define custom command-line arguments or pass
through existing ones::
def configure_options(self):
super(MRYourJob, self).configure_options
self.add_passthrough_option(...)
self.add_file_option(...)
self.pass_through_option(...)
...
"""
self.option_parser.add_option(
'-h', '--help', dest='help', action='store_true', default=False,
help='show this message and exit')
self.option_parser.add_option(
'--deprecated', dest='deprecated', action='store_true',
default=False,
help='include help for deprecated options')
_add_basic_options(self.option_parser)
_add_job_options(self.option_parser)
_add_runner_options(self.option_parser, _pick_runner_opts())
def is_task(self):
"""True if this is a mapper, combiner, or reducer.
This is mostly useful inside :py:meth:`load_options`, to disable
loading options when we aren't running inside Hadoop Streaming.
"""
return False
def add_passthrough_option(self, *args, **kwargs):
"""Function to create options which both the job runner
and the job itself respect (we use this for protocols, for example).
Use it like you would use :py:func:`optparse.OptionParser.add_option`::
def configure_options(self):
super(MRYourJob, self).configure_options()
self.add_passthrough_option(
'--max-ngram-size', type='int', default=4, help='...')
Specify an *opt_group* keyword argument to add the option to that
:py:class:`OptionGroup` rather than the top-level
:py:class:`OptionParser`.
If you want to pass files through to the mapper/reducer, use
:py:meth:`add_file_option` instead.
If you want to pass through a built-in option (e.g. ``--runner``, use
:py:meth:`pass_through_option` instead.
"""
if 'opt_group' in kwargs:
pass_opt = kwargs.pop('opt_group').add_option(*args, **kwargs)
else:
pass_opt = self.option_parser.add_option(*args, **kwargs)
self._passthrough_options.append(pass_opt)
def pass_through_option(self, opt_str):
"""Pass through a built-in option to tasks. For example, for
tasks to see which runner launched them::
def configure_options(self):
super(MRYourJob, self).configure_options()
self.pass_through_option('--runner')
def mapper_init(self):
if self.options.runner == 'emr':
...
*opt_str* can be a long option switch like ``--runner`` or a short
one like ``-r``.
.. versionadded:: 0.5.4
"""
self._passthrough_options.append(
self.option_parser.get_option(opt_str))
def add_file_option(self, *args, **kwargs):
"""Add a command-line option that sends an external file
(e.g. a SQLite DB) to Hadoop::
def configure_options(self):
super(MRYourJob, self).configure_options()
self.add_file_option('--scoring-db', help=...)
This does the right thing: the file will be uploaded to the working
dir of the script on Hadoop, and the script will be passed the same
option, but with the local name of the file in the script's working
directory.
We suggest against sending Berkeley DBs to your job, as
Berkeley DB is not forwards-compatible (so a Berkeley DB that you
construct on your computer may not be readable from within
Hadoop). Use SQLite databases instead. If all you need is an on-disk
hash table, try out the :py:mod:`sqlite3dbm` module.
"""
pass_opt = self.option_parser.add_option(*args, **kwargs)
if not pass_opt.type == 'string':
raise OptionError(
'passthrough file options must take strings' % pass_opt.type)
if pass_opt.action not in ('store', 'append'):
raise OptionError("passthrough file options must use the options"
" 'store' or 'append'")
self._file_options.append(pass_opt)
def _process_args(self, args):
"""mrjob.launch takes the first arg as the script path, but mrjob.job
uses all args as input files. This method determines the behavior:
MRJobLauncher takes off the first arg as the script path.
"""
if not self._script_path:
if len(args) < 1:
self.option_parser.error('Must supply script path')
else:
self._script_path = os.path.abspath(args[0])
self.args = args[1:]
def load_options(self, args):
"""Load command-line options into ``self.options``,
``self._script_path``, and ``self.args``.
Called from :py:meth:`__init__()` after :py:meth:`configure_options`.
:type args: list of str
:param args: a list of command line arguments. ``None`` will be
treated the same as ``[]``.
Re-define if you want to post-process command-line arguments::
def load_options(self, args):
super(MRYourJob, self).load_options(args)
self.stop_words = self.options.stop_words.split(',')
...
"""
self.options, args = self.option_parser.parse_args(args)
if self.options.help:
self._print_help(self.options)
sys.exit(0)
self._process_args(args)
def inline_job_runner_kwargs(self):
"""Keyword arguments to create create runners when
:py:meth:`make_runner` is called, when we run a job locally
(``-r inline``).
:return: map from arg name to value
Re-define this if you want finer control when running jobs locally.
"""
return self._job_runner_kwargs_for_runner('inline')
def local_job_runner_kwargs(self):
"""Keyword arguments to create create runners when
:py:meth:`make_runner` is called, when we run a job locally
(``-r local``).
:return: map from arg name to value
Re-define this if you want finer control when running jobs locally.
"""
return self._job_runner_kwargs_for_runner('local')
def emr_job_runner_kwargs(self):
"""Keyword arguments to create create runners when
:py:meth:`make_runner` is called, when we run a job on EMR
(``-r emr``).
:return: map from arg name to value
Re-define this if you want finer control when running jobs on EMR.
"""
return self._job_runner_kwargs_for_runner('emr')
def dataproc_job_runner_kwargs(self):
"""Keyword arguments to create create runners when
:py:meth:`make_runner` is called, when we run a job on EMR
(``-r emr``).
:return: map from arg name to value
Re-define this if you want finer control when running jobs on EMR.
"""
return self._job_runner_kwargs_for_runner('dataproc')
def hadoop_job_runner_kwargs(self):
"""Keyword arguments to create create runners when
:py:meth:`make_runner` is called, when we run a job on EMR
(``-r hadoop``).
:return: map from arg name to value
Re-define this if you want finer control when running jobs on hadoop.
"""
return self._job_runner_kwargs_for_runner('hadoop')
def _job_runner_kwargs_for_runner(self, runner_alias):
"""Helper method that powers the *_job_runner_kwargs()
methods."""
# user can no longer silently ignore switches by overriding
# job_runner_kwargs()
return combine_dicts(
self._kwargs_from_switches(_allowed_keys(runner_alias)),
self.job_runner_kwargs(),
)
def job_runner_kwargs(self):
"""Keyword arguments used to create runners when
:py:meth:`make_runner` is called.
:return: map from arg name to value
Re-define this if you want finer control of runner initialization.
You might find :py:meth:`mrjob.conf.combine_dicts` useful if you
want to add or change lots of keyword arguments.
"""
return combine_dicts(
self._non_option_kwargs(),
self._kwargs_from_switches(_allowed_keys('base')),
self._job_kwargs(),
)
def _non_option_kwargs(self):
"""Keyword arguments to runner constructor that can't be set
in mrjob.conf.
These should match the (named) arguments to
:py:meth:`~mrjob.runner.MRJobRunner.__init__`.
"""
return dict(
conf_paths=self.options.conf_paths,
extra_args=self.generate_passthrough_arguments(),
file_upload_args=self.generate_file_upload_args(),
hadoop_input_format=self.hadoop_input_format(),
hadoop_output_format=self.hadoop_output_format(),
input_paths=self.args,
mr_job_script=self._script_path,
output_dir=self.options.output_dir,
partitioner=self.partitioner(),
stdin=self.stdin,
step_output_dir=self.options.step_output_dir,
)
def _kwargs_from_switches(self, keys):
return dict(
(key, getattr(self.options, key))
for key in keys if hasattr(self.options, key)
)
def _job_kwargs(self):
"""Keyword arguments to the runner class that can be specified
by the job/launcher itself."""
return dict(
jobconf=self.jobconf(),
libjars=self.libjars(),
partitioner=self.partitioner(),
sort_values=self.sort_values(),
)
### Hooks for options defined by the job ###
def hadoop_input_format(self):
"""See :py:meth:`mrjob.job.MRJob.hadoop_input_format`."""
return None
def hadoop_output_format(self):
"""See :py:meth:`mrjob.job.MRJob.hadoop_output_format`."""
return None
def jobconf(self):
"""See :py:meth:`mrjob.job.MRJob.jobconf`."""
return {}
def libjars(self):
"""See :py:meth:`mrjob.job.MRJob.libjars`."""
return []
def partitioner(self):
"""See :py:meth:`mrjob.job.MRJob.partitioner`."""
return None
def sort_values(self):
"""See :py:meth:`mrjob.job.MRJob.sort_values`."""
return None
### More option stuff ###
def generate_passthrough_arguments(self):
"""Returns a list of arguments to pass to subprocesses, either on
hadoop or executed via subprocess.
These are passed to :py:meth:`mrjob.runner.MRJobRunner.__init__`
as *extra_args*.
"""
arg_map = parse_and_save_options(self.option_parser, self._cl_args)
output_args = []
passthrough_dests = sorted(
set(option.dest for option in self._passthrough_options))
for option_dest in passthrough_dests:
output_args.extend(arg_map.get(option_dest, []))
return output_args
def generate_file_upload_args(self):
"""Figure out file upload args to pass through to the job runner.
Instead of generating a list of args, we're generating a list
of tuples of ``('--argname', path)``
These are passed to :py:meth:`mrjob.runner.MRJobRunner.__init__`
as ``file_upload_args``.
"""
file_upload_args = []
master_option_dict = self.options.__dict__
for opt in self._file_options:
opt_prefix = opt.get_opt_string()
opt_value = master_option_dict[opt.dest]
if opt_value:
paths = opt_value if opt.action == 'append' else [opt_value]
for path in paths:
file_upload_args.append((opt_prefix, path))
return file_upload_args
### Testing ###
def sandbox(self, stdin=None, stdout=None, stderr=None):
"""Redirect stdin, stdout, and stderr for automated testing.
You can set stdin, stdout, and stderr to file objects. By
default, they'll be set to empty ``BytesIO`` objects.
You can then access the job's file handles through ``self.stdin``,
``self.stdout``, and ``self.stderr``. See :ref:`testing` for more
information about testing.
You may call sandbox multiple times (this will essentially clear
the file handles).
``stdin`` is empty by default. You can set it to anything that yields
lines::
mr_job.sandbox(stdin=BytesIO(b'some_data\\n'))
or, equivalently::
mr_job.sandbox(stdin=[b'some_data\\n'])
For convenience, this sandbox() returns self, so you can do::
mr_job = MRJobClassToTest().sandbox()
Simple testing example::
mr_job = MRYourJob.sandbox()
self.assertEqual(list(mr_job.reducer('foo', ['a', 'b'])), [...])
More complex testing example::
from BytesIO import BytesIO
from mrjob.parse import parse_mr_job_stderr
from mrjob.protocol import JSONProtocol
mr_job = MRYourJob(args=[...])
fake_input = '"foo"\\t"bar"\\n"foo"\\t"baz"\\n'
mr_job.sandbox(stdin=BytesIO(fake_input))
mr_job.run_reducer(link_num=0)
self.assertEqual(mrjob.stdout.getvalue(), ...)
self.assertEqual(parse_mr_job_stderr(mr_job.stderr), ...)
"""
self.stdin = stdin or BytesIO()
self.stdout = stdout or BytesIO()
self.stderr = stderr or BytesIO()
return self
def processArguments():
'''init process arguments '''
parser = OptionParser(usage="Usage: deepin-screenshot [options] [arg]",
version="deepin-screenshot v2.1")
parser.add_option("-f",
"--full",
action="store_true",
dest="fullscreen",
help=_("Take a screenshot of full screen"))
parser.add_option("-w",
"--window",
action="store_true",
dest="window",
help=_("Take a screenshot of the window"))
parser.add_option("-d",
"--delay",
dest="delay",
type="int",
help=_("Take a screenshot after NUM seconds"),
metavar="NUM")
parser.add_option("-s",
"--save",
dest="save_file",
help=_("save screenshot to FILE"),
metavar="FILE")
parser.add_option("--sub",
action="store_true",
dest="sub",
help=_("run as a subprocess"))
parser.add_option("-n",
"--new",
action="store_true",
dest="new",
help=_("run a new process"))
parser.add_option("-I", "--icon", action="store_true", dest="icon")
#parser.add_option("-a", "--area", help="Grab an area of the screen instead of the entire screen", action="store_true")
#parser.add_option("-e", "--border-effect", action="store_true", dest="border_effect", help="Effect to add to the border")
#parser.add_option("-i", "--interactive", action="store_true", help="Interactively set options")
#parser.add_option("-b", "--include-border", action="store_true", help="Include the window border with the screenshot")
#parser.add_option("-B", "--remove-border", action="store_true", help="Remove the window border from the screenshot")
#parser.add_option("-c", "--clipboard", help="Send the grab directly to the clipboard", action="store_true")
parser.get_option('-h').help = _("show this help message and exit")
parser.get_option('--version').help = _(
"show program's version number and exit")
import sys
if '-h' in sys.argv or '--help' in sys.argv:
parser.remove_option("-I")
(options, args) = parser.parse_args()
if not options.new and IS_EXISTS:
print "deepint-screenshot has run"
exit(1)
if options.fullscreen and options.window:
parser.error("options -f and -w are mutually exclusive")
config.OPTION_ICON = options.icon
config.OPTION_FULLSCREEN = options.fullscreen
config.OPTION_WINDOWN = options.window
config.OPTION_NEW = options.new
config.OPTION_FILE = options.save_file
config.OPTION_SUB = options.sub
if options.delay:
notify("Deepin Screenshot",
0,
summary=_("DScreenshot"),
body=_("Deepin Screenshot will start in %d seconds.") %
options.delay,
timeout=(options.delay - 0.5) * 1000)
loop = gobject.MainLoop()
gobject.timeout_add_seconds(options.delay, loop.quit)
loop.run()
if options.save_file:
parserFile = parser_path(str(options.save_file))
if options.fullscreen:
pixbuf = get_screenshot_pixbuf(True)
pixbuf.save(parserFile[0], parserFile[1])
elif options.window:
pixbuf = get_screenshot_pixbuf(False)
pixbuf.save(parserFile[0], parserFile[1])
else:
main()
elif options.fullscreen:
open_file_dialog()
elif options.window:
open_file_dialog(False)
else:
main()
def cmdLineParser(argv=None):
"""
This function parses the command line parameters and arguments
"""
if not argv:
argv = sys.argv
checkSystemEncoding()
# Reference: https://stackoverflow.com/a/4012683 (Note: previously used "...sys.getfilesystemencoding() or UNICODE_ENCODING")
_ = getUnicode(os.path.basename(argv[0]), encoding=sys.stdin.encoding)
usage = "%s%s [options]" % ("python " if not IS_WIN else "", \
"\"%s\"" % _ if " " in _ else _)
parser = OptionParser(usage=usage)
try:
parser.add_option("--hh",
dest="advancedHelp",
action="store_true",
help="Show advanced help message and exit")
parser.add_option("--version",
dest="showVersion",
action="store_true",
help="Show program's version number and exit")
parser.add_option("-v",
dest="verbose",
type="int",
help="Verbosity level: 0-6 (default %d)" %
defaults.verbose)
# Target options
target = OptionGroup(
parser, "Target", "At least one of these "
"options has to be provided to define the target(s)")
target.add_option("-d",
dest="direct",
help="Connection string "
"for direct database connection")
target.add_option(
"-u",
"--url",
dest="url",
help="Target URL (e.g. \"http://www.site.com/vuln.php?id=1\")")
target.add_option("-l",
dest="logFile",
help="Parse target(s) from Burp "
"or WebScarab proxy log file")
target.add_option(
"-x",
dest="sitemapUrl",
help="Parse target(s) from remote sitemap(.xml) file")
target.add_option("-m",
dest="bulkFile",
help="Scan multiple targets given "
"in a textual file ")
target.add_option("-r",
dest="requestFile",
help="Load HTTP request from a file")
target.add_option("-g",
dest="googleDork",
help="Process Google dork results as target URLs")
target.add_option("-c",
dest="configFile",
help="Load options from a configuration INI file")
# Request options
request = OptionGroup(
parser, "Request", "These options can be used "
"to specify how to connect to the target URL")
request.add_option("--method",
dest="method",
help="Force usage of given HTTP method (e.g. PUT)")
request.add_option("--data",
dest="data",
help="Data string to be sent through POST")
request.add_option(
"--param-del",
dest="paramDel",
help="Character used for splitting parameter values")
request.add_option("--cookie",
dest="cookie",
help="HTTP Cookie header value")
request.add_option("--cookie-del",
dest="cookieDel",
help="Character used for splitting cookie values")
request.add_option(
"--load-cookies",
dest="loadCookies",
help="File containing cookies in Netscape/wget format")
request.add_option("--drop-set-cookie",
dest="dropSetCookie",
action="store_true",
help="Ignore Set-Cookie header from response")
request.add_option("--user-agent",
dest="agent",
help="HTTP User-Agent header value")
request.add_option(
"--random-agent",
dest="randomAgent",
action="store_true",
help="Use randomly selected HTTP User-Agent header value")
request.add_option("--host",
dest="host",
help="HTTP Host header value")
request.add_option("--referer",
dest="referer",
help="HTTP Referer header value")
request.add_option(
"-H",
"--header",
dest="header",
help="Extra header (e.g. \"X-Forwarded-For: 127.0.0.1\")")
request.add_option(
"--headers",
dest="headers",
help="Extra headers (e.g. \"Accept-Language: fr\\nETag: 123\")")
request.add_option("--auth-type",
dest="authType",
help="HTTP authentication type "
"(Basic, Digest, NTLM or PKI)")
request.add_option("--auth-cred",
dest="authCred",
help="HTTP authentication credentials "
"(name:password)")
request.add_option(
"--auth-file",
dest="authFile",
help="HTTP authentication PEM cert/private key file")
request.add_option("--ignore-400",
dest="ignore400",
action="store_true",
help="Suppress HTTP Error 400 (Bad Request)")
request.add_option("--ignore-401",
dest="ignore401",
action="store_true",
help="Ignore HTTP Error 401 (Unauthorized)")
request.add_option("--ignore-proxy",
dest="ignoreProxy",
action="store_true",
help="Ignore system default proxy settings")
request.add_option("--ignore-redirects",
dest="ignoreRedirects",
action="store_true",
help="Ignore redirection attempts")
request.add_option("--ignore-timeouts",
dest="ignoreTimeouts",
action="store_true",
help="Ignore connection timeouts")
request.add_option("--proxy",
dest="proxy",
help="Use a proxy to connect to the target URL")
request.add_option("--proxy-cred",
dest="proxyCred",
help="Proxy authentication credentials "
"(name:password)")
request.add_option("--proxy-file",
dest="proxyFile",
help="Load proxy list from a file")
request.add_option("--tor",
dest="tor",
action="store_true",
help="Use Tor anonymity network")
request.add_option("--tor-port",
dest="torPort",
help="Set Tor proxy port other than default")
request.add_option(
"--tor-type",
dest="torType",
help="Set Tor proxy type (HTTP, SOCKS4 or SOCKS5 (default))")
request.add_option("--check-tor",
dest="checkTor",
action="store_true",
help="Check to see if Tor is used properly")
request.add_option("--delay",
dest="delay",
type="float",
help="Delay in seconds between each HTTP request")
request.add_option("--timeout",
dest="timeout",
type="float",
help="Seconds to wait before timeout connection "
"(default %d)" % defaults.timeout)
request.add_option("--retries",
dest="retries",
type="int",
help="Retries when the connection timeouts "
"(default %d)" % defaults.retries)
request.add_option("--randomize",
dest="rParam",
help="Randomly change value for given parameter(s)")
request.add_option(
"--safe-url",
dest="safeUrl",
help="URL address to visit frequently during testing")
request.add_option("--safe-post",
dest="safePost",
help="POST data to send to a safe URL")
request.add_option("--safe-req",
dest="safeReqFile",
help="Load safe HTTP request from a file")
request.add_option(
"--safe-freq",
dest="safeFreq",
type="int",
help="Test requests between two visits to a given safe URL")
request.add_option("--skip-urlencode",
dest="skipUrlEncode",
action="store_true",
help="Skip URL encoding of payload data")
request.add_option("--csrf-token",
dest="csrfToken",
help="Parameter used to hold anti-CSRF token")
request.add_option(
"--csrf-url",
dest="csrfUrl",
help="URL address to visit to extract anti-CSRF token")
request.add_option("--force-ssl",
dest="forceSSL",
action="store_true",
help="Force usage of SSL/HTTPS")
request.add_option("--hpp",
dest="hpp",
action="store_true",
help="Use HTTP parameter pollution method")
request.add_option(
"--eval",
dest="evalCode",
help=
"Evaluate provided Python code before the request (e.g. \"import hashlib;id2=hashlib.md5(id).hexdigest()\")"
)
# Optimization options
optimization = OptionGroup(
parser, "Optimization", "These "
"options can be used to optimize the "
"performance of sqlmap")
optimization.add_option("-o",
dest="optimize",
action="store_true",
help="Turn on all optimization switches")
optimization.add_option("--predict-output",
dest="predictOutput",
action="store_true",
help="Predict common queries output")
optimization.add_option("--keep-alive",
dest="keepAlive",
action="store_true",
help="Use persistent HTTP(s) connections")
optimization.add_option(
"--null-connection",
dest="nullConnection",
action="store_true",
help="Retrieve page length without actual HTTP response body")
optimization.add_option("--threads",
dest="threads",
type="int",
help="Max number of concurrent HTTP(s) "
"requests (default %d)" % defaults.threads)
# Injection options
injection = OptionGroup(
parser, "Injection", "These options can be "
"used to specify which parameters to test "
"for, provide custom injection payloads and "
"optional tampering scripts")
injection.add_option("-p",
dest="testParameter",
help="Testable parameter(s)")
injection.add_option("--skip",
dest="skip",
help="Skip testing for given parameter(s)")
injection.add_option(
"--skip-static",
dest="skipStatic",
action="store_true",
help="Skip testing parameters that not appear to be dynamic")
injection.add_option(
"--param-exclude",
dest="paramExclude",
help="Regexp to exclude parameters from testing (e.g. \"ses\")")
injection.add_option("--dbms",
dest="dbms",
help="Force back-end DBMS to this value")
injection.add_option(
"--dbms-cred",
dest="dbmsCred",
help="DBMS authentication credentials (user:password)")
injection.add_option("--os",
dest="os",
help="Force back-end DBMS operating system "
"to this value")
injection.add_option("--invalid-bignum",
dest="invalidBignum",
action="store_true",
help="Use big numbers for invalidating values")
injection.add_option(
"--invalid-logical",
dest="invalidLogical",
action="store_true",
help="Use logical operations for invalidating values")
injection.add_option("--invalid-string",
dest="invalidString",
action="store_true",
help="Use random strings for invalidating values")
injection.add_option("--no-cast",
dest="noCast",
action="store_true",
help="Turn off payload casting mechanism")
injection.add_option("--no-escape",
dest="noEscape",
action="store_true",
help="Turn off string escaping mechanism")
injection.add_option("--prefix",
dest="prefix",
help="Injection payload prefix string")
injection.add_option("--suffix",
dest="suffix",
help="Injection payload suffix string")
injection.add_option(
"--tamper",
dest="tamper",
help="Use given script(s) for tampering injection data")
# Detection options
detection = OptionGroup(
parser, "Detection", "These options can be "
"used to customize the detection phase")
detection.add_option("--level",
dest="level",
type="int",
help="Level of tests to perform (1-5, "
"default %d)" % defaults.level)
detection.add_option("--risk",
dest="risk",
type="int",
help="Risk of tests to perform (1-3, "
"default %d)" % defaults.level)
detection.add_option("--string",
dest="string",
help="String to match when "
"query is evaluated to True")
detection.add_option("--not-string",
dest="notString",
help="String to match when "
"query is evaluated to False")
detection.add_option("--regexp",
dest="regexp",
help="Regexp to match when "
"query is evaluated to True")
detection.add_option("--code",
dest="code",
type="int",
help="HTTP code to match when "
"query is evaluated to True")
detection.add_option(
"--text-only",
dest="textOnly",
action="store_true",
help="Compare pages based only on the textual content")
detection.add_option("--titles",
dest="titles",
action="store_true",
help="Compare pages based only on their titles")
# Techniques options
techniques = OptionGroup(
parser, "Techniques", "These options can be "
"used to tweak testing of specific SQL "
"injection techniques")
techniques.add_option("--technique",
dest="tech",
help="SQL injection techniques to use "
"(default \"%s\")" % defaults.tech)
techniques.add_option("--time-sec",
dest="timeSec",
type="int",
help="Seconds to delay the DBMS response "
"(default %d)" % defaults.timeSec)
techniques.add_option(
"--union-cols",
dest="uCols",
help="Range of columns to test for UNION query SQL injection")
techniques.add_option(
"--union-char",
dest="uChar",
help="Character to use for bruteforcing number of columns")
techniques.add_option(
"--union-from",
dest="uFrom",
help="Table to use in FROM part of UNION query SQL injection")
techniques.add_option(
"--dns-domain",
dest="dnsDomain",
help="Domain name used for DNS exfiltration attack")
techniques.add_option(
"--second-order",
dest="secondOrder",
help="Resulting page URL searched for second-order "
"response")
# Fingerprint options
fingerprint = OptionGroup(parser, "Fingerprint")
fingerprint.add_option(
"-f",
"--fingerprint",
dest="extensiveFp",
action="store_true",
help="Perform an extensive DBMS version fingerprint")
# Enumeration options
enumeration = OptionGroup(
parser, "Enumeration", "These options can "
"be used to enumerate the back-end database "
"management system information, structure "
"and data contained in the tables. Moreover "
"you can run your own SQL statements")
enumeration.add_option("-a",
"--all",
dest="getAll",
action="store_true",
help="Retrieve everything")
enumeration.add_option("-b",
"--banner",
dest="getBanner",
action="store_true",
help="Retrieve DBMS banner")
enumeration.add_option("--current-user",
dest="getCurrentUser",
action="store_true",
help="Retrieve DBMS current user")
enumeration.add_option("--current-db",
dest="getCurrentDb",
action="store_true",
help="Retrieve DBMS current database")
enumeration.add_option("--hostname",
dest="getHostname",
action="store_true",
help="Retrieve DBMS server hostname")
enumeration.add_option("--is-dba",
dest="isDba",
action="store_true",
help="Detect if the DBMS current user is DBA")
enumeration.add_option("--users",
dest="getUsers",
action="store_true",
help="Enumerate DBMS users")
enumeration.add_option("--passwords",
dest="getPasswordHashes",
action="store_true",
help="Enumerate DBMS users password hashes")
enumeration.add_option("--privileges",
dest="getPrivileges",
action="store_true",
help="Enumerate DBMS users privileges")
enumeration.add_option("--roles",
dest="getRoles",
action="store_true",
help="Enumerate DBMS users roles")
enumeration.add_option("--dbs",
dest="getDbs",
action="store_true",
help="Enumerate DBMS databases")
enumeration.add_option("--tables",
dest="getTables",
action="store_true",
help="Enumerate DBMS database tables")
enumeration.add_option("--columns",
dest="getColumns",
action="store_true",
help="Enumerate DBMS database table columns")
enumeration.add_option("--schema",
dest="getSchema",
action="store_true",
help="Enumerate DBMS schema")
enumeration.add_option("--count",
dest="getCount",
action="store_true",
help="Retrieve number of entries for table(s)")
enumeration.add_option("--dump",
dest="dumpTable",
action="store_true",
help="Dump DBMS database table entries")
enumeration.add_option("--dump-all",
dest="dumpAll",
action="store_true",
help="Dump all DBMS databases tables entries")
enumeration.add_option(
"--search",
dest="search",
action="store_true",
help="Search column(s), table(s) and/or database name(s)")
enumeration.add_option("--comments",
dest="getComments",
action="store_true",
help="Retrieve DBMS comments")
enumeration.add_option("-D",
dest="db",
help="DBMS database to enumerate")
enumeration.add_option("-T",
dest="tbl",
help="DBMS database table(s) to enumerate")
enumeration.add_option(
"-C",
dest="col",
help="DBMS database table column(s) to enumerate")
enumeration.add_option(
"-X",
dest="excludeCol",
help="DBMS database table column(s) to not enumerate")
enumeration.add_option("-U",
dest="user",
help="DBMS user to enumerate")
enumeration.add_option("--exclude-sysdbs",
dest="excludeSysDbs",
action="store_true",
help="Exclude DBMS system databases when "
"enumerating tables")
enumeration.add_option("--pivot-column",
dest="pivotColumn",
help="Pivot column name")
enumeration.add_option("--where",
dest="dumpWhere",
help="Use WHERE condition while table dumping")
enumeration.add_option("--start",
dest="limitStart",
type="int",
help="First dump table entry to retrieve")
enumeration.add_option("--stop",
dest="limitStop",
type="int",
help="Last dump table entry to retrieve")
enumeration.add_option(
"--first",
dest="firstChar",
type="int",
help="First query output word character to retrieve")
enumeration.add_option(
"--last",
dest="lastChar",
type="int",
help="Last query output word character to retrieve")
enumeration.add_option("--sql-query",
dest="query",
help="SQL statement to be executed")
enumeration.add_option("--sql-shell",
dest="sqlShell",
action="store_true",
help="Prompt for an interactive SQL shell")
enumeration.add_option(
"--sql-file",
dest="sqlFile",
help="Execute SQL statements from given file(s)")
# Brute force options
brute = OptionGroup(
parser, "Brute force", "These "
"options can be used to run brute force "
"checks")
brute.add_option("--common-tables",
dest="commonTables",
action="store_true",
help="Check existence of common tables")
brute.add_option("--common-columns",
dest="commonColumns",
action="store_true",
help="Check existence of common columns")
# User-defined function options
udf = OptionGroup(
parser, "User-defined function injection", "These "
"options can be used to create custom user-defined "
"functions")
udf.add_option("--udf-inject",
dest="udfInject",
action="store_true",
help="Inject custom user-defined functions")
udf.add_option("--shared-lib",
dest="shLib",
help="Local path of the shared library")
# File system options
filesystem = OptionGroup(
parser, "File system access", "These options "
"can be used to access the back-end database "
"management system underlying file system")
filesystem.add_option("--file-read",
dest="rFile",
help="Read a file from the back-end DBMS "
"file system")
filesystem.add_option("--file-write",
dest="wFile",
help="Write a local file on the back-end "
"DBMS file system")
filesystem.add_option("--file-dest",
dest="dFile",
help="Back-end DBMS absolute filepath to "
"write to")
# Takeover options
takeover = OptionGroup(
parser, "Operating system access", "These "
"options can be used to access the back-end "
"database management system underlying "
"operating system")
takeover.add_option("--os-cmd",
dest="osCmd",
help="Execute an operating system command")
takeover.add_option("--os-shell",
dest="osShell",
action="store_true",
help="Prompt for an interactive operating "
"system shell")
takeover.add_option("--os-pwn",
dest="osPwn",
action="store_true",
help="Prompt for an OOB shell, "
"Meterpreter or VNC")
takeover.add_option("--os-smbrelay",
dest="osSmb",
action="store_true",
help="One click prompt for an OOB shell, "
"Meterpreter or VNC")
takeover.add_option("--os-bof",
dest="osBof",
action="store_true",
help="Stored procedure buffer overflow "
"exploitation")
takeover.add_option("--priv-esc",
dest="privEsc",
action="store_true",
help="Database process user privilege escalation")
takeover.add_option("--msf-path",
dest="msfPath",
help="Local path where Metasploit Framework "
"is installed")
takeover.add_option("--tmp-path",
dest="tmpPath",
help="Remote absolute path of temporary files "
"directory")
# Windows registry options
windows = OptionGroup(
parser, "Windows registry access", "These "
"options can be used to access the back-end "
"database management system Windows "
"registry")
windows.add_option("--reg-read",
dest="regRead",
action="store_true",
help="Read a Windows registry key value")
windows.add_option("--reg-add",
dest="regAdd",
action="store_true",
help="Write a Windows registry key value data")
windows.add_option("--reg-del",
dest="regDel",
action="store_true",
help="Delete a Windows registry key value")
windows.add_option("--reg-key",
dest="regKey",
help="Windows registry key")
windows.add_option("--reg-value",
dest="regVal",
help="Windows registry key value")
windows.add_option("--reg-data",
dest="regData",
help="Windows registry key value data")
windows.add_option("--reg-type",
dest="regType",
help="Windows registry key value type")
# General options
general = OptionGroup(
parser, "General", "These options can be used "
"to set some general working parameters")
general.add_option("-s",
dest="sessionFile",
help="Load session from a stored (.sqlite) file")
general.add_option("-t",
dest="trafficFile",
help="Log all HTTP traffic into a "
"textual file")
general.add_option(
"--batch",
dest="batch",
action="store_true",
help="Never ask for user input, use the default behaviour")
general.add_option(
"--binary-fields",
dest="binaryFields",
help="Result fields having binary values (e.g. \"digest\")")
general.add_option(
"--charset",
dest="charset",
help="Force character encoding used for data retrieval")
general.add_option(
"--check-internet",
dest="checkInternet",
action="store_true",
help="Check Internet connection before assessing the target")
general.add_option(
"--crawl",
dest="crawlDepth",
type="int",
help="Crawl the website starting from the target URL")
general.add_option(
"--crawl-exclude",
dest="crawlExclude",
help="Regexp to exclude pages from crawling (e.g. \"logout\")")
general.add_option("--csv-del",
dest="csvDel",
help="Delimiting character used in CSV output "
"(default \"%s\")" % defaults.csvDel)
general.add_option(
"--dump-format",
dest="dumpFormat",
help="Format of dumped data (CSV (default), HTML or SQLITE)")
general.add_option(
"--eta",
dest="eta",
action="store_true",
help="Display for each output the estimated time of arrival")
general.add_option("--flush-session",
dest="flushSession",
action="store_true",
help="Flush session files for current target")
general.add_option("--forms",
dest="forms",
action="store_true",
help="Parse and test forms on target URL")
general.add_option("--fresh-queries",
dest="freshQueries",
action="store_true",
help="Ignore query results stored in session file")
general.add_option("--har",
dest="harFile",
help="Log all HTTP traffic into a HAR file")
general.add_option("--hex",
dest="hexConvert",
action="store_true",
help="Use DBMS hex function(s) for data retrieval")
general.add_option("--output-dir",
dest="outputDir",
action="store",
help="Custom output directory path")
general.add_option(
"--parse-errors",
dest="parseErrors",
action="store_true",
help="Parse and display DBMS error messages from responses")
general.add_option("--save",
dest="saveConfig",
help="Save options to a configuration INI file")
general.add_option(
"--scope",
dest="scope",
help="Regexp to filter targets from provided proxy log")
general.add_option(
"--test-filter",
dest="testFilter",
help="Select tests by payloads and/or titles (e.g. ROW)")
general.add_option(
"--test-skip",
dest="testSkip",
help="Skip tests by payloads and/or titles (e.g. BENCHMARK)")
general.add_option("--update",
dest="updateAll",
action="store_true",
help="Update sqlmap")
# Miscellaneous options
miscellaneous = OptionGroup(parser, "Miscellaneous")
miscellaneous.add_option(
"-z",
dest="mnemonics",
help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
miscellaneous.add_option(
"--alert",
dest="alert",
help="Run host OS command(s) when SQL injection is found")
miscellaneous.add_option(
"--answers",
dest="answers",
help="Set question answers (e.g. \"quit=N,follow=N\")")
miscellaneous.add_option(
"--beep",
dest="beep",
action="store_true",
help="Beep on question and/or when SQL injection is found")
miscellaneous.add_option("--cleanup",
dest="cleanup",
action="store_true",
help="Clean up the DBMS from sqlmap specific "
"UDF and tables")
miscellaneous.add_option(
"--dependencies",
dest="dependencies",
action="store_true",
help="Check for missing (non-core) sqlmap dependencies")
miscellaneous.add_option("--disable-coloring",
dest="disableColoring",
action="store_true",
help="Disable console output coloring")
miscellaneous.add_option(
"--gpage",
dest="googlePage",
type="int",
help="Use Google dork results from specified page number")
miscellaneous.add_option(
"--identify-waf",
dest="identifyWaf",
action="store_true",
help="Make a thorough testing for a WAF/IPS/IDS protection")
miscellaneous.add_option(
"--mobile",
dest="mobile",
action="store_true",
help="Imitate smartphone through HTTP User-Agent header")
miscellaneous.add_option(
"--offline",
dest="offline",
action="store_true",
help="Work in offline mode (only use session data)")
miscellaneous.add_option(
"--purge-output",
dest="purgeOutput",
action="store_true",
help="Safely remove all content from output directory")
miscellaneous.add_option(
"--skip-waf",
dest="skipWaf",
action="store_true",
help="Skip heuristic detection of WAF/IPS/IDS protection")
miscellaneous.add_option(
"--smart",
dest="smart",
action="store_true",
help="Conduct thorough tests only if positive heuristic(s)")
miscellaneous.add_option("--sqlmap-shell",
dest="sqlmapShell",
action="store_true",
help="Prompt for an interactive sqlmap shell")
miscellaneous.add_option(
"--tmp-dir",
dest="tmpDir",
help="Local directory for storing temporary files")
miscellaneous.add_option(
"--web-root",
dest="webRoot",
help="Web server document root directory (e.g. \"/var/www\")")
miscellaneous.add_option(
"--wizard",
dest="wizard",
action="store_true",
help="Simple wizard interface for beginner users")
# Hidden and/or experimental options
parser.add_option("--dummy",
dest="dummy",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--murphy-rate",
dest="murphyRate",
type="int",
help=SUPPRESS_HELP)
parser.add_option("--disable-precon",
dest="disablePrecon",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--disable-stats",
dest="disableStats",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--profile",
dest="profile",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--force-dns",
dest="forceDns",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--force-threads",
dest="forceThreads",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--smoke-test",
dest="smokeTest",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--live-test",
dest="liveTest",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--stop-fail",
dest="stopFail",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--run-case", dest="runCase", help=SUPPRESS_HELP)
# API options
parser.add_option("--api",
dest="api",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--taskid", dest="taskid", help=SUPPRESS_HELP)
parser.add_option("--database", dest="database", help=SUPPRESS_HELP)
parser.add_option_group(target)
parser.add_option_group(request)
parser.add_option_group(optimization)
parser.add_option_group(injection)
parser.add_option_group(detection)
parser.add_option_group(techniques)
parser.add_option_group(fingerprint)
parser.add_option_group(enumeration)
parser.add_option_group(brute)
parser.add_option_group(udf)
parser.add_option_group(filesystem)
parser.add_option_group(takeover)
parser.add_option_group(windows)
parser.add_option_group(general)
parser.add_option_group(miscellaneous)
# Dirty hack to display longer options without breaking into two lines
def _(self, *args):
retVal = parser.formatter._format_option_strings(*args)
if len(retVal) > MAX_HELP_OPTION_LENGTH:
retVal = ("%%.%ds.." %
(MAX_HELP_OPTION_LENGTH -
parser.formatter.indent_increment)) % retVal
return retVal
parser.formatter._format_option_strings = parser.formatter.format_option_strings
parser.formatter.format_option_strings = type(
parser.formatter.format_option_strings)(_, parser, type(parser))
# Dirty hack for making a short option '-hh'
option = parser.get_option("--hh")
option._short_opts = ["-hh"]
option._long_opts = []
# Dirty hack for inherent help message of switch '-h'
option = parser.get_option("-h")
option.help = option.help.capitalize().replace("this help",
"basic help")
_ = []
prompt = False
advancedHelp = True
extraHeaders = []
# Reference: https://stackoverflow.com/a/4012683 (Note: previously used "...sys.getfilesystemencoding() or UNICODE_ENCODING")
for arg in argv:
_.append(getUnicode(arg, encoding=sys.stdin.encoding))
argv = _
checkDeprecatedOptions(argv)
prompt = "--sqlmap-shell" in argv
if prompt:
parser.usage = ""
cmdLineOptions.sqlmapShell = True
_ = ["x", "q", "exit", "quit", "clear"]
for option in parser.option_list:
_.extend(option._long_opts)
_.extend(option._short_opts)
for group in parser.option_groups:
for option in group.option_list:
_.extend(option._long_opts)
_.extend(option._short_opts)
autoCompletion(AUTOCOMPLETE_TYPE.SQLMAP, commands=_)
while True:
command = None
try:
command = raw_input("sqlmap-shell> ").strip()
command = getUnicode(command, encoding=sys.stdin.encoding)
except (KeyboardInterrupt, EOFError):
print
raise SqlmapShellQuitException
if not command:
continue
elif command.lower() == "clear":
clearHistory()
dataToStdout("[i] history cleared\n")
saveHistory(AUTOCOMPLETE_TYPE.SQLMAP)
elif command.lower() in ("x", "q", "exit", "quit"):
raise SqlmapShellQuitException
elif command[0] != '-':
dataToStdout("[!] invalid option(s) provided\n")
dataToStdout(
"[i] proper example: '-u http://www.site.com/vuln.php?id=1 --banner'\n"
)
else:
saveHistory(AUTOCOMPLETE_TYPE.SQLMAP)
loadHistory(AUTOCOMPLETE_TYPE.SQLMAP)
break
try:
for arg in shlex.split(command):
argv.append(getUnicode(arg, encoding=sys.stdin.encoding))
except ValueError, ex:
raise SqlmapSyntaxException, "something went wrong during command line parsing ('%s')" % ex.message
for i in xrange(len(argv)):
if argv[i] == "-hh":
argv[i] = "-h"
elif len(argv[i]) > 1 and all(
ord(_) in xrange(0x2018, 0x2020)
for _ in ((argv[i].split('=', 1)[-1].strip() or ' ')[0],
argv[i][-1])):
dataToStdout(
"[!] copy-pasting illegal (non-console) quote characters from Internet is, well, illegal (%s)\n"
% argv[i])
raise SystemExit
elif len(argv[i]) > 1 and u"\uff0c" in argv[i].split('=', 1)[-1]:
dataToStdout(
"[!] copy-pasting illegal (non-console) comma characters from Internet is, well, illegal (%s)\n"
% argv[i])
raise SystemExit
elif re.search(r"\A-\w=.+", argv[i]):
dataToStdout(
"[!] potentially miswritten (illegal '=') short option detected ('%s')\n"
% argv[i])
raise SystemExit
elif argv[i] == "-H":
if i + 1 < len(argv):
extraHeaders.append(argv[i + 1])
elif re.match(r"\A\d+!\Z", argv[i]) and argv[max(
0, i - 1)] == "--threads" or re.match(
r"\A--threads.+\d+!\Z", argv[i]):
argv[i] = argv[i][:-1]
conf.skipThreadCheck = True
elif argv[i] == "--version":
print VERSION_STRING.split('/')[-1]
raise SystemExit
elif argv[i] in ("-h", "--help"):
advancedHelp = False
for group in parser.option_groups[:]:
found = False
for option in group.option_list:
if option.dest not in BASIC_HELP_ITEMS:
option.help = SUPPRESS_HELP
else:
found = True
if not found:
parser.option_groups.remove(group)
for verbosity in (_ for _ in argv if re.search(r"\A\-v+\Z", _)):
try:
if argv.index(verbosity) == len(argv) - 1 or not argv[
argv.index(verbosity) + 1].isdigit():
conf.verbose = verbosity.count('v') + 1
del argv[argv.index(verbosity)]
except (IndexError, ValueError):
pass
try:
(args, _) = parser.parse_args(argv)
except UnicodeEncodeError, ex:
dataToStdout("\n[!] %s\n" % ex.object.encode("unicode-escape"))
raise SystemExit
def main():
# TODO : query the App Engine metadata to determine all kinds
kinds = ('Comment', 'DailyActivityLog', 'DiscussAnswer', 'DiscussQuestion',
'ExerciseVideo', 'Exercise', 'FeedbackNotification', 'Feedback',
'HourlyActivityLog', 'ProblemLog', 'Question', 'Setting',
'StemmedIndex', 'UserBadge', 'UserData', 'UserExercise',
'UserVideo', 'VideoLog', 'Video', 'YouTubeSyncStepLog')
parser = OptionParser(usage="%prog [options]",
description="Downloads a data and progress file for "
"each entity kind in SQLite3 format.")
parser.add_option("-U",
"--url",
default="http://*****:*****@example.com",
help="The username to use.")
parser.add_option("-k",
"--kinds",
default=','.join(kinds),
help="The comma separated list of kinds.")
parser.add_option(
"-p",
"--python",
default=(sys.executable if platform.system() == "Windows" else None),
help="Path of python executable.")
parser.add_option("-a",
"--appcfg",
default='appcfg.py',
help="Path of appcfg.py (Google App Engine).")
parser.add_option("-A",
"--application",
default='khanexercises',
help="GAE application name")
# Performance options
parser.add_option("-n",
"--num_threads",
default='55',
help="num_threads, passed to bulkloader")
parser.add_option("-s",
"--batch_size",
default='500',
help="batch_size, passed to bulkloader")
parser.add_option("-b",
"--bandwidth_limit",
default='10000000',
help="bandwidth_limit, passed to bulkloader")
parser.add_option("-r",
"--rps_limit",
default='15000',
help="rps_limit of threads, passed to bulkloader")
parser.add_option("-c",
"--http_limit",
default='40',
help="http_limit, passed to bulkloader")
parser.add_option("-x",
"--passin",
default='',
help="Path to file containing Google App Engine "
"password")
(options, args) = parser.parse_args()
files = []
for kind in options.kinds.split(','):
filename = 'bulkloader-results.%s.data' % kind
db_filename = 'bulkloader-progress.%s.data' % kind
files.append(filename)
files.append(db_filename)
if os.path.exists(filename):
os.remove(filename)
if os.path.exists(db_filename):
os.remove(db_filename)
call_args = [
options.appcfg,
'--url=%s' % options.url,
'--email=%s' % options.email,
'--kind=%s' % kind,
'--db_filename=%s' % db_filename,
'--filename=%s' % filename,
'--num_threads=%s' % options.num_threads,
'--batch_size=%s' % options.batch_size,
'--bandwidth_limit=%s' % options.bandwidth_limit,
'--rps_limit=%s' % options.rps_limit,
'--http_limit=%s' % options.http_limit
]
if options.application is not None:
call_args.append('--application=%s' % options.application)
if (options.email == parser.get_option('--email').default
or options.passin):
call_args.append('--passin')
if options.python is not None:
call_args.insert(0, options.python)
call_args.append('download_data')
print ' '.join(call_args)
if options.email == parser.get_option('--email').default:
process = subprocess.Popen(call_args, stdin=subprocess.PIPE)
# Send a newline for the password prompt
process.communicate("\n")
elif options.passin:
f = open(options.passin, "r")
password = f.read()
f.close()
# Send contents of password file for the password prompt
process = subprocess.Popen(call_args, stdin=subprocess.PIPE)
process.communicate(password)
else:
process = subprocess.Popen(call_args)
process.wait()
# zip up all created progress and results files
os.system("zip bulkloader.zip bulkloader-*.data")
# warning: this will blow away the logs and extra results db's, too!
os.system("rm bulkloader-*")
def get_option_parser(defaults):
"""Creates OptionParser and adds shell options (flags)
Default values are loaded in initially
"""
parser = OptionParser()
parser.set_defaults(**defaults)
parser.add_option("-i",
"--impalad",
dest="impalad",
help="<host:port> of impalad to connect to \t\t")
parser.add_option("-q",
"--query",
dest="query",
help="Execute a query without the shell")
parser.add_option(
"-f",
"--query_file",
dest="query_file",
help="Execute the queries in the query file, delimited by ;")
parser.add_option("-k",
"--kerberos",
dest="use_kerberos",
action="store_true",
help="Connect to a kerberized impalad")
parser.add_option(
"-o",
"--output_file",
dest="output_file",
help=("If set, query results are written to the "
"given file. Results from multiple semicolon-terminated "
"queries will be appended to the same file"))
parser.add_option("-B",
"--delimited",
dest="write_delimited",
action="store_true",
help="Output rows in delimited mode")
parser.add_option("--print_header",
dest="print_header",
action="store_true",
help="Print column names in delimited mode"
" when pretty-printed.")
parser.add_option(
"--output_delimiter",
dest="output_delimiter",
help="Field delimiter to use for output in delimited mode")
parser.add_option("-s",
"--kerberos_service_name",
dest="kerberos_service_name",
help="Service name of a kerberized impalad")
parser.add_option("-V",
"--verbose",
dest="verbose",
action="store_true",
help="Verbose output")
parser.add_option("-p",
"--show_profiles",
dest="show_profiles",
action="store_true",
help="Always display query profiles after execution")
parser.add_option("--quiet",
dest="verbose",
action="store_false",
help="Disable verbose output")
parser.add_option("-v",
"--version",
dest="version",
action="store_true",
help="Print version information")
parser.add_option("-c",
"--ignore_query_failure",
dest="ignore_query_failure",
action="store_true",
help="Continue on query failure")
parser.add_option("-r",
"--refresh_after_connect",
dest="refresh_after_connect",
action="store_true",
help="Refresh Impala catalog after connecting \t")
parser.add_option("-d",
"--database",
dest="default_db",
help="Issues a use database command on startup \t")
parser.add_option(
"-l",
"--ldap",
dest="use_ldap",
action="store_true",
help="Use LDAP to authenticate with Impala. Impala must be configured"
" to allow LDAP authentication. \t\t")
parser.add_option("-u",
"--user",
dest="user",
help="User to authenticate with.")
parser.add_option("--ssl",
dest="ssl",
action="store_true",
help="Connect to Impala via SSL-secured connection \t")
parser.add_option(
"--ca_cert",
dest="ca_cert",
help=
("Full path to "
"certificate file used to authenticate Impala's SSL certificate."
" May either be a copy of Impala's certificate (for self-signed "
"certs) or the certificate of a trusted third-party CA. If not set, "
"but SSL is enabled, the shell will NOT verify Impala's server "
"certificate"))
parser.add_option(
"--config_file",
dest="config_file",
help=("Specify the configuration file to load options. "
"File must have case-sensitive '[impala]' header. "
"Specifying this option within a config file will have "
"no effect. Only specify this as a option in the commandline."))
parser.add_option(
"--live_summary",
dest="print_summary",
action="store_true",
help="Print a query summary every 1s while the query is running.")
parser.add_option(
"--live_progress",
dest="print_progress",
action="store_true",
help="Print a query progress every 1s while the query is running.")
parser.add_option(
"--auth_creds_ok_in_clear",
dest="creds_ok_in_clear",
action="store_true",
help="If set, LDAP authentication " +
"may be used with an insecure connection to Impala. " +
"WARNING: Authentication credentials will therefore be sent " +
"unencrypted, and may be vulnerable to attack.")
parser.add_option(
"--ldap_password_cmd",
help="Shell command to run to retrieve the LDAP password")
parser.add_option(
"--var",
dest="keyval",
action="append",
help="Define variable(s) to be used within the Impala session.")
# add default values to the help text
for option in parser.option_list:
# since the quiet flag is the same as the verbose flag
# we need to make sure to print the opposite value for it
# (print quiet is false since verbose is true)
if option == parser.get_option('--quiet'):
option.help += " [default: %s]" % (not defaults['verbose'])
elif option != parser.get_option('--help'):
# don't want to print default value for help
option.help += " [default: %default]"
return parser
def cmdLineParser(argv=None):
"""
此函数解析命令行参数和参数
"""
if not argv:
argv = sys.argv
checkSystemEncoding()
# Reference: https://stackoverflow.com/a/4012683 (Note: previously used "...sys.getfilesystemencoding() or UNICODE_ENCODING")
_ = getUnicode(os.path.basename(argv[0]), encoding=sys.stdin.encoding)
usage = "%s%s [options]" % ("python " if not IS_WIN else "", \
"\"%s\"" % _ if " " in _ else _)
parser = OptionParser(usage=usage)
try:
parser.add_option("--hh",
dest=u"advancedHelp",
action="store_true",
help=u"显示高级帮助消息并退出")
# dest ： 用于保存临时变量，其值可以作为options的属性进行访问。存储的内容就是如-f,-n 等紧挨着的那个参数内容。
parser.add_option("--version",
dest=u"showVersion",
action="store_true",
help=u"显示程序版本号并退出")
# type 参数类型
parser.add_option("-v",
dest="verbose",
type="int",
help=u"详细程度: 0-6 (默认 %d)" % defaults.verbose)
# 目标选项
target = OptionGroup(parser, u"目标", u"至少提供下列一个选项作为注入目标")
#例如：mysql://数据库用户名:数据库密码@数据库ip地址:数据库端口号/数据库名)
target.add_option(
"-d",
dest="direct",
help=
u"直接连接数据库\r\n(例如：mysql://USER:PASSWORD@DBMS_IP:DBMS_PORT/DATABASE_NAME)"
)
target.add_option("-u",
"--url",
dest="url",
help=u"目标网址(例如:http://www.site.com/vuln.php?id=1)")
target.add_option("-l",
dest="logFile",
help=u"从Burp或WebScarab代理日志文件解析目标")
target.add_option("-x", dest="sitemapUrl", help=u"从远程站点地图（.xml）文件解析目标")
target.add_option("-m", dest="bulkFile", help=u"扫描文本文件中给出的多个目标")
target.add_option("-r", dest="requestFile", help=u"从文件加载HTTP请求")
target.add_option(
"-g",
dest="googleDork",
help=u"将Google dork搜索结果作为目标，这个选项使得sqlmap可以通过和搜索引擎通信,"
u"通过google dork搜索可能存在sql注入的网站 ,然后sqlmap会提取前100个结果 ,"
u"并询问用户是否针对这些目标进行检测 "
u"\r\n(例如：python sqlmap.py -g \"inurl:\".php?id=1\"\")")
target.add_option("-c",
dest="configFile",
help=u"加载sqlmap.conf文件里面的相关配置")
# 请求选项
request = OptionGroup(parser, u"请求", u"该选项指定以何种方式连接到目标URL")
request.add_option("--method",
dest="method",
help=u"强制使用指定的HTTP请求方法(例如：GET PUT)")
request.add_option("--data", dest="data", help=u"以POST方式提交数据")
request.add_option("--param-del", dest="paramDel", help=u"用于分割参数值的字符")
request.add_option("--cookie",
dest="cookie",
help=u"HTTP Cookie header value")
request.add_option("--cookie-del",
dest="cookieDel",
help=u"用于分割cookie值的字符")
request.add_option("--load-cookies",
dest="loadCookies",
help=u"包含Netscape/wget格式的Cookie的文件")
request.add_option("--drop-set-cookie",
dest="dropSetCookie",
action="store_true",
help=u"从响应response中忽略Set-Cookie header")
request.add_option(
"--user-agent",
dest="agent",
help=u"自定义修改HTTP请求头中User-Agent值,只有--level等级为3以上设置才会生效")
request.add_option("--random-agent",
dest="randomAgent",
action="store_true",
help=u"使用随机选择的HTTP User-Agent header值")
request.add_option("--host",
dest="host",
help=u"自定义修改HTTP请求头中的Host值,只有在--level值为5的时候设置才会生效")
request.add_option("--referer",
dest="referer",
help=u"sqlmap可以在请求中伪造HTTP中的referer,"
u"当--level参数设定为3或者3以上的时候会尝试对referer注入。")
request.add_option(
"-H",
"--header",
dest="header",
help=u"Extra header (例如: \"X-Forwarded-For: 127.0.0.1\")")
request.add_option(
"--headers",
dest="headers",
help=u"Extra headers (例如: \"Accept-Language: fr\\nETag: 123\")")
request.add_option("--auth-type",
dest="authType",
help=u"HTTP认证类型(Basic, Digest, NTLM or PKI)")
request.add_option("--auth-cred",
dest="authCred",
help=u"HTTP身份验证凭证(用户名:密码)")
request.add_option("--auth-file",
dest="authFile",
help=u"HTTP认证PEM认证/私钥文件")
request.add_option("--ignore-401",
dest="ignore401",
action="store_true",
help=u"忽略HTTP错误401（未经授权）")
request.add_option("--ignore-proxy",
dest="ignoreProxy",
action="store_true",
help=u"忽略系统默认代理设置")
request.add_option("--ignore-redirects",
dest="ignoreRedirects",
action="store_true",
help=u"忽略重定向尝试")
request.add_option("--ignore-timeouts",
dest="ignoreTimeouts",
action="store_true",
help=u"忽略连接超时")
request.add_option("--proxy", dest="proxy", help=u"使用代理连接到目标URL")
request.add_option("--proxy-cred",
dest="proxyCred",
help=u"代理认证凭证(用户名:密码)")
request.add_option("--proxy-file", dest="proxyFile", help=u"从文件加载代理列表")
request.add_option("--tor",
dest="tor",
action="store_true",
help=u"使用Tor匿名网络")
request.add_option("--tor-port",
dest="torPort",
help=u"设置tor的端口，如果不是默认端口的话")
request.add_option("--tor-type",
dest="torType",
help=u"设置Tor代理类型(HTTP, SOCKS4 or SOCKS5 (default))")
request.add_option("--check-tor",
dest="checkTor",
action="store_true",
help=u"检查Tor是否可用")
request.add_option("--delay",
dest="delay",
type="float",
help=u"每个HTTP请求之间的延迟秒数")
request.add_option("--timeout",
dest="timeout",
type="float",
help=u"设置超时时间，默认%d秒" % defaults.timeout)
request.add_option("--retries",
dest="retries",
type="int",
help=u"设置连接超时时重试次数，默认%d次" % defaults.retries)
request.add_option("--randomize", dest="rParam", help=u"随机更改给定参数的值")
request.add_option("--safe-url",
dest="safeUrl",
help=u"有的web程序会在多次错误访问后屏蔽所有请求，这样就导致之后所有的测试无法进行，"
u"绕过这个策略可以使用--safe-url，每隔一段时间去访问一个正常的页面。")
request.add_option("--safe-post",
dest="safePost",
help=u"发送POST数据到一个安全的URL")
request.add_option("--safe-req",
dest="safeReqFile",
help=u"从文件加载安全的HTTP请求")
request.add_option("--safe-freq",
dest="safeFreq",
type="int",
help=u"提供一个安全无错误的连接，在测试URL和安全链接之间交叉访问")
request.add_option(
"--skip-urlencode",
dest="skipUrlEncode",
action="store_true",
help=u"根据参数位置，他的值默认将会被URL编码，但是有些时候后端的web服务器不遵守RFC标准，"
u"只接受不经过URL编码的值，这时候就需要用--skip-urlencode参数,跳过Payload数据的URL编码")
request.add_option("--csrf-token",
dest="csrfToken",
help=u"设置CSRF的token")
request.add_option("--csrf-url",
dest="csrfUrl",
help=u"访问URL地址提取anti-CSRF token")
request.add_option("--force-ssl",
dest="forceSSL",
action="store_true",
help=u"强制使用SSL/HTTPS")
request.add_option(
"--hpp",
dest="hpp",
action="store_true",
help=
u"使用HTTP参数污染方法绕过WAF的检测机制，HTTP参数污染是一种可以绕过WAF/IPS/IDS的方法，这在面对ASP/IIS 或者是ASP.NET/IIS 组合的时候非常有用，如果你怀疑目标使用了某种保护(WAF/IDS/IPS) 那么你可以试试这个选项"
)
request.add_option(
"--eval",
dest="evalCode",
help=u"发送请求之前，先运行这段python代码，比如下面的hash参数就是id的md5值 "
u"(python sqlmap.py -u \"http://www.target.com/vuln.php?id=1&hash=c4ca4238a0b923820dcc509a6f75849b\" "
u"--eval = \"import hashlib;hash=hashlib.md5(id).hexdigest()\")")
# 优化选项
optimization = OptionGroup(parser, u"优化", u"这些选项可用于优化sqlmap的性能")
optimization.add_option("-o",
dest="optimize",
action="store_true",
help=u"开启所有优化选项")
optimization.add_option("--predict-output",
dest="predictOutput",
action="store_true",
help=u"预测常见查询输出")
optimization.add_option("--keep-alive",
dest="keepAlive",
action="store_true",
help=u"使用持久的HTTP(s)连接")
optimization.add_option("--null-connection",
dest="nullConnection",
action="store_true",
help=u"检索页面长度,排除实际的 HTTP 响应内容")
optimization.add_option("--threads",
dest="threads",
type="int",
help=u"最大并发HTTP(s)请求数（默认为%d）" %
defaults.threads)
# 注入选项
injection = OptionGroup(parser, u"注入",
u"这些选项可用于指定要测试的参数,提供自定义注入Payload和篡改脚本")
injection.add_option(
"-p",
dest="testParameter",
help=u"手动指定要测试的参数，默认情况下sqlmap会测试所有的GET和POST参数(例如: -p \"id\")")
injection.add_option("--skip", dest="skip", help=u"跳过你不想进行注入测试的参数")
injection.add_option("--skip-static",
dest="skipStatic",
action="store_true",
help=u"跳过那些不是动态的测试参数，对静态参数进行注入测试是徒劳的")
injection.add_option("--param-exclude",
dest="paramExclude",
help=u"使用正则表达式来排除不需要测试的参数 (例如:\"ses\")")
injection.add_option("--dbms",
dest="dbms",
help=u"指定后端的数据库类型（如mysql，oracle等）")
injection.add_option("--dbms-cred",
dest="dbmsCred",
help=u"数据库认证凭证(user:password)")
injection.add_option("--os",
dest="os",
help=u"手动指定后端DBMS操作系统(Windows，linux)")
injection.add_option(
"--invalid-bignum",
dest="invalidBignum",
action="store_true",
help=u"指定无效的大数字id=13,sqlmap会变成id=-13来报错,你也可以指定比如id=9999999来报错")
injection.add_option(
"--invalid-logical",
dest="invalidLogical",
action="store_true",
help=u"指定无效的逻辑,可以指定id=13把原来的id=-13的报错改成id=13 AND 18=19")
injection.add_option("--invalid-string",
dest="invalidString",
action="store_true",
help=u"使用随机字符串无效值来给参数赋值")
injection.add_option("--no-cast",
dest="noCast",
action="store_true",
help=u"关闭payload构造机制")
injection.add_option("--no-escape",
dest="noEscape",
action="store_true",
help=u"关闭字符串转义机制")
injection.add_option("--prefix",
dest="prefix",
help=u"设置注入的前缀，比如单引号注入点，就设置前缀为单引号")
"""
注入payload
参数：--prefix, --suffix
在有些环境中，需要在注入的payload的前面或者后面加一些字符，来保证payload的正常执行。
例如，代码中是这样调用数据库的：
$query = "SELECT * FROM users WHERE id=(’". $_GET[’id’]."’) LIMIT 0, 1";
这时你就需要--prefix和--suffix参数了：
python sqlmap.py -u "http://xxx.com/sqlmap/mysql/get_str_brackets.php?id=1" -p id --prefix "’)" --suffix "AND (’abc’=’abc"
这样执行的SQL语句变成：
$query = "SELECT * FROM users WHERE id=('1') <PAYLOAD> AND ('abc'='abc') LIMIT 0, 1";
"""
injection.add_option("--suffix", dest="suffix", help=u"设置注入payload的后缀")
injection.add_option("--tamper",
dest="tamper",
help=u"使用tamper脚本修改请求从而逃避WAF的规则检测")
# 探测选项
detection = OptionGroup(parser, u"探测", u"这些选项可用于指定检测目标的等级")
detection.add_option("--level",
dest="level",
type="int",
help=u"探测等级(1-5,默认%d级)" % defaults.level)
detection.add_option("--risk",
dest="risk",
type="int",
help=u"风险等级(1-3, 默认%d级)" % defaults.risk)
detection.add_option(
"--string",
dest="string",
help=
u"设置原始页面与条件为真情况下页面中都存在的字符串，而错误页面中不存在，如果页面返回这个字符串，说明我们的注入判断语句是正确的")
detection.add_option("--not-string",
dest="notString",
help=u"设置一段在原始页面与真条件页面中都不存在的字符串，而错误页面中存在的字符串")
detection.add_option("--regexp",
dest="regexp",
help=u"利用正则匹配页面返回内容，如果存在匹配字符串，则可能存在注入点")
detection.add_option("--code",
dest="code",
type="int",
help=u"用HTTP响应码来判断注入语句是否正确，"
u"例如，响应200的时候为真，响应401的时候为假，可以添加参数--code=200")
detection.add_option(
"--text-only",
dest="textOnly",
action="store_true",
help=
u"基于文本内容比较页面,如果在HTTP响应中存在大量脚本、或者是各种内嵌的东西 ,可以通过使用--text-only来进行筛选只显示文本内容"
)
detection.add_option(
"--titles",
dest="titles",
action="store_true",
help=
u"基于标题比较页面,如果使用者知道正常、错误响应之间的HTML标题的区别 (例如Welcom为正常，Forbidden为错误)那么他可以通过使用--titles来比较HTML标题的不同来提示sqlmap是否能够注入"
)
# 注入技术
techniques = OptionGroup(parser, u"注入技术", u"这些选项可用于指定具体的SQL注入技术的测试")
techniques.add_option("--technique",
dest="tech",
help=u"要使用的SQL注入技术(默认 \"%s\")" % defaults.tech)
techniques.add_option("--time-sec",
dest="timeSec",
type="int",
help=u"设定延迟注入的时间，"
u"当使用基于时间的盲注时，时刻使用--time-sec参数设定延时时间，默认是%d秒" %
defaults.timeSec)
techniques.add_option(
"--union-cols",
dest="uCols",
help=u"设定SQL注入时UNION查询字段数范围，如：12-16，是测试12-16个字段数")
techniques.add_option(
"--union-char",
dest="uChar",
help=
u"设定UNION查询使用的字符，用于爆破字段数目的字符,默认使用NULL字符，但是有些情况下会造成页面返回失败，而一个随机整数是成功的，这时你可以用--union-char指定UNION查询的字符"
)
techniques.add_option(
"--union-from",
dest="uFrom",
help=
u"在某些UNION查询SQL注入情况下，需要在FROM子句中强制使用有效且可访问的表名。 例如，Microsoft Access需要使用这样的表。 如果不提供一个UNION查询，SQL注入将无法正确执行(例如 --union-from = users)"
)
techniques.add_option("--dns-domain",
dest="dnsDomain",
help=u"用于DNS渗透攻击的域名")
techniques.add_option(
"--second-order",
dest="secondOrder",
help=
u"二阶注入是攻击者首先提交恶意的请求，在数据库保存成功后 再提交另外一个用于检索之前的恶意请求的请求，如果攻击成功，那么响应会在第二次响应中返回结果，使用这个选项的时候后面跟着的是显示结果页面的URL。有些时候注入点输入的数据返回结果的时候并不是当前的页面，而是另外的一个页面，这时候就需要你指定到哪个页面来获取响应结果判断真假。--second-order后面跟一个判断页面的URL地址。"
)
# 指纹识别选项
fingerprint = OptionGroup(parser, u"指纹识别")
fingerprint.add_option("-f",
"--fingerprint",
dest="extensiveFp",
action="store_true",
help=u"利用数据库特有的指纹信息识别其数据库类型和版本号")
# 枚举选项
enumeration = OptionGroup(
parser, u"检索数据", u"这些选项可用于枚举表中包含的DBMS信息结构和数据。此外，您可以运行自己的SQL语句")
enumeration.add_option("-a",
"--all",
dest="getAll",
action="store_true",
help=u"检索所有内容")
enumeration.add_option("-b",
"--banner",
dest="getBanner",
action="store_true",
help=u"检索数据库管理系统的标识（如mysql，oracle）")
enumeration.add_option("--current-user",
dest="getCurrentUser",
action="store_true",
help=u"检索当前连接数据库的用户CURRENT_USER()")
enumeration.add_option("--current-db",
dest="getCurrentDb",
action="store_true",
help=u"检索当前连接的数据库DATABASE()")
enumeration.add_option("--hostname",
dest="getHostname",
action="store_true",
help=u"检索服务器的主机名@@HOSTNAME")
enumeration.add_option("--is-dba",
dest="isDba",
action="store_true",
help=u"判断当前用户是否为管理，是的话会返回True")
enumeration.add_option("--users",
dest="getUsers",
action="store_true",
help=u"枚举数据库用户，"
u"当前用户有权限读取包含所有用户的表的权限时，就可以列出所有管理用户")
enumeration.add_option("--passwords",
dest="getPasswordHashes",
action="store_true",
help=u"枚举数据库用户密码的哈希值并尝试破解")
enumeration.add_option("--privileges",
dest="getPrivileges",
action="store_true",
help=u"枚举数据库用户的权限")
enumeration.add_option("--roles",
dest="getRoles",
action="store_true",
help=u"枚举数据库用户的角色")
enumeration.add_option("--dbs",
dest="getDbs",
action="store_true",
help=u"列出所有的数据库")
enumeration.add_option("--tables",
dest="getTables",
action="store_true",
help=u"列举数据库中的所有表")
enumeration.add_option("--columns",
dest="getColumns",
action="store_true",
help=u"列举数据库表中的字段，同时也会列出字段的数据类型")
enumeration.add_option("--schema",
dest="getSchema",
action="store_true",
help=u"列举数据库系统的架构，包含所有的数据库，表和字段，以及各自的类型")
enumeration.add_option("--count",
dest="getCount",
action="store_true",
help=u"检索表的条目数")
enumeration.add_option("--dump",
dest="dumpTable",
action="store_true",
help=u"获取整个表的数据")
enumeration.add_option("--dump-all",
dest="dumpAll",
action="store_true",
help=u"获取所有数据库表中的内容")
enumeration.add_option("--search",
dest="search",
action="store_true",
help=u"搜索字段，表，数据库，配合下面的-D,-C,-T")
enumeration.add_option("--comments",
dest="getComments",
action="store_true",
help=u"枚举数据库的注释")
enumeration.add_option("-D", dest="db", help=u"要进行枚举的数据库名")
enumeration.add_option("-T", dest="tbl", help=u"要进行枚举的数据库表")
enumeration.add_option("-C", dest="col", help=u"要进行枚举的数据库字段")
enumeration.add_option("-X", dest="excludeCol", help=u"指定不枚举那个字段")
enumeration.add_option("-U", dest="user", help=u"枚举数据库用户")
enumeration.add_option("--exclude-sysdbs",
dest="excludeSysDbs",
action="store_true",
help=u"枚举表时排除系统数据库")
enumeration.add_option("--pivot-column",
dest="pivotColumn",
help=u"行转列名称")
enumeration.add_option("--where",
dest="dumpWhere",
help=u"使用WHERE条件查询/获取指定表中的内容")
enumeration.add_option("--start",
dest="limitStart",
type="int",
help=u"指定开始从第几行开始输出，如--start=3，前两行就不输出了")
enumeration.add_option("--stop",
dest="limitStop",
type="int",
help=u"指定从第几行开始停止输出")
enumeration.add_option("--first",
dest="firstChar",
type="int",
help=u"指定从第几个字符之后开始输出")
enumeration.add_option("--last",
dest="lastChar",
type="int",
help=u"指定输出到第几个字符后停止输出，盲注才有效，亲测，跟上面的配合指定范围，"
u"如 ：--first 3 --last 5 只输出3到5位置的字符")
enumeration.add_option("--sql-query",
dest="query",
help=u"指定执行我们的sql语句")
enumeration.add_option("--sql-shell",
dest="sqlShell",
action="store_true",
help=u"返回一个sql的shell")
enumeration.add_option("--sql-file",
dest="sqlFile",
help=u"从文件中读取执行sql语句")
# 爆破选项
brute = OptionGroup(parser, u"爆破", u"这些选项可用于执行爆破检查")
brute.add_option("--common-tables",
dest="commonTables",
action="store_true",
help=u"检测常见的表名，暴力破解表名")
brute.add_option("--common-columns",
dest="commonColumns",
action="store_true",
help=u"检测常见的字段名，暴力破解列名")
# 用户自定义的功能选项
udf = OptionGroup(parser, u"使用用户自定义的功能进行注入")
udf.add_option("--udf-inject",
dest="udfInject",
action="store_true",
help=u"注入用户自定义的功能")
udf.add_option("--shared-lib", dest="shLib", help=u"共享库的本地路径")
# 文件系统访问"", "这些选项可用于访问基础文件系统的后端数据库管理系统"
filesystem = OptionGroup(parser, u"文件系统访问", u"这些选项可用于访问基础文件系统的后端DBMS")
filesystem.add_option("--file-read",
dest="rFile",
help=u"从数据库服务器中读取文件")
filesystem.add_option("--file-write",
dest="wFile",
help=u"把文件写入/上传到数据库服务器中")
filesystem.add_option("--file-dest",
dest="dFile",
help=u"写入数据库服务器的绝对路径")
# Takeover options
takeover = OptionGroup(parser, u"操作系统访问", u"这些选项可用于访问基础操作系统的后端DBMS")
takeover.add_option("--os-cmd", dest="osCmd", help=u"执行操作系统命令")
takeover.add_option("--os-shell",
dest="osShell",
action="store_true",
help=u"返回一个shell")
takeover.add_option("--os-pwn",
dest="osPwn",
action="store_true",
help=u"调出一个带外shell/meterpreter/VNC")
takeover.add_option("--os-smbrelay",
dest="osSmb",
action="store_true",
help=u"一键调出OOB shell/meterpreter/VNC")
takeover.add_option("--os-bof",
dest="osBof",
action="store_true",
help=u"存储过程缓冲区溢出利用")
takeover.add_option("--priv-esc",
dest="privEsc",
action="store_true",
help=u"对当前连接数据库进程的用户进行权限提升")
takeover.add_option("--msf-path",
dest="msfPath",
help=u"安装Metasploit Framework的本地路径")
takeover.add_option("--tmp-path",
dest="tmpPath",
help=u"临时文件目录的远程绝对路径")
# Windows registry options
windows = OptionGroup(parser, u"Windows注册表访问",
u"这些选项可用于访问后端数据库管理系统Windows注册表")
windows.add_option("--reg-read",
dest="regRead",
action="store_true",
help=u"读取Windows注册表项值")
windows.add_option("--reg-add",
dest="regAdd",
action="store_true",
help=u"写入注册表值")
windows.add_option("--reg-del",
dest="regDel",
action="store_true",
help=u"删除注册表值")
windows.add_option("--reg-key", dest="regKey",
help=u"指定键，配合之前三个参数使用，"
u"例如：python sqlmap.py -u http://192.168.136.129/sqlmap/pgsql/get_int.aspx?id=1"
u" --reg-add --reg-key=\"HKEY_LOCAL_MACHINE\SOFTWARE\sqlmap\" --reg-value=Test" \
u" --reg-type=REG_SZ --reg-data=1")
windows.add_option("--reg-value", dest="regVal", help=u"指定键值")
windows.add_option("--reg-data", dest="regData", help=u"指定键值的数据")
windows.add_option("--reg-type", dest="regType", help=u"指定键值的类型")
# 常用选项
general = OptionGroup(parser, u"通用", u"一些常用选项")
general.add_option("-s",
dest="sessionFile",
help=u"从（.sqlite）文件中读取session会话")
general.add_option(
"-t",
dest="trafficFile",
help=u"保存HTTP(S)日志，这个参数需要跟一个文本文件，sqlmap会把HTTP(S)请求与响应的日志保存到那里")
general.add_option("--batch",
dest="batch",
action="store_true",
help=u"非交互模式，用此参数，不需要用户输入，将会使用sqlmap提示的默认值一直运行下去")
general.add_option(
"--binary-fields",
dest="binaryFields",
help=u"Result fields having binary values (例如： \"digest\")")
general.add_option("--charset",
dest="charset",
help=u"强制指定字符编码（如：--charset=GBK）")
general.add_option("--check-internet",
dest="checkInternet",
action="store_true",
help=u"访问\"http://ipinfo.io/\"确认是否连接到互联网")
general.add_option(
"--crawl",
dest="crawlDepth",
type="int",
help=u"爬行网站URL，sqlmap可以收集潜在的可能存在漏洞的连接，后面跟的参数是爬行的深度--batch --crawl=3"
)
general.add_option(
"--crawl-exclude",
dest="crawlExclude",
help=
u"使用正则表达式排除网页中我们不想抓取的内容 (例如我们不想爬行url中包含logout的页面，可以这样写--crawl-exclude=logout)"
)
general.add_option(
"--csv-del",
dest="csvDel",
help=u"当保存为CSV格式时（--dump-format=CSV），需要一个分隔符(默认是逗号： \"%s\")，"
u"用户也可以改为别的,如分号：--csv-del=\";\" " % defaults.csvDel)
general.add_option("--dump-format",
dest="dumpFormat",
help=u"输出数据的格式（CSV，HTML或SQLITE）默认输出CSV格式")
general.add_option(
"--eta",
dest="eta",
action="store_true",
help=u"计算注入数据的剩余时间，sqlmap先输出长度，预计完成时间，显示百分比，输出字符"
u"17% [========> ] 11/64 ETA 00:19")
general.add_option(
"--flush-session",
dest="flushSession",
action="store_true",
help=u"刷新session文件，如果不想用之前缓存这个目标的session文件，可以使用这个参数。 "
u"会清空之前的session，重新测试该目标。")
general.add_option(
"--forms",
dest="forms",
action="store_true",
help=
u"在目标网址上解析和测试表单，如果你想通过form表单来测试SQL注入或者是弱密码(名字加密码) ，你可以通过使用-r把请求保存在文件中进行测试 或者--data发送POST数据 或者让sqlmap自动选择个选项对HTML响应中的<;form>;或者是<;input>;这样的标签进行测试，通过给sqlmap提供目标地址参数-u以及使用--form参数，它会自动请求对应的目标地址并且对form表单的输入进行测试"
)
general.add_option("--fresh-queries",
dest="freshQueries",
action="store_true",
help=u"忽略存储在会话文件中的查询结果")
general.add_option("--har", dest="harFile", help=u"将所有HTTP流量记录到HAR文件中")
general.add_option(
"--hex",
dest="hexConvert",
action="store_true",
help=
u"使用DBMS十六进制功能进行数据检索，很多情况下你检索的数据可能是非ASCII码的， 解决这个问题的一个办法是使用DBMS十六进制功能。 在这个开关打开的情况下，数据在被检索之前被编码为十六进制形式，然后被编码为原始形式。"
)
general.add_option("--output-dir",
dest="outputDir",
action="store",
help=u"自定义输出目录路径")
general.add_option("--parse-errors",
dest="parseErrors",
action="store_true",
help=u"从响应中解析并显示DBMS错误消息")
general.add_option("--save", dest="saveConfig", help=u"将选项保存到配置文件中INI")
general.add_option("--scope",
dest="scope",
help=u"利用正则表达式从提供的代理日志中过滤目标")
general.add_option("--test-filter",
dest="testFilter",
help=u"通过Payload和/或标题选择测试 (例如:ROW)")
general.add_option("--test-skip",
dest="testSkip",
help=u"跳过Payload和/或标题测试 (例如:BENCHMARK)")
general.add_option("--update",
dest="updateAll",
action="store_true",
help=u"更新sqlmap")
# 杂项
miscellaneous = OptionGroup(parser, u"解忧杂货铺")
miscellaneous.add_option("-z",
dest="mnemonics",
help=u"使用简短的助记符 (例如: \"flu,bat,ban,tec=EU\")")
miscellaneous.add_option("--alert",
dest="alert",
help=u"发现SQL注入时运行主机操作系统命令")
miscellaneous.add_option(
"--answers",
dest="answers",
help=
u"设置问题的答案，在使用--batch的时候可以通过使用--answers来指定某个回答所需要的答案，如果是多个的话 可以通过使用,来进行分隔 (例如: \"quit=N,follow=N\")"
)
miscellaneous.add_option("--beep",
dest="beep",
action="store_true",
help=u"检测到注入点时发出蜂鸣声提示")
miscellaneous.add_option("--cleanup",
dest="cleanup",
action="store_true",
help=u"清除sqlmap注入时产生的udf与表")
miscellaneous.add_option(
"--dependencies",
dest="dependencies",
action="store_true",
help=
u"检查sqlmap是否缺少第三方库，sqlmap在某些特定的情况下需要用到第三方的库 (例如 -d --os-pwn用到的icmpsh 通道 --auth-type 用到的NTLM HTTP认证类型) 在这些情况下都会有警告，建议使用--dependencies来进行检查"
)
miscellaneous.add_option("--disable-coloring",
dest="disableColoring",
action="store_true",
help=u"禁用控制台输出着色")
miscellaneous.add_option(
"--gpage",
dest="googlePage",
type="int",
help=u"默认sqlmap使用前100个URL地址作为注入测试，结合此选项，可以对指定页码的URL测试")
miscellaneous.add_option("--identify-waf",
dest="identifyWaf",
action="store_true",
help=u"对WAF/IPS/IDS保护进行全面测试")
miscellaneous.add_option(
"--mobile",
dest="mobile",
action="store_true",
help=u"通过HTTP User-Agent header模拟智能手机,"
u"有时服务端只接收移动端的访问，此时可以设定一个手机的User-Agent来模仿手机登陆")
miscellaneous.add_option("--offline",
dest="offline",
action="store_true",
help=u"离线模式工作(仅使用会话数据)")
miscellaneous.add_option(
"--purge-output",
dest="purgeOutput",
action="store_true",
help=u"从输出目录中安全删除所有内容,有时需要删除结果文件，而不被恢复，原有文件将会被随机的一些文件覆盖")
miscellaneous.add_option("--skip-waf",
dest="skipWaf",
action="store_true",
help=u"跳过启发式检测WAF/IPS/IDS保护")
miscellaneous.add_option(
"--smart",
dest="smart",
action="store_true",
help=u"启发式判断注入,有时对目标非常多的URL进行测试，为节省时间，只对能够快速判断为注入的报错点进行注入")
miscellaneous.add_option("--sqlmap-shell",
dest="sqlmapShell",
action="store_true",
help=u"交互式sqlmap shell")
miscellaneous.add_option("--tmp-dir",
dest="tmpDir",
help=u"用于存储临时文件的本地目录")
miscellaneous.add_option("--web-root",
dest="webRoot",
help=u"Web服务器文件根目录(例如 \"/var/www\")")
miscellaneous.add_option("--wizard",
dest="wizard",
action="store_true",
help=u"简单的向导界面，用于初级用户")
# Hidden and/or experimental options
parser.add_option("--dummy",
dest="dummy",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--murphy-rate",
dest="murphyRate",
type="int",
help=SUPPRESS_HELP)
parser.add_option("--disable-precon",
dest="disablePrecon",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--disable-stats",
dest="disableStats",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--profile",
dest="profile",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--force-dns",
dest="forceDns",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--force-threads",
dest="forceThreads",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--smoke-test",
dest="smokeTest",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--live-test",
dest="liveTest",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--stop-fail",
dest="stopFail",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--run-case", dest="runCase", help=SUPPRESS_HELP)
# API选项
parser.add_option("--api",
dest="api",
action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--taskid", dest="taskid", help=SUPPRESS_HELP)
parser.add_option("--database", dest="database", help=SUPPRESS_HELP)
parser.add_option_group(target)
parser.add_option_group(request)
parser.add_option_group(optimization)
parser.add_option_group(injection)
parser.add_option_group(detection)
parser.add_option_group(techniques)
parser.add_option_group(fingerprint)
parser.add_option_group(enumeration)
parser.add_option_group(brute)
parser.add_option_group(udf)
parser.add_option_group(filesystem)
parser.add_option_group(takeover)
parser.add_option_group(windows)
parser.add_option_group(general)
parser.add_option_group(miscellaneous)
# Dirty hack可以显示更长的选项，而不会分成两行
def _(self, *args):
retVal = parser.formatter._format_option_strings(*args)
# sqlmap参数的最大长度
# MAX_HELP_OPTION_LENGTH = 18
if len(retVal) > MAX_HELP_OPTION_LENGTH:
retVal = ("%%.%ds.." %
(MAX_HELP_OPTION_LENGTH -
parser.formatter.indent_increment)) % retVal
return retVal
parser.formatter._format_option_strings = parser.formatter.format_option_strings
parser.formatter.format_option_strings = type(
parser.formatter.format_option_strings)(_, parser, type(parser))
# Dirty hack for making a short option '-hh'简短选项
option = parser.get_option("--hh")
option._short_opts = ["-hh"]
option._long_opts = []
# Dirty hack for inherent help message of switch '-h'固有的帮助信息
option = parser.get_option("-h")
option.help = option.help.capitalize().replace("此帮助", "基本的帮助")
_ = []
prompt = False
advancedHelp = True
extraHeaders = []
# Reference: https://stackoverflow.com/a/4012683 (Note: previously used "...sys.getfilesystemencoding() or UNICODE_ENCODING")
for arg in argv:
_.append(getUnicode(arg, encoding=sys.stdin.encoding))
argv = _
checkDeprecatedOptions(argv)
prompt = "--sqlmap-shell" in argv
if prompt:
parser.usage = ""
cmdLineOptions.sqlmapShell = True
_ = ["x", "q", "exit", "quit", "clear"]
for option in parser.option_list:
_.extend(option._long_opts)
_.extend(option._short_opts)
for group in parser.option_groups:
for option in group.option_list:
_.extend(option._long_opts)
_.extend(option._short_opts)
autoCompletion(AUTOCOMPLETE_TYPE.SQLMAP, commands=_)
while True:
command = None
try:
command = raw_input("sqlmap-shell> ").strip()
command = getUnicode(command, encoding=sys.stdin.encoding)
except (KeyboardInterrupt, EOFError):
print
raise SqlmapShellQuitException
if not command:
continue
elif command.lower() == "clear":
clearHistory()
dataToStdout(u"[i] 清除历史记录\n")
saveHistory(AUTOCOMPLETE_TYPE.SQLMAP)
elif command.lower() in ("x", "q", "exit", "quit"):
raise SqlmapShellQuitException
elif command[0] != '-':
dataToStdout(u"[!] 提供无效选项\n")
dataToStdout(
u"[i] 正确的例子: '-u http://www.site.com/vuln.php?id=1 --banner'\n"
)
else:
saveHistory(AUTOCOMPLETE_TYPE.SQLMAP)
loadHistory(AUTOCOMPLETE_TYPE.SQLMAP)
break
try:
for arg in shlex.split(command):
argv.append(getUnicode(arg, encoding=sys.stdin.encoding))
except ValueError, ex:
raise SqlmapSyntaxException, u"命令行解析时出错 ('%s')" % ex.message
for i in xrange(len(argv)):
if argv[i] == "-hh":
argv[i] = "-h"
elif len(argv[i]) > 1 and all(
ord(_) in xrange(0x2018, 0x2020)
for _ in ((argv[i].split('=', 1)[-1].strip() or ' ')[0],
argv[i][-1])):
dataToStdout("[!] 从互联网上复制粘贴非法(非控制台)引号字符是非法的(%s)\n" % argv[i])
raise SystemExit
elif len(argv[i]) > 1 and u"\uff0c" in argv[i].split('=', 1)[-1]:
dataToStdout("[!] 从互联网复制粘贴非法(非控制台)逗号字符是非法的(%s)\n" % argv[i])
raise SystemExit
elif re.search(r"\A-\w=.+", argv[i]):
dataToStdout("[!] 检测到潜在的错误(非法的 '=') 短选项 ('%s')\n" % argv[i])
raise SystemExit
elif argv[i] == "-H":
if i + 1 < len(argv):
extraHeaders.append(argv[i + 1])
elif re.match(r"\A\d+!\Z", argv[i]) and argv[max(
0, i - 1)] == "--threads" or re.match(
r"\A--threads.+\d+!\Z", argv[i]):
argv[i] = argv[i][:-1]
conf.skipThreadCheck = True
elif argv[i] == "--version":
print VERSION_STRING.split('/')[-1]
raise SystemExit
elif argv[i] in ("-h", "--help"):
advancedHelp = False
for group in parser.option_groups[:]:
found = False
for option in group.option_list:
if option.dest not in BASIC_HELP_ITEMS:
option.help = SUPPRESS_HELP
else:
found = True
if not found:
parser.option_groups.remove(group)
for verbosity in (_ for _ in argv if re.search(r"\A\-v+\Z", _)):
try:
if argv.index(verbosity) == len(argv) - 1 or not argv[
argv.index(verbosity) + 1].isdigit():
conf.verbose = verbosity.count('v') + 1
del argv[argv.index(verbosity)]
except (IndexError, ValueError):
pass
try:
(args, _) = parser.parse_args(argv)
except UnicodeEncodeError, ex:
dataToStdout("\n[!] %s\n" % ex.object.encode("unicode-escape"))
raise SystemExit
LOCALES_PATH = os.path.join(INSTALL_PREFIX, 'share', 'locale')
gettext.install('lazygal', LOCALES_PATH)
locale.setlocale(locale.LC_ALL, '')
import lazygal
from lazygal.generators import Album
import lazygal.config
import lazygal.eyecandy
import lazygal.log
usage = _("usage: %prog [options] albumdir")
parser = OptionParser(usage=usage)
# The help option must be changed to comply with i18n.
parser.get_option('-h').help = _("Show this help message and exit.")
parser.add_option("",
"--quiet",
action="store_true",
dest="quiet",
help=_("Don't output anything except for errors."))
parser.add_option("",
"--debug",
action="store_true",
dest="debug",
help=_("Output everything that lazygal is doing."))
parser.add_option(
"-o",
"--output-directory",
action="store",
else:
if options.output:
filename = options.output
else:
filename = inputfile
output = "%s.png" % (os.path.splitext(filename)[0])
c = 1
while os.path.exists(output):
output = "%s_%i.png" % (os.path.splitext(filename)[0], c)
c += 1
parser = SimpleParser(inputfile)
if options.solution:
solution = options.solution
elif parser.has_option("solution"):
solution = parser.get_option("solution")
else:
solution = None
wordlist = parser.get_questions()
cwd = CrossWord(options.columns, options.rows, " ", 5000, wordlist)
score = cwd.compute_crossword(best_of=options.bestof, force_solved=False)
tmplist = [w.word.lower() for w in cwd.placed_words]
missing = [w.word for w in cwd.wordlist if w.word.lower() not in tmplist]
if missing != []:
print("Could not place some words. Probably your grid is too small. Sometimes setting \"--bestof\" to a higer value also help.")
print("Words that could not be placed: '%s'" % missing)
if options.stats:
stats(cwd, False)
# Parse
(options, args) = parser.parse_args()
# Target config or netcdf
if len(args)>0:
ncfile = args[0]
else:
ncfile = options.ncfile
if ncfile is not None:
options.force = True
# Bounds
defaults = [('xmin', -720), ('xmax', 720), ('ymin', -90), ('ymax', 90)]
for bname, bdef in defaults:
if getattr(options, bname) is None:
desc = parser.get_option('--'+bname).help
if options.force:
value = bdef
else:
while True:
try:
value = float(raw_input('Please set %s (or use -f or --%s options): '%(desc.lower(), bname)).strip())
break
except ValueError:
print 'Please enter a float'
if getattr(options, bname) is None:
setattr(options, bname, value)
lon = (options.xmin, options.xmax)
lat = (options.ymin, options.ymax)
# Netcdf file direct access
class CmdBase(object):
doesLogging = True
"""
Class used for all Zenoss commands
"""
def __init__(self, noopts=0, args=None, should_log=None):
zope.component.provideAdapter(DefaultTraversable, (None,))
# This explicitly loads all of the products - must happen first!
from OFS.Application import import_products
import_products()
#make sure we aren't in debug mode
import Globals
Globals.DevelopmentMode = False
# We must import ZenossStartup at this point so that all Zenoss daemons
# and tools will have any ZenPack monkey-patched methods available.
import Products.ZenossStartup
unused(Products.ZenossStartup)
zcml.load_site()
import Products.ZenWidgets
load_config_override('scriptmessaging.zcml', Products.ZenWidgets)
self.usage = "%prog [options]"
self.noopts = noopts
self.inputArgs = args
# inputArgs was created to allow unit tests to pass in command line
# arguments and get around whatever Zope was doing to sys.argv.
if self.inputArgs is None:
self.inputArgs = sys.argv[1:]
self.parser = None
self.args = []
self.buildParser()
self.buildOptions()
# Get defaults from global.conf. They will be overridden by
# daemon-specific config file or command line arguments.
applyGlobalConfToParser(self.parser)
self.parseOptions()
if self.options.configfile:
self.parser.defaults = self.getConfigFileDefaults(self.options.configfile)
# We've updated the parser with defaults from configs, now we need
# to reparse our command-line to get the correct overrides from
# the command-line
self.parseOptions()
if should_log is not None:
self.doesLogging = should_log
if self.doesLogging:
self.setupLogging()
def buildParser(self):
"""
Create the options parser
"""
if not self.parser:
from Products.ZenModel.ZenossInfo import ZenossInfo
try:
zinfo= ZenossInfo('')
version= str(zinfo.getZenossVersion())
except Exception:
from Products.ZenModel.ZVersion import VERSION
version= VERSION
self.parser = OptionParser(usage=self.usage,
version="%prog " + version,
option_class=LogSeverityOption)
def buildOptions(self):
"""
Basic options setup. Other classes should call this before adding
more options
"""
self.buildParser()
if self.doesLogging:
group = OptionGroup(self.parser, "Logging Options")
group.add_option(
'-v', '--logseverity',
dest='logseverity', default='INFO', type='loglevel',
help='Logging severity threshold',
)
group.add_option(
'--logpath', dest='logpath', default=zenPath('log'), type='str',
help='Override the default logging path; default $ZENHOME/log'
)
group.add_option(
'--maxlogsize',
dest='maxLogKiloBytes', default=10240, type='int',
help='Max size of log file in KB; default 10240',
)
group.add_option(
'--maxbackuplogs',
dest='maxBackupLogs', default=3, type='int',
help='Max number of back up log files; default 3',
)
self.parser.add_option_group(group)
self.parser.add_option("-C", "--configfile",
dest="configfile",
help="Use an alternate configuration file" )
self.parser.add_option("--genconf",
action="store_true",
default=False,
help="Generate a template configuration file" )
self.parser.add_option("--genxmltable",
action="store_true",
default=False,
help="Generate a Docbook table showing command-line switches." )
self.parser.add_option("--genxmlconfigs",
action="store_true",
default=False,
help="Generate an XML file containing command-line switches." )
def parseOptions(self):
"""
Uses the optparse parse previously populated and performs common options.
"""
if self.noopts:
args = []
else:
args = self.inputArgs
(self.options, self.args) = self.parser.parse_args(args=args)
if self.options.genconf:
self.generate_configs( self.parser, self.options )
if self.options.genxmltable:
self.generate_xml_table( self.parser, self.options )
if self.options.genxmlconfigs:
self.generate_xml_configs( self.parser, self.options )
def getConfigFileDefaults(self, filename, correctErrors=True):
# TODO: This should be refactored - duplicated code with CmdBase.
"""
Parse a config file which has key-value pairs delimited by white space,
and update the parser's option defaults with these values.
@parameter filename: name of configuration file
@type filename: string
"""
options = self.parser.get_default_values()
lines = self.loadConfigFile(filename)
if lines:
lines, errors = self.validateConfigFile(filename, lines,
correctErrors=correctErrors)
args = self.getParamatersFromConfig(lines)
try:
self.parser._process_args([], args, options)
except (BadOptionError, OptionValueError) as err:
print >>sys.stderr, 'WARN: %s in config file %s' % (err, filename)
return options.__dict__
def getGlobalConfigFileDefaults(self):
# Deprecated: This method is going away - it is duplicated in GlobalConfig.py
"""
Parse a config file which has key-value pairs delimited by white space,
and update the parser's option defaults with these values.
"""
filename = zenPath('etc', 'global.conf')
options = self.parser.get_default_values()
lines = self.loadConfigFile(filename)
if lines:
args = self.getParamatersFromConfig(lines)
try:
self.parser._process_args([], args, options)
except (BadOptionError, OptionValueError):
# Ignore it, we only care about our own options as defined in the parser
pass
return options.__dict__
def loadConfigFile(self, filename):
# TODO: This should be refactored - duplicated code with CmdBase.
"""
Parse a config file which has key-value pairs delimited by white space.
@parameter filename: path to the configuration file
@type filename: string
"""
lines = []
if not os.path.exists(filename):
return lines
try:
with open(filename) as file:
for line in file:
if line.lstrip().startswith('#') or line.strip() == '':
lines.append(dict(type='comment', line=line))
else:
try:
# add default blank string for keys with no default value
# valid delimiters are space, ':' and/or '=' (see ZenUtils/config.py)
key, value = (re.split(r'[\s:=]+', line.strip(), 1) + ['',])[:2]
except ValueError:
lines.append(dict(type='option', line=line, key=line.strip(), value=None, option=None))
else:
option = self.parser.get_option('--%s' % key)
lines.append(dict(type='option', line=line, key=key, value=value, option=option))
except IOError as e:
errorMessage = ('WARN: unable to read config file {filename} '
'-- skipping. ({exceptionName}: {exception})').format(
filename=filename,
exceptionName=e.__class__.__name__,
exception=e
)
print >>sys.stderr, errorMessage
return []
return lines
def validateConfigFile(self, filename, lines, correctErrors=True, warnErrors=True):
"""
Validate config file lines which has key-value pairs delimited by white space,
and validate that the keys exist for this command's option parser. If
the option does not exist or has an empty value it will comment it out
in the config file.
@parameter filename: path to the configuration file
@type filename: string
@parameter lines: lines from config parser
@type lines: list
@parameter correctErrors: Whether or not invalid conf values should be
commented out.
@type correctErrors: boolean
"""
output = []
errors = []
validLines = []
date = datetime.datetime.now().isoformat()
errorTemplate = '## Commenting out by config parser (%s) on %s: %%s\n' % (
sys.argv[0], date)
for lineno, line in enumerate(lines):
if line['type'] == 'comment':
output.append(line['line'])
elif line['type'] == 'option':
if line['value'] is None:
errors.append((lineno + 1, 'missing value for "%s"' % line['key']))
output.append(errorTemplate % 'missing value')
output.append('## %s' % line['line'])
elif line['option'] is None:
errors.append((lineno + 1, 'unknown option "%s"' % line['key']))
output.append(errorTemplate % 'unknown option')
output.append('## %s' % line['line'])
else:
validLines.append(line)
output.append(line['line'])
else:
errors.append((lineno + 1, 'unknown line "%s"' % line['line']))
output.append(errorTemplate % 'unknown line')
output.append('## %s' % line['line'])
if errors:
if correctErrors:
for lineno, message in errors:
print >>sys.stderr, 'INFO: Commenting out %s on line %d in %s' % (message, lineno, filename)
with open(filename, 'w') as file:
file.writelines(output)
if warnErrors:
for lineno, message in errors:
print >>sys.stderr, 'WARN: %s on line %d in %s' % (message, lineno, filename)
return validLines, errors
def getParamatersFromConfig(self, lines):
# Deprecated: This method is going away
return _convertConfigLinesToArguments(self.parser, lines)
def setupLogging(self):
"""
Set common logging options
"""
rlog = logging.getLogger()
rlog.setLevel(logging.WARN)
mname = self.__class__.__name__
self.log = logging.getLogger("zen."+ mname)
zlog = logging.getLogger("zen")
try:
loglevel = int(self.options.logseverity)
except ValueError:
loglevel = getattr(logging, self.options.logseverity.upper(), logging.INFO)
zlog.setLevel(loglevel)
logdir = self.checkLogpath()
if logdir:
logfile = os.path.join(logdir, mname.lower()+".log")
maxBytes = self.options.maxLogKiloBytes * 1024
backupCount = self.options.maxBackupLogs
h = logging.handlers.RotatingFileHandler(logfile, maxBytes=maxBytes,
backupCount=backupCount)
h.setFormatter(logging.Formatter(
"%(asctime)s %(levelname)s %(name)s: %(message)s",
"%Y-%m-%d %H:%M:%S"))
rlog.addHandler(h)
else:
logging.basicConfig()
def checkLogpath(self):
"""
Validate the logpath is valid
"""
if not self.options.logpath:
return None
else:
logdir = self.options.logpath
if not os.path.exists(logdir):
# try creating the directory hierarchy if it doesn't exist...
try:
os.makedirs(logdir)
except OSError:
raise SystemExit("logpath:%s doesn't exist and cannot be created" % logdir)
elif not os.path.isdir(logdir):
raise SystemExit("logpath:%s exists but is not a directory" % logdir)
return logdir
def pretty_print_config_comment( self, comment ):
"""
Quick and dirty pretty printer for comments that happen to be longer than can comfortably
be seen on the display.
"""
max_size= 40
#
# As a heuristic we'll accept strings that are +- text_window
# size in length.
#
text_window= 5
if len( comment ) <= max_size + text_window:
return comment
#
# First, take care of embedded newlines and expand them out to array entries
#
new_comment= []
all_lines= comment.split( '\n' )
for line in all_lines:
if len(line) <= max_size + text_window:
new_comment.append( line )
continue
start_position= max_size - text_window
while len(line) > max_size + text_window:
index= line.find( ' ', start_position )
if index > 0:
new_comment.append( line[ 0:index ] )
line= line[ index: ]
else:
if start_position == 0:
#
# If we get here it means that the line is just one big string with no spaces
# in it. There's nothing that we can do except print it out. Doh!
#
new_comment.append( line )
break
#
# Okay, haven't found anything to split on -- go back and try again
#
start_position= start_position - text_window
if start_position < 0:
start_position= 0
else:
new_comment.append( line )
return "\n# ".join( new_comment )
def generate_configs( self, parser, options ):
"""
Create a configuration file based on the long-form of the option names
@parameter parser: an optparse parser object which contains defaults, help
@parameter options: parsed options list containing actual values
"""
#
# Header for the configuration file
#
unused(options)
daemon_name= os.path.basename( sys.argv[0] )
daemon_name= daemon_name.replace( '.py', '' )
print """#
# Configuration file for %s
#
# To enable a particular option, uncomment the desired entry.
#
# Parameter Setting
# --------- -------""" % ( daemon_name )
options_to_ignore= ( 'help', 'version', '', 'genconf', 'genxmltable' )
#
# Create an entry for each of the command line flags
#
# NB: Ideally, this should print out only the option parser dest
# entries, rather than the command line options.
#
import re
for opt in getAllParserOptionsGen(parser):
if opt.help is SUPPRESS_HELP:
continue
#
# Get rid of the short version of the command
#
option_name= re.sub( r'.*/--', '', "%s" % opt )
#
# And what if there's no short version?
#
option_name= re.sub( r'^--', '', "%s" % option_name )
#
# Don't display anything we shouldn't be displaying
#
if option_name in options_to_ignore:
continue
#
# Find the actual value specified on the command line, if any,
# and display it
#
value= getattr( parser.values, opt.dest )
default_value= parser.defaults.get( opt.dest )
if default_value is NO_DEFAULT or default_value is None:
default_value= ""
default_string= ""
if default_value != "":
default_string= ", default: " + str( default_value )
comment= self.pretty_print_config_comment( opt.help + default_string )
#
# NB: I would prefer to use tabs to separate the parameter name
# and value, but I don't know that this would work.
#
print """#
# %s
#%s %s""" % ( comment, option_name, value )
#
# Pretty print and exit
#
print "#"
sys.exit( 0 )
def generate_xml_table( self, parser, options ):
"""
Create a Docbook table based on the long-form of the option names
@parameter parser: an optparse parser object which contains defaults, help
@parameter options: parsed options list containing actual values
"""
#
# Header for the configuration file
#
unused(options)
daemon_name= os.path.basename( sys.argv[0] )
daemon_name= daemon_name.replace( '.py', '' )
print """<?xml version="1.0" encoding="UTF-8"?>
<section version="4.0" xmlns="http://docbook.org/ns/docbook"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xi="http://www.w3.org/2001/XInclude"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns:mml="http://www.w3.org/1998/Math/MathML"
xmlns:html="http://www.w3.org/1999/xhtml"
xmlns:db="http://docbook.org/ns/docbook"
xml:id="%s.options"
>
<title>%s Options</title>
<para />
<table frame="all">
<caption>%s <indexterm><primary>Daemons</primary><secondary>%s</secondary></indexterm> options</caption>
<tgroup cols="2">
<colspec colname="option" colwidth="1*" />
<colspec colname="description" colwidth="2*" />
<thead>
<row>
<entry> <para>Option</para> </entry>
<entry> <para>Description</para> </entry>
</row>
</thead>
<tbody>
""" % ( daemon_name, daemon_name, daemon_name, daemon_name )
options_to_ignore= ( 'help', 'version', '', 'genconf', 'genxmltable' )
#
# Create an entry for each of the command line flags
#
# NB: Ideally, this should print out only the option parser dest
# entries, rather than the command line options.
#
import re
for opt in getAllParserOptionsGen(parser):
if opt.help is SUPPRESS_HELP:
continue
#
# Create a Docbook-happy version of the option strings
# Yes, <arg></arg> would be better semantically, but the output
# just looks goofy in a table. Use literal instead.
#
all_options= '<literal>' + re.sub( r'/', '</literal>,</para> <para><literal>', "%s" % opt ) + '</literal>'
#
# Don't display anything we shouldn't be displaying
#
option_name= re.sub( r'.*/--', '', "%s" % opt )
option_name= re.sub( r'^--', '', "%s" % option_name )
if option_name in options_to_ignore:
continue
default_value= parser.defaults.get( opt.dest )
if default_value is NO_DEFAULT or default_value is None:
default_value= ""
default_string= ""
if default_value != "":
default_string= "<para> Default: <literal>" + str( default_value ) + "</literal></para>\n"
comment= self.pretty_print_config_comment( opt.help )
#
# TODO: Determine the variable name used and display the --option_name=variable_name
#
if opt.action in [ 'store_true', 'store_false' ]:
print """<row>
<entry> <para>%s</para> </entry>
<entry>
<para>%s</para>
%s</entry>
</row>
""" % ( all_options, comment, default_string )
else:
target= '=<replaceable>' + opt.dest.lower() + '</replaceable>'
all_options= all_options + target
all_options= re.sub( r',', target + ',', all_options )
print """<row>
<entry> <para>%s</para> </entry>
<entry>
<para>%s</para>
%s</entry>
</row>
""" % ( all_options, comment, default_string )
#
# Close the table elements
#
print """</tbody></tgroup>
</table>
<para />
</section>
"""
sys.exit( 0 )
def generate_xml_configs( self, parser, options ):
"""
Create an XML file that can be used to create Docbook files
as well as used as the basis for GUI-based daemon option
configuration.
"""
#
# Header for the configuration file
#
unused(options)
daemon_name= os.path.basename( sys.argv[0] )
daemon_name= daemon_name.replace( '.py', '' )
export_date = datetime.datetime.now()
print """<?xml version="1.0" encoding="UTF-8"?>
<!-- Default daemon configuration generated on %s -->
<configuration id="%s" >
""" % ( export_date, daemon_name )
options_to_ignore= (
'help', 'version', '', 'genconf', 'genxmltable',
'genxmlconfigs',
)
#
# Create an entry for each of the command line flags
#
# NB: Ideally, this should print out only the option parser dest
# entries, rather than the command line options.
#
import re
for opt in getAllParserOptionsGen(parser):
if opt.help is SUPPRESS_HELP:
continue
#
# Don't display anything we shouldn't be displaying
#
option_name= re.sub( r'.*/--', '', "%s" % opt )
option_name= re.sub( r'^--', '', "%s" % option_name )
if option_name in options_to_ignore:
continue
default_value= parser.defaults.get( opt.dest )
if default_value is NO_DEFAULT or default_value is None:
default_string= ""
else:
default_string= str( default_value )
#
# TODO: Determine the variable name used and display the --option_name=variable_name
#
if opt.action in [ 'store_true', 'store_false' ]:
print """ <option id="%s" type="%s" default="%s" help="%s" />
""" % ( option_name, "boolean", default_string, quote(opt.help), )
else:
target= opt.dest.lower()
print """ <option id="%s" type="%s" default="%s" target="%s" help="%s" />
""" % ( option_name, opt.type, quote(default_string), target, quote(opt.help), )
#
# Close the table elements
#
print """
</configuration>
"""
sys.exit( 0 )
"""
def _(self, *args):
_ = parser.formatter._format_option_strings(*args)
if len(_) > settings.MAX_OPTION_LENGTH:
_ = ("%%.%ds.." % (settings.MAX_OPTION_LENGTH -
parser.formatter.indent_increment)) % _
return _
parser.formatter._format_option_strings = parser.formatter.format_option_strings
parser.formatter.format_option_strings = type(
parser.formatter.format_option_strings)(_, parser)
option = parser.get_option("-h")
option.help = option.help.capitalize().replace(
"Show this help message and exit", "Show help and exit.")
(options, args) = parser.parse_args()
# Checkall the banner
if not options.version:
banner()
# argv input errors
settings.sys_argv_errors()
"""
The "os_shell" available options.
"""
def main():
kinds = (
'Exercise',
'Video',
'ExerciseVideo',
'StemmedIndex',
'LiteralIndex',
)
parser = OptionParser(
usage="%prog [options] upload|download",
description=
"Uploads the sample data to a server or downloads it from the server.")
parser.add_option("-U",
"--url",
default="http://*****:*****@example.com",
help="The username to use.")
parser.add_option("-k",
"--kinds",
default=','.join(kinds),
help="The comma separated list of kinds.")
parser.add_option(
"-p",
"--python",
default=(sys.executable if platform.system() == "Windows" else None),
help="Path of python executable.")
parser.add_option("-a",
"--appcfg",
default='appcfg.py',
help="Path of appcfg.py (Google App Engine).")
parser.add_option("-A",
"--application",
default='dev~khan-academy',
help="GAE application name")
(options, args) = parser.parse_args()
if len(args) < 1:
parser.print_help()
return
for kind in options.kinds.split(','):
filename = '%s.data' % kind
call_args = [
options.appcfg,
'--url=%s' % options.url,
'--email=%s' % options.email,
'--kind=%s' % kind,
'--filename=%s' % filename
]
if options.application is not None:
call_args.append('--application=%s' % options.application)
if options.email == parser.get_option('--email').default:
call_args.append('--passin')
if options.python is not None:
call_args.insert(0, options.python)
call_args.append('--num_threads=1')
if args[0] == 'upload':
call_args.append('upload_data')
elif args[0] == 'download':
if os.path.exists(filename):
os.remove(filename)
call_args.append('download_data')
else:
parser.print_help()
return
print ' '.join(call_args)
if options.email == parser.get_option('--email').default:
process = subprocess.Popen(call_args, stdin=subprocess.PIPE)
# Send a newline for the password prompt
process.communicate("\n")
else:
process = subprocess.Popen(call_args)
process.wait()
