def makeParser():
parser = OptionParser()
parser.usage=usage()
parser.add_option("-u", "--trackerUrl", type="string", default="http://ifp-32:7625/announce",
help="bittorrent tracker URL. (default: %default)")
parser.add_option("-b", "--bitTorrentScriptsPath", type="string", default="/opt/rocks/bittorrent",
help="The path to bittorrent scripts. (default: %default)")
parser.add_option("-r", "--rTorrentPath", type="string", default="/cworkspace/ifp-32-1/hasegawa/programs/bin.x86_64/rtorrent",
help="The path to rtorrent binary. (default: %default)")
parser.add_option("-t", "--torrent_dir", type="string", default="/cworkspace/ifp-32-1/hasegawa/programs/scripts/cluster/torrents",
help="Where to store torrent files. (default: %default)")
parser.add_option("-s", "--save_in", type="string", default="/scratch/hasegawa",
help="The path where to store the files downloaded from a torrent. (default: %default)")
parser.add_option("-m", "--pathToMirror", type="string",
help="The path to directory or file to mirror. Required argument.")
parser.add_option("--parse_dir_interval", type="int", default="60",
help="How often to check the torrent_dir for changes, in seconds. (default: %default sec)")
parser.add_option("--display_interval", type="int", default="60",
help="How often the swarm participants print status, in seconds. (default: %default sec)")
parser.add_option("--download_rate", type="int", default="9500",
help="Global maximim download rate limit, in kb. Currently there seems to be a bug in rtorrent, when the download rate is too fast (faster than the disk can write?), you get a 'File chunk write error: Cannot allocate memory.' and peers get dropped, and not picked up again until the next tracker announce. This affects performance. (default: %default kb)")
parser.add_option("--shutdown", action="store_true", default=False,
help="When in node mode, shutdown btlaunchmany.py (default: %default)")
intUse = OptionGroup(parser,"OPTIONS FOR INTERNAL USE")
intUse.add_option("-n", "--nodeMode", action="store_true", default=False,
help="Run script in node mode, i.e. ensure that exactly one copy of btlaunchmany.py is running (default: False)")
parser.add_option_group(intUse)
#command to include command line help in auto-generated documentation
return parser
def test_options(self, mock_parser, mock_group):
parser = OptionParser()
mock_parser.return_value = parser
parser.add_option = Mock(side_effect=parser.add_option)
parser.add_option_group = Mock(side_effect=parser.add_option_group)
parser._get_args = Mock(return_value=[])
group = OptionGroup(parser, '')
mock_group.return_value = group
group.add_option = Mock(side_effect=group.add_option)
# test
builder.get_options()
# validation
parser.add_option.assert_any_call(
'-w', '--working-dir', dest='working_dir', help=builder.WORKING_DIR)
parser.add_option.assert_any_call(
'-o', '--output-dir', dest='output_dir', help=builder.OUTPUT_DIR)
parser.add_option.assert_any_call(
'-c', '--clean', default=False, action='store_true', help=builder.CLEAN)
parser.add_option.assert_any_call('-p', '--path', help=builder.PATH)
parser.add_option_group.assert_called_with(group)
group.add_option.assert_any_call('-u', '--url', help=builder.URL)
group.add_option.assert_any_call('-b', '--branch', help=builder.BRANCH)
group.add_option.assert_any_call('-t', '--tag', help=builder.TAG)
def _get_partial_conversion_options_group(self):
group = OptionGroup(self.parser, 'Partial conversions')
group.add_option(ManOption(
'--pass', type='string',
action='callback', callback=self.callback_passes,
help='execute only specified PASS of conversion',
man_help=(
'Execute only pass \\fIpass\\fR of the conversion. '
'\\fIpass\\fR can be specified by name or by number (see '
'\\fB--help-passes\\fR).'
),
metavar='PASS',
))
group.add_option(ManOption(
'--passes', '-p', type='string',
action='callback', callback=self.callback_passes,
help=(
'execute passes START through END, inclusive (PASS, '
'START, and END can be pass names or numbers)'
),
man_help=(
'Execute passes \\fIstart\\fR through \\fIend\\fR of the '
'conversion (inclusive). \\fIstart\\fR and \\fIend\\fR can be '
'specified by name or by number (see \\fB--help-passes\\fR). '
'If \\fIstart\\fR or \\fIend\\fR is missing, it defaults to '
'the first or last pass, respectively. For this to work the '
'earlier passes must have been completed before on the '
'same CVS repository, and the generated data files must be '
'in the temporary directory (see \\fB--tmpdir\\fR).'
),
metavar='[START]:[END]',
))
return group
def register_options(cls, parser):
group = OptionGroup(parser, "Nagios specific options")
group.add_option("-w", "--warning", dest="warning")
group.add_option("-c", "--critical", dest="critical")
parser.add_option_group(group)
def format_command_help(cmd):
"""
Format help string for the command.
:param cmd: string: command name
:return: string
"""
usage = [cmd, '[options]']
alls = all_options(cmd)
standards = [_ for _ in alls if _.cli_match]
extras = [_ for _ in alls if not _.cli_match]
for opt in alls:
if not opt.name.startswith('-'):
optname = "[{0}]".format(opt.name) if opt.is_optional else opt.name
usage.append(optname)
usage = ' '.join(usage)
parser = OptParser(prog=cmd, add_help_option=False, usage=usage,
conflict_handler='resolve')
for opt in standards:
if opt.name.startswith('-'):
parser.add_option(*opt.args, **opt.kwargs)
if extras:
g = OptionGroup(parser, "Non-standard options")
for opt in extras:
g.add_option(*opt.args, **opt.kwargs)
parser.add_option_group(g)
return parser.format_help()
def get_proxy_options(parser):
proxy_group = OptionGroup(parser, "Proxy", "Options for specifying proxies. Format: [scheme://][user:password@]host:port")
proxy_group.add_option("", "--http-proxy", help="A (non-SSL) HTTP proxy", type="string", dest="http_proxy")
proxy_group.add_option("", "--https-proxy", help="An HTTPS proxy", type="string", dest="https_proxy")
return proxy_group
def _get_options_file_options_group(self):
group = OptionGroup(
self.parser, 'Configuration via options file'
)
self.parser.set_default('options_file_found', False)
group.add_option(ManOption(
'--options', type='string',
action='callback', callback=self.callback_options,
help=(
'read the conversion options from PATH. This '
'method allows more flexibility than using '
'command-line options. See documentation for info'
),
man_help=(
'Read the conversion options from \\fIpath\\fR instead of from '
'the command line. This option allows far more conversion '
'flexibility than can be achieved using the command-line alone. '
'See the documentation for more information. Only the following '
'command-line options are allowed in combination with '
'\\fB--options\\fR: \\fB-h\\fR/\\fB--help\\fR, '
'\\fB--help-passes\\fR, \\fB--version\\fR, '
'\\fB-v\\fR/\\fB--verbose\\fR, \\fB-q\\fR/\\fB--quiet\\fR, '
'\\fB-p\\fR/\\fB--pass\\fR/\\fB--passes\\fR, \\fB--dry-run\\fR, '
'\\fB--profile\\fR, \\fB--sort\\fR, \\fB--trunk-only\\fR, '
'\\fB--encoding\\fR, and \\fB--fallback-encoding\\fR. '
'Options are processed in the order specified on the command '
'line.'
),
metavar='PATH',
))
return group
def parser(mp=None):
if mp==None: mp = OptionParser()
mg1 = OptionGroup(mp,cyan+"dependencyFactory settings"+plain)
mg1.add_option('--bmap',help='Calculate btagmap JetMon/QCD.',default=False,action='store_true')
mg1.add_option('--kfac',help='Calculate k-factor.',default=False,action='store_true')
mp.add_option_group(mg1)
return mp
def generate_parser():
parser = OptionParser()
parser.add_option("-f","--hostname-file",dest="hostfile",
help="File containing Hostnames's used in the test, one per line.")
parser.add_option("-d","--dry-run",action="store_true",dest="dryrun",
default=False, help="Don't actually issue the DGI startup commands, "
"just show the procedure.")
parser.add_option("-n","--hostname",dest="hostnames",action="append",
help="UUIDs to use for this experiment set.")
parser.add_option("-t","--time",dest="time",default=None,
help="The option that will be provided to the unix command"
"timeout to terminate the run. Fomrat is of an argument to the"
"UNIX timeout command. e.g. 10m for a 10 minute run.")
parser.add_option("-c","--line-client",dest="lineclient",
help="If specified, indicates the hostname which should be connected"
"to in order to start the line server")
parser.add_option("-e","--experiment",dest="experiment",action="store_true",
help="Enable experiment mode, which can be used to observe the affects"
"of network link reliability on the system")
expgroup = OptionGroup(parser, "Network Experiments",
"These options will only be used in network experiment mode (-e). In"
"this mode, the program will run for a specified period of time with"
"incrementing network settings (from 0 up to full reliability")
expgroup.add_option("-o","--output-file",dest="outputfile",default="exp.dat",
help="File to write the experiment table to, if not provided, "
"defaults to exp.dat")
expgroup.add_option("-g","--granularity",dest="granularity",default=5,type="int",
help="The simulator will run experiments with a step up or down in the"
"granularity for each network connection.")
parser.add_option_group(expgroup)
return parser
def register_options(cls, parser):
group = OptionGroup(parser, 'Cacti specific options')
group.add_option('-l', '--leader', dest='leader',
action="store_true", help="only query the cluster leader")
parser.add_option_group(group)
def run(self, *args):
"""Create a new site."""
parser = OptionParser(usage=self.usage)
group = OptionGroup(parser, "Site Options")
group.add_option(
"--empty", action="store_true", dest='empty', default=True,
help="Create an empty site with only a config.")
group.add_option("--demo", action="store_false", dest='empty',
help="Create a site filled with example data.")
parser.add_option_group(group)
(options, args) = parser.parse_args(list(args))
if not args:
print("Usage: nikola init folder [options]")
return
target = args[0]
if target is None:
print(self.usage)
else:
if options.empty:
self.create_empty_site(target)
print('Created empty site at %s.' % target)
else:
self.copy_sample_site(target)
print("A new site with example data has been created at %s."
% target)
print("See README.txt in that folder for more information.")
self.create_configuration(target)
def parse_options():
parser = OptionParser()
parser.add_option(
"-a", "--ampfile", dest="ampsname", metavar="AMP", help="single input .amp file (default: %default)"
)
parser.add_option(
"-s", "--segfile", dest="segsname", metavar="SEG", help="single input .seg file (default: %default)"
)
parser.add_option(
"-w", "--wavfile", dest="wavname", metavar="WAV", help="single input .wav file (default: %default)"
)
group = OptionGroup(parser, "Batch processing")
group.add_option(
"-i",
"--input-dir",
dest="indir",
help="input directory; must contain wav/ and amps/ subdirectories" " with matching filesets.",
)
group.add_option("-o", "--output-dir", dest="outdir", help="output directory (will be created if it doesn't exist)")
parser.add_option_group(group)
parser.set_defaults(ampsname="test.amp", segsname="test.seg", wavname="test.wav")
return parser.parse_args()
def register_options(cls, parser):
group = OptionGroup(parser, 'Nagios specific options')
group.add_option('-w', '--warning', dest='warning')
group.add_option('-c', '--critical', dest='critical')
parser.add_option_group(group)
def optParse():
"""
Parses command line options.
Generally we're supporting all the command line options that doxypy.py
supports in an analogous way to make it easy to switch back and forth.
We additionally support a top-level namespace argument that is used
to trim away excess path information.
"""
parser = OptionParser(prog=basename(argv[0]))
parser.set_usage("%prog [options] filename")
parser.add_option(
"-a", "--autobrief",
action="store_true", dest="autobrief",
help="parse the docstring for @brief description and other information"
)
parser.add_option(
"-c", "--autocode",
action="store_true", dest="autocode",
help="parse the docstring for code samples"
)
parser.add_option(
"-n", "--ns",
action="store", type="string", dest="topLevelNamespace",
help="specify a top-level namespace that will be used to trim paths"
)
parser.add_option(
"-t", "--tablength",
action="store", type="int", dest="tablength", default=4,
help="specify a tab length in spaces; only needed if tabs are used"
)
group = OptionGroup(parser, "Debug Options")
group.add_option(
"-d", "--debug",
action="store_true", dest="debug",
help="enable debug output on stderr"
)
parser.add_option_group(group)
## Parse options based on our definition.
(options, filename) = parser.parse_args()
# Just abort immediately if we are don't have an input file.
if not filename:
stderr.write("No filename given." + linesep)
sysExit(-1)
# Turn the full path filename into a full path module location.
fullPathNamespace = filename[0].replace(sep, '.')[:-3]
# Use any provided top-level namespace argument to trim off excess.
realNamespace = fullPathNamespace
if options.topLevelNamespace:
namespaceStart = fullPathNamespace.find(options.topLevelNamespace)
if namespaceStart >= 0:
realNamespace = fullPathNamespace[namespaceStart:]
options.fullPathNamespace = realNamespace
return options, filename[0]
def run(self, argv):
parser = OptionParser(usage="%prog [options] [mode] [project]",
description='The jscc is a tool that helps you compile js code using google closure compiler.')
mode_group = OptionGroup(parser, "Mode options (only specify one)")
mode_group.add_option('-c', '--create', action='store_const', dest='mode', const='create', help='Create a new project.')
mode_group.add_option('-u', '--update', action='store_const', dest='mode', const='update', help='Update the project. This is default mode.')
mode_group.add_option('-w', '--watch', action='store_const', dest='mode', const='watch', help='Monitor the project for changes and update.')
parser.add_option_group(mode_group)
parser.add_option('--compiler', default='closure-compiler', help='Path to the google closure compiler. By default used: closure-compiler (see INSTALL for more details)')
parser.add_option('-f', '--force', dest='force', action='store_true', help='Force recompile the project.')
parser.add_option('-D', '--debug-mode', dest='debug_mode', action='store_true', help='Do not compile files. Just concatinate them.')
parser.add_option('-d', '--debug', action='store_true', help='Display debug output')
(options, args) = parser.parse_args(argv)
if len(args) > 2:
parser.error("incorrect number of arguments")
self.compiler = options.compiler
self.debug_mode = options.debug_mode or False
force = options.force or False
if options.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
mode = options.mode or 'update'
project = args[1] if len(args) > 1 else DEFAULT_PROJECT_FILENAME
getattr(self, mode)(project, force=force)
def setup_parser(self):
parser = ModTool.setup_parser(self)
ogroup = OptionGroup(parser, "Rename module options")
ogroup.add_option("-o", "--old-name", type="string", default=None, help="Current name of the block to rename.")
ogroup.add_option("-u", "--new-name", type="string", default=None, help="New name of the block.")
parser.add_option_group(ogroup)
return parser
def run(parser, rhn, logger):
parser.add_satellite_options()
parser.set_required(["sat_server", "sat_username", "sat_password", "cloneset"])
group = OptionGroup(parser.parser, "Show Options")
group.add_option("-c", "--cloneset", action="store", type="string", dest="cloneset", help="Cloneset")
parser.add_group(group)
(options, args) = parser.parse()
rhn = Satellite(options.sat_server, options.sat_username, options.sat_password)
try:
cloneset = rhn.cloneset_info(options.cloneset)
except KeyError:
print "Cloneset not found: %s" % options.cloneset
sys.exit()
print ""
print "Selected cloneset: " + options.cloneset
print ""
show.run(parser, rhn, logger)
response = raw_input("Are you sure? (This can't be undone) [Y/n] ")
if response.rstrip() == "Y":
cloneset.delete()
else:
print "Aborted."
def add_opts(self, parser, module_list = None):
if not module_list:
module_list = ['general', 'logging']
else:
module_list = ['general', 'logging'] + module_list
for m in self.dataseq:
if not m[0] in module_list:
continue
options = self.getval(m[0], 'options')
if not options: options = []
else: options = options.split(',')
group = None
if len(options):
group = OptionGroup(parser, m[0])
for k, v in m[1]:
if k[-5:] == '_desc':
continue
if not (k in options):
continue
desc = self.getval(m[0], k + '_desc')
if not desc:
desc = 'Refer to Idlak voice build documentation'
if v == 'True' or v == 'False':
#print v, k
group.add_option("--" + k, dest=k, action="store_true",
help=desc)
else:
#print v, k
group.add_option("--" + k, dest=k, default=None,
help=desc)
if group:
parser.add_option_group(group)
def test_help_description_groups(self):
self.parser.set_description(
"This is the program description for %prog. %prog has " "an option group as well as single options."
)
group = OptionGroup(
self.parser,
"Dangerous Options",
"Caution: use of these options is at your own risk. " "It is believed that some of them bite.",
)
group.add_option("-g", action="store_true", help="Group option.")
self.parser.add_option_group(group)
self.assertHelpEquals(
"""\
usage: bar.py [options]
This is the program description for bar.py. bar.py has an option group as
well as single options.
options:
-a APPLE throw APPLEs at basket
-b NUM, --boo=NUM shout "boo!" NUM times (in order to frighten away all the
evil spirits that cause trouble and mayhem)
--foo=FOO store FOO in the foo list for later fooing
-h, --help show this help message and exit
Dangerous Options:
Caution: use of these options is at your own risk. It is believed
that some of them bite.
-g Group option.
"""
)
def parse_options():
description = \
"""This example script gather information provided by a SAP Netweaver
Application Server with the Message Server protocol using the Dump
command. Execution of the dump commands require accessing the internal
Message Server port.
"""
epilog = \
"""pysap - http://corelabs.coresecurity.com/index.php?module=Wiki&action=view&type=tool&name=pysap"""
usage = "Usage: %prog [options] -d <remote host>"
parser = OptionParser(usage=usage, description=description, epilog=epilog)
target = OptionGroup(parser, "Target")
target.add_option("-d", "--remote-host", dest="remote_host", help="Remote host")
target.add_option("-p", "--remote-port", dest="remote_port", type="int", help="Remote port [%default]", default=3900)
parser.add_option_group(target)
misc = OptionGroup(parser, "Misc options")
misc.add_option("-c", "--client", dest="client", default="pysap's-dumper", help="Client name [%default]")
misc.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, help="Verbose output [%default]")
parser.add_option_group(misc)
(options, _) = parser.parse_args()
if not options.remote_host:
parser.error("Remote host is required")
return options
def _create_parser(programs):
description = "pyMonteCarlo Command Line Interface. Runs simulation(s) " + \
"with different Monte Carlo programs from the same interface." + \
"After the simulations, the results are automatically saved " + \
"in the output directory."
epilog = "For more information, see http://pymontecarlo.bitbucket.org"
parser = OptionParser(usage="%prog [options] [OPTION_FILE...]",
description=description, epilog=epilog)
# Base options
parser.add_option("-o", '--outputdir', dest="outputdir", default=os.getcwd(), metavar="DIR",
help="Output directory where results from simulation(s) will be saved [default: current directory]")
parser.add_option("-w", '--workdir', dest="workdir", default=None, metavar="DIR",
help="Work directory where temporary files from simulation(s) will be stored [default: temporary folder]")
parser.add_option('-v', '--verbose', dest='verbose', default=False,
action='store_true', help='Debug mode')
parser.add_option('-q', '--quiet', dest='quiet', default=False,
action='store_true', help='Quite mode (no progress bar is shown)')
parser.add_option("-n", dest="nbprocesses", default=1, type="int",
help="Number of processes/threads to use (not applicable for all Monte Carlo programs) [default: %default]")
parser.add_option('-s', '--skip', dest='skip', default=False,
action='store_true', help='Skip simulation if results already exist')
# Program group
group = OptionGroup(parser, "Monte Carlo programs",
"Note: Only supported programs are shown.")
for alias in sorted(map(attrgetter('alias'), programs)):
group.add_option('--%s' % alias, dest=alias, action="store_true")
parser.add_option_group(group)
return parser
def makeFlaskOpts(parser):
flaskOpts = OptionGroup(parser, 'Flask Options')
flaskOpts.add_option('--public', action='store_true', dest='public', default=False,
help='By default a flask server is not accessible from any external ip address, if this is set, the Flask web server will accept connections from external addresses, can be dangerous if used in conjunction with the debug mode.')
return flaskOpts
def _options_builder(self):
self._options_parser = OptionParser()
self._options_parser.add_option(
"--config", help="Set options from JSON file (generate example by --gen-conf).",
action="callback", type="string", callback=self._load_config, metavar="file".upper()
)
for section, keys in self._data_dict.items():
group = OptionGroup(
title=self._sections_help[section],
parser=self._options_parser,
description=self._sections_descriptions[section]
)
for key, params in keys.items():
self.__validators[self._key(section, key)] = params.pop("validator")
group.add_option(*params.pop('keys'), **params)
self._options_parser.add_option_group(group)
self._options_parser.add_option(
"--gen-conf", "--gen-config", help="Print sample config file and exit.",
action="callback", callback=self._gen_conf
)
return self._options_parser
def add_options(self, parser):
group = OptionGroup(parser, "'upgrade' command",
self.__doc__.strip())
group.add_option("--no-db-upgrade", action="store_false",
dest="upgrade_db", default=True,
help="don't upgrade the database")
parser.add_option_group(group)
def fwrapc(argv):
"""
Main entry point -- called by cmdline script.
"""
subcommands = ('configure', 'gen', 'build')
parser = OptionParser()
parser.add_option('--version', dest="version",
action="store_true", default=False,
help="get version and license info and exit")
# configure options
configure_opts = OptionGroup(parser, "Configure Options")
configure_opts.add_option("--name",
help='name for the extension module [default %default]')
configure_opts.add_option("--outdir",
help='directory for the intermediate files [default %default]')
parser.add_option_group(configure_opts)
conf_defaults = dict(name=PROJECT_NAME, outdir=PROJECT_OUTDIR)
parser.set_defaults(**conf_defaults)
opts, args = parser.parse_args(args=argv)
if opts.version:
print_version()
return 0
if not ('configure' in args or 'build' in args):
parser.print_usage()
return 1
return call_waf(opts, args, argv)
def build_parser():
usage = '%prog [OPTION...] MSGSTR_FILE MSGID_FILE...'
p = OptionParser(usage=usage,
description='Combine msgstrs from MSGSTR_FILE with '
'msgids from MSGID_FILEs where applicable, printing '
'the resulting catalog to stdout. The result is '
'guaranteed to be template-compatible with MSGID_FILE.')
p.add_option('--msgmerge', action='store_true',
help='use msgmerge for fuzzy matching.')
mode_opts = OptionGroup(p, 'Merge modes')
modes = ['left', 'right', 'translationproject']
#, 'launchpad', 'versionport']
text = dict(left='prefer translations from MSGSTR_FILE in conflicts '
'(default).',
right='prefer translations from MSGID_FILE in conflicts.',
translationproject='merge MSGSTR_FILE into all MSGID_FILES '
'and write to separate files. Useful for merging to many '
'older versions on translationproject.org',
launchpad='transfer as many strings as possible, '
'keeping header of MSGID_FILE '
'compatible with upload to Launchpad.',
versionport='transfer as many strings as possible, '
'but update header to contain ... blahblah') # XXX
for mode in modes:
mode_opts.add_option('--%s' % mode, action='store_const',
default=modes[0],
dest='mode', const=mode, help=text[mode])
p.add_option_group(mode_opts)
# option: translation project version merge mode
# option: launchpad upload prepare mode
#p.add_option('--keep-header', action='store_true',
# help='')
return p
def main():
try:
print banner()
parser = OptionParser(usage="usage %prog [options]", version="%prog 1.0")
parameters = OptionGroup(parser, "Options")
parameters.add_option("-t", "--target", type="string", help="Specify target as 'target' or 'target:port' (port 22 is default)", dest="target")
parameters.add_option("-l", "--target-list", type="string", help="File with targets: 'target' or 'target:port' seperated by a newline (port 22 is default)", dest="targetlist")
parser.add_option_group(parameters)
options, arguments = parser.parse_args()
target = options.target
targetlist = options.targetlist
if target:
parse_target(target)
else:
if targetlist:
list_parser(targetlist)
else:
print "[-] No target specified!"
sys.exit(0)
except KeyboardInterrupt:
print "\n[-] ^C Pressed, quitting!"
sys.exit(3)
def init_cmd_options(self):
super(MyProjectPlugin,self).init_cmd_options()
group = OptionGroup(self._cmd_parser, 'Specific to my project')
group.add_option('-c', action='store', dest='db_json_file', metavar="FILE",
help='override default path to the db.json file')
self._cmd_parser.add_option_group(group)
def parse_args():
parser = OptionParser(usage="%prog [options] MAP.level")
parser.add_option("--video", dest="video", metavar="FRAMEWORK",
choices=["pygame", "pyglet"],
help='use either "pygame" or "pyglet" framework '
'for video')
pygame = OptionGroup(parser, "Pygame video options",
"Only used when Pygame is selected as the "
"video framework.")
pygame.add_option("--sdl-videodriver", dest="videodriver", metavar="DRIVER",
help='override SDL_VIDEODRIVER environment variable')
pygame.add_option("--sdl-hwsurface", dest="hwsurface", action="store_true",
help='use hardware accelerated surfaces')
parser.add_option_group(pygame)
(options, args) = parser.parse_args()
# TODO: The real game should not have any required positional args
try:
map_file = args[0]
except IndexError:
parser.error("missing map file")
return (options, map_file)
def AddOptions(parser):
group = OptionGroup(parser, "S7 scanner options")
group.add_option("--src-tsap", help="Try this src-tsap (list) (default: 0x100,0x200)",
type="string", metavar="LIST")
group.add_option("--dst-tsap", help="Try this dst-tsap (list) (default: 0x102,0x200,0x201)",
type="string", metavar="LIST")
parser.add_option_group(group)
parser.add_option('-r',
'--run',
dest='validation_run',
help='ask for validation run',
default=False,
action="store_true")
parser.add_option('-j',
dest='speed',
help='set -j for make',
default=num_threads)
parser.add_option('--ispc-build-compiler',
dest='ispc_build_compiler',
help='set compiler to build ispc binary (clang/gcc)',
default="clang")
# options for activity "build LLVM"
llvm_group = OptionGroup(parser, "Options for building LLVM",
"These options must be used with -b option.")
llvm_group.add_option(
'--version',
dest='version',
help=
'version of llvm to build: 3.2 3.3 3.4 3.5 3.6 trunk. Default: trunk',
default="trunk")
llvm_group.add_option(
'--with-gcc-toolchain',
dest='gcc_toolchain_path',
help=
'GCC install dir to use when building clang. It is important to set when '
+
'you have alternative gcc installation. Note that otherwise gcc from standard '
+ 'location will be used, not from your PATH',
default="")
if __name__ == '__main__': # parse command line options, and set the high level properties
from optparse import OptionParser, OptionGroup
parser = OptionParser()
parser.add_option('-d',
'--verbose',
dest='verbose',
default=False,
action='store_true',
help='enable debug for all modules')
parser.add_option('--test',
dest='test',
default=False,
action='store_true',
help='run any tests and exit')
group1 = OptionGroup(
parser, 'Registrar',
'Use these options to specify registrar options such as listening address and hosted domains. A hosted domain is the host portion of the URI for which this registrar will accept registrations. Any domain that is not hosted here is treated as foreign domain.'
)
group1.add_option(
'-t',
'--transport',
dest='transport',
default=[],
action='append',
metavar='TYPE',
help=
'transport type is "udp", "tcp", "tls", "ws" or "wss". This can appear multiple times, e.g., "-t udp -t tcp". Default is "udp"'
)
group1.add_option(
'-l',
'--local',
dest='local',
def configure_options(self):
"""Define arguments for this script. Called from :py:meth:`__init__()`.
Run ``python -m mrjob.job.MRJob --help`` to see all options.
Re-define to define custom command-line arguments::
def configure_options(self):
super(MRYourJob, self).configure_options
self.add_passthrough_option(...)
self.add_file_option(...)
...
"""
self.option_parser.add_option(
'--help', dest='help_main', action='store_true', default=False,
help='show this message and exit')
self.option_parser.add_option(
'--help-emr', dest='help_emr', action='store_true', default=False,
help='show EMR-related options')
self.option_parser.add_option(
'--help-hadoop', dest='help_hadoop', action='store_true',
default=False,
help='show Hadoop-related options')
self.option_parser.add_option(
'--help-local', dest='help_local', action='store_true',
default=False,
help='show local/inline runner-related options')
self.option_parser.add_option(
'--help-runner', dest='help_runner', action='store_true',
default=False, help='show runner-related options')
# protocol stuff
self.proto_opt_group = OptionGroup(
self.option_parser, 'Protocols')
self.option_parser.add_option_group(self.proto_opt_group)
_add_protocol_opts(self.proto_opt_group)
# options for running the entire job
self.runner_opt_group = OptionGroup(
self.option_parser, 'Running the entire job')
self.option_parser.add_option_group(self.runner_opt_group)
_add_runner_opts(self.runner_opt_group, self._DEFAULT_RUNNER)
_add_basic_opts(self.runner_opt_group)
# options for inline/local runners
self.local_opt_group = OptionGroup(
self.option_parser,
'Running locally (these apply when you set -r inline or -r local)')
self.option_parser.add_option_group(self.local_opt_group)
_add_local_opts(self.local_opt_group)
# options common to Hadoop and EMR
self.hadoop_emr_opt_group = OptionGroup(
self.option_parser,
'Running on Hadoop or EMR (these apply when you set -r hadoop or'
' -r emr)')
self.option_parser.add_option_group(self.hadoop_emr_opt_group)
_add_hadoop_emr_opts(self.hadoop_emr_opt_group)
# options for running the job on Hadoop
self.hadoop_opt_group = OptionGroup(
self.option_parser,
'Running on Hadoop (these apply when you set -r hadoop)')
self.option_parser.add_option_group(self.hadoop_opt_group)
_add_hadoop_opts(self.hadoop_opt_group)
# options for running the job on EMR
self.emr_opt_group = OptionGroup(
self.option_parser,
'Running on EMR (these apply when you set -r emr)')
self.option_parser.add_option_group(self.emr_opt_group)
_add_emr_opts(self.emr_opt_group)
def getOptionParser():
usage = "usage: %prog [options] job-url-or-path"
parser = OptionParser(usage=usage)
parser.add_option("-v", "--version", action="store_true",
help="Print version and exit")
prodEnvOptionGroup = OptionGroup(parser, "Product Env Options",
"Normal user use these options to set jvm parameters, job runtime mode etc. "
"Make sure these options can be used in Product Env.")
prodEnvOptionGroup.add_option("-j", "--jvm", metavar="<jvm parameters>", dest="jvmParameters", action="store",
help="Set jvm parameters if necessary.")
prodEnvOptionGroup.add_option("--jobid", metavar="<job unique id>", dest="jobid", action="store", default="-1",
help="Set job unique id when running by Distribute/Local Mode.")
prodEnvOptionGroup.add_option("-m", "--mode", metavar="<job runtime mode>",
action="store", default="standalone",
help="Set job runtime mode such as: standalone, local, distribute. "
"Default mode is standalone.")
prodEnvOptionGroup.add_option("-p", "--params", metavar="<parameter used in job config>",
action="store", dest="params",
help='Set job parameter, eg: the source tableName you want to set it by command, '
'then you can use like this: -p"-DtableName=your-table-name", '
'if you have mutiple parameters: -p"-DtableName=your-table-name -DcolumnName=your-column-name".'
'Note: you should config in you job tableName with ${tableName}.')
prodEnvOptionGroup.add_option("-r", "--reader", metavar="<parameter used in view job config[reader] template>",
action="store", dest="reader", type="string",
help='View job config[reader] template, eg: mysqlreader,streamreader')
prodEnvOptionGroup.add_option("-w", "--writer", metavar="<parameter used in view job config[writer] template>",
action="store", dest="writer", type="string",
help='View job config[writer] template, eg: mysqlwriter,streamwriter')
prodEnvOptionGroup.add_option("-l", "--logdir", metavar="<log directory>",
action="store", dest="logdir", type="string",
help="the directory which log writes to",
default=DATAX_HOME + '/log')
parser.add_option_group(prodEnvOptionGroup)
devEnvOptionGroup = OptionGroup(parser, "Develop/Debug Options",
"Developer use these options to trace more details of DataX.")
devEnvOptionGroup.add_option("-d", "--debug", dest="remoteDebug", action="store_true",
help="Set to remote debug mode.")
devEnvOptionGroup.add_option("--loglevel", metavar="<log level>", dest="loglevel", action="store",
default="info", help="Set log level such as: debug, info, all etc.")
parser.add_option_group(devEnvOptionGroup)
return parser
def main():
print ("kosgen, Part of k_os (Konnex Operating-System based on the OSEK/VDX-Standard).")
usage = 'usage: %prog [options] oil_file'
options = []
args = []
op = OptionParser(usage = usage, version = '%prog ' + __version__,
description = "Generate K-OS-Configuration from '.oil'-File."
)
op.add_option('-f', '--command-file', type = 'string', metavar = 'FILE', dest = 'command_file',
help = 'read options from command-FILE'
)
input_group = OptionGroup(op, 'Input')
input_group.add_option('-I', '--include-path', dest = 'inc_path', action = 'append',
metavar = 'dir',
help = """Add directory to the list of directories to be searched for include files.
Environment-Variable 'KOS_INCLUDE' is also used to locate Include-Files."""
)
op.add_option_group(input_group)
group = OptionGroup(op, 'Output')
group.add_option('-o', '--orti', help = 'generate orti-FILE', dest = 'orti', action = 'store_true', default = False)
group.add_option('-r', '--resource-usage', help = 'generate resource statistics', dest = 'res_usage',
action = 'store_true', default = False)
group.add_option('-t', '--test', help = "verify only, don't generate anything", dest = 'test',
action = 'store_true', default = False)
group.add_option('-V', '--verbose', help = 'print Information messages', dest = 'verbose',
action = 'store_true', default = False)
group.add_option('-S', '--silent', help = "don't print any messages.", dest = 'silent', action = 'store_true',
default = False)
## keep immediate file.
op.add_option_group(group)
(options, args) = op.parse_args()
SetIncludePaths(options.inc_path)
if len(args) == 0:
op.error('no input file')
if len(args) != 1:
op.error('incorrect number of arguments')
error = Logger()
logger = logging.getLogger('kos.oil.logger')
if options.verbose:
logger.setLevel(logging.DEBUG)
elif options.silent:
logger.setLevel(logging.CRITICAL)
else:
logger.setLevel(logging.WARNING)
inFile = open(args[0])
outFilename = os.path.splitext(os.path.abspath(args[0]))[0] + '.i'
outFile = open(outFilename, 'w')
Preproc.Preproc(inFile, outFile)
outFile.close()
try:
inFile = open(outFilename)
except IOError:
error.fatalError("Could not open file '%s'.\n" % outFilename)
else:
(implDefMap, appDefMap, info) = Parse(outFilename)
info.version = __version__
if options.test == False:
if error.errorCounter > 0:
error.error('\n%u Error(s) occured during parsing, generating nothing.'
% error.errorCounter)
else:
GenCfg.Generate(os.path.splitext(args[0])[0], appDefMap, info)
def define_options(self, option_parser):
opt_group = OptionGroup(option_parser, 'Full Refresh Manager Options')
opt_group.add_option(
'--cluster',
dest='cluster',
default='refresh_primary',
help='Required: Specifies table cluster (default: %default).')
opt_group.add_option(
'--database',
dest='database',
help='Specify the database to switch to after connecting to the '
'cluster.')
opt_group.add_option(
'--config-path',
dest='config_path',
default='/nail/srv/configs/data_pipeline_tools.yaml',
help='Config file path for the refresh manager. '
'(default: %default)')
opt_group.add_option(
'--dry-run',
action="store_true",
default=False,
dest="dry_run",
help="Will execute all refreshes as dry runs, will still affect "
"the schematizer's records. (default: %default)")
opt_group.add_option(
'--per-source-throughput-cap',
default=DEFAULT_PER_SOURCE_THROUGHPUT_CAP,
dest="per_source_throughput_cap",
help=
"The cap that each source within the namespace is given. Any source in this "
"namespace cannot have a throughput higher than this cap. (default: %default)"
)
opt_group.add_option(
'--total-throughput-cap',
default=DEFAULT_TOTAL_THROUGHPUT_CAP,
dest="total_throughput_cap",
help=
"The cap that the namespace gives in general. The cumulative running refreshes "
"cannot have a throughput higher than this number. (default: %default)"
)
return opt_group
def process_options(): """ Process options and check for required values. """ opt = OptionParser() opt.add_option( "-c", "--input-cache", help="Read triggers from the files in this cache." ) opt.add_option( "-v", "--verbose", action="store_true", help="Be verbose." ) veto_settings = OptionGroup( opt, "HVeto settings" ) veto_settings.add_option( "-i", "--instrument", help="Instrument against which to veto. Required." ) veto_settings.add_option( "-r", "--reference-channel", help="Channel against which to veto. Required." ) veto_settings.add_option( "-t", "--reference-triggers", help="File path to load reference triggers. Required." ) veto_settings.add_option( "-s", "--significance-threshold", type=float, default=15, help="Significance below which to terminate the rounds. Default is 15." ) veto_settings.add_option( "--snr-thresh", action="append", help="Add an SNR threshold to use in veto round. Can be given multiple times for different values. WARNING: This will override the default settings, *not* append to them." ) veto_settings.add_option( "--time-window", action="append", help="Add a time window to use in veto round. Can be given multiple times for different values. WARNING: This will override the default settings, *not* append to them." ) veto_settings.add_option( "-S", "--min-ref-snr", type=float, default=8, help="Minimum SNR threshold to load a trigger in the reference channel." ) # FIXME: Strictly speaking the ignore list is required because I'm not # sure what the function will do with out one? veto_settings.add_option( "-I", "--ignore-list", help="Text file, one channel per line with a list of channels to ignore when loading triggers." ) # FIXME: #veto_settings.add_option( "-C", "--ignore-channel", action="append", help="Ignore these channels. Given several times, will ignore several channels. Do not prepend instrument. E.g. -C LSC-DARM_CTRL." ) veto_settings.add_option( "--write-coinc", action="store_true", default=False, help="If set, output table will include coinc tables indicating which triggers were coincided in the process of execution." ) opt.add_option_group( veto_settings ) livetime_settings = OptionGroup( opt, "livetime settings" ) # FIXME: #livetime_settings.add_option( "-L", "--livetime-definer", action="append", help="Name of segment definer entry from which to draw live segments. See '-l' option. If none is indicated, use all segments. Provide several times for deveral different segment definers." ) livetime_settings.add_option( "-l", "--livetime-segments", help="File from which to parse livetime segments. Will assume, at first, a LIGOLW XML file with valid segment definer and segment tables. If this fails, will try segwizard format. Required." ) livetime_settings.add_option( "--segment-definer", help="In tandem with --livetime-segments will retrieve segments with this definer. If none is provided, all segments will be used. Note: this option is REQUIRED if querying a databse. Example: H1:DMT-SCIENCE:1 (version is required)" ) livetime_settings.add_option( "--segment-database", help="Query this URL for segments. Takes precedence over providing a file." ) livetime_settings.add_option( "--gps-start", type=int, help="GPS start of analysis." ) livetime_settings.add_option( "--gps-end", type=int, help="GPS end of analysis." ) opt.add_option_group( livetime_settings ) opts, args = opt.parse_args() if opts.instrument is None: print >>sys.stderr, "Instrument must be indicated." exit() if opts.reference_channel is None: print >>sys.stderr, "Reference channel must be indicated." exit() if opts.reference_triggers is None: print >>sys.stderr, "Reference triggers must be present." exit() if (opts.livetime_segments or opts.segment_database) is None: print >>sys.stderr, "Must provide livetime segments file or segment database location." exit() if opts.segment_database and (opts.segment_definer is None): print >>sys.stderr, "Must provide definer for segment database querying." exit() if len(args) == 0 and opts.input_cache is None: print >>sys.stderr, "Must provide input arguments or set --input-cache." exit() if opts.input_cache is not None: with open(opts.input_cache) as cache: c = Cache.fromfile(cache) args.extend( c.pfnlist() ) if opts.ignore_list is None: print >>sys.stderr, "Must provide a channel ignore list." exit() return opts, args
def cmdLineParser(argv=None):
"""
This function parses the command line parameters and arguments
"""
if not argv:
argv = sys.argv
checkSystemEncoding()
# Reference: https://stackoverflow.com/a/4012683 (Note: previously used "...sys.getfilesystemencoding() or UNICODE_ENCODING")
_ = getUnicode(os.path.basename(argv[0]), encoding=sys.stdin.encoding)
usage = "%s%s [options]" % ("python " if not IS_WIN else "", "\"%s\"" % _ if " " in _ else _)
parser = OptionParser(usage=usage)
try:
parser.add_option("--hh", dest="advancedHelp",
action="store_true",
help="Show advanced help message and exit")
parser.add_option("--version", dest="showVersion",
action="store_true",
help="Show program's version number and exit")
parser.add_option("-v", dest="verbose", type="int",
help="Verbosity level: 0-6 (default %d)" % defaults.verbose)
# Target options
target = OptionGroup(parser, "Target", "At least one of these "
"options has to be provided to define the target(s)")
target.add_option("-d", dest="direct", help="Connection string "
"for direct database connection")
target.add_option("-u", "--url", dest="url", help="Target URL (e.g. \"http://www.site.com/vuln.php?id=1\")")
target.add_option("-l", dest="logFile", help="Parse target(s) from Burp "
"or WebScarab proxy log file")
target.add_option("-x", dest="sitemapUrl", help="Parse target(s) from remote sitemap(.xml) file")
target.add_option("-m", dest="bulkFile", help="Scan multiple targets given "
"in a textual file ")
target.add_option("-r", dest="requestFile",
help="Load HTTP request from a file")
target.add_option("-g", dest="googleDork",
help="Process Google dork results as target URLs")
target.add_option("-c", dest="configFile",
help="Load options from a configuration INI file")
# Request options
request = OptionGroup(parser, "Request", "These options can be used "
"to specify how to connect to the target URL")
request.add_option("--method", dest="method",
help="Force usage of given HTTP method (e.g. PUT)")
request.add_option("--data", dest="data",
help="Data string to be sent through POST (e.g. \"id=1\")")
request.add_option("--param-del", dest="paramDel",
help="Character used for splitting parameter values (e.g. &)")
request.add_option("--cookie", dest="cookie",
help="HTTP Cookie header value (e.g. \"PHPSESSID=a8d127e..\")")
request.add_option("--cookie-del", dest="cookieDel",
help="Character used for splitting cookie values (e.g. ;)")
request.add_option("--load-cookies", dest="loadCookies",
help="File containing cookies in Netscape/wget format")
request.add_option("--drop-set-cookie", dest="dropSetCookie", action="store_true",
help="Ignore Set-Cookie header from response")
request.add_option("--user-agent", dest="agent",
help="HTTP User-Agent header value")
request.add_option("--random-agent", dest="randomAgent", action="store_true",
help="Use randomly selected HTTP User-Agent header value")
request.add_option("--host", dest="host",
help="HTTP Host header value")
request.add_option("--referer", dest="referer",
help="HTTP Referer header value")
request.add_option("-H", "--header", dest="header",
help="Extra header (e.g. \"X-Forwarded-For: 127.0.0.1\")")
request.add_option("--headers", dest="headers",
help="Extra headers (e.g. \"Accept-Language: fr\\nETag: 123\")")
request.add_option("--auth-type", dest="authType",
help="HTTP authentication type (Basic, Digest, NTLM or PKI)")
request.add_option("--auth-cred", dest="authCred",
help="HTTP authentication credentials (name:password)")
request.add_option("--auth-file", dest="authFile",
help="HTTP authentication PEM cert/private key file")
request.add_option("--ignore-code", dest="ignoreCode", type="int",
help="Ignore (problematic) HTTP error code (e.g. 401)")
request.add_option("--ignore-proxy", dest="ignoreProxy", action="store_true",
help="Ignore system default proxy settings")
request.add_option("--ignore-redirects", dest="ignoreRedirects", action="store_true",
help="Ignore redirection attempts")
request.add_option("--ignore-timeouts", dest="ignoreTimeouts", action="store_true",
help="Ignore connection timeouts")
request.add_option("--proxy", dest="proxy",
help="Use a proxy to connect to the target URL")
request.add_option("--proxy-cred", dest="proxyCred",
help="Proxy authentication credentials (name:password)")
request.add_option("--proxy-file", dest="proxyFile",
help="Load proxy list from a file")
request.add_option("--tor", dest="tor", action="store_true",
help="Use Tor anonymity network")
request.add_option("--tor-port", dest="torPort",
help="Set Tor proxy port other than default")
request.add_option("--tor-type", dest="torType",
help="Set Tor proxy type (HTTP, SOCKS4 or SOCKS5 (default))")
request.add_option("--check-tor", dest="checkTor", action="store_true",
help="Check to see if Tor is used properly")
request.add_option("--delay", dest="delay", type="float",
help="Delay in seconds between each HTTP request")
request.add_option("--timeout", dest="timeout", type="float",
help="Seconds to wait before timeout connection (default %d)" % defaults.timeout)
request.add_option("--retries", dest="retries", type="int",
help="Retries when the connection timeouts (default %d)" % defaults.retries)
request.add_option("--randomize", dest="rParam",
help="Randomly change value for given parameter(s)")
request.add_option("--safe-url", dest="safeUrl",
help="URL address to visit frequently during testing")
request.add_option("--safe-post", dest="safePost",
help="POST data to send to a safe URL")
request.add_option("--safe-req", dest="safeReqFile",
help="Load safe HTTP request from a file")
request.add_option("--safe-freq", dest="safeFreq", type="int",
help="Test requests between two visits to a given safe URL")
request.add_option("--skip-urlencode", dest="skipUrlEncode", action="store_true",
help="Skip URL encoding of payload data")
request.add_option("--csrf-token", dest="csrfToken",
help="Parameter used to hold anti-CSRF token")
request.add_option("--csrf-url", dest="csrfUrl",
help="URL address to visit for extraction of anti-CSRF token")
request.add_option("--force-ssl", dest="forceSSL", action="store_true",
help="Force usage of SSL/HTTPS")
request.add_option("--hpp", dest="hpp", action="store_true",
help="Use HTTP parameter pollution method")
request.add_option("--eval", dest="evalCode",
help="Evaluate provided Python code before the request (e.g. \"import hashlib;id2=hashlib.md5(id).hexdigest()\")")
# Optimization options
optimization = OptionGroup(parser, "Optimization", "These options can be used to optimize the performance of sqlmap")
optimization.add_option("-o", dest="optimize", action="store_true",
help="Turn on all optimization switches")
optimization.add_option("--predict-output", dest="predictOutput", action="store_true",
help="Predict common queries output")
optimization.add_option("--keep-alive", dest="keepAlive", action="store_true",
help="Use persistent HTTP(s) connections")
optimization.add_option("--null-connection", dest="nullConnection", action="store_true",
help="Retrieve page length without actual HTTP response body")
optimization.add_option("--threads", dest="threads", type="int",
help="Max number of concurrent HTTP(s) "
"requests (default %d)" % defaults.threads)
# Injection options
injection = OptionGroup(parser, "Injection", "These options can be used to specify which parameters to test for, provide custom injection payloads and optional tampering scripts")
injection.add_option("-p", dest="testParameter",
help="Testable parameter(s)")
injection.add_option("--skip", dest="skip",
help="Skip testing for given parameter(s)")
injection.add_option("--skip-static", dest="skipStatic", action="store_true",
help="Skip testing parameters that not appear to be dynamic")
injection.add_option("--param-exclude", dest="paramExclude",
help="Regexp to exclude parameters from testing (e.g. \"ses\")")
injection.add_option("--dbms", dest="dbms",
help="Force back-end DBMS to provided value")
injection.add_option("--dbms-cred", dest="dbmsCred",
help="DBMS authentication credentials (user:password)")
injection.add_option("--os", dest="os",
help="Force back-end DBMS operating system to provided value")
injection.add_option("--invalid-bignum", dest="invalidBignum", action="store_true",
help="Use big numbers for invalidating values")
injection.add_option("--invalid-logical", dest="invalidLogical", action="store_true",
help="Use logical operations for invalidating values")
injection.add_option("--invalid-string", dest="invalidString", action="store_true",
help="Use random strings for invalidating values")
injection.add_option("--no-cast", dest="noCast", action="store_true",
help="Turn off payload casting mechanism")
injection.add_option("--no-escape", dest="noEscape", action="store_true",
help="Turn off string escaping mechanism")
injection.add_option("--prefix", dest="prefix",
help="Injection payload prefix string")
injection.add_option("--suffix", dest="suffix",
help="Injection payload suffix string")
injection.add_option("--tamper", dest="tamper",
help="Use given script(s) for tampering injection data")
# Detection options
detection = OptionGroup(parser, "Detection", "These options can be used to customize the detection phase")
detection.add_option("--level", dest="level", type="int",
help="Level of tests to perform (1-5, default %d)" % defaults.level)
detection.add_option("--risk", dest="risk", type="int",
help="Risk of tests to perform (1-3, default %d)" % defaults.risk)
detection.add_option("--string", dest="string",
help="String to match when query is evaluated to True")
detection.add_option("--not-string", dest="notString",
help="String to match when query is evaluated to False")
detection.add_option("--regexp", dest="regexp",
help="Regexp to match when query is evaluated to True")
detection.add_option("--code", dest="code", type="int",
help="HTTP code to match when query is evaluated to True")
detection.add_option("--text-only", dest="textOnly", action="store_true",
help="Compare pages based only on the textual content")
detection.add_option("--titles", dest="titles", action="store_true",
help="Compare pages based only on their titles")
# Techniques options
techniques = OptionGroup(parser, "Techniques", "These options can be used to tweak testing of specific SQL injection techniques")
techniques.add_option("--technique", dest="tech",
help="SQL injection techniques to use (default \"%s\")" % defaults.tech)
techniques.add_option("--time-sec", dest="timeSec", type="int",
help="Seconds to delay the DBMS response (default %d)" % defaults.timeSec)
techniques.add_option("--union-cols", dest="uCols",
help="Range of columns to test for UNION query SQL injection")
techniques.add_option("--union-char", dest="uChar",
help="Character to use for bruteforcing number of columns")
techniques.add_option("--union-from", dest="uFrom",
help="Table to use in FROM part of UNION query SQL injection")
techniques.add_option("--dns-domain", dest="dnsDomain",
help="Domain name used for DNS exfiltration attack")
techniques.add_option("--second-url", dest="secondUrl",
help="Resulting page URL searched for second-order response")
techniques.add_option("--second-req", dest="secondReq",
help="Load second-order HTTP request from file")
# Fingerprint options
fingerprint = OptionGroup(parser, "Fingerprint")
fingerprint.add_option("-f", "--fingerprint", dest="extensiveFp", action="store_true",
help="Perform an extensive DBMS version fingerprint")
# Enumeration options
enumeration = OptionGroup(parser, "Enumeration", "These options can be used to enumerate the back-end database management system information, structure and data contained in the tables. Moreover you can run your own SQL statements")
enumeration.add_option("-a", "--all", dest="getAll", action="store_true",
help="Retrieve everything")
enumeration.add_option("-b", "--banner", dest="getBanner", action="store_true",
help="Retrieve DBMS banner")
enumeration.add_option("--current-user", dest="getCurrentUser", action="store_true",
help="Retrieve DBMS current user")
enumeration.add_option("--current-db", dest="getCurrentDb", action="store_true",
help="Retrieve DBMS current database")
enumeration.add_option("--hostname", dest="getHostname", action="store_true",
help="Retrieve DBMS server hostname")
enumeration.add_option("--is-dba", dest="isDba", action="store_true",
help="Detect if the DBMS current user is DBA")
enumeration.add_option("--users", dest="getUsers", action="store_true",
help="Enumerate DBMS users")
enumeration.add_option("--passwords", dest="getPasswordHashes", action="store_true",
help="Enumerate DBMS users password hashes")
enumeration.add_option("--privileges", dest="getPrivileges", action="store_true",
help="Enumerate DBMS users privileges")
enumeration.add_option("--roles", dest="getRoles", action="store_true",
help="Enumerate DBMS users roles")
enumeration.add_option("--dbs", dest="getDbs", action="store_true",
help="Enumerate DBMS databases")
enumeration.add_option("--tables", dest="getTables", action="store_true",
help="Enumerate DBMS database tables")
enumeration.add_option("--columns", dest="getColumns", action="store_true",
help="Enumerate DBMS database table columns")
enumeration.add_option("--schema", dest="getSchema", action="store_true",
help="Enumerate DBMS schema")
enumeration.add_option("--count", dest="getCount", action="store_true",
help="Retrieve number of entries for table(s)")
enumeration.add_option("--dump", dest="dumpTable", action="store_true",
help="Dump DBMS database table entries")
enumeration.add_option("--dump-all", dest="dumpAll", action="store_true",
help="Dump all DBMS databases tables entries")
enumeration.add_option("--search", dest="search", action="store_true",
help="Search column(s), table(s) and/or database name(s)")
enumeration.add_option("--comments", dest="getComments", action="store_true",
help="Check for DBMS comments during enumeration")
enumeration.add_option("-D", dest="db",
help="DBMS database to enumerate")
enumeration.add_option("-T", dest="tbl",
help="DBMS database table(s) to enumerate")
enumeration.add_option("-C", dest="col",
help="DBMS database table column(s) to enumerate")
enumeration.add_option("-X", dest="exclude",
help="DBMS database identifier(s) to not enumerate")
enumeration.add_option("-U", dest="user",
help="DBMS user to enumerate")
enumeration.add_option("--exclude-sysdbs", dest="excludeSysDbs", action="store_true",
help="Exclude DBMS system databases when enumerating tables")
enumeration.add_option("--pivot-column", dest="pivotColumn",
help="Pivot column name")
enumeration.add_option("--where", dest="dumpWhere",
help="Use WHERE condition while table dumping")
enumeration.add_option("--start", dest="limitStart", type="int",
help="First dump table entry to retrieve")
enumeration.add_option("--stop", dest="limitStop", type="int",
help="Last dump table entry to retrieve")
enumeration.add_option("--first", dest="firstChar", type="int",
help="First query output word character to retrieve")
enumeration.add_option("--last", dest="lastChar", type="int",
help="Last query output word character to retrieve")
enumeration.add_option("--sql-query", dest="query",
help="SQL statement to be executed")
enumeration.add_option("--sql-shell", dest="sqlShell", action="store_true",
help="Prompt for an interactive SQL shell")
enumeration.add_option("--sql-file", dest="sqlFile",
help="Execute SQL statements from given file(s)")
# Brute force options
brute = OptionGroup(parser, "Brute force", "These options can be used to run brute force checks")
brute.add_option("--common-tables", dest="commonTables", action="store_true",
help="Check existence of common tables")
brute.add_option("--common-columns", dest="commonColumns", action="store_true",
help="Check existence of common columns")
# User-defined function options
udf = OptionGroup(parser, "User-defined function injection", "These options can be used to create custom user-defined functions")
udf.add_option("--udf-inject", dest="udfInject", action="store_true",
help="Inject custom user-defined functions")
udf.add_option("--shared-lib", dest="shLib",
help="Local path of the shared library")
# File system options
filesystem = OptionGroup(parser, "File system access", "These options can be used to access the back-end database management system underlying file system")
filesystem.add_option("--file-read", dest="fileRead",
help="Read a file from the back-end DBMS file system")
filesystem.add_option("--file-write", dest="fileWrite",
help="Write a local file on the back-end DBMS file system")
filesystem.add_option("--file-dest", dest="fileDest",
help="Back-end DBMS absolute filepath to write to")
# Takeover options
takeover = OptionGroup(parser, "Operating system access", "These options can be used to access the back-end database management system underlying operating system")
takeover.add_option("--os-cmd", dest="osCmd",
help="Execute an operating system command")
takeover.add_option("--os-shell", dest="osShell", action="store_true",
help="Prompt for an interactive operating system shell")
takeover.add_option("--os-pwn", dest="osPwn", action="store_true",
help="Prompt for an OOB shell, Meterpreter or VNC")
takeover.add_option("--os-smbrelay", dest="osSmb", action="store_true",
help="One click prompt for an OOB shell, Meterpreter or VNC")
takeover.add_option("--os-bof", dest="osBof", action="store_true",
help="Stored procedure buffer overflow "
"exploitation")
takeover.add_option("--priv-esc", dest="privEsc", action="store_true",
help="Database process user privilege escalation")
takeover.add_option("--msf-path", dest="msfPath",
help="Local path where Metasploit Framework is installed")
takeover.add_option("--tmp-path", dest="tmpPath",
help="Remote absolute path of temporary files directory")
# Windows registry options
windows = OptionGroup(parser, "Windows registry access", "These options can be used to access the back-end database management system Windows registry")
windows.add_option("--reg-read", dest="regRead", action="store_true",
help="Read a Windows registry key value")
windows.add_option("--reg-add", dest="regAdd", action="store_true",
help="Write a Windows registry key value data")
windows.add_option("--reg-del", dest="regDel", action="store_true",
help="Delete a Windows registry key value")
windows.add_option("--reg-key", dest="regKey",
help="Windows registry key")
windows.add_option("--reg-value", dest="regVal",
help="Windows registry key value")
windows.add_option("--reg-data", dest="regData",
help="Windows registry key value data")
windows.add_option("--reg-type", dest="regType",
help="Windows registry key value type")
# General options
general = OptionGroup(parser, "General", "These options can be used to set some general working parameters")
general.add_option("-s", dest="sessionFile",
help="Load session from a stored (.sqlite) file")
general.add_option("-t", dest="trafficFile",
help="Log all HTTP traffic into a textual file")
general.add_option("--batch", dest="batch", action="store_true",
help="Never ask for user input, use the default behavior")
general.add_option("--binary-fields", dest="binaryFields",
help="Result fields having binary values (e.g. \"digest\")")
general.add_option("--check-internet", dest="checkInternet", action="store_true",
help="Check Internet connection before assessing the target")
general.add_option("--crawl", dest="crawlDepth", type="int",
help="Crawl the website starting from the target URL")
general.add_option("--crawl-exclude", dest="crawlExclude",
help="Regexp to exclude pages from crawling (e.g. \"logout\")")
general.add_option("--csv-del", dest="csvDel",
help="Delimiting character used in CSV output (default \"%s\")" % defaults.csvDel)
general.add_option("--charset", dest="charset",
help="Blind SQL injection charset (e.g. \"0123456789abcdef\")")
general.add_option("--dump-format", dest="dumpFormat",
help="Format of dumped data (CSV (default), HTML or SQLITE)")
general.add_option("--encoding", dest="encoding",
help="Character encoding used for data retrieval (e.g. GBK)")
general.add_option("--eta", dest="eta", action="store_true",
help="Display for each output the estimated time of arrival")
general.add_option("--flush-session", dest="flushSession", action="store_true",
help="Flush session files for current target")
general.add_option("--forms", dest="forms", action="store_true",
help="Parse and test forms on target URL")
general.add_option("--fresh-queries", dest="freshQueries", action="store_true",
help="Ignore query results stored in session file")
general.add_option("--har", dest="harFile",
help="Log all HTTP traffic into a HAR file")
general.add_option("--hex", dest="hexConvert", action="store_true",
help="Use hex conversion during data retrieval")
general.add_option("--output-dir", dest="outputDir", action="store",
help="Custom output directory path")
general.add_option("--parse-errors", dest="parseErrors", action="store_true",
help="Parse and display DBMS error messages from responses")
general.add_option("--save", dest="saveConfig",
help="Save options to a configuration INI file")
general.add_option("--scope", dest="scope",
help="Regexp to filter targets from provided proxy log")
general.add_option("--test-filter", dest="testFilter",
help="Select tests by payloads and/or titles (e.g. ROW)")
general.add_option("--test-skip", dest="testSkip",
help="Skip tests by payloads and/or titles (e.g. BENCHMARK)")
general.add_option("--update", dest="updateAll", action="store_true",
help="Update sqlmap")
# Miscellaneous options
miscellaneous = OptionGroup(parser, "Miscellaneous")
miscellaneous.add_option("-z", dest="mnemonics",
help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
miscellaneous.add_option("--alert", dest="alert",
help="Run host OS command(s) when SQL injection is found")
miscellaneous.add_option("--answers", dest="answers",
help="Set predefined answers (e.g. \"quit=N,follow=N\")")
miscellaneous.add_option("--beep", dest="beep", action="store_true",
help="Beep on question and/or when SQL injection is found")
miscellaneous.add_option("--cleanup", dest="cleanup", action="store_true",
help="Clean up the DBMS from sqlmap specific UDF and tables")
miscellaneous.add_option("--dependencies", dest="dependencies", action="store_true",
help="Check for missing (optional) sqlmap dependencies")
miscellaneous.add_option("--disable-coloring", dest="disableColoring", action="store_true",
help="Disable console output coloring")
miscellaneous.add_option("--gpage", dest="googlePage", type="int",
help="Use Google dork results from specified page number")
miscellaneous.add_option("--identify-waf", dest="identifyWaf", action="store_true",
help="Make a thorough testing for a WAF/IPS protection")
miscellaneous.add_option("--list-tampers", dest="listTampers", action="store_true",
help="Display list of available tamper scripts")
miscellaneous.add_option("--mobile", dest="mobile", action="store_true",
help="Imitate smartphone through HTTP User-Agent header")
miscellaneous.add_option("--offline", dest="offline", action="store_true",
help="Work in offline mode (only use session data)")
miscellaneous.add_option("--purge", dest="purge", action="store_true",
help="Safely remove all content from sqlmap data directory")
miscellaneous.add_option("--skip-waf", dest="skipWaf", action="store_true",
help="Skip heuristic detection of WAF/IPS protection")
miscellaneous.add_option("--smart", dest="smart", action="store_true",
help="Conduct thorough tests only if positive heuristic(s)")
miscellaneous.add_option("--sqlmap-shell", dest="sqlmapShell", action="store_true",
help="Prompt for an interactive sqlmap shell")
miscellaneous.add_option("--tmp-dir", dest="tmpDir",
help="Local directory for storing temporary files")
miscellaneous.add_option("--web-root", dest="webRoot",
help="Web server document root directory (e.g. \"/var/www\")")
miscellaneous.add_option("--wizard", dest="wizard", action="store_true",
help="Simple wizard interface for beginner users")
# Hidden and/or experimental options
parser.add_option("--crack", dest="hashFile",
help=SUPPRESS_HELP)
# help="Load and crack hashes from a file (standalone)")
parser.add_option("--dummy", dest="dummy", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--murphy-rate", dest="murphyRate", type="int",
help=SUPPRESS_HELP)
parser.add_option("--disable-precon", dest="disablePrecon", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--disable-stats", dest="disableStats", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--profile", dest="profile", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--force-dbms", dest="forceDbms",
help=SUPPRESS_HELP)
parser.add_option("--force-dns", dest="forceDns", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--force-pivoting", dest="forcePivoting", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--force-threads", dest="forceThreads", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--smoke-test", dest="smokeTest", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--live-test", dest="liveTest", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--stop-fail", dest="stopFail", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--run-case", dest="runCase", help=SUPPRESS_HELP)
# API options
parser.add_option("--api", dest="api", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--taskid", dest="taskid", help=SUPPRESS_HELP)
parser.add_option("--database", dest="database", help=SUPPRESS_HELP)
parser.add_option_group(target)
parser.add_option_group(request)
parser.add_option_group(optimization)
parser.add_option_group(injection)
parser.add_option_group(detection)
parser.add_option_group(techniques)
parser.add_option_group(fingerprint)
parser.add_option_group(enumeration)
parser.add_option_group(brute)
parser.add_option_group(udf)
parser.add_option_group(filesystem)
parser.add_option_group(takeover)
parser.add_option_group(windows)
parser.add_option_group(general)
parser.add_option_group(miscellaneous)
# Dirty hack to display longer options without breaking into two lines
def _(self, *args):
retVal = parser.formatter._format_option_strings(*args)
if len(retVal) > MAX_HELP_OPTION_LENGTH:
retVal = ("%%.%ds.." % (MAX_HELP_OPTION_LENGTH - parser.formatter.indent_increment)) % retVal
return retVal
parser.formatter._format_option_strings = parser.formatter.format_option_strings
parser.formatter.format_option_strings = type(parser.formatter.format_option_strings)(_, parser, type(parser))
# Dirty hack for making a short option '-hh'
option = parser.get_option("--hh")
option._short_opts = ["-hh"]
option._long_opts = []
# Dirty hack for inherent help message of switch '-h'
option = parser.get_option("-h")
option.help = option.help.capitalize().replace("this help", "basic help")
_ = []
prompt = False
advancedHelp = True
extraHeaders = []
tamperIndex = None
# Reference: https://stackoverflow.com/a/4012683 (Note: previously used "...sys.getfilesystemencoding() or UNICODE_ENCODING")
for arg in argv:
_.append(getUnicode(arg, encoding=sys.stdin.encoding))
argv = _
checkDeprecatedOptions(argv)
prompt = "--sqlmap-shell" in argv
if prompt:
parser.usage = ""
cmdLineOptions.sqlmapShell = True
_ = ["x", "q", "exit", "quit", "clear"]
for option in parser.option_list:
_.extend(option._long_opts)
_.extend(option._short_opts)
for group in parser.option_groups:
for option in group.option_list:
_.extend(option._long_opts)
_.extend(option._short_opts)
autoCompletion(AUTOCOMPLETE_TYPE.SQLMAP, commands=_)
while True:
command = None
try:
command = raw_input("sqlmap-shell> ").strip()
command = getUnicode(command, encoding=sys.stdin.encoding)
except (KeyboardInterrupt, EOFError):
print
raise SqlmapShellQuitException
if not command:
continue
elif command.lower() == "clear":
clearHistory()
dataToStdout("[i] history cleared\n")
saveHistory(AUTOCOMPLETE_TYPE.SQLMAP)
elif command.lower() in ("x", "q", "exit", "quit"):
raise SqlmapShellQuitException
elif command[0] != '-':
dataToStdout("[!] invalid option(s) provided\n")
dataToStdout("[i] proper example: '-u http://www.site.com/vuln.php?id=1 --banner'\n")
else:
saveHistory(AUTOCOMPLETE_TYPE.SQLMAP)
loadHistory(AUTOCOMPLETE_TYPE.SQLMAP)
break
try:
for arg in shlex.split(command):
argv.append(getUnicode(arg, encoding=sys.stdin.encoding))
except ValueError, ex:
raise SqlmapSyntaxException("something went wrong during command line parsing ('%s')" % ex.message)
for i in xrange(len(argv)):
if argv[i] == "-hh":
argv[i] = "-h"
elif len(argv[i]) > 1 and all(ord(_) in xrange(0x2018, 0x2020) for _ in ((argv[i].split('=', 1)[-1].strip() or ' ')[0], argv[i][-1])):
dataToStdout("[!] copy-pasting illegal (non-console) quote characters from Internet is, well, illegal (%s)\n" % argv[i])
raise SystemExit
elif len(argv[i]) > 1 and u"\uff0c" in argv[i].split('=', 1)[-1]:
dataToStdout("[!] copy-pasting illegal (non-console) comma characters from Internet is, well, illegal (%s)\n" % argv[i])
raise SystemExit
elif re.search(r"\A-\w=.+", argv[i]):
dataToStdout("[!] potentially miswritten (illegal '=') short option detected ('%s')\n" % argv[i])
raise SystemExit
elif argv[i].startswith("--tamper"):
if tamperIndex is None:
tamperIndex = i if '=' in argv[i] else (i + 1 if i + 1 < len(argv) and not argv[i + 1].startswith('-') else None)
else:
argv[tamperIndex] = "%s,%s" % (argv[tamperIndex], argv[i].split('=')[1] if '=' in argv[i] else (argv[i + 1] if i + 1 < len(argv) and not argv[i + 1].startswith('-') else ""))
argv[i] = ""
elif argv[i] == "-H":
if i + 1 < len(argv):
extraHeaders.append(argv[i + 1])
elif re.match(r"\A\d+!\Z", argv[i]) and argv[max(0, i - 1)] == "--threads" or re.match(r"\A--threads.+\d+!\Z", argv[i]):
argv[i] = argv[i][:-1]
conf.skipThreadCheck = True
elif argv[i] == "--version":
print VERSION_STRING.split('/')[-1]
raise SystemExit
elif argv[i] in ("-h", "--help"):
advancedHelp = False
for group in parser.option_groups[:]:
found = False
for option in group.option_list:
if option.dest not in BASIC_HELP_ITEMS:
option.help = SUPPRESS_HELP
else:
found = True
if not found:
parser.option_groups.remove(group)
for verbosity in (_ for _ in argv if re.search(r"\A\-v+\Z", _)):
try:
if argv.index(verbosity) == len(argv) - 1 or not argv[argv.index(verbosity) + 1].isdigit():
conf.verbose = verbosity.count('v') + 1
del argv[argv.index(verbosity)]
except (IndexError, ValueError):
pass
try:
(args, _) = parser.parse_args(argv)
except UnicodeEncodeError, ex:
dataToStdout("\n[!] %s\n" % ex.object.encode("unicode-escape"))
raise SystemExit
This program provides basic genome-wide association (GWAS) functionality.
You provide a phenotype and genotype file as well as a pre-computed (use pylmmKinship.py)
kinship matrix and the program outputs a result file with information about each SNP, including the association p-value.
The input file are all standard plink formatted with the first two columns specifying the individual and family ID.
For the phenotype file, we accept either NA or -9 to denote missing values.
For Ancestry, assumes Ancestry information is in the last column of the covariate file
Basic usage:
python pylmmGxTheta.py -v --bfile plinkFile --kfile preComputedKinship.kin --phenofile plinkFormattedPhenotypeFile --covfile covariatefile --Ancestry resultFile
"""
parser = OptionParser(usage=usage)
basicGroup = OptionGroup(parser, "Basic Options")
advancedGroup = OptionGroup(parser, "Advanced Options")
experimentalGroup = OptionGroup(parser, "Experimental Options")
annotationGroup = OptionGroup(parser, "Annotation Options")
GxEGroup = OptionGroup(parser, "GxE Options")
GxAGroup = OptionGroup(parser, "GxA Options")
#basicGroup.add_option("--pfile", dest="pfile",
# help="The base for a PLINK ped file")
basicGroup.add_option("--tfile",
dest="tfile",
help="The base for a PLINK tped file")
basicGroup.add_option("--bfile",
dest="bfile",
help="The base for a PLINK binary bed file")
basicGroup.add_option(
import pandas as pd
from Bio import SeqIO, pairwise2
import os
import sys
from optparse import OptionParser, OptionGroup
if __name__ == "__main__":
parser = OptionParser("Cut the fastq sequences into single size.")
# REQUIRED
group_reqd = OptionGroup(parser, "Required flags", "")
group_reqd.add_option(
"-f",
"--fastq_path",
type="string",
default=
"/home/vered/EMIRGE/data/s26_mock/dada2/SampRS26_rev_fasta_mod.fastq",
help="path to output new database location")
group_reqd.add_option("-l",
"--length",
type="int",
default=145,
help="length of sequence")
parser.add_option_group(group_reqd)
(options, args) = parser.parse_args(sys.argv[1:])
new_fastq_path = "{}_{}{}".format(
options.fastq_path.split('.fastq')[0], options.length, ".fastq")
with open(options.fastq_path, 'r') as f_i:
#
import sys, subprocess, os, time, hashlib
from optparse import OptionParser, OptionGroup
######################
# User Interface
######################
# Setup command line options and arguments
parser = OptionParser(usage='Usage: python %prog [argument(s)] -o <filename>')
parser.add_option(
'-o',
help='specified file will contain the results of RapidTriage',
dest='outfile')
group1 = OptionGroup(parser, "Arguments (one or more system areas required)")
group1.add_option('-a',
'--all_areas',
action='store_true',
help='collect information from all areas',
dest='allchecks')
group1.add_option('-f',
'--filesystem',
action='store_true',
help='collect filesystem related information',
dest='filesystem')
group1.add_option('-l',
'--log_events',
action='store_true',
help='collect histories and log data',
dest='logs')
def _options():
global modules
parser = OptionParser(usage="usage: %prog [options]")
parser.add_option("-g",
"--graph",
action="store_true",
dest="graph",
default=False,
help="Generate graph. default: %default")
parser.add_option("-G",
"--show-graph",
action="store_true",
dest="show_graph",
default=False,
help="Display the graph (implies -g). default: %default")
parser.add_option("-X",
"--check",
action="store_true",
dest="validate",
default=False,
help="Validate results against perfect sine.\t"
"default: %default")
parser.add_option("-l",
"--std-limit",
action="store",
type="float",
dest="std_limit",
metavar="LIMIT",
default=3.0,
help="Standard error limit in % of the results.\t"
"default: %default%")
parser.add_option("-L",
"--max-limit",
action="store",
type="float",
dest="max_limit",
metavar="LIMIT",
default=10.0,
help="Maximum error limit in % of the results.\t"
"default: %default%")
parser.add_option("-O",
"--output",
action="store",
type="string",
dest="output",
metavar="DIR",
default="",
help="Where to write test results.\t\t"
"default: None")
parser.add_option("-A",
"--autoname",
action="store_true",
dest="autoname",
default=False,
help="Automatically create subfolder for test results.\t"
"Format: YYYYMMDD-hhmmss-variant-serial")
parser.add_option("-P",
"--print-options",
action="store_true",
dest="print_options",
default=False,
help="Print command line options in the output.\t\t"
"default: %default")
group = OptionGroup(parser, "Device Options")
group.add_option("-d",
"--drivers",
action="store",
type="string",
dest="drivers",
metavar="DRIVERS",
default=",".join(modules),
help="Comma separated list of drivers.\t\t"
"default: All supported")
group.add_option("-v",
"--variant",
action="store",
type="string",
dest="variant",
metavar="VARIANT",
default="",
help="String defining variant to open.\t\t"
"default: None (open any)")
group.add_option("-b",
"--serial",
"--batch",
action="store",
type="string",
dest="serial",
metavar="SERIAL",
default="",
help="String to open ps by serial.\t\t\t"
"default: None (match any)")
group.add_option(
"-w",
"--warnings",
action="store_true",
dest="warnings",
default=False,
help="Raise power supply and usb port warnings as errors.\t"
"default: %default")
parser.add_option_group(group)
group = OptionGroup(parser, "Channels Options")
group.add_option("-c",
"--channels",
action="store",
type="string",
dest="channels",
metavar="CHANNELS",
default="",
help="Comma separated list of channels to test.\t"
"default: all available")
group.add_option("-r",
"--range",
action="store",
type="int",
dest="range",
metavar="RANGE",
default=7,
help="Channels range, default: +/-2V(7)")
parser.add_option_group(group)
group = OptionGroup(parser, "SigGen Options")
group.add_option("-f",
"--frequency",
action="store",
type="float",
dest="frequency",
metavar="FREQ",
default=1000.0,
help="Siggen frequency to test.\t\t\t"
"default: %default")
group.add_option("-a",
"--amplitude",
action="store",
type="int",
dest="pk2pk",
metavar="PK2PK",
default=3000000,
help="Siggen amplitude to set in microvolts.\t\t"
"default: %default")
group.add_option("-o",
"--offset",
action="store",
type="int",
dest="offset",
metavar="OFFSET",
default=0,
help="Siggen offset to apply, in microvolts.\t\t"
"default: %default")
parser.add_option_group(group)
group = OptionGroup(parser, "Trigger Options")
group.add_option("-T",
"--trigger",
action="store_true",
dest="trigger",
default=False,
help="Use trigger, default: %default")
group.add_option(
"-C",
"--triggc",
action="store",
type="int",
dest="triggc",
metavar="ENUM",
default=-1,
help="Channel on trigger; set -1 to pick first available.")
group.add_option("-V",
"--triggv",
action="store",
type=float,
dest="triggv",
metavar="VRATIO",
default=0.0,
help="Vertical trigger ratio:\t\t\t\t\t"
"threshold as (-1.0, 1.0) of current range.\t"
"default: %default")
group.add_option("-H",
"--triggh",
action="store",
type=float,
dest="triggh",
metavar="HRATIO",
default=0.5,
help="Horizontal trigger ratio (0.0, 1.0)\t\t"
"default: %default")
group.add_option("-D",
"--triggd",
action="store",
type="int",
dest="triggd",
metavar="DIR",
default=2,
help="Trigger direction; 2: rising, 3: falling\t"
"default: 2: rising")
group.add_option(
"-W",
"--triggw",
action="store",
type="int",
dest="triggw",
metavar="BLOCKS",
default=3,
help=
"Trigger wait how many (samples * interval) blocks to wait for trigger. "
"default: %default")
parser.add_option_group(group)
group = OptionGroup(parser, "Sampling Options")
group.add_option("-s",
"--samples",
action="store",
type="int",
dest="max_samples",
metavar="COUNT",
default=10000,
help="Maximum number of raw samples per channel.\t\t"
"Limited to device capabilities where necessary.\t"
"default: %default")
group.add_option("-m",
"--modes",
action="store",
type="string",
dest="modes",
metavar="MODES",
default="raw",
help="Reduction mode to run, use raw \t\t\t\t"
"or comma separated list from avg, agg, dec\t"
"default: %default")
group.add_option("-R",
"--ratios",
action="store",
type="string",
dest="ratios",
metavar="RATIOS",
default="1",
help="Reduction ratios as comma separated list,\t\t"
"ignored for RAW mode. default: %default")
group.add_option("-S",
"--max-segments",
action="store",
type="int",
dest="segments",
metavar="SEGMENTS",
default=1,
help="Number of segments to set.\t\t\t"
"default: %default")
group.add_option(
"-B",
"--segment",
action="store",
type="string",
dest="segment",
metavar="SEGMENT|START-STOP",
default="0",
help="Segment number to collect from (0, max-segments-1).\t"
"Specify as range for bulk collection: start-stop.\t\t"
"default: %default")
group.add_option("-i",
"--interval",
action="store",
type="float",
dest="interval",
metavar="NS",
default=0.0,
help="Desired sample interval in nanoseconds.\t\t\t"
"Set to 0 for count selection.")
group.add_option("-t",
"--cycles",
action="store",
type="int",
dest="cycles",
metavar="CYCLES",
default=3,
help="Calculate interval from number of cycles\t\t\t"
"of the waveform of siggen frequency to test.\t\t"
"Ignored if explicit interval given. default: %default")
group.add_option("-M",
"--overlapped",
action="store_true",
dest="overlapped",
default=False,
help="Use overlapped buffers. default: %default")
group.add_option("-Y",
"--ets-mode",
action="store",
type="string",
dest="ets_mode",
metavar="MODE",
default="off",
help="Set ETS mode for collection: off, fast, slow.\t\t"
"default: %default")
group.add_option("-y",
"--ets-cycles",
action="store",
type="int",
dest="ets_cycles",
metavar="CYCLES",
default=100,
help="ETS cycles, default: %default")
group.add_option("-u",
"--ets-interleaves",
action="store",
type="int",
dest="ets_interleaves",
metavar="INTERLEAVES",
default=10,
help="ETS interleaves, default: %default")
parser.add_option_group(group)
return parser
def main(builtinParameters={}):
# Use processes by default on Unix platforms.
isWindows = platform.system() == 'Windows'
useProcessesIsDefault = not isWindows
global options
from optparse import OptionParser, OptionGroup
parser = OptionParser("usage: %prog [options] {file-or-path}")
parser.add_option("",
"--version",
dest="show_version",
help="Show version and exit",
action="store_true",
default=False)
parser.add_option("-j",
"--threads",
dest="numThreads",
metavar="N",
help="Number of testing threads",
type=int,
action="store",
default=None)
parser.add_option("",
"--config-prefix",
dest="configPrefix",
metavar="NAME",
help="Prefix for 'lit' config files",
action="store",
default=None)
parser.add_option("-D",
"--param",
dest="userParameters",
metavar="NAME=VAL",
help="Add 'NAME' = 'VAL' to the user defined parameters",
type=str,
action="append",
default=[])
group = OptionGroup(parser, "Output Format")
# FIXME: I find these names very confusing, although I like the
# functionality.
group.add_option("-q",
"--quiet",
dest="quiet",
help="Suppress no error output",
action="store_true",
default=False)
group.add_option("-s",
"--succinct",
dest="succinct",
help="Reduce amount of output",
action="store_true",
default=False)
group.add_option("-v",
"--verbose",
dest="showOutput",
help="Show all test output",
action="store_true",
default=False)
group.add_option("-o",
"--output",
dest="output_path",
help="Write test results to the provided path",
action="store",
type=str,
metavar="PATH")
group.add_option("",
"--no-progress-bar",
dest="useProgressBar",
help="Do not use curses based progress bar",
action="store_false",
default=True)
group.add_option("",
"--show-unsupported",
dest="show_unsupported",
help="Show unsupported tests",
action="store_true",
default=False)
group.add_option("",
"--show-xfail",
dest="show_xfail",
help="Show tests that were expected to fail",
action="store_true",
default=False)
parser.add_option_group(group)
group = OptionGroup(parser, "Test Execution")
group.add_option("",
"--path",
dest="path",
help="Additional paths to add to testing environment",
action="append",
type=str,
default=[])
group.add_option("",
"--vg",
dest="useValgrind",
help="Run tests under valgrind",
action="store_true",
default=False)
group.add_option("",
"--vg-leak",
dest="valgrindLeakCheck",
help="Check for memory leaks under valgrind",
action="store_true",
default=False)
group.add_option("",
"--vg-arg",
dest="valgrindArgs",
metavar="ARG",
help="Specify an extra argument for valgrind",
type=str,
action="append",
default=[])
group.add_option("",
"--time-tests",
dest="timeTests",
help="Track elapsed wall time for each test",
action="store_true",
default=False)
group.add_option("",
"--no-execute",
dest="noExecute",
help="Don't execute any tests (assume PASS)",
action="store_true",
default=False)
group.add_option("",
"--xunit-xml-output",
dest="xunit_output_file",
help=("Write XUnit-compatible XML test reports to the"
" specified file"),
default=None)
parser.add_option_group(group)
group = OptionGroup(parser, "Test Selection")
group.add_option("",
"--max-tests",
dest="maxTests",
metavar="N",
help="Maximum number of tests to run",
action="store",
type=int,
default=None)
group.add_option("",
"--max-time",
dest="maxTime",
metavar="N",
help="Maximum time to spend testing (in seconds)",
action="store",
type=float,
default=None)
group.add_option("",
"--shuffle",
dest="shuffle",
help="Run tests in random order",
action="store_true",
default=False)
group.add_option("-i",
"--incremental",
dest="incremental",
help="Run modified and failing tests first (updates "
"mtimes)",
action="store_true",
default=False)
group.add_option("",
"--filter",
dest="filter",
metavar="REGEX",
help=("Only run tests with paths matching the given "
"regular expression"),
action="store",
default=None)
parser.add_option_group(group)
group = OptionGroup(parser, "Debug and Experimental Options")
group.add_option("",
"--debug",
dest="debug",
help="Enable debugging (for 'lit' development)",
action="store_true",
default=False)
group.add_option("",
"--show-suites",
dest="showSuites",
help="Show discovered test suites",
action="store_true",
default=False)
group.add_option("",
"--show-tests",
dest="showTests",
help="Show all discovered tests",
action="store_true",
default=False)
group.add_option("",
"--use-processes",
dest="useProcesses",
help="Run tests in parallel with processes (not threads)",
action="store_true",
default=useProcessesIsDefault)
group.add_option("",
"--use-threads",
dest="useProcesses",
help="Run tests in parallel with threads (not processes)",
action="store_false",
default=useProcessesIsDefault)
group.add_option("",
"--junit-xml-output",
dest="xmlFile",
help=("Write JUnit-compatible XML test reports to the"
" specified file"),
default=None)
parser.add_option_group(group)
(opts, args) = parser.parse_args()
if opts.show_version:
print("lit %s" % (lit.__version__, ))
return
if not args:
parser.error('No inputs specified')
if opts.numThreads is None:
# Python <2.5 has a race condition causing lit to always fail with numThreads>1
# http://bugs.python.org/issue1731717
# I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
# threads by default there.
if sys.hexversion >= 0x2050200:
opts.numThreads = lit.util.detectCPUs()
else:
opts.numThreads = 1
inputs = args
# Create the user defined parameters.
userParams = dict(builtinParameters)
for entry in opts.userParameters:
if '=' not in entry:
name, val = entry, ''
else:
name, val = entry.split('=', 1)
userParams[name] = val
# Create the global config object.
litConfig = lit.LitConfig.LitConfig(
progname=os.path.basename(sys.argv[0]),
path=opts.path,
quiet=opts.quiet,
useValgrind=opts.useValgrind,
valgrindLeakCheck=opts.valgrindLeakCheck,
valgrindArgs=opts.valgrindArgs,
noExecute=opts.noExecute,
debug=opts.debug,
isWindows=isWindows,
params=userParams,
config_prefix=opts.configPrefix)
# Perform test discovery.
run = lit.run.Run(litConfig,
lit.discovery.find_tests_for_inputs(litConfig, inputs))
if opts.showSuites or opts.showTests:
# Aggregate the tests by suite.
suitesAndTests = {}
for result_test in run.tests:
if result_test.suite not in suitesAndTests:
suitesAndTests[result_test.suite] = []
suitesAndTests[result_test.suite].append(result_test)
suitesAndTests = list(suitesAndTests.items())
suitesAndTests.sort(key=lambda item: item[0].name)
# Show the suites, if requested.
if opts.showSuites:
print('-- Test Suites --')
for ts, ts_tests in suitesAndTests:
print(' %s - %d tests' % (ts.name, len(ts_tests)))
print(' Source Root: %s' % ts.source_root)
print(' Exec Root : %s' % ts.exec_root)
# Show the tests, if requested.
if opts.showTests:
print('-- Available Tests --')
for ts, ts_tests in suitesAndTests:
ts_tests.sort(key=lambda test: test.path_in_suite)
for test in ts_tests:
print(' %s' % (test.getFullName(), ))
# Exit.
sys.exit(0)
# Select and order the tests.
numTotalTests = len(run.tests)
# First, select based on the filter expression if given.
if opts.filter:
try:
rex = re.compile(opts.filter)
except:
parser.error("invalid regular expression for --filter: %r" %
(opts.filter))
run.tests = [
result_test for result_test in run.tests
if rex.search(result_test.getFullName())
]
# Then select the order.
if opts.shuffle:
random.shuffle(run.tests)
elif opts.incremental:
sort_by_incremental_cache(run)
else:
run.tests.sort(key=lambda result_test: result_test.getFullName())
# Finally limit the number of tests, if desired.
if opts.maxTests is not None:
run.tests = run.tests[:opts.maxTests]
# Don't create more threads than tests.
opts.numThreads = min(len(run.tests), opts.numThreads)
# Because some tests use threads internally, and at least on Linux each
# of these threads counts toward the current process limit, try to
# raise the (soft) process limit so that tests don't fail due to
# resource exhaustion.
try:
cpus = lit.util.detectCPUs()
desired_limit = opts.numThreads * cpus * 2 # the 2 is a safety factor
# Import the resource module here inside this try block because it
# will likely fail on Windows.
import resource
max_procs_soft, max_procs_hard = resource.getrlimit(
resource.RLIMIT_NPROC)
desired_limit = min(desired_limit, max_procs_hard)
if max_procs_soft < desired_limit:
resource.setrlimit(resource.RLIMIT_NPROC,
(desired_limit, max_procs_hard))
litConfig.note('raised the process limit from %d to %d' % \
(max_procs_soft, desired_limit))
except:
pass
extra = ''
if len(run.tests) != numTotalTests:
extra = ' of %d' % numTotalTests
header = '-- Testing: %d%s tests, %d threads --' % (len(
run.tests), extra, opts.numThreads)
progressBar = None
if not opts.quiet:
if opts.succinct and opts.useProgressBar:
try:
tc = lit.ProgressBar.TerminalController()
progressBar = lit.ProgressBar.ProgressBar(tc, header)
except ValueError:
print(header)
progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
else:
print(header)
startTime = time.time()
display = TestingProgressDisplay(opts, len(run.tests), progressBar)
try:
run.execute_tests(display, opts.numThreads, opts.maxTime,
opts.useProcesses)
except KeyboardInterrupt:
sys.exit(2)
display.finish()
testing_time = time.time() - startTime
if not opts.quiet:
print('Testing Time: %.2fs' % (testing_time, ))
# Write out the test data, if requested.
if opts.output_path is not None:
write_test_results(run, litConfig, testing_time, opts.output_path)
# List test results organized by kind.
hasFailures = False
byCode = {}
for test in run.tests:
if test.result.code not in byCode:
byCode[test.result.code] = []
byCode[test.result.code].append(test)
if test.result.code.isFailure:
hasFailures = True
# Print each test in any of the failing groups.
for title, code in (('Unexpected Passing Tests',
lit.Test.XPASS), ('Failing Tests', lit.Test.FAIL),
('Unresolved Tests', lit.Test.UNRESOLVED),
('Unsupported Tests', lit.Test.UNSUPPORTED),
('Expected Failing Tests', lit.Test.XFAIL)):
if (lit.Test.XFAIL == code and not opts.show_xfail) or \
(lit.Test.UNSUPPORTED == code and not opts.show_unsupported):
continue
elts = byCode.get(code)
if not elts:
continue
print('*' * 20)
print('%s (%d):' % (title, len(elts)))
for test in elts:
print(' %s' % test.getFullName())
sys.stdout.write('\n')
if opts.timeTests and run.tests:
# Order by time.
test_times = [(test.getFullName(), test.result.elapsed)
for test in run.tests]
lit.util.printHistogram(test_times, title='Tests')
for name, code in (('Expected Passes ', lit.Test.PASS),
('Passes With Retry ', lit.Test.FLAKYPASS),
('Expected Failures ', lit.Test.XFAIL),
('Unsupported Tests ', lit.Test.UNSUPPORTED),
('Unresolved Tests ', lit.Test.UNRESOLVED),
('Unexpected Passes ',
lit.Test.XPASS), ('Unexpected Failures',
lit.Test.FAIL)):
if opts.quiet and not code.isFailure:
continue
N = len(byCode.get(code, []))
if N:
print(' %s: %d' % (name, N))
if opts.xmlFile:
# Collect the tests, indexed by test suite
bySuite = {}
for t in run.tests:
suite = t.suite.config.name
if suite not in bySuite:
bySuite[suite] = {'passes': 0, 'failures': 0, 'tests': []}
bySuite[suite]['tests'].append(t)
if t.result.code.isFailure:
bySuite[suite]['failures'] += 1
else:
bySuite[suite]['passes'] += 1
xmlFile = open(opts.xmlFile, "w")
xmlFile.write("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n")
xmlFile.write("<testsuites>\n")
for suiteName in bySuite:
s = bySuite[suiteName]
xmlFile.write("<testsuite name='" + suiteName + "'")
xmlFile.write(" tests='" + str(s['passes'] + s['failures']) + "'")
xmlFile.write(" failures='" + str(s['failures']) + "'>\n")
for t in s['tests']:
xmlFile.write(t.getJUnitXML() + "\n")
xmlFile.write("</testsuite>\n")
xmlFile.write("</testsuites>")
xmlFile.close()
if opts.xunit_output_file:
# Collect the tests, indexed by test suite
by_suite = {}
for result_test in run.tests:
suite = result_test.suite.config.name
if suite not in by_suite:
by_suite[suite] = {'passes': 0, 'failures': 0, 'tests': []}
by_suite[suite]['tests'].append(result_test)
if result_test.result.code.isFailure:
by_suite[suite]['failures'] += 1
else:
by_suite[suite]['passes'] += 1
xunit_output_file = open(opts.xunit_output_file, "w")
xunit_output_file.write(
"<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n")
xunit_output_file.write("<testsuites>\n")
for suite_name, suite in by_suite.items():
safe_suite_name = suite_name.replace(".", "-")
xunit_output_file.write("<testsuite name='" + safe_suite_name +
"'")
xunit_output_file.write(" tests='" +
str(suite['passes'] + suite['failures']) +
"'")
xunit_output_file.write(" failures='" + str(suite['failures']) +
"'>\n")
for result_test in suite['tests']:
xunit_output_file.write(result_test.getJUnitXML() + "\n")
xunit_output_file.write("</testsuite>\n")
xunit_output_file.write("</testsuites>")
xunit_output_file.close()
# If we encountered any additional errors, exit abnormally.
if litConfig.numErrors:
sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
sys.exit(2)
# Warn about warnings.
if litConfig.numWarnings:
sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
if hasFailures:
sys.exit(1)
sys.exit(0)
dest="warn_unusual_code",
default=False,
help="""\
Enable warnings for unusual code detected at compile time.""",
)
parser.add_option(
"--assume-yes-for-downloads",
action="store_true",
dest="assume_yes_for_downloads",
default=False,
help="""\
Allow Nuitka to download code if necessary, e.g. dependency walker on Windows.""",
)
include_group = OptionGroup(parser,
"Control the inclusion of modules and packages")
include_group.add_option("--include-package",
action="append",
dest="include_packages",
metavar="PACKAGE",
default=[],
help="""\
Include a whole package. Give as a Python namespace, e.g. ``some_package.sub_package``
and Nuitka will then find it and include it and all the modules found below that
disk location in the binary or extension module it creates, and make it available
for import by the code. Default empty.""")
include_group.add_option("--include-module",
action="append",
dest="include_modules",
def populate_parser(parser, defaultparameters):
opts = OptionGroup(parser, 'GPAW parameters')
append_to_optiongroup(defaultparameters, opts)
parser.add_option_group(opts)
return parser
print "[!] Failed to start the server: %s" % e
if __name__ == "__main__":
header = " _ _ __ \n"
header += " | | version %s | | / _| \n" % DNSCHEF_VERSION
header += " __| |_ __ ___ ___| |__ ___| |_ \n"
header += " / _` | '_ \/ __|/ __| '_ \ / _ \ _|\n"
header += " | (_| | | | \__ \ (__| | | | __/ | \n"
header += " \__,_|_| |_|___/\___|_| |_|\___|_| \n"
header += " [email protected] \n"
# Parse command line arguments
parser = OptionParser(usage = "dnschef.py [options]:\n" + header, description="DNSChef is a highly configurable DNS Proxy for Penetration Testers and Malware Analysts. It is capable of fine configuration of which DNS replies to modify or to simply proxy with real responses. In order to take advantage of the tool you must either manually configure or poison DNS server entry to point to DNSChef. The tool requires root privileges to run on privileged ports." )
fakegroup = OptionGroup(parser, "Fake DNS records:")
fakegroup.add_option('--fakeip', metavar="192.0.2.1", action="store", help='IP address to use for matching DNS queries. If you use this parameter without specifying domain names, then all \'A\' queries will be spoofed. Consider using --file argument if you need to define more than one IP address.')
fakegroup.add_option('--fakeipv6', metavar="2001:db8::1", action="store", help='IPv6 address to use for matching DNS queries. If you use this parameter without specifying domain names, then all \'AAAA\' queries will be spoofed. Consider using --file argument if you need to define more than one IPv6 address.')
fakegroup.add_option('--fakemail', metavar="mail.fake.com", action="store", help='MX name to use for matching DNS queries. If you use this parameter without specifying domain names, then all \'MX\' queries will be spoofed. Consider using --file argument if you need to define more than one MX record.')
fakegroup.add_option('--fakealias', metavar="www.fake.com", action="store", help='CNAME name to use for matching DNS queries. If you use this parameter without specifying domain names, then all \'CNAME\' queries will be spoofed. Consider using --file argument if you need to define more than one CNAME record.')
fakegroup.add_option('--fakens', metavar="ns.fake.com", action="store", help='NS name to use for matching DNS queries. If you use this parameter without specifying domain names, then all \'NS\' queries will be spoofed. Consider using --file argument if you need to define more than one NS record.')
fakegroup.add_option('--file', action="store", help="Specify a file containing a list of DOMAIN=IP pairs (one pair per line) used for DNS responses. For example: google.com=1.1.1.1 will force all queries to 'google.com' to be resolved to '1.1.1.1'. IPv6 addresses will be automatically detected. You can be even more specific by combining --file with other arguments. However, data obtained from the file will take precedence over others.")
parser.add_option_group(fakegroup)
parser.add_option('--fakedomains', metavar="thesprawl.org,google.com", action="store", help='A comma separated list of domain names which will be resolved to FAKE values specified in the the above parameters. All other domain names will be resolved to their true values.')
parser.add_option('--truedomains', metavar="thesprawl.org,google.com", action="store", help='A comma separated list of domain names which will be resolved to their TRUE values. All other domain names will be resolved to fake values specified in the above parameters.')
rungroup = OptionGroup(parser,"Optional runtime parameters.")
rungroup.add_option("--dnsfile", action="store", help="Specify a dns file to search tcp domain")
rungroup.add_option("--logfile", action="store", help="Specify a log file to record all activity")
rungroup.add_option("--nameservers", metavar="8.8.8.8#53 or 4.2.2.1#53#tcp or 2001:4860:4860::8888", default='8.8.8.8', action="store", help='A comma separated list of alternative DNS servers to use with proxied requests. Nameservers can have either IP or IP#PORT format. A randomly selected server from the list will be used for proxy requests when provided with multiple servers. By default, the tool uses Google\'s public DNS server 8.8.8.8 when running in IPv4 mode and 2001:4860:4860::8888 when running in IPv6 mode.')
-t 页面输出模板
-o 输出页面
-i 页面标题
通用参数:
-N 试用模式:不作为,测试输出相关参数
-D 调试模式:打印debug信息到屏幕
-Q 安静模式:不输出任何信息
-V 运营模式:输出 INFO|WARNING|ERROR 到sys.stderr
--doctest TDD支持
-v doctest 详细模式
'''
usage = "$python %prog [-V|N|D|Q][--dot|fdp|circo|twopi|neato][-t path/2/templete.htm] [-o path/2/export.html] [-i 'title of page'] -d path/2/foo.dot"
parser = OptionParser(usage,
version="%s {powered by Zoom.Quiet+Leo}" % VERSION)
gRun = OptionGroup(
parser, "Group for Running", '''DEFAULT::
-V False -N False -D False -Q False''')
gRun.add_option("-V",
"--verbose",
metavar="",
help="mod_run: export INFO|WARNING|ERROR ->sys.stderr",
dest="verbose",
action="store_true",
default=False)
gRun.add_option("-v",
metavar="",
help="verbose for doctest",
dest="verbose_doctest",
action="store_true",
default=False)
gRun.add_option("-N",
def parse_options():
description = "This script is an example implementation of SAP's Message Server Monitor program (msmon). It " \
"allows the monitoring of a Message Server service and allows sending different commands and " \
"opcodes. Includes some commands not available on the msmon program. Some commands requires the " \
"server running in monitor mode, the most requires access to the Message Server internal port."
epilog = "pysap %(version)s - %(url)s - %(repo)s" % {"version": pysap.__version__,
"url": pysap.__url__,
"repo": pysap.__repo__}
usage = "Usage: %prog [options] -d <remote host>"
parser = OptionParser(usage=usage, description=description, epilog=epilog)
target = OptionGroup(parser, "Target")
target.add_option("-d", "--remote-host", dest="remote_host",
help="Remote host")
target.add_option("-p", "--remote-port", dest="remote_port", type="int", default=3900,
help="Remote port [%default]")
target.add_option("--route-string", dest="route_string",
help="Route string for connecting through a SAP Router")
target.add_option("--domain", dest="domain", default="ABAP",
help="Domain to connect to (ABAP, J2EE or JSTARTUP) [%default]")
parser.add_option_group(target)
misc = OptionGroup(parser, "Misc options")
misc.add_option("-c", "--client", dest="client", default="pysap's-monitor",
help="Client name [%default]")
misc.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
help="Verbose output [%default]")
misc.add_option("--log-file", dest="logfile", metavar="FILE",
help="Log file")
misc.add_option("--console-log", dest="consolelog", metavar="FILE",
help="Console log file")
misc.add_option("--script", dest="script", metavar="FILE",
help="Script file to run")
parser.add_option_group(misc)
(options, _) = parser.parse_args()
if not (options.remote_host or options.route_string):
parser.error("Remote host or route string is required")
if options.domain not in ms_domain_values_inv.keys():
parser.error("Invalid domain specified")
return options
def parse_options(argv):
"""Parses options from the command line """
from optparse import OptionParser, OptionGroup
parser = OptionParser()
required = OptionGroup(parser, 'REQUIRED')
required.add_option('-b',
'--best_score',
dest='best_scores',
metavar='FILE',
help='file to store the best scoring parameters',
default='-')
required.add_option('-m',
'--matrix',
dest='matrix',
metavar='FILE',
help='file to store the full performance matrix',
default='-')
required.add_option('-f',
'--features',
dest='features',
metavar='FILE',
help='alignment intron features',
default='-')
required.add_option('-i',
'--annotation_introns',
dest='anno_int',
metavar='FILE',
help='annotation intron list',
default='-')
optional = OptionGroup(parser, 'OPTIONAL')
optional.add_option(
'-E',
'--exclude_introns',
dest='exclude_introns',
metavar='STRINGLIST',
help=
'list of comma separated intron files to exclude from submitted features',
default='-')
optional.add_option('-I',
'--max_intron_len',
dest='max_intron_len',
metavar='INT',
type='int',
help='maximal intron length [10000000]',
default=10000000)
optional.add_option('-s',
'--ignore_strand',
dest='ignore_strand',
action='store_true',
help='ignore strand information present in annotation',
default=False)
optional.add_option(
'-X',
'--max_feat_mismatches',
dest='max_feat_mismatches',
metavar='INT',
type='int',
help=
'max number of mismatches for feat generation [80] (do only change, if you are absolutely sure!)',
default=80)
optional.add_option('-v',
'--verbose',
dest='verbose',
action='store_true',
help='verbosity',
default=False)
parser.add_option_group(required)
parser.add_option_group(optional)
(options, args) = parser.parse_args()
if len(argv) < 2:
parser.print_help()
sys.exit(2)
return options
action="store_true",
help="increase verbosity. [Default: False]")
parser.add_option(
"--channels",
dest="channels",
default="mt, et, em, mm, tt",
type="string",
help="Channels to produce the tables for. [Default: \"mt, et, em, mm, tt\"]"
)
parser.add_option(
"--categories",
dest="categories",
default="0, 1, 2, 3, 5",
type="string",
help="Categories to produce the tables for. [Default: \"0, 1, 2, 3, 5\"]")
cats1 = OptionGroup(parser, "LUMI PER CHANNEL", "Luminosities per channel.")
cats1.add_option("--lumi-7TeV-mm",
dest="lumi_7TeV_mm",
default=4.9,
type="float",
help="Luminosity used for mm channel. [Default: 4.9]")
cats1.add_option("--lumi-7TeV-ee",
dest="lumi_7TeV_ee",
default=4.9,
type="float",
help="Luminosity used for ee channel. [Default: 4.9]")
cats1.add_option("--lumi-7TeV-em",
dest="lumi_7TeV_em",
default=4.9,
type="float",
help="Luminosity used for em channel. [Default: 4.9]")
def parseCommandLine():
epilog = ''
epilog += ('\n Classification Algorithms:\n' + '\n'.join([
' %-10s %s' % (name, classifier_menu[name][1])
for name in classifier_names
]) + '\n')
epilog += ('\n Tokenizer Options:\n' + '\n'.join([
' %-10s %s' % (name, tokenize_menu[name][1])
for name in tokenize_names
]) + '\n')
epilog += ('\n Feature Space Options:\n' + '\n'.join([
' %-10s %s' % (name, feature_menu[name][1])
for name in feature_names
]) + '\n')
parser = MyParser(usage='usage: %prog [options]', epilog=epilog)
group = OptionGroup(parser, 'Experiment Options')
group.add_option(
'-r',
dest='rounds',
type='int',
default=1,
help=
'Number of independent rounds of 10-fold cross-validation [default: %default]'
)
group.add_option('-i',
dest='filename',
default='SMS_Spam.dat',
help='read data from file [default: %default]')
group.add_option('-s',
dest='seed',
type=int,
default=None,
help='seed for all randomization [default: clock]')
group.add_option(
'-v',
dest='verbose',
default=False,
action='store_true',
help=
('Verbose; print success rate for each independent trial [default: %default]'
))
parser.add_option_group(group)
group = OptionGroup(parser, 'Algorithmic Options')
group.add_option(
'-c',
dest='algorithm',
default=classifier_default,
choices=classifier_names,
help=('Classification algorithm (see below) [default: %default]'))
group.add_option('-t',
dest='tokenize',
default=tokenize_default,
choices=tokenize_names,
help=('Tokenizer (see below) [default: %default]'))
group.add_option(
'-d',
dest='downcase',
default=False,
action='store_true',
help=('Downcase all messages before tokenizing [default: %default]'))
group.add_option(
'-f',
dest='feature',
default=feature_default,
choices=feature_names,
help=('Feature selection (see below) [default: %default]'))
parser.add_option_group(group)
group = OptionGroup(parser, 'Additional Nearest Neighbor Settings')
group.add_option(
'-k',
dest='knn',
type=int,
default=1,
metavar="NEIGHBORS",
help=
('Number of neighbors for k-nearest neighbors algorithm [default: %default]'
))
parser.add_option_group(group)
group = OptionGroup(parser, 'Additional Naive Bayesian Settings')
group.add_option(
'-b',
dest='bayesStrength',
type=float,
default=0,
metavar="STRENGTH",
help=(
'Strength factor of prior for Naive Bayesian [default: %default]'))
parser.add_option_group(group)
group = OptionGroup(parser, 'Additional Decision Tree Settings')
group.add_option(
'-m',
dest='treeThreshold',
type=int,
default=1,
metavar="SIZE",
help=
('Split decision subtrees having more than this many samples [default: %default]'
))
group.add_option(
'-u',
dest='treeUniformity',
type=float,
default=1.0,
metavar="PCT",
help=
('Do not split decision subtree if uniformity is at or above given ratio [default: %default]'
))
parser.add_option_group(group)
return parser.parse_args()
import sys
import collections
import math
import numpy
from optparse import OptionParser, OptionGroup
#Author: Martin Kapun
######################################################### HELP #########################################################################
usage = "python %prog --input candidates_BE.cmh --pops 8,9,10 --threshold 0.95 > candidates_BE_095.cmh"
parser = OptionParser(usage=usage)
group = OptionGroup(
parser, """
H E L P :
_________
Description:
This script extracts the SNP with a major allele equal or higher than the threshold (--threshold) in all replicate populations (defined with --pops) from a sync input file (--input)
""")
######################################################### CODE #########################################################################
parser.add_option("-i", "--input", dest="inp", help="input: cmh or sync file")
parser.add_option(
"-p",
"--pops",
dest="pops",
help=
"define, for which population in the sync you want to test for fixed SNPs")
parser.add_option("-t",
"--threshold",
def add_options(cls, parser):
super(ReplicaPrepare, cls).add_options(parser, debug_option=True)
parser.add_option("-p", "--password", dest="password",
help="Directory Manager password (for the existing master)")
parser.add_option("--ip-address", dest="ip_addresses", type="ip",
action="append", default=[], metavar="IP_ADDRESS",
help="add A and PTR records of the future replica. This option can be used multiple times")
parser.add_option("--reverse-zone", dest="reverse_zones",
action="append", default=[], metavar="REVERSE_ZONE",
help="the reverse DNS zone to use. This option can be used multiple times")
parser.add_option("--no-reverse", dest="no_reverse",
action="store_true", default=False,
help="do not create reverse DNS zone")
parser.add_option("--auto-reverse", dest="auto_reverse", default=False,
action="store_true", help="create necessary DNS zones")
parser.add_option("--allow-zone-overlap", dest="allow_zone_overlap",
action="store_true", default=False, help="create DNS "
"zone even if it already exists")
parser.add_option("--ca", dest="ca_file", default=paths.CACERT_P12,
metavar="FILE",
help="location of CA PKCS#12 file, default /root/cacert.p12")
parser.add_option('--no-wait-for-dns', dest='wait_for_dns',
action='store_false', default=True,
help="do not wait until the replica is resolvable in DNS")
group = OptionGroup(parser, "SSL certificate options",
"Only used if the server was installed using custom SSL certificates")
group.add_option("--dirsrv-cert-file", dest="dirsrv_cert_files",
action="append", metavar="FILE",
help="File containing the Directory Server SSL certificate and private key")
group.add_option("--dirsrv_pkcs12", dest="dirsrv_cert_files",
action="append",
help=SUPPRESS_HELP)
group.add_option("--http-cert-file", dest="http_cert_files",
action="append", metavar="FILE",
help="File containing the Apache Server SSL certificate and private key")
group.add_option("--http_pkcs12", dest="http_cert_files",
action="append",
help=SUPPRESS_HELP)
group.add_option("--dirsrv-pin", dest="dirsrv_pin", sensitive=True,
metavar="PIN",
help="The password to unlock the Directory Server private key")
group.add_option("--dirsrv_pin", dest="dirsrv_pin", sensitive=True,
help=SUPPRESS_HELP)
group.add_option("--http-pin", dest="http_pin", sensitive=True,
metavar="PIN",
help="The password to unlock the Apache Server private key")
group.add_option("--http_pin", dest="http_pin", sensitive=True,
help=SUPPRESS_HELP)
group.add_option("--dirsrv-cert-name", dest="dirsrv_cert_name",
metavar="NAME",
help="Name of the Directory Server SSL certificate to install")
group.add_option("--http-cert-name", dest="http_cert_name",
metavar="NAME",
help="Name of the Apache Server SSL certificate to install")
parser.add_option_group(group)
def main():
from optparse import OptionParser, OptionGroup
parser = OptionParser("usage: %prog [options]")
group = OptionGroup(parser, "Input Options")
group.add_option("",
"--source-root",
dest="source_root",
metavar="PATH",
help="Path to the LLVM source (inferred if not given)",
action="store",
default=None)
group.add_option(
"",
"--llvmbuild-source-root",
dest="llvmbuild_source_root",
help=("If given, an alternate path to search for LLVMBuild.txt files"),
action="store",
default=None,
metavar="PATH")
group.add_option("",
"--build-root",
dest="build_root",
metavar="PATH",
help="Path to the build directory (if needed) [%default]",
action="store",
default=None)
parser.add_option_group(group)
group = OptionGroup(parser, "Output Options")
group.add_option("",
"--print-tree",
dest="print_tree",
help="Print out the project component tree [%default]",
action="store_true",
default=False)
group.add_option("",
"--write-llvmbuild",
dest="write_llvmbuild",
help="Write out the LLVMBuild.txt files to PATH",
action="store",
default=None,
metavar="PATH")
group.add_option("",
"--write-library-table",
dest="write_library_table",
metavar="PATH",
help="Write the C++ library dependency table to PATH",
action="store",
default=None)
group.add_option("",
"--write-cmake-fragment",
dest="write_cmake_fragment",
metavar="PATH",
help="Write the CMake project information to PATH",
action="store",
default=None)
group.add_option("",
"--write-cmake-exports-fragment",
dest="write_cmake_exports_fragment",
metavar="PATH",
help="Write the CMake exports information to PATH",
action="store",
default=None)
group.add_option("",
"--configure-target-def-file",
dest="configure_target_def_files",
help="""Configure the given file at SUBPATH (relative to
the inferred or given source root, and with a '.in' suffix) by replacing certain
substitution variables with lists of targets that support certain features (for
example, targets with AsmPrinters) and write the result to the build root (as
given by --build-root) at the same SUBPATH""",
metavar="SUBPATH",
action="append",
default=None)
parser.add_option_group(group)
group = OptionGroup(parser, "Configuration Options")
group.add_option("",
"--native-target",
dest="native_target",
metavar="NAME",
help=("Treat the named target as the 'native' one, if "
"given [%default]"),
action="store",
default=None)
group.add_option("",
"--enable-targets",
dest="enable_targets",
metavar="NAMES",
help=("Enable the given space or semi-colon separated "
"list of targets, or all targets if not present"),
action="store",
default=None)
group.add_option("",
"--enable-optional-components",
dest="optional_components",
metavar="NAMES",
help=("Enable the given space or semi-colon separated "
"list of optional components"),
action="store",
default="")
parser.add_option_group(group)
(opts, args) = parser.parse_args()
# Determine the LLVM source path, if not given.
source_root = opts.source_root
if source_root:
if not os.path.exists(
os.path.join(source_root, 'lib', 'IR', 'Function.cpp')):
parser.error('invalid LLVM source root: %r' % source_root)
else:
llvmbuild_path = os.path.dirname(__file__)
llvm_build_path = os.path.dirname(llvmbuild_path)
utils_path = os.path.dirname(llvm_build_path)
source_root = os.path.dirname(utils_path)
if not os.path.exists(
os.path.join(source_root, 'lib', 'IR', 'Function.cpp')):
parser.error('unable to infer LLVM source root, please specify')
# Construct the LLVM project information.
llvmbuild_source_root = opts.llvmbuild_source_root or source_root
project_info = LLVMProjectInfo.load_from_path(source_root,
llvmbuild_source_root)
# Add the magic target based components.
add_magic_target_components(parser, project_info, opts)
# Validate the project component info.
project_info.validate_components()
# Print the component tree, if requested.
if opts.print_tree:
project_info.print_tree()
# Write out the components, if requested. This is useful for auto-upgrading
# the schema.
if opts.write_llvmbuild:
project_info.write_components(opts.write_llvmbuild)
# Write out the required library table, if requested.
if opts.write_library_table:
project_info.write_library_table(opts.write_library_table,
opts.optional_components)
# Write out the cmake fragment, if requested.
if opts.write_cmake_fragment:
project_info.write_cmake_fragment(opts.write_cmake_fragment,
opts.optional_components)
if opts.write_cmake_exports_fragment:
project_info.write_cmake_exports_fragment(
opts.write_cmake_exports_fragment, opts.optional_components)
# Configure target definition files, if requested.
if opts.configure_target_def_files:
# Verify we were given a build root.
if not opts.build_root:
parser.error("must specify --build-root when using "
"--configure-target-def-file")
# Create the substitution list.
available_targets = [
ci for ci in project_info.component_infos
if ci.type_name == 'TargetGroup'
]
substitutions = [
("@LLVM_ENUM_TARGETS@", ' '.join('LLVM_TARGET(%s)' % ci.name
for ci in available_targets)),
("@LLVM_ENUM_ASM_PRINTERS@",
' '.join('LLVM_ASM_PRINTER(%s)' % ci.name
for ci in available_targets if ci.has_asmprinter)),
("@LLVM_ENUM_ASM_PARSERS@",
' '.join('LLVM_ASM_PARSER(%s)' % ci.name
for ci in available_targets if ci.has_asmparser)),
("@LLVM_ENUM_DISASSEMBLERS@",
' '.join('LLVM_DISASSEMBLER(%s)' % ci.name
for ci in available_targets if ci.has_disassembler))
]
# Configure the given files.
for subpath in opts.configure_target_def_files:
inpath = os.path.join(source_root, subpath + '.in')
outpath = os.path.join(opts.build_root, subpath)
result = configutil.configure_file(inpath, outpath, substitutions)
if not result:
note("configured file %r hasn't changed" % outpath)
# Command line interpretter code
if __name__ == "__main__":
stime = time.time()
global_enumerate = ParFunction(enumerate)
global_print_stats = ParRootFunction(print_stats)
global_print_value = ParRootFunction(print_value)
global_print_blank = ParRootFunction(print_blank)
from optparse import OptionParser, OptionGroup
## Parse command line options.
usage = "usage: %prog --states= --symbols= [options]"
parser = OptionParser(usage=usage)
req_parser = OptionGroup(parser, "Required Parameters") # Oxymoron?
req_parser.add_option("--states", type=int, help="Number of states")
req_parser.add_option("--symbols", type=int, help="Number of symbols")
parser.add_option_group(req_parser)
enum_parser = OptionGroup(parser, "Enumeration Options")
enum_parser.add_option("--randomize",
action="store_true",
default=False,
help="Randomize the order of enumeration.")
enum_parser.add_option("--seed", type=int, help="Seed to randomize with.")
parser.add_option_group(enum_parser)
Macro_Simulator.add_option_group(parser)
out_parser = OptionGroup(parser, "Output Options")
def main(argv=sys.argv[1:]):
parser = OptionParser(usage=USAGE, version="Version: " + __version__)
# Required arguments
requiredOptions = OptionGroup(
parser, "Required options",
"These options are required to run BOTA, and may be supplied in any order."
)
requiredOptions.add_option(
"-c",
"--config",
type="string",
metavar="FILE",
help="The configuration file to define a project.")
requiredOptions.add_option(
"-o",
"--outdir",
type="string",
metavar="DIR",
help=
"The output directory of BOTA. If it doesn't exist, BOTA will create it."
)
parser.add_option_group(requiredOptions)
# Optional arguments that need to be supplied if not the same as default
optOptions = OptionGroup(
parser, "Optional parameters",
"These options are optional, and may be supplied in any order.")
optOptions.add_option(
"-t",
"--nproc",
type="int",
default=1,
metavar="INT",
help=
"Number of processor for BOTA to use [default: 1; set 0 if you want to use all CPUs available]."
)
parser.add_option_group(optOptions)
(options, args) = parser.parse_args(argv)
# load allele first
if options.config == None:
parser.error('[FATAL] No input configuration file specified.\n')
exit(1)
if options.outdir == None:
parser.error('[FATAL] No output directory specified.\n')
exit(1)
if not os.path.exists(options.outdir):
try:
os.mkdir(options.outdir)
except:
sys.stderr.write(
'[FATAL] Error in creating output directory:%s\n' %
options.outdir)
exit()
logfile = os.path.join(options.outdir, 'project.log')
logfh = open(logfile, 'w')
project_info = parse_config(options.config)
project_info.print_projInfo()
project_info.print_projInfo(stream=logfh)
logfh.close()
sys.stdout.write('################# Data Preparation ################\n')
# copy hmmtop db files
current_dir = os.getcwd()
for o in project_info.hmmtop_arch:
t = os.path.join(current_dir, os.path.basename(o))
if not os.path.exists(t): shutil.copyfile(o, t)
# iterate through genomes
for genomeID in project_info.genomes:
genome_pkl = os.path.join(options.outdir, '%s.pkl' % genomeID)
if os.path.exists(genome_pkl): continue
genome_proj = project_info.genomes[genomeID]
genome_dir = os.path.join(options.outdir, genomeID)
if not os.path.exists(genome_dir): os.mkdir(genome_dir)
# ID the AA sequences and genes
sys.stdout.write(' [%s] Now extracting A.A. sequences.\n' % genomeID)
faa_file = os.path.join(genome_dir, '%s.faa' % genomeID)
if not os.path.exists(faa_file):
if genome_proj.gff != None:
extract_gff_features(genome_proj, faa_file)
else:
prodigal = project_info.prodigal
args = [
prodigal, genomeID, genome_proj.fna, genome_dir, 'single'
]
run_prodigal(args)
# get Gram stain type
sys.stdout.write(' [%s] Now deciding Gram stain type.\n' % genomeID)
gram_file = os.path.join(genome_dir, '%s.gram' % genomeID)
if not os.path.exists(gram_file):
if genome_proj.gram == None:
blat = project_info.blat
db = project_info.gram_db
genome_proj.gram = call_Gram(
[blat, db, faa_file, genome_dir, genomeID])
with open(gram_file, 'w') as gfh:
gfh.write('%s\n' % genome_proj.gram)
else:
with open(gram_file, 'r') as gfh:
genome_proj.gram = gfh.readline().rstrip('\n')
# hmmscan for domains
sys.stdout.write(' [%s] HMMscan against Pfam-A for domain ID.\n' %
genomeID)
hmmscan_out = os.path.join(genome_dir, '%s.hmmscan' % genomeID)
if not os.path.exists(hmmscan_out):
pfam_db = project_info.pfam
hmmscan = project_info.hmmscan
args = [hmmscan, pfam_db, faa_file, hmmscan_out, options.nproc]
run_hmmscan(args)
# psort
sys.stdout.write(
' [%s] Determining protein subcellular location using PSORT.\n' %
genomeID)
psort_out = os.path.join(genome_dir, '%s.psort' % genomeID)
if not os.path.exists(psort_out):
psort = project_info.psort
gram = genome_proj.gram
psort_args = [
psort, genome_dir, faa_file, gram, psort_out, options.nproc
]
run_psort(psort_args)
# hmmtop
sys.stdout.write(
' [%s] Now predicting transmembrane structures using HMMTOP...\n'
% genomeID)
hmmtop_file = os.path.join(genome_dir, '%s.hmmtop' % genomeID)
if not os.path.exists(hmmtop_file):
hmmtop = project_info.hmmtop
hmmtop_args = [
project_info.hmmtop, faa_file, hmmtop_file, genome_dir,
options.nproc
]
run_hmmtop(hmmtop_args)
# PW Matrix
sys.stdout.write(' [%s] Now PW Matrix scoring...\n' % genomeID)
pwm_file = os.path.join(genome_dir, '%s.pwm' % genomeID)
if not os.path.exists(pwm_file):
loc_dict = {}
# load the location information
psort_out = os.path.join(genome_dir, '%s.psort' % genomeID)
for line in open(psort_out, 'r'):
cols = line.rstrip().split('\t')
geneID = cols[0].rstrip(' ')
loc = cols[1]
loc_dict[geneID] = loc
aa_list, pwmatrix = cPickle.load(open(project_info.pwm_db, 'rb'),
encoding='latin-1')
pw_dict = {}
for aa_ind, aa in enumerate(aa_list):
for pos_ind in range(9):
pw_dict[(aa, pos_ind)] = pwmatrix[aa_ind, pos_ind]
pwmscore_args = [
faa_file, pw_dict, loc_dict, pwm_file, options.nproc
]
run_pwmscore(pwmscore_args)
# make input data per GenomeID
sys.stdout.write(' [%s] Integrating all data for DNN module...\n' %
genomeID)
# make data into genome_pkl
integrate_data(genomeID, genome_dir, genome_pkl)
# clean up
current_dir = os.getcwd()
for o in project_info.hmmtop_arch:
t = os.path.join(current_dir, os.path.basename(o))
os.unlink(t)
sys.stdout.write(
'################# DNN Model Prediction ################\n')
# load models
models = {}
for genomeID in project_info.genomes:
for allele in project_info.genomes[genomeID].alleles:
model_arch, model_weights = project_info.models[allele]
models[allele] = model_from_json(open(model_arch).read())
# models[allele] = model_from_config(open(model_arch).read())
models[allele].load_weights(model_weights)
AAs = 'ARNDCQEGHILKMFPSTWYV'
aa_dict = {}
for ind, aa in enumerate(list(AAs)):
aa_dict[aa] = ind
for genomeID in project_info.genomes:
genome_pkl = os.path.join(options.outdir, '%s.pkl' % genomeID)
genome_data = cPickle.load(open(genome_pkl, 'rb'), encoding="latin-1")
candidates = []
for geneID in genome_data:
G = genome_data[geneID]
if G.cellular_loc not in [
'CellWall', 'Cellwall', 'Extracellular', 'OuterMembrane'
]:
continue
candidates += G.select_candidate()
R = [] # store the results
for pep in candidates:
#if pep.gene != 'JXUN01000121.1|29853-30995|+|gene=TA05_10230': continue
D = transform_seq(pep.seq, aa_dict)
for allele in models:
model = models[allele]
try:
Z = model.predict_classes(D, batch_size=128, verbose=0)
S = map(itemgetter(1),
model.predict(D, batch_size=128, verbose=0))
except:
continue
ZX = []
G = networkx.Graph()
locs = []
for ind, x in enumerate(Z):
sx, nx = pep.scores[ind]
#y = sx+nx/1e13
if x == 1 and sx > 1e-10 and nx >= 75:
ZX.append(1)
locs.append(ind)
else:
ZX.append(0)
G.add_nodes_from(locs)
for l1 in locs:
for l2 in locs:
if abs(l2 - l1) <= 6: G.add_edge(l1, l2)
for x in list(networkx.connected_components(G)):
left, right = min(x), max(x)
seq = pep.seq[left:right + 9]
z = sum(S[left:right + 9]) / (9 + right - left)
#z = max(S[left:right+9])
if z < 0.6: continue
s_coord = left + pep.start
e_coord = right + 9 + pep.start
chr = pep.gene.split('|')[0]
R.append([allele, pep.gene, chr, s_coord, e_coord, seq, z])
peptide_output = os.path.join(options.outdir,
'%s.peptides.txt' % genomeID)
pfh = open(peptide_output, 'w')
pfh.write(
'#peptide\tgene_name\tchr_acc\tgene_start\tgene_stop\tstrand\tpep_start\tpep_stop\tscore\n'
)
for allele, gene, chr, s, e, seq, score in R:
gene_start, gene_stop, strand, geneID = re.search(
'(\d+)\-(\d+)\|(.{1})\|gene\=(.+)$', gene).group(1, 2, 3, 4)
pep_start = s * 3 - 2
pep_stop = e * 3
pfh.write('%s\t%s\t%s\t%s\t%s\t%s\t%i\t%i\t%.6f\n' % \
(seq, geneID, chr, gene_start, gene_stop, strand, pep_start, pep_stop, score))
pfh.close()
sys.stdout.write(' [%s] %i peptides predicted.\n' %
(genomeID, len(R)))
sys.stdout.write('Done.\n\n')
return 0
def get_parser() -> ModifiedOptionParser:
parser = ModifiedOptionParser(
add_help_option=False,
version="%prog " + __version__,
usage="%prog [options] [connection string]",
epilog=
("The connection string can be in the form of a list of "
"Key/Value parameters or an URI as described in the PostgreSQL documentation. "
"The parsing is delegated to the libpq: different versions of the client library "
"may support different formats or parameters (for example, connection URIs are "
"only supported from libpq 9.2)"),
description=("htop like application for PostgreSQL "
"server activity monitoring."),
)
# -U / --username
parser.add_option(
"-U",
"--username",
dest="username",
help="Database user name",
metavar="USERNAME",
)
# -p / --port
parser.add_option(
"-p",
"--port",
dest="port",
help="Database server port",
metavar="PORT",
)
# -h / --host
parser.add_option(
"-h",
"--host",
dest="host",
help="Database server host or socket directory",
metavar="HOSTNAME",
)
# -d / --dbname
parser.add_option(
"-d",
"--dbname",
dest="dbname",
help="Database name to connect to",
metavar="DBNAME",
)
# --blocksize
parser.add_option(
"--blocksize",
dest="blocksize",
help="Filesystem blocksize (default: %default)",
metavar="BLOCKSIZE",
default=4096,
)
# --rds
parser.add_option(
"--rds",
dest="rds",
action="store_true",
help="Enable support for AWS RDS",
default=False,
)
# --output
parser.add_option(
"--output",
dest="output",
help="Store running queries as CSV",
metavar="FILEPATH",
default=None,
)
# --help
parser.add_option(
"--help",
dest="help",
action="store_true",
help="Show this help message and exit",
default=False,
)
# --no-db-size
parser.add_option(
"--no-db-size",
dest="nodbsize",
action="store_true",
help="Skip total size of DB",
default=False,
)
# --verbose-mode
parser.add_option(
"--verbose-mode",
dest="verbosemode",
help=
"Queries display mode. Values: 1-TRUNCATED, 2-FULL(default), 3-INDENTED",
metavar="VERBOSE_MODE",
choices=["1", "2", "3"],
default="2",
)
# --duration-mode
parser.add_option(
"--duration-mode",
dest="durationmode",
help=
"Duration mode. Values: 1-QUERY(default), 2-TRANSACTION, 3-BACKEND",
metavar="DURATION_MODE",
choices=["1", "2", "3"],
default="1",
)
# --min-duration
parser.add_option(
"--min-duration",
dest="minduration",
help=
"Don't display queries with smaller than specified duration (in seconds)",
metavar="SECONDS",
type=float,
default=0,
)
group = OptionGroup(
parser, "Display Options, you can exclude some columns by using them ")
# --no-pid
group.add_option(
"--no-pid",
dest="nopid",
action="store_true",
help="Disable PID.",
default=False,
)
# --no-database
group.add_option(
"--no-database",
dest="nodb",
action="store_true",
help="Disable DATABASE",
default=False,
)
# --no-user
group.add_option(
"--no-user",
dest="nouser",
action="store_true",
help="Disable USER",
default=False,
)
# --no-client
group.add_option(
"--no-client",
dest="noclient",
action="store_true",
help="Disable CLIENT",
default=False,
)
# --no-cpu
group.add_option(
"--no-cpu",
dest="nocpu",
action="store_true",
help="Disable CPU%",
default=False,
)
# --no-mem
group.add_option(
"--no-mem",
dest="nomem",
action="store_true",
help="Disable MEM%",
default=False,
)
# --no-read
group.add_option(
"--no-read",
dest="noread",
action="store_true",
help="Disable READ/s",
default=False,
)
# --no-write
group.add_option(
"--no-write",
dest="nowrite",
action="store_true",
help="Disable WRITE/s",
default=False,
)
# --no-time
group.add_option(
"--no-time",
dest="notime",
action="store_true",
help="Disable TIME+",
default=False,
)
# --no-wait
group.add_option("--no-wait",
dest="nowait",
action="store_true",
help="Disable W",
default=False)
# --no-app-name
group.add_option(
"--no-app-name",
dest="noappname",
action="store_true",
help="Disable App",
default=False,
)
# --hide-queries-in-logs
group.add_option(
"--hide-queries-in-logs",
dest="hide_queries_in_logs",
action="store_true",
help=
"Disable log_min_duration_statements and log_min_duration_sample for pg_activity",
default=False,
)
parser.add_option_group(group)
return parser
def add_options(self, parser):
"""
Populate option parse with options available for this command
"""
group = OptionGroup(parser, "Global Options")
group.add_option("--logfile",
metavar="FILE",
help="log file. if omitted stderr will be used")
group.add_option("-L",
"--loglevel",
metavar="LEVEL",
default=None,
help="log level (default: %s)" %
self.settings['LOG_LEVEL'])
group.add_option("--nolog",
action="store_true",
help="disable logging completely")
group.add_option("--profile",
metavar="FILE",
default=None,
help="write python cProfile stats to FILE")
group.add_option("--pidfile",
metavar="FILE",
help="write process ID to FILE")
group.add_option("-s",
"--set",
action="append",
default=[],
metavar="NAME=VALUE",
help="set/override setting (may be repeated)")
group.add_option("--pdb",
action="store_true",
help="enable pdb on failure")
parser.add_option_group(group)
def parser(mp=None):
if mp == None: mp = OptionParser()
mpTransfer = OptionGroup(mp, cyan + "Transfer function options" + plain)
mpTransfer.add_option('--mvaBins',
help='mva bins: var#3#1;3;6;9,...',
type='str',
action='callback',
callback=optsplitdict)
mpTransfer.add_option('--catTags',
help='tags for categories',
type='str',
action='callback',
callback=optsplit,
default=[])
mpTransfer.add_option('--fBound',
help='fit MIN and MAX',
type='str',
action='callback',
callback=optsplit,
default=['90', '200'])
mpTransfer.add_option('--complexWghts',
help='Wght info.',
type='str',
action='callback',
callback=optsplitmore,
default=[])
mpTransfer.add_option('--typetag',
help='Type tag (Dat,QCD,Z,T).',
type='str',
default='Data')
mpTransfer.add_option('--unblind',
help='No blind window.',
action='store_true',
default=False)
mpTransfer.add_option('--merge',
help='Merge higher CATs.',
action='store_true',
default=False)
mp.add_option_group(mpTransfer)
return mp
if __name__ == "__main__":
# Import Psyco if available
try:
import psyco
psyco.full()
print >> sys.stderr, "Found Psyco, using"
except ImportError:
print >> sys.stderr, "Psyco not installed"
from optparse import OptionParser, OptionGroup
optparser = OptionParser(
description="Train a new event/relation extraction model")
# main options
group = OptionGroup(optparser, "Main Options", "")
group.add_option("-t",
"--task",
default=None,
dest="task",
help="task number")
group.add_option("-p",
"--parse",
default="McCC",
dest="parse",
help="Parse XML element name")
group.add_option("-c",
"--connection",
default=None,
dest="connection",
help="")
