def main():
usage = "usage: %prog [options] gold_amr_file test_amr_file\nBoth AMR files are one-amr-per-line.\n Try %prog --help for more options."
parser = OptionParser(usage=usage)
parser.add_option(
"-m",
"--method",
dest="method",
default="root",
help=
"Set method to initialize the variable alignment in each hillclimbing restart.\n 'random': Randomly map variables.\n'concept': Random mapping between variables with the same concept\n'root'(default): Random mapping, roots always mapped.\n'optimistic': Assume AMRs have the same structure and map variables. Do not restart.",
metavar="METHOD")
parser.add_option("-d",
"--detailed",
action="store_true",
dest="detailed",
default=False,
help="Print scores per sentence.")
parser.add_option(
"-r",
"--restarts",
action="store",
dest="restarts",
default="10",
metavar="RESTARTS",
help=
"Restart hill climbing RESTARTS time. 1 means single start. Default: 10."
)
parser.add_option(
"-t",
"--threshold",
action="store",
dest="threshold",
default=1.0,
metavar="VALUE",
help=
"Do not restart hill climbing after F-Score reaches VALUE in [0.0-1.0]. Default: 1.0"
)
parser.add_option(
"-c",
"--concept_edges",
action="store_true",
dest="concept_edges",
default=False,
help=
"Do not restart hill climbing after F-Score reaches this threshold [0.0-1.0]"
)
parser.add_option(
"-i",
"--ignore-missing",
action="store_true",
dest="missing",
default=False,
help=
"Ignore line pairs with empty test AMR when computing overall Smatch.")
group = OptionGroup(
parser, "Dangerous Options",
"Caution: These options are believed to bite! Use at your own risk.")
group.add_option(
"-p",
"--precise",
action="store_true",
dest="precise",
default=False,
help="Compute precise SMATCH score. Do not use this option!")
parser.add_option_group(group)
(options, args) = parser.parse_args()
if len(args) != 2:
print parser.get_usage()
sys.exit(1)
try:
restarts = int(options.restarts)
if restarts <= 0:
raise ValueError
except:
sys.stderr.write(
"ERROR: Invalid number of restarts. Must be an integer > 0.\n")
print parser.get_usage()
sys.exit(1)
try:
threshold = float(options.threshold)
if threshold < 0.0 or threshold > 1.0:
raise ValueError / tmp / gold.amr
except:
sys.stderr.write(
"ERROR: Invalid restart threshold. Must be a float 0.0<=i<=1.0\n")
print parser.get_usage()
sys.exit(1)
concept_edges = options.concept_edges
missing = options.missing
precise = options.precise
detailed = options.detailed
if precise:
sys.stderr.write(
"WARNING: Computing precise SMATCH (i.e. graph isomorphism!). This can take very long!\n"
)
method = options.method
method_map = {
'random': get_random_start,
'concept': get_concept_match_start,
'root': get_root_align_start,
'optimistic': get_parallel_start
}
if not method in method_map:
sys.stderr.write(
"ERROR: -m METHOD must be one of the following: random, concept, root, optimistic.\n"
)
print parser.get_usage()
sys.exit(1)
if method == "optimistic":
print "Using optimistic initializer. Setting number of restarts to 1."
restarts = 1
compute_smatch_batch(args[0],
args[1],
starts=restarts,
method=get_random_start,
restart_threshold=threshold,
concept_edges=concept_edges,
precise=precise,
missing=missing,
detailed=detailed)
def _init():
'''
Parse CLI options.
'''
parser = OptionParser()
parser.add_option('--platform',
dest='platform',
help='Platform (\'os\' grain)')
parser.add_option('--log-level',
dest='log_level',
default='warning',
help='Control verbosity of logging. Default: %default')
# All arguments dealing with file paths (except for platform-specific ones
# like those for SPEC files) should be placed in this group so that
# relative paths are properly expanded.
path_group = OptionGroup(parser, 'File/Directory Options')
path_group.add_option('--source-dir',
default='/testing',
help='Source directory. Must be a git checkout. '
'(default: %default)')
path_group.add_option('--build-dir',
default='/tmp/salt-buildpackage',
help='Build root, will be removed if it exists '
'prior to running script. (default: %default)')
path_group.add_option('--artifact-dir',
default='/tmp/salt-packages',
help='Location where build artifacts should be '
'placed for Jenkins to retrieve them '
'(default: %default)')
parser.add_option_group(path_group)
# This group should also consist of nothing but file paths, which will be
# normalized below.
rpm_group = OptionGroup(parser, 'RPM-specific File/Directory Options')
rpm_group.add_option('--spec',
dest='spec_file',
default='/tmp/salt.spec',
help='Spec file to use as a template to build RPM. '
'(default: %default)')
parser.add_option_group(rpm_group)
opts = parser.parse_args()[0]
# Expand any relative paths
for group in (path_group, rpm_group):
for path_opt in [opt.dest for opt in group.option_list]:
path = getattr(opts, path_opt)
if not os.path.isabs(path):
# Expand ~ or ~user
path = os.path.expanduser(path)
if not os.path.isabs(path):
# Still not absolute, resolve '..'
path = os.path.realpath(path)
# Update attribute with absolute path
setattr(opts, path_opt, path)
# Sanity checks
problems = []
if not opts.platform:
problems.append('Platform (\'os\' grain) required')
if not os.path.isdir(opts.source_dir):
problems.append('Source directory {0} not found'
.format(opts.source_dir))
try:
shutil.rmtree(opts.build_dir)
except OSError as exc:
if exc.errno not in (errno.ENOENT, errno.ENOTDIR):
problems.append('Unable to remove pre-existing destination '
'directory {0}: {1}'.format(opts.build_dir, exc))
finally:
try:
os.makedirs(opts.build_dir)
except OSError as exc:
problems.append('Unable to create destination directory {0}: {1}'
.format(opts.build_dir, exc))
try:
shutil.rmtree(opts.artifact_dir)
except OSError as exc:
if exc.errno not in (errno.ENOENT, errno.ENOTDIR):
problems.append('Unable to remove pre-existing artifact directory '
'{0}: {1}'.format(opts.artifact_dir, exc))
finally:
try:
os.makedirs(opts.artifact_dir)
except OSError as exc:
problems.append('Unable to create artifact directory {0}: {1}'
.format(opts.artifact_dir, exc))
# Create log file in the artifact dir so it is sent back to master if the
# job fails
opts.log_file = os.path.join(opts.artifact_dir, 'salt-buildpackage.log')
if problems:
_abort(problems)
return opts
def create_configuration(**kwargs):
"""Creates and returns a valid config object.
It merges cli arguments, actual parameters of this function and
key/value pairs of a given config file.
Used in conjunction with L{validate_configuration}
Precedence of origin in descending order:
- Actual keyword arguments
- command line arguments
- config file values
In addition this function sets the global configuration object.
@type protocol_dir: str
@keyword protocol_dir: the directory of the protocol file to process
@type port: int
@keyword port: port number
@type host: str
@keyword host: host to connect to/ listen on
@type analyze: bool
@keyword analyze: only run analyze phase
@type client: bool
@keyword client: if True tasty runs as client, otherwise in server mode
@type threads: bool
@keyword threads: number of threads/processes to use
@type filter_callgraph: bool
@keyword filter_callgraph: If True only shows relevant information,
otherwise shows all calling graph nodes
@type security_level: "ultra-short" | "short" | "medium" | "long"
@keyword security_level: ultra-short/short/medium/long-term security
@type ot_type: "Paillier" | "EC" | "EC_c"
@keyword ot_type: type of OT protocol
@type ot_chain: list
@keyword ot_chain: list of used OT protocols
@type use_driver: bool
@param use_driver: if True, and a procedure with the signature
driver(client, server) is present in precified tasty protocol, tasty will
execute the that method instead of the original run procedure, and creates a
graph of the result
@type symmetric_security_parameter: int
@keyword symmetric_security_parameter: bit length of
@type asymmetric_security_parameter: int
@keyword asymmetric_security_parameter: number of threads/processes to use
@type compiler_warnings_are_errors: bool
@keyword compiler_warnings_are_errors: turns warnings into errors
@type homomorphic_type: HomomorphicType
@type homomorphic_type: the actual implementation of HomomorphicType
@type testing: bool
@keyword testing: set this to True if you create a test config and want
to relax config var validation
@rtype: instance
@return: instance of an old style class providing values as attributes
"""
#if state.config:
#warnings.warn("Reconfiguring tasty. Hopefully you know what you are doing", UserWarningOnce)
usage = "usage: %%prog [options] %s" % os.path.join(
"path", "to", "protocol", "directory")
global g_parser
parser = g_parser = OptionParser(usage=usage, version="%%prog %s" % tasty.__version__)
parser.add_option("-I", "--info",
action="store_true",
dest="info",
default=False,
help="show program information")
net_opts = OptionGroup(parser, "network options")
mode_opts = OptionGroup(parser, "operational options")
protocol_opts = OptionGroup(parser, "protocol options")
compiler_opts = OptionGroup(parser, "compiler options")
net_opts.add_option("-p", "--port",
action="store",
dest="port",
type="int",
help="port number")
net_opts.add_option("-H", "--host",
action="store",
dest="host",
help="host address")
#mode_opts.add_option("-L", "--log-file",
#action="store",
#dest="log_file",
#help="explicitly specify the complete log file path and name (default=protocol_dir/results/tasty.log)")
mode_opts.add_option("-c", "--client",
action="store_true",
dest="client",
default=False,
help="run in client mode")
mode_opts.add_option("-s", "--server",
action="store_true",
dest="server",
default=False,
help="run in server mode")
mode_opts.add_option("-F", "--forever",
action="store_true",
dest="serve_forever",
default=False,
help="run until killed by user (in conjunction with both options '--server' or '--client'")
mode_opts.add_option("-t", "--threads",
action="store",
dest="threads",
type="int",
default=1,
help="number of cpus to use. Not fully used yet!")
mode_opts.add_option("-C", "--color",
action="store",
dest="color",
choices=COLOR_CHOICES,
default=None,
help="Output color. Especially when tasty puts its output into the same terminal, its nice to have a different color for each party."
" Choose one of (n)one, (r)ed, (g)reen, (y)ellow, (c)yan, (m)agenta")
mode_opts.add_option("-v", "--verbose",
action="count",
dest="verbose",
default=0,
help="debug output level, can be repeated to increase verbosity")
mode_opts.add_option("-f", "--force-protocol",
action="store_true",
dest="force_protocol",
default=False,
help="force usage of given protocol and send it to other party if protocols differ. If both parties use this, it will be set to False.")
mode_opts.add_option("-A", "--accept-protocol",
action="store_true",
dest="accept_protocol",
default=False,
help="if other party wants to force you using its protocol version tasty denyies and exits with error. This flag makes this party accept forced protocols. Warning: Only activate this flag, if you are controlling both parties and accept that this is a potential security thread.")
mode_opts.add_option("--test_mode",
action="store_true",
dest="test_mode",
default=False,
help="Internal flag!!! This flag must not be used directly, but should be set for functional tests.")
mode_opts.add_option("--client-fail-if-no-server",
action="store_false",
dest="client_waiting",
default=True,
help="makes the client to fail if no server is reachable. Default behaviour is waiting for a server to come up.")
protocol_opts.add_option("-l", "--security_level",
action="store",
dest="security_level",
type="string",
default="short",
help="security level can be either 'ultra-short', 'short' (default), 'medium' or 'long'")
protocol_opts.add_option("-d", "--driver",
action="store_true",
dest="use_driver",
default=False,
help="turn tasty into driver/batch mode")
protocol_opts.add_option("-P", "--protocol_name",
action="store",
dest="protocol_name",
default="protocol",
help="name of actual protocol method to run. default='protocol'?")
protocol_opts.add_option("-D", "--driver_name",
action="store",
dest="driver_name",
help="name of actual Driver implementation to use if more than one provided.?")
protocol_opts.add_option("-O", "--oblivious_transfer",
action="store",
dest="ot_chain",
default=False,
help="overwrite oblivious chain")
#compiler_opts.add_option("-E", "--exclude_compiler",
#action="store_true",
#dest="exclude_compiler",
#default=False,
#help="only run compiled protocol without compiling it")
compiler_opts.add_option("-a", "--analyze",
action="store_true",
dest="analyze",
default=False,
help="showing costs on exit")
#compiler_opts.add_option("-u", "--find_unused",
#action="store_true",
#dest="find_unused",
#default=False,
#help="searches for potentially unused attributes and constants. For now there are many false positives so this is optionally")
compiler_opts.add_option("--homomorphic_type",
action="store",
dest="homomorphic_type",
default=types.Paillier,
help="sets the homomorphic type to use")
compiler_opts.add_option("--homomorphic_vec_type",
action="store",
dest="homomorphic_vec_type",
default=types.PaillierVec,
help="sets the homomorphic vec type to use")
compiler_opts.add_option("--WError", action="store_true",
help="turn warnings into errors", dest="compiler_warnings_are_errors", default=False)
parser.add_option_group(compiler_opts)
parser.add_option_group(mode_opts)
parser.add_option_group(net_opts)
parser.add_option_group(protocol_opts)
configuration, args = parser.parse_args()
#if len(sys.argv) <= 1:
#parser.print_help()
#sys.exit(0)
if configuration.info:
sys.exit(0)
if "verbose" in kwargs:
configuration.verbose = kwargs["verbose"]
if "protocol_dir" in kwargs:
configuration.protocol_dir = os.path.abspath(kwargs["protocol_dir"])
else:
try:
configuration.protocol_dir = os.path.abspath(args[0])
except IndexError:
pass
if "host" in kwargs:
configuration.host = kwargs["host"]
if "port" in kwargs:
configuration.port = kwargs["port"]
if "client" in kwargs:
configuration.client = kwargs["client"]
if "threads" in kwargs:
configuration.threads = kwargs["threads"]
if "analyze" in kwargs:
configuration.analyze = kwargs["analyze"]
if "ot_chain" in kwargs:
configuration.ot_chain = kwargs["ot_chain"]
if "color" in kwargs:
if kwargs["color"] not in COLOR_CHOICES:
raise OptionValueError("unknown color %r specified" % kwargs["color"])
configuration.color = COLOR[kwargs["color"]]
if "verbose" in kwargs:
configuration.ot_chain = kwargs["verbose"]
if "client-fail-if-no-server" in kwargs:
configuration.client_waiting = kwargs["client-fail-if-no-server"]
if "security_level" in kwargs:
configuration.security_level = kwargs["security_level"]
if "asymmetric_security_parameter" in kwargs:
configuration.asymmetric_security_parameter = kwargs["asymmetric_security_parameter"]
if "symmetric_security_parameter" in kwargs:
configuration.symmetric_security_parameter = kwargs["symmetric_security_parameter"]
if "compiler_warnings_are_errors" in kwargs:
configuration.compiler_warnings_are_errors = kwargs["compiler_warnings_are_errors"]
if "test_mode" in kwargs:
configuration.test_mode = kwargs["test_mode"]
log_name = "tasty_%s.log" % ("client" if configuration.client else "server")
try:
configuration.log_file = os.path.join(configuration.protocol_dir, "results", log_name)
except AttributeError:
parser.print_help()
sys.exit(0)
if not configuration.client_waiting and configuration.server:
raise ValueError("specifying client waiting option makes only sence for clients!")
if configuration.color == "n" or sys.platform == "win32":
state.color = ""
state.color_reset = ""
elif configuration.server:
state.party_name = "server"
if not configuration.color:
# make server log output blue, for client it's already set to green
state.color = "\033[35;1m"
if configuration.color:
state.color = COLOR[configuration.color]
# important - don't delete that
state.config = configuration
return configuration
def main():
usage = "usage: %prog [options] command [command parameters]"
description = """
%prog [options] upload ARTIFACT_FILE_TO_UPLOAD [ARTIFACT_UUID]
%prog [options] update ARTIFACT_UUID
%prog [options] delete ARTIFACT_UUID
%prog [options] info ARTIFACT_UUID
%prog [options] link ARTIFACT_UUID OBJECT_UUID
%prog [options] unlink ARTIFACT_UUID OBJECT_UUID
%prog [options] list [LIMIT]
%prog [options] download ARTIFACT_UUID ARTIFACT_FILE_TO_SAVE
"""
op = OptionParser(description=description,
usage=usage,
formatter=formatter())
op.add_option("-v",
"--verbose",
default=0,
action="count",
help="enable verbose mode")
op.add_option("-p",
"--pt-server-url",
default="http://127.0.0.1:9000",
help="perftracker url, default %default")
og = OptionGroup(op, "'upload' and 'update' options")
og.add_option("-d", "--description", help="artifact description (i")
og.add_option("-m",
"--mime",
default=None,
help="artifact mime type, default is guessed or "
"'application/octet-stream'")
og.add_option("-f",
"--filename",
help="override artifact file name by given name")
og.add_option("-z",
"--compression",
action="store_true",
help="inline decompression on every file view or "
"download")
og.add_option("-i",
"--inline",
default=False,
action="store_true",
help="inline view in browser "
"(do not download on click)")
og.add_option("-t",
"--ttl",
default=180,
help="time to live (days), default=%default, 0 - infinite")
op.add_option_group(og)
opts, args = op.parse_args()
loglevel = logging.DEBUG if opts.verbose >= 2 else (
logging.INFO if opts.verbose == 1 else logging.WARNING)
logging.basicConfig(
level=loglevel,
format="%(asctime)s - %(module)17s - %(levelname).3s - %(message)s",
datefmt='%H:%M:%S')
def abort(msg=None):
op.print_usage()
print(description)
if msg:
print("error: %s" % msg)
sys.exit(-1)
try:
run(opts, args, abort)
except ptRuntimeException as e:
logging.error(str(e))
sys.exit(-1)
def main():
from optparse import OptionParser, OptionGroup
parser = OptionParser(
"%prog --size SIZE1 --square SQUARE1 [ --size SIZE2 --square SQUARE2 ]",
description=None)
parser.add_option(
"-c",
"--camera_name",
type="string",
default='narrow_stereo',
help="name of the camera to appear in the calibration file")
group = OptionGroup(
parser, "Chessboard Options",
"You must specify one or more chessboards as pairs of --size and --square options."
)
group.add_option(
"-p",
"--pattern",
type="string",
default="chessboard",
help=
"calibration pattern to detect - 'chessboard', 'circles', 'acircles'")
group.add_option(
"-s",
"--size",
action="append",
default=[],
help=
"chessboard size as NxM, counting interior corners (e.g. a standard chessboard is 7x7)"
)
group.add_option("-q",
"--square",
action="append",
default=[],
help="chessboard square size in meters")
parser.add_option_group(group)
group = OptionGroup(parser, "ROS Communication Options")
group.add_option(
"--approximate",
type="float",
default=0.0,
help=
"allow specified slop (in seconds) when pairing images from unsynchronized stereo cameras"
)
group.add_option(
"--no-service-check",
action="store_false",
dest="service_check",
default=True,
help="disable check for set_camera_info services at startup")
parser.add_option_group(group)
group = OptionGroup(parser, "Calibration Optimizer Options")
group.add_option("--fix-principal-point",
action="store_true",
default=False,
help="fix the principal point at the image center")
group.add_option("--fix-aspect-ratio",
action="store_true",
default=False,
help="enforce focal lengths (fx, fy) are equal")
group.add_option(
"--zero-tangent-dist",
action="store_true",
default=False,
help="set tangential distortion coefficients (p1, p2) to zero")
group.add_option(
"-k",
"--k-coefficients",
type="int",
default=2,
metavar="NUM_COEFFS",
help=
"number of radial distortion coefficients to use (up to 6, default %default)"
)
group.add_option(
"--disable_calib_cb_fast_check",
action='store_true',
default=False,
help="uses the CALIB_CB_FAST_CHECK flag for findChessboardCorners")
parser.add_option_group(group)
options, args = parser.parse_args()
if len(options.size) != len(options.square):
parser.error("Number of size and square inputs must be the same!")
if not options.square:
options.square.append("0.108")
options.size.append("8x6")
boards = []
for (sz, sq) in zip(options.size, options.square):
size = tuple([int(c) for c in sz.split('x')])
boards.append(ChessboardInfo(size[0], size[1], float(sq)))
if options.approximate == 0.0:
sync = message_filters.TimeSynchronizer
else:
sync = functools.partial(ApproximateTimeSynchronizer,
slop=options.approximate)
num_ks = options.k_coefficients
calib_flags = 0
if options.fix_principal_point:
calib_flags |= cv2.CALIB_FIX_PRINCIPAL_POINT
if options.fix_aspect_ratio:
calib_flags |= cv2.CALIB_FIX_ASPECT_RATIO
if options.zero_tangent_dist:
calib_flags |= cv2.CALIB_ZERO_TANGENT_DIST
if (num_ks > 3):
calib_flags |= cv2.CALIB_RATIONAL_MODEL
if (num_ks < 6):
calib_flags |= cv2.CALIB_FIX_K6
if (num_ks < 5):
calib_flags |= cv2.CALIB_FIX_K5
if (num_ks < 4):
calib_flags |= cv2.CALIB_FIX_K4
if (num_ks < 3):
calib_flags |= cv2.CALIB_FIX_K3
if (num_ks < 2):
calib_flags |= cv2.CALIB_FIX_K2
if (num_ks < 1):
calib_flags |= cv2.CALIB_FIX_K1
pattern = Patterns.Chessboard
if options.pattern == 'circles':
pattern = Patterns.Circles
elif options.pattern == 'acircles':
pattern = Patterns.ACircles
elif options.pattern != 'chessboard':
print('Unrecognized pattern %s, defaulting to chessboard' %
options.pattern)
if options.disable_calib_cb_fast_check:
checkerboard_flags = 0
else:
checkerboard_flags = cv2.CALIB_CB_FAST_CHECK
rospy.init_node('cameracalibrator')
node = OpenCVCalibrationNode(boards,
options.service_check,
sync,
calib_flags,
pattern,
options.camera_name,
checkerboard_flags=checkerboard_flags)
rospy.spin()
from __future__ import with_statement
import logging
from optparse import OptionParser, OptionGroup
import sys
from gnotty import __version__, __version_string__, __url__
parser = OptionParser(usage="%prog [options]", version=__version__)
options = OptionGroup(parser, "")
options.add_option("-a", "--http-host", dest="HTTP_HOST", metavar="HOST",
default="127.0.0.1",
help="HTTP host address to serve from [default: %default]")
options.add_option("-p", "--http-port", dest="HTTP_PORT", metavar="PORT",
default=8080, type=int,
help="HTTP port to serve from [default: %default]")
options.add_option("-A", "--irc-host", dest="IRC_HOST", metavar="HOST",
default="irc.freenode.net",
help="IRC host address to connect to [default: %default]")
options.add_option("-P", "--irc-port", dest="IRC_PORT", metavar="PORT",
default=6667, type=int,
help="IRC port to connect to [default: %default]")
options.add_option("-C", "--irc-channel", dest="IRC_CHANNEL",
metavar="CHANNEL", default="#gnotty",
help="IRC channel to join [default: %default]")
options.add_option("-K", "--irc-channel-key", dest="IRC_CHANNEL_KEY",
metavar="CHANNEL_KEY", default="",
help="Optional key required to access the IRC channel")
options.add_option("-c", "--bot-class", dest="BOT_CLASS",
metavar="DOTTED_PYTHON_PATH",
if debug_printlevel > 0: traceback.print_tb(tb)
sys.stderr.write('# %s: %s\n' % (exception_type.__name__, exception_value))
sys.stderr.write('# Goodbye.\n')
sys.exit(1)
sys.excepthook = excepthook # replace sys.excepthook with my definition above
class WeightedHistogramError(Exception):
""" For errors in this script """
parser = OptionParser()
group = OptionGroup(parser, 'Input Options', 'Specify input variables')
group.add_option('-d',
'--density-of-states',
default=None,
dest='density',
metavar='FILE',
help='File containing the density of states.')
group.add_option('-p',
'--property-file',
default=None,
dest='property',
metavar='FILE',
help='File containing synchronized time-' +
'series of the property of interest with the energy.')
group.add_option('-t',
'--temperature',
def main():
from optparse import OptionParser, OptionGroup
parser = OptionParser("usage: %prog [options]")
group = OptionGroup(parser, "Input Options")
group.add_option("",
"--source-root",
dest="source_root",
metavar="PATH",
help="Path to the LLVM source (inferred if not given)",
action="store",
default=None)
group.add_option(
"",
"--llvmbuild-source-root",
dest="llvmbuild_source_root",
help=("If given, an alternate path to search for LLVMBuild.txt files"),
action="store",
default=None,
metavar="PATH")
group.add_option("",
"--build-root",
dest="build_root",
metavar="PATH",
help="Path to the build directory (if needed) [%default]",
action="store",
default=None)
parser.add_option_group(group)
group = OptionGroup(parser, "Output Options")
group.add_option("",
"--print-tree",
dest="print_tree",
help="Print out the project component tree [%default]",
action="store_true",
default=False)
group.add_option("",
"--write-llvmbuild",
dest="write_llvmbuild",
help="Write out the LLVMBuild.txt files to PATH",
action="store",
default=None,
metavar="PATH")
group.add_option("",
"--write-library-table",
dest="write_library_table",
metavar="PATH",
help="Write the C++ library dependency table to PATH",
action="store",
default=None)
group.add_option("",
"--write-cmake-fragment",
dest="write_cmake_fragment",
metavar="PATH",
help="Write the CMake project information to PATH",
action="store",
default=None)
group.add_option("",
"--write-cmake-exports-fragment",
dest="write_cmake_exports_fragment",
metavar="PATH",
help="Write the CMake exports information to PATH",
action="store",
default=None)
group.add_option("",
"--write-make-fragment",
dest="write_make_fragment",
metavar="PATH",
help="Write the Makefile project information to PATH",
action="store",
default=None)
group.add_option("",
"--configure-target-def-file",
dest="configure_target_def_files",
help="""Configure the given file at SUBPATH (relative to
the inferred or given source root, and with a '.in' suffix) by replacing certain
substitution variables with lists of targets that support certain features (for
example, targets with AsmPrinters) and write the result to the build root (as
given by --build-root) at the same SUBPATH""",
metavar="SUBPATH",
action="append",
default=None)
parser.add_option_group(group)
group = OptionGroup(parser, "Configuration Options")
group.add_option("",
"--native-target",
dest="native_target",
metavar="NAME",
help=("Treat the named target as the 'native' one, if "
"given [%default]"),
action="store",
default=None)
group.add_option("",
"--enable-targets",
dest="enable_targets",
metavar="NAMES",
help=("Enable the given space or semi-colon separated "
"list of targets, or all targets if not present"),
action="store",
default=None)
group.add_option("",
"--enable-optional-components",
dest="optional_components",
metavar="NAMES",
help=("Enable the given space or semi-colon separated "
"list of optional components"),
action="store",
default="")
parser.add_option_group(group)
(opts, args) = parser.parse_args()
# Determine the LLVM source path, if not given.
source_root = opts.source_root
if source_root:
if not os.path.exists(
os.path.join(source_root, 'lib', 'IR', 'Function.cpp')):
parser.error('invalid LLVM source root: %r' % source_root)
else:
llvmbuild_path = os.path.dirname(__file__)
llvm_build_path = os.path.dirname(llvmbuild_path)
utils_path = os.path.dirname(llvm_build_path)
source_root = os.path.dirname(utils_path)
if not os.path.exists(
os.path.join(source_root, 'lib', 'IR', 'Function.cpp')):
parser.error('unable to infer LLVM source root, please specify')
# Construct the LLVM project information.
llvmbuild_source_root = opts.llvmbuild_source_root or source_root
project_info = LLVMProjectInfo.load_from_path(source_root,
llvmbuild_source_root)
# Add the magic target based components.
add_magic_target_components(parser, project_info, opts)
# Validate the project component info.
project_info.validate_components()
# Print the component tree, if requested.
if opts.print_tree:
project_info.print_tree()
# Write out the components, if requested. This is useful for auto-upgrading
# the schema.
if opts.write_llvmbuild:
project_info.write_components(opts.write_llvmbuild)
# Write out the required library table, if requested.
if opts.write_library_table:
project_info.write_library_table(opts.write_library_table,
opts.optional_components)
# Write out the make fragment, if requested.
if opts.write_make_fragment:
project_info.write_make_fragment(opts.write_make_fragment,
opts.optional_components)
# Write out the cmake fragment, if requested.
if opts.write_cmake_fragment:
project_info.write_cmake_fragment(opts.write_cmake_fragment,
opts.optional_components)
if opts.write_cmake_exports_fragment:
project_info.write_cmake_exports_fragment(
opts.write_cmake_exports_fragment, opts.optional_components)
# Configure target definition files, if requested.
if opts.configure_target_def_files:
# Verify we were given a build root.
if not opts.build_root:
parser.error("must specify --build-root when using "
"--configure-target-def-file")
# Create the substitution list.
available_targets = [
ci for ci in project_info.component_infos
if ci.type_name == 'TargetGroup'
]
substitutions = [
("@LLVM_ENUM_TARGETS@", ' '.join('LLVM_TARGET(%s)' % ci.name
for ci in available_targets)),
("@LLVM_ENUM_ASM_PRINTERS@",
' '.join('LLVM_ASM_PRINTER(%s)' % ci.name
for ci in available_targets if ci.has_asmprinter)),
("@LLVM_ENUM_ASM_PARSERS@",
' '.join('LLVM_ASM_PARSER(%s)' % ci.name
for ci in available_targets if ci.has_asmparser)),
("@LLVM_ENUM_DISASSEMBLERS@",
' '.join('LLVM_DISASSEMBLER(%s)' % ci.name
for ci in available_targets if ci.has_disassembler))
]
# Configure the given files.
for subpath in opts.configure_target_def_files:
inpath = os.path.join(source_root, subpath + '.in')
outpath = os.path.join(opts.build_root, subpath)
result = configutil.configure_file(inpath, outpath, substitutions)
if not result:
note("configured file %r hasn't changed" % outpath)
sys.stdout = Unbuffered(sys.stdout)
# -------------------------------------------------------------------------------
# Files extensions to encrypt
encrypt_extensions = ('.py', )
# Encrypted files extension
encrypt_output_extension = '.enc'
# Encryption key encoded to base64.
encrypt_key = b'U0cxdGJVbDBTWE5NYjI5clRHbHJaVXRsZVE9PQ=='
from optparse import OptionParser, OptionGroup
parser = OptionParser(usage='%prog [options] arg1',
version='Simple encrypt / decrypt file or directory')
group = OptionGroup(parser, 'Test information options:')
group.add_option('--dir',
dest='dir',
default=None,
help='Directory or file for encrypt/decrypt')
group.add_option('--decrypt',
dest='decrypt',
action="store_true",
default=False,
help='Decrypt file or directory. False by default.')
parser.add_option_group(group)
options, args = parser.parse_args()
if not options.dir:
parser.print_help()
print(
parser.add_option("-t",
'--threads',
action="store",
dest="threads",
default=1,
help='number of threads running [default to %default]')
parser.add_option(
"-m",
'--watermarker',
action="store_false",
dest="watermarker",
help='print out water marker [default to not print water marker]')
##parameters for htqc
group = OptionGroup(parser, 'Advanced Options', 'Parameters for htqc')
group.add_option('-w',
action='store',
metavar='INT',
dest='htqc_W',
default=5,
help='window size for quality trim [default: %default]')
group.add_option('-C',
action='store',
metavar='INT',
dest='htqc_C',
default=20,
help='quality threshold for trim [default: %default]')
group.add_option('-L',
action='store',
metavar='INT',
def main():
from optparse import OptionParser, OptionGroup
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage, version="%prog 01")
parser.add_option(
"-r", "--rma", dest="rma_dataset", type="string", default=None,
help="Specify the RMA-normalized data to analyze.")
parser.add_option(
"-m", "--mas5", dest="mas5_dataset", type="string", default=None,
help="Specify the MAS5-normalized data to analyze.")
parser.add_option(
"-i", "--illu", dest="illu_dataset", type="string", default=None,
help="Specify the Illumina data to analyze.")
parser.add_option(
"", "--sigdb_path", dest="sigdb_path", type="string", default=None,
help="Location of the sigdb/ directory.")
parser.add_option(
"", "--sigtag", dest="signature_tags", default=[], action="append",
help="Specify a specific tag to use.")
parser.add_option(
"", "--sigid", dest="signature_ids", default=[], action="append",
help="Specify a specific signature to use.")
parser.add_option(
"", "--max_signatures", dest="max_signatures", type="int",
default=None,
help="Maximum number of signatures to run (for DEBUGGING).")
parser.add_option(
"-j", "", dest="num_procs", type="int", default=1,
help="Number of jobs to run in parallel.")
parser.add_option(
"-z", "", dest="archive", action="store_true", default=False,
help="Archive the individual signatures. Helpful for GenePattern.")
parser.add_option(
"", "--libpath", dest="libpath", action="append", default=[],
help="Add to the Python library search path.")
parser.add_option(
"-o", "--outpath", dest="outpath", type="string", default=None,
help="Save files in this path.")
parser.add_option(
"", "--gp_imod_all_vars", dest="gp_imod_all_vars", type="string",
default=None,
help="Special internal variable for use with GenePattern "
"interactive modules.")
parser.add_option(
"", "--debug_gp_imod_all_vars", action="store_true", default=False,
dest="debug_gp_imod_all_vars",
)
#group = OptionGroup(parser, "Normalization")
#group.add_option(
# "", "--normalization", dest="normalization", default="MAS5",
# help="How was the data set normalized (default MAS5).")
#group.add_option(
# "-l", "--log_data", dest="log_data", action="store_true",
# default=False,
# help="Log the MAS5 data before analyzing.")
#parser.add_option_group(group)
group = OptionGroup(parser, "Pybinreg")
group.add_option(
"", "--python", dest="python", default=None,
help="Specify the command to run python.")
group.add_option(
"", "--matlab", dest="matlab", default=None,
help="Specify the command to run matlab.")
group.add_option(
"", "--povray", dest="povray", default=None,
help="Specify the command to run povray.")
group.add_option(
"", "--cluster", dest="cluster", default=None,
help="Specify the command to run cluster.")
group.add_option(
"", "--binreg", dest="binreg_path", default=None,
help="Specify the path to the BinReg2.0 code.")
group.add_option(
"", "--pybinreg", dest="pybinreg", default=None,
help="Specify the command to run pybinreg.py.")
group.add_option(
"", "--arrayplot", dest="arrayplot", default=None,
help="Specify the command to run arrayplot.")
parser.add_option_group(group)
options, args = parser.parse_args()
#if len(args) < 1:
# #print sys.argv
# #print len(args), args
# parser.error("Please specify sigdb_path.")
#elif len(args) > 1:
# parser.error("Too many arguments.")
if args:
parser.error("Too many arguments.")
# DEBUG the gp_imod_all_vars variable.
if options.debug_gp_imod_all_vars:
assert not options.gp_imod_all_vars
options.gp_imod_all_vars = (
"mas5_expression_file_cb=file&mas5_expression_file_url=&"
"rma_expression_file_cb=file&rma_expression_file_url=&"
# Skip AKT signature.
"sig_AKT=no&"
# Change BCAT normalization.
"sig_BCAT=yes (custom parameters)&"
"sig_BCAT_apply_quantile_normalization=no&"
"sig_BCAT_apply_shiftscale_normalization=no&"
"sig_BCAT_num_genes=85&sig_BCAT_num_metagenes=2&"
# No changes in E2F1.
"sig_E2F1=yes (custom parameters)&"
"sig_E2F1_apply_quantile_normalization=yes&"
"sig_E2F1_apply_shiftscale_normalization=yes&"
"sig_E2F1_num_genes=150&sig_E2F1_num_metagenes=2&"
# Change genes in EGFR.
"sig_EGFR=yes (custom parameters)&"
"sig_EGFR_apply_quantile_normalization=no&"
"sig_EGFR_apply_shiftscale_normalization=yes&"
#"sig_EGFR_num_genes=50000&sig_EGFR_num_metagenes=2&"
"sig_EGFR_num_genes=501&sig_EGFR_num_metagenes=2&"
# Change quantile, genes, metagenes in ER.
"sig_ER=yes (custom parameters)&"
"sig_ER_apply_quantile_normalization=no&"
"sig_ER_apply_shiftscale_normalization=yes&"
"sig_ER_num_genes=150&sig_ER_num_metagenes=3&"
"sig_HER2=yes (default parameters)&"
"sig_IFNalpha=yes (default parameters)&"
"sig_IFNgamma=yes (default parameters)&"
"sig_MYC=yes (default parameters)&"
"sig_P53=yes (default parameters)&"
"sig_P63=yes (default parameters)&"
"sig_PI3K=yes (default parameters)&"
"sig_PR=yes (default parameters)&"
"sig_RAS=yes (default parameters)&"
"sig_SRC=yes (default parameters)&"
"sig_STAT3=yes (default parameters)&"
"sig_TGFB=yes (default parameters)&"
"sig_TNFa=yes (default parameters)&"
"which_signatures=I choose myself"
)
datafile_rma = datafile_mas5 = datafile_illu = None
if options.rma_dataset is not None:
assert os.path.exists(options.rma_dataset), \
"RMA file not found: %s" % options.rma_dataset
datafile_rma = os.path.realpath(options.rma_dataset)
if options.mas5_dataset is not None:
assert os.path.exists(options.mas5_dataset), \
"MAS5 file not found: %s" % options.mas5_dataset
datafile_mas5 = os.path.realpath(options.mas5_dataset)
if options.illu_dataset is not None:
assert os.path.exists(options.illu_dataset), \
"ILLU file not found: %s" % options.illu_dataset
datafile_illu = os.path.realpath(options.illu_dataset)
assert datafile_rma or datafile_mas5 or datafile_illu, \
"Please specify at least one data set."
if options.libpath:
sys.path = options.libpath + sys.path
# Import after the library path is set.
import time
import arrayio
from genomicode import config
from genomicode import parallel
from genomicode import archive
from genomicode import hashlib
from genomicode import matrixlib
from genomicode import genepattern
#sigdb_path, = args
x = options.sigdb_path or config.sigdb_path
sigdb_path = os.path.realpath(x)
assert os.path.exists(sigdb_path), \
"I could not find the signatures database: %s." % sigdb_path
start_time = time.time()
genepattern.fix_environ_path()
file_layout = make_file_layout(options.outpath)
init_paths(file_layout)
# Read the signatures and select the ones to score.
# BUG: Should allow this to be specified on the command line.
desired_tags = ["Pathway"] # default
if options.signature_tags:
desired_tags = options.signature_tags[:]
all_normalization = ["RMA", "MAS5", "ILLU"]
desired_normalization = []
if datafile_rma is not None: # RMA datafile is specified.
desired_normalization.append("RMA")
if datafile_mas5 is not None: # MAS5 datafile is specified.
desired_normalization.append("MAS5")
if datafile_illu is not None: # ILLU datafile is specified.
desired_normalization.append("ILLU")
# If any signature IDs are specified, then use only those IDs and
# ignore the desired tags.
print "Reading signature database: %s." % sigdb_path
desired_ids = []
if options.signature_ids:
desired_ids = options.signature_ids[:]
x = read_signatures(
sigdb_path, all_normalization, desired_ids, desired_tags)
signatures = x
orig_signatures = signatures[:]
# Filter for just the normalization that we have data files for.
# Keep track of why we filtered out certain signatures.
why_dropped = {} # ID -> explanation as string
good = []
for sig in signatures:
if sig.Normalization.upper() in desired_normalization:
good.append(sig)
continue
x = "Signature requires %s normalized data, but it was not provided."%(
sig.Normalization.upper())
why_dropped[sig.xID] = x
signatures = good
assert signatures, "No signatures available."
# Process additional parameters from GenePattern.
# o Do this before max_signatures, so that the maximum signatures
# is selected only out of the ones that the user specified.
# o Do this before names and paths, so the variables will be
# aligned.
# gp_imod_all_vars can be None or "".
if options.gp_imod_all_vars:
x = process_gp_imod_all_vars(
options.gp_imod_all_vars, signatures, why_dropped)
signatures, why_dropped = x
sys.stdout.flush()
DATA_rma = DATA_mas5 = DATA_illu = None
if datafile_rma is not None:
print "Reading RMA file: %s" % datafile_rma
DATA_rma = arrayio.read(datafile_rma)
DATA_rma = arrayio.convert(DATA_rma, to_format=arrayio.gct_format)
if datafile_mas5 is not None:
print "Reading MAS5 file: %s" % datafile_mas5
DATA_mas5 = arrayio.read(datafile_mas5)
DATA_mas5 = arrayio.convert(DATA_mas5, to_format=arrayio.gct_format)
if datafile_illu is not None:
print "Reading ILLU file: %s" % datafile_illu
DATA_illu = arrayio.read(datafile_illu)
DATA_illu = arrayio.convert(DATA_illu, to_format=arrayio.gct_format)
# Don't handle the log. Let pybinreg do it.
# Make sure the data sets contain the same samples. Align them if
# necessary.
DATA_all = [
("DATA_rma", DATA_rma), ("DATA_mas5", DATA_mas5),
("DATA_illu", DATA_illu)]
DATA_all = [x for x in DATA_all if x[1]]
for i in range(1, len(DATA_all)):
key1, data1 = DATA_all[0]
key2, data2 = DATA_all[i]
assert key1 != key2
assert data1 and data2
assert data1.ncol() == data2.ncol(), \
"%s and %s data sets have different numbers of samples." % (
key1, key2)
if matrixlib.are_cols_aligned(data1, data2):
continue
x = matrixlib.align_cols(data1, data2)
data1_new, data2_new = x
assert matrixlib.are_cols_aligned(data1_new, data2_new)
# The samples in data1 (the reference) should not be changed.
assert data1.ncol() == data1_new.ncol(), \
"%s and %s data sets have different samples" % (
key1, key2)
assert matrixlib.are_cols_aligned(data1, data1_new)
DATA_all[i] = key2, data2_new
for key, data in DATA_all:
if key == "DATA_rma":
DATA_rma = data
elif key == "DATA_mas5":
DATA_mas5 = data
elif key == "DATA_illu":
DATA_illu = data
else:
raise AssertionError, "Unknown key: %s" % key
print "Writing aligned signal files."
if DATA_rma:
arrayio.gct_format.write(
DATA_rma, open(file_layout.DATASET_RMA, 'w'))
if DATA_mas5:
arrayio.gct_format.write(
DATA_mas5, open(file_layout.DATASET_MAS5, 'w'))
if DATA_illu:
arrayio.gct_format.write(
DATA_illu, open(file_layout.DATASET_ILLU, 'w'))
# Figure out the names and paths for each signature.
print "Finding signatures."
names = [None] * len(signatures) # SIG19_AKT[_modified]
paths = [None] * len(signatures) # <path>/SIG19_AKT[_modified]
for i, sig in enumerate(signatures):
name = "SIG%02d_%s" % (sig.xID, hashlib.hash_var(sig.Name))
# If the user has modified the signature from the default
# parameters, then make a note of it.
if getattr(sig, "Changed", False):
name = "%s_modified" % name
outpath = os.path.join(file_layout.OUTPATH, name)
names[i] = name
paths[i] = outpath
if options.max_signatures is not None:
signatures = signatures[:options.max_signatures]
# Make a list of the jobs.
jobs = [] # list of cmd, outpath, outfile
for i, sig in enumerate(signatures):
name, outpath = names[i], paths[i]
#print "Generating signature %s [%d:%d]" % (
# name, i+1, len(signatures))
#sys.stdout.flush()
quantile_normalize = False
assert sig.Quantile.upper() in ["YES", "NO"]
if sig.Quantile.upper() == "YES":
quantile_normalize = True
shift_scale_normalize = False
assert sig.Shift_Scale.upper() in ["YES", "NO"]
if sig.Shift_Scale.upper() == "YES":
shift_scale_normalize = True
#outfile = os.path.join(files.outpath, "%s.out.txt" % name)
outfile = os.path.join(outpath, "out.txt")
if sig.Normalization.upper() == "RMA":
datafile = file_layout.DATASET_RMA
assert DATA_rma
elif sig.Normalization.upper() == "MAS5":
datafile = file_layout.DATASET_MAS5
assert DATA_mas5
elif sig.Normalization.upper() == "ILLU":
datafile = file_layout.DATASET_ILLU
assert DATA_illu
else:
raise AssertionError, "Unknown normalization."
# If the entire analysis should be archived, then go ahead and
# archive each of the pybinreg runs too. This will prevent
# large analyses from taking up too much disk space. The
# drawback is that the files that are archived are no longer
# available for use here. Hopefully this won't be a problem.
cmd = make_pybinreg_cmd(
options.pybinreg, options.python, options.binreg_path,
options.matlab, options.arrayplot, options.povray,
options.cluster, options.libpath,
outpath, options.archive, sig.Genes, sig.Metagenes,
quantile_normalize, shift_scale_normalize,
sig.Train0, sig.Train1, datafile)
x = cmd, outpath, outfile
jobs.append(x)
# Run each of the jobs.
if options.num_procs < 1 or options.num_procs > 100:
parser.error("Please specify between 1 and 100 processes.")
if options.num_procs > 1:
if parallel._find_parallel():
num_sigs = min(options.num_procs, len(jobs))
if num_sigs > 1:
print "Predicting %d signatures at a time." % num_sigs
else:
print("I could not find GNU parallel. "
"Predicting 1 signature at a time.")
options.num_procs = 1
sys.stdout.flush()
DEBUG = False # Can disable pybinreg temporarily for debugging.
if not DEBUG:
if options.num_procs <= 1:
for x in jobs:
cmd, outpath, outfile = x
run_one_pybinreg(cmd, outpath, outfile)
else:
run_many_pybinreg(jobs, options.num_procs)
if signatures:
print "Extracting the reports from each signature."
report_files = extract_reports(names, paths, file_layout)
print "Combining probabilities from each of the signatures."
summarize_probabilities(signatures, names, paths, file_layout)
print "Making heatmap of the results."
sys.stdout.flush()
summarize_heatmap(
options.python, options.arrayplot, options.cluster,
options.libpath, file_layout)
print "Summarizing signatures."
summarize_signatures(signatures, file_layout)
print "Making a report."
analysis_name = make_analysis_name(options)
summarize_report(
analysis_name, signatures, orig_signatures, report_files,
start_time, why_dropped, file_layout)
if options.archive:
print "Compressing results."
sys.stdout.flush()
archive.zip_path(file_layout.ATTIC)
for i, sig in enumerate(signatures):
name, outpath = names[i], paths[i]
archive.zip_path(outpath)
print "Done."
def parser():
usage = "usage: %prog -h|--help \n"\
" %prog [--raw|--stat] elflet-address\n"\
" %prog [-p PASSWORD] -j FILE elflet-address\n"\
" %prog [-p PASSWORD] [options] elflet-address"
p = OptionParser(usage)
# for get mode
g = OptionGroup(p, 'Options for retrieving information')
g.add_option('--raw',
action='store_true',
dest='rawOutput',
help='show configuration as raw json data')
g.add_option('--stat',
action='store_true',
dest='stat',
help='show statistics data')
p.add_option_group(g)
# for updagte mode
g = OptionGroup(p, 'Options for changing configuration')
g.add_option('-p',
'--password',
action='store',
dest='password',
help="specify a admin password of elflet, "
"password will be asked interactively "
"if you ommit this option")
g.add_option('-j',
'--json',
action='store',
dest='jsonFile',
metavar='FILE',
help="specify a JSON file to reflect to elflet configuration")
g.add_option('-n',
'--name',
action='store',
dest='name',
help="change elflet node name")
g.add_option('-t',
'--timezone',
action='store',
dest='tz',
help="change timezone")
g.add_option('-N',
'--ntp',
action='store',
dest='ntp',
help="change NTP server address")
g.add_option('-m',
'--mode',
action='store',
dest='mode',
choices=['full', 'sensor-only'],
help="change elflet function mode, MODE must be specified "
"eather 'full' or 'sensor-only'")
g.add_option('-s',
'--sensor-interval',
action='store',
dest='sensorInterval',
metavar='ITNERVAL',
type='int',
help="change sensor capturing interval time in second")
g.add_option('-i',
'--irmode',
action='store',
dest='irmode',
metavar='MODE',
choices=['on-demand', 'continuous'],
help="change IR receiver mode, MODE must be specified "
"eather 'on-demand' or 'continuous'")
g.add_option('-b',
'--enable-ble-keyboard',
action='store_true',
dest='blehid',
help="enable BLE keyboard emurator function")
g.add_option('-B',
'--disable-ble-keyboard',
action='store_false',
dest='blehid',
help="disable BLE keyboard emurator function")
g.add_option('-w',
'--enable-wifi',
action='store_true',
dest='wifi',
help="enable WiFi connectivity")
g.add_option('-W',
'--disable-wifi',
action='store_false',
dest='wifi',
help="disable WiFi connectivity, "
"this option can be specified if BLE keyboard "
"emurator is enabled (-b) and button is in 'BLEHID' mode")
p.add_option_group(g)
g = OptionGroup(p, 'Button settings Options')
g.add_option(
'--button-mode',
action='store',
dest='buttonMode',
metavar='MODE',
choices=['remote-receiver', 'blehid'],
help=
"change funciton vnvoked when a button is pressed, MODE must be specified "
"eather 'remote-receiver' or 'blehid'")
g.add_option(
'--button-keycode',
action='store',
dest='buttonKeyCode',
metavar='CODE',
help="set HID key combination sending when button is pressed. "
"Key combination is specified as key code stream separated by commas, "
"such as '31,32,33'. "
"If modifier code such as shift key is specified, "
"add logical sum of modifier code wihth '+' at the beggining, "
"such as '3+31,32,33'.")
g.add_option('--button-consumercode',
action='store',
dest='buttonConsumerCode',
metavar='CODE',
type='int',
help="set HID consumer code sending when button is pressed. ")
g.add_option(
'--button-code-duration',
action='store',
dest='buttonDuration',
metavar='DURATION',
type='int',
help="change keypress duration in milli second for sending HID "
"code when button is pressed.")
p.add_option_group(g)
g = OptionGroup(p, 'MQTT PubSub related Options')
g.add_option('--pubsub-server',
action='store',
dest='pubsubServer',
metavar='SERVER',
help="change PubSub server address "
"and enable PubSub function, "
"PubSub function will be disabled "
"if zero length string is specified")
g.add_option('--pubsub-session',
action='store',
dest='pubsubSession',
metavar='TYPE',
choices=['TCP', 'TSL', 'WebSocket', 'WebSocketSecure'],
help="change type of session with PubSub server, "
"TYPE must be specified 'TCP', 'TSL', 'WebSocket' or "
"'WebSocketSecure'")
g.add_option('--pubsub-user',
action='store',
dest='pubsubUser',
metavar='USER:PASS',
help="change user name and password to use authentication "
"in PubSub server")
g.add_option('--pubsub-server-cert',
action='store',
dest='pubsubCert',
metavar='FILE',
help="change certification of PubSub server, "
"this is necessory when session type is "
"'TSL' or 'WebSocketSecure'")
g.add_option('--sensor-topic',
action='store',
dest='sensorTopic',
metavar='TOPIC',
help="change topic name to publish sensor data")
g.add_option('--shadow-topic',
action='store',
dest='shadowTopic',
metavar='TOPIC',
help="change topic name to publish shadow state")
g.add_option('--irrc-received-data-topic',
action='store',
dest='irrcReceivedDataTopic',
metavar='TOPIC',
help="change topic name to publish received IR data")
g.add_option('--irrc-receive-topic',
action='store',
dest='irrcReceiveTopic',
metavar='TOPIC',
help="change topic name to subscribe request to "
"transit IR receiving mode")
g.add_option('--irrc-send-topic',
action='store',
dest='irrcSendTopic',
metavar='TOPIC',
help="change topic name to subscribe transmitting IR data")
g.add_option('--download-firmware-topic',
action='store',
dest='downloadFirmwareTopic',
metavar='TOPIC',
help="change topic name to subscribe request "
"to start firmware download")
p.add_option_group(g)
return p
def parser(mp=None):
if not mp: mp = OptionParser()
# mp.add_option('','',help=colours[5]+''+colours[0],default='',type='',dest='')
#
# mg1 = OptionGroup(mp,'GroupTitle')
# mg1.add_option('','',help=colours[5]+''+colours[0],default='',type='',dest='')
# mp.add_option_group(mg1)
#
mp.add_option('--workdir',
help=colours[5] + 'Case workdir.' + colours[0],
default='case0',
type='str')
mp.add_option('--long',
help=colours[5] + 'Long filenames.' + colours[0],
default=False,
action='store_true')
mp.add_option('-v',
'--verbosity',
help=colours[5] + 'Verbosity.' + colours[0],
default=0,
action='count')
mp.add_option('-q',
'--quiet',
help=colours[5] + 'Quiet.' + colours[0],
default=True,
action='store_false')
#
mg1 = OptionGroup(mp, 'Selection setup')
mg1.add_option(
'--SELCATs',
help=colours[5] +
'Selection/Category setup: "NOM;NOM;-0.6#0.0#0.7#0.84#1.0,VBF;PRK;-0.1#0.4#0.8#1.0,...".'
+ colours[0],
default='NOM;NOM;-0.6#0.0#0.7#0.84#1.0,VBF;PRK;-0.1#0.4#0.8#1.0',
type='str',
action='callback',
callback=SELsetup,
dest='SC')
mp.add_option_group(mg1)
#
mg2 = OptionGroup(mp, 'Bkg template setup')
mg2.add_option('--bounds',
help=colours[5] + 'Template boundaries: 80,200 (min,max)' +
colours[0],
default=[80., 200.],
type='str',
action='callback',
callback=optsplitfloat,
dest='X')
mg2.add_option('--binsize',
help=colours[5] + 'Template NBINS: 20,20 (NOM,VBF)' +
colours[0],
default=[20, 20],
type='str',
action='callback',
callback=optsplitint,
dest='NBINS')
mg2.add_option('--lumi',
help=colours[5] + 'Luminosity: 19784.,18281. (NOM,VBF)' +
colours[0],
default=[19784., 18281.],
type='str',
action='callback',
callback=optsplitfloat,
dest='LUMI')
mp.add_option_group(mg2)
#
return mp
from __future__ import print_function
from PhysicsTools.HeppyCore.utils.edmIntegrityCheck import PublishToFileSystem, IntegrityCheck
import das
import copy, os
if __name__ == '__main__':
from optparse import OptionParser, OptionGroup
usage = """usage: %prog [options] /Sample/Name/On/Castor
e.g.: %prog -u wreece -p -w 'PFAOD_*.root' /MultiJet/Run2011A-05Aug2011-v1/AOD/V2
"""
das = das.DASOptionParser(usage=usage)
group = OptionGroup(das.parser, 'edmIntegrityCheck Options',
'Options related to checking files on CASTOR')
group.add_option("-d",
"--device",
dest="device",
default='cmst3',
help="The storage device to write to, e.g. 'cmst3'")
group.add_option(
"-n",
"--name",
dest="name",
default=None,
help='The name of the dataset in DAS. Will be guessed if not specified'
)
group.add_option("-p",
"--printout",
def cmdLineParser(argv=None):
"""
This function parses the command line parameters and arguments
"""
if not argv:
argv = sys.argv
checkSystemEncoding()
_ = getUnicode(os.path.basename(argv[0]), encoding=sys.getfilesystemencoding())
usage = "%s%s [options]" % ("python " if not IS_WIN else "", \
"\"%s\"" % _ if " " in _ else _)
parser = OptionParser(usage=usage)
try:
parser.add_option("--hh", dest="advancedHelp",
action="store_true",
help="Show advanced help message and exit")
parser.add_option("--version", dest="showVersion",
action="store_true",
help="Show program's version number and exit")
parser.add_option("-v", dest="verbose", type="int",
help="Verbosity level: 0-6 (default %d)" % defaults.verbose)
# Target options
target = OptionGroup(parser, "Target", "At least one of these "
"options has to be provided to define the target(s)")
target.add_option("-d", dest="direct", help="Connection string "
"for direct database connection")
target.add_option("-u", "--url", dest="url", help="Target URL (e.g. \"http://www.site.com/vuln.php?id=1\")")
target.add_option("-l", dest="logFile", help="Parse target(s) from Burp "
"or WebScarab proxy log file")
target.add_option("-x", dest="sitemapUrl", help="Parse target(s) from remote sitemap(.xml) file")
target.add_option("-m", dest="bulkFile", help="Scan multiple targets given "
"in a textual file ")
target.add_option("-r", dest="requestFile",
help="Load HTTP request from a file")
target.add_option("-g", dest="googleDork",
help="Process Google dork results as target URLs")
target.add_option("-c", dest="configFile",
help="Load options from a configuration INI file")
# Request options
request = OptionGroup(parser, "Request", "These options can be used "
"to specify how to connect to the target URL")
request.add_option("--method", dest="method",
help="Force usage of given HTTP method (e.g. PUT)")
request.add_option("--data", dest="data",
help="Data string to be sent through POST")
request.add_option("--param-del", dest="paramDel",
help="Character used for splitting parameter values")
request.add_option("--cookie", dest="cookie",
help="HTTP Cookie header value")
request.add_option("--cookie-del", dest="cookieDel",
help="Character used for splitting cookie values")
request.add_option("--load-cookies", dest="loadCookies",
help="File containing cookies in Netscape/wget format")
request.add_option("--drop-set-cookie", dest="dropSetCookie",
action="store_true",
help="Ignore Set-Cookie header from response")
request.add_option("--user-agent", dest="agent",
help="HTTP User-Agent header value")
request.add_option("--random-agent", dest="randomAgent",
action="store_true",
help="Use randomly selected HTTP User-Agent header value")
request.add_option("--host", dest="host",
help="HTTP Host header value")
request.add_option("--referer", dest="referer",
help="HTTP Referer header value")
request.add_option("-H", "--header", dest="header",
help="Extra header (e.g. \"X-Forwarded-For: 127.0.0.1\")")
request.add_option("--headers", dest="headers",
help="Extra headers (e.g. \"Accept-Language: fr\\nETag: 123\")")
request.add_option("--auth-type", dest="authType",
help="HTTP authentication type "
"(Basic, Digest, NTLM or PKI)")
request.add_option("--auth-cred", dest="authCred",
help="HTTP authentication credentials "
"(name:password)")
request.add_option("--auth-file", dest="authFile",
help="HTTP authentication PEM cert/private key file")
request.add_option("--ignore-401", dest="ignore401", action="store_true",
help="Ignore HTTP Error 401 (Unauthorized)")
request.add_option("--proxy", dest="proxy",
help="Use a proxy to connect to the target URL")
request.add_option("--proxy-cred", dest="proxyCred",
help="Proxy authentication credentials "
"(name:password)")
request.add_option("--proxy-file", dest="proxyFile",
help="Load proxy list from a file")
request.add_option("--ignore-proxy", dest="ignoreProxy", action="store_true",
help="Ignore system default proxy settings")
request.add_option("--tor", dest="tor",
action="store_true",
help="Use Tor anonymity network")
request.add_option("--tor-port", dest="torPort",
help="Set Tor proxy port other than default")
request.add_option("--tor-type", dest="torType",
help="Set Tor proxy type (HTTP (default), SOCKS4 or SOCKS5)")
request.add_option("--check-tor", dest="checkTor",
action="store_true",
help="Check to see if Tor is used properly")
request.add_option("--delay", dest="delay", type="float",
help="Delay in seconds between each HTTP request")
request.add_option("--timeout", dest="timeout", type="float",
help="Seconds to wait before timeout connection "
"(default %d)" % defaults.timeout)
request.add_option("--retries", dest="retries", type="int",
help="Retries when the connection timeouts "
"(default %d)" % defaults.retries)
request.add_option("--randomize", dest="rParam",
help="Randomly change value for given parameter(s)")
request.add_option("--safe-url", dest="safeUrl",
help="URL address to visit frequently during testing")
request.add_option("--safe-post", dest="safePost",
help="POST data to send to a safe URL")
request.add_option("--safe-req", dest="safeReqFile",
help="Load safe HTTP request from a file")
request.add_option("--safe-freq", dest="safeFreq", type="int",
help="Test requests between two visits to a given safe URL")
request.add_option("--skip-urlencode", dest="skipUrlEncode",
action="store_true",
help="Skip URL encoding of payload data")
request.add_option("--csrf-token", dest="csrfToken",
help="Parameter used to hold anti-CSRF token")
request.add_option("--csrf-url", dest="csrfUrl",
help="URL address to visit to extract anti-CSRF token")
request.add_option("--force-ssl", dest="forceSSL",
action="store_true",
help="Force usage of SSL/HTTPS")
request.add_option("--hpp", dest="hpp",
action="store_true",
help="Use HTTP parameter pollution method")
request.add_option("--eval", dest="evalCode",
help="Evaluate provided Python code before the request (e.g. \"import hashlib;id2=hashlib.md5(id).hexdigest()\")")
# Optimization options
optimization = OptionGroup(parser, "Optimization", "These "
"options can be used to optimize the "
"performance of sqlmap")
optimization.add_option("-o", dest="optimize",
action="store_true",
help="Turn on all optimization switches")
optimization.add_option("--predict-output", dest="predictOutput", action="store_true",
help="Predict common queries output")
optimization.add_option("--keep-alive", dest="keepAlive", action="store_true",
help="Use persistent HTTP(s) connections")
optimization.add_option("--null-connection", dest="nullConnection", action="store_true",
help="Retrieve page length without actual HTTP response body")
optimization.add_option("--threads", dest="threads", type="int",
help="Max number of concurrent HTTP(s) "
"requests (default %d)" % defaults.threads)
# Injection options
injection = OptionGroup(parser, "Injection", "These options can be "
"used to specify which parameters to test "
"for, provide custom injection payloads and "
"optional tampering scripts")
injection.add_option("-p", dest="testParameter",
help="Testable parameter(s)")
injection.add_option("--skip", dest="skip",
help="Skip testing for given parameter(s)")
injection.add_option("--skip-static", dest="skipStatic", action="store_true",
help="Skip testing parameters that not appear dynamic")
injection.add_option("--dbms", dest="dbms",
help="Force back-end DBMS to this value")
injection.add_option("--dbms-cred", dest="dbmsCred",
help="DBMS authentication credentials (user:password)")
injection.add_option("--os", dest="os",
help="Force back-end DBMS operating system "
"to this value")
injection.add_option("--invalid-bignum", dest="invalidBignum",
action="store_true",
help="Use big numbers for invalidating values")
injection.add_option("--invalid-logical", dest="invalidLogical",
action="store_true",
help="Use logical operations for invalidating values")
injection.add_option("--invalid-string", dest="invalidString",
action="store_true",
help="Use random strings for invalidating values")
injection.add_option("--no-cast", dest="noCast",
action="store_true",
help="Turn off payload casting mechanism")
injection.add_option("--no-escape", dest="noEscape",
action="store_true",
help="Turn off string escaping mechanism")
injection.add_option("--prefix", dest="prefix",
help="Injection payload prefix string")
injection.add_option("--suffix", dest="suffix",
help="Injection payload suffix string")
injection.add_option("--tamper", dest="tamper",
help="Use given script(s) for tampering injection data")
# Detection options
detection = OptionGroup(parser, "Detection", "These options can be "
"used to customize the detection phase")
detection.add_option("--level", dest="level", type="int",
help="Level of tests to perform (1-5, "
"default %d)" % defaults.level)
detection.add_option("--risk", dest="risk", type="int",
help="Risk of tests to perform (1-3, "
"default %d)" % defaults.level)
detection.add_option("--string", dest="string",
help="String to match when "
"query is evaluated to True")
detection.add_option("--not-string", dest="notString",
help="String to match when "
"query is evaluated to False")
detection.add_option("--regexp", dest="regexp",
help="Regexp to match when "
"query is evaluated to True")
detection.add_option("--code", dest="code", type="int",
help="HTTP code to match when "
"query is evaluated to True")
detection.add_option("--text-only", dest="textOnly",
action="store_true",
help="Compare pages based only on the textual content")
detection.add_option("--titles", dest="titles",
action="store_true",
help="Compare pages based only on their titles")
# Techniques options
techniques = OptionGroup(parser, "Techniques", "These options can be "
"used to tweak testing of specific SQL "
"injection techniques")
techniques.add_option("--technique", dest="tech",
help="SQL injection techniques to use "
"(default \"%s\")" % defaults.tech)
techniques.add_option("--time-sec", dest="timeSec",
type="int",
help="Seconds to delay the DBMS response "
"(default %d)" % defaults.timeSec)
techniques.add_option("--union-cols", dest="uCols",
help="Range of columns to test for UNION query SQL injection")
techniques.add_option("--union-char", dest="uChar",
help="Character to use for bruteforcing number of columns")
techniques.add_option("--union-from", dest="uFrom",
help="Table to use in FROM part of UNION query SQL injection")
techniques.add_option("--dns-domain", dest="dnsName",
help="Domain name used for DNS exfiltration attack")
techniques.add_option("--second-order", dest="secondOrder",
help="Resulting page URL searched for second-order "
"response")
# Fingerprint options
fingerprint = OptionGroup(parser, "Fingerprint")
fingerprint.add_option("-f", "--fingerprint", dest="extensiveFp",
action="store_true",
help="Perform an extensive DBMS version fingerprint")
# Enumeration options
enumeration = OptionGroup(parser, "Enumeration", "These options can "
"be used to enumerate the back-end database "
"management system information, structure "
"and data contained in the tables. Moreover "
"you can run your own SQL statements")
enumeration.add_option("-a", "--all", dest="getAll",
action="store_true", help="Retrieve everything")
enumeration.add_option("-b", "--banner", dest="getBanner",
action="store_true", help="Retrieve DBMS banner")
enumeration.add_option("--current-user", dest="getCurrentUser",
action="store_true",
help="Retrieve DBMS current user")
enumeration.add_option("--current-db", dest="getCurrentDb",
action="store_true",
help="Retrieve DBMS current database")
enumeration.add_option("--hostname", dest="getHostname",
action="store_true",
help="Retrieve DBMS server hostname")
enumeration.add_option("--is-dba", dest="isDba",
action="store_true",
help="Detect if the DBMS current user is DBA")
enumeration.add_option("--users", dest="getUsers", action="store_true",
help="Enumerate DBMS users")
enumeration.add_option("--passwords", dest="getPasswordHashes",
action="store_true",
help="Enumerate DBMS users password hashes")
enumeration.add_option("--privileges", dest="getPrivileges",
action="store_true",
help="Enumerate DBMS users privileges")
enumeration.add_option("--roles", dest="getRoles",
action="store_true",
help="Enumerate DBMS users roles")
enumeration.add_option("--dbs", dest="getDbs", action="store_true",
help="Enumerate DBMS databases")
enumeration.add_option("--tables", dest="getTables", action="store_true",
help="Enumerate DBMS database tables")
enumeration.add_option("--columns", dest="getColumns", action="store_true",
help="Enumerate DBMS database table columns")
enumeration.add_option("--schema", dest="getSchema", action="store_true",
help="Enumerate DBMS schema")
enumeration.add_option("--count", dest="getCount", action="store_true",
help="Retrieve number of entries for table(s)")
enumeration.add_option("--dump", dest="dumpTable", action="store_true",
help="Dump DBMS database table entries")
enumeration.add_option("--dump-all", dest="dumpAll", action="store_true",
help="Dump all DBMS databases tables entries")
enumeration.add_option("--search", dest="search", action="store_true",
help="Search column(s), table(s) and/or database name(s)")
enumeration.add_option("--comments", dest="getComments", action="store_true",
help="Retrieve DBMS comments")
enumeration.add_option("-D", dest="db",
help="DBMS database to enumerate")
enumeration.add_option("-T", dest="tbl",
help="DBMS database table(s) to enumerate")
enumeration.add_option("-C", dest="col",
help="DBMS database table column(s) to enumerate")
enumeration.add_option("-X", dest="excludeCol",
help="DBMS database table column(s) to not enumerate")
enumeration.add_option("-U", dest="user",
help="DBMS user to enumerate")
enumeration.add_option("--exclude-sysdbs", dest="excludeSysDbs",
action="store_true",
help="Exclude DBMS system databases when "
"enumerating tables")
enumeration.add_option("--where", dest="dumpWhere",
help="Use WHERE condition while table dumping")
enumeration.add_option("--start", dest="limitStart", type="int",
help="First query output entry to retrieve")
enumeration.add_option("--stop", dest="limitStop", type="int",
help="Last query output entry to retrieve")
enumeration.add_option("--first", dest="firstChar", type="int",
help="First query output word character to retrieve")
enumeration.add_option("--last", dest="lastChar", type="int",
help="Last query output word character to retrieve")
enumeration.add_option("--sql-query", dest="query",
help="SQL statement to be executed")
enumeration.add_option("--sql-shell", dest="sqlShell",
action="store_true",
help="Prompt for an interactive SQL shell")
enumeration.add_option("--sql-file", dest="sqlFile",
help="Execute SQL statements from given file(s)")
# Brute force options
brute = OptionGroup(parser, "Brute force", "These "
"options can be used to run brute force "
"checks")
brute.add_option("--common-tables", dest="commonTables", action="store_true",
help="Check existence of common tables")
brute.add_option("--common-columns", dest="commonColumns", action="store_true",
help="Check existence of common columns")
# User-defined function options
udf = OptionGroup(parser, "User-defined function injection", "These "
"options can be used to create custom user-defined "
"functions")
udf.add_option("--udf-inject", dest="udfInject", action="store_true",
help="Inject custom user-defined functions")
udf.add_option("--shared-lib", dest="shLib",
help="Local path of the shared library")
# File system options
filesystem = OptionGroup(parser, "File system access", "These options "
"can be used to access the back-end database "
"management system underlying file system")
filesystem.add_option("--file-read", dest="rFile",
help="Read a file from the back-end DBMS "
"file system")
filesystem.add_option("--file-write", dest="wFile",
help="Write a local file on the back-end "
"DBMS file system")
filesystem.add_option("--file-dest", dest="dFile",
help="Back-end DBMS absolute filepath to "
"write to")
# Takeover options
takeover = OptionGroup(parser, "Operating system access", "These "
"options can be used to access the back-end "
"database management system underlying "
"operating system")
takeover.add_option("--os-cmd", dest="osCmd",
help="Execute an operating system command")
takeover.add_option("--os-shell", dest="osShell",
action="store_true",
help="Prompt for an interactive operating "
"system shell")
takeover.add_option("--os-pwn", dest="osPwn",
action="store_true",
help="Prompt for an OOB shell, "
"Meterpreter or VNC")
takeover.add_option("--os-smbrelay", dest="osSmb",
action="store_true",
help="One click prompt for an OOB shell, "
"Meterpreter or VNC")
takeover.add_option("--os-bof", dest="osBof",
action="store_true",
help="Stored procedure buffer overflow "
"exploitation")
takeover.add_option("--priv-esc", dest="privEsc",
action="store_true",
help="Database process user privilege escalation")
takeover.add_option("--msf-path", dest="msfPath",
help="Local path where Metasploit Framework "
"is installed")
takeover.add_option("--tmp-path", dest="tmpPath",
help="Remote absolute path of temporary files "
"directory")
# Windows registry options
windows = OptionGroup(parser, "Windows registry access", "These "
"options can be used to access the back-end "
"database management system Windows "
"registry")
windows.add_option("--reg-read", dest="regRead",
action="store_true",
help="Read a Windows registry key value")
windows.add_option("--reg-add", dest="regAdd",
action="store_true",
help="Write a Windows registry key value data")
windows.add_option("--reg-del", dest="regDel",
action="store_true",
help="Delete a Windows registry key value")
windows.add_option("--reg-key", dest="regKey",
help="Windows registry key")
windows.add_option("--reg-value", dest="regVal",
help="Windows registry key value")
windows.add_option("--reg-data", dest="regData",
help="Windows registry key value data")
windows.add_option("--reg-type", dest="regType",
help="Windows registry key value type")
# General options
general = OptionGroup(parser, "General", "These options can be used "
"to set some general working parameters")
#general.add_option("-x", dest="xmlFile",
# help="Dump the data into an XML file")
general.add_option("-s", dest="sessionFile",
help="Load session from a stored (.sqlite) file")
general.add_option("-t", dest="trafficFile",
help="Log all HTTP traffic into a "
"textual file")
general.add_option("--batch", dest="batch",
action="store_true",
help="Never ask for user input, use the default behaviour")
general.add_option("--charset", dest="charset",
help="Force character encoding used for data retrieval")
general.add_option("--crawl", dest="crawlDepth", type="int",
help="Crawl the website starting from the target URL")
general.add_option("--crawl-exclude", dest="crawlExclude",
help="Regexp to exclude pages from crawling (e.g. \"logout\")")
general.add_option("--csv-del", dest="csvDel",
help="Delimiting character used in CSV output "
"(default \"%s\")" % defaults.csvDel)
general.add_option("--dump-format", dest="dumpFormat",
help="Format of dumped data (CSV (default), HTML or SQLITE)")
general.add_option("--eta", dest="eta",
action="store_true",
help="Display for each output the "
"estimated time of arrival")
general.add_option("--flush-session", dest="flushSession",
action="store_true",
help="Flush session files for current target")
general.add_option("--forms", dest="forms",
action="store_true",
help="Parse and test forms on target URL")
general.add_option("--fresh-queries", dest="freshQueries",
action="store_true",
help="Ignore query results stored in session file")
general.add_option("--hex", dest="hexConvert",
action="store_true",
help="Use DBMS hex function(s) for data retrieval")
general.add_option("--output-dir", dest="outputDir",
action="store",
help="Custom output directory path")
general.add_option("--parse-errors", dest="parseErrors",
action="store_true",
help="Parse and display DBMS error messages from responses")
general.add_option("--pivot-column", dest="pivotColumn",
help="Pivot column name")
general.add_option("--save", dest="saveConfig",
help="Save options to a configuration INI file")
general.add_option("--scope", dest="scope",
help="Regexp to filter targets from provided proxy log")
general.add_option("--test-filter", dest="testFilter",
help="Select tests by payloads and/or titles (e.g. ROW)")
general.add_option("--test-skip", dest="testSkip",
help="Skip tests by payloads and/or titles (e.g. BENCHMARK)")
general.add_option("--update", dest="updateAll",
action="store_true",
help="Update sqlmap")
# Miscellaneous options
miscellaneous = OptionGroup(parser, "Miscellaneous")
miscellaneous.add_option("-z", dest="mnemonics",
help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
miscellaneous.add_option("--alert", dest="alert",
help="Run host OS command(s) when SQL injection is found")
miscellaneous.add_option("--answers", dest="answers",
help="Set question answers (e.g. \"quit=N,follow=N\")")
miscellaneous.add_option("--beep", dest="beep", action="store_true",
help="Beep on question and/or when SQL injection is found")
miscellaneous.add_option("--cleanup", dest="cleanup",
action="store_true",
help="Clean up the DBMS from sqlmap specific "
"UDF and tables")
miscellaneous.add_option("--dependencies", dest="dependencies",
action="store_true",
help="Check for missing (non-core) sqlmap dependencies")
miscellaneous.add_option("--disable-coloring", dest="disableColoring",
action="store_true",
help="Disable console output coloring")
miscellaneous.add_option("--gpage", dest="googlePage", type="int",
help="Use Google dork results from specified page number")
miscellaneous.add_option("--identify-waf", dest="identifyWaf",
action="store_true",
help="Make a thorough testing for a WAF/IPS/IDS protection")
miscellaneous.add_option("--skip-waf", dest="skipWaf",
action="store_true",
help="Skip heuristic detection of WAF/IPS/IDS protection")
miscellaneous.add_option("--mobile", dest="mobile",
action="store_true",
help="Imitate smartphone through HTTP User-Agent header")
miscellaneous.add_option("--offline", dest="offline",
action="store_true",
help="Work in offline mode (only use session data)")
miscellaneous.add_option("--page-rank", dest="pageRank",
action="store_true",
help="Display page rank (PR) for Google dork results")
miscellaneous.add_option("--purge-output", dest="purgeOutput",
action="store_true",
help="Safely remove all content from output directory")
miscellaneous.add_option("--smart", dest="smart",
action="store_true",
help="Conduct thorough tests only if positive heuristic(s)")
miscellaneous.add_option("--sqlmap-shell", dest="sqlmapShell", action="store_true",
help="Prompt for an interactive sqlmap shell")
miscellaneous.add_option("--wizard", dest="wizard",
action="store_true",
help="Simple wizard interface for beginner users")
# Hidden and/or experimental options
parser.add_option("--dummy", dest="dummy", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--pickled-options", dest="pickledOptions",
help=SUPPRESS_HELP)
parser.add_option("--disable-precon", dest="disablePrecon", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--profile", dest="profile", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--binary-fields", dest="binaryFields",
help=SUPPRESS_HELP)
parser.add_option("--force-dns", dest="forceDns", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--force-threads", dest="forceThreads", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--smoke-test", dest="smokeTest", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--live-test", dest="liveTest", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--stop-fail", dest="stopFail", action="store_true",
help=SUPPRESS_HELP)
parser.add_option("--run-case", dest="runCase", help=SUPPRESS_HELP)
parser.add_option_group(target)
parser.add_option_group(request)
parser.add_option_group(optimization)
parser.add_option_group(injection)
parser.add_option_group(detection)
parser.add_option_group(techniques)
parser.add_option_group(fingerprint)
parser.add_option_group(enumeration)
parser.add_option_group(brute)
parser.add_option_group(udf)
parser.add_option_group(filesystem)
parser.add_option_group(takeover)
parser.add_option_group(windows)
parser.add_option_group(general)
parser.add_option_group(miscellaneous)
# Dirty hack to display longer options without breaking into two lines
def _(self, *args):
retVal = parser.formatter._format_option_strings(*args)
if len(retVal) > MAX_HELP_OPTION_LENGTH:
retVal = ("%%.%ds.." % (MAX_HELP_OPTION_LENGTH - parser.formatter.indent_increment)) % retVal
return retVal
parser.formatter._format_option_strings = parser.formatter.format_option_strings
parser.formatter.format_option_strings = type(parser.formatter.format_option_strings)(_, parser, type(parser))
# Dirty hack for making a short option -hh
option = parser.get_option("--hh")
option._short_opts = ["-hh"]
option._long_opts = []
# Dirty hack for inherent help message of switch -h
option = parser.get_option("-h")
option.help = option.help.capitalize().replace("this help", "basic help")
_ = []
prompt = False
advancedHelp = True
extraHeaders = []
for arg in argv:
_.append(getUnicode(arg, encoding=sys.getfilesystemencoding()))
argv = _
checkDeprecatedOptions(argv)
prompt = "--sqlmap-shell" in argv
if prompt:
parser.usage = ""
cmdLineOptions.sqlmapShell = True
_ = ["x", "q", "exit", "quit", "clear"]
for option in parser.option_list:
_.extend(option._long_opts)
_.extend(option._short_opts)
for group in parser.option_groups:
for option in group.option_list:
_.extend(option._long_opts)
_.extend(option._short_opts)
autoCompletion(AUTOCOMPLETE_TYPE.SQLMAP, commands=_)
while True:
command = None
try:
command = raw_input("sqlmap-shell> ").strip()
command = getUnicode(command, encoding=sys.stdin.encoding)
except (KeyboardInterrupt, EOFError):
print
raise SqlmapShellQuitException
if not command:
continue
elif command.lower() == "clear":
clearHistory()
print "[i] history cleared"
saveHistory(AUTOCOMPLETE_TYPE.SQLMAP)
elif command.lower() in ("x", "q", "exit", "quit"):
raise SqlmapShellQuitException
elif command[0] != '-':
print "[!] invalid option(s) provided"
print "[i] proper example: '-u http://www.site.com/vuln.php?id=1 --banner'"
else:
saveHistory(AUTOCOMPLETE_TYPE.SQLMAP)
loadHistory(AUTOCOMPLETE_TYPE.SQLMAP)
break
try:
for arg in shlex.split(command):
argv.append(getUnicode(arg, encoding=sys.stdin.encoding))
except ValueError, ex:
raise SqlmapSyntaxException, "something went wrong during command line parsing ('%s')" % ex.message
# Hide non-basic options in basic help case
for i in xrange(len(argv)):
if argv[i] == "-hh":
argv[i] = "-h"
elif re.search(r"\A-\w=.+", argv[i]):
print "[!] potentially miswritten (illegal '=') short option detected ('%s')" % argv[i]
elif argv[i] == "-H":
if i + 1 < len(argv):
extraHeaders.append(argv[i + 1])
elif re.match(r"\A\d+!\Z", argv[i]) and argv[max(0, i - 1)] == "--threads" or re.match(r"\A--threads.+\d+!\Z", argv[i]):
argv[i] = argv[i][:-1]
conf.skipThreadCheck = True
elif argv[i] == "--version":
print VERSION_STRING.split('/')[-1]
raise SystemExit
elif argv[i] == "-h":
advancedHelp = False
for group in parser.option_groups[:]:
found = False
for option in group.option_list:
if option.dest not in BASIC_HELP_ITEMS:
option.help = SUPPRESS_HELP
else:
found = True
if not found:
parser.option_groups.remove(group)
try:
(args, _) = parser.parse_args(argv)
except UnicodeEncodeError, ex:
print "\n[!] %s" % ex.object.encode("unicode-escape")
raise SystemExit
def _populate_parser(self):
super(Dispatcher, self)._populate_parser()
# don't do a whole lot with this
tcase = _YUVOutputComparisonFactory()
yuv_tests = [
name[5:] for name, meth in filter(
lambda pair: pair[0][:5] == "test_", inspect.getmembers(tcase))
]
group = OptionGroup(self.optparse, "x264 testing-specific options")
group.add_option("-v",
"--video",
metavar="FILENAME",
action="callback",
dest="video",
type=str,
callback=lambda option, opt, value, parser: setattr(
self, "video", value),
help="yuv video to perform testing on (default: %s)" %
self.video)
group.add_option(
"-s",
"--seed",
metavar="SEED",
action="callback",
dest="seed",
type=int,
callback=lambda option, opt, value, parser: setattr(
self, "seed", value),
help=
"seed for the random number generator (default: unix timestamp)")
group.add_option(
"-p",
"--product-tests",
metavar="NUM",
action="callback",
dest="video",
type=int,
callback=lambda option, opt, value, parser: setattr(
self, "products", value),
help=
"number of cartesian products to generate for yuv comparison testing (default: %d)"
% self.products)
group.add_option("--configure-with",
metavar="FLAGS",
action="callback",
dest="configure",
type=str,
callback=lambda option, opt, value, parser: setattr(
self, "configure", shlex.split(value)),
help="options to run ./configure with")
group.add_option(
"--yuv-tests",
action="callback",
dest="yuv_tests",
type=str,
callback=lambda option, opt, value, parser: setattr(
self, "yuv_tests", [val.strip() for val in value.split(",")]),
help=
"select tests to run with yuv comparisons (default: %s, available: %s)"
% (", ".join(self.yuv_tests), ", ".join(yuv_tests)))
group.add_option("--x264-with",
metavar="FLAGS",
action="callback",
dest="x264",
type=str,
callback=lambda option, opt, value, parser: setattr(
self, "x264", shlex.split(value)),
help="additional options to run ./x264 with")
self.optparse.add_option_group(group)
import sys
from collections import defaultdict as d
from optparse import OptionParser, OptionGroup
#Author: Martin Kapun
######################################################### HELP #########################################################################
usage = "python %prog --input input.sync > output.af"
parser = OptionParser(usage=usage)
group = OptionGroup(
parser, """
H E L P :
_________
Calculate allele frequencies of the alleles more common in the selected population than in the control populations. The script assumes that there is an equal number of selected and control in the input sync file and that the first half is comprised by the control and the second by the selected populations. The output contains the allelic states of the "Selected" and "Control" alleles (separated by a dash) in the third column
""")
######################################################### CODE #########################################################################
parser.add_option("--input", dest="input", help="input SYNC file")
(options, args) = parser.parse_args()
parser.add_option_group(group)
def sync2freqh(x):
''' convert string in SYNC format to dictionary of freqencies where x is a string in sync format'''
from collections import defaultdict as d
nuc = ["A", "T", "C", "G"]
counts = map(int, x.split(":")[:4])
if sum(counts) == 0:
action='store_false',
help='Set this to disable thumbnails. [Default: %default]')
parser.add_option(
'--snap-delay',
dest='snapshot_delay',
default=0,
type='int',
metavar='SECONDS',
help='Add n seconds for a delayed snapshot. [Default %default]')
parser.add_option(
'--thumbnails-only',
dest='thumbnails_only',
action='store_true',
default=False,
help='Set this to get only thumbnails. [Default: %default]')
log_group = OptionGroup(parser, ('These options handles logging options'))
log_group.add_option(
'--loglevel',
dest='loglevel',
default=0,
metavar='LEVEL',
type='string',
help=('Set this to see some messages. Possible Values are: ' +
'NOTSET, DEBUG, INFO, WARNING, ERROR, CRITICAL ' +
'[Default: NOTSET]'),
callback=loglevel_callback,
action='callback')
log_group.add_option(
'--no-logfile',
dest='disable_logfile',
action='store_true',
if retryIdx == numRetries:
raise retryException
print "Download complete."
if __name__ == '__main__':
def check_option(parser, value, name):
if None == value:
print 'Option ' + name + ' required.\n'
parser.print_help()
sys.exit(1)
parser = OptionParser()
group = OptionGroup(parser, "Credential options")
group.add_option('-K',
'--client-key',
help='the developer.basespace.illumina.com client key',
dest='clientKey',
default=None)
group.add_option('-S',
'--client-secret',
help='the developer.basespace.illumina.com client token',
dest='clientSecret',
default=None)
group.add_option('-A',
'--access-token',
help='the developer.basespace.illumina.com access token',
dest='accessToken',
default=None)
def configOpt():
"""Init for option
"""
usage = 'Usage: %prog [-f] [other option] [-h]'
p = OptionParser(usage)
# basic options
p.add_option('-t',
'--totalsj',
dest='totalsj',
action='store',
type='string',
help='totalsj file')
p.add_option('-s',
'--suvasj',
dest='suvasj',
action='store',
type='string',
help='suvasj file')
p.add_option('-o',
'--outfile',
dest='outfile',
default='Mapping_distribution.txt',
action='store',
type='string',
help='gene expression file')
group = OptionGroup(p, "Preset options")
# preset options
group.add_option('-O',
'--outDir',
dest='outDir',
default='./',
action='store',
type='string',
help='output directory',
metavar="DIR")
group.add_option('-L',
'--logDir',
dest='logDir',
default='',
action='store',
type='string',
help='log dir ,default is same as outDir')
group.add_option('-P',
'--logPrefix',
dest='logPrefix',
default='',
action='store',
type='string',
help='log file prefix')
group.add_option(
'-E',
'--email',
dest='email',
default='none',
action='store',
type='string',
help=
'email address, if you want get a email when this job is finished,default is no email',
metavar="EMAIL")
group.add_option('-Q',
'--quiet',
dest='quiet',
default=False,
action='store_true',
help='do not print messages to stdout')
group.add_option('-K',
'--keepTemp',
dest='keepTemp',
default=False,
action='store_true',
help='keep temp dir')
group.add_option('-T',
'--test',
dest='isTest',
default=False,
action='store_true',
help='run this program for test')
p.add_option_group(group)
if len(sys.argv) == 1:
p.print_help()
sys.exit(1)
opt, args = p.parse_args()
return (p, opt, args)
def add_argument_group(self, title=None, **kwargs):
optiongroup = OptionGroup(self._parser, title, **kwargs)
self._parser.add_option_group(optiongroup)
return _ArgumentGroup(optiongroup)
def initCmdLineParser():
"""
Initiate the optparse object, add all the groups and general command line flags
and returns the optparse object
"""
# Init parser and all general flags
usage = "usage: %prog [options] [--help]"
parser = OptionParser(usage=usage,
version="%prog {0}".format(
version_info.version_string()))
parser.add_option("--gen-answer-file",
help="Generate a template of an answer file.")
parser.add_option(
"--answer-file",
help=
"Runs the configuration in non-interactive mode, extracting all information from the"
"configuration file. using this option excludes all other options")
parser.add_option(
"--install-hosts",
help=
"Install on a set of hosts in a single step. The format should be a comma separated list "
"of hosts, the first is setup as a controller, and the others are setup as compute nodes."
"if only a single host is supplied then it is setup as an all in one installation. An answerfile "
"will also be generated and should be used if Packstack needs to be run a second time "
)
parser.add_option(
"--allinone",
action="store_true",
help=
"Shorthand for --install-hosts=<local ipaddr> --novanetwork-pubif=<dev> "
"--novacompute-privif=lo --novanetwork-privif=lo --os-swift-install=y --nagios-install=y "
", this option can be used to install an all in one OpenStack on this host"
)
parser.add_option("-t",
"--timeout",
default=300,
help="The timeout for puppet Exec calls")
parser.add_option(
"-o",
"--options",
action="store_true",
dest="options",
help="Print details on options available in answer file(rst format)")
parser.add_option("-d",
"--debug",
action="store_true",
default=False,
help="Enable debug in logging")
parser.add_option("-y",
"--dry-run",
action="store_true",
default=False,
help="Don't execute, just generate manifests")
# For each group, create a group option
for group in controller.getAllGroups():
groupParser = OptionGroup(parser, group.DESCRIPTION)
for param in group.parameters.itervalues():
cmdOption = param.CMD_OPTION
paramUsage = param.USAGE
optionsList = param.OPTION_LIST
useDefault = param.USE_DEFAULT
if not useDefault:
groupParser.add_option("--%s" % cmdOption, help=paramUsage)
# Add group parser to main parser
parser.add_option_group(groupParser)
return parser
maybe_absolute_branchpath)) == 'mysql.branches':
default_branchname = os.path.basename(maybe_absolute_branchpath)
else:
default_branchname = 'mainline'
elif toplevel_basename[:7] == 'tokudb.':
default_branchname = toplevel_basename[7:]
else:
default_branchname = 'unknown branch'
parser.add_option('--branch',
type='string',
dest='branch',
default=default_branchname,
help=('what to call this branch [default=%s]' %
default_branchname))
test_group = OptionGroup(parser, 'Scheduler Options',
'Control how the scheduler runs jobs.')
test_group.add_option(
'-t',
'--test_time',
type='int',
dest='test_time',
default=60,
help='time to run each test, in seconds [default=60]'),
test_group.add_option('-j',
'--jobs',
type='int',
dest='jobs',
default=8,
help='how many concurrent tests to run [default=8]')
test_group.add_option(
'--maxlarge',
def main():
"""Main function"""
# usage
usage = "usage: %prog [options] hdflist_file"
if 1 == len(sys.argv) and WXPYTHON:
option_parser_class = optparse_gui.OptionParser
else:
option_parser_class = optparse_required.OptionParser
parser = option_parser_class(usage=usage, description='modis_mosaic')
groupR = OptionGroup(parser, 'General options')
groupG = OptionGroup(parser, 'Options for GDAL')
groupM = OptionGroup(parser, 'Options for MRT')
# output
groupR.add_option("-o",
"--output",
dest="output",
required=True,
help="the name or prefix (for VRT) of output mosaic",
metavar="OUTPUT_FILE")
# subset
groupR.add_option("-s",
"--subset",
dest="subset",
help="a subset of product layers. The string should"
" be similar to: 1 0 [default: all layers]")
# options only for GDAL
groupG.add_option("-f",
"--output-format",
dest="output_format",
metavar="OUTPUT_FORMAT",
default="GTiff",
help="output format supported: GTiff, HDF4Image"
" [default=%default]")
# groupG.add_option("-g", "--grain", dest="grain",
# type="int", help="force the spatial resolution of output"
# " file")
# options for set VRT
groupG.add_option("-v",
"--vrt",
dest="vrt",
action="store_true",
default=False,
help="Create a GDAL VRT file. No other "
"GDAL options have to been set")
# mrt path
groupM.add_option("-m",
"--mrt",
dest="mrt_path",
type='directory',
help="the path to MRT software",
metavar="MRT_PATH")
parser.add_option_group(groupR)
parser.add_option_group(groupG)
parser.add_option_group(groupM)
(options, args) = parser.parse_args()
# check the number of tiles
if len(args) == 0 and not WXPYTHON:
parser.print_help()
sys.exit(1)
if not args:
parser.error(ERROR)
sys.exit()
else:
if type(args) != ListType:
parser.error(ERROR)
sys.exit()
elif len(args) > 1:
parser.error(ERROR)
sys.exit()
if not os.path.isfile(args[0]):
parser.error("You have to define the name of a text file containing "
"HDF files. (One HDF file for line)")
# check is a subset it is set
if not options.subset:
options.subset = False
else:
if string.find(options.subset, '(') != -1 or string.find(
options.subset, ')') != -1:
parser.error('ERROR: The spectral string should be similar to: '
'"1 0" without "(" and ")"')
# if not options.grain and options.vrt:
# options.grain = False
# elif not options.grain and options.vrt:
# parser.error("You have to define the resolution of output file. Please"
# " set -g/--grain option")
if options.mrt_path:
modisOgg = convertmodis.createMosaic(args[0], options.output,
options.mrt_path, options.subset)
modisOgg.run()
else:
tiles = []
dire = os.path.dirname(args[0])
with open(args[0]) as f:
for l in f:
name = os.path.splitext(l.strip())[0]
if '.hdf' not in name:
if dire not in l:
fname = os.path.join(dire, l.strip())
else:
fname = l.strip()
tiles.append(fname)
modisOgg = convertmodis_gdal.createMosaicGDAL(tiles, options.subset,
options.output_format)
if options.vrt:
modisOgg.write_vrt(options.output)
else:
modisOgg.run(options.output)
def main():
usage = '''Usage: python -m capsul [options] processname [arg1] [arg2] ...
[argx=valuex] [argy=valuey] ...
Example:
python -m capsul threshold ~/data/irm.ima /tmp/th.nii threshold1=80
Named arguments (in the shape argx=valuex) may address sub-processes of a
pipeline, using the dot separator:
PrepareSubject.t1mri=/home/myself/mymri.nii
For a more precise description, please look at the web documentation:
http://brainvisa.info/capsul/user_doc/user_guide_tree/index.html
'''
# Set up logging on stderr. This must be called before any logging takes
# place, to avoid "No handlers could be found for logger" errors.
logging.basicConfig()
parser = OptionParser(description='Run a single CAPSUL process',
usage=usage)
group1 = OptionGroup(
parser,
'Config',
description='Processing configuration, database options')
group1.add_option(
'--studyconfig',
dest='studyconfig',
help='load StudyConfig configuration from the given file (JSON)')
group1.add_option('-i',
'--input',
dest='input_directory',
help='input data directory (if not specified in '
'studyconfig file). If not specified neither on the '
'commandline nor study configfile, taken as the same as '
'output.')
group1.add_option('-o',
'--output',
dest='output_directory',
help='output data directory (if not specified in '
'studyconfig file). If not specified neither on the '
'commandline nor study configfile, taken as the same as '
'input.')
parser.add_option_group(group1)
group2 = OptionGroup(
parser,
'Processing',
description='Processing options, distributed execution')
group2.add_option('--swf',
'--soma_workflow',
dest='soma_workflow',
default=False,
action='store_true',
help='use soma_workflow. Soma-Workflow '
'configuration has to be setup and valid for non-local '
'execution, and additional file transfer options '
'may be used. The default is *not* to use SWF and '
'process mono-processor, sequential execution.')
group2.add_option('-r',
'--resource_id',
dest='resource_id',
default=None,
help='soma-workflow resource ID, defaults to localhost')
group2.add_option('-p',
'--password',
dest='password',
default=None,
help='password to access the remote computing resource. '
'Do not specify it if using a ssh key')
group2.add_option('--rsa-pass',
dest='rsa_key_pass',
default=None,
help='RSA key password, for ssh key access')
group2.add_option('--queue',
dest='queue',
default=None,
help='Queue to use on the computing resource. If not '
'specified, use the default queue.')
#group2.add_option('--input-processing', dest='input_file_processing',
#default=None, help='Input files processing: local_path, '
#'transfer, translate, or translate_shared. The default is '
#'local_path if the computing resource is the localhost, or '
#'translate_shared otherwise.')
#group2.add_option('--output-processing', dest='output_file_processing',
#default=None, help='Output files processing: local_path, '
#'transfer, or translate. The default is local_path.')
group2.add_option('--keep-succeeded-workflow',
dest='keep_succeded_workflow',
action='store_true',
default=False,
help='keep the workflow in the computing resource '
'database after execution. By default it is removed.')
group2.add_option('--delete-failed-workflow',
dest='delete_failed_workflow',
action='store_true',
default=False,
help='delete the workflow in the computing resource '
'database after execution, if it has failed. By default '
'it is kept.')
parser.add_option_group(group2)
group3 = OptionGroup(parser, 'Iteration', description='Iteration')
group3.add_option('-I',
'--iterate',
dest='iterate_on',
action='append',
help='Iterate the given process, iterating over the '
'given parameter(s). Multiple parameters may be '
'iterated jointly using several -I options. In the '
'process parameters, values are replaced by lists, all '
'iterated lists should have the same size.\n'
'Ex:\n'
'python -m capsul -I par_a -I par_c a_process '
'par_a="[1, 2]" par_b="something" '
'par_c="[\\"one\\", \\"two\\"]"')
parser.add_option_group(group3)
group4 = OptionGroup(parser, 'Attributes completion')
group4.add_option('-a',
'--attribute',
dest='attributes',
action='append',
default=[],
help='set completion (including FOM) attribute. '
'Syntax: attribute=value, value the same syntax as '
'process parameters (python syntax for lists, for '
'instance), with proper quotes if needed for shell '
'escaping.\n'
'Ex: -a acquisition="default" '
'-a subject=\'["s1", "s2"]\'')
parser.add_option_group(group4)
group5 = OptionGroup(parser,
'Help',
description='Help and documentation options')
group5.add_option('--process-help',
dest='process_help',
action='store_true',
default=False,
help='display specified process help')
parser.add_option_group(group5)
parser.disable_interspersed_args()
(options, args) = parser.parse_args()
if options.studyconfig:
study_config = StudyConfig(modules=StudyConfig.default_modules +
['FomConfig', 'BrainVISAConfig'])
if yaml:
scdict = yaml.load(open(options.studyconfig))
else:
scdict = json.load(open(options.studyconfig))
study_config.set_study_configuration(scdict)
else:
study_config = StudyConfig()
study_config.read_configuration()
if options.input_directory:
study_config.input_directory = options.input_directory
if options.output_directory:
study_config.output_directory = options.output_directory
if study_config.output_directory in (None, Undefined) \
and study_config.input_directory not in (None, Undefined):
study_config.output_directory = study_config.input_directory
if study_config.input_directory in (None, Undefined) \
and study_config.output_directory not in (None, Undefined):
study_config.input_directory = study_config.output_directory
study_config.somaworkflow_keep_succeeded_workflows \
= options.keep_succeded_workflow
study_config.somaworkflow_keep_failed_workflows \
= not options.delete_failed_workflow
kwre = re.compile('([a-zA-Z_](\.?[a-zA-Z0-9_])*)\s*=\s*(.*)$')
attributes = {}
for att in options.attributes:
m = kwre.match(att)
if m is None:
raise SyntaxError('syntax error in attribute definition: %s' % att)
attributes[m.group(1)] = convert_commandline_parameter(m.group(3))
args = tuple((convert_commandline_parameter(i) for i in args))
kwargs = {}
todel = []
for arg in args:
if type(arg) in types.StringTypes:
m = kwre.match(arg)
if m is not None:
kwargs[m.group(1)] = convert_commandline_parameter(m.group(3))
todel.append(arg)
args = [arg for arg in args if arg not in todel]
if not args:
parser.print_usage()
sys.exit(2)
# get the main process
process_name = args[0]
args = args[1:]
iterated = options.iterate_on
try:
process = get_process_with_params(process_name,
study_config,
iterated,
attributes=attributes,
*args,
**kwargs)
except ProcessParamError as e:
print("error: {0}".format(e), file=sys.stderr)
sys.exit(1)
if options.process_help:
process.help()
print()
completion_engine \
= ProcessCompletionEngine.get_completion_engine(process)
attribs = completion_engine.get_attribute_values()
aval = attribs.export_to_dict()
print('Completion attributes:')
print('----------------------')
print()
print('(note: may differ depending on study config file contents, '
'completion rules (FOM)...)')
print()
skipped = set(['generated_by_parameter', 'generated_by_process'])
for name, value in six.iteritems(aval):
if name in skipped:
continue
ttype = attribs.trait(name).trait_type.__class__.__name__
if isinstance(attribs.trait(name).trait_type, List):
ttype += '(%s)' \
% attribs.trait(name).inner_traits[
0].trait_type.__class__.__name__
print('%s:' % name, ttype)
if value not in (None, Undefined):
print(' ', value)
print()
del aval, attribs, completion_engine, process
sys.exit(0)
resource_id = options.resource_id
password = options.password
rsa_key_pass = options.rsa_key_pass
queue = options.queue
file_processing = []
study_config.use_soma_workflow = options.soma_workflow
if options.soma_workflow:
file_processing = [None, None]
else:
file_processing = [None, None]
res = run_process_with_distribution(
study_config,
process,
options.soma_workflow,
resource_id=resource_id,
password=password,
rsa_key_pass=rsa_key_pass,
queue=queue,
input_file_processing=file_processing[0],
output_file_processing=file_processing[1])
sys.exit(0)
def parser(mp=None):
if not mp: mp = OptionParser()
# mp.add_option('','',help=colours[5]+''+colours[0],default='',type='',dest='')
#
mp.add_option('--workdir',
help=colours[5] + 'Case workdir.' + colours[0],
default='case0',
type='str')
mp.add_option('--long',
help=colours[5] + 'Long filenames.' + colours[0],
default=False,
action='store_true')
mp.add_option('--bias',
help=colours[5] + 'Bias filenames.' + colours[0],
default=False,
action='store_true')
mp.add_option('-v',
'--verbosity',
help=colours[5] + 'Verbosity.' + colours[0],
default=0,
action='count')
mp.add_option('-q',
'--quiet',
help=colours[5] + 'Quiet.' + colours[0],
default=True,
action='store_false')
mg1 = OptionGroup(mp, 'Selection setup')
mg1.add_option(
'--SELCATs',
help=colours[5] + 'Selection/Category setup.' + colours[0],
default=
'double;DoubleB;-1#0.0875#0.5775#0.7875#1.,single;SingleB;-1#0.0775#0.5775#0.7775#0.8775#1.',
type='str',
action='callback',
callback=SELsetup,
dest='SC') # final
mp.add_option_group(mg1)
mg2 = OptionGroup(mp, 'Datacard settings')
mg2.add_option('--bounds',
help=colours[5] + 'Template boundaries: 80,200 (min,max)' +
colours[0],
default=[80., 200.],
type='str',
action='callback',
callback=optsplitfloat,
dest='X')
mg2.add_option('--lumi',
help=colours[5] + 'Luminosity: 35900(single,double)' +
colours[0],
default=[35900, 35900.],
type='str',
action='callback',
callback=optsplitfloat,
dest='LUMI')
mg2.add_option('--TF',
help=colours[5] +
'Transfer function label: POL1,POL1 (Single,Double)' +
colours[0],
default=['POL1', 'POL1'],
type='str',
action='callback',
callback=optsplit)
mg2.add_option('--BRN',
help=colours[5] + 'Bernstein order: 5,5 ' + colours[0],
default=[5, 5],
type='str',
action='callback',
callback=optsplitint)
# mg2.add_option('--MASS',help=colours[5]+'Signal masspoint: 125'+colours[0],default=[115,120,125,130,135],type='int')
mg2.add_option('--MASS',
help=colours[5] + 'Signal masspoint: 125' + colours[0],
default=[125],
type='int')
mg2.add_option('--CATveto',
help=colours[5] + 'CATveto: (blank)' + colours[0],
default=[],
type='str',
action='callback',
callback=optsplitint)
mg2.add_option('--CATS',
help=colours[5] + 'CATMIN-CATMAX: 0,8' + colours[0],
default=[0, 8],
type='str',
action='callback',
callback=optsplitint)
mg2.add_option('--function',
help=colours[5] + 'Function for bias study : expPow' +
colours[0],
default='expPow',
type='str')
mg2.add_option('--prefix',
help=colours[5] + 'Datacard prefix (folder): datacards/' +
colours[0],
default="datacards/",
type='str')
mp.add_option_group(mg2)
# mg1.add_option('','',help=colours[5]+''+colours[0],default='',type='',dest='')
mp.add_option_group(mg1)
#
return mp
def parse_args(self, argv):
from optparse import OptionParser, OptionGroup
from os.path import basename
parser = OptionParser(
usage='%prog [parameters] [[flags...] input_file]...')
def add_flag(parser, key):
if len(parser.largs) == 0 or not isinstance(
parser.largs[-1], list):
parser.largs.append(list())
parser.largs[-1].append(key)
def opt_callback_align(option, opt_str, value, parser):
try:
value = convert_size(value)
except ValueError:
parser.error('Invalid format for --align')
add_flag(parser, 'align')
add_flag(parser, value)
if 'align' in parser.largs[-1][:-2:2]:
parser.error('Only one align flag can be specified per file')
group = OptionGroup(parser, 'Required Parameters')
group.add_option('-o',
action='store',
dest='outfilename',
help='write output to FILENAME',
metavar='FILENAME')
group.add_option(
'-p',
action='store',
dest='physaddr',
help='set base physical address of output file to PHYSADDR',
metavar='PHYSADDR')
parser.add_option_group(group)
group = OptionGroup(parser, 'Flags affecting the following input file')
group.add_option(
'--align',
action='callback',
callback=opt_callback_align,
type='string',
help=
'mark the next input file as being aligned to BOUNDARY in physical memory',
metavar='BOUNDARY')
parser.add_option_group(group)
(opts, args) = parser.parse_args()
self.outfilename = opts.outfilename
if self.outfilename == None:
parser.error('Must provide an output file name using -o')
try:
if opts.physaddr == None:
raise ValueError()
self.physaddr = convert_size(opts.physaddr)
if (self.physaddr & 0xFFFFF000) != self.physaddr:
raise ValueError()
except ValueError:
parser.error(
'Must provide a valid 4K-aligned physical address using -p')
if not args:
parser.error('Cannot create an empty filesystem')
if isinstance(args[-1], list):
parser.error('Flags found with no file name following')
self.args = []
flags = ()
for x in args:
if isinstance(x, list):
flags = tuple(x)
else:
if '=' in x:
runtime_file_name, build_file_name = x.split('=', 1)
else:
runtime_file_name, build_file_name = (basename(x), x)
self.args.append([flags, runtime_file_name, build_file_name])
flags = ()
def parse_options():
description = "This example script connects with a SAP Router server and allows perform administrative tasks. " \
"The options are the ones found on the regular SAP Router tool with the addition of some " \
"undocumented operation codes."
epilog = "pysap %(version)s - %(url)s - %(repo)s" % {
"version": pysap.__version__,
"url": pysap.__url__,
"repo": pysap.__repo__
}
usage = "Usage: %prog [options] -d <remote host>"
parser = OptionParser(usage=usage, description=description, epilog=epilog)
target = OptionGroup(parser, "Target")
target.add_option("-d",
"--remote-host",
dest="remote_host",
default="127.0.0.1",
help="Remote host [%default]")
target.add_option("-p",
"--remote-port",
dest="remote_port",
type="int",
default=3299,
help="Remote port [%default]")
target.add_option(
"--router-version",
dest="router_version",
type="int",
help="SAP Router version to use [retrieve from the remote SAP Router]")
parser.add_option_group(target)
command = OptionGroup(parser, "Command")
command.add_option("-s",
"--stop-router",
dest="stop",
action="store_true",
help="Stop router request")
command.add_option("-o",
"--soft-shutdown",
dest="soft",
action="store_true",
help="Soft shutdown request")
command.add_option("-l",
"--router-info",
dest="info",
action="store_true",
help="Router info request")
command.add_option("-P",
"--info-pass",
dest="info_password",
action="store",
help="Password for info request")
command.add_option("-n",
"--new-route",
dest="new_route",
action="store_true",
help="New route table request")
command.add_option("-t",
"--toggle-trace",
dest="trace",
action="store_true",
help="Toggle trace request")
command.add_option("-c",
"--cancel-route",
dest="cancel",
action="store",
help="Cancel route request")
command.add_option("-b",
"--dump-buffer",
dest="dump",
action="store_true",
help="Dump buffers request")
command.add_option("-f",
"--flush-buffer",
dest="flush",
action="store_true",
help="Flush buffers request")
command.add_option("-z",
"--hide-errors",
dest="hide",
action="store_true",
help="Hide errors info")
command.add_option("--set-peer-trace",
dest="set_peer",
action="store",
help="Set peer trace (undocumented command)")
command.add_option("--clear-peer-trace",
dest="clear_peer",
action="store",
help="Clear peer trace (undocumented command)")
command.add_option("--trace-connection",
dest="trace_conn",
action="store",
help="Trace connection (undocumented command)")
parser.add_option_group(command)
misc = OptionGroup(parser, "Misc options")
misc.add_option("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="Verbose output [%default]")
parser.add_option_group(misc)
(options, _) = parser.parse_args()
if not options.remote_host:
parser.error("Remote host is required")
return options
def main():
parser = OptionParser(usage="usage: %prog [options] area ref.Length",
version="%prog 1.0",
description=__doc__)
parser.add_option(
"-s",
"--data-set",
action="append",
dest="dataSets",
type="string",
help="Specifies a set of cases in the current working directory,\
that are considered to be of one data set. The selection is\
done via regular expressions. Multiple data sets can be\
specified.")
parser.add_option("-i",
"--inlet",
action="store",
dest="inletPatch",
type="string",
default="XMAX",
help="Name of the inlet patch. (default=XMAX)")
parser.add_option("-f",
action="store",
type="choice",
choices=['1', '-1', '2', '-2', '3', '-3'],
dest="direction",
default=-1,
help="Main flow direction (1=x,2=y,3=z). (default=1)")
parser.add_option("-x",
"--x-data",
action="store",
type="choice",
choices=['v', 'Fr', 'Re'],
dest="xDataToPlot",
default='Fr',
help="Which data to show on the x-axis. (default=Fr)")
parser.add_option("-y",
"--y-data",
action="store",
type="choice",
choices=['CF', 'CT', 'RF', 'RT'],
dest="yDataToPlot",
default='CF',
help="Which data to show on the y-axis. (default=CF)")
group = OptionGroup(parser, "Flag Options")
group.add_option("--without-ittc57",
action="store_true",
dest="withoutIttc",
help="Do not print ITTC\'57 correlation line")
group.add_option("--without-errorbars",
action="store_true",
dest="withoutErrorbars",
help="Do not print the error bars for the averaging")
parser.add_option_group(group)
group = OptionGroup(parser, "Statistics Options")
group.add_option("--with-absolute",
action="store_true",
dest="withAbsolute",
help="If specified, all statistics intervalls have to be\
defined in terms of absolute times, rather than\
relatives")
group.add_option("--average-start",
action="store",
dest="averageStart",
type="float",
default=0.5,
help="Start of the averaging interval. (default=0.5)")
group.add_option("--average-end",
action="store",
dest="averageEnd",
type="float",
default=1.0,
help="End of the averaging interval. (default=1.0)")
parser.add_option_group(group)
(options, args) = parser.parse_args()
if len(args) != 2:
parser.error("wrong number of arguments")
# Get the reference area and reference lengths from the command line
# parameters. U needs to be evaluated, as either a string containg "False"
# is passed or a string with a 3D list in it.
area = float(args[0])
L = float(args[1])
############################
# Variable initialisations #
############################
startAtElement = 0
dataSets = []
dataSetNames = []
thisSet = []
for sI in options.dataSets:
setPattern = compile(r"%s" %
(sI.replace("_F_", Settings.floatPattern)))
for fI in listdir(getcwd()):
if findall(setPattern, fI):
thisSet.append(fI)
dataSetNames.append(thisSet)
thisSet = []
# Gather all cases
for sI in dataSetNames:
casesOfThisSet = []
for cI in sI:
casesOfThisSet.append(
Case.case(path.join(getcwd(), cI),
archive=None,
paraviewLink=False,
inletPatch=options.inletPatch,
L=L,
A=area,
direction=options.direction))
# Calculation of force coefficients has to be called explicitely, as the
# area and a reference length have to be specified. Furtheron it is not
# essential that forces are present.
casesOfThisSet[-1].calculateCoeffs()
dataSets.append(casesOfThisSet)
casesOfThisSet = []
if options.xDataToPlot == 'v':
unitX = 'm/s'
else:
unitX = '-'
if options.yDataToPlot in ('RT', 'RF'):
unitY = 'N'
else:
unitY = '-'
# Initialise the plot
fig = plt.figure(figsize=(11, 6))
plt.title("Comparison of resistances over velocity")
plt.grid(True)
ax1 = fig.add_subplot(111)
ax1.set_xlabel("%s [%s]" % (options.xDataToPlot, unitX))
ax1.set_ylabel("%s [%s]" % (options.yDataToPlot, unitY))
# Shink current axis by 20%
box = ax1.get_position()
ax1.set_position([box.x0, box.y0, box.width * 0.7, box.height])
plt.grid(True)
allPlots = []
dataToPlot = []
thisSetToPlot = []
Re = [1e15, 0]
Fr = [1e15, 0]
uInf = [1e15, 0]
for sI in dataSets:
for cI in sI:
aveStart = DataFile.element(cI.t, options.averageStart,
options.withAbsolute)
aveEnd = DataFile.element(cI.t, options.averageEnd,
options.withAbsolute)
ave = average(cI.resistances[options.yDataToPlot][aveStart:aveEnd])
# Update velocities
Re[0] = min(Re[0], cI.Re)
Re[1] = max(Re[1], cI.Re)
Fr[0] = min(Fr[0], cI.Fr)
Fr[1] = max(Fr[1], cI.Fr)
uInf[0] = min(uInf[0], cI.uInf)
uInf[1] = max(uInf[1], cI.uInf)
tmpList = [
cI.Fr, cI.Re, cI.uInf, ave,
std(cI.resistances[options.yDataToPlot][aveStart:aveEnd])
]
thisSetToPlot.append(tmpList)
dataToPlot.append(thisSetToPlot)
thisSetToPlot = []
dataToPlot = array(dataToPlot)
i = 0
for dI in dataToPlot:
# The data needs to be sorted
# http://stackoverflow.com/questions/2706605/sorting-a-2d-numpy-array-by-multiple-axes
ind = lexsort((dI[:, 1], dI[:, 0]))
sorted = dI[ind]
if options.xDataToPlot == 'v':
xData = sorted[:, 2]
elif options.xDataToPlot == 'Fr':
xData = sorted[:, 0]
else:
xData = sorted[:, 1]
if not options.withoutErrorbars:
allPlots.append(
ax1.errorbar(
xData,
sorted[:, 3],
sorted[:, 4],
fmt='-,',
))
else:
allPlots.append(
ax1.plot(xData, sorted[:, 3], '-o', label=options.dataSets[i]))
i += 1
if not options.withoutIttc:
ittcRe = linspace(Re[0], Re[1], 100)
if options.xDataToPlot == 'Fr':
xData = linspace(Fr[0], Fr[1], 100)
elif options.xDataToPlot == 'v':
xData = linspace(uInf[0], uInf[1], 100)
else:
xData = ittcRe
ittc = SkinFriction.ittc57(Re=ittcRe)
allPlots.append(
ax1.plot(xData, ittc, '-', color='black', label='ITTC\'57'))
# Assemble all plots for a proper legend
lines = [pI[0] for pI in allPlots]
labels = [l.get_label() for l in lines]
ax1.legend(
lines,
labels,
bbox_to_anchor=(1.02, 1),
loc=2,
#mode="expand",
#borderaxespad=0.,
ncol=1)
plt.show()
import collections
import sys
from optparse import OptionParser, OptionGroup
import time
import rpy2.robjects as robjects
from rpy2.robjects.packages import importr
import math
#Author: Martin Kapun
#version 1.0
######################################################### HELP #########################################################################
parser = OptionParser()
group=OptionGroup(parser,""" D E S C R I P T I O N
1) usage: python stauffer.py --inp data1.fet+data2.fet --pops 1,2+1,4 --sync fulldata.sync --out data.stauffer
2) the rpy2 package needs to be installed: http://sourceforge.net/projects/rpy/files/rpy2/ and type \"sudo python setup.py install\" in the directory
""")
parser.add_option("-o", "--out", dest="out",help="output file")
parser.add_option("-i", "--inp", dest="inp",help="output from Ram's FET script. The files must be separated by a \"+\"")
parser.add_option("-p", "--pops", dest="pops",help="ID of the population in the FET comparison. Must be linked to the position of the corresponding columns in the *.sync file. E.g. \"1,2\" stands for the FET pvalues of the first two populations in the sync file. The ID's must also be separated by a \"+\" (e.g. 1,2+1,4)")
parser.add_option("-s", "--sync", dest="sync", help="synchronized file")
parser.add_option_group(group)
(options, args) = parser.parse_args()
######################################################### CODE #########################################################################
inp=(str(options.inp).split("+"))
pop=(str(options.pops).split("+"))
fulllist=[]
