def parseArguments(command_list, cur_pizza, skip_flags=False):
formatter = optparse.IndentedHelpFormatter(max_help_position=30)
parser = optparse.OptionParser(usage=USAGE,
version=VERSION_STR,
add_help_option=False,
formatter=formatter)
if not skip_flags:
parser.add_option("-U", "--username", help="Specify your user name")
parser.add_option("-P", "--password", help="Specify your password")
parser.add_option("-O",
"--coupon",
default="",
help="Specify an "
"online coupon. Input x to see the coupon menu")
parser.add_option("-F",
"--force",
action="store_true",
help="Order the pizza with no user confirmation")
parser.add_option("-L",
"--login",
action="store_true",
help="Specify login information within the "
"program as opposed to using the command "
"line arguments")
parser.add_option("-I",
"--input-file",
help="Input file to read "
"batch of pizza (see man page for info)")
parser.add_option("-H",
"--help",
action="help",
help="Display help text")
topping_opts = optparse.OptionGroup(parser, "Topping options")
for topping in TOPPINGS:
short = "-" + topping.short_name
long = "--" + topping.long_name
topping_opts.add_option(short,
long,
dest=topping.option_dest,
action="store_true",
help=topping.help_name)
parser.add_option_group(topping_opts)
(options, args) = parser.parse_args(command_list)
# For each topping, check if the option was set. If so, add it to the
# pizza.
for topping in TOPPINGS:
if getattr(options, topping.option_dest):
cur_pizza.addTopping(topping)
# Parse positional arguments
for argument in args:
if argument in sizes:
cur_pizza.setSize(argument)
elif argument in crusts:
cur_pizza.setCrust(argument)
elif argument.isdigit():
cur_pizza.setQuantity(argument)
else:
print >> sys.stderr, "'%s' is not a valid argument. Exiting." % argument
sys.exit(42)
return [
options.username, options.password, options.coupon, options.force,
options.login, options.input_file
]
def parse_cmdline(argv):
# Help message and version string
version = ("Process execution tracer\n"
"by Mario Vilas (mvilas at gmail.com)\n"
"%s\n") % winappdbg.version
usage = (
"\n"
"\n"
" Create a new process (parameters for the target must be escaped):\n"
" %prog [options] -c <executable> [parameters for the target]\n"
" %prog [options] -w <executable> [parameters for the target]\n"
"\n"
" Attach to a running process (by filename):\n"
" %prog [options] -a <executable>\n"
"\n"
" Attach to a running process (by ID):\n"
" %prog [options] -a <process id>")
## formatter = optparse.IndentedHelpFormatter()
## formatter = optparse.TitledHelpFormatter()
parser = optparse.OptionParser(
usage=usage,
version=version,
## formatter=formatter,
)
# Commands
commands = optparse.OptionGroup(parser, "Commands")
commands.add_option("-a",
"--attach",
action="append",
type="string",
metavar="PROCESS",
help="Attach to a running process")
commands.add_option("-w",
"--windowed",
action="callback",
type="string",
metavar="CMDLINE",
callback=callback_execute_target,
help="Create a new windowed process")
commands.add_option("-c",
"--console",
action="callback",
type="string",
metavar="CMDLINE",
callback=callback_execute_target,
help="Create a new console process [default]")
parser.add_option_group(commands)
# Tracing options
tracing = optparse.OptionGroup(parser, "Tracing options")
tracing.add_option("--trace",
action="store_const",
const="trace",
dest="mode",
help="Set the single step mode [default]")
if System.arch == win32.ARCH_I386:
tracing.add_option(
"--branch",
action="store_const",
const="branch",
dest="mode",
help=
"Set the step-on-branch mode (doesn't work on virtual machines)")
tracing.add_option("--syscall",
action="store_const",
const="syscall",
dest="mode",
help="Set the syscall trap mode")
## tracing.add_options("--module", action="append", metavar="MODULES",
## dest="modules",
## help="only trace into these modules (comma-separated)")
## debugging.add_option("--from-start", action="store_true",
## help="start tracing when the process is created [default]")
## debugging.add_option("--from-entry", action="store_true",
## help="start tracing when the entry point is reached")
parser.add_option_group(tracing)
# Debugging options
debugging = optparse.OptionGroup(parser, "Debugging options")
debugging.add_option(
"--autodetach",
action="store_true",
help="automatically detach from debugees on exit [default]")
debugging.add_option(
"--follow",
action="store_true",
help="automatically attach to child processes [default]")
debugging.add_option("--trusted",
action="store_false",
dest="hostile",
help="treat debugees as trusted code [default]")
debugging.add_option(
"--dont-autodetach",
action="store_false",
dest="autodetach",
help="don't automatically detach from debugees on exit")
debugging.add_option("--dont-follow",
action="store_false",
dest="follow",
help="don't automatically attach to child processes")
debugging.add_option("--hostile",
action="store_true",
help="treat debugees as hostile code")
parser.add_option_group(debugging)
# Defaults
parser.set_defaults(
autodetach=True,
follow=True,
hostile=False,
windowed=list(),
console=list(),
attach=list(),
## modules = list(),
mode="trace",
)
# Parse and validate the command line options
if len(argv) == 1:
argv = argv + ['--help']
(options, args) = parser.parse_args(argv)
args = args[1:]
if not options.windowed and not options.console and not options.attach:
if not args:
parser.error("missing target application(s)")
options.console = [args]
else:
if args:
parser.error("don't know what to do with extra parameters: %s" %
args)
# Get the list of attach targets
system = System()
system.request_debug_privileges()
system.scan_processes()
attach_targets = list()
for token in options.attach:
try:
dwProcessId = HexInput.integer(token)
except ValueError:
dwProcessId = None
if dwProcessId is not None:
if not system.has_process(dwProcessId):
parser.error("can't find process %d" % dwProcessId)
try:
process = Process(dwProcessId)
process.open_handle()
process.close_handle()
except WindowsError, e:
parser.error("can't open process %d: %s" % (dwProcessId, e))
attach_targets.append(dwProcessId)
else:
matched = system.find_processes_by_filename(token)
if not matched:
parser.error("can't find process %s" % token)
for process, name in matched:
dwProcessId = process.get_pid()
try:
process = Process(dwProcessId)
process.open_handle()
process.close_handle()
except WindowsError, e:
parser.error("can't open process %d: %s" %
(dwProcessId, e))
attach_targets.append(process.get_pid())
default=False,
help='show map')
parser.add_option("--experimental",
default=False,
action='store_true',
help='enable experimental tests')
parser.add_option("--timeout",
default=3000,
type='int',
help='maximum runtime in seconds')
parser.add_option("--frame",
type='string',
default=None,
help='specify frame type')
group_build = optparse.OptionGroup(parser, "Build options")
group_build.add_option("--no-configure",
default=False,
action='store_true',
help='do not configure before building',
dest="no_configure")
group_build.add_option("-j", default=None, type='int', help='build CPUs')
group_build.add_option("--no-clean",
default=False,
action='store_true',
help='do not clean before building',
dest="no_clean")
group_build.add_option("--debug",
default=False,
action='store_true',
help='make built binaries debug binaries')
def __init__(self, fname=None):
self.options = {
'email_from': False,
'db_host': False,
'db_port': False,
'db_name': False,
'db_user': False,
'db_password': False,
'db_maxconn': 64,
'reportgz': False,
'translate_in': None,
'translate_out': None,
'overwrite_existing_translations': False,
'load_language': None,
'language': None,
'pg_path': None,
'admin_passwd': 'admin',
'csv_internal_sep': ',',
'addons_path': None,
'root_path': None,
'debug_mode': False,
'import_partial': "",
'pidfile': None,
'logfile': None,
'logrotate': True,
'smtp_server': 'localhost',
'smtp_user': False,
'smtp_port': 25,
'smtp_ssl': False,
'smtp_password': False,
'stop_after_init':
False, # this will stop the server after initialization
'syslog': False,
'log_level': logging.INFO,
'assert_exit_level':
logging.ERROR, # level above which a failed assert will be raised
'login_message': False,
'list_db': True,
'timezone': False, # to override the default TZ
'test_file': False,
'test_report_directory': False,
'test_disable': False,
'test_commit': False,
'publisher_warranty_url':
'http://services.openerp.com/publisher-warranty/',
'osv_memory_count_limit':
None, # number of records in each osv_memory virtual table
'osv_memory_age_limit': 1, # hours
}
self.aliases = {
'cache_timeout': 'cache.timeout',
'httpd_interface': 'httpd.interface',
'httpd_port': 'httpd.port',
'httpds_interface': 'httpsd.interface',
'httpds_port': 'httpsd.port',
'httpd': 'httpd.enable',
'httpds': 'httpsd.enable',
'netrpc_interface': 'netrpcd.interface',
'netrpc_port': 'netrpcd.port',
'netrpc': 'netrpcd.enable',
'secure_cert_file': 'httpsd.sslcert',
'secure_pkey_file': 'httpsd.sslkey',
'osv_memory_count_limit': 'osv_memory.count_limit',
'osv_memory_age_limit': 'osv_memory.age_limit',
}
self.blacklist_for_save = set(
["publisher_warranty_url", "load_language"])
self.misc = {}
self.config_file = fname
self.has_ssl = check_ssl()
self._LOGLEVELS = dict([
(getattr(netsvc, 'LOG_%s' % x), getattr(logging, x))
for x in ('CRITICAL', 'ERROR', 'WARNING', 'INFO', 'TEST', 'DEBUG',
'DEBUG_RPC', 'DEBUG_SQL', 'NOTSET')
])
version = "%s %s" % (release.description, release.version)
self.parser = parser = optparse.OptionParser(version=version)
parser.add_option("-c",
"--config",
dest="config",
help="specify alternate config file")
parser.add_option("-s",
"--save",
action="store_true",
dest="save",
default=False,
help="save configuration to ~/.openerp_serverrc")
parser.add_option("--pidfile",
dest="pidfile",
help="file where the server pid will be stored")
parser.add_option(
"-D",
"--define",
dest="defines",
action="append",
help="Define an arbitrary option, like 'section.var[=value]'")
group = optparse.OptionGroup(parser, "XML-RPC Configuration")
group.add_option(
"--httpd-interface",
dest="httpd_interface",
help="specify the TCP IP address for the XML-RPC protocol")
group.add_option("--httpd-port",
dest="httpd_port",
help="specify the TCP port for the HTTP protocol",
type="int")
group.add_option("--no-httpd",
dest="httpd",
action="store_false",
help="disable the HTTP protocol")
parser.add_option_group(group)
title = "HTTP Secure Configuration"
if not self.has_ssl:
title += " (disabled as ssl is unavailable)"
group = optparse.OptionGroup(parser, title)
group.add_option(
"--httpds-interface",
dest="httpds_interface",
help="specify the TCP IP address for the XML-RPC Secure protocol")
group.add_option(
"--httpds-port",
dest="httpds_port",
help="specify the TCP port for the HTTP Secure protocol",
type="int")
group.add_option("--no-httpds",
dest="httpds",
action="store_false",
help="disable the HTTP Secure protocol")
group.add_option(
"--cert-file",
dest="secure_cert_file",
help="specify the certificate file for the SSL connection")
group.add_option(
"--pkey-file",
dest="secure_pkey_file",
help="specify the private key file for the SSL connection")
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "NET-RPC Configuration")
group.add_option(
"--netrpc-interface",
dest="netrpc_interface",
help="specify the TCP IP address for the NETRPC protocol")
group.add_option("--netrpc-port",
dest="netrpc_port",
help="specify the TCP port for the NETRPC protocol",
type="int")
group.add_option("--no-netrpc",
dest="netrpc",
action="store_false",
help="disable the NETRPC protocol")
parser.add_option_group(group)
parser.add_option("-i",
"--init",
dest="init",
help="init a module (use \"all\" for all modules)")
parser.add_option(
"--without-demo",
dest="without_demo",
help="load demo data for a module (use \"all\" for all modules)",
default=False)
parser.add_option("-u",
"--update",
dest="update",
help="update a module (use \"all\" for all modules)")
parser.add_option("--cache-timeout",
dest="cache_timeout",
help="set the timeout for the cache system",
type="int")
parser.add_option(
"-t",
"--timezone",
dest="timezone",
help=
"specify reference timezone for the server (e.g. Europe/Brussels")
# stops the server from launching after initialization
parser.add_option("--stop-after-init",
action="store_true",
dest="stop_after_init",
default=False,
help="stop the server after it initializes")
parser.add_option('--debug',
dest='debug_mode',
action='store_true',
default=False,
help='enable debug mode')
parser.add_option(
"--assert-exit-level",
dest='assert_exit_level',
type="choice",
choices=self._LOGLEVELS.keys(),
help=
"specify the level at which a failed assertion will stop the server. Accepted values: %s"
% (self._LOGLEVELS.keys(), ))
# Testing Group
group = optparse.OptionGroup(parser, "Testing Configuration")
group.add_option("--test-file",
dest="test_file",
help="Launch a YML test file.")
group.add_option(
"--test-report-directory",
dest="test_report_directory",
help="If set, will save sample of all reports in this directory.")
group.add_option("--test-disable",
action="store_true",
dest="test_disable",
default=False,
help="Disable loading test files.")
group.add_option("--test-commit",
action="store_true",
dest="test_commit",
default=False,
help="Commit database changes performed by tests.")
parser.add_option_group(group)
# Logging Group
group = optparse.OptionGroup(parser, "Logging Configuration")
group.add_option("--logfile",
dest="logfile",
help="file where the server log will be stored")
group.add_option("--no-logrotate",
dest="logrotate",
action="store_false",
help="do not rotate the logfile")
group.add_option("--syslog",
action="store_true",
dest="syslog",
default=False,
help="Send the log to the syslog server")
group.add_option(
'--log-level',
dest='log_level',
type='choice',
choices=self._LOGLEVELS.keys(),
help='specify the level of the logging. Accepted values: ' +
str(self._LOGLEVELS.keys()))
parser.add_option_group(group)
# SMTP Group
group = optparse.OptionGroup(parser, "SMTP Configuration")
group.add_option(
'--email-from',
dest='email_from',
help='specify the SMTP email address for sending email')
group.add_option('--smtp',
dest='smtp_server',
help='specify the SMTP server for sending email')
group.add_option('--smtp-port',
dest='smtp_port',
help='specify the SMTP port',
type="int")
group.add_option('--smtp-ssl',
dest='smtp_ssl',
action='store_true',
help='specify the SMTP server support SSL or not')
group.add_option('--smtp-user',
dest='smtp_user',
help='specify the SMTP username for sending email')
group.add_option('--smtp-password',
dest='smtp_password',
help='specify the SMTP password for sending email')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Database related options")
group.add_option("-d",
"--database",
dest="db_name",
help="specify the database name")
group.add_option("-r",
"--db_user",
dest="db_user",
help="specify the database user name")
group.add_option("-w",
"--db_password",
dest="db_password",
help="specify the database password")
group.add_option("--pg_path",
dest="pg_path",
help="specify the pg executable path")
group.add_option("--db_host",
dest="db_host",
help="specify the database host")
group.add_option("--db_port",
dest="db_port",
help="specify the database port",
type="int")
group.add_option(
"--db_maxconn",
dest="db_maxconn",
type='int',
help=
"specify the the maximum number of physical connections to posgresql"
)
group.add_option(
"-P",
"--import-partial",
dest="import_partial",
help=
"Use this for big data importation, if it crashes you will be able to continue at the current state. Provide a filename to store intermediate importation states.",
default=False)
parser.add_option_group(group)
group = optparse.OptionGroup(
parser, "Internationalisation options",
"Use these options to translate OpenERP to another language."
"See i18n section of the user manual. Option '-d' is mandatory."
"Option '-l' is mandatory in case of importation")
group.add_option(
'--load-language',
dest="load_language",
help=
"specifies the languages for the translations you want to be loaded"
)
group.add_option(
'-l',
"--language",
dest="language",
help=
"specify the language of the translation file. Use it with --i18n-export or --i18n-import"
)
group.add_option(
"--i18n-export",
dest="translate_out",
help=
"export all sentences to be translated to a CSV file, a PO file or a TGZ archive and exit"
)
group.add_option(
"--i18n-import",
dest="translate_in",
help=
"import a CSV or a PO file with translations and exit. The '-l' option is required."
)
group.add_option(
"--i18n-overwrite",
dest="overwrite_existing_translations",
action="store_true",
default=False,
help=
"overwrites existing translation terms on importing a CSV or a PO file."
)
group.add_option(
"--modules",
dest="translate_modules",
help=
"specify modules to export. Use in combination with --i18n-export")
group.add_option("--addons-path",
dest="addons_path",
help="specify an alternative addons path.",
action="callback",
callback=self._check_addons_path,
nargs=1,
type="string")
group.add_option(
"--osv-memory-count-limit",
dest="osv_memory_count_limit",
default=False,
help=
"Force a limit on the maximum number of records kept in the virtual "
"osv_memory tables. The default is False, which means no count-based limit.",
type="int")
group.add_option(
"--osv-memory-age-limit",
dest="osv_memory_age_limit",
default=1.0,
help=
"Force a limit on the maximum age of records kept in the virtual "
"osv_memory tables. This is a decimal value expressed in hours, "
"and the default is 1 hour.",
type="float")
parser.add_option_group(group)
security = optparse.OptionGroup(parser, 'Security-related options')
security.add_option(
'--no-database-list',
action="store_false",
dest='list_db',
help="disable the ability to return the list of databases")
parser.add_option_group(security)
import os
import re
import Configuration.Applications
from Configuration.Applications.ConfigBuilder import ConfigBuilder, defaultOptions
import traceback
# Prepare a parser to read the options
usage=\
"""%prog <TYPE> [options].
Example:
%prog reco -s RAW2DIGI,RECO --conditions STARTUP_V4::All --eventcontent RECOSIM
"""
parser = optparse.OptionParser(usage)
expertSettings = optparse.OptionGroup(
parser, '===============\n Expert Options',
'Caution: please use only if you know what you are doing.')
famosSettings = optparse.OptionGroup(
parser, '===============\n FastSimulation options', '')
parser.add_option_group(expertSettings)
threeValued = []
parser.add_option("-s", "--step",
help="The desired step. The possible values are: "+\
"GEN,SIM,DIGI,L1,DIGI2RAW,HLT,RAW2DIGI,RECO,POSTRECO,DQM,ALCA,VALIDATION,HARVESTING, NONE or ALL.",
default="ALL",
dest="step")
parser.add_option("--conditions",
help="What conditions to use. This has to be specified",
default=None,
NOTE: no security measures are implemented. Anyone can remotely connect
to this service over the network.
Only one connection at once is supported. When the connection is terminated
it waits for the next connect.
""")
parser.add_option("-q", "--quiet",
dest = "quiet",
action = "store_true",
help = "suppress non error messages",
default = False
)
group = optparse.OptionGroup(parser,
"Serial Port",
"Serial port settings"
)
parser.add_option_group(group)
group.add_option("-p", "--port",
dest = "port",
help = "port, a number (default 0) or a device name",
default = None
)
group.add_option("-b", "--baud",
dest = "baudrate",
action = "store",
type = 'int',
help = "set baud rate, default: %default",
default = 9600
def parseOpts(overrideArguments=None):
def _readOptions(filename_bytes, default=[]):
try:
optionf = open(filename_bytes)
except IOError:
return default # silently skip if file is not present
try:
res = []
for l in optionf:
res += shlex.split(l, comments=True)
finally:
optionf.close()
return res
def _readUserConf():
xdg_config_home = compat_getenv('XDG_CONFIG_HOME')
if xdg_config_home:
userConfFile = os.path.join(xdg_config_home, 'ananse', 'config')
if not os.path.isfile(userConfFile):
userConfFile = os.path.join(xdg_config_home, 'ananse.conf')
else:
userConfFile = os.path.join(compat_expanduser('~'), '.config',
'ananse', 'config')
if not os.path.isfile(userConfFile):
userConfFile = os.path.join(compat_expanduser('~'), '.config',
'ananse.conf')
userConf = _readOptions(userConfFile, None)
if userConf is None:
appdata_dir = compat_getenv('appdata')
if appdata_dir:
userConf = _readOptions(os.path.join(appdata_dir, 'ananse',
'config'),
default=None)
if userConf is None:
userConf = _readOptions(os.path.join(
appdata_dir, 'ananse', 'config.txt'),
default=None)
if userConf is None:
userConf = _readOptions(os.path.join(compat_expanduser('~'),
'ananse.conf'),
default=None)
if userConf is None:
userConf = _readOptions(os.path.join(compat_expanduser('~'),
'ananse.conf.txt'),
default=None)
if userConf is None:
userConf = []
return userConf
def _format_option_string(option):
''' ('-o', '--option') -> -o, --format METAVAR'''
opts = []
if option._short_opts:
opts.append(option._short_opts[0])
if option._long_opts:
opts.append(option._long_opts[0])
if len(opts) > 1:
opts.insert(1, ', ')
if option.takes_value():
opts.append(' %s' % option.metavar)
return "".join(opts)
def _comma_separated_values_options_callback(option, opt_str, value,
parser):
setattr(parser.values, option.dest, value.split(','))
def _hide_login_info(opts):
opts = list(opts)
for private_opt in [
'-p', '--password', '-u', '--username', '--video-password'
]:
try:
i = opts.index(private_opt)
opts[i + 1] = 'PRIVATE'
except ValueError:
pass
return opts
# No need to wrap help messages if we're on a wide console
columns = get_term_width()
max_width = columns if columns else 80
max_help_position = 80
fmt = optparse.IndentedHelpFormatter(width=max_width,
max_help_position=max_help_position)
fmt.format_option_strings = _format_option_string
kw = {
'version': __version__,
'formatter': fmt,
'usage': '%prog [options] url [url...]',
'conflict_handler': 'resolve',
}
parser = optparse.OptionParser(**compat_kwargs(kw))
general = optparse.OptionGroup(parser, 'General Options')
general.add_option('-h',
'--help',
action='help',
help='print this help text and exit')
general.add_option('-v',
'--version',
action='version',
help='print program version and exit')
general.add_option(
'-U',
'--update',
action='store_true',
dest='update_self',
help=
'update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)'
)
general.add_option(
'-i',
'--ignore-errors',
action='store_true',
dest='ignoreerrors',
default=False,
help=
'continue on download errors, for example to skip unavailable videos in a playlist'
)
general.add_option(
'--abort-on-error',
action='store_false',
dest='ignoreerrors',
help=
'Abort downloading of further videos (in the playlist or the command line) if an error occurs'
)
general.add_option('--dump-user-agent',
action='store_true',
dest='dump_user_agent',
default=False,
help='display the current browser identification')
general.add_option(
'--list-extractors',
action='store_true',
dest='list_extractors',
default=False,
help='List all supported extractors and the URLs they would handle')
general.add_option('--extractor-descriptions',
action='store_true',
dest='list_extractor_descriptions',
default=False,
help='Output descriptions of all supported extractors')
general.add_option(
'--proxy',
dest='proxy',
default=None,
metavar='URL',
help=
'Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection'
)
general.add_option('--socket-timeout',
dest='socket_timeout',
type=float,
default=None,
help='Time to wait before giving up, in seconds')
general.add_option(
'--default-search',
dest='default_search',
metavar='PREFIX',
help=
'Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for ananse "large apple". Use the value "auto" to let ananse guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.'
)
general.add_option(
'--ignore-config',
action='store_true',
help='Do not read configuration files. '
'When given in the global configuration file /etc/ananse.conf: '
'Do not read the user configuration in ~/.config/ananse/config '
'(%APPDATA%/ananse/config.txt on Windows)')
general.add_option(
'--flat-playlist',
action='store_const',
dest='extract_flat',
const='in_playlist',
default=False,
help='Do not extract the videos of a playlist, only list them.')
selection = optparse.OptionGroup(parser, 'Video Selection')
selection.add_option(
'--playlist-start',
dest='playliststart',
metavar='NUMBER',
default=1,
type=int,
help='playlist video to start at (default is %default)')
selection.add_option('--playlist-end',
dest='playlistend',
metavar='NUMBER',
default=None,
type=int,
help='playlist video to end at (default is last)')
selection.add_option(
'--match-title',
dest='matchtitle',
metavar='REGEX',
help='download only matching titles (regex or caseless sub-string)')
selection.add_option(
'--reject-title',
dest='rejecttitle',
metavar='REGEX',
help='skip download for matching titles (regex or caseless sub-string)'
)
selection.add_option('--max-downloads',
dest='max_downloads',
metavar='NUMBER',
type=int,
default=None,
help='Abort after downloading NUMBER files')
selection.add_option(
'--min-filesize',
metavar='SIZE',
dest='min_filesize',
default=None,
help='Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)'
)
selection.add_option(
'--max-filesize',
metavar='SIZE',
dest='max_filesize',
default=None,
help='Do not download any videos larger than SIZE (e.g. 50k or 44.6m)')
selection.add_option('--date',
metavar='DATE',
dest='date',
default=None,
help='download only videos uploaded in this date')
selection.add_option(
'--datebefore',
metavar='DATE',
dest='datebefore',
default=None,
help=
'download only videos uploaded on or before this date (i.e. inclusive)'
)
selection.add_option(
'--dateafter',
metavar='DATE',
dest='dateafter',
default=None,
help=
'download only videos uploaded on or after this date (i.e. inclusive)')
selection.add_option(
'--min-views',
metavar='COUNT',
dest='min_views',
default=None,
type=int,
help='Do not download any videos with less than COUNT views',
)
selection.add_option(
'--max-views',
metavar='COUNT',
dest='max_views',
default=None,
type=int,
help='Do not download any videos with more than COUNT views')
selection.add_option(
'--no-playlist',
action='store_true',
dest='noplaylist',
default=False,
help=
'If the URL refers to a video and a playlist, download only the video.'
)
selection.add_option(
'--age-limit',
metavar='YEARS',
dest='age_limit',
default=None,
type=int,
help='download only videos suitable for the given age')
selection.add_option(
'--download-archive',
metavar='FILE',
dest='download_archive',
help=
'Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.'
)
selection.add_option('--include-ads',
dest='include_ads',
action='store_true',
help='Download advertisements as well (experimental)')
authentication = optparse.OptionGroup(parser, 'Authentication Options')
authentication.add_option('-u',
'--username',
dest='username',
metavar='USERNAME',
help='login with this account ID')
authentication.add_option('-p',
'--password',
dest='password',
metavar='PASSWORD',
help='account password')
authentication.add_option('-2',
'--twofactor',
dest='twofactor',
metavar='TWOFACTOR',
help='two-factor auth code')
authentication.add_option('-n',
'--netrc',
action='store_true',
dest='usenetrc',
default=False,
help='use .netrc authentication data')
authentication.add_option('--video-password',
dest='videopassword',
metavar='PASSWORD',
help='video password (vimeo, smotri)')
video_format = optparse.OptionGroup(parser, 'Video Format Options')
video_format.add_option(
'-f',
'--format',
action='store',
dest='format',
metavar='FORMAT',
default=None,
help=(
'video format code, specify the order of preference using'
' slashes: -f 22/17/18 . -f mp4 , -f m4a and -f flv are also'
' supported. You can also use the special names "best",'
' "bestvideo", "bestaudio", "worst", "worstvideo" and'
' "worstaudio". By default, ananse will pick the best quality.'
' Use commas to download multiple audio formats, such as'
' -f 136/137/mp4/bestvideo,140/m4a/bestaudio.'
' You can merge the video and audio of two formats into a single'
' file using -f <video-format>+<audio-format> (requires ffmpeg or'
' avconv), for example -f bestvideo+bestaudio.'))
video_format.add_option('--all-formats',
action='store_const',
dest='format',
const='all',
help='download all available video formats')
video_format.add_option(
'--prefer-free-formats',
action='store_true',
dest='prefer_free_formats',
default=False,
help='prefer free video formats unless a specific one is requested')
video_format.add_option('--max-quality',
action='store',
dest='format_limit',
metavar='FORMAT',
help='highest quality format to download')
video_format.add_option('-F',
'--list-formats',
action='store_true',
dest='listformats',
help='list all available formats')
video_format.add_option('--youtube-include-dash-manifest',
action='store_true',
dest='youtube_include_dash_manifest',
default=True,
help=optparse.SUPPRESS_HELP)
video_format.add_option(
'--youtube-skip-dash-manifest',
action='store_false',
dest='youtube_include_dash_manifest',
help='Do not download the DASH manifest on YouTube videos')
subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
subtitles.add_option('--write-sub',
'--write-srt',
action='store_true',
dest='writesubtitles',
default=False,
help='write subtitle file')
subtitles.add_option('--write-auto-sub',
'--write-automatic-sub',
action='store_true',
dest='writeautomaticsub',
default=False,
help='write automatic subtitle file (youtube only)')
subtitles.add_option(
'--all-subs',
action='store_true',
dest='allsubtitles',
default=False,
help='downloads all the available subtitles of the video')
subtitles.add_option('--list-subs',
action='store_true',
dest='listsubtitles',
default=False,
help='lists all available subtitles for the video')
subtitles.add_option(
'--sub-format',
action='store',
dest='subtitlesformat',
metavar='FORMAT',
default='srt',
help='subtitle format (default=srt) ([sbv/vtt] youtube only)')
subtitles.add_option(
'--sub-lang',
'--sub-langs',
'--srt-lang',
action='callback',
dest='subtitleslangs',
metavar='LANGS',
type='str',
default=[],
callback=_comma_separated_values_options_callback,
help=
'languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\''
)
downloader = optparse.OptionGroup(parser, 'Download Options')
downloader.add_option(
'-r',
'--rate-limit',
dest='ratelimit',
metavar='LIMIT',
help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
downloader.add_option('-R',
'--retries',
dest='retries',
metavar='RETRIES',
default=10,
help='number of retries (default is %default)')
downloader.add_option(
'--buffer-size',
dest='buffersize',
metavar='SIZE',
default='1024',
help='size of download buffer (e.g. 1024 or 16K) (default is %default)'
)
downloader.add_option(
'--no-resize-buffer',
action='store_true',
dest='noresizebuffer',
default=False,
help=
'do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.'
)
downloader.add_option('--test',
action='store_true',
dest='test',
default=False,
help=optparse.SUPPRESS_HELP)
downloader.add_option('--playlist-reverse',
action='store_true',
help='Download playlist videos in reverse order')
workarounds = optparse.OptionGroup(parser, 'Workarounds')
workarounds.add_option('--encoding',
dest='encoding',
metavar='ENCODING',
help='Force the specified encoding (experimental)')
workarounds.add_option('--no-check-certificate',
action='store_true',
dest='no_check_certificate',
default=False,
help='Suppress HTTPS certificate validation.')
workarounds.add_option(
'--prefer-insecure',
'--prefer-unsecure',
action='store_true',
dest='prefer_insecure',
help=
'Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)'
)
workarounds.add_option('--user-agent',
metavar='UA',
dest='user_agent',
help='specify a custom user agent')
workarounds.add_option(
'--referer',
metavar='URL',
dest='referer',
default=None,
help=
'specify a custom referer, use if the video access is restricted to one domain',
)
workarounds.add_option(
'--add-header',
metavar='FIELD:VALUE',
dest='headers',
action='append',
help=
'specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
)
workarounds.add_option(
'--bidi-workaround',
dest='bidi_workaround',
action='store_true',
help=
'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH'
)
verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
verbosity.add_option('-q',
'--quiet',
action='store_true',
dest='quiet',
default=False,
help='activates quiet mode')
verbosity.add_option('--no-warnings',
dest='no_warnings',
action='store_true',
default=False,
help='Ignore warnings')
verbosity.add_option(
'-s',
'--simulate',
action='store_true',
dest='simulate',
default=False,
help='do not download the video and do not write anything to disk',
)
verbosity.add_option(
'--skip-download',
action='store_true',
dest='skip_download',
default=False,
help='do not download the video',
)
verbosity.add_option('-g',
'--get-url',
action='store_true',
dest='geturl',
default=False,
help='simulate, quiet but print URL')
verbosity.add_option('-e',
'--get-title',
action='store_true',
dest='gettitle',
default=False,
help='simulate, quiet but print title')
verbosity.add_option('--get-id',
action='store_true',
dest='getid',
default=False,
help='simulate, quiet but print id')
verbosity.add_option('--get-thumbnail',
action='store_true',
dest='getthumbnail',
default=False,
help='simulate, quiet but print thumbnail URL')
verbosity.add_option('--get-description',
action='store_true',
dest='getdescription',
default=False,
help='simulate, quiet but print video description')
verbosity.add_option('--get-duration',
action='store_true',
dest='getduration',
default=False,
help='simulate, quiet but print video length')
verbosity.add_option('--get-filename',
action='store_true',
dest='getfilename',
default=False,
help='simulate, quiet but print output filename')
verbosity.add_option('--get-format',
action='store_true',
dest='getformat',
default=False,
help='simulate, quiet but print output format')
verbosity.add_option(
'-j',
'--dump-json',
action='store_true',
dest='dumpjson',
default=False,
help=
'simulate, quiet but print JSON information. See --output for a description of available keys.'
)
verbosity.add_option(
'-J',
'--dump-single-json',
action='store_true',
dest='dump_single_json',
default=False,
help=
'simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist information in a single line.'
)
verbosity.add_option('--newline',
action='store_true',
dest='progress_with_newline',
default=False,
help='output progress bar as new lines')
verbosity.add_option('--no-progress',
action='store_true',
dest='noprogress',
default=False,
help='do not print progress bar')
verbosity.add_option('--console-title',
action='store_true',
dest='consoletitle',
default=False,
help='display progress in console titlebar')
verbosity.add_option('-v',
'--verbose',
action='store_true',
dest='verbose',
default=False,
help='print various debugging information')
verbosity.add_option(
'--dump-intermediate-pages',
action='store_true',
dest='dump_intermediate_pages',
default=False,
help='print downloaded pages to debug problems (very verbose)')
verbosity.add_option(
'--write-pages',
action='store_true',
dest='write_pages',
default=False,
help=
'Write downloaded intermediary pages to files in the current directory to debug problems'
)
verbosity.add_option('--youtube-print-sig-code',
action='store_true',
dest='youtube_print_sig_code',
default=False,
help=optparse.SUPPRESS_HELP)
verbosity.add_option('--print-traffic',
dest='debug_printtraffic',
action='store_true',
default=False,
help='Display sent and read HTTP traffic')
filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
filesystem.add_option(
'-a',
'--batch-file',
dest='batchfile',
metavar='FILE',
help='file containing URLs to download (\'-\' for stdin)')
filesystem.add_option('--id',
default=False,
action='store_true',
dest='useid',
help='use only video ID in file name')
filesystem.add_option(
'-o',
'--output',
dest='outtmpl',
metavar='TEMPLATE',
help=
('output filename template. Use %(title)s to get the title, '
'%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
'%(autonumber)s to get an automatically incremented number, '
'%(ext)s for the filename extension, '
'%(format)s for the format description (like "22 - 1280x720" or "HD"), '
'%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
'%(upload_date)s for the upload date (YYYYMMDD), '
'%(extractor)s for the provider (youtube, metacafe, etc), '
'%(id)s for the video id, '
'%(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in, '
'%(playlist_index)s for the position in the playlist. '
'%(height)s and %(width)s for the width and height of the video format. '
'%(resolution)s for a textual description of the resolution of the video format. '
'%% for a literal percent. '
'Use - to output to stdout. Can also be used to download to a different directory, '
'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'
))
filesystem.add_option(
'--autonumber-size',
dest='autonumber_size',
metavar='NUMBER',
help=
'Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given'
)
filesystem.add_option(
'--restrict-filenames',
action='store_true',
dest='restrictfilenames',
default=False,
help=
'Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames'
)
filesystem.add_option(
'-A',
'--auto-number',
action='store_true',
dest='autonumber',
default=False,
help=
'[deprecated; use -o "%(autonumber)s-%(title)s.%(ext)s" ] number downloaded files starting from 00000'
)
filesystem.add_option('-t',
'--title',
action='store_true',
dest='usetitle',
default=False,
help='[deprecated] use title in file name (default)')
filesystem.add_option('-l',
'--literal',
default=False,
action='store_true',
dest='usetitle',
help='[deprecated] alias of --title')
filesystem.add_option('-w',
'--no-overwrites',
action='store_true',
dest='nooverwrites',
default=False,
help='do not overwrite files')
filesystem.add_option(
'-c',
'--continue',
action='store_true',
dest='continue_dl',
default=True,
help=
'force resume of partially downloaded files. By default, ananse will resume downloads if possible.'
)
filesystem.add_option(
'--no-continue',
action='store_false',
dest='continue_dl',
help='do not resume partially downloaded files (restart from beginning)'
)
filesystem.add_option(
'--no-part',
action='store_true',
dest='nopart',
default=False,
help='do not use .part files - write directly into output file')
filesystem.add_option(
'--no-mtime',
action='store_false',
dest='updatetime',
default=True,
help=
'do not use the Last-modified header to set the file modification time'
)
filesystem.add_option(
'--write-description',
action='store_true',
dest='writedescription',
default=False,
help='write video description to a .description file')
filesystem.add_option('--write-info-json',
action='store_true',
dest='writeinfojson',
default=False,
help='write video metadata to a .info.json file')
filesystem.add_option('--write-annotations',
action='store_true',
dest='writeannotations',
default=False,
help='write video annotations to a .annotation file')
filesystem.add_option('--write-thumbnail',
action='store_true',
dest='writethumbnail',
default=False,
help='write thumbnail image to disk')
filesystem.add_option(
'--load-info',
dest='load_info_filename',
metavar='FILE',
help=
'json file containing the video information (created with the "--write-json" option)'
)
filesystem.add_option(
'--cookies',
dest='cookiefile',
metavar='FILE',
help='file to read cookies from and dump cookie jar in')
filesystem.add_option(
'--cache-dir',
dest='cachedir',
default=None,
metavar='DIR',
help=
'Location in the filesystem where ananse can store some downloaded information permanently. By default $XDG_CACHE_HOME/ananse or ~/.cache/ananse . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.'
)
filesystem.add_option('--no-cache-dir',
action='store_const',
const=False,
dest='cachedir',
help='Disable filesystem caching')
filesystem.add_option('--rm-cache-dir',
action='store_true',
dest='rm_cachedir',
help='Delete all filesystem cache files')
postproc = optparse.OptionGroup(parser, 'Post-processing Options')
postproc.add_option(
'-x',
'--extract-audio',
action='store_true',
dest='extractaudio',
default=False,
help=
'convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)'
)
postproc.add_option(
'--audio-format',
metavar='FORMAT',
dest='audioformat',
default='best',
help=
'"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "%default" by default'
)
postproc.add_option(
'--audio-quality',
metavar='QUALITY',
dest='audioquality',
default='5',
help=
'ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default %default)'
)
postproc.add_option(
'--recode-video',
metavar='FORMAT',
dest='recodevideo',
default=None,
help=
'Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)'
)
postproc.add_option(
'-k',
'--keep-video',
action='store_true',
dest='keepvideo',
default=False,
help=
'keeps the video file on disk after the post-processing; the video is erased by default'
)
postproc.add_option(
'--no-post-overwrites',
action='store_true',
dest='nopostoverwrites',
default=False,
help=
'do not overwrite post-processed files; the post-processed files are overwritten by default'
)
postproc.add_option(
'--embed-subs',
action='store_true',
dest='embedsubtitles',
default=False,
help='embed subtitles in the video (only for mp4 videos)')
postproc.add_option('--embed-thumbnail',
action='store_true',
dest='embedthumbnail',
default=False,
help='embed thumbnail in the audio as cover art')
postproc.add_option('--add-metadata',
action='store_true',
dest='addmetadata',
default=False,
help='write metadata to the video file')
postproc.add_option(
'--xattrs',
action='store_true',
dest='xattrs',
default=False,
help=
'write metadata to the video file\'s xattrs (using dublin core and xdg standards)'
)
postproc.add_option(
'--prefer-avconv',
action='store_false',
dest='prefer_ffmpeg',
help=
'Prefer avconv over ffmpeg for running the postprocessors (default)')
postproc.add_option(
'--prefer-ffmpeg',
action='store_true',
dest='prefer_ffmpeg',
help='Prefer ffmpeg over avconv for running the postprocessors')
postproc.add_option(
'--exec',
metavar='CMD',
dest='exec_cmd',
help=
'Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\''
)
parser.add_option_group(general)
parser.add_option_group(selection)
parser.add_option_group(downloader)
parser.add_option_group(filesystem)
parser.add_option_group(verbosity)
parser.add_option_group(workarounds)
parser.add_option_group(video_format)
parser.add_option_group(subtitles)
parser.add_option_group(authentication)
parser.add_option_group(postproc)
if overrideArguments is not None:
opts, args = parser.parse_args(overrideArguments)
if opts.verbose:
write_string('[debug] Override config: ' +
repr(overrideArguments) + '\n')
else:
commandLineConf = sys.argv[1:]
if '--ignore-config' in commandLineConf:
systemConf = []
userConf = []
else:
systemConf = _readOptions('/etc/ananse.conf')
if '--ignore-config' in systemConf:
userConf = []
else:
userConf = _readUserConf()
argv = systemConf + userConf + commandLineConf
opts, args = parser.parse_args(argv)
if opts.verbose:
write_string('[debug] System config: ' +
repr(_hide_login_info(systemConf)) + '\n')
write_string('[debug] User config: ' +
repr(_hide_login_info(userConf)) + '\n')
write_string('[debug] Command-line args: ' +
repr(_hide_login_info(commandLineConf)) + '\n')
return parser, opts, args
def get_options():
p = optparse.OptionParser(
description="Description: Search in ASCII files and CKAN packages",
prog='search.py',
usage=
'%prog [ OPTIONS ] PATTERN\nPATTERN is your search pattern. Regular expressions and fields for disk searches are not supported.'
)
group_disk = optparse.OptionGroup(
p, "Disk Options",
"Use these options if you want to search in files on disk")
group_disk.add_option(
'--dir',
'-d',
help=
'Directory where you want to search in. The program will search recursively.',
default=None,
metavar='PATH')
group_disk.add_option('--disk_limit',
help='Limit of shown files',
default=20,
type='int',
metavar='INTEGER')
group_disk.add_option('--filetype',
'-f',
help='Filextension (e.g. "json", "xml", etc.)',
metavar='FILE EXTENSION')
group_disk.add_option(
'--offset',
help=
'How much characters will be shown before and after the found pattern',
default=35,
type='int',
metavar='INTEGER')
group_ckan = optparse.OptionGroup(
p, "CKAN Options", "Use these options if you want to search in CKAN")
group_ckan.add_option('--ckan',
help='Search in this CKAN portal',
default=None,
metavar='IP/URL')
group_ckan.add_option('--community',
'-c',
help="Community where you want to search in",
default='',
metavar='STRING')
group_ckan.add_option('--ckan_limit',
help='Limit of shown datasets',
default=20,
type='int',
metavar='INTEGER')
p.add_option_group(group_disk)
p.add_option_group(group_ckan)
options, args = p.parse_args()
if (len(args) != 1):
print("[ERROR] Need a pattern as an argument!")
exit()
return options, args[0]
def __init__(self, fname=None):
# Options not exposed on the command line. Command line options will be added
# from optparse's parser.
self.options = {
'admin_passwd': 'admin',
'csv_internal_sep': ',',
'publisher_warranty_url':
'http://services.openerp.com/publisher-warranty/',
'reportgz': False,
'root_path': None,
}
# Not exposed in the configuration file.
self.blacklist_for_save = set([
'publisher_warranty_url', 'load_language', 'root_path', 'init',
'save', 'config', 'update', 'stop_after_init'
])
# dictionary mapping option destination (keys in self.options) to MyOptions.
self.casts = {}
self.misc = {}
self.config_file = fname
self.has_ssl = check_ssl()
self._LOGLEVELS = dict([(getattr(loglevels,
'LOG_%s' % x), getattr(logging, x))
for x in ('CRITICAL', 'ERROR', 'WARNING',
'INFO', 'DEBUG', 'NOTSET')])
version = "%s %s" % (release.description, release.version)
self.parser = parser = optparse.OptionParser(version=version,
option_class=MyOption)
# Server startup config
group = optparse.OptionGroup(parser, "Common options")
group.add_option("-c",
"--config",
dest="config",
help="specify alternate config file")
group.add_option("-s",
"--save",
action="store_true",
dest="save",
default=False,
help="save configuration to ~/.openerp_serverrc")
group.add_option(
"-i",
"--init",
dest="init",
help=
"install one or more modules (comma-separated list, use \"all\" for all modules), requires -d"
)
group.add_option(
"-u",
"--update",
dest="update",
help=
"update one or more modules (comma-separated list, use \"all\" for all modules). Requires -d."
)
group.add_option(
"--without-demo",
dest="without_demo",
help=
"disable loading demo data for modules to be installed (comma-separated, use \"all\" for all modules). Requires -d and -i. Default is %default",
my_default=False)
group.add_option(
"-P",
"--import-partial",
dest="import_partial",
my_default='',
help=
"Use this for big data importation, if it crashes you will be able to continue at the current state. Provide a filename to store intermediate importation states."
)
group.add_option("--pidfile",
dest="pidfile",
help="file where the server pid will be stored")
group.add_option(
"--addons-path",
dest="addons_path",
help="specify additional addons paths (separated by commas).",
action="callback",
callback=self._check_addons_path,
nargs=1,
type="string")
group.add_option(
"--load",
dest="server_wide_modules",
help="Comma-separated list of server-wide modules default=web")
parser.add_option_group(group)
# XML-RPC / HTTP
group = optparse.OptionGroup(parser, "XML-RPC Configuration")
group.add_option(
"--xmlrpc-interface",
dest="xmlrpc_interface",
my_default='',
help=
"Specify the TCP IP address for the XML-RPC protocol. The empty string binds to all interfaces."
)
group.add_option("--xmlrpc-port",
dest="xmlrpc_port",
my_default=8069,
help="specify the TCP port for the XML-RPC protocol",
type="int")
group.add_option("--no-xmlrpc",
dest="xmlrpc",
action="store_false",
my_default=True,
help="disable the XML-RPC protocol")
group.add_option(
"--proxy-mode",
dest="proxy_mode",
action="store_true",
my_default=False,
help="Enable correct behavior when behind a reverse proxy")
group.add_option("--longpolling-port",
dest="longpolling_port",
my_default=8072,
help="specify the TCP port for longpolling requests",
type="int")
parser.add_option_group(group)
# XML-RPC / HTTPS
title = "XML-RPC Secure Configuration"
if not self.has_ssl:
title += " (disabled as ssl is unavailable)"
group = optparse.OptionGroup(parser, title)
group.add_option(
"--xmlrpcs-interface",
dest="xmlrpcs_interface",
my_default='',
help=
"Specify the TCP IP address for the XML-RPC Secure protocol. The empty string binds to all interfaces."
)
group.add_option(
"--xmlrpcs-port",
dest="xmlrpcs_port",
my_default=8071,
help="specify the TCP port for the XML-RPC Secure protocol",
type="int")
group.add_option("--no-xmlrpcs",
dest="xmlrpcs",
action="store_false",
my_default=True,
help="disable the XML-RPC Secure protocol")
group.add_option(
"--cert-file",
dest="secure_cert_file",
my_default='server.cert',
help="specify the certificate file for the SSL connection")
group.add_option(
"--pkey-file",
dest="secure_pkey_file",
my_default='server.pkey',
help="specify the private key file for the SSL connection")
parser.add_option_group(group)
# WEB
# TODO move to web addons after MetaOption merge
group = optparse.OptionGroup(parser, "Web interface Configuration")
group.add_option("--db-filter",
dest="dbfilter",
default='.*',
help="Filter listed database",
metavar="REGEXP")
parser.add_option_group(group)
# Static HTTP
group = optparse.OptionGroup(parser, "Static HTTP service")
group.add_option(
"--static-http-enable",
dest="static_http_enable",
action="store_true",
my_default=False,
help="enable static HTTP service for serving plain HTML files")
group.add_option(
"--static-http-document-root",
dest="static_http_document_root",
help=
"specify the directory containing your static HTML files (e.g '/var/www/')"
)
group.add_option(
"--static-http-url-prefix",
dest="static_http_url_prefix",
help=
"specify the URL root prefix where you want web browsers to access your static HTML files (e.g '/')"
)
parser.add_option_group(group)
# Testing Group
group = optparse.OptionGroup(parser, "Testing Configuration")
group.add_option("--test-file",
dest="test_file",
my_default=False,
help="Launch a YML test file.")
group.add_option(
"--test-report-directory",
dest="test_report_directory",
my_default=False,
help="If set, will save sample of all reports in this directory.")
group.add_option("--test-enable",
action="store_true",
dest="test_enable",
my_default=False,
help="Enable YAML and unit tests.")
group.add_option(
"--test-commit",
action="store_true",
dest="test_commit",
my_default=False,
help="Commit database changes performed by YAML or XML tests.")
parser.add_option_group(group)
# Logging Group
group = optparse.OptionGroup(parser, "Logging Configuration")
group.add_option("--logfile",
dest="logfile",
help="file where the server log will be stored")
group.add_option("--logrotate",
dest="logrotate",
action="store_true",
my_default=False,
help="enable logfile rotation")
group.add_option("--syslog",
action="store_true",
dest="syslog",
my_default=False,
help="Send the log to the syslog server")
group.add_option(
'--log-handler',
action="append",
default=DEFAULT_LOG_HANDLER,
my_default=DEFAULT_LOG_HANDLER,
metavar="PREFIX:LEVEL",
help=
'setup a handler at LEVEL for a given PREFIX. An empty PREFIX indicates the root logger. This option can be repeated. Example: "openerp.orm:DEBUG" or "werkzeug:CRITICAL" (default: ":INFO")'
)
group.add_option(
'--log-request',
action="append_const",
dest="log_handler",
const="openerp.netsvc.rpc.request:DEBUG",
help='shortcut for --log-handler=openerp.netsvc.rpc.request:DEBUG')
group.add_option(
'--log-response',
action="append_const",
dest="log_handler",
const="openerp.netsvc.rpc.response:DEBUG",
help='shortcut for --log-handler=openerp.netsvc.rpc.response:DEBUG'
)
group.add_option(
'--log-web',
action="append_const",
dest="log_handler",
const="openerp.addons.web.http:DEBUG",
help='shortcut for --log-handler=openerp.addons.web.http:DEBUG')
group.add_option(
'--log-sql',
action="append_const",
dest="log_handler",
const="openerp.sql_db:DEBUG",
help='shortcut for --log-handler=openerp.sql_db:DEBUG')
# For backward-compatibility, map the old log levels to something
# quite close.
levels = [
'info', 'debug_rpc', 'warn', 'test', 'critical', 'debug_sql',
'error', 'debug', 'debug_rpc_answer', 'notset'
]
group.add_option(
'--log-level',
dest='log_level',
type='choice',
choices=levels,
my_default='info',
help='specify the level of the logging. Accepted values: ' +
str(levels) + ' (deprecated option).')
parser.add_option_group(group)
# SMTP Group
group = optparse.OptionGroup(parser, "SMTP Configuration")
group.add_option(
'--email-from',
dest='email_from',
my_default=False,
help='specify the SMTP email address for sending email')
group.add_option('--smtp',
dest='smtp_server',
my_default='localhost',
help='specify the SMTP server for sending email')
group.add_option('--smtp-port',
dest='smtp_port',
my_default=25,
help='specify the SMTP port',
type="int")
group.add_option(
'--smtp-ssl',
dest='smtp_ssl',
action='store_true',
my_default=False,
help=
'if passed, SMTP connections will be encrypted with SSL (STARTTLS)'
)
group.add_option('--smtp-user',
dest='smtp_user',
my_default=False,
help='specify the SMTP username for sending email')
group.add_option('--smtp-password',
dest='smtp_password',
my_default=False,
help='specify the SMTP password for sending email')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Database related options")
group.add_option("-d",
"--database",
dest="db_name",
my_default=False,
help="specify the database name")
group.add_option("-r",
"--db_user",
dest="db_user",
my_default=False,
help="specify the database user name")
group.add_option("-w",
"--db_password",
dest="db_password",
my_default=False,
help="specify the database password")
group.add_option("--pg_path",
dest="pg_path",
help="specify the pg executable path")
group.add_option("--db_host",
dest="db_host",
my_default=False,
help="specify the database host")
group.add_option("--db_port",
dest="db_port",
my_default=False,
help="specify the database port",
type="int")
group.add_option(
"--db_maxconn",
dest="db_maxconn",
type='int',
my_default=64,
help=
"specify the the maximum number of physical connections to posgresql"
)
group.add_option(
"--db-template",
dest="db_template",
my_default="template1",
help="specify a custom database template to create a new database")
parser.add_option_group(group)
group = optparse.OptionGroup(
parser, "Internationalisation options",
"Use these options to translate OpenERP to another language."
"See i18n section of the user manual. Option '-d' is mandatory."
"Option '-l' is mandatory in case of importation")
group.add_option(
'--load-language',
dest="load_language",
help=
"specifies the languages for the translations you want to be loaded"
)
group.add_option(
'-l',
"--language",
dest="language",
help=
"specify the language of the translation file. Use it with --i18n-export or --i18n-import"
)
group.add_option(
"--i18n-export",
dest="translate_out",
help=
"export all sentences to be translated to a CSV file, a PO file or a TGZ archive and exit"
)
group.add_option(
"--i18n-import",
dest="translate_in",
help=
"import a CSV or a PO file with translations and exit. The '-l' option is required."
)
group.add_option(
"--i18n-overwrite",
dest="overwrite_existing_translations",
action="store_true",
my_default=False,
help=
"overwrites existing translation terms on updating a module or importing a CSV or a PO file."
)
group.add_option(
"--modules",
dest="translate_modules",
help=
"specify modules to export. Use in combination with --i18n-export")
parser.add_option_group(group)
security = optparse.OptionGroup(parser, 'Security-related options')
security.add_option(
'--no-database-list',
action="store_false",
dest='list_db',
my_default=True,
help="disable the ability to return the list of databases")
parser.add_option_group(security)
# Advanced options
group = optparse.OptionGroup(parser, "Advanced options")
group.add_option('--auto-reload',
dest='auto_reload',
action='store_true',
my_default=False,
help='enable auto reload')
group.add_option('--debug',
dest='debug_mode',
action='store_true',
my_default=False,
help='enable debug mode')
group.add_option("--stop-after-init",
action="store_true",
dest="stop_after_init",
my_default=False,
help="stop the server after its initialization")
group.add_option(
"-t",
"--timezone",
dest="timezone",
my_default=False,
help=
"specify reference timezone for the server (e.g. Europe/Brussels")
group.add_option(
"--osv-memory-count-limit",
dest="osv_memory_count_limit",
my_default=False,
help=
"Force a limit on the maximum number of records kept in the virtual "
"osv_memory tables. The default is False, which means no count-based limit.",
type="int")
group.add_option(
"--osv-memory-age-limit",
dest="osv_memory_age_limit",
my_default=1.0,
help=
"Force a limit on the maximum age of records kept in the virtual "
"osv_memory tables. This is a decimal value expressed in hours, "
"and the default is 1 hour.",
type="float")
group.add_option(
"--max-cron-threads",
dest="max_cron_threads",
my_default=2,
help=
"Maximum number of threads processing concurrently cron jobs (default 2).",
type="int")
group.add_option(
"--unaccent",
dest="unaccent",
my_default=False,
action="store_true",
help=
"Use the unaccent function provided by the database when available."
)
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Multiprocessing options")
# TODO sensible default for the three following limits.
group.add_option(
"--workers",
dest="workers",
my_default=0,
help="Specify the number of workers, 0 disable prefork mode.",
type="int")
group.add_option(
"--limit-memory-soft",
dest="limit_memory_soft",
my_default=640 * 1024 * 1024,
help=
"Maximum allowed virtual memory per worker, when reached the worker be reset after the current request (default 671088640 aka 640MB).",
type="int")
group.add_option(
"--limit-memory-hard",
dest="limit_memory_hard",
my_default=768 * 1024 * 1024,
help=
"Maximum allowed virtual memory per worker, when reached, any memory allocation will fail (default 805306368 aka 768MB).",
type="int")
group.add_option(
"--limit-time-cpu",
dest="limit_time_cpu",
my_default=60,
help="Maximum allowed CPU time per request (default 60).",
type="int")
group.add_option(
"--limit-time-real",
dest="limit_time_real",
my_default=120,
help="Maximum allowed Real time per request (default 120).",
type="int")
group.add_option(
"--limit-request",
dest="limit_request",
my_default=8192,
help=
"Maximum number of request to be processed per worker (default 8192).",
type="int")
parser.add_option_group(group)
# Copy all optparse options (i.e. MyOption) into self.options.
for group in parser.option_groups:
for option in group.option_list:
if option.dest not in self.options:
self.options[option.dest] = option.my_default
self.casts[option.dest] = option
self.parse_config(None, False)
def main():
"""A script that exposes all of the senseme commands as a CLI to enable scripting"""
fan = None
parser = optparse.OptionParser()
fan_group = optparse.OptionGroup(parser, "Commands specific to the fan")
addon_light_group = optparse.OptionGroup(
parser, "Commands specific to the add on light to the fan")
wall_controller_group = optparse.OptionGroup(
parser, "Commands specific to the wall controller")
device_group = optparse.OptionGroup(parser,
"Commands generic to senseme devices")
device_group.add_option("--ip", default=None, help="IP Address of Fan")
device_group.add_option(
"--name",
default=None,
help="Name of SenseMe Device as from the Haiku App")
device_group.add_option("--model",
default="FAN",
help="Model of the SenseMe device")
device_group.add_option(
"--get-beeper-sound",
default=None,
action="store_true",
help="Returns if the beeper sound is ON or OFF",
)
device_group.add_option("--set-beeper-sound",
default=None,
help="Sets beepersound to ON or OFF")
device_group.add_option(
"--get-device-time",
default=None,
action="store_true",
help="Returns the present time on the device",
)
device_group.add_option(
"--get-firmware-name",
default=None,
action="store_true",
help="Returns the name of the presently running firmware",
)
device_group.add_option(
"--get-firmware-version",
default=None,
action="store_true",
help="Returns the version of the presently running firmware",
)
device_group.add_option(
"--list-devices",
default=None,
action="store_true",
help=
"listens on the local network for Haiku devices and prints information about them.",
)
device_group.add_option(
"--get-led-indicators",
default=None,
action="store_true",
help="Returns if the indicator leds are ON or OFF",
)
device_group.add_option(
"--set-led-indicators",
default=None,
help="Sets if the indicator leds are ON or OFF",
)
device_group.add_option(
"--get-network-apstatus",
default=None,
action="store_true",
help="Returns if the wireless access point is enabled on the device",
)
device_group.add_option(
"--get-network-dhcp-state",
default=None,
action="store_true",
help="Returns if the device is running a local dhcp service",
)
device_group.add_option(
"--get-network-parameters",
default=None,
action="store_true",
help="Return a string of all network settings for the device",
)
device_group.add_option(
"--get-network-ssid",
default=None,
action="store_true",
help="Return the wireless SSID the device is connected to",
)
device_group.add_option(
"--get-network-token",
default=None,
action="store_true",
help="Return the network token of the device",
)
fan_group.add_option(
"--get-fanmode",
default=None,
action="store_true",
help="Returns if the light is ON or OFF",
)
fan_group.add_option("--set-fanmode",
default=None,
help="Sets the fan to ON or OFF")
fan_group.add_option(
"--toggle-fanmode",
default=None,
action="store_true",
help="Toggles the fan mode between ON or OFF",
)
fan_group.add_option(
"--get-height",
default=None,
action="store_true",
help="retrieve the height of the fan in centimeters",
)
fan_group.add_option(
"--set-height",
default=None,
type=int,
help="Sets the height of the fan in centimeters (ex. 10ft=304)",
)
fan_group.add_option(
"--get-speed",
default=None,
action="store_true",
help="retrieve the speed of the fan (0-7)",
)
fan_group.add_option("--set-speed",
default=None,
type="int",
help="set the speed of the fan (0-7)")
fan_group.add_option(
"--get-minspeed",
default=None,
action="store_true",
help="Returns the fan's minimum speed setting (0-7)",
)
fan_group.add_option(
"--get-maxspeed",
default=None,
action="store_true",
help="Returns the fan's maximum speed setting (0-7)",
)
fan_group.add_option(
"--get-room-fanspeed-limits",
default=None,
action="store_true",
help="Returns the fans min and max speeds for the room settings.",
)
fan_group.add_option(
"--set-room-fanspeed-limits",
default=None,
nargs=2,
dest="roomfanspeedlimits",
help="Sets the fans min and max speeds for the room settings. "
"This take 2 arguments min max with each being (0-7)",
)
fan_group.add_option(
"--decrease-fanspeed",
default=None,
action="store_true",
help="Decreases the speed of the fan by 1",
)
fan_group.add_option(
"--increase-fanspeed",
default=None,
action="store_true",
help="increases the speed of the fan by 1",
)
fan_group.add_option(
"--get-learnmode",
default=None,
action="store_true",
help="Returns if the fan's learning mode is ON or OFF",
)
fan_group.add_option(
"--set-learnmode",
default=None,
help="Sets if the fan's learning mode is ON or OFF",
)
fan_group.add_option(
"--get-learnmode-zerotemp",
default=None,
action="store_true",
help="Returns the temp in fahrenheit where learning mode "
"auto-shuts off the fan (valid temps 50-90)",
)
fan_group.add_option(
"--set-learnmode-zerotemp",
default=None,
type="int",
help="Sets the temp in fahrenheit where learning mode "
"auto-shuts off the fan (valid temps 50-90)",
)
fan_group.add_option(
"--get-learnmode-minspeed",
default=None,
action="store_true",
help="Returns the minimum speed setting for learning mode (0-7)",
)
fan_group.add_option(
"--set-learnmode-minspeed",
default=None,
type="int",
help="Sets the minimum speed setting for learning mode (0-7)",
)
fan_group.add_option(
"--get-learnmode-maxspeed",
default=None,
action="store_true",
help="Sets the maximum speed setting for learning mode (0-7)",
)
fan_group.add_option(
"--set-learnmode-maxspeed",
default=None,
type="int",
help="Sets the maximum speed setting for learning mode (0-7)",
)
fan_group.add_option(
"--get-smartsleep-mode",
default=None,
action="store_true",
help="Returns if the fan's smartsleep mode is ON or OFF",
)
fan_group.add_option(
"--set-smartsleep-mode",
default=None,
help="Sets if the fan's smartsleep mode is ON or OFF",
)
fan_group.add_option(
"--get-smartsleep-idealtemp",
default=None,
action="store_true",
help="retrieve the smart sleep mode ideal temp in fahrenheit",
)
fan_group.add_option(
"--set-smartsleep-idealtemp",
default=None,
type="int",
help=
"Sets the smart sleep mode ideal temp in fahrenheit (valid temps 50-90)",
)
fan_group.add_option(
"--get-smartsleep-minspeed",
default=None,
action="store_true",
help=
"retrieve the minimum the speed of the fan (0-7) when in smartsleep mode",
)
fan_group.add_option(
"--set-smartsleep-minspeed",
default=None,
type="int",
help="set the minimum speed of the fan (0-7) when in smartsleep mode",
)
fan_group.add_option(
"--get-smartsleep-maxspeed",
default=None,
action="store_true",
help=
"retrieve the maximum the speed of the fan (0-7) when in smartsleep mode",
)
fan_group.add_option(
"--set-smartsleep-maxspeed",
default=None,
type="int",
help="set the maximum speed of the fan (0-7) when in smartsleep mode",
)
fan_group.add_option(
"--get-smartsleep-wakeup-brightness",
default=None,
action="store_true",
help="get the add-on light brightness in wakeup mode "
"(0-16) 0=off, 1-16 are the brightness levels",
)
fan_group.add_option(
"--set-smartsleep-wakeup-brightness",
default=None,
type="int",
help="set the add-on light brightness in wakeup mode "
"(0-16) 0=off, 1-16 are the brightness levels",
)
fan_group.add_option(
"--get-fan-direction",
default=None,
action="store_true",
help="Gets if the fan blades are spinning forward or reverse.",
)
fan_group.add_option(
"--set-fan-direction",
default=None,
help=
"Sets if the fan blades are spinning forward or reverse. DO NOT DO WITH BLADES SPINNING",
)
fan_group.add_option(
"--get-fan-motionmode",
default=None,
action="store_true",
help="Returns if the fan's motion sensor is ON or OFF",
)
fan_group.add_option(
"--set-fan-motionmode",
default=None,
help="Sets if the fan's motion sensor is ON or OFF",
)
fan_group.add_option(
"--get-motionmode-mintimer",
default=None,
action="store_true",
help="Returns the minimum number of minutes the timer "
"can be set to auto-shutoff the fan and/or add-on light",
)
fan_group.add_option(
"--get-motionmode-maxtimer",
default=None,
action="store_true",
help="Returns the maximum number of minutes the timer "
"can be set to auto-shutoff the fan and/or add-on light",
)
fan_group.add_option(
"--get-motionmode-currenttimer",
default=None,
action="store_true",
help="Returns the current number of minutes the timer "
"can be set to auto-shutoff the fan and/or add-on light",
)
fan_group.add_option(
"--set-motionmode-timer",
default=None,
help="Sets the the number of minutes the timer "
"will run prior to auto-shutoff of the fan and/or add-on light",
)
fan_group.add_option(
"--get-motionmode-occupied-status",
default=None,
action="store_true",
help="Returns if the room is presently occupied or "
"unoccupied based on the built in motion sensor.",
)
fan_group.add_option(
"--get-wintermode",
default=None,
action="store_true",
help="Returns if the wintermode state is ON or OFF",
)
fan_group.add_option("--set-wintermode",
default=None,
help="Sets wintermode to ON or OFF")
fan_group.add_option(
"--get-smartmode",
default=None,
action="store_true",
help="Returns if the smartmode is off, cooling or heating",
)
fan_group.add_option(
"--set-smartmode",
default=None,
help="Sets smart mode to off,cooling, or heating",
)
fan_group.add_option(
"--get-whooshmode",
default=None,
action="store_true",
help="Returns of whoosh mode is ON or OFF",
)
fan_group.add_option("--set-whooshmode",
default=None,
help="Sets whoosh mode to ON or OFF")
addon_light_group.add_option(
"--get-brightness",
default=None,
action="store_true",
help="retrieve the brightness of the fan (0-16)",
)
addon_light_group.add_option(
"--set-brightness",
default=None,
type="int",
help="Sets the brightness of the light on the fan (0-16)",
)
addon_light_group.add_option(
"--get-min-brightness",
default=None,
action="store_true",
help="Returns the add-on lights minimum brightness setting (0-16)",
)
addon_light_group.add_option(
"--get-max-brightness",
default=None,
action="store_true",
help="Returns the add-on lights maximum brightness setting (0-16)",
)
addon_light_group.add_option(
"--get-room-brightness-limits",
default=None,
action="store_true",
help=
"Returns the add-on lights min and max brightness for the room settings",
)
addon_light_group.add_option(
"--set-room-brightness-limits",
default=None,
type="int",
nargs=2,
dest="roombrightnesslimits",
help="Sets the add-on lights min and max brightness "
"for the room settings this takes 2 arguments min max with each being (0-16)",
)
addon_light_group.add_option(
"--decrease-brightness",
default=None,
action="store_true",
help="Decreases the brightness of the light by 1",
)
addon_light_group.add_option(
"--increase-brightness",
default=None,
action="store_true",
help="increase the brightnes of the light by 1",
)
addon_light_group.add_option(
"--is-fan-light-installed",
default=None,
action="store_true",
help="Returns if the optional light module is present",
)
addon_light_group.add_option(
"--get-light-motionmode",
default=None,
action="store_true",
help="Returns if the light's motion sensor is ON or OFF",
)
addon_light_group.add_option(
"--set-light-motionmode",
default=None,
help="Sets if the light's motion sensor is ON or OFF",
)
addon_light_group.add_option(
"--get-lightmode",
default=None,
action="store_true",
help="Returns if the light is ON or OFF",
)
addon_light_group.add_option("--set-lightmode",
default=None,
help="Sets the light to ON or OFF")
addon_light_group.add_option(
"--toggle-lightmode",
default=None,
action="store_true",
help="Toggles the light mode between ON or OFF",
)
parser.add_option_group(device_group)
parser.add_option_group(fan_group)
parser.add_option_group(addon_light_group)
parser.add_option_group(wall_controller_group)
opts, args = parser.parse_args()
if not all([opts.ip, opts.name]):
print("IP and the fan name are required to work")
return 1
elif len(args) != 0:
print("Unknown options specified: " + " ".join(args))
return 1
else:
fan = SenseMe(ip=opts.ip, name=opts.name, model=opts.model)
# Process all the command line arguments
# Device commands first
if opts.get_beeper_sound:
mode = fan.beeper_sound
print(mode)
return 0
elif opts.set_beeper_sound:
mode = opts.set_beeper_sound.upper()
if mode == "ON" or mode == "OFF":
fan.beeper_sound = mode
return 0
else:
print("Invalid beeper sound specified, valid option are ON or OFF")
return 1
elif opts.get_device_time:
print(fan.device_time)
return 0
elif opts.get_firmware_name:
print(fan.firmware_name)
return 0
elif opts.get_firmware_version:
print(fan.firmware_version)
return 0
elif opts.list_devices:
devices = discover()
for device in devices:
print('%s,%s,%s,%s,"%s"' % (device.name, device.ip, device.mac,
device.series, device.model))
return 0
elif opts.get_led_indicators:
mode = fan.led_indicators
print(mode)
return 0
elif opts.set_led_indicators:
mode = opts.set_led_indicators.upper()
if mode == "OFF" or mode == "ON":
fan.led_indicators = mode
return 0
else:
print(
"Invalid led indicator state specified, valid option are ON or OFF"
)
return 1
elif opts.get_network_apstatus:
print(fan.network_ap_status)
return 0
elif opts.get_network_dhcp_state:
print(fan.network_dhcp_state)
return 0
elif opts.get_network_parameters:
(ip, subnetmask, defaultgw) = fan.network_parameters
print(ip + "," + subnetmask + "," + defaultgw)
return 0
elif opts.get_network_ssid:
print(fan.network_ssid)
return 0
elif opts.get_network_token:
print(fan.network_token)
return 0
# Fan commands
elif opts.get_fanmode:
mode = fan.fan_powered_on
if mode:
print("ON")
else:
print("OFF")
return 0
elif opts.set_fanmode:
mode = opts.set_fanmode.upper()
if mode == "ON":
fan.fan_powered_on = True
else:
fan.fan_powered_on = False
return 0
elif opts.get_height:
print(fan.height)
return 0
elif opts.set_height:
if opts.set_height > 0:
fan.height = opts.set_height
return 0
elif opts.toggle_fanmode:
fan.fan_toggle()
return 0
elif opts.get_speed:
print(fan.speed)
return 0
elif opts.set_speed is not None:
if opts.set_speed >= 7:
fan.speed = 7
elif opts.set_speed <= 0:
fan.speed = 0
else:
fan.speed = opts.set_speed
return 0
elif opts.get_minspeed:
print(fan.min_speed)
return 0
elif opts.get_maxspeed:
print(fan.max_speed)
return 0
elif opts.get_room_fanspeed_limits:
(min_speed, max_speed) = fan.room_settings_fan_speed_limits
print("%d %d" % (min_speed, max_speed))
elif opts.roomfanspeedlimits:
min_speed = opts.roomfanspeedlimits[0]
max_speed = opts.roomfanspeedlimits[1]
fan.room_settings_fan_speed_limits = (min_speed, max_speed)
elif opts.decrease_fanspeed:
fan.dec_speed()
return 0
elif opts.increase_fanspeed:
fan.inc_speed()
return 0
elif opts.get_learnmode:
mode = fan.learnmode
print(mode.upper())
return 0
elif opts.set_learnmode:
mode = opts.set_learnmode.upper()
if mode == "OFF" or mode == "ON":
fan.learnmode = mode
return 0
else:
print("Invalid learn mode specified, valid option are ON or OFF")
return 1
elif opts.get_learnmode_zerotemp:
print(fan.learnmode_zerotemp)
return 0
elif opts.set_learnmode_zerotemp:
if opts.set_learnmode_zerotemp >= 90:
fan.learn_mode_zero_temp = 90
elif opts.set_learnmode_zerotemp <= 50:
fan.learnmode_zerotemp = 50
else:
fan.learnmode_zerotemp = opts.set_learnmode_zero_temp
return 0
elif opts.get_learnmode_minspeed:
print(fan.learnmode_minspeed)
return 0
elif opts.set_learnmode_minspeed is not None:
if opts.set_learnmode_minspeed >= 7:
fan.learnmode_minspeed = 7
elif opts.set_learnmode_minspeed <= 0:
fan.learnmode_minspeed = 0
else:
fan.learnmode_minspeed = opts.set_learnmode_minspeed
return 0
elif opts.get_learnmode_maxspeed:
print(fan.learnmode_maxspeed)
return 0
elif opts.set_learnmode_maxspeed is not None:
if opts.set_learnmode_maxspeed >= 7:
fan.learnmode_maxspeed = 7
elif opts.set_learnmode_maxspeed <= 0:
fan.learnmode_maxspeed = 0
else:
fan.learnmode_maxspeed = opts.set_learnmode_maxspeed
return 0
elif opts.get_smartsleep_mode:
mode = fan.smartsleep_mode
print(mode.upper())
return 0
elif opts.set_smartsleep_mode:
mode = opts.set_smartsleep_mode.upper()
if mode == "OFF" or mode == "ON":
fan.smartsleep_mode = mode
return 0
else:
print(
"Invalid smartsleep mode specified, valid option are ON or OFF"
)
return 1
elif opts.get_smartsleep_idealtemp:
print(fan.smartsleep_idealtemp)
return 0
elif opts.set_smartsleep_idealtemp:
if opts.set_smartsleep_idealtemp >= 90:
fan.smartsleep_idealtemp = 90
elif opts.set_smartsleep_idealtemp <= 50:
fan.smartsleep_idealtemp = 50
else:
fan.smartsleep_idealtemp = opts.set_smartsleep_idealtemp
return 0
elif opts.get_smartsleep_minspeed:
print(fan.smartsleep_minspeed)
return 0
elif opts.set_smartsleep_minspeed is not None:
if opts.set_smartsleep_minspeed >= 7:
fan.smartsleep_minspeed = 7
elif opts.set_smartsleep_minspeed <= 0:
fan.smartsleep_minspeed = 0
else:
fan.smartsleep_minspeed = opts.set_smartsleep_minspeed
return 0
elif opts.get_smartsleep_maxspeed:
print(fan.smartsleep_maxspeed)
return 0
elif opts.set_smartsleep_maxspeed is not None:
if opts.set_smartsleep_maxspeed >= 7:
fan.smartsleep_maxspeed = 7
elif opts.set_smartsleep_maxspeed <= 0:
fan.smartsleep_maxspeed = 0
else:
fan.smartsleep_maxspeed = opts.set_smartsleep_maxspeed
return 0
elif opts.get_smartsleep_wakeup_brightness:
print(fan.smartsleep_wakeup_brightness)
return 0
elif opts.set_smartsleep_wakeup_brightness is not None:
if opts.set_smartsleep_wakeup_brightness >= 16:
fan.smartsleep_wakeup_brightness = 16
elif opts.set_smartsleep_wakeup_brightness <= 0:
fan.smartsleep_wakeup_brightness = 0
else:
fan.smartsleep_wakeup_brightness = opts.set_smartsleep_wakeup_brightness
return 0
elif opts.get_fan_direction:
mode = fan.fan_direction
if mode.upper() == "FWD":
print("FORWARD")
else:
print("REVERSE")
return 0
elif opts.set_fan_direction:
mode = opts.set_fan_direction.upper()
if mode == "FORWARD":
fan.fan_direction = "FWD"
return 0
elif mode == "REVERSE":
fan.fan_direction = "REV"
return 0
else:
print(
"Invalid fan direction specified, valid option are FORWARD or REVERSE"
)
return 1
elif opts.get_fan_motionmode:
mode = fan.fan_motionmode
print(mode.upper())
return 0
elif opts.set_fan_motionmode:
mode = opts.set_fan_motionmode.upper()
if mode == "OFF" or mode == "ON":
fan.fan_motionmode = mode
return 0
else:
print(
"Invalid fan motion detection mode specified, valid option are ON or OFF"
)
return 1
elif opts.get_motionmode_mintimer:
print(fan.motionmode_mintimer)
return 0
elif opts.get_motionmode_maxtimer:
print(fan.motionmode_maxtimer)
return 0
elif opts.get_motionmode_currenttimer:
print(fan.motionmode_currenttimer)
return 0
elif opts.set_motionmode_timer:
fan.motionmode_currenttimer = opts.set_motionmode_timer
return 1
elif opts.get_motionmode_occupied_status:
mode = fan.motionmode_occupied_status
print(mode.upper())
return 0
elif opts.get_wintermode:
mode = fan.wintermode
print(mode)
return 0
elif opts.set_wintermode:
mode = opts.set_wintermode.upper()
if mode == "OFF" or mode == "ON":
fan.wintermode = mode
return 0
else:
print("Invalid winter mode specified, valid option are ON or OFF")
return 1
elif opts.get_smartmode:
mode = fan.smartmode
print(mode)
return 0
elif opts.set_smartmode:
mode = opts.set_smartmode.upper()
if mode == "OFF" or mode == "COOLING" or mode == "HEATING":
fan.smartmode = mode
return 0
else:
print(
"Invalid smart mode specified, valid option are OFF, COOLING or HEATING"
)
return 1
elif opts.get_whooshmode:
mode = fan.whoosh
if mode:
print("ON")
else:
print("OFF")
return 0
elif opts.set_whooshmode:
mode = opts.set_whooshmode.upper()
if mode == "ON":
fan.whoosh = True
else:
fan.whoosh = False
return 0
# Add-on Light Commands
elif opts.get_brightness:
print(fan.brightness)
return 0
elif opts.set_brightness is not None:
if opts.set_brightness >= 16:
fan.brightness = 16
elif opts.set_brightness <= 0:
fan.brightness = 0
else:
fan.brightness = opts.set_brightness
return 0
elif opts.get_min_brightness:
print(fan.min_brightness)
return 0
elif opts.get_max_brightness:
print(fan.max_brightness)
return 0
elif opts.get_room_brightness_limits:
(min_brightness, max_brightness) = fan.room_settings_brightness_limits
print("%d %d" % (min_brightness, max_brightness))
return 0
elif opts.roombrightnesslimits:
min_brightness = opts.room_brightness_limits[0]
max_brightness = opts.room_brightness_limits[1]
fan.room_settings_brightness_limits = (min_brightness, max_brightness)
return 0
elif opts.decrease_brightness:
fan.dec_brightness()
return 0
elif opts.increase_brightness:
fan.inc_brightness()
return 0
elif opts.is_fan_light_installed:
mode = fan.is_fan_light_installed
if mode:
print("INSTALLED")
return 1
else:
print("NOT INSTALLED")
return 0
elif opts.get_light_motionmode:
mode = fan.light_motionmode
print(mode.upper())
return 0
elif opts.set_light_motionmode:
mode = opts.set_light_motionmode.upper()
if mode == "OFF" or mode == "ON":
fan.light_motionmode = mode
return 0
else:
print(
"Invalid light motion detection mode specified, valid option are ON or OFF"
)
return 1
elif opts.get_lightmode:
mode = fan.light_powered_on
if mode:
print("ON")
else:
print("OFF")
return 0
elif opts.set_lightmode:
mode = opts.set_lightmode.upper()
if mode == "ON":
fan.light_powered_on = True
else:
fan.light_powered_on = False
return 0
elif opts.toggle_lightmode:
fan.light_toggle()
return 0
else:
print("Invalid argument specified")
return 1
def AddCommandLineArgs(cls, parser):
group = optparse.OptionGroup(parser, '%s test options' % cls.Name())
cls.AddBenchmarkCommandLineArgs(group)
if group.option_list:
parser.add_option_group(group)
def _parse_args(self, argv=None):
parser = optparse.OptionParser(
usage='usage: %prog [options] [args...]')
upload_group = optparse.OptionGroup(parser, 'Upload Options')
upload_group.add_options(upload_options())
parser.add_option_group(upload_group)
parser.add_option('-a',
'--all',
action='store_true',
default=False,
help='run all the tests')
parser.add_option(
'-c',
'--coverage',
action='store_true',
default=False,
help=
'generate code coverage info (requires http://pypi.python.org/pypi/coverage)'
)
parser.add_option('-i',
'--integration-tests',
action='store_true',
default=False,
help='run integration tests as well as unit tests'),
parser.add_option(
'-j',
'--child-processes',
action='store',
type='int',
default=(1 if sys.platform.startswith('win') else
multiprocessing.cpu_count()),
help='number of tests to run in parallel (default=%default)')
parser.add_option(
'-p',
'--pass-through',
action='store_true',
default=False,
help=
'be debugger friendly by passing captured output through to the system'
)
parser.add_option(
'-q',
'--quiet',
action='store_true',
default=False,
help='run quietly (errors, warnings, and progress only)')
parser.add_option(
'-t',
'--timing',
action='store_true',
default=False,
help='display per-test execution time (implies --verbose)')
parser.add_option(
'-v',
'--verbose',
action='count',
default=0,
help=
'verbose output (specify once for individual test results, twice for debug messages)'
)
# FIXME: Remove '--json' argument.
parser.add_option('--json',
action='store_true',
default=False,
help='write JSON formatted test results to stdout')
parser.add_option(
'--json-output',
action='store',
type='string',
dest='json_file_name',
help=
'Create a file at specified path, listing test results in JSON format.'
)
parser.epilog = (
'[args...] is an optional list of modules, test_classes, or individual tests. '
'If no args are given, all the tests will be run.')
return parser.parse_args(argv)
def main():
try:
usage = "[>] python3 %prog [options] arg\n\n\r[>] Example for spawn or attach app with -s(--script) options:\npython3 hook.py -p com.apple.AppStore / [-n App Store] -s trace_class.js\n\n\r[>] Example for spawn or attach app with -m(--method) options:\npython3 hook.py -p com.apple.AppStore / [-n App Store] -m app-static"
parser = optparse.OptionParser(usage, add_help_option=False)
info = optparse.OptionGroup(parser, "Information")
quick = optparse.OptionGroup(parser, "Quick Method")
parser.add_option('-h',
"--help",
action="help",
dest="help",
help="Show basic help message and exit")
#Using options -p(--package) for spawn application and load script
parser.add_option("-p",
"--package",
dest="package",
help="Identifier of the target app",
metavar="PACKAGE",
action="store",
type="string")
#Using options -n(--name) for attach script to application is running
parser.add_option("-n",
"--name",
dest="name",
help="Name of the target app",
metavar="NAME",
action="store",
type="string")
parser.add_option("-s",
"--script",
dest="script",
help="Frida Script Hooking",
metavar="SCIPRT.JS")
parser.add_option("-d",
"--dump",
action="store_true",
help="Dump decrypt application.ipa",
dest="dump")
parser.add_option("-c",
"--check-version",
action="store_true",
help="Check iOS hook for the newest version",
dest="checkversion")
parser.add_option("-u",
"--update",
action="store_true",
help="Update iOS hook to the newest version",
dest="update")
quick.add_option(
"-m",
"--method",
dest="method",
type="choice",
choices=[
'app-static', 'bypass-jb', 'bypass-ssl', 'i-url-req',
'i-crypto'
],
help=
"__app-static: Static Ananlysis Application(-n)\n\n\r\r__bypass-jb: Bypass Jailbreak Detection(-p)\n\n\r\r\r\r\r\r__bypass-ssl: Bypass SSL Pinning(-p)\n\n\n\n\n\n\n\n\n\r\r\r\r\r\r__i-url-req: Intercept URLRequest in App(-p)\n\n\n\n\n\n\n\n\n\r\r\r\r\r\r__i-crypto: Intercept Crypto in App(-p)",
metavar="app-static / bypass-jb / bypass-ssl / i-url-req / i-crypto"
)
#Some options to get info from device and applications
info.add_option("--list-devices",
action="store_true",
help="List All Devices",
dest="listdevices")
info.add_option("--list-apps",
action="store_true",
help="List The Installed apps",
dest="listapps")
info.add_option("--list-appinfo",
action="store_true",
help="List Info of Apps on Itunes",
dest="listappinfo")
info.add_option("--list-scripts",
action="store_true",
help="List All Scripts",
dest="listscripts")
parser.add_option_group(info)
parser.add_option_group(quick)
options, args = parser.parse_args()
methods = [
"method/ios_list_apps.js", #0
"method/static_analysis.js", #1
"method/bypass_ssl.js", #2
"method/bypass_jailbreak.js", #3
"method/intercept_url_request.js", #4
"method/intercept_crypto.js" #5
]
if options.listdevices:
logger.info('[*] List All Devices: ')
os.system('frida-ls-devices')
elif options.listapps:
logger.info('[*] List All Apps on Devies: ')
device = get_usb_iphone()
list_applications(device)
elif options.listappinfo:
method = methods[0]
if os.path.isfile(method):
logger.info('[*] List Info of Apps on Itunes: ')
process = 'itunesstored'
os.system('frida -U -n ' + process + ' -l ' + method)
#sys.stdin.read()
else:
logger.error('[?] Script not found!')
elif options.listscripts:
path = 'frida-scripts/'
if os.path.exists(path):
logger.info('[*] List All Scripts: ')
for file_name in os.listdir(path):
if fnmatch.fnmatch(file_name, '*.js'):
print('[*] ' + file_name)
else:
logger.error('[?] Path frida-script not exists!')
#Spawning application and load script
elif options.package and options.script:
if os.path.isfile(options.script):
logger.info('[*] Spawning: ' + options.package)
logger.info('[*] Script: ' + options.script)
time.sleep(2)
pid = frida.get_usb_device().spawn(options.package)
session = frida.get_usb_device().attach(pid)
hook = open(options.script, 'r')
script = session.create_script(hook.read())
script.load()
frida.get_usb_device().resume(pid)
sys.stdin.read()
else:
logger.error('[?] Script not found!')
#Attaching script to application
elif options.name and options.script:
if os.path.isfile(options.script):
logger.info('[*] Attaching: ' + options.name)
logger.info('[*] Script: ' + options.script)
time.sleep(2)
process = frida.get_usb_device().attach(options.name)
hook = open(options.script, 'r')
script = process.create_script(hook.read())
script.load()
sys.stdin.read()
else:
logger.error('[?] Script not found!')
#Static Analysis Application
elif options.name and options.method == "app-static":
method = methods[1]
if os.path.isfile(method):
logger.info('[*] Attaching: ' + options.name)
logger.info('[*] Method: ' + options.method)
time.sleep(2)
process = frida.get_usb_device().attach(options.name)
method = open(method, 'r')
script = process.create_script(method.read())
script.load()
sys.stdin.read()
else:
logger.error('[?] Script not found!')
#Bypass jailbreak
elif options.package and options.method == "bypass-jb":
method = methods[3]
if os.path.isfile(method):
logger.info('[*] Bypass Jailbreak: ')
logger.info('[*] Spawning: ' + options.package)
logger.info('[*] Script: ' + method)
time.sleep(2)
pid = frida.get_usb_device().spawn(options.package)
session = frida.get_usb_device().attach(pid)
hook = open(method, 'r')
script = session.create_script(hook.read())
script.load()
frida.get_usb_device().resume(pid)
sys.stdin.read()
else:
logger.error('[?] Script for method not found!')
#Bypass SSL Pinning
elif options.package and options.method == "bypass-ssl":
method = methods[2]
if os.path.isfile(method):
logger.info('[*] Bypass SSL Pinning: ')
logger.info('[*] Spawning: ' + options.package)
logger.info('[*] Script: ' + method)
os.system('frida -U -f ' + options.package + ' -l ' + method +
' --no-pause')
#sys.stdin.read()
else:
logger.error('[?] Script for method not found!')
#Intercept url request in app
elif options.package and options.method == "i-url-req":
method = methods[4]
print(method)
if os.path.isfile(method):
logger.info('[*] Intercept UrlRequest: ')
logger.info('[*] Spawning: ' + options.package)
logger.info('[*] Script: ' + method)
time.sleep(2)
pid = frida.get_usb_device().spawn(options.package)
session = frida.get_usb_device().attach(pid)
hook = open(method, 'r')
script = session.create_script(hook.read())
script.load()
frida.get_usb_device().resume(pid)
sys.stdin.read()
else:
logger.error('[?] Script for method not found!')
#Intercept Crypto Operations
elif options.package and options.method == "i-crypto":
method = methods[5]
if os.path.isfile(method):
logger.info('[*] Intercept Crypto Operations: ')
logger.info('[*] Spawning: ' + options.package)
logger.info('[*] Script: ' + method)
os.system('frida -U -f ' + options.package + ' -l ' + method +
' --no-pause')
#sys.stdin.read()
else:
logger.error('[?] Script for method not found!')
#check newversion
elif options.checkversion:
logger.info('[*] Checking for updates...')
is_newest = check_version(speak=True)
# if not is_newest:
# logger.info('[*] There is an update available for iOS hook')
#update newversion
elif options.update:
logger.info('[*] Update in progress...')
cmd = shlex.split("git pull origin master")
subprocess.call(cmd)
sys.exit(0)
#dump decrypt application
elif options.dump:
logger.warning('[!] The Method Is Updating!!')
else:
logger.warning("[!] Specify the options. use (-h) for more help!")
# sys.exit(0)
#EXCEPTION FOR FRIDA
except frida.ServerNotRunningError:
logger.error("Frida server is not running.")
except frida.TimedOutError:
logger.error("Timed out while waiting for device to appear.")
except frida.TransportError:
logger.error("[x_x] The application may crash or lose connection.")
except (frida.ProcessNotFoundError, frida.InvalidOperationError):
logger.error("[x_x] Unable to find process with name " + options.name +
". You need run app first.!!")
#EXCEPTION FOR OPTIONPARSING
#EXCEPTION FOR SYSTEM
except Exception as e:
logger.error(
"[x_x] Something went wrong, please check your error message.\n Message - {0}"
.format(e))
except KeyboardInterrupt:
logger.info("Bye bro!!")
def main(*largs, **kwargs):
global log
bin_path = os.environ['BINPATH']
sys.path.insert(0, bin_path)
# get map of name to module import path
decoder_map = getDecoders(setDecoderPath(os.environ['DECODERPATH']))
# The main argument parser. It will have every command line option
# available and should be used when actually parsing
parser = dshellOptionParser(
usage="usage: %prog [options] [decoder options] file1 file2 ... filen [-- [decoder args]+]",
version="%prog " + str(dshell.__version__), add_help_option=False)
# A short argument parser, meant to only hold the shorter list of
# arguments for when a decoder is called without a pcap file. DO
# NOT USE for any serious argument parsing.
parser_short = dshellOptionParser(
usage="usage: %prog [options] [decoder options] file1 file2 ... filen [-- [decoder args]+]",
version="%prog " + str(dshell.__version__), add_help_option=False)
parser.add_option('-h', '-?', '--help', dest='help',
help="Print common command-line flags and exit", action='store_true',
default=False)
parser_short.add_option('-h', '-?', '--help', dest='help',
help="Print common command-line flags and exit", action='store_true',
default=False)
parser.add_option('-d', '--decoder', dest="decoder",
action='append', help="Use a specific decoder module")
parser.add_option('-l', '--ls', '--list', action="store_true",
help='List all available decoders', dest='list')
parser.add_option(
'-C', '--config', dest='config', help='specify config.ini file')
parser.add_option('--tmpdir', dest='tmpdir', type='string', default=tempfile.gettempdir(),
help='alternate temp directory (for use when processing compressed pcap files)')
parser.add_option('-r', '--recursive', dest='recursive', action='store_true',
help='recursively process all PCAP files under input directory')
group = optparse.OptionGroup(parser, "Multiprocessing options")
group.add_option('-p', '--parallel', dest='parallel',
action='store_true', help='process multiple files in parallel')
group.add_option('-t', '--threaded', dest='threaded',
action='store_true', help='run multiple decoders in parallel')
group.add_option('-n', '--nprocs', dest='numprocs', type='int',
default=4, help='number of simultaneous processes')
parser.add_option_group(group)
# decode-pcap specific options
group = optparse.OptionGroup(parser, "Input options")
group.add_option('-i', '--interface', dest='interface',
default=None, help='listen live on INTERFACE')
group.add_option('-c', '--count', dest='count', type='int',
help='number of packets to process', default=0)
group.add_option('-f', '--bpf', dest='bpf',
help='replace default decoder filter (use carefully)')
group.add_option('--nofilterfn', dest='nofilterfn',
action="store_true", help='Set filterfn to pass-thru')
group.add_option('-F', dest='filefilter',
help='Use filefilter as input for the filter expression. An additional expression given on the command line is ignored.')
group.add_option(
'--ebpf', dest='ebpf', help='BPF filter to exclude traffic, extends other filters')
group.add_option('--no-vlan', dest='novlan', action="store_true",
help='do not examine traffic which has VLAN headers present')
group.add_option('--layer2', dest='layer2', default='ethernet.Ethernet',
help='select the layer-2 protocol module')
group.add_option('--strip', dest='striplayers', default=0,
help='extra data-link layers to strip')
parser.add_option_group(group)
group = optparse.OptionGroup(parser_short, "Input options")
group.add_option('-i', '--interface', dest='interface',
default=None, help='listen live on INTERFACE')
group.add_option('-c', '--count', dest='count', type='int',
help='number of packets to process', default=0)
group.add_option('-f', '--bpf', dest='bpf',
help='replace default decoder filter (use carefully)')
group.add_option('--nofilterfn', dest='nofilterfn',
action="store_true", help='Set filterfn to pass-thru')
group.add_option('-F', dest='filefilter',
help='Use filefilter as input for the filter expression. An additional expression given on the command line is ignored.')
group.add_option(
'--ebpf', dest='ebpf', help='BPF filter to exclude traffic, extends other filters')
group.add_option('--no-vlan', dest='novlan', action="store_true",
help='do not examine traffic which has VLAN headers present')
group.add_option('--layer2', dest='layer2', default='ethernet.Ethernet',
help='select the layer-2 protocol module')
group.add_option('--strip', dest='striplayers', default=0,
help='extra data-link layers to strip')
parser_short.add_option_group(group)
group = optparse.OptionGroup(parser, "Output options")
group.add_option('-o', '--outfile', dest='outfile', help='write output to the file OUTFILE. Additional output can be set with KEYWORD=VALUE,...\n' +
'\tmode=<w: write (default), a: append, noclobber: do not overwrite, use a a OUTFILE.1 (.2,.3) file if file(s) exists\n' +
'\tpcap=PCAPFILE to write packets to a PCAP file\n' +
'\tsession=SESSION to write session text\n' +
'\tdirection=data direction to write (c,s,both,split)')
group.add_option('--nobuf', help='turn off output buffering', dest='nobuffer',
action='store_true', default=False)
group.add_option('-w', '--session', dest='session',
help='write session file, same as -o session=')
group.add_option('-W', '--pcap', dest='pcap', default=None,
help='output decoded packets to PCAP (same as -o pcap=....)')
group.add_option('--db', dest='db', default=None,
help='output to db. Supply "config=file" or "param=...,param=..." ')
group.add_option(
'--oformat', dest='oformat', help='define the output format')
group.add_option('-x', '--extra', dest='oextra',
action='store_true', help='output a lot of extra information')
group.add_option('-O', '--output', dest='output', default=None,
help='Use a custom output module. Supply "modulename,option=value,..."')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Logging options")
group.add_option('-L', '--logfile', dest="logfile", help="log to file")
group.add_option('--debug', action="store_true", dest="debug",
help="debug logging (debug may also affect decoding behavior)")
group.add_option('-v', '--verbose', action="store_true",
dest="verbose", help="verbose logging")
group.add_option('-q', '--quiet', action="store_true",
dest="quiet", help="practically zero logging")
parser.add_option_group(group)
group = optparse.OptionGroup(parser_short, "Logging options")
group.add_option('-L', '--logfile', dest="logfile", help="log to file")
group.add_option('--debug', action="store_true", dest="debug",
help="debug logging (debug may also affect decoding behavior)")
group.add_option(
'-v', '--verbose', action="store_true", dest="verbose", help="verbose logging")
group.add_option('-q', '--quiet', action="store_true",
dest="quiet", help="practically zero logging")
parser_short.add_option_group(group)
# [decoder][option]=value dict of decoder options, set by config file
decoder_options = {}
decoder_args = []
args = []
extra_args = False
for x in largs:
if x == '--':
extra_args = True
continue
if extra_args:
decoder_args.append(x)
else:
args.append(x)
# parse basic options and crdate the options object
options = parser.parse_args(args, **kwargs)[0]
if options == None:
print "\nError processing provided arguments"
return
# dump list
if options.list:
printDecoders(decoder_map, not options.debug)
return
# parse config file, updating the options and decoder_options dicts
if options.config:
try:
import ConfigParser
config = ConfigParser.ConfigParser()
config.read(options.config)
for s in config.sections():
# this is the main section, set the options
if s.lower() == 'dshell':
for k, v in config.items(s, raw=True):
if k in options.__dict__:
options.__dict__[k] = v
except:
raise # :-(
# are we a thread outputting to a queue?
if 'queue' in options.__dict__:
out = output.QueueOutput(options.queue)
# if not, parse output args
else:
outfile = None
outkw = {}
# set output file (and other args if -o filename,key=val...)
if options.outfile:
outfile, outkw = util.strtok(options.outfile)
if options.nobuffer:
outkw.update(nobuffer=True)
# output extra?
if options.oextra:
outkw.update(extra=True)
# set session writer?
if options.session:
outkw.update(session=options.session)
# add default pcap writer?
if options.pcap:
outkw.update(pcap=options.pcap)
# use database?
if options.db:
a, kw = util.strtok(options.db, as_list=True)
# add output options
kw.update(outkw)
out = output.DBOutput(*a, **kw)
# if not db mode and no out module specd
# use default output lib to get default module
elif not options.output:
options.output = 'output'
# init output module
if options.output:
# parse output arglist (-O module,args..,k=val...)
a, kw = util.strtok(options.output, as_list=True)
kw.update(outkw) # set output options
if outfile:
# set filename arg if -o given (can also be first arg in module
# arglist)
kw.update(file=outfile)
outmod = import_module(name=os.path.basename(a[0])) # load module
if outmod:
# pass remaining args and keyword args to init object
out = outmod.obj(*a[1:], **kw)
# set the output format
if options.oformat != None:
out.setformat(options.oformat)
# set global log functions
out.logger = logging.getLogger('dshell')
log = out.log
# start up the logger
if options.debug:
level = logging.DEBUG
elif options.verbose:
level = logging.INFO
elif options.quiet:
level = logging.FATAL
else:
level = logging.WARNING
logging.basicConfig(filename=options.logfile, level=level)
decoders = {}
decoderNames = set()
# check for a decoder
if options.decoder != None:
# single decoder or came from config file
if type(options.decoder) == str:
options.decoder = util.strtok(
options.decoder, as_list=True)[0] # make it a list
# we have a list of decoders
for dnames in options.decoder:
chain = dnames.split('+')
# last module does not have a subdecoder
module = chain.pop()
try:
module, n = module.split(':', 1)
except:
n = None
m = import_module(module, search=decoder_map)
if m:
# create copy in case we import multiple times under different
# names
dObj = copy.copy(m.dObj)
if n:
dObj.name = n
else:
dObj = None
try:
decoderNames.add(dObj.name)
except AttributeError:
decoderNames.add(module)
# walk up the chain, setting sub-decoders
while chain:
subObj = dObj
module = chain.pop()
try:
module, n = module.split(':', 1)
except: # :-(
n = None
m = import_module(module, search=decoder_map)
if m:
dObj = copy.copy(m.dObj)
if n:
dObj.name = n
else:
dObj = None
try:
decoderNames.add(dObj.name)
except AttributeError:
decoderNames.add(module)
if dObj and dObj.chainable:
dObj.subDecoder = subObj
elif dObj:
sys.stderr.write("Error %s is not chainable\n" % module)
# insert the top decoder in the dict
if dObj:
decoders[dObj.name] = dObj
# save option dict
options_dict = options.__dict__.copy()
# add in options for loaded decoders and subdecoders
for d in decoders.itervalues():
parser.add_decoder_options(d)
for d in decoders.itervalues():
parser_short.add_decoder_options(d)
# reparse args to handle decoder options
optionerror = False
try:
options, args = parser.parse_args(args, **kwargs)
except:
optionerror = True
# replace base options
options.__dict__.update(options_dict)
# look for name_option keys and put them in decoder_options[name][option]
for k, v in options.__dict__.iteritems():
for decName in decoderNames:
try:
n = k.split(decName + '_', 1)[1]
decoder_options.setdefault(decName, {}).setdefault(n, v)
except IndexError:
continue
# reparse config file to handle decoder options
if options.config:
for s in config.sections():
# set the options for loaded decoders if they are present in the
# config file
if s.lower() in decoder_options:
for k, v in config.items(s, raw=True):
if k in decoder_options[s]: # if this is a valid option
if v.isdigit():
v = int(v) # try conversion to int/float
elif '.' in v:
try:
v = float(v)
except:
pass
decoder_options[s][k] = v
if any(x in ('-h', '-?', '--help') for x in sys.argv[1:]):
# Print the verbose help message
parser.print_help()
printDecoderBriefs(decoders)
return
if optionerror or (not args and not options.interface):
# Print the short help message
parser_short.print_help()
printDecoderBriefs(decoders)
return
#######################################################################
# listen live on the interface
# this will not process any files
if options.interface != None:
if len(decoders) != 1:
print 'Can only run one module live on an interface'
return
# handles options and arguments for dumping live on an interface
decode_live(out, options, dObj, decoder_args, decoder_options)
# close output
out.close()
return
#######################################################################
# take all other command line arguments as files to process
####################################################
# Works if directory (e.g. ~/data/) or * (e.g. ~/data/*
# used on command line. Does not work for partial
# wildcards (e.g. ~/data/*.dat) because bash
# expands wildcards before passing arguments into
# decode-pcap.py. Will work if no matches in root of
# path specified.
####################################################
inputs = []
for file_path in args:
# If this argument is a directory and RECURSIVE specified, then add
# entire directory tree to the list of input files
if os.path.isdir(file_path) and options.recursive:
addFilesFromDirectory(inputs, file_path)
# If a wildcard is specified, then handle accordingly
elif file_path.find('*') > -1:
(path, wildcard) = os.path.split(file_path)
# If just file is specified (no path)
if len(path) == 0:
inputs.extend(glob.glob(wildcard))
# If there is a path, but recursion not specified,
# then just add matching files from specified dir
elif not len(path) == 0 and not options.recursive:
inputs.extend(glob.glob(file_path))
# Otherwise, recursion specified and there is a directory.
# Recurse directory and add files
else:
addFilesFromDirectory(inputs, path, wildcard)
# Just a normal file, append to list of inputs
else:
inputs.append(file_path)
if options.parallel or options.threaded:
import multiprocessing
procs = []
q = multiprocessing.Queue()
kwargs = options.__dict__.copy() # put parsed base options in kwargs
kwargs.update(config=None, outfile=None, queue=q) # pass the q,
# do not pass the config file or outfile because we handled that here
for d in decoder_options: # put pre-parsed decoder options in kwargs
for k, v in decoder_options[d].items():
kwargs[d + '_' + k] = v
# check here to see if we are running in parallel-file mode
if options.parallel and len(inputs) > 1:
for f in inputs:
# create a child process for each input file
procs.append(
multiprocessing.Process(target=main, kwargs=kwargs, args=[f]))
runChildProcs(procs, q, out, numprocs=options.numprocs)
# check here to see if we are running decoders multithreaded
elif options.threaded and len(options.decoder) > 1:
for d in options.decoder:
# create a child for each decoder
kwargs.update(decoder=d)
procs.append(
multiprocessing.Process(target=main, kwargs=kwargs, args=inputs))
runChildProcs(procs, q, out, numprocs=options.numprocs)
# fall through to here (single threaded or child process)
else:
#
# Here is where we use the decoder(s) to process the pcap
#
temporaryFiles = [] # used when uncompressing files
for module in decoders.keys():
decoder = decoders[module]
initDecoderOptions(
decoder, out, options, decoder_args, decoder_options)
# If the decoder has a preModule function, will execute it now
decoder.preModule()
for input_file in inputs:
# Decoder-specific options may be seen as input files
# Skip anything starts with "--"
if input_file[:2] == '--':
continue
# Recursive directory processing is handled elsewhere,
# so we should only be dealing with files at this point.
if os.path.isdir(input_file):
continue
log('+Processing file %s' % input_file)
# assume the input_file is not compressed
# Allows the processing of .pcap files that are compressed with
# gzip, bzip2, or zip. Writes uncompressed file to a
# NamedTemporaryFile and unlinks the file once it is no longer
# needed. Might consider using mkstemp() since this implementation
# requires Python >= 2.6.
try:
exts = ['.gz', '.bz2', '.zip']
if os.path.splitext(input_file)[1] not in exts:
pcapfile = input_file
else:
# we have a compressed file
tmpfile = expandCompressedFile(
input_file, options.verbose, options.tmpdir)
temporaryFiles.append(tmpfile)
pcapfile = tmpfile
except:
if options.verbose:
sys.stderr.write(
'+Error processing file %s' % (input_file))
continue
# give the decoder access to the input filename
# motivation: run a decoder against a large number of pcap
# files and have the decoder print the filename
# so you can go straight to the pcap file for
# further analysis
decoder.input_file = input_file
# Check to see if the decoder has a preFile function
# This will be called before the decoder processes each
# input file
decoder.preFile()
try:
if not pcap:
raise NotImplementedError(
"pcap support not implemented")
decoder.capture = pcap.pcap(pcapfile)
if decoder.filter:
decoder.capture.setfilter(decoder.filter)
while not options.count or decoder.count < options.count:
try:
# read next packet and break on EOF
ts, pkt = decoder.capture.next()
except:
break # no data
# for ts, pkt in dpkt.pcap.Reader(open(pcapfile,'rb')): # someday we may need this
decoder.decode(ts, pkt)
except KeyboardInterrupt:
raise
except:
traceback.print_exc()
if options.verbose:
log('+Done processing %s' % (input_file))
# call that decoder's processFile()
decoder.postFile()
# check to see if the decoder is using the Messages class
# if so, we need to clean up the connection store to
# purge any unfinished connections
if 'cleanConnectionStore' in dir(decoder):
decoder.cleanConnectionStore()
# Check to see if the decoder has a postModule function
# A postModule function will be called when the module
# has finished running against all of the input files
if 'postModule' in dir(decoder):
decoder.postModule()
# remove any temporary files that were created during execution
for tmpfile in temporaryFiles:
if options.verbose:
log('+Unlinking %s' % (tmpfile))
os.unlink(tmpfile)
# close output
out.close()
return
help="""set vehicle frame type
%s""" % (generate_frame_help()))
parser.add_option("-C",
"--sim_vehicle_sh_compatible",
action='store_true',
default=False,
help="be compatible with the way sim_vehicle.sh works; "
"make this the first option")
parser.add_option("-H",
"--hil",
action='store_true',
default=False,
help="start HIL")
group_build = optparse.OptionGroup(parser, "Build options")
group_build.add_option("-N",
"--no-rebuild",
action='store_true',
default=False,
help="don't rebuild before starting ardupilot")
group_build.add_option("-D",
"--debug",
action='store_true',
default=False,
help="build with debugging")
group_build.add_option("-c",
"--clean",
action='store_true',
default=False,
help="do a make clean before building")
def main(args=None):
if args is None:
args = sys.argv[1:]
usage = 'Usage: %prog command [options] [arguments]'
version = 'Rudix Package Manager (%prog) version ' + __version__ + '\n'
version += __copyright__
parser = optparse.OptionParser(usage=usage, version=version)
parser.add_option('-v',
'--verbose',
action='store_true',
default=False,
help='displays more information when available')
parser.add_option('--volume',
default=Volume,
help='set volume to use. Default "%default"')
parser.add_option('--force',
action='store_true',
default=False,
help='force operation')
commands = optparse.OptionGroup(parser, 'Commands',
'The Package manager commands.')
commands.add_option('-l',
'--list',
action='store_const',
dest='command',
const=command_list,
help='list all packages installed')
commands.add_option('-I',
'--info',
'--show',
action='store_const',
dest='command',
const=command_info,
help='show information about installed packages')
commands.add_option('-L',
'--files',
'--content',
action='store_const',
dest='command',
const=command_files,
help="show packages's files")
commands.add_option('-i',
'--install',
action='store_const',
dest='command',
const=command_install,
help='install local or remote package(s)')
commands.add_option('-d',
'--download',
action='store_const',
dest='command',
const=command_download,
help='download package(s) but do not install')
commands.add_option('-u',
'--update',
'--upgrade',
action='store_const',
dest='command',
const=command_update,
help='update all packages')
commands.add_option('-r',
'--remove',
'--uninstall',
action='store_const',
dest='command',
const=command_remove,
help='remove (uninstall) package(s)')
commands.add_option('-R',
'--remove-all',
'--uninstall-all',
action='store_const',
dest='command',
const=command_remove_all,
help='remove (uninstall) ALL packages')
commands.add_option('-t',
'--status',
action='store_const',
dest='command',
const=command_status,
help='show repository status')
commands.add_option('-s',
'--search',
action='store_const',
dest='command',
const=command_search,
help='search for remote packages')
commands.add_option(
'-S',
'--search-path',
action='store_const',
dest='command',
const=command_search_path,
help='search for path in all packages and print if matched')
commands.add_option('-a',
'--alias',
action='store_const',
dest='command',
const=command_alias,
help='list aliases')
commands.add_option('-z',
'--freeze',
action='store_const',
dest='command',
const=command_freeze,
help='freeze package list.')
parser.add_option_group(commands)
parser.set_defaults(command=command_list)
# Allow commands without dashes
if args:
command = args[0]
if command.startswith('-') is False:
args[0] = '--' + command
(options, args) = parser.parse_args(args)
return options.command(options, args)
def create_optparser(sdk_dir):
"""Create the parser for the command line.
"""
def store_abspath(option, opt_str, value, parser):
setattr(parser.values, option.dest, os.path.abspath(value))
def store_abspath_dir(option, opt_str, value, parser):
if not os.path.isdir(value):
raise optparse.OptionValueError("'%s' is not a directory" % value)
setattr(parser.values, option.dest, os.path.abspath(value))
def store_abspath_file(option, opt_str, value, parser):
if not os.path.isfile(value):
raise optparse.OptionValueError("'%s' is not a file" % value)
setattr(parser.values, option.dest, os.path.abspath(value))
def store_version(option, opt_str, value, parser):
version = siputils.version_from_string(value)
if version is None:
raise optparse.OptionValueError(
"'%s' is not a valid version number" % value)
setattr(parser.values, option.dest, version)
p = optparse.OptionParser(usage="python %prog [opts] [macro=value] "
"[macro+=value]",
version=sip_version_str)
# Note: we don't use %default to be compatible with Python 2.3.
p.add_option("-k",
"--static",
action="store_true",
default=False,
dest="static",
help="build the SIP module as a static library")
p.add_option("-p",
"--platform",
action="store",
type="string",
metavar="PLATFORM",
dest="platform",
help="the platform/compiler "
"configuration [default: %s]" % build_platform)
p.add_option("-u",
"--debug",
action="store_true",
default=False,
help="build with debugging symbols")
p.add_option("--sip-module",
action="store",
default="sip",
type="string",
metavar="NAME",
dest="sip_module",
help="the package.module name "
"of the sip module [default: sip]")
p.add_option("--configuration",
dest='config_file',
type='string',
action='callback',
callback=store_abspath_file,
metavar="FILE",
help="FILE contains the target configuration")
p.add_option("--target-py-version",
dest='target_py_version',
type='string',
action='callback',
callback=store_version,
metavar="VERSION",
help="the major.minor version of the target Python [default: "
"%s]" % siputils.version_to_string(py_version, parts=2))
p.add_option("--sysroot",
dest='sysroot',
type='string',
action='callback',
callback=store_abspath_dir,
metavar="DIR",
help="DIR is the target system root directory")
p.add_option("--no-module",
action="store_true",
default=False,
dest="no_module",
help="disable the installation of the sip "
"module [default: enabled]")
p.add_option("--no-tools",
action="store_true",
default=False,
dest="no_tools",
help="disable the building of the code generator "
"and the installation of the build system [default: enabled]")
p.add_option("--use-qmake",
action="store_true",
default=False,
dest="use_qmake",
help="generate qmake .pro files instead of "
"Makefiles")
if sys.platform == 'darwin':
# Get the latest SDK to use as the default.
sdks = glob.glob(sdk_dir + '/MacOSX*.sdk')
if len(sdks) > 0:
sdks.sort()
_, default_sdk = os.path.split(sdks[-1])
else:
default_sdk = 'MacOSX10.4u.sdk'
g = optparse.OptionGroup(p, title="MacOS X Configuration")
g.add_option("--arch",
action="append",
default=[],
dest="arch",
choices=["i386", "x86_64", "ppc"],
help="build for architecture ARCH")
g.add_option("--deployment-target",
action="store",
default='',
metavar="VERSION",
dest="deployment_target",
help="set the value of the MACOSX_DEPLOYMENT_TARGET "
"environment variable in generated Makefiles")
g.add_option(
"-n",
"--universal",
action="store_true",
default=False,
dest="universal",
help="build the SIP code generator and module as universal "
"binaries")
g.add_option("-s",
"--sdk",
action="store",
default=default_sdk,
type="string",
metavar="SDK",
dest="sdk",
help="the name of the SDK used when building universal "
"binaries [default: %s]" % default_sdk)
p.add_option_group(g)
# Querying.
g = optparse.OptionGroup(p, title="Query")
g.add_option("--show-platforms",
action="store_true",
default=False,
dest="show_platforms",
help="show the list of supported "
"platform/compiler configurations")
g.add_option("--show-build-macros",
action="store_true",
default=False,
dest="show_build_macros",
help="show the list of supported build "
"macros")
p.add_option_group(g)
# Installation.
g = optparse.OptionGroup(p, title="Installation")
g.add_option(
"-b",
"--bindir",
action="callback",
type="string",
metavar="DIR",
dest="sipbindir",
callback=store_abspath,
help="where the SIP code generator will be installed [default: "
"%s]" % plat_bin_dir)
g.add_option("-d",
"--destdir",
action="callback",
type="string",
metavar="DIR",
dest="destdir",
callback=store_abspath,
help="where the SIP module will be installed [default: "
"%s]" % plat_py_site_dir)
g.add_option("-e",
"--incdir",
action="callback",
type="string",
metavar="DIR",
dest="sipincdir",
callback=store_abspath,
help="where the SIP header file will be installed [default: "
"%s]" % plat_py_venv_inc_dir)
g.add_option("-v",
"--sipdir",
action="callback",
type="string",
metavar="DIR",
dest="sipsipdir",
callback=store_abspath,
help="where .sip files are normally installed [default: "
"%s]" % plat_sip_dir)
g.add_option("--no-dist-info",
action="store_false",
default=True,
dest="distinfo",
help="do not install the dist-info directory")
g.add_option("--no-stubs",
"--no-pyi",
action="store_false",
default=True,
dest="pyi",
help="do not install the sip.pyi stub file")
g.add_option(
"--stubsdir",
"--pyidir",
action="callback",
type="string",
metavar="DIR",
dest="pyidir",
callback=store_abspath,
help="where the sip.pyi stub file will be installed [default: "
"%s]" % plat_py_site_dir)
p.add_option_group(g)
return p
def _mixin_setup(self):
self.add_option('-r',
'--roster',
default=False,
help='The name of the Salt Roster to use.')
self.add_option(
'--roster-file',
dest='roster_file',
help='Absolute path to the Roster file to use.',
)
self.add_option(
'--sync',
default=False,
action='store_true',
help=('Return the replies from the devices immediately they are '
'received, or everything at once.'),
)
self.add_option(
'--cache-grains',
default=False,
action='store_true',
help=('Cache the collected Grains. This is going to override the '
'existing cached Grains.'),
)
self.add_option(
'--cache-pillar',
default=False,
action='store_true',
help=('Cache the compiled Pillar. This is going to override the '
'existing cached Pillar.'),
)
self.add_option(
'--no-cached-grains',
default=False,
action='store_true',
help='Do not use the available cached Grains (if any).',
)
self.add_option(
'--no-cached-pillar',
default=False,
action='store_true',
help='Do not use the available cached Pillar (if any)',
)
self.add_option(
'--no-grains',
default=False,
action='store_true',
help=('Do not attempt to collect Grains at all. Use with care, it '
'may lead to unexpected results.'),
)
self.add_option(
'--no-pillar',
default=False,
action='store_true',
help=(
'Do not compile Pillar at all. Use with care, it may lead to '
'unexpected results.'),
)
self.add_option(
'-b',
'--batch',
'--batch-size',
default=CPU_COUNT,
dest='batch_size',
help=('The number of devices to connect to in parallel. '
'Default: {}'.format(CPU_COUNT)),
)
self.add_option(
'--preview-target',
dest='preview_target',
action='store_true',
help='Show the devices expected to match the target.',
)
self.add_option(
'--sync-roster',
dest='sync_roster',
action='store_true',
help=('Synchronise the Roster modules (both salt-sproxy native '
'and provided by the user in their own environment).'),
)
self.add_option(
'--events',
dest='events',
action='store_true',
help=('Whether should put the events on the Salt bus (mostly '
'useful when having a Master running).'),
)
self.add_option(
'--use-proxy',
'--use-existing-proxy',
dest='use_existing_proxy',
action='store_true',
help=('Use the existing Proxy Minions to execute the commands, '
'whenever available.'),
)
self.add_option(
'--file-roots',
'--display-file-roots',
dest='display_file_roots',
action='store_true',
help=(
'Display the file_roots option you would need to configure '
'in order to use the salt-sproxy extension modules directly, '
'and, implicitly, leverage the event-driven methodology and '
'the Salt REST API.'),
)
self.add_option(
'--save-file-roots',
dest='save_file_roots',
action='store_true',
help=(
'Saves the file_roots configuration so you can start '
'leveraging the event-driven automation and the Salt REST API.'
),
)
group = self.output_options_group = optparse.OptionGroup(
self, 'Output Options', 'Configure your preferred output format.')
self.add_option_group(group)
group.add_option(
'-q',
'--quiet',
default=False,
action='store_true',
help='Do not display the results of the run.',
)
def main():
# Silence upload.py.
rietveld.upload.verbosity = 0
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option(
'-u',
'--user',
metavar='<email>',
# Look for USER and USERNAME (Windows) environment variables.
default=os.environ.get('USER', os.environ.get('USERNAME')),
help='Filter on user, default=%default')
parser.add_option('-b',
'--begin',
metavar='<date>',
help='Filter issues created after the date (mm/dd/yy)')
parser.add_option('-e',
'--end',
metavar='<date>',
help='Filter issues created before the date (mm/dd/yy)')
quarter_begin, quarter_end = get_quarter_of(datetime.today() -
relativedelta(months=2))
parser.add_option(
'-Q',
'--last_quarter',
action='store_true',
help='Use last quarter\'s dates, i.e. %s to %s' %
(quarter_begin.strftime('%Y-%m-%d'), quarter_end.strftime('%Y-%m-%d')))
parser.add_option('-Y',
'--this_year',
action='store_true',
help='Use this year\'s dates')
parser.add_option('-w',
'--week_of',
metavar='<date>',
help='Show issues for week of the date (mm/dd/yy)')
parser.add_option(
'-W',
'--last_week',
action='count',
help='Show last week\'s issues. Use more times for more weeks.')
parser.add_option(
'-a',
'--auth',
action='store_true',
help='Ask to authenticate for instances with no auth cookie')
parser.add_option('-d',
'--deltas',
action='store_true',
help='Fetch deltas for changes.')
parser.add_option(
'--no-referenced-issues',
action='store_true',
help='Do not fetch issues referenced by owned changes. Useful in '
'combination with --changes-by-issue when you only want to list '
'issues that have also been modified in the same time period.')
parser.add_option(
'--skip-own-issues-without-changes',
action='store_true',
help='Skips listing own issues without changes when showing changes '
'grouped by referenced issue(s). See --changes-by-issue for more '
'details.')
activity_types_group = optparse.OptionGroup(
parser, 'Activity Types',
'By default, all activity will be looked up and '
'printed. If any of these are specified, only '
'those specified will be searched.')
activity_types_group.add_option('-c',
'--changes',
action='store_true',
help='Show changes.')
activity_types_group.add_option('-i',
'--issues',
action='store_true',
help='Show issues.')
activity_types_group.add_option('-r',
'--reviews',
action='store_true',
help='Show reviews.')
activity_types_group.add_option(
'--changes-by-issue',
action='store_true',
help='Show changes grouped by referenced issue(s).')
parser.add_option_group(activity_types_group)
output_format_group = optparse.OptionGroup(
parser, 'Output Format',
'By default, all activity will be printed in the '
'following format: {url} {title}. This can be '
'changed for either all activity types or '
'individually for each activity type. The format '
'is defined as documented for '
'string.format(...). The variables available for '
'all activity types are url, title, author, '
'created and modified. Format options for '
'specific activity types will override the '
'generic format.')
output_format_group.add_option(
'-f',
'--output-format',
metavar='<format>',
default=u'{url} {title}',
help='Specifies the format to use when printing all your activity.')
output_format_group.add_option(
'--output-format-changes',
metavar='<format>',
default=None,
help='Specifies the format to use when printing changes. Supports the '
'additional variable {reviewers}')
output_format_group.add_option(
'--output-format-issues',
metavar='<format>',
default=None,
help='Specifies the format to use when printing issues. Supports the '
'additional variable {owner}.')
output_format_group.add_option(
'--output-format-reviews',
metavar='<format>',
default=None,
help='Specifies the format to use when printing reviews.')
output_format_group.add_option(
'--output-format-heading',
metavar='<format>',
default=u'{heading}:',
help='Specifies the format to use when printing headings.')
output_format_group.add_option(
'--output-format-no-url',
default='{title}',
help='Specifies the format to use when printing activity without url.')
output_format_group.add_option(
'-m',
'--markdown',
action='store_true',
help='Use markdown-friendly output (overrides --output-format '
'and --output-format-heading)')
output_format_group.add_option(
'-j',
'--json',
action='store_true',
help='Output json data (overrides other format options)')
parser.add_option_group(output_format_group)
auth.add_auth_options(parser)
parser.add_option('-v',
'--verbose',
action='store_const',
dest='verbosity',
default=logging.WARN,
const=logging.INFO,
help='Output extra informational messages.')
parser.add_option('-q',
'--quiet',
action='store_const',
dest='verbosity',
const=logging.ERROR,
help='Suppress non-error messages.')
parser.add_option('-M',
'--merged-only',
action='store_true',
dest='merged_only',
default=False,
help='Shows only changes that have been merged.')
parser.add_option(
'-C',
'--completed-issues',
action='store_true',
dest='completed_issues',
default=False,
help='Shows only monorail issues that have completed (Fixed|Verified) '
'by the user.')
parser.add_option(
'-o',
'--output',
metavar='<file>',
help='Where to output the results. By default prints to stdout.')
# Remove description formatting
parser.format_description = (lambda _: parser.description) # pylint: disable=no-member
options, args = parser.parse_args()
options.local_user = os.environ.get('USER')
if args:
parser.error('Args unsupported')
if not options.user:
parser.error('USER/USERNAME is not set, please use -u')
options.user = username(options.user)
logging.basicConfig(level=options.verbosity)
# python-keyring provides easy access to the system keyring.
try:
import keyring # pylint: disable=unused-import,unused-variable,F0401
except ImportError:
logging.warning('Consider installing python-keyring')
if not options.begin:
if options.last_quarter:
begin, end = quarter_begin, quarter_end
elif options.this_year:
begin, end = get_year_of(datetime.today())
elif options.week_of:
begin, end = (get_week_of(
datetime.strptime(options.week_of, '%m/%d/%y')))
elif options.last_week:
begin, end = (
get_week_of(datetime.today() -
timedelta(days=1 + 7 * options.last_week)))
else:
begin, end = (get_week_of(datetime.today() - timedelta(days=1)))
else:
begin = dateutil.parser.parse(options.begin)
if options.end:
end = dateutil.parser.parse(options.end)
else:
end = datetime.today()
options.begin, options.end = begin, end
if begin >= end:
# The queries fail in peculiar ways when the begin date is in the future.
# Give a descriptive error message instead.
logging.error(
'Start date (%s) is the same or later than end date (%s)' %
(begin, end))
return 1
if options.markdown:
options.output_format_heading = '### {heading}\n'
options.output_format = ' * [{title}]({url})'
options.output_format_no_url = ' * {title}'
logging.info('Searching for activity by %s', options.user)
logging.info('Using range %s to %s', options.begin, options.end)
my_activity = MyActivity(options)
my_activity.show_progress('Loading data')
if not (options.changes or options.reviews or options.issues
or options.changes_by_issue):
options.changes = True
options.issues = True
options.reviews = True
# First do any required authentication so none of the user interaction has to
# wait for actual work.
if options.changes or options.changes_by_issue:
my_activity.auth_for_changes()
if options.reviews:
my_activity.auth_for_reviews()
logging.info('Looking up activity.....')
try:
if options.changes or options.changes_by_issue:
my_activity.get_changes()
if options.reviews:
my_activity.get_reviews()
if options.issues or options.changes_by_issue:
my_activity.get_issues()
if not options.no_referenced_issues:
my_activity.get_referenced_issues()
except auth.AuthenticationError as e:
logging.error('auth.AuthenticationError: %s', e)
my_activity.show_progress('\n')
my_activity.print_access_errors()
output_file = None
try:
if options.output:
output_file = open(options.output, 'w')
logging.info('Printing output to "%s"', options.output)
sys.stdout = output_file
except (IOError, OSError) as e:
logging.error('Unable to write output: %s', e)
else:
if options.json:
my_activity.dump_json()
else:
if options.changes:
my_activity.print_changes()
if options.reviews:
my_activity.print_reviews()
if options.issues:
my_activity.print_issues()
if options.changes_by_issue:
my_activity.print_changes_by_issue(
options.skip_own_issues_without_changes)
finally:
if output_file:
logging.info('Done printing to file.')
sys.stdout = sys.__stdout__
output_file.close()
return 0
def parseOpts(overrideArguments=None):
def _readOptions(filename_bytes, default=[]):
try:
optionf = open(filename_bytes)
except IOError:
return default # silently skip if file is not present
try:
res = []
for l in optionf:
res += shlex.split(l, comments=True)
finally:
optionf.close()
return res
def _readUserConf():
xdg_config_home = compat_getenv('XDG_CONFIG_HOME')
if xdg_config_home:
userConfFile = os.path.join(xdg_config_home, 'youtube-dl',
'config')
if not os.path.isfile(userConfFile):
userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
else:
userConfFile = os.path.join(compat_expanduser('~'), '.config',
'youtube-dl', 'config')
if not os.path.isfile(userConfFile):
userConfFile = os.path.join(compat_expanduser('~'), '.config',
'youtube-dl.conf')
userConf = _readOptions(userConfFile, None)
if userConf is None:
appdata_dir = compat_getenv('appdata')
if appdata_dir:
userConf = _readOptions(os.path.join(appdata_dir, 'youtube-dl',
'config'),
default=None)
if userConf is None:
userConf = _readOptions(os.path.join(
appdata_dir, 'youtube-dl', 'config.txt'),
default=None)
if userConf is None:
userConf = _readOptions(os.path.join(compat_expanduser('~'),
'youtube-dl.conf'),
default=None)
if userConf is None:
userConf = _readOptions(os.path.join(compat_expanduser('~'),
'youtube-dl.conf.txt'),
default=None)
if userConf is None:
userConf = []
return userConf
def _format_option_string(option):
''' ('-o', '--option') -> -o, --format METAVAR'''
opts = []
if option._short_opts:
opts.append(option._short_opts[0])
if option._long_opts:
opts.append(option._long_opts[0])
if len(opts) > 1:
opts.insert(1, ', ')
if option.takes_value():
opts.append(' %s' % option.metavar)
return "".join(opts)
def _comma_separated_values_options_callback(option, opt_str, value,
parser):
setattr(parser.values, option.dest, value.split(','))
def _hide_login_info(opts):
opts = list(opts)
for private_opt in [
'-p', '--password', '-u', '--username', '--video-password'
]:
try:
i = opts.index(private_opt)
opts[i + 1] = 'PRIVATE'
except ValueError:
pass
return opts
# No need to wrap help messages if we're on a wide console
columns = compat_get_terminal_size().columns
max_width = columns if columns else 80
max_help_position = 80
fmt = optparse.IndentedHelpFormatter(width=max_width,
max_help_position=max_help_position)
fmt.format_option_strings = _format_option_string
kw = {
'version': __version__,
'formatter': fmt,
'usage': '%prog [OPTIONS] URL [URL...]',
'conflict_handler': 'resolve',
}
parser = optparse.OptionParser(**compat_kwargs(kw))
general = optparse.OptionGroup(parser, 'General Options')
general.add_option('-h',
'--help',
action='help',
help='Print this help text and exit')
general.add_option('-v',
'--version',
action='version',
help='Print program version and exit')
general.add_option(
'-U',
'--update',
action='store_true',
dest='update_self',
help=
'Update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)'
)
general.add_option(
'-i',
'--ignore-errors',
action='store_true',
dest='ignoreerrors',
default=False,
help=
'Continue on download errors, for example to skip unavailable videos in a playlist'
)
general.add_option(
'--abort-on-error',
action='store_false',
dest='ignoreerrors',
help=
'Abort downloading of further videos (in the playlist or the command line) if an error occurs'
)
general.add_option('--dump-user-agent',
action='store_true',
dest='dump_user_agent',
default=False,
help='Display the current browser identification')
general.add_option('--list-extractors',
action='store_true',
dest='list_extractors',
default=False,
help='List all supported extractors')
general.add_option('--extractor-descriptions',
action='store_true',
dest='list_extractor_descriptions',
default=False,
help='Output descriptions of all supported extractors')
general.add_option('--force-generic-extractor',
action='store_true',
dest='force_generic_extractor',
default=False,
help='Force extraction to use the generic extractor')
general.add_option(
'--default-search',
dest='default_search',
metavar='PREFIX',
help=
'Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.'
)
general.add_option(
'--ignore-config',
action='store_true',
help='Do not read configuration files. '
'When given in the global configuration file /etc/youtube-dl.conf: '
'Do not read the user configuration in ~/.config/youtube-dl/config '
'(%APPDATA%/youtube-dl/config.txt on Windows)')
general.add_option(
'--flat-playlist',
action='store_const',
dest='extract_flat',
const='in_playlist',
default=False,
help='Do not extract the videos of a playlist, only list them.')
general.add_option('--no-color',
'--no-colors',
action='store_true',
dest='no_color',
default=False,
help='Do not emit color codes in output')
network = optparse.OptionGroup(parser, 'Network Options')
network.add_option(
'--proxy',
dest='proxy',
default=None,
metavar='URL',
help=
'Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection'
)
network.add_option('--socket-timeout',
dest='socket_timeout',
type=float,
default=None,
metavar='SECONDS',
help='Time to wait before giving up, in seconds')
network.add_option(
'--source-address',
metavar='IP',
dest='source_address',
default=None,
help='Client-side IP address to bind to (experimental)',
)
network.add_option(
'-4',
'--force-ipv4',
action='store_const',
const='0.0.0.0',
dest='source_address',
help='Make all connections via IPv4 (experimental)',
)
network.add_option(
'-6',
'--force-ipv6',
action='store_const',
const='::',
dest='source_address',
help='Make all connections via IPv6 (experimental)',
)
network.add_option(
'--cn-verification-proxy',
dest='cn_verification_proxy',
default=None,
metavar='URL',
help='Use this proxy to verify the IP address for some Chinese sites. '
'The default proxy specified by --proxy (or none, if the options is not present) is used for the actual downloading. (experimental)'
)
selection = optparse.OptionGroup(parser, 'Video Selection')
selection.add_option(
'--playlist-start',
dest='playliststart',
metavar='NUMBER',
default=1,
type=int,
help='Playlist video to start at (default is %default)')
selection.add_option('--playlist-end',
dest='playlistend',
metavar='NUMBER',
default=None,
type=int,
help='Playlist video to end at (default is last)')
selection.add_option(
'--playlist-items',
dest='playlist_items',
metavar='ITEM_SPEC',
default=None,
help=
'Playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.'
)
selection.add_option(
'--match-title',
dest='matchtitle',
metavar='REGEX',
help='Download only matching titles (regex or caseless sub-string)')
selection.add_option(
'--reject-title',
dest='rejecttitle',
metavar='REGEX',
help='Skip download for matching titles (regex or caseless sub-string)'
)
selection.add_option('--max-downloads',
dest='max_downloads',
metavar='NUMBER',
type=int,
default=None,
help='Abort after downloading NUMBER files')
selection.add_option(
'--min-filesize',
metavar='SIZE',
dest='min_filesize',
default=None,
help='Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)'
)
selection.add_option(
'--max-filesize',
metavar='SIZE',
dest='max_filesize',
default=None,
help='Do not download any videos larger than SIZE (e.g. 50k or 44.6m)')
selection.add_option('--date',
metavar='DATE',
dest='date',
default=None,
help='Download only videos uploaded in this date')
selection.add_option(
'--datebefore',
metavar='DATE',
dest='datebefore',
default=None,
help=
'Download only videos uploaded on or before this date (i.e. inclusive)'
)
selection.add_option(
'--dateafter',
metavar='DATE',
dest='dateafter',
default=None,
help=
'Download only videos uploaded on or after this date (i.e. inclusive)')
selection.add_option(
'--min-views',
metavar='COUNT',
dest='min_views',
default=None,
type=int,
help='Do not download any videos with less than COUNT views')
selection.add_option(
'--max-views',
metavar='COUNT',
dest='max_views',
default=None,
type=int,
help='Do not download any videos with more than COUNT views')
selection.add_option(
'--match-filter',
metavar='FILTER',
dest='match_filter',
default=None,
help=(
'Generic video filter (experimental). '
'Specify any key (see help for -o for a list of available keys) to'
' match if the key is present, '
'!key to check if the key is not present,'
'key > NUMBER (like "comment_count > 12", also works with '
'>=, <, <=, !=, =) to compare against a number, and '
'& to require multiple matches. '
'Values which are not known are excluded unless you'
' put a question mark (?) after the operator.'
'For example, to only match videos that have been liked more than '
'100 times and disliked less than 50 times (or the dislike '
'functionality is not available at the given service), but who '
'also have a description, use --match-filter '
'"like_count > 100 & dislike_count <? 50 & description" .'))
selection.add_option(
'--no-playlist',
action='store_true',
dest='noplaylist',
default=False,
help=
'Download only the video, if the URL refers to a video and a playlist.'
)
selection.add_option(
'--yes-playlist',
action='store_false',
dest='noplaylist',
default=False,
help=
'Download the playlist, if the URL refers to a video and a playlist.')
selection.add_option(
'--age-limit',
metavar='YEARS',
dest='age_limit',
default=None,
type=int,
help='Download only videos suitable for the given age')
selection.add_option(
'--download-archive',
metavar='FILE',
dest='download_archive',
help=
'Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.'
)
selection.add_option('--include-ads',
dest='include_ads',
action='store_true',
help='Download advertisements as well (experimental)')
authentication = optparse.OptionGroup(parser, 'Authentication Options')
authentication.add_option('-u',
'--username',
dest='username',
metavar='USERNAME',
help='Login with this account ID')
authentication.add_option(
'-p',
'--password',
dest='password',
metavar='PASSWORD',
help=
'Account password. If this option is left out, youtube-dl will ask interactively.'
)
authentication.add_option('-2',
'--twofactor',
dest='twofactor',
metavar='TWOFACTOR',
help='Two-factor auth code')
authentication.add_option('-n',
'--netrc',
action='store_true',
dest='usenetrc',
default=False,
help='Use .netrc authentication data')
authentication.add_option('--video-password',
dest='videopassword',
metavar='PASSWORD',
help='Video password (vimeo, smotri)')
video_format = optparse.OptionGroup(parser, 'Video Format Options')
video_format.add_option(
'-f',
'--format',
action='store',
dest='format',
metavar='FORMAT',
default=None,
help='Video format code, see the "FORMAT SELECTION" for all the info')
video_format.add_option('--all-formats',
action='store_const',
dest='format',
const='all',
help='Download all available video formats')
video_format.add_option(
'--prefer-free-formats',
action='store_true',
dest='prefer_free_formats',
default=False,
help='Prefer free video formats unless a specific one is requested')
video_format.add_option('-F',
'--list-formats',
action='store_true',
dest='listformats',
help='List all available formats')
video_format.add_option('--youtube-include-dash-manifest',
action='store_true',
dest='youtube_include_dash_manifest',
default=True,
help=optparse.SUPPRESS_HELP)
video_format.add_option(
'--youtube-skip-dash-manifest',
action='store_false',
dest='youtube_include_dash_manifest',
help=
'Do not download the DASH manifests and related data on YouTube videos'
)
video_format.add_option(
'--merge-output-format',
action='store',
dest='merge_output_format',
metavar='FORMAT',
default=None,
help=
('If a merge is required (e.g. bestvideo+bestaudio), '
'output to given container format. One of mkv, mp4, ogg, webm, flv. '
'Ignored if no merge is required'))
subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
subtitles.add_option('--write-sub',
'--write-srt',
action='store_true',
dest='writesubtitles',
default=False,
help='Write subtitle file')
subtitles.add_option('--write-auto-sub',
'--write-automatic-sub',
action='store_true',
dest='writeautomaticsub',
default=False,
help='Write automatic subtitle file (YouTube only)')
subtitles.add_option(
'--all-subs',
action='store_true',
dest='allsubtitles',
default=False,
help='Download all the available subtitles of the video')
subtitles.add_option('--list-subs',
action='store_true',
dest='listsubtitles',
default=False,
help='List all available subtitles for the video')
subtitles.add_option(
'--sub-format',
action='store',
dest='subtitlesformat',
metavar='FORMAT',
default='best',
help=
'Subtitle format, accepts formats preference, for example: "srt" or "ass/srt/best"'
)
subtitles.add_option(
'--sub-lang',
'--sub-langs',
'--srt-lang',
action='callback',
dest='subtitleslangs',
metavar='LANGS',
type='str',
default=[],
callback=_comma_separated_values_options_callback,
help=
'Languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\''
)
downloader = optparse.OptionGroup(parser, 'Download Options')
downloader.add_option(
'-r',
'--rate-limit',
dest='ratelimit',
metavar='LIMIT',
help='Maximum download rate in bytes per second (e.g. 50K or 4.2M)')
downloader.add_option(
'-R',
'--retries',
dest='retries',
metavar='RETRIES',
default=10,
help='Number of retries (default is %default), or "infinite".')
downloader.add_option(
'--buffer-size',
dest='buffersize',
metavar='SIZE',
default='1024',
help='Size of download buffer (e.g. 1024 or 16K) (default is %default)'
)
downloader.add_option(
'--no-resize-buffer',
action='store_true',
dest='noresizebuffer',
default=False,
help=
'Do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.'
)
downloader.add_option('--test',
action='store_true',
dest='test',
default=False,
help=optparse.SUPPRESS_HELP)
downloader.add_option('--playlist-reverse',
action='store_true',
help='Download playlist videos in reverse order')
downloader.add_option(
'--xattr-set-filesize',
dest='xattr_set_filesize',
action='store_true',
help=
'Set file xattribute ytdl.filesize with expected filesize (experimental)'
)
downloader.add_option(
'--hls-prefer-native',
dest='hls_prefer_native',
action='store_true',
help='Use the native HLS downloader instead of ffmpeg (experimental)')
downloader.add_option('--external-downloader',
dest='external_downloader',
metavar='COMMAND',
help='Use the specified external downloader. '
'Currently supports %s' %
','.join(list_external_downloaders()))
downloader.add_option(
'--external-downloader-args',
dest='external_downloader_args',
metavar='ARGS',
help='Give these arguments to the external downloader')
workarounds = optparse.OptionGroup(parser, 'Workarounds')
workarounds.add_option('--encoding',
dest='encoding',
metavar='ENCODING',
help='Force the specified encoding (experimental)')
workarounds.add_option('--no-check-certificate',
action='store_true',
dest='no_check_certificate',
default=False,
help='Suppress HTTPS certificate validation')
workarounds.add_option(
'--prefer-insecure',
'--prefer-unsecure',
action='store_true',
dest='prefer_insecure',
help=
'Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)'
)
workarounds.add_option('--user-agent',
metavar='UA',
dest='user_agent',
help='Specify a custom user agent')
workarounds.add_option(
'--referer',
metavar='URL',
dest='referer',
default=None,
help=
'Specify a custom referer, use if the video access is restricted to one domain',
)
workarounds.add_option(
'--add-header',
metavar='FIELD:VALUE',
dest='headers',
action='append',
help=
'Specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
)
workarounds.add_option(
'--bidi-workaround',
dest='bidi_workaround',
action='store_true',
help=
'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH'
)
workarounds.add_option(
'--sleep-interval',
metavar='SECONDS',
dest='sleep_interval',
type=float,
help='Number of seconds to sleep before each download.')
verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
verbosity.add_option('-q',
'--quiet',
action='store_true',
dest='quiet',
default=False,
help='Activate quiet mode')
verbosity.add_option('--no-warnings',
dest='no_warnings',
action='store_true',
default=False,
help='Ignore warnings')
verbosity.add_option(
'-s',
'--simulate',
action='store_true',
dest='simulate',
default=False,
help='Do not download the video and do not write anything to disk')
verbosity.add_option('--skip-download',
action='store_true',
dest='skip_download',
default=False,
help='Do not download the video')
verbosity.add_option('-g',
'--get-url',
action='store_true',
dest='geturl',
default=False,
help='Simulate, quiet but print URL')
verbosity.add_option('-e',
'--get-title',
action='store_true',
dest='gettitle',
default=False,
help='Simulate, quiet but print title')
verbosity.add_option('--get-id',
action='store_true',
dest='getid',
default=False,
help='Simulate, quiet but print id')
verbosity.add_option('--get-thumbnail',
action='store_true',
dest='getthumbnail',
default=False,
help='Simulate, quiet but print thumbnail URL')
verbosity.add_option('--get-description',
action='store_true',
dest='getdescription',
default=False,
help='Simulate, quiet but print video description')
verbosity.add_option('--get-duration',
action='store_true',
dest='getduration',
default=False,
help='Simulate, quiet but print video length')
verbosity.add_option('--get-filename',
action='store_true',
dest='getfilename',
default=False,
help='Simulate, quiet but print output filename')
verbosity.add_option('--get-format',
action='store_true',
dest='getformat',
default=False,
help='Simulate, quiet but print output format')
verbosity.add_option(
'-j',
'--dump-json',
action='store_true',
dest='dumpjson',
default=False,
help=
'Simulate, quiet but print JSON information. See --output for a description of available keys.'
)
verbosity.add_option(
'-J',
'--dump-single-json',
action='store_true',
dest='dump_single_json',
default=False,
help=
'Simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist information in a single line.'
)
verbosity.add_option(
'--print-json',
action='store_true',
dest='print_json',
default=False,
help=
'Be quiet and print the video information as JSON (video is still being downloaded).',
)
verbosity.add_option('--newline',
action='store_true',
dest='progress_with_newline',
default=False,
help='Output progress bar as new lines')
verbosity.add_option('--no-progress',
action='store_true',
dest='noprogress',
default=False,
help='Do not print progress bar')
verbosity.add_option('--console-title',
action='store_true',
dest='consoletitle',
default=False,
help='Display progress in console titlebar')
verbosity.add_option('-v',
'--verbose',
action='store_true',
dest='verbose',
default=False,
help='Print various debugging information')
verbosity.add_option(
'--dump-pages',
'--dump-intermediate-pages',
action='store_true',
dest='dump_intermediate_pages',
default=False,
help=
'Print downloaded pages encoded using base64 to debug problems (very verbose)'
)
verbosity.add_option(
'--write-pages',
action='store_true',
dest='write_pages',
default=False,
help=
'Write downloaded intermediary pages to files in the current directory to debug problems'
)
verbosity.add_option('--youtube-print-sig-code',
action='store_true',
dest='youtube_print_sig_code',
default=False,
help=optparse.SUPPRESS_HELP)
verbosity.add_option('--print-traffic',
'--dump-headers',
dest='debug_printtraffic',
action='store_true',
default=False,
help='Display sent and read HTTP traffic')
verbosity.add_option('-C',
'--call-home',
dest='call_home',
action='store_true',
default=False,
help='Contact the youtube-dl server for debugging')
verbosity.add_option(
'--no-call-home',
dest='call_home',
action='store_false',
default=False,
help='Do NOT contact the youtube-dl server for debugging')
filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
filesystem.add_option(
'-a',
'--batch-file',
dest='batchfile',
metavar='FILE',
help='File containing URLs to download (\'-\' for stdin)')
filesystem.add_option('--id',
default=False,
action='store_true',
dest='useid',
help='Use only video ID in file name')
filesystem.add_option(
'-o',
'--output',
dest='outtmpl',
metavar='TEMPLATE',
help=
('Output filename template. Use %(title)s to get the title, '
'%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
'%(autonumber)s to get an automatically incremented number, '
'%(ext)s for the filename extension, '
'%(format)s for the format description (like "22 - 1280x720" or "HD"), '
'%(format_id)s for the unique id of the format (like YouTube\'s itags: "137"), '
'%(upload_date)s for the upload date (YYYYMMDD), '
'%(extractor)s for the provider (youtube, metacafe, etc), '
'%(id)s for the video id, '
'%(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in, '
'%(playlist_index)s for the position in the playlist. '
'%(height)s and %(width)s for the width and height of the video format. '
'%(resolution)s for a textual description of the resolution of the video format. '
'%% for a literal percent. '
'Use - to output to stdout. Can also be used to download to a different directory, '
'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'
))
filesystem.add_option(
'--autonumber-size',
dest='autonumber_size',
metavar='NUMBER',
help=
'Specify the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given'
)
filesystem.add_option(
'--restrict-filenames',
action='store_true',
dest='restrictfilenames',
default=False,
help=
'Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames'
)
filesystem.add_option(
'-A',
'--auto-number',
action='store_true',
dest='autonumber',
default=False,
help=
'[deprecated; use -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000'
)
filesystem.add_option('-t',
'--title',
action='store_true',
dest='usetitle',
default=False,
help='[deprecated] Use title in file name (default)')
filesystem.add_option('-l',
'--literal',
default=False,
action='store_true',
dest='usetitle',
help='[deprecated] Alias of --title')
filesystem.add_option('-w',
'--no-overwrites',
action='store_true',
dest='nooverwrites',
default=False,
help='Do not overwrite files')
filesystem.add_option(
'-c',
'--continue',
action='store_true',
dest='continue_dl',
default=True,
help=
'Force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.'
)
filesystem.add_option(
'--no-continue',
action='store_false',
dest='continue_dl',
help='Do not resume partially downloaded files (restart from beginning)'
)
filesystem.add_option(
'--no-part',
action='store_true',
dest='nopart',
default=False,
help='Do not use .part files - write directly into output file')
filesystem.add_option(
'--no-mtime',
action='store_false',
dest='updatetime',
default=True,
help=
'Do not use the Last-modified header to set the file modification time'
)
filesystem.add_option(
'--write-description',
action='store_true',
dest='writedescription',
default=False,
help='Write video description to a .description file')
filesystem.add_option('--write-info-json',
action='store_true',
dest='writeinfojson',
default=False,
help='Write video metadata to a .info.json file')
filesystem.add_option(
'--write-annotations',
action='store_true',
dest='writeannotations',
default=False,
help='Write video annotations to a .annotations.xml file')
filesystem.add_option(
'--load-info',
dest='load_info_filename',
metavar='FILE',
help=
'JSON file containing the video information (created with the "--write-info-json" option)'
)
filesystem.add_option(
'--cookies',
dest='cookiefile',
metavar='FILE',
help='File to read cookies from and dump cookie jar in')
filesystem.add_option(
'--cache-dir',
dest='cachedir',
default=None,
metavar='DIR',
help=
'Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.'
)
filesystem.add_option('--no-cache-dir',
action='store_const',
const=False,
dest='cachedir',
help='Disable filesystem caching')
filesystem.add_option('--rm-cache-dir',
action='store_true',
dest='rm_cachedir',
help='Delete all filesystem cache files')
thumbnail = optparse.OptionGroup(parser, 'Thumbnail images')
thumbnail.add_option('--write-thumbnail',
action='store_true',
dest='writethumbnail',
default=False,
help='Write thumbnail image to disk')
thumbnail.add_option('--write-all-thumbnails',
action='store_true',
dest='write_all_thumbnails',
default=False,
help='Write all thumbnail image formats to disk')
thumbnail.add_option(
'--list-thumbnails',
action='store_true',
dest='list_thumbnails',
default=False,
help='Simulate and list all available thumbnail formats')
postproc = optparse.OptionGroup(parser, 'Post-processing Options')
postproc.add_option(
'-x',
'--extract-audio',
action='store_true',
dest='extractaudio',
default=False,
help=
'Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)'
)
postproc.add_option(
'--audio-format',
metavar='FORMAT',
dest='audioformat',
default='best',
help=
'Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "%default" by default'
)
postproc.add_option(
'--audio-quality',
metavar='QUALITY',
dest='audioquality',
default='5',
help=
'Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default %default)'
)
postproc.add_option(
'--recode-video',
metavar='FORMAT',
dest='recodevideo',
default=None,
help=
'Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)'
)
postproc.add_option(
'-k',
'--keep-video',
action='store_true',
dest='keepvideo',
default=False,
help=
'Keep the video file on disk after the post-processing; the video is erased by default'
)
postproc.add_option(
'--no-post-overwrites',
action='store_true',
dest='nopostoverwrites',
default=False,
help=
'Do not overwrite post-processed files; the post-processed files are overwritten by default'
)
postproc.add_option(
'--embed-subs',
action='store_true',
dest='embedsubtitles',
default=False,
help='Embed subtitles in the video (only for mkv and mp4 videos)')
postproc.add_option('--embed-thumbnail',
action='store_true',
dest='embedthumbnail',
default=False,
help='Embed thumbnail in the audio as cover art')
postproc.add_option('--add-metadata',
action='store_true',
dest='addmetadata',
default=False,
help='Write metadata to the video file')
postproc.add_option(
'--metadata-from-title',
metavar='FORMAT',
dest='metafromtitle',
help=
'Parse additional metadata like song title / artist from the video title. '
'The format syntax is the same as --output, '
'the parsed parameters replace existing values. '
'Additional templates: %(album)s, %(artist)s. '
'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like '
'"Coldplay - Paradise"')
postproc.add_option(
'--xattrs',
action='store_true',
dest='xattrs',
default=False,
help=
'Write metadata to the video file\'s xattrs (using dublin core and xdg standards)'
)
postproc.add_option(
'--fixup',
metavar='POLICY',
dest='fixup',
default='detect_or_warn',
help='Automatically correct known faults of the file. '
'One of never (do nothing), warn (only emit a warning), '
'detect_or_warn (the default; fix file if we can, warn otherwise)')
postproc.add_option(
'--prefer-avconv',
action='store_false',
dest='prefer_ffmpeg',
help=
'Prefer avconv over ffmpeg for running the postprocessors (default)')
postproc.add_option(
'--prefer-ffmpeg',
action='store_true',
dest='prefer_ffmpeg',
help='Prefer ffmpeg over avconv for running the postprocessors')
postproc.add_option(
'--ffmpeg-location',
'--avconv-location',
metavar='PATH',
dest='ffmpeg_location',
help=
'Location of the ffmpeg/avconv binary; either the path to the binary or its containing directory.'
)
postproc.add_option(
'--exec',
metavar='CMD',
dest='exec_cmd',
help=
'Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\''
)
postproc.add_option(
'--convert-subtitles',
'--convert-subs',
metavar='FORMAT',
dest='convertsubtitles',
default=None,
help=
'Convert the subtitles to other format (currently supported: srt|ass|vtt)'
)
parser.add_option_group(general)
parser.add_option_group(network)
parser.add_option_group(selection)
parser.add_option_group(downloader)
parser.add_option_group(filesystem)
parser.add_option_group(thumbnail)
parser.add_option_group(verbosity)
parser.add_option_group(workarounds)
parser.add_option_group(video_format)
parser.add_option_group(subtitles)
parser.add_option_group(authentication)
parser.add_option_group(postproc)
if overrideArguments is not None:
opts, args = parser.parse_args(overrideArguments)
if opts.verbose:
write_string('[debug] Override config: ' +
repr(overrideArguments) + '\n')
else:
def compat_conf(conf):
if sys.version_info < (3, ):
return [a.decode(preferredencoding(), 'replace') for a in conf]
return conf
command_line_conf = compat_conf(sys.argv[1:])
if '--ignore-config' in command_line_conf:
system_conf = []
user_conf = []
else:
system_conf = compat_conf(_readOptions('/etc/youtube-dl.conf'))
if '--ignore-config' in system_conf:
user_conf = []
else:
user_conf = compat_conf(_readUserConf())
argv = system_conf + user_conf + command_line_conf
opts, args = parser.parse_args(argv)
if opts.verbose:
write_string('[debug] System config: ' +
repr(_hide_login_info(system_conf)) + '\n')
write_string('[debug] User config: ' +
repr(_hide_login_info(user_conf)) + '\n')
write_string('[debug] Command-line args: ' +
repr(_hide_login_info(command_line_conf)) + '\n')
return parser, opts, args
def main():
usage = """scholar.py [options] <query string>
A command-line interface to Google Scholar.
Examples:
# Retrieve one article written by Einstein on quantum theory:
scholar.py -c 1 --author "albert einstein" --phrase "quantum theory"
# Retrieve a BibTeX entry for that quantum theory paper:
scholar.py -c 1 -C 17749203648027613321 --citation bt
# Retrieve five articles written by Einstein after 1970 where the title
# does not contain the words "quantum" and "theory":
scholar.py -c 5 -a "albert einstein" -t --none "quantum theory" --after 1970"""
fmt = optparse.IndentedHelpFormatter(max_help_position=50, width=100)
parser = optparse.OptionParser(usage=usage, formatter=fmt)
group = optparse.OptionGroup(parser, 'Query arguments',
'These options define search query arguments and parameters.')
group.add_option('-a', '--author', metavar='AUTHORS', default=None,
help='Author name(s)')
group.add_option('-A', '--all', metavar='WORDS', default=None, dest='allw',
help='Results must contain all of these words')
group.add_option('-s', '--some', metavar='WORDS', default=None,
help='Results must contain at least one of these words')
group.add_option('-n', '--none', metavar='WORDS', default=None,
help='Results must contain none of these words')
group.add_option('-p', '--phrase', metavar='PHRASE', default=None,
help='Results must contain exact phrase')
group.add_option('-t', '--title-only', action='store_true', default=False,
help='Search title only')
group.add_option('-P', '--pub', metavar='PUBLICATIONS', default=None,
help='Results must have appeared in this publication')
group.add_option('--after', metavar='YEAR', default=None,
help='Results must have appeared in or after given year')
group.add_option('--before', metavar='YEAR', default=None,
help='Results must have appeared in or before given year')
group.add_option('-C', '--cluster-id', metavar='CLUSTER_ID', default=None,
help='Do not search, just use articles in given cluster ID')
group.add_option('-c', '--count', type='int', default=None,
help='Maximum number of results')
group.add_option('-S', '--start', type='int', default=None,
help='Start of results')
group.add_option('-N', '--pages', type='int', default=None,
help='Number of pages of results')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Output format',
'These options control the appearance of the results.')
group.add_option('--txt', action='store_true',
help='Print article data in text format (default)')
group.add_option('--csv', action='store_true',
help='Print article data in CSV form (separator is "|")')
group.add_option('--csv-header', action='store_true',
help='Like --csv, but print header with column names')
group.add_option('--citation', metavar='FORMAT', default=None,
help='Print article details in standard citation format. Argument Must be one of "bt" (BibTeX), "en" (EndNote), "rm" (RefMan), or "rw" (RefWorks).')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Miscellaneous')
group.add_option('--cookie-file', metavar='FILE', default=None,
help='File to use for cookie storage. If given, will read any existing cookies if found at startup, and save resulting cookies in the end.')
group.add_option('-d', '--debug', action='count', default=0,
help='Enable verbose logging to stderr. Repeated options increase detail of debug output.')
group.add_option('-v', '--version', action='store_true', default=False,
help='Show version information')
parser.add_option_group(group)
options, _ = parser.parse_args()
# Show help if we have neither keyword search nor author name
if len(sys.argv) == 1:
parser.print_help()
return 1
if options.debug > 0:
options.debug = min(options.debug, ScholarUtils.LOG_LEVELS['debug'])
ScholarConf.LOG_LEVEL = options.debug
ScholarUtils.log('info', 'using log level %d' % ScholarConf.LOG_LEVEL)
if options.version:
print 'This is scholar.py %s.' % ScholarConf.VERSION
return 0
if options.cookie_file:
ScholarConf.COOKIE_JAR_FILE = options.cookie_file
# Sanity-check the options: if they include a cluster ID query, it
# makes no sense to have search arguments:
if options.cluster_id is not None:
if options.author or options.allw or options.some or options.none \
or options.phrase or options.title_only or options.pub \
or options.after or options.before:
print 'Cluster ID queries do not allow additional search arguments.'
return 1
querier = ScholarQuerier()
settings = ScholarSettings()
if options.citation == 'bt':
settings.set_citation_format(ScholarSettings.CITFORM_BIBTEX)
elif options.citation == 'en':
settings.set_citation_format(ScholarSettings.CITFORM_ENDNOTE)
elif options.citation == 'rm':
settings.set_citation_format(ScholarSettings.CITFORM_REFMAN)
elif options.citation == 'rw':
settings.set_citation_format(ScholarSettings.CITFORM_REFWORKS)
elif options.citation is not None:
print 'Invalid citation link format, must be one of "bt", "en", "rm", or "rw".'
return 1
querier.apply_settings(settings)
if options.cluster_id:
query = ClusterScholarQuery(cluster=options.cluster_id)
else:
query = SearchScholarQuery()
if options.author:
query.set_author(options.author)
if options.allw:
query.set_words(options.allw)
if options.some:
query.set_words_some(options.some)
if options.none:
query.set_words_none(options.none)
if options.phrase:
query.set_phrase(options.phrase)
if options.title_only:
query.set_scope(True)
if options.pub:
query.set_pub(options.pub)
if options.after or options.before:
query.set_timeframe(options.after, options.before)
if options.count is not None:
options.count = min(options.count, ScholarConf.MAX_PAGE_RESULTS)
query.set_num_page_results(options.count)
else:
options.count = settings.per_page_results
if options.start is not None:
options.start = max(options.start, 0)
query.set_start_results(options.start)
else:
options.start = 0
if options.pages is None:
options.pages = 1
for i in range(options.pages):
querier.send_query(query)
options.start += options.count
query.set_start_results(options.start)
if options.csv:
csv(querier)
elif options.csv_header:
csv(querier, header=(i == 0))
elif options.citation is not None:
citation_export(querier)
else:
txt(querier)
if options.cookie_file:
querier.save_cookies()
return 0
def _parse_args():
"""
Parse the command line for options
"""
usage = """AlignmentFilter.py -x [bowtie2index] -f [FqFile] -s <390> -F <20> -T [TargetPoolFile] -o <output>
"""
fmt = optparse.IndentedHelpFormatter(max_help_position=50, width=100)
parser = optparse.OptionParser(usage=usage, formatter=fmt)
group = optparse.OptionGroup(
parser, 'Query arguments',
'These options define search query arguments and parameters. '
'bowtie2 command has been the optimal params.')
group.add_option('-f',
'--file',
action='store',
dest='file',
type='str',
help='The fastq file to be processed')
group.add_option('-s',
'--salt',
action='store',
default=390,
type=int,
dest='salt',
help='The mM Na+ concentration to be used for Tm '
'calculation, default is 390')
group.add_option(
'-T',
'--targets',
action='store',
dest='targets',
help='Excluding these reads were mapped on non-target genes')
group.add_option('-F',
'--formamide',
action='store',
default=50,
type=float,
dest='formamide',
help='The percent formamide to be used for Tm '
'calculation, default is 50')
group.add_option('-o',
'--output',
action='store',
default=None,
type=str,
dest='output',
help='Specify the stem of the output filename')
group.add_option('-x',
'--index',
action='store',
default=None,
type=str,
dest='index',
help='The genome index dirs which generated by bowtie2')
group.add_option(
'-v',
'--verbose',
action='store_true',
default=False,
dest='verbose',
help=
'if True, the bowtie2 alignment results will stroe in the tmp.sam file.default:False'
)
group.add_option('-p',
'--probelength',
action='store',
default=70,
type=int,
dest='probelength',
help='probelength.')
group.add_option('-h',
'--hp',
action='store',
default=47,
type=float,
dest='hytemp',
help="hybridize temp")
parser.add_option_group(group)
options, args = parser.parse_args()
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
return options
def main(argv):
loadConfigFile()
#Command Line arguments and options
#
usage = "usage: prog command [options]\n\n"
parser = optparse.OptionParser(usage=usage, version="%prog 1.2")
group1 = optparse.OptionGroup(parser, "Options for Accounts and Media")
group1.add_option("--query", action="store", dest="query", default=_CFG['query'],
help="Provide a query to limit the search (ex id:2444)")
group1.add_option("--accountid", action="store", dest="acctId",default=_CFG['accountId'],
help="Process a single account if accountID is provided.")
group1.add_option("--mediaid", action="store", dest="mediaId", default="",
help="Display all info on a specific media item")
#######################
group2 = optparse.OptionGroup(parser, "Update Commands - Require permissions",
"These options update the media items")
group2.add_option("--delete-masters", action="store_true", dest="deletemaster", default=False,
help="Delete the digital masters")
group2.add_option("--high-as-master", action="store_true", dest="highasmaster", default=False,
help="Set the highest MP4 rendition as the new master")
group2.add_option("--reingest-from-highest", action="store_true", dest="reingesthighest", default=False,
help="Re-ingest content using highest rendition.")
#######################
group3 = optparse.OptionGroup(parser, "Level of Information to return",
"These options define what information is returned")
group3.add_option("--countonly", action="store_true", dest="countonly", default=False,
help="Just return the number of items in each account")
group3.add_option("--itemsonly", action="store_true", dest="itemsonly", default=False,
help="Process items only (no masters or renditions")
#######################
group4 = optparse.OptionGroup(parser, "Default Overrides",
"Override defaults in configuration file")
group4.add_option("--outputdir", action="store", dest="outputdir", default=_CFG['outputdir'],
help="Output Directory (Default=./)")
group4.add_option("--keyname", action="store", dest="keyname", default=_CFG['keyname'],
help="Key Name in config file to use")
group4.add_option("--ingest-profile", action="store", dest="ingestprofile", default=_CFG['ingestprofile'],
help="Ingest profile to use for ingestion/re-ingestion")
group4.add_option("--keyfile", action="store", dest="keyfile", default=_CFG['keyfile'],
help="Special case. Ask Dave")
group4.add_option("--itemcsv", action="store", dest="itemcsv", default="",
help="Added for Josef. Path to a file with MediaIDs to process. (just skips items that arent in file)")
#######################
group5 = optparse.OptionGroup(parser, "Debugging and screen output")
group5.add_option("--verbose", action="store_true", dest="verbose", default=_CFG['verbose'],
help="Display more info on screen")
group5.add_option("--debug", action="store", dest="debug", type="int",default=_CFG['debug'],
help="Enable debugging (1=basic or 2=json 3=print keys)")
group5.add_option("--passitems", action="store", dest="passitems", default=_CFG['passitems'], type="int",
help="Number of items to process per query call (max is 100, default is 50)")
group5.add_option("--limit", action="store", dest="limit",default=_CFG['limit'],type="int",
help="Limit the number of media items to this count")
group5.add_option("--skipto", action="store", dest="skiptoid",default=_CFG['skiptoid'],
help="Skip until this ID then start processing)")
group5.add_option("--testme", action="store_true", dest="testme", default="",
help="Test Code.")
#######################
group6 = optparse.OptionGroup(parser, "Ingest and Content Item Related ")
group6.add_option("--ingest-from-json", action="store", dest="ingestjsonfile", default="",
help="Ingest from a JSON Feed File (supports --limit)")
group6.add_option("--ingest-from-MRSS", action="store", dest="ingestmrssfile", default="",
help="Ingest from a MRSS Feed File (supports --limit)")
group6.add_option("--get-ingest-profiles", action="store_true", dest="getingestprofiles", default=False,
help="Get ingestion profile info for accountid")
group6.add_option("--copy-item-fields", action="store_true", dest="copyitemfields", default=False,
help="Copy the field to an alternate field (specified in the config file)")
group6.add_option("--delete-media-items", action="store_true", dest="deletemediaitems", default=False,
help="Delete Content Items for account (BE CAREFUL). uses --query")
parser.add_option_group(group1)
parser.add_option_group(group2)
parser.add_option_group(group3)
parser.add_option_group(group4)
parser.add_option_group(group5)
parser.add_option_group(group6)
(options, args) = parser.parse_args()
# Override config file with command line options
#
_CFG['keyname'] = options.keyname
_CFG['accountId'] = options.acctId
_CFG['debug'] = options.debug
_CFG['limit'] = options.limit
_CFG['skiptoid'] = options.skiptoid
_CFG['mediaId'] = options.mediaId
_CFG['query'] = options.query
_CFG['outputDir'] = options.outputdir
_CFG['itemsonly'] = options.itemsonly
_CFG['passitems'] = options.passitems
_CFG['keyfile'] = options.keyfile
_CFG['ingestprofile'] = options.ingestprofile
_CFG['verbose'] = options.verbose
_CFG['countonly'] = options.countonly
_CFG['deletemaster'] = options.deletemaster
_CFG['highasmaster'] = options.highasmaster
_CFG['reingesthighest'] = options.reingesthighest
_CFG['itemcsv'] = options.itemcsv
_CFG['handles'] = { 'real':0, 'shared':0, 'error':0, 'results':0 }
# _CFG['fieldStr'] = "{id},{name},{reference_id},{masterId},{masterRate},{masterSize},{duration},{created_at},{updated_at},{state},{rendSize},{rendCount},{rendBitrates},{HLSRendSize},{HLSRendCount},{HLSResolutions},{HLSBitrates},{rendLgEncRate},{rendLgRes},{rendSize},{rendLgSize},{rendLgUrl}"
# _CFG['resultStr'] = "{accountId},{totalCount},{itemcount},{shared},{skipped},{duration},{masterSize},{rendCount},{rendSize},{HLSRendCount},{HLSRendSize},{rendLgSize}"
_CFG['totalFields'] = { 'masterSize':0, 'masterCount':0, 'duration':0,'rendCount':0, 'rendSize':0, 'rendLgSize':0, 'HLSRendSize':0, 'HLSRendCount':0,
'skipped':0, 'shared':0, 'itemcount':0 , 'totalCount':0}
_CFG['itemFields'] = { 'masterSize':0,'duration':0,'masterId':0, 'masterRate':0, 'id':0, 'name':0,'created_at':0,'updated_at':0, 'state':0 }
_CFG['rendFields'] = { 'rendCount':0, 'rendSize':0, 'rendLgSize':0, 'rendBitrates':0, 'rendCodecs':0, 'rendLgId':0, 'rendLgEncRate':0, 'rendLgRes':0,'rendLgUrl':0,
'HLSRendCount':0, 'HLSRendSize':0, 'HLSResolutions':0,'HLSBitrates':0 }
# AuthToken (gets renewed every 4 minutes)
_CFG['token'] = ""
_CFG['tokenLastUpdated'] = 0
#if len(argv) == 1: parser.print_help(); sys.exit()
#if not _CFG['mediaId']: printCfg(_CFG)
printCfg(_CFG)
sys.stdout.flush()
# Change the config options based on passed in flags where necessary
#
if _CFG['mediaId']:
_CFG['level'] = { 'items': 1, 'masters': 1, 'renditions': 1, 'hlsrenditions': 1}
if options.countonly:
_CFG['level'] = { 'items': 0, 'masters': 0, 'renditions': 0, 'hlsrenditions': 0}
if options.itemsonly:
_CFG['level'] = { 'items': 1, 'masters': 0, 'renditions': 0, 'hlsrenditions': 0}
# need to fetch renditions to set high master.
if _CFG.get('highasmaster',0) or _CFG.get('reingest',0):
_CFG['level']['renditions'] = 1;
if options.testme:
testme()
sys.exit()
elif options.getingestprofiles:
getIngestionProfiles()
sys.exit()
elif options.ingestjsonfile:
ingestFromJsonFile(options.ingestjsonfile, _CFG['accountId'])
sys.exit()
elif options.ingestmrssfile:
ingestFromMRSSFile(options.ingestmrssfile, _CFG['accountId'])
sys.exit()
elif options.copyitemfields:
copyItemFields()
sys.exit()
elif options.deletemediaitems:
deleteMediaItems()
sys.exit()
else:
processAll()
sys.exit()
def main():
parser = optparse.OptionParser(description=__doc__)
control_options = optparse.OptionGroup(parser, "Control options")
shell_options = optparse.OptionGroup(parser, "Shell options")
copy_options = optparse.OptionGroup(parser, "Copy options")
parser.add_option("--userHost",
dest="user_host",
default=None,
help="User and remote host to execute commands on [REQUIRED]."
" Examples, '[email protected]' or '*****@*****.**'.")
parser.add_option("--operation",
dest="operation",
default="shell",
choices=_OPERATIONS,
help="Remote operation to perform, choose one of '{}',"
" defaults to '%default'.".format(", ".join(_OPERATIONS)))
control_options.add_option("--sshOptions",
dest="ssh_options",
default=None,
action="append",
help="SSH connection options."
" More than one option can be specified either"
" in one quoted string or by specifying"
" this option more than once. Example options:"
" '-i $HOME/.ssh/access.pem -o ConnectTimeout=10"
" -o ConnectionAttempts=10'")
control_options.add_option("--retries",
dest="retries",
type=int,
default=0,
help="Number of retries to attempt for operation,"
" defaults to '%default'.")
control_options.add_option("--retrySleep",
dest="retry_sleep",
type=int,
default=10,
help="Number of seconds to wait between retries,"
" defaults to '%default'.")
control_options.add_option("--debug",
dest="debug",
action="store_true",
default=False,
help="Provides debug output.")
control_options.add_option("--verbose",
dest="verbose",
action="store_true",
default=False,
help="Print exit status and output at end.")
shell_options.add_option("--commands",
dest="remote_commands",
default=None,
action="append",
help="Commands to excute on the remote host. The"
" commands must be separated by a ';' and can either"
" be specifed in a quoted string or by specifying"
" this option more than once. A ';' will be added"
" between commands when this option is specifed"
" more than once.")
shell_options.add_option("--commandDir",
dest="command_dir",
default=None,
help="Working directory on remote to execute commands"
" form. Defaults to remote login directory.")
copy_options.add_option("--file",
dest="files",
default=None,
action="append",
help="The file to copy to/from remote host. To"
" support spaces in the file, each file must be"
" specified using this option more than once.")
copy_options.add_option("--remoteDir",
dest="remote_dir",
default=None,
help="Remote directory to copy to, only applies when"
" operation is 'copy_to'. Defaults to the login"
" directory on the remote host.")
copy_options.add_option("--localDir",
dest="local_dir",
default=".",
help="Local directory to copy to, only applies when"
" operation is 'copy_from'. Defaults to the"
" current directory, '%default'.")
parser.add_option_group(control_options)
parser.add_option_group(shell_options)
parser.add_option_group(copy_options)
(options, args) = parser.parse_args()
if not getattr(options, "user_host", None):
parser.print_help()
parser.error("Missing required option")
if options.operation == "shell":
if not getattr(options, "remote_commands", None):
parser.print_help()
parser.error("Missing required '{}' option '{}'".format(
options.operation, "--commands"))
operation_param = ";".join(options.remote_commands)
operation_dir = options.command_dir
else:
if not getattr(options, "files", None):
parser.print_help()
parser.error("Missing required '{}' option '{}'".format(
options.operation, "--file"))
operation_param = options.files
if options.operation == "copy_to":
operation_dir = options.remote_dir
else:
operation_dir = options.local_dir
ssh_options = None if not options.ssh_options else " ".join(options.ssh_options)
remote_op = RemoteOperations(
user_host=options.user_host,
ssh_options=ssh_options,
retries=options.retries,
retry_sleep=options.retry_sleep,
debug=options.debug)
ret_code, buffer = remote_op.operation(options.operation, operation_param, operation_dir)
if options.verbose:
print("Return code: {} for command {}".format(ret_code, sys.argv))
print(buffer)
sys.exit(ret_code)
def main(argv):
parser = optparse.OptionParser()
parser.add_option('-v',
'--version',
action='store_true',
dest='version',
default=False,
help='The version of this python tool.')
info = (
'The packaging mode of the web application. The value \'shared\' '
'means that the runtime is shared across multiple application '
'instances and that the runtime needs to be distributed separately. '
'The value \'embedded\' means that the runtime is embedded into the '
'application itself and distributed along with it.'
'Set the default mode as \'embedded\'. For example: --mode=embedded')
parser.add_option('--mode', default='embedded', help=info)
info = (
'The target architecture of the embedded runtime. Supported values '
'are \'x86\' and \'arm\'. Note, if undefined, APKs for all possible '
'architestures will be generated.')
parser.add_option('--arch', help=info)
group = optparse.OptionGroup(
parser, 'Application Source Options',
'This packaging tool supports 3 kinds of web application source: '
'1) XPK package; 2) manifest.json; 3) various command line options, '
'for example, \'--app-url\' for website, \'--app-root\' and '
'\'--app-local-path\' for local web application.')
info = (
'The path of the XPK package. For example, --xpk=/path/to/xpk/file')
group.add_option('--xpk', help=info)
info = (
'The manifest file with the detail description of the application. '
'For example, --manifest=/path/to/your/manifest/file')
group.add_option('--manifest', help=info)
info = ('The url of application. '
'This flag allows to package website as apk. For example, '
'--app-url=http://www.intel.com')
group.add_option('--app-url', help=info)
info = ('The root path of the web app. '
'This flag allows to package local web app as apk. For example, '
'--app-root=/root/path/of/the/web/app')
group.add_option('--app-root', help=info)
info = (
'The relative path of entry file based on the value from '
'\'app_root\'. This flag should work with \'--app-root\' together. '
'For example, --app-local-path=/relative/path/of/entry/file')
group.add_option('--app-local-path', help=info)
parser.add_option_group(group)
group = optparse.OptionGroup(
parser, 'Mandatory arguments',
'They are used for describing the APK information through '
'command line options.')
info = ('The apk name. For example, --name=YourApplicationName')
group.add_option('--name', help=info)
info = ('The package name. For example, '
'--package=com.example.YourPackage')
group.add_option('--package', help=info)
parser.add_option_group(group)
group = optparse.OptionGroup(
parser, 'Optional arguments',
'They are used for various settings for applications through '
'command line options.')
info = ('The version name of the application. '
'For example, --app-version=1.0.0')
group.add_option('--app-version', help=info)
info = ('The description of the application. For example, '
'--description=YourApplicationDescription')
group.add_option('--description', help=info)
group.add_option('--enable-remote-debugging',
action='store_true',
dest='enable_remote_debugging',
default=False,
help='Enable remote debugging.')
info = ('The list of external extension paths splitted by OS separators. '
'The separators are \':\' , \';\' and \':\' on Linux, Windows and '
'Mac OS respectively. For example, '
'--extensions=/path/to/extension1:/path/to/extension2.')
group.add_option('--extensions', help=info)
group.add_option('-f',
'--fullscreen',
action='store_true',
dest='fullscreen',
default=False,
help='Make application fullscreen.')
info = ('The path of application icon. '
'Such as: --icon=/path/to/your/customized/icon')
group.add_option('--icon', help=info)
info = ('The orientation of the web app\'s display on the device. '
'For example, --orientation=landscape. The default value is '
'\'unspecified\'. The permitted values are from Android: '
'http://developer.android.com/guide/topics/manifest/'
'activity-element.html#screen')
group.add_option('--orientation', help=info)
info = (
'The list of permissions to be used by web application. For example, '
'--permissions=geolocation:webgl')
group.add_option('--permissions', help=info)
parser.add_option_group(group)
group = optparse.OptionGroup(
parser, 'Keystore Options',
'The keystore is a signature from web developer, it\'s used when '
'developer wants to distribute the applications.')
info = ('The path to the developer keystore. For example, '
'--keystore-path=/path/to/your/developer/keystore')
group.add_option('--keystore-path', help=info)
info = ('The alias name of keystore. For example, --keystore-alias=name')
group.add_option('--keystore-alias', help=info)
info = ('The passcode of keystore. For example, --keystore-passcode=code')
group.add_option('--keystore-passcode', help=info)
parser.add_option_group(group)
options, _ = parser.parse_args()
if len(argv) == 1:
parser.print_help()
return 0
if options.version:
if os.path.isfile('VERSION'):
print GetVersion('VERSION')
return 0
else:
parser.error('Can\'t get version due to the VERSION file missing!')
xpk_temp_dir = ''
if options.xpk:
xpk_name = os.path.splitext(os.path.basename(options.xpk))[0]
xpk_temp_dir = xpk_name + '_xpk'
ParseXPK(options, xpk_temp_dir)
if options.app_root and not options.manifest:
manifest_path = os.path.join(options.app_root, 'manifest.json')
if os.path.exists(manifest_path):
print('Using manifest.json distributed with the application.')
options.manifest = manifest_path
if not options.manifest:
if not options.package:
parser.error('The package name is required! '
'Please use "--package" option.')
if not options.name:
parser.error(
'The APK name is required! Pleaes use "--name" option.')
if not ((options.app_url and not options.app_root
and not options.app_local_path) or
((not options.app_url) and options.app_root
and options.app_local_path)):
parser.error(
'The entry is required. If the entry is a remote url, '
'please use "--app-url" option; If the entry is local, '
'please use "--app-root" and '
'"--app-local-path" options together!')
if options.permissions:
permission_list = options.permissions.split(':')
else:
print(
'Warning: all supported permissions on Android port are added. '
'Refer to https://github.com/crosswalk-project/'
'crosswalk-website/wiki/Crosswalk-manifest')
permission_list = permission_mapping_table.keys()
options.permissions = HandlePermissionList(permission_list)
else:
try:
ParseManifest(options)
except SystemExit, ec:
return ec.code
def base_parser(usage="",
output_opts=False,
runas_opts=False,
meta_opts=False,
runtask_opts=False,
vault_opts=False,
module_opts=False,
async_opts=False,
connect_opts=False,
subset_opts=False,
check_opts=False,
inventory_opts=False,
epilog=None,
fork_opts=False,
runas_prompt_opts=False):
''' create an options parser for most ansible scripts '''
# TODO: implement epilog parsing
# OptionParser.format_epilog = lambda self, formatter: self.epilog
# base opts
parser = SortedOptParser(usage, version=CLI.version("%prog"))
parser.add_option(
'-v',
'--verbose',
dest='verbosity',
default=0,
action="count",
help=
"verbose mode (-vvv for more, -vvvv to enable connection debugging)"
)
if inventory_opts:
parser.add_option(
'-i',
'--inventory-file',
dest='inventory',
help=
"specify inventory host path (default=%s) or comma separated host list."
% C.DEFAULT_HOST_LIST,
default=C.DEFAULT_HOST_LIST,
action="callback",
callback=CLI.expand_tilde,
type=str)
parser.add_option(
'--list-hosts',
dest='listhosts',
action='store_true',
help=
'outputs a list of matching hosts; does not execute anything else'
)
parser.add_option(
'-l',
'--limit',
default=C.DEFAULT_SUBSET,
dest='subset',
help='further limit selected hosts to an additional pattern')
if module_opts:
parser.add_option(
'-M',
'--module-path',
dest='module_path',
default=None,
help="specify path(s) to module library (default=%s)" %
C.DEFAULT_MODULE_PATH,
action="callback",
callback=CLI.expand_tilde,
type=str)
if runtask_opts:
parser.add_option(
'-e',
'--extra-vars',
dest="extra_vars",
action="append",
help="set additional variables as key=value or YAML/JSON",
default=[])
if fork_opts:
parser.add_option(
'-f',
'--forks',
dest='forks',
default=C.DEFAULT_FORKS,
type='int',
help="specify number of parallel processes to use (default=%s)"
% C.DEFAULT_FORKS)
if vault_opts:
parser.add_option('--ask-vault-pass',
default=C.DEFAULT_ASK_VAULT_PASS,
dest='ask_vault_pass',
action='store_true',
help='ask for vault password')
parser.add_option('--vault-password-file',
default=C.DEFAULT_VAULT_PASSWORD_FILE,
dest='vault_password_file',
help="vault password file",
action="callback",
callback=CLI.expand_tilde,
type=str)
parser.add_option('--new-vault-password-file',
dest='new_vault_password_file',
help="new vault password file for rekey",
action="callback",
callback=CLI.expand_tilde,
type=str)
parser.add_option(
'--output',
default=None,
dest='output_file',
help=
'output file name for encrypt or decrypt; use - for stdout',
action="callback",
callback=CLI.expand_tilde,
type=str)
if subset_opts:
parser.add_option(
'-t',
'--tags',
dest='tags',
default='all',
help="only run plays and tasks tagged with these values")
parser.add_option(
'--skip-tags',
dest='skip_tags',
help=
"only run plays and tasks whose tags do not match these values"
)
if output_opts:
parser.add_option('-o',
'--one-line',
dest='one_line',
action='store_true',
help='condense output')
parser.add_option('-t',
'--tree',
dest='tree',
default=None,
help='log output to this directory')
if connect_opts:
connect_group = optparse.OptionGroup(
parser, "Connection Options",
"control as whom and how to connect to hosts")
connect_group.add_option('-k',
'--ask-pass',
default=C.DEFAULT_ASK_PASS,
dest='ask_pass',
action='store_true',
help='ask for connection password')
connect_group.add_option(
'--private-key',
'--key-file',
default=C.DEFAULT_PRIVATE_KEY_FILE,
dest='private_key_file',
help='use this file to authenticate the connection')
connect_group.add_option('-u',
'--user',
default=C.DEFAULT_REMOTE_USER,
dest='remote_user',
help='connect as this user (default=%s)' %
C.DEFAULT_REMOTE_USER)
connect_group.add_option(
'-c',
'--connection',
dest='connection',
default=C.DEFAULT_TRANSPORT,
help="connection type to use (default=%s)" %
C.DEFAULT_TRANSPORT)
connect_group.add_option(
'-T',
'--timeout',
default=C.DEFAULT_TIMEOUT,
type='int',
dest='timeout',
help="override the connection timeout in seconds (default=%s)"
% C.DEFAULT_TIMEOUT)
connect_group.add_option(
'--ssh-common-args',
default='',
dest='ssh_common_args',
help=
"specify common arguments to pass to sftp/scp/ssh (e.g. ProxyCommand)"
)
connect_group.add_option(
'--sftp-extra-args',
default='',
dest='sftp_extra_args',
help=
"specify extra arguments to pass to sftp only (e.g. -f, -l)")
connect_group.add_option(
'--scp-extra-args',
default='',
dest='scp_extra_args',
help="specify extra arguments to pass to scp only (e.g. -l)")
connect_group.add_option(
'--ssh-extra-args',
default='',
dest='ssh_extra_args',
help="specify extra arguments to pass to ssh only (e.g. -R)")
parser.add_option_group(connect_group)
runas_group = None
rg = optparse.OptionGroup(
parser, "Privilege Escalation Options",
"control how and which user you become as on target hosts")
if runas_opts:
runas_group = rg
# priv user defaults to root later on to enable detecting when this option was given here
runas_group.add_option(
"-s",
"--sudo",
default=C.DEFAULT_SUDO,
action="store_true",
dest='sudo',
help=
"run operations with sudo (nopasswd) (deprecated, use become)")
runas_group.add_option(
'-U',
'--sudo-user',
dest='sudo_user',
default=None,
help='desired sudo user (default=root) (deprecated, use become)'
)
runas_group.add_option(
'-S',
'--su',
default=C.DEFAULT_SU,
action='store_true',
help='run operations with su (deprecated, use become)')
runas_group.add_option(
'-R',
'--su-user',
default=None,
help=
'run operations with su as this user (default=%s) (deprecated, use become)'
% C.DEFAULT_SU_USER)
# consolidated privilege escalation (become)
runas_group.add_option(
"-b",
"--become",
default=C.DEFAULT_BECOME,
action="store_true",
dest='become',
help=
"run operations with become (does not imply password prompting)"
)
runas_group.add_option(
'--become-method',
dest='become_method',
default=C.DEFAULT_BECOME_METHOD,
type='choice',
choices=C.BECOME_METHODS,
help=
"privilege escalation method to use (default=%s), valid choices: [ %s ]"
% (C.DEFAULT_BECOME_METHOD, ' | '.join(C.BECOME_METHODS)))
runas_group.add_option(
'--become-user',
default=None,
dest='become_user',
type='string',
help='run operations as this user (default=%s)' %
C.DEFAULT_BECOME_USER)
if runas_opts or runas_prompt_opts:
if not runas_group:
runas_group = rg
runas_group.add_option(
'--ask-sudo-pass',
default=C.DEFAULT_ASK_SUDO_PASS,
dest='ask_sudo_pass',
action='store_true',
help='ask for sudo password (deprecated, use become)')
runas_group.add_option(
'--ask-su-pass',
default=C.DEFAULT_ASK_SU_PASS,
dest='ask_su_pass',
action='store_true',
help='ask for su password (deprecated, use become)')
runas_group.add_option(
'-K',
'--ask-become-pass',
default=False,
dest='become_ask_pass',
action='store_true',
help='ask for privilege escalation password')
if runas_group:
parser.add_option_group(runas_group)
if async_opts:
parser.add_option(
'-P',
'--poll',
default=C.DEFAULT_POLL_INTERVAL,
type='int',
dest='poll_interval',
help="set the poll interval if using -B (default=%s)" %
C.DEFAULT_POLL_INTERVAL)
parser.add_option(
'-B',
'--background',
dest='seconds',
type='int',
default=0,
help='run asynchronously, failing after X seconds (default=N/A)'
)
if check_opts:
parser.add_option(
"-C",
"--check",
default=False,
dest='check',
action='store_true',
help=
"don't make any changes; instead, try to predict some of the changes that may occur"
)
parser.add_option(
'--syntax-check',
dest='syntax',
action='store_true',
help=
"perform a syntax check on the playbook, but do not execute it"
)
parser.add_option(
"-D",
"--diff",
default=False,
dest='diff',
action='store_true',
help=
"when changing (small) files and templates, show the differences in those files; works great with --check"
)
if meta_opts:
parser.add_option('--force-handlers',
default=C.DEFAULT_FORCE_HANDLERS,
dest='force_handlers',
action='store_true',
help="run handlers even if a task fails")
parser.add_option('--flush-cache',
dest='flush_cache',
action='store_true',
help="clear the fact cache")
return parser
default=None,
help='specify frame type')
parser.add_option("--show-test-timings",
action="store_true",
default=False,
help="show how long each test took to run")
parser.add_option("--validate-parameters",
action="store_true",
default=False,
help="validate vehicle parameter files")
parser.add_option("--Werror",
action='store_true',
default=False,
help='configure with --Werror')
group_build = optparse.OptionGroup(parser, "Build options")
group_build.add_option("--no-configure",
default=False,
action='store_true',
help='do not configure before building',
dest="no_configure")
group_build.add_option("",
"--waf-configure-args",
action="append",
dest="waf_configure_args",
type="string",
default=[],
help="extra arguments passed to waf in configure")
group_build.add_option("-j", default=None, type='int', help='build CPUs')
group_build.add_option("--no-clean",
default=False,
def __init__(self, testsuite_directory, *args, **kwargs):
if kwargs.pop('html_output_from_env', None) is not None or \
kwargs.pop('html_output_dir', None) is not None:
warnings.warn(
'The unit tests HTML support was removed from {0}. Please '
'stop passing \'html_output_dir\' or \'html_output_from_env\' '
'as arguments to {0}'.format(self.__class__.__name__),
category=DeprecationWarning,
stacklevel=2)
# Get XML output settings
xml_output_dir_env_var = kwargs.pop('xml_output_from_env',
'XML_TESTS_OUTPUT_DIR')
xml_output_dir = kwargs.pop('xml_output_dir', None)
if xml_output_dir_env_var in os.environ:
xml_output_dir = os.environ.get(xml_output_dir_env_var)
if not xml_output_dir:
xml_output_dir = os.path.join(
tempfile.gettempdir() if platform.system() != 'Darwin' else
'/tmp', 'xml-tests-output')
self.xml_output_dir = xml_output_dir
# Get the desired logfile to use while running tests
self.tests_logfile = kwargs.pop('tests_logfile', None)
optparse.OptionParser.__init__(self, *args, **kwargs)
self.testsuite_directory = testsuite_directory
self.testsuite_results = []
self.test_selection_group = optparse.OptionGroup(
self, 'Tests Selection Options',
'Select which tests are to be executed')
if self.support_destructive_tests_selection is True:
self.test_selection_group.add_option(
'--run-destructive',
action='store_true',
default=False,
help=('Run destructive tests. These tests can include adding '
'or removing users from your system for example. '
'Default: %default'))
if self.support_expensive_tests_selection is True:
self.test_selection_group.add_option(
'--run-expensive',
action='store_true',
default=False,
help=(
'Run expensive tests. Expensive tests are any tests that, '
'once configured, cost money to run, such as creating or '
'destroying cloud instances on a cloud provider.'))
self.test_selection_group.add_option(
'-n',
'--name',
dest='name',
action='append',
default=None,
help=('Specific test name to run. A named test is the module path '
'relative to the tests directory'))
self.test_selection_group.add_option(
'--names-file',
dest='names_file',
default=None,
help=('The location of a newline delimited file of test names to '
'run'))
self.test_selection_group.add_option(
'--from-filenames',
dest='from_filenames',
action='append',
default=None,
help=('Pass a comma-separated list of file paths, and any '
'unit/integration test module which corresponds to the '
'specified file(s) will be run. For example, a path of '
'salt/modules/git.py would result in unit.modules.test_git '
'and integration.modules.test_git being run.'))
self.test_selection_group.add_option(
'--filename-map',
dest='filename_map',
default=None,
help=('Path to a YAML file mapping paths/path globs to a list '
'of test names to run. See tests/files_map.yml '
'for example usage.'))
self.add_option_group(self.test_selection_group)
if self.support_docker_execution is True:
self.docked_selection_group = optparse.OptionGroup(
self, 'Docked Tests Execution',
'Run the tests suite under a Docker container. This allows, '
'for example, to run destructive tests on your machine '
'without actually breaking it in any way.')
self.docked_selection_group.add_option(
'--docked',
default=None,
metavar='CONTAINER',
help='Run the tests suite in the chosen Docker container')
self.docked_selection_group.add_option(
'--docked-interpreter',
default=None,
metavar='PYTHON_INTERPRETER',
help='The python binary name to use when calling the tests '
'suite.')
self.docked_selection_group.add_option(
'--docked-skip-delete',
default=False,
action='store_true',
help='Skip docker container deletion on exit. Default: False')
self.docked_selection_group.add_option(
'--docked-skip-delete-on-errors',
default=False,
action='store_true',
help='Skip docker container deletion on exit if errors '
'occurred. Default: False')
self.docked_selection_group.add_option(
'--docker-binary',
help='The docker binary on the host system. Default: %default',
default='/usr/bin/docker',
)
self.add_option_group(self.docked_selection_group)
self.output_options_group = optparse.OptionGroup(
self, 'Output Options')
self.output_options_group.add_option('-F',
'--fail-fast',
dest='failfast',
default=False,
action='store_true',
help='Stop on first failure')
self.output_options_group.add_option('-v',
'--verbose',
dest='verbosity',
default=1,
action='count',
help='Verbose test runner output')
self.output_options_group.add_option(
'--output-columns',
default=PNUM,
type=int,
help='Number of maximum columns to use on the output')
self.output_options_group.add_option(
'--tests-logfile',
default=self.tests_logfile,
help='The path to the tests suite logging logfile')
if self.xml_output_dir is not None:
self.output_options_group.add_option(
'-x',
'--xml',
'--xml-out',
dest='xml_out',
default=False,
help='XML test runner output(Output directory: {0})'.format(
self.xml_output_dir))
self.output_options_group.add_option(
'--no-report',
default=False,
action='store_true',
help='Do NOT show the overall tests result')
self.add_option_group(self.output_options_group)
self.fs_cleanup_options_group = optparse.OptionGroup(
self, 'File system cleanup Options')
self.fs_cleanup_options_group.add_option(
'--clean',
dest='clean',
default=True,
action='store_true',
help=('Clean up test environment before and after running the '
'tests suite (default behaviour)'))
self.fs_cleanup_options_group.add_option(
'--no-clean',
dest='clean',
action='store_false',
help=('Don\'t clean up test environment before and after the '
'tests suite execution (speed up test process)'))
self.add_option_group(self.fs_cleanup_options_group)
self.setup_additional_options()
def GetOptionParser():
class PlainHelpFormatter(optparse.IndentedHelpFormatter):
def format_description(self, description):
if description:
return description + '\n'
else:
return ''
option_parser = optparse.OptionParser(
usage='%prog [options] replay_file',
formatter=PlainHelpFormatter(),
description=__doc__,
epilog='http://code.google.com/p/web-page-replay/')
option_parser.add_option(
'-r',
'--record',
default=False,
action='store_true',
help='Download real responses and record them to replay_file')
option_parser.add_option('--append',
default=False,
action='store_true',
help='Append responses to replay_file.')
option_parser.add_option('-l',
'--log_level',
default='debug',
action='store',
type='choice',
choices=('debug', 'info', 'warning', 'error',
'critical'),
help='Minimum verbosity level to log')
option_parser.add_option(
'-f',
'--log_file',
default=None,
action='store',
type='string',
help='Log file to use in addition to writting logs to stderr.')
network_group = optparse.OptionGroup(
option_parser, 'Network Simulation Options',
'These options configure the network simulation in replay mode')
network_group.add_option(
'-u',
'--up',
default='0',
action='store',
type='string',
help='Upload Bandwidth in [K|M]{bit/s|Byte/s}. Zero means unlimited.')
network_group.add_option(
'-d',
'--down',
default='0',
action='store',
type='string',
help='Download Bandwidth in [K|M]{bit/s|Byte/s}. Zero means unlimited.'
)
network_group.add_option(
'-m',
'--delay_ms',
default='0',
action='store',
type='string',
help='Propagation delay (latency) in milliseconds. Zero means no delay.'
)
network_group.add_option(
'-p',
'--packet_loss_rate',
default='0',
action='store',
type='string',
help='Packet loss rate in range [0..1]. Zero means no loss.')
network_group.add_option(
'-w',
'--init_cwnd',
default='0',
action='store',
type='string',
help='Set initial cwnd (linux only, requires kernel patch)')
network_group.add_option('--net',
default=None,
action='store',
type='choice',
choices=net_configs.NET_CONFIG_NAMES,
help='Select a set of network options: %s.' %
', '.join(net_configs.NET_CONFIG_NAMES))
network_group.add_option(
'--shaping_type',
default='dummynet',
action='store',
choices=('dummynet', 'proxy'),
help='When shaping is configured (i.e. --up, --down, etc.) decides '
'whether to use |dummynet| (default), or |proxy| servers.')
option_parser.add_option_group(network_group)
harness_group = optparse.OptionGroup(
option_parser, 'Replay Harness Options',
'These advanced options configure various aspects of the replay harness'
)
harness_group.add_option(
'-S',
'--server',
default=None,
action='store',
type='string',
help='IP address of host running "replay.py --server_mode". '
'This only changes the primary DNS nameserver to use the given IP.')
harness_group.add_option(
'-M',
'--server_mode',
default=False,
action='store_true',
help='Run replay DNS & http proxies, and trafficshaping on --port '
'without changing the primary DNS nameserver. '
'Other hosts may connect to this using "replay.py --server" '
'or by pointing their DNS to this server.')
harness_group.add_option(
'-i',
'--inject_scripts',
default='deterministic.js',
action='store',
dest='inject_scripts',
help='A comma separated list of JavaScript sources to inject in all '
'pages. By default a script is injected that eliminates sources '
'of entropy such as Date() and Math.random() deterministic. '
'CAUTION: Without deterministic.js, many pages will not replay.')
harness_group.add_option(
'-D',
'--no-diff_unknown_requests',
default=True,
action='store_false',
dest='diff_unknown_requests',
help='During replay, do not show a diff of unknown requests against '
'their nearest match in the archive.')
harness_group.add_option(
'-C',
'--use_closest_match',
default=False,
action='store_true',
dest='use_closest_match',
help='During replay, if a request is not found, serve the closest match'
'in the archive instead of giving a 404.')
harness_group.add_option(
'-U',
'--use_server_delay',
default=False,
action='store_true',
dest='use_server_delay',
help='During replay, simulate server delay by delaying response time to'
'requests.')
harness_group.add_option(
'-I',
'--screenshot_dir',
default=None,
action='store',
type='string',
help='Save PNG images of the loaded page in the given directory.')
harness_group.add_option(
'-P',
'--no-dns_private_passthrough',
default=True,
action='store_false',
dest='dns_private_passthrough',
help='Don\'t forward DNS requests that resolve to private network '
'addresses. CAUTION: With this option important services like '
'Kerberos will resolve to the HTTP proxy address.')
harness_group.add_option(
'-x',
'--no-dns_forwarding',
default=True,
action='store_false',
dest='dns_forwarding',
help='Don\'t forward DNS requests to the local replay server. '
'CAUTION: With this option an external mechanism must be used to '
'forward traffic to the replay server.')
harness_group.add_option(
'--host',
default=None,
action='store',
type='str',
help='The IP address to bind all servers to. Defaults to 0.0.0.0 or '
'127.0.0.1, depending on --server_mode and platform.')
harness_group.add_option('-o',
'--port',
default=80,
action='store',
type='int',
help='Port number to listen on.')
harness_group.add_option('--ssl_port',
default=443,
action='store',
type='int',
help='SSL port number to listen on.')
harness_group.add_option(
'--http_to_https_port',
default=None,
action='store',
type='int',
help='Port on which WPR will listen for HTTP requests that it will send '
'along as HTTPS requests.')
harness_group.add_option('--dns_port',
default=53,
action='store',
type='int',
help='DNS port number to listen on.')
harness_group.add_option(
'-c',
'--https_root_ca_cert_path',
default=None,
action='store',
type='string',
help='Certificate file to use with SSL (gets auto-generated if needed).'
)
harness_group.add_option('--no-ssl',
default=True,
action='store_false',
dest='ssl',
help='Do not setup an SSL proxy.')
option_parser.add_option_group(harness_group)
harness_group.add_option(
'--should_generate_certs',
default=False,
action='store_true',
help='Use OpenSSL to generate certificate files for requested hosts.')
harness_group.add_option(
'--no-admin-check',
default=True,
action='store_false',
dest='admin_check',
help='Do not check if administrator access is needed.')
harness_group.add_option('--scramble_images',
default=False,
action='store_true',
dest='scramble_images',
help='Scramble image responses.')
return option_parser
def parse_args(args):
option_group_definitions = []
option_group_definitions.append(("Platform options", platform_options()))
option_group_definitions.append(
("Configuration options", configuration_options()))
option_group_definitions.append(
("Printing Options", printing.print_options()))
option_group_definitions.append(("EFL-specific Options", [
optparse.make_option(
"--webprocess-cmd-prefix",
type="string",
default=False,
help="Prefix used when spawning the Web process (Debug mode only)"
),
]))
option_group_definitions.append(("Feature Switches", [
optparse.make_option(
"--complex-text",
action="store_true",
default=False,
help=
"Use the complex text code path for all text (OS X and Windows only)"
),
optparse.make_option("--accelerated-drawing",
action="store_true",
default=False,
help="Use accelerated drawing (OS X only)"),
optparse.make_option(
"--remote-layer-tree",
action="store_true",
default=False,
help="Use the remote layer tree drawing model (OS X WebKit2 only)"
),
]))
option_group_definitions.append((
"WebKit Options",
[
optparse.make_option(
"--gc-between-tests",
action="store_true",
default=False,
help="Force garbage collection between each test"),
optparse.make_option(
"-l",
"--leaks",
action="store_true",
default=False,
help="Enable leaks checking (OS X and Gtk+ only)"),
optparse.make_option("-g",
"--guard-malloc",
action="store_true",
default=False,
help="Enable Guard Malloc (OS X only)"),
optparse.make_option(
"--threaded",
action="store_true",
default=False,
help="Run a concurrent JavaScript thread with each test"),
optparse.make_option(
"--webkit-test-runner",
"-2",
action="store_true",
help="Use WebKitTestRunner rather than DumpRenderTree."),
# FIXME: We should merge this w/ --build-directory and only have one flag.
optparse.make_option(
"--root",
action="store",
help=
"Path to a directory containing the executables needed to run tests."
),
]))
option_group_definitions.append((
"Results Options",
[
optparse.make_option("-p",
"--pixel",
"--pixel-tests",
action="store_true",
dest="pixel_tests",
help="Enable pixel-to-pixel PNG comparisons"),
optparse.make_option(
"--no-pixel",
"--no-pixel-tests",
action="store_false",
dest="pixel_tests",
help="Disable pixel-to-pixel PNG comparisons"),
optparse.make_option(
"--no-sample-on-timeout",
action="store_false",
dest="sample_on_timeout",
help="Don't run sample on timeout (OS X only)"),
optparse.make_option("--no-ref-tests",
action="store_true",
dest="no_ref_tests",
help="Skip all ref tests"),
optparse.make_option(
"--tolerance",
help="Ignore image differences less than this percentage (some "
"ports may ignore this option)",
type="float"),
optparse.make_option("--results-directory",
help="Location of test results"),
optparse.make_option(
"--build-directory",
help=
"Path to the directory under which build files are kept (should not include configuration)"
),
optparse.make_option(
"--add-platform-exceptions",
action="store_true",
default=False,
help=
"Save generated results into the *most-specific-platform* directory rather than the *generic-platform* directory"
),
optparse.make_option(
"--new-baseline",
action="store_true",
default=False,
help="Save generated results as new baselines "
"into the *most-specific-platform* directory, overwriting whatever's "
"already there. Equivalent to --reset-results --add-platform-exceptions"
),
optparse.make_option(
"--reset-results",
action="store_true",
default=False,
help="Reset expectations to the "
"generated results in their existing location."),
optparse.make_option(
"--no-new-test-results",
action="store_false",
dest="new_test_results",
default=True,
help="Don't create new baselines when no expected results exist"
),
optparse.make_option(
"--treat-ref-tests-as-pixel-tests",
action="store_true",
default=False,
help=
"Run ref tests, but treat them as if they were traditional pixel tests"
),
#FIXME: we should support a comma separated list with --pixel-test-directory as well.
optparse.make_option(
"--pixel-test-directory",
action="append",
default=[],
dest="pixel_test_directories",
help=
"A directory where it is allowed to execute tests as pixel tests. "
"Specify multiple times to add multiple directories. "
"This option implies --pixel-tests. If specified, only those tests "
"will be executed as pixel tests that are located in one of the "
"directories enumerated with the option. Some ports may ignore this "
"option while others can have a default value that can be overridden here."
),
optparse.make_option(
"--skip-failing-tests",
action="store_true",
default=False,
help="Skip tests that are expected to fail. "
"Note: When using this option, you might miss new crashes "
"in these tests."),
optparse.make_option(
"--additional-drt-flag",
action="append",
default=[],
help="Additional command line flag to pass to DumpRenderTree "
"Specify multiple times to add multiple flags."),
optparse.make_option(
"--driver-name",
type="string",
help="Alternative DumpRenderTree binary to use"),
optparse.make_option(
"--additional-platform-directory",
action="append",
default=[],
help="Additional directory where to look for test "
"baselines (will take precendence over platform baselines). "
"Specify multiple times to add multiple search path entries."),
optparse.make_option(
"--additional-expectations",
action="append",
default=[],
help=
"Path to a test_expectations file that will override previous expectations. "
"Specify multiple times for multiple sets of overrides."),
optparse.make_option(
"--compare-port",
action="store",
default=None,
help="Use the specified port's baselines first"),
optparse.make_option(
"--no-show-results",
action="store_false",
default=True,
dest="show_results",
help="Don't launch a browser with results after the tests "
"are done"),
optparse.make_option(
"--full-results-html",
action="store_true",
default=False,
help=
"Show all failures in results.html, rather than only regressions"
),
optparse.make_option(
"--clobber-old-results",
action="store_true",
default=False,
help="Clobbers test results from previous runs."),
optparse.make_option(
"--http",
action="store_true",
dest="http",
default=True,
help="Run HTTP and WebSocket tests (default)"),
optparse.make_option("--no-http",
action="store_false",
dest="http",
help="Don't run HTTP and WebSocket tests"),
optparse.make_option(
"--ignore-metrics",
action="store_true",
dest="ignore_metrics",
default=False,
help="Ignore rendering metrics related information from test "
"output, only compare the structure of the rendertree."),
optparse.make_option(
"--nocheck-sys-deps",
action="store_true",
default=False,
help="Don't check the system dependencies (themes)"),
optparse.make_option("--nojava",
action="store_true",
default=False,
help="Don't build java support files"),
]))
option_group_definitions.append((
"Testing Options",
[
optparse.make_option(
"--build",
dest="build",
action="store_true",
default=True,
help="Check to ensure the DumpRenderTree build is up-to-date "
"(default)."),
optparse.make_option("--no-build",
dest="build",
action="store_false",
help="Don't check to see if the "
"DumpRenderTree build is up-to-date."),
optparse.make_option(
"-n",
"--dry-run",
action="store_true",
default=False,
help=
"Do everything but actually run the tests or upload results."),
optparse.make_option(
"--wrapper",
help="wrapper command to insert before invocations of "
"DumpRenderTree; option is split on whitespace before "
"running. (Example: --wrapper='valgrind --smc-check=all')"),
optparse.make_option(
"-i",
"--ignore-tests",
action="append",
default=[],
help=
"directories or test to ignore (may specify multiple times)"),
optparse.make_option("--test-list",
action="append",
help="read list of tests to run from file",
metavar="FILE"),
optparse.make_option(
"--skipped",
action="store",
default="default",
help=
("control how tests marked SKIP are run. "
"'default' == Skip tests unless explicitly listed on the command line, "
"'ignore' == Run them anyway, "
"'only' == only run the SKIP tests, "
"'always' == always skip, even if listed on the command line."
)),
optparse.make_option(
"--force",
action="store_true",
default=False,
help=
"Run all tests with PASS as expected result, even those marked SKIP in the test list (implies --skipped=ignore)"
),
optparse.make_option("--time-out-ms",
help="Set the timeout for each test"),
optparse.make_option(
"--order",
action="store",
default="natural",
help=
("determine the order in which the test cases will be run. "
"'none' == use the order in which the tests were listed either in arguments or test list, "
"'natural' == use the natural order (default), "
"'random' == randomize the test order.")),
optparse.make_option(
"--run-chunk",
help=("Run a specified chunk (n:l), the nth of len l, "
"of the layout tests")),
optparse.make_option(
"--run-part",
help=("Run a specified part (n:m), "
"the nth of m parts, of the layout tests")),
optparse.make_option(
"--batch-size",
help=("Run a the tests in batches (n), after every n tests, "
"DumpRenderTree is relaunched."),
type="int",
default=None),
optparse.make_option(
"--run-singly",
action="store_true",
default=False,
help=
"run a separate DumpRenderTree for each test (implies --verbose)"
),
optparse.make_option(
"--child-processes",
help="Number of DumpRenderTrees to run in parallel."),
# FIXME: Display default number of child processes that will run.
optparse.make_option("-f",
"--fully-parallel",
action="store_true",
help="run all tests in parallel"),
optparse.make_option(
"--exit-after-n-failures",
type="int",
default=None,
help="Exit after the first N failures instead of running all "
"tests"),
optparse.make_option(
"--exit-after-n-crashes-or-timeouts",
type="int",
default=None,
help="Exit after the first N crashes instead of "
"running all tests"),
optparse.make_option(
"--iterations",
type="int",
default=1,
help="Number of times to run the set of tests (e.g. ABCABCABC)"
),
optparse.make_option(
"--repeat-each",
type="int",
default=1,
help="Number of times to run each test (e.g. AAABBBCCC)"),
optparse.make_option(
"--retry-failures",
action="store_true",
default=True,
help=
"Re-try any tests that produce unexpected results (default)"),
optparse.make_option(
"--no-retry-failures",
action="store_false",
dest="retry_failures",
help="Don't re-try any tests that produce unexpected results."
),
optparse.make_option(
"--max-locked-shards",
type="int",
default=0,
help="Set the maximum number of locked shards"),
optparse.make_option(
"--additional-env-var",
type="string",
action="append",
default=[],
help=
"Passes that environment variable to the tests (--additional-env-var=NAME=VALUE)"
),
optparse.make_option("--profile",
action="store_true",
help="Output per-test profile information."),
optparse.make_option(
"--profiler",
action="store",
help=
"Output per-test profile information, using the specified profiler."
),
optparse.make_option("--no-timeout",
action="store_true",
default=False,
help="Disable test timeouts"),
]))
option_group_definitions.append(("iOS Simulator Options", [
optparse.make_option(
'--runtime',
help='iOS Simulator runtime identifier (default: latest runtime)'),
optparse.make_option(
'--device-type',
help=
'iOS Simulator device type identifier (default: i386 -> iPhone 5, x86_64 -> iPhone 5s)'
),
]))
option_group_definitions.append(("Miscellaneous Options", [
optparse.make_option("--lint-test-files",
action="store_true",
default=False,
help=("Makes sure the test files parse for all "
"configurations. Does not run any tests.")),
]))
# FIXME: Move these into json_results_generator.py
option_group_definitions.append(("Result JSON Options", [
optparse.make_option("--master-name",
help="The name of the buildbot master."),
optparse.make_option(
"--builder-name",
default="",
help=
("The name of the builder shown on the waterfall running this script. e.g. Apple MountainLion Release WK2 (Tests)."
)),
optparse.make_option(
"--build-name",
default="DUMMY_BUILD_NAME",
help=(
"The name of the builder used in its path, e.g. webkit-rel.")),
optparse.make_option(
"--build-slave",
default="DUMMY_BUILD_SLAVE",
help=("The name of the buildslave used. e.g. apple-macpro-6.")),
optparse.make_option(
"--build-number",
default="DUMMY_BUILD_NUMBER",
help=("The build number of the builder running this script.")),
optparse.make_option(
"--test-results-server",
default="",
help=
("If specified, upload results json files to this appengine server."
)),
optparse.make_option(
"--results-server-host",
default="",
help=(
"If specified, upload results JSON file to this results server."
)),
optparse.make_option(
"--additional-repository-name",
help=("The name of an additional subversion or git checkout")),
optparse.make_option(
"--additional-repository-path",
help=
("The path to an additional subversion or git checkout (requires --additional-repository-name)"
)),
]))
option_parser = optparse.OptionParser()
for group_name, group_options in option_group_definitions:
option_group = optparse.OptionGroup(option_parser, group_name)
option_group.add_options(group_options)
option_parser.add_option_group(option_group)
return option_parser.parse_args(args)
