def add_file_option(self, *args, **kwargs):
"""Add a command-line option that sends an external file
(e.g. a SQLite DB) to Hadoop::
def configure_options(self):
super(MRYourJob, self).configure_options()
self.add_file_option('--scoring-db', help=...)
This does the right thing: the file will be uploaded to the working
dir of the script on Hadoop, and the script will be passed the same
option, but with the local name of the file in the script's working
directory.
We suggest against sending Berkeley DBs to your job, as
Berkeley DB is not forwards-compatible (so a Berkeley DB that you
construct on your computer may not be readable from within
Hadoop). Use SQLite databases instead. If all you need is an on-disk
hash table, try out the :py:mod:`sqlite3dbm` module.
"""
pass_opt = self.option_parser.add_option(*args, **kwargs)
if not pass_opt.type == 'string':
raise OptionError('passthrough file options must take strings' %
pass_opt.type)
if pass_opt.action not in ('store', 'append'):
raise OptionError("passthrough file options must use the options"
" 'store' or 'append'")
self._file_options.append(pass_opt)
def _check_options(self):
if self.action == 'command':
if self.options and not isinstance(self.options, (tuple, list)):
raise OptionError('options must be a tuple or a list', self)
elif self.options is not None:
raise OptionError("'options' must not be supplied for action "
"%r" % self.action, self)
def main():
"""Main function declares options and initialisation routine for daemon."""
global SESSION, LOG
# Find networks on this machine, to determine good defaults
# and help verify options
networks = NetworkHelper()
# See if this system has simple defaults we can use
default_iface, default_ip = networks.get_default()
# program options
oprs = opts_common("control24d Communication Daemon")
oprs.add_option(
"-n",
"--network",
dest="network",
help=
"Ethernet interface to the same network as the Control24. Default = %s"
% default_iface)
default_listener = networks.ipstr_from_tuple(default_ip,
DEFAULTS.get('daemon'))
oprs.add_option("-l",
"--listen",
dest="listen",
help="listen on given host:port. Default = %s" %
default_listener)
oprs.set_defaults(network=default_iface)
oprs.set_defaults(listen=default_listener)
# Parse and verify options
# TODO move to argparse and use that to verify
(opts, __) = oprs.parse_args()
if not networks.get(opts.network):
print networks
raise OptionError(
'Specified network does not exist. Known networks are listed to the output.',
'network')
if not networks.verify_ip(opts.listen.split(':')[0]):
raise OptionError('No network has the IP address specified.', 'listen')
# Build the C24Session
if SESSION is None:
SESSION = C24session(opts, networks)
# Main thread when everything is initiated. Wait for interrupt
if sys.platform.startswith('win'):
# Set up Interrupt signal handler so daemon can close cleanly
signal.signal(signal.SIGINT, signal_handler)
while True:
try:
time.sleep(TIMING_MAIN_LOOP)
except KeyboardInterrupt:
break
else:
signal.pause()
SESSION.close()
def _check_attrs(self):
if self.action == 'command':
if self.args is None:
self.args = 'optional'
elif self.args not in ('optional', 'required', 'no'):
raise OptionError("args must be on of: 'optional', "
"'required', no'", self)
elif self.args is not None:
raise OptionError("'args' must not be supplied for action "
"%r" % self.action, self)
def _check_choice(self):
"""FIXME: need to override this due to optik misdesign"""
if self.type in ("choice", "multiple_choice"):
if self.choices is None:
raise OptionError(
"must supply a list of choices for type 'choice'", self)
elif type(self.choices) not in (types.TupleType, types.ListType):
raise OptionError(
"choices must be a list of strings ('%s' supplied)" %
str(type(self.choices)).split("'")[1], self)
elif self.choices is not None:
raise OptionError(
"must not supply choices for type %r" % self.type, self)
def _check_multiple_choice(self):
if self.type == "multiple_choice":
if self.mchoices is None:
raise OptionError(
"must supply a list of mchoices for type '%s'" % self.type,
self)
elif type(self.mchoices) not in (types.TupleType, types.ListType):
raise OptionError(
"choices must be a list of strings ('%s' supplied)" %
str(type(self.mchoices)).split("'")[1], self)
if self.split_char is None:
self.split_char = ','
elif self.mchoices is not None:
raise OptionError(
"must not supply mchoices for type %r" % self.type, self)
def _check_choice(self) -> None:
"""FIXME: need to override this due to optik misdesign"""
if self.type in ("choice", "multiple_choice"):
# mypy: "Option" has no attribute "choices"
# we know that option of this type has this attribute
if self.choices is None: # type: ignore
raise OptionError("must supply a list of choices for type 'choice'", self)
elif not isinstance(self.choices, (tuple, list)): # type: ignore
raise OptionError(
"choices must be a list of strings ('%s' supplied)"
% str(type(self.choices)).split("'")[1], # type: ignore
self,
)
elif self.choices is not None: # type: ignore
raise OptionError("must not supply choices for type %r" % self.type, self)
def _check_attrs(self):
if self.args is None:
self.args = 'optional'
if not self.args in ('optional', 'required', 'no'):
raise OptionError(
"args must be on of: 'optional', 'required', "
"'no'", self)
def _set_opt_string(self, opts):
if self.REMAINING in opts:
self._long_opts.append(self.REMAINING)
optparse.Option._set_opt_string(self, opts)
if len(self._short_opts) > len(self._long_opts):
raise OptionError("goption.Option needs more long option names "
"than short option names")
def parse_args(option_parser):
options, args = option_parser.parse_args()
if len(args) != 0:
raise OptionError('This program takes no arguments', option_parser)
return options
def check_key(option, opt_str, value, parser):
try:
if len(value) != 32:
raise Error
long(value, 16)
except:
raise OptionError("invalid --key option", option)
setattr(parser.values, option.dest, value)
def _check_nargs(self):
nargs_limit = self.NARGS_SPECIFIED_ACTIONS[self.action]
if nargs_limit is None:
_Option._check_nargs(self)
elif self.nargs is None:
self.nargs = nargs_limit
elif self.nargs != nargs_limit:
msg = "'nargs' must be %d for action %r" % (nargs_limit,
self.action)
raise OptionError(msg, self)
def assert_option_format(self):
"""
I don't want environment vars to be provided as
"-e KEY VALUE", I want "-e KEY=VALUE" instead.
Would argparse help here?
"""
dict_values = vars(self.values)
if 'environment' in dict_values and dict_values['environment']:
for envar in dict_values['environment']:
if '=' not in envar:
raise OptionError(
'Usage: -e KEY1=VALUE1 -e KEY2=VALUE2...', '-e')
def parse_args(option_parser, cl_args=None):
option_parser = make_option_parser()
options, args = option_parser.parse_args(cl_args)
# should be one argument, the job flow ID
if len(args) != 1:
raise OptionError('Must supply one positional argument as the job'
' flow ID', option_parser)
options.emr_job_flow_id = args[0]
return options
def handle(self, *test_labels, **options):
if options['dump_options']:
# Dump the parsed options and then exit without running tests
self.stdout.write(repr(options))
return
from django.test.utils import get_runner
TestRunner = get_runner(settings)
TestRunner = result_hook_wrap(TestRunner)
if options['pdb'] and options['xmlreports']:
from optparse import OptionError
raise OptionError("--pdb, -x", "cannot have pdb and xmlreports specified")
if options['pdb']:
TestRunner = self.pdb_wrap(TestRunner)
if options['xmlreports']:
TestRunner = self.xml_wrap(TestRunner)
if options['tags'] or self.have_tag_exclusions(options):
exclusions = None
if self.have_tag_exclusions(options) and not options['no_exclude']:
exclusions = options.get('exclude_tags') or ','.join(
getattr(settings, 'TEST_EXCLUDE_TAGS'), [])
TestRunner = self.tag_wrap(TestRunner, options.get('tags'),
exclusions)
if options['coverage']:
TestRunner = self.coverage_wrap(TestRunner, options['coverage'])
if options['profile']:
TestRunner = self.profile_wrap(TestRunner)
if options['headless']:
self.start_headless_display()
self._core_handle(TestRunner, *test_labels, **options)
def parse_commandline():
"""Parse the command line. Returns options, args."""
parser = OptionParser(
usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
parser.add_option("-l",
"--latest",
action="store_true",
default=False,
help="Build only the latest version of each package")
parser.add_option("-s",
"--stop",
action="store_true",
default=False,
help="Make the build stop on exceptions")
parser.add_option(
"-t",
"--test",
action="store_true",
default=False,
help=
"Test mode - put output in the tmp directory only, and always build, even if the output already exists."
)
parser.add_option("--server",
action="store_true",
default=False,
help="Use build server")
parser.add_option(
"--resetserver",
action="store_true",
default=False,
help=
"Reset and create a brand new build server, even if the existing one appears to be ok."
)
parser.add_option("--on-server",
dest="onserver",
action="store_true",
default=False,
help="Specify that we're running on the build server")
parser.add_option(
"--skip-scan",
dest="skipscan",
action="store_true",
default=False,
help="Skip scanning the source code for binaries and other problems")
parser.add_option(
"--no-tarball",
dest="notarball",
action="store_true",
default=False,
help="Don't create a source tarball, useful when testing a build")
parser.add_option(
"-f",
"--force",
action="store_true",
default=False,
help=
"Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode."
)
parser.add_option("-a",
"--all",
action="store_true",
default=False,
help="Build all applications available")
parser.add_option("-w",
"--wiki",
default=False,
action="store_true",
help="Update the wiki")
options, args = parser.parse_args()
# Force --stop with --on-server to get correct exit code
if options.onserver:
options.stop = True
if options.force and not options.test:
raise OptionError("Force is only allowed in test mode", "force")
return options, args
def main():
global options, config
options, args = parse_commandline()
if not args and not options.all:
raise OptionError(
"If you really want to build all the apps, use --all", "all")
config = common.read_config(options)
if config['build_server_always']:
options.server = True
if options.resetserver and not options.server:
raise OptionError(
"Using --resetserver without --server makes no sense",
"resetserver")
log_dir = 'logs'
if not os.path.isdir(log_dir):
logging.info("Creating log directory")
os.makedirs(log_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
if options.test:
output_dir = tmp_dir
else:
output_dir = 'unsigned'
if not os.path.isdir(output_dir):
logging.info("Creating output directory")
os.makedirs(output_dir)
if config['archive_older'] != 0:
also_check_dir = 'archive'
else:
also_check_dir = None
repo_dir = 'repo'
build_dir = 'build'
if not os.path.isdir(build_dir):
logging.info("Creating build directory")
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
# Read all app and srclib metadata
allapps = metadata.read_metadata(xref=not options.onserver)
apps = common.read_app_args(args, allapps, True)
for appid, app in apps.items():
if (app['Disabled'] and not options.force
) or not app['Repo Type'] or not app['builds']:
del apps[appid]
if not apps:
raise FDroidException("No apps to process.")
if options.latest:
for app in apps.itervalues():
for build in reversed(app['builds']):
if build['disable'] and not options.force:
continue
app['builds'] = [build]
break
if options.wiki:
import mwclient
site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
path=config['wiki_path'])
site.login(config['wiki_user'], config['wiki_password'])
# Build applications...
failed_apps = {}
build_succeeded = []
for appid, app in apps.iteritems():
first = True
for thisbuild in app['builds']:
wikilog = None
try:
# For the first build of a particular app, we need to set up
# the source repo. We can reuse it on subsequent builds, if
# there are any.
if first:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib',
app['Repo'])
else:
build_dir = os.path.join('build', appid)
# Set up vcs interface and make sure we have the latest code...
logging.debug("Getting {0} vcs interface for {1}".format(
app['Repo Type'], app['Repo']))
vcs = common.getvcs(app['Repo Type'], app['Repo'],
build_dir)
first = False
logging.debug("Checking " + thisbuild['version'])
if trybuild(app, thisbuild, build_dir, output_dir,
also_check_dir, srclib_dir, extlib_dir, tmp_dir,
repo_dir, vcs, options.test, options.server,
options.force, options.onserver):
build_succeeded.append(app)
wikilog = "Build succeeded"
except BuildException as be:
logfile = open(os.path.join(log_dir, appid + '.log'), 'a+')
logfile.write(str(be))
logfile.close()
print("Could not build app %s due to BuildException: %s" %
(appid, be))
if options.stop:
sys.exit(1)
failed_apps[appid] = be
wikilog = be.get_wikitext()
except VCSException as vcse:
reason = str(vcse).split(
'\n', 1)[0] if options.verbose else str(vcse)
logging.error("VCS error while building app %s: %s" %
(appid, reason))
if options.stop:
sys.exit(1)
failed_apps[appid] = vcse
wikilog = str(vcse)
except Exception as e:
logging.error(
"Could not build app %s due to unknown error: %s" %
(appid, traceback.format_exc()))
if options.stop:
sys.exit(1)
failed_apps[appid] = e
wikilog = str(e)
if options.wiki and wikilog:
try:
# Write a page with the last build log for this version code
lastbuildpage = appid + '/lastbuild_' + thisbuild['vercode']
newpage = site.Pages[lastbuildpage]
txt = "Build completed at " + time.strftime(
"%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + wikilog
newpage.save(txt, summary='Build log')
# Redirect from /lastbuild to the most recent build log
newpage = site.Pages[appid + '/lastbuild']
newpage.save('#REDIRECT [[' + lastbuildpage + ']]',
summary='Update redirect')
except:
logging.error(
"Error while attempting to publish build log")
for app in build_succeeded:
logging.info("success: %s" % (app['id']))
if not options.verbose:
for fa in failed_apps:
logging.info("Build for app %s failed:\n%s" %
(fa, failed_apps[fa]))
logging.info("Finished.")
if len(build_succeeded) > 0:
logging.info(str(len(build_succeeded)) + ' builds succeeded')
if len(failed_apps) > 0:
logging.info(str(len(failed_apps)) + ' builds failed')
sys.exit(0)
default=True,
dest='accept_connections',
action='store_false',
help="don't accept any incoming connection requests",
)
parser.add_option(
'--get-delay',
default=0,
dest='get_delay',
metavar='GET_DELAY',
type='int',
help='delay get command by GET_DELAY seconds',
)
parser.add_option(
'-p', '--port',
default=11212,
dest='port',
metavar='PORT',
type='int',
help='listen on PORT',
)
(options, args) = parser.parse_args()
if len(args) > 0:
raise OptionError('unrecognized arguments: %s' % ' '.join(args))
server = MockMemcached('127.0.0.1',
options.port,
options.accept_connections,
options.get_delay)
server.run()
def _check_const(self):
if self.action not in self.CONST_ACTIONS and self.const is not None:
msg = "'const' must not be supplied for action %r" % self.action
raise OptionError(msg, self)
def _check_required(self):
if self.required and not self.takes_value():
raise OptionError(
"required flag set for option that doesn't take a value", self)
def print_help(self, file=None):
raise OptionError(self.format_help(), '-h')
return None
version="%prog 1.1")
optionsParser.add_option("-k", "--keep-clone", action="store_true", dest="keepClone",
help="keep clone when exiting, which uses disk space, default %default", default=False)
optionsParser.add_option("-j", "--dont-ignore-joliet", action="store_true", dest="dontIgnoreJoliet",
help="don't ignore Joliet-extension information, default %default", default=False)
optionsParser.add_option("-l", "--linux-distro", action="store_true", dest="linuxDistro",
help="clone a Linux distro, bootable with boot image and catalog, default %default", default=False)
optionsParser.add_option("-u", "--udf", action="store_true", dest="udf",
help="use UDF instead of ISO 9660, default %default", default=False)
optionsParser.add_option("-w", "--windows-installer", action="store_true", dest="windowsInstaller",
help="clone a Windows installer, bootable with boot image, default %default", default=False)
optionsParser.add_option("-p", "--pause", action="store_true", dest="pause",
help="pause before and after applying modifications, default %default", default=False)
(options, args) = optionsParser.parse_args()
if len(args) is not 1:
raise OptionError("needs exactly one argument, a .iso file")
try:
isoFile = args[0]
isoImageClone = isoImage = None # define to avoid distracting exceptions
if options.linuxDistro:
isoImage = DistroIsoImage(isoFile)
isoImageClone = isoImage.cloneWithModifications(modifications=[],
udf=options.udf,
ignoreJoliet=not options.dontIgnoreJoliet)
elif options.windowsInstaller:
isoImage = WinUdfImage(isoFile)
isoImageClone = isoImage.cloneWithModifications(modifications=[isoImage.modificationForElToritoBootImage()],
pause=options.pause)
else:
isoImage = IsoImage(isoFile)
def load_options(self, args):
super(MRSpannogram, self).load_options(args=args)
if not self.options.rowcount:
raise OptionError("All Options must be present")
def main():
global options, config
# Parse command line...
parser = OptionParser(
usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
parser.add_option("-a",
"--all",
action="store_true",
default=False,
help="Install all signed applications available")
(options, args) = parser.parse_args()
if not args and not options.all:
raise OptionError(
"If you really want to install all the signed apps, use --all",
"all")
config = common.read_config(options)
output_dir = 'repo'
if not os.path.isdir(output_dir):
logging.info("No signed output directory - nothing to do")
sys.exit(0)
if args:
vercodes = common.read_pkg_args(args, True)
apks = {appid: None for appid in vercodes}
# Get the signed apk with the highest vercode
for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))):
try:
appid, vercode = common.apknameinfo(apkfile)
except FDroidException:
continue
if appid not in apks:
continue
if vercodes[appid] and vercode not in vercodes[appid]:
continue
apks[appid] = apkfile
for appid, apk in apks.iteritems():
if not apk:
raise FDroidException("No signed apk available for %s" % appid)
else:
apks = {
common.apknameinfo(apkfile)[0]: apkfile
for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk')))
}
for appid, apk in apks.iteritems():
# Get device list each time to avoid device not found errors
devs = devices()
if not devs:
raise FDroidException("No attached devices found")
logging.info("Installing %s..." % apk)
for dev in devs:
logging.info("Installing %s on %s..." % (apk, dev))
p = FDroidPopen([config['adb'], "-s", dev, "install", apk])
fail = ""
for line in p.output.splitlines():
if line.startswith("Failure"):
fail = line[9:-1]
if not fail:
continue
if fail == "INSTALL_FAILED_ALREADY_EXISTS":
logging.warn("%s is already installed on %s." % (apk, dev))
else:
raise FDroidException("Failed to install %s on %s: %s" %
(apk, dev, fail))
logging.info("\nFinished")
def load_options(self, *args, **kwargs):
super(MRGrepJob, self).load_options(*args, **kwargs)
if not self.options.expression:
raise OptionError("The -e flag is required.")
def _check_const(self):
if self.action != "store_const" and self.action != "store_const_once" \
and getattr(self, 'const') is not None:
raise OptionError(
"'const' must not be supplied for action %r" % self.action,
self)
def handle(self, *args, **options):
# Passing -1 to chown leaves the ownership unchanged, hence the default
uid = -1
gid = -1
chown = options.get('chown', None)
chgrp = options.get('chgrp', None)
verbosity = int(options.get('verbosity', 1))
if chown or chgrp:
# pwd is only available on POSIX-compliant systems
try:
import pwd
except ImportError:
raise OptionError(
'Ownership changes are not supported by your operating system.',
'--chown')
try:
if chown:
uid = pwd.getpwnam(chown).pw_uid
if chgrp:
gid = pwd.getpwnam(chgrp).pw_gid
except (KeyError, TypeError):
raise OptionError(
'The specified username "%s" does not exist or is invalid.'
% chown, '--chown')
if not hasattr(os, 'chmod'):
raise NotImplementedError(
'Permission changes are not supported by your operating system'
)
if not hasattr(settings,
'COMPILER_FORMATS') or not settings.COMPILER_FORMATS:
raise ImproperlyConfigured(
'COMPILER_FORMATS not specified in settings.')
if verbosity:
print 'Looking for slateable CSS files in %s' % settings.MEDIA_ROOT
# Find all files in MEDIA_ROOT that have a COMPILER_FORMATS-supported
# extension, and return them as a list of (full path to file without
# extension, extension) tuples.
files_to_compile = []
for root, dirs, files in os.walk(settings.MEDIA_ROOT):
for _dir in dirs:
for _file in files:
name, ext = os.path.splitext(_file)
if ext in settings.COMPILER_FORMATS:
files_to_compile.append((os.path.join(root,
name), ext))
if verbosity:
print 'Found %s files to be slated...' % len(files_to_compile)
for filename, extension in files_to_compile:
if verbosity > 1:
print 'Compiling %s%s' % (filename, extension)
CssCompressor.compile(filename,
settings.COMPILER_FORMATS[extension])
css_file = '%s.css' % filename
if chown or chgrp:
# Change file ownership to specified group and/or user
os.chown(css_file, uid, gid)
# Make sure owner can write and everyone can read
os.chmod(css_file, 0644)
else:
# Allow everyone to read and write
os.chmod(css_file, 0666)
if verbosity:
print 'Finished slating.'
