def add_file_option(self, *args, **kwargs): """Add a command-line option that sends an external file (e.g. a SQLite DB) to Hadoop:: def configure_options(self): super(MRYourJob, self).configure_options() self.add_file_option('--scoring-db', help=...) This does the right thing: the file will be uploaded to the working dir of the script on Hadoop, and the script will be passed the same option, but with the local name of the file in the script's working directory. We suggest against sending Berkeley DBs to your job, as Berkeley DB is not forwards-compatible (so a Berkeley DB that you construct on your computer may not be readable from within Hadoop). Use SQLite databases instead. If all you need is an on-disk hash table, try out the :py:mod:`sqlite3dbm` module. """ pass_opt = self.option_parser.add_option(*args, **kwargs) if not pass_opt.type == 'string': raise OptionError('passthrough file options must take strings' % pass_opt.type) if pass_opt.action not in ('store', 'append'): raise OptionError("passthrough file options must use the options" " 'store' or 'append'") self._file_options.append(pass_opt)
def _check_options(self): if self.action == 'command': if self.options and not isinstance(self.options, (tuple, list)): raise OptionError('options must be a tuple or a list', self) elif self.options is not None: raise OptionError("'options' must not be supplied for action " "%r" % self.action, self)
def main(): """Main function declares options and initialisation routine for daemon.""" global SESSION, LOG # Find networks on this machine, to determine good defaults # and help verify options networks = NetworkHelper() # See if this system has simple defaults we can use default_iface, default_ip = networks.get_default() # program options oprs = opts_common("control24d Communication Daemon") oprs.add_option( "-n", "--network", dest="network", help= "Ethernet interface to the same network as the Control24. Default = %s" % default_iface) default_listener = networks.ipstr_from_tuple(default_ip, DEFAULTS.get('daemon')) oprs.add_option("-l", "--listen", dest="listen", help="listen on given host:port. Default = %s" % default_listener) oprs.set_defaults(network=default_iface) oprs.set_defaults(listen=default_listener) # Parse and verify options # TODO move to argparse and use that to verify (opts, __) = oprs.parse_args() if not networks.get(opts.network): print networks raise OptionError( 'Specified network does not exist. Known networks are listed to the output.', 'network') if not networks.verify_ip(opts.listen.split(':')[0]): raise OptionError('No network has the IP address specified.', 'listen') # Build the C24Session if SESSION is None: SESSION = C24session(opts, networks) # Main thread when everything is initiated. Wait for interrupt if sys.platform.startswith('win'): # Set up Interrupt signal handler so daemon can close cleanly signal.signal(signal.SIGINT, signal_handler) while True: try: time.sleep(TIMING_MAIN_LOOP) except KeyboardInterrupt: break else: signal.pause() SESSION.close()
def _check_attrs(self): if self.action == 'command': if self.args is None: self.args = 'optional' elif self.args not in ('optional', 'required', 'no'): raise OptionError("args must be on of: 'optional', " "'required', no'", self) elif self.args is not None: raise OptionError("'args' must not be supplied for action " "%r" % self.action, self)
def _check_choice(self): """FIXME: need to override this due to optik misdesign""" if self.type in ("choice", "multiple_choice"): if self.choices is None: raise OptionError( "must supply a list of choices for type 'choice'", self) elif type(self.choices) not in (types.TupleType, types.ListType): raise OptionError( "choices must be a list of strings ('%s' supplied)" % str(type(self.choices)).split("'")[1], self) elif self.choices is not None: raise OptionError( "must not supply choices for type %r" % self.type, self)
def _check_multiple_choice(self): if self.type == "multiple_choice": if self.mchoices is None: raise OptionError( "must supply a list of mchoices for type '%s'" % self.type, self) elif type(self.mchoices) not in (types.TupleType, types.ListType): raise OptionError( "choices must be a list of strings ('%s' supplied)" % str(type(self.mchoices)).split("'")[1], self) if self.split_char is None: self.split_char = ',' elif self.mchoices is not None: raise OptionError( "must not supply mchoices for type %r" % self.type, self)
def _check_choice(self) -> None: """FIXME: need to override this due to optik misdesign""" if self.type in ("choice", "multiple_choice"): # mypy: "Option" has no attribute "choices" # we know that option of this type has this attribute if self.choices is None: # type: ignore raise OptionError("must supply a list of choices for type 'choice'", self) elif not isinstance(self.choices, (tuple, list)): # type: ignore raise OptionError( "choices must be a list of strings ('%s' supplied)" % str(type(self.choices)).split("'")[1], # type: ignore self, ) elif self.choices is not None: # type: ignore raise OptionError("must not supply choices for type %r" % self.type, self)
def _check_attrs(self): if self.args is None: self.args = 'optional' if not self.args in ('optional', 'required', 'no'): raise OptionError( "args must be on of: 'optional', 'required', " "'no'", self)
def _set_opt_string(self, opts): if self.REMAINING in opts: self._long_opts.append(self.REMAINING) optparse.Option._set_opt_string(self, opts) if len(self._short_opts) > len(self._long_opts): raise OptionError("goption.Option needs more long option names " "than short option names")
def parse_args(option_parser): options, args = option_parser.parse_args() if len(args) != 0: raise OptionError('This program takes no arguments', option_parser) return options
def check_key(option, opt_str, value, parser): try: if len(value) != 32: raise Error long(value, 16) except: raise OptionError("invalid --key option", option) setattr(parser.values, option.dest, value)
def _check_nargs(self): nargs_limit = self.NARGS_SPECIFIED_ACTIONS[self.action] if nargs_limit is None: _Option._check_nargs(self) elif self.nargs is None: self.nargs = nargs_limit elif self.nargs != nargs_limit: msg = "'nargs' must be %d for action %r" % (nargs_limit, self.action) raise OptionError(msg, self)
def assert_option_format(self): """ I don't want environment vars to be provided as "-e KEY VALUE", I want "-e KEY=VALUE" instead. Would argparse help here? """ dict_values = vars(self.values) if 'environment' in dict_values and dict_values['environment']: for envar in dict_values['environment']: if '=' not in envar: raise OptionError( 'Usage: -e KEY1=VALUE1 -e KEY2=VALUE2...', '-e')
def parse_args(option_parser, cl_args=None): option_parser = make_option_parser() options, args = option_parser.parse_args(cl_args) # should be one argument, the job flow ID if len(args) != 1: raise OptionError('Must supply one positional argument as the job' ' flow ID', option_parser) options.emr_job_flow_id = args[0] return options
def handle(self, *test_labels, **options): if options['dump_options']: # Dump the parsed options and then exit without running tests self.stdout.write(repr(options)) return from django.test.utils import get_runner TestRunner = get_runner(settings) TestRunner = result_hook_wrap(TestRunner) if options['pdb'] and options['xmlreports']: from optparse import OptionError raise OptionError("--pdb, -x", "cannot have pdb and xmlreports specified") if options['pdb']: TestRunner = self.pdb_wrap(TestRunner) if options['xmlreports']: TestRunner = self.xml_wrap(TestRunner) if options['tags'] or self.have_tag_exclusions(options): exclusions = None if self.have_tag_exclusions(options) and not options['no_exclude']: exclusions = options.get('exclude_tags') or ','.join( getattr(settings, 'TEST_EXCLUDE_TAGS'), []) TestRunner = self.tag_wrap(TestRunner, options.get('tags'), exclusions) if options['coverage']: TestRunner = self.coverage_wrap(TestRunner, options['coverage']) if options['profile']: TestRunner = self.profile_wrap(TestRunner) if options['headless']: self.start_headless_display() self._core_handle(TestRunner, *test_labels, **options)
def parse_commandline(): """Parse the command line. Returns options, args.""" parser = OptionParser( usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") parser.add_option("-l", "--latest", action="store_true", default=False, help="Build only the latest version of each package") parser.add_option("-s", "--stop", action="store_true", default=False, help="Make the build stop on exceptions") parser.add_option( "-t", "--test", action="store_true", default=False, help= "Test mode - put output in the tmp directory only, and always build, even if the output already exists." ) parser.add_option("--server", action="store_true", default=False, help="Use build server") parser.add_option( "--resetserver", action="store_true", default=False, help= "Reset and create a brand new build server, even if the existing one appears to be ok." ) parser.add_option("--on-server", dest="onserver", action="store_true", default=False, help="Specify that we're running on the build server") parser.add_option( "--skip-scan", dest="skipscan", action="store_true", default=False, help="Skip scanning the source code for binaries and other problems") parser.add_option( "--no-tarball", dest="notarball", action="store_true", default=False, help="Don't create a source tarball, useful when testing a build") parser.add_option( "-f", "--force", action="store_true", default=False, help= "Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode." ) parser.add_option("-a", "--all", action="store_true", default=False, help="Build all applications available") parser.add_option("-w", "--wiki", default=False, action="store_true", help="Update the wiki") options, args = parser.parse_args() # Force --stop with --on-server to get correct exit code if options.onserver: options.stop = True if options.force and not options.test: raise OptionError("Force is only allowed in test mode", "force") return options, args
def main(): global options, config options, args = parse_commandline() if not args and not options.all: raise OptionError( "If you really want to build all the apps, use --all", "all") config = common.read_config(options) if config['build_server_always']: options.server = True if options.resetserver and not options.server: raise OptionError( "Using --resetserver without --server makes no sense", "resetserver") log_dir = 'logs' if not os.path.isdir(log_dir): logging.info("Creating log directory") os.makedirs(log_dir) tmp_dir = 'tmp' if not os.path.isdir(tmp_dir): logging.info("Creating temporary directory") os.makedirs(tmp_dir) if options.test: output_dir = tmp_dir else: output_dir = 'unsigned' if not os.path.isdir(output_dir): logging.info("Creating output directory") os.makedirs(output_dir) if config['archive_older'] != 0: also_check_dir = 'archive' else: also_check_dir = None repo_dir = 'repo' build_dir = 'build' if not os.path.isdir(build_dir): logging.info("Creating build directory") os.makedirs(build_dir) srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') # Read all app and srclib metadata allapps = metadata.read_metadata(xref=not options.onserver) apps = common.read_app_args(args, allapps, True) for appid, app in apps.items(): if (app['Disabled'] and not options.force ) or not app['Repo Type'] or not app['builds']: del apps[appid] if not apps: raise FDroidException("No apps to process.") if options.latest: for app in apps.itervalues(): for build in reversed(app['builds']): if build['disable'] and not options.force: continue app['builds'] = [build] break if options.wiki: import mwclient site = mwclient.Site((config['wiki_protocol'], config['wiki_server']), path=config['wiki_path']) site.login(config['wiki_user'], config['wiki_password']) # Build applications... failed_apps = {} build_succeeded = [] for appid, app in apps.iteritems(): first = True for thisbuild in app['builds']: wikilog = None try: # For the first build of a particular app, we need to set up # the source repo. We can reuse it on subsequent builds, if # there are any. if first: if app['Repo Type'] == 'srclib': build_dir = os.path.join('build', 'srclib', app['Repo']) else: build_dir = os.path.join('build', appid) # Set up vcs interface and make sure we have the latest code... logging.debug("Getting {0} vcs interface for {1}".format( app['Repo Type'], app['Repo'])) vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) first = False logging.debug("Checking " + thisbuild['version']) if trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test, options.server, options.force, options.onserver): build_succeeded.append(app) wikilog = "Build succeeded" except BuildException as be: logfile = open(os.path.join(log_dir, appid + '.log'), 'a+') logfile.write(str(be)) logfile.close() print("Could not build app %s due to BuildException: %s" % (appid, be)) if options.stop: sys.exit(1) failed_apps[appid] = be wikilog = be.get_wikitext() except VCSException as vcse: reason = str(vcse).split( '\n', 1)[0] if options.verbose else str(vcse) logging.error("VCS error while building app %s: %s" % (appid, reason)) if options.stop: sys.exit(1) failed_apps[appid] = vcse wikilog = str(vcse) except Exception as e: logging.error( "Could not build app %s due to unknown error: %s" % (appid, traceback.format_exc())) if options.stop: sys.exit(1) failed_apps[appid] = e wikilog = str(e) if options.wiki and wikilog: try: # Write a page with the last build log for this version code lastbuildpage = appid + '/lastbuild_' + thisbuild['vercode'] newpage = site.Pages[lastbuildpage] txt = "Build completed at " + time.strftime( "%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + wikilog newpage.save(txt, summary='Build log') # Redirect from /lastbuild to the most recent build log newpage = site.Pages[appid + '/lastbuild'] newpage.save('#REDIRECT [[' + lastbuildpage + ']]', summary='Update redirect') except: logging.error( "Error while attempting to publish build log") for app in build_succeeded: logging.info("success: %s" % (app['id'])) if not options.verbose: for fa in failed_apps: logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa])) logging.info("Finished.") if len(build_succeeded) > 0: logging.info(str(len(build_succeeded)) + ' builds succeeded') if len(failed_apps) > 0: logging.info(str(len(failed_apps)) + ' builds failed') sys.exit(0)
default=True, dest='accept_connections', action='store_false', help="don't accept any incoming connection requests", ) parser.add_option( '--get-delay', default=0, dest='get_delay', metavar='GET_DELAY', type='int', help='delay get command by GET_DELAY seconds', ) parser.add_option( '-p', '--port', default=11212, dest='port', metavar='PORT', type='int', help='listen on PORT', ) (options, args) = parser.parse_args() if len(args) > 0: raise OptionError('unrecognized arguments: %s' % ' '.join(args)) server = MockMemcached('127.0.0.1', options.port, options.accept_connections, options.get_delay) server.run()
def _check_const(self): if self.action not in self.CONST_ACTIONS and self.const is not None: msg = "'const' must not be supplied for action %r" % self.action raise OptionError(msg, self)
def _check_required(self): if self.required and not self.takes_value(): raise OptionError( "required flag set for option that doesn't take a value", self)
def print_help(self, file=None): raise OptionError(self.format_help(), '-h') return None
version="%prog 1.1") optionsParser.add_option("-k", "--keep-clone", action="store_true", dest="keepClone", help="keep clone when exiting, which uses disk space, default %default", default=False) optionsParser.add_option("-j", "--dont-ignore-joliet", action="store_true", dest="dontIgnoreJoliet", help="don't ignore Joliet-extension information, default %default", default=False) optionsParser.add_option("-l", "--linux-distro", action="store_true", dest="linuxDistro", help="clone a Linux distro, bootable with boot image and catalog, default %default", default=False) optionsParser.add_option("-u", "--udf", action="store_true", dest="udf", help="use UDF instead of ISO 9660, default %default", default=False) optionsParser.add_option("-w", "--windows-installer", action="store_true", dest="windowsInstaller", help="clone a Windows installer, bootable with boot image, default %default", default=False) optionsParser.add_option("-p", "--pause", action="store_true", dest="pause", help="pause before and after applying modifications, default %default", default=False) (options, args) = optionsParser.parse_args() if len(args) is not 1: raise OptionError("needs exactly one argument, a .iso file") try: isoFile = args[0] isoImageClone = isoImage = None # define to avoid distracting exceptions if options.linuxDistro: isoImage = DistroIsoImage(isoFile) isoImageClone = isoImage.cloneWithModifications(modifications=[], udf=options.udf, ignoreJoliet=not options.dontIgnoreJoliet) elif options.windowsInstaller: isoImage = WinUdfImage(isoFile) isoImageClone = isoImage.cloneWithModifications(modifications=[isoImage.modificationForElToritoBootImage()], pause=options.pause) else: isoImage = IsoImage(isoFile)
def load_options(self, args): super(MRSpannogram, self).load_options(args=args) if not self.options.rowcount: raise OptionError("All Options must be present")
def main(): global options, config # Parse command line... parser = OptionParser( usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") parser.add_option("-a", "--all", action="store_true", default=False, help="Install all signed applications available") (options, args) = parser.parse_args() if not args and not options.all: raise OptionError( "If you really want to install all the signed apps, use --all", "all") config = common.read_config(options) output_dir = 'repo' if not os.path.isdir(output_dir): logging.info("No signed output directory - nothing to do") sys.exit(0) if args: vercodes = common.read_pkg_args(args, True) apks = {appid: None for appid in vercodes} # Get the signed apk with the highest vercode for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))): try: appid, vercode = common.apknameinfo(apkfile) except FDroidException: continue if appid not in apks: continue if vercodes[appid] and vercode not in vercodes[appid]: continue apks[appid] = apkfile for appid, apk in apks.iteritems(): if not apk: raise FDroidException("No signed apk available for %s" % appid) else: apks = { common.apknameinfo(apkfile)[0]: apkfile for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))) } for appid, apk in apks.iteritems(): # Get device list each time to avoid device not found errors devs = devices() if not devs: raise FDroidException("No attached devices found") logging.info("Installing %s..." % apk) for dev in devs: logging.info("Installing %s on %s..." % (apk, dev)) p = FDroidPopen([config['adb'], "-s", dev, "install", apk]) fail = "" for line in p.output.splitlines(): if line.startswith("Failure"): fail = line[9:-1] if not fail: continue if fail == "INSTALL_FAILED_ALREADY_EXISTS": logging.warn("%s is already installed on %s." % (apk, dev)) else: raise FDroidException("Failed to install %s on %s: %s" % (apk, dev, fail)) logging.info("\nFinished")
def load_options(self, *args, **kwargs): super(MRGrepJob, self).load_options(*args, **kwargs) if not self.options.expression: raise OptionError("The -e flag is required.")
def _check_const(self): if self.action != "store_const" and self.action != "store_const_once" \ and getattr(self, 'const') is not None: raise OptionError( "'const' must not be supplied for action %r" % self.action, self)
def handle(self, *args, **options): # Passing -1 to chown leaves the ownership unchanged, hence the default uid = -1 gid = -1 chown = options.get('chown', None) chgrp = options.get('chgrp', None) verbosity = int(options.get('verbosity', 1)) if chown or chgrp: # pwd is only available on POSIX-compliant systems try: import pwd except ImportError: raise OptionError( 'Ownership changes are not supported by your operating system.', '--chown') try: if chown: uid = pwd.getpwnam(chown).pw_uid if chgrp: gid = pwd.getpwnam(chgrp).pw_gid except (KeyError, TypeError): raise OptionError( 'The specified username "%s" does not exist or is invalid.' % chown, '--chown') if not hasattr(os, 'chmod'): raise NotImplementedError( 'Permission changes are not supported by your operating system' ) if not hasattr(settings, 'COMPILER_FORMATS') or not settings.COMPILER_FORMATS: raise ImproperlyConfigured( 'COMPILER_FORMATS not specified in settings.') if verbosity: print 'Looking for slateable CSS files in %s' % settings.MEDIA_ROOT # Find all files in MEDIA_ROOT that have a COMPILER_FORMATS-supported # extension, and return them as a list of (full path to file without # extension, extension) tuples. files_to_compile = [] for root, dirs, files in os.walk(settings.MEDIA_ROOT): for _dir in dirs: for _file in files: name, ext = os.path.splitext(_file) if ext in settings.COMPILER_FORMATS: files_to_compile.append((os.path.join(root, name), ext)) if verbosity: print 'Found %s files to be slated...' % len(files_to_compile) for filename, extension in files_to_compile: if verbosity > 1: print 'Compiling %s%s' % (filename, extension) CssCompressor.compile(filename, settings.COMPILER_FORMATS[extension]) css_file = '%s.css' % filename if chown or chgrp: # Change file ownership to specified group and/or user os.chown(css_file, uid, gid) # Make sure owner can write and everyone can read os.chmod(css_file, 0644) else: # Allow everyone to read and write os.chmod(css_file, 0666) if verbosity: print 'Finished slating.'