def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output: if opts.output == '-': self.settings.set('FEED_URI', 'stdout:', priority='cmdline') else: self.settings.set('FEED_URI', opts.output, priority='cmdline') valid_output_formats = ( list(self.settings.getdict('FEED_EXPORTERS').keys()) + list(self.settings.getdict('FEED_EXPORTERS_BASE').keys())) if not opts.output_format: opts.output_format = os.path.splitext(opts.output)[1].replace( ".", "") if opts.output_format not in valid_output_formats: raise UsageError( "Unrecognized output format '%s', set one" " using the '-t' switch or as a file extension" " from the supported list %s" % (opts.output_format, tuple(valid_output_formats))) self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.overwrite_output: if opts.output: raise UsageError("Please use only one of --output and --overwrite-output") opts.output = opts.overwrite_output self.settings.overrides['FEED_OVERWRITE'] = True if opts.output: if opts.output == '-': self.settings.set('FEED_URI', 'stdout:', priority='cmdline') else: self.settings.set('FEED_URI', opts.output, priority='cmdline') valid_output_formats = ( list(self.settings.getdict('FEED_EXPORTERS').keys()) + list(self.settings.getdict('FEED_EXPORTERS_BASE').keys()) ) if not opts.output_format: opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") if opts.output_format not in valid_output_formats: raise UsageError("Unrecognized output format '%s', set one" " using the '-t' switch or as a file extension" " from the supported list %s" % (opts.output_format, tuple(valid_output_formats))) self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.nofollow: settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.nofollow: settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output: if opts.output == '-': self.settings.overrides['FEED_URI'] = 'stdout:' else: self.settings.overrides['FEED_URI'] = opts.output self.settings.overrides['FEED_FORMAT'] = opts.output_format
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output: if opts.output == '-': self.settings.overrides['FEED_URI'] = 'stdout:' else: self.settings.overrides['FEED_URI'] = opts.output self.settings.overrides['FEED_FORMAT'] = opts.output_format
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output: if opts.output == '-': self.settings.overrides['FEED_URI'] = 'stdout:' else: self.settings.overrides['FEED_URI'] = opts.output valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys() if opts.output_format not in valid_output_formats: raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format, valid_output_formats)) self.settings.overrides['FEED_FORMAT'] = opts.output_format
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output: if opts.output == '-': self.settings.overrides['FEED_URI'] = 'stdout:' else: self.settings.overrides['FEED_URI'] = opts.output valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys() if opts.output_format not in valid_output_formats: raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format,valid_output_formats)) self.settings.overrides['FEED_FORMAT'] = opts.output_format
def execute(argv=None): if argv is None: argv = sys.argv cmds = _get_commands_dict() cmdname = _get_command_name(argv) _update_default_settings("scrapy.conf.commands", cmdname) _update_default_settings(settings["COMMANDS_SETTINGS_MODULE"], cmdname) parser = optparse.OptionParser( formatter=optparse.TitledHelpFormatter(), conflict_handler="resolve", add_help_option=False ) if cmdname in cmds: cmd = cmds[cmdname] cmd.add_options(parser) opts, args = parser.parse_args(args=argv[1:]) cmd.process_options(args, opts) parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax()) parser.description = cmd.long_desc() if cmd.requires_project and not settings.settings_module: print "Error running: scrapy-ctl.py %s\n" % cmdname print "Cannot find project settings module in python path: %s" % settings.settings_module_path sys.exit(1) if opts.help: parser.print_help() sys.exit() elif not cmdname: cmd = ScrapyCommand() cmd.add_options(parser) opts, args = parser.parse_args(args=argv) cmd.process_options(args, opts) _print_usage(settings.settings_module) sys.exit(2) else: print "Unknown command: %s\n" % cmdname print 'Use "scrapy-ctl.py -h" for help' sys.exit(2) del args[0] # remove command name from args send_catch_log(signal=command_executed, cmdname=cmdname, cmdobj=cmd, args=args, opts=opts) from scrapy.core.manager import scrapymanager scrapymanager.configure(control_reactor=True) ret = _run_command(cmd, args, opts) if ret is False: parser.print_help()
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output: if opts.output == '-': self.settings.overrides['FEED_URI'] = 'stdout:' else: self.settings.overrides['FEED_URI'] = opts.output valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys() if not opts.output_format: opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") if opts.output_format not in valid_output_formats: raise UsageError("Unrecognized output format '%s', set one" " using the '-t' switch or as a file extension" " from the supported list %s" % (opts.output_format, tuple(valid_output_formats))) self.settings.overrides['FEED_FORMAT'] = opts.output_format
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output: if opts.output == "-": self.settings.set("FEED_URI", "stdout:", priority="cmdline") else: self.settings.set("FEED_URI", opts.output, priority="cmdline") valid_output_formats = list(self.settings.getdict("FEED_EXPORTERS").keys()) + list( self.settings.getdict("FEED_EXPORTERS_BASE").keys() ) if not opts.output_format: opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") if opts.output_format not in valid_output_formats: raise UsageError( "Unrecognized output format '%s', set one" " using the '-t' switch or as a file extension" " from the supported list %s" % (opts.output_format, tuple(valid_output_formats)) ) self.settings.set("FEED_FORMAT", opts.output_format, priority="cmdline")
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) if opts.nofollow: settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts)
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts)