class CommandSettings(unittest.TestCase): def setUp(self): self.command = ScrapyCommand() self.command.settings = Settings() self.parser = argparse.ArgumentParser( formatter_class=ScrapyHelpFormatter, conflict_handler='resolve') self.command.add_options(self.parser) def test_settings_json_string(self): feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}' opts, args = self.parser.parse_known_args( args=['-s', f'FEEDS={feeds_json}', 'spider.py']) self.command.process_options(args, opts) self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings) self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json)) def test_help_formatter(self): formatter = ScrapyHelpFormatter(prog='scrapy') part_strings = [ 'usage: scrapy genspider [options] <name> <domain>\n\n', '\n', 'optional arguments:\n', '\n', 'Global Options:\n' ] self.assertEqual( formatter._join_parts(part_strings), ('Usage\n=====\n scrapy genspider [options] <name> <domain>\n\n\n' 'Optional Arguments\n==================\n\n' 'Global Options\n--------------\n'))
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output: if opts.output == '-': self.settings.set('FEED_URI', 'stdout:', priority='cmdline') else: self.settings.set('FEED_URI', opts.output, priority='cmdline') valid_output_formats = ( list(self.settings.getdict('FEED_EXPORTERS').keys()) + list(self.settings.getdict('FEED_EXPORTERS_BASE').keys())) if not opts.output_format: opts.output_format = os.path.splitext(opts.output)[1].replace( ".", "") if opts.output_format not in valid_output_formats: raise UsageError( "Unrecognized output format '%s', set one" " using the '-t' switch or as a file extension" " from the supported list %s" % (opts.output_format, tuple(valid_output_formats))) self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError( "Invalid -a value, use -a NAME=VALUE", print_help=False)
def process_options(self, args, opts): '''对输入的命令进行解析''' ScrapyCommand.process_options(self, args, opts) if opts.travelmode: travelmode = opts.travelmode.strip() if travelmode not in ("飞机", "火车"): raise UsageError("you must input correct travelmode") else: self.settings.set("travelmode", travelmode, priority="cmdline") if opts.traveltime: traveltime = opts.traveltime.strip() if self.verify_date_str_lawyer(traveltime): self.settings.set("traveltime", traveltime, priority="cmdline") else: raise UsageError("you must input correct traveltime") if opts.startstation: startstation = opts.startstation.strip() self.settings.set("startstation", startstation, priority="cmdline") if startstation in list(CityCode.keys()): startcitycode = CityCode[startstation] self.settings.set("startcitycode", startcitycode, priority="cmdline") else: raise UsageError("you must input correct startstation") if opts.endstation: endstation = opts.endstation.strip() self.settings.set("endstation", endstation, priority="cmdline") if endstation in list(CityCode.keys()): endcitycode = CityCode[endstation] self.settings.set("endcitycode", endcitycode, priority="cmdline") else: raise UsageError("you must input correct endstation")
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: pass except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
def process_options(self, args, opts): # 处理从命令行中传入的选项参数 ScrapyCommand.process_options(self, args, opts) # print(self.settings.__dict__) # if not os.path.exists(os.path.dirname(self.settings.attributes.get('LOG_FILE').value)): # os.makedirs(os.path.dirname(self.settings.attributes.get('LOG_FILE').value)) # 加载默认配置 # self.parse_default_args() self.parse_from_cmdline(args,opts)
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) if args: self._locations[args[0]] = opts.output_filename else: locations = self.settings.get('SPIDERDOCS_LOCATIONS', None) if locations: self._locations = locations else: raise UsageError("Module name is required.", print_help=False)
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output: feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) self.settings.set('FEEDS', feeds, priority='cmdline')
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) if opts.urlqueue: self.settings.set('Redis_key', opts.urlqueue, priority='cmdline') except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
class CommandSettings(unittest.TestCase): def setUp(self): self.command = ScrapyCommand() self.command.settings = Settings() self.parser = optparse.OptionParser( formatter=optparse.TitledHelpFormatter(), conflict_handler='resolve', ) self.command.add_options(self.parser) def test_settings_json_string(self): feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}' opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) self.command.process_options(args, opts) self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings) self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json))
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) opts.spargs = {} if len(args) == 2: self.settings['RSS'] = args.pop() self._takeover_logging() if opts.output: self.settings['OUTPUT'] = opts.output[0] self.settings.pop('FEEDS') self.settings['CMDLINE_ARGS'] = {'args': args, 'opts': vars(opts)} if opts.verbose: self.settings['VERBOSE'] = True self.settings.set('LOG_VIOLATIONS', True, priority='cmdline') self.settings.set('STATS_DUMP', True, priority='cmdline')
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output: if opts.output == '-': self.settings.set('FEED_URI', 'stdout:', priority='cmdline') else: self.settings.set('FEED_URI', opts.output, priority='cmdline') feed_exporters = without_none_values(self.settings._getcomposite('FEED_EXPORTERS')) valid_output_formats = feed_exporters.keys() if not opts.output_format: opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") if opts.output_format not in valid_output_formats: raise UsageError("Unrecognized output format '%s', set one" " using the '-t' switch or as a file extension" " from the supported list %s" % (opts.output_format, tuple(valid_output_formats))) self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) self.process_spider_arguments(opts) self.process_request_meta(opts) self.process_request_cb_kwargs(opts)
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: print "Invalid -a value, use -a NAME=VALUE"
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts)
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) self.process_spider_arguments(opts) self.process_request_meta(opts)
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) opts.spargs = arglist_to_dict(opts.spargs)
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: print("Invalid -a value, use -a NAME=VALUE")
def process_options(self, args, opts): BaseParser.process_options(self, args, opts) self.process_spider_arguments(opts) self.process_request_meta(opts)
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: print "*************** Multitask exception *******************"
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) # try: opts.spargs = arglist_to_dict(opts.spargs)
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except Exception: raise Exception
def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: pass