def test_build_spider_registry_with_name(self):
cfg = self.build_config(['test.util_module', 'zz'])
reg = build_spider_registry(cfg)
self.assertEqual(2, len(reg))
cfg = self.build_config(['test.util_module:SecondSpider', 'zz'])
reg = build_spider_registry(cfg)
self.assertEqual(1, len(reg))
def test_build_spider_registry_with_name(self):
cfg = self.build_config(['test.util_module', 'zz'])
reg = build_spider_registry(cfg)
self.assertEqual(2, len(reg))
cfg = self.build_config(['test.util_module:SecondSpider', 'zz'])
reg = build_spider_registry(cfg)
self.assertEqual(1, len(reg))
def grab_control(request):
form = ControlForm(request.GET or None)
spider_registry = build_spider_registry(build_global_config())
spider_choices = [(x, x) for x in spider_registry.keys()]
form.fields['spider'].choices = spider_choices
form.fields['spider'].widget.choices = spider_choices
command_choices = [(x, x) for x in Spider.get_available_command_names()]
form.fields['command'].choices = command_choices
form.fields['command'].widget.choices = command_choices
context = {
'form': form,
}
return render(request, 'grabstat/control_form.html', context)
def grab_control(request):
form = ControlForm(request.GET or None)
spider_registry = build_spider_registry(build_global_config())
spider_choices = [(x, x) for x in spider_registry.keys()]
form.fields['spider'].choices = spider_choices
form.fields['spider'].widget.choices = spider_choices
command_choices = [(x, x) for x in Spider.get_available_command_names()]
form.fields['command'].choices = command_choices
form.fields['command'].widget.choices = command_choices
context = {
'form': form,
}
return render(request, 'grabstat/control_form.html', context)
def test_load_spider_class_error(self):
cfg = self.build_config(['test.util_module'])
reg = build_spider_registry(cfg)
self.assertRaises(SpiderInternalError, load_spider_class, cfg,
'first_spider_zzz')
def test_build_spider_registry_failed_module(self):
cfg = self.build_config(
['test.util_module', 'zz', 'test.files.invalid_import'])
reg = build_spider_registry(cfg)
self.assertEqual(2, len(reg))
def test_build_spider_registry(self):
cfg = self.build_config(['test.util_module', 'zz'])
reg = build_spider_registry(cfg)
self.assertEqual(reg, SPIDER_REGISTRY)
def test_load_spider_class_error(self):
cfg = self.build_config(['test.util_module'])
reg = build_spider_registry(cfg)
self.assertRaises(SpiderInternalError, load_spider_class,
cfg, 'first_spider_zzz')
def test_build_spider_registry_failed_module(self):
cfg = self.build_config(['test.util_module', 'zz',
'test.files.invalid_import'])
reg = build_spider_registry(cfg)
self.assertEqual(2, len(reg))
def test_build_spider_registry(self):
cfg = self.build_config(['test.util_module', 'zz'])
reg = build_spider_registry(cfg)
self.assertEqual(reg, SPIDER_REGISTRY)
