def __init__(self):
self.CLASSES.update(settings['RESPONSE_CLASSES'])
self.classes = {}
self.mimetypes = MimeTypes()
mimedata = get_data('scrapy', 'mime.types')
self.mimetypes.readfp(StringIO(mimedata))
for mimetype, cls in self.CLASSES.iteritems():
self.classes[mimetype] = load_object(cls)
def __init__(self):
self.CLASSES.update(settings.get('RESPONSE_CLASSES', {}))
self.classes = {}
self.mimetypes = MimeTypes()
mimedata = get_data('scrapy', 'mime.types')
self.mimetypes.readfp(StringIO(mimedata))
for mimetype, cls in self.CLASSES.iteritems():
self.classes[mimetype] = load_object(cls)
def __init__(self):
self.CLASSES.update(settings.get("RESPONSE_CLASSES", {}))
self.classes = {}
self.mimetypes = MimeTypes()
mimedata = get_data("scrapy", "mime.types")
self.mimetypes.readfp(StringIO(mimedata))
for mimetype, cls in self.CLASSES.iteritems():
self.classes[mimetype] = load_object(cls)
def __init__(self, values=None):
if values is None:
sources = self._getsources()
default_config = get_data(__package__, 'default_scrapyd.conf')
self.cp = SafeConfigParser()
self.cp.readfp(StringIO(default_config))
self.cp.read(sources)
else:
self.cp = SafeConfigParser(values)
self.cp.add_section(self.SECTION)
def __init__(self, values=None, extra_sources=()):
if values is None:
sources = self._getsources()
default_config = get_data(__package__, 'default_scrapyd.conf')
self.cp = SafeConfigParser()
self.cp.readfp(StringIO(default_config))
self.cp.read(sources)
for fp in extra_sources:
self.cp.readfp(fp)
else:
self.cp = SafeConfigParser(values)
self.cp.add_section(self.SECTION)
def test_get_spider_list(self):
path = os.path.abspath(self.mktemp())
j = os.path.join
eggs_dir = j(path, 'eggs')
os.makedirs(eggs_dir)
dbs_dir = j(path, 'dbs')
os.makedirs(dbs_dir)
logs_dir = j(path, 'logs')
os.makedirs(logs_dir)
os.chdir(path)
with open('scrapyd.conf', 'w') as f:
f.write("[scrapyd]\n")
f.write("eggs_dir = %s\n" % eggs_dir)
f.write("dbs_dir = %s\n" % dbs_dir)
f.write("logs_dir = %s\n" % logs_dir)
app = get_application()
eggstorage = app.getComponent(IEggStorage)
eggfile = StringIO(get_data(__package__, 'mybot.egg'))
eggstorage.put(eggfile, 'mybot', 'r1')
self.assertEqual(sorted(get_spider_list('mybot')), ['spider1', 'spider2'])
def test_get_spider_list_from_eggfile(self):
eggfile = StringIO(get_data(__package__, 'mybot.egg'))
spiders = get_spider_list_from_eggfile(eggfile, 'mybot')
self.assertEqual(set(spiders), set(['spider1', 'spider2']))