Ejemplo n.º 1
0
    def from_crawler(cls, crawler: Crawler):
        base_settings: BaseSettings = crawler.settings
        cls.normalize(base_settings)

        settings = BaseSettings(priority='spider')
        cls.from_object(settings, crawler.spidercls.SpiderConfig)
        settings.update(
            {k: v
             for k, v in base_settings.items() if k in settings},
            priority='cmdline')

        preset = base_settings.get('PRESET')
        if preset:
            preset_dict = BaseSettings(priority=35)
            cls.from_pyfile(preset_dict, preset)
            settings.update(preset_dict)

        adapted = BaseSettings(priority=50)
        for k, v in settings.items():
            adapt = getattr(SettingsAdapter, k.lower(), None)
            if adapt:
                adapted.update(adapt(v))
            else:
                adapted[k] = v

        base_settings.update(adapted.copy_to_dict(), priority=50)
        base_settings['SPIDER_CONFIG'] = adapted
        return cls()