def test_isinstance(self):
        with warnings.catch_warnings(record=True):
            DeprecatedName = create_deprecated_class('DeprecatedName', NewName)

            class UpdatedUserClass2(NewName):
                pass

            class UpdatedUserClass2a(NewName):
                pass

            class OutdatedUserClass2(DeprecatedName):
                pass

            class OutdatedUserClass2a(DeprecatedName):
                pass

            class UnrelatedClass(object):
                pass

            class OldStyleClass:
                pass

        assert isinstance(UpdatedUserClass2(), NewName)
        assert isinstance(UpdatedUserClass2a(), NewName)
        assert isinstance(UpdatedUserClass2(), DeprecatedName)
        assert isinstance(UpdatedUserClass2a(), DeprecatedName)
        assert isinstance(OutdatedUserClass2(), DeprecatedName)
        assert isinstance(OutdatedUserClass2a(), DeprecatedName)
        assert not isinstance(OutdatedUserClass2a(), OutdatedUserClass2)
        assert not isinstance(OutdatedUserClass2(), OutdatedUserClass2a)
        assert not isinstance(UnrelatedClass(), DeprecatedName)
        assert not isinstance(OldStyleClass(), DeprecatedName)
    def test_issubclass(self):
        with warnings.catch_warnings(record=True):
            DeprecatedName = create_deprecated_class('DeprecatedName', NewName)

            class UpdatedUserClass1(NewName):
                pass

            class UpdatedUserClass1a(NewName):
                pass

            class OutdatedUserClass1(DeprecatedName):
                pass

            class OutdatedUserClass1a(DeprecatedName):
                pass

            class UnrelatedClass(object):
                pass

            class OldStyleClass:
                pass

        assert issubclass(UpdatedUserClass1, NewName)
        assert issubclass(UpdatedUserClass1a, NewName)
        assert issubclass(UpdatedUserClass1, DeprecatedName)
        assert issubclass(UpdatedUserClass1a, DeprecatedName)
        assert issubclass(OutdatedUserClass1, DeprecatedName)
        assert not issubclass(UnrelatedClass, DeprecatedName)
        assert not issubclass(OldStyleClass, DeprecatedName)
        assert not issubclass(OldStyleClass, DeprecatedName)
        assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a)
        assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)

        self.assertRaises(TypeError, issubclass, object(), DeprecatedName)
Exemple #3
0
    def test_issubclass(self):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
            DeprecatedName = create_deprecated_class('DeprecatedName', NewName)

            class UpdatedUserClass1(NewName):
                pass

            class UpdatedUserClass1a(NewName):
                pass

            class OutdatedUserClass1(DeprecatedName):
                pass

            class OutdatedUserClass1a(DeprecatedName):
                pass

            class UnrelatedClass(object):
                pass

            class OldStyleClass:
                pass

        assert issubclass(UpdatedUserClass1, NewName)
        assert issubclass(UpdatedUserClass1a, NewName)
        assert issubclass(UpdatedUserClass1, DeprecatedName)
        assert issubclass(UpdatedUserClass1a, DeprecatedName)
        assert issubclass(OutdatedUserClass1, DeprecatedName)
        assert not issubclass(UnrelatedClass, DeprecatedName)
        assert not issubclass(OldStyleClass, DeprecatedName)
        assert not issubclass(OldStyleClass, DeprecatedName)
        assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a)
        assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)

        self.assertRaises(TypeError, issubclass, object(), DeprecatedName)
Exemple #4
0
    def test_isinstance(self):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
            DeprecatedName = create_deprecated_class('DeprecatedName', NewName)

            class UpdatedUserClass2(NewName):
                pass

            class UpdatedUserClass2a(NewName):
                pass

            class OutdatedUserClass2(DeprecatedName):
                pass

            class OutdatedUserClass2a(DeprecatedName):
                pass

            class UnrelatedClass(object):
                pass

            class OldStyleClass:
                pass

        assert isinstance(UpdatedUserClass2(), NewName)
        assert isinstance(UpdatedUserClass2a(), NewName)
        assert isinstance(UpdatedUserClass2(), DeprecatedName)
        assert isinstance(UpdatedUserClass2a(), DeprecatedName)
        assert isinstance(OutdatedUserClass2(), DeprecatedName)
        assert isinstance(OutdatedUserClass2a(), DeprecatedName)
        assert not isinstance(OutdatedUserClass2a(), OutdatedUserClass2)
        assert not isinstance(OutdatedUserClass2(), OutdatedUserClass2a)
        assert not isinstance(UnrelatedClass(), DeprecatedName)
        assert not isinstance(OldStyleClass(), DeprecatedName)
Exemple #5
0
    def test_inspect_stack(self):
        with mock.patch('inspect.stack', side_effect=IndexError):
            with warnings.catch_warnings(record=True) as w:
                DeprecatedName = create_deprecated_class('DeprecatedName', NewName)
                class SubClass(DeprecatedName):
                    pass

        self.assertIn("Error detecting parent module", str(w[0].message))
    def test_warning_auto_message(self):
        with warnings.catch_warnings(record=True) as w:
            Deprecated = create_deprecated_class('Deprecated', NewName)

            class UserClass2(Deprecated):
                pass

        msg = str(w[0].message)
        self.assertIn("scrapy.tests.test_utils_deprecate.NewName", msg)
        self.assertIn("scrapy.tests.test_utils_deprecate.Deprecated", msg)
    def test_deprecate_subclass_of_deprecated_class(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            Deprecated = create_deprecated_class('Deprecated', NewName,
                                                 warn_category=MyWarning)
            AlsoDeprecated = create_deprecated_class('AlsoDeprecated', Deprecated,
                                                     new_class_path='foo.Bar',
                                                     warn_category=MyWarning)

        w = self._mywarnings(w)
        self.assertEqual(len(w), 0, str(map(str, w)))

        with warnings.catch_warnings(record=True) as w:
            AlsoDeprecated()
            class UserClass(AlsoDeprecated):
                pass

        w = self._mywarnings(w)
        self.assertEqual(len(w), 2)
        self.assertIn('AlsoDeprecated', str(w[0].message))
        self.assertIn('foo.Bar', str(w[0].message))
        self.assertIn('AlsoDeprecated', str(w[1].message))
        self.assertIn('foo.Bar', str(w[1].message))
    def test_subclassing_warns_only_on_direct_childs(self):
        Deprecated = create_deprecated_class("Deprecated", NewName, warn_once=False, warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:

            class UserClass(Deprecated):
                pass

            class NoWarnOnMe(UserClass):
                pass

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertIn("UserClass", str(w[0].message))
    def test_subclassing_warns_once_by_default(self):
        Deprecated = create_deprecated_class('Deprecated', NewName,
                                             warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:
            class UserClass(Deprecated):
                pass

            class FooClass(Deprecated):
                pass

            class BarClass(Deprecated):
                pass

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertIn('UserClass', str(w[0].message))
    def test_subclassing_warns_only_on_direct_childs(self):
        Deprecated = create_deprecated_class('Deprecated',
                                             NewName,
                                             warn_once=False,
                                             warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:

            class UserClass(Deprecated):
                pass

            class NoWarnOnMe(UserClass):
                pass

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertIn('UserClass', str(w[0].message))
    def test_subclassing_warns_once_by_default(self):
        Deprecated = create_deprecated_class('Deprecated', NewName,
                                             warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:
            class UserClass(Deprecated):
                pass

            class FooClass(Deprecated):
                pass

            class BarClass(Deprecated):
                pass

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertIn('UserClass', str(w[0].message))
    def test_custom_class_paths(self):
        Deprecated = create_deprecated_class('Deprecated', NewName,
                                             new_class_path='foo.NewClass',
                                             old_class_path='bar.OldClass',
                                             warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:
            class UserClass(Deprecated):
                pass

            _ = Deprecated()

        w = self._mywarnings(w)
        self.assertEqual(len(w), 2)
        self.assertIn('foo.NewClass', str(w[0].message))
        self.assertIn('bar.OldClass', str(w[0].message))
        self.assertIn('foo.NewClass', str(w[1].message))
        self.assertIn('bar.OldClass', str(w[1].message))
Exemple #13
0
    def test_warning_on_subclassing(self):
        with warnings.catch_warnings(record=True) as w:
            Deprecated = create_deprecated_class('Deprecated', NewName,
                                                 warn_category=MyWarning)

            class UserClass(Deprecated):
                pass

        self.assertEqual(len(w), 1)
        msg = w[0]
        assert issubclass(msg.category, MyWarning)
        self.assertEqual(
            str(msg.message),
            "scrapy.tests.test_utils_deprecate.UserClass inherits from "
            "deprecated class scrapy.tests.test_utils_deprecate.Deprecated, "
            "please inherit from scrapy.tests.test_utils_deprecate.NewName."
        )
        self.assertEqual(msg.lineno, inspect.getsourcelines(UserClass)[1])
    def test_subclassing_warning_message(self):
        Deprecated = create_deprecated_class('Deprecated', NewName,
                                             warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:
            class UserClass(Deprecated):
                pass

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertEqual(
            str(w[0].message),
            "scrapy.tests.test_utils_deprecate.UserClass inherits from "
            "deprecated class scrapy.tests.test_utils_deprecate.Deprecated, "
            "please inherit from scrapy.tests.test_utils_deprecate.NewName."
            " (warning only on first subclass, there may be others)"
        )
        self.assertEqual(w[0].lineno, inspect.getsourcelines(UserClass)[1])
    def test_custom_class_paths(self):
        Deprecated = create_deprecated_class('Deprecated', NewName,
                                             new_class_path='foo.NewClass',
                                             old_class_path='bar.OldClass',
                                             warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:
            class UserClass(Deprecated):
                pass

            _ = Deprecated()

        w = self._mywarnings(w)
        self.assertEqual(len(w), 2)
        self.assertIn('foo.NewClass', str(w[0].message))
        self.assertIn('bar.OldClass', str(w[0].message))
        self.assertIn('foo.NewClass', str(w[1].message))
        self.assertIn('bar.OldClass', str(w[1].message))
    def test_subclassing_warning_message(self):
        Deprecated = create_deprecated_class('Deprecated', NewName,
                                             warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:
            class UserClass(Deprecated):
                pass

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertEqual(
            str(w[0].message),
            "tests.test_utils_deprecate.UserClass inherits from "
            "deprecated class tests.test_utils_deprecate.Deprecated, "
            "please inherit from tests.test_utils_deprecate.NewName."
            " (warning only on first subclass, there may be others)"
        )
        self.assertEqual(w[0].lineno, inspect.getsourcelines(UserClass)[1])
Exemple #17
0
    def test_warning_on_instance(self):
        with warnings.catch_warnings(record=True) as w:
            Deprecated = create_deprecated_class('Deprecated', NewName,
                                                 warn_category=MyWarning)

            class UserClass(Deprecated):
                pass

            _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
            _ = UserClass()

        self.assertEqual(len(w), 2)
        msg = w[1]
        assert issubclass(msg.category, MyWarning)
        self.assertEqual(
            str(msg.message),
            "scrapy.tests.test_utils_deprecate.Deprecated is deprecated, "
            "instantiate scrapy.tests.test_utils_deprecate.NewName instead."
        )
        self.assertEqual(msg.lineno, lineno)
    def test_warning_on_instance(self):
        Deprecated = create_deprecated_class('Deprecated', NewName,
                                             warn_category=MyWarning)

        # ignore subclassing warnings
        with warnings.catch_warnings(record=True):
            class UserClass(Deprecated):
                pass

        with warnings.catch_warnings(record=True) as w:
            _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
            _ = UserClass()  # subclass instances don't warn

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertEqual(
            str(w[0].message),
            "scrapy.tests.test_utils_deprecate.Deprecated is deprecated, "
            "instantiate scrapy.tests.test_utils_deprecate.NewName instead."
        )
        self.assertEqual(w[0].lineno, lineno)
    def test_warning_on_instance(self):
        Deprecated = create_deprecated_class('Deprecated',
                                             NewName,
                                             warn_category=MyWarning)

        # ignore subclassing warnings
        with warnings.catch_warnings(record=True):

            class UserClass(Deprecated):
                pass

        with warnings.catch_warnings(record=True) as w:
            _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
            _ = UserClass()  # subclass instances don't warn

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertEqual(
            str(w[0].message),
            "scrapy.tests.test_utils_deprecate.Deprecated is deprecated, "
            "instantiate scrapy.tests.test_utils_deprecate.NewName instead.")
        self.assertEqual(w[0].lineno, lineno)
    def test_clsdict(self):
        with warnings.catch_warnings(record=True):
            Deprecated = create_deprecated_class('Deprecated', NewName,
                                                 {'foo': 'bar'})

        self.assertEqual(Deprecated.foo, 'bar')
    def test_clsdict(self):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
            Deprecated = create_deprecated_class('Deprecated', NewName, {'foo': 'bar'})

        self.assertEqual(Deprecated.foo, 'bar')
Exemple #22
0
"""
This module provides some commonly used processors for Item Loaders.

See documentation in docs/topics/loaders.rst
"""
from itemloaders import processors

from scrapy.utils.deprecate import create_deprecated_class

MapCompose = create_deprecated_class('MapCompose', processors.MapCompose)

Compose = create_deprecated_class('Compose', processors.Compose)

TakeFirst = create_deprecated_class('TakeFirst', processors.TakeFirst)

Identity = create_deprecated_class('Identity', processors.Identity)

SelectJmes = create_deprecated_class('SelectJmes', processors.SelectJmes)

Join = create_deprecated_class('Join', processors.Join)
Exemple #23
0
"""
This module provides some commonly used processors for Item Loaders.

See documentation in docs/topics/loaders.rst
"""
from itemloaders import processors

from scrapy.utils.deprecate import create_deprecated_class

MapCompose = create_deprecated_class("MapCompose", processors.MapCompose)
Compose = create_deprecated_class("Compose", processors.Compose)
TakeFirst = create_deprecated_class("TakeFirst", processors.TakeFirst)
Identity = create_deprecated_class("Identity", processors.Identity)
SelectJmes = create_deprecated_class("SelectJmes", processors.SelectJmes)
Join = create_deprecated_class("Join", processors.Join)
Exemple #24
0
"""
Backwards compatibility shim. Use scrapy.spiderloader instead.
"""
from scrapy.spiderloader import SpiderLoader
from scrapy.utils.deprecate import create_deprecated_class

SpiderManager = create_deprecated_class('SpiderManager', SpiderLoader)
 def test_deprecate_a_class_with_custom_metaclass(self):
     Meta1 = type('Meta1', (type, ), {})
     New = Meta1('New', (), {})
     Deprecated = create_deprecated_class('Deprecated', New)
        self._seen = WeakKeyDictionary()
        self.hsref = hsref.hsref
        self.pipe_writer = pipe_writer
        self.request_id_sequence = request_id_sequence

    def process_spider_input(self, response, spider):
        self.pipe_writer.write_request(
            url=response.url,
            status=response.status,
            method=response.request.method,
            rs=len(response.body),
            duration=response.meta.get('download_latency', 0) * 1000,
            parent=response.meta.get(HS_PARENT_ID_KEY),
            fp=request_fingerprint(response.request),
        )
        self._seen[response] = next(self.request_id_sequence)

    def process_spider_output(self, response, result, spider):
        parent = self._seen.pop(response)
        for x in result:
            if isinstance(x, Request):
                x.meta[HS_PARENT_ID_KEY] = parent
            yield x


HubstorageMiddleware = create_deprecated_class(
    "HubstorageMiddleware",
    HubstorageMiddleware,
    warn_category=SHScrapyDeprecationWarning,
    subclass_warn_message=_HUBSTORAGE_MIDDLEWARE_WARNING)
Exemple #27
0
    def __getitem__(self, opt_name):
        if opt_name in self.overrides:
            return self.overrides[opt_name]
        if self.settings_module and hasattr(self.settings_module, opt_name):
            return getattr(self.settings_module, opt_name)
        if opt_name in self.defaults:
            return self.defaults[opt_name]
        return Settings.__getitem__(self, opt_name)

    def __str__(self):
        return "<CrawlerSettings module=%r>" % self.settings_module


CrawlerSettings = create_deprecated_class(
    'CrawlerSettings',
    CrawlerSettings,
    new_class_path='scrapy.settings.Settings')


def iter_default_settings():
    """Return the default settings as an iterator of (name, value) tuples"""
    for name in dir(default_settings):
        if name.isupper():
            yield name, getattr(default_settings, name)


def overridden_settings(settings):
    """Return a dict of the settings that have been overridden"""
    for name, defvalue in iter_default_settings():
        value = settings[name]
        if not isinstance(defvalue, dict) and value != defvalue:
Exemple #28
0
from parsel.csstranslator import XPathExpr, GenericTranslator, HTMLTranslator
from scrapy.utils.deprecate import create_deprecated_class


ScrapyXPathExpr = create_deprecated_class(
    'ScrapyXPathExpr', XPathExpr,
    new_class_path='parsel.csstranslator.XPathExpr')

ScrapyGenericTranslator = create_deprecated_class(
    'ScrapyGenericTranslator', GenericTranslator,
    new_class_path='parsel.csstranslator.GenericTranslator')

ScrapyHTMLTranslator = create_deprecated_class(
    'ScrapyHTMLTranslator', HTMLTranslator,
    new_class_path='parsel.csstranslator.HTMLTranslator')
    def test_no_warning_on_definition(self):
        with warnings.catch_warnings(record=True) as w:
            Deprecated = create_deprecated_class('Deprecated', NewName)

        w = self._mywarnings(w)
        self.assertEqual(w, [])
Exemple #30
0
# public queue classes
PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQueue)
PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue)
MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue)
MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue)
FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)
LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)


# deprecated queue classes
_subclass_warn_message = "{cls} inherits from deprecated class {old}"
_instance_warn_message = "{cls} is deprecated"
PickleFifoDiskQueueNonRequest = create_deprecated_class(
    name="PickleFifoDiskQueueNonRequest",
    new_class=_PickleFifoSerializationDiskQueue,
    subclass_warn_message=_subclass_warn_message,
    instance_warn_message=_instance_warn_message,
)
PickleLifoDiskQueueNonRequest = create_deprecated_class(
    name="PickleLifoDiskQueueNonRequest",
    new_class=_PickleLifoSerializationDiskQueue,
    subclass_warn_message=_subclass_warn_message,
    instance_warn_message=_instance_warn_message,
)
MarshalFifoDiskQueueNonRequest = create_deprecated_class(
    name="MarshalFifoDiskQueueNonRequest",
    new_class=_MarshalFifoSerializationDiskQueue,
    subclass_warn_message=_subclass_warn_message,
    instance_warn_message=_instance_warn_message,
)
MarshalLifoDiskQueueNonRequest = create_deprecated_class(
Exemple #31
0
        values = self._get_xpathvalues(xpath, **kw)
        return self.get_value(values, *processors, **kw)

    @deprecated(use_instead='._get_xpathvalues()')
    def _get_values(self, xpaths, **kw):
        return self._get_xpathvalues(xpaths, **kw)

    def _get_xpathvalues(self, xpaths, **kw):
        self._check_selector_method()
        xpaths = arg_to_iter(xpaths)
        return flatten(self.selector.xpath(xpath).extract() for xpath in xpaths)

    def add_css(self, field_name, css, *processors, **kw):
        values = self._get_cssvalues(css, **kw)
        self.add_value(field_name, values, *processors, **kw)

    def replace_css(self, field_name, css, *processors, **kw):
        values = self._get_cssvalues(css, **kw)
        self.replace_value(field_name, values, *processors, **kw)

    def get_css(self, css, *processors, **kw):
        values = self._get_cssvalues(css, **kw)
        return self.get_value(values, *processors, **kw)

    def _get_cssvalues(self, csss, **kw):
        self._check_selector_method()
        csss = arg_to_iter(csss)
        return flatten(self.selector.css(css).extract() for css in csss)

XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)
from parsel.csstranslator import XPathExpr, GenericTranslator, HTMLTranslator
from scrapy.utils.deprecate import create_deprecated_class

ScrapyXPathExpr = create_deprecated_class(
    'ScrapyXPathExpr',
    XPathExpr,
    new_class_path='parsel.csstranslator.XPathExpr')

ScrapyGenericTranslator = create_deprecated_class(
    'ScrapyGenericTranslator',
    GenericTranslator,
    new_class_path='parsel.csstranslator.GenericTranslator')

ScrapyHTMLTranslator = create_deprecated_class(
    'ScrapyHTMLTranslator',
    HTMLTranslator,
    new_class_path='parsel.csstranslator.HTMLTranslator')
Exemple #33
0
#静态类方法,
    @staticmethod
    def close(spider, reason):
        closed = getattr(spider, 'closed', None)
        if callable(closed):
            return closed(reason)

#类似java的toString方法
    def __str__(self):
        return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self))

    __repr__ = __str__

#兼容以前的废弃类BaseSpider
BaseSpider = create_deprecated_class('BaseSpider', Spider)

class ObsoleteClass(object):
    def __init__(self, message):
        self.message = message

    def __getattr__(self, name):
        raise AttributeError(self.message)

spiders = ObsoleteClass(
    '"from scrapy.spider import spiders" no longer works - use '
    '"from scrapy.spiderloader import SpiderLoader" and instantiate '
    'it with your project settings"'
)

# Top-level imports
        elif not body_passed and data_passed:
            kwargs['body'] = self._dumps(data)

            if 'method' not in kwargs:
                kwargs['method'] = 'POST'

        super().__init__(*args, **kwargs)
        self.headers.setdefault('Content-Type', 'application/json')
        self.headers.setdefault(
            'Accept', 'application/json, text/javascript, */*; q=0.01')

    def replace(self, *args, **kwargs):
        body_passed = kwargs.get('body', None) is not None
        data = kwargs.pop('data', None)
        data_passed = data is not None

        if body_passed and data_passed:
            warnings.warn('Both body and data passed. data will be ignored')

        elif not body_passed and data_passed:
            kwargs['body'] = self._dumps(data)

        return super().replace(*args, **kwargs)

    def _dumps(self, data):
        """Convert to JSON """
        return json.dumps(data, **self._dumps_kwargs)


JSONRequest = create_deprecated_class("JSONRequest", JsonRequest)
Exemple #35
0
    @deprecated(use_instead='._get_xpathvalues()')
    def _get_values(self, xpaths, **kw):
        return self._get_xpathvalues(xpaths, **kw)

    def _get_xpathvalues(self, xpaths, **kw):
        self._check_selector_method()
        xpaths = arg_to_iter(xpaths)
        return flatten(
            self.selector.xpath(xpath).extract() for xpath in xpaths)

    def add_css(self, field_name, css, *processors, **kw):
        values = self._get_cssvalues(css, **kw)
        self.add_value(field_name, values, *processors, **kw)

    def replace_css(self, field_name, css, *processors, **kw):
        values = self._get_cssvalues(css, **kw)
        self.replace_value(field_name, values, *processors, **kw)

    def get_css(self, css, *processors, **kw):
        values = self._get_cssvalues(css, **kw)
        return self.get_value(values, *processors, **kw)

    def _get_cssvalues(self, csss, **kw):
        self._check_selector_method()
        csss = arg_to_iter(csss)
        return flatten(self.selector.css(css).extract() for css in csss)


XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)
Exemple #36
0
    'XPathSelectorList'
]


def _xpathselector_css(self, *a, **kw):
    raise RuntimeError('.css() method not available for %s, '
                       'instantiate scrapy.selector.Selector '
                       'instead' % type(self).__name__)


XPathSelector = create_deprecated_class(
    'XPathSelector',
    Selector,
    {
        '__slots__': (),
        '_default_type': 'html',
        'css': _xpathselector_css,
    },
    new_class_path='scrapy.selector.Selector',
    old_class_path='scrapy.selector.XPathSelector',
)

XmlXPathSelector = create_deprecated_class(
    'XmlXPathSelector',
    XPathSelector,
    clsdict={
        '__slots__': (),
        '_default_type': 'xml',
    },
    new_class_path='scrapy.selector.Selector',
    old_class_path='scrapy.selector.XmlXPathSelector',