コード例 #1
0
    def test_isinstance(self):
        with warnings.catch_warnings(record=True):
            DeprecatedName = create_deprecated_class('DeprecatedName', NewName)

            class UpdatedUserClass2(NewName):
                pass

            class UpdatedUserClass2a(NewName):
                pass

            class OutdatedUserClass2(DeprecatedName):
                pass

            class OutdatedUserClass2a(DeprecatedName):
                pass

            class UnrelatedClass(object):
                pass

            class OldStyleClass:
                pass

        assert isinstance(UpdatedUserClass2(), NewName)
        assert isinstance(UpdatedUserClass2a(), NewName)
        assert isinstance(UpdatedUserClass2(), DeprecatedName)
        assert isinstance(UpdatedUserClass2a(), DeprecatedName)
        assert isinstance(OutdatedUserClass2(), DeprecatedName)
        assert isinstance(OutdatedUserClass2a(), DeprecatedName)
        assert not isinstance(OutdatedUserClass2a(), OutdatedUserClass2)
        assert not isinstance(OutdatedUserClass2(), OutdatedUserClass2a)
        assert not isinstance(UnrelatedClass(), DeprecatedName)
        assert not isinstance(OldStyleClass(), DeprecatedName)
コード例 #2
0
    def test_issubclass(self):
        with warnings.catch_warnings(record=True):
            DeprecatedName = create_deprecated_class('DeprecatedName', NewName)

            class UpdatedUserClass1(NewName):
                pass

            class UpdatedUserClass1a(NewName):
                pass

            class OutdatedUserClass1(DeprecatedName):
                pass

            class OutdatedUserClass1a(DeprecatedName):
                pass

            class UnrelatedClass(object):
                pass

            class OldStyleClass:
                pass

        assert issubclass(UpdatedUserClass1, NewName)
        assert issubclass(UpdatedUserClass1a, NewName)
        assert issubclass(UpdatedUserClass1, DeprecatedName)
        assert issubclass(UpdatedUserClass1a, DeprecatedName)
        assert issubclass(OutdatedUserClass1, DeprecatedName)
        assert not issubclass(UnrelatedClass, DeprecatedName)
        assert not issubclass(OldStyleClass, DeprecatedName)
        assert not issubclass(OldStyleClass, DeprecatedName)
        assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a)
        assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)

        self.assertRaises(TypeError, issubclass, object(), DeprecatedName)
コード例 #3
0
ファイル: test_utils_deprecate.py プロジェクト: 01-/scrapy
    def test_issubclass(self):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
            DeprecatedName = create_deprecated_class('DeprecatedName', NewName)

            class UpdatedUserClass1(NewName):
                pass

            class UpdatedUserClass1a(NewName):
                pass

            class OutdatedUserClass1(DeprecatedName):
                pass

            class OutdatedUserClass1a(DeprecatedName):
                pass

            class UnrelatedClass(object):
                pass

            class OldStyleClass:
                pass

        assert issubclass(UpdatedUserClass1, NewName)
        assert issubclass(UpdatedUserClass1a, NewName)
        assert issubclass(UpdatedUserClass1, DeprecatedName)
        assert issubclass(UpdatedUserClass1a, DeprecatedName)
        assert issubclass(OutdatedUserClass1, DeprecatedName)
        assert not issubclass(UnrelatedClass, DeprecatedName)
        assert not issubclass(OldStyleClass, DeprecatedName)
        assert not issubclass(OldStyleClass, DeprecatedName)
        assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a)
        assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)

        self.assertRaises(TypeError, issubclass, object(), DeprecatedName)
コード例 #4
0
ファイル: test_utils_deprecate.py プロジェクト: 01-/scrapy
    def test_isinstance(self):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
            DeprecatedName = create_deprecated_class('DeprecatedName', NewName)

            class UpdatedUserClass2(NewName):
                pass

            class UpdatedUserClass2a(NewName):
                pass

            class OutdatedUserClass2(DeprecatedName):
                pass

            class OutdatedUserClass2a(DeprecatedName):
                pass

            class UnrelatedClass(object):
                pass

            class OldStyleClass:
                pass

        assert isinstance(UpdatedUserClass2(), NewName)
        assert isinstance(UpdatedUserClass2a(), NewName)
        assert isinstance(UpdatedUserClass2(), DeprecatedName)
        assert isinstance(UpdatedUserClass2a(), DeprecatedName)
        assert isinstance(OutdatedUserClass2(), DeprecatedName)
        assert isinstance(OutdatedUserClass2a(), DeprecatedName)
        assert not isinstance(OutdatedUserClass2a(), OutdatedUserClass2)
        assert not isinstance(OutdatedUserClass2(), OutdatedUserClass2a)
        assert not isinstance(UnrelatedClass(), DeprecatedName)
        assert not isinstance(OldStyleClass(), DeprecatedName)
コード例 #5
0
ファイル: test_utils_deprecate.py プロジェクト: 0xfab/scrapy
    def test_inspect_stack(self):
        with mock.patch('inspect.stack', side_effect=IndexError):
            with warnings.catch_warnings(record=True) as w:
                DeprecatedName = create_deprecated_class('DeprecatedName', NewName)
                class SubClass(DeprecatedName):
                    pass

        self.assertIn("Error detecting parent module", str(w[0].message))
コード例 #6
0
    def test_warning_auto_message(self):
        with warnings.catch_warnings(record=True) as w:
            Deprecated = create_deprecated_class('Deprecated', NewName)

            class UserClass2(Deprecated):
                pass

        msg = str(w[0].message)
        self.assertIn("scrapy.tests.test_utils_deprecate.NewName", msg)
        self.assertIn("scrapy.tests.test_utils_deprecate.Deprecated", msg)
コード例 #7
0
    def test_deprecate_subclass_of_deprecated_class(self):
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            Deprecated = create_deprecated_class('Deprecated', NewName,
                                                 warn_category=MyWarning)
            AlsoDeprecated = create_deprecated_class('AlsoDeprecated', Deprecated,
                                                     new_class_path='foo.Bar',
                                                     warn_category=MyWarning)

        w = self._mywarnings(w)
        self.assertEqual(len(w), 0, str(map(str, w)))

        with warnings.catch_warnings(record=True) as w:
            AlsoDeprecated()
            class UserClass(AlsoDeprecated):
                pass

        w = self._mywarnings(w)
        self.assertEqual(len(w), 2)
        self.assertIn('AlsoDeprecated', str(w[0].message))
        self.assertIn('foo.Bar', str(w[0].message))
        self.assertIn('AlsoDeprecated', str(w[1].message))
        self.assertIn('foo.Bar', str(w[1].message))
コード例 #8
0
    def test_subclassing_warns_only_on_direct_childs(self):
        Deprecated = create_deprecated_class("Deprecated", NewName, warn_once=False, warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:

            class UserClass(Deprecated):
                pass

            class NoWarnOnMe(UserClass):
                pass

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertIn("UserClass", str(w[0].message))
コード例 #9
0
    def test_subclassing_warns_once_by_default(self):
        Deprecated = create_deprecated_class('Deprecated', NewName,
                                             warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:
            class UserClass(Deprecated):
                pass

            class FooClass(Deprecated):
                pass

            class BarClass(Deprecated):
                pass

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertIn('UserClass', str(w[0].message))
コード例 #10
0
    def test_subclassing_warns_only_on_direct_childs(self):
        Deprecated = create_deprecated_class('Deprecated',
                                             NewName,
                                             warn_once=False,
                                             warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:

            class UserClass(Deprecated):
                pass

            class NoWarnOnMe(UserClass):
                pass

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertIn('UserClass', str(w[0].message))
コード例 #11
0
    def test_subclassing_warns_once_by_default(self):
        Deprecated = create_deprecated_class('Deprecated', NewName,
                                             warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:
            class UserClass(Deprecated):
                pass

            class FooClass(Deprecated):
                pass

            class BarClass(Deprecated):
                pass

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertIn('UserClass', str(w[0].message))
コード例 #12
0
    def test_custom_class_paths(self):
        Deprecated = create_deprecated_class('Deprecated', NewName,
                                             new_class_path='foo.NewClass',
                                             old_class_path='bar.OldClass',
                                             warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:
            class UserClass(Deprecated):
                pass

            _ = Deprecated()

        w = self._mywarnings(w)
        self.assertEqual(len(w), 2)
        self.assertIn('foo.NewClass', str(w[0].message))
        self.assertIn('bar.OldClass', str(w[0].message))
        self.assertIn('foo.NewClass', str(w[1].message))
        self.assertIn('bar.OldClass', str(w[1].message))
コード例 #13
0
ファイル: test_utils_deprecate.py プロジェクト: 7zhang/scrapy
    def test_warning_on_subclassing(self):
        with warnings.catch_warnings(record=True) as w:
            Deprecated = create_deprecated_class('Deprecated', NewName,
                                                 warn_category=MyWarning)

            class UserClass(Deprecated):
                pass

        self.assertEqual(len(w), 1)
        msg = w[0]
        assert issubclass(msg.category, MyWarning)
        self.assertEqual(
            str(msg.message),
            "scrapy.tests.test_utils_deprecate.UserClass inherits from "
            "deprecated class scrapy.tests.test_utils_deprecate.Deprecated, "
            "please inherit from scrapy.tests.test_utils_deprecate.NewName."
        )
        self.assertEqual(msg.lineno, inspect.getsourcelines(UserClass)[1])
コード例 #14
0
    def test_subclassing_warning_message(self):
        Deprecated = create_deprecated_class('Deprecated', NewName,
                                             warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:
            class UserClass(Deprecated):
                pass

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertEqual(
            str(w[0].message),
            "scrapy.tests.test_utils_deprecate.UserClass inherits from "
            "deprecated class scrapy.tests.test_utils_deprecate.Deprecated, "
            "please inherit from scrapy.tests.test_utils_deprecate.NewName."
            " (warning only on first subclass, there may be others)"
        )
        self.assertEqual(w[0].lineno, inspect.getsourcelines(UserClass)[1])
コード例 #15
0
    def test_custom_class_paths(self):
        Deprecated = create_deprecated_class('Deprecated', NewName,
                                             new_class_path='foo.NewClass',
                                             old_class_path='bar.OldClass',
                                             warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:
            class UserClass(Deprecated):
                pass

            _ = Deprecated()

        w = self._mywarnings(w)
        self.assertEqual(len(w), 2)
        self.assertIn('foo.NewClass', str(w[0].message))
        self.assertIn('bar.OldClass', str(w[0].message))
        self.assertIn('foo.NewClass', str(w[1].message))
        self.assertIn('bar.OldClass', str(w[1].message))
コード例 #16
0
    def test_subclassing_warning_message(self):
        Deprecated = create_deprecated_class('Deprecated', NewName,
                                             warn_category=MyWarning)

        with warnings.catch_warnings(record=True) as w:
            class UserClass(Deprecated):
                pass

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertEqual(
            str(w[0].message),
            "tests.test_utils_deprecate.UserClass inherits from "
            "deprecated class tests.test_utils_deprecate.Deprecated, "
            "please inherit from tests.test_utils_deprecate.NewName."
            " (warning only on first subclass, there may be others)"
        )
        self.assertEqual(w[0].lineno, inspect.getsourcelines(UserClass)[1])
コード例 #17
0
ファイル: test_utils_deprecate.py プロジェクト: 7zhang/scrapy
    def test_warning_on_instance(self):
        with warnings.catch_warnings(record=True) as w:
            Deprecated = create_deprecated_class('Deprecated', NewName,
                                                 warn_category=MyWarning)

            class UserClass(Deprecated):
                pass

            _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
            _ = UserClass()

        self.assertEqual(len(w), 2)
        msg = w[1]
        assert issubclass(msg.category, MyWarning)
        self.assertEqual(
            str(msg.message),
            "scrapy.tests.test_utils_deprecate.Deprecated is deprecated, "
            "instantiate scrapy.tests.test_utils_deprecate.NewName instead."
        )
        self.assertEqual(msg.lineno, lineno)
コード例 #18
0
    def test_warning_on_instance(self):
        Deprecated = create_deprecated_class('Deprecated', NewName,
                                             warn_category=MyWarning)

        # ignore subclassing warnings
        with warnings.catch_warnings(record=True):
            class UserClass(Deprecated):
                pass

        with warnings.catch_warnings(record=True) as w:
            _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
            _ = UserClass()  # subclass instances don't warn

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertEqual(
            str(w[0].message),
            "scrapy.tests.test_utils_deprecate.Deprecated is deprecated, "
            "instantiate scrapy.tests.test_utils_deprecate.NewName instead."
        )
        self.assertEqual(w[0].lineno, lineno)
コード例 #19
0
    def test_warning_on_instance(self):
        Deprecated = create_deprecated_class('Deprecated',
                                             NewName,
                                             warn_category=MyWarning)

        # ignore subclassing warnings
        with warnings.catch_warnings(record=True):

            class UserClass(Deprecated):
                pass

        with warnings.catch_warnings(record=True) as w:
            _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
            _ = UserClass()  # subclass instances don't warn

        w = self._mywarnings(w)
        self.assertEqual(len(w), 1)
        self.assertEqual(
            str(w[0].message),
            "scrapy.tests.test_utils_deprecate.Deprecated is deprecated, "
            "instantiate scrapy.tests.test_utils_deprecate.NewName instead.")
        self.assertEqual(w[0].lineno, lineno)
コード例 #20
0
    def test_clsdict(self):
        with warnings.catch_warnings(record=True):
            Deprecated = create_deprecated_class('Deprecated', NewName,
                                                 {'foo': 'bar'})

        self.assertEqual(Deprecated.foo, 'bar')
コード例 #21
0
    def test_clsdict(self):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
            Deprecated = create_deprecated_class('Deprecated', NewName, {'foo': 'bar'})

        self.assertEqual(Deprecated.foo, 'bar')
コード例 #22
0
"""
This module provides some commonly used processors for Item Loaders.

See documentation in docs/topics/loaders.rst
"""
from itemloaders import processors

from scrapy.utils.deprecate import create_deprecated_class

MapCompose = create_deprecated_class('MapCompose', processors.MapCompose)

Compose = create_deprecated_class('Compose', processors.Compose)

TakeFirst = create_deprecated_class('TakeFirst', processors.TakeFirst)

Identity = create_deprecated_class('Identity', processors.Identity)

SelectJmes = create_deprecated_class('SelectJmes', processors.SelectJmes)

Join = create_deprecated_class('Join', processors.Join)
コード例 #23
0
"""
This module provides some commonly used processors for Item Loaders.

See documentation in docs/topics/loaders.rst
"""
from itemloaders import processors

from scrapy.utils.deprecate import create_deprecated_class

MapCompose = create_deprecated_class("MapCompose", processors.MapCompose)
Compose = create_deprecated_class("Compose", processors.Compose)
TakeFirst = create_deprecated_class("TakeFirst", processors.TakeFirst)
Identity = create_deprecated_class("Identity", processors.Identity)
SelectJmes = create_deprecated_class("SelectJmes", processors.SelectJmes)
Join = create_deprecated_class("Join", processors.Join)
コード例 #24
0
"""
Backwards compatibility shim. Use scrapy.spiderloader instead.
"""
from scrapy.spiderloader import SpiderLoader
from scrapy.utils.deprecate import create_deprecated_class

SpiderManager = create_deprecated_class('SpiderManager', SpiderLoader)
コード例 #25
0
 def test_deprecate_a_class_with_custom_metaclass(self):
     Meta1 = type('Meta1', (type, ), {})
     New = Meta1('New', (), {})
     Deprecated = create_deprecated_class('Deprecated', New)
コード例 #26
0
        self._seen = WeakKeyDictionary()
        self.hsref = hsref.hsref
        self.pipe_writer = pipe_writer
        self.request_id_sequence = request_id_sequence

    def process_spider_input(self, response, spider):
        self.pipe_writer.write_request(
            url=response.url,
            status=response.status,
            method=response.request.method,
            rs=len(response.body),
            duration=response.meta.get('download_latency', 0) * 1000,
            parent=response.meta.get(HS_PARENT_ID_KEY),
            fp=request_fingerprint(response.request),
        )
        self._seen[response] = next(self.request_id_sequence)

    def process_spider_output(self, response, result, spider):
        parent = self._seen.pop(response)
        for x in result:
            if isinstance(x, Request):
                x.meta[HS_PARENT_ID_KEY] = parent
            yield x


HubstorageMiddleware = create_deprecated_class(
    "HubstorageMiddleware",
    HubstorageMiddleware,
    warn_category=SHScrapyDeprecationWarning,
    subclass_warn_message=_HUBSTORAGE_MIDDLEWARE_WARNING)
コード例 #27
0
ファイル: __init__.py プロジェクト: PeterLUYP/2016YCProject
    def __getitem__(self, opt_name):
        if opt_name in self.overrides:
            return self.overrides[opt_name]
        if self.settings_module and hasattr(self.settings_module, opt_name):
            return getattr(self.settings_module, opt_name)
        if opt_name in self.defaults:
            return self.defaults[opt_name]
        return Settings.__getitem__(self, opt_name)

    def __str__(self):
        return "<CrawlerSettings module=%r>" % self.settings_module


CrawlerSettings = create_deprecated_class(
    'CrawlerSettings',
    CrawlerSettings,
    new_class_path='scrapy.settings.Settings')


def iter_default_settings():
    """Return the default settings as an iterator of (name, value) tuples"""
    for name in dir(default_settings):
        if name.isupper():
            yield name, getattr(default_settings, name)


def overridden_settings(settings):
    """Return a dict of the settings that have been overridden"""
    for name, defvalue in iter_default_settings():
        value = settings[name]
        if not isinstance(defvalue, dict) and value != defvalue:
コード例 #28
0
ファイル: csstranslator.py プロジェクト: 01-/scrapy
from parsel.csstranslator import XPathExpr, GenericTranslator, HTMLTranslator
from scrapy.utils.deprecate import create_deprecated_class


ScrapyXPathExpr = create_deprecated_class(
    'ScrapyXPathExpr', XPathExpr,
    new_class_path='parsel.csstranslator.XPathExpr')

ScrapyGenericTranslator = create_deprecated_class(
    'ScrapyGenericTranslator', GenericTranslator,
    new_class_path='parsel.csstranslator.GenericTranslator')

ScrapyHTMLTranslator = create_deprecated_class(
    'ScrapyHTMLTranslator', HTMLTranslator,
    new_class_path='parsel.csstranslator.HTMLTranslator')
コード例 #29
0
    def test_no_warning_on_definition(self):
        with warnings.catch_warnings(record=True) as w:
            Deprecated = create_deprecated_class('Deprecated', NewName)

        w = self._mywarnings(w)
        self.assertEqual(w, [])
コード例 #30
0
ファイル: squeues.py プロジェクト: shantikumar/Web-Scraper
# public queue classes
PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQueue)
PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue)
MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue)
MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue)
FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)
LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)


# deprecated queue classes
_subclass_warn_message = "{cls} inherits from deprecated class {old}"
_instance_warn_message = "{cls} is deprecated"
PickleFifoDiskQueueNonRequest = create_deprecated_class(
    name="PickleFifoDiskQueueNonRequest",
    new_class=_PickleFifoSerializationDiskQueue,
    subclass_warn_message=_subclass_warn_message,
    instance_warn_message=_instance_warn_message,
)
PickleLifoDiskQueueNonRequest = create_deprecated_class(
    name="PickleLifoDiskQueueNonRequest",
    new_class=_PickleLifoSerializationDiskQueue,
    subclass_warn_message=_subclass_warn_message,
    instance_warn_message=_instance_warn_message,
)
MarshalFifoDiskQueueNonRequest = create_deprecated_class(
    name="MarshalFifoDiskQueueNonRequest",
    new_class=_MarshalFifoSerializationDiskQueue,
    subclass_warn_message=_subclass_warn_message,
    instance_warn_message=_instance_warn_message,
)
MarshalLifoDiskQueueNonRequest = create_deprecated_class(
コード例 #31
0
ファイル: __init__.py プロジェクト: 01-/scrapy
        values = self._get_xpathvalues(xpath, **kw)
        return self.get_value(values, *processors, **kw)

    @deprecated(use_instead='._get_xpathvalues()')
    def _get_values(self, xpaths, **kw):
        return self._get_xpathvalues(xpaths, **kw)

    def _get_xpathvalues(self, xpaths, **kw):
        self._check_selector_method()
        xpaths = arg_to_iter(xpaths)
        return flatten(self.selector.xpath(xpath).extract() for xpath in xpaths)

    def add_css(self, field_name, css, *processors, **kw):
        values = self._get_cssvalues(css, **kw)
        self.add_value(field_name, values, *processors, **kw)

    def replace_css(self, field_name, css, *processors, **kw):
        values = self._get_cssvalues(css, **kw)
        self.replace_value(field_name, values, *processors, **kw)

    def get_css(self, css, *processors, **kw):
        values = self._get_cssvalues(css, **kw)
        return self.get_value(values, *processors, **kw)

    def _get_cssvalues(self, csss, **kw):
        self._check_selector_method()
        csss = arg_to_iter(csss)
        return flatten(self.selector.css(css).extract() for css in csss)

XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)
コード例 #32
0
from parsel.csstranslator import XPathExpr, GenericTranslator, HTMLTranslator
from scrapy.utils.deprecate import create_deprecated_class

ScrapyXPathExpr = create_deprecated_class(
    'ScrapyXPathExpr',
    XPathExpr,
    new_class_path='parsel.csstranslator.XPathExpr')

ScrapyGenericTranslator = create_deprecated_class(
    'ScrapyGenericTranslator',
    GenericTranslator,
    new_class_path='parsel.csstranslator.GenericTranslator')

ScrapyHTMLTranslator = create_deprecated_class(
    'ScrapyHTMLTranslator',
    HTMLTranslator,
    new_class_path='parsel.csstranslator.HTMLTranslator')
コード例 #33
0
#静态类方法,
    @staticmethod
    def close(spider, reason):
        closed = getattr(spider, 'closed', None)
        if callable(closed):
            return closed(reason)

#类似java的toString方法
    def __str__(self):
        return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self))

    __repr__ = __str__

#兼容以前的废弃类BaseSpider
BaseSpider = create_deprecated_class('BaseSpider', Spider)

class ObsoleteClass(object):
    def __init__(self, message):
        self.message = message

    def __getattr__(self, name):
        raise AttributeError(self.message)

spiders = ObsoleteClass(
    '"from scrapy.spider import spiders" no longer works - use '
    '"from scrapy.spiderloader import SpiderLoader" and instantiate '
    'it with your project settings"'
)

# Top-level imports
コード例 #34
0
        elif not body_passed and data_passed:
            kwargs['body'] = self._dumps(data)

            if 'method' not in kwargs:
                kwargs['method'] = 'POST'

        super().__init__(*args, **kwargs)
        self.headers.setdefault('Content-Type', 'application/json')
        self.headers.setdefault(
            'Accept', 'application/json, text/javascript, */*; q=0.01')

    def replace(self, *args, **kwargs):
        body_passed = kwargs.get('body', None) is not None
        data = kwargs.pop('data', None)
        data_passed = data is not None

        if body_passed and data_passed:
            warnings.warn('Both body and data passed. data will be ignored')

        elif not body_passed and data_passed:
            kwargs['body'] = self._dumps(data)

        return super().replace(*args, **kwargs)

    def _dumps(self, data):
        """Convert to JSON """
        return json.dumps(data, **self._dumps_kwargs)


JSONRequest = create_deprecated_class("JSONRequest", JsonRequest)
コード例 #35
0
    @deprecated(use_instead='._get_xpathvalues()')
    def _get_values(self, xpaths, **kw):
        return self._get_xpathvalues(xpaths, **kw)

    def _get_xpathvalues(self, xpaths, **kw):
        self._check_selector_method()
        xpaths = arg_to_iter(xpaths)
        return flatten(
            self.selector.xpath(xpath).extract() for xpath in xpaths)

    def add_css(self, field_name, css, *processors, **kw):
        values = self._get_cssvalues(css, **kw)
        self.add_value(field_name, values, *processors, **kw)

    def replace_css(self, field_name, css, *processors, **kw):
        values = self._get_cssvalues(css, **kw)
        self.replace_value(field_name, values, *processors, **kw)

    def get_css(self, css, *processors, **kw):
        values = self._get_cssvalues(css, **kw)
        return self.get_value(values, *processors, **kw)

    def _get_cssvalues(self, csss, **kw):
        self._check_selector_method()
        csss = arg_to_iter(csss)
        return flatten(self.selector.css(css).extract() for css in csss)


XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)
コード例 #36
0
ファイル: lxmlsel.py プロジェクト: xtmhm2000/scrapy-0.22
    'XPathSelectorList'
]


def _xpathselector_css(self, *a, **kw):
    raise RuntimeError('.css() method not available for %s, '
                       'instantiate scrapy.selector.Selector '
                       'instead' % type(self).__name__)


XPathSelector = create_deprecated_class(
    'XPathSelector',
    Selector,
    {
        '__slots__': (),
        '_default_type': 'html',
        'css': _xpathselector_css,
    },
    new_class_path='scrapy.selector.Selector',
    old_class_path='scrapy.selector.XPathSelector',
)

XmlXPathSelector = create_deprecated_class(
    'XmlXPathSelector',
    XPathSelector,
    clsdict={
        '__slots__': (),
        '_default_type': 'xml',
    },
    new_class_path='scrapy.selector.Selector',
    old_class_path='scrapy.selector.XmlXPathSelector',