def test_isinstance(self): with warnings.catch_warnings(record=True): DeprecatedName = create_deprecated_class('DeprecatedName', NewName) class UpdatedUserClass2(NewName): pass class UpdatedUserClass2a(NewName): pass class OutdatedUserClass2(DeprecatedName): pass class OutdatedUserClass2a(DeprecatedName): pass class UnrelatedClass(object): pass class OldStyleClass: pass assert isinstance(UpdatedUserClass2(), NewName) assert isinstance(UpdatedUserClass2a(), NewName) assert isinstance(UpdatedUserClass2(), DeprecatedName) assert isinstance(UpdatedUserClass2a(), DeprecatedName) assert isinstance(OutdatedUserClass2(), DeprecatedName) assert isinstance(OutdatedUserClass2a(), DeprecatedName) assert not isinstance(OutdatedUserClass2a(), OutdatedUserClass2) assert not isinstance(OutdatedUserClass2(), OutdatedUserClass2a) assert not isinstance(UnrelatedClass(), DeprecatedName) assert not isinstance(OldStyleClass(), DeprecatedName)
def test_issubclass(self): with warnings.catch_warnings(record=True): DeprecatedName = create_deprecated_class('DeprecatedName', NewName) class UpdatedUserClass1(NewName): pass class UpdatedUserClass1a(NewName): pass class OutdatedUserClass1(DeprecatedName): pass class OutdatedUserClass1a(DeprecatedName): pass class UnrelatedClass(object): pass class OldStyleClass: pass assert issubclass(UpdatedUserClass1, NewName) assert issubclass(UpdatedUserClass1a, NewName) assert issubclass(UpdatedUserClass1, DeprecatedName) assert issubclass(UpdatedUserClass1a, DeprecatedName) assert issubclass(OutdatedUserClass1, DeprecatedName) assert not issubclass(UnrelatedClass, DeprecatedName) assert not issubclass(OldStyleClass, DeprecatedName) assert not issubclass(OldStyleClass, DeprecatedName) assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a) assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1) self.assertRaises(TypeError, issubclass, object(), DeprecatedName)
def test_issubclass(self): with warnings.catch_warnings(): warnings.simplefilter('ignore', ScrapyDeprecationWarning) DeprecatedName = create_deprecated_class('DeprecatedName', NewName) class UpdatedUserClass1(NewName): pass class UpdatedUserClass1a(NewName): pass class OutdatedUserClass1(DeprecatedName): pass class OutdatedUserClass1a(DeprecatedName): pass class UnrelatedClass(object): pass class OldStyleClass: pass assert issubclass(UpdatedUserClass1, NewName) assert issubclass(UpdatedUserClass1a, NewName) assert issubclass(UpdatedUserClass1, DeprecatedName) assert issubclass(UpdatedUserClass1a, DeprecatedName) assert issubclass(OutdatedUserClass1, DeprecatedName) assert not issubclass(UnrelatedClass, DeprecatedName) assert not issubclass(OldStyleClass, DeprecatedName) assert not issubclass(OldStyleClass, DeprecatedName) assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a) assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1) self.assertRaises(TypeError, issubclass, object(), DeprecatedName)
def test_isinstance(self): with warnings.catch_warnings(): warnings.simplefilter('ignore', ScrapyDeprecationWarning) DeprecatedName = create_deprecated_class('DeprecatedName', NewName) class UpdatedUserClass2(NewName): pass class UpdatedUserClass2a(NewName): pass class OutdatedUserClass2(DeprecatedName): pass class OutdatedUserClass2a(DeprecatedName): pass class UnrelatedClass(object): pass class OldStyleClass: pass assert isinstance(UpdatedUserClass2(), NewName) assert isinstance(UpdatedUserClass2a(), NewName) assert isinstance(UpdatedUserClass2(), DeprecatedName) assert isinstance(UpdatedUserClass2a(), DeprecatedName) assert isinstance(OutdatedUserClass2(), DeprecatedName) assert isinstance(OutdatedUserClass2a(), DeprecatedName) assert not isinstance(OutdatedUserClass2a(), OutdatedUserClass2) assert not isinstance(OutdatedUserClass2(), OutdatedUserClass2a) assert not isinstance(UnrelatedClass(), DeprecatedName) assert not isinstance(OldStyleClass(), DeprecatedName)
def test_inspect_stack(self): with mock.patch('inspect.stack', side_effect=IndexError): with warnings.catch_warnings(record=True) as w: DeprecatedName = create_deprecated_class('DeprecatedName', NewName) class SubClass(DeprecatedName): pass self.assertIn("Error detecting parent module", str(w[0].message))
def test_warning_auto_message(self): with warnings.catch_warnings(record=True) as w: Deprecated = create_deprecated_class('Deprecated', NewName) class UserClass2(Deprecated): pass msg = str(w[0].message) self.assertIn("scrapy.tests.test_utils_deprecate.NewName", msg) self.assertIn("scrapy.tests.test_utils_deprecate.Deprecated", msg)
def test_deprecate_subclass_of_deprecated_class(self): with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') Deprecated = create_deprecated_class('Deprecated', NewName, warn_category=MyWarning) AlsoDeprecated = create_deprecated_class('AlsoDeprecated', Deprecated, new_class_path='foo.Bar', warn_category=MyWarning) w = self._mywarnings(w) self.assertEqual(len(w), 0, str(map(str, w))) with warnings.catch_warnings(record=True) as w: AlsoDeprecated() class UserClass(AlsoDeprecated): pass w = self._mywarnings(w) self.assertEqual(len(w), 2) self.assertIn('AlsoDeprecated', str(w[0].message)) self.assertIn('foo.Bar', str(w[0].message)) self.assertIn('AlsoDeprecated', str(w[1].message)) self.assertIn('foo.Bar', str(w[1].message))
def test_subclassing_warns_only_on_direct_childs(self): Deprecated = create_deprecated_class("Deprecated", NewName, warn_once=False, warn_category=MyWarning) with warnings.catch_warnings(record=True) as w: class UserClass(Deprecated): pass class NoWarnOnMe(UserClass): pass w = self._mywarnings(w) self.assertEqual(len(w), 1) self.assertIn("UserClass", str(w[0].message))
def test_subclassing_warns_once_by_default(self): Deprecated = create_deprecated_class('Deprecated', NewName, warn_category=MyWarning) with warnings.catch_warnings(record=True) as w: class UserClass(Deprecated): pass class FooClass(Deprecated): pass class BarClass(Deprecated): pass w = self._mywarnings(w) self.assertEqual(len(w), 1) self.assertIn('UserClass', str(w[0].message))
def test_subclassing_warns_only_on_direct_childs(self): Deprecated = create_deprecated_class('Deprecated', NewName, warn_once=False, warn_category=MyWarning) with warnings.catch_warnings(record=True) as w: class UserClass(Deprecated): pass class NoWarnOnMe(UserClass): pass w = self._mywarnings(w) self.assertEqual(len(w), 1) self.assertIn('UserClass', str(w[0].message))
def test_custom_class_paths(self): Deprecated = create_deprecated_class('Deprecated', NewName, new_class_path='foo.NewClass', old_class_path='bar.OldClass', warn_category=MyWarning) with warnings.catch_warnings(record=True) as w: class UserClass(Deprecated): pass _ = Deprecated() w = self._mywarnings(w) self.assertEqual(len(w), 2) self.assertIn('foo.NewClass', str(w[0].message)) self.assertIn('bar.OldClass', str(w[0].message)) self.assertIn('foo.NewClass', str(w[1].message)) self.assertIn('bar.OldClass', str(w[1].message))
def test_warning_on_subclassing(self): with warnings.catch_warnings(record=True) as w: Deprecated = create_deprecated_class('Deprecated', NewName, warn_category=MyWarning) class UserClass(Deprecated): pass self.assertEqual(len(w), 1) msg = w[0] assert issubclass(msg.category, MyWarning) self.assertEqual( str(msg.message), "scrapy.tests.test_utils_deprecate.UserClass inherits from " "deprecated class scrapy.tests.test_utils_deprecate.Deprecated, " "please inherit from scrapy.tests.test_utils_deprecate.NewName." ) self.assertEqual(msg.lineno, inspect.getsourcelines(UserClass)[1])
def test_subclassing_warning_message(self): Deprecated = create_deprecated_class('Deprecated', NewName, warn_category=MyWarning) with warnings.catch_warnings(record=True) as w: class UserClass(Deprecated): pass w = self._mywarnings(w) self.assertEqual(len(w), 1) self.assertEqual( str(w[0].message), "scrapy.tests.test_utils_deprecate.UserClass inherits from " "deprecated class scrapy.tests.test_utils_deprecate.Deprecated, " "please inherit from scrapy.tests.test_utils_deprecate.NewName." " (warning only on first subclass, there may be others)" ) self.assertEqual(w[0].lineno, inspect.getsourcelines(UserClass)[1])
def test_subclassing_warning_message(self): Deprecated = create_deprecated_class('Deprecated', NewName, warn_category=MyWarning) with warnings.catch_warnings(record=True) as w: class UserClass(Deprecated): pass w = self._mywarnings(w) self.assertEqual(len(w), 1) self.assertEqual( str(w[0].message), "tests.test_utils_deprecate.UserClass inherits from " "deprecated class tests.test_utils_deprecate.Deprecated, " "please inherit from tests.test_utils_deprecate.NewName." " (warning only on first subclass, there may be others)" ) self.assertEqual(w[0].lineno, inspect.getsourcelines(UserClass)[1])
def test_warning_on_instance(self): with warnings.catch_warnings(record=True) as w: Deprecated = create_deprecated_class('Deprecated', NewName, warn_category=MyWarning) class UserClass(Deprecated): pass _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) _ = UserClass() self.assertEqual(len(w), 2) msg = w[1] assert issubclass(msg.category, MyWarning) self.assertEqual( str(msg.message), "scrapy.tests.test_utils_deprecate.Deprecated is deprecated, " "instantiate scrapy.tests.test_utils_deprecate.NewName instead." ) self.assertEqual(msg.lineno, lineno)
def test_warning_on_instance(self): Deprecated = create_deprecated_class('Deprecated', NewName, warn_category=MyWarning) # ignore subclassing warnings with warnings.catch_warnings(record=True): class UserClass(Deprecated): pass with warnings.catch_warnings(record=True) as w: _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) _ = UserClass() # subclass instances don't warn w = self._mywarnings(w) self.assertEqual(len(w), 1) self.assertEqual( str(w[0].message), "scrapy.tests.test_utils_deprecate.Deprecated is deprecated, " "instantiate scrapy.tests.test_utils_deprecate.NewName instead." ) self.assertEqual(w[0].lineno, lineno)
def test_warning_on_instance(self): Deprecated = create_deprecated_class('Deprecated', NewName, warn_category=MyWarning) # ignore subclassing warnings with warnings.catch_warnings(record=True): class UserClass(Deprecated): pass with warnings.catch_warnings(record=True) as w: _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) _ = UserClass() # subclass instances don't warn w = self._mywarnings(w) self.assertEqual(len(w), 1) self.assertEqual( str(w[0].message), "scrapy.tests.test_utils_deprecate.Deprecated is deprecated, " "instantiate scrapy.tests.test_utils_deprecate.NewName instead.") self.assertEqual(w[0].lineno, lineno)
def test_clsdict(self): with warnings.catch_warnings(record=True): Deprecated = create_deprecated_class('Deprecated', NewName, {'foo': 'bar'}) self.assertEqual(Deprecated.foo, 'bar')
def test_clsdict(self): with warnings.catch_warnings(): warnings.simplefilter('ignore', ScrapyDeprecationWarning) Deprecated = create_deprecated_class('Deprecated', NewName, {'foo': 'bar'}) self.assertEqual(Deprecated.foo, 'bar')
""" This module provides some commonly used processors for Item Loaders. See documentation in docs/topics/loaders.rst """ from itemloaders import processors from scrapy.utils.deprecate import create_deprecated_class MapCompose = create_deprecated_class('MapCompose', processors.MapCompose) Compose = create_deprecated_class('Compose', processors.Compose) TakeFirst = create_deprecated_class('TakeFirst', processors.TakeFirst) Identity = create_deprecated_class('Identity', processors.Identity) SelectJmes = create_deprecated_class('SelectJmes', processors.SelectJmes) Join = create_deprecated_class('Join', processors.Join)
""" This module provides some commonly used processors for Item Loaders. See documentation in docs/topics/loaders.rst """ from itemloaders import processors from scrapy.utils.deprecate import create_deprecated_class MapCompose = create_deprecated_class("MapCompose", processors.MapCompose) Compose = create_deprecated_class("Compose", processors.Compose) TakeFirst = create_deprecated_class("TakeFirst", processors.TakeFirst) Identity = create_deprecated_class("Identity", processors.Identity) SelectJmes = create_deprecated_class("SelectJmes", processors.SelectJmes) Join = create_deprecated_class("Join", processors.Join)
""" Backwards compatibility shim. Use scrapy.spiderloader instead. """ from scrapy.spiderloader import SpiderLoader from scrapy.utils.deprecate import create_deprecated_class SpiderManager = create_deprecated_class('SpiderManager', SpiderLoader)
def test_deprecate_a_class_with_custom_metaclass(self): Meta1 = type('Meta1', (type, ), {}) New = Meta1('New', (), {}) Deprecated = create_deprecated_class('Deprecated', New)
self._seen = WeakKeyDictionary() self.hsref = hsref.hsref self.pipe_writer = pipe_writer self.request_id_sequence = request_id_sequence def process_spider_input(self, response, spider): self.pipe_writer.write_request( url=response.url, status=response.status, method=response.request.method, rs=len(response.body), duration=response.meta.get('download_latency', 0) * 1000, parent=response.meta.get(HS_PARENT_ID_KEY), fp=request_fingerprint(response.request), ) self._seen[response] = next(self.request_id_sequence) def process_spider_output(self, response, result, spider): parent = self._seen.pop(response) for x in result: if isinstance(x, Request): x.meta[HS_PARENT_ID_KEY] = parent yield x HubstorageMiddleware = create_deprecated_class( "HubstorageMiddleware", HubstorageMiddleware, warn_category=SHScrapyDeprecationWarning, subclass_warn_message=_HUBSTORAGE_MIDDLEWARE_WARNING)
def __getitem__(self, opt_name): if opt_name in self.overrides: return self.overrides[opt_name] if self.settings_module and hasattr(self.settings_module, opt_name): return getattr(self.settings_module, opt_name) if opt_name in self.defaults: return self.defaults[opt_name] return Settings.__getitem__(self, opt_name) def __str__(self): return "<CrawlerSettings module=%r>" % self.settings_module CrawlerSettings = create_deprecated_class( 'CrawlerSettings', CrawlerSettings, new_class_path='scrapy.settings.Settings') def iter_default_settings(): """Return the default settings as an iterator of (name, value) tuples""" for name in dir(default_settings): if name.isupper(): yield name, getattr(default_settings, name) def overridden_settings(settings): """Return a dict of the settings that have been overridden""" for name, defvalue in iter_default_settings(): value = settings[name] if not isinstance(defvalue, dict) and value != defvalue:
from parsel.csstranslator import XPathExpr, GenericTranslator, HTMLTranslator from scrapy.utils.deprecate import create_deprecated_class ScrapyXPathExpr = create_deprecated_class( 'ScrapyXPathExpr', XPathExpr, new_class_path='parsel.csstranslator.XPathExpr') ScrapyGenericTranslator = create_deprecated_class( 'ScrapyGenericTranslator', GenericTranslator, new_class_path='parsel.csstranslator.GenericTranslator') ScrapyHTMLTranslator = create_deprecated_class( 'ScrapyHTMLTranslator', HTMLTranslator, new_class_path='parsel.csstranslator.HTMLTranslator')
def test_no_warning_on_definition(self): with warnings.catch_warnings(record=True) as w: Deprecated = create_deprecated_class('Deprecated', NewName) w = self._mywarnings(w) self.assertEqual(w, [])
# public queue classes PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQueue) PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue) MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue) MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) # deprecated queue classes _subclass_warn_message = "{cls} inherits from deprecated class {old}" _instance_warn_message = "{cls} is deprecated" PickleFifoDiskQueueNonRequest = create_deprecated_class( name="PickleFifoDiskQueueNonRequest", new_class=_PickleFifoSerializationDiskQueue, subclass_warn_message=_subclass_warn_message, instance_warn_message=_instance_warn_message, ) PickleLifoDiskQueueNonRequest = create_deprecated_class( name="PickleLifoDiskQueueNonRequest", new_class=_PickleLifoSerializationDiskQueue, subclass_warn_message=_subclass_warn_message, instance_warn_message=_instance_warn_message, ) MarshalFifoDiskQueueNonRequest = create_deprecated_class( name="MarshalFifoDiskQueueNonRequest", new_class=_MarshalFifoSerializationDiskQueue, subclass_warn_message=_subclass_warn_message, instance_warn_message=_instance_warn_message, ) MarshalLifoDiskQueueNonRequest = create_deprecated_class(
values = self._get_xpathvalues(xpath, **kw) return self.get_value(values, *processors, **kw) @deprecated(use_instead='._get_xpathvalues()') def _get_values(self, xpaths, **kw): return self._get_xpathvalues(xpaths, **kw) def _get_xpathvalues(self, xpaths, **kw): self._check_selector_method() xpaths = arg_to_iter(xpaths) return flatten(self.selector.xpath(xpath).extract() for xpath in xpaths) def add_css(self, field_name, css, *processors, **kw): values = self._get_cssvalues(css, **kw) self.add_value(field_name, values, *processors, **kw) def replace_css(self, field_name, css, *processors, **kw): values = self._get_cssvalues(css, **kw) self.replace_value(field_name, values, *processors, **kw) def get_css(self, css, *processors, **kw): values = self._get_cssvalues(css, **kw) return self.get_value(values, *processors, **kw) def _get_cssvalues(self, csss, **kw): self._check_selector_method() csss = arg_to_iter(csss) return flatten(self.selector.css(css).extract() for css in csss) XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)
#静态类方法, @staticmethod def close(spider, reason): closed = getattr(spider, 'closed', None) if callable(closed): return closed(reason) #类似java的toString方法 def __str__(self): return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self)) __repr__ = __str__ #兼容以前的废弃类BaseSpider BaseSpider = create_deprecated_class('BaseSpider', Spider) class ObsoleteClass(object): def __init__(self, message): self.message = message def __getattr__(self, name): raise AttributeError(self.message) spiders = ObsoleteClass( '"from scrapy.spider import spiders" no longer works - use ' '"from scrapy.spiderloader import SpiderLoader" and instantiate ' 'it with your project settings"' ) # Top-level imports
elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) if 'method' not in kwargs: kwargs['method'] = 'POST' super().__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault( 'Accept', 'application/json, text/javascript, */*; q=0.01') def replace(self, *args, **kwargs): body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None if body_passed and data_passed: warnings.warn('Both body and data passed. data will be ignored') elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) return super().replace(*args, **kwargs) def _dumps(self, data): """Convert to JSON """ return json.dumps(data, **self._dumps_kwargs) JSONRequest = create_deprecated_class("JSONRequest", JsonRequest)
@deprecated(use_instead='._get_xpathvalues()') def _get_values(self, xpaths, **kw): return self._get_xpathvalues(xpaths, **kw) def _get_xpathvalues(self, xpaths, **kw): self._check_selector_method() xpaths = arg_to_iter(xpaths) return flatten( self.selector.xpath(xpath).extract() for xpath in xpaths) def add_css(self, field_name, css, *processors, **kw): values = self._get_cssvalues(css, **kw) self.add_value(field_name, values, *processors, **kw) def replace_css(self, field_name, css, *processors, **kw): values = self._get_cssvalues(css, **kw) self.replace_value(field_name, values, *processors, **kw) def get_css(self, css, *processors, **kw): values = self._get_cssvalues(css, **kw) return self.get_value(values, *processors, **kw) def _get_cssvalues(self, csss, **kw): self._check_selector_method() csss = arg_to_iter(csss) return flatten(self.selector.css(css).extract() for css in csss) XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)
'XPathSelectorList' ] def _xpathselector_css(self, *a, **kw): raise RuntimeError('.css() method not available for %s, ' 'instantiate scrapy.selector.Selector ' 'instead' % type(self).__name__) XPathSelector = create_deprecated_class( 'XPathSelector', Selector, { '__slots__': (), '_default_type': 'html', 'css': _xpathselector_css, }, new_class_path='scrapy.selector.Selector', old_class_path='scrapy.selector.XPathSelector', ) XmlXPathSelector = create_deprecated_class( 'XmlXPathSelector', XPathSelector, clsdict={ '__slots__': (), '_default_type': 'xml', }, new_class_path='scrapy.selector.Selector', old_class_path='scrapy.selector.XmlXPathSelector',