def msg(message, level=INFO, component=BOT_NAME, domain=None, spider=None):
"""Log message according to the level"""
if level > log_level:
return
if domain is not None:
import warnings
warnings.warn("'domain' argument of scrapy.log.msg() is deprecated, " \
"use 'spider' argument instead", DeprecationWarning, stacklevel=2)
dispatcher.send(signal=logmessage_received, message=message, level=level, \
spider=spider)
system = domain or (spider.domain_name if spider else component)
msg_txt = unicode_to_str("%s: %s" % (level_names[level], message))
log.msg(msg_txt, system=system)
def stats_spider_closed(self, spider):
if spider.enable_metadata:
self.exporter.finish_exporting()
f = self.files.pop(spider)
f.close()
self.move_local_data_file_to_nfs(spider)
if spider in self.unified_marketplace_files:
self.exporter_market.finish_exporting()
f = self.unified_marketplace_files.pop(spider)
f.close()
self.move_local_unified_marketplace_data_file_to_nfs(spider)
dispatcher.send('export_finished', None, spider,
self.stats.get_stats())
def modules(self):
if not hasattr(self, '_modules'):
self._modules = {}
for _, obj in dispatcher.send(signal=webconsole_discover_module, \
sender=self.__class__):
self._modules[obj.webconsole_id] = obj
return self._modules
def _spider_idle(self, spider):
"""Called when a spider gets idle. This function is called when there
are no remaining pages to download or schedule. It can be called
multiple times. If some extension raises a DontCloseSpider exception
(in the spider_idle signal handler) the spider is not closed until the
next loop and this function is guaranteed to be called (at least) once
again for this spider.
"""
try:
dispatcher.send(signal=signals.spider_idle, sender=self.__class__, \
spider=spider)
except DontCloseSpider:
reactor.callLater(5, self.next_request, spider)
return
except Exception, e:
log.msg("Exception caught on 'spider_idle' signal dispatch: %r" % e, \
level=log.ERROR)
#/bin/python
#from pydispatch import dispatcher
from scrapy.xlib.pydispatch import dispatcher
from signals import discount_signal
def handle_specific_event( sender, moo):
"""Handle a simple event, requiring a "moo" parameter"""
print 'Specialized event for %(sender)s moo=%(moo)r'%locals()
dispatcher.connect( handle_specific_event, signal=discount_signal, sender=dispatcher.Any, )
first_sender = object()
dispatcher.send( signal=discount_signal, sender=first_sender, moo={'price': 1, 'url': 'a.url'})
#/bin/python
#from pydispatch import dispatcher
from scrapy.xlib.pydispatch import dispatcher
from signals import discount_signal
def handle_specific_event(sender, moo):
"""Handle a simple event, requiring a "moo" parameter"""
print 'Specialized event for %(sender)s moo=%(moo)r' % locals()
dispatcher.connect(
handle_specific_event,
signal=discount_signal,
sender=dispatcher.Any,
)
first_sender = object()
dispatcher.send(signal=discount_signal,
sender=first_sender,
moo={
'price': 1,
'url': 'a.url'
})
