def msg(message, level=INFO, component=BOT_NAME, domain=None, spider=None): """Log message according to the level""" if level > log_level: return if domain is not None: import warnings warnings.warn("'domain' argument of scrapy.log.msg() is deprecated, " \ "use 'spider' argument instead", DeprecationWarning, stacklevel=2) dispatcher.send(signal=logmessage_received, message=message, level=level, \ spider=spider) system = domain or (spider.domain_name if spider else component) msg_txt = unicode_to_str("%s: %s" % (level_names[level], message)) log.msg(msg_txt, system=system)
def stats_spider_closed(self, spider): if spider.enable_metadata: self.exporter.finish_exporting() f = self.files.pop(spider) f.close() self.move_local_data_file_to_nfs(spider) if spider in self.unified_marketplace_files: self.exporter_market.finish_exporting() f = self.unified_marketplace_files.pop(spider) f.close() self.move_local_unified_marketplace_data_file_to_nfs(spider) dispatcher.send('export_finished', None, spider, self.stats.get_stats())
def modules(self): if not hasattr(self, '_modules'): self._modules = {} for _, obj in dispatcher.send(signal=webconsole_discover_module, \ sender=self.__class__): self._modules[obj.webconsole_id] = obj return self._modules
def _spider_idle(self, spider): """Called when a spider gets idle. This function is called when there are no remaining pages to download or schedule. It can be called multiple times. If some extension raises a DontCloseSpider exception (in the spider_idle signal handler) the spider is not closed until the next loop and this function is guaranteed to be called (at least) once again for this spider. """ try: dispatcher.send(signal=signals.spider_idle, sender=self.__class__, \ spider=spider) except DontCloseSpider: reactor.callLater(5, self.next_request, spider) return except Exception, e: log.msg("Exception caught on 'spider_idle' signal dispatch: %r" % e, \ level=log.ERROR)
#/bin/python #from pydispatch import dispatcher from scrapy.xlib.pydispatch import dispatcher from signals import discount_signal def handle_specific_event( sender, moo): """Handle a simple event, requiring a "moo" parameter""" print 'Specialized event for %(sender)s moo=%(moo)r'%locals() dispatcher.connect( handle_specific_event, signal=discount_signal, sender=dispatcher.Any, ) first_sender = object() dispatcher.send( signal=discount_signal, sender=first_sender, moo={'price': 1, 'url': 'a.url'})
#/bin/python #from pydispatch import dispatcher from scrapy.xlib.pydispatch import dispatcher from signals import discount_signal def handle_specific_event(sender, moo): """Handle a simple event, requiring a "moo" parameter""" print 'Specialized event for %(sender)s moo=%(moo)r' % locals() dispatcher.connect( handle_specific_event, signal=discount_signal, sender=dispatcher.Any, ) first_sender = object() dispatcher.send(signal=discount_signal, sender=first_sender, moo={ 'price': 1, 'url': 'a.url' })