def __init__(self):
     if 'SHUB_JOB' not in os.environ:
         raise NotConfigured
     auth = os.environ['SHUB_JOBAUTH']
     apiurl = os.environ['SHUB_STORAGE']
     client = Client(auth, url=apiurl)
     path = "/items/%(SHUB_PROJECT)s/%(SHUB_SPIDER)s/%(SHUB_JOB)s" % os.environ
     self.writer = client.open_item_writer(path)
     log.msg("HubStorage: writing items to %s" % urlparse.urljoin(apiurl, path))
     dispatcher.connect(self.item_scraped, signals.item_scraped)
     dispatcher.connect(self.spider_closed, signals.spider_closed)
Пример #2
0
def main():
    opts, path = parse_opts()
    logging.basicConfig(level=getattr(logging, opts.loglevel))
    client = Client(opts.apikey, url=opts.server)
    if opts.load:
        with client.open_item_writer(path) as writer:
            for line in sys.stdin:
                writer.write_json_item(line.strip())
    elif opts.dump:
        for jsonitem in client.iter_json_items(path):
            print jsonitem.rstrip()
Пример #3
0
 def start(self):
     client = Client(self.auth, url=self.url)
     self.writer = client.open_item_writer(self.path)
     txlog.startLoggingWithObserver(self.emit, setStdout=False)
     from twisted.internet import reactor
     reactor.addSystemEventTrigger('after', 'shutdown', self.stop)