def __init__(self):
if 'SHUB_JOB' not in os.environ:
raise NotConfigured
auth = os.environ['SHUB_JOBAUTH']
apiurl = os.environ['SHUB_STORAGE']
client = Client(auth, url=apiurl)
path = "/items/%(SHUB_PROJECT)s/%(SHUB_SPIDER)s/%(SHUB_JOB)s" % os.environ
self.writer = client.open_item_writer(path)
log.msg("HubStorage: writing items to %s" % urlparse.urljoin(apiurl, path))
dispatcher.connect(self.item_scraped, signals.item_scraped)
dispatcher.connect(self.spider_closed, signals.spider_closed)
def main():
opts, path = parse_opts()
logging.basicConfig(level=getattr(logging, opts.loglevel))
client = Client(opts.apikey, url=opts.server)
if opts.load:
with client.open_item_writer(path) as writer:
for line in sys.stdin:
writer.write_json_item(line.strip())
elif opts.dump:
for jsonitem in client.iter_json_items(path):
print jsonitem.rstrip()
def start(self):
client = Client(self.auth, url=self.url)
self.writer = client.open_item_writer(self.path)
txlog.startLoggingWithObserver(self.emit, setStdout=False)
from twisted.internet import reactor
reactor.addSystemEventTrigger('after', 'shutdown', self.stop)