def spider(worker, job):
"Gearman entry point"
url = job.data
client = GearmanClient(['localhost'])
for href in url2hrefs(url): # for URL the content of that URL refers to,
(blah,ext) = path.splitext(href)
if ext != '.json':
href = href + '.json'
# submit a new spider job for that href
client.submit_job('spider',href,background=True,wait_until_complete=False)
return job.data
#!/usr/bin/python
from gearman.client import GearmanClient
client = GearmanClient(['localhost'])
URL = 'http://ifcb-data.whoi.edu/feed.json'
client.submit_job('spider', URL)
def __init__(self, config):
self.config = config
self.client = GearmanClient(
self.config.get('gearman', 'hosts').split(','))
