class PttCrawlerJob():
def __init__(self, crawl_url):
""" Initialize and build a connection with Scrapinghub via its api
"""
self._client = ScrapinghubClient(settings.SCRAPINGHUB_APIKEY)
# TODO: need to be revised
self._project_id = self._client.projects.list()[0]
self._project = self._client.get_project(self._project_id)
self._target = crawl_url
self._job = None
self._meta = None
self._state = 'initialized'
def run(self):
""" Run the crawler (spider)
"""
if not self._job:
self._job = self._project.jobs.run(
'ptt', job_args={'test_url': self._target})
return self._job.key
else:
return None
def update_meta(self):
""" Update job's meta data
"""
if self._job:
self._meta = dict(self._job.metadata.iter())
self._state = self._meta['state']
def cancle(self):
""" Cancle a job
"""
self._job.cancel()
@property
def meta(self):
""" Get job's meta data
"""
if self._meta:
return self._meta
else:
return None
@property
def state(self):
""" Get job's current state
"""
return self._state
@property
def item(self):
""" Get scrapped items
"""
if self._state == 'finished':
# items.iter() returns a iterable, but is not a list and does not support indexing
# so it has to be transformed into a list. Each element in the list is a dict.
return list(self._job.items.iter())
else:
return None
def __init__(self, input_uri, settings):
super().__init__(settings)
client = ScrapinghubClient()
jobkey = parse_job_key(os.environ['SHUB_JOBKEY'])
project = client.get_project(jobkey.project_id)
collection_name = input_uri.replace('collections://', '')
self._store = project.collections.get_store(collection_name)
def jobRuning00():
# Enter ScrapingHub
# Enter ScrapingHub
apikey = '40f9881d52794d7bb09b9f5ee6d12a3e' # your API key as a string
client = ScrapinghubClient(apikey)
projectID = 410647
project = client.get_project(projectID)
# get spider
spiderID = 'quotes'
spider = project.spiders.get(spiderID)
spider.jobs.run()
def schedule_spider(project, endpoint, apikey, spider, arguments=(), settings=(),
priority=DEFAULT_PRIORITY, units=None, tag=(), environment=()):
client = ScrapinghubClient(apikey, dash_endpoint=endpoint)
try:
project = client.get_project(project)
args = dict(x.split('=', 1) for x in arguments)
cmd_args = args.pop('cmd_args', None)
meta = args.pop('meta', None)
job = project.jobs.run(
spider=spider,
meta=json.loads(meta) if meta else {},
cmd_args=cmd_args,
job_args=args,
job_settings=dict(x.split('=', 1) for x in settings),
priority=priority,
units=units,
add_tag=tag,
environment=dict(x.split('=', 1) for x in environment),
)
return job.key
except ScrapinghubAPIError as e:
raise RemoteErrorException(str(e))
class SHConnection():
''' Wrapper for scrapinghub client, project and api calls
to simplify use.
'''
def __init__(self, api_key, default_project_key=None):
self.api_key = api_key
self.project_key = resolve_project_key(
default_project_key=default_project_key
)
def __enter__(self):
self.client = ScrapinghubClient(self.api_key)
self.project = self.client.get_project(self.project_key)
return self
def __exit__(self, *args):
self.client.close()
def jobs_iter(self, **kwargs):
return self.project.jobs.iter(**kwargs)
def get_job(self, job_id):
return self.client.get_job(job_id)