def get_job_list(self, project_name, spider_status=None):
data = request("get",
self._scrapyd_url() +
"/listjobs.json?project=%s" % project_name,
return_type="json")
result = {
SpiderStatus.PENDING: [],
SpiderStatus.RUNNING: [],
SpiderStatus.FINISHED: []
}
if data and data['status'] == 'ok':
for _status in self.spider_status_name_dict.keys():
for item in data[self.spider_status_name_dict[_status]]:
start_time, end_time = None, None
if item.get('start_time'):
start_time = datetime.datetime.strptime(
item['start_time'], '%Y-%m-%d %H:%M:%S.%f')
if item.get('end_time'):
end_time = datetime.datetime.strptime(
item['end_time'], '%Y-%m-%d %H:%M:%S.%f')
result[_status].append(
dict(id=item['id'],
start_time=start_time,
end_time=end_time))
return result if not spider_status else result[spider_status]
def cancel_spider(self, project_name, job_id):
post_data = dict(project=project_name, job=job_id)
data = request("post",
self._scrapyd_url() + "/cancel.json",
data=post_data,
return_type="json")
return data != None
def delete_project(self, project_name):
post_data = dict(project=project_name)
data = request("post",
self._scrapyd_url() + "/delproject.json",
data=post_data,
return_type="json")
return True if data and data['status'] == 'ok' else False
def get_project_list(self):
data = request("get",
self._scrapyd_url() + "/listprojects.json",
return_type="json")
result = []
if data:
for project_name in data['projects']:
project = Project()
project.project_name = project_name
result.append(project)
return result
def get_spider_list(self, project_name):
data = request("get",
self._scrapyd_url() +
"/listspiders.json?project=%s" % project_name,
return_type="json")
result = []
if data and data['status'] == 'ok':
for spider_name in data['spiders']:
spider_instance = SpiderInstance()
spider_instance.spider_name = spider_name
result.append(spider_instance)
return result
def start_spider(self, project_name, spider_name, arguments):
post_data = dict(project=project_name, spider=spider_name)
post_data.update(arguments)
data = request("post",
self._scrapyd_url() + "/schedule.json",
data=post_data,
return_type="json")
if data and data['status'] == 'ok':
return data['jobid']
else:
import time
time.sleep(3)
return self.start_spider(project_name, spider_name, arguments)
def start_spider(self, project_name, spider_name, arguments):
post_data = dict(project=project_name, spider=spider_name)
post_data.update(arguments)
data = request("post",
self._scrapyd_url() + "/schedule.json",
data=post_data,
return_type="json")
# app.logger.warn("The response from {} was {} after request with {}".format(self._scrapyd_url(), data['status'] if data else data.status_code, post_data))
if data and data['status'] == 'ok':
return data['jobid']
else:
import time
time.sleep(3)
return self.start_spider(project_name, spider_name, arguments)
