def get_project_list(self):
data = request("get", self._scrapyd_url() + "/listprojects.json", return_type="json")
result = []
if data:
for project_name in data['projects']:
project = Project()
project.project_name = project_name
result.append(project)
return result
def get_spider_list(self, project_name):
data = request("get", self._scrapyd_url() + "/listspiders.json?project=%s" % project_name,
return_type="json")
result = []
if data and data['status'] == 'ok':
for spider_name in data['spiders']:
spider_instance = SpiderInstance()
spider_instance.spider_name = spider_name
result.append(spider_instance)
return result
def get_job_list(self, project_name, spider_status=None):
data = request("get", self._scrapyd_url() + "/listjobs.json?project=%s" % project_name,
return_type="json")
result = {SpiderStatus.PENDING: [], SpiderStatus.RUNNING: [], SpiderStatus.FINISHED: []}
if data and data['status'] == 'ok':
for _status in self.spider_status_name_dict.keys():
for item in data[self.spider_status_name_dict[_status]]:
start_time, end_time = None, None
if item.get('start_time'):
start_time = datetime.datetime.strptime(item['start_time'], '%Y-%m-%d %H:%M:%S.%f')
if item.get('end_time'):
end_time = datetime.datetime.strptime(item['end_time'], '%Y-%m-%d %H:%M:%S.%f')
result[_status].append(dict(id=item['id'], start_time=start_time, end_time=end_time))
return result if not spider_status else result[spider_status]
def cancel_spider(self, project_name, job_id):
post_data = dict(project=project_name, job=job_id)
data = request("post", self._scrapyd_url() + "/cancel.json", data=post_data, return_type="json")
return data != None
def start_spider(self, project_name, spider_name, arguments):
post_data = dict(project=project_name, spider=spider_name)
post_data.update(arguments)
data = request("post", self._scrapyd_url() + "/schedule.json", data=post_data, return_type="json")
return data['jobid'] if data and data['status'] == 'ok' else None
def delete_project(self, project_name):
post_data = dict(project=project_name)
data = request("post", self._scrapyd_url() + "/delproject.json", data=post_data, return_type="json")
return True if data and data['status'] == 'ok' else False