def delete_project(self, project_name):
post_data = dict(project=project_name)
data = request("post",
self._scrapyd_url() + "/delproject.json",
data=post_data,
return_type="json")
return True if data and data['status'] == 'ok' else False
def get_job_list(self, project_name, project_id, spider_status=None):
data = request("get",
self._scrapyd_url() +
"/listjobs.json?project=%s" % project_name,
return_type="json")
result = {
SpiderStatus.PENDING: [],
SpiderStatus.RUNNING: [],
SpiderStatus.FINISHED: []
}
if data and data['status'] == 'ok':
for _status in self.spider_status_name_dict.keys():
for item in data[self.spider_status_name_dict[_status]]:
# create a new job execution for newly discovered spiders
self.create_job_execution(item, project_id)
start_time, end_time = None, None
if item.get('start_time'):
start_time = datetime.datetime.strptime(
item['start_time'], '%Y-%m-%d %H:%M:%S.%f')
if item.get('end_time'):
end_time = datetime.datetime.strptime(
item['end_time'], '%Y-%m-%d %H:%M:%S.%f')
result[_status].append(
dict(id=item['id'],
start_time=start_time,
end_time=end_time))
return result if not spider_status else result[spider_status]
def cancel_spider(self, project_name, job_id):
post_data = dict(project=project_name, job=job_id)
data = request("post",
self._scrapyd_url() + "/cancel.json",
data=post_data,
return_type="json")
return data != None
def start_spider(self, project_name, spider_name, arguments):
post_data = dict(project=project_name, spider=spider_name)
post_data.update(arguments)
data = request("post",
self._scrapyd_url() + "/schedule.json",
data=post_data,
return_type="json")
return data['jobid'] if data and data['status'] == 'ok' else None
def get_project_list(self):
data = request("get", self._scrapyd_url() + "/listprojects.json", return_type="json")
result = []
if data:
for project_name in data['projects']:
project = Project()
project.project_name = project_name
result.append(project)
return result
def get_spider_list(self, project_name):
data = request("get", self._scrapyd_url() + "/listspiders.json?project=%s" % project_name,
return_type="json")
result = []
if data and data['status'] == 'ok':
for spider_name in data['spiders']:
spider_instance = SpiderInstance()
spider_instance.spider_name = spider_name
result.append(spider_instance)
return result
def cancel_spider(self, project_name, job_id, signal=None):
post_data = dict(project=project_name, job=job_id, signal='KILL')
'''
if signal is not None:
post_data['signal'] = signal
'''
data = request("post",
self._scrapyd_url() + "/cancel.json",
data=post_data,
return_type="json")
return data is not None
def start_spider(self, project_name, spider_name, arguments):
post_data = dict(project=project_name, spider=spider_name)
post_data.update(arguments)
post_data.update({
"setting":
"CLOSESPIDER_TIMEOUT={}".format(app.config["CLOSE_TIME"])
})
data = request("post",
self._scrapyd_url() + "/schedule.json",
data=post_data,
return_type="json")
return data['jobid'] if data and data['status'] == 'ok' else None
def get_job_list(self, project_name, spider_status=None):
data = request("get", self._scrapyd_url() + "/listjobs.json?project=%s" % project_name,
return_type="json")
result = {SpiderStatus.PENDING: [], SpiderStatus.RUNNING: [], SpiderStatus.FINISHED: []}
if data and data['status'] == 'ok':
for _status in self.spider_status_name_dict.keys():
for item in data[self.spider_status_name_dict[_status]]:
start_time, end_time = None, None
if item.get('start_time'):
start_time = datetime.datetime.strptime(item['start_time'], '%Y-%m-%d %H:%M:%S.%f')
if item.get('end_time'):
end_time = datetime.datetime.strptime(item['end_time'], '%Y-%m-%d %H:%M:%S.%f')
result[_status].append(dict(id=item['id'], start_time=start_time, end_time=end_time))
return result if not spider_status else result[spider_status]
def get_job_list(self, project_name, spider_status):
data = request("get",
self._scrapyd_url() +
"/listjobs.json?project=%s" % project_name,
return_type="json")
result = []
if data and data['status'] == 'ok':
for item in data[self.spider_status_name_dict[spider_status]]:
start_time, end_time = None, None
if item.get('start_time'):
start_time = datetime.datetime.strptime(
item['start_time'], '%Y-%m-%d %H:%M:%S.%f')
if item.get('end_time'):
end_time = datetime.datetime.strptime(
item['end_time'], '%Y-%m-%d %H:%M:%S.%f')
result.append(
dict(id=item['id'],
start_time=start_time,
end_time=end_time))
return result
def cancel_spider(self, project_name, job_id):
post_data = dict(project=project_name, job=job_id)
data = request("post", self._scrapyd_url() + "/cancel.json", data=post_data, return_type="json")
return data != None
def start_spider(self, project_name, spider_name, arguments):
post_data = dict(project=project_name, spider=spider_name)
post_data.update(arguments)
data = request("post", self._scrapyd_url() + "/schedule.json", data=post_data, return_type="json")
return data['jobid'] if data and data['status'] == 'ok' else None
def delete_project(self, project_name):
post_data = dict(project=project_name)
data = request("post", self._scrapyd_url() + "/delproject.json", data=post_data, return_type="json")
return True if data and data['status'] == 'ok' else False
