def delete_project(self, project_name): post_data = dict(project=project_name) data = request("post", self._scrapyd_url() + "/delproject.json", data=post_data, return_type="json") return True if data and data['status'] == 'ok' else False
def get_job_list(self, project_name, project_id, spider_status=None): data = request("get", self._scrapyd_url() + "/listjobs.json?project=%s" % project_name, return_type="json") result = { SpiderStatus.PENDING: [], SpiderStatus.RUNNING: [], SpiderStatus.FINISHED: [] } if data and data['status'] == 'ok': for _status in self.spider_status_name_dict.keys(): for item in data[self.spider_status_name_dict[_status]]: # create a new job execution for newly discovered spiders self.create_job_execution(item, project_id) start_time, end_time = None, None if item.get('start_time'): start_time = datetime.datetime.strptime( item['start_time'], '%Y-%m-%d %H:%M:%S.%f') if item.get('end_time'): end_time = datetime.datetime.strptime( item['end_time'], '%Y-%m-%d %H:%M:%S.%f') result[_status].append( dict(id=item['id'], start_time=start_time, end_time=end_time)) return result if not spider_status else result[spider_status]
def cancel_spider(self, project_name, job_id): post_data = dict(project=project_name, job=job_id) data = request("post", self._scrapyd_url() + "/cancel.json", data=post_data, return_type="json") return data != None
def start_spider(self, project_name, spider_name, arguments): post_data = dict(project=project_name, spider=spider_name) post_data.update(arguments) data = request("post", self._scrapyd_url() + "/schedule.json", data=post_data, return_type="json") return data['jobid'] if data and data['status'] == 'ok' else None
def get_project_list(self): data = request("get", self._scrapyd_url() + "/listprojects.json", return_type="json") result = [] if data: for project_name in data['projects']: project = Project() project.project_name = project_name result.append(project) return result
def get_spider_list(self, project_name): data = request("get", self._scrapyd_url() + "/listspiders.json?project=%s" % project_name, return_type="json") result = [] if data and data['status'] == 'ok': for spider_name in data['spiders']: spider_instance = SpiderInstance() spider_instance.spider_name = spider_name result.append(spider_instance) return result
def cancel_spider(self, project_name, job_id, signal=None): post_data = dict(project=project_name, job=job_id, signal='KILL') ''' if signal is not None: post_data['signal'] = signal ''' data = request("post", self._scrapyd_url() + "/cancel.json", data=post_data, return_type="json") return data is not None
def start_spider(self, project_name, spider_name, arguments): post_data = dict(project=project_name, spider=spider_name) post_data.update(arguments) post_data.update({ "setting": "CLOSESPIDER_TIMEOUT={}".format(app.config["CLOSE_TIME"]) }) data = request("post", self._scrapyd_url() + "/schedule.json", data=post_data, return_type="json") return data['jobid'] if data and data['status'] == 'ok' else None
def get_job_list(self, project_name, spider_status=None): data = request("get", self._scrapyd_url() + "/listjobs.json?project=%s" % project_name, return_type="json") result = {SpiderStatus.PENDING: [], SpiderStatus.RUNNING: [], SpiderStatus.FINISHED: []} if data and data['status'] == 'ok': for _status in self.spider_status_name_dict.keys(): for item in data[self.spider_status_name_dict[_status]]: start_time, end_time = None, None if item.get('start_time'): start_time = datetime.datetime.strptime(item['start_time'], '%Y-%m-%d %H:%M:%S.%f') if item.get('end_time'): end_time = datetime.datetime.strptime(item['end_time'], '%Y-%m-%d %H:%M:%S.%f') result[_status].append(dict(id=item['id'], start_time=start_time, end_time=end_time)) return result if not spider_status else result[spider_status]
def get_job_list(self, project_name, spider_status): data = request("get", self._scrapyd_url() + "/listjobs.json?project=%s" % project_name, return_type="json") result = [] if data and data['status'] == 'ok': for item in data[self.spider_status_name_dict[spider_status]]: start_time, end_time = None, None if item.get('start_time'): start_time = datetime.datetime.strptime( item['start_time'], '%Y-%m-%d %H:%M:%S.%f') if item.get('end_time'): end_time = datetime.datetime.strptime( item['end_time'], '%Y-%m-%d %H:%M:%S.%f') result.append( dict(id=item['id'], start_time=start_time, end_time=end_time)) return result
def cancel_spider(self, project_name, job_id): post_data = dict(project=project_name, job=job_id) data = request("post", self._scrapyd_url() + "/cancel.json", data=post_data, return_type="json") return data != None
def start_spider(self, project_name, spider_name, arguments): post_data = dict(project=project_name, spider=spider_name) post_data.update(arguments) data = request("post", self._scrapyd_url() + "/schedule.json", data=post_data, return_type="json") return data['jobid'] if data and data['status'] == 'ok' else None
def delete_project(self, project_name): post_data = dict(project=project_name) data = request("post", self._scrapyd_url() + "/delproject.json", data=post_data, return_type="json") return True if data and data['status'] == 'ok' else False