Ejemplo n.º 1
0
    def render_POST(self, txrequest):
        project = txrequest.args['project'][0]

        check_db_connection()
        finished = [{
            'job_id': s.job,
            'project': s.project,
            'spider': s.spider,
            'start_time': s.start_time,
            'end_time': s.end_time
        } for s in self.root.launcher.finished
                    if s.project == project or project == 'all']

        failed_items = []
        for fj in finished:
            item = JobItem()
            for key in fj.keys():
                item[key] = fj.get(key)
            if not item.save_only():
                failed_items.append(item['job_id'])

        if failed_items:
            return {"node_name": self.root.nodename, "status":"error", \
                    "message": "%d items failed" % len(failed_items)}

        backup = []
        for s in self.root.launcher.finished:
            if project != 'all' and s.project != project:
                backup.append(s)
        del self.root.launcher.finished
        self.root.launcher.finished = backup
        return {"node_name": self.root.nodename, "status": "ok"}
Ejemplo n.º 2
0
    def render_POST(self, txrequest):
        project = txrequest.args['project'][0]

        check_db_connection()
        finished = [{'job_id': s.job, 'project': s.project, 'spider': s.spider,
                     'start_time': s.start_time, 'end_time': s.end_time}
                     for s in self.root.launcher.finished if s.project == project or project == 'all']

        failed_items = []
        for fj in finished:
            item = JobItem()
            for key in fj.keys():
                item[key] = fj.get(key)
            if not item.save_only():
                failed_items.append(item['job_id'])

        if failed_items:
            return {"node_name": self.root.nodename, "status":"error", \
                    "message": "%d items failed" % len(failed_items)}

        backup = []
        for s in self.root.launcher.finished:
            if project != 'all' and s.project != project:
                backup.append(s)
        del self.root.launcher.finished
        self.root.launcher.finished = backup
        return {"node_name": self.root.nodename, "status":"ok"}
Ejemplo n.º 3
0
    def render_GET(self, txrequest):
        project = txrequest.args['project'][0]
        spider = txrequest.args['spider'][0]
        page_id = int(txrequest.args['page_id'][0])
        page_count = int(txrequest.args['page_count'][0])

        check_db_connection()
        cnt, qs = JobItem.get_jobs_by_project(project, spider, page_id, page_count)
        finished = [{'id': j.job_id, 'project': j.project, 'spider': j.spider, 'start_time': j.start_time.isoformat(' '),
                     'end_time': j.end_time.isoformat(' ')} for j in qs]
        return {"node_name": self.root.nodename, "status":"ok", "count": cnt, "finished": finished}
Ejemplo n.º 4
0
    def render_GET(self, txrequest):
        project = txrequest.args['project'][0]
        spider = txrequest.args['spider'][0]
        page_id = int(txrequest.args['page_id'][0])
        page_count = int(txrequest.args['page_count'][0])

        check_db_connection()
        cnt, qs = JobItem.get_jobs_by_project(project, spider, page_id, page_count)
        finished = [{'id': j.job_id, 'project': j.project, 'spider': j.spider, 'start_time': j.start_time.isoformat(' '),
                     'end_time': j.end_time.isoformat(' ')} for j in qs]
        return {"node_name": self.root.nodename, "status":"ok", "count": cnt, "finished": finished}