Esempio n. 1
0
    def urls_add(self):
        add_url_list = list(set(self.__request_json['urls_add']))  # 去重

        # 已存在queue中的
        exist_queue_url_list = []
        res = Mongo.get()['queue_' + self.project_name].find({'url_md5': {'$in': [md5(l) for l in add_url_list]}},
                                                             {'url': 1})

        for doc in res:
            exist_queue_url_list.append(doc['url'])

        # 已存在parsed中的
        exist_parsed_url_list = []

        res = Mongo.get()['parsed_' + self.project_name].find({'url_md5': {'$in': [md5(l) for l in add_url_list]}},
                                                              {'url': 1})
        for doc in res:  # todo 需要判断存在的时间, 允许重复抓取
            exist_parsed_url_list.append(doc['url'])

        # 加入队列
        add_urls_data = []
        for url in add_url_list:
            if url not in exist_queue_url_list and url not in exist_parsed_url_list:  # 不存在queue不存在parsed中才加入队列
                add_urls_data.append(
                    {'domain': get_domain(url),
                     'url': url,
                     'url_md5': md5(url),
                     'flag_time': 0,
                     'add_time': int(time.time()),
                     'slave_ip': self.__request_address[0]})

        add_urls_data and Mongo.get()['queue_' + self.project_name].insert(add_urls_data)
Esempio n. 2
0
def toggle_project(project_id):
    try:
        project = Mongo.get()["projects"].find_one({"_id": ObjectId(project_id)})
        if not project:
            raise Exception("不存在的记录!")
    except:
        return jsonify({"success": False, "msg": "不存在的记录!"})

    project["static"] = "抓取中" if project["static"] == "暂停中" else "暂停中"

    Mongo.get()["projects"].update({"_id": ObjectId(project_id)}, {"$set": {"static": project["static"]}})
    return jsonify({"success": True, "msg": "切换成功!"})
Esempio n. 3
0
def get_projects():
    project_dict = {}
    for project in get_project_list():
        project_dict[project['name']] = {
            '_id': str(project['_id']),
            'name': project['name'],
            'static': project['static'],
            'queue_len': Mongo.get()['queue_' + project['name']].count(),
            'parsed_len': Mongo.get()['parsed_' + project['name']].count(),
            'result_len': Mongo.get()['result_' + project['name']].count(),
        }

    return jsonify(project_dict)
Esempio n. 4
0
def get_projects():
    project_dict = {}
    for project in get_project_list():
        project_dict[project["name"]] = {
            "_id": str(project["_id"]),
            "name": project["name"],
            "static": project["static"],
            "queue_len": Mongo.get()["queue_" + project["name"]].count(),
            "parsed_len": Mongo.get()["parsed_" + project["name"]].count(),
            "result_len": Mongo.get()["result_" + project["name"]].count(),
        }

    return jsonify(project_dict)
Esempio n. 5
0
def toggle_project(project_id):
    try:
        project = Mongo.get()['projects'].find_one({'_id': ObjectId(project_id)})
        if not project:
            raise Exception('不存在的记录!')
    except:
        return jsonify({'success': False, 'msg': '不存在的记录!'})

    project['static'] = '抓取中' if project['static'] == '暂停中' else '暂停中'

    Mongo.get()['projects'].update({'_id': ObjectId(project_id)},
                                   {'$set': {'static': project['static']}})
    return jsonify({'success': True, 'msg': '切换成功!'})
Esempio n. 6
0
def get_projects():
    project_dict = {}
    for project in get_project_list():
        project_dict[project['name']] = {
            '_id': str(project['_id']),
            'name': project['name'],
            'static': project['static'],
            'queue_len': Mongo.get()['queue_' + project['name']].count(),
            'parsed_len': Mongo.get()['parsed_' + project['name']].count(),
            'result_len': Mongo.get()['result_' + project['name']].count(),
        }

    return jsonify(project_dict)
Esempio n. 7
0
def get_project_tasks(project_name):
    res = []
    for doc in Mongo.get()['parsed_' + project_name].find().sort('_id', -1).limit(100):
        del doc['_id']
        doc['slave_ip'] = mix_ip(doc['slave_ip'])
        res.append(doc)
    return json.dumps(res)
Esempio n. 8
0
def toggle_project(project_id):
    try:
        project = Mongo.get()['projects'].find_one(
            {'_id': ObjectId(project_id)})
        if not project:
            raise Exception('不存在的记录!')
    except:
        return jsonify({'success': False, 'msg': '不存在的记录!'})

    project['static'] = '抓取中' if project['static'] == '暂停中' else '暂停中'

    Mongo.get()['projects'].update({'_id': ObjectId(project_id)},
                                   {'$set': {
                                       'static': project['static']
                                   }})
    return jsonify({'success': True, 'msg': '切换成功!'})
Esempio n. 9
0
def get_project_tasks(project_name):
    res = []
    for doc in Mongo.get()["parsed_" + project_name].find().sort("_id", -1).limit(100):
        del doc["_id"]
        doc["slave_ip"] = mix_ip(doc["slave_ip"])
        res.append(doc)
    return json.dumps(res)
Esempio n. 10
0
def save_project():
    return jsonify({'success': False, 'msg': '禁止修改!'})
    form_data = json.loads(request.data)  # todo 需要验证表单数据

    name_r = re.compile(r'^[0-9a-zA-Z_-]+$')
    if not name_r.search(form_data['name']):
        return jsonify({'success': False, 'msg': '计划名称必须满足正则规则: ^[0-9a-zA-Z_-]+$ '})

    exists_project = list(Mongo.get()['projects'].find({'name': form_data['name']}, {'_id': 1, 'add_time': 1}).limit(1))

    if 'edit' not in form_data and exists_project:
        return jsonify({'success': False, 'msg': '计划名称已经存在!'})

    # 新增计划或更新计划
    data = {
        'name': form_data['name'],
        'init_url': form_data['init_url'],
        'desc': form_data['desc'] if 'desc' in form_data else '',
        'code': form_data['code'],
        'static': '暂停中',
        'update_time': int(time.time()),
        'add_time': exists_project[0]['add_time'] if exists_project else int(time.time()),
    }
    Mongo.get()['projects'].update({'name': form_data['name']}, data, True)

    # 当是新计划时的初始化
    if 'edit' not in form_data:
        Mongo.get()['queue_' + form_data['name']].insert(
            {
                'url': form_data['init_url'],
                'url_md5': md5(form_data['init_url']),
                'flag_time': 0,
                'add_time': int(time.time()),
                'slave_ip': '0.0.0.0'
            })

        # 在没创建集合前设置索引mongodb会自动创建该集合并赋索引
        Mongo.get()['parsed_' + form_data['name']].ensure_index('url_md5', unique=True)
        Mongo.get()['queue_' + form_data['name']].ensure_index('url_md5', unique=True)

        # 有新计划加入, 重启全部slave
        restart_slave_list = GlobalHelper.get('restart_slave_list') or []
        for slave_record in Mongo.get()['slave_record'].find():
            restart_slave_list.append(slave_record['ip'])
        GlobalHelper.set('restart_slave_list', list(set(restart_slave_list)))

    return jsonify({'success': True, 'msg': '保存成功!'})
Esempio n. 11
0
def save_project():
    form_data = json.loads(request.data)  # todo 需要验证表单数据

    name_r = re.compile(r"^[0-9a-zA-Z_-]+$")
    if not name_r.search(form_data["name"]):
        return jsonify({"success": False, "msg": "计划名称必须满足正则规则: ^[0-9a-zA-Z_-]+$ "})

    exists_project = list(Mongo.get()["projects"].find({"name": form_data["name"]}, {"_id": 1, "add_time": 1}).limit(1))

    if "edit" not in form_data and exists_project:
        return jsonify({"success": False, "msg": "计划名称已经存在!"})

    # 新增计划或更新计划
    data = {
        "name": form_data["name"],
        "init_url": form_data["init_url"],
        "desc": form_data["desc"] if "desc" in form_data else "",
        "code": form_data["code"],
        "static": "暂停中",
        "update_time": int(time.time()),
        "add_time": exists_project[0]["add_time"] if exists_project else int(time.time()),
    }
    Mongo.get()["projects"].update({"name": form_data["name"]}, data, True)

    # 当是新计划时的初始化
    if "edit" not in form_data:
        Mongo.get()["queue_" + form_data["name"]].insert(
            {
                "url": form_data["init_url"],
                "url_md5": md5(form_data["init_url"]),
                "flag_time": 0,
                "add_time": int(time.time()),
                "slave_ip": "0.0.0.0",
            }
        )

        # 在没创建集合前设置索引mongodb会自动创建该集合并赋索引
        Mongo.get()["parsed_" + form_data["name"]].ensure_index("url_md5", unique=True)
        Mongo.get()["queue_" + form_data["name"]].ensure_index("url_md5", unique=True)

        # 有新计划加入, 重启全部slave
        restart_slave_list = GlobalHelper.get("restart_slave_list") or []
        for slave_record in Mongo.get()["slave_record"].find():
            restart_slave_list.append(slave_record["ip"])
        GlobalHelper.set("restart_slave_list", list(set(restart_slave_list)))

    return jsonify({"success": True, "msg": "保存成功!"})
Esempio n. 12
0
    def __storage_record(self):
        for ip, data in self.slave_record.items():
            res = Mongo.get().slave_record.update(
                {'ip': ip},
                {'ip': ip, 'data': data}, True)  # 有着更新, 无则插入

            if not res['updatedExisting'] and 'upserted' in res:  # 插入时
                self.slave_record[ip]['_id'] = str(res['upserted'])
Esempio n. 13
0
def get_project_tasks(project_name):
    res = []
    for doc in Mongo.get()['parsed_' + project_name].find().sort(
            '_id', -1).limit(100):
        del doc['_id']
        doc['slave_ip'] = mix_ip(doc['slave_ip'])
        res.append(doc)
    return json.dumps(res)
Esempio n. 14
0
 def __init__(self, project_name, request_json, request_address):
     self.project_name = project_name
     temp = list(
         Mongo.get()['projects'].find({'name': self.project_name}).limit(1))
     self.project = temp[0] if temp else {}
     self.__request_json = request_json
     self.__request_address = request_address
     self.__slave_record = SlaveRecord.get_instance()
Esempio n. 15
0
def get_project_list():
    """
    todo 测试是否支持中文
    :return:
    """
    # get_project_list.cache = [v for v in Mongo.get().projects.find({}, {'_id': 0})]
    get_project_list.cache = [v for v in Mongo.get().projects.find()]

    return get_project_list.cache
Esempio n. 16
0
def get_json_results(project_name):
    res = make_response(
        json.dumps(
            list(Mongo.get()['result_' + project_name].find({}, {'_id': 0}))),
        200)
    # header("Content-Disposition:attachment;filename={$fileName}");
    res.headers['Content-Disposition'] = 'attachment'
    res.headers['filename'] = project_name + '/json'
    return res
Esempio n. 17
0
def get_slave_tasks(slave_id):
    res = []
    try:
        slave_record = Mongo.get()['slave_record'].find_one({'_id': ObjectId(slave_id)})
        if not slave_record:
            raise Exception('不存在的记录!')
    except:
        return json.dumps(res)

    for project in get_project_list():
        for doc in Mongo.get()['parsed_' + project['name']].find({'slave_ip': slave_record['ip']}).sort('_id',
                                                                                                        -1).limit(20):
            del doc['_id']
            doc['slave_ip'] = mix_ip(doc['slave_ip'])
            res.append(doc)

    res.sort(key=lambda x: x['add_time'], reverse=True)
    return json.dumps(res)
Esempio n. 18
0
def toggle_slave(slave_id):
    try:
        slave_record = Mongo.get()['slave_record'].find_one({'_id': ObjectId(slave_id)})
        if not slave_record:
            raise Exception('不存在的记录!')
    except:
        return jsonify({'success': False, 'msg': '不存在的记录!'})

    slave_record['data']['static'] = '抓取中' if slave_record['data']['static'] == '暂停中' else '暂停中'
    try:
        Mongo.get()['slave_record'].update({'_id': ObjectId(slave_id)},
                                           {'$set': {'data.static': slave_record['data']['static']}})
        global_salve_record = GlobalHelper.get('salve_record')
        global_salve_record[slave_record['ip']]['static'] = slave_record['data']['static']
        GlobalHelper.set('salve_record', global_salve_record)
    except:
        print traceback.format_exc()
    return jsonify({'success': True, 'msg': '切换成功!'})
Esempio n. 19
0
def get_results(project_name, page=1):
    res = []
    pagination = paginate(Mongo.get()['result_' + project_name].find().sort('_id', -1), page, 20)
    if pagination:
        for doc in pagination.result():
            del doc['_id']
            res.append(doc)

    return json.dumps({'result': res, 'render_json': pagination.render_json(10)})
Esempio n. 20
0
def get_results(project_name, page=1):
    res = []
    pagination = paginate(Mongo.get()["result_" + project_name].find().sort("_id", -1), page, 20)
    if pagination:
        for doc in pagination.result():
            del doc["_id"]
            res.append(doc)

    return json.dumps({"result": res, "render_json": pagination.render_json(10)})
Esempio n. 21
0
    def __storage_record(self):
        for ip, data in self.slave_record.items():
            res = Mongo.get().slave_record.update({'ip': ip}, {
                'ip': ip,
                'data': data
            }, True)  # 有着更新, 无则插入

            if not res['updatedExisting'] and 'upserted' in res:  # 插入时
                self.slave_record[ip]['_id'] = str(res['upserted'])
Esempio n. 22
0
 def code_ctrl(self):
     result = []
     for project in Mongo.get().projects.find({}, {
             '_id': 0,
             'code': 1,
             'name': 1,
             'init_url': 1
     }):
         result.append(project)
     return result
Esempio n. 23
0
def toggle_slave(slave_id):
    try:
        slave_record = Mongo.get()["slave_record"].find_one({"_id": ObjectId(slave_id)})
        if not slave_record:
            raise Exception("不存在的记录!")
    except:
        return jsonify({"success": False, "msg": "不存在的记录!"})

    slave_record["data"]["static"] = "抓取中" if slave_record["data"]["static"] == "暂停中" else "暂停中"
    try:
        Mongo.get()["slave_record"].update(
            {"_id": ObjectId(slave_id)}, {"$set": {"data.static": slave_record["data"]["static"]}}
        )
        global_salve_record = GlobalHelper.get("salve_record")
        global_salve_record[slave_record["ip"]]["static"] = slave_record["data"]["static"]
        GlobalHelper.set("salve_record", global_salve_record)
    except:
        print traceback.format_exc()
    return jsonify({"success": True, "msg": "切换成功!"})
Esempio n. 24
0
def get_slave_tasks(slave_id):
    res = []
    try:
        slave_record = Mongo.get()["slave_record"].find_one({"_id": ObjectId(slave_id)})
        if not slave_record:
            raise Exception("不存在的记录!")
    except:
        return json.dumps(res)

    for project in get_project_list():
        for doc in (
            Mongo.get()["parsed_" + project["name"]].find({"slave_ip": slave_record["ip"]}).sort("_id", -1).limit(20)
        ):
            del doc["_id"]
            doc["slave_ip"] = mix_ip(doc["slave_ip"])
            res.append(doc)

    res.sort(key=lambda x: x["add_time"], reverse=True)
    return json.dumps(res)
Esempio n. 25
0
    def get_urls(self):
        # SerHandle.__init_project(self.project_name)
        response_url_list = []
        ids = []
        # print self.__slave_record.slave_record[self.__request_address[0]]['deny_domains']

        # todo need to test
        deny_domains = [x['domain'] for x in
                        self.__slave_record.slave_record[self.__request_address[0]]['deny_domains']]

        for doc in Mongo.get()['queue_' + self.project_name].find(
                {'domain': {'$nin': deny_domains}, 'flag_time': {'$lt': int(time.time() - 300)}}).limit(3) \
                .sort('_id', pymongo.ASCENDING):  # 取标识时间早于当前时间300秒之前的url
            ids.append(doc['_id'])
            response_url_list.append(doc['url'])

        ids and Mongo.get()['queue_' + self.project_name].update({'_id': {'$in': ids}},
                                                                 {'$set': {'flag_time': int(time.time())}},
                                                                 multi=True)
        return response_url_list
Esempio n. 26
0
def get_slave_tasks(slave_id):
    res = []
    try:
        slave_record = Mongo.get()['slave_record'].find_one(
            {'_id': ObjectId(slave_id)})
        if not slave_record:
            raise Exception('不存在的记录!')
    except:
        return json.dumps(res)

    for project in get_project_list():
        for doc in Mongo.get()['parsed_' + project['name']].find({
                'slave_ip':
                slave_record['ip']
        }).sort('_id', -1).limit(20):
            del doc['_id']
            doc['slave_ip'] = mix_ip(doc['slave_ip'])
            res.append(doc)

    res.sort(key=lambda x: x['add_time'], reverse=True)
    return json.dumps(res)
Esempio n. 27
0
def get_results(project_name, page=1):
    res = []
    pagination = paginate(
        Mongo.get()['result_' + project_name].find().sort('_id', -1), page, 20)
    if pagination:
        for doc in pagination.result():
            del doc['_id']
            res.append(doc)

    return json.dumps({
        'result': res,
        'render_json': pagination.render_json(10)
    })
Esempio n. 28
0
def restart_slave(slave_id):
    try:
        slave_record = Mongo.get()["slave_record"].find_one({"_id": ObjectId(slave_id)})
        if not slave_record:
            raise Exception("不存在的记录!")
    except:
        return jsonify({"success": False, "msg": "不存在的记录!"})

    restart_slave_list = GlobalHelper.get("restart_slave_list") or []
    restart_slave_list.append(slave_record["ip"])

    GlobalHelper.set("restart_slave_list", list(set(restart_slave_list)))
    return jsonify({"success": True, "msg": "重启中!"})
Esempio n. 29
0
def restart_slave(slave_id):
    try:
        slave_record = Mongo.get()['slave_record'].find_one({'_id': ObjectId(slave_id)})
        if not slave_record:
            raise Exception('不存在的记录!')
    except:
        return jsonify({'success': False, 'msg': '不存在的记录!'})

    restart_slave_list = GlobalHelper.get('restart_slave_list') or []
    restart_slave_list.append(slave_record['ip'])

    GlobalHelper.set('restart_slave_list', list(set(restart_slave_list)))
    return jsonify({'success': True, 'msg': '重启中!'})
Esempio n. 30
0
    def urls_parsed(self):
        urls_data = []
        url_list = []
        for url in self.__request_json['urls_parsed']:
            self.__slave_record.add_parsed_record(self.__request_address[0])
            url_list.append(url)
            urls_data.append(
                {'url': url, 'url_md5': md5(url), 'add_time': int(time.time()), 'slave_ip': self.__request_address[0]})

        Mongo.get()['queue_' + self.project_name].remove({'url_md5': {'$in': [md5(l) for l in url_list]}},
                                                         multi=True)  # 删除抓取完毕的队列

        try:
            urls_data and Mongo.get()['parsed_' + self.project_name].insert(urls_data)
        except:
            try:
                for single_url in urls_data:
                    single_url and Mongo.get()['parsed_' + self.project_name].insert_one(single_url)
            except Exception, error:
                print traceback.format_exc()
                print error
                print u'下面链接重复抓取的并重复保存到parsed_*中的记录'
                print single_url, '\r\n\r\n'
Esempio n. 31
0
def restart_slave(slave_id):
    try:
        slave_record = Mongo.get()['slave_record'].find_one(
            {'_id': ObjectId(slave_id)})
        if not slave_record:
            raise Exception('不存在的记录!')
    except:
        return jsonify({'success': False, 'msg': '不存在的记录!'})

    restart_slave_list = GlobalHelper.get('restart_slave_list') or []
    restart_slave_list.append(slave_record['ip'])

    GlobalHelper.set('restart_slave_list', list(set(restart_slave_list)))
    return jsonify({'success': True, 'msg': '重启中!'})
Esempio n. 32
0
def toggle_slave(slave_id):
    try:
        slave_record = Mongo.get()['slave_record'].find_one(
            {'_id': ObjectId(slave_id)})
        if not slave_record:
            raise Exception('不存在的记录!')
    except:
        return jsonify({'success': False, 'msg': '不存在的记录!'})

    slave_record['data'][
        'static'] = '抓取中' if slave_record['data']['static'] == '暂停中' else '暂停中'
    try:
        Mongo.get()['slave_record'].update(
            {'_id': ObjectId(slave_id)},
            {'$set': {
                'data.static': slave_record['data']['static']
            }})
        global_salve_record = GlobalHelper.get('salve_record')
        global_salve_record[
            slave_record['ip']]['static'] = slave_record['data']['static']
        GlobalHelper.set('salve_record', global_salve_record)
    except:
        print traceback.format_exc()
    return jsonify({'success': True, 'msg': '切换成功!'})
Esempio n. 33
0
def api_test():
    try:
        client = MongoClient('localhost', 27017)
        res = client['pyfetch']['result_cnbeta'].find().limit(10)

        # print GlobalHelper.get('salve_record')
        # time.sleep(10)
        pagination = paginate(Mongo.get()['result_cnbeta'].find(), 1, 30)
        if pagination:
            for row in pagination.current_page():
                print row

                # print pagination.next()
                # print pagination.prev()

    except:
        print traceback.format_exc()
    return jsonify({'fd': 1})
Esempio n. 34
0
def api_test():
    try:
        client = MongoClient('localhost', 27017)
        res = client['pyfetch']['result_cnbeta'].find().limit(10)

        # print GlobalHelper.get('salve_record')
        # time.sleep(10)
        pagination = paginate(Mongo.get()['result_cnbeta'].find(), 1, 30)
        if pagination:
            for row in pagination.current_page():
                print row

                # print pagination.next()
                # print pagination.prev()

    except:
        print traceback.format_exc()
    return jsonify({'fd': 1})
Esempio n. 35
0
    def __init__(self):
        self.__init_format = {
            '_id': '',
            'ip': '',
            'parsed_count': 0,
            'connected_count': 0,
            'last_connected_time': 0,
            'work_time_count': 1,
            'deny_domains': [],
            'error_domains': {},
            'static': '抓取中'}
        self.slave_record = {}

        self.deny_urls_temp = {}

        if not self.slave_record:
            for item in Mongo.get().slave_record.find():
                item['data']['_id'] = str(item['_id'])
                item['data']['ip'] = str(item['ip'])
                self.slave_record[item['ip']] = item['data']

        self.refresh_connect_status()
Esempio n. 36
0
    def __init__(self):
        self.__init_format = {
            '_id': '',
            'ip': '',
            'parsed_count': 0,
            'connected_count': 0,
            'last_connected_time': 0,
            'work_time_count': 1,
            'deny_domains': [],
            'error_domains': {},
            'static': '抓取中'
        }
        self.slave_record = {}

        self.deny_urls_temp = {}

        if not self.slave_record:
            for item in Mongo.get().slave_record.find():
                item['data']['_id'] = str(item['_id'])
                item['data']['ip'] = str(item['ip'])
                self.slave_record[item['ip']] = item['data']

        self.refresh_connect_status()
Esempio n. 37
0
 def code_ctrl(self):
     result = []
     for project in Mongo.get().projects.find({}, {'_id': 0, 'code': 1, 'name': 1, 'init_url': 1}):
         result.append(project)
     return result
Esempio n. 38
0
def get_json_results(project_name):
    res = make_response(json.dumps(list(Mongo.get()['result_' + project_name].find({}, {'_id': 0}))), 200)
    # header("Content-Disposition:attachment;filename={$fileName}");
    res.headers['Content-Disposition'] = 'attachment'
    res.headers['filename'] = project_name + '/json'
    return res
Esempio n. 39
0
def get_project_by_name(name):
    res = list(Mongo.get().projects.find({"name": name}, {"_id": 0}))
    if not res:
        return jsonify({})
    return jsonify(res[0])
Esempio n. 40
0
def get_json_results(project_name):
    res = make_response(json.dumps(list(Mongo.get()["result_" + project_name].find({}, {"_id": 0}))), 200)
    # header("Content-Disposition:attachment;filename={$fileName}");
    res.headers["Content-Disposition"] = "attachment"
    res.headers["filename"] = project_name + "/json"
    return res
Esempio n. 41
0
def get_project_by_name(name):
    res = list(Mongo.get().projects.find({'name': name}, {'_id': 0}))
    if not res:
        return jsonify({})
    return jsonify(res[0])
Esempio n. 42
0
 def result_save(self):
     Mongo.get()['result_' + self.project_name].insert(self.__request_json['save'])
Esempio n. 43
0
def save_project():
    form_data = json.loads(request.data)  # todo 需要验证表单数据

    name_r = re.compile(r'^[0-9a-zA-Z_-]+$')
    if not name_r.search(form_data['name']):
        return jsonify({
            'success': False,
            'msg': '计划名称必须满足正则规则: ^[0-9a-zA-Z_-]+$ '
        })

    exists_project = list(Mongo.get()['projects'].find(
        {
            'name': form_data['name']
        }, {
            '_id': 1,
            'add_time': 1
        }).limit(1))

    if 'edit' not in form_data and exists_project:
        return jsonify({'success': False, 'msg': '计划名称已经存在!'})

    # 新增计划或更新计划
    data = {
        'name':
        form_data['name'],
        'init_url':
        form_data['init_url'],
        'desc':
        form_data['desc'] if 'desc' in form_data else '',
        'code':
        form_data['code'],
        'static':
        '暂停中',
        'update_time':
        int(time.time()),
        'add_time':
        exists_project[0]['add_time'] if exists_project else int(time.time()),
    }
    Mongo.get()['projects'].update({'name': form_data['name']}, data, True)

    # 当是新计划时的初始化
    if 'edit' not in form_data:
        Mongo.get()['queue_' + form_data['name']].insert({
            'url':
            form_data['init_url'],
            'url_md5':
            md5(form_data['init_url']),
            'flag_time':
            0,
            'add_time':
            int(time.time()),
            'slave_ip':
            '0.0.0.0'
        })

        # 在没创建集合前设置索引mongodb会自动创建该集合并赋索引
        Mongo.get()['parsed_' + form_data['name']].ensure_index('url_md5',
                                                                unique=True)
        Mongo.get()['queue_' + form_data['name']].ensure_index('url_md5',
                                                               unique=True)

        # 有新计划加入, 重启全部slave
        restart_slave_list = GlobalHelper.get('restart_slave_list') or []
        for slave_record in Mongo.get()['slave_record'].find():
            restart_slave_list.append(slave_record['ip'])
        GlobalHelper.set('restart_slave_list', list(set(restart_slave_list)))

    return jsonify({'success': True, 'msg': '保存成功!'})
Esempio n. 44
0
def get_project_by_name(name):
    res = list(Mongo.get().projects.find({'name': name}, {'_id': 0}))
    if not res:
        return jsonify({})
    return jsonify(res[0])