Пример #1
0
def add():
    error = None
    succ = None
    seed_type = request.args.get("type")
    site_id = request.args.get("site_id")
    if request.method == "POST":
        table_name = request.form.get("table_name")
        if table_name:
            field_ids = request.form.getlist("field_id[]");
            site_fields = request.form.getlist("site_field[]");
            seed_type =request.form.get("seed_type")
            site_id = request.form.get("site_id")
            site_map = Site_map(current_app)
            for i,k in enumerate(field_ids):
                save = {
                    "siteid": site_id,
                    "table_name": table_name,
                    "field_id": field_ids[i],
                    "seed_type": seed_type,
                    "site_field": site_fields[i]
                }
                site_map_id = site_map.add(**save)
            succ = u"添加成功!"
        else:
            error = u"数据表名不能为空"
    fields = getSeedFieldsByType(seed_type)
    return render_template("site_map/add.html", fields=fields, seed_type=seed_type, site_id=site_id, error=error, succ=succ)
Пример #2
0
def edit(site_id):
    site = Site(current_app)
    hook_funcs = [];
    attrs = dir(hooks);
    for attr in attrs:
        if attr.find("__") == -1:
            hook_funcs.append(attr)
    if request.method == "POST":
        name = request.form.get("name")
        url = request.form.get("url")
        descript = request.form.get("descript")
        sync_type = request.form.get("sync_type")
        if not name:
            error = "请输入站点名称"
        elif not url:
            error = "请输入站点链接"
        elif not sync_type:
            error = "请选择入库方式"
        else:
            site = Site(current_app)
            sync_profile = {
                "staticUrl": request.form.get("staticUrl"),
                "staticType": request.form.get("staticType"),
                "mysql_server": request.form.get("mysql_server"),
                "mysql_dbname": request.form.get("mysql_dbname"),
                "mysql_prefix": request.form.get("mysql_prefix"),
                "mysql_username": request.form.get("mysql_username"),
                "mysql_password": request.form.get("mysql_password"),
                "ftp_server": request.form.get("ftp_server"),
                "ftp_port": request.form.get("ftp_port"),
                "ftp_path": request.form.get("ftp_path"),
                "ftp_username": request.form.get("ftp_username"),
                "ftp_password": request.form.get("ftp_password"),
                "access_id": request.form.get("access_id"),
                "secret_access_key": request.form.get("secret_access_key"),
                "hook_func": request.form.get("hook_func"),
                "api_url": request.form.get("api_url")
            }
            sync_profile = phpserialize.dumps(sync_profile)
            site.edit(id=site_id, name=name, url=url, descript=descript, sync_type=sync_type, sync_profile=sync_profile)
            return redirect(url_for('sites.index'))
    per = site.view(site_id)[0]
    if per["sync_profile"]:
        per["sync_profile"] = phpserialize.loads(per["sync_profile"])
    field = Field(current_app)
    types = field.getSeedType()
    site_map = Site_map(current_app)
    website_maps = site_map.list(site_id)
    maps = {}
    for website_map in website_maps:
        if website_map["seed_type"] not in maps:
            maps[website_map["seed_type"]] = website_map
    return render_template("site/add.html", site=per, types=types, maps=maps, hook_funcs=hook_funcs)
Пример #3
0
    def __init__(self):
        '''
	初始化 把所有的网站都列出来, 并且进行数据推送
	'''
        #site cache
        self.sites = {}
        # site map cache by category, kaifu
        self.site_by_category = {}

        #初始化站点信息
        self.init_sites()
        self.mapdb = Site_map()
Пример #4
0
    def __init__(self):
	'''
	初始化 把所有的网站都列出来, 并且进行数据推送
	'''
	#site cache
	self.sites = {}
	# site map cache by category, kaifu
	self.site_by_category = {}

	#初始化站点信息
	self.init_sites()
	self.mapdb = Site_map()
Пример #5
0
def edit():
    error = None
    succ = None
    table_name = ""
    seed_type = request.args.get("type")
    site_id = request.args.get("site_id")
    site_map = Site_map(current_app)
    if request.method == "POST":
        table_name = request.form.get("table_name")
        if table_name:
            field_ids = request.form.getlist("field_id[]");
            site_fields = request.form.getlist("site_field[]");
            seed_type =request.form.get("seed_type")
            site_id = request.form.get("site_id")
            for i,k in enumerate(field_ids):
                fid = field_ids[i]
                ishere = site_map.view(site_id, seed_type, fid).list()
                save = {
                    "siteid": site_id,
                    "table_name": table_name,
                    "field_id": field_ids[i],
                    "seed_type": seed_type,
                    "site_field": site_fields[i]
                }
                if len(ishere) > 0:
                    ishere = ishere[0]
                    site_map_id = site_map.edit(ishere["id"],**save)
                else:
                    site_map_id = site_map.add(**save)
            succ = u"修改成功!"
        else:
            error = u"数据表名不能为空"
    chk = {"siteid":site_id, "seed_type": seed_type}
    website_maps = site_map.getlist(chk)
    maps = {}
    fields = getSeedFieldsByType(seed_type)
    for website_map in website_maps:
        table_name = website_map["table_name"]
        maps[website_map["field_id"]] = website_map["site_field"]
    return render_template("site_map/add.html", fields=fields, seed_type=seed_type, site_id=site_id, error=error, succ=succ, table_name=table_name, maps=maps)
Пример #6
0
class PublishServer():
    '''
    发布服务器, 每次采集成功后, 会调用此服务中的push转发到各个站点
    '''
    def __init__(self):
	'''
	初始化 把所有的网站都列出来, 并且进行数据推送
	'''
	#site cache
	self.sites = {}
	# site map cache by category, kaifu
	self.site_by_category = {}

	#初始化站点信息
	self.init_sites()
	self.mapdb = Site_map()

    def init_sites(self):
	'''
	初始化将所有的站点数据保存在sites中
	'''
	site_db = Site_Model();
	query = site_db.select();
	r = query.list();
	for site in r:
	    if site and "id" in site:
		self.sites[site["id"]] = Site(site)

    def update_sites(self):
	'''
	这里更新需要做一次安全的检测。 当push不在调用的时候执行此函数
	'''
	self.init_sites();

    def get_site(self, site_id):
	'''
	get Site instance by site_id
	'''
	if site_id in self.sites:
	    return self.sites[site_id]

    def init_site_fieldmap(self, seed_type):
	'''
	初始化站点字段映射表
	'''
	self.site_by_category[seed_type] = {}
	query = self.mapdb.select(where={ "seed_type" : seed_type }, what="siteid", group="siteid")
	if len(query) > 0:
	    r = query.list();
	    for site in r:
		site_id = site["siteid"]
		#映射数据表
		fields = self.mapdb.select( where = { "seed_type" : seed_type, "siteid" : site_id} ).list()
		self.site_by_category[seed_type][site_id] = fields

    def update_site_fieldmap(self, seed_type):
	'''
	这里更新时候需要做安全检测
	'''

    def push(self, guid, data):
	'''
	发布接口
	将发布的数据解析类型。 根据类型找到site_id
	然后发布出去
	'''
	seed_type = data["type"]
	if seed_type not in self.site_by_category:
	    self.init_site_fieldmap(seed_type)

	if seed_type in self.site_by_category:
	    sites = self.site_by_category[seed_type]
	    for site_id in sites:
		site_profile = sites[site_id]
		site = self.get_site(site_id)
		if site is not None:
		    site.push(guid, data, site_profile)
Пример #7
0
class PublishServer():
    '''
    发布服务器, 每次采集成功后, 会调用此服务中的push转发到各个站点
    '''
    def __init__(self):
        '''
	初始化 把所有的网站都列出来, 并且进行数据推送
	'''
        #site cache
        self.sites = {}
        # site map cache by category, kaifu
        self.site_by_category = {}

        #初始化站点信息
        self.init_sites()
        self.mapdb = Site_map()

    def init_sites(self):
        '''
	初始化将所有的站点数据保存在sites中
	'''
        site_db = Site_Model()
        query = site_db.select()
        r = query.list()
        for site in r:
            if site and "id" in site:
                self.sites[site["id"]] = Site(site)

    def update_sites(self):
        '''
	这里更新需要做一次安全的检测。 当push不在调用的时候执行此函数
	'''
        self.init_sites()

    def get_site(self, site_id):
        '''
	get Site instance by site_id
	'''
        if site_id in self.sites:
            return self.sites[site_id]

    def init_site_fieldmap(self, seed_type):
        '''
	初始化站点字段映射表
	'''
        self.site_by_category[seed_type] = {}
        query = self.mapdb.select(where={"seed_type": seed_type},
                                  what="siteid",
                                  group="siteid")
        if len(query) > 0:
            r = query.list()
            for site in r:
                site_id = site["siteid"]
                #映射数据表
                fields = self.mapdb.select(where={
                    "seed_type": seed_type,
                    "siteid": site_id
                }).list()
                self.site_by_category[seed_type][site_id] = fields

    def update_site_fieldmap(self, seed_type):
        '''
	这里更新时候需要做安全检测
	'''

    def push(self, guid, data):
        '''
	发布接口
	将发布的数据解析类型。 根据类型找到site_id
	然后发布出去
	'''
        seed_type = data["type"]
        if seed_type not in self.site_by_category:
            self.init_site_fieldmap(seed_type)

        if seed_type in self.site_by_category:
            sites = self.site_by_category[seed_type]
            for site_id in sites:
                site_profile = sites[site_id]
                site = self.get_site(site_id)
                if site is not None:
                    site.push(guid, data, site_profile)