def add():
error = None
succ = None
seed_type = request.args.get("type")
site_id = request.args.get("site_id")
if request.method == "POST":
table_name = request.form.get("table_name")
if table_name:
field_ids = request.form.getlist("field_id[]");
site_fields = request.form.getlist("site_field[]");
seed_type =request.form.get("seed_type")
site_id = request.form.get("site_id")
site_map = Site_map(current_app)
for i,k in enumerate(field_ids):
save = {
"siteid": site_id,
"table_name": table_name,
"field_id": field_ids[i],
"seed_type": seed_type,
"site_field": site_fields[i]
}
site_map_id = site_map.add(**save)
succ = u"添加成功！"
else:
error = u"数据表名不能为空"
fields = getSeedFieldsByType(seed_type)
return render_template("site_map/add.html", fields=fields, seed_type=seed_type, site_id=site_id, error=error, succ=succ)
def edit(site_id):
site = Site(current_app)
hook_funcs = [];
attrs = dir(hooks);
for attr in attrs:
if attr.find("__") == -1:
hook_funcs.append(attr)
if request.method == "POST":
name = request.form.get("name")
url = request.form.get("url")
descript = request.form.get("descript")
sync_type = request.form.get("sync_type")
if not name:
error = "请输入站点名称"
elif not url:
error = "请输入站点链接"
elif not sync_type:
error = "请选择入库方式"
else:
site = Site(current_app)
sync_profile = {
"staticUrl": request.form.get("staticUrl"),
"staticType": request.form.get("staticType"),
"mysql_server": request.form.get("mysql_server"),
"mysql_dbname": request.form.get("mysql_dbname"),
"mysql_prefix": request.form.get("mysql_prefix"),
"mysql_username": request.form.get("mysql_username"),
"mysql_password": request.form.get("mysql_password"),
"ftp_server": request.form.get("ftp_server"),
"ftp_port": request.form.get("ftp_port"),
"ftp_path": request.form.get("ftp_path"),
"ftp_username": request.form.get("ftp_username"),
"ftp_password": request.form.get("ftp_password"),
"access_id": request.form.get("access_id"),
"secret_access_key": request.form.get("secret_access_key"),
"hook_func": request.form.get("hook_func"),
"api_url": request.form.get("api_url")
}
sync_profile = phpserialize.dumps(sync_profile)
site.edit(id=site_id, name=name, url=url, descript=descript, sync_type=sync_type, sync_profile=sync_profile)
return redirect(url_for('sites.index'))
per = site.view(site_id)[0]
if per["sync_profile"]:
per["sync_profile"] = phpserialize.loads(per["sync_profile"])
field = Field(current_app)
types = field.getSeedType()
site_map = Site_map(current_app)
website_maps = site_map.list(site_id)
maps = {}
for website_map in website_maps:
if website_map["seed_type"] not in maps:
maps[website_map["seed_type"]] = website_map
return render_template("site/add.html", site=per, types=types, maps=maps, hook_funcs=hook_funcs)
def __init__(self):
'''
初始化 把所有的网站都列出来， 并且进行数据推送
'''
#site cache
self.sites = {}
# site map cache by category, kaifu
self.site_by_category = {}
#初始化站点信息
self.init_sites()
self.mapdb = Site_map()
def __init__(self):
'''
初始化 把所有的网站都列出来， 并且进行数据推送
'''
#site cache
self.sites = {}
# site map cache by category, kaifu
self.site_by_category = {}
#初始化站点信息
self.init_sites()
self.mapdb = Site_map()
def edit():
error = None
succ = None
table_name = ""
seed_type = request.args.get("type")
site_id = request.args.get("site_id")
site_map = Site_map(current_app)
if request.method == "POST":
table_name = request.form.get("table_name")
if table_name:
field_ids = request.form.getlist("field_id[]");
site_fields = request.form.getlist("site_field[]");
seed_type =request.form.get("seed_type")
site_id = request.form.get("site_id")
for i,k in enumerate(field_ids):
fid = field_ids[i]
ishere = site_map.view(site_id, seed_type, fid).list()
save = {
"siteid": site_id,
"table_name": table_name,
"field_id": field_ids[i],
"seed_type": seed_type,
"site_field": site_fields[i]
}
if len(ishere) > 0:
ishere = ishere[0]
site_map_id = site_map.edit(ishere["id"],**save)
else:
site_map_id = site_map.add(**save)
succ = u"修改成功！"
else:
error = u"数据表名不能为空"
chk = {"siteid":site_id, "seed_type": seed_type}
website_maps = site_map.getlist(chk)
maps = {}
fields = getSeedFieldsByType(seed_type)
for website_map in website_maps:
table_name = website_map["table_name"]
maps[website_map["field_id"]] = website_map["site_field"]
return render_template("site_map/add.html", fields=fields, seed_type=seed_type, site_id=site_id, error=error, succ=succ, table_name=table_name, maps=maps)
class PublishServer():
'''
发布服务器, 每次采集成功后， 会调用此服务中的push转发到各个站点
'''
def __init__(self):
'''
初始化 把所有的网站都列出来， 并且进行数据推送
'''
#site cache
self.sites = {}
# site map cache by category, kaifu
self.site_by_category = {}
#初始化站点信息
self.init_sites()
self.mapdb = Site_map()
def init_sites(self):
'''
初始化将所有的站点数据保存在sites中
'''
site_db = Site_Model();
query = site_db.select();
r = query.list();
for site in r:
if site and "id" in site:
self.sites[site["id"]] = Site(site)
def update_sites(self):
'''
这里更新需要做一次安全的检测。 当push不在调用的时候执行此函数
'''
self.init_sites();
def get_site(self, site_id):
'''
get Site instance by site_id
'''
if site_id in self.sites:
return self.sites[site_id]
def init_site_fieldmap(self, seed_type):
'''
初始化站点字段映射表
'''
self.site_by_category[seed_type] = {}
query = self.mapdb.select(where={ "seed_type" : seed_type }, what="siteid", group="siteid")
if len(query) > 0:
r = query.list();
for site in r:
site_id = site["siteid"]
#映射数据表
fields = self.mapdb.select( where = { "seed_type" : seed_type, "siteid" : site_id} ).list()
self.site_by_category[seed_type][site_id] = fields
def update_site_fieldmap(self, seed_type):
'''
这里更新时候需要做安全检测
'''
def push(self, guid, data):
'''
发布接口
将发布的数据解析类型。 根据类型找到site_id
然后发布出去
'''
seed_type = data["type"]
if seed_type not in self.site_by_category:
self.init_site_fieldmap(seed_type)
if seed_type in self.site_by_category:
sites = self.site_by_category[seed_type]
for site_id in sites:
site_profile = sites[site_id]
site = self.get_site(site_id)
if site is not None:
site.push(guid, data, site_profile)
class PublishServer():
'''
发布服务器, 每次采集成功后， 会调用此服务中的push转发到各个站点
'''
def __init__(self):
'''
初始化 把所有的网站都列出来， 并且进行数据推送
'''
#site cache
self.sites = {}
# site map cache by category, kaifu
self.site_by_category = {}
#初始化站点信息
self.init_sites()
self.mapdb = Site_map()
def init_sites(self):
'''
初始化将所有的站点数据保存在sites中
'''
site_db = Site_Model()
query = site_db.select()
r = query.list()
for site in r:
if site and "id" in site:
self.sites[site["id"]] = Site(site)
def update_sites(self):
'''
这里更新需要做一次安全的检测。 当push不在调用的时候执行此函数
'''
self.init_sites()
def get_site(self, site_id):
'''
get Site instance by site_id
'''
if site_id in self.sites:
return self.sites[site_id]
def init_site_fieldmap(self, seed_type):
'''
初始化站点字段映射表
'''
self.site_by_category[seed_type] = {}
query = self.mapdb.select(where={"seed_type": seed_type},
what="siteid",
group="siteid")
if len(query) > 0:
r = query.list()
for site in r:
site_id = site["siteid"]
#映射数据表
fields = self.mapdb.select(where={
"seed_type": seed_type,
"siteid": site_id
}).list()
self.site_by_category[seed_type][site_id] = fields
def update_site_fieldmap(self, seed_type):
'''
这里更新时候需要做安全检测
'''
def push(self, guid, data):
'''
发布接口
将发布的数据解析类型。 根据类型找到site_id
然后发布出去
'''
seed_type = data["type"]
if seed_type not in self.site_by_category:
self.init_site_fieldmap(seed_type)
if seed_type in self.site_by_category:
sites = self.site_by_category[seed_type]
for site_id in sites:
site_profile = sites[site_id]
site = self.get_site(site_id)
if site is not None:
site.push(guid, data, site_profile)