def copynew(seed_id):
if seed_id:
seed = Seed(current_app)
sid = seed.copynew(seed_id)
if sid:
seed_fields = Seed_fields(current_app)
seed_fields.copynew(seed_id, sid)
seed_filter = Seed_filter(current_app)
seed_filter.copynew(seed_id, sid)
url = request.referrer and request.referrer or url_for("seeds.index")
return redirect(url)
def index(page=1):
seed = Seed(current_app)
field = Field(current_app)
seed_fields = Seed_fields(current_app)
filte = {}
seed_type = None
if request.method == "POST":
error = None
action = request.form.get("do")
if action == "delete":
seed_filter_model = Seed_filter(current_app)
sids = request.form.getlist("sid[]")
if len(sids) > 0:
for sid in sids:
if sid:
seed.remove(sid)
seed_fields.remove(sid)
seed_filter_model.remove(sid)
url = request.referrer and request.referrer or url_for('seeds.index')
return redirect(url)
else:
error = "请选择要删除的数据"
if action == "enable" or action == "disable":
edid = (action == "enable") and 1 or 0
sids = request.form.getlist("sid[]")
if len(sids) > 0:
for sid in sids:
if sid:
save = {"enabled":edid}
seed.edit(sid,**save)
url = request.referrer and request.referrer or url_for('seeds.index')
return redirect(url)
else:
error = "请选择要启用的数据"
return error
if request.args.get("keywords"):
filte["seed_name"] = request.args.get("keywords")
if request.args.get("type"):
seed_type = request.args.get("type")
filte["type"] = seed_type
seeds1 = seed.list(page, PER_PAGE, filte)
seeds = []
if not seeds1 and page != 1:
abort(404)
for seed_item in seeds1:
seed_item["tags"] = getTagsBySeedId(seed_item["sid"])
seeds.append(seed_item)
count = seed.totalcount(filte)
pagination = Pagination(page, PER_PAGE, count)
fields = field.getSeedType()
return render_template("seed/list.html", pagination=pagination, seeds=seeds, fields=fields, seed_type=seed_type)
def __init__(self, rule, seed):
if not rule:
raise RuleEmpty
self.seed = seed;
'''
extrarules 额外规则表
这些额外的规则是动态的， 每个都有一个field id
'''
field_db = Seed_fields()
r = field_db.list(seed["sid"])
if len(r) > 0:
self.extrarules = r.list();
rule = rule.encode("utf-8");
rule = unserialize(rule, decode_strings=True)
self.rule = rule;
def getSeedFieldsBySid(sid, seed_type):
seed_fields = Seed_fields(current_app)
#fields = seed_fields.list(sid)
field = Field(current_app)
datas = field.list(seed_type)
new = []
for data in datas:
new2 = {}
#tmp = field.view(data.field_id)[0]
new2["name"] = data["name"]
new2["title"] = data["title"]
new2["type"] = data["type"]
new2["id"] = data["id"]
v = seed_fields.view(sid, data["id"]).list()
if len(v) > 0:
new2["value"] = v[0]["value"]
new2["page_type"] = v[0]["page_type"]
new2["fetch_all"] = v[0]["fetch_all"]
else:
new2["value"] = ""
new2["page_type"] = ""
new2["fetch_all"] = 0
new.append(new2)
return new
def edit(seed_id):
tags = []
alltags = []
if request.method == "POST" and request.form.get("sid"):
sid = request.form.get("sid")
enabled = checkboxVal(request.form.get("enabled"))
list1 = {
"urlformat": request.form.get("urlformat"),
"urltype": request.form.get("urltype"),
"startpage": request.form.get("startpage"),
"maxpage": request.form.get("maxpage"),
"step": request.form.get("step"),
"zero": request.form.get("zero"),
"listparent": request.form.get("listparent"),
"entryparent": request.form.get("entryparent"),
"contenturl": request.form.get("contenturl"),
"contentparent": request.form.get("contentparent"),
"pageparent": request.form.get("pageparent"),
"filters": request.form.get("filters"),
}
time1 = int(time.time())
save = {
"type": request.form.get("type"),
"seed_name": request.form.get("seed_name"),
"charset": request.form.get("charset"),
"frequency": float(request.form.get("frequency")) * 3600,
"timeout": request.form.get("timeout"),
"tries": request.form.get("tries"),
"enabled": enabled,
"listtype": request.form.get("listtype"),
"lang": request.form.get("lang"),
"update_time": time1,
"rule": phpserialize.dumps(list1),
}
field = Field(current_app)
fields = field.list(save["type"]).list()
guid_rule_datas = request.form.get("guid_rule")
save["guid_rule"] = getFeildIdByTitle(guid_rule_datas, save["type"])
seed = Seed(current_app)
msg = seed.edit(sid, **save)
seed_field = Seed_fields(current_app)
# filter_model = Filter(current_app)
# filters_org_data = filter_model.list()
seed_filter_model = Seed_filter(current_app)
# filters_data = {}
# for filter_item in filters_org_data:
# filters_data[str(filter_item["id"])] = filter_item
for field in fields:
field_data = seed_field.view(sid, field.id).list()
if len(field_data) > 0:
seed_field.edit(
sid,
field.id,
request.form.get(field.name),
request.form.get("page_type_" + field.name),
request.form.get("fetch_all_" + field.name),
)
else:
seed_value = {}
seed_value["seed_id"] = sid
seed_value["field_id"] = field.id
seed_value["value"] = request.form.get(field.name)
seed_value["page_type"] = request.form.get("page_type_" + field.name)
seed_value["fetch_all"] = (
request.form.get("fetch_all_" + field.name) and request.form.get("fetch_all_" + field.name) or 0
)
seed_field.add(**seed_value)
# 更改过滤规则
filter_ids = request.form.getlist("filter" + str(field.id) + "[]")
order_ct = 0
if len(filter_ids) > 0 and filter_ids[0] != "none":
seed_filter_model.remove(sid, field.id)
for filter_id in filter_ids:
filter_item = request.form.getlist("profile_" + str(field.id) + "_" + str(filter_id) + "[]")
tmp = {}
ct = 0
for config in filter_item:
v = request.form.get(config + "_" + str(field.id) + "_" + filter_id)
if not v:
ct = ct + 1
tmp[config] = v
if ct != len(filter_item):
order_ct = order_ct + 1
filter_save = {
"sid": int(sid),
"field_id": int(field.id),
"filter_id": int(filter_id),
"profile": phpserialize.dumps(tmp),
"list_order": int(order_ct),
}
seed_filter_model.add(**filter_save)
# 更改标签
tags_data = request.form.get("tags")
tags_data = tags_data.split(",")
tags_model = Tags(current_app)
seed_tag = Seed_tag(current_app)
current_tags_data = seed_tag.list(sid)
del_tags_data = {}
for current_tag in current_tags_data:
del_tags_data[current_tag["tid"]] = current_tag
for tag in tags_data:
tag = tag.strip()
if tag:
ishere = tags_model.findByName(tag).list()
if len(ishere) > 0:
tid = ishere[0]["id"]
else:
tid = tags_model.add(name=tag)
if tid in del_tags_data:
del del_tags_data[tid]
ishere = seed_tag.view(sid, tid).list()
if len(ishere) == 0:
seed_tag.add(sid=sid, tid=tid)
for k in del_tags_data:
seed_tag.remove(del_tags_data[k]["sid"], del_tags_data[k]["tid"])
url = getReferer()
url = url and url or url_for("seeds.index")
return redirect(url)
if request.method == "GET" and seed_id > 0:
setReferer()
size = request.cookies.get("editorSize")
if not size:
size = 50
seed = Seed(current_app)
seed_data = seed.view(seed_id)[0]
seed_type = seed_data["type"]
seed_field = getSeedFieldsBySid(seed_id, seed_type)
seed_fields = Seed_fields(current_app)
page_types = seed_fields.getpageType()
seed_data["frequency"] = float(seed_data["frequency"]) / float(3600)
if seed_data["rule"]:
seed_data["rule"] = seed_data["rule"].encode("utf-8")
seed_data["rule"] = phpserialize.loads(seed_data["rule"], decode_strings=True)
# 获取GUID规则
if seed_data["guid_rule"]:
seed_data["guid_rule"] = getFeildTitleById(seed_data["guid_rule"], seed_type)
# 取出标签
seed_tag = Seed_tag(current_app)
tags_model = Tags(current_app)
tags_data = seed_tag.list(seed_id)
for tag in tags_data:
tmp = tags_model.view(tag["tid"])[0]
tags.append(tmp["name"])
alltags_data = tags_model.list()
for tag in alltags_data:
alltags.append(tag["name"])
return render_template(
"seed/add.html",
seed_type=seed_type,
fields=seed_field,
seed_data=seed_data,
sid=seed_id,
page_types=page_types,
tags=tags,
alltags=alltags,
size=size,
)
def add():
field = Field(current_app)
if request.method == "POST":
list1 = {
"urlformat": request.form.get("urlformat"),
"urltype": request.form.get("urltype"),
"startpage": request.form.get("startpage"),
"maxpage": request.form.get("maxpage"),
"step": request.form.get("step"),
"zero": request.form.get("zero"),
"listparent": request.form.get("listparent"),
"entryparent": request.form.get("entryparent"),
"contenturl": request.form.get("contenturl"),
"contentparent": request.form.get("contentparent"),
"pageparent": request.form.get("pageparent"),
"filters": request.form.get("filters"),
}
time1 = int(time.time())
save = {
"type": request.form.get("type"),
"seed_name": request.form.get("seed_name"),
"charset": request.form.get("charset"),
"frequency": float(request.form.get("frequency")) * 3600,
"timeout": request.form.get("timeout"),
"tries": request.form.get("tries"),
"enabled": request.form.get("enabled") is 1 and 1 or 0,
"listtype": request.form.get("listtype"),
"lang": request.form.get("lang"),
"created_time": time1,
"update_time": time1,
"rule": phpserialize.dumps(list1),
}
fields = field.list(save["type"]).list()
guid_rule_datas = request.form.get("guid_rule")
save["guid_rule"] = getFeildIdByTitle(guid_rule_datas, save["type"])
seed = Seed(current_app)
sid = seed.add(**save)
if sid:
seed_field = Seed_fields(current_app)
# page_types = request.form.getlist("page_type[]")
for field in fields:
seed_value = {}
seed_value["seed_id"] = sid
seed_value["field_id"] = field.id
seed_value["value"] = request.form.get(field.name)
seed_value["page_type"] = request.form.get("page_type_" + field.name)
seed_value["fetch_all"] = (
request.form.get("fetch_all_" + field.name) and request.form.get("fetch_all_" + field.name) or 0
)
seed_field.add(**seed_value)
# 插入标签
tags_data = request.form.get("tags")
tags_data = tags_data.split(",")
tags_model = Tags(current_app)
seed_tag = Seed_tag(current_app)
for tag in tags_data:
tag = tag.strip()
if tag:
ishere = tags_model.findByName(tag).list()
if len(ishere) > 0:
tid = ishere[0]["id"]
else:
tid = tags_model.add(name=tag)
ishere = seed_tag.view(sid, tid).list()
if len(ishere) == 0:
seed_tag.add(sid=sid, tid=tid)
url = getReferer()
url = url and url or url_for("seeds.index")
return redirect(url)
fields = field.getSeedType()
if request.method == "GET" and request.args.get("type"):
setReferer()
size = request.cookies.get("editorSize")
if not size:
size = 50
seed_data = {}
seed_data["rule"] = {}
seed_type = request.args.get("type")
fields = field.list(seed_type).list()
seed_field = Seed_fields(current_app)
page_types = seed_field.getpageType()
alltags = []
tags_model = Tags(current_app)
alltags_data = tags_model.list()
for tag in alltags_data:
alltags.append(tag["name"])
return render_template(
"seed/add.html",
seed_type=seed_type,
fields=fields,
seed_data=seed_data,
page_types=page_types,
alltags=alltags,
size=size,
)
return render_template("seed/select_type.html", fields=fields)
