def index():
data = 0
sites = Site(current_app)
site = sites.totalcount()
seeds = Seed(current_app)
seed = seeds.totalcount()
users = User(current_app)
uid = session['uid']
user = users.view(uid)
lastlogin = user[0]["lastlogintime"]
return render_template("index.html", site=site, seed=seed, data=data, lastlogin=lastlogin)
def copynew(seed_id):
if seed_id:
seed = Seed(current_app)
sid = seed.copynew(seed_id)
if sid:
seed_fields = Seed_fields(current_app)
seed_fields.copynew(seed_id, sid)
seed_filter = Seed_filter(current_app)
seed_filter.copynew(seed_id, sid)
url = request.referrer and request.referrer or url_for("seeds.index")
return redirect(url)
def view(seed_id,guid):
items = None
fields = {}
if guid and seed_id:
db = Seed_Model();
r = db.view(seed_id);
seed = Seed(r.list()[0])
t = Grab(seed)
if len(t):
db_field = Field_Model(current_app)
items = t[guid]
field_datas = db_field.list(items["type"])
for field_data in field_datas:
fields[field_data["name"]] = field_data["title"]
return render_template("test_seed/view.html", items=items, fields=fields)
def seed_logs(page=1):
seed = Seed(current_app)
seed_logs = Seed_log(current_app)
field = Field(current_app)
filte = []
status = None
start_time = ""
end_time = ""
if request.args.get("start_time"):
start_time = request.args.get("start_time")
tmp = time.mktime(datetime.datetime.strptime(start_time, "%Y-%m-%d %H:%M").timetuple())
filte.append("start_time >= "+str(tmp))
if request.args.get("end_time"):
end_time = request.args.get("end_time")
tmp = time.mktime(datetime.datetime.strptime(end_time, "%Y-%m-%d %H:%M").timetuple())
filte.append("start_time <= "+str(tmp))
if request.args.get("status"):
status = request.args.get("status")
filte.append("status = "+str(status))
if request.args.get("seed_id"):
seed_id = request.args.get("seed_id")
filte.append("sid = "+str(seed_id))
filte = " and ".join(filte)
seeds1 = seed_logs.list(page, PER_PAGE, filte, "start_time DESC")
seeds = []
if not seeds1 and page != 1:
abort(404)
for seed_item in seeds1:
tmp = seed.view(seed_item["sid"]).list()
if len(tmp) > 0:
seed_item["seed_name"] = tmp[0]["seed_name"]
else:
seed_item["seed_name"] = u"未知"
seeds.append(seed_item)
count = seed_logs.totalcount(filte)
pagination = Pagination(page, PER_PAGE, count)
fields = field.getSeedType()
return render_template("logs/seed.html", pagination=pagination, seeds=seeds, fields=fields, status=status, start_time=start_time, end_time=end_time)
def list(seed_id):
items = {}
num = 0
seed_type = None
if seed_id:
db = Seed_Model();
r = db.view(seed_id);
seed = Seed(r.list()[0])
t = Grab(seed)
if len(t):
num = len(t.keys())
seed_type = t.seed_type
for guid in t.keys():
tmp = {}
dont_craw_content = [
'kaifu', 'kaice', "gift"
]
if seed_type in dont_craw_content:
for f in t.items[guid].fields:
tmp[f] = t.items[guid][f].value
else:
tmp["url"] = t.items[guid]["url"]
items[guid] = tmp
return render_template("test_seed/list.html", items=items, num=num, seed_id=seed_id, seed_type=seed_type, dont_craw_content=dont_craw_content)
def index(page=1):
seed = Seed(current_app)
field = Field(current_app)
seed_fields = Seed_fields(current_app)
filte = {}
seed_type = None
if request.method == "POST":
error = None
action = request.form.get("do")
if action == "delete":
seed_filter_model = Seed_filter(current_app)
sids = request.form.getlist("sid[]")
if len(sids) > 0:
for sid in sids:
if sid:
seed.remove(sid)
seed_fields.remove(sid)
seed_filter_model.remove(sid)
url = request.referrer and request.referrer or url_for('seeds.index')
return redirect(url)
else:
error = "请选择要删除的数据"
if action == "enable" or action == "disable":
edid = (action == "enable") and 1 or 0
sids = request.form.getlist("sid[]")
if len(sids) > 0:
for sid in sids:
if sid:
save = {"enabled":edid}
seed.edit(sid,**save)
url = request.referrer and request.referrer or url_for('seeds.index')
return redirect(url)
else:
error = "请选择要启用的数据"
return error
if request.args.get("keywords"):
filte["seed_name"] = request.args.get("keywords")
if request.args.get("type"):
seed_type = request.args.get("type")
filte["type"] = seed_type
seeds1 = seed.list(page, PER_PAGE, filte)
seeds = []
if not seeds1 and page != 1:
abort(404)
for seed_item in seeds1:
seed_item["tags"] = getTagsBySeedId(seed_item["sid"])
seeds.append(seed_item)
count = seed.totalcount(filte)
pagination = Pagination(page, PER_PAGE, count)
fields = field.getSeedType()
return render_template("seed/list.html", pagination=pagination, seeds=seeds, fields=fields, seed_type=seed_type)
def edit(seed_id):
tags = []
alltags = []
if request.method == "POST" and request.form.get("sid"):
sid = request.form.get("sid")
enabled = checkboxVal(request.form.get("enabled"))
list1 = {
"urlformat": request.form.get("urlformat"),
"urltype": request.form.get("urltype"),
"startpage": request.form.get("startpage"),
"maxpage": request.form.get("maxpage"),
"step": request.form.get("step"),
"zero": request.form.get("zero"),
"listparent": request.form.get("listparent"),
"entryparent": request.form.get("entryparent"),
"contenturl": request.form.get("contenturl"),
"contentparent": request.form.get("contentparent"),
"pageparent": request.form.get("pageparent"),
"filters": request.form.get("filters"),
}
time1 = int(time.time())
save = {
"type": request.form.get("type"),
"seed_name": request.form.get("seed_name"),
"charset": request.form.get("charset"),
"frequency": float(request.form.get("frequency")) * 3600,
"timeout": request.form.get("timeout"),
"tries": request.form.get("tries"),
"enabled": enabled,
"listtype": request.form.get("listtype"),
"lang": request.form.get("lang"),
"update_time": time1,
"rule": phpserialize.dumps(list1),
}
field = Field(current_app)
fields = field.list(save["type"]).list()
guid_rule_datas = request.form.get("guid_rule")
save["guid_rule"] = getFeildIdByTitle(guid_rule_datas, save["type"])
seed = Seed(current_app)
msg = seed.edit(sid, **save)
seed_field = Seed_fields(current_app)
# filter_model = Filter(current_app)
# filters_org_data = filter_model.list()
seed_filter_model = Seed_filter(current_app)
# filters_data = {}
# for filter_item in filters_org_data:
# filters_data[str(filter_item["id"])] = filter_item
for field in fields:
field_data = seed_field.view(sid, field.id).list()
if len(field_data) > 0:
seed_field.edit(
sid,
field.id,
request.form.get(field.name),
request.form.get("page_type_" + field.name),
request.form.get("fetch_all_" + field.name),
)
else:
seed_value = {}
seed_value["seed_id"] = sid
seed_value["field_id"] = field.id
seed_value["value"] = request.form.get(field.name)
seed_value["page_type"] = request.form.get("page_type_" + field.name)
seed_value["fetch_all"] = (
request.form.get("fetch_all_" + field.name) and request.form.get("fetch_all_" + field.name) or 0
)
seed_field.add(**seed_value)
# 更改过滤规则
filter_ids = request.form.getlist("filter" + str(field.id) + "[]")
order_ct = 0
if len(filter_ids) > 0 and filter_ids[0] != "none":
seed_filter_model.remove(sid, field.id)
for filter_id in filter_ids:
filter_item = request.form.getlist("profile_" + str(field.id) + "_" + str(filter_id) + "[]")
tmp = {}
ct = 0
for config in filter_item:
v = request.form.get(config + "_" + str(field.id) + "_" + filter_id)
if not v:
ct = ct + 1
tmp[config] = v
if ct != len(filter_item):
order_ct = order_ct + 1
filter_save = {
"sid": int(sid),
"field_id": int(field.id),
"filter_id": int(filter_id),
"profile": phpserialize.dumps(tmp),
"list_order": int(order_ct),
}
seed_filter_model.add(**filter_save)
# 更改标签
tags_data = request.form.get("tags")
tags_data = tags_data.split(",")
tags_model = Tags(current_app)
seed_tag = Seed_tag(current_app)
current_tags_data = seed_tag.list(sid)
del_tags_data = {}
for current_tag in current_tags_data:
del_tags_data[current_tag["tid"]] = current_tag
for tag in tags_data:
tag = tag.strip()
if tag:
ishere = tags_model.findByName(tag).list()
if len(ishere) > 0:
tid = ishere[0]["id"]
else:
tid = tags_model.add(name=tag)
if tid in del_tags_data:
del del_tags_data[tid]
ishere = seed_tag.view(sid, tid).list()
if len(ishere) == 0:
seed_tag.add(sid=sid, tid=tid)
for k in del_tags_data:
seed_tag.remove(del_tags_data[k]["sid"], del_tags_data[k]["tid"])
url = getReferer()
url = url and url or url_for("seeds.index")
return redirect(url)
if request.method == "GET" and seed_id > 0:
setReferer()
size = request.cookies.get("editorSize")
if not size:
size = 50
seed = Seed(current_app)
seed_data = seed.view(seed_id)[0]
seed_type = seed_data["type"]
seed_field = getSeedFieldsBySid(seed_id, seed_type)
seed_fields = Seed_fields(current_app)
page_types = seed_fields.getpageType()
seed_data["frequency"] = float(seed_data["frequency"]) / float(3600)
if seed_data["rule"]:
seed_data["rule"] = seed_data["rule"].encode("utf-8")
seed_data["rule"] = phpserialize.loads(seed_data["rule"], decode_strings=True)
# 获取GUID规则
if seed_data["guid_rule"]:
seed_data["guid_rule"] = getFeildTitleById(seed_data["guid_rule"], seed_type)
# 取出标签
seed_tag = Seed_tag(current_app)
tags_model = Tags(current_app)
tags_data = seed_tag.list(seed_id)
for tag in tags_data:
tmp = tags_model.view(tag["tid"])[0]
tags.append(tmp["name"])
alltags_data = tags_model.list()
for tag in alltags_data:
alltags.append(tag["name"])
return render_template(
"seed/add.html",
seed_type=seed_type,
fields=seed_field,
seed_data=seed_data,
sid=seed_id,
page_types=page_types,
tags=tags,
alltags=alltags,
size=size,
)
def add():
field = Field(current_app)
if request.method == "POST":
list1 = {
"urlformat": request.form.get("urlformat"),
"urltype": request.form.get("urltype"),
"startpage": request.form.get("startpage"),
"maxpage": request.form.get("maxpage"),
"step": request.form.get("step"),
"zero": request.form.get("zero"),
"listparent": request.form.get("listparent"),
"entryparent": request.form.get("entryparent"),
"contenturl": request.form.get("contenturl"),
"contentparent": request.form.get("contentparent"),
"pageparent": request.form.get("pageparent"),
"filters": request.form.get("filters"),
}
time1 = int(time.time())
save = {
"type": request.form.get("type"),
"seed_name": request.form.get("seed_name"),
"charset": request.form.get("charset"),
"frequency": float(request.form.get("frequency")) * 3600,
"timeout": request.form.get("timeout"),
"tries": request.form.get("tries"),
"enabled": request.form.get("enabled") is 1 and 1 or 0,
"listtype": request.form.get("listtype"),
"lang": request.form.get("lang"),
"created_time": time1,
"update_time": time1,
"rule": phpserialize.dumps(list1),
}
fields = field.list(save["type"]).list()
guid_rule_datas = request.form.get("guid_rule")
save["guid_rule"] = getFeildIdByTitle(guid_rule_datas, save["type"])
seed = Seed(current_app)
sid = seed.add(**save)
if sid:
seed_field = Seed_fields(current_app)
# page_types = request.form.getlist("page_type[]")
for field in fields:
seed_value = {}
seed_value["seed_id"] = sid
seed_value["field_id"] = field.id
seed_value["value"] = request.form.get(field.name)
seed_value["page_type"] = request.form.get("page_type_" + field.name)
seed_value["fetch_all"] = (
request.form.get("fetch_all_" + field.name) and request.form.get("fetch_all_" + field.name) or 0
)
seed_field.add(**seed_value)
# 插入标签
tags_data = request.form.get("tags")
tags_data = tags_data.split(",")
tags_model = Tags(current_app)
seed_tag = Seed_tag(current_app)
for tag in tags_data:
tag = tag.strip()
if tag:
ishere = tags_model.findByName(tag).list()
if len(ishere) > 0:
tid = ishere[0]["id"]
else:
tid = tags_model.add(name=tag)
ishere = seed_tag.view(sid, tid).list()
if len(ishere) == 0:
seed_tag.add(sid=sid, tid=tid)
url = getReferer()
url = url and url or url_for("seeds.index")
return redirect(url)
fields = field.getSeedType()
if request.method == "GET" and request.args.get("type"):
setReferer()
size = request.cookies.get("editorSize")
if not size:
size = 50
seed_data = {}
seed_data["rule"] = {}
seed_type = request.args.get("type")
fields = field.list(seed_type).list()
seed_field = Seed_fields(current_app)
page_types = seed_field.getpageType()
alltags = []
tags_model = Tags(current_app)
alltags_data = tags_model.list()
for tag in alltags_data:
alltags.append(tag["name"])
return render_template(
"seed/add.html",
seed_type=seed_type,
fields=fields,
seed_data=seed_data,
page_types=page_types,
alltags=alltags,
size=size,
)
return render_template("seed/select_type.html", fields=fields)
extrarules = self.parent.extrarules
for i, rule in enumerate(extrarules):
if rule and rule["page_type"] == "content":
field_id = rule["field_id"]
value = rule["value"]
fetch_all = rule["fetch_all"]
self.extrarules.append((field_id, value, fetch_all))
if "filters" in parent.rule:
self.filters = []
if parent.rule["filters"]:
parent.rule["filters"] = parent.rule["filters"].split("|");
for f in parent.rule["filters"]:
self.filters.append(f)
else:
self.filters = []
if __name__ == "__main__":
from web.models import Seed as Seed_Model
db = Seed_Model();
r = db.view(8);
t = Seed(r.list()[0])
# test Seed info
rule = t.getRule();
#test rule
rule.getListRule();
#test article
rule.getArticleRule();