def editconfig():
id = request.args.get("id", type=int)
if id:
db = DB()
try:
ret = six.next(db._select2dic("scraper_setting", where="id = ?", where_values=[id]))
form = AddConfigForm()
if form.validate_on_submit():
# spider_ip = form.spider_ip.data
url = form.url.data
active = form.active.data
source = form.source.data
if int(active) == 0:
db._update("scraper_setting", where="id=?", where_values=[id], source=source, url=url,
active=active, spider_ip="")
else:
db._update("scraper_setting", where="id=?", where_values=[id], source=source, url=url,
active=active)
db.commit()
return redirect(url_for("configlist"))
else:
flash_errors(form)
form.source.data = ret['source']
# form.spider_ip.data = ret['spider_ip']
form.url.data = ret['url']
form.active.data = str(ret['active'])
return render_template("addconfig.html", form=form)
except StopIteration:
return redirect(url_for("configlist"))
return redirect(url_for("configlist"))
def getconfig():
if request.headers.getlist("X-Forwarded-For"):
ip = request.headers.getlist("X-Forwarded-For")[0]
else:
ip = request.remote_addr
sources = ["craigslist", "ebay", "kijiji", "facebook"]
db = DB()
data = []
for source in sources:
try:
ret = six.next(
db._select2dic("scraper_setting", where="source = ? and spider_ip = ? and active = 1",
where_values=[source, ip]))
url = ret['url']
id = ret['id']
except:
try:
ret = six.next(
db._select2dic("scraper_setting", where="source = ? and active = 0", where_values=[source]))
url = ret['url']
id = ret['id']
db._update("scraper_setting", where="id = ?", where_values=[id], active=1, spider_ip=ip)
db.commit()
except StopIteration:
url = ""
id = 0
data.append({
"url": url,
"source": source,
"id": id,
})
return jsonify(data), 200
def save():
id = request.args.get("id", type=int)
if id:
db = DB()
try:
six.next(db._select2dic("scraper_craigslist", where="id = ?", where_values=[id]))
except StopIteration:
return jsonify({"code": 404, "message": "not found", "data": None})
db._update("scraper_craigslist", where="id = ?", where_values=[id], is_save=1)
db.commit()
return jsonify({"code": 200, "message": "success", "data": None})
return jsonify({"code": 500, "message": "missing id", "data": None})
def addcomment():
id = request.form["id"]
comment = request.form["comment"]
if id:
db = DB()
try:
six.next(db._select2dic("scraper_craigslist", where="id = ?", where_values=[id]))
except StopIteration:
return jsonify({"code": 404, "message": "not found", "data": None})
db._update("scraper_craigslist", where="id = ?", where_values=[id], comments=comment)
db.commit()
return jsonify({"code": 200, "message": "success", "data": None})
return jsonify({"code": 500, "message": "missing id", "data": None})
def keyword_filter():
form = AddKeywordsFilterForm()
db = DB()
ret = None
try:
ret = six.next(db._select2dic("keyword_filter"))
except:
pass
if form.validate_on_submit():
keywords = form.keywords.data
if ret:
db._update("keyword_filter", where="id = ?", where_values=[ret['id']], keywords=keywords)
db.commit()
return render_template("keyword_filter.html", form=form)
else:
db._insert("keyword_filter", **{"keywords": keywords})
db.commit()
return redirect(url_for("keyword_filter"))
else:
flash_errors(form)
if ret:
form.keywords.data = ret['keywords']
return render_template("keyword_filter.html", form=form)
def statusswitch():
id = request.args.get("id", type=int)
active = request.args.get("active", type=int)
if id and active is not None:
db = DB()
try:
active = 0 if active == 1 else 1
ret = six.next(db._select2dic("scraper_setting", where="id = ?", where_values=[id]))
if int(active) == 0:
db._update("scraper_setting", where="id=?", where_values=[id], active=active, spider_ip="")
else:
db._update("scraper_setting", where="id = ?", where_values=[id], active=active)
ret['active'] = active
ret['spider_ip'] = "" if active == 0 else ret['spider_ip']
db.commit()
active_status = "inactive" if ret['active'] == 0 else "active"
ret['active_status'] = active_status
html = """
<td>%s</td>
<td>%s<br><a id="switch" href="#" data-id="%s" data-active="%s">Switch</a></td>
<td>%s</td>
<td>
<a href="/deleteconfig?id=%s">delete</a>
<a href="/editconfig?id=%s">edit</a>
</td>
<td>%s</td>
""" % (
ret['source'], ret["active_status"], ret['id'], ret['active'], ret['spider_ip'],
ret['id'], ret['id'], ret["url"],)
return jsonify({"code": 200, "message": "success", "data": html})
except StopIteration:
return jsonify({"code": 404, "message": "not found", "data": None})
return jsonify({"code": 500, "message": "missing id", "data": None})
def spider_status():
id = request.form["id"]
status = request.form["status"]
if request.headers.getlist("X-Forwarded-For"):
ip = request.headers.getlist("X-Forwarded-For")[0]
else:
ip = request.remote_addr
if id and status:
db = DB()
try:
six.next(
db._select2dic("scraper_setting", where="id = ?", where_values=[id]))
if status == "Sleeping":
db._update("scraper_setting", where="spider_ip = ?", where_values=[ip], spider_status=status,
last_full_scan_time=int(time.mktime(time.gmtime())))
else:
db._update("scraper_setting", where="spider_ip = ?", where_values=[ip], spider_status=status)
db.commit()
return jsonify({"code": 200, "message": "success", "data": None})
except StopIteration:
return jsonify({"code": 404, "message": "not found", "data": None})
return jsonify({"code": 500, "message": "missing params", "data": None})