def article_keyword_list():
try:
res = request.get_json()
page = res.get('page', 1)
pagesize = res.get('pagesize', 10)
Logging.logger.info('request_args:{0}'.format(res))
try:
page = int(page)
pagesize = int(pagesize)
except Exception as e:
Logging.logger.error('errmsg:{0}'.format(e))
page, pagesize = 1, 10
results = ArticleKeyWords.query.paginate(page, pagesize, False)
count = results.total
keyword_list = list()
redis_client = RedisClient.create_redis_cli()
redis_client.delete('group_id_of_keyword')
first_ls = list()
second_ls = list()
i = (page - 1) * pagesize + 1
for result in results.items:
keyword_dict = dict()
first_keyword = result.first_keywords
second_keyword = result.second_keywords
keyword_dict['keyword_id'] = result.id
keyword_dict['article_group_id'] = result.article_group_id
group = Group.query.get(result.article_group_id)
keyword_dict['article_group_name'] = group.name
keyword_dict['first_keywords'] = first_keyword
keyword_dict['second_keywords'] = second_keyword
keyword_dict['location'] = i
keyword_list.append(keyword_dict)
i += 1
# 关键词对应分类ID存Redis
for first_word in first_keyword.split(','):
redis_client.hset('group_id_of_keyword', first_word,
result.article_group_id)
for second_word in second_keyword.split(','):
redis_client.hset('group_id_of_keyword', second_word,
result.article_group_id)
first_ls.append(first_keyword)
second_ls.append(second_keyword)
# 关键词数据存入Redis
first_keywords = ','.join(first_ls)
second_keywords = ','.join(second_ls)
redis_client.set('first_keywords', first_keywords)
redis_client.set('second_keywords', second_keywords)
return jsonify(errno=0, errmsg="OK", data=keyword_list, count=count)
except Exception as e:
Logging.logger.error('errmsg:{0}'.format(e))
db.session.rollback()
return jsonify(errno=-1, errmsg='网络异常')
def del_article_keyword():
try:
res = request.get_json()
keywords_id_list = res.get('keywords_id_list')
Logging.logger.info('request_args:{0}'.format(res))
if not isinstance(keywords_id_list, list):
return jsonify(errno=-1, errmsg='参数错误')
redis_client = RedisClient.create_redis_cli()
for keyword_id in keywords_id_list:
if not keyword_id:
return jsonify(errno=-1, errmsg="参数错误,请传入删除文章关键词ID")
obj = ArticleKeyWords.query.get(keyword_id)
if not obj:
return jsonify(errno=-1, errmsg="参数错误,文章关键词ID不存在")
first_keywords = obj.first_keywords
second_keywords = obj.second_keywords
for first_word in first_keywords.split(','):
redis_client.hdel('group_id_of_keyword', first_word)
for second_word in second_keywords.split(','):
redis_client.hdel('group_id_of_keyword', second_word)
db.session.delete(obj)
db.session.commit()
return jsonify(errno=0, errmsg="OK")
except Exception as e:
Logging.logger.error('errmsg:{0}'.format(e))
db.session.rollback()
return jsonify(errno=-1, errmsg='网络异常')
def has_limit(openid, api):
result = redis_store.hexists(name=openid, key=api)
now = int(time.time())
data = {"times": 1, "ttl": now}
redis_cli = RedisClient.create_redis_cli()
if not result:
redis_cli.hset(name=openid, key=api, value=json.dumps(data))
else:
res = redis_cli.hget(name=openid, key=api)
res = json.loads(res)
Logging.logger.info('用户接口访问状态:{0}'.format(res))
times = res.get('times')
ttl = res.get('ttl')
if times >= 10:
if now - ttl <= 90:
return True # 超出访问限制, 30s后可访问
else:
redis_cli.hdel(openid, api)
if now - ttl <= 60:
data['times'] = times + 1
data['ttl'] = ttl
redis_cli.hset(name=openid, key=api, value=json.dumps(data))
else:
redis_cli.hdel(openid, api)
return False
def __init__(self, title, content):
self.redis_client = RedisClient.create_redis_cli()
self.first_keywords = self.redis_client.get('first_keywords').split(
',')
self.second_keywords = self.redis_client.get('second_keywords').split(
',')
self.title = title
self.content = content
self.group_id_list = list()
def start(self, url, admin_id):
res = self.get_article(url)
if res == 1:
Logging.logger.info('该文章已被发布者删除')
return 1
elif res == -1:
Logging.logger.info('该文章已存在MySQL')
return -1
else:
Logging.logger.info("#" * 20)
web_name = res.get('name')
title = res.get('title')
# 测试文章去重
content = res.get('content')
Logging.logger.info(title)
sim_ls = ArticleModel().run(content)
Logging.logger.info('对比文章相似度的结果:{0}'.format(sim_ls))
if not sim_ls:
Logging.logger.info('获取到的文章公众号是:{0},文章标题是:{1}'.format(
web_name, title))
resp = self.upload_mongo(res, title)
if resp:
article_id = self.upload_mysql(res['title'], res['name'],
res['date'],
res['web_name'], url,
admin_id,
res['is_original'])
print(('article_id=', article_id))
if article_id:
redis_client = RedisClient.create_redis_cli()
redis_client.rpush('article_id_list', article_id)
Logging.logger.info('爬取完成')
else:
# sim_ls = [(1051, 1.0000005)]
redis_client = RedisClient.create_redis_cli()
sim_article_ids = [
redis_client.lindex('article_id_list', item[0])
for item in sim_ls
]
Logging.logger.info(
'已存在相似度大于0.6的文章,相似文章ID:{0}'.format(sim_article_ids))
return 2
def upload_mysql_by_keyword(title, author, wechat_art_date, min_pic, summary, web_name, link, round_head_img, alias,
is_original, group_id_list):
now1 = datetime.datetime.now()
Logging.logger.info("获取到公众号:%s, 文章标题是:%s, 准备插入" % (web_name, title))
data = list()
conn = connect(host='rm-uf6gw23j409s5ui7qmoo.mysql.rds.aliyuncs.com', port=3306,
database='zjlivenew',
user='******', password='******', charset='utf8')
# 获得Cursor对象 "%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s"
cursor = conn.cursor()
try:
sql1 = "insert into zj_article_info (title,author,wechat_art_date,min_pic, summary, web_name, link, round_head_img, alias,is_show,is_big,create_time,is_original) " \
"values ('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')" % (
title, author, wechat_art_date, min_pic, summary, web_name, link, round_head_img, alias, 0,
0, now1, is_original)
cursor.execute(sql1)
article_id = int(conn.insert_id())
redis_client = RedisClient.create_redis_cli()
redis_client.rpush('article_id_list', article_id)
data.append(article_id)
Logging.logger.info("公众号:%s, 文章:%s, 插入成功, 获得文章ID: %s" % (web_name, title, article_id))
if group_id_list and article_id > 0:
group_id_list = set(group_id_list)
now2 = datetime.datetime.now()
for group_id in group_id_list:
if group_id:
# 文章自动上线到关键词对应的文章分类,并显示在最前
sql2 = 'insert into zj_article_group (article_id, group_id, sort_num, create_time) values ("%s", "%s", "%s", "%s")' % (
article_id, group_id, 1, now2)
cursor.execute(sql2)
sql3 = 'update zj_article_info set is_show = 1, zj_art_date="%s" where id="%s"' % (
now2, article_id)
cursor.execute(sql3)
break # 3.11北京要求:文章筛选出现多个分类时,随便上架到一个分类即可
Logging.logger.info("公众号:%s,文章标题:%s, 自动上线成功" % (web_name, title))
data.append(666)
conn.commit()
return data
except Exception as e:
conn.rollback()
Logging.logger.error('公众号:%s,文章标题:%s,自动上线失败:' % (web_name, title))
Logging.logger.error('errmsg:{0}'.format(e))
return False
finally:
cursor.close()
conn.close()
def sensitive_words_ls():
try:
result = ArticleSensitiveWords.query.first()
redis_client = RedisClient.create_redis_cli()
keyword_dict = dict()
if result:
sensitive_words = result.sensitive_words
keyword_dict['sensitive_id'] = result.id
keyword_dict['sensitive_words'] = sensitive_words
# 敏感词数据存入Redis
redis_client.set('sensitive_words', sensitive_words)
return jsonify(errno=0, errmsg="OK", data=keyword_dict)
except Exception as e:
Logging.logger.error('errmsg:{0}'.format(e))
db.session.rollback()
return jsonify(errno=-1, errmsg='网络异常')
def wrapper(*args, **kwargs):
token = request.headers.get("TOKEN")
openid = request.headers.get("OPENID")
ip = request.headers.get("Remoteip")
Logging.logger.info('headers:{0}'.format(request.headers))
if not all([token, openid]):
return jsonify(errno=403, errmsg='用户未授权或非正常访问')
user = User.query.filter(User.token == token,
User.openid == openid).first()
if not user:
return jsonify(errno=403, errmsg='用户未授权或非正常访问')
else:
Logging.logger.info("当前用户昵称:{0}, IP:{1}".format(
user.nick_name, ip))
# 用户限制一分钟内总的接口访问量
redis_cli = RedisClient.create_redis_cli()
num = redis_cli.get(openid)
if not num:
redis_cli.setex(openid, 60, 1)
else:
Logging.logger.info("num:{0}".format(num))
if int(num) >= 300: # 5次/秒/人的接口访问量
# 重置过期时间, 30秒后可访问
Logging.logger.info("num:{0}".format(type(num)))
redis_cli.setex(openid, 30, num)
# 记录访问黑名单
# try:
# obj = BlackAccount()
# obj.ip = ip
# obj.openid = openid
# obj.num = num
# db.session.add(obj)
# db.session.commit()
# except Exception as e:
# Logging.logger.error('record to black_account fail, errmsg:{0}'.format(e))
# db.session.rollback()
return jsonify(errno=403, errmsg='操作过于频繁,请稍后再试')
else:
redis_cli.incrby(openid, 1)
return view_func(*args, **kwargs)
def del_sensitive_words():
try:
res = request.get_json()
sensitive_id = res.get('sensitive_id')
Logging.logger.info('request_args:{0}'.format(res))
obj = ArticleSensitiveWords.query.get(sensitive_id)
if not obj:
return jsonify(errno=-1, errmsg='参数错误')
redis_client = RedisClient.create_redis_cli()
redis_client.delete('sensitive_words')
db.session.delete(obj)
db.session.commit()
return jsonify(errno=0, errmsg="OK")
except Exception as e:
Logging.logger.error('errmsg:{0}'.format(e))
db.session.rollback()
return jsonify(errno=-1, errmsg='网络异常')