def zRangeRemByScore(key, _min=0, _max=None):
if _max == None: _max = TMSL()
if MyRedis.redis_status:
try:
lines = MyRedis.rs.zrangebyscore(key, _min, _max)
MyRedis.rs.zremrangebyscore(key, _min, _max)
return [eval(line) for line in lines]
except:
L.error('zrangeRemByScore error', key)
return []
def request_data(url, name):
err_count = 0
while True:
try:
rst = request_url(url)
rst = json.loads(rst, encoding="utf8")
return rst
except:
err_count += 1
L.info("Error request found when {}, the {} times".format(
name, err_count))
if err_count > 10: return None
time.sleep(3)
return None
def run(self, sql):
cnx = self.getConnection()
cursor = cnx.cursor()
emp_no = 0
try:
cursor.execute(sql)
emp_no = cursor.lastrowid
cnx.commit()
except Exception as e:
emp_no = -1
L.error("Run Error: " + sql)
L.error(e)
'''End Try'''
cursor.close()
cnx.close()
return emp_no
def detection_data_citys():
db = Database()
unknowns = {}
sql = '''SELECT region_parent, region_name FROM patients WHERE region_level=2
group by region_parent, region_name'''
for (parent, name) in db.select(sql):
if parent not in REGIONS:
L.error("Not in Region Source: {}".format(parent))
continue
if check_city(name, parent): continue
if parent not in unknowns: unknowns[parent] = set()
unknowns[parent].add(name)
for p in unknowns:
print('----------------------')
print([v['name'] for v in REGIONS[p]['children'].values()])
print(list(unknowns[p]))
def getConnection(self):
'''检查是否初始化'''
if not Database.pool:
Database.pool = PooledDB(mysql.connector,
cfg.POOL_SIZE,
host=cfg.HOST,
port=cfg.PORT,
user=cfg.USER_NAME,
passwd=cfg.PASSWORD,
db=cfg.DB_NAME,
use_unicode=True,
charset='utf8')
cnx = None
try:
cnx = Database.pool.connection()
self.lsConn.append(cnx)
except:
L.error('pool error:' + str(len(self.lsConn)))
return cnx
def generate_weibo_topics():
L.info("Start update weibo topic.")
db_weibo = pymysql.connect("localhost", "root", "root", "weibo")
cursor = db_weibo.cursor()
cursor.execute(
"select publish_time, content from weibo order by publish_time asc")
data = cursor.fetchall()
db = Database()
now = datetime.date(2019, 12, 31)
news = ""
topic = {}
sql = "insert into topic (date, topic) values (%s, %s)"
for line in data:
if line[0].date() == now:
news += line[1]
else:
topic.clear()
for keyword, weight in textrank(news, topK=20, withWeight=True):
topic[keyword] = weight
db.execute(sql, [now.strftime("%Y-%m-%d %H:%M:%S"), str(topic)])
L.info("\tNow processing {}".format(now.strftime("%Y-%m-%d")))
now = line[0].date()
news = line[1]
topic.clear()
for keyword, weight in textrank(news, topK=20, withWeight=True):
topic[keyword] = weight
db.execute(sql, [now.strftime("%Y-%m-%d %H:%M:%S"), str(topic)])
L.info("\tFinished update weibo topic.")
def web_login_common(request, no_code=False):
ip = get_client_ip(request)
if check_ip_limit(ip): return ErrorResponseJson("温馨提示：操作过于频繁，请稍后再试！")
R = request.form if request.method == 'POST' else request.args
client = str(R.get('mcode', '0'))
securityCode, codeTms = str(R.get('securityCode',
'')), str(R.get('tms', ''))
correctCode = str(rs.hget('securityCode', codeTms, False))
if not no_code and (codeTms == '' or securityCode == ''
or correctCode != securityCode):
return ErrorResponseJson('验证码有误， 请重新输入！', None, '103-02')
username, password = R.get('username', ''), R.get('password', '')
data = user_login(username, password, ip, "", no_code)
if data['error']: return ErrorResponseData(data)
L.info('User Login: %s, %s' % (username, ip))
'''TODO: 定期删除TKS或设置redis过期时间，防止太长'''
U = data['data']
user_info = {
'name': username,
'level': U['level'],
'type': U['type'],
'id': U['id']
}
'''记录用户和登录信息'''
token = create_token(username, user_info)
data = {
'u': username,
'tk': token,
'lv': U['level'],
"ip": ip,
"client": client,
"loginTime": TM()
}
return data
def request_news_time_series():
L.info("Start update news time series.")
with open('./json/DXYNews-TimeSeries.json', 'r') as file:
data = json.load(file)
L.info("\tRead {} lines from file".format(len(data)))
db = Database()
update_num = 0
for line in data:
key = [
'id', 'provinceId', 'title', 'summary', 'infoSource', 'sourceUrl',
'pubDate', 'province'
]
data = db.select("select * from news where id={}".format(line['id']))
if not data:
update_num += 1
sql = "insert into news (" + ','.join(
key) + ") values (" + ', '.join(['%s' for k in key]) + ")"
line['pubDate'] = TS2S(line['pubDate'] / 1000.0)
if line['provinceId'] == "":
line['provinceId'] = None
line['summary'] = line['summary'][0:4096]
if 'province' not in line:
line['province'] = None
params = [line[k] for k in key]
db.execute(sql, params)
L.info('\tUpdate {} news data.'.format(update_num))
def generate_topics():
L.info("Start update topic.")
db = Database()
db.run("delete from topic")
data = db.select("select pubDate, summary from news order by pubDate asc")
now = datetime.date(2019, 12, 31)
news = ""
topic = {}
today = []
yesterday = []
dead = []
new = []
sql = "insert into topic (date, topic, dead, new) values (%s, %s, %s, %s)"
for line in data:
if line[0].date() == now:
news += line[1]
else:
topic.clear()
today.clear()
for keyword, weight in textrank(news, topK=40, withWeight=True):
topic[keyword] = weight
today.append(keyword)
for keyword in today:
if keyword not in yesterday:
new.append(keyword)
for keyword in yesterday:
if keyword not in today:
dead.append(keyword)
db.execute(sql, [
now.strftime("%Y-%m-%d %H:%M:%S"),
str(topic),
str(dead),
str(new)
])
L.info("\tNow processing {}".format(now.strftime("%Y-%m-%d")))
now = line[0].date()
news = line[1]
yesterday = today.copy()
new.clear()
dead.clear()
topic.clear()
today.clear()
for keyword, weight in textrank(news, topK=20, withWeight=True):
topic[keyword] = weight
today.append(keyword)
for keyword in today:
if keyword not in yesterday:
new.append(keyword)
for keyword in yesterday:
if keyword not in today:
dead.append(keyword)
db.execute(
sql,
[now.strftime("%Y-%m-%d %H:%M:%S"),
str(topic),
str(dead),
str(new)])
L.info("\tFinished update topic.")
def Transaction(self, lines, cnx=None, cursor=None):
if cnx == None: cnx = self.getConnection()
if cursor == None:
cursor = cnx.cursor()
self.lsCurs.append(cursor)
flag, lsRst = True, []
'''开启事务'''
cursor.execute("BEGIN;")
try:
for line in lines:
cursor.execute(line[0], line[1])
lsRst.append(cursor.lastrowid)
cnx.commit()
except Error as e:
cnx.rollback()
flag = False
L.error('Error: ' + str(e))
'''End For'''
cursor.close()
cnx.close()
return flag, lsRst
def zadd(key, value, score):
# 暂时不支持本地模式
MyRedis.checkConn()
if MyRedis.redis_status:
try:
if MyRedis.rs.zcard(key) < MyRedis.ZMaxCount:
return MyRedis.rs.zadd(key, value, score)
else:
return L.error(
'Collection size is too big over ZMaxCount when zadd')
except:
MyRedis.SwitchToLocal()
if key not in MyRedis.dataHash: MyRedis.dataHash[key] = {}
MyRedis.dataHash[key][score] = value
return False
def user_login(user, passwd, ip, address='', isApp=None):
db = Database()
sql = ("select password, status, level, nickname, type, id from user \
where username = %s and isDel!=1")
params = (user, )
L.info("user {} login from ip: {}".format(user, ip))
dt = db.selectEx(sql, params)
rst = {'error': 1, 'message': 'success', 'data': {}}
print(dt)
if len(dt) == 0:
rst['message'] = '用户不存在'
elif dt[0][0] != passwd and get_md5(dt[0][0]) != passwd:
rst['message'] = '密码错误'
else:
if dt[0][1] != 0:
rst['message'] = '用户登陆受限'
else:
rst['error'] = 0
rst['data'] = {
'level': dt[0][2],
'type': dt[0][4],
'id': dt[0][5],
'nickname': dt[0][3].encode("utf8") if dt[0][3] else '未命名'
}
'''End If'''
'''登陆成功'''
if rst['error'] == 0:
sqlU, paramsU = (
"update user set loginTime = %s, onlineStatus = %s where \
username = %s"), (TM(), 1, user)
sqlL, paramsL = ("insert into logs (user, ip, address, client) values (%s, %s, %s, %s)"), \
(user, ip, address, 'app' if isApp else 'pc')
db.Transaction([[sqlU, paramsU], [sqlL, paramsL]])
return rst
def request_data_time_series():
L.info("Start update data time series.")
needed = {}
for p in SP:
if p['name'] == '香港特别行政区':
needed['香港'] = p
if p['name'] == '澳门特别行政区':
needed['澳门'] = p
if p['name'] == '台湾省':
needed['台湾'] = p
needed[p['name']] = p
with open('./json/DXYArea-TimeSeries.json', 'r') as file:
data = json.load(file)
db = Database()
history = getLatest(db)
lines = []
for province in data:
if province["provinceName"] in needed:
p = needed[province["provinceName"]]
translate([province], p, lines)
L.info("\tRead {} lines from file".format(len(lines)))
comands = []
for line in lines:
'''排除已有历史数据'''
key = '_'.join([
str(line['region_code']), line['region_name'],
str(line['region_parent'])
])
if key in history and not date_less(history[key], line['data_date']):
continue
ks = line.keys()
sql = "insert into patients (" + ','.join(
ks) + ") values (" + ', '.join(['%s' for k in ks]) + ")"
params = [line[k] for k in ks]
comands.append([sql, params])
L.info("\tNew data lines count:" + str(len(comands)))
if len(comands) > 0:
db.Transaction(comands)
def request_rumor_time_series():
L.info("Start update rumor time series.")
with open('./json/DXYRumors-TimeSeries.json', 'r') as file:
data = json.load(file)
L.info("\tRead {} lines from file".format(len(data)))
db = Database()
update_num = 0
for line in data:
key = [
'id', 'title', 'mainSummary', 'body', 'sourceUrl', 'rumorType',
'crawlTime'
]
data = db.select("select * from rumor where id={}".format(line['id']))
if not data:
update_num += 1
sql = "insert into rumor (" + ','.join(
key) + ") values (" + ', '.join(['%s' for k in key]) + ")"
line['crawlTime'] = TS2S(line['crawlTime'] / 1000.0)
line['mainSummary'] = line['mainSummary'][0:1024]
line['body'] = line['body'][0:1024]
params = [line[k] for k in key]
db.execute(sql, params)
L.info('\tUpdate {} rumor data.'.format(update_num))
def request_data_province():
names = {'香港特别行政区': '香港', '澳门特别行政区': '澳门', '台湾省': '台湾'}
url = "https://lab.isaaclin.cn/nCoV/api/area?latest=1&province="
db = Database()
history = getLatest(db)
idx = 0
for p in SP:
idx += 1
# if idx <= 23: continue
purl = url + parse.quote(names.get(p['name'], p['name']))
rst = request_data(purl, p['name'])
if not rst: continue
data, lines = rst['results'], []
translate(data, p, lines)
L.info("Get data collects count:" + str(len(data)))
comands = []
for line in lines:
'''排除已有历史数据'''
key = '_'.join([
str(line['region_code']), line['region_name'],
str(line['region_parent'])
])
if key in history and not date_less(history[key],
line['data_date']):
continue
ks = line.keys()
sql = "insert into patients (" + ','.join(
ks) + ") values (" + ', '.join(['%s' for k in ks]) + ")"
params = [line[k] for k in ks]
comands.append([sql, params])
L.info("New data lines count:" + str(len(comands)))
if len(comands) > 0: db.Transaction(comands)
L.info("{}\t {} finished!".format(idx, p['name']))
time.sleep(3)
def is_login(user):
L.info("User %s Request login status" % user)
return NormalResponseJson({})
def request_news_data():
L.info("Collecting news data.")
error_times = 0
db = Database()
url = "https://lab.isaaclin.cn/nCoV/api/news?num=50&page="
i = 0
while i < 100:
i += 1
L.info("Preparing for page {}".format(str(i)))
time.sleep(3)
rst = request_data(url + str(i), str(i))
if rst['success'] == False:
if error_times == 10: break
error_times += 1
L.info(
"This is the {} times fail at getting page {}, will try it again."
.format(str(error_times), str(i)))
i -= 1
continue
else:
error_times = 0
if not rst['results']:
L.info("Collecting news data finished.")
break
comands = []
for line in rst['results']:
ks = line.keys()
params = line['title']
sql = "select * from news where title ='{}'".format(params)
data = db.select(sql)
if not data:
sql = "insert into news (" + ','.join(
ks) + ") values (" + ', '.join(['%s' for k in ks]) + ")"
params = [line[k] for k in ks]
comands.append([sql, params])
try:
db.Transaction(comands)
L.info("Writing database for {} news.".format(str(len(comands))))
except Exception as e:
if error_times == 10: break
error_times += 1
L.info(
"This is the {} times fail at wirting database due to {}, will try this page again."
.format(str(error_times), str(e)))
i -= 1
continue
if error_times == 10:
L.info("Collecting news data finished with error.")
else:
L.info("Collecting news data finished.")
def request_rumor_data():
L.info("Collecting rumor data.")
for i in range(0, 3):
request_rumor_type_data(i)
def SwitchToLocal():
raise RuntimeError('Redis Error')
if not C.DEV_MODE: return
MyRedis.redis_status = False
L.error("RedisEx find error in Function: " + inspect.stack()[1][3])
def worker():
try:
request_data_province()
L.info("Sleep now, worker will run after 30 minutes")
except Exception as e:
L.info("Someting went wrong because {}".format(str(e)))
@auth_required('lv0')
def _get_task_control(user, R):
tid = R.get('tid', '')
multi = R.get('multi')
data = T.get_task_control(tid, user)
return NormalResponseJson(data)
@app.route('/submitReport', methods=['GET', 'POST'])
@auth_required('lv0')
def submit_report(user, R):
params = R.get('params', '{}')
data = json.loads(params, encoding="utf8")
if not data: return ErrorResponseJson("参数错误")
jobs = data.get('jobs', [])
plans = data.get('plans', [])
flag, data = RC.submit_record(user, jobs, plans)
if not flag: return ErrorResponseJson("提交失败:" + data)
return NormalResponseJson(data)
if __name__ == '__main__':
L.info('server start on: %s !' % (C.PORT))
app.debug=False
app.run(host='0.0.0.0', port=C.PORT)
def worker():
try:
request_data_province()
L.info("Sleep now, worker will run after 30 minutes")
except:
pass
data = DC.get_time_data(level, int(code))
return NormalResponseJson(request, data)
@app.route('/getDataPos')
def get_data_pos():
R = request.form if request.method == 'POST' else request.args
code = R.get('code', '420000')
data = DP.get_region_data(int(code))
return NormalResponseJson(request, data)
@app.route('/getMap')
def get_map():
R = request.form if request.method == 'POST' else request.args
name = R.get('id', '')
path = FILE_PATH + "/data/geojson/{}.json".format(name)
if not os.path.exists(path):
return ErrorResponseJson("地图文件不存在")
with open(path, encoding='utf8') as fp:
data = ''.join(fp.readlines())
return NormalResponseJson(request, data)
if __name__ == '__main__':
L.info("Server Start...")
app.run(port=C.web.PORT)