def get_latest():
r = redis.Redis(host=parameter.REDIS_SERVER(),
port=parameter.REDIS_PORT(),
decode_responses=True)
latest = r.hgetall('latest')
res_dict = {}
if 'timestamp' not in latest:
res_dict['date'] = 'No record'
else:
res_dict['date'] = datetime.fromtimestamp(int(
latest['timestamp']), LOCAL_TZ).strftime(TIME_FORMAT)
site_list = [item['site'] for item in parameter.SITE_CFG()]
for site in site_list:
if site in latest:
if latest[site] == 'NOT RUNNING':
res_dict[site] = 'NOT RUNNING'
continue
json_data = json.loads(latest[site])
err_item_list = []
for counter in json_data:
err_item_list.append('{} fail {} times'.format(
counter[5:], json_data[counter]))
res_dict[site] = err_item_list
else:
res_dict[site] = 'pass'
return jsonify(res_dict)
def fail_count():
time_range = request.args.get('range', default=0, type=int)
time_period = request.args.get('periodUnit', default='days', type=str)
if time_range == 0:
return render_template('failCount.html',
time_range=time_range,
time_period=time_period)
if time_period == 'min':
delta_time = timedelta(minutes=time_range)
elif time_period == 'hr':
delta_time = timedelta(hours=time_range)
else:
delta_time = timedelta(days=time_range)
site_list = [item['site'] for item in parameter.SITE_CFG()]
r = redis.Redis(host=parameter.REDIS_SERVER(),
port=parameter.REDIS_PORT(),
decode_responses=True)
index, redis_key = r.scan(cursor=0,
match='LOG:*',
count=parameter.REDIS_SCAN_AMOUNT())
redis_key.sort(reverse=True)
target_ts = int((datetime.now() - delta_time).timestamp())
res_data = {}
for key in redis_key:
hash_data = r.hgetall(key)
timestamp = int(key[4:])
if timestamp < target_ts:
break
for site in hash_data:
json_data = json.loads(hash_data[site])
if site not in res_data:
res_data[site] = json_data
else:
res_data[site] = dict(
Counter(res_data[site]) + Counter(json_data))
for site in res_data:
err_item_list = []
for counter in res_data[site]:
err_item_list.append('{} fail {}次'.format(counter[5:],
res_data[site][counter]))
res_data[site] = err_item_list
return render_template('failCount.html',
timeRange=time_range,
timePeriod=time_period,
redisLog=res_data,
siteList=site_list)
def log_list():
site_list = [item['site'] for item in parameter.SITE_CFG()]
page = request.args.get('page', default=1, type=int)
r = redis.Redis(host=parameter.REDIS_SERVER(),
port=parameter.REDIS_PORT(),
decode_responses=True)
log_period = parameter.LOGGING_PERIOD()
redis_log_list = []
index, redis_key = r.scan(cursor=0,
match='LOG:*',
count=parameter.REDIS_SCAN_AMOUNT())
redis_key.sort(reverse=True)
log_num = len(redis_key)
total_page = math.ceil(log_num / PAGE_SIZE)
if page == 0:
key_list = redis_key
else:
start = (page - 1) * PAGE_SIZE
if page > total_page:
key_list = []
elif page == total_page:
key_list = redis_key[start:]
else:
key_list = redis_key[start:(start + PAGE_SIZE)]
for key in key_list:
res_data = {}
for site in site_list:
res_data[site] = ''
hash_data = r.hgetall(key)
timestamp = int(key[4:])
redislog_time = datetime.fromtimestamp(timestamp,
LOCAL_TZ).strftime(TIME_FORMAT)
res_data['logTime'] = redislog_time
for site in hash_data:
json_data = json.loads(hash_data[site])
err_item_list = []
for counter in json_data:
err_item_list.append('{} fail {}次'.format(
counter[5:], json_data[counter]))
res_data[site] = err_item_list
redis_log_list.append(res_data)
return render_template('logList.html',
redisLogList=redis_log_list,
totalPage=total_page,
currentPage=page,
pageAmount=PAGE_SIZE,
siteList=site_list,
logPeriod=log_period)
def fail_alert(self, send_email=True):
check_period = parameter.FAIL_ALERT_PERIOD_MINS()
fail_thres = parameter.FAIL_ALERT_THRESHOLD()
delta_time = timedelta(minutes=check_period)
has_failure = False
mail_content = ''
site_list = [item['site'] for item in parameter.SITE_CFG()]
r = redis.Redis(host=parameter.REDIS_SERVER(),
port=parameter.REDIS_PORT(),
decode_responses=True)
index, redis_key = r.scan(cursor=0,
match='LOG:*',
count=parameter.REDIS_SCAN_AMOUNT())
redis_key.sort(reverse=True)
target_ts = int((datetime.now() - delta_time).timestamp())
res_data = {}
for key in redis_key:
hash_data = r.hgetall(key)
timestamp = int(key[4:])
if timestamp < target_ts:
break
for site in hash_data:
json_data = json.loads(hash_data[site])
if site not in res_data:
res_data[site] = json_data
else:
res_data[site] = dict(
Counter(res_data[site]) + Counter(json_data))
for site in res_data:
period_err = 0
for counter in res_data[site]:
period_err += res_data[site][counter]
if period_err >= fail_thres:
has_failure = True
mail_content += '{}: hit fail alert threshold\n'.format(site)
if has_failure:
if send_email:
self.send_fail_letter(parameter.FAIL_ALERT_SUB(), mail_content,
None)
def login():
if request.method == 'GET':
return render_template('auth.html')
r = redis.Redis(host=parameter.REDIS_SERVER(),
port=parameter.REDIS_PORT(),
decode_responses=True)
user_id = request.form['userId']
if r.hexists('users', user_id):
hash_pw = hashlib.md5(request.form['password'].encode()).hexdigest()
store_pw = json.loads(r.hget('users', user_id))['password']
if hash_pw == store_pw:
user = User()
user.id = user_id
login_user(user)
return redirect('/log/newest')
flash('Input wrong User ID or PASSWORD', 'warning')
return render_template('auth.html')
def hook():
all_args = request.args
r = redis.Redis(host=parameter.REDIS_SERVER(),
port=parameter.REDIS_PORT(),
decode_responses=True)
r.delete('hook')
res_data = {}
for key, val in request.args.items():
r.hset('hook', key, unquote(val))
res_data.update(request.args)
if request.get_json():
data = request.get_json()
for key, val in data.items():
r.hset('hook', key, json.dumps(val))
res_data[key] = val
else:
data = request.get_data()
if len(data) > 0:
res_data['raw_res'] = str(data)
return jsonify(res_data)
def check_log(self, send_email=True):
finish_pattern = re.compile(r'Ran (\d*) tests in (\d*).(\d*)s')
fail_pattern = re.compile(r'FAIL: (\w*) ')
error_pattern = re.compile(r'ERROR: (\w*) ')
site_cfg = parameter.SITE_CFG()
mail_content = ''
mail_attach_list = []
has_failure = False
r = redis.Redis(host=parameter.REDIS_SERVER(),
port=parameter.REDIS_PORT(),
decode_responses=True)
redis_log_time = int(datetime.now().timestamp())
# clear last latest_redis_log
r.delete('latest')
r.hset('latest', 'timestamp', redis_log_time)
for config in site_cfg:
log_list = [
f for f in glob.glob('/log/' + config['site'] +
'_test_result_*.log')
]
error_count = defaultdict(int)
for log in log_list:
log_content = ''
fail_list = []
error_list = []
log_time = get_log_ts(log)
if datetime.now() < (log_time + timedelta(
minutes=parameter.TEST_DURATION_MINUTES())):
continue
with open(log, 'r') as f:
log_content = f.read()
# Parsing test which finish successfully
if len(finish_pattern.findall(log_content)) == 1:
fail_list = fail_pattern.findall(log_content)
error_list = error_pattern.findall(log_content)
if len(fail_list) > 0 or len(error_list) > 0:
has_failure = True
mail_content += 'Site: {} at {}: fail item\n'.format(
config['site'],
log_time.astimezone(
pytz.timezone(parameter.LOCAL_TIMEZONE())))
mail_content += ' FAIL: {}\n'.format(fail_list)
mail_content += ' ERROR: {}\n'.format(error_list)
mail_content += '\n'
mail_content += '\n'
mail_attach_list.append(log)
mail_attach_list.append(get_api_res_record(log))
for fail in fail_list:
error_count[fail] += 1
for error in error_list:
error_count[error] += 1
else:
os.remove(log)
os.remove(get_api_res_record(log))
# Record test which hanged for long time.
else:
has_failure = True
mail_content += 'Site: {}, at {} unfinished\n\n\n'.format(
config['site'],
log_time.astimezone(
pytz.timezone(parameter.LOCAL_TIMEZONE())))
os.remove(log)
os.remove(get_api_res_record(log))
if error_count:
mail_content += 'Fail of site {} in check period\n'.format(
config['site'])
redis_data_dict = {}
for key in error_count:
mail_content += '{}: {}\n'.format(key, error_count[key])
redis_data_dict[key] = error_count[key]
mail_content += '\n\n'
redis_data_key = 'LOG:' + str(redis_log_time)
redis_data_field = config['site']
r.hset(redis_data_key, redis_data_field,
json.dumps(redis_data_dict))
r.hset('latest', redis_data_field, json.dumps(redis_data_dict))
r.expire(redis_data_key, time=LOG_TIMEOUT_SECS)
# If a site doesn't have testing log in recent check period, mark it in NOT RUNNING status
elif not log_list:
redis_data_field = config['site']
r.hset('latest', redis_data_field, 'NOT RUNNING')
has_failure = True
mail_content += f'Site: {config["site"]} has no testing result in this checking period.\n\n\n'
if has_failure:
try:
send_slack_msg(mail_content)
except Exception as e:
mail_content += f'Send msg to slack fail at: {e}'
if send_email:
self.send_fail_letter(parameter.NOTIF_MAIL_SUB(), mail_content,
mail_attach_list)
# Backup fail_log for download
print(f'MA\n{mail_attach_list}')
if mail_attach_list:
zip_file = '/webpage/static/testLog/{}.zip'.format(
redis_log_time)
compress_log_to_zip(zip_file, mail_attach_list)
for rm_log in mail_attach_list:
os.remove(rm_log)