def login():
if request.method == 'POST':
api_response = {"code": 0, "msg": "success"}
request_json = request.get_json()
if 'accName' not in request_json or request_json['accName'] is "":
raise ValidationError("参数不能为空")
accName = request_json['accName']
if 'password' not in request_json or request_json['password'] is "":
raise ValidationError("参数不能为空")
password = request_json['password']
try:
user = User.query.filter(User.accName == accName).first()
if user == None:
api_response["code"] = 401
api_response["msg"] = "用户不存在"
return jsonify(api_response), 401
very = user.very_password(password)
if very == False:
api_response["code"] = 401
api_response["msg"] = "账号或密码不正确"
return jsonify(api_response), 401
elif user.status == 0:
api_response["code"] = 401
api_response["msg"] = "账号已被封禁,请联系管理员"
return jsonify(api_response), 401
else:
last_login = Last_Online.query.filter(
Last_Online.accName == accName).first()
if last_login is None:
new_last_login = Last_Online(accName, user.userName,
datetime.now(), 1)
db.session.add(new_last_login)
else:
last_login.last_login_time = datetime.now()
last_login.login_count = last_login.login_count + 1
try:
db.session.commit()
except Exception as ie:
Logger('error.log', level='error').logger.error(
"[事务提交失败]accName:【%s】%s" % (accName, ie))
db.session.rollback()
token = bytes.decode(user.generate_auth_token(2592000))
api_response['expire'] = 3600
api_response['token'] = token
return jsonify(api_response)
except Exception as e:
Logger('error.log', level='error').logger.error(
"[登录异常]accName:【%s】%s" % (accName, e))
api_response["code"] = 500
api_response["msg"] = "服务器未知错误"
return jsonify(api_response), 500
def upload():
if request.method == 'POST':
api_response = {
"code": 0,
"msg": "success"
}
try:
currResource = Resource.query.filter(Resource.perms == "modules:enterprise:save").first()
if currResource==None:
api_response["code"] = 401
api_response["msg"] = "当前操作权限实体不存在"
return jsonify(api_response), 401
very = very_permission(currResource.id)
if very == False:
api_response["code"] = 401
api_response["msg"] = "该账户无操作权限"
return jsonify(api_response), 401
else:
files = request.files.getlist("excelFile")
for file in files:
filename = file.filename
if not os.path.exists(UPLOADFILES_DIRS):
os.makedirs(UPLOADFILES_DIRS)
newFilePath = os.path.join(UPLOADFILES_DIRS, filename)
file.save(newFilePath)
except Exception as e:
Logger('error.log', level='error').logger.error("[上传文件异常]accName:【%s】%s" % (g.user, e))
api_response["code"] = 400
api_response["msg"] = "系统异常"
return jsonify(api_response)
api_response["result"] = filename
return jsonify(api_response)
def nav():
if request.method == 'GET':
api_response = {"code": 0, "msg": "success"}
try:
user = User.query.filter(User.accName == g.user).first()
if user == None:
api_response["code"] = 400
api_response["msg"] = "该账户不存在"
return jsonify(api_response), 400
menuList = []
location = []
currrole = Role.query.filter(Role.id == user.role_id).first()
roleToResourceS = currrole.participateResource.all()
for roleToResource in roleToResourceS:
currResource = Resource.query.filter(
Resource.id == roleToResource.id).first()
if currResource.type == 1:
menuList.append(currResource.name)
location.append(currResource.url)
response = dict(menuList=menuList, location=location)
api_response['result'] = response
return jsonify(api_response)
except Exception as e:
Logger('error.log', level='error').logger.error(
"[获取菜单异常]accName:【%s】%s" % (g.user, e))
api_response["code"] = 500
api_response["msg"] = "服务器未知错误"
return jsonify(api_response), 500
def scp(identity_file, source, target, user, host, verbose=False):
cmd = 'scp -q -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i %s -r "%s" "%s@%s:%s"' % (
identity_file, source, user, host, target)
if verbose:
print cmd
os.system(cmd)
Logger().cmd(cmd)
def custom_error_handler(e):
if e.level in [BaseError.LEVEL_INFO, BaseError.LEVEL_WARN, BaseError.LEVEL_ERROR]:
if isinstance(e, OrmError):
app.logger.error('%s %s' % (e.parent_error, e))
else:
if e.level is BaseError.LEVEL_INFO:
Logger().logger.info('INFO信息: %s %s' % (e.extras, e))
#app.logger.info('INFO信息: %s %s' % (e.extras, e))
elif e.level is BaseError.LEVEL_WARN:
Logger('error.log', level='error').logger.error('告警信息: %s %s' % (e.extras, e))
#app.logger.error('告警信息: %s %s' % (e.extras, e))
else:
Logger('error.log', level='error').logger.error('错误信息: %s %s' % (e.extras, e))
#app.logger.error('错误信息: %s %s' % (e.extras, e))
response = jsonify(e.to_dict())
response.status_code = e.status_code
return response
def editUserPermission():
if request.method == 'POST':
api_response = {"code": 0, "msg": "success"}
try:
currResource = Resource.query.filter(
Resource.perms == "modules:usermanage:update").first()
if currResource == None:
api_response["code"] = 401
api_response["msg"] = "当前操作权限实体不存在"
return jsonify(api_response), 401
very = very_permission(currResource.id)
if very == False:
api_response["code"] = 401
api_response["msg"] = "该账户无操作权限"
return jsonify(api_response), 401
else:
request_json = request.get_json()
if 'accName' not in request_json or request_json[
'accName'] is "":
raise ValidationError("参数不能为空")
accName = request_json['accName']
if 'accType' not in request_json or request_json[
'accType'] is "":
raise ValidationError("参数不能为空")
accType = request_json['accType']
user = User.query.filter(User.accName == accName).first()
if user == None:
api_response["code"] = 400
api_response["msg"] = "所修改账户不存在"
return jsonify(api_response), 400
user.role_id = accType
try:
db.session.commit()
except Exception as ie:
Logger('error.log', level='error').logger.error(
"[事务提交失败]accName:【%s】%s" % (accName, ie))
db.session.rollback()
return jsonify(api_response)
except Exception as e:
Logger('error.log', level='error').logger.error(
"[修改角色异常]accName:【%s】%s" % (accName, e))
api_response["code"] = 500
api_response["msg"] = "服务器未知错误"
return jsonify(api_response), 500
def SSH_connect(identity_file, user, host):
if verbose:
print(
"""ssh -q -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i %s "%s@%s" """
% (identity_file, user, host))
cmd = """ssh -q -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i %s "%s@%s" """ % (
identity_file, user, host)
os.system(cmd)
Logger().cmd(cmd)
def deleteBatch():
if request.method == 'POST':
api_response = {
"code": 0,
"msg": "success"
}
try:
currResource = Resource.query.filter(Resource.perms == "modules:enterprise:delete").first()
if currResource==None:
api_response["code"] = 401
api_response["msg"] = "当前操作权限实体不存在"
return jsonify(api_response), 401
very = very_permission(currResource.id)
if very == False:
api_response["code"] = 401
api_response["msg"] = "该账户无操作权限"
return jsonify(api_response), 401
else:
request_json = request.get_json()
etpCodeList = (request_json['etpCodeList'] if ('etpCodeList' in request_json) else "")
for etpCode in etpCodeList:
enter = Enterprise.query.filter(Enterprise.etpCode==etpCode).first()
server = Server.query.filter(Server.etpCode == etpCode).first()
if enter == None:
api_response["code"] = 400
api_response["msg"] = "该组织机构不存在"
return jsonify(api_response), 400
if server != None:
Server.query.filter(Server.etpCode == etpCode).delete()
db.session.delete(enter)
try:
db.session.commit()
except Exception as ie:
Logger('error.log', level='error').logger.error("[事务提交失败]accName:【%s】%s" % (g.user, ie))
db.session.rollback()
return jsonify(api_response)
except Exception as e:
Logger('error.log', level='error').logger.error("[批量删除企业异常]accName:【%s】%s" % (g.user, e))
api_response["code"] = 500
api_response["msg"] = "服务器未知错误"
return jsonify(api_response),500
def SSH_exec_cmd(identity_file, user, host, cmd, showCmd=True):
if len(cmd) == 0:
return
if showCmd or verbose:
print(
"""ssh -q -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i %s "%s@%s" "%s" """
% (identity_file, user, host, cmd))
cmd = """ssh -q -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i %s "%s@%s" "%s" """ % (
identity_file, user, host, cmd)
os.system(cmd)
Logger().cmd(cmd)
def exec_cmd_local(execmd, supressWarning=False):
if supressWarning:
cmd += " 2>/dev/null"
if verbose:
print execmd
try:
output = subprocess.check_output(execmd, shell=True)
except subprocess.CalledProcessError as e:
output = "Return code: " + str(
e.returncode) + ", output: " + e.output.strip()
Logger().cmd(execmd)
# print output
return output
def getEnterpriseList():
if request.method == 'GET':
page = int(request.args.get("page",1))
limit = int(request.args.get("limit",10))
etpCode = request.args.get("etpCode", "")
etpName = request.args.get("etpName", "")
filters = {1 == 1}
api_response = {
"code": 0,
"msg": "success"
}
try:
currResource = Resource.query.filter(Resource.perms == "modules:enterprise:list").first()
if currResource==None:
api_response["code"] = 401
api_response["msg"] = "当前操作权限实体不存在"
return jsonify(api_response), 401
very = very_permission(currResource.id)
if very == False:
api_response["code"] = 401
api_response["msg"] = "该账户无操作权限"
return jsonify(api_response), 401
else:
enterpriselist = []
if etpCode is not "":
filters.add(Enterprise.etpCode == etpCode)
if etpName is not "":
filters.add(Enterprise.etpName.like("%" + etpName + "%"))
enterpriseList = Enterprise.query.filter(*filters).limit(limit).offset((page - 1) * limit).all()
sumList = Enterprise.query.filter(*filters).all()
for enterprise in enterpriseList:
enterpriselist.append(
dict(etpCode=enterprise.etpCode, etpName=enterprise.etpName, LDAPCode=enterprise.LDAPCode,
dhcpServerIP=enterprise.dhcpServerIP, TFTPServerIP=enterprise.TFTPServerIP, FTPServerIP=enterprise.FTPServerIP,
createAdmin=enterprise.createAdmin, createTime=enterprise.createTime, updateTime=enterprise.updateTime))
result = dict(sumcount=len(sumList), detailcount=len(enterpriseList), list=enterpriselist)
api_response['result'] = result
return jsonify(api_response)
except Exception as e:
Logger('error.log', level='error').logger.error("[查询企业列表异常]accName:【%s】%s" % (g.user, e))
api_response["code"] = 500
api_response["msg"] = "服务器未知错误"
return jsonify(api_response),500
def push_docker_with_config(dockername, config, verbose=False, nocache=False):
usedockername = dockername.lower()
# build_docker( config["dockers"]["container"][dockername]["name"], config["dockers"]["container"][dockername]["dirname"], verbose, nocache )
if verbose:
print "Pushing docker ... " + config["dockers"]["container"][
dockername]["name"] + " to " + config["dockers"]["container"][
dockername]["fullname"]
cmd = "docker tag " + config["dockers"]["container"][dockername][
"name"] + " " + config["dockers"]["container"][dockername]["fullname"]
cmd += "; docker push " + config["dockers"]["container"][dockername][
"fullname"]
os.system(cmd)
Logger().cmd(cmd)
return config["dockers"]["container"][dockername]["name"]
def getEnterprise(etpCode):
if request.method == 'GET':
api_response = {
"code": 0,
"msg": "success"
}
try:
currResource = Resource.query.filter(Resource.perms == "modules:enterprise:list").first()
if currResource==None:
api_response["code"] = 401
api_response["msg"] = "当前操作权限实体不存在"
return jsonify(api_response), 401
very = very_permission(currResource.id)
if very == False:
api_response["code"] = 401
api_response["msg"] = "该账户无操作权限"
return jsonify(api_response), 401
else:
enterprise = Enterprise.query.filter(Enterprise.etpCode == etpCode).first()
serverList = Server.query.filter(Server.etpCode == etpCode).all()
if enterprise == None:
api_response["code"] = 400
api_response["msg"] = "该组织机构不存在"
return jsonify(api_response), 400
else:
data = []
serverinfoList = []
if serverList:
for server in serverList:
serverinfoList.append(dict(etpCode=etpCode, serverType=server.serverType, serverIP=server.serverIP,
serverUsername=server.serverUsername, serverPasswd=server.serverPasswd))
data.append(dict(etpCode=enterprise.etpCode, etpName=enterprise.etpName, LDAPCode=enterprise.LDAPCode,
dhcpServerIP=enterprise.dhcpServerIP, TFTPServerIP=enterprise.TFTPServerIP, FTPServerIP=enterprise.FTPServerIP,
createAdmin=enterprise.createAdmin, createTime=enterprise.createTime, updateTime=enterprise.updateTime, serverInfolist=serverinfoList))
response = dict(list=data)
api_response['result'] = response
return jsonify(api_response)
except Exception as e:
Logger('error.log', level='error').logger.error("[查询企业信息异常]accName:【%s】%s" % (g.user, e))
api_response["code"] = 500
api_response["msg"] = "服务器未知错误"
return jsonify(api_response), 500
def getCurrentUserInfo():
if request.method == 'GET':
api_response = {"code": 0, "msg": "success"}
try:
currResource = Resource.query.filter(
Resource.perms == "modules:usermanage:info").first()
if currResource == None:
api_response["code"] = 401
api_response["msg"] = "当前操作权限实体不存在"
return jsonify(api_response), 401
very = very_permission(currResource.id)
if very == False:
api_response["code"] = 401
api_response["msg"] = "该账户无操作权限"
return jsonify(api_response), 401
else:
data = []
user = User.query.filter(User.accName == g.user).first()
if user == None:
api_response["code"] = 400
api_response["msg"] = "该账户不存在"
return jsonify(api_response), 400
data.append(
dict(accName=user.accName,
userID=user.userID,
accType=user.role_id,
userName=user.userName,
accAttr=user.accAttr,
etpName=user.etpName,
userDP=user.userDP,
userMail=user.userMail,
userPhone=user.userPhone,
userTel=user.userTel))
response = dict(list=data)
api_response['result'] = response
return jsonify(api_response)
except Exception as e:
Logger('error.log', level='error').logger.error(
"[获取当前用户信息异常]accName:【%s】%s" % (g.user, e))
api_response["code"] = 500
api_response["msg"] = "服务器未知错误"
return jsonify(api_response), 500
def downLoad(fileName):
if request.method == 'GET':
api_response = {
"code": 0,
"msg": "success"
}
try:
currResource = Resource.query.filter(Resource.perms == "modules:enterprise:list").first()
if currResource==None:
api_response["code"] = 401
api_response["msg"] = "当前操作权限实体不存在"
return jsonify(api_response), 401
very = very_permission(currResource.id)
if very == False:
api_response["code"] = 401
api_response["msg"] = "该账户无操作权限"
return jsonify(api_response), 401
else:
files = os.path.join(UPLOADFILES_DIRS, fileName)
if os.path.exists(files):
# 流式读取
def send_file():
store_path = files
with open(store_path, 'rb') as targetfile:
while 1:
data = targetfile.read(20 * 1024 * 1024) # 每次读取20M
if not data:
break
yield data
response = Response(send_file(), content_type='application/octet-stream')
response.headers["Content-disposition"] = 'attachment; filename=%s' % fileName # 如果不加上这行代码，导致下图的问题
return response
api_response["code"] = 400
api_response["msg"] = "文件不存在！"
return jsonify(api_response)
except Exception as e:
Logger('error.log', level='error').logger.error("[下载文件异常]accName:【%s】%s" % (g.user, e))
api_response["code"] = 400
api_response["msg"] = "系统异常"
return jsonify(api_response)
def CheckOutput2(Modelname, config, outdir, report_dir, diff_plan,
fake_modelname):
'''
1.如果是单个流程（可能包括多个模块，需要循环调用），则直接调用默认输出文件list进行判断即可output_file_every_module.list
2.并在outdir生成一个Report目录用于王龙飞提取数据的需求
3.return返回值为字符串，如果为空的话表示数据完成，否则输出缺失的文件列表str
'''
# 设置log
log = Logger(Modelname + "_output_file_check", outdir)
# 读取config文件，判断是否存在多个分组，并得到分组列表
config_paras = configparser.ConfigParser()
config_paras.read(config)
# 设置检验不存在的文件列表
no_files = []
# 新建一个Report目录
# 在输出目录的前一级目录/maindata/F18FTSECWLJ1207/result/../Report
# report_dir = outdir + "/../Report"
if not os.path.exists(report_dir):
os.makedirs(report_dir)
# output_file.list是自定义的
list_file = sys.path[0] + "/output_file_every_module.list"
# 获取对应模块的输出文件列表
# 如果是文件夹的话要怎么判断呢？？JudgeFile()函数都可以判断
results, cp_report = ReadList2(list_file, outdir, report_dir, Modelname,
diff_plan, fake_modelname)
r_index = 0
for file in results[Modelname]:
# 遍历所有的results，判断[文件/文件夹]是否存在，以及是否为空
tag = JudgeFile(file)
if tag == 0:
no_files.append(file)
log.error(file + " does not exist! Please check!")
else:
# 如果存在则拷贝到Report目录下面
os.system(str(cp_report[Modelname][r_index]))
log.info(file + " exist, and copy command: " +
cp_report[Modelname][r_index])
r_index += 1
return no_files
def CheckOutput(Modelname, config, outdir, report_dir, diff_plan):
'''
1.（废弃）如果是主程序，则需要判断所有的运行模块(这个步骤放在了ReAnalysis_main.py程序中进行)
2.当调用主流程时，分别按照main结果中的模块多次调用此程序，默认输出文件list进行判断即可output_file.list
3.并在outdir生成一个Report目录用于王龙飞提取数据的需求
4.return返回值为字符串，如果为空的话表示数据完成，否则输出缺失的文件列表str
'''
# 设置log
log = Logger(Modelname + "_output_file_check", outdir)
# 读取config文件，判断是否存在多个分组，并得到分组列表
config_paras = configparser.ConfigParser()
config_paras.read(config)
# diff_plan = []
# if config_paras[Modelname].get(Modelname + "_diff_plan"):
# anosim_diff_plan = HF-NF,HF-HFH-HFL-HFM,HFH-HFL-HFM-NF,HF-HFH-HFL-HFM-NF
# diff_plan = config_paras[Modelname].get(Modelname + "_diff_plan").split(",")
# elif config_paras[Modelname].get(Modelname + "diff_plan"):
# main程序中的参数
# diff_plan = HF-NF,HF-HFH-HFL-HFM,HFH-HFL-HFM-NF,HF-HFH-HFL-HFM-NF
# diff_plan = config_paras[Modelname].get(Modelname + "diff_plan").split(",")
# 设置检验不存在的文件列表
no_files = []
# 新建一个Report目录
# 在输出目录的前一级目录/maindata/F18FTSECWLJ1207/result/../Report
# report_dir = outdir + "/../Report"
if not os.path.exists(report_dir):
os.makedirs(report_dir)
# output_file.list是自定义的
list_file = sys.path[0] + "/output_file.list"
# list_file = sys.path[0] + "/output_file2.list"
# 获取对应模块的输出文件列表
# 如果是文件夹的话要怎么判断呢？？JudgeFile()函数都可以判断
results, cp_report = ReadList(list_file, outdir, report_dir, Modelname,
diff_plan)
r_index = 0
for file in results[Modelname]:
# 遍历所有的results，判断[文件/文件夹]是否存在，以及是否为空
tag = JudgeFile(file)
if tag == 0:
no_files.append(file)
# log.error(file + " does not exist! Please check!")
log.warning(file + " does not exist! Please check!")
else:
# 如果存在则拷贝到Report目录下面
# 判断Report文件中的文件路径是否存在，不存在则创建
report_filename = str(cp_report[Modelname][r_index]).split(" ")[2]
report_filedir = os.path.dirname(os.path.abspath(report_filename))
if not os.path.exists(report_filedir):
os.makedirs(report_filedir)
# 判断Report中的文件是否存在，存在则删除
if os.path.exists(report_filename):
os.remove(report_filename)
os.system(str(cp_report[Modelname][r_index]))
log.info(file + " exist, and copy command: " +
cp_report[Modelname][r_index])
r_index += 1
return no_files
def editEnterprise():
if request.method == 'POST':
api_response = {
"code": 0,
"msg": "success"
}
try:
currResource = Resource.query.filter(Resource.perms == "modules:enterprise:update").first()
if currResource==None:
api_response["code"] = 401
api_response["msg"] = "当前操作权限实体不存在"
return jsonify(api_response), 401
very = very_permission(currResource.id)
if very == False:
api_response["code"] = 401
api_response["msg"] = "该账户无操作权限"
return jsonify(api_response), 401
else:
request_json = request.get_json()
if 'etpCode' not in request_json or request_json['etpCode'] is "":
raise ValidationError("参数不能为空")
etpCode = request_json['etpCode']
enterprise = Enterprise.query.filter(Enterprise.etpCode == etpCode).first()
if enterprise == None:
api_response["code"] = 400
api_response["msg"] = "该组织机构不存在"
return jsonify(api_response), 400
updateTime = (request_json['updateTime'] if ('updateTime' in request_json) else datetime.now())
enterprise.updateTime = updateTime
if ('dhcpServerIP' in request_json):
enterprise.dhcpServerIP = request_json['dhcpServerIP']
if ('TFTPServerIP' in request_json):
enterprise.TFTPServerIP = request_json['TFTPServerIP']
if ('FTPServerIP' in request_json):
enterprise.FTPServerIP = request_json['FTPServerIP']
if ('serverInfolist' in request_json):
serverInfoList = request_json['serverInfolist']
for server in serverInfoList:
serverType = (server['serverType'] if ('serverType' in server) else "")
serverInfo = Server.query.filter(Server.etpCode == etpCode,Server.serverType == serverType).first()
if serverInfo==None:
api_response["code"] = 400
api_response["msg"] = "该服务器信息不存在"
return jsonify(api_response), 400
if ('serverIP' in server):
serverInfo.serverIP = server['serverIP']
if ('serverUsername' in server):
serverInfo.serverUsername = server['serverUsername']
if ('serverPasswd' in server):
serverInfo.serverPasswd = server['serverPasswd']
try:
db.session.commit()
except Exception as ie:
Logger('error.log', level='error').logger.error("[事务提交失败]accName:【%s】%s" % (g.user, ie))
db.session.rollback()
api_response["result"] = etpCode
return jsonify(api_response)
except Exception as e:
Logger('error.log', level='error').logger.error("[修改企业信息异常]accName:【%s】%s" % (g.user, e))
api_response["code"] = 500
api_response["msg"] = "服务器未知错误"
return jsonify(api_response)
def execute_backup_and_encrypt(clusterName, fname, key):
clusterID = get_cluster_ID_from_file()
backupdir = "./deploy_backup/backup"
cmd = "mkdir -p %s/clusterID" % backupdir
os.system(cmd)
Logger().cmd(cmd)
cmd = "cp -r ./*.yaml %s" % backupdir
os.system(cmd)
Logger().cmd(cmd)
cmd = "cp -r ./deploy/sshkey %s/sshkey" % backupdir
os.system(cmd)
Logger().cmd(cmd)
cmd = "cp -r ./deploy/etc %s/etc" % backupdir
os.system(cmd)
Logger().cmd(cmd)
cmd = "cp -r ./deploy/ssl %s/ssl" % backupdir
os.system(cmd)
Logger().cmd(cmd)
cmd = "cp -r ./deploy/clusterID.yml %s/clusterID/" % backupdir
os.system(cmd)
Logger().cmd(cmd)
if os.path.exists("./deploy/acs_kubeclusterconfig"):
cmd = "cp -r ./deploy/acs_kubeclusterconfig %s/" % backupdir
os.system(cmd)
Logger().cmd(cmd)
cmd = "tar -czvf %s.tar.gz %s" % (fname, backupdir)
os.system(cmd)
Logger().cmd(cmd)
if not key is None:
cmd = "openssl enc -aes-256-cbc -k %s -in %s.tar.gz -out %s.tar.gz.enc" % (
key, fname, fname)
os.system(cmd)
Logger().cmd(cmd)
cmd = "rm %s.tar.gz" % fname
os.system(cmd)
Logger().cmd(cmd)
cmd = "rm -rf ./deploy_backup/backup"
os.system(cmd)
Logger().cmd(cmd)
return
def render_template(template_file, target_file, config, verbose=False):
filename, file_extension = os.path.splitext(template_file)
basename = os.path.basename(template_file)
if ("render-exclude" in config and basename in config["render-exclude"]):
# Don't render/copy the file.
return
if ("render-by-copy-ext" in config
and file_extension in config["render-by-copy-ext"]) or (
"render-by-copy" in config
and basename in config["render-by-copy"]):
copyfile(template_file, target_file)
Logger().cmd("copy file %s %s" % (template_file, target_file))
if verbose:
print "Copy tempalte " + template_file + " --> " + target_file
elif "render-by-copy-full" in config and template_file in config[
"render-by-copy-full"]:
copyfile(template_file, target_file)
Logger().cmd("copy file %s %s" % (template_file, target_file))
if verbose:
print "Copy tempalte " + template_file + " --> " + target_file
elif ("render-by-line-ext" in config
and file_extension in config["render-by-line-ext"]) or (
"render-by-line" in config
and basename in config["render-by-line"]):
if verbose:
print "Render template " + template_file + " --> " + target_file + " Line by Line .... "
## 逐行拷贝
ENV_local = Environment(loader=FileSystemLoader("/"))
with open(target_file, 'w') as f:
with open(template_file, 'r') as fr:
for line in fr:
print "Read: " + line
try:
template = ENV_local.Template(line)
content = template.render(cnf=config)
print content
f.write(content + "\n")
except:
pass
fr.close()
f.close()
Logger().cmd("copy file %s %s" % (template_file, target_file))
else:
if verbose:
print "Render template " + template_file + " --> " + target_file
try:
ENV_local = Environment(loader=FileSystemLoader("/"))
template = ENV_local.get_template(os.path.abspath(template_file))
content = template.render(cnf=config)
target_dir = os.path.dirname(target_file)
cmd = "mkdir -p {0}".format(target_dir)
os.system(cmd)
Logger().cmd(cmd)
Logger().cmd("copy file %s %s" % (template_file, target_file))
with open(target_file, 'w') as f:
f.write(content)
f.close()
except Exception as e:
print "!!! Failure !!! in render template " + template_file
print e
pass
def getUserList():
if request.method == 'GET':
page = int(request.args.get("page", 1))
limit = int(request.args.get("limit", 10))
accName = request.args.get("accName", "")
etpCode = request.args.get("etpCode", "")
etpName = request.args.get("etpName", "")
accType = request.args.get("accType", "")
filters = {User.status == 1}
api_response = {"code": 0, "msg": "success"}
try:
currResource = Resource.query.filter(
Resource.perms == "modules:usermanage:list").first()
if currResource == None:
api_response["code"] = 401
api_response["msg"] = "当前操作权限实体不存在"
return jsonify(api_response), 401
very = very_permission(currResource.id)
if very == False:
api_response["code"] = 401
api_response["msg"] = "该账户无操作权限"
return jsonify(api_response), 401
else:
userList = []
if accName is not "":
filters.add(User.accName.like("%" + accName + "%"))
if etpCode is not "":
filters.add(User.accAttr == etpCode)
if etpName is not "":
filters.add(User.etpName.like("%" + etpName + "%"))
if accType is not "":
filters.add(User.role_id == int(accType))
userlist = User.query.filter(*filters).limit(limit).offset(
(page - 1) * limit).all()
sumList = User.query.filter(*filters).all()
for user in userlist:
last_login = Last_Online.query.filter(
Last_Online.accName == user.accName).first()
if last_login is None:
lastLogintime = None
else:
lastLogintime = last_login.last_login_time
userList.append(
dict(accName=user.accName,
userName=user.userName,
accType=user.role_id,
userDP=user.userDP,
etpName=user.etpName,
createTime=user.create_date,
lastLogintime=lastLogintime))
result = dict(sumcount=len(sumList),
detailcount=len(userlist),
list=userList)
api_response['result'] = result
return jsonify(api_response)
except Exception as e:
Logger('error.log', level='error').logger.error(
"[获取用户列表异常]accName:【%s】%s" % (accName, e))
api_response["code"] = 500
api_response["msg"] = "服务器未知错误"
return jsonify(api_response), 500
def __init__(self, modelname, config_file, fake_modelname):
# 全局变量
self.modelname = modelname
self.fake_modelname = fake_modelname.lower()
# 读取config的参数
self.config = configparser.ConfigParser()
self.config.read(config_file)
# -- 如果是main程序，则读入project_id, 默认为空 --
if self.config[modelname].get("project_id"):
self.project_id = self.config[modelname].get("project_id")
else:
self.project_id = ""
# -- 读入输出文件路径 --
# 这里要求读取的config文件包含输出路径，参数示例如下pca_outdir = /result/
if self.config[modelname].get(modelname + "_outdir"):
self.outdir = os.path.abspath(
self.config[modelname].get(modelname + "_outdir"))
# 当输入config文件为主流程（main）参数文件时，输出路径格式有所差异，没有下划线
elif self.config[modelname].get("outdir"):
self.outdir = os.path.abspath(self.config[modelname].get("outdir"))
else:
print("config文件(" + config_file + ")中找不到输出文件夹参数 outdir，请检查！！\n")
exit(1)
# 如果输出文件夹不存在则重新建立
if not os.path.exists(self.outdir):
os.makedirs(self.outdir)
# -- 设置全局的Log --
self.log = Logger(modelname, self.outdir)
# -- 设置全局输出Tag文件(eg: result/pca/pca.Run.tag) --
# 对Run.tag文件进行更新
if os.path.exists(self.outdir + "/" + self.modelname + ".Run.tag"):
os.remove(self.outdir + "/" + self.modelname + ".Run.tag")
self.tag = open(self.outdir + "/" + self.modelname + ".Run.tag", "w")
# 对运行状态进行判断。
self.tag.write(self.project_id + "_" + self.modelname + "：运行中\n")
self.tag.flush()
# -- 判断fake_modelname的有效性 --
if self.fake_modelname == "Not Found":
print("没有找到" + self.fake_modelname + "，请检查！\n")
self.log.error("没有找到" + self.fake_modelname + "，请检查！\n")
self.tag.write(self.project_id + "_" + self.modelname + "：失败\n")
self.tag.flush()
exit(1)
# -- 读入差异分组组 --
if self.config[modelname].get(modelname + "_diff_plan"):
self.diff_plan = self.config[modelname].get(
modelname + "_diff_plan").split(",")
# 当输入config文件为主流程参数文件时，输出格式有所差异
elif self.config[modelname].get("diff_plan"):
self.diff_plan = self.config[modelname].get("diff_plan").split(",")
else:
self.diff_plan = []
# -- 读入main流程需要分析的模块，默认为空 --
if self.config[modelname].get("module_option"):
self.module_option = self.config[modelname].get("module_option")
# 所有模块存成列表，如果是all，则存默认，否则直接split
# 定义了“all”
if self.module_option == "all":
self.module_option = [
'otu', 'anno', 'basic', 'pca', 'barplot', 'beta', 'alpha',
"plsda", "nmds", "picrust", "difference", "heatmap",
"network", 'rank', 'cumulative', 'ccarda', 'lefse',
'anosim', "randomforest", "permanova", "sourcetracker",
"graphlan", "enterotypes"
]
else:
self.module_option = self.module_option.replace(' ',
'').split(",")
else:
self.module_option = []
# 如果为空的话报错！
print("module_option参数为空或缺失，请检查config文件配置！\n")
self.log.error("module_option参数为空或缺失，请检查config文件配置！")
self.tag.write(self.project_id + "_" + self.modelname + "：失败\n")
self.tag.flush()
exit(1)
def render_template_directory(template_dir,
target_dir,
config,
verbose=False,
exclude_dir=None):
if target_dir in StaticVariable.rendered_target_directory:
return
else:
StaticVariable.rendered_target_directory[target_dir] = template_dir
cmd = "mkdir -p " + target_dir
os.system(cmd)
Logger().cmd(cmd)
markfile = os.path.join(target_dir, "DO_NOT_WRITE")
# print "Evaluate %s" % markfile
if not os.path.exists(markfile):
# print "Write DO_NOT_WRITE"
open(markfile, 'w').close()
filenames = os.listdir(template_dir)
for filename in filenames:
if filename == "copy_dir":
fullname = os.path.join(template_dir, filename)
with open(fullname) as f:
content = f.readlines()
content = [x.strip() for x in content]
for copy_dir in content:
fullname_copy_dir = os.path.join(template_dir, copy_dir)
# print "To render via copy %s" % fullname_copy_dir
# Allow target directory to be re-rendered
StaticVariable.rendered_target_directory.pop(
target_dir, None)
render_template_directory(fullname_copy_dir,
target_dir,
config,
verbose,
exclude_dir=template_dir)
elif os.path.isfile(os.path.join(template_dir, filename)):
if exclude_dir is not None:
check_file = os.path.join(exclude_dir, filename)
if os.path.exists(check_file):
continue
render_template(os.path.join(template_dir, filename),
os.path.join(target_dir, filename), config,
verbose)
else:
srcdir = os.path.join(template_dir, filename)
dstdir = os.path.join(target_dir, filename)
if ("render-by-copy" in config
and filename in config["render-by-copy"]):
cmd = "rm -rf %s" % dstdir
os.system(cmd)
Logger().cmd(cmd)
cmd = "cp -r %s %s" % (srcdir, dstdir)
os.system(cmd)
Logger().cmd(cmd)
else:
if exclude_dir is None:
render_template_directory(srcdir, dstdir, config,
verbose)
else:
exdir = os.path.join(exclude_dir, filename)
render_template_directory(srcdir,
dstdir,
config,
verbose,
exclude_dir=exdir)
def editUser():
if request.method == 'POST':
api_response = {"code": 0, "msg": "success"}
try:
currResource = Resource.query.filter(
Resource.perms == "modules:usermanage:update").first()
if currResource == None:
api_response["code"] = 401
api_response["msg"] = "当前操作权限实体不存在"
return jsonify(api_response), 401
very = very_permission(currResource.id)
if very == False:
api_response["code"] = 401
api_response["msg"] = "该账户无操作权限"
return jsonify(api_response), 401
else:
request_json = request.get_json()
if 'accName' not in request_json or request_json[
'accName'] is "":
raise ValidationError("参数不能为空")
accName = request_json['accName']
# if 'userID' not in request_json or request_json['userID'] is "":
# raise ValidationError("参数不能为空")
# userID = request_json['userID']
# if 'userName' not in request_json or request_json['userName'] is "":
# raise ValidationError("参数不能为空")
# userName = request_json['userName']
# if 'accAttr' not in request_json or request_json['accAttr'] is "":
# raise ValidationError("参数不能为空")
# accAttr = request_json['accAttr']
# if 'etpName' not in request_json or request_json['etpName'] is "":
# raise ValidationError("参数不能为空")
# etpName = request_json['etpName']
# if 'userDP' not in request_json or request_json['userDP'] is "":
# raise ValidationError("参数不能为空")
# userDP = request_json['userDP']
if 'userMail' not in request_json or request_json[
'userMail'] is "":
raise ValidationError("参数不能为空")
userMail = request_json['userMail']
if 'userPhone' not in request_json or request_json[
'userPhone'] is "":
raise ValidationError("参数不能为空")
userPhone = request_json['userPhone']
if 'userTel' not in request_json or request_json[
'userTel'] is "":
raise ValidationError("参数不能为空")
userTel = request_json['userTel']
# user = User.query.filter(User.accName == accName,User.userID == userID,User.userName == userName,
# User.accAttr == accAttr,User.etpName == etpName,User.userDP == userDP).first()
user = User.query.filter(User.accName == accName).first()
if user == None:
api_response["code"] = 400
api_response["msg"] = "该账户不存在"
return jsonify(api_response), 400
if ('accType' in request_json):
user.role_id = request_json['accType']
user.userMail = userMail
user.userPhone = userPhone
user.userTel = userTel
if ('password' in request_json):
user.hash_pwd(request_json['password'])
if ('status' in request_json):
user.status = request_json['status']
if ('remarks' in request_json):
user.remarks = request_json['remarks']
try:
db.session.commit()
except Exception as ie:
Logger('error.log', level='error').logger.error(
"[事务提交失败]accName:【%s】%s" % (accName, ie))
db.session.rollback()
return jsonify(api_response)
except Exception as e:
Logger('error.log', level='error').logger.error(
"[修改用户信息异常]accName:【%s】%s" % (accName, e))
api_response["code"] = 500
api_response["msg"] = "服务器未知错误"
return jsonify(api_response), 500
def createUser():
if request.method == 'POST':
api_response = {"code": 0, "msg": "success"}
try:
currResource = Resource.query.filter(
Resource.perms == "modules:usermanage:save").first()
if currResource == None:
api_response["code"] = 401
api_response["msg"] = "当前操作权限实体不存在"
return jsonify(api_response), 401
very = very_permission(currResource.id)
if very == False:
api_response["code"] = 401
api_response["msg"] = "该账户无操作权限"
return jsonify(api_response), 401
else:
request_json = request.get_json()
if 'accName' not in request_json or request_json[
'accName'] is "":
raise ValidationError("参数不能为空")
accName = request_json['accName']
if 'password' not in request_json or request_json[
'password'] is "":
raise ValidationError("参数不能为空")
password = request_json['password']
if 'accType' not in request_json or request_json[
'accType'] is "":
raise ValidationError("参数不能为空")
accType = request_json['accType']
if 'userID' not in request_json or request_json['userID'] is "":
raise ValidationError("参数不能为空")
userID = request_json['userID']
if 'userName' not in request_json or request_json[
'userName'] is "":
raise ValidationError("参数不能为空")
userName = request_json['userName']
if 'accAttr' not in request_json or request_json[
'accAttr'] is "":
raise ValidationError("参数不能为空")
accAttr = request_json['accAttr']
if 'etpName' not in request_json or request_json[
'etpName'] is "":
raise ValidationError("参数不能为空")
etpName = request_json['etpName']
if 'userDP' not in request_json or request_json['userDP'] is "":
raise ValidationError("参数不能为空")
userDP = request_json['userDP']
if 'userMail' not in request_json or request_json[
'userMail'] is "":
raise ValidationError("参数不能为空")
userMail = request_json['userMail']
if 'userPhone' not in request_json or request_json[
'userPhone'] is "":
raise ValidationError("参数不能为空")
userPhone = request_json['userPhone']
if 'userTel' not in request_json or request_json[
'userTel'] is "":
raise ValidationError("参数不能为空")
userTel = request_json['userTel']
status = (request_json['status'] if
('status' in request_json) else 1)
create_user_id = g.user
create_date = (request_json['create_date'] if
('create_date'
in request_json) else datetime.now())
remarks = (request_json['remarks'] if
('remarks' in request_json) else "")
newUser = User(accName, userID, userName, userMail, userPhone,
userTel, password, status, accType, accAttr,
etpName, userDP, create_date, create_user_id,
remarks)
newUser.hash_pwd(password)
db.session.add(newUser)
try:
db.session.commit()
except Exception as ie:
Logger('error.log', level='error').logger.error(
"[事务提交失败]accName:【%s】%s" % (accName, ie))
db.session.rollback()
return jsonify(api_response)
except Exception as e:
Logger('error.log', level='error').logger.error(
"[添加用户异常]accName:【%s】%s" % (accName, e))
api_response["code"] = 500
api_response["msg"] = "服务器未知错误"
return jsonify(api_response), 500
conf_str = f"_CLUSTERS_WS_{WINDOW_SIZE}"
folder_time = get_time_as_str()
model_folder_name = folder_time + conf_str
now = time.time()
now_date = datetime.datetime.now()
prefix = '..' if os.name == 'nt' else '.'
gene_clusters_documents_files_folder = "gene_clusters_documents"
input_folder = os.path.join(prefix, "results_files", BACTERIA, gene_clusters_documents_files_folder)
models_folder = os.path.join(prefix, "results_files", BACTERIA, "gene_clusters_models", model_folder_name)
if not os.path.exists(models_folder):
os.makedirs(models_folder)
log_path = os.path.join(models_folder, f"log_{model_folder_name}.txt")
sys.stdout = Logger(log_path)
print(f"Started running on: {now_date.strftime('%Y-%m-%d %H:%M:%S')}")
print(f"Started GeneClusters dov2vec training for bacteria: {BACTERIA} num of workers: {workers} model_folder_name: {model_folder_name}")
files_list = os.listdir(input_folder)
files_list = [x for x in files_list if ".pickle" in x]
if USE_ONLY_LABELED_STRAINS:
original_files_list_len = len(files_list)
amr_df = pd.read_csv(os.path.join(prefix, "results_files", BACTERIA, "amr_labels.csv"))
labeled_files_list = list(amr_df["NCBI File Name"])
files_list = [x for x in files_list if x.replace(".pickle", "") in labeled_files_list]
new_files_list_len = len(files_list)
print(f"Using only labeled files ; Original number of strains: {original_files_list_len} New number of strains: {new_files_list_len}")
conf_dict = {
"bacteria": BACTERIA,
class Processing():
def __init__(self, modelname, config_file, fake_modelname):
# 全局变量
self.modelname = modelname
self.fake_modelname = fake_modelname.lower()
# 读取config的参数
self.config = configparser.ConfigParser()
self.config.read(config_file)
# -- 如果是main程序，则读入project_id, 默认为空 --
if self.config[modelname].get("project_id"):
self.project_id = self.config[modelname].get("project_id")
else:
self.project_id = ""
# -- 读入输出文件路径 --
# 这里要求读取的config文件包含输出路径，参数示例如下pca_outdir = /result/
if self.config[modelname].get(modelname + "_outdir"):
self.outdir = os.path.abspath(
self.config[modelname].get(modelname + "_outdir"))
# 当输入config文件为主流程（main）参数文件时，输出路径格式有所差异，没有下划线
elif self.config[modelname].get("outdir"):
self.outdir = os.path.abspath(self.config[modelname].get("outdir"))
else:
print("config文件(" + config_file + ")中找不到输出文件夹参数 outdir，请检查！！\n")
exit(1)
# 如果输出文件夹不存在则重新建立
if not os.path.exists(self.outdir):
os.makedirs(self.outdir)
# -- 设置全局的Log --
self.log = Logger(modelname, self.outdir)
# -- 设置全局输出Tag文件(eg: result/pca/pca.Run.tag) --
# 对Run.tag文件进行更新
if os.path.exists(self.outdir + "/" + self.modelname + ".Run.tag"):
os.remove(self.outdir + "/" + self.modelname + ".Run.tag")
self.tag = open(self.outdir + "/" + self.modelname + ".Run.tag", "w")
# 对运行状态进行判断。
self.tag.write(self.project_id + "_" + self.modelname + "：运行中\n")
self.tag.flush()
# -- 判断fake_modelname的有效性 --
if self.fake_modelname == "Not Found":
print("没有找到" + self.fake_modelname + "，请检查！\n")
self.log.error("没有找到" + self.fake_modelname + "，请检查！\n")
self.tag.write(self.project_id + "_" + self.modelname + "：失败\n")
self.tag.flush()
exit(1)
# -- 读入差异分组组 --
if self.config[modelname].get(modelname + "_diff_plan"):
self.diff_plan = self.config[modelname].get(
modelname + "_diff_plan").split(",")
# 当输入config文件为主流程参数文件时，输出格式有所差异
elif self.config[modelname].get("diff_plan"):
self.diff_plan = self.config[modelname].get("diff_plan").split(",")
else:
self.diff_plan = []
# -- 读入main流程需要分析的模块，默认为空 --
if self.config[modelname].get("module_option"):
self.module_option = self.config[modelname].get("module_option")
# 所有模块存成列表，如果是all，则存默认，否则直接split
# 定义了“all”
if self.module_option == "all":
self.module_option = [
'otu', 'anno', 'basic', 'pca', 'barplot', 'beta', 'alpha',
"plsda", "nmds", "picrust", "difference", "heatmap",
"network", 'rank', 'cumulative', 'ccarda', 'lefse',
'anosim', "randomforest", "permanova", "sourcetracker",
"graphlan", "enterotypes"
]
else:
self.module_option = self.module_option.replace(' ',
'').split(",")
else:
self.module_option = []
# 如果为空的话报错！
print("module_option参数为空或缺失，请检查config文件配置！\n")
self.log.error("module_option参数为空或缺失，请检查config文件配置！")
self.tag.write(self.project_id + "_" + self.modelname + "：失败\n")
self.tag.flush()
exit(1)
def PrintTime(self):
'''
用于增加monitor运行-p参数时间戳后缀
'''
import time
now = int(round(time.time() * 1000))
# eg: 2020_03_20_09_26_47
time_now = time.strftime('%Y_%m_%d_%H_%M_%S',
time.localtime(now / 1000))
return time_now
def read_conf(self, config_file):
'''
读取/root/.new.pymonitor.conf文件，获取当前正在跑的所有任务的project ID
:param config_file
:return: project ID list (project_all)
'''
project_all = []
project_tag = 0
for line in open(config_file, 'r'):
line = line.strip()
if re.match("^\s*$", line) or re.match("^\s*#", line):
continue
elif re.match("^\[project\]", line):
project_tag = 1
elif re.match("^\[base\]", line):
project_tag = 0
else:
if project_tag == 1:
line.replace(" ", "")
re.sub("#.*", "", line)
line = line.split('=')
project_all.append(line[0].strip())
return project_all
def MkReport(self, run_stat, run_project, run_time):
'''
根据运行状态判断是否需要拷贝结果到Report目录：
1. Running: 继续刷新
2. Fail: 退出程序，报错
3. Done: 运行完成，检验结果，并拷贝到Report目录
4. Exit: 重复刷新5次后仍然Fail，报错退出
:param run_stat: 运行状态，由JudgeRun()函数(外置了)得到
:param run_project: project_id
:return:
'''
while run_stat == "Running":
# 如果是正在运行则刷新项目，并不断调用函数返回状态，直至最后显示"Fail"或"Done"
time.sleep(5)
# os.system(monitor_soft + " stat updateproject -p " + run_project)
# run_stat = self.JudgeRun2(run_project)
run_stat = JudgeRun(run_project, self.outdir, self.log, run_time)
if run_stat == "Fail":
# fail_shell = os.popen(monitor_soft + " stat -m 3 -p " + run_project + " | grep fail")
print(run_project + "项目monitor运行报错。请检查程序是否正确投递或使用如下命令检查报错程序！\n" +
monitor_soft + " stat -m 3 -p " + run_project +
" | grep fail\n")
self.log.error(run_project +
"项目monitor运行报错。请检查程序是否正确投递或使用如下命令检查报错程序！\n" +
monitor_soft + " stat -m 3 -p " + run_project +
" | grep fail")
self.tag.write(self.project_id + "_" + self.modelname + "：失败\n")
self.tag.flush()
exit(1)
# 删除当前project
# os.system("/root/Software/monitor2020/monitor removeproject -m 1 -b -p " + run_project)
if run_stat == "Exit":
print(run_project + "项目连续5次刷新均报错！程序终止！找不到该项目或投递出错，请检查！")
self.tag.write(self.project_id + "_" + self.modelname + "：失败\n")
self.tag.flush()
exit(1)
if run_stat == "Done":
# 如果完成的话，则结束运行，返回log
self.log.info(run_project + "项目monitor运行完成。结果路径：" + self.outdir)
self.log.info(run_project + "开始检查输出文件是否完整。")
# CheckOutput()判断输出数据是否完整，单独拎出来写一个模块
# 判断输出结果是否完整，如果是的话，则成功运行，否则输出哪些文件缺失（no_files）。
# main模块和单个模块分别使用不同的函数进行判断
# 指定reort_dir，并清空目录
report_dir = self.outdir + "/../Report/"
if not os.path.exists(report_dir):
os.makedirs(report_dir)
else:
shutil.rmtree(report_dir)
os.makedirs(report_dir)
# 检验现有模块输出文件是否完整
file_check_tag = 0
# main和单个模块使用不同的CheckOutput函数！
for module in self.module_option:
# print(module)
module = module.lower()
if not os.path.exists(self.outdir + "/" + module):
continue
if self.fake_modelname == "main":
# no_files = CheckOutput(module, self.outdir + "/Configdir/" + module + ".config",
# self.outdir + "/" + module, report_dir, self.diff_plan)
no_files, file_stat = CheckOutput(
module,
self.outdir + "/Configdir/" + module + ".config",
self.outdir + "/" + module, report_dir, self.diff_plan,
self.log)
else:
# 这里单个模块的config是一个文件，不需要读取Configdir
no_files, file_stat = CheckOutput2(
module, config, self.outdir + "/" + module, report_dir,
self.diff_plan, self.fake_modelname, self.log)
if len(no_files):
# print(len(no_files))
# 不为空，有文件缺失，判断是否有必要文件缺失
no_files = "\n".join(no_files)
if file_stat == "Error":
file_check_tag = 1
print(run_project + "中" + module +
"【重要】结果文件缺失。缺失文件列表：\n" + no_files)
self.log.warning(run_project + "中" + module +
"【重要】结果文件缺失。缺失文件列表：\n" + no_files)
# self.tag.write(self.project_id + "_" + self.modelname + "：失败\n")
# self.tag.flush()
# exit(1)
else:
print(run_project + "中" + module + "结果文件缺失。缺失文件列表：\n" +
no_files)
self.log.warning(run_project + "中" + module +
"结果文件缺失。缺失文件列表：\n" + no_files)
else:
# 为空表示文件都存在
self.log.info(run_project + "中" + module + "结果文件完整。结果路径：" +
self.outdir + "/" + module)
# 扫描完之后更新tag文件
if file_check_tag:
# 也可以把失败更新放到"Error"鉴别，速度会更快，但是Log列出的缺失文件会不全
self.tag.write(self.project_id + "_" + self.modelname +
"：失败\n")
self.tag.flush()
else:
self.tag.write(self.project_id + "_" + self.modelname +
"：成功\n")
self.tag.flush()
# 输出log
self.log.info(run_project + " monitor运行log日志： " + run_project +
".db.log")
os.system(monitor_soft + " logdump -p " + run_project)
# 最后不管报错与否，删除当前project
# os.system(monitor_soft + " removeproject -m 1 -b -p " + run_project)
def RunModule(self, software, database, config, project, queue,
monitor_soft):
'''
1.如果是运行主流程，则需要用夏展峰的模块Generate_main.py，根据config文件生成各个模块的程序
同时根据生成的All_dependence.txt进行提交monitor，并返回log
2.（废弃）如果是运行单个流程，则调用夏展峰的Module模块，生成脚本后提交monitor运行，并返回log
3.如果是单个流程，可能存在需要依赖关系，需要调用其他的流程，所以这里统一用夏展峰的模块Generate_main.py进行调用
'''
# 优先使用config中的project_id作为“-p”参数前缀,否则使用默认的
if self.project_id != "":
project = self.project_id
self.log.info('根据config重新生成运行脚本。输出结果路径：' + self.outdir)
# -- 定义运行monitor的项目名称 --
# 根据时间戳设置monitor项目名称：Modelname + time (eg: -p anosim_2020_03_20_09_20_46)
# 跑main流程/单个的时候monitor运行的-p参数中加上了项目id
run_project = project + "_" + self.fake_modelname + "_" + self.PrintTime(
)
# 如果是主流程，则先调用夏展峰写的Generate_main.py模块，生成所有模块的运行脚本和路径
status = os.system(
python_soft +
" /root/16s/Modules/Generate_pipeline/Generate_main.py --config " +
config + " --database " + database + " --software " + software)
if status != 0:
# 如果Generate_main.py模块报错，则查看result1/generate_main.log信息查看报错情况
print(
"/root/16s/Modules/Generate_pipeline/Generate_main.py运行错误。报错信息查看log：当前运行路径/generate_main.log\n"
)
self.log.error(
"/root/16s/Modules/Generate_pipeline/Generate_main.py运行错误。报错信息查看log：当前运行路径/generate_main.log"
)
self.tag.write(self.project_id + "_" + self.modelname + "：失败\n")
self.tag.flush()
exit(1)
else:
# 处理result1/Listdir/All.dependence.txt，有的情况下为空，如果为空则报错退出
if not os.path.getsize(self.outdir +
"/Listdir/All.dependence.txt"):
print(self.outdir + "/Listdir/All.dependence.txt为空！请检查！\n")
self.log.info(self.outdir +
"/Listdir/All.dependence.txt为空！请检查！")
self.tag.write(self.project_id + "_" + self.modelname +
"：失败\n")
self.tag.flush()
exit(1)
# 只有一个module或多个module，直接拷贝到All.dependence.txt(这里的Module之间无关联关系！！)
# os.system("cat " + self.outdir + "/Listdir/*.list > " + self.outdir + "/Listdir/All.dependence.txt")
self.log.info(
"/root/16s/Modules/Generate_pipeline/Generate_main.py运行完成。")
# Generate_main.py模块会根据参数文件生成result1/qsub.sh，直接提交qsub.sh就可以运行
# 但是qsub.sh中的project是固定的，如果同时有多个项目投递的时候可能存在重复
# 所以这里我没有用这个脚本，而是直接调用result1/Listdir/All.dependence.txt，按照时间戳进行提交
# run_command = "/root/Software/monitor2020/monitor taskmonitor -P " + project + " -q " + queue + " -p " + run_project + " -i " + self.outdir + "/Listdir/All.dependence.txt"
run_command = monitor_soft + " taskmonitor -q " + queue + " -p " + run_project + " -i " + self.outdir + "/Listdir/All.dependence.txt"
self.log.info(run_project + "项目monitor开始运行。运行代码： " + run_command)
# -- 写入qsub命令（用于备份） --
if os.path.exists(self.outdir + "/qsub_" + self.modelname + ".sh"):
os.remove(self.outdir + "/qsub_" + self.modelname + ".sh")
qsub_cmd = open(self.outdir + "/qsub_" + self.modelname + ".sh", "w")
qsub_cmd.write(run_command)
qsub_cmd.flush()
# -- 运行monitor任务 --
os.system(run_command)
run_time = self.PrintTime()
# JudgeRun()判断是否运行完成
# run_stat = self.JudgeRun2(run_project)
run_stat = JudgeRun(run_project, self.outdir, self.log, run_time)
# MkReport()判断运行状态，并生成Report目录，拷贝文件
self.MkReport(run_stat, run_project, run_time)
def RunReAnalysis(self, software, database, config, project, queue,
monitor_soft):
'''
判断是否当前任务之前是否已经投递
1.是的话则直接进入到判断JudgeRun()和生成Report目录
2.否则直接重新投递新任务RunModule()
:return: None
'''
# 判断当前project_id是否已经投递，如果是的话则直接跳转到监控模块
# 有两种思路，一种是用monitor stat -m 2参数(未完成),一种是扫描/root/.new.pymonitor.conf文件获取
project_all = self.read_conf("/root/.new.pymonitor.conf")
# 用于标记判断是否是新项目，0为新项目，1表示之前已经投递
project_all_tag = 0
# try except监控monitor状态判断及结果判断是否出错
try:
for p in project_all:
# print(p)
matchObj = re.match(
r"^" + self.project_id.lower() + "_" +
self.fake_modelname.lower() + "_" +
"(\d{4}_\d{2}_\d{2}_\d{2}_\d{2}_\d{2})", p, re.I)
if matchObj:
# print(p)
project_all_tag = 1
print("项目已被投递monitor，投递编号：" + p)
self.log.info("项目已被投递monitor，投递编号：" + p)
run_project = p
# 刷新Run.tag文件
# 对Run.tag文件进行更新
if os.path.exists(self.outdir + "/" + self.modelname +
".Run.tag"):
os.remove(self.outdir + "/" + self.modelname +
".Run.tag")
self.tag = open(
self.outdir + "/" + self.modelname + ".Run.tag", "w")
self.tag.write(self.project_id + "_" + self.modelname +
"：运行中\n")
self.tag.flush()
# 删除旧的定时记录文件
if os.path.exists(self.outdir + "/retry_monitor_stat.txt"):
os.remove(self.outdir + "/retry_monitor_stat.txt")
# 进行运行状态判断
run_time = matchObj.group(1)
# print(run_time)
run_stat = JudgeRun(run_project, self.outdir, self.log,
run_time)
# 对运行状态进行处理
self.MkReport(run_stat, run_project, run_time)
# 当已经识别到旧的monitor任务投递时，不再重复循环，直接退出
break
if project_all_tag == 0:
self.RunModule(software, database, config, project, queue,
monitor_soft)
except Exception as e:
# 一旦上述程序块运行中有报错，即返回错误，记录到run.tag文件中，并删除project
print("程序运行错误!!! \n")
self.log.error("程序运行错误!!!")
self.log.error(e)
self.log.error(traceback.format_exc())
self.tag.write(self.project_id + "_" + self.modelname + "：失败\n")
self.tag.flush()
# os.system(monitor_soft + " removeproject -m 1 -b -p " + run_project)
exit(1)
