def post(self, request):
username = request.data['username']
password = request.data['password']
print(username, password)
### Dev mock user
message = '请检查填写的内容！'
uinfo = API_tools.check_user(username, password)
if "错误" in uinfo:
message = "用户名或密码错误！"
print(message)
response = {
"status": "error",
"type": "account",
"currentAuthority": "guest"
}
return Response(data=response)
else:
UID = int(uinfo["payload"]["userInfo"]["userId"])
DUser = models.User.objects.filter(id=UID)
print("UID,len", UID, DUser)
#mntpath = str(username)+"_mnt",
if len(DUser) == 0:
new_user = models.User.objects.create_user(
username=username,
tocken=API_tools.get_tocken(username, password),
password=password,
first_name=password,
id=UID)
new_user.save()
else:
DUser[0].username = username
DUser[0].set_password(password)
DUser[0].first_name = password
DUser[0].tocken = API_tools.get_tocken(username, password)
DUser[0].save()
user = auth.authenticate(username=username,
password=password) # 验证是否存在用户
print(user)
if (user):
print("login!!!!!!!!!!!!!!!!!!")
auth.login(request, user)
print(request.session)
response = {
"status": "ok",
"type": "account",
"currentAuthority": "user",
}
return Response(data=response)
def refresh_train_job(user):
'''
Args:
user: 用户实例
'''
# 以本地数据库的信息为主
un=user.username
pa=user.first_name
token=API_tools.get_tocken(un,pa)
job_list=models.customize_job.objects.all()
for rec in job_list:
# print("JOB: ", rec.jobid)
info=API_tools.get_jobinfo(rec.job_id,token,un,pa)
if info['code'] == 'S000': # 存在该任务
job=info['payload']
rec.state = job["jobStatus"]["state"]
timeStamp2 = int(job['jobStatus']["completedTime"])
if timeStamp2 == 0: continue
rec.completed_at = datetime.datetime.fromtimestamp(
timeStamp2/1000 )
rec.save()
else:
rec=models.customize_job.objects.get(id=rec.id)
rec.delete()
def updata_jobtable(tocken,un,pa):
#同步云脑数据库job信息
job = models.User_Job.objects.all().order_by("id")
job = job.exclude(state="STOPPED").exclude(state="FAIL").exclude(state="SUCCEEDED")
for jd in job:
print(jd)
jd_detail = API_tools.get_jobinfo(jd.jobid,tocken,un,pa)
if jd_detail["code"] == "S000":
jd.state = jd_detail["payload"]["jobStatus"]["state"]
timeStamp2 = int(jd_detail['payload']['jobStatus']["completedTime"])
if timeStamp2 != 0:
timeArray2 = time.localtime(timeStamp2 / 1000)
otherStyleTime2 = time.strftime("%Y-%m-%d %H:%M:%S", timeArray2)
jd.completedTime = otherStyleTime2
jd.save()
print("$$$$$$$$ Update Dataset Success")
def post(self,request):
# 创建任务
# 后送到云脑
# 成功后再录入数据库
user = auth.get_user(request)
form_dict=request.data
print("MISSION DICT: ",form_dict)
name=form_dict["name"]
try:
algo=models.customize_algo.objects.get(id=form_dict["algoID"])
except BaseException as e:
print("DATABASE ERROR:",repr(e))
res=errParser()
return Response(data=res)
project_path=algo.project_path
main_file=algo.start_path
param,res=parse_param(form_dict)
if res:
return Response(data=res)
index=form_dict['resource']
resource={ k:v for k,v in zip(RES_TYPE[0],RES_TYPE[index])}
info=API_tools.mission_submit(
job_name=name.lower(),
project_dir=project_path,
main_file=main_file,
param=param,
resource=resource,
username=user.username,
password=user.first_name,
)
print("TYPE is: ", type(info))
print("RETURN INFO IS: ", info)
'''RETURN INFO IS: {
'code': 'S000',
'msg': 'update job kktestnet2 successfully',
'payload': {
'jobId': 'fbb98fc00e9d0011eb0891304939b5259323'}}'''
if type(info) != dict or info.__contains__("code") and info["code"]!="S000":
res=errParser(errmessage="")
else:
job=info['payload']
# print("JOB INFO: ",job)
res=back_untils.insert_train_job(user,job["jobId"],algo)
res=errParser(errmessage=res[0]) if len(res) else Parser(res)
return Response(data=res)
def delete(self, request):
user = auth.get_user(request)
# print(user)
form_dict=request.data
print(form_dict)
try:
job=models.customize_job.objects.get(id=form_dict['id'])
rep=API_tools.delete_job(job.job_id,user.tocken,user.username,user.first_name)
if rep['code']!="S000":
# raise Exception(rep['msg'])
raise Exception("只能让任务停止，无法删除任务")
print("DELETE Result: ",rep)
res=Parser([])
except BaseException as e:
res=errParser(errmessage=repr(e))
finally:
return Response(data=res)
def evaluate(self, params, ii, jj):
print(params)
new_outpath = self.ouputdir + "/bbo_out_" + str(ii) + "_" + str(jj)
if os.path.exists(self.ouputdir) != True:
os.makedirs(self.ouputdir)
if os.path.exists(new_outpath) != True:
os.makedirs(new_outpath)
command = "cd " + self.algpath[
0:-8] + ";PYTHONPATH=./ python " + self.algpath[
-8:] + " --lr " + str(
params["lr"]) + " --outputdir " + str(new_outpath)
print(new_outpath, command)
#os.system(command)
info = API_tools.creat_mission(str("bbo_" + str(ii) + "_" + str(jj)),
command, "qwer", "wudch", "woodchen")
return 1
def insert_train_job(user,job_id,algo=None):
res_info=API_tools.get_jobinfo(job_id,"",user.username,user.first_name)
job=res_info["payload"]
print("RESPONSE JOB INFO: ",res_info)
jid=job['id']
if jid != job_id:
return ["Job id Error"]
createdTime = datetime.datetime.fromtimestamp(job['jobStatus']["completedTime"]/1000)
# compeletedTime = createdTime
models.customize_job.objects.create(
job_id=jid,
name=job['name'],
state=job['jobStatus']['state'],
created_at=createdTime,
# completed_at=compeletedTime,
uid=user,
algo_id=algo,
)
return []
def post(self,request):
# AutoML 新建任务
# res 字典数据格式详见 @/Frontend/src/pages/AutoML/CreateMission/data.d.ts
# export interface Former {
# //Base set
# type:string;
# name:string;
# description?:string;
# //Dataset set
# dataName?:string; //新建数据集的名称
# dataOutput?:string; //新建数据集的输出路径
# dataInput?:string; //新建数据集的输入路径
# dataSelection?:string; // 已有数据集的id
# //Model set
# modelsize:number;
# }
# @指项目文件夹路径
user = auth.get_user(request)
form_dict=request.data
# 创建任务
#{'type': 'Image_Classification', 'name': 'dsad', 'modelsize': 12321, 'dataSelection': 3}
FRONT_DEBUG=True
if(not FRONT_DEBUG):
datasetname = None
algtype = form_dict["type"]
jobname = form_dict['name']
maxflops = int(form_dict['modelsize'])
datasetid = form_dict['dataSelection']
if form_dict['dataSelection'] != None:
datasetname = models.Dataset.objects.filter(id = int(datasetid))[0]
# print(datasetname.name)
if algtype == 'Image_Classification':
algdict = ["efficientnet_b3a","mobilenetv2_120d","efficientnet_lite0","mobilenetv2_100","mobilenetv3_large_100"]
if maxflops > 900:
algname = 'efficientnet_b3a'
elif maxflops > 600:
algname = 'mobilenetv2_120d'
elif maxflops > 400:
algname = 'efficientnet_lite0'
elif maxflops > 300:
algname = 'mobilenetv2_100'
else:
algname = 'mobilenetv3_large_100'
#-------挂载CP算法操作----------
#back_untils.alg_cp(r'./../../algorithm/classification/pytorch_automodel/image_classification',"")
#-----------------------------
#command = "cd ../userhome/fakejobspace/algorithm/classification/pytorch_automodel/image_classification/;"
command = "cd ../userhome;mkdir jobspace;cd jobspace;rm -r algorithm;mkdir algorithm;cd algorithm;" \
"git clone https://github.com/MAC-AutoML/PCL_AutoML_System.git;cd ..;" \
"mkdir image_classification;cd ..;"
# 测试时使用fakejobspace中的算法运行
command = command+"cd jobspace/algorithm/PCL_AutoML_System/algorithm/classification/pytorch_automodel/image_classification;"
command = command + "PYTHONPATH=./ python Timm.py "
expdirname = str(jobname) + "_" + str(datasetname) + "_" + str(maxflops) + "_exp_" + str(time.time())
outputdir = "/userhome/jobspace/image_classification/"+expdirname
command = command + " --outputdir " + outputdir
command = command + " --dataset " + str(datasetname)
command = command + " --algname " + str(algname)
print(command)
info = API_tools.creat_mission(str(jobname), command, user.tocken, user, user.first_name)
if not info["payload"]:
print("error~!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
return Response(data=errParser(errcode=404))
timeArray = time.localtime()
otherStyleTime = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)
jobid = API_tools.get_keyword(str(info["payload"]["jobId"]))
name = API_tools.get_keyword(str(jobname))
username = API_tools.get_keyword(str(user.username))
user_id = str(user.id)
state = "WAITTING"
createdTime = API_tools.get_keyword(str(otherStyleTime))
completedTime = str(0)
_path = API_tools.get_keyword(str(outputdir))
Da = models.User_algorithm.objects.filter(user_id=user.id).filter(name=algname)[0]
algorithm_id = Da.id
dataset_id = form_dict['dataSelection']
with connection.cursor() as cursor:
sqltext = "INSERT INTO `automl_web`.`_app_user_job`(`jobid`, `name`, `username`, `user_id`, `state`, `createdTime`, `completedTime`,`_path`, `algorithm_id`, `dataset_id`) " \
"VALUES('{0}', '{1}', '{2}', '{3}', '{4}', '{5}', '{6}', '{7}', '{8}','{9}');".format(
jobid, name, username, user_id, state, createdTime, completedTime, _path, algorithm_id, dataset_id
)
print("$$$$$$$$$$$", sqltext)
cursor.execute(sqltext)
# 创建完成
# 【】前端 后端 需要添加判断任务是否创建成功
res=[]
res=Parser(res)
return Response(data=res)
def AutoSearch(method:str,hyper:dict,package:dict,api_config:dict,interval:int=60):
print("子进程（%s） 开始执行，父进程为（%s）" % (os.getpid(), os.getppid()))
time.sleep(5)
print("========MISSION REPORT========")
print("pid: %s METHOD is: "%(os.getpid()),method)
time.sleep(5)
print("-"*24)
time.sleep(5)
print("pid: %s Hypers: "%(os.getpid()))
a=[print(k,v) for k,v in enumerate(hyper.items())]
print("-"*24)
time.sleep(5)
print("pid: %s Job config is: "%(os.getpid()))
a=[print(k,v) for k,v in package.items()]
print("-"*24)
time.sleep(5)
print("pid: %s Search config is: "%(os.getpid()))
a=[print(k,v) for k,v in api_config.items()]
print("="*24)
return
optimizer=None # 【需要字典形式列出的黑盒方法集合】
for ii in range(hyper['epoch']):
points=optimizer.suggest(hyper['suggest'])
success=[]
jobid=[]
for jj,point in enumerate(points):
# 创建每个建议参数的任务
print(point)
param=get_param(package['param'],point) # 【需要知道point的形式】
info=API_tools.mission_submit(
job_name="as_"+package['job_name'],
project_dir=package['project_dir'],
main_file = package['main_file'],
param=param,
resource=package['resource'],
username=package['username'],
password=package['password'],)
if type(info) != dict or info.__contains__("code") and info["code"]!="S000":
print("ERROR: mission creation failed.\n* Epoch:{}, P:{}, {}".format(ii,jj,point))
success.append(False)
else:
jobid.append(info['payload']['jobId'])
success.append(True)
if not all(success):
print("Some mission failed, exit this search procedure.")
break
finish=[False]*hyper['suggest']
retry=[10]*hyper['suggest']
stop=False
while(all(finish) or stop):
for i,jid in enumerate(jobid):
info=API_tools.get_jobinfo(jid,username=package['username'],password=package['password'])
if info['code'] == 'S000':
status=info['payload']['jobStatus']['state'].lower()
if status == 'succeeded':
finish[i] = True
if status == 'waiting':
pass
if status == 'running':
pass
if status == 'failed':
stop=True
if status == 'stopped':
stop=True
else:
retry[i]-=1
if stop : break
stop=any([x<0 for x in retry])
time.sleep(interval)
if all(finish) and not stop:
ylist=get_result(package['username'],hyper['result'])
optimizer.observe(points,ylist)