def index(request):
context = {}
if request.method == 'POST':
if 'delete-project' in request.POST:
form = NN_modelForm()
deleted_model = NN_model.objects.get(
user=request.user, pk=request.POST['delete-project'])
deleted_model.delete()
else:
form = NN_modelForm(request.POST)
if form.is_valid():
duplicate = NN_model.objects.filter(
user=request.user, title=request.POST['title']).count()
if duplicate != 0:
messages.error(
request, 'title "{}" already exists!'.format(
request.POST['title']))
error_message = 'title "{}" already exists!'.format(
request.POST['title'])
context['error_message'] = error_message
else:
model_dir = "models/{}/{}".format(request.user.id,
request.POST['title'])
newModel = NN_model(title=request.POST['title'],
user=request.user)
newModel.save(create=True)
update_status(newModel.state_file, 'system idle')
return HttpResponseRedirect(reverse("model:index"))
elif request.method == 'GET':
form = NN_modelForm()
projects = NN_model.objects.filter(user_id=request.user.id)
context['projects'] = projects
context['form'] = form
return render(request, 'model/index.html', context)
def index(request):
context = {}
if request.method == 'POST':
if 'delete-project' in request.POST:
form = NN_modelForm()
deleted_model = NN_model.objects.get(
id=request.POST['delete-project'])
shutil.rmtree(
op.join(MEDIA_ROOT, op.dirname(deleted_model.structure_file)))
deleted_model.delete()
else:
form = NN_modelForm(request.POST)
if form.is_valid():
duplicate = NN_model.objects.filter(
user=request.user.id, title=request.POST['title']).count()
if duplicate != 0:
messages.error(
request, 'title "{}" already exists!'.format(
request.POST['title']))
error_message = 'title "{}" already exists!'.format(
request.POST['title'])
context['error_message'] = error_message
else:
model_dir = "models/{}/{}".format(request.user.id,
request.POST['title'])
newModel = NN_model(title=request.POST['title'],
user=request.user,
state_file=model_dir + "/state.json",
structure_file=model_dir +
"/result.json")
newModel.save()
os.makedirs(op.join(MEDIA_ROOT, model_dir))
update_status(newModel.state_file, 'system idle')
shutil.copy2(
op.abspath(
op.join(op.abspath(__file__),
'../../../.config.json')),
op.join(MEDIA_ROOT, model_dir))
return HttpResponseRedirect(reverse("model:index"))
elif request.method == 'GET':
form = NN_modelForm()
projects = NN_model.objects.filter(user_id=request.user.id)
for project in projects:
status = update_status(project.state_file)['status']
# state_path = op.join(MEDIA_ROOT,project.state_file)
# if op.exists(state_path):
# with open(state_path) as f:
# status = json.load(f)['status']
# else:
# with open(state_path,'w') as f:
# json.dump({'status':'system idle'},f)
# status = 'idle'
if status == 'start training model' or status == 'start testing':
project.status = 'running'
else:
project.status = 'idle'
project.save()
context['projects'] = projects
context['form'] = form
return render(request, 'model/index.html', context)
def api(request):
project = get_object_or_404(NN_model,
user=request.user,
pk=request.POST['project_id'])
try:
info = update_status(project.state_file)
except:
return HttpResponse('failed parsing status')
if info['status'] in ['error', 'finish training']:
if request.POST.get('type') == 'init':
logger.info('back to idle')
# if info['status'] not in ['start training model', 'start testing', 'loading model']:
info['status'] = 'system idle'
update_status(project.state_file, info['status'])
elif info['status'] == 'error':
if info.get('error_log_file'):
with open(info['error_log_file']) as f:
info['detail'] = f.read()
project.status = 'idle'
project.save()
return JsonResponse(info)
def api(request):
project = get_object_or_404(NN_model,
user=request.user,
pk=request.POST['project_id'])
try:
info = update_status(project.state_file)
except:
return HttpResponse('failed parsing status')
history = project.history_set.last()
if not history:
log = None
else:
logfile_name = op.join(MEDIA_ROOT, op.dirname(project.structure_file),
history.save_path, 'logs/realtime_logging.txt')
if op.exists(logfile_name):
# log = {'epoch':[],'acc':[],'loss':[]}
log = []
with open(logfile_name, 'r') as f:
for i, line in enumerate(f):
log.append(line)
# epoch,acc,loss = line.split(',')
# log['epoch'].append(epoch)
# log['acc'].append(acc)
# log['loss'].append(loss)
else:
log = None
info['log'] = log
info['status'] = project.get_status_display()
# if info['status'] in ['error','finish training']:
if project.status in ['error', 'finish']:
if request.POST.get('type') == 'init':
logger.info('back to idle')
# # if info['status'] not in ['start training model', 'start testing', 'loading model']:
info['status'] = 'system idle'
# update_status(project.state_file,info['status'])
# elif info['status'] == 'error':
project.status = 'idle'
project.save()
elif project.status == 'error':
if info.get('error_log_file'):
with open(info['error_log_file']) as f:
info['detail'] = f.read()
if project.status in [
'training', 'finish'
] and info['loss']['value'] and np.isnan(info['loss']['value']):
info['loss']['value'] = 'nan'
return JsonResponse(info)
def backend_api(request):
if request.method == "POST":
project = NN_model.objects.filter(user=request.user).get(
pk=request.POST['project'])
script_path = op.abspath(
op.join(__file__, op.pardir, op.pardir, op.pardir,
'backend/petpen0.1.py'))
executed = datetime.datetime.now()
save_path = executed.strftime('%y%m%d_%H%M%S')
if not project:
return Http404('project not found')
if request.POST['command'] == 'predict':
history = History.objects.filter(project=project).get(
pk=request.POST['history'])
history_dir = op.join(MEDIA_ROOT,
op.dirname(history.project.structure_file),
history.save_path)
dataset_type = request.POST['dataset']
predict_dir = op.join(MEDIA_ROOT,
op.dirname(project.structure_file), 'result')
if op.exists(predict_dir):
shutil.rmtree(predict_dir)
os.mkdir(predict_dir)
if dataset_type == 'test':
# only support single text input node now, so only read one dataset path
with open(op.join(history_dir, 'preprocessed/result.json'),
'r') as f:
structure = json.load(f)
import pprint
pprint.pprint(structure['dataset'])
dataset = [
v['valid_x'] for v in structure['dataset'].values()
if 'valid_x' in v.keys()
][0]
# p = push(project.id,['python',script_path,'-m',op.join(MEDIA_ROOT,op.dirname(project.structure_file)),'-testx',op.join(predict_dir,'input.npy'),'-w',op.join(history_dir,'weights.h5'),'predict'])
# p = push(project.id,['python',script_path,'-m',history_dir,'-t',save_path,'-testx',dataset,'-w',op.join(history_dir,'weights.h5'),'predict'])
elif dataset_type == 'custom':
with open(op.join(history_dir, 'preprocessed/result.json'),
'r') as f:
structure = json.load(f)
input_shape = [
v['params']['shape']
for v in structure['layers'].values()
if v['type'] == 'Input'
][0]
dataset = request.FILES['file']
ext = op.splitext(dataset.name)[1].lower()
try:
if ext in ['.jpg', '.jpeg', '.img', '.png']:
with open(op.join(predict_dir, 'input' + ext),
'wb') as f:
for chunk in dataset.chunks():
f.write(chunk)
import matplotlib.pyplot as plt
data_value = plt.imread(op.join(
predict_dir, 'input' + ext),
format=ext[1:])
data_value = data_value.reshape([1] + input_shape)
np.save(op.join(predict_dir, 'input.npy'), data_value)
elif ext == '.csv':
# data_value = pd.read_csv(op.join(predict_dir,'input'+ext),header = None).values
if request.POST.get('header') == 'YES':
data_value = pd.read_csv(dataset).values
else:
data_value = pd.read_csv(dataset,
header=None).values
np.save(op.join(predict_dir, 'input.npy'), data_value)
if len(data_value.shape
) == 3 and data_value.shape[2] in [1, 3, 4]:
plt.imsave(op.join(predict_dir, 'input.png'),
data_value,
format='png')
elif ext in ['.pickle', '.pkl']:
try:
data_value = pickle.load(open(dataset, 'rb'))
except:
data_value = pd.read_pickle(dataset)
data_value = np.array(data_value)
np.save(op.join(predict_dir, 'input.npy'), data_value)
if len(data_value.shape
) == 3 and data_value.shape[2] in [1, 3, 4]:
plt.imsave(op.join(predict_dir, 'input.png'),
data_value,
format='png')
elif ext == '.npy':
with open(op.join(predict_dir, 'input' + ext),
'wb') as f:
for chunk in dataset.chunks():
f.write(chunk)
except Exception as err:
if not op.exists(predict_dir):
os.mkdir(predict_dir)
with open(op.join(predict_dir, 'error_log'), 'w') as f:
error_info = {
'error_type': str(type(err)),
'error_message': list(err.args),
}
json.dump(error_info, f)
logger.debug(err)
logger.debug(list(err.args))
# logger.error(err)
return HttpResponseRedirect(
reverse('model:predict',
kwargs={
'project_id': project.id,
'history_path': history.save_path
}))
p = push(project.id, [
'python', script_path, '-m',
op.join(MEDIA_ROOT, op.dirname(project.structure_file)),
'-testx',
op.join(predict_dir, 'input.npy'), '-w',
op.join(history_dir, 'weights.h5'), 'predict'
])
return HttpResponseRedirect(
reverse('model:predict',
kwargs={
'project_id': project.id,
'history_path': history.save_path
}))
elif request.POST['command'] == 'train':
history_name = request.POST.get('name') or save_path
logger.debug('start training on model {}, save path: {}'.format(
project, save_path))
structure_file = op.join(MEDIA_ROOT, project.structure_file)
if project.status not in ['idle', 'finish', 'error']:
return HttpResponse('waiting project back to idle')
else:
project.status = 'loading'
project.save()
with open(structure_file) as f:
model_parser = json.load(f)
for key, value in model_parser['layers'].items():
if value['type'] == 'Output':
loss = value['params']['loss']
if 'entropy' in loss:
problem = 'classification'
else:
problem = 'regression'
history = History(project=project,
name=history_name,
executed=executed,
save_path=save_path,
status='running',
execution_type=problem)
history.save()
project.training_counts += 1
project_path = op.dirname(structure_file)
history_dir = op.join(project_path, save_path)
flow_file = op.join(project_path, 'flows_petpen.json')
os.mkdir(op.join(project_path, save_path))
shutil.copy2(structure_file, history_dir)
shutil.copy2(flow_file, history_dir)
#----- file path transformation -----
prcs = preprocess_structure(
structure_file, NN_model.objects.filter(user=request.user),
Dataset.objects.filter(user=request.user))
print(prcs)
if prcs['status'] != 'successed':
os.makedirs(op.join(project_path, save_path, 'logs/'))
history.status = 'aborted'
history.save()
project.status = 'error'
project.save()
if prcs['status'] != 'file missing':
# update_status(project.state_file,status='error',detail='structure assignment error found on nodes {}'.format(', '.join(prcs)))
with open(op.join(history_dir, 'logs/error_log'),
'w') as f:
f.write(
'Structure assignment error found on nodes {}\n{}'.
format(', '.join(prcs['layers']),
(prcs['status'])))
return JsonResponse({'missing': prcs['status']})
else:
# update_status(project.state_file,status='error',detail='please depoly your model structure before running')
with open(op.join(history_dir, 'logs/error_log'),
'w') as f:
f.write(
'No deployed neural network found. Finish your neural network editing before running experiments.'
)
return JsonResponse({'missing': 'no structure file'})
else:
os.mkdir(op.join(history_dir, 'preprocessed'))
shutil.copy2(
op.join(op.dirname(structure_file),
'preprocessed/result.json'),
op.join(project_path, save_path, 'preprocessed'))
with open(op.join(history_dir, 'preprocessed/result.json'),
'r') as f:
structure = json.load(f)
import pprint
pprint.pprint(structure['dataset'])
dataset = [
v['train_x'] for v in structure['dataset'].values()
if 'train_x' in v.keys()
][0]
update_status(project.state_file,
status='loading',
epoch=[0, 0],
progress=[0, 0])
try:
if prcs['has_labels']:
p = push(project.id, [
'python', script_path, '-m', project_path, '-t',
save_path, '-header', 'True', 'train'
])
else:
p = push(project.id, [
'python', script_path, '-m', project_path, '-t',
save_path, 'train'
])
except Exception as e:
logger.error('Failed to run the backend', exc_info=True)
elif request.POST['command'] == 'stop':
p = kill(project.id)
project.status = 'idle'
project.save()
history = project.history_set.latest('id')
history.status = 'aborted'
history.save()
history_path = op.join(MEDIA_ROOT,
op.dirname(project.structure_file),
history.save_path)
if not op.exists(op.join(history_path, 'logs/')):
os.mkdir(op.join(history_path, 'logs'))
with open(op.join(history_path, 'logs/error_log'), 'w') as f:
f.write('training process stopped by user.')
return HttpResponse("response sent from backend")
def preprocess_structure(file_path, projects, datasets):
'''
parse model structure file and preprocess project and dataset file path to result.json
return a dictionary including:
1. status always included in result.keys (success/file missing/missing dataset)
2. has_labels included if success
3. layers included when missing dataset
'''
result = {}
if not op.exists(file_path):
update_status(op.join(op.dirname(file_path), 'state.json'),
'error: missing model structure')
return {'status': 'file missing'}
with open(file_path, 'r') as f:
structure = json.load(f)
import pprint
inputs = [
k for (k, v) in structure['layers'].items() if v['type'] == 'Input'
]
outputs = [
k for (k, v) in structure['layers'].items()
if v['type'] == 'Output'
]
dataset_setting = structure['dataset']
missing_dataset = []
for i in inputs:
if i not in dataset_setting:
missing_dataset.append(i)
else:
try:
dataset = datasets.get(title=dataset_setting[i][0])
except:
missing_dataset.append(i)
continue
dataset_setting[i] = {
'train_x':
op.join(MEDIA_ROOT, str(dataset.training_input_file)),
'valid_x':
op.join(MEDIA_ROOT, str(dataset.testing_input_file))
}
for o in outputs:
if o not in dataset_setting:
missing_dataset.append(o)
else:
try:
dataset = datasets.get(title=dataset_setting[o][0])
result['has_labels'] = dataset.has_labels
except:
missing_dataset.append(o)
continue
dataset_setting[o] = {
'train_y':
op.join(MEDIA_ROOT, str(dataset.training_output_file)),
'valid_y':
op.join(MEDIA_ROOT, str(dataset.testing_output_file))
}
structure['dataset'] = dataset_setting
pretrains = [
k for (k, v) in structure['layers'].items()
if v['type'] == 'Pretrained'
]
for p in pretrains:
pretrain_project_name = structure['layers'][p]['params'][
'project_name']
pretrain_history_name = structure['layers'][p]['params'][
'weight_file']
try:
pretrain_project = projects.get(title=pretrain_project_name)
pretrain_history = History.objects.filter(
project=pretrain_project,
name=pretrain_history_name,
status='success').latest('id')
except:
missing_dataset.append(p)
continue
structure['layers'][p]['params']['weight_file'] = op.join(
MEDIA_ROOT, op.dirname(pretrain_project.structure_file),
pretrain_history.save_path, 'weights.h5')
pprint.pprint(structure)
if missing_dataset:
print(missing_dataset)
result['status'] = 'missing dataset'
result['layers'] = missing_dataset
return result
preprocessed_dir = op.join(op.dirname(file_path), 'preprocessed')
if not op.exists(preprocessed_dir):
os.makedirs(preprocessed_dir)
with open(op.join(preprocessed_dir, 'result.json'), 'w') as pre_f:
json.dump(structure, pre_f)
result['status'] = 'successed'
return result
def backend_api(request):
if request.method == "POST":
script_path = op.abspath(
op.join(__file__, op.pardir, op.pardir, op.pardir,
'backend/petpen0.1.py'))
executed = datetime.datetime.now()
save_path = executed.strftime('%y%m%d_%H%M%S')
history_name = request.POST['name'] or save_path
project = NN_model.objects.filter(user=request.user).get(
id=request.POST['project'])
if not project:
return Http404('project not found')
history = History(project=project,
name=history_name,
executed=executed,
save_path=save_path,
status='running')
history.save()
project.training_counts += 1
project.status = 'running'
project.save()
logger.debug((history_name, save_path, executed))
logger.debug(request.POST['project'], project.title)
structure_file = op.join(MEDIA_ROOT, project.structure_file)
info = update_status(project.state_file)
if info['status'] != 'system idle':
return HttpResponse('waiting back to idle')
else:
update_status(project.state_file, 'loading model')
# pass
# state_file = op.join(MEDIA_ROOT,project.state_file)
# with open(state_file,'r+') as f:
# info = json.load(f)
# if info['status'] != 'system idle':
# return HttpResponse('waiting back to idle')
# info['status'] = 'loading model'
# f.seek(0)
# json.dump(info,f)
# f.truncate()
project_path = op.dirname(structure_file)
os.mkdir(op.join(project_path, save_path))
shutil.copy2(structure_file, op.join(project_path, save_path))
#----- file path transformation -----
prcs = preprocess_structure(structure_file,
Dataset.objects.filter(user=request.user))
print(prcs)
if prcs != 'successed':
history.status = 'aborted'
project.status = 'idle'
project.save()
history.save()
if prcs != 'file missing':
update_status(
project.state_file,
status='error',
detail='structure assignment error found on nodes {}'.
format(', '.join(prcs)))
return JsonResponse({'missing': prcs})
else:
update_status(
project.state_file,
status='error',
detail='please depoly your model structure before running')
return JsonResponse({'missing': 'no structure file'})
try:
p = subprocess.Popen([
'python', script_path, '-m', project_path, '-t', save_path,
'train'
], )
except Exception as e:
logger.error('Failed to run the backend', exc_info=True)
# project.status='running'
# project.save()
return HttpResponse("running")
def preprocess_structure(file_path, datasets):
if not op.exists(file_path):
update_status(op.join(op.dirname(file_path), 'state.json'),
'error: missing model structure')
return 'file missing'
# with open(op.join(op.dirname(file_path),'state.json'),'r+') as f:
# info = json.load(f)
# info['status'] = 'error: missing model structure'
# f.seek(0)
# json.dump(info,f)
# f.truncate()
# return 'file missing'
with open(file_path, 'r') as f:
structure = json.load(f)
import pprint
pprint.pprint(structure)
inputs = [
k for (k, v) in structure['layers'].items() if v['type'] == 'Input'
]
outputs = [
k for (k, v) in structure['layers'].items()
if v['type'] == 'Output'
]
dataset_setting = structure['dataset']
missing_dataset = []
for i in inputs:
if i not in dataset_setting:
missing_dataset.append(i)
else:
try:
dataset = datasets.get(title=dataset_setting[i][0])
except:
logger.error('Failed to load dataset', exc_info=True)
missing_dataset.append(i)
continue
dataset_setting[i] = {
'train_x':
op.join(MEDIA_ROOT, str(dataset.training_input_file)),
'valid_x':
op.join(MEDIA_ROOT, str(dataset.testing_input_file))
}
for o in outputs:
if o not in dataset_setting:
missing_dataset.append(o)
else:
try:
dataset = datasets.get(title=dataset_setting[o][0])
except:
logger.error('Failed to load dataset', exc_info=True)
missing_dataset.append(o)
continue
dataset_setting[o] = {
'train_y':
op.join(MEDIA_ROOT, str(dataset.training_output_file)),
'valid_y':
op.join(MEDIA_ROOT, str(dataset.testing_output_file))
}
structure['dataset'] = dataset_setting
pretrains = [
k for (k, v) in structure['layers'].items()
if v['type'] == 'Pretrained'
]
for p in pretrains:
pretrain_project_name = structure['layers'][p]['params'][
'project_name']
pretrain_history_name = structure['layers'][p]['params'][
'weight_file']
try:
pretrain_project = NN_model.objects.get(
user=request.user, title=pretrain_project_name)
pretrain_history = History.objects.filter(
project=pretrain_project,
name=pretrain_history_name,
status='success').latest('id')
except:
missing_dataset.append(p)
continue
structure['layers'][p]['params']['weight_file'] = op.join(
MEDIA_ROOT, op.dirname(pretrain_project.structure_file),
pretrain_history.save_path, 'weights.h5')
pprint.pprint(structure)
if missing_dataset:
print(missing_dataset)
return missing_dataset
preprocessed_dir = op.join(op.dirname(file_path), 'preprocessed')
if not op.exists(preprocessed_dir):
os.makedirs(preprocessed_dir)
with open(op.join(preprocessed_dir, 'result.json'), 'w') as pre_f:
json.dump(structure, pre_f)
return 'successed'