def eval_model(args, model, weight_bits, group_size, weight_terms, data_bits,
data_terms):
# replace Conv2d with TRConv2dLayer
tr_params = cnn_models.static_conv_layer_settings(model, weight_bits,
group_size, weight_terms)
avg_terms = compute_avg_terms(tr_params)
qmodel = cnn_models.convert_model(model, tr_params, data_bits, data_terms)
# profiling
qmc = deepcopy(qmodel)
qmc.cuda()
x = torch.randn(1, 3, 224, 224).cuda()
tmacs, params = profile_model.get_model_ops(qmc, inputs=(x, ))
qmodel = nn.DataParallel(qmodel)
# compute activation scale factors
_ = util.validate(val_loader,
qmodel,
criterion,
args,
verbose=args.verbose,
pct=0.05)
tr_layer.set_tr_tracking(qmodel, False)
# evaluate model performance
_, acc = util.validate(val_loader,
qmodel,
criterion,
args,
verbose=args.verbose)
return acc, tmacs, avg_terms, params
def pruning_process(model):
print("------------------------- Before pruning --------------------------------")
criterion = nn.CrossEntropyLoss().cuda()
accuracy = util.validate(args, test_loader, model, criterion)
print("------------------------- pruning --------------------------------------")
if args.model_mode == 'c' or args.model_mode =='a':
model.prune_by_percentile( ['conv1'], q=100-58.0)
model.prune_by_percentile( ['conv2'], q=100-22.0)
model.prune_by_percentile( ['conv3'], q=100-34.0)
model.prune_by_percentile( ['conv4'], q=100-36.0)
model.prune_by_percentile( ['conv5'], q=100-53.0)
model.prune_by_percentile( ['conv6'], q=100-24.0)
model.prune_by_percentile( ['conv7'], q=100-42.0)
model.prune_by_percentile( ['conv8'], q=100-32.0)
model.prune_by_percentile( ['conv9'], q=100-27.0)
model.prune_by_percentile( ['conv10'], q=100-34.0)
model.prune_by_percentile( ['conv11'], q=100-35.0)
model.prune_by_percentile( ['conv12'], q=100-29.0)
model.prune_by_percentile( ['conv13'], q=100-36.0)
if args.model_mode == 'd' or args.model_mode == 'a':
model.prune_by_percentile( ['fc1'], q=100-10.0)
model.prune_by_percentile( ['fc2'], q=100-10.0)
model.prune_by_percentile( ['fc3'], q=100-10.0)
print("------------------------- After pruning --------------------------------")
util.print_nonzeros(model, f"{args.save_dir}/{args.log}")
accuracy = util.validate(args, test_loader, model, criterion)
torch.save(model, f"{args.save_dir}/model_pruned.ptmodel")
util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_pruned_folder}")
util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_pruned.ptmodel")
util.log(f"{args.save_dir}/{args.log}", f"accuracy after pruning\t{accuracy}")
util.layer2torch(model, f"{args.save_dir}/{args.out_pruned_folder}")
weight_list = util.parameters2list(model.children())
util.save_parameters(f"{args.save_dir}/{args.out_pruned_folder}", weight_list)
# Retrain
print("------------------------- Prune and Retrain ----------------------------")
tok="prune_re"
util.initial_train(model, args, train_loader, test_loader, tok, use_cuda=True)
print("------------------------- After Retraining -----------------------------")
util.print_nonzeros(model, f"{args.save_dir}/{args.log}")
accuracy = util.validate(args, test_loader, model, criterion)
torch.save(model, f"{args.save_dir}/model_prune_retrain_{args.reepochs}.ptmodel")
util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_pruned_re_folder}")
util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/mmodel_prune_retrain_{args.reepochs}.ptmodel")
util.log(f"{args.save_dir}/{args.log}", f"accuracy after prune retrain\t{accuracy}")
util.layer2torch(model, f"{args.save_dir}/{args.out_pruned_re_folder}")
weight_list = util.parameters2list(model.children())
util.save_parameters(f"{args.save_dir}/{args.out_pruned_re_folder}", weight_list)
return model
def pointQuery(api, attribute, display=False, validation=DO_VALIDATION):
result = api.liststreamkeyitems(STREAM, attribute, False, FILE_SIZE)
if validation:
validate(getData(result["result"]), attribute[1:])
if display:
display(result["result"])
return result["result"]
def pruning_process(model):
print(
"------------------------- Before pruning --------------------------------"
)
util.print_nonzeros(model, f"{args.save_dir}/{args.log}")
accuracy = util.validate(val_loader, model, args)
print(
"------------------------- pruning CNN--------------------------------------"
)
model.prune_by_percentile(['conv1'], q=100 - 66.0)
model.prune_by_percentile(['conv2'], q=100 - 12.0)
print(
"------------------------------- After prune CNN ----------------------------"
)
util.print_nonzeros(model, f"{args.save_dir}/{args.log}")
prec1 = util.validate(val_loader, model, args)
torch.save(model, f"{args.save_dir}/model_pruned.ptmodel")
util.log(f"{args.save_dir}/{args.log}",
f"weight\t{args.save_dir}/{args.out_pruned_folder}")
util.log(f"{args.save_dir}/{args.log}",
f"model\t{args.save_dir}/model_pruned.ptmodel")
util.log(f"{args.save_dir}/{args.log}", f"prune acc\t{prec1}")
util.layer2torch(f"{args.save_dir}/{args.out_pruned_folder}", model)
weight_list = util.parameters2list(model.children())
util.save_parameters(f"{args.save_dir}/{args.out_pruned_folder}",
weight_list)
print(
"------------------------- start retrain after prune CNN----------------------------"
)
util.initial_train(model, args, train_loader, val_loader, 'prune_re')
print(
"------------------------- After Retraining -----------------------------"
)
util.print_nonzeros(model, f"{args.save_dir}/{args.log}")
accuracy = util.validate(val_loader, model, args)
torch.save(model,
f"{args.save_dir}/model_prune_retrain_{args.reepochs}.ptmodel")
util.log(f"{args.save_dir}/{args.log}",
f"weight\t{args.save_dir}/{args.out_pruned_re_folder}")
util.log(
f"{args.save_dir}/{args.log}",
f"model\t{args.save_dir}/model_prune_retrain_{args.reepochs}.ptmodel")
util.log(f"{args.save_dir}/{args.log}",
f"prune and retrain acc\t{accuracy}")
util.layer2torch(f"{args.save_dir}/{args.out_pruned_re_folder}", model)
weight_list = util.parameters2list(model.children())
util.save_parameters(f"{args.save_dir}/{args.out_pruned_re_folder}",
weight_list)
return model
def quantize_process(model):
#util.print_model_parameters(model)
print(
'---------------------- Before weight sharing ---------------------------'
)
criterion = nn.CrossEntropyLoss().cuda()
acc = util.validate(args, test_loader, model, criterion)
util.log(f"{args.save_dir}/{args.log}",
f"accuracy before weight sharing\t{acc}")
# Weight sharing
old_weight_list, new_weight_list, quantized_index_list, quantized_center_list = apply_weight_sharing(
model, args.model_mode, args.bits)
print(
'----------------------- After weight sharing ---------------------------'
)
acc = util.validate(args, test_loader, model, criterion)
torch.save(model, f"{args.save_dir}/model_quantized.ptmodel")
util.log(f"{args.save_dir}/{args.log}",
f"weight\t{args.save_dir}/{args.out_quantized_folder}")
util.log(f"{args.save_dir}/{args.log}",
f"model\t{args.save_dir}/model_quantized.ptmodel")
util.log(f"{args.save_dir}/{args.log}",
f"accuracy after weight sharing {args.bits}bits\t{acc}")
util.layer2torch(model, f"{args.save_dir}/{args.out_quantized_folder}")
util.save_parameters(f"{args.save_dir}/{args.out_quantized_folder}",
new_weight_list)
print(
'----------------------- quantize retrain -------------------------------'
)
util.quantized_retrain(model, args, quantized_index_list,
quantized_center_list, train_loader, use_cuda)
#acc = util.test(model, test_loader, use_cuda=True)
acc = util.validate(args, test_loader, model, criterion)
torch.save(
model,
f"{args.save_dir}/model_quantized_retrain{args.reepochs}.ptmodel")
util.layer2torch(model, f"{args.save_dir}/{args.out_quantized_re_folder}")
util.log(f"{args.save_dir}/{args.log}",
f"weight\t{args.save_dir}/{args.out_quantized_re_folder}")
util.log(
f"{args.save_dir}/{args.log}",
f"model\t{args.save_dir}/model_quantized_bit{args.bits}_retrain{args.reepochs}.ptmodel"
)
util.log(f"{args.save_dir}/{args.log}",
f"accuracy retrain after weight sharing\t{acc}")
weight_list = util.parameters2list(model.children())
util.save_parameters(f"{args.save_dir}/{args.out_quantized_re_folder}",
weight_list)
return model
def validate(number: str):
util.validate(number)
intList = [int(c) for c in number]
powers = []
k = 1
while(k < len(intList)):
powers.append(k)
k *= 2
combinations = []
for c, n in enumerate(intList):
i = c + 1
if util.isPowerOfTwo(i):
continue
combination = []
for w in powers:
if(w & i == w):
combination.append(w)
combinations.append({
'index': i,
'combination': combination
})
sums = [intList[p - 1] for p in powers]
for ce, n in enumerate(powers):
for c in combinations:
if n in c['combination']:
sums[ce] += intList[c['index'] - 1]
sums = [0 if s % 2 == 0 else 1 for s in sums]
totalSum = 0
for s in sums:
totalSum += s
if(totalSum == 0):
print('Correct sequence!')
print('No error was encountered.')
return
position = 0
i = 0
for c in range(len(sums) - 1, -1, -1):
n = sums[i]
position += n**c
i += 1
position -= 1
intList[position] = 0 if intList[position] == 1 else 1
print('Corrupted sequence.')
print(f'Invalid bit encountered at index {position}.')
print(f'Right sequence would be {"".join([str(n) for n in intList])}')
def quantize_process(model):
print('------------------------------- accuracy before weight sharing ----------------------------------')
acc = util.validate(val_loader, model, args)
util.log(f"{args.save_dir}/{args.log}", f"accuracy before weight sharing\t{acc}")
print('------------------------------- accuacy after weight sharing -------------------------------')
tempfc1=torch.index_select(model.fc1.weight, 0, model.invrow1.cuda())
model.fc1.weight=torch.nn.Parameter(torch.index_select(tempfc1, 1, model.invcol1.cuda()))
tempfc2=torch.index_select(model.fc2.weight, 0, model.invrow2.cuda())
model.fc2.weight=torch.nn.Parameter(torch.index_select(tempfc2, 1, model.invcol2.cuda()))
tempfc3=torch.index_select(model.fc3.weight, 0, model.invrow3.cuda())
model.fc3.weight=torch.nn.Parameter(torch.index_select(tempfc3, 1, model.invcol3.cuda()))
old_weight_list, new_weight_list, quantized_index_list, quantized_center_list = apply_weight_sharing(model, args.model_mode, args.bits)
temp1=torch.index_select(model.fc1.weight, 0, model.rowp1.cuda())
model.fc1.weight=torch.nn.Parameter(torch.index_select(temp1, 1, model.colp1.cuda()))
temp2=torch.index_select(model.fc2.weight, 0, model.rowp2.cuda())
model.fc2.weight=torch.nn.Parameter(torch.index_select(temp2, 1, model.colp2.cuda()))
temp3=torch.index_select(model.fc3.weight, 0, model.rowp3.cuda())
model.fc3.weight=torch.nn.Parameter(torch.index_select(temp3, 1, model.colp3.cuda()))
acc = util.validate(val_loader, model, args)
util.save_checkpoint({
'state_dict': model.state_dict(),
'best_prec1': acc,
}, True, filename=os.path.join(args.save_dir, 'checkpoint_{}_alpha_{}.tar'.format('quantized',args.alpha)))
util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_quantized_folder}")
util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_quantized.ptmodel")
util.log(f"{args.save_dir}/{args.log}", f"accuracy after weight sharing {args.bits}bits\t{acc}")
util.layer2torch(f"{args.save_dir}/{args.out_quantized_folder}" , model)
util.save_parameters(f"{args.save_dir}/{args.out_quantized_folder}", new_weight_list)
print('------------------------------- retraining -------------------------------------------')
util.quantized_retrain(model, args, quantized_index_list, quantized_center_list, train_loader, val_loader)
acc = util.validate(val_loader, model, args)
util.save_checkpoint({
'state_dict': model.state_dict(),
'best_prec1': acc,
}, True, filename=os.path.join(args.save_dir, 'checkpoint_{}_alpha_{}.tar'.format('quantized_re',args.alpha)))
util.layer2torch(f"{args.save_dir}/{args.out_quantized_re_folder}" , model)
util.log(f"{args.save_dir}/{args.log}", f"weight:{args.save_dir}/{args.out_quantized_re_folder}")
util.log(f"{args.save_dir}/{args.log}", f"model:{args.save_dir}/model_quantized_bit{args.bits}_retrain{args.reepochs}.ptmodel")
util.log(f"{args.save_dir}/{args.log}", f"acc after qauntize and retrain\t{acc}")
weight_list = util.parameters2list(model.children())
util.save_parameters(f"{args.save_dir}/{args.out_quantized_re_folder}", weight_list)
return model
def __init__(self):
getFlights()
url = "http://localhost:8080/RESTApp/REST_Resources/airTickets/airTicket"
code = int(raw_input("\nPlease press the code of a Flight: "))
type_ = int(
raw_input("Press 1 to One-Way Ticket or 2 to Two-Way Ticket: "))
if (type_ != 1 and type_ != 2):
print "Invalid type of ticket!\n"
return None
try:
beginDate = raw_input("Start date (dd/mm/aaaa): ")
validate(beginDate)
endDate = raw_input("Final date (dd/mm/aaaa): ")
validate(endDate)
except ValueError:
print "Incorrect data format, should be dd/mm/aaaa\n"
return None
numberOfPeople = int(raw_input("Inform number of seats: "))
age = []
for i in xrange(0, numberOfPeople):
age.append(
int(raw_input("Inform the age of client #" + str(i) + " ")))
print "Inform the credicard information:"
cardNumber = int(raw_input("Credicard number: "))
dateValid = raw_input("Validity (mm/aaaa): ")
secret = int(raw_input("Inform the secret number: "))
parts = int(raw_input("Inform how many months of paying: "))
##########################
# For testing purposes
# code = 1
# type_ = 1
# beginDate = "10/12/2016"
# endDate = "12/12/2016"
# numberOfPeople = 2
# age = [22, 23]
# cardNumber = 31232131
# dateValid = "12/2018"
# secret = 323
# parts = 1
##########################
airTicket = airFlightTicketsDesire(code, type_, beginDate, endDate,
numberOfPeople, age, cardNumber,
dateValid, secret, parts)
response = exec_request(url, "post", json=airTicket.to_JSON())
print str(response.text)
def pruning_process(model):
print("------------------------- Before pruning --------------------------------")
util.print_nonzeros(model, f"{args.save_dir}/{args.log}")
accuracy = util.validate(val_loader, model, args)
print("------------------------- pruning CNN--------------------------------------")
model.prune_by_percentile( ['conv1'], q=100-58.0)
model.prune_by_percentile( ['conv2'], q=100-22.0)
model.prune_by_percentile( ['conv3'], q=100-34.0)
model.prune_by_percentile( ['conv4'], q=100-36.0)
model.prune_by_percentile( ['conv5'], q=100-53.0)
model.prune_by_percentile( ['conv6'], q=100-24.0)
model.prune_by_percentile( ['conv7'], q=100-42.0)
model.prune_by_percentile( ['conv8'], q=100-32.0)
model.prune_by_percentile( ['conv9'], q=100-27.0)
model.prune_by_percentile( ['conv10'], q=100-34.0)
model.prune_by_percentile( ['conv11'], q=100-35.0)
model.prune_by_percentile( ['conv12'], q=100-29.0)
model.prune_by_percentile( ['conv13'], q=100-36.0)
print("------------------------------- After prune CNN ----------------------------")
util.print_nonzeros(model, f"{args.save_dir}/{args.log}")
prec1 = util.validate(val_loader, model, args)
util.save_checkpoint({
'state_dict': model.state_dict(),
'best_prec1': prec1,
}, True, filename=os.path.join(args.save_dir, 'checkpoint_{}_alpha_{}.tar'.format('pruned',args.alpha)))
util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_pruned_folder}")
util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_pruned.ptmodel")
util.log(f"{args.save_dir}/{args.log}", f"prune acc\t{prec1}")
util.layer2torch(f"{args.save_dir}/{args.out_pruned_folder}" , model)
weight_list = util.parameters2list(model.children())
util.save_parameters(f"{args.save_dir}/{args.out_pruned_folder}", weight_list)
print("------------------------- start retrain after prune CNN----------------------------")
util.initial_train(model, args, train_loader, val_loader, 'prune_re')
print("------------------------- After Retraining -----------------------------")
util.print_nonzeros(model, f"{args.save_dir}/{args.log}")
accuracy = util.validate(val_loader, model, args)
util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_pruned_re_folder}")
util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_prune_retrain_{args.reepochs}.ptmodel")
util.log(f"{args.save_dir}/{args.log}", f"prune and retrain acc\t{accuracy}")
util.layer2torch(f"{args.save_dir}/{args.out_pruned_re_folder}" , model)
weight_list = util.parameters2list(model.children())
util.save_parameters(f"{args.save_dir}/{args.out_pruned_re_folder}", weight_list)
return model
def pointQuery(api, attribute, sort=True, reverse=False):
result = api.liststreamkeyitems(
att_dict[attribute[0]], attribute[1:], False, FILE_SIZE)
if DO_VALIDATION:
validate(getData(result["result"]), attribute[1:])
result = getData(result["result"])
att_name = att_dict[attribute[0]]
if type(ATTRIBUTE_TYPE[att_name]) is int and sort:
result = sortResult(result, att_name, reverse)
# print(*result, sep='\n')
# input()
return result
def quantize_process(model):
print(
'------------------------------- accuracy before weight sharing ----------------------------------'
)
acc = util.validate(val_loader, model, args)
util.log(f"{args.save_dir}/{args.log}",
f"accuracy before weight sharing\t{acc}")
print(
'------------------------------- accuacy after weight sharing -------------------------------'
)
old_weight_list, new_weight_list, quantized_index_list, quantized_center_list = apply_weight_sharing(
model, args.model_mode, args.bits)
acc = util.validate(val_loader, model, args)
util.log(f"{args.save_dir}/{args.log}",
f"weight\t{args.save_dir}/{args.out_quantized_folder}")
util.log(f"{args.save_dir}/{args.log}",
f"model\t{args.save_dir}/model_quantized.ptmodel")
util.log(f"{args.save_dir}/{args.log}",
f"accuracy after weight sharing {args.bits}bits\t{acc}")
util.layer2torch(f"{args.save_dir}/{args.out_quantized_folder}", model)
util.save_parameters(f"{args.save_dir}/{args.out_quantized_folder}",
new_weight_list)
print(
'------------------------------- retraining -------------------------------------------'
)
util.quantized_retrain(model, args, quantized_index_list,
quantized_center_list, train_loader, val_loader)
acc = util.validate(val_loader, model, args)
util.layer2torch(f"{args.save_dir}/{args.out_quantized_re_folder}", model)
util.log(f"{args.save_dir}/{args.log}",
f"weight:{args.save_dir}/{args.out_quantized_re_folder}")
util.log(
f"{args.save_dir}/{args.log}",
f"model:{args.save_dir}/model_quantized_retrain{args.reepochs}.ptmodel"
)
util.log(f"{args.save_dir}/{args.log}",
f"acc after qauntize and retrain\t{acc}")
weight_list = util.parameters2list(model.children())
util.save_parameters(f"{args.save_dir}/{args.out_quantized_re_folder}",
weight_list)
return model
def __init__(self):
getFlights()
url = "http://localhost:8080/RESTApp/REST_Resources/airTickets/airTicket"
code = int(raw_input("\nPlease press the code of a Flight: "))
type_ = int(raw_input("Press 1 to One-Way Ticket or 2 to Two-Way Ticket: "))
if (type_ != 1 and type_ != 2):
print "Invalid type of ticket!\n"
return None
try:
beginDate = raw_input("Start date (dd/mm/aaaa): ")
validate(beginDate)
endDate = raw_input("Final date (dd/mm/aaaa): ")
validate(endDate)
except ValueError:
print "Incorrect data format, should be dd/mm/aaaa\n"
return None
numberOfPeople = int(raw_input("Inform number of seats: "))
age = []
for i in xrange(0, numberOfPeople):
age.append(int(raw_input("Inform the age of client #"+str(i)+" ")))
print "Inform the credicard information:"
cardNumber = int(raw_input("Credicard number: "))
dateValid = raw_input("Validity (mm/aaaa): ")
secret = int(raw_input("Inform the secret number: "))
parts = int(raw_input("Inform how many months of paying: "))
##########################
# For testing purposes
# code = 1
# type_ = 1
# beginDate = "10/12/2016"
# endDate = "12/12/2016"
# numberOfPeople = 2
# age = [22, 23]
# cardNumber = 31232131
# dateValid = "12/2018"
# secret = 323
# parts = 1
##########################
airTicket = airFlightTicketsDesire(code, type_, beginDate, endDate, numberOfPeople, age, cardNumber, dateValid, secret, parts)
response = exec_request(url, "post", json = airTicket.to_JSON())
print str(response.text)
def project(htmlname):
if session.get('author', 'nologin') == 'nologin':
return redirect('/login')
headers['authorization'] = session['author']
validate_result = json.loads(
util.validate(session['author'], app.config['passport_key']))
if int(validate_result['code']) == 0:
# data['method'] = 'userprojects.getlist'
data['method'] = 'project.getlist' # 只在申请列表中通过jinja2渲染
data['params'] = {}
r = requests.post(get_url(), headers=headers, json=data)
result = json.loads(r.text)
result = json.loads(result['result'])
if int(result['code']) == 0:
return render_template(htmlname + '.html',
info=session,
user=session['user'],
result=result['result'])
else:
return render_template(htmlname + '.html',
info=session,
result=result['errmsg'])
else:
return render_template(htmlname + '.html',
errmsg=validate_result['errmsg'])
def initial_process(model):
print(model)
util.print_model_parameters(model)
print(
"------------------------- Initial training -------------------------------"
)
tok = "initial"
criterion = nn.CrossEntropyLoss().cuda()
util.initial_train(model,
args,
train_loader,
test_loader,
tok,
use_cuda=True)
accuracy = util.validate(args, test_loader, model, criterion)
torch.save(model, f"{args.save_dir}/model_initial_end.ptmodel")
util.log(f"{args.save_dir}/{args.log}",
f"weight:{args.save_dir}/{args.out_oldweight_folder}")
util.log(f"{args.save_dir}/{args.log}",
f"model:{args.save_dir}/model_initial_end.ptmodel")
util.log(f"{args.save_dir}/{args.log}", f"initial_accuracy {accuracy}")
util.layer2torch(model, f"{args.save_dir}/{args.out_oldweight_folder}")
weight_list = util.parameters2list(model.children())
util.save_parameters(f"{args.save_dir}/{args.out_oldweight_folder}",
weight_list)
return model
def login():
username = request.form.get('username')
password = request.form.get('password')
if util.validate([username, password]):
res = db.execute_query("select * from user where username=?",
(username, ))
if len(res) == 1:
if res[0]['password'] == password:
# r0 = db.execute("update ")
global time_start
time_start = datetime.datetime.now()
return {
'status': 200,
'msg': '登陆成功',
'data': {
'uid': res[0]['uid']
}
}
else:
return {'status': 404, 'msg': '密码错误'}
elif len(res) == 0:
return {'status': 404, 'msg': '用户名不存在'}
else:
return {'status': 404, 'msg': '未知错误'}
else:
return {'status': 404, 'msg': '字段格式错误'}
def register():
username = request.form.get('username')
password = request.form.get('password')
phone = request.form.get('phone')
if util.validate([username, password, phone]):
res = db.execute_query("select * from user where username=?",
(username, ))
if len(res) == 0:
# uid其实就是username的hash
uid = util.get_md5(username)
res = db.execute("insert into user values (?, ?, ?, ?, ?, ?, ?)",
(username, password, uid, phone, '', '', 0))
# 每个用户自己的单词表
res1 = db.execute(
"create table [" + uid + "_table]" + " (" + "word integer," +
"level integer)", ())
# res1 = True
if res and res1:
return {'status': 200, 'msg': '注册成功', 'uid': uid}
else:
return {'status': 404, 'msg': '未知错误'}
else:
return {'status': 404, 'msg': '用户已存在'}
else:
return {'status': 404, 'msg': '字段格式错误'}
def randomSolve(rides_clone, F, B, T):
scores = {}
count = 100
for i in range(count):
rides = [ride for ride in rides_clone]
vehicles = [[] for i in range(F)]
vehicle_index = 0
while len(rides) > 0:
ride_index = randint(0, len(rides) - 1)
ride = rides.pop(ride_index)
vehicles[vehicle_index % F].append(ride.index)
vehicle_index += 1
vehicles_rides = [[len(vehicle), *vehicle] for vehicle in vehicles]
score = validate(rides_clone, vehicles_rides, B, T)
scores[score] = vehicles_rides
if i % (count // 10) == 0:
progressBar(i, count - count // 10)
m_score = max(scores, key=float)
return scores[m_score]
def __init__(self):
getHotels()
url = "http://localhost:8080/RESTApp/REST_Resources/accomodation/buyAccomodation"
code = int(raw_input("\nPlease press the code of the Hotel: "))
try:
beginDate = raw_input("Start date (dd/mm/aaaa): ")
validate(beginDate)
endDate = raw_input("Final date (dd/mm/aaaa): ")
validate(endDate)
except ValueError:
print "Incorrect data format, should be dd/mm/aaaa\n"
return None
numberPerson = int(raw_input("Inform number of rooms: "))
age = []
for i in xrange(0, numberPerson):
age.append(
int(raw_input("Inform the age of client #" + str(i) + " ")))
print "Inform the credicard information:"
cardNumber = int(raw_input("Credicard number: "))
dateValid = raw_input("Validity (mm/aaaa): ")
secret = int(raw_input("Inform the secret number: "))
parts = int(raw_input("Inform how many months of paying: "))
##########################
# For testing purposes
# code = 1
# destination = "Sao Paulo"
# beginDate = "10/12/2016"
# endDate = "12/12/2016"
# numberPerson = 2
# age = [22, 23]
# cardNumber = 31232131
# dateValid = "12/2018"
# secret = 323
# parts = 1
##########################
accDes = accommodationDesire(code, beginDate, endDate, numberPerson,
age, cardNumber, dateValid, secret, parts)
response = exec_request(url, "post", json=accDes.to_JSON())
print str(response.text)
def api(htmlname):
if session.get('author','nologin') == 'nologin':
return redirect('/login')
headers['authorization'] = session['author']
validate_result = json.loads(util.validate(session['author'], app.config['passport_key']))
if int(validate_result['code']) == 0:
return render_template(htmlname+'.html',info=session,user=session['user'])
else:
return render_template(htmlname+'.html',errmsg=validate_result['errmsg'])
def rota():
if session.get('author','nologin') == 'nologin':
return redirect('/login')
headers['authorization'] = session['author']
validate_result = json.loads(util.validate(session['author'], app.config['passport_key']))
if int(validate_result['code']) == 0:
return render_template('rota.html',info=session)
else:
return render_template('rota.html',errmsg=validate_result['errmsg'])
def __init__(self):
getHotels()
url = "http://localhost:8080/RESTApp/REST_Resources/accomodation/buyAccomodation"
code = int(raw_input("\nPlease press the code of the Hotel: "))
try:
beginDate = raw_input("Start date (dd/mm/aaaa): ")
validate(beginDate)
endDate = raw_input("Final date (dd/mm/aaaa): ")
validate(endDate)
except ValueError:
print "Incorrect data format, should be dd/mm/aaaa\n"
return None
numberPerson = int(raw_input("Inform number of rooms: "))
age = []
for i in xrange(0, numberPerson):
age.append(int(raw_input("Inform the age of client #"+str(i)+" ")))
print "Inform the credicard information:"
cardNumber = int(raw_input("Credicard number: "))
dateValid = raw_input("Validity (mm/aaaa): ")
secret = int(raw_input("Inform the secret number: "))
parts = int(raw_input("Inform how many months of paying: "))
##########################
# For testing purposes
# code = 1
# destination = "Sao Paulo"
# beginDate = "10/12/2016"
# endDate = "12/12/2016"
# numberPerson = 2
# age = [22, 23]
# cardNumber = 31232131
# dateValid = "12/2018"
# secret = 323
# parts = 1
##########################
accDes = accommodationDesire(code, beginDate, endDate,
numberPerson, age, cardNumber, dateValid, secret, parts)
response = exec_request(url, "post", json = accDes.to_JSON())
print str(response.text)
def train_model(wrapped_model, model, model_path, train_loader, test_loader, init_lr, epochs, args):
train_loss_f = refinery_loss.RefineryLoss()
val_loss_f = nn.CrossEntropyLoss()
best_model_path = '.'.join(model_path.split('.')[:-1]) + '.best.pth'
# tracking stats
if not hasattr(model, 'stats'):
model.stats = {'train_loss': [], 'test_acc': [], 'test_loss': [],
'weight': [], 'lr': [], 'macs': [], 'efficiency': []}
start_epoch = 1
best_acc = 0
else:
start_epoch = len(model.stats['test_loss'])
best_acc = max(model.stats['test_acc']).item()
curr_weights, _ = util.num_nonzeros(model)
if hasattr(model, 'packed_layer_size'):
macs = np.sum([x*y for x, y in model.packed_layer_size])
else:
macs = curr_weights
# optimizer
optimizer = optim.RMSprop(util.group_weight(model), lr=init_lr, momentum=0.9, alpha=0.9,
weight_decay=0, eps=1.0)
print("Optimizer:")
print(optimizer)
# pruning stage
for epoch in range(start_epoch, epochs + 1):
print('[Epoch {}]'.format(epoch))
for g in optimizer.param_groups:
lr = g['lr']
break
train_loss = util.train(train_loader, wrapped_model, train_loss_f, optimizer, epoch-1, args)
test_loss, test_acc = util.validate(test_loader, model, val_loss_f, epoch-1, args)
print('LR :: {}'.format(lr))
print('Train Loss:: {}'.format(train_loss))
print('Test Loss:: {}'.format(test_loss))
print('Test Acc.:: {}'.format(test_acc))
print('Nonzeros :: {}'.format(curr_weights))
print('')
print('')
model.stats['lr'].append(lr)
model.optimizer = optimizer.state_dict()
model.cpu()
torch.save(model, model_path)
if test_acc > best_acc:
print('New best model found')
torch.save(model, best_model_path)
best_acc = test_acc
model.cuda()
def deploy_list():
if session.get('author','nologin') == 'nologin':
return redirect('/login')
session['pre_click'] = 'deploy'
headers['authorization'] = session['author']
validate_result = json.loads(util.validate(session['author'], app.config['passport_key']))
if int(validate_result['code']) == 0:
return render_template('deploylist.html',info=session,qs=request.args)
else:
return render_template('deploylist.html',errmsg=validate_result['errmsg'])
def pointQuery(api, attribute, display=False):
pointers = getPointers(api, attribute)
result = []
for p in pointers:
result += getValue(api, p)
if DO_VALIDATION:
result = validate(result, attribute[1:])
if display:
display(result)
return result
def generate(number: str):
util.validate(number)
numberList = []
placeHolders = []
index = 1
for n in number:
nInt = int(n)
while (util.isPowerOfTwo(index)):
placeHolders.append(index)
numberList.append(-1)
index += 1
numberList.append(nInt)
index += 1
combinations = []
greaterPower = placeHolders[len(placeHolders) - 1]
for c, n in enumerate(numberList):
i = c + 1
if util.isPowerOfTwo(i):
continue
combination = []
k = 1
while (k <= greaterPower):
if (k & i == k):
combination.append(k)
k *= 2
combinations.append({'index': i, 'combination': combination})
for n in placeHolders:
i = n - 1
appear = []
for c in combinations:
if n in c['combination']:
appear.append(c['index'] - 1)
value = numberList[appear[0]] ^ numberList[appear[1]]
for k in range(2, len(appear)):
value = value ^ numberList[appear[k]]
numberList[i] = value
return ''.join([str(n) for n in numberList])
def git():
if session.get('username') == None:
return redirect('/login')
session['pre_click'] = 'git'
headers['authorization'] = session['author']
validate_result = json.loads(
util.validate(session['author'], app.config['passport_key']))
name = session['username']
if int(validate_result['code']) == 0:
return render_template('git.html', info=session)
else:
return render_template('git.html', errmsg=validate_result['errmsg'])
def opsjob_hisrtory():
if session.get('author', 'nologin') == 'nologin':
return redirect('/login')
headers['authorization'] = session['author']
validate_result = json.loads(
util.validate(session['author'], app.config['passport_key']))
session['pre_click'] = 'opsjob'
if int(validate_result['code']) == 0:
return render_template('opsjob_alllist.html', info=session)
else:
return render_template('opsjob_alllist.html',
errmsg=validate_result['errmsg'])
def wrapper(*arg, **kwargs):
try:
authorization = request.headers.get('authorization', 'None')
res = util.validate(authorization, app.config['passport_key'])
res = json.loads(res)
if int(res['code']) == 1:
util.write_log('api').warning("Request forbiden:%s" % res['errmsg'])
return json.dumps({'code': 1, 'errmsg': '%s' % res['errmsg']})
except:
util.write_log('api').warning("Validate error: %s" % traceback.format_exc())
return json.dumps({'code': 1, 'errmsg': '验证异常'})
return func(res, *arg, **kwargs)
def wrapper(*arg, **kwargs):
try:
authorization = request.headers.get('authorization', 'None')
res = util.validate(authorization, app.config['passport_key'])
res = json.loads(res)
#res = {u'username': u'admin', u'code': 0, u'uid': u'1', u'r_id': u'1'}
if int(res['code']) == 1:
util.write_log('api').warning("Request forbiden:%s" % res['errmsg'])
return json.dumps({'code': 1, 'errmsg': '%s' % res['errmsg']})
except:
util.write_log('api').warning("Validate error: %s" % traceback.format_exc())
return json.dumps({'code': 1, 'errmsg': '验证异常'})
return func(res, *arg, **kwargs)
def cmdb(htmlname):
if session.get('author', 'nologin') == 'nologin':
return redirect('/login')
headers['authorization'] = session['author']
validate_result = json.loads(
util.validate(session['author'], current_app.config['passport_key']))
if int(validate_result['code']) == 0:
return render_template(htmlname + '.html',
info=session,
user=session['user'])
else:
return render_template(htmlname + '.html',
errmsg=validate_result['errmsg'])
def wrapper(*arg, **args):
try:
authorization = request.headers.get('authorization', 'None')
res = util.validate(authorization, app.config['passport_key'])
res = json.loads(res)
if int(res['code']) == 1:
logging.getLogger().warning("Request forbiden:%s" %
res['errmsg'])
return json.dumps({'code': 1, 'errmsg': '%s' % res['errmsg']})
except:
logging.getLogger().warning("Validate error: %s" %
traceback.format_exc())
return json.dumps({'code': 1, 'errmsg': '验证异常'})
return func(res, *arg, **args)
def dbreset():
if session.get('author', 'nologin') == 'nologin':
return redirect('/login')
session['pre_click'] = 'ext'
headers['authorization'] = session['author']
validate_result = json.loads(
util.validate(session['author'], app.config['passport_key']))
if request.method == 'GET':
if int(validate_result['code']) == 0:
return render_template('dbreset.html', info=session)
else:
return render_template('dbreset.html',
errmsg=validate_result['errmsg'])
def initial_process(model):
print(model)
util.print_model_parameters(model)
print("------------------------- Initial training -------------------------------")
util.initial_train(model, args, train_loader, val_loader, 'initial')
accuracy = util.validate(val_loader, model, args)
util.log(f"{args.save_dir}/{args.log}", f"weight\t{args.save_dir}/{args.out_oldweight_folder}")
util.log(f"{args.save_dir}/{args.log}", f"model\t{args.save_dir}/model_initial_end.ptmodel")
util.log(f"{args.save_dir}/{args.log}", f"initial_accuracy\t{accuracy}")
util.layer2torch(f"{args.save_dir}/{args.out_oldweight_folder}",model)
weight_list = util.parameters2list(model.children())
util.save_parameters(f"{args.save_dir}/{args.out_oldweight_folder}", weight_list)
return model
def login():
if request.method == 'POST':
username = request.form.get('username')
password = request.form.get('passwd')
url = "http://%s/api/login?username=%s&passwd=%s" % (app.config['api_host'],username,password)
r = requests.get(url, headers=headers) #请求API验证用户，并获取token
result = json.loads(r.content)
if result['code'] == 0:
token = result["authorization"]
res = util.validate(token,app.config['passport_key']) #解密token
res = json.loads(res) #return : dict(username:*,uid:*,role:*)
session['author'] = token
session['username'] = username
return json.dumps({'code':0})
else:
return json.dumps({'code':1,'errmsg':result['errmsg']})
return render_template('login.html')
def project(htmlname):
if session.get('author','nologin') == 'nologin':
return redirect('/login')
headers['authorization'] = session['author']
validate_result = json.loads(util.validate(session['author'], app.config['passport_key']))
if int(validate_result['code']) == 0:
data['method'] = 'userprojects.getlist'
data['params'] = {}
r = requests.post(get_url(),headers=headers,json=data)
result = json.loads(r.text)
result = json.loads(result['result'])
if int(result['code']) == 0:
return render_template(htmlname+'.html',info=session,user=session['user'],result=result['result'])
else:
return render_template(htmlname+'.html',info=session,result=result['errmsg'])
else:
return render_template(htmlname+'.html',errmsg=validate_result['errmsg'])
def get_user():
uid = request.args.get('uid')
if util.validate([uid]):
res = db.execute_query("select * from user where uid=?", (uid, ))
if len(res) == 1:
return {
'status': 200,
'msg': '查询成功',
'data': {
'username': res[0]['username'],
'phone': res[0]['phone'],
'nickname': res[0]['nickname']
}
}
else:
return {'status': 404, 'msg': '未知错误'}
else:
return {'status': 404, 'msg': '字段格式错误'}
def main():
net = Mininet(topo=TestTopo(lopts={"bw": 100}), controller=RemoteController, switch=OVSSwitch,
link=TCLink, autoSetMacs=True)
print "Starting network.."
net.start()
if len(sys.argv) == 2 and sys.argv[1] == 'legacy':
util.turn_legacy_on()
print "Waiting {} s ...".format(LEGACY_INITIALIZATION_DELAY)
time.sleep(LEGACY_INITIALIZATION_DELAY)
server_names = ['hs1', 'hs2', 'hs3']
host_names = []
print "Waiting {} s for initialization of mininet and controller...".format(INITIALIZATION_DELAY)
time.sleep(INITIALIZATION_DELAY)
util.test_init()
checksums = util.get_checksums()
# Configure servers
for hs in server_names:
print "Configuring server " + hs
util.launch_twistd_ftp(net, hs)
util.launch_twistd_web(net, hs)
util.launch_iperf(net, hs)
for hs in host_names:
print "Configuring hosts " + hs
util.launch_iperf(net, hs)
for (k, v) in background_traffic.items():
print 'Launching background traffic with thread for {} with {}'.format(k, v)
host_thread = util.HostThread(net, v[0])
host_thread.set_actions(v[1])
host_thread.start()
# sys.stdin.read(1)
results = []
valid = True
for i in xrange(TEST_ITERATIONS):
# TODO initialize environment (for download results)
threads = []
start = time.time()
for (k, v) in actions_map.items():
print 'Launching thread for {} with {}'.format(k, v)
host_thread = util.HostThread(net, v[0])
host_thread.set_actions(v[1])
threads.append(host_thread)
host_thread.start()
for t in threads:
t.join()
elapsed = time.time() - start
results.append(elapsed)
print "{}: elapsed time {}".format(i, elapsed)
# TODO check correctness
if not util.validate(checksums):
valid = False
if i < TEST_ITERATIONS-1:
print "Waiting before next execution: {} s ...".format(INTER_TEST_DELAY)
time.sleep(INTER_TEST_DELAY)
print ''
print "--- Test Summary ({}): ---".format("PASSED" if valid else "FAILED")
for i in xrange(TEST_ITERATIONS):
print "{}: {}".format(i, results[i])
print "Avg: {}".format(sum(results) / len(results))
print "------------------------------"
print ''
# makeTerm(net.getNodeByName('h1'))
# print "Press enter.."
# sys.stdin.read(1)
print "Killing twistd deamons"
call(['pkill', '-9', 'twistd'])
call(['pkill', '-9', 'iperf'])
# TODO stop timer
print "Stopping network.."
net.stop()
def hybrid_prop(instance, value):
validate(fset, value, fset.__pg_types__, fset.__pg_enum__)
fset(instance, value)