def put(self, cmd_list): #上传
filename = cmd_list[1]
self.conn.send(b'ok')
obj = self.conn.recv(4)
header_size = struct.unpack('i', obj)[0] # 获取报头长度
header_bytes = self.conn.recv(header_size) # 报头字节
header_json = header_bytes.decode('utf-8') # 报头字符串
header_dict = json.loads(header_json) # 报头字典
total_size = header_dict['total_size'] # 数据总长度
filename = header_dict['filename']
config = self._get_config()
quota_size = int(config[cmd_list[2]]['quota']) * 1024 * 1024
root_dir = os.path.join(os.path.join(settings.BASE_DIR, 'share'),
str(config[cmd_list[2]]['home_dir']))
current_home_size = self._get_dir_size(root_dir)
if current_home_size + total_size < quota_size:
self.conn.send('ok'.encode(self.coding))
with open('{}\{}'.format(self.current_dir, filename), 'wb') as f:
# print(total_size)
recv_size = 0
while recv_size < total_size: # 循环获取数据，直到完成
res = self.conn.recv(1024)
f.write(res)
recv_size += len(res)
# print('总大小：{} 已下载：{}'.format(total_size, recv_size))
new_file_md5 = self._get_md5('{}\{}'.format(
self.current_dir, filename))
if new_file_md5 == header_dict['md5']:
self.conn.send(new_file_md5.encode(self.coding))
logger.logger('{} put file {} successful'.format(
cmd_list[2], filename))
else:
self.conn.send('not'.encode(self.coding))
else:
self.conn.send('not'.encode(self.coding))
def get(self,cmd_list): #下载
filename = cmd_list[1]
file_list = os.listdir( self.current_dir)
for i in file_list:
if not os.path.isfile(os.path.join( self.current_dir, i)):
file_list.pop(file_list.index(i))
if filename not in file_list:
header_dict = {
'filename': filename,
'file_list': file_list,
} # 生成报头
header_json = json.dumps(header_dict) # 将报头转成字符串
header_bytes = header_json.encode('utf-8') # 将报头转成字节
self.conn.send(struct.pack('i', len(header_bytes)))
self.conn.send(header_bytes)
self.conn.send('file not exists'.encode(self.coding))
return
header_dict = {
'username':cmd_list[2],
'total_size':os.path.getsize('{}\{}'.format( self.current_dir,filename)),
'filename':filename,
'md5':self._get_md5('{}\{}'.format( self.current_dir,filename)),
'file_list':file_list,
} #生成报头
header_json = json.dumps(header_dict) #将报头转成字符串
header_bytes = header_json.encode('utf-8') #将报头转成字节
self.conn.send(struct.pack('i',len(header_bytes)))
self.conn.send(header_bytes)
with open('{}\{}'.format( self.current_dir,filename),'rb') as f:
for i in f:
self.conn.send(i)
logger.logger('{} get file {} successful'.format(cmd_list[2],filename))
def main(args):
device = torch.device('cuda:{}'.format(args.device)) \
if args.cuda else torch.device('cpu')
model = EfficientDet.from_pretrained(args.model).to(device) \
if args.pretrained else EfficientDet.from_name(args.model).to(device)
if args.mode == 'trainval':
logger("Model's trainable parameters: {}".format(
count_parameters(model)))
loader = get_loader(path=cfg.TRAIN_SET,
annotations=cfg.TRAIN_ANNOTATIONS,
batch_size=cfg.BATCH_SIZE)
optimizer, scheduler, criterion, ema_decay = build_tools(model)
writer = setup_writer(args.experiment, args)
best_score = -1
for epoch in range(cfg.NUM_EPOCHS):
model, optimizer, scheduler, writer = \
train(model, optimizer, loader, scheduler,
criterion, ema_decay, device, writer)
if epoch > cfg.VAL_DELAY and \
(epoch + 1) % cfg.VAL_INTERVAL == 0:
ema_decay.assign(model)
model, writer, best_score = \
validate(model, device, writer,
cfg.MODEL.SAVE_PATH, best_score=best_score)
ema_decay.resume(model)
elif args.mode == 'eval':
validate(model, device)
def put_money(self): #交学费
if self.tuition == False:
money = self.grade[0].course[0].price
print('未交学费,您的课程{}，需要{}'.format(self.grade[0].course[0].name,
money))
self.tuition = True
print('学费缴纳成功')
logger.logger('{} successful tuition fee'.format(self.name))
else:
print('已交学费')
def from_pretrained(name):
""" Interface for pre-trained model """
cfg.MODEL.choose_model(name)
model_to_return = EfficientDet(name)
if not cfg.MODEL.WEIGHTS.exists():
logger('Downloading pre-trained {}...'.format(cfg.MODEL.NAME))
download_model_weights(name, cfg.MODEL.WEIGHTS)
model_to_return._load_weights(cfg.MODEL.WEIGHTS)
return model_to_return
def create_grade(self): #创建班级
teacher_list = []
with open(settings.GRADE_FILE, 'rb') as f:
grade_dict = pickle.loads(f.read())
with open(settings.COURSE_FILE, 'rb') as f:
course_dict = pickle.loads(f.read())
with open(settings.USER_FILE, 'rb') as f:
user_dict = pickle.loads(f.read())
with open(settings.SCHOOL_FILE, 'rb') as f:
school_list = pickle.loads(f.read())
for i in school_list:
print(school_list.index(i), i.area)
area_num = input('请输入创建班级的校区序号').strip()
if area_num.isdigit() and int(area_num) in range(len(school_list)):
area = school_list[int(area_num)]
else:
print('无效的输入')
grade_name = input('输入班级名:').strip()
area.show_course()
grade_course = input('关联课程:').strip()
if grade_course not in course_dict:
print('无此课程')
return
for i in user_dict:
if user_dict[i]['role'] == 'teacher':
if user_dict[i]['username_obj'].school[0].area == area.area:
# print(user_dict[i]['username_obj'].school[0].area,area.area)
teacher_list.append(i)
if teacher_list:
print('本校区讲师：')
for i in teacher_list:
print(i)
else:
print('本校区暂无讲师，请先创建讲师')
return
grade_teacher = input('关联讲师:').strip()
if grade_teacher not in user_dict:
print('无此讲师')
return
grade_obj = course.grade(grade_name)
grade_obj.course.append(course_dict[grade_course])
grade_obj.teacher.append(user_dict[grade_teacher]['username_obj'])
# area.grade.append(grade_obj)
school_list[int(area_num)].grade.append(grade_obj)
grade_dict[grade_name] = grade_obj
print('创建班级{}成功'.format(grade_name))
logger.logger('{} successful create class {}'.format(
self.name, grade_name))
with open(settings.SCHOOL_FILE, 'wb') as f:
f.write(pickle.dumps(school_list))
with open(settings.GRADE_FILE, 'wb') as f:
f.write(pickle.dumps(grade_dict))
def from_name(name):
""" Interface for model prepared to train on COCO """
cfg.MODEL.choose_model(name)
model_to_return = EfficientDet(name)
if not cfg.MODEL.BACKBONE_WEIGHTS.exists():
logger('Downloading backbone {}...'.format(cfg.MODEL.BACKBONE))
download_model_weights(cfg.MODEL.BACKBONE, cfg.MODEL.BACKBONE_WEIGHTS)
model_to_return._load_backbone(cfg.MODEL.BACKBONE_WEIGHTS)
model_to_return._initialize_weights()
return model_to_return
def main(args):
device = torch.device('cuda:{}'.format(args.device)) \
if args.cuda else torch.device('cpu')
model = EfficientDet.from_pretrained(args.model).to(device) \
if args.pretrained else EfficientDet.from_name(args.model).to(device)
if args.mode == 'trainval':
logger("Model's trainable parameters: {}".format(
count_parameters(model)))
loader = get_loader(path=cfg.TRAIN_SET,
annotations=cfg.TRAIN_ANNOTATIONS,
batch_size=cfg.BATCH_SIZE)
optimizer, scheduler, criterion, ema_decay = build_tools(model)
writer = setup_writer(args.experiment, args)
best_score = -1
for epoch in range(cfg.NUM_EPOCHS):
model, optimizer, scheduler, writer = \
train(model, optimizer, loader, scheduler,
criterion, ema_decay, device, writer)
if epoch > cfg.VAL_DELAY and \
(epoch + 1) % cfg.VAL_INTERVAL == 0:
ema_decay.assign(model)
model, writer, best_score = \
validate(model, device, writer,
cfg.MODEL.SAVE_PATH, best_score=best_score)
ema_decay.resume(model)
elif args.mode == 'eval':
model.eval()
#model.to(memory_format=torch.channels_last)
with torch.no_grad():
model = torch.jit.script(model)
#model = torch.utils.mobile_optimizer.optimize_for_mobile(model)
#model(torch.randn((1,3,896,896)))
import sys, time
#N = 16
#t =- time.time()
#with torch.no_grad():
# for _ in range(N):
# model(torch.randn((1,3,896,896)))
#t += time.time()
#print(t/N)
#sys.exit(0)
import pdb
pdb.set_trace()
validate(model, device)
def get_msg(self): #查看个人信息
print('所在校区：{}'.format(self.school[0].area))
course_list = []
with open(settings.GRADE_FILE, 'rb') as f:
grade_dict = pickle.loads(f.read())
for i in grade_dict:
if grade_dict[i].teacher[0].name == self.name:
course_list.append(i)
if course_list:
print('所带班级：')
for i in course_list:
print(i)
else:
print('所带班级：无')
logger.logger('{} view personal information'.format(self.name))
def select_grade(self): #选择上课班级
course_list = []
with open(settings.GRADE_FILE, 'rb') as f:
grade_dict = pickle.loads(f.read())
print('以下为您所教的班级')
for i in grade_dict:
if grade_dict[i].teacher[0].name == self.name:
course_list.append(i)
print(i)
choose = input('选择您要上课的班级：').strip()
if choose in course_list:
print('您选择了{}班，去上课吧'.format(choose))
logger.logger('{} choose class {}'.format(self.name, choose))
else:
print('这不是您教的班级或无此班级')
def put(self, cmd):
if len(cmd.split()) != 2:
print('参数个数不正确')
return
file_list = os.listdir(settings.DOWNLOAD_DIR)
for i in file_list:
if not os.path.isfile(os.path.join(settings.DOWNLOAD_DIR, i)):
file_list.pop(file_list.index(i))
if cmd.split()[1] not in file_list:
print('此文件不存在')
return
self.client.send(cmd.encode('utf-8'))
self.client.recv(1024)
filename = cmd.split()[1]
header_dict = {
'total_size':
os.path.getsize('{}\{}'.format(settings.DOWNLOAD_DIR, filename)),
'filename':
filename,
'md5':
self.get_md5('{}\{}'.format(settings.DOWNLOAD_DIR, filename)),
} # 生成报头
total_size = header_dict['total_size']
header_json = json.dumps(header_dict) # 将报头转成字符串
header_bytes = header_json.encode('utf-8') # 将报头转成字节
self.client.send(struct.pack('i', len(header_bytes)))
self.client.send(header_bytes)
progress = self._progress_bar()
progress.__next__()
judge_quota = self.client.recv(1024).decode(self.coding)
if judge_quota == 'ok':
with open('{}\{}'.format(settings.DOWNLOAD_DIR, filename),
'rb') as f:
send_size = 0
for i in f:
self.client.send(i)
send_size += len(i)
current_percent = send_size / total_size * 100
progress.send(current_percent)
res = self.client.recv(1024).decode(self.coding)
if res == 'not': print('上传文件损坏')
else:
print('\n')
print('传输完整：{}'.format(res))
logger.logger('put file {} successful'.format(filename))
else:
print('空间不足，无法上传')
def _login(self): #登陆
config = self._get_config()
try:
user_info_bytes = self.conn.recv(1024)
except ConnectionResetError:
return
if user_info_bytes:
user_info = json.loads(user_info_bytes.decode(self.coding))
username = user_info['username']
password = user_info['password']
if username in config.sections() and password == config[username]['password']:
self.conn.send(username.encode(self.coding))
user_dict = {'username':username,'quota':config[username]['quota']}
logger.logger('{} login successful'.format(username))
return user_dict
else:self.conn.send('not'.encode(self.coding))
def get_msg(self): #获取个人信息
if len(self.school) != 0 and len(self.grade) != 0:
print('所在校区：{}'.format(self.school[0]))
print('所选课程：{}'.format(self.grade[0].course[0].name))
print('所在班级：{}'.format(self.grade[0].name))
if self.tuition == False:
print('学费状态：未交学费')
else:
print('学费状态：已交学费')
if self.fraction:
print('课程成绩：{}'.format(self.fraction))
else:
print('课程成绩：您还没有成绩')
logger.logger('{} view personal information'.format(self.name))
else:
print('您还未注册')
def login(self):
for i in range(3):
username = input('input your username:'******'input your password:'******'用户名或密码不能为空')
continue
m = hashlib.md5(password.encode(self.coding))
user_info = {'username':username,'password':m.hexdigest()}
user_info_json = json.dumps(user_info)
user_info_bytes = user_info_json.encode(self.coding)
self.client.send(user_info_bytes)
res = self.client.recv(1024).decode(self.coding)
if res == 'not':
print('用户名密码错误')
continue
else:
logger.logger('{} login successful'.format(username))
return res
def get(self, cmd):
if len(cmd.split()) != 2:
print('参数个数不正确')
return
self.client.send(cmd.encode('utf-8'))
obj = self.client.recv(4)
header_size = struct.unpack('i', obj)[0] #获取报头长度
header_bytes = self.client.recv(header_size) #报头字节
header_json = header_bytes.decode('utf-8') #报头字符串
header_dict = json.loads(header_json) #报头字典
filename = header_dict['filename']
if filename not in header_dict['file_list']:
data = self.client.recv(1024)
print(data.decode(self.coding))
else:
# self.shelve_obj[]
total_size = header_dict['total_size'] # 数据总长度
progress = self._progress_bar()
progress.__next__()
with open('{}\{}'.format(settings.DOWNLOAD_DIR, filename),
'wb') as f:
# print(total_size)
recv_size = 0
while recv_size < total_size: #循环获取数据，直到完成
res = self.client.recv(1024)
f.write(res)
recv_size += len(res)
current_percent = recv_size / total_size * 100
progress.send(current_percent)
new_file_md5 = self.get_md5('{}\{}'.format(settings.DOWNLOAD_DIR,
filename))
if new_file_md5 == header_dict['md5']:
print('\n')
print('传输完整：{}'.format(new_file_md5))
logger.logger('get file {} successful'.format(filename))
else:
print('传输过程中有数据丢失')
def registered(self): #注册
if len(self.school) == 0 and len(self.grade) == 0:
with open(settings.SCHOOL_FILE, 'rb') as f:
school_list = pickle.loads(f.read())
for i in school_list:
print(school_list.index(i), i.area)
select = input('选择您要报名的校区序号：').strip()
if select.isdigit() and int(select) in range(len(school_list)):
self.school.append(i.area)
for k in school_list[int(select)].grade:
print(school_list[int(select)].grade.index(k), k.name)
select1 = input('选择您要报名的班级序号：').strip()
if select.isdigit() and int(select) in range(len(school_list)):
self.grade.append(
school_list[int(select)].grade[int(select1)])
else:
print('无效的输入')
# print(self.school, self.grade)
print('注册成功')
logger.logger('{} successful regist'.format(self.name))
else:
print('您已注册过，请勿重复注册')
def view_student(self): #查看班级学员
course_list = []
with open(settings.GRADE_FILE, 'rb') as f:
grade_dict = pickle.loads(f.read())
with open(settings.USER_FILE, 'rb') as f:
user_dict = pickle.loads(f.read())
print('以下为您所教的班级')
for i in grade_dict:
if grade_dict[i].teacher[0].name == self.name:
print(i)
course_list.append(i)
choose = input('选择您要查看的班级：').strip()
if choose in course_list:
print('班级{}有以下学员：'.format(choose))
for i in user_dict:
if user_dict[i]['role'] == 'student':
if user_dict[i]['username_obj'].grade[0].name == choose:
print(i)
logger.logger('{} view class {} students'.format(
self.name, choose))
else:
print('这不是您教的班级或无此班级')
def update_fraction(self): #修改学员成绩
course_list = []
student_list = []
with open(settings.GRADE_FILE, 'rb') as f:
grade_dict = pickle.loads(f.read())
with open(settings.USER_FILE, 'rb') as f:
user_dict = pickle.loads(f.read())
print('以下为您所教的班级')
for i in grade_dict:
if grade_dict[i].teacher[0].name == self.name:
print(i)
course_list.append(i)
choose = input('选择您要查看的班级：').strip()
if choose in course_list:
print('班级{}有以下学员：'.format(choose))
for i in user_dict:
if user_dict[i]['role'] == 'student':
if user_dict[i]['username_obj'].grade[0].name == choose:
print(i)
student_list.append(i)
else:
print('这不是您教的班级或无此班级')
choose1 = input('选择您要修改成绩的学员:').strip()
if choose1 in student_list:
choose2 = input('输入学员分数：').strip()
if choose2.isdigit(): choose2 = int(choose2)
user_dict[choose1]['username_obj'].fraction = choose2
# print(user_dict[choose1]['username_obj'].fraction,choose2)
# print(user_dict['student4']['username_obj'].fraction)
with open(settings.USER_FILE, 'wb') as f:
f.write(pickle.dumps(user_dict))
logger.logger('{} Modify student {} performance'.format(
self.name, choose1))
print('修改成绩成功')
else:
print('无此学员或不是本班学员')
def main():
context = zmq.Context()
server = context.socket(zmq.REP)
server.bind("tcp://*:5555")
for cycles in itertools.count():
request = server.recv()
# Simulate various problems, after a few cycles
if cycles > 3 and randint(0, 3) == 0:
logger(Fore.BLUE + Back.WHITE + "Simulating a crash" +
Style.RESET_ALL)
break
elif cycles > 3 and randint(0, 3) == 0:
logger(Fore.BLUE + Back.WHITE + "Simulating CPU overload" +
Style.RESET_ALL)
time.sleep(2)
logger(Fore.BLUE + Back.WHITE + f"Normal request {request}" +
Style.RESET_ALL)
time.sleep(1) # Do some heavy work
server.send(request)
def _load_backbone(self, path):
self.backbone.model.load_state_dict(torch.load(path), strict=False)
logger('Loaded backbone checkpoint {}'.format(path))
def _load_weights(self, path):
self.load_state_dict(torch.load(path))
logger('Loaded checkpoint {}'.format(path))
def validate(model, device, writer=None, save_filename=None, best_score=0.0):
""" COCO VAL2017 """
model.eval()
wrapper = DetectionWrapper(model, device)
coco_gt = COCO(cfg.VAL_ANNOTATIONS)
image_ids = coco_gt.getImgIds()
batch_paths = []
batch_ids = []
processed_img_ids = []
results = []
with torch.no_grad():
for idx, image_id in tqdm(enumerate(image_ids)):
image_info = coco_gt.loadImgs(image_id)[0]
image_path = cfg.VAL_SET / image_info['file_name']
batch_paths.append(image_path)
batch_ids.append(image_id)
if (idx + 1) % cfg.BATCH_SIZE == 0:
output = wrapper(batch_paths, batch_ids)
for batch_out in output:
for det in batch_out:
image_id = int(det[0])
score = float(det[5])
coco_det = {
'image_id': image_id,
'bbox': det[1:5].tolist(),
'score': score,
'category_id': int(det[6]),
}
processed_img_ids.append(image_id)
results.append(coco_det)
batch_paths = []
batch_ids = []
json.dump(results, open(cfg.COCO_RESULTS, 'w'), indent=4)
coco_pred = coco_gt.loadRes(str(cfg.COCO_RESULTS))
coco_eval = COCOeval(coco_gt, coco_pred, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
if save_filename is not None and best_score < coco_eval.stats[0]:
logger('Saving model weights with score: {}'.format(
coco_eval.stats[0]))
torch.save(model.state_dict(), cfg.WEIGHTS_PATH / save_filename)
best_score = coco_eval.stats[0]
if writer is not None:
writer.add_scalar("Eval/mAP", coco_eval.stats[0], writer.eval_step)
writer.eval_step += 1
return model, writer, best_score
import pytest
import pandas as pd
import os
from pathlib import Path
import logging
from log import logger
from config import config
from utils import helper as hp
from utils import parser
logger = logger.logger(__name__,logging.DEBUG)
# root package
PACKAGE_ROOT = Path().resolve()
logger.info(f'Root path: {PACKAGE_ROOT}')
PACKAGE_ROOT_DATA = os.path.join(PACKAGE_ROOT,'data')
logger.info(f'Data path: {PACKAGE_ROOT_DATA}')
"""
Test Objective:
1. Test file and its path in config file
2. Unit test for the helper functions
3. Unit test for parser functions
# -*- coding: utf-8 -*- from log.logger import logger log = logger() from task.task_auto import task_auto auto = task_auto()
