class Version(BaseModel, db.Model):
"""版本"""
__tablename__ = "ds_versions"
__table_args__ = {'extend_existing': True}
id = db.Column(db.Integer, primary_key=True, autoincrement=True) # 版本编号
collection_id = db.Column(db.Integer,
db.ForeignKey('ds_collections.id'),
nullable=False)
name = db.Column(db.String(32), nullable=False) # 版本名称
description = db.Column(db.String(128)) # 版本描述
zip_path = db.Column(db.String(64)) # 版本地址
average_number = db.Column(db.Integer, default=0) # 平均框值
mean_size = db.Column(db.String(16), default='0*0') # 平均大小
label_info = db.Column(db.String(512)) # 标签使用详情
images = db.relationship("Image", backref="version") # 版本/图片一对多
def to_dict(self):
"""将对象转换为字典数据"""
version_dict = {
"version_id": self.id,
"version_name": self.name,
"version_description": self.description,
"version_average_number": self.average_number,
"version_mean_size": self.mean_size
}
return version_dict
class Image(BaseModel, db.Model):
"""图片"""
__tablename__ = "ds_images"
__table_args__ = {'extend_existing': True}
id = db.Column(db.Integer, primary_key=True, autoincrement=True) # 图片编号
version_id = db.Column(db.Integer,
db.ForeignKey('ds_versions.id'),
nullable=True) # 版本图片
collection_id = db.Column(db.Integer,
db.ForeignKey('ds_collections.id'),
nullable=False) # 集合图片
name = db.Column(db.String(128), nullable=False) # 图片名称
site = db.Column(db.String(128), nullable=False) # 图片地址
label_path = db.Column(db.String(128), nullable=True) # 图片标注路径
status = db.Column(db.Integer, default=0) # 图片状态(是否标注)
structured_path = db.Column(db.String(128), nullable=True) # 图片结构化路径
md5 = db.Column(db.String(32), index=True)
def to_dict(self):
"""将对象转换为字典数据"""
image_dict = {
"id": self.id,
"name": self.name,
"site": self.site,
"label_path": self.label_path,
"state": self.state,
"structured_path": self.structured_path
}
return image_dict
class Collection(BaseModel, db.Model):
"""集合"""
__tablename__ = "ds_collections"
__table_args__ = {'extend_existing': True}
id = db.Column(db.Integer, primary_key=True, autoincrement=True) # 编号
user_id = db.Column(db.Integer,
db.ForeignKey('ds_users.id'),
nullable=False)
name = db.Column(db.String(32), nullable=False) # 集合名字
type = db.Column(db.Integer, default=1) # 类型 (分类/人脸 | 检测/切割)
desc = db.Column(db.String(128)) # 数据集描述
images = db.relationship("Image", backref="collection") # 集合/图片一对多
versions = db.relationship("Version", backref="collection") # 集合/版本一对多
labels = db.relationship("Label", backref="collection") # 集合/标签一对多
parameter_1 = db.Column(db.String(32))
parameter_2 = db.Column(db.Integer)
def to_dict(self):
"""将对象转换为字典数据"""
collection_dict = {
"id": self.id,
"name": self.name,
"type": self.type,
"desc": self.desc,
}
return collection_dict
class TrainResult(db.Model, BaseModel):
__tablename__ = 'result_models'
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
user_id = db.Column(db.Integer, nullable=False)
model_id = db.Column(db.Integer, nullable=False)
task_id = db.Column(db.Integer, nullable=False)
plt_path = db.Column(db.String(512))
log_path = db.Column(db.String(512))
trained_path = db.Column(db.String(512))
@staticmethod
def check_expire():
all_result = TrainResult.query.order_by(TrainResult.create_time).all()
if all_result is None:
return None
for obj in all_result:
if obj.create_time is None:
continue
if (datetime.now() - obj.create_time).days >= 30:
res = TrainResult().del_result(obj)
if res:
db.session.delete(obj)
db.session.commit()
print 'delete all expire trained file and server.log file'
@staticmethod
def del_result(obj):
try:
if obj.trained_path:
if os.path.exists(obj.trained_path):
shutil.rmtree(obj.trained_path)
if obj.log_path:
if os.path.exists(obj.log_path):
os.remove(obj.log_path)
except Exception as e:
print e
return None
return True
@staticmethod
def update_log(trained_path, log_path):
update_obj = TrainResult.query.filter_by(
trained_path=trained_path).first()
if update_obj is None:
return None
try:
update_obj.log_path = log_path
db.session.add(update_obj)
db.session.commit()
except Exception as e:
print e
return None
return 'complete'
class Dataset(db.Model, BaseModel):
__tablename__ = 'dataset_models'
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
user_id = db.Column(db.Integer, nullable=False)
dataset_id = db.Column(db.Integer, nullable=False)
dataset_var = db.Column(db.String(128), nullable=False)
dataset_path = db.Column(db.String(512), nullable=False)
@staticmethod
def check_expire():
all_dataset = Dataset.query.order_by(Dataset.create_time).all()
if all_dataset is None:
return None
for obj in all_dataset:
if obj.create_time is None:
continue
if (datetime.now() - obj.create_time).days >= 30:
res = Dataset().del_dataset(obj)
if res:
db.session.delete(obj)
db.session.commit()
print 'delete all expire dataset'
@staticmethod
def del_dataset(obj):
if obj.dataset_path is None:
return None
try:
ds_objs = json.loads(obj.dataset_path)
for _, v in ds_objs.items():
if os.path.exists(v):
try:
os.remove(v)
except Exception as e:
print e
continue
except Exception as e:
print e
return None
return True
@staticmethod
def create_test_data():
for i in range(5):
ds = Dataset(user_id=1,
dataset_id=1,
dataset_var='[1,2,%s]' % i,
dataset_path='/path/test/%s' % i)
db.session.add(ds)
db.session.commit()
class Share(db.Model):
__tablename__ = "ds_shares"
__table_args__ = {'extend_existing': True}
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
my_share = db.Column(db.String(128)) # 我的分享 "{1:[4,5,6], 2:[7,8]}"
share_collection = db.Column(db.String(128)) # 被分享的数据集 "[1,2,3,4]"
users = db.relationship('User', backref='share',
uselist=False) # 参数uselist=False 表示一对一
def to_dict(self):
"""将对象转换为字典数据"""
share_dict = {"share_collection": self.share_collection}
return share_dict
class Role(db.Model):
__tablename__ = 'roles'
__table_args__ = {'extend_existing': True}
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(64), unique=True)
default = db.Column(db.Boolean, default=False,
index=True) # 用户默认角色,default是False
permissions = db.Column(db.Integer) # 用户权限设置，是一个数值
# lazy='dynamic'表示表Role对象点users属性，返回的是对象，仍可进行过滤等相关操作，默认lazy='select'
users = db.relationship('User', backref='role',
lazy='dynamic') # lazy='select'时查询的时对应User表的结果
def __init__(self, **kwargs):
super(Role, self).__init__(**kwargs)
if self.permissions is None:
self.permissions = 0
def __repr__(self):
return '<Role %r>' % self.name
@staticmethod
def insert_roles():
roles = {
'User': (Permission.UPDATE, ), # 权限为1
'Moderator': (Permission.UPDATE, Permission.CRUD), # 权限 1，2
'Administrator': (Permission.UPDATE, Permission.CRUD,
Permission.ADMINISTER) # 权限 1，2，4
}
default_role = 'User'
for r in roles: # 通过角色查找现有角色去设置权限，方便修改权限和添加新角色
role = Role.query.filter_by(name=r).first()
if role is None: # 当数据库中没有某个角色时创建个新角色，并添加user权限
role = Role(name=r)
role.reset_permissions()
for perm in roles[r]: # 遍历 User对应的权限元组
role.add_permission(perm)
role.default = (role.name == default_role) # 判断是不是默认用户
db.session.add(role)
db.session.commit()
def has_permission(self, perm): # 判断当前角色是否已经有该权限
return self.permissions & perm == perm
def add_permission(self, perm): # 角色添加权限
if not self.has_permission(perm):
self.permissions += perm
def remove_permiss(self, perm): # 角色删除权限
if self.has_permission(perm):
self.permissions -= perm
def reset_permissions(self): # 重置角色
self.permissions = 1
class Label(BaseModel, db.Model):
"""标签"""
__tablename__ = "ds_labels"
__table_args__ = {'extend_existing': True}
id = db.Column(db.Integer, primary_key=True, autoincrement=True) # 标签编号
collection_id = db.Column(db.Integer, db.ForeignKey('ds_collections.id'), nullable=True) # 集合标签
label_id = db.Column(db.Integer) # 集合内的标签ID
name = db.Column(db.String(32), nullable=False) # 标签名称/类别
count = db.Column(db.Integer, default=0) # 标签使用量
def to_dict(self):
"""将对象转换为字典数据"""
label_dict = {
"label_id": self.label_id,
"label_name": self.name,
"label_count": self.count
}
return label_dict
class Tasks(db.Model, BaseModel):
__tablename__ = 'task_models'
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
name = db.Column(db.String(64), nullable=False)
description = db.Column(db.String(128))
state = db.Column(db.String(32), default="训练中")
train_dataset_id = db.Column(db.Integer, nullable=False)
train_dataset_var = db.Column(db.String(128), nullable=False)
train_dataset_path = db.Column(db.String(256))
test_dataset_id = db.Column(db.Integer, nullable=False)
test_dataset_var = db.Column(db.String(128), nullable=False)
test_dataset_path = db.Column(db.String(256))
parameter = db.Column(db.String(256))
user_id = db.Column(db.Integer)
model_id = db.Column(db.Integer, nullable=False)
def to_dict(self):
task_dict = {
"task_id": self.id,
"task_name": self.name,
"task_description": self.description,
"task_state": self.state,
"task_create_time": self.create_time
}
return task_dict
def to_dict_detail(self):
task_dict_detail = {
"task_id": self.id,
"task_name": self.name,
"task_description": self.description,
"task_create_time": self.create_time,
"task_state": self.state,
"train_data_id": self.train_dataset_id,
"parameter": self.parameter,
}
return task_dict_detail
@staticmethod
def from_json(json_model, frame):
name = json_model.get('name')
description = json_model.get('description')
model_id = json_model.get('model_id')
train_dataset_id = json_model.get('train_dataset_id')
train_dataset_var = json_model.get('train_var_ids')
test_dataset_id = json_model.get('test_dataset_id')
test_dataset_var = json_model.get('test_var_ids')
if not all([name, model_id, train_dataset_id, train_dataset_var, test_dataset_id, test_dataset_var]):
return None
if not (isinstance(train_dataset_var, list) or isinstance(test_dataset_var, list)):
return None
if frame == 1:
# kears
if 'batch_size' in json_model and not isinstance(json_model['batch_size'], int):
return None
if 'epoch'in json_model and not isinstance(json_model['epoch'], int):
return None
if 'mean' in json_model and not isinstance(json_model['mean'], list):
return None
if 'scale' in json_model and not (0 < json_model['scale'] < 1):
return None
if 'mirror' in json_model and not isinstance(json_model['mirror'], bool):
return None
if 'vertical_flip' in json_model and not isinstance(json_model['vertical_filp'], bool):
return None
if 'lr_policy' in json_model and not (json_model['lr_policy'] in [0, 1]):
return None
if 'optimizer' in json_model and not (json_model['optimizer'] in [0, 1]):
return None
if frame == 2:
#caffe
if 'batch_size' in json_model and not isinstance(json_model['batch_size'], int):
return None
if 'max_iter'in json_model and not isinstance(json_model['max_iter'], int):
return None
if 'mean' in json_model and not isinstance(json_model['mean'], list):
return None
if 'scale' in json_model and not (0 < json_model['scale'] < 1):
return None
if 'mirror' in json_model and not isinstance(json_model['mirror'], bool):
return None
if 'base_lr' in json_model and not isinstance(json_model['base_lr'], float):
return None
if 'lr_policy' in json_model and not (json_model['lr_policy'] in [0, 1]):
return None
if 'gamma' in json_model and not isinstance(json_model['gamma'], float):
return None
if 'optimizer' in json_model and not (json_model['optimizer'] in [0, 1]):
return None
if 'test_interval' in json_model and not isinstance(json_model['test_interval'], int):
return None
return Tasks(name=name, description=description, model_id=model_id, train_dataset_id=train_dataset_id,
train_dataset_var=str(train_dataset_var), test_dataset_id=test_dataset_id, test_dataset_var=str(test_dataset_var))
@staticmethod
def from_json_caffe(json_model):
name = json_model.get('name')
description = json_model.get('description')
model_id = json_model.get('model_id')
train_dataset_id = json_model.get('train_dataset_id')
train_dataset_var = json_model.get('train_dataset_var')
test_dataset_id = json_model.get('test_dataset_id')
test_dataset_var = json_model.get('test_dataset_var')
if not all([name, model_id, train_dataset_id, train_dataset_var, test_dataset_id, test_dataset_var]):
return None
if not (isinstance(train_dataset_var, list) or isinstance(test_dataset_var, list)):
return Nonejson_model.get('parameter')
return Tasks(name=name, description=description, model_id=model_id, train_dataset_id=train_dataset_id,
train_dataset_var=train_dataset_var, test_dataset_id=test_dataset_id,
test_dataset_var=test_dataset_var)
class Models(db.Model, BaseModel):
__tablename__ = 'train_models'
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
name = db.Column(db.String(64), nullable=False)
description = db.Column(db.String(128))
frame = db.Column(db.Integer, default=1, nullable=False)
type = db.Column(db.Integer, default=1, nullable=False)
define_path = db.Column(db.String(256), nullable=False)
weight_path = db.Column(db.String(256), nullable=False)
label = db.Column(db.Integer, nullable=False)
is_default = db.Column(db.Boolean, default=False)
user_id = db.Column(db.Integer)
version = db.Column(db.String(64), default='None')
def to_dict(self):
model_dict = {
"model_id": self.id,
"model_name": self.name,
"model_description": self.description,
"model_frame": self.frame,
"model_type": self.type,
"model_label": self.label,
"model_version": self.version,
}
return model_dict
def to_default_dict(self):
default_dict = {
"default_id": self.id,
"default_define": self.define_path,
"default_weight": self.weight_path,
"default_width": self.width,
"default_height": self.height,
"is_default": self.is_default
}
return default_dict
@staticmethod
def insert_default():
init_model = [{'id':1, 'name':'keras', 'description':'vgg16', 'frame':1, 'type':1, 'define_path': 'vgg16',
'weight_path': 'vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5', 'label':1, 'is_default':1},
{'id':2, 'name':'keras', 'description':'resnet50', 'frame':1, 'type':1, 'define_path': 'resnet50',
'weight_path': 'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5', 'label':2, 'is_default':1},
{'id':3, 'name':'caffe', 'description':'vgg16', 'frame':2, 'type':1, 'define_path': 'vgg16',
'weight_path': 'vgg16net', 'label':3, 'is_default':1},
{'id':4, 'name':'caffe', 'description':'resnet50', 'frame':2, 'type':1, 'define_path': 'resnet50',
'weight_path': 'resnet50net', 'label':4, 'is_default':1}]
for row in init_model:
obj = Models(id=row['id'], name=row['name'], description=row['description'], frame=row['frame'], type=row['type'],
define_path=row['define_path'], weight_path=row['weight_path'], label=row['label'], is_default=row['is_default'])
db.session.add(obj)
db.session.commit()
@staticmethod
def from_json_default(json_model):
model = Models.query.filter_by(id=json_model.get('default_id')).first()
if model is None:
return None
name = json_model.get('name')
description = json_model.get('description')
label = json_model.get('label')
frame = model.frame
type = model.type
define_path = model.define_path
weight_path = model.weight_path
if not all([name, frame, type, define_path, weight_path, label]):
return None
if not isinstance(label, int) or label <= 0:
return None
return Models(name=name, description=description, frame=frame, type=type,
define_path=define_path, weight_path=weight_path, width=width,
height=height, label=label)
@staticmethod
def from_json(json_model):
name = json_model.get('name')
description = json_model.get('description')
frame = json_model.get('frame')
type = json_model.get('type')
define_path = json_model.get('define_path')
weight_path = json_model.get('weight_path')
label = json_model.get('label')
if not all([name, frame, type, define_path, weight_path, label]):
return None
if not isinstance(label, int) or label <= 0:
return None
return Models(name=name, description=description, frame=frame, type=type,
define_path=define_path, weight_path=weight_path, width=width,
height=height, label=label)
class User(db.Model, BaseModel):
__tablename__ = 'ds_users'
__table_args__ = {'extend_existing': True}
id = db.Column(db.Integer, primary_key=True, autoincrement=True) # 用户编号
role_id = db.Column(db.Integer, db.ForeignKey('roles.id')) # ???
shares_id = db.Column(db.Integer,
db.ForeignKey('ds_shares.id')) # 用户/分享集合一对一
name = db.Column(db.String(32), unique=True, nullable=False) # 用户昵称
email = db.Column(db.String(128), index=True, unique=True) # 邮箱
password_hash = db.Column(db.String(128), nullable=False) # 加密的密码
login_time = db.Column(db.Integer) # 登录时间验证
collections = db.relationship("Collection", backref="user") # 用户/集合一对多
def __repr__(self):
return '<User %r>' % self.name # 实例化User类后，直接打印实例返回name属性，默认返回一个对象，现在是重写
def __init__(self, **kwargs):
super(User, self).__init__(**kwargs)
if self.role is None:
if self.name == current_app.config['ADMIN']:
self.role = Role.query.filter_by(name='Administrator').first()
elif self.email == current_app.config['ADMIN_EMAIL']:
self.role = Role.query.filter_by(name='Administrator').first()
else:
self.role = Role.query.filter_by(default=True).first()
if self.share is None:
self.share = Share()
@property
def password(self):
"""获取password属性时被调用"""
raise AttributeError("不可读")
@password.setter
def password(self, passwd):
"""设置password属性时被调用，设置密码加密"""
self.password_hash = generate_password_hash(passwd)
def check_password(self, passwd):
"""检查密码的正确性"""
return check_password_hash(self.password_hash, passwd)
# 查看用户是否有权限
def can(self, permission):
return self.role is not None and \
(self.role.permissions & permission) == permission
# 验证用户是否管理员
def is_administrator(self):
return self.can(Permission.ADMINISTER)
@staticmethod
def get(id):
return User.query.filter_by(id=id).first()
def add(self, user):
db.session.add(user)
return session_commit()
def update(self):
return session_commit()
def delete(self, id):
self.query.filter_by(id=id).delete()
return session_commit()
def to_dict(self):
"""将对象转换为字典数据"""
user_dict = {
"id": self.id,
"name": self.name,
}
return user_dict