class SubJobResult(db.Model):
STATUS_OK = 1
STATUS_DELETE = 2
id = db.Column(db.BigInteger, primary_key=True)
sub_job_id = db.Column(db.BigInteger, db.ForeignKey('sub_job.id'))
result = db.Column(db.Text)
created_time = db.Column(db.DateTime)
status = db.Column(db.Integer, nullable=False)
def __init__(self, sub_job, result, created_time=None, status=None):
self.sub_job_id = sub_job.id
self.result = json.dumps(result) if isinstance(result, (
dict,
list,
tuple,
)) else result
self.created_time = timezone.now(
) if created_time is None else created_time
self.status = self.STATUS_OK if status is None else status
@classmethod
def create(cls, sub_job, result):
job_result = cls(sub_job=sub_job, result=result)
db.session.add(job_result)
db.session.commit()
return job_result
class Application(db.Model):
STATUS_NEW = 0
STATUS_ACK = 1
STATUS_DELETE = 2
id = db.Column(db.BigInteger, primary_key=True)
name = db.Column(db.String(128), nullable=False, default='')
protocol = db.Column(db.String(32), nullable=False, default='')
port = db.Column(db.Integer, nullable=False, default=0)
state = db.Column(db.String(32), nullable=False, default='')
product = db.Column(db.String(128), nullable=False, default='')
version = db.Column(db.String(64), nullable=False, default='')
asset_id = db.Column(db.BigInteger, db.ForeignKey('sys_asset.id'))
created_time = db.Column(db.DateTime, nullable=False, default='')
last_discove_time = db.Column(db.DateTime, nullable=False, default='')
status = db.Column(db.Integer, nullable=False)
@classmethod
def stats(cls):
return {
'total' : cls.query.filter(cls.status.notin_([cls.STATUS_DELETE])).count(),
'24_hour' : cls.query.filter(cls.created_time>=timezone.now() - timedelta(days=1), cls.status.notin_([cls.STATUS_DELETE])).count(),
}
@classmethod
def stats_port(cls):
rs = db.session.execute('select port, count(*) from application where status != :status group by port order by count desc', {'status' : cls.STATUS_DELETE})
return [dict(r) for r in rs]
@classmethod
def all(cls):
return cls.query.filter(cls.status.notin_([cls.STATUS_DELETE])).all()
@classmethod
def create_or_replace(cls, name, protocol, port, state, product, version, asset, *args, **kwargs):
obj = db.session.query(cls).filter_by(asset_id=asset.id, port=port).first()
if obj is None or obj.status == cls.STATUS_DELETE:
obj = cls()
obj.port = port
obj.created_time = timezone.now()
obj.status = cls.STATUS_NEW
obj.asset_id = asset.id
obj.name = name
obj.protocol = protocol
obj.state = state
obj.product = product
obj.last_discove_time = timezone.now()
db.session.add(obj)
db.session.commit()
class WebAsset(db.Model):
STATUS_NEW = 0
STATUS_ACK = 1
STATUS_OFFLINE = 2
STATUS_DELETE = 3
id = db.Column(db.BigInteger, primary_key=True)
name = db.Column(db.String(32), nullable=False, default='')
url = db.Column(db.String(256), nullable=False, default='')
group_id = db.Column(db.BigInteger, db.ForeignKey('web_group.id'))
sys_asset_id = db.Column(db.BigInteger, db.ForeignKey('sys_asset.id'))
manager = db.Column(db.String(64), nullable=False, default='')
created_time = db.Column(db.DateTime, nullable=False, default='')
last_discove_time = db.Column(db.DateTime, nullable=False, default='')
status = db.Column(db.Integer, nullable=False)
@classmethod
def all(cls):
return cls.query.filter(cls.status.notin_([cls.STATUS_DELETE])).all()
class SysAsset(db.Model, AsDictMixin):
WEIGHT_HIGH = 10
WEIGHT_MEDIUM = 6
WEIGHT_LOW = 1
STATUS_NEW = 0
STATUS_ACK = 1
STATUS_DELETE = 2
id = db.Column(db.BigInteger, primary_key=True)
sn = db.Column(db.String(128), nullable=False, default='')
name = db.Column(db.String(32), nullable=False, default='')
ip = db.Column(db.String(256), nullable=False, default='')
mac = db.Column(db.String(32), nullable=False, default='')
os = db.Column(db.String(128), nullable=False, default='')
group_id = db.Column(db.BigInteger, db.ForeignKey('sys_group.id'))
manager = db.Column(db.String(64), nullable=False, default='')
weight = db.Column(db.Integer, nullable=False)
created_time = db.Column(db.DateTime, nullable=False)
last_discove_time = db.Column(db.DateTime, nullable=False)
status = db.Column(db.Integer, nullable=False)
applications = db.relationship('Application', backref='asset', lazy='dynamic')
@classmethod
def stats(cls):
return {
'total' : cls.query.filter(cls.status.notin_([cls.STATUS_DELETE])).count(),
'24_hour' : cls.query.filter(cls.created_time>=timezone.now() - timedelta(days=1), cls.status.notin_([cls.STATUS_DELETE])).count(),
}
@classmethod
def all(cls):
return cls.query.filter(cls.status.notin_([cls.STATUS_DELETE])).all()
@classmethod
def get_by_key(cls, value, key='id'):
return db.session.query(cls).filter_by(**{key : value}).first()
@classmethod
def create_or_replace(cls, **kwargs):
ip = kwargs.get('ip', '')
obj = cls.get_by_key(ip, key='ip')
try:
if obj is None or obj.status == cls.STATUS_DELETE:
obj = cls()
obj.ip = ip
obj.status = cls.STATUS_NEW
obj.created_time = timezone.now()
obj.weight = cls.WEIGHT_LOW
obj.last_discove_time = timezone.now()
obj.name = kwargs.get('name', obj.name)
obj.os = kwargs.get('os', obj.os)
obj.mac = kwargs.get('mac', obj.mac)
db.session.add(obj)
db.session.commit()
for app in kwargs.get('apps', []):
Application.create_or_replace(app.get('name', ''),
app.get('protocol', ''),
app.get('port', 0),
app.get('state', ''),
app.get('product', ''),
app.get('version', ''),
obj,
)
except BaseException as e:
logger.error(e)
logger.exception(traceback.format_exc())
db.session.rollback()
return obj
@classmethod
def delete_by_key(cls, id):
obj = cls.get_by_key.get(id)
if obj:
obj.status = cls.STATUS_DELETE
db.session.add(obj)
db.session.commit()
return obj
class Job(db.Model, AsDictMixin):
STATUS_WATING = 1
STATUS_PREPROCESS = 2
STATUS_DOING = 3
STATUS_CANCEL = 4
STATUS_SUCCESS = 5
STATUS_FAILURE = 6
STATUS_DELETE = 7
PROGRESS_PREPROCESS = 0
PROGRESS_DOING = 5
PROGRESS_COMPLATE = 100
id = db.Column(db.BigInteger, primary_key=True)
name = db.Column(db.String(64), nullable=False, default='')
user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
type = db.Column(db.Integer, nullable=False)
job_params = db.Column(db.Text)
progress = db.Column(db.Integer, nullable=False, default=0)
status = db.Column(db.Integer, nullable=False)
explain = db.Column(db.String(512), nullable=False, default='')
created_time = db.Column(db.DateTime, nullable=False)
start_time = db.Column(db.DateTime)
finish_time = db.Column(db.DateTime)
sub_jobs = db.relationship('SubJob', backref="job", lazy='dynamic')
def __init__(self,
user=None,
type=None,
name='',
job_params=None,
status=None,
explain='',
created_time=None,
start_time=None,
finish_time=None,
progress=0,
id=None,
user_id=None,
from_cache=False,
*args,
**kwargs):
self.id = id
self.name = name
self.user_id = user.id if user else user_id
self.type = type
job_params = {} if job_params is None else job_params
self.job_params = json.dumps(job_params) if isinstance(
job_params, (dict, )) else job_params
self.progress = progress
self.status = self.STATUS_WATING if status is None else status
self.explain = explain
self.created_time = timezone.now(
) if created_time is None else created_time
self.start_time = start_time
self.finish_time = finish_time
self.from_cache = from_cache
@property
def job_params_object(self):
try:
return json.loads(self.job_params)
except BaseException as e:
return {}
@classmethod
def create(cls, user, type, name, job_params):
job = cls(user=user, type=type, name=name, job_params=job_params)
db.session.add(job)
db.session.commit()
redis.hmset(REDIS_KEYS['JOB_CONTENT'].format(id=job.id),
job.as_dict_string())
redis.lpush(REDIS_KEYS['JOB_PREPROCESS'], job.id)
return job
def preprocess(self):
job = self
if getattr(self, 'from_cache', False):
job = self.get_by_key(self.id)
job.status = self.STATUS_PREPROCESS
job.progress = self.PROGRESS_PREPROCESS
db.session.add(job)
db.session.commit()
redis.hmset(REDIS_KEYS['JOB_CONTENT'].format(id=self.id), {
'status': self.STATUS_PREPROCESS,
'progress': self.PROGRESS_PREPROCESS
})
redis.lpush(REDIS_KEYS['JOB_DOING'], self.id)
def start(self):
job = self
if getattr(self, 'from_cache', False):
job = self.get_by_key(self.id)
if job.start_time is None:
job.start_time = timezone.now()
job.status = self.STATUS_DOING
redis_content = {'status': self.STATUS_DOING}
if job.progress < self.PROGRESS_DOING:
job.progress = self.PROGRESS_DOING
redis_content['progress'] = self.PROGRESS_DOING
db.session.add(job)
db.session.commit()
redis.hmset(REDIS_KEYS['JOB_CONTENT'].format(id=self.id), {
'status': self.STATUS_DOING,
'progress': self.PROGRESS_DOING
})
@classmethod
def get_by_key(cls, value, key='id'):
return db.session.query(cls).filter_by(**{key: value}).first()
@classmethod
def get_by_cache(cls, id):
bjob = redis.hgetall(REDIS_KEYS['JOB_CONTENT'].format(id=id))
if bjob:
job = {'from_cache': True}
for k, v in bjob.items():
job[k] = int(v) if k in ('id', 'user_id', 'type', 'progress',
'status') else v
return cls(**job)
return None
def finish(self, status=None, explain=''):
job = self
if getattr(self, 'from_cache', False):
job = self.get_by_key(self.id)
job.finish_time = timezone.now()
job.status = self.STATUS_SUCCESS if status is None else self.STATUS_FAILURE
job.progress = self.PROGRESS_COMPLATE
job.explain = explain
db.session.add(job)
db.session.commit()
redis.delete(REDIS_KEYS['JOB_CONTENT'].format(id=self.id))
redis.lrem(REDIS_KEYS['JOB_DOING'], 0, self.id)
for _, key in JobType.KEYS.items():
redis.delete(REDIS_KEYS['JOB_QUEUE'].format(type=key, id=self.id))
return True
def cancel(self):
job = self
if getattr(self, 'from_cache', False):
job = self.get_by_key(self.id)
job.status = self.STATUS_CANCEL
job.progress = self.PROGRESS_COMPLATE
SubJob.cancel(job)
db.session.add(job)
db.session.commit()
redis.delete(REDIS_KEYS['JOB_CONTENT'].format(id=self.id))
redis.lrem(REDIS_KEYS['JOB_DOING'], 0, self.id)
for _, key in JobType.KEYS.items():
redis.delete(REDIS_KEYS['JOB_QUEUE'].format(type=key, id=self.id))
return True
def delete(self):
job = self
if getattr(self, 'from_cache', False):
job = self.get_by_key(self.id)
job.status = self.STATUS_DELETE
job.progress = self.PROGRESS_COMPLATE
SubJob.delete(job)
db.session.add(job)
db.session.commit()
redis.delete(REDIS_KEYS['JOB_CONTENT'].format(id=self.id))
redis.lrem(REDIS_KEYS['JOB_DOING'], 0, self.id)
for _, key in JobType.KEYS.items():
redis.delete(REDIS_KEYS['JOB_QUEUE'].format(type=key, id=self.id))
return True
def doing(self, progress=None):
job = self
if getattr(self, 'from_cache', False):
job = self.get_by_key(self.id)
job.status = self.STATUS_DOING
if progress is not None:
job.progress = progress
db.session.add(job)
db.session.commit()
class SubJob(db.Model, AsDictMixin):
STATUS_WATING = 1
STATUS_DOING = 2
STATUS_CANCEL = 3
STATUS_SUCCESS = 4
STATUS_FAILURE = 5
STATUS_DELETE = 6
id = db.Column(db.BigInteger, primary_key=True)
job_id = db.Column(db.BigInteger, db.ForeignKey('job.id'))
type = db.Column(db.Integer, nullable=False)
job_params = db.Column(db.Text)
created_time = db.Column(db.DateTime, nullable=False)
start_time = db.Column(db.DateTime)
finish_time = db.Column(db.DateTime)
status = db.Column(db.Integer, nullable=False)
executor = db.Column(db.String(128), nullable=False, default='')
explain = db.Column(db.String(512), nullable=False, default='')
def __init__(self,
job,
type,
job_params=None,
created_time=None,
start_time=None,
finish_time=None,
status=None,
executor='',
explain='',
*args,
**kwargs):
self.job_id = job.id
self.type = type
job_params = {} if job_params is None else job_params
self.job_params = json.dumps(job_params) if isinstance(
job_params, (dict, )) else job_params
self.created_time = timezone.now(
) if created_time is None else created_time
self.start_time = start_time
self.finish_time = finish_time
self.status = self.STATUS_WATING if status is None else status
self.executor = executor
self.explain = explain
@property
def job_params_object(self):
try:
return json.loads(self.job_params)
except BaseException as e:
return {}
@classmethod
def create(cls, job, type, job_params):
sub_job = cls(job=job, type=type, job_params=job_params)
db.session.add(sub_job)
db.session.commit()
redis.zadd(
REDIS_KEYS['JOB_QUEUE'].format(type=JobType.KEYS.get(type),
id=job.id), 0, sub_job.id)
return sub_job
@classmethod
def get_by_key(cls, value, key='id'):
return db.session.query(cls).filter_by(**{key: value}).first()
@classmethod
def cancel(cls, job):
for obj in cls.query.filter(
cls.job_id == job.id,
cls.status.notin_([
cls.STATUS_SUCCESS, cls.STATUS_CANCEL, cls.STATUS_FAILURE,
cls.STATUS_DELETE
])).all():
obj.status = cls.STATUS_CANCEL
Executor.running_decr_by_ident(obj.executor, obj.type)
db.session.add(obj)
db.session.commit()
return True
@classmethod
def delete(cls, job):
for obj in cls.query.filter(
cls.job_id == job.id,
cls.status.notin_([
cls.STATUS_SUCCESS, cls.STATUS_CANCEL, cls.STATUS_FAILURE,
cls.STATUS_DELETE
])).all():
obj.status = cls.STATUS_DELETE
Executor.running_decr_by_ident(obj.executor, obj.type)
db.session.add(obj)
db.session.commit()
return True
def start(self, executor=None):
self.status = self.STATUS_DOING
self.executor = executor if executor else self.executor
self.start_time = timezone.now()
redis.zadd(
REDIS_KEYS['JOB_QUEUE'].format(type=JobType.KEYS.get(self.type),
id=self.job.id), -1, self.id)
self.job.start()
db.session.add(self)
db.session.commit()
def finish(self, status=None, explain=''):
if self.status not in [
self.STATUS_SUCCESS, self.STATUS_CANCEL, self.STATUS_FAILURE,
self.STATUS_DELETE
]:
Executor.running_decr_by_ident(self.executor, self.type)
self.status = self.STATUS_SUCCESS if status is None else status
self.finish_time = timezone.now()
self.explain = explain
db.session.add(self)
db.session.commit()
redis.zadd(
REDIS_KEYS['JOB_QUEUE'].format(type=JobType.KEYS.get(self.type),
id=self.job.id), 1, self.id)
class User(db.Model, ValidatorMixin):
STATUS_REGISTED = 0
STATUS_LOCKED = 1
STATUS_OK = 2
STATUS_DELETE = 3
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(32), nullable=False, default='')
password = db.Column(db.String(512), nullable=False, default='')
email = db.Column(db.String(64), nullable=False, default='')
role_id = db.Column(db.Integer, db.ForeignKey('role.id'))
created_time = db.Column(db.DateTime, nullable=False)
last_login_time = db.Column(db.DateTime)
is_super = db.Column(db.Boolean, nullable=False, default=False)
status = db.Column(db.Integer, nullable=False)
def __init__(self, name, email, is_super=False, created_time=None, last_login_time=None, status=None, id=None, password=None, role_id=None):
self.id = id
self.name = name
self.password = password
self.email = email
self.role_id = role_id
self.created_time = created_time if created_time else timezone.now()
self.last_login_time = last_login_time
self.is_super = is_super
self.status = self.STATUS_REGISTED if status is None else status
@classmethod
def authenticate(cls, name, password):
user = cls.query.filter_by(name=name).first()
if not user:
user = cls.query.filter_by(email=name).first()
if user and check_password_hash(user.password, password):
user.last_login_time = timezone.now()
db.session.add(user)
db.session.commit()
return user
return None
def set_password(self, password):
self.password = generate_password_hash(password)
def clean_name(self):
name = self.name.strip()
if not ValidatorUtils.is_length(name, 3, 32):
raise ValidatorException('用户名必须为3到32个字符')
user = User.query.filter_by(name=self.name).first()
if user and (self.id is None or user.id != self.id):
raise ValidatorException('用户名已存在')
return name
def clean_email(self):
email = self.email.strip()
if not ValidatorUtils.is_email(email):
raise ValidatorException('邮箱格式不正确')
user = User.query.filter_by(email=self.email).first()
if user and (self.id is None or user.id != self.id):
raise ValidatorException('邮箱已存在')
return email
@classmethod
def create(cls, name, password, email, is_super):
user = cls(name=name, email=email, is_super=is_super, status=cls.STATUS_OK)
user.set_password(password)
has_error, errors = user.valid()
if has_error:
return None, has_error, errors
db.session.add(user)
db.session.commit()
return user, has_error, errors
@classmethod
def get_by_key(cls, value, key='id'):
return cls.query.filter_by(**{key:value}).first()
@classmethod
def all(cls):
return cls.query.filter(cls.status!=cls.STATUS_DELETE).all()