class SubJobResult(db.Model): STATUS_OK = 1 STATUS_DELETE = 2 id = db.Column(db.BigInteger, primary_key=True) sub_job_id = db.Column(db.BigInteger, db.ForeignKey('sub_job.id')) result = db.Column(db.Text) created_time = db.Column(db.DateTime) status = db.Column(db.Integer, nullable=False) def __init__(self, sub_job, result, created_time=None, status=None): self.sub_job_id = sub_job.id self.result = json.dumps(result) if isinstance(result, ( dict, list, tuple, )) else result self.created_time = timezone.now( ) if created_time is None else created_time self.status = self.STATUS_OK if status is None else status @classmethod def create(cls, sub_job, result): job_result = cls(sub_job=sub_job, result=result) db.session.add(job_result) db.session.commit() return job_result
class Application(db.Model): STATUS_NEW = 0 STATUS_ACK = 1 STATUS_DELETE = 2 id = db.Column(db.BigInteger, primary_key=True) name = db.Column(db.String(128), nullable=False, default='') protocol = db.Column(db.String(32), nullable=False, default='') port = db.Column(db.Integer, nullable=False, default=0) state = db.Column(db.String(32), nullable=False, default='') product = db.Column(db.String(128), nullable=False, default='') version = db.Column(db.String(64), nullable=False, default='') asset_id = db.Column(db.BigInteger, db.ForeignKey('sys_asset.id')) created_time = db.Column(db.DateTime, nullable=False, default='') last_discove_time = db.Column(db.DateTime, nullable=False, default='') status = db.Column(db.Integer, nullable=False) @classmethod def stats(cls): return { 'total' : cls.query.filter(cls.status.notin_([cls.STATUS_DELETE])).count(), '24_hour' : cls.query.filter(cls.created_time>=timezone.now() - timedelta(days=1), cls.status.notin_([cls.STATUS_DELETE])).count(), } @classmethod def stats_port(cls): rs = db.session.execute('select port, count(*) from application where status != :status group by port order by count desc', {'status' : cls.STATUS_DELETE}) return [dict(r) for r in rs] @classmethod def all(cls): return cls.query.filter(cls.status.notin_([cls.STATUS_DELETE])).all() @classmethod def create_or_replace(cls, name, protocol, port, state, product, version, asset, *args, **kwargs): obj = db.session.query(cls).filter_by(asset_id=asset.id, port=port).first() if obj is None or obj.status == cls.STATUS_DELETE: obj = cls() obj.port = port obj.created_time = timezone.now() obj.status = cls.STATUS_NEW obj.asset_id = asset.id obj.name = name obj.protocol = protocol obj.state = state obj.product = product obj.last_discove_time = timezone.now() db.session.add(obj) db.session.commit()
class WebAsset(db.Model): STATUS_NEW = 0 STATUS_ACK = 1 STATUS_OFFLINE = 2 STATUS_DELETE = 3 id = db.Column(db.BigInteger, primary_key=True) name = db.Column(db.String(32), nullable=False, default='') url = db.Column(db.String(256), nullable=False, default='') group_id = db.Column(db.BigInteger, db.ForeignKey('web_group.id')) sys_asset_id = db.Column(db.BigInteger, db.ForeignKey('sys_asset.id')) manager = db.Column(db.String(64), nullable=False, default='') created_time = db.Column(db.DateTime, nullable=False, default='') last_discove_time = db.Column(db.DateTime, nullable=False, default='') status = db.Column(db.Integer, nullable=False) @classmethod def all(cls): return cls.query.filter(cls.status.notin_([cls.STATUS_DELETE])).all()
class SysAsset(db.Model, AsDictMixin): WEIGHT_HIGH = 10 WEIGHT_MEDIUM = 6 WEIGHT_LOW = 1 STATUS_NEW = 0 STATUS_ACK = 1 STATUS_DELETE = 2 id = db.Column(db.BigInteger, primary_key=True) sn = db.Column(db.String(128), nullable=False, default='') name = db.Column(db.String(32), nullable=False, default='') ip = db.Column(db.String(256), nullable=False, default='') mac = db.Column(db.String(32), nullable=False, default='') os = db.Column(db.String(128), nullable=False, default='') group_id = db.Column(db.BigInteger, db.ForeignKey('sys_group.id')) manager = db.Column(db.String(64), nullable=False, default='') weight = db.Column(db.Integer, nullable=False) created_time = db.Column(db.DateTime, nullable=False) last_discove_time = db.Column(db.DateTime, nullable=False) status = db.Column(db.Integer, nullable=False) applications = db.relationship('Application', backref='asset', lazy='dynamic') @classmethod def stats(cls): return { 'total' : cls.query.filter(cls.status.notin_([cls.STATUS_DELETE])).count(), '24_hour' : cls.query.filter(cls.created_time>=timezone.now() - timedelta(days=1), cls.status.notin_([cls.STATUS_DELETE])).count(), } @classmethod def all(cls): return cls.query.filter(cls.status.notin_([cls.STATUS_DELETE])).all() @classmethod def get_by_key(cls, value, key='id'): return db.session.query(cls).filter_by(**{key : value}).first() @classmethod def create_or_replace(cls, **kwargs): ip = kwargs.get('ip', '') obj = cls.get_by_key(ip, key='ip') try: if obj is None or obj.status == cls.STATUS_DELETE: obj = cls() obj.ip = ip obj.status = cls.STATUS_NEW obj.created_time = timezone.now() obj.weight = cls.WEIGHT_LOW obj.last_discove_time = timezone.now() obj.name = kwargs.get('name', obj.name) obj.os = kwargs.get('os', obj.os) obj.mac = kwargs.get('mac', obj.mac) db.session.add(obj) db.session.commit() for app in kwargs.get('apps', []): Application.create_or_replace(app.get('name', ''), app.get('protocol', ''), app.get('port', 0), app.get('state', ''), app.get('product', ''), app.get('version', ''), obj, ) except BaseException as e: logger.error(e) logger.exception(traceback.format_exc()) db.session.rollback() return obj @classmethod def delete_by_key(cls, id): obj = cls.get_by_key.get(id) if obj: obj.status = cls.STATUS_DELETE db.session.add(obj) db.session.commit() return obj
class Job(db.Model, AsDictMixin): STATUS_WATING = 1 STATUS_PREPROCESS = 2 STATUS_DOING = 3 STATUS_CANCEL = 4 STATUS_SUCCESS = 5 STATUS_FAILURE = 6 STATUS_DELETE = 7 PROGRESS_PREPROCESS = 0 PROGRESS_DOING = 5 PROGRESS_COMPLATE = 100 id = db.Column(db.BigInteger, primary_key=True) name = db.Column(db.String(64), nullable=False, default='') user_id = db.Column(db.Integer, db.ForeignKey('user.id')) type = db.Column(db.Integer, nullable=False) job_params = db.Column(db.Text) progress = db.Column(db.Integer, nullable=False, default=0) status = db.Column(db.Integer, nullable=False) explain = db.Column(db.String(512), nullable=False, default='') created_time = db.Column(db.DateTime, nullable=False) start_time = db.Column(db.DateTime) finish_time = db.Column(db.DateTime) sub_jobs = db.relationship('SubJob', backref="job", lazy='dynamic') def __init__(self, user=None, type=None, name='', job_params=None, status=None, explain='', created_time=None, start_time=None, finish_time=None, progress=0, id=None, user_id=None, from_cache=False, *args, **kwargs): self.id = id self.name = name self.user_id = user.id if user else user_id self.type = type job_params = {} if job_params is None else job_params self.job_params = json.dumps(job_params) if isinstance( job_params, (dict, )) else job_params self.progress = progress self.status = self.STATUS_WATING if status is None else status self.explain = explain self.created_time = timezone.now( ) if created_time is None else created_time self.start_time = start_time self.finish_time = finish_time self.from_cache = from_cache @property def job_params_object(self): try: return json.loads(self.job_params) except BaseException as e: return {} @classmethod def create(cls, user, type, name, job_params): job = cls(user=user, type=type, name=name, job_params=job_params) db.session.add(job) db.session.commit() redis.hmset(REDIS_KEYS['JOB_CONTENT'].format(id=job.id), job.as_dict_string()) redis.lpush(REDIS_KEYS['JOB_PREPROCESS'], job.id) return job def preprocess(self): job = self if getattr(self, 'from_cache', False): job = self.get_by_key(self.id) job.status = self.STATUS_PREPROCESS job.progress = self.PROGRESS_PREPROCESS db.session.add(job) db.session.commit() redis.hmset(REDIS_KEYS['JOB_CONTENT'].format(id=self.id), { 'status': self.STATUS_PREPROCESS, 'progress': self.PROGRESS_PREPROCESS }) redis.lpush(REDIS_KEYS['JOB_DOING'], self.id) def start(self): job = self if getattr(self, 'from_cache', False): job = self.get_by_key(self.id) if job.start_time is None: job.start_time = timezone.now() job.status = self.STATUS_DOING redis_content = {'status': self.STATUS_DOING} if job.progress < self.PROGRESS_DOING: job.progress = self.PROGRESS_DOING redis_content['progress'] = self.PROGRESS_DOING db.session.add(job) db.session.commit() redis.hmset(REDIS_KEYS['JOB_CONTENT'].format(id=self.id), { 'status': self.STATUS_DOING, 'progress': self.PROGRESS_DOING }) @classmethod def get_by_key(cls, value, key='id'): return db.session.query(cls).filter_by(**{key: value}).first() @classmethod def get_by_cache(cls, id): bjob = redis.hgetall(REDIS_KEYS['JOB_CONTENT'].format(id=id)) if bjob: job = {'from_cache': True} for k, v in bjob.items(): job[k] = int(v) if k in ('id', 'user_id', 'type', 'progress', 'status') else v return cls(**job) return None def finish(self, status=None, explain=''): job = self if getattr(self, 'from_cache', False): job = self.get_by_key(self.id) job.finish_time = timezone.now() job.status = self.STATUS_SUCCESS if status is None else self.STATUS_FAILURE job.progress = self.PROGRESS_COMPLATE job.explain = explain db.session.add(job) db.session.commit() redis.delete(REDIS_KEYS['JOB_CONTENT'].format(id=self.id)) redis.lrem(REDIS_KEYS['JOB_DOING'], 0, self.id) for _, key in JobType.KEYS.items(): redis.delete(REDIS_KEYS['JOB_QUEUE'].format(type=key, id=self.id)) return True def cancel(self): job = self if getattr(self, 'from_cache', False): job = self.get_by_key(self.id) job.status = self.STATUS_CANCEL job.progress = self.PROGRESS_COMPLATE SubJob.cancel(job) db.session.add(job) db.session.commit() redis.delete(REDIS_KEYS['JOB_CONTENT'].format(id=self.id)) redis.lrem(REDIS_KEYS['JOB_DOING'], 0, self.id) for _, key in JobType.KEYS.items(): redis.delete(REDIS_KEYS['JOB_QUEUE'].format(type=key, id=self.id)) return True def delete(self): job = self if getattr(self, 'from_cache', False): job = self.get_by_key(self.id) job.status = self.STATUS_DELETE job.progress = self.PROGRESS_COMPLATE SubJob.delete(job) db.session.add(job) db.session.commit() redis.delete(REDIS_KEYS['JOB_CONTENT'].format(id=self.id)) redis.lrem(REDIS_KEYS['JOB_DOING'], 0, self.id) for _, key in JobType.KEYS.items(): redis.delete(REDIS_KEYS['JOB_QUEUE'].format(type=key, id=self.id)) return True def doing(self, progress=None): job = self if getattr(self, 'from_cache', False): job = self.get_by_key(self.id) job.status = self.STATUS_DOING if progress is not None: job.progress = progress db.session.add(job) db.session.commit()
class SubJob(db.Model, AsDictMixin): STATUS_WATING = 1 STATUS_DOING = 2 STATUS_CANCEL = 3 STATUS_SUCCESS = 4 STATUS_FAILURE = 5 STATUS_DELETE = 6 id = db.Column(db.BigInteger, primary_key=True) job_id = db.Column(db.BigInteger, db.ForeignKey('job.id')) type = db.Column(db.Integer, nullable=False) job_params = db.Column(db.Text) created_time = db.Column(db.DateTime, nullable=False) start_time = db.Column(db.DateTime) finish_time = db.Column(db.DateTime) status = db.Column(db.Integer, nullable=False) executor = db.Column(db.String(128), nullable=False, default='') explain = db.Column(db.String(512), nullable=False, default='') def __init__(self, job, type, job_params=None, created_time=None, start_time=None, finish_time=None, status=None, executor='', explain='', *args, **kwargs): self.job_id = job.id self.type = type job_params = {} if job_params is None else job_params self.job_params = json.dumps(job_params) if isinstance( job_params, (dict, )) else job_params self.created_time = timezone.now( ) if created_time is None else created_time self.start_time = start_time self.finish_time = finish_time self.status = self.STATUS_WATING if status is None else status self.executor = executor self.explain = explain @property def job_params_object(self): try: return json.loads(self.job_params) except BaseException as e: return {} @classmethod def create(cls, job, type, job_params): sub_job = cls(job=job, type=type, job_params=job_params) db.session.add(sub_job) db.session.commit() redis.zadd( REDIS_KEYS['JOB_QUEUE'].format(type=JobType.KEYS.get(type), id=job.id), 0, sub_job.id) return sub_job @classmethod def get_by_key(cls, value, key='id'): return db.session.query(cls).filter_by(**{key: value}).first() @classmethod def cancel(cls, job): for obj in cls.query.filter( cls.job_id == job.id, cls.status.notin_([ cls.STATUS_SUCCESS, cls.STATUS_CANCEL, cls.STATUS_FAILURE, cls.STATUS_DELETE ])).all(): obj.status = cls.STATUS_CANCEL Executor.running_decr_by_ident(obj.executor, obj.type) db.session.add(obj) db.session.commit() return True @classmethod def delete(cls, job): for obj in cls.query.filter( cls.job_id == job.id, cls.status.notin_([ cls.STATUS_SUCCESS, cls.STATUS_CANCEL, cls.STATUS_FAILURE, cls.STATUS_DELETE ])).all(): obj.status = cls.STATUS_DELETE Executor.running_decr_by_ident(obj.executor, obj.type) db.session.add(obj) db.session.commit() return True def start(self, executor=None): self.status = self.STATUS_DOING self.executor = executor if executor else self.executor self.start_time = timezone.now() redis.zadd( REDIS_KEYS['JOB_QUEUE'].format(type=JobType.KEYS.get(self.type), id=self.job.id), -1, self.id) self.job.start() db.session.add(self) db.session.commit() def finish(self, status=None, explain=''): if self.status not in [ self.STATUS_SUCCESS, self.STATUS_CANCEL, self.STATUS_FAILURE, self.STATUS_DELETE ]: Executor.running_decr_by_ident(self.executor, self.type) self.status = self.STATUS_SUCCESS if status is None else status self.finish_time = timezone.now() self.explain = explain db.session.add(self) db.session.commit() redis.zadd( REDIS_KEYS['JOB_QUEUE'].format(type=JobType.KEYS.get(self.type), id=self.job.id), 1, self.id)
class User(db.Model, ValidatorMixin): STATUS_REGISTED = 0 STATUS_LOCKED = 1 STATUS_OK = 2 STATUS_DELETE = 3 id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(32), nullable=False, default='') password = db.Column(db.String(512), nullable=False, default='') email = db.Column(db.String(64), nullable=False, default='') role_id = db.Column(db.Integer, db.ForeignKey('role.id')) created_time = db.Column(db.DateTime, nullable=False) last_login_time = db.Column(db.DateTime) is_super = db.Column(db.Boolean, nullable=False, default=False) status = db.Column(db.Integer, nullable=False) def __init__(self, name, email, is_super=False, created_time=None, last_login_time=None, status=None, id=None, password=None, role_id=None): self.id = id self.name = name self.password = password self.email = email self.role_id = role_id self.created_time = created_time if created_time else timezone.now() self.last_login_time = last_login_time self.is_super = is_super self.status = self.STATUS_REGISTED if status is None else status @classmethod def authenticate(cls, name, password): user = cls.query.filter_by(name=name).first() if not user: user = cls.query.filter_by(email=name).first() if user and check_password_hash(user.password, password): user.last_login_time = timezone.now() db.session.add(user) db.session.commit() return user return None def set_password(self, password): self.password = generate_password_hash(password) def clean_name(self): name = self.name.strip() if not ValidatorUtils.is_length(name, 3, 32): raise ValidatorException('用户名必须为3到32个字符') user = User.query.filter_by(name=self.name).first() if user and (self.id is None or user.id != self.id): raise ValidatorException('用户名已存在') return name def clean_email(self): email = self.email.strip() if not ValidatorUtils.is_email(email): raise ValidatorException('邮箱格式不正确') user = User.query.filter_by(email=self.email).first() if user and (self.id is None or user.id != self.id): raise ValidatorException('邮箱已存在') return email @classmethod def create(cls, name, password, email, is_super): user = cls(name=name, email=email, is_super=is_super, status=cls.STATUS_OK) user.set_password(password) has_error, errors = user.valid() if has_error: return None, has_error, errors db.session.add(user) db.session.commit() return user, has_error, errors @classmethod def get_by_key(cls, value, key='id'): return cls.query.filter_by(**{key:value}).first() @classmethod def all(cls): return cls.query.filter(cls.status!=cls.STATUS_DELETE).all()