def create_or_replace(cls, **kwargs):
ip = kwargs.get('ip', '')
obj = cls.get_by_key(ip, key='ip')
try:
if obj is None or obj.status == cls.STATUS_DELETE:
obj = cls()
obj.ip = ip
obj.status = cls.STATUS_NEW
obj.created_time = timezone.now()
obj.weight = cls.WEIGHT_LOW
obj.last_discove_time = timezone.now()
obj.name = kwargs.get('name', obj.name)
obj.os = kwargs.get('os', obj.os)
obj.mac = kwargs.get('mac', obj.mac)
db.session.add(obj)
db.session.commit()
for app in kwargs.get('apps', []):
Application.create_or_replace(app.get('name', ''),
app.get('protocol', ''),
app.get('port', 0),
app.get('state', ''),
app.get('product', ''),
app.get('version', ''),
obj,
)
except BaseException as e:
logger.error(e)
logger.exception(traceback.format_exc())
db.session.rollback()
return obj
def create_or_replace(cls, key, plugin_ident, payloads):
obj = cls.query.filter_by(ip=key, plugin_ident=plugin_ident).first()
if obj is None:
obj = cls()
obj.ip = key
obj.plugin_ident = plugin_ident
obj.created_time = timezone.now()
obj.last_discover_time = timezone.now()
obj.payloads = json.dumps(payloads) if isinstance(
payloads, (list, dict)) else payloads
db.session.add(obj)
db.session.commit()
async def create_price_history(self, db_connection, stock):
query = price_history.insert().values(product_stock_id=stock.id,
price_base=stock.price_base,
price_sale=stock.price_sale,
price_card=stock.price_card,
dt=now())
await db_connection.execute(query)
def consume(self,callback,isjson=True):
"""
Return True if some resource has been consumed; otherwise return False
"""
status = self.status
if status[0]:
#the latest resource has been consumed
return False
resource_client = AzureBlob(status[1][0],connection_string,container_name)
if isjson:
callback(resource_client.json)
else:
res_file = resource_client.download()
try:
with open(res_file,'rb') as f:
callback(f)
finally:
#after processing,remove the downloaded local resource file
os.remove(res_file)
#update the client consume data
client_metdata = {
"resource_id" : status[1][0],
"publish_date" : status[1][1],
"consume_date": timezone.now()
}
self.update(client_metadata)
return True
def archive_by_date(d,
delete_after_archive=False,
check=False,
overwrite=False,
backup_to_archive_table=True,
rearchive=False):
"""
Archive the logged point within the specified date
delete_after_archive: delete the archived data from table tracking_loggedpoint
check: check whether archiving is succeed or not
overwrite: if true, overwrite the existing archived file;if false, throw exception if already archived
rearchive: if true, rearchive the existing archived file;if false, throw exception if already archived
return a tuple (archived or not, archive type(archive,overwrite,rearchive),archived_metadata)
"""
now = timezone.now()
today = now.date()
if d >= today:
raise Exception(
"Can only archive the logged points happened before today")
archive_group = get_archive_group(d)
archive_id = get_archive_id(d)
start_date = timezone.datetime(d.year, d.month, d.day)
end_date = start_date + timedelta(days=1)
backup_table = get_backup_table(d) if backup_to_archive_table else None
return archive(archive_group,
archive_id,
start_date,
end_date,
delete_after_archive=delete_after_archive,
check=check,
overwrite=overwrite,
rearchive=rearchive,
backup_table=backup_table)
def get(self, request, *args, **kwargs):
"""
下载。/?download
HTML预 /?html
PDF预 /
Return a HTTPResponse either of a PDF file or HTML.
:rtype: HttpResponse
"""
self.get_options_filename()
if 'html' in request.GET:
# Output HTML
content = self.render_html(*args, **kwargs)
return HttpResponse(content)
else:
# Output PDF
content = self.render_pdf(*args, **kwargs)
response = HttpResponse(content, content_type='application/pdf')
if (not self.inline or 'download' in request.GET) and 'inline' not in request.GET:
response['Content-Disposition'] = 'attachment; filename="%s"' % self.get_filename().format(
name='{}'.format(str(timezone.now()).split('.')[0])
)
response['Content-Length'] = len(content)
return response
def save_with_log(self, user_id):
if not self.create_user_id:
self.create_user_id = user_id
self.update_user_id = user_id
if not self.create_time:
self.create_time = timezone.now()
self.save()
def get_periodic_sorted_tasks(self, schedule):
dt_now = localtime(now())
tasks = get_tasks(schedule)
tasks_periodic = []
count_every = 0
for task in tasks:
type_task = task.type
if type_task == 'every':
time_task = task.time
offset_minutes = count_every + OFFSET_FOR_EVERY
dt_task = dt_now + datetime.timedelta(minutes=offset_minutes)
dt_limit = dt_task + datetime.timedelta(days=1)
while dt_task < dt_limit:
struct_time = dt_task.timetuple()
time = {
'hour': struct_time.tm_hour, 'minute': struct_time.tm_min,
'second': struct_time.tm_sec, 'microsecond': 0,
}
tasks_periodic.append(TaskType(task.name, 'periodic', task.executor, time))
time_task_kw = {
'hours': time_task.get('hour', 0),
'minutes': time_task.get('minute', 0),
'seconds': time_task.get('second', 0),
}
dt_task = dt_task + datetime.timedelta(**time_task_kw)
count_every += 1
else:
tasks_periodic.append(task)
return sorted(tasks_periodic, key=lambda item: dt_now.replace(**item.time))
def __init__(self,
user=None,
type=None,
name='',
job_params=None,
status=None,
explain='',
created_time=None,
start_time=None,
finish_time=None,
progress=0,
id=None,
user_id=None,
from_cache=False,
*args,
**kwargs):
self.id = id
self.name = name
self.user_id = user.id if user else user_id
self.type = type
job_params = {} if job_params is None else job_params
self.job_params = json.dumps(job_params) if isinstance(
job_params, (dict, )) else job_params
self.progress = progress
self.status = self.STATUS_WATING if status is None else status
self.explain = explain
self.created_time = timezone.now(
) if created_time is None else created_time
self.start_time = start_time
self.finish_time = finish_time
self.from_cache = from_cache
async def notice_msg_for_send_iter(self):
bulk_size = 10
send_ids = []
while True:
dt_send = now()
async with db.engine.acquire() as db_connection:
# .where(notice_msg.c.dt_send.is_(None))\
query = sa.select((notice_msg.c.id, user.c.user_id_orig, user.c.first_name, notice_msg.c.text))\
.select_from(notice_msg.join(user, user.c.id == notice_msg.c.user_id))\
.where(notice_msg.c.status == StatusNoticeEnum.not_send.value)\
.order_by(user.c.id)\
.limit(bulk_size)
res = await db_connection.execute(query)
recs = await res.fetchall()
if len(recs) == 0:
break
send_ids.clear()
for item in recs:
yield item.user_id_orig, item.first_name, item.text
send_ids.append(item.id)
async with db.engine.acquire() as db_connection:
query = notice_msg.update().values(
dt_send=dt_send, status=StatusNoticeEnum.in_progress.value
)\
.where(notice_msg.c.id.in_(send_ids))
await db_connection.execute(query)
def result(self, sub_job, type, result):
data = result.get('data', [])
if sub_job.type == JobType.SYS_ASSET_DISCOVER:
job = sub_job.job
for asset in data:
# 存储资产
obj = SysAsset.create_or_replace(**asset)
job_params = {
'ip': obj.ip,
'plugins': job.job_params_object.get('plugins', []),
}
# 下载漏洞检查任务
SubJob.create(job, JobType.SYS_VUL_CHECK, job_params)
elif sub_job.type == JobType.SYS_VUL_CHECK:
now = timezone.now()
for asset in data:
key = asset.get('key', '')
for k, v in asset.items():
if k == 'key':
continue
AssetSysVul.create_or_replace(key=key,
plugin_ident=k,
payloads=v.get(
'payloads', {}))
AssetSysVul.delete_not_found(key, now)
sub_job.finish(status=result.get('status'),
explain=result.get('explain', ''))
SubJobResult.create(sub_job, data)
def __get__(self, instance, owner):
"""
Get the value, or generate if needed
"""
if instance is None:
return self
try:
return getattr(instance, self.propname)
except AttributeError:
pass
key = self._make_key(instance)
client = create_client()
document = client.collection("cache").document(key).get()
if not document.exists:
value = None
else:
data = document.to_dict()
if data["expires"] <= timezone.now():
value = None
else:
value = data["value"]
if value is None:
value = self.getter(instance)
self.validate_and_set(document.reference, value)
self.set_value(instance, value)
return value
def create_or_replace(cls, name, protocol, port, state, product, version, asset, *args, **kwargs):
obj = db.session.query(cls).filter_by(asset_id=asset.id, port=port).first()
if obj is None or obj.status == cls.STATUS_DELETE:
obj = cls()
obj.port = port
obj.created_time = timezone.now()
obj.status = cls.STATUS_NEW
obj.asset_id = asset.id
obj.name = name
obj.protocol = protocol
obj.state = state
obj.product = product
obj.last_discove_time = timezone.now()
db.session.add(obj)
db.session.commit()
def stats(cls):
return {
'total':
cls.query.count(),
'24_hour':
cls.query.filter(cls.created_time >= timezone.now() -
timedelta(days=1)).count(),
}
def _archive(self):
"""
Archive the az logs
return True if some az logs are archived; False if nothing is archived
"""
query_start, query_end = self.get_query_interval()
if not query_end:
logger.info(
"All az logs have been archived.the end time of the last archiving is {}"
.format(query_start))
return False
logger.info("Archive az logs between {} and {}".format(
query_start, query_end))
resource_id = self.get_resource_id(query_start)
metadata = {
"start_archive": timezone.now(),
"resource_id": resource_id,
self.ARCHIVE_STARTTIME: query_start,
self.ARCHIVE_ENDTIME: query_end
}
self.set_metadata(metadata)
try:
dump_file = None
with tempfile.NamedTemporaryFile(
suffix=".json",
prefix=self.settings.RESOURCE_NAME,
delete=False) as f:
dump_file = f.name
if self.settings.TENANT:
login_cmd = "az login --service-principal -u {} -p {} --tenant {}".format(
self.settings.USER, self.settings.PASSWORD,
self.settings.TENANT)
else:
login_cmd = "az login -u {} -p {}".format(
self.settings.USER, self.settings.PASSWORD)
cmd = "{}&&az monitor log-analytics query -w {} --analytics-query '{}' -t {}/{} > {}".format(
login_cmd, self.settings.WORKSPACE, self.settings.QUERY,
timezone.utctime(query_start).strftime("%Y-%m-%dT%H:%M:%SZ"),
timezone.utctime(query_end).strftime("%Y-%m-%dT%H:%M:%SZ"),
dump_file)
subprocess.check_output(cmd, shell=True)
with open(dump_file, 'r') as f:
data = json.loads(f.read())
metadata["log_records"] = len(data)
resourcemetadata = self.resource_repository.push_file(
dump_file,
metadata,
f_post_push=_set_end_datetime("end_archive"))
return True
finally:
remove_file(dump_file)
def stats(cls):
return {
'total':
db.session.query(cls).filter(cls.status == cls.STATUS_OK).count(),
'online':
db.session.query(cls).filter(
cls.status == cls.STATUS_OK, cls.heartbeat_time >=
timezone.now() - timedelta(minutes=3)).count()
}
def __init__(self, sub_job, result, created_time=None, status=None):
self.sub_job_id = sub_job.id
self.result = json.dumps(result) if isinstance(result, (
dict,
list,
tuple,
)) else result
self.created_time = timezone.now(
) if created_time is None else created_time
self.status = self.STATUS_OK if status is None else status
def start(self, executor=None):
self.status = self.STATUS_DOING
self.executor = executor if executor else self.executor
self.start_time = timezone.now()
redis.zadd(
REDIS_KEYS['JOB_QUEUE'].format(type=JobType.KEYS.get(self.type),
id=self.job.id), -1, self.id)
self.job.start()
db.session.add(self)
db.session.commit()
def __init__(self, name, email, is_super=False, created_time=None, last_login_time=None, status=None, id=None, password=None, role_id=None):
self.id = id
self.name = name
self.password = password
self.email = email
self.role_id = role_id
self.created_time = created_time if created_time else timezone.now()
self.last_login_time = last_login_time
self.is_super = is_super
self.status = self.STATUS_REGISTED if status is None else status
def _archive_file(storage,
f,
resource_id,
checking_policy,
check_md5,
metadata={}):
#push the updated or new files into storage
file_status = os.stat(f)
file_modify_date = file_status.st_mtime_ns
file_size = file_status.st_size
if check_md5:
file_md5 = utils.file_md5(f)
else:
file_md5 = None
try:
res_metadata = storage.get_resource_metadata(resource_id)
except ResourceNotFound as ex:
res_metadata = None
is_changed = False
for policy in checking_policy:
if policy == FILE_MD5:
if not res_metadata or res_metadata.get("file_md5") != file_md5:
is_changed = True
break
elif policy == FILE_MODIFY_DATE:
if not res_metadata or res_metadata.get(
"file_modify_date") != file_modify_date:
is_changed = True
break
elif policy == FILE_SIZE:
if not res_metadata or res_metadata.get("file_msize") != file_size:
is_changed = True
break
else:
raise Exception("Checking policy({}) Not Support".format(policy))
if not is_changed:
logger.debug(
"File({},{}) is not changed, no need to archive again".format(
f, resource_id))
return False
metadata["archive_time"] = timezone.now()
metadata["resource_id"] = resource_id
metadata["file_modify_date"] = file_modify_date
metadata["file_size"] = file_size
if check_md5:
metadata["file_md5"] = file_md5
storage.push_file(f, metadata=metadata)
logger.debug("File({},{}) was archived successfully.".format(
f, resource_id))
return True
def authenticate(cls, name, password):
user = cls.query.filter_by(name=name).first()
if not user:
user = cls.query.filter_by(email=name).first()
if user and check_password_hash(user.password, password):
user.last_login_time = timezone.now()
db.session.add(user)
db.session.commit()
return user
return None
class User(Model):
uid = CharField(verbose_name='用户id',
max_length=32,
index=True,
unique=True,
primary_key=True)
username = CharField(verbose_name='用户名', max_length=24)
password = CharField(128, verbose_name='密码')
last_login = DateTimeField(verbose_name='last_login',
default=timezone.now())
phone = CharField(verbose_name='手机号', null=True, max_length=13)
nickname = CharField(verbose_name='昵称', max_length=24, null=True)
is_active = BooleanField(default=True)
is_admin = BooleanField(default=False)
register_time = DateTimeField(null=True, default=timezone.now())
version = IntegerField(verbose_name='密码版本', default=1)
email = CharField(20, verbose_name='邮箱', null=True)
# 0女1男
gender = CharField(1, verbose_name='性别', default='1')
avatar = CharField(256, verbose_name='头像', default='')
# 0普通用户 1管理员
role = CharField(1, verbose_name='角色', default='0')
deleted = CharField(1, verbose_name='是否删除', default='0')
def __str__(self):
return self.get_username()
def set_password(self, raw_password):
self.password = generate_password_hash(raw_password)
return self.password
def check_password(self, raw_password):
return check_password_hash(self.password, raw_password)
def save(self, *args, **kwargs):
super().save(*args, **kwargs)
class Meta:
database = BaseModel.db
table_name = 'system_user'
def __init__(self,
key,
value=None,
status=None,
created_time=None,
from_cache=False,
id=None):
self.id = id
self.key = key
self.value = value
self.status = self.STATUS_OK if status is None else status
self.created_time = timezone.now(
) if created_time is None else created_time
self.from_cache = from_cache
def need_delete(meta):
resourceid = meta["resource_id"]
file_folder, file_name = os.path.split(resourceid)
if file_name[0] == ".":
return True
else:
if not settings.FILE_RE.search(file_name):
return True
if meta.get(ResourceConstant.DELETED_KEY, False) and meta.get(
ResourceConstant.DELETE_TIME_KEY) and timezone.now() > meta.get(
ResourceConstant.DELETE_TIME_KEY) + DELETED_RESROURCE_EXPIRED:
return True
return False
def finish(self, status=None, explain=''):
if self.status not in [
self.STATUS_SUCCESS, self.STATUS_CANCEL, self.STATUS_FAILURE,
self.STATUS_DELETE
]:
Executor.running_decr_by_ident(self.executor, self.type)
self.status = self.STATUS_SUCCESS if status is None else status
self.finish_time = timezone.now()
self.explain = explain
db.session.add(self)
db.session.commit()
redis.zadd(
REDIS_KEYS['JOB_QUEUE'].format(type=JobType.KEYS.get(self.type),
id=self.job.id), 1, self.id)
def finish(self, status=None, explain=''):
job = self
if getattr(self, 'from_cache', False):
job = self.get_by_key(self.id)
job.finish_time = timezone.now()
job.status = self.STATUS_SUCCESS if status is None else self.STATUS_FAILURE
job.progress = self.PROGRESS_COMPLATE
job.explain = explain
db.session.add(job)
db.session.commit()
redis.delete(REDIS_KEYS['JOB_CONTENT'].format(id=self.id))
redis.lrem(REDIS_KEYS['JOB_DOING'], 0, self.id)
for _, key in JobType.KEYS.items():
redis.delete(REDIS_KEYS['JOB_QUEUE'].format(type=key, id=self.id))
return True
def get_next(self, call_in_start=False):
while True:
try:
next_item = self.queue.popleft()
wait_seconds = self.get_wait_seconds(next_item.time)
if wait_seconds < 0 and call_in_start:
continue
break
except IndexError: # очередь закончилась
self.queue = deque(self.tasks)
next_item = self.queue.popleft()
wait_seconds = self.get_wait_seconds(next_item.time)
if wait_seconds < 0:
dt_reference = localtime(now()) + datetime.timedelta(days=1)
wait_seconds = self.get_wait_seconds(next_item.time, dt_reference)
break
return next_item.executor, wait_seconds
def start(self):
job = self
if getattr(self, 'from_cache', False):
job = self.get_by_key(self.id)
if job.start_time is None:
job.start_time = timezone.now()
job.status = self.STATUS_DOING
redis_content = {'status': self.STATUS_DOING}
if job.progress < self.PROGRESS_DOING:
job.progress = self.PROGRESS_DOING
redis_content['progress'] = self.PROGRESS_DOING
db.session.add(job)
db.session.commit()
redis.hmset(REDIS_KEYS['JOB_CONTENT'].format(id=self.id), {
'status': self.STATUS_DOING,
'progress': self.PROGRESS_DOING
})
def get_query_interval(self):
"""
Return the current query interval(start,end) based on the last archiving;
return (start,None) if all logs are archived
"""
last_resource = self.resource_repository.last_resource
if last_resource:
query_start = last_resource[1][self.ARCHIVE_ENDTIME]
else:
query_start = self.settings.QUERY_START
query_end = query_start + self.settings.QUERY_DURATION
if query_end < timezone.now() - self.settings.LOG_DELAY_TIME:
return (query_start, query_end)
else:
return (query_start, None)
def create(cls, ident, hostname, pid, type, total, busy, idle):
executor = cls.query.filter_by(ident=ident, type=type).first()
if executor is None:
executor = cls(ident=ident,
hostname=hostname,
pid=pid,
type=type,
total=total,
busy=busy,
idle=idle)
executor.total = total
executor.busy = busy
executor.pid = pid
executor.heartbeat_time = timezone.now()
db.session.add(executor)
db.session.commit()
return executor
