def filter_asset(self, queryset):
is_query_all = self.get_query_param('all', True)
asset_id = self.get_query_param('asset_id')
hostname = self.get_query_param('hostname')
ip = self.get_query_param('ip')
if asset_id:
assets = Asset.objects.filter(pk=asset_id)
elif hostname:
assets = Asset.objects.filter(hostname=hostname)
elif ip:
assets = Asset.objects.filter(ip=ip)
else:
return queryset
if not assets:
return queryset.none()
asset = assets.get()
if not is_query_all:
queryset = queryset.filter(assets=asset)
return queryset
inherit_all_nodekeys = set()
inherit_nodekeys = asset.nodes.values_list('key', flat=True)
for key in inherit_nodekeys:
ancestor_keys = Node.get_node_ancestor_keys(key, with_self=True)
inherit_all_nodekeys.update(ancestor_keys)
inherit_all_nodeids = Node.objects.filter(
key__in=inherit_all_nodekeys).values_list('id', flat=True)
inherit_all_nodeids = list(inherit_all_nodeids)
qs1 = queryset.filter(assets=asset).distinct()
qs2 = queryset.filter(nodes__id__in=inherit_all_nodeids).distinct()
qs = UnionQuerySet(qs1, qs2)
return qs
def get_user_nodes_with_assets(cls, user):
"""
:param user:
:return: {node: {asset: set(su1, su2)}}
"""
from assets.models import Node
unnode = Node(value='Unnode')
nodes = defaultdict(dict)
for _node in cls.get_user_nodes(user):
children = _node.get_family()
for node in children:
nodes[node] = defaultdict(set)
nodes[unnode] = defaultdict(set)
_assets = cls.get_user_assets(user)
for asset, _system_users in _assets.items():
_nodes = asset.get_nodes()
in_node = False
for node in _nodes:
if node in nodes:
in_node = True
nodes[node][asset].update(_system_users)
if not in_node:
nodes[unnode][asset].update(_system_users)
return nodes
def keep_subscribe_node_assets_relation():
while True:
try:
subscribe = node_assets_mapping_for_memory_pub_sub.subscribe()
msgs = subscribe.listen()
# 开始之前关闭连接，因为server端可能关闭了连接，而 client 还在 CONN_MAX_AGE 中
close_old_connections()
for message in msgs:
if message["type"] != "message":
continue
org_id = message['data'].decode()
root_org_id = Organization.ROOT_ID
Node.expire_node_all_asset_ids_mapping_from_memory(org_id)
Node.expire_node_all_asset_ids_mapping_from_memory(
root_org_id)
logger.debug(
"Expire node assets id mapping from memory of org={}, pid={}"
"".format(str(org_id), os.getpid()))
except Exception as e:
logger.exception(f'subscribe_node_assets_mapping_expire: {e}')
Node.expire_all_orgs_node_all_asset_ids_mapping_from_memory()
finally:
# 请求结束，关闭连接
close_old_connections()
def compute_assets_amount(self):
if self.org.is_root():
return Asset.objects.all().count()
node = Node.org_root()
return node.assets_amount
def get_nodes(self):
from assets.models import Node
nodes = self.nodes.all() or [Node.root()]
return nodes
def get_assets(self):
nodes_keys, assets_ids = self.get_permissions_nodes_and_assets()
queryset = Node.get_nodes_all_assets(nodes_keys,
extra_assets_ids=assets_ids)
return queryset.valid()
def get_data_on_node_direct_granted(self, key):
# 如果这个节点是直接授权的(或者说祖先节点直接授权的), 获取下面的所有资产
return Node.get_node_all_assets_by_key_v2(key)
def handle_node_relation_change(org_id):
root_org_id = Organization.ROOT_ID
Node.expire_node_all_asset_ids_mapping_from_memory(org_id)
Node.expire_node_all_asset_ids_mapping_from_memory(root_org_id)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.tree = Node.tree()
def get_ungrouped_node(user, asset_perms_id=None):
assets_amount = count_user_direct_granted_assets(user, asset_perms_id)
return Node(id=UNGROUPED_NODE_KEY,
key=UNGROUPED_NODE_KEY,
value=UNGROUPED_NODE_VALUE,
assets_amount=assets_amount)
def on_node_pre_save(sender, instance: Node, **kwargs):
instance.parent_key = instance.compute_parent_key()
def form_valid(self, form):
sync_time = datetime.datetime.now()
account_id = self.request.GET.get("account_id")
# Will create ROOT node while user first time importing assets.
if not get_object_or_none(Node, key="1"):
Node.root()
try:
account_id = account_id if account_id else form.cleaned_data['account_id']
account_id = ''.join(account_id.split("-"))
hostname_map = json.loads(form.cleaned_data["hostname_map"])
if not hostname_map:
data = {'valid': False, 'msg': _('Did not selected any assets')}
return self.render_to_response(data)
except Exception as e:
data = {'valid': False,
'msg': _("some error,reason {error}".format(**{"error":str(e)}))}
return self.render_to_response(data)
ak = get_object_or_none(CloudAccount, id=account_id)
if ak is None:
pass
ak.last_sync_time = sync_time
ak.save()
account_name = ak.name
cloud_name = ak.cloud_provider
cloud_node = create_cloud_node(cloud_name, account_id,account_name)
created, updated, failed = [], [], []
instances = cache.get(account_id, [])
if instances:
for instance in instances:
for res_ins in hostname_map:
if res_ins["number"] == instance["InstanceId"]:
projectID = get_projectId(cloud_name, instance)
project_node_key = create_project_node(cloud_node, projectID)
asset_dict = genernate_asset_dic(cloud_name, instance)
asset = None
asset_id = asset_dict.pop('id', None)
if asset_id:
asset = get_object_or_none(Asset, id=asset_id)
if not asset:
try:
if len(Asset.objects.filter(number=asset_dict.get('number'))):
raise Exception(_('already exists'))
with transaction.atomic():
asset = Asset.objects.create(**asset_dict)
assign_asset_to_node(project_node_key, asset_dict["hostname"])
created.append(asset_dict['hostname'])
except Exception as e:
failed.append('%s: %s' % (asset_dict['hostname'], str(e)))
else:
for k, v in asset_dict.items():
if v != '':
setattr(asset, k, v)
try:
asset.save()
updated.append(asset_dict['hostname'])
except Exception as e:
failed.append('%s: %s' % (asset_dict['hostname'], str(e)))
else:
continue
data = {
'created': created,
'created_info': 'Created {}'.format(len(created)),
'updated': updated,
'updated_info': 'Updated {}'.format(len(updated)),
'failed': failed,
'failed_info': 'Failed {}'.format(len(failed)),
'valid': True,
'msg': 'Created: {}. Updated: {}, Error: {}'.format(len(created), len(updated), len(failed))
}
return self.render_json_response(data)
else:
data = {'valid': False,
'msg': _('No instance for this account')}
return self.render_json_response(data)
def compute_assets_amount(self):
node = Node.org_root()
return node.assets_amount
def sync_dc2_list_info_manual():
logger.info('ready to sync didi cloud dc2 list')
didi_util = DiDiCloudUtil()
created, updated, failed = [], [], []
j_created, j_updated, j_failed = [], [], []
node = Node.root()
Dc2.objects.all().update(status='Destory')
for info in didi_util.get_dc2_instances():
logger.info(json.dumps(info))
dc2 = get_object_or_none(Dc2, instance_id=info.get('instance_id'))
if not dc2:
try:
with transaction.atomic():
dc2 = Dc2.objects.create(**info)
node = auto_allocate_asset_node(info.get('instance_name'), 'dc2')
# need to add auto join node
dc2.nodes.set([node])
created.append(info['instance_name'])
except Exception as e:
failed.append('%s: %s' % (info['instance_name'], str(e)))
else:
for k, v in info.items():
if v != '':
setattr(dc2, k, v)
try:
if not dc2.nodes.exclude(key='1').first():
logger.info('update node for root node dc2')
node = auto_allocate_asset_node(info.get('instance_name'), 'dc2')
dc2.nodes.set([node])
dc2.save()
updated.append(info['instance_name'])
except Exception as e:
failed.append('%s: %s' % (info['instance_name'], str(e)))
if settings.AUTO_UPDATE_JUMPSERVER_ASSETS:
asset = get_object_or_none(JAssets, number=info.get('instance_id'))
if not asset:
try:
with transaction.atomic():
hostname = info.get('instance_name')
admin_user, domain = auto_get_admin_and_domain(hostname)
attr = {
'number': info.get('instance_id'),
'ip': info.get('inner_ip'),
'port': 3299,
'protocol': 'ssh',
'protocols': 'ssh/3299',
'hostname': hostname,
'platform': 'Linux',
'domain': domain,
'admin_user': admin_user
}
asset = JAssets.objects.create(**attr)
# need to add auto join node
asset.nodes.set([node])
j_created.append(info['instance_name'])
except Exception as e:
j_failed.append('%s: %s' % (info['instance_name'], str(e)))
else:
setattr(asset, 'hostname', info.get('instance_name'))
setattr(asset, 'ip', info.get('inner_ip'))
try:
asset.save()
j_updated.append(info['instance_name'])
except Exception as e:
j_failed.append('%s: %s' % (info['instance_name'], str(e)))
data = {
'created': created,
'created_info': 'Created {}'.format(len(created)),
'updated': updated,
'updated_info': 'Updated {}'.format(len(updated)),
'failed': failed,
'failed_info': 'Failed {}'.format(len(failed)),
'valid': True,
'msg': 'Created: {}. Updated: {}, Error: {}'.format(
len(created), len(updated), len(failed))
}
logger.info('dc2 sync finish')
logger.info(json.dumps(data))
if settings.AUTO_UPDATE_JUMPSERVER_ASSETS:
clean_destory_assets_in_jumpserver()
j_data = {
'created': j_created,
'created_info': 'Created {}'.format(len(j_created)),
'updated': j_updated,
'updated_info': 'Updated {}'.format(len(j_updated)),
'failed': j_failed,
'failed_info': 'Failed {}'.format(len(j_failed)),
'valid': True,
'msg': 'Created: {}. Updated: {}, Error: {}'.format(
len(j_created), len(j_updated), len(j_failed))
}
logger.info('jump server asset update finish')
logger.info(json.dumps(j_data))
return data
def get_ungrouped_node(user):
assets_amount = count_user_direct_granted_assets(user)
return Node(id=UNGROUPED_NODE_KEY,
key=UNGROUPED_NODE_KEY,
value=_(UNGROUPED_NODE_KEY),
assets_amount=assets_amount)
def sync_ecs_list_info_manual():
logger.info('ready to sync aly cloud ecs list')
ali_util = AliCloudUtil()
result = ali_util.get_ecs_instances()
created, updated, failed = [], [], []
j_created, j_updated, j_failed = [], [], []
node = Node.root()
Ecs.objects.all().update(status='Destory')
for info in result:
logger.info(json.dumps(info))
ecs = get_object_or_none(Ecs, instance_id=info.get('instance_id'))
if not ecs:
try:
with transaction.atomic():
ecs = Ecs.objects.create(**info)
node = auto_allocate_asset_node(info.get('instance_name'), 'ecs')
# need to add auto join node
ecs.nodes.set([node])
created.append(info['instance_name'])
except Exception as e:
failed.append('%s: %s' % (info['instance_name'], str(e)))
else:
for k, v in info.items():
if v != '':
setattr(ecs, k, v)
try:
if not ecs.nodes.exclude(key='1').first():
logger.info('update node for root node ecs')
node = auto_allocate_asset_node(info.get('instance_name'), 'ecs')
ecs.nodes.set([node])
ecs.save()
updated.append(info['instance_name'])
except Exception as e:
failed.append('%s: %s' % (info['instance_name'], str(e)))
if settings.AUTO_UPDATE_JUMPSERVER_ASSETS:
asset = get_object_or_none(Asset, number=info.get('instance_id'))
if not asset:
try:
with transaction.atomic():
hostname = info.get('instance_name')
domain = None
admin_user = None
if settings.ENVIROMENT == 'PROD':
if 'ebs' in hostname:
domain = 'ebs'
admin_user = '******'
elif 'vip6' in hostname:
domain = 'vip6'
admin_user = '******'
elif 'hsb' in hostname:
domain = 'hsb'
admin_user = '******'
elif 'vip5' in hostname:
domain = 'vip5'
admin_user = '******'
elif 'usa' in hostname:
domain = 'osu'
admin_user = '******'
else:
domain = None
admin_user = None
attr = {
'number': 'instance_id',
'ip': info.get('inner_ip'),
'port': 3299,
'hostname': hostname,
'platform': 'Linux',
'domain': domain,
'admin_user': admin_user
}
asset = Asset.objects.create(**attr)
# need to add auto join node
asset.nodes.set([node])
j_created.append(info['instance_name'])
except Exception as e:
j_failed.append('%s: %s' % (info['instance_name'], str(e)))
else:
setattr(ecs, 'hostname', info.get('hostname'))
try:
asset.save()
j_updated.append(info['instance_name'])
except Exception as e:
j_failed.append('%s: %s' % (info['instance_name'], str(e)))
data = {
'created': created,
'created_info': 'Created {}'.format(len(created)),
'updated': updated,
'updated_info': 'Updated {}'.format(len(updated)),
'failed': failed,
'failed_info': 'Failed {}'.format(len(failed)),
'valid': True,
'msg': 'Created: {}. Updated: {}, Error: {}'.format(
len(created), len(updated), len(failed))
}
logger.info('ecs sync finish')
logger.info(json.dumps(data))
j_data = {
'created': j_created,
'created_info': 'Created {}'.format(len(j_created)),
'updated': j_updated,
'updated_info': 'Updated {}'.format(len(j_updated)),
'failed': j_failed,
'failed_info': 'Failed {}'.format(len(j_failed)),
'valid': True,
'msg': 'Created: {}. Updated: {}, Error: {}'.format(
len(j_created), len(j_updated), len(j_failed))
}
logger.info('jump server asset update finish')
logger.info(json.dumps(j_data))
return data
