def filter_asset(self, queryset): is_query_all = self.get_query_param('all', True) asset_id = self.get_query_param('asset_id') hostname = self.get_query_param('hostname') ip = self.get_query_param('ip') if asset_id: assets = Asset.objects.filter(pk=asset_id) elif hostname: assets = Asset.objects.filter(hostname=hostname) elif ip: assets = Asset.objects.filter(ip=ip) else: return queryset if not assets: return queryset.none() asset = assets.get() if not is_query_all: queryset = queryset.filter(assets=asset) return queryset inherit_all_nodekeys = set() inherit_nodekeys = asset.nodes.values_list('key', flat=True) for key in inherit_nodekeys: ancestor_keys = Node.get_node_ancestor_keys(key, with_self=True) inherit_all_nodekeys.update(ancestor_keys) inherit_all_nodeids = Node.objects.filter( key__in=inherit_all_nodekeys).values_list('id', flat=True) inherit_all_nodeids = list(inherit_all_nodeids) qs1 = queryset.filter(assets=asset).distinct() qs2 = queryset.filter(nodes__id__in=inherit_all_nodeids).distinct() qs = UnionQuerySet(qs1, qs2) return qs
def get_user_nodes_with_assets(cls, user): """ :param user: :return: {node: {asset: set(su1, su2)}} """ from assets.models import Node unnode = Node(value='Unnode') nodes = defaultdict(dict) for _node in cls.get_user_nodes(user): children = _node.get_family() for node in children: nodes[node] = defaultdict(set) nodes[unnode] = defaultdict(set) _assets = cls.get_user_assets(user) for asset, _system_users in _assets.items(): _nodes = asset.get_nodes() in_node = False for node in _nodes: if node in nodes: in_node = True nodes[node][asset].update(_system_users) if not in_node: nodes[unnode][asset].update(_system_users) return nodes
def keep_subscribe_node_assets_relation(): while True: try: subscribe = node_assets_mapping_for_memory_pub_sub.subscribe() msgs = subscribe.listen() # 开始之前关闭连接,因为server端可能关闭了连接,而 client 还在 CONN_MAX_AGE 中 close_old_connections() for message in msgs: if message["type"] != "message": continue org_id = message['data'].decode() root_org_id = Organization.ROOT_ID Node.expire_node_all_asset_ids_mapping_from_memory(org_id) Node.expire_node_all_asset_ids_mapping_from_memory( root_org_id) logger.debug( "Expire node assets id mapping from memory of org={}, pid={}" "".format(str(org_id), os.getpid())) except Exception as e: logger.exception(f'subscribe_node_assets_mapping_expire: {e}') Node.expire_all_orgs_node_all_asset_ids_mapping_from_memory() finally: # 请求结束,关闭连接 close_old_connections()
def compute_assets_amount(self): if self.org.is_root(): return Asset.objects.all().count() node = Node.org_root() return node.assets_amount
def get_nodes(self): from assets.models import Node nodes = self.nodes.all() or [Node.root()] return nodes
def get_assets(self): nodes_keys, assets_ids = self.get_permissions_nodes_and_assets() queryset = Node.get_nodes_all_assets(nodes_keys, extra_assets_ids=assets_ids) return queryset.valid()
def get_data_on_node_direct_granted(self, key): # 如果这个节点是直接授权的(或者说祖先节点直接授权的), 获取下面的所有资产 return Node.get_node_all_assets_by_key_v2(key)
def handle_node_relation_change(org_id): root_org_id = Organization.ROOT_ID Node.expire_node_all_asset_ids_mapping_from_memory(org_id) Node.expire_node_all_asset_ids_mapping_from_memory(root_org_id)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.tree = Node.tree()
def get_ungrouped_node(user, asset_perms_id=None): assets_amount = count_user_direct_granted_assets(user, asset_perms_id) return Node(id=UNGROUPED_NODE_KEY, key=UNGROUPED_NODE_KEY, value=UNGROUPED_NODE_VALUE, assets_amount=assets_amount)
def on_node_pre_save(sender, instance: Node, **kwargs): instance.parent_key = instance.compute_parent_key()
def form_valid(self, form): sync_time = datetime.datetime.now() account_id = self.request.GET.get("account_id") # Will create ROOT node while user first time importing assets. if not get_object_or_none(Node, key="1"): Node.root() try: account_id = account_id if account_id else form.cleaned_data['account_id'] account_id = ''.join(account_id.split("-")) hostname_map = json.loads(form.cleaned_data["hostname_map"]) if not hostname_map: data = {'valid': False, 'msg': _('Did not selected any assets')} return self.render_to_response(data) except Exception as e: data = {'valid': False, 'msg': _("some error,reason {error}".format(**{"error":str(e)}))} return self.render_to_response(data) ak = get_object_or_none(CloudAccount, id=account_id) if ak is None: pass ak.last_sync_time = sync_time ak.save() account_name = ak.name cloud_name = ak.cloud_provider cloud_node = create_cloud_node(cloud_name, account_id,account_name) created, updated, failed = [], [], [] instances = cache.get(account_id, []) if instances: for instance in instances: for res_ins in hostname_map: if res_ins["number"] == instance["InstanceId"]: projectID = get_projectId(cloud_name, instance) project_node_key = create_project_node(cloud_node, projectID) asset_dict = genernate_asset_dic(cloud_name, instance) asset = None asset_id = asset_dict.pop('id', None) if asset_id: asset = get_object_or_none(Asset, id=asset_id) if not asset: try: if len(Asset.objects.filter(number=asset_dict.get('number'))): raise Exception(_('already exists')) with transaction.atomic(): asset = Asset.objects.create(**asset_dict) assign_asset_to_node(project_node_key, asset_dict["hostname"]) created.append(asset_dict['hostname']) except Exception as e: failed.append('%s: %s' % (asset_dict['hostname'], str(e))) else: for k, v in asset_dict.items(): if v != '': setattr(asset, k, v) try: asset.save() updated.append(asset_dict['hostname']) except Exception as e: failed.append('%s: %s' % (asset_dict['hostname'], str(e))) else: continue data = { 'created': created, 'created_info': 'Created {}'.format(len(created)), 'updated': updated, 'updated_info': 'Updated {}'.format(len(updated)), 'failed': failed, 'failed_info': 'Failed {}'.format(len(failed)), 'valid': True, 'msg': 'Created: {}. Updated: {}, Error: {}'.format(len(created), len(updated), len(failed)) } return self.render_json_response(data) else: data = {'valid': False, 'msg': _('No instance for this account')} return self.render_json_response(data)
def compute_assets_amount(self): node = Node.org_root() return node.assets_amount
def sync_dc2_list_info_manual(): logger.info('ready to sync didi cloud dc2 list') didi_util = DiDiCloudUtil() created, updated, failed = [], [], [] j_created, j_updated, j_failed = [], [], [] node = Node.root() Dc2.objects.all().update(status='Destory') for info in didi_util.get_dc2_instances(): logger.info(json.dumps(info)) dc2 = get_object_or_none(Dc2, instance_id=info.get('instance_id')) if not dc2: try: with transaction.atomic(): dc2 = Dc2.objects.create(**info) node = auto_allocate_asset_node(info.get('instance_name'), 'dc2') # need to add auto join node dc2.nodes.set([node]) created.append(info['instance_name']) except Exception as e: failed.append('%s: %s' % (info['instance_name'], str(e))) else: for k, v in info.items(): if v != '': setattr(dc2, k, v) try: if not dc2.nodes.exclude(key='1').first(): logger.info('update node for root node dc2') node = auto_allocate_asset_node(info.get('instance_name'), 'dc2') dc2.nodes.set([node]) dc2.save() updated.append(info['instance_name']) except Exception as e: failed.append('%s: %s' % (info['instance_name'], str(e))) if settings.AUTO_UPDATE_JUMPSERVER_ASSETS: asset = get_object_or_none(JAssets, number=info.get('instance_id')) if not asset: try: with transaction.atomic(): hostname = info.get('instance_name') admin_user, domain = auto_get_admin_and_domain(hostname) attr = { 'number': info.get('instance_id'), 'ip': info.get('inner_ip'), 'port': 3299, 'protocol': 'ssh', 'protocols': 'ssh/3299', 'hostname': hostname, 'platform': 'Linux', 'domain': domain, 'admin_user': admin_user } asset = JAssets.objects.create(**attr) # need to add auto join node asset.nodes.set([node]) j_created.append(info['instance_name']) except Exception as e: j_failed.append('%s: %s' % (info['instance_name'], str(e))) else: setattr(asset, 'hostname', info.get('instance_name')) setattr(asset, 'ip', info.get('inner_ip')) try: asset.save() j_updated.append(info['instance_name']) except Exception as e: j_failed.append('%s: %s' % (info['instance_name'], str(e))) data = { 'created': created, 'created_info': 'Created {}'.format(len(created)), 'updated': updated, 'updated_info': 'Updated {}'.format(len(updated)), 'failed': failed, 'failed_info': 'Failed {}'.format(len(failed)), 'valid': True, 'msg': 'Created: {}. Updated: {}, Error: {}'.format( len(created), len(updated), len(failed)) } logger.info('dc2 sync finish') logger.info(json.dumps(data)) if settings.AUTO_UPDATE_JUMPSERVER_ASSETS: clean_destory_assets_in_jumpserver() j_data = { 'created': j_created, 'created_info': 'Created {}'.format(len(j_created)), 'updated': j_updated, 'updated_info': 'Updated {}'.format(len(j_updated)), 'failed': j_failed, 'failed_info': 'Failed {}'.format(len(j_failed)), 'valid': True, 'msg': 'Created: {}. Updated: {}, Error: {}'.format( len(j_created), len(j_updated), len(j_failed)) } logger.info('jump server asset update finish') logger.info(json.dumps(j_data)) return data
def get_ungrouped_node(user): assets_amount = count_user_direct_granted_assets(user) return Node(id=UNGROUPED_NODE_KEY, key=UNGROUPED_NODE_KEY, value=_(UNGROUPED_NODE_KEY), assets_amount=assets_amount)
def sync_ecs_list_info_manual(): logger.info('ready to sync aly cloud ecs list') ali_util = AliCloudUtil() result = ali_util.get_ecs_instances() created, updated, failed = [], [], [] j_created, j_updated, j_failed = [], [], [] node = Node.root() Ecs.objects.all().update(status='Destory') for info in result: logger.info(json.dumps(info)) ecs = get_object_or_none(Ecs, instance_id=info.get('instance_id')) if not ecs: try: with transaction.atomic(): ecs = Ecs.objects.create(**info) node = auto_allocate_asset_node(info.get('instance_name'), 'ecs') # need to add auto join node ecs.nodes.set([node]) created.append(info['instance_name']) except Exception as e: failed.append('%s: %s' % (info['instance_name'], str(e))) else: for k, v in info.items(): if v != '': setattr(ecs, k, v) try: if not ecs.nodes.exclude(key='1').first(): logger.info('update node for root node ecs') node = auto_allocate_asset_node(info.get('instance_name'), 'ecs') ecs.nodes.set([node]) ecs.save() updated.append(info['instance_name']) except Exception as e: failed.append('%s: %s' % (info['instance_name'], str(e))) if settings.AUTO_UPDATE_JUMPSERVER_ASSETS: asset = get_object_or_none(Asset, number=info.get('instance_id')) if not asset: try: with transaction.atomic(): hostname = info.get('instance_name') domain = None admin_user = None if settings.ENVIROMENT == 'PROD': if 'ebs' in hostname: domain = 'ebs' admin_user = '******' elif 'vip6' in hostname: domain = 'vip6' admin_user = '******' elif 'hsb' in hostname: domain = 'hsb' admin_user = '******' elif 'vip5' in hostname: domain = 'vip5' admin_user = '******' elif 'usa' in hostname: domain = 'osu' admin_user = '******' else: domain = None admin_user = None attr = { 'number': 'instance_id', 'ip': info.get('inner_ip'), 'port': 3299, 'hostname': hostname, 'platform': 'Linux', 'domain': domain, 'admin_user': admin_user } asset = Asset.objects.create(**attr) # need to add auto join node asset.nodes.set([node]) j_created.append(info['instance_name']) except Exception as e: j_failed.append('%s: %s' % (info['instance_name'], str(e))) else: setattr(ecs, 'hostname', info.get('hostname')) try: asset.save() j_updated.append(info['instance_name']) except Exception as e: j_failed.append('%s: %s' % (info['instance_name'], str(e))) data = { 'created': created, 'created_info': 'Created {}'.format(len(created)), 'updated': updated, 'updated_info': 'Updated {}'.format(len(updated)), 'failed': failed, 'failed_info': 'Failed {}'.format(len(failed)), 'valid': True, 'msg': 'Created: {}. Updated: {}, Error: {}'.format( len(created), len(updated), len(failed)) } logger.info('ecs sync finish') logger.info(json.dumps(data)) j_data = { 'created': j_created, 'created_info': 'Created {}'.format(len(j_created)), 'updated': j_updated, 'updated_info': 'Updated {}'.format(len(j_updated)), 'failed': j_failed, 'failed_info': 'Failed {}'.format(len(j_failed)), 'valid': True, 'msg': 'Created: {}. Updated: {}, Error: {}'.format( len(j_created), len(j_updated), len(j_failed)) } logger.info('jump server asset update finish') logger.info(json.dumps(j_data)) return data