def auto_allocate_asset_node(name, asset_type):
logger.info('auto allocate {} {}'.format(name, asset_type))
num_list = re.findall(r"\d+", name)
if len(num_list):
name = name[:name.index(num_list[-1])]
logger.info('auto allocate find name {}'.format(name))
asset = None
if asset_type == 'ecs':
asset = Ecs.objects.filter(instance_name__contains=name).exclude(
Q(nodes=None) | Q(nodes__key='1')).first()
elif asset_type == 'slb':
asset = Slb.objects.filter(instance_name__contains=name).exclude(
Q(nodes=None) | Q(nodes__key='1')).first()
elif asset_type == 'kvstore':
asset = KvStore.objects.filter(instance_name__contains=name).exclude(
Q(nodes=None) | Q(nodes__key='1')).first()
elif asset_type == 'oss':
asset = Oss.objects.filter(instance_name__contains=name).exclude(
Q(nodes=None) | Q(nodes__key='1')).first()
elif asset_type == 'rds':
asset = Rds.objects.filter(instance_name__contains=name).exclude(
Q(nodes=None) | Q(nodes__key='1')).first()
else:
pass
if asset:
node = asset.nodes.exclude(key='1').first()
if node:
logger.info('auto allocate {} to {}'.format(name, node.value))
return node
else:
logger.info('auto allocate {} to root node'.format(name))
return Node.root()
else:
logger.info('auto allocate {} to root node'.format(name))
return Node.root()
def get_context_data(self, **kwargs):
Node.root()
context = {
'app': _('Assets'),
'action': _('Asset list'),
'labels': Label.objects.all().order_by('name'),
'nodes': Node.objects.all().order_by('-key'),
}
kwargs.update(context)
return super().get_context_data(**kwargs)
def filter_node(self, queryset):
unallocated = self.request.query_params.get("unallocated")
if unallocated:
queryset = queryset.annotate(num_nodes=Count('nodes'))
queryset = queryset.filter((Q(num_nodes=0)) | (Q(nodes=Node.root())
& Q(num_nodes=1)))
return queryset
node_id = self.request.query_params.get("node_id")
if not node_id:
return queryset
node = get_object_or_404(Node, id=node_id)
show_current_asset = self.request.query_params.get(
"show_current_asset") in ('1', 'true')
if node.is_root() and show_current_asset:
queryset = queryset.filter(
Q(nodes=node_id) | Q(nodes__isnull=True))
elif node.is_root() and not show_current_asset:
pass
elif not node.is_root() and show_current_asset:
queryset = queryset.filter(nodes=node)
else:
queryset = queryset.filter(
nodes__key__regex='^{}(:[0-9]+)*$'.format(node.key), )
return queryset
def default_node():
try:
from assets.models import Node
root = Node.root()
return root
except:
return None
def set_assets_node(self, assets):
if not isinstance(assets, list):
assets = [assets]
node = Node.root()
node_id = self.request.query_params.get('node_id')
if node_id:
node = get_object_or_none(Node, pk=node_id)
node.assets.add(*assets)
def perform_update(self, serializer):
assets = serializer.validated_data.get('assets')
instance = self.get_object()
if instance != Node.root():
instance.kvstore.remove(*tuple(assets))
else:
assets = [asset for asset in assets if asset.nodes.count() > 1]
instance.kvstore.remove(*tuple(assets))
def get_form(self, form_class=None):
form = super().get_form(form_class=form_class)
node_id = self.request.GET.get("node_id")
if node_id:
node = get_object_or_none(Node, id=node_id)
else:
node = Node.root()
form['nodes'].initial = node
return form
def sync_oss_list_info_manual():
logger.info('ready to sync aly cloud oss list')
ali_util = AliCloudUtil()
created, updated, failed = [], [], []
node = Node.root()
Oss.objects.all().update(status='Destory')
for info in ali_util.get_oss_instances():
logger.info(json.dumps(info))
oss = get_object_or_none(Oss, instance_id=info.get('instance_id'))
if not oss:
try:
with transaction.atomic():
oss = Oss.objects.create(**info)
# need to add auto join node
node = auto_allocate_asset_node(info.get('instance_name'),
'oss')
oss.nodes.set([node])
created.append(info['instance_name'])
except Exception as e:
failed.append('%s: %s' % (info['instance_name'], str(e)))
else:
for k, v in info.items():
if v != '':
setattr(oss, k, v)
try:
if not oss.nodes.exclude(key='1').first():
logger.info('update node for root node oss')
node = auto_allocate_asset_node(info.get('instance_name'),
'oss')
oss.nodes.set([node])
oss.save()
updated.append(info['instance_name'])
except Exception as e:
failed.append('%s: %s' % (info['instance_name'], str(e)))
data = {
'created':
created,
'created_info':
'Created {}'.format(len(created)),
'updated':
updated,
'updated_info':
'Updated {}'.format(len(updated)),
'failed':
failed,
'failed_info':
'Failed {}'.format(len(failed)),
'valid':
True,
'msg':
'Created: {}. Updated: {}, Error: {}'.format(len(created),
len(updated), len(failed))
}
logger.info('sync finish')
logger.info(json.dumps(data))
return data
def get_queryset(self):
node_key = self.request.query_params.get('key')
if node_key:
self.node = Node.objects.get(key=node_key)
queryset = self.node.get_children(with_self=False)
else:
self.is_root = True
self.node = Node.root()
queryset = list(self.node.get_children(with_self=True))
nodes_invalid = Node.objects.exclude(key__startswith=self.node.key)
queryset.extend(list(nodes_invalid))
queryset = [self.as_tree_node(node) for node in queryset]
queryset = sorted(queryset)
return queryset
def get_nodes(self):
from assets.models import Node
nodes = self.nodes.all() or [Node.root()]
return nodes
def form_valid(self, form):
sync_time = datetime.datetime.now()
account_id = self.request.GET.get("account_id")
# Will create ROOT node while user first time importing assets.
if not get_object_or_none(Node, key="1"):
Node.root()
try:
account_id = account_id if account_id else form.cleaned_data['account_id']
account_id = ''.join(account_id.split("-"))
hostname_map = json.loads(form.cleaned_data["hostname_map"])
if not hostname_map:
data = {'valid': False, 'msg': _('Did not selected any assets')}
return self.render_to_response(data)
except Exception as e:
data = {'valid': False,
'msg': _("some error,reason {error}".format(**{"error":str(e)}))}
return self.render_to_response(data)
ak = get_object_or_none(CloudAccount, id=account_id)
if ak is None:
pass
ak.last_sync_time = sync_time
ak.save()
account_name = ak.name
cloud_name = ak.cloud_provider
cloud_node = create_cloud_node(cloud_name, account_id,account_name)
created, updated, failed = [], [], []
instances = cache.get(account_id, [])
if instances:
for instance in instances:
for res_ins in hostname_map:
if res_ins["number"] == instance["InstanceId"]:
projectID = get_projectId(cloud_name, instance)
project_node_key = create_project_node(cloud_node, projectID)
asset_dict = genernate_asset_dic(cloud_name, instance)
asset = None
asset_id = asset_dict.pop('id', None)
if asset_id:
asset = get_object_or_none(Asset, id=asset_id)
if not asset:
try:
if len(Asset.objects.filter(number=asset_dict.get('number'))):
raise Exception(_('already exists'))
with transaction.atomic():
asset = Asset.objects.create(**asset_dict)
assign_asset_to_node(project_node_key, asset_dict["hostname"])
created.append(asset_dict['hostname'])
except Exception as e:
failed.append('%s: %s' % (asset_dict['hostname'], str(e)))
else:
for k, v in asset_dict.items():
if v != '':
setattr(asset, k, v)
try:
asset.save()
updated.append(asset_dict['hostname'])
except Exception as e:
failed.append('%s: %s' % (asset_dict['hostname'], str(e)))
else:
continue
data = {
'created': created,
'created_info': 'Created {}'.format(len(created)),
'updated': updated,
'updated_info': 'Updated {}'.format(len(updated)),
'failed': failed,
'failed_info': 'Failed {}'.format(len(failed)),
'valid': True,
'msg': 'Created: {}. Updated: {}, Error: {}'.format(len(created), len(updated), len(failed))
}
return self.render_json_response(data)
else:
data = {'valid': False,
'msg': _('No instance for this account')}
return self.render_json_response(data)
def sync_dc2_list_info_manual():
logger.info('ready to sync didi cloud dc2 list')
didi_util = DiDiCloudUtil()
created, updated, failed = [], [], []
j_created, j_updated, j_failed = [], [], []
node = Node.root()
Dc2.objects.all().update(status='Destory')
for info in didi_util.get_dc2_instances():
logger.info(json.dumps(info))
dc2 = get_object_or_none(Dc2, instance_id=info.get('instance_id'))
if not dc2:
try:
with transaction.atomic():
dc2 = Dc2.objects.create(**info)
node = auto_allocate_asset_node(info.get('instance_name'), 'dc2')
# need to add auto join node
dc2.nodes.set([node])
created.append(info['instance_name'])
except Exception as e:
failed.append('%s: %s' % (info['instance_name'], str(e)))
else:
for k, v in info.items():
if v != '':
setattr(dc2, k, v)
try:
if not dc2.nodes.exclude(key='1').first():
logger.info('update node for root node dc2')
node = auto_allocate_asset_node(info.get('instance_name'), 'dc2')
dc2.nodes.set([node])
dc2.save()
updated.append(info['instance_name'])
except Exception as e:
failed.append('%s: %s' % (info['instance_name'], str(e)))
if settings.AUTO_UPDATE_JUMPSERVER_ASSETS:
asset = get_object_or_none(JAssets, number=info.get('instance_id'))
if not asset:
try:
with transaction.atomic():
hostname = info.get('instance_name')
admin_user, domain = auto_get_admin_and_domain(hostname)
attr = {
'number': info.get('instance_id'),
'ip': info.get('inner_ip'),
'port': 3299,
'protocol': 'ssh',
'protocols': 'ssh/3299',
'hostname': hostname,
'platform': 'Linux',
'domain': domain,
'admin_user': admin_user
}
asset = JAssets.objects.create(**attr)
# need to add auto join node
asset.nodes.set([node])
j_created.append(info['instance_name'])
except Exception as e:
j_failed.append('%s: %s' % (info['instance_name'], str(e)))
else:
setattr(asset, 'hostname', info.get('instance_name'))
setattr(asset, 'ip', info.get('inner_ip'))
try:
asset.save()
j_updated.append(info['instance_name'])
except Exception as e:
j_failed.append('%s: %s' % (info['instance_name'], str(e)))
data = {
'created': created,
'created_info': 'Created {}'.format(len(created)),
'updated': updated,
'updated_info': 'Updated {}'.format(len(updated)),
'failed': failed,
'failed_info': 'Failed {}'.format(len(failed)),
'valid': True,
'msg': 'Created: {}. Updated: {}, Error: {}'.format(
len(created), len(updated), len(failed))
}
logger.info('dc2 sync finish')
logger.info(json.dumps(data))
if settings.AUTO_UPDATE_JUMPSERVER_ASSETS:
clean_destory_assets_in_jumpserver()
j_data = {
'created': j_created,
'created_info': 'Created {}'.format(len(j_created)),
'updated': j_updated,
'updated_info': 'Updated {}'.format(len(j_updated)),
'failed': j_failed,
'failed_info': 'Failed {}'.format(len(j_failed)),
'valid': True,
'msg': 'Created: {}. Updated: {}, Error: {}'.format(
len(j_created), len(j_updated), len(j_failed))
}
logger.info('jump server asset update finish')
logger.info(json.dumps(j_data))
return data
def sync_ecs_list_info_manual():
logger.info('ready to sync aly cloud ecs list')
ali_util = AliCloudUtil()
result = ali_util.get_ecs_instances()
created, updated, failed = [], [], []
j_created, j_updated, j_failed = [], [], []
node = Node.root()
Ecs.objects.all().update(status='Destory')
for info in result:
logger.info(json.dumps(info))
ecs = get_object_or_none(Ecs, instance_id=info.get('instance_id'))
if not ecs:
try:
with transaction.atomic():
ecs = Ecs.objects.create(**info)
node = auto_allocate_asset_node(info.get('instance_name'), 'ecs')
# need to add auto join node
ecs.nodes.set([node])
created.append(info['instance_name'])
except Exception as e:
failed.append('%s: %s' % (info['instance_name'], str(e)))
else:
for k, v in info.items():
if v != '':
setattr(ecs, k, v)
try:
if not ecs.nodes.exclude(key='1').first():
logger.info('update node for root node ecs')
node = auto_allocate_asset_node(info.get('instance_name'), 'ecs')
ecs.nodes.set([node])
ecs.save()
updated.append(info['instance_name'])
except Exception as e:
failed.append('%s: %s' % (info['instance_name'], str(e)))
if settings.AUTO_UPDATE_JUMPSERVER_ASSETS:
asset = get_object_or_none(Asset, number=info.get('instance_id'))
if not asset:
try:
with transaction.atomic():
hostname = info.get('instance_name')
domain = None
admin_user = None
if settings.ENVIROMENT == 'PROD':
if 'ebs' in hostname:
domain = 'ebs'
admin_user = '******'
elif 'vip6' in hostname:
domain = 'vip6'
admin_user = '******'
elif 'hsb' in hostname:
domain = 'hsb'
admin_user = '******'
elif 'vip5' in hostname:
domain = 'vip5'
admin_user = '******'
elif 'usa' in hostname:
domain = 'osu'
admin_user = '******'
else:
domain = None
admin_user = None
attr = {
'number': 'instance_id',
'ip': info.get('inner_ip'),
'port': 3299,
'hostname': hostname,
'platform': 'Linux',
'domain': domain,
'admin_user': admin_user
}
asset = Asset.objects.create(**attr)
# need to add auto join node
asset.nodes.set([node])
j_created.append(info['instance_name'])
except Exception as e:
j_failed.append('%s: %s' % (info['instance_name'], str(e)))
else:
setattr(ecs, 'hostname', info.get('hostname'))
try:
asset.save()
j_updated.append(info['instance_name'])
except Exception as e:
j_failed.append('%s: %s' % (info['instance_name'], str(e)))
data = {
'created': created,
'created_info': 'Created {}'.format(len(created)),
'updated': updated,
'updated_info': 'Updated {}'.format(len(updated)),
'failed': failed,
'failed_info': 'Failed {}'.format(len(failed)),
'valid': True,
'msg': 'Created: {}. Updated: {}, Error: {}'.format(
len(created), len(updated), len(failed))
}
logger.info('ecs sync finish')
logger.info(json.dumps(data))
j_data = {
'created': j_created,
'created_info': 'Created {}'.format(len(j_created)),
'updated': j_updated,
'updated_info': 'Updated {}'.format(len(j_updated)),
'failed': j_failed,
'failed_info': 'Failed {}'.format(len(j_failed)),
'valid': True,
'msg': 'Created: {}. Updated: {}, Error: {}'.format(
len(j_created), len(j_updated), len(j_failed))
}
logger.info('jump server asset update finish')
logger.info(json.dumps(j_data))
return data