def sync_subscription_regions(subscription_id: str, access_token: str = None) -> Dict:
"""Azure Region for Subscription Sync"""
_created = 0
_updated = 0
provider = Provider.objects.get(name=constants.Provider.AZURE)
if access_token:
session = cli.get_session(access_token)
else:
_, session = get_subscription_and_session(subscription_id)
for region in cli.get_regions_list(subscription_id, session=session):
_, created = Region.objects.get_or_create(
name=region['name'], provider=provider,
defaults=dict(
display_name=region.get('displayName'),
longitude=float(region.get('longitude')),
latitude=float(region.get('latitude'))
)
)
if created:
_created += 1
else:
_updated += 1
logger.info(f"sync - azure - Region - created[{_created}]- updated[{_updated}]")
return dict(errors=0, created=_created, updated=_updated, deleted=0)
def get_subscription_and_session(subscription_id):
# TODO: raise if active=False
subscription = models.SubscriptionAzure.objects.get(
subscription_id=subscription_id)
auth = subscription.get_auth()
token = get_access_token(**auth)
session = cli.get_session(token=token['access_token'])
return subscription, session
def test_get_resourcegroups_list(mock_response_class, json_file):
data = json_file("resource_group_list.json")
with patch("requests.Session.get") as func:
func.return_value = mock_response_class(200, data)
session = core.get_session("test")
response = core.get_resourcegroups_list(
"00000000-0000-0000-0000-000000000000", session=session)
assert response == data["value"]
def test_get_resource_by_id(mock_response_class, json_file):
data = json_file("resource-vm.json")
resource_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/MY_RG_GROUP/providers/Microsoft.Compute/virtualMachines/MY_VM"
with patch("requests.Session.get") as func:
func.return_value = mock_response_class(200, data)
session = core.get_session("test")
response = core.get_resource_by_id(resource_id, session=session)
assert response == data
def get_subscription_and_session(subscription_id: str, timeout: float = None):
"""
Args:
subscription_id:
Returns:
(models.SubscriptionAzure, requests.Session)
"""
# TODO: raise if active=False
subscription = models.SubscriptionAzure.objects.get(subscription_id=subscription_id)
auth = subscription.get_auth()
token = get_access_token(timeout=timeout, **auth)
session = cli.get_session(token=token['access_token'])
return subscription, session
def sync_resource_list(
subscription_id: str = None,
resources: List = None,
subscription: models.SubscriptionAzure = None,
session: requests.Session = None,
access_token: str = None,
disable_changes: bool = False,
disable_delete: bool = False,
timeout: float = None) -> Dict:
provider = Provider.objects.filter(name=constants.Provider.AZURE).first()
if not provider:
raise ProviderNotFound(f"Provider not found : {constants.Provider.AZURE}")
if not subscription_id and not subscription:
raise AttributeError("subscription_id or subscription is require")
if not subscription or not session:
# TODO: catch error
if access_token:
session = cli.get_session(access_token)
subscription = models.SubscriptionAzure.objects.get(subscription_id=subscription_id)
else:
subscription, session = get_subscription_and_session(subscription_id)
company = subscription.company
# TODO: include company.resource_types
# TODO: include company.regions
_created = 0
_updated = 0
_errors = 0
_deleted = 0
found_ids = []
regions = {}
resource_groups = {}
if not resources:
resources = cli.get_resources_list(subscription_id, session, timeout=timeout)
for i, resource in enumerate(resources):
is_complete = False
if "properties" in resource:
is_complete = True
resource_id = resource['id']
found_ids.append(resource_id)
location = resource['location']
logger.debug(f"{i} - create or update azure resource [{resource_id}]")
# --- Region
if location in regions:
region = regions[location]
else:
region = Region.objects.filter(name=location, provider=provider).first()
if not region:
_errors += 1
msg = f"region not found [{location}] - bypass resource [{resource_id}]"
logger.error(msg)
continue
if not region.name in regions:
regions[region.name] = region
# --- ResourceGroup
group_name = resource_id.split('/')[4]
group_id = f"/subscriptions/{subscription.subscription_id}/resourceGroups/{group_name}"
if group_id in resource_groups:
resource_group = resource_groups[group_id]
else:
resource_group = models.ResourceAzure.objects.filter(
resource_id__iexact=group_id,
company=company,
subscription=subscription
).first()
if not resource_group:
_errors += 1
msg = f"resource group [{group_id}] not found - resource [{resource_id}]"
logger.error(msg)
continue
if not group_id in resource_groups:
resource_groups[group_id] = resource_group
try:
result = sync_resource(
subscription_id=subscription_id,
resource=resource if is_complete else None,
resource_id=resource_id,
resource_type=resource.get('type'),
subscription=subscription,
resource_group=resource_group,
region=region,
session=session,
access_token=access_token,
disable_changes=disable_changes
)
if result["created"]:
_created += 1
else:
_updated += 1
except Exception as err:
_errors += 1
logger.error(f"sync resource error: {str(err)}")
continue
if not disable_delete:
# Create events delete
qs = models.ResourceAzure.objects.filter(subscription=subscription, resource_group__isnull=False).exclude(
resource_id__in=found_ids
)
if not disable_changes and getattr(settings, "MCE_CHANGES_ENABLE", False):
ResourceEventChange.create_event_change_delete(qs)
_deleted, objects = qs.delete()
print("deleted objects : %s" % objects)
# FIXME: _deleted = nombre total d'entrée avec cascade !!!
logger.info("[%s] old ResourceAzure and dependencies deleted" % _deleted)
return dict(errors=_errors, created=_created, updated=_updated, deleted=_deleted)
def sync_resource(
resource: Dict = None,
subscription_id: str = None,
resource_id: str = None,
resource_group: models.ResourceAzure = None,
resource_type: str = None,
region: Region = None,
subscription: models.SubscriptionAzure = None,
session: requests.Session = None,
access_token: str = None,
disable_changes: bool = False,
timeout: float = None) -> Dict:
"""Synchronize a `models.ResourceAzure` with database.
Args:
resource:
subscription_id:
resource_id:
resource_group:
resource_type:
subscription:
session:
Returns:
Dict: Return Pk and created is True|False and changes dict.
"""
if not resource:
if not resource_id:
raise AttributeError("requires value: resource_id or resource")
if not resource_type:
raise AttributeError("requires value: resource_type or resource")
provider = Provider.objects.filter(name=constants.Provider.AZURE).first()
if not provider:
raise ProviderNotFound(f"Provider not found : {constants.Provider.AZURE}")
if resource and not resource_id:
resource_id = resource['id']
if resource and resource_id and not subscription_id:
subscription_id = resource_id.split('/')[4]
# TODO: récupérer le token de session ?: session.headers.get('authorization')
# TODO: exception
if not subscription or not session:
if access_token:
session = cli.get_session(access_token)
subscription = models.SubscriptionAzure.objects.get(subscription_id=subscription_id)
else:
subscription, session = get_subscription_and_session(subscription_id)
company = subscription.company
if resource and not resource_type:
resource_type = resource['type']
logger.debug(f"start for resource [{resource_id}]")
if '|' in resource_type:
resource_type = resource_type.split('|')[0]
_type = ResourceType.objects.filter(name__iexact=resource_type, provider=provider).first()
if not _type:
msg = f"resource type [{resource_type}] not found - resource [{resource_id}]"
logger.error(msg)
raise ResourceTypeNotFound(msg)
# Microsoft.Automation/automationAccounts
if _type.exclude_sync:
msg = f"resource type [{resource_type}] disabled for synchronize - resource [{resource_id}]"
logger.error(msg)
raise DisableSyncError(msg)
if not resource_group:
group_name = resource_id.split('/')[4]
group_id = f"/subscriptions/{subscription.subscription_id}/resourceGroups/{group_name}"
resource_group = models.ResourceAzure.objects.filter(
resource_id__iexact=group_id,
company=company,
subscription=subscription
).first()
if not resource_group:
msg = f"resource group [{group_id}] not found - resource [{resource_id}]"
logger.error(msg)
raise ResourceGroupNotFound(msg)
if not resource:
try:
resource = cli.get_resource_by_id(resource_id, session=session, timeout=timeout)
except Exception as err:
msg = f"fetch resource {resource_id} error : {err}"
logger.exception(msg)
raise FetchResourceError(msg)
if not region:
region = Region.objects.filter(name=resource['location'], provider=provider).first()
if not region:
msg = f"region not found [{resource['location']}] - bypass resource [{resource_id}]"
raise RegionNotFound(msg)
metas = resource.get('properties', {}) or {}
tags_objects = []
datas = dict(
name=resource['name'],
subscription=subscription,
company=company,
resource_type=_type,
region=region,
provider=provider,
resource_group=resource_group,
metas=metas,
)
if resource.get('sku'):
datas['sku'] = resource.get('sku')
if resource.get('kind'):
datas['kind'] = resource.get('kind')
if resource.get('plan'):
datas['plan'] = resource.get('plan')
if resource.get('managed_by'):
datas['managed_by'] = resource.get('managed_by')
tags = resource.get('tags', {}) or {}
for k, v in tags.items():
tag, created = Tag.objects.update_or_create(
name=k, company=company, provider=provider, value=v
)
tags_objects.append(tag)
# if created: todo event tag
old_resource = models.ResourceAzure.objects.filter(resource_id=resource_id).first()
old_object = None
if old_resource:
old_object = old_resource.to_dict(exclude=["created", "updated"])
new_resource, created = models.ResourceAzure.objects.update_or_create(
resource_id=resource_id, defaults=datas
)
if tags_objects:
new_resource.tags.set(tags_objects)
result = {"pk": new_resource.pk, "created": created, "changes": None}
if not disable_changes and getattr(settings, "MCE_CHANGES_ENABLE", False):
changes = None
if not created:
new_object = new_resource.to_dict(exclude=["created", "updated", "tagged_items"])
changes = ResourceEventChange.create_event_change_update(old_object, new_object, new_resource)
if changes:
result["changes"] = changes.to_dict(exclude=['created', 'updated', 'tagged_items'])
return result
def sync_resource_group(
subscription_id: str,
resources_groups: List = None,
max_error: int = None,
disable_changes: bool = False,
disable_delete: bool = False,
access_token: str = None,
timeout: float = None) -> Dict:
# TODO: catch error
if access_token:
session = cli.get_session(access_token)
subscription = models.SubscriptionAzure.objects.get(subscription_id=subscription_id)
else:
subscription, session = get_subscription_and_session(subscription_id)
company = subscription.company
provider = Provider.objects.get(name=constants.Provider.AZURE)
# TODO: catch error
if not resources_groups:
resources_groups = cli.get_resourcegroups_list(subscription_id, session=session, timeout=timeout)
_created = 0
_updated = 0
_errors = 0
_deleted = 0
found_ids = []
RESOURCE_GROUP_TYPE = "Microsoft.Resources/resourceGroups".lower()
_type = ResourceType.objects.get(name__iexact=RESOURCE_GROUP_TYPE)
for r in resources_groups:
resource_id = r['id'].lower()
found_ids.append(resource_id)
region = Region.objects.filter(provider=provider).filter(Q(name=r['location']) | Q(display_name=r['location'])).first()
if not region:
_errors += 1
msg = f"region not found [{r['location']}] - bypass resource [{resource_id}]"
logger.error(msg)
continue
if r['type'].lower() != RESOURCE_GROUP_TYPE:
_errors += 1
msg = f"bad resource type [{r['type']}] - bypass resource [{resource_id}]"
logger.error(msg)
continue
if max_error and _errors >= max_error:
raise Exception("MAX ERROR...")
# TODO: ajouter autres champs ?
metas = r.get('properties', {})
assert isinstance(metas, dict) is True
tags_objects = []
# TODO: events et logs
tags = r.get('tags', {}) or {}
for k, v in tags.items():
tag, created = Tag.objects.update_or_create(
name=k, company=company, provider=provider, value=v
)
tags_objects.append(tag)
# if created: todo event tag
old_resource = models.ResourceAzure.objects.filter(resource_id=resource_id).first()
old_object = None
if old_resource:
old_object = old_resource.to_dict(exclude=["created", "updated"])
new_resource, created = models.ResourceAzure.objects.update_or_create(
resource_id=resource_id,
defaults=dict(
name=r['name'], # TODO: lower ?
subscription=subscription,
company=company,
resource_type=_type,
#location=r['location'],
region=region,
provider=provider,
metas=metas,
)
)
#new_resource.metas = metas
#new_resource.save()
#new_resource.refresh_from_db()
if tags_objects:
new_resource.tags.set(tags_objects)
if created:
_created += 1
elif not disable_changes and getattr(settings, "MCE_CHANGES_ENABLE", False):
new_object = new_resource.to_dict(exclude=["created", "updated", "tagged_items"])
#print('!!! old, new, equal : ', type(old_object['metas']), type(new_object['metas']), old_object['metas']==new_object['metas'])
changes = ResourceEventChange.create_event_change_update(old_object, new_object, new_resource)
if changes:
_updated += 1
logger.info(
"sync - azure - ResourceAzure Group - _errors[%s] - created[%s]- updated[%s]"
% (_errors, _created, _updated)
)
if not disable_delete:
# Create events delete
qs = models.ResourceAzure.objects.filter(resource_group__isnull=True).exclude(
resource_id__in=found_ids, subscription=subscription
)
# TODO: CHANGES_MODES: CREATE/UPDATE/DELETE
if not disable_changes and getattr(settings, "MCE_CHANGES_ENABLE", False):
ResourceEventChange.create_event_change_delete(qs)
# Mark for deleted
# TODO: delegate tasks ?
_deleted, objects = qs.delete()
# FIXME: _deleted = nombre total d'entrée avec cascade !!!
print("deleted objects : %s" % objects)
logger.info(f"mark for deleted. [{_deleted}] old ResourceAzure Group and dependencies")
return dict(errors=_errors, created=_created, updated=_updated, deleted=_deleted)