def _query_native(subscriptions: Iterable[str], query: str,
max_pages) -> List[QueryResponse]:
client = default_sdk_client(ResourceGraphClient)
query_options = QueryRequestOptions()
query_request = QueryRequest(subscriptions=realize_sequence(subscriptions),
query=query,
options=query_options)
query_response: QueryResponse = client.resources(query_request)
if query_response.result_truncated is ResultTruncated.true:
raise RuntimeError(
"results are truncated. project id to enable paging.")
if query_response.skip_token:
page_size = query_response.count
if query_response.total_records > page_size * max_pages:
raise RuntimeError("too many results. increase max pages.")
responses = [query_response]
while query_response.skip_token:
query_options = QueryRequestOptions(
skip_token=query_response.skip_token)
query_request = QueryRequest(subscriptions=subscriptions,
query=query,
options=query_options)
query_response = client.resources(query_request)
responses.append(query_response)
return responses
def __query_request_from(query, experiment_configuration: Configuration):
configuration = load_configuration(experiment_configuration)
arg_query_options = arg.models.QueryRequestOptions(result_format="table")
result = QueryRequest(query=query,
subscriptions=[configuration.get('subscription_id')],
options=arg_query_options)
return result
def __create_resource_graph_query(query, resource_type, configuration):
subscription_id = configuration['azure']['subscription_id']
_query = __create_query(resource_type, query)
query = QueryRequest(query=_query,
subscriptions=[subscription_id],
additional_properties=True)
return query
def __query_request_from(query, configuration):
subscription_id = configuration.get("azure_subscription_id")
if not subscription_id:
subscription_id = configuration['azure']['subscription_id']
result = QueryRequest(query=query, subscriptions=[subscription_id])
return result
def create_request(resource_type: str, user_query: str,
experiment_configuration: Configuration) -> QueryRequest:
prepared_query = __prepare(resource_type, user_query)
subscription_id = load_subscription_id()
result = QueryRequest(query=prepared_query,
subscriptions=[subscription_id])
return result
def read_query(self, model, search_domains=[], start_row=0):
queryStr = self.forge_query(model)
query = QueryRequest(query=queryStr,
subscriptions=[self.subscription_id])
query_response = self.client.resources(query)
print("Basic query :\n{}".format(query_response))
return query_response.data
def get_count(self, model, search_domains=[]):
queryStr = self.forge_query(model, count=True)
query = QueryRequest(query=queryStr,
subscriptions=[self.subscription_id])
query_response = self.client.resources(query)
total_count = query_response.data[0]['Count']
return total_count
def get_resources(self, _):
session = self.manager.get_session()
client = session.client('azure.mgmt.resourcegraph.ResourceGraphClient')
# empty scope will return all resource
query_scope = ""
if self.manager.resource_type.resource_type != 'armresource':
query_scope = "where type =~ '%s'" % self.manager.resource_type.resource_type
query = QueryRequest(query=query_scope,
subscriptions=[session.get_subscription_id()])
res = client.resources(query)
cols = [c['name'] for c in res.data['columns']]
data = [dict(zip(cols, r)) for r in res.data['rows']]
return data
def get_resources(self, _):
log.warning('The Azure Resource Graph source '
'should not be used in production scenarios at this time.')
session = self.manager.get_session()
client = session.client('azure.mgmt.resourcegraph.ResourceGraphClient')
# empty scope will return all resource
query_scope = ""
if self.manager.resource_type.resource_type != 'armresource':
query_scope = "where type =~ '%s'" % self.manager.resource_type.resource_type
query = QueryRequest(query=query_scope,
subscriptions=[session.get_subscription_id()])
res = client.resources(query)
cols = [c['name'] for c in res.data['columns']]
data = [dict(zip(cols, r)) for r in res.data['rows']]
return data
def dataprotection_backup_instance_list_from_resourcegraph(
client,
datasource_type,
resource_groups=None,
vaults=None,
subscriptions=None,
protection_status=None,
datasource_id=None):
if subscriptions is None:
subscriptions = [backupcenter_helper.get_selected_subscription()]
query = backupcenter_helper.get_backup_instance_query(
datasource_type, resource_groups, vaults, protection_status,
datasource_id)
request_options = QueryRequestOptions(top=1000, skip=0)
request = QueryRequest(query=query,
subscriptions=subscriptions,
options=request_options)
response = client.resources(request)
return response.data
def resource_graph_query(self, query: str) -> Iterator:
LOGGER.debug("query=%r", query)
request = QueryRequest(
subscriptions=[self.subscription_id],
query=query,
options=QueryRequestOptions(result_format="objectArray"),
)
def paged_query() -> Iterator[list]:
while True:
response = self.resource_graph.resources(request)
yield response.data
if not response.skip_token:
# See https://docs.microsoft.com/en-us/azure/governance/resource-graph/concepts/work-with-data#paging-results
assert response.result_truncated == "false", "paging is not possible because you missed id column"
break
LOGGER.debug("get next page of query=%r", query)
request.options.skip_token = response.skip_token
return chain.from_iterable(paged_query())
def __triggerArgResourcesMethod(self, query: str, subscriptionIds: List[str], skipToken: str = None) -> QueryResponse:
"""Trigger ARG Resources method.
Args:
query (str): Query to be run.
subscriptionIds (List[str]): List of subscriptions within which the resources should be queried.
skipToken (str, optional): Token from last run in case of pagination. Defaults to None.
Raises:
Exception: Maximum number of retries is exceeded.
Returns:
QueryResponse: Response from ARG.
"""
# Build the query options.
self.tracer.info(
"%s Entered __triggerArgResourcesMethod" % self.logTag)
self.tracer.info("%s Building query request options." % self.logTag)
argQueryOptions = None
if skipToken is None:
argQueryOptions = QueryRequestOptions(
result_format=RESULT_FORMAT)
else:
argQueryOptions = QueryRequestOptions(
result_format=RESULT_FORMAT, skip_token=skipToken)
# Build the query request.
self.tracer.info("%s Building ARG query request." % self.logTag)
argQuery = QueryRequest(
query=query,
subscriptions=subscriptionIds,
options=argQueryOptions
)
# Call the ARG method in a loop to handle throttling.
retries = 0
while retries <= MAX_RETRIES:
self.tracer.info(
"%s Invoking resources method of ARG SDK." % self.logTag)
argQueryResponse = None
try:
# Track latency of the SDK call.
latencyStartTime = time.time()
argQueryResponse = self.argClient.resources(
argQuery, cls=self.__customResponse)
latency = TimeUtils.getElapsedMilliseconds(latencyStartTime)
except Exception as e:
latency = TimeUtils.getElapsedMilliseconds(latencyStartTime)
self.tracer.error("%s ARG call failed. subscription=%s; query=%s; latency=%s ms. (e)",
self.logTag, subscriptionIds, query, latency, e, exc_info=True)
if self.__shouldRetry(argQueryResponse):
self.tracer.info(
"%s Throttling limit is hit or failed to extract the status code header. ARG call took %s ms. subscription=%s; query=%s;" % (self.logTag, latency, subscriptionIds, query))
self.__waitForQuotaReset(argQueryResponse.quotaResetsAfter)
retries += 1
continue
self.tracer.info("%s Throttling limit not hit. subscription=%s; query=%s;" %
(self.logTag, subscriptionIds, query))
self.tracer.info(
"%s ARG response receieved. subscription=%s; query=%s; totalNumberOfResources=%s; numberOfResourcesInCurrentPage=%s; latency=%s ms." % (self.logTag, subscriptionIds, query, argQueryResponse.total_records, argQueryResponse.count, latency))
return argQueryResponse
# Max number of retries exceeded.
if retries > MAX_RETRIES:
errorMessage = "%s Maximum number of retries exceeded (MaxRetriesConfig=%s; CurrentRetries=%s). Aborting the ARG call. subscription=%s; query=%s" % (
self.logTag, MAX_RETRIES, retries, subscriptionIds, query)
self.tracer.error(errorMessage)
raise Exception(errorMessage)
def __query_request_from(query, experiment_configuration: Configuration):
configuration = load_configuration(experiment_configuration)
result = QueryRequest(query=query,
subscriptions=[configuration.get('subscription_id')])
return result