def test_http_get_request(monkeypatch):
monkeypatch.setattr('processor.helper.httpapi.http_utils.request.urlopen',
mock_urlopen)
from processor.helper.httpapi.http_utils import http_get_request
st, ret = http_get_request('http://a.b.c')
assert True == isinstance(ret, dict)
assert 200 == st
st, ret = http_get_request(None)
assert ret is None
assert st is None
st, ret = http_get_request('http://a.b.c',
{'Content-Type': 'application/json'})
assert True == isinstance(ret, dict)
assert 200 == st
def get_node(credentials, node, snapshot_source, snapshot):
"""
Fetch node from google using connection. In this case using google client API's
functions.
"""
collection = node['collection'] if 'collection' in node else COLLECTION
parts = snapshot_source.split('.')
project_id = get_field_value_with_default(snapshot, 'project-id',"")
path = get_field_value_with_default(node, 'path',"")
zone = re.findall(r"(?<=zones\/)[a-zA-Z0-9\-]*(?=\/)", path)
db_record = {
"structure": "google",
"error": None,
"reference": project_id,
"source": parts[0],
"path": path,
"timestamp": int(time.time() * 1000),
"queryuser": get_field_value(snapshot, 'testUser'),
"checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(),
"node": node,
"snapshotId": node['snapshotId'],
"collection": collection.replace('.', '').lower(),
"region" : zone[0] if zone else "",
"json": {} # Refactor when node is absent it should None, when empty object put it as {}
}
try:
access_token = credentials.get_access_token().access_token
header = {
"Authorization" : ("Bearer %s" % access_token)
}
node_type = node['type'] if node and 'type' in node else ""
base_node_type_list = node_type.split("/")
if len(base_node_type_list) > 1:
base_node_type = base_node_type_list[0]
else:
logger.error("Invalid node type %s", node_type)
return db_record
base_url = "%s%s" % (base_node_type, ".googleapis.com")
request_url = "https://%s/%s" % (base_url, path)
logger.info("Invoke request for get snapshot: %s", request_url)
status, data = http_get_request(request_url, header)
logger.info('Get snapshot status: %s', status)
if status and isinstance(status, int) and status == 200:
if data:
db_record['json'] = data
checksum = get_checksum(data)
if checksum:
db_record['checksum'] = checksum
else:
logger.error("Get snapshot returned invalid status: %s", status)
db_record['error'] = ("Get snapshot returned invalid status: %s" % status)
except Exception as ex:
logger.error('Failed to populate the snapshot : %s', ex)
db_record['error'] = 'Failed to populate the snapshot: %s' % ex
return db_record
def get_uami_vault_access_token():
"""
Get the vault access token to get all the other passwords/secrets.
"""
hdrs = {"Metadata": "true", "Cache-Control": "no-cache"}
vaulttoken = get_from_currentdata(UAMIVAULTACCESSTOKEN)
# print(vaulttoken)
expiry_time = get_from_currentdata(UAMIVAULTOKENEXPIRY)
is_token_valid = isinstance(expiry_time, str) and \
datetime.now() < datetime.fromtimestamp(float(expiry_time))
if (not vaulttoken) or (not is_token_valid):
url = 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fvault.azure.net'
# logger.info('Get Azure UAMI token REST API invoked!')
print('Get Azure UAMI token REST API invoked!')
status, data = http_get_request(url, headers=hdrs)
print(data)
if status and isinstance(status, int) and status == 200:
vaulttoken = data['access_token']
expiry_time = data['expires_on']
put_in_currentdata(UAMIVAULTACCESSTOKEN, vaulttoken)
put_in_currentdata(UAMIVAULTOKENEXPIRY, expiry_time)
else:
put_in_currentdata('errors', data)
# logger.info("Get Azure token returned invalid status: %s", status)
print("Get Azure token returned invalid status: %s" % status)
return vaulttoken
def test_http_get_request_URLError_exception(monkeypatch):
monkeypatch.setattr('processor.helper.httpapi.http_utils.request.urlopen',
mock_urlopen_URLError_exception)
from processor.helper.httpapi.http_utils import http_get_request
st, ret = http_get_request('http://a.b.c')
assert type(ret) is str
assert st == 500
def test_http_get_request_exception(monkeypatch):
monkeypatch.setattr('processor.helper.httpapi.http_utils.request.urlopen',
mock_urlopen_exception)
from processor.helper.httpapi.http_utils import http_get_request
st, ret = http_get_request('http://a.b.c')
assert ret is None
assert st == 404
def get_snapshot_node(snapshot, token, sub_name, sub_id, node, user,
snapshot_source, connector_type):
""" Fetch node from azure portal using rest API."""
version = get_node_version(node, snapshot)
if sub_id and token and node and node['path'] and version:
db_record = get_data_record(sub_name, node, user, snapshot_source,
connector_type)
hdrs = {'Authorization': 'Bearer %s' % token}
if node['path'].startswith('/subscriptions'):
urlstr = 'https://management.azure.com%s?api-version=%s'
url = urlstr % (node['path'], version)
else:
urlstr = 'https://management.azure.com/subscriptions/%s%s?api-version=%s'
url = urlstr % (sub_id, node['path'], version)
db_record['path'] = node['path']
logger.info('Get Id REST API invoked!')
status, data = http_get_request(url, hdrs)
logger.info('Get Id status: %s', status)
if status and isinstance(status, int) and status == 200:
db_record['json'] = data
data_str = json.dumps(data)
db_record['checksum'] = hashlib.md5(
data_str.encode('utf-8')).hexdigest()
else:
put_in_currentdata('errors', data)
logger.info("Get Id returned invalid status: %s, response: %s",
status, data)
logger.error(
"Failed to get Azure resourse with given path : %s, please verify your azure connector detail and path given in snapshot.",
node['path'])
else:
logger.info('Get requires valid subscription, token and path.!')
return db_record
def get_all_secrets(keyvault, vaulttoken):
hdrs = {
'Authorization': 'Bearer %s' % vaulttoken
}
logger.info('Get Id REST API invoked!')
urlstr = 'https://%s.vault.azure.net/secrets?api-version=7.0'
url = urlstr % (keyvault)
keys_response = []
keys = []
while url != None:
status, data = http_get_request(url, hdrs)
if status and isinstance(status, int) and status == 200:
logger.debug('Data: %s', data)
values = data.get("value", [])
url = data.get("nextLink",None)
keys_response.extend(values)
else:
put_in_currentdata('errors', data)
url = None
logger.info("Get Id returned invalid status: %s", status)
for each_key in keys_response:
key_url = each_key.get("id",None)
if key_url:
secret_key = key_url.split("secrets/",1)[1].split("/")[0]
keys.append(secret_key)
return keys
def call_azure_api(self, url):
hdrs = {'Authorization': 'Bearer %s' % self.token}
status, data = http_get_request(url, hdrs, name='\tRESOURCE:')
if status and isinstance(status, int) and status == 200:
for resource in data.get("value", []):
put_in_currentdata('resources', resource)
self.resources += data.get("value", [])
def get_all_nodes(token, sub_name, sub_id, node, user, snapshot_source):
""" Fetch all nodes from azure portal using rest API."""
collection = node['collection'] if 'collection' in node else COLLECTION
parts = snapshot_source.split('.')
db_records = []
d_record = {
"structure": "azure",
"reference": sub_name,
"source": parts[0],
"path": '',
"timestamp": int(time.time() * 1000),
"queryuser": user,
"checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(),
"node": node,
"snapshotId": None,
"mastersnapshot": True,
"masterSnapshotId": [node['masterSnapshotId']],
"collection": collection.replace('.', '').lower(),
"json": {} # Refactor when node is absent it should None, when empty object put it as {}
}
# version = get_version_for_type(node)
# if sub_id and token and node and version:
nodetype = None
if node and 'type' in node and node['type']:
nodetype = node['type']
if sub_id and token and nodetype:
hdrs = {
'Authorization': 'Bearer %s' % token
}
# urlstr = 'https://management.azure.com/subscriptions/%s/providers/%s?api-version=%s'
# url = urlstr % (sub_id, node['type'], version)
# db_record['path'] = node['path']
resources = get_from_currentdata('resources')
if not resources:
urlstr = 'https://management.azure.com/subscriptions/%s/resources?api-version=2017-05-10'
url = urlstr % sub_id
logger.info('Get Id REST API invoked!')
status, data = http_get_request(url, hdrs)
logger.info('Get Id status: %s', status)
if status and isinstance(status, int) and status == 200:
resources = data['value']
put_in_currentdata('resources', resources)
else:
put_in_currentdata('errors', data)
logger.info("Get Id returned invalid status: %s", status)
if resources:
for idx, value in enumerate(resources):
if nodetype in value['type']:
db_record = copy.deepcopy(d_record)
db_record['snapshotId'] = '%s%s' % (node['masterSnapshotId'], str(idx))
db_record['path'] = value['id']
db_record['json'] = value
data_str = json.dumps(value)
db_record['checksum'] = hashlib.md5(data_str.encode('utf-8')).hexdigest()
db_records.append(db_record)
else:
logger.info('Get requires valid subscription, token and path.!')
return db_records
def get_keyvault_secret(keyvault, secret_key, vaulttoken):
hdrs = {'Authorization': 'Bearer %s' % vaulttoken}
logger.info('Get Id REST API invoked!')
urlstr = 'https://%s.vault.azure.net/secrets/%s?api-version=7.0'
url = urlstr % (keyvault, secret_key)
status, data = http_get_request(url, hdrs)
logger.debug('Get Id status: %s', status)
if status and isinstance(status, int) and status == 200:
logger.debug('Data: %s', data)
else:
put_in_currentdata('errors', data)
logger.info("Get Id returned invalid status: %s", status)
return data
def get_node(token, sub_name, sub_id, node, user, snapshot_source):
""" Fetch node from azure portal using rest API."""
collection = node['collection'] if 'collection' in node else COLLECTION
parts = snapshot_source.split('.')
db_records = []
db_record = {
"structure": "azure",
"reference": sub_name,
"source": parts[0],
"path": '',
"timestamp": int(time.time() * 1000),
"queryuser": user,
"checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(),
"node": node,
"snapshotId": node['snapshotId'],
"mastersnapshot": False,
"masterSnapshotId": None,
"collection": collection.replace('.', '').lower(),
"region" : "",
"json": {} # Refactor when node is absent it should None, when empty object put it as {}
}
version = get_version_for_type(node)
if sub_id and token and node and node['path'] and version:
hdrs = {
'Authorization': 'Bearer %s' % token
}
if node['path'].startswith('/subscriptions'):
urlstr = 'https://management.azure.com%s?api-version=%s'
url = urlstr % (node['path'], version)
else:
urlstr = 'https://management.azure.com/subscriptions/%s%s?api-version=%s'
url = urlstr % (sub_id, node['path'], version)
db_record['path'] = node['path']
# logger.info('Get Id REST API invoked!')
status, data = http_get_request(url, hdrs, name='\tRESOURCE:')
# logger.info('Get Id status: %s', status)
if status and isinstance(status, int) and status == 200:
db_record['json'] = data
db_record['region'] = db_record['json'].get("location")
data_str = json.dumps(data)
db_record['checksum'] = hashlib.md5(data_str.encode('utf-8')).hexdigest()
else:
put_in_currentdata('errors', data)
logger.info("Get Id returned invalid status: %s, response: %s", status, data)
logger.error("Failed to get Azure resourse with given path : %s, please verify your azure connector detail and path given in snapshot.", node['path'])
else:
db_record = {}
logger.info('Get requires valid subscription, token and path.!')
return db_record
def get_snapshot_nodes(snapshot, token, sub_name, sub_id, node, user,
snapshot_source, connector_type):
""" Fetch all nodes from azure portal using rest API."""
db_records = []
d_record = get_data_record(sub_name, node, user, snapshot_source,
connector_type)
nodetype = node[
'type'] if node and 'type' in node and node['type'] else None
if sub_id and token and nodetype:
hdrs = {'Authorization': 'Bearer %s' % token}
resources = snapshot.get_value('resources')
if not resources:
urlstr = 'https://management.azure.com/subscriptions/%s/resources?api-version=2017-05-10'
url = urlstr % sub_id
logger.info('Get Id REST API invoked!')
status, data = http_get_request(url, hdrs)
logger.info('Get Id status: %s', status)
if status and isinstance(status, int) and status == 200:
resources = data['value']
snapshot.store_value('resources', resources)
else:
put_in_currentdata('errors', data)
logger.info("Get Id returned invalid status: %s", status)
if resources:
for idx, value in enumerate(resources):
if nodetype in value['type']:
db_record = copy.deepcopy(d_record)
db_record['snapshotId'] = '%s%s' % (
node['masterSnapshotId'], str(idx))
db_record['path'] = value['id']
db_record['json'] = value
data_str = json.dumps(value)
db_record['checksum'] = hashlib.md5(
data_str.encode('utf-8')).hexdigest()
db_records.append(db_record)
else:
logger.info('Get requires valid subscription, token and path.!')
return db_records
def get_all_nodes(credentials, node, snapshot_source, snapshot, snapshot_data):
"""
Fetch all nodes from google using connection using google client API's functions.
"""
collection = node['collection'] if 'collection' in node else COLLECTION
parts = snapshot_source.split('.')
project_id = get_field_value_with_default(snapshot, 'project-id',"")
node_type = get_field_value_with_default(node, 'type',"")
db_record = {
"structure": "google",
"error": None,
"reference": project_id,
"source": parts[0],
"path": "",
"timestamp": int(time.time() * 1000),
"queryuser": get_field_value(snapshot, 'testUser'),
"checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(),
"node": node,
"snapshotId": None,
"masterSnapshotId": [node['masterSnapshotId']],
"collection": collection.replace('.', '').lower(),
"json": {}, # Refactor when node is absent it should None, when empty object put it as {}
"items": []
}
if node_type:
access_token = credentials.get_access_token().access_token
header = {
"Authorization" : ("Bearer %s" % access_token)
}
base_node_type_list = node_type.split("/")
if len(base_node_type_list) > 1:
base_node_type = base_node_type_list[1]
else:
logger.error("Invalid node type '%s'", node_type)
return db_record
request_url = get_api_path(base_node_type)
request_url = generate_request_url(request_url, project_id)
logger.info("Invoke request for get snapshot: %s", request_url)
status, data = http_get_request(request_url, header)
logger.info('Get snapshot status: %s', status)
fn_str_list = ""
if node and 'type' in node and node['type']:
fn_str_list = get_field_value(node, 'type').split(".")
response_param = ""
if fn_str_list and len(fn_str_list) > 1:
response_param = fn_str_list[-2]
elif fn_str_list and len(fn_str_list) == 1:
response_param = fn_str_list[0]
if data:
check_node_type = node_type
node_type_list = node_type.split(".")
if len(node_type_list) > 1:
del node_type_list[-1]
check_node_type = ".".join(node_type_list)
db_record['json'] = data
if response_param in data:
db_record['items'] = data[response_param]
elif "items" in data:
if isinstance(data['items'], dict):
for name, scoped_dict in data['items'].items():
if response_param in scoped_dict:
db_record['items'] = db_record['items'] + scoped_dict[check_node_type]
if not db_record['items']:
db_record['items'] = data['items']
set_snapshot_data(node, db_record['items'], snapshot_data)
checksum = get_checksum(data)
if checksum:
db_record['checksum'] = checksum
return db_record
