def test_http_get_request(monkeypatch): monkeypatch.setattr('processor.helper.httpapi.http_utils.request.urlopen', mock_urlopen) from processor.helper.httpapi.http_utils import http_get_request st, ret = http_get_request('http://a.b.c') assert True == isinstance(ret, dict) assert 200 == st st, ret = http_get_request(None) assert ret is None assert st is None st, ret = http_get_request('http://a.b.c', {'Content-Type': 'application/json'}) assert True == isinstance(ret, dict) assert 200 == st
def get_node(credentials, node, snapshot_source, snapshot): """ Fetch node from google using connection. In this case using google client API's functions. """ collection = node['collection'] if 'collection' in node else COLLECTION parts = snapshot_source.split('.') project_id = get_field_value_with_default(snapshot, 'project-id',"") path = get_field_value_with_default(node, 'path',"") zone = re.findall(r"(?<=zones\/)[a-zA-Z0-9\-]*(?=\/)", path) db_record = { "structure": "google", "error": None, "reference": project_id, "source": parts[0], "path": path, "timestamp": int(time.time() * 1000), "queryuser": get_field_value(snapshot, 'testUser'), "checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(), "node": node, "snapshotId": node['snapshotId'], "collection": collection.replace('.', '').lower(), "region" : zone[0] if zone else "", "json": {} # Refactor when node is absent it should None, when empty object put it as {} } try: access_token = credentials.get_access_token().access_token header = { "Authorization" : ("Bearer %s" % access_token) } node_type = node['type'] if node and 'type' in node else "" base_node_type_list = node_type.split("/") if len(base_node_type_list) > 1: base_node_type = base_node_type_list[0] else: logger.error("Invalid node type %s", node_type) return db_record base_url = "%s%s" % (base_node_type, ".googleapis.com") request_url = "https://%s/%s" % (base_url, path) logger.info("Invoke request for get snapshot: %s", request_url) status, data = http_get_request(request_url, header) logger.info('Get snapshot status: %s', status) if status and isinstance(status, int) and status == 200: if data: db_record['json'] = data checksum = get_checksum(data) if checksum: db_record['checksum'] = checksum else: logger.error("Get snapshot returned invalid status: %s", status) db_record['error'] = ("Get snapshot returned invalid status: %s" % status) except Exception as ex: logger.error('Failed to populate the snapshot : %s', ex) db_record['error'] = 'Failed to populate the snapshot: %s' % ex return db_record
def get_uami_vault_access_token(): """ Get the vault access token to get all the other passwords/secrets. """ hdrs = {"Metadata": "true", "Cache-Control": "no-cache"} vaulttoken = get_from_currentdata(UAMIVAULTACCESSTOKEN) # print(vaulttoken) expiry_time = get_from_currentdata(UAMIVAULTOKENEXPIRY) is_token_valid = isinstance(expiry_time, str) and \ datetime.now() < datetime.fromtimestamp(float(expiry_time)) if (not vaulttoken) or (not is_token_valid): url = 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fvault.azure.net' # logger.info('Get Azure UAMI token REST API invoked!') print('Get Azure UAMI token REST API invoked!') status, data = http_get_request(url, headers=hdrs) print(data) if status and isinstance(status, int) and status == 200: vaulttoken = data['access_token'] expiry_time = data['expires_on'] put_in_currentdata(UAMIVAULTACCESSTOKEN, vaulttoken) put_in_currentdata(UAMIVAULTOKENEXPIRY, expiry_time) else: put_in_currentdata('errors', data) # logger.info("Get Azure token returned invalid status: %s", status) print("Get Azure token returned invalid status: %s" % status) return vaulttoken
def test_http_get_request_URLError_exception(monkeypatch): monkeypatch.setattr('processor.helper.httpapi.http_utils.request.urlopen', mock_urlopen_URLError_exception) from processor.helper.httpapi.http_utils import http_get_request st, ret = http_get_request('http://a.b.c') assert type(ret) is str assert st == 500
def test_http_get_request_exception(monkeypatch): monkeypatch.setattr('processor.helper.httpapi.http_utils.request.urlopen', mock_urlopen_exception) from processor.helper.httpapi.http_utils import http_get_request st, ret = http_get_request('http://a.b.c') assert ret is None assert st == 404
def get_snapshot_node(snapshot, token, sub_name, sub_id, node, user, snapshot_source, connector_type): """ Fetch node from azure portal using rest API.""" version = get_node_version(node, snapshot) if sub_id and token and node and node['path'] and version: db_record = get_data_record(sub_name, node, user, snapshot_source, connector_type) hdrs = {'Authorization': 'Bearer %s' % token} if node['path'].startswith('/subscriptions'): urlstr = 'https://management.azure.com%s?api-version=%s' url = urlstr % (node['path'], version) else: urlstr = 'https://management.azure.com/subscriptions/%s%s?api-version=%s' url = urlstr % (sub_id, node['path'], version) db_record['path'] = node['path'] logger.info('Get Id REST API invoked!') status, data = http_get_request(url, hdrs) logger.info('Get Id status: %s', status) if status and isinstance(status, int) and status == 200: db_record['json'] = data data_str = json.dumps(data) db_record['checksum'] = hashlib.md5( data_str.encode('utf-8')).hexdigest() else: put_in_currentdata('errors', data) logger.info("Get Id returned invalid status: %s, response: %s", status, data) logger.error( "Failed to get Azure resourse with given path : %s, please verify your azure connector detail and path given in snapshot.", node['path']) else: logger.info('Get requires valid subscription, token and path.!') return db_record
def get_all_secrets(keyvault, vaulttoken): hdrs = { 'Authorization': 'Bearer %s' % vaulttoken } logger.info('Get Id REST API invoked!') urlstr = 'https://%s.vault.azure.net/secrets?api-version=7.0' url = urlstr % (keyvault) keys_response = [] keys = [] while url != None: status, data = http_get_request(url, hdrs) if status and isinstance(status, int) and status == 200: logger.debug('Data: %s', data) values = data.get("value", []) url = data.get("nextLink",None) keys_response.extend(values) else: put_in_currentdata('errors', data) url = None logger.info("Get Id returned invalid status: %s", status) for each_key in keys_response: key_url = each_key.get("id",None) if key_url: secret_key = key_url.split("secrets/",1)[1].split("/")[0] keys.append(secret_key) return keys
def call_azure_api(self, url): hdrs = {'Authorization': 'Bearer %s' % self.token} status, data = http_get_request(url, hdrs, name='\tRESOURCE:') if status and isinstance(status, int) and status == 200: for resource in data.get("value", []): put_in_currentdata('resources', resource) self.resources += data.get("value", [])
def get_all_nodes(token, sub_name, sub_id, node, user, snapshot_source): """ Fetch all nodes from azure portal using rest API.""" collection = node['collection'] if 'collection' in node else COLLECTION parts = snapshot_source.split('.') db_records = [] d_record = { "structure": "azure", "reference": sub_name, "source": parts[0], "path": '', "timestamp": int(time.time() * 1000), "queryuser": user, "checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(), "node": node, "snapshotId": None, "mastersnapshot": True, "masterSnapshotId": [node['masterSnapshotId']], "collection": collection.replace('.', '').lower(), "json": {} # Refactor when node is absent it should None, when empty object put it as {} } # version = get_version_for_type(node) # if sub_id and token and node and version: nodetype = None if node and 'type' in node and node['type']: nodetype = node['type'] if sub_id and token and nodetype: hdrs = { 'Authorization': 'Bearer %s' % token } # urlstr = 'https://management.azure.com/subscriptions/%s/providers/%s?api-version=%s' # url = urlstr % (sub_id, node['type'], version) # db_record['path'] = node['path'] resources = get_from_currentdata('resources') if not resources: urlstr = 'https://management.azure.com/subscriptions/%s/resources?api-version=2017-05-10' url = urlstr % sub_id logger.info('Get Id REST API invoked!') status, data = http_get_request(url, hdrs) logger.info('Get Id status: %s', status) if status and isinstance(status, int) and status == 200: resources = data['value'] put_in_currentdata('resources', resources) else: put_in_currentdata('errors', data) logger.info("Get Id returned invalid status: %s", status) if resources: for idx, value in enumerate(resources): if nodetype in value['type']: db_record = copy.deepcopy(d_record) db_record['snapshotId'] = '%s%s' % (node['masterSnapshotId'], str(idx)) db_record['path'] = value['id'] db_record['json'] = value data_str = json.dumps(value) db_record['checksum'] = hashlib.md5(data_str.encode('utf-8')).hexdigest() db_records.append(db_record) else: logger.info('Get requires valid subscription, token and path.!') return db_records
def get_keyvault_secret(keyvault, secret_key, vaulttoken): hdrs = {'Authorization': 'Bearer %s' % vaulttoken} logger.info('Get Id REST API invoked!') urlstr = 'https://%s.vault.azure.net/secrets/%s?api-version=7.0' url = urlstr % (keyvault, secret_key) status, data = http_get_request(url, hdrs) logger.debug('Get Id status: %s', status) if status and isinstance(status, int) and status == 200: logger.debug('Data: %s', data) else: put_in_currentdata('errors', data) logger.info("Get Id returned invalid status: %s", status) return data
def get_node(token, sub_name, sub_id, node, user, snapshot_source): """ Fetch node from azure portal using rest API.""" collection = node['collection'] if 'collection' in node else COLLECTION parts = snapshot_source.split('.') db_records = [] db_record = { "structure": "azure", "reference": sub_name, "source": parts[0], "path": '', "timestamp": int(time.time() * 1000), "queryuser": user, "checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(), "node": node, "snapshotId": node['snapshotId'], "mastersnapshot": False, "masterSnapshotId": None, "collection": collection.replace('.', '').lower(), "region" : "", "json": {} # Refactor when node is absent it should None, when empty object put it as {} } version = get_version_for_type(node) if sub_id and token and node and node['path'] and version: hdrs = { 'Authorization': 'Bearer %s' % token } if node['path'].startswith('/subscriptions'): urlstr = 'https://management.azure.com%s?api-version=%s' url = urlstr % (node['path'], version) else: urlstr = 'https://management.azure.com/subscriptions/%s%s?api-version=%s' url = urlstr % (sub_id, node['path'], version) db_record['path'] = node['path'] # logger.info('Get Id REST API invoked!') status, data = http_get_request(url, hdrs, name='\tRESOURCE:') # logger.info('Get Id status: %s', status) if status and isinstance(status, int) and status == 200: db_record['json'] = data db_record['region'] = db_record['json'].get("location") data_str = json.dumps(data) db_record['checksum'] = hashlib.md5(data_str.encode('utf-8')).hexdigest() else: put_in_currentdata('errors', data) logger.info("Get Id returned invalid status: %s, response: %s", status, data) logger.error("Failed to get Azure resourse with given path : %s, please verify your azure connector detail and path given in snapshot.", node['path']) else: db_record = {} logger.info('Get requires valid subscription, token and path.!') return db_record
def get_snapshot_nodes(snapshot, token, sub_name, sub_id, node, user, snapshot_source, connector_type): """ Fetch all nodes from azure portal using rest API.""" db_records = [] d_record = get_data_record(sub_name, node, user, snapshot_source, connector_type) nodetype = node[ 'type'] if node and 'type' in node and node['type'] else None if sub_id and token and nodetype: hdrs = {'Authorization': 'Bearer %s' % token} resources = snapshot.get_value('resources') if not resources: urlstr = 'https://management.azure.com/subscriptions/%s/resources?api-version=2017-05-10' url = urlstr % sub_id logger.info('Get Id REST API invoked!') status, data = http_get_request(url, hdrs) logger.info('Get Id status: %s', status) if status and isinstance(status, int) and status == 200: resources = data['value'] snapshot.store_value('resources', resources) else: put_in_currentdata('errors', data) logger.info("Get Id returned invalid status: %s", status) if resources: for idx, value in enumerate(resources): if nodetype in value['type']: db_record = copy.deepcopy(d_record) db_record['snapshotId'] = '%s%s' % ( node['masterSnapshotId'], str(idx)) db_record['path'] = value['id'] db_record['json'] = value data_str = json.dumps(value) db_record['checksum'] = hashlib.md5( data_str.encode('utf-8')).hexdigest() db_records.append(db_record) else: logger.info('Get requires valid subscription, token and path.!') return db_records
def get_all_nodes(credentials, node, snapshot_source, snapshot, snapshot_data): """ Fetch all nodes from google using connection using google client API's functions. """ collection = node['collection'] if 'collection' in node else COLLECTION parts = snapshot_source.split('.') project_id = get_field_value_with_default(snapshot, 'project-id',"") node_type = get_field_value_with_default(node, 'type',"") db_record = { "structure": "google", "error": None, "reference": project_id, "source": parts[0], "path": "", "timestamp": int(time.time() * 1000), "queryuser": get_field_value(snapshot, 'testUser'), "checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(), "node": node, "snapshotId": None, "masterSnapshotId": [node['masterSnapshotId']], "collection": collection.replace('.', '').lower(), "json": {}, # Refactor when node is absent it should None, when empty object put it as {} "items": [] } if node_type: access_token = credentials.get_access_token().access_token header = { "Authorization" : ("Bearer %s" % access_token) } base_node_type_list = node_type.split("/") if len(base_node_type_list) > 1: base_node_type = base_node_type_list[1] else: logger.error("Invalid node type '%s'", node_type) return db_record request_url = get_api_path(base_node_type) request_url = generate_request_url(request_url, project_id) logger.info("Invoke request for get snapshot: %s", request_url) status, data = http_get_request(request_url, header) logger.info('Get snapshot status: %s', status) fn_str_list = "" if node and 'type' in node and node['type']: fn_str_list = get_field_value(node, 'type').split(".") response_param = "" if fn_str_list and len(fn_str_list) > 1: response_param = fn_str_list[-2] elif fn_str_list and len(fn_str_list) == 1: response_param = fn_str_list[0] if data: check_node_type = node_type node_type_list = node_type.split(".") if len(node_type_list) > 1: del node_type_list[-1] check_node_type = ".".join(node_type_list) db_record['json'] = data if response_param in data: db_record['items'] = data[response_param] elif "items" in data: if isinstance(data['items'], dict): for name, scoped_dict in data['items'].items(): if response_param in scoped_dict: db_record['items'] = db_record['items'] + scoped_dict[check_node_type] if not db_record['items']: db_record['items'] = data['items'] set_snapshot_data(node, db_record['items'], snapshot_data) checksum = get_checksum(data) if checksum: db_record['checksum'] = checksum return db_record