def get_vault_access_token(tenant_id, vault_client_id, client_secret=None):
"""
Get the vault access token to get all the other passwords/secrets.
"""
vaulttoken = get_from_currentdata(VAULTACCESSTOKEN)
expiry_time = get_from_currentdata(VAULTOKENEXPIRY)
is_token_valid = isinstance(expiry_time, str) and \
datetime.now() < datetime.fromtimestamp(float(expiry_time))
if (not vaulttoken) or (not is_token_valid):
vault_client_secret = client_secret if client_secret else get_vault_client_secret(
)
data = {
'grant_type': 'client_credentials',
'client_id': vault_client_id,
'client_secret': vault_client_secret,
'resource': 'https://vault.azure.net'
}
hdrs = {'Cache-Control': "no-cache", "Accept": "application/json"}
if tenant_id:
url = 'https://login.microsoftonline.com/%s/oauth2/token' % tenant_id
logger.info('Get Azure token REST API invoked!')
status, data = http_post_request(url, data, headers=hdrs)
if status and isinstance(status, int) and status == 200:
vaulttoken = data['access_token']
expiry_time = data['expires_on']
put_in_currentdata(VAULTACCESSTOKEN, vaulttoken)
put_in_currentdata(VAULTOKENEXPIRY, expiry_time)
else:
put_in_currentdata('errors', data)
logger.info("Get Azure token returned invalid status: %s",
status)
return vaulttoken
def get_uami_vault_access_token():
"""
Get the vault access token to get all the other passwords/secrets.
"""
hdrs = {"Metadata": "true", "Cache-Control": "no-cache"}
vaulttoken = get_from_currentdata(UAMIVAULTACCESSTOKEN)
# print(vaulttoken)
expiry_time = get_from_currentdata(UAMIVAULTOKENEXPIRY)
is_token_valid = isinstance(expiry_time, str) and \
datetime.now() < datetime.fromtimestamp(float(expiry_time))
if (not vaulttoken) or (not is_token_valid):
url = 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fvault.azure.net'
# logger.info('Get Azure UAMI token REST API invoked!')
print('Get Azure UAMI token REST API invoked!')
status, data = http_get_request(url, headers=hdrs)
print(data)
if status and isinstance(status, int) and status == 200:
vaulttoken = data['access_token']
expiry_time = data['expires_on']
put_in_currentdata(UAMIVAULTACCESSTOKEN, vaulttoken)
put_in_currentdata(UAMIVAULTOKENEXPIRY, expiry_time)
else:
put_in_currentdata('errors', data)
# logger.info("Get Azure token returned invalid status: %s", status)
print("Get Azure token returned invalid status: %s" % status)
return vaulttoken
def test_check_and_add_error():
from processor.helper.httpapi.http_utils import check_and_add_error
from processor.helper.config.rundata_utils import save_currentdata, get_from_currentdata
save_currentdata(None)
check_and_add_error(200, 'Failed http get')
value = get_from_currentdata('errors')
assert value is None
check_and_add_error(400, 'Failed http get')
value = get_from_currentdata('errors')
assert value is not None
def get_client_secret1(key='CLIENTKEY'):
""" Return the client secret used for the current run"""
client_secret = get_from_currentdata(CLIENTSECRET)
if not client_secret:
client_secret = os.getenv(key, None)
# if not client_secret:
# client_secret = input('Enter the client secret for the app: ')
return client_secret
def get_vault_client_secret():
""" Return the vault client secret used."""
vault_client_secret = get_from_currentdata(VAULTCLIENTSECRET)
if not vault_client_secret:
vault_client_secret = os.getenv('VAULTCLIENTKEY', None)
if not vault_client_secret:
vault_client_secret = input('Enter the vault client secret: ')
return vault_client_secret
def get_git_pwd(key='GIT_PWD'):
""" Return the git password for https connection"""
git_pwd = get_from_currentdata('GIT_PWD')
if not git_pwd:
git_pwd = os.getenv(key, None)
if git_pwd:
logger.info("Git password from envirnment: %s", '*' * len(git_pwd))
return git_pwd
def get_all_nodes(token, sub_name, sub_id, node, user, snapshot_source):
""" Fetch all nodes from azure portal using rest API."""
collection = node['collection'] if 'collection' in node else COLLECTION
parts = snapshot_source.split('.')
db_records = []
d_record = {
"structure": "azure",
"reference": sub_name,
"source": parts[0],
"path": '',
"timestamp": int(time.time() * 1000),
"queryuser": user,
"checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(),
"node": node,
"snapshotId": None,
"mastersnapshot": True,
"masterSnapshotId": [node['masterSnapshotId']],
"collection": collection.replace('.', '').lower(),
"json": {} # Refactor when node is absent it should None, when empty object put it as {}
}
# version = get_version_for_type(node)
# if sub_id and token and node and version:
nodetype = None
if node and 'type' in node and node['type']:
nodetype = node['type']
if sub_id and token and nodetype:
hdrs = {
'Authorization': 'Bearer %s' % token
}
# urlstr = 'https://management.azure.com/subscriptions/%s/providers/%s?api-version=%s'
# url = urlstr % (sub_id, node['type'], version)
# db_record['path'] = node['path']
resources = get_from_currentdata('resources')
if not resources:
urlstr = 'https://management.azure.com/subscriptions/%s/resources?api-version=2017-05-10'
url = urlstr % sub_id
logger.info('Get Id REST API invoked!')
status, data = http_get_request(url, hdrs)
logger.info('Get Id status: %s', status)
if status and isinstance(status, int) and status == 200:
resources = data['value']
put_in_currentdata('resources', resources)
else:
put_in_currentdata('errors', data)
logger.info("Get Id returned invalid status: %s", status)
if resources:
for idx, value in enumerate(resources):
if nodetype in value['type']:
db_record = copy.deepcopy(d_record)
db_record['snapshotId'] = '%s%s' % (node['masterSnapshotId'], str(idx))
db_record['path'] = value['id']
db_record['json'] = value
data_str = json.dumps(value)
db_record['checksum'] = hashlib.md5(data_str.encode('utf-8')).hexdigest()
db_records.append(db_record)
else:
logger.info('Get requires valid subscription, token and path.!')
return db_records
def container_snapshots_filesystem(container):
"""
Get snapshot and mastersnapshot list used in all test/mastertest files of a container from the filesystem.
This gets list of all the snapshots/mastersnapshots used in the container.
The list will be used to not populate the snapshots/mastersnapshots multiple times, if the same
snapshots/mastersnapshots are used in different test/mastertest files of a container.
The configuration of the default path is configured in config.ini.
"""
snapshots = []
logger.info("Starting to get list of snapshots")
reporting_path = config_value('REPORTING', 'reportOutputFolder')
json_dir = '%s/%s/%s' % (framework_dir(), reporting_path, container)
logger.info(json_dir)
singletest = get_from_currentdata(SINGLETEST)
test_files = get_json_files(json_dir, JSONTEST)
logger.info('\n'.join(test_files))
for test_file in test_files:
test_json_data = json_from_file(test_file)
if test_json_data:
snapshot = test_json_data['snapshot'] if 'snapshot' in test_json_data else ''
if snapshot:
file_name = snapshot if snapshot.endswith('.json') else '%s.json' % snapshot
if singletest:
testsets = get_field_value_with_default(test_json_data, 'testSet', [])
for testset in testsets:
for testcase in testset['cases']:
if ('testId' in testcase and testcase['testId'] == singletest) or \
('masterTestId' in testcase and testcase['masterTestId'] == singletest):
if file_name not in snapshots:
snapshots.append(file_name)
else:
snapshots.append(file_name)
test_files = get_json_files(json_dir, MASTERTEST)
logger.info('\n'.join(test_files))
for test_file in test_files:
test_json_data = json_from_file(test_file)
if test_json_data:
snapshot = test_json_data['masterSnapshot'] if 'masterSnapshot' in test_json_data else ''
if snapshot:
file_name = snapshot if snapshot.endswith('.json') else '%s.json' % snapshot
parts = file_name.split('.')
file_name = '%s_gen.%s' % (parts[0], parts[-1])
if singletest:
testsets = get_field_value_with_default(test_json_data, 'testSet', [])
for testset in testsets:
for testcase in testset['cases']:
if ('testId' in testcase and testcase['testId'] == singletest) or \
('masterTestId' in testcase and testcase['masterTestId'] == singletest):
if file_name not in snapshots:
snapshots.append(file_name)
else:
snapshots.append(file_name)
return list(set(snapshots))
def get_client_secret(key='CLIENTKEY', client_id=None):
""" Return the client secret used for the current run"""
# logger.info('before get_from_currentdata CLIENTSECRET invoked! ')
client_secret = get_from_currentdata(CLIENTSECRET)
# logger.info('after get_from_currentdata CLIENTSECRET invoked! %s', client_secret)
if not client_secret:
if 'UAMI' in os.environ and os.environ['UAMI'] == 'true':
# client_secret = get_vault_data(client_id)
vaulttoken = get_uami_vault_access_token()
keyvault = config_value('VAULT', 'keyvault')
# secret_key = config_value('VAULT', 'secret_key')
logger.info('Keyvault: %s, key:%s', keyvault, client_id)
secret_data = get_keyvault_secret(keyvault, client_id, vaulttoken)
if secret_data and 'value' in secret_data:
client_secret = secret_data['value']
else:
if not client_secret:
client_secret = os.getenv(key, None)
if not client_secret and not get_from_currentdata(CUSTOMER):
client_secret = input('Enter the client secret for the app: ')
return client_secret
def get_snaphotid_doc(self, sid):
doc = None
isdb_fetch = get_dbtests()
if isdb_fetch:
dbname = self.dbname
coll = self.collection_data[sid] if sid in self.collection_data else COLLECTION
docs = get_documents(coll, {'snapshotId': sid}, dbname,
sort=[('timestamp', pymongo.DESCENDING)], limit=1)
logger.debug('Number of Snapshot Documents: %s', len(docs))
if docs and len(docs):
doc = docs[0]['json']
snapshot = {
'id': docs[0]['snapshotId'],
'structure': docs[0]['structure'],
'reference': docs[0]['reference'],
'source': docs[0]['source'],
'collection': docs[0]['collection'],
'type': docs[0].get("node", {}).get('type'),
'region' : docs[0].get('region', "")
}
if 'paths' in docs[0]:
snapshot['paths'] = docs[0]['paths']
else:
snapshot['path'] = docs[0]['path']
self.snapshots.append(snapshot)
else:
json_dir = '%s%s' % (get_test_json_dir(), self.container)
if exists_dir(json_dir):
fname = '%s/snapshots/%s' % (json_dir, sid)
if exists_file(fname):
json_data = json_from_file(fname)
if json_data and 'json' in json_data:
doc = json_data['json']
snapshot_val = {
'id': json_data['snapshotId'],
'structure': json_data['structure'],
'reference': json_data['reference'],
'source': json_data['source'],
'collection': json_data['collection'],
'type': json_data.get("node", {}).get('type'),
'region' : json_data.get('region', "")
}
if 'paths' in json_data:
snapshot_val['paths'] = json_data['paths']
else:
snapshot_val['path'] = json_data['path']
singletest = get_from_currentdata(SINGLETEST)
if singletest:
snapshot_val['json'] = doc
self.snapshots.append(snapshot_val)
return doc
def generate_gce(google_data, project, user):
"""
Generate client secret json from the google data
"""
logger.info("Generating GCE")
gce = {
"type": get_field_value(user, "type"),
"project_id": get_field_value(project, "project-id"),
"private_key_id": get_field_value(user, "private_key_id"),
"private_key": get_field_value(user, "private_key"),
"client_email": get_field_value(user, "client_email"),
"client_id": get_field_value(user, "client_id"),
"auth_uri": get_field_value(google_data, "auth_uri"),
"token_uri": get_field_value(google_data, "token_uri"),
"auth_provider_x509_cert_url": get_field_value(google_data, "auth_provider_x509_cert_url"),
"client_x509_cert_url": get_field_value(user, "client_x509_cert_url"),
}
# Read the private key from the key path
if not gce['private_key'] and get_field_value(user, "private_key_path"):
private_key_path = get_field_value(user, "private_key_path")
logger.info("Private key path : %s ", private_key_path)
try:
gce['private_key'] = open(private_key_path, 'r', encoding="utf-8").read().replace("\\n","\n")
if gce['private_key']:
logger.info('Private key from Private key path, Secret: %s', '*' * len(gce['private_key']))
except Exception as e:
raise Exception("Private key does not exist at given private key path : %s " % str(e))
if not gce['private_key']:
raise Exception("Private key does not exist at given private key path : %s " % private_key_path)
# Read the private key from the vault
if not gce['private_key']:
private_key = get_vault_data(gce['private_key_id'])
if private_key:
gce["private_key"] = private_key.replace("\\n","\n")
if gce["private_key"]:
logger.info('Private key from vault Secret: %s', '*' * len(gce["private_key"]))
elif get_from_currentdata(CUSTOMER):
raise Exception("Private key does not set in a vault")
if not gce['private_key']:
raise Exception("No `private_key` field in the connector file to access Google resource!...")
if (None in list(gce.values())):
raise Exception("Connector file does not contains valid values or some fields are missing for access Google resources.")
return gce
def get_config_value(section, key, env_var, prompt_str=None):
""" Return the client secret used for the current run"""
client_secret = config_value(section, key)
if not client_secret and env_var:
client_secret = os.getenv(env_var, None)
if not client_secret and prompt_str:
key_str = '%s_%s' % (section, key)
client_secret = get_from_currentdata(key_str)
if not client_secret:
client_secret = input(prompt_str)
if client_secret:
put_in_currentdata(key_str, client_secret)
logger.info('Key:%s, sec:%s', key_str, client_secret)
add_to_exclude_list(key_str)
return client_secret
def get_snaphotid_doc(self, sid):
doc = None
isdb_fetch = get_dbtests()
if isdb_fetch:
dbname = self.kwargs['dbname']
coll = self.kwargs['snapshots'][sid] if sid in self.kwargs[
'snapshots'] else COLLECTION
docs = get_documents(coll, {'snapshotId': sid},
dbname,
sort=[('timestamp', pymongo.DESCENDING)],
limit=1)
logger.debug('Number of Snapshot Documents: %s', len(docs))
if docs and len(docs):
doc = docs[0]['json']
self.snapshots.append({
'id': docs[0]['snapshotId'],
'path': docs[0]['path'],
'structure': docs[0]['structure'],
'reference': docs[0]['reference'],
'source': docs[0]['source']
})
else:
json_dir = '%s%s' % (get_test_json_dir(), self.kwargs['container'])
if exists_dir(json_dir):
fname = '%s/snapshots/%s' % (json_dir, sid)
if exists_file(fname):
json_data = json_from_file(fname)
if json_data and 'json' in json_data:
doc = json_data['json']
# self.snapshots.append({
# 'id': json_data['snapshotId'],
# 'path': json_data['path'],
# 'structure': json_data['structure'],
# 'reference': json_data['reference'],
# 'source': json_data['source']
# })
snapshot_val = {
'id': json_data['snapshotId'],
'path': json_data['path'],
'structure': json_data['structure'],
'reference': json_data['reference'],
'source': json_data['source']
}
singletest = get_from_currentdata(SINGLETEST)
if singletest:
snapshot_val['json'] = doc
self.snapshots.append(snapshot_val)
return doc
def run_file_validation_tests(test_file, container, filesystem=True, snapshot_status=None):
# logger.info("*" * 50)
logger.info("\tTEST: %s", test_file)
dirpath = None
test_json_data = json_from_file(test_file)
if not test_json_data:
logger.info("\t\tTest file %s looks to be empty, next!...", test_file)
if test_json_data and "connector" in test_json_data and "remoteFile" in test_json_data and test_json_data["connector"] and test_json_data["remoteFile"]:
dirpath, pull_response = pull_json_data(test_json_data)
if not pull_response:
return {}
singletest = get_from_currentdata(SINGLETEST)
if singletest:
testsets = get_field_value_with_default(test_json_data, 'testSet', [])
for testset in testsets:
newtestcases = []
for testcase in testset['cases']:
if ('testId' in testcase and testcase['testId'] == singletest) or \
('masterTestId' in testcase and testcase['masterTestId'] == singletest):
newtestcases.append(testcase)
testset['cases'] = newtestcases
resultset = run_json_validation_tests(test_json_data, container, filesystem, snapshot_status, dirpath=dirpath)
finalresult = True
if resultset:
snapshot = test_json_data['snapshot'] if 'snapshot' in test_json_data else ''
if singletest:
print(json.dumps(resultset, indent=2))
else:
dump_output_results(resultset, container, test_file, snapshot, filesystem)
for result in resultset:
if 'result' in result:
if not re.match(r'passed', result['result'], re.I):
finalresult = False
break
else:
# TODO: NO test cases in this file.
# LOG HERE that no test cases are present in this file.
finalresult = False
return finalresult
def get_access_token():
"""
Get the access token if stored in rundata, otherwise get the token from
management.azure.com portal for the webapp.
"""
token = get_from_currentdata(ACCESSTOKEN)
if not token:
tenant_id = get_tenant_id()
client_id = get_client_id()
if client_id:
# client_secret = get_client_secret()
client_secret = get_client_secret(key='CLIENTKEY',
client_id=client_id)
else:
logger.info('client Id required for REST API access!')
return None
data = {
'grant_type': 'client_credentials',
'client_id': client_id,
'client_secret': client_secret,
'resource': 'https://management.azure.com'
}
hdrs = {'Cache-Control': "no-cache", "Accept": "application/json"}
if tenant_id:
url = 'https://login.microsoftonline.com/%s/oauth2/token' % tenant_id
logger.info('Get Azure token REST API invoked!')
status, data = http_post_request(url,
data,
headers=hdrs,
json_type=True)
if status and isinstance(status, int) and status == 200:
token = data['access_token']
put_in_currentdata(ACCESSTOKEN, token)
else:
put_in_currentdata('errors', data)
logger.info("Get Azure token returned invalid status: %s",
status)
return token
def get_custom_data(snapshot_source, tabs=2):
sub_data = {}
if json_source():
container = get_from_currentdata('container')
dbname = config_value(DATABASE, DBNAME)
collection = config_value(DATABASE, collectiontypes[STRUCTURE])
parts = snapshot_source.split('.')
qry = {'name': parts[0], 'container' : container }
sort = [sort_field('timestamp', False)]
docs = get_documents(collection, dbname=dbname, sort=sort, query=qry, limit=1)
logger.info('Number of Custom Documents: %d', len(docs))
if docs and len(docs):
sub_data = docs[0]['json']
else:
json_test_dir = get_test_json_dir()
file_name = '%s.json' % snapshot_source if snapshot_source and not \
snapshot_source.endswith('.json') else snapshot_source
custom_source = '%s/../%s' % (json_test_dir, file_name)
logger.info('\t\tCUSTOM CONNECTOR: %s ', custom_source)
# logger.info('Custom source: %s', custom_source)
if exists_file(custom_source):
sub_data = json_from_file(custom_source)
return sub_data
def get_tenant_id():
""" Return the tenant_id"""
return get_from_currentdata(TENANT)
def get_git_pwd(key='GIT_PWD'):
""" Return the git password for https connection"""
git_pwd = get_from_currentdata('GIT_PWD')
if not git_pwd:
git_pwd = os.getenv(key, None)
return git_pwd
def get_subscription_id():
""" Return the subscription Id used for the current run"""
return get_from_currentdata(SUBSCRIPTION)
def run_container_validation_tests_filesystem(container, snapshot_status=None):
"""Get test files from the filesystem."""
logger.info("Starting validation tests")
reporting_path = config_value('REPORTING', 'reportOutputFolder')
json_dir = '%s/%s/%s' % (framework_dir(), reporting_path, container)
logger.info(json_dir)
test_files = get_json_files(json_dir, JSONTEST)
logger.info('\n'.join(test_files))
result = True
for test_file in test_files:
val = run_file_validation_tests(test_file, container, True,
snapshot_status)
result = result and val
# mastertest files
test_files = get_json_files(json_dir, MASTERTEST)
logger.info('\n'.join(test_files))
finalresult = True
for test_file in test_files:
logger.info("*" * 50)
logger.info("validator tests: %s", test_file)
test_json_data = json_from_file(test_file)
if not test_json_data:
logger.info("Test file %s looks to be empty, next!...", test_file)
continue
snapshot_key = '%s_gen' % test_json_data['masterSnapshot']
mastersnapshots = defaultdict(list)
snapshot_data = snapshot_status[
snapshot_key] if snapshot_key in snapshot_status else {}
for snapshot_id, mastersnapshot_id in snapshot_data.items():
if isinstance(mastersnapshot_id, list):
for master_snapshot_id in mastersnapshot_id:
mastersnapshots[master_snapshot_id].append(snapshot_id)
elif isinstance(mastersnapshot_id, str):
mastersnapshots[mastersnapshot_id].append(snapshot_id)
test_json_data['snapshot'] = snapshot_key
testsets = get_field_value_with_default(test_json_data, 'testSet', [])
for testset in testsets:
testcases = get_field_value_with_default(testset, 'cases', [])
testset['cases'] = _get_new_testcases(testcases, mastersnapshots)
# print(json.dumps(test_json_data, indent=2))
singletest = get_from_currentdata(SINGLETEST)
if singletest:
for testset in testsets:
newtestcases = []
for testcase in testset['cases']:
if ('testId' in testcase and testcase['testId'] == singletest) or \
('masterTestId' in testcase and testcase['masterTestId'] == singletest):
newtestcases.append(testcase)
testset['cases'] = newtestcases
resultset = run_json_validation_tests(test_json_data, container, False,
snapshot_status)
if resultset:
snapshot = test_json_data[
'snapshot'] if 'snapshot' in test_json_data else ''
if singletest:
print(json.dumps(resultset, indent=2))
else:
dump_output_results(resultset, container, test_file, snapshot,
True)
for result in resultset:
if 'result' in result:
if not re.match(r'passed', result['result'], re.I):
finalresult = False
break
else:
logger.info('No mastertest Documents found!')
finalresult = False
return finalresult
def populate_aws_snapshot(snapshot, container=None):
"""
This is an entrypoint for populating a snapshot of type aws.
All snapshot connectors should take snapshot object and based on
'source' field create a method to connect to the service for the
connector.
The 'source' field could be used by more than one snapshot, so the
'testuser' attribute should match to the user the 'source'
"""
dbname = config_value('MONGODB', 'dbname')
snapshot_source = get_field_value(snapshot, 'source')
snapshot_user = get_field_value(snapshot, 'testUser')
account_id = get_field_value(snapshot, 'accountId')
sub_data = get_aws_data(snapshot_source)
snapshot_nodes = get_field_value(snapshot, 'nodes')
snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes)
# valid_snapshotids = True
# if snapshot_nodes:
# for node in snapshot_nodes:
# snapshot_data[node['snapshotId']] = False
# if not isinstance(node['snapshotId'], str):
# valid_snapshotids = False
# if not valid_snapshotids:
# logger.error('All snap')
if valid_snapshotids and sub_data and snapshot_nodes:
logger.debug(sub_data)
access_key, secret_access, region, connector_client_str = \
get_aws_client_data(sub_data, snapshot_user, account_id)
if not access_key:
logger.info(
"No access_key in the snapshot to access aws resource!...")
raise Exception(
"No access_key in the snapshot to access aws resource!...")
# return snapshot_data
# Read the client secrets from envirnment variable
if not secret_access:
secret_access = os.getenv(snapshot_user, None)
if secret_access:
logger.info(
'Secret Access key from environment variable, Secret: %s',
'*' * len(secret_access))
# Read the client secrets from the vault
if not secret_access:
secret_access = get_vault_data(access_key)
if secret_access:
logger.info('Secret Access key from vault Secret: %s',
'*' * len(secret_access))
elif get_from_currentdata(CUSTOMER):
logger.error("Secret Access key does not set in a vault")
raise Exception("Secret Access key does not set in a vault")
if not secret_access:
raise Exception(
"No `secret-access` key in the connector file to access aws resource!..."
)
if access_key and secret_access:
# existing_aws_client = {}
for node in snapshot['nodes']:
validate = node['validate'] if 'validate' in node else True
mastercode = False
if 'snapshotId' in node and validate:
client_str, aws_region = _get_aws_client_data_from_node(
node,
default_client=connector_client_str,
default_region=region)
if not _validate_client_name(client_str):
logger.error("Invalid Client Name")
return snapshot_data
try:
awsclient = client(client_str.lower(),
aws_access_key_id=access_key,
aws_secret_access_key=secret_access,
region_name=aws_region)
except Exception as ex:
logger.info('Unable to create AWS client: %s', ex)
awsclient = None
logger.info(awsclient)
if awsclient:
data = get_node(awsclient, node, snapshot_source,
snapshot)
if data:
error_str = data.pop('error', None)
if get_dbtests():
if get_collection_size(
data['collection']) == 0:
#Creating indexes for collection
create_indexes(
data['collection'],
config_value(DATABASE, DBNAME),
[('snapshotId', pymongo.ASCENDING),
('timestamp', pymongo.DESCENDING)])
create_indexes(
data['collection'],
config_value(DATABASE, DBNAME),
[('_id', pymongo.DESCENDING),
('timestamp', pymongo.DESCENDING),
('snapshotId', pymongo.ASCENDING)])
check_key = is_check_keys_required(data)
insert_one_document(data, data['collection'],
dbname, check_key)
else:
snapshot_dir = make_snapshots_dir(container)
if snapshot_dir:
store_snapshot(snapshot_dir, data)
if 'masterSnapshotId' in node:
snapshot_data[node['snapshotId']] = node[
'masterSnapshotId']
else:
snapshot_data[node[
'snapshotId']] = False if error_str else True
else:
node['status'] = 'inactive'
elif 'masterSnapshotId' in node:
mastercode = True
client_str, aws_region = _get_aws_client_data_from_node(
node,
default_client=connector_client_str,
default_region=region)
if not _validate_client_name(client_str):
logger.error("Invalid Client Name")
return snapshot_data
if aws_region:
all_regions = [aws_region]
else:
all_regions = Session().get_available_regions(
client_str.lower())
if client_str.lower() in ['s3', 'cloudtrail']:
all_regions = ['us-west-1']
logger.info("Length of all regions is %s" %
(str(len(all_regions))))
count = 0
snapshot_data[node['masterSnapshotId']] = []
for each_region in all_regions:
logger.info(each_region)
try:
awsclient = client(
client_str.lower(),
aws_access_key_id=access_key,
aws_secret_access_key=secret_access,
region_name=each_region)
except Exception as ex:
logger.info('Unable to create AWS client: %s', ex)
logger.info(awsclient)
if awsclient:
all_data = get_all_nodes(awsclient, node, snapshot,
sub_data)
if all_data:
for data in all_data:
snapshot_data[
node['masterSnapshotId']].append({
'snapshotId':
'%s%s' % (node['masterSnapshotId'],
str(count)),
'validate':
validate,
'detailMethods':
data['detailMethods'],
'structure':
'aws',
'masterSnapshotId':
node['masterSnapshotId'],
'collection':
data['collection'],
'arn':
data['arn'],
'status':
'active'
})
count += 1
if mastercode:
snapshot_data = eliminate_duplicate_snapshots(snapshot_data)
return snapshot_data
def run_container_validation_tests_filesystem(container, snapshot_status=None):
"""Get test files from the filesystem."""
# logger.info("Starting validation tests")
logger.info("VALIDATION:")
logger.info("\tCollection: %s, Type: FILESYSTEM", container)
reporting_path = config_value('REPORTING', 'reportOutputFolder')
json_dir = '%s/%s/%s' % (framework_dir(), reporting_path, container)
logger.info('\tLOCATION: %s', json_dir)
test_files = get_json_files(json_dir, JSONTEST)
# logger.info('\n'.join(test_files))
result = True
for test_file in test_files:
logger.info('\tCOLLECTION: %s', test_file)
val = run_file_validation_tests(test_file, container, True, snapshot_status)
result = result and val
if test_files:
# return the result value if "test" file is processed collection
logger.critical("VALIDATION COMPLETE:")
return result
# mastertest files
test_files = get_json_files(json_dir, MASTERTEST)
# logger.info('\n'.join(test_files))
if not test_files:
logger.error("ERROR: No `test` or `mastertest` file found. collection should contain either `test` or `mastertest` file")
return False
finalresult = result
for test_file in test_files:
logger.info('\tCOLLECTION: %s', test_file)
# logger.info("*" * 50)
# logger.info("validator tests: %s", test_file)
dirpath = None
test_json_data = json_from_file(test_file)
if not test_json_data:
logger.info("Test file %s looks to be empty, next!...", test_file)
continue
if "connector" in test_json_data and "remoteFile" in test_json_data and test_json_data["connector"] and test_json_data["remoteFile"]:
dirpath, pull_response = pull_json_data(test_json_data)
if not pull_response:
return {}
snapshot_key = '%s_gen' % test_json_data['masterSnapshot']
mastersnapshots = defaultdict(list)
snapshot_data = snapshot_status[snapshot_key] if snapshot_key in snapshot_status else {}
for snapshot_id, mastersnapshot_id in snapshot_data.items():
if isinstance(mastersnapshot_id, list):
for master_snapshot_id in mastersnapshot_id:
mastersnapshots[master_snapshot_id].append(snapshot_id)
elif isinstance(mastersnapshot_id, str):
mastersnapshots[mastersnapshot_id].append(snapshot_id)
if not mastersnapshots:
logger.error("No generated snapshots found for validation.")
continue
test_json_data['snapshot'] = snapshot_key
testsets = get_field_value_with_default(test_json_data, 'testSet', [])
for testset in testsets:
testcases = get_field_value_with_default(testset, 'cases', [])
testset['cases'] = _get_new_testcases(testcases, mastersnapshots)
# print(json.dumps(test_json_data, indent=2))
singletest = get_from_currentdata(SINGLETEST)
if singletest:
for testset in testsets:
newtestcases = []
for testcase in testset['cases']:
if ('testId' in testcase and testcase['testId'] == singletest) or \
('masterTestId' in testcase and testcase['masterTestId'] == singletest):
newtestcases.append(testcase)
testset['cases'] = newtestcases
resultset = run_json_validation_tests(test_json_data, container, True, snapshot_status, dirpath=dirpath)
if test_json_data.get('testSet') and not resultset:
logger.error('\tERROR: Testset does not contains any testcases or all testcases are skipped due to invalid rules.')
elif resultset:
snapshot = test_json_data['snapshot'] if 'snapshot' in test_json_data else ''
if singletest:
print(json.dumps(resultset, indent=2))
else:
dump_output_results(resultset, container, test_file, snapshot, True)
for result in resultset:
if 'result' in result:
if not re.match(r'passed', result['result'], re.I):
finalresult = False
break
else:
logger.error('\tERROR: No mastertest Documents found!')
finalresult = False
logger.critical("VALIDATION COMPLETE:")
return finalresult
def json_source():
"""Return the json source, file system or mongo """
val = get_from_currentdata(JSONSOURCE)
return val if val else False
def get_resource_group():
""" Return the resource group"""
return get_from_currentdata(RESOURCEGROUP)
def populate_azure_snapshot(snapshot, container=None, snapshot_type='azure'):
""" Populates the resources from azure."""
dbname = config_value('MONGODB', 'dbname')
snapshot_source = get_field_value(snapshot, 'source')
snapshot_user = get_field_value(snapshot, 'testUser')
snapshot_nodes = get_field_value(snapshot, 'nodes')
snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes)
client_id, client_secret, sub_name, sub_id, tenant_id = \
get_web_client_data(snapshot_type, snapshot_source, snapshot_user)
if not client_id:
# logger.info("No client_id in the snapshot to access azure resource!...")
raise Exception("No client id in the snapshot to access azure resource!...")
# Read the client secrets from envirnment variable
if not client_secret:
client_secret = os.getenv(snapshot_user, None)
if client_secret:
logger.info('Client Secret from environment variable, Secret: %s', '*' * len(client_secret))
# Read the client secrets from the vault
if not client_secret:
client_secret = get_vault_data(client_id)
if client_secret:
logger.info('Client Secret from Vault, Secret: %s', '*' * len(client_secret))
elif get_from_currentdata(CUSTOMER):
logger.error("Client Secret key does not set in a vault")
raise Exception("Client Secret key does not set in a vault")
if not client_secret:
raise Exception("No `client_secret` key in the connector file to access azure resource!...")
logger.info('\t\tSubscription: %s', sub_id)
logger.info('\t\tTenant: %s', tenant_id)
logger.info('\t\tclient: %s', client_id)
put_in_currentdata('clientId', client_id)
put_in_currentdata('clientSecret', client_secret)
put_in_currentdata('subscriptionId', sub_id)
put_in_currentdata('tenant_id', tenant_id)
token = get_access_token()
logger.debug('TOKEN: %s', token)
if not token:
logger.info("Unable to get access token, will not run tests....")
raise Exception("Unable to get access token, will not run tests....")
# return {}
# snapshot_nodes = get_field_value(snapshot, 'nodes')
# snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes)
if valid_snapshotids and token and snapshot_nodes:
for node in snapshot_nodes:
validate = node['validate'] if 'validate' in node else True
if 'path' in node:
data = get_node(token, sub_name, sub_id, node, snapshot_user, snapshot_source)
if data:
if validate:
if get_dbtests():
if get_collection_size(data['collection']) == 0:
# Creating indexes for collection
create_indexes(
data['collection'],
config_value(DATABASE, DBNAME),
[
('snapshotId', pymongo.ASCENDING),
('timestamp', pymongo.DESCENDING)
]
)
create_indexes(
data['collection'],
config_value(DATABASE, DBNAME),
[
('_id', pymongo.DESCENDING),
('timestamp', pymongo.DESCENDING),
('snapshotId', pymongo.ASCENDING)
]
)
insert_one_document(data, data['collection'], dbname, check_keys=False)
else:
snapshot_dir = make_snapshots_dir(container)
if snapshot_dir:
store_snapshot(snapshot_dir, data)
if 'masterSnapshotId' in node:
snapshot_data[node['snapshotId']] = node['masterSnapshotId']
else:
snapshot_data[node['snapshotId']] = True
# else:
# snapshot_data[node['snapshotId']] = False
node['status'] = 'active'
else:
# TODO alert if notification enabled or summary for inactive.
node['status'] = 'inactive'
logger.debug('Type: %s', type(data))
else:
alldata = get_all_nodes(
token, sub_name, sub_id, node, snapshot_user, snapshot_source)
if alldata:
snapshot_data[node['masterSnapshotId']] = []
for data in alldata:
# insert_one_document(data, data['collection'], dbname)
found_old_record = False
for masterSnapshotId, snapshot_list in snapshot_data.items():
old_record = None
if isinstance(snapshot_list, list):
for item in snapshot_list:
if item["path"] == data['path']:
old_record = item
if old_record:
found_old_record = True
if node['masterSnapshotId'] not in old_record['masterSnapshotId']:
old_record['masterSnapshotId'].append(
node['masterSnapshotId'])
if not found_old_record:
snapshot_data[node['masterSnapshotId']].append(
{
'masterSnapshotId': [node['masterSnapshotId']],
'snapshotId': data['snapshotId'],
'path': data['path'],
'validate': validate,
'status': 'active'
})
# snapshot_data[node['masterSnapshotId']] = True
logger.debug('Type: %s', type(alldata))
delete_from_currentdata('resources')
delete_from_currentdata('clientId')
delete_from_currentdata('client_secret')
delete_from_currentdata('subscriptionId')
delete_from_currentdata('tenant_id')
delete_from_currentdata('token')
return snapshot_data
def git_clone_dir(connector):
clonedir = None
repopath = tempfile.mkdtemp()
subdir = False
if connector and isinstance(connector, dict):
giturl = get_field_value(connector, 'gitProvider')
if not giturl:
logger.error("Git connector does not have valid git provider URL")
return repopath, clonedir
branch = get_from_currentdata('branch')
if not branch:
branch = get_field_value_with_default(connector, 'branchName', 'master')
isprivate = get_field_value(connector, 'private')
isprivate = True if isprivate is None or not isinstance(isprivate, bool) else isprivate
# logger.info("Repopath: %s", repopath)
logger.info("\t\t\tRepopath: %s", repopath)
http_match = re.match(r'^http(s)?://', giturl, re.I)
if http_match:
logger.info("\t\t\tHttp (private:%s) giturl: %s", "YES" if isprivate else "NO", giturl)
accessToken = get_field_value(connector, 'httpsAccessToken')
username = get_field_value(connector, 'httpsUser')
if accessToken:
logger.info("AccessToken: %s" % accessToken)
pwd = get_field_value(connector, 'httpsPassword')
pwd = pwd if pwd else get_git_pwd(key=accessToken)
if not pwd:
pwd = get_pwd_from_vault(accessToken)
if pwd:
logger.info("Git access token from vault: %s", '*' * len(pwd))
if pwd:
gh = GithubFunctions()
gh.set_access_token(pwd)
_ = gh.populate_user()
rpo = gh.clone_repo(giturl, repopath, branch)
if rpo:
logger.info('Successfully cloned in %s dir' % repopath)
checkdir = '%s/tmpclone' % repopath if subdir else repopath
clonedir = checkdir if exists_dir('%s/.git' % checkdir) else None
if not exists_dir(clonedir):
logger.error("No valid data provided for connect to git : %s", giturl)
return repopath, clonedir
elif isprivate:
logger.error("Please provide password for connect to git repository.")
return repopath, clonedir
else:
git_cmd = 'git clone %s %s' % (giturl, repopath)
elif username:
pwd = get_field_value(connector, 'httpsPassword')
schema = giturl[:http_match.span()[-1]]
other_part = giturl[http_match.span()[-1]:]
# pwd = pwd if (pwd and not json_source()) else (get_git_pwd() if not json_source() else get_pwd_from_vault(pwd))
pwd = pwd if pwd else get_git_pwd(key=username)
# populate the password from vault
if not pwd:
pwd = get_pwd_from_vault(username)
if pwd:
logger.info("Git password from vault: %s", '*' * len(pwd))
if pwd:
git_cmd = 'git clone --depth 1 %s%s:%s@%s %s' % (schema, urllib.parse.quote_plus(username),
urllib.parse.quote_plus(pwd), other_part, repopath)
elif isprivate:
logger.error("Please provide password for connect to git repository.")
return repopath, clonedir
else:
git_cmd = 'git clone --depth 1 %s%s:%s@%s %s' % (schema, urllib.parse.quote_plus(username), "",
other_part, repopath)
else:
git_cmd = 'git clone --depth 1 %s %s' % (giturl, repopath)
else:
logger.info("SSH (private:%s) giturl: %s, Repopath: %s", "YES" if isprivate else "NO",
giturl, repopath)
if isprivate:
ssh_key_file = get_field_value(connector, 'sshKeyfile')
ssh_key_name = get_field_value(connector, 'sshKeyName')
ssh_key_file_data = None
if ssh_key_file:
if not exists_file(ssh_key_file):
logger.error("Git connector points to a non-existent ssh keyfile!")
return repopath, clonedir
elif ssh_key_name:
ssh_key_file_data = get_vault_data(ssh_key_name)
if not ssh_key_file_data:
logger.info('Git connector points to a non-existent ssh keyName in the vault!')
return repopath, clonedir
ssh_host = get_field_value(connector, 'sshHost')
ssh_user = get_field_value_with_default(connector, 'sshUser', 'git')
if not ssh_host:
logger.error("SSH host not set, could be like github.com, gitlab.com, 192.168.1.45 etc")
return repopath, clonedir
ssh_dir = '%s/.ssh' % repopath
if exists_dir(ssh_dir):
logger.error("Git ssh dir: %s already exists, cannot recreate it!", ssh_dir)
return repopath, clonedir
os.mkdir('%s/.ssh' % repopath, 0o700)
if not ssh_key_file and ssh_key_name and ssh_key_file_data:
ssh_key_file = create_ssh_file_vault_data(ssh_dir, ssh_key_file_data, ssh_key_name)
if not ssh_key_file:
logger.info('Git connector points to a non-existent ssh keyName in the vault!')
return repopath, clonedir
ssh_cfg = create_ssh_config(ssh_dir, ssh_key_file, ssh_user)
if not ssh_cfg:
logger.error("Creation of Git ssh config in dir: %s failed!", ssh_dir)
return repopath, clonedir
git_ssh_cmd = 'ssh -o "StrictHostKeyChecking=no" -F %s' % ssh_cfg
git_cmd = 'git clone %s %s/tmpclone' % (giturl, repopath)
subdir = True
else:
git_ssh_cmd = 'ssh -o "StrictHostKeyChecking=no"'
git_cmd = 'git clone %s %s' % (giturl, repopath)
os.environ['GIT_SSH_COMMAND'] = git_ssh_cmd
git_cmd = '%s --branch %s' % (git_cmd, branch)
if git_cmd:
error_result, result = run_subprocess_cmd(git_cmd)
checkdir = '%s/tmpclone' % repopath if subdir else repopath
clonedir = checkdir if exists_dir('%s/.git' % checkdir) else None
if not exists_dir(clonedir):
logger.error("No valid data provided for connect to git : %s", error_result)
if 'GIT_SSH_COMMAND' in os.environ:
os.environ.pop('GIT_SSH_COMMAND')
return repopath, clonedir
def populate_custom_snapshot(snapshot, container=None):
""" Populates the resources from git."""
dbname = config_value('MONGODB', 'dbname')
snapshot_source = get_field_value(snapshot, 'source')
connector_data = get_from_currentdata('connector')
if connector_data:
sub_data = get_custom_data(connector_data)
if not sub_data:
logger.error("No connector data found in '%s'", connector_data)
else:
sub_data = get_custom_data(snapshot_source)
snapshot_nodes = get_field_value(snapshot, 'nodes')
snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes)
if valid_snapshotids and sub_data and snapshot_nodes:
baserepo, repopath = _get_repo_path(sub_data, snapshot)
if repopath:
brnch = get_field_value_with_default(sub_data, 'branchName', 'master')
for node in snapshot_nodes:
node_type = node['type'] if 'type' in node and node['type'] else ''
if node_type in TEMPLATE_NODE_TYPES:
template_data = {
"container" : container,
"dbname" : dbname,
"snapshot_source" : snapshot_source,
"connector_data" : sub_data,
"snapshot_data" : snapshot_data,
"repopath" : repopath,
"snapshot" : snapshot
}
template_processor = TEMPLATE_NODE_TYPES[node_type](node, **template_data)
if 'snapshotId' in node:
snapshot_data = template_processor.populate_template_snapshot()
elif 'masterSnapshotId' in node:
snapshot_data = template_processor.populate_all_template_snapshot()
elif 'paths' in node:
logger.error("ERROR: Invalid json : `%s` is not a valid node type." % (node_type))
else:
# logger.debug(node)
# data = get_node(repopath, node, snapshot_source, brnch)
# if data:
# insert_one_document(data, data['collection'], dbname)
# snapshot_data[node['snapshotId']] = True
validate = node['validate'] if 'validate' in node else True
if 'snapshotId' in node:
logger.debug(node)
data = get_node(repopath, node, snapshot, brnch, sub_data)
if data:
if validate:
if get_dbtests():
if get_collection_size(data['collection']) == 0:
#Creating indexes for collection
create_indexes(data['collection'],
config_value(DATABASE, DBNAME),
[('snapshotId', pymongo.ASCENDING),
('timestamp', pymongo.DESCENDING)])
create_indexes(
data['collection'],
config_value(DATABASE, DBNAME),
[
('_id', pymongo.DESCENDING),
('timestamp', pymongo.DESCENDING),
('snapshotId', pymongo.ASCENDING)
]
)
insert_one_document(data, data['collection'], dbname)
else:
snapshot_dir = make_snapshots_dir(container)
if snapshot_dir:
store_snapshot(snapshot_dir, data)
if 'masterSnapshotId' in node:
snapshot_data[node['snapshotId']] = node['masterSnapshotId']
else:
snapshot_data[node['snapshotId']] = True
# else:
# snapshot_data[node['snapshotId']] = False
node['status'] = 'active'
else:
node['status'] = 'inactive'
logger.debug('Type: %s', type(data))
elif 'masterSnapshotId' in node:
alldata = get_all_nodes(repopath, node, snapshot, brnch, sub_data)
if alldata:
snapshot_data[node['masterSnapshotId']] = []
for data in alldata:
snapshot_data[node['masterSnapshotId']].append(
{
'snapshotId': data['snapshotId'],
'path': data['path'],
'validate': validate
})
logger.debug('Type: %s', type(alldata))
if baserepo and os.path.exists(baserepo):
# logger.info('\t\tCLEANING Repo: %s', baserepo)
shutil.rmtree(baserepo)
return snapshot_data
def get_client_id():
""" Return the client Id used for the current run"""
return get_from_currentdata(CLIENTID)
