def _get_new_testcases(testcases, mastersnapshots): newcases = [] for testcase in testcases: test_parser_type = testcase.get('type', None) if test_parser_type == 'rego': new_cases = _get_rego_testcase(testcase, mastersnapshots) newcases.extend(new_cases) else: rule_str = get_field_value_with_default(testcase, 'rule', '') ms_ids = re.findall(r'\{(.*)\}', rule_str) # detail_method = get_field_value(testcase, 'detailMethod') for ms_id in ms_ids: for s_id in mastersnapshots[ms_id]: # new_rule_str = re.sub('{%s}' % ms_id, '{%s}' % s_id, rule_str) # if not detail_method or detail_method == snapshots_details_map[s_id]: new_rule_str = rule_str.replace('{%s}' % ms_id, '{%s}' % s_id) new_testcase = { 'title': testcase.get('title') if testcase.get('title') else "", 'description': testcase.get('description') if testcase.get('description') else "", 'rule': new_rule_str, 'testId': testcase['masterTestId'], 'status' : get_field_value_with_default(testcase, 'status', "enable") } newcases.append(new_testcase) return newcases
def get_web_client_data(snapshot_type, snapshot_source, snapshot_user): client_id = None client_secret = None sub_id = None sub_name = None tenant_id = None found = False if snapshot_type == 'azure': sub_data = get_azure_data(snapshot_source) if sub_data: accounts = get_field_value_with_default(sub_data, 'accounts', []) for account in accounts: subscriptions = get_field_value_with_default(account, 'subscription', []) for subscription in subscriptions: users = get_field_value_with_default(subscription, 'users', []) if users: for user in users: name = get_field_value(user, 'name') if name and name == snapshot_user: client_id = get_field_value(user, 'client_id') client_secret = get_field_value(user, 'client_secret') sub_id = get_field_value(subscription, 'subscription_id') sub_name = get_field_value(subscription, 'subscription_name') tenant_id = get_field_value(sub_data, 'tenant_id') found = True if found: break if found: break if found: break if not found: logger.error("No connector data found, check the connector configuration and snapshot configuration files.") return client_id, client_secret, sub_name, sub_id, tenant_id
def get_node(credentials, node, snapshot_source, snapshot): """ Fetch node from google using connection. In this case using google client API's functions. """ collection = node['collection'] if 'collection' in node else COLLECTION parts = snapshot_source.split('.') project_id = get_field_value_with_default(snapshot, 'project-id',"") path = get_field_value_with_default(node, 'path',"") zone = re.findall(r"(?<=zones\/)[a-zA-Z0-9\-]*(?=\/)", path) db_record = { "structure": "google", "error": None, "reference": project_id, "source": parts[0], "path": path, "timestamp": int(time.time() * 1000), "queryuser": get_field_value(snapshot, 'testUser'), "checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(), "node": node, "snapshotId": node['snapshotId'], "collection": collection.replace('.', '').lower(), "region" : zone[0] if zone else "", "json": {} # Refactor when node is absent it should None, when empty object put it as {} } try: access_token = credentials.get_access_token().access_token header = { "Authorization" : ("Bearer %s" % access_token) } node_type = node['type'] if node and 'type' in node else "" base_node_type_list = node_type.split("/") if len(base_node_type_list) > 1: base_node_type = base_node_type_list[0] else: logger.error("Invalid node type %s", node_type) return db_record base_url = "%s%s" % (base_node_type, ".googleapis.com") request_url = "https://%s/%s" % (base_url, path) logger.info("Invoke request for get snapshot: %s", request_url) status, data = http_get_request(request_url, header) logger.info('Get snapshot status: %s', status) if status and isinstance(status, int) and status == 200: if data: db_record['json'] = data checksum = get_checksum(data) if checksum: db_record['checksum'] = checksum else: logger.error("Get snapshot returned invalid status: %s", status) db_record['error'] = ("Get snapshot returned invalid status: %s" % status) except Exception as ex: logger.error('Failed to populate the snapshot : %s', ex) db_record['error'] = 'Failed to populate the snapshot: %s' % ex return db_record
def get_node(compute_fn, node, snapshot_source, snapshot): """ Fetch node from google using connection. In this case using google client API's functions. """ collection = node['collection'] if 'collection' in node else COLLECTION parts = snapshot_source.split('.') project_id = get_field_value_with_default(snapshot, 'project-id', "") path = get_field_value_with_default(node, 'path', "") db_record = { "structure": "google", "error": None, "reference": project_id, "source": parts[0], "path": path, "timestamp": int(time.time() * 1000), "queryuser": "", "checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(), "node": node, "snapshotId": node['snapshotId'], "collection": collection.replace('.', '').lower(), "json": { } # Refactor when node is absent it should None, when empty object put it as {} } fn_str_list, kwargs = get_google_call_function(node) if fn_str_list: for i in range(0, len(fn_str_list)): compute_fn = getattr(compute_fn, fn_str_list[i], None) if compute_fn and i != len(fn_str_list) - 1: compute_fn = compute_fn() response_param = "" if fn_str_list and len(fn_str_list) > 1: response_param = fn_str_list[-2] elif fn_str_list and len(fn_str_list) == 1: response_param = fn_str_list[0] if compute_fn and callable(compute_fn): try: data = compute_fn(**kwargs["params"]).execute() if data: db_record['json'] = data checksum = get_checksum(data) if checksum: db_record['checksum'] = checksum except Exception as ex: logger.info('Compute function exception: %s', ex) db_record['error'] = 'Compute function exception: %s' % ex else: logger.info('Invalid Compute function exception: %s', str(fn_str_list)) db_record[ 'error'] = 'Invalid Compute function exception: %s' % str( fn_str_list) else: logger.info('Missing Compute function') db_record['error'] = 'Missing Compute function' return db_record
def container_snapshots_filesystem(container): """ Get snapshot and mastersnapshot list used in all test/mastertest files of a container from the filesystem. This gets list of all the snapshots/mastersnapshots used in the container. The list will be used to not populate the snapshots/mastersnapshots multiple times, if the same snapshots/mastersnapshots are used in different test/mastertest files of a container. The configuration of the default path is configured in config.ini. """ snapshots = [] logger.info("Starting to get list of snapshots") reporting_path = config_value('REPORTING', 'reportOutputFolder') json_dir = '%s/%s/%s' % (framework_dir(), reporting_path, container) logger.info(json_dir) singletest = get_from_currentdata(SINGLETEST) test_files = get_json_files(json_dir, JSONTEST) logger.info('\n'.join(test_files)) for test_file in test_files: test_json_data = json_from_file(test_file) if test_json_data: snapshot = test_json_data['snapshot'] if 'snapshot' in test_json_data else '' if snapshot: file_name = snapshot if snapshot.endswith('.json') else '%s.json' % snapshot if singletest: testsets = get_field_value_with_default(test_json_data, 'testSet', []) for testset in testsets: for testcase in testset['cases']: if ('testId' in testcase and testcase['testId'] == singletest) or \ ('masterTestId' in testcase and testcase['masterTestId'] == singletest): if file_name not in snapshots: snapshots.append(file_name) else: snapshots.append(file_name) test_files = get_json_files(json_dir, MASTERTEST) logger.info('\n'.join(test_files)) for test_file in test_files: test_json_data = json_from_file(test_file) if test_json_data: snapshot = test_json_data['masterSnapshot'] if 'masterSnapshot' in test_json_data else '' if snapshot: file_name = snapshot if snapshot.endswith('.json') else '%s.json' % snapshot parts = file_name.split('.') file_name = '%s_gen.%s' % (parts[0], parts[-1]) if singletest: testsets = get_field_value_with_default(test_json_data, 'testSet', []) for testset in testsets: for testcase in testset['cases']: if ('testId' in testcase and testcase['testId'] == singletest) or \ ('masterTestId' in testcase and testcase['masterTestId'] == singletest): if file_name not in snapshots: snapshots.append(file_name) else: snapshots.append(file_name) return list(set(snapshots))
def process_files(self, test_files, doctype, replace=False): """ Process Test or masterTest json files.""" snapshots = [] for test_file in test_files: test_json_data = json_from_file(test_file) if test_json_data: snapshot = test_json_data[ doctype] if doctype in test_json_data else '' if snapshot: file_name = snapshot if snapshot.endswith( '.json') else '%s.json' % snapshot if replace: file_name = file_name.replace('.json', '_gen.json') if self.singleTest: testsets = get_field_value_with_default( test_json_data, 'testSet', []) for testset in testsets: for testcase in testset['cases']: if ('testId' in testcase and testcase['testId'] == self.singleTest) or \ ('masterTestId' in testcase and testcase['masterTestId'] == self.singleTest): if file_name not in snapshots: snapshots.append(file_name) else: snapshots.append(file_name) return snapshots
def get_all_nodes(repopath, node, snapshot, ref, connector): """ Fetch all the nodes from the cloned git repository in the given path.""" db_records = [] collection = node['collection'] if 'collection' in node else COLLECTION given_type = get_field_value(connector, "type") base_path = get_field_value_with_default(connector, "folderPath", "") snapshot_source = get_field_value(snapshot, 'source') parts = snapshot_source.split('.') d_record = { "structure": given_type, "reference": ref if not base_path else "", "source": parts[0], "path": '', "timestamp": int(time.time() * 1000), "queryuser": "", "checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(), "node": node, "snapshotId": None, "masterSnapshotId": node['masterSnapshotId'], "collection": collection.replace('.', '').lower(), "json": {} } node_type = node['type'] if 'type' in node and node['type'] else 'json' json_path = '%s/%s' % (repopath, node['path']) file_path = json_path.replace('//', '/') logger.info('Dir: %s', file_path) if exists_dir(file_path): count = 0 for filename in glob.glob('%s/*.json' % file_path.replace('//', '/')): parts = filename.rsplit('/', 1) path = '%s/%s' % (node['path'], parts[-1]) json_data = convert_to_json(filename, node_type) logger.info('type: %s, json:%s', node_type, json_data) if json_data: db_record = copy.deepcopy(d_record) db_record['snapshotId'] = '%s%s' % (node['masterSnapshotId'], str(count)) db_record['path'] = path.replace('//', '/') db_record['json'] = json_data data_str = json.dumps(json_data) db_record['checksum'] = hashlib.md5( data_str.encode('utf-8')).hexdigest() db_records.append(db_record) count += 1 else: logger.info('Get requires valid directory for snapshot not present!') return db_records
def generate_crawler_snapshot(snapshot,node,snapshot_data): node_type= get_field_value(node,'type') node_type = get_field_value_with_default(node, 'type',"") resource_items = [] resource_items=get_lits(snapshot=snapshot,node=node) snapshot_data[node['masterSnapshotId']] = [] for index,resource_namespaced_item in enumerate(resource_items) : snapshot_data[node['masterSnapshotId']].append({ "masterSnapshotId" : [node['masterSnapshotId']], "snapshotId": '%s%s_%s' % (node['masterSnapshotId'],resource_namespaced_item['namespace'], str(index)), "type": node_type, "collection": node['collection'], "paths": resource_namespaced_item['paths'], "status" : "active", "validate" : node['validate'] if 'validate' in node else True }) return snapshot_data
def run_file_validation_tests(test_file, container, filesystem=True, snapshot_status=None): # logger.info("*" * 50) logger.info("\tTEST: %s", test_file) dirpath = None test_json_data = json_from_file(test_file) if not test_json_data: logger.info("\t\tTest file %s looks to be empty, next!...", test_file) if test_json_data and "connector" in test_json_data and "remoteFile" in test_json_data and test_json_data["connector"] and test_json_data["remoteFile"]: dirpath, pull_response = pull_json_data(test_json_data) if not pull_response: return {} singletest = get_from_currentdata(SINGLETEST) if singletest: testsets = get_field_value_with_default(test_json_data, 'testSet', []) for testset in testsets: newtestcases = [] for testcase in testset['cases']: if ('testId' in testcase and testcase['testId'] == singletest) or \ ('masterTestId' in testcase and testcase['masterTestId'] == singletest): newtestcases.append(testcase) testset['cases'] = newtestcases resultset = run_json_validation_tests(test_json_data, container, filesystem, snapshot_status, dirpath=dirpath) finalresult = True if resultset: snapshot = test_json_data['snapshot'] if 'snapshot' in test_json_data else '' if singletest: print(json.dumps(resultset, indent=2)) else: dump_output_results(resultset, container, test_file, snapshot, filesystem) for result in resultset: if 'result' in result: if not re.match(r'passed', result['result'], re.I): finalresult = False break else: # TODO: NO test cases in this file. # LOG HERE that no test cases are present in this file. finalresult = False return finalresult
def get_node(repopath, node, snapshot, ref, connector): """ Fetch node from the cloned git repository.""" collection = node['collection'] if 'collection' in node else COLLECTION given_type = get_field_value(connector, "type") base_path = get_field_value_with_default(connector, "folderPath", "") snapshot_source = get_field_value(snapshot, 'source') parts = snapshot_source.split('.') db_record = { "structure": given_type, "reference": ref if not base_path else "", "source": parts[0], "path": base_path + node['path'], "timestamp": int(time.time() * 1000), "queryuser": "", "checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(), "node": node, "snapshotId": node['snapshotId'], "collection": collection.replace('.', '').lower(), "json": {} } json_path = '%s/%s' % (repopath, node['path']) file_path = json_path.replace('//', '/') logger.info('File: %s', file_path) if exists_file(file_path): node_type = node['type'] if 'type' in node and node['type'] else 'json' json_data = convert_to_json(file_path, node_type) logger.info('type: %s, json:%s', node_type, json_data) # json_data = json_from_file(file_path) if json_data: db_record['json'] = json_data data_str = json.dumps(json_data) db_record['checksum'] = hashlib.md5( data_str.encode('utf-8')).hexdigest() else: logger.info('Get requires valid file for snapshot not present!') logger.debug('DB: %s', db_record) return db_record
def git_clone_dir(connector): clonedir = None repopath = tempfile.mkdtemp() subdir = False if connector and isinstance(connector, dict): giturl = get_field_value(connector, 'gitProvider') if not giturl: logger.error("Git connector does not have valid git provider URL") return repopath, clonedir brnch = get_field_value_with_default(connector, 'branchName', 'master') isprivate = get_field_value(connector, 'private') isprivate = True if isprivate is None or not isinstance( isprivate, bool) else isprivate logger.info("Repopath: %s", repopath) http_match = re.match(r'^http(s)?://', giturl, re.I) if http_match: logger.info("Http (private:%s) giturl: %s, Repopath: %s", "YES" if isprivate else "NO", giturl, repopath) username = get_field_value(connector, 'httpsUser') if username: pwd = get_field_value(connector, 'httpsPassword') schema = giturl[:http_match.span()[-1]] other_part = giturl[http_match.span()[-1]:] pwd = pwd if pwd else get_git_pwd() if pwd: git_cmd = 'git clone %s%s:%s@%s %s' % ( schema, urllib.parse.quote_plus(username), urllib.parse.quote_plus(pwd), other_part, repopath) else: git_cmd = 'git clone %s%s@%s %s' % ( schema, urllib.parse.quote_plus(username), other_part, repopath) else: git_cmd = 'git clone %s %s' % (giturl, repopath) else: logger.info("SSH (private:%s) giturl: %s, Repopath: %s", "YES" if isprivate else "NO", giturl, repopath) if isprivate: ssh_key_file = get_field_value(connector, 'sshKeyfile') if not exists_file(ssh_key_file): logger.error( "Git connector points to a non-existent ssh keyfile!") return repopath, clonedir ssh_host = get_field_value(connector, 'sshHost') ssh_user = get_field_value_with_default( connector, 'sshUser', 'git') if not ssh_host: logger.error( "SSH host not set, could be like github.com, gitlab.com, 192.168.1.45 etc" ) return repopath, clonedir ssh_dir = '%s/.ssh' % repopath if exists_dir(ssh_dir): logger.error( "Git ssh dir: %s already exists, cannot recreate it!", ssh_dir) return repopath, clonedir os.mkdir('%s/.ssh' % repopath, 0o700) ssh_cfg = create_ssh_config(ssh_dir, ssh_key_file, ssh_user) if not ssh_cfg: logger.error( "Creation of Git ssh config in dir: %s failed!", ssh_dir) return repopath, clonedir git_ssh_cmd = 'ssh -o "StrictHostKeyChecking=no" -F %s' % ssh_cfg git_cmd = 'git clone %s %s/tmpclone' % (giturl, repopath) subdir = True else: git_ssh_cmd = 'ssh -o "StrictHostKeyChecking=no"' git_cmd = 'git clone %s %s' % (giturl, repopath) os.environ['GIT_SSH_COMMAND'] = git_ssh_cmd logger.info("GIT_SSH_COMMAND=%s", git_ssh_cmd) git_cmd = '%s --branch %s' % (git_cmd, brnch) logger.info("os.system(%s)", git_cmd) if git_cmd: run_subprocess_cmd(git_cmd) checkdir = '%s/tmpclone' % repopath if subdir else repopath clonedir = checkdir if exists_dir('%s/.git' % checkdir) else None if 'GIT_SSH_COMMAND' in os.environ: os.environ.pop('GIT_SSH_COMMAND') return repopath, clonedir
def run_container_validation_tests_filesystem(container, snapshot_status=None): """Get test files from the filesystem.""" # logger.info("Starting validation tests") logger.info("VALIDATION:") logger.info("\tCollection: %s, Type: FILESYSTEM", container) reporting_path = config_value('REPORTING', 'reportOutputFolder') json_dir = '%s/%s/%s' % (framework_dir(), reporting_path, container) logger.info('\tLOCATION: %s', json_dir) test_files = get_json_files(json_dir, JSONTEST) # logger.info('\n'.join(test_files)) result = True for test_file in test_files: logger.info('\tCOLLECTION: %s', test_file) val = run_file_validation_tests(test_file, container, True, snapshot_status) result = result and val if test_files: # return the result value if "test" file is processed collection logger.critical("VALIDATION COMPLETE:") return result # mastertest files test_files = get_json_files(json_dir, MASTERTEST) # logger.info('\n'.join(test_files)) if not test_files: logger.error("ERROR: No `test` or `mastertest` file found. collection should contain either `test` or `mastertest` file") return False finalresult = result for test_file in test_files: logger.info('\tCOLLECTION: %s', test_file) # logger.info("*" * 50) # logger.info("validator tests: %s", test_file) dirpath = None test_json_data = json_from_file(test_file) if not test_json_data: logger.info("Test file %s looks to be empty, next!...", test_file) continue if "connector" in test_json_data and "remoteFile" in test_json_data and test_json_data["connector"] and test_json_data["remoteFile"]: dirpath, pull_response = pull_json_data(test_json_data) if not pull_response: return {} snapshot_key = '%s_gen' % test_json_data['masterSnapshot'] mastersnapshots = defaultdict(list) snapshot_data = snapshot_status[snapshot_key] if snapshot_key in snapshot_status else {} for snapshot_id, mastersnapshot_id in snapshot_data.items(): if isinstance(mastersnapshot_id, list): for master_snapshot_id in mastersnapshot_id: mastersnapshots[master_snapshot_id].append(snapshot_id) elif isinstance(mastersnapshot_id, str): mastersnapshots[mastersnapshot_id].append(snapshot_id) if not mastersnapshots: logger.error("No generated snapshots found for validation.") continue test_json_data['snapshot'] = snapshot_key testsets = get_field_value_with_default(test_json_data, 'testSet', []) for testset in testsets: testcases = get_field_value_with_default(testset, 'cases', []) testset['cases'] = _get_new_testcases(testcases, mastersnapshots) # print(json.dumps(test_json_data, indent=2)) singletest = get_from_currentdata(SINGLETEST) if singletest: for testset in testsets: newtestcases = [] for testcase in testset['cases']: if ('testId' in testcase and testcase['testId'] == singletest) or \ ('masterTestId' in testcase and testcase['masterTestId'] == singletest): newtestcases.append(testcase) testset['cases'] = newtestcases resultset = run_json_validation_tests(test_json_data, container, True, snapshot_status, dirpath=dirpath) if test_json_data.get('testSet') and not resultset: logger.error('\tERROR: Testset does not contains any testcases or all testcases are skipped due to invalid rules.') elif resultset: snapshot = test_json_data['snapshot'] if 'snapshot' in test_json_data else '' if singletest: print(json.dumps(resultset, indent=2)) else: dump_output_results(resultset, container, test_file, snapshot, True) for result in resultset: if 'result' in result: if not re.match(r'passed', result['result'], re.I): finalresult = False break else: logger.error('\tERROR: No mastertest Documents found!') finalresult = False logger.critical("VALIDATION COMPLETE:") return finalresult
def git_clone_dir(connector): clonedir = None repopath = tempfile.mkdtemp() subdir = False if connector and isinstance(connector, dict): giturl = get_field_value(connector, 'gitProvider') if not giturl: logger.error("Git connector does not have valid git provider URL") return repopath, clonedir branch = get_from_currentdata('branch') if not branch: branch = get_field_value_with_default(connector, 'branchName', 'master') isprivate = get_field_value(connector, 'private') isprivate = True if isprivate is None or not isinstance(isprivate, bool) else isprivate # logger.info("Repopath: %s", repopath) logger.info("\t\t\tRepopath: %s", repopath) http_match = re.match(r'^http(s)?://', giturl, re.I) if http_match: logger.info("\t\t\tHttp (private:%s) giturl: %s", "YES" if isprivate else "NO", giturl) accessToken = get_field_value(connector, 'httpsAccessToken') username = get_field_value(connector, 'httpsUser') if accessToken: logger.info("AccessToken: %s" % accessToken) pwd = get_field_value(connector, 'httpsPassword') pwd = pwd if pwd else get_git_pwd(key=accessToken) if not pwd: pwd = get_pwd_from_vault(accessToken) if pwd: logger.info("Git access token from vault: %s", '*' * len(pwd)) if pwd: gh = GithubFunctions() gh.set_access_token(pwd) _ = gh.populate_user() rpo = gh.clone_repo(giturl, repopath, branch) if rpo: logger.info('Successfully cloned in %s dir' % repopath) checkdir = '%s/tmpclone' % repopath if subdir else repopath clonedir = checkdir if exists_dir('%s/.git' % checkdir) else None if not exists_dir(clonedir): logger.error("No valid data provided for connect to git : %s", giturl) return repopath, clonedir elif isprivate: logger.error("Please provide password for connect to git repository.") return repopath, clonedir else: git_cmd = 'git clone %s %s' % (giturl, repopath) elif username: pwd = get_field_value(connector, 'httpsPassword') schema = giturl[:http_match.span()[-1]] other_part = giturl[http_match.span()[-1]:] # pwd = pwd if (pwd and not json_source()) else (get_git_pwd() if not json_source() else get_pwd_from_vault(pwd)) pwd = pwd if pwd else get_git_pwd(key=username) # populate the password from vault if not pwd: pwd = get_pwd_from_vault(username) if pwd: logger.info("Git password from vault: %s", '*' * len(pwd)) if pwd: git_cmd = 'git clone --depth 1 %s%s:%s@%s %s' % (schema, urllib.parse.quote_plus(username), urllib.parse.quote_plus(pwd), other_part, repopath) elif isprivate: logger.error("Please provide password for connect to git repository.") return repopath, clonedir else: git_cmd = 'git clone --depth 1 %s%s:%s@%s %s' % (schema, urllib.parse.quote_plus(username), "", other_part, repopath) else: git_cmd = 'git clone --depth 1 %s %s' % (giturl, repopath) else: logger.info("SSH (private:%s) giturl: %s, Repopath: %s", "YES" if isprivate else "NO", giturl, repopath) if isprivate: ssh_key_file = get_field_value(connector, 'sshKeyfile') ssh_key_name = get_field_value(connector, 'sshKeyName') ssh_key_file_data = None if ssh_key_file: if not exists_file(ssh_key_file): logger.error("Git connector points to a non-existent ssh keyfile!") return repopath, clonedir elif ssh_key_name: ssh_key_file_data = get_vault_data(ssh_key_name) if not ssh_key_file_data: logger.info('Git connector points to a non-existent ssh keyName in the vault!') return repopath, clonedir ssh_host = get_field_value(connector, 'sshHost') ssh_user = get_field_value_with_default(connector, 'sshUser', 'git') if not ssh_host: logger.error("SSH host not set, could be like github.com, gitlab.com, 192.168.1.45 etc") return repopath, clonedir ssh_dir = '%s/.ssh' % repopath if exists_dir(ssh_dir): logger.error("Git ssh dir: %s already exists, cannot recreate it!", ssh_dir) return repopath, clonedir os.mkdir('%s/.ssh' % repopath, 0o700) if not ssh_key_file and ssh_key_name and ssh_key_file_data: ssh_key_file = create_ssh_file_vault_data(ssh_dir, ssh_key_file_data, ssh_key_name) if not ssh_key_file: logger.info('Git connector points to a non-existent ssh keyName in the vault!') return repopath, clonedir ssh_cfg = create_ssh_config(ssh_dir, ssh_key_file, ssh_user) if not ssh_cfg: logger.error("Creation of Git ssh config in dir: %s failed!", ssh_dir) return repopath, clonedir git_ssh_cmd = 'ssh -o "StrictHostKeyChecking=no" -F %s' % ssh_cfg git_cmd = 'git clone %s %s/tmpclone' % (giturl, repopath) subdir = True else: git_ssh_cmd = 'ssh -o "StrictHostKeyChecking=no"' git_cmd = 'git clone %s %s' % (giturl, repopath) os.environ['GIT_SSH_COMMAND'] = git_ssh_cmd git_cmd = '%s --branch %s' % (git_cmd, branch) if git_cmd: error_result, result = run_subprocess_cmd(git_cmd) checkdir = '%s/tmpclone' % repopath if subdir else repopath clonedir = checkdir if exists_dir('%s/.git' % checkdir) else None if not exists_dir(clonedir): logger.error("No valid data provided for connect to git : %s", error_result) if 'GIT_SSH_COMMAND' in os.environ: os.environ.pop('GIT_SSH_COMMAND') return repopath, clonedir
def populate_custom_snapshot(snapshot, container=None): """ Populates the resources from git.""" dbname = config_value('MONGODB', 'dbname') snapshot_source = get_field_value(snapshot, 'source') connector_data = get_from_currentdata('connector') if connector_data: sub_data = get_custom_data(connector_data) if not sub_data: logger.error("No connector data found in '%s'", connector_data) else: sub_data = get_custom_data(snapshot_source) snapshot_nodes = get_field_value(snapshot, 'nodes') snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes) if valid_snapshotids and sub_data and snapshot_nodes: baserepo, repopath = _get_repo_path(sub_data, snapshot) if repopath: brnch = get_field_value_with_default(sub_data, 'branchName', 'master') for node in snapshot_nodes: node_type = node['type'] if 'type' in node and node['type'] else '' if node_type in TEMPLATE_NODE_TYPES: template_data = { "container" : container, "dbname" : dbname, "snapshot_source" : snapshot_source, "connector_data" : sub_data, "snapshot_data" : snapshot_data, "repopath" : repopath, "snapshot" : snapshot } template_processor = TEMPLATE_NODE_TYPES[node_type](node, **template_data) if 'snapshotId' in node: snapshot_data = template_processor.populate_template_snapshot() elif 'masterSnapshotId' in node: snapshot_data = template_processor.populate_all_template_snapshot() elif 'paths' in node: logger.error("ERROR: Invalid json : `%s` is not a valid node type." % (node_type)) else: # logger.debug(node) # data = get_node(repopath, node, snapshot_source, brnch) # if data: # insert_one_document(data, data['collection'], dbname) # snapshot_data[node['snapshotId']] = True validate = node['validate'] if 'validate' in node else True if 'snapshotId' in node: logger.debug(node) data = get_node(repopath, node, snapshot, brnch, sub_data) if data: if validate: if get_dbtests(): if get_collection_size(data['collection']) == 0: #Creating indexes for collection create_indexes(data['collection'], config_value(DATABASE, DBNAME), [('snapshotId', pymongo.ASCENDING), ('timestamp', pymongo.DESCENDING)]) create_indexes( data['collection'], config_value(DATABASE, DBNAME), [ ('_id', pymongo.DESCENDING), ('timestamp', pymongo.DESCENDING), ('snapshotId', pymongo.ASCENDING) ] ) insert_one_document(data, data['collection'], dbname) else: snapshot_dir = make_snapshots_dir(container) if snapshot_dir: store_snapshot(snapshot_dir, data) if 'masterSnapshotId' in node: snapshot_data[node['snapshotId']] = node['masterSnapshotId'] else: snapshot_data[node['snapshotId']] = True # else: # snapshot_data[node['snapshotId']] = False node['status'] = 'active' else: node['status'] = 'inactive' logger.debug('Type: %s', type(data)) elif 'masterSnapshotId' in node: alldata = get_all_nodes(repopath, node, snapshot, brnch, sub_data) if alldata: snapshot_data[node['masterSnapshotId']] = [] for data in alldata: snapshot_data[node['masterSnapshotId']].append( { 'snapshotId': data['snapshotId'], 'path': data['path'], 'validate': validate }) logger.debug('Type: %s', type(alldata)) if baserepo and os.path.exists(baserepo): # logger.info('\t\tCLEANING Repo: %s', baserepo) shutil.rmtree(baserepo) return snapshot_data
def get_all_nodes(credentials, node, snapshot_source, snapshot, snapshot_data): """ Fetch all nodes from google using connection using google client API's functions. """ collection = node['collection'] if 'collection' in node else COLLECTION parts = snapshot_source.split('.') project_id = get_field_value_with_default(snapshot, 'project-id',"") node_type = get_field_value_with_default(node, 'type',"") db_record = { "structure": "google", "error": None, "reference": project_id, "source": parts[0], "path": "", "timestamp": int(time.time() * 1000), "queryuser": get_field_value(snapshot, 'testUser'), "checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(), "node": node, "snapshotId": None, "masterSnapshotId": [node['masterSnapshotId']], "collection": collection.replace('.', '').lower(), "json": {}, # Refactor when node is absent it should None, when empty object put it as {} "items": [] } if node_type: access_token = credentials.get_access_token().access_token header = { "Authorization" : ("Bearer %s" % access_token) } base_node_type_list = node_type.split("/") if len(base_node_type_list) > 1: base_node_type = base_node_type_list[1] else: logger.error("Invalid node type '%s'", node_type) return db_record request_url = get_api_path(base_node_type) request_url = generate_request_url(request_url, project_id) logger.info("Invoke request for get snapshot: %s", request_url) status, data = http_get_request(request_url, header) logger.info('Get snapshot status: %s', status) fn_str_list = "" if node and 'type' in node and node['type']: fn_str_list = get_field_value(node, 'type').split(".") response_param = "" if fn_str_list and len(fn_str_list) > 1: response_param = fn_str_list[-2] elif fn_str_list and len(fn_str_list) == 1: response_param = fn_str_list[0] if data: check_node_type = node_type node_type_list = node_type.split(".") if len(node_type_list) > 1: del node_type_list[-1] check_node_type = ".".join(node_type_list) db_record['json'] = data if response_param in data: db_record['items'] = data[response_param] elif "items" in data: if isinstance(data['items'], dict): for name, scoped_dict in data['items'].items(): if response_param in scoped_dict: db_record['items'] = db_record['items'] + scoped_dict[check_node_type] if not db_record['items']: db_record['items'] = data['items'] set_snapshot_data(node, db_record['items'], snapshot_data) checksum = get_checksum(data) if checksum: db_record['checksum'] = checksum return db_record
def populate_google_snapshot(snapshot, container=None): """ This is an entrypoint for populating a snapshot of type google. All snapshot connectors should take snapshot object and based on 'source' field create a method to connect to the service for the connector. The 'source' field could be used by more than one snapshot, so the 'testuser' attribute should match to the user the 'source' """ dbname = config_value('MONGODB', 'dbname') snapshot_source = get_field_value(snapshot, 'source') snapshot_user = get_field_value(snapshot, 'testUser') project_id = get_field_value(snapshot, 'project-id') sub_data = get_google_data(snapshot_source) snapshot_nodes = get_field_value(snapshot, 'nodes') snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes) if valid_snapshotids and sub_data and snapshot_nodes: logger.debug(sub_data) try: for node in snapshot['nodes']: validate = node['validate'] if 'validate' in node else True logger.info(node) node_type = get_field_value_with_default(node, 'type',"") credentials = get_google_client_data(sub_data, snapshot_user, node_type, project_id) if not credentials: logger.info("No GCE connection in the snapshot to access Google resource!...") return snapshot_data if 'snapshotId' in node: if validate: data = get_node(credentials, node, snapshot_source, snapshot) if data: error_str = data.pop('error', None) if get_dbtests(): if get_collection_size(data['collection']) == 0: #Creating indexes for collection create_indexes( data['collection'], config_value(DATABASE, DBNAME), [ ('snapshotId', pymongo.ASCENDING), ('timestamp', pymongo.DESCENDING) ] ) create_indexes( data['collection'], config_value(DATABASE, DBNAME), [ ('_id', pymongo.DESCENDING), ('timestamp', pymongo.DESCENDING), ('snapshotId', pymongo.ASCENDING) ] ) insert_one_document(data, data['collection'], dbname) else: snapshot_dir = make_snapshots_dir(container) if snapshot_dir: store_snapshot(snapshot_dir, data) if 'masterSnapshotId' in node: snapshot_data[node['snapshotId']] = node['masterSnapshotId'] else: snapshot_data[node['snapshotId']] = False if error_str else True else: node['status'] = 'inactive' elif 'masterSnapshotId' in node: data = get_all_nodes(credentials, node, snapshot_source, snapshot, snapshot_data) logger.debug('Type: %s', type(data)) except Exception as ex: logger.info('Unable to create Google client: %s', ex) raise ex return snapshot_data
def populate_custom_snapshot(snapshot, container=None): """ Populates the resources from git.""" dbname = config_value('MONGODB', 'dbname') snapshot_source = get_field_value(snapshot, 'source') sub_data = get_custom_data(snapshot_source) snapshot_nodes = get_field_value(snapshot, 'nodes') snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes) if valid_snapshotids and sub_data and snapshot_nodes: baserepo, repopath = _get_repo_path(sub_data, snapshot) if repopath: brnch = get_field_value_with_default(sub_data, 'branchName', 'master') for node in snapshot_nodes: node_type = node[ 'type'] if 'type' in node and node['type'] else 'json' if node_type == 'arm': if 'snapshotId' in node: populate_arm_snapshot(container, dbname, snapshot_source, sub_data, snapshot_data, node, repopath) elif 'masterSnapshotId' in node: populate_all_arm_snapshot(snapshot, dbname, sub_data, node, repopath, snapshot_data) else: # logger.debug(node) # data = get_node(repopath, node, snapshot_source, brnch) # if data: # insert_one_document(data, data['collection'], dbname) # snapshot_data[node['snapshotId']] = True validate = node['validate'] if 'validate' in node else True if 'snapshotId' in node: logger.debug(node) data = get_node(repopath, node, snapshot, brnch, sub_data) if data: if validate: if get_dbtests(): if get_collection_size( data['collection']) == 0: #Creating indexes for collection create_indexes( data['collection'], config_value(DATABASE, DBNAME), [('snapshotId', pymongo.ASCENDING), ('timestamp', pymongo.DESCENDING) ]) insert_one_document( data, data['collection'], dbname) else: snapshot_dir = make_snapshots_dir( container) if snapshot_dir: store_snapshot(snapshot_dir, data) if 'masterSnapshotId' in node: snapshot_data[node['snapshotId']] = node[ 'masterSnapshotId'] else: snapshot_data[node['snapshotId']] = True else: snapshot_data[node['snapshotId']] = False node['status'] = 'active' else: node['status'] = 'inactive' logger.debug('Type: %s', type(data)) elif 'masterSnapshotId' in node: alldata = get_all_nodes(repopath, node, snapshot, brnch, sub_data) if alldata: snapshot_data[node['masterSnapshotId']] = [] for data in alldata: snapshot_data[node['masterSnapshotId']].append( { 'snapshotId': data['snapshotId'], 'path': data['path'], 'validate': True }) logger.debug('Type: %s', type(alldata)) if baserepo and os.path.exists(baserepo): logger.info('Repo path: %s', baserepo) shutil.rmtree(baserepo) return snapshot_data
def run_container_validation_tests_filesystem(container, snapshot_status=None): """Get test files from the filesystem.""" logger.info("Starting validation tests") reporting_path = config_value('REPORTING', 'reportOutputFolder') json_dir = '%s/%s/%s' % (framework_dir(), reporting_path, container) logger.info(json_dir) test_files = get_json_files(json_dir, JSONTEST) logger.info('\n'.join(test_files)) result = True for test_file in test_files: val = run_file_validation_tests(test_file, container, True, snapshot_status) result = result and val # mastertest files test_files = get_json_files(json_dir, MASTERTEST) logger.info('\n'.join(test_files)) finalresult = True for test_file in test_files: logger.info("*" * 50) logger.info("validator tests: %s", test_file) test_json_data = json_from_file(test_file) if not test_json_data: logger.info("Test file %s looks to be empty, next!...", test_file) continue snapshot_key = '%s_gen' % test_json_data['masterSnapshot'] mastersnapshots = defaultdict(list) snapshot_data = snapshot_status[ snapshot_key] if snapshot_key in snapshot_status else {} for snapshot_id, mastersnapshot_id in snapshot_data.items(): if isinstance(mastersnapshot_id, list): for master_snapshot_id in mastersnapshot_id: mastersnapshots[master_snapshot_id].append(snapshot_id) elif isinstance(mastersnapshot_id, str): mastersnapshots[mastersnapshot_id].append(snapshot_id) test_json_data['snapshot'] = snapshot_key testsets = get_field_value_with_default(test_json_data, 'testSet', []) for testset in testsets: testcases = get_field_value_with_default(testset, 'cases', []) testset['cases'] = _get_new_testcases(testcases, mastersnapshots) # print(json.dumps(test_json_data, indent=2)) singletest = get_from_currentdata(SINGLETEST) if singletest: for testset in testsets: newtestcases = [] for testcase in testset['cases']: if ('testId' in testcase and testcase['testId'] == singletest) or \ ('masterTestId' in testcase and testcase['masterTestId'] == singletest): newtestcases.append(testcase) testset['cases'] = newtestcases resultset = run_json_validation_tests(test_json_data, container, False, snapshot_status) if resultset: snapshot = test_json_data[ 'snapshot'] if 'snapshot' in test_json_data else '' if singletest: print(json.dumps(resultset, indent=2)) else: dump_output_results(resultset, container, test_file, snapshot, True) for result in resultset: if 'result' in result: if not re.match(r'passed', result['result'], re.I): finalresult = False break else: logger.info('No mastertest Documents found!') finalresult = False return finalresult
def run_container_validation_tests_database(container, snapshot_status=None): """ Get the test files from the database""" dbname = config_value(DATABASE, DBNAME) # For test files collection = config_value(DATABASE, collectiontypes[TEST]) qry = {'container': container} sort = [sort_field('timestamp', False)] docs = get_documents(collection, dbname=dbname, sort=sort, query=qry) finalresult = True if docs and len(docs): logger.info('Number of test Documents: %s', len(docs)) for doc in docs: if doc['json']: resultset = run_json_validation_tests(doc['json'], container, False) if resultset: snapshot = doc['json']['snapshot'] if 'snapshot' in doc[ 'json'] else '' test_file = doc['name'] if 'name' in doc else '' dump_output_results(resultset, container, test_file, snapshot, False) for result in resultset: if 'result' in result: if not re.match(r'passed', result['result'], re.I): finalresult = False break else: logger.info('No test Documents found!') finalresult = False # For mastertest files collection = config_value(DATABASE, collectiontypes[MASTERTEST]) docs = get_documents(collection, dbname=dbname, sort=sort, query=qry) # snapshots_details_map = _get_snapshot_type_map(container) if docs and len(docs): logger.info('Number of mastertest Documents: %s', len(docs)) for doc in docs: test_json_data = doc['json'] if test_json_data: snapshot_key = '%s_gen' % test_json_data['masterSnapshot'] mastersnapshots = defaultdict(list) snapshot_data = snapshot_status[ snapshot_key] if snapshot_key in snapshot_status else {} for snapshot_id, mastersnapshot_id in snapshot_data.items(): if isinstance(mastersnapshot_id, list): for msnp_id in mastersnapshot_id: mastersnapshots[msnp_id].append(snapshot_id) else: mastersnapshots[mastersnapshot_id].append(snapshot_id) test_json_data['snapshot'] = snapshot_key testsets = get_field_value_with_default( test_json_data, 'testSet', []) for testset in testsets: testcases = get_field_value_with_default( testset, 'cases', []) testset['cases'] = _get_new_testcases( testcases, mastersnapshots) # print(json.dumps(test_json_data, indent=2)) resultset = run_json_validation_tests(test_json_data, container, False, snapshot_status) if resultset: snapshot = doc['json']['snapshot'] if 'snapshot' in doc[ 'json'] else '' test_file = doc['name'] if 'name' in doc else '' dump_output_results(resultset, container, test_file, snapshot, False) for result in resultset: if 'result' in result: if not re.match(r'passed', result['result'], re.I): finalresult = False break else: logger.info('No mastertest Documents found!') finalresult = False return finalresult
def run_container_validation_tests_database(container, snapshot_status=None): """ Get the test files from the database""" dirpath = None dbname = config_value(DATABASE, DBNAME) test_files_found = True mastertest_files_found = True # For test files collection = config_value(DATABASE, collectiontypes[TEST]) qry = {'container': container} sort = [sort_field('timestamp', False)] docs = get_documents(collection, dbname=dbname, sort=sort, query=qry) finalresult = True if docs and len(docs): logger.info('Number of test Documents: %s', len(docs)) for doc in docs: if doc['json']: try: snapshot = doc['json']['snapshot'] if 'snapshot' in doc['json'] else '' if "connector" in doc['json'] and "remoteFile" in doc['json'] and doc['json']["connector"] and doc['json']["remoteFile"]: dirpath, pull_response = pull_json_data(doc['json']) if not pull_response: return {} resultset = run_json_validation_tests(doc['json'], container, False, dirpath=dirpath) if resultset: test_file = doc['name'] if 'name' in doc else '' dump_output_results(resultset, container, test_file, snapshot, False) for result in resultset: if 'result' in result: if not re.match(r'passed', result['result'], re.I): finalresult = False break except Exception as e: dump_output_results([], container, "-", snapshot, False) raise e else: logger.info('No test Documents found!') test_files_found = False finalresult = False # For mastertest files collection = config_value(DATABASE, collectiontypes[MASTERTEST]) docs = get_documents(collection, dbname=dbname, sort=sort, query=qry) # snapshots_details_map = _get_snapshot_type_map(container) if docs and len(docs): logger.info('Number of mastertest Documents: %s', len(docs)) for doc in docs: test_json_data = doc['json'] if test_json_data: snapshot = doc['json']['snapshot'] if 'snapshot' in doc['json'] else '' test_file = doc['name'] if 'name' in doc else '-' try: if "connector" in test_json_data and "remoteFile" in test_json_data and test_json_data["connector"] and test_json_data["remoteFile"]: dirpath, pull_response = pull_json_data(test_json_data) if not pull_response: return {} snapshot_key = '%s_gen' % test_json_data['masterSnapshot'] mastersnapshots = defaultdict(list) snapshot_data = snapshot_status[snapshot_key] if snapshot_key in snapshot_status else {} for snapshot_id, mastersnapshot_id in snapshot_data.items(): if isinstance(mastersnapshot_id, list): for msnp_id in mastersnapshot_id: mastersnapshots[msnp_id].append(snapshot_id) else: mastersnapshots[mastersnapshot_id].append(snapshot_id) test_json_data['snapshot'] = snapshot_key testsets = get_field_value_with_default(test_json_data, 'testSet', []) for testset in testsets: testcases = get_field_value_with_default(testset, 'cases', []) testset['cases'] = _get_new_testcases(testcases, mastersnapshots) # print(json.dumps(test_json_data, indent=2)) resultset = run_json_validation_tests(test_json_data, container, False, snapshot_status, dirpath=dirpath) if resultset: dump_output_results(resultset, container, test_file, snapshot, False) for result in resultset: if 'result' in result: if not re.match(r'passed', result['result'], re.I): finalresult = False break except Exception as e: dump_output_results([], container, test_file, snapshot, False) raise e else: logger.info('No mastertest Documents found!') mastertest_files_found = False finalresult = False if not test_files_found and not mastertest_files_found: raise Exception("No complaince tests for this container: %s, add and run!", container) return finalresult
def get_all_nodes(compute_fn, node, snapshot_source, snapshot, snapshot_data): """ Fetch all nodes from google using connection using google client API's functions. """ collection = node['collection'] if 'collection' in node else COLLECTION parts = snapshot_source.split('.') project_id = get_field_value_with_default(snapshot, 'project-id', "") node_type = get_field_value_with_default(node, 'type', "") db_record = { "structure": "google", "error": None, "reference": project_id, "source": parts[0], "path": "", "timestamp": int(time.time() * 1000), "queryuser": "", "checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(), "node": node, "snapshotId": None, "masterSnapshotId": [node['masterSnapshotId']], "collection": collection.replace('.', '').lower(), "json": {}, # Refactor when node is absent it should None, when empty object put it as {} "items": [] } if node_type: fn_str_list, kwargs = get_google_call_function_for_crawler( node, project_id) if fn_str_list and kwargs: for i in range(0, len(fn_str_list)): compute_fn = getattr(compute_fn, fn_str_list[i], None) if compute_fn and i != len(fn_str_list) - 1: compute_fn = compute_fn() response_param = "" if fn_str_list and len(fn_str_list) > 1: response_param = fn_str_list[-2] elif fn_str_list and len(fn_str_list) == 1: response_param = fn_str_list[0] if compute_fn and callable(compute_fn): try: data = compute_fn(**kwargs).execute() if data: check_node_type = node_type node_type_list = node_type.split(".") if len(node_type_list) > 1: del node_type_list[-1] check_node_type = ".".join(node_type_list) db_record['json'] = data if response_param in data: db_record['items'] = data[response_param] elif "items" in data: if isinstance(data['items'], dict): for name, scoped_dict in data['items'].items(): if response_param in scoped_dict: db_record['items'] = db_record[ 'items'] + scoped_dict[ check_node_type] if not db_record['items']: db_record['items'] = data['items'] set_snapshot_data(node, db_record['items'], snapshot_data) checksum = get_checksum(data) if checksum: db_record['checksum'] = checksum else: put_in_currentdata('errors', data) logger.info("Compute function does not exist: %s", str(fn_str_list)) db_record[ 'error'] = "Compute function does not exist: %s" % str( fn_str_list) except Exception as ex: logger.info('Compute function exception: %s', ex) db_record['error'] = 'Compute function exception: %s' % ex else: logger.info('Invalid Compute function exception: %s', str(fn_str_list)) db_record[ 'error'] = 'Invalid Compute function exception: %s' % str( fn_str_list) else: logger.info('Missing Compute function') db_record['error'] = 'Missing Compute function' return db_record