def generate_snapshots_from_mastersnapshot_file(mastersnapshot_file): """ Each snapshot file from the filesystem is loaded as a json datastructue and generate all the nodes in this json datastructure. """ mastersnapshot_file_name = '%s.json' % mastersnapshot_file if mastersnapshot_file and not \ mastersnapshot_file.endswith('.json') else mastersnapshot_file mastersnapshot_json_data = json_from_file(mastersnapshot_file_name) if not mastersnapshot_json_data: logger.error("masterSnapshot file %s looks to be empty, next!...", mastersnapshot_file) return {}, {} if "connector" in mastersnapshot_json_data and "remoteFile" in mastersnapshot_json_data and mastersnapshot_json_data[ "connector"] and mastersnapshot_json_data["remoteFile"]: _, pull_response = pull_json_data(mastersnapshot_json_data) if not pull_response: return {}, {} logger.debug(json.dumps(mastersnapshot_json_data, indent=2)) parts = mastersnapshot_file_name.rsplit('.', 1) snapshot_file_name = '%s_gen.%s' % (parts[0], parts[1]) snapshot_json_data = json_from_file(snapshot_file_name) if not snapshot_json_data: snapshot_json_data = {} snapshot_data = generate_mastersnapshots_from_json( mastersnapshot_json_data, snapshot_json_data) # save_json_to_file(mastersnapshot_json_data, mastersnapshot_file) if exists_file(snapshot_file_name): remove_file(snapshot_file_name) save_json_to_file(snapshot_json_data, snapshot_file_name) return snapshot_data, mastersnapshot_json_data
def generate_template_json(self): """ generate the template json from template and parameter files """ stars = '*' * 25 template_json = json_from_file(self.get_template()) self.replace_spacial_characters(template_json) gen_template_json = None if template_json: gen_template_json = copy.deepcopy(template_json) if 'parameters' not in template_json: template_json['parameters'] = {} self.gparams = template_json['parameters'] for param in self.get_parameter(): param_json = json_from_file(param) if 'parameters' in param_json and param_json['parameters']: for key, value in param_json['parameters'].items(): # if key in template_json['parameters']: if "value" in value: if key not in template_json['parameters']: template_json['parameters'][key] = {'value': value['value']} else: template_json['parameters'][key]['value'] = value['value'] else: logger.error("From parameter %s was not replaced.", key) gen_template_json['parameters'] = self.gparams # print('%s Updated Parameters %s' % (stars, stars)) # print(json.dumps(template_json['parameters'], indent=2)) if 'variables' in template_json: self.gvariables = template_json['variables'] new_resource = self.process_resource(template_json['variables']) # print('%s Original Variables %s' % (stars, stars)) # print(json.dumps(template_json['variables'], indent=2)) # print('%s Updated Variables %s' % (stars, stars)) # print(json.dumps(new_resource, indent=2)) # Second pass, becoz some variables have been defined in terms of parameters, functions like concat, substr self.gvariables = self.process_resource(new_resource) gen_template_json['variables'] = self.gvariables if 'resources' in template_json: new_resources = [] for resource in template_json['resources']: is_copy, copy_resources = self.handle_copy(resource) if is_copy: for resourc in copy_resources: new_resource = self.process_resource(resourc) new_resources.append(new_resource) else: new_resource = self.process_resource(resource) new_resources.append(new_resource) # print('%s Original Resource %s' % (stars, stars)) # print(json.dumps(resource, indent=2)) # print('%s Updated Resource %s' % (stars, stars)) # print(json.dumps(new_resource, indent=2)) gen_template_json['resources'] = new_resources return gen_template_json
def container_snapshots_filesystem(container): """ Get snapshot and mastersnapshot list used in all test/mastertest files of a container from the filesystem. This gets list of all the snapshots/mastersnapshots used in the container. The list will be used to not populate the snapshots/mastersnapshots multiple times, if the same snapshots/mastersnapshots are used in different test/mastertest files of a container. The configuration of the default path is configured in config.ini. """ snapshots = [] logger.info("Starting to get list of snapshots") reporting_path = config_value('REPORTING', 'reportOutputFolder') json_dir = '%s/%s/%s' % (framework_dir(), reporting_path, container) logger.info(json_dir) singletest = get_from_currentdata(SINGLETEST) test_files = get_json_files(json_dir, JSONTEST) logger.info('\n'.join(test_files)) for test_file in test_files: test_json_data = json_from_file(test_file) if test_json_data: snapshot = test_json_data['snapshot'] if 'snapshot' in test_json_data else '' if snapshot: file_name = snapshot if snapshot.endswith('.json') else '%s.json' % snapshot if singletest: testsets = get_field_value_with_default(test_json_data, 'testSet', []) for testset in testsets: for testcase in testset['cases']: if ('testId' in testcase and testcase['testId'] == singletest) or \ ('masterTestId' in testcase and testcase['masterTestId'] == singletest): if file_name not in snapshots: snapshots.append(file_name) else: snapshots.append(file_name) test_files = get_json_files(json_dir, MASTERTEST) logger.info('\n'.join(test_files)) for test_file in test_files: test_json_data = json_from_file(test_file) if test_json_data: snapshot = test_json_data['masterSnapshot'] if 'masterSnapshot' in test_json_data else '' if snapshot: file_name = snapshot if snapshot.endswith('.json') else '%s.json' % snapshot parts = file_name.split('.') file_name = '%s_gen.%s' % (parts[0], parts[-1]) if singletest: testsets = get_field_value_with_default(test_json_data, 'testSet', []) for testset in testsets: for testcase in testset['cases']: if ('testId' in testcase and testcase['testId'] == singletest) or \ ('masterTestId' in testcase and testcase['masterTestId'] == singletest): if file_name not in snapshots: snapshots.append(file_name) else: snapshots.append(file_name) return list(set(snapshots))
def generate_template_json(self): """ generate the template json from template and parameter files """ gen_template_json = None template_json = None if self.get_template().endswith(".yaml") and exists_file( self.get_template()): template_json = self.yaml_to_json(self.get_template()) elif self.get_template().endswith(".json"): template_json = json_from_file(self.get_template(), object_pairs_hook=None) # template_json = self.json_from_file(self.get_template()) logger.info(self.get_template()) if not template_json: logger.error("Invalid path! No file found at : %s", self.get_template()) return gen_template_json if "AWSTemplateFormatVersion" not in template_json: logger.error( "Invalid file content : file does not contains 'AWSTemplateFormatVersion' field." ) return gen_template_json if template_json: gen_template_json = copy.deepcopy(template_json) if 'Parameters' in template_json: self.gparams = template_json['Parameters'] if self.parameter_file: parameters = json_from_file(self.parameter_file, object_pairs_hook=None) # parameters = self.json_from_file(self.parameter_file) if parameters: for param in parameters: if "ParameterKey" in param and "ParameterValue" in param: self.gparams[param["ParameterKey"]] = { "Default": param["ParameterValue"] } logger.info(self.gparams) if 'Mappings' in template_json: self.mappings = template_json['Mappings'] if 'Resources' in template_json: new_resources = [] for key, resource in template_json['Resources'].items(): new_resource = self.process_resource(resource) new_resources.append(new_resource) gen_template_json['Resources'] = new_resources return gen_template_json
def get_snaphotid_doc_old(self, sid, container): doc = None json_dir = get_test_json_dir() if exists_dir(json_dir): fname = '%s/%s/snapshots/%s' % (json_dir, container, sid) if exists_file(fname): json_data = json_from_file(fname) if json_data and 'json' in json_data: doc = json_data['json'] snapshot = { 'id': json_data['snapshotId'], 'structure': json_data['structure'], 'reference': json_data['reference'], 'source': json_data['source'], 'collection': json_data['collection'], 'type': json_data.get("node", {}).get('type'), 'region' : json_data.get('region', "") } if 'paths' in json_data: snapshot['paths'] = json_data['paths'] else: snapshot['path'] = json_data['path'] self.snapshots.append(snapshot) return doc
def get_paramter_json_list(self): """ process parameter files and returns parameters json list """ parameter_json_list = [] for parameter in self.parameter_file: if len(parameter.split(".")) > 0 and parameter.split(".")[-1] in [ "tf", "tfvars" ]: json_data = hcl_to_json(parameter) if json_data: parameter_json_list.append( {parameter.split(".")[-1]: json_data}) elif len(parameter.split(".")) > 1 and [ ele for ele in [".tfvars.json", ".tf.json"] if (ele in parameter) ]: json_data = json_from_file(parameter, escape_chars=['$']) if json_data: splited_list = parameter.split(".") parameter_json_list.append({ '.'.join(splited_list[len(splited_list) - 2:]): json_data }) return parameter_json_list
def get_call_kwargs(node): """Get argument names and their values in kwargs""" kwargs = {"params": {}} logger.info("Get node's kwargs") params_source = config_value('GOOGLE', 'params') paramsversions = None if json_source(): dbname = config_value(DATABASE, DBNAME) collection = config_value(DATABASE, collectiontypes[STRUCTURE]) parts = params_source.rsplit('/') name = parts[-1].split('.') qry = {'name': name[0]} sort = [sort_field('timestamp', False)] docs = get_documents(collection, dbname=dbname, sort=sort, query=qry, limit=1) logger.info('Number of Google Params versions: %s', len(docs)) if docs and len(docs): paramsversions = docs[0]['json'] else: paramsversions_file = '%s/%s' % (framework_dir(), params_source) logger.info(paramsversions_file) if exists_file(paramsversions_file): paramsversions = json_from_file(paramsversions_file) path = node['path'] if paramsversions and "queryprameters" in paramsversions: if node['type'] in paramsversions["queryprameters"]: for param, parameter_type in paramsversions["queryprameters"][ node['type']].items(): add_argument_parameter(path, kwargs, param, parameter_type) return kwargs
def mastersnapshots_used_in_mastertests_filesystem(container): """ Get mastersnapshot list used in all mastertest files of a container from the filesystem. This gets list of all the mastersnapshots used in the container. The list will be used to make sure the snapshots are not generated multiple times, if the same mastersnapshots are used in different mastertest files of a container. The configuration of the default path is configured in config.ini. """ snapshots = [] # logger.info("Starting to get list of mastersnapshots used in test files.") reporting_path = config_value('REPORTING', 'reportOutputFolder') json_dir = '%s/%s/%s' % (framework_dir(), reporting_path, container) # logger.info(json_dir) # Only get list of mastertest files. test_files = get_json_files(json_dir, MASTERTEST) # logger.info('\n'.join(test_files)) for test_file in test_files: logger.info('\tMASTERTEST:%s', test_file) test_json_data = json_from_file(test_file) if test_json_data: snapshot = test_json_data[ 'masterSnapshot'] if 'masterSnapshot' in test_json_data else '' if snapshot: file_name = snapshot if snapshot.endswith( '.json') else '%s.json' % snapshot snapshots.append(file_name) return list( set(snapshots)) # set so that unique list of files are returned.
def get_currentdata(): """Get the current run data, if present else empty json object""" runctx = framework_currentdata() curr_data = json_from_file(runctx) if not curr_data: curr_data = {} return curr_data
def get_azure_data(snapshot_source): sub_data = {} if json_source(): dbname = config_value(DATABASE, DBNAME) collection = config_value(DATABASE, collectiontypes[STRUCTURE]) parts = snapshot_source.split('.') qry = {'name': parts[0]} sort = [sort_field('timestamp', False)] docs = get_documents(collection, dbname=dbname, sort=sort, query=qry, limit=1) logger.info('Number of Snapshot Documents: %s', len(docs)) if docs and len(docs): sub_data = docs[0]['json'] else: json_test_dir = get_test_json_dir() file_name = '%s.json' % snapshot_source if snapshot_source and not \ snapshot_source.endswith('.json') else snapshot_source azure_source = '%s/../%s' % (json_test_dir, file_name) logger.info('Azure source: %s', azure_source) if exists_file(azure_source): sub_data = json_from_file(azure_source) return sub_data
def get_version_for_type(node): """Url version of the resource.""" version = None apiversions = None logger.info("Get type's version") api_source = config_value('AZURE', 'api') if json_source(): dbname = config_value(DATABASE, DBNAME) collection = config_value(DATABASE, collectiontypes[STRUCTURE]) parts = api_source.rsplit('/') name = parts[-1].split('.') qry = {'name': name[0]} sort = [sort_field('timestamp', False)] docs = get_documents(collection, dbname=dbname, sort=sort, query=qry, limit=1) logger.info('Number of Azure API versions: %s', len(docs)) if docs and len(docs): apiversions = docs[0]['json'] else: apiversions_file = '%s/%s' % (framework_dir(), api_source) logger.info(apiversions_file) if exists_file(apiversions_file): apiversions = json_from_file(apiversions_file) if apiversions: if node and 'type' in node and node['type'] in apiversions: version = apiversions[node['type']]['version'] return version
def get_google_data(snapshot_source): """ The Google source object to be fetched from database or the filesystem The initial configuration for database is 'validator' and collection is 'structures', whereas for the filesystem the path to fetch the 'structures' is $SOLUTIONDIR/realm/<structure>.json """ sub_data = {} if json_source(): dbname = config_value(DATABASE, DBNAME) collection = config_value(DATABASE, collectiontypes[STRUCTURE]) parts = snapshot_source.split('.') qry = {'name': parts[0]} sort = [sort_field('timestamp', False)] docs = get_documents(collection, dbname=dbname, sort=sort, query=qry, limit=1) logger.info('Number of Google structure Documents: %d', len(docs)) if docs and len(docs): sub_data = docs[0]['json'] else: json_test_dir = get_test_json_dir() file_name = '%s.json' % snapshot_source if snapshot_source and not \ snapshot_source.endswith('.json') else snapshot_source google_source = '%s/../%s' % (json_test_dir, file_name) logger.info('Google source: %s', google_source) if exists_file(google_source): sub_data = json_from_file(google_source) if not sub_data: logger.error("Google connector file %s does not exist, or it does not contains the valid JSON.", snapshot_source) return sub_data
def get_connector_data(self): """ get connector data from snapshot """ connector_data = {} if self.snapshots: isdb_fetch = get_dbtests() if isdb_fetch: connectors = get_documents( "structures", query={ "name" : self.snapshots[0].get("source"), "type" : "structure", "container": self.container }, dbname=self.dbname, limit=1 ) connector_data = connectors[0].get("json", {}) if connectors else {} else: json_test_dir = get_test_json_dir() snapshot_source = self.snapshots[0].get("source") file_name = '%s.json' % snapshot_source if snapshot_source and not \ snapshot_source.endswith('.json') else snapshot_source connector_path = '%s/../%s' % (json_test_dir, file_name) if exists_file(connector_path): connector_data = json_from_file(connector_path) return connector_data
def get_node_version(node, snapshot): """Url version of the resource.""" version = None apiversions = None logger.info("Get type's version") api_source = config_value('AZURE', 'api') if snapshot.isDb: parts = api_source.rsplit('/') name = parts[-1].split('.') qry = {'name': name[0]} docs = get_documents(snapshot.collection(STRUCTURE), dbname=snapshot.dbname, sort=snapshot.sort, query=qry, limit=1) logger.info('Number of Azure API versions: %s', len(docs)) if docs and len(docs): apiversions = docs[0]['json'] else: apiversions_file = '%s/%s' % (framework_dir(), api_source) logger.info(apiversions_file) if exists_file(apiversions_file): apiversions = json_from_file(apiversions_file) if apiversions: if node and 'type' in node and node['type'] in apiversions: version = apiversions[node['type']]['version'] return version
def process_files(self, test_files, doctype, replace=False): """ Process Test or masterTest json files.""" snapshots = [] for test_file in test_files: test_json_data = json_from_file(test_file) if test_json_data: snapshot = test_json_data[ doctype] if doctype in test_json_data else '' if snapshot: file_name = snapshot if snapshot.endswith( '.json') else '%s.json' % snapshot if replace: file_name = file_name.replace('.json', '_gen.json') if self.singleTest: testsets = get_field_value_with_default( test_json_data, 'testSet', []) for testset in testsets: for testcase in testset['cases']: if ('testId' in testcase and testcase['testId'] == self.singleTest) or \ ('masterTestId' in testcase and testcase['masterTestId'] == self.singleTest): if file_name not in snapshots: snapshots.append(file_name) else: snapshots.append(file_name) return snapshots
def is_template_file(self, filename): if filename and filename.endswith('_gen.json'): return False json_data = json_from_file(filename) if json_data and '$schema' in json_data and json_data['$schema']: match = re.match(r'.*deploymentTemplate.json#$', json_data['$schema'], re.I) return True if match else False return None
def is_template_file(self, file_path): """ check for valid template file for parse arm template """ if len(file_path.split(".")) > 0 and file_path.split( ".")[-1] == "json": json_data = json_from_file(file_path) return True if (json_data) else False return False
def test_json_from_file(create_temp_dir, create_temp_json, create_temp_text): newpath = create_temp_dir() fname = create_temp_text(newpath) fullpath = '%s/%s' % (newpath, fname) file_exists = os.path.exists(fullpath) assert True == file_exists json_data = json_from_file(fullpath) assert json_data is None fname = create_temp_json(newpath) fullpath = '%s/%s' % (newpath, fname) file_exists = os.path.exists(fullpath) assert True == file_exists json_data = json_from_file(fullpath) assert json_data is not None assert isinstance(json_data, collections.OrderedDict) json_data = json_from_file(None) assert json_data is None json_data = json_from_file('/tmp/xyza.json') assert json_data is None
def process_rego_test_case(self): inputjson = {} result = False opa_exe = opa_binary() if not opa_exe: # print('*' * 50) return result rule_expr = get_field_value(self.testcase, 'eval') if not rule_expr: rule_expr = 'data.rule.rulepass' if len(self.testcase['snapshotId']) == 1: sid = self.testcase['snapshotId'][0] inputjson = self.get_snaphotid_doc(sid) else: ms_id = dict( zip(self.testcase['snapshotId'], self.testcase['masterSnapshotId'])) for sid in self.testcase['snapshotId']: inputjson.update({ms_id[sid]: self.get_snaphotid_doc(sid)}) if inputjson: save_json_to_file(inputjson, '/tmp/input.json') rego_rule = self.rule rego_match = re.match(r'^file\((.*)\)$', rego_rule, re.I) if rego_match: # rego_file = get_rego_rule_filename(rego_match.groups()[0], self.container) rego_file = self.rego_rule_filename(rego_match.groups()[0], self.container) if rego_file: pass else: rego_file = None # rego_file1 = self.rego_rule_filename(rego_match.groups()[0], "google_crawler_container") # rego_file1 = self.rego_rule_filename('google_crawler.rego', "google_crawler_container") # print(rego_file1) else: rego_txt = [ "package rule", "default rulepass = false", "rulepass = true{", " %s" % rego_rule, "}", "" ] rego_file = '/tmp/input.rego' open(rego_file, 'w').write('\n'.join(rego_txt)) if rego_file: os.system( '%s eval -i /tmp/input.json -d %s "%s" > /tmp/a.json' % (opa_exe, rego_file, rule_expr)) resultval = json_from_file('/tmp/a.json') if resultval: resultbool = resultval['result'][0]['expressions'][0][ 'value'] # [get_field_value(resultval, 'result[0].expressions[0].value') if resultbool: result = parsebool(resultbool) else: result = False return result
def is_template_file(self, file_path): """ check for valid template file for parse terraform template """ if len(file_path.split(".")) > 0 and file_path.split(".")[-1] == "tf": json_data = hcl_to_json(file_path) return True if (json_data and ("resource" in json_data or "module" in json_data)) else False elif len(file_path.split(".")) > 0 and file_path.split(".")[-1] == "json": json_data = json_from_file(file_path, escape_chars=['$']) return True if (json_data and ("resource" in json_data or "module" in json_data)) else False return False
def is_parameter_file(self, file_path): """ check for valid variable file for parse terraform template """ if len(file_path.split(".")) > 0 and file_path.split(".")[-1] in ["tf", "tfvars"]: json_data = hcl_to_json(file_path) return True if (json_data and not "resource" in json_data) else False elif len(file_path.split(".")) > 1 and [ele for ele in [".tfvars.json", ".tf.json"] if(ele in file_path)]: json_data = json_from_file(file_path, escape_chars=['$']) return True if (json_data and not "resource" in json_data) else False return False
def is_parameter_file(self, file_path): """ check for valid parameter file for parse cloudformation template """ if len(file_path.split(".")) > 0 and file_path.split( ".")[-1] == "json": json_data = json_from_file(file_path) if json_data and '$schema' in json_data and json_data['$schema']: match = re.match(r'.*deploymentParameters.json#', json_data['$schema'], re.I) return True if match else False return False
def convert_to_json(file_path, node_type): json_data = {} if node_type == 'json': json_data = json_from_file(file_path, escape_chars=['$']) elif node_type == 'terraform': with open(file_path, 'r') as fp: json_data = hcl.load(fp) elif node_type == 'yaml' or node_type == 'yml': json_data = yaml_from_file(file_path) else: logger.error("Snapshot error type:%s and file: %s", node_type, file_path) return json_data
def is_template_file(self, file_path): """ check for valid template file for parse arm template """ if len(file_path.split(".")) > 0 and file_path.split( ".")[-1] == "json": json_data = json_from_file(file_path) if json_data and '$schema' in json_data and json_data['$schema']: match = re.match(r'.*deploymentTemplate.json#$', json_data['$schema'], re.I) return True if match else False return False
def get_template(self): """ return the template json data """ json_data = None if len(self.template_file.split(".")) > 0 and self.template_file.split( ".")[-1] == "tf": json_data = hcl_to_json(self.template_file) elif len(self.template_file.split( ".")) > 1 and ".tf.json" in self.template_file: json_data = json_from_file(self.template_file, escape_chars=['$']) return json_data
def get_snaphotid_doc(self, sid): doc = None isdb_fetch = get_dbtests() if isdb_fetch: dbname = self.dbname coll = self.collection_data[sid] if sid in self.collection_data else COLLECTION docs = get_documents(coll, {'snapshotId': sid}, dbname, sort=[('timestamp', pymongo.DESCENDING)], limit=1) logger.debug('Number of Snapshot Documents: %s', len(docs)) if docs and len(docs): doc = docs[0]['json'] snapshot = { 'id': docs[0]['snapshotId'], 'structure': docs[0]['structure'], 'reference': docs[0]['reference'], 'source': docs[0]['source'], 'collection': docs[0]['collection'], 'type': docs[0].get("node", {}).get('type'), 'region' : docs[0].get('region', "") } if 'paths' in docs[0]: snapshot['paths'] = docs[0]['paths'] else: snapshot['path'] = docs[0]['path'] self.snapshots.append(snapshot) else: json_dir = '%s%s' % (get_test_json_dir(), self.container) if exists_dir(json_dir): fname = '%s/snapshots/%s' % (json_dir, sid) if exists_file(fname): json_data = json_from_file(fname) if json_data and 'json' in json_data: doc = json_data['json'] snapshot_val = { 'id': json_data['snapshotId'], 'structure': json_data['structure'], 'reference': json_data['reference'], 'source': json_data['source'], 'collection': json_data['collection'], 'type': json_data.get("node", {}).get('type'), 'region' : json_data.get('region', "") } if 'paths' in json_data: snapshot_val['paths'] = json_data['paths'] else: snapshot_val['path'] = json_data['path'] singletest = get_from_currentdata(SINGLETEST) if singletest: snapshot_val['json'] = doc self.snapshots.append(snapshot_val) return doc
def get_structure_data(self, snapshot_object): """ Get the structure from the filesystem.""" structure_data = {} json_test_dir = get_test_json_dir() snapshot_source = get_field_value(snapshot_object, "source") file_name = '%s.json' % snapshot_source if snapshot_source and not \ snapshot_source.endswith('.json') else snapshot_source custom_source = '%s/../%s' % (json_test_dir, file_name) logger.info('%s structure file is %s', Snapshot.LOGPREFIX, custom_source) if exists_file(custom_source): structure_data = json_from_file(custom_source) return structure_data
def get_custom_data(snapshot_source): """ Get source JSON data """ sub_data = {} json_test_dir = get_test_json_dir() file_name = '%s.json' % snapshot_source if snapshot_source and not \ snapshot_source.endswith('.json') else snapshot_source custom_source = '%s/../%s' % (json_test_dir, file_name) logger.info('Custom source: %s', custom_source) if exists_file(custom_source): sub_data = json_from_file(custom_source) return sub_data
def main(arg_vals=None): """Main driver utility for running validator tests.""" logger.info("Comand: '%s %s'", sys.executable.rsplit('/', 1)[-1], ' '.join(sys.argv)) cmd_parser = argparse.ArgumentParser("Comparator functional tests.") cmd_parser.add_argument('container', action='store', help='Container tests directory.') cmd_parser.add_argument('testfile', action='store', help='test file in the container') args = cmd_parser.parse_args(arg_vals) # Delete the rundata at the end of the script. atexit.register(delete_currentdata) logger.info(args) init_currentdata() init_db() snapshot_dir, snapshot_files = get_container_snapshot_json_files( args.container) if not snapshot_files: logger.info("No Snapshot files in %s, exiting!...", snapshot_dir) return False logger.info('Snapshot files: %s', snapshot_files) dbname = config_value(DATABASE, DBNAME) snapshot_ids = [] for fl in snapshot_files: snapshot_ids = populate_snapshots_from_file(fl) logger.debug(snapshot_ids) for sid, coll in snapshot_ids.items(): docs = get_documents(coll, {'snapshotId': sid}, dbname, sort=[('timestamp', pymongo.DESCENDING)], limit=1) logger.debug('Number of Snapshot Documents: %s', len(docs)) if docs and len(docs): doc = docs[0]['json'] logger.info('#' * 80) logger.info(json.dumps(doc, indent=2)) test6 = '%s/%s' % (get_container_dir(args.container), args.testfile) test_json = json_from_file(test6) if not test_json: return logger.debug(test_json) otherdata = {'dbname': dbname, 'snapshots': snapshot_ids} # for testcase in test_json['testSet'][0]['cases']: for testset in test_json['testSet']: for testcase in testset['cases']: rulestr = get_field_value(testcase, 'rule') if rulestr: main_comparator(rulestr, otherdata)
def is_parameter_file(self, file_path): """ check for valid parameter file for parse cloudformation template """ if len(file_path.split(".")) > 0 and file_path.split(".")[-1] in [ "json", "yaml" ]: json_data = json_from_file(file_path) if json_data and isinstance(json_data, list): parameter = json_data[0] if isinstance( parameter, dict ) and "ParameterKey" in parameter and "ParameterValue" in parameter: return True return False