def get_google_data(snapshot_source): """ The Google source object to be fetched from database or the filesystem The initial configuration for database is 'validator' and collection is 'structures', whereas for the filesystem the path to fetch the 'structures' is $SOLUTIONDIR/realm/<structure>.json """ sub_data = {} if json_source(): dbname = config_value(DATABASE, DBNAME) collection = config_value(DATABASE, collectiontypes[STRUCTURE]) parts = snapshot_source.split('.') qry = {'name': parts[0]} sort = [sort_field('timestamp', False)] docs = get_documents(collection, dbname=dbname, sort=sort, query=qry, limit=1) logger.info('Number of Google structure Documents: %d', len(docs)) if docs and len(docs): sub_data = docs[0]['json'] else: json_test_dir = get_test_json_dir() file_name = '%s.json' % snapshot_source if snapshot_source and not \ snapshot_source.endswith('.json') else snapshot_source google_source = '%s/../%s' % (json_test_dir, file_name) logger.info('Google source: %s', google_source) if exists_file(google_source): sub_data = json_from_file(google_source) if not sub_data: logger.error("Google connector file %s does not exist, or it does not contains the valid JSON.", snapshot_source) return sub_data
def get_snaphotid_doc_old(self, sid, container): doc = None json_dir = get_test_json_dir() if exists_dir(json_dir): fname = '%s/%s/snapshots/%s' % (json_dir, container, sid) if exists_file(fname): json_data = json_from_file(fname) if json_data and 'json' in json_data: doc = json_data['json'] snapshot = { 'id': json_data['snapshotId'], 'structure': json_data['structure'], 'reference': json_data['reference'], 'source': json_data['source'], 'collection': json_data['collection'], 'type': json_data.get("node", {}).get('type'), 'region' : json_data.get('region', "") } if 'paths' in json_data: snapshot['paths'] = json_data['paths'] else: snapshot['path'] = json_data['path'] self.snapshots.append(snapshot) return doc
def get_azure_data(snapshot_source): sub_data = {} if json_source(): dbname = config_value(DATABASE, DBNAME) collection = config_value(DATABASE, collectiontypes[STRUCTURE]) parts = snapshot_source.split('.') qry = {'name': parts[0]} sort = [sort_field('timestamp', False)] docs = get_documents(collection, dbname=dbname, sort=sort, query=qry, limit=1) logger.info('Number of Snapshot Documents: %s', len(docs)) if docs and len(docs): sub_data = docs[0]['json'] else: json_test_dir = get_test_json_dir() file_name = '%s.json' % snapshot_source if snapshot_source and not \ snapshot_source.endswith('.json') else snapshot_source azure_source = '%s/../%s' % (json_test_dir, file_name) logger.info('Azure source: %s', azure_source) if exists_file(azure_source): sub_data = json_from_file(azure_source) return sub_data
def get_connector_data(self): """ get connector data from snapshot """ connector_data = {} if self.snapshots: isdb_fetch = get_dbtests() if isdb_fetch: connectors = get_documents( "structures", query={ "name" : self.snapshots[0].get("source"), "type" : "structure", "container": self.container }, dbname=self.dbname, limit=1 ) connector_data = connectors[0].get("json", {}) if connectors else {} else: json_test_dir = get_test_json_dir() snapshot_source = self.snapshots[0].get("source") file_name = '%s.json' % snapshot_source if snapshot_source and not \ snapshot_source.endswith('.json') else snapshot_source connector_path = '%s/../%s' % (json_test_dir, file_name) if exists_file(connector_path): connector_data = json_from_file(connector_path) return connector_data
def make_snapshots_dir(container): snapshot_dir = None json_dir = '%s%s' % (get_test_json_dir(), container) if exists_dir(json_dir): snapshot_dir = '%s/snapshots' % json_dir mkdir_path(snapshot_dir) return snapshot_dir
def test_get_test_json_dir(): test_dir = os.getenv('TESTDIR', '/realm/validation/') val_dir = '%s/%s' % (TESTSDIR, test_dir) os.chdir(val_dir) tests_curdir = os.getcwd() os.chdir(get_test_json_dir()) prod_curdir = os.getcwd() assert tests_curdir == prod_curdir
def get_container_dir(container, tabs=1): """Translate container name to container directory""" json_test_dir = get_test_json_dir() logger.info('%s LOCATION: %s', '\t' * tabs, json_test_dir) container_dir = '%s/%s' % (json_test_dir, container) container_dir = container_dir.replace('//', '/') logger.info('%s COLLECTION: %s', '\t' * tabs, container_dir) return container_dir
def get_rego_rule_filename(rego_file, container): rego_file_name = None json_dir = get_test_json_dir() if exists_dir(json_dir): rego_file_name = '%s/%s/%s' % (json_dir, container, rego_file) if exists_file(rego_file_name): pass else: rego_file_name = None return rego_file_name
def get_snaphotid_doc(self, sid): doc = None isdb_fetch = get_dbtests() if isdb_fetch: dbname = self.dbname coll = self.collection_data[sid] if sid in self.collection_data else COLLECTION docs = get_documents(coll, {'snapshotId': sid}, dbname, sort=[('timestamp', pymongo.DESCENDING)], limit=1) logger.debug('Number of Snapshot Documents: %s', len(docs)) if docs and len(docs): doc = docs[0]['json'] snapshot = { 'id': docs[0]['snapshotId'], 'structure': docs[0]['structure'], 'reference': docs[0]['reference'], 'source': docs[0]['source'], 'collection': docs[0]['collection'], 'type': docs[0].get("node", {}).get('type'), 'region' : docs[0].get('region', "") } if 'paths' in docs[0]: snapshot['paths'] = docs[0]['paths'] else: snapshot['path'] = docs[0]['path'] self.snapshots.append(snapshot) else: json_dir = '%s%s' % (get_test_json_dir(), self.container) if exists_dir(json_dir): fname = '%s/snapshots/%s' % (json_dir, sid) if exists_file(fname): json_data = json_from_file(fname) if json_data and 'json' in json_data: doc = json_data['json'] snapshot_val = { 'id': json_data['snapshotId'], 'structure': json_data['structure'], 'reference': json_data['reference'], 'source': json_data['source'], 'collection': json_data['collection'], 'type': json_data.get("node", {}).get('type'), 'region' : json_data.get('region', "") } if 'paths' in json_data: snapshot_val['paths'] = json_data['paths'] else: snapshot_val['path'] = json_data['path'] singletest = get_from_currentdata(SINGLETEST) if singletest: snapshot_val['json'] = doc self.snapshots.append(snapshot_val) return doc
def get_structure_data(self, snapshot_object): """ Get the structure from the filesystem.""" structure_data = {} json_test_dir = get_test_json_dir() snapshot_source = get_field_value(snapshot_object, "source") file_name = '%s.json' % snapshot_source if snapshot_source and not \ snapshot_source.endswith('.json') else snapshot_source custom_source = '%s/../%s' % (json_test_dir, file_name) logger.info('%s structure file is %s', Snapshot.LOGPREFIX, custom_source) if exists_file(custom_source): structure_data = json_from_file(custom_source) return structure_data
def get_custom_data(snapshot_source): """ Get source JSON data """ sub_data = {} json_test_dir = get_test_json_dir() file_name = '%s.json' % snapshot_source if snapshot_source and not \ snapshot_source.endswith('.json') else snapshot_source custom_source = '%s/../%s' % (json_test_dir, file_name) logger.info('Custom source: %s', custom_source) if exists_file(custom_source): sub_data = json_from_file(custom_source) return sub_data
def get_snaphotid_doc(self, sid): doc = None isdb_fetch = get_dbtests() if isdb_fetch: dbname = self.kwargs['dbname'] coll = self.kwargs['snapshots'][sid] if sid in self.kwargs[ 'snapshots'] else COLLECTION docs = get_documents(coll, {'snapshotId': sid}, dbname, sort=[('timestamp', pymongo.DESCENDING)], limit=1) logger.debug('Number of Snapshot Documents: %s', len(docs)) if docs and len(docs): doc = docs[0]['json'] self.snapshots.append({ 'id': docs[0]['snapshotId'], 'path': docs[0]['path'], 'structure': docs[0]['structure'], 'reference': docs[0]['reference'], 'source': docs[0]['source'] }) else: json_dir = '%s%s' % (get_test_json_dir(), self.kwargs['container']) if exists_dir(json_dir): fname = '%s/snapshots/%s' % (json_dir, sid) if exists_file(fname): json_data = json_from_file(fname) if json_data and 'json' in json_data: doc = json_data['json'] # self.snapshots.append({ # 'id': json_data['snapshotId'], # 'path': json_data['path'], # 'structure': json_data['structure'], # 'reference': json_data['reference'], # 'source': json_data['source'] # }) snapshot_val = { 'id': json_data['snapshotId'], 'path': json_data['path'], 'structure': json_data['structure'], 'reference': json_data['reference'], 'source': json_data['source'] } singletest = get_from_currentdata(SINGLETEST) if singletest: snapshot_val['json'] = doc self.snapshots.append(snapshot_val) return doc
def get_snaphotid_doc_old(self, sid, container): doc = None json_dir = get_test_json_dir() if exists_dir(json_dir): fname = '%s/%s/snapshots/%s' % (json_dir, container, sid) if exists_file(fname): json_data = json_from_file(fname) if json_data and 'json' in json_data: doc = json_data['json'] self.snapshots.append({ 'id': json_data['snapshotId'], 'path': json_data['path'], 'structure': json_data['structure'], 'reference': json_data['reference'], 'source': json_data['source'] }) return doc
def get_snapshot_id_to_collection_dict(snapshot_file, container, dbname, filesystem=True): snapshot_data = {} snapshot_json_data = {} if filesystem: file_name = '%s.json' % snapshot_file if snapshot_file and not \ snapshot_file.endswith('.json') else snapshot_file snapshot_file = '%s/%s/%s' % (get_test_json_dir(), container, file_name) snapshot_json_data = json_from_file(snapshot_file) else: parts = snapshot_file.split('.') collection = config_value(DATABASE, collectiontypes[SNAPSHOT]) qry = {'container': container, 'name': parts[0]} sort = [sort_field('timestamp', False)] docs = get_documents(collection, dbname=dbname, sort=sort, query=qry, limit=1) logger.info('Number of Snapshot Documents: %s', len(docs)) if docs and len(docs): snapshot_json_data = docs[0]['json'] snapshots = get_field_value(snapshot_json_data, 'snapshots') if not snapshots: logger.info("Snapshot does not contain snapshots...") return snapshot_data for snapshot in snapshots: nodes = get_field_value(snapshot, 'nodes') if not nodes: logger.info("No nodes in snapshot, continuing to next!...") continue for node in nodes: sid = get_field_value(node, 'snapshotId') coll = node['collection'] if 'collection' in node else COLLECTION collection = coll.replace('.', '').lower() snapshot_data[sid] = collection if get_dbtests(): create_indexes(collection, dbname, [('timestamp', pymongo.TEXT)]) return snapshot_data
def rego_rule_filename(self, rego_file, container): rego_file_name = None if 'dirpath' in self.testcase and self.testcase['dirpath']: rego_file_name = '%s/%s' % (self.testcase['dirpath'], rego_file) if exists_file(rego_file_name): pass else: rego_file_name = None return rego_file_name isdb_fetch = get_dbtests() #It give same value for DB and SNAPSHOT, So for SNAPSHOT, we'll check it in #db first and if file isn't there, then we are fetching it from file path ''' if isdb_fetch: dbname = self.dbname coll = 'structures' docs = get_documents(coll, { 'type': 'others', "container" : container}, dbname, sort=[('timestamp', pymongo.DESCENDING)], limit=1) # print('Number of other Documents: %s' % len(docs)) logger.debug('Number of other Documents: %s', len(docs)) if docs and len(docs): doc = docs[0]['json'] if doc and 'file' in doc and isinstance(doc['file'], list): for file_doc in doc['file']: name = get_field_value(file_doc, 'name') # print(name, rego_file) if name == rego_file: content = get_field_value(file_doc, 'container_file') if content: rego_file_name = '/tmp/%s' % rego_file open(rego_file_name, 'w', encoding="utf-8").write(content) return rego_file_name # print(doc) json_dir = get_test_json_dir() if exists_dir(json_dir): rego_file_name = '%s/%s/%s' % (json_dir, container, rego_file) if exists_file(rego_file_name): pass else: rego_file_name = None return rego_file_name
def get_custom_data(snapshot_source, tabs=2): sub_data = {} if json_source(): container = get_from_currentdata('container') dbname = config_value(DATABASE, DBNAME) collection = config_value(DATABASE, collectiontypes[STRUCTURE]) parts = snapshot_source.split('.') qry = {'name': parts[0], 'container' : container } sort = [sort_field('timestamp', False)] docs = get_documents(collection, dbname=dbname, sort=sort, query=qry, limit=1) logger.info('Number of Custom Documents: %d', len(docs)) if docs and len(docs): sub_data = docs[0]['json'] else: json_test_dir = get_test_json_dir() file_name = '%s.json' % snapshot_source if snapshot_source and not \ snapshot_source.endswith('.json') else snapshot_source custom_source = '%s/../%s' % (json_test_dir, file_name) logger.info('\t\tCUSTOM CONNECTOR: %s ', custom_source) # logger.info('Custom source: %s', custom_source) if exists_file(custom_source): sub_data = json_from_file(custom_source) return sub_data