def dump_output_results(results, container, test_file, snapshot, filesystem=True): """ Dump the report in the json format for test execution results.""" od = OrderedDict() od["$schema"] = "" od["contentVersion"] = "1.0.0.0" od["fileType"] = OUTPUT od["timestamp"] = int(time.time() * 1000) od["snapshot"] = snapshot od["container"] = container dblog = get_dblogger() od["log"] = dblog if dblog else "" if filesystem: test_file_parts = test_file.rsplit('/', 1) od["test"] = test_file_parts[-1] output_file = '%s/output-%s' % (test_file_parts[0], test_file_parts[-1]) od["results"] = results save_json_to_file(od, output_file) else: od["test"] = test_file od["results"] = results del od["$schema"] doc = json_record(container, OUTPUT, test_file, od) dbname = config_value(DATABASE, DBNAME) collection = config_value(DATABASE, collectiontypes[OUTPUT]) insert_one_document(doc, collection, dbname)
def generate_crawler_run_output(container): """ This creates a entry in the output collection, whenever a crawler runs to fetch data. """ timestamp = int(time.time() * 1000) sort = [sort_field('timestamp', False)] qry = {'container': container} output_collection = config_value(DATABASE, collectiontypes[OUTPUT]) dbname = config_value(DATABASE, DBNAME) collection = config_value(DATABASE, collectiontypes[MASTERTEST]) tests = get_documents(collection, dbname=dbname, sort=sort, query=qry, _id=True) master_tests = [{ "id": str(test['_id']), "name": test['name'] } for test in tests] mastersp_collection = config_value(DATABASE, collectiontypes[MASTERSNAPSHOT]) snapshots = get_documents(mastersp_collection, dbname=dbname, sort=sort, query=qry, _id=True) master_snapshots = [{ "id": str(snapshot['_id']), "name": snapshot['name'] } for snapshot in snapshots] db_record = { "timestamp": timestamp, "checksum": hashlib.md5("{}".encode('utf-8')).hexdigest(), "collection": output_collection, "container": container, "name": "Crawlertest_%s" % (container), "type": "crawlerOutput", "json": { "container": container, "contentVersion": "", "fileType": "output", "snapshot": "", "test": "Crawlertest_%s" % (container), "log": get_dblog_handler().get_log_collection(), "timestamp": timestamp, "master_test_list": master_tests, "master_snapshot_list": master_snapshots, "output_type": "crawlerrun", "results": [] } } insert_one_document(db_record, db_record['collection'], dbname, False)
def create_container_json_to_db(dbname): container_json = {'filterType': 'container', 'containers': []} container = { 'collection': 'structures', 'container': '', 'name': '', 'type': 'container', 'timestamp': int(datetime.datetime.now().timestamp() * 1000), 'json': container_json } insert_one_document(container, container['collection'], dbname, False)
def store_data_node(self, data): """ Store to database""" if get_collection_size(data['collection']) == 0: # Creating indexes for collection create_indexes(data['collection'], config_value(DATABASE, DBNAME), [('snapshotId', pymongo.ASCENDING), ('timestamp', pymongo.DESCENDING)]) create_indexes(data['collection'], config_value(DATABASE, DBNAME), [('_id', pymongo.DESCENDING), ('timestamp', pymongo.DESCENDING), ('snapshotId', pymongo.ASCENDING)]) insert_one_document(data, data['collection'], self.dbname, check_keys=False)
def test_mongoconnection(monkeypatch): monkeypatch.setattr('processor.database.database.config_value', mock_config_value) monkeypatch.setattr('processor.database.database.get_dburl', mock_get_dburl) monkeypatch.setattr('processor.database.database.MongoClient', MyMongoClient) from processor.database.database import mongoconnection, mongodb, init_db,\ get_collection, collection_names, insert_one_document, insert_documents,\ check_document, get_documents, count_documents, index_information, distinct_documents # assert MONGO is None mongo = mongoconnection() assert mongo is not None dbname = 'abcd' testdb = mongodb(dbname) assert testdb is not None testdb = mongodb() assert testdb is not None val = testdb['abcd'] assert val is not None init_db() coll = get_collection(dbname, 'a1') assert coll is not None colls = collection_names(dbname) assert colls is not None val = insert_one_document({'a':'b'}, 'a1', dbname) assert val is not None val = distinct_documents('a1', 'a', dbname) assert val is not None with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) val = insert_one_document({'a': 'b'}, 'a1', dbname, False) assert val is not None val = '123456789012345678901234' # 24 character string doc = check_document('a1', val, dbname) assert doc is not None vals = insert_documents([{'a': 'b'}, {'c': 'd'}], 'a1', dbname) assert len(vals) == 2 vals = get_documents('a1',dbname=dbname, sort=None) assert vals is not None vals = get_documents('a1', dbname=dbname, sort='abcd') assert vals is not None count = count_documents('a1', dbname=dbname) assert type(count) is int assert count > 0 info = index_information('a1', dbname) assert info is not None mongo.drop_database(dbname)
def store_data_record(self): """ creates the indexes on collection and stores the data record in database or creates the generated snapshot at file system """ data_record = self.create_database_record() if get_dbtests(): if get_collection_size(data_record['collection']) == 0: #Creating indexes for collection create_indexes(data_record['collection'], config_value(DATABASE, DBNAME), [('snapshotId', pymongo.ASCENDING), ('timestamp', pymongo.DESCENDING)]) create_indexes(data_record['collection'], config_value(DATABASE, DBNAME), [('_id', pymongo.DESCENDING), ('timestamp', pymongo.DESCENDING), ('snapshotId', pymongo.ASCENDING)]) insert_one_document(data_record, data_record['collection'], self.dbname, check_keys=False) else: snapshot_dir = make_snapshots_dir(self.container) if snapshot_dir: store_snapshot(snapshot_dir, data_record) if 'masterSnapshotId' in self.node: self.snapshot_data[ self.node['snapshotId']] = self.node['masterSnapshotId'] else: self.snapshot_data[self.node['snapshotId']] = False if ( 'error' in data_record and data_record['error']) else True self.node['status'] = 'active'
def populate_custom_snapshot(snapshot, container=None): """ Populates the resources from git.""" dbname = config_value('MONGODB', 'dbname') snapshot_source = get_field_value(snapshot, 'source') connector_data = get_from_currentdata('connector') if connector_data: sub_data = get_custom_data(connector_data) if not sub_data: logger.error("No connector data found in '%s'", connector_data) else: sub_data = get_custom_data(snapshot_source) snapshot_nodes = get_field_value(snapshot, 'nodes') snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes) if valid_snapshotids and sub_data and snapshot_nodes: baserepo, repopath = _get_repo_path(sub_data, snapshot) if repopath: brnch = get_field_value_with_default(sub_data, 'branchName', 'master') for node in snapshot_nodes: node_type = node['type'] if 'type' in node and node['type'] else '' if node_type in TEMPLATE_NODE_TYPES: template_data = { "container" : container, "dbname" : dbname, "snapshot_source" : snapshot_source, "connector_data" : sub_data, "snapshot_data" : snapshot_data, "repopath" : repopath, "snapshot" : snapshot } template_processor = TEMPLATE_NODE_TYPES[node_type](node, **template_data) if 'snapshotId' in node: snapshot_data = template_processor.populate_template_snapshot() elif 'masterSnapshotId' in node: snapshot_data = template_processor.populate_all_template_snapshot() elif 'paths' in node: logger.error("ERROR: Invalid json : `%s` is not a valid node type." % (node_type)) else: # logger.debug(node) # data = get_node(repopath, node, snapshot_source, brnch) # if data: # insert_one_document(data, data['collection'], dbname) # snapshot_data[node['snapshotId']] = True validate = node['validate'] if 'validate' in node else True if 'snapshotId' in node: logger.debug(node) data = get_node(repopath, node, snapshot, brnch, sub_data) if data: if validate: if get_dbtests(): if get_collection_size(data['collection']) == 0: #Creating indexes for collection create_indexes(data['collection'], config_value(DATABASE, DBNAME), [('snapshotId', pymongo.ASCENDING), ('timestamp', pymongo.DESCENDING)]) create_indexes( data['collection'], config_value(DATABASE, DBNAME), [ ('_id', pymongo.DESCENDING), ('timestamp', pymongo.DESCENDING), ('snapshotId', pymongo.ASCENDING) ] ) insert_one_document(data, data['collection'], dbname) else: snapshot_dir = make_snapshots_dir(container) if snapshot_dir: store_snapshot(snapshot_dir, data) if 'masterSnapshotId' in node: snapshot_data[node['snapshotId']] = node['masterSnapshotId'] else: snapshot_data[node['snapshotId']] = True # else: # snapshot_data[node['snapshotId']] = False node['status'] = 'active' else: node['status'] = 'inactive' logger.debug('Type: %s', type(data)) elif 'masterSnapshotId' in node: alldata = get_all_nodes(repopath, node, snapshot, brnch, sub_data) if alldata: snapshot_data[node['masterSnapshotId']] = [] for data in alldata: snapshot_data[node['masterSnapshotId']].append( { 'snapshotId': data['snapshotId'], 'path': data['path'], 'validate': validate }) logger.debug('Type: %s', type(alldata)) if baserepo and os.path.exists(baserepo): # logger.info('\t\tCLEANING Repo: %s', baserepo) shutil.rmtree(baserepo) return snapshot_data
def populate_google_snapshot(snapshot, container=None): """ This is an entrypoint for populating a snapshot of type google. All snapshot connectors should take snapshot object and based on 'source' field create a method to connect to the service for the connector. The 'source' field could be used by more than one snapshot, so the 'testuser' attribute should match to the user the 'source' """ dbname = config_value('MONGODB', 'dbname') snapshot_source = get_field_value(snapshot, 'source') snapshot_user = get_field_value(snapshot, 'testUser') project_id = get_field_value(snapshot, 'project-id') sub_data = get_google_data(snapshot_source) snapshot_nodes = get_field_value(snapshot, 'nodes') snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes) if valid_snapshotids and sub_data and snapshot_nodes: logger.debug(sub_data) try: for node in snapshot['nodes']: validate = node['validate'] if 'validate' in node else True logger.info(node) node_type = get_field_value_with_default(node, 'type',"") credentials = get_google_client_data(sub_data, snapshot_user, node_type, project_id) if not credentials: logger.info("No GCE connection in the snapshot to access Google resource!...") return snapshot_data if 'snapshotId' in node: if validate: data = get_node(credentials, node, snapshot_source, snapshot) if data: error_str = data.pop('error', None) if get_dbtests(): if get_collection_size(data['collection']) == 0: #Creating indexes for collection create_indexes( data['collection'], config_value(DATABASE, DBNAME), [ ('snapshotId', pymongo.ASCENDING), ('timestamp', pymongo.DESCENDING) ] ) create_indexes( data['collection'], config_value(DATABASE, DBNAME), [ ('_id', pymongo.DESCENDING), ('timestamp', pymongo.DESCENDING), ('snapshotId', pymongo.ASCENDING) ] ) insert_one_document(data, data['collection'], dbname) else: snapshot_dir = make_snapshots_dir(container) if snapshot_dir: store_snapshot(snapshot_dir, data) if 'masterSnapshotId' in node: snapshot_data[node['snapshotId']] = node['masterSnapshotId'] else: snapshot_data[node['snapshotId']] = False if error_str else True else: node['status'] = 'inactive' elif 'masterSnapshotId' in node: data = get_all_nodes(credentials, node, snapshot_source, snapshot, snapshot_data) logger.debug('Type: %s', type(data)) except Exception as ex: logger.info('Unable to create Google client: %s', ex) raise ex return snapshot_data
def populate_azure_snapshot(snapshot, container=None, snapshot_type='azure'): """ Populates the resources from azure.""" dbname = config_value('MONGODB', 'dbname') snapshot_source = get_field_value(snapshot, 'source') snapshot_user = get_field_value(snapshot, 'testUser') snapshot_nodes = get_field_value(snapshot, 'nodes') snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes) client_id, client_secret, sub_name, sub_id, tenant_id = \ get_web_client_data(snapshot_type, snapshot_source, snapshot_user) if not client_id: # logger.info("No client_id in the snapshot to access azure resource!...") raise Exception("No client id in the snapshot to access azure resource!...") # Read the client secrets from envirnment variable if not client_secret: client_secret = os.getenv(snapshot_user, None) if client_secret: logger.info('Client Secret from environment variable, Secret: %s', '*' * len(client_secret)) # Read the client secrets from the vault if not client_secret: client_secret = get_vault_data(client_id) if client_secret: logger.info('Client Secret from Vault, Secret: %s', '*' * len(client_secret)) elif get_from_currentdata(CUSTOMER): logger.error("Client Secret key does not set in a vault") raise Exception("Client Secret key does not set in a vault") if not client_secret: raise Exception("No `client_secret` key in the connector file to access azure resource!...") logger.info('\t\tSubscription: %s', sub_id) logger.info('\t\tTenant: %s', tenant_id) logger.info('\t\tclient: %s', client_id) put_in_currentdata('clientId', client_id) put_in_currentdata('clientSecret', client_secret) put_in_currentdata('subscriptionId', sub_id) put_in_currentdata('tenant_id', tenant_id) token = get_access_token() logger.debug('TOKEN: %s', token) if not token: logger.info("Unable to get access token, will not run tests....") raise Exception("Unable to get access token, will not run tests....") # return {} # snapshot_nodes = get_field_value(snapshot, 'nodes') # snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes) if valid_snapshotids and token and snapshot_nodes: for node in snapshot_nodes: validate = node['validate'] if 'validate' in node else True if 'path' in node: data = get_node(token, sub_name, sub_id, node, snapshot_user, snapshot_source) if data: if validate: if get_dbtests(): if get_collection_size(data['collection']) == 0: # Creating indexes for collection create_indexes( data['collection'], config_value(DATABASE, DBNAME), [ ('snapshotId', pymongo.ASCENDING), ('timestamp', pymongo.DESCENDING) ] ) create_indexes( data['collection'], config_value(DATABASE, DBNAME), [ ('_id', pymongo.DESCENDING), ('timestamp', pymongo.DESCENDING), ('snapshotId', pymongo.ASCENDING) ] ) insert_one_document(data, data['collection'], dbname, check_keys=False) else: snapshot_dir = make_snapshots_dir(container) if snapshot_dir: store_snapshot(snapshot_dir, data) if 'masterSnapshotId' in node: snapshot_data[node['snapshotId']] = node['masterSnapshotId'] else: snapshot_data[node['snapshotId']] = True # else: # snapshot_data[node['snapshotId']] = False node['status'] = 'active' else: # TODO alert if notification enabled or summary for inactive. node['status'] = 'inactive' logger.debug('Type: %s', type(data)) else: alldata = get_all_nodes( token, sub_name, sub_id, node, snapshot_user, snapshot_source) if alldata: snapshot_data[node['masterSnapshotId']] = [] for data in alldata: # insert_one_document(data, data['collection'], dbname) found_old_record = False for masterSnapshotId, snapshot_list in snapshot_data.items(): old_record = None if isinstance(snapshot_list, list): for item in snapshot_list: if item["path"] == data['path']: old_record = item if old_record: found_old_record = True if node['masterSnapshotId'] not in old_record['masterSnapshotId']: old_record['masterSnapshotId'].append( node['masterSnapshotId']) if not found_old_record: snapshot_data[node['masterSnapshotId']].append( { 'masterSnapshotId': [node['masterSnapshotId']], 'snapshotId': data['snapshotId'], 'path': data['path'], 'validate': validate, 'status': 'active' }) # snapshot_data[node['masterSnapshotId']] = True logger.debug('Type: %s', type(alldata)) delete_from_currentdata('resources') delete_from_currentdata('clientId') delete_from_currentdata('client_secret') delete_from_currentdata('subscriptionId') delete_from_currentdata('tenant_id') delete_from_currentdata('token') return snapshot_data
def populate_aws_snapshot(snapshot, container=None): """ This is an entrypoint for populating a snapshot of type aws. All snapshot connectors should take snapshot object and based on 'source' field create a method to connect to the service for the connector. The 'source' field could be used by more than one snapshot, so the 'testuser' attribute should match to the user the 'source' """ dbname = config_value('MONGODB', 'dbname') snapshot_source = get_field_value(snapshot, 'source') snapshot_user = get_field_value(snapshot, 'testUser') account_id = get_field_value(snapshot, 'accountId') sub_data = get_aws_data(snapshot_source) snapshot_nodes = get_field_value(snapshot, 'nodes') snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes) # valid_snapshotids = True # if snapshot_nodes: # for node in snapshot_nodes: # snapshot_data[node['snapshotId']] = False # if not isinstance(node['snapshotId'], str): # valid_snapshotids = False # if not valid_snapshotids: # logger.error('All snap') if valid_snapshotids and sub_data and snapshot_nodes: logger.debug(sub_data) access_key, secret_access, region, connector_client_str = \ get_aws_client_data(sub_data, snapshot_user, account_id) if not access_key: logger.info( "No access_key in the snapshot to access aws resource!...") raise Exception( "No access_key in the snapshot to access aws resource!...") # return snapshot_data # Read the client secrets from envirnment variable or Standard input # if not secret_access and ('UAMI' not in os.environ or os.environ['UAMI'] != 'true'): # secret_access = get_client_secret() # logger.info('Environment variable or Standard input, Secret: %s', '*' * len(secret_access)) # Read the client secrets from the vault if not secret_access: secret_access = get_vault_data(access_key) if secret_access: logger.info('Vault Secret: %s', '*' * len(secret_access)) else: logger.info("Secret Access key does not set in a vault") raise Exception("Secret Access key does not set in a vault") if not secret_access: logger.info( "No secret_access in the snapshot to access aws resource!...") return snapshot_data if access_key and secret_access: # existing_aws_client = {} for node in snapshot['nodes']: mastercode = False if 'snapshotId' in node: client_str, aws_region = _get_aws_client_data_from_node( node, default_client=connector_client_str, default_region=region) if not _validate_client_name(client_str): logger.error("Invalid Client Name") return snapshot_data try: awsclient = client(client_str.lower(), aws_access_key_id=access_key, aws_secret_access_key=secret_access, region_name=aws_region) except Exception as ex: logger.info('Unable to create AWS client: %s', ex) awsclient = None logger.info(awsclient) if awsclient: data = get_node(awsclient, node, snapshot_source) if data: error_str = data.pop('error', None) if get_dbtests(): if get_collection_size( data['collection']) == 0: #Creating indexes for collection create_indexes( data['collection'], config_value(DATABASE, DBNAME), [('snapshotId', pymongo.ASCENDING), ('timestamp', pymongo.DESCENDING)]) check_key = is_check_keys_required(data) insert_one_document(data, data['collection'], dbname, check_key) else: snapshot_dir = make_snapshots_dir(container) if snapshot_dir: store_snapshot(snapshot_dir, data) if 'masterSnapshotId' in node: snapshot_data[node['snapshotId']] = node[ 'masterSnapshotId'] else: snapshot_data[node[ 'snapshotId']] = False if error_str else True elif 'masterSnapshotId' in node: mastercode = True client_str, aws_region = _get_aws_client_data_from_node( node, default_client=connector_client_str, default_region=region) if not _validate_client_name(client_str): logger.error("Invalid Client Name") return snapshot_data if aws_region: all_regions = [aws_region] else: all_regions = Session().get_available_regions( client_str.lower()) if client_str.lower() in ['s3', 'cloudtrail']: all_regions = ['us-west-1'] logger.info("Length of all regions is %s" % (str(len(all_regions)))) count = 0 snapshot_data[node['masterSnapshotId']] = [] for each_region in all_regions: logger.info(each_region) try: awsclient = client( client_str.lower(), aws_access_key_id=access_key, aws_secret_access_key=secret_access, region_name=each_region) except Exception as ex: logger.info('Unable to create AWS client: %s', ex) logger.info(awsclient) if awsclient: all_data = get_all_nodes(awsclient, node, snapshot, sub_data) if all_data: for data in all_data: snapshot_data[ node['masterSnapshotId']].append({ 'snapshotId': '%s%s' % (node['masterSnapshotId'], str(count)), 'validate': True, 'detailMethods': data['detailMethods'], 'structure': 'aws', 'masterSnapshotId': node['masterSnapshotId'], 'collection': data['collection'], 'arn': data['arn'] }) count += 1 if mastercode: snapshot_data = eliminate_duplicate_snapshots(snapshot_data) return snapshot_data
def populate_sub_directory_snapshot(base_dir_path, sub_dir_path, snapshot, dbname, node, snapshot_data): dir_path = str('%s/%s' % (base_dir_path, sub_dir_path)).replace('//', '/') if exists_dir(dir_path): list_of_file = os.listdir(dir_path) template_file_path = "" deployment_file_path_list = [] for entry in list_of_file: new_dir_path = ('%s/%s' % (dir_path, entry)).replace('//', '/') new_sub_directory_path = ('%s/%s' % (sub_dir_path, entry)).replace( '//', '/') if exists_dir(new_dir_path): populate_sub_directory_snapshot(base_dir_path, new_sub_directory_path, snapshot, dbname, node, snapshot_data) elif exists_file(new_dir_path): if len(entry.split(".")) > 0 and "json" in entry.split( ".")[-1]: json_data = json_from_file(new_dir_path) if json_data and "$schema" in json_data: if "deploymentTemplate.json" in json_data[ '$schema'].split("/")[-1]: template_file_path = new_sub_directory_path elif "deploymentParameters.json" in json_data[ '$schema'].split("/")[-1]: deployment_file_path_list.append( new_sub_directory_path) if template_file_path and deployment_file_path_list: location = get_field_value(node, 'location') new_deployment_file_path_list = [] template_file_json_path = str( '%s/%s' % (base_dir_path, template_file_path)).replace( '//', '/') for deployment_file_path in deployment_file_path_list: deployment_file_json_path = str( '%s/%s' % (base_dir_path, deployment_file_path)).replace( '//', '/') response = invoke_az_cli("deployment validate --location " + location + " --template-file " + template_file_json_path + " --parameters @" + deployment_file_json_path) if not response['error']: new_deployment_file_path_list.append({ "path": deployment_file_path, "status": "active" }) else: new_deployment_file_path_list.append({ "path": deployment_file_path, "status": "inactive" }) data_record = create_snapshot_record( snapshot, new_sub_directory_path, node, template_file_path, new_deployment_file_path_list) if node['masterSnapshotId'] not in snapshot_data or not isinstance( snapshot_data[node['masterSnapshotId']], list): snapshot_data[node['masterSnapshotId']] = [] snapshot_data[node['masterSnapshotId']] = snapshot_data[node[ 'masterSnapshotId']] + data_record['snapshots'][0]['nodes'] if get_dbtests(): insert_one_document(data_record, node['collection'], dbname) else: snapshot_file = '%s/%s' % (dir_path, "snapshot.json") save_json_to_file(data_record, snapshot_file)
def populate_arm_snapshot(container, dbname, snapshot_source, sub_data, snapshot_data, node, repopath): """ Populate snapshot by running arm command """ dir_path = get_field_value(sub_data, 'folderPath') if not dir_path: dir_path = repopath location = get_field_value(node, 'location') paths = get_field_value(node, 'paths') template_file_path = "" deployment_file_path = "" if paths and isinstance(paths, list): if not location: logger.error("Invalid json : 'location' field is required in node") node['status'] = 'inactive' return snapshot_data for json_file in paths: json_file_path = '%s/%s.json' % (dir_path, json_file) json_data = json_from_file(json_file_path) if not json_data: logger.error("Invalid path or json") node['status'] = 'inactive' return snapshot_data elif "$schema" not in json_data: logger.error( "Invalid json : does not contains '$schema' field in json." ) node['status'] = 'inactive' return snapshot_data else: if "deploymentTemplate.json" in json_data['$schema'].split( "/")[-1]: template_file_path = json_file_path elif "deploymentParameters.json" in json_data['$schema'].split( "/")[-1]: deployment_file_path = json_file_path else: logger.error( "Invalid json : $schema does not contains the correct value" ) if template_file_path and deployment_file_path: response = invoke_az_cli("deployment validate --location " + location + " --template-file " + template_file_path + " --parameters @" + deployment_file_path) data_record = create_database_record(node, snapshot_source, response, sub_data) if get_dbtests(): if get_collection_size(node['collection']) == 0: #Creating indexes for collection create_indexes(node['collection'], config_value(DATABASE, DBNAME), [('snapshotId', pymongo.ASCENDING), ('timestamp', pymongo.DESCENDING)]) insert_one_document(data_record, node['collection'], dbname) else: snapshot_dir = make_snapshots_dir(container) if snapshot_dir: store_snapshot(snapshot_dir, data_record) snapshot_data[ node['snapshotId']] = False if data_record['error'] else True node['status'] = 'active' else: node['status'] = 'inactive' else: node['status'] = 'inactive' logger.error( "Invalid json : `paths` field is missing for 'arm' node type or it is not a list" ) return snapshot_data
def populate_custom_snapshot(snapshot, container=None): """ Populates the resources from git.""" dbname = config_value('MONGODB', 'dbname') snapshot_source = get_field_value(snapshot, 'source') sub_data = get_custom_data(snapshot_source) snapshot_nodes = get_field_value(snapshot, 'nodes') snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes) if valid_snapshotids and sub_data and snapshot_nodes: baserepo, repopath = _get_repo_path(sub_data, snapshot) if repopath: brnch = get_field_value_with_default(sub_data, 'branchName', 'master') for node in snapshot_nodes: node_type = node[ 'type'] if 'type' in node and node['type'] else 'json' if node_type == 'arm': if 'snapshotId' in node: populate_arm_snapshot(container, dbname, snapshot_source, sub_data, snapshot_data, node, repopath) elif 'masterSnapshotId' in node: populate_all_arm_snapshot(snapshot, dbname, sub_data, node, repopath, snapshot_data) else: # logger.debug(node) # data = get_node(repopath, node, snapshot_source, brnch) # if data: # insert_one_document(data, data['collection'], dbname) # snapshot_data[node['snapshotId']] = True validate = node['validate'] if 'validate' in node else True if 'snapshotId' in node: logger.debug(node) data = get_node(repopath, node, snapshot, brnch, sub_data) if data: if validate: if get_dbtests(): if get_collection_size( data['collection']) == 0: #Creating indexes for collection create_indexes( data['collection'], config_value(DATABASE, DBNAME), [('snapshotId', pymongo.ASCENDING), ('timestamp', pymongo.DESCENDING) ]) insert_one_document( data, data['collection'], dbname) else: snapshot_dir = make_snapshots_dir( container) if snapshot_dir: store_snapshot(snapshot_dir, data) if 'masterSnapshotId' in node: snapshot_data[node['snapshotId']] = node[ 'masterSnapshotId'] else: snapshot_data[node['snapshotId']] = True else: snapshot_data[node['snapshotId']] = False node['status'] = 'active' else: node['status'] = 'inactive' logger.debug('Type: %s', type(data)) elif 'masterSnapshotId' in node: alldata = get_all_nodes(repopath, node, snapshot, brnch, sub_data) if alldata: snapshot_data[node['masterSnapshotId']] = [] for data in alldata: snapshot_data[node['masterSnapshotId']].append( { 'snapshotId': data['snapshotId'], 'path': data['path'], 'validate': True }) logger.debug('Type: %s', type(alldata)) if baserepo and os.path.exists(baserepo): logger.info('Repo path: %s', baserepo) shutil.rmtree(baserepo) return snapshot_data
def populate_json_files(args): dbname = config_value(DATABASE, DBNAME) containerId = None if args.container: container_struture_list = get_documents('structures', {'type': 'container'}, dbname) if not container_struture_list: # create container_json create_container_json_to_db(dbname) container_struture_list = get_documents('structures', {'type': 'container'}, dbname) container_json = container_struture_list[0]['json'] container_list = container_json['containers'] filtered_list = list( filter(lambda i: i['name'] == args.container, container_list)) if not filtered_list: # add new container if container not exist add_new_container(args.container, dbname) container_struture_list = get_documents('structures', {'type': 'container'}, dbname) container_json = container_struture_list[0]['json'] container_list = container_json['containers'] filtered_list = list( filter(lambda i: i['name'] == args.container, container_list)) containerId = filtered_list[0]['containerId'] # return containerId # if args.dir: # logger.info("Checking this directory: %s for json files", args.dir) # json_dir = args.dir # if exists_dir(args.dir): # for filename in glob.glob('%s/*.json' % json_dir.replace('//', '/')): # json_data = json_from_file(filename) # if json_data and 'fileType' in json_data: # filetype = json_data['fileType'] # else: # filetype = 'structure' # logger.info('Storing file:%s from directory: %s', json_dir, filename) # db_record = json_record(args.container, filetype, filename, json_data) # if validate_json_data(db_record['json'], db_record['type']): # insert_one_document(db_record, db_record['collection'], dbname, False) # logger.debug('DB Record: %s', json.dumps(db_record, indent=2)) # else: # logger.info('Invalid json for type:%s', db_record['type']) # logger.info('*' * 80) if args.file: logger.info("Populating %s json file.", args.file) json_file = args.file if exists_file(json_file): if json_file.endswith('.json'): json_data = json_from_file(json_file) if json_data and 'fileType' in json_data: filetype = json_data['fileType'] # elif args.type: # filetype = args.type else: filetype = 'structure' logger.info('Storing file:%s', json_file) db_record = json_record(args.container, filetype, json_file, json_data) if validate_json_data(db_record['json'], db_record['type']): docId = insert_one_document(db_record, db_record['collection'], dbname, False) data = { 'object_id': ObjectId(docId), 'name': db_record['name'] } if filetype == 'masterSnapshot': save_container_object(args.container, 'masterSnapshots', data, dbname) elif filetype == 'mastertest': save_container_object(args.container, 'masterTests', data, dbname) elif filetype == 'snapshot': save_container_object(args.container, 'Snapshots', data, dbname) elif filetype == 'test': save_container_object(args.container, 'Tests', data, dbname) logger.debug('DB Record: %s', json.dumps(db_record, indent=2)) else: logger.info('Invalid json for type:%s', db_record['type']) logger.info('*' * 80) elif json_file.endswith('.rego'): with open(json_file) as f: file_content = f.read() content_type = 'application/octet-stream' save_container_to_db(args.container, containerId, json_file, content_type, file_content, dbname)
def generate_container_mastersnapshots_database(container): """ Get the mastersnapshot files from the container with storage system as database. The table or collection and database is configured in the config.ini, for the default location configuration is "validator" database with "mastersnapshots" as its collections. """ snapshots_status = {} dbname = config_value(DATABASE, DBNAME) collection = config_value(DATABASE, collectiontypes[MASTERSNAPSHOT]) snp_collection = config_value(DATABASE, collectiontypes[SNAPSHOT]) qry = {'container': container} sort = [sort_field('timestamp', False)] docs = get_documents(collection, dbname=dbname, sort=sort, query=qry) try: if docs and len(docs): logger.info('Number of mastersnapshot Documents: %s', len(docs)) snapshots = mastersnapshots_used_in_mastertests_database(container) populated = [] for doc in docs: if doc['json']: snapshot = doc['name'] if "connector" in doc['json'] and "remoteFile" in doc[ 'json'] and doc['json']["connector"] and doc[ 'json']["remoteFile"]: _, pull_response = pull_json_data(doc['json']) if not pull_response: logger.info( "Failed to populate master snapshot json from the git repository" ) break if snapshot in snapshots: if snapshot not in populated: snp_collection = config_value( DATABASE, collectiontypes[SNAPSHOT]) snp_name = '%s_gen' % snapshot snp_qry = { 'container': container, 'name': snp_name } snp_sort = [sort_field('timestamp', False)] snp_docs = get_documents(snp_collection, dbname=dbname, sort=snp_sort, query=snp_qry, _id=True) snp_json_data = {} if snp_docs and len(snp_docs): logger.info('Number of snapshot Documents: %s', len(snp_docs)) snp_json_data = snp_docs[0] # Take the mastersnapshot and populate the mastersnapshot snapshot_file_data = generate_mastersnapshots_from_json( doc['json'], snp_json_data) # Insert or update the new generated snapshot document with name='*_gen' and same container name. generate_snapshot(doc['json'], snapshot_file_data) if snp_json_data: set_snapshot_activate_and_validate_data( doc['json'], snp_json_data['json']) snp_json_data['json'] = doc['json'] snp_json_data["timestamp"] = int(time.time() * 1000) update_one_document( snp_json_data, snp_json_data['collection'], dbname) else: db_record = { "timestamp": int(time.time() * 1000), "container": container, "checksum": hashlib.md5( "{}".encode('utf-8')).hexdigest(), "type": "snapshot", "name": snp_name, "collection": "snapshots", "json": doc['json'] } insert_one_document(db_record, db_record['collection'], dbname, False) populated.append(snapshot) snapshots_status[snapshot] = snapshot_file_data else: logger.error("No master testcase found for %s " % snapshot) except Exception as e: generate_crawler_run_output(container) raise e generate_crawler_run_output(container) return snapshots_status
def populate_kubernetes_snapshot(snapshot, container=None): snapshot_nodes = get_field_value(snapshot,'nodes') snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes) dbname = config_value('MONGODB', 'dbname') if valid_snapshotids and snapshot_nodes: logger.debug(valid_snapshotids) try : for node in snapshot_nodes: validate = node['validate'] if 'validate' in node else True logger.info(node) if 'snapshotId' in node: if validate: kubernetes_snapshot_data = get_kubernetes_snapshot_data(snapshot,node) if kubernetes_snapshot_data : error_str = kubernetes_snapshot_data.pop('error', None) kubernetes_snapshot_template = make_kubernetes_snapshot_template( snapshot, node, kubernetes_snapshot_data ) if get_dbtests(): if get_collection_size(kubernetes_snapshot_template['collection']) == 0: #Creating indexes for collection create_indexes( kubernetes_snapshot_template['collection'], config_value(DATABASE, DBNAME), [ ('snapshotId', pymongo.ASCENDING), ('timestamp', pymongo.DESCENDING) ] ) create_indexes( kubernetes_snapshot_template['collection'], config_value(DATABASE, DBNAME), [ ('_id', pymongo.DESCENDING), ('timestamp', pymongo.DESCENDING), ('snapshotId', pymongo.ASCENDING) ] ) insert_one_document(kubernetes_snapshot_template, kubernetes_snapshot_template['collection'], dbname,check_keys=False) snapshot_dir = make_snapshots_dir(container) if snapshot_dir: store_snapshot(snapshot_dir, kubernetes_snapshot_template) if "masterSnapshotId" in node : snapshot_data[node['snapshotId']] = node['masterSnapshotId'] elif "snapshotId" in node : snapshot_data[node['snapshotId']] = False if error_str else True else: node['status'] = 'inactive' elif 'masterSnapshotId' in node: snapshot_data = generate_crawler_snapshot(snapshot,node,snapshot_data) except Exception as ex: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logger.error('can not connect to kubernetes cluster: %s ', ex ) logger.error('\t ERROR INFO : \n \tfile name : %s\n \tline : %s\n \ttype : %s\n \tobject : %s',fname,exc_tb.tb_lineno,exc_type,exc_obj) print(traceback.format_exc()) raise ex return snapshot_data
def save_container_to_db(container_name, container_id, file_name, content_type, file_content, dbname): file_content_list = [] structure_model_obj = get_documents('structures', { 'json.containerId': container_id, 'type': 'others' }, dbname, _id=True) file_obj = { 'name': file_name, 'container_file': file_content, 'content_type': content_type } if structure_model_obj: exist = False for file_data in structure_model_obj[0]['json']['file']: for key, value in file_data.items(): if value == file_name: exist = True if exist: for file_data in structure_model_obj[0]['json']['file']: for key, value in file_data.items(): if value == file_name: file_data['container_file'] = file_content else: structure_model_obj[0]['json']['file'].append(file_obj) update_one_document(structure_model_obj[0], 'structures', dbname) # print(structure_model_obj) data = {'object_id': structure_model_obj[0]['_id'], 'name': file_name} else: file_obj = { 'name': file_name, 'container_file': file_content, 'content_type': content_type } file_content_list.append(file_obj) container_json = { 'name': container_name, 'containerId': container_id, 'file': file_content_list } structure_model_obj = { 'checksum': '', 'collection': 'structures', 'container': container_name, 'name': 'file_upload', 'timestamp': int(datetime.datetime.now().timestamp() * 1000), 'type': 'others', 'json': container_json } docId = insert_one_document(structure_model_obj, structure_model_obj['collection'], dbname, False) # print(docId) data = {'object_id': ObjectId(docId), 'name': file_name} save_container_object(container_name, 'others', data, dbname)