def test_get_vault_data_cyberark(monkeypatch): monkeypatch.setattr('processor.connector.vault.config_value', mock_config_value_cybeark) monkeypatch.setattr('processor.connector.vault.Popen', Popen) from processor.connector.vault import get_vault_data val = get_vault_data(None) assert val == 'secret' val = get_vault_data('abcd') assert val == 'secret'
def test_get_vault_data(monkeypatch): monkeypatch.setattr('processor.connector.vault.config_value', mock_config_value) monkeypatch.setattr('processor.connector.vault.get_vault_access_token', mock_get_vault_access_token) monkeypatch.setattr('processor.connector.vault.get_keyvault_secret', mock_get_keyvault_secret) from processor.connector.vault import get_vault_data val = get_vault_data(None) assert val is None val = get_vault_data('abcd') assert val == 'secret'
def generate_gce(google_data, project, user): """ Generate client secret json from the google data """ logger.info("Generating GCE") gce = { "type": get_field_value(user, "type"), "project_id": get_field_value(project, "project-id"), "private_key_id": get_field_value(user, "private_key_id"), "private_key": get_field_value(user, "private_key"), "client_email": get_field_value(user, "client_email"), "client_id": get_field_value(user, "client_id"), "auth_uri": get_field_value(google_data, "auth_uri"), "token_uri": get_field_value(google_data, "token_uri"), "auth_provider_x509_cert_url": get_field_value(google_data, "auth_provider_x509_cert_url"), "client_x509_cert_url": get_field_value(user, "client_x509_cert_url"), } # Read the private key from the key path if not gce['private_key'] and get_field_value(user, "private_key_path"): private_key_path = get_field_value(user, "private_key_path") logger.info("Private key path : %s ", private_key_path) try: gce['private_key'] = open(private_key_path, 'r', encoding="utf-8").read().replace("\\n","\n") if gce['private_key']: logger.info('Private key from Private key path, Secret: %s', '*' * len(gce['private_key'])) except Exception as e: raise Exception("Private key does not exist at given private key path : %s " % str(e)) if not gce['private_key']: raise Exception("Private key does not exist at given private key path : %s " % private_key_path) # Read the private key from the vault if not gce['private_key']: private_key = get_vault_data(gce['private_key_id']) if private_key: gce["private_key"] = private_key.replace("\\n","\n") if gce["private_key"]: logger.info('Private key from vault Secret: %s', '*' * len(gce["private_key"])) elif get_from_currentdata(CUSTOMER): raise Exception("Private key does not set in a vault") if not gce['private_key']: raise Exception("No `private_key` field in the connector file to access Google resource!...") if (None in list(gce.values())): raise Exception("Connector file does not contains valid values or some fields are missing for access Google resources.") return gce
def get_private_key(gce): """ Fetches the Private Key for get the google service account credentials """ if ('UAMI' not in os.environ or os.environ['UAMI'] != 'true'): # if private_key does not exist then it will set to None: gce["private_key"] = get_field_value(gce, 'private_key') if ('UAMI' in os.environ and os.environ['UAMI'] == 'true') or not gce["private_key"]: private_key = get_vault_data(gce['private_key_id']) if private_key: gce["private_key"] = private_key.replace("\\n", "\n") else: raise Exception("Private key does not set in a vault") return gce
def generate_azure_vault_key(): key = input("Enter the key to add or update its password: "******"Regenerating password for key: ", key) elif is_created: set_key_visbility(key, EDITABLE) print("Creating and generating password for key: ", key) else: print("Getting issue while generating key:", key)
def get_client_secret(kubernetes_structure_data,snapshot_serviceAccount,snapshot_namespace): """ get_client_secret get service account from master snapshot and will compare with other service accounts which allocated in kubernetes structure file and get secret of service account if it’s exist in structure file. Also check environment variables if service account secret isn’t exist in structure file with the name got from snap shot file. This function return secret as string which will use to get connection with kubernetes cluster. """ global Cache_namespace,Cache_secret if snapshot_namespace == Cache_namespace: return Cache_secret namespaces = get_field_value(kubernetes_structure_data,'namespaces') service_account_secret = "" for namespace in namespaces : service_accounts = get_field_value(namespace,'serviceAccounts') for service_account in service_accounts : if snapshot_serviceAccount == service_account['name'] and namespace['namespace'] in snapshot_namespace : service_account_secret = get_field_value(service_account,'secret') if service_account_secret is not None: Cache_secret= service_account_secret Cache_namespace = snapshot_namespace return service_account_secret else : service_account_secret = get_vault_data(service_account['id']) if service_account_secret is not None: Cache_secret= service_account_secret Cache_namespace = snapshot_namespace if service_account_secret == "" : logger.error("\t\t ERROR : can not find secret for service account : %s" % (snapshot_serviceAccount)) return service_account_secret
def git_clone_dir(connector): clonedir = None repopath = tempfile.mkdtemp() subdir = False if connector and isinstance(connector, dict): giturl = get_field_value(connector, 'gitProvider') if not giturl: logger.error("Git connector does not have valid git provider URL") return repopath, clonedir branch = get_from_currentdata('branch') if not branch: branch = get_field_value_with_default(connector, 'branchName', 'master') isprivate = get_field_value(connector, 'private') isprivate = True if isprivate is None or not isinstance(isprivate, bool) else isprivate # logger.info("Repopath: %s", repopath) logger.info("\t\t\tRepopath: %s", repopath) http_match = re.match(r'^http(s)?://', giturl, re.I) if http_match: logger.info("\t\t\tHttp (private:%s) giturl: %s", "YES" if isprivate else "NO", giturl) accessToken = get_field_value(connector, 'httpsAccessToken') username = get_field_value(connector, 'httpsUser') if accessToken: logger.info("AccessToken: %s" % accessToken) pwd = get_field_value(connector, 'httpsPassword') pwd = pwd if pwd else get_git_pwd(key=accessToken) if not pwd: pwd = get_pwd_from_vault(accessToken) if pwd: logger.info("Git access token from vault: %s", '*' * len(pwd)) if pwd: gh = GithubFunctions() gh.set_access_token(pwd) _ = gh.populate_user() rpo = gh.clone_repo(giturl, repopath, branch) if rpo: logger.info('Successfully cloned in %s dir' % repopath) checkdir = '%s/tmpclone' % repopath if subdir else repopath clonedir = checkdir if exists_dir('%s/.git' % checkdir) else None if not exists_dir(clonedir): logger.error("No valid data provided for connect to git : %s", giturl) return repopath, clonedir elif isprivate: logger.error("Please provide password for connect to git repository.") return repopath, clonedir else: git_cmd = 'git clone %s %s' % (giturl, repopath) elif username: pwd = get_field_value(connector, 'httpsPassword') schema = giturl[:http_match.span()[-1]] other_part = giturl[http_match.span()[-1]:] # pwd = pwd if (pwd and not json_source()) else (get_git_pwd() if not json_source() else get_pwd_from_vault(pwd)) pwd = pwd if pwd else get_git_pwd(key=username) # populate the password from vault if not pwd: pwd = get_pwd_from_vault(username) if pwd: logger.info("Git password from vault: %s", '*' * len(pwd)) if pwd: git_cmd = 'git clone --depth 1 %s%s:%s@%s %s' % (schema, urllib.parse.quote_plus(username), urllib.parse.quote_plus(pwd), other_part, repopath) elif isprivate: logger.error("Please provide password for connect to git repository.") return repopath, clonedir else: git_cmd = 'git clone --depth 1 %s%s:%s@%s %s' % (schema, urllib.parse.quote_plus(username), "", other_part, repopath) else: git_cmd = 'git clone --depth 1 %s %s' % (giturl, repopath) else: logger.info("SSH (private:%s) giturl: %s, Repopath: %s", "YES" if isprivate else "NO", giturl, repopath) if isprivate: ssh_key_file = get_field_value(connector, 'sshKeyfile') ssh_key_name = get_field_value(connector, 'sshKeyName') ssh_key_file_data = None if ssh_key_file: if not exists_file(ssh_key_file): logger.error("Git connector points to a non-existent ssh keyfile!") return repopath, clonedir elif ssh_key_name: ssh_key_file_data = get_vault_data(ssh_key_name) if not ssh_key_file_data: logger.info('Git connector points to a non-existent ssh keyName in the vault!') return repopath, clonedir ssh_host = get_field_value(connector, 'sshHost') ssh_user = get_field_value_with_default(connector, 'sshUser', 'git') if not ssh_host: logger.error("SSH host not set, could be like github.com, gitlab.com, 192.168.1.45 etc") return repopath, clonedir ssh_dir = '%s/.ssh' % repopath if exists_dir(ssh_dir): logger.error("Git ssh dir: %s already exists, cannot recreate it!", ssh_dir) return repopath, clonedir os.mkdir('%s/.ssh' % repopath, 0o700) if not ssh_key_file and ssh_key_name and ssh_key_file_data: ssh_key_file = create_ssh_file_vault_data(ssh_dir, ssh_key_file_data, ssh_key_name) if not ssh_key_file: logger.info('Git connector points to a non-existent ssh keyName in the vault!') return repopath, clonedir ssh_cfg = create_ssh_config(ssh_dir, ssh_key_file, ssh_user) if not ssh_cfg: logger.error("Creation of Git ssh config in dir: %s failed!", ssh_dir) return repopath, clonedir git_ssh_cmd = 'ssh -o "StrictHostKeyChecking=no" -F %s' % ssh_cfg git_cmd = 'git clone %s %s/tmpclone' % (giturl, repopath) subdir = True else: git_ssh_cmd = 'ssh -o "StrictHostKeyChecking=no"' git_cmd = 'git clone %s %s' % (giturl, repopath) os.environ['GIT_SSH_COMMAND'] = git_ssh_cmd git_cmd = '%s --branch %s' % (git_cmd, branch) if git_cmd: error_result, result = run_subprocess_cmd(git_cmd) checkdir = '%s/tmpclone' % repopath if subdir else repopath clonedir = checkdir if exists_dir('%s/.git' % checkdir) else None if not exists_dir(clonedir): logger.error("No valid data provided for connect to git : %s", error_result) if 'GIT_SSH_COMMAND' in os.environ: os.environ.pop('GIT_SSH_COMMAND') return repopath, clonedir
def get_pwd_from_vault(password_key): """ Return the git password from vault """ password = get_vault_data(password_key) if not password: logger.info("Password does not set in the vault") return password
def populate_azure_snapshot(snapshot, container=None, snapshot_type='azure'): """ Populates the resources from azure.""" dbname = config_value('MONGODB', 'dbname') snapshot_source = get_field_value(snapshot, 'source') snapshot_user = get_field_value(snapshot, 'testUser') snapshot_nodes = get_field_value(snapshot, 'nodes') snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes) client_id, client_secret, sub_name, sub_id, tenant_id = \ get_web_client_data(snapshot_type, snapshot_source, snapshot_user) if not client_id: # logger.info("No client_id in the snapshot to access azure resource!...") raise Exception("No client id in the snapshot to access azure resource!...") # Read the client secrets from envirnment variable if not client_secret: client_secret = os.getenv(snapshot_user, None) if client_secret: logger.info('Client Secret from environment variable, Secret: %s', '*' * len(client_secret)) # Read the client secrets from the vault if not client_secret: client_secret = get_vault_data(client_id) if client_secret: logger.info('Client Secret from Vault, Secret: %s', '*' * len(client_secret)) elif get_from_currentdata(CUSTOMER): logger.error("Client Secret key does not set in a vault") raise Exception("Client Secret key does not set in a vault") if not client_secret: raise Exception("No `client_secret` key in the connector file to access azure resource!...") logger.info('\t\tSubscription: %s', sub_id) logger.info('\t\tTenant: %s', tenant_id) logger.info('\t\tclient: %s', client_id) put_in_currentdata('clientId', client_id) put_in_currentdata('clientSecret', client_secret) put_in_currentdata('subscriptionId', sub_id) put_in_currentdata('tenant_id', tenant_id) token = get_access_token() logger.debug('TOKEN: %s', token) if not token: logger.info("Unable to get access token, will not run tests....") raise Exception("Unable to get access token, will not run tests....") # return {} # snapshot_nodes = get_field_value(snapshot, 'nodes') # snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes) if valid_snapshotids and token and snapshot_nodes: for node in snapshot_nodes: validate = node['validate'] if 'validate' in node else True if 'path' in node: data = get_node(token, sub_name, sub_id, node, snapshot_user, snapshot_source) if data: if validate: if get_dbtests(): if get_collection_size(data['collection']) == 0: # Creating indexes for collection create_indexes( data['collection'], config_value(DATABASE, DBNAME), [ ('snapshotId', pymongo.ASCENDING), ('timestamp', pymongo.DESCENDING) ] ) create_indexes( data['collection'], config_value(DATABASE, DBNAME), [ ('_id', pymongo.DESCENDING), ('timestamp', pymongo.DESCENDING), ('snapshotId', pymongo.ASCENDING) ] ) insert_one_document(data, data['collection'], dbname, check_keys=False) else: snapshot_dir = make_snapshots_dir(container) if snapshot_dir: store_snapshot(snapshot_dir, data) if 'masterSnapshotId' in node: snapshot_data[node['snapshotId']] = node['masterSnapshotId'] else: snapshot_data[node['snapshotId']] = True # else: # snapshot_data[node['snapshotId']] = False node['status'] = 'active' else: # TODO alert if notification enabled or summary for inactive. node['status'] = 'inactive' logger.debug('Type: %s', type(data)) else: alldata = get_all_nodes( token, sub_name, sub_id, node, snapshot_user, snapshot_source) if alldata: snapshot_data[node['masterSnapshotId']] = [] for data in alldata: # insert_one_document(data, data['collection'], dbname) found_old_record = False for masterSnapshotId, snapshot_list in snapshot_data.items(): old_record = None if isinstance(snapshot_list, list): for item in snapshot_list: if item["path"] == data['path']: old_record = item if old_record: found_old_record = True if node['masterSnapshotId'] not in old_record['masterSnapshotId']: old_record['masterSnapshotId'].append( node['masterSnapshotId']) if not found_old_record: snapshot_data[node['masterSnapshotId']].append( { 'masterSnapshotId': [node['masterSnapshotId']], 'snapshotId': data['snapshotId'], 'path': data['path'], 'validate': validate, 'status': 'active' }) # snapshot_data[node['masterSnapshotId']] = True logger.debug('Type: %s', type(alldata)) delete_from_currentdata('resources') delete_from_currentdata('clientId') delete_from_currentdata('client_secret') delete_from_currentdata('subscriptionId') delete_from_currentdata('tenant_id') delete_from_currentdata('token') return snapshot_data
def populate_aws_snapshot(snapshot, container=None): """ This is an entrypoint for populating a snapshot of type aws. All snapshot connectors should take snapshot object and based on 'source' field create a method to connect to the service for the connector. The 'source' field could be used by more than one snapshot, so the 'testuser' attribute should match to the user the 'source' """ dbname = config_value('MONGODB', 'dbname') snapshot_source = get_field_value(snapshot, 'source') snapshot_user = get_field_value(snapshot, 'testUser') account_id = get_field_value(snapshot, 'accountId') sub_data = get_aws_data(snapshot_source) snapshot_nodes = get_field_value(snapshot, 'nodes') snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes) # valid_snapshotids = True # if snapshot_nodes: # for node in snapshot_nodes: # snapshot_data[node['snapshotId']] = False # if not isinstance(node['snapshotId'], str): # valid_snapshotids = False # if not valid_snapshotids: # logger.error('All snap') if valid_snapshotids and sub_data and snapshot_nodes: logger.debug(sub_data) access_key, secret_access, region, connector_client_str = \ get_aws_client_data(sub_data, snapshot_user, account_id) if not access_key: logger.info( "No access_key in the snapshot to access aws resource!...") raise Exception( "No access_key in the snapshot to access aws resource!...") # return snapshot_data # Read the client secrets from envirnment variable or Standard input # if not secret_access and ('UAMI' not in os.environ or os.environ['UAMI'] != 'true'): # secret_access = get_client_secret() # logger.info('Environment variable or Standard input, Secret: %s', '*' * len(secret_access)) # Read the client secrets from the vault if not secret_access: secret_access = get_vault_data(access_key) if secret_access: logger.info('Vault Secret: %s', '*' * len(secret_access)) else: logger.info("Secret Access key does not set in a vault") raise Exception("Secret Access key does not set in a vault") if not secret_access: logger.info( "No secret_access in the snapshot to access aws resource!...") return snapshot_data if access_key and secret_access: # existing_aws_client = {} for node in snapshot['nodes']: mastercode = False if 'snapshotId' in node: client_str, aws_region = _get_aws_client_data_from_node( node, default_client=connector_client_str, default_region=region) if not _validate_client_name(client_str): logger.error("Invalid Client Name") return snapshot_data try: awsclient = client(client_str.lower(), aws_access_key_id=access_key, aws_secret_access_key=secret_access, region_name=aws_region) except Exception as ex: logger.info('Unable to create AWS client: %s', ex) awsclient = None logger.info(awsclient) if awsclient: data = get_node(awsclient, node, snapshot_source) if data: error_str = data.pop('error', None) if get_dbtests(): if get_collection_size( data['collection']) == 0: #Creating indexes for collection create_indexes( data['collection'], config_value(DATABASE, DBNAME), [('snapshotId', pymongo.ASCENDING), ('timestamp', pymongo.DESCENDING)]) check_key = is_check_keys_required(data) insert_one_document(data, data['collection'], dbname, check_key) else: snapshot_dir = make_snapshots_dir(container) if snapshot_dir: store_snapshot(snapshot_dir, data) if 'masterSnapshotId' in node: snapshot_data[node['snapshotId']] = node[ 'masterSnapshotId'] else: snapshot_data[node[ 'snapshotId']] = False if error_str else True elif 'masterSnapshotId' in node: mastercode = True client_str, aws_region = _get_aws_client_data_from_node( node, default_client=connector_client_str, default_region=region) if not _validate_client_name(client_str): logger.error("Invalid Client Name") return snapshot_data if aws_region: all_regions = [aws_region] else: all_regions = Session().get_available_regions( client_str.lower()) if client_str.lower() in ['s3', 'cloudtrail']: all_regions = ['us-west-1'] logger.info("Length of all regions is %s" % (str(len(all_regions)))) count = 0 snapshot_data[node['masterSnapshotId']] = [] for each_region in all_regions: logger.info(each_region) try: awsclient = client( client_str.lower(), aws_access_key_id=access_key, aws_secret_access_key=secret_access, region_name=each_region) except Exception as ex: logger.info('Unable to create AWS client: %s', ex) logger.info(awsclient) if awsclient: all_data = get_all_nodes(awsclient, node, snapshot, sub_data) if all_data: for data in all_data: snapshot_data[ node['masterSnapshotId']].append({ 'snapshotId': '%s%s' % (node['masterSnapshotId'], str(count)), 'validate': True, 'detailMethods': data['detailMethods'], 'structure': 'aws', 'masterSnapshotId': node['masterSnapshotId'], 'collection': data['collection'], 'arn': data['arn'] }) count += 1 if mastercode: snapshot_data = eliminate_duplicate_snapshots(snapshot_data) return snapshot_data
def populate_snapshot_azure(snapshot_json, fssnapshot): """ Populates the resources from azure.""" snapshot_data, valid_snapshotids = fssnapshot.validate_snapshot_ids_in_nodes( snapshot_json) client_id, client_secret, sub_name, sub_id, tenant_id, connector_type = get_web_client_data( snapshot_json, fssnapshot) if not client_id: logger.info( "No client_id in the snapshot to access azure resource!...") # raise Exception("No client id in the snapshot to access azure resource!...") raise SnapshotsException( "Container %s failure as no client id in the snapshot to access azure resource!..." % fssnapshot.container) # Read the client secrets from the vault if not client_secret: client_secret = get_vault_data(client_id) if client_secret: logger.info('Client Secret from Vault, Secret: %s', '*' * len(client_secret)) elif fssnapshot.get_value(CUSTOMER): logger.error("Client Secret key does not set in a vault") raise SnapshotsException( "Client Secret key does not set in a vault") if not client_secret: raise SnapshotsException( "No `client_secret` key in the connector file to access azure resource!..." ) logger.info('Sub:%s, tenant:%s, client: %s', sub_id, tenant_id, client_id) fssnapshot.store_value('clientId', client_id) fssnapshot.store_value('clientSecret', client_secret) fssnapshot.store_value('subscriptionId', sub_id) fssnapshot.store_value('tenant_id', tenant_id) token = get_access_token() logger.debug('TOKEN: %s', token) if not token: logger.info("Unable to get access token, will not run tests....") raise SnapshotsException( "Unable to get access token, will not run tests....") snapshot_source = get_field_value(snapshot_json, 'source') snapshot_user = get_field_value(snapshot_json, 'testUser') for node in fssnapshot.get_snapshot_nodes(snapshot_json): validate = node['validate'] if 'validate' in node else True if 'path' in node: data = get_snapshot_node(fssnapshot, token, sub_name, sub_id, node, snapshot_user, snapshot_source, connector_type) if data and validate: fssnapshot.store_data_node(data) snapshot_data[node['snapshotId']] = node[ 'masterSnapshotId'] if 'masterSnapshotId' in node else True node['status'] = 'active' else: # TODO alert if notification enabled or summary for inactive. node['status'] = 'inactive' logger.debug('Type: %s', type(data)) else: # Crawler Operation alldata = get_snapshot_nodes(fssnapshot, token, sub_name, sub_id, node, snapshot_user, snapshot_source, connector_type) if alldata: snapshot_data[node['masterSnapshotId']] = [] for data in alldata: found_old_record = False for masterSnapshotId, snapshot_list in snapshot_data.items( ): old_record = None if isinstance(snapshot_list, list): for item in snapshot_list: if item["path"] == data['path']: old_record = item if old_record: found_old_record = True if node['masterSnapshotId'] not in old_record[ 'masterSnapshotId']: old_record['masterSnapshotId'].append( node['masterSnapshotId']) if not found_old_record: snapshot_data[node['masterSnapshotId']].append({ 'masterSnapshotId': [node['masterSnapshotId']], 'snapshotId': data['snapshotId'], 'path': data['path'], 'validate': validate, 'status': 'active' }) logger.debug('Type: %s', type(alldata)) return snapshot_data