def encrypt(request, json_key_attributes_dict, pin, version):
"""
This method will be called by the application entry point
for encrypting the payload.
request.value has the plaintext payload
request.alg contains the padding algorithm for encryption.
"""
key_path = json_key_attributes_dict["keypath"]
vault_url = json_key_attributes_dict["vaulturl"]
key_name = json_key_attributes_dict["keyname"]
hvac_client = hvac.Client(url=vault_url, token=pin)
hvac_client = hvac.Client(url=vault_url, token=pin)
read_response = hvac_client.secrets.kv.read_secret_version(path=key_path)
rsa_key_pem = read_response['data']['data'][key_name]
key = RSA.import_key(rsa_key_pem)
if request.alg == CryptoConstants.WRAP_RSA_OAEP:
cipher_algo = PKCS1_OAEP.new(key)
cipher_text = cipher_algo.encrypt(request.value)
response = EncryptDecryptResponse(cipher_text)
return response
def get_client(args):
client = hvac.Client(url=args.vault_address, verify=False if args.no_tls_verify else True)
if (args.auth_type == K8S):
return _auth_k8s(client, args.k8_auth_mount_point, role=args.k8_role)
elif (args.auth_type == APPROLE):
return _auth_approle(client, args.approle_role_id, args.approle_secret_id)
elif (args.auth_type == GCP_GCE):
return _auth_gcp_gce(client, args.gcp_role, args.vault_address)
else:
raise Exception("{} is not a supported auth type.", args.auth_type)
def _vault_client(config):
"""Helper wrapper to create Vault Client
:param: config: configparser object of vaultlocker config
:returns: hvac.Client. configured Vault Client object
"""
client = hvac.Client(url=config.get('vault', 'url'),
verify=config.get('vault', 'ca_bundle',
fallback=True))
client.auth_approle(config.get('vault', 'approle'),
secret_id=config.get('vault', 'secret_id'))
return client
def get_data_from_vault(path):
client = hvac.Client(url=app.config["VAULT_HOST"])
if client.is_sealed():
raise InvalidUsage('Error getting credentials. Check Vault',
status_code=400)
client.token = app.config["VAULT_TOKEN"]
vault_data = client.read(path)
if not vault_data:
print("No data found in vault path {path}".format(path=path))
client.logout()
return vault_data["data"]["data"] if vault_data else None
def ext_pillar(minion_id, pillar, *args, **kwargs):
""" Main handler. Compile pillar data for the specified minion ID
"""
vault_pillar = {}
# Load configuration values
for key in CONF:
if kwargs.get(key, None):
CONF[key] = kwargs.get(key)
# Resolve salt:// fileserver path, if necessary
if CONF["config"].startswith("salt://"):
local_opts = __opts__.copy()
local_opts["file_client"] = "local"
minion = salt.minion.MasterMinion(local_opts)
CONF["config"] = minion.functions["cp.cache_file"](CONF["config"])
# Read the secret map
renderers = salt.loader.render(__opts__, __salt__)
raw_yml = salt.template.compile_template(CONF["config"],
renderers,
'jinja',
whitelist=[],
blacklist=[])
if raw_yml:
secret_map = yaml.safe_load(raw_yml.getvalue()) or {}
else:
LOG.error("Unable to read secret mappings file '%s'", CONF["config"])
return vault_pillar
if not CONF["url"]:
LOG.error("'url' must be specified for Vault configuration")
return vault_pillar
# Connect and authenticate to Vault
conn = hvac.Client(url=CONF["url"])
_authenticate(conn)
# Apply the compound filters to determine which secrets to expose for this minion
ckminions = salt.utils.minions.CkMinions(__opts__)
for filter, secrets in secret_map.items():
minions = ckminions.check_minions(filter, "compound")
if 'minions' in minions:
# In Salt 2018 this is now in a kwarg
minions = minions['minions']
if minion_id in minions:
for variable, location in secrets.items():
return_data = couple(location, conn)
if return_data:
vault_pillar[variable] = return_data
return vault_pillar
def default_page():
#############################
# VAULT AUTH
#############################
client = hvac.Client(url=os.environ['VAULT_ADDR'],
token=os.environ['VAULT_TOKEN'])
if (not cache.get('db_creds_expiry')) or int(
time.time()) > (cache.get('db_creds_expiry') - 10):
db_creds = client.read('database/creds/people_list-readwrite')
cache.set('db_creds_result', json.dumps(db_creds))
cache.set('db_creds_expiry',
int(time.time() + db_creds["lease_duration"]))
else:
db_creds = json.loads(cache.get('db_creds_result'))
#############################
# DB ACCESS/USAGE
#############################
conn = psycopg2.connect(dbname='people_list',
user=db_creds['data']['username'],
host='postgresql',
password=db_creds['data']['password'],
sslmode='disable')
# Add data to DB if there is POSTDATA found
if request.form:
ins_cur = conn.cursor()
ins_cur.execute(
"""INSERT INTO staff (LastName, FirstName, Company) VALUES (%(last_name)s, %(first_name)s, %(company)s);""",
request.form)
conn.commit()
ins_cur.close()
sel_cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
sel_cur.execute('SELECT * from staff;')
rows = [dict(row) for row in sel_cur.fetchall()]
sel_cur.close()
#############################
# TEMPLATE RENDERING
#############################
template_params = {
'creds_expiry':
time.strftime("%Z - %Y/%m/%d, %H:%M:%S",
time.localtime(cache.get('db_creds_expiry'))),
'db_creds':
json.dumps(db_creds, indent=2),
'db_result':
rows
}
return render_template('index.html', **template_params)
def initialize(provider='virtualbox', bucket=None):
global client
global root_token
global keys
vault_var = "/vault"
if client is None:
VAULT_ADDR = os.environ["VAULT_ADDR"]
client = hvac.Client(url=VAULT_ADDR)
if not client.sys.is_initialized():
shares = 5
threshold = 3
result = client.sys.initialize(shares, threshold)
root_token = result['root_token']
keys = result['keys']
with open(vault_var + '/keys.json', "w+") as keysfile:
json.dump(keys, keysfile)
with open(vault_var + '/root_token.txt', "w+") as tokenfile:
tokenfile.write(root_token)
if provider == 'aws':
s3_client = boto3.client('s3')
s3_client.upload_file(vault_var + '/keys.json', bucket,
'keys.json')
s3_client.upload_file(vault_var + '/root_token.txt', bucket,
'root_token.txt')
else:
if keys is None or root_token is None:
if provider == 'aws':
s3_client = boto3.client('s3')
s3_client.download_file(bucket, 'keys.json',
vault_var + '/keys.json')
s3_client.download_file(bucket, 'root_token.txt',
vault_var + '/root_token.txt')
with open(vault_var + '/keys.json') as keysfile:
keys = json.load(keysfile)
with open(vault_var + '/root_token.txt') as tokenfile:
root_token = tokenfile.read()
if client.token is None:
client.token = root_token
def handler(event, context):
env = os.environ.get('ENV')
vault_url = os.environ.get('VAULT_URL')
path = os.environ.get('DATASERVICE_PG_SECRET')
client = hvac.Client(url=vault_url)
session = boto3.Session()
credentials = session.get_credentials()
r = client.auth_aws_iam(credentials.access_key,
credentials.secret_key,
credentials.token)
secret = client.read(path)['data']
conn = psycopg2.connect(
host=secret['hostname'],
dbname='kfpostgres'+env,
user=secret['username'],
password=secret['password'])
cur = conn.cursor()
cur.execute("""
SELECT phenotype.source_text_phenotype, count(phenotype.source_text_phenotype), study.kf_id AS study_id FROM phenotype, participant, study
WHERE phenotype.participant_id = participant.kf_id and participant.study_id = study.kf_id
GROUP BY phenotype.source_text_phenotype,study.kf_id
HAVING count(phenotype.source_text_phenotype) > 100
ORDER BY count(phenotype.source_text_phenotype);
""")
data = cur.fetchall()
cur.close()
conn.close()
files = {}
by_pheno = defaultdict(int)
by_study = defaultdict(lambda: defaultdict(int))
path = '/tmp/phenotypes.csv'
with open(path, 'w') as f:
writer = csv.writer(f)
writer.writerow(['source text phenotype', 'count', 'study_id'])
for v in data:
by_pheno[v[0]] += v[1]
by_study[v[2]][v[0]] += v[1]
writer.writerow(v)
files['Phenotype Data'] = path
files['All Studies'] = make_plot(by_pheno, 'all studies')
for study_id, d in by_study.items():
files[study_id] = make_plot(d, study_id)
return [], files
def auth_to_vault():
# TODO: plumb in consul here
vault_addr = get_option('vault', 'url')
vault_port = get_option('vault', 'port')
if not vault_port or not vault_addr:
raise Exception('Please provide values for vault_addr and vault_port in the config.ini.')
global client
address = str(vault_addr) + ':' + str(vault_port)
print 'Initializing client with address: ' + address
client = hvac.Client(url=address)
get_token()
def dynamic_connect_long():
client = hvac.Client(url='http://main-secrets:8200', token=_vault_token)
credentials = client.read('jesper/creds/readonly')
con = mdb.connect('main-db', credentials['data']['username'],
credentials['data']['password'], 'vaulthackathon')
cur = con.cursor()
cur.execute("SELECT VERSION()")
ver = cur.fetchone()
return "user %s pass %s lease_duration %s found %s ]" % (
credentials['data']['username'], credentials['data']['password'],
credentials['lease_duration'], ver)
def __init__(self, configuration, token, logger=None):
self.logger = logger
self.logger.info("=== VAULT START INITIALIZE ===")
self.host = configuration['url']
self.token = token
if self.host is None:
self.exit(self.VAULT_NO_SERVER_PROVIDED, "No Server was provided")
if self.token is not None:
self.client = hvac.Client(self.host, token)
else:
self.exit(self.VAULT_NO_TOKEN, "No Token provided")
self.logger.info("=== VAULT END INITIALIZE ===")
def retrieve_credentials_from_vault(
credentials_dict: Dict[str, Any]) -> Dict[str, str]:
"""Expects that you're logged into Hashicorp Vault."""
import hvac
client = hvac.Client()
if not client.is_authenticated():
raise ImmutaCredentialsError(
"Vault is not authenticated. Log in to vault first")
secret = client.secrets.kv.v2.read_secret_version(
path=credentials_dict["key"])
retrieved_credentials = secret["data"]["data"]
return retrieved_credentials
def get_secret(secret: str) -> str:
"""Vault secret fetcher.
Args:
secret (str): The secret's path.
Returns:
str: The secret's value.
"""
vault = hvac.Client()
response = vault.secrets.kv.v2.read_secret_version(
path=secret, )["data"]["data"]
return response["secret"]
def authenticated_client(self, *args: Any, **kwargs: Any) -> hvac.Client:
if self.authtype == "token":
cl = hvac.Client(token=self.credentials, *args, **kwargs)
elif self.authtype == "app-id":
cl = hvac.Client(*args, **kwargs)
cl.auth_app_id(*self.credentials)
elif self.authtype == "ssl":
cl = hvac.Client(cert=self.credentials, *args, **kwargs)
cl.auth.tls.login()
else:
cl = hvac.Client(*args, **kwargs)
try:
auth_adapter = getattr(cl.auth, self.authtype)
except AttributeError:
raise VaultCredentialProviderException("unknown auth method %s" % self.authtype)
auth_adapter.login(*self.credentials, mount_point=self.authmount)
if not cl.is_authenticated():
raise VaultCredentialProviderException("Unable to authenticate Vault client using provided credentials "
"(type=%s)" % self.authtype)
return cl
def upgrade(vault_name: str, vaults: List[Vault]):
vault = next(v for v in vaults if v['name'] == vault_name)
vault_client = hvac.Client(url=vault['url'], token=vault['token'])
for role, kwargs in roles.items():
print('INFO: create role `{}` at `{}` vault'.format(role, vault_name))
kwargs['name'] = role
vault_client.write(path='flant/role', **kwargs)
vault_client.write(path='flant/role/ssh.open/include/servers.query')
vault_client.write(path='flant/role/ssh.open/include/tenant.read.auth')
vault_client.write(
path='flant/role/servers.register/include/tenant.read.auth')
def __init__(self, mount_point=os.environ.get('VAULT_MOUNT_POINT', 'pwdmng/')):
path_prefix = self._path_prefix = mount_point
self._client = client = hvac.Client(url=settings.VAULT_HOST, token=settings.VAULT_TOKEN)
try:
#assert client.is_authenticated() # => True
engines = client.list_secret_backends()
if engines.get(path_prefix) == None:
client.enable_secret_backend('kv', mount_point=path_prefix, options={'version': '1'})
client.kv.default_kv_version = "1"
except ConnectionError as err:
print("VaultClient error: {0}".format(err))
pass
def upgrade(vault_name: str, vaults: List[Vault]):
vault = next(v for v in vaults if v['name'] == vault_name)
vault_client = hvac.Client(url=vault['url'], token=vault['token'])
if 'root' in vault_name:
plugins = root_vault_plugins
else:
plugins = auth_vault_plugins
for plugin in plugins:
print("INFO: configure kafka access for '{}' plugin at '{}' vault".format(plugin, vault_name))
if plugin == 'flant_iam_auth':
vault_client.write(path='auth/flant/kafka/configure_access', **kafka_configure_options)
elif plugin == 'flant_iam':
vault_client.write(path='flant/kafka/configure_access', **kafka_configure_options)
def get_certificate(ca, serial, cn):
vault = hvac.Client(app.config['VAULT_ADDR'])
vault.token = app.config['VAULT_TOKEN']
assert vault.is_authenticated()
certificate = vault.read('{}/cert/{}'.format(ca, serial))
return Response(certificate['data']['certificate'],
mimetype='application/x-pem-file',
headers={
'Content-Disposition':
'attachment;filename={}.pem'.format(cn)
})
def _do_connect(self):
try:
logging.info("Creating connection")
client = hvac.Client(url=self._url,
token=self._token,
verify=self._verify_tls)
client.sys.list_mounted_secrets_engines()
self._client = client
self._on_connect()
except Exception as e:
logging.error(e)
self._error = e
self._on_connect_error(e)
def vault():
with open('/vault/secrets/token') as open_file:
client = hvac.Client(
url='http://vault.default.svc:8200', # this url should be updated
# to the local sidecar as it will proxy requests to vault
token=open_file.read().rstrip()
)
vault_read_response = client.secrets.kv.v2.read_secret_version(
path='bar',
)
return str(vault_read_response['data']['data']['baz'])
def secret_to_vault(settings, path, value):
print("-----")
print(path)
print(value)
print("-----")
if settings["VAULT_DRY_RUN"] == "True":
print("Skipped.")
else:
vault_client = hvac.Client(url=settings["VAULT_URL"], token=settings["VAULT_TOKEN"], verify=False)
vault_client.write(path, value=value)
print("Written.")
def store_keys(server, token, deployment, pubkey, privkey):
"""Store the keys in the vault server."""
client = hvac.Client(url=server, token=token)
with open(pubkey, "r") as key:
pubkey_data = key.read()
with open(privkey, "r") as key:
privkey_data = key.read()
client.write("secret/{}/ssh/public_key".format(deployment),
key=pubkey_data)
client.write("secret/{}/ssh/private_key".format(deployment),
key=privkey_data)
def make_service_jenkins_jobs(env, local_vault_url, local_vault_token):
client = hvac.Client(url=local_vault_url, token=local_vault_token)
res = client.list('msa-services/')
service_list = res.get('data').get('keys')
slack_token = client.read('secret/common')['data']['JENKINS_SLACK_TOKEN']
jenkins_url = "http://jenkins.mmt.local:9082/"
jenkins_un = "user"
jenkins_pw = os.getenv("GITHUB_TOKEN")
server = jenkins.Jenkins(jenkins_url, username=jenkins_un, password=jenkins_pw)
for service in service_list:
_make_service_jenkins_job(server, env, service, slack_token)
def __init__(self, filename=None, path=None):
"""Initialize class properties."""
self._filename = filename or 'tcex.json'
self._path = path or os.getcwd()
# properties
self._contents = None
self.ij = InstallJson()
self.vault_client = hvac.Client(
url=os.getenv('VAULT_URL', 'http://localhost:8200'),
token=os.getenv('VAULT_TOKEN'),
cert=os.getenv('VAULT_CERT'),
)
def login_token():
"""Login to Vault using a token."""
url = request.form['url']
token = request.form['token']
client = hvac.Client(
url=url,
token=token,
verify=ast.literal_eval(os.environ['VERIFY']))
if client.is_authenticated():
return 200
else:
return 401
def client(self, **kwargs) -> Iterator[hvac.Client]: # type: ignore
"""
Get a context manager that creates and closes a root-access hvac client
Args:
**kwargs: forwarded to hvac.Client constructor
"""
client = hvac.Client(url=self.local_url(),
token=self.root_token,
**kwargs)
try:
yield client
finally:
client.adapter.close()
def vault_authenticate(vault_url, credentials, region, ca_cert):
"""
Return Vault client using supplied IAM credentials.
"""
vault_client = hvac.Client(url=vault_url, verify=ca_cert)
vault_client.auth_aws_iam(credentials.access_key,
credentials.secret_key,
credentials.token,
region=region)
vault_token = vault_client.lookup_token()["data"]["id"]
logging.info("authenticated with vault")
return vault_token
def __init__(
self,
paths: str = None,
token: str = os.environ.get("VAULTIFY_SECRET"),
addr: str = None,
):
self.token = os.environ.get("VAULT_TOKEN", token)
self.addr = os.environ.get("VAULT_ADDR", addr)
self.paths = os.environ.get("VAULT_PATHS", paths).split(",")
self.client = hvac.Client(url=self.addr, token=self.token)
logger.debug("VaultProvider initialized")
def init_vault(self, addr, token, namespace, path, key_name):
if not addr or not token:
logger.warn('Skipping initialization...')
return
else:
logger.warn("Connecting to vault server: {}".format(addr))
self.vault_client = hvac.Client(url=addr,
token=token,
namespace=namespace)
self.key_name = key_name
self.mount_point = path
logger.debug("Initialized vault_client: {}".format(
self.vault_client))
def fetch_api_address(self):
"""
Fetch the Vault API address and instanciate hvac client
"""
if "VAULT_ADDR" in os.environ:
self.logger.debug("'VAULT_ADDR' found in env")
vault_address = os.environ["VAULT_ADDR"]
else:
self.logger.debug("'VAULT_ADDR' not in env. Asking for a token")
vault_address = input("Vault address to use "
"(http://vault_address:vault_port): ")
self.logger.debug("Vault address to be used: " + vault_address)
self.vault_client = hvac.Client(url=vault_address)
