def encrypt(conf_name, delete):
"""Function to encrypt a given conf file"""
conf_name = conf_name.strip()
yaycl_crypt.encrypt_yaml(conf, conf_name, delete=delete)
print('{} conf encrypted'.format(conf_name))
if not delete:
print('WARNING: unencrypted file left which will override encrypted')
def encrypt(conf_name, delete):
"""Function to encrypt a given conf file"""
conf_name = conf_name.strip()
yaycl_crypt.encrypt_yaml(conf, conf_name, delete=delete)
print('{} conf encrypted'.format(conf_name))
if not delete:
print('WARNING: unencrypted file left which will override encrypted')
def test_load_warnings(conf, test_conf, recwarn, tmpdir):
# encrypt without deleting to create the eyaml next to the yaml
yaycl_crypt.encrypt_yaml(conf, 'test', delete=False)
del(conf['test'])
assert os.path.exists(test_conf.encrypted)
assert os.path.exists(test_conf.unencrypted)
# test env is sane, clear warnings and make sure one is caught on yaml load
warnings.resetwarnings()
assert conf.test.test_key == 'test value'
assert recwarn.pop(UserWarning)
def test_load_warnings(conf, test_conf, recwarn, tmpdir):
# encrypt without deleting to create the eyaml next to the yaml
yaycl_crypt.encrypt_yaml(conf, 'test', delete=False)
del (conf['test'])
assert os.path.exists(test_conf.encrypted)
assert os.path.exists(test_conf.unencrypted)
# test env is sane, clear warnings and make sure one is caught on yaml load
warnings.resetwarnings()
assert conf.test.test_key == 'test value'
assert recwarn.pop(UserWarning)
def test_encrypt_decrypt_yaml(request, conf, test_conf, tmpdir):
assert os.path.exists(test_conf.unencrypted)
# encryption deletes the unenecrypted yaml
yaycl_crypt.encrypt_yaml(conf, 'test')
assert os.path.exists(test_conf.encrypted)
assert not os.path.exists(test_conf.unencrypted)
del(conf['test'])
# decryption deletes the encrypted yaml
yaycl_crypt.decrypt_yaml(conf, 'test')
assert os.path.exists(test_conf.unencrypted)
assert not os.path.exists(test_conf.encrypted)
del(conf['test'])
# decryption refuses to delete an unencrypted yaml
with pytest.raises(Exception):
yaycl_crypt.decrypt_yaml(conf, 'test')
def test_encrypt_decrypt_yaml(request, conf, test_conf, tmpdir):
assert os.path.exists(test_conf.unencrypted)
# encryption deletes the unenecrypted yaml
yaycl_crypt.encrypt_yaml(conf, 'test')
assert os.path.exists(test_conf.encrypted)
assert not os.path.exists(test_conf.unencrypted)
del (conf['test'])
# decryption deletes the encrypted yaml
yaycl_crypt.decrypt_yaml(conf, 'test')
assert os.path.exists(test_conf.unencrypted)
assert not os.path.exists(test_conf.encrypted)
del (conf['test'])
# decryption refuses to delete an unencrypted yaml
with pytest.raises(Exception):
yaycl_crypt.decrypt_yaml(conf, 'test')
def credentials(provider_data):
if not os.path.exists("conf/.yaml_key"):
pytest.fail("No yaml key in conf/.yaml_key")
conf = yaycl.Config("conf/cfme-qe-yamls/complete",
crypt_key_file="conf/.yaml_key")
yaycl_crypt.decrypt_yaml(conf, "credentials", delete=False)
try:
with open("conf/cfme-qe-yamls/complete/credentials.yaml",
"r") as stream:
try:
creds = yaml.safe_load(stream)
except UnicodeDecodeError:
os.remove("conf/cfme-qe-yamls/complete/credentials.yaml")
pytest.fail("Unable to read decrypted credential file, "
"did you put the correct key in conf/.yaml_key?")
yield creds.get(provider_data.get("credentials"))
# cleanup the file after the test finishes
yaycl_crypt.encrypt_yaml(conf, "credentials", delete=True)
except IOError:
pytest.fail("Credential YAML file not found or not decrypted!")
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""Script to encrypt config files.
Usage:
scripts/encrypt_conf.py confname1 confname2 ... confnameN
scripts/encrypt_conf.py credentials
"""
import sys
import yaycl_crypt
from utils import conf
for conf_name in sys.argv[1:]:
conf_name = conf_name.strip()
yaycl_crypt.encrypt_yaml(conf, conf_name)
print('{} conf encrypted'.format(conf_name))
import argparse
import yaycl_crypt
from cfme.utils import conf
def parse_cmd_line():
parser = argparse.ArgumentParser(argument_default=None)
parser.add_argument('-e', '--encrypt', default=False, dest='encrypt', action='store_true',
help='encrypts the file specified')
parser.add_argument('-d', '--decrypt', default=False, dest='decrypt', action='store_true',
help='decrypts the file specified')
parser.add_argument('--file', dest='file', default='credentials',
help='file name in "conf" to be encrypted/decrypted')
parser.add_argument('--delete', dest='delete', default=False,
help='If set to False, encrypt_yaml will not delete the unencrypted '
'config of the same name, and decrypt_yaml will similarly not '
'delete its encrypted counterpart.')
args = parser.parse_args()
return args
args = parse_cmd_line()
conf_name = args.file.strip()
if args.encrypt:
yaycl_crypt.encrypt_yaml(conf, conf_name, delete=args.delete)
print('{} conf encrypted'.format(conf_name))
if args.decrypt:
yaycl_crypt.decrypt_yaml(conf, conf_name, delete=args.delete)
print('{} conf decrypted'.format(conf_name))
help='encrypts the file specified')
parser.add_argument('-d',
'--decrypt',
default=False,
dest='decrypt',
action='store_true',
help='decrypts the file specified')
parser.add_argument('--file',
dest='file',
default='cfme_performance',
help='file name in "conf" to be encrypted/decrypted')
parser.add_argument(
'--delete',
dest='delete',
default=False,
help='If set to False, encrypt_yaml will not delete the unencrypted '
'config of the same name, and decrypt_yaml will similarly not '
'delete its encrypted counterpart.')
args = parser.parse_args()
return args
args = parse_cmd_line()
conf_name = args.file.strip()
if args.encrypt:
yaycl_crypt.encrypt_yaml(conf, conf_name, delete=args.delete)
print('{} conf encrypted'.format(conf_name))
if args.decrypt:
yaycl_crypt.decrypt_yaml(conf, conf_name, delete=args.delete)
print('{} conf decrypted'.format(conf_name))
from pyspark.sql.session import SparkSession
from pyspark import SparkContext
import pyspark
import yaml
import yaycl_crypt
import yaycl
sc = SparkContext(master = 'local[*]')
spark = SparkSession(sc)
conf = yaycl.Config('/Internship_assessments/DonorsChooseCode/', crypt_key='/Internship_assessments/DonorsChooseCode/secret_text')
#yaycl_crypt.encrypt_yaml(conf, 'config')
yaycl_crypt.decrypt_yaml(conf, 'config')
file1=open("/Internship_assessments/DonorsChooseCode/config.yaml")
yaycl_crypt.encrypt_yaml(conf, 'config')
cfg=yaml.load(file1)
def colRename_wrtprq(DF ,prq_name):
for col in DF.columns:
DF = DF.withColumnRenamed(col,col.replace(" ", "_"))
try:
DF.write.parquet("hdfs:///inputs/DonorChoose/"+prq_name+".parquet")
except:
yield
#read data from hdfs
try:
donationsDF = spark.read.parquet("hdfs:///inputs/DonorChoose/donations.parquet")
except:
donationsDF = spark.read.csv("hdfs:///inputs/DonorChoose/Donations.csv", header= True, inferSchema= True)
import yaycl, yaycl_crypt
sc = SparkContext(master='local')
spark = SparkSession(sc)
'''reading from HDFS and writing into '''
filteredDF = spark.read.parquet("hdfs:///files/accident/filteredDF")
conf = yaycl.Config('/home/hduser/', crypt_key='my secret')
yaycl_crypt.decrypt_yaml(conf, 'test')
with open("/home/hduser/test.yaml", 'r') as ymlfile:
cfg = yaml.load(ymlfile)
yaycl_crypt.encrypt_yaml(conf, 'test')
filteredDF.write.format(cfg['type'].get('conn_type')).options(
url=cfg['url'].get('path') + cfg['url'].get('databaseName') +
cfg['url'].get('user') + cfg['url'].get('password'),
dbtable=cfg['dbtable'].get('name')).save()
newDF = spark.read.format(cfg['type'].get('conn_type')).options(
url=cfg['url'].get('path') + cfg['url'].get('databaseName') +
cfg['url'].get('user') + cfg['url'].get('password'),
dbtable=cfg['dbtable'].get('name')).load()
'''writing into postgres '''
newDF.write.format(cfg['type'].get('conn_type')).options(
url=cfg['url'].get('database_postgre'),
dbtable=cfg['dbtable'].get('pname')).save()
