def test_filesystem_decrypt(conf, encrypted_test_conf):
# test decryption in isolation to ensure filesystem operations work correctly
with warnings.catch_warnings(record=True) as caught_warnings:
yaycl_crypt.decrypt_yaml(conf, encrypted_test_conf)
for warning in caught_warnings:
assert warning.category is not yaycl_crypt.YayclCryptWarning, \
'Unencrypted yaml found while decrypting: {}'.format(warning.message)
assert conf.encrypted.test_key == 'test value'
def test_filesystem_decrypt(conf, encrypted_test_conf):
# test decryption in isolation to ensure filesystem operations work correctly
with warnings.catch_warnings(record=True) as caught_warnings:
yaycl_crypt.decrypt_yaml(conf, encrypted_test_conf)
for warning in caught_warnings:
assert warning.category is not yaycl_crypt.YayclCryptWarning, \
'Unencrypted yaml found while decrypting: {}'.format(warning.message)
assert conf.encrypted.test_key == 'test value'
def decrypt(conf_name, delete, skip):
"""Function to decrypt a given conf file"""
conf_name = conf_name.strip()
try:
yaycl_crypt.decrypt_yaml(conf, conf_name, delete=delete)
except yaycl_crypt.YayclCryptError as ex:
if skip and 'overwrite' in ex.message:
print('{} conf decrypt skipped, decrypted file already exists'.format(conf_name))
return
else:
raise
print('{} conf decrypted'.format(conf_name))
def decrypt(conf_name, delete, skip):
"""Function to decrypt a given conf file"""
conf_name = conf_name.strip()
try:
yaycl_crypt.decrypt_yaml(conf, conf_name, delete=delete)
except yaycl_crypt.YayclCryptError as ex:
if skip and 'overwrite' in ex.message:
print('{} conf decrypt skipped, decrypted file already exists'.format(conf_name))
return
else:
raise
print('{} conf decrypted'.format(conf_name))
def decrypt(conf_name, delete, skip):
"""Function to decrypt a given conf file"""
conf_name = conf_name.strip()
try:
yaycl_crypt.decrypt_yaml(conf, conf_name, delete=delete)
except yaycl_crypt.YayclCryptError as ex:
if skip and 'overwrite' in str(ex):
print(
f'SKIPPED {conf_name} conf decrypt, decrypted file already exists'
)
return
else:
raise
print(f'{conf_name} conf decrypted')
def test_encrypt_decrypt_yaml(request, conf, test_conf, tmpdir):
assert os.path.exists(test_conf.unencrypted)
# encryption deletes the unenecrypted yaml
yaycl_crypt.encrypt_yaml(conf, 'test')
assert os.path.exists(test_conf.encrypted)
assert not os.path.exists(test_conf.unencrypted)
del(conf['test'])
# decryption deletes the encrypted yaml
yaycl_crypt.decrypt_yaml(conf, 'test')
assert os.path.exists(test_conf.unencrypted)
assert not os.path.exists(test_conf.encrypted)
del(conf['test'])
# decryption refuses to delete an unencrypted yaml
with pytest.raises(Exception):
yaycl_crypt.decrypt_yaml(conf, 'test')
def test_encrypt_decrypt_yaml(request, conf, test_conf, tmpdir):
assert os.path.exists(test_conf.unencrypted)
# encryption deletes the unenecrypted yaml
yaycl_crypt.encrypt_yaml(conf, 'test')
assert os.path.exists(test_conf.encrypted)
assert not os.path.exists(test_conf.unencrypted)
del (conf['test'])
# decryption deletes the encrypted yaml
yaycl_crypt.decrypt_yaml(conf, 'test')
assert os.path.exists(test_conf.unencrypted)
assert not os.path.exists(test_conf.encrypted)
del (conf['test'])
# decryption refuses to delete an unencrypted yaml
with pytest.raises(Exception):
yaycl_crypt.decrypt_yaml(conf, 'test')
def credentials(provider_data):
if not os.path.exists("conf/.yaml_key"):
pytest.fail("No yaml key in conf/.yaml_key")
conf = yaycl.Config("conf/cfme-qe-yamls/complete",
crypt_key_file="conf/.yaml_key")
yaycl_crypt.decrypt_yaml(conf, "credentials", delete=False)
try:
with open("conf/cfme-qe-yamls/complete/credentials.yaml",
"r") as stream:
try:
creds = yaml.safe_load(stream)
except UnicodeDecodeError:
os.remove("conf/cfme-qe-yamls/complete/credentials.yaml")
pytest.fail("Unable to read decrypted credential file, "
"did you put the correct key in conf/.yaml_key?")
yield creds.get(provider_data.get("credentials"))
# cleanup the file after the test finishes
yaycl_crypt.encrypt_yaml(conf, "credentials", delete=True)
except IOError:
pytest.fail("Credential YAML file not found or not decrypted!")
import argparse
import yaycl_crypt
from cfme.utils import conf
def parse_cmd_line():
parser = argparse.ArgumentParser(argument_default=None)
parser.add_argument('-e', '--encrypt', default=False, dest='encrypt', action='store_true',
help='encrypts the file specified')
parser.add_argument('-d', '--decrypt', default=False, dest='decrypt', action='store_true',
help='decrypts the file specified')
parser.add_argument('--file', dest='file', default='credentials',
help='file name in "conf" to be encrypted/decrypted')
parser.add_argument('--delete', dest='delete', default=False,
help='If set to False, encrypt_yaml will not delete the unencrypted '
'config of the same name, and decrypt_yaml will similarly not '
'delete its encrypted counterpart.')
args = parser.parse_args()
return args
args = parse_cmd_line()
conf_name = args.file.strip()
if args.encrypt:
yaycl_crypt.encrypt_yaml(conf, conf_name, delete=args.delete)
print('{} conf encrypted'.format(conf_name))
if args.decrypt:
yaycl_crypt.decrypt_yaml(conf, conf_name, delete=args.delete)
print('{} conf decrypted'.format(conf_name))
help='encrypts the file specified')
parser.add_argument('-d',
'--decrypt',
default=False,
dest='decrypt',
action='store_true',
help='decrypts the file specified')
parser.add_argument('--file',
dest='file',
default='cfme_performance',
help='file name in "conf" to be encrypted/decrypted')
parser.add_argument(
'--delete',
dest='delete',
default=False,
help='If set to False, encrypt_yaml will not delete the unencrypted '
'config of the same name, and decrypt_yaml will similarly not '
'delete its encrypted counterpart.')
args = parser.parse_args()
return args
args = parse_cmd_line()
conf_name = args.file.strip()
if args.encrypt:
yaycl_crypt.encrypt_yaml(conf, conf_name, delete=args.delete)
print('{} conf encrypted'.format(conf_name))
if args.decrypt:
yaycl_crypt.decrypt_yaml(conf, conf_name, delete=args.delete)
print('{} conf decrypted'.format(conf_name))
from pyspark.sql.session import SparkSession
from pyspark import SparkContext
import pyspark
import yaml
import yaycl_crypt
import yaycl
sc = SparkContext(master = 'local[*]')
spark = SparkSession(sc)
conf = yaycl.Config('/Internship_assessments/DonorsChooseCode/', crypt_key='/Internship_assessments/DonorsChooseCode/secret_text')
#yaycl_crypt.encrypt_yaml(conf, 'config')
yaycl_crypt.decrypt_yaml(conf, 'config')
file1=open("/Internship_assessments/DonorsChooseCode/config.yaml")
yaycl_crypt.encrypt_yaml(conf, 'config')
cfg=yaml.load(file1)
def colRename_wrtprq(DF ,prq_name):
for col in DF.columns:
DF = DF.withColumnRenamed(col,col.replace(" ", "_"))
try:
DF.write.parquet("hdfs:///inputs/DonorChoose/"+prq_name+".parquet")
except:
yield
#read data from hdfs
try:
donationsDF = spark.read.parquet("hdfs:///inputs/DonorChoose/donations.parquet")
except:
donationsDF = spark.read.csv("hdfs:///inputs/DonorChoose/Donations.csv", header= True, inferSchema= True)
from pyspark import SparkContext
from pyspark.sql.session import SparkSession
import yaml
import yaycl, yaycl_crypt
sc = SparkContext(master='local')
spark = SparkSession(sc)
'''reading from HDFS and writing into '''
filteredDF = spark.read.parquet("hdfs:///files/accident/filteredDF")
conf = yaycl.Config('/home/hduser/', crypt_key='my secret')
yaycl_crypt.decrypt_yaml(conf, 'test')
with open("/home/hduser/test.yaml", 'r') as ymlfile:
cfg = yaml.load(ymlfile)
yaycl_crypt.encrypt_yaml(conf, 'test')
filteredDF.write.format(cfg['type'].get('conn_type')).options(
url=cfg['url'].get('path') + cfg['url'].get('databaseName') +
cfg['url'].get('user') + cfg['url'].get('password'),
dbtable=cfg['dbtable'].get('name')).save()
newDF = spark.read.format(cfg['type'].get('conn_type')).options(
url=cfg['url'].get('path') + cfg['url'].get('databaseName') +
cfg['url'].get('user') + cfg['url'].get('password'),
dbtable=cfg['dbtable'].get('name')).load()
'''writing into postgres '''
def decrypt(conf_name, delete):
"""Function to decrypt a given conf file"""
conf_name = conf_name.strip()
yaycl_crypt.decrypt_yaml(conf, conf_name, delete=delete)
print('{} conf decrypted'.format(conf_name))
def decrypt(conf_name, delete):
"""Function to decrypt a given conf file"""
conf_name = conf_name.strip()
yaycl_crypt.decrypt_yaml(conf, conf_name, delete=delete)
print('{} conf decrypted'.format(conf_name))
