def test_file_params(self):
temp_test_dir = "yac/tests/stacks/aws/vectors/deploy"
service_parmeters = {"render-me": {"value": "baby"},
"servicefile-path": {"value": temp_test_dir},
"service-alias": {"value": "testing"}}
serialize_boot_files = {
"files": [
{
"src": "deploy_for_boot_file_params.txt",
"dest": "/tmp/stacks/aws/vectors/deploy/rendered/deploy_one_way.txt",
"file-params": {
"also-render-me": {
"comment": "render first exclamation",
"value": "oooh!"
}
}
},
{
"src": "deploy_for_boot_file_params.txt",
"dest": "/tmp/stacks/aws/vectors/deploy/rendered/deploy_other_way.txt",
"file-params": {
"also-render-me": {
"comment": "render second exclamation",
"value": "no!"
}
}
}
]
}
boot_files = BootFiles(serialize_boot_files,
)
try:
boot_files.deploy(Params(service_parmeters),
context="")
except FileError as e:
print(e.msg)
# read file contents from destination files
file_contents1 = get_file_contents(serialize_boot_files['files'][0]['dest'])
file_contents2 = get_file_contents(serialize_boot_files['files'][1]['dest'])
# clean up
shutil.rmtree("/tmp/stacks/aws/vectors/deploy/rendered")
self.assertTrue(file_contents1 == "baby, %s, baby, %s"%("baby","oooh!"))
self.assertTrue(file_contents2 == "baby, %s, baby, %s"%("baby","no!"))
def test_files(self):
temp_test_dir = "yac/tests/stacks/aws/vectors/deploy"
service_parmeters = {"render-me": {"value": "baby!"},
"servicefile-path": {"value": temp_test_dir},
"service-alias": {"value": "testing"}}
serialize_boot_files = {
"files": [
{
"src": "deploy_for_boot.txt",
"dest": "/tmp/stacks/aws/vectors/deploy/rendered/deploy_for_boot.txt"
}
]
}
boot_files = BootFiles(serialize_boot_files)
try:
boot_files.deploy(Params(service_parmeters),context="")
except FileError as e:
print(e.msg)
# read file contents from destination file
file_contents = get_file_contents(serialize_boot_files['files'][0]['dest'])
# clean up
shutil.rmtree("/tmp/stacks/aws/vectors/deploy/rendered")
self.assertTrue(file_contents == "render my params, then deploy me %s"%(service_parmeters["render-me"]["value"]))
def test_stemplate(self):
test_file = 'yac/tests/template/vectors/sample_map_file.txt'
test_variables = {
"user-name": {
"value": "henry-grantham"
},
"neighborhood-map": {
"lookup": "user-name",
"value": {
"tom-jackson": "phinney",
"henry-grantham": "capital-hill"
}
}
}
# read file into string
file_contents = get_file_contents(test_file)
# run test
updated_file_contents = apply_stemplate(file_contents,
Params(test_variables))
# test that the correct neighborhood got rendered into the file contents
render_check = "capital-hill" in updated_file_contents
self.assertTrue(render_check)
def register_service(service_name, servicefile_w_path, challenge):
err = ""
if os.path.exists(servicefile_w_path):
service_contents_str = get_file_contents(servicefile_w_path)
if service_contents_str:
reg_key = service_name + YAC_SERVICE_SUFFIX
# get the base path of the service file
servicefile_path = os.path.dirname(servicefile_w_path)
updated_service_contents_str = convert_local_files(service_name,
service_contents_str,
servicefile_path,
challenge)
# set the service in the registry
set_remote_string_w_challenge(reg_key, updated_service_contents_str, challenge)
else:
err = "service path %s doesn't exist"%servicefile_path
return err
def edit_secrets(vault):
err = ""
new_secrets = {}
if vault.vault_path:
# open an editor session so user can edit the file
if os.environ.get('EDITOR'):
EDITOR = os.environ.get('EDITOR')
else:
EDITOR = 'nano'
call([EDITOR, vault.vault_path])
# load the file contents back into a dictionary
file_contents = get_file_contents(vault.vault_path)
# parse the secrets to make sure the string is well formatted
if file_contents:
if vault.get_format() == "json":
try:
new_secrets = json.loads(file_contents)
except ValueError as e:
err = str(e)
elif vault.get_format() == "yaml":
try:
new_secrets = yaml.load(file_contents)
except yaml.scanner.ScannerError as e:
err = str(e)
return file_contents, err
def view_secrets(vault):
if vault.vault_path:
# load the file contents into a dictionary
file_contents = get_file_contents(vault.vault_path)
print(file_contents)
def do_calc(calc_arg, params):
# Return a list of boot script lines, one per script line provided in params.
# List is intended to be incorporated into the UserData portion of an EC2 or LaunchConfiguration
# template.
# args:
# calc_arg: array containing single element: a string holding path to boot script file
boot_script_list = []
err = ""
servicefile_path = params.get("servicefile-path")
boot_file = calc_arg[0] if len(calc_arg) == 1 else ""
if boot_file:
boot_script_contents = get_file_contents(boot_file, servicefile_path)
if boot_script_contents:
# render template variables into the file contents
boot_script_contents = apply_stemplate(boot_script_contents,
params)
# split script into lines
boot_script_lines_list = boot_script_contents.split('\n')
for i, line in enumerate(boot_script_lines_list):
if ("{" in line and "}" in line and "Ref" in line):
# this line contains a cloud formation reference which needs to be broken out
# i.e. something like ...
# CLUSTER_NAME={"Ref": "ECS"}
prefix = line[:line.index('{')]
reference = line[line.index('{'):line.index('}') + 1]
reference_dict = json.loads(reference)
boot_script_list = boot_script_list + [
prefix, {
"Ref": reference_dict["Ref"]
}
] + ["\n"]
else:
boot_script_list = boot_script_list + [line] + ["\n"]
else:
err = "boot file %s does not exist or has no content" % boot_file
else:
err = "No boot script provided"
return boot_script_list, err
def initialize(self, params):
err = ""
if os.path.exists(self.vault_path):
# pull contents into a dictionary
file_contents = get_file_contents(self.vault_path)
self.load(file_contents)
self.initialized = True
else:
err = "vault at %s does not exist" % self.vault_path
return err
def do_calc(arg_array, params):
# this calc should only be called via the calcs module, so we can be
# less defensive with input checking
boot_file = arg_array[0]
boot_script_list = []
servicefile_path = params.get("servicefile-path")
# get the boot script from the user params
if boot_file:
boot_script_contents = get_file_contents(boot_file, servicefile_path)
if boot_script_contents:
# render template variables into the file contents
boot_script_contents = apply_stemplate(boot_script_contents,
params)
# split script into lines
boot_script_lines_list = boot_script_contents.split('\n')
for i, line in enumerate(boot_script_lines_list):
if ("{" in line and "}" in line and "Ref" in line):
# this line contains a cloud formation reference which needs to be broken out
# i.e. something like ...
# CLUSTER_NAME={"Ref": "ECS"}
prefix = line[:line.index('{')]
reference = line[line.index('{'):line.index('}') + 1]
reference_dict = json.loads(reference)
boot_script_list = boot_script_list + [
prefix, {
"Ref": reference_dict["Ref"]
}
] + ["\n"]
else:
boot_script_list = boot_script_list + [line] + ["\n"]
else:
boot_script_list = boot_script_list + [
"# No boot script provided. See yac docs for more info.\n"
]
return boot_script_list
def test_register_file(self):
file_path = "yac/tests/registry/vectors/local_file.txt"
challenge_phrase = 'test-challenge' + str(random.randint(1, 1000))
file_key = '%s:%s' % (file_path, str(random.randint(1, 1000)))
register_file(file_key, file_path, challenge_phrase)
# read file back out
file_contents_returned = get_file_from_registry(file_key)
# clean up - remove the test file from the registry
clear_file_from_registry(file_key, challenge_phrase)
file_contents_sent = get_file_contents(file_path)
# test that the create was successful
self.assertTrue(file_contents_sent == file_contents_returned)
def test_file_json(self):
# test rendering templates in a json file containing intrinsics
test_file = 'yac/tests/template/vectors/render.json'
test_variables = {"latency-sample-period": {"value": 10}}
# run test
updated_file = apply_templates_in_file(test_file,
Params(test_variables), "/tmp")
file_contents = get_file_contents(updated_file)
updated_file_json = json.loads(file_contents)
# test that user name got rendered into the file contents
render_check = updated_file_json["period-mins"] == 10
self.assertTrue(render_check)
def install_kube_login(self):
# get the contents of the .kube/config file
file_contents = get_file_contents('yac/lib/stacks/k8s/configs/.kubeloginrc.yaml')
# write file
self.write_home_file(file_contents,".kubeloginrc.yaml")
# copy the kubelogin app under the user's home dir
kubelogin_dest = self.get_kubelogin_path()
if os.path.exists(kubelogin_dest):
# remove existing installatiaon of kubelogin
os.remove(kubelogin_dest)
print("installing: %s"%kubelogin_dest)
shutil.copyfile('yac/lib/stacks/k8s/configs/kubelogin',
kubelogin_dest)
os.chmod(kubelogin_dest, stat.S_IREAD | stat.S_IEXEC )
def apply_ftemplate(file_w_variables, params):
""" Replaces template variables in a string with values in params
Args:
file_w_variables: A file with variables in mustaches
params: A Params instance
Returns:
A string with mustaches replaced with values.
Raises:
TemplateError: if file does not exist, or
if file contains variables not defined in params
"""
# read file into string
string_w_instrinsics = get_file_contents(file_w_variables)
return apply_stemplate(string_w_instrinsics, params)
def main():
parser = argparse.ArgumentParser(description='View, test drive, or export service examples')
# required args
parser.add_argument('-v','--view', help='key of service example to view (run "yac grok" with no args to show all examples)')
parser.add_argument('-f',
'--file', help='view individual file from an example, or from yac sources (i.e. schemas)')
parser.add_argument('-x',
'--export', help='export example service for customization',
action='store_true')
args = parser.parse_args()
if not args.view and not args.file and not args.export:
# show the keys of all available examples
keys = get_all_keys()
print("Available examples include:")
for key in keys:
print("examples/%s"%key)
if args.view and not args.export:
# show the contents of the servicefile
file_contents = get_file_contents(args.view)
if file_contents:
print(file_contents)
if args.file:
if args.view:
base_path = os.path.dirname(args.view)
else:
# assume the path is relative to yac
base_path = ""
# show the contents of the file
file_contents = get_file_contents(os.path.join(base_path,args.file))
if file_contents:
print(file_contents)
if args.export and args.view:
# put the files in the std dump path so they
# can be viewed after the yac container stops
src = os.path.dirname(os.path.join(args.view))
# get servicealias
service,err = get_service(args.view)
print("made it here")
dest = get_home_dump_path("%s/%s"%("examples",
service.get_description().get_alias()))
# make sure destination is empty
if os.path.exists(dest):
shutil.rmtree(dest)
# recreate destination
# os.makedirs(dest)
#
# src_files = os.listdir(src)
# for file_name in src_files:
# full_file_name = os.path.join(src, file_name)
# if (os.path.isfile(full_file_name)):
# shutil.copy(full_file_name, dest)
shutil.copytree(src,dest)
print("example exported to: %s"%dest)
def apply_templates_in_file(file_w_variables,
params,
rendered_file_dest="tmp"):
""" Replaces template variables in a string with values in params
Args:
file_w_variables: A file with variables in mustaches
params: A Params instance
rendered_file_dest: The relative path to where the rendered file should be placed
Returns:
The path to a file with mustaches replaced with values.
Raises:
TemplateError: if source file does not exist
"""
# get the file type
file_abs_path = localize_file(file_w_variables,
params.get('servicefile-path'))
file_type = get_file_type(file_abs_path)
# if the file is a text file render any variables in the file contents using the
# provided template variables
if (file_type and
len(file_type) >= 1 and
('text' in file_type or
'json' in file_type or
'xml' in file_type or
'html' in file_type)):
# read file into string
file_contents = get_file_contents(file_abs_path)
# if the file is json, first attempt to render using intrinsics instead of templates
rendered_file_contents = ""
if 'json' in file_type:
rendered_file_contents = apply_intrinsics(file_contents, params)
else:
# render template variables
rendered_file_contents = apply_stemplate(file_contents, params)
# create the directory to hold the rendered file contents
if not os.path.exists(rendered_file_dest):
os.makedirs(rendered_file_dest)
file_name = os.path.basename(file_w_variables)
rendered_file = os.path.join(rendered_file_dest, file_name)
# write the rendered string into the temp file
with open(rendered_file, 'w') as outfile:
outfile.write(rendered_file_contents)
else:
# this isn't a text file, so don't attemp to render any variables
# instead copy from source to destination
# create a 'tmp' directory to hold the files
if not os.path.exists(rendered_file_dest):
os.makedirs(rendered_file_dest)
file_name = os.path.basename(file_w_variables)
rendered_file = os.path.join(rendered_file_dest, file_name)
# print "nrf: %s"%rendered_file
shutil.copy(file_w_variables, rendered_file)
return rendered_file
def _apply_intrinsics_leaf(key, source_dict, params):
# see if any of the values have intrinsics
if key == 'yac-ref':
# Pull referenced value from the params. Default to a string
# containing an error message in case the reference does not have
# a corresponding value.
# apply any intrinsics in the reference
#rendered_ref_args = _apply_intrinsics_list(source_dict[key], params)
setpoint = params.get(source_dict[key], "M.I.A.")
if setpoint == "M.I.A.":
reference_failure(params, source_dict[key])
return setpoint
if key == 'yac-calc':
# make sure arg is a list
if type(source_dict[key]) is not list:
msg = "%s\n%s" % (source_dict[key],
"argument to yac-calc must be type list")
calc_failure(params, msg)
setpoint = ""
else:
# apply any intrinsics in the calc arguments
rendered_calc_args = _apply_intrinsics_list(
source_dict[key], params)
setpoint, err = do_calc(rendered_calc_args, params)
if err:
msg = "%s\n%s" % (source_dict[key], err)
calc_failure(params, msg)
return setpoint
elif key == 'yac-join':
delimiters = source_dict[key][0]
name_parts = source_dict[key][1]
# apply intrinsics in the list of items to be joined
filled_parts = _apply_intrinsics_list(name_parts, params)
# get rid of empty strings before joining with delimitter
filled_parts = [_f for _f in filled_parts if _f]
return delimiters.join(filled_parts)
elif key == 'yac-fxn':
# this value should be filled by custom function supplied by service
fxn_script = source_dict[key]
return apply_custom_fxn(fxn_script, params)
elif key == 'yac-name':
# get the name for this resource
resource = source_dict[key]
return get_resource_name(params, resource)
elif key == 'yac-include':
# pull the templates out of the included file
included_file = source_dict[key]
servicefile_path = params.get('servicefile-path')
included_dict = {}
include_err = ""
if get_file_contents(included_file, servicefile_path):
# attempt to load the contents of the included file as a
# dictionary
included_dict, include_err = load_dict_from_file(
included_file, servicefile_path)
if not include_err:
# render any intrisics in the included dictionary
_apply_intrinsics(included_dict, params)
else:
include_err = "%s is either non-existant or has no content" % included_file
# if any errors where generated in the inclusion processing,
# register the error
if include_err:
include_failure(params, include_err)
return included_dict
else:
return source_dict[key]
