def run_step(context):
"""Simple echo. Outputs context['echoMe'].
Args:
context: dictionary-like. context is mandatory.
context must contain key 'echoMe'
context['echoMe'] will echo the value to logger.
This logger could well be stdout.
When you execute the pipeline, it should look something like this:
pypyr [name here] --context 'echoMe=test'.
"""
logger.debug("started")
assert context, ("context must be set for echo. Did you set "
"--context 'echoMe=text here'?")
context.assert_key_exists('echoMe', __name__)
if isinstance(context['echoMe'], str):
val = context.get_formatted('echoMe')
else:
val = context['echoMe']
logger.info(val)
logger.debug("done")
def run_step(context):
"""Remove specified keys from context.
Args:
Context is a dictionary or dictionary-like.
context['contextClear'] must exist. It's a dictionary.
Will iterate context['contextClear'] and remove those keys from
context.
For example, say input context is:
key1: value1
key2: value2
key3: value3
key4: value4
contextClear:
- key2
- key4
- contextClear
This will result in return context:
key1: value1
key3: value3
"""
logger.debug("started")
context.assert_key_has_value(key='contextClear', caller=__name__)
for k in context['contextClear']:
logger.debug(f"removing {k} from context")
# slightly unorthodox pop returning None means you don't get a KeyError
# if key doesn't exist
context.pop(k, None)
logger.info(f"removed {k} from context")
logger.debug("done")
def run_step(context):
"""Outputs pypyr version in format 'pypyr x.y.z python a.b.c'"""
logger.debug("started")
logger.info(f"pypyr version is: {pypyr.version.get_version()}")
logger.debug("done")
def tar_archive(context):
"""Archive specified path to a tar archive.
Args:
context: dictionary-like. context is mandatory.
context['tarArchive'] must exist. It's a dictionary.
keys are the paths to archive.
values are the destination output paths.
Example:
tarArchive:
- in: path/to/dir
out: path/to/destination.tar.xs
- in: another/my.file
out: ./my.tar.xs
This will archive directory path/to/dir to path/to/destination.tar.xs,
and also archive file another/my.file to ./my.tar.xs
"""
logger.debug("start")
mode = get_file_mode_for_writing(context)
for item in context['tarArchive']:
# value is the destination tar. Allow string interpolation.
destination = context.get_formatted_string(item['out'])
# key is the source to archive
source = context.get_formatted_string(item['in'])
with tarfile.open(destination, mode) as archive_me:
logger.debug(f"Archiving '{source}' to '{destination}'")
archive_me.add(source)
logger.info(f"Archived '{source}' to '{destination}'")
logger.debug("end")
def tar_extract(context):
"""Extract all members of tar archive to specified path.
Args:
context: dictionary-like. context is mandatory.
context['tarExtract'] must exist. It's a dictionary.
keys are the path to the tar to extract.
values are the destination paths.
Example:
tarExtract:
- in: path/to/my.tar.xs
out: /path/extract/here
- in: another/tar.xs
out: .
This will extract path/to/my.tar.xs to /path/extract/here, and also
extract another/tar.xs to $PWD.
"""
logger.debug("start")
mode = get_file_mode_for_reading(context)
for item in context['tarExtract']:
# in is the path to the tar to extract. Allows string interpolation.
source = context.get_formatted_string(item['in'])
# out is the outdir, dhur. Allows string interpolation.
destination = context.get_formatted_string(item['out'])
with tarfile.open(source, mode) as extract_me:
logger.debug(f"Extracting '{source}' to '{destination}'")
extract_me.extractall(destination)
logger.info(f"Extracted '{source}' to '{destination}'")
logger.debug("end")
def run_step(context):
"""Wipe the entire context.
Args:
Context is a dictionary or dictionary-like.
Does not require any specific keys in context.
"""
logger.debug("started")
context.clear()
logger.info(f"Context wiped. New context size: {len(context)}")
logger.debug("done")
def run_step(context):
"""Parses input file and replaces a search string.
This also does string substitutions from context on the fileReplacePairs.
It does this before it search & replaces the in file.
Be careful of order. If fileReplacePairs is not an ordered collection,
replacements could evaluate in any given order. If this is coming in from
pipeline yaml it will be an ordered dictionary, so life is good.
Args:
context: pypyr.context.Context. Mandatory.
The following context keys expected:
- fileReplaceIn. mandatory. path-like.
Path to source file on disk.
- fileReplaceOut. mandatory. path-like. Write output file to
here. Will create directories in path for you.
- fileReplacePairs. mandatory. Dictionary where items are:
'find_string': 'replace_string'
Returns:
None.
Raises:
FileNotFoundError: take a guess
pypyr.errors.KeyNotInContextError: Any of the required keys missing in
context.
pypyr.errors.KeyInContextHasNoValueError: Any of the required keys
exists but is None.
"""
logger.debug("started")
context.assert_keys_have_values(__name__, 'fileReplaceIn',
'fileReplaceOut', 'fileReplacePairs')
in_path = context.get_formatted('fileReplaceIn')
out_path = context.get_formatted('fileReplaceOut')
logger.debug("Running subsitutions from context on fileReplacePairs")
formatted_replacements = context.get_formatted_iterable(
context['fileReplacePairs'])
logger.debug(f"opening source file: {in_path}")
with open(in_path) as infile:
logger.debug(f"opening destination file for writing: {out_path}")
os.makedirs(os.path.abspath(os.path.dirname(out_path)), exist_ok=True)
with open(out_path, 'w') as outfile:
outfile.writelines(
iter_replace_strings(infile, formatted_replacements))
logger.info(f"Read {in_path}, replaced strings and wrote to {out_path}")
logger.debug("done")
def run_step(context):
"""Parses input yaml file and substitutes {tokens} from context.
Loads yaml into memory to do parsing, so be aware of big files.
Args:
context: pypyr.context.Context. Mandatory.
The following context keys expected:
- fileFormatYamlIn. mandatory. path-like.
Path to source file on disk.
- fileFormatYamlOut. mandatory. path-like. Write output file to
here. Will create directories in path for you.
Returns:
None.
Raises:
FileNotFoundError: take a guess
pypyr.errors.KeyNotInContextError: fileFormatYamlIn or
fileFormatYamlOut missing in context.
pypyr.errors.KeyInContextHasNoValueError: fileFormatYamlIn or
fileFormatYamlOut exists but is None.
"""
logger.debug("started")
context.assert_keys_have_values(__name__, 'fileFormatYamlIn',
'fileFormatYamlOut')
in_path = context.get_formatted('fileFormatYamlIn')
out_path = context.get_formatted('fileFormatYamlOut')
logger.debug(f"opening yaml source file: {in_path}")
with open(in_path) as infile:
payload = yaml.load(infile, Loader=yaml.RoundTripLoader)
logger.debug(f"opening destination file for writing: {out_path}")
os.makedirs(os.path.abspath(os.path.dirname(out_path)), exist_ok=True)
with open(out_path, 'w') as outfile:
formatted_iterable = context.get_formatted_iterable(payload)
yaml.dump(formatted_iterable,
outfile,
Dumper=yaml.RoundTripDumper,
allow_unicode=True,
width=50)
logger.info(
f"Read {in_path} yaml, formatted contents and wrote to {out_path}")
logger.debug("done")
def run_step(context):
"""Loads a yaml file into the pypyr context.
Yaml parsed from the file will be merged into the pypyr context. This will
overwrite existing values if the same keys are already in there.
I.e if file yaml has {'eggs' : 'boiled'} and context {'eggs': 'fried'}
already exists, returned context['eggs'] will be 'boiled'.
Args:
context: pypyr.context.Context. Mandatory.
The following context key must exist
- fetchYamlPath. path-like. Path to file on disk.
Returns:
None. updates context arg.
Raises:
FileNotFoundError: take a guess
pypyr.errors.KeyNotInContextError: fetchYamlPath missing in context.
pypyr.errors.KeyInContextHasNoValueError: fetchYamlPath exists but is
None.
"""
logger.debug("started")
context.assert_key_has_value(key='fetchYamlPath', caller=__name__)
file_path = context.get_formatted('fetchYamlPath')
logger.debug(f"attempting to open file: {file_path}")
with open(file_path) as yaml_file:
payload = yaml.safe_load(yaml_file)
if not isinstance(payload, MutableMapping):
raise TypeError("yaml input should describe a dictionary at the top "
"level. You should have something like "
"\n'key1: value1'\n key2: value2'\n"
"in the yaml top-level, not \n'- value1\n - value2'")
logger.debug("yaml file loaded. Merging into pypyr context. . .")
context.update(payload)
logger.info(f"yaml file merged into pypyr context. Count: {len(payload)}")
logger.debug("done")
def run_step(context):
"""Loads a json file into the pypyr context.
json parsed from the file will be merged into the pypyr context. This will
overwrite existing values if the same keys are already in there.
I.e if file json has {'eggs' : 'boiled'} and context {'eggs': 'fried'}
already exists, returned context['eggs'] will be 'boiled'.
The json should not be an array [] on the top level, but rather an Object.
Args:
context: pypyr.context.Context. Mandatory.
The following context key must exist
- fetchJsonPath. path-like. Path to file on disk.
Returns:
None. updates context arg.
Raises:
FileNotFoundError: take a guess
pypyr.errors.KeyNotInContextError: fetchJsonPath missing in context.
pypyr.errors.KeyInContextHasNoValueError: fetchJsonPath exists but is
None.
"""
logger.debug("started")
context.assert_key_has_value(key='fetchJsonPath', caller=__name__)
file_path = context.get_formatted('fetchJsonPath')
logger.debug(f"attempting to open file: {file_path}")
with open(file_path) as json_file:
payload = json.load(json_file)
logger.debug("json file loaded. Merging into pypyr context. . .")
context.update(payload)
logger.info(f"json file merged into pypyr context. Count: {len(payload)}")
logger.debug("done")
def run_step(context):
"""Parses input file and substitutes {tokens} from context.
Args:
context: pypyr.context.Context. Mandatory.
The following context keys expected:
- fileFormatIn. mandatory. path-like.
Path to source file on disk.
- fileFormatOut. mandatory. path-like. Write output file to
here. Will create directories in path for you.
Returns:
None.
Raises:
FileNotFoundError: take a guess
pypyr.errors.KeyNotInContextError: fileFormatIn or fileFormatOut
missing in context.
pypyr.errors.KeyInContextHasNoValueError: fileFormatIn or fileFormatOut
exists but is None.
"""
logger.debug("started")
context.assert_keys_have_values(__name__, 'fileFormatIn', 'fileFormatOut')
in_path = context.get_formatted('fileFormatIn')
out_path = context.get_formatted('fileFormatOut')
logger.debug(f"opening source file: {in_path}")
with open(in_path) as infile:
logger.debug(f"opening destination file for writing: {out_path}")
os.makedirs(os.path.abspath(os.path.dirname(out_path)), exist_ok=True)
with open(out_path, 'w') as outfile:
outfile.writelines(context.iter_formatted_strings(infile))
logger.info(f"Read {in_path}, formatted and wrote to {out_path}")
logger.debug("done")