def dump_yaml(s, build_dir):
# Enable dumping multiline strings as block literals
#
# Block literals look like this:
#
# foo: |
# line 1
# line 2
# line 3
#
# The indentation of the first line is used as the common indentation
# for the entire block, so when read as a string it looks like this:
#
# >>> line 1
# >>> line 2
# >>> line 3
#
def str_representer(dumper, data):
tmp = {'tag': 'tag:yaml.org,2002:str', 'value': data}
if len(data.splitlines()) > 1:
tmp.update({'style': '|'})
return dumper.represent_scalar(**tmp)
yaml.add_representer(str, str_representer)
# Dump the content
write_yaml(
data=s._config,
path=build_dir + '/configure.yml',
)
def save_construct_path(s, construct_path):
yaml_path = '.mflowgen.yml'
try:
data = read_yaml(yaml_path)
except Exception:
data = {}
data['construct'] = construct_path
write_yaml(data=data, path=yaml_path)
def update_stash( s ):
write_yaml(
data = s.stash,
path = s.stash_yaml_path,
)
def set_stash_path( s, path ):
s.link_path = path
write_yaml(
data = { 'path' : s.link_path },
path = s.link_path_yaml,
)
def launch_push(s, help_, step, msg, all_):
try:
author = os.environ['USER']
except KeyError:
author = 'Unknown'
# Help message
def print_help():
print()
print(bold('Usage:'), 'mflowgen stash push',
'--step/-s <int> --message/-m "<str>"', '[--all]')
print()
print(bold('Example:'), 'mflowgen stash push',
'--step 5 -m "foo bar"')
print()
print('Pushes a built step to the mflowgen stash. The given step')
print('is copied to the stash, preserving all permissions and')
print('following all symlinks. By default, only the outputs of a')
print('step are stashed, but the entire step can be stashed with')
print('--all. The stashed copy is given a hash stamp and is')
print('marked as authored by $USER (' + author + '). An optional')
print('message can also be attached to each push.')
print()
if help_ or step == None or not msg:
print_help()
return
# Sanity-check the stash
s.verify_stash()
# Get step to push
#
# Check the current directory and search for a dirname matching the
# given step number.
#
# - E.g., if "step" is 5 then look for any sub-directory that starts
# with "5-". If there is a directory called
# '4-synopsys-dc-synthesis' then "step_name" will be
# "synopsys-dc-synthesis"
#
build_dirs = os.listdir('.')
targets = [_ for _ in build_dirs if _.startswith(str(step) + '-')]
try:
push_target = targets[0]
except IndexError:
print(bold('Error:'), 'No build directory found for step',
'{}'.format(step))
sys.exit(1)
# Create a unique name for the stashed copy of this step
today = datetime.today()
datestamp = datetime.strftime(today, '%Y-%m%d')
hashstamp = s.gen_unique_hash()
step_name = '-'.join(push_target.split('-')[1:])
dst_dirname = '-'.join([datestamp, step_name, hashstamp])
# Try to get some information to help describe "where this step came
# from"
def get_shell_output(cmd):
try:
output = subprocess.check_output(cmd.split(),
stderr=subprocess.DEVNULL,
universal_newlines=True)
output = output.strip()
except Exception:
output = ''
return output
def get_hostname():
import socket
return socket.gethostname()
git_cmd = 'git rev-parse --short HEAD' # git commit hash
git_hash = get_shell_output(git_cmd)
git_cmd = 'git rev-parse --show-toplevel' # git root dir
git_repo = get_shell_output(git_cmd)
git_repo = os.path.basename(git_repo)
build_path = os.getcwd() # build dir path
hostname = get_hostname() # hostname
stashed_from = {
'stashed-from-git-root-dir': git_repo,
'stashed-from-git-root-dir-hash': git_hash,
'stashed-from-dir': build_path,
'stashed-from-hostname': hostname,
}
# Helper function to ignore copying files other than the outputs
def f_ignore(path, files):
# For nested subdirectories, ignore all files
if '/' in path:
if path.split('/')[1] == 'outputs':
return [] # ignore nothing in outputs
elif path.split('/')[1] in ['logs', 'reports']: # TEMPORARY
return [] # ignore nothing # TEMPORARY
else:
return files # ignore everything for any other directory
# At the top level, keep the outputs and a few other misc files
keep = [
'outputs',
'logs', # TEMPORARY
'reports', # TEMPORARY
'configure.yml',
'mflowgen-run.log',
'.time_start',
'.time_end',
'.stamp',
'.execstamp',
'.postconditions.stamp',
]
ignore = [_ for _ in files if _ not in keep]
return ignore
# Now copy src to dst
#
# - symlinks = False # Follow all symlinks
# - ignore_dangling_symlinks = False # Stop with error if we cannot
# # follow a link to something
# # we need
# - ignore = (func) # Ignore all but the outputs
# # unless "--all" was given
#
remote_path = s.get_stash_path() + '/' + dst_dirname
try:
shutil.copytree(src=push_target,
dst=remote_path,
symlinks=False,
ignore=None if all_ else f_ignore,
ignore_dangling_symlinks=False)
except shutil.Error:
# According to online discussion, ignore_dangling_symlinks does not
# apply recursively within sub-directories (a bug):
#
# - https://bugs.python.org/issue38523
#
# But according to more online discussion, copytree finishes copying
# everything else before raising the exception.
#
# - https://bugs.python.org/issue6547
#
# So if we just pass here, we will have everything except dangling
# symlinks, which is fine for our use case. Dangling symlinks can
# happen in a few situations:
#
# 1. In inputs, if users cleaned earlier dependent steps. In this
# situation, we are just doing our best to copy what is available.
#
# 2. Some symlink to somewhere we do not have permission to view.
# It would be nice to raise an exception in this case, but that is
# hard to differentiate.
#
pass
except Exception as e:
print(bold('Error:'), 'Failed to complete stash push')
shutil.rmtree(path=remote_path, ignore_errors=True) # clean up
raise
# Update the metadata in the stash
push_metadata = {
'date': datestamp,
'dir': dst_dirname,
'hash': hashstamp,
'author': author,
'step': step_name,
'msg': msg,
'stashed-from': stashed_from,
}
s.stash.append(push_metadata)
s.update_stash()
# Try adding the metadata to the stashed step itself, so that when the
# step gets pulled somewhere, we have all of its metadata to know
# where it came from
try:
data = push_metadata
data.update({'stash-dir': s.link_path}) # add stash dir
write_yaml(
data=data,
path=remote_path + '/.mflowgen.stash.node.yml',
)
except Exception as e:
pass
print('Stashed step {step} "{step_name}" as author "{author}"'.format(
step=step,
step_name=step_name,
author=author,
))
