def test_mock_overwrite(self):
# assert 0
prefix = None
if prefix is None:
prefix = File('/tmp/spiper.symbolic/root')
prefix.dirname().rmtree_p()
_d = spiper.rcParams.copy()
spiper.rcParams['dir_layout'] = 'clean'
tarball_main( mock_run, prefix)
fs = sorted(prefix.fileglob('*',0,0)); print(pprint(fs))
assert fs == [
File('/tmp/spiper.symbolic/root.gen_files.out_txt'),
File('/tmp/spiper.symbolic/root.gen_files.out_txt.empty.mock'),
File('/tmp/spiper.symbolic/root.gen_files.out_txt.old.mock'),
File('/tmp/spiper.symbolic/root.tarball_dangerous_cache.cmd'),
File('/tmp/spiper.symbolic/root.tarball_dangerous_cache.cmd.empty.mock'),
File('/tmp/spiper.symbolic/root.tarball_dangerous_cache.cmd.old.mock'),
File('/tmp/spiper.symbolic/root.tarball_dangerous_cache.tar_gz'),
File('/tmp/spiper.symbolic/root.tarball_dangerous_cache.tar_gz.empty.mock'),
File('/tmp/spiper.symbolic/root.tarball_dangerous_cache.tar_gz.old.mock'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.cmd'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.cmd.empty.mock'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.cmd.old.mock'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.tar_gz'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.tar_gz.empty.mock'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.tar_gz.old.mock')]
# tarball_main( lambda *a:cache_run(*a,check_changed=2), prefix)
tarball_main( cache_run, prefix)
fs = sorted(prefix.fileglob('*',0,0)); print(pprint(fs))
assert fs == [
File('/tmp/spiper.symbolic/root.gen_files.out_txt'),
File('/tmp/spiper.symbolic/root.tarball_dangerous_cache.cmd'),
File('/tmp/spiper.symbolic/root.tarball_dangerous_cache.tar_gz'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.cmd'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.tar_gz')]
tarball_main( mock_run , prefix)
fs = sorted(prefix.fileglob('*',0,0)); print(pprint(fs))
assert fs == [
File('/tmp/spiper.symbolic/root.gen_files.out_txt'),
File('/tmp/spiper.symbolic/root.tarball_dangerous_cache.cmd'),
File('/tmp/spiper.symbolic/root.tarball_dangerous_cache.tar_gz'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.cmd'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.tar_gz')]
with open((prefix + '.gen_files.out_txt'),'w') as f:
f.write('100'*2000)
tarball_main( mock_run , prefix)
fs = sorted(prefix.fileglob('*',0,0)); print(pprint(fs))
assert fs == [
File('/tmp/spiper.symbolic/root.gen_files.out_txt'),
File('/tmp/spiper.symbolic/root.gen_files.out_txt.old.mock'),
File('/tmp/spiper.symbolic/root.tarball_dangerous_cache.cmd'),
File('/tmp/spiper.symbolic/root.tarball_dangerous_cache.tar_gz'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.cmd'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.cmd.old.mock'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.tar_gz'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.tar_gz.old.mock')]
tarball_main( cache_run, prefix)
fs = sorted(prefix.fileglob('*',0,0)); print(pprint(fs))
assert fs == [
File('/tmp/spiper.symbolic/root.gen_files.out_txt'),
File('/tmp/spiper.symbolic/root.tarball_dangerous_cache.cmd'),
File('/tmp/spiper.symbolic/root.tarball_dangerous_cache.tar_gz'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.cmd'),
File('/tmp/spiper.symbolic/root.tarball_prefix_cache.tar_gz')]
# res = LoggedShellCommand(['ls -lhtr',prefix+'*'])
# .fileglob('*'),])
# print(res)
# prefix
# tarball_main( mock_run , prefix)
# assert 0
# (prefix.dirname()/'root.tarball_dangerous_cache.tar_gz').touch()
# self.assertRaises(spiper._types.OverwriteError, tarball_main, mock_run, prefix)
spiper.rcParams.update(_d)
def main(self=None,
prefix = None):
from spiper.runner import cache_run, mock_run, get_changed_files, get_all_files
from spiper.shell import LoggedShellCommand
from spiper.types import File,CacheFile
from pprint import pprint
spiper.rcParams['dir_layout']='clean'
# if prefix is None:
prefix = Path('/tmp/spiper.symbolic/root')
# backup_prefix = File('/home/user/.temp/backup_03_mock_flow/root')
backup_prefix = File('~/.temp/backup_03_mock_flow/root').expand()
prefix.dirname().rmtree_p()
backup_prefix.dirname().rmtree_p()
print('\n...[start]%r'%prefix)
#### once a workflow is defined, we can view the proposed file changes
fs = get_changed_files(workflow, prefix, 1, 100, verbose=0)
pprint(fs)
assert fs ==[
File('/tmp/spiper.symbolic/root.workflow.log'),
File('/tmp/spiper.symbolic/root.random_seq.seq'),
File('/tmp/spiper.symbolic/root.random_seq_const.seq'),
File('/tmp/spiper.symbolic/root.transcribe.fasta'),
File('/tmp/spiper.symbolic/root.mutate.fasta'),
File('/tmp/spiper.symbolic/root.source.py'),
# File('/home/user/.temp/backup_03_mock_flow/root.source.py')
]
### backup is conveniently defined as a workflow taking an executed workflow as an input.
### To check the proposed backup, mock_run() the workflow first.
workflow_out = mock_run(workflow, prefix, 1, 100)
fs = get_changed_files(backup, backup_prefix, workflow_out)
pprint(fs)
assert fs == [
File('/home/user/.temp/backup_03_mock_flow/root.subflow.random_seq.output.seq'),
File('/home/user/.temp/backup_03_mock_flow/root.subflow.random_seq_const.output.seq'),
File('/home/user/.temp/backup_03_mock_flow/root.subflow.transcribe.output.fasta'),
File('/home/user/.temp/backup_03_mock_flow/root.subflow.mutate.output.fasta'),
File('/home/user/.temp/backup_03_mock_flow/root.output.log'),
# File('/tmp/spiper.symbolic/root.source.py')
File('/home/user/.temp/backup_03_mock_flow/root.source.py')
]
### a convenient Flow may be defined to execute the two in chain
### If there is certain change to the workflow,
### the backup can also be runned
fs = get_changed_files (run_and_backup, prefix, 1, 100, backup_prefix, verbose=0)
pprint(fs)
assert fs == [
File('/tmp/spiper.symbolic/root.workflow.log'),
File('/tmp/spiper.symbolic/root.random_seq.seq'),
File('/tmp/spiper.symbolic/root.random_seq_const.seq'),
File('/tmp/spiper.symbolic/root.transcribe.fasta'),
File('/tmp/spiper.symbolic/root.mutate.fasta'),
File('/tmp/spiper.symbolic/root.source.py'),
File('/home/user/.temp/backup_03_mock_flow/root.subflow.random_seq.output.seq'),
File('/home/user/.temp/backup_03_mock_flow/root.subflow.random_seq_const.output.seq'),
File('/home/user/.temp/backup_03_mock_flow/root.subflow.transcribe.output.fasta'),
File('/home/user/.temp/backup_03_mock_flow/root.subflow.mutate.output.fasta'),
File('/home/user/.temp/backup_03_mock_flow/root.output.log'),
File('/home/user/.temp/backup_03_mock_flow/root.source.py'),
File('/home/user/.temp/backup_03_mock_flow/root.plot_graph.deptree_json'),
File('/home/user/.temp/backup_03_mock_flow/root.plot_graph.deptree_dot_txt'),
]
###### constants that are preserved between runs should be detected unchanged
_ = cache_run (run_and_backup, prefix, 1, 100, backup_prefix, verbose=0)
fs = get_changed_files (run_and_backup, prefix, 2, 200, backup_prefix, verbose=0)
pprint(fs)
assert fs == [File('/tmp/spiper.symbolic/root.workflow.log'),
File('/tmp/spiper.symbolic/root.random_seq.seq'),
# File('/tmp/spiper.symbolic/root.random_seq_const.seq'),
File('/tmp/spiper.symbolic/root.transcribe.fasta'),
File('/tmp/spiper.symbolic/root.mutate.fasta'),
# File('/tmp/spiper.symbolic/root.source.py'),
File('/home/user/.temp/backup_03_mock_flow/root.subflow.random_seq.output.seq'),
# File('/home/user/.temp/backup_03_mock_flow/root.subflow.random_seq_const.output.seq'),
File('/home/user/.temp/backup_03_mock_flow/root.subflow.transcribe.output.fasta'),
File('/home/user/.temp/backup_03_mock_flow/root.subflow.mutate.output.fasta'),
File('/home/user/.temp/backup_03_mock_flow/root.output.log'),
# File('/home/user/.temp/backup_03_mock_flow/root.source.py'),
File('/home/user/.temp/backup_03_mock_flow/root.plot_graph.deptree_json'),
File('/home/user/.temp/backup_03_mock_flow/root.plot_graph.deptree_dot_txt'),
]
##### get_all_files() return a leaf file regardless of whether is is changed
fs = get_all_files (run_and_backup, prefix, 2, 200, backup_prefix, verbose=0)
pprint(fs)
assert fs == [
File('/tmp/spiper.symbolic/root.workflow.log'),
File('/tmp/spiper.symbolic/root.random_seq.seq'),
File('/tmp/spiper.symbolic/root.random_seq_const.seq'),
File('/tmp/spiper.symbolic/root.transcribe.fasta'),
File('/tmp/spiper.symbolic/root.mutate.fasta'),
File('/tmp/spiper.symbolic/root.source.py'),
File('/home/user/.temp/backup_03_mock_flow/root.subflow.random_seq.output.seq'),
File('/home/user/.temp/backup_03_mock_flow/root.subflow.random_seq_const.output.seq'),
File('/home/user/.temp/backup_03_mock_flow/root.subflow.transcribe.output.fasta'),
File('/home/user/.temp/backup_03_mock_flow/root.subflow.mutate.output.fasta'),
File('/home/user/.temp/backup_03_mock_flow/root.output.log'),
File('/home/user/.temp/backup_03_mock_flow/root.source.py'),
File('/home/user/.temp/backup_03_mock_flow/root.plot_graph.deptree_json'),
File('/home/user/.temp/backup_03_mock_flow/root.plot_graph.deptree_dot_txt'),
]
_ = cache_run (run_and_backup, prefix, 2, 200, backup_prefix, verbose=0)