def run_from_cache_a_bunch_of_times():
n = random.randint(2, 10)
print "running %s times:" % n
for i in range(n):
print '', i + 1
wrapper = Wrapper(log_level=LOGLEVEL, debug=True)
wrapper.run_from_new()
for node in wrapper.nodes.values():
assert_node_state(node, 'consolidated', "In iter %s" % i)
wrapper.report()
def run_from_cache_a_bunch_of_times():
n = random.randint(2, 10)
print "running %s times:" % n
for i in range(n):
print '', i+1
wrapper = Wrapper(log_level=LOGLEVEL)
wrapper.run_from_new()
for node in wrapper.nodes.values():
assert_node_state(node, 'consolidated', "In iter %s" % i)
wrapper.report()
def test_hard_tabs_in_config():
with wrap():
with capture_stderr() as stderr:
os.makedirs("abc/def")
with open("abc/def/dexy.yaml", "w") as f:
f.write("""foo:\t- .txt""")
wrapper = Wrapper()
try:
wrapper.run_from_new()
except UserFeedback as e:
assert "hard tabs" in str(e)
assert "abc/def/dexy.yaml" in stderr.getvalue()
def test_hard_tabs_in_config():
with wrap():
with capture_stderr() as stderr:
os.makedirs("abc/def")
with open("abc/def/dexy.yaml", "w") as f:
f.write("""foo:\t- .txt""")
wrapper = Wrapper()
try:
wrapper.run_from_new()
except UserFeedback as e:
assert "hard tabs" in str(e)
assert "abc/def/dexy.yaml" in stderr.getvalue()
def test_ran():
with tempdir():
with open("dexy.yaml", "w") as f:
f.write("foo.txt")
with open("foo.txt", "w") as f:
f.write("foo")
wrapper = Wrapper()
wrapper.create_dexy_dirs()
wrapper.run_from_new()
for node in wrapper.roots:
assert node.state == 'ran'
wrapper.validate_state('ran')
wrapper = Wrapper()
wrapper.run_from_new()
for node in wrapper.roots:
assert node.state == 'consolidated'
wrapper.validate_state('ran')
def test_batch_with_docs():
with tempdir():
wrapper = Wrapper(log_level='DEBUG', debug=True)
wrapper.create_dexy_dirs()
with open("hello.txt", "w") as f:
f.write("hello")
with open("dexy.yaml", "w") as f:
f.write("hello.txt")
wrapper = Wrapper()
wrapper.run_from_new()
batch = dexy.batch.Batch.load_most_recent(wrapper)
assert batch
for doc_key in batch.docs:
assert batch.input_data(doc_key)
assert batch.output_data(doc_key)
def test_ran():
with tempdir():
with open("dexy.yaml", "w") as f:
f.write("foo.txt")
with open("foo.txt", "w") as f:
f.write("foo")
wrapper = Wrapper()
wrapper.create_dexy_dirs()
wrapper.run_from_new()
for node in wrapper.roots:
assert node.state == 'ran'
wrapper.validate_state('ran')
wrapper = Wrapper()
wrapper.run_from_new()
for node in wrapper.roots:
assert node.state == 'consolidated'
wrapper.validate_state('ran')
def test_batch_with_docs():
with tempdir():
wrapper = Wrapper(log_level='DEBUG', debug=True)
wrapper.create_dexy_dirs()
with open("hello.txt", "w") as f:
f.write("hello")
with open("dexy.yaml", "w") as f:
f.write("hello.txt")
wrapper = Wrapper()
wrapper.run_from_new()
batch = dexy.batch.Batch.load_most_recent(wrapper)
assert batch
for doc_key in batch.docs:
assert batch.input_data(doc_key)
assert batch.output_data(doc_key)
def test_subdir_config_with_bundle():
with wrap():
with open("dexy.yaml", "w") as f:
f.write("""
foo:
- .txt
""")
os.makedirs("abc/def")
with open("abc/def/dexy.yaml", "w") as f:
f.write("""
bar:
- .py
""")
with open("abc/def/hello.py", "w") as f:
f.write("print 'hello'")
wrapper = Wrapper()
wrapper.run_from_new()
assert "doc:abc/def/hello.py" in wrapper.nodes
wrapper = Wrapper(recurse=False)
wrapper.run_from_new()
assert not "doc:abc/def/hello.py" in wrapper.nodes
wrapper = Wrapper(recurse=False, configs="abc/def/dexy.yaml")
wrapper.run_from_new()
assert "doc:abc/def/hello.py" in wrapper.nodes
def test_script_node_caching__slow():
with wrap():
with open("start.sh", "w") as f:
f.write("pwd")
with open("middle.sh", "w") as f:
f.write("echo `time`")
with open("end.sh", "w") as f:
f.write("echo 'done'")
with open("dexy.yaml", "w") as f:
f.write(SCRIPT_YAML)
wrapper1 = Wrapper()
wrapper1.run_from_new()
for node in wrapper1.nodes.values():
assert node.state == 'ran'
wrapper2 = Wrapper()
wrapper2.run_from_new()
for node in wrapper2.nodes.values():
assert node.state == 'consolidated'
time.sleep(1.1)
with open("middle.sh", "w") as f:
f.write("echo 'new'")
wrapper3 = Wrapper()
wrapper3.run_from_new()
for node in wrapper1.nodes.values():
assert node.state == 'ran'
def test_subdir_config_with_bundle():
with wrap():
with open("dexy.yaml", "w") as f:
f.write("""
foo:
- .txt
""")
os.makedirs("abc/def")
with open("abc/def/dexy.yaml", "w") as f:
f.write("""
bar:
- .py
""")
with open("abc/def/hello.py", "w") as f:
f.write("print 'hello'")
wrapper = Wrapper()
wrapper.run_from_new()
assert "doc:abc/def/hello.py" in wrapper.nodes
wrapper = Wrapper(recurse=False)
wrapper.run_from_new()
assert not "doc:abc/def/hello.py" in wrapper.nodes
wrapper = Wrapper(recurse=False, configs="abc/def/dexy.yaml")
wrapper.run_from_new()
assert "doc:abc/def/hello.py" in wrapper.nodes
def test_script_node_caching__slow():
with wrap():
with open("start.sh", "w") as f:
f.write("pwd")
with open("middle.sh", "w") as f:
f.write("echo `time`")
with open("end.sh", "w") as f:
f.write("echo 'done'")
with open("dexy.yaml", "w") as f:
f.write(SCRIPT_YAML)
wrapper1 = Wrapper()
wrapper1.run_from_new()
for node in wrapper1.nodes.values():
assert node.state == 'ran'
wrapper2 = Wrapper()
wrapper2.run_from_new()
for node in wrapper2.nodes.values():
assert node.state == 'consolidated'
time.sleep(1.1)
with open("middle.sh", "w") as f:
f.write("echo 'new'")
wrapper3 = Wrapper()
wrapper3.run_from_new()
for node in wrapper1.nodes.values():
assert node.state == 'ran'
def test_example_project():
with tempdir():
def run_from_cache_a_bunch_of_times():
n = random.randint(2, 10)
print "running %s times:" % n
for i in range(n):
print '', i + 1
wrapper = Wrapper(log_level=LOGLEVEL, debug=True)
wrapper.run_from_new()
for node in wrapper.nodes.values():
assert_node_state(node, 'consolidated', "In iter %s" % i)
wrapper.report()
example_src = os.path.join(TEST_DATA_DIR, 'example')
shutil.copytree(example_src, "example")
os.chdir("example")
wrapper = Wrapper(log_level=LOGLEVEL)
wrapper.create_dexy_dirs()
wrapper.run_from_new()
wrapper.report()
for node in wrapper.nodes.values():
assert_node_state(node, 'ran', "in first run")
run_from_cache_a_bunch_of_times()
# touch this file so it triggers cache updating
os.utime("multiply.py", None)
unaffected_keys = (
'latex',
'pygments.sty|pyg',
's1/loop.py|pycon',
's1/loop.py|py',
'main.rst|idio|h',
'main.rst|idio|l',
'main.rst|pyg|l',
'main.rst|pyg|h',
's1/loop.py|idio|h',
's1/loop.py|idio|l',
's1/loop.py|pyg|l',
's1/loop.py|pyg|h',
'dexy.yaml|idio|h',
'dexy.yaml|idio|l',
'dexy.yaml|pyg|l',
'dexy.yaml|pyg|h',
)
affected_keys = (
'code',
'docs',
"*|pyg|l",
"*|pyg|h",
"*|idio|l",
"*|idio|h",
"main.rst|jinja|rst|latex",
"*.rst|jinja|rst|latex",
"*.py|pycon",
"*.py|py",
"main.rst|jinja|rstbody|easyhtml",
"*.rst|jinja|rstbody|easyhtml",
"foo.txt",
"multiply.py|idio|h",
"multiply.py|idio|l",
"multiply.py|pycon",
"multiply.py|py",
"multiply.py|pyg|h",
"multiply.py|pyg|l",
)
wrapper = Wrapper(log_level=LOGLEVEL)
wrapper.run_from_new()
wrapper.report()
for node in wrapper.nodes.values():
if node.key in unaffected_keys:
assert_node_state(node, 'consolidated',
"after touching multiply.py")
else:
assert node.key in affected_keys, node.key
assert_node_state(node, 'ran', "after touchimg multiply.py")
run_from_cache_a_bunch_of_times()
import time
time.sleep(0.5)
with open("multiply.py", "r") as f:
old_content = f.read()
with open("multiply.py", "w") as f:
f.write("raise")
wrapper = Wrapper(log_level=LOGLEVEL)
wrapper.run_from_new()
assert wrapper.state == 'error'
import time
time.sleep(0.9)
with open("multiply.py", "w") as f:
f.write(old_content)
wrapper = Wrapper(log_level=LOGLEVEL)
wrapper.run_from_new()
for node in wrapper.nodes.values():
if node.key in unaffected_keys:
assert_node_state(node, 'consolidated',
"after restoring old multiply.py content")
else:
assert node.key in affected_keys, node.key
assert_node_state(node, 'ran',
"after restoring old multiply.py contnet")
wrapper.remove_dexy_dirs()
wrapper.remove_reports_dirs(keep_empty_dir=True)
wrapper.create_dexy_dirs()
assert len(os.listdir(".dexy")) == 1
wrapper = Wrapper(log_level=LOGLEVEL, dry_run=True)
wrapper.run_from_new()
wrapper.report()
assert len(os.listdir(".dexy")) == 6
with open(".dexy/reports/graph.txt", "r") as f:
graph_text = f.read()
assert "BundleNode(docs) (uncached)" in graph_text
os.chdir("..")
def test_example_project():
with tempdir():
def run_from_cache_a_bunch_of_times():
n = random.randint(2, 10)
print "running %s times:" % n
for i in range(n):
print '', i+1
wrapper = Wrapper(log_level=LOGLEVEL)
wrapper.run_from_new()
for node in wrapper.nodes.values():
assert_node_state(node, 'consolidated', "In iter %s" % i)
wrapper.report()
example_src = os.path.join(TEST_DATA_DIR, 'example')
shutil.copytree(example_src, "example")
os.chdir("example")
wrapper = Wrapper(log_level=LOGLEVEL)
wrapper.create_dexy_dirs()
wrapper.run_from_new()
wrapper.report()
for node in wrapper.nodes.values():
assert_node_state(node, 'ran')
run_from_cache_a_bunch_of_times()
# touch this file so it triggers cache updating
os.utime("multiply.py", None)
unaffected_keys = ('latex', 'pygments.sty|pyg', 's1/loop.py|pycon', 's1/loop.py|py',
'main.rst|idio|h', 'main.rst|idio|l', 'main.rst|pyg|l', 'main.rst|pyg|h',
's1/loop.py|idio|h', 's1/loop.py|idio|l', 's1/loop.py|pyg|l', 's1/loop.py|pyg|h',
'dexy.yaml|idio|h', 'dexy.yaml|idio|l', 'dexy.yaml|pyg|l', 'dexy.yaml|pyg|h',
)
affected_keys = ('code', 'docs', "*|pyg|l", "*|pyg|h", "*|idio|l", "*|idio|h",
"main.rst|jinja|rst|latex", "*.rst|jinja|rst|latex",
"*.py|pycon", "*.py|py", "main.rst|jinja|rstbody|easyhtml",
"*.rst|jinja|rstbody|easyhtml", "foo.txt",
"multiply.py|idio|h", "multiply.py|idio|l", "multiply.py|pycon", "multiply.py|py",
"multiply.py|pyg|h", "multiply.py|pyg|l",
)
wrapper = Wrapper(log_level=LOGLEVEL)
wrapper.run_from_new()
wrapper.report()
for node in wrapper.nodes.values():
if node.key in unaffected_keys:
assert_node_state(node, 'consolidated')
else:
assert node.key in affected_keys, node.key
assert_node_state(node, 'ran')
run_from_cache_a_bunch_of_times()
import time
time.sleep(0.5)
with open("multiply.py", "r") as f:
old_content = f.read()
with open("multiply.py", "w") as f:
f.write("raise")
wrapper = Wrapper(log_level=LOGLEVEL)
wrapper.run_from_new()
assert wrapper.state == 'error'
import time
time.sleep(0.9)
with open("multiply.py", "w") as f:
f.write(old_content)
wrapper = Wrapper(log_level=LOGLEVEL)
wrapper.run_from_new()
for node in wrapper.nodes.values():
if node.key in unaffected_keys:
assert_node_state(node, 'consolidated')
else:
assert node.key in affected_keys, node.key
assert_node_state(node, 'ran')
wrapper.remove_dexy_dirs()
wrapper.remove_reports_dirs(keep_empty_dir=True)
wrapper.create_dexy_dirs()
assert len(os.listdir(".dexy")) == 1
wrapper = Wrapper(log_level=LOGLEVEL, dry_run=True)
wrapper.run_from_new()
wrapper.report()
assert len(os.listdir(".dexy")) == 6
with open(".dexy/reports/graph.txt", "r") as f:
graph_text = f.read()
assert "BundleNode(docs) (uncached)" in graph_text
os.chdir("..")
