def test_setup_message_with_config():
with tempdir():
with divert_stdout() as stdout:
with open(Constants.DEFAULT_CONFIG, "w") as f:
f.write(SIMPLE_PY_CONFIG)
setup_command()
assert "You are now ready" in stdout.getvalue()
def test_remove_trash_with_trash():
with tempdir():
wrapper = Wrapper()
os.mkdir(".trash")
assert os.path.exists(".trash")
wrapper.empty_trash()
assert not os.path.exists(".trash")
def test_repo_from_invalid_path():
with tempdir():
try:
repo, remote = repo_from_path(".")
assert False
except UserFeedback as e:
assert "no git repository was found at '.'" in str(e)
def test_zip_archive_filter():
with tempdir():
with open("hello.py", "w") as f:
f.write("print 'hello'")
with open("hello.rb", "w") as f:
f.write("puts 'hello'")
wrapper = Wrapper()
wrapper.create_dexy_dirs()
wrapper = Wrapper()
doc = Doc("archive.zip|zip",
wrapper, [
Doc("hello.py", wrapper),
Doc("hello.rb", wrapper),
Doc("hello.py|pyg", wrapper),
Doc("hello.rb|pyg", wrapper)
],
contents=" ")
wrapper.run_docs(doc)
wrapper.report()
path_exists = os.path.exists("output/archive.zip")
assert path_exists
z = zipfile.ZipFile("output/archive.zip", "r")
names = z.namelist()
assert "archive/hello.py" in names
assert "archive/hello.rb" in names
assert "archive/hello.py-pyg.html" in names
assert "archive/hello.rb-pyg.html" in names
z.close()
def test_wrapper_register():
with tempdir():
doc = Doc("abc.txt")
wrapper = Wrapper()
wrapper.setup_run()
wrapper.register(doc)
assert doc in wrapper.registered
def test_init_sqlite_row_read():
with tempdir():
with open("example.sqlite3", "w") as f:
f.write("fake data")
storage = RowData("example.sqlite3")
assert storage.ext == ".sqlite3"
assert storage.mode == "read"
def test_init_kyotocabinet_read():
with tempdir():
with open("example.kch", "w") as f:
f.write("fake data")
storage = KeyValueData("example.kch")
assert storage.ext == ".kch"
assert storage.mode == "read"
def test_config_file():
with tempdir():
with open("dexy.conf", "w") as f:
f.write("""{ "logfile" : "a.log" }""")
wrapper = init_wrapper({'conf': 'dexy.conf'})
assert wrapper.log_file == "a.log"
def test_init_json_read():
with tempdir():
with open("example.json", "w") as f:
f.write("""{}""")
storage = RowData("example.json")
assert storage.ext == ".json"
assert storage.mode == "read"
def test_caching_virtual_file():
with tempdir():
runner1 = Runner()
doc1 = Doc("abc.txt|dexy",
contents="these are the contents",
runner=runner1)
runner1.docs = [doc1]
runner1.run()
assert isinstance(doc1.artifacts[0], InitialVirtualArtifact)
hashstring_0_1 = doc1.artifacts[0].hashstring
assert isinstance(doc1.artifacts[1], FilterArtifact)
hashstring_1_1 = doc1.artifacts[1].hashstring
runner2 = Runner()
doc2 = Doc("abc.txt|dexy",
contents="these are the contents",
runner=runner2)
runner2.docs = [doc2]
runner2.run()
assert isinstance(doc2.artifacts[0], InitialVirtualArtifact)
hashstring_0_2 = doc2.artifacts[0].hashstring
assert isinstance(doc2.artifacts[1], FilterArtifact)
hashstring_1_2 = doc2.artifacts[1].hashstring
assert hashstring_0_1 == hashstring_0_2
assert hashstring_1_1 == hashstring_1_2
def test_repo_from_invalid_path():
with tempdir():
try:
repo, remote = repo_from_path(".")
assert False
except UserFeedback as e:
assert "no git repository was found at '.'" in str(e)
def test_caching():
with tempdir():
wrapper1 = Wrapper()
with open("abc.txt", "w") as f:
f.write("these are the contents")
doc1 = Doc("abc.txt|dexy", wrapper=wrapper1)
wrapper1.docs = [doc1]
wrapper1.run()
assert isinstance(doc1.artifacts[0], InitialArtifact)
hashstring_0_1 = doc1.artifacts[0].hashstring
assert isinstance(doc1.artifacts[1], FilterArtifact)
hashstring_1_1 = doc1.artifacts[1].hashstring
wrapper2 = Wrapper()
doc2 = Doc("abc.txt|dexy", wrapper=wrapper2)
wrapper2.docs = [doc2]
wrapper2.run()
assert isinstance(doc2.artifacts[0], InitialArtifact)
hashstring_0_2 = doc2.artifacts[0].hashstring
assert isinstance(doc2.artifacts[1], FilterArtifact)
hashstring_1_2 = doc2.artifacts[1].hashstring
assert hashstring_0_1 == hashstring_0_2
assert hashstring_1_1 == hashstring_1_2
def test_logging_to_file():
with tempdir():
os.mkdir(Constants.DEFAULT_LDIR)
r = Reporter()
assert len(r.log.handlers) == 1
assert isinstance(r.log.handlers[0], RotatingFileHandler)
remove_all_handlers(r.log)
def test_caching_virtual_file():
with tempdir():
wrapper1 = Wrapper()
doc1 = Doc("abc.txt|dexy",
contents = "these are the contents",
wrapper=wrapper1)
wrapper1.docs = [doc1]
wrapper1.run()
assert isinstance(doc1.artifacts[0], InitialVirtualArtifact)
hashstring_0_1 = doc1.artifacts[0].hashstring
assert isinstance(doc1.artifacts[1], FilterArtifact)
hashstring_1_1 = doc1.artifacts[1].hashstring
wrapper2 = Wrapper()
doc2 = Doc(
"abc.txt|dexy",
contents = "these are the contents",
wrapper=wrapper2)
wrapper2.docs = [doc2]
wrapper2.run()
assert isinstance(doc2.artifacts[0], InitialVirtualArtifact)
hashstring_0_2 = doc2.artifacts[0].hashstring
assert isinstance(doc2.artifacts[1], FilterArtifact)
hashstring_1_2 = doc2.artifacts[1].hashstring
assert hashstring_0_1 == hashstring_0_2
assert hashstring_1_1 == hashstring_1_2
def test_args_copied():
config = {
"." : {
"*.md|jinja" : { "inputs" : ["*.txt"] },
"*.txt" : { "meta" : 5 }
},
}
with tempdir():
for filename in ["abc.txt", "def.txt", "template.md"]:
with open(filename, "w") as f:
f.write("hello")
i = 0
for doc in run_dexy_without_tempdir(config):
i += 1
doc.run()
if doc.key().endswith(".txt"):
assert doc.output() == "hello"
assert doc.last_artifact.args['meta'] == 5
assert len(doc.inputs) == 0
elif doc.key().endswith(".md"):
assert len(doc.inputs) == 2
assert i == 3
def test_init_csv_read():
with tempdir():
with open("example.csv", "w") as f:
f.write("some data")
storage = RowData("example.csv")
assert storage.ext == ".csv"
assert storage.mode == "read"
def test_runner_register():
with tempdir():
doc = Doc("abc.txt")
runner = Runner()
runner.setup_dexy_dirs()
runner.register(doc)
assert doc in runner.registered
def test_runner_register():
with tempdir():
doc = Doc("abc.txt")
runner = Runner()
runner.setup_dexy_dirs()
runner.register(doc)
assert doc in runner.registered
def test_caching():
with tempdir():
runner1 = Runner()
with open("abc.txt", "w") as f:
f.write("these are the contents")
doc1 = Doc("abc.txt|dexy", runner=runner1)
runner1.docs = [doc1]
runner1.run()
assert isinstance(doc1.artifacts[0], InitialArtifact)
hashstring_0_1 = doc1.artifacts[0].hashstring
assert isinstance(doc1.artifacts[1], FilterArtifact)
hashstring_1_1 = doc1.artifacts[1].hashstring
runner2 = Runner()
doc2 = Doc("abc.txt|dexy", runner=runner2)
runner2.docs = [doc2]
runner2.run()
assert isinstance(doc2.artifacts[0], InitialArtifact)
hashstring_0_2 = doc2.artifacts[0].hashstring
assert isinstance(doc2.artifacts[1], FilterArtifact)
hashstring_1_2 = doc2.artifacts[1].hashstring
assert hashstring_0_1 == hashstring_0_2
assert hashstring_1_1 == hashstring_1_2
def test_error_if_to_valid_called_without_dirs_setup():
with tempdir():
wrapper = Wrapper()
try:
wrapper.to_valid()
assert False, "should not get here"
except InternalDexyProblem:
assert True
def test_create_remove_dexy_dirs():
with tempdir():
wrapper = Wrapper()
wrapper.create_dexy_dirs()
wrapper.to_valid()
assert wrapper.dexy_dirs_exist()
wrapper.remove_dexy_dirs()
assert not wrapper.dexy_dirs_exist()
def test_assert_dexy_dirs():
with tempdir():
wrapper = Wrapper()
try:
wrapper.assert_dexy_dirs_exist()
assert False
except UserFeedback:
assert True
def test_init_wrapper():
with tempdir():
with open("dexy.conf", "w") as f:
f.write("artifactsdir: custom")
modargs = {}
wrapper = dexy.commands.utils.init_wrapper(modargs)
assert wrapper.artifacts_dir == 'custom'
def test_conf_command_if_path_exists(stdout):
with tempdir():
with open("dexy.conf", "w") as f:
f.write("foo")
assert os.path.exists("dexy.conf")
dexy.commands.run()
assert "dexy.conf already exists" in stdout.getvalue()
assert "artifactsdir" in stdout.getvalue()
def test_commands_dexy_without_setup():
with tempdir():
try:
dexy_command()
assertFalse # should not get here
except UserFeedback as e:
assert "Please run 'dexy setup'" in e.message
assert not check_setup(logsdir=Constants.DEFAULT_LDIR, artifactsdir=Constants.DEFAULT_LDIR)
def test_invalid_json():
with tempdir():
with open(".dexy", "w") as f:
f.write(INVALID_JSON_CONFIG)
setup_command()
try:
dexy_command()
except UserFeedback as e:
assert "Your config file .dexy has invalid JSON" in e.message
def test_write_read_csv():
with tempdir():
storage = RowData("example.csv", ["x", "y"])
storage.append(1, 1)
storage.append(2, 4)
storage.save()
storage = RowData("example.csv")
assert storage.read() == "x,y\r\n1,1\r\n2,4\r\n"
def test_basic():
with tempdir():
for doc in run_dexy_without_tempdir(BASIC_CONFIG):
doc.run()
assert doc.artifacts[-1].source == 'run'
for doc in run_dexy_without_tempdir(BASIC_CONFIG):
doc.run()
assert doc.artifacts[-1].source == 'cache'
def test_next_batch_id():
with tempdir():
os.mkdir(Constants.DEFAULT_LDIR)
db = CsvDatabase()
batch_id = db.next_batch_id()
assert batch_id == 1
batch_id = db.next_batch_id()
assert batch_id == 2
def test_deprecated_dot_dexy_file():
with tempdir():
with open(".dexy", 'w') as f:
f.write("{}")
wrapper = Wrapper()
try:
wrapper.assert_dexy_dirs_exist()
except UserFeedback as e:
assert "this format is no longer supported" in str(e)
def test_setup_with_dexy_conf_file():
with tempdir():
with open("dexy.conf", "w") as f:
f.write("artifactsdir: custom")
dexy.commands.run()
assert os.path.exists("custom")
assert os.path.isdir("custom")
assert not os.path.exists("artifacts")
def test_commands_dexy_without_setup():
with tempdir():
with divert_stdout() as stdout:
try:
dexy_command()
assertFalse # should not get here
except SystemExit as e:
assert "Please run 'dexy setup'" in stdout.getvalue()
assert not check_setup(logsdir=Constants.DEFAULT_LDIR, artifactsdir=Constants.DEFAULT_LDIR)
assert e.code == 1
def test_commands_reset_custom_location():
with tempdir():
logsdir = "logsx"
artifactsdir = "artifactsx"
setup_command(logsdir=logsdir, artifactsdir=artifactsdir)
assert check_setup(logsdir=logsdir, artifactsdir=artifactsdir)
assert os.path.exists(artifactsdir)
assert os.path.exists(logsdir)
reset_command(logsdir=logsdir, artifactsdir=artifactsdir)
assert check_setup(logsdir=logsdir, artifactsdir=artifactsdir)
def test_parse_doc_configs_no_configs():
with tempdir():
with capture_stdout() as stdout:
wrapper = Wrapper()
wrapper.create_dexy_dirs()
wrapper = Wrapper()
wrapper.to_valid()
wrapper.to_walked()
value = stdout.getvalue()
assert "didn't find any document config files" in value
def test_run():
with tempdir():
fn = modargs.function_for(dexy.commands, "dexy")
args = modargs.determine_kwargs(fn)
args['globals'] = []
os.mkdir(args['logsdir'])
c = Controller(args)
c.config = SIMPLE_PY_CONFIG
c.process_config()
assert c.members.has_key("simple.py|py")
assert isinstance(c.members["simple.py|py"], Document)
def test_parent_doc_hash():
with tempdir():
args = [["hello.txt|newdoc", { "contents" : "hello" }]]
wrapper = Wrapper(*args)
wrapper.run()
doc = wrapper.docs[-1]
wrapper.setup_db()
rows = wrapper.get_child_hashes_in_previous_batch(doc.final_artifact.hashstring)
assert len(rows) == 3
def test_parent_doc_hash():
with tempdir():
params = RunParams()
args = [["hello.txt|newdoc", { "contents" : "hello" }]]
runner = Runner(params, args)
runner.run()
doc = runner.docs[-1]
runner.setup_db()
rows = runner.get_child_hashes_in_previous_batch(doc.final_artifact.hashstring)
assert len(rows) == 3
def test_wrapper_run():
with tempdir():
wrapper = Wrapper()
wrapper.setup_dexy_dirs()
d1 = Doc("abc.txt|outputabc", contents="these are the contents", wrapper=wrapper)
d2 = Doc("hello.txt|outputabc", contents="these are more contents", wrapper=wrapper)
assert d1.state == 'setup'
assert d2.state == 'setup'
wrapper.docs = [d1, d2]
wrapper.run()
assert d1.state == 'complete'
assert d2.state == 'complete'
def test_config_applies_in_subdirectory():
with tempdir():
with open(".dexy", "wb") as f:
json.dump({"*.txt" : {}}, f)
os.makedirs("abc")
args = controller_args()
c = Controller(args)
c.load_config()
assert c.config['.'].has_key("*.txt")
assert c.config['./abc'].has_key("*.txt")
def test_kwargs_override_config_file():
with tempdir():
with open("dexy.conf", "w") as f:
f.write("""{ "logfile" : "a.log" }""")
wrapper = init_wrapper({
'__cli_options': {
'logfile': 'b.log'
},
'logfile': "b.log",
'conf': 'dexy.conf'
})
assert wrapper.log_file == "b.log"
def test_move_cache_dir():
with capture_stdout() as stdout:
with tempdir():
os.mkdir(".cache")
with open(".cache/.dexy-generated", 'w') as f:
f.write("")
wrapper = Wrapper()
wrapper.assert_dexy_dirs_exist()
assert "Moving directory '.cache'" in stdout.getvalue()
assert not os.path.exists(".cache")
assert os.path.exists(".dexy")
def test_parent_doc_hash():
with tempdir():
params = RunParams()
args = [["hello.txt|newdoc", {"contents": "hello"}]]
runner = Runner(params, args)
runner.run()
doc = runner.docs[-1]
runner.setup_db()
rows = runner.get_child_hashes_in_previous_batch(
doc.final_artifact.hashstring)
assert len(rows) == 3
def test_commands_setup_cleanup():
with tempdir():
with divert_stdout() as stdout:
setup_command()
assert "You are almost ready" in stdout.getvalue()
assert os.path.exists(Constants.DEFAULT_ADIR)
assert os.path.exists(Constants.DEFAULT_LDIR)
assert check_setup(logsdir=Constants.DEFAULT_LDIR, artifactsdir=Constants.DEFAULT_ADIR)
cleanup_command()
assert not os.path.exists(Constants.DEFAULT_ADIR)
assert not os.path.exists(Constants.DEFAULT_LDIR)
assert not check_setup(logsdir=Constants.DEFAULT_LDIR, artifactsdir=Constants.DEFAULT_LDIR)
def test_walked():
with tempdir():
with open("dexy.yaml", "w") as f:
f.write("foo.txt")
with open("foo.txt", "w") as f:
f.write("foo")
wrapper = Wrapper()
wrapper.create_dexy_dirs()
wrapper.to_valid()
wrapper.to_walked()
wrapper.validate_state('walked')
def test_batch():
with tempdir():
wrapper = Wrapper()
wrapper.create_dexy_dirs()
wrapper = Wrapper()
batch = dexy.batch.Batch(wrapper)
os.makedirs(batch.batch_dir())
batch.save_to_file()
assert batch.filename() in os.listdir(".dexy/batches")
wrapper = Wrapper()
batch = dexy.batch.Batch.load_most_recent(wrapper)
def test_batch():
with tempdir():
wrapper = Wrapper()
wrapper.create_dexy_dirs()
wrapper = Wrapper()
batch = dexy.batch.Batch(wrapper)
os.makedirs(batch.batch_dir())
batch.save_to_file()
assert batch.filename() in os.listdir(".dexy/batches")
wrapper = Wrapper()
batch = dexy.batch.Batch.load_most_recent(wrapper)
def test_run_with_userfeedback_exception(stderr):
with tempdir():
with open("docs.txt", "w") as f:
f.write("*.py|py")
with open("hello.py", "w") as f:
f.write("raise")
try:
dexy.commands.run()
assert False, 'should raise SystemExit'
except SystemExit as e:
assert e.message == 1
assert "Dexy is stopping" in stderr.getvalue()
def test_parse_doc_configs_single_empty_config():
with tempdir():
wrapper = Wrapper()
wrapper.create_dexy_dirs()
with open("dexy.yaml", "w") as f:
f.write("foo.txt")
with open("foo.txt", "w") as f:
f.write("foo")
wrapper = Wrapper()
wrapper.to_valid()
wrapper.to_walked()
def test_old_cache_dir_with_settings():
with capture_stdout() as stdout:
with tempdir():
os.mkdir(".cache")
with open(".cache/.dexy-generated", 'w') as f:
f.write("")
wrapper = Wrapper(artifacts_dir=".cache")
wrapper.assert_dexy_dirs_exist()
assert os.path.exists(".cache")
assert not os.path.exists(".dexy")
assert "You may have a dexy.conf file" in stdout.getvalue()
def test_init_wrapper_if_dexy_dirs_exist():
with tempdir():
wrapper = Wrapper()
wrapper.create_dexy_dirs()
with open("hello.txt", "w") as f:
f.write("hello")
wrapper = Wrapper()
wrapper.to_valid()
assert wrapper.project_root
wrapper.to_walked()
assert 'hello.txt' in wrapper.filemap
assert 'dexy.log' in os.listdir('.dexy')
assert not '.dexy/dexy.log' in wrapper.filemap
def test_runner_run():
with tempdir():
runner = Runner()
runner.setup_dexy_dirs()
d1 = Doc("abc.txt|outputabc",
contents="these are the contents",
runner=runner)
d2 = Doc("hello.txt|outputabc",
contents="these are more contents",
runner=runner)
assert d1.state == 'setup'
assert d2.state == 'setup'
runner.docs = [d1, d2]
runner.run()
assert d1.state == 'complete'
assert d2.state == 'complete'
def test_parent_doc_hash_2():
with tempdir():
params = RunParams()
args = [["hello.txt|newdoc", {"contents": "hello"}]]
runner = Runner(params, args)
runner.run()
for doc in runner.registered:
if doc.__class__.__name__ == 'FilterArtifact':
assert doc.source == 'generated'
runner = Runner(params, args)
runner.run()
for doc in runner.registered:
if doc.__class__.__name__ == 'FilterArtifact':
assert doc.source == 'cached'
def test_ran():
with tempdir():
with open("dexy.yaml", "w") as f:
f.write("foo.txt")
with open("foo.txt", "w") as f:
f.write("foo")
wrapper = Wrapper()
wrapper.create_dexy_dirs()
wrapper.run_from_new()
for node in wrapper.roots:
assert node.state == 'ran'
wrapper.validate_state('ran')
wrapper = Wrapper()
wrapper.run_from_new()
for node in wrapper.roots:
assert node.state == 'consolidated'
wrapper.validate_state('ran')
def test_batch_with_docs():
with tempdir():
wrapper = Wrapper(log_level='DEBUG', debug=True)
wrapper.create_dexy_dirs()
with open("hello.txt", "w") as f:
f.write("hello")
with open("dexy.yaml", "w") as f:
f.write("hello.txt")
wrapper = Wrapper()
wrapper.run_from_new()
batch = dexy.batch.Batch.load_most_recent(wrapper)
assert batch
for doc_key in batch.docs:
assert batch.input_data(doc_key)
assert batch.output_data(doc_key)
def test_nodexy_files():
with tempdir():
wrapper = Wrapper()
wrapper.create_dexy_dirs()
with open("hello.txt", "w") as f:
f.write("hello")
os.makedirs("s1/s2/s3")
nodexy_path = "s1/s2/.nodexy"
with open(nodexy_path, 'w') as f:
f.write("dexy stop here")
with open("s1/s2/ignore.txt", "w") as f:
f.write("dexy should ignore this")
with open("s1/s2/s3/ignore.txt", "w") as f:
f.write("dexy should also ignore this")
# Only the hello.txt file is visible to dexy
wrapper = Wrapper()
wrapper.to_valid()
wrapper.to_walked()
assert len(wrapper.filemap) == 1
assert 'hello.txt' in wrapper.filemap
os.remove(nodexy_path)
# Now we can see all 3 text files.
wrapper = Wrapper()
wrapper.to_valid()
wrapper.to_walked()
assert len(wrapper.filemap) == 3
assert 'hello.txt' in wrapper.filemap
assert 's1/s2/ignore.txt' in wrapper.filemap
assert 's1/s2/s3/ignore.txt' in wrapper.filemap
def test_cache_and_dexy_dirs_present():
with tempdir():
os.mkdir(".dexy")
os.mkdir(".cache")
with open(".dexy/.dexy-generated", 'w') as f:
f.write("")
with open(".cache/.dexy-generated", 'w') as f:
f.write("")
wrapper = Wrapper()
try:
wrapper.assert_dexy_dirs_exist()
except UserFeedback as e:
assert "Please remove '.cache'" in str(e)
os.remove(".cache/.dexy-generated")
wrapper.assert_dexy_dirs_exist()
# Cache still exists but dexy just ignores it.
assert os.path.exists(".cache")
# Dexy uses .dexy dir
assert os.path.exists(".dexy")
def test_run_dexy(stdout):
with tempdir():
wrapper = Wrapper()
wrapper.create_dexy_dirs()
dexy.commands.run()
