def test_caching():
with tempdir():
wrapper1 = Wrapper()
with open("abc.txt", "w") as f:
f.write("these are the contents")
doc1 = Doc("abc.txt|dexy", wrapper=wrapper1)
wrapper1.docs = [doc1]
wrapper1.run()
assert isinstance(doc1.artifacts[0], InitialArtifact)
hashstring_0_1 = doc1.artifacts[0].hashstring
assert isinstance(doc1.artifacts[1], FilterArtifact)
hashstring_1_1 = doc1.artifacts[1].hashstring
wrapper2 = Wrapper()
doc2 = Doc("abc.txt|dexy", wrapper=wrapper2)
wrapper2.docs = [doc2]
wrapper2.run()
assert isinstance(doc2.artifacts[0], InitialArtifact)
hashstring_0_2 = doc2.artifacts[0].hashstring
assert isinstance(doc2.artifacts[1], FilterArtifact)
hashstring_1_2 = doc2.artifacts[1].hashstring
assert hashstring_0_1 == hashstring_0_2
assert hashstring_1_1 == hashstring_1_2
def test_caching_virtual_file():
with tempdir():
wrapper1 = Wrapper()
doc1 = Doc("abc.txt|dexy",
contents = "these are the contents",
wrapper=wrapper1)
wrapper1.docs = [doc1]
wrapper1.run()
assert isinstance(doc1.artifacts[0], InitialVirtualArtifact)
hashstring_0_1 = doc1.artifacts[0].hashstring
assert isinstance(doc1.artifacts[1], FilterArtifact)
hashstring_1_1 = doc1.artifacts[1].hashstring
wrapper2 = Wrapper()
doc2 = Doc(
"abc.txt|dexy",
contents = "these are the contents",
wrapper=wrapper2)
wrapper2.docs = [doc2]
wrapper2.run()
assert isinstance(doc2.artifacts[0], InitialVirtualArtifact)
hashstring_0_2 = doc2.artifacts[0].hashstring
assert isinstance(doc2.artifacts[1], FilterArtifact)
hashstring_1_2 = doc2.artifacts[1].hashstring
assert hashstring_0_1 == hashstring_0_2
assert hashstring_1_1 == hashstring_1_2
def test_parent_doc_hash():
with tempdir():
args = [["hello.txt|newdoc", { "contents" : "hello" }]]
wrapper = Wrapper(*args)
wrapper.run()
doc = wrapper.docs[-1]
wrapper.setup_db()
rows = wrapper.get_child_hashes_in_previous_batch(doc.final_artifact.hashstring)
assert len(rows) == 3
def test_wrapper_run():
with tempdir():
wrapper = Wrapper()
wrapper.setup_dexy_dirs()
d1 = Doc("abc.txt|outputabc", contents="these are the contents", wrapper=wrapper)
d2 = Doc("hello.txt|outputabc", contents="these are more contents", wrapper=wrapper)
assert d1.state == 'setup'
assert d2.state == 'setup'
wrapper.docs = [d1, d2]
wrapper.run()
assert d1.state == 'complete'
assert d2.state == 'complete'
def test_parent_doc_hash_2():
with tempdir():
args = [["hello.txt|newdoc", { "contents" : "hello" }]]
wrapper = Wrapper(*args)
wrapper.run()
for doc in wrapper.registered:
if doc.__class__.__name__ == 'FilterArtifact':
assert doc.source == 'generated'
wrapper = Wrapper(*args)
wrapper.run()
for doc in wrapper.registered:
if doc.__class__.__name__ == 'FilterArtifact':
assert doc.source == 'cached'
def __enter__(self):
# Create a temporary working dir and move to it
self.tempdir = tempfile.mkdtemp()
self.location = os.path.abspath(os.curdir)
os.chdir(self.tempdir)
# Create a document. Skip testing documents with inactive filters.
try:
doc_key = "subdir/example%s|%s" % (self.ext, self.filter_alias)
doc_spec = [doc_key, {"contents": self.doc_contents}]
wrapper = Wrapper(doc_spec)
wrapper.run()
except InactiveFilter:
print "Skipping tests for inactive filter", self.filter_alias
raise SkipTest
return wrapper.docs[0]
def test_config_for_directory():
with wrap() as wrapper:
with open("dexy.yaml", "w") as f:
f.write(""".abc""")
with open("root.abc", "w") as f:
f.write("hello")
with open("root.def", "w") as f:
f.write("hello")
os.makedirs("s1")
os.makedirs("s2")
with open("s1/s1.abc", "w") as f:
f.write("hello")
with open("s1/s1.def", "w") as f:
f.write("hello")
with open("s2/s2.abc", "w") as f:
f.write("hello")
with open("s2/s2.def", "w") as f:
f.write("hello")
with open(os.path.join('s1', 'dexy.yaml'), 'w') as f:
f.write(""".def|dexy""")
wrapper = Wrapper()
wrapper.to_valid()
wrapper.to_walked()
wrapper.to_checked()
wrapper.run()
assert len(wrapper.nodes) == 6
p = wrapper.nodes["pattern:*.abc"]
c = wrapper.nodes["doc:s2/s2.abc"]
assert c in p.children
def test_config_for_directory():
with wrap() as wrapper:
with open("dexy.yaml", "w") as f:
f.write(""".abc""")
with open("root.abc", "w") as f:
f.write("hello")
with open("root.def", "w") as f:
f.write("hello")
os.makedirs("s1")
os.makedirs("s2")
with open("s1/s1.abc", "w") as f:
f.write("hello")
with open("s1/s1.def", "w") as f:
f.write("hello")
with open("s2/s2.abc", "w") as f:
f.write("hello")
with open("s2/s2.def", "w") as f:
f.write("hello")
with open(os.path.join('s1', 'dexy.yaml'), 'w') as f:
f.write(""".def|dexy""")
wrapper = Wrapper()
wrapper.to_valid()
wrapper.to_walked()
wrapper.to_checked()
wrapper.run()
assert len(wrapper.nodes) == 6
p = wrapper.nodes["pattern:*.abc"]
c = wrapper.nodes["doc:s2/s2.abc"]
assert c in p.children
def test_except_patterndoc_pattern():
with wrap() as wrapper:
with open("exceptme.abc", "w") as f:
f.write("hello")
wrapper = Wrapper()
wrapper.to_valid()
wrapper.nodes = {}
wrapper.roots = []
wrapper.batch = dexy.batch.Batch(wrapper)
wrapper.filemap = wrapper.map_files()
ast = AbstractSyntaxTree(wrapper)
parser = Yaml(wrapper, ast)
parser.parse('.', """.abc:\n - except : 'exceptme.*' """)
ast.walk()
wrapper.transition('walked')
wrapper.to_checked()
wrapper.run()
assert len(wrapper.nodes) == 1
def test_except_patterndoc_pattern():
with wrap() as wrapper:
with open("exceptme.abc", "w") as f:
f.write("hello")
wrapper = Wrapper()
wrapper.to_valid()
wrapper.nodes = {}
wrapper.roots = []
wrapper.batch = dexy.batch.Batch(wrapper)
wrapper.filemap = wrapper.map_files()
ast = AbstractSyntaxTree(wrapper)
parser = Yaml(wrapper, ast)
parser.parse('.', """.abc:\n - except : 'exceptme.*' """)
ast.walk()
wrapper.transition('walked')
wrapper.to_checked()
wrapper.run()
assert len(wrapper.nodes) == 1
def run(*args, **kwargs):
from dexy.wrapper import Wrapper
wrapper = Wrapper(*args, **kwargs)
wrapper.run()
wrapper.report()
def dexy_command(
artifactsdir=Wrapper.DEFAULT_ARTIFACTS_DIR, # location of directory in which to store artifacts
conf=Wrapper.DEFAULT_CONFIG_FILE, # name to use for configuration file
danger=False, # whether to allow running remote files
dbalias=Wrapper.DEFAULT_DB_ALIAS, # type of database to use
dbfile=Wrapper.DEFAULT_DB_FILE, # name of the database file (it lives in the logs dir)
# directory=".", # the directory to process, you can just process a subdirectory of your project
disabletests=False, # Whether to disable the dexy 'test' filter
dryrun=Wrapper.DEFAULT_DRYRUN, # if True, just parse config and print batch info, don't run dexy
exclude=Wrapper.DEFAULT_EXCLUDE, # directories to exclude from dexy processing
globals=Wrapper.DEFAULT_GLOBALS, # global values to make available within dexy documents, should be KEY=VALUE pairs separated by spaces
help=False, # for people who type -help out of habit
h=False, # for people who type -h out of habit
hashfunction=Wrapper.DEFAULT_HASHFUNCTION, # What hash function to use, set to crc32 or adler32 for more speed but less reliability
ignore=Wrapper.DEFAULT_IGNORE_NONZERO_EXIT, # whether to ignore nonzero exit status or raise an error - may not be supported by all filters
logfile=Wrapper.DEFAULT_LOG_FILE, # name of log file
logformat=Wrapper.DEFAULT_LOG_FORMAT, # format of log entries
loglevel=Wrapper.DEFAULT_LOG_LEVEL, # log level
logsdir=Wrapper.DEFAULT_LOG_DIR, # location of directory in which to store logs
nocache=Wrapper.DEFAULT_DONT_USE_CACHE, # whether to force artifacts to run even if there is a matching file in the cache
# output=False, # Shortcut to mean "I just want the OutputReporter, nothing else"
recurse=Wrapper.DEFAULT_RECURSE, # whether to recurse into subdirectories when running Dexy
reports=Wrapper.DEFAULT_REPORTS, # reports to be run after dexy runs, enclose in quotes and separate with spaces
# reset=False, # whether to purge existing artifacts and logs before running Dexy
# run="", # specific document to run. if specified, this document + its dependencies will be all that is run
silent=False, # Whether to not print any output when running dexy
# uselocals=True, # use cached local copies of remote URLs, faster but might not be up to date, 304 from server will override this setting
version=False # For people who type -version out of habit
):
"""
Runs Dexy, by processing your .dexy configuration file and running content
through the filters you have specified. Results are cached in the
artifacts/ directory but are presented in a more usable format by
reporters. Basic reports are run automatically but you can specify
additional reports. Type 'dexy reporters' for a list of available reporters.
If your project is large, then running reports will start to take up a lot
of time, so you should specify only the reports you really need. You can
always run more reports after a batch has finished running (you can run
historical reports as far back as the last time you cleared out your
artifacts cache with a 'dexy reset' or similar).
After running Dexy, the output/ directory will hold what dexy thinks are
the most important generated files (with pretty filenames), the output-long
directory will hold all of your generated files (with ugly filenames), and
the logs/ directory will hold the basic dexy.log logfile and also a more
colorful and descriptive HTML log file in logs/run-latest/. Please look at
these logfiles to learn more about how dexy works, and if you run into
problems the dexy.log file might provide clues as to what has gone wrong.
Your original files will be copied to logs/source-batch-00001/ by the
SourceReporter (enabled by default). Each time you run dexy, your source
code files will be copied so you have a mini-version history. (You can also
use the 'dexy history' command to get a history for a given file, and you
can run the SourceReporter again at any time to restore a given batch's
source files.)
If you run into trouble, visit http://dexy.it/help
"""
if h or help:
help_command()
elif version:
version_command()
else:
wrapper = Wrapper(**locals())
import time
start_time = time.time()
try:
wrapper.setup_config()
wrapper.run()
wrapper.report()
print "finished in %0.4f" % (time.time() - start_time)
except dexy.exceptions.UserFeedback as e:
wrapper.cleanup_partial_run()
sys.stderr.write(e.message)
if not e.message.endswith("\n"):
sys.stderr.write("\n")
sys.stderr.write("Dexy is stopping.\n")
sys.exit(1)