def run_dexy(args):
# validate args and do any conversions required
args['globals'] = dict([g.split("=") for g in args['globals'].split()])
args['exclude'] = [x.strip("/") for x in args['exclude'].split()]
controller = Controller(args)
controller.run()
return controller
def test_run():
with tempdir():
fn = modargs.function_for(dexy.commands, "dexy")
args = modargs.determine_kwargs(fn)
args['globals'] = []
os.mkdir(args['logsdir'])
c = Controller(args)
c.config = SIMPLE_PY_CONFIG
c.process_config()
assert c.members.has_key("simple.py|py")
assert isinstance(c.members["simple.py|py"], Document)
def test_config_applies_in_subdirectory():
with tempdir():
with open(".dexy", "wb") as f:
json.dump({"*.txt" : {}}, f)
os.makedirs("abc")
args = controller_args()
c = Controller(args)
c.load_config()
assert c.config['.'].has_key("*.txt")
assert c.config['./abc'].has_key("*.txt")
def run_example(name, script_output, run=False):
os.chdir("../docs/guide/creating-content/")
os.chdir(name)
populate_filters_log, populate_filters_logstream = stream_logger("%s.populate.filters.log" % name)
log, logstream = stream_logger("%s.log" % name)
with divert_stdout() as stdout:
Document.filter_list = dexy.introspect.filters(populate_filters_log)
controller = Controller(args)
controller.log = log
controller.load_config()
controller.process_config()
controller.virtual_docs = []
for doc in controller.docs:
doc.log = log
[doc.setup() for doc in controller.docs]
controller.docs = [doc.run() for doc in controller.docs]
script_output["%s-run" % name] = stdout.getvalue()
script_output['docs'] = dict((doc.key(), doc.last_artifact.data_dict) for doc in controller.docs)
print script_output['docs']
script_output["%s-populate-filters" % name] = populate_filters_logstream.getvalue()
script_output["%s-log" % name] = logstream.getvalue()
os.chdir("..")
os.chdir("../../../artifacts")
def run_example(name, script_output, run=False):
log, logstream = stream_logger("%s.log" % name)
populate_filters_log, populate_filters_logstream= stream_logger("%s.populate.filters.log" % name)
os.chdir(name)
with divert_stdout() as stdout:
### @export "controller-config"
controller = Controller(args)
controller.log = log
controller.load_config()
controller.process_config()
controller.virtual_docs = []
### @end
if run:
for doc in controller.docs:
doc.log = log
### @export "populate-filters"
Document.filter_list = dexy.introspect.filters(populate_filters_log)
### @export "controller-run"
[doc.setup() for doc in controller.docs]
controller.docs = [doc.run() for doc in controller.docs]
### @end
script_output["%s-run" % name] = stdout.getvalue()
os.chdir("..")
script_output["%s-depends" % name] = repr(controller.depends)
script_output["%s-docs" % name] = print_controller_docs(controller)
script_output["%s-log" % name] = logstream.getvalue()
script_output["%s-populate-filters" % name] = populate_filters_logstream.getvalue()
script_output["%s-members" % name] = print_controller_members(controller)
script_output["%s-ordering" % name] = repr(controller.ordering)
script_output["%s-doc-logs" % name] = dict((doc.key(), doc.logstream.getvalue()) for doc in controller.docs)
def setup_controller(config_file = None):
controller = Controller()
controller.artifacts_dir = 'artifacts'
if not os.path.isdir(controller.artifacts_dir):
os.mkdir(controller.artifacts_dir)
controller.artifact_class = FileSystemJsonArtifact
if config_file:
controller.config_file = config_file
else:
controller.config_file = '.dexy'
controller.load_config("tests/data")
controller.setup_and_run()
return controller
def test_circular_dependencies():
with tempdir():
fn = modargs.function_for(dexy.commands, "dexy")
args = modargs.determine_kwargs(fn)
args['globals'] = []
os.mkdir(args['logsdir'])
args['danger'] = True
c = Controller(args)
c.config = CIRCULAR_CONFIG
with divert_stdout() as stdout:
try:
c.process_config()
assert False
except CycleError:
assert True
stdout_text = stdout.getvalue()
assert "abc depends on ghi" in stdout_text
assert "def depends on abc" in stdout_text
assert "ghi depends on def" in stdout_text
def test_docs_with_no_filters():
with tempdir():
fn = modargs.function_for(dexy.commands, "dexy")
args = modargs.determine_kwargs(fn)
args['globals'] = []
os.mkdir(args['logsdir'])
c = Controller(args)
c.config = NO_FILTERS_CONFIG
c.process_config()
assert c.members.has_key("hello.txt")
assert isinstance(c.members["hello.txt"], Document)
assert sorted(c.batch_info().keys()) == [
"args",
"config",
"docs",
"elapsed",
"finish_time",
"id",
"start_time",
"timing"
]
def setup_controller():
controller = Controller()
controller.artifacts_dir = 'artifacts'
if not os.path.isdir(controller.artifacts_dir):
os.mkdir(controller.artifacts_dir)
controller.artifact_class = FileSystemJsonArtifact
controller.allow_remote = True
controller.config = {
'tests/data' : {
"@simple.py|pyg" : {
"contents" : "x = 5\nx^2"
}
}
}
controller.setup_and_run()
return controller
def run_dexy_without_tempdir(config_dict, additional_args={}):
if not hasattr(Document, 'filter_list'):
Document.filter_list = dexy.introspect.filters()
args = controller_args(additional_args)
c = Controller(args)
c.config = config_dict
c.process_config()
[doc.setup() for doc in c.docs]
for doc in c.docs:
yield(doc)
c.persist()
def run_dexy_without_tempdir(config_dict, additional_args={}):
if not hasattr(Document, "filter_list"):
Document.filter_list = dexy.introspect.filters()
fn = modargs.function_for(dexy.commands, "dexy")
args = modargs.determine_kwargs(fn)
args.update(additional_args)
if not os.path.exists(args["logsdir"]):
os.mkdir(args["logsdir"])
if not os.path.exists(args["artifactsdir"]):
os.mkdir(args["artifactsdir"])
c = Controller(args)
c.config = config_dict
c.process_config()
[doc.setup() for doc in c.docs]
for doc in c.docs:
yield (doc)
c.persist()
def test_handlers():
"""controller: find_handlers() should not raise errors"""
controller = Controller()
handlers = controller.find_handlers()
assert handlers['dexy'] == DexyHandler
def setup_option_parser():
project_base = os.path.abspath(os.curdir)
os.chdir(project_base)
def add_option(parser, *args, **kwargs):
if hasattr(parser, "add_option"):
parser.add_option(*args, **kwargs)
else:
parser.add_argument(*args, **kwargs)
try:
# These are argparse-specific
import argparse
option_parser = "argparse"
parser = argparse.ArgumentParser()
parser.add_argument("dir", help="directory of files to process with dexy", default=".", nargs="?")
parser.add_argument("-x", "--exclude", help=EXCLUDE_HELP, nargs="+")
parser.add_argument("-v", "--version", action="version", version="%%(prog)s %s" % VERSION)
except ImportError:
# These are optparse-specific
import optparse
option_parser = "optparse"
parser = optparse.OptionParser(version="%%prog %s" % VERSION)
parser.add_option("-x", "--exclude", help=EXCLUDE_HELP + " Separate multiple directories with commas.")
# Remaining options are the same for argparse and optparse
add_option(
parser,
"-n",
"--no-recurse",
dest="recurse",
default=True,
action="store_false",
help="do not recurse into subdirectories (default: recurse)",
)
add_option(
parser,
"-u",
"--utf8",
default=False,
action="store_true",
help="switch encoding to UTF-8 (default: don't change encoding)",
)
add_option(
parser, "-p", "--purge", default=False, action="store_true", help="purge all artifacts before running dexy"
)
add_option(
parser,
"--cleanup",
default=False,
action="store_true",
help="delete all dexy-generated directories and files (does not run dexy)",
)
add_option(
parser, "--filters", default=False, action="store_true", help="list all available filters (does not run dexy)"
)
add_option(
parser,
"--reporters",
default=False,
action="store_true",
help="list all available reporters (does not run dexy)",
)
add_option(
parser,
"--artifact-class",
default="FileSystemJsonArtifact",
help="name of artifact class to use (default: FileSystemJsonArtifact)",
)
add_option(
parser,
"-a",
"--artifacts-dir",
default="artifacts",
help="location of artifacts directory (default: artifacts)",
)
add_option(
parser,
"-l",
"--logs-dir",
default="logs",
help="""location of logs directory (default: logs)
dexy will create a dexy.log file in this directory
reporters may create reports in this directory""",
)
add_option(
parser,
"-c",
"--cache-dir",
default="cache",
help="DEPRECATED - does not do anything anymore, will be removed soon",
)
add_option(
parser,
"--local",
default=False,
action="store_true",
help="Use cached local copies of remote urls - faster but might not be up to date",
)
add_option(
parser,
"--ignore-errors",
default=False,
action="store_true",
help="""Don't raise an error if scripts return nonzero exit status
(depends on the filter being written to support this)""",
)
add_option(
parser,
"--setup",
default=False,
action="store_true",
help="Create artifacts and logs directory and generic .dexy file",
)
add_option(
parser,
"--serve",
default=False,
action="store_true",
help="Start a static server within the output directory, will be visible on port 8000. (similar to python -m SimpleHTTPServer)",
)
add_option(parser, "-g", "--config", default=".dexy", help="name of configuration file")
add_option(
parser,
"-d",
"--dangerous",
default=False,
action="store_true",
help="Allow running remote URLs which may execute dangerous code, use with care.",
)
add_option(
parser,
"--no-reports",
default=False,
action="store_true",
help="""Don't run reports when finished running Dexy.""",
)
if option_parser == "argparse":
args = parser.parse_args()
dir_name = args.dir
if args.exclude:
additional_excludes = args.exclude
else:
additional_excludes = []
elif option_parser == "optparse":
(args, argv) = parser.parse_args()
if len(argv) == 0:
dir_name = "."
else:
dir_name = argv[0]
if args.exclude:
additional_excludes = args.exclude.split(",")
else:
additional_excludes = []
else:
raise Exception("unexpected option_parser %s" % option_parser)
args.run_dexy = True
if args.utf8:
if sys.getdefaultencoding() == "UTF-8":
print "encoding is already UTF-8"
else:
print "changing encoding from %s to UTF-8" % sys.getdefaultencoding()
reload(sys)
sys.setdefaultencoding("UTF-8")
if not os.path.exists(dir_name):
raise Exception("file %s not found!" % dir_name)
if args.setup:
if not os.path.exists(args.artifacts_dir):
os.mkdir(args.artifacts_dir)
if not os.path.exists(args.logs_dir):
os.mkdir(args.logs_dir)
if not os.path.exists(".dexy"):
f = open(".dexy", "w")
f.write("{\n}\n")
f.close()
if args.config.startswith("http"):
print "fetching remote config file %s" % args.config
filename = os.path.basename(args.config)
print "writing to %s" % filename
f = urllib.urlopen(args.config)
config_file = open(filename, "w")
config_file.write(f.read())
config_file.close()
f.close()
args.config = filename
if not os.path.exists(args.artifacts_dir):
path_to_artifacts_dir = os.path.join(project_base, args.artifacts_dir)
raise Exception(
"""artifacts directory not found at %s,
please create a directory called %s if you want to use
this location as a dexy project root"""
% (path_to_artifacts_dir, args.artifacts_dir)
)
if not os.path.exists(args.logs_dir):
path_to_logs_dir = os.path.join(project_base, args.logs_dir)
raise Exception(
"""logs directory not found at %s,
please create a directory called %s if you want to use
this location as a dexy project root"""
% (path_to_logs_dir, args.logs_dir)
)
# Set up main dexy log
dexy_log = logging.getLogger("dexy")
dexy_log.setLevel(logging.DEBUG)
logfile = os.path.join(args.logs_dir, "dexy.log")
handler = RotatingFileHandler(logfile)
dexy_log.addHandler(handler)
if args.setup:
# This might be the first time someone has run Dexy,
# let them know where the logfile is
print "dexy will log debugging information to", logfile
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler.setFormatter(formatter)
args.artifact_class = artifact_class(args.artifact_class, dexy_log)
if args.purge:
dexy_log.warn("purging contents of %s" % args.artifacts_dir)
shutil.rmtree(args.artifacts_dir)
os.mkdir(args.artifacts_dir)
# Each artifact class may need to do its own purging also
if hasattr(args.artifact_class, "purge"):
args.artifact_class.purge()
if args.cleanup:
args.run_dexy = False
print "purging contents of %s" % args.artifacts_dir
shutil.rmtree(args.artifacts_dir)
if os.path.exists(args.cache_dir):
print "purging contents of %s" % args.cache_dir
shutil.rmtree(args.cache_dir)
if os.path.exists(args.logs_dir):
print "purging contents of %s" % args.logs_dir
shutil.rmtree(args.logs_dir)
controller = Controller()
controller.find_reporters()
for d in controller.reports_dirs:
if d and os.path.exists(d):
print "purging contents of %s" % d
shutil.rmtree(d)
filters_dir = "filters"
# TODO improve this so it detects __init__.py customizations etc.
if os.path.exists(filters_dir):
if sorted(os.listdir(filters_dir)) == ["README", "__init__.py"]:
dexy_log.warn("purging contents of %s" % filters_dir)
shutil.rmtree(filters_dir)
else:
print ("Directory %s has been modified, not removing." % filters_dir)
if args.reporters:
args.run_dexy = False
controller = Controller()
reporters = controller.find_reporters()
for r in reporters:
print r.__name__
if r.REPORTS_DIR:
print "any generated reports will be saved in", r.REPORTS_DIR
else:
print "any generated reports will be saved in", args.logs_dir
if r.__doc__:
print r.__doc__
else:
print "no documentation available"
print # finish with blank line
print "Running reports can be disabled with the --no-reports option"
if args.filters:
args.run_dexy = False
controller = Controller()
handlers = controller.find_handlers()
for k in sorted(handlers.keys()):
klass = handlers[k]
print
print k, ":", klass.__name__
if klass.executable():
print " calls", klass.executable()
if klass.version_command():
try:
raw_version = klass.version().strip()
if raw_version.find("\n") > 0:
# Clean up long version strings like R's
version = raw_version.split("\n")[0]
else:
version = raw_version
print " version", version, "detected using", klass.version_command()
except Exception: # TODO raise/catch specific Exception
print " may not be installed, was unable to run", klass.version_command()
if klass.__doc__:
print " ", klass.__doc__.strip()
return args, dir_name, additional_excludes, dexy_log
def setup_controller():
args, dir_name, additional_excludes, log = setup_option_parser()
if not args.run_dexy:
return None, args, log
controller = Controller()
controller.dir_name = dir_name
controller.args = args
controller.allow_remote = args.dangerous
controller.artifact_class = args.artifact_class
controller.artifacts_dir = args.artifacts_dir
controller.logs_dir = args.logs_dir
controller.log = log
controller.config_file = args.config
controller.use_local_files = args.local
controller.find_reporters()
controller.additional_excludes = additional_excludes
return controller, args, log