def run_dexy(args): # validate args and do any conversions required args['globals'] = dict([g.split("=") for g in args['globals'].split()]) args['exclude'] = [x.strip("/") for x in args['exclude'].split()] controller = Controller(args) controller.run() return controller
def test_run(): with tempdir(): fn = modargs.function_for(dexy.commands, "dexy") args = modargs.determine_kwargs(fn) args['globals'] = [] os.mkdir(args['logsdir']) c = Controller(args) c.config = SIMPLE_PY_CONFIG c.process_config() assert c.members.has_key("simple.py|py") assert isinstance(c.members["simple.py|py"], Document)
def test_config_applies_in_subdirectory(): with tempdir(): with open(".dexy", "wb") as f: json.dump({"*.txt" : {}}, f) os.makedirs("abc") args = controller_args() c = Controller(args) c.load_config() assert c.config['.'].has_key("*.txt") assert c.config['./abc'].has_key("*.txt")
def run_example(name, script_output, run=False): os.chdir("../docs/guide/creating-content/") os.chdir(name) populate_filters_log, populate_filters_logstream = stream_logger("%s.populate.filters.log" % name) log, logstream = stream_logger("%s.log" % name) with divert_stdout() as stdout: Document.filter_list = dexy.introspect.filters(populate_filters_log) controller = Controller(args) controller.log = log controller.load_config() controller.process_config() controller.virtual_docs = [] for doc in controller.docs: doc.log = log [doc.setup() for doc in controller.docs] controller.docs = [doc.run() for doc in controller.docs] script_output["%s-run" % name] = stdout.getvalue() script_output['docs'] = dict((doc.key(), doc.last_artifact.data_dict) for doc in controller.docs) print script_output['docs'] script_output["%s-populate-filters" % name] = populate_filters_logstream.getvalue() script_output["%s-log" % name] = logstream.getvalue() os.chdir("..") os.chdir("../../../artifacts")
def run_example(name, script_output, run=False): log, logstream = stream_logger("%s.log" % name) populate_filters_log, populate_filters_logstream= stream_logger("%s.populate.filters.log" % name) os.chdir(name) with divert_stdout() as stdout: ### @export "controller-config" controller = Controller(args) controller.log = log controller.load_config() controller.process_config() controller.virtual_docs = [] ### @end if run: for doc in controller.docs: doc.log = log ### @export "populate-filters" Document.filter_list = dexy.introspect.filters(populate_filters_log) ### @export "controller-run" [doc.setup() for doc in controller.docs] controller.docs = [doc.run() for doc in controller.docs] ### @end script_output["%s-run" % name] = stdout.getvalue() os.chdir("..") script_output["%s-depends" % name] = repr(controller.depends) script_output["%s-docs" % name] = print_controller_docs(controller) script_output["%s-log" % name] = logstream.getvalue() script_output["%s-populate-filters" % name] = populate_filters_logstream.getvalue() script_output["%s-members" % name] = print_controller_members(controller) script_output["%s-ordering" % name] = repr(controller.ordering) script_output["%s-doc-logs" % name] = dict((doc.key(), doc.logstream.getvalue()) for doc in controller.docs)
def setup_controller(config_file = None): controller = Controller() controller.artifacts_dir = 'artifacts' if not os.path.isdir(controller.artifacts_dir): os.mkdir(controller.artifacts_dir) controller.artifact_class = FileSystemJsonArtifact if config_file: controller.config_file = config_file else: controller.config_file = '.dexy' controller.load_config("tests/data") controller.setup_and_run() return controller
def test_circular_dependencies(): with tempdir(): fn = modargs.function_for(dexy.commands, "dexy") args = modargs.determine_kwargs(fn) args['globals'] = [] os.mkdir(args['logsdir']) args['danger'] = True c = Controller(args) c.config = CIRCULAR_CONFIG with divert_stdout() as stdout: try: c.process_config() assert False except CycleError: assert True stdout_text = stdout.getvalue() assert "abc depends on ghi" in stdout_text assert "def depends on abc" in stdout_text assert "ghi depends on def" in stdout_text
def test_docs_with_no_filters(): with tempdir(): fn = modargs.function_for(dexy.commands, "dexy") args = modargs.determine_kwargs(fn) args['globals'] = [] os.mkdir(args['logsdir']) c = Controller(args) c.config = NO_FILTERS_CONFIG c.process_config() assert c.members.has_key("hello.txt") assert isinstance(c.members["hello.txt"], Document) assert sorted(c.batch_info().keys()) == [ "args", "config", "docs", "elapsed", "finish_time", "id", "start_time", "timing" ]
def setup_controller(): controller = Controller() controller.artifacts_dir = 'artifacts' if not os.path.isdir(controller.artifacts_dir): os.mkdir(controller.artifacts_dir) controller.artifact_class = FileSystemJsonArtifact controller.allow_remote = True controller.config = { 'tests/data' : { "@simple.py|pyg" : { "contents" : "x = 5\nx^2" } } } controller.setup_and_run() return controller
def run_dexy_without_tempdir(config_dict, additional_args={}): if not hasattr(Document, 'filter_list'): Document.filter_list = dexy.introspect.filters() args = controller_args(additional_args) c = Controller(args) c.config = config_dict c.process_config() [doc.setup() for doc in c.docs] for doc in c.docs: yield(doc) c.persist()
def run_dexy_without_tempdir(config_dict, additional_args={}): if not hasattr(Document, "filter_list"): Document.filter_list = dexy.introspect.filters() fn = modargs.function_for(dexy.commands, "dexy") args = modargs.determine_kwargs(fn) args.update(additional_args) if not os.path.exists(args["logsdir"]): os.mkdir(args["logsdir"]) if not os.path.exists(args["artifactsdir"]): os.mkdir(args["artifactsdir"]) c = Controller(args) c.config = config_dict c.process_config() [doc.setup() for doc in c.docs] for doc in c.docs: yield (doc) c.persist()
def test_handlers(): """controller: find_handlers() should not raise errors""" controller = Controller() handlers = controller.find_handlers() assert handlers['dexy'] == DexyHandler
def setup_option_parser(): project_base = os.path.abspath(os.curdir) os.chdir(project_base) def add_option(parser, *args, **kwargs): if hasattr(parser, "add_option"): parser.add_option(*args, **kwargs) else: parser.add_argument(*args, **kwargs) try: # These are argparse-specific import argparse option_parser = "argparse" parser = argparse.ArgumentParser() parser.add_argument("dir", help="directory of files to process with dexy", default=".", nargs="?") parser.add_argument("-x", "--exclude", help=EXCLUDE_HELP, nargs="+") parser.add_argument("-v", "--version", action="version", version="%%(prog)s %s" % VERSION) except ImportError: # These are optparse-specific import optparse option_parser = "optparse" parser = optparse.OptionParser(version="%%prog %s" % VERSION) parser.add_option("-x", "--exclude", help=EXCLUDE_HELP + " Separate multiple directories with commas.") # Remaining options are the same for argparse and optparse add_option( parser, "-n", "--no-recurse", dest="recurse", default=True, action="store_false", help="do not recurse into subdirectories (default: recurse)", ) add_option( parser, "-u", "--utf8", default=False, action="store_true", help="switch encoding to UTF-8 (default: don't change encoding)", ) add_option( parser, "-p", "--purge", default=False, action="store_true", help="purge all artifacts before running dexy" ) add_option( parser, "--cleanup", default=False, action="store_true", help="delete all dexy-generated directories and files (does not run dexy)", ) add_option( parser, "--filters", default=False, action="store_true", help="list all available filters (does not run dexy)" ) add_option( parser, "--reporters", default=False, action="store_true", help="list all available reporters (does not run dexy)", ) add_option( parser, "--artifact-class", default="FileSystemJsonArtifact", help="name of artifact class to use (default: FileSystemJsonArtifact)", ) add_option( parser, "-a", "--artifacts-dir", default="artifacts", help="location of artifacts directory (default: artifacts)", ) add_option( parser, "-l", "--logs-dir", default="logs", help="""location of logs directory (default: logs) dexy will create a dexy.log file in this directory reporters may create reports in this directory""", ) add_option( parser, "-c", "--cache-dir", default="cache", help="DEPRECATED - does not do anything anymore, will be removed soon", ) add_option( parser, "--local", default=False, action="store_true", help="Use cached local copies of remote urls - faster but might not be up to date", ) add_option( parser, "--ignore-errors", default=False, action="store_true", help="""Don't raise an error if scripts return nonzero exit status (depends on the filter being written to support this)""", ) add_option( parser, "--setup", default=False, action="store_true", help="Create artifacts and logs directory and generic .dexy file", ) add_option( parser, "--serve", default=False, action="store_true", help="Start a static server within the output directory, will be visible on port 8000. (similar to python -m SimpleHTTPServer)", ) add_option(parser, "-g", "--config", default=".dexy", help="name of configuration file") add_option( parser, "-d", "--dangerous", default=False, action="store_true", help="Allow running remote URLs which may execute dangerous code, use with care.", ) add_option( parser, "--no-reports", default=False, action="store_true", help="""Don't run reports when finished running Dexy.""", ) if option_parser == "argparse": args = parser.parse_args() dir_name = args.dir if args.exclude: additional_excludes = args.exclude else: additional_excludes = [] elif option_parser == "optparse": (args, argv) = parser.parse_args() if len(argv) == 0: dir_name = "." else: dir_name = argv[0] if args.exclude: additional_excludes = args.exclude.split(",") else: additional_excludes = [] else: raise Exception("unexpected option_parser %s" % option_parser) args.run_dexy = True if args.utf8: if sys.getdefaultencoding() == "UTF-8": print "encoding is already UTF-8" else: print "changing encoding from %s to UTF-8" % sys.getdefaultencoding() reload(sys) sys.setdefaultencoding("UTF-8") if not os.path.exists(dir_name): raise Exception("file %s not found!" % dir_name) if args.setup: if not os.path.exists(args.artifacts_dir): os.mkdir(args.artifacts_dir) if not os.path.exists(args.logs_dir): os.mkdir(args.logs_dir) if not os.path.exists(".dexy"): f = open(".dexy", "w") f.write("{\n}\n") f.close() if args.config.startswith("http"): print "fetching remote config file %s" % args.config filename = os.path.basename(args.config) print "writing to %s" % filename f = urllib.urlopen(args.config) config_file = open(filename, "w") config_file.write(f.read()) config_file.close() f.close() args.config = filename if not os.path.exists(args.artifacts_dir): path_to_artifacts_dir = os.path.join(project_base, args.artifacts_dir) raise Exception( """artifacts directory not found at %s, please create a directory called %s if you want to use this location as a dexy project root""" % (path_to_artifacts_dir, args.artifacts_dir) ) if not os.path.exists(args.logs_dir): path_to_logs_dir = os.path.join(project_base, args.logs_dir) raise Exception( """logs directory not found at %s, please create a directory called %s if you want to use this location as a dexy project root""" % (path_to_logs_dir, args.logs_dir) ) # Set up main dexy log dexy_log = logging.getLogger("dexy") dexy_log.setLevel(logging.DEBUG) logfile = os.path.join(args.logs_dir, "dexy.log") handler = RotatingFileHandler(logfile) dexy_log.addHandler(handler) if args.setup: # This might be the first time someone has run Dexy, # let them know where the logfile is print "dexy will log debugging information to", logfile formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") handler.setFormatter(formatter) args.artifact_class = artifact_class(args.artifact_class, dexy_log) if args.purge: dexy_log.warn("purging contents of %s" % args.artifacts_dir) shutil.rmtree(args.artifacts_dir) os.mkdir(args.artifacts_dir) # Each artifact class may need to do its own purging also if hasattr(args.artifact_class, "purge"): args.artifact_class.purge() if args.cleanup: args.run_dexy = False print "purging contents of %s" % args.artifacts_dir shutil.rmtree(args.artifacts_dir) if os.path.exists(args.cache_dir): print "purging contents of %s" % args.cache_dir shutil.rmtree(args.cache_dir) if os.path.exists(args.logs_dir): print "purging contents of %s" % args.logs_dir shutil.rmtree(args.logs_dir) controller = Controller() controller.find_reporters() for d in controller.reports_dirs: if d and os.path.exists(d): print "purging contents of %s" % d shutil.rmtree(d) filters_dir = "filters" # TODO improve this so it detects __init__.py customizations etc. if os.path.exists(filters_dir): if sorted(os.listdir(filters_dir)) == ["README", "__init__.py"]: dexy_log.warn("purging contents of %s" % filters_dir) shutil.rmtree(filters_dir) else: print ("Directory %s has been modified, not removing." % filters_dir) if args.reporters: args.run_dexy = False controller = Controller() reporters = controller.find_reporters() for r in reporters: print r.__name__ if r.REPORTS_DIR: print "any generated reports will be saved in", r.REPORTS_DIR else: print "any generated reports will be saved in", args.logs_dir if r.__doc__: print r.__doc__ else: print "no documentation available" print # finish with blank line print "Running reports can be disabled with the --no-reports option" if args.filters: args.run_dexy = False controller = Controller() handlers = controller.find_handlers() for k in sorted(handlers.keys()): klass = handlers[k] print print k, ":", klass.__name__ if klass.executable(): print " calls", klass.executable() if klass.version_command(): try: raw_version = klass.version().strip() if raw_version.find("\n") > 0: # Clean up long version strings like R's version = raw_version.split("\n")[0] else: version = raw_version print " version", version, "detected using", klass.version_command() except Exception: # TODO raise/catch specific Exception print " may not be installed, was unable to run", klass.version_command() if klass.__doc__: print " ", klass.__doc__.strip() return args, dir_name, additional_excludes, dexy_log
def setup_controller(): args, dir_name, additional_excludes, log = setup_option_parser() if not args.run_dexy: return None, args, log controller = Controller() controller.dir_name = dir_name controller.args = args controller.allow_remote = args.dangerous controller.artifact_class = args.artifact_class controller.artifacts_dir = args.artifacts_dir controller.logs_dir = args.logs_dir controller.log = log controller.config_file = args.config controller.use_local_files = args.local controller.find_reporters() controller.additional_excludes = additional_excludes return controller, args, log