示例#1
0
def scan_workflows(debug_imports):
    workflows = []
    warnings.simplefilter("ignore")
    UnitRegistry.enabled = False
    root_dir = root.common.dirs.veles
    for path, _, files in os.walk(root_dir, followlinks=True):
        relpath = os.path.relpath(path, root_dir)
        skip = False
        for bdir in BLACKLISTED_DIRS:
            if relpath.startswith(bdir):
                skip = True
                break
        if skip:
            continue
        for f in set(files) - BLACKLISTED_FILES:
            f_path = os.path.join(path, f)
            modname, ext = os.path.splitext(f)
            if ext != '.py':
                continue
            if debug_imports:
                sys.stdout.write("[%s] importing %s... " % (relpath, modname))
            mod = try_to_import_file(f_path)
            if not is_module(mod):
                if debug_imports:
                    print("SKIP (import)")
                continue
            for func in dir(mod):
                if func == "run" and \
                        getargspec(mod.run).args == ["load", "main"]:
                    wf_path = os.path.relpath(f_path, root_dir)
                    workflows.append(os.path.abspath(wf_path))
                    if debug_imports:
                        print("OK")
                    break
            else:
                if debug_imports:
                    print("SKIP (run)")
    gc.collect()
    warnings.simplefilter("default")
    print("Found %d workflows:\n" % len(workflows), workflows)
    # Fix ResourceWarning on /dev/null
    from IPython.utils.io import devnull
    devnull.close()
    if len(threading.enumerate()) > 1:
        print("Warning: more than 1 thread is currently running, a join lock "
              "may happen.")
    return workflows
示例#2
0
def scan_workflows(debug_imports):
    workflows = []
    warnings.simplefilter("ignore")
    UnitRegistry.enabled = False
    root_dir = root.common.dirs.veles
    for path, _, files in os.walk(root_dir, followlinks=True):
        relpath = os.path.relpath(path, root_dir)
        skip = False
        for bdir in BLACKLISTED_DIRS:
            if relpath.startswith(bdir):
                skip = True
                break
        if skip:
            continue
        for f in set(files) - BLACKLISTED_FILES:
            f_path = os.path.join(path, f)
            modname, ext = os.path.splitext(f)
            if ext != '.py':
                continue
            if debug_imports:
                sys.stdout.write("[%s] importing %s... " % (relpath, modname))
            mod = try_to_import_file(f_path)
            if not is_module(mod):
                if debug_imports:
                    print("SKIP (import)")
                continue
            for func in dir(mod):
                if func == "run" and \
                        getargspec(mod.run).args == ["load", "main"]:
                    workflows.append(f_path)
                    if debug_imports:
                        print("OK")
                    break
            else:
                if debug_imports:
                    print("SKIP (run)")
    gc.collect()
    warnings.simplefilter("default")
    print("Found %d workflows:\n" % len(workflows), workflows)
    # Fix ResourceWarning on /dev/null
    from IPython.utils.io import devnull
    devnull.close()
    if len(threading.enumerate()) > 1:
        print("Warning: more than 1 thread is currently running, a join lock "
              "may happen.")
    return workflows
示例#3
0
    def assist(self):
        """
        Interactively create a metadata file for the workflow, so that it can
        be submitted to VelesForge.
        """
        metadata = {}
        base = os.path.dirname(self.path)
        modname = os.path.splitext(os.path.basename(self.path))[0]
        print("Importing %s..." % modname)
        sys.stdout.flush()
        module = try_to_import_file(self.path)
        if not is_module(module):
            self.error("Failed to import %s.\npackage: %s\ndirect: %s",
                       self.path, *module)
            self.return_code = 1
            return

        # Discover the module's dependencies
        metadata["requires"] = self._scan_deps(module)

        # Discover the workflow class
        wfcls = [None]

        def fake_load(klass, *args, **kwargs):
            wfcls[0] = klass
            return None, None

        def fake_run(*args, **kwargs):
            pass

        module.run(fake_load, fake_run)
        wfcls = wfcls[0]

        # Fill "name"
        name = wfcls.__name__.replace("Workflow", "")
        max_chars = 64
        inp = '0' * (max_chars + 1)
        while len(inp) > max_chars:
            if len(inp) > max_chars:
                print("Package name may not be longer than %d chars." %
                      max_chars)
            inp = input("Please enter the desired package name (%s): " % name)
        metadata["name"] = inp or name

        # Fill "short_description"
        max_chars = 140
        inp = "0" * (max_chars + 1)
        while len(inp) > max_chars:
            inp = input("Please enter a *short* description (<= 140 chars): ")
        metadata["short_description"] = inp

        # Fill "long_description"
        print("The long description will be taken from %s's docstring." %
              wfcls.__name__)
        metadata["long_description"] = wfcls.__doc__.strip()

        inp = input("Please introduce yourself (e.g., "
                    "\"Ivan Ivanov <*****@*****.**>\"): ")
        metadata["author"] = inp
        metadata["workflow"] = os.path.basename(self.path)

        # Discover the configuration file
        wfn, wfext = os.path.splitext(self.path)
        wfn += "_config"
        cfgfile = wfn + wfext
        inp = cfgfile
        while (inp and not os.path.exists(inp) and
               not os.path.exists(os.path.join(base, inp))):
            inp = input("Please enter the path to the configuration file "
                        "(may be blank) (%s): " % cfgfile)
        fullcfg = inp or cfgfile
        metadata["configuration"] = os.path.basename(fullcfg)

        print("Generating %s..." % metadata["name"])
        os.mkdir(metadata["name"])
        shutil.copyfile(self.path, os.path.join(metadata["name"],
                                                metadata["workflow"]))
        destcfg = os.path.join(metadata["name"], metadata["configuration"])
        if os.path.exists(fullcfg):
            shutil.copyfile(fullcfg, destcfg)
        else:
            open(destcfg, 'w').close()
        with open(os.path.join(metadata["name"],
                               root.common.forge.manifest), "w") as fout:
            json.dump(metadata, fout, sort_keys=True, indent=4)