def scan_workflows(debug_imports):
workflows = []
warnings.simplefilter("ignore")
UnitRegistry.enabled = False
root_dir = root.common.dirs.veles
for path, _, files in os.walk(root_dir, followlinks=True):
relpath = os.path.relpath(path, root_dir)
skip = False
for bdir in BLACKLISTED_DIRS:
if relpath.startswith(bdir):
skip = True
break
if skip:
continue
for f in set(files) - BLACKLISTED_FILES:
f_path = os.path.join(path, f)
modname, ext = os.path.splitext(f)
if ext != '.py':
continue
if debug_imports:
sys.stdout.write("[%s] importing %s... " % (relpath, modname))
mod = try_to_import_file(f_path)
if not is_module(mod):
if debug_imports:
print("SKIP (import)")
continue
for func in dir(mod):
if func == "run" and \
getargspec(mod.run).args == ["load", "main"]:
wf_path = os.path.relpath(f_path, root_dir)
workflows.append(os.path.abspath(wf_path))
if debug_imports:
print("OK")
break
else:
if debug_imports:
print("SKIP (run)")
gc.collect()
warnings.simplefilter("default")
print("Found %d workflows:\n" % len(workflows), workflows)
# Fix ResourceWarning on /dev/null
from IPython.utils.io import devnull
devnull.close()
if len(threading.enumerate()) > 1:
print("Warning: more than 1 thread is currently running, a join lock "
"may happen.")
return workflows
def scan_workflows(debug_imports):
workflows = []
warnings.simplefilter("ignore")
UnitRegistry.enabled = False
root_dir = root.common.dirs.veles
for path, _, files in os.walk(root_dir, followlinks=True):
relpath = os.path.relpath(path, root_dir)
skip = False
for bdir in BLACKLISTED_DIRS:
if relpath.startswith(bdir):
skip = True
break
if skip:
continue
for f in set(files) - BLACKLISTED_FILES:
f_path = os.path.join(path, f)
modname, ext = os.path.splitext(f)
if ext != '.py':
continue
if debug_imports:
sys.stdout.write("[%s] importing %s... " % (relpath, modname))
mod = try_to_import_file(f_path)
if not is_module(mod):
if debug_imports:
print("SKIP (import)")
continue
for func in dir(mod):
if func == "run" and \
getargspec(mod.run).args == ["load", "main"]:
workflows.append(f_path)
if debug_imports:
print("OK")
break
else:
if debug_imports:
print("SKIP (run)")
gc.collect()
warnings.simplefilter("default")
print("Found %d workflows:\n" % len(workflows), workflows)
# Fix ResourceWarning on /dev/null
from IPython.utils.io import devnull
devnull.close()
if len(threading.enumerate()) > 1:
print("Warning: more than 1 thread is currently running, a join lock "
"may happen.")
return workflows
def assist(self):
"""
Interactively create a metadata file for the workflow, so that it can
be submitted to VelesForge.
"""
metadata = {}
base = os.path.dirname(self.path)
modname = os.path.splitext(os.path.basename(self.path))[0]
print("Importing %s..." % modname)
sys.stdout.flush()
module = try_to_import_file(self.path)
if not is_module(module):
self.error("Failed to import %s.\npackage: %s\ndirect: %s",
self.path, *module)
self.return_code = 1
return
# Discover the module's dependencies
metadata["requires"] = self._scan_deps(module)
# Discover the workflow class
wfcls = [None]
def fake_load(klass, *args, **kwargs):
wfcls[0] = klass
return None, None
def fake_run(*args, **kwargs):
pass
module.run(fake_load, fake_run)
wfcls = wfcls[0]
# Fill "name"
name = wfcls.__name__.replace("Workflow", "")
max_chars = 64
inp = '0' * (max_chars + 1)
while len(inp) > max_chars:
if len(inp) > max_chars:
print("Package name may not be longer than %d chars." %
max_chars)
inp = input("Please enter the desired package name (%s): " % name)
metadata["name"] = inp or name
# Fill "short_description"
max_chars = 140
inp = "0" * (max_chars + 1)
while len(inp) > max_chars:
inp = input("Please enter a *short* description (<= 140 chars): ")
metadata["short_description"] = inp
# Fill "long_description"
print("The long description will be taken from %s's docstring." %
wfcls.__name__)
metadata["long_description"] = wfcls.__doc__.strip()
inp = input("Please introduce yourself (e.g., "
"\"Ivan Ivanov <*****@*****.**>\"): ")
metadata["author"] = inp
metadata["workflow"] = os.path.basename(self.path)
# Discover the configuration file
wfn, wfext = os.path.splitext(self.path)
wfn += "_config"
cfgfile = wfn + wfext
inp = cfgfile
while (inp and not os.path.exists(inp) and
not os.path.exists(os.path.join(base, inp))):
inp = input("Please enter the path to the configuration file "
"(may be blank) (%s): " % cfgfile)
fullcfg = inp or cfgfile
metadata["configuration"] = os.path.basename(fullcfg)
print("Generating %s..." % metadata["name"])
os.mkdir(metadata["name"])
shutil.copyfile(self.path, os.path.join(metadata["name"],
metadata["workflow"]))
destcfg = os.path.join(metadata["name"], metadata["configuration"])
if os.path.exists(fullcfg):
shutil.copyfile(fullcfg, destcfg)
else:
open(destcfg, 'w').close()
with open(os.path.join(metadata["name"],
root.common.forge.manifest), "w") as fout:
json.dump(metadata, fout, sort_keys=True, indent=4)