def run(self, parent, forward, extra_args, params):
from pyloco.task import load_taskclass, Task
out = 0
out_forward = None
for task, argv, subargv in self._tasks:
task_class, argv, subargv, objs = load_taskclass(task, argv + extra_args, subargv)
if task_class is None:
raise UsageError("Task '%s' is not loaded." % str(task))
taskobj = task_class(parent)
if taskobj is None:
raise InternalError("Task '%s' is not created." % str(task))
#with open("taskpath.%d.log"%os.getpid(), "w") as f:
# f.write("\n\n".join([str(taskobj), str(parent)]))
# f.flush()
if isinstance(taskobj, Task):
forward["_pathid_"] = self._pathid_
taskobj._env.update(objs)
taskobj._env.update(params)
out, forward = taskobj.run(argv, subargv=subargv, forward=forward)
out_forward = forward
else:
raise UsageError("Task '%s' is not a Task type." % str(task))
return out, out_forward
def perform(self, targs):
if targs.task:
taskpath = targs.task
elif self.subargv:
if "--" in self.subargv:
raise UsageError(
"'upload' task requires only one sub-task "
"to upload, but found multiple sub-tasks: '%s'" %
" ".join(self.subargv)
)
taskpath = self.subargv[0]
else:
raise UsageError(
"'upload' task requires one sub-task to upload."
)
with _Pack(self, taskpath, distname=targs.name) as (taskname, tmpdir):
cwd = os.getcwd()
os.chdir(os.path.join(tmpdir, taskname))
ret, sout, serr = system(sys.executable + " setup.py sdist")
ret, sout, serr = system(sys.executable + " setup.py bdist_wheel --universal")
ret = isystem("twine upload --repository-url %s dist/*" %
targs.repository)
os.chdir(cwd)
if ret == 0:
print("'%s' task is uploaded successfully." % taskpath)
else:
print("Uploading of '%s' task is failed." % taskpath)
def perform(self, targs):
if os.path.exists(targs.command):
raise UsageError("Specified command '%s' already exists." %
targs.command)
# get OS and select shell script type
if OS == "windows":
import pdb; pdb.set_trace()
else:
if len(self.subargv) < 1:
raise UsageError("No target task is specified.")
if not os.path.exists(self.subargv[0]):
raise UsageError("Target task does not exist: %s" % self.subargv[0])
self.subargv[0] = os.path.abspath(os.path.realpath(self.subargv[0]))
with open(targs.command, "w") as f:
retval, stdout, _ = system("which bash")
if retval == 0:
f.write("#!" + stdout + "\n")
f.write("%s %s $*" % (sys.argv[0], " ".join(self.subargv)))
os.chmod(targs.command, 0o744)
def perform(self, targs):
if targs.task:
taskpath = targs.task
elif self.subargv:
if "--" in self.subargv:
raise UsageError(
"'upload' task requires only one sub-task "
"to upload, but found multiple sub-tasks: '%s'" %
" ".join(self.subargv)
)
taskpath = self.subargv[0]
else:
raise UsageError(
"'upload' task requires one sub-task to upload."
)
with _Pack(self, taskpath, distname=targs.name) as (taskname, tmpdir):
topdir = os.path.join(tmpdir, taskname)
outdir = targs.outdir if targs.outdir else ""
if targs.name:
outfile = os.path.join(outdir, targs.name+".plz")
else:
outfile = os.path.join(outdir, taskname+".plz")
shutil.make_archive(topdir, 'zip', *os.path.split(topdir))
shutil.move(topdir+".zip", outfile)
print("'%s' task is packed successfully." % taskpath)
def _append_task(self, path, items):
if len(items) < 1:
raise UsageError("task is not found: %s" % str(self.subargv))
if items[0].startswith("-"):
raise UsageError("First argument is not a task reference: %s" % str(items))
path.append_task(items[0], argv=items[1:])
del items[:]
def run(self, argv, subargv=None, forward=None):
if not argv:
raise UsageError("PlZ Task is not found."
" Please check plz path.")
elif not os.path.isfile(argv[0]):
raise UsageError("PlZ Task '%s' is not found."
" Please check plz path." % str(argv[0]))
out = -1
taskpath = argv.pop(0)
if zipfile.is_zipfile(taskpath):
tempdir = None
with TemporaryDirectory() as tempdir:
plz = zipfile.ZipFile(taskpath)
plz.extractall(path=tempdir)
plz.close()
dirnames = os.listdir(tempdir)
if len(dirnames) != 1:
raise UsageError("'%s' is not a plz file." % taskpath)
taskname = dirnames[0]
pkgdir = os.path.join(tempdir, taskname)
moddir = os.path.join(pkgdir, taskname)
sys.path.insert(0, pkgdir)
mod = pyloco_import(taskname)
taskcls = getattr(mod, "entry_task")
if taskcls is PlXTask:
argv.insert(0, os.path.join(moddir, getattr(mod, "plx")))
sys.path.pop(0)
from pyloco.main import perform
parent = self.get_proxy()
out = perform(taskcls,
argv=argv,
subargv=subargv,
parent=parent,
forward=forward,
shared=self.parent.shared)
return out
else:
raise UsageError("'%s' is not a plz file format." % taskpath)
def _register_check(self, dest):
if not dest:
raise UsageError("Incorrect name: %s" % dest)
if dest.startswith("_"):
raise UsageError("'Forward-name' should not start with an "
"underscore ('_'): %s" % dest)
if dest in self._fwddefs:
raise UsageError("'%s' is already registered for forwarding" %
dest)
if dest in self._shrdefs:
raise UsageError("'%s' is already registered for sharing" % dest)
def parse_param_option(val, evaluate, env):
def _p(*argv, **kw_str):
return list(argv), kw_str
obj = Option()
if isinstance(val, str) or type(val) == type(u"A"): # noqa: E721
items = [v.strip() for v in val.split("@")]
obj.context = items[1:]
for c in obj.context:
if not is_valid_variable_name(c):
items = [val]
obj.context = []
break
if evaluate:
eval_out = teval('_p({0})'.format(items[0]), env, _p=_p)
if eval_out:
obj.vargs, obj.kwargs = eval_out
else:
raise UsageError("syntax error at '{0}'.".format(items[0]))
else:
obj.vargs, obj.kwargs = parse_literal_args(items[0])
else:
obj.vargs = [val]
return obj
def _install(self, topdir, taskname, targs):
dirnames = os.listdir(topdir)
if "setup.py" not in dirnames:
raise UsageError("'setup.py' is not found.'")
if targs.name and targs.name != taskname:
new_taskname = targs.name
os.rename(os.path.join(topdir, taskname),
os.path.join(topdir, new_taskname))
taskname = new_taskname
setup = os.path.join(topdir, "setup.py")
oldsetup = os.path.join(topdir, "setup.py.old")
shutil.move(setup, oldsetup)
prefix = ' __taskname__ = "'
with open(setup, "w") as fw:
with open(oldsetup, "r") as fr:
for line in fr:
if line.startswith(prefix):
fw.write(prefix + taskname + '"\n')
else:
fw.write(line)
os.remove(oldsetup)
retval, stdout, stderr = system(sys.executable + " setup.py sdist"
" --formats=tar",
cwd=topdir)
if retval != 0:
raise InternalError(stderr)
distdir = os.path.join(topdir, "dist")
sdist = None
for dname in os.listdir(distdir):
if dname.endswith(".tar"):
sdist = dname
break
if sdist is None:
raise InternalError("'sdist' file is not found: %s" % taskname)
if is_venv():
retval, stdout, stderr = system(get_pip() + " install " + sdist,
cwd=distdir)
# system("python setup.py install", cwd=topdir)
else:
retval, stdout, stderr = system(get_pip() +
" install %s --user" % sdist,
cwd=distdir)
# system("python setup.py install --user", cwd=topdir)
if retval == 0:
# TODO: print installed version with some progress remarks
print("'%s' task is installed successfully." % taskname)
else:
raise InternalError(stderr)
def __init__(self, zfile):
self.tmpdir = tempfile.mkdtemp()
shutil.unpack_archive(zfile, self.tmpdir, "zip")
dirnames = os.listdir(self.tmpdir)
if len(dirnames) != 1:
raise UsageError("'%s' is not a plz file." % zfile)
self.taskname = dirnames[0]
def __getattr__(self, attr):
try:
return getattr(self.task.parent, attr)
except Exception:
raise UsageError(
"'ServerProxy' object has no attribute '%s'" % attr)
def parse_literal_args(expr):
lv = []
lk = {}
expr_items = expr.split(",")
text = ""
while expr_items:
expr_item = expr_items.pop(0).strip()
if not expr_item:
continue
if text:
text = text + "," + expr_item
else:
text = expr_item
#if not text:
# continue
try:
tree = ast.parse("func({0})".format(text))
args = tree.body[0].value.args
keywords = tree.body[0].value.keywords
if len(args) > 0 and len(keywords):
raise UsageError("Both of args and keywords are found"
" during argument parsing.")
text = text.strip()
if not text:
continue
if len(args) > 0:
lv.append(text)
elif len(keywords) > 0:
key, val = text.split("=", 1)
if val:
lk[key] = val
text = ""
except Exception:
pass
#if not lv and not lk:
# lv.append(expr)
return lv, lk
def perform(self, targs):
if targs.task:
if len(targs.task) > 1:
raise UsageError(
"'uninstall' task only one task: %s." % targs.task
)
taskname = targs.task[0]
elif self.subargv:
if "--" in self.subargv:
raise UsageError(
"'uninstall' task requires only one sub-task "
"to uninstall, but found multiple sub-tasks: '%s'" %
" ".join(self.subargv)
)
taskname = self.subargv[0]
del self.subargv[:]
else:
raise UsageError(
"'uninstall' task requires one sub-task to uninstall."
)
for ep in pkg_resources.iter_entry_points(group='pyloco.task'):
if ep.name == taskname:
retval, sout, serr = system(
get_pip() + " uninstall -y " + ep.dist.project_name
)
if retval == 0:
print("'%s' task is uninstalled successfully." % taskname)
else:
print("Uninstallation of '%s' task was failed: %s" %
(taskname, serr))
return retval
raise UsageError("'%s' is not found." % taskname)
def _handle_sharedarg(self, shared):
# TODO: need this?
try:
for shr in shared:
for varg in vargs:
self.parent.shared[varg] = env[varg]
for dest, value in shr.kwargs.items():
self.parent.shared[dest] = eval(value, env, lenv)
except Exception as err:
raise UsageError("failed on sharing variable: %s" % str(err))
def run(self, argv, subargv=None, forward=None):
if not argv:
raise UsageError("PlX Task is not found."
" Please check plx path.")
elif not os.path.isfile(argv[0]):
raise UsageError("PlX Task '%s' is not found."
" Please check plx path." % str(argv[0]))
self._setup(argv[0])
prog = os.path.basename(getattr(self, "_path_", self._name_))
self._parser.prog = self.get_mgrname() + " " + prog[-20:]
if hasattr(self, "__doc__") and self.__doc__:
self._parser.desc, self._parser.long_desc = pydoc.splitdoc(
self.__doc__)
return super(PlXTask, self).run(argv[1:],
subargv=subargv,
forward=forward)
def perform(self, targs):
if os.path.exists(targs.task):
raise UsageError("Task '%s' is already exists: " % targs.task)
outdir, filename = os.path.split(targs.task)
outdir = os.path.abspath(outdir)
if not os.path.exists(outdir):
os.makedirs(outdir)
taskname, tasktype = os.path.splitext(filename)
if not taskname and tasktype:
taskname = tasktype
tasktype = "package"
clsname = taskname[0].upper() + taskname[1:]
if tasktype == ".py":
output = _init_template.format(clsname=clsname,
taskname=taskname,
taskversion=targs.task_version)
with open(targs.task, "w") as f:
f.write(output)
elif tasktype == ".plx":
import pdb
pdb.set_trace()
elif tasktype == "package":
import pdb
pdb.set_trace()
else:
raise UsageError("Unknown task type: %s" % tasktype)
print("Initialized '%s' task development in '%s'" % (taskname, outdir))
def _handle_sharedarg(self, shared, forwards):
try:
env = dict(self._env)
env.update(forwards)
for shr in shared:
for varg in shr.vargs:
self.parent.shared[varg] = env[varg]
for dest, value in shr.kwargs.items():
self.parent.shared[dest] = eval(value, env, {})
except Exception as err:
raise UsageError("failed on sharing variable: %s" % str(err))
def _add_transfer(self, defs, cont, **kwargs):
for dest, value in kwargs.items():
if dest not in defs:
raise UsageError("'%s' is not registered for data transfer." %
dest)
if type_check(value, defs[dest][0]):
cont[dest] = value
else:
if isinstance(value, str) and os.path.isfile(value):
import pdb; pdb.set_trace() # noqa: E702
else:
raise TestError("Data transfer type check failure: %s" % dest)
def import_testplx(path, manager, argv, subargv):
if argv is None:
argv = []
if subargv is None:
subargv = []
if not os.path.isfile(path):
raise UsageError("PlX Task '%s' is not found."
" Please check plx path." % str(path))
PlXTest._plx_ = plx = PlXTestTask(manager)
plx._setup(path)
prog = os.path.basename(getattr(plx, "_path_", plx._name_))
plx._parser.prog = plx.get_mgrname() + " " + prog[-20:]
if hasattr(plx, "__doc__") and plx.__doc__:
plx._parser.desc, plx._parser.long_desc = pydoc.splitdoc(
plx.__doc__)
super(PlXTask, plx).run(argv, subargv=subargv)
del PlXTest._setups[:]
del PlXTest._teardowns[:]
# create test methods
for hdr, body in plx.plx_sections:
if hdr.endswith("*"):
if hdr.startswith("setup"):
PlXTest._setups.append(partial(PlXTest._test_template, header=hdr[:-1], body=body))
elif hdr.startswith("teardown"):
PlXTest._teardowns.append(partial(PlXTest._test_template, header=hdr[:-1], body=body))
else:
setattr(PlXTest, "test_%s" % hdr.replace("@", "_"),
partial(PlXTest._test_template, header=hdr, body=body))
import imp
mod = imp.new_module("")
setattr(mod, "PlXTest", PlXTest)
return mod
def perform(self, targs):
for plx_dest, opt in self.plx_argdefs.items():
dest = opt.kwargs.get("dest", opt.vargs[0])
if dest in self._env["__arguments__"]:
argval = self._env["__arguments__"].pop(dest, None)
self._env["__arguments__"][plx_dest] = argval
out = self.run_section(self.plx_entry_body)
if out == 0:
for hdr, body in self.plx_sections:
# check hdr
if hdr.endswith("*"):
special_sec = True
opt = parse_param_option(hdr[:-1], False,
self.parent.shared)
else:
special_sec = False
opt = parse_param_option(hdr, False, self.parent.shared)
env = dict(self.parent.shared)
env["__builtins__"] = pyloco_builtins
sec_check = all([eval(c, env) for c in opt.context])
if sec_check:
sec_name = opt.vargs[0]
# find sec_handler
if special_sec:
if sec_name in self._section_handlers:
sec_handler = self._section_handlers[sec_name]
else:
raise UsageError(
"Special section '%s' is not registered."
" Please register first." % sec_name)
else:
sec_handler = self.run_section
out = sec_handler(body, *opt.vargs[1:], **opt.kwargs)
return out
def import_modulepath(path):
#debug(path == "test.plx")
fragment = None
spath = [p.strip() for p in path.split("#")]
if len(spath) > 1:
path = spath[0]
fragment = spath[1]
if os.path.exists(path):
head, base = os.path.split(path)
mod = None
if os.path.isfile(path) and path.endswith(".py"):
modname = base[:-3]
mod = load_pymod(head, modname)
elif (os.path.isdir(path)
and os.path.isfile(os.path.join(path, "__init__.py"))):
if base[-1] == os.sep:
modname = base[:-1]
else:
modname = base
mod = load_pymod(head, modname)
else:
try:
modname = path
mod = pyloco_import(modname)
except ModuleNotFoundError as err:
raise UsageError("path does not exist: %s" % path)
if mod:
if fragment:
return fragment, getattr(mod, fragment)
else:
return modname, mod
def perform(self, targs):
try:
#original_argparser = Task._argparser_
#Task._argparser_ = TestArgParser
if targs.test:
if targs.test.endswith(".plx"):
module = import_testplx(targs.test, self.get_proxy(),
targs.testargv, targs.testsubtask)
else:
_, module = import_modulepath(targs.test)
unittest.main(module=module, argv=["dummy"])
elif self.subargv:
import pyloco.grouptask as testmodule
del testmodule.DefaultGroupTestCase._testargs_[:]
testmodule.DefaultGroupTestCase._testargs_.extend(self.subargv)
unittest.main(module=testmodule, argv=["dummy"])
else:
raise UsageError("No test is specified.")
#
#
# results = []
#
# # invoke test runner
# if targs.params:
# for param in targs.params:
# for varg in param.vargs:
# results.append((vargs, param.kwargs, tester.run(varg, param.kwargs)))
#
finally:
Task._argparser_ = self.original_argparser
def test_section(self, hdr, body):
# check hdr
if hdr.endswith("*"):
special_sec = True
opt = parse_param_option(hdr[:-1], False,
self.parent.shared)
else:
special_sec = False
opt = parse_param_option(hdr, False, self.parent.shared)
env = dict(self.parent.shared)
env["__builtins__"] = pyloco_builtins
sec_check = all([eval(c, env) for c in opt.context])
if sec_check:
sec_name = opt.vargs[0]
# find sec_handler
if special_sec:
if sec_name in self._section_handlers:
sec_handler = self._section_handlers[sec_name]
else:
raise UsageError(
"Special section '%s' is not registered."
" Please register first." % sec_name
)
else:
sec_handler = self.run_section
return sec_handler(body, *opt.vargs[1:], **opt.kwargs)
else:
return 0
def post_perform(self, targs):
if targs.webapp:
appath = targs.webapp
wait2close = self.taskattr.get("webapp.wait2close", True)
if wait2close:
if self._websocket_server:
pyloco_print("Waiting for '%s' to be completed..." %
appath,
end="")
sys.stdout.flush()
self._websocket_server.communicate(input=None)
if self._websocket_client:
self._websocket_client.close()
if self._webserver:
self._webserver.communicate(input=None)
pyloco_print("DONE.")
sys.stdout.flush()
env = dict(self._env)
env.update(self.parent.shared)
env.update(self._fwds)
lenv = {}
if targs.forward:
try:
for fwd in targs.forward:
for varg in fwd.vargs:
self._fwds[varg] = env[varg]
for dest, value in fwd.kwargs.items():
self._fwds[dest] = eval(value, env, lenv)
except Exception as err:
raise UsageError("failed on forwarding: %s" % str(err))
if targs.shared:
self._handle_sharedarg(targs.shared)
if hasattr(targs, "assert_output") and targs.assert_output:
aenv = {"__builtins__": pyloco_builtins}
for k, v in self._env.items():
if not k.startswith("_"):
aenv[k] = v
aenv.update(self.parent.shared)
aenv.update(self._fwds)
for boolexpr in targs.assert_output:
for varg in boolexpr.vargs:
assert_result = eval(varg, aenv)
if assert_result:
if self._verbose:
pyloco_print('\nOUTPUT TEST PASSED with "%s"' %
varg)
else:
pairs = split_assert_expr(varg)
if not pairs:
raise TestError("\nOUTPUT TEST FAILED with '%s' =>"
" not True" % varg)
elif len(pairs) == 1:
sep, (lexpr, rexpr) = pairs.popitem()
msg = ("\nOUTPUT TEST(%s) is FAILED.\n "
"Left expr(%s) of '%s' is evaluated to '%s'"
" and\n right expr(%s) of '%s' "
"is evaluated to '%s'.\n") % (
varg, lexpr, sep, eval(lexpr, aenv),
rexpr, sep, eval(rexpr, aenv))
raise TestError(msg)
else:
msg = (
"\nOUTPUT TEST(%s) FAILED: detected multiple"
" possibilities of this test failure\n") % varg
idx = 0
for sep, (lexpr, rexpr) in pairs.items():
idx += 1
try:
msg += ("CASE%d:\n Left expr(%s)"
" of"
" '%s' is evaluated to '%s' and\n"
" right expr(%s) of '%s' is "
"evaluated to '%s'.\n") % (
idx, lexpr, sep,
eval(lexpr, aenv), rexpr, sep,
eval(rexpr, aenv))
except Exception:
pass
raise TestError(msg)
if targs.write_pickle:
ppf = PylocoPickle()
data = dict(self.parent.shared)
data.update(self._fwds)
data.pop("parent_name", None)
pdata = self.write_pickle(ppf, data)
ppf.dump(pdata, targs.write_pickle)
def load_taskclass(taskpath, argv, subargv):
if not taskpath:
return None, None, None, None
# TODO: handle aliased task
if isinstance(taskpath, type):
if issubclass(taskpath, Task):
return taskpath, argv, subargv, None
raise UsageError("Not compatible task type: %s" % type(taskpath))
# TODO: move to callsite to load_taskclass
objs = {}
while "--import" in argv:
idx = argv.index("--import")
mpath = argv.pop(idx + 1)
argv.pop(idx)
key, obj = import_modulepath(mpath)
objs[key] = obj
task_class = None
_p = taskpath.split("#", 1)
if len(_p) == 2:
taskpath, fragment = [x.strip() for x in _p]
else:
fragment = ""
if os.path.exists(taskpath):
mods = []
if os.path.isfile(taskpath):
head, base = os.path.split(taskpath)
if base.endswith(".py"):
mods.append(load_pymod(head, base[:-3]))
elif base.endswith(".plx"):
from pyloco.plxtask import PlXTask
task_class = PlXTask
argv.insert(0, taskpath)
elif base.endswith(".plz"):
from pyloco.plztask import PlZTask
task_class = PlZTask
argv.insert(0, taskpath)
elif os.path.isdir(taskpath):
# TODO: support Python package
pass
import pdb
pdb.set_trace()
candidates = {}
for mod in mods:
for name in dir(mod):
if not name.startswith("_"):
obj = getattr(mod, name)
if (type(obj) == type(Task) and issubclass(obj, Task)
and (obj.__module__ is None
or not obj.__module__.startswith("pyloco."))):
candidates[name] = obj
if candidates:
if fragment:
if hasattr(candidates, fragment):
task_class = getattr(candidates, fragment)
else:
raise UsageError("No task is found with a fragment of "
"'%s'." % fragment)
elif len(candidates) == 1:
task_class = candidates.popitem()[1]
else:
raise UsageError("More than one frame are found."
"Please add fragment to select one: %s" %
list(candidates.keys()))
if task_class:
setattr(task_class, "_path_", os.path.abspath(taskpath))
#else:
# raise UsageError("Task class is not found. Please check path: %s" % taskpath)
if task_class is None:
from pyloco.manage import _ManagerBase
if taskpath in _ManagerBase._default_tasks_:
task_class = _ManagerBase._default_tasks_[taskpath]
if task_class is None:
for ep in pkg_resources.iter_entry_points(group='pyloco.task'):
if taskpath == ep.name:
task_class = ep.load()
from pyloco.plxtask import PlXTask
if task_class is PlXTask:
task_mod = pyloco_import(taskpath)
task_dir = os.path.dirname(task_mod.__file__)
argv.insert(
0, os.path.join(task_dir, getattr(task_mod, "plx")))
break
if not task_class:
from pyloco.mgmttask import mgmt_tasks
from pyloco.stdtask import standard_tasks
if taskpath in mgmt_tasks:
task_class = mgmt_tasks[taskpath]
elif taskpath in standard_tasks:
task_class = standard_tasks[taskpath]
# TODO support remote task
# if not task_class:
#
# url = urlparse(taskpath)
#
# if url.netloc or url.scheme:
# argv.insert(0, taskpath)
# task_class = RemoteTask
if not task_class:
raise UsageError("Task '%s' is not found. Please check path." %
taskpath)
return task_class, argv, subargv, objs
def load_default_task(cls, *tasks, **kwargs):
frame = inspect.stack()[1]
module = inspect.getmodule(frame[0])
mgrdir = os.path.realpath(
os.path.abspath(os.path.dirname(module.__file__)))
for task in tasks:
_p = task.split("#", 1)
if len(_p) == 2:
taskpath, fragment = [x.strip() for x in _p]
else:
taskpath = task
fragment = ""
taskmodpath = os.path.realpath(
os.path.abspath(os.path.join(mgrdir, taskpath)))
if os.path.exists(taskmodpath):
modname, mod = import_modulepath(taskmodpath)
if modname not in sys.modules:
raise InternalError("'%s' task is not loaded." % task)
task_class = None
candidates = {}
for name in dir(mod):
if not name.startswith("_"):
obj = getattr(mod, name)
if (type(obj) == type(Task) and issubclass(obj, Task)
and
(obj.__module__ is None
or not obj.__module__.startswith("pyloco."))):
candidates[name] = obj
if candidates:
if fragment:
if hasattr(candidates, fragment):
task_class = getattr(candidates, fragment)
else:
raise UsageError(
"No task is found with a fragment of "
"'%s'." % fragment)
elif len(candidates) == 1:
task_class = candidates.popitem()[1]
else:
raise UsageError(
"More than one frame are found."
"Please add fragment to select one: %s" %
list(candidates.keys()))
else:
raise UsageError("'%s' does not exist within this manager." %
task)
if task_class:
cls._default_tasks_[task_class._name_] = task_class
else:
raise UsageError("'%s' task is not loaded." % task)
def __init__(self, task, taskpath, distname=None):
from pyloco.task import load_taskclass
parent = task.get_proxy()
argv = task.subargv[1:]
_dirname, _basename = os.path.split(taskpath)
_base, _ext = os.path.splitext(_basename)
# consumes all subargv
del task.subargv[:]
task_class, argv, subargv, objs = load_taskclass(taskpath, argv, [])
if task_class is None:
raise UsageError("ERROR: Task '%s' is not loaded. "
" Please check task-path." % taskpath)
task = task_class(parent)
if task is None:
raise UsageError("ERROR: Task '%s' is not created." % taskpath)
task._env.update(objs)
from pyloco.plxtask import PlXTask
if task_class is PlXTask:
task._setup(taskpath)
self.tmpdir = tempfile.mkdtemp()
self.taskname = task._name_
topdir = os.path.join(self.tmpdir, self.taskname)
srcdir = os.path.join(topdir, self.taskname)
os.makedirs(srcdir)
package_data = ""
# generate __init__.py
if os.path.isfile(taskpath):
shutil.copy(taskpath, srcdir)
clsname = task.__class__.__name__
init_extra = []
main_extra = []
if task_class is PlXTask:
modname = "pyloco.plxtask"
init_extra.append('plx = "%s"' % _basename)
main_extra.append('plx = "%s"' % _basename)
package_data = ('"%s": ["%s"]' %
(self.taskname, _basename))
else:
modname = "." + _base
# TODO: add metadata such as __doc__
with open(os.path.join(srcdir, "__init__.py"), "w") as f:
extra = "\n".join(init_extra)
f.write(_taskinit_template % (modname, clsname, extra))
with open(os.path.join(srcdir, "__main__.py"), "w") as f:
extra = "\n".join(main_extra)
f.write(_taskmain_template % (modname, clsname, extra))
elif os.path.isdir(taskpath):
import pdb; pdb.set_trace() # noqa: E702
else:
raise UsageError(
"Task '%s' supports packing of Python module and package." %
self.taskname
)
setup_kwargs = {}
modules = {}
mod2dist = {}
dists = {}
collect_modules(srcdir, modules)
ret, sout, serr = system("pip list")
sout = sout.replace("(", " ").replace(")", " ")
dist_list = [d.split() for d in sout.split("\n")[2:] if d]
for d, v in dict(d for d in dist_list if len(d) == 2).items():
ret, sout, serr = system("pip show -f " + d)
for line in sout.split("\n"):
if line.endswith("__init__.py"):
pkgline = line.split(os.sep)
if len(pkgline) == 2:
mod2dist[pkgline[0].strip()] = (d, v)
elif line.strip() == d + ".py":
mod2dist[d] = (d, v)
site = os.path.dirname(ast.__file__)
for m, v in modules.items():
d = mod2dist.get(m, None)
if m == "pyloco":
from pyloco.manage import PylocoManager
dists[m] = PylocoManager._version_
elif m == parent.get_managerattr("name"):
dists[m] = parent.get_managerattr("version")
elif d is None:
d = sys.modules.get(m, None)
if m in sys.builtin_module_names:
pass
else:
pass
# TODO: check this
# moddir = os.path.dirname(d.__file__)
# sitem = os.path.join(site, m)
#
# if moddir != site and moddir != sitem:
# import pdb; pdb.set_trace()
# raise InternalError(
# "Distribution package for '%s' module is not "
# "found." % m)
else:
dists[d[0]] = d[1]
# TODO: copy other tasks and data files in srcdir
install_requires = []
if hasattr(task, "_install_requires_"):
install_requires.extend(['"%s"' % r for r in task._install_requires_])
for dname, dver in dists.items():
if dver is not None:
install_requires.append('"{0}>={1}"'.format(dname, dver))
else:
install_requires.append('"{}"'.format(dname))
setup_kwargs["install_requires"] = ", ".join(install_requires)
# generate setup.py
with open(os.path.join(topdir, "setup.py"), "w") as f:
setup_kwargs["distname"] = distname if distname else self.taskname
setup_kwargs["taskname"] = self.taskname
setup_kwargs["version"] = getattr(task, "_version_", "0.1.0")
setup_kwargs["package_data"] = package_data
setup_kwargs["entry_points"] = (
'"{taskname} = {taskname}:entry_task".format(taskname=__taskname__)'
)
f.write(_setup_template.format(**setup_kwargs))
def load_testclass(testpath):
if not testpath:
return None
if isinstance(testpath, type):
if issubclass(testpath, Test):
return testpath
raise UsageError("Not compatible test type: %s" % type(testpath))
test_class = None
_p = testpath.split("#", 1)
if len(_p) == 2:
testpath, fragment = [x.strip() for x in _p]
else:
fragment = ""
if os.path.exists(testpath):
mods = []
if os.path.isfile(testpath):
head, base = os.path.split(testpath)
if base.endswith(".py"):
mods.append(load_pymod(head, base[:-3]))
candidates = {}
for mod in mods:
for name in dir(mod):
if not name.startswith("_"):
obj = getattr(mod, name)
if (type(obj) == type(Test) and issubclass(obj, Test) and
(obj.__module__ is None or
not obj.__module__.startswith("pyloco."))):
candidates[name] = obj
if candidates:
if fragment:
if hasattr(candidates, fragment):
test_class = getattr(candidates, fragment)
else:
raise UsageError("No test is found with a fragment of "
"'%s'." % fragment)
elif len(candidates) == 1:
test_class = candidates.popitem()[1]
else:
raise UsageError(
"More than one frame are found."
"Please add fragment to select one: %s" %
list(candidates.keys())
)
if test_class:
setattr(test_class, "_path_", os.path.abspath(testpath))
else:
raise UsageError("Test class is not found. Please check path: %s" % testpath)
return test_class
def run_section(self, body, *vargs, **kwargs):
env = dict(self._env)
env.update(self.parent.shared)
lenv = kwargs.get("lenv", {})
lines = []
hidx = 0
for b in body:
l1 = b.replace("{", "{{").replace("}", "}}")
l2 = l1.replace("__{{", "{").replace("}}__", "}")
l3 = pyloco_formatter.vformat(l2, [], env)
_match, _line = collect_plx_command(l3)
if _match:
opt = _match[0]
name = opt.vargs[0]
cmd_handler = None
for ctx in opt.context:
if ctx in self._command_handlers:
cmd_handler = self._command_handlers[ctx]
break
# rhs = parse_param_option(_match[1], True, None)
if cmd_handler:
idx_space = l3.find(name)
fname = "__plx_cmd_handler%d__" % hidx
hidx += 1
env[fname] = cmd_handler
vargs = ", ".join(opt.vargs[1:])
kwargs = ", ".join(
["%s=%s" % (k, v) for k, v in opt.kwargs.items()])
if kwargs:
args = "%s, %s" % (vargs, kwargs)
else:
args = vargs
cmd = (_match[1].replace('\\"', '__EDQ__').replace(
'"', '\\"').replace('__EDQ__', '\\\\\\"'))
cmd = cmd.strip()
if args:
lines.append(l3[:idx_space] + "%s = " % name + fname +
'("%s", %s)' % (cmd, args))
else:
lines.append(l3[:idx_space] + "%s = " % name + fname +
'("%s")' % cmd)
else:
raise UsageError("command handler for '%s' is not found." %
opt.context[0])
else:
lines.append(l3)
exec("\n".join(lines), env, lenv)
self.parent.shared.update(lenv)
return lenv["out"][0] if "out" in lenv else 0
def perform(self, targs):
if targs.task:
taskpath = targs.task
elif self.subargv:
if "--" in self.subargv:
raise UsageError(
"'install' task requires only one sub-task "
"to install, but found multiple sub-tasks: '%s'" %
" ".join(self.subargv)
)
taskpath = self.subargv[0]
else:
raise UsageError(
"'install' task requires one sub-task to install."
)
if os.path.exists(taskpath):
if zipfile.is_zipfile(taskpath):
if taskpath.endswith(".plz"):
with _Unpack(taskpath) as (taskname, tmpdir):
topdir = os.path.join(tmpdir, taskname)
dirnames = os.listdir(topdir)
if taskname not in dirnames:
raise UsageError(
"'%s' is not a plz file." % taskpath
)
self._install(topdir, taskname, targs)
else:
raise UsageError("Unknown file format: %s" % taskpath)
else:
with _Pack(self, taskpath) as (taskname, tmpdir):
topdir = os.path.join(tmpdir, taskname)
self._install(topdir, taskname, targs)
else:
# TODO?: ask user permission to install
installed = False
try:
distname = taskpath.replace("_", "-")
url = "%s/%s/json" % (repo_check, distname)
if not PY3:
url = strencode(url)
if json.load(urlopen(url)):
prefix = (" install -i %s --extra-index-url %s " %
(repo_install, extra_repo_install))
args = []
if targs.user or not (is_venv() or "CONDA_DEFAULT_ENV" in os.environ):
args.append("--user")
if targs.upgrade:
args.append("-U")
ret, _, _ = system(get_pip() + prefix + " ".join(args) +
" " + distname)
if ret == 0:
print("'%s' task is installed successfully." % taskpath)
installed = True
except HTTPError:
pass
if not installed:
url = urlparse(taskpath)
if url.netloc or url.scheme:
import pdb; pdb.set_trace() # noqa: E702
else:
raise UsageError("Task '%s' is not found." % taskpath)
del self.subargv[:]
