def run(self, parent, forward, extra_args, params): from pyloco.task import load_taskclass, Task out = 0 out_forward = None for task, argv, subargv in self._tasks: task_class, argv, subargv, objs = load_taskclass(task, argv + extra_args, subargv) if task_class is None: raise UsageError("Task '%s' is not loaded." % str(task)) taskobj = task_class(parent) if taskobj is None: raise InternalError("Task '%s' is not created." % str(task)) #with open("taskpath.%d.log"%os.getpid(), "w") as f: # f.write("\n\n".join([str(taskobj), str(parent)])) # f.flush() if isinstance(taskobj, Task): forward["_pathid_"] = self._pathid_ taskobj._env.update(objs) taskobj._env.update(params) out, forward = taskobj.run(argv, subargv=subargv, forward=forward) out_forward = forward else: raise UsageError("Task '%s' is not a Task type." % str(task)) return out, out_forward
def perform(self, targs): if targs.task: taskpath = targs.task elif self.subargv: if "--" in self.subargv: raise UsageError( "'upload' task requires only one sub-task " "to upload, but found multiple sub-tasks: '%s'" % " ".join(self.subargv) ) taskpath = self.subargv[0] else: raise UsageError( "'upload' task requires one sub-task to upload." ) with _Pack(self, taskpath, distname=targs.name) as (taskname, tmpdir): cwd = os.getcwd() os.chdir(os.path.join(tmpdir, taskname)) ret, sout, serr = system(sys.executable + " setup.py sdist") ret, sout, serr = system(sys.executable + " setup.py bdist_wheel --universal") ret = isystem("twine upload --repository-url %s dist/*" % targs.repository) os.chdir(cwd) if ret == 0: print("'%s' task is uploaded successfully." % taskpath) else: print("Uploading of '%s' task is failed." % taskpath)
def perform(self, targs): if os.path.exists(targs.command): raise UsageError("Specified command '%s' already exists." % targs.command) # get OS and select shell script type if OS == "windows": import pdb; pdb.set_trace() else: if len(self.subargv) < 1: raise UsageError("No target task is specified.") if not os.path.exists(self.subargv[0]): raise UsageError("Target task does not exist: %s" % self.subargv[0]) self.subargv[0] = os.path.abspath(os.path.realpath(self.subargv[0])) with open(targs.command, "w") as f: retval, stdout, _ = system("which bash") if retval == 0: f.write("#!" + stdout + "\n") f.write("%s %s $*" % (sys.argv[0], " ".join(self.subargv))) os.chmod(targs.command, 0o744)
def perform(self, targs): if targs.task: taskpath = targs.task elif self.subargv: if "--" in self.subargv: raise UsageError( "'upload' task requires only one sub-task " "to upload, but found multiple sub-tasks: '%s'" % " ".join(self.subargv) ) taskpath = self.subargv[0] else: raise UsageError( "'upload' task requires one sub-task to upload." ) with _Pack(self, taskpath, distname=targs.name) as (taskname, tmpdir): topdir = os.path.join(tmpdir, taskname) outdir = targs.outdir if targs.outdir else "" if targs.name: outfile = os.path.join(outdir, targs.name+".plz") else: outfile = os.path.join(outdir, taskname+".plz") shutil.make_archive(topdir, 'zip', *os.path.split(topdir)) shutil.move(topdir+".zip", outfile) print("'%s' task is packed successfully." % taskpath)
def _append_task(self, path, items): if len(items) < 1: raise UsageError("task is not found: %s" % str(self.subargv)) if items[0].startswith("-"): raise UsageError("First argument is not a task reference: %s" % str(items)) path.append_task(items[0], argv=items[1:]) del items[:]
def run(self, argv, subargv=None, forward=None): if not argv: raise UsageError("PlZ Task is not found." " Please check plz path.") elif not os.path.isfile(argv[0]): raise UsageError("PlZ Task '%s' is not found." " Please check plz path." % str(argv[0])) out = -1 taskpath = argv.pop(0) if zipfile.is_zipfile(taskpath): tempdir = None with TemporaryDirectory() as tempdir: plz = zipfile.ZipFile(taskpath) plz.extractall(path=tempdir) plz.close() dirnames = os.listdir(tempdir) if len(dirnames) != 1: raise UsageError("'%s' is not a plz file." % taskpath) taskname = dirnames[0] pkgdir = os.path.join(tempdir, taskname) moddir = os.path.join(pkgdir, taskname) sys.path.insert(0, pkgdir) mod = pyloco_import(taskname) taskcls = getattr(mod, "entry_task") if taskcls is PlXTask: argv.insert(0, os.path.join(moddir, getattr(mod, "plx"))) sys.path.pop(0) from pyloco.main import perform parent = self.get_proxy() out = perform(taskcls, argv=argv, subargv=subargv, parent=parent, forward=forward, shared=self.parent.shared) return out else: raise UsageError("'%s' is not a plz file format." % taskpath)
def _register_check(self, dest): if not dest: raise UsageError("Incorrect name: %s" % dest) if dest.startswith("_"): raise UsageError("'Forward-name' should not start with an " "underscore ('_'): %s" % dest) if dest in self._fwddefs: raise UsageError("'%s' is already registered for forwarding" % dest) if dest in self._shrdefs: raise UsageError("'%s' is already registered for sharing" % dest)
def parse_param_option(val, evaluate, env): def _p(*argv, **kw_str): return list(argv), kw_str obj = Option() if isinstance(val, str) or type(val) == type(u"A"): # noqa: E721 items = [v.strip() for v in val.split("@")] obj.context = items[1:] for c in obj.context: if not is_valid_variable_name(c): items = [val] obj.context = [] break if evaluate: eval_out = teval('_p({0})'.format(items[0]), env, _p=_p) if eval_out: obj.vargs, obj.kwargs = eval_out else: raise UsageError("syntax error at '{0}'.".format(items[0])) else: obj.vargs, obj.kwargs = parse_literal_args(items[0]) else: obj.vargs = [val] return obj
def _install(self, topdir, taskname, targs): dirnames = os.listdir(topdir) if "setup.py" not in dirnames: raise UsageError("'setup.py' is not found.'") if targs.name and targs.name != taskname: new_taskname = targs.name os.rename(os.path.join(topdir, taskname), os.path.join(topdir, new_taskname)) taskname = new_taskname setup = os.path.join(topdir, "setup.py") oldsetup = os.path.join(topdir, "setup.py.old") shutil.move(setup, oldsetup) prefix = ' __taskname__ = "' with open(setup, "w") as fw: with open(oldsetup, "r") as fr: for line in fr: if line.startswith(prefix): fw.write(prefix + taskname + '"\n') else: fw.write(line) os.remove(oldsetup) retval, stdout, stderr = system(sys.executable + " setup.py sdist" " --formats=tar", cwd=topdir) if retval != 0: raise InternalError(stderr) distdir = os.path.join(topdir, "dist") sdist = None for dname in os.listdir(distdir): if dname.endswith(".tar"): sdist = dname break if sdist is None: raise InternalError("'sdist' file is not found: %s" % taskname) if is_venv(): retval, stdout, stderr = system(get_pip() + " install " + sdist, cwd=distdir) # system("python setup.py install", cwd=topdir) else: retval, stdout, stderr = system(get_pip() + " install %s --user" % sdist, cwd=distdir) # system("python setup.py install --user", cwd=topdir) if retval == 0: # TODO: print installed version with some progress remarks print("'%s' task is installed successfully." % taskname) else: raise InternalError(stderr)
def __init__(self, zfile): self.tmpdir = tempfile.mkdtemp() shutil.unpack_archive(zfile, self.tmpdir, "zip") dirnames = os.listdir(self.tmpdir) if len(dirnames) != 1: raise UsageError("'%s' is not a plz file." % zfile) self.taskname = dirnames[0]
def __getattr__(self, attr): try: return getattr(self.task.parent, attr) except Exception: raise UsageError( "'ServerProxy' object has no attribute '%s'" % attr)
def parse_literal_args(expr): lv = [] lk = {} expr_items = expr.split(",") text = "" while expr_items: expr_item = expr_items.pop(0).strip() if not expr_item: continue if text: text = text + "," + expr_item else: text = expr_item #if not text: # continue try: tree = ast.parse("func({0})".format(text)) args = tree.body[0].value.args keywords = tree.body[0].value.keywords if len(args) > 0 and len(keywords): raise UsageError("Both of args and keywords are found" " during argument parsing.") text = text.strip() if not text: continue if len(args) > 0: lv.append(text) elif len(keywords) > 0: key, val = text.split("=", 1) if val: lk[key] = val text = "" except Exception: pass #if not lv and not lk: # lv.append(expr) return lv, lk
def perform(self, targs): if targs.task: if len(targs.task) > 1: raise UsageError( "'uninstall' task only one task: %s." % targs.task ) taskname = targs.task[0] elif self.subargv: if "--" in self.subargv: raise UsageError( "'uninstall' task requires only one sub-task " "to uninstall, but found multiple sub-tasks: '%s'" % " ".join(self.subargv) ) taskname = self.subargv[0] del self.subargv[:] else: raise UsageError( "'uninstall' task requires one sub-task to uninstall." ) for ep in pkg_resources.iter_entry_points(group='pyloco.task'): if ep.name == taskname: retval, sout, serr = system( get_pip() + " uninstall -y " + ep.dist.project_name ) if retval == 0: print("'%s' task is uninstalled successfully." % taskname) else: print("Uninstallation of '%s' task was failed: %s" % (taskname, serr)) return retval raise UsageError("'%s' is not found." % taskname)
def _handle_sharedarg(self, shared): # TODO: need this? try: for shr in shared: for varg in vargs: self.parent.shared[varg] = env[varg] for dest, value in shr.kwargs.items(): self.parent.shared[dest] = eval(value, env, lenv) except Exception as err: raise UsageError("failed on sharing variable: %s" % str(err))
def run(self, argv, subargv=None, forward=None): if not argv: raise UsageError("PlX Task is not found." " Please check plx path.") elif not os.path.isfile(argv[0]): raise UsageError("PlX Task '%s' is not found." " Please check plx path." % str(argv[0])) self._setup(argv[0]) prog = os.path.basename(getattr(self, "_path_", self._name_)) self._parser.prog = self.get_mgrname() + " " + prog[-20:] if hasattr(self, "__doc__") and self.__doc__: self._parser.desc, self._parser.long_desc = pydoc.splitdoc( self.__doc__) return super(PlXTask, self).run(argv[1:], subargv=subargv, forward=forward)
def perform(self, targs): if os.path.exists(targs.task): raise UsageError("Task '%s' is already exists: " % targs.task) outdir, filename = os.path.split(targs.task) outdir = os.path.abspath(outdir) if not os.path.exists(outdir): os.makedirs(outdir) taskname, tasktype = os.path.splitext(filename) if not taskname and tasktype: taskname = tasktype tasktype = "package" clsname = taskname[0].upper() + taskname[1:] if tasktype == ".py": output = _init_template.format(clsname=clsname, taskname=taskname, taskversion=targs.task_version) with open(targs.task, "w") as f: f.write(output) elif tasktype == ".plx": import pdb pdb.set_trace() elif tasktype == "package": import pdb pdb.set_trace() else: raise UsageError("Unknown task type: %s" % tasktype) print("Initialized '%s' task development in '%s'" % (taskname, outdir))
def _handle_sharedarg(self, shared, forwards): try: env = dict(self._env) env.update(forwards) for shr in shared: for varg in shr.vargs: self.parent.shared[varg] = env[varg] for dest, value in shr.kwargs.items(): self.parent.shared[dest] = eval(value, env, {}) except Exception as err: raise UsageError("failed on sharing variable: %s" % str(err))
def _add_transfer(self, defs, cont, **kwargs): for dest, value in kwargs.items(): if dest not in defs: raise UsageError("'%s' is not registered for data transfer." % dest) if type_check(value, defs[dest][0]): cont[dest] = value else: if isinstance(value, str) and os.path.isfile(value): import pdb; pdb.set_trace() # noqa: E702 else: raise TestError("Data transfer type check failure: %s" % dest)
def import_testplx(path, manager, argv, subargv): if argv is None: argv = [] if subargv is None: subargv = [] if not os.path.isfile(path): raise UsageError("PlX Task '%s' is not found." " Please check plx path." % str(path)) PlXTest._plx_ = plx = PlXTestTask(manager) plx._setup(path) prog = os.path.basename(getattr(plx, "_path_", plx._name_)) plx._parser.prog = plx.get_mgrname() + " " + prog[-20:] if hasattr(plx, "__doc__") and plx.__doc__: plx._parser.desc, plx._parser.long_desc = pydoc.splitdoc( plx.__doc__) super(PlXTask, plx).run(argv, subargv=subargv) del PlXTest._setups[:] del PlXTest._teardowns[:] # create test methods for hdr, body in plx.plx_sections: if hdr.endswith("*"): if hdr.startswith("setup"): PlXTest._setups.append(partial(PlXTest._test_template, header=hdr[:-1], body=body)) elif hdr.startswith("teardown"): PlXTest._teardowns.append(partial(PlXTest._test_template, header=hdr[:-1], body=body)) else: setattr(PlXTest, "test_%s" % hdr.replace("@", "_"), partial(PlXTest._test_template, header=hdr, body=body)) import imp mod = imp.new_module("") setattr(mod, "PlXTest", PlXTest) return mod
def perform(self, targs): for plx_dest, opt in self.plx_argdefs.items(): dest = opt.kwargs.get("dest", opt.vargs[0]) if dest in self._env["__arguments__"]: argval = self._env["__arguments__"].pop(dest, None) self._env["__arguments__"][plx_dest] = argval out = self.run_section(self.plx_entry_body) if out == 0: for hdr, body in self.plx_sections: # check hdr if hdr.endswith("*"): special_sec = True opt = parse_param_option(hdr[:-1], False, self.parent.shared) else: special_sec = False opt = parse_param_option(hdr, False, self.parent.shared) env = dict(self.parent.shared) env["__builtins__"] = pyloco_builtins sec_check = all([eval(c, env) for c in opt.context]) if sec_check: sec_name = opt.vargs[0] # find sec_handler if special_sec: if sec_name in self._section_handlers: sec_handler = self._section_handlers[sec_name] else: raise UsageError( "Special section '%s' is not registered." " Please register first." % sec_name) else: sec_handler = self.run_section out = sec_handler(body, *opt.vargs[1:], **opt.kwargs) return out
def import_modulepath(path): #debug(path == "test.plx") fragment = None spath = [p.strip() for p in path.split("#")] if len(spath) > 1: path = spath[0] fragment = spath[1] if os.path.exists(path): head, base = os.path.split(path) mod = None if os.path.isfile(path) and path.endswith(".py"): modname = base[:-3] mod = load_pymod(head, modname) elif (os.path.isdir(path) and os.path.isfile(os.path.join(path, "__init__.py"))): if base[-1] == os.sep: modname = base[:-1] else: modname = base mod = load_pymod(head, modname) else: try: modname = path mod = pyloco_import(modname) except ModuleNotFoundError as err: raise UsageError("path does not exist: %s" % path) if mod: if fragment: return fragment, getattr(mod, fragment) else: return modname, mod
def perform(self, targs): try: #original_argparser = Task._argparser_ #Task._argparser_ = TestArgParser if targs.test: if targs.test.endswith(".plx"): module = import_testplx(targs.test, self.get_proxy(), targs.testargv, targs.testsubtask) else: _, module = import_modulepath(targs.test) unittest.main(module=module, argv=["dummy"]) elif self.subargv: import pyloco.grouptask as testmodule del testmodule.DefaultGroupTestCase._testargs_[:] testmodule.DefaultGroupTestCase._testargs_.extend(self.subargv) unittest.main(module=testmodule, argv=["dummy"]) else: raise UsageError("No test is specified.") # # # results = [] # # # invoke test runner # if targs.params: # for param in targs.params: # for varg in param.vargs: # results.append((vargs, param.kwargs, tester.run(varg, param.kwargs))) # finally: Task._argparser_ = self.original_argparser
def test_section(self, hdr, body): # check hdr if hdr.endswith("*"): special_sec = True opt = parse_param_option(hdr[:-1], False, self.parent.shared) else: special_sec = False opt = parse_param_option(hdr, False, self.parent.shared) env = dict(self.parent.shared) env["__builtins__"] = pyloco_builtins sec_check = all([eval(c, env) for c in opt.context]) if sec_check: sec_name = opt.vargs[0] # find sec_handler if special_sec: if sec_name in self._section_handlers: sec_handler = self._section_handlers[sec_name] else: raise UsageError( "Special section '%s' is not registered." " Please register first." % sec_name ) else: sec_handler = self.run_section return sec_handler(body, *opt.vargs[1:], **opt.kwargs) else: return 0
def post_perform(self, targs): if targs.webapp: appath = targs.webapp wait2close = self.taskattr.get("webapp.wait2close", True) if wait2close: if self._websocket_server: pyloco_print("Waiting for '%s' to be completed..." % appath, end="") sys.stdout.flush() self._websocket_server.communicate(input=None) if self._websocket_client: self._websocket_client.close() if self._webserver: self._webserver.communicate(input=None) pyloco_print("DONE.") sys.stdout.flush() env = dict(self._env) env.update(self.parent.shared) env.update(self._fwds) lenv = {} if targs.forward: try: for fwd in targs.forward: for varg in fwd.vargs: self._fwds[varg] = env[varg] for dest, value in fwd.kwargs.items(): self._fwds[dest] = eval(value, env, lenv) except Exception as err: raise UsageError("failed on forwarding: %s" % str(err)) if targs.shared: self._handle_sharedarg(targs.shared) if hasattr(targs, "assert_output") and targs.assert_output: aenv = {"__builtins__": pyloco_builtins} for k, v in self._env.items(): if not k.startswith("_"): aenv[k] = v aenv.update(self.parent.shared) aenv.update(self._fwds) for boolexpr in targs.assert_output: for varg in boolexpr.vargs: assert_result = eval(varg, aenv) if assert_result: if self._verbose: pyloco_print('\nOUTPUT TEST PASSED with "%s"' % varg) else: pairs = split_assert_expr(varg) if not pairs: raise TestError("\nOUTPUT TEST FAILED with '%s' =>" " not True" % varg) elif len(pairs) == 1: sep, (lexpr, rexpr) = pairs.popitem() msg = ("\nOUTPUT TEST(%s) is FAILED.\n " "Left expr(%s) of '%s' is evaluated to '%s'" " and\n right expr(%s) of '%s' " "is evaluated to '%s'.\n") % ( varg, lexpr, sep, eval(lexpr, aenv), rexpr, sep, eval(rexpr, aenv)) raise TestError(msg) else: msg = ( "\nOUTPUT TEST(%s) FAILED: detected multiple" " possibilities of this test failure\n") % varg idx = 0 for sep, (lexpr, rexpr) in pairs.items(): idx += 1 try: msg += ("CASE%d:\n Left expr(%s)" " of" " '%s' is evaluated to '%s' and\n" " right expr(%s) of '%s' is " "evaluated to '%s'.\n") % ( idx, lexpr, sep, eval(lexpr, aenv), rexpr, sep, eval(rexpr, aenv)) except Exception: pass raise TestError(msg) if targs.write_pickle: ppf = PylocoPickle() data = dict(self.parent.shared) data.update(self._fwds) data.pop("parent_name", None) pdata = self.write_pickle(ppf, data) ppf.dump(pdata, targs.write_pickle)
def load_taskclass(taskpath, argv, subargv): if not taskpath: return None, None, None, None # TODO: handle aliased task if isinstance(taskpath, type): if issubclass(taskpath, Task): return taskpath, argv, subargv, None raise UsageError("Not compatible task type: %s" % type(taskpath)) # TODO: move to callsite to load_taskclass objs = {} while "--import" in argv: idx = argv.index("--import") mpath = argv.pop(idx + 1) argv.pop(idx) key, obj = import_modulepath(mpath) objs[key] = obj task_class = None _p = taskpath.split("#", 1) if len(_p) == 2: taskpath, fragment = [x.strip() for x in _p] else: fragment = "" if os.path.exists(taskpath): mods = [] if os.path.isfile(taskpath): head, base = os.path.split(taskpath) if base.endswith(".py"): mods.append(load_pymod(head, base[:-3])) elif base.endswith(".plx"): from pyloco.plxtask import PlXTask task_class = PlXTask argv.insert(0, taskpath) elif base.endswith(".plz"): from pyloco.plztask import PlZTask task_class = PlZTask argv.insert(0, taskpath) elif os.path.isdir(taskpath): # TODO: support Python package pass import pdb pdb.set_trace() candidates = {} for mod in mods: for name in dir(mod): if not name.startswith("_"): obj = getattr(mod, name) if (type(obj) == type(Task) and issubclass(obj, Task) and (obj.__module__ is None or not obj.__module__.startswith("pyloco."))): candidates[name] = obj if candidates: if fragment: if hasattr(candidates, fragment): task_class = getattr(candidates, fragment) else: raise UsageError("No task is found with a fragment of " "'%s'." % fragment) elif len(candidates) == 1: task_class = candidates.popitem()[1] else: raise UsageError("More than one frame are found." "Please add fragment to select one: %s" % list(candidates.keys())) if task_class: setattr(task_class, "_path_", os.path.abspath(taskpath)) #else: # raise UsageError("Task class is not found. Please check path: %s" % taskpath) if task_class is None: from pyloco.manage import _ManagerBase if taskpath in _ManagerBase._default_tasks_: task_class = _ManagerBase._default_tasks_[taskpath] if task_class is None: for ep in pkg_resources.iter_entry_points(group='pyloco.task'): if taskpath == ep.name: task_class = ep.load() from pyloco.plxtask import PlXTask if task_class is PlXTask: task_mod = pyloco_import(taskpath) task_dir = os.path.dirname(task_mod.__file__) argv.insert( 0, os.path.join(task_dir, getattr(task_mod, "plx"))) break if not task_class: from pyloco.mgmttask import mgmt_tasks from pyloco.stdtask import standard_tasks if taskpath in mgmt_tasks: task_class = mgmt_tasks[taskpath] elif taskpath in standard_tasks: task_class = standard_tasks[taskpath] # TODO support remote task # if not task_class: # # url = urlparse(taskpath) # # if url.netloc or url.scheme: # argv.insert(0, taskpath) # task_class = RemoteTask if not task_class: raise UsageError("Task '%s' is not found. Please check path." % taskpath) return task_class, argv, subargv, objs
def load_default_task(cls, *tasks, **kwargs): frame = inspect.stack()[1] module = inspect.getmodule(frame[0]) mgrdir = os.path.realpath( os.path.abspath(os.path.dirname(module.__file__))) for task in tasks: _p = task.split("#", 1) if len(_p) == 2: taskpath, fragment = [x.strip() for x in _p] else: taskpath = task fragment = "" taskmodpath = os.path.realpath( os.path.abspath(os.path.join(mgrdir, taskpath))) if os.path.exists(taskmodpath): modname, mod = import_modulepath(taskmodpath) if modname not in sys.modules: raise InternalError("'%s' task is not loaded." % task) task_class = None candidates = {} for name in dir(mod): if not name.startswith("_"): obj = getattr(mod, name) if (type(obj) == type(Task) and issubclass(obj, Task) and (obj.__module__ is None or not obj.__module__.startswith("pyloco."))): candidates[name] = obj if candidates: if fragment: if hasattr(candidates, fragment): task_class = getattr(candidates, fragment) else: raise UsageError( "No task is found with a fragment of " "'%s'." % fragment) elif len(candidates) == 1: task_class = candidates.popitem()[1] else: raise UsageError( "More than one frame are found." "Please add fragment to select one: %s" % list(candidates.keys())) else: raise UsageError("'%s' does not exist within this manager." % task) if task_class: cls._default_tasks_[task_class._name_] = task_class else: raise UsageError("'%s' task is not loaded." % task)
def __init__(self, task, taskpath, distname=None): from pyloco.task import load_taskclass parent = task.get_proxy() argv = task.subargv[1:] _dirname, _basename = os.path.split(taskpath) _base, _ext = os.path.splitext(_basename) # consumes all subargv del task.subargv[:] task_class, argv, subargv, objs = load_taskclass(taskpath, argv, []) if task_class is None: raise UsageError("ERROR: Task '%s' is not loaded. " " Please check task-path." % taskpath) task = task_class(parent) if task is None: raise UsageError("ERROR: Task '%s' is not created." % taskpath) task._env.update(objs) from pyloco.plxtask import PlXTask if task_class is PlXTask: task._setup(taskpath) self.tmpdir = tempfile.mkdtemp() self.taskname = task._name_ topdir = os.path.join(self.tmpdir, self.taskname) srcdir = os.path.join(topdir, self.taskname) os.makedirs(srcdir) package_data = "" # generate __init__.py if os.path.isfile(taskpath): shutil.copy(taskpath, srcdir) clsname = task.__class__.__name__ init_extra = [] main_extra = [] if task_class is PlXTask: modname = "pyloco.plxtask" init_extra.append('plx = "%s"' % _basename) main_extra.append('plx = "%s"' % _basename) package_data = ('"%s": ["%s"]' % (self.taskname, _basename)) else: modname = "." + _base # TODO: add metadata such as __doc__ with open(os.path.join(srcdir, "__init__.py"), "w") as f: extra = "\n".join(init_extra) f.write(_taskinit_template % (modname, clsname, extra)) with open(os.path.join(srcdir, "__main__.py"), "w") as f: extra = "\n".join(main_extra) f.write(_taskmain_template % (modname, clsname, extra)) elif os.path.isdir(taskpath): import pdb; pdb.set_trace() # noqa: E702 else: raise UsageError( "Task '%s' supports packing of Python module and package." % self.taskname ) setup_kwargs = {} modules = {} mod2dist = {} dists = {} collect_modules(srcdir, modules) ret, sout, serr = system("pip list") sout = sout.replace("(", " ").replace(")", " ") dist_list = [d.split() for d in sout.split("\n")[2:] if d] for d, v in dict(d for d in dist_list if len(d) == 2).items(): ret, sout, serr = system("pip show -f " + d) for line in sout.split("\n"): if line.endswith("__init__.py"): pkgline = line.split(os.sep) if len(pkgline) == 2: mod2dist[pkgline[0].strip()] = (d, v) elif line.strip() == d + ".py": mod2dist[d] = (d, v) site = os.path.dirname(ast.__file__) for m, v in modules.items(): d = mod2dist.get(m, None) if m == "pyloco": from pyloco.manage import PylocoManager dists[m] = PylocoManager._version_ elif m == parent.get_managerattr("name"): dists[m] = parent.get_managerattr("version") elif d is None: d = sys.modules.get(m, None) if m in sys.builtin_module_names: pass else: pass # TODO: check this # moddir = os.path.dirname(d.__file__) # sitem = os.path.join(site, m) # # if moddir != site and moddir != sitem: # import pdb; pdb.set_trace() # raise InternalError( # "Distribution package for '%s' module is not " # "found." % m) else: dists[d[0]] = d[1] # TODO: copy other tasks and data files in srcdir install_requires = [] if hasattr(task, "_install_requires_"): install_requires.extend(['"%s"' % r for r in task._install_requires_]) for dname, dver in dists.items(): if dver is not None: install_requires.append('"{0}>={1}"'.format(dname, dver)) else: install_requires.append('"{}"'.format(dname)) setup_kwargs["install_requires"] = ", ".join(install_requires) # generate setup.py with open(os.path.join(topdir, "setup.py"), "w") as f: setup_kwargs["distname"] = distname if distname else self.taskname setup_kwargs["taskname"] = self.taskname setup_kwargs["version"] = getattr(task, "_version_", "0.1.0") setup_kwargs["package_data"] = package_data setup_kwargs["entry_points"] = ( '"{taskname} = {taskname}:entry_task".format(taskname=__taskname__)' ) f.write(_setup_template.format(**setup_kwargs))
def load_testclass(testpath): if not testpath: return None if isinstance(testpath, type): if issubclass(testpath, Test): return testpath raise UsageError("Not compatible test type: %s" % type(testpath)) test_class = None _p = testpath.split("#", 1) if len(_p) == 2: testpath, fragment = [x.strip() for x in _p] else: fragment = "" if os.path.exists(testpath): mods = [] if os.path.isfile(testpath): head, base = os.path.split(testpath) if base.endswith(".py"): mods.append(load_pymod(head, base[:-3])) candidates = {} for mod in mods: for name in dir(mod): if not name.startswith("_"): obj = getattr(mod, name) if (type(obj) == type(Test) and issubclass(obj, Test) and (obj.__module__ is None or not obj.__module__.startswith("pyloco."))): candidates[name] = obj if candidates: if fragment: if hasattr(candidates, fragment): test_class = getattr(candidates, fragment) else: raise UsageError("No test is found with a fragment of " "'%s'." % fragment) elif len(candidates) == 1: test_class = candidates.popitem()[1] else: raise UsageError( "More than one frame are found." "Please add fragment to select one: %s" % list(candidates.keys()) ) if test_class: setattr(test_class, "_path_", os.path.abspath(testpath)) else: raise UsageError("Test class is not found. Please check path: %s" % testpath) return test_class
def run_section(self, body, *vargs, **kwargs): env = dict(self._env) env.update(self.parent.shared) lenv = kwargs.get("lenv", {}) lines = [] hidx = 0 for b in body: l1 = b.replace("{", "{{").replace("}", "}}") l2 = l1.replace("__{{", "{").replace("}}__", "}") l3 = pyloco_formatter.vformat(l2, [], env) _match, _line = collect_plx_command(l3) if _match: opt = _match[0] name = opt.vargs[0] cmd_handler = None for ctx in opt.context: if ctx in self._command_handlers: cmd_handler = self._command_handlers[ctx] break # rhs = parse_param_option(_match[1], True, None) if cmd_handler: idx_space = l3.find(name) fname = "__plx_cmd_handler%d__" % hidx hidx += 1 env[fname] = cmd_handler vargs = ", ".join(opt.vargs[1:]) kwargs = ", ".join( ["%s=%s" % (k, v) for k, v in opt.kwargs.items()]) if kwargs: args = "%s, %s" % (vargs, kwargs) else: args = vargs cmd = (_match[1].replace('\\"', '__EDQ__').replace( '"', '\\"').replace('__EDQ__', '\\\\\\"')) cmd = cmd.strip() if args: lines.append(l3[:idx_space] + "%s = " % name + fname + '("%s", %s)' % (cmd, args)) else: lines.append(l3[:idx_space] + "%s = " % name + fname + '("%s")' % cmd) else: raise UsageError("command handler for '%s' is not found." % opt.context[0]) else: lines.append(l3) exec("\n".join(lines), env, lenv) self.parent.shared.update(lenv) return lenv["out"][0] if "out" in lenv else 0
def perform(self, targs): if targs.task: taskpath = targs.task elif self.subargv: if "--" in self.subargv: raise UsageError( "'install' task requires only one sub-task " "to install, but found multiple sub-tasks: '%s'" % " ".join(self.subargv) ) taskpath = self.subargv[0] else: raise UsageError( "'install' task requires one sub-task to install." ) if os.path.exists(taskpath): if zipfile.is_zipfile(taskpath): if taskpath.endswith(".plz"): with _Unpack(taskpath) as (taskname, tmpdir): topdir = os.path.join(tmpdir, taskname) dirnames = os.listdir(topdir) if taskname not in dirnames: raise UsageError( "'%s' is not a plz file." % taskpath ) self._install(topdir, taskname, targs) else: raise UsageError("Unknown file format: %s" % taskpath) else: with _Pack(self, taskpath) as (taskname, tmpdir): topdir = os.path.join(tmpdir, taskname) self._install(topdir, taskname, targs) else: # TODO?: ask user permission to install installed = False try: distname = taskpath.replace("_", "-") url = "%s/%s/json" % (repo_check, distname) if not PY3: url = strencode(url) if json.load(urlopen(url)): prefix = (" install -i %s --extra-index-url %s " % (repo_install, extra_repo_install)) args = [] if targs.user or not (is_venv() or "CONDA_DEFAULT_ENV" in os.environ): args.append("--user") if targs.upgrade: args.append("-U") ret, _, _ = system(get_pip() + prefix + " ".join(args) + " " + distname) if ret == 0: print("'%s' task is installed successfully." % taskpath) installed = True except HTTPError: pass if not installed: url = urlparse(taskpath) if url.netloc or url.scheme: import pdb; pdb.set_trace() # noqa: E702 else: raise UsageError("Task '%s' is not found." % taskpath) del self.subargv[:]