def run__test__(self, d, name): """d, name -> Treat dict d like module.__test__. Return (#failures, #tries). See testmod.__doc__ for details. """ failures = tries = 0 prefix = name + "." savepvt = self.isprivate try: self.isprivate = lambda *args: 0 # Run the tests by alpha order of names, for consistency in # verbose-mode output. keys = d.keys() keys.sort() for k in keys: v = d[k] thisname = prefix + k if type(v) in _StringTypes: f, t = self.runstring(v, thisname) elif _isfunction(v) or _isclass(v): f, t = self.rundoc(v, thisname) else: raise TypeError("Tester.run__test__: values in " "dict must be strings, functions " "or classes; " + `v`) failures = failures + f tries = tries + t finally: self.isprivate = savepvt return failures, tries
def run__test__(self, d, name): """d, name -> Treat dict d like module.__test__. Return (#failures, #tries). See testmod.__doc__ for details. """ failures = tries = 0 prefix = name + "." savepvt = self.isprivate try: self.isprivate = lambda *args: 0 # Run the tests by alpha order of names, for consistency in # verbose-mode output. keys = d.keys() keys.sort() for k in keys: v = d[k] thisname = prefix + k if type(v) in _StringTypes: f, t = self.runstring(v, thisname) elif _isfunction(v) or _isclass(v): f, t = self.rundoc(v, thisname) else: raise TypeError("Tester.run__test__: values in " "dict must be strings, functions " "or classes; " + ` v `) failures = failures + f tries = tries + t finally: self.isprivate = savepvt return failures, tries
def inner_decorator(init_or_class): if _isclass(init_or_class): func = getattr(init_or_class, '__init__') elif _isfunction(init_or_class): func = init_or_class else: raise ValueError("autoinit decorator should be applied to class or its __init__ method") if (func.__name__ != '__init__' or func.__code__.co_name != '__init__') and not no_warn: _warn(AutoinitWarning("autoinit decorator intended to be applied only to __init__" " method (use autoinit(no_warn=True) to suppress this warning)")) args_names = func.__code__.co_varnames[1:func.__code__.co_argcount] @_wraps(func) def inner(self, *args, **kwargs): if reverse: func(self, *args, **kwargs) args_vals = args[:] if func.__defaults__: args_vals += func.__defaults__[len(args) - len(args_names):] for key, val in zip(args_names, args_vals): if key not in exclude: if (type(self.__class__).__name__ != 'classobj' and hasattr(self, '__slots__') and key not in self.__slots__): raise AttributeError("Can not assign attribute '%s': it is not " "listed in %s.__slots__" % (key, self.__class__)) setattr(self, key, val) if not reverse: func(self, *args, **kwargs) if _isclass(init_or_class): init_or_class.__init__ = inner return init_or_class return inner
def args(arguments: _split=argv[1:], stdin_char: str='-', **mappings): """Parses the cmd-arguments to execute the requested command. :param arguments: The arguments to use (default: argv[1:]). :param stdin_char: The char to denote stdin-input. Use None to disable (default: '-'). :param mappings: Additional/custom mappings (i.e. {'command': function}). """ # getting arguments try: if stdin_char is not None and arguments[-1] == stdin_char: del arguments[-1] while True: try: i = input() arguments.append(i) except EOFError: break f, *a = arguments except (ValueError, IndexError): print("ERROR: Requires function or --help!", file=stderr) exit(1) # get caller module = _get_calling_module() # getting functions funcs = {k: v for k, v in vars(module).items() if not k.startswith('_') and _isfunction(v)} funcs['--help'] = funcs['-h'] = funcs['help'] = usage funcs['--list-commands'] = commands funcs.update(mappings) # use defined mappings if f not in funcs: raise Exception("Invalid function '{}'.".format(f)) # getting this function ff = funcs[f] # converting arguments fspecs = _getargs(ff) for i, (val, name) in enumerate(zip(a, fspecs.args)): if name in fspecs.annotations: a[i] = fspecs.annotations[name](val) if fspecs.varargs: l = len(fspecs.args) vals = a[l:] if fspecs.varargs in fspecs.annotations: a[l:] = fspecs.annotations[fspecs.varargs](vals) # executing if ff in {usage, commands}: ff(globalvars=funcs) else: return ff(*a)
def __to_dict__(self): params = self.get_params() param_dict = {} isFunction = False for k,v in params.iteritems(): param_dict[k] = v if _isfunction(v): param_dict[k] = v.func_name isFunction = True task_json = {} task_json["name"] = self.get_name() task_json["description"] = self.get_description() task_json["inputs"] = self.get_inputs() task_json["code"] = self.__code_snapshot__() task_json["outputs"] = self.get_outputs() task_json["params"] = param_dict task_json["isFunction"] = isFunction task_json["required_packages"] = list(self.get_required_packages()) return task_json
def doxtend(*bases, func_name=None, sep='\n\n'): """ Inherit doc-string(s) from other functions/methods. bases: base objects to search in, 'None' looks in globals() func_name: alternative name to search for(defaults to name of decorated function) sep: str to place between inherited docstring Attempts to order doc-strings in a cummulative/hierarchical fashion - moving left to right through 'bases' - excluding the same string(s) from different objects(except for the final (non)derived doc-string which is guaranteed to be included last)" """ def doxtend_decorator(func): nonlocal bases, func_name uniq_doc_strs = [] if func_name is None: func_name = func.__name__ for bc in bases: try: f = globals()[func_name] if bc is None else getattr(bc,func_name) except (KeyError, AttributeError): raise DoxtendError("couldn't find function '" + func_name + "'") if f.__doc__ not in uniq_doc_strs: uniq_doc_strs.append(f.__doc__) if uniq_doc_strs: if func.__doc__: uniq_doc_strs.append(func.__doc__) func.__doc__ = sep.join(uniq_doc_strs) return func ### doxtend_decorator() ### if bases: if _isfunction(bases[0]): f = bases[0] bases = (None,) return doxtend_decorator(f) else: return doxtend_decorator else: bases = (None,) return doxtend_decorator
def get_status(self): if hasattr(self, '_status') and self._status is not None: return self._status if(self._terminated): return self._status # if pid still exists then return 'Running' # else if check process exit code in exec dir if _psutil.pid_exists(self._pid): proc = _psutil.Process(pid=self._pid) if _isfunction(proc.status) or _ismethod(proc.status): status = str(proc.status()).capitalize() else: status = str(proc.status).capitalize() if status in ['Zombie']: return self._get_status() elif status in ['Sleeping', 'Disk-sleep']: return 'Running' else: return status else: return self._get_status()
def set_code(self, code, function_name='run', directory=None, pattern="*.py"): """ Set the code block to run when Task is executed. The code to be run needs to be a function that takes one argument. When this function is called, the argument will be an object with three dictionary attributes (inputs, outputs, params). The inputs dictionary will have instantiated data sources by name. The outputs dictionary needs to be assigned by name to the results to save. The params dictionary allows capturing any additional keys/values with the Task. Parameters ---------- code : function | string Function to be called when this Task is executed. This can either be a defined function or it can be a full file path. As a file path this method will inspect the file and confirm it has a function named according to the parameter function_name, which must take only one argument. This function will then also take, by default, the working directory of that file and run find_packages() on it, collecting all the source files indicated. If code is set to None, then this Task's code is removed. function_name : str, optional Name of the function to call. Default value is 'run'. directory : os.path, optional Optional parameter for the directory from which to grab the pattern of files. pattern : str, optional Optional parameter for a "glob" pattern (ex. *.py) to specify the subset of files from which code should be captured. By default, all .py files from the same directory as the file containing the specified function are captured. This pattern is matched using ``fnmatch``, so rules from fnmatch documentation apply. Returns ------- self : Task Examples -------- Using a defined function: >>> def func(task): >>> input = task.inputs['input'] >>> task.outputs['output'] = input.apply(lambda x : x * 2) >>> t1 = graphlab.deploy.Task("set_code_ex1") >>> t1.set_code(func) Another example, with different named tasks: >>> t2 = graphlab.deploy.Task('set_code_ex2') >>> t2.set_inputs(['one', 'two', 'three']) >>> t2.set_outputs(['result_one', 'result_two']) >>> def func(task): >>> one = task.inputs['one'] >>> two = task.inputs['two'] >>> three = task.inputs['three'] >>> ... >>> task.outputs['result_one'] = first_result >>> task.outputs['result_two'] = second_result The example below illustrates how to specify code in a file: >>> t3 = graphlab.deploy.Task('set_code_ex3') >>> t3.set_code('/full/path/to/munge.py') Using a file with a custom fuction name, and custom directory and pattern to include: >>> t4 = graphlab.deploy.Task('set_code_ex4') >>> t4.set_code('/full/path/to/transform.py', function_name='to_csv', >>> directory='src', pattern='*.py') """ if code is None: keys = ['code', 'codestr', 'funcname', 'filename'] self._data = Task.__remove_unwanted_keys(self._data, keys) self._data['code'] = None return self if _isfunction(code): # code is callable, so store it as is self._data['code'] = code try: self._data['codestr'] = _inspect.getsource(code) except IOError: pass elif _inspect.ismethod(code): raise TypeError("Code cannot be an instance method, please use a function or a file") else: # specifying code using file path, and not function code = _os.path.expanduser(code) if not _os.path.exists(code) and directory is not None: if not _os.path.exists(_os.path.join(directory, code)): raise TypeError("Unable to find the full path for the code file specified, please verify the full path to the file to use. Location searched: %s" % _os.path.join(directory, code)) else: code = _os.path.join(directory, code) if _os.path.exists(_os.path.abspath(code)): # code is a file path to a python file, capture the source etc if directory is None: directory = _os.path.join(_os.path.dirname(_os.path.abspath(code))) # confirm that function exists in file provided use getattr and try to load the function in the module module = _imp.load_source('', code) if not hasattr(module, function_name): raise TypeError("Unable to find function name \"%s\" from the code file spcified, please verify the function name and the code file. Code file searched: %s" % (function_name, code)) self._data['codestr'] = _inspect.getsource(getattr(module, function_name)) files = {} for f in Task.__find_files(directory, pattern): with open(f, 'r') as src: rel_path = _os.path.relpath(f, start=directory) files[rel_path] = src.read() self._data['funcname'] = function_name self._data['filename'] = _os.path.basename(code) # now all source files are referred to dictionary of files with filename self._data['code'] = files else: raise TypeError("Unable to find the full path for the code file specified, please verify the path is correct. Path searched: %s" % code) self._set_dirty_bit() return self
def rundict(self, d, name, module=None): """ d, name, module=None -> search for docstring examples in d.values(). For k, v in d.items() such that v is a function or class, do self.rundoc(v, name + "." + k). Whether this includes objects with private names depends on the constructor's "isprivate" argument. If module is specified, functions and classes that are not defined in module are excluded. Return aggregate (#failures, #examples). Build and populate two modules with sample functions to test that exclusion of external functions and classes works. >>> import new >>> m1 = new.module('_m1') >>> m2 = new.module('_m2') >>> test_data = \""" ... def _f(): ... '''>>> assert 1 == 1 ... ''' ... def g(): ... '''>>> assert 2 != 1 ... ''' ... class H: ... '''>>> assert 2 > 1 ... ''' ... def bar(self): ... '''>>> assert 1 < 2 ... ''' ... \""" >>> exec test_data in m1.__dict__ >>> exec test_data in m2.__dict__ >>> m1.__dict__.update({"f2": m2._f, "g2": m2.g, "h2": m2.H}) Tests that objects outside m1 are excluded: >>> t = Tester(globs={}, verbose=0) >>> t.rundict(m1.__dict__, "rundict_test", m1) # _f, f2 and g2 and h2 skipped (0, 3) Again, but with a custom isprivate function allowing _f: >>> t = Tester(globs={}, verbose=0, isprivate=lambda x,y: 0) >>> t.rundict(m1.__dict__, "rundict_test_pvt", m1) # Only f2, g2 and h2 skipped (0, 4) And once more, not excluding stuff outside m1: >>> t = Tester(globs={}, verbose=0, isprivate=lambda x,y: 0) >>> t.rundict(m1.__dict__, "rundict_test_pvt") # None are skipped. (0, 8) The exclusion of objects from outside the designated module is meant to be invoked automagically by testmod. >>> testmod(m1) (0, 3) """ if not hasattr(d, "items"): raise TypeError("Tester.rundict: d must support .items(); " + ` d `) f = t = 0 # Run the tests by alpha order of names, for consistency in # verbose-mode output. names = d.keys() names.sort() for thisname in names: value = d[thisname] if _isfunction(value) or _isclass(value): if module and not _from_module(module, value): continue f2, t2 = self.__runone(value, name + "." + thisname) f = f + f2 t = t + t2 return f, t
def _from_module(module, object): if _isfunction(object): return module.__dict__ is object.func_globals if _isclass(object): return module.__name__ == object.__module__ raise ValueError("object must be a class or function")
def rundict(self, d, name, module=None): """ d, name, module=None -> search for docstring examples in d.values(). For k, v in d.items() such that v is a function or class, do self.rundoc(v, name + "." + k). Whether this includes objects with private names depends on the constructor's "isprivate" argument. If module is specified, functions and classes that are not defined in module are excluded. Return aggregate (#failures, #examples). Build and populate two modules with sample functions to test that exclusion of external functions and classes works. >>> import new >>> m1 = new.module('_m1') >>> m2 = new.module('_m2') >>> test_data = \""" ... def _f(): ... '''>>> assert 1 == 1 ... ''' ... def g(): ... '''>>> assert 2 != 1 ... ''' ... class H: ... '''>>> assert 2 > 1 ... ''' ... def bar(self): ... '''>>> assert 1 < 2 ... ''' ... \""" >>> exec test_data in m1.__dict__ >>> exec test_data in m2.__dict__ >>> m1.__dict__.update({"f2": m2._f, "g2": m2.g, "h2": m2.H}) Tests that objects outside m1 are excluded: >>> t = Tester(globs={}, verbose=0) >>> t.rundict(m1.__dict__, "rundict_test", m1) # _f, f2 and g2 and h2 skipped (0, 3) Again, but with a custom isprivate function allowing _f: >>> t = Tester(globs={}, verbose=0, isprivate=lambda x,y: 0) >>> t.rundict(m1.__dict__, "rundict_test_pvt", m1) # Only f2, g2 and h2 skipped (0, 4) And once more, not excluding stuff outside m1: >>> t = Tester(globs={}, verbose=0, isprivate=lambda x,y: 0) >>> t.rundict(m1.__dict__, "rundict_test_pvt") # None are skipped. (0, 8) The exclusion of objects from outside the designated module is meant to be invoked automagically by testmod. >>> testmod(m1) (0, 3) """ if not hasattr(d, "items"): raise TypeError("Tester.rundict: d must support .items(); " + `d`) f = t = 0 # Run the tests by alpha order of names, for consistency in # verbose-mode output. names = d.keys() names.sort() for thisname in names: value = d[thisname] if _isfunction(value) or _isclass(value): if module and not _from_module(module, value): continue f2, t2 = self.__runone(value, name + "." + thisname) f = f + f2 t = t + t2 return f, t
def _should_be_wrapped(member): name, obj = member return _is_exported_name(name) and _isfunction(obj) and _getmodule( obj) == _check
# Module doctest.
def autoinit(*decoargs, **decokwargs): ''' Decorator for automatic initialization instance attributes @autoinit def __init__(self, a, b=10): pass is equivalent to def __init__(self, a, b=10): self.a = a self.b = b The decorator can be equally applied to both the __init__ method and the entire class. Options: exclude: str or iterable of strs # skip specified attributes no_warn: bool = False # do not warn when decorator applied to not __init__, reverse: bool = False # call wrapped method before the assignment ''' reverse = decokwargs.get('reverse', False) no_warn = decokwargs.get('no_warn', False) exclude = decokwargs.get('exclude', []) if version_info.major > 2: unicode = str else: unicode = type(u"") acceptable_str_types = (str, unicode) if isinstance(exclude, acceptable_str_types): exclude = [exclude] def inner_decorator(init_or_class): if _isclass(init_or_class): func = getattr(init_or_class, '__init__') elif _isfunction(init_or_class): func = init_or_class else: raise ValueError("autoinit decorator should be applied to class or its __init__ method") if (func.__name__ != '__init__' or func.__code__.co_name != '__init__') and not no_warn: _warn(AutoinitWarning("autoinit decorator intended to be applied only to __init__" " method (use autoinit(no_warn=True) to suppress this warning)")) args_names = func.__code__.co_varnames[1:func.__code__.co_argcount] @_wraps(func) def inner(self, *args, **kwargs): if reverse: func(self, *args, **kwargs) args_vals = args[:] if func.__defaults__: args_vals += func.__defaults__[len(args) - len(args_names):] for key, val in zip(args_names, args_vals): if key not in exclude: if (type(self.__class__).__name__ != 'classobj' and hasattr(self, '__slots__') and key not in self.__slots__): raise AttributeError("Can not assign attribute '%s': it is not " "listed in %s.__slots__" % (key, self.__class__)) setattr(self, key, val) if not reverse: func(self, *args, **kwargs) if _isclass(init_or_class): init_or_class.__init__ = inner return init_or_class return inner if decoargs and (_isfunction(decoargs[0]) or _isclass(decoargs[0])): return inner_decorator(decoargs[0]) return inner_decorator