def run__test__(self, d, name):
"""d, name -> Treat dict d like module.__test__.
Return (#failures, #tries).
See testmod.__doc__ for details.
"""
failures = tries = 0
prefix = name + "."
savepvt = self.isprivate
try:
self.isprivate = lambda *args: 0
# Run the tests by alpha order of names, for consistency in
# verbose-mode output.
keys = d.keys()
keys.sort()
for k in keys:
v = d[k]
thisname = prefix + k
if type(v) in _StringTypes:
f, t = self.runstring(v, thisname)
elif _isfunction(v) or _isclass(v):
f, t = self.rundoc(v, thisname)
else:
raise TypeError("Tester.run__test__: values in "
"dict must be strings, functions "
"or classes; " + `v`)
failures = failures + f
tries = tries + t
finally:
self.isprivate = savepvt
return failures, tries
def run__test__(self, d, name):
"""d, name -> Treat dict d like module.__test__.
Return (#failures, #tries).
See testmod.__doc__ for details.
"""
failures = tries = 0
prefix = name + "."
savepvt = self.isprivate
try:
self.isprivate = lambda *args: 0
# Run the tests by alpha order of names, for consistency in
# verbose-mode output.
keys = d.keys()
keys.sort()
for k in keys:
v = d[k]
thisname = prefix + k
if type(v) in _StringTypes:
f, t = self.runstring(v, thisname)
elif _isfunction(v) or _isclass(v):
f, t = self.rundoc(v, thisname)
else:
raise TypeError("Tester.run__test__: values in "
"dict must be strings, functions "
"or classes; " + ` v `)
failures = failures + f
tries = tries + t
finally:
self.isprivate = savepvt
return failures, tries
def inner_decorator(init_or_class):
if _isclass(init_or_class):
func = getattr(init_or_class, '__init__')
elif _isfunction(init_or_class):
func = init_or_class
else:
raise ValueError("autoinit decorator should be applied to class or its __init__ method")
if (func.__name__ != '__init__' or func.__code__.co_name != '__init__') and not no_warn:
_warn(AutoinitWarning("autoinit decorator intended to be applied only to __init__"
" method (use autoinit(no_warn=True) to suppress this warning)"))
args_names = func.__code__.co_varnames[1:func.__code__.co_argcount]
@_wraps(func)
def inner(self, *args, **kwargs):
if reverse:
func(self, *args, **kwargs)
args_vals = args[:]
if func.__defaults__:
args_vals += func.__defaults__[len(args) - len(args_names):]
for key, val in zip(args_names, args_vals):
if key not in exclude:
if (type(self.__class__).__name__ != 'classobj' and
hasattr(self, '__slots__') and key not in self.__slots__):
raise AttributeError("Can not assign attribute '%s': it is not "
"listed in %s.__slots__" % (key, self.__class__))
setattr(self, key, val)
if not reverse:
func(self, *args, **kwargs)
if _isclass(init_or_class):
init_or_class.__init__ = inner
return init_or_class
return inner
def args(arguments: _split=argv[1:], stdin_char: str='-',
**mappings):
"""Parses the cmd-arguments to execute the requested command.
:param arguments: The arguments to use (default: argv[1:]).
:param stdin_char: The char to denote stdin-input. Use
None to disable (default: '-').
:param mappings: Additional/custom mappings
(i.e. {'command': function}).
"""
# getting arguments
try:
if stdin_char is not None and arguments[-1] == stdin_char:
del arguments[-1]
while True:
try:
i = input()
arguments.append(i)
except EOFError:
break
f, *a = arguments
except (ValueError, IndexError):
print("ERROR: Requires function or --help!", file=stderr)
exit(1)
# get caller
module = _get_calling_module()
# getting functions
funcs = {k: v for k, v in vars(module).items()
if not k.startswith('_') and _isfunction(v)}
funcs['--help'] = funcs['-h'] = funcs['help'] = usage
funcs['--list-commands'] = commands
funcs.update(mappings) # use defined mappings
if f not in funcs:
raise Exception("Invalid function '{}'.".format(f))
# getting this function
ff = funcs[f]
# converting arguments
fspecs = _getargs(ff)
for i, (val, name) in enumerate(zip(a, fspecs.args)):
if name in fspecs.annotations:
a[i] = fspecs.annotations[name](val)
if fspecs.varargs:
l = len(fspecs.args)
vals = a[l:]
if fspecs.varargs in fspecs.annotations:
a[l:] = fspecs.annotations[fspecs.varargs](vals)
# executing
if ff in {usage, commands}:
ff(globalvars=funcs)
else:
return ff(*a)
def __to_dict__(self):
params = self.get_params()
param_dict = {}
isFunction = False
for k,v in params.iteritems():
param_dict[k] = v
if _isfunction(v):
param_dict[k] = v.func_name
isFunction = True
task_json = {}
task_json["name"] = self.get_name()
task_json["description"] = self.get_description()
task_json["inputs"] = self.get_inputs()
task_json["code"] = self.__code_snapshot__()
task_json["outputs"] = self.get_outputs()
task_json["params"] = param_dict
task_json["isFunction"] = isFunction
task_json["required_packages"] = list(self.get_required_packages())
return task_json
def doxtend(*bases, func_name=None, sep='\n\n'):
""" Inherit doc-string(s) from other functions/methods.
bases: base objects to search in, 'None' looks in globals()
func_name: alternative name to search for(defaults to name of decorated function)
sep: str to place between inherited docstring
Attempts to order doc-strings in a cummulative/hierarchical fashion - moving
left to right through 'bases' - excluding the same string(s) from different
objects(except for the final (non)derived doc-string which is guaranteed to
be included last)"
"""
def doxtend_decorator(func):
nonlocal bases, func_name
uniq_doc_strs = []
if func_name is None:
func_name = func.__name__
for bc in bases:
try:
f = globals()[func_name] if bc is None else getattr(bc,func_name)
except (KeyError, AttributeError):
raise DoxtendError("couldn't find function '" + func_name + "'")
if f.__doc__ not in uniq_doc_strs:
uniq_doc_strs.append(f.__doc__)
if uniq_doc_strs:
if func.__doc__:
uniq_doc_strs.append(func.__doc__)
func.__doc__ = sep.join(uniq_doc_strs)
return func
### doxtend_decorator() ###
if bases:
if _isfunction(bases[0]):
f = bases[0]
bases = (None,)
return doxtend_decorator(f)
else:
return doxtend_decorator
else:
bases = (None,)
return doxtend_decorator
def get_status(self):
if hasattr(self, '_status') and self._status is not None:
return self._status
if(self._terminated):
return self._status
# if pid still exists then return 'Running'
# else if check process exit code in exec dir
if _psutil.pid_exists(self._pid):
proc = _psutil.Process(pid=self._pid)
if _isfunction(proc.status) or _ismethod(proc.status):
status = str(proc.status()).capitalize()
else:
status = str(proc.status).capitalize()
if status in ['Zombie']:
return self._get_status()
elif status in ['Sleeping', 'Disk-sleep']:
return 'Running'
else:
return status
else:
return self._get_status()
def set_code(self, code, function_name='run', directory=None, pattern="*.py"):
"""
Set the code block to run when Task is executed.
The code to be run needs to be a function that takes one argument. When
this function is called, the argument will be an object with three
dictionary attributes (inputs, outputs, params). The inputs dictionary will
have instantiated data sources by name. The outputs dictionary needs to
be assigned by name to the results to save. The params dictionary
allows capturing any additional keys/values with the Task.
Parameters
----------
code : function | string
Function to be called when this Task is executed. This can
either be a defined function or it can be a full file path. As a
file path this method will inspect the file and confirm it has a
function named according to the parameter function_name, which must
take only one argument. This function will then also take, by
default, the working directory of that file and run find_packages()
on it, collecting all the source files indicated. If code is set to
None, then this Task's code is removed.
function_name : str, optional
Name of the function to call. Default value is 'run'.
directory : os.path, optional
Optional parameter for the directory from which to grab the pattern
of files.
pattern : str, optional
Optional parameter for a "glob" pattern (ex. *.py) to specify the
subset of files from which code should be captured. By default, all
.py files from the same directory as the file containing the
specified function are captured. This pattern is matched using
``fnmatch``, so rules from fnmatch documentation apply.
Returns
-------
self : Task
Examples
--------
Using a defined function:
>>> def func(task):
>>> input = task.inputs['input']
>>> task.outputs['output'] = input.apply(lambda x : x * 2)
>>> t1 = graphlab.deploy.Task("set_code_ex1")
>>> t1.set_code(func)
Another example, with different named tasks:
>>> t2 = graphlab.deploy.Task('set_code_ex2')
>>> t2.set_inputs(['one', 'two', 'three'])
>>> t2.set_outputs(['result_one', 'result_two'])
>>> def func(task):
>>> one = task.inputs['one']
>>> two = task.inputs['two']
>>> three = task.inputs['three']
>>> ...
>>> task.outputs['result_one'] = first_result
>>> task.outputs['result_two'] = second_result
The example below illustrates how to specify code in a file:
>>> t3 = graphlab.deploy.Task('set_code_ex3')
>>> t3.set_code('/full/path/to/munge.py')
Using a file with a custom fuction name, and custom directory and
pattern to include:
>>> t4 = graphlab.deploy.Task('set_code_ex4')
>>> t4.set_code('/full/path/to/transform.py', function_name='to_csv',
>>> directory='src', pattern='*.py')
"""
if code is None:
keys = ['code', 'codestr', 'funcname', 'filename']
self._data = Task.__remove_unwanted_keys(self._data, keys)
self._data['code'] = None
return self
if _isfunction(code):
# code is callable, so store it as is
self._data['code'] = code
try:
self._data['codestr'] = _inspect.getsource(code)
except IOError:
pass
elif _inspect.ismethod(code):
raise TypeError("Code cannot be an instance method, please use a function or a file")
else:
# specifying code using file path, and not function
code = _os.path.expanduser(code)
if not _os.path.exists(code) and directory is not None:
if not _os.path.exists(_os.path.join(directory, code)):
raise TypeError("Unable to find the full path for the code file specified, please verify the full path to the file to use. Location searched: %s" % _os.path.join(directory, code))
else:
code = _os.path.join(directory, code)
if _os.path.exists(_os.path.abspath(code)):
# code is a file path to a python file, capture the source etc
if directory is None:
directory = _os.path.join(_os.path.dirname(_os.path.abspath(code)))
# confirm that function exists in file provided use getattr and try to load the function in the module
module = _imp.load_source('', code)
if not hasattr(module, function_name):
raise TypeError("Unable to find function name \"%s\" from the code file spcified, please verify the function name and the code file. Code file searched: %s" % (function_name, code))
self._data['codestr'] = _inspect.getsource(getattr(module, function_name))
files = {}
for f in Task.__find_files(directory, pattern):
with open(f, 'r') as src:
rel_path = _os.path.relpath(f, start=directory)
files[rel_path] = src.read()
self._data['funcname'] = function_name
self._data['filename'] = _os.path.basename(code)
# now all source files are referred to dictionary of files with filename
self._data['code'] = files
else:
raise TypeError("Unable to find the full path for the code file specified, please verify the path is correct. Path searched: %s" % code)
self._set_dirty_bit()
return self
def rundict(self, d, name, module=None):
"""
d, name, module=None -> search for docstring examples in d.values().
For k, v in d.items() such that v is a function or class,
do self.rundoc(v, name + "." + k). Whether this includes
objects with private names depends on the constructor's
"isprivate" argument. If module is specified, functions and
classes that are not defined in module are excluded.
Return aggregate (#failures, #examples).
Build and populate two modules with sample functions to test that
exclusion of external functions and classes works.
>>> import new
>>> m1 = new.module('_m1')
>>> m2 = new.module('_m2')
>>> test_data = \"""
... def _f():
... '''>>> assert 1 == 1
... '''
... def g():
... '''>>> assert 2 != 1
... '''
... class H:
... '''>>> assert 2 > 1
... '''
... def bar(self):
... '''>>> assert 1 < 2
... '''
... \"""
>>> exec test_data in m1.__dict__
>>> exec test_data in m2.__dict__
>>> m1.__dict__.update({"f2": m2._f, "g2": m2.g, "h2": m2.H})
Tests that objects outside m1 are excluded:
>>> t = Tester(globs={}, verbose=0)
>>> t.rundict(m1.__dict__, "rundict_test", m1) # _f, f2 and g2 and h2 skipped
(0, 3)
Again, but with a custom isprivate function allowing _f:
>>> t = Tester(globs={}, verbose=0, isprivate=lambda x,y: 0)
>>> t.rundict(m1.__dict__, "rundict_test_pvt", m1) # Only f2, g2 and h2 skipped
(0, 4)
And once more, not excluding stuff outside m1:
>>> t = Tester(globs={}, verbose=0, isprivate=lambda x,y: 0)
>>> t.rundict(m1.__dict__, "rundict_test_pvt") # None are skipped.
(0, 8)
The exclusion of objects from outside the designated module is
meant to be invoked automagically by testmod.
>>> testmod(m1)
(0, 3)
"""
if not hasattr(d, "items"):
raise TypeError("Tester.rundict: d must support .items(); " +
` d `)
f = t = 0
# Run the tests by alpha order of names, for consistency in
# verbose-mode output.
names = d.keys()
names.sort()
for thisname in names:
value = d[thisname]
if _isfunction(value) or _isclass(value):
if module and not _from_module(module, value):
continue
f2, t2 = self.__runone(value, name + "." + thisname)
f = f + f2
t = t + t2
return f, t
def _from_module(module, object):
if _isfunction(object):
return module.__dict__ is object.func_globals
if _isclass(object):
return module.__name__ == object.__module__
raise ValueError("object must be a class or function")
def rundict(self, d, name, module=None):
"""
d, name, module=None -> search for docstring examples in d.values().
For k, v in d.items() such that v is a function or class,
do self.rundoc(v, name + "." + k). Whether this includes
objects with private names depends on the constructor's
"isprivate" argument. If module is specified, functions and
classes that are not defined in module are excluded.
Return aggregate (#failures, #examples).
Build and populate two modules with sample functions to test that
exclusion of external functions and classes works.
>>> import new
>>> m1 = new.module('_m1')
>>> m2 = new.module('_m2')
>>> test_data = \"""
... def _f():
... '''>>> assert 1 == 1
... '''
... def g():
... '''>>> assert 2 != 1
... '''
... class H:
... '''>>> assert 2 > 1
... '''
... def bar(self):
... '''>>> assert 1 < 2
... '''
... \"""
>>> exec test_data in m1.__dict__
>>> exec test_data in m2.__dict__
>>> m1.__dict__.update({"f2": m2._f, "g2": m2.g, "h2": m2.H})
Tests that objects outside m1 are excluded:
>>> t = Tester(globs={}, verbose=0)
>>> t.rundict(m1.__dict__, "rundict_test", m1) # _f, f2 and g2 and h2 skipped
(0, 3)
Again, but with a custom isprivate function allowing _f:
>>> t = Tester(globs={}, verbose=0, isprivate=lambda x,y: 0)
>>> t.rundict(m1.__dict__, "rundict_test_pvt", m1) # Only f2, g2 and h2 skipped
(0, 4)
And once more, not excluding stuff outside m1:
>>> t = Tester(globs={}, verbose=0, isprivate=lambda x,y: 0)
>>> t.rundict(m1.__dict__, "rundict_test_pvt") # None are skipped.
(0, 8)
The exclusion of objects from outside the designated module is
meant to be invoked automagically by testmod.
>>> testmod(m1)
(0, 3)
"""
if not hasattr(d, "items"):
raise TypeError("Tester.rundict: d must support .items(); " +
`d`)
f = t = 0
# Run the tests by alpha order of names, for consistency in
# verbose-mode output.
names = d.keys()
names.sort()
for thisname in names:
value = d[thisname]
if _isfunction(value) or _isclass(value):
if module and not _from_module(module, value):
continue
f2, t2 = self.__runone(value, name + "." + thisname)
f = f + f2
t = t + t2
return f, t
def _from_module(module, object):
if _isfunction(object):
return module.__dict__ is object.func_globals
if _isclass(object):
return module.__name__ == object.__module__
raise ValueError("object must be a class or function")
def _should_be_wrapped(member):
name, obj = member
return _is_exported_name(name) and _isfunction(obj) and _getmodule(
obj) == _check
# Module doctest.
def autoinit(*decoargs, **decokwargs):
'''
Decorator for automatic initialization instance attributes
@autoinit
def __init__(self, a, b=10):
pass
is equivalent to
def __init__(self, a, b=10):
self.a = a
self.b = b
The decorator can be equally applied to both the __init__ method and the entire class.
Options:
exclude: str or iterable of strs # skip specified attributes
no_warn: bool = False # do not warn when decorator applied to not __init__,
reverse: bool = False # call wrapped method before the assignment
'''
reverse = decokwargs.get('reverse', False)
no_warn = decokwargs.get('no_warn', False)
exclude = decokwargs.get('exclude', [])
if version_info.major > 2:
unicode = str
else:
unicode = type(u"")
acceptable_str_types = (str, unicode)
if isinstance(exclude, acceptable_str_types):
exclude = [exclude]
def inner_decorator(init_or_class):
if _isclass(init_or_class):
func = getattr(init_or_class, '__init__')
elif _isfunction(init_or_class):
func = init_or_class
else:
raise ValueError("autoinit decorator should be applied to class or its __init__ method")
if (func.__name__ != '__init__' or func.__code__.co_name != '__init__') and not no_warn:
_warn(AutoinitWarning("autoinit decorator intended to be applied only to __init__"
" method (use autoinit(no_warn=True) to suppress this warning)"))
args_names = func.__code__.co_varnames[1:func.__code__.co_argcount]
@_wraps(func)
def inner(self, *args, **kwargs):
if reverse:
func(self, *args, **kwargs)
args_vals = args[:]
if func.__defaults__:
args_vals += func.__defaults__[len(args) - len(args_names):]
for key, val in zip(args_names, args_vals):
if key not in exclude:
if (type(self.__class__).__name__ != 'classobj' and
hasattr(self, '__slots__') and key not in self.__slots__):
raise AttributeError("Can not assign attribute '%s': it is not "
"listed in %s.__slots__" % (key, self.__class__))
setattr(self, key, val)
if not reverse:
func(self, *args, **kwargs)
if _isclass(init_or_class):
init_or_class.__init__ = inner
return init_or_class
return inner
if decoargs and (_isfunction(decoargs[0]) or _isclass(decoargs[0])):
return inner_decorator(decoargs[0])
return inner_decorator
# Module doctest.
