def __init__(self,
description=None,
threads=1,
input_files=(),
output_files=(),
executables=(),
auxiliary_files=(),
requirements=(),
subnodes=(),
dependencies=()):
if not isinstance(description, _DESC_TYPES):
raise TypeError("'description' must be None or a string, not %r" \
% (description.__class__.__name__,))
self.__description = description
self.input_files = self._validate_files(input_files)
self.output_files = self._validate_files(output_files)
self.executables = self._validate_files(executables)
self.auxiliary_files = self._validate_files(auxiliary_files)
self.requirements = self._validate_requirements(requirements)
self.subnodes = frozenset()
self.dependencies = frozenset()
self.threads = self._validate_nthreads(threads)
try:
# Ensure that the node can be used in a multiprocessing context
fast_pickle_test(self)
except pickle.PicklingError, error:
raise NodeError(
"Node could not be pickled, please file a bug-report:\n"
"\tNode: %s\n\tError: %s" % (self, error))
def _start_new_tasks(self, remaining, running, nodegraph, max_threads,
pool):
started_nodes = []
idle_processes = max_threads \
- sum(node.threads for (node, _) in running.itervalues())
for node in remaining:
if not running or (idle_processes >= node.threads):
state = nodegraph.get_node_state(node)
if state == nodegraph.RUNABLE:
try:
# The multi-processing module relies on pickling
fast_pickle_test(node)
except pickle.PicklingError, error:
self._logger.error("Node cannot be pickled; please "
"file a bug-report:\n"
"\tNode: %s\n\tError: %s"
% (self, error))
nodegraph.set_node_state(node, nodegraph.ERROR)
started_nodes.append(node)
continue
key = id(node)
proc_args = (key, node, self._config)
running[key] = (node, pool.apply_async(_call_run,
args=proc_args))
started_nodes.append(node)
nodegraph.set_node_state(node, nodegraph.RUNNING)
idle_processes -= node.threads
elif state in (nodegraph.DONE, nodegraph.ERROR):
started_nodes.append(node)
elif idle_processes <= 0:
break
def __init__(self, description = None, threads = 1,
input_files = (), output_files = (),
executables = (), auxiliary_files = (),
requirements = (), subnodes = (), dependencies = ()):
if not isinstance(description, _DESC_TYPES):
raise TypeError("'description' must be None or a string, not %r" \
% (description.__class__.__name__,))
self.__description = description
self.input_files = self._validate_files(input_files)
self.output_files = self._validate_files(output_files)
self.executables = self._validate_files(executables)
self.auxiliary_files = self._validate_files(auxiliary_files)
self.requirements = self._validate_requirements(requirements)
self.subnodes = frozenset()
self.dependencies = frozenset()
self.threads = self._validate_nthreads(threads)
try:
# Ensure that the node can be used in a multiprocessing context
fast_pickle_test(self)
except pickle.PicklingError, error:
raise NodeError("Node could not be pickled, please file a bug-report:\n"
"\tNode: %s\n\tError: %s" % (self, error))
def _start_new_tasks(self, remaining, running, nodegraph, max_threads,
pool):
started_nodes = []
idle_processes = max_threads \
- sum(node.threads for (node, _) in running.itervalues())
if not idle_processes:
return False
for node in remaining:
if not running or (idle_processes >= node.threads):
state = nodegraph.get_node_state(node)
if state == nodegraph.RUNABLE:
try:
# The multi-processing module relies on pickling
fast_pickle_test(node)
except pickle.PicklingError, error:
self._logger.error("Node cannot be pickled; please "
"file a bug-report:\n"
"\tNode: %s\n\tError: %s" %
(self, error))
nodegraph.set_node_state(node, nodegraph.ERROR)
started_nodes.append(node)
continue
key = id(node)
proc_args = (key, node, self._config)
running[key] = (node,
pool.apply_async(_call_run,
args=proc_args))
started_nodes.append(node)
nodegraph.set_node_state(node, nodegraph.RUNNING)
idle_processes -= node.threads
elif state in (nodegraph.DONE, nodegraph.ERROR):
started_nodes.append(node)
elif idle_processes <= 0:
break
def test_fast_pickle_test__unpicklable_2():
def _func():
return None # pragma: no coverage
utils.fast_pickle_test(_func)
def test_fast_pickle_test__unpicklable_1():
_func = lambda: None # pragma: no coverage
utils.fast_pickle_test(_func)
def test_fast_pickle_test__picklable():
utils.fast_pickle_test(1)
utils.fast_pickle_test({})
utils.fast_pickle_test(test_cumsum__empty)
def test_fast_pickle_test__unpicklable_2():
def _func():
return None # pragma: no coverage
utils.fast_pickle_test(_func)
def test_fast_pickle_test__unpicklable_1():
_func = lambda: None # pragma: no coverage
utils.fast_pickle_test(_func)
def test_fast_pickle_test__picklable():
utils.fast_pickle_test(1)
utils.fast_pickle_test({})
utils.fast_pickle_test(test_cumsum__empty)