def ensure_atom(self, atom):
"""Ensure that there is an atomdetail in storage for the given atom.
Returns uuid for the atomdetail that is/was created.
"""
if isinstance(atom, task.BaseTask):
return self._ensure_task(atom.name, misc.get_version_string(atom),
atom.save_as)
elif isinstance(atom, retry.Retry):
return self._ensure_retry(atom.name, misc.get_version_string(atom),
atom.save_as)
else:
raise TypeError("Object of type 'atom' expected not"
" '%s' (%s)" % (atom, type(atom)))
def ensure_atom(self, atom):
"""Ensure that there is an atomdetail in storage for the given atom.
Returns uuid for the atomdetail that is/was created.
"""
if isinstance(atom, task.BaseTask):
return self._ensure_task(atom.name,
misc.get_version_string(atom),
atom.save_as)
elif isinstance(atom, retry.Retry):
return self._ensure_retry(atom.name,
misc.get_version_string(atom),
atom.save_as)
else:
raise TypeError("Object of type 'atom' expected."
" Got %s, %r." % (type(atom), atom))
def compile(self):
"""Compiles the contained flow into a structure which the engine can
use to run or if this can not be done then an exception is thrown
indicating why this compilation could not be achieved.
"""
if self._root is not None:
return
assert self._graph_action is not None, ('Graph action class must be'
' specified')
self._change_state(states.RESUMING) # does nothing in PENDING state
task_graph = flow_utils.flatten(self._flow)
if task_graph.number_of_nodes() == 0:
raise exc.EmptyFlow("Flow %s is empty." % self._flow.name)
self._root = self._graph_action(task_graph)
for task in task_graph.nodes_iter():
try:
task_id = self.storage.get_uuid_by_name(task.name)
except exc.NotFound:
task_id = uuidutils.generate_uuid()
task_version = misc.get_version_string(task)
self.storage.add_task(task_name=task.name, uuid=task_id,
task_version=task_version)
self.storage.set_result_mapping(task_id, task.save_as)
self._root.add(task, task_action.TaskAction(task, task_id))
self._change_state(states.SUSPENDED) # does nothing in PENDING state
def _ensure_storage_for(self, task_graph):
# NOTE(harlowja): signal to the tasks that exist that we are about to
# resume, if they have a previous state, they will now transition to
# a resuming state (and then to suspended).
self._change_state(states.RESUMING) # does nothing in PENDING state
for task in task_graph.nodes_iter():
task_version = misc.get_version_string(task)
self.storage.ensure_task(task.name, task_version, task.save_as)
self._change_state(states.SUSPENDED) # does nothing in PENDING state
def _ensure_storage_for(self, task_graph):
# NOTE(harlowja): signal to the tasks that exist that we are about to
# resume, if they have a previous state, they will now transition to
# a resuming state (and then to suspended).
self._change_state(states.RESUMING) # does nothing in PENDING state
for task in task_graph.nodes_iter():
task_version = misc.get_version_string(task)
self.storage.ensure_task(task.name, task_version, task.save_as)
self._change_state(states.SUSPENDED) # does nothing in PENDING state
def pformat(self, indent=0, linesep=os.linesep):
"""Pretty formats this atom detail into a string."""
cls_name = self.__class__.__name__
lines = ["%s%s: '%s'" % (" " * (indent), cls_name, self.name)]
lines.extend(_format_shared(self, indent=indent + 1))
lines.append("%s- version = %s" % (" " * (indent + 1), misc.get_version_string(self)))
lines.append("%s- results = %s" % (" " * (indent + 1), self.results))
lines.append("%s- failure = %s" % (" " * (indent + 1), bool(self.failure)))
lines.extend(_format_meta(self.meta, indent=indent + 1))
return linesep.join(lines)
def ensure_atoms(self, atoms):
"""Ensure there is an atomdetail for **each** of the given atoms.
Returns list of atomdetail uuids for each atom processed.
"""
atom_ids = []
missing_ads = []
for i, atom in enumerate(atoms):
match = misc.match_type(atom, self._ensure_matchers)
if not match:
raise TypeError("Unknown atom '%s' (%s) requested to ensure" %
(atom, type(atom)))
atom_detail_cls, kind = match
atom_name = atom.name
if not atom_name:
raise ValueError("%s name must be non-empty" % (kind))
try:
atom_id = self._atom_name_to_uuid[atom_name]
except KeyError:
missing_ads.append((i, atom, atom_detail_cls))
# This will be later replaced with the uuid that is created...
atom_ids.append(None)
else:
ad = self._flowdetail.find(atom_id)
if not isinstance(ad, atom_detail_cls):
raise exceptions.Duplicate(
"Atom detail '%s' already exists in flow"
" detail '%s'" % (atom_name, self._flowdetail.name))
else:
atom_ids.append(ad.uuid)
self._set_result_mapping(atom_name, atom.save_as)
if missing_ads:
needs_to_be_created_ads = []
for (i, atom, atom_detail_cls) in missing_ads:
ad = self._create_atom_detail(
atom.name,
atom_detail_cls,
atom_version=misc.get_version_string(atom))
needs_to_be_created_ads.append((i, atom, ad))
# Add the atom detail(s) to a clone, which upon success will be
# updated into the contained flow detail; if it does not get saved
# then no update will happen.
source, clone = self._fetch_flowdetail(clone=True)
for (_i, _atom, ad) in needs_to_be_created_ads:
clone.add(ad)
self._with_connection(self._save_flow_detail, source, clone)
# Insert the needed data, and get outta here...
for (i, atom, ad) in needs_to_be_created_ads:
atom_name = atom.name
atom_ids[i] = ad.uuid
self._atom_name_to_uuid[atom_name] = ad.uuid
self._set_result_mapping(atom_name, atom.save_as)
self._failures.setdefault(atom_name, {})
return atom_ids
def pformat_task_detail(task_detail, indent=0):
"""Pretty formats a task detail."""
lines = ["%sTask: '%s'" % (" " * (indent), task_detail.name)]
lines.extend(_format_shared(task_detail, indent=indent + 1))
lines.append("%s- version = %s" %
(" " * (indent + 1), misc.get_version_string(task_detail)))
lines.append("%s- results = %s" % (" " *
(indent + 1), task_detail.results))
lines.append("%s- failure = %s" %
(" " * (indent + 1), bool(task_detail.failure)))
lines.extend(_format_meta(task_detail.meta, indent=indent + 1))
return "\n".join(lines)
def ensure_atoms(self, atoms):
"""Ensure there is an atomdetail for **each** of the given atoms.
Returns list of atomdetail uuids for each atom processed.
"""
atom_ids = []
missing_ads = []
for i, atom in enumerate(atoms):
match = misc.match_type(atom, self._ensure_matchers)
if not match:
raise TypeError("Unknown atom '%s' (%s) requested to ensure"
% (atom, type(atom)))
atom_detail_cls, kind = match
atom_name = atom.name
if not atom_name:
raise ValueError("%s name must be non-empty" % (kind))
try:
atom_id = self._atom_name_to_uuid[atom_name]
except KeyError:
missing_ads.append((i, atom, atom_detail_cls))
# This will be later replaced with the uuid that is created...
atom_ids.append(None)
else:
ad = self._flowdetail.find(atom_id)
if not isinstance(ad, atom_detail_cls):
raise exceptions.Duplicate(
"Atom detail '%s' already exists in flow"
" detail '%s'" % (atom_name, self._flowdetail.name))
else:
atom_ids.append(ad.uuid)
self._set_result_mapping(atom_name, atom.save_as)
if missing_ads:
needs_to_be_created_ads = []
for (i, atom, atom_detail_cls) in missing_ads:
ad = self._create_atom_detail(
atom.name, atom_detail_cls,
atom_version=misc.get_version_string(atom))
needs_to_be_created_ads.append((i, atom, ad))
# Add the atom detail(s) to a clone, which upon success will be
# updated into the contained flow detail; if it does not get saved
# then no update will happen.
source, clone = self._fetch_flowdetail(clone=True)
for (_i, _atom, ad) in needs_to_be_created_ads:
clone.add(ad)
self._with_connection(self._save_flow_detail, source, clone)
# Insert the needed data, and get outta here...
for (i, atom, ad) in needs_to_be_created_ads:
atom_name = atom.name
atom_ids[i] = ad.uuid
self._atom_name_to_uuid[atom_name] = ad.uuid
self._set_result_mapping(atom_name, atom.save_as)
self._failures.setdefault(atom_name, {})
return atom_ids
def pformat_task_detail(task_detail, indent=0):
"""Pretty formats a task detail"""
lines = ["%sTask: '%s'" % (" " * (indent), task_detail.name)]
lines.extend(_format_shared(task_detail, indent=indent + 1))
lines.append("%s- version = %s"
% (" " * (indent + 1), misc.get_version_string(task_detail)))
lines.append("%s- results = %s"
% (" " * (indent + 1), task_detail.results))
lines.append("%s- failure = %s" % (" " * (indent + 1),
bool(task_detail.failure)))
lines.extend(_format_meta(task_detail.meta, indent=indent + 1))
return "\n".join(lines)
def pformat(self, indent=0, linesep=os.linesep):
"""Pretty formats this atom detail into a string."""
cls_name = self.__class__.__name__
lines = ["%s%s: '%s'" % (" " * (indent), cls_name, self.name)]
lines.extend(_format_shared(self, indent=indent + 1))
lines.append("%s- version = %s" %
(" " * (indent + 1), misc.get_version_string(self)))
lines.append("%s- results = %s" % (" " * (indent + 1), self.results))
lines.append("%s- failure = %s" % (" " *
(indent + 1), bool(self.failure)))
lines.extend(_format_meta(self.meta, indent=indent + 1))
return linesep.join(lines)
def ensure_atom(self, atom):
"""Ensure that there is an atomdetail in storage for the given atom.
Returns uuid for the atomdetail that is/was created.
"""
functor = misc.match_type_handler(atom, self._ensure_matchers)
if not functor:
raise TypeError("Unknown item '%s' (%s) requested to ensure" %
(atom, type(atom)))
else:
return functor(atom.name, misc.get_version_string(atom),
atom.save_as)
def pformat_atom_detail(atom_detail, indent=0):
"""Pretty formats a atom detail."""
detail_type = logbook.atom_detail_type(atom_detail)
lines = ["%s%s: '%s'" % (" " * (indent), detail_type, atom_detail.name)]
lines.extend(_format_shared(atom_detail, indent=indent + 1))
lines.append("%s- version = %s" %
(" " * (indent + 1), misc.get_version_string(atom_detail)))
lines.append("%s- results = %s" % (" " *
(indent + 1), atom_detail.results))
lines.append("%s- failure = %s" %
(" " * (indent + 1), bool(atom_detail.failure)))
lines.extend(_format_meta(atom_detail.meta, indent=indent + 1))
return "\n".join(lines)
def ensure_atom(self, atom):
"""Ensure that there is an atomdetail in storage for the given atom.
Returns uuid for the atomdetail that is/was created.
"""
functor = misc.match_type_handler(atom, self._ensure_matchers)
if not functor:
raise TypeError("Unknown item '%s' (%s) requested to ensure"
% (atom, type(atom)))
else:
return functor(atom.name,
misc.get_version_string(atom),
atom.save_as)
def pformat_atom_detail(atom_detail, indent=0):
"""Pretty formats a atom detail."""
detail_type = logbook.atom_detail_type(atom_detail)
lines = ["%s%s: '%s'" % (" " * (indent), detail_type, atom_detail.name)]
lines.extend(_format_shared(atom_detail, indent=indent + 1))
lines.append("%s- version = %s"
% (" " * (indent + 1), misc.get_version_string(atom_detail)))
lines.append("%s- results = %s"
% (" " * (indent + 1), atom_detail.results))
lines.append("%s- failure = %s" % (" " * (indent + 1),
bool(atom_detail.failure)))
lines.extend(_format_meta(atom_detail.meta, indent=indent + 1))
return "\n".join(lines)
def _ensure_storage(self):
# NOTE(harlowja): signal to the tasks that exist that we are about to
# resume, if they have a previous state, they will now transition to
# a resuming state (and then to suspended).
self._change_state(states.RESUMING) # does nothing in PENDING state
for node in self._compilation.execution_graph.nodes_iter():
version = misc.get_version_string(node)
if isinstance(node, retry.Retry):
self.storage.ensure_retry(node.name, version, node.save_as)
else:
self.storage.ensure_task(node.name, version, node.save_as)
if node.inject:
self.storage.inject_atom_args(node.name, node.inject)
self._change_state(states.SUSPENDED) # does nothing in PENDING state
def ensure_atom(self, atom):
"""Ensure that there is an atomdetail in storage for the given atom.
Returns uuid for the atomdetail that is/was created.
"""
match = misc.match_type(atom, self._ensure_matchers)
if not match:
raise TypeError("Unknown atom '%s' (%s) requested to ensure"
% (atom, type(atom)))
else:
detail_cls, kind = match
atom_id = self._ensure_atom_detail(kind, detail_cls, atom.name,
misc.get_version_string(atom),
atom.save_as)
return atom_id
def compile(self):
"""Compiles the contained flow into a structure which the engine can
use to run or if this can not be done then an exception is thrown
indicating why this compilation could not be achieved.
"""
if self._root is not None:
return
assert self._graph_action is not None, ('Graph action class must be'
' specified')
self._change_state(states.RESUMING) # does nothing in PENDING state
task_graph = flow_utils.flatten(self._flow)
self._root = self._graph_action(task_graph)
loaded_failures = {}
for task in task_graph.nodes_iter():
try:
task_id = self.storage.get_uuid_by_name(task.name)
except exc.NotFound:
task_id = uuidutils.generate_uuid()
task_version = misc.get_version_string(task)
self.storage.add_task(task_name=task.name,
uuid=task_id,
task_version=task_version)
try:
result = self.storage.get(task_id)
except exc.NotFound:
result = None
if isinstance(result, misc.Failure):
# NOTE(imelnikov): old failure may have exc_info which
# might get lost during serialization, so we preserve
# old failure object if possible.
old_failure = self._failures.get(task_id, None)
if result.matches(old_failure):
loaded_failures[task_id] = old_failure
else:
loaded_failures[task_id] = result
self.storage.set_result_mapping(task_id, task.save_as)
self._root.add(task, task_action.TaskAction(task, task_id))
self._failures = loaded_failures
self._change_state(states.SUSPENDED) # does nothing in PENDING state
def compile(self):
"""Compiles the contained flow into a structure which the engine can
use to run or if this can not be done then an exception is thrown
indicating why this compilation could not be achieved.
"""
if self._root is not None:
return
assert self._graph_action is not None, ('Graph action class must be'
' specified')
self._change_state(states.RESUMING) # does nothing in PENDING state
task_graph = flow_utils.flatten(self._flow)
self._root = self._graph_action(task_graph)
loaded_failures = {}
for task in task_graph.nodes_iter():
try:
task_id = self.storage.get_uuid_by_name(task.name)
except exc.NotFound:
task_id = uuidutils.generate_uuid()
task_version = misc.get_version_string(task)
self.storage.add_task(task_name=task.name, uuid=task_id,
task_version=task_version)
try:
result = self.storage.get(task_id)
except exc.NotFound:
result = None
if isinstance(result, misc.Failure):
# NOTE(imelnikov): old failure may have exc_info which
# might get lost during serialization, so we preserve
# old failure object if possible.
old_failure = self._failures.get(task_id, None)
if result.matches(old_failure):
loaded_failures[task_id] = old_failure
else:
loaded_failures[task_id] = result
self.storage.set_result_mapping(task_id, task.save_as)
self._root.add(task, task_action.TaskAction(task, task_id))
self._failures = loaded_failures
self._change_state(states.SUSPENDED) # does nothing in PENDING state
def compile(self):
"""Compiles the contained flow into a structure which the engine can
use to run or if this can not be done then an exception is thrown
indicating why this compilation could not be achieved.
"""
if self._root is not None:
return
assert self._graph_action_cls is not None, (
'Graph action class must be specified')
self._change_state(states.RESUMING) # does nothing in PENDING state
task_graph = flow_utils.flatten(self._flow)
if task_graph.number_of_nodes() == 0:
raise exc.EmptyFlow("Flow %s is empty." % self._flow.name)
self._root = self._graph_action_cls(task_graph)
for task in task_graph.nodes_iter():
task_version = misc.get_version_string(task)
self.storage.ensure_task(task.name, task_version, task.save_as)
self._change_state(states.SUSPENDED) # does nothing in PENDING state
def __str__(self):
return "%s==%s" % (self.name, misc.get_version_string(self))
def __str__(self):
return "%s==%s" % (self.name, misc.get_version_string(self))