def test_task_detail_meta_update(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = models.LogBook(name=lb_name, uuid=lb_id)
fd = models.FlowDetail('test', uuid=uuidutils.generate_uuid())
lb.add(fd)
td = models.TaskDetail("detail-1", uuid=uuidutils.generate_uuid())
td.meta = {'test': 42}
fd.add(td)
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
conn.update_flow_details(fd)
conn.update_atom_details(td)
td.meta['test'] = 43
with contextlib.closing(self._get_connection()) as conn:
conn.update_atom_details(td)
with contextlib.closing(self._get_connection()) as conn:
lb2 = conn.get_logbook(lb_id)
fd2 = lb2.find(fd.uuid)
td2 = fd2.find(td.uuid)
self.assertEqual(43, td2.meta.get('test'))
self.assertIsInstance(td2, models.TaskDetail)
def test_retry_detail_save_intention(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = models.LogBook(name=lb_name, uuid=lb_id)
fd = models.FlowDetail('test', uuid=uuidutils.generate_uuid())
lb.add(fd)
rd = models.RetryDetail("retry-1", uuid=uuidutils.generate_uuid())
fd.add(rd)
# save it
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
conn.update_flow_details(fd)
conn.update_atom_details(rd)
# change intention and save
rd.intention = states.REVERT
with contextlib.closing(self._get_connection()) as conn:
conn.update_atom_details(rd)
# now read it back
with contextlib.closing(self._get_connection()) as conn:
lb2 = conn.get_logbook(lb_id)
fd2 = lb2.find(fd.uuid)
rd2 = fd2.find(rd.uuid)
self.assertEqual(states.REVERT, rd2.intention)
self.assertIsInstance(rd2, models.RetryDetail)
def test_logbook_merge_flow_detail(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = models.LogBook(name=lb_name, uuid=lb_id)
fd = models.FlowDetail('test', uuid=uuidutils.generate_uuid())
lb.add(fd)
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
lb2 = models.LogBook(name=lb_name, uuid=lb_id)
fd2 = models.FlowDetail('test2', uuid=uuidutils.generate_uuid())
lb2.add(fd2)
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb2)
with contextlib.closing(self._get_connection()) as conn:
lb3 = conn.get_logbook(lb_id)
self.assertEqual(2, len(lb3))
def test_flow_name_uuid_and_meta(self):
flow_detail = models.FlowDetail(name='test-fd', uuid='aaaa')
flow_detail.meta = {'a': 1}
s = self._get_storage(flow_detail)
self.assertEqual('test-fd', s.flow_name)
self.assertEqual('aaaa', s.flow_uuid)
self.assertEqual({'a': 1}, s.flow_meta)
def run_poster():
# This just posts a single job and then ends...
print("Starting poster with pid: %s" % ME)
my_name = "poster-%s" % ME
persist_backend = persistence_backends.fetch(PERSISTENCE_URI)
with contextlib.closing(persist_backend):
with contextlib.closing(persist_backend.get_connection()) as conn:
conn.upgrade()
job_backend = job_backends.fetch(my_name, JB_CONF,
persistence=persist_backend)
job_backend.connect()
with contextlib.closing(job_backend):
# Create information in the persistence backend about the
# unit of work we want to complete and the factory that
# can be called to create the tasks that the work unit needs
# to be done.
lb = models.LogBook("post-from-%s" % my_name)
fd = models.FlowDetail("song-from-%s" % my_name,
uuidutils.generate_uuid())
lb.add(fd)
with contextlib.closing(persist_backend.get_connection()) as conn:
conn.save_logbook(lb)
engines.save_factory_details(fd, make_bottles,
[HOW_MANY_BOTTLES], {},
backend=persist_backend)
# Post, and be done with it!
jb = job_backend.post("song-from-%s" % my_name, book=lb)
print("Posted: %s" % jb)
print("Goodbye...")
def test_logbook_add_task_detail(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = models.LogBook(name=lb_name, uuid=lb_id)
fd = models.FlowDetail('test', uuid=uuidutils.generate_uuid())
td = models.TaskDetail("detail-1", uuid=uuidutils.generate_uuid())
td.version = '4.2'
fd.add(td)
lb.add(fd)
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
with contextlib.closing(self._get_connection()) as conn:
lb2 = conn.get_logbook(lb_id)
self.assertEqual(1, len(lb2))
tasks = 0
for fd in lb:
tasks += len(fd)
self.assertEqual(1, tasks)
with contextlib.closing(self._get_connection()) as conn:
lb2 = conn.get_logbook(lb_id)
fd2 = lb2.find(fd.uuid)
td2 = fd2.find(td.uuid)
self.assertIsNotNone(td2)
self.assertEqual('detail-1', td2.name)
self.assertEqual('4.2', td2.version)
self.assertEqual(states.EXECUTE, td2.intention)
def test_task_detail_with_failure(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = models.LogBook(name=lb_name, uuid=lb_id)
fd = models.FlowDetail('test', uuid=uuidutils.generate_uuid())
lb.add(fd)
td = models.TaskDetail("detail-1", uuid=uuidutils.generate_uuid())
try:
raise RuntimeError('Woot!')
except Exception:
td.failure = failure.Failure()
fd.add(td)
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
conn.update_flow_details(fd)
conn.update_atom_details(td)
# Read failure back
with contextlib.closing(self._get_connection()) as conn:
lb2 = conn.get_logbook(lb_id)
fd2 = lb2.find(fd.uuid)
td2 = fd2.find(td.uuid)
self.assertEqual('Woot!', td2.failure.exception_str)
self.assertIs(td2.failure.check(RuntimeError), RuntimeError)
self.assertEqual(td.failure.traceback_str, td2.failure.traceback_str)
self.assertIsInstance(td2, models.TaskDetail)
def test_retry_detail_save_with_task_failure(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = models.LogBook(name=lb_name, uuid=lb_id)
fd = models.FlowDetail('test', uuid=uuidutils.generate_uuid())
lb.add(fd)
rd = models.RetryDetail("retry-1", uuid=uuidutils.generate_uuid())
fail = failure.Failure.from_exception(RuntimeError('fail'))
rd.results.append((42, {'some-task': fail}))
fd.add(rd)
# save it
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
conn.update_flow_details(fd)
conn.update_atom_details(rd)
# now read it back
with contextlib.closing(self._get_connection()) as conn:
lb2 = conn.get_logbook(lb_id)
fd2 = lb2.find(fd.uuid)
rd2 = fd2.find(rd.uuid)
self.assertIsInstance(rd2, models.RetryDetail)
fail2 = rd2.results[0][1].get('some-task')
self.assertIsInstance(fail2, failure.Failure)
self.assertTrue(fail.matches(fail2))
def test_flow_detail_update_not_existing(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = models.LogBook(name=lb_name, uuid=lb_id)
fd = models.FlowDetail('test', uuid=uuidutils.generate_uuid())
lb.add(fd)
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
fd2 = models.FlowDetail('test-2', uuid=uuidutils.generate_uuid())
lb.add(fd2)
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
with contextlib.closing(self._get_connection()) as conn:
lb2 = conn.get_logbook(lb.uuid)
self.assertIsNotNone(lb2.find(fd.uuid))
self.assertIsNotNone(lb2.find(fd2.uuid))
def test_logbook_lazy_fetch(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = models.LogBook(name=lb_name, uuid=lb_id)
fd = models.FlowDetail('test', uuid=uuidutils.generate_uuid())
lb.add(fd)
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
with contextlib.closing(self._get_connection()) as conn:
lb2 = conn.get_logbook(lb_id, lazy=True)
self.assertEqual(0, len(lb2))
self.assertEqual(1, len(lb))
def test_logbook_add_flow_detail(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = models.LogBook(name=lb_name, uuid=lb_id)
fd = models.FlowDetail('test', uuid=uuidutils.generate_uuid())
lb.add(fd)
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
with contextlib.closing(self._get_connection()) as conn:
lb2 = conn.get_logbook(lb_id)
self.assertEqual(1, len(lb2))
self.assertEqual(1, len(lb))
self.assertEqual(fd.name, lb2.find(fd.uuid).name)
def create_logbook(self, job_name, store=None):
logbook_id = uuidutils.generate_uuid()
connection = self._persistence.get_connection()
try:
book = connection.get_logbook(logbook_id, lazy=True)
except excp.NotFound:
book = models.LogBook(job_name, logbook_id)
flow_detail = models.FlowDetail(job_name, logbook_id)
flow_detail.meta['store'] = store or {}
book.add(flow_detail)
connection.save_logbook(book)
return flow_detail, book
def test_flow_detail_lazy_fetch(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = models.LogBook(name=lb_name, uuid=lb_id)
fd = models.FlowDetail('test', uuid=uuidutils.generate_uuid())
td = models.TaskDetail("detail-1", uuid=uuidutils.generate_uuid())
td.version = '4.2'
fd.add(td)
lb.add(fd)
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
with contextlib.closing(self._get_connection()) as conn:
fd2 = conn.get_flow_details(fd.uuid, lazy=True)
self.assertEqual(0, len(fd2))
self.assertEqual(1, len(fd))
def test_flow_detail_save(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = models.LogBook(name=lb_name, uuid=lb_id)
fd = models.FlowDetail('test', uuid=uuidutils.generate_uuid())
lb.add(fd)
# Ensure we can't save it since its owning logbook hasn't been
# saved (flow details can not exist on their own without a connection
# to a logbook).
with contextlib.closing(self._get_connection()) as conn:
self.assertRaises(exc.NotFound, conn.get_logbook, lb_id)
self.assertRaises(exc.NotFound, conn.update_flow_details, fd)
# Ok now we should be able to save both.
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
conn.update_flow_details(fd)
def test_flow_detail_meta_update(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = models.LogBook(name=lb_name, uuid=lb_id)
fd = models.FlowDetail('test', uuid=uuidutils.generate_uuid())
fd.meta = {'test': 42}
lb.add(fd)
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
conn.update_flow_details(fd)
fd.meta['test'] = 43
with contextlib.closing(self._get_connection()) as conn:
conn.update_flow_details(fd)
with contextlib.closing(self._get_connection()) as conn:
lb2 = conn.get_logbook(lb_id)
fd2 = lb2.find(fd.uuid)
self.assertEqual(43, fd2.meta.get('test'))
def test_task_detail_save(self):
lb_id = uuidutils.generate_uuid()
lb_name = 'lb-%s' % (lb_id)
lb = models.LogBook(name=lb_name, uuid=lb_id)
fd = models.FlowDetail('test', uuid=uuidutils.generate_uuid())
lb.add(fd)
td = models.TaskDetail("detail-1", uuid=uuidutils.generate_uuid())
fd.add(td)
# Ensure we can't save it since its owning logbook hasn't been
# saved (flow details/task details can not exist on their own without
# their parent existing).
with contextlib.closing(self._get_connection()) as conn:
self.assertRaises(exc.NotFound, conn.update_flow_details, fd)
self.assertRaises(exc.NotFound, conn.update_atom_details, td)
# Ok now we should be able to save them.
with contextlib.closing(self._get_connection()) as conn:
conn.save_logbook(lb)
conn.update_flow_details(fd)
conn.update_atom_details(td)
def create_flow_detail(flow, book=None, backend=None, meta=None):
"""Creates a flow detail for a flow & adds & saves it in a logbook.
This will create a flow detail for the given flow using the flow name,
and add it to the provided logbook and then uses the given backend to save
the logbook and then returns the created flow detail.
If no book is provided a temporary one will be created automatically (no
reference to the logbook will be returned, so this should nearly *always*
be provided or only used in situations where no logbook is needed, for
example in tests). If no backend is provided then no saving will occur and
the created flow detail will not be persisted even if the flow detail was
added to a given (or temporarily generated) logbook.
"""
flow_id = uuidutils.generate_uuid()
flow_name = getattr(flow, 'name', None)
if flow_name is None:
LOG.warning("No name provided for flow %s (id %s)", flow, flow_id)
flow_name = flow_id
flow_detail = models.FlowDetail(name=flow_name, uuid=flow_id)
if meta is not None:
if flow_detail.meta is None:
flow_detail.meta = {}
flow_detail.meta.update(meta)
if backend is not None and book is None:
LOG.warning("No logbook provided for flow %s, creating one.", flow)
book = temporary_log_book(backend)
if book is not None:
book.add(flow_detail)
if backend is not None:
with contextlib.closing(backend.get_connection()) as conn:
conn.save_logbook(book)
# Return the one from the saved logbook instead of the local one so
# that the freshest version is given back.
return book.find(flow_id)
else:
return flow_detail
def make_save_book(saver, review_id):
# Record what we want to happen (sometime in the future).
book = models.LogBook("book_%s" % review_id)
detail = models.FlowDetail("flow_%s" % review_id,
uuidutils.generate_uuid())
book.add(detail)
# Associate the factory method we want to be called (in the future)
# with the book, so that the conductor will be able to call into
# that factory to retrieve the workflow objects that represent the
# work.
#
# These args and kwargs *can* be used to save any specific parameters
# into the factory when it is being called to create the workflow
# objects (typically used to tell a factory how to create a unique
# workflow that represents this review).
factory_args = ()
factory_kwargs = {}
engines.save_factory_details(detail, create_review_workflow,
factory_args, factory_kwargs)
with contextlib.closing(saver.get_connection()) as conn:
conn.save_logbook(book)
return book
def temporary_flow_detail(backend=None, meta=None):
"""Creates a temporary flow detail and logbook in the given backend.
Mainly useful for tests and other use cases where a temporary flow detail
and a temporary logbook is needed for a short-period of time.
"""
flow_id = uuidutils.generate_uuid()
book = temporary_log_book(backend)
flow_detail = models.FlowDetail(name='tmp-flow-detail', uuid=flow_id)
if meta is not None:
if flow_detail.meta is None:
flow_detail.meta = {}
flow_detail.meta.update(meta)
book.add(flow_detail)
if backend is not None:
with contextlib.closing(backend.get_connection()) as conn:
conn.save_logbook(book)
# Return the one from the saved logbook instead of the local one so
# that the freshest version is given back.
return book, book.find(flow_id)
return 'ok'
def flow_factory():
return lf.Flow('resume from backend example').add(
TestTask(name='first'), InterruptTask(name='boom'),
TestTask(name='second'))
# INITIALIZE PERSISTENCE ####################################
with eu.get_backend() as backend:
# Create a place where the persistence information will be stored.
book = models.LogBook("example")
flow_detail = models.FlowDetail("resume from backend example",
uuid=uuidutils.generate_uuid())
book.add(flow_detail)
with contextlib.closing(backend.get_connection()) as conn:
conn.save_logbook(book)
# CREATE AND RUN THE FLOW: FIRST ATTEMPT ####################
flow = flow_factory()
engine = zag.engines.load(flow,
flow_detail=flow_detail,
book=book,
backend=backend)
print_task_states(flow_detail, "At the beginning, there is no state")
eu.print_wrapped("Running")
engine.run()
book_id, flow_id = sys.argv[2].split("+", 1)
except (IndexError, ValueError):
book_id = None
flow_id = None
if not all([book_id, flow_id]):
# If no 'tracking id' (think a fedex or ups tracking id) is provided
# then we create one by creating a logbook (where flow details are
# stored) and creating a flow detail (where flow and task state is
# stored). The combination of these 2 objects unique ids (uuids) allows
# the users of zag to reassociate the workflows that were
# potentially running (and which may have partially completed) back
# with zag so that those workflows can be resumed (or reverted)
# after a process/thread/engine has failed in someway.
book = models.LogBook('resume-volume-create')
flow_detail = models.FlowDetail("root", uuid=uuidutils.generate_uuid())
book.add(flow_detail)
with contextlib.closing(backend.get_connection()) as conn:
conn.save_logbook(book)
print("!! Your tracking id is: '%s+%s'" % (book.uuid,
flow_detail.uuid))
print("!! Please submit this on later runs for tracking purposes")
else:
flow_detail = find_flow_detail(backend, book_id, flow_id)
# Load and run.
engine = engines.load(flow,
flow_detail=flow_detail,
backend=backend, engine='serial')
engine.run()