def test_task_detail_with_failure(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid()) lb.add(fd) td = logbook.TaskDetail("detail-1", uuid=uuidutils.generate_uuid()) try: raise RuntimeError('Woot!') except Exception: td.failure = misc.Failure() fd.add(td) with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) conn.update_flow_details(fd) conn.update_task_details(td) # Read failure back with contextlib.closing(self._get_connection()) as conn: lb2 = conn.get_logbook(lb_id) fd2 = lb2.find(fd.uuid) td2 = fd2.find(td.uuid) failure = td2.failure self.assertEqual(failure.exception_str, 'Woot!') self.assertIs(failure.check(RuntimeError), RuntimeError) self.assertEqual(failure.traceback_str, td.failure.traceback_str)
def test_retry_detail_save_with_task_failure(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid()) lb.add(fd) rd = logbook.RetryDetail("retry-1", uuid=uuidutils.generate_uuid()) fail = misc.Failure.from_exception(RuntimeError('fail')) rd.results.append((42, {'some-task': fail})) fd.add(rd) # save it with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) conn.update_flow_details(fd) conn.update_atom_details(rd) # now read it back with contextlib.closing(self._get_connection()) as conn: lb2 = conn.get_logbook(lb_id) fd2 = lb2.find(fd.uuid) rd2 = fd2.find(rd.uuid) self.assertIsInstance(rd2, logbook.RetryDetail) fail2 = rd2.results[0][1].get('some-task') self.assertIsInstance(fail2, misc.Failure) self.assertTrue(fail.matches(fail2))
def test_task_detail_meta_update(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid()) lb.add(fd) td = logbook.TaskDetail("detail-1", uuid=uuidutils.generate_uuid()) td.meta = {'test': 42} fd.add(td) with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) conn.update_flow_details(fd) conn.update_atom_details(td) td.meta['test'] = 43 with contextlib.closing(self._get_connection()) as conn: conn.update_atom_details(td) with contextlib.closing(self._get_connection()) as conn: lb2 = conn.get_logbook(lb_id) fd2 = lb2.find(fd.uuid) td2 = fd2.find(td.uuid) self.assertEqual(td2.meta.get('test'), 43) self.assertIsInstance(td2, logbook.TaskDetail)
def create_flow_detail(flow, book=None, backend=None): """Creates a flow detail for the given flow and adds it to the provided logbook (if provided) and then uses the given backend (if provided) to save the logbook then returns the created flow detail. """ try: flow_name = getattr(flow, 'name') except AttributeError: LOG.warn("Flow %s does not have a name attribute, creating one.", flow) flow_name = uuidutils.generate_uuid() try: flow_id = getattr(flow, 'uuid') except AttributeError: LOG.warn("Flow %s does not have a uuid attribute, creating one.", flow) flow_id = uuidutils.generate_uuid() flow_detail = logbook.FlowDetail(name=flow_name, uuid=flow_id) if book is not None: book.add(flow_detail) if backend is not None: with contextlib.closing(backend.get_connection()) as conn: conn.save_logbook(book) # Return the one from the saved logbook instead of the local one so # that the freshest version is given back return book.find(flow_id) else: if backend is not None: LOG.warn("Can not save %s without a provided logbook", flow) return flow_detail
def test_logbook_add_task_detail(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid()) td = logbook.TaskDetail("detail-1", uuid=uuidutils.generate_uuid()) td.version = '4.2' fd.add(td) lb.add(fd) with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) with contextlib.closing(self._get_connection()) as conn: lb2 = conn.get_logbook(lb_id) self.assertEqual(1, len(lb2)) tasks = 0 for fd in lb: tasks += len(fd) self.assertEqual(1, tasks) with contextlib.closing(self._get_connection()) as conn: lb2 = conn.get_logbook(lb_id) fd2 = lb2.find(fd.uuid) td2 = fd2.find(td.uuid) self.assertIsNot(td2, None) self.assertEqual(td2.name, 'detail-1') self.assertEqual(td2.version, '4.2')
def test_retry_detail_save_intention(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid()) lb.add(fd) rd = logbook.RetryDetail("retry-1", uuid=uuidutils.generate_uuid()) fd.add(rd) # save it with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) conn.update_flow_details(fd) conn.update_atom_details(rd) # change intention and save rd.intention = states.REVERT with contextlib.closing(self._get_connection()) as conn: conn.update_atom_details(rd) # now read it back with contextlib.closing(self._get_connection()) as conn: lb2 = conn.get_logbook(lb_id) fd2 = lb2.find(fd.uuid) rd2 = fd2.find(rd.uuid) self.assertEqual(rd2.intention, states.REVERT) self.assertIsInstance(rd2, logbook.RetryDetail)
def test_logbook_add_task_detail(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid()) td = logbook.TaskDetail("detail-1", uuid=uuidutils.generate_uuid()) td.version = '4.2' fd.add(td) lb.add(fd) with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) with contextlib.closing(self._get_connection()) as conn: lb2 = conn.get_logbook(lb_id) self.assertEqual(1, len(lb2)) tasks = 0 for fd in lb: tasks += len(fd) self.assertEqual(1, tasks) with contextlib.closing(self._get_connection()) as conn: lb2 = conn.get_logbook(lb_id) fd2 = lb2.find(fd.uuid) td2 = fd2.find(td.uuid) self.assertIsNot(td2, None) self.assertEqual(td2.name, 'detail-1') self.assertEqual(td2.version, '4.2')
def test_task_detail_with_failure(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid()) lb.add(fd) td = logbook.TaskDetail("detail-1", uuid=uuidutils.generate_uuid()) try: raise RuntimeError('Woot!') except Exception: td.failure = misc.Failure() fd.add(td) with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) conn.update_flow_details(fd) conn.update_task_details(td) # Read failure back with contextlib.closing(self._get_connection()) as conn: lb2 = conn.get_logbook(lb_id) fd2 = lb2.find(fd.uuid) td2 = fd2.find(td.uuid) failure = td2.failure self.assertEqual(failure.exception_str, 'Woot!') self.assertIs(failure.check(RuntimeError), RuntimeError) self.assertEqual(failure.traceback_str, td.failure.traceback_str)
def test_logbook_add_flow_detail(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid()) lb.add(fd) with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) with contextlib.closing(self._get_connection()) as conn: lb2 = conn.get_logbook(lb_id) self.assertEqual(1, len(lb2)) self.assertEqual(1, len(lb)) self.assertEqual(fd.name, lb2.find(fd.uuid).name)
def test_logbook_add_flow_detail(self): lb_id = uuidutils.generate_uuid() lb_name = "lb-%s" % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail("test", uuid=uuidutils.generate_uuid()) lb.add(fd) with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) with contextlib.closing(self._get_connection()) as conn: lb2 = conn.get_logbook(lb_id) self.assertEquals(1, len(lb2)) self.assertEquals(1, len(lb)) self.assertEquals(fd.name, lb2.find(fd.uuid).name)
def setUpClass(cls): # Create a workflow for flowdetails to use wf_id = uuidutils.generate_uuid() wf_name = 'wf-%s' % (wf_id) wf = flow.Flow(wf_name, None, wf_id) cls.wfs.append(wf) # Create a task for taskdetails to use task_id = uuidutils.generate_uuid() task_name = 'task-%s' % (task_id) tsk = utils.DummyTask(task_name, task_id) cls.tsks.append(tsk)
def setUpClass(cls): # Create a workflow for flowdetails to use wf_id = uuidutils.generate_uuid() wf_name = 'wf-%s' % (wf_id) wf = flow.Flow(wf_name, None, wf_id) cls.wfs.append(wf) # Create a task for taskdetails to use task_id = uuidutils.generate_uuid() task_name = 'task-%s' % (task_id) tsk = utils.DummyTask(task_name, task_id) cls.tsks.append(tsk)
def test_logbook_add_flow_detail(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id, backend=self._get_backend()) fd = flowdetail.FlowDetail('test', uuid=uuidutils.generate_uuid()) lb.add(fd) lb.save() lb2 = logbook.load(lb_id, backend=self._get_backend()) self.assertEquals(1, len(lb2)) self.assertEquals(1, len(lb)) self.assertEquals(fd.name, lb2.find(fd.uuid).name)
def test_logbook_save(self): # Create a generic logbook to save lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(lb_name, lb_id) # Save the logbook and record its uuid and name b_api.logbook_save(lb) self.lb_names.append(lb_name) self.lb_ids.append(lb_id) # Check that the saved logbook exists in the backend actual = b_api.logbook_get(lb_id) self.assertIsNotNone(actual) # Check that the saved logbook has no flowdetails self.assertEquals(len(actual), 0) # Add a flowdetail to the logbook fd = b_api.flowdetail_get(self.fd_ids[0]) lb.add_flow_detail(fd) # Save the updated logbook b_api.logbook_save(lb) # Check that the updated logbook is still in the backend actual = b_api.logbook_get(lb_id) self.assertIsNotNone(actual) # Check that the added flowdetail was recorded self.assertEquals(len(actual), 1)
def setUpClass(cls): # Create a workflow to create flowdetails with wf_id = uuidutils.generate_uuid() wf_name = 'wf-%s' % (wf_id) wf = flow.Flow(wf_name, None, wf_id) cls.wfs.append(wf)
def setUp(self): # Create a logbook and record its uuid and name lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) b_api.logbook_create(lb_name, lb_id) self.lb_names.append(lb_name) self.lb_ids.append(lb_id) # Create a flowdetail and record its uuid and name fd_id = uuidutils.generate_uuid() fd_name = 'fd-%s' % (fd_id) b_api.flowdetail_create(fd_name, self.wfs[0], fd_id) self.fd_names.append(fd_name) self.fd_ids.append(fd_id)
def compile(self): """Compiles the contained flow into a structure which the engine can use to run or if this can not be done then an exception is thrown indicating why this compilation could not be achieved. """ if self._root is not None: return assert self._graph_action is not None, ('Graph action class must be' ' specified') self._change_state(states.RESUMING) # does nothing in PENDING state task_graph = flow_utils.flatten(self._flow) if task_graph.number_of_nodes() == 0: raise exc.EmptyFlow("Flow %s is empty." % self._flow.name) self._root = self._graph_action(task_graph) for task in task_graph.nodes_iter(): try: task_id = self.storage.get_uuid_by_name(task.name) except exc.NotFound: task_id = uuidutils.generate_uuid() task_version = misc.get_version_string(task) self.storage.add_task(task_name=task.name, uuid=task_id, task_version=task_version) self.storage.set_result_mapping(task_id, task.save_as) self._root.add(task, task_action.TaskAction(task, task_id)) self._change_state(states.SUSPENDED) # does nothing in PENDING state
def flush(client, path=None): # This uses the linearity guarantee of zookeeper (and associated libraries) # to create a temporary node, wait until a watcher notifies it's created, # then yield back for more work, and then at the end of that work delete # the created node. This ensures that the operations done in the yield # of this context manager will be applied and all watchers will have fired # before this context manager exits. if not path: path = FLUSH_PATH_TPL % uuidutils.generate_uuid() created = threading.Event() deleted = threading.Event() def on_created(data, stat): if stat is not None: created.set() return False # cause this watcher to cease to exist def on_deleted(data, stat): if stat is None: deleted.set() return False # cause this watcher to cease to exist watchers.DataWatch(client, path, func=on_created) client.create(path, makepath=True) created.wait() try: yield finally: watchers.DataWatch(client, path, func=on_deleted) client.delete(path, recursive=True) deleted.wait()
def _ensure_retry(self, retry_name, retry_version, result_mapping): """Ensures there is a retrydetail that corresponds to the retry info. If retry does not exist, adds a record for it. Added retry will have PENDING state. Sets result mapping for the retry from result_mapping argument. Initializes retry result as an empty collections of results and failures history. Returns uuid for the retry details corresponding to the retry with given name. """ if not retry_name: raise ValueError("Retry name must be non-empty") with self._lock.write_lock(): try: retry_id = self._atom_name_to_uuid[retry_name] except KeyError: retry_id = uuidutils.generate_uuid() self._create_atom_detail(logbook.RetryDetail, retry_name, retry_id, retry_version) else: ad = self._flowdetail.find(retry_id) if not isinstance(ad, logbook.RetryDetail): raise exceptions.Duplicate( "Atom detail %s already exists in flow detail %s." % (retry_name, self._flowdetail.name)) self._set_result_mapping(retry_name, result_mapping) return retry_id
def setUp(self): # Create a logbook and record its uuid and name lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) b_api.logbook_create(lb_name, lb_id) self.lb_names.append(lb_name) self.lb_ids.append(lb_id) # Create a flowdetail and record its uuid and name fd_id = uuidutils.generate_uuid() fd_name = 'fd-%s' % (fd_id) b_api.flowdetail_create(fd_name, self.wfs[0], fd_id) self.fd_names.append(fd_name) self.fd_ids.append(fd_id)
def test_logbook_save(self): # Create a generic logbook to save lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(lb_name, lb_id) # Save the logbook and record its uuid and name b_api.logbook_save(lb) self.lb_names.append(lb_name) self.lb_ids.append(lb_id) # Check that the saved logbook exists in the backend actual = b_api.logbook_get(lb_id) self.assertIsNotNone(actual) # Check that the saved logbook has no flowdetails self.assertEquals(len(actual), 0) # Add a flowdetail to the logbook fd = b_api.flowdetail_get(self.fd_ids[0]) lb.add_flow_detail(fd) # Save the updated logbook b_api.logbook_save(lb) # Check that the updated logbook is still in the backend actual = b_api.logbook_get(lb_id) self.assertIsNotNone(actual) # Check that the added flowdetail was recorded self.assertEquals(len(actual), 1)
def setUp(self): # Create a flowdetail and record its uuid and name fd_id = uuidutils.generate_uuid() fd_name = 'fd-%s' % (fd_id) b_api.flowdetail_create(fd_name, self.wfs[0], fd_id) self.fd_names.append(fd_name) self.fd_ids.append(fd_id) # Create a taskdetail and record its uuid and name td_id = uuidutils.generate_uuid() td_name = 'td-%s' % (td_id) b_api.taskdetail_create(td_name, self.tsks[0], td_id) self.td_names.append(td_name) self.td_ids.append(td_id)
def test_flowdetail_save(self): # Create a generic flowdetail to save fd_id = uuidutils.generate_uuid() fd_name = 'fd-%s' % (fd_id) wf = self.wfs[0] fd = flowdetail.FlowDetail(fd_name, wf, fd_id) # Save the generic flowdetail to the backend and record its uuid/name b_api.flowdetail_save(fd) self.fd_names.append(fd_name) self.fd_ids.append(fd_id) # Check that the saved flowdetail is in the backend actual = b_api.flowdetail_get(fd_id) self.assertIsNotNone(actual) # Check that the saved flowdetail has no taskdetails self.assertEquals(len(actual), 0) # Add a generic taskdetail to the flowdetail td = b_api.taskdetail_get(self.td_ids[0]) fd.add_task_detail(td) # Save the updated flowdetail b_api.flowdetail_save(fd) # Check that the saved flowdetail is still there actual = b_api.flowdetail_get(fd_id) self.assertIsNotNone(actual) # Check that the addition of a taskdetail was recorded self.assertEquals(len(actual), 1)
def test_flow_detail_save(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id, backend=self._get_backend()) fd = flowdetail.FlowDetail('test', uuid=uuidutils.generate_uuid()) lb.add(fd) # Ensure we can't save it since its owning logbook hasn't been # saved. self.assertRaises(exc.NotFound, fd.save) # Ok now we should be able to save it lb.save() fd.save()
def create_flow_detail(flow, book=None, backend=None, meta=None): """Creates a flow detail for the given flow and adds it to the provided logbook (if provided) and then uses the given backend (if provided) to save the logbook then returns the created flow detail. """ flow_id = uuidutils.generate_uuid() flow_name = getattr(flow, 'name', None) if flow_name is None: LOG.warn("No name provided for flow %s (id %s)" % (flow, flow_id)) flow_name = flow_id flow_detail = logbook.FlowDetail(name=flow_name, uuid=flow_id) if meta is not None: if flow_detail.meta is None: flow_detail.meta = {} flow_detail.meta.update(meta) if backend is not None and book is None: LOG.warn("No logbook provided for flow %s, creating one.", flow) book = temporary_log_book(backend) if book is not None: book.add(flow_detail) if backend is not None: with contextlib.closing(backend.get_connection()) as conn: conn.save_logbook(book) # Return the one from the saved logbook instead of the local one so # that the freshest version is given back. return book.find(flow_id) else: return flow_detail
def _ensure_task(self, task_name, task_version, result_mapping): """Ensures there is a taskdetail that corresponds to the task info. If task does not exist, adds a record for it. Added task will have PENDING state. Sets result mapping for the task from result_mapping argument. Returns uuid for the task details corresponding to the task with given name. """ if not task_name: raise ValueError("Task name must be non-empty") with self._lock.write_lock(): try: task_id = self._atom_name_to_uuid[task_name] except KeyError: task_id = uuidutils.generate_uuid() self._create_atom_detail(logbook.TaskDetail, task_name, task_id, task_version) else: ad = self._flowdetail.find(task_id) if not isinstance(ad, logbook.TaskDetail): raise exceptions.Duplicate( "Atom detail %s already exists in flow detail %s." % (task_name, self._flowdetail.name)) self._set_result_mapping(task_name, result_mapping) return task_id
def test_logbook_simple_save(self): lb_id = uuidutils.generate_uuid() lb_meta = {'1': 2} lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id, backend=self._get_backend()) lb.meta = lb_meta # Should not already exist self.assertRaises(exc.NotFound, logbook.load, lb_id, backend=self._get_backend()) lb.save() del lb lb = None lb = logbook.load(lb_id, backend=self._get_backend()) self.assertEquals(lb_name, lb.name) self.assertEquals(0, len(lb)) self.assertEquals(lb_meta, lb.meta) self.assertIsNone(lb.updated_at) self.assertIsNotNone(lb.created_at)
def test_logbook_add_flow_detail(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id, backend=self._get_backend()) fd = flowdetail.FlowDetail('test', uuid=uuidutils.generate_uuid()) lb.add(fd) lb.save() lb2 = logbook.load(lb_id, backend=self._get_backend()) self.assertEquals(1, len(lb2)) self.assertEquals(1, len(lb)) self.assertEquals(fd.name, lb2.find(fd.uuid).name)
def _ensure_task(self, task_name, task_version, result_mapping): """Ensures there is a taskdetail that corresponds to the task info. If task does not exist, adds a record for it. Added task will have PENDING state. Sets result mapping for the task from result_mapping argument. Returns uuid for the task details corresponding to the task with given name. """ if not task_name: raise ValueError("Task name must be non-empty") with self._lock.write_lock(): try: task_id = self._atom_name_to_uuid[task_name] except KeyError: task_id = uuidutils.generate_uuid() self._create_atom_detail(logbook.TaskDetail, task_name, task_id, task_version) else: ad = self._flowdetail.find(task_id) if not isinstance(ad, logbook.TaskDetail): raise exceptions.Duplicate( "Atom detail %s already exists in flow detail %s." % (task_name, self._flowdetail.name)) self._set_result_mapping(task_name, result_mapping) return task_id
def __init__(self, name, parents=None, uuid=None): self._name = str(name) # The state of this flow. self._state = states.PENDING # If this flow has a parent flow/s which need to be reverted if # this flow fails then please include them here to allow this child # to call the parents... if parents: self.parents = tuple(parents) else: self.parents = tuple([]) # Any objects that want to listen when a wf/task starts/stops/completes # or errors should be registered here. This can be used to monitor # progress and record tasks finishing (so that it becomes possible to # store the result of a task in some persistent or semi-persistent # storage backend). self.notifier = utils.TransitionNotifier() self.task_notifier = utils.TransitionNotifier() # Ensure that modifications and/or multiple runs aren't happening # at the same time in the same flow at the same time. self._lock = threading.RLock() # Assign this flow a unique identifer. if uuid: self._id = str(uuid) else: self._id = uuidutils.generate_uuid()
def _ensure_retry(self, retry_name, retry_version, result_mapping): """Ensures there is a retrydetail that corresponds to the retry info. If retry does not exist, adds a record for it. Added retry will have PENDING state. Sets result mapping for the retry from result_mapping argument. Initializes retry result as an empty collections of results and failures history. Returns uuid for the retry details corresponding to the retry with given name. """ if not retry_name: raise ValueError("Retry name must be non-empty") with self._lock.write_lock(): try: retry_id = self._atom_name_to_uuid[retry_name] except KeyError: retry_id = uuidutils.generate_uuid() self._create_atom_detail(logbook.RetryDetail, retry_name, retry_id, retry_version) else: ad = self._flowdetail.find(retry_id) if not isinstance(ad, logbook.RetryDetail): raise exceptions.Duplicate( "Atom detail %s already exists in flow detail %s." % (retry_name, self._flowdetail.name)) self._set_result_mapping(retry_name, result_mapping) return retry_id
def setUpClass(cls): # Create a workflow to create flowdetails with wf_id = uuidutils.generate_uuid() wf_name = 'wf-%s' % (wf_id) wf = flow.Flow(wf_name, None, wf_id) cls.wfs.append(wf)
def test_logbook_merge_flow_detail(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid()) lb.add(fd) with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) lb2 = logbook.LogBook(name=lb_name, uuid=lb_id) fd2 = logbook.FlowDetail('test2', uuid=uuidutils.generate_uuid()) lb2.add(fd2) with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb2) with contextlib.closing(self._get_connection()) as conn: lb3 = conn.get_logbook(lb_id) self.assertEqual(2, len(lb3))
def test_multi_message(self): message_count = 30 barrier = latch.Latch(message_count) countdown = lambda data, message: barrier.countdown() on_notify = mock.MagicMock() on_notify.side_effect = countdown on_response = mock.MagicMock() on_response.side_effect = countdown on_request = mock.MagicMock() on_request.side_effect = countdown handlers = { pr.NOTIFY: on_notify, pr.RESPONSE: on_response, pr.REQUEST: on_request, } p = proxy.Proxy(TEST_TOPIC, TEST_EXCHANGE, handlers, transport='memory', transport_options={ 'polling_interval': POLLING_INTERVAL, }) t = threading.Thread(target=p.start) t.daemon = True t.start() p.wait() for i in range(0, message_count): j = i % 3 if j == 0: p.publish(pr.Notify(), TEST_TOPIC) elif j == 1: p.publish(pr.Response(pr.RUNNING), TEST_TOPIC) else: p.publish(pr.Request(test_utils.DummyTask("dummy_%s" % i), uuidutils.generate_uuid(), pr.EXECUTE, [], None, None), TEST_TOPIC) self.assertTrue(barrier.wait(test_utils.WAIT_TIMEOUT)) self.assertEqual(0, barrier.needed) p.stop() t.join() self.assertTrue(on_notify.called) self.assertTrue(on_response.called) self.assertTrue(on_request.called) self.assertEqual(10, on_notify.call_count) self.assertEqual(10, on_response.call_count) self.assertEqual(10, on_request.call_count) call_count = sum([ on_notify.call_count, on_response.call_count, on_request.call_count, ]) self.assertEqual(message_count, call_count)
def test_logbook_merge_flow_detail(self): lb_id = uuidutils.generate_uuid() lb_name = "lb-%s" % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail("test", uuid=uuidutils.generate_uuid()) lb.add(fd) with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) lb2 = logbook.LogBook(name=lb_name, uuid=lb_id) fd2 = logbook.FlowDetail("test2", uuid=uuidutils.generate_uuid()) lb2.add(fd2) with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb2) with contextlib.closing(self._get_connection()) as conn: lb3 = conn.get_logbook(lb_id) self.assertEquals(2, len(lb3))
def create_flow_detail(flow, book=None, backend=None, meta=None): """Creates a flow detail for the given flow and adds it to the provided logbook (if provided) and then uses the given backend (if provided) to save the logbook then returns the created flow detail. """ flow_id = uuidutils.generate_uuid() flow_name = getattr(flow, 'name', None) if flow_name is None: LOG.warn("No name provided for flow %s (id %s)" % (flow, flow_id)) flow_name = flow_id flow_detail = logbook.FlowDetail(name=flow_name, uuid=flow_id) if meta is not None: if flow_detail.meta is None: flow_detail.meta = {} flow_detail.meta.update(meta) if backend is not None and book is None: LOG.warn("No logbook provided for flow %s, creating one.", flow) book = temporary_log_book(backend) if book is not None: book.add(flow_detail) if backend is not None: with contextlib.closing(backend.get_connection()) as conn: conn.save_logbook(book) # Return the one from the saved logbook instead of the local one so # that the freshest version is given back return book.find(flow_id) else: return flow_detail
def setUp(self): # Create a flowdetail and record its uuid and name fd_id = uuidutils.generate_uuid() fd_name = 'fd-%s' % (fd_id) b_api.flowdetail_create(fd_name, self.wfs[0], fd_id) self.fd_names.append(fd_name) self.fd_ids.append(fd_id) # Create a taskdetail and record its uuid and name td_id = uuidutils.generate_uuid() td_name = 'td-%s' % (td_id) b_api.taskdetail_create(td_name, self.tsks[0], td_id) self.td_names.append(td_name) self.td_ids.append(td_id)
def test_flowdetail_save(self): # Create a generic flowdetail to save fd_id = uuidutils.generate_uuid() fd_name = 'fd-%s' % (fd_id) wf = self.wfs[0] fd = flowdetail.FlowDetail(fd_name, wf, fd_id) # Save the generic flowdetail to the backend and record its uuid/name b_api.flowdetail_save(fd) self.fd_names.append(fd_name) self.fd_ids.append(fd_id) # Check that the saved flowdetail is in the backend actual = b_api.flowdetail_get(fd_id) self.assertIsNotNone(actual) # Check that the saved flowdetail has no taskdetails self.assertEquals(len(actual), 0) # Add a generic taskdetail to the flowdetail td = b_api.taskdetail_get(self.td_ids[0]) fd.add_task_detail(td) # Save the updated flowdetail b_api.flowdetail_save(fd) # Check that the saved flowdetail is still there actual = b_api.flowdetail_get(fd_id) self.assertIsNotNone(actual) # Check that the addition of a taskdetail was recorded self.assertEquals(len(actual), 1)
def test_flow_detail_save(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id, backend=self._get_backend()) fd = flowdetail.FlowDetail('test', uuid=uuidutils.generate_uuid()) lb.add(fd) # Ensure we can't save it since its owning logbook hasn't been # saved. self.assertRaises(exc.NotFound, fd.save) # Ok now we should be able to save it lb.save() fd.save()
def __init__(self, name, jb_id=None): if jb_id: self._uuid = jb_id else: self._uuid = uuidutils.generate_uuid() self._name = name self._jobs = []
def __init__(self, name, jb_id=None): if jb_id: self._uuid = jb_id else: self._uuid = uuidutils.generate_uuid() self._name = name self._jobs = []
def test_logbook_add_task_detail(self): lb_id = uuidutils.generate_uuid() lb_name = "lb-%s" % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail("test", uuid=uuidutils.generate_uuid()) td = logbook.TaskDetail("detail-1", uuid=uuidutils.generate_uuid()) fd.add(td) lb.add(fd) with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) with contextlib.closing(self._get_connection()) as conn: lb2 = conn.get_logbook(lb_id) self.assertEquals(1, len(lb2)) tasks = 0 for fd in lb: tasks += len(fd) self.assertEquals(1, tasks)
def __init__(self, name, lb_id=None): if lb_id: self._uuid = lb_id else: self._uuid = uuidutils.generate_uuid() self._name = name self.updated_at = datetime.now() self._flowdetails = []
def __init__(self, name, lb_id=None): if lb_id: self._uuid = lb_id else: self._uuid = uuidutils.generate_uuid() self._name = name self.updated_at = datetime.now() self._flowdetails = []
def __init__(self, name, wf, fd_id=None): if fd_id: self._uuid = fd_id else: self._uuid = uuidutils.generate_uuid() self._name = name self._flow = wf self.updated_at = datetime.now() self._taskdetails = []
def test_logbook_add_task_detail(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id, backend=self._get_backend()) fd = flowdetail.FlowDetail('test', uuid=uuidutils.generate_uuid()) td = taskdetail.TaskDetail("detail-1", uuid=uuidutils.generate_uuid()) fd.add(td) lb.add(fd) lb.save() lb2 = logbook.load(lb_id, backend=self._get_backend()) self.assertEquals(1, len(lb2)) tasks = 0 for fd in lb: tasks += len(fd) self.assertEquals(1, tasks)
def __init__(self, name, uuid=None, details=None): if uuid: self._uuid = uuid else: self._uuid = uuidutils.generate_uuid() self._name = name if not details: details = {} self._details = details
def _fetch_executor(self): executor = worker_executor.WorkerTaskExecutor( uuidutils.generate_uuid(), TEST_EXCHANGE, [TEST_TOPIC], transport='memory', transport_options={ 'polling_interval': POLLING_INTERVAL, }) return executor
def test_flow_detail_save(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid()) lb.add(fd) # Ensure we can't save it since its owning logbook hasn't been # saved (flow details can not exist on their own without a connection # to a logbook). with contextlib.closing(self._get_connection()) as conn: self.assertRaises(exc.NotFound, conn.get_logbook, lb_id) self.assertRaises(exc.NotFound, conn.update_flow_details, fd) # Ok now we should be able to save both. with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) conn.update_flow_details(fd)
def __init__(self, name, uuid=None, details=None): if uuid: self._uuid = uuid else: self._uuid = uuidutils.generate_uuid() self._name = name if not details: details = {} self._details = details
def test_flow_detail_save(self): lb_id = uuidutils.generate_uuid() lb_name = "lb-%s" % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail("test", uuid=uuidutils.generate_uuid()) lb.add(fd) # Ensure we can't save it since its owning logbook hasn't been # saved (flow details can not exist on there own without a connection # to a logbook). with contextlib.closing(self._get_connection()) as conn: self.assertRaises(exc.NotFound, conn.get_logbook, lb_id) self.assertRaises(exc.NotFound, conn.update_flow_details, fd) # Ok now we should be able to save both. with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) conn.update_flow_details(fd)
def __init__(self, task, uuid=None): assert isinstance(task, collections.Callable) self.task = task self.providers = {} self.runs_before = [] self.result = None if not uuid: self._id = uuidutils.generate_uuid() else: self._id = str(uuid)
def __init__(self, name, uuid=None): if uuid: self._uuid = uuid else: self._uuid = uuidutils.generate_uuid() self._name = name self._flowdetails_by_id = {} self.created_at = timeutils.utcnow() self.updated_at = None self.meta = {}
def test_logbook_add_task_detail(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id, backend=self._get_backend()) fd = flowdetail.FlowDetail('test', uuid=uuidutils.generate_uuid()) td = taskdetail.TaskDetail("detail-1", uuid=uuidutils.generate_uuid()) fd.add(td) lb.add(fd) lb.save() lb2 = logbook.load(lb_id, backend=self._get_backend()) self.assertEquals(1, len(lb2)) tasks = 0 for fd in lb: tasks += len(fd) self.assertEquals(1, tasks)
def test_flow_detail_meta_update(self): lb_id = uuidutils.generate_uuid() lb_name = 'lb-%s' % (lb_id) lb = logbook.LogBook(name=lb_name, uuid=lb_id) fd = logbook.FlowDetail('test', uuid=uuidutils.generate_uuid()) fd.meta = {'test': 42} lb.add(fd) with contextlib.closing(self._get_connection()) as conn: conn.save_logbook(lb) conn.update_flow_details(fd) fd.meta['test'] = 43 with contextlib.closing(self._get_connection()) as conn: conn.update_flow_details(fd) with contextlib.closing(self._get_connection()) as conn: lb2 = conn.get_logbook(lb_id) fd2 = lb2.find(fd.uuid) self.assertEqual(fd2.meta.get('test'), 43)
def __init__(self, name, uuid=None, updated_at=None, created_at=None): if uuid: self._uuid = uuid else: self._uuid = uuidutils.generate_uuid() self._name = name self._flowdetails = [] self._updated_at = updated_at self._created_at = created_at self.meta = None